ksmbd: discard write access to the directory open

may_open() does not allow a directory to be opened with the write access. However, some writing flags set by client result in adding write access on server, making ksmbd incompatible with FUSE file system. Simply, let's discard the write access when opening a directory. list_add corruption. next is NULL. ------------[ cut here ]------------ kernel BUG at lib/list_debug.c:26! pc : __list_add_valid+0x88/0xbc lr : __list_add_valid+0x88/0xbc Call trace: __list_add_valid+0x88/0xbc fuse_finish_open+0x11c/0x170 fuse_open_common+0x284/0x5e8 fuse_dir_open+0x14/0x24 do_dentry_open+0x2a4/0x4e0 dentry_open+0x50/0x80 smb2_open+0xbe4/0x15a4 handle_ksmbd_work+0x478/0x5ec process_one_work+0x1b4/0x448 worker_thread+0x25c/0x430 kthread+0x104/0x1d4 ret_from_fork+0x10/0x20 Cc: stable@vger.kernel.org Signed-off-by: Yoonho Shin <yoonho.shin@samsung.com> Signed-off-by: Hobin Woo <hobin.woo@samsung.com> Acked-by: Namjae Jeon <linkinjeon@kernel.org> Signed-off-by: Steve French <stfrench@microsoft.com>
author: Hobin Woo <hobin.woo@samsung.com> 2024-07-05 12:27:25 +0900
committer: Steve French <stfrench@microsoft.com> 2024-07-05 09:56:13 -0500
commit: e2e33caa5dc2eae7bddf88b22ce11ec3d760e5cd (patch)
tree: 4384112736f21d7b9116ee907d048d300f7451f3
parent: 25a6e135569b3901452e4863c94560df7c11c492 (diff)
1 files changed, 11 insertions, 2 deletions
diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c
index 786cd45fe18f..840c71c66b30 100644
--- a/fs/smb/server/smb2pdu.c
+++ b/fs/smb/server/smb2pdu.c
@@ -2051,15 +2051,22 @@ out_err1:
  * @access:		file access flags
  * @disposition:	file disposition flags
  * @may_flags:		set with MAY_ flags
+ * @is_dir:		is creating open flags for directory
  *
  * Return:      file open flags
  */
 static int smb2_create_open_flags(bool file_present, __le32 access,
 				  __le32 disposition,
-				  int *may_flags)
+				  int *may_flags,
+				  bool is_dir)
 {
 	int oflags = O_NONBLOCK | O_LARGEFILE;
 
+	if (is_dir) {
+		access &= ~FILE_WRITE_DESIRE_ACCESS_LE;
+		ksmbd_debug(SMB, "Discard write access to a directory\n");
+	}
+
 	if (access & FILE_READ_DESIRED_ACCESS_LE &&
 	    access & FILE_WRITE_DESIRE_ACCESS_LE) {
 		oflags |= O_RDWR;
@@ -3167,7 +3174,9 @@ int smb2_open(struct ksmbd_work *work)
 
 	open_flags = smb2_create_open_flags(file_present, daccess,
 					    req->CreateDisposition,
-					    &may_flags);
+					    &may_flags,
+		req->CreateOptions & FILE_DIRECTORY_FILE_LE ||
+		(file_present && S_ISDIR(d_inode(path.dentry)->i_mode)));
 
 	if (!test_tree_conn_flag(tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) {
 		if (open_flags & (O_CREAT | O_TRUNC)) {
diff --git a/arch/x86/Kbuild b/arch/x86/Kbuild
index cf0ad89f5639..36b985d0e7bf 100644
--- a/arch/x86/Kbuild
+++ b/arch/x86/Kbuild
@@ -1,4 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0
+
+# Branch profiling isn't noinstr-safe.  Disable it for arch/x86/*
+subdir-ccflags-$(CONFIG_TRACE_BRANCH_PROFILING) += -DDISABLE_BRANCH_PROFILING
+
+obj-y += boot/startup/
+
 obj-$(CONFIG_ARCH_HAS_CC_PLATFORM) += coco/
 
 obj-y += entry/
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 2852fcd82cbd..80527299f859 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -14,7 +14,6 @@ config X86_32
 	select ARCH_WANT_IPC_PARSE_VERSION
 	select CLKSRC_I8253
 	select CLONE_BACKWARDS
-	select GENERIC_VDSO_32
 	select HAVE_DEBUG_STACKOVERFLOW
 	select KMAP_LOCAL
 	select MODULES_USE_ELF_REL
@@ -26,6 +25,7 @@ config X86_64
 	depends on 64BIT
 	# Options that are inherently 64-bit kernel only:
 	select ARCH_HAS_GIGANTIC_PAGE
+	select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
 	select ARCH_SUPPORTS_PER_VMA_LOCK
 	select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE
@@ -36,6 +36,7 @@ config X86_64
 	select ARCH_HAS_ELFCORE_COMPAT
 	select ZONE_DMA32
 	select EXECMEM if DYNAMIC_FTRACE
+	select ACPI_MRRM if ACPI
 
 config FORCE_DYNAMIC_FTRACE
 	def_bool y
@@ -72,6 +73,7 @@ config X86
 	select ARCH_ENABLE_SPLIT_PMD_PTLOCK if (PGTABLE_LEVELS > 2) && (X86_64 || X86_PAE)
 	select ARCH_ENABLE_THP_MIGRATION if X86_64 && TRANSPARENT_HUGEPAGE
 	select ARCH_HAS_ACPI_TABLE_UPGRADE	if ACPI
+	select ARCH_HAS_CPU_ATTACK_VECTORS	if CPU_MITIGATIONS
 	select ARCH_HAS_CACHE_LINE_SIZE
 	select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION
 	select ARCH_HAS_CPU_FINALIZE_INIT
@@ -83,6 +85,7 @@ config X86
 	select ARCH_HAS_DMA_OPS			if GART_IOMMU || XEN
 	select ARCH_HAS_EARLY_DEBUG		if KGDB
 	select ARCH_HAS_ELF_RANDOMIZE
+	select ARCH_HAS_EXECMEM_ROX		if X86_64 && STRICT_MODULE_RWX
 	select ARCH_HAS_FAST_MULTIPLIER
 	select ARCH_HAS_FORTIFY_SOURCE
 	select ARCH_HAS_GCOV_PROFILE_ALL
@@ -93,7 +96,8 @@ config X86
 	select ARCH_HAS_NMI_SAFE_THIS_CPU_OPS
 	select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
 	select ARCH_HAS_PMEM_API		if X86_64
-	select ARCH_HAS_PTE_DEVMAP		if X86_64
+	select ARCH_HAS_PREEMPT_LAZY
+	select ARCH_HAS_PTDUMP
 	select ARCH_HAS_PTE_SPECIAL
 	select ARCH_HAS_HW_PTE_YOUNG
 	select ARCH_HAS_NONLEAF_PMD_YOUNG	if PGTABLE_LEVELS > 2
@@ -118,16 +122,19 @@ config X86
 	select ARCH_SUPPORTS_ACPI
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_DEBUG_PAGEALLOC
+	select ARCH_SUPPORTS_HUGETLBFS
 	select ARCH_SUPPORTS_PAGE_TABLE_CHECK	if X86_64
 	select ARCH_SUPPORTS_NUMA_BALANCING	if X86_64
 	select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP	if NR_CPUS <= 4096
-	select ARCH_SUPPORTS_CFI_CLANG		if X86_64
-	select ARCH_USES_CFI_TRAPS		if X86_64 && CFI_CLANG
+	select ARCH_SUPPORTS_CFI		if X86_64
+	select ARCH_USES_CFI_TRAPS		if X86_64 && CFI
 	select ARCH_SUPPORTS_LTO_CLANG
 	select ARCH_SUPPORTS_LTO_CLANG_THIN
 	select ARCH_SUPPORTS_RT
+	select ARCH_SUPPORTS_AUTOFDO_CLANG
+	select ARCH_SUPPORTS_PROPELLER_CLANG    if X86_64
 	select ARCH_USE_BUILTIN_BSWAP
-	select ARCH_USE_CMPXCHG_LOCKREF		if X86_CMPXCHG64
+	select ARCH_USE_CMPXCHG_LOCKREF		if X86_CX8
 	select ARCH_USE_MEMTEST
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
@@ -137,15 +144,16 @@ config X86
 	select ARCH_WANTS_DYNAMIC_TASK_STRUCT
 	select ARCH_WANTS_NO_INSTR
 	select ARCH_WANT_GENERAL_HUGETLB
-	select ARCH_WANT_HUGE_PMD_SHARE
+	select ARCH_WANT_HUGE_PMD_SHARE		if X86_64
 	select ARCH_WANT_LD_ORPHAN_WARN
 	select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP	if X86_64
 	select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP	if X86_64
+	select ARCH_WANT_HUGETLB_VMEMMAP_PREINIT if X86_64
 	select ARCH_WANTS_THP_SWAP		if X86_64
 	select ARCH_HAS_PARANOID_L1D_FLUSH
+	select ARCH_WANT_IRQS_OFF_ACTIVATE_MM
 	select BUILDTIME_TABLE_SORT
 	select CLKEVT_I8253
-	select CLOCKSOURCE_VALIDATE_LAST_CYCLE
 	select CLOCKSOURCE_WATCHDOG
 	# Word-size accesses may read uninitialized data past the trailing \0
 	# in strings and cause false KMSAN reports.
@@ -170,11 +178,9 @@ config X86
 	select GENERIC_IRQ_RESERVATION_MODE
 	select GENERIC_IRQ_SHOW
 	select GENERIC_PENDING_IRQ		if SMP
-	select GENERIC_PTDUMP
 	select GENERIC_SMP_IDLE_THREAD
 	select GENERIC_TIME_VSYSCALL
 	select GENERIC_GETTIMEOFDAY
-	select GENERIC_VDSO_TIME_NS
 	select GENERIC_VDSO_OVERFLOW_PROTECT
 	select GUP_GET_PXX_LOW_HIGH		if X86_PAE
 	select HARDIRQS_SW_RESEND
@@ -193,13 +199,13 @@ config X86
 	select HAVE_ARCH_KFENCE
 	select HAVE_ARCH_KMSAN			if X86_64
 	select HAVE_ARCH_KGDB
+	select HAVE_ARCH_KSTACK_ERASE
 	select HAVE_ARCH_MMAP_RND_BITS		if MMU
 	select HAVE_ARCH_MMAP_RND_COMPAT_BITS	if MMU && COMPAT
 	select HAVE_ARCH_COMPAT_MMAP_BASES	if MMU && COMPAT
 	select HAVE_ARCH_PREL32_RELOCATIONS
 	select HAVE_ARCH_SECCOMP_FILTER
 	select HAVE_ARCH_THREAD_STRUCT_WHITELIST
-	select HAVE_ARCH_STACKLEAK
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_ARCH_TRANSPARENT_HUGEPAGE
 	select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD if X86_64
@@ -222,17 +228,20 @@ config X86
 	select HAVE_DYNAMIC_FTRACE
 	select HAVE_DYNAMIC_FTRACE_WITH_REGS
 	select HAVE_DYNAMIC_FTRACE_WITH_ARGS	if X86_64
+	select HAVE_FTRACE_REGS_HAVING_PT_REGS	if X86_64
 	select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+	select HAVE_DYNAMIC_FTRACE_WITH_JMP	if X86_64
 	select HAVE_SAMPLE_FTRACE_DIRECT	if X86_64
 	select HAVE_SAMPLE_FTRACE_DIRECT_MULTI	if X86_64
 	select HAVE_EBPF_JIT
 	select HAVE_EFFICIENT_UNALIGNED_ACCESS
-	select HAVE_EISA
+	select HAVE_EISA			if X86_32
 	select HAVE_EXIT_THREAD
+	select HAVE_GENERIC_TIF_BITS
 	select HAVE_GUP_FAST
 	select HAVE_FENTRY			if X86_64 || DYNAMIC_FTRACE
-	select HAVE_FTRACE_MCOUNT_RECORD
-	select HAVE_FUNCTION_GRAPH_RETVAL	if HAVE_FUNCTION_GRAPH_TRACER
+	select HAVE_FTRACE_GRAPH_FUNC		if HAVE_FUNCTION_GRAPH_TRACER
+	select HAVE_FUNCTION_GRAPH_FREGS	if HAVE_FUNCTION_GRAPH_TRACER
 	select HAVE_FUNCTION_GRAPH_TRACER	if X86_32 || (X86_64 && DYNAMIC_FTRACE)
 	select HAVE_FUNCTION_TRACER
 	select HAVE_GCC_PLUGINS
@@ -253,6 +262,7 @@ config X86
 	select HAVE_FUNCTION_ERROR_INJECTION
 	select HAVE_KRETPROBES
 	select HAVE_RETHOOK
+	select HAVE_KLP_BUILD			if X86_64
 	select HAVE_LIVEPATCH			if X86_64
 	select HAVE_MIXED_BREAKPOINTS_REGS
 	select HAVE_MOD_ARCH_SPECIFIC
@@ -271,7 +281,8 @@ config X86
 	select HAVE_PCI
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
-	select MMU_GATHER_RCU_TABLE_FREE	if PARAVIRT
+	select ASYNC_KERNEL_PGTABLE_FREE	if IOMMU_SVA
+	select MMU_GATHER_RCU_TABLE_FREE
 	select MMU_GATHER_MERGE_VMAS
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
 	select HAVE_REGS_AND_STACK_ACCESS_API
@@ -279,7 +290,7 @@ config X86
 	select HAVE_FUNCTION_ARG_ACCESS_API
 	select HAVE_SETUP_PER_CPU_AREA
 	select HAVE_SOFTIRQ_ON_OWN_STACK
-	select HAVE_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR
+	select HAVE_STACKPROTECTOR
 	select HAVE_STACK_VALIDATION		if HAVE_OBJTOOL
 	select HAVE_STATIC_CALL
 	select HAVE_STATIC_CALL_INLINE		if HAVE_OBJTOOL
@@ -289,6 +300,7 @@ config X86
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UACCESS_VALIDATION		if HAVE_OBJTOOL
 	select HAVE_UNSTABLE_SCHED_CLOCK
+	select HAVE_UNWIND_USER_FP		if X86_64
 	select HAVE_USER_RETURN_NOTIFIER
 	select HAVE_GENERIC_VDSO
 	select VDSO_GETRANDOM			if X86_64
@@ -319,6 +331,11 @@ config X86
 	select FUNCTION_ALIGNMENT_4B
 	imply IMA_SECURE_AND_OR_TRUSTED_BOOT    if EFI
 	select HAVE_DYNAMIC_FTRACE_NO_PATCHABLE
+	select ARCH_SUPPORTS_PT_RECLAIM		if X86_64
+	select ARCH_SUPPORTS_SCHED_SMT		if SMP
+	select SCHED_SMT			if SMP
+	select ARCH_SUPPORTS_SCHED_CLUSTER	if SMP
+	select ARCH_SUPPORTS_SCHED_MC		if SMP
 
 config INSTRUCTION_DECODER
 	def_bool y
@@ -366,7 +383,7 @@ config GENERIC_CSUM
 config GENERIC_BUG
 	def_bool y
 	depends on BUG
-	select GENERIC_BUG_RELATIVE_POINTERS if X86_64
+	select GENERIC_BUG_RELATIVE_POINTERS
 
 config GENERIC_BUG_RELATIVE_POINTERS
 	bool
@@ -399,10 +416,6 @@ config HAVE_INTEL_TXT
 	def_bool y
 	depends on INTEL_IOMMU && ACPI
 
-config X86_64_SMP
-	def_bool y
-	depends on X86_64 && SMP
-
 config ARCH_SUPPORTS_UPROBES
 	def_bool y
 
@@ -414,20 +427,10 @@ config DYNAMIC_PHYSICAL_MASK
 
 config PGTABLE_LEVELS
 	int
-	default 5 if X86_5LEVEL
-	default 4 if X86_64
+	default 5 if X86_64
 	default 3 if X86_PAE
 	default 2
 
-config CC_HAS_SANE_STACKPROTECTOR
-	bool
-	default $(success,$(srctree)/scripts/gcc-x86_64-has-stack-protector.sh $(CC) $(CLANG_FLAGS)) if 64BIT
-	default $(success,$(srctree)/scripts/gcc-x86_32-has-stack-protector.sh $(CC) $(CLANG_FLAGS))
-	help
-	  We have to make sure stack protector is unconditionally disabled if
-	  the compiler produces broken code or if it does not let us control
-	  the segment on 32-bit kernels.
-
 menu "Processor type and features"
 
 config SMP
@@ -459,18 +462,39 @@ config SMP
 	  If you don't know what to do here, say N.
 
 config X86_X2APIC
-	bool "Support x2apic"
+	bool "x2APIC interrupt controller architecture support"
 	depends on X86_LOCAL_APIC && X86_64 && (IRQ_REMAP || HYPERVISOR_GUEST)
+	default y
 	help
-	  This enables x2apic support on CPUs that have this feature.
+	  x2APIC is an interrupt controller architecture, a component of which
+	  (the local APIC) is present in the CPU. It allows faster access to
+	  the local APIC and supports a larger number of CPUs in the system
+	  than the predecessors.
 
-	  This allows 32-bit apic IDs (so it can support very large systems),
-	  and accesses the local apic via MSRs not via mmio.
+	  x2APIC was introduced in Intel CPUs around 2008 and in AMD EPYC CPUs
+	  in 2019, but it can be disabled by the BIOS. It is also frequently
+	  emulated in virtual machines, even when the host CPU does not support
+	  it. Support in the CPU can be checked by executing
+		grep x2apic /proc/cpuinfo
 
-	  Some Intel systems circa 2022 and later are locked into x2APIC mode
-	  and can not fall back to the legacy APIC modes if SGX or TDX are
-	  enabled in the BIOS. They will boot with very reduced functionality
-	  without enabling this option.
+	  If this configuration option is disabled, the kernel will boot with
+	  very reduced functionality and performance on some platforms that
+	  have x2APIC enabled. On the other hand, on hardware that does not
+	  support x2APIC, a kernel with this option enabled will just fallback
+	  to older APIC implementations.
+
+	  If in doubt, say Y.
+
+config AMD_SECURE_AVIC
+	bool "AMD Secure AVIC"
+	depends on AMD_MEM_ENCRYPT && X86_X2APIC
+	help
+	  Enable this to get AMD Secure AVIC support on guests that have this feature.
+
+	  AMD Secure AVIC provides hardware acceleration for performance sensitive
+	  APIC accesses and support for managing guest owned APIC state for SEV-SNP
+	  guests. Secure AVIC does not support xAPIC mode. It has functional
+	  dependency on x2apic being enabled in the guest.
 
 	  If you don't know what to do here, say N.
 
@@ -496,8 +520,10 @@ config X86_MPPARSE
 config X86_CPU_RESCTRL
 	bool "x86 CPU resource control support"
 	depends on X86 && (CPU_SUP_INTEL || CPU_SUP_AMD)
-	select KERNFS
-	select PROC_CPU_RESCTRL		if PROC_FS
+	depends on MISC_FILESYSTEMS
+	select ARCH_HAS_CPU_RESCTRL
+	select RESCTRL_FS
+	select RESCTRL_FS_PSEUDO_LOCK
 	help
 	  Enable x86 CPU resource control support.
 
@@ -523,12 +549,6 @@ config X86_FRED
 	  ring transitions and exception/interrupt handling if the
 	  system supports it.
 
-config X86_BIGSMP
-	bool "Support for big SMP systems with more than 8 CPUs"
-	depends on SMP && X86_32
-	help
-	  This option is needed for the systems that have more than 8 CPUs.
-
 config X86_EXTENDED_PLATFORM
 	bool "Support for extended (non-PC) x86 platforms"
 	default y
@@ -542,17 +562,17 @@ config X86_EXTENDED_PLATFORM
 	  CONFIG_64BIT.
 
 	  32-bit platforms (CONFIG_64BIT=n):
-		Goldfish (Android emulator)
-		AMD Elan
+		Goldfish (mostly Android emulator)
+		Intel CE media processor (CE4100) SoC
+		Intel Quark
 		RDC R-321x SoC
-		SGI 320/540 (Visual Workstation)
-		STA2X11-based (e.g. Northville)
-		Moorestown MID devices
 
 	  64-bit platforms (CONFIG_64BIT=y):
 		Numascale NumaChip
 		ScaleMP vSMP
 		SGI Ultraviolet
+		Merrifield/Moorefield MID devices
+		Goldfish (mostly Android emulator)
 
 	  If you have one of these systems, or if you want to build a
 	  generic distribution kernel, say Y here - otherwise say N.
@@ -597,8 +617,31 @@ config X86_UV
 	  This option is needed in order to support SGI Ultraviolet systems.
 	  If you don't have one of these, you should say N here.
 
-# Following is an alphabetically sorted list of 32 bit extended platforms
-# Please maintain the alphabetic order if and when there are additions
+config X86_INTEL_MID
+	bool "Intel Z34xx/Z35xx MID platform support"
+	depends on X86_EXTENDED_PLATFORM
+	depends on X86_PLATFORM_DEVICES
+	depends on PCI
+	depends on X86_64 || (EXPERT && PCI_GOANY)
+	depends on X86_IO_APIC
+	select I2C
+	select DW_APB_TIMER
+	select INTEL_SCU_PCI
+	help
+	  Select to build a kernel capable of supporting 64-bit Intel MID
+	  (Mobile Internet Device) platform systems which do not have
+	  the PCI legacy interfaces.
+
+	  The only supported devices are the 22nm Merrified (Z34xx)
+	  and Moorefield (Z35xx) SoC used in the Intel Edison board and
+	  a small number of Android devices such as the Asus Zenfone 2,
+	  Asus FonePad 8 and Dell Venue 7.
+
+	  If you are building for a PC class system or non-MID tablet
+	  SoCs like Bay Trail (Z36xx/Z37xx), say N here.
+
+	  Intel MID platforms are based on an Intel processor and chipset which
+	  consume less power than most of the x86 derivatives.
 
 config X86_GOLDFISH
 	bool "Goldfish (Virtual Platform)"
@@ -608,6 +651,9 @@ config X86_GOLDFISH
 	  for Android development. Unless you are building for the Android
 	  Goldfish emulator say N here.
 
+# Following is an alphabetically sorted list of 32 bit extended platforms
+# Please maintain the alphabetic order if and when there are additions
+
 config X86_INTEL_CE
 	bool "CE4100 TV platform"
 	depends on PCI
@@ -623,24 +669,6 @@ config X86_INTEL_CE
 	  This option compiles in support for the CE4100 SOC for settop
 	  boxes and media devices.
 
-config X86_INTEL_MID
-	bool "Intel MID platform support"
-	depends on X86_EXTENDED_PLATFORM
-	depends on X86_PLATFORM_DEVICES
-	depends on PCI
-	depends on X86_64 || (PCI_GOANY && X86_32)
-	depends on X86_IO_APIC
-	select I2C
-	select DW_APB_TIMER
-	select INTEL_SCU_PCI
-	help
-	  Select to build a kernel capable of supporting Intel MID (Mobile
-	  Internet Device) platform systems which do not have the PCI legacy
-	  interfaces. If you are building for a PC class system say N here.
-
-	  Intel MID platforms are based on an Intel processor and chipset which
-	  consume less power than most of the x86 derivatives.
-
 config X86_INTEL_QUARK
 	bool "Intel Quark platform support"
 	depends on X86_32
@@ -658,6 +686,17 @@ config X86_INTEL_QUARK
 	  Say Y here if you have a Quark based system such as the Arduino
 	  compatible Intel Galileo.
 
+config X86_RDC321X
+	bool "RDC R-321x SoC"
+	depends on X86_32
+	depends on X86_EXTENDED_PLATFORM
+	select M486
+	select X86_REBOOTFIXUPS
+	help
+	  This option is needed for RDC R-321x system-on-chip, also known
+	  as R-8610-(G).
+	  If you don't have one of these chips, you should say N here.
+
 config X86_INTEL_LPSS
 	bool "Intel Low Power Subsystem Support"
 	depends on X86 && ACPI && PCI
@@ -711,29 +750,6 @@ config IOSF_MBI_DEBUG
 
 	  If you don't require the option or are in doubt, say N.
 
-config X86_RDC321X
-	bool "RDC R-321x SoC"
-	depends on X86_32
-	depends on X86_EXTENDED_PLATFORM
-	select M486
-	select X86_REBOOTFIXUPS
-	help
-	  This option is needed for RDC R-321x system-on-chip, also known
-	  as R-8610-(G).
-	  If you don't have one of these chips, you should say N here.
-
-config X86_32_NON_STANDARD
-	bool "Support non-standard 32-bit SMP architectures"
-	depends on X86_32 && SMP
-	depends on X86_EXTENDED_PLATFORM
-	help
-	  This option compiles in the bigsmp and STA2X11 default
-	  subarchitectures.  It is intended for a generic binary
-	  kernel. If you select them all, kernel will probe it one by
-	  one and will fallback to default.
-
-# Alphabetically sorted list of Non standard 32 bit platforms
-
 config X86_SUPPORTS_MEMORY_FAILURE
 	def_bool y
 	# MCE code calls memory_failure():
@@ -743,19 +759,6 @@ config X86_SUPPORTS_MEMORY_FAILURE
 	depends on X86_64 || !SPARSEMEM
 	select ARCH_SUPPORTS_MEMORY_FAILURE
 
-config STA2X11
-	bool "STA2X11 Companion Chip Support"
-	depends on X86_32_NON_STANDARD && PCI
-	select SWIOTLB
-	select MFD_STA2X11
-	select GPIOLIB
-	help
-	  This adds support for boards based on the STA2X11 IO-Hub,
-	  a.k.a. "ConneXt". The chip is used in place of the standard
-	  PC chipset, so all "standard" peripherals are missing. If this
-	  option is selected the kernel will still be able to boot on
-	  standard PC machines.
-
 config X86_32_IRIS
 	tristate "Eurobraille/Iris poweroff module"
 	depends on X86_32
@@ -804,6 +807,7 @@ config PARAVIRT
 
 config PARAVIRT_XXL
 	bool
+	depends on X86_64
 
 config PARAVIRT_DEBUG
 	bool "paravirt-ops debugging"
@@ -890,11 +894,21 @@ config ACRN_GUEST
 	  IOT with small footprint and real-time features. More details can be
 	  found in https://projectacrn.org/.
 
+config BHYVE_GUEST
+	bool "Bhyve (BSD Hypervisor) Guest support"
+	depends on X86_64
+	help
+	  This option allows to run Linux to recognise when it is running as a
+	  guest in the Bhyve hypervisor, and to support more than 255 vCPUs when
+	  when doing so. More details about Bhyve can be found at https://bhyve.org
+	  and https://wiki.freebsd.org/bhyve/.
+
 config INTEL_TDX_GUEST
 	bool "Intel TDX (Trust Domain Extensions) - Guest Support"
 	depends on X86_64 && CPU_SUP_INTEL
 	depends on X86_X2APIC
 	depends on EFI_STUB
+	depends on PARAVIRT
 	select ARCH_HAS_CC_PLATFORM
 	select X86_MEM_ENCRYPT
 	select X86_MCE
@@ -1005,8 +1019,7 @@ config NR_CPUS_RANGE_BEGIN
 config NR_CPUS_RANGE_END
 	int
 	depends on X86_32
-	default   64 if  SMP &&  X86_BIGSMP
-	default    8 if  SMP && !X86_BIGSMP
+	default    8 if  SMP
 	default    1 if !SMP
 
 config NR_CPUS_RANGE_END
@@ -1019,7 +1032,6 @@ config NR_CPUS_RANGE_END
 config NR_CPUS_DEFAULT
 	int
 	depends on X86_32
-	default   32 if  X86_BIGSMP
 	default    8 if  SMP
 	default    1 if !SMP
 
@@ -1043,29 +1055,6 @@ config NR_CPUS
 	  This is purely to save memory: each supported CPU adds about 8KB
 	  to the kernel image.
 
-config SCHED_CLUSTER
-	bool "Cluster scheduler support"
-	depends on SMP
-	default y
-	help
-	  Cluster scheduler support improves the CPU scheduler's decision
-	  making when dealing with machines that have clusters of CPUs.
-	  Cluster usually means a couple of CPUs which are placed closely
-	  by sharing mid-level caches, last-level cache tags or internal
-	  busses.
-
-config SCHED_SMT
-	def_bool y if SMP
-
-config SCHED_MC
-	def_bool y
-	prompt "Multi-core scheduler support"
-	depends on SMP
-	help
-	  Multi-core scheduler support improves the CPU scheduler's decision
-	  making when dealing with multi-core CPU chips at a cost of slightly
-	  increased overhead in some places. If unsure say N here.
-
 config SCHED_MC_PRIO
 	bool "CPU core priorities scheduler support"
 	depends on SCHED_MC
@@ -1095,7 +1084,7 @@ config UP_LATE_INIT
 config X86_UP_APIC
 	bool "Local APIC support on uniprocessors" if !PCI_MSI
 	default PCI_MSI
-	depends on X86_32 && !SMP && !X86_32_NON_STANDARD
+	depends on X86_32 && !SMP
 	help
 	  A local APIC (Advanced Programmable Interrupt Controller) is an
 	  integrated interrupt controller in the CPU. If you have a single-CPU
@@ -1120,7 +1109,7 @@ config X86_UP_IOAPIC
 
 config X86_LOCAL_APIC
 	def_bool y
-	depends on X86_64 || SMP || X86_32_NON_STANDARD || X86_UP_APIC || PCI_MSI
+	depends on X86_64 || SMP || X86_UP_APIC || PCI_MSI
 	select IRQ_DOMAIN_HIERARCHY
 
 config ACPI_MADT_WAKEUP
@@ -1186,7 +1175,7 @@ config X86_MCE_INTEL
 config X86_MCE_AMD
 	def_bool y
 	prompt "AMD MCE features"
-	depends on X86_MCE && X86_LOCAL_APIC && AMD_NB
+	depends on X86_MCE && X86_LOCAL_APIC
 	help
 	  Additional support for AMD specific MCE features such as
 	  the DRAM Error Threshold.
@@ -1334,6 +1323,7 @@ config X86_REBOOTFIXUPS
 config MICROCODE
 	def_bool y
 	depends on CPU_SUP_AMD || CPU_SUP_INTEL
+	select CRYPTO_LIB_SHA256 if CPU_SUP_AMD
 
 config MICROCODE_INITRD32
 	def_bool y
@@ -1351,7 +1341,7 @@ config MICROCODE_LATE_LOADING
 	  use this at your own risk. Late loading taints the kernel unless the
 	  microcode header indicates that it is safe for late loading via the
 	  minimal revision check. This minimal revision check can be enforced on
-	  the kernel command line with "microcode.minrev=Y".
+	  the kernel command line with "microcode=force_minrev".
 
 config MICROCODE_LATE_FORCE_MINREV
 	bool "Enforce late microcode loading minimal revision check"
@@ -1367,10 +1357,22 @@ config MICROCODE_LATE_FORCE_MINREV
 	  revision check fails.
 
 	  This minimal revision check can also be controlled via the
-	  "microcode.minrev" parameter on the kernel command line.
+	  "microcode=force_minrev" parameter on the kernel command line.
 
 	  If unsure say Y.
 
+config MICROCODE_DBG
+	bool "Enable microcode loader debugging"
+	default n
+	depends on MICROCODE
+	help
+	  Enable code which allows for debugging the microcode loader in
+	  a guest. Meaning the patch loading is simulated but everything else
+	  related to patch parsing and handling is done as on baremetal with
+	  the purpose of debugging solely the software side of things.
+
+	  You almost certainly want to say n here.
+
 config X86_MSR
 	tristate "/dev/cpu/*/msr - Model-specific register support"
 	help
@@ -1388,15 +1390,11 @@ config X86_CPUID
 	  with major 203 and minors 0 to 31 for /dev/cpu/0/cpuid to
 	  /dev/cpu/31/cpuid.
 
-choice
-	prompt "High Memory Support"
-	default HIGHMEM4G
+config HIGHMEM4G
+	bool "High Memory Support"
 	depends on X86_32
-
-config NOHIGHMEM
-	bool "off"
 	help
-	  Linux can use up to 64 Gigabytes of physical memory on x86 systems.
+	  Linux can use up to 4 Gigabytes of physical memory on x86 systems.
 	  However, the address space of 32-bit x86 processors is only 4
 	  Gigabytes large. That means that, if you have a large amount of
 	  physical memory, not all of it can be "permanently mapped" by the
@@ -1412,38 +1410,9 @@ config NOHIGHMEM
 	  possible.
 
 	  If the machine has between 1 and 4 Gigabytes physical RAM, then
-	  answer "4GB" here.
+	  answer "Y" here.
 
-	  If more than 4 Gigabytes is used then answer "64GB" here. This
-	  selection turns Intel PAE (Physical Address Extension) mode on.
-	  PAE implements 3-level paging on IA32 processors. PAE is fully
-	  supported by Linux, PAE mode is implemented on all recent Intel
-	  processors (Pentium Pro and better). NOTE: If you say "64GB" here,
-	  then the kernel will not boot on CPUs that don't support PAE!
-
-	  The actual amount of total physical memory will either be
-	  auto detected or can be forced by using a kernel command line option
-	  such as "mem=256M". (Try "man bootparam" or see the documentation of
-	  your boot loader (lilo or loadlin) about how to pass options to the
-	  kernel at boot time.)
-
-	  If unsure, say "off".
-
-config HIGHMEM4G
-	bool "4GB"
-	help
-	  Select this if you have a 32-bit processor and between 1 and 4
-	  gigabytes of physical RAM.
-
-config HIGHMEM64G
-	bool "64GB"
-	depends on X86_HAVE_PAE
-	select X86_PAE
-	help
-	  Select this if you have a 32-bit processor and more than 4
-	  gigabytes of physical RAM.
-
-endchoice
+	  If unsure, say N.
 
 choice
 	prompt "Memory split" if EXPERT
@@ -1489,41 +1458,18 @@ config PAGE_OFFSET
 	depends on X86_32
 
 config HIGHMEM
-	def_bool y
-	depends on X86_32 && (HIGHMEM64G || HIGHMEM4G)
+	def_bool HIGHMEM4G
 
 config X86_PAE
 	bool "PAE (Physical Address Extension) Support"
 	depends on X86_32 && X86_HAVE_PAE
 	select PHYS_ADDR_T_64BIT
-	select SWIOTLB
 	help
 	  PAE is required for NX support, and furthermore enables
 	  larger swapspace support for non-overcommit purposes. It
 	  has the cost of more pagetable lookup overhead, and also
 	  consumes more pagetable space per process.
 
-config X86_5LEVEL
-	bool "Enable 5-level page tables support"
-	default y
-	select DYNAMIC_MEMORY_LAYOUT
-	select SPARSEMEM_VMEMMAP
-	depends on X86_64
-	help
-	  5-level paging enables access to larger address space:
-	  up to 128 PiB of virtual address space and 4 PiB of
-	  physical address space.
-
-	  It will be supported by future Intel CPUs.
-
-	  A kernel with the option enabled can be booted on machines that
-	  support 4- or 5-level paging.
-
-	  See Documentation/arch/x86/x86_64/5level-paging.rst for more
-	  information.
-
-	  Say N if unsure.
-
 config X86_DIRECT_GBPAGES
 	def_bool y
 	depends on X86_64
@@ -1556,6 +1502,7 @@ config AMD_MEM_ENCRYPT
 	select ARCH_HAS_CC_PLATFORM
 	select X86_MEM_ENCRYPT
 	select UNACCEPTED_MEMORY
+	select CRYPTO_LIB_AESGCM
 	help
 	  Say yes to enable support for the encryption of system memory.
 	  This requires an AMD processor that supports Secure Memory
@@ -1565,8 +1512,7 @@ config AMD_MEM_ENCRYPT
 config NUMA
 	bool "NUMA Memory Allocation and Scheduler Support"
 	depends on SMP
-	depends on X86_64 || (X86_32 && HIGHMEM64G && X86_BIGSMP)
-	default y if X86_BIGSMP
+	depends on X86_64
 	select USE_PERCPU_NUMA_NODE_ID
 	select OF_NUMA if OF
 	help
@@ -1579,9 +1525,6 @@ config NUMA
 	  For 64-bit this is recommended if the system is Intel Core i7
 	  (or later), AMD Opteron, or EM64T NUMA.
 
-	  For 32-bit this is only needed if you boot a 32-bit
-	  kernel on a 64-bit NUMA platform.
-
 	  Otherwise, you should say N.
 
 config AMD_NUMA
@@ -1620,7 +1563,6 @@ config ARCH_FLATMEM_ENABLE
 
 config ARCH_SPARSEMEM_ENABLE
 	def_bool y
-	depends on X86_64 || NUMA || X86_32 || X86_32_NON_STANDARD
 	select SPARSEMEM_STATIC if X86_32
 	select SPARSEMEM_VMEMMAP_ENABLE if X86_64
 
@@ -1666,15 +1608,6 @@ config X86_PMEM_LEGACY
 
 	  Say Y if unsure.
 
-config HIGHPTE
-	bool "Allocate 3rd-level pagetables from highmem"
-	depends on HIGHMEM
-	help
-	  The VM uses one page table entry for each page of physical memory.
-	  For systems with a lot of RAM, this can be wasteful of precious
-	  low memory.  Setting this option will put user-space page table
-	  entries in high memory.
-
 config X86_CHECK_BIOS_CORRUPTION
 	bool "Check for low memory corruption"
 	help
@@ -1832,11 +1765,7 @@ config X86_UMIP
 config CC_HAS_IBT
 	# GCC >= 9 and binutils >= 2.29
 	# Retpoline check to work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93654
-	# Clang/LLVM >= 14
-	# https://github.com/llvm/llvm-project/commit/e0b89df2e0f0130881bf6c39bf31d7f6aac00e0f
-	# https://github.com/llvm/llvm-project/commit/dfcf69770bc522b9e411c66454934a37c1f35332
-	def_bool ((CC_IS_GCC && $(cc-option, -fcf-protection=branch -mindirect-branch-register)) || \
-		  (CC_IS_CLANG && CLANG_VERSION >= 140000)) && \
+	def_bool ((CC_IS_GCC && $(cc-option, -fcf-protection=branch -mindirect-branch-register)) || CC_IS_CLANG) && \
 		  $(as-instr,endbr64)
 
 config X86_CET
@@ -1848,8 +1777,6 @@ config X86_KERNEL_IBT
 	prompt "Indirect Branch Tracking"
 	def_bool y
 	depends on X86_64 && CC_HAS_IBT && HAVE_OBJTOOL
-	# https://github.com/llvm/llvm-project/commit/9d7001eba9c4cb311e03cd8cdc231f9e579f2d0f
-	depends on !LD_IS_LLD || LLD_VERSION >= 140000
 	select OBJTOOL
 	select X86_CET
 	help
@@ -1934,8 +1861,7 @@ endchoice
 config X86_SGX
 	bool "Software Guard eXtensions (SGX)"
 	depends on X86_64 && CPU_SUP_INTEL && X86_X2APIC
-	depends on CRYPTO=y
-	depends on CRYPTO_SHA256=y
+	select CRYPTO_LIB_SHA256
 	select MMU_NOTIFIER
 	select NUMA_KEEP_MEMINFO if NUMA
 	select XARRAY_MULTI
@@ -1954,6 +1880,7 @@ config X86_USER_SHADOW_STACK
 	depends on AS_WRUSS
 	depends on X86_64
 	select ARCH_USES_HIGH_VMA_FLAGS
+	select ARCH_HAS_USER_SHADOW_STACK
 	select X86_CET
 	help
 	  Shadow stack protection is a hardware feature that detects function
@@ -1975,7 +1902,6 @@ config INTEL_TDX_HOST
 	depends on X86_X2APIC
 	select ARCH_KEEP_MEMBLOCK
 	depends on CONTIG_ALLOC
-	depends on !KEXEC_CORE
 	depends on X86_MCE
 	help
 	  Intel Trust Domain Extensions (TDX) protects guest VMs from malicious
@@ -2081,9 +2007,15 @@ config ARCH_SUPPORTS_KEXEC_BZIMAGE_VERIFY_SIG
 config ARCH_SUPPORTS_KEXEC_JUMP
 	def_bool y
 
+config ARCH_SUPPORTS_KEXEC_HANDOVER
+	def_bool X86_64
+
 config ARCH_SUPPORTS_CRASH_DUMP
 	def_bool X86_64 || (X86_32 && HIGHMEM)
 
+config ARCH_DEFAULT_CRASH_DUMP
+	def_bool y
+
 config ARCH_SUPPORTS_CRASH_HOTPLUG
 	def_bool y
 
@@ -2187,6 +2119,7 @@ config RANDOMIZE_BASE
 config X86_NEED_RELOCS
 	def_bool y
 	depends on RANDOMIZE_BASE || (X86_32 && RELOCATABLE)
+	select ARCH_VMLINUX_NEEDS_RELOCS
 
 config PHYSICAL_ALIGN
 	hex "Alignment value to which kernel should be aligned"
@@ -2215,17 +2148,10 @@ config PHYSICAL_ALIGN
 
 	  Don't change this unless you know what you are doing.
 
-config DYNAMIC_MEMORY_LAYOUT
-	bool
-	help
-	  This option makes base addresses of vmalloc and vmemmap as well as
-	  __PAGE_OFFSET movable during boot.
-
 config RANDOMIZE_MEMORY
 	bool "Randomize the kernel memory sections"
 	depends on X86_64
 	depends on RANDOMIZE_BASE
-	select DYNAMIC_MEMORY_LAYOUT
 	default RANDOMIZE_BASE
 	help
 	  Randomizes the base virtual address of kernel memory sections
@@ -2257,6 +2183,7 @@ config RANDOMIZE_MEMORY_PHYSICAL_PADDING
 config ADDRESS_MASKING
 	bool "Linear Address Masking support"
 	depends on X86_64
+	depends on COMPILE_TEST || !CPU_MITIGATIONS # wait for LASS
 	help
 	  Linear Address Masking (LAM) modifies the checking that is applied
 	  to 64-bit linear addresses, allowing software to use of the
@@ -2271,7 +2198,7 @@ config HOTPLUG_CPU
 
 config COMPAT_VDSO
 	def_bool n
-	prompt "Disable the 32-bit vDSO (needed for glibc 2.3.3)"
+	prompt "Workaround for glibc 2.3.2 / 2.3.3 (released in year 2003/2004)"
 	depends on COMPAT_32
 	help
 	  Certain buggy versions of glibc will crash if they are
@@ -2415,6 +2342,7 @@ config STRICT_SIGALTSTACK_SIZE
 config CFI_AUTO_DEFAULT
 	bool "Attempt to use FineIBT by default at boot time"
 	depends on FINEIBT
+	depends on !RUST || RUSTC_VERSION >= 108800
 	default y
 	help
 	  Attempt to use FineIBT by default at boot time. If enabled,
@@ -2423,24 +2351,34 @@ config CFI_AUTO_DEFAULT
 
 source "kernel/livepatch/Kconfig"
 
+config X86_BUS_LOCK_DETECT
+	bool "Split Lock Detect and Bus Lock Detect support"
+	depends on CPU_SUP_INTEL || CPU_SUP_AMD
+	default y
+	help
+	  Enable Split Lock Detect and Bus Lock Detect functionalities.
+	  See <file:Documentation/arch/x86/buslock.rst> for more information.
+
 endmenu
 
 config CC_HAS_NAMED_AS
 	def_bool $(success,echo 'int __seg_fs fs; int __seg_gs gs;' | $(CC) -x c - -S -o /dev/null)
 	depends on CC_IS_GCC
 
+#
+# -fsanitize=kernel-address (KASAN) and -fsanitize=thread (KCSAN)
+# are incompatible with named address spaces with GCC < 13.3
+# (see GCC PR sanitizer/111736 and also PR sanitizer/115172).
+#
+
 config CC_HAS_NAMED_AS_FIXED_SANITIZERS
-	def_bool CC_IS_GCC && GCC_VERSION >= 130300
+	def_bool y
+	depends on !(KASAN || KCSAN) || GCC_VERSION >= 130300
+	depends on !(UBSAN_BOOL && KASAN) || GCC_VERSION >= 140200
 
 config USE_X86_SEG_SUPPORT
-	def_bool y
-	depends on CC_HAS_NAMED_AS
-	#
-	# -fsanitize=kernel-address (KASAN) and -fsanitize=thread
-	# (KCSAN) are incompatible with named address spaces with
-	# GCC < 13.3 - see GCC PR sanitizer/111736.
-	#
-	depends on !(KASAN || KCSAN) || CC_HAS_NAMED_AS_FIXED_SANITIZERS
+	def_bool CC_HAS_NAMED_AS
+	depends on CC_HAS_NAMED_AS_FIXED_SANITIZERS
 
 config CC_HAS_SLS
 	def_bool $(cc-option,-mharden-sls=all)
@@ -2451,6 +2389,10 @@ config CC_HAS_RETURN_THUNK
 config CC_HAS_ENTRY_PADDING
 	def_bool $(cc-option,-fpatchable-function-entry=16,16)
 
+config CC_HAS_KCFI_ARITY
+	def_bool $(cc-option,-fsanitize=kcfi -fsanitize-kcfi-arity)
+	depends on CC_IS_CLANG && !RUST
+
 config FUNCTION_PADDING_CFI
 	int
 	default 59 if FUNCTION_ALIGNMENT_64B
@@ -2459,11 +2401,11 @@ config FUNCTION_PADDING_CFI
 	default  3 if FUNCTION_ALIGNMENT_8B
 	default  0
 
-# Basically: FUNCTION_ALIGNMENT - 5*CFI_CLANG
+# Basically: FUNCTION_ALIGNMENT - 5*CFI
 # except Kconfig can't do arithmetic :/
 config FUNCTION_PADDING_BYTES
 	int
-	default FUNCTION_PADDING_CFI if CFI_CLANG
+	default FUNCTION_PADDING_CFI if CFI
 	default FUNCTION_ALIGNMENT
 
 config CALL_PADDING
@@ -2473,9 +2415,13 @@ config CALL_PADDING
 
 config FINEIBT
 	def_bool y
-	depends on X86_KERNEL_IBT && CFI_CLANG && MITIGATION_RETPOLINE
+	depends on X86_KERNEL_IBT && CFI && MITIGATION_RETPOLINE
 	select CALL_PADDING
 
+config FINEIBT_BHI
+	def_bool y
+	depends on FINEIBT && CC_HAS_KCFI_ARITY
+
 config HAVE_CALL_THUNKS
 	def_bool y
 	depends on CC_HAS_ENTRY_PADDING && MITIGATION_RETHUNK && OBJTOOL
@@ -2486,7 +2432,7 @@ config CALL_THUNKS
 
 config PREFIX_SYMBOLS
 	def_bool y
-	depends on CALL_PADDING && !CFI_CLANG
+	depends on CALL_PADDING && !CFI
 
 menuconfig CPU_MITIGATIONS
 	bool "Mitigations for CPU vulnerabilities"
@@ -2551,15 +2497,14 @@ config MITIGATION_CALL_DEPTH_TRACKING
 	default y
 	help
 	  Compile the kernel with call depth tracking to mitigate the Intel
-	  SKL Return-Speculation-Buffer (RSB) underflow issue. The
-	  mitigation is off by default and needs to be enabled on the
-	  kernel command line via the retbleed=stuff option. For
-	  non-affected systems the overhead of this option is marginal as
-	  the call depth tracking is using run-time generated call thunks
-	  in a compiler generated padding area and call patching. This
-	  increases text size by ~5%. For non affected systems this space
-	  is unused. On affected SKL systems this results in a significant
-	  performance gain over the IBRS mitigation.
+	  SKL Return-Stack-Buffer (RSB) underflow issue. The mitigation is off
+	  by default and needs to be enabled on the kernel command line via the
+	  retbleed=stuff option. For non-affected systems the overhead of this
+	  option is marginal as the call depth tracking is using run-time
+	  generated call thunks in a compiler generated padding area and call
+	  patching. This increases text size by ~5%. For non affected systems
+	  this space is unused. On affected SKL systems this results in a
+	  significant performance gain over the IBRS mitigation.
 
 config CALL_THUNKS_DEBUG
 	bool "Enable call thunks and call depth tracking debugging"
@@ -2579,7 +2524,8 @@ config MITIGATION_IBPB_ENTRY
 	depends on CPU_SUP_AMD && X86_64
 	default y
 	help
-	  Compile the kernel with support for the retbleed=ibpb mitigation.
+	  Compile the kernel with support for the retbleed=ibpb and
+	  spec_rstack_overflow={ibpb,ibpb-vmexit} mitigations.
 
 config MITIGATION_IBRS_ENTRY
 	bool "Enable IBRS on kernel entry"
@@ -2739,6 +2685,36 @@ config MITIGATION_SSB
 	  of speculative execution in a similar way to the Meltdown and Spectre
 	  security vulnerabilities.
 
+config MITIGATION_ITS
+	bool "Enable Indirect Target Selection mitigation"
+	depends on CPU_SUP_INTEL && X86_64
+	depends on MITIGATION_RETPOLINE && MITIGATION_RETHUNK
+	select EXECMEM
+	default y
+	help
+	  Enable Indirect Target Selection (ITS) mitigation. ITS is a bug in
+	  BPU on some Intel CPUs that may allow Spectre V2 style attacks. If
+	  disabled, mitigation cannot be enabled via cmdline.
+	  See <file:Documentation/admin-guide/hw-vuln/indirect-target-selection.rst>
+
+config MITIGATION_TSA
+	bool "Mitigate Transient Scheduler Attacks"
+	depends on CPU_SUP_AMD
+	default y
+	help
+	  Enable mitigation for Transient Scheduler Attacks. TSA is a hardware
+	  security vulnerability on AMD CPUs which can lead to forwarding of
+	  invalid info to subsequent instructions and thus can affect their
+	  timing and thereby cause a leakage.
+
+config MITIGATION_VMSCAPE
+	bool "Mitigate VMSCAPE"
+	depends on KVM
+	default y
+	help
+	  Enable mitigation for VMSCAPE attacks. VMSCAPE is a hardware security
+	  vulnerability on Intel and AMD CPUs that may allow a guest to do
+	  Spectre v2 style attacks on userspace hypervisor.
 endif
 
 config ARCH_HAS_ADD_PAGES
@@ -2942,6 +2918,19 @@ config PCI_MMCONFIG
 	default y
 	depends on PCI && (ACPI || JAILHOUSE_GUEST)
 	depends on X86_64 || (PCI_GOANY || PCI_GOMMCONFIG)
+	help
+	  Add support for accessing the PCI configuration space as a memory
+	  mapped area. It is the recommended method if the system supports
+	  this (it must have PCI Express and ACPI for it to be available).
+
+	  In the unlikely case that enabling this configuration option causes
+	  problems, the mechanism can be switched off with the 'pci=nommconf'
+	  command line parameter.
+
+	  Say N only if you are sure that your platform does not support this
+	  access method or you have problems caused by it.
+
+	  Say Y otherwise.
 
 config PCI_OLPC
 	def_bool y
@@ -2956,13 +2945,21 @@ config MMCONF_FAM10H
 	depends on X86_64 && PCI_MMCONFIG && ACPI
 
 config PCI_CNB20LE_QUIRK
-	bool "Read CNB20LE Host Bridge Windows" if EXPERT
-	depends on PCI
+	bool "Read PCI host bridge windows from the CNB20LE chipset" if EXPERT
+	depends on X86_32 && PCI
 	help
 	  Read the PCI windows out of the CNB20LE host bridge. This allows
 	  PCI hotplug to work on systems with the CNB20LE chipset which do
 	  not have ACPI.
 
+	  The ServerWorks (later Broadcom) CNB20LE was a chipset designed
+	  most probably only for Pentium III.
+
+	  To find out if you have such a chipset, search for a PCI device with
+	  1166:0009 PCI IDs, for example by executing
+		lspci -nn | grep '1166:0009'
+	  The code is inactive if there is none.
+
 	  There's no public spec for this chipset, and this functionality
 	  is known to be incomplete.
 
@@ -3114,6 +3111,10 @@ endif # X86_32
 
 config AMD_NB
 	def_bool y
+	depends on AMD_NODE
+
+config AMD_NODE
+	def_bool y
 	depends on CPU_SUP_AMD && PCI
 
 endmenu
@@ -3176,4 +3177,6 @@ config HAVE_ATOMIC_IOMAP
 
 source "arch/x86/kvm/Kconfig"
 
+source "arch/x86/Kconfig.cpufeatures"
+
 source "arch/x86/Kconfig.assembler"
diff --git a/arch/x86/Kconfig.assembler b/arch/x86/Kconfig.assembler
index 6d20a6ce0507..b1c59fb0a4c9 100644
--- a/arch/x86/Kconfig.assembler
+++ b/arch/x86/Kconfig.assembler
@@ -1,40 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 # Copyright (C) 2020 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 
-config AS_AVX512
-	def_bool $(as-instr,vpmovm2b %k1$(comma)%zmm5)
-	help
-	  Supported by binutils >= 2.25 and LLVM integrated assembler
-
-config AS_SHA1_NI
-	def_bool $(as-instr,sha1msg1 %xmm0$(comma)%xmm1)
-	help
-	  Supported by binutils >= 2.24 and LLVM integrated assembler
-
-config AS_SHA256_NI
-	def_bool $(as-instr,sha256msg1 %xmm0$(comma)%xmm1)
-	help
-	  Supported by binutils >= 2.24 and LLVM integrated assembler
-config AS_TPAUSE
-	def_bool $(as-instr,tpause %ecx)
-	help
-	  Supported by binutils >= 2.31.1 and LLVM integrated assembler >= V7
-
-config AS_GFNI
-	def_bool $(as-instr,vgf2p8mulb %xmm0$(comma)%xmm1$(comma)%xmm2)
-	help
-	  Supported by binutils >= 2.30 and LLVM integrated assembler
-
-config AS_VAES
-	def_bool $(as-instr,vaesenc %ymm0$(comma)%ymm1$(comma)%ymm2)
-	help
-	  Supported by binutils >= 2.30 and LLVM integrated assembler
-
-config AS_VPCLMULQDQ
-	def_bool $(as-instr,vpclmulqdq \$0x10$(comma)%ymm0$(comma)%ymm1$(comma)%ymm2)
-	help
-	  Supported by binutils >= 2.30 and LLVM integrated assembler
-
 config AS_WRUSS
 	def_bool $(as-instr64,wrussq %rax$(comma)(%rbx))
 	help
diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu
index 2a7279d80460..f928cf6e3252 100644
--- a/arch/x86/Kconfig.cpu
+++ b/arch/x86/Kconfig.cpu
@@ -1,9 +1,9 @@
 # SPDX-License-Identifier: GPL-2.0
 # Put here option for CPU selection and depending optimization
 choice
-	prompt "Processor family"
-	default M686 if X86_32
-	default GENERIC_CPU if X86_64
+	prompt "x86-32 Processor family"
+	depends on X86_32
+	default M686
 	help
 	  This is the processor type of your CPU. This information is
 	  used for optimizing purposes. In order to compile a kernel
@@ -31,7 +31,6 @@ choice
 	  - "Pentium-4" for the Intel Pentium 4 or P4-based Celeron.
 	  - "K6" for the AMD K6, K6-II and K6-III (aka K6-3D).
 	  - "Athlon" for the AMD K7 family (Athlon/Duron/Thunderbird).
-	  - "Opteron/Athlon64/Hammer/K8" for all K8 and newer AMD CPUs.
 	  - "Crusoe" for the Transmeta Crusoe series.
 	  - "Efficeon" for the Transmeta Efficeon series.
 	  - "Winchip-C6" for original IDT Winchip.
@@ -42,13 +41,10 @@ choice
 	  - "CyrixIII/VIA C3" for VIA Cyrix III or VIA C3.
 	  - "VIA C3-2" for VIA C3-2 "Nehemiah" (model 9 and above).
 	  - "VIA C7" for VIA C7.
-	  - "Intel P4" for the Pentium 4/Netburst microarchitecture.
-	  - "Core 2/newer Xeon" for all core2 and newer Intel CPUs.
 	  - "Intel Atom" for the Atom-microarchitecture CPUs.
-	  - "Generic-x86-64" for a kernel which runs on any x86-64 CPU.
 
 	  See each option's help text for additional details. If you don't know
-	  what to do, choose "486".
+	  what to do, choose "Pentium-Pro".
 
 config M486SX
 	bool "486SX"
@@ -114,11 +110,11 @@ config MPENTIUMIII
 	  extensions.
 
 config MPENTIUMM
-	bool "Pentium M"
+	bool "Pentium M/Pentium Dual Core/Core Solo/Core Duo"
 	depends on X86_32
 	help
 	  Select this for Intel Pentium M (not Pentium-4 M)
-	  notebook chips.
+	  "Merom" Core Solo/Duo notebook chips
 
 config MPENTIUM4
 	bool "Pentium-4/Celeron(P4-based)/Pentium-4 M/older Xeon"
@@ -139,22 +135,10 @@ config MPENTIUM4
 		-Mobile Pentium 4
 		-Mobile Pentium 4 M
 		-Extreme Edition (Gallatin)
-		-Prescott
-		-Prescott 2M
-		-Cedar Mill
-		-Presler
-		-Smithfiled
 	    Xeons (Intel Xeon, Xeon MP, Xeon LV, Xeon MV) corename:
 		-Foster
 		-Prestonia
 		-Gallatin
-		-Nocona
-		-Irwindale
-		-Cranford
-		-Potomac
-		-Paxville
-		-Dempsey
-
 
 config MK6
 	bool "K6/K6-II/K6-III"
@@ -172,13 +156,6 @@ config MK7
 	  some extended instructions, and passes appropriate optimization
 	  flags to GCC.
 
-config MK8
-	bool "Opteron/Athlon64/Hammer/K8"
-	help
-	  Select this for an AMD Opteron or Athlon64 Hammer-family processor.
-	  Enables use of some extended instructions, and passes appropriate
-	  optimization flags to GCC.
-
 config MCRUSOE
 	bool "Crusoe"
 	depends on X86_32
@@ -258,43 +235,39 @@ config MVIAC7
 	  Select this for a VIA C7.  Selecting this uses the correct cache
 	  shift and tells gcc to treat the CPU as a 686.
 
-config MPSC
-	bool "Intel P4 / older Netburst based Xeon"
-	depends on X86_64
-	help
-	  Optimize for Intel Pentium 4, Pentium D and older Nocona/Dempsey
-	  Xeon CPUs with Intel 64bit which is compatible with x86-64.
-	  Note that the latest Xeons (Xeon 51xx and 53xx) are not based on the
-	  Netburst core and shouldn't use this option. You can distinguish them
-	  using the cpu family field
-	  in /proc/cpuinfo. Family 15 is an older Xeon, Family 6 a newer one.
-
-config MCORE2
-	bool "Core 2/newer Xeon"
-	help
-
-	  Select this for Intel Core 2 and newer Core 2 Xeons (Xeon 51xx and
-	  53xx) CPUs. You can distinguish newer from older Xeons by the CPU
-	  family in /proc/cpuinfo. Newer ones have 6 and older ones 15
-	  (not a typo)
-
 config MATOM
 	bool "Intel Atom"
 	help
-
 	  Select this for the Intel Atom platform. Intel Atom CPUs have an
 	  in-order pipelining architecture and thus can benefit from
 	  accordingly optimized code. Use a recent GCC with specific Atom
 	  support in order to fully benefit from selecting this option.
 
-config GENERIC_CPU
-	bool "Generic-x86-64"
+endchoice
+
+config CC_HAS_MARCH_NATIVE
+	# This flag might not be available in cross-compilers:
+	def_bool $(cc-option, -march=native)
+	# LLVM 18 has an easily triggered internal compiler error in core
+	# networking code with '-march=native' on certain systems:
+	# https://github.com/llvm/llvm-project/issues/72026
+	# LLVM 19 introduces an optimization that resolves some high stack
+	# usage warnings that only appear wth '-march=native'.
+	depends on CC_IS_GCC || CLANG_VERSION >= 190100
+
+config X86_NATIVE_CPU
+	bool "Build and optimize for local/native CPU"
 	depends on X86_64
+	depends on CC_HAS_MARCH_NATIVE
 	help
-	  Generic x86-64 CPU.
-	  Run equally well on all x86-64 CPUs.
+	  Optimize for the current CPU used to compile the kernel.
+	  Use this option if you intend to build the kernel for your
+	  local machine.
 
-endchoice
+	  Note that such a kernel might not work optimally on a
+	  different x86 machine.
+
+	  If unsure, say N.
 
 config X86_GENERIC
 	bool "Generic x86 support"
@@ -317,8 +290,8 @@ config X86_INTERNODE_CACHE_SHIFT
 
 config X86_L1_CACHE_SHIFT
 	int
-	default "7" if MPENTIUM4 || MPSC
-	default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU
+	default "7" if MPENTIUM4
+	default "6" if MK7 || MPENTIUMM || MATOM || MVIAC7 || X86_GENERIC || X86_64
 	default "4" if MELAN || M486SX || M486 || MGEODEGX1
 	default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX
 
@@ -336,51 +309,35 @@ config X86_ALIGNMENT_16
 
 config X86_INTEL_USERCOPY
 	def_bool y
-	depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2
+	depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK7 || MEFFICEON
 
 config X86_USE_PPRO_CHECKSUM
 	def_bool y
-	depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM
-
-#
-# P6_NOPs are a relatively minor optimization that require a family >=
-# 6 processor, except that it is broken on certain VIA chips.
-# Furthermore, AMD chips prefer a totally different sequence of NOPs
-# (which work on all CPUs).  In addition, it looks like Virtual PC
-# does not understand them.
-#
-# As a result, disallow these if we're not compiling for X86_64 (these
-# NOPs do work on all x86-64 capable chips); the list of processors in
-# the right-hand clause are the cores that benefit from this optimization.
-#
-config X86_P6_NOP
-	def_bool y
-	depends on X86_64
-	depends on (MCORE2 || MPENTIUM4 || MPSC)
+	depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MATOM
 
 config X86_TSC
 	def_bool y
-	depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM) || X86_64
+	depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MATOM) || X86_64
 
 config X86_HAVE_PAE
 	def_bool y
-	depends on MCRUSOE || MEFFICEON || MCYRIXIII || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC7 || MCORE2 || MATOM || X86_64
+	depends on MCRUSOE || MEFFICEON || MCYRIXIII || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC7 || MATOM || X86_64
 
-config X86_CMPXCHG64
+config X86_CX8
 	def_bool y
-	depends on X86_HAVE_PAE || M586TSC || M586MMX || MK6 || MK7
+	depends on X86_HAVE_PAE || M586TSC || M586MMX || MK6 || MK7 || MGEODEGX1 || MGEODE_LX
 
 # this should be set for all -march=.. options where the compiler
 # generates cmov.
 config X86_CMOV
 	def_bool y
-	depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX)
+	depends on (MK7 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || MATOM || MGEODE_LX || X86_64)
 
 config X86_MINIMUM_CPU_FAMILY
 	int
 	default "64" if X86_64
-	default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCORE2 || MK7 || MK8)
-	default "5" if X86_32 && X86_CMPXCHG64
+	default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MK7)
+	default "5" if X86_32 && X86_CX8
 	default "4"
 
 config X86_DEBUGCTLMSR
@@ -401,6 +358,10 @@ menuconfig PROCESSOR_SELECT
 	  This lets you choose what x86 vendor support code your kernel
 	  will include.
 
+config BROADCAST_TLB_FLUSH
+	def_bool y
+	depends on CPU_SUP_AMD && 64BIT
+
 config CPU_SUP_INTEL
 	default y
 	bool "Support Intel processors" if PROCESSOR_SELECT
diff --git a/arch/x86/Kconfig.cpufeatures b/arch/x86/Kconfig.cpufeatures
new file mode 100644
index 000000000000..733d5aff2456
--- /dev/null
+++ b/arch/x86/Kconfig.cpufeatures
@@ -0,0 +1,201 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# x86 feature bits (see arch/x86/include/asm/cpufeatures.h) that are
+# either REQUIRED to be enabled, or DISABLED (always ignored) for this
+# particular compile-time configuration.  The tests for these features
+# are turned into compile-time constants via the generated
+# <asm/cpufeaturemasks.h>.
+#
+# The naming of these variables *must* match asm/cpufeatures.h, e.g.,
+#     X86_FEATURE_ALWAYS <==> X86_REQUIRED_FEATURE_ALWAYS
+#     X86_FEATURE_FRED   <==> X86_DISABLED_FEATURE_FRED
+#
+# And these REQUIRED and DISABLED config options are manipulated in an
+# AWK script as the following example:
+#
+#                          +----------------------+
+#                          |    X86_FRED = y ?    |
+#                          +----------------------+
+#                              /             \
+#                           Y /               \ N
+#  +-------------------------------------+   +-------------------------------+
+#  | X86_DISABLED_FEATURE_FRED undefined |   | X86_DISABLED_FEATURE_FRED = y |
+#  +-------------------------------------+   +-------------------------------+
+#                                                        |
+#                                                        |
+#     +-------------------------------------------+      |
+#     | X86_FEATURE_FRED: feature word 12, bit 17 | ---->|
+#     +-------------------------------------------+      |
+#                                                        |
+#                                                        |
+#                                     +-------------------------------+
+#                                     | set bit 17 of DISABLED_MASK12 |
+#                                     +-------------------------------+
+#
+
+config X86_REQUIRED_FEATURE_ALWAYS
+	def_bool y
+
+config X86_REQUIRED_FEATURE_NOPL
+	def_bool y
+	depends on X86_64 || X86_P6_NOP
+
+config X86_REQUIRED_FEATURE_CX8
+	def_bool y
+	depends on X86_CX8
+
+# this should be set for all -march=.. options where the compiler
+# generates cmov.
+config X86_REQUIRED_FEATURE_CMOV
+	def_bool y
+	depends on X86_CMOV
+
+# this should be set for all -march= options where the compiler
+# generates movbe.
+config X86_REQUIRED_FEATURE_MOVBE
+	def_bool y
+	depends on MATOM
+
+config X86_REQUIRED_FEATURE_CPUID
+	def_bool y
+	depends on X86_64
+
+config X86_REQUIRED_FEATURE_UP
+	def_bool y
+	depends on !SMP
+
+config X86_REQUIRED_FEATURE_FPU
+	def_bool y
+	depends on !MATH_EMULATION
+
+config X86_REQUIRED_FEATURE_PAE
+	def_bool y
+	depends on X86_64 || X86_PAE
+
+config X86_REQUIRED_FEATURE_PSE
+	def_bool y
+	depends on X86_64 && !PARAVIRT_XXL
+
+config X86_REQUIRED_FEATURE_PGE
+	def_bool y
+	depends on X86_64 && !PARAVIRT_XXL
+
+config X86_REQUIRED_FEATURE_MSR
+	def_bool y
+	depends on X86_64
+
+config X86_REQUIRED_FEATURE_FXSR
+	def_bool y
+	depends on X86_64
+
+config X86_REQUIRED_FEATURE_XMM
+	def_bool y
+	depends on X86_64
+
+config X86_REQUIRED_FEATURE_XMM2
+	def_bool y
+	depends on X86_64
+
+config X86_REQUIRED_FEATURE_LM
+	def_bool y
+	depends on X86_64
+
+config X86_DISABLED_FEATURE_UMIP
+	def_bool y
+	depends on !X86_UMIP
+
+config X86_DISABLED_FEATURE_VME
+	def_bool y
+	depends on X86_64
+
+config X86_DISABLED_FEATURE_K6_MTRR
+	def_bool y
+	depends on X86_64
+
+config X86_DISABLED_FEATURE_CYRIX_ARR
+	def_bool y
+	depends on X86_64
+
+config X86_DISABLED_FEATURE_CENTAUR_MCR
+	def_bool y
+	depends on X86_64
+
+config X86_DISABLED_FEATURE_PCID
+	def_bool y
+	depends on !X86_64
+
+config X86_DISABLED_FEATURE_LASS
+	def_bool y
+	depends on X86_32
+
+config X86_DISABLED_FEATURE_PKU
+	def_bool y
+	depends on !X86_INTEL_MEMORY_PROTECTION_KEYS
+
+config X86_DISABLED_FEATURE_OSPKE
+	def_bool y
+	depends on !X86_INTEL_MEMORY_PROTECTION_KEYS
+
+config X86_DISABLED_FEATURE_PTI
+	def_bool y
+	depends on !MITIGATION_PAGE_TABLE_ISOLATION
+
+config X86_DISABLED_FEATURE_RETPOLINE
+	def_bool y
+	depends on !MITIGATION_RETPOLINE
+
+config X86_DISABLED_FEATURE_RETPOLINE_LFENCE
+	def_bool y
+	depends on !MITIGATION_RETPOLINE
+
+config X86_DISABLED_FEATURE_RETHUNK
+	def_bool y
+	depends on !MITIGATION_RETHUNK
+
+config X86_DISABLED_FEATURE_UNRET
+	def_bool y
+	depends on !MITIGATION_UNRET_ENTRY
+
+config X86_DISABLED_FEATURE_CALL_DEPTH
+	def_bool y
+	depends on !MITIGATION_CALL_DEPTH_TRACKING
+
+config X86_DISABLED_FEATURE_LAM
+	def_bool y
+	depends on !ADDRESS_MASKING
+
+config X86_DISABLED_FEATURE_ENQCMD
+	def_bool y
+	depends on !INTEL_IOMMU_SVM
+
+config X86_DISABLED_FEATURE_SGX
+	def_bool y
+	depends on !X86_SGX
+
+config X86_DISABLED_FEATURE_XENPV
+	def_bool y
+	depends on !XEN_PV
+
+config X86_DISABLED_FEATURE_TDX_GUEST
+	def_bool y
+	depends on !INTEL_TDX_GUEST
+
+config X86_DISABLED_FEATURE_USER_SHSTK
+	def_bool y
+	depends on !X86_USER_SHADOW_STACK
+
+config X86_DISABLED_FEATURE_IBT
+	def_bool y
+	depends on !X86_KERNEL_IBT
+
+config X86_DISABLED_FEATURE_FRED
+	def_bool y
+	depends on !X86_FRED
+
+config X86_DISABLED_FEATURE_SEV_SNP
+	def_bool y
+	depends on !KVM_AMD_SEV
+
+config X86_DISABLED_FEATURE_INVLPGB
+	def_bool y
+	depends on !BROADCAST_TLB_FLUSH
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 74777a97e394..c95c3aaadf97 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -59,7 +59,7 @@ config EARLY_PRINTK_USB_XDBC
 config EFI_PGT_DUMP
 	bool "Dump the EFI pagetable"
 	depends on EFI
-	select PTDUMP_CORE
+	select PTDUMP
 	help
 	  Enable this if you want to dump the EFI page table before
 	  enabling virtual mode. This can be used to debug miscellaneous
@@ -97,7 +97,7 @@ config IOMMU_DEBUG
 	  code. When you use it make sure you have a big enough
 	  IOMMU/AGP aperture.  Most of the options enabled by this can
 	  be set more finegrained using the iommu= command line
-	  options. See Documentation/arch/x86/x86_64/boot-options.rst for more
+	  options. See Documentation/admin-guide/kernel-parameters.txt for more
 	  details.
 
 config IOMMU_LEAK
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index cd75e78a06c1..1d403a3612ea 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -13,8 +13,8 @@ else
 endif
 
 ifdef CONFIG_CC_IS_GCC
-RETPOLINE_CFLAGS	:= $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register)
-RETPOLINE_VDSO_CFLAGS	:= $(call cc-option,-mindirect-branch=thunk-inline -mindirect-branch-register)
+RETPOLINE_CFLAGS	:= -mindirect-branch=thunk-extern -mindirect-branch-register
+RETPOLINE_VDSO_CFLAGS	:= -mindirect-branch=thunk-inline -mindirect-branch-register
 endif
 ifdef CONFIG_CC_IS_CLANG
 RETPOLINE_CFLAGS	:= -mretpoline-external-thunk
@@ -37,17 +37,19 @@ export RETPOLINE_VDSO_CFLAGS
 
 # For gcc stack alignment is specified with -mpreferred-stack-boundary,
 # clang has the option -mstack-alignment for that purpose.
-ifneq ($(call cc-option, -mpreferred-stack-boundary=4),)
+ifdef CONFIG_CC_IS_GCC
       cc_stack_align4 := -mpreferred-stack-boundary=2
       cc_stack_align8 := -mpreferred-stack-boundary=3
-else ifneq ($(call cc-option, -mstack-alignment=16),)
+endif
+ifdef CONFIG_CC_IS_CLANG
       cc_stack_align4 := -mstack-alignment=4
       cc_stack_align8 := -mstack-alignment=8
 endif
 
 # How to compile the 16-bit code.  Note we always compile for -march=i386;
 # that way we can complain to the user if the CPU is insufficient.
-REALMODE_CFLAGS	:= -std=gnu11 -m16 -g -Os -DDISABLE_BRANCH_PROFILING -D__DISABLE_EXPORTS \
+REALMODE_CFLAGS	:= -std=gnu11 -fms-extensions -m16 -g -Os \
+		   -DDISABLE_BRANCH_PROFILING -D__DISABLE_EXPORTS \
 		   -Wall -Wstrict-prototypes -march=i386 -mregparm=3 \
 		   -fno-strict-aliasing -fomit-frame-pointer -fno-pic \
 		   -mno-mmx -mno-sse $(call cc-option,-fcf-protection=none)
@@ -59,6 +61,7 @@ REALMODE_CFLAGS += $(cc_stack_align4)
 REALMODE_CFLAGS += $(CLANG_FLAGS)
 ifdef CONFIG_CC_IS_CLANG
 REALMODE_CFLAGS += -Wno-gnu
+REALMODE_CFLAGS += -Wno-microsoft-anon-tag
 endif
 export REALMODE_CFLAGS
 
@@ -74,7 +77,7 @@ export BITS
 #
 #    https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53383
 #
-KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx
+KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx -mno-sse4a
 KBUILD_RUSTFLAGS += --target=$(objtree)/scripts/target.json
 KBUILD_RUSTFLAGS += -Ctarget-feature=-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-avx,-avx2
 
@@ -83,19 +86,7 @@ KBUILD_RUSTFLAGS += -Ctarget-feature=-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-av
 #
 CC_FLAGS_FPU := -msse -msse2
 ifdef CONFIG_CC_IS_GCC
-# Stack alignment mismatch, proceed with caution.
-# GCC < 7.1 cannot compile code using `double` and -mpreferred-stack-boundary=3
-# (8B stack alignment).
-# See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53383
-#
-# The "-msse" in the first argument is there so that the
-# -mpreferred-stack-boundary=3 build error:
-#
-#  -mpreferred-stack-boundary=3 is not between 4 and 12
-#
-# can be triggered. Otherwise gcc doesn't complain.
 CC_FLAGS_FPU += -mhard-float
-CC_FLAGS_FPU += $(call cc-option,-msse -mpreferred-stack-boundary=3,-mpreferred-stack-boundary=4)
 endif
 
 ifeq ($(CONFIG_X86_KERNEL_IBT),y)
@@ -109,7 +100,7 @@ ifeq ($(CONFIG_X86_KERNEL_IBT),y)
 #   https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104816
 #
 KBUILD_CFLAGS += $(call cc-option,-fcf-protection=branch -fno-jump-tables)
-KBUILD_RUSTFLAGS += -Zcf-protection=branch -Zno-jump-tables
+KBUILD_RUSTFLAGS += -Zcf-protection=branch $(if $(call rustc-min-version,109300),-Cjump-tables=n,-Zno-jump-tables)
 else
 KBUILD_CFLAGS += $(call cc-option,-fcf-protection=none)
 endif
@@ -137,16 +128,12 @@ ifeq ($(CONFIG_X86_32),y)
         include $(srctree)/arch/x86/Makefile_32.cpu
         KBUILD_CFLAGS += $(cflags-y)
 
-        # temporary until string.h is fixed
+    ifneq ($(call clang-min-version, 160000),y)
+        # https://github.com/llvm/llvm-project/issues/53645
         KBUILD_CFLAGS += -ffreestanding
-
-    ifeq ($(CONFIG_STACKPROTECTOR),y)
-        ifeq ($(CONFIG_SMP),y)
-			KBUILD_CFLAGS += -mstack-protector-guard-reg=fs -mstack-protector-guard-symbol=__stack_chk_guard
-        else
-			KBUILD_CFLAGS += -mstack-protector-guard=global
-        endif
     endif
+
+        percpu_seg := fs
 else
         BITS := 64
         UTS_MACHINE := x86_64
@@ -163,7 +150,7 @@ else
 
         # Don't autogenerate traditional x87 instructions
         KBUILD_CFLAGS += -mno-80387
-        KBUILD_CFLAGS += $(call cc-option,-mno-fp-ret-in-387)
+        KBUILD_CFLAGS += -mno-fp-ret-in-387
 
         # By default gcc and clang use a stack alignment of 16 bytes for x86.
         # However the standard kernel entry on x86-64 leaves the stack on an
@@ -175,27 +162,31 @@ else
         KBUILD_CFLAGS += $(cc_stack_align8)
 
 	# Use -mskip-rax-setup if supported.
-	KBUILD_CFLAGS += $(call cc-option,-mskip-rax-setup)
-
-        # FIXME - should be integrated in Makefile.cpu (Makefile_32.cpu)
-        cflags-$(CONFIG_MK8)		+= -march=k8
-        cflags-$(CONFIG_MPSC)		+= -march=nocona
-        cflags-$(CONFIG_MCORE2)		+= -march=core2
-        cflags-$(CONFIG_MATOM)		+= -march=atom
-        cflags-$(CONFIG_GENERIC_CPU)	+= -mtune=generic
-        KBUILD_CFLAGS += $(cflags-y)
+	KBUILD_CFLAGS += -mskip-rax-setup
 
-        rustflags-$(CONFIG_MK8)		+= -Ctarget-cpu=k8
-        rustflags-$(CONFIG_MPSC)	+= -Ctarget-cpu=nocona
-        rustflags-$(CONFIG_MCORE2)	+= -Ctarget-cpu=core2
-        rustflags-$(CONFIG_MATOM)	+= -Ctarget-cpu=atom
-        rustflags-$(CONFIG_GENERIC_CPU)	+= -Ztune-cpu=generic
-        KBUILD_RUSTFLAGS += $(rustflags-y)
+ifdef CONFIG_X86_NATIVE_CPU
+        KBUILD_CFLAGS += -march=native
+        KBUILD_RUSTFLAGS += -Ctarget-cpu=native
+else
+        KBUILD_CFLAGS += -march=x86-64 -mtune=generic
+        KBUILD_RUSTFLAGS += -Ctarget-cpu=x86-64 -Ztune-cpu=generic
+endif
 
         KBUILD_CFLAGS += -mno-red-zone
         KBUILD_CFLAGS += -mcmodel=kernel
         KBUILD_RUSTFLAGS += -Cno-redzone=y
         KBUILD_RUSTFLAGS += -Ccode-model=kernel
+
+        percpu_seg := gs
+endif
+
+ifeq ($(CONFIG_STACKPROTECTOR),y)
+    ifeq ($(CONFIG_SMP),y)
+	KBUILD_CFLAGS += -mstack-protector-guard-reg=$(percpu_seg)
+	KBUILD_CFLAGS += -mstack-protector-guard-symbol=__ref_stack_chk_guard
+    else
+	KBUILD_CFLAGS += -mstack-protector-guard=global
+    endif
 endif
 
 #
@@ -250,12 +241,6 @@ endif
 
 KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE)
 
-ifdef CONFIG_X86_NEED_RELOCS
-LDFLAGS_vmlinux := --emit-relocs --discard-none
-else
-LDFLAGS_vmlinux :=
-endif
-
 #
 # The 64-bit kernel must be aligned to 2MB.  Pass -z max-page-size=0x200000 to
 # the linker to force 2MB page size regardless of the default page size used
@@ -276,6 +261,20 @@ archheaders:
 	$(Q)$(MAKE) $(build)=arch/x86/entry/syscalls all
 
 ###
+# <asm/cpufeaturemasks.h> header generation
+
+cpufeaturemasks.hdr := arch/x86/include/generated/asm/cpufeaturemasks.h
+cpufeaturemasks.awk := $(srctree)/arch/x86/tools/cpufeaturemasks.awk
+cpufeatures_hdr := $(srctree)/arch/x86/include/asm/cpufeatures.h
+targets += $(cpufeaturemasks.hdr)
+      filechk_gen_featuremasks = $(AWK) -f $(cpufeaturemasks.awk) $(cpufeatures_hdr) $(KCONFIG_CONFIG)
+
+$(cpufeaturemasks.hdr): $(cpufeaturemasks.awk) $(cpufeatures_hdr) $(KCONFIG_CONFIG) FORCE
+	$(shell mkdir -p $(dir $@))
+	$(call filechk,gen_featuremasks)
+archprepare: $(cpufeaturemasks.hdr)
+
+###
 # Kernel objects
 
 libs-y  += arch/x86/lib/
diff --git a/arch/x86/Makefile.postlink b/arch/x86/Makefile.postlink
deleted file mode 100644
index fef2e977cc7d..000000000000
--- a/arch/x86/Makefile.postlink
+++ /dev/null
@@ -1,44 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-# ===========================================================================
-# Post-link x86 pass
-# ===========================================================================
-#
-# 1. Separate relocations from vmlinux into vmlinux.relocs.
-# 2. Strip relocations from vmlinux.
-
-PHONY := __archpost
-__archpost:
-
--include include/config/auto.conf
-include $(srctree)/scripts/Kbuild.include
-
-CMD_RELOCS = arch/x86/tools/relocs
-OUT_RELOCS = arch/x86/boot/compressed
-quiet_cmd_relocs = RELOCS  $(OUT_RELOCS)/$@.relocs
-      cmd_relocs = \
-	mkdir -p $(OUT_RELOCS); \
-	$(CMD_RELOCS) $@ > $(OUT_RELOCS)/$@.relocs; \
-	$(CMD_RELOCS) --abs-relocs $@
-
-quiet_cmd_strip_relocs = RSTRIP  $@
-      cmd_strip_relocs = \
-	$(OBJCOPY) --remove-section='.rel.*' --remove-section='.rel__*' \
-		   --remove-section='.rela.*' --remove-section='.rela__*' $@
-
-# `@true` prevents complaint when there is nothing to be done
-
-vmlinux: FORCE
-	@true
-ifeq ($(CONFIG_X86_NEED_RELOCS),y)
-	$(call cmd,relocs)
-	$(call cmd,strip_relocs)
-endif
-
-clean:
-	@rm -f $(OUT_RELOCS)/vmlinux.relocs
-
-PHONY += FORCE clean
-
-FORCE:
-
-.PHONY: $(PHONY)
diff --git a/arch/x86/Makefile.um b/arch/x86/Makefile.um
index a46b1397ad01..c86cbd9cbba3 100644
--- a/arch/x86/Makefile.um
+++ b/arch/x86/Makefile.um
@@ -7,12 +7,13 @@ core-y += arch/x86/crypto/
 # GCC versions < 11. See:
 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=99652
 #
-ifeq ($(CONFIG_CC_IS_CLANG),y)
-KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx
-KBUILD_RUSTFLAGS += --target=$(objtree)/scripts/target.json
+ifeq ($(call gcc-min-version, 110000)$(CONFIG_CC_IS_CLANG),y)
+KBUILD_CFLAGS +=  -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx
 KBUILD_RUSTFLAGS += -Ctarget-feature=-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-avx,-avx2
 endif
 
+KBUILD_RUSTFLAGS += --target=$(objtree)/scripts/target.json
+
 ifeq ($(CONFIG_X86_32),y)
 START := 0x8048000
 
diff --git a/arch/x86/Makefile_32.cpu b/arch/x86/Makefile_32.cpu
index 94834c4b5e5e..af7de9a42752 100644
--- a/arch/x86/Makefile_32.cpu
+++ b/arch/x86/Makefile_32.cpu
@@ -24,7 +24,6 @@ cflags-$(CONFIG_MK6)		+= -march=k6
 # Please note, that patches that add -march=athlon-xp and friends are pointless.
 # They make zero difference whatsosever to performance at this time.
 cflags-$(CONFIG_MK7)		+= -march=athlon
-cflags-$(CONFIG_MK8)		+= $(call cc-option,-march=k8,-march=athlon)
 cflags-$(CONFIG_MCRUSOE)	+= -march=i686 $(align)
 cflags-$(CONFIG_MEFFICEON)	+= -march=i686 $(call tune,pentium3) $(align)
 cflags-$(CONFIG_MWINCHIPC6)	+= $(call cc-option,-march=winchip-c6,-march=i586)
@@ -32,9 +31,7 @@ cflags-$(CONFIG_MWINCHIP3D)	+= $(call cc-option,-march=winchip2,-march=i586)
 cflags-$(CONFIG_MCYRIXIII)	+= $(call cc-option,-march=c3,-march=i486) $(align)
 cflags-$(CONFIG_MVIAC3_2)	+= $(call cc-option,-march=c3-2,-march=i686)
 cflags-$(CONFIG_MVIAC7)		+= -march=i686
-cflags-$(CONFIG_MCORE2)		+= -march=i686 $(call tune,core2)
-cflags-$(CONFIG_MATOM)		+= $(call cc-option,-march=atom,$(call cc-option,-march=core2,-march=i686)) \
-	$(call cc-option,-mtune=atom,$(call cc-option,-mtune=generic))
+cflags-$(CONFIG_MATOM)		+= -march=atom
 
 # AMD Elan support
 cflags-$(CONFIG_MELAN)		+= -march=i486
diff --git a/arch/x86/boot/.gitignore b/arch/x86/boot/.gitignore
index 1189be057ebd..070ef534c915 100644
--- a/arch/x86/boot/.gitignore
+++ b/arch/x86/boot/.gitignore
@@ -12,3 +12,4 @@ fdimage
 mtools.conf
 image.iso
 hdimage
+tools/
diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile
index 9cc0ff6e9067..3f9fb3698d66 100644
--- a/arch/x86/boot/Makefile
+++ b/arch/x86/boot/Makefile
@@ -35,7 +35,6 @@ setup-y		+= video-vesa.o
 setup-y		+= video-bios.o
 
 targets		+= $(setup-y)
-hostprogs	:= tools/build
 hostprogs	+= mkcpustr
 
 HOST_EXTRACFLAGS += -I$(srctree)/tools/include \
@@ -54,18 +53,15 @@ targets += cpustr.h
 
 KBUILD_CFLAGS	:= $(REALMODE_CFLAGS) -D_SETUP
 KBUILD_AFLAGS	:= $(KBUILD_CFLAGS) -D__ASSEMBLY__
-KBUILD_CFLAGS	+= $(call cc-option,-fmacro-prefix-map=$(srctree)/=)
 KBUILD_CFLAGS	+= -fno-asynchronous-unwind-tables
 KBUILD_CFLAGS	+= $(CONFIG_CC_IMPLICIT_FALLTHROUGH)
 
 $(obj)/bzImage: asflags-y  := $(SVGA_MODE)
 
 quiet_cmd_image = BUILD   $@
-silent_redirect_image = >/dev/null
-cmd_image = $(obj)/tools/build $(obj)/setup.bin $(obj)/vmlinux.bin \
-			       $(obj)/zoffset.h $@ $($(quiet)redirect_image)
+      cmd_image = (dd if=$< bs=4k conv=sync status=none; cat $(filter-out $<,$(real-prereqs))) >$@
 
-$(obj)/bzImage: $(obj)/setup.bin $(obj)/vmlinux.bin $(obj)/tools/build FORCE
+$(obj)/bzImage: $(obj)/setup.bin $(obj)/vmlinux.bin FORCE
 	$(call if_changed,image)
 	@$(kecho) 'Kernel: $@ is ready' ' (#'$(or $(KBUILD_BUILD_VERSION),`cat .version`)')'
 
@@ -75,7 +71,7 @@ $(obj)/vmlinux.bin: $(obj)/compressed/vmlinux FORCE
 
 SETUP_OBJS = $(addprefix $(obj)/,$(setup-y))
 
-sed-zoffset := -e 's/^\([0-9a-fA-F]*\) [a-zA-Z] \(startup_32\|efi.._stub_entry\|efi\(32\)\?_pe_entry\|input_data\|kernel_info\|_end\|_ehead\|_text\|_e\?data\|z_.*\)$$/\#define ZO_\2 0x\1/p'
+sed-zoffset := -e 's/^\([0-9a-fA-F]*\) [a-zA-Z] \(startup_32\|efi.._stub_entry\|efi\(32\)\?_pe_entry\|input_data\|kernel_info\|_end\|_ehead\|_text\|_e\?data\|_e\?sbat\|z_.*\)$$/\#define ZO_\2 0x\1/p'
 
 quiet_cmd_zoffset = ZOFFSET $@
       cmd_zoffset = $(NM) $< | sed -n $(sed-zoffset) > $@
diff --git a/arch/x86/boot/a20.c b/arch/x86/boot/a20.c
index a2b6b428922a..bda042933a05 100644
--- a/arch/x86/boot/a20.c
+++ b/arch/x86/boot/a20.c
@@ -135,29 +135,29 @@ int enable_a20(void)
 		  (legacy free, etc.) */
 	       if (a20_test_short())
 		       return 0;
-	       
+
 	       /* Next, try the BIOS (INT 0x15, AX=0x2401) */
 	       enable_a20_bios();
 	       if (a20_test_short())
 		       return 0;
-	       
+
 	       /* Try enabling A20 through the keyboard controller */
 	       kbc_err = empty_8042();
 
 	       if (a20_test_short())
 		       return 0; /* BIOS worked, but with delayed reaction */
-	
+
 	       if (!kbc_err) {
 		       enable_a20_kbc();
 		       if (a20_test_long())
 			       return 0;
 	       }
-	       
+
 	       /* Finally, try enabling the "fast A20 gate" */
 	       enable_a20_fast();
 	       if (a20_test_long())
 		       return 0;
        }
-       
+
        return -1;
 }
diff --git a/arch/x86/boot/bioscall.S b/arch/x86/boot/bioscall.S
index aa9b96457584..cf4a6155714e 100644
--- a/arch/x86/boot/bioscall.S
+++ b/arch/x86/boot/bioscall.S
@@ -32,7 +32,7 @@ intcall:
 	movw	%dx, %si
 	movw	%sp, %di
 	movw	$11, %cx
-	rep; movsl
+	rep movsl
 
 	/* Pop full state from the stack */
 	popal
@@ -67,7 +67,7 @@ intcall:
 	jz	4f
 	movw	%sp, %si
 	movw	$11, %cx
-	rep; movsl
+	rep movsl
 4:	addw	$44, %sp
 
 	/* Restore state and return */
diff --git a/arch/x86/boot/bitops.h b/arch/x86/boot/bitops.h
index 8518ae214c9b..79e15971529d 100644
--- a/arch/x86/boot/bitops.h
+++ b/arch/x86/boot/bitops.h
@@ -27,7 +27,7 @@ static inline bool variable_test_bit(int nr, const void *addr)
 	bool v;
 	const u32 *p = addr;
 
-	asm("btl %2,%1" CC_SET(c) : CC_OUT(c) (v) : "m" (*p), "Ir" (nr));
+	asm("btl %2,%1" : "=@ccc" (v) : "m" (*p), "Ir" (nr));
 	return v;
 }
 
diff --git a/arch/x86/boot/boot.h b/arch/x86/boot/boot.h
index 148ba5c5106e..8e3eab34dff4 100644
--- a/arch/x86/boot/boot.h
+++ b/arch/x86/boot/boot.h
@@ -16,7 +16,7 @@
 
 #define STACK_SIZE	1024	/* Minimum number of bytes for stack */
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/stdarg.h>
 #include <linux/types.h>
@@ -34,7 +34,7 @@
 extern struct setup_header hdr;
 extern struct boot_params boot_params;
 
-#define cpu_relax()	asm volatile("rep; nop")
+#define cpu_relax()	asm volatile("pause")
 
 static inline void io_delay(void)
 {
@@ -155,15 +155,15 @@ static inline void wrgs32(u32 v, addr_t addr)
 static inline bool memcmp_fs(const void *s1, addr_t s2, size_t len)
 {
 	bool diff;
-	asm volatile("fs; repe; cmpsb" CC_SET(nz)
-		     : CC_OUT(nz) (diff), "+D" (s1), "+S" (s2), "+c" (len));
+	asm volatile("fs repe cmpsb"
+		     : "=@ccnz" (diff), "+D" (s1), "+S" (s2), "+c" (len));
 	return diff;
 }
 static inline bool memcmp_gs(const void *s1, addr_t s2, size_t len)
 {
 	bool diff;
-	asm volatile("gs; repe; cmpsb" CC_SET(nz)
-		     : CC_OUT(nz) (diff), "+D" (s1), "+S" (s2), "+c" (len));
+	asm volatile("gs repe cmpsb"
+		     : "=@ccnz" (diff), "+D" (s1), "+S" (s2), "+c" (len));
 	return diff;
 }
 
@@ -193,8 +193,6 @@ static inline bool heap_free(size_t n)
 
 void copy_to_fs(addr_t dst, void *src, size_t len);
 void *copy_from_fs(void *dst, addr_t src, size_t len);
-void copy_to_gs(addr_t dst, void *src, size_t len);
-void *copy_from_gs(void *dst, addr_t src, size_t len);
 
 /* a20.c */
 int enable_a20(void);
@@ -305,7 +303,6 @@ void initregs(struct biosregs *regs);
 int strcmp(const char *str1, const char *str2);
 int strncmp(const char *cs, const char *ct, size_t count);
 size_t strnlen(const char *s, size_t maxlen);
-unsigned int atou(const char *s);
 unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base);
 size_t strlen(const char *s);
 char *strchr(const char *s, int c);
@@ -328,6 +325,6 @@ void probe_cards(int unsafe);
 /* video-vesa.c */
 void vesa_store_edid(void);
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* BOOT_BOOT_H */
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index f2051644de94..68f9d7a1683b 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -25,6 +25,7 @@ targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \
 # avoid errors with '-march=i386', and future flags may depend on the target to
 # be valid.
 KBUILD_CFLAGS := -m$(BITS) -O2 $(CLANG_FLAGS)
+KBUILD_CFLAGS += -std=gnu11 -fms-extensions
 KBUILD_CFLAGS += -fno-strict-aliasing -fPIE
 KBUILD_CFLAGS += -Wundef
 KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
@@ -35,19 +36,21 @@ KBUILD_CFLAGS += -mno-mmx -mno-sse
 KBUILD_CFLAGS += -ffreestanding -fshort-wchar
 KBUILD_CFLAGS += -fno-stack-protector
 KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member)
-KBUILD_CFLAGS += $(call cc-disable-warning, gnu)
+ifdef CONFIG_CC_IS_CLANG
+KBUILD_CFLAGS += -Wno-gnu
+KBUILD_CFLAGS += -Wno-microsoft-anon-tag
+endif
 KBUILD_CFLAGS += -Wno-pointer-sign
-KBUILD_CFLAGS += $(call cc-option,-fmacro-prefix-map=$(srctree)/=)
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 KBUILD_CFLAGS += -D__DISABLE_EXPORTS
 # Disable relocation relaxation in case the link is not PIE.
 KBUILD_CFLAGS += $(call cc-option,-Wa$(comma)-mrelax-relocations=no)
 KBUILD_CFLAGS += -include $(srctree)/include/linux/hidden.h
 
-# sev.c indirectly includes inat-table.h which is generated during
+# sev-decode-insn.c indirectly includes inat-table.c which is generated during
 # compilation and stored in $(objtree). Add the directory to the includes so
 # that the compiler finds it even with out-of-tree builds (make O=/some/path).
-CFLAGS_sev.o += -I$(objtree)/arch/x86/lib/
+CFLAGS_sev-handle-vc.o += -I$(objtree)/arch/x86/lib/
 
 KBUILD_AFLAGS  := $(KBUILD_CFLAGS) -D__ASSEMBLY__
 
@@ -73,7 +76,7 @@ LDFLAGS_vmlinux += -T
 hostprogs	:= mkpiggy
 HOST_EXTRACFLAGS += -I$(srctree)/tools/include
 
-sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVW] \(_text\|__start_rodata\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p'
+sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABbCDGRSTtVW] \(_text\|__start_rodata\|_sinittext\|__inittext_end\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p'
 
 quiet_cmd_voffset = VOFFSET $@
       cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@
@@ -96,7 +99,7 @@ ifdef CONFIG_X86_64
 	vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o
 	vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/mem_encrypt.o
 	vmlinux-objs-y += $(obj)/pgtable_64.o
-	vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev.o
+	vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev.o $(obj)/sev-handle-vc.o
 endif
 
 vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o
@@ -104,8 +107,13 @@ vmlinux-objs-$(CONFIG_INTEL_TDX_GUEST) += $(obj)/tdx.o $(obj)/tdcall.o $(obj)/td
 vmlinux-objs-$(CONFIG_UNACCEPTED_MEMORY) += $(obj)/mem.o
 
 vmlinux-objs-$(CONFIG_EFI) += $(obj)/efi.o
-vmlinux-objs-$(CONFIG_EFI_MIXED) += $(obj)/efi_mixed.o
 vmlinux-libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a
+vmlinux-libs-$(CONFIG_X86_64)	+= $(objtree)/arch/x86/boot/startup/lib.a
+vmlinux-objs-$(CONFIG_EFI_SBAT) += $(obj)/sbat.o
+
+ifdef CONFIG_EFI_SBAT
+$(obj)/sbat.o: $(CONFIG_EFI_SBAT_FILE)
+endif
 
 $(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE
 	$(call if_changed,ld)
@@ -116,9 +124,12 @@ $(obj)/vmlinux.bin: vmlinux FORCE
 
 targets += $(patsubst $(obj)/%,%,$(vmlinux-objs-y)) vmlinux.bin.all vmlinux.relocs
 
-# vmlinux.relocs is created by the vmlinux postlink step.
-$(obj)/vmlinux.relocs: vmlinux
-	@true
+CMD_RELOCS = arch/x86/tools/relocs
+quiet_cmd_relocs = RELOCS  $@
+      cmd_relocs = $(CMD_RELOCS) $< > $@;$(CMD_RELOCS) --abs-relocs $<
+
+$(obj)/vmlinux.relocs: vmlinux.unstripped FORCE
+	$(call if_changed,relocs)
 
 vmlinux.bin.all-y := $(obj)/vmlinux.bin
 vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += $(obj)/vmlinux.relocs
diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S
deleted file mode 100644
index 876fc6d46a13..000000000000
--- a/arch/x86/boot/compressed/efi_mixed.S
+++ /dev/null
@@ -1,341 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) 2014, 2015 Intel Corporation; author Matt Fleming
- *
- * Early support for invoking 32-bit EFI services from a 64-bit kernel.
- *
- * Because this thunking occurs before ExitBootServices() we have to
- * restore the firmware's 32-bit GDT and IDT before we make EFI service
- * calls.
- *
- * On the plus side, we don't have to worry about mangling 64-bit
- * addresses into 32-bits because we're executing with an identity
- * mapped pagetable and haven't transitioned to 64-bit virtual addresses
- * yet.
- */
-
-#include <linux/linkage.h>
-#include <asm/asm-offsets.h>
-#include <asm/msr.h>
-#include <asm/page_types.h>
-#include <asm/processor-flags.h>
-#include <asm/segment.h>
-#include <asm/setup.h>
-
-	.code64
-	.text
-/*
- * When booting in 64-bit mode on 32-bit EFI firmware, startup_64_mixed_mode()
- * is the first thing that runs after switching to long mode. Depending on
- * whether the EFI handover protocol or the compat entry point was used to
- * enter the kernel, it will either branch to the common 64-bit EFI stub
- * entrypoint efi_stub_entry() directly, or via the 64-bit EFI PE/COFF
- * entrypoint efi_pe_entry(). In the former case, the bootloader must provide a
- * struct bootparams pointer as the third argument, so the presence of such a
- * pointer is used to disambiguate.
- *
- *                                                             +--------------+
- *  +------------------+     +------------+            +------>| efi_pe_entry |
- *  | efi32_pe_entry   |---->|            |            |       +-----------+--+
- *  +------------------+     |            |     +------+----------------+  |
- *                           | startup_32 |---->| startup_64_mixed_mode |  |
- *  +------------------+     |            |     +------+----------------+  |
- *  | efi32_stub_entry |---->|            |            |                   |
- *  +------------------+     +------------+            |                   |
- *                                                     V                   |
- *                           +------------+     +----------------+         |
- *                           | startup_64 |<----| efi_stub_entry |<--------+
- *                           +------------+     +----------------+
- */
-SYM_FUNC_START(startup_64_mixed_mode)
-	lea	efi32_boot_args(%rip), %rdx
-	mov	0(%rdx), %edi
-	mov	4(%rdx), %esi
-
-	/* Switch to the firmware's stack */
-	movl	efi32_boot_sp(%rip), %esp
-	andl	$~7, %esp
-
-#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
-	mov	8(%rdx), %edx		// saved bootparams pointer
-	test	%edx, %edx
-	jnz	efi_stub_entry
-#endif
-	/*
-	 * efi_pe_entry uses MS calling convention, which requires 32 bytes of
-	 * shadow space on the stack even if all arguments are passed in
-	 * registers. We also need an additional 8 bytes for the space that
-	 * would be occupied by the return address, and this also results in
-	 * the correct stack alignment for entry.
-	 */
-	sub	$40, %rsp
-	mov	%rdi, %rcx		// MS calling convention
-	mov	%rsi, %rdx
-	jmp	efi_pe_entry
-SYM_FUNC_END(startup_64_mixed_mode)
-
-SYM_FUNC_START(__efi64_thunk)
-	push	%rbp
-	push	%rbx
-
-	movl	%ds, %eax
-	push	%rax
-	movl	%es, %eax
-	push	%rax
-	movl	%ss, %eax
-	push	%rax
-
-	/* Copy args passed on stack */
-	movq	0x30(%rsp), %rbp
-	movq	0x38(%rsp), %rbx
-	movq	0x40(%rsp), %rax
-
-	/*
-	 * Convert x86-64 ABI params to i386 ABI
-	 */
-	subq	$64, %rsp
-	movl	%esi, 0x0(%rsp)
-	movl	%edx, 0x4(%rsp)
-	movl	%ecx, 0x8(%rsp)
-	movl	%r8d, 0xc(%rsp)
-	movl	%r9d, 0x10(%rsp)
-	movl	%ebp, 0x14(%rsp)
-	movl	%ebx, 0x18(%rsp)
-	movl	%eax, 0x1c(%rsp)
-
-	leaq	0x20(%rsp), %rbx
-	sgdt	(%rbx)
-	sidt	16(%rbx)
-
-	leaq	1f(%rip), %rbp
-
-	/*
-	 * Switch to IDT and GDT with 32-bit segments. These are the firmware
-	 * GDT and IDT that were installed when the kernel started executing.
-	 * The pointers were saved by the efi32_entry() routine below.
-	 *
-	 * Pass the saved DS selector to the 32-bit code, and use far return to
-	 * restore the saved CS selector.
-	 */
-	lidt	efi32_boot_idt(%rip)
-	lgdt	efi32_boot_gdt(%rip)
-
-	movzwl	efi32_boot_ds(%rip), %edx
-	movzwq	efi32_boot_cs(%rip), %rax
-	pushq	%rax
-	leaq	efi_enter32(%rip), %rax
-	pushq	%rax
-	lretq
-
-1:	addq	$64, %rsp
-	movq	%rdi, %rax
-
-	pop	%rbx
-	movl	%ebx, %ss
-	pop	%rbx
-	movl	%ebx, %es
-	pop	%rbx
-	movl	%ebx, %ds
-	/* Clear out 32-bit selector from FS and GS */
-	xorl	%ebx, %ebx
-	movl	%ebx, %fs
-	movl	%ebx, %gs
-
-	pop	%rbx
-	pop	%rbp
-	RET
-SYM_FUNC_END(__efi64_thunk)
-
-	.code32
-#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
-SYM_FUNC_START(efi32_stub_entry)
-	call	1f
-1:	popl	%ecx
-	leal	(efi32_boot_args - 1b)(%ecx), %ebx
-
-	/* Clear BSS */
-	xorl	%eax, %eax
-	leal	(_bss - 1b)(%ecx), %edi
-	leal	(_ebss - 1b)(%ecx), %ecx
-	subl	%edi, %ecx
-	shrl	$2, %ecx
-	cld
-	rep	stosl
-
-	add	$0x4, %esp		/* Discard return address */
-	popl	%ecx
-	popl	%edx
-	popl	%esi
-	movl	%esi, 8(%ebx)
-	jmp	efi32_entry
-SYM_FUNC_END(efi32_stub_entry)
-#endif
-
-/*
- * EFI service pointer must be in %edi.
- *
- * The stack should represent the 32-bit calling convention.
- */
-SYM_FUNC_START_LOCAL(efi_enter32)
-	/* Load firmware selector into data and stack segment registers */
-	movl	%edx, %ds
-	movl	%edx, %es
-	movl	%edx, %fs
-	movl	%edx, %gs
-	movl	%edx, %ss
-
-	/* Reload pgtables */
-	movl	%cr3, %eax
-	movl	%eax, %cr3
-
-	/* Disable paging */
-	movl	%cr0, %eax
-	btrl	$X86_CR0_PG_BIT, %eax
-	movl	%eax, %cr0
-
-	/* Disable long mode via EFER */
-	movl	$MSR_EFER, %ecx
-	rdmsr
-	btrl	$_EFER_LME, %eax
-	wrmsr
-
-	call	*%edi
-
-	/* We must preserve return value */
-	movl	%eax, %edi
-
-	/*
-	 * Some firmware will return with interrupts enabled. Be sure to
-	 * disable them before we switch GDTs and IDTs.
-	 */
-	cli
-
-	lidtl	16(%ebx)
-	lgdtl	(%ebx)
-
-	movl	%cr4, %eax
-	btsl	$(X86_CR4_PAE_BIT), %eax
-	movl	%eax, %cr4
-
-	movl	%cr3, %eax
-	movl	%eax, %cr3
-
-	movl	$MSR_EFER, %ecx
-	rdmsr
-	btsl	$_EFER_LME, %eax
-	wrmsr
-
-	xorl	%eax, %eax
-	lldt	%ax
-
-	pushl	$__KERNEL_CS
-	pushl	%ebp
-
-	/* Enable paging */
-	movl	%cr0, %eax
-	btsl	$X86_CR0_PG_BIT, %eax
-	movl	%eax, %cr0
-	lret
-SYM_FUNC_END(efi_enter32)
-
-/*
- * This is the common EFI stub entry point for mixed mode.
- *
- * Arguments:	%ecx	image handle
- * 		%edx	EFI system table pointer
- *
- * Since this is the point of no return for ordinary execution, no registers
- * are considered live except for the function parameters. [Note that the EFI
- * stub may still exit and return to the firmware using the Exit() EFI boot
- * service.]
- */
-SYM_FUNC_START_LOCAL(efi32_entry)
-	call	1f
-1:	pop	%ebx
-
-	/* Save firmware GDTR and code/data selectors */
-	sgdtl	(efi32_boot_gdt - 1b)(%ebx)
-	movw	%cs, (efi32_boot_cs - 1b)(%ebx)
-	movw	%ds, (efi32_boot_ds - 1b)(%ebx)
-
-	/* Store firmware IDT descriptor */
-	sidtl	(efi32_boot_idt - 1b)(%ebx)
-
-	/* Store firmware stack pointer */
-	movl	%esp, (efi32_boot_sp - 1b)(%ebx)
-
-	/* Store boot arguments */
-	leal	(efi32_boot_args - 1b)(%ebx), %ebx
-	movl	%ecx, 0(%ebx)
-	movl	%edx, 4(%ebx)
-	movb	$0x0, 12(%ebx)          // efi_is64
-
-	/*
-	 * Allocate some memory for a temporary struct boot_params, which only
-	 * needs the minimal pieces that startup_32() relies on.
-	 */
-	subl	$PARAM_SIZE, %esp
-	movl	%esp, %esi
-	movl	$PAGE_SIZE, BP_kernel_alignment(%esi)
-	movl	$_end - 1b, BP_init_size(%esi)
-	subl	$startup_32 - 1b, BP_init_size(%esi)
-
-	/* Disable paging */
-	movl	%cr0, %eax
-	btrl	$X86_CR0_PG_BIT, %eax
-	movl	%eax, %cr0
-
-	jmp	startup_32
-SYM_FUNC_END(efi32_entry)
-
-/*
- * efi_status_t efi32_pe_entry(efi_handle_t image_handle,
- *			       efi_system_table_32_t *sys_table)
- */
-SYM_FUNC_START(efi32_pe_entry)
-	pushl	%ebp
-	movl	%esp, %ebp
-	pushl	%ebx				// save callee-save registers
-	pushl	%edi
-
-	call	verify_cpu			// check for long mode support
-	testl	%eax, %eax
-	movl	$0x80000003, %eax		// EFI_UNSUPPORTED
-	jnz	2f
-
-	movl	8(%ebp), %ecx			// image_handle
-	movl	12(%ebp), %edx			// sys_table
-	jmp	efi32_entry			// pass %ecx, %edx
-						// no other registers remain live
-
-2:	popl	%edi				// restore callee-save registers
-	popl	%ebx
-	leave
-	RET
-SYM_FUNC_END(efi32_pe_entry)
-
-#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
-	.org	efi32_stub_entry + 0x200
-	.code64
-SYM_FUNC_START_NOALIGN(efi64_stub_entry)
-	jmp	efi_handover_entry
-SYM_FUNC_END(efi64_stub_entry)
-#endif
-
-	.data
-	.balign	8
-SYM_DATA_START_LOCAL(efi32_boot_gdt)
-	.word	0
-	.quad	0
-SYM_DATA_END(efi32_boot_gdt)
-
-SYM_DATA_START_LOCAL(efi32_boot_idt)
-	.word	0
-	.quad	0
-SYM_DATA_END(efi32_boot_idt)
-
-SYM_DATA_LOCAL(efi32_boot_cs, .word 0)
-SYM_DATA_LOCAL(efi32_boot_ds, .word 0)
-SYM_DATA_LOCAL(efi32_boot_sp, .long 0)
-SYM_DATA_LOCAL(efi32_boot_args, .long 0, 0, 0)
-SYM_DATA(efi_is64, .byte 1)
diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
index 1dcb794c5479..d9dab940ff62 100644
--- a/arch/x86/boot/compressed/head_64.S
+++ b/arch/x86/boot/compressed/head_64.S
@@ -35,7 +35,6 @@
 #include <asm/bootparam.h>
 #include <asm/desc_defs.h>
 #include <asm/trapnr.h>
-#include "pgtable.h"
 
 /*
  * Fix alignment at 16 bytes. Following CONFIG_FUNCTION_ALIGNMENT will result
@@ -263,13 +262,6 @@ SYM_FUNC_START(startup_32)
 	 * used to perform that far jump.
 	 */
 	leal	rva(startup_64)(%ebp), %eax
-#ifdef CONFIG_EFI_MIXED
-	cmpb	$1, rva(efi_is64)(%ebp)
-	je	1f
-	leal	rva(startup_64_mixed_mode)(%ebp), %eax
-1:
-#endif
-
 	pushl	$__KERNEL_CS
 	pushl	%eax
 
@@ -483,110 +475,7 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated)
 	jmp	*%rax
 SYM_FUNC_END(.Lrelocated)
 
-/*
- * This is the 32-bit trampoline that will be copied over to low memory. It
- * will be called using the ordinary 64-bit calling convention from code
- * running in 64-bit mode.
- *
- * Return address is at the top of the stack (might be above 4G).
- * The first argument (EDI) contains the address of the temporary PGD level
- * page table in 32-bit addressable memory which will be programmed into
- * register CR3.
- */
-	.section ".rodata", "a", @progbits
-SYM_CODE_START(trampoline_32bit_src)
-	/*
-	 * Preserve callee save 64-bit registers on the stack: this is
-	 * necessary because the architecture does not guarantee that GPRs will
-	 * retain their full 64-bit values across a 32-bit mode switch.
-	 */
-	pushq	%r15
-	pushq	%r14
-	pushq	%r13
-	pushq	%r12
-	pushq	%rbp
-	pushq	%rbx
-
-	/* Preserve top half of RSP in a legacy mode GPR to avoid truncation */
-	movq	%rsp, %rbx
-	shrq	$32, %rbx
-
-	/* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */
-	pushq	$__KERNEL32_CS
-	leaq	0f(%rip), %rax
-	pushq	%rax
-	lretq
-
-	/*
-	 * The 32-bit code below will do a far jump back to long mode and end
-	 * up here after reconfiguring the number of paging levels. First, the
-	 * stack pointer needs to be restored to its full 64-bit value before
-	 * the callee save register contents can be popped from the stack.
-	 */
-.Lret:
-	shlq	$32, %rbx
-	orq	%rbx, %rsp
-
-	/* Restore the preserved 64-bit registers */
-	popq	%rbx
-	popq	%rbp
-	popq	%r12
-	popq	%r13
-	popq	%r14
-	popq	%r15
-	retq
-
 	.code32
-0:
-	/* Disable paging */
-	movl	%cr0, %eax
-	btrl	$X86_CR0_PG_BIT, %eax
-	movl	%eax, %cr0
-
-	/* Point CR3 to the trampoline's new top level page table */
-	movl	%edi, %cr3
-
-	/* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */
-	movl	$MSR_EFER, %ecx
-	rdmsr
-	btsl	$_EFER_LME, %eax
-	/* Avoid writing EFER if no change was made (for TDX guest) */
-	jc	1f
-	wrmsr
-1:
-	/* Toggle CR4.LA57 */
-	movl	%cr4, %eax
-	btcl	$X86_CR4_LA57_BIT, %eax
-	movl	%eax, %cr4
-
-	/* Enable paging again. */
-	movl	%cr0, %eax
-	btsl	$X86_CR0_PG_BIT, %eax
-	movl	%eax, %cr0
-
-	/*
-	 * Return to the 64-bit calling code using LJMP rather than LRET, to
-	 * avoid the need for a 32-bit addressable stack. The destination
-	 * address will be adjusted after the template code is copied into a
-	 * 32-bit addressable buffer.
-	 */
-.Ljmp:	ljmpl	$__KERNEL_CS, $(.Lret - trampoline_32bit_src)
-SYM_CODE_END(trampoline_32bit_src)
-
-/*
- * This symbol is placed right after trampoline_32bit_src() so its address can
- * be used to infer the size of the trampoline code.
- */
-SYM_DATA(trampoline_ljmp_imm_offset, .word  .Ljmp + 1 - trampoline_32bit_src)
-
-	/*
-         * The trampoline code has a size limit.
-         * Make sure we fail to compile if the trampoline code grows
-         * beyond TRAMPOLINE_32BIT_CODE_SIZE bytes.
-	 */
-	.org	trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_SIZE
-
-	.text
 SYM_FUNC_START_LOCAL_NOALIGN(.Lno_longmode)
 	/* This isn't an x86-64 CPU, so hang intentionally, we cannot continue */
 1:
diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c
index f4d82379bf44..3b0948ad449f 100644
--- a/arch/x86/boot/compressed/kaslr.c
+++ b/arch/x86/boot/compressed/kaslr.c
@@ -25,10 +25,6 @@
 #include "efi.h"
 
 #include <generated/compile.h>
-#include <linux/module.h>
-#include <linux/uts.h>
-#include <linux/utsname.h>
-#include <linux/ctype.h>
 #include <generated/utsversion.h>
 #include <generated/utsrelease.h>
 
@@ -764,6 +760,49 @@ static void process_e820_entries(unsigned long minimum,
 	}
 }
 
+/*
+ * If KHO is active, only process its scratch areas to ensure we are not
+ * stepping onto preserved memory.
+ */
+static bool process_kho_entries(unsigned long minimum, unsigned long image_size)
+{
+	struct kho_scratch *kho_scratch;
+	struct setup_data *ptr;
+	struct kho_data *kho;
+	int i, nr_areas = 0;
+
+	if (!IS_ENABLED(CONFIG_KEXEC_HANDOVER))
+		return false;
+
+	ptr = (struct setup_data *)(unsigned long)boot_params_ptr->hdr.setup_data;
+	while (ptr) {
+		if (ptr->type == SETUP_KEXEC_KHO) {
+			kho = (struct kho_data *)(unsigned long)ptr->data;
+			kho_scratch = (void *)(unsigned long)kho->scratch_addr;
+			nr_areas = kho->scratch_size / sizeof(*kho_scratch);
+			break;
+		}
+
+		ptr = (struct setup_data *)(unsigned long)ptr->next;
+	}
+
+	if (!nr_areas)
+		return false;
+
+	for (i = 0; i < nr_areas; i++) {
+		struct kho_scratch *area = &kho_scratch[i];
+		struct mem_vector region = {
+			.start = area->addr,
+			.size = area->size,
+		};
+
+		if (process_mem_region(&region, minimum, image_size))
+			break;
+	}
+
+	return true;
+}
+
 static unsigned long find_random_phys_addr(unsigned long minimum,
 					   unsigned long image_size)
 {
@@ -779,7 +818,12 @@ static unsigned long find_random_phys_addr(unsigned long minimum,
 		return 0;
 	}
 
-	if (!process_efi_entries(minimum, image_size))
+	/*
+	 * During kexec handover only process KHO scratch areas that are known
+	 * not to contain any data that must be preserved.
+	 */
+	if (!process_kho_entries(minimum, image_size) &&
+	    !process_efi_entries(minimum, image_size))
 		process_e820_entries(minimum, image_size);
 
 	phys_addr = slots_fetch_random();
diff --git a/arch/x86/boot/compressed/mem.c b/arch/x86/boot/compressed/mem.c
index dbba332e4a12..0e9f84ab4bdc 100644
--- a/arch/x86/boot/compressed/mem.c
+++ b/arch/x86/boot/compressed/mem.c
@@ -38,7 +38,7 @@ void arch_accept_memory(phys_addr_t start, phys_addr_t end)
 	if (early_is_tdx_guest()) {
 		if (!tdx_accept_memory(start, end))
 			panic("TDX: Failed to accept memory\n");
-	} else if (sev_snp_enabled()) {
+	} else if (early_is_sevsnp_guest()) {
 		snp_accept_memory(start, end);
 	} else {
 		error("Cannot accept memory: unknown platform\n");
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 04a35b2c26e9..0f41ca0e52c0 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -14,7 +14,6 @@
 
 #include "misc.h"
 #include "error.h"
-#include "pgtable.h"
 #include "../string.h"
 #include "../voffset.h"
 #include <asm/bootparam_utils.h>
@@ -235,7 +234,7 @@ static void handle_relocations(void *output, unsigned long output_len,
 
 	/*
 	 * Process relocations: 32 bit relocations first then 64 bit after.
-	 * Three sets of binary relocations are added to the end of the kernel
+	 * Two sets of binary relocations are added to the end of the kernel
 	 * before compression. Each relocation table entry is the kernel
 	 * address of the location which needs to be updated stored as a
 	 * 32-bit value which is sign extended to 64 bits.
@@ -245,8 +244,6 @@ static void handle_relocations(void *output, unsigned long output_len,
 	 * kernel bits...
 	 * 0 - zero terminator for 64 bit relocations
 	 * 64 bit relocation repeated
-	 * 0 - zero terminator for inverse 32 bit relocations
-	 * 32 bit inverse relocation repeated
 	 * 0 - zero terminator for 32 bit relocations
 	 * 32 bit relocation repeated
 	 *
@@ -263,16 +260,6 @@ static void handle_relocations(void *output, unsigned long output_len,
 		*(uint32_t *)ptr += delta;
 	}
 #ifdef CONFIG_X86_64
-	while (*--reloc) {
-		long extended = *reloc;
-		extended += map;
-
-		ptr = (unsigned long)extended;
-		if (ptr < min_addr || ptr > max_addr)
-			error("inverse 32-bit relocation outside of kernel!\n");
-
-		*(int32_t *)ptr -= delta;
-	}
 	for (reloc--; *reloc; reloc--) {
 		long extended = *reloc;
 		extended += map;
@@ -345,6 +332,8 @@ static size_t parse_elf(void *output)
 }
 
 const unsigned long kernel_text_size = VO___start_rodata - VO__text;
+const unsigned long kernel_inittext_offset = VO__sinittext - VO__text;
+const unsigned long kernel_inittext_size = VO___inittext_end - VO__sinittext;
 const unsigned long kernel_total_size = VO__end - VO__text;
 
 static u8 boot_heap[BOOT_HEAP_SIZE] __aligned(4);
@@ -385,6 +374,19 @@ static void parse_mem_encrypt(struct setup_header *hdr)
 		hdr->xloadflags |= XLF_MEM_ENCRYPTION;
 }
 
+static void early_sev_detect(void)
+{
+	/*
+	 * Accessing video memory causes guest termination because
+	 * the boot stage2 #VC handler of SEV-ES/SNP guests does not
+	 * support MMIO handling and kexec -c adds screen_info to the
+	 * boot parameters passed to the kexec kernel, which causes
+	 * console output to be dumped to both video and serial.
+	 */
+	if (sev_status & MSR_AMD64_SEV_ES_ENABLED)
+		lines = cols = 0;
+}
+
 /*
  * The compressed kernel image (ZO), has been moved so that its position
  * is against the end of the buffer used to hold the uncompressed kernel
@@ -440,6 +442,8 @@ asmlinkage __visible void *extract_kernel(void *rmode, unsigned char *output)
 	 */
 	early_tdx_detect();
 
+	early_sev_detect();
+
 	console_init();
 
 	/*
diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h
index dd8d1a85f671..fd855e32c9b9 100644
--- a/arch/x86/boot/compressed/misc.h
+++ b/arch/x86/boot/compressed/misc.h
@@ -136,6 +136,9 @@ static inline void console_init(void)
 #endif
 
 #ifdef CONFIG_AMD_MEM_ENCRYPT
+struct es_em_ctxt;
+struct insn;
+
 void sev_enable(struct boot_params *bp);
 void snp_check_features(void);
 void sev_es_shutdown_ghcb(void);
@@ -143,18 +146,12 @@ extern bool sev_es_check_ghcb_fault(unsigned long address);
 void snp_set_page_private(unsigned long paddr);
 void snp_set_page_shared(unsigned long paddr);
 void sev_prep_identity_maps(unsigned long top_level_pgt);
+
+enum es_result vc_decode_insn(struct es_em_ctxt *ctxt);
+bool insn_has_rep_prefix(struct insn *insn);
+void sev_insn_decode_init(void);
+bool early_setup_ghcb(void);
 #else
-static inline void sev_enable(struct boot_params *bp)
-{
-	/*
-	 * bp->cc_blob_address should only be set by boot/compressed kernel.
-	 * Initialize it to 0 unconditionally (thus here in this stub too) to
-	 * ensure that uninitialized values from buggy bootloaders aren't
-	 * propagated.
-	 */
-	if (bp)
-		bp->cc_blob_address = 0;
-}
 static inline void snp_check_features(void) { }
 static inline void sev_es_shutdown_ghcb(void) { }
 static inline bool sev_es_check_ghcb_fault(unsigned long address)
diff --git a/arch/x86/boot/compressed/pgtable.h b/arch/x86/boot/compressed/pgtable.h
deleted file mode 100644
index 6d595abe06b3..000000000000
--- a/arch/x86/boot/compressed/pgtable.h
+++ /dev/null
@@ -1,18 +0,0 @@
-#ifndef BOOT_COMPRESSED_PAGETABLE_H
-#define BOOT_COMPRESSED_PAGETABLE_H
-
-#define TRAMPOLINE_32BIT_SIZE		(2 * PAGE_SIZE)
-
-#define TRAMPOLINE_32BIT_CODE_OFFSET	PAGE_SIZE
-#define TRAMPOLINE_32BIT_CODE_SIZE	0xA0
-
-#ifndef __ASSEMBLER__
-
-extern unsigned long *trampoline_32bit;
-
-extern void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
-
-extern const u16 trampoline_ljmp_imm_offset;
-
-#endif /* __ASSEMBLER__ */
-#endif /* BOOT_COMPRESSED_PAGETABLE_H */
diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
index c882e1f67af0..0e89e197e112 100644
--- a/arch/x86/boot/compressed/pgtable_64.c
+++ b/arch/x86/boot/compressed/pgtable_64.c
@@ -1,21 +1,20 @@
 // SPDX-License-Identifier: GPL-2.0
 #include "misc.h"
 #include <asm/bootparam.h>
+#include <asm/bootparam_utils.h>
 #include <asm/e820/types.h>
+#include <asm/pgtable.h>
 #include <asm/processor.h>
-#include "pgtable.h"
 #include "../string.h"
 #include "efi.h"
 
 #define BIOS_START_MIN		0x20000U	/* 128K, less than this is insane */
 #define BIOS_START_MAX		0x9f000U	/* 640K, absolute maximum */
 
-#ifdef CONFIG_X86_5LEVEL
 /* __pgtable_l5_enabled needs to be in .data to avoid being cleared along with .bss */
 unsigned int __section(".data") __pgtable_l5_enabled;
 unsigned int __section(".data") pgdir_shift = 39;
 unsigned int __section(".data") ptrs_per_p4d = 1;
-#endif
 
 /* Buffer to preserve trampoline memory */
 static char trampoline_save[TRAMPOLINE_32BIT_SIZE];
@@ -107,24 +106,20 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable)
 	bool l5_required = false;
 
 	/* Initialize boot_params. Required for cmdline_find_option_bool(). */
+	sanitize_boot_params(bp);
 	boot_params_ptr = bp;
 
 	/*
 	 * Check if LA57 is desired and supported.
 	 *
 	 * There are several parts to the check:
-	 *   - if the kernel supports 5-level paging: CONFIG_X86_5LEVEL=y
 	 *   - if user asked to disable 5-level paging: no5lvl in cmdline
 	 *   - if the machine supports 5-level paging:
 	 *     + CPUID leaf 7 is supported
 	 *     + the leaf has the feature bit set
-	 *
-	 * That's substitute for boot_cpu_has() in early boot code.
 	 */
-	if (IS_ENABLED(CONFIG_X86_5LEVEL) &&
-			!cmdline_find_option_bool("no5lvl") &&
-			native_cpuid_eax(0) >= 7 &&
-			(native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) {
+	if (!cmdline_find_option_bool("no5lvl") &&
+	    native_cpuid_eax(0) >= 7 && (native_cpuid_ecx(7) & BIT(16))) {
 		l5_required = true;
 
 		/* Initialize variables for 5-level paging */
@@ -174,9 +169,10 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable)
 		 * For 4- to 5-level paging transition, set up current CR3 as
 		 * the first and the only entry in a new top-level page table.
 		 */
-		*trampoline_32bit = __native_read_cr3() | _PAGE_TABLE_NOENC;
+		*trampoline_32bit = native_read_cr3_pa() | _PAGE_TABLE_NOENC;
 	} else {
-		unsigned long src;
+		u64 *new_cr3;
+		pgd_t *pgdp;
 
 		/*
 		 * For 5- to 4-level paging transition, copy page table pointed
@@ -186,8 +182,9 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable)
 		 * We cannot just point to the page table from trampoline as it
 		 * may be above 4G.
 		 */
-		src = *(unsigned long *)__native_read_cr3() & PAGE_MASK;
-		memcpy(trampoline_32bit, (void *)src, PAGE_SIZE);
+		pgdp = (pgd_t *)native_read_cr3_pa();
+		new_cr3 = (u64 *)(native_pgd_val(pgdp[0]) & PTE_PFN_MASK);
+		memcpy(trampoline_32bit, new_cr3, PAGE_SIZE);
 	}
 
 	toggle_la57(trampoline_32bit);
diff --git a/arch/x86/boot/compressed/sbat.S b/arch/x86/boot/compressed/sbat.S
new file mode 100644
index 000000000000..838f70a997dd
--- /dev/null
+++ b/arch/x86/boot/compressed/sbat.S
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Embed SBAT data in the kernel.
+ */
+	.pushsection ".sbat", "a", @progbits
+	.incbin CONFIG_EFI_SBAT_FILE
+	.popsection
diff --git a/arch/x86/boot/compressed/sev-handle-vc.c b/arch/x86/boot/compressed/sev-handle-vc.c
new file mode 100644
index 000000000000..030001b46554
--- /dev/null
+++ b/arch/x86/boot/compressed/sev-handle-vc.c
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "misc.h"
+#include "error.h"
+#include "sev.h"
+
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <asm/insn.h>
+#include <asm/pgtable_types.h>
+#include <asm/ptrace.h>
+#include <asm/sev.h>
+#include <asm/trapnr.h>
+#include <asm/trap_pf.h>
+#include <asm/fpu/xcr.h>
+
+#define __BOOT_COMPRESSED
+#undef __init
+#define __init
+
+/* Basic instruction decoding support needed */
+#include "../../lib/inat.c"
+#include "../../lib/insn.c"
+
+/*
+ * Copy a version of this function here - insn-eval.c can't be used in
+ * pre-decompression code.
+ */
+bool insn_has_rep_prefix(struct insn *insn)
+{
+	insn_byte_t p;
+
+	insn_get_prefixes(insn);
+
+	for_each_insn_prefix(insn, p) {
+		if (p == 0xf2 || p == 0xf3)
+			return true;
+	}
+
+	return false;
+}
+
+enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
+{
+	char buffer[MAX_INSN_SIZE];
+	int ret;
+
+	memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
+
+	ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
+	if (ret < 0)
+		return ES_DECODE_FAILED;
+
+	return ES_OK;
+}
+
+extern void sev_insn_decode_init(void) __alias(inat_init_tables);
+
+/*
+ * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and
+ * doesn't use segments.
+ */
+static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx)
+{
+	return 0UL;
+}
+
+static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
+				   void *dst, char *buf, size_t size)
+{
+	memcpy(dst, buf, size);
+
+	return ES_OK;
+}
+
+static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
+				  void *src, char *buf, size_t size)
+{
+	memcpy(buf, src, size);
+
+	return ES_OK;
+}
+
+static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
+{
+	return ES_OK;
+}
+
+static bool fault_in_kernel_space(unsigned long address)
+{
+	return false;
+}
+
+#define sev_printk(fmt, ...)
+
+#include "../../coco/sev/vc-shared.c"
+
+void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code)
+{
+	struct es_em_ctxt ctxt;
+	enum es_result result;
+
+	if (!boot_ghcb && !early_setup_ghcb())
+		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
+
+	vc_ghcb_invalidate(boot_ghcb);
+	result = vc_init_em_ctxt(&ctxt, regs, exit_code);
+	if (result != ES_OK)
+		goto finish;
+
+	result = vc_check_opcode_bytes(&ctxt, exit_code);
+	if (result != ES_OK)
+		goto finish;
+
+	switch (exit_code) {
+	case SVM_EXIT_RDTSC:
+	case SVM_EXIT_RDTSCP:
+		result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code);
+		break;
+	case SVM_EXIT_IOIO:
+		result = vc_handle_ioio(boot_ghcb, &ctxt);
+		break;
+	case SVM_EXIT_CPUID:
+		result = vc_handle_cpuid(boot_ghcb, &ctxt);
+		break;
+	default:
+		result = ES_UNSUPPORTED;
+		break;
+	}
+
+finish:
+	if (result == ES_OK)
+		vc_finish_insn(&ctxt);
+	else if (result != ES_RETRY)
+		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
+}
diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c
index cd44e120fe53..c8c1464b3a56 100644
--- a/arch/x86/boot/compressed/sev.c
+++ b/arch/x86/boot/compressed/sev.c
@@ -14,6 +14,7 @@
 
 #include <asm/bootparam.h>
 #include <asm/pgtable_types.h>
+#include <asm/shared/msr.h>
 #include <asm/sev.h>
 #include <asm/trapnr.h>
 #include <asm/trap_pf.h>
@@ -21,189 +22,61 @@
 #include <asm/fpu/xcr.h>
 #include <asm/ptrace.h>
 #include <asm/svm.h>
-#include <asm/cpuid.h>
+#include <asm/cpuid/api.h>
 
 #include "error.h"
-#include "../msr.h"
+#include "sev.h"
 
 static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
 struct ghcb *boot_ghcb;
 
-/*
- * Copy a version of this function here - insn-eval.c can't be used in
- * pre-decompression code.
- */
-static bool insn_has_rep_prefix(struct insn *insn)
-{
-	insn_byte_t p;
-	int i;
-
-	insn_get_prefixes(insn);
-
-	for_each_insn_prefix(insn, i, p) {
-		if (p == 0xf2 || p == 0xf3)
-			return true;
-	}
-
-	return false;
-}
-
-/*
- * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and
- * doesn't use segments.
- */
-static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx)
-{
-	return 0UL;
-}
-
-static inline u64 sev_es_rd_ghcb_msr(void)
-{
-	struct msr m;
-
-	boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m);
-
-	return m.q;
-}
-
-static inline void sev_es_wr_ghcb_msr(u64 val)
-{
-	struct msr m;
-
-	m.q = val;
-	boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m);
-}
-
-static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
-{
-	char buffer[MAX_INSN_SIZE];
-	int ret;
-
-	memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
-
-	ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
-	if (ret < 0)
-		return ES_DECODE_FAILED;
-
-	return ES_OK;
-}
-
-static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
-				   void *dst, char *buf, size_t size)
-{
-	memcpy(dst, buf, size);
-
-	return ES_OK;
-}
-
-static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
-				  void *src, char *buf, size_t size)
-{
-	memcpy(buf, src, size);
-
-	return ES_OK;
-}
-
-static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
-{
-	return ES_OK;
-}
-
-static bool fault_in_kernel_space(unsigned long address)
-{
-	return false;
-}
-
 #undef __init
 #define __init
 
-#undef __head
-#define __head
-
 #define __BOOT_COMPRESSED
 
-/* Basic instruction decoding support needed */
-#include "../../lib/inat.c"
-#include "../../lib/insn.c"
-
-/* Include code for early handlers */
-#include "../../coco/sev/shared.c"
-
-static struct svsm_ca *svsm_get_caa(void)
-{
-	return boot_svsm_caa;
-}
+u8 snp_vmpl;
+u16 ghcb_version;
 
-static u64 svsm_get_caa_pa(void)
-{
-	return boot_svsm_caa_pa;
-}
+u64 boot_svsm_caa_pa;
 
-static int svsm_perform_call_protocol(struct svsm_call *call)
-{
-	struct ghcb *ghcb;
-	int ret;
-
-	if (boot_ghcb)
-		ghcb = boot_ghcb;
-	else
-		ghcb = NULL;
-
-	do {
-		ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
-			   : svsm_perform_msr_protocol(call);
-	} while (ret == -EAGAIN);
-
-	return ret;
-}
+/* Include code for early handlers */
+#include "../../boot/startup/sev-shared.c"
 
-bool sev_snp_enabled(void)
+static bool sev_snp_enabled(void)
 {
 	return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
 }
 
-static void __page_state_change(unsigned long paddr, enum psc_op op)
+void snp_set_page_private(unsigned long paddr)
 {
-	u64 val;
+	struct psc_desc d = {
+		SNP_PAGE_STATE_PRIVATE,
+		(struct svsm_ca *)boot_svsm_caa_pa,
+		boot_svsm_caa_pa
+	};
 
 	if (!sev_snp_enabled())
 		return;
 
-	/*
-	 * If private -> shared then invalidate the page before requesting the
-	 * state change in the RMP table.
-	 */
-	if (op == SNP_PAGE_STATE_SHARED)
-		pvalidate_4k_page(paddr, paddr, false);
-
-	/* Issue VMGEXIT to change the page state in RMP table. */
-	sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
-	VMGEXIT();
-
-	/* Read the response of the VMGEXIT. */
-	val = sev_es_rd_ghcb_msr();
-	if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val))
-		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
-
-	/*
-	 * Now that page state is changed in the RMP table, validate it so that it is
-	 * consistent with the RMP entry.
-	 */
-	if (op == SNP_PAGE_STATE_PRIVATE)
-		pvalidate_4k_page(paddr, paddr, true);
-}
-
-void snp_set_page_private(unsigned long paddr)
-{
-	__page_state_change(paddr, SNP_PAGE_STATE_PRIVATE);
+	__page_state_change(paddr, paddr, &d);
 }
 
 void snp_set_page_shared(unsigned long paddr)
 {
-	__page_state_change(paddr, SNP_PAGE_STATE_SHARED);
+	struct psc_desc d = {
+		SNP_PAGE_STATE_SHARED,
+		(struct svsm_ca *)boot_svsm_caa_pa,
+		boot_svsm_caa_pa
+	};
+
+	if (!sev_snp_enabled())
+		return;
+
+	__page_state_change(paddr, paddr, &d);
 }
 
-static bool early_setup_ghcb(void)
+bool early_setup_ghcb(void)
 {
 	if (set_page_decrypted((unsigned long)&boot_ghcb_page))
 		return false;
@@ -214,7 +87,7 @@ static bool early_setup_ghcb(void)
 	boot_ghcb = &boot_ghcb_page;
 
 	/* Initialize lookup tables for the instruction decoder */
-	inat_init_tables();
+	sev_insn_decode_init();
 
 	/* SNP guest requires the GHCB GPA must be registered */
 	if (sev_snp_enabled())
@@ -223,56 +96,16 @@ static bool early_setup_ghcb(void)
 	return true;
 }
 
-static phys_addr_t __snp_accept_memory(struct snp_psc_desc *desc,
-				       phys_addr_t pa, phys_addr_t pa_end)
-{
-	struct psc_hdr *hdr;
-	struct psc_entry *e;
-	unsigned int i;
-
-	hdr = &desc->hdr;
-	memset(hdr, 0, sizeof(*hdr));
-
-	e = desc->entries;
-
-	i = 0;
-	while (pa < pa_end && i < VMGEXIT_PSC_MAX_ENTRY) {
-		hdr->end_entry = i;
-
-		e->gfn = pa >> PAGE_SHIFT;
-		e->operation = SNP_PAGE_STATE_PRIVATE;
-		if (IS_ALIGNED(pa, PMD_SIZE) && (pa_end - pa) >= PMD_SIZE) {
-			e->pagesize = RMP_PG_SIZE_2M;
-			pa += PMD_SIZE;
-		} else {
-			e->pagesize = RMP_PG_SIZE_4K;
-			pa += PAGE_SIZE;
-		}
-
-		e++;
-		i++;
-	}
-
-	if (vmgexit_psc(boot_ghcb, desc))
-		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
-
-	pvalidate_pages(desc);
-
-	return pa;
-}
-
 void snp_accept_memory(phys_addr_t start, phys_addr_t end)
 {
-	struct snp_psc_desc desc = {};
-	unsigned int i;
-	phys_addr_t pa;
-
-	if (!boot_ghcb && !early_setup_ghcb())
-		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
+	struct psc_desc d = {
+		SNP_PAGE_STATE_PRIVATE,
+		(struct svsm_ca *)boot_svsm_caa_pa,
+		boot_svsm_caa_pa
+	};
 
-	pa = start;
-	while (pa < end)
-		pa = __snp_accept_memory(&desc, pa, end);
+	for (phys_addr_t pa = start; pa < end; pa += PAGE_SIZE)
+		__page_state_change(pa, pa, &d);
 }
 
 void sev_es_shutdown_ghcb(void)
@@ -333,46 +166,6 @@ bool sev_es_check_ghcb_fault(unsigned long address)
 	return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page);
 }
 
-void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code)
-{
-	struct es_em_ctxt ctxt;
-	enum es_result result;
-
-	if (!boot_ghcb && !early_setup_ghcb())
-		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
-
-	vc_ghcb_invalidate(boot_ghcb);
-	result = vc_init_em_ctxt(&ctxt, regs, exit_code);
-	if (result != ES_OK)
-		goto finish;
-
-	result = vc_check_opcode_bytes(&ctxt, exit_code);
-	if (result != ES_OK)
-		goto finish;
-
-	switch (exit_code) {
-	case SVM_EXIT_RDTSC:
-	case SVM_EXIT_RDTSCP:
-		result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code);
-		break;
-	case SVM_EXIT_IOIO:
-		result = vc_handle_ioio(boot_ghcb, &ctxt);
-		break;
-	case SVM_EXIT_CPUID:
-		result = vc_handle_cpuid(boot_ghcb, &ctxt);
-		break;
-	default:
-		result = ES_UNSUPPORTED;
-		break;
-	}
-
-finish:
-	if (result == ES_OK)
-		vc_finish_insn(&ctxt);
-	else if (result != ES_RETRY)
-		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
-}
-
 /*
  * SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need
  * guest side implementation for proper functioning of the guest. If any
@@ -394,14 +187,23 @@ finish:
 				 MSR_AMD64_SNP_VMSA_REG_PROT |		\
 				 MSR_AMD64_SNP_RESERVED_BIT13 |		\
 				 MSR_AMD64_SNP_RESERVED_BIT15 |		\
+				 MSR_AMD64_SNP_SECURE_AVIC |		\
 				 MSR_AMD64_SNP_RESERVED_MASK)
 
+#ifdef CONFIG_AMD_SECURE_AVIC
+#define SNP_FEATURE_SECURE_AVIC		MSR_AMD64_SNP_SECURE_AVIC
+#else
+#define SNP_FEATURE_SECURE_AVIC		0
+#endif
+
 /*
  * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented
  * by the guest kernel. As and when a new feature is implemented in the
  * guest kernel, a corresponding bit should be added to the mask.
  */
-#define SNP_FEATURES_PRESENT	MSR_AMD64_SNP_DEBUG_SWAP
+#define SNP_FEATURES_PRESENT	(MSR_AMD64_SNP_DEBUG_SWAP |	\
+				 MSR_AMD64_SNP_SECURE_TSC |	\
+				 SNP_FEATURE_SECURE_AVIC)
 
 u64 snp_get_unsupported_features(u64 status)
 {
@@ -505,7 +307,7 @@ static bool early_snp_init(struct boot_params *bp)
 	 * running at VMPL0. The CA will be used to communicate with the
 	 * SVSM and request its services.
 	 */
-	svsm_setup_ca(cc_info);
+	svsm_setup_ca(cc_info, rip_rel_ptr(&boot_ghcb_page));
 
 	/*
 	 * Pass run-time kernel a pointer to CC info via boot_params so EFI
@@ -549,6 +351,8 @@ static int sev_check_cpu_support(void)
 	if (!(eax & BIT(1)))
 		return -ENODEV;
 
+	sev_snp_needs_sfw = !(ebx & BIT(31));
+
 	return ebx & 0x3f;
 }
 
@@ -594,7 +398,7 @@ void sev_enable(struct boot_params *bp)
 	}
 
 	/* Set the SME mask if this is an SEV guest. */
-	boot_rdmsr(MSR_AMD64_SEV, &m);
+	raw_rdmsr(MSR_AMD64_SEV, &m);
 	sev_status = m.q;
 	if (!(sev_status & MSR_AMD64_SEV_ENABLED))
 		return;
@@ -611,30 +415,16 @@ void sev_enable(struct boot_params *bp)
 	 */
 	if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
 		u64 hv_features;
-		int ret;
 
 		hv_features = get_hv_features();
 		if (!(hv_features & GHCB_HV_FT_SNP))
 			sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
 
 		/*
-		 * Enforce running at VMPL0 or with an SVSM.
-		 *
-		 * Use RMPADJUST (see the rmpadjust() function for a description of
-		 * what the instruction does) to update the VMPL1 permissions of a
-		 * page. If the guest is running at VMPL0, this will succeed. If the
-		 * guest is running at any other VMPL, this will fail. Linux SNP guests
-		 * only ever run at a single VMPL level so permission mask changes of a
-		 * lesser-privileged VMPL are a don't-care.
-		 */
-		ret = rmpadjust((unsigned long)&boot_ghcb_page, RMP_PG_SIZE_4K, 1);
-
-		/*
-		 * Running at VMPL0 is not required if an SVSM is present and the hypervisor
-		 * supports the required SVSM GHCB events.
+		 * Running at VMPL0 is required unless an SVSM is present and
+		 * the hypervisor supports the required SVSM GHCB events.
 		 */
-		if (ret &&
-		    !(snp_vmpl && (hv_features & GHCB_HV_FT_SNP_MULTI_VMPL)))
+		if (snp_vmpl && !(hv_features & GHCB_HV_FT_SNP_MULTI_VMPL))
 			sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0);
 	}
 
@@ -657,7 +447,7 @@ u64 sev_get_status(void)
 	if (sev_check_cpu_support() < 0)
 		return 0;
 
-	boot_rdmsr(MSR_AMD64_SEV, &m);
+	raw_rdmsr(MSR_AMD64_SEV, &m);
 	return m.q;
 }
 
@@ -681,3 +471,42 @@ void sev_prep_identity_maps(unsigned long top_level_pgt)
 
 	sev_verify_cbit(top_level_pgt);
 }
+
+bool early_is_sevsnp_guest(void)
+{
+	static bool sevsnp;
+
+	if (sevsnp)
+		return true;
+
+	if (!(sev_get_status() & MSR_AMD64_SEV_SNP_ENABLED))
+		return false;
+
+	sevsnp = true;
+
+	if (!snp_vmpl) {
+		unsigned int eax, ebx, ecx, edx;
+
+		/*
+		 * CPUID Fn8000_001F_EAX[28] - SVSM support
+		 */
+		eax = 0x8000001f;
+		ecx = 0;
+		native_cpuid(&eax, &ebx, &ecx, &edx);
+		if (eax & BIT(28)) {
+			struct msr m;
+
+			/* Obtain the address of the calling area to use */
+			raw_rdmsr(MSR_SVSM_CAA, &m);
+			boot_svsm_caa_pa = m.q;
+
+			/*
+			 * The real VMPL level cannot be discovered, but the
+			 * memory acceptance routines make no use of that so
+			 * any non-zero value suffices here.
+			 */
+			snp_vmpl = U8_MAX;
+		}
+	}
+	return true;
+}
diff --git a/arch/x86/boot/compressed/sev.h b/arch/x86/boot/compressed/sev.h
index fc725a981b09..22637b416b46 100644
--- a/arch/x86/boot/compressed/sev.h
+++ b/arch/x86/boot/compressed/sev.h
@@ -10,13 +10,34 @@
 
 #ifdef CONFIG_AMD_MEM_ENCRYPT
 
-bool sev_snp_enabled(void);
+#include <asm/shared/msr.h>
+
 void snp_accept_memory(phys_addr_t start, phys_addr_t end);
+u64 sev_get_status(void);
+bool early_is_sevsnp_guest(void);
+
+static inline u64 sev_es_rd_ghcb_msr(void)
+{
+	struct msr m;
+
+	raw_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m);
+
+	return m.q;
+}
+
+static inline void sev_es_wr_ghcb_msr(u64 val)
+{
+	struct msr m;
+
+	m.q = val;
+	raw_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m);
+}
 
 #else
 
-static inline bool sev_snp_enabled(void) { return false; }
 static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { }
+static inline u64 sev_get_status(void) { return 0; }
+static inline bool early_is_sevsnp_guest(void) { return false; }
 
 #endif
 
diff --git a/arch/x86/boot/compressed/string.c b/arch/x86/boot/compressed/string.c
index 81fc1eaa3229..9af19d9614cb 100644
--- a/arch/x86/boot/compressed/string.c
+++ b/arch/x86/boot/compressed/string.c
@@ -15,9 +15,9 @@ static void *____memcpy(void *dest, const void *src, size_t n)
 {
 	int d0, d1, d2;
 	asm volatile(
-		"rep ; movsl\n\t"
+		"rep movsl\n\t"
 		"movl %4,%%ecx\n\t"
-		"rep ; movsb\n\t"
+		"rep movsb"
 		: "=&c" (d0), "=&D" (d1), "=&S" (d2)
 		: "0" (n >> 2), "g" (n & 3), "1" (dest), "2" (src)
 		: "memory");
@@ -29,9 +29,9 @@ static void *____memcpy(void *dest, const void *src, size_t n)
 {
 	long d0, d1, d2;
 	asm volatile(
-		"rep ; movsq\n\t"
+		"rep movsq\n\t"
 		"movq %4,%%rcx\n\t"
-		"rep ; movsb\n\t"
+		"rep movsb"
 		: "=&c" (d0), "=&D" (d1), "=&S" (d2)
 		: "0" (n >> 3), "g" (n & 7), "1" (dest), "2" (src)
 		: "memory");
diff --git a/arch/x86/boot/compressed/vmlinux.lds.S b/arch/x86/boot/compressed/vmlinux.lds.S
index 083ec6d7722a..587ce3e7c504 100644
--- a/arch/x86/boot/compressed/vmlinux.lds.S
+++ b/arch/x86/boot/compressed/vmlinux.lds.S
@@ -43,12 +43,20 @@ SECTIONS
 		*(.rodata.*)
 		_erodata = . ;
 	}
+#ifdef CONFIG_EFI_SBAT
+	.sbat : ALIGN(0x1000) {
+		_sbat = . ;
+		*(.sbat)
+		_esbat = ALIGN(0x1000);
+		. = _esbat;
+	}
+#endif
 	.data :	ALIGN(0x1000) {
 		_data = . ;
 		*(.data)
 		*(.data.*)
 
-		/* Add 4 bytes of extra space for a CRC-32 checksum */
+		/* Add 4 bytes of extra space for the obsolete CRC-32 checksum */
 		. = ALIGN(. + 4, 0x200);
 		_edata = . ;
 	}
diff --git a/arch/x86/boot/copy.S b/arch/x86/boot/copy.S
index 6afd05e819d2..3973a67cd04e 100644
--- a/arch/x86/boot/copy.S
+++ b/arch/x86/boot/copy.S
@@ -22,10 +22,10 @@ SYM_FUNC_START_NOALIGN(memcpy)
 	movw	%dx, %si
 	pushw	%cx
 	shrw	$2, %cx
-	rep; movsl
+	rep movsl
 	popw	%cx
 	andw	$3, %cx
-	rep; movsb
+	rep movsb
 	popw	%di
 	popw	%si
 	retl
@@ -38,10 +38,10 @@ SYM_FUNC_START_NOALIGN(memset)
 	imull	$0x01010101,%eax
 	pushw	%cx
 	shrw	$2, %cx
-	rep; stosl
+	rep stosl
 	popw	%cx
 	andw	$3, %cx
-	rep; stosb
+	rep stosb
 	popw	%di
 	retl
 SYM_FUNC_END(memset)
diff --git a/arch/x86/boot/cpucheck.c b/arch/x86/boot/cpucheck.c
index 0aae4d4ed615..2e1bb936cba2 100644
--- a/arch/x86/boot/cpucheck.c
+++ b/arch/x86/boot/cpucheck.c
@@ -22,12 +22,13 @@
 # include "boot.h"
 #endif
 #include <linux/types.h>
+#include <asm/cpufeaturemasks.h>
 #include <asm/intel-family.h>
 #include <asm/processor-flags.h>
-#include <asm/required-features.h>
 #include <asm/msr-index.h>
+#include <asm/shared/msr.h>
+
 #include "string.h"
-#include "msr.h"
 
 static u32 err_flags[NCAPINTS];
 
@@ -133,9 +134,9 @@ int check_cpu(int *cpu_level_ptr, int *req_level_ptr, u32 **err_flags_ptr)
 
 		struct msr m;
 
-		boot_rdmsr(MSR_K7_HWCR, &m);
+		raw_rdmsr(MSR_K7_HWCR, &m);
 		m.l &= ~(1 << 15);
-		boot_wrmsr(MSR_K7_HWCR, &m);
+		raw_wrmsr(MSR_K7_HWCR, &m);
 
 		get_cpuflags();	/* Make sure it really did something */
 		err = check_cpuflags();
@@ -147,9 +148,9 @@ int check_cpu(int *cpu_level_ptr, int *req_level_ptr, u32 **err_flags_ptr)
 
 		struct msr m;
 
-		boot_rdmsr(MSR_VIA_FCR, &m);
+		raw_rdmsr(MSR_VIA_FCR, &m);
 		m.l |= (1 << 1) | (1 << 7);
-		boot_wrmsr(MSR_VIA_FCR, &m);
+		raw_wrmsr(MSR_VIA_FCR, &m);
 
 		set_bit(X86_FEATURE_CX8, cpu.flags);
 		err = check_cpuflags();
@@ -159,14 +160,14 @@ int check_cpu(int *cpu_level_ptr, int *req_level_ptr, u32 **err_flags_ptr)
 		struct msr m, m_tmp;
 		u32 level = 1;
 
-		boot_rdmsr(0x80860004, &m);
+		raw_rdmsr(0x80860004, &m);
 		m_tmp = m;
 		m_tmp.l = ~0;
-		boot_wrmsr(0x80860004, &m_tmp);
+		raw_wrmsr(0x80860004, &m_tmp);
 		asm("cpuid"
 		    : "+a" (level), "=d" (cpu.flags[0])
 		    : : "ecx", "ebx");
-		boot_wrmsr(0x80860004, &m);
+		raw_wrmsr(0x80860004, &m);
 
 		err = check_cpuflags();
 	} else if (err == 0x01 &&
diff --git a/arch/x86/boot/cpuflags.c b/arch/x86/boot/cpuflags.c
index d75237ba7ce9..916bac09b464 100644
--- a/arch/x86/boot/cpuflags.c
+++ b/arch/x86/boot/cpuflags.c
@@ -3,7 +3,6 @@
 #include "bitops.h"
 
 #include <asm/processor-flags.h>
-#include <asm/required-features.h>
 #include <asm/msr-index.h>
 #include "cpuflags.h"
 
@@ -29,40 +28,32 @@ static int has_fpu(void)
 	return fsw == 0 && (fcw & 0x103f) == 0x003f;
 }
 
+#ifdef CONFIG_X86_32
 /*
  * For building the 16-bit code we want to explicitly specify 32-bit
  * push/pop operations, rather than just saying 'pushf' or 'popf' and
- * letting the compiler choose. But this is also included from the
- * compressed/ directory where it may be 64-bit code, and thus needs
- * to be 'pushfq' or 'popfq' in that case.
+ * letting the compiler choose.
  */
-#ifdef __x86_64__
-#define PUSHF "pushfq"
-#define POPF "popfq"
-#else
-#define PUSHF "pushfl"
-#define POPF "popfl"
-#endif
-
-int has_eflag(unsigned long mask)
+bool has_eflag(unsigned long mask)
 {
 	unsigned long f0, f1;
 
-	asm volatile(PUSHF "	\n\t"
-		     PUSHF "	\n\t"
+	asm volatile("pushfl	\n\t"
+		     "pushfl	\n\t"
 		     "pop %0	\n\t"
 		     "mov %0,%1	\n\t"
 		     "xor %2,%1	\n\t"
 		     "push %1	\n\t"
-		     POPF "	\n\t"
-		     PUSHF "	\n\t"
+		     "popfl	\n\t"
+		     "pushfl	\n\t"
 		     "pop %1	\n\t"
-		     POPF
+		     "popfl"
 		     : "=&r" (f0), "=&r" (f1)
 		     : "ri" (mask));
 
 	return !!((f0^f1) & mask);
 }
+#endif
 
 void cpuid_count(u32 id, u32 count, u32 *a, u32 *b, u32 *c, u32 *d)
 {
diff --git a/arch/x86/boot/cpuflags.h b/arch/x86/boot/cpuflags.h
index 475b8fde90f7..a398d9204ad0 100644
--- a/arch/x86/boot/cpuflags.h
+++ b/arch/x86/boot/cpuflags.h
@@ -15,8 +15,13 @@ struct cpu_features {
 extern struct cpu_features cpu;
 extern u32 cpu_vendor[3];
 
-int has_eflag(unsigned long mask);
+#ifdef CONFIG_X86_32
+bool has_eflag(unsigned long mask);
+#else
+static inline bool has_eflag(unsigned long mask) { return true; }
+#endif
 void get_cpuflags(void);
 void cpuid_count(u32 id, u32 count, u32 *a, u32 *b, u32 *c, u32 *d);
+bool has_cpuflag(int flag);
 
 #endif
diff --git a/arch/x86/boot/genimage.sh b/arch/x86/boot/genimage.sh
index c9299aeb7333..3882ead513f7 100644
--- a/arch/x86/boot/genimage.sh
+++ b/arch/x86/boot/genimage.sh
@@ -22,6 +22,7 @@
 # This script requires:
 #   bash
 #   syslinux
+#   genisoimage
 #   mtools (for fdimage* and hdimage)
 #   edk2/OVMF (for hdimage)
 #
@@ -251,7 +252,9 @@ geniso() {
 	cp "$isolinux" "$ldlinux" "$tmp_dir"
 	cp "$FBZIMAGE" "$tmp_dir"/linux
 	echo default linux "$KCMDLINE" > "$tmp_dir"/isolinux.cfg
-	cp "${FDINITRDS[@]}" "$tmp_dir"/
+	if [ ${#FDINITRDS[@]} -gt 0 ]; then
+		cp "${FDINITRDS[@]}" "$tmp_dir"/
+	fi
 	genisoimage -J -r -appid 'LINUX_BOOT' -input-charset=utf-8 \
 		    -quiet -o "$FIMAGE" -b isolinux.bin \
 		    -c boot.cat -no-emul-boot -boot-load-size 4 \
diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S
index b5c79f43359b..9bea5a1e2c52 100644
--- a/arch/x86/boot/header.S
+++ b/arch/x86/boot/header.S
@@ -43,7 +43,7 @@ SYSSEG		= 0x1000		/* historical load address >> 4 */
 	.section ".bstext", "ax"
 #ifdef CONFIG_EFI_STUB
 	# "MZ", MS-DOS header
-	.word	MZ_MAGIC
+	.word	IMAGE_DOS_SIGNATURE
 	.org	0x38
 	#
 	# Offset to the PE header.
@@ -51,16 +51,16 @@ SYSSEG		= 0x1000		/* historical load address >> 4 */
 	.long	LINUX_PE_MAGIC
 	.long	pe_header
 pe_header:
-	.long	PE_MAGIC
+	.long	IMAGE_NT_SIGNATURE
 
 coff_header:
 #ifdef CONFIG_X86_32
 	.set	image_file_add_flags, IMAGE_FILE_32BIT_MACHINE
-	.set	pe_opt_magic, PE_OPT_MAGIC_PE32
+	.set	pe_opt_magic, IMAGE_NT_OPTIONAL_HDR32_MAGIC
 	.word	IMAGE_FILE_MACHINE_I386
 #else
 	.set	image_file_add_flags, 0
-	.set	pe_opt_magic, PE_OPT_MAGIC_PE32PLUS
+	.set	pe_opt_magic, IMAGE_NT_OPTIONAL_HDR64_MAGIC
 	.word	IMAGE_FILE_MACHINE_AMD64
 #endif
 	.word	section_count			# nr_sections
@@ -111,7 +111,7 @@ extra_header_fields:
 	.long	salign				# SizeOfHeaders
 	.long	0				# CheckSum
 	.word	IMAGE_SUBSYSTEM_EFI_APPLICATION	# Subsystem (EFI application)
-	.word	IMAGE_DLL_CHARACTERISTICS_NX_COMPAT	# DllCharacteristics
+	.word	IMAGE_DLLCHARACTERISTICS_NX_COMPAT	# DllCharacteristics
 #ifdef CONFIG_X86_32
 	.long	0				# SizeOfStackReserve
 	.long	0				# SizeOfStackCommit
@@ -179,15 +179,11 @@ pecompat_fstart:
 #else
 	.set	pecompat_fstart, setup_size
 #endif
-	.ascii	".text"
-	.byte	0
-	.byte	0
-	.byte	0
-	.long	ZO__data
-	.long	setup_size
-	.long	ZO__data			# Size of initialized data
-						# on disk
-	.long	setup_size
+	.ascii	".text\0\0\0"
+	.long	textsize            		# VirtualSize
+	.long	setup_size			# VirtualAddress
+	.long	textsize			# SizeOfRawData
+	.long	setup_size			# PointerToRawData
 	.long	0				# PointerToRelocations
 	.long	0				# PointerToLineNumbers
 	.word	0				# NumberOfRelocations
@@ -196,6 +192,23 @@ pecompat_fstart:
 		IMAGE_SCN_MEM_READ		| \
 		IMAGE_SCN_MEM_EXECUTE		# Characteristics
 
+#ifdef CONFIG_EFI_SBAT
+	.ascii	".sbat\0\0\0"
+	.long	ZO__esbat - ZO__sbat            # VirtualSize
+	.long	setup_size + ZO__sbat           # VirtualAddress
+	.long	ZO__esbat - ZO__sbat            # SizeOfRawData
+	.long	setup_size + ZO__sbat           # PointerToRawData
+
+	.long	0, 0, 0
+	.long	IMAGE_SCN_CNT_INITIALIZED_DATA	| \
+		IMAGE_SCN_MEM_READ		| \
+		IMAGE_SCN_MEM_DISCARDABLE	# Characteristics
+
+	.set	textsize, ZO__sbat
+#else
+	.set	textsize, ZO__data
+#endif
+
 	.ascii	".data\0\0\0"
 	.long	ZO__end - ZO__data		# VirtualSize
 	.long	setup_size + ZO__data		# VirtualAddress
@@ -361,12 +374,8 @@ xloadflags:
 #endif
 
 #ifdef CONFIG_X86_64
-#ifdef CONFIG_X86_5LEVEL
 #define XLF56 (XLF_5LEVEL|XLF_5LEVEL_ENABLED)
 #else
-#define XLF56 XLF_5LEVEL
-#endif
-#else
 #define XLF56 0
 #endif
 
@@ -585,7 +594,7 @@ start_of_setup:
 	xorl	%eax, %eax
 	subw	%di, %cx
 	shrw	$2, %cx
-	rep; stosl
+	rep stosl
 
 # Jump to C code (should not return)
 	calll	main
diff --git a/arch/x86/boot/mkcpustr.c b/arch/x86/boot/mkcpustr.c
index da0ccc5de538..22d730b227e3 100644
--- a/arch/x86/boot/mkcpustr.c
+++ b/arch/x86/boot/mkcpustr.c
@@ -12,8 +12,6 @@
 
 #include <stdio.h>
 
-#include "../include/asm/required-features.h"
-#include "../include/asm/disabled-features.h"
 #include "../include/asm/cpufeatures.h"
 #include "../include/asm/vmxfeatures.h"
 #include "../kernel/cpu/capflags.c"
@@ -23,6 +21,7 @@ int main(void)
 	int i, j;
 	const char *str;
 
+	printf("#include <asm/cpufeaturemasks.h>\n\n");
 	printf("static const char x86_cap_strs[] =\n");
 
 	for (i = 0; i < NCAPINTS; i++) {
diff --git a/arch/x86/boot/msr.h b/arch/x86/boot/msr.h
deleted file mode 100644
index aed66f7ae199..000000000000
--- a/arch/x86/boot/msr.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Helpers/definitions related to MSR access.
- */
-
-#ifndef BOOT_MSR_H
-#define BOOT_MSR_H
-
-#include <asm/shared/msr.h>
-
-/*
- * The kernel proper already defines rdmsr()/wrmsr(), but they are not for the
- * boot kernel since they rely on tracepoint/exception handling infrastructure
- * that's not available here.
- */
-static inline void boot_rdmsr(unsigned int reg, struct msr *m)
-{
-	asm volatile("rdmsr" : "=a" (m->l), "=d" (m->h) : "c" (reg));
-}
-
-static inline void boot_wrmsr(unsigned int reg, const struct msr *m)
-{
-	asm volatile("wrmsr" : : "c" (reg), "a"(m->l), "d" (m->h) : "memory");
-}
-
-#endif /* BOOT_MSR_H */
diff --git a/arch/x86/boot/setup.ld b/arch/x86/boot/setup.ld
index 3a2d1360abb0..e1d594a60204 100644
--- a/arch/x86/boot/setup.ld
+++ b/arch/x86/boot/setup.ld
@@ -45,6 +45,8 @@ SECTIONS
 
 		setup_size = ALIGN(ABSOLUTE(.), 4096);
 		setup_sects = ABSOLUTE(setup_size / 512);
+		ASSERT(setup_sects >= 5, "The setup must be at least 5 sectors in size");
+		ASSERT(setup_sects <= 64, "The setup must be at most 64 sectors in size");
 	}
 
 	. = ALIGN(16);
diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
new file mode 100644
index 000000000000..5e499cfb29b5
--- /dev/null
+++ b/arch/x86/boot/startup/Makefile
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: GPL-2.0
+
+KBUILD_AFLAGS		+= -D__DISABLE_EXPORTS
+KBUILD_CFLAGS		+= -D__DISABLE_EXPORTS -mcmodel=small -fPIC \
+			   -Os -DDISABLE_BRANCH_PROFILING \
+			   $(DISABLE_STACKLEAK_PLUGIN) \
+			   $(DISABLE_LATENT_ENTROPY_PLUGIN) \
+			   -fno-stack-protector -D__NO_FORTIFY \
+			   -fno-jump-tables \
+			   -include $(srctree)/include/linux/hidden.h
+
+# disable ftrace hooks and LTO
+KBUILD_CFLAGS	:= $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS))
+KBUILD_CFLAGS	:= $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS))
+KASAN_SANITIZE	:= n
+KCSAN_SANITIZE	:= n
+KMSAN_SANITIZE	:= n
+UBSAN_SANITIZE	:= n
+KCOV_INSTRUMENT	:= n
+
+obj-$(CONFIG_X86_64)		+= gdt_idt.o map_kernel.o
+obj-$(CONFIG_AMD_MEM_ENCRYPT)	+= sme.o sev-startup.o
+pi-objs				:= $(patsubst %.o,$(obj)/%.o,$(obj-y))
+
+lib-$(CONFIG_X86_64)		+= la57toggle.o
+lib-$(CONFIG_EFI_MIXED)		+= efi-mixed.o
+
+#
+# Disable objtool validation for all library code, which is intended
+# to be linked into the decompressor or the EFI stub but not vmlinux
+#
+$(patsubst %.o,$(obj)/%.o,$(lib-y)): OBJECT_FILES_NON_STANDARD := y
+
+#
+# Invoke objtool for each object individually to check for absolute
+# relocations, even if other objtool actions are being deferred.
+#
+$(pi-objs): objtool-enabled	= 1
+$(pi-objs): objtool-args	= $(if $(delay-objtool),--dry-run,$(objtool-args-y)) --noabs
+
+#
+# Confine the startup code by prefixing all symbols with __pi_ (for position
+# independent). This ensures that startup code can only call other startup
+# code, or code that has explicitly been made accessible to it via a symbol
+# alias.
+#
+$(obj)/%.pi.o: OBJCOPYFLAGS := --prefix-symbols=__pi_
+$(obj)/%.pi.o: $(obj)/%.o FORCE
+	$(call if_changed,objcopy)
+
+targets	+= $(obj-y)
+obj-y	:= $(patsubst %.o,%.pi.o,$(obj-y))
diff --git a/arch/x86/boot/startup/efi-mixed.S b/arch/x86/boot/startup/efi-mixed.S
new file mode 100644
index 000000000000..e04ed99bc449
--- /dev/null
+++ b/arch/x86/boot/startup/efi-mixed.S
@@ -0,0 +1,253 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2014, 2015 Intel Corporation; author Matt Fleming
+ *
+ * Early support for invoking 32-bit EFI services from a 64-bit kernel.
+ *
+ * Because this thunking occurs before ExitBootServices() we have to
+ * restore the firmware's 32-bit GDT and IDT before we make EFI service
+ * calls.
+ *
+ * On the plus side, we don't have to worry about mangling 64-bit
+ * addresses into 32-bits because we're executing with an identity
+ * mapped pagetable and haven't transitioned to 64-bit virtual addresses
+ * yet.
+ */
+
+#include <linux/linkage.h>
+#include <asm/desc_defs.h>
+#include <asm/msr.h>
+#include <asm/page_types.h>
+#include <asm/pgtable_types.h>
+#include <asm/processor-flags.h>
+#include <asm/segment.h>
+
+	.text
+	.code32
+#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
+SYM_FUNC_START(efi32_stub_entry)
+	call	1f
+1:	popl	%ecx
+
+	/* Clear BSS */
+	xorl	%eax, %eax
+	leal	(_bss - 1b)(%ecx), %edi
+	leal	(_ebss - 1b)(%ecx), %ecx
+	subl	%edi, %ecx
+	shrl	$2, %ecx
+	cld
+	rep	stosl
+
+	add	$0x4, %esp		/* Discard return address */
+	movl	8(%esp), %ebx		/* struct boot_params pointer */
+	jmp	efi32_startup
+SYM_FUNC_END(efi32_stub_entry)
+#endif
+
+/*
+ * Called using a far call from __efi64_thunk() below, using the x86_64 SysV
+ * ABI (except for R8/R9 which are inaccessible to 32-bit code - EAX/EBX are
+ * used instead).  EBP+16 points to the arguments passed via the stack.
+ *
+ * The first argument (EDI) is a pointer to the boot service or protocol, to
+ * which the remaining arguments are passed, each truncated to 32 bits.
+ */
+SYM_FUNC_START_LOCAL(efi_enter32)
+	/*
+	 * Convert x86-64 SysV ABI params to i386 ABI
+	 */
+	pushl	32(%ebp)	/* Up to 3 args passed via the stack */
+	pushl	24(%ebp)
+	pushl	16(%ebp)
+	pushl	%ebx		/* R9 */
+	pushl	%eax		/* R8 */
+	pushl	%ecx
+	pushl	%edx
+	pushl	%esi
+
+	/* Disable paging */
+	movl	%cr0, %eax
+	btrl	$X86_CR0_PG_BIT, %eax
+	movl	%eax, %cr0
+
+	/* Disable long mode via EFER */
+	movl	$MSR_EFER, %ecx
+	rdmsr
+	btrl	$_EFER_LME, %eax
+	wrmsr
+
+	call	*%edi
+
+	/* We must preserve return value */
+	movl	%eax, %edi
+
+	call	efi32_enable_long_mode
+
+	addl	$32, %esp
+	movl	%edi, %eax
+	lret
+SYM_FUNC_END(efi_enter32)
+
+	.code64
+SYM_FUNC_START(__efi64_thunk)
+	push	%rbp
+	movl	%esp, %ebp
+	push	%rbx
+
+	/* Move args #5 and #6 into 32-bit accessible registers */
+	movl	%r8d, %eax
+	movl	%r9d, %ebx
+
+	lcalll	*efi32_call(%rip)
+
+	pop	%rbx
+	pop	%rbp
+	RET
+SYM_FUNC_END(__efi64_thunk)
+
+	.code32
+SYM_FUNC_START_LOCAL(efi32_enable_long_mode)
+	movl	%cr4, %eax
+	btsl	$(X86_CR4_PAE_BIT), %eax
+	movl	%eax, %cr4
+
+	movl	$MSR_EFER, %ecx
+	rdmsr
+	btsl	$_EFER_LME, %eax
+	wrmsr
+
+	/* Disable interrupts - the firmware's IDT does not work in long mode */
+	cli
+
+	/* Enable paging */
+	movl	%cr0, %eax
+	btsl	$X86_CR0_PG_BIT, %eax
+	movl	%eax, %cr0
+	ret
+SYM_FUNC_END(efi32_enable_long_mode)
+
+/*
+ * This is the common EFI stub entry point for mixed mode. It sets up the GDT
+ * and page tables needed for 64-bit execution, after which it calls the
+ * common 64-bit EFI entrypoint efi_stub_entry().
+ *
+ * Arguments:	0(%esp)	image handle
+ * 		4(%esp)	EFI system table pointer
+ *		%ebx	struct boot_params pointer (or NULL)
+ *
+ * Since this is the point of no return for ordinary execution, no registers
+ * are considered live except for the function parameters. [Note that the EFI
+ * stub may still exit and return to the firmware using the Exit() EFI boot
+ * service.]
+ */
+SYM_FUNC_START_LOCAL(efi32_startup)
+	movl	%esp, %ebp
+
+	subl	$8, %esp
+	sgdtl	(%esp)			/* Save GDT descriptor to the stack */
+	movl	2(%esp), %esi		/* Existing GDT pointer */
+	movzwl	(%esp), %ecx		/* Existing GDT limit */
+	inc	%ecx			/* Existing GDT size */
+	andl	$~7, %ecx		/* Ensure size is multiple of 8 */
+
+	subl	%ecx, %esp		/* Allocate new GDT */
+	andl	$~15, %esp		/* Realign the stack */
+	movl	%esp, %edi		/* New GDT address */
+	leal	7(%ecx), %eax		/* New GDT limit */
+	pushw	%cx			/* Push 64-bit CS (for LJMP below) */
+	pushl	%edi			/* Push new GDT address */
+	pushw	%ax			/* Push new GDT limit */
+
+	/* Copy GDT to the stack and add a 64-bit code segment at the end */
+	movl	$GDT_ENTRY(DESC_CODE64, 0, 0xfffff) & 0xffffffff, (%edi,%ecx)
+	movl	$GDT_ENTRY(DESC_CODE64, 0, 0xfffff) >> 32, 4(%edi,%ecx)
+	shrl	$2, %ecx
+	cld
+	rep	movsl			/* Copy the firmware GDT */
+	lgdtl	(%esp)			/* Switch to the new GDT */
+
+	call	1f
+1:	pop	%edi
+
+	/* Record mixed mode entry */
+	movb	$0x0, (efi_is64 - 1b)(%edi)
+
+	/* Set up indirect far call to re-enter 32-bit mode */
+	leal	(efi32_call - 1b)(%edi), %eax
+	addl	%eax, (%eax)
+	movw	%cs, 4(%eax)
+
+	/* Disable paging */
+	movl	%cr0, %eax
+	btrl	$X86_CR0_PG_BIT, %eax
+	movl	%eax, %cr0
+
+	/* Set up 1:1 mapping */
+	leal	(pte - 1b)(%edi), %eax
+	movl	$_PAGE_PRESENT | _PAGE_RW | _PAGE_PSE, %ecx
+	leal	(_PAGE_PRESENT | _PAGE_RW)(%eax), %edx
+2:	movl	%ecx, (%eax)
+	addl	$8, %eax
+	addl	$PMD_SIZE, %ecx
+	jnc	2b
+
+	movl	$PAGE_SIZE, %ecx
+	.irpc	l, 0123
+	movl	%edx, \l * 8(%eax)
+	addl	%ecx, %edx
+	.endr
+	addl	%ecx, %eax
+	movl	%edx, (%eax)
+	movl	%eax, %cr3
+
+	call	efi32_enable_long_mode
+
+	/* Set up far jump to 64-bit mode (CS is already on the stack) */
+	leal	(efi_stub_entry - 1b)(%edi), %eax
+	movl	%eax, 2(%esp)
+
+	movl	0(%ebp), %edi
+	movl	4(%ebp), %esi
+	movl	%ebx, %edx
+	ljmpl	*2(%esp)
+SYM_FUNC_END(efi32_startup)
+
+/*
+ * efi_status_t efi32_pe_entry(efi_handle_t image_handle,
+ *			       efi_system_table_32_t *sys_table)
+ */
+SYM_FUNC_START(efi32_pe_entry)
+	pushl	%ebx				// save callee-save registers
+
+	/* Check whether the CPU supports long mode */
+	movl	$0x80000001, %eax		// assume extended info support
+	cpuid
+	btl	$29, %edx			// check long mode bit
+	jnc	1f
+	leal	8(%esp), %esp			// preserve stack alignment
+	xor	%ebx, %ebx			// no struct boot_params pointer
+	jmp	efi32_startup			// only ESP and EBX remain live
+1:	movl	$0x80000003, %eax		// EFI_UNSUPPORTED
+	popl	%ebx
+	RET
+SYM_FUNC_END(efi32_pe_entry)
+
+#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
+	.org	efi32_stub_entry + 0x200
+	.code64
+SYM_FUNC_START_NOALIGN(efi64_stub_entry)
+	jmp	efi_handover_entry
+SYM_FUNC_END(efi64_stub_entry)
+#endif
+
+	.data
+	.balign	8
+SYM_DATA_START_LOCAL(efi32_call)
+	.long	efi_enter32 - .
+	.word	0x0
+SYM_DATA_END(efi32_call)
+SYM_DATA(efi_is64, .byte 1)
+
+	.bss
+	.balign PAGE_SIZE
+SYM_DATA_LOCAL(pte, .fill 6 * PAGE_SIZE, 1, 0)
diff --git a/arch/x86/boot/startup/exports.h b/arch/x86/boot/startup/exports.h
new file mode 100644
index 000000000000..01d2363dc445
--- /dev/null
+++ b/arch/x86/boot/startup/exports.h
@@ -0,0 +1,14 @@
+
+/*
+ * The symbols below are functions that are implemented by the startup code,
+ * but called at runtime by the SEV code residing in the core kernel.
+ */
+PROVIDE(early_set_pages_state		= __pi_early_set_pages_state);
+PROVIDE(early_snp_set_memory_private	= __pi_early_snp_set_memory_private);
+PROVIDE(early_snp_set_memory_shared	= __pi_early_snp_set_memory_shared);
+PROVIDE(get_hv_features			= __pi_get_hv_features);
+PROVIDE(sev_es_terminate		= __pi_sev_es_terminate);
+PROVIDE(snp_cpuid			= __pi_snp_cpuid);
+PROVIDE(snp_cpuid_get_table		= __pi_snp_cpuid_get_table);
+PROVIDE(svsm_issue_call			= __pi_svsm_issue_call);
+PROVIDE(svsm_process_result_codes	= __pi_svsm_process_result_codes);
diff --git a/arch/x86/boot/startup/gdt_idt.c b/arch/x86/boot/startup/gdt_idt.c
new file mode 100644
index 000000000000..d16102abdaec
--- /dev/null
+++ b/arch/x86/boot/startup/gdt_idt.c
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/linkage.h>
+#include <linux/types.h>
+
+#include <asm/desc.h>
+#include <asm/init.h>
+#include <asm/setup.h>
+#include <asm/sev.h>
+#include <asm/trapnr.h>
+
+/*
+ * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is
+ * used until the idt_table takes over. On the boot CPU this happens in
+ * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases
+ * this happens in the functions called from head_64.S.
+ *
+ * The idt_table can't be used that early because all the code modifying it is
+ * in idt.c and can be instrumented by tracing or KASAN, which both don't work
+ * during early CPU bringup. Also the idt_table has the runtime vectors
+ * configured which require certain CPU state to be setup already (like TSS),
+ * which also hasn't happened yet in early CPU bringup.
+ */
+static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
+
+/* This may run while still in the direct mapping */
+void startup_64_load_idt(void *vc_handler)
+{
+	struct desc_ptr desc = {
+		.address = (unsigned long)rip_rel_ptr(bringup_idt_table),
+		.size    = sizeof(bringup_idt_table) - 1,
+	};
+	struct idt_data data;
+	gate_desc idt_desc;
+
+	/* @vc_handler is set only for a VMM Communication Exception */
+	if (vc_handler) {
+		init_idt_data(&data, X86_TRAP_VC, vc_handler);
+		idt_init_desc(&idt_desc, &data);
+		native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc);
+	}
+
+	native_load_idt(&desc);
+}
+
+/*
+ * Setup boot CPU state needed before kernel switches to virtual addresses.
+ */
+void __init startup_64_setup_gdt_idt(void)
+{
+	struct gdt_page *gp = rip_rel_ptr((void *)(__force unsigned long)&gdt_page);
+	void *handler = NULL;
+
+	struct desc_ptr startup_gdt_descr = {
+		.address = (unsigned long)gp->gdt,
+		.size    = GDT_SIZE - 1,
+	};
+
+	/* Load GDT */
+	native_load_gdt(&startup_gdt_descr);
+
+	/* New GDT is live - reload data segment registers */
+	asm volatile("movl %%eax, %%ds\n"
+		     "movl %%eax, %%ss\n"
+		     "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
+
+	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
+		handler = rip_rel_ptr(vc_no_ghcb);
+
+	startup_64_load_idt(handler);
+}
diff --git a/arch/x86/boot/startup/la57toggle.S b/arch/x86/boot/startup/la57toggle.S
new file mode 100644
index 000000000000..370075b4d95b
--- /dev/null
+++ b/arch/x86/boot/startup/la57toggle.S
@@ -0,0 +1,111 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/linkage.h>
+#include <asm/segment.h>
+#include <asm/boot.h>
+#include <asm/msr.h>
+#include <asm/processor-flags.h>
+
+/*
+ * This is the 32-bit trampoline that will be copied over to low memory. It
+ * will be called using the ordinary 64-bit calling convention from code
+ * running in 64-bit mode.
+ *
+ * Return address is at the top of the stack (might be above 4G).
+ * The first argument (EDI) contains the address of the temporary PGD level
+ * page table in 32-bit addressable memory which will be programmed into
+ * register CR3.
+ */
+
+	.section ".rodata", "a", @progbits
+SYM_CODE_START(trampoline_32bit_src)
+	/*
+	 * Preserve callee save 64-bit registers on the stack: this is
+	 * necessary because the architecture does not guarantee that GPRs will
+	 * retain their full 64-bit values across a 32-bit mode switch.
+	 */
+	pushq	%r15
+	pushq	%r14
+	pushq	%r13
+	pushq	%r12
+	pushq	%rbp
+	pushq	%rbx
+
+	/* Preserve top half of RSP in a legacy mode GPR to avoid truncation */
+	movq	%rsp, %rbx
+	shrq	$32, %rbx
+
+	/* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */
+	pushq	$__KERNEL32_CS
+	leaq	0f(%rip), %rax
+	pushq	%rax
+	lretq
+
+	/*
+	 * The 32-bit code below will do a far jump back to long mode and end
+	 * up here after reconfiguring the number of paging levels. First, the
+	 * stack pointer needs to be restored to its full 64-bit value before
+	 * the callee save register contents can be popped from the stack.
+	 */
+.Lret:
+	shlq	$32, %rbx
+	orq	%rbx, %rsp
+
+	/* Restore the preserved 64-bit registers */
+	popq	%rbx
+	popq	%rbp
+	popq	%r12
+	popq	%r13
+	popq	%r14
+	popq	%r15
+	retq
+
+	.code32
+0:
+	/* Disable paging */
+	movl	%cr0, %eax
+	btrl	$X86_CR0_PG_BIT, %eax
+	movl	%eax, %cr0
+
+	/* Point CR3 to the trampoline's new top level page table */
+	movl	%edi, %cr3
+
+	/* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */
+	movl	$MSR_EFER, %ecx
+	rdmsr
+	btsl	$_EFER_LME, %eax
+	/* Avoid writing EFER if no change was made (for TDX guest) */
+	jc	1f
+	wrmsr
+1:
+	/* Toggle CR4.LA57 */
+	movl	%cr4, %eax
+	btcl	$X86_CR4_LA57_BIT, %eax
+	movl	%eax, %cr4
+
+	/* Enable paging again. */
+	movl	%cr0, %eax
+	btsl	$X86_CR0_PG_BIT, %eax
+	movl	%eax, %cr0
+
+	/*
+	 * Return to the 64-bit calling code using LJMP rather than LRET, to
+	 * avoid the need for a 32-bit addressable stack. The destination
+	 * address will be adjusted after the template code is copied into a
+	 * 32-bit addressable buffer.
+	 */
+.Ljmp:	ljmpl	$__KERNEL_CS, $(.Lret - trampoline_32bit_src)
+SYM_CODE_END(trampoline_32bit_src)
+
+/*
+ * This symbol is placed right after trampoline_32bit_src() so its address can
+ * be used to infer the size of the trampoline code.
+ */
+SYM_DATA(trampoline_ljmp_imm_offset, .word  .Ljmp + 1 - trampoline_32bit_src)
+
+	/*
+         * The trampoline code has a size limit.
+         * Make sure we fail to compile if the trampoline code grows
+         * beyond TRAMPOLINE_32BIT_CODE_SIZE bytes.
+	 */
+	.org	trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_SIZE
diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c
new file mode 100644
index 000000000000..83ba98d61572
--- /dev/null
+++ b/arch/x86/boot/startup/map_kernel.c
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/init.h>
+#include <linux/linkage.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/pgtable.h>
+
+#include <asm/init.h>
+#include <asm/sections.h>
+#include <asm/setup.h>
+#include <asm/sev.h>
+
+extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
+extern unsigned int next_early_pgt;
+
+static inline bool check_la57_support(void)
+{
+	/*
+	 * 5-level paging is detected and enabled at kernel decompression
+	 * stage. Only check if it has been enabled there.
+	 */
+	if (!(native_read_cr4() & X86_CR4_LA57))
+		return false;
+
+	__pgtable_l5_enabled	= 1;
+	pgdir_shift		= 48;
+	ptrs_per_p4d		= 512;
+
+	return true;
+}
+
+static unsigned long __init sme_postprocess_startup(struct boot_params *bp,
+						    pmdval_t *pmd,
+						    unsigned long p2v_offset)
+{
+	unsigned long paddr, paddr_end;
+	int i;
+
+	/* Encrypt the kernel and related (if SME is active) */
+	sme_encrypt_kernel(bp);
+
+	/*
+	 * Clear the memory encryption mask from the .bss..decrypted section.
+	 * The bss section will be memset to zero later in the initialization so
+	 * there is no need to zero it after changing the memory encryption
+	 * attribute.
+	 */
+	if (sme_get_me_mask()) {
+		paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
+		paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
+
+		for (; paddr < paddr_end; paddr += PMD_SIZE) {
+			/*
+			 * On SNP, transition the page to shared in the RMP table so that
+			 * it is consistent with the page table attribute change.
+			 *
+			 * __start_bss_decrypted has a virtual address in the high range
+			 * mapping (kernel .text). PVALIDATE, by way of
+			 * early_snp_set_memory_shared(), requires a valid virtual
+			 * address but the kernel is currently running off of the identity
+			 * mapping so use the PA to get a *currently* valid virtual address.
+			 */
+			early_snp_set_memory_shared(paddr, paddr, PTRS_PER_PMD);
+
+			i = pmd_index(paddr - p2v_offset);
+			pmd[i] -= sme_get_me_mask();
+		}
+	}
+
+	/*
+	 * Return the SME encryption mask (if SME is active) to be used as a
+	 * modifier for the initial pgdir entry programmed into CR3.
+	 */
+	return sme_get_me_mask();
+}
+
+/*
+ * This code is compiled using PIC codegen because it will execute from the
+ * early 1:1 mapping of memory, which deviates from the mapping expected by the
+ * linker. Due to this deviation, taking the address of a global variable will
+ * produce an ambiguous result when using the plain & operator.  Instead,
+ * rip_rel_ptr() must be used, which will return the RIP-relative address in
+ * the 1:1 mapping of memory. Kernel virtual addresses can be determined by
+ * subtracting p2v_offset from the RIP-relative address.
+ */
+unsigned long __init __startup_64(unsigned long p2v_offset,
+				  struct boot_params *bp)
+{
+	pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
+	unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
+	unsigned long va_text, va_end;
+	unsigned long pgtable_flags;
+	unsigned long load_delta;
+	pgdval_t *pgd;
+	p4dval_t *p4d;
+	pudval_t *pud;
+	pmdval_t *pmd, pmd_entry;
+	bool la57;
+	int i;
+
+	la57 = check_la57_support();
+
+	/* Is the address too large? */
+	if (physaddr >> MAX_PHYSMEM_BITS)
+		for (;;);
+
+	/*
+	 * Compute the delta between the address I am compiled to run at
+	 * and the address I am actually running at.
+	 */
+	phys_base = load_delta = __START_KERNEL_map + p2v_offset;
+
+	/* Is the address not 2M aligned? */
+	if (load_delta & ~PMD_MASK)
+		for (;;);
+
+	va_text = physaddr - p2v_offset;
+	va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
+
+	/* Include the SME encryption mask in the fixup value */
+	load_delta += sme_get_me_mask();
+
+	/* Fixup the physical addresses in the page table */
+
+	pgd = rip_rel_ptr(early_top_pgt);
+	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
+
+	if (la57) {
+		p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
+		p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
+
+		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
+	}
+
+	level3_kernel_pgt[PTRS_PER_PUD - 2].pud += load_delta;
+	level3_kernel_pgt[PTRS_PER_PUD - 1].pud += load_delta;
+
+	for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
+		level2_fixmap_pgt[i].pmd += load_delta;
+
+	/*
+	 * Set up the identity mapping for the switchover.  These
+	 * entries should *NOT* have the global bit set!  This also
+	 * creates a bunch of nonsense entries but that is fine --
+	 * it avoids problems around wraparound.
+	 */
+
+	pud = &early_pgts[0]->pmd;
+	pmd = &early_pgts[1]->pmd;
+	next_early_pgt = 2;
+
+	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
+
+	if (la57) {
+		p4d = &early_pgts[next_early_pgt++]->pmd;
+
+		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
+		pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
+
+		i = physaddr >> P4D_SHIFT;
+		p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
+		p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
+	} else {
+		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+		pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
+		pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
+	}
+
+	i = physaddr >> PUD_SHIFT;
+	pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
+	pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
+
+	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
+	pmd_entry += sme_get_me_mask();
+	pmd_entry +=  physaddr;
+
+	for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) {
+		int idx = i + (physaddr >> PMD_SHIFT);
+
+		pmd[idx % PTRS_PER_PMD] = pmd_entry + i * PMD_SIZE;
+	}
+
+	/*
+	 * Fixup the kernel text+data virtual addresses. Note that
+	 * we might write invalid pmds, when the kernel is relocated
+	 * cleanup_highmap() fixes this up along with the mappings
+	 * beyond _end.
+	 *
+	 * Only the region occupied by the kernel image has so far
+	 * been checked against the table of usable memory regions
+	 * provided by the firmware, so invalidate pages outside that
+	 * region. A page table entry that maps to a reserved area of
+	 * memory would allow processor speculation into that area,
+	 * and on some hardware (particularly the UV platform) even
+	 * speculative access to some reserved areas is caught as an
+	 * error, causing the BIOS to halt the system.
+	 */
+
+	pmd = rip_rel_ptr(level2_kernel_pgt);
+
+	/* invalidate pages before the kernel image */
+	for (i = 0; i < pmd_index(va_text); i++)
+		pmd[i] &= ~_PAGE_PRESENT;
+
+	/* fixup pages that are part of the kernel image */
+	for (; i <= pmd_index(va_end); i++)
+		if (pmd[i] & _PAGE_PRESENT)
+			pmd[i] += load_delta;
+
+	/* invalidate pages after the kernel image */
+	for (; i < PTRS_PER_PMD; i++)
+		pmd[i] &= ~_PAGE_PRESENT;
+
+	return sme_postprocess_startup(bp, pmd, p2v_offset);
+}
diff --git a/arch/x86/boot/startup/sev-shared.c b/arch/x86/boot/startup/sev-shared.c
new file mode 100644
index 000000000000..a0fa8bb2b945
--- /dev/null
+++ b/arch/x86/boot/startup/sev-shared.c
@@ -0,0 +1,762 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD Encrypted Register State Support
+ *
+ * Author: Joerg Roedel <jroedel@suse.de>
+ *
+ * This file is not compiled stand-alone. It contains code shared
+ * between the pre-decompression boot code and the running Linux kernel
+ * and is included directly into both code-bases.
+ */
+
+#include <asm/setup_data.h>
+
+#ifndef __BOOT_COMPRESSED
+#define has_cpuflag(f)			cpu_feature_enabled(f)
+#else
+#undef WARN
+#define WARN(condition, format...) (!!(condition))
+#endif
+
+/* Copy of the SNP firmware's CPUID page. */
+static struct snp_cpuid_table cpuid_table_copy __ro_after_init;
+
+/*
+ * These will be initialized based on CPUID table so that non-present
+ * all-zero leaves (for sparse tables) can be differentiated from
+ * invalid/out-of-range leaves. This is needed since all-zero leaves
+ * still need to be post-processed.
+ */
+static u32 cpuid_std_range_max __ro_after_init;
+static u32 cpuid_hyp_range_max __ro_after_init;
+static u32 cpuid_ext_range_max __ro_after_init;
+
+bool sev_snp_needs_sfw;
+
+void __noreturn
+sev_es_terminate(unsigned int set, unsigned int reason)
+{
+	u64 val = GHCB_MSR_TERM_REQ;
+
+	/* Tell the hypervisor what went wrong. */
+	val |= GHCB_SEV_TERM_REASON(set, reason);
+
+	/* Request Guest Termination from Hypervisor */
+	sev_es_wr_ghcb_msr(val);
+	VMGEXIT();
+
+	while (true)
+		asm volatile("hlt\n" : : : "memory");
+}
+
+/*
+ * The hypervisor features are available from GHCB version 2 onward.
+ */
+u64 __init get_hv_features(void)
+{
+	u64 val;
+
+	if (ghcb_version < 2)
+		return 0;
+
+	sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ);
+	VMGEXIT();
+
+	val = sev_es_rd_ghcb_msr();
+	if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP)
+		return 0;
+
+	return GHCB_MSR_HV_FT_RESP_VAL(val);
+}
+
+int svsm_process_result_codes(struct svsm_call *call)
+{
+	switch (call->rax_out) {
+	case SVSM_SUCCESS:
+		return 0;
+	case SVSM_ERR_INCOMPLETE:
+	case SVSM_ERR_BUSY:
+		return -EAGAIN;
+	default:
+		return -EINVAL;
+	}
+}
+
+/*
+ * Issue a VMGEXIT to call the SVSM:
+ *   - Load the SVSM register state (RAX, RCX, RDX, R8 and R9)
+ *   - Set the CA call pending field to 1
+ *   - Issue VMGEXIT
+ *   - Save the SVSM return register state (RAX, RCX, RDX, R8 and R9)
+ *   - Perform atomic exchange of the CA call pending field
+ *
+ *   - See the "Secure VM Service Module for SEV-SNP Guests" specification for
+ *     details on the calling convention.
+ *     - The calling convention loosely follows the Microsoft X64 calling
+ *       convention by putting arguments in RCX, RDX, R8 and R9.
+ *     - RAX specifies the SVSM protocol/callid as input and the return code
+ *       as output.
+ */
+void svsm_issue_call(struct svsm_call *call, u8 *pending)
+{
+	register unsigned long rax asm("rax") = call->rax;
+	register unsigned long rcx asm("rcx") = call->rcx;
+	register unsigned long rdx asm("rdx") = call->rdx;
+	register unsigned long r8  asm("r8")  = call->r8;
+	register unsigned long r9  asm("r9")  = call->r9;
+
+	call->caa->call_pending = 1;
+
+	asm volatile("rep; vmmcall\n\t"
+		     : "+r" (rax), "+r" (rcx), "+r" (rdx), "+r" (r8), "+r" (r9)
+		     : : "memory");
+
+	*pending = xchg(&call->caa->call_pending, *pending);
+
+	call->rax_out = rax;
+	call->rcx_out = rcx;
+	call->rdx_out = rdx;
+	call->r8_out  = r8;
+	call->r9_out  = r9;
+}
+
+int svsm_perform_msr_protocol(struct svsm_call *call)
+{
+	u8 pending = 0;
+	u64 val, resp;
+
+	/*
+	 * When using the MSR protocol, be sure to save and restore
+	 * the current MSR value.
+	 */
+	val = sev_es_rd_ghcb_msr();
+
+	sev_es_wr_ghcb_msr(GHCB_MSR_VMPL_REQ_LEVEL(0));
+
+	svsm_issue_call(call, &pending);
+
+	resp = sev_es_rd_ghcb_msr();
+
+	sev_es_wr_ghcb_msr(val);
+
+	if (pending)
+		return -EINVAL;
+
+	if (GHCB_RESP_CODE(resp) != GHCB_MSR_VMPL_RESP)
+		return -EINVAL;
+
+	if (GHCB_MSR_VMPL_RESP_VAL(resp))
+		return -EINVAL;
+
+	return svsm_process_result_codes(call);
+}
+
+static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg)
+{
+	u64 val;
+
+	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx));
+	VMGEXIT();
+	val = sev_es_rd_ghcb_msr();
+	if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
+		return -EIO;
+
+	*reg = (val >> 32);
+
+	return 0;
+}
+
+static int __sev_cpuid_hv_msr(struct cpuid_leaf *leaf)
+{
+	int ret;
+
+	/*
+	 * MSR protocol does not support fetching non-zero subfunctions, but is
+	 * sufficient to handle current early-boot cases. Should that change,
+	 * make sure to report an error rather than ignoring the index and
+	 * grabbing random values. If this issue arises in the future, handling
+	 * can be added here to use GHCB-page protocol for cases that occur late
+	 * enough in boot that GHCB page is available.
+	 */
+	if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn)
+		return -EINVAL;
+
+	ret =         __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax);
+	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx);
+	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx);
+	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx);
+
+	return ret;
+}
+
+
+
+/*
+ * This may be called early while still running on the initial identity
+ * mapping. Use RIP-relative addressing to obtain the correct address
+ * while running with the initial identity mapping as well as the
+ * switch-over to kernel virtual addresses later.
+ */
+const struct snp_cpuid_table *snp_cpuid_get_table(void)
+{
+	return rip_rel_ptr(&cpuid_table_copy);
+}
+
+/*
+ * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of
+ * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0
+ * and 1 based on the corresponding features enabled by a particular
+ * combination of XCR0 and XSS registers so that a guest can look up the
+ * version corresponding to the features currently enabled in its XCR0/XSS
+ * registers. The only values that differ between these versions/table
+ * entries is the enabled XSAVE area size advertised via EBX.
+ *
+ * While hypervisors may choose to make use of this support, it is more
+ * robust/secure for a guest to simply find the entry corresponding to the
+ * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the
+ * XSAVE area size using subfunctions 2 through 64, as documented in APM
+ * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here.
+ *
+ * Since base/legacy XSAVE area size is documented as 0x240, use that value
+ * directly rather than relying on the base size in the CPUID table.
+ *
+ * Return: XSAVE area size on success, 0 otherwise.
+ */
+static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted)
+{
+	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
+	u64 xfeatures_found = 0;
+	u32 xsave_size = 0x240;
+	int i;
+
+	for (i = 0; i < cpuid_table->count; i++) {
+		const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
+
+		if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64))
+			continue;
+		if (!(xfeatures_en & (BIT_ULL(e->ecx_in))))
+			continue;
+		if (xfeatures_found & (BIT_ULL(e->ecx_in)))
+			continue;
+
+		xfeatures_found |= (BIT_ULL(e->ecx_in));
+
+		if (compacted)
+			xsave_size += e->eax;
+		else
+			xsave_size = max(xsave_size, e->eax + e->ebx);
+	}
+
+	/*
+	 * Either the guest set unsupported XCR0/XSS bits, or the corresponding
+	 * entries in the CPUID table were not present. This is not a valid
+	 * state to be in.
+	 */
+	if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2)))
+		return 0;
+
+	return xsave_size;
+}
+
+static bool
+snp_cpuid_get_validated_func(struct cpuid_leaf *leaf)
+{
+	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
+	int i;
+
+	for (i = 0; i < cpuid_table->count; i++) {
+		const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
+
+		if (e->eax_in != leaf->fn)
+			continue;
+
+		if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn)
+			continue;
+
+		/*
+		 * For 0xD subfunctions 0 and 1, only use the entry corresponding
+		 * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0).
+		 * See the comments above snp_cpuid_calc_xsave_size() for more
+		 * details.
+		 */
+		if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1))
+			if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in)
+				continue;
+
+		leaf->eax = e->eax;
+		leaf->ebx = e->ebx;
+		leaf->ecx = e->ecx;
+		leaf->edx = e->edx;
+
+		return true;
+	}
+
+	return false;
+}
+
+static void snp_cpuid_hv_msr(void *ctx, struct cpuid_leaf *leaf)
+{
+	if (__sev_cpuid_hv_msr(leaf))
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV);
+}
+
+static int
+snp_cpuid_postprocess(void (*cpuid_fn)(void *ctx, struct cpuid_leaf *leaf),
+		      void *ctx, struct cpuid_leaf *leaf)
+{
+	struct cpuid_leaf leaf_hv = *leaf;
+
+	switch (leaf->fn) {
+	case 0x1:
+		cpuid_fn(ctx, &leaf_hv);
+
+		/* initial APIC ID */
+		leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0));
+		/* APIC enabled bit */
+		leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9));
+
+		/* OSXSAVE enabled bit */
+		if (native_read_cr4() & X86_CR4_OSXSAVE)
+			leaf->ecx |= BIT(27);
+		break;
+	case 0x7:
+		/* OSPKE enabled bit */
+		leaf->ecx &= ~BIT(4);
+		if (native_read_cr4() & X86_CR4_PKE)
+			leaf->ecx |= BIT(4);
+		break;
+	case 0xB:
+		leaf_hv.subfn = 0;
+		cpuid_fn(ctx, &leaf_hv);
+
+		/* extended APIC ID */
+		leaf->edx = leaf_hv.edx;
+		break;
+	case 0xD: {
+		bool compacted = false;
+		u64 xcr0 = 1, xss = 0;
+		u32 xsave_size;
+
+		if (leaf->subfn != 0 && leaf->subfn != 1)
+			return 0;
+
+		if (native_read_cr4() & X86_CR4_OSXSAVE)
+			xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
+		if (leaf->subfn == 1) {
+			/* Get XSS value if XSAVES is enabled. */
+			if (leaf->eax & BIT(3)) {
+				unsigned long lo, hi;
+
+				asm volatile("rdmsr" : "=a" (lo), "=d" (hi)
+						     : "c" (MSR_IA32_XSS));
+				xss = (hi << 32) | lo;
+			}
+
+			/*
+			 * The PPR and APM aren't clear on what size should be
+			 * encoded in 0xD:0x1:EBX when compaction is not enabled
+			 * by either XSAVEC (feature bit 1) or XSAVES (feature
+			 * bit 3) since SNP-capable hardware has these feature
+			 * bits fixed as 1. KVM sets it to 0 in this case, but
+			 * to avoid this becoming an issue it's safer to simply
+			 * treat this as unsupported for SNP guests.
+			 */
+			if (!(leaf->eax & (BIT(1) | BIT(3))))
+				return -EINVAL;
+
+			compacted = true;
+		}
+
+		xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted);
+		if (!xsave_size)
+			return -EINVAL;
+
+		leaf->ebx = xsave_size;
+		}
+		break;
+	case 0x8000001E:
+		cpuid_fn(ctx, &leaf_hv);
+
+		/* extended APIC ID */
+		leaf->eax = leaf_hv.eax;
+		/* compute ID */
+		leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0));
+		/* node ID */
+		leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0));
+		break;
+	default:
+		/* No fix-ups needed, use values as-is. */
+		break;
+	}
+
+	return 0;
+}
+
+/*
+ * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value
+ * should be treated as fatal by caller.
+ */
+int snp_cpuid(void (*cpuid_fn)(void *ctx, struct cpuid_leaf *leaf),
+	      void *ctx, struct cpuid_leaf *leaf)
+{
+	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
+
+	if (!cpuid_table->count)
+		return -EOPNOTSUPP;
+
+	if (!snp_cpuid_get_validated_func(leaf)) {
+		/*
+		 * Some hypervisors will avoid keeping track of CPUID entries
+		 * where all values are zero, since they can be handled the
+		 * same as out-of-range values (all-zero). This is useful here
+		 * as well as it allows virtually all guest configurations to
+		 * work using a single SNP CPUID table.
+		 *
+		 * To allow for this, there is a need to distinguish between
+		 * out-of-range entries and in-range zero entries, since the
+		 * CPUID table entries are only a template that may need to be
+		 * augmented with additional values for things like
+		 * CPU-specific information during post-processing. So if it's
+		 * not in the table, set the values to zero. Then, if they are
+		 * within a valid CPUID range, proceed with post-processing
+		 * using zeros as the initial values. Otherwise, skip
+		 * post-processing and just return zeros immediately.
+		 */
+		leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;
+
+		/* Skip post-processing for out-of-range zero leafs. */
+		if (!(leaf->fn <= cpuid_std_range_max ||
+		      (leaf->fn >= 0x40000000 && leaf->fn <= cpuid_hyp_range_max) ||
+		      (leaf->fn >= 0x80000000 && leaf->fn <= cpuid_ext_range_max)))
+			return 0;
+	}
+
+	return snp_cpuid_postprocess(cpuid_fn, ctx, leaf);
+}
+
+/*
+ * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
+ * page yet, so it only supports the MSR based communication with the
+ * hypervisor and only the CPUID exit-code.
+ */
+void do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
+{
+	unsigned int subfn = lower_bits(regs->cx, 32);
+	unsigned int fn = lower_bits(regs->ax, 32);
+	u16 opcode = *(unsigned short *)regs->ip;
+	struct cpuid_leaf leaf;
+	int ret;
+
+	/* Only CPUID is supported via MSR protocol */
+	if (exit_code != SVM_EXIT_CPUID)
+		goto fail;
+
+	/* Is it really a CPUID insn? */
+	if (opcode != 0xa20f)
+		goto fail;
+
+	leaf.fn = fn;
+	leaf.subfn = subfn;
+
+	/*
+	 * If SNP is active, then snp_cpuid() uses the CPUID table to obtain the
+	 * CPUID values (with possible HV interaction during post-processing of
+	 * the values). But if SNP is not active (no CPUID table present), then
+	 * snp_cpuid() returns -EOPNOTSUPP so that an SEV-ES guest can call the
+	 * HV to obtain the CPUID information.
+	 */
+	ret = snp_cpuid(snp_cpuid_hv_msr, NULL, &leaf);
+	if (!ret)
+		goto cpuid_done;
+
+	if (ret != -EOPNOTSUPP)
+		goto fail;
+
+	/*
+	 * This is reached by a SEV-ES guest and needs to invoke the HV for
+	 * the CPUID data.
+	 */
+	if (__sev_cpuid_hv_msr(&leaf))
+		goto fail;
+
+cpuid_done:
+	regs->ax = leaf.eax;
+	regs->bx = leaf.ebx;
+	regs->cx = leaf.ecx;
+	regs->dx = leaf.edx;
+
+	/*
+	 * This is a VC handler and the #VC is only raised when SEV-ES is
+	 * active, which means SEV must be active too. Do sanity checks on the
+	 * CPUID results to make sure the hypervisor does not trick the kernel
+	 * into the no-sev path. This could map sensitive data unencrypted and
+	 * make it accessible to the hypervisor.
+	 *
+	 * In particular, check for:
+	 *	- Availability of CPUID leaf 0x8000001f
+	 *	- SEV CPUID bit.
+	 *
+	 * The hypervisor might still report the wrong C-bit position, but this
+	 * can't be checked here.
+	 */
+
+	if (fn == 0x80000000 && (regs->ax < 0x8000001f))
+		/* SEV leaf check */
+		goto fail;
+	else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
+		/* SEV bit */
+		goto fail;
+
+	/* Skip over the CPUID two-byte opcode */
+	regs->ip += 2;
+
+	return;
+
+fail:
+	/* Terminate the guest */
+	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
+}
+
+struct cc_setup_data {
+	struct setup_data header;
+	u32 cc_blob_address;
+};
+
+/*
+ * Search for a Confidential Computing blob passed in as a setup_data entry
+ * via the Linux Boot Protocol.
+ */
+static __init
+struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp)
+{
+	struct cc_setup_data *sd = NULL;
+	struct setup_data *hdr;
+
+	hdr = (struct setup_data *)bp->hdr.setup_data;
+
+	while (hdr) {
+		if (hdr->type == SETUP_CC_BLOB) {
+			sd = (struct cc_setup_data *)hdr;
+			return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address;
+		}
+		hdr = (struct setup_data *)hdr->next;
+	}
+
+	return NULL;
+}
+
+/*
+ * Initialize the kernel's copy of the SNP CPUID table, and set up the
+ * pointer that will be used to access it.
+ *
+ * Maintaining a direct mapping of the SNP CPUID table used by firmware would
+ * be possible as an alternative, but the approach is brittle since the
+ * mapping needs to be updated in sync with all the changes to virtual memory
+ * layout and related mapping facilities throughout the boot process.
+ */
+static void __init setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
+{
+	const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table;
+	int i;
+
+	if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE)
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
+
+	cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys;
+	if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX)
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
+
+	cpuid_table = snp_cpuid_get_table();
+	memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table));
+
+	/* Initialize CPUID ranges for range-checking. */
+	for (i = 0; i < cpuid_table->count; i++) {
+		const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
+
+		if (fn->eax_in == 0x0)
+			cpuid_std_range_max = fn->eax;
+		else if (fn->eax_in == 0x40000000)
+			cpuid_hyp_range_max = fn->eax;
+		else if (fn->eax_in == 0x80000000)
+			cpuid_ext_range_max = fn->eax;
+	}
+}
+
+static int svsm_call_msr_protocol(struct svsm_call *call)
+{
+	int ret;
+
+	do {
+		ret = svsm_perform_msr_protocol(call);
+	} while (ret == -EAGAIN);
+
+	return ret;
+}
+
+static void svsm_pval_4k_page(unsigned long paddr, bool validate,
+			      struct svsm_ca *caa, u64 caa_pa)
+{
+	struct svsm_pvalidate_call *pc;
+	struct svsm_call call = {};
+	unsigned long flags;
+	u64 pc_pa;
+
+	/*
+	 * This can be called very early in the boot, use native functions in
+	 * order to avoid paravirt issues.
+	 */
+	flags = native_local_irq_save();
+
+	call.caa = caa;
+
+	pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer;
+	pc_pa = caa_pa + offsetof(struct svsm_ca, svsm_buffer);
+
+	pc->num_entries = 1;
+	pc->cur_index   = 0;
+	pc->entry[0].page_size = RMP_PG_SIZE_4K;
+	pc->entry[0].action    = validate;
+	pc->entry[0].ignore_cf = 0;
+	pc->entry[0].rsvd      = 0;
+	pc->entry[0].pfn       = paddr >> PAGE_SHIFT;
+
+	/* Protocol 0, Call ID 1 */
+	call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE);
+	call.rcx = pc_pa;
+
+	/*
+	 * Use the MSR protocol exclusively, so that this code is usable in
+	 * startup code where VA/PA translations of the GHCB page's address may
+	 * be problematic.
+	 */
+	if (svsm_call_msr_protocol(&call))
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
+
+	native_local_irq_restore(flags);
+}
+
+static void pvalidate_4k_page(unsigned long vaddr, unsigned long paddr,
+			      bool validate, struct svsm_ca *caa, u64 caa_pa)
+{
+	int ret;
+
+	if (snp_vmpl) {
+		svsm_pval_4k_page(paddr, validate, caa, caa_pa);
+	} else {
+		ret = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
+		if (ret)
+			sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
+	}
+
+	/*
+	 * If validating memory (making it private) and affected by the
+	 * cache-coherency vulnerability, perform the cache eviction mitigation.
+	 */
+	if (validate && sev_snp_needs_sfw)
+		sev_evict_cache((void *)vaddr, 1);
+}
+
+static void __page_state_change(unsigned long vaddr, unsigned long paddr,
+			        const struct psc_desc *desc)
+{
+	u64 val, msr;
+
+	/*
+	 * If private -> shared then invalidate the page before requesting the
+	 * state change in the RMP table.
+	 */
+	if (desc->op == SNP_PAGE_STATE_SHARED)
+		pvalidate_4k_page(vaddr, paddr, false, desc->ca, desc->caa_pa);
+
+	/* Save the current GHCB MSR value */
+	msr = sev_es_rd_ghcb_msr();
+
+	/* Issue VMGEXIT to change the page state in RMP table. */
+	sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, desc->op));
+	VMGEXIT();
+
+	/* Read the response of the VMGEXIT. */
+	val = sev_es_rd_ghcb_msr();
+	if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val))
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
+
+	/* Restore the GHCB MSR value */
+	sev_es_wr_ghcb_msr(msr);
+
+	/*
+	 * Now that page state is changed in the RMP table, validate it so that it is
+	 * consistent with the RMP entry.
+	 */
+	if (desc->op == SNP_PAGE_STATE_PRIVATE)
+		pvalidate_4k_page(vaddr, paddr, true, desc->ca, desc->caa_pa);
+}
+
+/*
+ * Maintain the GPA of the SVSM Calling Area (CA) in order to utilize the SVSM
+ * services needed when not running in VMPL0.
+ */
+static bool __init svsm_setup_ca(const struct cc_blob_sev_info *cc_info,
+				 void *page)
+{
+	struct snp_secrets_page *secrets_page;
+	struct snp_cpuid_table *cpuid_table;
+	unsigned int i;
+	u64 caa;
+
+	BUILD_BUG_ON(sizeof(*secrets_page) != PAGE_SIZE);
+
+	/*
+	 * Check if running at VMPL0.
+	 *
+	 * Use RMPADJUST (see the rmpadjust() function for a description of what
+	 * the instruction does) to update the VMPL1 permissions of a page. If
+	 * the guest is running at VMPL0, this will succeed and implies there is
+	 * no SVSM. If the guest is running at any other VMPL, this will fail.
+	 * Linux SNP guests only ever run at a single VMPL level so permission mask
+	 * changes of a lesser-privileged VMPL are a don't-care.
+	 *
+	 * Use a rip-relative reference to obtain the proper address, since this
+	 * routine is running identity mapped when called, both by the decompressor
+	 * code and the early kernel code.
+	 */
+	if (!rmpadjust((unsigned long)page, RMP_PG_SIZE_4K, 1))
+		return false;
+
+	/*
+	 * Not running at VMPL0, ensure everything has been properly supplied
+	 * for running under an SVSM.
+	 */
+	if (!cc_info || !cc_info->secrets_phys || cc_info->secrets_len != PAGE_SIZE)
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SECRETS_PAGE);
+
+	secrets_page = (struct snp_secrets_page *)cc_info->secrets_phys;
+	if (!secrets_page->svsm_size)
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NO_SVSM);
+
+	if (!secrets_page->svsm_guest_vmpl)
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_VMPL0);
+
+	snp_vmpl = secrets_page->svsm_guest_vmpl;
+
+	caa = secrets_page->svsm_caa;
+
+	/*
+	 * An open-coded PAGE_ALIGNED() in order to avoid including
+	 * kernel-proper headers into the decompressor.
+	 */
+	if (caa & (PAGE_SIZE - 1))
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CAA);
+
+	boot_svsm_caa_pa = caa;
+
+	/* Advertise the SVSM presence via CPUID. */
+	cpuid_table = (struct snp_cpuid_table *)snp_cpuid_get_table();
+	for (i = 0; i < cpuid_table->count; i++) {
+		struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
+
+		if (fn->eax_in == 0x8000001f)
+			fn->eax |= BIT(28);
+	}
+
+	return true;
+}
diff --git a/arch/x86/boot/startup/sev-startup.c b/arch/x86/boot/startup/sev-startup.c
new file mode 100644
index 000000000000..09725428d3e6
--- /dev/null
+++ b/arch/x86/boot/startup/sev-startup.c
@@ -0,0 +1,220 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AMD Memory Encryption Support
+ *
+ * Copyright (C) 2019 SUSE
+ *
+ * Author: Joerg Roedel <jroedel@suse.de>
+ */
+
+#define pr_fmt(fmt)	"SEV: " fmt
+
+#include <linux/percpu-defs.h>
+#include <linux/cc_platform.h>
+#include <linux/printk.h>
+#include <linux/mm_types.h>
+#include <linux/set_memory.h>
+#include <linux/memblock.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/cpumask.h>
+#include <linux/efi.h>
+#include <linux/io.h>
+#include <linux/psp-sev.h>
+#include <uapi/linux/sev-guest.h>
+
+#include <asm/init.h>
+#include <asm/cpu_entry_area.h>
+#include <asm/stacktrace.h>
+#include <asm/sev.h>
+#include <asm/sev-internal.h>
+#include <asm/insn-eval.h>
+#include <asm/fpu/xcr.h>
+#include <asm/processor.h>
+#include <asm/realmode.h>
+#include <asm/setup.h>
+#include <asm/traps.h>
+#include <asm/svm.h>
+#include <asm/smp.h>
+#include <asm/cpu.h>
+#include <asm/apic.h>
+#include <asm/cpuid/api.h>
+#include <asm/cmdline.h>
+
+/* Include code shared with pre-decompression boot stage */
+#include "sev-shared.c"
+
+void
+early_set_pages_state(unsigned long vaddr, unsigned long paddr,
+		      unsigned long npages, const struct psc_desc *desc)
+{
+	unsigned long paddr_end;
+
+	vaddr = vaddr & PAGE_MASK;
+
+	paddr = paddr & PAGE_MASK;
+	paddr_end = paddr + (npages << PAGE_SHIFT);
+
+	while (paddr < paddr_end) {
+		__page_state_change(vaddr, paddr, desc);
+
+		vaddr += PAGE_SIZE;
+		paddr += PAGE_SIZE;
+	}
+}
+
+void __init early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr,
+					 unsigned long npages)
+{
+	struct psc_desc d = {
+		SNP_PAGE_STATE_PRIVATE,
+		rip_rel_ptr(&boot_svsm_ca_page),
+		boot_svsm_caa_pa
+	};
+
+	/*
+	 * This can be invoked in early boot while running identity mapped, so
+	 * use an open coded check for SNP instead of using cc_platform_has().
+	 * This eliminates worries about jump tables or checking boot_cpu_data
+	 * in the cc_platform_has() function.
+	 */
+	if (!(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
+		return;
+
+	 /*
+	  * Ask the hypervisor to mark the memory pages as private in the RMP
+	  * table.
+	  */
+	early_set_pages_state(vaddr, paddr, npages, &d);
+}
+
+void __init early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr,
+					unsigned long npages)
+{
+	struct psc_desc d = {
+		SNP_PAGE_STATE_SHARED,
+		rip_rel_ptr(&boot_svsm_ca_page),
+		boot_svsm_caa_pa
+	};
+
+	/*
+	 * This can be invoked in early boot while running identity mapped, so
+	 * use an open coded check for SNP instead of using cc_platform_has().
+	 * This eliminates worries about jump tables or checking boot_cpu_data
+	 * in the cc_platform_has() function.
+	 */
+	if (!(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
+		return;
+
+	 /* Ask hypervisor to mark the memory pages shared in the RMP table. */
+	early_set_pages_state(vaddr, paddr, npages, &d);
+}
+
+/*
+ * Initial set up of SNP relies on information provided by the
+ * Confidential Computing blob, which can be passed to the kernel
+ * in the following ways, depending on how it is booted:
+ *
+ * - when booted via the boot/decompress kernel:
+ *   - via boot_params
+ *
+ * - when booted directly by firmware/bootloader (e.g. CONFIG_PVH):
+ *   - via a setup_data entry, as defined by the Linux Boot Protocol
+ *
+ * Scan for the blob in that order.
+ */
+static struct cc_blob_sev_info *__init find_cc_blob(struct boot_params *bp)
+{
+	struct cc_blob_sev_info *cc_info;
+
+	/* Boot kernel would have passed the CC blob via boot_params. */
+	if (bp->cc_blob_address) {
+		cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address;
+		goto found_cc_info;
+	}
+
+	/*
+	 * If kernel was booted directly, without the use of the
+	 * boot/decompression kernel, the CC blob may have been passed via
+	 * setup_data instead.
+	 */
+	cc_info = find_cc_blob_setup_data(bp);
+	if (!cc_info)
+		return NULL;
+
+found_cc_info:
+	if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
+		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
+
+	return cc_info;
+}
+
+static void __init svsm_setup(struct cc_blob_sev_info *cc_info)
+{
+	struct snp_secrets_page *secrets = (void *)cc_info->secrets_phys;
+	struct svsm_call call = {};
+	u64 pa;
+
+	/*
+	 * Record the SVSM Calling Area address (CAA) if the guest is not
+	 * running at VMPL0. The CA will be used to communicate with the
+	 * SVSM to perform the SVSM services.
+	 */
+	if (!svsm_setup_ca(cc_info, rip_rel_ptr(&boot_svsm_ca_page)))
+		return;
+
+	/*
+	 * It is very early in the boot and the kernel is running identity
+	 * mapped but without having adjusted the pagetables to where the
+	 * kernel was loaded (physbase), so the get the CA address using
+	 * RIP-relative addressing.
+	 */
+	pa = (u64)rip_rel_ptr(&boot_svsm_ca_page);
+
+	/*
+	 * Switch over to the boot SVSM CA while the current CA is still 1:1
+	 * mapped and thus addressable with VA == PA. There is no GHCB at this
+	 * point so use the MSR protocol.
+	 *
+	 * SVSM_CORE_REMAP_CA call:
+	 *   RAX = 0 (Protocol=0, CallID=0)
+	 *   RCX = New CA GPA
+	 */
+	call.caa = (struct svsm_ca *)secrets->svsm_caa;
+	call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA);
+	call.rcx = pa;
+
+	if (svsm_call_msr_protocol(&call))
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CA_REMAP_FAIL);
+
+	boot_svsm_caa_pa = pa;
+}
+
+bool __init snp_init(struct boot_params *bp)
+{
+	struct cc_blob_sev_info *cc_info;
+
+	if (!bp)
+		return false;
+
+	cc_info = find_cc_blob(bp);
+	if (!cc_info)
+		return false;
+
+	if (cc_info->secrets_phys && cc_info->secrets_len == PAGE_SIZE)
+		sev_secrets_pa = cc_info->secrets_phys;
+	else
+		return false;
+
+	setup_cpuid_table(cc_info);
+
+	svsm_setup(cc_info);
+
+	/*
+	 * The CC blob will be used later to access the secrets page. Cache
+	 * it here like the boot kernel does.
+	 */
+	bp->cc_blob_address = (u32)(unsigned long)cc_info;
+
+	return true;
+}
diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/boot/startup/sme.c
index ac33b2263a43..e7ea65f3f1d6 100644
--- a/arch/x86/mm/mem_encrypt_identity.c
+++ b/arch/x86/boot/startup/sme.c
@@ -7,8 +7,6 @@
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
  */
 
-#define DISABLE_BRANCH_PROFILING
-
 /*
  * Since we're dealing with identity mappings, physical and virtual
  * addresses are the same, so override these defines which are ultimately
@@ -47,8 +45,6 @@
 #include <asm/coco.h>
 #include <asm/sev.h>
 
-#include "mm_internal.h"
-
 #define PGD_FLAGS		_KERNPG_TABLE_NOENC
 #define P4D_FLAGS		_KERNPG_TABLE_NOENC
 #define PUD_FLAGS		_KERNPG_TABLE_NOENC
@@ -95,7 +91,7 @@ struct sme_populate_pgd_data {
  */
 static char sme_workarea[2 * PMD_SIZE] __section(".init.scratch");
 
-static void __head sme_clear_pgd(struct sme_populate_pgd_data *ppd)
+static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd)
 {
 	unsigned long pgd_start, pgd_end, pgd_size;
 	pgd_t *pgd_p;
@@ -110,7 +106,7 @@ static void __head sme_clear_pgd(struct sme_populate_pgd_data *ppd)
 	memset(pgd_p, 0, pgd_size);
 }
 
-static pud_t __head *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
+static pud_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
 {
 	pgd_t *pgd;
 	p4d_t *p4d;
@@ -147,7 +143,7 @@ static pud_t __head *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
 	return pud;
 }
 
-static void __head sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
+static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
 {
 	pud_t *pud;
 	pmd_t *pmd;
@@ -163,7 +159,7 @@ static void __head sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
 	set_pmd(pmd, __pmd(ppd->paddr | ppd->pmd_flags));
 }
 
-static void __head sme_populate_pgd(struct sme_populate_pgd_data *ppd)
+static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd)
 {
 	pud_t *pud;
 	pmd_t *pmd;
@@ -189,7 +185,7 @@ static void __head sme_populate_pgd(struct sme_populate_pgd_data *ppd)
 		set_pte(pte, __pte(ppd->paddr | ppd->pte_flags));
 }
 
-static void __head __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
+static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
 {
 	while (ppd->vaddr < ppd->vaddr_end) {
 		sme_populate_pgd_large(ppd);
@@ -199,7 +195,7 @@ static void __head __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
 	}
 }
 
-static void __head __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
+static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
 {
 	while (ppd->vaddr < ppd->vaddr_end) {
 		sme_populate_pgd(ppd);
@@ -209,7 +205,7 @@ static void __head __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
 	}
 }
 
-static void __head __sme_map_range(struct sme_populate_pgd_data *ppd,
+static void __init __sme_map_range(struct sme_populate_pgd_data *ppd,
 				   pmdval_t pmd_flags, pteval_t pte_flags)
 {
 	unsigned long vaddr_end;
@@ -233,22 +229,22 @@ static void __head __sme_map_range(struct sme_populate_pgd_data *ppd,
 	__sme_map_range_pte(ppd);
 }
 
-static void __head sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
+static void __init sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
 {
 	__sme_map_range(ppd, PMD_FLAGS_ENC, PTE_FLAGS_ENC);
 }
 
-static void __head sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
+static void __init sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
 {
 	__sme_map_range(ppd, PMD_FLAGS_DEC, PTE_FLAGS_DEC);
 }
 
-static void __head sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
+static void __init sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
 {
 	__sme_map_range(ppd, PMD_FLAGS_DEC_WP, PTE_FLAGS_DEC_WP);
 }
 
-static unsigned long __head sme_pgtable_calc(unsigned long len)
+static unsigned long __init sme_pgtable_calc(unsigned long len)
 {
 	unsigned long entries = 0, tables = 0;
 
@@ -285,7 +281,7 @@ static unsigned long __head sme_pgtable_calc(unsigned long len)
 	return entries + tables;
 }
 
-void __head sme_encrypt_kernel(struct boot_params *bp)
+void __init sme_encrypt_kernel(struct boot_params *bp)
 {
 	unsigned long workarea_start, workarea_end, workarea_len;
 	unsigned long execute_start, execute_end, execute_len;
@@ -301,8 +297,7 @@ void __head sme_encrypt_kernel(struct boot_params *bp)
 	 * instrumentation or checking boot_cpu_data in the cc_platform_has()
 	 * function.
 	 */
-	if (!sme_get_me_mask() ||
-	    RIP_REL_REF(sev_status) & MSR_AMD64_SEV_ENABLED)
+	if (!sme_get_me_mask() || sev_status & MSR_AMD64_SEV_ENABLED)
 		return;
 
 	/*
@@ -320,8 +315,8 @@ void __head sme_encrypt_kernel(struct boot_params *bp)
 	 *     memory from being cached.
 	 */
 
-	kernel_start = (unsigned long)RIP_REL_REF(_text);
-	kernel_end = ALIGN((unsigned long)RIP_REL_REF(_end), PMD_SIZE);
+	kernel_start = (unsigned long)rip_rel_ptr(_text);
+	kernel_end = ALIGN((unsigned long)rip_rel_ptr(_end), PMD_SIZE);
 	kernel_len = kernel_end - kernel_start;
 
 	initrd_start = 0;
@@ -347,7 +342,7 @@ void __head sme_encrypt_kernel(struct boot_params *bp)
 	 *   pagetable structures for the encryption of the kernel
 	 *   pagetable structures for workarea (in case not currently mapped)
 	 */
-	execute_start = workarea_start = (unsigned long)RIP_REL_REF(sme_workarea);
+	execute_start = workarea_start = (unsigned long)rip_rel_ptr(sme_workarea);
 	execute_end = execute_start + (PAGE_SIZE * 2) + PMD_SIZE;
 	execute_len = execute_end - execute_start;
 
@@ -490,15 +485,15 @@ void __head sme_encrypt_kernel(struct boot_params *bp)
 	native_write_cr3(__native_read_cr3());
 }
 
-void __head sme_enable(struct boot_params *bp)
+void __init sme_enable(struct boot_params *bp)
 {
 	unsigned int eax, ebx, ecx, edx;
 	unsigned long feature_mask;
 	unsigned long me_mask;
-	bool snp;
+	bool snp_en;
 	u64 msr;
 
-	snp = snp_init(bp);
+	snp_en = snp_init(bp);
 
 	/* Check for the SME/SEV support leaf */
 	eax = 0x80000000;
@@ -526,14 +521,18 @@ void __head sme_enable(struct boot_params *bp)
 		return;
 
 	me_mask = 1UL << (ebx & 0x3f);
+	sev_snp_needs_sfw = !(ebx & BIT(31));
 
 	/* Check the SEV MSR whether SEV or SME is enabled */
-	RIP_REL_REF(sev_status) = msr = __rdmsr(MSR_AMD64_SEV);
+	sev_status = msr = native_rdmsrq(MSR_AMD64_SEV);
 	feature_mask = (msr & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT;
 
-	/* The SEV-SNP CC blob should never be present unless SEV-SNP is enabled. */
-	if (snp && !(msr & MSR_AMD64_SEV_SNP_ENABLED))
-		snp_abort();
+	/*
+	 * Any discrepancies between the presence of a CC blob and SNP
+	 * enablement abort the guest.
+	 */
+	if (snp_en ^ !!(msr & MSR_AMD64_SEV_SNP_ENABLED))
+		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
 
 	/* Check if memory encryption is enabled */
 	if (feature_mask == AMD_SME_BIT) {
@@ -556,13 +555,21 @@ void __head sme_enable(struct boot_params *bp)
 			return;
 
 		/* For SME, check the SYSCFG MSR */
-		msr = __rdmsr(MSR_AMD64_SYSCFG);
+		msr = native_rdmsrq(MSR_AMD64_SYSCFG);
 		if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
 			return;
 	}
 
-	RIP_REL_REF(sme_me_mask) = me_mask;
-	physical_mask &= ~me_mask;
-	cc_vendor = CC_VENDOR_AMD;
+	sme_me_mask	= me_mask;
+	physical_mask	&= ~me_mask;
+	cc_vendor	= CC_VENDOR_AMD;
 	cc_set_mask(me_mask);
 }
+
+#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
+/* Local version for startup code, which never operates on user page tables */
+pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
+{
+	return pgd;
+}
+#endif
diff --git a/arch/x86/boot/string.c b/arch/x86/boot/string.c
index c23f3b9c84fe..b25c6a9303b7 100644
--- a/arch/x86/boot/string.c
+++ b/arch/x86/boot/string.c
@@ -32,8 +32,8 @@
 int memcmp(const void *s1, const void *s2, size_t len)
 {
 	bool diff;
-	asm("repe; cmpsb" CC_SET(nz)
-	    : CC_OUT(nz) (diff), "+D" (s1), "+S" (s2), "+c" (len));
+	asm("repe cmpsb"
+	    : "=@ccnz" (diff), "+D" (s1), "+S" (s2), "+c" (len));
 	return diff;
 }
 
@@ -88,14 +88,6 @@ size_t strnlen(const char *s, size_t maxlen)
 	return (es - s);
 }
 
-unsigned int atou(const char *s)
-{
-	unsigned int i = 0;
-	while (isdigit(*s))
-		i = i * 10 + (*s++ - '0');
-	return i;
-}
-
 /* Works only for digits and letters, but small and fast */
 #define TOLOWER(x) ((x) | 0x20)
 
diff --git a/arch/x86/boot/string.h b/arch/x86/boot/string.h
index e5d2c6b8c2f1..a5b05ebc037d 100644
--- a/arch/x86/boot/string.h
+++ b/arch/x86/boot/string.h
@@ -24,7 +24,6 @@ extern size_t strlen(const char *s);
 extern char *strstr(const char *s1, const char *s2);
 extern char *strchr(const char *s, int c);
 extern size_t strnlen(const char *s, size_t maxlen);
-extern unsigned int atou(const char *s);
 extern unsigned long long simple_strtoull(const char *cp, char **endp,
 					  unsigned int base);
 long simple_strtol(const char *cp, char **endp, unsigned int base);
diff --git a/arch/x86/boot/tools/.gitignore b/arch/x86/boot/tools/.gitignore
deleted file mode 100644
index ae91f4d0d78b..000000000000
--- a/arch/x86/boot/tools/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-build
diff --git a/arch/x86/boot/tools/build.c b/arch/x86/boot/tools/build.c
deleted file mode 100644
index 10311d77c67f..000000000000
--- a/arch/x86/boot/tools/build.c
+++ /dev/null
@@ -1,247 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- *  Copyright (C) 1991, 1992  Linus Torvalds
- *  Copyright (C) 1997 Martin Mares
- *  Copyright (C) 2007 H. Peter Anvin
- */
-
-/*
- * This file builds a disk-image from three different files:
- *
- * - setup: 8086 machine code, sets up system parm
- * - system: 80386 code for actual system
- * - zoffset.h: header with ZO_* defines
- *
- * It does some checking that all files are of the correct type, and writes
- * the result to the specified destination, removing headers and padding to
- * the right amount. It also writes some system data to stdout.
- */
-
-/*
- * Changes by tytso to allow root device specification
- * High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996
- * Cross compiling fixes by Gertjan van Wingerde, July 1996
- * Rewritten by Martin Mares, April 1997
- * Substantially overhauled by H. Peter Anvin, April 2007
- */
-
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/mman.h>
-#include <tools/le_byteshift.h>
-
-typedef unsigned char  u8;
-typedef unsigned short u16;
-typedef unsigned int   u32;
-
-/* Minimal number of setup sectors */
-#define SETUP_SECT_MIN 5
-#define SETUP_SECT_MAX 64
-
-/* This must be large enough to hold the entire setup */
-u8 buf[SETUP_SECT_MAX*512];
-
-static unsigned long _edata;
-
-/*----------------------------------------------------------------------*/
-
-static const u32 crctab32[] = {
-	0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419,
-	0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4,
-	0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07,
-	0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
-	0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856,
-	0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
-	0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4,
-	0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
-	0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3,
-	0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a,
-	0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599,
-	0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
-	0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190,
-	0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f,
-	0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e,
-	0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
-	0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed,
-	0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
-	0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3,
-	0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
-	0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a,
-	0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5,
-	0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010,
-	0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
-	0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17,
-	0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6,
-	0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615,
-	0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
-	0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344,
-	0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
-	0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a,
-	0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
-	0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1,
-	0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c,
-	0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef,
-	0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
-	0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe,
-	0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31,
-	0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c,
-	0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
-	0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b,
-	0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
-	0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1,
-	0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
-	0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278,
-	0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7,
-	0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66,
-	0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
-	0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605,
-	0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8,
-	0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b,
-	0x2d02ef8d
-};
-
-static u32 partial_crc32_one(u8 c, u32 crc)
-{
-	return crctab32[(crc ^ c) & 0xff] ^ (crc >> 8);
-}
-
-static u32 partial_crc32(const u8 *s, int len, u32 crc)
-{
-	while (len--)
-		crc = partial_crc32_one(*s++, crc);
-	return crc;
-}
-
-static void die(const char * str, ...)
-{
-	va_list args;
-	va_start(args, str);
-	vfprintf(stderr, str, args);
-	va_end(args);
-	fputc('\n', stderr);
-	exit(1);
-}
-
-static void usage(void)
-{
-	die("Usage: build setup system zoffset.h image");
-}
-
-/*
- * Parse zoffset.h and find the entry points. We could just #include zoffset.h
- * but that would mean tools/build would have to be rebuilt every time. It's
- * not as if parsing it is hard...
- */
-#define PARSE_ZOFS(p, sym) do { \
-	if (!strncmp(p, "#define ZO_" #sym " ", 11+sizeof(#sym)))	\
-		sym = strtoul(p + 11 + sizeof(#sym), NULL, 16);		\
-} while (0)
-
-static void parse_zoffset(char *fname)
-{
-	FILE *file;
-	char *p;
-	int c;
-
-	file = fopen(fname, "r");
-	if (!file)
-		die("Unable to open `%s': %m", fname);
-	c = fread(buf, 1, sizeof(buf) - 1, file);
-	if (ferror(file))
-		die("read-error on `zoffset.h'");
-	fclose(file);
-	buf[c] = 0;
-
-	p = (char *)buf;
-
-	while (p && *p) {
-		PARSE_ZOFS(p, _edata);
-
-		p = strchr(p, '\n');
-		while (p && (*p == '\r' || *p == '\n'))
-			p++;
-	}
-}
-
-int main(int argc, char ** argv)
-{
-	unsigned int i, sz, setup_sectors;
-	int c;
-	struct stat sb;
-	FILE *file, *dest;
-	int fd;
-	void *kernel;
-	u32 crc = 0xffffffffUL;
-
-	if (argc != 5)
-		usage();
-	parse_zoffset(argv[3]);
-
-	dest = fopen(argv[4], "w");
-	if (!dest)
-		die("Unable to write `%s': %m", argv[4]);
-
-	/* Copy the setup code */
-	file = fopen(argv[1], "r");
-	if (!file)
-		die("Unable to open `%s': %m", argv[1]);
-	c = fread(buf, 1, sizeof(buf), file);
-	if (ferror(file))
-		die("read-error on `setup'");
-	if (c < 1024)
-		die("The setup must be at least 1024 bytes");
-	if (get_unaligned_le16(&buf[510]) != 0xAA55)
-		die("Boot block hasn't got boot flag (0xAA55)");
-	fclose(file);
-
-	/* Pad unused space with zeros */
-	setup_sectors = (c + 4095) / 4096;
-	setup_sectors *= 8;
-	if (setup_sectors < SETUP_SECT_MIN)
-		setup_sectors = SETUP_SECT_MIN;
-	i = setup_sectors*512;
-	memset(buf+c, 0, i-c);
-
-	/* Open and stat the kernel file */
-	fd = open(argv[2], O_RDONLY);
-	if (fd < 0)
-		die("Unable to open `%s': %m", argv[2]);
-	if (fstat(fd, &sb))
-		die("Unable to stat `%s': %m", argv[2]);
-	if (_edata != sb.st_size)
-		die("Unexpected file size `%s': %u != %u", argv[2], _edata,
-		    sb.st_size);
-	sz = _edata - 4;
-	kernel = mmap(NULL, sz, PROT_READ, MAP_SHARED, fd, 0);
-	if (kernel == MAP_FAILED)
-		die("Unable to mmap '%s': %m", argv[2]);
-
-	crc = partial_crc32(buf, i, crc);
-	if (fwrite(buf, 1, i, dest) != i)
-		die("Writing setup failed");
-
-	/* Copy the kernel code */
-	crc = partial_crc32(kernel, sz, crc);
-	if (fwrite(kernel, 1, sz, dest) != sz)
-		die("Writing kernel failed");
-
-	/* Write the CRC */
-	put_unaligned_le32(crc, buf);
-	if (fwrite(buf, 1, 4, dest) != 4)
-		die("Writing CRC failed");
-
-	/* Catch any delayed write failures */
-	if (fclose(dest))
-		die("Writing image failed");
-
-	close(fd);
-
-	/* Everything is OK */
-	return 0;
-}
diff --git a/arch/x86/boot/video.c b/arch/x86/boot/video.c
index f2e96905b3fe..0641c8c46aee 100644
--- a/arch/x86/boot/video.c
+++ b/arch/x86/boot/video.c
@@ -292,7 +292,7 @@ static void restore_screen(void)
 			     "shrw %%cx ; "
 			     "jnc 1f ; "
 			     "stosw \n\t"
-			     "1: rep;stosl ; "
+			     "1: rep stosl ; "
 			     "popw %%es"
 			     : "+D" (dst), "+c" (npad)
 			     : "bdS" (video_segment),
diff --git a/arch/x86/coco/core.c b/arch/x86/coco/core.c
index 0f81f70aca82..989ca9f72ba3 100644
--- a/arch/x86/coco/core.c
+++ b/arch/x86/coco/core.c
@@ -18,7 +18,9 @@
 #include <asm/processor.h>
 
 enum cc_vendor cc_vendor __ro_after_init = CC_VENDOR_NONE;
+SYM_PIC_ALIAS(cc_vendor);
 u64 cc_mask __ro_after_init;
+SYM_PIC_ALIAS(cc_mask);
 
 static struct cc_attr_flags {
 	__u64 host_sev_snp	: 1,
@@ -65,7 +67,6 @@ static __maybe_unused __always_inline bool amd_cc_platform_vtom(enum cc_attr att
  * up under SME the trampoline area cannot be encrypted, whereas under SEV
  * the trampoline area must be encrypted.
  */
-
 static bool noinstr amd_cc_platform_has(enum cc_attr attr)
 {
 #ifdef CONFIG_AMD_MEM_ENCRYPT
@@ -97,9 +98,15 @@ static bool noinstr amd_cc_platform_has(enum cc_attr attr)
 	case CC_ATTR_GUEST_SEV_SNP:
 		return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
 
+	case CC_ATTR_GUEST_SNP_SECURE_TSC:
+		return sev_status & MSR_AMD64_SNP_SECURE_TSC;
+
 	case CC_ATTR_HOST_SEV_SNP:
 		return cc_flags.host_sev_snp;
 
+	case CC_ATTR_SNP_SECURE_AVIC:
+		return sev_status & MSR_AMD64_SNP_SECURE_AVIC;
+
 	default:
 		return false;
 	}
diff --git a/arch/x86/coco/sev/Makefile b/arch/x86/coco/sev/Makefile
index 4e375e7305ac..3b8ae214a6a6 100644
--- a/arch/x86/coco/sev/Makefile
+++ b/arch/x86/coco/sev/Makefile
@@ -1,15 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0
 
-obj-y += core.o
+obj-y += core.o noinstr.o vc-handle.o
 
-ifdef CONFIG_FUNCTION_TRACER
-CFLAGS_REMOVE_core.o = -pg
-endif
+# Clang 14 and older may fail to respect __no_sanitize_undefined when inlining
+UBSAN_SANITIZE_noinstr.o	:= n
 
-KASAN_SANITIZE_core.o	:= n
-KMSAN_SANITIZE_core.o	:= n
-KCOV_INSTRUMENT_core.o	:= n
-
-# With some compiler versions the generated code results in boot hangs, caused
-# by several compilation units. To be safe, disable all instrumentation.
-KCSAN_SANITIZE		:= n
+# GCC may fail to respect __no_sanitize_address or __no_kcsan when inlining
+KASAN_SANITIZE_noinstr.o	:= n
+KCSAN_SANITIZE_noinstr.o	:= n
diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
index de1df0cb45da..9ae3b11754e6 100644
--- a/arch/x86/coco/sev/core.c
+++ b/arch/x86/coco/sev/core.c
@@ -25,11 +25,13 @@
 #include <linux/psp-sev.h>
 #include <linux/dmi.h>
 #include <uapi/linux/sev-guest.h>
+#include <crypto/gcm.h>
 
 #include <asm/init.h>
 #include <asm/cpu_entry_area.h>
 #include <asm/stacktrace.h>
 #include <asm/sev.h>
+#include <asm/sev-internal.h>
 #include <asm/insn-eval.h>
 #include <asm/fpu/xcr.h>
 #include <asm/processor.h>
@@ -40,10 +42,51 @@
 #include <asm/smp.h>
 #include <asm/cpu.h>
 #include <asm/apic.h>
-#include <asm/cpuid.h>
+#include <asm/cpuid/api.h>
 #include <asm/cmdline.h>
+#include <asm/msr.h>
 
-#define DR7_RESET_VALUE        0x400
+/* Bitmap of SEV features supported by the hypervisor */
+u64 sev_hv_features __ro_after_init;
+SYM_PIC_ALIAS(sev_hv_features);
+
+/* Secrets page physical address from the CC blob */
+u64 sev_secrets_pa __ro_after_init;
+SYM_PIC_ALIAS(sev_secrets_pa);
+
+/* For early boot SVSM communication */
+struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);
+SYM_PIC_ALIAS(boot_svsm_ca_page);
+
+/*
+ * SVSM related information:
+ *   During boot, the page tables are set up as identity mapped and later
+ *   changed to use kernel virtual addresses. Maintain separate virtual and
+ *   physical addresses for the CAA to allow SVSM functions to be used during
+ *   early boot, both with identity mapped virtual addresses and proper kernel
+ *   virtual addresses.
+ */
+u64 boot_svsm_caa_pa __ro_after_init;
+SYM_PIC_ALIAS(boot_svsm_caa_pa);
+
+DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
+DEFINE_PER_CPU(u64, svsm_caa_pa);
+
+static inline struct svsm_ca *svsm_get_caa(void)
+{
+	if (sev_cfg.use_cas)
+		return this_cpu_read(svsm_caa);
+	else
+		return rip_rel_ptr(&boot_svsm_ca_page);
+}
+
+static inline u64 svsm_get_caa_pa(void)
+{
+	if (sev_cfg.use_cas)
+		return this_cpu_read(svsm_caa_pa);
+	else
+		return boot_svsm_caa_pa;
+}
 
 /* AP INIT values as documented in the APM2  section "Processor Initialization State" */
 #define AP_INIT_CS_LIMIT		0xffff
@@ -78,599 +121,297 @@ static const char * const sev_status_feat_names[] = {
 	[MSR_AMD64_SNP_IBS_VIRT_BIT]		= "IBSVirt",
 	[MSR_AMD64_SNP_VMSA_REG_PROT_BIT]	= "VMSARegProt",
 	[MSR_AMD64_SNP_SMT_PROT_BIT]		= "SMTProt",
+	[MSR_AMD64_SNP_SECURE_AVIC_BIT]		= "SecureAVIC",
 };
 
-/* For early boot hypervisor communication in SEV-ES enabled guests */
-static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
-
 /*
- * Needs to be in the .data section because we need it NULL before bss is
- * cleared
+ * For Secure TSC guests, the BSP fetches TSC_INFO using SNP guest messaging and
+ * initializes snp_tsc_scale and snp_tsc_offset. These values are replicated
+ * across the APs VMSA fields (TSC_SCALE and TSC_OFFSET).
  */
-static struct ghcb *boot_ghcb __section(".data");
+static u64 snp_tsc_scale __ro_after_init;
+static u64 snp_tsc_offset __ro_after_init;
+static unsigned long snp_tsc_freq_khz __ro_after_init;
 
-/* Bitmap of SEV features supported by the hypervisor */
-static u64 sev_hv_features __ro_after_init;
-
-/* #VC handler runtime per-CPU data */
-struct sev_es_runtime_data {
-	struct ghcb ghcb_page;
-
-	/*
-	 * Reserve one page per CPU as backup storage for the unencrypted GHCB.
-	 * It is needed when an NMI happens while the #VC handler uses the real
-	 * GHCB, and the NMI handler itself is causing another #VC exception. In
-	 * that case the GHCB content of the first handler needs to be backed up
-	 * and restored.
-	 */
-	struct ghcb backup_ghcb;
-
-	/*
-	 * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions.
-	 * There is no need for it to be atomic, because nothing is written to
-	 * the GHCB between the read and the write of ghcb_active. So it is safe
-	 * to use it when a nested #VC exception happens before the write.
-	 *
-	 * This is necessary for example in the #VC->NMI->#VC case when the NMI
-	 * happens while the first #VC handler uses the GHCB. When the NMI code
-	 * raises a second #VC handler it might overwrite the contents of the
-	 * GHCB written by the first handler. To avoid this the content of the
-	 * GHCB is saved and restored when the GHCB is detected to be in use
-	 * already.
-	 */
-	bool ghcb_active;
-	bool backup_ghcb_active;
-
-	/*
-	 * Cached DR7 value - write it on DR7 writes and return it on reads.
-	 * That value will never make it to the real hardware DR7 as debugging
-	 * is currently unsupported in SEV-ES guests.
-	 */
-	unsigned long dr7;
-};
-
-struct ghcb_state {
-	struct ghcb *ghcb;
-};
-
-/* For early boot SVSM communication */
-static struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);
-
-static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
-static DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
-static DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
-static DEFINE_PER_CPU(u64, svsm_caa_pa);
-
-struct sev_config {
-	__u64 debug		: 1,
-
-	      /*
-	       * Indicates when the per-CPU GHCB has been created and registered
-	       * and thus can be used by the BSP instead of the early boot GHCB.
-	       *
-	       * For APs, the per-CPU GHCB is created before they are started
-	       * and registered upon startup, so this flag can be used globally
-	       * for the BSP and APs.
-	       */
-	      ghcbs_initialized	: 1,
-
-	      /*
-	       * Indicates when the per-CPU SVSM CA is to be used instead of the
-	       * boot SVSM CA.
-	       *
-	       * For APs, the per-CPU SVSM CA is created as part of the AP
-	       * bringup, so this flag can be used globally for the BSP and APs.
-	       */
-	      use_cas		: 1,
-
-	      __reserved	: 61;
-};
-
-static struct sev_config sev_cfg __read_mostly;
-
-static __always_inline bool on_vc_stack(struct pt_regs *regs)
-{
-	unsigned long sp = regs->sp;
-
-	/* User-mode RSP is not trusted */
-	if (user_mode(regs))
-		return false;
-
-	/* SYSCALL gap still has user-mode RSP */
-	if (ip_within_syscall_gap(regs))
-		return false;
-
-	return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC)));
-}
+DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
+DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
 
 /*
- * This function handles the case when an NMI is raised in the #VC
- * exception handler entry code, before the #VC handler has switched off
- * its IST stack. In this case, the IST entry for #VC must be adjusted,
- * so that any nested #VC exception will not overwrite the stack
- * contents of the interrupted #VC handler.
- *
- * The IST entry is adjusted unconditionally so that it can be also be
- * unconditionally adjusted back in __sev_es_ist_exit(). Otherwise a
- * nested sev_es_ist_exit() call may adjust back the IST entry too
- * early.
- *
- * The __sev_es_ist_enter() and __sev_es_ist_exit() functions always run
- * on the NMI IST stack, as they are only called from NMI handling code
- * right now.
+ * SVSM related information:
+ *   When running under an SVSM, the VMPL that Linux is executing at must be
+ *   non-zero. The VMPL is therefore used to indicate the presence of an SVSM.
  */
-void noinstr __sev_es_ist_enter(struct pt_regs *regs)
-{
-	unsigned long old_ist, new_ist;
-
-	/* Read old IST entry */
-	new_ist = old_ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]);
-
-	/*
-	 * If NMI happened while on the #VC IST stack, set the new IST
-	 * value below regs->sp, so that the interrupted stack frame is
-	 * not overwritten by subsequent #VC exceptions.
-	 */
-	if (on_vc_stack(regs))
-		new_ist = regs->sp;
-
-	/*
-	 * Reserve additional 8 bytes and store old IST value so this
-	 * adjustment can be unrolled in __sev_es_ist_exit().
-	 */
-	new_ist -= sizeof(old_ist);
-	*(unsigned long *)new_ist = old_ist;
-
-	/* Set new IST entry */
-	this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], new_ist);
-}
-
-void noinstr __sev_es_ist_exit(void)
-{
-	unsigned long ist;
-
-	/* Read IST entry */
-	ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]);
-
-	if (WARN_ON(ist == __this_cpu_ist_top_va(VC)))
-		return;
-
-	/* Read back old IST entry and write it to the TSS */
-	this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist);
-}
+u8 snp_vmpl __ro_after_init;
+EXPORT_SYMBOL_GPL(snp_vmpl);
+SYM_PIC_ALIAS(snp_vmpl);
 
 /*
- * Nothing shall interrupt this code path while holding the per-CPU
- * GHCB. The backup GHCB is only for NMIs interrupting this path.
+ * Since feature negotiation related variables are set early in the boot
+ * process they must reside in the .data section so as not to be zeroed
+ * out when the .bss section is later cleared.
  *
- * Callers must disable local interrupts around it.
+ * GHCB protocol version negotiated with the hypervisor.
  */
-static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
-{
-	struct sev_es_runtime_data *data;
-	struct ghcb *ghcb;
-
-	WARN_ON(!irqs_disabled());
+u16 ghcb_version __ro_after_init;
+SYM_PIC_ALIAS(ghcb_version);
 
-	data = this_cpu_read(runtime_data);
-	ghcb = &data->ghcb_page;
-
-	if (unlikely(data->ghcb_active)) {
-		/* GHCB is already in use - save its contents */
-
-		if (unlikely(data->backup_ghcb_active)) {
-			/*
-			 * Backup-GHCB is also already in use. There is no way
-			 * to continue here so just kill the machine. To make
-			 * panic() work, mark GHCBs inactive so that messages
-			 * can be printed out.
-			 */
-			data->ghcb_active        = false;
-			data->backup_ghcb_active = false;
-
-			instrumentation_begin();
-			panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use");
-			instrumentation_end();
-		}
+/* For early boot hypervisor communication in SEV-ES enabled guests */
+static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
 
-		/* Mark backup_ghcb active before writing to it */
-		data->backup_ghcb_active = true;
+/*
+ * Needs to be in the .data section because we need it NULL before bss is
+ * cleared
+ */
+struct ghcb *boot_ghcb __section(".data");
 
-		state->ghcb = &data->backup_ghcb;
+static u64 __init get_snp_jump_table_addr(void)
+{
+	struct snp_secrets_page *secrets;
+	void __iomem *mem;
+	u64 addr;
 
-		/* Backup GHCB content */
-		*state->ghcb = *ghcb;
-	} else {
-		state->ghcb = NULL;
-		data->ghcb_active = true;
+	mem = ioremap_encrypted(sev_secrets_pa, PAGE_SIZE);
+	if (!mem) {
+		pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n");
+		return 0;
 	}
 
-	return ghcb;
-}
+	secrets = (__force struct snp_secrets_page *)mem;
 
-static inline u64 sev_es_rd_ghcb_msr(void)
-{
-	return __rdmsr(MSR_AMD64_SEV_ES_GHCB);
+	addr = secrets->os_area.ap_jump_table_pa;
+	iounmap(mem);
+
+	return addr;
 }
 
-static __always_inline void sev_es_wr_ghcb_msr(u64 val)
+static u64 __init get_jump_table_addr(void)
 {
-	u32 low, high;
-
-	low  = (u32)(val);
-	high = (u32)(val >> 32);
+	struct ghcb_state state;
+	unsigned long flags;
+	struct ghcb *ghcb;
+	u64 ret = 0;
 
-	native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high);
-}
+	if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
+		return get_snp_jump_table_addr();
 
-static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,
-				unsigned char *buffer)
-{
-	return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
-}
+	local_irq_save(flags);
 
-static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt)
-{
-	char buffer[MAX_INSN_SIZE];
-	int insn_bytes;
+	ghcb = __sev_get_ghcb(&state);
 
-	insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer);
-	if (insn_bytes == 0) {
-		/* Nothing could be copied */
-		ctxt->fi.vector     = X86_TRAP_PF;
-		ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER;
-		ctxt->fi.cr2        = ctxt->regs->ip;
-		return ES_EXCEPTION;
-	} else if (insn_bytes == -EINVAL) {
-		/* Effective RIP could not be calculated */
-		ctxt->fi.vector     = X86_TRAP_GP;
-		ctxt->fi.error_code = 0;
-		ctxt->fi.cr2        = 0;
-		return ES_EXCEPTION;
-	}
+	vc_ghcb_invalidate(ghcb);
+	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE);
+	ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE);
+	ghcb_set_sw_exit_info_2(ghcb, 0);
 
-	if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes))
-		return ES_DECODE_FAILED;
+	sev_es_wr_ghcb_msr(__pa(ghcb));
+	VMGEXIT();
 
-	if (ctxt->insn.immediate.got)
-		return ES_OK;
-	else
-		return ES_DECODE_FAILED;
-}
+	if (ghcb_sw_exit_info_1_is_valid(ghcb) &&
+	    ghcb_sw_exit_info_2_is_valid(ghcb))
+		ret = ghcb->save.sw_exit_info_2;
 
-static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt)
-{
-	char buffer[MAX_INSN_SIZE];
-	int res, ret;
+	__sev_put_ghcb(&state);
 
-	res = vc_fetch_insn_kernel(ctxt, buffer);
-	if (res) {
-		ctxt->fi.vector     = X86_TRAP_PF;
-		ctxt->fi.error_code = X86_PF_INSTR;
-		ctxt->fi.cr2        = ctxt->regs->ip;
-		return ES_EXCEPTION;
-	}
+	local_irq_restore(flags);
 
-	ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
-	if (ret < 0)
-		return ES_DECODE_FAILED;
-	else
-		return ES_OK;
+	return ret;
 }
 
-static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
+static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call)
 {
-	if (user_mode(ctxt->regs))
-		return __vc_decode_user_insn(ctxt);
-	else
-		return __vc_decode_kern_insn(ctxt);
-}
+	struct es_em_ctxt ctxt;
+	u8 pending = 0;
 
-static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
-				   char *dst, char *buf, size_t size)
-{
-	unsigned long error_code = X86_PF_PROT | X86_PF_WRITE;
+	vc_ghcb_invalidate(ghcb);
 
 	/*
-	 * This function uses __put_user() independent of whether kernel or user
-	 * memory is accessed. This works fine because __put_user() does no
-	 * sanity checks of the pointer being accessed. All that it does is
-	 * to report when the access failed.
-	 *
-	 * Also, this function runs in atomic context, so __put_user() is not
-	 * allowed to sleep. The page-fault handler detects that it is running
-	 * in atomic context and will not try to take mmap_sem and handle the
-	 * fault, so additional pagefault_enable()/disable() calls are not
-	 * needed.
-	 *
-	 * The access can't be done via copy_to_user() here because
-	 * vc_write_mem() must not use string instructions to access unsafe
-	 * memory. The reason is that MOVS is emulated by the #VC handler by
-	 * splitting the move up into a read and a write and taking a nested #VC
-	 * exception on whatever of them is the MMIO access. Using string
-	 * instructions here would cause infinite nesting.
+	 * Fill in protocol and format specifiers. This can be called very early
+	 * in the boot, so use rip-relative references as needed.
 	 */
-	switch (size) {
-	case 1: {
-		u8 d1;
-		u8 __user *target = (u8 __user *)dst;
-
-		memcpy(&d1, buf, 1);
-		if (__put_user(d1, target))
-			goto fault;
-		break;
-	}
-	case 2: {
-		u16 d2;
-		u16 __user *target = (u16 __user *)dst;
+	ghcb->protocol_version = ghcb_version;
+	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
 
-		memcpy(&d2, buf, 2);
-		if (__put_user(d2, target))
-			goto fault;
-		break;
-	}
-	case 4: {
-		u32 d4;
-		u32 __user *target = (u32 __user *)dst;
+	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_SNP_RUN_VMPL);
+	ghcb_set_sw_exit_info_1(ghcb, 0);
+	ghcb_set_sw_exit_info_2(ghcb, 0);
 
-		memcpy(&d4, buf, 4);
-		if (__put_user(d4, target))
-			goto fault;
-		break;
-	}
-	case 8: {
-		u64 d8;
-		u64 __user *target = (u64 __user *)dst;
+	sev_es_wr_ghcb_msr(__pa(ghcb));
+
+	svsm_issue_call(call, &pending);
+
+	if (pending)
+		return -EINVAL;
 
-		memcpy(&d8, buf, 8);
-		if (__put_user(d8, target))
-			goto fault;
+	switch (verify_exception_info(ghcb, &ctxt)) {
+	case ES_OK:
 		break;
-	}
+	case ES_EXCEPTION:
+		vc_forward_exception(&ctxt);
+		fallthrough;
 	default:
-		WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
-		return ES_UNSUPPORTED;
+		return -EINVAL;
 	}
 
-	return ES_OK;
-
-fault:
-	if (user_mode(ctxt->regs))
-		error_code |= X86_PF_USER;
-
-	ctxt->fi.vector = X86_TRAP_PF;
-	ctxt->fi.error_code = error_code;
-	ctxt->fi.cr2 = (unsigned long)dst;
-
-	return ES_EXCEPTION;
+	return svsm_process_result_codes(call);
 }
 
-static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
-				  char *src, char *buf, size_t size)
+static int svsm_perform_call_protocol(struct svsm_call *call)
 {
-	unsigned long error_code = X86_PF_PROT;
-
-	/*
-	 * This function uses __get_user() independent of whether kernel or user
-	 * memory is accessed. This works fine because __get_user() does no
-	 * sanity checks of the pointer being accessed. All that it does is
-	 * to report when the access failed.
-	 *
-	 * Also, this function runs in atomic context, so __get_user() is not
-	 * allowed to sleep. The page-fault handler detects that it is running
-	 * in atomic context and will not try to take mmap_sem and handle the
-	 * fault, so additional pagefault_enable()/disable() calls are not
-	 * needed.
-	 *
-	 * The access can't be done via copy_from_user() here because
-	 * vc_read_mem() must not use string instructions to access unsafe
-	 * memory. The reason is that MOVS is emulated by the #VC handler by
-	 * splitting the move up into a read and a write and taking a nested #VC
-	 * exception on whatever of them is the MMIO access. Using string
-	 * instructions here would cause infinite nesting.
-	 */
-	switch (size) {
-	case 1: {
-		u8 d1;
-		u8 __user *s = (u8 __user *)src;
-
-		if (__get_user(d1, s))
-			goto fault;
-		memcpy(buf, &d1, 1);
-		break;
-	}
-	case 2: {
-		u16 d2;
-		u16 __user *s = (u16 __user *)src;
+	struct ghcb_state state;
+	unsigned long flags;
+	struct ghcb *ghcb;
+	int ret;
 
-		if (__get_user(d2, s))
-			goto fault;
-		memcpy(buf, &d2, 2);
-		break;
-	}
-	case 4: {
-		u32 d4;
-		u32 __user *s = (u32 __user *)src;
+	flags = native_local_irq_save();
 
-		if (__get_user(d4, s))
-			goto fault;
-		memcpy(buf, &d4, 4);
-		break;
-	}
-	case 8: {
-		u64 d8;
-		u64 __user *s = (u64 __user *)src;
-		if (__get_user(d8, s))
-			goto fault;
-		memcpy(buf, &d8, 8);
-		break;
-	}
-	default:
-		WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
-		return ES_UNSUPPORTED;
-	}
+	if (sev_cfg.ghcbs_initialized)
+		ghcb = __sev_get_ghcb(&state);
+	else if (boot_ghcb)
+		ghcb = boot_ghcb;
+	else
+		ghcb = NULL;
 
-	return ES_OK;
+	do {
+		ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
+			   : __pi_svsm_perform_msr_protocol(call);
+	} while (ret == -EAGAIN);
 
-fault:
-	if (user_mode(ctxt->regs))
-		error_code |= X86_PF_USER;
+	if (sev_cfg.ghcbs_initialized)
+		__sev_put_ghcb(&state);
 
-	ctxt->fi.vector = X86_TRAP_PF;
-	ctxt->fi.error_code = error_code;
-	ctxt->fi.cr2 = (unsigned long)src;
+	native_local_irq_restore(flags);
 
-	return ES_EXCEPTION;
+	return ret;
 }
 
-static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
-					   unsigned long vaddr, phys_addr_t *paddr)
+static inline void __pval_terminate(u64 pfn, bool action, unsigned int page_size,
+				    int ret, u64 svsm_ret)
 {
-	unsigned long va = (unsigned long)vaddr;
-	unsigned int level;
-	phys_addr_t pa;
-	pgd_t *pgd;
-	pte_t *pte;
+	WARN(1, "PVALIDATE failure: pfn: 0x%llx, action: %u, size: %u, ret: %d, svsm_ret: 0x%llx\n",
+	     pfn, action, page_size, ret, svsm_ret);
 
-	pgd = __va(read_cr3_pa());
-	pgd = &pgd[pgd_index(va)];
-	pte = lookup_address_in_pgd(pgd, va, &level);
-	if (!pte) {
-		ctxt->fi.vector     = X86_TRAP_PF;
-		ctxt->fi.cr2        = vaddr;
-		ctxt->fi.error_code = 0;
-
-		if (user_mode(ctxt->regs))
-			ctxt->fi.error_code |= X86_PF_USER;
-
-		return ES_EXCEPTION;
-	}
-
-	if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC))
-		/* Emulated MMIO to/from encrypted memory not supported */
-		return ES_UNSUPPORTED;
+	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
+}
 
-	pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
-	pa |= va & ~page_level_mask(level);
+static void svsm_pval_terminate(struct svsm_pvalidate_call *pc, int ret, u64 svsm_ret)
+{
+	unsigned int page_size;
+	bool action;
+	u64 pfn;
 
-	*paddr = pa;
+	pfn = pc->entry[pc->cur_index].pfn;
+	action = pc->entry[pc->cur_index].action;
+	page_size = pc->entry[pc->cur_index].page_size;
 
-	return ES_OK;
+	__pval_terminate(pfn, action, page_size, ret, svsm_ret);
 }
 
-static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
+static void pval_pages(struct snp_psc_desc *desc)
 {
-	BUG_ON(size > 4);
+	struct psc_entry *e;
+	unsigned long vaddr;
+	unsigned int size;
+	unsigned int i;
+	bool validate;
+	u64 pfn;
+	int rc;
 
-	if (user_mode(ctxt->regs)) {
-		struct thread_struct *t = &current->thread;
-		struct io_bitmap *iobm = t->io_bitmap;
-		size_t idx;
+	for (i = 0; i <= desc->hdr.end_entry; i++) {
+		e = &desc->entries[i];
 
-		if (!iobm)
-			goto fault;
+		pfn = e->gfn;
+		vaddr = (unsigned long)pfn_to_kaddr(pfn);
+		size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
+		validate = e->operation == SNP_PAGE_STATE_PRIVATE;
 
-		for (idx = port; idx < port + size; ++idx) {
-			if (test_bit(idx, iobm->bitmap))
-				goto fault;
+		rc = pvalidate(vaddr, size, validate);
+		if (!rc)
+			continue;
+
+		if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) {
+			unsigned long vaddr_end = vaddr + PMD_SIZE;
+
+			for (; vaddr < vaddr_end; vaddr += PAGE_SIZE, pfn++) {
+				rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
+				if (rc)
+					__pval_terminate(pfn, validate, RMP_PG_SIZE_4K, rc, 0);
+			}
+		} else {
+			__pval_terminate(pfn, validate, size, rc, 0);
 		}
 	}
+}
 
-	return ES_OK;
+static u64 svsm_build_ca_from_pfn_range(u64 pfn, u64 pfn_end, bool action,
+					struct svsm_pvalidate_call *pc)
+{
+	struct svsm_pvalidate_entry *pe;
 
-fault:
-	ctxt->fi.vector = X86_TRAP_GP;
-	ctxt->fi.error_code = 0;
+	/* Nothing in the CA yet */
+	pc->num_entries = 0;
+	pc->cur_index   = 0;
 
-	return ES_EXCEPTION;
-}
+	pe = &pc->entry[0];
 
-static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt)
-{
-	long error_code = ctxt->fi.error_code;
-	int trapnr = ctxt->fi.vector;
+	while (pfn < pfn_end) {
+		pe->page_size = RMP_PG_SIZE_4K;
+		pe->action    = action;
+		pe->ignore_cf = 0;
+		pe->rsvd      = 0;
+		pe->pfn       = pfn;
 
-	ctxt->regs->orig_ax = ctxt->fi.error_code;
+		pe++;
+		pfn++;
 
-	switch (trapnr) {
-	case X86_TRAP_GP:
-		exc_general_protection(ctxt->regs, error_code);
-		break;
-	case X86_TRAP_UD:
-		exc_invalid_op(ctxt->regs);
-		break;
-	case X86_TRAP_PF:
-		write_cr2(ctxt->fi.cr2);
-		exc_page_fault(ctxt->regs, error_code);
-		break;
-	case X86_TRAP_AC:
-		exc_alignment_check(ctxt->regs, error_code);
-		break;
-	default:
-		pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n");
-		BUG();
+		pc->num_entries++;
+		if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
+			break;
 	}
-}
-
-/* Include code shared with pre-decompression boot stage */
-#include "shared.c"
 
-static inline struct svsm_ca *svsm_get_caa(void)
-{
-	/*
-	 * Use rIP-relative references when called early in the boot. If
-	 * ->use_cas is set, then it is late in the boot and no need
-	 * to worry about rIP-relative references.
-	 */
-	if (RIP_REL_REF(sev_cfg).use_cas)
-		return this_cpu_read(svsm_caa);
-	else
-		return RIP_REL_REF(boot_svsm_caa);
+	return pfn;
 }
 
-static u64 svsm_get_caa_pa(void)
+static int svsm_build_ca_from_psc_desc(struct snp_psc_desc *desc, unsigned int desc_entry,
+				       struct svsm_pvalidate_call *pc)
 {
-	/*
-	 * Use rIP-relative references when called early in the boot. If
-	 * ->use_cas is set, then it is late in the boot and no need
-	 * to worry about rIP-relative references.
-	 */
-	if (RIP_REL_REF(sev_cfg).use_cas)
-		return this_cpu_read(svsm_caa_pa);
-	else
-		return RIP_REL_REF(boot_svsm_caa_pa);
-}
+	struct svsm_pvalidate_entry *pe;
+	struct psc_entry *e;
 
-static noinstr void __sev_put_ghcb(struct ghcb_state *state)
-{
-	struct sev_es_runtime_data *data;
-	struct ghcb *ghcb;
+	/* Nothing in the CA yet */
+	pc->num_entries = 0;
+	pc->cur_index   = 0;
 
-	WARN_ON(!irqs_disabled());
+	pe = &pc->entry[0];
+	e  = &desc->entries[desc_entry];
 
-	data = this_cpu_read(runtime_data);
-	ghcb = &data->ghcb_page;
+	while (desc_entry <= desc->hdr.end_entry) {
+		pe->page_size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
+		pe->action    = e->operation == SNP_PAGE_STATE_PRIVATE;
+		pe->ignore_cf = 0;
+		pe->rsvd      = 0;
+		pe->pfn       = e->gfn;
 
-	if (state->ghcb) {
-		/* Restore GHCB from Backup */
-		*ghcb = *state->ghcb;
-		data->backup_ghcb_active = false;
-		state->ghcb = NULL;
-	} else {
-		/*
-		 * Invalidate the GHCB so a VMGEXIT instruction issued
-		 * from userspace won't appear to be valid.
-		 */
-		vc_ghcb_invalidate(ghcb);
-		data->ghcb_active = false;
+		pe++;
+		e++;
+
+		desc_entry++;
+		pc->num_entries++;
+		if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
+			break;
 	}
+
+	return desc_entry;
 }
 
-static int svsm_perform_call_protocol(struct svsm_call *call)
+static void svsm_pval_pages(struct snp_psc_desc *desc)
 {
-	struct ghcb_state state;
+	struct svsm_pvalidate_entry pv_4k[VMGEXIT_PSC_MAX_ENTRY];
+	unsigned int i, pv_4k_count = 0;
+	struct svsm_pvalidate_call *pc;
+	struct svsm_call call = {};
 	unsigned long flags;
-	struct ghcb *ghcb;
+	bool action;
+	u64 pc_pa;
 	int ret;
 
 	/*
@@ -680,217 +421,166 @@ static int svsm_perform_call_protocol(struct svsm_call *call)
 	flags = native_local_irq_save();
 
 	/*
-	 * Use rip-relative references when called early in the boot. If
-	 * ghcbs_initialized is set, then it is late in the boot and no need
-	 * to worry about rip-relative references in called functions.
+	 * The SVSM calling area (CA) can support processing 510 entries at a
+	 * time. Loop through the Page State Change descriptor until the CA is
+	 * full or the last entry in the descriptor is reached, at which time
+	 * the SVSM is invoked. This repeats until all entries in the descriptor
+	 * are processed.
 	 */
-	if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
-		ghcb = __sev_get_ghcb(&state);
-	else if (RIP_REL_REF(boot_ghcb))
-		ghcb = RIP_REL_REF(boot_ghcb);
-	else
-		ghcb = NULL;
+	call.caa = svsm_get_caa();
 
-	do {
-		ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
-			   : svsm_perform_msr_protocol(call);
-	} while (ret == -EAGAIN);
+	pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer;
+	pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer);
 
-	if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
-		__sev_put_ghcb(&state);
+	/* Protocol 0, Call ID 1 */
+	call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE);
+	call.rcx = pc_pa;
 
-	native_local_irq_restore(flags);
+	for (i = 0; i <= desc->hdr.end_entry;) {
+		i = svsm_build_ca_from_psc_desc(desc, i, pc);
 
-	return ret;
-}
+		do {
+			ret = svsm_perform_call_protocol(&call);
+			if (!ret)
+				continue;
 
-void noinstr __sev_es_nmi_complete(void)
-{
-	struct ghcb_state state;
-	struct ghcb *ghcb;
+			/*
+			 * Check if the entry failed because of an RMP mismatch (a
+			 * PVALIDATE at 2M was requested, but the page is mapped in
+			 * the RMP as 4K).
+			 */
 
-	ghcb = __sev_get_ghcb(&state);
+			if (call.rax_out == SVSM_PVALIDATE_FAIL_SIZEMISMATCH &&
+			    pc->entry[pc->cur_index].page_size == RMP_PG_SIZE_2M) {
+				/* Save this entry for post-processing at 4K */
+				pv_4k[pv_4k_count++] = pc->entry[pc->cur_index];
 
-	vc_ghcb_invalidate(ghcb);
-	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);
-	ghcb_set_sw_exit_info_1(ghcb, 0);
-	ghcb_set_sw_exit_info_2(ghcb, 0);
+				/* Skip to the next one unless at the end of the list */
+				pc->cur_index++;
+				if (pc->cur_index < pc->num_entries)
+					ret = -EAGAIN;
+				else
+					ret = 0;
+			}
+		} while (ret == -EAGAIN);
 
-	sev_es_wr_ghcb_msr(__pa_nodebug(ghcb));
-	VMGEXIT();
+		if (ret)
+			svsm_pval_terminate(pc, ret, call.rax_out);
+	}
 
-	__sev_put_ghcb(&state);
-}
+	/* Process any entries that failed to be validated at 2M and validate them at 4K */
+	for (i = 0; i < pv_4k_count; i++) {
+		u64 pfn, pfn_end;
 
-static u64 __init get_secrets_page(void)
-{
-	u64 pa_data = boot_params.cc_blob_address;
-	struct cc_blob_sev_info info;
-	void *map;
+		action  = pv_4k[i].action;
+		pfn     = pv_4k[i].pfn;
+		pfn_end = pfn + 512;
 
-	/*
-	 * The CC blob contains the address of the secrets page, check if the
-	 * blob is present.
-	 */
-	if (!pa_data)
-		return 0;
+		while (pfn < pfn_end) {
+			pfn = svsm_build_ca_from_pfn_range(pfn, pfn_end, action, pc);
 
-	map = early_memremap(pa_data, sizeof(info));
-	if (!map) {
-		pr_err("Unable to locate SNP secrets page: failed to map the Confidential Computing blob.\n");
-		return 0;
+			ret = svsm_perform_call_protocol(&call);
+			if (ret)
+				svsm_pval_terminate(pc, ret, call.rax_out);
+		}
 	}
-	memcpy(&info, map, sizeof(info));
-	early_memunmap(map, sizeof(info));
 
-	/* smoke-test the secrets page passed */
-	if (!info.secrets_phys || info.secrets_len != PAGE_SIZE)
-		return 0;
-
-	return info.secrets_phys;
+	native_local_irq_restore(flags);
 }
 
-static u64 __init get_snp_jump_table_addr(void)
+static void pvalidate_pages(struct snp_psc_desc *desc)
 {
-	struct snp_secrets_page *secrets;
-	void __iomem *mem;
-	u64 pa, addr;
-
-	pa = get_secrets_page();
-	if (!pa)
-		return 0;
+	struct psc_entry *e;
+	unsigned int i;
 
-	mem = ioremap_encrypted(pa, PAGE_SIZE);
-	if (!mem) {
-		pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n");
-		return 0;
-	}
+	if (snp_vmpl)
+		svsm_pval_pages(desc);
+	else
+		pval_pages(desc);
 
-	secrets = (__force struct snp_secrets_page *)mem;
+	/*
+	 * If not affected by the cache-coherency vulnerability there is no need
+	 * to perform the cache eviction mitigation.
+	 */
+	if (cpu_feature_enabled(X86_FEATURE_COHERENCY_SFW_NO))
+		return;
 
-	addr = secrets->os_area.ap_jump_table_pa;
-	iounmap(mem);
+	for (i = 0; i <= desc->hdr.end_entry; i++) {
+		e = &desc->entries[i];
 
-	return addr;
+		/*
+		 * If validating memory (making it private) perform the cache
+		 * eviction mitigation.
+		 */
+		if (e->operation == SNP_PAGE_STATE_PRIVATE)
+			sev_evict_cache(pfn_to_kaddr(e->gfn), e->pagesize ? 512 : 1);
+	}
 }
 
-static u64 __init get_jump_table_addr(void)
+static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
 {
-	struct ghcb_state state;
-	unsigned long flags;
-	struct ghcb *ghcb;
-	u64 ret = 0;
-
-	if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
-		return get_snp_jump_table_addr();
-
-	local_irq_save(flags);
-
-	ghcb = __sev_get_ghcb(&state);
+	int cur_entry, end_entry, ret = 0;
+	struct snp_psc_desc *data;
+	struct es_em_ctxt ctxt;
 
 	vc_ghcb_invalidate(ghcb);
-	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE);
-	ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE);
-	ghcb_set_sw_exit_info_2(ghcb, 0);
 
-	sev_es_wr_ghcb_msr(__pa(ghcb));
-	VMGEXIT();
+	/* Copy the input desc into GHCB shared buffer */
+	data = (struct snp_psc_desc *)ghcb->shared_buffer;
+	memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
 
-	if (ghcb_sw_exit_info_1_is_valid(ghcb) &&
-	    ghcb_sw_exit_info_2_is_valid(ghcb))
-		ret = ghcb->save.sw_exit_info_2;
-
-	__sev_put_ghcb(&state);
-
-	local_irq_restore(flags);
-
-	return ret;
-}
-
-static void __head
-early_set_pages_state(unsigned long vaddr, unsigned long paddr,
-		      unsigned long npages, enum psc_op op)
-{
-	unsigned long paddr_end;
-	u64 val;
-
-	vaddr = vaddr & PAGE_MASK;
+	/*
+	 * As per the GHCB specification, the hypervisor can resume the guest
+	 * before processing all the entries. Check whether all the entries
+	 * are processed. If not, then keep retrying. Note, the hypervisor
+	 * will update the data memory directly to indicate the status, so
+	 * reference the data->hdr everywhere.
+	 *
+	 * The strategy here is to wait for the hypervisor to change the page
+	 * state in the RMP table before guest accesses the memory pages. If the
+	 * page state change was not successful, then later memory access will
+	 * result in a crash.
+	 */
+	cur_entry = data->hdr.cur_entry;
+	end_entry = data->hdr.end_entry;
 
-	paddr = paddr & PAGE_MASK;
-	paddr_end = paddr + (npages << PAGE_SHIFT);
+	while (data->hdr.cur_entry <= data->hdr.end_entry) {
+		ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
 
-	while (paddr < paddr_end) {
-		/* Page validation must be rescinded before changing to shared */
-		if (op == SNP_PAGE_STATE_SHARED)
-			pvalidate_4k_page(vaddr, paddr, false);
+		/* This will advance the shared buffer data points to. */
+		ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
 
 		/*
-		 * Use the MSR protocol because this function can be called before
-		 * the GHCB is established.
+		 * Page State Change VMGEXIT can pass error code through
+		 * exit_info_2.
 		 */
-		sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
-		VMGEXIT();
-
-		val = sev_es_rd_ghcb_msr();
-
-		if (WARN(GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP,
-			 "Wrong PSC response code: 0x%x\n",
-			 (unsigned int)GHCB_RESP_CODE(val)))
-			goto e_term;
-
-		if (WARN(GHCB_MSR_PSC_RESP_VAL(val),
-			 "Failed to change page state to '%s' paddr 0x%lx error 0x%llx\n",
-			 op == SNP_PAGE_STATE_PRIVATE ? "private" : "shared",
-			 paddr, GHCB_MSR_PSC_RESP_VAL(val)))
-			goto e_term;
+		if (WARN(ret || ghcb->save.sw_exit_info_2,
+			 "SNP: PSC failed ret=%d exit_info_2=%llx\n",
+			 ret, ghcb->save.sw_exit_info_2)) {
+			ret = 1;
+			goto out;
+		}
 
-		/* Page validation must be performed after changing to private */
-		if (op == SNP_PAGE_STATE_PRIVATE)
-			pvalidate_4k_page(vaddr, paddr, true);
+		/* Verify that reserved bit is not set */
+		if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
+			ret = 1;
+			goto out;
+		}
 
-		vaddr += PAGE_SIZE;
-		paddr += PAGE_SIZE;
+		/*
+		 * Sanity check that entry processing is not going backwards.
+		 * This will happen only if hypervisor is tricking us.
+		 */
+		if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry,
+"SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
+			 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
+			ret = 1;
+			goto out;
+		}
 	}
 
-	return;
-
-e_term:
-	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
-}
-
-void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr,
-					 unsigned long npages)
-{
-	/*
-	 * This can be invoked in early boot while running identity mapped, so
-	 * use an open coded check for SNP instead of using cc_platform_has().
-	 * This eliminates worries about jump tables or checking boot_cpu_data
-	 * in the cc_platform_has() function.
-	 */
-	if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
-		return;
-
-	 /*
-	  * Ask the hypervisor to mark the memory pages as private in the RMP
-	  * table.
-	  */
-	early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_PRIVATE);
-}
-
-void __init early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr,
-					unsigned long npages)
-{
-	/*
-	 * This can be invoked in early boot while running identity mapped, so
-	 * use an open coded check for SNP instead of using cc_platform_has().
-	 * This eliminates worries about jump tables or checking boot_cpu_data
-	 * in the cc_platform_has() function.
-	 */
-	if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
-		return;
-
-	 /* Ask hypervisor to mark the memory pages shared in the RMP table. */
-	early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED);
+out:
+	return ret;
 }
 
 static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
@@ -971,8 +661,11 @@ static void set_pages_state(unsigned long vaddr, unsigned long npages, int op)
 	unsigned long vaddr_end;
 
 	/* Use the MSR protocol when a GHCB is not available. */
-	if (!boot_ghcb)
-		return early_set_pages_state(vaddr, __pa(vaddr), npages, op);
+	if (!boot_ghcb) {
+		struct psc_desc d = { op, svsm_get_caa(), svsm_get_caa_pa() };
+
+		return early_set_pages_state(vaddr, __pa(vaddr), npages, &d);
+	}
 
 	vaddr = vaddr & PAGE_MASK;
 	vaddr_end = vaddr + (npages << PAGE_SHIFT);
@@ -1010,6 +703,46 @@ void snp_accept_memory(phys_addr_t start, phys_addr_t end)
 	set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE);
 }
 
+static int vmgexit_ap_control(u64 event, struct sev_es_save_area *vmsa, u32 apic_id)
+{
+	bool create = event != SVM_VMGEXIT_AP_DESTROY;
+	struct ghcb_state state;
+	unsigned long flags;
+	struct ghcb *ghcb;
+	int ret = 0;
+
+	local_irq_save(flags);
+
+	ghcb = __sev_get_ghcb(&state);
+
+	vc_ghcb_invalidate(ghcb);
+
+	if (create)
+		ghcb_set_rax(ghcb, vmsa->sev_features);
+
+	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_CREATION);
+	ghcb_set_sw_exit_info_1(ghcb,
+				((u64)apic_id << 32)	|
+				((u64)snp_vmpl << 16)	|
+				event);
+	ghcb_set_sw_exit_info_2(ghcb, __pa(vmsa));
+
+	sev_es_wr_ghcb_msr(__pa(ghcb));
+	VMGEXIT();
+
+	if (!ghcb_sw_exit_info_1_is_valid(ghcb) ||
+	    lower_32_bits(ghcb->save.sw_exit_info_1)) {
+		pr_err("SNP AP %s error\n", (create ? "CREATE" : "DESTROY"));
+		ret = -EINVAL;
+	}
+
+	__sev_put_ghcb(&state);
+
+	local_irq_restore(flags);
+
+	return ret;
+}
+
 static int snp_set_vmsa(void *va, void *caa, int apic_id, bool make_vmsa)
 {
 	int ret;
@@ -1055,6 +788,212 @@ static int snp_set_vmsa(void *va, void *caa, int apic_id, bool make_vmsa)
 	return ret;
 }
 
+static void snp_cleanup_vmsa(struct sev_es_save_area *vmsa, int apic_id)
+{
+	int err;
+
+	err = snp_set_vmsa(vmsa, NULL, apic_id, false);
+	if (err)
+		pr_err("clear VMSA page failed (%u), leaking page\n", err);
+	else
+		free_page((unsigned long)vmsa);
+}
+
+static void set_pte_enc(pte_t *kpte, int level, void *va)
+{
+	struct pte_enc_desc d = {
+		.kpte	   = kpte,
+		.pte_level = level,
+		.va	   = va,
+		.encrypt   = true
+	};
+
+	prepare_pte_enc(&d);
+	set_pte_enc_mask(kpte, d.pfn, d.new_pgprot);
+}
+
+static void unshare_all_memory(void)
+{
+	unsigned long addr, end, size, ghcb;
+	struct sev_es_runtime_data *data;
+	unsigned int npages, level;
+	bool skipped_addr;
+	pte_t *pte;
+	int cpu;
+
+	/* Unshare the direct mapping. */
+	addr = PAGE_OFFSET;
+	end  = PAGE_OFFSET + get_max_mapped();
+
+	while (addr < end) {
+		pte = lookup_address(addr, &level);
+		size = page_level_size(level);
+		npages = size / PAGE_SIZE;
+		skipped_addr = false;
+
+		if (!pte || !pte_decrypted(*pte) || pte_none(*pte)) {
+			addr += size;
+			continue;
+		}
+
+		/*
+		 * Ensure that all the per-CPU GHCBs are made private at the
+		 * end of the unsharing loop so that the switch to the slower
+		 * MSR protocol happens last.
+		 */
+		for_each_possible_cpu(cpu) {
+			data = per_cpu(runtime_data, cpu);
+			ghcb = (unsigned long)&data->ghcb_page;
+
+			/* Handle the case of a huge page containing the GHCB page */
+			if (addr <= ghcb && ghcb < addr + size) {
+				skipped_addr = true;
+				break;
+			}
+		}
+
+		if (!skipped_addr) {
+			set_pte_enc(pte, level, (void *)addr);
+			snp_set_memory_private(addr, npages);
+		}
+		addr += size;
+	}
+
+	/* Unshare all bss decrypted memory. */
+	addr = (unsigned long)__start_bss_decrypted;
+	end  = (unsigned long)__start_bss_decrypted_unused;
+	npages = (end - addr) >> PAGE_SHIFT;
+
+	for (; addr < end; addr += PAGE_SIZE) {
+		pte = lookup_address(addr, &level);
+		if (!pte || !pte_decrypted(*pte) || pte_none(*pte))
+			continue;
+
+		set_pte_enc(pte, level, (void *)addr);
+	}
+	addr = (unsigned long)__start_bss_decrypted;
+	snp_set_memory_private(addr, npages);
+
+	__flush_tlb_all();
+}
+
+/* Stop new private<->shared conversions */
+void snp_kexec_begin(void)
+{
+	if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
+		return;
+
+	if (!IS_ENABLED(CONFIG_KEXEC_CORE))
+		return;
+
+	/*
+	 * Crash kernel ends up here with interrupts disabled: can't wait for
+	 * conversions to finish.
+	 *
+	 * If race happened, just report and proceed.
+	 */
+	if (!set_memory_enc_stop_conversion())
+		pr_warn("Failed to stop shared<->private conversions\n");
+}
+
+/*
+ * Shutdown all APs except the one handling kexec/kdump and clearing
+ * the VMSA tag on AP's VMSA pages as they are not being used as
+ * VMSA page anymore.
+ */
+static void shutdown_all_aps(void)
+{
+	struct sev_es_save_area *vmsa;
+	int apic_id, this_cpu, cpu;
+
+	this_cpu = get_cpu();
+
+	/*
+	 * APs are already in HLT loop when enc_kexec_finish() callback
+	 * is invoked.
+	 */
+	for_each_present_cpu(cpu) {
+		vmsa = per_cpu(sev_vmsa, cpu);
+
+		/*
+		 * The BSP or offlined APs do not have guest allocated VMSA
+		 * and there is no need  to clear the VMSA tag for this page.
+		 */
+		if (!vmsa)
+			continue;
+
+		/*
+		 * Cannot clear the VMSA tag for the currently running vCPU.
+		 */
+		if (this_cpu == cpu) {
+			unsigned long pa;
+			struct page *p;
+
+			pa = __pa(vmsa);
+			/*
+			 * Mark the VMSA page of the running vCPU as offline
+			 * so that is excluded and not touched by makedumpfile
+			 * while generating vmcore during kdump.
+			 */
+			p = pfn_to_online_page(pa >> PAGE_SHIFT);
+			if (p)
+				__SetPageOffline(p);
+			continue;
+		}
+
+		apic_id = cpuid_to_apicid[cpu];
+
+		/*
+		 * Issue AP destroy to ensure AP gets kicked out of guest mode
+		 * to allow using RMPADJUST to remove the VMSA tag on it's
+		 * VMSA page.
+		 */
+		vmgexit_ap_control(SVM_VMGEXIT_AP_DESTROY, vmsa, apic_id);
+		snp_cleanup_vmsa(vmsa, apic_id);
+	}
+
+	put_cpu();
+}
+
+void snp_kexec_finish(void)
+{
+	struct sev_es_runtime_data *data;
+	unsigned long size, addr;
+	unsigned int level, cpu;
+	struct ghcb *ghcb;
+	pte_t *pte;
+
+	if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
+		return;
+
+	if (!IS_ENABLED(CONFIG_KEXEC_CORE))
+		return;
+
+	shutdown_all_aps();
+
+	unshare_all_memory();
+
+	/*
+	 * Switch to using the MSR protocol to change per-CPU GHCBs to
+	 * private. All the per-CPU GHCBs have been switched back to private,
+	 * so can't do any more GHCB calls to the hypervisor beyond this point
+	 * until the kexec'ed kernel starts running.
+	 */
+	boot_ghcb = NULL;
+	sev_cfg.ghcbs_initialized = false;
+
+	for_each_possible_cpu(cpu) {
+		data = per_cpu(runtime_data, cpu);
+		ghcb = &data->ghcb_page;
+		pte = lookup_address((unsigned long)ghcb, &level);
+		size = page_level_size(level);
+		/* Handle the case of a huge page containing the GHCB page */
+		addr = (unsigned long)ghcb & page_level_mask(level);
+		set_pte_enc(pte, level, (void *)addr);
+		snp_set_memory_private(addr, (size / PAGE_SIZE));
+	}
+}
+
 #define __ATTR_BASE		(SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK)
 #define INIT_CS_ATTRIBS		(__ATTR_BASE | SVM_SELECTOR_READ_MASK | SVM_SELECTOR_CODE_MASK)
 #define INIT_DS_ATTRIBS		(__ATTR_BASE | SVM_SELECTOR_WRITE_MASK)
@@ -1086,26 +1025,12 @@ static void *snp_alloc_vmsa_page(int cpu)
 	return page_address(p + 1);
 }
 
-static void snp_cleanup_vmsa(struct sev_es_save_area *vmsa, int apic_id)
-{
-	int err;
-
-	err = snp_set_vmsa(vmsa, NULL, apic_id, false);
-	if (err)
-		pr_err("clear VMSA page failed (%u), leaking page\n", err);
-	else
-		free_page((unsigned long)vmsa);
-}
-
-static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip)
+static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip, unsigned int cpu)
 {
 	struct sev_es_save_area *cur_vmsa, *vmsa;
-	struct ghcb_state state;
 	struct svsm_ca *caa;
-	unsigned long flags;
-	struct ghcb *ghcb;
 	u8 sipi_vector;
-	int cpu, ret;
+	int ret;
 	u64 cr4;
 
 	/*
@@ -1126,15 +1051,6 @@ static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip)
 
 	/* Override start_ip with known protected guest start IP */
 	start_ip = real_mode_header->sev_es_trampoline_start;
-
-	/* Find the logical CPU for the APIC ID */
-	for_each_present_cpu(cpu) {
-		if (arch_match_cpu_phys_id(cpu, apic_id))
-			break;
-	}
-	if (cpu >= nr_cpu_ids)
-		return -EINVAL;
-
 	cur_vmsa = per_cpu(sev_vmsa, cpu);
 
 	/*
@@ -1190,6 +1106,9 @@ static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip)
 	vmsa->x87_ftw		= AP_INIT_X87_FTW_DEFAULT;
 	vmsa->x87_fcw		= AP_INIT_X87_FCW_DEFAULT;
 
+	if (cc_platform_has(CC_ATTR_SNP_SECURE_AVIC))
+		vmsa->vintr_ctrl |= V_GIF_MASK | V_NMI_ENABLE_MASK;
+
 	/* SVME must be set. */
 	vmsa->efer		= EFER_SVME;
 
@@ -1201,6 +1120,12 @@ static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip)
 	vmsa->vmpl		= snp_vmpl;
 	vmsa->sev_features	= sev_status >> 2;
 
+	/* Populate AP's TSC scale/offset to get accurate TSC values. */
+	if (cc_platform_has(CC_ATTR_GUEST_SNP_SECURE_TSC)) {
+		vmsa->tsc_scale = snp_tsc_scale;
+		vmsa->tsc_offset = snp_tsc_offset;
+	}
+
 	/* Switch the page over to a VMSA page now that it is initialized */
 	ret = snp_set_vmsa(vmsa, caa, apic_id, true);
 	if (ret) {
@@ -1211,33 +1136,7 @@ static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip)
 	}
 
 	/* Issue VMGEXIT AP Creation NAE event */
-	local_irq_save(flags);
-
-	ghcb = __sev_get_ghcb(&state);
-
-	vc_ghcb_invalidate(ghcb);
-	ghcb_set_rax(ghcb, vmsa->sev_features);
-	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_CREATION);
-	ghcb_set_sw_exit_info_1(ghcb,
-				((u64)apic_id << 32)	|
-				((u64)snp_vmpl << 16)	|
-				SVM_VMGEXIT_AP_CREATE);
-	ghcb_set_sw_exit_info_2(ghcb, __pa(vmsa));
-
-	sev_es_wr_ghcb_msr(__pa(ghcb));
-	VMGEXIT();
-
-	if (!ghcb_sw_exit_info_1_is_valid(ghcb) ||
-	    lower_32_bits(ghcb->save.sw_exit_info_1)) {
-		pr_err("SNP AP Creation error\n");
-		ret = -EINVAL;
-	}
-
-	__sev_put_ghcb(&state);
-
-	local_irq_restore(flags);
-
-	/* Perform cleanup if there was an error */
+	ret = vmgexit_ap_control(SVM_VMGEXIT_AP_CREATE, vmsa, apic_id);
 	if (ret) {
 		snp_cleanup_vmsa(vmsa, apic_id);
 		vmsa = NULL;
@@ -1305,11 +1204,13 @@ int __init sev_es_setup_ap_jump_table(struct real_mode_header *rmh)
  * This is needed by the OVMF UEFI firmware which will use whatever it finds in
  * the GHCB MSR as its GHCB to talk to the hypervisor. So make sure the per-cpu
  * runtime GHCBs used by the kernel are also mapped in the EFI page-table.
+ *
+ * When running under SVSM the CA page is needed too, so map it as well.
  */
-int __init sev_es_efi_map_ghcbs(pgd_t *pgd)
+int __init sev_es_efi_map_ghcbs_cas(pgd_t *pgd)
 {
+	unsigned long address, pflags, pflags_enc;
 	struct sev_es_runtime_data *data;
-	unsigned long address, pflags;
 	int cpu;
 	u64 pfn;
 
@@ -1317,6 +1218,7 @@ int __init sev_es_efi_map_ghcbs(pgd_t *pgd)
 		return 0;
 
 	pflags = _PAGE_NX | _PAGE_RW;
+	pflags_enc = cc_mkenc(pflags);
 
 	for_each_possible_cpu(cpu) {
 		data = per_cpu(runtime_data, cpu);
@@ -1326,45 +1228,118 @@ int __init sev_es_efi_map_ghcbs(pgd_t *pgd)
 
 		if (kernel_map_pages_in_pgd(pgd, pfn, address, 1, pflags))
 			return 1;
+
+		if (snp_vmpl) {
+			address = per_cpu(svsm_caa_pa, cpu);
+			if (!address)
+				return 1;
+
+			pfn = address >> PAGE_SHIFT;
+			if (kernel_map_pages_in_pgd(pgd, pfn, address, 1, pflags_enc))
+				return 1;
+		}
 	}
 
 	return 0;
 }
 
-static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+u64 savic_ghcb_msr_read(u32 reg)
 {
-	struct pt_regs *regs = ctxt->regs;
-	enum es_result ret;
-	u64 exit_info_1;
-
-	/* Is it a WRMSR? */
-	exit_info_1 = (ctxt->insn.opcode.bytes[1] == 0x30) ? 1 : 0;
+	u64 msr = APIC_BASE_MSR + (reg >> 4);
+	struct pt_regs regs = { .cx = msr };
+	struct es_em_ctxt ctxt = { .regs = &regs };
+	struct ghcb_state state;
+	enum es_result res;
+	struct ghcb *ghcb;
 
-	if (regs->cx == MSR_SVSM_CAA) {
-		/* Writes to the SVSM CAA msr are ignored */
-		if (exit_info_1)
-			return ES_OK;
+	guard(irqsave)();
 
-		regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa));
-		regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa));
+	ghcb = __sev_get_ghcb(&state);
+	vc_ghcb_invalidate(ghcb);
 
-		return ES_OK;
+	res = sev_es_ghcb_handle_msr(ghcb, &ctxt, false);
+	if (res != ES_OK) {
+		pr_err("Secure AVIC MSR (0x%llx) read returned error (%d)\n", msr, res);
+		/* MSR read failures are treated as fatal errors */
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL);
 	}
 
-	ghcb_set_rcx(ghcb, regs->cx);
-	if (exit_info_1) {
-		ghcb_set_rax(ghcb, regs->ax);
-		ghcb_set_rdx(ghcb, regs->dx);
-	}
+	__sev_put_ghcb(&state);
 
-	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, exit_info_1, 0);
+	return regs.ax | regs.dx << 32;
+}
 
-	if ((ret == ES_OK) && (!exit_info_1)) {
-		regs->ax = ghcb->save.rax;
-		regs->dx = ghcb->save.rdx;
+void savic_ghcb_msr_write(u32 reg, u64 value)
+{
+	u64 msr = APIC_BASE_MSR + (reg >> 4);
+	struct pt_regs regs = {
+		.cx = msr,
+		.ax = lower_32_bits(value),
+		.dx = upper_32_bits(value)
+	};
+	struct es_em_ctxt ctxt = { .regs = &regs };
+	struct ghcb_state state;
+	enum es_result res;
+	struct ghcb *ghcb;
+
+	guard(irqsave)();
+
+	ghcb = __sev_get_ghcb(&state);
+	vc_ghcb_invalidate(ghcb);
+
+	res = sev_es_ghcb_handle_msr(ghcb, &ctxt, true);
+	if (res != ES_OK) {
+		pr_err("Secure AVIC MSR (0x%llx) write returned error (%d)\n", msr, res);
+		/* MSR writes should never fail. Any failure is fatal error for SNP guest */
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL);
 	}
 
-	return ret;
+	__sev_put_ghcb(&state);
+}
+
+enum es_result savic_register_gpa(u64 gpa)
+{
+	struct ghcb_state state;
+	struct es_em_ctxt ctxt;
+	enum es_result res;
+	struct ghcb *ghcb;
+
+	guard(irqsave)();
+
+	ghcb = __sev_get_ghcb(&state);
+	vc_ghcb_invalidate(ghcb);
+
+	ghcb_set_rax(ghcb, SVM_VMGEXIT_SAVIC_SELF_GPA);
+	ghcb_set_rbx(ghcb, gpa);
+	res = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_SAVIC,
+				  SVM_VMGEXIT_SAVIC_REGISTER_GPA, 0);
+
+	__sev_put_ghcb(&state);
+
+	return res;
+}
+
+enum es_result savic_unregister_gpa(u64 *gpa)
+{
+	struct ghcb_state state;
+	struct es_em_ctxt ctxt;
+	enum es_result res;
+	struct ghcb *ghcb;
+
+	guard(irqsave)();
+
+	ghcb = __sev_get_ghcb(&state);
+	vc_ghcb_invalidate(ghcb);
+
+	ghcb_set_rax(ghcb, SVM_VMGEXIT_SAVIC_SELF_GPA);
+	res = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_SAVIC,
+				  SVM_VMGEXIT_SAVIC_UNREGISTER_GPA, 0);
+	if (gpa && res == ES_OK)
+		*gpa = ghcb->save.rbx;
+
+	__sev_put_ghcb(&state);
+
+	return res;
 }
 
 static void snp_register_per_cpu_ghcb(void)
@@ -1493,9 +1468,8 @@ static void __init alloc_runtime_data(int cpu)
 		struct svsm_ca *caa;
 
 		/* Allocate the SVSM CA page if an SVSM is present */
-		caa = memblock_alloc(sizeof(*caa), PAGE_SIZE);
-		if (!caa)
-			panic("Can't allocate SVSM CA page\n");
+		caa = cpu ? memblock_alloc_or_panic(sizeof(*caa), PAGE_SIZE)
+			  : &boot_svsm_ca_page;
 
 		per_cpu(svsm_caa, cpu) = caa;
 		per_cpu(svsm_caa_pa, cpu) = __pa(caa);
@@ -1549,775 +1523,15 @@ void __init sev_es_init_vc_handling(void)
 		init_ghcb(cpu);
 	}
 
-	/* If running under an SVSM, switch to the per-cpu CA */
-	if (snp_vmpl) {
-		struct svsm_call call = {};
-		unsigned long flags;
-		int ret;
-
-		local_irq_save(flags);
-
-		/*
-		 * SVSM_CORE_REMAP_CA call:
-		 *   RAX = 0 (Protocol=0, CallID=0)
-		 *   RCX = New CA GPA
-		 */
-		call.caa = svsm_get_caa();
-		call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA);
-		call.rcx = this_cpu_read(svsm_caa_pa);
-		ret = svsm_perform_call_protocol(&call);
-		if (ret)
-			panic("Can't remap the SVSM CA, ret=%d, rax_out=0x%llx\n",
-			      ret, call.rax_out);
-
+	if (snp_vmpl)
 		sev_cfg.use_cas = true;
 
-		local_irq_restore(flags);
-	}
-
 	sev_es_setup_play_dead();
 
 	/* Secondary CPUs use the runtime #VC handler */
 	initial_vc_handler = (unsigned long)kernel_exc_vmm_communication;
 }
 
-static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt)
-{
-	int trapnr = ctxt->fi.vector;
-
-	if (trapnr == X86_TRAP_PF)
-		native_write_cr2(ctxt->fi.cr2);
-
-	ctxt->regs->orig_ax = ctxt->fi.error_code;
-	do_early_exception(ctxt->regs, trapnr);
-}
-
-static long *vc_insn_get_rm(struct es_em_ctxt *ctxt)
-{
-	long *reg_array;
-	int offset;
-
-	reg_array = (long *)ctxt->regs;
-	offset    = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs);
-
-	if (offset < 0)
-		return NULL;
-
-	offset /= sizeof(long);
-
-	return reg_array + offset;
-}
-static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
-				 unsigned int bytes, bool read)
-{
-	u64 exit_code, exit_info_1, exit_info_2;
-	unsigned long ghcb_pa = __pa(ghcb);
-	enum es_result res;
-	phys_addr_t paddr;
-	void __user *ref;
-
-	ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs);
-	if (ref == (void __user *)-1L)
-		return ES_UNSUPPORTED;
-
-	exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE;
-
-	res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr);
-	if (res != ES_OK) {
-		if (res == ES_EXCEPTION && !read)
-			ctxt->fi.error_code |= X86_PF_WRITE;
-
-		return res;
-	}
-
-	exit_info_1 = paddr;
-	/* Can never be greater than 8 */
-	exit_info_2 = bytes;
-
-	ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer));
-
-	return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2);
-}
-
-/*
- * The MOVS instruction has two memory operands, which raises the
- * problem that it is not known whether the access to the source or the
- * destination caused the #VC exception (and hence whether an MMIO read
- * or write operation needs to be emulated).
- *
- * Instead of playing games with walking page-tables and trying to guess
- * whether the source or destination is an MMIO range, split the move
- * into two operations, a read and a write with only one memory operand.
- * This will cause a nested #VC exception on the MMIO address which can
- * then be handled.
- *
- * This implementation has the benefit that it also supports MOVS where
- * source _and_ destination are MMIO regions.
- *
- * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a
- * rare operation. If it turns out to be a performance problem the split
- * operations can be moved to memcpy_fromio() and memcpy_toio().
- */
-static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt,
-					  unsigned int bytes)
-{
-	unsigned long ds_base, es_base;
-	unsigned char *src, *dst;
-	unsigned char buffer[8];
-	enum es_result ret;
-	bool rep;
-	int off;
-
-	ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS);
-	es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
-
-	if (ds_base == -1L || es_base == -1L) {
-		ctxt->fi.vector = X86_TRAP_GP;
-		ctxt->fi.error_code = 0;
-		return ES_EXCEPTION;
-	}
-
-	src = ds_base + (unsigned char *)ctxt->regs->si;
-	dst = es_base + (unsigned char *)ctxt->regs->di;
-
-	ret = vc_read_mem(ctxt, src, buffer, bytes);
-	if (ret != ES_OK)
-		return ret;
-
-	ret = vc_write_mem(ctxt, dst, buffer, bytes);
-	if (ret != ES_OK)
-		return ret;
-
-	if (ctxt->regs->flags & X86_EFLAGS_DF)
-		off = -bytes;
-	else
-		off =  bytes;
-
-	ctxt->regs->si += off;
-	ctxt->regs->di += off;
-
-	rep = insn_has_rep_prefix(&ctxt->insn);
-	if (rep)
-		ctxt->regs->cx -= 1;
-
-	if (!rep || ctxt->regs->cx == 0)
-		return ES_OK;
-	else
-		return ES_RETRY;
-}
-
-static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
-{
-	struct insn *insn = &ctxt->insn;
-	enum insn_mmio_type mmio;
-	unsigned int bytes = 0;
-	enum es_result ret;
-	u8 sign_byte;
-	long *reg_data;
-
-	mmio = insn_decode_mmio(insn, &bytes);
-	if (mmio == INSN_MMIO_DECODE_FAILED)
-		return ES_DECODE_FAILED;
-
-	if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) {
-		reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs);
-		if (!reg_data)
-			return ES_DECODE_FAILED;
-	}
-
-	if (user_mode(ctxt->regs))
-		return ES_UNSUPPORTED;
-
-	switch (mmio) {
-	case INSN_MMIO_WRITE:
-		memcpy(ghcb->shared_buffer, reg_data, bytes);
-		ret = vc_do_mmio(ghcb, ctxt, bytes, false);
-		break;
-	case INSN_MMIO_WRITE_IMM:
-		memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes);
-		ret = vc_do_mmio(ghcb, ctxt, bytes, false);
-		break;
-	case INSN_MMIO_READ:
-		ret = vc_do_mmio(ghcb, ctxt, bytes, true);
-		if (ret)
-			break;
-
-		/* Zero-extend for 32-bit operation */
-		if (bytes == 4)
-			*reg_data = 0;
-
-		memcpy(reg_data, ghcb->shared_buffer, bytes);
-		break;
-	case INSN_MMIO_READ_ZERO_EXTEND:
-		ret = vc_do_mmio(ghcb, ctxt, bytes, true);
-		if (ret)
-			break;
-
-		/* Zero extend based on operand size */
-		memset(reg_data, 0, insn->opnd_bytes);
-		memcpy(reg_data, ghcb->shared_buffer, bytes);
-		break;
-	case INSN_MMIO_READ_SIGN_EXTEND:
-		ret = vc_do_mmio(ghcb, ctxt, bytes, true);
-		if (ret)
-			break;
-
-		if (bytes == 1) {
-			u8 *val = (u8 *)ghcb->shared_buffer;
-
-			sign_byte = (*val & 0x80) ? 0xff : 0x00;
-		} else {
-			u16 *val = (u16 *)ghcb->shared_buffer;
-
-			sign_byte = (*val & 0x8000) ? 0xff : 0x00;
-		}
-
-		/* Sign extend based on operand size */
-		memset(reg_data, sign_byte, insn->opnd_bytes);
-		memcpy(reg_data, ghcb->shared_buffer, bytes);
-		break;
-	case INSN_MMIO_MOVS:
-		ret = vc_handle_mmio_movs(ctxt, bytes);
-		break;
-	default:
-		ret = ES_UNSUPPORTED;
-		break;
-	}
-
-	return ret;
-}
-
-static enum es_result vc_handle_dr7_write(struct ghcb *ghcb,
-					  struct es_em_ctxt *ctxt)
-{
-	struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
-	long val, *reg = vc_insn_get_rm(ctxt);
-	enum es_result ret;
-
-	if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
-		return ES_VMM_ERROR;
-
-	if (!reg)
-		return ES_DECODE_FAILED;
-
-	val = *reg;
-
-	/* Upper 32 bits must be written as zeroes */
-	if (val >> 32) {
-		ctxt->fi.vector = X86_TRAP_GP;
-		ctxt->fi.error_code = 0;
-		return ES_EXCEPTION;
-	}
-
-	/* Clear out other reserved bits and set bit 10 */
-	val = (val & 0xffff23ffL) | BIT(10);
-
-	/* Early non-zero writes to DR7 are not supported */
-	if (!data && (val & ~DR7_RESET_VALUE))
-		return ES_UNSUPPORTED;
-
-	/* Using a value of 0 for ExitInfo1 means RAX holds the value */
-	ghcb_set_rax(ghcb, val);
-	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0);
-	if (ret != ES_OK)
-		return ret;
-
-	if (data)
-		data->dr7 = val;
-
-	return ES_OK;
-}
-
-static enum es_result vc_handle_dr7_read(struct ghcb *ghcb,
-					 struct es_em_ctxt *ctxt)
-{
-	struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
-	long *reg = vc_insn_get_rm(ctxt);
-
-	if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
-		return ES_VMM_ERROR;
-
-	if (!reg)
-		return ES_DECODE_FAILED;
-
-	if (data)
-		*reg = data->dr7;
-	else
-		*reg = DR7_RESET_VALUE;
-
-	return ES_OK;
-}
-
-static enum es_result vc_handle_wbinvd(struct ghcb *ghcb,
-				       struct es_em_ctxt *ctxt)
-{
-	return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);
-}
-
-static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
-{
-	enum es_result ret;
-
-	ghcb_set_rcx(ghcb, ctxt->regs->cx);
-
-	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0);
-	if (ret != ES_OK)
-		return ret;
-
-	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb)))
-		return ES_VMM_ERROR;
-
-	ctxt->regs->ax = ghcb->save.rax;
-	ctxt->regs->dx = ghcb->save.rdx;
-
-	return ES_OK;
-}
-
-static enum es_result vc_handle_monitor(struct ghcb *ghcb,
-					struct es_em_ctxt *ctxt)
-{
-	/*
-	 * Treat it as a NOP and do not leak a physical address to the
-	 * hypervisor.
-	 */
-	return ES_OK;
-}
-
-static enum es_result vc_handle_mwait(struct ghcb *ghcb,
-				      struct es_em_ctxt *ctxt)
-{
-	/* Treat the same as MONITOR/MONITORX */
-	return ES_OK;
-}
-
-static enum es_result vc_handle_vmmcall(struct ghcb *ghcb,
-					struct es_em_ctxt *ctxt)
-{
-	enum es_result ret;
-
-	ghcb_set_rax(ghcb, ctxt->regs->ax);
-	ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0);
-
-	if (x86_platform.hyper.sev_es_hcall_prepare)
-		x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs);
-
-	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0);
-	if (ret != ES_OK)
-		return ret;
-
-	if (!ghcb_rax_is_valid(ghcb))
-		return ES_VMM_ERROR;
-
-	ctxt->regs->ax = ghcb->save.rax;
-
-	/*
-	 * Call sev_es_hcall_finish() after regs->ax is already set.
-	 * This allows the hypervisor handler to overwrite it again if
-	 * necessary.
-	 */
-	if (x86_platform.hyper.sev_es_hcall_finish &&
-	    !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs))
-		return ES_VMM_ERROR;
-
-	return ES_OK;
-}
-
-static enum es_result vc_handle_trap_ac(struct ghcb *ghcb,
-					struct es_em_ctxt *ctxt)
-{
-	/*
-	 * Calling ecx_alignment_check() directly does not work, because it
-	 * enables IRQs and the GHCB is active. Forward the exception and call
-	 * it later from vc_forward_exception().
-	 */
-	ctxt->fi.vector = X86_TRAP_AC;
-	ctxt->fi.error_code = 0;
-	return ES_EXCEPTION;
-}
-
-static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
-					 struct ghcb *ghcb,
-					 unsigned long exit_code)
-{
-	enum es_result result = vc_check_opcode_bytes(ctxt, exit_code);
-
-	if (result != ES_OK)
-		return result;
-
-	switch (exit_code) {
-	case SVM_EXIT_READ_DR7:
-		result = vc_handle_dr7_read(ghcb, ctxt);
-		break;
-	case SVM_EXIT_WRITE_DR7:
-		result = vc_handle_dr7_write(ghcb, ctxt);
-		break;
-	case SVM_EXIT_EXCP_BASE + X86_TRAP_AC:
-		result = vc_handle_trap_ac(ghcb, ctxt);
-		break;
-	case SVM_EXIT_RDTSC:
-	case SVM_EXIT_RDTSCP:
-		result = vc_handle_rdtsc(ghcb, ctxt, exit_code);
-		break;
-	case SVM_EXIT_RDPMC:
-		result = vc_handle_rdpmc(ghcb, ctxt);
-		break;
-	case SVM_EXIT_INVD:
-		pr_err_ratelimited("#VC exception for INVD??? Seriously???\n");
-		result = ES_UNSUPPORTED;
-		break;
-	case SVM_EXIT_CPUID:
-		result = vc_handle_cpuid(ghcb, ctxt);
-		break;
-	case SVM_EXIT_IOIO:
-		result = vc_handle_ioio(ghcb, ctxt);
-		break;
-	case SVM_EXIT_MSR:
-		result = vc_handle_msr(ghcb, ctxt);
-		break;
-	case SVM_EXIT_VMMCALL:
-		result = vc_handle_vmmcall(ghcb, ctxt);
-		break;
-	case SVM_EXIT_WBINVD:
-		result = vc_handle_wbinvd(ghcb, ctxt);
-		break;
-	case SVM_EXIT_MONITOR:
-		result = vc_handle_monitor(ghcb, ctxt);
-		break;
-	case SVM_EXIT_MWAIT:
-		result = vc_handle_mwait(ghcb, ctxt);
-		break;
-	case SVM_EXIT_NPF:
-		result = vc_handle_mmio(ghcb, ctxt);
-		break;
-	default:
-		/*
-		 * Unexpected #VC exception
-		 */
-		result = ES_UNSUPPORTED;
-	}
-
-	return result;
-}
-
-static __always_inline bool is_vc2_stack(unsigned long sp)
-{
-	return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2));
-}
-
-static __always_inline bool vc_from_invalid_context(struct pt_regs *regs)
-{
-	unsigned long sp, prev_sp;
-
-	sp      = (unsigned long)regs;
-	prev_sp = regs->sp;
-
-	/*
-	 * If the code was already executing on the VC2 stack when the #VC
-	 * happened, let it proceed to the normal handling routine. This way the
-	 * code executing on the VC2 stack can cause #VC exceptions to get handled.
-	 */
-	return is_vc2_stack(sp) && !is_vc2_stack(prev_sp);
-}
-
-static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code)
-{
-	struct ghcb_state state;
-	struct es_em_ctxt ctxt;
-	enum es_result result;
-	struct ghcb *ghcb;
-	bool ret = true;
-
-	ghcb = __sev_get_ghcb(&state);
-
-	vc_ghcb_invalidate(ghcb);
-	result = vc_init_em_ctxt(&ctxt, regs, error_code);
-
-	if (result == ES_OK)
-		result = vc_handle_exitcode(&ctxt, ghcb, error_code);
-
-	__sev_put_ghcb(&state);
-
-	/* Done - now check the result */
-	switch (result) {
-	case ES_OK:
-		vc_finish_insn(&ctxt);
-		break;
-	case ES_UNSUPPORTED:
-		pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n",
-				   error_code, regs->ip);
-		ret = false;
-		break;
-	case ES_VMM_ERROR:
-		pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
-				   error_code, regs->ip);
-		ret = false;
-		break;
-	case ES_DECODE_FAILED:
-		pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
-				   error_code, regs->ip);
-		ret = false;
-		break;
-	case ES_EXCEPTION:
-		vc_forward_exception(&ctxt);
-		break;
-	case ES_RETRY:
-		/* Nothing to do */
-		break;
-	default:
-		pr_emerg("Unknown result in %s():%d\n", __func__, result);
-		/*
-		 * Emulating the instruction which caused the #VC exception
-		 * failed - can't continue so print debug information
-		 */
-		BUG();
-	}
-
-	return ret;
-}
-
-static __always_inline bool vc_is_db(unsigned long error_code)
-{
-	return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB;
-}
-
-/*
- * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode
- * and will panic when an error happens.
- */
-DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)
-{
-	irqentry_state_t irq_state;
-
-	/*
-	 * With the current implementation it is always possible to switch to a
-	 * safe stack because #VC exceptions only happen at known places, like
-	 * intercepted instructions or accesses to MMIO areas/IO ports. They can
-	 * also happen with code instrumentation when the hypervisor intercepts
-	 * #DB, but the critical paths are forbidden to be instrumented, so #DB
-	 * exceptions currently also only happen in safe places.
-	 *
-	 * But keep this here in case the noinstr annotations are violated due
-	 * to bug elsewhere.
-	 */
-	if (unlikely(vc_from_invalid_context(regs))) {
-		instrumentation_begin();
-		panic("Can't handle #VC exception from unsupported context\n");
-		instrumentation_end();
-	}
-
-	/*
-	 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
-	 */
-	if (vc_is_db(error_code)) {
-		exc_debug(regs);
-		return;
-	}
-
-	irq_state = irqentry_nmi_enter(regs);
-
-	instrumentation_begin();
-
-	if (!vc_raw_handle_exception(regs, error_code)) {
-		/* Show some debug info */
-		show_regs(regs);
-
-		/* Ask hypervisor to sev_es_terminate */
-		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
-
-		/* If that fails and we get here - just panic */
-		panic("Returned from Terminate-Request to Hypervisor\n");
-	}
-
-	instrumentation_end();
-	irqentry_nmi_exit(regs, irq_state);
-}
-
-/*
- * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode
- * and will kill the current task with SIGBUS when an error happens.
- */
-DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)
-{
-	/*
-	 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
-	 */
-	if (vc_is_db(error_code)) {
-		noist_exc_debug(regs);
-		return;
-	}
-
-	irqentry_enter_from_user_mode(regs);
-	instrumentation_begin();
-
-	if (!vc_raw_handle_exception(regs, error_code)) {
-		/*
-		 * Do not kill the machine if user-space triggered the
-		 * exception. Send SIGBUS instead and let user-space deal with
-		 * it.
-		 */
-		force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0);
-	}
-
-	instrumentation_end();
-	irqentry_exit_to_user_mode(regs);
-}
-
-bool __init handle_vc_boot_ghcb(struct pt_regs *regs)
-{
-	unsigned long exit_code = regs->orig_ax;
-	struct es_em_ctxt ctxt;
-	enum es_result result;
-
-	vc_ghcb_invalidate(boot_ghcb);
-
-	result = vc_init_em_ctxt(&ctxt, regs, exit_code);
-	if (result == ES_OK)
-		result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code);
-
-	/* Done - now check the result */
-	switch (result) {
-	case ES_OK:
-		vc_finish_insn(&ctxt);
-		break;
-	case ES_UNSUPPORTED:
-		early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n",
-				exit_code, regs->ip);
-		goto fail;
-	case ES_VMM_ERROR:
-		early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
-				exit_code, regs->ip);
-		goto fail;
-	case ES_DECODE_FAILED:
-		early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
-				exit_code, regs->ip);
-		goto fail;
-	case ES_EXCEPTION:
-		vc_early_forward_exception(&ctxt);
-		break;
-	case ES_RETRY:
-		/* Nothing to do */
-		break;
-	default:
-		BUG();
-	}
-
-	return true;
-
-fail:
-	show_regs(regs);
-
-	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
-}
-
-/*
- * Initial set up of SNP relies on information provided by the
- * Confidential Computing blob, which can be passed to the kernel
- * in the following ways, depending on how it is booted:
- *
- * - when booted via the boot/decompress kernel:
- *   - via boot_params
- *
- * - when booted directly by firmware/bootloader (e.g. CONFIG_PVH):
- *   - via a setup_data entry, as defined by the Linux Boot Protocol
- *
- * Scan for the blob in that order.
- */
-static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
-{
-	struct cc_blob_sev_info *cc_info;
-
-	/* Boot kernel would have passed the CC blob via boot_params. */
-	if (bp->cc_blob_address) {
-		cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address;
-		goto found_cc_info;
-	}
-
-	/*
-	 * If kernel was booted directly, without the use of the
-	 * boot/decompression kernel, the CC blob may have been passed via
-	 * setup_data instead.
-	 */
-	cc_info = find_cc_blob_setup_data(bp);
-	if (!cc_info)
-		return NULL;
-
-found_cc_info:
-	if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
-		snp_abort();
-
-	return cc_info;
-}
-
-static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
-{
-	struct svsm_call call = {};
-	int ret;
-	u64 pa;
-
-	/*
-	 * Record the SVSM Calling Area address (CAA) if the guest is not
-	 * running at VMPL0. The CA will be used to communicate with the
-	 * SVSM to perform the SVSM services.
-	 */
-	if (!svsm_setup_ca(cc_info))
-		return;
-
-	/*
-	 * It is very early in the boot and the kernel is running identity
-	 * mapped but without having adjusted the pagetables to where the
-	 * kernel was loaded (physbase), so the get the CA address using
-	 * RIP-relative addressing.
-	 */
-	pa = (u64)&RIP_REL_REF(boot_svsm_ca_page);
-
-	/*
-	 * Switch over to the boot SVSM CA while the current CA is still
-	 * addressable. There is no GHCB at this point so use the MSR protocol.
-	 *
-	 * SVSM_CORE_REMAP_CA call:
-	 *   RAX = 0 (Protocol=0, CallID=0)
-	 *   RCX = New CA GPA
-	 */
-	call.caa = svsm_get_caa();
-	call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA);
-	call.rcx = pa;
-	ret = svsm_perform_call_protocol(&call);
-	if (ret)
-		panic("Can't remap the SVSM CA, ret=%d, rax_out=0x%llx\n", ret, call.rax_out);
-
-	RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa;
-	RIP_REL_REF(boot_svsm_caa_pa) = pa;
-}
-
-bool __head snp_init(struct boot_params *bp)
-{
-	struct cc_blob_sev_info *cc_info;
-
-	if (!bp)
-		return false;
-
-	cc_info = find_cc_blob(bp);
-	if (!cc_info)
-		return false;
-
-	setup_cpuid_table(cc_info);
-
-	svsm_setup(cc_info);
-
-	/*
-	 * The CC blob will be used later to access the secrets page. Cache
-	 * it here like the boot kernel does.
-	 */
-	bp->cc_blob_address = (u32)(unsigned long)cc_info;
-
-	return true;
-}
-
-void __head __noreturn snp_abort(void)
-{
-	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
-}
-
 /*
  * SEV-SNP guests should only execute dmi_setup() if EFI_CONFIG_TABLES are
  * enabled, as the alternative (fallback) logic for DMI probing in the legacy
@@ -2374,23 +1588,6 @@ static int __init report_snp_info(void)
 }
 arch_initcall(report_snp_info);
 
-static int __init init_sev_config(char *str)
-{
-	char *s;
-
-	while ((s = strsep(&str, ","))) {
-		if (!strcmp(s, "debug")) {
-			sev_cfg.debug = true;
-			continue;
-		}
-
-		pr_info("SEV command-line option '%s' was not recognized\n", s);
-	}
-
-	return 1;
-}
-__setup("sev=", init_sev_config);
-
 static void update_attest_input(struct svsm_call *call, struct svsm_attest_call *input)
 {
 	/* If (new) lengths have been returned, propagate them up */
@@ -2441,15 +1638,16 @@ int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call,
 }
 EXPORT_SYMBOL_GPL(snp_issue_svsm_attest_req);
 
-int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct snp_guest_request_ioctl *rio)
+static int snp_issue_guest_request(struct snp_guest_req *req)
 {
+	struct snp_req_data *input = &req->input;
 	struct ghcb_state state;
 	struct es_em_ctxt ctxt;
 	unsigned long flags;
 	struct ghcb *ghcb;
 	int ret;
 
-	rio->exitinfo2 = SEV_RET_NO_FW_CALL;
+	req->exitinfo2 = SEV_RET_NO_FW_CALL;
 
 	/*
 	 * __sev_get_ghcb() needs to run with IRQs disabled because it is using
@@ -2465,17 +1663,17 @@ int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct sn
 
 	vc_ghcb_invalidate(ghcb);
 
-	if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) {
+	if (req->exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) {
 		ghcb_set_rax(ghcb, input->data_gpa);
 		ghcb_set_rbx(ghcb, input->data_npages);
 	}
 
-	ret = sev_es_ghcb_hv_call(ghcb, &ctxt, exit_code, input->req_gpa, input->resp_gpa);
+	ret = sev_es_ghcb_hv_call(ghcb, &ctxt, req->exit_code, input->req_gpa, input->resp_gpa);
 	if (ret)
 		goto e_put;
 
-	rio->exitinfo2 = ghcb->save.sw_exit_info_2;
-	switch (rio->exitinfo2) {
+	req->exitinfo2 = ghcb->save.sw_exit_info_2;
+	switch (req->exitinfo2) {
 	case 0:
 		break;
 
@@ -2485,7 +1683,7 @@ int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct sn
 
 	case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN):
 		/* Number of expected pages are returned in RBX */
-		if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) {
+		if (req->exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) {
 			input->data_npages = ghcb_get_rbx(ghcb);
 			ret = -ENOSPC;
 			break;
@@ -2503,33 +1701,88 @@ e_restore_irq:
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(snp_issue_guest_request);
+
+/**
+ * snp_svsm_vtpm_probe() - Probe if SVSM provides a vTPM device
+ *
+ * Check that there is SVSM and that it supports at least TPM_SEND_COMMAND
+ * which is the only request used so far.
+ *
+ * Return: true if the platform provides a vTPM SVSM device, false otherwise.
+ */
+static bool snp_svsm_vtpm_probe(void)
+{
+	struct svsm_call call = {};
+
+	/* The vTPM device is available only if a SVSM is present */
+	if (!snp_vmpl)
+		return false;
+
+	call.caa = svsm_get_caa();
+	call.rax = SVSM_VTPM_CALL(SVSM_VTPM_QUERY);
+
+	if (svsm_perform_call_protocol(&call))
+		return false;
+
+	/* Check platform commands contains TPM_SEND_COMMAND - platform command 8 */
+	return call.rcx_out & BIT_ULL(8);
+}
+
+/**
+ * snp_svsm_vtpm_send_command() - Execute a vTPM operation on SVSM
+ * @buffer: A buffer used to both send the command and receive the response.
+ *
+ * Execute a SVSM_VTPM_CMD call as defined by
+ * "Secure VM Service Module for SEV-SNP Guests" Publication # 58019 Revision: 1.00
+ *
+ * All command request/response buffers have a common structure as specified by
+ * the following table:
+ *     Byte      Size       In/Out    Description
+ *     Offset    (Bytes)
+ *     0x000     4          In        Platform command
+ *                          Out       Platform command response size
+ *
+ * Each command can build upon this common request/response structure to create
+ * a structure specific to the command. See include/linux/tpm_svsm.h for more
+ * details.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+int snp_svsm_vtpm_send_command(u8 *buffer)
+{
+	struct svsm_call call = {};
+
+	call.caa = svsm_get_caa();
+	call.rax = SVSM_VTPM_CALL(SVSM_VTPM_CMD);
+	call.rcx = __pa(buffer);
+
+	return svsm_perform_call_protocol(&call);
+}
+EXPORT_SYMBOL_GPL(snp_svsm_vtpm_send_command);
 
 static struct platform_device sev_guest_device = {
 	.name		= "sev-guest",
 	.id		= -1,
 };
 
+static struct platform_device tpm_svsm_device = {
+	.name		= "tpm-svsm",
+	.id		= -1,
+};
+
 static int __init snp_init_platform_device(void)
 {
-	struct sev_guest_platform_data data;
-	u64 gpa;
-
 	if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
 		return -ENODEV;
 
-	gpa = get_secrets_page();
-	if (!gpa)
-		return -ENODEV;
-
-	data.secrets_gpa = gpa;
-	if (platform_device_add_data(&sev_guest_device, &data, sizeof(data)))
+	if (platform_device_register(&sev_guest_device))
 		return -ENODEV;
 
-	if (platform_device_register(&sev_guest_device))
+	if (snp_svsm_vtpm_probe() &&
+	    platform_device_register(&tpm_svsm_device))
 		return -ENODEV;
 
-	pr_info("SNP guest platform device initialized.\n");
+	pr_info("SNP guest platform devices initialized.\n");
 	return 0;
 }
 device_initcall(snp_init_platform_device);
@@ -2550,15 +1803,6 @@ void sev_show_status(void)
 	pr_cont("\n");
 }
 
-void __init snp_update_svsm_ca(void)
-{
-	if (!snp_vmpl)
-		return;
-
-	/* Update the CAA to a proper kernel address */
-	boot_svsm_caa = &boot_svsm_ca_page;
-}
-
 #ifdef CONFIG_SYSFS
 static ssize_t vmpl_show(struct kobject *kobj,
 			 struct kobj_attribute *attr, char *buf)
@@ -2604,3 +1848,584 @@ static int __init sev_sysfs_init(void)
 }
 arch_initcall(sev_sysfs_init);
 #endif // CONFIG_SYSFS
+
+static void free_shared_pages(void *buf, size_t sz)
+{
+	unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
+	int ret;
+
+	if (!buf)
+		return;
+
+	ret = set_memory_encrypted((unsigned long)buf, npages);
+	if (ret) {
+		WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n");
+		return;
+	}
+
+	__free_pages(virt_to_page(buf), get_order(sz));
+}
+
+static void *alloc_shared_pages(size_t sz)
+{
+	unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
+	struct page *page;
+	int ret;
+
+	page = alloc_pages(GFP_KERNEL_ACCOUNT, get_order(sz));
+	if (!page)
+		return NULL;
+
+	ret = set_memory_decrypted((unsigned long)page_address(page), npages);
+	if (ret) {
+		pr_err("failed to mark page shared, ret=%d\n", ret);
+		__free_pages(page, get_order(sz));
+		return NULL;
+	}
+
+	return page_address(page);
+}
+
+static u8 *get_vmpck(int id, struct snp_secrets_page *secrets, u32 **seqno)
+{
+	u8 *key = NULL;
+
+	switch (id) {
+	case 0:
+		*seqno = &secrets->os_area.msg_seqno_0;
+		key = secrets->vmpck0;
+		break;
+	case 1:
+		*seqno = &secrets->os_area.msg_seqno_1;
+		key = secrets->vmpck1;
+		break;
+	case 2:
+		*seqno = &secrets->os_area.msg_seqno_2;
+		key = secrets->vmpck2;
+		break;
+	case 3:
+		*seqno = &secrets->os_area.msg_seqno_3;
+		key = secrets->vmpck3;
+		break;
+	default:
+		break;
+	}
+
+	return key;
+}
+
+static struct aesgcm_ctx *snp_init_crypto(u8 *key, size_t keylen)
+{
+	struct aesgcm_ctx *ctx;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return NULL;
+
+	if (aesgcm_expandkey(ctx, key, keylen, AUTHTAG_LEN)) {
+		pr_err("Crypto context initialization failed\n");
+		kfree(ctx);
+		return NULL;
+	}
+
+	return ctx;
+}
+
+int snp_msg_init(struct snp_msg_desc *mdesc, int vmpck_id)
+{
+	/* Adjust the default VMPCK key based on the executing VMPL level */
+	if (vmpck_id == -1)
+		vmpck_id = snp_vmpl;
+
+	mdesc->vmpck = get_vmpck(vmpck_id, mdesc->secrets, &mdesc->os_area_msg_seqno);
+	if (!mdesc->vmpck) {
+		pr_err("Invalid VMPCK%d communication key\n", vmpck_id);
+		return -EINVAL;
+	}
+
+	/* Verify that VMPCK is not zero. */
+	if (!memchr_inv(mdesc->vmpck, 0, VMPCK_KEY_LEN)) {
+		pr_err("Empty VMPCK%d communication key\n", vmpck_id);
+		return -EINVAL;
+	}
+
+	mdesc->vmpck_id = vmpck_id;
+
+	mdesc->ctx = snp_init_crypto(mdesc->vmpck, VMPCK_KEY_LEN);
+	if (!mdesc->ctx)
+		return -ENOMEM;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(snp_msg_init);
+
+struct snp_msg_desc *snp_msg_alloc(void)
+{
+	struct snp_msg_desc *mdesc;
+	void __iomem *mem;
+
+	BUILD_BUG_ON(sizeof(struct snp_guest_msg) > PAGE_SIZE);
+
+	mdesc = kzalloc(sizeof(struct snp_msg_desc), GFP_KERNEL);
+	if (!mdesc)
+		return ERR_PTR(-ENOMEM);
+
+	mem = ioremap_encrypted(sev_secrets_pa, PAGE_SIZE);
+	if (!mem)
+		goto e_free_mdesc;
+
+	mdesc->secrets = (__force struct snp_secrets_page *)mem;
+
+	/* Allocate the shared page used for the request and response message. */
+	mdesc->request = alloc_shared_pages(sizeof(struct snp_guest_msg));
+	if (!mdesc->request)
+		goto e_unmap;
+
+	mdesc->response = alloc_shared_pages(sizeof(struct snp_guest_msg));
+	if (!mdesc->response)
+		goto e_free_request;
+
+	return mdesc;
+
+e_free_request:
+	free_shared_pages(mdesc->request, sizeof(struct snp_guest_msg));
+e_unmap:
+	iounmap(mem);
+e_free_mdesc:
+	kfree(mdesc);
+
+	return ERR_PTR(-ENOMEM);
+}
+EXPORT_SYMBOL_GPL(snp_msg_alloc);
+
+void snp_msg_free(struct snp_msg_desc *mdesc)
+{
+	if (!mdesc)
+		return;
+
+	kfree(mdesc->ctx);
+	free_shared_pages(mdesc->response, sizeof(struct snp_guest_msg));
+	free_shared_pages(mdesc->request, sizeof(struct snp_guest_msg));
+	iounmap((__force void __iomem *)mdesc->secrets);
+
+	memset(mdesc, 0, sizeof(*mdesc));
+	kfree(mdesc);
+}
+EXPORT_SYMBOL_GPL(snp_msg_free);
+
+/* Mutex to serialize the shared buffer access and command handling. */
+static DEFINE_MUTEX(snp_cmd_mutex);
+
+/*
+ * If an error is received from the host or AMD Secure Processor (ASP) there
+ * are two options. Either retry the exact same encrypted request or discontinue
+ * using the VMPCK.
+ *
+ * This is because in the current encryption scheme GHCB v2 uses AES-GCM to
+ * encrypt the requests. The IV for this scheme is the sequence number. GCM
+ * cannot tolerate IV reuse.
+ *
+ * The ASP FW v1.51 only increments the sequence numbers on a successful
+ * guest<->ASP back and forth and only accepts messages at its exact sequence
+ * number.
+ *
+ * So if the sequence number were to be reused the encryption scheme is
+ * vulnerable. If the sequence number were incremented for a fresh IV the ASP
+ * will reject the request.
+ */
+static void snp_disable_vmpck(struct snp_msg_desc *mdesc)
+{
+	pr_alert("Disabling VMPCK%d communication key to prevent IV reuse.\n",
+		  mdesc->vmpck_id);
+	memzero_explicit(mdesc->vmpck, VMPCK_KEY_LEN);
+	mdesc->vmpck = NULL;
+}
+
+static inline u64 __snp_get_msg_seqno(struct snp_msg_desc *mdesc)
+{
+	u64 count;
+
+	lockdep_assert_held(&snp_cmd_mutex);
+
+	/* Read the current message sequence counter from secrets pages */
+	count = *mdesc->os_area_msg_seqno;
+
+	return count + 1;
+}
+
+/* Return a non-zero on success */
+static u64 snp_get_msg_seqno(struct snp_msg_desc *mdesc)
+{
+	u64 count = __snp_get_msg_seqno(mdesc);
+
+	/*
+	 * The message sequence counter for the SNP guest request is a  64-bit
+	 * value but the version 2 of GHCB specification defines a 32-bit storage
+	 * for it. If the counter exceeds the 32-bit value then return zero.
+	 * The caller should check the return value, but if the caller happens to
+	 * not check the value and use it, then the firmware treats zero as an
+	 * invalid number and will fail the  message request.
+	 */
+	if (count >= UINT_MAX) {
+		pr_err("request message sequence counter overflow\n");
+		return 0;
+	}
+
+	return count;
+}
+
+static void snp_inc_msg_seqno(struct snp_msg_desc *mdesc)
+{
+	/*
+	 * The counter is also incremented by the PSP, so increment it by 2
+	 * and save in secrets page.
+	 */
+	*mdesc->os_area_msg_seqno += 2;
+}
+
+static int verify_and_dec_payload(struct snp_msg_desc *mdesc, struct snp_guest_req *req)
+{
+	struct snp_guest_msg *resp_msg = &mdesc->secret_response;
+	struct snp_guest_msg *req_msg = &mdesc->secret_request;
+	struct snp_guest_msg_hdr *req_msg_hdr = &req_msg->hdr;
+	struct snp_guest_msg_hdr *resp_msg_hdr = &resp_msg->hdr;
+	struct aesgcm_ctx *ctx = mdesc->ctx;
+	u8 iv[GCM_AES_IV_SIZE] = {};
+
+	pr_debug("response [seqno %lld type %d version %d sz %d]\n",
+		 resp_msg_hdr->msg_seqno, resp_msg_hdr->msg_type, resp_msg_hdr->msg_version,
+		 resp_msg_hdr->msg_sz);
+
+	/* Copy response from shared memory to encrypted memory. */
+	memcpy(resp_msg, mdesc->response, sizeof(*resp_msg));
+
+	/* Verify that the sequence counter is incremented by 1 */
+	if (unlikely(resp_msg_hdr->msg_seqno != (req_msg_hdr->msg_seqno + 1)))
+		return -EBADMSG;
+
+	/* Verify response message type and version number. */
+	if (resp_msg_hdr->msg_type != (req_msg_hdr->msg_type + 1) ||
+	    resp_msg_hdr->msg_version != req_msg_hdr->msg_version)
+		return -EBADMSG;
+
+	/*
+	 * If the message size is greater than our buffer length then return
+	 * an error.
+	 */
+	if (unlikely((resp_msg_hdr->msg_sz + ctx->authsize) > req->resp_sz))
+		return -EBADMSG;
+
+	/* Decrypt the payload */
+	memcpy(iv, &resp_msg_hdr->msg_seqno, min(sizeof(iv), sizeof(resp_msg_hdr->msg_seqno)));
+	if (!aesgcm_decrypt(ctx, req->resp_buf, resp_msg->payload, resp_msg_hdr->msg_sz,
+			    &resp_msg_hdr->algo, AAD_LEN, iv, resp_msg_hdr->authtag))
+		return -EBADMSG;
+
+	return 0;
+}
+
+static int enc_payload(struct snp_msg_desc *mdesc, u64 seqno, struct snp_guest_req *req)
+{
+	struct snp_guest_msg *msg = &mdesc->secret_request;
+	struct snp_guest_msg_hdr *hdr = &msg->hdr;
+	struct aesgcm_ctx *ctx = mdesc->ctx;
+	u8 iv[GCM_AES_IV_SIZE] = {};
+
+	memset(msg, 0, sizeof(*msg));
+
+	hdr->algo = SNP_AEAD_AES_256_GCM;
+	hdr->hdr_version = MSG_HDR_VER;
+	hdr->hdr_sz = sizeof(*hdr);
+	hdr->msg_type = req->msg_type;
+	hdr->msg_version = req->msg_version;
+	hdr->msg_seqno = seqno;
+	hdr->msg_vmpck = req->vmpck_id;
+	hdr->msg_sz = req->req_sz;
+
+	/* Verify the sequence number is non-zero */
+	if (!hdr->msg_seqno)
+		return -ENOSR;
+
+	pr_debug("request [seqno %lld type %d version %d sz %d]\n",
+		 hdr->msg_seqno, hdr->msg_type, hdr->msg_version, hdr->msg_sz);
+
+	if (WARN_ON((req->req_sz + ctx->authsize) > sizeof(msg->payload)))
+		return -EBADMSG;
+
+	memcpy(iv, &hdr->msg_seqno, min(sizeof(iv), sizeof(hdr->msg_seqno)));
+	aesgcm_encrypt(ctx, msg->payload, req->req_buf, req->req_sz, &hdr->algo,
+		       AAD_LEN, iv, hdr->authtag);
+
+	return 0;
+}
+
+static int __handle_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req)
+{
+	unsigned long req_start = jiffies;
+	unsigned int override_npages = 0;
+	u64 override_err = 0;
+	int rc;
+
+retry_request:
+	/*
+	 * Call firmware to process the request. In this function the encrypted
+	 * message enters shared memory with the host. So after this call the
+	 * sequence number must be incremented or the VMPCK must be deleted to
+	 * prevent reuse of the IV.
+	 */
+	rc = snp_issue_guest_request(req);
+	switch (rc) {
+	case -ENOSPC:
+		/*
+		 * If the extended guest request fails due to having too
+		 * small of a certificate data buffer, retry the same
+		 * guest request without the extended data request in
+		 * order to increment the sequence number and thus avoid
+		 * IV reuse.
+		 */
+		override_npages = req->input.data_npages;
+		req->exit_code	= SVM_VMGEXIT_GUEST_REQUEST;
+
+		/*
+		 * Override the error to inform callers the given extended
+		 * request buffer size was too small and give the caller the
+		 * required buffer size.
+		 */
+		override_err = SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN);
+
+		/*
+		 * If this call to the firmware succeeds, the sequence number can
+		 * be incremented allowing for continued use of the VMPCK. If
+		 * there is an error reflected in the return value, this value
+		 * is checked further down and the result will be the deletion
+		 * of the VMPCK and the error code being propagated back to the
+		 * user as an ioctl() return code.
+		 */
+		goto retry_request;
+
+	/*
+	 * The host may return SNP_GUEST_VMM_ERR_BUSY if the request has been
+	 * throttled. Retry in the driver to avoid returning and reusing the
+	 * message sequence number on a different message.
+	 */
+	case -EAGAIN:
+		if (jiffies - req_start > SNP_REQ_MAX_RETRY_DURATION) {
+			rc = -ETIMEDOUT;
+			break;
+		}
+		schedule_timeout_killable(SNP_REQ_RETRY_DELAY);
+		goto retry_request;
+	}
+
+	/*
+	 * Increment the message sequence number. There is no harm in doing
+	 * this now because decryption uses the value stored in the response
+	 * structure and any failure will wipe the VMPCK, preventing further
+	 * use anyway.
+	 */
+	snp_inc_msg_seqno(mdesc);
+
+	if (override_err) {
+		req->exitinfo2 = override_err;
+
+		/*
+		 * If an extended guest request was issued and the supplied certificate
+		 * buffer was not large enough, a standard guest request was issued to
+		 * prevent IV reuse. If the standard request was successful, return -EIO
+		 * back to the caller as would have originally been returned.
+		 */
+		if (!rc && override_err == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN))
+			rc = -EIO;
+	}
+
+	if (override_npages)
+		req->input.data_npages = override_npages;
+
+	return rc;
+}
+
+int snp_send_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req)
+{
+	u64 seqno;
+	int rc;
+
+	/*
+	 * enc_payload() calls aesgcm_encrypt(), which can potentially offload to HW.
+	 * The offload's DMA SG list of data to encrypt has to be in linear mapping.
+	 */
+	if (!virt_addr_valid(req->req_buf) || !virt_addr_valid(req->resp_buf)) {
+		pr_warn("AES-GSM buffers must be in linear mapping");
+		return -EINVAL;
+	}
+
+	guard(mutex)(&snp_cmd_mutex);
+
+	/* Check if the VMPCK is not empty */
+	if (!mdesc->vmpck || !memchr_inv(mdesc->vmpck, 0, VMPCK_KEY_LEN)) {
+		pr_err_ratelimited("VMPCK is disabled\n");
+		return -ENOTTY;
+	}
+
+	/* Get message sequence and verify that its a non-zero */
+	seqno = snp_get_msg_seqno(mdesc);
+	if (!seqno)
+		return -EIO;
+
+	/* Clear shared memory's response for the host to populate. */
+	memset(mdesc->response, 0, sizeof(struct snp_guest_msg));
+
+	/* Encrypt the userspace provided payload in mdesc->secret_request. */
+	rc = enc_payload(mdesc, seqno, req);
+	if (rc)
+		return rc;
+
+	/*
+	 * Write the fully encrypted request to the shared unencrypted
+	 * request page.
+	 */
+	memcpy(mdesc->request, &mdesc->secret_request, sizeof(mdesc->secret_request));
+
+	/* Initialize the input address for guest request */
+	req->input.req_gpa = __pa(mdesc->request);
+	req->input.resp_gpa = __pa(mdesc->response);
+	req->input.data_gpa = req->certs_data ? __pa(req->certs_data) : 0;
+
+	rc = __handle_guest_request(mdesc, req);
+	if (rc) {
+		if (rc == -EIO &&
+		    req->exitinfo2 == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN))
+			return rc;
+
+		pr_alert("Detected error from ASP request. rc: %d, exitinfo2: 0x%llx\n",
+			 rc, req->exitinfo2);
+
+		snp_disable_vmpck(mdesc);
+		return rc;
+	}
+
+	rc = verify_and_dec_payload(mdesc, req);
+	if (rc) {
+		pr_alert("Detected unexpected decode failure from ASP. rc: %d\n", rc);
+		snp_disable_vmpck(mdesc);
+		return rc;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(snp_send_guest_request);
+
+static int __init snp_get_tsc_info(void)
+{
+	struct snp_tsc_info_resp *tsc_resp;
+	struct snp_tsc_info_req *tsc_req;
+	struct snp_msg_desc *mdesc;
+	struct snp_guest_req req = {};
+	int rc = -ENOMEM;
+
+	tsc_req = kzalloc(sizeof(*tsc_req), GFP_KERNEL);
+	if (!tsc_req)
+		return rc;
+
+	/*
+	 * The intermediate response buffer is used while decrypting the
+	 * response payload. Make sure that it has enough space to cover
+	 * the authtag.
+	 */
+	tsc_resp = kzalloc(sizeof(*tsc_resp) + AUTHTAG_LEN, GFP_KERNEL);
+	if (!tsc_resp)
+		goto e_free_tsc_req;
+
+	mdesc = snp_msg_alloc();
+	if (IS_ERR_OR_NULL(mdesc))
+		goto e_free_tsc_resp;
+
+	rc = snp_msg_init(mdesc, snp_vmpl);
+	if (rc)
+		goto e_free_mdesc;
+
+	req.msg_version = MSG_HDR_VER;
+	req.msg_type = SNP_MSG_TSC_INFO_REQ;
+	req.vmpck_id = snp_vmpl;
+	req.req_buf = tsc_req;
+	req.req_sz = sizeof(*tsc_req);
+	req.resp_buf = (void *)tsc_resp;
+	req.resp_sz = sizeof(*tsc_resp) + AUTHTAG_LEN;
+	req.exit_code = SVM_VMGEXIT_GUEST_REQUEST;
+
+	rc = snp_send_guest_request(mdesc, &req);
+	if (rc)
+		goto e_request;
+
+	pr_debug("%s: response status 0x%x scale 0x%llx offset 0x%llx factor 0x%x\n",
+		 __func__, tsc_resp->status, tsc_resp->tsc_scale, tsc_resp->tsc_offset,
+		 tsc_resp->tsc_factor);
+
+	if (!tsc_resp->status) {
+		snp_tsc_scale = tsc_resp->tsc_scale;
+		snp_tsc_offset = tsc_resp->tsc_offset;
+	} else {
+		pr_err("Failed to get TSC info, response status 0x%x\n", tsc_resp->status);
+		rc = -EIO;
+	}
+
+e_request:
+	/* The response buffer contains sensitive data, explicitly clear it. */
+	memzero_explicit(tsc_resp, sizeof(*tsc_resp) + AUTHTAG_LEN);
+e_free_mdesc:
+	snp_msg_free(mdesc);
+e_free_tsc_resp:
+	kfree(tsc_resp);
+e_free_tsc_req:
+	kfree(tsc_req);
+
+	return rc;
+}
+
+void __init snp_secure_tsc_prepare(void)
+{
+	if (!cc_platform_has(CC_ATTR_GUEST_SNP_SECURE_TSC))
+		return;
+
+	if (snp_get_tsc_info()) {
+		pr_alert("Unable to retrieve Secure TSC info from ASP\n");
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SECURE_TSC);
+	}
+
+	pr_debug("SecureTSC enabled");
+}
+
+static unsigned long securetsc_get_tsc_khz(void)
+{
+	return snp_tsc_freq_khz;
+}
+
+void __init snp_secure_tsc_init(void)
+{
+	struct snp_secrets_page *secrets;
+	unsigned long tsc_freq_mhz;
+	void *mem;
+
+	if (!cc_platform_has(CC_ATTR_GUEST_SNP_SECURE_TSC))
+		return;
+
+	mem = early_memremap_encrypted(sev_secrets_pa, PAGE_SIZE);
+	if (!mem) {
+		pr_err("Unable to get TSC_FACTOR: failed to map the SNP secrets page.\n");
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SECURE_TSC);
+	}
+
+	secrets = (__force struct snp_secrets_page *)mem;
+
+	setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);
+	rdmsrq(MSR_AMD64_GUEST_TSC_FREQ, tsc_freq_mhz);
+
+	/* Extract the GUEST TSC MHZ from BIT[17:0], rest is reserved space */
+	tsc_freq_mhz &= GENMASK_ULL(17, 0);
+
+	snp_tsc_freq_khz = SNP_SCALE_TSC_FREQ(tsc_freq_mhz * 1000, secrets->tsc_factor);
+
+	x86_platform.calibrate_cpu = securetsc_get_tsc_khz;
+	x86_platform.calibrate_tsc = securetsc_get_tsc_khz;
+
+	early_memunmap(mem, PAGE_SIZE);
+}
diff --git a/arch/x86/coco/sev/noinstr.c b/arch/x86/coco/sev/noinstr.c
new file mode 100644
index 000000000000..b527eafb6312
--- /dev/null
+++ b/arch/x86/coco/sev/noinstr.c
@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AMD Memory Encryption Support
+ *
+ * Copyright (C) 2019 SUSE
+ *
+ * Author: Joerg Roedel <jroedel@suse.de>
+ */
+
+#define pr_fmt(fmt)	"SEV: " fmt
+
+#include <linux/bug.h>
+#include <linux/kernel.h>
+
+#include <asm/cpu_entry_area.h>
+#include <asm/msr.h>
+#include <asm/ptrace.h>
+#include <asm/sev.h>
+#include <asm/sev-internal.h>
+
+static __always_inline bool on_vc_stack(struct pt_regs *regs)
+{
+	unsigned long sp = regs->sp;
+
+	/* User-mode RSP is not trusted */
+	if (user_mode(regs))
+		return false;
+
+	/* SYSCALL gap still has user-mode RSP */
+	if (ip_within_syscall_gap(regs))
+		return false;
+
+	return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC)));
+}
+
+/*
+ * This function handles the case when an NMI is raised in the #VC
+ * exception handler entry code, before the #VC handler has switched off
+ * its IST stack. In this case, the IST entry for #VC must be adjusted,
+ * so that any nested #VC exception will not overwrite the stack
+ * contents of the interrupted #VC handler.
+ *
+ * The IST entry is adjusted unconditionally so that it can be also be
+ * unconditionally adjusted back in __sev_es_ist_exit(). Otherwise a
+ * nested sev_es_ist_exit() call may adjust back the IST entry too
+ * early.
+ *
+ * The __sev_es_ist_enter() and __sev_es_ist_exit() functions always run
+ * on the NMI IST stack, as they are only called from NMI handling code
+ * right now.
+ */
+void noinstr __sev_es_ist_enter(struct pt_regs *regs)
+{
+	unsigned long old_ist, new_ist;
+
+	/* Read old IST entry */
+	new_ist = old_ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]);
+
+	/*
+	 * If NMI happened while on the #VC IST stack, set the new IST
+	 * value below regs->sp, so that the interrupted stack frame is
+	 * not overwritten by subsequent #VC exceptions.
+	 */
+	if (on_vc_stack(regs))
+		new_ist = regs->sp;
+
+	/*
+	 * Reserve additional 8 bytes and store old IST value so this
+	 * adjustment can be unrolled in __sev_es_ist_exit().
+	 */
+	new_ist -= sizeof(old_ist);
+	*(unsigned long *)new_ist = old_ist;
+
+	/* Set new IST entry */
+	this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], new_ist);
+}
+
+void noinstr __sev_es_ist_exit(void)
+{
+	unsigned long ist;
+
+	/* Read IST entry */
+	ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]);
+
+	if (WARN_ON(ist == __this_cpu_ist_top_va(VC)))
+		return;
+
+	/* Read back old IST entry and write it to the TSS */
+	this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist);
+}
+
+void noinstr __sev_es_nmi_complete(void)
+{
+	struct ghcb_state state;
+	struct ghcb *ghcb;
+
+	ghcb = __sev_get_ghcb(&state);
+
+	vc_ghcb_invalidate(ghcb);
+	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);
+	ghcb_set_sw_exit_info_1(ghcb, 0);
+	ghcb_set_sw_exit_info_2(ghcb, 0);
+
+	sev_es_wr_ghcb_msr(__pa_nodebug(ghcb));
+	VMGEXIT();
+
+	__sev_put_ghcb(&state);
+}
+
+/*
+ * Nothing shall interrupt this code path while holding the per-CPU
+ * GHCB. The backup GHCB is only for NMIs interrupting this path.
+ *
+ * Callers must disable local interrupts around it.
+ */
+noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
+{
+	struct sev_es_runtime_data *data;
+	struct ghcb *ghcb;
+
+	WARN_ON(!irqs_disabled());
+
+	data = this_cpu_read(runtime_data);
+	ghcb = &data->ghcb_page;
+
+	if (unlikely(data->ghcb_active)) {
+		/* GHCB is already in use - save its contents */
+
+		if (unlikely(data->backup_ghcb_active)) {
+			/*
+			 * Backup-GHCB is also already in use. There is no way
+			 * to continue here so just kill the machine. To make
+			 * panic() work, mark GHCBs inactive so that messages
+			 * can be printed out.
+			 */
+			data->ghcb_active        = false;
+			data->backup_ghcb_active = false;
+
+			instrumentation_begin();
+			panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use");
+			instrumentation_end();
+		}
+
+		/* Mark backup_ghcb active before writing to it */
+		data->backup_ghcb_active = true;
+
+		state->ghcb = &data->backup_ghcb;
+
+		/* Backup GHCB content */
+		*state->ghcb = *ghcb;
+	} else {
+		state->ghcb = NULL;
+		data->ghcb_active = true;
+	}
+
+	return ghcb;
+}
+
+noinstr void __sev_put_ghcb(struct ghcb_state *state)
+{
+	struct sev_es_runtime_data *data;
+	struct ghcb *ghcb;
+
+	WARN_ON(!irqs_disabled());
+
+	data = this_cpu_read(runtime_data);
+	ghcb = &data->ghcb_page;
+
+	if (state->ghcb) {
+		/* Restore GHCB from Backup */
+		*ghcb = *state->ghcb;
+		data->backup_ghcb_active = false;
+		state->ghcb = NULL;
+	} else {
+		/*
+		 * Invalidate the GHCB so a VMGEXIT instruction issued
+		 * from userspace won't appear to be valid.
+		 */
+		vc_ghcb_invalidate(ghcb);
+		data->ghcb_active = false;
+	}
+}
diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c
deleted file mode 100644
index 71de53194089..000000000000
--- a/arch/x86/coco/sev/shared.c
+++ /dev/null
@@ -1,1717 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * AMD Encrypted Register State Support
- *
- * Author: Joerg Roedel <jroedel@suse.de>
- *
- * This file is not compiled stand-alone. It contains code shared
- * between the pre-decompression boot code and the running Linux kernel
- * and is included directly into both code-bases.
- */
-
-#include <asm/setup_data.h>
-
-#ifndef __BOOT_COMPRESSED
-#define error(v)			pr_err(v)
-#define has_cpuflag(f)			boot_cpu_has(f)
-#define sev_printk(fmt, ...)		printk(fmt, ##__VA_ARGS__)
-#define sev_printk_rtl(fmt, ...)	printk_ratelimited(fmt, ##__VA_ARGS__)
-#else
-#undef WARN
-#define WARN(condition, format...) (!!(condition))
-#define sev_printk(fmt, ...)
-#define sev_printk_rtl(fmt, ...)
-#undef vc_forward_exception
-#define vc_forward_exception(c)		panic("SNP: Hypervisor requested exception\n")
-#endif
-
-/*
- * SVSM related information:
- *   When running under an SVSM, the VMPL that Linux is executing at must be
- *   non-zero. The VMPL is therefore used to indicate the presence of an SVSM.
- *
- *   During boot, the page tables are set up as identity mapped and later
- *   changed to use kernel virtual addresses. Maintain separate virtual and
- *   physical addresses for the CAA to allow SVSM functions to be used during
- *   early boot, both with identity mapped virtual addresses and proper kernel
- *   virtual addresses.
- */
-u8 snp_vmpl __ro_after_init;
-EXPORT_SYMBOL_GPL(snp_vmpl);
-static struct svsm_ca *boot_svsm_caa __ro_after_init;
-static u64 boot_svsm_caa_pa __ro_after_init;
-
-static struct svsm_ca *svsm_get_caa(void);
-static u64 svsm_get_caa_pa(void);
-static int svsm_perform_call_protocol(struct svsm_call *call);
-
-/* I/O parameters for CPUID-related helpers */
-struct cpuid_leaf {
-	u32 fn;
-	u32 subfn;
-	u32 eax;
-	u32 ebx;
-	u32 ecx;
-	u32 edx;
-};
-
-/*
- * Individual entries of the SNP CPUID table, as defined by the SNP
- * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
- */
-struct snp_cpuid_fn {
-	u32 eax_in;
-	u32 ecx_in;
-	u64 xcr0_in;
-	u64 xss_in;
-	u32 eax;
-	u32 ebx;
-	u32 ecx;
-	u32 edx;
-	u64 __reserved;
-} __packed;
-
-/*
- * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
- * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
- * of 64 entries per CPUID table.
- */
-#define SNP_CPUID_COUNT_MAX 64
-
-struct snp_cpuid_table {
-	u32 count;
-	u32 __reserved1;
-	u64 __reserved2;
-	struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
-} __packed;
-
-/*
- * Since feature negotiation related variables are set early in the boot
- * process they must reside in the .data section so as not to be zeroed
- * out when the .bss section is later cleared.
- *
- * GHCB protocol version negotiated with the hypervisor.
- */
-static u16 ghcb_version __ro_after_init;
-
-/* Copy of the SNP firmware's CPUID page. */
-static struct snp_cpuid_table cpuid_table_copy __ro_after_init;
-
-/*
- * These will be initialized based on CPUID table so that non-present
- * all-zero leaves (for sparse tables) can be differentiated from
- * invalid/out-of-range leaves. This is needed since all-zero leaves
- * still need to be post-processed.
- */
-static u32 cpuid_std_range_max __ro_after_init;
-static u32 cpuid_hyp_range_max __ro_after_init;
-static u32 cpuid_ext_range_max __ro_after_init;
-
-static bool __init sev_es_check_cpu_features(void)
-{
-	if (!has_cpuflag(X86_FEATURE_RDRAND)) {
-		error("RDRAND instruction not supported - no trusted source of randomness available\n");
-		return false;
-	}
-
-	return true;
-}
-
-static void __head __noreturn
-sev_es_terminate(unsigned int set, unsigned int reason)
-{
-	u64 val = GHCB_MSR_TERM_REQ;
-
-	/* Tell the hypervisor what went wrong. */
-	val |= GHCB_SEV_TERM_REASON(set, reason);
-
-	/* Request Guest Termination from Hypervisor */
-	sev_es_wr_ghcb_msr(val);
-	VMGEXIT();
-
-	while (true)
-		asm volatile("hlt\n" : : : "memory");
-}
-
-/*
- * The hypervisor features are available from GHCB version 2 onward.
- */
-static u64 get_hv_features(void)
-{
-	u64 val;
-
-	if (ghcb_version < 2)
-		return 0;
-
-	sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ);
-	VMGEXIT();
-
-	val = sev_es_rd_ghcb_msr();
-	if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP)
-		return 0;
-
-	return GHCB_MSR_HV_FT_RESP_VAL(val);
-}
-
-static void snp_register_ghcb_early(unsigned long paddr)
-{
-	unsigned long pfn = paddr >> PAGE_SHIFT;
-	u64 val;
-
-	sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn));
-	VMGEXIT();
-
-	val = sev_es_rd_ghcb_msr();
-
-	/* If the response GPA is not ours then abort the guest */
-	if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) ||
-	    (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn))
-		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
-}
-
-static bool sev_es_negotiate_protocol(void)
-{
-	u64 val;
-
-	/* Do the GHCB protocol version negotiation */
-	sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
-	VMGEXIT();
-	val = sev_es_rd_ghcb_msr();
-
-	if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
-		return false;
-
-	if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||
-	    GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
-		return false;
-
-	ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX);
-
-	return true;
-}
-
-static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
-{
-	ghcb->save.sw_exit_code = 0;
-	__builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
-}
-
-static bool vc_decoding_needed(unsigned long exit_code)
-{
-	/* Exceptions don't require to decode the instruction */
-	return !(exit_code >= SVM_EXIT_EXCP_BASE &&
-		 exit_code <= SVM_EXIT_LAST_EXCP);
-}
-
-static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
-				      struct pt_regs *regs,
-				      unsigned long exit_code)
-{
-	enum es_result ret = ES_OK;
-
-	memset(ctxt, 0, sizeof(*ctxt));
-	ctxt->regs = regs;
-
-	if (vc_decoding_needed(exit_code))
-		ret = vc_decode_insn(ctxt);
-
-	return ret;
-}
-
-static void vc_finish_insn(struct es_em_ctxt *ctxt)
-{
-	ctxt->regs->ip += ctxt->insn.length;
-}
-
-static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
-{
-	u32 ret;
-
-	ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0);
-	if (!ret)
-		return ES_OK;
-
-	if (ret == 1) {
-		u64 info = ghcb->save.sw_exit_info_2;
-		unsigned long v = info & SVM_EVTINJ_VEC_MASK;
-
-		/* Check if exception information from hypervisor is sane. */
-		if ((info & SVM_EVTINJ_VALID) &&
-		    ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
-		    ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
-			ctxt->fi.vector = v;
-
-			if (info & SVM_EVTINJ_VALID_ERR)
-				ctxt->fi.error_code = info >> 32;
-
-			return ES_EXCEPTION;
-		}
-	}
-
-	return ES_VMM_ERROR;
-}
-
-static inline int svsm_process_result_codes(struct svsm_call *call)
-{
-	switch (call->rax_out) {
-	case SVSM_SUCCESS:
-		return 0;
-	case SVSM_ERR_INCOMPLETE:
-	case SVSM_ERR_BUSY:
-		return -EAGAIN;
-	default:
-		return -EINVAL;
-	}
-}
-
-/*
- * Issue a VMGEXIT to call the SVSM:
- *   - Load the SVSM register state (RAX, RCX, RDX, R8 and R9)
- *   - Set the CA call pending field to 1
- *   - Issue VMGEXIT
- *   - Save the SVSM return register state (RAX, RCX, RDX, R8 and R9)
- *   - Perform atomic exchange of the CA call pending field
- *
- *   - See the "Secure VM Service Module for SEV-SNP Guests" specification for
- *     details on the calling convention.
- *     - The calling convention loosely follows the Microsoft X64 calling
- *       convention by putting arguments in RCX, RDX, R8 and R9.
- *     - RAX specifies the SVSM protocol/callid as input and the return code
- *       as output.
- */
-static __always_inline void svsm_issue_call(struct svsm_call *call, u8 *pending)
-{
-	register unsigned long rax asm("rax") = call->rax;
-	register unsigned long rcx asm("rcx") = call->rcx;
-	register unsigned long rdx asm("rdx") = call->rdx;
-	register unsigned long r8  asm("r8")  = call->r8;
-	register unsigned long r9  asm("r9")  = call->r9;
-
-	call->caa->call_pending = 1;
-
-	asm volatile("rep; vmmcall\n\t"
-		     : "+r" (rax), "+r" (rcx), "+r" (rdx), "+r" (r8), "+r" (r9)
-		     : : "memory");
-
-	*pending = xchg(&call->caa->call_pending, *pending);
-
-	call->rax_out = rax;
-	call->rcx_out = rcx;
-	call->rdx_out = rdx;
-	call->r8_out  = r8;
-	call->r9_out  = r9;
-}
-
-static int svsm_perform_msr_protocol(struct svsm_call *call)
-{
-	u8 pending = 0;
-	u64 val, resp;
-
-	/*
-	 * When using the MSR protocol, be sure to save and restore
-	 * the current MSR value.
-	 */
-	val = sev_es_rd_ghcb_msr();
-
-	sev_es_wr_ghcb_msr(GHCB_MSR_VMPL_REQ_LEVEL(0));
-
-	svsm_issue_call(call, &pending);
-
-	resp = sev_es_rd_ghcb_msr();
-
-	sev_es_wr_ghcb_msr(val);
-
-	if (pending)
-		return -EINVAL;
-
-	if (GHCB_RESP_CODE(resp) != GHCB_MSR_VMPL_RESP)
-		return -EINVAL;
-
-	if (GHCB_MSR_VMPL_RESP_VAL(resp))
-		return -EINVAL;
-
-	return svsm_process_result_codes(call);
-}
-
-static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call)
-{
-	struct es_em_ctxt ctxt;
-	u8 pending = 0;
-
-	vc_ghcb_invalidate(ghcb);
-
-	/*
-	 * Fill in protocol and format specifiers. This can be called very early
-	 * in the boot, so use rip-relative references as needed.
-	 */
-	ghcb->protocol_version = RIP_REL_REF(ghcb_version);
-	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
-
-	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_SNP_RUN_VMPL);
-	ghcb_set_sw_exit_info_1(ghcb, 0);
-	ghcb_set_sw_exit_info_2(ghcb, 0);
-
-	sev_es_wr_ghcb_msr(__pa(ghcb));
-
-	svsm_issue_call(call, &pending);
-
-	if (pending)
-		return -EINVAL;
-
-	switch (verify_exception_info(ghcb, &ctxt)) {
-	case ES_OK:
-		break;
-	case ES_EXCEPTION:
-		vc_forward_exception(&ctxt);
-		fallthrough;
-	default:
-		return -EINVAL;
-	}
-
-	return svsm_process_result_codes(call);
-}
-
-static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
-					  struct es_em_ctxt *ctxt,
-					  u64 exit_code, u64 exit_info_1,
-					  u64 exit_info_2)
-{
-	/* Fill in protocol and format specifiers */
-	ghcb->protocol_version = ghcb_version;
-	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
-
-	ghcb_set_sw_exit_code(ghcb, exit_code);
-	ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
-	ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
-
-	sev_es_wr_ghcb_msr(__pa(ghcb));
-	VMGEXIT();
-
-	return verify_exception_info(ghcb, ctxt);
-}
-
-static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg)
-{
-	u64 val;
-
-	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx));
-	VMGEXIT();
-	val = sev_es_rd_ghcb_msr();
-	if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
-		return -EIO;
-
-	*reg = (val >> 32);
-
-	return 0;
-}
-
-static int __sev_cpuid_hv_msr(struct cpuid_leaf *leaf)
-{
-	int ret;
-
-	/*
-	 * MSR protocol does not support fetching non-zero subfunctions, but is
-	 * sufficient to handle current early-boot cases. Should that change,
-	 * make sure to report an error rather than ignoring the index and
-	 * grabbing random values. If this issue arises in the future, handling
-	 * can be added here to use GHCB-page protocol for cases that occur late
-	 * enough in boot that GHCB page is available.
-	 */
-	if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn)
-		return -EINVAL;
-
-	ret =         __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax);
-	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx);
-	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx);
-	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx);
-
-	return ret;
-}
-
-static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
-{
-	u32 cr4 = native_read_cr4();
-	int ret;
-
-	ghcb_set_rax(ghcb, leaf->fn);
-	ghcb_set_rcx(ghcb, leaf->subfn);
-
-	if (cr4 & X86_CR4_OSXSAVE)
-		/* Safe to read xcr0 */
-		ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
-	else
-		/* xgetbv will cause #UD - use reset value for xcr0 */
-		ghcb_set_xcr0(ghcb, 1);
-
-	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
-	if (ret != ES_OK)
-		return ret;
-
-	if (!(ghcb_rax_is_valid(ghcb) &&
-	      ghcb_rbx_is_valid(ghcb) &&
-	      ghcb_rcx_is_valid(ghcb) &&
-	      ghcb_rdx_is_valid(ghcb)))
-		return ES_VMM_ERROR;
-
-	leaf->eax = ghcb->save.rax;
-	leaf->ebx = ghcb->save.rbx;
-	leaf->ecx = ghcb->save.rcx;
-	leaf->edx = ghcb->save.rdx;
-
-	return ES_OK;
-}
-
-static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
-{
-	return ghcb ? __sev_cpuid_hv_ghcb(ghcb, ctxt, leaf)
-		    : __sev_cpuid_hv_msr(leaf);
-}
-
-/*
- * This may be called early while still running on the initial identity
- * mapping. Use RIP-relative addressing to obtain the correct address
- * while running with the initial identity mapping as well as the
- * switch-over to kernel virtual addresses later.
- */
-static const struct snp_cpuid_table *snp_cpuid_get_table(void)
-{
-	return &RIP_REL_REF(cpuid_table_copy);
-}
-
-/*
- * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of
- * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0
- * and 1 based on the corresponding features enabled by a particular
- * combination of XCR0 and XSS registers so that a guest can look up the
- * version corresponding to the features currently enabled in its XCR0/XSS
- * registers. The only values that differ between these versions/table
- * entries is the enabled XSAVE area size advertised via EBX.
- *
- * While hypervisors may choose to make use of this support, it is more
- * robust/secure for a guest to simply find the entry corresponding to the
- * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the
- * XSAVE area size using subfunctions 2 through 64, as documented in APM
- * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here.
- *
- * Since base/legacy XSAVE area size is documented as 0x240, use that value
- * directly rather than relying on the base size in the CPUID table.
- *
- * Return: XSAVE area size on success, 0 otherwise.
- */
-static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted)
-{
-	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
-	u64 xfeatures_found = 0;
-	u32 xsave_size = 0x240;
-	int i;
-
-	for (i = 0; i < cpuid_table->count; i++) {
-		const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
-
-		if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64))
-			continue;
-		if (!(xfeatures_en & (BIT_ULL(e->ecx_in))))
-			continue;
-		if (xfeatures_found & (BIT_ULL(e->ecx_in)))
-			continue;
-
-		xfeatures_found |= (BIT_ULL(e->ecx_in));
-
-		if (compacted)
-			xsave_size += e->eax;
-		else
-			xsave_size = max(xsave_size, e->eax + e->ebx);
-	}
-
-	/*
-	 * Either the guest set unsupported XCR0/XSS bits, or the corresponding
-	 * entries in the CPUID table were not present. This is not a valid
-	 * state to be in.
-	 */
-	if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2)))
-		return 0;
-
-	return xsave_size;
-}
-
-static bool __head
-snp_cpuid_get_validated_func(struct cpuid_leaf *leaf)
-{
-	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
-	int i;
-
-	for (i = 0; i < cpuid_table->count; i++) {
-		const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
-
-		if (e->eax_in != leaf->fn)
-			continue;
-
-		if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn)
-			continue;
-
-		/*
-		 * For 0xD subfunctions 0 and 1, only use the entry corresponding
-		 * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0).
-		 * See the comments above snp_cpuid_calc_xsave_size() for more
-		 * details.
-		 */
-		if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1))
-			if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in)
-				continue;
-
-		leaf->eax = e->eax;
-		leaf->ebx = e->ebx;
-		leaf->ecx = e->ecx;
-		leaf->edx = e->edx;
-
-		return true;
-	}
-
-	return false;
-}
-
-static void snp_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
-{
-	if (sev_cpuid_hv(ghcb, ctxt, leaf))
-		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV);
-}
-
-static int snp_cpuid_postprocess(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
-				 struct cpuid_leaf *leaf)
-{
-	struct cpuid_leaf leaf_hv = *leaf;
-
-	switch (leaf->fn) {
-	case 0x1:
-		snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
-
-		/* initial APIC ID */
-		leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0));
-		/* APIC enabled bit */
-		leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9));
-
-		/* OSXSAVE enabled bit */
-		if (native_read_cr4() & X86_CR4_OSXSAVE)
-			leaf->ecx |= BIT(27);
-		break;
-	case 0x7:
-		/* OSPKE enabled bit */
-		leaf->ecx &= ~BIT(4);
-		if (native_read_cr4() & X86_CR4_PKE)
-			leaf->ecx |= BIT(4);
-		break;
-	case 0xB:
-		leaf_hv.subfn = 0;
-		snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
-
-		/* extended APIC ID */
-		leaf->edx = leaf_hv.edx;
-		break;
-	case 0xD: {
-		bool compacted = false;
-		u64 xcr0 = 1, xss = 0;
-		u32 xsave_size;
-
-		if (leaf->subfn != 0 && leaf->subfn != 1)
-			return 0;
-
-		if (native_read_cr4() & X86_CR4_OSXSAVE)
-			xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
-		if (leaf->subfn == 1) {
-			/* Get XSS value if XSAVES is enabled. */
-			if (leaf->eax & BIT(3)) {
-				unsigned long lo, hi;
-
-				asm volatile("rdmsr" : "=a" (lo), "=d" (hi)
-						     : "c" (MSR_IA32_XSS));
-				xss = (hi << 32) | lo;
-			}
-
-			/*
-			 * The PPR and APM aren't clear on what size should be
-			 * encoded in 0xD:0x1:EBX when compaction is not enabled
-			 * by either XSAVEC (feature bit 1) or XSAVES (feature
-			 * bit 3) since SNP-capable hardware has these feature
-			 * bits fixed as 1. KVM sets it to 0 in this case, but
-			 * to avoid this becoming an issue it's safer to simply
-			 * treat this as unsupported for SNP guests.
-			 */
-			if (!(leaf->eax & (BIT(1) | BIT(3))))
-				return -EINVAL;
-
-			compacted = true;
-		}
-
-		xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted);
-		if (!xsave_size)
-			return -EINVAL;
-
-		leaf->ebx = xsave_size;
-		}
-		break;
-	case 0x8000001E:
-		snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
-
-		/* extended APIC ID */
-		leaf->eax = leaf_hv.eax;
-		/* compute ID */
-		leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0));
-		/* node ID */
-		leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0));
-		break;
-	default:
-		/* No fix-ups needed, use values as-is. */
-		break;
-	}
-
-	return 0;
-}
-
-/*
- * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value
- * should be treated as fatal by caller.
- */
-static int __head
-snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
-{
-	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
-
-	if (!cpuid_table->count)
-		return -EOPNOTSUPP;
-
-	if (!snp_cpuid_get_validated_func(leaf)) {
-		/*
-		 * Some hypervisors will avoid keeping track of CPUID entries
-		 * where all values are zero, since they can be handled the
-		 * same as out-of-range values (all-zero). This is useful here
-		 * as well as it allows virtually all guest configurations to
-		 * work using a single SNP CPUID table.
-		 *
-		 * To allow for this, there is a need to distinguish between
-		 * out-of-range entries and in-range zero entries, since the
-		 * CPUID table entries are only a template that may need to be
-		 * augmented with additional values for things like
-		 * CPU-specific information during post-processing. So if it's
-		 * not in the table, set the values to zero. Then, if they are
-		 * within a valid CPUID range, proceed with post-processing
-		 * using zeros as the initial values. Otherwise, skip
-		 * post-processing and just return zeros immediately.
-		 */
-		leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;
-
-		/* Skip post-processing for out-of-range zero leafs. */
-		if (!(leaf->fn <= RIP_REL_REF(cpuid_std_range_max) ||
-		      (leaf->fn >= 0x40000000 && leaf->fn <= RIP_REL_REF(cpuid_hyp_range_max)) ||
-		      (leaf->fn >= 0x80000000 && leaf->fn <= RIP_REL_REF(cpuid_ext_range_max))))
-			return 0;
-	}
-
-	return snp_cpuid_postprocess(ghcb, ctxt, leaf);
-}
-
-/*
- * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
- * page yet, so it only supports the MSR based communication with the
- * hypervisor and only the CPUID exit-code.
- */
-void __head do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
-{
-	unsigned int subfn = lower_bits(regs->cx, 32);
-	unsigned int fn = lower_bits(regs->ax, 32);
-	u16 opcode = *(unsigned short *)regs->ip;
-	struct cpuid_leaf leaf;
-	int ret;
-
-	/* Only CPUID is supported via MSR protocol */
-	if (exit_code != SVM_EXIT_CPUID)
-		goto fail;
-
-	/* Is it really a CPUID insn? */
-	if (opcode != 0xa20f)
-		goto fail;
-
-	leaf.fn = fn;
-	leaf.subfn = subfn;
-
-	ret = snp_cpuid(NULL, NULL, &leaf);
-	if (!ret)
-		goto cpuid_done;
-
-	if (ret != -EOPNOTSUPP)
-		goto fail;
-
-	if (__sev_cpuid_hv_msr(&leaf))
-		goto fail;
-
-cpuid_done:
-	regs->ax = leaf.eax;
-	regs->bx = leaf.ebx;
-	regs->cx = leaf.ecx;
-	regs->dx = leaf.edx;
-
-	/*
-	 * This is a VC handler and the #VC is only raised when SEV-ES is
-	 * active, which means SEV must be active too. Do sanity checks on the
-	 * CPUID results to make sure the hypervisor does not trick the kernel
-	 * into the no-sev path. This could map sensitive data unencrypted and
-	 * make it accessible to the hypervisor.
-	 *
-	 * In particular, check for:
-	 *	- Availability of CPUID leaf 0x8000001f
-	 *	- SEV CPUID bit.
-	 *
-	 * The hypervisor might still report the wrong C-bit position, but this
-	 * can't be checked here.
-	 */
-
-	if (fn == 0x80000000 && (regs->ax < 0x8000001f))
-		/* SEV leaf check */
-		goto fail;
-	else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
-		/* SEV bit */
-		goto fail;
-
-	/* Skip over the CPUID two-byte opcode */
-	regs->ip += 2;
-
-	return;
-
-fail:
-	/* Terminate the guest */
-	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
-}
-
-static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt,
-					   unsigned long address,
-					   bool write)
-{
-	if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) {
-		ctxt->fi.vector     = X86_TRAP_PF;
-		ctxt->fi.error_code = X86_PF_USER;
-		ctxt->fi.cr2        = address;
-		if (write)
-			ctxt->fi.error_code |= X86_PF_WRITE;
-
-		return ES_EXCEPTION;
-	}
-
-	return ES_OK;
-}
-
-static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
-					  void *src, char *buf,
-					  unsigned int data_size,
-					  unsigned int count,
-					  bool backwards)
-{
-	int i, b = backwards ? -1 : 1;
-	unsigned long address = (unsigned long)src;
-	enum es_result ret;
-
-	ret = vc_insn_string_check(ctxt, address, false);
-	if (ret != ES_OK)
-		return ret;
-
-	for (i = 0; i < count; i++) {
-		void *s = src + (i * data_size * b);
-		char *d = buf + (i * data_size);
-
-		ret = vc_read_mem(ctxt, s, d, data_size);
-		if (ret != ES_OK)
-			break;
-	}
-
-	return ret;
-}
-
-static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
-					   void *dst, char *buf,
-					   unsigned int data_size,
-					   unsigned int count,
-					   bool backwards)
-{
-	int i, s = backwards ? -1 : 1;
-	unsigned long address = (unsigned long)dst;
-	enum es_result ret;
-
-	ret = vc_insn_string_check(ctxt, address, true);
-	if (ret != ES_OK)
-		return ret;
-
-	for (i = 0; i < count; i++) {
-		void *d = dst + (i * data_size * s);
-		char *b = buf + (i * data_size);
-
-		ret = vc_write_mem(ctxt, d, b, data_size);
-		if (ret != ES_OK)
-			break;
-	}
-
-	return ret;
-}
-
-#define IOIO_TYPE_STR  BIT(2)
-#define IOIO_TYPE_IN   1
-#define IOIO_TYPE_INS  (IOIO_TYPE_IN | IOIO_TYPE_STR)
-#define IOIO_TYPE_OUT  0
-#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
-
-#define IOIO_REP       BIT(3)
-
-#define IOIO_ADDR_64   BIT(9)
-#define IOIO_ADDR_32   BIT(8)
-#define IOIO_ADDR_16   BIT(7)
-
-#define IOIO_DATA_32   BIT(6)
-#define IOIO_DATA_16   BIT(5)
-#define IOIO_DATA_8    BIT(4)
-
-#define IOIO_SEG_ES    (0 << 10)
-#define IOIO_SEG_DS    (3 << 10)
-
-static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
-{
-	struct insn *insn = &ctxt->insn;
-	size_t size;
-	u64 port;
-
-	*exitinfo = 0;
-
-	switch (insn->opcode.bytes[0]) {
-	/* INS opcodes */
-	case 0x6c:
-	case 0x6d:
-		*exitinfo |= IOIO_TYPE_INS;
-		*exitinfo |= IOIO_SEG_ES;
-		port	   = ctxt->regs->dx & 0xffff;
-		break;
-
-	/* OUTS opcodes */
-	case 0x6e:
-	case 0x6f:
-		*exitinfo |= IOIO_TYPE_OUTS;
-		*exitinfo |= IOIO_SEG_DS;
-		port	   = ctxt->regs->dx & 0xffff;
-		break;
-
-	/* IN immediate opcodes */
-	case 0xe4:
-	case 0xe5:
-		*exitinfo |= IOIO_TYPE_IN;
-		port	   = (u8)insn->immediate.value & 0xffff;
-		break;
-
-	/* OUT immediate opcodes */
-	case 0xe6:
-	case 0xe7:
-		*exitinfo |= IOIO_TYPE_OUT;
-		port	   = (u8)insn->immediate.value & 0xffff;
-		break;
-
-	/* IN register opcodes */
-	case 0xec:
-	case 0xed:
-		*exitinfo |= IOIO_TYPE_IN;
-		port	   = ctxt->regs->dx & 0xffff;
-		break;
-
-	/* OUT register opcodes */
-	case 0xee:
-	case 0xef:
-		*exitinfo |= IOIO_TYPE_OUT;
-		port	   = ctxt->regs->dx & 0xffff;
-		break;
-
-	default:
-		return ES_DECODE_FAILED;
-	}
-
-	*exitinfo |= port << 16;
-
-	switch (insn->opcode.bytes[0]) {
-	case 0x6c:
-	case 0x6e:
-	case 0xe4:
-	case 0xe6:
-	case 0xec:
-	case 0xee:
-		/* Single byte opcodes */
-		*exitinfo |= IOIO_DATA_8;
-		size       = 1;
-		break;
-	default:
-		/* Length determined by instruction parsing */
-		*exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
-						     : IOIO_DATA_32;
-		size       = (insn->opnd_bytes == 2) ? 2 : 4;
-	}
-
-	switch (insn->addr_bytes) {
-	case 2:
-		*exitinfo |= IOIO_ADDR_16;
-		break;
-	case 4:
-		*exitinfo |= IOIO_ADDR_32;
-		break;
-	case 8:
-		*exitinfo |= IOIO_ADDR_64;
-		break;
-	}
-
-	if (insn_has_rep_prefix(insn))
-		*exitinfo |= IOIO_REP;
-
-	return vc_ioio_check(ctxt, (u16)port, size);
-}
-
-static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
-{
-	struct pt_regs *regs = ctxt->regs;
-	u64 exit_info_1, exit_info_2;
-	enum es_result ret;
-
-	ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
-	if (ret != ES_OK)
-		return ret;
-
-	if (exit_info_1 & IOIO_TYPE_STR) {
-
-		/* (REP) INS/OUTS */
-
-		bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
-		unsigned int io_bytes, exit_bytes;
-		unsigned int ghcb_count, op_count;
-		unsigned long es_base;
-		u64 sw_scratch;
-
-		/*
-		 * For the string variants with rep prefix the amount of in/out
-		 * operations per #VC exception is limited so that the kernel
-		 * has a chance to take interrupts and re-schedule while the
-		 * instruction is emulated.
-		 */
-		io_bytes   = (exit_info_1 >> 4) & 0x7;
-		ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
-
-		op_count    = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
-		exit_info_2 = min(op_count, ghcb_count);
-		exit_bytes  = exit_info_2 * io_bytes;
-
-		es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
-
-		/* Read bytes of OUTS into the shared buffer */
-		if (!(exit_info_1 & IOIO_TYPE_IN)) {
-			ret = vc_insn_string_read(ctxt,
-					       (void *)(es_base + regs->si),
-					       ghcb->shared_buffer, io_bytes,
-					       exit_info_2, df);
-			if (ret)
-				return ret;
-		}
-
-		/*
-		 * Issue an VMGEXIT to the HV to consume the bytes from the
-		 * shared buffer or to have it write them into the shared buffer
-		 * depending on the instruction: OUTS or INS.
-		 */
-		sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
-		ghcb_set_sw_scratch(ghcb, sw_scratch);
-		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
-					  exit_info_1, exit_info_2);
-		if (ret != ES_OK)
-			return ret;
-
-		/* Read bytes from shared buffer into the guest's destination. */
-		if (exit_info_1 & IOIO_TYPE_IN) {
-			ret = vc_insn_string_write(ctxt,
-						   (void *)(es_base + regs->di),
-						   ghcb->shared_buffer, io_bytes,
-						   exit_info_2, df);
-			if (ret)
-				return ret;
-
-			if (df)
-				regs->di -= exit_bytes;
-			else
-				regs->di += exit_bytes;
-		} else {
-			if (df)
-				regs->si -= exit_bytes;
-			else
-				regs->si += exit_bytes;
-		}
-
-		if (exit_info_1 & IOIO_REP)
-			regs->cx -= exit_info_2;
-
-		ret = regs->cx ? ES_RETRY : ES_OK;
-
-	} else {
-
-		/* IN/OUT into/from rAX */
-
-		int bits = (exit_info_1 & 0x70) >> 1;
-		u64 rax = 0;
-
-		if (!(exit_info_1 & IOIO_TYPE_IN))
-			rax = lower_bits(regs->ax, bits);
-
-		ghcb_set_rax(ghcb, rax);
-
-		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
-		if (ret != ES_OK)
-			return ret;
-
-		if (exit_info_1 & IOIO_TYPE_IN) {
-			if (!ghcb_rax_is_valid(ghcb))
-				return ES_VMM_ERROR;
-			regs->ax = lower_bits(ghcb->save.rax, bits);
-		}
-	}
-
-	return ret;
-}
-
-static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
-{
-	struct pt_regs *regs = ctxt->regs;
-	struct cpuid_leaf leaf;
-	int ret;
-
-	leaf.fn = regs->ax;
-	leaf.subfn = regs->cx;
-	ret = snp_cpuid(ghcb, ctxt, &leaf);
-	if (!ret) {
-		regs->ax = leaf.eax;
-		regs->bx = leaf.ebx;
-		regs->cx = leaf.ecx;
-		regs->dx = leaf.edx;
-	}
-
-	return ret;
-}
-
-static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
-				      struct es_em_ctxt *ctxt)
-{
-	struct pt_regs *regs = ctxt->regs;
-	u32 cr4 = native_read_cr4();
-	enum es_result ret;
-	int snp_cpuid_ret;
-
-	snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt);
-	if (!snp_cpuid_ret)
-		return ES_OK;
-	if (snp_cpuid_ret != -EOPNOTSUPP)
-		return ES_VMM_ERROR;
-
-	ghcb_set_rax(ghcb, regs->ax);
-	ghcb_set_rcx(ghcb, regs->cx);
-
-	if (cr4 & X86_CR4_OSXSAVE)
-		/* Safe to read xcr0 */
-		ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
-	else
-		/* xgetbv will cause #GP - use reset value for xcr0 */
-		ghcb_set_xcr0(ghcb, 1);
-
-	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
-	if (ret != ES_OK)
-		return ret;
-
-	if (!(ghcb_rax_is_valid(ghcb) &&
-	      ghcb_rbx_is_valid(ghcb) &&
-	      ghcb_rcx_is_valid(ghcb) &&
-	      ghcb_rdx_is_valid(ghcb)))
-		return ES_VMM_ERROR;
-
-	regs->ax = ghcb->save.rax;
-	regs->bx = ghcb->save.rbx;
-	regs->cx = ghcb->save.rcx;
-	regs->dx = ghcb->save.rdx;
-
-	return ES_OK;
-}
-
-static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
-				      struct es_em_ctxt *ctxt,
-				      unsigned long exit_code)
-{
-	bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
-	enum es_result ret;
-
-	ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
-	if (ret != ES_OK)
-		return ret;
-
-	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
-	     (!rdtscp || ghcb_rcx_is_valid(ghcb))))
-		return ES_VMM_ERROR;
-
-	ctxt->regs->ax = ghcb->save.rax;
-	ctxt->regs->dx = ghcb->save.rdx;
-	if (rdtscp)
-		ctxt->regs->cx = ghcb->save.rcx;
-
-	return ES_OK;
-}
-
-struct cc_setup_data {
-	struct setup_data header;
-	u32 cc_blob_address;
-};
-
-/*
- * Search for a Confidential Computing blob passed in as a setup_data entry
- * via the Linux Boot Protocol.
- */
-static __head
-struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp)
-{
-	struct cc_setup_data *sd = NULL;
-	struct setup_data *hdr;
-
-	hdr = (struct setup_data *)bp->hdr.setup_data;
-
-	while (hdr) {
-		if (hdr->type == SETUP_CC_BLOB) {
-			sd = (struct cc_setup_data *)hdr;
-			return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address;
-		}
-		hdr = (struct setup_data *)hdr->next;
-	}
-
-	return NULL;
-}
-
-/*
- * Initialize the kernel's copy of the SNP CPUID table, and set up the
- * pointer that will be used to access it.
- *
- * Maintaining a direct mapping of the SNP CPUID table used by firmware would
- * be possible as an alternative, but the approach is brittle since the
- * mapping needs to be updated in sync with all the changes to virtual memory
- * layout and related mapping facilities throughout the boot process.
- */
-static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
-{
-	const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table;
-	int i;
-
-	if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE)
-		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
-
-	cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys;
-	if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX)
-		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
-
-	cpuid_table = snp_cpuid_get_table();
-	memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table));
-
-	/* Initialize CPUID ranges for range-checking. */
-	for (i = 0; i < cpuid_table->count; i++) {
-		const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
-
-		if (fn->eax_in == 0x0)
-			RIP_REL_REF(cpuid_std_range_max) = fn->eax;
-		else if (fn->eax_in == 0x40000000)
-			RIP_REL_REF(cpuid_hyp_range_max) = fn->eax;
-		else if (fn->eax_in == 0x80000000)
-			RIP_REL_REF(cpuid_ext_range_max) = fn->eax;
-	}
-}
-
-static inline void __pval_terminate(u64 pfn, bool action, unsigned int page_size,
-				    int ret, u64 svsm_ret)
-{
-	WARN(1, "PVALIDATE failure: pfn: 0x%llx, action: %u, size: %u, ret: %d, svsm_ret: 0x%llx\n",
-	     pfn, action, page_size, ret, svsm_ret);
-
-	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
-}
-
-static void svsm_pval_terminate(struct svsm_pvalidate_call *pc, int ret, u64 svsm_ret)
-{
-	unsigned int page_size;
-	bool action;
-	u64 pfn;
-
-	pfn = pc->entry[pc->cur_index].pfn;
-	action = pc->entry[pc->cur_index].action;
-	page_size = pc->entry[pc->cur_index].page_size;
-
-	__pval_terminate(pfn, action, page_size, ret, svsm_ret);
-}
-
-static void svsm_pval_4k_page(unsigned long paddr, bool validate)
-{
-	struct svsm_pvalidate_call *pc;
-	struct svsm_call call = {};
-	unsigned long flags;
-	u64 pc_pa;
-	int ret;
-
-	/*
-	 * This can be called very early in the boot, use native functions in
-	 * order to avoid paravirt issues.
-	 */
-	flags = native_local_irq_save();
-
-	call.caa = svsm_get_caa();
-
-	pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer;
-	pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer);
-
-	pc->num_entries = 1;
-	pc->cur_index   = 0;
-	pc->entry[0].page_size = RMP_PG_SIZE_4K;
-	pc->entry[0].action    = validate;
-	pc->entry[0].ignore_cf = 0;
-	pc->entry[0].pfn       = paddr >> PAGE_SHIFT;
-
-	/* Protocol 0, Call ID 1 */
-	call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE);
-	call.rcx = pc_pa;
-
-	ret = svsm_perform_call_protocol(&call);
-	if (ret)
-		svsm_pval_terminate(pc, ret, call.rax_out);
-
-	native_local_irq_restore(flags);
-}
-
-static void pvalidate_4k_page(unsigned long vaddr, unsigned long paddr, bool validate)
-{
-	int ret;
-
-	/*
-	 * This can be called very early during boot, so use rIP-relative
-	 * references as needed.
-	 */
-	if (RIP_REL_REF(snp_vmpl)) {
-		svsm_pval_4k_page(paddr, validate);
-	} else {
-		ret = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
-		if (ret)
-			__pval_terminate(PHYS_PFN(paddr), validate, RMP_PG_SIZE_4K, ret, 0);
-	}
-}
-
-static void pval_pages(struct snp_psc_desc *desc)
-{
-	struct psc_entry *e;
-	unsigned long vaddr;
-	unsigned int size;
-	unsigned int i;
-	bool validate;
-	u64 pfn;
-	int rc;
-
-	for (i = 0; i <= desc->hdr.end_entry; i++) {
-		e = &desc->entries[i];
-
-		pfn = e->gfn;
-		vaddr = (unsigned long)pfn_to_kaddr(pfn);
-		size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
-		validate = e->operation == SNP_PAGE_STATE_PRIVATE;
-
-		rc = pvalidate(vaddr, size, validate);
-		if (!rc)
-			continue;
-
-		if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) {
-			unsigned long vaddr_end = vaddr + PMD_SIZE;
-
-			for (; vaddr < vaddr_end; vaddr += PAGE_SIZE, pfn++) {
-				rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
-				if (rc)
-					__pval_terminate(pfn, validate, RMP_PG_SIZE_4K, rc, 0);
-			}
-		} else {
-			__pval_terminate(pfn, validate, size, rc, 0);
-		}
-	}
-}
-
-static u64 svsm_build_ca_from_pfn_range(u64 pfn, u64 pfn_end, bool action,
-					struct svsm_pvalidate_call *pc)
-{
-	struct svsm_pvalidate_entry *pe;
-
-	/* Nothing in the CA yet */
-	pc->num_entries = 0;
-	pc->cur_index   = 0;
-
-	pe = &pc->entry[0];
-
-	while (pfn < pfn_end) {
-		pe->page_size = RMP_PG_SIZE_4K;
-		pe->action    = action;
-		pe->ignore_cf = 0;
-		pe->pfn       = pfn;
-
-		pe++;
-		pfn++;
-
-		pc->num_entries++;
-		if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
-			break;
-	}
-
-	return pfn;
-}
-
-static int svsm_build_ca_from_psc_desc(struct snp_psc_desc *desc, unsigned int desc_entry,
-				       struct svsm_pvalidate_call *pc)
-{
-	struct svsm_pvalidate_entry *pe;
-	struct psc_entry *e;
-
-	/* Nothing in the CA yet */
-	pc->num_entries = 0;
-	pc->cur_index   = 0;
-
-	pe = &pc->entry[0];
-	e  = &desc->entries[desc_entry];
-
-	while (desc_entry <= desc->hdr.end_entry) {
-		pe->page_size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
-		pe->action    = e->operation == SNP_PAGE_STATE_PRIVATE;
-		pe->ignore_cf = 0;
-		pe->pfn       = e->gfn;
-
-		pe++;
-		e++;
-
-		desc_entry++;
-		pc->num_entries++;
-		if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
-			break;
-	}
-
-	return desc_entry;
-}
-
-static void svsm_pval_pages(struct snp_psc_desc *desc)
-{
-	struct svsm_pvalidate_entry pv_4k[VMGEXIT_PSC_MAX_ENTRY];
-	unsigned int i, pv_4k_count = 0;
-	struct svsm_pvalidate_call *pc;
-	struct svsm_call call = {};
-	unsigned long flags;
-	bool action;
-	u64 pc_pa;
-	int ret;
-
-	/*
-	 * This can be called very early in the boot, use native functions in
-	 * order to avoid paravirt issues.
-	 */
-	flags = native_local_irq_save();
-
-	/*
-	 * The SVSM calling area (CA) can support processing 510 entries at a
-	 * time. Loop through the Page State Change descriptor until the CA is
-	 * full or the last entry in the descriptor is reached, at which time
-	 * the SVSM is invoked. This repeats until all entries in the descriptor
-	 * are processed.
-	 */
-	call.caa = svsm_get_caa();
-
-	pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer;
-	pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer);
-
-	/* Protocol 0, Call ID 1 */
-	call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE);
-	call.rcx = pc_pa;
-
-	for (i = 0; i <= desc->hdr.end_entry;) {
-		i = svsm_build_ca_from_psc_desc(desc, i, pc);
-
-		do {
-			ret = svsm_perform_call_protocol(&call);
-			if (!ret)
-				continue;
-
-			/*
-			 * Check if the entry failed because of an RMP mismatch (a
-			 * PVALIDATE at 2M was requested, but the page is mapped in
-			 * the RMP as 4K).
-			 */
-
-			if (call.rax_out == SVSM_PVALIDATE_FAIL_SIZEMISMATCH &&
-			    pc->entry[pc->cur_index].page_size == RMP_PG_SIZE_2M) {
-				/* Save this entry for post-processing at 4K */
-				pv_4k[pv_4k_count++] = pc->entry[pc->cur_index];
-
-				/* Skip to the next one unless at the end of the list */
-				pc->cur_index++;
-				if (pc->cur_index < pc->num_entries)
-					ret = -EAGAIN;
-				else
-					ret = 0;
-			}
-		} while (ret == -EAGAIN);
-
-		if (ret)
-			svsm_pval_terminate(pc, ret, call.rax_out);
-	}
-
-	/* Process any entries that failed to be validated at 2M and validate them at 4K */
-	for (i = 0; i < pv_4k_count; i++) {
-		u64 pfn, pfn_end;
-
-		action  = pv_4k[i].action;
-		pfn     = pv_4k[i].pfn;
-		pfn_end = pfn + 512;
-
-		while (pfn < pfn_end) {
-			pfn = svsm_build_ca_from_pfn_range(pfn, pfn_end, action, pc);
-
-			ret = svsm_perform_call_protocol(&call);
-			if (ret)
-				svsm_pval_terminate(pc, ret, call.rax_out);
-		}
-	}
-
-	native_local_irq_restore(flags);
-}
-
-static void pvalidate_pages(struct snp_psc_desc *desc)
-{
-	if (snp_vmpl)
-		svsm_pval_pages(desc);
-	else
-		pval_pages(desc);
-}
-
-static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
-{
-	int cur_entry, end_entry, ret = 0;
-	struct snp_psc_desc *data;
-	struct es_em_ctxt ctxt;
-
-	vc_ghcb_invalidate(ghcb);
-
-	/* Copy the input desc into GHCB shared buffer */
-	data = (struct snp_psc_desc *)ghcb->shared_buffer;
-	memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
-
-	/*
-	 * As per the GHCB specification, the hypervisor can resume the guest
-	 * before processing all the entries. Check whether all the entries
-	 * are processed. If not, then keep retrying. Note, the hypervisor
-	 * will update the data memory directly to indicate the status, so
-	 * reference the data->hdr everywhere.
-	 *
-	 * The strategy here is to wait for the hypervisor to change the page
-	 * state in the RMP table before guest accesses the memory pages. If the
-	 * page state change was not successful, then later memory access will
-	 * result in a crash.
-	 */
-	cur_entry = data->hdr.cur_entry;
-	end_entry = data->hdr.end_entry;
-
-	while (data->hdr.cur_entry <= data->hdr.end_entry) {
-		ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
-
-		/* This will advance the shared buffer data points to. */
-		ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
-
-		/*
-		 * Page State Change VMGEXIT can pass error code through
-		 * exit_info_2.
-		 */
-		if (WARN(ret || ghcb->save.sw_exit_info_2,
-			 "SNP: PSC failed ret=%d exit_info_2=%llx\n",
-			 ret, ghcb->save.sw_exit_info_2)) {
-			ret = 1;
-			goto out;
-		}
-
-		/* Verify that reserved bit is not set */
-		if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
-			ret = 1;
-			goto out;
-		}
-
-		/*
-		 * Sanity check that entry processing is not going backwards.
-		 * This will happen only if hypervisor is tricking us.
-		 */
-		if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry,
-"SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
-			 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
-			ret = 1;
-			goto out;
-		}
-	}
-
-out:
-	return ret;
-}
-
-static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt,
-					    unsigned long exit_code)
-{
-	unsigned int opcode = (unsigned int)ctxt->insn.opcode.value;
-	u8 modrm = ctxt->insn.modrm.value;
-
-	switch (exit_code) {
-
-	case SVM_EXIT_IOIO:
-	case SVM_EXIT_NPF:
-		/* handled separately */
-		return ES_OK;
-
-	case SVM_EXIT_CPUID:
-		if (opcode == 0xa20f)
-			return ES_OK;
-		break;
-
-	case SVM_EXIT_INVD:
-		if (opcode == 0x080f)
-			return ES_OK;
-		break;
-
-	case SVM_EXIT_MONITOR:
-		/* MONITOR and MONITORX instructions generate the same error code */
-		if (opcode == 0x010f && (modrm == 0xc8 || modrm == 0xfa))
-			return ES_OK;
-		break;
-
-	case SVM_EXIT_MWAIT:
-		/* MWAIT and MWAITX instructions generate the same error code */
-		if (opcode == 0x010f && (modrm == 0xc9 || modrm == 0xfb))
-			return ES_OK;
-		break;
-
-	case SVM_EXIT_MSR:
-		/* RDMSR */
-		if (opcode == 0x320f ||
-		/* WRMSR */
-		    opcode == 0x300f)
-			return ES_OK;
-		break;
-
-	case SVM_EXIT_RDPMC:
-		if (opcode == 0x330f)
-			return ES_OK;
-		break;
-
-	case SVM_EXIT_RDTSC:
-		if (opcode == 0x310f)
-			return ES_OK;
-		break;
-
-	case SVM_EXIT_RDTSCP:
-		if (opcode == 0x010f && modrm == 0xf9)
-			return ES_OK;
-		break;
-
-	case SVM_EXIT_READ_DR7:
-		if (opcode == 0x210f &&
-		    X86_MODRM_REG(ctxt->insn.modrm.value) == 7)
-			return ES_OK;
-		break;
-
-	case SVM_EXIT_VMMCALL:
-		if (opcode == 0x010f && modrm == 0xd9)
-			return ES_OK;
-
-		break;
-
-	case SVM_EXIT_WRITE_DR7:
-		if (opcode == 0x230f &&
-		    X86_MODRM_REG(ctxt->insn.modrm.value) == 7)
-			return ES_OK;
-		break;
-
-	case SVM_EXIT_WBINVD:
-		if (opcode == 0x90f)
-			return ES_OK;
-		break;
-
-	default:
-		break;
-	}
-
-	sev_printk(KERN_ERR "Wrong/unhandled opcode bytes: 0x%x, exit_code: 0x%lx, rIP: 0x%lx\n",
-		   opcode, exit_code, ctxt->regs->ip);
-
-	return ES_UNSUPPORTED;
-}
-
-/*
- * Maintain the GPA of the SVSM Calling Area (CA) in order to utilize the SVSM
- * services needed when not running in VMPL0.
- */
-static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info)
-{
-	struct snp_secrets_page *secrets_page;
-	struct snp_cpuid_table *cpuid_table;
-	unsigned int i;
-	u64 caa;
-
-	BUILD_BUG_ON(sizeof(*secrets_page) != PAGE_SIZE);
-
-	/*
-	 * Check if running at VMPL0.
-	 *
-	 * Use RMPADJUST (see the rmpadjust() function for a description of what
-	 * the instruction does) to update the VMPL1 permissions of a page. If
-	 * the guest is running at VMPL0, this will succeed and implies there is
-	 * no SVSM. If the guest is running at any other VMPL, this will fail.
-	 * Linux SNP guests only ever run at a single VMPL level so permission mask
-	 * changes of a lesser-privileged VMPL are a don't-care.
-	 *
-	 * Use a rip-relative reference to obtain the proper address, since this
-	 * routine is running identity mapped when called, both by the decompressor
-	 * code and the early kernel code.
-	 */
-	if (!rmpadjust((unsigned long)&RIP_REL_REF(boot_ghcb_page), RMP_PG_SIZE_4K, 1))
-		return false;
-
-	/*
-	 * Not running at VMPL0, ensure everything has been properly supplied
-	 * for running under an SVSM.
-	 */
-	if (!cc_info || !cc_info->secrets_phys || cc_info->secrets_len != PAGE_SIZE)
-		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SECRETS_PAGE);
-
-	secrets_page = (struct snp_secrets_page *)cc_info->secrets_phys;
-	if (!secrets_page->svsm_size)
-		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NO_SVSM);
-
-	if (!secrets_page->svsm_guest_vmpl)
-		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_VMPL0);
-
-	RIP_REL_REF(snp_vmpl) = secrets_page->svsm_guest_vmpl;
-
-	caa = secrets_page->svsm_caa;
-
-	/*
-	 * An open-coded PAGE_ALIGNED() in order to avoid including
-	 * kernel-proper headers into the decompressor.
-	 */
-	if (caa & (PAGE_SIZE - 1))
-		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CAA);
-
-	/*
-	 * The CA is identity mapped when this routine is called, both by the
-	 * decompressor code and the early kernel code.
-	 */
-	RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)caa;
-	RIP_REL_REF(boot_svsm_caa_pa) = caa;
-
-	/* Advertise the SVSM presence via CPUID. */
-	cpuid_table = (struct snp_cpuid_table *)snp_cpuid_get_table();
-	for (i = 0; i < cpuid_table->count; i++) {
-		struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
-
-		if (fn->eax_in == 0x8000001f)
-			fn->eax |= BIT(28);
-	}
-
-	return true;
-}
diff --git a/arch/x86/coco/sev/vc-handle.c b/arch/x86/coco/sev/vc-handle.c
new file mode 100644
index 000000000000..f08c7505ed82
--- /dev/null
+++ b/arch/x86/coco/sev/vc-handle.c
@@ -0,0 +1,1080 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AMD Memory Encryption Support
+ *
+ * Copyright (C) 2019 SUSE
+ *
+ * Author: Joerg Roedel <jroedel@suse.de>
+ */
+
+#define pr_fmt(fmt)	"SEV: " fmt
+
+#include <linux/sched/debug.h>	/* For show_regs() */
+#include <linux/cc_platform.h>
+#include <linux/printk.h>
+#include <linux/mm_types.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/io.h>
+#include <linux/psp-sev.h>
+#include <linux/efi.h>
+#include <uapi/linux/sev-guest.h>
+
+#include <asm/init.h>
+#include <asm/stacktrace.h>
+#include <asm/sev.h>
+#include <asm/sev-internal.h>
+#include <asm/insn-eval.h>
+#include <asm/fpu/xcr.h>
+#include <asm/processor.h>
+#include <asm/setup.h>
+#include <asm/traps.h>
+#include <asm/svm.h>
+#include <asm/smp.h>
+#include <asm/cpu.h>
+#include <asm/apic.h>
+#include <asm/cpuid/api.h>
+
+static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
+					   unsigned long vaddr, phys_addr_t *paddr)
+{
+	unsigned long va = (unsigned long)vaddr;
+	unsigned int level;
+	phys_addr_t pa;
+	pgd_t *pgd;
+	pte_t *pte;
+
+	pgd = __va(read_cr3_pa());
+	pgd = &pgd[pgd_index(va)];
+	pte = lookup_address_in_pgd(pgd, va, &level);
+	if (!pte) {
+		ctxt->fi.vector     = X86_TRAP_PF;
+		ctxt->fi.cr2        = vaddr;
+		ctxt->fi.error_code = 0;
+
+		if (user_mode(ctxt->regs))
+			ctxt->fi.error_code |= X86_PF_USER;
+
+		return ES_EXCEPTION;
+	}
+
+	if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC))
+		/* Emulated MMIO to/from encrypted memory not supported */
+		return ES_UNSUPPORTED;
+
+	pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
+	pa |= va & ~page_level_mask(level);
+
+	*paddr = pa;
+
+	return ES_OK;
+}
+
+static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
+{
+	BUG_ON(size > 4);
+
+	if (user_mode(ctxt->regs)) {
+		struct thread_struct *t = &current->thread;
+		struct io_bitmap *iobm = t->io_bitmap;
+		size_t idx;
+
+		if (!iobm)
+			goto fault;
+
+		for (idx = port; idx < port + size; ++idx) {
+			if (test_bit(idx, iobm->bitmap))
+				goto fault;
+		}
+	}
+
+	return ES_OK;
+
+fault:
+	ctxt->fi.vector = X86_TRAP_GP;
+	ctxt->fi.error_code = 0;
+
+	return ES_EXCEPTION;
+}
+
+void vc_forward_exception(struct es_em_ctxt *ctxt)
+{
+	long error_code = ctxt->fi.error_code;
+	int trapnr = ctxt->fi.vector;
+
+	ctxt->regs->orig_ax = ctxt->fi.error_code;
+
+	switch (trapnr) {
+	case X86_TRAP_GP:
+		exc_general_protection(ctxt->regs, error_code);
+		break;
+	case X86_TRAP_UD:
+		exc_invalid_op(ctxt->regs);
+		break;
+	case X86_TRAP_PF:
+		write_cr2(ctxt->fi.cr2);
+		exc_page_fault(ctxt->regs, error_code);
+		break;
+	case X86_TRAP_AC:
+		exc_alignment_check(ctxt->regs, error_code);
+		break;
+	default:
+		pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n");
+		BUG();
+	}
+}
+
+static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,
+				unsigned char *buffer)
+{
+	return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
+}
+
+static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt)
+{
+	char buffer[MAX_INSN_SIZE];
+	int insn_bytes;
+
+	insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer);
+	if (insn_bytes == 0) {
+		/* Nothing could be copied */
+		ctxt->fi.vector     = X86_TRAP_PF;
+		ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER;
+		ctxt->fi.cr2        = ctxt->regs->ip;
+		return ES_EXCEPTION;
+	} else if (insn_bytes == -EINVAL) {
+		/* Effective RIP could not be calculated */
+		ctxt->fi.vector     = X86_TRAP_GP;
+		ctxt->fi.error_code = 0;
+		ctxt->fi.cr2        = 0;
+		return ES_EXCEPTION;
+	}
+
+	if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes))
+		return ES_DECODE_FAILED;
+
+	if (ctxt->insn.immediate.got)
+		return ES_OK;
+	else
+		return ES_DECODE_FAILED;
+}
+
+static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt)
+{
+	char buffer[MAX_INSN_SIZE];
+	int res, ret;
+
+	res = vc_fetch_insn_kernel(ctxt, buffer);
+	if (res) {
+		ctxt->fi.vector     = X86_TRAP_PF;
+		ctxt->fi.error_code = X86_PF_INSTR;
+		ctxt->fi.cr2        = ctxt->regs->ip;
+		return ES_EXCEPTION;
+	}
+
+	ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
+	if (ret < 0)
+		return ES_DECODE_FAILED;
+	else
+		return ES_OK;
+}
+
+/*
+ * User instruction decoding is also required for the EFI runtime. Even though
+ * the EFI runtime is running in kernel mode, it uses special EFI virtual
+ * address mappings that require the use of efi_mm to properly address and
+ * decode.
+ */
+static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
+{
+	if (user_mode(ctxt->regs) || mm_is_efi(current->active_mm))
+		return __vc_decode_user_insn(ctxt);
+	else
+		return __vc_decode_kern_insn(ctxt);
+}
+
+static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
+				   char *dst, char *buf, size_t size)
+{
+	unsigned long error_code = X86_PF_PROT | X86_PF_WRITE;
+
+	/*
+	 * This function uses __put_user() independent of whether kernel or user
+	 * memory is accessed. This works fine because __put_user() does no
+	 * sanity checks of the pointer being accessed. All that it does is
+	 * to report when the access failed.
+	 *
+	 * Also, this function runs in atomic context, so __put_user() is not
+	 * allowed to sleep. The page-fault handler detects that it is running
+	 * in atomic context and will not try to take mmap_sem and handle the
+	 * fault, so additional pagefault_enable()/disable() calls are not
+	 * needed.
+	 *
+	 * The access can't be done via copy_to_user() here because
+	 * vc_write_mem() must not use string instructions to access unsafe
+	 * memory. The reason is that MOVS is emulated by the #VC handler by
+	 * splitting the move up into a read and a write and taking a nested #VC
+	 * exception on whatever of them is the MMIO access. Using string
+	 * instructions here would cause infinite nesting.
+	 */
+	switch (size) {
+	case 1: {
+		u8 d1;
+		u8 __user *target = (u8 __user *)dst;
+
+		memcpy(&d1, buf, 1);
+		if (__put_user(d1, target))
+			goto fault;
+		break;
+	}
+	case 2: {
+		u16 d2;
+		u16 __user *target = (u16 __user *)dst;
+
+		memcpy(&d2, buf, 2);
+		if (__put_user(d2, target))
+			goto fault;
+		break;
+	}
+	case 4: {
+		u32 d4;
+		u32 __user *target = (u32 __user *)dst;
+
+		memcpy(&d4, buf, 4);
+		if (__put_user(d4, target))
+			goto fault;
+		break;
+	}
+	case 8: {
+		u64 d8;
+		u64 __user *target = (u64 __user *)dst;
+
+		memcpy(&d8, buf, 8);
+		if (__put_user(d8, target))
+			goto fault;
+		break;
+	}
+	default:
+		WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
+		return ES_UNSUPPORTED;
+	}
+
+	return ES_OK;
+
+fault:
+	if (user_mode(ctxt->regs))
+		error_code |= X86_PF_USER;
+
+	ctxt->fi.vector = X86_TRAP_PF;
+	ctxt->fi.error_code = error_code;
+	ctxt->fi.cr2 = (unsigned long)dst;
+
+	return ES_EXCEPTION;
+}
+
+static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
+				  char *src, char *buf, size_t size)
+{
+	unsigned long error_code = X86_PF_PROT;
+
+	/*
+	 * This function uses __get_user() independent of whether kernel or user
+	 * memory is accessed. This works fine because __get_user() does no
+	 * sanity checks of the pointer being accessed. All that it does is
+	 * to report when the access failed.
+	 *
+	 * Also, this function runs in atomic context, so __get_user() is not
+	 * allowed to sleep. The page-fault handler detects that it is running
+	 * in atomic context and will not try to take mmap_sem and handle the
+	 * fault, so additional pagefault_enable()/disable() calls are not
+	 * needed.
+	 *
+	 * The access can't be done via copy_from_user() here because
+	 * vc_read_mem() must not use string instructions to access unsafe
+	 * memory. The reason is that MOVS is emulated by the #VC handler by
+	 * splitting the move up into a read and a write and taking a nested #VC
+	 * exception on whatever of them is the MMIO access. Using string
+	 * instructions here would cause infinite nesting.
+	 */
+	switch (size) {
+	case 1: {
+		u8 d1;
+		u8 __user *s = (u8 __user *)src;
+
+		if (__get_user(d1, s))
+			goto fault;
+		memcpy(buf, &d1, 1);
+		break;
+	}
+	case 2: {
+		u16 d2;
+		u16 __user *s = (u16 __user *)src;
+
+		if (__get_user(d2, s))
+			goto fault;
+		memcpy(buf, &d2, 2);
+		break;
+	}
+	case 4: {
+		u32 d4;
+		u32 __user *s = (u32 __user *)src;
+
+		if (__get_user(d4, s))
+			goto fault;
+		memcpy(buf, &d4, 4);
+		break;
+	}
+	case 8: {
+		u64 d8;
+		u64 __user *s = (u64 __user *)src;
+		if (__get_user(d8, s))
+			goto fault;
+		memcpy(buf, &d8, 8);
+		break;
+	}
+	default:
+		WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
+		return ES_UNSUPPORTED;
+	}
+
+	return ES_OK;
+
+fault:
+	if (user_mode(ctxt->regs))
+		error_code |= X86_PF_USER;
+
+	ctxt->fi.vector = X86_TRAP_PF;
+	ctxt->fi.error_code = error_code;
+	ctxt->fi.cr2 = (unsigned long)src;
+
+	return ES_EXCEPTION;
+}
+
+#define sev_printk(fmt, ...)		printk(fmt, ##__VA_ARGS__)
+#define error(v)
+
+#include "vc-shared.c"
+
+/* Writes to the SVSM CAA MSR are ignored */
+static enum es_result __vc_handle_msr_caa(struct pt_regs *regs, bool write)
+{
+	if (write)
+		return ES_OK;
+
+	regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa));
+	regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa));
+
+	return ES_OK;
+}
+
+/*
+ * TSC related accesses should not exit to the hypervisor when a guest is
+ * executing with Secure TSC enabled, so special handling is required for
+ * accesses of MSR_IA32_TSC and MSR_AMD64_GUEST_TSC_FREQ.
+ */
+static enum es_result __vc_handle_secure_tsc_msrs(struct es_em_ctxt *ctxt, bool write)
+{
+	struct pt_regs *regs = ctxt->regs;
+	u64 tsc;
+
+	/*
+	 * Writing to MSR_IA32_TSC can cause subsequent reads of the TSC to
+	 * return undefined values, and GUEST_TSC_FREQ is read-only. Generate
+	 * a #GP on all writes.
+	 */
+	if (write) {
+		ctxt->fi.vector = X86_TRAP_GP;
+		ctxt->fi.error_code = 0;
+		return ES_EXCEPTION;
+	}
+
+	/*
+	 * GUEST_TSC_FREQ read should not be intercepted when Secure TSC is
+	 * enabled. Terminate the guest if a read is attempted.
+	 */
+	if (regs->cx == MSR_AMD64_GUEST_TSC_FREQ)
+		return ES_VMM_ERROR;
+
+	/* Reads of MSR_IA32_TSC should return the current TSC value. */
+	tsc = rdtsc_ordered();
+	regs->ax = lower_32_bits(tsc);
+	regs->dx = upper_32_bits(tsc);
+
+	return ES_OK;
+}
+
+enum es_result sev_es_ghcb_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt, bool write)
+{
+	struct pt_regs *regs = ctxt->regs;
+	enum es_result ret;
+
+	switch (regs->cx) {
+	case MSR_SVSM_CAA:
+		return __vc_handle_msr_caa(regs, write);
+	case MSR_IA32_TSC:
+	case MSR_AMD64_GUEST_TSC_FREQ:
+		if (sev_status & MSR_AMD64_SNP_SECURE_TSC)
+			return __vc_handle_secure_tsc_msrs(ctxt, write);
+		break;
+	case MSR_AMD64_SAVIC_CONTROL:
+		/*
+		 * AMD64_SAVIC_CONTROL should not be intercepted when
+		 * Secure AVIC is enabled. Terminate the Secure AVIC guest
+		 * if the interception is enabled.
+		 */
+		if (cc_platform_has(CC_ATTR_SNP_SECURE_AVIC))
+			return ES_VMM_ERROR;
+		break;
+	default:
+		break;
+	}
+
+	ghcb_set_rcx(ghcb, regs->cx);
+	if (write) {
+		ghcb_set_rax(ghcb, regs->ax);
+		ghcb_set_rdx(ghcb, regs->dx);
+	}
+
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, write, 0);
+
+	if ((ret == ES_OK) && !write) {
+		regs->ax = ghcb->save.rax;
+		regs->dx = ghcb->save.rdx;
+	}
+
+	return ret;
+}
+
+static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+{
+	return sev_es_ghcb_handle_msr(ghcb, ctxt, ctxt->insn.opcode.bytes[1] == 0x30);
+}
+
+static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt)
+{
+	int trapnr = ctxt->fi.vector;
+
+	if (trapnr == X86_TRAP_PF)
+		native_write_cr2(ctxt->fi.cr2);
+
+	ctxt->regs->orig_ax = ctxt->fi.error_code;
+	do_early_exception(ctxt->regs, trapnr);
+}
+
+static long *vc_insn_get_rm(struct es_em_ctxt *ctxt)
+{
+	long *reg_array;
+	int offset;
+
+	reg_array = (long *)ctxt->regs;
+	offset    = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs);
+
+	if (offset < 0)
+		return NULL;
+
+	offset /= sizeof(long);
+
+	return reg_array + offset;
+}
+static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
+				 unsigned int bytes, bool read)
+{
+	u64 exit_code, exit_info_1, exit_info_2;
+	unsigned long ghcb_pa = __pa(ghcb);
+	enum es_result res;
+	phys_addr_t paddr;
+	void __user *ref;
+
+	ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs);
+	if (ref == (void __user *)-1L)
+		return ES_UNSUPPORTED;
+
+	exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE;
+
+	res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr);
+	if (res != ES_OK) {
+		if (res == ES_EXCEPTION && !read)
+			ctxt->fi.error_code |= X86_PF_WRITE;
+
+		return res;
+	}
+
+	exit_info_1 = paddr;
+	/* Can never be greater than 8 */
+	exit_info_2 = bytes;
+
+	ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer));
+
+	return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2);
+}
+
+/*
+ * The MOVS instruction has two memory operands, which raises the
+ * problem that it is not known whether the access to the source or the
+ * destination caused the #VC exception (and hence whether an MMIO read
+ * or write operation needs to be emulated).
+ *
+ * Instead of playing games with walking page-tables and trying to guess
+ * whether the source or destination is an MMIO range, split the move
+ * into two operations, a read and a write with only one memory operand.
+ * This will cause a nested #VC exception on the MMIO address which can
+ * then be handled.
+ *
+ * This implementation has the benefit that it also supports MOVS where
+ * source _and_ destination are MMIO regions.
+ *
+ * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a
+ * rare operation. If it turns out to be a performance problem the split
+ * operations can be moved to memcpy_fromio() and memcpy_toio().
+ */
+static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt,
+					  unsigned int bytes)
+{
+	unsigned long ds_base, es_base;
+	unsigned char *src, *dst;
+	unsigned char buffer[8];
+	enum es_result ret;
+	bool rep;
+	int off;
+
+	ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS);
+	es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
+
+	if (ds_base == -1L || es_base == -1L) {
+		ctxt->fi.vector = X86_TRAP_GP;
+		ctxt->fi.error_code = 0;
+		return ES_EXCEPTION;
+	}
+
+	src = ds_base + (unsigned char *)ctxt->regs->si;
+	dst = es_base + (unsigned char *)ctxt->regs->di;
+
+	ret = vc_read_mem(ctxt, src, buffer, bytes);
+	if (ret != ES_OK)
+		return ret;
+
+	ret = vc_write_mem(ctxt, dst, buffer, bytes);
+	if (ret != ES_OK)
+		return ret;
+
+	if (ctxt->regs->flags & X86_EFLAGS_DF)
+		off = -bytes;
+	else
+		off =  bytes;
+
+	ctxt->regs->si += off;
+	ctxt->regs->di += off;
+
+	rep = insn_has_rep_prefix(&ctxt->insn);
+	if (rep)
+		ctxt->regs->cx -= 1;
+
+	if (!rep || ctxt->regs->cx == 0)
+		return ES_OK;
+	else
+		return ES_RETRY;
+}
+
+static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+{
+	struct insn *insn = &ctxt->insn;
+	enum insn_mmio_type mmio;
+	unsigned int bytes = 0;
+	enum es_result ret;
+	u8 sign_byte;
+	long *reg_data;
+
+	mmio = insn_decode_mmio(insn, &bytes);
+	if (mmio == INSN_MMIO_DECODE_FAILED)
+		return ES_DECODE_FAILED;
+
+	if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) {
+		reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs);
+		if (!reg_data)
+			return ES_DECODE_FAILED;
+	}
+
+	if (user_mode(ctxt->regs))
+		return ES_UNSUPPORTED;
+
+	switch (mmio) {
+	case INSN_MMIO_WRITE:
+		memcpy(ghcb->shared_buffer, reg_data, bytes);
+		ret = vc_do_mmio(ghcb, ctxt, bytes, false);
+		break;
+	case INSN_MMIO_WRITE_IMM:
+		memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes);
+		ret = vc_do_mmio(ghcb, ctxt, bytes, false);
+		break;
+	case INSN_MMIO_READ:
+		ret = vc_do_mmio(ghcb, ctxt, bytes, true);
+		if (ret)
+			break;
+
+		/* Zero-extend for 32-bit operation */
+		if (bytes == 4)
+			*reg_data = 0;
+
+		memcpy(reg_data, ghcb->shared_buffer, bytes);
+		break;
+	case INSN_MMIO_READ_ZERO_EXTEND:
+		ret = vc_do_mmio(ghcb, ctxt, bytes, true);
+		if (ret)
+			break;
+
+		/* Zero extend based on operand size */
+		memset(reg_data, 0, insn->opnd_bytes);
+		memcpy(reg_data, ghcb->shared_buffer, bytes);
+		break;
+	case INSN_MMIO_READ_SIGN_EXTEND:
+		ret = vc_do_mmio(ghcb, ctxt, bytes, true);
+		if (ret)
+			break;
+
+		if (bytes == 1) {
+			u8 *val = (u8 *)ghcb->shared_buffer;
+
+			sign_byte = (*val & 0x80) ? 0xff : 0x00;
+		} else {
+			u16 *val = (u16 *)ghcb->shared_buffer;
+
+			sign_byte = (*val & 0x8000) ? 0xff : 0x00;
+		}
+
+		/* Sign extend based on operand size */
+		memset(reg_data, sign_byte, insn->opnd_bytes);
+		memcpy(reg_data, ghcb->shared_buffer, bytes);
+		break;
+	case INSN_MMIO_MOVS:
+		ret = vc_handle_mmio_movs(ctxt, bytes);
+		break;
+	default:
+		ret = ES_UNSUPPORTED;
+		break;
+	}
+
+	return ret;
+}
+
+static enum es_result vc_handle_dr7_write(struct ghcb *ghcb,
+					  struct es_em_ctxt *ctxt)
+{
+	struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
+	long val, *reg = vc_insn_get_rm(ctxt);
+	enum es_result ret;
+
+	if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
+		return ES_VMM_ERROR;
+
+	if (!reg)
+		return ES_DECODE_FAILED;
+
+	val = *reg;
+
+	/* Upper 32 bits must be written as zeroes */
+	if (val >> 32) {
+		ctxt->fi.vector = X86_TRAP_GP;
+		ctxt->fi.error_code = 0;
+		return ES_EXCEPTION;
+	}
+
+	/* Clear out other reserved bits and set bit 10 */
+	val = (val & 0xffff23ffL) | BIT(10);
+
+	/* Early non-zero writes to DR7 are not supported */
+	if (!data && (val & ~DR7_RESET_VALUE))
+		return ES_UNSUPPORTED;
+
+	/* Using a value of 0 for ExitInfo1 means RAX holds the value */
+	ghcb_set_rax(ghcb, val);
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0);
+	if (ret != ES_OK)
+		return ret;
+
+	if (data)
+		data->dr7 = val;
+
+	return ES_OK;
+}
+
+static enum es_result vc_handle_dr7_read(struct ghcb *ghcb,
+					 struct es_em_ctxt *ctxt)
+{
+	struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
+	long *reg = vc_insn_get_rm(ctxt);
+
+	if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
+		return ES_VMM_ERROR;
+
+	if (!reg)
+		return ES_DECODE_FAILED;
+
+	if (data)
+		*reg = data->dr7;
+	else
+		*reg = DR7_RESET_VALUE;
+
+	return ES_OK;
+}
+
+static enum es_result vc_handle_wbinvd(struct ghcb *ghcb,
+				       struct es_em_ctxt *ctxt)
+{
+	return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);
+}
+
+static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+{
+	enum es_result ret;
+
+	ghcb_set_rcx(ghcb, ctxt->regs->cx);
+
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0);
+	if (ret != ES_OK)
+		return ret;
+
+	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb)))
+		return ES_VMM_ERROR;
+
+	ctxt->regs->ax = ghcb->save.rax;
+	ctxt->regs->dx = ghcb->save.rdx;
+
+	return ES_OK;
+}
+
+static enum es_result vc_handle_monitor(struct ghcb *ghcb,
+					struct es_em_ctxt *ctxt)
+{
+	/*
+	 * Treat it as a NOP and do not leak a physical address to the
+	 * hypervisor.
+	 */
+	return ES_OK;
+}
+
+static enum es_result vc_handle_mwait(struct ghcb *ghcb,
+				      struct es_em_ctxt *ctxt)
+{
+	/* Treat the same as MONITOR/MONITORX */
+	return ES_OK;
+}
+
+static enum es_result vc_handle_vmmcall(struct ghcb *ghcb,
+					struct es_em_ctxt *ctxt)
+{
+	enum es_result ret;
+
+	ghcb_set_rax(ghcb, ctxt->regs->ax);
+	ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0);
+
+	if (x86_platform.hyper.sev_es_hcall_prepare)
+		x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs);
+
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0);
+	if (ret != ES_OK)
+		return ret;
+
+	if (!ghcb_rax_is_valid(ghcb))
+		return ES_VMM_ERROR;
+
+	ctxt->regs->ax = ghcb->save.rax;
+
+	/*
+	 * Call sev_es_hcall_finish() after regs->ax is already set.
+	 * This allows the hypervisor handler to overwrite it again if
+	 * necessary.
+	 */
+	if (x86_platform.hyper.sev_es_hcall_finish &&
+	    !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs))
+		return ES_VMM_ERROR;
+
+	return ES_OK;
+}
+
+static enum es_result vc_handle_trap_ac(struct ghcb *ghcb,
+					struct es_em_ctxt *ctxt)
+{
+	/*
+	 * Calling ecx_alignment_check() directly does not work, because it
+	 * enables IRQs and the GHCB is active. Forward the exception and call
+	 * it later from vc_forward_exception().
+	 */
+	ctxt->fi.vector = X86_TRAP_AC;
+	ctxt->fi.error_code = 0;
+	return ES_EXCEPTION;
+}
+
+static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
+					 struct ghcb *ghcb,
+					 unsigned long exit_code)
+{
+	enum es_result result = vc_check_opcode_bytes(ctxt, exit_code);
+
+	if (result != ES_OK)
+		return result;
+
+	switch (exit_code) {
+	case SVM_EXIT_READ_DR7:
+		result = vc_handle_dr7_read(ghcb, ctxt);
+		break;
+	case SVM_EXIT_WRITE_DR7:
+		result = vc_handle_dr7_write(ghcb, ctxt);
+		break;
+	case SVM_EXIT_EXCP_BASE + X86_TRAP_AC:
+		result = vc_handle_trap_ac(ghcb, ctxt);
+		break;
+	case SVM_EXIT_RDTSC:
+	case SVM_EXIT_RDTSCP:
+		result = vc_handle_rdtsc(ghcb, ctxt, exit_code);
+		break;
+	case SVM_EXIT_RDPMC:
+		result = vc_handle_rdpmc(ghcb, ctxt);
+		break;
+	case SVM_EXIT_INVD:
+		pr_err_ratelimited("#VC exception for INVD??? Seriously???\n");
+		result = ES_UNSUPPORTED;
+		break;
+	case SVM_EXIT_CPUID:
+		result = vc_handle_cpuid(ghcb, ctxt);
+		break;
+	case SVM_EXIT_IOIO:
+		result = vc_handle_ioio(ghcb, ctxt);
+		break;
+	case SVM_EXIT_MSR:
+		result = vc_handle_msr(ghcb, ctxt);
+		break;
+	case SVM_EXIT_VMMCALL:
+		result = vc_handle_vmmcall(ghcb, ctxt);
+		break;
+	case SVM_EXIT_WBINVD:
+		result = vc_handle_wbinvd(ghcb, ctxt);
+		break;
+	case SVM_EXIT_MONITOR:
+		result = vc_handle_monitor(ghcb, ctxt);
+		break;
+	case SVM_EXIT_MWAIT:
+		result = vc_handle_mwait(ghcb, ctxt);
+		break;
+	case SVM_EXIT_NPF:
+		result = vc_handle_mmio(ghcb, ctxt);
+		break;
+	default:
+		/*
+		 * Unexpected #VC exception
+		 */
+		result = ES_UNSUPPORTED;
+	}
+
+	return result;
+}
+
+static __always_inline bool is_vc2_stack(unsigned long sp)
+{
+	return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2));
+}
+
+static __always_inline bool vc_from_invalid_context(struct pt_regs *regs)
+{
+	unsigned long sp, prev_sp;
+
+	sp      = (unsigned long)regs;
+	prev_sp = regs->sp;
+
+	/*
+	 * If the code was already executing on the VC2 stack when the #VC
+	 * happened, let it proceed to the normal handling routine. This way the
+	 * code executing on the VC2 stack can cause #VC exceptions to get handled.
+	 */
+	return is_vc2_stack(sp) && !is_vc2_stack(prev_sp);
+}
+
+static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code)
+{
+	struct ghcb_state state;
+	struct es_em_ctxt ctxt;
+	enum es_result result;
+	struct ghcb *ghcb;
+	bool ret = true;
+
+	ghcb = __sev_get_ghcb(&state);
+
+	vc_ghcb_invalidate(ghcb);
+	result = vc_init_em_ctxt(&ctxt, regs, error_code);
+
+	if (result == ES_OK)
+		result = vc_handle_exitcode(&ctxt, ghcb, error_code);
+
+	__sev_put_ghcb(&state);
+
+	/* Done - now check the result */
+	switch (result) {
+	case ES_OK:
+		vc_finish_insn(&ctxt);
+		break;
+	case ES_UNSUPPORTED:
+		pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n",
+				   error_code, regs->ip);
+		ret = false;
+		break;
+	case ES_VMM_ERROR:
+		pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
+				   error_code, regs->ip);
+		ret = false;
+		break;
+	case ES_DECODE_FAILED:
+		pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
+				   error_code, regs->ip);
+		ret = false;
+		break;
+	case ES_EXCEPTION:
+		vc_forward_exception(&ctxt);
+		break;
+	case ES_RETRY:
+		/* Nothing to do */
+		break;
+	default:
+		pr_emerg("Unknown result in %s():%d\n", __func__, result);
+		/*
+		 * Emulating the instruction which caused the #VC exception
+		 * failed - can't continue so print debug information
+		 */
+		BUG();
+	}
+
+	return ret;
+}
+
+static __always_inline bool vc_is_db(unsigned long error_code)
+{
+	return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB;
+}
+
+/*
+ * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode
+ * and will panic when an error happens.
+ */
+DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)
+{
+	irqentry_state_t irq_state;
+
+	/*
+	 * With the current implementation it is always possible to switch to a
+	 * safe stack because #VC exceptions only happen at known places, like
+	 * intercepted instructions or accesses to MMIO areas/IO ports. They can
+	 * also happen with code instrumentation when the hypervisor intercepts
+	 * #DB, but the critical paths are forbidden to be instrumented, so #DB
+	 * exceptions currently also only happen in safe places.
+	 *
+	 * But keep this here in case the noinstr annotations are violated due
+	 * to bug elsewhere.
+	 */
+	if (unlikely(vc_from_invalid_context(regs))) {
+		instrumentation_begin();
+		panic("Can't handle #VC exception from unsupported context\n");
+		instrumentation_end();
+	}
+
+	/*
+	 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
+	 */
+	if (vc_is_db(error_code)) {
+		exc_debug(regs);
+		return;
+	}
+
+	irq_state = irqentry_nmi_enter(regs);
+
+	instrumentation_begin();
+
+	if (!vc_raw_handle_exception(regs, error_code)) {
+		/* Show some debug info */
+		show_regs(regs);
+
+		/* Ask hypervisor to sev_es_terminate */
+		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
+
+		/* If that fails and we get here - just panic */
+		panic("Returned from Terminate-Request to Hypervisor\n");
+	}
+
+	instrumentation_end();
+	irqentry_nmi_exit(regs, irq_state);
+}
+
+/*
+ * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode
+ * and will kill the current task with SIGBUS when an error happens.
+ */
+DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)
+{
+	/*
+	 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
+	 */
+	if (vc_is_db(error_code)) {
+		noist_exc_debug(regs);
+		return;
+	}
+
+	irqentry_enter_from_user_mode(regs);
+	instrumentation_begin();
+
+	if (!vc_raw_handle_exception(regs, error_code)) {
+		/*
+		 * Do not kill the machine if user-space triggered the
+		 * exception. Send SIGBUS instead and let user-space deal with
+		 * it.
+		 */
+		force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0);
+	}
+
+	instrumentation_end();
+	irqentry_exit_to_user_mode(regs);
+}
+
+bool __init handle_vc_boot_ghcb(struct pt_regs *regs)
+{
+	unsigned long exit_code = regs->orig_ax;
+	struct es_em_ctxt ctxt;
+	enum es_result result;
+
+	vc_ghcb_invalidate(boot_ghcb);
+
+	result = vc_init_em_ctxt(&ctxt, regs, exit_code);
+	if (result == ES_OK)
+		result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code);
+
+	/* Done - now check the result */
+	switch (result) {
+	case ES_OK:
+		vc_finish_insn(&ctxt);
+		break;
+	case ES_UNSUPPORTED:
+		early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n",
+				exit_code, regs->ip);
+		goto fail;
+	case ES_VMM_ERROR:
+		early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
+				exit_code, regs->ip);
+		goto fail;
+	case ES_DECODE_FAILED:
+		early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
+				exit_code, regs->ip);
+		goto fail;
+	case ES_EXCEPTION:
+		vc_early_forward_exception(&ctxt);
+		break;
+	case ES_RETRY:
+		/* Nothing to do */
+		break;
+	default:
+		BUG();
+	}
+
+	return true;
+
+fail:
+	show_regs(regs);
+
+	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
+}
+
diff --git a/arch/x86/coco/sev/vc-shared.c b/arch/x86/coco/sev/vc-shared.c
new file mode 100644
index 000000000000..58b2f985d546
--- /dev/null
+++ b/arch/x86/coco/sev/vc-shared.c
@@ -0,0 +1,656 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#ifndef __BOOT_COMPRESSED
+#define has_cpuflag(f)                  cpu_feature_enabled(f)
+#endif
+
+static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt,
+					    unsigned long exit_code)
+{
+	unsigned int opcode = (unsigned int)ctxt->insn.opcode.value;
+	u8 modrm = ctxt->insn.modrm.value;
+
+	switch (exit_code) {
+
+	case SVM_EXIT_IOIO:
+	case SVM_EXIT_NPF:
+		/* handled separately */
+		return ES_OK;
+
+	case SVM_EXIT_CPUID:
+		if (opcode == 0xa20f)
+			return ES_OK;
+		break;
+
+	case SVM_EXIT_INVD:
+		if (opcode == 0x080f)
+			return ES_OK;
+		break;
+
+	case SVM_EXIT_MONITOR:
+		/* MONITOR and MONITORX instructions generate the same error code */
+		if (opcode == 0x010f && (modrm == 0xc8 || modrm == 0xfa))
+			return ES_OK;
+		break;
+
+	case SVM_EXIT_MWAIT:
+		/* MWAIT and MWAITX instructions generate the same error code */
+		if (opcode == 0x010f && (modrm == 0xc9 || modrm == 0xfb))
+			return ES_OK;
+		break;
+
+	case SVM_EXIT_MSR:
+		/* RDMSR */
+		if (opcode == 0x320f ||
+		/* WRMSR */
+		    opcode == 0x300f)
+			return ES_OK;
+		break;
+
+	case SVM_EXIT_RDPMC:
+		if (opcode == 0x330f)
+			return ES_OK;
+		break;
+
+	case SVM_EXIT_RDTSC:
+		if (opcode == 0x310f)
+			return ES_OK;
+		break;
+
+	case SVM_EXIT_RDTSCP:
+		if (opcode == 0x010f && modrm == 0xf9)
+			return ES_OK;
+		break;
+
+	case SVM_EXIT_READ_DR7:
+		if (opcode == 0x210f &&
+		    X86_MODRM_REG(ctxt->insn.modrm.value) == 7)
+			return ES_OK;
+		break;
+
+	case SVM_EXIT_VMMCALL:
+		if (opcode == 0x010f && modrm == 0xd9)
+			return ES_OK;
+
+		break;
+
+	case SVM_EXIT_WRITE_DR7:
+		if (opcode == 0x230f &&
+		    X86_MODRM_REG(ctxt->insn.modrm.value) == 7)
+			return ES_OK;
+		break;
+
+	case SVM_EXIT_WBINVD:
+		if (opcode == 0x90f)
+			return ES_OK;
+		break;
+
+	default:
+		break;
+	}
+
+	sev_printk(KERN_ERR "Wrong/unhandled opcode bytes: 0x%x, exit_code: 0x%lx, rIP: 0x%lx\n",
+		   opcode, exit_code, ctxt->regs->ip);
+
+	return ES_UNSUPPORTED;
+}
+
+static bool vc_decoding_needed(unsigned long exit_code)
+{
+	/* Exceptions don't require to decode the instruction */
+	return !(exit_code >= SVM_EXIT_EXCP_BASE &&
+		 exit_code <= SVM_EXIT_LAST_EXCP);
+}
+
+static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
+				      struct pt_regs *regs,
+				      unsigned long exit_code)
+{
+	enum es_result ret = ES_OK;
+
+	memset(ctxt, 0, sizeof(*ctxt));
+	ctxt->regs = regs;
+
+	if (vc_decoding_needed(exit_code))
+		ret = vc_decode_insn(ctxt);
+
+	return ret;
+}
+
+static void vc_finish_insn(struct es_em_ctxt *ctxt)
+{
+	ctxt->regs->ip += ctxt->insn.length;
+}
+
+static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt,
+					   unsigned long address,
+					   bool write)
+{
+	if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) {
+		ctxt->fi.vector     = X86_TRAP_PF;
+		ctxt->fi.error_code = X86_PF_USER;
+		ctxt->fi.cr2        = address;
+		if (write)
+			ctxt->fi.error_code |= X86_PF_WRITE;
+
+		return ES_EXCEPTION;
+	}
+
+	return ES_OK;
+}
+
+static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
+					  void *src, char *buf,
+					  unsigned int data_size,
+					  unsigned int count,
+					  bool backwards)
+{
+	int i, b = backwards ? -1 : 1;
+	unsigned long address = (unsigned long)src;
+	enum es_result ret;
+
+	ret = vc_insn_string_check(ctxt, address, false);
+	if (ret != ES_OK)
+		return ret;
+
+	for (i = 0; i < count; i++) {
+		void *s = src + (i * data_size * b);
+		char *d = buf + (i * data_size);
+
+		ret = vc_read_mem(ctxt, s, d, data_size);
+		if (ret != ES_OK)
+			break;
+	}
+
+	return ret;
+}
+
+static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
+					   void *dst, char *buf,
+					   unsigned int data_size,
+					   unsigned int count,
+					   bool backwards)
+{
+	int i, s = backwards ? -1 : 1;
+	unsigned long address = (unsigned long)dst;
+	enum es_result ret;
+
+	ret = vc_insn_string_check(ctxt, address, true);
+	if (ret != ES_OK)
+		return ret;
+
+	for (i = 0; i < count; i++) {
+		void *d = dst + (i * data_size * s);
+		char *b = buf + (i * data_size);
+
+		ret = vc_write_mem(ctxt, d, b, data_size);
+		if (ret != ES_OK)
+			break;
+	}
+
+	return ret;
+}
+
+#define IOIO_TYPE_STR  BIT(2)
+#define IOIO_TYPE_IN   1
+#define IOIO_TYPE_INS  (IOIO_TYPE_IN | IOIO_TYPE_STR)
+#define IOIO_TYPE_OUT  0
+#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
+
+#define IOIO_REP       BIT(3)
+
+#define IOIO_ADDR_64   BIT(9)
+#define IOIO_ADDR_32   BIT(8)
+#define IOIO_ADDR_16   BIT(7)
+
+#define IOIO_DATA_32   BIT(6)
+#define IOIO_DATA_16   BIT(5)
+#define IOIO_DATA_8    BIT(4)
+
+#define IOIO_SEG_ES    (0 << 10)
+#define IOIO_SEG_DS    (3 << 10)
+
+static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
+{
+	struct insn *insn = &ctxt->insn;
+	size_t size;
+	u64 port;
+
+	*exitinfo = 0;
+
+	switch (insn->opcode.bytes[0]) {
+	/* INS opcodes */
+	case 0x6c:
+	case 0x6d:
+		*exitinfo |= IOIO_TYPE_INS;
+		*exitinfo |= IOIO_SEG_ES;
+		port	   = ctxt->regs->dx & 0xffff;
+		break;
+
+	/* OUTS opcodes */
+	case 0x6e:
+	case 0x6f:
+		*exitinfo |= IOIO_TYPE_OUTS;
+		*exitinfo |= IOIO_SEG_DS;
+		port	   = ctxt->regs->dx & 0xffff;
+		break;
+
+	/* IN immediate opcodes */
+	case 0xe4:
+	case 0xe5:
+		*exitinfo |= IOIO_TYPE_IN;
+		port	   = (u8)insn->immediate.value & 0xffff;
+		break;
+
+	/* OUT immediate opcodes */
+	case 0xe6:
+	case 0xe7:
+		*exitinfo |= IOIO_TYPE_OUT;
+		port	   = (u8)insn->immediate.value & 0xffff;
+		break;
+
+	/* IN register opcodes */
+	case 0xec:
+	case 0xed:
+		*exitinfo |= IOIO_TYPE_IN;
+		port	   = ctxt->regs->dx & 0xffff;
+		break;
+
+	/* OUT register opcodes */
+	case 0xee:
+	case 0xef:
+		*exitinfo |= IOIO_TYPE_OUT;
+		port	   = ctxt->regs->dx & 0xffff;
+		break;
+
+	default:
+		return ES_DECODE_FAILED;
+	}
+
+	*exitinfo |= port << 16;
+
+	switch (insn->opcode.bytes[0]) {
+	case 0x6c:
+	case 0x6e:
+	case 0xe4:
+	case 0xe6:
+	case 0xec:
+	case 0xee:
+		/* Single byte opcodes */
+		*exitinfo |= IOIO_DATA_8;
+		size       = 1;
+		break;
+	default:
+		/* Length determined by instruction parsing */
+		*exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
+						     : IOIO_DATA_32;
+		size       = (insn->opnd_bytes == 2) ? 2 : 4;
+	}
+
+	switch (insn->addr_bytes) {
+	case 2:
+		*exitinfo |= IOIO_ADDR_16;
+		break;
+	case 4:
+		*exitinfo |= IOIO_ADDR_32;
+		break;
+	case 8:
+		*exitinfo |= IOIO_ADDR_64;
+		break;
+	}
+
+	if (insn_has_rep_prefix(insn))
+		*exitinfo |= IOIO_REP;
+
+	return vc_ioio_check(ctxt, (u16)port, size);
+}
+
+static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+{
+	struct pt_regs *regs = ctxt->regs;
+	u64 exit_info_1, exit_info_2;
+	enum es_result ret;
+
+	ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
+	if (ret != ES_OK)
+		return ret;
+
+	if (exit_info_1 & IOIO_TYPE_STR) {
+
+		/* (REP) INS/OUTS */
+
+		bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
+		unsigned int io_bytes, exit_bytes;
+		unsigned int ghcb_count, op_count;
+		unsigned long es_base;
+		u64 sw_scratch;
+
+		/*
+		 * For the string variants with rep prefix the amount of in/out
+		 * operations per #VC exception is limited so that the kernel
+		 * has a chance to take interrupts and re-schedule while the
+		 * instruction is emulated.
+		 */
+		io_bytes   = (exit_info_1 >> 4) & 0x7;
+		ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
+
+		op_count    = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
+		exit_info_2 = min(op_count, ghcb_count);
+		exit_bytes  = exit_info_2 * io_bytes;
+
+		es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
+
+		/* Read bytes of OUTS into the shared buffer */
+		if (!(exit_info_1 & IOIO_TYPE_IN)) {
+			ret = vc_insn_string_read(ctxt,
+					       (void *)(es_base + regs->si),
+					       ghcb->shared_buffer, io_bytes,
+					       exit_info_2, df);
+			if (ret)
+				return ret;
+		}
+
+		/*
+		 * Issue an VMGEXIT to the HV to consume the bytes from the
+		 * shared buffer or to have it write them into the shared buffer
+		 * depending on the instruction: OUTS or INS.
+		 */
+		sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
+		ghcb_set_sw_scratch(ghcb, sw_scratch);
+		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
+					  exit_info_1, exit_info_2);
+		if (ret != ES_OK)
+			return ret;
+
+		/* Read bytes from shared buffer into the guest's destination. */
+		if (exit_info_1 & IOIO_TYPE_IN) {
+			ret = vc_insn_string_write(ctxt,
+						   (void *)(es_base + regs->di),
+						   ghcb->shared_buffer, io_bytes,
+						   exit_info_2, df);
+			if (ret)
+				return ret;
+
+			if (df)
+				regs->di -= exit_bytes;
+			else
+				regs->di += exit_bytes;
+		} else {
+			if (df)
+				regs->si -= exit_bytes;
+			else
+				regs->si += exit_bytes;
+		}
+
+		if (exit_info_1 & IOIO_REP)
+			regs->cx -= exit_info_2;
+
+		ret = regs->cx ? ES_RETRY : ES_OK;
+
+	} else {
+
+		/* IN/OUT into/from rAX */
+
+		int bits = (exit_info_1 & 0x70) >> 1;
+		u64 rax = 0;
+
+		if (!(exit_info_1 & IOIO_TYPE_IN))
+			rax = lower_bits(regs->ax, bits);
+
+		ghcb_set_rax(ghcb, rax);
+
+		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
+		if (ret != ES_OK)
+			return ret;
+
+		if (exit_info_1 & IOIO_TYPE_IN) {
+			if (!ghcb_rax_is_valid(ghcb))
+				return ES_VMM_ERROR;
+			regs->ax = lower_bits(ghcb->save.rax, bits);
+		}
+	}
+
+	return ret;
+}
+
+enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+{
+	u32 ret;
+
+	ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0);
+	if (!ret)
+		return ES_OK;
+
+	if (ret == 1) {
+		u64 info = ghcb->save.sw_exit_info_2;
+		unsigned long v = info & SVM_EVTINJ_VEC_MASK;
+
+		/* Check if exception information from hypervisor is sane. */
+		if ((info & SVM_EVTINJ_VALID) &&
+		    ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
+		    ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
+			ctxt->fi.vector = v;
+
+			if (info & SVM_EVTINJ_VALID_ERR)
+				ctxt->fi.error_code = info >> 32;
+
+			return ES_EXCEPTION;
+		}
+	}
+
+	return ES_VMM_ERROR;
+}
+
+enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
+				   struct es_em_ctxt *ctxt,
+				   u64 exit_code, u64 exit_info_1,
+				   u64 exit_info_2)
+{
+	/* Fill in protocol and format specifiers */
+	ghcb->protocol_version = ghcb_version;
+	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
+
+	ghcb_set_sw_exit_code(ghcb, exit_code);
+	ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
+	ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
+
+	sev_es_wr_ghcb_msr(__pa(ghcb));
+	VMGEXIT();
+
+	return verify_exception_info(ghcb, ctxt);
+}
+
+static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
+{
+	u32 cr4 = native_read_cr4();
+	int ret;
+
+	ghcb_set_rax(ghcb, leaf->fn);
+	ghcb_set_rcx(ghcb, leaf->subfn);
+
+	if (cr4 & X86_CR4_OSXSAVE)
+		/* Safe to read xcr0 */
+		ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
+	else
+		/* xgetbv will cause #UD - use reset value for xcr0 */
+		ghcb_set_xcr0(ghcb, 1);
+
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
+	if (ret != ES_OK)
+		return ret;
+
+	if (!(ghcb_rax_is_valid(ghcb) &&
+	      ghcb_rbx_is_valid(ghcb) &&
+	      ghcb_rcx_is_valid(ghcb) &&
+	      ghcb_rdx_is_valid(ghcb)))
+		return ES_VMM_ERROR;
+
+	leaf->eax = ghcb->save.rax;
+	leaf->ebx = ghcb->save.rbx;
+	leaf->ecx = ghcb->save.rcx;
+	leaf->edx = ghcb->save.rdx;
+
+	return ES_OK;
+}
+
+struct cpuid_ctx {
+	struct ghcb *ghcb;
+	struct es_em_ctxt *ctxt;
+};
+
+static void snp_cpuid_hv_ghcb(void *p, struct cpuid_leaf *leaf)
+{
+	struct cpuid_ctx *ctx = p;
+
+	if (__sev_cpuid_hv_ghcb(ctx->ghcb, ctx->ctxt, leaf))
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV);
+}
+
+static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+{
+	struct cpuid_ctx ctx = { ghcb, ctxt };
+	struct pt_regs *regs = ctxt->regs;
+	struct cpuid_leaf leaf;
+	int ret;
+
+	leaf.fn = regs->ax;
+	leaf.subfn = regs->cx;
+	ret = snp_cpuid(snp_cpuid_hv_ghcb, &ctx, &leaf);
+	if (!ret) {
+		regs->ax = leaf.eax;
+		regs->bx = leaf.ebx;
+		regs->cx = leaf.ecx;
+		regs->dx = leaf.edx;
+	}
+
+	return ret;
+}
+
+static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
+				      struct es_em_ctxt *ctxt)
+{
+	struct pt_regs *regs = ctxt->regs;
+	u32 cr4 = native_read_cr4();
+	enum es_result ret;
+	int snp_cpuid_ret;
+
+	snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt);
+	if (!snp_cpuid_ret)
+		return ES_OK;
+	if (snp_cpuid_ret != -EOPNOTSUPP)
+		return ES_VMM_ERROR;
+
+	ghcb_set_rax(ghcb, regs->ax);
+	ghcb_set_rcx(ghcb, regs->cx);
+
+	if (cr4 & X86_CR4_OSXSAVE)
+		/* Safe to read xcr0 */
+		ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
+	else
+		/* xgetbv will cause #GP - use reset value for xcr0 */
+		ghcb_set_xcr0(ghcb, 1);
+
+	if (has_cpuflag(X86_FEATURE_SHSTK) && regs->ax == 0xd && regs->cx == 1) {
+		struct msr m;
+
+		raw_rdmsr(MSR_IA32_XSS, &m);
+		ghcb_set_xss(ghcb, m.q);
+	}
+
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
+	if (ret != ES_OK)
+		return ret;
+
+	if (!(ghcb_rax_is_valid(ghcb) &&
+	      ghcb_rbx_is_valid(ghcb) &&
+	      ghcb_rcx_is_valid(ghcb) &&
+	      ghcb_rdx_is_valid(ghcb)))
+		return ES_VMM_ERROR;
+
+	regs->ax = ghcb->save.rax;
+	regs->bx = ghcb->save.rbx;
+	regs->cx = ghcb->save.rcx;
+	regs->dx = ghcb->save.rdx;
+
+	return ES_OK;
+}
+
+static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
+				      struct es_em_ctxt *ctxt,
+				      unsigned long exit_code)
+{
+	bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
+	enum es_result ret;
+
+	/*
+	 * The hypervisor should not be intercepting RDTSC/RDTSCP when Secure
+	 * TSC is enabled. A #VC exception will be generated if the RDTSC/RDTSCP
+	 * instructions are being intercepted. If this should occur and Secure
+	 * TSC is enabled, guest execution should be terminated as the guest
+	 * cannot rely on the TSC value provided by the hypervisor.
+	 */
+	if (sev_status & MSR_AMD64_SNP_SECURE_TSC)
+		return ES_VMM_ERROR;
+
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
+	if (ret != ES_OK)
+		return ret;
+
+	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
+	     (!rdtscp || ghcb_rcx_is_valid(ghcb))))
+		return ES_VMM_ERROR;
+
+	ctxt->regs->ax = ghcb->save.rax;
+	ctxt->regs->dx = ghcb->save.rdx;
+	if (rdtscp)
+		ctxt->regs->cx = ghcb->save.rcx;
+
+	return ES_OK;
+}
+
+void snp_register_ghcb_early(unsigned long paddr)
+{
+	unsigned long pfn = paddr >> PAGE_SHIFT;
+	u64 val;
+
+	sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn));
+	VMGEXIT();
+
+	val = sev_es_rd_ghcb_msr();
+
+	/* If the response GPA is not ours then abort the guest */
+	if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) ||
+	    (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn))
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
+}
+
+bool __init sev_es_check_cpu_features(void)
+{
+	if (!has_cpuflag(X86_FEATURE_RDRAND)) {
+		error("RDRAND instruction not supported - no trusted source of randomness available\n");
+		return false;
+	}
+
+	return true;
+}
+
+bool sev_es_negotiate_protocol(void)
+{
+	u64 val;
+
+	/* Do the GHCB protocol version negotiation */
+	sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
+	VMGEXIT();
+	val = sev_es_rd_ghcb_msr();
+
+	if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
+		return false;
+
+	if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||
+	    GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
+		return false;
+
+	ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX);
+
+	return true;
+}
diff --git a/arch/x86/coco/tdx/Makefile b/arch/x86/coco/tdx/Makefile
index 2c7dcbf1458b..b3c47d3700e2 100644
--- a/arch/x86/coco/tdx/Makefile
+++ b/arch/x86/coco/tdx/Makefile
@@ -1,3 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0
 
-obj-y += tdx.o tdx-shared.o tdcall.o
+obj-y += debug.o tdcall.o tdx.o tdx-shared.o
diff --git a/arch/x86/coco/tdx/debug.c b/arch/x86/coco/tdx/debug.c
new file mode 100644
index 000000000000..cef847c8bb67
--- /dev/null
+++ b/arch/x86/coco/tdx/debug.c
@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#undef pr_fmt
+#define pr_fmt(fmt)     "tdx: " fmt
+
+#include <linux/array_size.h>
+#include <linux/printk.h>
+#include <asm/tdx.h>
+
+#define DEF_TDX_ATTR_NAME(_name) [TDX_ATTR_##_name##_BIT] = __stringify(_name)
+
+static __initdata const char *tdx_attributes[] = {
+	DEF_TDX_ATTR_NAME(DEBUG),
+	DEF_TDX_ATTR_NAME(HGS_PLUS_PROF),
+	DEF_TDX_ATTR_NAME(PERF_PROF),
+	DEF_TDX_ATTR_NAME(PMT_PROF),
+	DEF_TDX_ATTR_NAME(ICSSD),
+	DEF_TDX_ATTR_NAME(LASS),
+	DEF_TDX_ATTR_NAME(SEPT_VE_DISABLE),
+	DEF_TDX_ATTR_NAME(MIGRTABLE),
+	DEF_TDX_ATTR_NAME(PKS),
+	DEF_TDX_ATTR_NAME(KL),
+	DEF_TDX_ATTR_NAME(TPA),
+	DEF_TDX_ATTR_NAME(PERFMON),
+};
+
+#define DEF_TD_CTLS_NAME(_name) [TD_CTLS_##_name##_BIT] = __stringify(_name)
+
+static __initdata const char *tdcs_td_ctls[] = {
+	DEF_TD_CTLS_NAME(PENDING_VE_DISABLE),
+	DEF_TD_CTLS_NAME(ENUM_TOPOLOGY),
+	DEF_TD_CTLS_NAME(VIRT_CPUID2),
+	DEF_TD_CTLS_NAME(REDUCE_VE),
+	DEF_TD_CTLS_NAME(LOCK),
+};
+
+void __init tdx_dump_attributes(u64 td_attr)
+{
+	pr_info("Attributes:");
+
+	for (int i = 0; i < ARRAY_SIZE(tdx_attributes); i++) {
+		if (!tdx_attributes[i])
+			continue;
+		if (td_attr & BIT(i))
+			pr_cont(" %s", tdx_attributes[i]);
+		td_attr &= ~BIT(i);
+	}
+
+	if (td_attr)
+		pr_cont(" unknown:%#llx", td_attr);
+	pr_cont("\n");
+
+}
+
+void __init tdx_dump_td_ctls(u64 td_ctls)
+{
+	pr_info("TD_CTLS:");
+
+	for (int i = 0; i < ARRAY_SIZE(tdcs_td_ctls); i++) {
+		if (!tdcs_td_ctls[i])
+			continue;
+		if (td_ctls & BIT(i))
+			pr_cont(" %s", tdcs_td_ctls[i]);
+		td_ctls &= ~BIT(i);
+	}
+	if (td_ctls)
+		pr_cont(" unknown:%#llx", td_ctls);
+	pr_cont("\n");
+}
diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c
index 327c45c5013f..7b2833705d47 100644
--- a/arch/x86/coco/tdx/tdx.c
+++ b/arch/x86/coco/tdx/tdx.c
@@ -14,6 +14,7 @@
 #include <asm/ia32.h>
 #include <asm/insn.h>
 #include <asm/insn-eval.h>
+#include <asm/paravirt_types.h>
 #include <asm/pgtable.h>
 #include <asm/set_memory.h>
 #include <asm/traps.h>
@@ -32,12 +33,10 @@
 #define VE_GET_PORT_NUM(e)	((e) >> 16)
 #define VE_IS_IO_STRING(e)	((e) & BIT(4))
 
-#define ATTR_DEBUG		BIT(0)
-#define ATTR_SEPT_VE_DISABLE	BIT(28)
-
 /* TDX Module call error codes */
 #define TDCALL_RETURN_CODE(a)	((a) >> 32)
 #define TDCALL_INVALID_OPERAND	0xc0000100
+#define TDCALL_OPERAND_BUSY	0x80000200
 
 #define TDREPORT_SUBTYPE_0	0
 
@@ -78,6 +77,32 @@ static inline void tdcall(u64 fn, struct tdx_module_args *args)
 		panic("TDCALL %lld failed (Buggy TDX module!)\n", fn);
 }
 
+/* Read TD-scoped metadata */
+static inline u64 tdg_vm_rd(u64 field, u64 *value)
+{
+	struct tdx_module_args args = {
+		.rdx = field,
+	};
+	u64 ret;
+
+	ret = __tdcall_ret(TDG_VM_RD, &args);
+	*value = args.r8;
+
+	return ret;
+}
+
+/* Write TD-scoped metadata */
+static inline u64 tdg_vm_wr(u64 field, u64 value, u64 mask)
+{
+	struct tdx_module_args args = {
+		.rdx = field,
+		.r8 = value,
+		.r9 = mask,
+	};
+
+	return __tdcall(TDG_VM_WR, &args);
+}
+
 /**
  * tdx_mcall_get_report0() - Wrapper to get TDREPORT0 (a.k.a. TDREPORT
  *                           subtype 0) using TDG.MR.REPORT TDCALL.
@@ -85,12 +110,13 @@ static inline void tdcall(u64 fn, struct tdx_module_args *args)
  *              REPORTDATA to be included into TDREPORT.
  * @tdreport: Address of the output buffer to store TDREPORT.
  *
- * Refer to section titled "TDG.MR.REPORT leaf" in the TDX Module
- * v1.0 specification for more information on TDG.MR.REPORT TDCALL.
+ * Refer to section titled "TDG.MR.REPORT leaf" in the TDX Module v1.0
+ * specification for more information on TDG.MR.REPORT TDCALL.
+ *
  * It is used in the TDX guest driver module to get the TDREPORT0.
  *
- * Return 0 on success, -EINVAL for invalid operands, or -EIO on
- * other TDCALL failures.
+ * Return 0 on success, -ENXIO for invalid operands, -EBUSY for busy operation,
+ * or -EIO on other TDCALL failures.
  */
 int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport)
 {
@@ -104,7 +130,9 @@ int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport)
 	ret = __tdcall(TDG_MR_REPORT, &args);
 	if (ret) {
 		if (TDCALL_RETURN_CODE(ret) == TDCALL_INVALID_OPERAND)
-			return -EINVAL;
+			return -ENXIO;
+		else if (TDCALL_RETURN_CODE(ret) == TDCALL_OPERAND_BUSY)
+			return -EBUSY;
 		return -EIO;
 	}
 
@@ -113,6 +141,42 @@ int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport)
 EXPORT_SYMBOL_GPL(tdx_mcall_get_report0);
 
 /**
+ * tdx_mcall_extend_rtmr() - Wrapper to extend RTMR registers using
+ *			     TDG.MR.RTMR.EXTEND TDCALL.
+ * @index: Index of RTMR register to be extended.
+ * @data: Address of the input buffer with RTMR register extend data.
+ *
+ * Refer to section titled "TDG.MR.RTMR.EXTEND leaf" in the TDX Module v1.0
+ * specification for more information on TDG.MR.RTMR.EXTEND TDCALL.
+ *
+ * It is used in the TDX guest driver module to allow user to extend the RTMR
+ * registers.
+ *
+ * Return 0 on success, -ENXIO for invalid operands, -EBUSY for busy operation,
+ * or -EIO on other TDCALL failures.
+ */
+int tdx_mcall_extend_rtmr(u8 index, u8 *data)
+{
+	struct tdx_module_args args = {
+		.rcx = virt_to_phys(data),
+		.rdx = index,
+	};
+	u64 ret;
+
+	ret = __tdcall(TDG_MR_RTMR_EXTEND, &args);
+	if (ret) {
+		if (TDCALL_RETURN_CODE(ret) == TDCALL_INVALID_OPERAND)
+			return -ENXIO;
+		if (TDCALL_RETURN_CODE(ret) == TDCALL_OPERAND_BUSY)
+			return -EBUSY;
+		return -EIO;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(tdx_mcall_extend_rtmr);
+
+/**
  * tdx_hcall_get_quote() - Wrapper to request TD Quote using GetQuote
  *                         hypercall.
  * @buf: Address of the directly mapped shared kernel buffer which
@@ -144,11 +208,11 @@ static void __noreturn tdx_panic(const char *msg)
 		/* Define register order according to the GHCI */
 		struct { u64 r14, r15, rbx, rdi, rsi, r8, r9, rdx; };
 
-		char str[64];
+		char bytes[64] __nonstring;
 	} message;
 
 	/* VMM assumes '\0' in byte 65, if the message took all 64 bytes */
-	strtomem_pad(message.str, msg, '\0');
+	strtomem_pad(message.bytes, msg, '\0');
 
 	args.r8  = message.r8;
 	args.r9  = message.r9;
@@ -168,7 +232,101 @@ static void __noreturn tdx_panic(const char *msg)
 		__tdx_hypercall(&args);
 }
 
-static void tdx_parse_tdinfo(u64 *cc_mask)
+/*
+ * The kernel cannot handle #VEs when accessing normal kernel memory. Ensure
+ * that no #VE will be delivered for accesses to TD-private memory.
+ *
+ * TDX 1.0 does not allow the guest to disable SEPT #VE on its own. The VMM
+ * controls if the guest will receive such #VE with TD attribute
+ * TDX_ATTR_SEPT_VE_DISABLE.
+ *
+ * Newer TDX modules allow the guest to control if it wants to receive SEPT
+ * violation #VEs.
+ *
+ * Check if the feature is available and disable SEPT #VE if possible.
+ *
+ * If the TD is allowed to disable/enable SEPT #VEs, the TDX_ATTR_SEPT_VE_DISABLE
+ * attribute is no longer reliable. It reflects the initial state of the
+ * control for the TD, but it will not be updated if someone (e.g. bootloader)
+ * changes it before the kernel starts. Kernel must check TDCS_TD_CTLS bit to
+ * determine if SEPT #VEs are enabled or disabled.
+ */
+static void disable_sept_ve(u64 td_attr)
+{
+	const char *msg = "TD misconfiguration: SEPT #VE has to be disabled";
+	bool debug = td_attr & TDX_ATTR_DEBUG;
+	u64 config, controls;
+
+	/* Is this TD allowed to disable SEPT #VE */
+	tdg_vm_rd(TDCS_CONFIG_FLAGS, &config);
+	if (!(config & TDCS_CONFIG_FLEXIBLE_PENDING_VE)) {
+		/* No SEPT #VE controls for the guest: check the attribute */
+		if (td_attr & TDX_ATTR_SEPT_VE_DISABLE)
+			return;
+
+		/* Relax SEPT_VE_DISABLE check for debug TD for backtraces */
+		if (debug)
+			pr_warn("%s\n", msg);
+		else
+			tdx_panic(msg);
+		return;
+	}
+
+	/* Check if SEPT #VE has been disabled before us */
+	tdg_vm_rd(TDCS_TD_CTLS, &controls);
+	if (controls & TD_CTLS_PENDING_VE_DISABLE)
+		return;
+
+	/* Keep #VEs enabled for splats in debugging environments */
+	if (debug)
+		return;
+
+	/* Disable SEPT #VEs */
+	tdg_vm_wr(TDCS_TD_CTLS, TD_CTLS_PENDING_VE_DISABLE,
+		  TD_CTLS_PENDING_VE_DISABLE);
+}
+
+/*
+ * TDX 1.0 generates a #VE when accessing topology-related CPUID leafs (0xB and
+ * 0x1F) and the X2APIC_APICID MSR. The kernel returns all zeros on CPUID #VEs.
+ * In practice, this means that the kernel can only boot with a plain topology.
+ * Any complications will cause problems.
+ *
+ * The ENUM_TOPOLOGY feature allows the VMM to provide topology information.
+ * Enabling the feature  eliminates topology-related #VEs: the TDX module
+ * virtualizes accesses to the CPUID leafs and the MSR.
+ *
+ * Enable ENUM_TOPOLOGY if it is available.
+ */
+static void enable_cpu_topology_enumeration(void)
+{
+	u64 configured;
+
+	/* Has the VMM provided a valid topology configuration? */
+	tdg_vm_rd(TDCS_TOPOLOGY_ENUM_CONFIGURED, &configured);
+	if (!configured) {
+		pr_err("VMM did not configure X2APIC_IDs properly\n");
+		return;
+	}
+
+	tdg_vm_wr(TDCS_TD_CTLS, TD_CTLS_ENUM_TOPOLOGY, TD_CTLS_ENUM_TOPOLOGY);
+}
+
+static void reduce_unnecessary_ve(void)
+{
+	u64 err = tdg_vm_wr(TDCS_TD_CTLS, TD_CTLS_REDUCE_VE, TD_CTLS_REDUCE_VE);
+
+	if (err == TDX_SUCCESS)
+		return;
+
+	/*
+	 * Enabling REDUCE_VE includes ENUM_TOPOLOGY. Only try to
+	 * enable ENUM_TOPOLOGY if REDUCE_VE was not successful.
+	 */
+	enable_cpu_topology_enumeration();
+}
+
+static void tdx_setup(u64 *cc_mask)
 {
 	struct tdx_module_args args = {};
 	unsigned int gpa_width;
@@ -193,21 +351,14 @@ static void tdx_parse_tdinfo(u64 *cc_mask)
 	gpa_width = args.rcx & GENMASK(5, 0);
 	*cc_mask = BIT_ULL(gpa_width - 1);
 
-	/*
-	 * The kernel can not handle #VE's when accessing normal kernel
-	 * memory.  Ensure that no #VE will be delivered for accesses to
-	 * TD-private memory.  Only VMM-shared memory (MMIO) will #VE.
-	 */
 	td_attr = args.rdx;
-	if (!(td_attr & ATTR_SEPT_VE_DISABLE)) {
-		const char *msg = "TD misconfiguration: SEPT_VE_DISABLE attribute must be set.";
 
-		/* Relax SEPT_VE_DISABLE check for debug TD. */
-		if (td_attr & ATTR_DEBUG)
-			pr_warn("%s\n", msg);
-		else
-			tdx_panic(msg);
-	}
+	/* Kernel does not use NOTIFY_ENABLES and does not need random #VEs */
+	tdg_vm_wr(TDCS_NOTIFY_ENABLES, 0, -1ULL);
+
+	disable_sept_ve(td_attr);
+
+	reduce_unnecessary_ve();
 }
 
 /*
@@ -282,13 +433,21 @@ static int handle_halt(struct ve_info *ve)
 {
 	const bool irq_disabled = irqs_disabled();
 
+	/*
+	 * HLT with IRQs enabled is unsafe, as an IRQ that is intended to be a
+	 * wake event may be consumed before requesting HLT emulation, leaving
+	 * the vCPU blocking indefinitely.
+	 */
+	if (WARN_ONCE(!irq_disabled, "HLT emulation with IRQs enabled"))
+		return -EIO;
+
 	if (__halt(irq_disabled))
 		return -EIO;
 
 	return ve_instr_len(ve);
 }
 
-void __cpuidle tdx_safe_halt(void)
+void __cpuidle tdx_halt(void)
 {
 	const bool irq_disabled = false;
 
@@ -299,6 +458,16 @@ void __cpuidle tdx_safe_halt(void)
 		WARN_ONCE(1, "HLT instruction emulation failed\n");
 }
 
+static void __cpuidle tdx_safe_halt(void)
+{
+	tdx_halt();
+	/*
+	 * "__cpuidle" section doesn't support instrumentation, so stick
+	 * with raw_* variant that avoids tracing hooks.
+	 */
+	raw_local_irq_enable();
+}
+
 static int read_msr(struct pt_regs *regs, struct ve_info *ve)
 {
 	struct tdx_module_args args = {
@@ -927,12 +1096,22 @@ static void tdx_kexec_finish(void)
 	}
 }
 
+static __init void tdx_announce(void)
+{
+	struct tdx_module_args args = {};
+	u64 controls;
+
+	pr_info("Guest detected\n");
+
+	tdcall(TDG_VP_INFO, &args);
+	tdx_dump_attributes(args.rdx);
+
+	tdg_vm_rd(TDCS_TD_CTLS, &controls);
+	tdx_dump_td_ctls(controls);
+}
+
 void __init tdx_early_init(void)
 {
-	struct tdx_module_args args = {
-		.rdx = TDCS_NOTIFY_ENABLES,
-		.r9 = -1ULL,
-	};
 	u64 cc_mask;
 	u32 eax, sig[3];
 
@@ -947,11 +1126,11 @@ void __init tdx_early_init(void)
 	setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE);
 
 	cc_vendor = CC_VENDOR_INTEL;
-	tdx_parse_tdinfo(&cc_mask);
-	cc_set_mask(cc_mask);
 
-	/* Kernel does not use NOTIFY_ENABLES and does not need random #VEs */
-	tdcall(TDG_VM_WR, &args);
+	/* Configure the TD */
+	tdx_setup(&cc_mask);
+
+	cc_set_mask(cc_mask);
 
 	/*
 	 * All bits above GPA width are reserved and kernel treats shared bit
@@ -990,6 +1169,19 @@ void __init tdx_early_init(void)
 	x86_platform.guest.enc_kexec_finish	     = tdx_kexec_finish;
 
 	/*
+	 * Avoid "sti;hlt" execution in TDX guests as HLT induces a #VE that
+	 * will enable interrupts before HLT TDCALL invocation if executed
+	 * in STI-shadow, possibly resulting in missed wakeup events.
+	 *
+	 * Modify all possible HLT execution paths to use TDX specific routines
+	 * that directly execute TDCALL and toggle the interrupt state as
+	 * needed after TDCALL completion. This also reduces HLT related #VEs
+	 * in addition to having a reliable halt logic execution.
+	 */
+	pv_ops.irq.safe_halt = tdx_safe_halt;
+	pv_ops.irq.halt = tdx_halt;
+
+	/*
 	 * TDX intercepts the RDMSR to read the X2APIC ID in the parallel
 	 * bringup low level code. That raises #VE which cannot be handled
 	 * there.
@@ -1000,5 +1192,5 @@ void __init tdx_early_init(void)
 	 */
 	x86_cpuinit.parallel_bringup = false;
 
-	pr_info("Guest detected\n");
+	tdx_announce();
 }
diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig
index 91801138b10b..79fa38ca954d 100644
--- a/arch/x86/configs/i386_defconfig
+++ b/arch/x86/configs/i386_defconfig
@@ -1,7 +1,6 @@
 CONFIG_WERROR=y
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_USELIB=y
 CONFIG_AUDIT=y
 CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
@@ -28,10 +27,12 @@ CONFIG_CGROUP_DEBUG=y
 CONFIG_BLK_DEV_INITRD=y
 CONFIG_KALLSYMS_ALL=y
 CONFIG_PROFILING=y
+CONFIG_KEXEC=y
+# Do not remove this as it results in non-bootable kernels
+# CONFIG_64BIT is not set
 CONFIG_SMP=y
 CONFIG_HYPERVISOR_GUEST=y
 CONFIG_PARAVIRT=y
-CONFIG_NR_CPUS=8
 CONFIG_X86_REROUTE_FOR_BROKEN_BOOT_IRQS=y
 CONFIG_X86_MSR=y
 CONFIG_X86_CPUID=y
@@ -40,9 +41,6 @@ CONFIG_X86_CHECK_BIOS_CORRUPTION=y
 CONFIG_EFI=y
 CONFIG_EFI_STUB=y
 CONFIG_HZ_1000=y
-CONFIG_KEXEC=y
-CONFIG_CRASH_DUMP=y
-# CONFIG_MITIGATION_RETHUNK is not set
 CONFIG_HIBERNATION=y
 CONFIG_PM_DEBUG=y
 CONFIG_PM_TRACE_RTC=y
@@ -53,7 +51,6 @@ CONFIG_CPU_FREQ_GOV_ONDEMAND=y
 CONFIG_X86_ACPI_CPUFREQ=y
 CONFIG_KPROBES=y
 CONFIG_JUMP_LABEL=y
-CONFIG_COMPAT_32BIT_TIME=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_MODULE_FORCE_UNLOAD=y
@@ -64,9 +61,7 @@ CONFIG_BINFMT_MISC=y
 # CONFIG_COMPAT_BRK is not set
 CONFIG_NET=y
 CONFIG_PACKET=y
-CONFIG_UNIX=y
 CONFIG_XFRM_USER=y
-CONFIG_INET=y
 CONFIG_IP_MULTICAST=y
 CONFIG_IP_ADVANCED_ROUTER=y
 CONFIG_IP_MULTIPLE_TABLES=y
@@ -135,7 +130,6 @@ CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 CONFIG_DEBUG_DEVRES=y
 CONFIG_CONNECTOR=y
-CONFIG_EFI_CAPSULE_LOADER=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_VIRTIO_BLK=y
 CONFIG_BLK_DEV_SD=y
@@ -211,7 +205,6 @@ CONFIG_SND_HDA_INTEL=y
 CONFIG_SND_HDA_HWDEP=y
 CONFIG_HIDRAW=y
 CONFIG_HID_GYRATION=y
-CONFIG_LOGITECH_FF=y
 CONFIG_HID_NTRIG=y
 CONFIG_HID_PANTHERLORD=y
 CONFIG_PANTHERLORD_FF=y
@@ -242,7 +235,6 @@ CONFIG_EXT4_FS_POSIX_ACL=y
 CONFIG_EXT4_FS_SECURITY=y
 CONFIG_QUOTA=y
 CONFIG_QUOTA_NETLINK_INTERFACE=y
-# CONFIG_PRINT_QUOTA_WARNING is not set
 CONFIG_QFMT_V2=y
 CONFIG_AUTOFS_FS=y
 CONFIG_ISO9660_FS=y
@@ -267,19 +259,13 @@ CONFIG_SECURITY=y
 CONFIG_SECURITY_NETWORK=y
 CONFIG_SECURITY_SELINUX=y
 CONFIG_SECURITY_SELINUX_BOOTPARAM=y
-CONFIG_SECURITY_SELINUX_DISABLE=y
 CONFIG_PRINTK_TIME=y
 CONFIG_DEBUG_KERNEL=y
-CONFIG_FRAME_WARN=1024
 CONFIG_MAGIC_SYSRQ=y
-CONFIG_DEBUG_WX=y
 CONFIG_DEBUG_STACK_USAGE=y
-# CONFIG_SCHED_DEBUG is not set
 CONFIG_SCHEDSTATS=y
 CONFIG_BLK_DEV_IO_TRACE=y
 CONFIG_PROVIDE_OHCI1394_DMA_INIT=y
 CONFIG_EARLY_PRINTK_DBGP=y
 CONFIG_DEBUG_BOOT_PARAMS=y
-CONFIG_UNWINDER_FRAME_POINTER=y
 CONFIG_DEBUG_ENTRY=y
-# CONFIG_64BIT is not set
diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig
index 61e25f6209ed..7d7310cdf8b0 100644
--- a/arch/x86/configs/x86_64_defconfig
+++ b/arch/x86/configs/x86_64_defconfig
@@ -27,6 +27,7 @@ CONFIG_CGROUP_DEBUG=y
 CONFIG_BLK_DEV_INITRD=y
 CONFIG_KALLSYMS_ALL=y
 CONFIG_PROFILING=y
+CONFIG_KEXEC=y
 CONFIG_SMP=y
 CONFIG_HYPERVISOR_GUEST=y
 CONFIG_PARAVIRT=y
@@ -40,8 +41,6 @@ CONFIG_EFI=y
 CONFIG_EFI_STUB=y
 CONFIG_EFI_MIXED=y
 CONFIG_HZ_1000=y
-CONFIG_KEXEC=y
-CONFIG_CRASH_DUMP=y
 CONFIG_HIBERNATION=y
 CONFIG_PM_DEBUG=y
 CONFIG_PM_TRACE_RTC=y
@@ -63,9 +62,7 @@ CONFIG_BINFMT_MISC=y
 # CONFIG_COMPAT_BRK is not set
 CONFIG_NET=y
 CONFIG_PACKET=y
-CONFIG_UNIX=y
 CONFIG_XFRM_USER=y
-CONFIG_INET=y
 CONFIG_IP_MULTICAST=y
 CONFIG_IP_ADVANCED_ROUTER=y
 CONFIG_IP_MULTIPLE_TABLES=y
@@ -205,7 +202,6 @@ CONFIG_SND_HDA_INTEL=y
 CONFIG_SND_HDA_HWDEP=y
 CONFIG_HIDRAW=y
 CONFIG_HID_GYRATION=y
-CONFIG_LOGITECH_FF=y
 CONFIG_HID_NTRIG=y
 CONFIG_HID_PANTHERLORD=y
 CONFIG_PANTHERLORD_FF=y
@@ -239,7 +235,6 @@ CONFIG_EXT4_FS_POSIX_ACL=y
 CONFIG_EXT4_FS_SECURITY=y
 CONFIG_QUOTA=y
 CONFIG_QUOTA_NETLINK_INTERFACE=y
-# CONFIG_PRINT_QUOTA_WARNING is not set
 CONFIG_QFMT_V2=y
 CONFIG_AUTOFS_FS=y
 CONFIG_ISO9660_FS=y
@@ -264,13 +259,11 @@ CONFIG_SECURITY=y
 CONFIG_SECURITY_NETWORK=y
 CONFIG_SECURITY_SELINUX=y
 CONFIG_SECURITY_SELINUX_BOOTPARAM=y
-CONFIG_SECURITY_SELINUX_DISABLE=y
 CONFIG_PRINTK_TIME=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_WX=y
 CONFIG_DEBUG_STACK_USAGE=y
-# CONFIG_SCHED_DEBUG is not set
 CONFIG_SCHEDSTATS=y
 CONFIG_BLK_DEV_IO_TRACE=y
 CONFIG_PROVIDE_OHCI1394_DMA_INIT=y
diff --git a/arch/x86/configs/xen.config b/arch/x86/configs/xen.config
index 581296255b39..98b6952ba9d2 100644
--- a/arch/x86/configs/xen.config
+++ b/arch/x86/configs/xen.config
@@ -1,6 +1,4 @@
 # global x86 required specific stuff
-# On 32-bit HIGHMEM4G is not allowed
-CONFIG_HIGHMEM64G=y
 CONFIG_64BIT=y
 
 # These enable us to allow some of the
@@ -14,7 +12,6 @@ CONFIG_CPU_FREQ=y
 
 # x86 xen specific config options
 CONFIG_XEN_PVH=y
-CONFIG_XEN_SAVE_RESTORE=y
 # CONFIG_XEN_DEBUG_FS is not set
 CONFIG_XEN_MCE_LOG=y
 CONFIG_XEN_ACPI_PROCESSOR=m
diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig
index 7b1bebed879d..3fd2423d3cf8 100644
--- a/arch/x86/crypto/Kconfig
+++ b/arch/x86/crypto/Kconfig
@@ -2,26 +2,13 @@
 
 menu "Accelerated Cryptographic Algorithms for CPU (x86)"
 
-config CRYPTO_CURVE25519_X86
-	tristate "Public key crypto: Curve25519 (ADX)"
-	depends on X86 && 64BIT
-	select CRYPTO_LIB_CURVE25519_GENERIC
-	select CRYPTO_ARCH_HAVE_LIB_CURVE25519
-	help
-	  Curve25519 algorithm
-
-	  Architecture: x86_64 using:
-	  - ADX (large integer arithmetic)
-
 config CRYPTO_AES_NI_INTEL
 	tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XCTR, XTS, GCM (AES-NI/VAES)"
-	depends on X86
 	select CRYPTO_AEAD
 	select CRYPTO_LIB_AES
 	select CRYPTO_LIB_GF128MUL
 	select CRYPTO_ALGAPI
 	select CRYPTO_SKCIPHER
-	select CRYPTO_SIMD
 	help
 	  Block cipher: AES cipher algorithms
 	  AEAD cipher: AES with GCM
@@ -36,7 +23,7 @@ config CRYPTO_AES_NI_INTEL
 
 config CRYPTO_BLOWFISH_X86_64
 	tristate "Ciphers: Blowfish, modes: ECB, CBC"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
 	select CRYPTO_BLOWFISH_COMMON
 	imply CRYPTO_CTR
@@ -48,7 +35,7 @@ config CRYPTO_BLOWFISH_X86_64
 
 config CRYPTO_CAMELLIA_X86_64
 	tristate "Ciphers: Camellia with modes: ECB, CBC"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
 	imply CRYPTO_CTR
 	help
@@ -59,10 +46,9 @@ config CRYPTO_CAMELLIA_X86_64
 
 config CRYPTO_CAMELLIA_AESNI_AVX_X86_64
 	tristate "Ciphers: Camellia with modes: ECB, CBC (AES-NI/AVX)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
 	select CRYPTO_CAMELLIA_X86_64
-	select CRYPTO_SIMD
 	imply CRYPTO_XTS
 	help
 	  Length-preserving ciphers: Camellia with ECB and CBC modes
@@ -73,7 +59,7 @@ config CRYPTO_CAMELLIA_AESNI_AVX_X86_64
 
 config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64
 	tristate "Ciphers: Camellia with modes: ECB, CBC (AES-NI/AVX2)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_CAMELLIA_AESNI_AVX_X86_64
 	help
 	  Length-preserving ciphers: Camellia with ECB and CBC modes
@@ -84,11 +70,10 @@ config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64
 
 config CRYPTO_CAST5_AVX_X86_64
 	tristate "Ciphers: CAST5 with modes: ECB, CBC (AVX)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
 	select CRYPTO_CAST5
 	select CRYPTO_CAST_COMMON
-	select CRYPTO_SIMD
 	imply CRYPTO_CTR
 	help
 	  Length-preserving ciphers: CAST5 (CAST-128) cipher algorithm
@@ -101,11 +86,10 @@ config CRYPTO_CAST5_AVX_X86_64
 
 config CRYPTO_CAST6_AVX_X86_64
 	tristate "Ciphers: CAST6 with modes: ECB, CBC (AVX)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
 	select CRYPTO_CAST6
 	select CRYPTO_CAST_COMMON
-	select CRYPTO_SIMD
 	imply CRYPTO_XTS
 	imply CRYPTO_CTR
 	help
@@ -119,7 +103,7 @@ config CRYPTO_CAST6_AVX_X86_64
 
 config CRYPTO_DES3_EDE_X86_64
 	tristate "Ciphers: Triple DES EDE with modes: ECB, CBC"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
 	select CRYPTO_LIB_DES
 	imply CRYPTO_CTR
@@ -133,10 +117,9 @@ config CRYPTO_DES3_EDE_X86_64
 
 config CRYPTO_SERPENT_SSE2_X86_64
 	tristate "Ciphers: Serpent with modes: ECB, CBC (SSE2)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
 	select CRYPTO_SERPENT
-	select CRYPTO_SIMD
 	imply CRYPTO_CTR
 	help
 	  Length-preserving ciphers: Serpent cipher algorithm
@@ -149,10 +132,9 @@ config CRYPTO_SERPENT_SSE2_X86_64
 
 config CRYPTO_SERPENT_SSE2_586
 	tristate "Ciphers: Serpent with modes: ECB, CBC (32-bit with SSE2)"
-	depends on X86 && !64BIT
+	depends on !64BIT
 	select CRYPTO_SKCIPHER
 	select CRYPTO_SERPENT
-	select CRYPTO_SIMD
 	imply CRYPTO_CTR
 	help
 	  Length-preserving ciphers: Serpent cipher algorithm
@@ -165,10 +147,9 @@ config CRYPTO_SERPENT_SSE2_586
 
 config CRYPTO_SERPENT_AVX_X86_64
 	tristate "Ciphers: Serpent with modes: ECB, CBC (AVX)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
 	select CRYPTO_SERPENT
-	select CRYPTO_SIMD
 	imply CRYPTO_XTS
 	imply CRYPTO_CTR
 	help
@@ -182,7 +163,7 @@ config CRYPTO_SERPENT_AVX_X86_64
 
 config CRYPTO_SERPENT_AVX2_X86_64
 	tristate "Ciphers: Serpent with modes: ECB, CBC (AVX2)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SERPENT_AVX_X86_64
 	help
 	  Length-preserving ciphers: Serpent cipher algorithm
@@ -195,9 +176,8 @@ config CRYPTO_SERPENT_AVX2_X86_64
 
 config CRYPTO_SM4_AESNI_AVX_X86_64
 	tristate "Ciphers: SM4 with modes: ECB, CBC, CTR (AES-NI/AVX)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
-	select CRYPTO_SIMD
 	select CRYPTO_ALGAPI
 	select CRYPTO_SM4
 	help
@@ -216,9 +196,8 @@ config CRYPTO_SM4_AESNI_AVX_X86_64
 
 config CRYPTO_SM4_AESNI_AVX2_X86_64
 	tristate "Ciphers: SM4 with modes: ECB, CBC, CTR (AES-NI/AVX2)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
-	select CRYPTO_SIMD
 	select CRYPTO_ALGAPI
 	select CRYPTO_SM4
 	select CRYPTO_SM4_AESNI_AVX_X86_64
@@ -238,7 +217,7 @@ config CRYPTO_SM4_AESNI_AVX2_X86_64
 
 config CRYPTO_TWOFISH_586
 	tristate "Ciphers: Twofish (32-bit)"
-	depends on (X86 || UML_X86) && !64BIT
+	depends on !64BIT
 	select CRYPTO_ALGAPI
 	select CRYPTO_TWOFISH_COMMON
 	imply CRYPTO_CTR
@@ -249,7 +228,7 @@ config CRYPTO_TWOFISH_586
 
 config CRYPTO_TWOFISH_X86_64
 	tristate "Ciphers: Twofish"
-	depends on (X86 || UML_X86) && 64BIT
+	depends on 64BIT
 	select CRYPTO_ALGAPI
 	select CRYPTO_TWOFISH_COMMON
 	imply CRYPTO_CTR
@@ -260,7 +239,7 @@ config CRYPTO_TWOFISH_X86_64
 
 config CRYPTO_TWOFISH_X86_64_3WAY
 	tristate "Ciphers: Twofish with modes: ECB, CBC (3-way parallel)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
 	select CRYPTO_TWOFISH_COMMON
 	select CRYPTO_TWOFISH_X86_64
@@ -275,9 +254,8 @@ config CRYPTO_TWOFISH_X86_64_3WAY
 
 config CRYPTO_TWOFISH_AVX_X86_64
 	tristate "Ciphers: Twofish with modes: ECB, CBC (AVX)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
-	select CRYPTO_SIMD
 	select CRYPTO_TWOFISH_COMMON
 	select CRYPTO_TWOFISH_X86_64
 	select CRYPTO_TWOFISH_X86_64_3WAY
@@ -293,9 +271,8 @@ config CRYPTO_TWOFISH_AVX_X86_64
 
 config CRYPTO_ARIA_AESNI_AVX_X86_64
 	tristate "Ciphers: ARIA with modes: ECB, CTR (AES-NI/AVX/GFNI)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
-	select CRYPTO_SIMD
 	select CRYPTO_ALGAPI
 	select CRYPTO_ARIA
 	help
@@ -311,9 +288,8 @@ config CRYPTO_ARIA_AESNI_AVX_X86_64
 
 config CRYPTO_ARIA_AESNI_AVX2_X86_64
 	tristate "Ciphers: ARIA with modes: ECB, CTR (AES-NI/AVX2/GFNI)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
-	select CRYPTO_SIMD
 	select CRYPTO_ALGAPI
 	select CRYPTO_ARIA
 	select CRYPTO_ARIA_AESNI_AVX_X86_64
@@ -330,9 +306,8 @@ config CRYPTO_ARIA_AESNI_AVX2_X86_64
 
 config CRYPTO_ARIA_GFNI_AVX512_X86_64
 	tristate "Ciphers: ARIA with modes: ECB, CTR (AVX512/GFNI)"
-	depends on X86 && 64BIT && AS_AVX512 && AS_GFNI
+	depends on 64BIT
 	select CRYPTO_SKCIPHER
-	select CRYPTO_SIMD
 	select CRYPTO_ALGAPI
 	select CRYPTO_ARIA
 	select CRYPTO_ARIA_AESNI_AVX_X86_64
@@ -347,36 +322,20 @@ config CRYPTO_ARIA_GFNI_AVX512_X86_64
 
 	  Processes 64 blocks in parallel.
 
-config CRYPTO_CHACHA20_X86_64
-	tristate "Ciphers: ChaCha20, XChaCha20, XChaCha12 (SSSE3/AVX2/AVX-512VL)"
-	depends on X86 && 64BIT
-	select CRYPTO_SKCIPHER
-	select CRYPTO_LIB_CHACHA_GENERIC
-	select CRYPTO_ARCH_HAVE_LIB_CHACHA
-	help
-	  Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12
-	  stream cipher algorithms
-
-	  Architecture: x86_64 using:
-	  - SSSE3 (Supplemental SSE3)
-	  - AVX2 (Advanced Vector Extensions 2)
-	  - AVX-512VL (Advanced Vector Extensions-512VL)
-
 config CRYPTO_AEGIS128_AESNI_SSE2
-	tristate "AEAD ciphers: AEGIS-128 (AES-NI/SSE2)"
-	depends on X86 && 64BIT
+	tristate "AEAD ciphers: AEGIS-128 (AES-NI/SSE4.1)"
+	depends on 64BIT
 	select CRYPTO_AEAD
-	select CRYPTO_SIMD
 	help
 	  AEGIS-128 AEAD algorithm
 
 	  Architecture: x86_64 using:
 	  - AES-NI (AES New Instructions)
-	  - SSE2 (Streaming SIMD Extensions 2)
+	  - SSE4.1 (Streaming SIMD Extensions 4.1)
 
 config CRYPTO_NHPOLY1305_SSE2
 	tristate "Hash functions: NHPoly1305 (SSE2)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_NHPOLY1305
 	help
 	  NHPoly1305 hash function for Adiantum
@@ -386,7 +345,7 @@ config CRYPTO_NHPOLY1305_SSE2
 
 config CRYPTO_NHPOLY1305_AVX2
 	tristate "Hash functions: NHPoly1305 (AVX2)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_NHPOLY1305
 	help
 	  NHPoly1305 hash function for Adiantum
@@ -394,86 +353,11 @@ config CRYPTO_NHPOLY1305_AVX2
 	  Architecture: x86_64 using:
 	  - AVX2 (Advanced Vector Extensions 2)
 
-config CRYPTO_BLAKE2S_X86
-	bool "Hash functions: BLAKE2s (SSSE3/AVX-512)"
-	depends on X86 && 64BIT
-	select CRYPTO_LIB_BLAKE2S_GENERIC
-	select CRYPTO_ARCH_HAVE_LIB_BLAKE2S
-	help
-	  BLAKE2s cryptographic hash function (RFC 7693)
-
-	  Architecture: x86_64 using:
-	  - SSSE3 (Supplemental SSE3)
-	  - AVX-512 (Advanced Vector Extensions-512)
-
-config CRYPTO_POLYVAL_CLMUL_NI
-	tristate "Hash functions: POLYVAL (CLMUL-NI)"
-	depends on X86 && 64BIT
-	select CRYPTO_POLYVAL
-	help
-	  POLYVAL hash function for HCTR2
-
-	  Architecture: x86_64 using:
-	  - CLMUL-NI (carry-less multiplication new instructions)
-
-config CRYPTO_POLY1305_X86_64
-	tristate "Hash functions: Poly1305 (SSE2/AVX2)"
-	depends on X86 && 64BIT
-	select CRYPTO_LIB_POLY1305_GENERIC
-	select CRYPTO_ARCH_HAVE_LIB_POLY1305
-	help
-	  Poly1305 authenticator algorithm (RFC7539)
-
-	  Architecture: x86_64 using:
-	  - SSE2 (Streaming SIMD Extensions 2)
-	  - AVX2 (Advanced Vector Extensions 2)
-
-config CRYPTO_SHA1_SSSE3
-	tristate "Hash functions: SHA-1 (SSSE3/AVX/AVX2/SHA-NI)"
-	depends on X86 && 64BIT
-	select CRYPTO_SHA1
-	select CRYPTO_HASH
-	help
-	  SHA-1 secure hash algorithm (FIPS 180)
-
-	  Architecture: x86_64 using:
-	  - SSSE3 (Supplemental SSE3)
-	  - AVX (Advanced Vector Extensions)
-	  - AVX2 (Advanced Vector Extensions 2)
-	  - SHA-NI (SHA Extensions New Instructions)
-
-config CRYPTO_SHA256_SSSE3
-	tristate "Hash functions: SHA-224 and SHA-256 (SSSE3/AVX/AVX2/SHA-NI)"
-	depends on X86 && 64BIT
-	select CRYPTO_SHA256
-	select CRYPTO_HASH
-	help
-	  SHA-224 and SHA-256 secure hash algorithms (FIPS 180)
-
-	  Architecture: x86_64 using:
-	  - SSSE3 (Supplemental SSE3)
-	  - AVX (Advanced Vector Extensions)
-	  - AVX2 (Advanced Vector Extensions 2)
-	  - SHA-NI (SHA Extensions New Instructions)
-
-config CRYPTO_SHA512_SSSE3
-	tristate "Hash functions: SHA-384 and SHA-512 (SSSE3/AVX/AVX2)"
-	depends on X86 && 64BIT
-	select CRYPTO_SHA512
-	select CRYPTO_HASH
-	help
-	  SHA-384 and SHA-512 secure hash algorithms (FIPS 180)
-
-	  Architecture: x86_64 using:
-	  - SSSE3 (Supplemental SSE3)
-	  - AVX (Advanced Vector Extensions)
-	  - AVX2 (Advanced Vector Extensions 2)
-
 config CRYPTO_SM3_AVX_X86_64
 	tristate "Hash functions: SM3 (AVX)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_HASH
-	select CRYPTO_SM3
+	select CRYPTO_LIB_SM3
 	help
 	  SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3
 
@@ -484,7 +368,7 @@ config CRYPTO_SM3_AVX_X86_64
 
 config CRYPTO_GHASH_CLMUL_NI_INTEL
 	tristate "Hash functions: GHASH (CLMUL-NI)"
-	depends on X86 && 64BIT
+	depends on 64BIT
 	select CRYPTO_CRYPTD
 	help
 	  GCM GHASH hash function (NIST SP800-38D)
@@ -492,36 +376,4 @@ config CRYPTO_GHASH_CLMUL_NI_INTEL
 	  Architecture: x86_64 using:
 	  - CLMUL-NI (carry-less multiplication new instructions)
 
-config CRYPTO_CRC32C_INTEL
-	tristate "CRC32c (SSE4.2/PCLMULQDQ)"
-	depends on X86
-	select CRYPTO_HASH
-	help
-	  CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720)
-
-	  Architecture: x86 (32-bit and 64-bit) using:
-	  - SSE4.2 (Streaming SIMD Extensions 4.2) CRC32 instruction
-	  - PCLMULQDQ (carry-less multiplication)
-
-config CRYPTO_CRC32_PCLMUL
-	tristate "CRC32 (PCLMULQDQ)"
-	depends on X86
-	select CRYPTO_HASH
-	select CRC32
-	help
-	  CRC32 CRC algorithm (IEEE 802.3)
-
-	  Architecture: x86 (32-bit and 64-bit) using:
-	  - PCLMULQDQ (carry-less multiplication)
-
-config CRYPTO_CRCT10DIF_PCLMUL
-	tristate "CRCT10DIF (PCLMULQDQ)"
-	depends on X86 && 64BIT && CRC_T10DIF
-	select CRYPTO_HASH
-	help
-	  CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF)
-
-	  Architecture: x86_64 using:
-	  - PCLMULQDQ (carry-less multiplication)
-
 endmenu
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 53b4a277809e..5f2fb4f148fe 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -42,60 +42,22 @@ cast6-avx-x86_64-y := cast6-avx-x86_64-asm_64.o cast6_avx_glue.o
 obj-$(CONFIG_CRYPTO_AEGIS128_AESNI_SSE2) += aegis128-aesni.o
 aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o
 
-obj-$(CONFIG_CRYPTO_CHACHA20_X86_64) += chacha-x86_64.o
-chacha-x86_64-y := chacha-avx2-x86_64.o chacha-ssse3-x86_64.o chacha_glue.o
-chacha-x86_64-$(CONFIG_AS_AVX512) += chacha-avx512vl-x86_64.o
-
 obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
 aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o
-aesni-intel-$(CONFIG_64BIT) += aes_ctrby8_avx-x86_64.o \
+aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \
 			       aes-gcm-aesni-x86_64.o \
+			       aes-gcm-vaes-avx2.o \
+			       aes-gcm-vaes-avx512.o \
 			       aes-xts-avx-x86_64.o
-ifeq ($(CONFIG_AS_VAES)$(CONFIG_AS_VPCLMULQDQ),yy)
-aesni-intel-$(CONFIG_64BIT) += aes-gcm-avx10-x86_64.o
-endif
-
-obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o
-sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ssse3_glue.o
-sha1-ssse3-$(CONFIG_AS_SHA1_NI) += sha1_ni_asm.o
-
-obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o
-sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ssse3_glue.o
-sha256-ssse3-$(CONFIG_AS_SHA256_NI) += sha256_ni_asm.o
-
-obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o
-sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o
-
-obj-$(CONFIG_CRYPTO_BLAKE2S_X86) += libblake2s-x86_64.o
-libblake2s-x86_64-y := blake2s-core.o blake2s-glue.o
 
 obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
 ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
 
-obj-$(CONFIG_CRYPTO_POLYVAL_CLMUL_NI) += polyval-clmulni.o
-polyval-clmulni-y := polyval-clmulni_asm.o polyval-clmulni_glue.o
-
-obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
-crc32c-intel-y := crc32c-intel_glue.o
-crc32c-intel-$(CONFIG_64BIT) += crc32c-pcl-intel-asm_64.o
-
-obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o
-crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o
-
-obj-$(CONFIG_CRYPTO_CRCT10DIF_PCLMUL) += crct10dif-pclmul.o
-crct10dif-pclmul-y := crct10dif-pcl-asm_64.o crct10dif-pclmul_glue.o
-
-obj-$(CONFIG_CRYPTO_POLY1305_X86_64) += poly1305-x86_64.o
-poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o
-targets += poly1305-x86_64-cryptogams.S
-
 obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o
 nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o
 obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o
 nhpoly1305-avx2-y := nh-avx2-x86_64.o nhpoly1305-avx2-glue.o
 
-obj-$(CONFIG_CRYPTO_CURVE25519_X86) += curve25519-x86_64.o
-
 obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o
 sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o
 
@@ -113,11 +75,3 @@ aria-aesni-avx2-x86_64-y := aria-aesni-avx2-asm_64.o aria_aesni_avx2_glue.o
 
 obj-$(CONFIG_CRYPTO_ARIA_GFNI_AVX512_X86_64) += aria-gfni-avx512-x86_64.o
 aria-gfni-avx512-x86_64-y := aria-gfni-avx512-asm_64.o aria_gfni_avx512_glue.o
-
-quiet_cmd_perlasm = PERLASM $@
-      cmd_perlasm = $(PERL) $< > $@
-$(obj)/%.S: $(src)/%.pl FORCE
-	$(call if_changed,perlasm)
-
-# Disable GCOV in odd or sensitive code
-GCOV_PROFILE_curve25519-x86_64.o := n
diff --git a/arch/x86/crypto/aegis128-aesni-asm.S b/arch/x86/crypto/aegis128-aesni-asm.S
index ad7f4c891625..7294dc0ee7ba 100644
--- a/arch/x86/crypto/aegis128-aesni-asm.S
+++ b/arch/x86/crypto/aegis128-aesni-asm.S
@@ -1,14 +1,13 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * AES-NI + SSE2 implementation of AEGIS-128
+ * AES-NI + SSE4.1 implementation of AEGIS-128
  *
  * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
  * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ * Copyright 2024 Google LLC
  */
 
 #include <linux/linkage.h>
-#include <linux/cfi_types.h>
-#include <asm/frame.h>
 
 #define STATE0	%xmm0
 #define STATE1	%xmm1
@@ -20,11 +19,6 @@
 #define T0	%xmm6
 #define T1	%xmm7
 
-#define STATEP	%rdi
-#define LEN	%rsi
-#define SRC	%rdx
-#define DST	%rcx
-
 .section .rodata.cst16.aegis128_const, "aM", @progbits, 32
 .align 16
 .Laegis128_const_0:
@@ -34,11 +28,11 @@
 	.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
 	.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
 
-.section .rodata.cst16.aegis128_counter, "aM", @progbits, 16
-.align 16
-.Laegis128_counter:
-	.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
-	.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
+.section .rodata.cst32.zeropad_mask, "aM", @progbits, 32
+.align 32
+.Lzeropad_mask:
+	.octa 0xffffffffffffffffffffffffffffffff
+	.octa 0
 
 .text
 
@@ -61,140 +55,102 @@
 .endm
 
 /*
- * __load_partial: internal ABI
- * input:
- *   LEN - bytes
- *   SRC - src
- * output:
- *   MSG  - message block
- * changed:
- *   T0
- *   %r8
- *   %r9
+ * Load 1 <= LEN (%ecx) <= 15 bytes from the pointer SRC into the xmm register
+ * MSG and zeroize any remaining bytes.  Clobbers %rax, %rcx, and %r8.
  */
-SYM_FUNC_START_LOCAL(__load_partial)
-	xor %r9d, %r9d
-	pxor MSG, MSG
-
-	mov LEN, %r8
-	and $0x1, %r8
-	jz .Lld_partial_1
-
-	mov LEN, %r8
-	and $0x1E, %r8
-	add SRC, %r8
-	mov (%r8), %r9b
-
-.Lld_partial_1:
-	mov LEN, %r8
-	and $0x2, %r8
-	jz .Lld_partial_2
-
-	mov LEN, %r8
-	and $0x1C, %r8
-	add SRC, %r8
-	shl $0x10, %r9
-	mov (%r8), %r9w
-
-.Lld_partial_2:
-	mov LEN, %r8
-	and $0x4, %r8
-	jz .Lld_partial_4
-
-	mov LEN, %r8
-	and $0x18, %r8
-	add SRC, %r8
-	shl $32, %r9
-	mov (%r8), %r8d
-	xor %r8, %r9
-
-.Lld_partial_4:
-	movq %r9, MSG
-
-	mov LEN, %r8
-	and $0x8, %r8
-	jz .Lld_partial_8
-
-	mov LEN, %r8
-	and $0x10, %r8
-	add SRC, %r8
-	pslldq $8, MSG
-	movq (%r8), T0
-	pxor T0, MSG
-
-.Lld_partial_8:
-	RET
-SYM_FUNC_END(__load_partial)
+.macro load_partial
+	sub $8, %ecx			/* LEN - 8 */
+	jle .Lle8\@
+
+	/* Load 9 <= LEN <= 15 bytes: */
+	movq (SRC), MSG			/* Load first 8 bytes */
+	mov (SRC, %rcx), %rax		/* Load last 8 bytes */
+	neg %ecx
+	shl $3, %ecx
+	shr %cl, %rax			/* Discard overlapping bytes */
+	pinsrq $1, %rax, MSG
+	jmp .Ldone\@
+
+.Lle8\@:
+	add $4, %ecx			/* LEN - 4 */
+	jl .Llt4\@
+
+	/* Load 4 <= LEN <= 8 bytes: */
+	mov (SRC), %eax			/* Load first 4 bytes */
+	mov (SRC, %rcx), %r8d		/* Load last 4 bytes */
+	jmp .Lcombine\@
+
+.Llt4\@:
+	/* Load 1 <= LEN <= 3 bytes: */
+	add $2, %ecx			/* LEN - 2 */
+	movzbl (SRC), %eax		/* Load first byte */
+	jl .Lmovq\@
+	movzwl (SRC, %rcx), %r8d	/* Load last 2 bytes */
+.Lcombine\@:
+	shl $3, %ecx
+	shl %cl, %r8
+	or %r8, %rax			/* Combine the two parts */
+.Lmovq\@:
+	movq %rax, MSG
+.Ldone\@:
+.endm
 
 /*
- * __store_partial: internal ABI
- * input:
- *   LEN - bytes
- *   DST - dst
- * output:
- *   T0   - message block
- * changed:
- *   %r8
- *   %r9
- *   %r10
+ * Store 1 <= LEN (%ecx) <= 15 bytes from the xmm register \msg to the pointer
+ * DST.  Clobbers %rax, %rcx, and %r8.
  */
-SYM_FUNC_START_LOCAL(__store_partial)
-	mov LEN, %r8
-	mov DST, %r9
-
-	movq T0, %r10
-
-	cmp $8, %r8
-	jl .Lst_partial_8
-
-	mov %r10, (%r9)
-	psrldq $8, T0
-	movq T0, %r10
-
-	sub $8, %r8
-	add $8, %r9
-
-.Lst_partial_8:
-	cmp $4, %r8
-	jl .Lst_partial_4
-
-	mov %r10d, (%r9)
-	shr $32, %r10
-
-	sub $4, %r8
-	add $4, %r9
-
-.Lst_partial_4:
-	cmp $2, %r8
-	jl .Lst_partial_2
-
-	mov %r10w, (%r9)
-	shr $0x10, %r10
-
-	sub $2, %r8
-	add $2, %r9
-
-.Lst_partial_2:
-	cmp $1, %r8
-	jl .Lst_partial_1
-
-	mov %r10b, (%r9)
-
-.Lst_partial_1:
-	RET
-SYM_FUNC_END(__store_partial)
+.macro store_partial msg
+	sub $8, %ecx			/* LEN - 8 */
+	jl .Llt8\@
+
+	/* Store 8 <= LEN <= 15 bytes: */
+	pextrq $1, \msg, %rax
+	mov %ecx, %r8d
+	shl $3, %ecx
+	ror %cl, %rax
+	mov %rax, (DST, %r8)		/* Store last LEN - 8 bytes */
+	movq \msg, (DST)		/* Store first 8 bytes */
+	jmp .Ldone\@
+
+.Llt8\@:
+	add $4, %ecx			/* LEN - 4 */
+	jl .Llt4\@
+
+	/* Store 4 <= LEN <= 7 bytes: */
+	pextrd $1, \msg, %eax
+	mov %ecx, %r8d
+	shl $3, %ecx
+	ror %cl, %eax
+	mov %eax, (DST, %r8)		/* Store last LEN - 4 bytes */
+	movd \msg, (DST)		/* Store first 4 bytes */
+	jmp .Ldone\@
+
+.Llt4\@:
+	/* Store 1 <= LEN <= 3 bytes: */
+	pextrb $0, \msg, 0(DST)
+	cmp $-2, %ecx			/* LEN - 4 == -2, i.e. LEN == 2? */
+	jl .Ldone\@
+	pextrb $1, \msg, 1(DST)
+	je .Ldone\@
+	pextrb $2, \msg, 2(DST)
+.Ldone\@:
+.endm
 
 /*
- * void crypto_aegis128_aesni_init(void *state, const void *key, const void *iv);
+ * void aegis128_aesni_init(struct aegis_state *state,
+ *			    const struct aegis_block *key,
+ *			    const u8 iv[AEGIS128_NONCE_SIZE]);
  */
-SYM_FUNC_START(crypto_aegis128_aesni_init)
-	FRAME_BEGIN
+SYM_FUNC_START(aegis128_aesni_init)
+	.set STATEP, %rdi
+	.set KEYP, %rsi
+	.set IVP, %rdx
 
 	/* load IV: */
-	movdqu (%rdx), T1
+	movdqu (IVP), T1
 
 	/* load key: */
-	movdqa (%rsi), KEY
+	movdqa (KEYP), KEY
 	pxor KEY, T1
 	movdqa T1, STATE0
 	movdqa KEY, STATE3
@@ -224,20 +180,22 @@ SYM_FUNC_START(crypto_aegis128_aesni_init)
 	movdqu STATE2, 0x20(STATEP)
 	movdqu STATE3, 0x30(STATEP)
 	movdqu STATE4, 0x40(STATEP)
-
-	FRAME_END
 	RET
-SYM_FUNC_END(crypto_aegis128_aesni_init)
+SYM_FUNC_END(aegis128_aesni_init)
 
 /*
- * void crypto_aegis128_aesni_ad(void *state, unsigned int length,
- *                               const void *data);
+ * void aegis128_aesni_ad(struct aegis_state *state, const u8 *data,
+ *			  unsigned int len);
+ *
+ * len must be a multiple of 16.
  */
-SYM_FUNC_START(crypto_aegis128_aesni_ad)
-	FRAME_BEGIN
+SYM_FUNC_START(aegis128_aesni_ad)
+	.set STATEP, %rdi
+	.set SRC, %rsi
+	.set LEN, %edx
 
-	cmp $0x10, LEN
-	jb .Lad_out
+	test LEN, LEN
+	jz .Lad_out
 
 	/* load the state: */
 	movdqu 0x00(STATEP), STATE0
@@ -246,89 +204,40 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
 	movdqu 0x30(STATEP), STATE3
 	movdqu 0x40(STATEP), STATE4
 
-	mov SRC, %r8
-	and $0xF, %r8
-	jnz .Lad_u_loop
-
-.align 8
-.Lad_a_loop:
-	movdqa 0x00(SRC), MSG
-	aegis128_update
-	pxor MSG, STATE4
-	sub $0x10, LEN
-	cmp $0x10, LEN
-	jl .Lad_out_1
-
-	movdqa 0x10(SRC), MSG
-	aegis128_update
-	pxor MSG, STATE3
-	sub $0x10, LEN
-	cmp $0x10, LEN
-	jl .Lad_out_2
-
-	movdqa 0x20(SRC), MSG
-	aegis128_update
-	pxor MSG, STATE2
-	sub $0x10, LEN
-	cmp $0x10, LEN
-	jl .Lad_out_3
-
-	movdqa 0x30(SRC), MSG
-	aegis128_update
-	pxor MSG, STATE1
-	sub $0x10, LEN
-	cmp $0x10, LEN
-	jl .Lad_out_4
-
-	movdqa 0x40(SRC), MSG
-	aegis128_update
-	pxor MSG, STATE0
-	sub $0x10, LEN
-	cmp $0x10, LEN
-	jl .Lad_out_0
-
-	add $0x50, SRC
-	jmp .Lad_a_loop
-
 .align 8
-.Lad_u_loop:
+.Lad_loop:
 	movdqu 0x00(SRC), MSG
 	aegis128_update
 	pxor MSG, STATE4
 	sub $0x10, LEN
-	cmp $0x10, LEN
-	jl .Lad_out_1
+	jz .Lad_out_1
 
 	movdqu 0x10(SRC), MSG
 	aegis128_update
 	pxor MSG, STATE3
 	sub $0x10, LEN
-	cmp $0x10, LEN
-	jl .Lad_out_2
+	jz .Lad_out_2
 
 	movdqu 0x20(SRC), MSG
 	aegis128_update
 	pxor MSG, STATE2
 	sub $0x10, LEN
-	cmp $0x10, LEN
-	jl .Lad_out_3
+	jz .Lad_out_3
 
 	movdqu 0x30(SRC), MSG
 	aegis128_update
 	pxor MSG, STATE1
 	sub $0x10, LEN
-	cmp $0x10, LEN
-	jl .Lad_out_4
+	jz .Lad_out_4
 
 	movdqu 0x40(SRC), MSG
 	aegis128_update
 	pxor MSG, STATE0
 	sub $0x10, LEN
-	cmp $0x10, LEN
-	jl .Lad_out_0
+	jz .Lad_out_0
 
 	add $0x50, SRC
-	jmp .Lad_u_loop
+	jmp .Lad_loop
 
 	/* store the state: */
 .Lad_out_0:
@@ -337,7 +246,6 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
 	movdqu STATE2, 0x20(STATEP)
 	movdqu STATE3, 0x30(STATEP)
 	movdqu STATE4, 0x40(STATEP)
-	FRAME_END
 	RET
 
 .Lad_out_1:
@@ -346,7 +254,6 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
 	movdqu STATE1, 0x20(STATEP)
 	movdqu STATE2, 0x30(STATEP)
 	movdqu STATE3, 0x40(STATEP)
-	FRAME_END
 	RET
 
 .Lad_out_2:
@@ -355,7 +262,6 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
 	movdqu STATE0, 0x20(STATEP)
 	movdqu STATE1, 0x30(STATEP)
 	movdqu STATE2, 0x40(STATEP)
-	FRAME_END
 	RET
 
 .Lad_out_3:
@@ -364,7 +270,6 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
 	movdqu STATE4, 0x20(STATEP)
 	movdqu STATE0, 0x30(STATEP)
 	movdqu STATE1, 0x40(STATEP)
-	FRAME_END
 	RET
 
 .Lad_out_4:
@@ -373,41 +278,38 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
 	movdqu STATE3, 0x20(STATEP)
 	movdqu STATE4, 0x30(STATEP)
 	movdqu STATE0, 0x40(STATEP)
-	FRAME_END
-	RET
-
 .Lad_out:
-	FRAME_END
 	RET
-SYM_FUNC_END(crypto_aegis128_aesni_ad)
+SYM_FUNC_END(aegis128_aesni_ad)
 
-.macro encrypt_block a s0 s1 s2 s3 s4 i
-	movdq\a (\i * 0x10)(SRC), MSG
+.macro encrypt_block s0 s1 s2 s3 s4 i
+	movdqu (\i * 0x10)(SRC), MSG
 	movdqa MSG, T0
 	pxor \s1, T0
 	pxor \s4, T0
 	movdqa \s2, T1
 	pand \s3, T1
 	pxor T1, T0
-	movdq\a T0, (\i * 0x10)(DST)
+	movdqu T0, (\i * 0x10)(DST)
 
 	aegis128_update
 	pxor MSG, \s4
 
 	sub $0x10, LEN
-	cmp $0x10, LEN
-	jl .Lenc_out_\i
+	jz .Lenc_out_\i
 .endm
 
 /*
- * void crypto_aegis128_aesni_enc(void *state, unsigned int length,
- *                                const void *src, void *dst);
+ * void aegis128_aesni_enc(struct aegis_state *state, const u8 *src, u8 *dst,
+ *			   unsigned int len);
+ *
+ * len must be nonzero and a multiple of 16.
  */
-SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc)
-	FRAME_BEGIN
-
-	cmp $0x10, LEN
-	jb .Lenc_out
+SYM_FUNC_START(aegis128_aesni_enc)
+	.set STATEP, %rdi
+	.set SRC, %rsi
+	.set DST, %rdx
+	.set LEN, %ecx
 
 	/* load the state: */
 	movdqu 0x00(STATEP), STATE0
@@ -416,34 +318,17 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc)
 	movdqu 0x30(STATEP), STATE3
 	movdqu 0x40(STATEP), STATE4
 
-	mov  SRC,  %r8
-	or   DST,  %r8
-	and $0xF, %r8
-	jnz .Lenc_u_loop
-
 .align 8
-.Lenc_a_loop:
-	encrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0
-	encrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1
-	encrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2
-	encrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3
-	encrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4
+.Lenc_loop:
+	encrypt_block STATE0 STATE1 STATE2 STATE3 STATE4 0
+	encrypt_block STATE4 STATE0 STATE1 STATE2 STATE3 1
+	encrypt_block STATE3 STATE4 STATE0 STATE1 STATE2 2
+	encrypt_block STATE2 STATE3 STATE4 STATE0 STATE1 3
+	encrypt_block STATE1 STATE2 STATE3 STATE4 STATE0 4
 
 	add $0x50, SRC
 	add $0x50, DST
-	jmp .Lenc_a_loop
-
-.align 8
-.Lenc_u_loop:
-	encrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0
-	encrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1
-	encrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2
-	encrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3
-	encrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4
-
-	add $0x50, SRC
-	add $0x50, DST
-	jmp .Lenc_u_loop
+	jmp .Lenc_loop
 
 	/* store the state: */
 .Lenc_out_0:
@@ -452,7 +337,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc)
 	movdqu STATE1, 0x20(STATEP)
 	movdqu STATE2, 0x30(STATEP)
 	movdqu STATE3, 0x40(STATEP)
-	FRAME_END
 	RET
 
 .Lenc_out_1:
@@ -461,7 +345,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc)
 	movdqu STATE0, 0x20(STATEP)
 	movdqu STATE1, 0x30(STATEP)
 	movdqu STATE2, 0x40(STATEP)
-	FRAME_END
 	RET
 
 .Lenc_out_2:
@@ -470,7 +353,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc)
 	movdqu STATE4, 0x20(STATEP)
 	movdqu STATE0, 0x30(STATEP)
 	movdqu STATE1, 0x40(STATEP)
-	FRAME_END
 	RET
 
 .Lenc_out_3:
@@ -479,7 +361,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc)
 	movdqu STATE3, 0x20(STATEP)
 	movdqu STATE4, 0x30(STATEP)
 	movdqu STATE0, 0x40(STATEP)
-	FRAME_END
 	RET
 
 .Lenc_out_4:
@@ -488,20 +369,19 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc)
 	movdqu STATE2, 0x20(STATEP)
 	movdqu STATE3, 0x30(STATEP)
 	movdqu STATE4, 0x40(STATEP)
-	FRAME_END
-	RET
-
 .Lenc_out:
-	FRAME_END
 	RET
-SYM_FUNC_END(crypto_aegis128_aesni_enc)
+SYM_FUNC_END(aegis128_aesni_enc)
 
 /*
- * void crypto_aegis128_aesni_enc_tail(void *state, unsigned int length,
- *                                     const void *src, void *dst);
+ * void aegis128_aesni_enc_tail(struct aegis_state *state, const u8 *src,
+ *				u8 *dst, unsigned int len);
  */
-SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc_tail)
-	FRAME_BEGIN
+SYM_FUNC_START(aegis128_aesni_enc_tail)
+	.set STATEP, %rdi
+	.set SRC, %rsi
+	.set DST, %rdx
+	.set LEN, %ecx	/* {load,store}_partial rely on this being %ecx */
 
 	/* load the state: */
 	movdqu 0x00(STATEP), STATE0
@@ -511,7 +391,8 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc_tail)
 	movdqu 0x40(STATEP), STATE4
 
 	/* encrypt message: */
-	call __load_partial
+	mov LEN, %r9d
+	load_partial
 
 	movdqa MSG, T0
 	pxor STATE1, T0
@@ -520,7 +401,8 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc_tail)
 	pand STATE3, T1
 	pxor T1, T0
 
-	call __store_partial
+	mov %r9d, LEN
+	store_partial T0
 
 	aegis128_update
 	pxor MSG, STATE4
@@ -531,37 +413,36 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc_tail)
 	movdqu STATE1, 0x20(STATEP)
 	movdqu STATE2, 0x30(STATEP)
 	movdqu STATE3, 0x40(STATEP)
-
-	FRAME_END
 	RET
-SYM_FUNC_END(crypto_aegis128_aesni_enc_tail)
+SYM_FUNC_END(aegis128_aesni_enc_tail)
 
-.macro decrypt_block a s0 s1 s2 s3 s4 i
-	movdq\a (\i * 0x10)(SRC), MSG
+.macro decrypt_block s0 s1 s2 s3 s4 i
+	movdqu (\i * 0x10)(SRC), MSG
 	pxor \s1, MSG
 	pxor \s4, MSG
 	movdqa \s2, T1
 	pand \s3, T1
 	pxor T1, MSG
-	movdq\a MSG, (\i * 0x10)(DST)
+	movdqu MSG, (\i * 0x10)(DST)
 
 	aegis128_update
 	pxor MSG, \s4
 
 	sub $0x10, LEN
-	cmp $0x10, LEN
-	jl .Ldec_out_\i
+	jz .Ldec_out_\i
 .endm
 
 /*
- * void crypto_aegis128_aesni_dec(void *state, unsigned int length,
- *                                const void *src, void *dst);
+ * void aegis128_aesni_dec(struct aegis_state *state, const u8 *src, u8 *dst,
+ *			   unsigned int len);
+ *
+ * len must be nonzero and a multiple of 16.
  */
-SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec)
-	FRAME_BEGIN
-
-	cmp $0x10, LEN
-	jb .Ldec_out
+SYM_FUNC_START(aegis128_aesni_dec)
+	.set STATEP, %rdi
+	.set SRC, %rsi
+	.set DST, %rdx
+	.set LEN, %ecx
 
 	/* load the state: */
 	movdqu 0x00(STATEP), STATE0
@@ -570,34 +451,17 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec)
 	movdqu 0x30(STATEP), STATE3
 	movdqu 0x40(STATEP), STATE4
 
-	mov  SRC, %r8
-	or   DST, %r8
-	and $0xF, %r8
-	jnz .Ldec_u_loop
-
 .align 8
-.Ldec_a_loop:
-	decrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0
-	decrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1
-	decrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2
-	decrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3
-	decrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4
+.Ldec_loop:
+	decrypt_block STATE0 STATE1 STATE2 STATE3 STATE4 0
+	decrypt_block STATE4 STATE0 STATE1 STATE2 STATE3 1
+	decrypt_block STATE3 STATE4 STATE0 STATE1 STATE2 2
+	decrypt_block STATE2 STATE3 STATE4 STATE0 STATE1 3
+	decrypt_block STATE1 STATE2 STATE3 STATE4 STATE0 4
 
 	add $0x50, SRC
 	add $0x50, DST
-	jmp .Ldec_a_loop
-
-.align 8
-.Ldec_u_loop:
-	decrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0
-	decrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1
-	decrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2
-	decrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3
-	decrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4
-
-	add $0x50, SRC
-	add $0x50, DST
-	jmp .Ldec_u_loop
+	jmp .Ldec_loop
 
 	/* store the state: */
 .Ldec_out_0:
@@ -606,7 +470,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec)
 	movdqu STATE1, 0x20(STATEP)
 	movdqu STATE2, 0x30(STATEP)
 	movdqu STATE3, 0x40(STATEP)
-	FRAME_END
 	RET
 
 .Ldec_out_1:
@@ -615,7 +478,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec)
 	movdqu STATE0, 0x20(STATEP)
 	movdqu STATE1, 0x30(STATEP)
 	movdqu STATE2, 0x40(STATEP)
-	FRAME_END
 	RET
 
 .Ldec_out_2:
@@ -624,7 +486,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec)
 	movdqu STATE4, 0x20(STATEP)
 	movdqu STATE0, 0x30(STATEP)
 	movdqu STATE1, 0x40(STATEP)
-	FRAME_END
 	RET
 
 .Ldec_out_3:
@@ -633,7 +494,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec)
 	movdqu STATE3, 0x20(STATEP)
 	movdqu STATE4, 0x30(STATEP)
 	movdqu STATE0, 0x40(STATEP)
-	FRAME_END
 	RET
 
 .Ldec_out_4:
@@ -642,20 +502,19 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec)
 	movdqu STATE2, 0x20(STATEP)
 	movdqu STATE3, 0x30(STATEP)
 	movdqu STATE4, 0x40(STATEP)
-	FRAME_END
-	RET
-
 .Ldec_out:
-	FRAME_END
 	RET
-SYM_FUNC_END(crypto_aegis128_aesni_dec)
+SYM_FUNC_END(aegis128_aesni_dec)
 
 /*
- * void crypto_aegis128_aesni_dec_tail(void *state, unsigned int length,
- *                                     const void *src, void *dst);
+ * void aegis128_aesni_dec_tail(struct aegis_state *state, const u8 *src,
+ *				u8 *dst, unsigned int len);
  */
-SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec_tail)
-	FRAME_BEGIN
+SYM_FUNC_START(aegis128_aesni_dec_tail)
+	.set STATEP, %rdi
+	.set SRC, %rsi
+	.set DST, %rdx
+	.set LEN, %ecx	/* {load,store}_partial rely on this being %ecx */
 
 	/* load the state: */
 	movdqu 0x00(STATEP), STATE0
@@ -665,7 +524,8 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec_tail)
 	movdqu 0x40(STATEP), STATE4
 
 	/* decrypt message: */
-	call __load_partial
+	mov LEN, %r9d
+	load_partial
 
 	pxor STATE1, MSG
 	pxor STATE4, MSG
@@ -673,17 +533,13 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec_tail)
 	pand STATE3, T1
 	pxor T1, MSG
 
-	movdqa MSG, T0
-	call __store_partial
+	mov %r9d, LEN
+	store_partial MSG
 
 	/* mask with byte count: */
-	movq LEN, T0
-	punpcklbw T0, T0
-	punpcklbw T0, T0
-	punpcklbw T0, T0
-	punpcklbw T0, T0
-	movdqa .Laegis128_counter(%rip), T1
-	pcmpgtb T1, T0
+	lea .Lzeropad_mask+16(%rip), %rax
+	sub %r9, %rax
+	movdqu (%rax), T0
 	pand T0, MSG
 
 	aegis128_update
@@ -695,17 +551,19 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec_tail)
 	movdqu STATE1, 0x20(STATEP)
 	movdqu STATE2, 0x30(STATEP)
 	movdqu STATE3, 0x40(STATEP)
-
-	FRAME_END
 	RET
-SYM_FUNC_END(crypto_aegis128_aesni_dec_tail)
+SYM_FUNC_END(aegis128_aesni_dec_tail)
 
 /*
- * void crypto_aegis128_aesni_final(void *state, void *tag_xor,
- *                                  u64 assoclen, u64 cryptlen);
+ * void aegis128_aesni_final(struct aegis_state *state,
+ *			     struct aegis_block *tag_xor,
+ *			     unsigned int assoclen, unsigned int cryptlen);
  */
-SYM_FUNC_START(crypto_aegis128_aesni_final)
-	FRAME_BEGIN
+SYM_FUNC_START(aegis128_aesni_final)
+	.set STATEP, %rdi
+	.set TAG_XOR, %rsi
+	.set ASSOCLEN, %edx
+	.set CRYPTLEN, %ecx
 
 	/* load the state: */
 	movdqu 0x00(STATEP), STATE0
@@ -715,10 +573,8 @@ SYM_FUNC_START(crypto_aegis128_aesni_final)
 	movdqu 0x40(STATEP), STATE4
 
 	/* prepare length block: */
-	movq %rdx, MSG
-	movq %rcx, T0
-	pslldq $8, T0
-	pxor T0, MSG
+	movd ASSOCLEN, MSG
+	pinsrd $2, CRYPTLEN, MSG
 	psllq $3, MSG /* multiply by 8 (to get bit count) */
 
 	pxor STATE3, MSG
@@ -733,7 +589,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_final)
 	aegis128_update; pxor MSG, STATE3
 
 	/* xor tag: */
-	movdqu (%rsi), MSG
+	movdqu (TAG_XOR), MSG
 
 	pxor STATE0, MSG
 	pxor STATE1, MSG
@@ -741,8 +597,6 @@ SYM_FUNC_START(crypto_aegis128_aesni_final)
 	pxor STATE3, MSG
 	pxor STATE4, MSG
 
-	movdqu MSG, (%rsi)
-
-	FRAME_END
+	movdqu MSG, (TAG_XOR)
 	RET
-SYM_FUNC_END(crypto_aegis128_aesni_final)
+SYM_FUNC_END(aegis128_aesni_final)
diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c
index 4623189000d8..f1adfba1a76e 100644
--- a/arch/x86/crypto/aegis128-aesni-glue.c
+++ b/arch/x86/crypto/aegis128-aesni-glue.c
@@ -1,14 +1,13 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  * The AEGIS-128 Authenticated-Encryption Algorithm
- *   Glue for AES-NI + SSE2 implementation
+ *   Glue for AES-NI + SSE4.1 implementation
  *
  * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
  * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
  */
 
 #include <crypto/internal/aead.h>
-#include <crypto/internal/simd.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/scatterwalk.h>
 #include <linux/module.h>
@@ -23,27 +22,6 @@
 #define AEGIS128_MIN_AUTH_SIZE 8
 #define AEGIS128_MAX_AUTH_SIZE 16
 
-asmlinkage void crypto_aegis128_aesni_init(void *state, void *key, void *iv);
-
-asmlinkage void crypto_aegis128_aesni_ad(
-		void *state, unsigned int length, const void *data);
-
-asmlinkage void crypto_aegis128_aesni_enc(
-		void *state, unsigned int length, const void *src, void *dst);
-
-asmlinkage void crypto_aegis128_aesni_dec(
-		void *state, unsigned int length, const void *src, void *dst);
-
-asmlinkage void crypto_aegis128_aesni_enc_tail(
-		void *state, unsigned int length, const void *src, void *dst);
-
-asmlinkage void crypto_aegis128_aesni_dec_tail(
-		void *state, unsigned int length, const void *src, void *dst);
-
-asmlinkage void crypto_aegis128_aesni_final(
-		void *state, void *tag_xor, unsigned int cryptlen,
-		unsigned int assoclen);
-
 struct aegis_block {
 	u8 bytes[AEGIS128_BLOCK_SIZE] __aligned(AEGIS128_BLOCK_ALIGN);
 };
@@ -56,15 +34,31 @@ struct aegis_ctx {
 	struct aegis_block key;
 };
 
-struct aegis_crypt_ops {
-	int (*skcipher_walk_init)(struct skcipher_walk *walk,
-				  struct aead_request *req, bool atomic);
+asmlinkage void aegis128_aesni_init(struct aegis_state *state,
+				    const struct aegis_block *key,
+				    const u8 iv[AEGIS128_NONCE_SIZE]);
 
-	void (*crypt_blocks)(void *state, unsigned int length, const void *src,
-			     void *dst);
-	void (*crypt_tail)(void *state, unsigned int length, const void *src,
-			   void *dst);
-};
+asmlinkage void aegis128_aesni_ad(struct aegis_state *state, const u8 *data,
+				  unsigned int len);
+
+asmlinkage void aegis128_aesni_enc(struct aegis_state *state, const u8 *src,
+				   u8 *dst, unsigned int len);
+
+asmlinkage void aegis128_aesni_dec(struct aegis_state *state, const u8 *src,
+				   u8 *dst, unsigned int len);
+
+asmlinkage void aegis128_aesni_enc_tail(struct aegis_state *state,
+					const u8 *src, u8 *dst,
+					unsigned int len);
+
+asmlinkage void aegis128_aesni_dec_tail(struct aegis_state *state,
+					const u8 *src, u8 *dst,
+					unsigned int len);
+
+asmlinkage void aegis128_aesni_final(struct aegis_state *state,
+				     struct aegis_block *tag_xor,
+				     unsigned int assoclen,
+				     unsigned int cryptlen);
 
 static void crypto_aegis128_aesni_process_ad(
 		struct aegis_state *state, struct scatterlist *sg_src,
@@ -76,25 +70,23 @@ static void crypto_aegis128_aesni_process_ad(
 
 	scatterwalk_start(&walk, sg_src);
 	while (assoclen != 0) {
-		unsigned int size = scatterwalk_clamp(&walk, assoclen);
+		unsigned int size = scatterwalk_next(&walk, assoclen);
+		const u8 *src = walk.addr;
 		unsigned int left = size;
-		void *mapped = scatterwalk_map(&walk);
-		const u8 *src = (const u8 *)mapped;
 
 		if (pos + size >= AEGIS128_BLOCK_SIZE) {
 			if (pos > 0) {
 				unsigned int fill = AEGIS128_BLOCK_SIZE - pos;
 				memcpy(buf.bytes + pos, src, fill);
-				crypto_aegis128_aesni_ad(state,
-							 AEGIS128_BLOCK_SIZE,
-							 buf.bytes);
+				aegis128_aesni_ad(state, buf.bytes,
+						  AEGIS128_BLOCK_SIZE);
 				pos = 0;
 				left -= fill;
 				src += fill;
 			}
 
-			crypto_aegis128_aesni_ad(state, left, src);
-
+			aegis128_aesni_ad(state, src,
+					  left & ~(AEGIS128_BLOCK_SIZE - 1));
 			src += left & ~(AEGIS128_BLOCK_SIZE - 1);
 			left &= AEGIS128_BLOCK_SIZE - 1;
 		}
@@ -103,33 +95,52 @@ static void crypto_aegis128_aesni_process_ad(
 		pos += left;
 		assoclen -= size;
 
-		scatterwalk_unmap(mapped);
-		scatterwalk_advance(&walk, size);
-		scatterwalk_done(&walk, 0, assoclen);
+		scatterwalk_done_src(&walk, size);
 	}
 
 	if (pos > 0) {
 		memset(buf.bytes + pos, 0, AEGIS128_BLOCK_SIZE - pos);
-		crypto_aegis128_aesni_ad(state, AEGIS128_BLOCK_SIZE, buf.bytes);
+		aegis128_aesni_ad(state, buf.bytes, AEGIS128_BLOCK_SIZE);
 	}
 }
 
-static void crypto_aegis128_aesni_process_crypt(
-		struct aegis_state *state, struct skcipher_walk *walk,
-		const struct aegis_crypt_ops *ops)
+static __always_inline int
+crypto_aegis128_aesni_process_crypt(struct aegis_state *state,
+				    struct skcipher_walk *walk, bool enc)
 {
+	int err = 0;
+
 	while (walk->nbytes >= AEGIS128_BLOCK_SIZE) {
-		ops->crypt_blocks(state,
-				  round_down(walk->nbytes, AEGIS128_BLOCK_SIZE),
-				  walk->src.virt.addr, walk->dst.virt.addr);
-		skcipher_walk_done(walk, walk->nbytes % AEGIS128_BLOCK_SIZE);
+		if (enc)
+			aegis128_aesni_enc(state, walk->src.virt.addr,
+					   walk->dst.virt.addr,
+					   round_down(walk->nbytes,
+						      AEGIS128_BLOCK_SIZE));
+		else
+			aegis128_aesni_dec(state, walk->src.virt.addr,
+					   walk->dst.virt.addr,
+					   round_down(walk->nbytes,
+						      AEGIS128_BLOCK_SIZE));
+		kernel_fpu_end();
+		err = skcipher_walk_done(walk,
+					 walk->nbytes % AEGIS128_BLOCK_SIZE);
+		kernel_fpu_begin();
 	}
 
 	if (walk->nbytes) {
-		ops->crypt_tail(state, walk->nbytes, walk->src.virt.addr,
-				walk->dst.virt.addr);
-		skcipher_walk_done(walk, 0);
+		if (enc)
+			aegis128_aesni_enc_tail(state, walk->src.virt.addr,
+						walk->dst.virt.addr,
+						walk->nbytes);
+		else
+			aegis128_aesni_dec_tail(state, walk->src.virt.addr,
+						walk->dst.virt.addr,
+						walk->nbytes);
+		kernel_fpu_end();
+		err = skcipher_walk_done(walk, 0);
+		kernel_fpu_begin();
 	}
+	return err;
 }
 
 static struct aegis_ctx *crypto_aegis128_aesni_ctx(struct crypto_aead *aead)
@@ -162,42 +173,46 @@ static int crypto_aegis128_aesni_setauthsize(struct crypto_aead *tfm,
 	return 0;
 }
 
-static void crypto_aegis128_aesni_crypt(struct aead_request *req,
-					struct aegis_block *tag_xor,
-					unsigned int cryptlen,
-					const struct aegis_crypt_ops *ops)
+static __always_inline int
+crypto_aegis128_aesni_crypt(struct aead_request *req,
+			    struct aegis_block *tag_xor,
+			    unsigned int cryptlen, bool enc)
 {
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(tfm);
 	struct skcipher_walk walk;
 	struct aegis_state state;
+	int err;
 
-	ops->skcipher_walk_init(&walk, req, true);
+	if (enc)
+		err = skcipher_walk_aead_encrypt(&walk, req, false);
+	else
+		err = skcipher_walk_aead_decrypt(&walk, req, false);
+	if (err)
+		return err;
 
 	kernel_fpu_begin();
 
-	crypto_aegis128_aesni_init(&state, ctx->key.bytes, req->iv);
+	aegis128_aesni_init(&state, &ctx->key, req->iv);
 	crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen);
-	crypto_aegis128_aesni_process_crypt(&state, &walk, ops);
-	crypto_aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen);
-
+	err = crypto_aegis128_aesni_process_crypt(&state, &walk, enc);
+	if (err == 0)
+		aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen);
 	kernel_fpu_end();
+	return err;
 }
 
 static int crypto_aegis128_aesni_encrypt(struct aead_request *req)
 {
-	static const struct aegis_crypt_ops OPS = {
-		.skcipher_walk_init = skcipher_walk_aead_encrypt,
-		.crypt_blocks = crypto_aegis128_aesni_enc,
-		.crypt_tail = crypto_aegis128_aesni_enc_tail,
-	};
-
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	struct aegis_block tag = {};
 	unsigned int authsize = crypto_aead_authsize(tfm);
 	unsigned int cryptlen = req->cryptlen;
+	int err;
 
-	crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);
+	err = crypto_aegis128_aesni_crypt(req, &tag, cryptlen, true);
+	if (err)
+		return err;
 
 	scatterwalk_map_and_copy(tag.bytes, req->dst,
 				 req->assoclen + cryptlen, authsize, 1);
@@ -208,77 +223,58 @@ static int crypto_aegis128_aesni_decrypt(struct aead_request *req)
 {
 	static const struct aegis_block zeros = {};
 
-	static const struct aegis_crypt_ops OPS = {
-		.skcipher_walk_init = skcipher_walk_aead_decrypt,
-		.crypt_blocks = crypto_aegis128_aesni_dec,
-		.crypt_tail = crypto_aegis128_aesni_dec_tail,
-	};
-
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	struct aegis_block tag;
 	unsigned int authsize = crypto_aead_authsize(tfm);
 	unsigned int cryptlen = req->cryptlen - authsize;
+	int err;
 
 	scatterwalk_map_and_copy(tag.bytes, req->src,
 				 req->assoclen + cryptlen, authsize, 0);
 
-	crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);
+	err = crypto_aegis128_aesni_crypt(req, &tag, cryptlen, false);
+	if (err)
+		return err;
 
 	return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0;
 }
 
-static int crypto_aegis128_aesni_init_tfm(struct crypto_aead *aead)
-{
-	return 0;
-}
-
-static void crypto_aegis128_aesni_exit_tfm(struct crypto_aead *aead)
-{
-}
-
 static struct aead_alg crypto_aegis128_aesni_alg = {
 	.setkey = crypto_aegis128_aesni_setkey,
 	.setauthsize = crypto_aegis128_aesni_setauthsize,
 	.encrypt = crypto_aegis128_aesni_encrypt,
 	.decrypt = crypto_aegis128_aesni_decrypt,
-	.init = crypto_aegis128_aesni_init_tfm,
-	.exit = crypto_aegis128_aesni_exit_tfm,
 
 	.ivsize = AEGIS128_NONCE_SIZE,
 	.maxauthsize = AEGIS128_MAX_AUTH_SIZE,
 	.chunksize = AEGIS128_BLOCK_SIZE,
 
 	.base = {
-		.cra_flags = CRYPTO_ALG_INTERNAL,
 		.cra_blocksize = 1,
 		.cra_ctxsize = sizeof(struct aegis_ctx) +
 			       __alignof__(struct aegis_ctx),
-		.cra_alignmask = 0,
 		.cra_priority = 400,
 
-		.cra_name = "__aegis128",
-		.cra_driver_name = "__aegis128-aesni",
+		.cra_name = "aegis128",
+		.cra_driver_name = "aegis128-aesni",
 
 		.cra_module = THIS_MODULE,
 	}
 };
 
-static struct simd_aead_alg *simd_alg;
-
 static int __init crypto_aegis128_aesni_module_init(void)
 {
-	if (!boot_cpu_has(X86_FEATURE_XMM2) ||
+	if (!boot_cpu_has(X86_FEATURE_XMM4_1) ||
 	    !boot_cpu_has(X86_FEATURE_AES) ||
 	    !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
 		return -ENODEV;
 
-	return simd_register_aeads_compat(&crypto_aegis128_aesni_alg, 1,
-					  &simd_alg);
+	return crypto_register_aead(&crypto_aegis128_aesni_alg);
 }
 
 static void __exit crypto_aegis128_aesni_module_exit(void)
 {
-	simd_unregister_aeads(&crypto_aegis128_aesni_alg, 1, &simd_alg);
+	crypto_unregister_aead(&crypto_aegis128_aesni_alg);
 }
 
 module_init(crypto_aegis128_aesni_module_init);
@@ -286,6 +282,6 @@ module_exit(crypto_aegis128_aesni_module_exit);
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
-MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm -- AESNI+SSE2 implementation");
+MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm -- AESNI+SSE4.1 implementation");
 MODULE_ALIAS_CRYPTO("aegis128");
 MODULE_ALIAS_CRYPTO("aegis128-aesni");
diff --git a/arch/x86/crypto/aes-ctr-avx-x86_64.S b/arch/x86/crypto/aes-ctr-avx-x86_64.S
new file mode 100644
index 000000000000..2745918f68ee
--- /dev/null
+++ b/arch/x86/crypto/aes-ctr-avx-x86_64.S
@@ -0,0 +1,571 @@
+/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
+//
+// Copyright 2025 Google LLC
+//
+// Author: Eric Biggers <ebiggers@google.com>
+//
+// This file is dual-licensed, meaning that you can use it under your choice of
+// either of the following two licenses:
+//
+// Licensed under the Apache License 2.0 (the "License").  You may obtain a copy
+// of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// or
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+//    this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//------------------------------------------------------------------------------
+//
+// This file contains x86_64 assembly implementations of AES-CTR and AES-XCTR
+// using the following sets of CPU features:
+//	- AES-NI && AVX
+//	- VAES && AVX2
+//	- VAES && AVX512BW && AVX512VL && BMI2
+//
+// See the function definitions at the bottom of the file for more information.
+
+#include <linux/linkage.h>
+#include <linux/cfi_types.h>
+
+.section .rodata
+.p2align 4
+
+.Lbswap_mask:
+	.octa	0x000102030405060708090a0b0c0d0e0f
+
+.Lctr_pattern:
+	.quad	0, 0
+.Lone:
+	.quad	1, 0
+.Ltwo:
+	.quad	2, 0
+	.quad	3, 0
+
+.Lfour:
+	.quad	4, 0
+
+.text
+
+// Move a vector between memory and a register.
+.macro	_vmovdqu	src, dst
+.if VL < 64
+	vmovdqu		\src, \dst
+.else
+	vmovdqu8	\src, \dst
+.endif
+.endm
+
+// Move a vector between registers.
+.macro	_vmovdqa	src, dst
+.if VL < 64
+	vmovdqa		\src, \dst
+.else
+	vmovdqa64	\src, \dst
+.endif
+.endm
+
+// Broadcast a 128-bit value from memory to all 128-bit lanes of a vector
+// register.
+.macro	_vbroadcast128	src, dst
+.if VL == 16
+	vmovdqu		\src, \dst
+.elseif VL == 32
+	vbroadcasti128	\src, \dst
+.else
+	vbroadcasti32x4	\src, \dst
+.endif
+.endm
+
+// XOR two vectors together.
+.macro	_vpxor	src1, src2, dst
+.if VL < 64
+	vpxor		\src1, \src2, \dst
+.else
+	vpxord		\src1, \src2, \dst
+.endif
+.endm
+
+// Load 1 <= %ecx <= 15 bytes from the pointer \src into the xmm register \dst
+// and zeroize any remaining bytes.  Clobbers %rax, %rcx, and \tmp{64,32}.
+.macro	_load_partial_block	src, dst, tmp64, tmp32
+	sub		$8, %ecx		// LEN - 8
+	jle		.Lle8\@
+
+	// Load 9 <= LEN <= 15 bytes.
+	vmovq		(\src), \dst		// Load first 8 bytes
+	mov		(\src, %rcx), %rax	// Load last 8 bytes
+	neg		%ecx
+	shl		$3, %ecx
+	shr		%cl, %rax		// Discard overlapping bytes
+	vpinsrq		$1, %rax, \dst, \dst
+	jmp		.Ldone\@
+
+.Lle8\@:
+	add		$4, %ecx		// LEN - 4
+	jl		.Llt4\@
+
+	// Load 4 <= LEN <= 8 bytes.
+	mov		(\src), %eax		// Load first 4 bytes
+	mov		(\src, %rcx), \tmp32	// Load last 4 bytes
+	jmp		.Lcombine\@
+
+.Llt4\@:
+	// Load 1 <= LEN <= 3 bytes.
+	add		$2, %ecx		// LEN - 2
+	movzbl		(\src), %eax		// Load first byte
+	jl		.Lmovq\@
+	movzwl		(\src, %rcx), \tmp32	// Load last 2 bytes
+.Lcombine\@:
+	shl		$3, %ecx
+	shl		%cl, \tmp64
+	or		\tmp64, %rax		// Combine the two parts
+.Lmovq\@:
+	vmovq		%rax, \dst
+.Ldone\@:
+.endm
+
+// Store 1 <= %ecx <= 15 bytes from the xmm register \src to the pointer \dst.
+// Clobbers %rax, %rcx, and \tmp{64,32}.
+.macro	_store_partial_block	src, dst, tmp64, tmp32
+	sub		$8, %ecx		// LEN - 8
+	jl		.Llt8\@
+
+	// Store 8 <= LEN <= 15 bytes.
+	vpextrq		$1, \src, %rax
+	mov		%ecx, \tmp32
+	shl		$3, %ecx
+	ror		%cl, %rax
+	mov		%rax, (\dst, \tmp64)	// Store last LEN - 8 bytes
+	vmovq		\src, (\dst)		// Store first 8 bytes
+	jmp		.Ldone\@
+
+.Llt8\@:
+	add		$4, %ecx		// LEN - 4
+	jl		.Llt4\@
+
+	// Store 4 <= LEN <= 7 bytes.
+	vpextrd		$1, \src, %eax
+	mov		%ecx, \tmp32
+	shl		$3, %ecx
+	ror		%cl, %eax
+	mov		%eax, (\dst, \tmp64)	// Store last LEN - 4 bytes
+	vmovd		\src, (\dst)		// Store first 4 bytes
+	jmp		.Ldone\@
+
+.Llt4\@:
+	// Store 1 <= LEN <= 3 bytes.
+	vpextrb		$0, \src, 0(\dst)
+	cmp		$-2, %ecx		// LEN - 4 == -2, i.e. LEN == 2?
+	jl		.Ldone\@
+	vpextrb		$1, \src, 1(\dst)
+	je		.Ldone\@
+	vpextrb		$2, \src, 2(\dst)
+.Ldone\@:
+.endm
+
+// Prepare the next two vectors of AES inputs in AESDATA\i0 and AESDATA\i1, and
+// XOR each with the zero-th round key.  Also update LE_CTR if !\final.
+.macro	_prepare_2_ctr_vecs	is_xctr, i0, i1, final=0
+.if \is_xctr
+  .if USE_AVX512
+	vmovdqa64	LE_CTR, AESDATA\i0
+	vpternlogd	$0x96, XCTR_IV, RNDKEY0, AESDATA\i0
+  .else
+	vpxor		XCTR_IV, LE_CTR, AESDATA\i0
+	vpxor		RNDKEY0, AESDATA\i0, AESDATA\i0
+  .endif
+	vpaddq		LE_CTR_INC1, LE_CTR, AESDATA\i1
+
+  .if USE_AVX512
+	vpternlogd	$0x96, XCTR_IV, RNDKEY0, AESDATA\i1
+  .else
+	vpxor		XCTR_IV, AESDATA\i1, AESDATA\i1
+	vpxor		RNDKEY0, AESDATA\i1, AESDATA\i1
+  .endif
+.else
+	vpshufb		BSWAP_MASK, LE_CTR, AESDATA\i0
+	_vpxor		RNDKEY0, AESDATA\i0, AESDATA\i0
+	vpaddq		LE_CTR_INC1, LE_CTR, AESDATA\i1
+	vpshufb		BSWAP_MASK, AESDATA\i1, AESDATA\i1
+	_vpxor		RNDKEY0, AESDATA\i1, AESDATA\i1
+.endif
+.if !\final
+	vpaddq		LE_CTR_INC2, LE_CTR, LE_CTR
+.endif
+.endm
+
+// Do all AES rounds on the data in the given AESDATA vectors, excluding the
+// zero-th and last rounds.
+.macro	_aesenc_loop	vecs:vararg
+	mov		KEY, %rax
+1:
+	_vbroadcast128	(%rax), RNDKEY
+.irp i, \vecs
+	vaesenc		RNDKEY, AESDATA\i, AESDATA\i
+.endr
+	add		$16, %rax
+	cmp		%rax, RNDKEYLAST_PTR
+	jne		1b
+.endm
+
+// Finalize the keystream blocks in the given AESDATA vectors by doing the last
+// AES round, then XOR those keystream blocks with the corresponding data.
+// Reduce latency by doing the XOR before the vaesenclast, utilizing the
+// property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a).
+.macro	_aesenclast_and_xor	vecs:vararg
+.irp i, \vecs
+	_vpxor		\i*VL(SRC), RNDKEYLAST, RNDKEY
+	vaesenclast	RNDKEY, AESDATA\i, AESDATA\i
+.endr
+.irp i, \vecs
+	_vmovdqu	AESDATA\i, \i*VL(DST)
+.endr
+.endm
+
+// XOR the keystream blocks in the specified AESDATA vectors with the
+// corresponding data.
+.macro	_xor_data	vecs:vararg
+.irp i, \vecs
+	_vpxor		\i*VL(SRC), AESDATA\i, AESDATA\i
+.endr
+.irp i, \vecs
+	_vmovdqu	AESDATA\i, \i*VL(DST)
+.endr
+.endm
+
+.macro	_aes_ctr_crypt		is_xctr
+
+	// Define register aliases V0-V15 that map to the xmm, ymm, or zmm
+	// registers according to the selected Vector Length (VL).
+.irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+  .if VL == 16
+	.set	V\i,		%xmm\i
+  .elseif VL == 32
+	.set	V\i,		%ymm\i
+  .elseif VL == 64
+	.set	V\i,		%zmm\i
+  .else
+	.error "Unsupported Vector Length (VL)"
+  .endif
+.endr
+
+	// Function arguments
+	.set	KEY,		%rdi	// Initially points to the start of the
+					// crypto_aes_ctx, then is advanced to
+					// point to the index 1 round key
+	.set	KEY32,		%edi	// Available as temp register after all
+					// keystream blocks have been generated
+	.set	SRC,		%rsi	// Pointer to next source data
+	.set	DST,		%rdx	// Pointer to next destination data
+	.set	LEN,		%ecx	// Remaining length in bytes.
+					// Note: _load_partial_block relies on
+					// this being in %ecx.
+	.set	LEN64,		%rcx	// Zero-extend LEN before using!
+	.set	LEN8,		%cl
+.if \is_xctr
+	.set	XCTR_IV_PTR,	%r8	// const u8 iv[AES_BLOCK_SIZE];
+	.set	XCTR_CTR,	%r9	// u64 ctr;
+.else
+	.set	LE_CTR_PTR,	%r8	// const u64 le_ctr[2];
+.endif
+
+	// Additional local variables
+	.set	RNDKEYLAST_PTR,	%r10
+	.set	AESDATA0,	V0
+	.set	AESDATA0_XMM,	%xmm0
+	.set	AESDATA1,	V1
+	.set	AESDATA1_XMM,	%xmm1
+	.set	AESDATA2,	V2
+	.set	AESDATA3,	V3
+	.set	AESDATA4,	V4
+	.set	AESDATA5,	V5
+	.set	AESDATA6,	V6
+	.set	AESDATA7,	V7
+.if \is_xctr
+	.set	XCTR_IV,	V8
+.else
+	.set	BSWAP_MASK,	V8
+.endif
+	.set	LE_CTR,		V9
+	.set	LE_CTR_XMM,	%xmm9
+	.set	LE_CTR_INC1,	V10
+	.set	LE_CTR_INC2,	V11
+	.set	RNDKEY0,	V12
+	.set	RNDKEYLAST,	V13
+	.set	RNDKEY,		V14
+
+	// Create the first vector of counters.
+.if \is_xctr
+  .if VL == 16
+	vmovq		XCTR_CTR, LE_CTR
+  .elseif VL == 32
+	vmovq		XCTR_CTR, LE_CTR_XMM
+	inc		XCTR_CTR
+	vmovq		XCTR_CTR, AESDATA0_XMM
+	vinserti128	$1, AESDATA0_XMM, LE_CTR, LE_CTR
+  .else
+	vpbroadcastq	XCTR_CTR, LE_CTR
+	vpsrldq		$8, LE_CTR, LE_CTR
+	vpaddq		.Lctr_pattern(%rip), LE_CTR, LE_CTR
+  .endif
+	_vbroadcast128	(XCTR_IV_PTR), XCTR_IV
+.else
+	_vbroadcast128	(LE_CTR_PTR), LE_CTR
+  .if VL > 16
+	vpaddq		.Lctr_pattern(%rip), LE_CTR, LE_CTR
+  .endif
+	_vbroadcast128	.Lbswap_mask(%rip), BSWAP_MASK
+.endif
+
+.if VL == 16
+	_vbroadcast128	.Lone(%rip), LE_CTR_INC1
+.elseif VL == 32
+	_vbroadcast128	.Ltwo(%rip), LE_CTR_INC1
+.else
+	_vbroadcast128	.Lfour(%rip), LE_CTR_INC1
+.endif
+	vpsllq		$1, LE_CTR_INC1, LE_CTR_INC2
+
+	// Load the AES key length: 16 (AES-128), 24 (AES-192), or 32 (AES-256).
+	movl		480(KEY), %eax
+
+	// Compute the pointer to the last round key.
+	lea		6*16(KEY, %rax, 4), RNDKEYLAST_PTR
+
+	// Load the zero-th and last round keys.
+	_vbroadcast128	(KEY), RNDKEY0
+	_vbroadcast128	(RNDKEYLAST_PTR), RNDKEYLAST
+
+	// Make KEY point to the first round key.
+	add		$16, KEY
+
+	// This is the main loop, which encrypts 8 vectors of data at a time.
+	add		$-8*VL, LEN
+	jl		.Lloop_8x_done\@
+.Lloop_8x\@:
+	_prepare_2_ctr_vecs	\is_xctr, 0, 1
+	_prepare_2_ctr_vecs	\is_xctr, 2, 3
+	_prepare_2_ctr_vecs	\is_xctr, 4, 5
+	_prepare_2_ctr_vecs	\is_xctr, 6, 7
+	_aesenc_loop	0,1,2,3,4,5,6,7
+	_aesenclast_and_xor 0,1,2,3,4,5,6,7
+	sub		$-8*VL, SRC
+	sub		$-8*VL, DST
+	add		$-8*VL, LEN
+	jge		.Lloop_8x\@
+.Lloop_8x_done\@:
+	sub		$-8*VL, LEN
+	jz		.Ldone\@
+
+	// 1 <= LEN < 8*VL.  Generate 2, 4, or 8 more vectors of keystream
+	// blocks, depending on the remaining LEN.
+
+	_prepare_2_ctr_vecs	\is_xctr, 0, 1
+	_prepare_2_ctr_vecs	\is_xctr, 2, 3
+	cmp		$4*VL, LEN
+	jle		.Lenc_tail_atmost4vecs\@
+
+	// 4*VL < LEN < 8*VL.  Generate 8 vectors of keystream blocks.  Use the
+	// first 4 to XOR 4 full vectors of data.  Then XOR the remaining data.
+	_prepare_2_ctr_vecs	\is_xctr, 4, 5
+	_prepare_2_ctr_vecs	\is_xctr, 6, 7, final=1
+	_aesenc_loop	0,1,2,3,4,5,6,7
+	_aesenclast_and_xor 0,1,2,3
+	vaesenclast	RNDKEYLAST, AESDATA4, AESDATA0
+	vaesenclast	RNDKEYLAST, AESDATA5, AESDATA1
+	vaesenclast	RNDKEYLAST, AESDATA6, AESDATA2
+	vaesenclast	RNDKEYLAST, AESDATA7, AESDATA3
+	sub		$-4*VL, SRC
+	sub		$-4*VL, DST
+	add		$-4*VL, LEN
+	cmp		$1*VL-1, LEN
+	jle		.Lxor_tail_partial_vec_0\@
+	_xor_data	0
+	cmp		$2*VL-1, LEN
+	jle		.Lxor_tail_partial_vec_1\@
+	_xor_data	1
+	cmp		$3*VL-1, LEN
+	jle		.Lxor_tail_partial_vec_2\@
+	_xor_data	2
+	cmp		$4*VL-1, LEN
+	jle		.Lxor_tail_partial_vec_3\@
+	_xor_data	3
+	jmp		.Ldone\@
+
+.Lenc_tail_atmost4vecs\@:
+	cmp		$2*VL, LEN
+	jle		.Lenc_tail_atmost2vecs\@
+
+	// 2*VL < LEN <= 4*VL.  Generate 4 vectors of keystream blocks.  Use the
+	// first 2 to XOR 2 full vectors of data.  Then XOR the remaining data.
+	_aesenc_loop	0,1,2,3
+	_aesenclast_and_xor 0,1
+	vaesenclast	RNDKEYLAST, AESDATA2, AESDATA0
+	vaesenclast	RNDKEYLAST, AESDATA3, AESDATA1
+	sub		$-2*VL, SRC
+	sub		$-2*VL, DST
+	add		$-2*VL, LEN
+	jmp		.Lxor_tail_upto2vecs\@
+
+.Lenc_tail_atmost2vecs\@:
+	// 1 <= LEN <= 2*VL.  Generate 2 vectors of keystream blocks.  Then XOR
+	// the remaining data.
+	_aesenc_loop	0,1
+	vaesenclast	RNDKEYLAST, AESDATA0, AESDATA0
+	vaesenclast	RNDKEYLAST, AESDATA1, AESDATA1
+
+.Lxor_tail_upto2vecs\@:
+	cmp		$1*VL-1, LEN
+	jle		.Lxor_tail_partial_vec_0\@
+	_xor_data	0
+	cmp		$2*VL-1, LEN
+	jle		.Lxor_tail_partial_vec_1\@
+	_xor_data	1
+	jmp		.Ldone\@
+
+.Lxor_tail_partial_vec_1\@:
+	add		$-1*VL, LEN
+	jz		.Ldone\@
+	sub		$-1*VL, SRC
+	sub		$-1*VL, DST
+	_vmovdqa	AESDATA1, AESDATA0
+	jmp		.Lxor_tail_partial_vec_0\@
+
+.Lxor_tail_partial_vec_2\@:
+	add		$-2*VL, LEN
+	jz		.Ldone\@
+	sub		$-2*VL, SRC
+	sub		$-2*VL, DST
+	_vmovdqa	AESDATA2, AESDATA0
+	jmp		.Lxor_tail_partial_vec_0\@
+
+.Lxor_tail_partial_vec_3\@:
+	add		$-3*VL, LEN
+	jz		.Ldone\@
+	sub		$-3*VL, SRC
+	sub		$-3*VL, DST
+	_vmovdqa	AESDATA3, AESDATA0
+
+.Lxor_tail_partial_vec_0\@:
+	// XOR the remaining 1 <= LEN < VL bytes.  It's easy if masked
+	// loads/stores are available; otherwise it's a bit harder...
+.if USE_AVX512
+	mov		$-1, %rax
+	bzhi		LEN64, %rax, %rax
+	kmovq		%rax, %k1
+	vmovdqu8	(SRC), AESDATA1{%k1}{z}
+	vpxord		AESDATA1, AESDATA0, AESDATA0
+	vmovdqu8	AESDATA0, (DST){%k1}
+.else
+  .if VL == 32
+	cmp		$16, LEN
+	jl		1f
+	vpxor		(SRC), AESDATA0_XMM, AESDATA1_XMM
+	vmovdqu		AESDATA1_XMM, (DST)
+	add		$16, SRC
+	add		$16, DST
+	sub		$16, LEN
+	jz		.Ldone\@
+	vextracti128	$1, AESDATA0, AESDATA0_XMM
+1:
+  .endif
+	mov		LEN, %r10d
+	_load_partial_block	SRC, AESDATA1_XMM, KEY, KEY32
+	vpxor		AESDATA1_XMM, AESDATA0_XMM, AESDATA0_XMM
+	mov		%r10d, %ecx
+	_store_partial_block	AESDATA0_XMM, DST, KEY, KEY32
+.endif
+
+.Ldone\@:
+.if VL > 16
+	vzeroupper
+.endif
+	RET
+.endm
+
+// Below are the definitions of the functions generated by the above macro.
+// They have the following prototypes:
+//
+//
+// void aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key,
+//				 const u8 *src, u8 *dst, int len,
+//				 const u64 le_ctr[2]);
+//
+// void aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key,
+//				const u8 *src, u8 *dst, int len,
+//				const u8 iv[AES_BLOCK_SIZE], u64 ctr);
+//
+// Both functions generate |len| bytes of keystream, XOR it with the data from
+// |src|, and write the result to |dst|.  On non-final calls, |len| must be a
+// multiple of 16.  On the final call, |len| can be any value.
+//
+// aes_ctr64_crypt_* implement "regular" CTR, where the keystream is generated
+// from a 128-bit big endian counter that increments by 1 for each AES block.
+// HOWEVER, to keep the assembly code simple, some of the counter management is
+// left to the caller.  aes_ctr64_crypt_* take the counter in little endian
+// form, only increment the low 64 bits internally, do the conversion to big
+// endian internally, and don't write the updated counter back to memory.  The
+// caller is responsible for converting the starting IV to the little endian
+// le_ctr, detecting the (very rare) case of a carry out of the low 64 bits
+// being needed and splitting at that point with a carry done in between, and
+// updating le_ctr after each part if the message is multi-part.
+//
+// aes_xctr_crypt_* implement XCTR as specified in "Length-preserving encryption
+// with HCTR2" (https://eprint.iacr.org/2021/1441.pdf).  XCTR is an
+// easier-to-implement variant of CTR that uses little endian byte order and
+// eliminates carries.  |ctr| is the per-message block counter starting at 1.
+
+.set	VL, 16
+.set	USE_AVX512, 0
+SYM_TYPED_FUNC_START(aes_ctr64_crypt_aesni_avx)
+	_aes_ctr_crypt	0
+SYM_FUNC_END(aes_ctr64_crypt_aesni_avx)
+SYM_TYPED_FUNC_START(aes_xctr_crypt_aesni_avx)
+	_aes_ctr_crypt	1
+SYM_FUNC_END(aes_xctr_crypt_aesni_avx)
+
+.set	VL, 32
+.set	USE_AVX512, 0
+SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx2)
+	_aes_ctr_crypt	0
+SYM_FUNC_END(aes_ctr64_crypt_vaes_avx2)
+SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx2)
+	_aes_ctr_crypt	1
+SYM_FUNC_END(aes_xctr_crypt_vaes_avx2)
+
+.set	VL, 64
+.set	USE_AVX512, 1
+SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx512)
+	_aes_ctr_crypt	0
+SYM_FUNC_END(aes_ctr64_crypt_vaes_avx512)
+SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx512)
+	_aes_ctr_crypt	1
+SYM_FUNC_END(aes_xctr_crypt_vaes_avx512)
diff --git a/arch/x86/crypto/aes-gcm-aesni-x86_64.S b/arch/x86/crypto/aes-gcm-aesni-x86_64.S
index 45940e2883a0..7c8a8a32bd3c 100644
--- a/arch/x86/crypto/aes-gcm-aesni-x86_64.S
+++ b/arch/x86/crypto/aes-gcm-aesni-x86_64.S
@@ -61,15 +61,15 @@
 // for the *_aesni functions or AVX for the *_aesni_avx ones.  (But it seems
 // there are no CPUs that support AES-NI without also PCLMULQDQ and SSE4.1.)
 //
-// The design generally follows that of aes-gcm-avx10-x86_64.S, and that file is
+// The design generally follows that of aes-gcm-vaes-avx512.S, and that file is
 // more thoroughly commented.  This file has the following notable changes:
 //
 //    - The vector length is fixed at 128-bit, i.e. xmm registers.  This means
 //      there is only one AES block (and GHASH block) per register.
 //
-//    - Without AVX512 / AVX10, only 16 SIMD registers are available instead of
-//      32.  We work around this by being much more careful about using
-//      registers, relying heavily on loads to load values as they are needed.
+//    - Without AVX512, only 16 SIMD registers are available instead of 32.  We
+//      work around this by being much more careful about using registers,
+//      relying heavily on loads to load values as they are needed.
 //
 //    - Masking is not available either.  We work around this by implementing
 //      partial block loads and stores using overlapping scalar loads and stores
@@ -90,8 +90,8 @@
 //      multiplication instead of schoolbook multiplication.  This saves one
 //      pclmulqdq instruction per block, at the cost of one 64-bit load, one
 //      pshufd, and 0.25 pxors per block.  (This is without the three-argument
-//      XOR support that would be provided by AVX512 / AVX10, which would be
-//      more beneficial to schoolbook than Karatsuba.)
+//      XOR support that would be provided by AVX512, which would be more
+//      beneficial to schoolbook than Karatsuba.)
 //
 //      As a rough approximation, we can assume that Karatsuba multiplication is
 //      faster than schoolbook multiplication in this context if one pshufd and
diff --git a/arch/x86/crypto/aes-gcm-vaes-avx2.S b/arch/x86/crypto/aes-gcm-vaes-avx2.S
new file mode 100644
index 000000000000..93c9504a488f
--- /dev/null
+++ b/arch/x86/crypto/aes-gcm-vaes-avx2.S
@@ -0,0 +1,1146 @@
+/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
+//
+// AES-GCM implementation for x86_64 CPUs that support the following CPU
+// features: VAES && VPCLMULQDQ && AVX2
+//
+// Copyright 2025 Google LLC
+//
+// Author: Eric Biggers <ebiggers@google.com>
+//
+//------------------------------------------------------------------------------
+//
+// This file is dual-licensed, meaning that you can use it under your choice of
+// either of the following two licenses:
+//
+// Licensed under the Apache License 2.0 (the "License").  You may obtain a copy
+// of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// or
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+//    this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// -----------------------------------------------------------------------------
+//
+// This is similar to aes-gcm-vaes-avx512.S, but it uses AVX2 instead of AVX512.
+// This means it can only use 16 vector registers instead of 32, the maximum
+// vector length is 32 bytes, and some instructions such as vpternlogd and
+// masked loads/stores are unavailable.  However, it is able to run on CPUs that
+// have VAES without AVX512, namely AMD Zen 3 (including "Milan" server CPUs),
+// various Intel client CPUs such as Alder Lake, and Intel Sierra Forest.
+//
+// This implementation also uses Karatsuba multiplication instead of schoolbook
+// multiplication for GHASH in its main loop.  This does not help much on Intel,
+// but it improves performance by ~5% on AMD Zen 3.  Other factors weighing
+// slightly in favor of Karatsuba multiplication in this implementation are the
+// lower maximum vector length (which means there are fewer key powers, so we
+// can cache the halves of each key power XOR'd together and still use less
+// memory than the AVX512 implementation), and the unavailability of the
+// vpternlogd instruction (which helped schoolbook a bit more than Karatsuba).
+
+#include <linux/linkage.h>
+
+.section .rodata
+.p2align 4
+
+	// The below three 16-byte values must be in the order that they are, as
+	// they are really two 32-byte tables and a 16-byte value that overlap:
+	//
+	// - The first 32-byte table begins at .Lselect_high_bytes_table.
+	//   For 0 <= len <= 16, the 16-byte value at
+	//   '.Lselect_high_bytes_table + len' selects the high 'len' bytes of
+	//   another 16-byte value when AND'ed with it.
+	//
+	// - The second 32-byte table begins at .Lrshift_and_bswap_table.
+	//   For 0 <= len <= 16, the 16-byte value at
+	//   '.Lrshift_and_bswap_table + len' is a vpshufb mask that does the
+	//   following operation: right-shift by '16 - len' bytes (shifting in
+	//   zeroes), then reflect all 16 bytes.
+	//
+	// - The 16-byte value at .Lbswap_mask is a vpshufb mask that reflects
+	//   all 16 bytes.
+.Lselect_high_bytes_table:
+	.octa	0
+.Lrshift_and_bswap_table:
+	.octa	0xffffffffffffffffffffffffffffffff
+.Lbswap_mask:
+	.octa	0x000102030405060708090a0b0c0d0e0f
+
+	// Sixteen 0x0f bytes.  By XOR'ing an entry of .Lrshift_and_bswap_table
+	// with this, we get a mask that left-shifts by '16 - len' bytes.
+.Lfifteens:
+	.octa	0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+
+	// This is the GHASH reducing polynomial without its constant term, i.e.
+	// x^128 + x^7 + x^2 + x, represented using the backwards mapping
+	// between bits and polynomial coefficients.
+	//
+	// Alternatively, it can be interpreted as the naturally-ordered
+	// representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the
+	// "reversed" GHASH reducing polynomial without its x^128 term.
+.Lgfpoly:
+	.octa	0xc2000000000000000000000000000001
+
+	// Same as above, but with the (1 << 64) bit set.
+.Lgfpoly_and_internal_carrybit:
+	.octa	0xc2000000000000010000000000000001
+
+	// Values needed to prepare the initial vector of counter blocks.
+.Lctr_pattern:
+	.octa	0
+	.octa	1
+
+	// The number of AES blocks per vector, as a 128-bit value.
+.Linc_2blocks:
+	.octa	2
+
+// Offsets in struct aes_gcm_key_vaes_avx2
+#define OFFSETOF_AESKEYLEN	480
+#define OFFSETOF_H_POWERS	512
+#define NUM_H_POWERS		8
+#define OFFSETOFEND_H_POWERS    (OFFSETOF_H_POWERS + (NUM_H_POWERS * 16))
+#define OFFSETOF_H_POWERS_XORED	OFFSETOFEND_H_POWERS
+
+.text
+
+// Do one step of GHASH-multiplying the 128-bit lanes of \a by the 128-bit lanes
+// of \b and storing the reduced products in \dst.  Uses schoolbook
+// multiplication.
+.macro	_ghash_mul_step	i, a, b, dst, gfpoly, t0, t1, t2
+.if \i == 0
+	vpclmulqdq	$0x00, \a, \b, \t0	  // LO = a_L * b_L
+	vpclmulqdq	$0x01, \a, \b, \t1	  // MI_0 = a_L * b_H
+.elseif \i == 1
+	vpclmulqdq	$0x10, \a, \b, \t2	  // MI_1 = a_H * b_L
+.elseif \i == 2
+	vpxor		\t2, \t1, \t1		  // MI = MI_0 + MI_1
+.elseif \i == 3
+	vpclmulqdq	$0x01, \t0, \gfpoly, \t2  // LO_L*(x^63 + x^62 + x^57)
+.elseif \i == 4
+	vpshufd		$0x4e, \t0, \t0		  // Swap halves of LO
+.elseif \i == 5
+	vpxor		\t0, \t1, \t1		  // Fold LO into MI (part 1)
+	vpxor		\t2, \t1, \t1		  // Fold LO into MI (part 2)
+.elseif \i == 6
+	vpclmulqdq	$0x11, \a, \b, \dst	  // HI = a_H * b_H
+.elseif \i == 7
+	vpclmulqdq	$0x01, \t1, \gfpoly, \t0  // MI_L*(x^63 + x^62 + x^57)
+.elseif \i == 8
+	vpshufd		$0x4e, \t1, \t1		  // Swap halves of MI
+.elseif \i == 9
+	vpxor		\t1, \dst, \dst		  // Fold MI into HI (part 1)
+	vpxor		\t0, \dst, \dst		  // Fold MI into HI (part 2)
+.endif
+.endm
+
+// GHASH-multiply the 128-bit lanes of \a by the 128-bit lanes of \b and store
+// the reduced products in \dst.  See _ghash_mul_step for full explanation.
+.macro	_ghash_mul	a, b, dst, gfpoly, t0, t1, t2
+.irp i, 0,1,2,3,4,5,6,7,8,9
+	_ghash_mul_step	\i, \a, \b, \dst, \gfpoly, \t0, \t1, \t2
+.endr
+.endm
+
+// GHASH-multiply the 128-bit lanes of \a by the 128-bit lanes of \b and add the
+// *unreduced* products to \lo, \mi, and \hi.
+.macro	_ghash_mul_noreduce	a, b, lo, mi, hi, t0
+	vpclmulqdq	$0x00, \a, \b, \t0	// a_L * b_L
+	vpxor		\t0, \lo, \lo
+	vpclmulqdq	$0x01, \a, \b, \t0	// a_L * b_H
+	vpxor		\t0, \mi, \mi
+	vpclmulqdq	$0x10, \a, \b, \t0	// a_H * b_L
+	vpxor		\t0, \mi, \mi
+	vpclmulqdq	$0x11, \a, \b, \t0	// a_H * b_H
+	vpxor		\t0, \hi, \hi
+.endm
+
+// Reduce the unreduced products from \lo, \mi, and \hi and store the 128-bit
+// reduced products in \hi.  See _ghash_mul_step for explanation of reduction.
+.macro	_ghash_reduce	lo, mi, hi, gfpoly, t0
+	vpclmulqdq	$0x01, \lo, \gfpoly, \t0
+	vpshufd		$0x4e, \lo, \lo
+	vpxor		\lo, \mi, \mi
+	vpxor		\t0, \mi, \mi
+	vpclmulqdq	$0x01, \mi, \gfpoly, \t0
+	vpshufd		$0x4e, \mi, \mi
+	vpxor		\mi, \hi, \hi
+	vpxor		\t0, \hi, \hi
+.endm
+
+// This is a specialized version of _ghash_mul that computes \a * \a, i.e. it
+// squares \a.  It skips computing MI = (a_L * a_H) + (a_H * a_L) = 0.
+.macro	_ghash_square	a, dst, gfpoly, t0, t1
+	vpclmulqdq	$0x00, \a, \a, \t0	  // LO = a_L * a_L
+	vpclmulqdq	$0x11, \a, \a, \dst	  // HI = a_H * a_H
+	vpclmulqdq	$0x01, \t0, \gfpoly, \t1  // LO_L*(x^63 + x^62 + x^57)
+	vpshufd		$0x4e, \t0, \t0		  // Swap halves of LO
+	vpxor		\t0, \t1, \t1		  // Fold LO into MI
+	vpclmulqdq	$0x01, \t1, \gfpoly, \t0  // MI_L*(x^63 + x^62 + x^57)
+	vpshufd		$0x4e, \t1, \t1		  // Swap halves of MI
+	vpxor		\t1, \dst, \dst		  // Fold MI into HI (part 1)
+	vpxor		\t0, \dst, \dst		  // Fold MI into HI (part 2)
+.endm
+
+// void aes_gcm_precompute_vaes_avx2(struct aes_gcm_key_vaes_avx2 *key);
+//
+// Given the expanded AES key |key->base.aes_key|, derive the GHASH subkey and
+// initialize |key->h_powers| and |key->h_powers_xored|.
+//
+// We use h_powers[0..7] to store H^8 through H^1, and h_powers_xored[0..7] to
+// store the 64-bit halves of the key powers XOR'd together (for Karatsuba
+// multiplication) in the order 8,6,7,5,4,2,3,1.
+SYM_FUNC_START(aes_gcm_precompute_vaes_avx2)
+
+	// Function arguments
+	.set	KEY,		%rdi
+
+	// Additional local variables
+	.set	POWERS_PTR,	%rsi
+	.set	RNDKEYLAST_PTR,	%rdx
+	.set	TMP0,		%ymm0
+	.set	TMP0_XMM,	%xmm0
+	.set	TMP1,		%ymm1
+	.set	TMP1_XMM,	%xmm1
+	.set	TMP2,		%ymm2
+	.set	TMP2_XMM,	%xmm2
+	.set	H_CUR,		%ymm3
+	.set	H_CUR_XMM,	%xmm3
+	.set	H_CUR2,		%ymm4
+	.set	H_INC,		%ymm5
+	.set	H_INC_XMM,	%xmm5
+	.set	GFPOLY,		%ymm6
+	.set	GFPOLY_XMM,	%xmm6
+
+	// Encrypt an all-zeroes block to get the raw hash subkey.
+	movl		OFFSETOF_AESKEYLEN(KEY), %eax
+	lea		6*16(KEY,%rax,4), RNDKEYLAST_PTR
+	vmovdqu		(KEY), H_CUR_XMM  // Zero-th round key XOR all-zeroes block
+	lea		16(KEY), %rax
+1:
+	vaesenc		(%rax), H_CUR_XMM, H_CUR_XMM
+	add		$16, %rax
+	cmp		%rax, RNDKEYLAST_PTR
+	jne		1b
+	vaesenclast	(RNDKEYLAST_PTR), H_CUR_XMM, H_CUR_XMM
+
+	// Reflect the bytes of the raw hash subkey.
+	vpshufb		.Lbswap_mask(%rip), H_CUR_XMM, H_CUR_XMM
+
+	// Finish preprocessing the byte-reflected hash subkey by multiplying it
+	// by x^-1 ("standard" interpretation of polynomial coefficients) or
+	// equivalently x^1 (natural interpretation).  This gets the key into a
+	// format that avoids having to bit-reflect the data blocks later.
+	vpshufd		$0xd3, H_CUR_XMM, TMP0_XMM
+	vpsrad		$31, TMP0_XMM, TMP0_XMM
+	vpaddq		H_CUR_XMM, H_CUR_XMM, H_CUR_XMM
+	vpand		.Lgfpoly_and_internal_carrybit(%rip), TMP0_XMM, TMP0_XMM
+	vpxor		TMP0_XMM, H_CUR_XMM, H_CUR_XMM
+
+	// Load the gfpoly constant.
+	vbroadcasti128	.Lgfpoly(%rip), GFPOLY
+
+	// Square H^1 to get H^2.
+	_ghash_square	H_CUR_XMM, H_INC_XMM, GFPOLY_XMM, TMP0_XMM, TMP1_XMM
+
+	// Create H_CUR = [H^2, H^1] and H_INC = [H^2, H^2].
+	vinserti128	$1, H_CUR_XMM, H_INC, H_CUR
+	vinserti128	$1, H_INC_XMM, H_INC, H_INC
+
+	// Compute H_CUR2 = [H^4, H^3].
+	_ghash_mul	H_INC, H_CUR, H_CUR2, GFPOLY, TMP0, TMP1, TMP2
+
+	// Store [H^2, H^1] and [H^4, H^3].
+	vmovdqu		H_CUR, OFFSETOF_H_POWERS+3*32(KEY)
+	vmovdqu		H_CUR2, OFFSETOF_H_POWERS+2*32(KEY)
+
+	// For Karatsuba multiplication: compute and store the two 64-bit halves
+	// of each key power XOR'd together.  Order is 4,2,3,1.
+	vpunpcklqdq	H_CUR, H_CUR2, TMP0
+	vpunpckhqdq	H_CUR, H_CUR2, TMP1
+	vpxor		TMP1, TMP0, TMP0
+	vmovdqu		TMP0, OFFSETOF_H_POWERS_XORED+32(KEY)
+
+	// Compute and store H_CUR = [H^6, H^5] and H_CUR2 = [H^8, H^7].
+	_ghash_mul	H_INC, H_CUR2, H_CUR, GFPOLY, TMP0, TMP1, TMP2
+	_ghash_mul	H_INC, H_CUR, H_CUR2, GFPOLY, TMP0, TMP1, TMP2
+	vmovdqu		H_CUR, OFFSETOF_H_POWERS+1*32(KEY)
+	vmovdqu		H_CUR2, OFFSETOF_H_POWERS+0*32(KEY)
+
+	// Again, compute and store the two 64-bit halves of each key power
+	// XOR'd together.  Order is 8,6,7,5.
+	vpunpcklqdq	H_CUR, H_CUR2, TMP0
+	vpunpckhqdq	H_CUR, H_CUR2, TMP1
+	vpxor		TMP1, TMP0, TMP0
+	vmovdqu		TMP0, OFFSETOF_H_POWERS_XORED(KEY)
+
+	vzeroupper
+	RET
+SYM_FUNC_END(aes_gcm_precompute_vaes_avx2)
+
+// Do one step of the GHASH update of four vectors of data blocks.
+//   \i: the step to do, 0 through 9
+//   \ghashdata_ptr: pointer to the data blocks (ciphertext or AAD)
+//   KEY: pointer to struct aes_gcm_key_vaes_avx2
+//   BSWAP_MASK: mask for reflecting the bytes of blocks
+//   H_POW[2-1]_XORED: cached values from KEY->h_powers_xored
+//   TMP[0-2]: temporary registers.  TMP[1-2] must be preserved across steps.
+//   LO, MI: working state for this macro that must be preserved across steps
+//   GHASH_ACC: the GHASH accumulator (input/output)
+.macro	_ghash_step_4x	i, ghashdata_ptr
+	.set		HI, GHASH_ACC # alias
+	.set		HI_XMM, GHASH_ACC_XMM
+.if \i == 0
+	// First vector
+	vmovdqu		0*32(\ghashdata_ptr), TMP1
+	vpshufb		BSWAP_MASK, TMP1, TMP1
+	vmovdqu		OFFSETOF_H_POWERS+0*32(KEY), TMP2
+	vpxor		GHASH_ACC, TMP1, TMP1
+	vpclmulqdq	$0x00, TMP2, TMP1, LO
+	vpclmulqdq	$0x11, TMP2, TMP1, HI
+	vpunpckhqdq	TMP1, TMP1, TMP0
+	vpxor		TMP1, TMP0, TMP0
+	vpclmulqdq	$0x00, H_POW2_XORED, TMP0, MI
+.elseif \i == 1
+.elseif \i == 2
+	// Second vector
+	vmovdqu		1*32(\ghashdata_ptr), TMP1
+	vpshufb		BSWAP_MASK, TMP1, TMP1
+	vmovdqu		OFFSETOF_H_POWERS+1*32(KEY), TMP2
+	vpclmulqdq	$0x00, TMP2, TMP1, TMP0
+	vpxor		TMP0, LO, LO
+	vpclmulqdq	$0x11, TMP2, TMP1, TMP0
+	vpxor		TMP0, HI, HI
+	vpunpckhqdq	TMP1, TMP1, TMP0
+	vpxor		TMP1, TMP0, TMP0
+	vpclmulqdq	$0x10, H_POW2_XORED, TMP0, TMP0
+	vpxor		TMP0, MI, MI
+.elseif \i == 3
+	// Third vector
+	vmovdqu		2*32(\ghashdata_ptr), TMP1
+	vpshufb		BSWAP_MASK, TMP1, TMP1
+	vmovdqu		OFFSETOF_H_POWERS+2*32(KEY), TMP2
+.elseif \i == 4
+	vpclmulqdq	$0x00, TMP2, TMP1, TMP0
+	vpxor		TMP0, LO, LO
+	vpclmulqdq	$0x11, TMP2, TMP1, TMP0
+	vpxor		TMP0, HI, HI
+.elseif \i == 5
+	vpunpckhqdq	TMP1, TMP1, TMP0
+	vpxor		TMP1, TMP0, TMP0
+	vpclmulqdq	$0x00, H_POW1_XORED, TMP0, TMP0
+	vpxor		TMP0, MI, MI
+
+	// Fourth vector
+	vmovdqu		3*32(\ghashdata_ptr), TMP1
+	vpshufb		BSWAP_MASK, TMP1, TMP1
+.elseif \i == 6
+	vmovdqu		OFFSETOF_H_POWERS+3*32(KEY), TMP2
+	vpclmulqdq	$0x00, TMP2, TMP1, TMP0
+	vpxor		TMP0, LO, LO
+	vpclmulqdq	$0x11, TMP2, TMP1, TMP0
+	vpxor		TMP0, HI, HI
+	vpunpckhqdq	TMP1, TMP1, TMP0
+	vpxor		TMP1, TMP0, TMP0
+	vpclmulqdq	$0x10, H_POW1_XORED, TMP0, TMP0
+	vpxor		TMP0, MI, MI
+.elseif \i == 7
+	// Finalize 'mi' following Karatsuba multiplication.
+	vpxor		LO, MI, MI
+	vpxor		HI, MI, MI
+
+	// Fold lo into mi.
+	vbroadcasti128	.Lgfpoly(%rip), TMP2
+	vpclmulqdq	$0x01, LO, TMP2, TMP0
+	vpshufd		$0x4e, LO, LO
+	vpxor		LO, MI, MI
+	vpxor		TMP0, MI, MI
+.elseif \i == 8
+	// Fold mi into hi.
+	vpclmulqdq	$0x01, MI, TMP2, TMP0
+	vpshufd		$0x4e, MI, MI
+	vpxor		MI, HI, HI
+	vpxor		TMP0, HI, HI
+.elseif \i == 9
+	vextracti128	$1, HI, TMP0_XMM
+	vpxor		TMP0_XMM, HI_XMM, GHASH_ACC_XMM
+.endif
+.endm
+
+// Update GHASH with four vectors of data blocks.  See _ghash_step_4x for full
+// explanation.
+.macro	_ghash_4x	ghashdata_ptr
+.irp i, 0,1,2,3,4,5,6,7,8,9
+	_ghash_step_4x	\i, \ghashdata_ptr
+.endr
+.endm
+
+// Load 1 <= %ecx <= 16 bytes from the pointer \src into the xmm register \dst
+// and zeroize any remaining bytes.  Clobbers %rax, %rcx, and \tmp{64,32}.
+.macro	_load_partial_block	src, dst, tmp64, tmp32
+	sub		$8, %ecx		// LEN - 8
+	jle		.Lle8\@
+
+	// Load 9 <= LEN <= 16 bytes.
+	vmovq		(\src), \dst		// Load first 8 bytes
+	mov		(\src, %rcx), %rax	// Load last 8 bytes
+	neg		%ecx
+	shl		$3, %ecx
+	shr		%cl, %rax		// Discard overlapping bytes
+	vpinsrq		$1, %rax, \dst, \dst
+	jmp		.Ldone\@
+
+.Lle8\@:
+	add		$4, %ecx		// LEN - 4
+	jl		.Llt4\@
+
+	// Load 4 <= LEN <= 8 bytes.
+	mov		(\src), %eax		// Load first 4 bytes
+	mov		(\src, %rcx), \tmp32	// Load last 4 bytes
+	jmp		.Lcombine\@
+
+.Llt4\@:
+	// Load 1 <= LEN <= 3 bytes.
+	add		$2, %ecx		// LEN - 2
+	movzbl		(\src), %eax		// Load first byte
+	jl		.Lmovq\@
+	movzwl		(\src, %rcx), \tmp32	// Load last 2 bytes
+.Lcombine\@:
+	shl		$3, %ecx
+	shl		%cl, \tmp64
+	or		\tmp64, %rax		// Combine the two parts
+.Lmovq\@:
+	vmovq		%rax, \dst
+.Ldone\@:
+.endm
+
+// Store 1 <= %ecx <= 16 bytes from the xmm register \src to the pointer \dst.
+// Clobbers %rax, %rcx, and \tmp{64,32}.
+.macro	_store_partial_block	src, dst, tmp64, tmp32
+	sub		$8, %ecx		// LEN - 8
+	jl		.Llt8\@
+
+	// Store 8 <= LEN <= 16 bytes.
+	vpextrq		$1, \src, %rax
+	mov		%ecx, \tmp32
+	shl		$3, %ecx
+	ror		%cl, %rax
+	mov		%rax, (\dst, \tmp64)	// Store last LEN - 8 bytes
+	vmovq		\src, (\dst)		// Store first 8 bytes
+	jmp		.Ldone\@
+
+.Llt8\@:
+	add		$4, %ecx		// LEN - 4
+	jl		.Llt4\@
+
+	// Store 4 <= LEN <= 7 bytes.
+	vpextrd		$1, \src, %eax
+	mov		%ecx, \tmp32
+	shl		$3, %ecx
+	ror		%cl, %eax
+	mov		%eax, (\dst, \tmp64)	// Store last LEN - 4 bytes
+	vmovd		\src, (\dst)		// Store first 4 bytes
+	jmp		.Ldone\@
+
+.Llt4\@:
+	// Store 1 <= LEN <= 3 bytes.
+	vpextrb		$0, \src, 0(\dst)
+	cmp		$-2, %ecx		// LEN - 4 == -2, i.e. LEN == 2?
+	jl		.Ldone\@
+	vpextrb		$1, \src, 1(\dst)
+	je		.Ldone\@
+	vpextrb		$2, \src, 2(\dst)
+.Ldone\@:
+.endm
+
+// void aes_gcm_aad_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
+//				     u8 ghash_acc[16],
+//				     const u8 *aad, int aadlen);
+//
+// This function processes the AAD (Additional Authenticated Data) in GCM.
+// Using the key |key|, it updates the GHASH accumulator |ghash_acc| with the
+// data given by |aad| and |aadlen|.  On the first call, |ghash_acc| must be all
+// zeroes.  |aadlen| must be a multiple of 16, except on the last call where it
+// can be any length.  The caller must do any buffering needed to ensure this.
+//
+// This handles large amounts of AAD efficiently, while also keeping overhead
+// low for small amounts which is the common case.  TLS and IPsec use less than
+// one block of AAD, but (uncommonly) other use cases may use much more.
+SYM_FUNC_START(aes_gcm_aad_update_vaes_avx2)
+
+	// Function arguments
+	.set	KEY,		%rdi
+	.set	GHASH_ACC_PTR,	%rsi
+	.set	AAD,		%rdx
+	.set	AADLEN,		%ecx	// Must be %ecx for _load_partial_block
+	.set	AADLEN64,	%rcx	// Zero-extend AADLEN before using!
+
+	// Additional local variables.
+	// %rax and %r8 are used as temporary registers.
+	.set	TMP0,		%ymm0
+	.set	TMP0_XMM,	%xmm0
+	.set	TMP1,		%ymm1
+	.set	TMP1_XMM,	%xmm1
+	.set	TMP2,		%ymm2
+	.set	TMP2_XMM,	%xmm2
+	.set	LO,		%ymm3
+	.set	LO_XMM,		%xmm3
+	.set	MI,		%ymm4
+	.set	MI_XMM,		%xmm4
+	.set	GHASH_ACC,	%ymm5
+	.set	GHASH_ACC_XMM,	%xmm5
+	.set	BSWAP_MASK,	%ymm6
+	.set	BSWAP_MASK_XMM,	%xmm6
+	.set	GFPOLY,		%ymm7
+	.set	GFPOLY_XMM,	%xmm7
+	.set	H_POW2_XORED,	%ymm8
+	.set	H_POW1_XORED,	%ymm9
+
+	// Load the bswap_mask and gfpoly constants.  Since AADLEN is usually
+	// small, usually only 128-bit vectors will be used.  So as an
+	// optimization, don't broadcast these constants to both 128-bit lanes
+	// quite yet.
+	vmovdqu		.Lbswap_mask(%rip), BSWAP_MASK_XMM
+	vmovdqu		.Lgfpoly(%rip), GFPOLY_XMM
+
+	// Load the GHASH accumulator.
+	vmovdqu		(GHASH_ACC_PTR), GHASH_ACC_XMM
+
+	// Check for the common case of AADLEN <= 16, as well as AADLEN == 0.
+	test		AADLEN, AADLEN
+	jz		.Laad_done
+	cmp		$16, AADLEN
+	jle		.Laad_lastblock
+
+	// AADLEN > 16, so we'll operate on full vectors.  Broadcast bswap_mask
+	// and gfpoly to both 128-bit lanes.
+	vinserti128	$1, BSWAP_MASK_XMM, BSWAP_MASK, BSWAP_MASK
+	vinserti128	$1, GFPOLY_XMM, GFPOLY, GFPOLY
+
+	// If AADLEN >= 128, update GHASH with 128 bytes of AAD at a time.
+	add		$-128, AADLEN	// 128 is 4 bytes, -128 is 1 byte
+	jl		.Laad_loop_4x_done
+	vmovdqu		OFFSETOF_H_POWERS_XORED(KEY), H_POW2_XORED
+	vmovdqu		OFFSETOF_H_POWERS_XORED+32(KEY), H_POW1_XORED
+.Laad_loop_4x:
+	_ghash_4x	AAD
+	sub		$-128, AAD
+	add		$-128, AADLEN
+	jge		.Laad_loop_4x
+.Laad_loop_4x_done:
+
+	// If AADLEN >= 32, update GHASH with 32 bytes of AAD at a time.
+	add		$96, AADLEN
+	jl		.Laad_loop_1x_done
+.Laad_loop_1x:
+	vmovdqu		(AAD), TMP0
+	vpshufb		BSWAP_MASK, TMP0, TMP0
+	vpxor		TMP0, GHASH_ACC, GHASH_ACC
+	vmovdqu		OFFSETOFEND_H_POWERS-32(KEY), TMP0
+	_ghash_mul	TMP0, GHASH_ACC, GHASH_ACC, GFPOLY, TMP1, TMP2, LO
+	vextracti128	$1, GHASH_ACC, TMP0_XMM
+	vpxor		TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
+	add		$32, AAD
+	sub		$32, AADLEN
+	jge		.Laad_loop_1x
+.Laad_loop_1x_done:
+	add		$32, AADLEN
+	// Now 0 <= AADLEN < 32.
+
+	jz		.Laad_done
+	cmp		$16, AADLEN
+	jle		.Laad_lastblock
+
+	// Update GHASH with the remaining 17 <= AADLEN <= 31 bytes of AAD.
+	mov		AADLEN, AADLEN	// Zero-extend AADLEN to AADLEN64.
+	vmovdqu		(AAD), TMP0_XMM
+	vmovdqu		-16(AAD, AADLEN64), TMP1_XMM
+	vpshufb		BSWAP_MASK_XMM, TMP0_XMM, TMP0_XMM
+	vpxor		TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
+	lea		.Lrshift_and_bswap_table(%rip), %rax
+	vpshufb		-16(%rax, AADLEN64), TMP1_XMM, TMP1_XMM
+	vinserti128	$1, TMP1_XMM, GHASH_ACC, GHASH_ACC
+	vmovdqu		OFFSETOFEND_H_POWERS-32(KEY), TMP0
+	_ghash_mul	TMP0, GHASH_ACC, GHASH_ACC, GFPOLY, TMP1, TMP2, LO
+	vextracti128	$1, GHASH_ACC, TMP0_XMM
+	vpxor		TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
+	jmp		.Laad_done
+
+.Laad_lastblock:
+	// Update GHASH with the remaining 1 <= AADLEN <= 16 bytes of AAD.
+	_load_partial_block	AAD, TMP0_XMM, %r8, %r8d
+	vpshufb		BSWAP_MASK_XMM, TMP0_XMM, TMP0_XMM
+	vpxor		TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
+	vmovdqu		OFFSETOFEND_H_POWERS-16(KEY), TMP0_XMM
+	_ghash_mul	TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM, GFPOLY_XMM, \
+			TMP1_XMM, TMP2_XMM, LO_XMM
+
+.Laad_done:
+	// Store the updated GHASH accumulator back to memory.
+	vmovdqu		GHASH_ACC_XMM, (GHASH_ACC_PTR)
+
+	vzeroupper
+	RET
+SYM_FUNC_END(aes_gcm_aad_update_vaes_avx2)
+
+// Do one non-last round of AES encryption on the blocks in the given AESDATA
+// vectors using the round key that has been broadcast to all 128-bit lanes of
+// \round_key.
+.macro	_vaesenc	round_key, vecs:vararg
+.irp i, \vecs
+	vaesenc		\round_key, AESDATA\i, AESDATA\i
+.endr
+.endm
+
+// Generate counter blocks in the given AESDATA vectors, then do the zero-th AES
+// round on them.  Clobbers TMP0.
+.macro	_ctr_begin	vecs:vararg
+	vbroadcasti128	.Linc_2blocks(%rip), TMP0
+.irp i, \vecs
+	vpshufb		BSWAP_MASK, LE_CTR, AESDATA\i
+	vpaddd		TMP0, LE_CTR, LE_CTR
+.endr
+.irp i, \vecs
+	vpxor		RNDKEY0, AESDATA\i, AESDATA\i
+.endr
+.endm
+
+// Generate and encrypt counter blocks in the given AESDATA vectors, excluding
+// the last AES round.  Clobbers %rax and TMP0.
+.macro	_aesenc_loop	vecs:vararg
+	_ctr_begin	\vecs
+	lea		16(KEY), %rax
+.Laesenc_loop\@:
+	vbroadcasti128	(%rax), TMP0
+	_vaesenc	TMP0, \vecs
+	add		$16, %rax
+	cmp		%rax, RNDKEYLAST_PTR
+	jne		.Laesenc_loop\@
+.endm
+
+// Finalize the keystream blocks in the given AESDATA vectors by doing the last
+// AES round, then XOR those keystream blocks with the corresponding data.
+// Reduce latency by doing the XOR before the vaesenclast, utilizing the
+// property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a).  Clobbers TMP0.
+.macro	_aesenclast_and_xor	vecs:vararg
+.irp i, \vecs
+	vpxor		\i*32(SRC), RNDKEYLAST, TMP0
+	vaesenclast	TMP0, AESDATA\i, AESDATA\i
+.endr
+.irp i, \vecs
+	vmovdqu		AESDATA\i, \i*32(DST)
+.endr
+.endm
+
+// void aes_gcm_{enc,dec}_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
+//					   const u32 le_ctr[4], u8 ghash_acc[16],
+//					   const u8 *src, u8 *dst, int datalen);
+//
+// This macro generates a GCM encryption or decryption update function with the
+// above prototype (with \enc selecting which one).  The function computes the
+// next portion of the CTR keystream, XOR's it with |datalen| bytes from |src|,
+// and writes the resulting encrypted or decrypted data to |dst|.  It also
+// updates the GHASH accumulator |ghash_acc| using the next |datalen| ciphertext
+// bytes.
+//
+// |datalen| must be a multiple of 16, except on the last call where it can be
+// any length.  The caller must do any buffering needed to ensure this.  Both
+// in-place and out-of-place en/decryption are supported.
+//
+// |le_ctr| must give the current counter in little-endian format.  This
+// function loads the counter from |le_ctr| and increments the loaded counter as
+// needed, but it does *not* store the updated counter back to |le_ctr|.  The
+// caller must update |le_ctr| if any more data segments follow.  Internally,
+// only the low 32-bit word of the counter is incremented, following the GCM
+// standard.
+.macro	_aes_gcm_update	enc
+
+	// Function arguments
+	.set	KEY,		%rdi
+	.set	LE_CTR_PTR,	%rsi
+	.set	LE_CTR_PTR32,	%esi
+	.set	GHASH_ACC_PTR,	%rdx
+	.set	SRC,		%rcx	// Assumed to be %rcx.
+					// See .Ltail_xor_and_ghash_1to16bytes
+	.set	DST,		%r8
+	.set	DATALEN,	%r9d
+	.set	DATALEN64,	%r9	// Zero-extend DATALEN before using!
+
+	// Additional local variables
+
+	// %rax is used as a temporary register.  LE_CTR_PTR is also available
+	// as a temporary register after the counter is loaded.
+
+	// AES key length in bytes
+	.set	AESKEYLEN,	%r10d
+	.set	AESKEYLEN64,	%r10
+
+	// Pointer to the last AES round key for the chosen AES variant
+	.set	RNDKEYLAST_PTR,	%r11
+
+	// BSWAP_MASK is the shuffle mask for byte-reflecting 128-bit values
+	// using vpshufb, copied to all 128-bit lanes.
+	.set	BSWAP_MASK,	%ymm0
+	.set	BSWAP_MASK_XMM,	%xmm0
+
+	// GHASH_ACC is the accumulator variable for GHASH.  When fully reduced,
+	// only the lowest 128-bit lane can be nonzero.  When not fully reduced,
+	// more than one lane may be used, and they need to be XOR'd together.
+	.set	GHASH_ACC,	%ymm1
+	.set	GHASH_ACC_XMM,	%xmm1
+
+	// TMP[0-2] are temporary registers.
+	.set	TMP0,		%ymm2
+	.set	TMP0_XMM,	%xmm2
+	.set	TMP1,		%ymm3
+	.set	TMP1_XMM,	%xmm3
+	.set	TMP2,		%ymm4
+	.set	TMP2_XMM,	%xmm4
+
+	// LO and MI are used to accumulate unreduced GHASH products.
+	.set	LO,		%ymm5
+	.set	LO_XMM,		%xmm5
+	.set	MI,		%ymm6
+	.set	MI_XMM,		%xmm6
+
+	// H_POW[2-1]_XORED contain cached values from KEY->h_powers_xored.  The
+	// descending numbering reflects the order of the key powers.
+	.set	H_POW2_XORED,	%ymm7
+	.set	H_POW2_XORED_XMM, %xmm7
+	.set	H_POW1_XORED,	%ymm8
+
+	// RNDKEY0 caches the zero-th round key, and RNDKEYLAST the last one.
+	.set	RNDKEY0,	%ymm9
+	.set	RNDKEYLAST,	%ymm10
+
+	// LE_CTR contains the next set of little-endian counter blocks.
+	.set	LE_CTR,		%ymm11
+
+	// AESDATA[0-3] hold the counter blocks that are being encrypted by AES.
+	.set	AESDATA0,	%ymm12
+	.set	AESDATA0_XMM,	%xmm12
+	.set	AESDATA1,	%ymm13
+	.set	AESDATA1_XMM,	%xmm13
+	.set	AESDATA2,	%ymm14
+	.set	AESDATA3,	%ymm15
+
+.if \enc
+	.set	GHASHDATA_PTR,	DST
+.else
+	.set	GHASHDATA_PTR,	SRC
+.endif
+
+	vbroadcasti128	.Lbswap_mask(%rip), BSWAP_MASK
+
+	// Load the GHASH accumulator and the starting counter.
+	vmovdqu		(GHASH_ACC_PTR), GHASH_ACC_XMM
+	vbroadcasti128	(LE_CTR_PTR), LE_CTR
+
+	// Load the AES key length in bytes.
+	movl		OFFSETOF_AESKEYLEN(KEY), AESKEYLEN
+
+	// Make RNDKEYLAST_PTR point to the last AES round key.  This is the
+	// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256
+	// respectively.  Then load the zero-th and last round keys.
+	lea		6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR
+	vbroadcasti128	(KEY), RNDKEY0
+	vbroadcasti128	(RNDKEYLAST_PTR), RNDKEYLAST
+
+	// Finish initializing LE_CTR by adding 1 to the second block.
+	vpaddd		.Lctr_pattern(%rip), LE_CTR, LE_CTR
+
+	// If there are at least 128 bytes of data, then continue into the loop
+	// that processes 128 bytes of data at a time.  Otherwise skip it.
+	add		$-128, DATALEN	// 128 is 4 bytes, -128 is 1 byte
+	jl		.Lcrypt_loop_4x_done\@
+
+	vmovdqu		OFFSETOF_H_POWERS_XORED(KEY), H_POW2_XORED
+	vmovdqu		OFFSETOF_H_POWERS_XORED+32(KEY), H_POW1_XORED
+
+	// Main loop: en/decrypt and hash 4 vectors (128 bytes) at a time.
+
+.if \enc
+	// Encrypt the first 4 vectors of plaintext blocks.
+	_aesenc_loop	0,1,2,3
+	_aesenclast_and_xor	0,1,2,3
+	sub		$-128, SRC	// 128 is 4 bytes, -128 is 1 byte
+	add		$-128, DATALEN
+	jl		.Lghash_last_ciphertext_4x\@
+.endif
+
+.align 16
+.Lcrypt_loop_4x\@:
+
+	// Start the AES encryption of the counter blocks.
+	_ctr_begin	0,1,2,3
+	cmp		$24, AESKEYLEN
+	jl		128f	// AES-128?
+	je		192f	// AES-192?
+	// AES-256
+	vbroadcasti128	-13*16(RNDKEYLAST_PTR), TMP0
+	_vaesenc	TMP0, 0,1,2,3
+	vbroadcasti128	-12*16(RNDKEYLAST_PTR), TMP0
+	_vaesenc	TMP0, 0,1,2,3
+192:
+	vbroadcasti128	-11*16(RNDKEYLAST_PTR), TMP0
+	_vaesenc	TMP0, 0,1,2,3
+	vbroadcasti128	-10*16(RNDKEYLAST_PTR), TMP0
+	_vaesenc	TMP0, 0,1,2,3
+128:
+
+	// Finish the AES encryption of the counter blocks in AESDATA[0-3],
+	// interleaved with the GHASH update of the ciphertext blocks.
+.irp i, 9,8,7,6,5,4,3,2,1
+	_ghash_step_4x  (9 - \i), GHASHDATA_PTR
+	vbroadcasti128	-\i*16(RNDKEYLAST_PTR), TMP0
+	_vaesenc	TMP0, 0,1,2,3
+.endr
+	_ghash_step_4x	9, GHASHDATA_PTR
+.if \enc
+	sub		$-128, DST	// 128 is 4 bytes, -128 is 1 byte
+.endif
+	_aesenclast_and_xor	0,1,2,3
+	sub		$-128, SRC
+.if !\enc
+	sub		$-128, DST
+.endif
+	add		$-128, DATALEN
+	jge		.Lcrypt_loop_4x\@
+
+.if \enc
+.Lghash_last_ciphertext_4x\@:
+	// Update GHASH with the last set of ciphertext blocks.
+	_ghash_4x	DST
+	sub		$-128, DST
+.endif
+
+.Lcrypt_loop_4x_done\@:
+
+	// Undo the extra subtraction by 128 and check whether data remains.
+	sub		$-128, DATALEN	// 128 is 4 bytes, -128 is 1 byte
+	jz		.Ldone\@
+
+	// The data length isn't a multiple of 128 bytes.  Process the remaining
+	// data of length 1 <= DATALEN < 128.
+	//
+	// Since there are enough key powers available for all remaining data,
+	// there is no need to do a GHASH reduction after each iteration.
+	// Instead, multiply each remaining block by its own key power, and only
+	// do a GHASH reduction at the very end.
+
+	// Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N
+	// is the number of blocks that remain.
+	.set		POWERS_PTR, LE_CTR_PTR	// LE_CTR_PTR is free to be reused.
+	.set		POWERS_PTR32, LE_CTR_PTR32
+	mov		DATALEN, %eax
+	neg		%rax
+	and		$~15, %rax  // -round_up(DATALEN, 16)
+	lea		OFFSETOFEND_H_POWERS(KEY,%rax), POWERS_PTR
+
+	// Start collecting the unreduced GHASH intermediate value LO, MI, HI.
+	.set		HI, H_POW2_XORED	// H_POW2_XORED is free to be reused.
+	.set		HI_XMM, H_POW2_XORED_XMM
+	vpxor		LO_XMM, LO_XMM, LO_XMM
+	vpxor		MI_XMM, MI_XMM, MI_XMM
+	vpxor		HI_XMM, HI_XMM, HI_XMM
+
+	// 1 <= DATALEN < 128.  Generate 2 or 4 more vectors of keystream blocks
+	// excluding the last AES round, depending on the remaining DATALEN.
+	cmp		$64, DATALEN
+	jg		.Ltail_gen_4_keystream_vecs\@
+	_aesenc_loop	0,1
+	cmp		$32, DATALEN
+	jge		.Ltail_xor_and_ghash_full_vec_loop\@
+	jmp		.Ltail_xor_and_ghash_partial_vec\@
+.Ltail_gen_4_keystream_vecs\@:
+	_aesenc_loop	0,1,2,3
+
+	// XOR the remaining data and accumulate the unreduced GHASH products
+	// for DATALEN >= 32, starting with one full 32-byte vector at a time.
+.Ltail_xor_and_ghash_full_vec_loop\@:
+.if \enc
+	_aesenclast_and_xor	0
+	vpshufb		BSWAP_MASK, AESDATA0, AESDATA0
+.else
+	vmovdqu		(SRC), TMP1
+	vpxor		TMP1, RNDKEYLAST, TMP0
+	vaesenclast	TMP0, AESDATA0, AESDATA0
+	vmovdqu		AESDATA0, (DST)
+	vpshufb		BSWAP_MASK, TMP1, AESDATA0
+.endif
+	// The ciphertext blocks (i.e. GHASH input data) are now in AESDATA0.
+	vpxor		GHASH_ACC, AESDATA0, AESDATA0
+	vmovdqu		(POWERS_PTR), TMP2
+	_ghash_mul_noreduce	TMP2, AESDATA0, LO, MI, HI, TMP0
+	vmovdqa		AESDATA1, AESDATA0
+	vmovdqa		AESDATA2, AESDATA1
+	vmovdqa		AESDATA3, AESDATA2
+	vpxor		GHASH_ACC_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
+	add		$32, SRC
+	add		$32, DST
+	add		$32, POWERS_PTR
+	sub		$32, DATALEN
+	cmp		$32, DATALEN
+	jge		.Ltail_xor_and_ghash_full_vec_loop\@
+	test		DATALEN, DATALEN
+	jz		.Ltail_ghash_reduce\@
+
+.Ltail_xor_and_ghash_partial_vec\@:
+	// XOR the remaining data and accumulate the unreduced GHASH products,
+	// for 1 <= DATALEN < 32.
+	vaesenclast	RNDKEYLAST, AESDATA0, AESDATA0
+	cmp		$16, DATALEN
+	jle		.Ltail_xor_and_ghash_1to16bytes\@
+
+	// Handle 17 <= DATALEN < 32.
+
+	// Load a vpshufb mask that will right-shift by '32 - DATALEN' bytes
+	// (shifting in zeroes), then reflect all 16 bytes.
+	lea		.Lrshift_and_bswap_table(%rip), %rax
+	vmovdqu		-16(%rax, DATALEN64), TMP2_XMM
+
+	// Move the second keystream block to its own register and left-align it
+	vextracti128	$1, AESDATA0, AESDATA1_XMM
+	vpxor		.Lfifteens(%rip), TMP2_XMM, TMP0_XMM
+	vpshufb		TMP0_XMM, AESDATA1_XMM, AESDATA1_XMM
+
+	// Using overlapping loads and stores, XOR the source data with the
+	// keystream and write the destination data.  Then prepare the GHASH
+	// input data: the full ciphertext block and the zero-padded partial
+	// ciphertext block, both byte-reflected, in AESDATA0.
+.if \enc
+	vpxor		-16(SRC, DATALEN64), AESDATA1_XMM, AESDATA1_XMM
+	vpxor		(SRC), AESDATA0_XMM, AESDATA0_XMM
+	vmovdqu		AESDATA1_XMM, -16(DST, DATALEN64)
+	vmovdqu		AESDATA0_XMM, (DST)
+	vpshufb		TMP2_XMM, AESDATA1_XMM, AESDATA1_XMM
+	vpshufb		BSWAP_MASK_XMM, AESDATA0_XMM, AESDATA0_XMM
+.else
+	vmovdqu		-16(SRC, DATALEN64), TMP1_XMM
+	vmovdqu		(SRC), TMP0_XMM
+	vpxor		TMP1_XMM, AESDATA1_XMM, AESDATA1_XMM
+	vpxor		TMP0_XMM, AESDATA0_XMM, AESDATA0_XMM
+	vmovdqu		AESDATA1_XMM, -16(DST, DATALEN64)
+	vmovdqu		AESDATA0_XMM, (DST)
+	vpshufb		TMP2_XMM, TMP1_XMM, AESDATA1_XMM
+	vpshufb		BSWAP_MASK_XMM, TMP0_XMM, AESDATA0_XMM
+.endif
+	vpxor		GHASH_ACC_XMM, AESDATA0_XMM, AESDATA0_XMM
+	vinserti128	$1, AESDATA1_XMM, AESDATA0, AESDATA0
+	vmovdqu		(POWERS_PTR), TMP2
+	jmp		.Ltail_ghash_last_vec\@
+
+.Ltail_xor_and_ghash_1to16bytes\@:
+	// Handle 1 <= DATALEN <= 16.  Carefully load and store the
+	// possibly-partial block, which we mustn't access out of bounds.
+	vmovdqu		(POWERS_PTR), TMP2_XMM
+	mov		SRC, KEY	// Free up %rcx, assuming SRC == %rcx
+	mov		DATALEN, %ecx
+	_load_partial_block	KEY, TMP0_XMM, POWERS_PTR, POWERS_PTR32
+	vpxor		TMP0_XMM, AESDATA0_XMM, AESDATA0_XMM
+	mov		DATALEN, %ecx
+	_store_partial_block	AESDATA0_XMM, DST, POWERS_PTR, POWERS_PTR32
+.if \enc
+	lea		.Lselect_high_bytes_table(%rip), %rax
+	vpshufb		BSWAP_MASK_XMM, AESDATA0_XMM, AESDATA0_XMM
+	vpand		(%rax, DATALEN64), AESDATA0_XMM, AESDATA0_XMM
+.else
+	vpshufb		BSWAP_MASK_XMM, TMP0_XMM, AESDATA0_XMM
+.endif
+	vpxor		GHASH_ACC_XMM, AESDATA0_XMM, AESDATA0_XMM
+
+.Ltail_ghash_last_vec\@:
+	// Accumulate the unreduced GHASH products for the last 1-2 blocks.  The
+	// GHASH input data is in AESDATA0.  If only one block remains, then the
+	// second block in AESDATA0 is zero and does not affect the result.
+	_ghash_mul_noreduce	TMP2, AESDATA0, LO, MI, HI, TMP0
+
+.Ltail_ghash_reduce\@:
+	// Finally, do the GHASH reduction.
+	vbroadcasti128	.Lgfpoly(%rip), TMP0
+	_ghash_reduce	LO, MI, HI, TMP0, TMP1
+	vextracti128	$1, HI, GHASH_ACC_XMM
+	vpxor		HI_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
+
+.Ldone\@:
+	// Store the updated GHASH accumulator back to memory.
+	vmovdqu		GHASH_ACC_XMM, (GHASH_ACC_PTR)
+
+	vzeroupper
+	RET
+.endm
+
+// void aes_gcm_enc_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
+//				    const u32 le_ctr[4], u8 ghash_acc[16],
+//				    u64 total_aadlen, u64 total_datalen);
+// bool aes_gcm_dec_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
+//				    const u32 le_ctr[4], const u8 ghash_acc[16],
+//				    u64 total_aadlen, u64 total_datalen,
+//				    const u8 tag[16], int taglen);
+//
+// This macro generates one of the above two functions (with \enc selecting
+// which one).  Both functions finish computing the GCM authentication tag by
+// updating GHASH with the lengths block and encrypting the GHASH accumulator.
+// |total_aadlen| and |total_datalen| must be the total length of the additional
+// authenticated data and the en/decrypted data in bytes, respectively.
+//
+// The encryption function then stores the full-length (16-byte) computed
+// authentication tag to |ghash_acc|.  The decryption function instead loads the
+// expected authentication tag (the one that was transmitted) from the 16-byte
+// buffer |tag|, compares the first 4 <= |taglen| <= 16 bytes of it to the
+// computed tag in constant time, and returns true if and only if they match.
+.macro	_aes_gcm_final	enc
+
+	// Function arguments
+	.set	KEY,		%rdi
+	.set	LE_CTR_PTR,	%rsi
+	.set	GHASH_ACC_PTR,	%rdx
+	.set	TOTAL_AADLEN,	%rcx
+	.set	TOTAL_DATALEN,	%r8
+	.set	TAG,		%r9
+	.set	TAGLEN,		%r10d	// Originally at 8(%rsp)
+	.set	TAGLEN64,	%r10
+
+	// Additional local variables.
+	// %rax and %xmm0-%xmm3 are used as temporary registers.
+	.set	AESKEYLEN,	%r11d
+	.set	AESKEYLEN64,	%r11
+	.set	GFPOLY,		%xmm4
+	.set	BSWAP_MASK,	%xmm5
+	.set	LE_CTR,		%xmm6
+	.set	GHASH_ACC,	%xmm7
+	.set	H_POW1,		%xmm8
+
+	// Load some constants.
+	vmovdqa		.Lgfpoly(%rip), GFPOLY
+	vmovdqa		.Lbswap_mask(%rip), BSWAP_MASK
+
+	// Load the AES key length in bytes.
+	movl		OFFSETOF_AESKEYLEN(KEY), AESKEYLEN
+
+	// Set up a counter block with 1 in the low 32-bit word.  This is the
+	// counter that produces the ciphertext needed to encrypt the auth tag.
+	// GFPOLY has 1 in the low word, so grab the 1 from there using a blend.
+	vpblendd	$0xe, (LE_CTR_PTR), GFPOLY, LE_CTR
+
+	// Build the lengths block and XOR it with the GHASH accumulator.
+	// Although the lengths block is defined as the AAD length followed by
+	// the en/decrypted data length, both in big-endian byte order, a byte
+	// reflection of the full block is needed because of the way we compute
+	// GHASH (see _ghash_mul_step).  By using little-endian values in the
+	// opposite order, we avoid having to reflect any bytes here.
+	vmovq		TOTAL_DATALEN, %xmm0
+	vpinsrq		$1, TOTAL_AADLEN, %xmm0, %xmm0
+	vpsllq		$3, %xmm0, %xmm0	// Bytes to bits
+	vpxor		(GHASH_ACC_PTR), %xmm0, GHASH_ACC
+
+	// Load the first hash key power (H^1), which is stored last.
+	vmovdqu		OFFSETOFEND_H_POWERS-16(KEY), H_POW1
+
+	// Load TAGLEN if decrypting.
+.if !\enc
+	movl		8(%rsp), TAGLEN
+.endif
+
+	// Make %rax point to the last AES round key for the chosen AES variant.
+	lea		6*16(KEY,AESKEYLEN64,4), %rax
+
+	// Start the AES encryption of the counter block by swapping the counter
+	// block to big-endian and XOR-ing it with the zero-th AES round key.
+	vpshufb		BSWAP_MASK, LE_CTR, %xmm0
+	vpxor		(KEY), %xmm0, %xmm0
+
+	// Complete the AES encryption and multiply GHASH_ACC by H^1.
+	// Interleave the AES and GHASH instructions to improve performance.
+	cmp		$24, AESKEYLEN
+	jl		128f	// AES-128?
+	je		192f	// AES-192?
+	// AES-256
+	vaesenc		-13*16(%rax), %xmm0, %xmm0
+	vaesenc		-12*16(%rax), %xmm0, %xmm0
+192:
+	vaesenc		-11*16(%rax), %xmm0, %xmm0
+	vaesenc		-10*16(%rax), %xmm0, %xmm0
+128:
+.irp i, 0,1,2,3,4,5,6,7,8
+	_ghash_mul_step	\i, H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \
+			%xmm1, %xmm2, %xmm3
+	vaesenc		(\i-9)*16(%rax), %xmm0, %xmm0
+.endr
+	_ghash_mul_step	9, H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \
+			%xmm1, %xmm2, %xmm3
+
+	// Undo the byte reflection of the GHASH accumulator.
+	vpshufb		BSWAP_MASK, GHASH_ACC, GHASH_ACC
+
+	// Do the last AES round and XOR the resulting keystream block with the
+	// GHASH accumulator to produce the full computed authentication tag.
+	//
+	// Reduce latency by taking advantage of the property vaesenclast(key,
+	// a) ^ b == vaesenclast(key ^ b, a).  I.e., XOR GHASH_ACC into the last
+	// round key, instead of XOR'ing the final AES output with GHASH_ACC.
+	//
+	// enc_final then returns the computed auth tag, while dec_final
+	// compares it with the transmitted one and returns a bool.  To compare
+	// the tags, dec_final XORs them together and uses vptest to check
+	// whether the result is all-zeroes.  This should be constant-time.
+	// dec_final applies the vaesenclast optimization to this additional
+	// value XOR'd too.
+.if \enc
+	vpxor		(%rax), GHASH_ACC, %xmm1
+	vaesenclast	%xmm1, %xmm0, GHASH_ACC
+	vmovdqu		GHASH_ACC, (GHASH_ACC_PTR)
+.else
+	vpxor		(TAG), GHASH_ACC, GHASH_ACC
+	vpxor		(%rax), GHASH_ACC, GHASH_ACC
+	vaesenclast	GHASH_ACC, %xmm0, %xmm0
+	lea		.Lselect_high_bytes_table(%rip), %rax
+	vmovdqu		(%rax, TAGLEN64), %xmm1
+	vpshufb		BSWAP_MASK, %xmm1, %xmm1 // select low bytes, not high
+	xor		%eax, %eax
+	vptest		%xmm1, %xmm0
+	sete		%al
+.endif
+	// No need for vzeroupper here, since only used xmm registers were used.
+	RET
+.endm
+
+SYM_FUNC_START(aes_gcm_enc_update_vaes_avx2)
+	_aes_gcm_update	1
+SYM_FUNC_END(aes_gcm_enc_update_vaes_avx2)
+SYM_FUNC_START(aes_gcm_dec_update_vaes_avx2)
+	_aes_gcm_update	0
+SYM_FUNC_END(aes_gcm_dec_update_vaes_avx2)
+
+SYM_FUNC_START(aes_gcm_enc_final_vaes_avx2)
+	_aes_gcm_final	1
+SYM_FUNC_END(aes_gcm_enc_final_vaes_avx2)
+SYM_FUNC_START(aes_gcm_dec_final_vaes_avx2)
+	_aes_gcm_final	0
+SYM_FUNC_END(aes_gcm_dec_final_vaes_avx2)
diff --git a/arch/x86/crypto/aes-gcm-avx10-x86_64.S b/arch/x86/crypto/aes-gcm-vaes-avx512.S
index 97e0ee515fc5..06b71314d65c 100644
--- a/arch/x86/crypto/aes-gcm-avx10-x86_64.S
+++ b/arch/x86/crypto/aes-gcm-vaes-avx512.S
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
 //
-// VAES and VPCLMULQDQ optimized AES-GCM for x86_64
+// AES-GCM implementation for x86_64 CPUs that support the following CPU
+// features: VAES && VPCLMULQDQ && AVX512BW && AVX512VL && BMI2
 //
 // Copyright 2024 Google LLC
 //
@@ -45,41 +46,6 @@
 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 // POSSIBILITY OF SUCH DAMAGE.
-//
-//------------------------------------------------------------------------------
-//
-// This file implements AES-GCM (Galois/Counter Mode) for x86_64 CPUs that
-// support VAES (vector AES), VPCLMULQDQ (vector carryless multiplication), and
-// either AVX512 or AVX10.  Some of the functions, notably the encryption and
-// decryption update functions which are the most performance-critical, are
-// provided in two variants generated from a macro: one using 256-bit vectors
-// (suffix: vaes_avx10_256) and one using 512-bit vectors (vaes_avx10_512).  The
-// other, "shared" functions (vaes_avx10) use at most 256-bit vectors.
-//
-// The functions that use 512-bit vectors are intended for CPUs that support
-// 512-bit vectors *and* where using them doesn't cause significant
-// downclocking.  They require the following CPU features:
-//
-//	VAES && VPCLMULQDQ && BMI2 && ((AVX512BW && AVX512VL) || AVX10/512)
-//
-// The other functions require the following CPU features:
-//
-//	VAES && VPCLMULQDQ && BMI2 && ((AVX512BW && AVX512VL) || AVX10/256)
-//
-// All functions use the "System V" ABI.  The Windows ABI is not supported.
-//
-// Note that we use "avx10" in the names of the functions as a shorthand to
-// really mean "AVX10 or a certain set of AVX512 features".  Due to Intel's
-// introduction of AVX512 and then its replacement by AVX10, there doesn't seem
-// to be a simple way to name things that makes sense on all CPUs.
-//
-// Note that the macros that support both 256-bit and 512-bit vectors could
-// fairly easily be changed to support 128-bit too.  However, this would *not*
-// be sufficient to allow the code to run on CPUs without AVX512 or AVX10,
-// because the code heavily uses several features of these extensions other than
-// the vector length: the increase in the number of SIMD registers from 16 to
-// 32, masking support, and new instructions such as vpternlogd (which can do a
-// three-argument XOR).  These features are very useful for AES-GCM.
 
 #include <linux/linkage.h>
 
@@ -88,7 +54,7 @@
 
 	// A shuffle mask that reflects the bytes of 16-byte blocks
 .Lbswap_mask:
-	.octa   0x000102030405060708090a0b0c0d0e0f
+	.octa	0x000102030405060708090a0b0c0d0e0f
 
 	// This is the GHASH reducing polynomial without its constant term, i.e.
 	// x^128 + x^7 + x^2 + x, represented using the backwards mapping
@@ -104,16 +70,14 @@
 .Lgfpoly_and_internal_carrybit:
 	.octa	0xc2000000000000010000000000000001
 
-	// The below constants are used for incrementing the counter blocks.
-	// ctr_pattern points to the four 128-bit values [0, 1, 2, 3].
-	// inc_2blocks and inc_4blocks point to the single 128-bit values 2 and
-	// 4.  Note that the same '2' is reused in ctr_pattern and inc_2blocks.
+	// Values needed to prepare the initial vector of counter blocks.
 .Lctr_pattern:
 	.octa	0
 	.octa	1
-.Linc_2blocks:
 	.octa	2
 	.octa	3
+
+	// The number of AES blocks per vector, as a 128-bit value.
 .Linc_4blocks:
 	.octa	4
 
@@ -130,29 +94,13 @@
 // Offset to end of hash key powers array in the key struct.
 //
 // This is immediately followed by three zeroized padding blocks, which are
-// included so that partial vectors can be handled more easily.  E.g. if VL=64
-// and two blocks remain, we load the 4 values [H^2, H^1, 0, 0].  The most
-// padding blocks needed is 3, which occurs if [H^1, 0, 0, 0] is loaded.
+// included so that partial vectors can be handled more easily.  E.g. if two
+// blocks remain, we load the 4 values [H^2, H^1, 0, 0].  The most padding
+// blocks needed is 3, which occurs if [H^1, 0, 0, 0] is loaded.
 #define OFFSETOFEND_H_POWERS	(OFFSETOF_H_POWERS + (NUM_H_POWERS * 16))
 
 .text
 
-// Set the vector length in bytes.  This sets the VL variable and defines
-// register aliases V0-V31 that map to the ymm or zmm registers.
-.macro	_set_veclen	vl
-	.set	VL,	\vl
-.irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \
-	16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
-.if VL == 32
-	.set	V\i,	%ymm\i
-.elseif VL == 64
-	.set	V\i,	%zmm\i
-.else
-	.error "Unsupported vector length"
-.endif
-.endr
-.endm
-
 // The _ghash_mul_step macro does one step of GHASH multiplication of the
 // 128-bit lanes of \a by the corresponding 128-bit lanes of \b and storing the
 // reduced products in \dst.  \t0, \t1, and \t2 are temporary registers of the
@@ -312,39 +260,44 @@
 	vpternlogd	$0x96, \t0, \mi, \hi
 .endm
 
-// void aes_gcm_precompute_##suffix(struct aes_gcm_key_avx10 *key);
-//
-// Given the expanded AES key |key->aes_key|, this function derives the GHASH
-// subkey and initializes |key->ghash_key_powers| with powers of it.
-//
-// The number of key powers initialized is NUM_H_POWERS, and they are stored in
-// the order H^NUM_H_POWERS to H^1.  The zeroized padding blocks after the key
-// powers themselves are also initialized.
+// This is a specialized version of _ghash_mul that computes \a * \a, i.e. it
+// squares \a.  It skips computing MI = (a_L * a_H) + (a_H * a_L) = 0.
+.macro	_ghash_square	a, dst, gfpoly, t0, t1
+	vpclmulqdq	$0x00, \a, \a, \t0	  // LO = a_L * a_L
+	vpclmulqdq	$0x11, \a, \a, \dst	  // HI = a_H * a_H
+	vpclmulqdq	$0x01, \t0, \gfpoly, \t1  // LO_L*(x^63 + x^62 + x^57)
+	vpshufd		$0x4e, \t0, \t0		  // Swap halves of LO
+	vpxord		\t0, \t1, \t1		  // Fold LO into MI
+	vpclmulqdq	$0x01, \t1, \gfpoly, \t0  // MI_L*(x^63 + x^62 + x^57)
+	vpshufd		$0x4e, \t1, \t1		  // Swap halves of MI
+	vpternlogd	$0x96, \t0, \t1, \dst	  // Fold MI into HI
+.endm
+
+// void aes_gcm_precompute_vaes_avx512(struct aes_gcm_key_vaes_avx512 *key);
 //
-// This macro supports both VL=32 and VL=64.  _set_veclen must have been invoked
-// with the desired length.  In the VL=32 case, the function computes twice as
-// many key powers than are actually used by the VL=32 GCM update functions.
-// This is done to keep the key format the same regardless of vector length.
-.macro	_aes_gcm_precompute
+// Given the expanded AES key |key->base.aes_key|, derive the GHASH subkey and
+// initialize |key->h_powers| and |key->padding|.
+SYM_FUNC_START(aes_gcm_precompute_vaes_avx512)
 
 	// Function arguments
 	.set	KEY,		%rdi
 
-	// Additional local variables.  V0-V2 and %rax are used as temporaries.
+	// Additional local variables.
+	// %zmm[0-2] and %rax are used as temporaries.
 	.set	POWERS_PTR,	%rsi
 	.set	RNDKEYLAST_PTR,	%rdx
-	.set	H_CUR,		V3
+	.set	H_CUR,		%zmm3
 	.set	H_CUR_YMM,	%ymm3
 	.set	H_CUR_XMM,	%xmm3
-	.set	H_INC,		V4
+	.set	H_INC,		%zmm4
 	.set	H_INC_YMM,	%ymm4
 	.set	H_INC_XMM,	%xmm4
-	.set	GFPOLY,		V5
+	.set	GFPOLY,		%zmm5
 	.set	GFPOLY_YMM,	%ymm5
 	.set	GFPOLY_XMM,	%xmm5
 
 	// Get pointer to lowest set of key powers (located at end of array).
-	lea		OFFSETOFEND_H_POWERS-VL(KEY), POWERS_PTR
+	lea		OFFSETOFEND_H_POWERS-64(KEY), POWERS_PTR
 
 	// Encrypt an all-zeroes block to get the raw hash subkey.
 	movl		OFFSETOF_AESKEYLEN(KEY), %eax
@@ -363,8 +316,8 @@
 
 	// Zeroize the padding blocks.
 	vpxor		%xmm0, %xmm0, %xmm0
-	vmovdqu		%ymm0, VL(POWERS_PTR)
-	vmovdqu		%xmm0, VL+2*16(POWERS_PTR)
+	vmovdqu		%ymm0, 64(POWERS_PTR)
+	vmovdqu		%xmm0, 64+2*16(POWERS_PTR)
 
 	// Finish preprocessing the first key power, H^1.  Since this GHASH
 	// implementation operates directly on values with the backwards bit
@@ -384,8 +337,8 @@
 	vpshufd		$0xd3, H_CUR_XMM, %xmm0
 	vpsrad		$31, %xmm0, %xmm0
 	vpaddq		H_CUR_XMM, H_CUR_XMM, H_CUR_XMM
-	vpand		.Lgfpoly_and_internal_carrybit(%rip), %xmm0, %xmm0
-	vpxor		%xmm0, H_CUR_XMM, H_CUR_XMM
+	// H_CUR_XMM ^= xmm0 & gfpoly_and_internal_carrybit
+	vpternlogd	$0x78, .Lgfpoly_and_internal_carrybit(%rip), %xmm0, H_CUR_XMM
 
 	// Load the gfpoly constant.
 	vbroadcasti32x4	.Lgfpoly(%rip), GFPOLY
@@ -397,54 +350,44 @@
 	// special needs to be done to make this happen, though: H^1 * H^1 would
 	// end up with two factors of x^-1, but the multiplication consumes one.
 	// So the product H^2 ends up with the desired one factor of x^-1.
-	_ghash_mul	H_CUR_XMM, H_CUR_XMM, H_INC_XMM, GFPOLY_XMM, \
-			%xmm0, %xmm1, %xmm2
+	_ghash_square	H_CUR_XMM, H_INC_XMM, GFPOLY_XMM, %xmm0, %xmm1
 
 	// Create H_CUR_YMM = [H^2, H^1] and H_INC_YMM = [H^2, H^2].
 	vinserti128	$1, H_CUR_XMM, H_INC_YMM, H_CUR_YMM
 	vinserti128	$1, H_INC_XMM, H_INC_YMM, H_INC_YMM
 
-.if VL == 64
 	// Create H_CUR = [H^4, H^3, H^2, H^1] and H_INC = [H^4, H^4, H^4, H^4].
 	_ghash_mul	H_INC_YMM, H_CUR_YMM, H_INC_YMM, GFPOLY_YMM, \
 			%ymm0, %ymm1, %ymm2
 	vinserti64x4	$1, H_CUR_YMM, H_INC, H_CUR
 	vshufi64x2	$0, H_INC, H_INC, H_INC
-.endif
 
 	// Store the lowest set of key powers.
 	vmovdqu8	H_CUR, (POWERS_PTR)
 
-	// Compute and store the remaining key powers.  With VL=32, repeatedly
-	// multiply [H^(i+1), H^i] by [H^2, H^2] to get [H^(i+3), H^(i+2)].
-	// With VL=64, repeatedly multiply [H^(i+3), H^(i+2), H^(i+1), H^i] by
+	// Compute and store the remaining key powers.
+	// Repeatedly multiply [H^(i+3), H^(i+2), H^(i+1), H^i] by
 	// [H^4, H^4, H^4, H^4] to get [H^(i+7), H^(i+6), H^(i+5), H^(i+4)].
-	mov		$(NUM_H_POWERS*16/VL) - 1, %eax
-.Lprecompute_next\@:
-	sub		$VL, POWERS_PTR
-	_ghash_mul	H_INC, H_CUR, H_CUR, GFPOLY, V0, V1, V2
+	mov		$3, %eax
+.Lprecompute_next:
+	sub		$64, POWERS_PTR
+	_ghash_mul	H_INC, H_CUR, H_CUR, GFPOLY, %zmm0, %zmm1, %zmm2
 	vmovdqu8	H_CUR, (POWERS_PTR)
 	dec		%eax
-	jnz		.Lprecompute_next\@
+	jnz		.Lprecompute_next
 
 	vzeroupper	// This is needed after using ymm or zmm registers.
 	RET
-.endm
+SYM_FUNC_END(aes_gcm_precompute_vaes_avx512)
 
 // XOR together the 128-bit lanes of \src (whose low lane is \src_xmm) and store
 // the result in \dst_xmm.  This implicitly zeroizes the other lanes of dst.
 .macro	_horizontal_xor	src, src_xmm, dst_xmm, t0_xmm, t1_xmm, t2_xmm
 	vextracti32x4	$1, \src, \t0_xmm
-.if VL == 32
-	vpxord		\t0_xmm, \src_xmm, \dst_xmm
-.elseif VL == 64
 	vextracti32x4	$2, \src, \t1_xmm
 	vextracti32x4	$3, \src, \t2_xmm
 	vpxord		\t0_xmm, \src_xmm, \dst_xmm
 	vpternlogd	$0x96, \t1_xmm, \t2_xmm, \dst_xmm
-.else
-	.error "Unsupported vector length"
-.endif
 .endm
 
 // Do one step of the GHASH update of the data blocks given in the vector
@@ -458,25 +401,21 @@
 //
 // The GHASH update does: GHASH_ACC = H_POW4*(GHASHDATA0 + GHASH_ACC) +
 // H_POW3*GHASHDATA1 + H_POW2*GHASHDATA2 + H_POW1*GHASHDATA3, where the
-// operations are vectorized operations on vectors of 16-byte blocks.  E.g.,
-// with VL=32 there are 2 blocks per vector and the vectorized terms correspond
-// to the following non-vectorized terms:
+// operations are vectorized operations on 512-bit vectors of 128-bit blocks.
+// The vectorized terms correspond to the following non-vectorized terms:
 //
-//	H_POW4*(GHASHDATA0 + GHASH_ACC) => H^8*(blk0 + GHASH_ACC_XMM) and H^7*(blk1 + 0)
-//	H_POW3*GHASHDATA1 => H^6*blk2 and H^5*blk3
-//	H_POW2*GHASHDATA2 => H^4*blk4 and H^3*blk5
-//	H_POW1*GHASHDATA3 => H^2*blk6 and H^1*blk7
-//
-// With VL=64, we use 4 blocks/vector, H^16 through H^1, and blk0 through blk15.
+//       H_POW4*(GHASHDATA0 + GHASH_ACC) => H^16*(blk0 + GHASH_ACC_XMM),
+//              H^15*(blk1 + 0), H^14*(blk2 + 0), and H^13*(blk3 + 0)
+//       H_POW3*GHASHDATA1 => H^12*blk4, H^11*blk5, H^10*blk6, and H^9*blk7
+//       H_POW2*GHASHDATA2 => H^8*blk8,  H^7*blk9,  H^6*blk10, and H^5*blk11
+//       H_POW1*GHASHDATA3 => H^4*blk12, H^3*blk13, H^2*blk14, and H^1*blk15
 //
 // More concretely, this code does:
 //   - Do vectorized "schoolbook" multiplications to compute the intermediate
 //     256-bit product of each block and its corresponding hash key power.
-//     There are 4*VL/16 of these intermediate products.
-//   - Sum (XOR) the intermediate 256-bit products across vectors.  This leaves
-//     VL/16 256-bit intermediate values.
+//   - Sum (XOR) the intermediate 256-bit products across vectors.
 //   - Do a vectorized reduction of these 256-bit intermediate values to
-//     128-bits each.  This leaves VL/16 128-bit intermediate values.
+//     128-bits each.
 //   - Sum (XOR) these values and store the 128-bit result in GHASH_ACC_XMM.
 //
 // See _ghash_mul_step for the full explanation of the operations performed for
@@ -532,59 +471,224 @@
 .endif
 .endm
 
-// Do one non-last round of AES encryption on the counter blocks in V0-V3 using
-// the round key that has been broadcast to all 128-bit lanes of \round_key.
+// Update GHASH with four vectors of data blocks.  See _ghash_step_4x for full
+// explanation.
+.macro	_ghash_4x
+.irp i, 0,1,2,3,4,5,6,7,8,9
+	_ghash_step_4x	\i
+.endr
+.endm
+
+// void aes_gcm_aad_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
+//				       u8 ghash_acc[16],
+//				       const u8 *aad, int aadlen);
+//
+// This function processes the AAD (Additional Authenticated Data) in GCM.
+// Using the key |key|, it updates the GHASH accumulator |ghash_acc| with the
+// data given by |aad| and |aadlen|.  On the first call, |ghash_acc| must be all
+// zeroes.  |aadlen| must be a multiple of 16, except on the last call where it
+// can be any length.  The caller must do any buffering needed to ensure this.
+//
+// This handles large amounts of AAD efficiently, while also keeping overhead
+// low for small amounts which is the common case.  TLS and IPsec use less than
+// one block of AAD, but (uncommonly) other use cases may use much more.
+SYM_FUNC_START(aes_gcm_aad_update_vaes_avx512)
+
+	// Function arguments
+	.set	KEY,		%rdi
+	.set	GHASH_ACC_PTR,	%rsi
+	.set	AAD,		%rdx
+	.set	AADLEN,		%ecx
+	.set	AADLEN64,	%rcx	// Zero-extend AADLEN before using!
+
+	// Additional local variables.
+	// %rax and %k1 are used as temporary registers.
+	.set	GHASHDATA0,	%zmm0
+	.set	GHASHDATA0_XMM,	%xmm0
+	.set	GHASHDATA1,	%zmm1
+	.set	GHASHDATA1_XMM,	%xmm1
+	.set	GHASHDATA2,	%zmm2
+	.set	GHASHDATA2_XMM,	%xmm2
+	.set	GHASHDATA3,	%zmm3
+	.set	BSWAP_MASK,	%zmm4
+	.set	BSWAP_MASK_XMM,	%xmm4
+	.set	GHASH_ACC,	%zmm5
+	.set	GHASH_ACC_XMM,	%xmm5
+	.set	H_POW4,		%zmm6
+	.set	H_POW3,		%zmm7
+	.set	H_POW2,		%zmm8
+	.set	H_POW1,		%zmm9
+	.set	H_POW1_XMM,	%xmm9
+	.set	GFPOLY,		%zmm10
+	.set	GFPOLY_XMM,	%xmm10
+	.set	GHASHTMP0,	%zmm11
+	.set	GHASHTMP1,	%zmm12
+	.set	GHASHTMP2,	%zmm13
+
+	// Load the GHASH accumulator.
+	vmovdqu		(GHASH_ACC_PTR), GHASH_ACC_XMM
+
+	// Check for the common case of AADLEN <= 16, as well as AADLEN == 0.
+	cmp		$16, AADLEN
+	jg		.Laad_more_than_16bytes
+	test		AADLEN, AADLEN
+	jz		.Laad_done
+
+	// Fast path: update GHASH with 1 <= AADLEN <= 16 bytes of AAD.
+	vmovdqu		.Lbswap_mask(%rip), BSWAP_MASK_XMM
+	vmovdqu		.Lgfpoly(%rip), GFPOLY_XMM
+	mov		$-1, %eax
+	bzhi		AADLEN, %eax, %eax
+	kmovd		%eax, %k1
+	vmovdqu8	(AAD), GHASHDATA0_XMM{%k1}{z}
+	vmovdqu		OFFSETOFEND_H_POWERS-16(KEY), H_POW1_XMM
+	vpshufb		BSWAP_MASK_XMM, GHASHDATA0_XMM, GHASHDATA0_XMM
+	vpxor		GHASHDATA0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
+	_ghash_mul	H_POW1_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM, GFPOLY_XMM, \
+			GHASHDATA0_XMM, GHASHDATA1_XMM, GHASHDATA2_XMM
+	jmp		.Laad_done
+
+.Laad_more_than_16bytes:
+	vbroadcasti32x4	.Lbswap_mask(%rip), BSWAP_MASK
+	vbroadcasti32x4	.Lgfpoly(%rip), GFPOLY
+
+	// If AADLEN >= 256, update GHASH with 256 bytes of AAD at a time.
+	sub		$256, AADLEN
+	jl		.Laad_loop_4x_done
+	vmovdqu8	OFFSETOFEND_H_POWERS-4*64(KEY), H_POW4
+	vmovdqu8	OFFSETOFEND_H_POWERS-3*64(KEY), H_POW3
+	vmovdqu8	OFFSETOFEND_H_POWERS-2*64(KEY), H_POW2
+	vmovdqu8	OFFSETOFEND_H_POWERS-1*64(KEY), H_POW1
+.Laad_loop_4x:
+	vmovdqu8	0*64(AAD), GHASHDATA0
+	vmovdqu8	1*64(AAD), GHASHDATA1
+	vmovdqu8	2*64(AAD), GHASHDATA2
+	vmovdqu8	3*64(AAD), GHASHDATA3
+	_ghash_4x
+	add		$256, AAD
+	sub		$256, AADLEN
+	jge		.Laad_loop_4x
+.Laad_loop_4x_done:
+
+	// If AADLEN >= 64, update GHASH with 64 bytes of AAD at a time.
+	add		$192, AADLEN
+	jl		.Laad_loop_1x_done
+	vmovdqu8	OFFSETOFEND_H_POWERS-1*64(KEY), H_POW1
+.Laad_loop_1x:
+	vmovdqu8	(AAD), GHASHDATA0
+	vpshufb		BSWAP_MASK, GHASHDATA0, GHASHDATA0
+	vpxord		GHASHDATA0, GHASH_ACC, GHASH_ACC
+	_ghash_mul	H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \
+			GHASHDATA0, GHASHDATA1, GHASHDATA2
+	_horizontal_xor GHASH_ACC, GHASH_ACC_XMM, GHASH_ACC_XMM, \
+			GHASHDATA0_XMM, GHASHDATA1_XMM, GHASHDATA2_XMM
+	add		$64, AAD
+	sub		$64, AADLEN
+	jge		.Laad_loop_1x
+.Laad_loop_1x_done:
+
+	// Update GHASH with the remaining 0 <= AADLEN < 64 bytes of AAD.
+	add		$64, AADLEN
+	jz		.Laad_done
+	mov		$-1, %rax
+	bzhi		AADLEN64, %rax, %rax
+	kmovq		%rax, %k1
+	vmovdqu8	(AAD), GHASHDATA0{%k1}{z}
+	neg		AADLEN64
+	and		$~15, AADLEN64  // -round_up(AADLEN, 16)
+	vmovdqu8	OFFSETOFEND_H_POWERS(KEY,AADLEN64), H_POW1
+	vpshufb		BSWAP_MASK, GHASHDATA0, GHASHDATA0
+	vpxord		GHASHDATA0, GHASH_ACC, GHASH_ACC
+	_ghash_mul	H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \
+			GHASHDATA0, GHASHDATA1, GHASHDATA2
+	_horizontal_xor GHASH_ACC, GHASH_ACC_XMM, GHASH_ACC_XMM, \
+			GHASHDATA0_XMM, GHASHDATA1_XMM, GHASHDATA2_XMM
+
+.Laad_done:
+	// Store the updated GHASH accumulator back to memory.
+	vmovdqu		GHASH_ACC_XMM, (GHASH_ACC_PTR)
+
+	vzeroupper	// This is needed after using ymm or zmm registers.
+	RET
+SYM_FUNC_END(aes_gcm_aad_update_vaes_avx512)
+
+// Do one non-last round of AES encryption on the blocks in %zmm[0-3] using the
+// round key that has been broadcast to all 128-bit lanes of \round_key.
 .macro	_vaesenc_4x	round_key
-	vaesenc		\round_key, V0, V0
-	vaesenc		\round_key, V1, V1
-	vaesenc		\round_key, V2, V2
-	vaesenc		\round_key, V3, V3
+	vaesenc		\round_key, %zmm0, %zmm0
+	vaesenc		\round_key, %zmm1, %zmm1
+	vaesenc		\round_key, %zmm2, %zmm2
+	vaesenc		\round_key, %zmm3, %zmm3
 .endm
 
 // Start the AES encryption of four vectors of counter blocks.
 .macro	_ctr_begin_4x
 
 	// Increment LE_CTR four times to generate four vectors of little-endian
-	// counter blocks, swap each to big-endian, and store them in V0-V3.
-	vpshufb		BSWAP_MASK, LE_CTR, V0
+	// counter blocks, swap each to big-endian, and store them in %zmm[0-3].
+	vpshufb		BSWAP_MASK, LE_CTR, %zmm0
 	vpaddd		LE_CTR_INC, LE_CTR, LE_CTR
-	vpshufb		BSWAP_MASK, LE_CTR, V1
+	vpshufb		BSWAP_MASK, LE_CTR, %zmm1
 	vpaddd		LE_CTR_INC, LE_CTR, LE_CTR
-	vpshufb		BSWAP_MASK, LE_CTR, V2
+	vpshufb		BSWAP_MASK, LE_CTR, %zmm2
 	vpaddd		LE_CTR_INC, LE_CTR, LE_CTR
-	vpshufb		BSWAP_MASK, LE_CTR, V3
+	vpshufb		BSWAP_MASK, LE_CTR, %zmm3
 	vpaddd		LE_CTR_INC, LE_CTR, LE_CTR
 
 	// AES "round zero": XOR in the zero-th round key.
-	vpxord		RNDKEY0, V0, V0
-	vpxord		RNDKEY0, V1, V1
-	vpxord		RNDKEY0, V2, V2
-	vpxord		RNDKEY0, V3, V3
+	vpxord		RNDKEY0, %zmm0, %zmm0
+	vpxord		RNDKEY0, %zmm1, %zmm1
+	vpxord		RNDKEY0, %zmm2, %zmm2
+	vpxord		RNDKEY0, %zmm3, %zmm3
 .endm
 
-// void aes_gcm_{enc,dec}_update_##suffix(const struct aes_gcm_key_avx10 *key,
-//					  const u32 le_ctr[4], u8 ghash_acc[16],
-//					  const u8 *src, u8 *dst, int datalen);
+// Do the last AES round for four vectors of counter blocks %zmm[0-3], XOR
+// source data with the resulting keystream, and write the result to DST and
+// GHASHDATA[0-3].  (Implementation differs slightly, but has the same effect.)
+.macro	_aesenclast_and_xor_4x
+	// XOR the source data with the last round key, saving the result in
+	// GHASHDATA[0-3].  This reduces latency by taking advantage of the
+	// property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a).
+	vpxord		0*64(SRC), RNDKEYLAST, GHASHDATA0
+	vpxord		1*64(SRC), RNDKEYLAST, GHASHDATA1
+	vpxord		2*64(SRC), RNDKEYLAST, GHASHDATA2
+	vpxord		3*64(SRC), RNDKEYLAST, GHASHDATA3
+
+	// Do the last AES round.  This handles the XOR with the source data
+	// too, as per the optimization described above.
+	vaesenclast	GHASHDATA0, %zmm0, GHASHDATA0
+	vaesenclast	GHASHDATA1, %zmm1, GHASHDATA1
+	vaesenclast	GHASHDATA2, %zmm2, GHASHDATA2
+	vaesenclast	GHASHDATA3, %zmm3, GHASHDATA3
+
+	// Store the en/decrypted data to DST.
+	vmovdqu8	GHASHDATA0, 0*64(DST)
+	vmovdqu8	GHASHDATA1, 1*64(DST)
+	vmovdqu8	GHASHDATA2, 2*64(DST)
+	vmovdqu8	GHASHDATA3, 3*64(DST)
+.endm
+
+// void aes_gcm_{enc,dec}_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
+//					     const u32 le_ctr[4], u8 ghash_acc[16],
+//					     const u8 *src, u8 *dst, int datalen);
 //
 // This macro generates a GCM encryption or decryption update function with the
-// above prototype (with \enc selecting which one).  This macro supports both
-// VL=32 and VL=64.  _set_veclen must have been invoked with the desired length.
-//
-// This function computes the next portion of the CTR keystream, XOR's it with
-// |datalen| bytes from |src|, and writes the resulting encrypted or decrypted
-// data to |dst|.  It also updates the GHASH accumulator |ghash_acc| using the
-// next |datalen| ciphertext bytes.
+// above prototype (with \enc selecting which one).  The function computes the
+// next portion of the CTR keystream, XOR's it with |datalen| bytes from |src|,
+// and writes the resulting encrypted or decrypted data to |dst|.  It also
+// updates the GHASH accumulator |ghash_acc| using the next |datalen| ciphertext
+// bytes.
 //
 // |datalen| must be a multiple of 16, except on the last call where it can be
 // any length.  The caller must do any buffering needed to ensure this.  Both
 // in-place and out-of-place en/decryption are supported.
 //
-// |le_ctr| must give the current counter in little-endian format.  For a new
-// message, the low word of the counter must be 2.  This function loads the
-// counter from |le_ctr| and increments the loaded counter as needed, but it
-// does *not* store the updated counter back to |le_ctr|.  The caller must
-// update |le_ctr| if any more data segments follow.  Internally, only the low
-// 32-bit word of the counter is incremented, following the GCM standard.
+// |le_ctr| must give the current counter in little-endian format.  This
+// function loads the counter from |le_ctr| and increments the loaded counter as
+// needed, but it does *not* store the updated counter back to |le_ctr|.  The
+// caller must update |le_ctr| if any more data segments follow.  Internally,
+// only the low 32-bit word of the counter is incremented, following the GCM
+// standard.
 .macro	_aes_gcm_update	enc
 
 	// Function arguments
@@ -608,74 +712,69 @@
 	// Pointer to the last AES round key for the chosen AES variant
 	.set	RNDKEYLAST_PTR,	%r11
 
-	// In the main loop, V0-V3 are used as AES input and output.  Elsewhere
-	// they are used as temporary registers.
+	// In the main loop, %zmm[0-3] are used as AES input and output.
+	// Elsewhere they are used as temporary registers.
 
 	// GHASHDATA[0-3] hold the ciphertext blocks and GHASH input data.
-	.set	GHASHDATA0,	V4
+	.set	GHASHDATA0,	%zmm4
 	.set	GHASHDATA0_XMM,	%xmm4
-	.set	GHASHDATA1,	V5
+	.set	GHASHDATA1,	%zmm5
 	.set	GHASHDATA1_XMM,	%xmm5
-	.set	GHASHDATA2,	V6
+	.set	GHASHDATA2,	%zmm6
 	.set	GHASHDATA2_XMM,	%xmm6
-	.set	GHASHDATA3,	V7
+	.set	GHASHDATA3,	%zmm7
 
 	// BSWAP_MASK is the shuffle mask for byte-reflecting 128-bit values
 	// using vpshufb, copied to all 128-bit lanes.
-	.set	BSWAP_MASK,	V8
+	.set	BSWAP_MASK,	%zmm8
 
 	// RNDKEY temporarily holds the next AES round key.
-	.set	RNDKEY,		V9
+	.set	RNDKEY,		%zmm9
 
 	// GHASH_ACC is the accumulator variable for GHASH.  When fully reduced,
 	// only the lowest 128-bit lane can be nonzero.  When not fully reduced,
 	// more than one lane may be used, and they need to be XOR'd together.
-	.set	GHASH_ACC,	V10
+	.set	GHASH_ACC,	%zmm10
 	.set	GHASH_ACC_XMM,	%xmm10
 
 	// LE_CTR_INC is the vector of 32-bit words that need to be added to a
 	// vector of little-endian counter blocks to advance it forwards.
-	.set	LE_CTR_INC,	V11
+	.set	LE_CTR_INC,	%zmm11
 
 	// LE_CTR contains the next set of little-endian counter blocks.
-	.set	LE_CTR,		V12
+	.set	LE_CTR,		%zmm12
 
-	// RNDKEY0, RNDKEYLAST, and RNDKEY_M[9-5] contain cached AES round keys,
+	// RNDKEY0, RNDKEYLAST, and RNDKEY_M[9-1] contain cached AES round keys,
 	// copied to all 128-bit lanes.  RNDKEY0 is the zero-th round key,
 	// RNDKEYLAST the last, and RNDKEY_M\i the one \i-th from the last.
-	.set	RNDKEY0,	V13
-	.set	RNDKEYLAST,	V14
-	.set	RNDKEY_M9,	V15
-	.set	RNDKEY_M8,	V16
-	.set	RNDKEY_M7,	V17
-	.set	RNDKEY_M6,	V18
-	.set	RNDKEY_M5,	V19
-
-	// RNDKEYLAST[0-3] temporarily store the last AES round key XOR'd with
-	// the corresponding block of source data.  This is useful because
-	// vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a), and key ^ b can
-	// be computed in parallel with the AES rounds.
-	.set	RNDKEYLAST0,	V20
-	.set	RNDKEYLAST1,	V21
-	.set	RNDKEYLAST2,	V22
-	.set	RNDKEYLAST3,	V23
+	.set	RNDKEY0,	%zmm13
+	.set	RNDKEYLAST,	%zmm14
+	.set	RNDKEY_M9,	%zmm15
+	.set	RNDKEY_M8,	%zmm16
+	.set	RNDKEY_M7,	%zmm17
+	.set	RNDKEY_M6,	%zmm18
+	.set	RNDKEY_M5,	%zmm19
+	.set	RNDKEY_M4,	%zmm20
+	.set	RNDKEY_M3,	%zmm21
+	.set	RNDKEY_M2,	%zmm22
+	.set	RNDKEY_M1,	%zmm23
 
 	// GHASHTMP[0-2] are temporary variables used by _ghash_step_4x.  These
 	// cannot coincide with anything used for AES encryption, since for
 	// performance reasons GHASH and AES encryption are interleaved.
-	.set	GHASHTMP0,	V24
-	.set	GHASHTMP1,	V25
-	.set	GHASHTMP2,	V26
+	.set	GHASHTMP0,	%zmm24
+	.set	GHASHTMP1,	%zmm25
+	.set	GHASHTMP2,	%zmm26
 
-	// H_POW[4-1] contain the powers of the hash key H^(4*VL/16)...H^1.  The
+	// H_POW[4-1] contain the powers of the hash key H^16...H^1.  The
 	// descending numbering reflects the order of the key powers.
-	.set	H_POW4,		V27
-	.set	H_POW3,		V28
-	.set	H_POW2,		V29
-	.set	H_POW1,		V30
+	.set	H_POW4,		%zmm27
+	.set	H_POW3,		%zmm28
+	.set	H_POW2,		%zmm29
+	.set	H_POW1,		%zmm30
 
 	// GFPOLY contains the .Lgfpoly constant, copied to all 128-bit lanes.
-	.set	GFPOLY,		V31
+	.set	GFPOLY,		%zmm31
 
 	// Load some constants.
 	vbroadcasti32x4	.Lbswap_mask(%rip), BSWAP_MASK
@@ -698,29 +797,23 @@
 	// Finish initializing LE_CTR by adding [0, 1, ...] to its low words.
 	vpaddd		.Lctr_pattern(%rip), LE_CTR, LE_CTR
 
-	// Initialize LE_CTR_INC to contain VL/16 in all 128-bit lanes.
-.if VL == 32
-	vbroadcasti32x4	.Linc_2blocks(%rip), LE_CTR_INC
-.elseif VL == 64
+	// Load 4 into all 128-bit lanes of LE_CTR_INC.
 	vbroadcasti32x4	.Linc_4blocks(%rip), LE_CTR_INC
-.else
-	.error "Unsupported vector length"
-.endif
 
-	// If there are at least 4*VL bytes of data, then continue into the loop
-	// that processes 4*VL bytes of data at a time.  Otherwise skip it.
+	// If there are at least 256 bytes of data, then continue into the loop
+	// that processes 256 bytes of data at a time.  Otherwise skip it.
 	//
-	// Pre-subtracting 4*VL from DATALEN saves an instruction from the main
+	// Pre-subtracting 256 from DATALEN saves an instruction from the main
 	// loop and also ensures that at least one write always occurs to
 	// DATALEN, zero-extending it and allowing DATALEN64 to be used later.
-	sub		$4*VL, DATALEN
+	sub		$256, DATALEN
 	jl		.Lcrypt_loop_4x_done\@
 
 	// Load powers of the hash key.
-	vmovdqu8	OFFSETOFEND_H_POWERS-4*VL(KEY), H_POW4
-	vmovdqu8	OFFSETOFEND_H_POWERS-3*VL(KEY), H_POW3
-	vmovdqu8	OFFSETOFEND_H_POWERS-2*VL(KEY), H_POW2
-	vmovdqu8	OFFSETOFEND_H_POWERS-1*VL(KEY), H_POW1
+	vmovdqu8	OFFSETOFEND_H_POWERS-4*64(KEY), H_POW4
+	vmovdqu8	OFFSETOFEND_H_POWERS-3*64(KEY), H_POW3
+	vmovdqu8	OFFSETOFEND_H_POWERS-2*64(KEY), H_POW2
+	vmovdqu8	OFFSETOFEND_H_POWERS-1*64(KEY), H_POW1
 
 	// Main loop: en/decrypt and hash 4 vectors at a time.
 	//
@@ -748,26 +841,15 @@
 	add		$16, %rax
 	cmp		%rax, RNDKEYLAST_PTR
 	jne		1b
-	vpxord		0*VL(SRC), RNDKEYLAST, RNDKEYLAST0
-	vpxord		1*VL(SRC), RNDKEYLAST, RNDKEYLAST1
-	vpxord		2*VL(SRC), RNDKEYLAST, RNDKEYLAST2
-	vpxord		3*VL(SRC), RNDKEYLAST, RNDKEYLAST3
-	vaesenclast	RNDKEYLAST0, V0, GHASHDATA0
-	vaesenclast	RNDKEYLAST1, V1, GHASHDATA1
-	vaesenclast	RNDKEYLAST2, V2, GHASHDATA2
-	vaesenclast	RNDKEYLAST3, V3, GHASHDATA3
-	vmovdqu8	GHASHDATA0, 0*VL(DST)
-	vmovdqu8	GHASHDATA1, 1*VL(DST)
-	vmovdqu8	GHASHDATA2, 2*VL(DST)
-	vmovdqu8	GHASHDATA3, 3*VL(DST)
-	add		$4*VL, SRC
-	add		$4*VL, DST
-	sub		$4*VL, DATALEN
+	_aesenclast_and_xor_4x
+	add		$256, SRC
+	add		$256, DST
+	sub		$256, DATALEN
 	jl		.Lghash_last_ciphertext_4x\@
 .endif
 
 	// Cache as many additional AES round keys as possible.
-.irp i, 9,8,7,6,5
+.irp i, 9,8,7,6,5,4,3,2,1
 	vbroadcasti32x4	-\i*16(RNDKEYLAST_PTR), RNDKEY_M\i
 .endr
 
@@ -776,10 +858,10 @@
 	// If decrypting, load more ciphertext blocks into GHASHDATA[0-3].  If
 	// encrypting, GHASHDATA[0-3] already contain the previous ciphertext.
 .if !\enc
-	vmovdqu8	0*VL(SRC), GHASHDATA0
-	vmovdqu8	1*VL(SRC), GHASHDATA1
-	vmovdqu8	2*VL(SRC), GHASHDATA2
-	vmovdqu8	3*VL(SRC), GHASHDATA3
+	vmovdqu8	0*64(SRC), GHASHDATA0
+	vmovdqu8	1*64(SRC), GHASHDATA1
+	vmovdqu8	2*64(SRC), GHASHDATA2
+	vmovdqu8	3*64(SRC), GHASHDATA3
 .endif
 
 	// Start the AES encryption of the counter blocks.
@@ -799,77 +881,44 @@
 	_vaesenc_4x	RNDKEY
 128:
 
-	// XOR the source data with the last round key, saving the result in
-	// RNDKEYLAST[0-3].  This reduces latency by taking advantage of the
-	// property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a).
-.if \enc
-	vpxord		0*VL(SRC), RNDKEYLAST, RNDKEYLAST0
-	vpxord		1*VL(SRC), RNDKEYLAST, RNDKEYLAST1
-	vpxord		2*VL(SRC), RNDKEYLAST, RNDKEYLAST2
-	vpxord		3*VL(SRC), RNDKEYLAST, RNDKEYLAST3
-.else
-	vpxord		GHASHDATA0, RNDKEYLAST, RNDKEYLAST0
-	vpxord		GHASHDATA1, RNDKEYLAST, RNDKEYLAST1
-	vpxord		GHASHDATA2, RNDKEYLAST, RNDKEYLAST2
-	vpxord		GHASHDATA3, RNDKEYLAST, RNDKEYLAST3
-.endif
-
-	// Finish the AES encryption of the counter blocks in V0-V3, interleaved
-	// with the GHASH update of the ciphertext blocks in GHASHDATA[0-3].
-.irp i, 9,8,7,6,5
+	// Finish the AES encryption of the counter blocks in %zmm[0-3],
+	// interleaved with the GHASH update of the ciphertext blocks in
+	// GHASHDATA[0-3].
+.irp i, 9,8,7,6,5,4,3,2,1
+	_ghash_step_4x  (9 - \i)
 	_vaesenc_4x	RNDKEY_M\i
-	_ghash_step_4x	(9 - \i)
-.endr
-.irp i, 4,3,2,1
-	vbroadcasti32x4	-\i*16(RNDKEYLAST_PTR), RNDKEY
-	_vaesenc_4x	RNDKEY
-	_ghash_step_4x	(9 - \i)
 .endr
 	_ghash_step_4x	9
-
-	// Do the last AES round.  This handles the XOR with the source data
-	// too, as per the optimization described above.
-	vaesenclast	RNDKEYLAST0, V0, GHASHDATA0
-	vaesenclast	RNDKEYLAST1, V1, GHASHDATA1
-	vaesenclast	RNDKEYLAST2, V2, GHASHDATA2
-	vaesenclast	RNDKEYLAST3, V3, GHASHDATA3
-
-	// Store the en/decrypted data to DST.
-	vmovdqu8	GHASHDATA0, 0*VL(DST)
-	vmovdqu8	GHASHDATA1, 1*VL(DST)
-	vmovdqu8	GHASHDATA2, 2*VL(DST)
-	vmovdqu8	GHASHDATA3, 3*VL(DST)
-
-	add		$4*VL, SRC
-	add		$4*VL, DST
-	sub		$4*VL, DATALEN
+	_aesenclast_and_xor_4x
+	add		$256, SRC
+	add		$256, DST
+	sub		$256, DATALEN
 	jge		.Lcrypt_loop_4x\@
 
 .if \enc
 .Lghash_last_ciphertext_4x\@:
 	// Update GHASH with the last set of ciphertext blocks.
-.irp i, 0,1,2,3,4,5,6,7,8,9
-	_ghash_step_4x	\i
-.endr
+	_ghash_4x
 .endif
 
 .Lcrypt_loop_4x_done\@:
 
-	// Undo the extra subtraction by 4*VL and check whether data remains.
-	add		$4*VL, DATALEN
+	// Undo the extra subtraction by 256 and check whether data remains.
+	add		$256, DATALEN
 	jz		.Ldone\@
 
-	// The data length isn't a multiple of 4*VL.  Process the remaining data
-	// of length 1 <= DATALEN < 4*VL, up to one vector (VL bytes) at a time.
-	// Going one vector at a time may seem inefficient compared to having
-	// separate code paths for each possible number of vectors remaining.
-	// However, using a loop keeps the code size down, and it performs
-	// surprising well; modern CPUs will start executing the next iteration
-	// before the previous one finishes and also predict the number of loop
-	// iterations.  For a similar reason, we roll up the AES rounds.
+	// The data length isn't a multiple of 256 bytes.  Process the remaining
+	// data of length 1 <= DATALEN < 256, up to one 64-byte vector at a
+	// time.  Going one vector at a time may seem inefficient compared to
+	// having separate code paths for each possible number of vectors
+	// remaining.  However, using a loop keeps the code size down, and it
+	// performs surprising well; modern CPUs will start executing the next
+	// iteration before the previous one finishes and also predict the
+	// number of loop iterations.  For a similar reason, we roll up the AES
+	// rounds.
 	//
-	// On the last iteration, the remaining length may be less than VL.
-	// Handle this using masking.
+	// On the last iteration, the remaining length may be less than 64
+	// bytes.  Handle this using masking.
 	//
 	// Since there are enough key powers available for all remaining data,
 	// there is no need to do a GHASH reduction after each iteration.
@@ -898,65 +947,60 @@
 .Lcrypt_loop_1x\@:
 
 	// Select the appropriate mask for this iteration: all 1's if
-	// DATALEN >= VL, otherwise DATALEN 1's.  Do this branchlessly using the
+	// DATALEN >= 64, otherwise DATALEN 1's.  Do this branchlessly using the
 	// bzhi instruction from BMI2.  (This relies on DATALEN <= 255.)
-.if VL < 64
-	mov		$-1, %eax
-	bzhi		DATALEN, %eax, %eax
-	kmovd		%eax, %k1
-.else
 	mov		$-1, %rax
 	bzhi		DATALEN64, %rax, %rax
 	kmovq		%rax, %k1
-.endif
 
 	// Encrypt a vector of counter blocks.  This does not need to be masked.
-	vpshufb		BSWAP_MASK, LE_CTR, V0
+	vpshufb		BSWAP_MASK, LE_CTR, %zmm0
 	vpaddd		LE_CTR_INC, LE_CTR, LE_CTR
-	vpxord		RNDKEY0, V0, V0
+	vpxord		RNDKEY0, %zmm0, %zmm0
 	lea		16(KEY), %rax
 1:
 	vbroadcasti32x4	(%rax), RNDKEY
-	vaesenc		RNDKEY, V0, V0
+	vaesenc		RNDKEY, %zmm0, %zmm0
 	add		$16, %rax
 	cmp		%rax, RNDKEYLAST_PTR
 	jne		1b
-	vaesenclast	RNDKEYLAST, V0, V0
+	vaesenclast	RNDKEYLAST, %zmm0, %zmm0
 
 	// XOR the data with the appropriate number of keystream bytes.
-	vmovdqu8	(SRC), V1{%k1}{z}
-	vpxord		V1, V0, V0
-	vmovdqu8	V0, (DST){%k1}
+	vmovdqu8	(SRC), %zmm1{%k1}{z}
+	vpxord		%zmm1, %zmm0, %zmm0
+	vmovdqu8	%zmm0, (DST){%k1}
 
 	// Update GHASH with the ciphertext block(s), without reducing.
 	//
-	// In the case of DATALEN < VL, the ciphertext is zero-padded to VL.
-	// (If decrypting, it's done by the above masked load.  If encrypting,
-	// it's done by the below masked register-to-register move.)  Note that
-	// if DATALEN <= VL - 16, there will be additional padding beyond the
-	// padding of the last block specified by GHASH itself; i.e., there may
-	// be whole block(s) that get processed by the GHASH multiplication and
-	// reduction instructions but should not actually be included in the
+	// In the case of DATALEN < 64, the ciphertext is zero-padded to 64
+	// bytes.  (If decrypting, it's done by the above masked load.  If
+	// encrypting, it's done by the below masked register-to-register move.)
+	// Note that if DATALEN <= 48, there will be additional padding beyond
+	// the padding of the last block specified by GHASH itself; i.e., there
+	// may be whole block(s) that get processed by the GHASH multiplication
+	// and reduction instructions but should not actually be included in the
 	// GHASH.  However, any such blocks are all-zeroes, and the values that
 	// they're multiplied with are also all-zeroes.  Therefore they just add
 	// 0 * 0 = 0 to the final GHASH result, which makes no difference.
-	vmovdqu8        (POWERS_PTR), H_POW1
+	vmovdqu8	(POWERS_PTR), H_POW1
 .if \enc
-	vmovdqu8	V0, V1{%k1}{z}
+	vmovdqu8	%zmm0, %zmm1{%k1}{z}
 .endif
-	vpshufb		BSWAP_MASK, V1, V0
-	vpxord		GHASH_ACC, V0, V0
-	_ghash_mul_noreduce	H_POW1, V0, LO, MI, HI, GHASHDATA3, V1, V2, V3
+	vpshufb		BSWAP_MASK, %zmm1, %zmm0
+	vpxord		GHASH_ACC, %zmm0, %zmm0
+	_ghash_mul_noreduce	H_POW1, %zmm0, LO, MI, HI, \
+				GHASHDATA3, %zmm1, %zmm2, %zmm3
 	vpxor		GHASH_ACC_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
 
-	add		$VL, POWERS_PTR
-	add		$VL, SRC
-	add		$VL, DST
-	sub		$VL, DATALEN
+	add		$64, POWERS_PTR
+	add		$64, SRC
+	add		$64, DST
+	sub		$64, DATALEN
 	jg		.Lcrypt_loop_1x\@
 
 	// Finally, do the GHASH reduction.
-	_ghash_reduce	LO, MI, HI, GFPOLY, V0
+	_ghash_reduce	LO, MI, HI, GFPOLY, %zmm0
 	_horizontal_xor	HI, HI_XMM, GHASH_ACC_XMM, %xmm0, %xmm1, %xmm2
 
 .Ldone\@:
@@ -967,14 +1011,14 @@
 	RET
 .endm
 
-// void aes_gcm_enc_final_vaes_avx10(const struct aes_gcm_key_avx10 *key,
-//				     const u32 le_ctr[4], u8 ghash_acc[16],
-//				     u64 total_aadlen, u64 total_datalen);
-// bool aes_gcm_dec_final_vaes_avx10(const struct aes_gcm_key_avx10 *key,
-//				     const u32 le_ctr[4],
-//				     const u8 ghash_acc[16],
-//				     u64 total_aadlen, u64 total_datalen,
-//				     const u8 tag[16], int taglen);
+// void aes_gcm_enc_final_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
+//				      const u32 le_ctr[4], u8 ghash_acc[16],
+//				      u64 total_aadlen, u64 total_datalen);
+// bool aes_gcm_dec_final_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
+//				      const u32 le_ctr[4],
+//				      const u8 ghash_acc[16],
+//				      u64 total_aadlen, u64 total_datalen,
+//				      const u8 tag[16], int taglen);
 //
 // This macro generates one of the above two functions (with \enc selecting
 // which one).  Both functions finish computing the GCM authentication tag by
@@ -1104,119 +1148,16 @@
 	RET
 .endm
 
-_set_veclen 32
-SYM_FUNC_START(aes_gcm_precompute_vaes_avx10_256)
-	_aes_gcm_precompute
-SYM_FUNC_END(aes_gcm_precompute_vaes_avx10_256)
-SYM_FUNC_START(aes_gcm_enc_update_vaes_avx10_256)
-	_aes_gcm_update	1
-SYM_FUNC_END(aes_gcm_enc_update_vaes_avx10_256)
-SYM_FUNC_START(aes_gcm_dec_update_vaes_avx10_256)
-	_aes_gcm_update	0
-SYM_FUNC_END(aes_gcm_dec_update_vaes_avx10_256)
-
-_set_veclen 64
-SYM_FUNC_START(aes_gcm_precompute_vaes_avx10_512)
-	_aes_gcm_precompute
-SYM_FUNC_END(aes_gcm_precompute_vaes_avx10_512)
-SYM_FUNC_START(aes_gcm_enc_update_vaes_avx10_512)
+SYM_FUNC_START(aes_gcm_enc_update_vaes_avx512)
 	_aes_gcm_update	1
-SYM_FUNC_END(aes_gcm_enc_update_vaes_avx10_512)
-SYM_FUNC_START(aes_gcm_dec_update_vaes_avx10_512)
+SYM_FUNC_END(aes_gcm_enc_update_vaes_avx512)
+SYM_FUNC_START(aes_gcm_dec_update_vaes_avx512)
 	_aes_gcm_update	0
-SYM_FUNC_END(aes_gcm_dec_update_vaes_avx10_512)
-
-// void aes_gcm_aad_update_vaes_avx10(const struct aes_gcm_key_avx10 *key,
-//				      u8 ghash_acc[16],
-//				      const u8 *aad, int aadlen);
-//
-// This function processes the AAD (Additional Authenticated Data) in GCM.
-// Using the key |key|, it updates the GHASH accumulator |ghash_acc| with the
-// data given by |aad| and |aadlen|.  |key->ghash_key_powers| must have been
-// initialized.  On the first call, |ghash_acc| must be all zeroes.  |aadlen|
-// must be a multiple of 16, except on the last call where it can be any length.
-// The caller must do any buffering needed to ensure this.
-//
-// AES-GCM is almost always used with small amounts of AAD, less than 32 bytes.
-// Therefore, for AAD processing we currently only provide this implementation
-// which uses 256-bit vectors (ymm registers) and only has a 1x-wide loop.  This
-// keeps the code size down, and it enables some micro-optimizations, e.g. using
-// VEX-coded instructions instead of EVEX-coded to save some instruction bytes.
-// To optimize for large amounts of AAD, we could implement a 4x-wide loop and
-// provide a version using 512-bit vectors, but that doesn't seem to be useful.
-SYM_FUNC_START(aes_gcm_aad_update_vaes_avx10)
-
-	// Function arguments
-	.set	KEY,		%rdi
-	.set	GHASH_ACC_PTR,	%rsi
-	.set	AAD,		%rdx
-	.set	AADLEN,		%ecx
-	.set	AADLEN64,	%rcx	// Zero-extend AADLEN before using!
-
-	// Additional local variables.
-	// %rax, %ymm0-%ymm3, and %k1 are used as temporary registers.
-	.set	BSWAP_MASK,	%ymm4
-	.set	GFPOLY,		%ymm5
-	.set	GHASH_ACC,	%ymm6
-	.set	GHASH_ACC_XMM,	%xmm6
-	.set	H_POW1,		%ymm7
-
-	// Load some constants.
-	vbroadcasti128	.Lbswap_mask(%rip), BSWAP_MASK
-	vbroadcasti128	.Lgfpoly(%rip), GFPOLY
-
-	// Load the GHASH accumulator.
-	vmovdqu		(GHASH_ACC_PTR), GHASH_ACC_XMM
-
-	// Update GHASH with 32 bytes of AAD at a time.
-	//
-	// Pre-subtracting 32 from AADLEN saves an instruction from the loop and
-	// also ensures that at least one write always occurs to AADLEN,
-	// zero-extending it and allowing AADLEN64 to be used later.
-	sub		$32, AADLEN
-	jl		.Laad_loop_1x_done
-	vmovdqu8	OFFSETOFEND_H_POWERS-32(KEY), H_POW1	// [H^2, H^1]
-.Laad_loop_1x:
-	vmovdqu		(AAD), %ymm0
-	vpshufb		BSWAP_MASK, %ymm0, %ymm0
-	vpxor		%ymm0, GHASH_ACC, GHASH_ACC
-	_ghash_mul	H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \
-			%ymm0, %ymm1, %ymm2
-	vextracti128	$1, GHASH_ACC, %xmm0
-	vpxor		%xmm0, GHASH_ACC_XMM, GHASH_ACC_XMM
-	add		$32, AAD
-	sub		$32, AADLEN
-	jge		.Laad_loop_1x
-.Laad_loop_1x_done:
-	add		$32, AADLEN
-	jz		.Laad_done
-
-	// Update GHASH with the remaining 1 <= AADLEN < 32 bytes of AAD.
-	mov		$-1, %eax
-	bzhi		AADLEN, %eax, %eax
-	kmovd		%eax, %k1
-	vmovdqu8	(AAD), %ymm0{%k1}{z}
-	neg		AADLEN64
-	and		$~15, AADLEN64  // -round_up(AADLEN, 16)
-	vmovdqu8	OFFSETOFEND_H_POWERS(KEY,AADLEN64), H_POW1
-	vpshufb		BSWAP_MASK, %ymm0, %ymm0
-	vpxor		%ymm0, GHASH_ACC, GHASH_ACC
-	_ghash_mul	H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \
-			%ymm0, %ymm1, %ymm2
-	vextracti128	$1, GHASH_ACC, %xmm0
-	vpxor		%xmm0, GHASH_ACC_XMM, GHASH_ACC_XMM
-
-.Laad_done:
-	// Store the updated GHASH accumulator back to memory.
-	vmovdqu		GHASH_ACC_XMM, (GHASH_ACC_PTR)
-
-	vzeroupper	// This is needed after using ymm or zmm registers.
-	RET
-SYM_FUNC_END(aes_gcm_aad_update_vaes_avx10)
+SYM_FUNC_END(aes_gcm_dec_update_vaes_avx512)
 
-SYM_FUNC_START(aes_gcm_enc_final_vaes_avx10)
+SYM_FUNC_START(aes_gcm_enc_final_vaes_avx512)
 	_aes_gcm_final	1
-SYM_FUNC_END(aes_gcm_enc_final_vaes_avx10)
-SYM_FUNC_START(aes_gcm_dec_final_vaes_avx10)
+SYM_FUNC_END(aes_gcm_enc_final_vaes_avx512)
+SYM_FUNC_START(aes_gcm_dec_final_vaes_avx512)
 	_aes_gcm_final	0
-SYM_FUNC_END(aes_gcm_dec_final_vaes_avx10)
+SYM_FUNC_END(aes_gcm_dec_final_vaes_avx512)
diff --git a/arch/x86/crypto/aes-xts-avx-x86_64.S b/arch/x86/crypto/aes-xts-avx-x86_64.S
index 48f97b79f7a9..a30753a3e207 100644
--- a/arch/x86/crypto/aes-xts-avx-x86_64.S
+++ b/arch/x86/crypto/aes-xts-avx-x86_64.S
@@ -1,11 +1,50 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * AES-XTS for modern x86_64 CPUs
- *
- * Copyright 2024 Google LLC
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
+/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
+//
+// AES-XTS for modern x86_64 CPUs
+//
+// Copyright 2024 Google LLC
+//
+// Author: Eric Biggers <ebiggers@google.com>
+//
+//------------------------------------------------------------------------------
+//
+// This file is dual-licensed, meaning that you can use it under your choice of
+// either of the following two licenses:
+//
+// Licensed under the Apache License 2.0 (the "License").  You may obtain a copy
+// of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// or
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+//    this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
 
 /*
  * This file implements AES-XTS for modern x86_64 CPUs.  To handle the
@@ -13,32 +52,25 @@
  * different code, it uses a macro to generate several implementations that
  * share similar source code but are targeted at different CPUs, listed below:
  *
- * AES-NI + AVX
+ * AES-NI && AVX
  *    - 128-bit vectors (1 AES block per vector)
  *    - VEX-coded instructions
  *    - xmm0-xmm15
  *    - This is for older CPUs that lack VAES but do have AVX.
  *
- * VAES + VPCLMULQDQ + AVX2
+ * VAES && VPCLMULQDQ && AVX2
  *    - 256-bit vectors (2 AES blocks per vector)
  *    - VEX-coded instructions
  *    - ymm0-ymm15
- *    - This is for CPUs that have VAES but lack AVX512 or AVX10,
- *      e.g. Intel's Alder Lake and AMD's Zen 3.
+ *    - This is for CPUs that have VAES but either lack AVX512 (e.g. Intel's
+ *      Alder Lake and AMD's Zen 3) or downclock too eagerly when using zmm
+ *      registers (e.g. Intel's Ice Lake).
  *
- * VAES + VPCLMULQDQ + AVX10/256 + BMI2
- *    - 256-bit vectors (2 AES blocks per vector)
+ * VAES && VPCLMULQDQ && AVX512BW && AVX512VL && BMI2
+ *    - 512-bit vectors (4 AES blocks per vector)
  *    - EVEX-coded instructions
- *    - ymm0-ymm31
- *    - This is for CPUs that have AVX512 but where using zmm registers causes
- *      downclocking, and for CPUs that have AVX10/256 but not AVX10/512.
- *    - By "AVX10/256" we really mean (AVX512BW + AVX512VL) || AVX10/256.
- *      To avoid confusion with 512-bit, we just write AVX10/256.
- *
- * VAES + VPCLMULQDQ + AVX10/512 + BMI2
- *    - Same as the previous one, but upgrades to 512-bit vectors
- *      (4 AES blocks per vector) in zmm0-zmm31.
- *    - This is for CPUs that have good AVX512 or AVX10/512 support.
+ *    - zmm0-zmm31
+ *    - This is for CPUs that have good AVX512 support.
  *
  * This file doesn't have an implementation for AES-NI alone (without AVX), as
  * the lack of VEX would make all the assembly code different.
@@ -68,9 +100,20 @@
 	// exists when there's a carry out of the low 64 bits of the tweak.
 	.quad	0x87, 1
 
+	// These are the shift amounts that are needed when multiplying by [x^0,
+	// x^1, x^2, x^3] to compute the first vector of tweaks when VL=64.
+	//
+	// The right shifts by 64 are expected to zeroize the destination.
+	// 'vpsrlvq' is indeed defined to do that; i.e. it doesn't truncate the
+	// amount to 64 & 63 = 0 like the 'shr' scalar shift instruction would.
+.Lrshift_amounts:
+	.byte	64, 64, 63, 63, 62, 62, 61, 61
+.Llshift_amounts:
+	.byte	0, 0, 1, 1, 2, 2, 3, 3
+
 	// This table contains constants for vpshufb and vpblendvb, used to
 	// handle variable byte shifts and blending during ciphertext stealing
-	// on CPUs that don't support AVX10-style masking.
+	// on CPUs that don't support AVX512-style masking.
 .Lcts_permute_table:
 	.byte	0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
 	.byte	0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
@@ -80,22 +123,6 @@
 	.byte	0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
 .text
 
-// Function parameters
-.set	KEY,		%rdi	// Initially points to crypto_aes_ctx, then is
-				// advanced to point to 7th-from-last round key
-.set	SRC,		%rsi	// Pointer to next source data
-.set	DST,		%rdx	// Pointer to next destination data
-.set	LEN,		%ecx	// Remaining length in bytes
-.set	LEN8,		%cl
-.set	LEN64,		%rcx
-.set	TWEAK,		%r8	// Pointer to next tweak
-
-// %rax holds the AES key length in bytes.
-.set	KEYLEN,		%eax
-.set	KEYLEN64,	%rax
-
-// %r9-r11 are available as temporaries.
-
 .macro	_define_Vi	i
 .if VL == 16
 	.set	V\i,		%xmm\i
@@ -112,41 +139,31 @@
 	// Define register aliases V0-V15, or V0-V31 if all 32 SIMD registers
 	// are available, that map to the xmm, ymm, or zmm registers according
 	// to the selected Vector Length (VL).
-	_define_Vi	0
-	_define_Vi	1
-	_define_Vi	2
-	_define_Vi	3
-	_define_Vi	4
-	_define_Vi	5
-	_define_Vi	6
-	_define_Vi	7
-	_define_Vi	8
-	_define_Vi	9
-	_define_Vi	10
-	_define_Vi	11
-	_define_Vi	12
-	_define_Vi	13
-	_define_Vi	14
-	_define_Vi	15
-.if USE_AVX10
-	_define_Vi	16
-	_define_Vi	17
-	_define_Vi	18
-	_define_Vi	19
-	_define_Vi	20
-	_define_Vi	21
-	_define_Vi	22
-	_define_Vi	23
-	_define_Vi	24
-	_define_Vi	25
-	_define_Vi	26
-	_define_Vi	27
-	_define_Vi	28
-	_define_Vi	29
-	_define_Vi	30
-	_define_Vi	31
+.irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+	_define_Vi	\i
+.endr
+.if USE_AVX512
+.irp i, 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
+	_define_Vi	\i
+.endr
 .endif
 
+	// Function parameters
+	.set	KEY,		%rdi	// Initially points to crypto_aes_ctx, then is
+					// advanced to point to 7th-from-last round key
+	.set	SRC,		%rsi	// Pointer to next source data
+	.set	DST,		%rdx	// Pointer to next destination data
+	.set	LEN,		%ecx	// Remaining length in bytes
+	.set	LEN8,		%cl
+	.set	LEN64,		%rcx
+	.set	TWEAK,		%r8	// Pointer to next tweak
+
+	// %rax holds the AES key length in bytes.
+	.set	KEYLEN,		%eax
+	.set	KEYLEN64,	%rax
+
+	// %r9-r11 are available as temporaries.
+
 	// V0-V3 hold the data blocks during the main loop, or temporary values
 	// otherwise.  V4-V5 hold temporary values.
 
@@ -180,7 +197,7 @@
 	// keys to the *end* of this register range.  I.e., AES-128 uses
 	// KEY5-KEY14, AES-192 uses KEY3-KEY14, and AES-256 uses KEY1-KEY14.
 	// (All also use KEY0 for the XOR-only "round" at the beginning.)
-.if USE_AVX10
+.if USE_AVX512
 	.set	KEY1_XMM,	%xmm16
 	.set	KEY1,		V16
 	.set	KEY2_XMM,	%xmm17
@@ -224,9 +241,9 @@
 
 // Broadcast a 128-bit value into a vector.
 .macro	_vbroadcast128	src, dst
-.if VL == 16 && !USE_AVX10
+.if VL == 16
 	vmovdqu		\src, \dst
-.elseif VL == 32 && !USE_AVX10
+.elseif VL == 32
 	vbroadcasti128	\src, \dst
 .else
 	vbroadcasti32x4	\src, \dst
@@ -235,16 +252,16 @@
 
 // XOR two vectors together.
 .macro	_vpxor	src1, src2, dst
-.if USE_AVX10
-	vpxord		\src1, \src2, \dst
-.else
+.if VL < 64
 	vpxor		\src1, \src2, \dst
+.else
+	vpxord		\src1, \src2, \dst
 .endif
 .endm
 
 // XOR three vectors together.
 .macro	_xor3	src1, src2, src3_and_dst
-.if USE_AVX10
+.if USE_AVX512
 	// vpternlogd with immediate 0x96 is a three-argument XOR.
 	vpternlogd	$0x96, \src1, \src2, \src3_and_dst
 .else
@@ -259,8 +276,12 @@
 	vpshufd		$0x13, \src, \tmp
 	vpaddq		\src, \src, \dst
 	vpsrad		$31, \tmp, \tmp
+.if USE_AVX512
+	vpternlogd	$0x78, GF_POLY_XMM, \tmp, \dst
+.else
 	vpand		GF_POLY_XMM, \tmp, \tmp
 	vpxor		\tmp, \dst, \dst
+.endif
 .endm
 
 // Given the XTS tweak(s) in the vector \src, compute the next vector of
@@ -284,52 +305,75 @@
 // Given the first XTS tweak at (TWEAK), compute the first set of tweaks and
 // store them in the vector registers TWEAK0-TWEAK3.  Clobbers V0-V5.
 .macro	_compute_first_set_of_tweaks
-	vmovdqu		(TWEAK), TWEAK0_XMM
-	_vbroadcast128	.Lgf_poly(%rip), GF_POLY
 .if VL == 16
-	// With VL=16, multiplying by x serially is fastest.
+	vmovdqu		(TWEAK), TWEAK0_XMM
+	vmovdqu		.Lgf_poly(%rip), GF_POLY
 	_next_tweak	TWEAK0, %xmm0, TWEAK1
 	_next_tweak	TWEAK1, %xmm0, TWEAK2
 	_next_tweak	TWEAK2, %xmm0, TWEAK3
-.else
-.if VL == 32
-	// Compute the second block of TWEAK0.
+.elseif VL == 32
+	vmovdqu		(TWEAK), TWEAK0_XMM
+	vbroadcasti128	.Lgf_poly(%rip), GF_POLY
+
+	// Compute the first vector of tweaks.
 	_next_tweak	TWEAK0_XMM, %xmm0, %xmm1
 	vinserti128	$1, %xmm1, TWEAK0, TWEAK0
-.elseif VL == 64
-	// Compute the remaining blocks of TWEAK0.
-	_next_tweak	TWEAK0_XMM, %xmm0, %xmm1
-	_next_tweak	%xmm1, %xmm0, %xmm2
-	_next_tweak	%xmm2, %xmm0, %xmm3
-	vinserti32x4	$1, %xmm1, TWEAK0, TWEAK0
-	vinserti32x4	$2, %xmm2, TWEAK0, TWEAK0
-	vinserti32x4	$3, %xmm3, TWEAK0, TWEAK0
-.endif
-	// Compute TWEAK[1-3] from TWEAK0.
-	vpsrlq		$64 - 1*VL/16, TWEAK0, V0
-	vpsrlq		$64 - 2*VL/16, TWEAK0, V2
-	vpsrlq		$64 - 3*VL/16, TWEAK0, V4
+
+	// Compute the next three vectors of tweaks:
+	//	TWEAK1 = TWEAK0 * [x^2, x^2]
+	//	TWEAK2 = TWEAK0 * [x^4, x^4]
+	//	TWEAK3 = TWEAK0 * [x^6, x^6]
+	vpsrlq		$64 - 2, TWEAK0, V0
+	vpsrlq		$64 - 4, TWEAK0, V2
+	vpsrlq		$64 - 6, TWEAK0, V4
 	vpclmulqdq	$0x01, GF_POLY, V0, V1
 	vpclmulqdq	$0x01, GF_POLY, V2, V3
 	vpclmulqdq	$0x01, GF_POLY, V4, V5
 	vpslldq		$8, V0, V0
 	vpslldq		$8, V2, V2
 	vpslldq		$8, V4, V4
-	vpsllq		$1*VL/16, TWEAK0, TWEAK1
-	vpsllq		$2*VL/16, TWEAK0, TWEAK2
-	vpsllq		$3*VL/16, TWEAK0, TWEAK3
-.if USE_AVX10
-	vpternlogd	$0x96, V0, V1, TWEAK1
-	vpternlogd	$0x96, V2, V3, TWEAK2
-	vpternlogd	$0x96, V4, V5, TWEAK3
-.else
+	vpsllq		$2, TWEAK0, TWEAK1
+	vpsllq		$4, TWEAK0, TWEAK2
+	vpsllq		$6, TWEAK0, TWEAK3
 	vpxor		V0, TWEAK1, TWEAK1
 	vpxor		V2, TWEAK2, TWEAK2
 	vpxor		V4, TWEAK3, TWEAK3
 	vpxor		V1, TWEAK1, TWEAK1
 	vpxor		V3, TWEAK2, TWEAK2
 	vpxor		V5, TWEAK3, TWEAK3
-.endif
+.else
+	vbroadcasti32x4	(TWEAK), TWEAK0
+	vbroadcasti32x4	.Lgf_poly(%rip), GF_POLY
+
+	// Compute the first vector of tweaks:
+	//	TWEAK0 = broadcast128(TWEAK) * [x^0, x^1, x^2, x^3]
+	vpmovzxbq	.Lrshift_amounts(%rip), V4
+	vpsrlvq		V4, TWEAK0, V0
+	vpclmulqdq	$0x01, GF_POLY, V0, V1
+	vpmovzxbq	.Llshift_amounts(%rip), V4
+	vpslldq		$8, V0, V0
+	vpsllvq		V4, TWEAK0, TWEAK0
+	vpternlogd	$0x96, V0, V1, TWEAK0
+
+	// Compute the next three vectors of tweaks:
+	//	TWEAK1 = TWEAK0 * [x^4, x^4, x^4, x^4]
+	//	TWEAK2 = TWEAK0 * [x^8, x^8, x^8, x^8]
+	//	TWEAK3 = TWEAK0 * [x^12, x^12, x^12, x^12]
+	// x^8 only needs byte-aligned shifts, so optimize accordingly.
+	vpsrlq		$64 - 4, TWEAK0, V0
+	vpsrldq		$(64 - 8) / 8, TWEAK0, V2
+	vpsrlq		$64 - 12, TWEAK0, V4
+	vpclmulqdq	$0x01, GF_POLY, V0, V1
+	vpclmulqdq	$0x01, GF_POLY, V2, V3
+	vpclmulqdq	$0x01, GF_POLY, V4, V5
+	vpslldq		$8, V0, V0
+	vpslldq		$8, V4, V4
+	vpsllq		$4, TWEAK0, TWEAK1
+	vpslldq		$8 / 8, TWEAK0, TWEAK2
+	vpsllq		$12, TWEAK0, TWEAK3
+	vpternlogd	$0x96, V0, V1, TWEAK1
+	vpxord		V3, TWEAK2, TWEAK2
+	vpternlogd	$0x96, V4, V5, TWEAK3
 .endif
 .endm
 
@@ -369,9 +413,14 @@
 
 // Do one step in computing the next set of tweaks using the VPCLMULQDQ method
 // (the same method _next_tweakvec uses for VL > 16).  This means multiplying
-// each tweak by x^(4*VL/16) independently.  Since 4*VL/16 is a multiple of 8
-// when VL > 16 (which it is here), the needed shift amounts are byte-aligned,
-// which allows the use of vpsrldq and vpslldq to do 128-bit wide shifts.
+// each tweak by x^(4*VL/16) independently.
+//
+// Since 4*VL/16 is a multiple of 8 when VL > 16 (which it is here), the needed
+// shift amounts are byte-aligned, which allows the use of vpsrldq and vpslldq
+// to do 128-bit wide shifts.  The 128-bit left shift (vpslldq) saves
+// instructions directly.  The 128-bit right shift (vpsrldq) performs better
+// than a 64-bit right shift on Intel CPUs in the context where it is used here,
+// because it runs on a different execution port from the AES instructions.
 .macro	_tweak_step_pclmul	i
 .if \i == 0
 	vpsrldq		$(128 - 4*VL/16) / 8, TWEAK0, NEXT_TWEAK0
@@ -406,7 +455,7 @@
 // \i that include at least 0 through 19, then 1000 which signals the last step.
 //
 // This is used to interleave the computation of the next set of tweaks with the
-// AES en/decryptions, which increases performance in some cases.
+// AES en/decryptions, which increases performance in some cases.  Clobbers V5.
 .macro	_tweak_step	i
 .if VL == 16
 	_tweak_step_mulx	\i
@@ -443,119 +492,124 @@
 	// the last round needs different instructions.
 	//
 	// An alternative approach would be to roll up all the round loops.  We
-	// don't do that because it isn't compatible with caching the round keys
-	// in registers which we do when possible (see below), and also because
-	// it seems unwise to rely *too* heavily on the CPU's branch predictor.
+	// don't do that because (a) it isn't compatible with caching the round
+	// keys in registers which we do when possible (see below), (b) we
+	// interleave the AES rounds with the XTS tweak computation, and (c) it
+	// seems unwise to rely *too* heavily on the CPU's branch predictor.
 	lea		OFFS-16(KEY, KEYLEN64, 4), KEY
 
 	// If all 32 SIMD registers are available, cache all the round keys.
-.if USE_AVX10
+.if USE_AVX512
 	cmp		$24, KEYLEN
 	jl		.Laes128\@
 	je		.Laes192\@
-	_vbroadcast128	-6*16(KEY), KEY1
-	_vbroadcast128	-5*16(KEY), KEY2
+	vbroadcasti32x4	-6*16(KEY), KEY1
+	vbroadcasti32x4	-5*16(KEY), KEY2
 .Laes192\@:
-	_vbroadcast128	-4*16(KEY), KEY3
-	_vbroadcast128	-3*16(KEY), KEY4
+	vbroadcasti32x4	-4*16(KEY), KEY3
+	vbroadcasti32x4	-3*16(KEY), KEY4
 .Laes128\@:
-	_vbroadcast128	-2*16(KEY), KEY5
-	_vbroadcast128	-1*16(KEY), KEY6
-	_vbroadcast128	0*16(KEY), KEY7
-	_vbroadcast128	1*16(KEY), KEY8
-	_vbroadcast128	2*16(KEY), KEY9
-	_vbroadcast128	3*16(KEY), KEY10
-	_vbroadcast128	4*16(KEY), KEY11
-	_vbroadcast128	5*16(KEY), KEY12
-	_vbroadcast128	6*16(KEY), KEY13
-	_vbroadcast128	7*16(KEY), KEY14
+	vbroadcasti32x4	-2*16(KEY), KEY5
+	vbroadcasti32x4	-1*16(KEY), KEY6
+	vbroadcasti32x4	0*16(KEY), KEY7
+	vbroadcasti32x4	1*16(KEY), KEY8
+	vbroadcasti32x4	2*16(KEY), KEY9
+	vbroadcasti32x4	3*16(KEY), KEY10
+	vbroadcasti32x4	4*16(KEY), KEY11
+	vbroadcasti32x4	5*16(KEY), KEY12
+	vbroadcasti32x4	6*16(KEY), KEY13
+	vbroadcasti32x4	7*16(KEY), KEY14
 .endif
 .endm
 
-// Do a single round of AES encryption (if \enc==1) or decryption (if \enc==0)
-// on the block(s) in \data using the round key(s) in \key.  The register length
-// determines the number of AES blocks en/decrypted.
-.macro	_vaes	enc, last, key, data
+// Do a single non-last round of AES encryption (if \enc==1) or decryption (if
+// \enc==0) on the block(s) in \data using the round key(s) in \key.  The
+// register length determines the number of AES blocks en/decrypted.
+.macro	_vaes	enc, key, data
 .if \enc
-.if \last
-	vaesenclast	\key, \data, \data
-.else
 	vaesenc		\key, \data, \data
-.endif
-.else
-.if \last
-	vaesdeclast	\key, \data, \data
 .else
 	vaesdec		\key, \data, \data
 .endif
+.endm
+
+// Same as _vaes, but does the last round.
+.macro	_vaeslast	enc, key, data
+.if \enc
+	vaesenclast	\key, \data, \data
+.else
+	vaesdeclast	\key, \data, \data
 .endif
 .endm
 
-// Do a single round of AES en/decryption on the block(s) in \data, using the
-// same key for all block(s).  The round key is loaded from the appropriate
-// register or memory location for round \i.  May clobber V4.
-.macro _vaes_1x		enc, last, i, xmm_suffix, data
-.if USE_AVX10
-	_vaes		\enc, \last, KEY\i\xmm_suffix, \data
+// Do a single non-last round of AES en/decryption on the block(s) in \data,
+// using the same key for all block(s).  The round key is loaded from the
+// appropriate register or memory location for round \i.  May clobber \tmp.
+.macro _vaes_1x		enc, i, xmm_suffix, data, tmp
+.if USE_AVX512
+	_vaes		\enc, KEY\i\xmm_suffix, \data
 .else
 .ifnb \xmm_suffix
-	_vaes		\enc, \last, (\i-7)*16(KEY), \data
+	_vaes		\enc, (\i-7)*16(KEY), \data
 .else
-	_vbroadcast128	(\i-7)*16(KEY), V4
-	_vaes		\enc, \last, V4, \data
+	_vbroadcast128	(\i-7)*16(KEY), \tmp
+	_vaes		\enc, \tmp, \data
 .endif
 .endif
 .endm
 
-// Do a single round of AES en/decryption on the blocks in registers V0-V3,
-// using the same key for all blocks.  The round key is loaded from the
+// Do a single non-last round of AES en/decryption on the blocks in registers
+// V0-V3, using the same key for all blocks.  The round key is loaded from the
 // appropriate register or memory location for round \i.  In addition, does two
-// steps of the computation of the next set of tweaks.  May clobber V4.
-.macro	_vaes_4x	enc, last, i
-.if USE_AVX10
+// steps of the computation of the next set of tweaks.  May clobber V4 and V5.
+.macro	_vaes_4x	enc, i
+.if USE_AVX512
 	_tweak_step	(2*(\i-5))
-	_vaes		\enc, \last, KEY\i, V0
-	_vaes		\enc, \last, KEY\i, V1
+	_vaes		\enc, KEY\i, V0
+	_vaes		\enc, KEY\i, V1
 	_tweak_step	(2*(\i-5) + 1)
-	_vaes		\enc, \last, KEY\i, V2
-	_vaes		\enc, \last, KEY\i, V3
+	_vaes		\enc, KEY\i, V2
+	_vaes		\enc, KEY\i, V3
 .else
 	_vbroadcast128	(\i-7)*16(KEY), V4
 	_tweak_step	(2*(\i-5))
-	_vaes		\enc, \last, V4, V0
-	_vaes		\enc, \last, V4, V1
+	_vaes		\enc, V4, V0
+	_vaes		\enc, V4, V1
 	_tweak_step	(2*(\i-5) + 1)
-	_vaes		\enc, \last, V4, V2
-	_vaes		\enc, \last, V4, V3
+	_vaes		\enc, V4, V2
+	_vaes		\enc, V4, V3
 .endif
 .endm
 
 // Do tweaked AES en/decryption (i.e., XOR with \tweak, then AES en/decrypt,
 // then XOR with \tweak again) of the block(s) in \data.  To process a single
 // block, use xmm registers and set \xmm_suffix=_XMM.  To process a vector of
-// length VL, use V* registers and leave \xmm_suffix empty.  May clobber V4.
-.macro	_aes_crypt	enc, xmm_suffix, tweak, data
+// length VL, use V* registers and leave \xmm_suffix empty.  Clobbers \tmp.
+.macro	_aes_crypt	enc, xmm_suffix, tweak, data, tmp
 	_xor3		KEY0\xmm_suffix, \tweak, \data
 	cmp		$24, KEYLEN
 	jl		.Laes128\@
 	je		.Laes192\@
-	_vaes_1x	\enc, 0, 1, \xmm_suffix, \data
-	_vaes_1x	\enc, 0, 2, \xmm_suffix, \data
+	_vaes_1x	\enc, 1, \xmm_suffix, \data, tmp=\tmp
+	_vaes_1x	\enc, 2, \xmm_suffix, \data, tmp=\tmp
 .Laes192\@:
-	_vaes_1x	\enc, 0, 3, \xmm_suffix, \data
-	_vaes_1x	\enc, 0, 4, \xmm_suffix, \data
+	_vaes_1x	\enc, 3, \xmm_suffix, \data, tmp=\tmp
+	_vaes_1x	\enc, 4, \xmm_suffix, \data, tmp=\tmp
 .Laes128\@:
-	_vaes_1x	\enc, 0, 5, \xmm_suffix, \data
-	_vaes_1x	\enc, 0, 6, \xmm_suffix, \data
-	_vaes_1x	\enc, 0, 7, \xmm_suffix, \data
-	_vaes_1x	\enc, 0, 8, \xmm_suffix, \data
-	_vaes_1x	\enc, 0, 9, \xmm_suffix, \data
-	_vaes_1x	\enc, 0, 10, \xmm_suffix, \data
-	_vaes_1x	\enc, 0, 11, \xmm_suffix, \data
-	_vaes_1x	\enc, 0, 12, \xmm_suffix, \data
-	_vaes_1x	\enc, 0, 13, \xmm_suffix, \data
-	_vaes_1x	\enc, 1, 14, \xmm_suffix, \data
-	_vpxor		\tweak, \data, \data
+.irp i, 5,6,7,8,9,10,11,12,13
+	_vaes_1x	\enc, \i, \xmm_suffix, \data, tmp=\tmp
+.endr
+.if USE_AVX512
+	vpxord		KEY14\xmm_suffix, \tweak, \tmp
+.else
+.ifnb \xmm_suffix
+	vpxor		7*16(KEY), \tweak, \tmp
+.else
+	_vbroadcast128	7*16(KEY), \tmp
+	vpxor		\tweak, \tmp, \tmp
+.endif
+.endif
+	_vaeslast	\enc, \tmp, \data
 .endm
 
 .macro	_aes_xts_crypt	enc
@@ -581,14 +635,14 @@
 	// Compute the first set of tweaks TWEAK[0-3].
 	_compute_first_set_of_tweaks
 
-	sub		$4*VL, LEN
+	add		$-4*VL, LEN  // shorter than 'sub 4*VL' when VL=32
 	jl		.Lhandle_remainder\@
 
 .Lmain_loop\@:
 	// This is the main loop, en/decrypting 4*VL bytes per iteration.
 
 	// XOR each source block with its tweak and the zero-th round key.
-.if USE_AVX10
+.if USE_AVX512
 	vmovdqu8	0*VL(SRC), V0
 	vmovdqu8	1*VL(SRC), V1
 	vmovdqu8	2*VL(SRC), V2
@@ -612,28 +666,43 @@
 	je		.Laes192\@
 	// Do all the AES rounds on the data blocks, interleaved with
 	// the computation of the next set of tweaks.
-	_vaes_4x	\enc, 0, 1
-	_vaes_4x	\enc, 0, 2
+	_vaes_4x	\enc, 1
+	_vaes_4x	\enc, 2
 .Laes192\@:
-	_vaes_4x	\enc, 0, 3
-	_vaes_4x	\enc, 0, 4
+	_vaes_4x	\enc, 3
+	_vaes_4x	\enc, 4
 .Laes128\@:
-	_vaes_4x	\enc, 0, 5
-	_vaes_4x	\enc, 0, 6
-	_vaes_4x	\enc, 0, 7
-	_vaes_4x	\enc, 0, 8
-	_vaes_4x	\enc, 0, 9
-	_vaes_4x	\enc, 0, 10
-	_vaes_4x	\enc, 0, 11
-	_vaes_4x	\enc, 0, 12
-	_vaes_4x	\enc, 0, 13
-	_vaes_4x	\enc, 1, 14
-
-	// XOR in the tweaks again.
-	_vpxor		TWEAK0, V0, V0
-	_vpxor		TWEAK1, V1, V1
-	_vpxor		TWEAK2, V2, V2
-	_vpxor		TWEAK3, V3, V3
+.irp i, 5,6,7,8,9,10,11,12,13
+	_vaes_4x	\enc, \i
+.endr
+	// Do the last AES round, then XOR the results with the tweaks again.
+	// Reduce latency by doing the XOR before the vaesenclast, utilizing the
+	// property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a)
+	// (and likewise for vaesdeclast).
+.if USE_AVX512
+	_tweak_step	18
+	_tweak_step	19
+	vpxord		TWEAK0, KEY14, V4
+	vpxord		TWEAK1, KEY14, V5
+	_vaeslast	\enc, V4, V0
+	_vaeslast	\enc, V5, V1
+	vpxord		TWEAK2, KEY14, V4
+	vpxord		TWEAK3, KEY14, V5
+	_vaeslast	\enc, V4, V2
+	_vaeslast	\enc, V5, V3
+.else
+	_vbroadcast128	7*16(KEY), V4
+	_tweak_step	18 // uses V5
+	_tweak_step	19 // uses V5
+	vpxor		TWEAK0, V4, V5
+	_vaeslast	\enc, V5, V0
+	vpxor		TWEAK1, V4, V5
+	_vaeslast	\enc, V5, V1
+	vpxor		TWEAK2, V4, V5
+	vpxor		TWEAK3, V4, V4
+	_vaeslast	\enc, V5, V2
+	_vaeslast	\enc, V4, V3
+.endif
 
 	// Store the destination blocks.
 	_vmovdqu	V0, 0*VL(DST)
@@ -644,9 +713,9 @@
 	// Finish computing the next set of tweaks.
 	_tweak_step	1000
 
-	add		$4*VL, SRC
-	add		$4*VL, DST
-	sub		$4*VL, LEN
+	sub		$-4*VL, SRC  // shorter than 'add 4*VL' when VL=32
+	sub		$-4*VL, DST
+	add		$-4*VL, LEN
 	jge		.Lmain_loop\@
 
 	// Check for the uncommon case where the data length isn't a multiple of
@@ -670,7 +739,7 @@
 	jl		.Lvec_at_a_time_done\@
 .Lvec_at_a_time\@:
 	_vmovdqu	(SRC), V0
-	_aes_crypt	\enc, , TWEAK0, V0
+	_aes_crypt	\enc, , TWEAK0, V0, tmp=V1
 	_vmovdqu	V0, (DST)
 	_next_tweakvec	TWEAK0, V0, V1, TWEAK0
 	add		$VL, SRC
@@ -687,7 +756,7 @@
 	jl		.Lblock_at_a_time_done\@
 .Lblock_at_a_time\@:
 	vmovdqu		(SRC), %xmm0
-	_aes_crypt	\enc, _XMM, TWEAK0_XMM, %xmm0
+	_aes_crypt	\enc, _XMM, TWEAK0_XMM, %xmm0, tmp=%xmm1
 	vmovdqu		%xmm0, (DST)
 	_next_tweak	TWEAK0_XMM, %xmm0, TWEAK0_XMM
 	add		$16, SRC
@@ -715,10 +784,10 @@
 	// Do it now by advancing the tweak and decrypting the last full block.
 	_next_tweak	TWEAK0_XMM, %xmm0, TWEAK1_XMM
 	vmovdqu		(SRC), %xmm0
-	_aes_crypt	\enc, _XMM, TWEAK1_XMM, %xmm0
+	_aes_crypt	\enc, _XMM, TWEAK1_XMM, %xmm0, tmp=%xmm1
 .endif
 
-.if USE_AVX10
+.if USE_AVX512
 	// Create a mask that has the first LEN bits set.
 	mov		$-1, %r9d
 	bzhi		LEN, %r9d, %r9d
@@ -758,47 +827,49 @@
 	vpblendvb	%xmm3, %xmm0, %xmm1, %xmm0
 .endif
 	// En/decrypt again and store the last full block.
-	_aes_crypt	\enc, _XMM, TWEAK0_XMM, %xmm0
+	_aes_crypt	\enc, _XMM, TWEAK0_XMM, %xmm0, tmp=%xmm1
 	vmovdqu		%xmm0, (DST)
 	jmp		.Ldone\@
 .endm
 
 // void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
 //			   u8 iv[AES_BLOCK_SIZE]);
+//
+// Encrypt |iv| using the AES key |tweak_key| to get the first tweak.  Assumes
+// that the CPU supports AES-NI and AVX, but not necessarily VAES or AVX512.
 SYM_TYPED_FUNC_START(aes_xts_encrypt_iv)
-	vmovdqu		(%rsi), %xmm0
-	vpxor		(%rdi), %xmm0, %xmm0
-	movl		480(%rdi), %eax		// AES key length
-	lea		-16(%rdi, %rax, 4), %rdi
-	cmp		$24, %eax
+	.set	TWEAK_KEY,	%rdi
+	.set	IV,		%rsi
+	.set	KEYLEN,		%eax
+	.set	KEYLEN64,	%rax
+
+	vmovdqu		(IV), %xmm0
+	vpxor		(TWEAK_KEY), %xmm0, %xmm0
+	movl		480(TWEAK_KEY), KEYLEN
+	lea		-16(TWEAK_KEY, KEYLEN64, 4), TWEAK_KEY
+	cmp		$24, KEYLEN
 	jl		.Lencrypt_iv_aes128
 	je		.Lencrypt_iv_aes192
-	vaesenc		-6*16(%rdi), %xmm0, %xmm0
-	vaesenc		-5*16(%rdi), %xmm0, %xmm0
+	vaesenc		-6*16(TWEAK_KEY), %xmm0, %xmm0
+	vaesenc		-5*16(TWEAK_KEY), %xmm0, %xmm0
 .Lencrypt_iv_aes192:
-	vaesenc		-4*16(%rdi), %xmm0, %xmm0
-	vaesenc		-3*16(%rdi), %xmm0, %xmm0
+	vaesenc		-4*16(TWEAK_KEY), %xmm0, %xmm0
+	vaesenc		-3*16(TWEAK_KEY), %xmm0, %xmm0
 .Lencrypt_iv_aes128:
-	vaesenc		-2*16(%rdi), %xmm0, %xmm0
-	vaesenc		-1*16(%rdi), %xmm0, %xmm0
-	vaesenc		0*16(%rdi), %xmm0, %xmm0
-	vaesenc		1*16(%rdi), %xmm0, %xmm0
-	vaesenc		2*16(%rdi), %xmm0, %xmm0
-	vaesenc		3*16(%rdi), %xmm0, %xmm0
-	vaesenc		4*16(%rdi), %xmm0, %xmm0
-	vaesenc		5*16(%rdi), %xmm0, %xmm0
-	vaesenc		6*16(%rdi), %xmm0, %xmm0
-	vaesenclast	7*16(%rdi), %xmm0, %xmm0
-	vmovdqu		%xmm0, (%rsi)
+.irp i, -2,-1,0,1,2,3,4,5,6
+	vaesenc		\i*16(TWEAK_KEY), %xmm0, %xmm0
+.endr
+	vaesenclast	7*16(TWEAK_KEY), %xmm0, %xmm0
+	vmovdqu		%xmm0, (IV)
 	RET
 SYM_FUNC_END(aes_xts_encrypt_iv)
 
 // Below are the actual AES-XTS encryption and decryption functions,
 // instantiated from the above macro.  They all have the following prototype:
 //
-// void (*xts_asm_func)(const struct crypto_aes_ctx *key,
-//			const u8 *src, u8 *dst, unsigned int len,
-//			u8 tweak[AES_BLOCK_SIZE]);
+// void (*xts_crypt_func)(const struct crypto_aes_ctx *key,
+//			  const u8 *src, u8 *dst, int len,
+//			  u8 tweak[AES_BLOCK_SIZE]);
 //
 // |key| is the data key.  |tweak| contains the next tweak; the encryption of
 // the original IV with the tweak key was already done.  This function supports
@@ -807,7 +878,7 @@ SYM_FUNC_END(aes_xts_encrypt_iv)
 // multiple of 16, then this function updates |tweak| to contain the next tweak.
 
 .set	VL, 16
-.set	USE_AVX10, 0
+.set	USE_AVX512, 0
 SYM_TYPED_FUNC_START(aes_xts_encrypt_aesni_avx)
 	_aes_xts_crypt	1
 SYM_FUNC_END(aes_xts_encrypt_aesni_avx)
@@ -815,9 +886,8 @@ SYM_TYPED_FUNC_START(aes_xts_decrypt_aesni_avx)
 	_aes_xts_crypt	0
 SYM_FUNC_END(aes_xts_decrypt_aesni_avx)
 
-#if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
 .set	VL, 32
-.set	USE_AVX10, 0
+.set	USE_AVX512, 0
 SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx2)
 	_aes_xts_crypt	1
 SYM_FUNC_END(aes_xts_encrypt_vaes_avx2)
@@ -825,21 +895,11 @@ SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx2)
 	_aes_xts_crypt	0
 SYM_FUNC_END(aes_xts_decrypt_vaes_avx2)
 
-.set	VL, 32
-.set	USE_AVX10, 1
-SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx10_256)
-	_aes_xts_crypt	1
-SYM_FUNC_END(aes_xts_encrypt_vaes_avx10_256)
-SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx10_256)
-	_aes_xts_crypt	0
-SYM_FUNC_END(aes_xts_decrypt_vaes_avx10_256)
-
 .set	VL, 64
-.set	USE_AVX10, 1
-SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx10_512)
+.set	USE_AVX512, 1
+SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx512)
 	_aes_xts_crypt	1
-SYM_FUNC_END(aes_xts_encrypt_vaes_avx10_512)
-SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx10_512)
+SYM_FUNC_END(aes_xts_encrypt_vaes_avx512)
+SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx512)
 	_aes_xts_crypt	0
-SYM_FUNC_END(aes_xts_decrypt_vaes_avx10_512)
-#endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */
+SYM_FUNC_END(aes_xts_decrypt_vaes_avx512)
diff --git a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S
deleted file mode 100644
index 2402b9418cd7..000000000000
--- a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S
+++ /dev/null
@@ -1,597 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */
-/*
- * AES CTR mode by8 optimization with AVX instructions. (x86_64)
- *
- * Copyright(c) 2014 Intel Corporation.
- *
- * Contact Information:
- * James Guilford <james.guilford@intel.com>
- * Sean Gulley <sean.m.gulley@intel.com>
- * Chandramouli Narayanan <mouli@linux.intel.com>
- */
-/*
- * This is AES128/192/256 CTR mode optimization implementation. It requires
- * the support of Intel(R) AESNI and AVX instructions.
- *
- * This work was inspired by the AES CTR mode optimization published
- * in Intel Optimized IPSEC Cryptographic library.
- * Additional information on it can be found at:
- *    https://github.com/intel/intel-ipsec-mb
- */
-
-#include <linux/linkage.h>
-
-#define VMOVDQ		vmovdqu
-
-/*
- * Note: the "x" prefix in these aliases means "this is an xmm register".  The
- * alias prefixes have no relation to XCTR where the "X" prefix means "XOR
- * counter".
- */
-#define xdata0		%xmm0
-#define xdata1		%xmm1
-#define xdata2		%xmm2
-#define xdata3		%xmm3
-#define xdata4		%xmm4
-#define xdata5		%xmm5
-#define xdata6		%xmm6
-#define xdata7		%xmm7
-#define xcounter	%xmm8	// CTR mode only
-#define xiv		%xmm8	// XCTR mode only
-#define xbyteswap	%xmm9	// CTR mode only
-#define xtmp		%xmm9	// XCTR mode only
-#define xkey0		%xmm10
-#define xkey4		%xmm11
-#define xkey8		%xmm12
-#define xkey12		%xmm13
-#define xkeyA		%xmm14
-#define xkeyB		%xmm15
-
-#define p_in		%rdi
-#define p_iv		%rsi
-#define p_keys		%rdx
-#define p_out		%rcx
-#define num_bytes	%r8
-#define counter		%r9	// XCTR mode only
-#define tmp		%r10
-#define	DDQ_DATA	0
-#define	XDATA		1
-#define KEY_128		1
-#define KEY_192		2
-#define KEY_256		3
-
-.section .rodata
-.align 16
-
-byteswap_const:
-	.octa 0x000102030405060708090A0B0C0D0E0F
-ddq_low_msk:
-	.octa 0x0000000000000000FFFFFFFFFFFFFFFF
-ddq_high_add_1:
-	.octa 0x00000000000000010000000000000000
-ddq_add_1:
-	.octa 0x00000000000000000000000000000001
-ddq_add_2:
-	.octa 0x00000000000000000000000000000002
-ddq_add_3:
-	.octa 0x00000000000000000000000000000003
-ddq_add_4:
-	.octa 0x00000000000000000000000000000004
-ddq_add_5:
-	.octa 0x00000000000000000000000000000005
-ddq_add_6:
-	.octa 0x00000000000000000000000000000006
-ddq_add_7:
-	.octa 0x00000000000000000000000000000007
-ddq_add_8:
-	.octa 0x00000000000000000000000000000008
-
-.text
-
-/* generate a unique variable for ddq_add_x */
-
-/* generate a unique variable for xmm register */
-.macro setxdata n
-	var_xdata = %xmm\n
-.endm
-
-/* club the numeric 'id' to the symbol 'name' */
-
-.macro club name, id
-.altmacro
-	.if \name == XDATA
-		setxdata %\id
-	.endif
-.noaltmacro
-.endm
-
-/*
- * do_aes num_in_par load_keys key_len
- * This increments p_in, but not p_out
- */
-.macro do_aes b, k, key_len, xctr
-	.set by, \b
-	.set load_keys, \k
-	.set klen, \key_len
-
-	.if (load_keys)
-		vmovdqa	0*16(p_keys), xkey0
-	.endif
-
-	.if \xctr
-		movq counter, xtmp
-		.set i, 0
-		.rept (by)
-			club XDATA, i
-			vpaddq	(ddq_add_1 + 16 * i)(%rip), xtmp, var_xdata
-			.set i, (i +1)
-		.endr
-		.set i, 0
-		.rept (by)
-			club	XDATA, i
-			vpxor	xiv, var_xdata, var_xdata
-			.set i, (i +1)
-		.endr
-	.else
-		vpshufb	xbyteswap, xcounter, xdata0
-		.set i, 1
-		.rept (by - 1)
-			club XDATA, i
-			vpaddq	(ddq_add_1 + 16 * (i - 1))(%rip), xcounter, var_xdata
-			vptest	ddq_low_msk(%rip), var_xdata
-			jnz 1f
-			vpaddq	ddq_high_add_1(%rip), var_xdata, var_xdata
-			vpaddq	ddq_high_add_1(%rip), xcounter, xcounter
-			1:
-			vpshufb	xbyteswap, var_xdata, var_xdata
-			.set i, (i +1)
-		.endr
-	.endif
-
-	vmovdqa	1*16(p_keys), xkeyA
-
-	vpxor	xkey0, xdata0, xdata0
-	.if \xctr
-		add $by, counter
-	.else
-		vpaddq	(ddq_add_1 + 16 * (by - 1))(%rip), xcounter, xcounter
-		vptest	ddq_low_msk(%rip), xcounter
-		jnz	1f
-		vpaddq	ddq_high_add_1(%rip), xcounter, xcounter
-		1:
-	.endif
-
-	.set i, 1
-	.rept (by - 1)
-		club XDATA, i
-		vpxor	xkey0, var_xdata, var_xdata
-		.set i, (i +1)
-	.endr
-
-	vmovdqa	2*16(p_keys), xkeyB
-
-	.set i, 0
-	.rept by
-		club XDATA, i
-		vaesenc	xkeyA, var_xdata, var_xdata		/* key 1 */
-		.set i, (i +1)
-	.endr
-
-	.if (klen == KEY_128)
-		.if (load_keys)
-			vmovdqa	3*16(p_keys), xkey4
-		.endif
-	.else
-		vmovdqa	3*16(p_keys), xkeyA
-	.endif
-
-	.set i, 0
-	.rept by
-		club XDATA, i
-		vaesenc	xkeyB, var_xdata, var_xdata		/* key 2 */
-		.set i, (i +1)
-	.endr
-
-	add	$(16*by), p_in
-
-	.if (klen == KEY_128)
-		vmovdqa	4*16(p_keys), xkeyB
-	.else
-		.if (load_keys)
-			vmovdqa	4*16(p_keys), xkey4
-		.endif
-	.endif
-
-	.set i, 0
-	.rept by
-		club XDATA, i
-		/* key 3 */
-		.if (klen == KEY_128)
-			vaesenc	xkey4, var_xdata, var_xdata
-		.else
-			vaesenc	xkeyA, var_xdata, var_xdata
-		.endif
-		.set i, (i +1)
-	.endr
-
-	vmovdqa	5*16(p_keys), xkeyA
-
-	.set i, 0
-	.rept by
-		club XDATA, i
-		/* key 4 */
-		.if (klen == KEY_128)
-			vaesenc	xkeyB, var_xdata, var_xdata
-		.else
-			vaesenc	xkey4, var_xdata, var_xdata
-		.endif
-		.set i, (i +1)
-	.endr
-
-	.if (klen == KEY_128)
-		.if (load_keys)
-			vmovdqa	6*16(p_keys), xkey8
-		.endif
-	.else
-		vmovdqa	6*16(p_keys), xkeyB
-	.endif
-
-	.set i, 0
-	.rept by
-		club XDATA, i
-		vaesenc	xkeyA, var_xdata, var_xdata		/* key 5 */
-		.set i, (i +1)
-	.endr
-
-	vmovdqa	7*16(p_keys), xkeyA
-
-	.set i, 0
-	.rept by
-		club XDATA, i
-		/* key 6 */
-		.if (klen == KEY_128)
-			vaesenc	xkey8, var_xdata, var_xdata
-		.else
-			vaesenc	xkeyB, var_xdata, var_xdata
-		.endif
-		.set i, (i +1)
-	.endr
-
-	.if (klen == KEY_128)
-		vmovdqa	8*16(p_keys), xkeyB
-	.else
-		.if (load_keys)
-			vmovdqa	8*16(p_keys), xkey8
-		.endif
-	.endif
-
-	.set i, 0
-	.rept by
-		club XDATA, i
-		vaesenc	xkeyA, var_xdata, var_xdata		/* key 7 */
-		.set i, (i +1)
-	.endr
-
-	.if (klen == KEY_128)
-		.if (load_keys)
-			vmovdqa	9*16(p_keys), xkey12
-		.endif
-	.else
-		vmovdqa	9*16(p_keys), xkeyA
-	.endif
-
-	.set i, 0
-	.rept by
-		club XDATA, i
-		/* key 8 */
-		.if (klen == KEY_128)
-			vaesenc	xkeyB, var_xdata, var_xdata
-		.else
-			vaesenc	xkey8, var_xdata, var_xdata
-		.endif
-		.set i, (i +1)
-	.endr
-
-	vmovdqa	10*16(p_keys), xkeyB
-
-	.set i, 0
-	.rept by
-		club XDATA, i
-		/* key 9 */
-		.if (klen == KEY_128)
-			vaesenc	xkey12, var_xdata, var_xdata
-		.else
-			vaesenc	xkeyA, var_xdata, var_xdata
-		.endif
-		.set i, (i +1)
-	.endr
-
-	.if (klen != KEY_128)
-		vmovdqa	11*16(p_keys), xkeyA
-	.endif
-
-	.set i, 0
-	.rept by
-		club XDATA, i
-		/* key 10 */
-		.if (klen == KEY_128)
-			vaesenclast	xkeyB, var_xdata, var_xdata
-		.else
-			vaesenc	xkeyB, var_xdata, var_xdata
-		.endif
-		.set i, (i +1)
-	.endr
-
-	.if (klen != KEY_128)
-		.if (load_keys)
-			vmovdqa	12*16(p_keys), xkey12
-		.endif
-
-		.set i, 0
-		.rept by
-			club XDATA, i
-			vaesenc	xkeyA, var_xdata, var_xdata	/* key 11 */
-			.set i, (i +1)
-		.endr
-
-		.if (klen == KEY_256)
-			vmovdqa	13*16(p_keys), xkeyA
-		.endif
-
-		.set i, 0
-		.rept by
-			club XDATA, i
-			.if (klen == KEY_256)
-				/* key 12 */
-				vaesenc	xkey12, var_xdata, var_xdata
-			.else
-				vaesenclast xkey12, var_xdata, var_xdata
-			.endif
-			.set i, (i +1)
-		.endr
-
-		.if (klen == KEY_256)
-			vmovdqa	14*16(p_keys), xkeyB
-
-			.set i, 0
-			.rept by
-				club XDATA, i
-				/* key 13 */
-				vaesenc	xkeyA, var_xdata, var_xdata
-				.set i, (i +1)
-			.endr
-
-			.set i, 0
-			.rept by
-				club XDATA, i
-				/* key 14 */
-				vaesenclast	xkeyB, var_xdata, var_xdata
-				.set i, (i +1)
-			.endr
-		.endif
-	.endif
-
-	.set i, 0
-	.rept (by / 2)
-		.set j, (i+1)
-		VMOVDQ	(i*16 - 16*by)(p_in), xkeyA
-		VMOVDQ	(j*16 - 16*by)(p_in), xkeyB
-		club XDATA, i
-		vpxor	xkeyA, var_xdata, var_xdata
-		club XDATA, j
-		vpxor	xkeyB, var_xdata, var_xdata
-		.set i, (i+2)
-	.endr
-
-	.if (i < by)
-		VMOVDQ	(i*16 - 16*by)(p_in), xkeyA
-		club XDATA, i
-		vpxor	xkeyA, var_xdata, var_xdata
-	.endif
-
-	.set i, 0
-	.rept by
-		club XDATA, i
-		VMOVDQ	var_xdata, i*16(p_out)
-		.set i, (i+1)
-	.endr
-.endm
-
-.macro do_aes_load val, key_len, xctr
-	do_aes \val, 1, \key_len, \xctr
-.endm
-
-.macro do_aes_noload val, key_len, xctr
-	do_aes \val, 0, \key_len, \xctr
-.endm
-
-/* main body of aes ctr load */
-
-.macro do_aes_ctrmain key_len, xctr
-	cmp	$16, num_bytes
-	jb	.Ldo_return2\xctr\key_len
-
-	.if \xctr
-		shr	$4, counter
-		vmovdqu	(p_iv), xiv
-	.else
-		vmovdqa	byteswap_const(%rip), xbyteswap
-		vmovdqu	(p_iv), xcounter
-		vpshufb	xbyteswap, xcounter, xcounter
-	.endif
-
-	mov	num_bytes, tmp
-	and	$(7*16), tmp
-	jz	.Lmult_of_8_blks\xctr\key_len
-
-	/* 1 <= tmp <= 7 */
-	cmp	$(4*16), tmp
-	jg	.Lgt4\xctr\key_len
-	je	.Leq4\xctr\key_len
-
-.Llt4\xctr\key_len:
-	cmp	$(2*16), tmp
-	jg	.Leq3\xctr\key_len
-	je	.Leq2\xctr\key_len
-
-.Leq1\xctr\key_len:
-	do_aes_load	1, \key_len, \xctr
-	add	$(1*16), p_out
-	and	$(~7*16), num_bytes
-	jz	.Ldo_return2\xctr\key_len
-	jmp	.Lmain_loop2\xctr\key_len
-
-.Leq2\xctr\key_len:
-	do_aes_load	2, \key_len, \xctr
-	add	$(2*16), p_out
-	and	$(~7*16), num_bytes
-	jz	.Ldo_return2\xctr\key_len
-	jmp	.Lmain_loop2\xctr\key_len
-
-
-.Leq3\xctr\key_len:
-	do_aes_load	3, \key_len, \xctr
-	add	$(3*16), p_out
-	and	$(~7*16), num_bytes
-	jz	.Ldo_return2\xctr\key_len
-	jmp	.Lmain_loop2\xctr\key_len
-
-.Leq4\xctr\key_len:
-	do_aes_load	4, \key_len, \xctr
-	add	$(4*16), p_out
-	and	$(~7*16), num_bytes
-	jz	.Ldo_return2\xctr\key_len
-	jmp	.Lmain_loop2\xctr\key_len
-
-.Lgt4\xctr\key_len:
-	cmp	$(6*16), tmp
-	jg	.Leq7\xctr\key_len
-	je	.Leq6\xctr\key_len
-
-.Leq5\xctr\key_len:
-	do_aes_load	5, \key_len, \xctr
-	add	$(5*16), p_out
-	and	$(~7*16), num_bytes
-	jz	.Ldo_return2\xctr\key_len
-	jmp	.Lmain_loop2\xctr\key_len
-
-.Leq6\xctr\key_len:
-	do_aes_load	6, \key_len, \xctr
-	add	$(6*16), p_out
-	and	$(~7*16), num_bytes
-	jz	.Ldo_return2\xctr\key_len
-	jmp	.Lmain_loop2\xctr\key_len
-
-.Leq7\xctr\key_len:
-	do_aes_load	7, \key_len, \xctr
-	add	$(7*16), p_out
-	and	$(~7*16), num_bytes
-	jz	.Ldo_return2\xctr\key_len
-	jmp	.Lmain_loop2\xctr\key_len
-
-.Lmult_of_8_blks\xctr\key_len:
-	.if (\key_len != KEY_128)
-		vmovdqa	0*16(p_keys), xkey0
-		vmovdqa	4*16(p_keys), xkey4
-		vmovdqa	8*16(p_keys), xkey8
-		vmovdqa	12*16(p_keys), xkey12
-	.else
-		vmovdqa	0*16(p_keys), xkey0
-		vmovdqa	3*16(p_keys), xkey4
-		vmovdqa	6*16(p_keys), xkey8
-		vmovdqa	9*16(p_keys), xkey12
-	.endif
-.align 16
-.Lmain_loop2\xctr\key_len:
-	/* num_bytes is a multiple of 8 and >0 */
-	do_aes_noload	8, \key_len, \xctr
-	add	$(8*16), p_out
-	sub	$(8*16), num_bytes
-	jne	.Lmain_loop2\xctr\key_len
-
-.Ldo_return2\xctr\key_len:
-	.if !\xctr
-		/* return updated IV */
-		vpshufb	xbyteswap, xcounter, xcounter
-		vmovdqu	xcounter, (p_iv)
-	.endif
-	RET
-.endm
-
-/*
- * routine to do AES128 CTR enc/decrypt "by8"
- * XMM registers are clobbered.
- * Saving/restoring must be done at a higher level
- * aes_ctr_enc_128_avx_by8(void *in, void *iv, void *keys, void *out,
- *			unsigned int num_bytes)
- */
-SYM_FUNC_START(aes_ctr_enc_128_avx_by8)
-	/* call the aes main loop */
-	do_aes_ctrmain KEY_128 0
-
-SYM_FUNC_END(aes_ctr_enc_128_avx_by8)
-
-/*
- * routine to do AES192 CTR enc/decrypt "by8"
- * XMM registers are clobbered.
- * Saving/restoring must be done at a higher level
- * aes_ctr_enc_192_avx_by8(void *in, void *iv, void *keys, void *out,
- *			unsigned int num_bytes)
- */
-SYM_FUNC_START(aes_ctr_enc_192_avx_by8)
-	/* call the aes main loop */
-	do_aes_ctrmain KEY_192 0
-
-SYM_FUNC_END(aes_ctr_enc_192_avx_by8)
-
-/*
- * routine to do AES256 CTR enc/decrypt "by8"
- * XMM registers are clobbered.
- * Saving/restoring must be done at a higher level
- * aes_ctr_enc_256_avx_by8(void *in, void *iv, void *keys, void *out,
- *			unsigned int num_bytes)
- */
-SYM_FUNC_START(aes_ctr_enc_256_avx_by8)
-	/* call the aes main loop */
-	do_aes_ctrmain KEY_256 0
-
-SYM_FUNC_END(aes_ctr_enc_256_avx_by8)
-
-/*
- * routine to do AES128 XCTR enc/decrypt "by8"
- * XMM registers are clobbered.
- * Saving/restoring must be done at a higher level
- * aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv, const void *keys,
- * 	u8* out, unsigned int num_bytes, unsigned int byte_ctr)
- */
-SYM_FUNC_START(aes_xctr_enc_128_avx_by8)
-	/* call the aes main loop */
-	do_aes_ctrmain KEY_128 1
-
-SYM_FUNC_END(aes_xctr_enc_128_avx_by8)
-
-/*
- * routine to do AES192 XCTR enc/decrypt "by8"
- * XMM registers are clobbered.
- * Saving/restoring must be done at a higher level
- * aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv, const void *keys,
- * 	u8* out, unsigned int num_bytes, unsigned int byte_ctr)
- */
-SYM_FUNC_START(aes_xctr_enc_192_avx_by8)
-	/* call the aes main loop */
-	do_aes_ctrmain KEY_192 1
-
-SYM_FUNC_END(aes_xctr_enc_192_avx_by8)
-
-/*
- * routine to do AES256 XCTR enc/decrypt "by8"
- * XMM registers are clobbered.
- * Saving/restoring must be done at a higher level
- * aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv, const void *keys,
- * 	u8* out, unsigned int num_bytes, unsigned int byte_ctr)
- */
-SYM_FUNC_START(aes_xctr_enc_256_avx_by8)
-	/* call the aes main loop */
-	do_aes_ctrmain KEY_256 1
-
-SYM_FUNC_END(aes_xctr_enc_256_avx_by8)
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index eb153eff9331..b37881bb9f15 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -17,6 +17,7 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/objtool.h>
 #include <asm/frame.h>
 
 #define STATE1	%xmm0
@@ -1071,6 +1072,7 @@ SYM_FUNC_END(_aesni_inc)
  *		      size_t len, u8 *iv)
  */
 SYM_FUNC_START(aesni_ctr_enc)
+	ANNOTATE_NOENDBR
 	FRAME_BEGIN
 	cmp $16, LEN
 	jb .Lctr_enc_just_ret
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index b0dd83555499..48405e02d6e4 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -23,7 +23,6 @@
 #include <linux/err.h>
 #include <crypto/algapi.h>
 #include <crypto/aes.h>
-#include <crypto/ctr.h>
 #include <crypto/b128ops.h>
 #include <crypto/gcm.h>
 #include <crypto/xts.h>
@@ -82,30 +81,8 @@ asmlinkage void aesni_xts_dec(const struct crypto_aes_ctx *ctx, u8 *out,
 			      const u8 *in, unsigned int len, u8 *iv);
 
 #ifdef CONFIG_X86_64
-
 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 			      const u8 *in, unsigned int len, u8 *iv);
-DEFINE_STATIC_CALL(aesni_ctr_enc_tfm, aesni_ctr_enc);
-
-asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
-		void *keys, u8 *out, unsigned int num_bytes);
-asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
-		void *keys, u8 *out, unsigned int num_bytes);
-asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
-		void *keys, u8 *out, unsigned int num_bytes);
-
-
-asmlinkage void aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv,
-	const void *keys, u8 *out, unsigned int num_bytes,
-	unsigned int byte_ctr);
-
-asmlinkage void aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv,
-	const void *keys, u8 *out, unsigned int num_bytes,
-	unsigned int byte_ctr);
-
-asmlinkage void aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv,
-	const void *keys, u8 *out, unsigned int num_bytes,
-	unsigned int byte_ctr);
 #endif
 
 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
@@ -376,24 +353,8 @@ static int cts_cbc_decrypt(struct skcipher_request *req)
 }
 
 #ifdef CONFIG_X86_64
-static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
-			      const u8 *in, unsigned int len, u8 *iv)
-{
-	/*
-	 * based on key length, override with the by8 version
-	 * of ctr mode encryption/decryption for improved performance
-	 * aes_set_key_common() ensures that key length is one of
-	 * {128,192,256}
-	 */
-	if (ctx->key_length == AES_KEYSIZE_128)
-		aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
-	else if (ctx->key_length == AES_KEYSIZE_192)
-		aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
-	else
-		aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
-}
-
-static int ctr_crypt(struct skcipher_request *req)
+/* This is the non-AVX version. */
+static int ctr_crypt_aesni(struct skcipher_request *req)
 {
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
@@ -407,10 +368,9 @@ static int ctr_crypt(struct skcipher_request *req)
 	while ((nbytes = walk.nbytes) > 0) {
 		kernel_fpu_begin();
 		if (nbytes & AES_BLOCK_MASK)
-			static_call(aesni_ctr_enc_tfm)(ctx, walk.dst.virt.addr,
-						       walk.src.virt.addr,
-						       nbytes & AES_BLOCK_MASK,
-						       walk.iv);
+			aesni_ctr_enc(ctx, walk.dst.virt.addr,
+				      walk.src.virt.addr,
+				      nbytes & AES_BLOCK_MASK, walk.iv);
 		nbytes &= ~AES_BLOCK_MASK;
 
 		if (walk.nbytes == walk.total && nbytes > 0) {
@@ -426,59 +386,6 @@ static int ctr_crypt(struct skcipher_request *req)
 	}
 	return err;
 }
-
-static void aesni_xctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
-				   const u8 *in, unsigned int len, u8 *iv,
-				   unsigned int byte_ctr)
-{
-	if (ctx->key_length == AES_KEYSIZE_128)
-		aes_xctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len,
-					 byte_ctr);
-	else if (ctx->key_length == AES_KEYSIZE_192)
-		aes_xctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len,
-					 byte_ctr);
-	else
-		aes_xctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len,
-					 byte_ctr);
-}
-
-static int xctr_crypt(struct skcipher_request *req)
-{
-	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
-	u8 keystream[AES_BLOCK_SIZE];
-	struct skcipher_walk walk;
-	unsigned int nbytes;
-	unsigned int byte_ctr = 0;
-	int err;
-	__le32 block[AES_BLOCK_SIZE / sizeof(__le32)];
-
-	err = skcipher_walk_virt(&walk, req, false);
-
-	while ((nbytes = walk.nbytes) > 0) {
-		kernel_fpu_begin();
-		if (nbytes & AES_BLOCK_MASK)
-			aesni_xctr_enc_avx_tfm(ctx, walk.dst.virt.addr,
-				walk.src.virt.addr, nbytes & AES_BLOCK_MASK,
-				walk.iv, byte_ctr);
-		nbytes &= ~AES_BLOCK_MASK;
-		byte_ctr += walk.nbytes - nbytes;
-
-		if (walk.nbytes == walk.total && nbytes > 0) {
-			memcpy(block, walk.iv, AES_BLOCK_SIZE);
-			block[0] ^= cpu_to_le32(1 + byte_ctr / AES_BLOCK_SIZE);
-			aesni_enc(ctx, keystream, (u8 *)block);
-			crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes -
-				       nbytes, walk.src.virt.addr + walk.nbytes
-				       - nbytes, keystream, nbytes);
-			byte_ctr += nbytes;
-			nbytes = 0;
-		}
-		kernel_fpu_end();
-		err = skcipher_walk_done(&walk, nbytes);
-	}
-	return err;
-}
 #endif
 
 static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key,
@@ -505,7 +412,7 @@ static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key,
 typedef void (*xts_encrypt_iv_func)(const struct crypto_aes_ctx *tweak_key,
 				    u8 iv[AES_BLOCK_SIZE]);
 typedef void (*xts_crypt_func)(const struct crypto_aes_ctx *key,
-			       const u8 *src, u8 *dst, unsigned int len,
+			       const u8 *src, u8 *dst, int len,
 			       u8 tweak[AES_BLOCK_SIZE]);
 
 /* This handles cases where the source and/or destination span pages. */
@@ -581,11 +488,8 @@ xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv,
 {
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 	const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
-	const unsigned int cryptlen = req->cryptlen;
-	struct scatterlist *src = req->src;
-	struct scatterlist *dst = req->dst;
 
-	if (unlikely(cryptlen < AES_BLOCK_SIZE))
+	if (unlikely(req->cryptlen < AES_BLOCK_SIZE))
 		return -EINVAL;
 
 	kernel_fpu_begin();
@@ -593,23 +497,16 @@ xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv,
 
 	/*
 	 * In practice, virtually all XTS plaintexts and ciphertexts are either
-	 * 512 or 4096 bytes, aligned such that they don't span page boundaries.
-	 * To optimize the performance of these cases, and also any other case
-	 * where no page boundary is spanned, the below fast-path handles
-	 * single-page sources and destinations as efficiently as possible.
+	 * 512 or 4096 bytes and do not use multiple scatterlist elements.  To
+	 * optimize the performance of these cases, the below fast-path handles
+	 * single-scatterlist-element messages as efficiently as possible.  The
+	 * code is 64-bit specific, as it assumes no page mapping is needed.
 	 */
-	if (likely(src->length >= cryptlen && dst->length >= cryptlen &&
-		   src->offset + cryptlen <= PAGE_SIZE &&
-		   dst->offset + cryptlen <= PAGE_SIZE)) {
-		struct page *src_page = sg_page(src);
-		struct page *dst_page = sg_page(dst);
-		void *src_virt = kmap_local_page(src_page) + src->offset;
-		void *dst_virt = kmap_local_page(dst_page) + dst->offset;
-
-		(*crypt_func)(&ctx->crypt_ctx, src_virt, dst_virt, cryptlen,
-			      req->iv);
-		kunmap_local(dst_virt);
-		kunmap_local(src_virt);
+	if (IS_ENABLED(CONFIG_X86_64) &&
+	    likely(req->src->length >= req->cryptlen &&
+		   req->dst->length >= req->cryptlen)) {
+		(*crypt_func)(&ctx->crypt_ctx, sg_virt(req->src),
+			      sg_virt(req->dst), req->cryptlen, req->iv);
 		kernel_fpu_end();
 		return 0;
 	}
@@ -624,14 +521,14 @@ static void aesni_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
 }
 
 static void aesni_xts_encrypt(const struct crypto_aes_ctx *key,
-			      const u8 *src, u8 *dst, unsigned int len,
+			      const u8 *src, u8 *dst, int len,
 			      u8 tweak[AES_BLOCK_SIZE])
 {
 	aesni_xts_enc(key, dst, src, len, tweak);
 }
 
 static void aesni_xts_decrypt(const struct crypto_aes_ctx *key,
-			      const u8 *src, u8 *dst, unsigned int len,
+			      const u8 *src, u8 *dst, int len,
 			      u8 tweak[AES_BLOCK_SIZE])
 {
 	aesni_xts_dec(key, dst, src, len, tweak);
@@ -669,10 +566,9 @@ static struct crypto_alg aesni_cipher_alg = {
 static struct skcipher_alg aesni_skciphers[] = {
 	{
 		.base = {
-			.cra_name		= "__ecb(aes)",
-			.cra_driver_name	= "__ecb-aes-aesni",
+			.cra_name		= "ecb(aes)",
+			.cra_driver_name	= "ecb-aes-aesni",
 			.cra_priority		= 400,
-			.cra_flags		= CRYPTO_ALG_INTERNAL,
 			.cra_blocksize		= AES_BLOCK_SIZE,
 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
 			.cra_module		= THIS_MODULE,
@@ -684,10 +580,9 @@ static struct skcipher_alg aesni_skciphers[] = {
 		.decrypt	= ecb_decrypt,
 	}, {
 		.base = {
-			.cra_name		= "__cbc(aes)",
-			.cra_driver_name	= "__cbc-aes-aesni",
+			.cra_name		= "cbc(aes)",
+			.cra_driver_name	= "cbc-aes-aesni",
 			.cra_priority		= 400,
-			.cra_flags		= CRYPTO_ALG_INTERNAL,
 			.cra_blocksize		= AES_BLOCK_SIZE,
 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
 			.cra_module		= THIS_MODULE,
@@ -700,10 +595,9 @@ static struct skcipher_alg aesni_skciphers[] = {
 		.decrypt	= cbc_decrypt,
 	}, {
 		.base = {
-			.cra_name		= "__cts(cbc(aes))",
-			.cra_driver_name	= "__cts-cbc-aes-aesni",
+			.cra_name		= "cts(cbc(aes))",
+			.cra_driver_name	= "cts-cbc-aes-aesni",
 			.cra_priority		= 400,
-			.cra_flags		= CRYPTO_ALG_INTERNAL,
 			.cra_blocksize		= AES_BLOCK_SIZE,
 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
 			.cra_module		= THIS_MODULE,
@@ -718,10 +612,9 @@ static struct skcipher_alg aesni_skciphers[] = {
 #ifdef CONFIG_X86_64
 	}, {
 		.base = {
-			.cra_name		= "__ctr(aes)",
-			.cra_driver_name	= "__ctr-aes-aesni",
+			.cra_name		= "ctr(aes)",
+			.cra_driver_name	= "ctr-aes-aesni",
 			.cra_priority		= 400,
-			.cra_flags		= CRYPTO_ALG_INTERNAL,
 			.cra_blocksize		= 1,
 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
 			.cra_module		= THIS_MODULE,
@@ -731,15 +624,14 @@ static struct skcipher_alg aesni_skciphers[] = {
 		.ivsize		= AES_BLOCK_SIZE,
 		.chunksize	= AES_BLOCK_SIZE,
 		.setkey		= aesni_skcipher_setkey,
-		.encrypt	= ctr_crypt,
-		.decrypt	= ctr_crypt,
+		.encrypt	= ctr_crypt_aesni,
+		.decrypt	= ctr_crypt_aesni,
 #endif
 	}, {
 		.base = {
-			.cra_name		= "__xts(aes)",
-			.cra_driver_name	= "__xts-aes-aesni",
+			.cra_name		= "xts(aes)",
+			.cra_driver_name	= "xts-aes-aesni",
 			.cra_priority		= 401,
-			.cra_flags		= CRYPTO_ALG_INTERNAL,
 			.cra_blocksize		= AES_BLOCK_SIZE,
 			.cra_ctxsize		= XTS_AES_CTX_SIZE,
 			.cra_module		= THIS_MODULE,
@@ -754,46 +646,112 @@ static struct skcipher_alg aesni_skciphers[] = {
 	}
 };
 
-static
-struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
-
 #ifdef CONFIG_X86_64
-/*
- * XCTR does not have a non-AVX implementation, so it must be enabled
- * conditionally.
- */
-static struct skcipher_alg aesni_xctr = {
-	.base = {
-		.cra_name		= "__xctr(aes)",
-		.cra_driver_name	= "__xctr-aes-aesni",
-		.cra_priority		= 400,
-		.cra_flags		= CRYPTO_ALG_INTERNAL,
-		.cra_blocksize		= 1,
-		.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
-		.cra_module		= THIS_MODULE,
-	},
-	.min_keysize	= AES_MIN_KEY_SIZE,
-	.max_keysize	= AES_MAX_KEY_SIZE,
-	.ivsize		= AES_BLOCK_SIZE,
-	.chunksize	= AES_BLOCK_SIZE,
-	.setkey		= aesni_skcipher_setkey,
-	.encrypt	= xctr_crypt,
-	.decrypt	= xctr_crypt,
-};
-
-static struct simd_skcipher_alg *aesni_simd_xctr;
-
 asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
 				   u8 iv[AES_BLOCK_SIZE]);
 
-#define DEFINE_XTS_ALG(suffix, driver_name, priority)			       \
+/* __always_inline to avoid indirect call */
+static __always_inline int
+ctr_crypt(struct skcipher_request *req,
+	  void (*ctr64_func)(const struct crypto_aes_ctx *key,
+			     const u8 *src, u8 *dst, int len,
+			     const u64 le_ctr[2]))
+{
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm));
+	unsigned int nbytes, p1_nbytes, nblocks;
+	struct skcipher_walk walk;
+	u64 le_ctr[2];
+	u64 ctr64;
+	int err;
+
+	ctr64 = le_ctr[0] = get_unaligned_be64(&req->iv[8]);
+	le_ctr[1] = get_unaligned_be64(&req->iv[0]);
+
+	err = skcipher_walk_virt(&walk, req, false);
+
+	while ((nbytes = walk.nbytes) != 0) {
+		if (nbytes < walk.total) {
+			/* Not the end yet, so keep the length block-aligned. */
+			nbytes = round_down(nbytes, AES_BLOCK_SIZE);
+			nblocks = nbytes / AES_BLOCK_SIZE;
+		} else {
+			/* It's the end, so include any final partial block. */
+			nblocks = DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
+		}
+		ctr64 += nblocks;
+
+		kernel_fpu_begin();
+		if (likely(ctr64 >= nblocks)) {
+			/* The low 64 bits of the counter won't overflow. */
+			(*ctr64_func)(key, walk.src.virt.addr,
+				      walk.dst.virt.addr, nbytes, le_ctr);
+		} else {
+			/*
+			 * The low 64 bits of the counter will overflow.  The
+			 * assembly doesn't handle this case, so split the
+			 * operation into two at the point where the overflow
+			 * will occur.  After the first part, add the carry bit.
+			 */
+			p1_nbytes = min(nbytes, (nblocks - ctr64) * AES_BLOCK_SIZE);
+			(*ctr64_func)(key, walk.src.virt.addr,
+				      walk.dst.virt.addr, p1_nbytes, le_ctr);
+			le_ctr[0] = 0;
+			le_ctr[1]++;
+			(*ctr64_func)(key, walk.src.virt.addr + p1_nbytes,
+				      walk.dst.virt.addr + p1_nbytes,
+				      nbytes - p1_nbytes, le_ctr);
+		}
+		kernel_fpu_end();
+		le_ctr[0] = ctr64;
+
+		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
+	}
+
+	put_unaligned_be64(ctr64, &req->iv[8]);
+	put_unaligned_be64(le_ctr[1], &req->iv[0]);
+
+	return err;
+}
+
+/* __always_inline to avoid indirect call */
+static __always_inline int
+xctr_crypt(struct skcipher_request *req,
+	   void (*xctr_func)(const struct crypto_aes_ctx *key,
+			     const u8 *src, u8 *dst, int len,
+			     const u8 iv[AES_BLOCK_SIZE], u64 ctr))
+{
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm));
+	struct skcipher_walk walk;
+	unsigned int nbytes;
+	u64 ctr = 1;
+	int err;
+
+	err = skcipher_walk_virt(&walk, req, false);
+	while ((nbytes = walk.nbytes) != 0) {
+		if (nbytes < walk.total)
+			nbytes = round_down(nbytes, AES_BLOCK_SIZE);
+
+		kernel_fpu_begin();
+		(*xctr_func)(key, walk.src.virt.addr, walk.dst.virt.addr,
+			     nbytes, req->iv, ctr);
+		kernel_fpu_end();
+
+		ctr += DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
+	}
+	return err;
+}
+
+#define DEFINE_AVX_SKCIPHER_ALGS(suffix, driver_name_suffix, priority)	       \
 									       \
 asmlinkage void								       \
 aes_xts_encrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src,      \
-			 u8 *dst, unsigned int len, u8 tweak[AES_BLOCK_SIZE]); \
+			 u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]);	       \
 asmlinkage void								       \
 aes_xts_decrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src,      \
-			 u8 *dst, unsigned int len, u8 tweak[AES_BLOCK_SIZE]); \
+			 u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]);	       \
 									       \
 static int xts_encrypt_##suffix(struct skcipher_request *req)		       \
 {									       \
@@ -805,33 +763,72 @@ static int xts_decrypt_##suffix(struct skcipher_request *req)		       \
 	return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_decrypt_##suffix);   \
 }									       \
 									       \
-static struct skcipher_alg aes_xts_alg_##suffix = {			       \
-	.base = {							       \
-		.cra_name		= "__xts(aes)",			       \
-		.cra_driver_name	= "__" driver_name,		       \
-		.cra_priority		= priority,			       \
-		.cra_flags		= CRYPTO_ALG_INTERNAL,		       \
-		.cra_blocksize		= AES_BLOCK_SIZE,		       \
-		.cra_ctxsize		= XTS_AES_CTX_SIZE,		       \
-		.cra_module		= THIS_MODULE,			       \
-	},								       \
-	.min_keysize	= 2 * AES_MIN_KEY_SIZE,				       \
-	.max_keysize	= 2 * AES_MAX_KEY_SIZE,				       \
-	.ivsize		= AES_BLOCK_SIZE,				       \
-	.walksize	= 2 * AES_BLOCK_SIZE,				       \
-	.setkey		= xts_setkey_aesni,				       \
-	.encrypt	= xts_encrypt_##suffix,				       \
-	.decrypt	= xts_decrypt_##suffix,				       \
-};									       \
+asmlinkage void								       \
+aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key,		       \
+			 const u8 *src, u8 *dst, int len, const u64 le_ctr[2]);\
+									       \
+static int ctr_crypt_##suffix(struct skcipher_request *req)		       \
+{									       \
+	return ctr_crypt(req, aes_ctr64_crypt_##suffix);		       \
+}									       \
+									       \
+asmlinkage void								       \
+aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key,		       \
+			const u8 *src, u8 *dst, int len,		       \
+			const u8 iv[AES_BLOCK_SIZE], u64 ctr);		       \
+									       \
+static int xctr_crypt_##suffix(struct skcipher_request *req)		       \
+{									       \
+	return xctr_crypt(req, aes_xctr_crypt_##suffix);		       \
+}									       \
 									       \
-static struct simd_skcipher_alg *aes_xts_simdalg_##suffix
+static struct skcipher_alg skcipher_algs_##suffix[] = {{		       \
+	.base.cra_name		= "xts(aes)",				       \
+	.base.cra_driver_name	= "xts-aes-" driver_name_suffix,	       \
+	.base.cra_priority	= priority,				       \
+	.base.cra_blocksize	= AES_BLOCK_SIZE,			       \
+	.base.cra_ctxsize	= XTS_AES_CTX_SIZE,			       \
+	.base.cra_module	= THIS_MODULE,				       \
+	.min_keysize		= 2 * AES_MIN_KEY_SIZE,			       \
+	.max_keysize		= 2 * AES_MAX_KEY_SIZE,			       \
+	.ivsize			= AES_BLOCK_SIZE,			       \
+	.walksize		= 2 * AES_BLOCK_SIZE,			       \
+	.setkey			= xts_setkey_aesni,			       \
+	.encrypt		= xts_encrypt_##suffix,			       \
+	.decrypt		= xts_decrypt_##suffix,			       \
+}, {									       \
+	.base.cra_name		= "ctr(aes)",				       \
+	.base.cra_driver_name	= "ctr-aes-" driver_name_suffix,	       \
+	.base.cra_priority	= priority,				       \
+	.base.cra_blocksize	= 1,					       \
+	.base.cra_ctxsize	= CRYPTO_AES_CTX_SIZE,			       \
+	.base.cra_module	= THIS_MODULE,				       \
+	.min_keysize		= AES_MIN_KEY_SIZE,			       \
+	.max_keysize		= AES_MAX_KEY_SIZE,			       \
+	.ivsize			= AES_BLOCK_SIZE,			       \
+	.chunksize		= AES_BLOCK_SIZE,			       \
+	.setkey			= aesni_skcipher_setkey,		       \
+	.encrypt		= ctr_crypt_##suffix,			       \
+	.decrypt		= ctr_crypt_##suffix,			       \
+}, {									       \
+	.base.cra_name		= "xctr(aes)",				       \
+	.base.cra_driver_name	= "xctr-aes-" driver_name_suffix,	       \
+	.base.cra_priority	= priority,				       \
+	.base.cra_blocksize	= 1,					       \
+	.base.cra_ctxsize	= CRYPTO_AES_CTX_SIZE,			       \
+	.base.cra_module	= THIS_MODULE,				       \
+	.min_keysize		= AES_MIN_KEY_SIZE,			       \
+	.max_keysize		= AES_MAX_KEY_SIZE,			       \
+	.ivsize			= AES_BLOCK_SIZE,			       \
+	.chunksize		= AES_BLOCK_SIZE,			       \
+	.setkey			= aesni_skcipher_setkey,		       \
+	.encrypt		= xctr_crypt_##suffix,			       \
+	.decrypt		= xctr_crypt_##suffix,			       \
+}}
 
-DEFINE_XTS_ALG(aesni_avx, "xts-aes-aesni-avx", 500);
-#if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
-DEFINE_XTS_ALG(vaes_avx2, "xts-aes-vaes-avx2", 600);
-DEFINE_XTS_ALG(vaes_avx10_256, "xts-aes-vaes-avx10_256", 700);
-DEFINE_XTS_ALG(vaes_avx10_512, "xts-aes-vaes-avx10_512", 800);
-#endif
+DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500);
+DEFINE_AVX_SKCIPHER_ALGS(vaes_avx2, "vaes-avx2", 600);
+DEFINE_AVX_SKCIPHER_ALGS(vaes_avx512, "vaes-avx512", 800);
 
 /* The common part of the x86_64 AES-GCM key struct */
 struct aes_gcm_key {
@@ -876,8 +873,38 @@ struct aes_gcm_key_aesni {
 #define AES_GCM_KEY_AESNI_SIZE	\
 	(sizeof(struct aes_gcm_key_aesni) + (15 & ~(CRYPTO_MINALIGN - 1)))
 
-/* Key struct used by the VAES + AVX10 implementations of AES-GCM */
-struct aes_gcm_key_avx10 {
+/* Key struct used by the VAES + AVX2 implementation of AES-GCM */
+struct aes_gcm_key_vaes_avx2 {
+	/*
+	 * Common part of the key.  The assembly code prefers 16-byte alignment
+	 * for the round keys; we get this by them being located at the start of
+	 * the struct and the whole struct being 32-byte aligned.
+	 */
+	struct aes_gcm_key base;
+
+	/*
+	 * Powers of the hash key H^8 through H^1.  These are 128-bit values.
+	 * They all have an extra factor of x^-1 and are byte-reversed.
+	 * The assembly code prefers 32-byte alignment for this.
+	 */
+	u64 h_powers[8][2] __aligned(32);
+
+	/*
+	 * Each entry in this array contains the two halves of an entry of
+	 * h_powers XOR'd together, in the following order:
+	 * H^8,H^6,H^7,H^5,H^4,H^2,H^3,H^1 i.e. indices 0,2,1,3,4,6,5,7.
+	 * This is used for Karatsuba multiplication.
+	 */
+	u64 h_powers_xored[8];
+};
+
+#define AES_GCM_KEY_VAES_AVX2(key) \
+	container_of((key), struct aes_gcm_key_vaes_avx2, base)
+#define AES_GCM_KEY_VAES_AVX2_SIZE \
+	(sizeof(struct aes_gcm_key_vaes_avx2) + (31 & ~(CRYPTO_MINALIGN - 1)))
+
+/* Key struct used by the VAES + AVX512 implementation of AES-GCM */
+struct aes_gcm_key_vaes_avx512 {
 	/*
 	 * Common part of the key.  The assembly code prefers 16-byte alignment
 	 * for the round keys; we get this by them being located at the start of
@@ -897,10 +924,10 @@ struct aes_gcm_key_avx10 {
 	/* Three padding blocks required by the assembly code */
 	u64 padding[3][2];
 };
-#define AES_GCM_KEY_AVX10(key)	\
-	container_of((key), struct aes_gcm_key_avx10, base)
-#define AES_GCM_KEY_AVX10_SIZE	\
-	(sizeof(struct aes_gcm_key_avx10) + (63 & ~(CRYPTO_MINALIGN - 1)))
+#define AES_GCM_KEY_VAES_AVX512(key) \
+	container_of((key), struct aes_gcm_key_vaes_avx512, base)
+#define AES_GCM_KEY_VAES_AVX512_SIZE \
+	(sizeof(struct aes_gcm_key_vaes_avx512) + (63 & ~(CRYPTO_MINALIGN - 1)))
 
 /*
  * These flags are passed to the AES-GCM helper functions to specify the
@@ -912,23 +939,16 @@ struct aes_gcm_key_avx10 {
 #define FLAG_RFC4106	BIT(0)
 #define FLAG_ENC	BIT(1)
 #define FLAG_AVX	BIT(2)
-#if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
-#  define FLAG_AVX10_256	BIT(3)
-#  define FLAG_AVX10_512	BIT(4)
-#else
-   /*
-    * This should cause all calls to the AVX10 assembly functions to be
-    * optimized out, avoiding the need to ifdef each call individually.
-    */
-#  define FLAG_AVX10_256	0
-#  define FLAG_AVX10_512	0
-#endif
+#define FLAG_VAES_AVX2	BIT(3)
+#define FLAG_VAES_AVX512 BIT(4)
 
 static inline struct aes_gcm_key *
 aes_gcm_key_get(struct crypto_aead *tfm, int flags)
 {
-	if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512))
+	if (flags & FLAG_VAES_AVX512)
 		return PTR_ALIGN(crypto_aead_ctx(tfm), 64);
+	else if (flags & FLAG_VAES_AVX2)
+		return PTR_ALIGN(crypto_aead_ctx(tfm), 32);
 	else
 		return PTR_ALIGN(crypto_aead_ctx(tfm), 16);
 }
@@ -938,26 +958,16 @@ aes_gcm_precompute_aesni(struct aes_gcm_key_aesni *key);
 asmlinkage void
 aes_gcm_precompute_aesni_avx(struct aes_gcm_key_aesni *key);
 asmlinkage void
-aes_gcm_precompute_vaes_avx10_256(struct aes_gcm_key_avx10 *key);
+aes_gcm_precompute_vaes_avx2(struct aes_gcm_key_vaes_avx2 *key);
 asmlinkage void
-aes_gcm_precompute_vaes_avx10_512(struct aes_gcm_key_avx10 *key);
+aes_gcm_precompute_vaes_avx512(struct aes_gcm_key_vaes_avx512 *key);
 
 static void aes_gcm_precompute(struct aes_gcm_key *key, int flags)
 {
-	/*
-	 * To make things a bit easier on the assembly side, the AVX10
-	 * implementations use the same key format.  Therefore, a single
-	 * function using 256-bit vectors would suffice here.  However, it's
-	 * straightforward to provide a 512-bit one because of how the assembly
-	 * code is structured, and it works nicely because the total size of the
-	 * key powers is a multiple of 512 bits.  So we take advantage of that.
-	 *
-	 * A similar situation applies to the AES-NI implementations.
-	 */
-	if (flags & FLAG_AVX10_512)
-		aes_gcm_precompute_vaes_avx10_512(AES_GCM_KEY_AVX10(key));
-	else if (flags & FLAG_AVX10_256)
-		aes_gcm_precompute_vaes_avx10_256(AES_GCM_KEY_AVX10(key));
+	if (flags & FLAG_VAES_AVX512)
+		aes_gcm_precompute_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key));
+	else if (flags & FLAG_VAES_AVX2)
+		aes_gcm_precompute_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key));
 	else if (flags & FLAG_AVX)
 		aes_gcm_precompute_aesni_avx(AES_GCM_KEY_AESNI(key));
 	else
@@ -971,15 +981,21 @@ asmlinkage void
 aes_gcm_aad_update_aesni_avx(const struct aes_gcm_key_aesni *key,
 			     u8 ghash_acc[16], const u8 *aad, int aadlen);
 asmlinkage void
-aes_gcm_aad_update_vaes_avx10(const struct aes_gcm_key_avx10 *key,
-			      u8 ghash_acc[16], const u8 *aad, int aadlen);
+aes_gcm_aad_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
+			     u8 ghash_acc[16], const u8 *aad, int aadlen);
+asmlinkage void
+aes_gcm_aad_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
+			       u8 ghash_acc[16], const u8 *aad, int aadlen);
 
 static void aes_gcm_aad_update(const struct aes_gcm_key *key, u8 ghash_acc[16],
 			       const u8 *aad, int aadlen, int flags)
 {
-	if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512))
-		aes_gcm_aad_update_vaes_avx10(AES_GCM_KEY_AVX10(key), ghash_acc,
-					      aad, aadlen);
+	if (flags & FLAG_VAES_AVX512)
+		aes_gcm_aad_update_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
+					       ghash_acc, aad, aadlen);
+	else if (flags & FLAG_VAES_AVX2)
+		aes_gcm_aad_update_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
+					     ghash_acc, aad, aadlen);
 	else if (flags & FLAG_AVX)
 		aes_gcm_aad_update_aesni_avx(AES_GCM_KEY_AESNI(key), ghash_acc,
 					     aad, aadlen);
@@ -997,13 +1013,13 @@ aes_gcm_enc_update_aesni_avx(const struct aes_gcm_key_aesni *key,
 			     const u32 le_ctr[4], u8 ghash_acc[16],
 			     const u8 *src, u8 *dst, int datalen);
 asmlinkage void
-aes_gcm_enc_update_vaes_avx10_256(const struct aes_gcm_key_avx10 *key,
-				  const u32 le_ctr[4], u8 ghash_acc[16],
-				  const u8 *src, u8 *dst, int datalen);
+aes_gcm_enc_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
+			     const u32 le_ctr[4], u8 ghash_acc[16],
+			     const u8 *src, u8 *dst, int datalen);
 asmlinkage void
-aes_gcm_enc_update_vaes_avx10_512(const struct aes_gcm_key_avx10 *key,
-				  const u32 le_ctr[4], u8 ghash_acc[16],
-				  const u8 *src, u8 *dst, int datalen);
+aes_gcm_enc_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
+			       const u32 le_ctr[4], u8 ghash_acc[16],
+			       const u8 *src, u8 *dst, int datalen);
 
 asmlinkage void
 aes_gcm_dec_update_aesni(const struct aes_gcm_key_aesni *key,
@@ -1014,13 +1030,13 @@ aes_gcm_dec_update_aesni_avx(const struct aes_gcm_key_aesni *key,
 			     const u32 le_ctr[4], u8 ghash_acc[16],
 			     const u8 *src, u8 *dst, int datalen);
 asmlinkage void
-aes_gcm_dec_update_vaes_avx10_256(const struct aes_gcm_key_avx10 *key,
-				  const u32 le_ctr[4], u8 ghash_acc[16],
-				  const u8 *src, u8 *dst, int datalen);
+aes_gcm_dec_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
+			     const u32 le_ctr[4], u8 ghash_acc[16],
+			     const u8 *src, u8 *dst, int datalen);
 asmlinkage void
-aes_gcm_dec_update_vaes_avx10_512(const struct aes_gcm_key_avx10 *key,
-				  const u32 le_ctr[4], u8 ghash_acc[16],
-				  const u8 *src, u8 *dst, int datalen);
+aes_gcm_dec_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
+			       const u32 le_ctr[4], u8 ghash_acc[16],
+			       const u8 *src, u8 *dst, int datalen);
 
 /* __always_inline to optimize out the branches based on @flags */
 static __always_inline void
@@ -1029,14 +1045,14 @@ aes_gcm_update(const struct aes_gcm_key *key,
 	       const u8 *src, u8 *dst, int datalen, int flags)
 {
 	if (flags & FLAG_ENC) {
-		if (flags & FLAG_AVX10_512)
-			aes_gcm_enc_update_vaes_avx10_512(AES_GCM_KEY_AVX10(key),
-							  le_ctr, ghash_acc,
-							  src, dst, datalen);
-		else if (flags & FLAG_AVX10_256)
-			aes_gcm_enc_update_vaes_avx10_256(AES_GCM_KEY_AVX10(key),
-							  le_ctr, ghash_acc,
-							  src, dst, datalen);
+		if (flags & FLAG_VAES_AVX512)
+			aes_gcm_enc_update_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
+						       le_ctr, ghash_acc,
+						       src, dst, datalen);
+		else if (flags & FLAG_VAES_AVX2)
+			aes_gcm_enc_update_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
+						     le_ctr, ghash_acc,
+						     src, dst, datalen);
 		else if (flags & FLAG_AVX)
 			aes_gcm_enc_update_aesni_avx(AES_GCM_KEY_AESNI(key),
 						     le_ctr, ghash_acc,
@@ -1045,14 +1061,14 @@ aes_gcm_update(const struct aes_gcm_key *key,
 			aes_gcm_enc_update_aesni(AES_GCM_KEY_AESNI(key), le_ctr,
 						 ghash_acc, src, dst, datalen);
 	} else {
-		if (flags & FLAG_AVX10_512)
-			aes_gcm_dec_update_vaes_avx10_512(AES_GCM_KEY_AVX10(key),
-							  le_ctr, ghash_acc,
-							  src, dst, datalen);
-		else if (flags & FLAG_AVX10_256)
-			aes_gcm_dec_update_vaes_avx10_256(AES_GCM_KEY_AVX10(key),
-							  le_ctr, ghash_acc,
-							  src, dst, datalen);
+		if (flags & FLAG_VAES_AVX512)
+			aes_gcm_dec_update_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
+						       le_ctr, ghash_acc,
+						       src, dst, datalen);
+		else if (flags & FLAG_VAES_AVX2)
+			aes_gcm_dec_update_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
+						     le_ctr, ghash_acc,
+						     src, dst, datalen);
 		else if (flags & FLAG_AVX)
 			aes_gcm_dec_update_aesni_avx(AES_GCM_KEY_AESNI(key),
 						     le_ctr, ghash_acc,
@@ -1073,9 +1089,13 @@ aes_gcm_enc_final_aesni_avx(const struct aes_gcm_key_aesni *key,
 			    const u32 le_ctr[4], u8 ghash_acc[16],
 			    u64 total_aadlen, u64 total_datalen);
 asmlinkage void
-aes_gcm_enc_final_vaes_avx10(const struct aes_gcm_key_avx10 *key,
-			     const u32 le_ctr[4], u8 ghash_acc[16],
-			     u64 total_aadlen, u64 total_datalen);
+aes_gcm_enc_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
+			    const u32 le_ctr[4], u8 ghash_acc[16],
+			    u64 total_aadlen, u64 total_datalen);
+asmlinkage void
+aes_gcm_enc_final_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
+			      const u32 le_ctr[4], u8 ghash_acc[16],
+			      u64 total_aadlen, u64 total_datalen);
 
 /* __always_inline to optimize out the branches based on @flags */
 static __always_inline void
@@ -1083,10 +1103,14 @@ aes_gcm_enc_final(const struct aes_gcm_key *key,
 		  const u32 le_ctr[4], u8 ghash_acc[16],
 		  u64 total_aadlen, u64 total_datalen, int flags)
 {
-	if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512))
-		aes_gcm_enc_final_vaes_avx10(AES_GCM_KEY_AVX10(key),
-					     le_ctr, ghash_acc,
-					     total_aadlen, total_datalen);
+	if (flags & FLAG_VAES_AVX512)
+		aes_gcm_enc_final_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
+					      le_ctr, ghash_acc,
+					      total_aadlen, total_datalen);
+	else if (flags & FLAG_VAES_AVX2)
+		aes_gcm_enc_final_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
+					    le_ctr, ghash_acc,
+					    total_aadlen, total_datalen);
 	else if (flags & FLAG_AVX)
 		aes_gcm_enc_final_aesni_avx(AES_GCM_KEY_AESNI(key),
 					    le_ctr, ghash_acc,
@@ -1108,10 +1132,15 @@ aes_gcm_dec_final_aesni_avx(const struct aes_gcm_key_aesni *key,
 			    u64 total_aadlen, u64 total_datalen,
 			    const u8 tag[16], int taglen);
 asmlinkage bool __must_check
-aes_gcm_dec_final_vaes_avx10(const struct aes_gcm_key_avx10 *key,
-			     const u32 le_ctr[4], const u8 ghash_acc[16],
-			     u64 total_aadlen, u64 total_datalen,
-			     const u8 tag[16], int taglen);
+aes_gcm_dec_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
+			    const u32 le_ctr[4], const u8 ghash_acc[16],
+			    u64 total_aadlen, u64 total_datalen,
+			    const u8 tag[16], int taglen);
+asmlinkage bool __must_check
+aes_gcm_dec_final_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
+			      const u32 le_ctr[4], const u8 ghash_acc[16],
+			      u64 total_aadlen, u64 total_datalen,
+			      const u8 tag[16], int taglen);
 
 /* __always_inline to optimize out the branches based on @flags */
 static __always_inline bool __must_check
@@ -1119,11 +1148,16 @@ aes_gcm_dec_final(const struct aes_gcm_key *key, const u32 le_ctr[4],
 		  u8 ghash_acc[16], u64 total_aadlen, u64 total_datalen,
 		  u8 tag[16], int taglen, int flags)
 {
-	if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512))
-		return aes_gcm_dec_final_vaes_avx10(AES_GCM_KEY_AVX10(key),
-						    le_ctr, ghash_acc,
-						    total_aadlen, total_datalen,
-						    tag, taglen);
+	if (flags & FLAG_VAES_AVX512)
+		return aes_gcm_dec_final_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
+						     le_ctr, ghash_acc,
+						     total_aadlen, total_datalen,
+						     tag, taglen);
+	else if (flags & FLAG_VAES_AVX2)
+		return aes_gcm_dec_final_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
+						   le_ctr, ghash_acc,
+						   total_aadlen, total_datalen,
+						   tag, taglen);
 	else if (flags & FLAG_AVX)
 		return aes_gcm_dec_final_aesni_avx(AES_GCM_KEY_AESNI(key),
 						   le_ctr, ghash_acc,
@@ -1206,10 +1240,14 @@ static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key,
 	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers) != 496);
 	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers_xored) != 624);
 	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_times_x64) != 688);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, base.aes_key.key_enc) != 0);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, base.aes_key.key_length) != 480);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, h_powers) != 512);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, padding) != 768);
+	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.key_enc) != 0);
+	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.key_length) != 480);
+	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, h_powers) != 512);
+	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, h_powers_xored) != 640);
+	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.key_enc) != 0);
+	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.key_length) != 480);
+	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, h_powers) != 512);
+	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, padding) != 768);
 
 	if (likely(crypto_simd_usable())) {
 		err = aes_check_keylen(keylen);
@@ -1242,8 +1280,9 @@ static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key,
 		gf128mul_lle(&h, (const be128 *)x_to_the_minus1);
 
 		/* Compute the needed key powers */
-		if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512)) {
-			struct aes_gcm_key_avx10 *k = AES_GCM_KEY_AVX10(key);
+		if (flags & FLAG_VAES_AVX512) {
+			struct aes_gcm_key_vaes_avx512 *k =
+				AES_GCM_KEY_VAES_AVX512(key);
 
 			for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) {
 				k->h_powers[i][0] = be64_to_cpu(h.b);
@@ -1251,6 +1290,22 @@ static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key,
 				gf128mul_lle(&h, &h1);
 			}
 			memset(k->padding, 0, sizeof(k->padding));
+		} else if (flags & FLAG_VAES_AVX2) {
+			struct aes_gcm_key_vaes_avx2 *k =
+				AES_GCM_KEY_VAES_AVX2(key);
+			static const u8 indices[8] = { 0, 2, 1, 3, 4, 6, 5, 7 };
+
+			for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) {
+				k->h_powers[i][0] = be64_to_cpu(h.b);
+				k->h_powers[i][1] = be64_to_cpu(h.a);
+				gf128mul_lle(&h, &h1);
+			}
+			for (i = 0; i < ARRAY_SIZE(k->h_powers_xored); i++) {
+				int j = indices[i];
+
+				k->h_powers_xored[i] = k->h_powers[j][0] ^
+						       k->h_powers[j][1];
+			}
 		} else {
 			struct aes_gcm_key_aesni *k = AES_GCM_KEY_AESNI(key);
 
@@ -1291,41 +1346,40 @@ static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16],
 	scatterwalk_start(&walk, sg_src);
 
 	while (assoclen) {
-		unsigned int len_this_page = scatterwalk_clamp(&walk, assoclen);
-		void *mapped = scatterwalk_map(&walk);
-		const void *src = mapped;
+		unsigned int orig_len_this_step = scatterwalk_next(
+			&walk, assoclen);
+		unsigned int len_this_step = orig_len_this_step;
 		unsigned int len;
+		const u8 *src = walk.addr;
 
-		assoclen -= len_this_page;
-		scatterwalk_advance(&walk, len_this_page);
 		if (unlikely(pos)) {
-			len = min(len_this_page, 16 - pos);
+			len = min(len_this_step, 16 - pos);
 			memcpy(&buf[pos], src, len);
 			pos += len;
 			src += len;
-			len_this_page -= len;
+			len_this_step -= len;
 			if (pos < 16)
 				goto next;
 			aes_gcm_aad_update(key, ghash_acc, buf, 16, flags);
 			pos = 0;
 		}
-		len = len_this_page;
+		len = len_this_step;
 		if (unlikely(assoclen)) /* Not the last segment yet? */
 			len = round_down(len, 16);
 		aes_gcm_aad_update(key, ghash_acc, src, len, flags);
 		src += len;
-		len_this_page -= len;
-		if (unlikely(len_this_page)) {
-			memcpy(buf, src, len_this_page);
-			pos = len_this_page;
+		len_this_step -= len;
+		if (unlikely(len_this_step)) {
+			memcpy(buf, src, len_this_step);
+			pos = len_this_step;
 		}
 next:
-		scatterwalk_unmap(mapped);
-		scatterwalk_pagedone(&walk, 0, assoclen);
+		scatterwalk_done_src(&walk, orig_len_this_step);
 		if (need_resched()) {
 			kernel_fpu_end();
 			kernel_fpu_begin();
 		}
+		assoclen -= orig_len_this_step;
 	}
 	if (unlikely(pos))
 		aes_gcm_aad_update(key, ghash_acc, buf, pos, flags);
@@ -1485,10 +1539,9 @@ static struct aead_alg aes_gcm_algs_##suffix[] = { {			       \
 	.chunksize		= AES_BLOCK_SIZE,			       \
 	.maxauthsize		= 16,					       \
 	.base = {							       \
-		.cra_name		= "__gcm(aes)",			       \
-		.cra_driver_name	= "__" generic_driver_name,	       \
+		.cra_name		= "gcm(aes)",			       \
+		.cra_driver_name	= generic_driver_name,		       \
 		.cra_priority		= (priority),			       \
-		.cra_flags		= CRYPTO_ALG_INTERNAL,		       \
 		.cra_blocksize		= 1,				       \
 		.cra_ctxsize		= (ctxsize),			       \
 		.cra_module		= THIS_MODULE,			       \
@@ -1502,17 +1555,14 @@ static struct aead_alg aes_gcm_algs_##suffix[] = { {			       \
 	.chunksize		= AES_BLOCK_SIZE,			       \
 	.maxauthsize		= 16,					       \
 	.base = {							       \
-		.cra_name		= "__rfc4106(gcm(aes))",	       \
-		.cra_driver_name	= "__" rfc_driver_name,		       \
+		.cra_name		= "rfc4106(gcm(aes))",		       \
+		.cra_driver_name	= rfc_driver_name,		       \
 		.cra_priority		= (priority),			       \
-		.cra_flags		= CRYPTO_ALG_INTERNAL,		       \
 		.cra_blocksize		= 1,				       \
 		.cra_ctxsize		= (ctxsize),			       \
 		.cra_module		= THIS_MODULE,			       \
 	},								       \
-} };									       \
-									       \
-static struct simd_aead_alg *aes_gcm_simdalgs_##suffix[2]		       \
+} }
 
 /* aes_gcm_algs_aesni */
 DEFINE_GCM_ALGS(aesni, /* no flags */ 0,
@@ -1524,37 +1574,15 @@ DEFINE_GCM_ALGS(aesni_avx, FLAG_AVX,
 		"generic-gcm-aesni-avx", "rfc4106-gcm-aesni-avx",
 		AES_GCM_KEY_AESNI_SIZE, 500);
 
-#if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
-/* aes_gcm_algs_vaes_avx10_256 */
-DEFINE_GCM_ALGS(vaes_avx10_256, FLAG_AVX10_256,
-		"generic-gcm-vaes-avx10_256", "rfc4106-gcm-vaes-avx10_256",
-		AES_GCM_KEY_AVX10_SIZE, 700);
-
-/* aes_gcm_algs_vaes_avx10_512 */
-DEFINE_GCM_ALGS(vaes_avx10_512, FLAG_AVX10_512,
-		"generic-gcm-vaes-avx10_512", "rfc4106-gcm-vaes-avx10_512",
-		AES_GCM_KEY_AVX10_SIZE, 800);
-#endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */
+/* aes_gcm_algs_vaes_avx2 */
+DEFINE_GCM_ALGS(vaes_avx2, FLAG_VAES_AVX2,
+		"generic-gcm-vaes-avx2", "rfc4106-gcm-vaes-avx2",
+		AES_GCM_KEY_VAES_AVX2_SIZE, 600);
 
-/*
- * This is a list of CPU models that are known to suffer from downclocking when
- * zmm registers (512-bit vectors) are used.  On these CPUs, the AES mode
- * implementations with zmm registers won't be used by default.  Implementations
- * with ymm registers (256-bit vectors) will be used by default instead.
- */
-static const struct x86_cpu_id zmm_exclusion_list[] = {
-	X86_MATCH_VFM(INTEL_SKYLAKE_X,		0),
-	X86_MATCH_VFM(INTEL_ICELAKE_X,		0),
-	X86_MATCH_VFM(INTEL_ICELAKE_D,		0),
-	X86_MATCH_VFM(INTEL_ICELAKE,		0),
-	X86_MATCH_VFM(INTEL_ICELAKE_L,		0),
-	X86_MATCH_VFM(INTEL_ICELAKE_NNPI,	0),
-	X86_MATCH_VFM(INTEL_TIGERLAKE_L,	0),
-	X86_MATCH_VFM(INTEL_TIGERLAKE,		0),
-	/* Allow Rocket Lake and later, and Sapphire Rapids and later. */
-	/* Also allow AMD CPUs (starting with Zen 4, the first with AVX-512). */
-	{},
-};
+/* aes_gcm_algs_vaes_avx512 */
+DEFINE_GCM_ALGS(vaes_avx512, FLAG_VAES_AVX512,
+		"generic-gcm-vaes-avx512", "rfc4106-gcm-vaes-avx512",
+		AES_GCM_KEY_VAES_AVX512_SIZE, 800);
 
 static int __init register_avx_algs(void)
 {
@@ -1562,24 +1590,32 @@ static int __init register_avx_algs(void)
 
 	if (!boot_cpu_has(X86_FEATURE_AVX))
 		return 0;
-	err = simd_register_skciphers_compat(&aes_xts_alg_aesni_avx, 1,
-					     &aes_xts_simdalg_aesni_avx);
+	err = crypto_register_skciphers(skcipher_algs_aesni_avx,
+					ARRAY_SIZE(skcipher_algs_aesni_avx));
 	if (err)
 		return err;
-	err = simd_register_aeads_compat(aes_gcm_algs_aesni_avx,
-					 ARRAY_SIZE(aes_gcm_algs_aesni_avx),
-					 aes_gcm_simdalgs_aesni_avx);
+	err = crypto_register_aeads(aes_gcm_algs_aesni_avx,
+				    ARRAY_SIZE(aes_gcm_algs_aesni_avx));
 	if (err)
 		return err;
-#if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
+	/*
+	 * Note: not all the algorithms registered below actually require
+	 * VPCLMULQDQ.  But in practice every CPU with VAES also has VPCLMULQDQ.
+	 * Similarly, the assembler support was added at about the same time.
+	 * For simplicity, just always check for VAES and VPCLMULQDQ together.
+	 */
 	if (!boot_cpu_has(X86_FEATURE_AVX2) ||
 	    !boot_cpu_has(X86_FEATURE_VAES) ||
 	    !boot_cpu_has(X86_FEATURE_VPCLMULQDQ) ||
 	    !boot_cpu_has(X86_FEATURE_PCLMULQDQ) ||
 	    !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
 		return 0;
-	err = simd_register_skciphers_compat(&aes_xts_alg_vaes_avx2, 1,
-					     &aes_xts_simdalg_vaes_avx2);
+	err = crypto_register_skciphers(skcipher_algs_vaes_avx2,
+					ARRAY_SIZE(skcipher_algs_vaes_avx2));
+	if (err)
+		return err;
+	err = crypto_register_aeads(aes_gcm_algs_vaes_avx2,
+				    ARRAY_SIZE(aes_gcm_algs_vaes_avx2));
 	if (err)
 		return err;
 
@@ -1590,69 +1626,45 @@ static int __init register_avx_algs(void)
 			       XFEATURE_MASK_AVX512, NULL))
 		return 0;
 
-	err = simd_register_skciphers_compat(&aes_xts_alg_vaes_avx10_256, 1,
-					     &aes_xts_simdalg_vaes_avx10_256);
-	if (err)
-		return err;
-	err = simd_register_aeads_compat(aes_gcm_algs_vaes_avx10_256,
-					 ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256),
-					 aes_gcm_simdalgs_vaes_avx10_256);
-	if (err)
-		return err;
-
-	if (x86_match_cpu(zmm_exclusion_list)) {
+	if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) {
 		int i;
 
-		aes_xts_alg_vaes_avx10_512.base.cra_priority = 1;
-		for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512); i++)
-			aes_gcm_algs_vaes_avx10_512[i].base.cra_priority = 1;
+		for (i = 0; i < ARRAY_SIZE(skcipher_algs_vaes_avx512); i++)
+			skcipher_algs_vaes_avx512[i].base.cra_priority = 1;
+		for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx512); i++)
+			aes_gcm_algs_vaes_avx512[i].base.cra_priority = 1;
 	}
 
-	err = simd_register_skciphers_compat(&aes_xts_alg_vaes_avx10_512, 1,
-					     &aes_xts_simdalg_vaes_avx10_512);
+	err = crypto_register_skciphers(skcipher_algs_vaes_avx512,
+					ARRAY_SIZE(skcipher_algs_vaes_avx512));
 	if (err)
 		return err;
-	err = simd_register_aeads_compat(aes_gcm_algs_vaes_avx10_512,
-					 ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512),
-					 aes_gcm_simdalgs_vaes_avx10_512);
+	err = crypto_register_aeads(aes_gcm_algs_vaes_avx512,
+				    ARRAY_SIZE(aes_gcm_algs_vaes_avx512));
 	if (err)
 		return err;
-#endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */
+
 	return 0;
 }
 
+#define unregister_skciphers(A) \
+	if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
+		crypto_unregister_skciphers((A), ARRAY_SIZE(A))
+#define unregister_aeads(A) \
+	if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
+		crypto_unregister_aeads((A), ARRAY_SIZE(A))
+
 static void unregister_avx_algs(void)
 {
-	if (aes_xts_simdalg_aesni_avx)
-		simd_unregister_skciphers(&aes_xts_alg_aesni_avx, 1,
-					  &aes_xts_simdalg_aesni_avx);
-	if (aes_gcm_simdalgs_aesni_avx[0])
-		simd_unregister_aeads(aes_gcm_algs_aesni_avx,
-				      ARRAY_SIZE(aes_gcm_algs_aesni_avx),
-				      aes_gcm_simdalgs_aesni_avx);
-#if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
-	if (aes_xts_simdalg_vaes_avx2)
-		simd_unregister_skciphers(&aes_xts_alg_vaes_avx2, 1,
-					  &aes_xts_simdalg_vaes_avx2);
-	if (aes_xts_simdalg_vaes_avx10_256)
-		simd_unregister_skciphers(&aes_xts_alg_vaes_avx10_256, 1,
-					  &aes_xts_simdalg_vaes_avx10_256);
-	if (aes_gcm_simdalgs_vaes_avx10_256[0])
-		simd_unregister_aeads(aes_gcm_algs_vaes_avx10_256,
-				      ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256),
-				      aes_gcm_simdalgs_vaes_avx10_256);
-	if (aes_xts_simdalg_vaes_avx10_512)
-		simd_unregister_skciphers(&aes_xts_alg_vaes_avx10_512, 1,
-					  &aes_xts_simdalg_vaes_avx10_512);
-	if (aes_gcm_simdalgs_vaes_avx10_512[0])
-		simd_unregister_aeads(aes_gcm_algs_vaes_avx10_512,
-				      ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512),
-				      aes_gcm_simdalgs_vaes_avx10_512);
-#endif
+	unregister_skciphers(skcipher_algs_aesni_avx);
+	unregister_aeads(aes_gcm_algs_aesni_avx);
+	unregister_skciphers(skcipher_algs_vaes_avx2);
+	unregister_skciphers(skcipher_algs_vaes_avx512);
+	unregister_aeads(aes_gcm_algs_vaes_avx2);
+	unregister_aeads(aes_gcm_algs_vaes_avx512);
 }
 #else /* CONFIG_X86_64 */
 static struct aead_alg aes_gcm_algs_aesni[0];
-static struct simd_aead_alg *aes_gcm_simdalgs_aesni[0];
 
 static int __init register_avx_algs(void)
 {
@@ -1676,38 +1688,21 @@ static int __init aesni_init(void)
 
 	if (!x86_match_cpu(aesni_cpu_id))
 		return -ENODEV;
-#ifdef CONFIG_X86_64
-	if (boot_cpu_has(X86_FEATURE_AVX)) {
-		/* optimize performance of ctr mode encryption transform */
-		static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm);
-		pr_info("AES CTR mode by8 optimization enabled\n");
-	}
-#endif /* CONFIG_X86_64 */
 
 	err = crypto_register_alg(&aesni_cipher_alg);
 	if (err)
 		return err;
 
-	err = simd_register_skciphers_compat(aesni_skciphers,
-					     ARRAY_SIZE(aesni_skciphers),
-					     aesni_simd_skciphers);
+	err = crypto_register_skciphers(aesni_skciphers,
+					ARRAY_SIZE(aesni_skciphers));
 	if (err)
 		goto unregister_cipher;
 
-	err = simd_register_aeads_compat(aes_gcm_algs_aesni,
-					 ARRAY_SIZE(aes_gcm_algs_aesni),
-					 aes_gcm_simdalgs_aesni);
+	err = crypto_register_aeads(aes_gcm_algs_aesni,
+				    ARRAY_SIZE(aes_gcm_algs_aesni));
 	if (err)
 		goto unregister_skciphers;
 
-#ifdef CONFIG_X86_64
-	if (boot_cpu_has(X86_FEATURE_AVX))
-		err = simd_register_skciphers_compat(&aesni_xctr, 1,
-						     &aesni_simd_xctr);
-	if (err)
-		goto unregister_aeads;
-#endif /* CONFIG_X86_64 */
-
 	err = register_avx_algs();
 	if (err)
 		goto unregister_avx;
@@ -1716,17 +1711,11 @@ static int __init aesni_init(void)
 
 unregister_avx:
 	unregister_avx_algs();
-#ifdef CONFIG_X86_64
-	if (aesni_simd_xctr)
-		simd_unregister_skciphers(&aesni_xctr, 1, &aesni_simd_xctr);
-unregister_aeads:
-#endif /* CONFIG_X86_64 */
-	simd_unregister_aeads(aes_gcm_algs_aesni,
-			      ARRAY_SIZE(aes_gcm_algs_aesni),
-			      aes_gcm_simdalgs_aesni);
+	crypto_unregister_aeads(aes_gcm_algs_aesni,
+				ARRAY_SIZE(aes_gcm_algs_aesni));
 unregister_skciphers:
-	simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
-				  aesni_simd_skciphers);
+	crypto_unregister_skciphers(aesni_skciphers,
+				    ARRAY_SIZE(aesni_skciphers));
 unregister_cipher:
 	crypto_unregister_alg(&aesni_cipher_alg);
 	return err;
@@ -1734,20 +1723,15 @@ unregister_cipher:
 
 static void __exit aesni_exit(void)
 {
-	simd_unregister_aeads(aes_gcm_algs_aesni,
-			      ARRAY_SIZE(aes_gcm_algs_aesni),
-			      aes_gcm_simdalgs_aesni);
-	simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
-				  aesni_simd_skciphers);
+	crypto_unregister_aeads(aes_gcm_algs_aesni,
+				ARRAY_SIZE(aes_gcm_algs_aesni));
+	crypto_unregister_skciphers(aesni_skciphers,
+				    ARRAY_SIZE(aesni_skciphers));
 	crypto_unregister_alg(&aesni_cipher_alg);
-#ifdef CONFIG_X86_64
-	if (boot_cpu_has(X86_FEATURE_AVX))
-		simd_unregister_skciphers(&aesni_xctr, 1, &aesni_simd_xctr);
-#endif /* CONFIG_X86_64 */
 	unregister_avx_algs();
 }
 
-late_initcall(aesni_init);
+module_init(aesni_init);
 module_exit(aesni_exit);
 
 MODULE_DESCRIPTION("AES cipher and modes, optimized with AES-NI or VAES instructions");
diff --git a/arch/x86/crypto/aria-aesni-avx-asm_64.S b/arch/x86/crypto/aria-aesni-avx-asm_64.S
index 9556dacd9841..932fb17308e7 100644
--- a/arch/x86/crypto/aria-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/aria-aesni-avx-asm_64.S
@@ -295,7 +295,6 @@
 	vpshufb t1, t0, t2;				\
 	vpxor t2, x7, x7;
 
-#ifdef CONFIG_AS_GFNI
 #define aria_sbox_8way_gfni(x0, x1, x2, x3,		\
 			    x4, x5, x6, x7,		\
 			    t0, t1, t2, t3,		\
@@ -318,8 +317,6 @@
 	vgf2p8affineinvqb $0, t2, x3, x3;		\
 	vgf2p8affineinvqb $0, t2, x7, x7
 
-#endif /* CONFIG_AS_GFNI */
-
 #define aria_sbox_8way(x0, x1, x2, x3,            	\
 		       x4, x5, x6, x7,			\
 		       t0, t1, t2, t3,			\
@@ -561,7 +558,6 @@
 			     y4, y5, y6, y7,		\
 			     mem_tmp, 8);
 
-#ifdef CONFIG_AS_GFNI
 #define aria_fe_gfni(x0, x1, x2, x3,			\
 		     x4, x5, x6, x7,			\
 		     y0, y1, y2, y3,			\
@@ -719,8 +715,6 @@
 			     y4, y5, y6, y7,		\
 			     mem_tmp, 8);
 
-#endif /* CONFIG_AS_GFNI */
-
 /* NB: section is mergeable, all elements must be aligned 16-byte blocks */
 .section	.rodata.cst16, "aM", @progbits, 16
 .align 16
@@ -772,7 +766,6 @@
 .Ltf_hi__x2__and__fwd_aff:
 	.octa 0x3F893781E95FE1576CDA64D2BA0CB204
 
-#ifdef CONFIG_AS_GFNI
 /* AES affine: */
 #define tf_aff_const BV8(1, 1, 0, 0, 0, 1, 1, 0)
 .Ltf_aff_bitmatrix:
@@ -871,7 +864,6 @@
 		    BV8(0, 0, 0, 0, 0, 1, 0, 0),
 		    BV8(0, 0, 0, 0, 0, 0, 1, 0),
 		    BV8(0, 0, 0, 0, 0, 0, 0, 1))
-#endif /* CONFIG_AS_GFNI */
 
 /* 4-bit mask */
 .section	.rodata.cst4.L0f0f0f0f, "aM", @progbits, 4
@@ -1140,7 +1132,6 @@ SYM_TYPED_FUNC_START(aria_aesni_avx_ctr_crypt_16way)
 	RET;
 SYM_FUNC_END(aria_aesni_avx_ctr_crypt_16way)
 
-#ifdef CONFIG_AS_GFNI
 SYM_FUNC_START_LOCAL(__aria_aesni_avx_gfni_crypt_16way)
 	/* input:
 	*      %r9: rk
@@ -1359,4 +1350,3 @@ SYM_TYPED_FUNC_START(aria_aesni_avx_gfni_ctr_crypt_16way)
 	FRAME_END
 	RET;
 SYM_FUNC_END(aria_aesni_avx_gfni_ctr_crypt_16way)
-#endif /* CONFIG_AS_GFNI */
diff --git a/arch/x86/crypto/aria-aesni-avx2-asm_64.S b/arch/x86/crypto/aria-aesni-avx2-asm_64.S
index c60fa2980630..ed53d4f46bd7 100644
--- a/arch/x86/crypto/aria-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/aria-aesni-avx2-asm_64.S
@@ -302,7 +302,6 @@
 	vpbroadcastb ((round * 16) + idx + 4)(rk), t0;	\
 	vpxor t0, x7, x7;
 
-#ifdef CONFIG_AS_GFNI
 #define aria_sbox_8way_gfni(x0, x1, x2, x3,		\
 			    x4, x5, x6, x7,		\
 			    t0, t1, t2, t3,		\
@@ -325,7 +324,6 @@
 	vgf2p8affineinvqb $0, t2, x3, x3;		\
 	vgf2p8affineinvqb $0, t2, x7, x7
 
-#endif /* CONFIG_AS_GFNI */
 #define aria_sbox_8way(x0, x1, x2, x3,			\
 		       x4, x5, x6, x7,			\
 		       t0, t1, t2, t3,			\
@@ -598,7 +596,7 @@
 	aria_load_state_8way(y0, y1, y2, y3,		\
 			     y4, y5, y6, y7,		\
 			     mem_tmp, 8);
-#ifdef CONFIG_AS_GFNI
+
 #define aria_fe_gfni(x0, x1, x2, x3,			\
 		     x4, x5, x6, x7,			\
 		     y0, y1, y2, y3,			\
@@ -752,7 +750,6 @@
 	aria_load_state_8way(y0, y1, y2, y3,		\
 			     y4, y5, y6, y7,		\
 			     mem_tmp, 8);
-#endif /* CONFIG_AS_GFNI */
 
 .section        .rodata.cst32.shufb_16x16b, "aM", @progbits, 32
 .align 32
@@ -806,7 +803,6 @@
 .Ltf_hi__x2__and__fwd_aff:
 	.octa 0x3F893781E95FE1576CDA64D2BA0CB204
 
-#ifdef CONFIG_AS_GFNI
 .section	.rodata.cst8, "aM", @progbits, 8
 .align 8
 /* AES affine: */
@@ -868,8 +864,6 @@
 		    BV8(0, 0, 0, 0, 0, 0, 1, 0),
 		    BV8(0, 0, 0, 0, 0, 0, 0, 1))
 
-#endif /* CONFIG_AS_GFNI */
-
 /* 4-bit mask */
 .section	.rodata.cst4.L0f0f0f0f, "aM", @progbits, 4
 .align 4
@@ -1219,7 +1213,6 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_ctr_crypt_32way)
 	RET;
 SYM_FUNC_END(aria_aesni_avx2_ctr_crypt_32way)
 
-#ifdef CONFIG_AS_GFNI
 SYM_FUNC_START_LOCAL(__aria_aesni_avx2_gfni_crypt_32way)
 	/* input:
 	 *      %r9: rk
@@ -1438,4 +1431,3 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_ctr_crypt_32way)
 	FRAME_END
 	RET;
 SYM_FUNC_END(aria_aesni_avx2_gfni_ctr_crypt_32way)
-#endif /* CONFIG_AS_GFNI */
diff --git a/arch/x86/crypto/aria_aesni_avx2_glue.c b/arch/x86/crypto/aria_aesni_avx2_glue.c
index 87a11804fc77..1487a49bfbac 100644
--- a/arch/x86/crypto/aria_aesni_avx2_glue.c
+++ b/arch/x86/crypto/aria_aesni_avx2_glue.c
@@ -6,10 +6,10 @@
  */
 
 #include <crypto/algapi.h>
-#include <crypto/internal/simd.h>
 #include <crypto/aria.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
+#include <linux/export.h>
 #include <linux/module.h>
 #include <linux/types.h>
 
@@ -26,7 +26,6 @@ asmlinkage void aria_aesni_avx2_ctr_crypt_32way(const void *ctx, u8 *dst,
 						const u8 *src,
 						u8 *keystream, u8 *iv);
 EXPORT_SYMBOL_GPL(aria_aesni_avx2_ctr_crypt_32way);
-#ifdef CONFIG_AS_GFNI
 asmlinkage void aria_aesni_avx2_gfni_encrypt_32way(const void *ctx, u8 *dst,
 						   const u8 *src);
 EXPORT_SYMBOL_GPL(aria_aesni_avx2_gfni_encrypt_32way);
@@ -37,7 +36,6 @@ asmlinkage void aria_aesni_avx2_gfni_ctr_crypt_32way(const void *ctx, u8 *dst,
 						     const u8 *src,
 						     u8 *keystream, u8 *iv);
 EXPORT_SYMBOL_GPL(aria_aesni_avx2_gfni_ctr_crypt_32way);
-#endif /* CONFIG_AS_GFNI */
 
 static struct aria_avx_ops aria_ops;
 
@@ -165,10 +163,9 @@ static int aria_avx2_init_tfm(struct crypto_skcipher *tfm)
 
 static struct skcipher_alg aria_algs[] = {
 	{
-		.base.cra_name		= "__ecb(aria)",
-		.base.cra_driver_name	= "__ecb-aria-avx2",
+		.base.cra_name		= "ecb(aria)",
+		.base.cra_driver_name	= "ecb-aria-avx2",
 		.base.cra_priority	= 500,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= ARIA_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct aria_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -178,11 +175,10 @@ static struct skcipher_alg aria_algs[] = {
 		.encrypt		= aria_avx2_ecb_encrypt,
 		.decrypt		= aria_avx2_ecb_decrypt,
 	}, {
-		.base.cra_name		= "__ctr(aria)",
-		.base.cra_driver_name	= "__ctr-aria-avx2",
+		.base.cra_name		= "ctr(aria)",
+		.base.cra_driver_name	= "ctr-aria-avx2",
 		.base.cra_priority	= 500,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL |
-					  CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE,
+		.base.cra_flags		= CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE,
 		.base.cra_blocksize	= 1,
 		.base.cra_ctxsize	= sizeof(struct aria_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -197,8 +193,6 @@ static struct skcipher_alg aria_algs[] = {
 	}
 };
 
-static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)];
-
 static int __init aria_avx2_init(void)
 {
 	const char *feature_name;
@@ -217,7 +211,7 @@ static int __init aria_avx2_init(void)
 		return -ENODEV;
 	}
 
-	if (boot_cpu_has(X86_FEATURE_GFNI) && IS_ENABLED(CONFIG_AS_GFNI)) {
+	if (boot_cpu_has(X86_FEATURE_GFNI)) {
 		aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way;
 		aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way;
 		aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_gfni_ctr_crypt_16way;
@@ -233,15 +227,12 @@ static int __init aria_avx2_init(void)
 		aria_ops.aria_ctr_crypt_32way = aria_aesni_avx2_ctr_crypt_32way;
 	}
 
-	return simd_register_skciphers_compat(aria_algs,
-					      ARRAY_SIZE(aria_algs),
-					      aria_simd_algs);
+	return crypto_register_skciphers(aria_algs, ARRAY_SIZE(aria_algs));
 }
 
 static void __exit aria_avx2_exit(void)
 {
-	simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs),
-				  aria_simd_algs);
+	crypto_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs));
 }
 
 module_init(aria_avx2_init);
diff --git a/arch/x86/crypto/aria_aesni_avx_glue.c b/arch/x86/crypto/aria_aesni_avx_glue.c
index 4e1516b76669..e4e3d78915a5 100644
--- a/arch/x86/crypto/aria_aesni_avx_glue.c
+++ b/arch/x86/crypto/aria_aesni_avx_glue.c
@@ -6,10 +6,10 @@
  */
 
 #include <crypto/algapi.h>
-#include <crypto/internal/simd.h>
 #include <crypto/aria.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
+#include <linux/export.h>
 #include <linux/module.h>
 #include <linux/types.h>
 
@@ -26,7 +26,6 @@ asmlinkage void aria_aesni_avx_ctr_crypt_16way(const void *ctx, u8 *dst,
 					       const u8 *src,
 					       u8 *keystream, u8 *iv);
 EXPORT_SYMBOL_GPL(aria_aesni_avx_ctr_crypt_16way);
-#ifdef CONFIG_AS_GFNI
 asmlinkage void aria_aesni_avx_gfni_encrypt_16way(const void *ctx, u8 *dst,
 						  const u8 *src);
 EXPORT_SYMBOL_GPL(aria_aesni_avx_gfni_encrypt_16way);
@@ -37,7 +36,6 @@ asmlinkage void aria_aesni_avx_gfni_ctr_crypt_16way(const void *ctx, u8 *dst,
 						    const u8 *src,
 						    u8 *keystream, u8 *iv);
 EXPORT_SYMBOL_GPL(aria_aesni_avx_gfni_ctr_crypt_16way);
-#endif /* CONFIG_AS_GFNI */
 
 static struct aria_avx_ops aria_ops;
 
@@ -152,10 +150,9 @@ static int aria_avx_init_tfm(struct crypto_skcipher *tfm)
 
 static struct skcipher_alg aria_algs[] = {
 	{
-		.base.cra_name		= "__ecb(aria)",
-		.base.cra_driver_name	= "__ecb-aria-avx",
+		.base.cra_name		= "ecb(aria)",
+		.base.cra_driver_name	= "ecb-aria-avx",
 		.base.cra_priority	= 400,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= ARIA_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct aria_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -165,10 +162,9 @@ static struct skcipher_alg aria_algs[] = {
 		.encrypt		= aria_avx_ecb_encrypt,
 		.decrypt		= aria_avx_ecb_decrypt,
 	}, {
-		.base.cra_name		= "__ctr(aria)",
-		.base.cra_driver_name	= "__ctr-aria-avx",
+		.base.cra_name		= "ctr(aria)",
+		.base.cra_driver_name	= "ctr-aria-avx",
 		.base.cra_priority	= 400,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= 1,
 		.base.cra_ctxsize	= sizeof(struct aria_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -184,8 +180,6 @@ static struct skcipher_alg aria_algs[] = {
 	}
 };
 
-static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)];
-
 static int __init aria_avx_init(void)
 {
 	const char *feature_name;
@@ -203,7 +197,7 @@ static int __init aria_avx_init(void)
 		return -ENODEV;
 	}
 
-	if (boot_cpu_has(X86_FEATURE_GFNI) && IS_ENABLED(CONFIG_AS_GFNI)) {
+	if (boot_cpu_has(X86_FEATURE_GFNI)) {
 		aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way;
 		aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way;
 		aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_gfni_ctr_crypt_16way;
@@ -213,15 +207,12 @@ static int __init aria_avx_init(void)
 		aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_ctr_crypt_16way;
 	}
 
-	return simd_register_skciphers_compat(aria_algs,
-					      ARRAY_SIZE(aria_algs),
-					      aria_simd_algs);
+	return crypto_register_skciphers(aria_algs, ARRAY_SIZE(aria_algs));
 }
 
 static void __exit aria_avx_exit(void)
 {
-	simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs),
-				  aria_simd_algs);
+	crypto_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs));
 }
 
 module_init(aria_avx_init);
diff --git a/arch/x86/crypto/aria_gfni_avx512_glue.c b/arch/x86/crypto/aria_gfni_avx512_glue.c
index f4a2208d2638..363cbf4399cc 100644
--- a/arch/x86/crypto/aria_gfni_avx512_glue.c
+++ b/arch/x86/crypto/aria_gfni_avx512_glue.c
@@ -6,7 +6,6 @@
  */
 
 #include <crypto/algapi.h>
-#include <crypto/internal/simd.h>
 #include <crypto/aria.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
@@ -165,10 +164,9 @@ static int aria_avx512_init_tfm(struct crypto_skcipher *tfm)
 
 static struct skcipher_alg aria_algs[] = {
 	{
-		.base.cra_name		= "__ecb(aria)",
-		.base.cra_driver_name	= "__ecb-aria-avx512",
+		.base.cra_name		= "ecb(aria)",
+		.base.cra_driver_name	= "ecb-aria-avx512",
 		.base.cra_priority	= 600,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= ARIA_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct aria_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -178,11 +176,10 @@ static struct skcipher_alg aria_algs[] = {
 		.encrypt		= aria_avx512_ecb_encrypt,
 		.decrypt		= aria_avx512_ecb_decrypt,
 	}, {
-		.base.cra_name		= "__ctr(aria)",
-		.base.cra_driver_name	= "__ctr-aria-avx512",
+		.base.cra_name		= "ctr(aria)",
+		.base.cra_driver_name	= "ctr-aria-avx512",
 		.base.cra_priority	= 600,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL |
-					  CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE,
+		.base.cra_flags		= CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE,
 		.base.cra_blocksize	= 1,
 		.base.cra_ctxsize	= sizeof(struct aria_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -197,8 +194,6 @@ static struct skcipher_alg aria_algs[] = {
 	}
 };
 
-static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)];
-
 static int __init aria_avx512_init(void)
 {
 	const char *feature_name;
@@ -229,15 +224,12 @@ static int __init aria_avx512_init(void)
 	aria_ops.aria_decrypt_64way = aria_gfni_avx512_decrypt_64way;
 	aria_ops.aria_ctr_crypt_64way = aria_gfni_avx512_ctr_crypt_64way;
 
-	return simd_register_skciphers_compat(aria_algs,
-					      ARRAY_SIZE(aria_algs),
-					      aria_simd_algs);
+	return crypto_register_skciphers(aria_algs, ARRAY_SIZE(aria_algs));
 }
 
 static void __exit aria_avx512_exit(void)
 {
-	simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs),
-				  aria_simd_algs);
+	crypto_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs));
 }
 
 module_init(aria_avx512_init);
diff --git a/arch/x86/crypto/blake2s-core.S b/arch/x86/crypto/blake2s-core.S
deleted file mode 100644
index b50b35ff1fdb..000000000000
--- a/arch/x86/crypto/blake2s-core.S
+++ /dev/null
@@ -1,256 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 OR MIT */
-/*
- * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
- * Copyright (C) 2017-2019 Samuel Neves <sneves@dei.uc.pt>. All Rights Reserved.
- */
-
-#include <linux/linkage.h>
-
-.section .rodata.cst32.BLAKE2S_IV, "aM", @progbits, 32
-.align 32
-IV:	.octa 0xA54FF53A3C6EF372BB67AE856A09E667
-	.octa 0x5BE0CD191F83D9AB9B05688C510E527F
-.section .rodata.cst16.ROT16, "aM", @progbits, 16
-.align 16
-ROT16:	.octa 0x0D0C0F0E09080B0A0504070601000302
-.section .rodata.cst16.ROR328, "aM", @progbits, 16
-.align 16
-ROR328:	.octa 0x0C0F0E0D080B0A090407060500030201
-.section .rodata.cst64.BLAKE2S_SIGMA, "aM", @progbits, 160
-.align 64
-SIGMA:
-.byte  0,  2,  4,  6,  1,  3,  5,  7, 14,  8, 10, 12, 15,  9, 11, 13
-.byte 14,  4,  9, 13, 10,  8, 15,  6,  5,  1,  0, 11,  3, 12,  2,  7
-.byte 11, 12,  5, 15,  8,  0,  2, 13,  9, 10,  3,  7,  4, 14,  6,  1
-.byte  7,  3, 13, 11,  9,  1, 12, 14, 15,  2,  5,  4,  8,  6, 10,  0
-.byte  9,  5,  2, 10,  0,  7,  4, 15,  3, 14, 11,  6, 13,  1, 12,  8
-.byte  2,  6,  0,  8, 12, 10, 11,  3,  1,  4,  7, 15,  9, 13,  5, 14
-.byte 12,  1, 14,  4,  5, 15, 13, 10,  8,  0,  6,  9, 11,  7,  3,  2
-.byte 13,  7, 12,  3, 11, 14,  1,  9,  2,  5, 15,  8, 10,  0,  4,  6
-.byte  6, 14, 11,  0, 15,  9,  3,  8, 10, 12, 13,  1,  5,  2,  7,  4
-.byte 10,  8,  7,  1,  2,  4,  6,  5, 13, 15,  9,  3,  0, 11, 14, 12
-#ifdef CONFIG_AS_AVX512
-.section .rodata.cst64.BLAKE2S_SIGMA2, "aM", @progbits, 640
-.align 64
-SIGMA2:
-.long  0,  2,  4,  6,  1,  3,  5,  7, 14,  8, 10, 12, 15,  9, 11, 13
-.long  8,  2, 13, 15, 10,  9, 12,  3,  6,  4,  0, 14,  5, 11,  1,  7
-.long 11, 13,  8,  6,  5, 10, 14,  3,  2,  4, 12, 15,  1,  0,  7,  9
-.long 11, 10,  7,  0,  8, 15,  1, 13,  3,  6,  2, 12,  4, 14,  9,  5
-.long  4, 10,  9, 14, 15,  0, 11,  8,  1,  7,  3, 13,  2,  5,  6, 12
-.long  2, 11,  4, 15, 14,  3, 10,  8, 13,  6,  5,  7,  0, 12,  1,  9
-.long  4,  8, 15,  9, 14, 11, 13,  5,  3,  2,  1, 12,  6, 10,  7,  0
-.long  6, 13,  0, 14, 12,  2,  1, 11, 15,  4,  5,  8,  7,  9,  3, 10
-.long 15,  5,  4, 13, 10,  7,  3, 11, 12,  2,  0,  6,  9,  8,  1, 14
-.long  8,  7, 14, 11, 13, 15,  0, 12, 10,  4,  5,  6,  3,  2,  1,  9
-#endif /* CONFIG_AS_AVX512 */
-
-.text
-SYM_FUNC_START(blake2s_compress_ssse3)
-	testq		%rdx,%rdx
-	je		.Lendofloop
-	movdqu		(%rdi),%xmm0
-	movdqu		0x10(%rdi),%xmm1
-	movdqa		ROT16(%rip),%xmm12
-	movdqa		ROR328(%rip),%xmm13
-	movdqu		0x20(%rdi),%xmm14
-	movq		%rcx,%xmm15
-	leaq		SIGMA+0xa0(%rip),%r8
-	jmp		.Lbeginofloop
-	.align		32
-.Lbeginofloop:
-	movdqa		%xmm0,%xmm10
-	movdqa		%xmm1,%xmm11
-	paddq		%xmm15,%xmm14
-	movdqa		IV(%rip),%xmm2
-	movdqa		%xmm14,%xmm3
-	pxor		IV+0x10(%rip),%xmm3
-	leaq		SIGMA(%rip),%rcx
-.Lroundloop:
-	movzbl		(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm4
-	movzbl		0x1(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm5
-	movzbl		0x2(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm6
-	movzbl		0x3(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm7
-	punpckldq	%xmm5,%xmm4
-	punpckldq	%xmm7,%xmm6
-	punpcklqdq	%xmm6,%xmm4
-	paddd		%xmm4,%xmm0
-	paddd		%xmm1,%xmm0
-	pxor		%xmm0,%xmm3
-	pshufb		%xmm12,%xmm3
-	paddd		%xmm3,%xmm2
-	pxor		%xmm2,%xmm1
-	movdqa		%xmm1,%xmm8
-	psrld		$0xc,%xmm1
-	pslld		$0x14,%xmm8
-	por		%xmm8,%xmm1
-	movzbl		0x4(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm5
-	movzbl		0x5(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm6
-	movzbl		0x6(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm7
-	movzbl		0x7(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm4
-	punpckldq	%xmm6,%xmm5
-	punpckldq	%xmm4,%xmm7
-	punpcklqdq	%xmm7,%xmm5
-	paddd		%xmm5,%xmm0
-	paddd		%xmm1,%xmm0
-	pxor		%xmm0,%xmm3
-	pshufb		%xmm13,%xmm3
-	paddd		%xmm3,%xmm2
-	pxor		%xmm2,%xmm1
-	movdqa		%xmm1,%xmm8
-	psrld		$0x7,%xmm1
-	pslld		$0x19,%xmm8
-	por		%xmm8,%xmm1
-	pshufd		$0x93,%xmm0,%xmm0
-	pshufd		$0x4e,%xmm3,%xmm3
-	pshufd		$0x39,%xmm2,%xmm2
-	movzbl		0x8(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm6
-	movzbl		0x9(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm7
-	movzbl		0xa(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm4
-	movzbl		0xb(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm5
-	punpckldq	%xmm7,%xmm6
-	punpckldq	%xmm5,%xmm4
-	punpcklqdq	%xmm4,%xmm6
-	paddd		%xmm6,%xmm0
-	paddd		%xmm1,%xmm0
-	pxor		%xmm0,%xmm3
-	pshufb		%xmm12,%xmm3
-	paddd		%xmm3,%xmm2
-	pxor		%xmm2,%xmm1
-	movdqa		%xmm1,%xmm8
-	psrld		$0xc,%xmm1
-	pslld		$0x14,%xmm8
-	por		%xmm8,%xmm1
-	movzbl		0xc(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm7
-	movzbl		0xd(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm4
-	movzbl		0xe(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm5
-	movzbl		0xf(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm6
-	punpckldq	%xmm4,%xmm7
-	punpckldq	%xmm6,%xmm5
-	punpcklqdq	%xmm5,%xmm7
-	paddd		%xmm7,%xmm0
-	paddd		%xmm1,%xmm0
-	pxor		%xmm0,%xmm3
-	pshufb		%xmm13,%xmm3
-	paddd		%xmm3,%xmm2
-	pxor		%xmm2,%xmm1
-	movdqa		%xmm1,%xmm8
-	psrld		$0x7,%xmm1
-	pslld		$0x19,%xmm8
-	por		%xmm8,%xmm1
-	pshufd		$0x39,%xmm0,%xmm0
-	pshufd		$0x4e,%xmm3,%xmm3
-	pshufd		$0x93,%xmm2,%xmm2
-	addq		$0x10,%rcx
-	cmpq		%r8,%rcx
-	jnz		.Lroundloop
-	pxor		%xmm2,%xmm0
-	pxor		%xmm3,%xmm1
-	pxor		%xmm10,%xmm0
-	pxor		%xmm11,%xmm1
-	addq		$0x40,%rsi
-	decq		%rdx
-	jnz		.Lbeginofloop
-	movdqu		%xmm0,(%rdi)
-	movdqu		%xmm1,0x10(%rdi)
-	movdqu		%xmm14,0x20(%rdi)
-.Lendofloop:
-	RET
-SYM_FUNC_END(blake2s_compress_ssse3)
-
-#ifdef CONFIG_AS_AVX512
-SYM_FUNC_START(blake2s_compress_avx512)
-	vmovdqu		(%rdi),%xmm0
-	vmovdqu		0x10(%rdi),%xmm1
-	vmovdqu		0x20(%rdi),%xmm4
-	vmovq		%rcx,%xmm5
-	vmovdqa		IV(%rip),%xmm14
-	vmovdqa		IV+16(%rip),%xmm15
-	jmp		.Lblake2s_compress_avx512_mainloop
-.align 32
-.Lblake2s_compress_avx512_mainloop:
-	vmovdqa		%xmm0,%xmm10
-	vmovdqa		%xmm1,%xmm11
-	vpaddq		%xmm5,%xmm4,%xmm4
-	vmovdqa		%xmm14,%xmm2
-	vpxor		%xmm15,%xmm4,%xmm3
-	vmovdqu		(%rsi),%ymm6
-	vmovdqu		0x20(%rsi),%ymm7
-	addq		$0x40,%rsi
-	leaq		SIGMA2(%rip),%rax
-	movb		$0xa,%cl
-.Lblake2s_compress_avx512_roundloop:
-	addq		$0x40,%rax
-	vmovdqa		-0x40(%rax),%ymm8
-	vmovdqa		-0x20(%rax),%ymm9
-	vpermi2d	%ymm7,%ymm6,%ymm8
-	vpermi2d	%ymm7,%ymm6,%ymm9
-	vmovdqa		%ymm8,%ymm6
-	vmovdqa		%ymm9,%ymm7
-	vpaddd		%xmm8,%xmm0,%xmm0
-	vpaddd		%xmm1,%xmm0,%xmm0
-	vpxor		%xmm0,%xmm3,%xmm3
-	vprord		$0x10,%xmm3,%xmm3
-	vpaddd		%xmm3,%xmm2,%xmm2
-	vpxor		%xmm2,%xmm1,%xmm1
-	vprord		$0xc,%xmm1,%xmm1
-	vextracti128	$0x1,%ymm8,%xmm8
-	vpaddd		%xmm8,%xmm0,%xmm0
-	vpaddd		%xmm1,%xmm0,%xmm0
-	vpxor		%xmm0,%xmm3,%xmm3
-	vprord		$0x8,%xmm3,%xmm3
-	vpaddd		%xmm3,%xmm2,%xmm2
-	vpxor		%xmm2,%xmm1,%xmm1
-	vprord		$0x7,%xmm1,%xmm1
-	vpshufd		$0x93,%xmm0,%xmm0
-	vpshufd		$0x4e,%xmm3,%xmm3
-	vpshufd		$0x39,%xmm2,%xmm2
-	vpaddd		%xmm9,%xmm0,%xmm0
-	vpaddd		%xmm1,%xmm0,%xmm0
-	vpxor		%xmm0,%xmm3,%xmm3
-	vprord		$0x10,%xmm3,%xmm3
-	vpaddd		%xmm3,%xmm2,%xmm2
-	vpxor		%xmm2,%xmm1,%xmm1
-	vprord		$0xc,%xmm1,%xmm1
-	vextracti128	$0x1,%ymm9,%xmm9
-	vpaddd		%xmm9,%xmm0,%xmm0
-	vpaddd		%xmm1,%xmm0,%xmm0
-	vpxor		%xmm0,%xmm3,%xmm3
-	vprord		$0x8,%xmm3,%xmm3
-	vpaddd		%xmm3,%xmm2,%xmm2
-	vpxor		%xmm2,%xmm1,%xmm1
-	vprord		$0x7,%xmm1,%xmm1
-	vpshufd		$0x39,%xmm0,%xmm0
-	vpshufd		$0x4e,%xmm3,%xmm3
-	vpshufd		$0x93,%xmm2,%xmm2
-	decb		%cl
-	jne		.Lblake2s_compress_avx512_roundloop
-	vpxor		%xmm10,%xmm0,%xmm0
-	vpxor		%xmm11,%xmm1,%xmm1
-	vpxor		%xmm2,%xmm0,%xmm0
-	vpxor		%xmm3,%xmm1,%xmm1
-	decq		%rdx
-	jne		.Lblake2s_compress_avx512_mainloop
-	vmovdqu		%xmm0,(%rdi)
-	vmovdqu		%xmm1,0x10(%rdi)
-	vmovdqu		%xmm4,0x20(%rdi)
-	vzeroupper
-	RET
-SYM_FUNC_END(blake2s_compress_avx512)
-#endif /* CONFIG_AS_AVX512 */
diff --git a/arch/x86/crypto/blake2s-glue.c b/arch/x86/crypto/blake2s-glue.c
deleted file mode 100644
index 0313f9673f56..000000000000
--- a/arch/x86/crypto/blake2s-glue.c
+++ /dev/null
@@ -1,74 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0 OR MIT
-/*
- * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
- */
-
-#include <crypto/internal/blake2s.h>
-
-#include <linux/types.h>
-#include <linux/jump_label.h>
-#include <linux/kernel.h>
-#include <linux/sizes.h>
-
-#include <asm/cpufeature.h>
-#include <asm/fpu/api.h>
-#include <asm/processor.h>
-#include <asm/simd.h>
-
-asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state,
-				       const u8 *block, const size_t nblocks,
-				       const u32 inc);
-asmlinkage void blake2s_compress_avx512(struct blake2s_state *state,
-					const u8 *block, const size_t nblocks,
-					const u32 inc);
-
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3);
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512);
-
-void blake2s_compress(struct blake2s_state *state, const u8 *block,
-		      size_t nblocks, const u32 inc)
-{
-	/* SIMD disables preemption, so relax after processing each page. */
-	BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8);
-
-	if (!static_branch_likely(&blake2s_use_ssse3) || !may_use_simd()) {
-		blake2s_compress_generic(state, block, nblocks, inc);
-		return;
-	}
-
-	do {
-		const size_t blocks = min_t(size_t, nblocks,
-					    SZ_4K / BLAKE2S_BLOCK_SIZE);
-
-		kernel_fpu_begin();
-		if (IS_ENABLED(CONFIG_AS_AVX512) &&
-		    static_branch_likely(&blake2s_use_avx512))
-			blake2s_compress_avx512(state, block, blocks, inc);
-		else
-			blake2s_compress_ssse3(state, block, blocks, inc);
-		kernel_fpu_end();
-
-		nblocks -= blocks;
-		block += blocks * BLAKE2S_BLOCK_SIZE;
-	} while (nblocks);
-}
-EXPORT_SYMBOL(blake2s_compress);
-
-static int __init blake2s_mod_init(void)
-{
-	if (boot_cpu_has(X86_FEATURE_SSSE3))
-		static_branch_enable(&blake2s_use_ssse3);
-
-	if (IS_ENABLED(CONFIG_AS_AVX512) &&
-	    boot_cpu_has(X86_FEATURE_AVX) &&
-	    boot_cpu_has(X86_FEATURE_AVX2) &&
-	    boot_cpu_has(X86_FEATURE_AVX512F) &&
-	    boot_cpu_has(X86_FEATURE_AVX512VL) &&
-	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
-			      XFEATURE_MASK_AVX512, NULL))
-		static_branch_enable(&blake2s_use_avx512);
-
-	return 0;
-}
-
-subsys_initcall(blake2s_mod_init);
diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c
index 552f2df0643f..26c5f2ee5d10 100644
--- a/arch/x86/crypto/blowfish_glue.c
+++ b/arch/x86/crypto/blowfish_glue.c
@@ -94,7 +94,6 @@ static struct crypto_alg bf_cipher_alg = {
 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize		= BF_BLOCK_SIZE,
 	.cra_ctxsize		= sizeof(struct bf_ctx),
-	.cra_alignmask		= 0,
 	.cra_module		= THIS_MODULE,
 	.cra_u = {
 		.cipher = {
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index 646477a13e11..1dfef28c1266 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -16,6 +16,7 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 #include <asm/frame.h>
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
@@ -882,7 +883,7 @@ SYM_FUNC_START_LOCAL(__camellia_dec_blk16)
 	jmp .Ldec_max24;
 SYM_FUNC_END(__camellia_dec_blk16)
 
-SYM_FUNC_START(camellia_ecb_enc_16way)
+SYM_TYPED_FUNC_START(camellia_ecb_enc_16way)
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	%rsi: dst (16 blocks)
@@ -907,7 +908,7 @@ SYM_FUNC_START(camellia_ecb_enc_16way)
 	RET;
 SYM_FUNC_END(camellia_ecb_enc_16way)
 
-SYM_FUNC_START(camellia_ecb_dec_16way)
+SYM_TYPED_FUNC_START(camellia_ecb_dec_16way)
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	%rsi: dst (16 blocks)
@@ -937,7 +938,7 @@ SYM_FUNC_START(camellia_ecb_dec_16way)
 	RET;
 SYM_FUNC_END(camellia_ecb_dec_16way)
 
-SYM_FUNC_START(camellia_cbc_dec_16way)
+SYM_TYPED_FUNC_START(camellia_cbc_dec_16way)
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	%rsi: dst (16 blocks)
diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
index a0eb94e53b1b..b1c9b9450555 100644
--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
@@ -6,6 +6,7 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 #include <asm/frame.h>
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
diff --git a/arch/x86/crypto/camellia-x86_64-asm_64.S b/arch/x86/crypto/camellia-x86_64-asm_64.S
index 816b6bb8bded..824cb94de6c2 100644
--- a/arch/x86/crypto/camellia-x86_64-asm_64.S
+++ b/arch/x86/crypto/camellia-x86_64-asm_64.S
@@ -6,6 +6,7 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 
 .file "camellia-x86_64-asm_64.S"
 .text
@@ -177,7 +178,7 @@
 	bswapq				RAB0; \
 	movq RAB0,			4*2(RIO);
 
-SYM_FUNC_START(__camellia_enc_blk)
+SYM_TYPED_FUNC_START(__camellia_enc_blk)
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	%rsi: dst
@@ -224,7 +225,7 @@ SYM_FUNC_START(__camellia_enc_blk)
 	RET;
 SYM_FUNC_END(__camellia_enc_blk)
 
-SYM_FUNC_START(camellia_dec_blk)
+SYM_TYPED_FUNC_START(camellia_dec_blk)
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	%rsi: dst
@@ -411,7 +412,7 @@ SYM_FUNC_END(camellia_dec_blk)
 		bswapq				RAB1; \
 		movq RAB1,			12*2(RIO);
 
-SYM_FUNC_START(__camellia_enc_blk_2way)
+SYM_TYPED_FUNC_START(__camellia_enc_blk_2way)
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	%rsi: dst
@@ -460,7 +461,7 @@ SYM_FUNC_START(__camellia_enc_blk_2way)
 	RET;
 SYM_FUNC_END(__camellia_enc_blk_2way)
 
-SYM_FUNC_START(camellia_dec_blk_2way)
+SYM_TYPED_FUNC_START(camellia_dec_blk_2way)
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	%rsi: dst
diff --git a/arch/x86/crypto/camellia_aesni_avx2_glue.c b/arch/x86/crypto/camellia_aesni_avx2_glue.c
index e7e4d64e9577..2d2f4e16537c 100644
--- a/arch/x86/crypto/camellia_aesni_avx2_glue.c
+++ b/arch/x86/crypto/camellia_aesni_avx2_glue.c
@@ -6,7 +6,6 @@
  */
 
 #include <crypto/algapi.h>
-#include <crypto/internal/simd.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
 #include <linux/module.h>
@@ -69,10 +68,9 @@ static int cbc_decrypt(struct skcipher_request *req)
 
 static struct skcipher_alg camellia_algs[] = {
 	{
-		.base.cra_name		= "__ecb(camellia)",
-		.base.cra_driver_name	= "__ecb-camellia-aesni-avx2",
+		.base.cra_name		= "ecb(camellia)",
+		.base.cra_driver_name	= "ecb-camellia-aesni-avx2",
 		.base.cra_priority	= 500,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -82,10 +80,9 @@ static struct skcipher_alg camellia_algs[] = {
 		.encrypt		= ecb_encrypt,
 		.decrypt		= ecb_decrypt,
 	}, {
-		.base.cra_name		= "__cbc(camellia)",
-		.base.cra_driver_name	= "__cbc-camellia-aesni-avx2",
+		.base.cra_name		= "cbc(camellia)",
+		.base.cra_driver_name	= "cbc-camellia-aesni-avx2",
 		.base.cra_priority	= 500,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -98,8 +95,6 @@ static struct skcipher_alg camellia_algs[] = {
 	},
 };
 
-static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];
-
 static int __init camellia_aesni_init(void)
 {
 	const char *feature_name;
@@ -118,15 +113,13 @@ static int __init camellia_aesni_init(void)
 		return -ENODEV;
 	}
 
-	return simd_register_skciphers_compat(camellia_algs,
-					      ARRAY_SIZE(camellia_algs),
-					      camellia_simd_algs);
+	return crypto_register_skciphers(camellia_algs,
+					 ARRAY_SIZE(camellia_algs));
 }
 
 static void __exit camellia_aesni_fini(void)
 {
-	simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
-				  camellia_simd_algs);
+	crypto_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs));
 }
 
 module_init(camellia_aesni_init);
diff --git a/arch/x86/crypto/camellia_aesni_avx_glue.c b/arch/x86/crypto/camellia_aesni_avx_glue.c
index c7ccf63e741e..5c321f255eb7 100644
--- a/arch/x86/crypto/camellia_aesni_avx_glue.c
+++ b/arch/x86/crypto/camellia_aesni_avx_glue.c
@@ -6,9 +6,9 @@
  */
 
 #include <crypto/algapi.h>
-#include <crypto/internal/simd.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
+#include <linux/export.h>
 #include <linux/module.h>
 #include <linux/types.h>
 
@@ -69,10 +69,9 @@ static int cbc_decrypt(struct skcipher_request *req)
 
 static struct skcipher_alg camellia_algs[] = {
 	{
-		.base.cra_name		= "__ecb(camellia)",
-		.base.cra_driver_name	= "__ecb-camellia-aesni",
+		.base.cra_name		= "ecb(camellia)",
+		.base.cra_driver_name	= "ecb-camellia-aesni",
 		.base.cra_priority	= 400,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -82,10 +81,9 @@ static struct skcipher_alg camellia_algs[] = {
 		.encrypt		= ecb_encrypt,
 		.decrypt		= ecb_decrypt,
 	}, {
-		.base.cra_name		= "__cbc(camellia)",
-		.base.cra_driver_name	= "__cbc-camellia-aesni",
+		.base.cra_name		= "cbc(camellia)",
+		.base.cra_driver_name	= "cbc-camellia-aesni",
 		.base.cra_priority	= 400,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -98,8 +96,6 @@ static struct skcipher_alg camellia_algs[] = {
 	}
 };
 
-static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];
-
 static int __init camellia_aesni_init(void)
 {
 	const char *feature_name;
@@ -117,15 +113,13 @@ static int __init camellia_aesni_init(void)
 		return -ENODEV;
 	}
 
-	return simd_register_skciphers_compat(camellia_algs,
-					      ARRAY_SIZE(camellia_algs),
-					      camellia_simd_algs);
+	return crypto_register_skciphers(camellia_algs,
+					 ARRAY_SIZE(camellia_algs));
 }
 
 static void __exit camellia_aesni_fini(void)
 {
-	simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
-				  camellia_simd_algs);
+	crypto_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs));
 }
 
 module_init(camellia_aesni_init);
diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c
index f110708c8038..cbede120e5f2 100644
--- a/arch/x86/crypto/camellia_glue.c
+++ b/arch/x86/crypto/camellia_glue.c
@@ -10,6 +10,7 @@
 
 #include <linux/unaligned.h>
 #include <linux/crypto.h>
+#include <linux/export.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/types.h>
@@ -1313,7 +1314,6 @@ static struct crypto_alg camellia_cipher_alg = {
 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize		= CAMELLIA_BLOCK_SIZE,
 	.cra_ctxsize		= sizeof(struct camellia_ctx),
-	.cra_alignmask		= 0,
 	.cra_module		= THIS_MODULE,
 	.cra_u			= {
 		.cipher = {
diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
index b4e460a87f18..fb95a614249d 100644
--- a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
@@ -487,79 +487,3 @@ SYM_FUNC_START(cast5_cbc_dec_16way)
 	FRAME_END
 	RET;
 SYM_FUNC_END(cast5_cbc_dec_16way)
-
-SYM_FUNC_START(cast5_ctr_16way)
-	/* input:
-	 *	%rdi: ctx
-	 *	%rsi: dst
-	 *	%rdx: src
-	 *	%rcx: iv (big endian, 64bit)
-	 */
-	FRAME_BEGIN
-	pushq %r12;
-	pushq %r15;
-
-	movq %rdi, CTX;
-	movq %rsi, %r11;
-	movq %rdx, %r12;
-
-	vpcmpeqd RTMP, RTMP, RTMP;
-	vpsrldq $8, RTMP, RTMP; /* low: -1, high: 0 */
-
-	vpcmpeqd RKR, RKR, RKR;
-	vpaddq RKR, RKR, RKR; /* low: -2, high: -2 */
-	vmovdqa .Lbswap_iv_mask(%rip), R1ST;
-	vmovdqa .Lbswap128_mask(%rip), RKM;
-
-	/* load IV and byteswap */
-	vmovq (%rcx), RX;
-	vpshufb R1ST, RX, RX;
-
-	/* construct IVs */
-	vpsubq RTMP, RX, RX;  /* le: IV1, IV0 */
-	vpshufb RKM, RX, RL1; /* be: IV0, IV1 */
-	vpsubq RKR, RX, RX;
-	vpshufb RKM, RX, RR1; /* be: IV2, IV3 */
-	vpsubq RKR, RX, RX;
-	vpshufb RKM, RX, RL2; /* be: IV4, IV5 */
-	vpsubq RKR, RX, RX;
-	vpshufb RKM, RX, RR2; /* be: IV6, IV7 */
-	vpsubq RKR, RX, RX;
-	vpshufb RKM, RX, RL3; /* be: IV8, IV9 */
-	vpsubq RKR, RX, RX;
-	vpshufb RKM, RX, RR3; /* be: IV10, IV11 */
-	vpsubq RKR, RX, RX;
-	vpshufb RKM, RX, RL4; /* be: IV12, IV13 */
-	vpsubq RKR, RX, RX;
-	vpshufb RKM, RX, RR4; /* be: IV14, IV15 */
-
-	/* store last IV */
-	vpsubq RTMP, RX, RX; /* le: IV16, IV14 */
-	vpshufb R1ST, RX, RX; /* be: IV16, IV16 */
-	vmovq RX, (%rcx);
-
-	call __cast5_enc_blk16;
-
-	/* dst = src ^ iv */
-	vpxor (0*16)(%r12), RR1, RR1;
-	vpxor (1*16)(%r12), RL1, RL1;
-	vpxor (2*16)(%r12), RR2, RR2;
-	vpxor (3*16)(%r12), RL2, RL2;
-	vpxor (4*16)(%r12), RR3, RR3;
-	vpxor (5*16)(%r12), RL3, RL3;
-	vpxor (6*16)(%r12), RR4, RR4;
-	vpxor (7*16)(%r12), RL4, RL4;
-	vmovdqu RR1, (0*16)(%r11);
-	vmovdqu RL1, (1*16)(%r11);
-	vmovdqu RR2, (2*16)(%r11);
-	vmovdqu RL2, (3*16)(%r11);
-	vmovdqu RR3, (4*16)(%r11);
-	vmovdqu RL3, (5*16)(%r11);
-	vmovdqu RR4, (6*16)(%r11);
-	vmovdqu RL4, (7*16)(%r11);
-
-	popq %r15;
-	popq %r12;
-	FRAME_END
-	RET;
-SYM_FUNC_END(cast5_ctr_16way)
diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c
index 3976a87f92ad..3aca04d43b34 100644
--- a/arch/x86/crypto/cast5_avx_glue.c
+++ b/arch/x86/crypto/cast5_avx_glue.c
@@ -8,7 +8,6 @@
 
 #include <crypto/algapi.h>
 #include <crypto/cast5.h>
-#include <crypto/internal/simd.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
 #include <linux/module.h>
@@ -64,10 +63,9 @@ static int cbc_decrypt(struct skcipher_request *req)
 
 static struct skcipher_alg cast5_algs[] = {
 	{
-		.base.cra_name		= "__ecb(cast5)",
-		.base.cra_driver_name	= "__ecb-cast5-avx",
+		.base.cra_name		= "ecb(cast5)",
+		.base.cra_driver_name	= "ecb-cast5-avx",
 		.base.cra_priority	= 200,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= CAST5_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct cast5_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -77,10 +75,9 @@ static struct skcipher_alg cast5_algs[] = {
 		.encrypt		= ecb_encrypt,
 		.decrypt		= ecb_decrypt,
 	}, {
-		.base.cra_name		= "__cbc(cast5)",
-		.base.cra_driver_name	= "__cbc-cast5-avx",
+		.base.cra_name		= "cbc(cast5)",
+		.base.cra_driver_name	= "cbc-cast5-avx",
 		.base.cra_priority	= 200,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= CAST5_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct cast5_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -93,8 +90,6 @@ static struct skcipher_alg cast5_algs[] = {
 	}
 };
 
-static struct simd_skcipher_alg *cast5_simd_algs[ARRAY_SIZE(cast5_algs)];
-
 static int __init cast5_init(void)
 {
 	const char *feature_name;
@@ -105,15 +100,13 @@ static int __init cast5_init(void)
 		return -ENODEV;
 	}
 
-	return simd_register_skciphers_compat(cast5_algs,
-					      ARRAY_SIZE(cast5_algs),
-					      cast5_simd_algs);
+	return crypto_register_skciphers(cast5_algs,
+					 ARRAY_SIZE(cast5_algs));
 }
 
 static void __exit cast5_exit(void)
 {
-	simd_unregister_skciphers(cast5_algs, ARRAY_SIZE(cast5_algs),
-				  cast5_simd_algs);
+	crypto_unregister_skciphers(cast5_algs, ARRAY_SIZE(cast5_algs));
 }
 
 module_init(cast5_init);
diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c
index 7e2aea372349..c4dd28c30303 100644
--- a/arch/x86/crypto/cast6_avx_glue.c
+++ b/arch/x86/crypto/cast6_avx_glue.c
@@ -14,7 +14,6 @@
 #include <linux/err.h>
 #include <crypto/algapi.h>
 #include <crypto/cast6.h>
-#include <crypto/internal/simd.h>
 
 #include "ecb_cbc_helpers.h"
 
@@ -64,10 +63,9 @@ static int cbc_decrypt(struct skcipher_request *req)
 
 static struct skcipher_alg cast6_algs[] = {
 	{
-		.base.cra_name		= "__ecb(cast6)",
-		.base.cra_driver_name	= "__ecb-cast6-avx",
+		.base.cra_name		= "ecb(cast6)",
+		.base.cra_driver_name	= "ecb-cast6-avx",
 		.base.cra_priority	= 200,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= CAST6_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct cast6_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -77,10 +75,9 @@ static struct skcipher_alg cast6_algs[] = {
 		.encrypt		= ecb_encrypt,
 		.decrypt		= ecb_decrypt,
 	}, {
-		.base.cra_name		= "__cbc(cast6)",
-		.base.cra_driver_name	= "__cbc-cast6-avx",
+		.base.cra_name		= "cbc(cast6)",
+		.base.cra_driver_name	= "cbc-cast6-avx",
 		.base.cra_priority	= 200,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= CAST6_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct cast6_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -93,8 +90,6 @@ static struct skcipher_alg cast6_algs[] = {
 	},
 };
 
-static struct simd_skcipher_alg *cast6_simd_algs[ARRAY_SIZE(cast6_algs)];
-
 static int __init cast6_init(void)
 {
 	const char *feature_name;
@@ -105,15 +100,12 @@ static int __init cast6_init(void)
 		return -ENODEV;
 	}
 
-	return simd_register_skciphers_compat(cast6_algs,
-					      ARRAY_SIZE(cast6_algs),
-					      cast6_simd_algs);
+	return crypto_register_skciphers(cast6_algs, ARRAY_SIZE(cast6_algs));
 }
 
 static void __exit cast6_exit(void)
 {
-	simd_unregister_skciphers(cast6_algs, ARRAY_SIZE(cast6_algs),
-				  cast6_simd_algs);
+	crypto_unregister_skciphers(cast6_algs, ARRAY_SIZE(cast6_algs));
 }
 
 module_init(cast6_init);
diff --git a/arch/x86/crypto/chacha-avx2-x86_64.S b/arch/x86/crypto/chacha-avx2-x86_64.S
deleted file mode 100644
index f3d8fc018249..000000000000
--- a/arch/x86/crypto/chacha-avx2-x86_64.S
+++ /dev/null
@@ -1,1021 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * ChaCha 256-bit cipher algorithm, x64 AVX2 functions
- *
- * Copyright (C) 2015 Martin Willi
- */
-
-#include <linux/linkage.h>
-
-.section	.rodata.cst32.ROT8, "aM", @progbits, 32
-.align 32
-ROT8:	.octa 0x0e0d0c0f0a09080b0605040702010003
-	.octa 0x0e0d0c0f0a09080b0605040702010003
-
-.section	.rodata.cst32.ROT16, "aM", @progbits, 32
-.align 32
-ROT16:	.octa 0x0d0c0f0e09080b0a0504070601000302
-	.octa 0x0d0c0f0e09080b0a0504070601000302
-
-.section	.rodata.cst32.CTRINC, "aM", @progbits, 32
-.align 32
-CTRINC:	.octa 0x00000003000000020000000100000000
-	.octa 0x00000007000000060000000500000004
-
-.section	.rodata.cst32.CTR2BL, "aM", @progbits, 32
-.align 32
-CTR2BL:	.octa 0x00000000000000000000000000000000
-	.octa 0x00000000000000000000000000000001
-
-.section	.rodata.cst32.CTR4BL, "aM", @progbits, 32
-.align 32
-CTR4BL:	.octa 0x00000000000000000000000000000002
-	.octa 0x00000000000000000000000000000003
-
-.text
-
-SYM_FUNC_START(chacha_2block_xor_avx2)
-	# %rdi: Input state matrix, s
-	# %rsi: up to 2 data blocks output, o
-	# %rdx: up to 2 data blocks input, i
-	# %rcx: input/output length in bytes
-	# %r8d: nrounds
-
-	# This function encrypts two ChaCha blocks by loading the state
-	# matrix twice across four AVX registers. It performs matrix operations
-	# on four words in each matrix in parallel, but requires shuffling to
-	# rearrange the words after each round.
-
-	vzeroupper
-
-	# x0..3[0-2] = s0..3
-	vbroadcasti128	0x00(%rdi),%ymm0
-	vbroadcasti128	0x10(%rdi),%ymm1
-	vbroadcasti128	0x20(%rdi),%ymm2
-	vbroadcasti128	0x30(%rdi),%ymm3
-
-	vpaddd		CTR2BL(%rip),%ymm3,%ymm3
-
-	vmovdqa		%ymm0,%ymm8
-	vmovdqa		%ymm1,%ymm9
-	vmovdqa		%ymm2,%ymm10
-	vmovdqa		%ymm3,%ymm11
-
-	vmovdqa		ROT8(%rip),%ymm4
-	vmovdqa		ROT16(%rip),%ymm5
-
-	mov		%rcx,%rax
-
-.Ldoubleround:
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 16)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxor		%ymm0,%ymm3,%ymm3
-	vpshufb		%ymm5,%ymm3,%ymm3
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm6
-	vpslld		$12,%ymm6,%ymm6
-	vpsrld		$20,%ymm1,%ymm1
-	vpor		%ymm6,%ymm1,%ymm1
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 8)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxor		%ymm0,%ymm3,%ymm3
-	vpshufb		%ymm4,%ymm3,%ymm3
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm7
-	vpslld		$7,%ymm7,%ymm7
-	vpsrld		$25,%ymm1,%ymm1
-	vpor		%ymm7,%ymm1,%ymm1
-
-	# x1 = shuffle32(x1, MASK(0, 3, 2, 1))
-	vpshufd		$0x39,%ymm1,%ymm1
-	# x2 = shuffle32(x2, MASK(1, 0, 3, 2))
-	vpshufd		$0x4e,%ymm2,%ymm2
-	# x3 = shuffle32(x3, MASK(2, 1, 0, 3))
-	vpshufd		$0x93,%ymm3,%ymm3
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 16)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxor		%ymm0,%ymm3,%ymm3
-	vpshufb		%ymm5,%ymm3,%ymm3
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm6
-	vpslld		$12,%ymm6,%ymm6
-	vpsrld		$20,%ymm1,%ymm1
-	vpor		%ymm6,%ymm1,%ymm1
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 8)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxor		%ymm0,%ymm3,%ymm3
-	vpshufb		%ymm4,%ymm3,%ymm3
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm7
-	vpslld		$7,%ymm7,%ymm7
-	vpsrld		$25,%ymm1,%ymm1
-	vpor		%ymm7,%ymm1,%ymm1
-
-	# x1 = shuffle32(x1, MASK(2, 1, 0, 3))
-	vpshufd		$0x93,%ymm1,%ymm1
-	# x2 = shuffle32(x2, MASK(1, 0, 3, 2))
-	vpshufd		$0x4e,%ymm2,%ymm2
-	# x3 = shuffle32(x3, MASK(0, 3, 2, 1))
-	vpshufd		$0x39,%ymm3,%ymm3
-
-	sub		$2,%r8d
-	jnz		.Ldoubleround
-
-	# o0 = i0 ^ (x0 + s0)
-	vpaddd		%ymm8,%ymm0,%ymm7
-	cmp		$0x10,%rax
-	jl		.Lxorpart2
-	vpxor		0x00(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x00(%rsi)
-	vextracti128	$1,%ymm7,%xmm0
-	# o1 = i1 ^ (x1 + s1)
-	vpaddd		%ymm9,%ymm1,%ymm7
-	cmp		$0x20,%rax
-	jl		.Lxorpart2
-	vpxor		0x10(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x10(%rsi)
-	vextracti128	$1,%ymm7,%xmm1
-	# o2 = i2 ^ (x2 + s2)
-	vpaddd		%ymm10,%ymm2,%ymm7
-	cmp		$0x30,%rax
-	jl		.Lxorpart2
-	vpxor		0x20(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x20(%rsi)
-	vextracti128	$1,%ymm7,%xmm2
-	# o3 = i3 ^ (x3 + s3)
-	vpaddd		%ymm11,%ymm3,%ymm7
-	cmp		$0x40,%rax
-	jl		.Lxorpart2
-	vpxor		0x30(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x30(%rsi)
-	vextracti128	$1,%ymm7,%xmm3
-
-	# xor and write second block
-	vmovdqa		%xmm0,%xmm7
-	cmp		$0x50,%rax
-	jl		.Lxorpart2
-	vpxor		0x40(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x40(%rsi)
-
-	vmovdqa		%xmm1,%xmm7
-	cmp		$0x60,%rax
-	jl		.Lxorpart2
-	vpxor		0x50(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x50(%rsi)
-
-	vmovdqa		%xmm2,%xmm7
-	cmp		$0x70,%rax
-	jl		.Lxorpart2
-	vpxor		0x60(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x60(%rsi)
-
-	vmovdqa		%xmm3,%xmm7
-	cmp		$0x80,%rax
-	jl		.Lxorpart2
-	vpxor		0x70(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x70(%rsi)
-
-.Ldone2:
-	vzeroupper
-	RET
-
-.Lxorpart2:
-	# xor remaining bytes from partial register into output
-	mov		%rax,%r9
-	and		$0x0f,%r9
-	jz		.Ldone2
-	and		$~0x0f,%rax
-
-	mov		%rsi,%r11
-
-	lea		8(%rsp),%r10
-	sub		$0x10,%rsp
-	and		$~31,%rsp
-
-	lea		(%rdx,%rax),%rsi
-	mov		%rsp,%rdi
-	mov		%r9,%rcx
-	rep movsb
-
-	vpxor		0x00(%rsp),%xmm7,%xmm7
-	vmovdqa		%xmm7,0x00(%rsp)
-
-	mov		%rsp,%rsi
-	lea		(%r11,%rax),%rdi
-	mov		%r9,%rcx
-	rep movsb
-
-	lea		-8(%r10),%rsp
-	jmp		.Ldone2
-
-SYM_FUNC_END(chacha_2block_xor_avx2)
-
-SYM_FUNC_START(chacha_4block_xor_avx2)
-	# %rdi: Input state matrix, s
-	# %rsi: up to 4 data blocks output, o
-	# %rdx: up to 4 data blocks input, i
-	# %rcx: input/output length in bytes
-	# %r8d: nrounds
-
-	# This function encrypts four ChaCha blocks by loading the state
-	# matrix four times across eight AVX registers. It performs matrix
-	# operations on four words in two matrices in parallel, sequentially
-	# to the operations on the four words of the other two matrices. The
-	# required word shuffling has a rather high latency, we can do the
-	# arithmetic on two matrix-pairs without much slowdown.
-
-	vzeroupper
-
-	# x0..3[0-4] = s0..3
-	vbroadcasti128	0x00(%rdi),%ymm0
-	vbroadcasti128	0x10(%rdi),%ymm1
-	vbroadcasti128	0x20(%rdi),%ymm2
-	vbroadcasti128	0x30(%rdi),%ymm3
-
-	vmovdqa		%ymm0,%ymm4
-	vmovdqa		%ymm1,%ymm5
-	vmovdqa		%ymm2,%ymm6
-	vmovdqa		%ymm3,%ymm7
-
-	vpaddd		CTR2BL(%rip),%ymm3,%ymm3
-	vpaddd		CTR4BL(%rip),%ymm7,%ymm7
-
-	vmovdqa		%ymm0,%ymm11
-	vmovdqa		%ymm1,%ymm12
-	vmovdqa		%ymm2,%ymm13
-	vmovdqa		%ymm3,%ymm14
-	vmovdqa		%ymm7,%ymm15
-
-	vmovdqa		ROT8(%rip),%ymm8
-	vmovdqa		ROT16(%rip),%ymm9
-
-	mov		%rcx,%rax
-
-.Ldoubleround4:
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 16)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxor		%ymm0,%ymm3,%ymm3
-	vpshufb		%ymm9,%ymm3,%ymm3
-
-	vpaddd		%ymm5,%ymm4,%ymm4
-	vpxor		%ymm4,%ymm7,%ymm7
-	vpshufb		%ymm9,%ymm7,%ymm7
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm10
-	vpslld		$12,%ymm10,%ymm10
-	vpsrld		$20,%ymm1,%ymm1
-	vpor		%ymm10,%ymm1,%ymm1
-
-	vpaddd		%ymm7,%ymm6,%ymm6
-	vpxor		%ymm6,%ymm5,%ymm5
-	vmovdqa		%ymm5,%ymm10
-	vpslld		$12,%ymm10,%ymm10
-	vpsrld		$20,%ymm5,%ymm5
-	vpor		%ymm10,%ymm5,%ymm5
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 8)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxor		%ymm0,%ymm3,%ymm3
-	vpshufb		%ymm8,%ymm3,%ymm3
-
-	vpaddd		%ymm5,%ymm4,%ymm4
-	vpxor		%ymm4,%ymm7,%ymm7
-	vpshufb		%ymm8,%ymm7,%ymm7
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm10
-	vpslld		$7,%ymm10,%ymm10
-	vpsrld		$25,%ymm1,%ymm1
-	vpor		%ymm10,%ymm1,%ymm1
-
-	vpaddd		%ymm7,%ymm6,%ymm6
-	vpxor		%ymm6,%ymm5,%ymm5
-	vmovdqa		%ymm5,%ymm10
-	vpslld		$7,%ymm10,%ymm10
-	vpsrld		$25,%ymm5,%ymm5
-	vpor		%ymm10,%ymm5,%ymm5
-
-	# x1 = shuffle32(x1, MASK(0, 3, 2, 1))
-	vpshufd		$0x39,%ymm1,%ymm1
-	vpshufd		$0x39,%ymm5,%ymm5
-	# x2 = shuffle32(x2, MASK(1, 0, 3, 2))
-	vpshufd		$0x4e,%ymm2,%ymm2
-	vpshufd		$0x4e,%ymm6,%ymm6
-	# x3 = shuffle32(x3, MASK(2, 1, 0, 3))
-	vpshufd		$0x93,%ymm3,%ymm3
-	vpshufd		$0x93,%ymm7,%ymm7
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 16)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxor		%ymm0,%ymm3,%ymm3
-	vpshufb		%ymm9,%ymm3,%ymm3
-
-	vpaddd		%ymm5,%ymm4,%ymm4
-	vpxor		%ymm4,%ymm7,%ymm7
-	vpshufb		%ymm9,%ymm7,%ymm7
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm10
-	vpslld		$12,%ymm10,%ymm10
-	vpsrld		$20,%ymm1,%ymm1
-	vpor		%ymm10,%ymm1,%ymm1
-
-	vpaddd		%ymm7,%ymm6,%ymm6
-	vpxor		%ymm6,%ymm5,%ymm5
-	vmovdqa		%ymm5,%ymm10
-	vpslld		$12,%ymm10,%ymm10
-	vpsrld		$20,%ymm5,%ymm5
-	vpor		%ymm10,%ymm5,%ymm5
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 8)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxor		%ymm0,%ymm3,%ymm3
-	vpshufb		%ymm8,%ymm3,%ymm3
-
-	vpaddd		%ymm5,%ymm4,%ymm4
-	vpxor		%ymm4,%ymm7,%ymm7
-	vpshufb		%ymm8,%ymm7,%ymm7
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm10
-	vpslld		$7,%ymm10,%ymm10
-	vpsrld		$25,%ymm1,%ymm1
-	vpor		%ymm10,%ymm1,%ymm1
-
-	vpaddd		%ymm7,%ymm6,%ymm6
-	vpxor		%ymm6,%ymm5,%ymm5
-	vmovdqa		%ymm5,%ymm10
-	vpslld		$7,%ymm10,%ymm10
-	vpsrld		$25,%ymm5,%ymm5
-	vpor		%ymm10,%ymm5,%ymm5
-
-	# x1 = shuffle32(x1, MASK(2, 1, 0, 3))
-	vpshufd		$0x93,%ymm1,%ymm1
-	vpshufd		$0x93,%ymm5,%ymm5
-	# x2 = shuffle32(x2, MASK(1, 0, 3, 2))
-	vpshufd		$0x4e,%ymm2,%ymm2
-	vpshufd		$0x4e,%ymm6,%ymm6
-	# x3 = shuffle32(x3, MASK(0, 3, 2, 1))
-	vpshufd		$0x39,%ymm3,%ymm3
-	vpshufd		$0x39,%ymm7,%ymm7
-
-	sub		$2,%r8d
-	jnz		.Ldoubleround4
-
-	# o0 = i0 ^ (x0 + s0), first block
-	vpaddd		%ymm11,%ymm0,%ymm10
-	cmp		$0x10,%rax
-	jl		.Lxorpart4
-	vpxor		0x00(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x00(%rsi)
-	vextracti128	$1,%ymm10,%xmm0
-	# o1 = i1 ^ (x1 + s1), first block
-	vpaddd		%ymm12,%ymm1,%ymm10
-	cmp		$0x20,%rax
-	jl		.Lxorpart4
-	vpxor		0x10(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x10(%rsi)
-	vextracti128	$1,%ymm10,%xmm1
-	# o2 = i2 ^ (x2 + s2), first block
-	vpaddd		%ymm13,%ymm2,%ymm10
-	cmp		$0x30,%rax
-	jl		.Lxorpart4
-	vpxor		0x20(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x20(%rsi)
-	vextracti128	$1,%ymm10,%xmm2
-	# o3 = i3 ^ (x3 + s3), first block
-	vpaddd		%ymm14,%ymm3,%ymm10
-	cmp		$0x40,%rax
-	jl		.Lxorpart4
-	vpxor		0x30(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x30(%rsi)
-	vextracti128	$1,%ymm10,%xmm3
-
-	# xor and write second block
-	vmovdqa		%xmm0,%xmm10
-	cmp		$0x50,%rax
-	jl		.Lxorpart4
-	vpxor		0x40(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x40(%rsi)
-
-	vmovdqa		%xmm1,%xmm10
-	cmp		$0x60,%rax
-	jl		.Lxorpart4
-	vpxor		0x50(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x50(%rsi)
-
-	vmovdqa		%xmm2,%xmm10
-	cmp		$0x70,%rax
-	jl		.Lxorpart4
-	vpxor		0x60(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x60(%rsi)
-
-	vmovdqa		%xmm3,%xmm10
-	cmp		$0x80,%rax
-	jl		.Lxorpart4
-	vpxor		0x70(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x70(%rsi)
-
-	# o0 = i0 ^ (x0 + s0), third block
-	vpaddd		%ymm11,%ymm4,%ymm10
-	cmp		$0x90,%rax
-	jl		.Lxorpart4
-	vpxor		0x80(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x80(%rsi)
-	vextracti128	$1,%ymm10,%xmm4
-	# o1 = i1 ^ (x1 + s1), third block
-	vpaddd		%ymm12,%ymm5,%ymm10
-	cmp		$0xa0,%rax
-	jl		.Lxorpart4
-	vpxor		0x90(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x90(%rsi)
-	vextracti128	$1,%ymm10,%xmm5
-	# o2 = i2 ^ (x2 + s2), third block
-	vpaddd		%ymm13,%ymm6,%ymm10
-	cmp		$0xb0,%rax
-	jl		.Lxorpart4
-	vpxor		0xa0(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0xa0(%rsi)
-	vextracti128	$1,%ymm10,%xmm6
-	# o3 = i3 ^ (x3 + s3), third block
-	vpaddd		%ymm15,%ymm7,%ymm10
-	cmp		$0xc0,%rax
-	jl		.Lxorpart4
-	vpxor		0xb0(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0xb0(%rsi)
-	vextracti128	$1,%ymm10,%xmm7
-
-	# xor and write fourth block
-	vmovdqa		%xmm4,%xmm10
-	cmp		$0xd0,%rax
-	jl		.Lxorpart4
-	vpxor		0xc0(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0xc0(%rsi)
-
-	vmovdqa		%xmm5,%xmm10
-	cmp		$0xe0,%rax
-	jl		.Lxorpart4
-	vpxor		0xd0(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0xd0(%rsi)
-
-	vmovdqa		%xmm6,%xmm10
-	cmp		$0xf0,%rax
-	jl		.Lxorpart4
-	vpxor		0xe0(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0xe0(%rsi)
-
-	vmovdqa		%xmm7,%xmm10
-	cmp		$0x100,%rax
-	jl		.Lxorpart4
-	vpxor		0xf0(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0xf0(%rsi)
-
-.Ldone4:
-	vzeroupper
-	RET
-
-.Lxorpart4:
-	# xor remaining bytes from partial register into output
-	mov		%rax,%r9
-	and		$0x0f,%r9
-	jz		.Ldone4
-	and		$~0x0f,%rax
-
-	mov		%rsi,%r11
-
-	lea		8(%rsp),%r10
-	sub		$0x10,%rsp
-	and		$~31,%rsp
-
-	lea		(%rdx,%rax),%rsi
-	mov		%rsp,%rdi
-	mov		%r9,%rcx
-	rep movsb
-
-	vpxor		0x00(%rsp),%xmm10,%xmm10
-	vmovdqa		%xmm10,0x00(%rsp)
-
-	mov		%rsp,%rsi
-	lea		(%r11,%rax),%rdi
-	mov		%r9,%rcx
-	rep movsb
-
-	lea		-8(%r10),%rsp
-	jmp		.Ldone4
-
-SYM_FUNC_END(chacha_4block_xor_avx2)
-
-SYM_FUNC_START(chacha_8block_xor_avx2)
-	# %rdi: Input state matrix, s
-	# %rsi: up to 8 data blocks output, o
-	# %rdx: up to 8 data blocks input, i
-	# %rcx: input/output length in bytes
-	# %r8d: nrounds
-
-	# This function encrypts eight consecutive ChaCha blocks by loading
-	# the state matrix in AVX registers eight times. As we need some
-	# scratch registers, we save the first four registers on the stack. The
-	# algorithm performs each operation on the corresponding word of each
-	# state matrix, hence requires no word shuffling. For final XORing step
-	# we transpose the matrix by interleaving 32-, 64- and then 128-bit
-	# words, which allows us to do XOR in AVX registers. 8/16-bit word
-	# rotation is done with the slightly better performing byte shuffling,
-	# 7/12-bit word rotation uses traditional shift+OR.
-
-	vzeroupper
-	# 4 * 32 byte stack, 32-byte aligned
-	lea		8(%rsp),%r10
-	and		$~31, %rsp
-	sub		$0x80, %rsp
-	mov		%rcx,%rax
-
-	# x0..15[0-7] = s[0..15]
-	vpbroadcastd	0x00(%rdi),%ymm0
-	vpbroadcastd	0x04(%rdi),%ymm1
-	vpbroadcastd	0x08(%rdi),%ymm2
-	vpbroadcastd	0x0c(%rdi),%ymm3
-	vpbroadcastd	0x10(%rdi),%ymm4
-	vpbroadcastd	0x14(%rdi),%ymm5
-	vpbroadcastd	0x18(%rdi),%ymm6
-	vpbroadcastd	0x1c(%rdi),%ymm7
-	vpbroadcastd	0x20(%rdi),%ymm8
-	vpbroadcastd	0x24(%rdi),%ymm9
-	vpbroadcastd	0x28(%rdi),%ymm10
-	vpbroadcastd	0x2c(%rdi),%ymm11
-	vpbroadcastd	0x30(%rdi),%ymm12
-	vpbroadcastd	0x34(%rdi),%ymm13
-	vpbroadcastd	0x38(%rdi),%ymm14
-	vpbroadcastd	0x3c(%rdi),%ymm15
-	# x0..3 on stack
-	vmovdqa		%ymm0,0x00(%rsp)
-	vmovdqa		%ymm1,0x20(%rsp)
-	vmovdqa		%ymm2,0x40(%rsp)
-	vmovdqa		%ymm3,0x60(%rsp)
-
-	vmovdqa		CTRINC(%rip),%ymm1
-	vmovdqa		ROT8(%rip),%ymm2
-	vmovdqa		ROT16(%rip),%ymm3
-
-	# x12 += counter values 0-3
-	vpaddd		%ymm1,%ymm12,%ymm12
-
-.Ldoubleround8:
-	# x0 += x4, x12 = rotl32(x12 ^ x0, 16)
-	vpaddd		0x00(%rsp),%ymm4,%ymm0
-	vmovdqa		%ymm0,0x00(%rsp)
-	vpxor		%ymm0,%ymm12,%ymm12
-	vpshufb		%ymm3,%ymm12,%ymm12
-	# x1 += x5, x13 = rotl32(x13 ^ x1, 16)
-	vpaddd		0x20(%rsp),%ymm5,%ymm0
-	vmovdqa		%ymm0,0x20(%rsp)
-	vpxor		%ymm0,%ymm13,%ymm13
-	vpshufb		%ymm3,%ymm13,%ymm13
-	# x2 += x6, x14 = rotl32(x14 ^ x2, 16)
-	vpaddd		0x40(%rsp),%ymm6,%ymm0
-	vmovdqa		%ymm0,0x40(%rsp)
-	vpxor		%ymm0,%ymm14,%ymm14
-	vpshufb		%ymm3,%ymm14,%ymm14
-	# x3 += x7, x15 = rotl32(x15 ^ x3, 16)
-	vpaddd		0x60(%rsp),%ymm7,%ymm0
-	vmovdqa		%ymm0,0x60(%rsp)
-	vpxor		%ymm0,%ymm15,%ymm15
-	vpshufb		%ymm3,%ymm15,%ymm15
-
-	# x8 += x12, x4 = rotl32(x4 ^ x8, 12)
-	vpaddd		%ymm12,%ymm8,%ymm8
-	vpxor		%ymm8,%ymm4,%ymm4
-	vpslld		$12,%ymm4,%ymm0
-	vpsrld		$20,%ymm4,%ymm4
-	vpor		%ymm0,%ymm4,%ymm4
-	# x9 += x13, x5 = rotl32(x5 ^ x9, 12)
-	vpaddd		%ymm13,%ymm9,%ymm9
-	vpxor		%ymm9,%ymm5,%ymm5
-	vpslld		$12,%ymm5,%ymm0
-	vpsrld		$20,%ymm5,%ymm5
-	vpor		%ymm0,%ymm5,%ymm5
-	# x10 += x14, x6 = rotl32(x6 ^ x10, 12)
-	vpaddd		%ymm14,%ymm10,%ymm10
-	vpxor		%ymm10,%ymm6,%ymm6
-	vpslld		$12,%ymm6,%ymm0
-	vpsrld		$20,%ymm6,%ymm6
-	vpor		%ymm0,%ymm6,%ymm6
-	# x11 += x15, x7 = rotl32(x7 ^ x11, 12)
-	vpaddd		%ymm15,%ymm11,%ymm11
-	vpxor		%ymm11,%ymm7,%ymm7
-	vpslld		$12,%ymm7,%ymm0
-	vpsrld		$20,%ymm7,%ymm7
-	vpor		%ymm0,%ymm7,%ymm7
-
-	# x0 += x4, x12 = rotl32(x12 ^ x0, 8)
-	vpaddd		0x00(%rsp),%ymm4,%ymm0
-	vmovdqa		%ymm0,0x00(%rsp)
-	vpxor		%ymm0,%ymm12,%ymm12
-	vpshufb		%ymm2,%ymm12,%ymm12
-	# x1 += x5, x13 = rotl32(x13 ^ x1, 8)
-	vpaddd		0x20(%rsp),%ymm5,%ymm0
-	vmovdqa		%ymm0,0x20(%rsp)
-	vpxor		%ymm0,%ymm13,%ymm13
-	vpshufb		%ymm2,%ymm13,%ymm13
-	# x2 += x6, x14 = rotl32(x14 ^ x2, 8)
-	vpaddd		0x40(%rsp),%ymm6,%ymm0
-	vmovdqa		%ymm0,0x40(%rsp)
-	vpxor		%ymm0,%ymm14,%ymm14
-	vpshufb		%ymm2,%ymm14,%ymm14
-	# x3 += x7, x15 = rotl32(x15 ^ x3, 8)
-	vpaddd		0x60(%rsp),%ymm7,%ymm0
-	vmovdqa		%ymm0,0x60(%rsp)
-	vpxor		%ymm0,%ymm15,%ymm15
-	vpshufb		%ymm2,%ymm15,%ymm15
-
-	# x8 += x12, x4 = rotl32(x4 ^ x8, 7)
-	vpaddd		%ymm12,%ymm8,%ymm8
-	vpxor		%ymm8,%ymm4,%ymm4
-	vpslld		$7,%ymm4,%ymm0
-	vpsrld		$25,%ymm4,%ymm4
-	vpor		%ymm0,%ymm4,%ymm4
-	# x9 += x13, x5 = rotl32(x5 ^ x9, 7)
-	vpaddd		%ymm13,%ymm9,%ymm9
-	vpxor		%ymm9,%ymm5,%ymm5
-	vpslld		$7,%ymm5,%ymm0
-	vpsrld		$25,%ymm5,%ymm5
-	vpor		%ymm0,%ymm5,%ymm5
-	# x10 += x14, x6 = rotl32(x6 ^ x10, 7)
-	vpaddd		%ymm14,%ymm10,%ymm10
-	vpxor		%ymm10,%ymm6,%ymm6
-	vpslld		$7,%ymm6,%ymm0
-	vpsrld		$25,%ymm6,%ymm6
-	vpor		%ymm0,%ymm6,%ymm6
-	# x11 += x15, x7 = rotl32(x7 ^ x11, 7)
-	vpaddd		%ymm15,%ymm11,%ymm11
-	vpxor		%ymm11,%ymm7,%ymm7
-	vpslld		$7,%ymm7,%ymm0
-	vpsrld		$25,%ymm7,%ymm7
-	vpor		%ymm0,%ymm7,%ymm7
-
-	# x0 += x5, x15 = rotl32(x15 ^ x0, 16)
-	vpaddd		0x00(%rsp),%ymm5,%ymm0
-	vmovdqa		%ymm0,0x00(%rsp)
-	vpxor		%ymm0,%ymm15,%ymm15
-	vpshufb		%ymm3,%ymm15,%ymm15
-	# x1 += x6, x12 = rotl32(x12 ^ x1, 16)%ymm0
-	vpaddd		0x20(%rsp),%ymm6,%ymm0
-	vmovdqa		%ymm0,0x20(%rsp)
-	vpxor		%ymm0,%ymm12,%ymm12
-	vpshufb		%ymm3,%ymm12,%ymm12
-	# x2 += x7, x13 = rotl32(x13 ^ x2, 16)
-	vpaddd		0x40(%rsp),%ymm7,%ymm0
-	vmovdqa		%ymm0,0x40(%rsp)
-	vpxor		%ymm0,%ymm13,%ymm13
-	vpshufb		%ymm3,%ymm13,%ymm13
-	# x3 += x4, x14 = rotl32(x14 ^ x3, 16)
-	vpaddd		0x60(%rsp),%ymm4,%ymm0
-	vmovdqa		%ymm0,0x60(%rsp)
-	vpxor		%ymm0,%ymm14,%ymm14
-	vpshufb		%ymm3,%ymm14,%ymm14
-
-	# x10 += x15, x5 = rotl32(x5 ^ x10, 12)
-	vpaddd		%ymm15,%ymm10,%ymm10
-	vpxor		%ymm10,%ymm5,%ymm5
-	vpslld		$12,%ymm5,%ymm0
-	vpsrld		$20,%ymm5,%ymm5
-	vpor		%ymm0,%ymm5,%ymm5
-	# x11 += x12, x6 = rotl32(x6 ^ x11, 12)
-	vpaddd		%ymm12,%ymm11,%ymm11
-	vpxor		%ymm11,%ymm6,%ymm6
-	vpslld		$12,%ymm6,%ymm0
-	vpsrld		$20,%ymm6,%ymm6
-	vpor		%ymm0,%ymm6,%ymm6
-	# x8 += x13, x7 = rotl32(x7 ^ x8, 12)
-	vpaddd		%ymm13,%ymm8,%ymm8
-	vpxor		%ymm8,%ymm7,%ymm7
-	vpslld		$12,%ymm7,%ymm0
-	vpsrld		$20,%ymm7,%ymm7
-	vpor		%ymm0,%ymm7,%ymm7
-	# x9 += x14, x4 = rotl32(x4 ^ x9, 12)
-	vpaddd		%ymm14,%ymm9,%ymm9
-	vpxor		%ymm9,%ymm4,%ymm4
-	vpslld		$12,%ymm4,%ymm0
-	vpsrld		$20,%ymm4,%ymm4
-	vpor		%ymm0,%ymm4,%ymm4
-
-	# x0 += x5, x15 = rotl32(x15 ^ x0, 8)
-	vpaddd		0x00(%rsp),%ymm5,%ymm0
-	vmovdqa		%ymm0,0x00(%rsp)
-	vpxor		%ymm0,%ymm15,%ymm15
-	vpshufb		%ymm2,%ymm15,%ymm15
-	# x1 += x6, x12 = rotl32(x12 ^ x1, 8)
-	vpaddd		0x20(%rsp),%ymm6,%ymm0
-	vmovdqa		%ymm0,0x20(%rsp)
-	vpxor		%ymm0,%ymm12,%ymm12
-	vpshufb		%ymm2,%ymm12,%ymm12
-	# x2 += x7, x13 = rotl32(x13 ^ x2, 8)
-	vpaddd		0x40(%rsp),%ymm7,%ymm0
-	vmovdqa		%ymm0,0x40(%rsp)
-	vpxor		%ymm0,%ymm13,%ymm13
-	vpshufb		%ymm2,%ymm13,%ymm13
-	# x3 += x4, x14 = rotl32(x14 ^ x3, 8)
-	vpaddd		0x60(%rsp),%ymm4,%ymm0
-	vmovdqa		%ymm0,0x60(%rsp)
-	vpxor		%ymm0,%ymm14,%ymm14
-	vpshufb		%ymm2,%ymm14,%ymm14
-
-	# x10 += x15, x5 = rotl32(x5 ^ x10, 7)
-	vpaddd		%ymm15,%ymm10,%ymm10
-	vpxor		%ymm10,%ymm5,%ymm5
-	vpslld		$7,%ymm5,%ymm0
-	vpsrld		$25,%ymm5,%ymm5
-	vpor		%ymm0,%ymm5,%ymm5
-	# x11 += x12, x6 = rotl32(x6 ^ x11, 7)
-	vpaddd		%ymm12,%ymm11,%ymm11
-	vpxor		%ymm11,%ymm6,%ymm6
-	vpslld		$7,%ymm6,%ymm0
-	vpsrld		$25,%ymm6,%ymm6
-	vpor		%ymm0,%ymm6,%ymm6
-	# x8 += x13, x7 = rotl32(x7 ^ x8, 7)
-	vpaddd		%ymm13,%ymm8,%ymm8
-	vpxor		%ymm8,%ymm7,%ymm7
-	vpslld		$7,%ymm7,%ymm0
-	vpsrld		$25,%ymm7,%ymm7
-	vpor		%ymm0,%ymm7,%ymm7
-	# x9 += x14, x4 = rotl32(x4 ^ x9, 7)
-	vpaddd		%ymm14,%ymm9,%ymm9
-	vpxor		%ymm9,%ymm4,%ymm4
-	vpslld		$7,%ymm4,%ymm0
-	vpsrld		$25,%ymm4,%ymm4
-	vpor		%ymm0,%ymm4,%ymm4
-
-	sub		$2,%r8d
-	jnz		.Ldoubleround8
-
-	# x0..15[0-3] += s[0..15]
-	vpbroadcastd	0x00(%rdi),%ymm0
-	vpaddd		0x00(%rsp),%ymm0,%ymm0
-	vmovdqa		%ymm0,0x00(%rsp)
-	vpbroadcastd	0x04(%rdi),%ymm0
-	vpaddd		0x20(%rsp),%ymm0,%ymm0
-	vmovdqa		%ymm0,0x20(%rsp)
-	vpbroadcastd	0x08(%rdi),%ymm0
-	vpaddd		0x40(%rsp),%ymm0,%ymm0
-	vmovdqa		%ymm0,0x40(%rsp)
-	vpbroadcastd	0x0c(%rdi),%ymm0
-	vpaddd		0x60(%rsp),%ymm0,%ymm0
-	vmovdqa		%ymm0,0x60(%rsp)
-	vpbroadcastd	0x10(%rdi),%ymm0
-	vpaddd		%ymm0,%ymm4,%ymm4
-	vpbroadcastd	0x14(%rdi),%ymm0
-	vpaddd		%ymm0,%ymm5,%ymm5
-	vpbroadcastd	0x18(%rdi),%ymm0
-	vpaddd		%ymm0,%ymm6,%ymm6
-	vpbroadcastd	0x1c(%rdi),%ymm0
-	vpaddd		%ymm0,%ymm7,%ymm7
-	vpbroadcastd	0x20(%rdi),%ymm0
-	vpaddd		%ymm0,%ymm8,%ymm8
-	vpbroadcastd	0x24(%rdi),%ymm0
-	vpaddd		%ymm0,%ymm9,%ymm9
-	vpbroadcastd	0x28(%rdi),%ymm0
-	vpaddd		%ymm0,%ymm10,%ymm10
-	vpbroadcastd	0x2c(%rdi),%ymm0
-	vpaddd		%ymm0,%ymm11,%ymm11
-	vpbroadcastd	0x30(%rdi),%ymm0
-	vpaddd		%ymm0,%ymm12,%ymm12
-	vpbroadcastd	0x34(%rdi),%ymm0
-	vpaddd		%ymm0,%ymm13,%ymm13
-	vpbroadcastd	0x38(%rdi),%ymm0
-	vpaddd		%ymm0,%ymm14,%ymm14
-	vpbroadcastd	0x3c(%rdi),%ymm0
-	vpaddd		%ymm0,%ymm15,%ymm15
-
-	# x12 += counter values 0-3
-	vpaddd		%ymm1,%ymm12,%ymm12
-
-	# interleave 32-bit words in state n, n+1
-	vmovdqa		0x00(%rsp),%ymm0
-	vmovdqa		0x20(%rsp),%ymm1
-	vpunpckldq	%ymm1,%ymm0,%ymm2
-	vpunpckhdq	%ymm1,%ymm0,%ymm1
-	vmovdqa		%ymm2,0x00(%rsp)
-	vmovdqa		%ymm1,0x20(%rsp)
-	vmovdqa		0x40(%rsp),%ymm0
-	vmovdqa		0x60(%rsp),%ymm1
-	vpunpckldq	%ymm1,%ymm0,%ymm2
-	vpunpckhdq	%ymm1,%ymm0,%ymm1
-	vmovdqa		%ymm2,0x40(%rsp)
-	vmovdqa		%ymm1,0x60(%rsp)
-	vmovdqa		%ymm4,%ymm0
-	vpunpckldq	%ymm5,%ymm0,%ymm4
-	vpunpckhdq	%ymm5,%ymm0,%ymm5
-	vmovdqa		%ymm6,%ymm0
-	vpunpckldq	%ymm7,%ymm0,%ymm6
-	vpunpckhdq	%ymm7,%ymm0,%ymm7
-	vmovdqa		%ymm8,%ymm0
-	vpunpckldq	%ymm9,%ymm0,%ymm8
-	vpunpckhdq	%ymm9,%ymm0,%ymm9
-	vmovdqa		%ymm10,%ymm0
-	vpunpckldq	%ymm11,%ymm0,%ymm10
-	vpunpckhdq	%ymm11,%ymm0,%ymm11
-	vmovdqa		%ymm12,%ymm0
-	vpunpckldq	%ymm13,%ymm0,%ymm12
-	vpunpckhdq	%ymm13,%ymm0,%ymm13
-	vmovdqa		%ymm14,%ymm0
-	vpunpckldq	%ymm15,%ymm0,%ymm14
-	vpunpckhdq	%ymm15,%ymm0,%ymm15
-
-	# interleave 64-bit words in state n, n+2
-	vmovdqa		0x00(%rsp),%ymm0
-	vmovdqa		0x40(%rsp),%ymm2
-	vpunpcklqdq	%ymm2,%ymm0,%ymm1
-	vpunpckhqdq	%ymm2,%ymm0,%ymm2
-	vmovdqa		%ymm1,0x00(%rsp)
-	vmovdqa		%ymm2,0x40(%rsp)
-	vmovdqa		0x20(%rsp),%ymm0
-	vmovdqa		0x60(%rsp),%ymm2
-	vpunpcklqdq	%ymm2,%ymm0,%ymm1
-	vpunpckhqdq	%ymm2,%ymm0,%ymm2
-	vmovdqa		%ymm1,0x20(%rsp)
-	vmovdqa		%ymm2,0x60(%rsp)
-	vmovdqa		%ymm4,%ymm0
-	vpunpcklqdq	%ymm6,%ymm0,%ymm4
-	vpunpckhqdq	%ymm6,%ymm0,%ymm6
-	vmovdqa		%ymm5,%ymm0
-	vpunpcklqdq	%ymm7,%ymm0,%ymm5
-	vpunpckhqdq	%ymm7,%ymm0,%ymm7
-	vmovdqa		%ymm8,%ymm0
-	vpunpcklqdq	%ymm10,%ymm0,%ymm8
-	vpunpckhqdq	%ymm10,%ymm0,%ymm10
-	vmovdqa		%ymm9,%ymm0
-	vpunpcklqdq	%ymm11,%ymm0,%ymm9
-	vpunpckhqdq	%ymm11,%ymm0,%ymm11
-	vmovdqa		%ymm12,%ymm0
-	vpunpcklqdq	%ymm14,%ymm0,%ymm12
-	vpunpckhqdq	%ymm14,%ymm0,%ymm14
-	vmovdqa		%ymm13,%ymm0
-	vpunpcklqdq	%ymm15,%ymm0,%ymm13
-	vpunpckhqdq	%ymm15,%ymm0,%ymm15
-
-	# interleave 128-bit words in state n, n+4
-	# xor/write first four blocks
-	vmovdqa		0x00(%rsp),%ymm1
-	vperm2i128	$0x20,%ymm4,%ymm1,%ymm0
-	cmp		$0x0020,%rax
-	jl		.Lxorpart8
-	vpxor		0x0000(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x0000(%rsi)
-	vperm2i128	$0x31,%ymm4,%ymm1,%ymm4
-
-	vperm2i128	$0x20,%ymm12,%ymm8,%ymm0
-	cmp		$0x0040,%rax
-	jl		.Lxorpart8
-	vpxor		0x0020(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x0020(%rsi)
-	vperm2i128	$0x31,%ymm12,%ymm8,%ymm12
-
-	vmovdqa		0x40(%rsp),%ymm1
-	vperm2i128	$0x20,%ymm6,%ymm1,%ymm0
-	cmp		$0x0060,%rax
-	jl		.Lxorpart8
-	vpxor		0x0040(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x0040(%rsi)
-	vperm2i128	$0x31,%ymm6,%ymm1,%ymm6
-
-	vperm2i128	$0x20,%ymm14,%ymm10,%ymm0
-	cmp		$0x0080,%rax
-	jl		.Lxorpart8
-	vpxor		0x0060(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x0060(%rsi)
-	vperm2i128	$0x31,%ymm14,%ymm10,%ymm14
-
-	vmovdqa		0x20(%rsp),%ymm1
-	vperm2i128	$0x20,%ymm5,%ymm1,%ymm0
-	cmp		$0x00a0,%rax
-	jl		.Lxorpart8
-	vpxor		0x0080(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x0080(%rsi)
-	vperm2i128	$0x31,%ymm5,%ymm1,%ymm5
-
-	vperm2i128	$0x20,%ymm13,%ymm9,%ymm0
-	cmp		$0x00c0,%rax
-	jl		.Lxorpart8
-	vpxor		0x00a0(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x00a0(%rsi)
-	vperm2i128	$0x31,%ymm13,%ymm9,%ymm13
-
-	vmovdqa		0x60(%rsp),%ymm1
-	vperm2i128	$0x20,%ymm7,%ymm1,%ymm0
-	cmp		$0x00e0,%rax
-	jl		.Lxorpart8
-	vpxor		0x00c0(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x00c0(%rsi)
-	vperm2i128	$0x31,%ymm7,%ymm1,%ymm7
-
-	vperm2i128	$0x20,%ymm15,%ymm11,%ymm0
-	cmp		$0x0100,%rax
-	jl		.Lxorpart8
-	vpxor		0x00e0(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x00e0(%rsi)
-	vperm2i128	$0x31,%ymm15,%ymm11,%ymm15
-
-	# xor remaining blocks, write to output
-	vmovdqa		%ymm4,%ymm0
-	cmp		$0x0120,%rax
-	jl		.Lxorpart8
-	vpxor		0x0100(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x0100(%rsi)
-
-	vmovdqa		%ymm12,%ymm0
-	cmp		$0x0140,%rax
-	jl		.Lxorpart8
-	vpxor		0x0120(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x0120(%rsi)
-
-	vmovdqa		%ymm6,%ymm0
-	cmp		$0x0160,%rax
-	jl		.Lxorpart8
-	vpxor		0x0140(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x0140(%rsi)
-
-	vmovdqa		%ymm14,%ymm0
-	cmp		$0x0180,%rax
-	jl		.Lxorpart8
-	vpxor		0x0160(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x0160(%rsi)
-
-	vmovdqa		%ymm5,%ymm0
-	cmp		$0x01a0,%rax
-	jl		.Lxorpart8
-	vpxor		0x0180(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x0180(%rsi)
-
-	vmovdqa		%ymm13,%ymm0
-	cmp		$0x01c0,%rax
-	jl		.Lxorpart8
-	vpxor		0x01a0(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x01a0(%rsi)
-
-	vmovdqa		%ymm7,%ymm0
-	cmp		$0x01e0,%rax
-	jl		.Lxorpart8
-	vpxor		0x01c0(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x01c0(%rsi)
-
-	vmovdqa		%ymm15,%ymm0
-	cmp		$0x0200,%rax
-	jl		.Lxorpart8
-	vpxor		0x01e0(%rdx),%ymm0,%ymm0
-	vmovdqu		%ymm0,0x01e0(%rsi)
-
-.Ldone8:
-	vzeroupper
-	lea		-8(%r10),%rsp
-	RET
-
-.Lxorpart8:
-	# xor remaining bytes from partial register into output
-	mov		%rax,%r9
-	and		$0x1f,%r9
-	jz		.Ldone8
-	and		$~0x1f,%rax
-
-	mov		%rsi,%r11
-
-	lea		(%rdx,%rax),%rsi
-	mov		%rsp,%rdi
-	mov		%r9,%rcx
-	rep movsb
-
-	vpxor		0x00(%rsp),%ymm0,%ymm0
-	vmovdqa		%ymm0,0x00(%rsp)
-
-	mov		%rsp,%rsi
-	lea		(%r11,%rax),%rdi
-	mov		%r9,%rcx
-	rep movsb
-
-	jmp		.Ldone8
-
-SYM_FUNC_END(chacha_8block_xor_avx2)
diff --git a/arch/x86/crypto/chacha-avx512vl-x86_64.S b/arch/x86/crypto/chacha-avx512vl-x86_64.S
deleted file mode 100644
index 259383e1ad44..000000000000
--- a/arch/x86/crypto/chacha-avx512vl-x86_64.S
+++ /dev/null
@@ -1,836 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * ChaCha 256-bit cipher algorithm, x64 AVX-512VL functions
- *
- * Copyright (C) 2018 Martin Willi
- */
-
-#include <linux/linkage.h>
-
-.section	.rodata.cst32.CTR2BL, "aM", @progbits, 32
-.align 32
-CTR2BL:	.octa 0x00000000000000000000000000000000
-	.octa 0x00000000000000000000000000000001
-
-.section	.rodata.cst32.CTR4BL, "aM", @progbits, 32
-.align 32
-CTR4BL:	.octa 0x00000000000000000000000000000002
-	.octa 0x00000000000000000000000000000003
-
-.section	.rodata.cst32.CTR8BL, "aM", @progbits, 32
-.align 32
-CTR8BL:	.octa 0x00000003000000020000000100000000
-	.octa 0x00000007000000060000000500000004
-
-.text
-
-SYM_FUNC_START(chacha_2block_xor_avx512vl)
-	# %rdi: Input state matrix, s
-	# %rsi: up to 2 data blocks output, o
-	# %rdx: up to 2 data blocks input, i
-	# %rcx: input/output length in bytes
-	# %r8d: nrounds
-
-	# This function encrypts two ChaCha blocks by loading the state
-	# matrix twice across four AVX registers. It performs matrix operations
-	# on four words in each matrix in parallel, but requires shuffling to
-	# rearrange the words after each round.
-
-	vzeroupper
-
-	# x0..3[0-2] = s0..3
-	vbroadcasti128	0x00(%rdi),%ymm0
-	vbroadcasti128	0x10(%rdi),%ymm1
-	vbroadcasti128	0x20(%rdi),%ymm2
-	vbroadcasti128	0x30(%rdi),%ymm3
-
-	vpaddd		CTR2BL(%rip),%ymm3,%ymm3
-
-	vmovdqa		%ymm0,%ymm8
-	vmovdqa		%ymm1,%ymm9
-	vmovdqa		%ymm2,%ymm10
-	vmovdqa		%ymm3,%ymm11
-
-.Ldoubleround:
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 16)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxord		%ymm0,%ymm3,%ymm3
-	vprold		$16,%ymm3,%ymm3
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxord		%ymm2,%ymm1,%ymm1
-	vprold		$12,%ymm1,%ymm1
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 8)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxord		%ymm0,%ymm3,%ymm3
-	vprold		$8,%ymm3,%ymm3
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxord		%ymm2,%ymm1,%ymm1
-	vprold		$7,%ymm1,%ymm1
-
-	# x1 = shuffle32(x1, MASK(0, 3, 2, 1))
-	vpshufd		$0x39,%ymm1,%ymm1
-	# x2 = shuffle32(x2, MASK(1, 0, 3, 2))
-	vpshufd		$0x4e,%ymm2,%ymm2
-	# x3 = shuffle32(x3, MASK(2, 1, 0, 3))
-	vpshufd		$0x93,%ymm3,%ymm3
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 16)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxord		%ymm0,%ymm3,%ymm3
-	vprold		$16,%ymm3,%ymm3
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxord		%ymm2,%ymm1,%ymm1
-	vprold		$12,%ymm1,%ymm1
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 8)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxord		%ymm0,%ymm3,%ymm3
-	vprold		$8,%ymm3,%ymm3
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxord		%ymm2,%ymm1,%ymm1
-	vprold		$7,%ymm1,%ymm1
-
-	# x1 = shuffle32(x1, MASK(2, 1, 0, 3))
-	vpshufd		$0x93,%ymm1,%ymm1
-	# x2 = shuffle32(x2, MASK(1, 0, 3, 2))
-	vpshufd		$0x4e,%ymm2,%ymm2
-	# x3 = shuffle32(x3, MASK(0, 3, 2, 1))
-	vpshufd		$0x39,%ymm3,%ymm3
-
-	sub		$2,%r8d
-	jnz		.Ldoubleround
-
-	# o0 = i0 ^ (x0 + s0)
-	vpaddd		%ymm8,%ymm0,%ymm7
-	cmp		$0x10,%rcx
-	jl		.Lxorpart2
-	vpxord		0x00(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x00(%rsi)
-	vextracti128	$1,%ymm7,%xmm0
-	# o1 = i1 ^ (x1 + s1)
-	vpaddd		%ymm9,%ymm1,%ymm7
-	cmp		$0x20,%rcx
-	jl		.Lxorpart2
-	vpxord		0x10(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x10(%rsi)
-	vextracti128	$1,%ymm7,%xmm1
-	# o2 = i2 ^ (x2 + s2)
-	vpaddd		%ymm10,%ymm2,%ymm7
-	cmp		$0x30,%rcx
-	jl		.Lxorpart2
-	vpxord		0x20(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x20(%rsi)
-	vextracti128	$1,%ymm7,%xmm2
-	# o3 = i3 ^ (x3 + s3)
-	vpaddd		%ymm11,%ymm3,%ymm7
-	cmp		$0x40,%rcx
-	jl		.Lxorpart2
-	vpxord		0x30(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x30(%rsi)
-	vextracti128	$1,%ymm7,%xmm3
-
-	# xor and write second block
-	vmovdqa		%xmm0,%xmm7
-	cmp		$0x50,%rcx
-	jl		.Lxorpart2
-	vpxord		0x40(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x40(%rsi)
-
-	vmovdqa		%xmm1,%xmm7
-	cmp		$0x60,%rcx
-	jl		.Lxorpart2
-	vpxord		0x50(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x50(%rsi)
-
-	vmovdqa		%xmm2,%xmm7
-	cmp		$0x70,%rcx
-	jl		.Lxorpart2
-	vpxord		0x60(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x60(%rsi)
-
-	vmovdqa		%xmm3,%xmm7
-	cmp		$0x80,%rcx
-	jl		.Lxorpart2
-	vpxord		0x70(%rdx),%xmm7,%xmm6
-	vmovdqu		%xmm6,0x70(%rsi)
-
-.Ldone2:
-	vzeroupper
-	RET
-
-.Lxorpart2:
-	# xor remaining bytes from partial register into output
-	mov		%rcx,%rax
-	and		$0xf,%rcx
-	jz		.Ldone2
-	mov		%rax,%r9
-	and		$~0xf,%r9
-
-	mov		$1,%rax
-	shld		%cl,%rax,%rax
-	sub		$1,%rax
-	kmovq		%rax,%k1
-
-	vmovdqu8	(%rdx,%r9),%xmm1{%k1}{z}
-	vpxord		%xmm7,%xmm1,%xmm1
-	vmovdqu8	%xmm1,(%rsi,%r9){%k1}
-
-	jmp		.Ldone2
-
-SYM_FUNC_END(chacha_2block_xor_avx512vl)
-
-SYM_FUNC_START(chacha_4block_xor_avx512vl)
-	# %rdi: Input state matrix, s
-	# %rsi: up to 4 data blocks output, o
-	# %rdx: up to 4 data blocks input, i
-	# %rcx: input/output length in bytes
-	# %r8d: nrounds
-
-	# This function encrypts four ChaCha blocks by loading the state
-	# matrix four times across eight AVX registers. It performs matrix
-	# operations on four words in two matrices in parallel, sequentially
-	# to the operations on the four words of the other two matrices. The
-	# required word shuffling has a rather high latency, we can do the
-	# arithmetic on two matrix-pairs without much slowdown.
-
-	vzeroupper
-
-	# x0..3[0-4] = s0..3
-	vbroadcasti128	0x00(%rdi),%ymm0
-	vbroadcasti128	0x10(%rdi),%ymm1
-	vbroadcasti128	0x20(%rdi),%ymm2
-	vbroadcasti128	0x30(%rdi),%ymm3
-
-	vmovdqa		%ymm0,%ymm4
-	vmovdqa		%ymm1,%ymm5
-	vmovdqa		%ymm2,%ymm6
-	vmovdqa		%ymm3,%ymm7
-
-	vpaddd		CTR2BL(%rip),%ymm3,%ymm3
-	vpaddd		CTR4BL(%rip),%ymm7,%ymm7
-
-	vmovdqa		%ymm0,%ymm11
-	vmovdqa		%ymm1,%ymm12
-	vmovdqa		%ymm2,%ymm13
-	vmovdqa		%ymm3,%ymm14
-	vmovdqa		%ymm7,%ymm15
-
-.Ldoubleround4:
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 16)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxord		%ymm0,%ymm3,%ymm3
-	vprold		$16,%ymm3,%ymm3
-
-	vpaddd		%ymm5,%ymm4,%ymm4
-	vpxord		%ymm4,%ymm7,%ymm7
-	vprold		$16,%ymm7,%ymm7
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxord		%ymm2,%ymm1,%ymm1
-	vprold		$12,%ymm1,%ymm1
-
-	vpaddd		%ymm7,%ymm6,%ymm6
-	vpxord		%ymm6,%ymm5,%ymm5
-	vprold		$12,%ymm5,%ymm5
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 8)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxord		%ymm0,%ymm3,%ymm3
-	vprold		$8,%ymm3,%ymm3
-
-	vpaddd		%ymm5,%ymm4,%ymm4
-	vpxord		%ymm4,%ymm7,%ymm7
-	vprold		$8,%ymm7,%ymm7
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxord		%ymm2,%ymm1,%ymm1
-	vprold		$7,%ymm1,%ymm1
-
-	vpaddd		%ymm7,%ymm6,%ymm6
-	vpxord		%ymm6,%ymm5,%ymm5
-	vprold		$7,%ymm5,%ymm5
-
-	# x1 = shuffle32(x1, MASK(0, 3, 2, 1))
-	vpshufd		$0x39,%ymm1,%ymm1
-	vpshufd		$0x39,%ymm5,%ymm5
-	# x2 = shuffle32(x2, MASK(1, 0, 3, 2))
-	vpshufd		$0x4e,%ymm2,%ymm2
-	vpshufd		$0x4e,%ymm6,%ymm6
-	# x3 = shuffle32(x3, MASK(2, 1, 0, 3))
-	vpshufd		$0x93,%ymm3,%ymm3
-	vpshufd		$0x93,%ymm7,%ymm7
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 16)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxord		%ymm0,%ymm3,%ymm3
-	vprold		$16,%ymm3,%ymm3
-
-	vpaddd		%ymm5,%ymm4,%ymm4
-	vpxord		%ymm4,%ymm7,%ymm7
-	vprold		$16,%ymm7,%ymm7
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxord		%ymm2,%ymm1,%ymm1
-	vprold		$12,%ymm1,%ymm1
-
-	vpaddd		%ymm7,%ymm6,%ymm6
-	vpxord		%ymm6,%ymm5,%ymm5
-	vprold		$12,%ymm5,%ymm5
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 8)
-	vpaddd		%ymm1,%ymm0,%ymm0
-	vpxord		%ymm0,%ymm3,%ymm3
-	vprold		$8,%ymm3,%ymm3
-
-	vpaddd		%ymm5,%ymm4,%ymm4
-	vpxord		%ymm4,%ymm7,%ymm7
-	vprold		$8,%ymm7,%ymm7
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
-	vpaddd		%ymm3,%ymm2,%ymm2
-	vpxord		%ymm2,%ymm1,%ymm1
-	vprold		$7,%ymm1,%ymm1
-
-	vpaddd		%ymm7,%ymm6,%ymm6
-	vpxord		%ymm6,%ymm5,%ymm5
-	vprold		$7,%ymm5,%ymm5
-
-	# x1 = shuffle32(x1, MASK(2, 1, 0, 3))
-	vpshufd		$0x93,%ymm1,%ymm1
-	vpshufd		$0x93,%ymm5,%ymm5
-	# x2 = shuffle32(x2, MASK(1, 0, 3, 2))
-	vpshufd		$0x4e,%ymm2,%ymm2
-	vpshufd		$0x4e,%ymm6,%ymm6
-	# x3 = shuffle32(x3, MASK(0, 3, 2, 1))
-	vpshufd		$0x39,%ymm3,%ymm3
-	vpshufd		$0x39,%ymm7,%ymm7
-
-	sub		$2,%r8d
-	jnz		.Ldoubleround4
-
-	# o0 = i0 ^ (x0 + s0), first block
-	vpaddd		%ymm11,%ymm0,%ymm10
-	cmp		$0x10,%rcx
-	jl		.Lxorpart4
-	vpxord		0x00(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x00(%rsi)
-	vextracti128	$1,%ymm10,%xmm0
-	# o1 = i1 ^ (x1 + s1), first block
-	vpaddd		%ymm12,%ymm1,%ymm10
-	cmp		$0x20,%rcx
-	jl		.Lxorpart4
-	vpxord		0x10(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x10(%rsi)
-	vextracti128	$1,%ymm10,%xmm1
-	# o2 = i2 ^ (x2 + s2), first block
-	vpaddd		%ymm13,%ymm2,%ymm10
-	cmp		$0x30,%rcx
-	jl		.Lxorpart4
-	vpxord		0x20(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x20(%rsi)
-	vextracti128	$1,%ymm10,%xmm2
-	# o3 = i3 ^ (x3 + s3), first block
-	vpaddd		%ymm14,%ymm3,%ymm10
-	cmp		$0x40,%rcx
-	jl		.Lxorpart4
-	vpxord		0x30(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x30(%rsi)
-	vextracti128	$1,%ymm10,%xmm3
-
-	# xor and write second block
-	vmovdqa		%xmm0,%xmm10
-	cmp		$0x50,%rcx
-	jl		.Lxorpart4
-	vpxord		0x40(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x40(%rsi)
-
-	vmovdqa		%xmm1,%xmm10
-	cmp		$0x60,%rcx
-	jl		.Lxorpart4
-	vpxord		0x50(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x50(%rsi)
-
-	vmovdqa		%xmm2,%xmm10
-	cmp		$0x70,%rcx
-	jl		.Lxorpart4
-	vpxord		0x60(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x60(%rsi)
-
-	vmovdqa		%xmm3,%xmm10
-	cmp		$0x80,%rcx
-	jl		.Lxorpart4
-	vpxord		0x70(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x70(%rsi)
-
-	# o0 = i0 ^ (x0 + s0), third block
-	vpaddd		%ymm11,%ymm4,%ymm10
-	cmp		$0x90,%rcx
-	jl		.Lxorpart4
-	vpxord		0x80(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x80(%rsi)
-	vextracti128	$1,%ymm10,%xmm4
-	# o1 = i1 ^ (x1 + s1), third block
-	vpaddd		%ymm12,%ymm5,%ymm10
-	cmp		$0xa0,%rcx
-	jl		.Lxorpart4
-	vpxord		0x90(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0x90(%rsi)
-	vextracti128	$1,%ymm10,%xmm5
-	# o2 = i2 ^ (x2 + s2), third block
-	vpaddd		%ymm13,%ymm6,%ymm10
-	cmp		$0xb0,%rcx
-	jl		.Lxorpart4
-	vpxord		0xa0(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0xa0(%rsi)
-	vextracti128	$1,%ymm10,%xmm6
-	# o3 = i3 ^ (x3 + s3), third block
-	vpaddd		%ymm15,%ymm7,%ymm10
-	cmp		$0xc0,%rcx
-	jl		.Lxorpart4
-	vpxord		0xb0(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0xb0(%rsi)
-	vextracti128	$1,%ymm10,%xmm7
-
-	# xor and write fourth block
-	vmovdqa		%xmm4,%xmm10
-	cmp		$0xd0,%rcx
-	jl		.Lxorpart4
-	vpxord		0xc0(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0xc0(%rsi)
-
-	vmovdqa		%xmm5,%xmm10
-	cmp		$0xe0,%rcx
-	jl		.Lxorpart4
-	vpxord		0xd0(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0xd0(%rsi)
-
-	vmovdqa		%xmm6,%xmm10
-	cmp		$0xf0,%rcx
-	jl		.Lxorpart4
-	vpxord		0xe0(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0xe0(%rsi)
-
-	vmovdqa		%xmm7,%xmm10
-	cmp		$0x100,%rcx
-	jl		.Lxorpart4
-	vpxord		0xf0(%rdx),%xmm10,%xmm9
-	vmovdqu		%xmm9,0xf0(%rsi)
-
-.Ldone4:
-	vzeroupper
-	RET
-
-.Lxorpart4:
-	# xor remaining bytes from partial register into output
-	mov		%rcx,%rax
-	and		$0xf,%rcx
-	jz		.Ldone4
-	mov		%rax,%r9
-	and		$~0xf,%r9
-
-	mov		$1,%rax
-	shld		%cl,%rax,%rax
-	sub		$1,%rax
-	kmovq		%rax,%k1
-
-	vmovdqu8	(%rdx,%r9),%xmm1{%k1}{z}
-	vpxord		%xmm10,%xmm1,%xmm1
-	vmovdqu8	%xmm1,(%rsi,%r9){%k1}
-
-	jmp		.Ldone4
-
-SYM_FUNC_END(chacha_4block_xor_avx512vl)
-
-SYM_FUNC_START(chacha_8block_xor_avx512vl)
-	# %rdi: Input state matrix, s
-	# %rsi: up to 8 data blocks output, o
-	# %rdx: up to 8 data blocks input, i
-	# %rcx: input/output length in bytes
-	# %r8d: nrounds
-
-	# This function encrypts eight consecutive ChaCha blocks by loading
-	# the state matrix in AVX registers eight times. Compared to AVX2, this
-	# mostly benefits from the new rotate instructions in VL and the
-	# additional registers.
-
-	vzeroupper
-
-	# x0..15[0-7] = s[0..15]
-	vpbroadcastd	0x00(%rdi),%ymm0
-	vpbroadcastd	0x04(%rdi),%ymm1
-	vpbroadcastd	0x08(%rdi),%ymm2
-	vpbroadcastd	0x0c(%rdi),%ymm3
-	vpbroadcastd	0x10(%rdi),%ymm4
-	vpbroadcastd	0x14(%rdi),%ymm5
-	vpbroadcastd	0x18(%rdi),%ymm6
-	vpbroadcastd	0x1c(%rdi),%ymm7
-	vpbroadcastd	0x20(%rdi),%ymm8
-	vpbroadcastd	0x24(%rdi),%ymm9
-	vpbroadcastd	0x28(%rdi),%ymm10
-	vpbroadcastd	0x2c(%rdi),%ymm11
-	vpbroadcastd	0x30(%rdi),%ymm12
-	vpbroadcastd	0x34(%rdi),%ymm13
-	vpbroadcastd	0x38(%rdi),%ymm14
-	vpbroadcastd	0x3c(%rdi),%ymm15
-
-	# x12 += counter values 0-3
-	vpaddd		CTR8BL(%rip),%ymm12,%ymm12
-
-	vmovdqa64	%ymm0,%ymm16
-	vmovdqa64	%ymm1,%ymm17
-	vmovdqa64	%ymm2,%ymm18
-	vmovdqa64	%ymm3,%ymm19
-	vmovdqa64	%ymm4,%ymm20
-	vmovdqa64	%ymm5,%ymm21
-	vmovdqa64	%ymm6,%ymm22
-	vmovdqa64	%ymm7,%ymm23
-	vmovdqa64	%ymm8,%ymm24
-	vmovdqa64	%ymm9,%ymm25
-	vmovdqa64	%ymm10,%ymm26
-	vmovdqa64	%ymm11,%ymm27
-	vmovdqa64	%ymm12,%ymm28
-	vmovdqa64	%ymm13,%ymm29
-	vmovdqa64	%ymm14,%ymm30
-	vmovdqa64	%ymm15,%ymm31
-
-.Ldoubleround8:
-	# x0 += x4, x12 = rotl32(x12 ^ x0, 16)
-	vpaddd		%ymm0,%ymm4,%ymm0
-	vpxord		%ymm0,%ymm12,%ymm12
-	vprold		$16,%ymm12,%ymm12
-	# x1 += x5, x13 = rotl32(x13 ^ x1, 16)
-	vpaddd		%ymm1,%ymm5,%ymm1
-	vpxord		%ymm1,%ymm13,%ymm13
-	vprold		$16,%ymm13,%ymm13
-	# x2 += x6, x14 = rotl32(x14 ^ x2, 16)
-	vpaddd		%ymm2,%ymm6,%ymm2
-	vpxord		%ymm2,%ymm14,%ymm14
-	vprold		$16,%ymm14,%ymm14
-	# x3 += x7, x15 = rotl32(x15 ^ x3, 16)
-	vpaddd		%ymm3,%ymm7,%ymm3
-	vpxord		%ymm3,%ymm15,%ymm15
-	vprold		$16,%ymm15,%ymm15
-
-	# x8 += x12, x4 = rotl32(x4 ^ x8, 12)
-	vpaddd		%ymm12,%ymm8,%ymm8
-	vpxord		%ymm8,%ymm4,%ymm4
-	vprold		$12,%ymm4,%ymm4
-	# x9 += x13, x5 = rotl32(x5 ^ x9, 12)
-	vpaddd		%ymm13,%ymm9,%ymm9
-	vpxord		%ymm9,%ymm5,%ymm5
-	vprold		$12,%ymm5,%ymm5
-	# x10 += x14, x6 = rotl32(x6 ^ x10, 12)
-	vpaddd		%ymm14,%ymm10,%ymm10
-	vpxord		%ymm10,%ymm6,%ymm6
-	vprold		$12,%ymm6,%ymm6
-	# x11 += x15, x7 = rotl32(x7 ^ x11, 12)
-	vpaddd		%ymm15,%ymm11,%ymm11
-	vpxord		%ymm11,%ymm7,%ymm7
-	vprold		$12,%ymm7,%ymm7
-
-	# x0 += x4, x12 = rotl32(x12 ^ x0, 8)
-	vpaddd		%ymm0,%ymm4,%ymm0
-	vpxord		%ymm0,%ymm12,%ymm12
-	vprold		$8,%ymm12,%ymm12
-	# x1 += x5, x13 = rotl32(x13 ^ x1, 8)
-	vpaddd		%ymm1,%ymm5,%ymm1
-	vpxord		%ymm1,%ymm13,%ymm13
-	vprold		$8,%ymm13,%ymm13
-	# x2 += x6, x14 = rotl32(x14 ^ x2, 8)
-	vpaddd		%ymm2,%ymm6,%ymm2
-	vpxord		%ymm2,%ymm14,%ymm14
-	vprold		$8,%ymm14,%ymm14
-	# x3 += x7, x15 = rotl32(x15 ^ x3, 8)
-	vpaddd		%ymm3,%ymm7,%ymm3
-	vpxord		%ymm3,%ymm15,%ymm15
-	vprold		$8,%ymm15,%ymm15
-
-	# x8 += x12, x4 = rotl32(x4 ^ x8, 7)
-	vpaddd		%ymm12,%ymm8,%ymm8
-	vpxord		%ymm8,%ymm4,%ymm4
-	vprold		$7,%ymm4,%ymm4
-	# x9 += x13, x5 = rotl32(x5 ^ x9, 7)
-	vpaddd		%ymm13,%ymm9,%ymm9
-	vpxord		%ymm9,%ymm5,%ymm5
-	vprold		$7,%ymm5,%ymm5
-	# x10 += x14, x6 = rotl32(x6 ^ x10, 7)
-	vpaddd		%ymm14,%ymm10,%ymm10
-	vpxord		%ymm10,%ymm6,%ymm6
-	vprold		$7,%ymm6,%ymm6
-	# x11 += x15, x7 = rotl32(x7 ^ x11, 7)
-	vpaddd		%ymm15,%ymm11,%ymm11
-	vpxord		%ymm11,%ymm7,%ymm7
-	vprold		$7,%ymm7,%ymm7
-
-	# x0 += x5, x15 = rotl32(x15 ^ x0, 16)
-	vpaddd		%ymm0,%ymm5,%ymm0
-	vpxord		%ymm0,%ymm15,%ymm15
-	vprold		$16,%ymm15,%ymm15
-	# x1 += x6, x12 = rotl32(x12 ^ x1, 16)
-	vpaddd		%ymm1,%ymm6,%ymm1
-	vpxord		%ymm1,%ymm12,%ymm12
-	vprold		$16,%ymm12,%ymm12
-	# x2 += x7, x13 = rotl32(x13 ^ x2, 16)
-	vpaddd		%ymm2,%ymm7,%ymm2
-	vpxord		%ymm2,%ymm13,%ymm13
-	vprold		$16,%ymm13,%ymm13
-	# x3 += x4, x14 = rotl32(x14 ^ x3, 16)
-	vpaddd		%ymm3,%ymm4,%ymm3
-	vpxord		%ymm3,%ymm14,%ymm14
-	vprold		$16,%ymm14,%ymm14
-
-	# x10 += x15, x5 = rotl32(x5 ^ x10, 12)
-	vpaddd		%ymm15,%ymm10,%ymm10
-	vpxord		%ymm10,%ymm5,%ymm5
-	vprold		$12,%ymm5,%ymm5
-	# x11 += x12, x6 = rotl32(x6 ^ x11, 12)
-	vpaddd		%ymm12,%ymm11,%ymm11
-	vpxord		%ymm11,%ymm6,%ymm6
-	vprold		$12,%ymm6,%ymm6
-	# x8 += x13, x7 = rotl32(x7 ^ x8, 12)
-	vpaddd		%ymm13,%ymm8,%ymm8
-	vpxord		%ymm8,%ymm7,%ymm7
-	vprold		$12,%ymm7,%ymm7
-	# x9 += x14, x4 = rotl32(x4 ^ x9, 12)
-	vpaddd		%ymm14,%ymm9,%ymm9
-	vpxord		%ymm9,%ymm4,%ymm4
-	vprold		$12,%ymm4,%ymm4
-
-	# x0 += x5, x15 = rotl32(x15 ^ x0, 8)
-	vpaddd		%ymm0,%ymm5,%ymm0
-	vpxord		%ymm0,%ymm15,%ymm15
-	vprold		$8,%ymm15,%ymm15
-	# x1 += x6, x12 = rotl32(x12 ^ x1, 8)
-	vpaddd		%ymm1,%ymm6,%ymm1
-	vpxord		%ymm1,%ymm12,%ymm12
-	vprold		$8,%ymm12,%ymm12
-	# x2 += x7, x13 = rotl32(x13 ^ x2, 8)
-	vpaddd		%ymm2,%ymm7,%ymm2
-	vpxord		%ymm2,%ymm13,%ymm13
-	vprold		$8,%ymm13,%ymm13
-	# x3 += x4, x14 = rotl32(x14 ^ x3, 8)
-	vpaddd		%ymm3,%ymm4,%ymm3
-	vpxord		%ymm3,%ymm14,%ymm14
-	vprold		$8,%ymm14,%ymm14
-
-	# x10 += x15, x5 = rotl32(x5 ^ x10, 7)
-	vpaddd		%ymm15,%ymm10,%ymm10
-	vpxord		%ymm10,%ymm5,%ymm5
-	vprold		$7,%ymm5,%ymm5
-	# x11 += x12, x6 = rotl32(x6 ^ x11, 7)
-	vpaddd		%ymm12,%ymm11,%ymm11
-	vpxord		%ymm11,%ymm6,%ymm6
-	vprold		$7,%ymm6,%ymm6
-	# x8 += x13, x7 = rotl32(x7 ^ x8, 7)
-	vpaddd		%ymm13,%ymm8,%ymm8
-	vpxord		%ymm8,%ymm7,%ymm7
-	vprold		$7,%ymm7,%ymm7
-	# x9 += x14, x4 = rotl32(x4 ^ x9, 7)
-	vpaddd		%ymm14,%ymm9,%ymm9
-	vpxord		%ymm9,%ymm4,%ymm4
-	vprold		$7,%ymm4,%ymm4
-
-	sub		$2,%r8d
-	jnz		.Ldoubleround8
-
-	# x0..15[0-3] += s[0..15]
-	vpaddd		%ymm16,%ymm0,%ymm0
-	vpaddd		%ymm17,%ymm1,%ymm1
-	vpaddd		%ymm18,%ymm2,%ymm2
-	vpaddd		%ymm19,%ymm3,%ymm3
-	vpaddd		%ymm20,%ymm4,%ymm4
-	vpaddd		%ymm21,%ymm5,%ymm5
-	vpaddd		%ymm22,%ymm6,%ymm6
-	vpaddd		%ymm23,%ymm7,%ymm7
-	vpaddd		%ymm24,%ymm8,%ymm8
-	vpaddd		%ymm25,%ymm9,%ymm9
-	vpaddd		%ymm26,%ymm10,%ymm10
-	vpaddd		%ymm27,%ymm11,%ymm11
-	vpaddd		%ymm28,%ymm12,%ymm12
-	vpaddd		%ymm29,%ymm13,%ymm13
-	vpaddd		%ymm30,%ymm14,%ymm14
-	vpaddd		%ymm31,%ymm15,%ymm15
-
-	# interleave 32-bit words in state n, n+1
-	vpunpckldq	%ymm1,%ymm0,%ymm16
-	vpunpckhdq	%ymm1,%ymm0,%ymm17
-	vpunpckldq	%ymm3,%ymm2,%ymm18
-	vpunpckhdq	%ymm3,%ymm2,%ymm19
-	vpunpckldq	%ymm5,%ymm4,%ymm20
-	vpunpckhdq	%ymm5,%ymm4,%ymm21
-	vpunpckldq	%ymm7,%ymm6,%ymm22
-	vpunpckhdq	%ymm7,%ymm6,%ymm23
-	vpunpckldq	%ymm9,%ymm8,%ymm24
-	vpunpckhdq	%ymm9,%ymm8,%ymm25
-	vpunpckldq	%ymm11,%ymm10,%ymm26
-	vpunpckhdq	%ymm11,%ymm10,%ymm27
-	vpunpckldq	%ymm13,%ymm12,%ymm28
-	vpunpckhdq	%ymm13,%ymm12,%ymm29
-	vpunpckldq	%ymm15,%ymm14,%ymm30
-	vpunpckhdq	%ymm15,%ymm14,%ymm31
-
-	# interleave 64-bit words in state n, n+2
-	vpunpcklqdq	%ymm18,%ymm16,%ymm0
-	vpunpcklqdq	%ymm19,%ymm17,%ymm1
-	vpunpckhqdq	%ymm18,%ymm16,%ymm2
-	vpunpckhqdq	%ymm19,%ymm17,%ymm3
-	vpunpcklqdq	%ymm22,%ymm20,%ymm4
-	vpunpcklqdq	%ymm23,%ymm21,%ymm5
-	vpunpckhqdq	%ymm22,%ymm20,%ymm6
-	vpunpckhqdq	%ymm23,%ymm21,%ymm7
-	vpunpcklqdq	%ymm26,%ymm24,%ymm8
-	vpunpcklqdq	%ymm27,%ymm25,%ymm9
-	vpunpckhqdq	%ymm26,%ymm24,%ymm10
-	vpunpckhqdq	%ymm27,%ymm25,%ymm11
-	vpunpcklqdq	%ymm30,%ymm28,%ymm12
-	vpunpcklqdq	%ymm31,%ymm29,%ymm13
-	vpunpckhqdq	%ymm30,%ymm28,%ymm14
-	vpunpckhqdq	%ymm31,%ymm29,%ymm15
-
-	# interleave 128-bit words in state n, n+4
-	# xor/write first four blocks
-	vmovdqa64	%ymm0,%ymm16
-	vperm2i128	$0x20,%ymm4,%ymm0,%ymm0
-	cmp		$0x0020,%rcx
-	jl		.Lxorpart8
-	vpxord		0x0000(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x0000(%rsi)
-	vmovdqa64	%ymm16,%ymm0
-	vperm2i128	$0x31,%ymm4,%ymm0,%ymm4
-
-	vperm2i128	$0x20,%ymm12,%ymm8,%ymm0
-	cmp		$0x0040,%rcx
-	jl		.Lxorpart8
-	vpxord		0x0020(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x0020(%rsi)
-	vperm2i128	$0x31,%ymm12,%ymm8,%ymm12
-
-	vperm2i128	$0x20,%ymm6,%ymm2,%ymm0
-	cmp		$0x0060,%rcx
-	jl		.Lxorpart8
-	vpxord		0x0040(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x0040(%rsi)
-	vperm2i128	$0x31,%ymm6,%ymm2,%ymm6
-
-	vperm2i128	$0x20,%ymm14,%ymm10,%ymm0
-	cmp		$0x0080,%rcx
-	jl		.Lxorpart8
-	vpxord		0x0060(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x0060(%rsi)
-	vperm2i128	$0x31,%ymm14,%ymm10,%ymm14
-
-	vperm2i128	$0x20,%ymm5,%ymm1,%ymm0
-	cmp		$0x00a0,%rcx
-	jl		.Lxorpart8
-	vpxord		0x0080(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x0080(%rsi)
-	vperm2i128	$0x31,%ymm5,%ymm1,%ymm5
-
-	vperm2i128	$0x20,%ymm13,%ymm9,%ymm0
-	cmp		$0x00c0,%rcx
-	jl		.Lxorpart8
-	vpxord		0x00a0(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x00a0(%rsi)
-	vperm2i128	$0x31,%ymm13,%ymm9,%ymm13
-
-	vperm2i128	$0x20,%ymm7,%ymm3,%ymm0
-	cmp		$0x00e0,%rcx
-	jl		.Lxorpart8
-	vpxord		0x00c0(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x00c0(%rsi)
-	vperm2i128	$0x31,%ymm7,%ymm3,%ymm7
-
-	vperm2i128	$0x20,%ymm15,%ymm11,%ymm0
-	cmp		$0x0100,%rcx
-	jl		.Lxorpart8
-	vpxord		0x00e0(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x00e0(%rsi)
-	vperm2i128	$0x31,%ymm15,%ymm11,%ymm15
-
-	# xor remaining blocks, write to output
-	vmovdqa64	%ymm4,%ymm0
-	cmp		$0x0120,%rcx
-	jl		.Lxorpart8
-	vpxord		0x0100(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x0100(%rsi)
-
-	vmovdqa64	%ymm12,%ymm0
-	cmp		$0x0140,%rcx
-	jl		.Lxorpart8
-	vpxord		0x0120(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x0120(%rsi)
-
-	vmovdqa64	%ymm6,%ymm0
-	cmp		$0x0160,%rcx
-	jl		.Lxorpart8
-	vpxord		0x0140(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x0140(%rsi)
-
-	vmovdqa64	%ymm14,%ymm0
-	cmp		$0x0180,%rcx
-	jl		.Lxorpart8
-	vpxord		0x0160(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x0160(%rsi)
-
-	vmovdqa64	%ymm5,%ymm0
-	cmp		$0x01a0,%rcx
-	jl		.Lxorpart8
-	vpxord		0x0180(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x0180(%rsi)
-
-	vmovdqa64	%ymm13,%ymm0
-	cmp		$0x01c0,%rcx
-	jl		.Lxorpart8
-	vpxord		0x01a0(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x01a0(%rsi)
-
-	vmovdqa64	%ymm7,%ymm0
-	cmp		$0x01e0,%rcx
-	jl		.Lxorpart8
-	vpxord		0x01c0(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x01c0(%rsi)
-
-	vmovdqa64	%ymm15,%ymm0
-	cmp		$0x0200,%rcx
-	jl		.Lxorpart8
-	vpxord		0x01e0(%rdx),%ymm0,%ymm0
-	vmovdqu64	%ymm0,0x01e0(%rsi)
-
-.Ldone8:
-	vzeroupper
-	RET
-
-.Lxorpart8:
-	# xor remaining bytes from partial register into output
-	mov		%rcx,%rax
-	and		$0x1f,%rcx
-	jz		.Ldone8
-	mov		%rax,%r9
-	and		$~0x1f,%r9
-
-	mov		$1,%rax
-	shld		%cl,%rax,%rax
-	sub		$1,%rax
-	kmovq		%rax,%k1
-
-	vmovdqu8	(%rdx,%r9),%ymm1{%k1}{z}
-	vpxord		%ymm0,%ymm1,%ymm1
-	vmovdqu8	%ymm1,(%rsi,%r9){%k1}
-
-	jmp		.Ldone8
-
-SYM_FUNC_END(chacha_8block_xor_avx512vl)
diff --git a/arch/x86/crypto/chacha-ssse3-x86_64.S b/arch/x86/crypto/chacha-ssse3-x86_64.S
deleted file mode 100644
index 7111949cd5b9..000000000000
--- a/arch/x86/crypto/chacha-ssse3-x86_64.S
+++ /dev/null
@@ -1,791 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * ChaCha 256-bit cipher algorithm, x64 SSSE3 functions
- *
- * Copyright (C) 2015 Martin Willi
- */
-
-#include <linux/linkage.h>
-#include <asm/frame.h>
-
-.section	.rodata.cst16.ROT8, "aM", @progbits, 16
-.align 16
-ROT8:	.octa 0x0e0d0c0f0a09080b0605040702010003
-.section	.rodata.cst16.ROT16, "aM", @progbits, 16
-.align 16
-ROT16:	.octa 0x0d0c0f0e09080b0a0504070601000302
-.section	.rodata.cst16.CTRINC, "aM", @progbits, 16
-.align 16
-CTRINC:	.octa 0x00000003000000020000000100000000
-
-.text
-
-/*
- * chacha_permute - permute one block
- *
- * Permute one 64-byte block where the state matrix is in %xmm0-%xmm3.  This
- * function performs matrix operations on four words in parallel, but requires
- * shuffling to rearrange the words after each round.  8/16-bit word rotation is
- * done with the slightly better performing SSSE3 byte shuffling, 7/12-bit word
- * rotation uses traditional shift+OR.
- *
- * The round count is given in %r8d.
- *
- * Clobbers: %r8d, %xmm4-%xmm7
- */
-SYM_FUNC_START_LOCAL(chacha_permute)
-
-	movdqa		ROT8(%rip),%xmm4
-	movdqa		ROT16(%rip),%xmm5
-
-.Ldoubleround:
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 16)
-	paddd		%xmm1,%xmm0
-	pxor		%xmm0,%xmm3
-	pshufb		%xmm5,%xmm3
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
-	paddd		%xmm3,%xmm2
-	pxor		%xmm2,%xmm1
-	movdqa		%xmm1,%xmm6
-	pslld		$12,%xmm6
-	psrld		$20,%xmm1
-	por		%xmm6,%xmm1
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 8)
-	paddd		%xmm1,%xmm0
-	pxor		%xmm0,%xmm3
-	pshufb		%xmm4,%xmm3
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
-	paddd		%xmm3,%xmm2
-	pxor		%xmm2,%xmm1
-	movdqa		%xmm1,%xmm7
-	pslld		$7,%xmm7
-	psrld		$25,%xmm1
-	por		%xmm7,%xmm1
-
-	# x1 = shuffle32(x1, MASK(0, 3, 2, 1))
-	pshufd		$0x39,%xmm1,%xmm1
-	# x2 = shuffle32(x2, MASK(1, 0, 3, 2))
-	pshufd		$0x4e,%xmm2,%xmm2
-	# x3 = shuffle32(x3, MASK(2, 1, 0, 3))
-	pshufd		$0x93,%xmm3,%xmm3
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 16)
-	paddd		%xmm1,%xmm0
-	pxor		%xmm0,%xmm3
-	pshufb		%xmm5,%xmm3
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
-	paddd		%xmm3,%xmm2
-	pxor		%xmm2,%xmm1
-	movdqa		%xmm1,%xmm6
-	pslld		$12,%xmm6
-	psrld		$20,%xmm1
-	por		%xmm6,%xmm1
-
-	# x0 += x1, x3 = rotl32(x3 ^ x0, 8)
-	paddd		%xmm1,%xmm0
-	pxor		%xmm0,%xmm3
-	pshufb		%xmm4,%xmm3
-
-	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
-	paddd		%xmm3,%xmm2
-	pxor		%xmm2,%xmm1
-	movdqa		%xmm1,%xmm7
-	pslld		$7,%xmm7
-	psrld		$25,%xmm1
-	por		%xmm7,%xmm1
-
-	# x1 = shuffle32(x1, MASK(2, 1, 0, 3))
-	pshufd		$0x93,%xmm1,%xmm1
-	# x2 = shuffle32(x2, MASK(1, 0, 3, 2))
-	pshufd		$0x4e,%xmm2,%xmm2
-	# x3 = shuffle32(x3, MASK(0, 3, 2, 1))
-	pshufd		$0x39,%xmm3,%xmm3
-
-	sub		$2,%r8d
-	jnz		.Ldoubleround
-
-	RET
-SYM_FUNC_END(chacha_permute)
-
-SYM_FUNC_START(chacha_block_xor_ssse3)
-	# %rdi: Input state matrix, s
-	# %rsi: up to 1 data block output, o
-	# %rdx: up to 1 data block input, i
-	# %rcx: input/output length in bytes
-	# %r8d: nrounds
-	FRAME_BEGIN
-
-	# x0..3 = s0..3
-	movdqu		0x00(%rdi),%xmm0
-	movdqu		0x10(%rdi),%xmm1
-	movdqu		0x20(%rdi),%xmm2
-	movdqu		0x30(%rdi),%xmm3
-	movdqa		%xmm0,%xmm8
-	movdqa		%xmm1,%xmm9
-	movdqa		%xmm2,%xmm10
-	movdqa		%xmm3,%xmm11
-
-	mov		%rcx,%rax
-	call		chacha_permute
-
-	# o0 = i0 ^ (x0 + s0)
-	paddd		%xmm8,%xmm0
-	cmp		$0x10,%rax
-	jl		.Lxorpart
-	movdqu		0x00(%rdx),%xmm4
-	pxor		%xmm4,%xmm0
-	movdqu		%xmm0,0x00(%rsi)
-	# o1 = i1 ^ (x1 + s1)
-	paddd		%xmm9,%xmm1
-	movdqa		%xmm1,%xmm0
-	cmp		$0x20,%rax
-	jl		.Lxorpart
-	movdqu		0x10(%rdx),%xmm0
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0x10(%rsi)
-	# o2 = i2 ^ (x2 + s2)
-	paddd		%xmm10,%xmm2
-	movdqa		%xmm2,%xmm0
-	cmp		$0x30,%rax
-	jl		.Lxorpart
-	movdqu		0x20(%rdx),%xmm0
-	pxor		%xmm2,%xmm0
-	movdqu		%xmm0,0x20(%rsi)
-	# o3 = i3 ^ (x3 + s3)
-	paddd		%xmm11,%xmm3
-	movdqa		%xmm3,%xmm0
-	cmp		$0x40,%rax
-	jl		.Lxorpart
-	movdqu		0x30(%rdx),%xmm0
-	pxor		%xmm3,%xmm0
-	movdqu		%xmm0,0x30(%rsi)
-
-.Ldone:
-	FRAME_END
-	RET
-
-.Lxorpart:
-	# xor remaining bytes from partial register into output
-	mov		%rax,%r9
-	and		$0x0f,%r9
-	jz		.Ldone
-	and		$~0x0f,%rax
-
-	mov		%rsi,%r11
-
-	lea		8(%rsp),%r10
-	sub		$0x10,%rsp
-	and		$~31,%rsp
-
-	lea		(%rdx,%rax),%rsi
-	mov		%rsp,%rdi
-	mov		%r9,%rcx
-	rep movsb
-
-	pxor		0x00(%rsp),%xmm0
-	movdqa		%xmm0,0x00(%rsp)
-
-	mov		%rsp,%rsi
-	lea		(%r11,%rax),%rdi
-	mov		%r9,%rcx
-	rep movsb
-
-	lea		-8(%r10),%rsp
-	jmp		.Ldone
-
-SYM_FUNC_END(chacha_block_xor_ssse3)
-
-SYM_FUNC_START(hchacha_block_ssse3)
-	# %rdi: Input state matrix, s
-	# %rsi: output (8 32-bit words)
-	# %edx: nrounds
-	FRAME_BEGIN
-
-	movdqu		0x00(%rdi),%xmm0
-	movdqu		0x10(%rdi),%xmm1
-	movdqu		0x20(%rdi),%xmm2
-	movdqu		0x30(%rdi),%xmm3
-
-	mov		%edx,%r8d
-	call		chacha_permute
-
-	movdqu		%xmm0,0x00(%rsi)
-	movdqu		%xmm3,0x10(%rsi)
-
-	FRAME_END
-	RET
-SYM_FUNC_END(hchacha_block_ssse3)
-
-SYM_FUNC_START(chacha_4block_xor_ssse3)
-	# %rdi: Input state matrix, s
-	# %rsi: up to 4 data blocks output, o
-	# %rdx: up to 4 data blocks input, i
-	# %rcx: input/output length in bytes
-	# %r8d: nrounds
-
-	# This function encrypts four consecutive ChaCha blocks by loading the
-	# the state matrix in SSE registers four times. As we need some scratch
-	# registers, we save the first four registers on the stack. The
-	# algorithm performs each operation on the corresponding word of each
-	# state matrix, hence requires no word shuffling. For final XORing step
-	# we transpose the matrix by interleaving 32- and then 64-bit words,
-	# which allows us to do XOR in SSE registers. 8/16-bit word rotation is
-	# done with the slightly better performing SSSE3 byte shuffling,
-	# 7/12-bit word rotation uses traditional shift+OR.
-
-	lea		8(%rsp),%r10
-	sub		$0x80,%rsp
-	and		$~63,%rsp
-	mov		%rcx,%rax
-
-	# x0..15[0-3] = s0..3[0..3]
-	movq		0x00(%rdi),%xmm1
-	pshufd		$0x00,%xmm1,%xmm0
-	pshufd		$0x55,%xmm1,%xmm1
-	movq		0x08(%rdi),%xmm3
-	pshufd		$0x00,%xmm3,%xmm2
-	pshufd		$0x55,%xmm3,%xmm3
-	movq		0x10(%rdi),%xmm5
-	pshufd		$0x00,%xmm5,%xmm4
-	pshufd		$0x55,%xmm5,%xmm5
-	movq		0x18(%rdi),%xmm7
-	pshufd		$0x00,%xmm7,%xmm6
-	pshufd		$0x55,%xmm7,%xmm7
-	movq		0x20(%rdi),%xmm9
-	pshufd		$0x00,%xmm9,%xmm8
-	pshufd		$0x55,%xmm9,%xmm9
-	movq		0x28(%rdi),%xmm11
-	pshufd		$0x00,%xmm11,%xmm10
-	pshufd		$0x55,%xmm11,%xmm11
-	movq		0x30(%rdi),%xmm13
-	pshufd		$0x00,%xmm13,%xmm12
-	pshufd		$0x55,%xmm13,%xmm13
-	movq		0x38(%rdi),%xmm15
-	pshufd		$0x00,%xmm15,%xmm14
-	pshufd		$0x55,%xmm15,%xmm15
-	# x0..3 on stack
-	movdqa		%xmm0,0x00(%rsp)
-	movdqa		%xmm1,0x10(%rsp)
-	movdqa		%xmm2,0x20(%rsp)
-	movdqa		%xmm3,0x30(%rsp)
-
-	movdqa		CTRINC(%rip),%xmm1
-	movdqa		ROT8(%rip),%xmm2
-	movdqa		ROT16(%rip),%xmm3
-
-	# x12 += counter values 0-3
-	paddd		%xmm1,%xmm12
-
-.Ldoubleround4:
-	# x0 += x4, x12 = rotl32(x12 ^ x0, 16)
-	movdqa		0x00(%rsp),%xmm0
-	paddd		%xmm4,%xmm0
-	movdqa		%xmm0,0x00(%rsp)
-	pxor		%xmm0,%xmm12
-	pshufb		%xmm3,%xmm12
-	# x1 += x5, x13 = rotl32(x13 ^ x1, 16)
-	movdqa		0x10(%rsp),%xmm0
-	paddd		%xmm5,%xmm0
-	movdqa		%xmm0,0x10(%rsp)
-	pxor		%xmm0,%xmm13
-	pshufb		%xmm3,%xmm13
-	# x2 += x6, x14 = rotl32(x14 ^ x2, 16)
-	movdqa		0x20(%rsp),%xmm0
-	paddd		%xmm6,%xmm0
-	movdqa		%xmm0,0x20(%rsp)
-	pxor		%xmm0,%xmm14
-	pshufb		%xmm3,%xmm14
-	# x3 += x7, x15 = rotl32(x15 ^ x3, 16)
-	movdqa		0x30(%rsp),%xmm0
-	paddd		%xmm7,%xmm0
-	movdqa		%xmm0,0x30(%rsp)
-	pxor		%xmm0,%xmm15
-	pshufb		%xmm3,%xmm15
-
-	# x8 += x12, x4 = rotl32(x4 ^ x8, 12)
-	paddd		%xmm12,%xmm8
-	pxor		%xmm8,%xmm4
-	movdqa		%xmm4,%xmm0
-	pslld		$12,%xmm0
-	psrld		$20,%xmm4
-	por		%xmm0,%xmm4
-	# x9 += x13, x5 = rotl32(x5 ^ x9, 12)
-	paddd		%xmm13,%xmm9
-	pxor		%xmm9,%xmm5
-	movdqa		%xmm5,%xmm0
-	pslld		$12,%xmm0
-	psrld		$20,%xmm5
-	por		%xmm0,%xmm5
-	# x10 += x14, x6 = rotl32(x6 ^ x10, 12)
-	paddd		%xmm14,%xmm10
-	pxor		%xmm10,%xmm6
-	movdqa		%xmm6,%xmm0
-	pslld		$12,%xmm0
-	psrld		$20,%xmm6
-	por		%xmm0,%xmm6
-	# x11 += x15, x7 = rotl32(x7 ^ x11, 12)
-	paddd		%xmm15,%xmm11
-	pxor		%xmm11,%xmm7
-	movdqa		%xmm7,%xmm0
-	pslld		$12,%xmm0
-	psrld		$20,%xmm7
-	por		%xmm0,%xmm7
-
-	# x0 += x4, x12 = rotl32(x12 ^ x0, 8)
-	movdqa		0x00(%rsp),%xmm0
-	paddd		%xmm4,%xmm0
-	movdqa		%xmm0,0x00(%rsp)
-	pxor		%xmm0,%xmm12
-	pshufb		%xmm2,%xmm12
-	# x1 += x5, x13 = rotl32(x13 ^ x1, 8)
-	movdqa		0x10(%rsp),%xmm0
-	paddd		%xmm5,%xmm0
-	movdqa		%xmm0,0x10(%rsp)
-	pxor		%xmm0,%xmm13
-	pshufb		%xmm2,%xmm13
-	# x2 += x6, x14 = rotl32(x14 ^ x2, 8)
-	movdqa		0x20(%rsp),%xmm0
-	paddd		%xmm6,%xmm0
-	movdqa		%xmm0,0x20(%rsp)
-	pxor		%xmm0,%xmm14
-	pshufb		%xmm2,%xmm14
-	# x3 += x7, x15 = rotl32(x15 ^ x3, 8)
-	movdqa		0x30(%rsp),%xmm0
-	paddd		%xmm7,%xmm0
-	movdqa		%xmm0,0x30(%rsp)
-	pxor		%xmm0,%xmm15
-	pshufb		%xmm2,%xmm15
-
-	# x8 += x12, x4 = rotl32(x4 ^ x8, 7)
-	paddd		%xmm12,%xmm8
-	pxor		%xmm8,%xmm4
-	movdqa		%xmm4,%xmm0
-	pslld		$7,%xmm0
-	psrld		$25,%xmm4
-	por		%xmm0,%xmm4
-	# x9 += x13, x5 = rotl32(x5 ^ x9, 7)
-	paddd		%xmm13,%xmm9
-	pxor		%xmm9,%xmm5
-	movdqa		%xmm5,%xmm0
-	pslld		$7,%xmm0
-	psrld		$25,%xmm5
-	por		%xmm0,%xmm5
-	# x10 += x14, x6 = rotl32(x6 ^ x10, 7)
-	paddd		%xmm14,%xmm10
-	pxor		%xmm10,%xmm6
-	movdqa		%xmm6,%xmm0
-	pslld		$7,%xmm0
-	psrld		$25,%xmm6
-	por		%xmm0,%xmm6
-	# x11 += x15, x7 = rotl32(x7 ^ x11, 7)
-	paddd		%xmm15,%xmm11
-	pxor		%xmm11,%xmm7
-	movdqa		%xmm7,%xmm0
-	pslld		$7,%xmm0
-	psrld		$25,%xmm7
-	por		%xmm0,%xmm7
-
-	# x0 += x5, x15 = rotl32(x15 ^ x0, 16)
-	movdqa		0x00(%rsp),%xmm0
-	paddd		%xmm5,%xmm0
-	movdqa		%xmm0,0x00(%rsp)
-	pxor		%xmm0,%xmm15
-	pshufb		%xmm3,%xmm15
-	# x1 += x6, x12 = rotl32(x12 ^ x1, 16)
-	movdqa		0x10(%rsp),%xmm0
-	paddd		%xmm6,%xmm0
-	movdqa		%xmm0,0x10(%rsp)
-	pxor		%xmm0,%xmm12
-	pshufb		%xmm3,%xmm12
-	# x2 += x7, x13 = rotl32(x13 ^ x2, 16)
-	movdqa		0x20(%rsp),%xmm0
-	paddd		%xmm7,%xmm0
-	movdqa		%xmm0,0x20(%rsp)
-	pxor		%xmm0,%xmm13
-	pshufb		%xmm3,%xmm13
-	# x3 += x4, x14 = rotl32(x14 ^ x3, 16)
-	movdqa		0x30(%rsp),%xmm0
-	paddd		%xmm4,%xmm0
-	movdqa		%xmm0,0x30(%rsp)
-	pxor		%xmm0,%xmm14
-	pshufb		%xmm3,%xmm14
-
-	# x10 += x15, x5 = rotl32(x5 ^ x10, 12)
-	paddd		%xmm15,%xmm10
-	pxor		%xmm10,%xmm5
-	movdqa		%xmm5,%xmm0
-	pslld		$12,%xmm0
-	psrld		$20,%xmm5
-	por		%xmm0,%xmm5
-	# x11 += x12, x6 = rotl32(x6 ^ x11, 12)
-	paddd		%xmm12,%xmm11
-	pxor		%xmm11,%xmm6
-	movdqa		%xmm6,%xmm0
-	pslld		$12,%xmm0
-	psrld		$20,%xmm6
-	por		%xmm0,%xmm6
-	# x8 += x13, x7 = rotl32(x7 ^ x8, 12)
-	paddd		%xmm13,%xmm8
-	pxor		%xmm8,%xmm7
-	movdqa		%xmm7,%xmm0
-	pslld		$12,%xmm0
-	psrld		$20,%xmm7
-	por		%xmm0,%xmm7
-	# x9 += x14, x4 = rotl32(x4 ^ x9, 12)
-	paddd		%xmm14,%xmm9
-	pxor		%xmm9,%xmm4
-	movdqa		%xmm4,%xmm0
-	pslld		$12,%xmm0
-	psrld		$20,%xmm4
-	por		%xmm0,%xmm4
-
-	# x0 += x5, x15 = rotl32(x15 ^ x0, 8)
-	movdqa		0x00(%rsp),%xmm0
-	paddd		%xmm5,%xmm0
-	movdqa		%xmm0,0x00(%rsp)
-	pxor		%xmm0,%xmm15
-	pshufb		%xmm2,%xmm15
-	# x1 += x6, x12 = rotl32(x12 ^ x1, 8)
-	movdqa		0x10(%rsp),%xmm0
-	paddd		%xmm6,%xmm0
-	movdqa		%xmm0,0x10(%rsp)
-	pxor		%xmm0,%xmm12
-	pshufb		%xmm2,%xmm12
-	# x2 += x7, x13 = rotl32(x13 ^ x2, 8)
-	movdqa		0x20(%rsp),%xmm0
-	paddd		%xmm7,%xmm0
-	movdqa		%xmm0,0x20(%rsp)
-	pxor		%xmm0,%xmm13
-	pshufb		%xmm2,%xmm13
-	# x3 += x4, x14 = rotl32(x14 ^ x3, 8)
-	movdqa		0x30(%rsp),%xmm0
-	paddd		%xmm4,%xmm0
-	movdqa		%xmm0,0x30(%rsp)
-	pxor		%xmm0,%xmm14
-	pshufb		%xmm2,%xmm14
-
-	# x10 += x15, x5 = rotl32(x5 ^ x10, 7)
-	paddd		%xmm15,%xmm10
-	pxor		%xmm10,%xmm5
-	movdqa		%xmm5,%xmm0
-	pslld		$7,%xmm0
-	psrld		$25,%xmm5
-	por		%xmm0,%xmm5
-	# x11 += x12, x6 = rotl32(x6 ^ x11, 7)
-	paddd		%xmm12,%xmm11
-	pxor		%xmm11,%xmm6
-	movdqa		%xmm6,%xmm0
-	pslld		$7,%xmm0
-	psrld		$25,%xmm6
-	por		%xmm0,%xmm6
-	# x8 += x13, x7 = rotl32(x7 ^ x8, 7)
-	paddd		%xmm13,%xmm8
-	pxor		%xmm8,%xmm7
-	movdqa		%xmm7,%xmm0
-	pslld		$7,%xmm0
-	psrld		$25,%xmm7
-	por		%xmm0,%xmm7
-	# x9 += x14, x4 = rotl32(x4 ^ x9, 7)
-	paddd		%xmm14,%xmm9
-	pxor		%xmm9,%xmm4
-	movdqa		%xmm4,%xmm0
-	pslld		$7,%xmm0
-	psrld		$25,%xmm4
-	por		%xmm0,%xmm4
-
-	sub		$2,%r8d
-	jnz		.Ldoubleround4
-
-	# x0[0-3] += s0[0]
-	# x1[0-3] += s0[1]
-	movq		0x00(%rdi),%xmm3
-	pshufd		$0x00,%xmm3,%xmm2
-	pshufd		$0x55,%xmm3,%xmm3
-	paddd		0x00(%rsp),%xmm2
-	movdqa		%xmm2,0x00(%rsp)
-	paddd		0x10(%rsp),%xmm3
-	movdqa		%xmm3,0x10(%rsp)
-	# x2[0-3] += s0[2]
-	# x3[0-3] += s0[3]
-	movq		0x08(%rdi),%xmm3
-	pshufd		$0x00,%xmm3,%xmm2
-	pshufd		$0x55,%xmm3,%xmm3
-	paddd		0x20(%rsp),%xmm2
-	movdqa		%xmm2,0x20(%rsp)
-	paddd		0x30(%rsp),%xmm3
-	movdqa		%xmm3,0x30(%rsp)
-
-	# x4[0-3] += s1[0]
-	# x5[0-3] += s1[1]
-	movq		0x10(%rdi),%xmm3
-	pshufd		$0x00,%xmm3,%xmm2
-	pshufd		$0x55,%xmm3,%xmm3
-	paddd		%xmm2,%xmm4
-	paddd		%xmm3,%xmm5
-	# x6[0-3] += s1[2]
-	# x7[0-3] += s1[3]
-	movq		0x18(%rdi),%xmm3
-	pshufd		$0x00,%xmm3,%xmm2
-	pshufd		$0x55,%xmm3,%xmm3
-	paddd		%xmm2,%xmm6
-	paddd		%xmm3,%xmm7
-
-	# x8[0-3] += s2[0]
-	# x9[0-3] += s2[1]
-	movq		0x20(%rdi),%xmm3
-	pshufd		$0x00,%xmm3,%xmm2
-	pshufd		$0x55,%xmm3,%xmm3
-	paddd		%xmm2,%xmm8
-	paddd		%xmm3,%xmm9
-	# x10[0-3] += s2[2]
-	# x11[0-3] += s2[3]
-	movq		0x28(%rdi),%xmm3
-	pshufd		$0x00,%xmm3,%xmm2
-	pshufd		$0x55,%xmm3,%xmm3
-	paddd		%xmm2,%xmm10
-	paddd		%xmm3,%xmm11
-
-	# x12[0-3] += s3[0]
-	# x13[0-3] += s3[1]
-	movq		0x30(%rdi),%xmm3
-	pshufd		$0x00,%xmm3,%xmm2
-	pshufd		$0x55,%xmm3,%xmm3
-	paddd		%xmm2,%xmm12
-	paddd		%xmm3,%xmm13
-	# x14[0-3] += s3[2]
-	# x15[0-3] += s3[3]
-	movq		0x38(%rdi),%xmm3
-	pshufd		$0x00,%xmm3,%xmm2
-	pshufd		$0x55,%xmm3,%xmm3
-	paddd		%xmm2,%xmm14
-	paddd		%xmm3,%xmm15
-
-	# x12 += counter values 0-3
-	paddd		%xmm1,%xmm12
-
-	# interleave 32-bit words in state n, n+1
-	movdqa		0x00(%rsp),%xmm0
-	movdqa		0x10(%rsp),%xmm1
-	movdqa		%xmm0,%xmm2
-	punpckldq	%xmm1,%xmm2
-	punpckhdq	%xmm1,%xmm0
-	movdqa		%xmm2,0x00(%rsp)
-	movdqa		%xmm0,0x10(%rsp)
-	movdqa		0x20(%rsp),%xmm0
-	movdqa		0x30(%rsp),%xmm1
-	movdqa		%xmm0,%xmm2
-	punpckldq	%xmm1,%xmm2
-	punpckhdq	%xmm1,%xmm0
-	movdqa		%xmm2,0x20(%rsp)
-	movdqa		%xmm0,0x30(%rsp)
-	movdqa		%xmm4,%xmm0
-	punpckldq	%xmm5,%xmm4
-	punpckhdq	%xmm5,%xmm0
-	movdqa		%xmm0,%xmm5
-	movdqa		%xmm6,%xmm0
-	punpckldq	%xmm7,%xmm6
-	punpckhdq	%xmm7,%xmm0
-	movdqa		%xmm0,%xmm7
-	movdqa		%xmm8,%xmm0
-	punpckldq	%xmm9,%xmm8
-	punpckhdq	%xmm9,%xmm0
-	movdqa		%xmm0,%xmm9
-	movdqa		%xmm10,%xmm0
-	punpckldq	%xmm11,%xmm10
-	punpckhdq	%xmm11,%xmm0
-	movdqa		%xmm0,%xmm11
-	movdqa		%xmm12,%xmm0
-	punpckldq	%xmm13,%xmm12
-	punpckhdq	%xmm13,%xmm0
-	movdqa		%xmm0,%xmm13
-	movdqa		%xmm14,%xmm0
-	punpckldq	%xmm15,%xmm14
-	punpckhdq	%xmm15,%xmm0
-	movdqa		%xmm0,%xmm15
-
-	# interleave 64-bit words in state n, n+2
-	movdqa		0x00(%rsp),%xmm0
-	movdqa		0x20(%rsp),%xmm1
-	movdqa		%xmm0,%xmm2
-	punpcklqdq	%xmm1,%xmm2
-	punpckhqdq	%xmm1,%xmm0
-	movdqa		%xmm2,0x00(%rsp)
-	movdqa		%xmm0,0x20(%rsp)
-	movdqa		0x10(%rsp),%xmm0
-	movdqa		0x30(%rsp),%xmm1
-	movdqa		%xmm0,%xmm2
-	punpcklqdq	%xmm1,%xmm2
-	punpckhqdq	%xmm1,%xmm0
-	movdqa		%xmm2,0x10(%rsp)
-	movdqa		%xmm0,0x30(%rsp)
-	movdqa		%xmm4,%xmm0
-	punpcklqdq	%xmm6,%xmm4
-	punpckhqdq	%xmm6,%xmm0
-	movdqa		%xmm0,%xmm6
-	movdqa		%xmm5,%xmm0
-	punpcklqdq	%xmm7,%xmm5
-	punpckhqdq	%xmm7,%xmm0
-	movdqa		%xmm0,%xmm7
-	movdqa		%xmm8,%xmm0
-	punpcklqdq	%xmm10,%xmm8
-	punpckhqdq	%xmm10,%xmm0
-	movdqa		%xmm0,%xmm10
-	movdqa		%xmm9,%xmm0
-	punpcklqdq	%xmm11,%xmm9
-	punpckhqdq	%xmm11,%xmm0
-	movdqa		%xmm0,%xmm11
-	movdqa		%xmm12,%xmm0
-	punpcklqdq	%xmm14,%xmm12
-	punpckhqdq	%xmm14,%xmm0
-	movdqa		%xmm0,%xmm14
-	movdqa		%xmm13,%xmm0
-	punpcklqdq	%xmm15,%xmm13
-	punpckhqdq	%xmm15,%xmm0
-	movdqa		%xmm0,%xmm15
-
-	# xor with corresponding input, write to output
-	movdqa		0x00(%rsp),%xmm0
-	cmp		$0x10,%rax
-	jl		.Lxorpart4
-	movdqu		0x00(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0x00(%rsi)
-
-	movdqu		%xmm4,%xmm0
-	cmp		$0x20,%rax
-	jl		.Lxorpart4
-	movdqu		0x10(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0x10(%rsi)
-
-	movdqu		%xmm8,%xmm0
-	cmp		$0x30,%rax
-	jl		.Lxorpart4
-	movdqu		0x20(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0x20(%rsi)
-
-	movdqu		%xmm12,%xmm0
-	cmp		$0x40,%rax
-	jl		.Lxorpart4
-	movdqu		0x30(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0x30(%rsi)
-
-	movdqa		0x20(%rsp),%xmm0
-	cmp		$0x50,%rax
-	jl		.Lxorpart4
-	movdqu		0x40(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0x40(%rsi)
-
-	movdqu		%xmm6,%xmm0
-	cmp		$0x60,%rax
-	jl		.Lxorpart4
-	movdqu		0x50(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0x50(%rsi)
-
-	movdqu		%xmm10,%xmm0
-	cmp		$0x70,%rax
-	jl		.Lxorpart4
-	movdqu		0x60(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0x60(%rsi)
-
-	movdqu		%xmm14,%xmm0
-	cmp		$0x80,%rax
-	jl		.Lxorpart4
-	movdqu		0x70(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0x70(%rsi)
-
-	movdqa		0x10(%rsp),%xmm0
-	cmp		$0x90,%rax
-	jl		.Lxorpart4
-	movdqu		0x80(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0x80(%rsi)
-
-	movdqu		%xmm5,%xmm0
-	cmp		$0xa0,%rax
-	jl		.Lxorpart4
-	movdqu		0x90(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0x90(%rsi)
-
-	movdqu		%xmm9,%xmm0
-	cmp		$0xb0,%rax
-	jl		.Lxorpart4
-	movdqu		0xa0(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0xa0(%rsi)
-
-	movdqu		%xmm13,%xmm0
-	cmp		$0xc0,%rax
-	jl		.Lxorpart4
-	movdqu		0xb0(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0xb0(%rsi)
-
-	movdqa		0x30(%rsp),%xmm0
-	cmp		$0xd0,%rax
-	jl		.Lxorpart4
-	movdqu		0xc0(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0xc0(%rsi)
-
-	movdqu		%xmm7,%xmm0
-	cmp		$0xe0,%rax
-	jl		.Lxorpart4
-	movdqu		0xd0(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0xd0(%rsi)
-
-	movdqu		%xmm11,%xmm0
-	cmp		$0xf0,%rax
-	jl		.Lxorpart4
-	movdqu		0xe0(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0xe0(%rsi)
-
-	movdqu		%xmm15,%xmm0
-	cmp		$0x100,%rax
-	jl		.Lxorpart4
-	movdqu		0xf0(%rdx),%xmm1
-	pxor		%xmm1,%xmm0
-	movdqu		%xmm0,0xf0(%rsi)
-
-.Ldone4:
-	lea		-8(%r10),%rsp
-	RET
-
-.Lxorpart4:
-	# xor remaining bytes from partial register into output
-	mov		%rax,%r9
-	and		$0x0f,%r9
-	jz		.Ldone4
-	and		$~0x0f,%rax
-
-	mov		%rsi,%r11
-
-	lea		(%rdx,%rax),%rsi
-	mov		%rsp,%rdi
-	mov		%r9,%rcx
-	rep movsb
-
-	pxor		0x00(%rsp),%xmm0
-	movdqa		%xmm0,0x00(%rsp)
-
-	mov		%rsp,%rsi
-	lea		(%r11,%rax),%rdi
-	mov		%r9,%rcx
-	rep movsb
-
-	jmp		.Ldone4
-
-SYM_FUNC_END(chacha_4block_xor_ssse3)
diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c
deleted file mode 100644
index 7b3a1cf0984b..000000000000
--- a/arch/x86/crypto/chacha_glue.c
+++ /dev/null
@@ -1,317 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * x64 SIMD accelerated ChaCha and XChaCha stream ciphers,
- * including ChaCha20 (RFC7539)
- *
- * Copyright (C) 2015 Martin Willi
- */
-
-#include <crypto/algapi.h>
-#include <crypto/internal/chacha.h>
-#include <crypto/internal/simd.h>
-#include <crypto/internal/skcipher.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/sizes.h>
-#include <asm/simd.h>
-
-asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
-				       unsigned int len, int nrounds);
-asmlinkage void chacha_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
-					unsigned int len, int nrounds);
-asmlinkage void hchacha_block_ssse3(const u32 *state, u32 *out, int nrounds);
-
-asmlinkage void chacha_2block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
-				       unsigned int len, int nrounds);
-asmlinkage void chacha_4block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
-				       unsigned int len, int nrounds);
-asmlinkage void chacha_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
-				       unsigned int len, int nrounds);
-
-asmlinkage void chacha_2block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
-					   unsigned int len, int nrounds);
-asmlinkage void chacha_4block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
-					   unsigned int len, int nrounds);
-asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
-					   unsigned int len, int nrounds);
-
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_simd);
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx2);
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx512vl);
-
-static unsigned int chacha_advance(unsigned int len, unsigned int maxblocks)
-{
-	len = min(len, maxblocks * CHACHA_BLOCK_SIZE);
-	return round_up(len, CHACHA_BLOCK_SIZE) / CHACHA_BLOCK_SIZE;
-}
-
-static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src,
-			  unsigned int bytes, int nrounds)
-{
-	if (IS_ENABLED(CONFIG_AS_AVX512) &&
-	    static_branch_likely(&chacha_use_avx512vl)) {
-		while (bytes >= CHACHA_BLOCK_SIZE * 8) {
-			chacha_8block_xor_avx512vl(state, dst, src, bytes,
-						   nrounds);
-			bytes -= CHACHA_BLOCK_SIZE * 8;
-			src += CHACHA_BLOCK_SIZE * 8;
-			dst += CHACHA_BLOCK_SIZE * 8;
-			state[12] += 8;
-		}
-		if (bytes > CHACHA_BLOCK_SIZE * 4) {
-			chacha_8block_xor_avx512vl(state, dst, src, bytes,
-						   nrounds);
-			state[12] += chacha_advance(bytes, 8);
-			return;
-		}
-		if (bytes > CHACHA_BLOCK_SIZE * 2) {
-			chacha_4block_xor_avx512vl(state, dst, src, bytes,
-						   nrounds);
-			state[12] += chacha_advance(bytes, 4);
-			return;
-		}
-		if (bytes) {
-			chacha_2block_xor_avx512vl(state, dst, src, bytes,
-						   nrounds);
-			state[12] += chacha_advance(bytes, 2);
-			return;
-		}
-	}
-
-	if (static_branch_likely(&chacha_use_avx2)) {
-		while (bytes >= CHACHA_BLOCK_SIZE * 8) {
-			chacha_8block_xor_avx2(state, dst, src, bytes, nrounds);
-			bytes -= CHACHA_BLOCK_SIZE * 8;
-			src += CHACHA_BLOCK_SIZE * 8;
-			dst += CHACHA_BLOCK_SIZE * 8;
-			state[12] += 8;
-		}
-		if (bytes > CHACHA_BLOCK_SIZE * 4) {
-			chacha_8block_xor_avx2(state, dst, src, bytes, nrounds);
-			state[12] += chacha_advance(bytes, 8);
-			return;
-		}
-		if (bytes > CHACHA_BLOCK_SIZE * 2) {
-			chacha_4block_xor_avx2(state, dst, src, bytes, nrounds);
-			state[12] += chacha_advance(bytes, 4);
-			return;
-		}
-		if (bytes > CHACHA_BLOCK_SIZE) {
-			chacha_2block_xor_avx2(state, dst, src, bytes, nrounds);
-			state[12] += chacha_advance(bytes, 2);
-			return;
-		}
-	}
-
-	while (bytes >= CHACHA_BLOCK_SIZE * 4) {
-		chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);
-		bytes -= CHACHA_BLOCK_SIZE * 4;
-		src += CHACHA_BLOCK_SIZE * 4;
-		dst += CHACHA_BLOCK_SIZE * 4;
-		state[12] += 4;
-	}
-	if (bytes > CHACHA_BLOCK_SIZE) {
-		chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);
-		state[12] += chacha_advance(bytes, 4);
-		return;
-	}
-	if (bytes) {
-		chacha_block_xor_ssse3(state, dst, src, bytes, nrounds);
-		state[12]++;
-	}
-}
-
-void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
-{
-	if (!static_branch_likely(&chacha_use_simd) || !crypto_simd_usable()) {
-		hchacha_block_generic(state, stream, nrounds);
-	} else {
-		kernel_fpu_begin();
-		hchacha_block_ssse3(state, stream, nrounds);
-		kernel_fpu_end();
-	}
-}
-EXPORT_SYMBOL(hchacha_block_arch);
-
-void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
-{
-	chacha_init_generic(state, key, iv);
-}
-EXPORT_SYMBOL(chacha_init_arch);
-
-void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
-		       int nrounds)
-{
-	if (!static_branch_likely(&chacha_use_simd) || !crypto_simd_usable() ||
-	    bytes <= CHACHA_BLOCK_SIZE)
-		return chacha_crypt_generic(state, dst, src, bytes, nrounds);
-
-	do {
-		unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
-
-		kernel_fpu_begin();
-		chacha_dosimd(state, dst, src, todo, nrounds);
-		kernel_fpu_end();
-
-		bytes -= todo;
-		src += todo;
-		dst += todo;
-	} while (bytes);
-}
-EXPORT_SYMBOL(chacha_crypt_arch);
-
-static int chacha_simd_stream_xor(struct skcipher_request *req,
-				  const struct chacha_ctx *ctx, const u8 *iv)
-{
-	u32 state[CHACHA_STATE_WORDS] __aligned(8);
-	struct skcipher_walk walk;
-	int err;
-
-	err = skcipher_walk_virt(&walk, req, false);
-
-	chacha_init_generic(state, ctx->key, iv);
-
-	while (walk.nbytes > 0) {
-		unsigned int nbytes = walk.nbytes;
-
-		if (nbytes < walk.total)
-			nbytes = round_down(nbytes, walk.stride);
-
-		if (!static_branch_likely(&chacha_use_simd) ||
-		    !crypto_simd_usable()) {
-			chacha_crypt_generic(state, walk.dst.virt.addr,
-					     walk.src.virt.addr, nbytes,
-					     ctx->nrounds);
-		} else {
-			kernel_fpu_begin();
-			chacha_dosimd(state, walk.dst.virt.addr,
-				      walk.src.virt.addr, nbytes,
-				      ctx->nrounds);
-			kernel_fpu_end();
-		}
-		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
-	}
-
-	return err;
-}
-
-static int chacha_simd(struct skcipher_request *req)
-{
-	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-	struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
-
-	return chacha_simd_stream_xor(req, ctx, req->iv);
-}
-
-static int xchacha_simd(struct skcipher_request *req)
-{
-	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-	struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
-	u32 state[CHACHA_STATE_WORDS] __aligned(8);
-	struct chacha_ctx subctx;
-	u8 real_iv[16];
-
-	chacha_init_generic(state, ctx->key, req->iv);
-
-	if (req->cryptlen > CHACHA_BLOCK_SIZE && crypto_simd_usable()) {
-		kernel_fpu_begin();
-		hchacha_block_ssse3(state, subctx.key, ctx->nrounds);
-		kernel_fpu_end();
-	} else {
-		hchacha_block_generic(state, subctx.key, ctx->nrounds);
-	}
-	subctx.nrounds = ctx->nrounds;
-
-	memcpy(&real_iv[0], req->iv + 24, 8);
-	memcpy(&real_iv[8], req->iv + 16, 8);
-	return chacha_simd_stream_xor(req, &subctx, real_iv);
-}
-
-static struct skcipher_alg algs[] = {
-	{
-		.base.cra_name		= "chacha20",
-		.base.cra_driver_name	= "chacha20-simd",
-		.base.cra_priority	= 300,
-		.base.cra_blocksize	= 1,
-		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
-		.base.cra_module	= THIS_MODULE,
-
-		.min_keysize		= CHACHA_KEY_SIZE,
-		.max_keysize		= CHACHA_KEY_SIZE,
-		.ivsize			= CHACHA_IV_SIZE,
-		.chunksize		= CHACHA_BLOCK_SIZE,
-		.setkey			= chacha20_setkey,
-		.encrypt		= chacha_simd,
-		.decrypt		= chacha_simd,
-	}, {
-		.base.cra_name		= "xchacha20",
-		.base.cra_driver_name	= "xchacha20-simd",
-		.base.cra_priority	= 300,
-		.base.cra_blocksize	= 1,
-		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
-		.base.cra_module	= THIS_MODULE,
-
-		.min_keysize		= CHACHA_KEY_SIZE,
-		.max_keysize		= CHACHA_KEY_SIZE,
-		.ivsize			= XCHACHA_IV_SIZE,
-		.chunksize		= CHACHA_BLOCK_SIZE,
-		.setkey			= chacha20_setkey,
-		.encrypt		= xchacha_simd,
-		.decrypt		= xchacha_simd,
-	}, {
-		.base.cra_name		= "xchacha12",
-		.base.cra_driver_name	= "xchacha12-simd",
-		.base.cra_priority	= 300,
-		.base.cra_blocksize	= 1,
-		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
-		.base.cra_module	= THIS_MODULE,
-
-		.min_keysize		= CHACHA_KEY_SIZE,
-		.max_keysize		= CHACHA_KEY_SIZE,
-		.ivsize			= XCHACHA_IV_SIZE,
-		.chunksize		= CHACHA_BLOCK_SIZE,
-		.setkey			= chacha12_setkey,
-		.encrypt		= xchacha_simd,
-		.decrypt		= xchacha_simd,
-	},
-};
-
-static int __init chacha_simd_mod_init(void)
-{
-	if (!boot_cpu_has(X86_FEATURE_SSSE3))
-		return 0;
-
-	static_branch_enable(&chacha_use_simd);
-
-	if (boot_cpu_has(X86_FEATURE_AVX) &&
-	    boot_cpu_has(X86_FEATURE_AVX2) &&
-	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
-		static_branch_enable(&chacha_use_avx2);
-
-		if (IS_ENABLED(CONFIG_AS_AVX512) &&
-		    boot_cpu_has(X86_FEATURE_AVX512VL) &&
-		    boot_cpu_has(X86_FEATURE_AVX512BW)) /* kmovq */
-			static_branch_enable(&chacha_use_avx512vl);
-	}
-	return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ?
-		crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0;
-}
-
-static void __exit chacha_simd_mod_fini(void)
-{
-	if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) && boot_cpu_has(X86_FEATURE_SSSE3))
-		crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
-}
-
-module_init(chacha_simd_mod_init);
-module_exit(chacha_simd_mod_fini);
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
-MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (x64 SIMD accelerated)");
-MODULE_ALIAS_CRYPTO("chacha20");
-MODULE_ALIAS_CRYPTO("chacha20-simd");
-MODULE_ALIAS_CRYPTO("xchacha20");
-MODULE_ALIAS_CRYPTO("xchacha20-simd");
-MODULE_ALIAS_CRYPTO("xchacha12");
-MODULE_ALIAS_CRYPTO("xchacha12-simd");
diff --git a/arch/x86/crypto/crc32-pclmul_asm.S b/arch/x86/crypto/crc32-pclmul_asm.S
deleted file mode 100644
index 5d31137e2c7d..000000000000
--- a/arch/x86/crypto/crc32-pclmul_asm.S
+++ /dev/null
@@ -1,218 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright 2012 Xyratex Technology Limited
- *
- * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
- * calculation.
- * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
- * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
- * at:
- * http://www.intel.com/products/processor/manuals/
- * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
- * Volume 2B: Instruction Set Reference, N-Z
- *
- * Authors:   Gregory Prestas <Gregory_Prestas@us.xyratex.com>
- *	      Alexander Boyko <Alexander_Boyko@xyratex.com>
- */
-
-#include <linux/linkage.h>
-
-
-.section .rodata
-.align 16
-/*
- * [x4*128+32 mod P(x) << 32)]'  << 1   = 0x154442bd4
- * #define CONSTANT_R1  0x154442bd4LL
- *
- * [(x4*128-32 mod P(x) << 32)]' << 1   = 0x1c6e41596
- * #define CONSTANT_R2  0x1c6e41596LL
- */
-.Lconstant_R2R1:
-	.octa 0x00000001c6e415960000000154442bd4
-/*
- * [(x128+32 mod P(x) << 32)]'   << 1   = 0x1751997d0
- * #define CONSTANT_R3  0x1751997d0LL
- *
- * [(x128-32 mod P(x) << 32)]'   << 1   = 0x0ccaa009e
- * #define CONSTANT_R4  0x0ccaa009eLL
- */
-.Lconstant_R4R3:
-	.octa 0x00000000ccaa009e00000001751997d0
-/*
- * [(x64 mod P(x) << 32)]'       << 1   = 0x163cd6124
- * #define CONSTANT_R5  0x163cd6124LL
- */
-.Lconstant_R5:
-	.octa 0x00000000000000000000000163cd6124
-.Lconstant_mask32:
-	.octa 0x000000000000000000000000FFFFFFFF
-/*
- * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
- *
- * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))` = 0x1F7011641LL
- * #define CONSTANT_RU  0x1F7011641LL
- */
-.Lconstant_RUpoly:
-	.octa 0x00000001F701164100000001DB710641
-
-#define CONSTANT %xmm0
-
-#ifdef __x86_64__
-#define BUF     %rdi
-#define LEN     %rsi
-#define CRC     %edx
-#else
-#define BUF     %eax
-#define LEN     %edx
-#define CRC     %ecx
-#endif
-
-
-
-.text
-/**
- *      Calculate crc32
- *      BUF - buffer (16 bytes aligned)
- *      LEN - sizeof buffer (16 bytes aligned), LEN should be grater than 63
- *      CRC - initial crc32
- *      return %eax crc32
- *      uint crc32_pclmul_le_16(unsigned char const *buffer,
- *	                     size_t len, uint crc32)
- */
-
-SYM_FUNC_START(crc32_pclmul_le_16) /* buffer and buffer size are 16 bytes aligned */
-	movdqa  (BUF), %xmm1
-	movdqa  0x10(BUF), %xmm2
-	movdqa  0x20(BUF), %xmm3
-	movdqa  0x30(BUF), %xmm4
-	movd    CRC, CONSTANT
-	pxor    CONSTANT, %xmm1
-	sub     $0x40, LEN
-	add     $0x40, BUF
-	cmp     $0x40, LEN
-	jb      .Lless_64
-
-#ifdef __x86_64__
-	movdqa .Lconstant_R2R1(%rip), CONSTANT
-#else
-	movdqa .Lconstant_R2R1, CONSTANT
-#endif
-
-.Lloop_64:/*  64 bytes Full cache line folding */
-	prefetchnta    0x40(BUF)
-	movdqa  %xmm1, %xmm5
-	movdqa  %xmm2, %xmm6
-	movdqa  %xmm3, %xmm7
-#ifdef __x86_64__
-	movdqa  %xmm4, %xmm8
-#endif
-	pclmulqdq $0x00, CONSTANT, %xmm1
-	pclmulqdq $0x00, CONSTANT, %xmm2
-	pclmulqdq $0x00, CONSTANT, %xmm3
-#ifdef __x86_64__
-	pclmulqdq $0x00, CONSTANT, %xmm4
-#endif
-	pclmulqdq $0x11, CONSTANT, %xmm5
-	pclmulqdq $0x11, CONSTANT, %xmm6
-	pclmulqdq $0x11, CONSTANT, %xmm7
-#ifdef __x86_64__
-	pclmulqdq $0x11, CONSTANT, %xmm8
-#endif
-	pxor    %xmm5, %xmm1
-	pxor    %xmm6, %xmm2
-	pxor    %xmm7, %xmm3
-#ifdef __x86_64__
-	pxor    %xmm8, %xmm4
-#else
-	/* xmm8 unsupported for x32 */
-	movdqa  %xmm4, %xmm5
-	pclmulqdq $0x00, CONSTANT, %xmm4
-	pclmulqdq $0x11, CONSTANT, %xmm5
-	pxor    %xmm5, %xmm4
-#endif
-
-	pxor    (BUF), %xmm1
-	pxor    0x10(BUF), %xmm2
-	pxor    0x20(BUF), %xmm3
-	pxor    0x30(BUF), %xmm4
-
-	sub     $0x40, LEN
-	add     $0x40, BUF
-	cmp     $0x40, LEN
-	jge     .Lloop_64
-.Lless_64:/*  Folding cache line into 128bit */
-#ifdef __x86_64__
-	movdqa  .Lconstant_R4R3(%rip), CONSTANT
-#else
-	movdqa  .Lconstant_R4R3, CONSTANT
-#endif
-	prefetchnta     (BUF)
-
-	movdqa  %xmm1, %xmm5
-	pclmulqdq $0x00, CONSTANT, %xmm1
-	pclmulqdq $0x11, CONSTANT, %xmm5
-	pxor    %xmm5, %xmm1
-	pxor    %xmm2, %xmm1
-
-	movdqa  %xmm1, %xmm5
-	pclmulqdq $0x00, CONSTANT, %xmm1
-	pclmulqdq $0x11, CONSTANT, %xmm5
-	pxor    %xmm5, %xmm1
-	pxor    %xmm3, %xmm1
-
-	movdqa  %xmm1, %xmm5
-	pclmulqdq $0x00, CONSTANT, %xmm1
-	pclmulqdq $0x11, CONSTANT, %xmm5
-	pxor    %xmm5, %xmm1
-	pxor    %xmm4, %xmm1
-
-	cmp     $0x10, LEN
-	jb      .Lfold_64
-.Lloop_16:/* Folding rest buffer into 128bit */
-	movdqa  %xmm1, %xmm5
-	pclmulqdq $0x00, CONSTANT, %xmm1
-	pclmulqdq $0x11, CONSTANT, %xmm5
-	pxor    %xmm5, %xmm1
-	pxor    (BUF), %xmm1
-	sub     $0x10, LEN
-	add     $0x10, BUF
-	cmp     $0x10, LEN
-	jge     .Lloop_16
-
-.Lfold_64:
-	/* perform the last 64 bit fold, also adds 32 zeroes
-	 * to the input stream */
-	pclmulqdq $0x01, %xmm1, CONSTANT /* R4 * xmm1.low */
-	psrldq  $0x08, %xmm1
-	pxor    CONSTANT, %xmm1
-
-	/* final 32-bit fold */
-	movdqa  %xmm1, %xmm2
-#ifdef __x86_64__
-	movdqa  .Lconstant_R5(%rip), CONSTANT
-	movdqa  .Lconstant_mask32(%rip), %xmm3
-#else
-	movdqa  .Lconstant_R5, CONSTANT
-	movdqa  .Lconstant_mask32, %xmm3
-#endif
-	psrldq  $0x04, %xmm2
-	pand    %xmm3, %xmm1
-	pclmulqdq $0x00, CONSTANT, %xmm1
-	pxor    %xmm2, %xmm1
-
-	/* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
-#ifdef __x86_64__
-	movdqa  .Lconstant_RUpoly(%rip), CONSTANT
-#else
-	movdqa  .Lconstant_RUpoly, CONSTANT
-#endif
-	movdqa  %xmm1, %xmm2
-	pand    %xmm3, %xmm1
-	pclmulqdq $0x10, CONSTANT, %xmm1
-	pand    %xmm3, %xmm1
-	pclmulqdq $0x00, CONSTANT, %xmm1
-	pxor    %xmm2, %xmm1
-	pextrd  $0x01, %xmm1, %eax
-
-	RET
-SYM_FUNC_END(crc32_pclmul_le_16)
diff --git a/arch/x86/crypto/crc32-pclmul_glue.c b/arch/x86/crypto/crc32-pclmul_glue.c
deleted file mode 100644
index 9f5e342b9845..000000000000
--- a/arch/x86/crypto/crc32-pclmul_glue.c
+++ /dev/null
@@ -1,202 +0,0 @@
-/* GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see http://www.gnu.org/licenses
- *
- * Please  visit http://www.xyratex.com/contact if you need additional
- * information or have any questions.
- *
- * GPL HEADER END
- */
-
-/*
- * Copyright 2012 Xyratex Technology Limited
- *
- * Wrappers for kernel crypto shash api to pclmulqdq crc32 implementation.
- */
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/string.h>
-#include <linux/kernel.h>
-#include <linux/crc32.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-
-#include <asm/cpufeatures.h>
-#include <asm/cpu_device_id.h>
-#include <asm/simd.h>
-
-#define CHKSUM_BLOCK_SIZE	1
-#define CHKSUM_DIGEST_SIZE	4
-
-#define PCLMUL_MIN_LEN		64L     /* minimum size of buffer
-					 * for crc32_pclmul_le_16 */
-#define SCALE_F			16L	/* size of xmm register */
-#define SCALE_F_MASK		(SCALE_F - 1)
-
-u32 crc32_pclmul_le_16(unsigned char const *buffer, size_t len, u32 crc32);
-
-static u32 __attribute__((pure))
-	crc32_pclmul_le(u32 crc, unsigned char const *p, size_t len)
-{
-	unsigned int iquotient;
-	unsigned int iremainder;
-	unsigned int prealign;
-
-	if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !crypto_simd_usable())
-		return crc32_le(crc, p, len);
-
-	if ((long)p & SCALE_F_MASK) {
-		/* align p to 16 byte */
-		prealign = SCALE_F - ((long)p & SCALE_F_MASK);
-
-		crc = crc32_le(crc, p, prealign);
-		len -= prealign;
-		p = (unsigned char *)(((unsigned long)p + SCALE_F_MASK) &
-				     ~SCALE_F_MASK);
-	}
-	iquotient = len & (~SCALE_F_MASK);
-	iremainder = len & SCALE_F_MASK;
-
-	kernel_fpu_begin();
-	crc = crc32_pclmul_le_16(p, iquotient, crc);
-	kernel_fpu_end();
-
-	if (iremainder)
-		crc = crc32_le(crc, p + iquotient, iremainder);
-
-	return crc;
-}
-
-static int crc32_pclmul_cra_init(struct crypto_tfm *tfm)
-{
-	u32 *key = crypto_tfm_ctx(tfm);
-
-	*key = 0;
-
-	return 0;
-}
-
-static int crc32_pclmul_setkey(struct crypto_shash *hash, const u8 *key,
-			unsigned int keylen)
-{
-	u32 *mctx = crypto_shash_ctx(hash);
-
-	if (keylen != sizeof(u32))
-		return -EINVAL;
-	*mctx = le32_to_cpup((__le32 *)key);
-	return 0;
-}
-
-static int crc32_pclmul_init(struct shash_desc *desc)
-{
-	u32 *mctx = crypto_shash_ctx(desc->tfm);
-	u32 *crcp = shash_desc_ctx(desc);
-
-	*crcp = *mctx;
-
-	return 0;
-}
-
-static int crc32_pclmul_update(struct shash_desc *desc, const u8 *data,
-			       unsigned int len)
-{
-	u32 *crcp = shash_desc_ctx(desc);
-
-	*crcp = crc32_pclmul_le(*crcp, data, len);
-	return 0;
-}
-
-/* No final XOR 0xFFFFFFFF, like crc32_le */
-static int __crc32_pclmul_finup(u32 *crcp, const u8 *data, unsigned int len,
-				u8 *out)
-{
-	*(__le32 *)out = cpu_to_le32(crc32_pclmul_le(*crcp, data, len));
-	return 0;
-}
-
-static int crc32_pclmul_finup(struct shash_desc *desc, const u8 *data,
-			      unsigned int len, u8 *out)
-{
-	return __crc32_pclmul_finup(shash_desc_ctx(desc), data, len, out);
-}
-
-static int crc32_pclmul_final(struct shash_desc *desc, u8 *out)
-{
-	u32 *crcp = shash_desc_ctx(desc);
-
-	*(__le32 *)out = cpu_to_le32p(crcp);
-	return 0;
-}
-
-static int crc32_pclmul_digest(struct shash_desc *desc, const u8 *data,
-			       unsigned int len, u8 *out)
-{
-	return __crc32_pclmul_finup(crypto_shash_ctx(desc->tfm), data, len,
-				    out);
-}
-
-static struct shash_alg alg = {
-	.setkey		= crc32_pclmul_setkey,
-	.init		= crc32_pclmul_init,
-	.update		= crc32_pclmul_update,
-	.final		= crc32_pclmul_final,
-	.finup		= crc32_pclmul_finup,
-	.digest		= crc32_pclmul_digest,
-	.descsize	= sizeof(u32),
-	.digestsize	= CHKSUM_DIGEST_SIZE,
-	.base		= {
-			.cra_name		= "crc32",
-			.cra_driver_name	= "crc32-pclmul",
-			.cra_priority		= 200,
-			.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
-			.cra_blocksize		= CHKSUM_BLOCK_SIZE,
-			.cra_ctxsize		= sizeof(u32),
-			.cra_module		= THIS_MODULE,
-			.cra_init		= crc32_pclmul_cra_init,
-	}
-};
-
-static const struct x86_cpu_id crc32pclmul_cpu_id[] = {
-	X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL),
-	{}
-};
-MODULE_DEVICE_TABLE(x86cpu, crc32pclmul_cpu_id);
-
-
-static int __init crc32_pclmul_mod_init(void)
-{
-
-	if (!x86_match_cpu(crc32pclmul_cpu_id)) {
-		pr_info("PCLMULQDQ-NI instructions are not detected.\n");
-		return -ENODEV;
-	}
-	return crypto_register_shash(&alg);
-}
-
-static void __exit crc32_pclmul_mod_fini(void)
-{
-	crypto_unregister_shash(&alg);
-}
-
-module_init(crc32_pclmul_mod_init);
-module_exit(crc32_pclmul_mod_fini);
-
-MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>");
-MODULE_DESCRIPTION("CRC32 algorithm (IEEE 802.3) accelerated with PCLMULQDQ");
-MODULE_LICENSE("GPL");
-
-MODULE_ALIAS_CRYPTO("crc32");
-MODULE_ALIAS_CRYPTO("crc32-pclmul");
diff --git a/arch/x86/crypto/crc32c-intel_glue.c b/arch/x86/crypto/crc32c-intel_glue.c
deleted file mode 100644
index feccb5254c7e..000000000000
--- a/arch/x86/crypto/crc32c-intel_glue.c
+++ /dev/null
@@ -1,250 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal.
- * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE)
- * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at:
- * http://www.intel.com/products/processor/manuals/
- * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
- * Volume 2A: Instruction Set Reference, A-M
- *
- * Copyright (C) 2008 Intel Corporation
- * Authors: Austin Zhang <austin_zhang@linux.intel.com>
- *          Kent Liu <kent.liu@intel.com>
- */
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/string.h>
-#include <linux/kernel.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-
-#include <asm/cpufeatures.h>
-#include <asm/cpu_device_id.h>
-#include <asm/simd.h>
-
-#define CHKSUM_BLOCK_SIZE	1
-#define CHKSUM_DIGEST_SIZE	4
-
-#define SCALE_F	sizeof(unsigned long)
-
-#ifdef CONFIG_X86_64
-#define CRC32_INST "crc32q %1, %q0"
-#else
-#define CRC32_INST "crc32l %1, %0"
-#endif
-
-#ifdef CONFIG_X86_64
-/*
- * use carryless multiply version of crc32c when buffer
- * size is >= 512 to account
- * for fpu state save/restore overhead.
- */
-#define CRC32C_PCL_BREAKEVEN	512
-
-asmlinkage unsigned int crc_pcl(const u8 *buffer, int len,
-				unsigned int crc_init);
-#endif /* CONFIG_X86_64 */
-
-static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length)
-{
-	while (length--) {
-		asm("crc32b %1, %0"
-		    : "+r" (crc) : "rm" (*data));
-		data++;
-	}
-
-	return crc;
-}
-
-static u32 __pure crc32c_intel_le_hw(u32 crc, unsigned char const *p, size_t len)
-{
-	unsigned int iquotient = len / SCALE_F;
-	unsigned int iremainder = len % SCALE_F;
-	unsigned long *ptmp = (unsigned long *)p;
-
-	while (iquotient--) {
-		asm(CRC32_INST
-		    : "+r" (crc) : "rm" (*ptmp));
-		ptmp++;
-	}
-
-	if (iremainder)
-		crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp,
-				 iremainder);
-
-	return crc;
-}
-
-/*
- * Setting the seed allows arbitrary accumulators and flexible XOR policy
- * If your algorithm starts with ~0, then XOR with ~0 before you set
- * the seed.
- */
-static int crc32c_intel_setkey(struct crypto_shash *hash, const u8 *key,
-			unsigned int keylen)
-{
-	u32 *mctx = crypto_shash_ctx(hash);
-
-	if (keylen != sizeof(u32))
-		return -EINVAL;
-	*mctx = le32_to_cpup((__le32 *)key);
-	return 0;
-}
-
-static int crc32c_intel_init(struct shash_desc *desc)
-{
-	u32 *mctx = crypto_shash_ctx(desc->tfm);
-	u32 *crcp = shash_desc_ctx(desc);
-
-	*crcp = *mctx;
-
-	return 0;
-}
-
-static int crc32c_intel_update(struct shash_desc *desc, const u8 *data,
-			       unsigned int len)
-{
-	u32 *crcp = shash_desc_ctx(desc);
-
-	*crcp = crc32c_intel_le_hw(*crcp, data, len);
-	return 0;
-}
-
-static int __crc32c_intel_finup(u32 *crcp, const u8 *data, unsigned int len,
-				u8 *out)
-{
-	*(__le32 *)out = ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len));
-	return 0;
-}
-
-static int crc32c_intel_finup(struct shash_desc *desc, const u8 *data,
-			      unsigned int len, u8 *out)
-{
-	return __crc32c_intel_finup(shash_desc_ctx(desc), data, len, out);
-}
-
-static int crc32c_intel_final(struct shash_desc *desc, u8 *out)
-{
-	u32 *crcp = shash_desc_ctx(desc);
-
-	*(__le32 *)out = ~cpu_to_le32p(crcp);
-	return 0;
-}
-
-static int crc32c_intel_digest(struct shash_desc *desc, const u8 *data,
-			       unsigned int len, u8 *out)
-{
-	return __crc32c_intel_finup(crypto_shash_ctx(desc->tfm), data, len,
-				    out);
-}
-
-static int crc32c_intel_cra_init(struct crypto_tfm *tfm)
-{
-	u32 *key = crypto_tfm_ctx(tfm);
-
-	*key = ~0;
-
-	return 0;
-}
-
-#ifdef CONFIG_X86_64
-static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data,
-			       unsigned int len)
-{
-	u32 *crcp = shash_desc_ctx(desc);
-
-	/*
-	 * use faster PCL version if datasize is large enough to
-	 * overcome kernel fpu state save/restore overhead
-	 */
-	if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) {
-		kernel_fpu_begin();
-		*crcp = crc_pcl(data, len, *crcp);
-		kernel_fpu_end();
-	} else
-		*crcp = crc32c_intel_le_hw(*crcp, data, len);
-	return 0;
-}
-
-static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len,
-				u8 *out)
-{
-	if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) {
-		kernel_fpu_begin();
-		*(__le32 *)out = ~cpu_to_le32(crc_pcl(data, len, *crcp));
-		kernel_fpu_end();
-	} else
-		*(__le32 *)out =
-			~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len));
-	return 0;
-}
-
-static int crc32c_pcl_intel_finup(struct shash_desc *desc, const u8 *data,
-			      unsigned int len, u8 *out)
-{
-	return __crc32c_pcl_intel_finup(shash_desc_ctx(desc), data, len, out);
-}
-
-static int crc32c_pcl_intel_digest(struct shash_desc *desc, const u8 *data,
-			       unsigned int len, u8 *out)
-{
-	return __crc32c_pcl_intel_finup(crypto_shash_ctx(desc->tfm), data, len,
-				    out);
-}
-#endif /* CONFIG_X86_64 */
-
-static struct shash_alg alg = {
-	.setkey			=	crc32c_intel_setkey,
-	.init			=	crc32c_intel_init,
-	.update			=	crc32c_intel_update,
-	.final			=	crc32c_intel_final,
-	.finup			=	crc32c_intel_finup,
-	.digest			=	crc32c_intel_digest,
-	.descsize		=	sizeof(u32),
-	.digestsize		=	CHKSUM_DIGEST_SIZE,
-	.base			=	{
-		.cra_name		=	"crc32c",
-		.cra_driver_name	=	"crc32c-intel",
-		.cra_priority		=	200,
-		.cra_flags		=	CRYPTO_ALG_OPTIONAL_KEY,
-		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,
-		.cra_ctxsize		=	sizeof(u32),
-		.cra_module		=	THIS_MODULE,
-		.cra_init		=	crc32c_intel_cra_init,
-	}
-};
-
-static const struct x86_cpu_id crc32c_cpu_id[] = {
-	X86_MATCH_FEATURE(X86_FEATURE_XMM4_2, NULL),
-	{}
-};
-MODULE_DEVICE_TABLE(x86cpu, crc32c_cpu_id);
-
-static int __init crc32c_intel_mod_init(void)
-{
-	if (!x86_match_cpu(crc32c_cpu_id))
-		return -ENODEV;
-#ifdef CONFIG_X86_64
-	if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
-		alg.update = crc32c_pcl_intel_update;
-		alg.finup = crc32c_pcl_intel_finup;
-		alg.digest = crc32c_pcl_intel_digest;
-	}
-#endif
-	return crypto_register_shash(&alg);
-}
-
-static void __exit crc32c_intel_mod_fini(void)
-{
-	crypto_unregister_shash(&alg);
-}
-
-module_init(crc32c_intel_mod_init);
-module_exit(crc32c_intel_mod_fini);
-
-MODULE_AUTHOR("Austin Zhang <austin.zhang@intel.com>, Kent Liu <kent.liu@intel.com>");
-MODULE_DESCRIPTION("CRC32c (Castagnoli) optimization using Intel Hardware.");
-MODULE_LICENSE("GPL");
-
-MODULE_ALIAS_CRYPTO("crc32c");
-MODULE_ALIAS_CRYPTO("crc32c-intel");
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
deleted file mode 100644
index bbcff1fb78cb..000000000000
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ /dev/null
@@ -1,463 +0,0 @@
-/*
- * Implement fast CRC32C with PCLMULQDQ instructions. (x86_64)
- *
- * The white papers on CRC32C calculations with PCLMULQDQ instruction can be
- * downloaded from:
- * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/crc-iscsi-polynomial-crc32-instruction-paper.pdf
- * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-paper.pdf
- *
- * Copyright (C) 2012 Intel Corporation.
- *
- * Authors:
- *	Wajdi Feghali <wajdi.k.feghali@intel.com>
- *	James Guilford <james.guilford@intel.com>
- *	David Cote <david.m.cote@intel.com>
- *	Tim Chen <tim.c.chen@linux.intel.com>
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <linux/linkage.h>
-#include <asm/nospec-branch.h>
-
-## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
-
-.macro LABEL prefix n
-.L\prefix\n\():
-.endm
-
-.macro JMPTBL_ENTRY i
-.quad .Lcrc_\i
-.endm
-
-.macro JNC_LESS_THAN j
-	jnc .Lless_than_\j
-.endm
-
-# Define threshold where buffers are considered "small" and routed to more
-# efficient "by-1" code. This "by-1" code only handles up to 255 bytes, so
-# SMALL_SIZE can be no larger than 255.
-
-#define SMALL_SIZE 200
-
-.if (SMALL_SIZE > 255)
-.error "SMALL_ SIZE must be < 256"
-.endif
-
-# unsigned int crc_pcl(u8 *buffer, int len, unsigned int crc_init);
-
-.text
-SYM_FUNC_START(crc_pcl)
-#define    bufp		rdi
-#define    bufp_dw	%edi
-#define    bufp_w	%di
-#define    bufp_b	%dil
-#define    bufptmp	%rcx
-#define    block_0	%rcx
-#define    block_1	%rdx
-#define    block_2	%r11
-#define    len		%rsi
-#define    len_dw	%esi
-#define    len_w	%si
-#define    len_b	%sil
-#define    crc_init_arg %rdx
-#define    tmp		%rbx
-#define    crc_init	%r8
-#define    crc_init_dw	%r8d
-#define    crc1		%r9
-#define    crc2		%r10
-
-	pushq   %rbx
-	pushq   %rdi
-	pushq   %rsi
-
-	## Move crc_init for Linux to a different
-	mov     crc_init_arg, crc_init
-
-	################################################################
-	## 1) ALIGN:
-	################################################################
-
-	mov     %bufp, bufptmp		# rdi = *buf
-	neg     %bufp
-	and     $7, %bufp		# calculate the unalignment amount of
-					# the address
-	je      .Lproc_block		# Skip if aligned
-
-	## If len is less than 8 and we're unaligned, we need to jump
-	## to special code to avoid reading beyond the end of the buffer
-	cmp     $8, len
-	jae     .Ldo_align
-	# less_than_8 expects length in upper 3 bits of len_dw
-	# less_than_8_post_shl1 expects length = carryflag * 8 + len_dw[31:30]
-	shl     $32-3+1, len_dw
-	jmp     .Lless_than_8_post_shl1
-
-.Ldo_align:
-	#### Calculate CRC of unaligned bytes of the buffer (if any)
-	movq    (bufptmp), tmp		# load a quadward from the buffer
-	add     %bufp, bufptmp		# align buffer pointer for quadword
-					# processing
-	sub     %bufp, len		# update buffer length
-.Lalign_loop:
-	crc32b  %bl, crc_init_dw 	# compute crc32 of 1-byte
-	shr     $8, tmp			# get next byte
-	dec     %bufp
-	jne     .Lalign_loop
-
-.Lproc_block:
-
-	################################################################
-	## 2) PROCESS  BLOCKS:
-	################################################################
-
-	## compute num of bytes to be processed
-	movq    len, tmp		# save num bytes in tmp
-
-	cmpq    $128*24, len
-	jae     .Lfull_block
-
-.Lcontinue_block:
-	cmpq    $SMALL_SIZE, len
-	jb      .Lsmall
-
-	## len < 128*24
-	movq    $2731, %rax		# 2731 = ceil(2^16 / 24)
-	mul     len_dw
-	shrq    $16, %rax
-
-	## eax contains floor(bytes / 24) = num 24-byte chunks to do
-
-	## process rax 24-byte chunks (128 >= rax >= 0)
-
-	## compute end address of each block
-	## block 0 (base addr + RAX * 8)
-	## block 1 (base addr + RAX * 16)
-	## block 2 (base addr + RAX * 24)
-	lea     (bufptmp, %rax, 8), block_0
-	lea     (block_0, %rax, 8), block_1
-	lea     (block_1, %rax, 8), block_2
-
-	xor     crc1, crc1
-	xor     crc2, crc2
-
-	## branch into array
-	leaq	jump_table(%rip), %bufp
-	mov	(%bufp,%rax,8), %bufp
-	JMP_NOSPEC bufp
-
-	################################################################
-	## 2a) PROCESS FULL BLOCKS:
-	################################################################
-.Lfull_block:
-	movl    $128,%eax
-	lea     128*8*2(block_0), block_1
-	lea     128*8*3(block_0), block_2
-	add     $128*8*1, block_0
-
-	xor     crc1,crc1
-	xor     crc2,crc2
-
-	# Fall through into top of crc array (crc_128)
-
-	################################################################
-	## 3) CRC Array:
-	################################################################
-
-	i=128
-.rept 128-1
-.altmacro
-LABEL crc_ %i
-.noaltmacro
-	ENDBR
-	crc32q   -i*8(block_0), crc_init
-	crc32q   -i*8(block_1), crc1
-	crc32q   -i*8(block_2), crc2
-	i=(i-1)
-.endr
-
-.altmacro
-LABEL crc_ %i
-.noaltmacro
-	ENDBR
-	crc32q   -i*8(block_0), crc_init
-	crc32q   -i*8(block_1), crc1
-# SKIP  crc32  -i*8(block_2), crc2 ; Don't do this one yet
-
-	mov     block_2, block_0
-
-	################################################################
-	## 4) Combine three results:
-	################################################################
-
-	lea	(K_table-8)(%rip), %bufp		# first entry is for idx 1
-	shlq    $3, %rax			# rax *= 8
-	pmovzxdq (%bufp,%rax), %xmm0		# 2 consts: K1:K2
-	leal	(%eax,%eax,2), %eax		# rax *= 3 (total *24)
-	subq    %rax, tmp			# tmp -= rax*24
-
-	movq    crc_init, %xmm1			# CRC for block 1
-	pclmulqdq $0x00, %xmm0, %xmm1		# Multiply by K2
-
-	movq    crc1, %xmm2			# CRC for block 2
-	pclmulqdq $0x10, %xmm0, %xmm2		# Multiply by K1
-
-	pxor    %xmm2,%xmm1
-	movq    %xmm1, %rax
-	xor     -i*8(block_2), %rax
-	mov     crc2, crc_init
-	crc32   %rax, crc_init
-
-	################################################################
-	## 5) Check for end:
-	################################################################
-
-LABEL crc_ 0
-	ENDBR
-	mov     tmp, len
-	cmp     $128*24, tmp
-	jae     .Lfull_block
-	cmp     $24, tmp
-	jae     .Lcontinue_block
-
-.Lless_than_24:
-	shl     $32-4, len_dw			# less_than_16 expects length
-						# in upper 4 bits of len_dw
-	jnc     .Lless_than_16
-	crc32q  (bufptmp), crc_init
-	crc32q  8(bufptmp), crc_init
-	jz      .Ldo_return
-	add     $16, bufptmp
-	# len is less than 8 if we got here
-	# less_than_8 expects length in upper 3 bits of len_dw
-	# less_than_8_post_shl1 expects length = carryflag * 8 + len_dw[31:30]
-	shl     $2, len_dw
-	jmp     .Lless_than_8_post_shl1
-
-	#######################################################################
-	## 6) LESS THAN 256-bytes REMAIN AT THIS POINT (8-bits of len are full)
-	#######################################################################
-.Lsmall:
-	shl $32-8, len_dw		# Prepare len_dw for less_than_256
-	j=256
-.rept 5					# j = {256, 128, 64, 32, 16}
-.altmacro
-LABEL less_than_ %j			# less_than_j: Length should be in
-					# upper lg(j) bits of len_dw
-	j=(j/2)
-	shl     $1, len_dw		# Get next MSB
-	JNC_LESS_THAN %j
-.noaltmacro
-	i=0
-.rept (j/8)
-	crc32q  i(bufptmp), crc_init	# Compute crc32 of 8-byte data
-	i=i+8
-.endr
-	jz      .Ldo_return		# Return if remaining length is zero
-	add     $j, bufptmp		# Advance buf
-.endr
-
-.Lless_than_8:				# Length should be stored in
-					# upper 3 bits of len_dw
-	shl     $1, len_dw
-.Lless_than_8_post_shl1:
-	jnc     .Lless_than_4
-	crc32l  (bufptmp), crc_init_dw	# CRC of 4 bytes
-	jz      .Ldo_return		# return if remaining data is zero
-	add     $4, bufptmp
-.Lless_than_4:				# Length should be stored in
-					# upper 2 bits of len_dw
-	shl     $1, len_dw
-	jnc     .Lless_than_2
-	crc32w  (bufptmp), crc_init_dw	# CRC of 2 bytes
-	jz      .Ldo_return		# return if remaining data is zero
-	add     $2, bufptmp
-.Lless_than_2:				# Length should be stored in the MSB
-					# of len_dw
-	shl     $1, len_dw
-	jnc     .Lless_than_1
-	crc32b  (bufptmp), crc_init_dw	# CRC of 1 byte
-.Lless_than_1:				# Length should be zero
-.Ldo_return:
-	movq    crc_init, %rax
-	popq    %rsi
-	popq    %rdi
-	popq    %rbx
-        RET
-SYM_FUNC_END(crc_pcl)
-
-.section	.rodata, "a", @progbits
-        ################################################################
-        ## jump table        Table is 129 entries x 2 bytes each
-        ################################################################
-.align 4
-jump_table:
-	i=0
-.rept 129
-.altmacro
-JMPTBL_ENTRY %i
-.noaltmacro
-	i=i+1
-.endr
-
-
-	################################################################
-	## PCLMULQDQ tables
-	## Table is 128 entries x 2 words (8 bytes) each
-	################################################################
-.align 8
-K_table:
-	.long 0x493c7d27, 0x00000001
-	.long 0xba4fc28e, 0x493c7d27
-	.long 0xddc0152b, 0xf20c0dfe
-	.long 0x9e4addf8, 0xba4fc28e
-	.long 0x39d3b296, 0x3da6d0cb
-	.long 0x0715ce53, 0xddc0152b
-	.long 0x47db8317, 0x1c291d04
-	.long 0x0d3b6092, 0x9e4addf8
-	.long 0xc96cfdc0, 0x740eef02
-	.long 0x878a92a7, 0x39d3b296
-	.long 0xdaece73e, 0x083a6eec
-	.long 0xab7aff2a, 0x0715ce53
-	.long 0x2162d385, 0xc49f4f67
-	.long 0x83348832, 0x47db8317
-	.long 0x299847d5, 0x2ad91c30
-	.long 0xb9e02b86, 0x0d3b6092
-	.long 0x18b33a4e, 0x6992cea2
-	.long 0xb6dd949b, 0xc96cfdc0
-	.long 0x78d9ccb7, 0x7e908048
-	.long 0xbac2fd7b, 0x878a92a7
-	.long 0xa60ce07b, 0x1b3d8f29
-	.long 0xce7f39f4, 0xdaece73e
-	.long 0x61d82e56, 0xf1d0f55e
-	.long 0xd270f1a2, 0xab7aff2a
-	.long 0xc619809d, 0xa87ab8a8
-	.long 0x2b3cac5d, 0x2162d385
-	.long 0x65863b64, 0x8462d800
-	.long 0x1b03397f, 0x83348832
-	.long 0xebb883bd, 0x71d111a8
-	.long 0xb3e32c28, 0x299847d5
-	.long 0x064f7f26, 0xffd852c6
-	.long 0xdd7e3b0c, 0xb9e02b86
-	.long 0xf285651c, 0xdcb17aa4
-	.long 0x10746f3c, 0x18b33a4e
-	.long 0xc7a68855, 0xf37c5aee
-	.long 0x271d9844, 0xb6dd949b
-	.long 0x8e766a0c, 0x6051d5a2
-	.long 0x93a5f730, 0x78d9ccb7
-	.long 0x6cb08e5c, 0x18b0d4ff
-	.long 0x6b749fb2, 0xbac2fd7b
-	.long 0x1393e203, 0x21f3d99c
-	.long 0xcec3662e, 0xa60ce07b
-	.long 0x96c515bb, 0x8f158014
-	.long 0xe6fc4e6a, 0xce7f39f4
-	.long 0x8227bb8a, 0xa00457f7
-	.long 0xb0cd4768, 0x61d82e56
-	.long 0x39c7ff35, 0x8d6d2c43
-	.long 0xd7a4825c, 0xd270f1a2
-	.long 0x0ab3844b, 0x00ac29cf
-	.long 0x0167d312, 0xc619809d
-	.long 0xf6076544, 0xe9adf796
-	.long 0x26f6a60a, 0x2b3cac5d
-	.long 0xa741c1bf, 0x96638b34
-	.long 0x98d8d9cb, 0x65863b64
-	.long 0x49c3cc9c, 0xe0e9f351
-	.long 0x68bce87a, 0x1b03397f
-	.long 0x57a3d037, 0x9af01f2d
-	.long 0x6956fc3b, 0xebb883bd
-	.long 0x42d98888, 0x2cff42cf
-	.long 0x3771e98f, 0xb3e32c28
-	.long 0xb42ae3d9, 0x88f25a3a
-	.long 0x2178513a, 0x064f7f26
-	.long 0xe0ac139e, 0x4e36f0b0
-	.long 0x170076fa, 0xdd7e3b0c
-	.long 0x444dd413, 0xbd6f81f8
-	.long 0x6f345e45, 0xf285651c
-	.long 0x41d17b64, 0x91c9bd4b
-	.long 0xff0dba97, 0x10746f3c
-	.long 0xa2b73df1, 0x885f087b
-	.long 0xf872e54c, 0xc7a68855
-	.long 0x1e41e9fc, 0x4c144932
-	.long 0x86d8e4d2, 0x271d9844
-	.long 0x651bd98b, 0x52148f02
-	.long 0x5bb8f1bc, 0x8e766a0c
-	.long 0xa90fd27a, 0xa3c6f37a
-	.long 0xb3af077a, 0x93a5f730
-	.long 0x4984d782, 0xd7c0557f
-	.long 0xca6ef3ac, 0x6cb08e5c
-	.long 0x234e0b26, 0x63ded06a
-	.long 0xdd66cbbb, 0x6b749fb2
-	.long 0x4597456a, 0x4d56973c
-	.long 0xe9e28eb4, 0x1393e203
-	.long 0x7b3ff57a, 0x9669c9df
-	.long 0xc9c8b782, 0xcec3662e
-	.long 0x3f70cc6f, 0xe417f38a
-	.long 0x93e106a4, 0x96c515bb
-	.long 0x62ec6c6d, 0x4b9e0f71
-	.long 0xd813b325, 0xe6fc4e6a
-	.long 0x0df04680, 0xd104b8fc
-	.long 0x2342001e, 0x8227bb8a
-	.long 0x0a2a8d7e, 0x5b397730
-	.long 0x6d9a4957, 0xb0cd4768
-	.long 0xe8b6368b, 0xe78eb416
-	.long 0xd2c3ed1a, 0x39c7ff35
-	.long 0x995a5724, 0x61ff0e01
-	.long 0x9ef68d35, 0xd7a4825c
-	.long 0x0c139b31, 0x8d96551c
-	.long 0xf2271e60, 0x0ab3844b
-	.long 0x0b0bf8ca, 0x0bf80dd2
-	.long 0x2664fd8b, 0x0167d312
-	.long 0xed64812d, 0x8821abed
-	.long 0x02ee03b2, 0xf6076544
-	.long 0x8604ae0f, 0x6a45d2b2
-	.long 0x363bd6b3, 0x26f6a60a
-	.long 0x135c83fd, 0xd8d26619
-	.long 0x5fabe670, 0xa741c1bf
-	.long 0x35ec3279, 0xde87806c
-	.long 0x00bcf5f6, 0x98d8d9cb
-	.long 0x8ae00689, 0x14338754
-	.long 0x17f27698, 0x49c3cc9c
-	.long 0x58ca5f00, 0x5bd2011f
-	.long 0xaa7c7ad5, 0x68bce87a
-	.long 0xb5cfca28, 0xdd07448e
-	.long 0xded288f8, 0x57a3d037
-	.long 0x59f229bc, 0xdde8f5b9
-	.long 0x6d390dec, 0x6956fc3b
-	.long 0x37170390, 0xa3e3e02c
-	.long 0x6353c1cc, 0x42d98888
-	.long 0xc4584f5c, 0xd73c7bea
-	.long 0xf48642e9, 0x3771e98f
-	.long 0x531377e2, 0x80ff0093
-	.long 0xdd35bc8d, 0xb42ae3d9
-	.long 0xb25b29f2, 0x8fe4c34d
-	.long 0x9a5ede41, 0x2178513a
-	.long 0xa563905d, 0xdf99fc11
-	.long 0x45cddf4e, 0xe0ac139e
-	.long 0xacfa3103, 0x6c23e841
-	.long 0xa51b6135, 0x170076fa
diff --git a/arch/x86/crypto/crct10dif-pcl-asm_64.S b/arch/x86/crypto/crct10dif-pcl-asm_64.S
deleted file mode 100644
index 5286db5b8165..000000000000
--- a/arch/x86/crypto/crct10dif-pcl-asm_64.S
+++ /dev/null
@@ -1,332 +0,0 @@
-########################################################################
-# Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions
-#
-# Copyright (c) 2013, Intel Corporation
-#
-# Authors:
-#     Erdinc Ozturk <erdinc.ozturk@intel.com>
-#     Vinodh Gopal <vinodh.gopal@intel.com>
-#     James Guilford <james.guilford@intel.com>
-#     Tim Chen <tim.c.chen@linux.intel.com>
-#
-# This software is available to you under a choice of one of two
-# licenses.  You may choose to be licensed under the terms of the GNU
-# General Public License (GPL) Version 2, available from the file
-# COPYING in the main directory of this source tree, or the
-# OpenIB.org BSD license below:
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-#   notice, this list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above copyright
-#   notice, this list of conditions and the following disclaimer in the
-#   documentation and/or other materials provided with the
-#   distribution.
-#
-# * Neither the name of the Intel Corporation nor the names of its
-#   contributors may be used to endorse or promote products derived from
-#   this software without specific prior written permission.
-#
-#
-# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-#       Reference paper titled "Fast CRC Computation for Generic
-#	Polynomials Using PCLMULQDQ Instruction"
-#       URL: http://www.intel.com/content/dam/www/public/us/en/documents
-#  /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
-#
-
-#include <linux/linkage.h>
-
-.text
-
-#define		init_crc	%edi
-#define		buf		%rsi
-#define		len		%rdx
-
-#define		FOLD_CONSTS	%xmm10
-#define		BSWAP_MASK	%xmm11
-
-# Fold reg1, reg2 into the next 32 data bytes, storing the result back into
-# reg1, reg2.
-.macro	fold_32_bytes	offset, reg1, reg2
-	movdqu	\offset(buf), %xmm9
-	movdqu	\offset+16(buf), %xmm12
-	pshufb	BSWAP_MASK, %xmm9
-	pshufb	BSWAP_MASK, %xmm12
-	movdqa	\reg1, %xmm8
-	movdqa	\reg2, %xmm13
-	pclmulqdq	$0x00, FOLD_CONSTS, \reg1
-	pclmulqdq	$0x11, FOLD_CONSTS, %xmm8
-	pclmulqdq	$0x00, FOLD_CONSTS, \reg2
-	pclmulqdq	$0x11, FOLD_CONSTS, %xmm13
-	pxor	%xmm9 , \reg1
-	xorps	%xmm8 , \reg1
-	pxor	%xmm12, \reg2
-	xorps	%xmm13, \reg2
-.endm
-
-# Fold src_reg into dst_reg.
-.macro	fold_16_bytes	src_reg, dst_reg
-	movdqa	\src_reg, %xmm8
-	pclmulqdq	$0x11, FOLD_CONSTS, \src_reg
-	pclmulqdq	$0x00, FOLD_CONSTS, %xmm8
-	pxor	%xmm8, \dst_reg
-	xorps	\src_reg, \dst_reg
-.endm
-
-#
-# u16 crc_t10dif_pcl(u16 init_crc, const *u8 buf, size_t len);
-#
-# Assumes len >= 16.
-#
-SYM_FUNC_START(crc_t10dif_pcl)
-
-	movdqa	.Lbswap_mask(%rip), BSWAP_MASK
-
-	# For sizes less than 256 bytes, we can't fold 128 bytes at a time.
-	cmp	$256, len
-	jl	.Lless_than_256_bytes
-
-	# Load the first 128 data bytes.  Byte swapping is necessary to make the
-	# bit order match the polynomial coefficient order.
-	movdqu	16*0(buf), %xmm0
-	movdqu	16*1(buf), %xmm1
-	movdqu	16*2(buf), %xmm2
-	movdqu	16*3(buf), %xmm3
-	movdqu	16*4(buf), %xmm4
-	movdqu	16*5(buf), %xmm5
-	movdqu	16*6(buf), %xmm6
-	movdqu	16*7(buf), %xmm7
-	add	$128, buf
-	pshufb	BSWAP_MASK, %xmm0
-	pshufb	BSWAP_MASK, %xmm1
-	pshufb	BSWAP_MASK, %xmm2
-	pshufb	BSWAP_MASK, %xmm3
-	pshufb	BSWAP_MASK, %xmm4
-	pshufb	BSWAP_MASK, %xmm5
-	pshufb	BSWAP_MASK, %xmm6
-	pshufb	BSWAP_MASK, %xmm7
-
-	# XOR the first 16 data *bits* with the initial CRC value.
-	pxor	%xmm8, %xmm8
-	pinsrw	$7, init_crc, %xmm8
-	pxor	%xmm8, %xmm0
-
-	movdqa	.Lfold_across_128_bytes_consts(%rip), FOLD_CONSTS
-
-	# Subtract 128 for the 128 data bytes just consumed.  Subtract another
-	# 128 to simplify the termination condition of the following loop.
-	sub	$256, len
-
-	# While >= 128 data bytes remain (not counting xmm0-7), fold the 128
-	# bytes xmm0-7 into them, storing the result back into xmm0-7.
-.Lfold_128_bytes_loop:
-	fold_32_bytes	0, %xmm0, %xmm1
-	fold_32_bytes	32, %xmm2, %xmm3
-	fold_32_bytes	64, %xmm4, %xmm5
-	fold_32_bytes	96, %xmm6, %xmm7
-	add	$128, buf
-	sub	$128, len
-	jge	.Lfold_128_bytes_loop
-
-	# Now fold the 112 bytes in xmm0-xmm6 into the 16 bytes in xmm7.
-
-	# Fold across 64 bytes.
-	movdqa	.Lfold_across_64_bytes_consts(%rip), FOLD_CONSTS
-	fold_16_bytes	%xmm0, %xmm4
-	fold_16_bytes	%xmm1, %xmm5
-	fold_16_bytes	%xmm2, %xmm6
-	fold_16_bytes	%xmm3, %xmm7
-	# Fold across 32 bytes.
-	movdqa	.Lfold_across_32_bytes_consts(%rip), FOLD_CONSTS
-	fold_16_bytes	%xmm4, %xmm6
-	fold_16_bytes	%xmm5, %xmm7
-	# Fold across 16 bytes.
-	movdqa	.Lfold_across_16_bytes_consts(%rip), FOLD_CONSTS
-	fold_16_bytes	%xmm6, %xmm7
-
-	# Add 128 to get the correct number of data bytes remaining in 0...127
-	# (not counting xmm7), following the previous extra subtraction by 128.
-	# Then subtract 16 to simplify the termination condition of the
-	# following loop.
-	add	$128-16, len
-
-	# While >= 16 data bytes remain (not counting xmm7), fold the 16 bytes
-	# xmm7 into them, storing the result back into xmm7.
-	jl	.Lfold_16_bytes_loop_done
-.Lfold_16_bytes_loop:
-	movdqa	%xmm7, %xmm8
-	pclmulqdq	$0x11, FOLD_CONSTS, %xmm7
-	pclmulqdq	$0x00, FOLD_CONSTS, %xmm8
-	pxor	%xmm8, %xmm7
-	movdqu	(buf), %xmm0
-	pshufb	BSWAP_MASK, %xmm0
-	pxor	%xmm0 , %xmm7
-	add	$16, buf
-	sub	$16, len
-	jge	.Lfold_16_bytes_loop
-
-.Lfold_16_bytes_loop_done:
-	# Add 16 to get the correct number of data bytes remaining in 0...15
-	# (not counting xmm7), following the previous extra subtraction by 16.
-	add	$16, len
-	je	.Lreduce_final_16_bytes
-
-.Lhandle_partial_segment:
-	# Reduce the last '16 + len' bytes where 1 <= len <= 15 and the first 16
-	# bytes are in xmm7 and the rest are the remaining data in 'buf'.  To do
-	# this without needing a fold constant for each possible 'len', redivide
-	# the bytes into a first chunk of 'len' bytes and a second chunk of 16
-	# bytes, then fold the first chunk into the second.
-
-	movdqa	%xmm7, %xmm2
-
-	# xmm1 = last 16 original data bytes
-	movdqu	-16(buf, len), %xmm1
-	pshufb	BSWAP_MASK, %xmm1
-
-	# xmm2 = high order part of second chunk: xmm7 left-shifted by 'len' bytes.
-	lea	.Lbyteshift_table+16(%rip), %rax
-	sub	len, %rax
-	movdqu	(%rax), %xmm0
-	pshufb	%xmm0, %xmm2
-
-	# xmm7 = first chunk: xmm7 right-shifted by '16-len' bytes.
-	pxor	.Lmask1(%rip), %xmm0
-	pshufb	%xmm0, %xmm7
-
-	# xmm1 = second chunk: 'len' bytes from xmm1 (low-order bytes),
-	# then '16-len' bytes from xmm2 (high-order bytes).
-	pblendvb	%xmm2, %xmm1	#xmm0 is implicit
-
-	# Fold the first chunk into the second chunk, storing the result in xmm7.
-	movdqa	%xmm7, %xmm8
-	pclmulqdq	$0x11, FOLD_CONSTS, %xmm7
-	pclmulqdq	$0x00, FOLD_CONSTS, %xmm8
-	pxor	%xmm8, %xmm7
-	pxor	%xmm1, %xmm7
-
-.Lreduce_final_16_bytes:
-	# Reduce the 128-bit value M(x), stored in xmm7, to the final 16-bit CRC
-
-	# Load 'x^48 * (x^48 mod G(x))' and 'x^48 * (x^80 mod G(x))'.
-	movdqa	.Lfinal_fold_consts(%rip), FOLD_CONSTS
-
-	# Fold the high 64 bits into the low 64 bits, while also multiplying by
-	# x^64.  This produces a 128-bit value congruent to x^64 * M(x) and
-	# whose low 48 bits are 0.
-	movdqa	%xmm7, %xmm0
-	pclmulqdq	$0x11, FOLD_CONSTS, %xmm7 # high bits * x^48 * (x^80 mod G(x))
-	pslldq	$8, %xmm0
-	pxor	%xmm0, %xmm7			  # + low bits * x^64
-
-	# Fold the high 32 bits into the low 96 bits.  This produces a 96-bit
-	# value congruent to x^64 * M(x) and whose low 48 bits are 0.
-	movdqa	%xmm7, %xmm0
-	pand	.Lmask2(%rip), %xmm0		  # zero high 32 bits
-	psrldq	$12, %xmm7			  # extract high 32 bits
-	pclmulqdq	$0x00, FOLD_CONSTS, %xmm7 # high 32 bits * x^48 * (x^48 mod G(x))
-	pxor	%xmm0, %xmm7			  # + low bits
-
-	# Load G(x) and floor(x^48 / G(x)).
-	movdqa	.Lbarrett_reduction_consts(%rip), FOLD_CONSTS
-
-	# Use Barrett reduction to compute the final CRC value.
-	movdqa	%xmm7, %xmm0
-	pclmulqdq	$0x11, FOLD_CONSTS, %xmm7 # high 32 bits * floor(x^48 / G(x))
-	psrlq	$32, %xmm7			  # /= x^32
-	pclmulqdq	$0x00, FOLD_CONSTS, %xmm7 # *= G(x)
-	psrlq	$48, %xmm0
-	pxor	%xmm7, %xmm0		     # + low 16 nonzero bits
-	# Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of xmm0.
-
-	pextrw	$0, %xmm0, %eax
-	RET
-
-.align 16
-.Lless_than_256_bytes:
-	# Checksumming a buffer of length 16...255 bytes
-
-	# Load the first 16 data bytes.
-	movdqu	(buf), %xmm7
-	pshufb	BSWAP_MASK, %xmm7
-	add	$16, buf
-
-	# XOR the first 16 data *bits* with the initial CRC value.
-	pxor	%xmm0, %xmm0
-	pinsrw	$7, init_crc, %xmm0
-	pxor	%xmm0, %xmm7
-
-	movdqa	.Lfold_across_16_bytes_consts(%rip), FOLD_CONSTS
-	cmp	$16, len
-	je	.Lreduce_final_16_bytes		# len == 16
-	sub	$32, len
-	jge	.Lfold_16_bytes_loop		# 32 <= len <= 255
-	add	$16, len
-	jmp	.Lhandle_partial_segment	# 17 <= len <= 31
-SYM_FUNC_END(crc_t10dif_pcl)
-
-.section	.rodata, "a", @progbits
-.align 16
-
-# Fold constants precomputed from the polynomial 0x18bb7
-# G(x) = x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + x^0
-.Lfold_across_128_bytes_consts:
-	.quad		0x0000000000006123	# x^(8*128)	mod G(x)
-	.quad		0x0000000000002295	# x^(8*128+64)	mod G(x)
-.Lfold_across_64_bytes_consts:
-	.quad		0x0000000000001069	# x^(4*128)	mod G(x)
-	.quad		0x000000000000dd31	# x^(4*128+64)	mod G(x)
-.Lfold_across_32_bytes_consts:
-	.quad		0x000000000000857d	# x^(2*128)	mod G(x)
-	.quad		0x0000000000007acc	# x^(2*128+64)	mod G(x)
-.Lfold_across_16_bytes_consts:
-	.quad		0x000000000000a010	# x^(1*128)	mod G(x)
-	.quad		0x0000000000001faa	# x^(1*128+64)	mod G(x)
-.Lfinal_fold_consts:
-	.quad		0x1368000000000000	# x^48 * (x^48 mod G(x))
-	.quad		0x2d56000000000000	# x^48 * (x^80 mod G(x))
-.Lbarrett_reduction_consts:
-	.quad		0x0000000000018bb7	# G(x)
-	.quad		0x00000001f65a57f8	# floor(x^48 / G(x))
-
-.section	.rodata.cst16.mask1, "aM", @progbits, 16
-.align 16
-.Lmask1:
-	.octa	0x80808080808080808080808080808080
-
-.section	.rodata.cst16.mask2, "aM", @progbits, 16
-.align 16
-.Lmask2:
-	.octa	0x00000000FFFFFFFFFFFFFFFFFFFFFFFF
-
-.section	.rodata.cst16.bswap_mask, "aM", @progbits, 16
-.align 16
-.Lbswap_mask:
-	.octa	0x000102030405060708090A0B0C0D0E0F
-
-.section	.rodata.cst32.byteshift_table, "aM", @progbits, 32
-.align 16
-# For 1 <= len <= 15, the 16-byte vector beginning at &byteshift_table[16 - len]
-# is the index vector to shift left by 'len' bytes, and is also {0x80, ...,
-# 0x80} XOR the index vector to shift right by '16 - len' bytes.
-.Lbyteshift_table:
-	.byte		 0x0, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87
-	.byte		0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f
-	.byte		 0x0,  0x1,  0x2,  0x3,  0x4,  0x5,  0x6,  0x7
-	.byte		 0x8,  0x9,  0xa,  0xb,  0xc,  0xd,  0xe , 0x0
diff --git a/arch/x86/crypto/crct10dif-pclmul_glue.c b/arch/x86/crypto/crct10dif-pclmul_glue.c
deleted file mode 100644
index 71291d5af9f4..000000000000
--- a/arch/x86/crypto/crct10dif-pclmul_glue.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Cryptographic API.
- *
- * T10 Data Integrity Field CRC16 Crypto Transform using PCLMULQDQ Instructions
- *
- * Copyright (C) 2013 Intel Corporation
- * Author: Tim Chen <tim.c.chen@linux.intel.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#include <linux/types.h>
-#include <linux/module.h>
-#include <linux/crc-t10dif.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <linux/init.h>
-#include <linux/string.h>
-#include <linux/kernel.h>
-#include <asm/cpufeatures.h>
-#include <asm/cpu_device_id.h>
-#include <asm/simd.h>
-
-asmlinkage u16 crc_t10dif_pcl(u16 init_crc, const u8 *buf, size_t len);
-
-struct chksum_desc_ctx {
-	__u16 crc;
-};
-
-static int chksum_init(struct shash_desc *desc)
-{
-	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
-
-	ctx->crc = 0;
-
-	return 0;
-}
-
-static int chksum_update(struct shash_desc *desc, const u8 *data,
-			 unsigned int length)
-{
-	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
-
-	if (length >= 16 && crypto_simd_usable()) {
-		kernel_fpu_begin();
-		ctx->crc = crc_t10dif_pcl(ctx->crc, data, length);
-		kernel_fpu_end();
-	} else
-		ctx->crc = crc_t10dif_generic(ctx->crc, data, length);
-	return 0;
-}
-
-static int chksum_final(struct shash_desc *desc, u8 *out)
-{
-	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
-
-	*(__u16 *)out = ctx->crc;
-	return 0;
-}
-
-static int __chksum_finup(__u16 crc, const u8 *data, unsigned int len, u8 *out)
-{
-	if (len >= 16 && crypto_simd_usable()) {
-		kernel_fpu_begin();
-		*(__u16 *)out = crc_t10dif_pcl(crc, data, len);
-		kernel_fpu_end();
-	} else
-		*(__u16 *)out = crc_t10dif_generic(crc, data, len);
-	return 0;
-}
-
-static int chksum_finup(struct shash_desc *desc, const u8 *data,
-			unsigned int len, u8 *out)
-{
-	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
-
-	return __chksum_finup(ctx->crc, data, len, out);
-}
-
-static int chksum_digest(struct shash_desc *desc, const u8 *data,
-			 unsigned int length, u8 *out)
-{
-	return __chksum_finup(0, data, length, out);
-}
-
-static struct shash_alg alg = {
-	.digestsize		=	CRC_T10DIF_DIGEST_SIZE,
-	.init		=	chksum_init,
-	.update		=	chksum_update,
-	.final		=	chksum_final,
-	.finup		=	chksum_finup,
-	.digest		=	chksum_digest,
-	.descsize		=	sizeof(struct chksum_desc_ctx),
-	.base			=	{
-		.cra_name		=	"crct10dif",
-		.cra_driver_name	=	"crct10dif-pclmul",
-		.cra_priority		=	200,
-		.cra_blocksize		=	CRC_T10DIF_BLOCK_SIZE,
-		.cra_module		=	THIS_MODULE,
-	}
-};
-
-static const struct x86_cpu_id crct10dif_cpu_id[] = {
-	X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL),
-	{}
-};
-MODULE_DEVICE_TABLE(x86cpu, crct10dif_cpu_id);
-
-static int __init crct10dif_intel_mod_init(void)
-{
-	if (!x86_match_cpu(crct10dif_cpu_id))
-		return -ENODEV;
-
-	return crypto_register_shash(&alg);
-}
-
-static void __exit crct10dif_intel_mod_fini(void)
-{
-	crypto_unregister_shash(&alg);
-}
-
-module_init(crct10dif_intel_mod_init);
-module_exit(crct10dif_intel_mod_fini);
-
-MODULE_AUTHOR("Tim Chen <tim.c.chen@linux.intel.com>");
-MODULE_DESCRIPTION("T10 DIF CRC calculation accelerated with PCLMULQDQ.");
-MODULE_LICENSE("GPL");
-
-MODULE_ALIAS_CRYPTO("crct10dif");
-MODULE_ALIAS_CRYPTO("crct10dif-pclmul");
diff --git a/arch/x86/crypto/curve25519-x86_64.c b/arch/x86/crypto/curve25519-x86_64.c
deleted file mode 100644
index dcfc0de333de..000000000000
--- a/arch/x86/crypto/curve25519-x86_64.c
+++ /dev/null
@@ -1,1725 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0 OR MIT
-/*
- * Copyright (C) 2020 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
- * Copyright (c) 2016-2020 INRIA, CMU and Microsoft Corporation
- */
-
-#include <crypto/curve25519.h>
-#include <crypto/internal/kpp.h>
-
-#include <linux/types.h>
-#include <linux/jump_label.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/scatterlist.h>
-
-#include <asm/cpufeature.h>
-#include <asm/processor.h>
-
-static __always_inline u64 eq_mask(u64 a, u64 b)
-{
-	u64 x = a ^ b;
-	u64 minus_x = ~x + (u64)1U;
-	u64 x_or_minus_x = x | minus_x;
-	u64 xnx = x_or_minus_x >> (u32)63U;
-	return xnx - (u64)1U;
-}
-
-static __always_inline u64 gte_mask(u64 a, u64 b)
-{
-	u64 x = a;
-	u64 y = b;
-	u64 x_xor_y = x ^ y;
-	u64 x_sub_y = x - y;
-	u64 x_sub_y_xor_y = x_sub_y ^ y;
-	u64 q = x_xor_y | x_sub_y_xor_y;
-	u64 x_xor_q = x ^ q;
-	u64 x_xor_q_ = x_xor_q >> (u32)63U;
-	return x_xor_q_ - (u64)1U;
-}
-
-/* Computes the addition of four-element f1 with value in f2
- * and returns the carry (if any) */
-static inline u64 add_scalar(u64 *out, const u64 *f1, u64 f2)
-{
-	u64 carry_r;
-
-	asm volatile(
-		/* Clear registers to propagate the carry bit */
-		"  xor %%r8d, %%r8d;"
-		"  xor %%r9d, %%r9d;"
-		"  xor %%r10d, %%r10d;"
-		"  xor %%r11d, %%r11d;"
-		"  xor %k1, %k1;"
-
-		/* Begin addition chain */
-		"  addq 0(%3), %0;"
-		"  movq %0, 0(%2);"
-		"  adcxq 8(%3), %%r8;"
-		"  movq %%r8, 8(%2);"
-		"  adcxq 16(%3), %%r9;"
-		"  movq %%r9, 16(%2);"
-		"  adcxq 24(%3), %%r10;"
-		"  movq %%r10, 24(%2);"
-
-		/* Return the carry bit in a register */
-		"  adcx %%r11, %1;"
-		: "+&r"(f2), "=&r"(carry_r)
-		: "r"(out), "r"(f1)
-		: "%r8", "%r9", "%r10", "%r11", "memory", "cc");
-
-	return carry_r;
-}
-
-/* Computes the field addition of two field elements */
-static inline void fadd(u64 *out, const u64 *f1, const u64 *f2)
-{
-	asm volatile(
-		/* Compute the raw addition of f1 + f2 */
-		"  movq 0(%0), %%r8;"
-		"  addq 0(%2), %%r8;"
-		"  movq 8(%0), %%r9;"
-		"  adcxq 8(%2), %%r9;"
-		"  movq 16(%0), %%r10;"
-		"  adcxq 16(%2), %%r10;"
-		"  movq 24(%0), %%r11;"
-		"  adcxq 24(%2), %%r11;"
-
-		/* Wrap the result back into the field */
-
-		/* Step 1: Compute carry*38 */
-		"  mov $0, %%rax;"
-		"  mov $38, %0;"
-		"  cmovc %0, %%rax;"
-
-		/* Step 2: Add carry*38 to the original sum */
-		"  xor %%ecx, %%ecx;"
-		"  add %%rax, %%r8;"
-		"  adcx %%rcx, %%r9;"
-		"  movq %%r9, 8(%1);"
-		"  adcx %%rcx, %%r10;"
-		"  movq %%r10, 16(%1);"
-		"  adcx %%rcx, %%r11;"
-		"  movq %%r11, 24(%1);"
-
-		/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
-		"  mov $0, %%rax;"
-		"  cmovc %0, %%rax;"
-		"  add %%rax, %%r8;"
-		"  movq %%r8, 0(%1);"
-		: "+&r"(f2)
-		: "r"(out), "r"(f1)
-		: "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc");
-}
-
-/* Computes the field subtraction of two field elements */
-static inline void fsub(u64 *out, const u64 *f1, const u64 *f2)
-{
-	asm volatile(
-		/* Compute the raw subtraction of f1-f2 */
-		"  movq 0(%1), %%r8;"
-		"  subq 0(%2), %%r8;"
-		"  movq 8(%1), %%r9;"
-		"  sbbq 8(%2), %%r9;"
-		"  movq 16(%1), %%r10;"
-		"  sbbq 16(%2), %%r10;"
-		"  movq 24(%1), %%r11;"
-		"  sbbq 24(%2), %%r11;"
-
-		/* Wrap the result back into the field */
-
-		/* Step 1: Compute carry*38 */
-		"  mov $0, %%rax;"
-		"  mov $38, %%rcx;"
-		"  cmovc %%rcx, %%rax;"
-
-		/* Step 2: Subtract carry*38 from the original difference */
-		"  sub %%rax, %%r8;"
-		"  sbb $0, %%r9;"
-		"  sbb $0, %%r10;"
-		"  sbb $0, %%r11;"
-
-		/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
-		"  mov $0, %%rax;"
-		"  cmovc %%rcx, %%rax;"
-		"  sub %%rax, %%r8;"
-
-		/* Store the result */
-		"  movq %%r8, 0(%0);"
-		"  movq %%r9, 8(%0);"
-		"  movq %%r10, 16(%0);"
-		"  movq %%r11, 24(%0);"
-		:
-		: "r"(out), "r"(f1), "r"(f2)
-		: "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc");
-}
-
-/* Computes a field multiplication: out <- f1 * f2
- * Uses the 8-element buffer tmp for intermediate results */
-static inline void fmul(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp)
-{
-	asm volatile(
-
-		/* Compute the raw multiplication: tmp <- src1 * src2 */
-
-		/* Compute src1[0] * src2 */
-		"  movq 0(%0), %%rdx;"
-		"  mulxq 0(%1), %%r8, %%r9;"
-		"  xor %%r10d, %%r10d;"
-		"  movq %%r8, 0(%2);"
-		"  mulxq 8(%1), %%r10, %%r11;"
-		"  adox %%r9, %%r10;"
-		"  movq %%r10, 8(%2);"
-		"  mulxq 16(%1), %%rbx, %%r13;"
-		"  adox %%r11, %%rbx;"
-		"  mulxq 24(%1), %%r14, %%rdx;"
-		"  adox %%r13, %%r14;"
-		"  mov $0, %%rax;"
-		"  adox %%rdx, %%rax;"
-
-		/* Compute src1[1] * src2 */
-		"  movq 8(%0), %%rdx;"
-		"  mulxq 0(%1), %%r8, %%r9;"
-		"  xor %%r10d, %%r10d;"
-		"  adcxq 8(%2), %%r8;"
-		"  movq %%r8, 8(%2);"
-		"  mulxq 8(%1), %%r10, %%r11;"
-		"  adox %%r9, %%r10;"
-		"  adcx %%rbx, %%r10;"
-		"  movq %%r10, 16(%2);"
-		"  mulxq 16(%1), %%rbx, %%r13;"
-		"  adox %%r11, %%rbx;"
-		"  adcx %%r14, %%rbx;"
-		"  mov $0, %%r8;"
-		"  mulxq 24(%1), %%r14, %%rdx;"
-		"  adox %%r13, %%r14;"
-		"  adcx %%rax, %%r14;"
-		"  mov $0, %%rax;"
-		"  adox %%rdx, %%rax;"
-		"  adcx %%r8, %%rax;"
-
-		/* Compute src1[2] * src2 */
-		"  movq 16(%0), %%rdx;"
-		"  mulxq 0(%1), %%r8, %%r9;"
-		"  xor %%r10d, %%r10d;"
-		"  adcxq 16(%2), %%r8;"
-		"  movq %%r8, 16(%2);"
-		"  mulxq 8(%1), %%r10, %%r11;"
-		"  adox %%r9, %%r10;"
-		"  adcx %%rbx, %%r10;"
-		"  movq %%r10, 24(%2);"
-		"  mulxq 16(%1), %%rbx, %%r13;"
-		"  adox %%r11, %%rbx;"
-		"  adcx %%r14, %%rbx;"
-		"  mov $0, %%r8;"
-		"  mulxq 24(%1), %%r14, %%rdx;"
-		"  adox %%r13, %%r14;"
-		"  adcx %%rax, %%r14;"
-		"  mov $0, %%rax;"
-		"  adox %%rdx, %%rax;"
-		"  adcx %%r8, %%rax;"
-
-		/* Compute src1[3] * src2 */
-		"  movq 24(%0), %%rdx;"
-		"  mulxq 0(%1), %%r8, %%r9;"
-		"  xor %%r10d, %%r10d;"
-		"  adcxq 24(%2), %%r8;"
-		"  movq %%r8, 24(%2);"
-		"  mulxq 8(%1), %%r10, %%r11;"
-		"  adox %%r9, %%r10;"
-		"  adcx %%rbx, %%r10;"
-		"  movq %%r10, 32(%2);"
-		"  mulxq 16(%1), %%rbx, %%r13;"
-		"  adox %%r11, %%rbx;"
-		"  adcx %%r14, %%rbx;"
-		"  movq %%rbx, 40(%2);"
-		"  mov $0, %%r8;"
-		"  mulxq 24(%1), %%r14, %%rdx;"
-		"  adox %%r13, %%r14;"
-		"  adcx %%rax, %%r14;"
-		"  movq %%r14, 48(%2);"
-		"  mov $0, %%rax;"
-		"  adox %%rdx, %%rax;"
-		"  adcx %%r8, %%rax;"
-		"  movq %%rax, 56(%2);"
-
-		/* Line up pointers */
-		"  mov %2, %0;"
-		"  mov %3, %2;"
-
-		/* Wrap the result back into the field */
-
-		/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
-		"  mov $38, %%rdx;"
-		"  mulxq 32(%0), %%r8, %%r13;"
-		"  xor %k1, %k1;"
-		"  adoxq 0(%0), %%r8;"
-		"  mulxq 40(%0), %%r9, %%rbx;"
-		"  adcx %%r13, %%r9;"
-		"  adoxq 8(%0), %%r9;"
-		"  mulxq 48(%0), %%r10, %%r13;"
-		"  adcx %%rbx, %%r10;"
-		"  adoxq 16(%0), %%r10;"
-		"  mulxq 56(%0), %%r11, %%rax;"
-		"  adcx %%r13, %%r11;"
-		"  adoxq 24(%0), %%r11;"
-		"  adcx %1, %%rax;"
-		"  adox %1, %%rax;"
-		"  imul %%rdx, %%rax;"
-
-		/* Step 2: Fold the carry back into dst */
-		"  add %%rax, %%r8;"
-		"  adcx %1, %%r9;"
-		"  movq %%r9, 8(%2);"
-		"  adcx %1, %%r10;"
-		"  movq %%r10, 16(%2);"
-		"  adcx %1, %%r11;"
-		"  movq %%r11, 24(%2);"
-
-		/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
-		"  mov $0, %%rax;"
-		"  cmovc %%rdx, %%rax;"
-		"  add %%rax, %%r8;"
-		"  movq %%r8, 0(%2);"
-		: "+&r"(f1), "+&r"(f2), "+&r"(tmp)
-		: "r"(out)
-		: "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13",
-		  "%r14", "memory", "cc");
-}
-
-/* Computes two field multiplications:
- *   out[0] <- f1[0] * f2[0]
- *   out[1] <- f1[1] * f2[1]
- * Uses the 16-element buffer tmp for intermediate results: */
-static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp)
-{
-	asm volatile(
-
-		/* Compute the raw multiplication tmp[0] <- f1[0] * f2[0] */
-
-		/* Compute src1[0] * src2 */
-		"  movq 0(%0), %%rdx;"
-		"  mulxq 0(%1), %%r8, %%r9;"
-		"  xor %%r10d, %%r10d;"
-		"  movq %%r8, 0(%2);"
-		"  mulxq 8(%1), %%r10, %%r11;"
-		"  adox %%r9, %%r10;"
-		"  movq %%r10, 8(%2);"
-		"  mulxq 16(%1), %%rbx, %%r13;"
-		"  adox %%r11, %%rbx;"
-		"  mulxq 24(%1), %%r14, %%rdx;"
-		"  adox %%r13, %%r14;"
-		"  mov $0, %%rax;"
-		"  adox %%rdx, %%rax;"
-
-		/* Compute src1[1] * src2 */
-		"  movq 8(%0), %%rdx;"
-		"  mulxq 0(%1), %%r8, %%r9;"
-		"  xor %%r10d, %%r10d;"
-		"  adcxq 8(%2), %%r8;"
-		"  movq %%r8, 8(%2);"
-		"  mulxq 8(%1), %%r10, %%r11;"
-		"  adox %%r9, %%r10;"
-		"  adcx %%rbx, %%r10;"
-		"  movq %%r10, 16(%2);"
-		"  mulxq 16(%1), %%rbx, %%r13;"
-		"  adox %%r11, %%rbx;"
-		"  adcx %%r14, %%rbx;"
-		"  mov $0, %%r8;"
-		"  mulxq 24(%1), %%r14, %%rdx;"
-		"  adox %%r13, %%r14;"
-		"  adcx %%rax, %%r14;"
-		"  mov $0, %%rax;"
-		"  adox %%rdx, %%rax;"
-		"  adcx %%r8, %%rax;"
-
-		/* Compute src1[2] * src2 */
-		"  movq 16(%0), %%rdx;"
-		"  mulxq 0(%1), %%r8, %%r9;"
-		"  xor %%r10d, %%r10d;"
-		"  adcxq 16(%2), %%r8;"
-		"  movq %%r8, 16(%2);"
-		"  mulxq 8(%1), %%r10, %%r11;"
-		"  adox %%r9, %%r10;"
-		"  adcx %%rbx, %%r10;"
-		"  movq %%r10, 24(%2);"
-		"  mulxq 16(%1), %%rbx, %%r13;"
-		"  adox %%r11, %%rbx;"
-		"  adcx %%r14, %%rbx;"
-		"  mov $0, %%r8;"
-		"  mulxq 24(%1), %%r14, %%rdx;"
-		"  adox %%r13, %%r14;"
-		"  adcx %%rax, %%r14;"
-		"  mov $0, %%rax;"
-		"  adox %%rdx, %%rax;"
-		"  adcx %%r8, %%rax;"
-
-		/* Compute src1[3] * src2 */
-		"  movq 24(%0), %%rdx;"
-		"  mulxq 0(%1), %%r8, %%r9;"
-		"  xor %%r10d, %%r10d;"
-		"  adcxq 24(%2), %%r8;"
-		"  movq %%r8, 24(%2);"
-		"  mulxq 8(%1), %%r10, %%r11;"
-		"  adox %%r9, %%r10;"
-		"  adcx %%rbx, %%r10;"
-		"  movq %%r10, 32(%2);"
-		"  mulxq 16(%1), %%rbx, %%r13;"
-		"  adox %%r11, %%rbx;"
-		"  adcx %%r14, %%rbx;"
-		"  movq %%rbx, 40(%2);"
-		"  mov $0, %%r8;"
-		"  mulxq 24(%1), %%r14, %%rdx;"
-		"  adox %%r13, %%r14;"
-		"  adcx %%rax, %%r14;"
-		"  movq %%r14, 48(%2);"
-		"  mov $0, %%rax;"
-		"  adox %%rdx, %%rax;"
-		"  adcx %%r8, %%rax;"
-		"  movq %%rax, 56(%2);"
-
-		/* Compute the raw multiplication tmp[1] <- f1[1] * f2[1] */
-
-		/* Compute src1[0] * src2 */
-		"  movq 32(%0), %%rdx;"
-		"  mulxq 32(%1), %%r8, %%r9;"
-		"  xor %%r10d, %%r10d;"
-		"  movq %%r8, 64(%2);"
-		"  mulxq 40(%1), %%r10, %%r11;"
-		"  adox %%r9, %%r10;"
-		"  movq %%r10, 72(%2);"
-		"  mulxq 48(%1), %%rbx, %%r13;"
-		"  adox %%r11, %%rbx;"
-		"  mulxq 56(%1), %%r14, %%rdx;"
-		"  adox %%r13, %%r14;"
-		"  mov $0, %%rax;"
-		"  adox %%rdx, %%rax;"
-
-		/* Compute src1[1] * src2 */
-		"  movq 40(%0), %%rdx;"
-		"  mulxq 32(%1), %%r8, %%r9;"
-		"  xor %%r10d, %%r10d;"
-		"  adcxq 72(%2), %%r8;"
-		"  movq %%r8, 72(%2);"
-		"  mulxq 40(%1), %%r10, %%r11;"
-		"  adox %%r9, %%r10;"
-		"  adcx %%rbx, %%r10;"
-		"  movq %%r10, 80(%2);"
-		"  mulxq 48(%1), %%rbx, %%r13;"
-		"  adox %%r11, %%rbx;"
-		"  adcx %%r14, %%rbx;"
-		"  mov $0, %%r8;"
-		"  mulxq 56(%1), %%r14, %%rdx;"
-		"  adox %%r13, %%r14;"
-		"  adcx %%rax, %%r14;"
-		"  mov $0, %%rax;"
-		"  adox %%rdx, %%rax;"
-		"  adcx %%r8, %%rax;"
-
-		/* Compute src1[2] * src2 */
-		"  movq 48(%0), %%rdx;"
-		"  mulxq 32(%1), %%r8, %%r9;"
-		"  xor %%r10d, %%r10d;"
-		"  adcxq 80(%2), %%r8;"
-		"  movq %%r8, 80(%2);"
-		"  mulxq 40(%1), %%r10, %%r11;"
-		"  adox %%r9, %%r10;"
-		"  adcx %%rbx, %%r10;"
-		"  movq %%r10, 88(%2);"
-		"  mulxq 48(%1), %%rbx, %%r13;"
-		"  adox %%r11, %%rbx;"
-		"  adcx %%r14, %%rbx;"
-		"  mov $0, %%r8;"
-		"  mulxq 56(%1), %%r14, %%rdx;"
-		"  adox %%r13, %%r14;"
-		"  adcx %%rax, %%r14;"
-		"  mov $0, %%rax;"
-		"  adox %%rdx, %%rax;"
-		"  adcx %%r8, %%rax;"
-
-		/* Compute src1[3] * src2 */
-		"  movq 56(%0), %%rdx;"
-		"  mulxq 32(%1), %%r8, %%r9;"
-		"  xor %%r10d, %%r10d;"
-		"  adcxq 88(%2), %%r8;"
-		"  movq %%r8, 88(%2);"
-		"  mulxq 40(%1), %%r10, %%r11;"
-		"  adox %%r9, %%r10;"
-		"  adcx %%rbx, %%r10;"
-		"  movq %%r10, 96(%2);"
-		"  mulxq 48(%1), %%rbx, %%r13;"
-		"  adox %%r11, %%rbx;"
-		"  adcx %%r14, %%rbx;"
-		"  movq %%rbx, 104(%2);"
-		"  mov $0, %%r8;"
-		"  mulxq 56(%1), %%r14, %%rdx;"
-		"  adox %%r13, %%r14;"
-		"  adcx %%rax, %%r14;"
-		"  movq %%r14, 112(%2);"
-		"  mov $0, %%rax;"
-		"  adox %%rdx, %%rax;"
-		"  adcx %%r8, %%rax;"
-		"  movq %%rax, 120(%2);"
-
-		/* Line up pointers */
-		"  mov %2, %0;"
-		"  mov %3, %2;"
-
-		/* Wrap the results back into the field */
-
-		/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
-		"  mov $38, %%rdx;"
-		"  mulxq 32(%0), %%r8, %%r13;"
-		"  xor %k1, %k1;"
-		"  adoxq 0(%0), %%r8;"
-		"  mulxq 40(%0), %%r9, %%rbx;"
-		"  adcx %%r13, %%r9;"
-		"  adoxq 8(%0), %%r9;"
-		"  mulxq 48(%0), %%r10, %%r13;"
-		"  adcx %%rbx, %%r10;"
-		"  adoxq 16(%0), %%r10;"
-		"  mulxq 56(%0), %%r11, %%rax;"
-		"  adcx %%r13, %%r11;"
-		"  adoxq 24(%0), %%r11;"
-		"  adcx %1, %%rax;"
-		"  adox %1, %%rax;"
-		"  imul %%rdx, %%rax;"
-
-		/* Step 2: Fold the carry back into dst */
-		"  add %%rax, %%r8;"
-		"  adcx %1, %%r9;"
-		"  movq %%r9, 8(%2);"
-		"  adcx %1, %%r10;"
-		"  movq %%r10, 16(%2);"
-		"  adcx %1, %%r11;"
-		"  movq %%r11, 24(%2);"
-
-		/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
-		"  mov $0, %%rax;"
-		"  cmovc %%rdx, %%rax;"
-		"  add %%rax, %%r8;"
-		"  movq %%r8, 0(%2);"
-
-		/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
-		"  mov $38, %%rdx;"
-		"  mulxq 96(%0), %%r8, %%r13;"
-		"  xor %k1, %k1;"
-		"  adoxq 64(%0), %%r8;"
-		"  mulxq 104(%0), %%r9, %%rbx;"
-		"  adcx %%r13, %%r9;"
-		"  adoxq 72(%0), %%r9;"
-		"  mulxq 112(%0), %%r10, %%r13;"
-		"  adcx %%rbx, %%r10;"
-		"  adoxq 80(%0), %%r10;"
-		"  mulxq 120(%0), %%r11, %%rax;"
-		"  adcx %%r13, %%r11;"
-		"  adoxq 88(%0), %%r11;"
-		"  adcx %1, %%rax;"
-		"  adox %1, %%rax;"
-		"  imul %%rdx, %%rax;"
-
-		/* Step 2: Fold the carry back into dst */
-		"  add %%rax, %%r8;"
-		"  adcx %1, %%r9;"
-		"  movq %%r9, 40(%2);"
-		"  adcx %1, %%r10;"
-		"  movq %%r10, 48(%2);"
-		"  adcx %1, %%r11;"
-		"  movq %%r11, 56(%2);"
-
-		/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
-		"  mov $0, %%rax;"
-		"  cmovc %%rdx, %%rax;"
-		"  add %%rax, %%r8;"
-		"  movq %%r8, 32(%2);"
-		: "+&r"(f1), "+&r"(f2), "+&r"(tmp)
-		: "r"(out)
-		: "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13",
-		  "%r14", "memory", "cc");
-}
-
-/* Computes the field multiplication of four-element f1 with value in f2
- * Requires f2 to be smaller than 2^17 */
-static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2)
-{
-	register u64 f2_r asm("rdx") = f2;
-
-	asm volatile(
-		/* Compute the raw multiplication of f1*f2 */
-		"  mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */
-		"  mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */
-		"  add %%rcx, %%r9;"
-		"  mov $0, %%rcx;"
-		"  mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */
-		"  adcx %%rbx, %%r10;"
-		"  mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */
-		"  adcx %%r13, %%r11;"
-		"  adcx %%rcx, %%rax;"
-
-		/* Wrap the result back into the field */
-
-		/* Step 1: Compute carry*38 */
-		"  mov $38, %%rdx;"
-		"  imul %%rdx, %%rax;"
-
-		/* Step 2: Fold the carry back into dst */
-		"  add %%rax, %%r8;"
-		"  adcx %%rcx, %%r9;"
-		"  movq %%r9, 8(%1);"
-		"  adcx %%rcx, %%r10;"
-		"  movq %%r10, 16(%1);"
-		"  adcx %%rcx, %%r11;"
-		"  movq %%r11, 24(%1);"
-
-		/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
-		"  mov $0, %%rax;"
-		"  cmovc %%rdx, %%rax;"
-		"  add %%rax, %%r8;"
-		"  movq %%r8, 0(%1);"
-		: "+&r"(f2_r)
-		: "r"(out), "r"(f1)
-		: "%rax", "%rbx", "%rcx", "%r8", "%r9", "%r10", "%r11", "%r13",
-		  "memory", "cc");
-}
-
-/* Computes p1 <- bit ? p2 : p1 in constant time */
-static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2)
-{
-	asm volatile(
-		/* Transfer bit into CF flag */
-		"  add $18446744073709551615, %0;"
-
-		/* cswap p1[0], p2[0] */
-		"  movq 0(%1), %%r8;"
-		"  movq 0(%2), %%r9;"
-		"  mov %%r8, %%r10;"
-		"  cmovc %%r9, %%r8;"
-		"  cmovc %%r10, %%r9;"
-		"  movq %%r8, 0(%1);"
-		"  movq %%r9, 0(%2);"
-
-		/* cswap p1[1], p2[1] */
-		"  movq 8(%1), %%r8;"
-		"  movq 8(%2), %%r9;"
-		"  mov %%r8, %%r10;"
-		"  cmovc %%r9, %%r8;"
-		"  cmovc %%r10, %%r9;"
-		"  movq %%r8, 8(%1);"
-		"  movq %%r9, 8(%2);"
-
-		/* cswap p1[2], p2[2] */
-		"  movq 16(%1), %%r8;"
-		"  movq 16(%2), %%r9;"
-		"  mov %%r8, %%r10;"
-		"  cmovc %%r9, %%r8;"
-		"  cmovc %%r10, %%r9;"
-		"  movq %%r8, 16(%1);"
-		"  movq %%r9, 16(%2);"
-
-		/* cswap p1[3], p2[3] */
-		"  movq 24(%1), %%r8;"
-		"  movq 24(%2), %%r9;"
-		"  mov %%r8, %%r10;"
-		"  cmovc %%r9, %%r8;"
-		"  cmovc %%r10, %%r9;"
-		"  movq %%r8, 24(%1);"
-		"  movq %%r9, 24(%2);"
-
-		/* cswap p1[4], p2[4] */
-		"  movq 32(%1), %%r8;"
-		"  movq 32(%2), %%r9;"
-		"  mov %%r8, %%r10;"
-		"  cmovc %%r9, %%r8;"
-		"  cmovc %%r10, %%r9;"
-		"  movq %%r8, 32(%1);"
-		"  movq %%r9, 32(%2);"
-
-		/* cswap p1[5], p2[5] */
-		"  movq 40(%1), %%r8;"
-		"  movq 40(%2), %%r9;"
-		"  mov %%r8, %%r10;"
-		"  cmovc %%r9, %%r8;"
-		"  cmovc %%r10, %%r9;"
-		"  movq %%r8, 40(%1);"
-		"  movq %%r9, 40(%2);"
-
-		/* cswap p1[6], p2[6] */
-		"  movq 48(%1), %%r8;"
-		"  movq 48(%2), %%r9;"
-		"  mov %%r8, %%r10;"
-		"  cmovc %%r9, %%r8;"
-		"  cmovc %%r10, %%r9;"
-		"  movq %%r8, 48(%1);"
-		"  movq %%r9, 48(%2);"
-
-		/* cswap p1[7], p2[7] */
-		"  movq 56(%1), %%r8;"
-		"  movq 56(%2), %%r9;"
-		"  mov %%r8, %%r10;"
-		"  cmovc %%r9, %%r8;"
-		"  cmovc %%r10, %%r9;"
-		"  movq %%r8, 56(%1);"
-		"  movq %%r9, 56(%2);"
-		: "+&r"(bit)
-		: "r"(p1), "r"(p2)
-		: "%r8", "%r9", "%r10", "memory", "cc");
-}
-
-/* Computes the square of a field element: out <- f * f
- * Uses the 8-element buffer tmp for intermediate results */
-static inline void fsqr(u64 *out, const u64 *f, u64 *tmp)
-{
-	asm volatile(
-		/* Compute the raw multiplication: tmp <- f * f */
-
-		/* Step 1: Compute all partial products */
-		"  movq 0(%0), %%rdx;" /* f[0] */
-		"  mulxq 8(%0), %%r8, %%r14;"
-		"  xor %%r15d, %%r15d;" /* f[1]*f[0] */
-		"  mulxq 16(%0), %%r9, %%r10;"
-		"  adcx %%r14, %%r9;" /* f[2]*f[0] */
-		"  mulxq 24(%0), %%rax, %%rcx;"
-		"  adcx %%rax, %%r10;" /* f[3]*f[0] */
-		"  movq 24(%0), %%rdx;" /* f[3] */
-		"  mulxq 8(%0), %%r11, %%rbx;"
-		"  adcx %%rcx, %%r11;" /* f[1]*f[3] */
-		"  mulxq 16(%0), %%rax, %%r13;"
-		"  adcx %%rax, %%rbx;" /* f[2]*f[3] */
-		"  movq 8(%0), %%rdx;"
-		"  adcx %%r15, %%r13;" /* f1 */
-		"  mulxq 16(%0), %%rax, %%rcx;"
-		"  mov $0, %%r14;" /* f[2]*f[1] */
-
-		/* Step 2: Compute two parallel carry chains */
-		"  xor %%r15d, %%r15d;"
-		"  adox %%rax, %%r10;"
-		"  adcx %%r8, %%r8;"
-		"  adox %%rcx, %%r11;"
-		"  adcx %%r9, %%r9;"
-		"  adox %%r15, %%rbx;"
-		"  adcx %%r10, %%r10;"
-		"  adox %%r15, %%r13;"
-		"  adcx %%r11, %%r11;"
-		"  adox %%r15, %%r14;"
-		"  adcx %%rbx, %%rbx;"
-		"  adcx %%r13, %%r13;"
-		"  adcx %%r14, %%r14;"
-
-		/* Step 3: Compute intermediate squares */
-		"  movq 0(%0), %%rdx;"
-		"  mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
-		"  movq %%rax, 0(%1);"
-		"  add %%rcx, %%r8;"
-		"  movq %%r8, 8(%1);"
-		"  movq 8(%0), %%rdx;"
-		"  mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
-		"  adcx %%rax, %%r9;"
-		"  movq %%r9, 16(%1);"
-		"  adcx %%rcx, %%r10;"
-		"  movq %%r10, 24(%1);"
-		"  movq 16(%0), %%rdx;"
-		"  mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
-		"  adcx %%rax, %%r11;"
-		"  movq %%r11, 32(%1);"
-		"  adcx %%rcx, %%rbx;"
-		"  movq %%rbx, 40(%1);"
-		"  movq 24(%0), %%rdx;"
-		"  mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
-		"  adcx %%rax, %%r13;"
-		"  movq %%r13, 48(%1);"
-		"  adcx %%rcx, %%r14;"
-		"  movq %%r14, 56(%1);"
-
-		/* Line up pointers */
-		"  mov %1, %0;"
-		"  mov %2, %1;"
-
-		/* Wrap the result back into the field */
-
-		/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
-		"  mov $38, %%rdx;"
-		"  mulxq 32(%0), %%r8, %%r13;"
-		"  xor %%ecx, %%ecx;"
-		"  adoxq 0(%0), %%r8;"
-		"  mulxq 40(%0), %%r9, %%rbx;"
-		"  adcx %%r13, %%r9;"
-		"  adoxq 8(%0), %%r9;"
-		"  mulxq 48(%0), %%r10, %%r13;"
-		"  adcx %%rbx, %%r10;"
-		"  adoxq 16(%0), %%r10;"
-		"  mulxq 56(%0), %%r11, %%rax;"
-		"  adcx %%r13, %%r11;"
-		"  adoxq 24(%0), %%r11;"
-		"  adcx %%rcx, %%rax;"
-		"  adox %%rcx, %%rax;"
-		"  imul %%rdx, %%rax;"
-
-		/* Step 2: Fold the carry back into dst */
-		"  add %%rax, %%r8;"
-		"  adcx %%rcx, %%r9;"
-		"  movq %%r9, 8(%1);"
-		"  adcx %%rcx, %%r10;"
-		"  movq %%r10, 16(%1);"
-		"  adcx %%rcx, %%r11;"
-		"  movq %%r11, 24(%1);"
-
-		/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
-		"  mov $0, %%rax;"
-		"  cmovc %%rdx, %%rax;"
-		"  add %%rax, %%r8;"
-		"  movq %%r8, 0(%1);"
-		: "+&r"(f), "+&r"(tmp)
-		: "r"(out)
-		: "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11",
-		  "%r13", "%r14", "%r15", "memory", "cc");
-}
-
-/* Computes two field squarings:
- *   out[0] <- f[0] * f[0]
- *   out[1] <- f[1] * f[1]
- * Uses the 16-element buffer tmp for intermediate results */
-static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
-{
-	asm volatile(
-		/* Step 1: Compute all partial products */
-		"  movq 0(%0), %%rdx;" /* f[0] */
-		"  mulxq 8(%0), %%r8, %%r14;"
-		"  xor %%r15d, %%r15d;" /* f[1]*f[0] */
-		"  mulxq 16(%0), %%r9, %%r10;"
-		"  adcx %%r14, %%r9;" /* f[2]*f[0] */
-		"  mulxq 24(%0), %%rax, %%rcx;"
-		"  adcx %%rax, %%r10;" /* f[3]*f[0] */
-		"  movq 24(%0), %%rdx;" /* f[3] */
-		"  mulxq 8(%0), %%r11, %%rbx;"
-		"  adcx %%rcx, %%r11;" /* f[1]*f[3] */
-		"  mulxq 16(%0), %%rax, %%r13;"
-		"  adcx %%rax, %%rbx;" /* f[2]*f[3] */
-		"  movq 8(%0), %%rdx;"
-		"  adcx %%r15, %%r13;" /* f1 */
-		"  mulxq 16(%0), %%rax, %%rcx;"
-		"  mov $0, %%r14;" /* f[2]*f[1] */
-
-		/* Step 2: Compute two parallel carry chains */
-		"  xor %%r15d, %%r15d;"
-		"  adox %%rax, %%r10;"
-		"  adcx %%r8, %%r8;"
-		"  adox %%rcx, %%r11;"
-		"  adcx %%r9, %%r9;"
-		"  adox %%r15, %%rbx;"
-		"  adcx %%r10, %%r10;"
-		"  adox %%r15, %%r13;"
-		"  adcx %%r11, %%r11;"
-		"  adox %%r15, %%r14;"
-		"  adcx %%rbx, %%rbx;"
-		"  adcx %%r13, %%r13;"
-		"  adcx %%r14, %%r14;"
-
-		/* Step 3: Compute intermediate squares */
-		"  movq 0(%0), %%rdx;"
-		"  mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
-		"  movq %%rax, 0(%1);"
-		"  add %%rcx, %%r8;"
-		"  movq %%r8, 8(%1);"
-		"  movq 8(%0), %%rdx;"
-		"  mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
-		"  adcx %%rax, %%r9;"
-		"  movq %%r9, 16(%1);"
-		"  adcx %%rcx, %%r10;"
-		"  movq %%r10, 24(%1);"
-		"  movq 16(%0), %%rdx;"
-		"  mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
-		"  adcx %%rax, %%r11;"
-		"  movq %%r11, 32(%1);"
-		"  adcx %%rcx, %%rbx;"
-		"  movq %%rbx, 40(%1);"
-		"  movq 24(%0), %%rdx;"
-		"  mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
-		"  adcx %%rax, %%r13;"
-		"  movq %%r13, 48(%1);"
-		"  adcx %%rcx, %%r14;"
-		"  movq %%r14, 56(%1);"
-
-		/* Step 1: Compute all partial products */
-		"  movq 32(%0), %%rdx;" /* f[0] */
-		"  mulxq 40(%0), %%r8, %%r14;"
-		"  xor %%r15d, %%r15d;" /* f[1]*f[0] */
-		"  mulxq 48(%0), %%r9, %%r10;"
-		"  adcx %%r14, %%r9;" /* f[2]*f[0] */
-		"  mulxq 56(%0), %%rax, %%rcx;"
-		"  adcx %%rax, %%r10;" /* f[3]*f[0] */
-		"  movq 56(%0), %%rdx;" /* f[3] */
-		"  mulxq 40(%0), %%r11, %%rbx;"
-		"  adcx %%rcx, %%r11;" /* f[1]*f[3] */
-		"  mulxq 48(%0), %%rax, %%r13;"
-		"  adcx %%rax, %%rbx;" /* f[2]*f[3] */
-		"  movq 40(%0), %%rdx;"
-		"  adcx %%r15, %%r13;" /* f1 */
-		"  mulxq 48(%0), %%rax, %%rcx;"
-		"  mov $0, %%r14;" /* f[2]*f[1] */
-
-		/* Step 2: Compute two parallel carry chains */
-		"  xor %%r15d, %%r15d;"
-		"  adox %%rax, %%r10;"
-		"  adcx %%r8, %%r8;"
-		"  adox %%rcx, %%r11;"
-		"  adcx %%r9, %%r9;"
-		"  adox %%r15, %%rbx;"
-		"  adcx %%r10, %%r10;"
-		"  adox %%r15, %%r13;"
-		"  adcx %%r11, %%r11;"
-		"  adox %%r15, %%r14;"
-		"  adcx %%rbx, %%rbx;"
-		"  adcx %%r13, %%r13;"
-		"  adcx %%r14, %%r14;"
-
-		/* Step 3: Compute intermediate squares */
-		"  movq 32(%0), %%rdx;"
-		"  mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
-		"  movq %%rax, 64(%1);"
-		"  add %%rcx, %%r8;"
-		"  movq %%r8, 72(%1);"
-		"  movq 40(%0), %%rdx;"
-		"  mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
-		"  adcx %%rax, %%r9;"
-		"  movq %%r9, 80(%1);"
-		"  adcx %%rcx, %%r10;"
-		"  movq %%r10, 88(%1);"
-		"  movq 48(%0), %%rdx;"
-		"  mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
-		"  adcx %%rax, %%r11;"
-		"  movq %%r11, 96(%1);"
-		"  adcx %%rcx, %%rbx;"
-		"  movq %%rbx, 104(%1);"
-		"  movq 56(%0), %%rdx;"
-		"  mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
-		"  adcx %%rax, %%r13;"
-		"  movq %%r13, 112(%1);"
-		"  adcx %%rcx, %%r14;"
-		"  movq %%r14, 120(%1);"
-
-		/* Line up pointers */
-		"  mov %1, %0;"
-		"  mov %2, %1;"
-
-		/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
-		"  mov $38, %%rdx;"
-		"  mulxq 32(%0), %%r8, %%r13;"
-		"  xor %%ecx, %%ecx;"
-		"  adoxq 0(%0), %%r8;"
-		"  mulxq 40(%0), %%r9, %%rbx;"
-		"  adcx %%r13, %%r9;"
-		"  adoxq 8(%0), %%r9;"
-		"  mulxq 48(%0), %%r10, %%r13;"
-		"  adcx %%rbx, %%r10;"
-		"  adoxq 16(%0), %%r10;"
-		"  mulxq 56(%0), %%r11, %%rax;"
-		"  adcx %%r13, %%r11;"
-		"  adoxq 24(%0), %%r11;"
-		"  adcx %%rcx, %%rax;"
-		"  adox %%rcx, %%rax;"
-		"  imul %%rdx, %%rax;"
-
-		/* Step 2: Fold the carry back into dst */
-		"  add %%rax, %%r8;"
-		"  adcx %%rcx, %%r9;"
-		"  movq %%r9, 8(%1);"
-		"  adcx %%rcx, %%r10;"
-		"  movq %%r10, 16(%1);"
-		"  adcx %%rcx, %%r11;"
-		"  movq %%r11, 24(%1);"
-
-		/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
-		"  mov $0, %%rax;"
-		"  cmovc %%rdx, %%rax;"
-		"  add %%rax, %%r8;"
-		"  movq %%r8, 0(%1);"
-
-		/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
-		"  mov $38, %%rdx;"
-		"  mulxq 96(%0), %%r8, %%r13;"
-		"  xor %%ecx, %%ecx;"
-		"  adoxq 64(%0), %%r8;"
-		"  mulxq 104(%0), %%r9, %%rbx;"
-		"  adcx %%r13, %%r9;"
-		"  adoxq 72(%0), %%r9;"
-		"  mulxq 112(%0), %%r10, %%r13;"
-		"  adcx %%rbx, %%r10;"
-		"  adoxq 80(%0), %%r10;"
-		"  mulxq 120(%0), %%r11, %%rax;"
-		"  adcx %%r13, %%r11;"
-		"  adoxq 88(%0), %%r11;"
-		"  adcx %%rcx, %%rax;"
-		"  adox %%rcx, %%rax;"
-		"  imul %%rdx, %%rax;"
-
-		/* Step 2: Fold the carry back into dst */
-		"  add %%rax, %%r8;"
-		"  adcx %%rcx, %%r9;"
-		"  movq %%r9, 40(%1);"
-		"  adcx %%rcx, %%r10;"
-		"  movq %%r10, 48(%1);"
-		"  adcx %%rcx, %%r11;"
-		"  movq %%r11, 56(%1);"
-
-		/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
-		"  mov $0, %%rax;"
-		"  cmovc %%rdx, %%rax;"
-		"  add %%rax, %%r8;"
-		"  movq %%r8, 32(%1);"
-		: "+&r"(f), "+&r"(tmp)
-		: "r"(out)
-		: "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11",
-		  "%r13", "%r14", "%r15", "memory", "cc");
-}
-
-static void point_add_and_double(u64 *q, u64 *p01_tmp1, u64 *tmp2)
-{
-	u64 *nq = p01_tmp1;
-	u64 *nq_p1 = p01_tmp1 + (u32)8U;
-	u64 *tmp1 = p01_tmp1 + (u32)16U;
-	u64 *x1 = q;
-	u64 *x2 = nq;
-	u64 *z2 = nq + (u32)4U;
-	u64 *z3 = nq_p1 + (u32)4U;
-	u64 *a = tmp1;
-	u64 *b = tmp1 + (u32)4U;
-	u64 *ab = tmp1;
-	u64 *dc = tmp1 + (u32)8U;
-	u64 *x3;
-	u64 *z31;
-	u64 *d0;
-	u64 *c0;
-	u64 *a1;
-	u64 *b1;
-	u64 *d;
-	u64 *c;
-	u64 *ab1;
-	u64 *dc1;
-	fadd(a, x2, z2);
-	fsub(b, x2, z2);
-	x3 = nq_p1;
-	z31 = nq_p1 + (u32)4U;
-	d0 = dc;
-	c0 = dc + (u32)4U;
-	fadd(c0, x3, z31);
-	fsub(d0, x3, z31);
-	fmul2(dc, dc, ab, tmp2);
-	fadd(x3, d0, c0);
-	fsub(z31, d0, c0);
-	a1 = tmp1;
-	b1 = tmp1 + (u32)4U;
-	d = tmp1 + (u32)8U;
-	c = tmp1 + (u32)12U;
-	ab1 = tmp1;
-	dc1 = tmp1 + (u32)8U;
-	fsqr2(dc1, ab1, tmp2);
-	fsqr2(nq_p1, nq_p1, tmp2);
-	a1[0U] = c[0U];
-	a1[1U] = c[1U];
-	a1[2U] = c[2U];
-	a1[3U] = c[3U];
-	fsub(c, d, c);
-	fmul_scalar(b1, c, (u64)121665U);
-	fadd(b1, b1, d);
-	fmul2(nq, dc1, ab1, tmp2);
-	fmul(z3, z3, x1, tmp2);
-}
-
-static void point_double(u64 *nq, u64 *tmp1, u64 *tmp2)
-{
-	u64 *x2 = nq;
-	u64 *z2 = nq + (u32)4U;
-	u64 *a = tmp1;
-	u64 *b = tmp1 + (u32)4U;
-	u64 *d = tmp1 + (u32)8U;
-	u64 *c = tmp1 + (u32)12U;
-	u64 *ab = tmp1;
-	u64 *dc = tmp1 + (u32)8U;
-	fadd(a, x2, z2);
-	fsub(b, x2, z2);
-	fsqr2(dc, ab, tmp2);
-	a[0U] = c[0U];
-	a[1U] = c[1U];
-	a[2U] = c[2U];
-	a[3U] = c[3U];
-	fsub(c, d, c);
-	fmul_scalar(b, c, (u64)121665U);
-	fadd(b, b, d);
-	fmul2(nq, dc, ab, tmp2);
-}
-
-static void montgomery_ladder(u64 *out, const u8 *key, u64 *init1)
-{
-	u64 tmp2[16U] = { 0U };
-	u64 p01_tmp1_swap[33U] = { 0U };
-	u64 *p0 = p01_tmp1_swap;
-	u64 *p01 = p01_tmp1_swap;
-	u64 *p03 = p01;
-	u64 *p11 = p01 + (u32)8U;
-	u64 *x0;
-	u64 *z0;
-	u64 *p01_tmp1;
-	u64 *p01_tmp11;
-	u64 *nq10;
-	u64 *nq_p11;
-	u64 *swap1;
-	u64 sw0;
-	u64 *nq1;
-	u64 *tmp1;
-	memcpy(p11, init1, (u32)8U * sizeof(init1[0U]));
-	x0 = p03;
-	z0 = p03 + (u32)4U;
-	x0[0U] = (u64)1U;
-	x0[1U] = (u64)0U;
-	x0[2U] = (u64)0U;
-	x0[3U] = (u64)0U;
-	z0[0U] = (u64)0U;
-	z0[1U] = (u64)0U;
-	z0[2U] = (u64)0U;
-	z0[3U] = (u64)0U;
-	p01_tmp1 = p01_tmp1_swap;
-	p01_tmp11 = p01_tmp1_swap;
-	nq10 = p01_tmp1_swap;
-	nq_p11 = p01_tmp1_swap + (u32)8U;
-	swap1 = p01_tmp1_swap + (u32)32U;
-	cswap2((u64)1U, nq10, nq_p11);
-	point_add_and_double(init1, p01_tmp11, tmp2);
-	swap1[0U] = (u64)1U;
-	{
-		u32 i;
-		for (i = (u32)0U; i < (u32)251U; i = i + (u32)1U) {
-			u64 *p01_tmp12 = p01_tmp1_swap;
-			u64 *swap2 = p01_tmp1_swap + (u32)32U;
-			u64 *nq2 = p01_tmp12;
-			u64 *nq_p12 = p01_tmp12 + (u32)8U;
-			u64 bit = (u64)(key[((u32)253U - i) / (u32)8U] >> ((u32)253U - i) % (u32)8U & (u8)1U);
-			u64 sw = swap2[0U] ^ bit;
-			cswap2(sw, nq2, nq_p12);
-			point_add_and_double(init1, p01_tmp12, tmp2);
-			swap2[0U] = bit;
-		}
-	}
-	sw0 = swap1[0U];
-	cswap2(sw0, nq10, nq_p11);
-	nq1 = p01_tmp1;
-	tmp1 = p01_tmp1 + (u32)16U;
-	point_double(nq1, tmp1, tmp2);
-	point_double(nq1, tmp1, tmp2);
-	point_double(nq1, tmp1, tmp2);
-	memcpy(out, p0, (u32)8U * sizeof(p0[0U]));
-
-	memzero_explicit(tmp2, sizeof(tmp2));
-	memzero_explicit(p01_tmp1_swap, sizeof(p01_tmp1_swap));
-}
-
-static void fsquare_times(u64 *o, const u64 *inp, u64 *tmp, u32 n1)
-{
-	u32 i;
-	fsqr(o, inp, tmp);
-	for (i = (u32)0U; i < n1 - (u32)1U; i = i + (u32)1U)
-		fsqr(o, o, tmp);
-}
-
-static void finv(u64 *o, const u64 *i, u64 *tmp)
-{
-	u64 t1[16U] = { 0U };
-	u64 *a0 = t1;
-	u64 *b = t1 + (u32)4U;
-	u64 *c = t1 + (u32)8U;
-	u64 *t00 = t1 + (u32)12U;
-	u64 *tmp1 = tmp;
-	u64 *a;
-	u64 *t0;
-	fsquare_times(a0, i, tmp1, (u32)1U);
-	fsquare_times(t00, a0, tmp1, (u32)2U);
-	fmul(b, t00, i, tmp);
-	fmul(a0, b, a0, tmp);
-	fsquare_times(t00, a0, tmp1, (u32)1U);
-	fmul(b, t00, b, tmp);
-	fsquare_times(t00, b, tmp1, (u32)5U);
-	fmul(b, t00, b, tmp);
-	fsquare_times(t00, b, tmp1, (u32)10U);
-	fmul(c, t00, b, tmp);
-	fsquare_times(t00, c, tmp1, (u32)20U);
-	fmul(t00, t00, c, tmp);
-	fsquare_times(t00, t00, tmp1, (u32)10U);
-	fmul(b, t00, b, tmp);
-	fsquare_times(t00, b, tmp1, (u32)50U);
-	fmul(c, t00, b, tmp);
-	fsquare_times(t00, c, tmp1, (u32)100U);
-	fmul(t00, t00, c, tmp);
-	fsquare_times(t00, t00, tmp1, (u32)50U);
-	fmul(t00, t00, b, tmp);
-	fsquare_times(t00, t00, tmp1, (u32)5U);
-	a = t1;
-	t0 = t1 + (u32)12U;
-	fmul(o, t0, a, tmp);
-}
-
-static void store_felem(u64 *b, u64 *f)
-{
-	u64 f30 = f[3U];
-	u64 top_bit0 = f30 >> (u32)63U;
-	u64 f31;
-	u64 top_bit;
-	u64 f0;
-	u64 f1;
-	u64 f2;
-	u64 f3;
-	u64 m0;
-	u64 m1;
-	u64 m2;
-	u64 m3;
-	u64 mask;
-	u64 f0_;
-	u64 f1_;
-	u64 f2_;
-	u64 f3_;
-	u64 o0;
-	u64 o1;
-	u64 o2;
-	u64 o3;
-	f[3U] = f30 & (u64)0x7fffffffffffffffU;
-	add_scalar(f, f, (u64)19U * top_bit0);
-	f31 = f[3U];
-	top_bit = f31 >> (u32)63U;
-	f[3U] = f31 & (u64)0x7fffffffffffffffU;
-	add_scalar(f, f, (u64)19U * top_bit);
-	f0 = f[0U];
-	f1 = f[1U];
-	f2 = f[2U];
-	f3 = f[3U];
-	m0 = gte_mask(f0, (u64)0xffffffffffffffedU);
-	m1 = eq_mask(f1, (u64)0xffffffffffffffffU);
-	m2 = eq_mask(f2, (u64)0xffffffffffffffffU);
-	m3 = eq_mask(f3, (u64)0x7fffffffffffffffU);
-	mask = ((m0 & m1) & m2) & m3;
-	f0_ = f0 - (mask & (u64)0xffffffffffffffedU);
-	f1_ = f1 - (mask & (u64)0xffffffffffffffffU);
-	f2_ = f2 - (mask & (u64)0xffffffffffffffffU);
-	f3_ = f3 - (mask & (u64)0x7fffffffffffffffU);
-	o0 = f0_;
-	o1 = f1_;
-	o2 = f2_;
-	o3 = f3_;
-	b[0U] = o0;
-	b[1U] = o1;
-	b[2U] = o2;
-	b[3U] = o3;
-}
-
-static void encode_point(u8 *o, const u64 *i)
-{
-	const u64 *x = i;
-	const u64 *z = i + (u32)4U;
-	u64 tmp[4U] = { 0U };
-	u64 tmp_w[16U] = { 0U };
-	finv(tmp, z, tmp_w);
-	fmul(tmp, tmp, x, tmp_w);
-	store_felem((u64 *)o, tmp);
-}
-
-static void curve25519_ever64(u8 *out, const u8 *priv, const u8 *pub)
-{
-	u64 init1[8U] = { 0U };
-	u64 tmp[4U] = { 0U };
-	u64 tmp3;
-	u64 *x;
-	u64 *z;
-	{
-		u32 i;
-		for (i = (u32)0U; i < (u32)4U; i = i + (u32)1U) {
-			u64 *os = tmp;
-			const u8 *bj = pub + i * (u32)8U;
-			u64 u = *(u64 *)bj;
-			u64 r = u;
-			u64 x0 = r;
-			os[i] = x0;
-		}
-	}
-	tmp3 = tmp[3U];
-	tmp[3U] = tmp3 & (u64)0x7fffffffffffffffU;
-	x = init1;
-	z = init1 + (u32)4U;
-	z[0U] = (u64)1U;
-	z[1U] = (u64)0U;
-	z[2U] = (u64)0U;
-	z[3U] = (u64)0U;
-	x[0U] = tmp[0U];
-	x[1U] = tmp[1U];
-	x[2U] = tmp[2U];
-	x[3U] = tmp[3U];
-	montgomery_ladder(init1, priv, init1);
-	encode_point(out, init1);
-}
-
-/* The below constants were generated using this sage script:
- *
- * #!/usr/bin/env sage
- * import sys
- * from sage.all import *
- * def limbs(n):
- * 	n = int(n)
- * 	l = ((n >> 0) % 2^64, (n >> 64) % 2^64, (n >> 128) % 2^64, (n >> 192) % 2^64)
- * 	return "0x%016xULL, 0x%016xULL, 0x%016xULL, 0x%016xULL" % l
- * ec = EllipticCurve(GF(2^255 - 19), [0, 486662, 0, 1, 0])
- * p_minus_s = (ec.lift_x(9) - ec.lift_x(1))[0]
- * print("static const u64 p_minus_s[] = { %s };\n" % limbs(p_minus_s))
- * print("static const u64 table_ladder[] = {")
- * p = ec.lift_x(9)
- * for i in range(252):
- * 	l = (p[0] + p[2]) / (p[0] - p[2])
- * 	print(("\t%s" + ("," if i != 251 else "")) % limbs(l))
- * 	p = p * 2
- * print("};")
- *
- */
-
-static const u64 p_minus_s[] = { 0x816b1e0137d48290ULL, 0x440f6a51eb4d1207ULL, 0x52385f46dca2b71dULL, 0x215132111d8354cbULL };
-
-static const u64 table_ladder[] = {
-	0xfffffffffffffff3ULL, 0xffffffffffffffffULL, 0xffffffffffffffffULL, 0x5fffffffffffffffULL,
-	0x6b8220f416aafe96ULL, 0x82ebeb2b4f566a34ULL, 0xd5a9a5b075a5950fULL, 0x5142b2cf4b2488f4ULL,
-	0x6aaebc750069680cULL, 0x89cf7820a0f99c41ULL, 0x2a58d9183b56d0f4ULL, 0x4b5aca80e36011a4ULL,
-	0x329132348c29745dULL, 0xf4a2e616e1642fd7ULL, 0x1e45bb03ff67bc34ULL, 0x306912d0f42a9b4aULL,
-	0xff886507e6af7154ULL, 0x04f50e13dfeec82fULL, 0xaa512fe82abab5ceULL, 0x174e251a68d5f222ULL,
-	0xcf96700d82028898ULL, 0x1743e3370a2c02c5ULL, 0x379eec98b4e86eaaULL, 0x0c59888a51e0482eULL,
-	0xfbcbf1d699b5d189ULL, 0xacaef0d58e9fdc84ULL, 0xc1c20d06231f7614ULL, 0x2938218da274f972ULL,
-	0xf6af49beff1d7f18ULL, 0xcc541c22387ac9c2ULL, 0x96fcc9ef4015c56bULL, 0x69c1627c690913a9ULL,
-	0x7a86fd2f4733db0eULL, 0xfdb8c4f29e087de9ULL, 0x095e4b1a8ea2a229ULL, 0x1ad7a7c829b37a79ULL,
-	0x342d89cad17ea0c0ULL, 0x67bedda6cced2051ULL, 0x19ca31bf2bb42f74ULL, 0x3df7b4c84980acbbULL,
-	0xa8c6444dc80ad883ULL, 0xb91e440366e3ab85ULL, 0xc215cda00164f6d8ULL, 0x3d867c6ef247e668ULL,
-	0xc7dd582bcc3e658cULL, 0xfd2c4748ee0e5528ULL, 0xa0fd9b95cc9f4f71ULL, 0x7529d871b0675ddfULL,
-	0xb8f568b42d3cbd78ULL, 0x1233011b91f3da82ULL, 0x2dce6ccd4a7c3b62ULL, 0x75e7fc8e9e498603ULL,
-	0x2f4f13f1fcd0b6ecULL, 0xf1a8ca1f29ff7a45ULL, 0xc249c1a72981e29bULL, 0x6ebe0dbb8c83b56aULL,
-	0x7114fa8d170bb222ULL, 0x65a2dcd5bf93935fULL, 0xbdc41f68b59c979aULL, 0x2f0eef79a2ce9289ULL,
-	0x42ecbf0c083c37ceULL, 0x2930bc09ec496322ULL, 0xf294b0c19cfeac0dULL, 0x3780aa4bedfabb80ULL,
-	0x56c17d3e7cead929ULL, 0xe7cb4beb2e5722c5ULL, 0x0ce931732dbfe15aULL, 0x41b883c7621052f8ULL,
-	0xdbf75ca0c3d25350ULL, 0x2936be086eb1e351ULL, 0xc936e03cb4a9b212ULL, 0x1d45bf82322225aaULL,
-	0xe81ab1036a024cc5ULL, 0xe212201c304c9a72ULL, 0xc5d73fba6832b1fcULL, 0x20ffdb5a4d839581ULL,
-	0xa283d367be5d0fadULL, 0x6c2b25ca8b164475ULL, 0x9d4935467caaf22eULL, 0x5166408eee85ff49ULL,
-	0x3c67baa2fab4e361ULL, 0xb3e433c67ef35cefULL, 0x5259729241159b1cULL, 0x6a621892d5b0ab33ULL,
-	0x20b74a387555cdcbULL, 0x532aa10e1208923fULL, 0xeaa17b7762281dd1ULL, 0x61ab3443f05c44bfULL,
-	0x257a6c422324def8ULL, 0x131c6c1017e3cf7fULL, 0x23758739f630a257ULL, 0x295a407a01a78580ULL,
-	0xf8c443246d5da8d9ULL, 0x19d775450c52fa5dULL, 0x2afcfc92731bf83dULL, 0x7d10c8e81b2b4700ULL,
-	0xc8e0271f70baa20bULL, 0x993748867ca63957ULL, 0x5412efb3cb7ed4bbULL, 0x3196d36173e62975ULL,
-	0xde5bcad141c7dffcULL, 0x47cc8cd2b395c848ULL, 0xa34cd942e11af3cbULL, 0x0256dbf2d04ecec2ULL,
-	0x875ab7e94b0e667fULL, 0xcad4dd83c0850d10ULL, 0x47f12e8f4e72c79fULL, 0x5f1a87bb8c85b19bULL,
-	0x7ae9d0b6437f51b8ULL, 0x12c7ce5518879065ULL, 0x2ade09fe5cf77aeeULL, 0x23a05a2f7d2c5627ULL,
-	0x5908e128f17c169aULL, 0xf77498dd8ad0852dULL, 0x74b4c4ceab102f64ULL, 0x183abadd10139845ULL,
-	0xb165ba8daa92aaacULL, 0xd5c5ef9599386705ULL, 0xbe2f8f0cf8fc40d1ULL, 0x2701e635ee204514ULL,
-	0x629fa80020156514ULL, 0xf223868764a8c1ceULL, 0x5b894fff0b3f060eULL, 0x60d9944cf708a3faULL,
-	0xaeea001a1c7a201fULL, 0xebf16a633ee2ce63ULL, 0x6f7709594c7a07e1ULL, 0x79b958150d0208cbULL,
-	0x24b55e5301d410e7ULL, 0xe3a34edff3fdc84dULL, 0xd88768e4904032d8ULL, 0x131384427b3aaeecULL,
-	0x8405e51286234f14ULL, 0x14dc4739adb4c529ULL, 0xb8a2b5b250634ffdULL, 0x2fe2a94ad8a7ff93ULL,
-	0xec5c57efe843faddULL, 0x2843ce40f0bb9918ULL, 0xa4b561d6cf3d6305ULL, 0x743629bde8fb777eULL,
-	0x343edd46bbaf738fULL, 0xed981828b101a651ULL, 0xa401760b882c797aULL, 0x1fc223e28dc88730ULL,
-	0x48604e91fc0fba0eULL, 0xb637f78f052c6fa4ULL, 0x91ccac3d09e9239cULL, 0x23f7eed4437a687cULL,
-	0x5173b1118d9bd800ULL, 0x29d641b63189d4a7ULL, 0xfdbf177988bbc586ULL, 0x2959894fcad81df5ULL,
-	0xaebc8ef3b4bbc899ULL, 0x4148995ab26992b9ULL, 0x24e20b0134f92cfbULL, 0x40d158894a05dee8ULL,
-	0x46b00b1185af76f6ULL, 0x26bac77873187a79ULL, 0x3dc0bf95ab8fff5fULL, 0x2a608bd8945524d7ULL,
-	0x26449588bd446302ULL, 0x7c4bc21c0388439cULL, 0x8e98a4f383bd11b2ULL, 0x26218d7bc9d876b9ULL,
-	0xe3081542997c178aULL, 0x3c2d29a86fb6606fULL, 0x5c217736fa279374ULL, 0x7dde05734afeb1faULL,
-	0x3bf10e3906d42babULL, 0xe4f7803e1980649cULL, 0xe6053bf89595bf7aULL, 0x394faf38da245530ULL,
-	0x7a8efb58896928f4ULL, 0xfbc778e9cc6a113cULL, 0x72670ce330af596fULL, 0x48f222a81d3d6cf7ULL,
-	0xf01fce410d72caa7ULL, 0x5a20ecc7213b5595ULL, 0x7bc21165c1fa1483ULL, 0x07f89ae31da8a741ULL,
-	0x05d2c2b4c6830ff9ULL, 0xd43e330fc6316293ULL, 0xa5a5590a96d3a904ULL, 0x705edb91a65333b6ULL,
-	0x048ee15e0bb9a5f7ULL, 0x3240cfca9e0aaf5dULL, 0x8f4b71ceedc4a40bULL, 0x621c0da3de544a6dULL,
-	0x92872836a08c4091ULL, 0xce8375b010c91445ULL, 0x8a72eb524f276394ULL, 0x2667fcfa7ec83635ULL,
-	0x7f4c173345e8752aULL, 0x061b47feee7079a5ULL, 0x25dd9afa9f86ff34ULL, 0x3780cef5425dc89cULL,
-	0x1a46035a513bb4e9ULL, 0x3e1ef379ac575adaULL, 0xc78c5f1c5fa24b50ULL, 0x321a967634fd9f22ULL,
-	0x946707b8826e27faULL, 0x3dca84d64c506fd0ULL, 0xc189218075e91436ULL, 0x6d9284169b3b8484ULL,
-	0x3a67e840383f2ddfULL, 0x33eec9a30c4f9b75ULL, 0x3ec7c86fa783ef47ULL, 0x26ec449fbac9fbc4ULL,
-	0x5c0f38cba09b9e7dULL, 0x81168cc762a3478cULL, 0x3e23b0d306fc121cULL, 0x5a238aa0a5efdcddULL,
-	0x1ba26121c4ea43ffULL, 0x36f8c77f7c8832b5ULL, 0x88fbea0b0adcf99aULL, 0x5ca9938ec25bebf9ULL,
-	0xd5436a5e51fccda0ULL, 0x1dbc4797c2cd893bULL, 0x19346a65d3224a08ULL, 0x0f5034e49b9af466ULL,
-	0xf23c3967a1e0b96eULL, 0xe58b08fa867a4d88ULL, 0xfb2fabc6a7341679ULL, 0x2a75381eb6026946ULL,
-	0xc80a3be4c19420acULL, 0x66b1f6c681f2b6dcULL, 0x7cf7036761e93388ULL, 0x25abbbd8a660a4c4ULL,
-	0x91ea12ba14fd5198ULL, 0x684950fc4a3cffa9ULL, 0xf826842130f5ad28ULL, 0x3ea988f75301a441ULL,
-	0xc978109a695f8c6fULL, 0x1746eb4a0530c3f3ULL, 0x444d6d77b4459995ULL, 0x75952b8c054e5cc7ULL,
-	0xa3703f7915f4d6aaULL, 0x66c346202f2647d8ULL, 0xd01469df811d644bULL, 0x77fea47d81a5d71fULL,
-	0xc5e9529ef57ca381ULL, 0x6eeeb4b9ce2f881aULL, 0xb6e91a28e8009bd6ULL, 0x4b80be3e9afc3fecULL,
-	0x7e3773c526aed2c5ULL, 0x1b4afcb453c9a49dULL, 0xa920bdd7baffb24dULL, 0x7c54699f122d400eULL,
-	0xef46c8e14fa94bc8ULL, 0xe0b074ce2952ed5eULL, 0xbea450e1dbd885d5ULL, 0x61b68649320f712cULL,
-	0x8a485f7309ccbdd1ULL, 0xbd06320d7d4d1a2dULL, 0x25232973322dbef4ULL, 0x445dc4758c17f770ULL,
-	0xdb0434177cc8933cULL, 0xed6fe82175ea059fULL, 0x1efebefdc053db34ULL, 0x4adbe867c65daf99ULL,
-	0x3acd71a2a90609dfULL, 0xe5e991856dd04050ULL, 0x1ec69b688157c23cULL, 0x697427f6885cfe4dULL,
-	0xd7be7b9b65e1a851ULL, 0xa03d28d522c536ddULL, 0x28399d658fd2b645ULL, 0x49e5b7e17c2641e1ULL,
-	0x6f8c3a98700457a4ULL, 0x5078f0a25ebb6778ULL, 0xd13c3ccbc382960fULL, 0x2e003258a7df84b1ULL,
-	0x8ad1f39be6296a1cULL, 0xc1eeaa652a5fbfb2ULL, 0x33ee0673fd26f3cbULL, 0x59256173a69d2cccULL,
-	0x41ea07aa4e18fc41ULL, 0xd9fc19527c87a51eULL, 0xbdaacb805831ca6fULL, 0x445b652dc916694fULL,
-	0xce92a3a7f2172315ULL, 0x1edc282de11b9964ULL, 0xa1823aafe04c314aULL, 0x790a2d94437cf586ULL,
-	0x71c447fb93f6e009ULL, 0x8922a56722845276ULL, 0xbf70903b204f5169ULL, 0x2f7a89891ba319feULL,
-	0x02a08eb577e2140cULL, 0xed9a4ed4427bdcf4ULL, 0x5253ec44e4323cd1ULL, 0x3e88363c14e9355bULL,
-	0xaa66c14277110b8cULL, 0x1ae0391610a23390ULL, 0x2030bd12c93fc2a2ULL, 0x3ee141579555c7abULL,
-	0x9214de3a6d6e7d41ULL, 0x3ccdd88607f17efeULL, 0x674f1288f8e11217ULL, 0x5682250f329f93d0ULL,
-	0x6cf00b136d2e396eULL, 0x6e4cf86f1014debfULL, 0x5930b1b5bfcc4e83ULL, 0x047069b48aba16b6ULL,
-	0x0d4ce4ab69b20793ULL, 0xb24db91a97d0fb9eULL, 0xcdfa50f54e00d01dULL, 0x221b1085368bddb5ULL,
-	0xe7e59468b1e3d8d2ULL, 0x53c56563bd122f93ULL, 0xeee8a903e0663f09ULL, 0x61efa662cbbe3d42ULL,
-	0x2cf8ddddde6eab2aULL, 0x9bf80ad51435f231ULL, 0x5deadacec9f04973ULL, 0x29275b5d41d29b27ULL,
-	0xcfde0f0895ebf14fULL, 0xb9aab96b054905a7ULL, 0xcae80dd9a1c420fdULL, 0x0a63bf2f1673bbc7ULL,
-	0x092f6e11958fbc8cULL, 0x672a81e804822fadULL, 0xcac8351560d52517ULL, 0x6f3f7722c8f192f8ULL,
-	0xf8ba90ccc2e894b7ULL, 0x2c7557a438ff9f0dULL, 0x894d1d855ae52359ULL, 0x68e122157b743d69ULL,
-	0xd87e5570cfb919f3ULL, 0x3f2cdecd95798db9ULL, 0x2121154710c0a2ceULL, 0x3c66a115246dc5b2ULL,
-	0xcbedc562294ecb72ULL, 0xba7143c36a280b16ULL, 0x9610c2efd4078b67ULL, 0x6144735d946a4b1eULL,
-	0x536f111ed75b3350ULL, 0x0211db8c2041d81bULL, 0xf93cb1000e10413cULL, 0x149dfd3c039e8876ULL,
-	0xd479dde46b63155bULL, 0xb66e15e93c837976ULL, 0xdafde43b1f13e038ULL, 0x5fafda1a2e4b0b35ULL,
-	0x3600bbdf17197581ULL, 0x3972050bbe3cd2c2ULL, 0x5938906dbdd5be86ULL, 0x34fce5e43f9b860fULL,
-	0x75a8a4cd42d14d02ULL, 0x828dabc53441df65ULL, 0x33dcabedd2e131d3ULL, 0x3ebad76fb814d25fULL,
-	0xd4906f566f70e10fULL, 0x5d12f7aa51690f5aULL, 0x45adb16e76cefcf2ULL, 0x01f768aead232999ULL,
-	0x2b6cc77b6248febdULL, 0x3cd30628ec3aaffdULL, 0xce1c0b80d4ef486aULL, 0x4c3bff2ea6f66c23ULL,
-	0x3f2ec4094aeaeb5fULL, 0x61b19b286e372ca7ULL, 0x5eefa966de2a701dULL, 0x23b20565de55e3efULL,
-	0xe301ca5279d58557ULL, 0x07b2d4ce27c2874fULL, 0xa532cd8a9dcf1d67ULL, 0x2a52fee23f2bff56ULL,
-	0x8624efb37cd8663dULL, 0xbbc7ac20ffbd7594ULL, 0x57b85e9c82d37445ULL, 0x7b3052cb86a6ec66ULL,
-	0x3482f0ad2525e91eULL, 0x2cb68043d28edca0ULL, 0xaf4f6d052e1b003aULL, 0x185f8c2529781b0aULL,
-	0xaa41de5bd80ce0d6ULL, 0x9407b2416853e9d6ULL, 0x563ec36e357f4c3aULL, 0x4cc4b8dd0e297bceULL,
-	0xa2fc1a52ffb8730eULL, 0x1811f16e67058e37ULL, 0x10f9a366cddf4ee1ULL, 0x72f4a0c4a0b9f099ULL,
-	0x8c16c06f663f4ea7ULL, 0x693b3af74e970fbaULL, 0x2102e7f1d69ec345ULL, 0x0ba53cbc968a8089ULL,
-	0xca3d9dc7fea15537ULL, 0x4c6824bb51536493ULL, 0xb9886314844006b1ULL, 0x40d2a72ab454cc60ULL,
-	0x5936a1b712570975ULL, 0x91b9d648debda657ULL, 0x3344094bb64330eaULL, 0x006ba10d12ee51d0ULL,
-	0x19228468f5de5d58ULL, 0x0eb12f4c38cc05b0ULL, 0xa1039f9dd5601990ULL, 0x4502d4ce4fff0e0bULL,
-	0xeb2054106837c189ULL, 0xd0f6544c6dd3b93cULL, 0x40727064c416d74fULL, 0x6e15c6114b502ef0ULL,
-	0x4df2a398cfb1a76bULL, 0x11256c7419f2f6b1ULL, 0x4a497962066e6043ULL, 0x705b3aab41355b44ULL,
-	0x365ef536d797b1d8ULL, 0x00076bd622ddf0dbULL, 0x3bbf33b0e0575a88ULL, 0x3777aa05c8e4ca4dULL,
-	0x392745c85578db5fULL, 0x6fda4149dbae5ae2ULL, 0xb1f0b00b8adc9867ULL, 0x09963437d36f1da3ULL,
-	0x7e824e90a5dc3853ULL, 0xccb5f6641f135cbdULL, 0x6736d86c87ce8fccULL, 0x625f3ce26604249fULL,
-	0xaf8ac8059502f63fULL, 0x0c05e70a2e351469ULL, 0x35292e9c764b6305ULL, 0x1a394360c7e23ac3ULL,
-	0xd5c6d53251183264ULL, 0x62065abd43c2b74fULL, 0xb5fbf5d03b973f9bULL, 0x13a3da3661206e5eULL,
-	0xc6bd5837725d94e5ULL, 0x18e30912205016c5ULL, 0x2088ce1570033c68ULL, 0x7fba1f495c837987ULL,
-	0x5a8c7423f2f9079dULL, 0x1735157b34023fc5ULL, 0xe4f9b49ad2fab351ULL, 0x6691ff72c878e33cULL,
-	0x122c2adedc5eff3eULL, 0xf8dd4bf1d8956cf4ULL, 0xeb86205d9e9e5bdaULL, 0x049b92b9d975c743ULL,
-	0xa5379730b0f6c05aULL, 0x72a0ffacc6f3a553ULL, 0xb0032c34b20dcd6dULL, 0x470e9dbc88d5164aULL,
-	0xb19cf10ca237c047ULL, 0xb65466711f6c81a2ULL, 0xb3321bd16dd80b43ULL, 0x48c14f600c5fbe8eULL,
-	0x66451c264aa6c803ULL, 0xb66e3904a4fa7da6ULL, 0xd45f19b0b3128395ULL, 0x31602627c3c9bc10ULL,
-	0x3120dc4832e4e10dULL, 0xeb20c46756c717f7ULL, 0x00f52e3f67280294ULL, 0x566d4fc14730c509ULL,
-	0x7e3a5d40fd837206ULL, 0xc1e926dc7159547aULL, 0x216730fba68d6095ULL, 0x22e8c3843f69cea7ULL,
-	0x33d074e8930e4b2bULL, 0xb6e4350e84d15816ULL, 0x5534c26ad6ba2365ULL, 0x7773c12f89f1f3f3ULL,
-	0x8cba404da57962aaULL, 0x5b9897a81999ce56ULL, 0x508e862f121692fcULL, 0x3a81907fa093c291ULL,
-	0x0dded0ff4725a510ULL, 0x10d8cc10673fc503ULL, 0x5b9d151c9f1f4e89ULL, 0x32a5c1d5cb09a44cULL,
-	0x1e0aa442b90541fbULL, 0x5f85eb7cc1b485dbULL, 0xbee595ce8a9df2e5ULL, 0x25e496c722422236ULL,
-	0x5edf3c46cd0fe5b9ULL, 0x34e75a7ed2a43388ULL, 0xe488de11d761e352ULL, 0x0e878a01a085545cULL,
-	0xba493c77e021bb04ULL, 0x2b4d1843c7df899aULL, 0x9ea37a487ae80d67ULL, 0x67a9958011e41794ULL,
-	0x4b58051a6697b065ULL, 0x47e33f7d8d6ba6d4ULL, 0xbb4da8d483ca46c1ULL, 0x68becaa181c2db0dULL,
-	0x8d8980e90b989aa5ULL, 0xf95eb14a2c93c99bULL, 0x51c6c7c4796e73a2ULL, 0x6e228363b5efb569ULL,
-	0xc6bbc0b02dd624c8ULL, 0x777eb47dec8170eeULL, 0x3cde15a004cfafa9ULL, 0x1dc6bc087160bf9bULL,
-	0x2e07e043eec34002ULL, 0x18e9fc677a68dc7fULL, 0xd8da03188bd15b9aULL, 0x48fbc3bb00568253ULL,
-	0x57547d4cfb654ce1ULL, 0xd3565b82a058e2adULL, 0xf63eaf0bbf154478ULL, 0x47531ef114dfbb18ULL,
-	0xe1ec630a4278c587ULL, 0x5507d546ca8e83f3ULL, 0x85e135c63adc0c2bULL, 0x0aa7efa85682844eULL,
-	0x72691ba8b3e1f615ULL, 0x32b4e9701fbe3ffaULL, 0x97b6d92e39bb7868ULL, 0x2cfe53dea02e39e8ULL,
-	0x687392cd85cd52b0ULL, 0x27ff66c910e29831ULL, 0x97134556a9832d06ULL, 0x269bb0360a84f8a0ULL,
-	0x706e55457643f85cULL, 0x3734a48c9b597d1bULL, 0x7aee91e8c6efa472ULL, 0x5cd6abc198a9d9e0ULL,
-	0x0e04de06cb3ce41aULL, 0xd8c6eb893402e138ULL, 0x904659bb686e3772ULL, 0x7215c371746ba8c8ULL,
-	0xfd12a97eeae4a2d9ULL, 0x9514b7516394f2c5ULL, 0x266fd5809208f294ULL, 0x5c847085619a26b9ULL,
-	0x52985410fed694eaULL, 0x3c905b934a2ed254ULL, 0x10bb47692d3be467ULL, 0x063b3d2d69e5e9e1ULL,
-	0x472726eedda57debULL, 0xefb6c4ae10f41891ULL, 0x2b1641917b307614ULL, 0x117c554fc4f45b7cULL,
-	0xc07cf3118f9d8812ULL, 0x01dbd82050017939ULL, 0xd7e803f4171b2827ULL, 0x1015e87487d225eaULL,
-	0xc58de3fed23acc4dULL, 0x50db91c294a7be2dULL, 0x0b94d43d1c9cf457ULL, 0x6b1640fa6e37524aULL,
-	0x692f346c5fda0d09ULL, 0x200b1c59fa4d3151ULL, 0xb8c46f760777a296ULL, 0x4b38395f3ffdfbcfULL,
-	0x18d25e00be54d671ULL, 0x60d50582bec8aba6ULL, 0x87ad8f263b78b982ULL, 0x50fdf64e9cda0432ULL,
-	0x90f567aac578dcf0ULL, 0xef1e9b0ef2a3133bULL, 0x0eebba9242d9de71ULL, 0x15473c9bf03101c7ULL,
-	0x7c77e8ae56b78095ULL, 0xb678e7666e6f078eULL, 0x2da0b9615348ba1fULL, 0x7cf931c1ff733f0bULL,
-	0x26b357f50a0a366cULL, 0xe9708cf42b87d732ULL, 0xc13aeea5f91cb2c0ULL, 0x35d90c991143bb4cULL,
-	0x47c1c404a9a0d9dcULL, 0x659e58451972d251ULL, 0x3875a8c473b38c31ULL, 0x1fbd9ed379561f24ULL,
-	0x11fabc6fd41ec28dULL, 0x7ef8dfe3cd2a2dcaULL, 0x72e73b5d8c404595ULL, 0x6135fa4954b72f27ULL,
-	0xccfc32a2de24b69cULL, 0x3f55698c1f095d88ULL, 0xbe3350ed5ac3f929ULL, 0x5e9bf806ca477eebULL,
-	0xe9ce8fb63c309f68ULL, 0x5376f63565e1f9f4ULL, 0xd1afcfb35a6393f1ULL, 0x6632a1ede5623506ULL,
-	0x0b7d6c390c2ded4cULL, 0x56cb3281df04cb1fULL, 0x66305a1249ecc3c7ULL, 0x5d588b60a38ca72aULL,
-	0xa6ecbf78e8e5f42dULL, 0x86eeb44b3c8a3eecULL, 0xec219c48fbd21604ULL, 0x1aaf1af517c36731ULL,
-	0xc306a2836769bde7ULL, 0x208280622b1e2adbULL, 0x8027f51ffbff94a6ULL, 0x76cfa1ce1124f26bULL,
-	0x18eb00562422abb6ULL, 0xf377c4d58f8c29c3ULL, 0x4dbbc207f531561aULL, 0x0253b7f082128a27ULL,
-	0x3d1f091cb62c17e0ULL, 0x4860e1abd64628a9ULL, 0x52d17436309d4253ULL, 0x356f97e13efae576ULL,
-	0xd351e11aa150535bULL, 0x3e6b45bb1dd878ccULL, 0x0c776128bed92c98ULL, 0x1d34ae93032885b8ULL,
-	0x4ba0488ca85ba4c3ULL, 0x985348c33c9ce6ceULL, 0x66124c6f97bda770ULL, 0x0f81a0290654124aULL,
-	0x9ed09ca6569b86fdULL, 0x811009fd18af9a2dULL, 0xff08d03f93d8c20aULL, 0x52a148199faef26bULL,
-	0x3e03f9dc2d8d1b73ULL, 0x4205801873961a70ULL, 0xc0d987f041a35970ULL, 0x07aa1f15a1c0d549ULL,
-	0xdfd46ce08cd27224ULL, 0x6d0a024f934e4239ULL, 0x808a7a6399897b59ULL, 0x0a4556e9e13d95a2ULL,
-	0xd21a991fe9c13045ULL, 0x9b0e8548fe7751b8ULL, 0x5da643cb4bf30035ULL, 0x77db28d63940f721ULL,
-	0xfc5eeb614adc9011ULL, 0x5229419ae8c411ebULL, 0x9ec3e7787d1dcf74ULL, 0x340d053e216e4cb5ULL,
-	0xcac7af39b48df2b4ULL, 0xc0faec2871a10a94ULL, 0x140a69245ca575edULL, 0x0cf1c37134273a4cULL,
-	0xc8ee306ac224b8a5ULL, 0x57eaee7ccb4930b0ULL, 0xa1e806bdaacbe74fULL, 0x7d9a62742eeb657dULL,
-	0x9eb6b6ef546c4830ULL, 0x885cca1fddb36e2eULL, 0xe6b9f383ef0d7105ULL, 0x58654fef9d2e0412ULL,
-	0xa905c4ffbe0e8e26ULL, 0x942de5df9b31816eULL, 0x497d723f802e88e1ULL, 0x30684dea602f408dULL,
-	0x21e5a278a3e6cb34ULL, 0xaefb6e6f5b151dc4ULL, 0xb30b8e049d77ca15ULL, 0x28c3c9cf53b98981ULL,
-	0x287fb721556cdd2aULL, 0x0d317ca897022274ULL, 0x7468c7423a543258ULL, 0x4a7f11464eb5642fULL,
-	0xa237a4774d193aa6ULL, 0xd865986ea92129a1ULL, 0x24c515ecf87c1a88ULL, 0x604003575f39f5ebULL,
-	0x47b9f189570a9b27ULL, 0x2b98cede465e4b78ULL, 0x026df551dbb85c20ULL, 0x74fcd91047e21901ULL,
-	0x13e2a90a23c1bfa3ULL, 0x0cb0074e478519f6ULL, 0x5ff1cbbe3af6cf44ULL, 0x67fe5438be812dbeULL,
-	0xd13cf64fa40f05b0ULL, 0x054dfb2f32283787ULL, 0x4173915b7f0d2aeaULL, 0x482f144f1f610d4eULL,
-	0xf6210201b47f8234ULL, 0x5d0ae1929e70b990ULL, 0xdcd7f455b049567cULL, 0x7e93d0f1f0916f01ULL,
-	0xdd79cbf18a7db4faULL, 0xbe8391bf6f74c62fULL, 0x027145d14b8291bdULL, 0x585a73ea2cbf1705ULL,
-	0x485ca03e928a0db2ULL, 0x10fc01a5742857e7ULL, 0x2f482edbd6d551a7ULL, 0x0f0433b5048fdb8aULL,
-	0x60da2e8dd7dc6247ULL, 0x88b4c9d38cd4819aULL, 0x13033ac001f66697ULL, 0x273b24fe3b367d75ULL,
-	0xc6e8f66a31b3b9d4ULL, 0x281514a494df49d5ULL, 0xd1726fdfc8b23da7ULL, 0x4b3ae7d103dee548ULL,
-	0xc6256e19ce4b9d7eULL, 0xff5c5cf186e3c61cULL, 0xacc63ca34b8ec145ULL, 0x74621888fee66574ULL,
-	0x956f409645290a1eULL, 0xef0bf8e3263a962eULL, 0xed6a50eb5ec2647bULL, 0x0694283a9dca7502ULL,
-	0x769b963643a2dcd1ULL, 0x42b7c8ea09fc5353ULL, 0x4f002aee13397eabULL, 0x63005e2c19b7d63aULL,
-	0xca6736da63023beaULL, 0x966c7f6db12a99b7ULL, 0xace09390c537c5e1ULL, 0x0b696063a1aa89eeULL,
-	0xebb03e97288c56e5ULL, 0x432a9f9f938c8be8ULL, 0xa6a5a93d5b717f71ULL, 0x1a5fb4c3e18f9d97ULL,
-	0x1c94e7ad1c60cdceULL, 0xee202a43fc02c4a0ULL, 0x8dafe4d867c46a20ULL, 0x0a10263c8ac27b58ULL,
-	0xd0dea9dfe4432a4aULL, 0x856af87bbe9277c5ULL, 0xce8472acc212c71aULL, 0x6f151b6d9bbb1e91ULL,
-	0x26776c527ceed56aULL, 0x7d211cb7fbf8faecULL, 0x37ae66a6fd4609ccULL, 0x1f81b702d2770c42ULL,
-	0x2fb0b057eac58392ULL, 0xe1dd89fe29744e9dULL, 0xc964f8eb17beb4f8ULL, 0x29571073c9a2d41eULL,
-	0xa948a18981c0e254ULL, 0x2df6369b65b22830ULL, 0xa33eb2d75fcfd3c6ULL, 0x078cd6ec4199a01fULL,
-	0x4a584a41ad900d2fULL, 0x32142b78e2c74c52ULL, 0x68c4e8338431c978ULL, 0x7f69ea9008689fc2ULL,
-	0x52f2c81e46a38265ULL, 0xfd78072d04a832fdULL, 0x8cd7d5fa25359e94ULL, 0x4de71b7454cc29d2ULL,
-	0x42eb60ad1eda6ac9ULL, 0x0aad37dfdbc09c3aULL, 0x81004b71e33cc191ULL, 0x44e6be345122803cULL,
-	0x03fe8388ba1920dbULL, 0xf5d57c32150db008ULL, 0x49c8c4281af60c29ULL, 0x21edb518de701aeeULL,
-	0x7fb63e418f06dc99ULL, 0xa4460d99c166d7b8ULL, 0x24dd5248ce520a83ULL, 0x5ec3ad712b928358ULL,
-	0x15022a5fbd17930fULL, 0xa4f64a77d82570e3ULL, 0x12bc8d6915783712ULL, 0x498194c0fc620abbULL,
-	0x38a2d9d255686c82ULL, 0x785c6bd9193e21f0ULL, 0xe4d5c81ab24a5484ULL, 0x56307860b2e20989ULL,
-	0x429d55f78b4d74c4ULL, 0x22f1834643350131ULL, 0x1e60c24598c71fffULL, 0x59f2f014979983efULL,
-	0x46a47d56eb494a44ULL, 0x3e22a854d636a18eULL, 0xb346e15274491c3bULL, 0x2ceafd4e5390cde7ULL,
-	0xba8a8538be0d6675ULL, 0x4b9074bb50818e23ULL, 0xcbdab89085d304c3ULL, 0x61a24fe0e56192c4ULL,
-	0xcb7615e6db525bcbULL, 0xdd7d8c35a567e4caULL, 0xe6b4153acafcdd69ULL, 0x2d668e097f3c9766ULL,
-	0xa57e7e265ce55ef0ULL, 0x5d9f4e527cd4b967ULL, 0xfbc83606492fd1e5ULL, 0x090d52beb7c3f7aeULL,
-	0x09b9515a1e7b4d7cULL, 0x1f266a2599da44c0ULL, 0xa1c49548e2c55504ULL, 0x7ef04287126f15ccULL,
-	0xfed1659dbd30ef15ULL, 0x8b4ab9eec4e0277bULL, 0x884d6236a5df3291ULL, 0x1fd96ea6bf5cf788ULL,
-	0x42a161981f190d9aULL, 0x61d849507e6052c1ULL, 0x9fe113bf285a2cd5ULL, 0x7c22d676dbad85d8ULL,
-	0x82e770ed2bfbd27dULL, 0x4c05b2ece996f5a5ULL, 0xcd40a9c2b0900150ULL, 0x5895319213d9bf64ULL,
-	0xe7cc5d703fea2e08ULL, 0xb50c491258e2188cULL, 0xcce30baa48205bf0ULL, 0x537c659ccfa32d62ULL,
-	0x37b6623a98cfc088ULL, 0xfe9bed1fa4d6aca4ULL, 0x04d29b8e56a8d1b0ULL, 0x725f71c40b519575ULL,
-	0x28c7f89cd0339ce6ULL, 0x8367b14469ddc18bULL, 0x883ada83a6a1652cULL, 0x585f1974034d6c17ULL,
-	0x89cfb266f1b19188ULL, 0xe63b4863e7c35217ULL, 0xd88c9da6b4c0526aULL, 0x3e035c9df0954635ULL,
-	0xdd9d5412fb45de9dULL, 0xdd684532e4cff40dULL, 0x4b5c999b151d671cULL, 0x2d8c2cc811e7f690ULL,
-	0x7f54be1d90055d40ULL, 0xa464c5df464aaf40ULL, 0x33979624f0e917beULL, 0x2c018dc527356b30ULL,
-	0xa5415024e330b3d4ULL, 0x73ff3d96691652d3ULL, 0x94ec42c4ef9b59f1ULL, 0x0747201618d08e5aULL,
-	0x4d6ca48aca411c53ULL, 0x66415f2fcfa66119ULL, 0x9c4dd40051e227ffULL, 0x59810bc09a02f7ebULL,
-	0x2a7eb171b3dc101dULL, 0x441c5ab99ffef68eULL, 0x32025c9b93b359eaULL, 0x5e8ce0a71e9d112fULL,
-	0xbfcccb92429503fdULL, 0xd271ba752f095d55ULL, 0x345ead5e972d091eULL, 0x18c8df11a83103baULL,
-	0x90cd949a9aed0f4cULL, 0xc5d1f4cb6660e37eULL, 0xb8cac52d56c52e0bULL, 0x6e42e400c5808e0dULL,
-	0xa3b46966eeaefd23ULL, 0x0c4f1f0be39ecdcaULL, 0x189dc8c9d683a51dULL, 0x51f27f054c09351bULL,
-	0x4c487ccd2a320682ULL, 0x587ea95bb3df1c96ULL, 0xc8ccf79e555cb8e8ULL, 0x547dc829a206d73dULL,
-	0xb822a6cd80c39b06ULL, 0xe96d54732000d4c6ULL, 0x28535b6f91463b4dULL, 0x228f4660e2486e1dULL,
-	0x98799538de8d3abfULL, 0x8cd8330045ebca6eULL, 0x79952a008221e738ULL, 0x4322e1a7535cd2bbULL,
-	0xb114c11819d1801cULL, 0x2016e4d84f3f5ec7ULL, 0xdd0e2df409260f4cULL, 0x5ec362c0ae5f7266ULL,
-	0xc0462b18b8b2b4eeULL, 0x7cc8d950274d1afbULL, 0xf25f7105436b02d2ULL, 0x43bbf8dcbff9ccd3ULL,
-	0xb6ad1767a039e9dfULL, 0xb0714da8f69d3583ULL, 0x5e55fa18b42931f5ULL, 0x4ed5558f33c60961ULL,
-	0x1fe37901c647a5ddULL, 0x593ddf1f8081d357ULL, 0x0249a4fd813fd7a6ULL, 0x69acca274e9caf61ULL,
-	0x047ba3ea330721c9ULL, 0x83423fc20e7e1ea0ULL, 0x1df4c0af01314a60ULL, 0x09a62dab89289527ULL,
-	0xa5b325a49cc6cb00ULL, 0xe94b5dc654b56cb6ULL, 0x3be28779adc994a0ULL, 0x4296e8f8ba3a4aadULL,
-	0x328689761e451eabULL, 0x2e4d598bff59594aULL, 0x49b96853d7a7084aULL, 0x4980a319601420a8ULL,
-	0x9565b9e12f552c42ULL, 0x8a5318db7100fe96ULL, 0x05c90b4d43add0d7ULL, 0x538b4cd66a5d4edaULL,
-	0xf4e94fc3e89f039fULL, 0x592c9af26f618045ULL, 0x08a36eb5fd4b9550ULL, 0x25fffaf6c2ed1419ULL,
-	0x34434459cc79d354ULL, 0xeeecbfb4b1d5476bULL, 0xddeb34a061615d99ULL, 0x5129cecceb64b773ULL,
-	0xee43215894993520ULL, 0x772f9c7cf14c0b3bULL, 0xd2e2fce306bedad5ULL, 0x715f42b546f06a97ULL,
-	0x434ecdceda5b5f1aULL, 0x0da17115a49741a9ULL, 0x680bd77c73edad2eULL, 0x487c02354edd9041ULL,
-	0xb8efeff3a70ed9c4ULL, 0x56a32aa3e857e302ULL, 0xdf3a68bd48a2a5a0ULL, 0x07f650b73176c444ULL,
-	0xe38b9b1626e0ccb1ULL, 0x79e053c18b09fb36ULL, 0x56d90319c9f94964ULL, 0x1ca941e7ac9ff5c4ULL,
-	0x49c4df29162fa0bbULL, 0x8488cf3282b33305ULL, 0x95dfda14cabb437dULL, 0x3391f78264d5ad86ULL,
-	0x729ae06ae2b5095dULL, 0xd58a58d73259a946ULL, 0xe9834262d13921edULL, 0x27fedafaa54bb592ULL,
-	0xa99dc5b829ad48bbULL, 0x5f025742499ee260ULL, 0x802c8ecd5d7513fdULL, 0x78ceb3ef3f6dd938ULL,
-	0xc342f44f8a135d94ULL, 0x7b9edb44828cdda3ULL, 0x9436d11a0537cfe7ULL, 0x5064b164ec1ab4c8ULL,
-	0x7020eccfd37eb2fcULL, 0x1f31ea3ed90d25fcULL, 0x1b930d7bdfa1bb34ULL, 0x5344467a48113044ULL,
-	0x70073170f25e6dfbULL, 0xe385dc1a50114cc8ULL, 0x2348698ac8fc4f00ULL, 0x2a77a55284dd40d8ULL,
-	0xfe06afe0c98c6ce4ULL, 0xc235df96dddfd6e4ULL, 0x1428d01e33bf1ed3ULL, 0x785768ec9300bdafULL,
-	0x9702e57a91deb63bULL, 0x61bdb8bfe5ce8b80ULL, 0x645b426f3d1d58acULL, 0x4804a82227a557bcULL,
-	0x8e57048ab44d2601ULL, 0x68d6501a4b3a6935ULL, 0xc39c9ec3f9e1c293ULL, 0x4172f257d4de63e2ULL,
-	0xd368b450330c6401ULL, 0x040d3017418f2391ULL, 0x2c34bb6090b7d90dULL, 0x16f649228fdfd51fULL,
-	0xbea6818e2b928ef5ULL, 0xe28ccf91cdc11e72ULL, 0x594aaa68e77a36cdULL, 0x313034806c7ffd0fULL,
-	0x8a9d27ac2249bd65ULL, 0x19a3b464018e9512ULL, 0xc26ccff352b37ec7ULL, 0x056f68341d797b21ULL,
-	0x5e79d6757efd2327ULL, 0xfabdbcb6553afe15ULL, 0xd3e7222c6eaf5a60ULL, 0x7046c76d4dae743bULL,
-	0x660be872b18d4a55ULL, 0x19992518574e1496ULL, 0xc103053a302bdcbbULL, 0x3ed8e9800b218e8eULL,
-	0x7b0b9239fa75e03eULL, 0xefe9fb684633c083ULL, 0x98a35fbe391a7793ULL, 0x6065510fe2d0fe34ULL,
-	0x55cb668548abad0cULL, 0xb4584548da87e527ULL, 0x2c43ecea0107c1ddULL, 0x526028809372de35ULL,
-	0x3415c56af9213b1fULL, 0x5bee1a4d017e98dbULL, 0x13f6b105b5cf709bULL, 0x5ff20e3482b29ab6ULL,
-	0x0aa29c75cc2e6c90ULL, 0xfc7d73ca3a70e206ULL, 0x899fc38fc4b5c515ULL, 0x250386b124ffc207ULL,
-	0x54ea28d5ae3d2b56ULL, 0x9913149dd6de60ceULL, 0x16694fc58f06d6c1ULL, 0x46b23975eb018fc7ULL,
-	0x470a6a0fb4b7b4e2ULL, 0x5d92475a8f7253deULL, 0xabeee5b52fbd3adbULL, 0x7fa20801a0806968ULL,
-	0x76f3faf19f7714d2ULL, 0xb3e840c12f4660c3ULL, 0x0fb4cd8df212744eULL, 0x4b065a251d3a2dd2ULL,
-	0x5cebde383d77cd4aULL, 0x6adf39df882c9cb1ULL, 0xa2dd242eb09af759ULL, 0x3147c0e50e5f6422ULL,
-	0x164ca5101d1350dbULL, 0xf8d13479c33fc962ULL, 0xe640ce4d13e5da08ULL, 0x4bdee0c45061f8baULL,
-	0xd7c46dc1a4edb1c9ULL, 0x5514d7b6437fd98aULL, 0x58942f6bb2a1c00bULL, 0x2dffb2ab1d70710eULL,
-	0xccdfcf2fc18b6d68ULL, 0xa8ebcba8b7806167ULL, 0x980697f95e2937e3ULL, 0x02fbba1cd0126e8cULL
-};
-
-static void curve25519_ever64_base(u8 *out, const u8 *priv)
-{
-	u64 swap = 1;
-	int i, j, k;
-	u64 tmp[16 + 32 + 4];
-	u64 *x1 = &tmp[0];
-	u64 *z1 = &tmp[4];
-	u64 *x2 = &tmp[8];
-	u64 *z2 = &tmp[12];
-	u64 *xz1 = &tmp[0];
-	u64 *xz2 = &tmp[8];
-	u64 *a = &tmp[0 + 16];
-	u64 *b = &tmp[4 + 16];
-	u64 *c = &tmp[8 + 16];
-	u64 *ab = &tmp[0 + 16];
-	u64 *abcd = &tmp[0 + 16];
-	u64 *ef = &tmp[16 + 16];
-	u64 *efgh = &tmp[16 + 16];
-	u64 *key = &tmp[0 + 16 + 32];
-
-	memcpy(key, priv, 32);
-	((u8 *)key)[0] &= 248;
-	((u8 *)key)[31] = (((u8 *)key)[31] & 127) | 64;
-
-	x1[0] = 1, x1[1] = x1[2] = x1[3] = 0;
-	z1[0] = 1, z1[1] = z1[2] = z1[3] = 0;
-	z2[0] = 1, z2[1] = z2[2] = z2[3] = 0;
-	memcpy(x2, p_minus_s, sizeof(p_minus_s));
-
-	j = 3;
-	for (i = 0; i < 4; ++i) {
-		while (j < (const int[]){ 64, 64, 64, 63 }[i]) {
-			u64 bit = (key[i] >> j) & 1;
-			k = (64 * i + j - 3);
-			swap = swap ^ bit;
-			cswap2(swap, xz1, xz2);
-			swap = bit;
-			fsub(b, x1, z1);
-			fadd(a, x1, z1);
-			fmul(c, &table_ladder[4 * k], b, ef);
-			fsub(b, a, c);
-			fadd(a, a, c);
-			fsqr2(ab, ab, efgh);
-			fmul2(xz1, xz2, ab, efgh);
-			++j;
-		}
-		j = 0;
-	}
-
-	point_double(xz1, abcd, efgh);
-	point_double(xz1, abcd, efgh);
-	point_double(xz1, abcd, efgh);
-	encode_point(out, xz1);
-
-	memzero_explicit(tmp, sizeof(tmp));
-}
-
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(curve25519_use_bmi2_adx);
-
-void curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE],
-		     const u8 secret[CURVE25519_KEY_SIZE],
-		     const u8 basepoint[CURVE25519_KEY_SIZE])
-{
-	if (static_branch_likely(&curve25519_use_bmi2_adx))
-		curve25519_ever64(mypublic, secret, basepoint);
-	else
-		curve25519_generic(mypublic, secret, basepoint);
-}
-EXPORT_SYMBOL(curve25519_arch);
-
-void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE],
-			  const u8 secret[CURVE25519_KEY_SIZE])
-{
-	if (static_branch_likely(&curve25519_use_bmi2_adx))
-		curve25519_ever64_base(pub, secret);
-	else
-		curve25519_generic(pub, secret, curve25519_base_point);
-}
-EXPORT_SYMBOL(curve25519_base_arch);
-
-static int curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,
-				 unsigned int len)
-{
-	u8 *secret = kpp_tfm_ctx(tfm);
-
-	if (!len)
-		curve25519_generate_secret(secret);
-	else if (len == CURVE25519_KEY_SIZE &&
-		 crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE))
-		memcpy(secret, buf, CURVE25519_KEY_SIZE);
-	else
-		return -EINVAL;
-	return 0;
-}
-
-static int curve25519_generate_public_key(struct kpp_request *req)
-{
-	struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
-	const u8 *secret = kpp_tfm_ctx(tfm);
-	u8 buf[CURVE25519_KEY_SIZE];
-	int copied, nbytes;
-
-	if (req->src)
-		return -EINVAL;
-
-	curve25519_base_arch(buf, secret);
-
-	/* might want less than we've got */
-	nbytes = min_t(size_t, CURVE25519_KEY_SIZE, req->dst_len);
-	copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst,
-								nbytes),
-				     buf, nbytes);
-	if (copied != nbytes)
-		return -EINVAL;
-	return 0;
-}
-
-static int curve25519_compute_shared_secret(struct kpp_request *req)
-{
-	struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
-	const u8 *secret = kpp_tfm_ctx(tfm);
-	u8 public_key[CURVE25519_KEY_SIZE];
-	u8 buf[CURVE25519_KEY_SIZE];
-	int copied, nbytes;
-
-	if (!req->src)
-		return -EINVAL;
-
-	copied = sg_copy_to_buffer(req->src,
-				   sg_nents_for_len(req->src,
-						    CURVE25519_KEY_SIZE),
-				   public_key, CURVE25519_KEY_SIZE);
-	if (copied != CURVE25519_KEY_SIZE)
-		return -EINVAL;
-
-	curve25519_arch(buf, secret, public_key);
-
-	/* might want less than we've got */
-	nbytes = min_t(size_t, CURVE25519_KEY_SIZE, req->dst_len);
-	copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst,
-								nbytes),
-				     buf, nbytes);
-	if (copied != nbytes)
-		return -EINVAL;
-	return 0;
-}
-
-static unsigned int curve25519_max_size(struct crypto_kpp *tfm)
-{
-	return CURVE25519_KEY_SIZE;
-}
-
-static struct kpp_alg curve25519_alg = {
-	.base.cra_name		= "curve25519",
-	.base.cra_driver_name	= "curve25519-x86",
-	.base.cra_priority	= 200,
-	.base.cra_module	= THIS_MODULE,
-	.base.cra_ctxsize	= CURVE25519_KEY_SIZE,
-
-	.set_secret		= curve25519_set_secret,
-	.generate_public_key	= curve25519_generate_public_key,
-	.compute_shared_secret	= curve25519_compute_shared_secret,
-	.max_size		= curve25519_max_size,
-};
-
-
-static int __init curve25519_mod_init(void)
-{
-	if (boot_cpu_has(X86_FEATURE_BMI2) && boot_cpu_has(X86_FEATURE_ADX))
-		static_branch_enable(&curve25519_use_bmi2_adx);
-	else
-		return 0;
-	return IS_REACHABLE(CONFIG_CRYPTO_KPP) ?
-		crypto_register_kpp(&curve25519_alg) : 0;
-}
-
-static void __exit curve25519_mod_exit(void)
-{
-	if (IS_REACHABLE(CONFIG_CRYPTO_KPP) &&
-	    static_branch_likely(&curve25519_use_bmi2_adx))
-		crypto_unregister_kpp(&curve25519_alg);
-}
-
-module_init(curve25519_mod_init);
-module_exit(curve25519_mod_exit);
-
-MODULE_ALIAS_CRYPTO("curve25519");
-MODULE_ALIAS_CRYPTO("curve25519-x86");
-MODULE_DESCRIPTION("Curve25519 algorithm, ADX optimized");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c
index abb8b1fe123b..34600f90d8a6 100644
--- a/arch/x86/crypto/des3_ede_glue.c
+++ b/arch/x86/crypto/des3_ede_glue.c
@@ -73,7 +73,7 @@ static int ecb_crypt(struct skcipher_request *req, const u32 *expkey)
 	err = skcipher_walk_virt(&walk, req, false);
 
 	while ((nbytes = walk.nbytes)) {
-		u8 *wsrc = walk.src.virt.addr;
+		const u8 *wsrc = walk.src.virt.addr;
 		u8 *wdst = walk.dst.virt.addr;
 
 		/* Process four block batch */
@@ -291,7 +291,6 @@ static struct crypto_alg des3_ede_cipher = {
 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize		= DES3_EDE_BLOCK_SIZE,
 	.cra_ctxsize		= sizeof(struct des3_ede_x86_ctx),
-	.cra_alignmask		= 0,
 	.cra_module		= THIS_MODULE,
 	.cra_u = {
 		.cipher = {
diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/arch/x86/crypto/ghash-clmulni-intel_asm.S
index 99cb983ded9e..c4fbaa82ed7a 100644
--- a/arch/x86/crypto/ghash-clmulni-intel_asm.S
+++ b/arch/x86/crypto/ghash-clmulni-intel_asm.S
@@ -103,8 +103,8 @@ SYM_FUNC_START(clmul_ghash_mul)
 SYM_FUNC_END(clmul_ghash_mul)
 
 /*
- * void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
- *			   const le128 *shash);
+ * int clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
+ *			  const le128 *shash);
  */
 SYM_FUNC_START(clmul_ghash_update)
 	FRAME_BEGIN
@@ -127,6 +127,7 @@ SYM_FUNC_START(clmul_ghash_update)
 	pshufb BSWAP, DATA
 	movups DATA, (%rdi)
 .Lupdate_just_ret:
+	mov %rdx, %rax
 	FRAME_END
 	RET
 SYM_FUNC_END(clmul_ghash_update)
diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c
index 41bc02e48916..aea5d4d06be7 100644
--- a/arch/x86/crypto/ghash-clmulni-intel_glue.c
+++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c
@@ -7,41 +7,27 @@
  *   Author: Huang Ying <ying.huang@intel.com>
  */
 
-#include <linux/err.h>
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/crypto.h>
-#include <crypto/algapi.h>
-#include <crypto/cryptd.h>
-#include <crypto/gf128mul.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
 #include <asm/cpu_device_id.h>
 #include <asm/simd.h>
+#include <crypto/b128ops.h>
+#include <crypto/ghash.h>
+#include <crypto/internal/hash.h>
+#include <crypto/utils.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h>
 #include <linux/unaligned.h>
 
-#define GHASH_BLOCK_SIZE	16
-#define GHASH_DIGEST_SIZE	16
+asmlinkage void clmul_ghash_mul(char *dst, const le128 *shash);
 
-void clmul_ghash_mul(char *dst, const le128 *shash);
+asmlinkage int clmul_ghash_update(char *dst, const char *src,
+				  unsigned int srclen, const le128 *shash);
 
-void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
-			const le128 *shash);
-
-struct ghash_async_ctx {
-	struct cryptd_ahash *cryptd_tfm;
-};
-
-struct ghash_ctx {
+struct x86_ghash_ctx {
 	le128 shash;
 };
 
-struct ghash_desc_ctx {
-	u8 buffer[GHASH_BLOCK_SIZE];
-	u32 bytes;
-};
-
 static int ghash_init(struct shash_desc *desc)
 {
 	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
@@ -54,7 +40,7 @@ static int ghash_init(struct shash_desc *desc)
 static int ghash_setkey(struct crypto_shash *tfm,
 			const u8 *key, unsigned int keylen)
 {
-	struct ghash_ctx *ctx = crypto_shash_ctx(tfm);
+	struct x86_ghash_ctx *ctx = crypto_shash_ctx(tfm);
 	u64 a, b;
 
 	if (keylen != GHASH_BLOCK_SIZE)
@@ -95,64 +81,38 @@ static int ghash_setkey(struct crypto_shash *tfm,
 static int ghash_update(struct shash_desc *desc,
 			 const u8 *src, unsigned int srclen)
 {
+	struct x86_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
 	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
-	struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
 	u8 *dst = dctx->buffer;
+	int remain;
 
 	kernel_fpu_begin();
-	if (dctx->bytes) {
-		int n = min(srclen, dctx->bytes);
-		u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes);
-
-		dctx->bytes -= n;
-		srclen -= n;
-
-		while (n--)
-			*pos++ ^= *src++;
-
-		if (!dctx->bytes)
-			clmul_ghash_mul(dst, &ctx->shash);
-	}
-
-	clmul_ghash_update(dst, src, srclen, &ctx->shash);
+	remain = clmul_ghash_update(dst, src, srclen, &ctx->shash);
 	kernel_fpu_end();
-
-	if (srclen & 0xf) {
-		src += srclen - (srclen & 0xf);
-		srclen &= 0xf;
-		dctx->bytes = GHASH_BLOCK_SIZE - srclen;
-		while (srclen--)
-			*dst++ ^= *src++;
-	}
-
-	return 0;
+	return remain;
 }
 
-static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx)
+static void ghash_flush(struct x86_ghash_ctx *ctx, struct ghash_desc_ctx *dctx,
+			const u8 *src, unsigned int len)
 {
 	u8 *dst = dctx->buffer;
 
-	if (dctx->bytes) {
-		u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes);
-
-		while (dctx->bytes--)
-			*tmp++ ^= 0;
-
-		kernel_fpu_begin();
+	kernel_fpu_begin();
+	if (len) {
+		crypto_xor(dst, src, len);
 		clmul_ghash_mul(dst, &ctx->shash);
-		kernel_fpu_end();
 	}
-
-	dctx->bytes = 0;
+	kernel_fpu_end();
 }
 
-static int ghash_final(struct shash_desc *desc, u8 *dst)
+static int ghash_finup(struct shash_desc *desc, const u8 *src,
+		       unsigned int len, u8 *dst)
 {
+	struct x86_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
 	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
-	struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
 	u8 *buf = dctx->buffer;
 
-	ghash_flush(ctx, dctx);
+	ghash_flush(ctx, dctx, src, len);
 	memcpy(dst, buf, GHASH_BLOCK_SIZE);
 
 	return 0;
@@ -162,175 +122,20 @@ static struct shash_alg ghash_alg = {
 	.digestsize	= GHASH_DIGEST_SIZE,
 	.init		= ghash_init,
 	.update		= ghash_update,
-	.final		= ghash_final,
+	.finup		= ghash_finup,
 	.setkey		= ghash_setkey,
 	.descsize	= sizeof(struct ghash_desc_ctx),
 	.base		= {
-		.cra_name		= "__ghash",
-		.cra_driver_name	= "__ghash-pclmulqdqni",
-		.cra_priority		= 0,
-		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_name		= "ghash",
+		.cra_driver_name	= "ghash-pclmulqdqni",
+		.cra_priority		= 400,
+		.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,
 		.cra_blocksize		= GHASH_BLOCK_SIZE,
-		.cra_ctxsize		= sizeof(struct ghash_ctx),
+		.cra_ctxsize		= sizeof(struct x86_ghash_ctx),
 		.cra_module		= THIS_MODULE,
 	},
 };
 
-static int ghash_async_init(struct ahash_request *req)
-{
-	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
-	struct ahash_request *cryptd_req = ahash_request_ctx(req);
-	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
-	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
-	struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
-
-	desc->tfm = child;
-	return crypto_shash_init(desc);
-}
-
-static int ghash_async_update(struct ahash_request *req)
-{
-	struct ahash_request *cryptd_req = ahash_request_ctx(req);
-	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
-	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
-
-	if (!crypto_simd_usable() ||
-	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
-		memcpy(cryptd_req, req, sizeof(*req));
-		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
-		return crypto_ahash_update(cryptd_req);
-	} else {
-		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
-		return shash_ahash_update(req, desc);
-	}
-}
-
-static int ghash_async_final(struct ahash_request *req)
-{
-	struct ahash_request *cryptd_req = ahash_request_ctx(req);
-	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
-	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
-
-	if (!crypto_simd_usable() ||
-	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
-		memcpy(cryptd_req, req, sizeof(*req));
-		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
-		return crypto_ahash_final(cryptd_req);
-	} else {
-		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
-		return crypto_shash_final(desc, req->result);
-	}
-}
-
-static int ghash_async_import(struct ahash_request *req, const void *in)
-{
-	struct ahash_request *cryptd_req = ahash_request_ctx(req);
-	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
-	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
-
-	ghash_async_init(req);
-	memcpy(dctx, in, sizeof(*dctx));
-	return 0;
-
-}
-
-static int ghash_async_export(struct ahash_request *req, void *out)
-{
-	struct ahash_request *cryptd_req = ahash_request_ctx(req);
-	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
-	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
-
-	memcpy(out, dctx, sizeof(*dctx));
-	return 0;
-
-}
-
-static int ghash_async_digest(struct ahash_request *req)
-{
-	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
-	struct ahash_request *cryptd_req = ahash_request_ctx(req);
-	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
-
-	if (!crypto_simd_usable() ||
-	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
-		memcpy(cryptd_req, req, sizeof(*req));
-		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
-		return crypto_ahash_digest(cryptd_req);
-	} else {
-		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
-		struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
-
-		desc->tfm = child;
-		return shash_ahash_digest(req, desc);
-	}
-}
-
-static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
-			      unsigned int keylen)
-{
-	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
-	struct crypto_ahash *child = &ctx->cryptd_tfm->base;
-
-	crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
-	crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
-			       & CRYPTO_TFM_REQ_MASK);
-	return crypto_ahash_setkey(child, key, keylen);
-}
-
-static int ghash_async_init_tfm(struct crypto_tfm *tfm)
-{
-	struct cryptd_ahash *cryptd_tfm;
-	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni",
-					CRYPTO_ALG_INTERNAL,
-					CRYPTO_ALG_INTERNAL);
-	if (IS_ERR(cryptd_tfm))
-		return PTR_ERR(cryptd_tfm);
-	ctx->cryptd_tfm = cryptd_tfm;
-	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
-				 sizeof(struct ahash_request) +
-				 crypto_ahash_reqsize(&cryptd_tfm->base));
-
-	return 0;
-}
-
-static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
-{
-	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	cryptd_free_ahash(ctx->cryptd_tfm);
-}
-
-static struct ahash_alg ghash_async_alg = {
-	.init		= ghash_async_init,
-	.update		= ghash_async_update,
-	.final		= ghash_async_final,
-	.setkey		= ghash_async_setkey,
-	.digest		= ghash_async_digest,
-	.export		= ghash_async_export,
-	.import		= ghash_async_import,
-	.halg = {
-		.digestsize	= GHASH_DIGEST_SIZE,
-		.statesize = sizeof(struct ghash_desc_ctx),
-		.base = {
-			.cra_name		= "ghash",
-			.cra_driver_name	= "ghash-clmulni",
-			.cra_priority		= 400,
-			.cra_ctxsize		= sizeof(struct ghash_async_ctx),
-			.cra_flags		= CRYPTO_ALG_ASYNC,
-			.cra_blocksize		= GHASH_BLOCK_SIZE,
-			.cra_module		= THIS_MODULE,
-			.cra_init		= ghash_async_init_tfm,
-			.cra_exit		= ghash_async_exit_tfm,
-		},
-	},
-};
-
 static const struct x86_cpu_id pcmul_cpu_id[] = {
 	X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL), /* Pickle-Mickle-Duck */
 	{}
@@ -339,29 +144,14 @@ MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);
 
 static int __init ghash_pclmulqdqni_mod_init(void)
 {
-	int err;
-
 	if (!x86_match_cpu(pcmul_cpu_id))
 		return -ENODEV;
 
-	err = crypto_register_shash(&ghash_alg);
-	if (err)
-		goto err_out;
-	err = crypto_register_ahash(&ghash_async_alg);
-	if (err)
-		goto err_shash;
-
-	return 0;
-
-err_shash:
-	crypto_unregister_shash(&ghash_alg);
-err_out:
-	return err;
+	return crypto_register_shash(&ghash_alg);
 }
 
 static void __exit ghash_pclmulqdqni_mod_exit(void)
 {
-	crypto_unregister_ahash(&ghash_async_alg);
 	crypto_unregister_shash(&ghash_alg);
 }
 
diff --git a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl
deleted file mode 100644
index b9abcd79c1f4..000000000000
--- a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl
+++ /dev/null
@@ -1,4248 +0,0 @@
-#!/usr/bin/env perl
-# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
-#
-# Copyright (C) 2017-2018 Samuel Neves <sneves@dei.uc.pt>. All Rights Reserved.
-# Copyright (C) 2017-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
-# Copyright (C) 2006-2017 CRYPTOGAMS by <appro@openssl.org>. All Rights Reserved.
-#
-# This code is taken from the OpenSSL project but the author, Andy Polyakov,
-# has relicensed it under the licenses specified in the SPDX header above.
-# The original headers, including the original license headers, are
-# included below for completeness.
-#
-# ====================================================================
-# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
-# project. The module is, however, dual licensed under OpenSSL and
-# CRYPTOGAMS licenses depending on where you obtain it. For further
-# details see http://www.openssl.org/~appro/cryptogams/.
-# ====================================================================
-#
-# This module implements Poly1305 hash for x86_64.
-#
-# March 2015
-#
-# Initial release.
-#
-# December 2016
-#
-# Add AVX512F+VL+BW code path.
-#
-# November 2017
-#
-# Convert AVX512F+VL+BW code path to pure AVX512F, so that it can be
-# executed even on Knights Landing. Trigger for modification was
-# observation that AVX512 code paths can negatively affect overall
-# Skylake-X system performance. Since we are likely to suppress
-# AVX512F capability flag [at least on Skylake-X], conversion serves
-# as kind of "investment protection". Note that next *lake processor,
-# Cannonlake, has AVX512IFMA code path to execute...
-#
-# Numbers are cycles per processed byte with poly1305_blocks alone,
-# measured with rdtsc at fixed clock frequency.
-#
-#		IALU/gcc-4.8(*)	AVX(**)		AVX2	AVX-512
-# P4		4.46/+120%	-
-# Core 2	2.41/+90%	-
-# Westmere	1.88/+120%	-
-# Sandy Bridge	1.39/+140%	1.10
-# Haswell	1.14/+175%	1.11		0.65
-# Skylake[-X]	1.13/+120%	0.96		0.51	[0.35]
-# Silvermont	2.83/+95%	-
-# Knights L	3.60/?		1.65		1.10	0.41(***)
-# Goldmont	1.70/+180%	-
-# VIA Nano	1.82/+150%	-
-# Sledgehammer	1.38/+160%	-
-# Bulldozer	2.30/+130%	0.97
-# Ryzen		1.15/+200%	1.08		1.18
-#
-# (*)	improvement coefficients relative to clang are more modest and
-#	are ~50% on most processors, in both cases we are comparing to
-#	__int128 code;
-# (**)	SSE2 implementation was attempted, but among non-AVX processors
-#	it was faster than integer-only code only on older Intel P4 and
-#	Core processors, 50-30%, less newer processor is, but slower on
-#	contemporary ones, for example almost 2x slower on Atom, and as
-#	former are naturally disappearing, SSE2 is deemed unnecessary;
-# (***)	strangely enough performance seems to vary from core to core,
-#	listed result is best case;
-
-$flavour = shift;
-$output  = shift;
-if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
-
-$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
-$kernel=0; $kernel=1 if (!$flavour && !$output);
-
-if (!$kernel) {
-	$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
-	( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
-	( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
-	die "can't locate x86_64-xlate.pl";
-
-	open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
-	*STDOUT=*OUT;
-
-	if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
-	    =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
-		$avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25);
-	}
-
-	if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
-	    `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) {
-		$avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12);
-		$avx += 1 if ($1==2.11 && $2>=8);
-	}
-
-	if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
-	    `ml64 2>&1` =~ /Version ([0-9]+)\./) {
-		$avx = ($1>=10) + ($1>=11);
-	}
-
-	if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
-		$avx = ($2>=3.0) + ($2>3.0);
-	}
-} else {
-	$avx = 4; # The kernel uses ifdefs for this.
-}
-
-sub declare_function() {
-	my ($name, $align, $nargs) = @_;
-	if($kernel) {
-		$code .= "SYM_FUNC_START($name)\n";
-		$code .= ".L$name:\n";
-	} else {
-		$code .= ".globl	$name\n";
-		$code .= ".type	$name,\@function,$nargs\n";
-		$code .= ".align	$align\n";
-		$code .= "$name:\n";
-	}
-}
-
-sub end_function() {
-	my ($name) = @_;
-	if($kernel) {
-		$code .= "SYM_FUNC_END($name)\n";
-	} else {
-		$code .= ".size   $name,.-$name\n";
-	}
-}
-
-$code.=<<___ if $kernel;
-#include <linux/linkage.h>
-___
-
-if ($avx) {
-$code.=<<___ if $kernel;
-.section .rodata
-___
-$code.=<<___;
-.align	64
-.Lconst:
-.Lmask24:
-.long	0x0ffffff,0,0x0ffffff,0,0x0ffffff,0,0x0ffffff,0
-.L129:
-.long	`1<<24`,0,`1<<24`,0,`1<<24`,0,`1<<24`,0
-.Lmask26:
-.long	0x3ffffff,0,0x3ffffff,0,0x3ffffff,0,0x3ffffff,0
-.Lpermd_avx2:
-.long	2,2,2,3,2,0,2,1
-.Lpermd_avx512:
-.long	0,0,0,1, 0,2,0,3, 0,4,0,5, 0,6,0,7
-
-.L2_44_inp_permd:
-.long	0,1,1,2,2,3,7,7
-.L2_44_inp_shift:
-.quad	0,12,24,64
-.L2_44_mask:
-.quad	0xfffffffffff,0xfffffffffff,0x3ffffffffff,0xffffffffffffffff
-.L2_44_shift_rgt:
-.quad	44,44,42,64
-.L2_44_shift_lft:
-.quad	8,8,10,64
-
-.align	64
-.Lx_mask44:
-.quad	0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff
-.quad	0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff
-.Lx_mask42:
-.quad	0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff
-.quad	0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff
-___
-}
-$code.=<<___ if (!$kernel);
-.asciz	"Poly1305 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
-.align	16
-___
-
-my ($ctx,$inp,$len,$padbit)=("%rdi","%rsi","%rdx","%rcx");
-my ($mac,$nonce)=($inp,$len);	# *_emit arguments
-my ($d1,$d2,$d3, $r0,$r1,$s1)=("%r8","%r9","%rdi","%r11","%r12","%r13");
-my ($h0,$h1,$h2)=("%r14","%rbx","%r10");
-
-sub poly1305_iteration {
-# input:	copy of $r1 in %rax, $h0-$h2, $r0-$r1
-# output:	$h0-$h2 *= $r0-$r1
-$code.=<<___;
-	mulq	$h0			# h0*r1
-	mov	%rax,$d2
-	 mov	$r0,%rax
-	mov	%rdx,$d3
-
-	mulq	$h0			# h0*r0
-	mov	%rax,$h0		# future $h0
-	 mov	$r0,%rax
-	mov	%rdx,$d1
-
-	mulq	$h1			# h1*r0
-	add	%rax,$d2
-	 mov	$s1,%rax
-	adc	%rdx,$d3
-
-	mulq	$h1			# h1*s1
-	 mov	$h2,$h1			# borrow $h1
-	add	%rax,$h0
-	adc	%rdx,$d1
-
-	imulq	$s1,$h1			# h2*s1
-	add	$h1,$d2
-	 mov	$d1,$h1
-	adc	\$0,$d3
-
-	imulq	$r0,$h2			# h2*r0
-	add	$d2,$h1
-	mov	\$-4,%rax		# mask value
-	adc	$h2,$d3
-
-	and	$d3,%rax		# last reduction step
-	mov	$d3,$h2
-	shr	\$2,$d3
-	and	\$3,$h2
-	add	$d3,%rax
-	add	%rax,$h0
-	adc	\$0,$h1
-	adc	\$0,$h2
-___
-}
-
-########################################################################
-# Layout of opaque area is following.
-#
-#	unsigned __int64 h[3];		# current hash value base 2^64
-#	unsigned __int64 r[2];		# key value base 2^64
-
-$code.=<<___;
-.text
-___
-$code.=<<___ if (!$kernel);
-.extern	OPENSSL_ia32cap_P
-
-.globl	poly1305_init_x86_64
-.hidden	poly1305_init_x86_64
-.globl	poly1305_blocks_x86_64
-.hidden	poly1305_blocks_x86_64
-.globl	poly1305_emit_x86_64
-.hidden	poly1305_emit_x86_64
-___
-&declare_function("poly1305_init_x86_64", 32, 3);
-$code.=<<___;
-	xor	%eax,%eax
-	mov	%rax,0($ctx)		# initialize hash value
-	mov	%rax,8($ctx)
-	mov	%rax,16($ctx)
-
-	test	$inp,$inp
-	je	.Lno_key
-___
-$code.=<<___ if (!$kernel);
-	lea	poly1305_blocks_x86_64(%rip),%r10
-	lea	poly1305_emit_x86_64(%rip),%r11
-___
-$code.=<<___	if (!$kernel && $avx);
-	mov	OPENSSL_ia32cap_P+4(%rip),%r9
-	lea	poly1305_blocks_avx(%rip),%rax
-	lea	poly1305_emit_avx(%rip),%rcx
-	bt	\$`60-32`,%r9		# AVX?
-	cmovc	%rax,%r10
-	cmovc	%rcx,%r11
-___
-$code.=<<___	if (!$kernel && $avx>1);
-	lea	poly1305_blocks_avx2(%rip),%rax
-	bt	\$`5+32`,%r9		# AVX2?
-	cmovc	%rax,%r10
-___
-$code.=<<___	if (!$kernel && $avx>3);
-	mov	\$`(1<<31|1<<21|1<<16)`,%rax
-	shr	\$32,%r9
-	and	%rax,%r9
-	cmp	%rax,%r9
-	je	.Linit_base2_44
-___
-$code.=<<___;
-	mov	\$0x0ffffffc0fffffff,%rax
-	mov	\$0x0ffffffc0ffffffc,%rcx
-	and	0($inp),%rax
-	and	8($inp),%rcx
-	mov	%rax,24($ctx)
-	mov	%rcx,32($ctx)
-___
-$code.=<<___	if (!$kernel && $flavour !~ /elf32/);
-	mov	%r10,0(%rdx)
-	mov	%r11,8(%rdx)
-___
-$code.=<<___	if (!$kernel && $flavour =~ /elf32/);
-	mov	%r10d,0(%rdx)
-	mov	%r11d,4(%rdx)
-___
-$code.=<<___;
-	mov	\$1,%eax
-.Lno_key:
-	RET
-___
-&end_function("poly1305_init_x86_64");
-
-&declare_function("poly1305_blocks_x86_64", 32, 4);
-$code.=<<___;
-.cfi_startproc
-.Lblocks:
-	shr	\$4,$len
-	jz	.Lno_data		# too short
-
-	push	%rbx
-.cfi_push	%rbx
-	push	%r12
-.cfi_push	%r12
-	push	%r13
-.cfi_push	%r13
-	push	%r14
-.cfi_push	%r14
-	push	%r15
-.cfi_push	%r15
-	push	$ctx
-.cfi_push	$ctx
-.Lblocks_body:
-
-	mov	$len,%r15		# reassign $len
-
-	mov	24($ctx),$r0		# load r
-	mov	32($ctx),$s1
-
-	mov	0($ctx),$h0		# load hash value
-	mov	8($ctx),$h1
-	mov	16($ctx),$h2
-
-	mov	$s1,$r1
-	shr	\$2,$s1
-	mov	$r1,%rax
-	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
-	jmp	.Loop
-
-.align	32
-.Loop:
-	add	0($inp),$h0		# accumulate input
-	adc	8($inp),$h1
-	lea	16($inp),$inp
-	adc	$padbit,$h2
-___
-
-	&poly1305_iteration();
-
-$code.=<<___;
-	mov	$r1,%rax
-	dec	%r15			# len-=16
-	jnz	.Loop
-
-	mov	0(%rsp),$ctx
-.cfi_restore	$ctx
-
-	mov	$h0,0($ctx)		# store hash value
-	mov	$h1,8($ctx)
-	mov	$h2,16($ctx)
-
-	mov	8(%rsp),%r15
-.cfi_restore	%r15
-	mov	16(%rsp),%r14
-.cfi_restore	%r14
-	mov	24(%rsp),%r13
-.cfi_restore	%r13
-	mov	32(%rsp),%r12
-.cfi_restore	%r12
-	mov	40(%rsp),%rbx
-.cfi_restore	%rbx
-	lea	48(%rsp),%rsp
-.cfi_adjust_cfa_offset	-48
-.Lno_data:
-.Lblocks_epilogue:
-	RET
-.cfi_endproc
-___
-&end_function("poly1305_blocks_x86_64");
-
-&declare_function("poly1305_emit_x86_64", 32, 3);
-$code.=<<___;
-.Lemit:
-	mov	0($ctx),%r8	# load hash value
-	mov	8($ctx),%r9
-	mov	16($ctx),%r10
-
-	mov	%r8,%rax
-	add	\$5,%r8		# compare to modulus
-	mov	%r9,%rcx
-	adc	\$0,%r9
-	adc	\$0,%r10
-	shr	\$2,%r10	# did 130-bit value overflow?
-	cmovnz	%r8,%rax
-	cmovnz	%r9,%rcx
-
-	add	0($nonce),%rax	# accumulate nonce
-	adc	8($nonce),%rcx
-	mov	%rax,0($mac)	# write result
-	mov	%rcx,8($mac)
-
-	RET
-___
-&end_function("poly1305_emit_x86_64");
-if ($avx) {
-
-########################################################################
-# Layout of opaque area is following.
-#
-#	unsigned __int32 h[5];		# current hash value base 2^26
-#	unsigned __int32 is_base2_26;
-#	unsigned __int64 r[2];		# key value base 2^64
-#	unsigned __int64 pad;
-#	struct { unsigned __int32 r^2, r^1, r^4, r^3; } r[9];
-#
-# where r^n are base 2^26 digits of degrees of multiplier key. There are
-# 5 digits, but last four are interleaved with multiples of 5, totalling
-# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
-
-my ($H0,$H1,$H2,$H3,$H4, $T0,$T1,$T2,$T3,$T4, $D0,$D1,$D2,$D3,$D4, $MASK) =
-    map("%xmm$_",(0..15));
-
-$code.=<<___;
-.type	__poly1305_block,\@abi-omnipotent
-.align	32
-__poly1305_block:
-	push $ctx
-___
-	&poly1305_iteration();
-$code.=<<___;
-	pop $ctx
-	RET
-.size	__poly1305_block,.-__poly1305_block
-
-.type	__poly1305_init_avx,\@abi-omnipotent
-.align	32
-__poly1305_init_avx:
-	push %rbp
-	mov %rsp,%rbp
-	mov	$r0,$h0
-	mov	$r1,$h1
-	xor	$h2,$h2
-
-	lea	48+64($ctx),$ctx	# size optimization
-
-	mov	$r1,%rax
-	call	__poly1305_block	# r^2
-
-	mov	\$0x3ffffff,%eax	# save interleaved r^2 and r base 2^26
-	mov	\$0x3ffffff,%edx
-	mov	$h0,$d1
-	and	$h0#d,%eax
-	mov	$r0,$d2
-	and	$r0#d,%edx
-	mov	%eax,`16*0+0-64`($ctx)
-	shr	\$26,$d1
-	mov	%edx,`16*0+4-64`($ctx)
-	shr	\$26,$d2
-
-	mov	\$0x3ffffff,%eax
-	mov	\$0x3ffffff,%edx
-	and	$d1#d,%eax
-	and	$d2#d,%edx
-	mov	%eax,`16*1+0-64`($ctx)
-	lea	(%rax,%rax,4),%eax	# *5
-	mov	%edx,`16*1+4-64`($ctx)
-	lea	(%rdx,%rdx,4),%edx	# *5
-	mov	%eax,`16*2+0-64`($ctx)
-	shr	\$26,$d1
-	mov	%edx,`16*2+4-64`($ctx)
-	shr	\$26,$d2
-
-	mov	$h1,%rax
-	mov	$r1,%rdx
-	shl	\$12,%rax
-	shl	\$12,%rdx
-	or	$d1,%rax
-	or	$d2,%rdx
-	and	\$0x3ffffff,%eax
-	and	\$0x3ffffff,%edx
-	mov	%eax,`16*3+0-64`($ctx)
-	lea	(%rax,%rax,4),%eax	# *5
-	mov	%edx,`16*3+4-64`($ctx)
-	lea	(%rdx,%rdx,4),%edx	# *5
-	mov	%eax,`16*4+0-64`($ctx)
-	mov	$h1,$d1
-	mov	%edx,`16*4+4-64`($ctx)
-	mov	$r1,$d2
-
-	mov	\$0x3ffffff,%eax
-	mov	\$0x3ffffff,%edx
-	shr	\$14,$d1
-	shr	\$14,$d2
-	and	$d1#d,%eax
-	and	$d2#d,%edx
-	mov	%eax,`16*5+0-64`($ctx)
-	lea	(%rax,%rax,4),%eax	# *5
-	mov	%edx,`16*5+4-64`($ctx)
-	lea	(%rdx,%rdx,4),%edx	# *5
-	mov	%eax,`16*6+0-64`($ctx)
-	shr	\$26,$d1
-	mov	%edx,`16*6+4-64`($ctx)
-	shr	\$26,$d2
-
-	mov	$h2,%rax
-	shl	\$24,%rax
-	or	%rax,$d1
-	mov	$d1#d,`16*7+0-64`($ctx)
-	lea	($d1,$d1,4),$d1		# *5
-	mov	$d2#d,`16*7+4-64`($ctx)
-	lea	($d2,$d2,4),$d2		# *5
-	mov	$d1#d,`16*8+0-64`($ctx)
-	mov	$d2#d,`16*8+4-64`($ctx)
-
-	mov	$r1,%rax
-	call	__poly1305_block	# r^3
-
-	mov	\$0x3ffffff,%eax	# save r^3 base 2^26
-	mov	$h0,$d1
-	and	$h0#d,%eax
-	shr	\$26,$d1
-	mov	%eax,`16*0+12-64`($ctx)
-
-	mov	\$0x3ffffff,%edx
-	and	$d1#d,%edx
-	mov	%edx,`16*1+12-64`($ctx)
-	lea	(%rdx,%rdx,4),%edx	# *5
-	shr	\$26,$d1
-	mov	%edx,`16*2+12-64`($ctx)
-
-	mov	$h1,%rax
-	shl	\$12,%rax
-	or	$d1,%rax
-	and	\$0x3ffffff,%eax
-	mov	%eax,`16*3+12-64`($ctx)
-	lea	(%rax,%rax,4),%eax	# *5
-	mov	$h1,$d1
-	mov	%eax,`16*4+12-64`($ctx)
-
-	mov	\$0x3ffffff,%edx
-	shr	\$14,$d1
-	and	$d1#d,%edx
-	mov	%edx,`16*5+12-64`($ctx)
-	lea	(%rdx,%rdx,4),%edx	# *5
-	shr	\$26,$d1
-	mov	%edx,`16*6+12-64`($ctx)
-
-	mov	$h2,%rax
-	shl	\$24,%rax
-	or	%rax,$d1
-	mov	$d1#d,`16*7+12-64`($ctx)
-	lea	($d1,$d1,4),$d1		# *5
-	mov	$d1#d,`16*8+12-64`($ctx)
-
-	mov	$r1,%rax
-	call	__poly1305_block	# r^4
-
-	mov	\$0x3ffffff,%eax	# save r^4 base 2^26
-	mov	$h0,$d1
-	and	$h0#d,%eax
-	shr	\$26,$d1
-	mov	%eax,`16*0+8-64`($ctx)
-
-	mov	\$0x3ffffff,%edx
-	and	$d1#d,%edx
-	mov	%edx,`16*1+8-64`($ctx)
-	lea	(%rdx,%rdx,4),%edx	# *5
-	shr	\$26,$d1
-	mov	%edx,`16*2+8-64`($ctx)
-
-	mov	$h1,%rax
-	shl	\$12,%rax
-	or	$d1,%rax
-	and	\$0x3ffffff,%eax
-	mov	%eax,`16*3+8-64`($ctx)
-	lea	(%rax,%rax,4),%eax	# *5
-	mov	$h1,$d1
-	mov	%eax,`16*4+8-64`($ctx)
-
-	mov	\$0x3ffffff,%edx
-	shr	\$14,$d1
-	and	$d1#d,%edx
-	mov	%edx,`16*5+8-64`($ctx)
-	lea	(%rdx,%rdx,4),%edx	# *5
-	shr	\$26,$d1
-	mov	%edx,`16*6+8-64`($ctx)
-
-	mov	$h2,%rax
-	shl	\$24,%rax
-	or	%rax,$d1
-	mov	$d1#d,`16*7+8-64`($ctx)
-	lea	($d1,$d1,4),$d1		# *5
-	mov	$d1#d,`16*8+8-64`($ctx)
-
-	lea	-48-64($ctx),$ctx	# size [de-]optimization
-	pop %rbp
-	RET
-.size	__poly1305_init_avx,.-__poly1305_init_avx
-___
-
-&declare_function("poly1305_blocks_avx", 32, 4);
-$code.=<<___;
-.cfi_startproc
-	mov	20($ctx),%r8d		# is_base2_26
-	cmp	\$128,$len
-	jae	.Lblocks_avx
-	test	%r8d,%r8d
-	jz	.Lblocks
-
-.Lblocks_avx:
-	and	\$-16,$len
-	jz	.Lno_data_avx
-
-	vzeroupper
-
-	test	%r8d,%r8d
-	jz	.Lbase2_64_avx
-
-	test	\$31,$len
-	jz	.Leven_avx
-
-	push	%rbp
-.cfi_push	%rbp
-	mov 	%rsp,%rbp
-	push	%rbx
-.cfi_push	%rbx
-	push	%r12
-.cfi_push	%r12
-	push	%r13
-.cfi_push	%r13
-	push	%r14
-.cfi_push	%r14
-	push	%r15
-.cfi_push	%r15
-.Lblocks_avx_body:
-
-	mov	$len,%r15		# reassign $len
-
-	mov	0($ctx),$d1		# load hash value
-	mov	8($ctx),$d2
-	mov	16($ctx),$h2#d
-
-	mov	24($ctx),$r0		# load r
-	mov	32($ctx),$s1
-
-	################################# base 2^26 -> base 2^64
-	mov	$d1#d,$h0#d
-	and	\$`-1*(1<<31)`,$d1
-	mov	$d2,$r1			# borrow $r1
-	mov	$d2#d,$h1#d
-	and	\$`-1*(1<<31)`,$d2
-
-	shr	\$6,$d1
-	shl	\$52,$r1
-	add	$d1,$h0
-	shr	\$12,$h1
-	shr	\$18,$d2
-	add	$r1,$h0
-	adc	$d2,$h1
-
-	mov	$h2,$d1
-	shl	\$40,$d1
-	shr	\$24,$h2
-	add	$d1,$h1
-	adc	\$0,$h2			# can be partially reduced...
-
-	mov	\$-4,$d2		# ... so reduce
-	mov	$h2,$d1
-	and	$h2,$d2
-	shr	\$2,$d1
-	and	\$3,$h2
-	add	$d2,$d1			# =*5
-	add	$d1,$h0
-	adc	\$0,$h1
-	adc	\$0,$h2
-
-	mov	$s1,$r1
-	mov	$s1,%rax
-	shr	\$2,$s1
-	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
-
-	add	0($inp),$h0		# accumulate input
-	adc	8($inp),$h1
-	lea	16($inp),$inp
-	adc	$padbit,$h2
-
-	call	__poly1305_block
-
-	test	$padbit,$padbit		# if $padbit is zero,
-	jz	.Lstore_base2_64_avx	# store hash in base 2^64 format
-
-	################################# base 2^64 -> base 2^26
-	mov	$h0,%rax
-	mov	$h0,%rdx
-	shr	\$52,$h0
-	mov	$h1,$r0
-	mov	$h1,$r1
-	shr	\$26,%rdx
-	and	\$0x3ffffff,%rax	# h[0]
-	shl	\$12,$r0
-	and	\$0x3ffffff,%rdx	# h[1]
-	shr	\$14,$h1
-	or	$r0,$h0
-	shl	\$24,$h2
-	and	\$0x3ffffff,$h0		# h[2]
-	shr	\$40,$r1
-	and	\$0x3ffffff,$h1		# h[3]
-	or	$r1,$h2			# h[4]
-
-	sub	\$16,%r15
-	jz	.Lstore_base2_26_avx
-
-	vmovd	%rax#d,$H0
-	vmovd	%rdx#d,$H1
-	vmovd	$h0#d,$H2
-	vmovd	$h1#d,$H3
-	vmovd	$h2#d,$H4
-	jmp	.Lproceed_avx
-
-.align	32
-.Lstore_base2_64_avx:
-	mov	$h0,0($ctx)
-	mov	$h1,8($ctx)
-	mov	$h2,16($ctx)		# note that is_base2_26 is zeroed
-	jmp	.Ldone_avx
-
-.align	16
-.Lstore_base2_26_avx:
-	mov	%rax#d,0($ctx)		# store hash value base 2^26
-	mov	%rdx#d,4($ctx)
-	mov	$h0#d,8($ctx)
-	mov	$h1#d,12($ctx)
-	mov	$h2#d,16($ctx)
-.align	16
-.Ldone_avx:
-	pop 		%r15
-.cfi_restore	%r15
-	pop 		%r14
-.cfi_restore	%r14
-	pop 		%r13
-.cfi_restore	%r13
-	pop 		%r12
-.cfi_restore	%r12
-	pop 		%rbx
-.cfi_restore	%rbx
-	pop 		%rbp
-.cfi_restore	%rbp
-.Lno_data_avx:
-.Lblocks_avx_epilogue:
-	RET
-.cfi_endproc
-
-.align	32
-.Lbase2_64_avx:
-.cfi_startproc
-	push	%rbp
-.cfi_push	%rbp
-	mov 	%rsp,%rbp
-	push	%rbx
-.cfi_push	%rbx
-	push	%r12
-.cfi_push	%r12
-	push	%r13
-.cfi_push	%r13
-	push	%r14
-.cfi_push	%r14
-	push	%r15
-.cfi_push	%r15
-.Lbase2_64_avx_body:
-
-	mov	$len,%r15		# reassign $len
-
-	mov	24($ctx),$r0		# load r
-	mov	32($ctx),$s1
-
-	mov	0($ctx),$h0		# load hash value
-	mov	8($ctx),$h1
-	mov	16($ctx),$h2#d
-
-	mov	$s1,$r1
-	mov	$s1,%rax
-	shr	\$2,$s1
-	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
-
-	test	\$31,$len
-	jz	.Linit_avx
-
-	add	0($inp),$h0		# accumulate input
-	adc	8($inp),$h1
-	lea	16($inp),$inp
-	adc	$padbit,$h2
-	sub	\$16,%r15
-
-	call	__poly1305_block
-
-.Linit_avx:
-	################################# base 2^64 -> base 2^26
-	mov	$h0,%rax
-	mov	$h0,%rdx
-	shr	\$52,$h0
-	mov	$h1,$d1
-	mov	$h1,$d2
-	shr	\$26,%rdx
-	and	\$0x3ffffff,%rax	# h[0]
-	shl	\$12,$d1
-	and	\$0x3ffffff,%rdx	# h[1]
-	shr	\$14,$h1
-	or	$d1,$h0
-	shl	\$24,$h2
-	and	\$0x3ffffff,$h0		# h[2]
-	shr	\$40,$d2
-	and	\$0x3ffffff,$h1		# h[3]
-	or	$d2,$h2			# h[4]
-
-	vmovd	%rax#d,$H0
-	vmovd	%rdx#d,$H1
-	vmovd	$h0#d,$H2
-	vmovd	$h1#d,$H3
-	vmovd	$h2#d,$H4
-	movl	\$1,20($ctx)		# set is_base2_26
-
-	call	__poly1305_init_avx
-
-.Lproceed_avx:
-	mov	%r15,$len
-	pop 		%r15
-.cfi_restore	%r15
-	pop 		%r14
-.cfi_restore	%r14
-	pop 		%r13
-.cfi_restore	%r13
-	pop 		%r12
-.cfi_restore	%r12
-	pop 		%rbx
-.cfi_restore	%rbx
-	pop 		%rbp
-.cfi_restore	%rbp
-.Lbase2_64_avx_epilogue:
-	jmp	.Ldo_avx
-.cfi_endproc
-
-.align	32
-.Leven_avx:
-.cfi_startproc
-	vmovd		4*0($ctx),$H0		# load hash value
-	vmovd		4*1($ctx),$H1
-	vmovd		4*2($ctx),$H2
-	vmovd		4*3($ctx),$H3
-	vmovd		4*4($ctx),$H4
-
-.Ldo_avx:
-___
-$code.=<<___	if (!$win64);
-	lea		8(%rsp),%r10
-.cfi_def_cfa_register	%r10
-	and		\$-32,%rsp
-	sub		\$-8,%rsp
-	lea		-0x58(%rsp),%r11
-	sub		\$0x178,%rsp
-___
-$code.=<<___	if ($win64);
-	lea		-0xf8(%rsp),%r11
-	sub		\$0x218,%rsp
-	vmovdqa		%xmm6,0x50(%r11)
-	vmovdqa		%xmm7,0x60(%r11)
-	vmovdqa		%xmm8,0x70(%r11)
-	vmovdqa		%xmm9,0x80(%r11)
-	vmovdqa		%xmm10,0x90(%r11)
-	vmovdqa		%xmm11,0xa0(%r11)
-	vmovdqa		%xmm12,0xb0(%r11)
-	vmovdqa		%xmm13,0xc0(%r11)
-	vmovdqa		%xmm14,0xd0(%r11)
-	vmovdqa		%xmm15,0xe0(%r11)
-.Ldo_avx_body:
-___
-$code.=<<___;
-	sub		\$64,$len
-	lea		-32($inp),%rax
-	cmovc		%rax,$inp
-
-	vmovdqu		`16*3`($ctx),$D4	# preload r0^2
-	lea		`16*3+64`($ctx),$ctx	# size optimization
-	lea		.Lconst(%rip),%rcx
-
-	################################################################
-	# load input
-	vmovdqu		16*2($inp),$T0
-	vmovdqu		16*3($inp),$T1
-	vmovdqa		64(%rcx),$MASK		# .Lmask26
-
-	vpsrldq		\$6,$T0,$T2		# splat input
-	vpsrldq		\$6,$T1,$T3
-	vpunpckhqdq	$T1,$T0,$T4		# 4
-	vpunpcklqdq	$T1,$T0,$T0		# 0:1
-	vpunpcklqdq	$T3,$T2,$T3		# 2:3
-
-	vpsrlq		\$40,$T4,$T4		# 4
-	vpsrlq		\$26,$T0,$T1
-	vpand		$MASK,$T0,$T0		# 0
-	vpsrlq		\$4,$T3,$T2
-	vpand		$MASK,$T1,$T1		# 1
-	vpsrlq		\$30,$T3,$T3
-	vpand		$MASK,$T2,$T2		# 2
-	vpand		$MASK,$T3,$T3		# 3
-	vpor		32(%rcx),$T4,$T4	# padbit, yes, always
-
-	jbe		.Lskip_loop_avx
-
-	# expand and copy pre-calculated table to stack
-	vmovdqu		`16*1-64`($ctx),$D1
-	vmovdqu		`16*2-64`($ctx),$D2
-	vpshufd		\$0xEE,$D4,$D3		# 34xx -> 3434
-	vpshufd		\$0x44,$D4,$D0		# xx12 -> 1212
-	vmovdqa		$D3,-0x90(%r11)
-	vmovdqa		$D0,0x00(%rsp)
-	vpshufd		\$0xEE,$D1,$D4
-	vmovdqu		`16*3-64`($ctx),$D0
-	vpshufd		\$0x44,$D1,$D1
-	vmovdqa		$D4,-0x80(%r11)
-	vmovdqa		$D1,0x10(%rsp)
-	vpshufd		\$0xEE,$D2,$D3
-	vmovdqu		`16*4-64`($ctx),$D1
-	vpshufd		\$0x44,$D2,$D2
-	vmovdqa		$D3,-0x70(%r11)
-	vmovdqa		$D2,0x20(%rsp)
-	vpshufd		\$0xEE,$D0,$D4
-	vmovdqu		`16*5-64`($ctx),$D2
-	vpshufd		\$0x44,$D0,$D0
-	vmovdqa		$D4,-0x60(%r11)
-	vmovdqa		$D0,0x30(%rsp)
-	vpshufd		\$0xEE,$D1,$D3
-	vmovdqu		`16*6-64`($ctx),$D0
-	vpshufd		\$0x44,$D1,$D1
-	vmovdqa		$D3,-0x50(%r11)
-	vmovdqa		$D1,0x40(%rsp)
-	vpshufd		\$0xEE,$D2,$D4
-	vmovdqu		`16*7-64`($ctx),$D1
-	vpshufd		\$0x44,$D2,$D2
-	vmovdqa		$D4,-0x40(%r11)
-	vmovdqa		$D2,0x50(%rsp)
-	vpshufd		\$0xEE,$D0,$D3
-	vmovdqu		`16*8-64`($ctx),$D2
-	vpshufd		\$0x44,$D0,$D0
-	vmovdqa		$D3,-0x30(%r11)
-	vmovdqa		$D0,0x60(%rsp)
-	vpshufd		\$0xEE,$D1,$D4
-	vpshufd		\$0x44,$D1,$D1
-	vmovdqa		$D4,-0x20(%r11)
-	vmovdqa		$D1,0x70(%rsp)
-	vpshufd		\$0xEE,$D2,$D3
-	 vmovdqa	0x00(%rsp),$D4		# preload r0^2
-	vpshufd		\$0x44,$D2,$D2
-	vmovdqa		$D3,-0x10(%r11)
-	vmovdqa		$D2,0x80(%rsp)
-
-	jmp		.Loop_avx
-
-.align	32
-.Loop_avx:
-	################################################################
-	# ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
-	# ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
-	#   \___________________/
-	# ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
-	# ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
-	#   \___________________/ \____________________/
-	#
-	# Note that we start with inp[2:3]*r^2. This is because it
-	# doesn't depend on reduction in previous iteration.
-	################################################################
-	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
-	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
-	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
-	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
-	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
-	#
-	# though note that $Tx and $Hx are "reversed" in this section,
-	# and $D4 is preloaded with r0^2...
-
-	vpmuludq	$T0,$D4,$D0		# d0 = h0*r0
-	vpmuludq	$T1,$D4,$D1		# d1 = h1*r0
-	  vmovdqa	$H2,0x20(%r11)				# offload hash
-	vpmuludq	$T2,$D4,$D2		# d3 = h2*r0
-	 vmovdqa	0x10(%rsp),$H2		# r1^2
-	vpmuludq	$T3,$D4,$D3		# d3 = h3*r0
-	vpmuludq	$T4,$D4,$D4		# d4 = h4*r0
-
-	  vmovdqa	$H0,0x00(%r11)				#
-	vpmuludq	0x20(%rsp),$T4,$H0	# h4*s1
-	  vmovdqa	$H1,0x10(%r11)				#
-	vpmuludq	$T3,$H2,$H1		# h3*r1
-	vpaddq		$H0,$D0,$D0		# d0 += h4*s1
-	vpaddq		$H1,$D4,$D4		# d4 += h3*r1
-	  vmovdqa	$H3,0x30(%r11)				#
-	vpmuludq	$T2,$H2,$H0		# h2*r1
-	vpmuludq	$T1,$H2,$H1		# h1*r1
-	vpaddq		$H0,$D3,$D3		# d3 += h2*r1
-	 vmovdqa	0x30(%rsp),$H3		# r2^2
-	vpaddq		$H1,$D2,$D2		# d2 += h1*r1
-	  vmovdqa	$H4,0x40(%r11)				#
-	vpmuludq	$T0,$H2,$H2		# h0*r1
-	 vpmuludq	$T2,$H3,$H0		# h2*r2
-	vpaddq		$H2,$D1,$D1		# d1 += h0*r1
-
-	 vmovdqa	0x40(%rsp),$H4		# s2^2
-	vpaddq		$H0,$D4,$D4		# d4 += h2*r2
-	vpmuludq	$T1,$H3,$H1		# h1*r2
-	vpmuludq	$T0,$H3,$H3		# h0*r2
-	vpaddq		$H1,$D3,$D3		# d3 += h1*r2
-	 vmovdqa	0x50(%rsp),$H2		# r3^2
-	vpaddq		$H3,$D2,$D2		# d2 += h0*r2
-	vpmuludq	$T4,$H4,$H0		# h4*s2
-	vpmuludq	$T3,$H4,$H4		# h3*s2
-	vpaddq		$H0,$D1,$D1		# d1 += h4*s2
-	 vmovdqa	0x60(%rsp),$H3		# s3^2
-	vpaddq		$H4,$D0,$D0		# d0 += h3*s2
-
-	 vmovdqa	0x80(%rsp),$H4		# s4^2
-	vpmuludq	$T1,$H2,$H1		# h1*r3
-	vpmuludq	$T0,$H2,$H2		# h0*r3
-	vpaddq		$H1,$D4,$D4		# d4 += h1*r3
-	vpaddq		$H2,$D3,$D3		# d3 += h0*r3
-	vpmuludq	$T4,$H3,$H0		# h4*s3
-	vpmuludq	$T3,$H3,$H1		# h3*s3
-	vpaddq		$H0,$D2,$D2		# d2 += h4*s3
-	 vmovdqu	16*0($inp),$H0				# load input
-	vpaddq		$H1,$D1,$D1		# d1 += h3*s3
-	vpmuludq	$T2,$H3,$H3		# h2*s3
-	 vpmuludq	$T2,$H4,$T2		# h2*s4
-	vpaddq		$H3,$D0,$D0		# d0 += h2*s3
-
-	 vmovdqu	16*1($inp),$H1				#
-	vpaddq		$T2,$D1,$D1		# d1 += h2*s4
-	vpmuludq	$T3,$H4,$T3		# h3*s4
-	vpmuludq	$T4,$H4,$T4		# h4*s4
-	 vpsrldq	\$6,$H0,$H2				# splat input
-	vpaddq		$T3,$D2,$D2		# d2 += h3*s4
-	vpaddq		$T4,$D3,$D3		# d3 += h4*s4
-	 vpsrldq	\$6,$H1,$H3				#
-	vpmuludq	0x70(%rsp),$T0,$T4	# h0*r4
-	vpmuludq	$T1,$H4,$T0		# h1*s4
-	 vpunpckhqdq	$H1,$H0,$H4		# 4
-	vpaddq		$T4,$D4,$D4		# d4 += h0*r4
-	 vmovdqa	-0x90(%r11),$T4		# r0^4
-	vpaddq		$T0,$D0,$D0		# d0 += h1*s4
-
-	vpunpcklqdq	$H1,$H0,$H0		# 0:1
-	vpunpcklqdq	$H3,$H2,$H3		# 2:3
-
-	#vpsrlq		\$40,$H4,$H4		# 4
-	vpsrldq		\$`40/8`,$H4,$H4	# 4
-	vpsrlq		\$26,$H0,$H1
-	vpand		$MASK,$H0,$H0		# 0
-	vpsrlq		\$4,$H3,$H2
-	vpand		$MASK,$H1,$H1		# 1
-	vpand		0(%rcx),$H4,$H4		# .Lmask24
-	vpsrlq		\$30,$H3,$H3
-	vpand		$MASK,$H2,$H2		# 2
-	vpand		$MASK,$H3,$H3		# 3
-	vpor		32(%rcx),$H4,$H4	# padbit, yes, always
-
-	vpaddq		0x00(%r11),$H0,$H0	# add hash value
-	vpaddq		0x10(%r11),$H1,$H1
-	vpaddq		0x20(%r11),$H2,$H2
-	vpaddq		0x30(%r11),$H3,$H3
-	vpaddq		0x40(%r11),$H4,$H4
-
-	lea		16*2($inp),%rax
-	lea		16*4($inp),$inp
-	sub		\$64,$len
-	cmovc		%rax,$inp
-
-	################################################################
-	# Now we accumulate (inp[0:1]+hash)*r^4
-	################################################################
-	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
-	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
-	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
-	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
-	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
-
-	vpmuludq	$H0,$T4,$T0		# h0*r0
-	vpmuludq	$H1,$T4,$T1		# h1*r0
-	vpaddq		$T0,$D0,$D0
-	vpaddq		$T1,$D1,$D1
-	 vmovdqa	-0x80(%r11),$T2		# r1^4
-	vpmuludq	$H2,$T4,$T0		# h2*r0
-	vpmuludq	$H3,$T4,$T1		# h3*r0
-	vpaddq		$T0,$D2,$D2
-	vpaddq		$T1,$D3,$D3
-	vpmuludq	$H4,$T4,$T4		# h4*r0
-	 vpmuludq	-0x70(%r11),$H4,$T0	# h4*s1
-	vpaddq		$T4,$D4,$D4
-
-	vpaddq		$T0,$D0,$D0		# d0 += h4*s1
-	vpmuludq	$H2,$T2,$T1		# h2*r1
-	vpmuludq	$H3,$T2,$T0		# h3*r1
-	vpaddq		$T1,$D3,$D3		# d3 += h2*r1
-	 vmovdqa	-0x60(%r11),$T3		# r2^4
-	vpaddq		$T0,$D4,$D4		# d4 += h3*r1
-	vpmuludq	$H1,$T2,$T1		# h1*r1
-	vpmuludq	$H0,$T2,$T2		# h0*r1
-	vpaddq		$T1,$D2,$D2		# d2 += h1*r1
-	vpaddq		$T2,$D1,$D1		# d1 += h0*r1
-
-	 vmovdqa	-0x50(%r11),$T4		# s2^4
-	vpmuludq	$H2,$T3,$T0		# h2*r2
-	vpmuludq	$H1,$T3,$T1		# h1*r2
-	vpaddq		$T0,$D4,$D4		# d4 += h2*r2
-	vpaddq		$T1,$D3,$D3		# d3 += h1*r2
-	 vmovdqa	-0x40(%r11),$T2		# r3^4
-	vpmuludq	$H0,$T3,$T3		# h0*r2
-	vpmuludq	$H4,$T4,$T0		# h4*s2
-	vpaddq		$T3,$D2,$D2		# d2 += h0*r2
-	vpaddq		$T0,$D1,$D1		# d1 += h4*s2
-	 vmovdqa	-0x30(%r11),$T3		# s3^4
-	vpmuludq	$H3,$T4,$T4		# h3*s2
-	 vpmuludq	$H1,$T2,$T1		# h1*r3
-	vpaddq		$T4,$D0,$D0		# d0 += h3*s2
-
-	 vmovdqa	-0x10(%r11),$T4		# s4^4
-	vpaddq		$T1,$D4,$D4		# d4 += h1*r3
-	vpmuludq	$H0,$T2,$T2		# h0*r3
-	vpmuludq	$H4,$T3,$T0		# h4*s3
-	vpaddq		$T2,$D3,$D3		# d3 += h0*r3
-	vpaddq		$T0,$D2,$D2		# d2 += h4*s3
-	 vmovdqu	16*2($inp),$T0				# load input
-	vpmuludq	$H3,$T3,$T2		# h3*s3
-	vpmuludq	$H2,$T3,$T3		# h2*s3
-	vpaddq		$T2,$D1,$D1		# d1 += h3*s3
-	 vmovdqu	16*3($inp),$T1				#
-	vpaddq		$T3,$D0,$D0		# d0 += h2*s3
-
-	vpmuludq	$H2,$T4,$H2		# h2*s4
-	vpmuludq	$H3,$T4,$H3		# h3*s4
-	 vpsrldq	\$6,$T0,$T2				# splat input
-	vpaddq		$H2,$D1,$D1		# d1 += h2*s4
-	vpmuludq	$H4,$T4,$H4		# h4*s4
-	 vpsrldq	\$6,$T1,$T3				#
-	vpaddq		$H3,$D2,$H2		# h2 = d2 + h3*s4
-	vpaddq		$H4,$D3,$H3		# h3 = d3 + h4*s4
-	vpmuludq	-0x20(%r11),$H0,$H4	# h0*r4
-	vpmuludq	$H1,$T4,$H0
-	 vpunpckhqdq	$T1,$T0,$T4		# 4
-	vpaddq		$H4,$D4,$H4		# h4 = d4 + h0*r4
-	vpaddq		$H0,$D0,$H0		# h0 = d0 + h1*s4
-
-	vpunpcklqdq	$T1,$T0,$T0		# 0:1
-	vpunpcklqdq	$T3,$T2,$T3		# 2:3
-
-	#vpsrlq		\$40,$T4,$T4		# 4
-	vpsrldq		\$`40/8`,$T4,$T4	# 4
-	vpsrlq		\$26,$T0,$T1
-	 vmovdqa	0x00(%rsp),$D4		# preload r0^2
-	vpand		$MASK,$T0,$T0		# 0
-	vpsrlq		\$4,$T3,$T2
-	vpand		$MASK,$T1,$T1		# 1
-	vpand		0(%rcx),$T4,$T4		# .Lmask24
-	vpsrlq		\$30,$T3,$T3
-	vpand		$MASK,$T2,$T2		# 2
-	vpand		$MASK,$T3,$T3		# 3
-	vpor		32(%rcx),$T4,$T4	# padbit, yes, always
-
-	################################################################
-	# lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
-	# and P. Schwabe
-
-	vpsrlq		\$26,$H3,$D3
-	vpand		$MASK,$H3,$H3
-	vpaddq		$D3,$H4,$H4		# h3 -> h4
-
-	vpsrlq		\$26,$H0,$D0
-	vpand		$MASK,$H0,$H0
-	vpaddq		$D0,$D1,$H1		# h0 -> h1
-
-	vpsrlq		\$26,$H4,$D0
-	vpand		$MASK,$H4,$H4
-
-	vpsrlq		\$26,$H1,$D1
-	vpand		$MASK,$H1,$H1
-	vpaddq		$D1,$H2,$H2		# h1 -> h2
-
-	vpaddq		$D0,$H0,$H0
-	vpsllq		\$2,$D0,$D0
-	vpaddq		$D0,$H0,$H0		# h4 -> h0
-
-	vpsrlq		\$26,$H2,$D2
-	vpand		$MASK,$H2,$H2
-	vpaddq		$D2,$H3,$H3		# h2 -> h3
-
-	vpsrlq		\$26,$H0,$D0
-	vpand		$MASK,$H0,$H0
-	vpaddq		$D0,$H1,$H1		# h0 -> h1
-
-	vpsrlq		\$26,$H3,$D3
-	vpand		$MASK,$H3,$H3
-	vpaddq		$D3,$H4,$H4		# h3 -> h4
-
-	ja		.Loop_avx
-
-.Lskip_loop_avx:
-	################################################################
-	# multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
-
-	vpshufd		\$0x10,$D4,$D4		# r0^n, xx12 -> x1x2
-	add		\$32,$len
-	jnz		.Long_tail_avx
-
-	vpaddq		$H2,$T2,$T2
-	vpaddq		$H0,$T0,$T0
-	vpaddq		$H1,$T1,$T1
-	vpaddq		$H3,$T3,$T3
-	vpaddq		$H4,$T4,$T4
-
-.Long_tail_avx:
-	vmovdqa		$H2,0x20(%r11)
-	vmovdqa		$H0,0x00(%r11)
-	vmovdqa		$H1,0x10(%r11)
-	vmovdqa		$H3,0x30(%r11)
-	vmovdqa		$H4,0x40(%r11)
-
-	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
-	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
-	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
-	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
-	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
-
-	vpmuludq	$T2,$D4,$D2		# d2 = h2*r0
-	vpmuludq	$T0,$D4,$D0		# d0 = h0*r0
-	 vpshufd	\$0x10,`16*1-64`($ctx),$H2		# r1^n
-	vpmuludq	$T1,$D4,$D1		# d1 = h1*r0
-	vpmuludq	$T3,$D4,$D3		# d3 = h3*r0
-	vpmuludq	$T4,$D4,$D4		# d4 = h4*r0
-
-	vpmuludq	$T3,$H2,$H0		# h3*r1
-	vpaddq		$H0,$D4,$D4		# d4 += h3*r1
-	 vpshufd	\$0x10,`16*2-64`($ctx),$H3		# s1^n
-	vpmuludq	$T2,$H2,$H1		# h2*r1
-	vpaddq		$H1,$D3,$D3		# d3 += h2*r1
-	 vpshufd	\$0x10,`16*3-64`($ctx),$H4		# r2^n
-	vpmuludq	$T1,$H2,$H0		# h1*r1
-	vpaddq		$H0,$D2,$D2		# d2 += h1*r1
-	vpmuludq	$T0,$H2,$H2		# h0*r1
-	vpaddq		$H2,$D1,$D1		# d1 += h0*r1
-	vpmuludq	$T4,$H3,$H3		# h4*s1
-	vpaddq		$H3,$D0,$D0		# d0 += h4*s1
-
-	 vpshufd	\$0x10,`16*4-64`($ctx),$H2		# s2^n
-	vpmuludq	$T2,$H4,$H1		# h2*r2
-	vpaddq		$H1,$D4,$D4		# d4 += h2*r2
-	vpmuludq	$T1,$H4,$H0		# h1*r2
-	vpaddq		$H0,$D3,$D3		# d3 += h1*r2
-	 vpshufd	\$0x10,`16*5-64`($ctx),$H3		# r3^n
-	vpmuludq	$T0,$H4,$H4		# h0*r2
-	vpaddq		$H4,$D2,$D2		# d2 += h0*r2
-	vpmuludq	$T4,$H2,$H1		# h4*s2
-	vpaddq		$H1,$D1,$D1		# d1 += h4*s2
-	 vpshufd	\$0x10,`16*6-64`($ctx),$H4		# s3^n
-	vpmuludq	$T3,$H2,$H2		# h3*s2
-	vpaddq		$H2,$D0,$D0		# d0 += h3*s2
-
-	vpmuludq	$T1,$H3,$H0		# h1*r3
-	vpaddq		$H0,$D4,$D4		# d4 += h1*r3
-	vpmuludq	$T0,$H3,$H3		# h0*r3
-	vpaddq		$H3,$D3,$D3		# d3 += h0*r3
-	 vpshufd	\$0x10,`16*7-64`($ctx),$H2		# r4^n
-	vpmuludq	$T4,$H4,$H1		# h4*s3
-	vpaddq		$H1,$D2,$D2		# d2 += h4*s3
-	 vpshufd	\$0x10,`16*8-64`($ctx),$H3		# s4^n
-	vpmuludq	$T3,$H4,$H0		# h3*s3
-	vpaddq		$H0,$D1,$D1		# d1 += h3*s3
-	vpmuludq	$T2,$H4,$H4		# h2*s3
-	vpaddq		$H4,$D0,$D0		# d0 += h2*s3
-
-	vpmuludq	$T0,$H2,$H2		# h0*r4
-	vpaddq		$H2,$D4,$D4		# h4 = d4 + h0*r4
-	vpmuludq	$T4,$H3,$H1		# h4*s4
-	vpaddq		$H1,$D3,$D3		# h3 = d3 + h4*s4
-	vpmuludq	$T3,$H3,$H0		# h3*s4
-	vpaddq		$H0,$D2,$D2		# h2 = d2 + h3*s4
-	vpmuludq	$T2,$H3,$H1		# h2*s4
-	vpaddq		$H1,$D1,$D1		# h1 = d1 + h2*s4
-	vpmuludq	$T1,$H3,$H3		# h1*s4
-	vpaddq		$H3,$D0,$D0		# h0 = d0 + h1*s4
-
-	jz		.Lshort_tail_avx
-
-	vmovdqu		16*0($inp),$H0		# load input
-	vmovdqu		16*1($inp),$H1
-
-	vpsrldq		\$6,$H0,$H2		# splat input
-	vpsrldq		\$6,$H1,$H3
-	vpunpckhqdq	$H1,$H0,$H4		# 4
-	vpunpcklqdq	$H1,$H0,$H0		# 0:1
-	vpunpcklqdq	$H3,$H2,$H3		# 2:3
-
-	vpsrlq		\$40,$H4,$H4		# 4
-	vpsrlq		\$26,$H0,$H1
-	vpand		$MASK,$H0,$H0		# 0
-	vpsrlq		\$4,$H3,$H2
-	vpand		$MASK,$H1,$H1		# 1
-	vpsrlq		\$30,$H3,$H3
-	vpand		$MASK,$H2,$H2		# 2
-	vpand		$MASK,$H3,$H3		# 3
-	vpor		32(%rcx),$H4,$H4	# padbit, yes, always
-
-	vpshufd		\$0x32,`16*0-64`($ctx),$T4	# r0^n, 34xx -> x3x4
-	vpaddq		0x00(%r11),$H0,$H0
-	vpaddq		0x10(%r11),$H1,$H1
-	vpaddq		0x20(%r11),$H2,$H2
-	vpaddq		0x30(%r11),$H3,$H3
-	vpaddq		0x40(%r11),$H4,$H4
-
-	################################################################
-	# multiply (inp[0:1]+hash) by r^4:r^3 and accumulate
-
-	vpmuludq	$H0,$T4,$T0		# h0*r0
-	vpaddq		$T0,$D0,$D0		# d0 += h0*r0
-	vpmuludq	$H1,$T4,$T1		# h1*r0
-	vpaddq		$T1,$D1,$D1		# d1 += h1*r0
-	vpmuludq	$H2,$T4,$T0		# h2*r0
-	vpaddq		$T0,$D2,$D2		# d2 += h2*r0
-	 vpshufd	\$0x32,`16*1-64`($ctx),$T2		# r1^n
-	vpmuludq	$H3,$T4,$T1		# h3*r0
-	vpaddq		$T1,$D3,$D3		# d3 += h3*r0
-	vpmuludq	$H4,$T4,$T4		# h4*r0
-	vpaddq		$T4,$D4,$D4		# d4 += h4*r0
-
-	vpmuludq	$H3,$T2,$T0		# h3*r1
-	vpaddq		$T0,$D4,$D4		# d4 += h3*r1
-	 vpshufd	\$0x32,`16*2-64`($ctx),$T3		# s1
-	vpmuludq	$H2,$T2,$T1		# h2*r1
-	vpaddq		$T1,$D3,$D3		# d3 += h2*r1
-	 vpshufd	\$0x32,`16*3-64`($ctx),$T4		# r2
-	vpmuludq	$H1,$T2,$T0		# h1*r1
-	vpaddq		$T0,$D2,$D2		# d2 += h1*r1
-	vpmuludq	$H0,$T2,$T2		# h0*r1
-	vpaddq		$T2,$D1,$D1		# d1 += h0*r1
-	vpmuludq	$H4,$T3,$T3		# h4*s1
-	vpaddq		$T3,$D0,$D0		# d0 += h4*s1
-
-	 vpshufd	\$0x32,`16*4-64`($ctx),$T2		# s2
-	vpmuludq	$H2,$T4,$T1		# h2*r2
-	vpaddq		$T1,$D4,$D4		# d4 += h2*r2
-	vpmuludq	$H1,$T4,$T0		# h1*r2
-	vpaddq		$T0,$D3,$D3		# d3 += h1*r2
-	 vpshufd	\$0x32,`16*5-64`($ctx),$T3		# r3
-	vpmuludq	$H0,$T4,$T4		# h0*r2
-	vpaddq		$T4,$D2,$D2		# d2 += h0*r2
-	vpmuludq	$H4,$T2,$T1		# h4*s2
-	vpaddq		$T1,$D1,$D1		# d1 += h4*s2
-	 vpshufd	\$0x32,`16*6-64`($ctx),$T4		# s3
-	vpmuludq	$H3,$T2,$T2		# h3*s2
-	vpaddq		$T2,$D0,$D0		# d0 += h3*s2
-
-	vpmuludq	$H1,$T3,$T0		# h1*r3
-	vpaddq		$T0,$D4,$D4		# d4 += h1*r3
-	vpmuludq	$H0,$T3,$T3		# h0*r3
-	vpaddq		$T3,$D3,$D3		# d3 += h0*r3
-	 vpshufd	\$0x32,`16*7-64`($ctx),$T2		# r4
-	vpmuludq	$H4,$T4,$T1		# h4*s3
-	vpaddq		$T1,$D2,$D2		# d2 += h4*s3
-	 vpshufd	\$0x32,`16*8-64`($ctx),$T3		# s4
-	vpmuludq	$H3,$T4,$T0		# h3*s3
-	vpaddq		$T0,$D1,$D1		# d1 += h3*s3
-	vpmuludq	$H2,$T4,$T4		# h2*s3
-	vpaddq		$T4,$D0,$D0		# d0 += h2*s3
-
-	vpmuludq	$H0,$T2,$T2		# h0*r4
-	vpaddq		$T2,$D4,$D4		# d4 += h0*r4
-	vpmuludq	$H4,$T3,$T1		# h4*s4
-	vpaddq		$T1,$D3,$D3		# d3 += h4*s4
-	vpmuludq	$H3,$T3,$T0		# h3*s4
-	vpaddq		$T0,$D2,$D2		# d2 += h3*s4
-	vpmuludq	$H2,$T3,$T1		# h2*s4
-	vpaddq		$T1,$D1,$D1		# d1 += h2*s4
-	vpmuludq	$H1,$T3,$T3		# h1*s4
-	vpaddq		$T3,$D0,$D0		# d0 += h1*s4
-
-.Lshort_tail_avx:
-	################################################################
-	# horizontal addition
-
-	vpsrldq		\$8,$D4,$T4
-	vpsrldq		\$8,$D3,$T3
-	vpsrldq		\$8,$D1,$T1
-	vpsrldq		\$8,$D0,$T0
-	vpsrldq		\$8,$D2,$T2
-	vpaddq		$T3,$D3,$D3
-	vpaddq		$T4,$D4,$D4
-	vpaddq		$T0,$D0,$D0
-	vpaddq		$T1,$D1,$D1
-	vpaddq		$T2,$D2,$D2
-
-	################################################################
-	# lazy reduction
-
-	vpsrlq		\$26,$D3,$H3
-	vpand		$MASK,$D3,$D3
-	vpaddq		$H3,$D4,$D4		# h3 -> h4
-
-	vpsrlq		\$26,$D0,$H0
-	vpand		$MASK,$D0,$D0
-	vpaddq		$H0,$D1,$D1		# h0 -> h1
-
-	vpsrlq		\$26,$D4,$H4
-	vpand		$MASK,$D4,$D4
-
-	vpsrlq		\$26,$D1,$H1
-	vpand		$MASK,$D1,$D1
-	vpaddq		$H1,$D2,$D2		# h1 -> h2
-
-	vpaddq		$H4,$D0,$D0
-	vpsllq		\$2,$H4,$H4
-	vpaddq		$H4,$D0,$D0		# h4 -> h0
-
-	vpsrlq		\$26,$D2,$H2
-	vpand		$MASK,$D2,$D2
-	vpaddq		$H2,$D3,$D3		# h2 -> h3
-
-	vpsrlq		\$26,$D0,$H0
-	vpand		$MASK,$D0,$D0
-	vpaddq		$H0,$D1,$D1		# h0 -> h1
-
-	vpsrlq		\$26,$D3,$H3
-	vpand		$MASK,$D3,$D3
-	vpaddq		$H3,$D4,$D4		# h3 -> h4
-
-	vmovd		$D0,`4*0-48-64`($ctx)	# save partially reduced
-	vmovd		$D1,`4*1-48-64`($ctx)
-	vmovd		$D2,`4*2-48-64`($ctx)
-	vmovd		$D3,`4*3-48-64`($ctx)
-	vmovd		$D4,`4*4-48-64`($ctx)
-___
-$code.=<<___	if ($win64);
-	vmovdqa		0x50(%r11),%xmm6
-	vmovdqa		0x60(%r11),%xmm7
-	vmovdqa		0x70(%r11),%xmm8
-	vmovdqa		0x80(%r11),%xmm9
-	vmovdqa		0x90(%r11),%xmm10
-	vmovdqa		0xa0(%r11),%xmm11
-	vmovdqa		0xb0(%r11),%xmm12
-	vmovdqa		0xc0(%r11),%xmm13
-	vmovdqa		0xd0(%r11),%xmm14
-	vmovdqa		0xe0(%r11),%xmm15
-	lea		0xf8(%r11),%rsp
-.Ldo_avx_epilogue:
-___
-$code.=<<___	if (!$win64);
-	lea		-8(%r10),%rsp
-.cfi_def_cfa_register	%rsp
-___
-$code.=<<___;
-	vzeroupper
-	RET
-.cfi_endproc
-___
-&end_function("poly1305_blocks_avx");
-
-&declare_function("poly1305_emit_avx", 32, 3);
-$code.=<<___;
-	cmpl	\$0,20($ctx)	# is_base2_26?
-	je	.Lemit
-
-	mov	0($ctx),%eax	# load hash value base 2^26
-	mov	4($ctx),%ecx
-	mov	8($ctx),%r8d
-	mov	12($ctx),%r11d
-	mov	16($ctx),%r10d
-
-	shl	\$26,%rcx	# base 2^26 -> base 2^64
-	mov	%r8,%r9
-	shl	\$52,%r8
-	add	%rcx,%rax
-	shr	\$12,%r9
-	add	%rax,%r8	# h0
-	adc	\$0,%r9
-
-	shl	\$14,%r11
-	mov	%r10,%rax
-	shr	\$24,%r10
-	add	%r11,%r9
-	shl	\$40,%rax
-	add	%rax,%r9	# h1
-	adc	\$0,%r10	# h2
-
-	mov	%r10,%rax	# could be partially reduced, so reduce
-	mov	%r10,%rcx
-	and	\$3,%r10
-	shr	\$2,%rax
-	and	\$-4,%rcx
-	add	%rcx,%rax
-	add	%rax,%r8
-	adc	\$0,%r9
-	adc	\$0,%r10
-
-	mov	%r8,%rax
-	add	\$5,%r8		# compare to modulus
-	mov	%r9,%rcx
-	adc	\$0,%r9
-	adc	\$0,%r10
-	shr	\$2,%r10	# did 130-bit value overflow?
-	cmovnz	%r8,%rax
-	cmovnz	%r9,%rcx
-
-	add	0($nonce),%rax	# accumulate nonce
-	adc	8($nonce),%rcx
-	mov	%rax,0($mac)	# write result
-	mov	%rcx,8($mac)
-
-	RET
-___
-&end_function("poly1305_emit_avx");
-
-if ($avx>1) {
-
-my ($H0,$H1,$H2,$H3,$H4, $MASK, $T4,$T0,$T1,$T2,$T3, $D0,$D1,$D2,$D3,$D4) =
-    map("%ymm$_",(0..15));
-my $S4=$MASK;
-
-sub poly1305_blocks_avxN {
-	my ($avx512) = @_;
-	my $suffix = $avx512 ? "_avx512" : "";
-$code.=<<___;
-.cfi_startproc
-	mov	20($ctx),%r8d		# is_base2_26
-	cmp	\$128,$len
-	jae	.Lblocks_avx2$suffix
-	test	%r8d,%r8d
-	jz	.Lblocks
-
-.Lblocks_avx2$suffix:
-	and	\$-16,$len
-	jz	.Lno_data_avx2$suffix
-
-	vzeroupper
-
-	test	%r8d,%r8d
-	jz	.Lbase2_64_avx2$suffix
-
-	test	\$63,$len
-	jz	.Leven_avx2$suffix
-
-	push	%rbp
-.cfi_push	%rbp
-	mov 	%rsp,%rbp
-	push	%rbx
-.cfi_push	%rbx
-	push	%r12
-.cfi_push	%r12
-	push	%r13
-.cfi_push	%r13
-	push	%r14
-.cfi_push	%r14
-	push	%r15
-.cfi_push	%r15
-.Lblocks_avx2_body$suffix:
-
-	mov	$len,%r15		# reassign $len
-
-	mov	0($ctx),$d1		# load hash value
-	mov	8($ctx),$d2
-	mov	16($ctx),$h2#d
-
-	mov	24($ctx),$r0		# load r
-	mov	32($ctx),$s1
-
-	################################# base 2^26 -> base 2^64
-	mov	$d1#d,$h0#d
-	and	\$`-1*(1<<31)`,$d1
-	mov	$d2,$r1			# borrow $r1
-	mov	$d2#d,$h1#d
-	and	\$`-1*(1<<31)`,$d2
-
-	shr	\$6,$d1
-	shl	\$52,$r1
-	add	$d1,$h0
-	shr	\$12,$h1
-	shr	\$18,$d2
-	add	$r1,$h0
-	adc	$d2,$h1
-
-	mov	$h2,$d1
-	shl	\$40,$d1
-	shr	\$24,$h2
-	add	$d1,$h1
-	adc	\$0,$h2			# can be partially reduced...
-
-	mov	\$-4,$d2		# ... so reduce
-	mov	$h2,$d1
-	and	$h2,$d2
-	shr	\$2,$d1
-	and	\$3,$h2
-	add	$d2,$d1			# =*5
-	add	$d1,$h0
-	adc	\$0,$h1
-	adc	\$0,$h2
-
-	mov	$s1,$r1
-	mov	$s1,%rax
-	shr	\$2,$s1
-	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
-
-.Lbase2_26_pre_avx2$suffix:
-	add	0($inp),$h0		# accumulate input
-	adc	8($inp),$h1
-	lea	16($inp),$inp
-	adc	$padbit,$h2
-	sub	\$16,%r15
-
-	call	__poly1305_block
-	mov	$r1,%rax
-
-	test	\$63,%r15
-	jnz	.Lbase2_26_pre_avx2$suffix
-
-	test	$padbit,$padbit		# if $padbit is zero,
-	jz	.Lstore_base2_64_avx2$suffix	# store hash in base 2^64 format
-
-	################################# base 2^64 -> base 2^26
-	mov	$h0,%rax
-	mov	$h0,%rdx
-	shr	\$52,$h0
-	mov	$h1,$r0
-	mov	$h1,$r1
-	shr	\$26,%rdx
-	and	\$0x3ffffff,%rax	# h[0]
-	shl	\$12,$r0
-	and	\$0x3ffffff,%rdx	# h[1]
-	shr	\$14,$h1
-	or	$r0,$h0
-	shl	\$24,$h2
-	and	\$0x3ffffff,$h0		# h[2]
-	shr	\$40,$r1
-	and	\$0x3ffffff,$h1		# h[3]
-	or	$r1,$h2			# h[4]
-
-	test	%r15,%r15
-	jz	.Lstore_base2_26_avx2$suffix
-
-	vmovd	%rax#d,%x#$H0
-	vmovd	%rdx#d,%x#$H1
-	vmovd	$h0#d,%x#$H2
-	vmovd	$h1#d,%x#$H3
-	vmovd	$h2#d,%x#$H4
-	jmp	.Lproceed_avx2$suffix
-
-.align	32
-.Lstore_base2_64_avx2$suffix:
-	mov	$h0,0($ctx)
-	mov	$h1,8($ctx)
-	mov	$h2,16($ctx)		# note that is_base2_26 is zeroed
-	jmp	.Ldone_avx2$suffix
-
-.align	16
-.Lstore_base2_26_avx2$suffix:
-	mov	%rax#d,0($ctx)		# store hash value base 2^26
-	mov	%rdx#d,4($ctx)
-	mov	$h0#d,8($ctx)
-	mov	$h1#d,12($ctx)
-	mov	$h2#d,16($ctx)
-.align	16
-.Ldone_avx2$suffix:
-	pop 		%r15
-.cfi_restore	%r15
-	pop 		%r14
-.cfi_restore	%r14
-	pop 		%r13
-.cfi_restore	%r13
-	pop 		%r12
-.cfi_restore	%r12
-	pop 		%rbx
-.cfi_restore	%rbx
-	pop 		%rbp
-.cfi_restore 	%rbp
-.Lno_data_avx2$suffix:
-.Lblocks_avx2_epilogue$suffix:
-	RET
-.cfi_endproc
-
-.align	32
-.Lbase2_64_avx2$suffix:
-.cfi_startproc
-	push	%rbp
-.cfi_push	%rbp
-	mov 	%rsp,%rbp
-	push	%rbx
-.cfi_push	%rbx
-	push	%r12
-.cfi_push	%r12
-	push	%r13
-.cfi_push	%r13
-	push	%r14
-.cfi_push	%r14
-	push	%r15
-.cfi_push	%r15
-.Lbase2_64_avx2_body$suffix:
-
-	mov	$len,%r15		# reassign $len
-
-	mov	24($ctx),$r0		# load r
-	mov	32($ctx),$s1
-
-	mov	0($ctx),$h0		# load hash value
-	mov	8($ctx),$h1
-	mov	16($ctx),$h2#d
-
-	mov	$s1,$r1
-	mov	$s1,%rax
-	shr	\$2,$s1
-	add	$r1,$s1			# s1 = r1 + (r1 >> 2)
-
-	test	\$63,$len
-	jz	.Linit_avx2$suffix
-
-.Lbase2_64_pre_avx2$suffix:
-	add	0($inp),$h0		# accumulate input
-	adc	8($inp),$h1
-	lea	16($inp),$inp
-	adc	$padbit,$h2
-	sub	\$16,%r15
-
-	call	__poly1305_block
-	mov	$r1,%rax
-
-	test	\$63,%r15
-	jnz	.Lbase2_64_pre_avx2$suffix
-
-.Linit_avx2$suffix:
-	################################# base 2^64 -> base 2^26
-	mov	$h0,%rax
-	mov	$h0,%rdx
-	shr	\$52,$h0
-	mov	$h1,$d1
-	mov	$h1,$d2
-	shr	\$26,%rdx
-	and	\$0x3ffffff,%rax	# h[0]
-	shl	\$12,$d1
-	and	\$0x3ffffff,%rdx	# h[1]
-	shr	\$14,$h1
-	or	$d1,$h0
-	shl	\$24,$h2
-	and	\$0x3ffffff,$h0		# h[2]
-	shr	\$40,$d2
-	and	\$0x3ffffff,$h1		# h[3]
-	or	$d2,$h2			# h[4]
-
-	vmovd	%rax#d,%x#$H0
-	vmovd	%rdx#d,%x#$H1
-	vmovd	$h0#d,%x#$H2
-	vmovd	$h1#d,%x#$H3
-	vmovd	$h2#d,%x#$H4
-	movl	\$1,20($ctx)		# set is_base2_26
-
-	call	__poly1305_init_avx
-
-.Lproceed_avx2$suffix:
-	mov	%r15,$len			# restore $len
-___
-$code.=<<___ if (!$kernel);
-	mov	OPENSSL_ia32cap_P+8(%rip),%r9d
-	mov	\$`(1<<31|1<<30|1<<16)`,%r11d
-___
-$code.=<<___;
-	pop 		%r15
-.cfi_restore	%r15
-	pop 		%r14
-.cfi_restore	%r14
-	pop 		%r13
-.cfi_restore	%r13
-	pop 		%r12
-.cfi_restore	%r12
-	pop 		%rbx
-.cfi_restore	%rbx
-	pop 		%rbp
-.cfi_restore 	%rbp
-.Lbase2_64_avx2_epilogue$suffix:
-	jmp	.Ldo_avx2$suffix
-.cfi_endproc
-
-.align	32
-.Leven_avx2$suffix:
-.cfi_startproc
-___
-$code.=<<___ if (!$kernel);
-	mov		OPENSSL_ia32cap_P+8(%rip),%r9d
-___
-$code.=<<___;
-	vmovd		4*0($ctx),%x#$H0	# load hash value base 2^26
-	vmovd		4*1($ctx),%x#$H1
-	vmovd		4*2($ctx),%x#$H2
-	vmovd		4*3($ctx),%x#$H3
-	vmovd		4*4($ctx),%x#$H4
-
-.Ldo_avx2$suffix:
-___
-$code.=<<___		if (!$kernel && $avx>2);
-	cmp		\$512,$len
-	jb		.Lskip_avx512
-	and		%r11d,%r9d
-	test		\$`1<<16`,%r9d		# check for AVX512F
-	jnz		.Lblocks_avx512
-.Lskip_avx512$suffix:
-___
-$code.=<<___ if ($avx > 2 && $avx512 && $kernel);
-	cmp		\$512,$len
-	jae		.Lblocks_avx512
-___
-$code.=<<___	if (!$win64);
-	lea		8(%rsp),%r10
-.cfi_def_cfa_register	%r10
-	sub		\$0x128,%rsp
-___
-$code.=<<___	if ($win64);
-	lea		8(%rsp),%r10
-	sub		\$0x1c8,%rsp
-	vmovdqa		%xmm6,-0xb0(%r10)
-	vmovdqa		%xmm7,-0xa0(%r10)
-	vmovdqa		%xmm8,-0x90(%r10)
-	vmovdqa		%xmm9,-0x80(%r10)
-	vmovdqa		%xmm10,-0x70(%r10)
-	vmovdqa		%xmm11,-0x60(%r10)
-	vmovdqa		%xmm12,-0x50(%r10)
-	vmovdqa		%xmm13,-0x40(%r10)
-	vmovdqa		%xmm14,-0x30(%r10)
-	vmovdqa		%xmm15,-0x20(%r10)
-.Ldo_avx2_body$suffix:
-___
-$code.=<<___;
-	lea		.Lconst(%rip),%rcx
-	lea		48+64($ctx),$ctx	# size optimization
-	vmovdqa		96(%rcx),$T0		# .Lpermd_avx2
-
-	# expand and copy pre-calculated table to stack
-	vmovdqu		`16*0-64`($ctx),%x#$T2
-	and		\$-512,%rsp
-	vmovdqu		`16*1-64`($ctx),%x#$T3
-	vmovdqu		`16*2-64`($ctx),%x#$T4
-	vmovdqu		`16*3-64`($ctx),%x#$D0
-	vmovdqu		`16*4-64`($ctx),%x#$D1
-	vmovdqu		`16*5-64`($ctx),%x#$D2
-	lea		0x90(%rsp),%rax		# size optimization
-	vmovdqu		`16*6-64`($ctx),%x#$D3
-	vpermd		$T2,$T0,$T2		# 00003412 -> 14243444
-	vmovdqu		`16*7-64`($ctx),%x#$D4
-	vpermd		$T3,$T0,$T3
-	vmovdqu		`16*8-64`($ctx),%x#$MASK
-	vpermd		$T4,$T0,$T4
-	vmovdqa		$T2,0x00(%rsp)
-	vpermd		$D0,$T0,$D0
-	vmovdqa		$T3,0x20-0x90(%rax)
-	vpermd		$D1,$T0,$D1
-	vmovdqa		$T4,0x40-0x90(%rax)
-	vpermd		$D2,$T0,$D2
-	vmovdqa		$D0,0x60-0x90(%rax)
-	vpermd		$D3,$T0,$D3
-	vmovdqa		$D1,0x80-0x90(%rax)
-	vpermd		$D4,$T0,$D4
-	vmovdqa		$D2,0xa0-0x90(%rax)
-	vpermd		$MASK,$T0,$MASK
-	vmovdqa		$D3,0xc0-0x90(%rax)
-	vmovdqa		$D4,0xe0-0x90(%rax)
-	vmovdqa		$MASK,0x100-0x90(%rax)
-	vmovdqa		64(%rcx),$MASK		# .Lmask26
-
-	################################################################
-	# load input
-	vmovdqu		16*0($inp),%x#$T0
-	vmovdqu		16*1($inp),%x#$T1
-	vinserti128	\$1,16*2($inp),$T0,$T0
-	vinserti128	\$1,16*3($inp),$T1,$T1
-	lea		16*4($inp),$inp
-
-	vpsrldq		\$6,$T0,$T2		# splat input
-	vpsrldq		\$6,$T1,$T3
-	vpunpckhqdq	$T1,$T0,$T4		# 4
-	vpunpcklqdq	$T3,$T2,$T2		# 2:3
-	vpunpcklqdq	$T1,$T0,$T0		# 0:1
-
-	vpsrlq		\$30,$T2,$T3
-	vpsrlq		\$4,$T2,$T2
-	vpsrlq		\$26,$T0,$T1
-	vpsrlq		\$40,$T4,$T4		# 4
-	vpand		$MASK,$T2,$T2		# 2
-	vpand		$MASK,$T0,$T0		# 0
-	vpand		$MASK,$T1,$T1		# 1
-	vpand		$MASK,$T3,$T3		# 3
-	vpor		32(%rcx),$T4,$T4	# padbit, yes, always
-
-	vpaddq		$H2,$T2,$H2		# accumulate input
-	sub		\$64,$len
-	jz		.Ltail_avx2$suffix
-	jmp		.Loop_avx2$suffix
-
-.align	32
-.Loop_avx2$suffix:
-	################################################################
-	# ((inp[0]*r^4+inp[4])*r^4+inp[ 8])*r^4
-	# ((inp[1]*r^4+inp[5])*r^4+inp[ 9])*r^3
-	# ((inp[2]*r^4+inp[6])*r^4+inp[10])*r^2
-	# ((inp[3]*r^4+inp[7])*r^4+inp[11])*r^1
-	#   \________/\__________/
-	################################################################
-	#vpaddq		$H2,$T2,$H2		# accumulate input
-	vpaddq		$H0,$T0,$H0
-	vmovdqa		`32*0`(%rsp),$T0	# r0^4
-	vpaddq		$H1,$T1,$H1
-	vmovdqa		`32*1`(%rsp),$T1	# r1^4
-	vpaddq		$H3,$T3,$H3
-	vmovdqa		`32*3`(%rsp),$T2	# r2^4
-	vpaddq		$H4,$T4,$H4
-	vmovdqa		`32*6-0x90`(%rax),$T3	# s3^4
-	vmovdqa		`32*8-0x90`(%rax),$S4	# s4^4
-
-	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
-	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
-	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
-	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
-	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
-	#
-	# however, as h2 is "chronologically" first one available pull
-	# corresponding operations up, so it's
-	#
-	# d4 = h2*r2   + h4*r0 + h3*r1             + h1*r3   + h0*r4
-	# d3 = h2*r1   + h3*r0           + h1*r2   + h0*r3   + h4*5*r4
-	# d2 = h2*r0           + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
-	# d1 = h2*5*r4 + h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3
-	# d0 = h2*5*r3 + h0*r0 + h4*5*r1 + h3*5*r2           + h1*5*r4
-
-	vpmuludq	$H2,$T0,$D2		# d2 = h2*r0
-	vpmuludq	$H2,$T1,$D3		# d3 = h2*r1
-	vpmuludq	$H2,$T2,$D4		# d4 = h2*r2
-	vpmuludq	$H2,$T3,$D0		# d0 = h2*s3
-	vpmuludq	$H2,$S4,$D1		# d1 = h2*s4
-
-	vpmuludq	$H0,$T1,$T4		# h0*r1
-	vpmuludq	$H1,$T1,$H2		# h1*r1, borrow $H2 as temp
-	vpaddq		$T4,$D1,$D1		# d1 += h0*r1
-	vpaddq		$H2,$D2,$D2		# d2 += h1*r1
-	vpmuludq	$H3,$T1,$T4		# h3*r1
-	vpmuludq	`32*2`(%rsp),$H4,$H2	# h4*s1
-	vpaddq		$T4,$D4,$D4		# d4 += h3*r1
-	vpaddq		$H2,$D0,$D0		# d0 += h4*s1
-	 vmovdqa	`32*4-0x90`(%rax),$T1	# s2
-
-	vpmuludq	$H0,$T0,$T4		# h0*r0
-	vpmuludq	$H1,$T0,$H2		# h1*r0
-	vpaddq		$T4,$D0,$D0		# d0 += h0*r0
-	vpaddq		$H2,$D1,$D1		# d1 += h1*r0
-	vpmuludq	$H3,$T0,$T4		# h3*r0
-	vpmuludq	$H4,$T0,$H2		# h4*r0
-	 vmovdqu	16*0($inp),%x#$T0	# load input
-	vpaddq		$T4,$D3,$D3		# d3 += h3*r0
-	vpaddq		$H2,$D4,$D4		# d4 += h4*r0
-	 vinserti128	\$1,16*2($inp),$T0,$T0
-
-	vpmuludq	$H3,$T1,$T4		# h3*s2
-	vpmuludq	$H4,$T1,$H2		# h4*s2
-	 vmovdqu	16*1($inp),%x#$T1
-	vpaddq		$T4,$D0,$D0		# d0 += h3*s2
-	vpaddq		$H2,$D1,$D1		# d1 += h4*s2
-	 vmovdqa	`32*5-0x90`(%rax),$H2	# r3
-	vpmuludq	$H1,$T2,$T4		# h1*r2
-	vpmuludq	$H0,$T2,$T2		# h0*r2
-	vpaddq		$T4,$D3,$D3		# d3 += h1*r2
-	vpaddq		$T2,$D2,$D2		# d2 += h0*r2
-	 vinserti128	\$1,16*3($inp),$T1,$T1
-	 lea		16*4($inp),$inp
-
-	vpmuludq	$H1,$H2,$T4		# h1*r3
-	vpmuludq	$H0,$H2,$H2		# h0*r3
-	 vpsrldq	\$6,$T0,$T2		# splat input
-	vpaddq		$T4,$D4,$D4		# d4 += h1*r3
-	vpaddq		$H2,$D3,$D3		# d3 += h0*r3
-	vpmuludq	$H3,$T3,$T4		# h3*s3
-	vpmuludq	$H4,$T3,$H2		# h4*s3
-	 vpsrldq	\$6,$T1,$T3
-	vpaddq		$T4,$D1,$D1		# d1 += h3*s3
-	vpaddq		$H2,$D2,$D2		# d2 += h4*s3
-	 vpunpckhqdq	$T1,$T0,$T4		# 4
-
-	vpmuludq	$H3,$S4,$H3		# h3*s4
-	vpmuludq	$H4,$S4,$H4		# h4*s4
-	 vpunpcklqdq	$T1,$T0,$T0		# 0:1
-	vpaddq		$H3,$D2,$H2		# h2 = d2 + h3*r4
-	vpaddq		$H4,$D3,$H3		# h3 = d3 + h4*r4
-	 vpunpcklqdq	$T3,$T2,$T3		# 2:3
-	vpmuludq	`32*7-0x90`(%rax),$H0,$H4	# h0*r4
-	vpmuludq	$H1,$S4,$H0		# h1*s4
-	vmovdqa		64(%rcx),$MASK		# .Lmask26
-	vpaddq		$H4,$D4,$H4		# h4 = d4 + h0*r4
-	vpaddq		$H0,$D0,$H0		# h0 = d0 + h1*s4
-
-	################################################################
-	# lazy reduction (interleaved with tail of input splat)
-
-	vpsrlq		\$26,$H3,$D3
-	vpand		$MASK,$H3,$H3
-	vpaddq		$D3,$H4,$H4		# h3 -> h4
-
-	vpsrlq		\$26,$H0,$D0
-	vpand		$MASK,$H0,$H0
-	vpaddq		$D0,$D1,$H1		# h0 -> h1
-
-	vpsrlq		\$26,$H4,$D4
-	vpand		$MASK,$H4,$H4
-
-	 vpsrlq		\$4,$T3,$T2
-
-	vpsrlq		\$26,$H1,$D1
-	vpand		$MASK,$H1,$H1
-	vpaddq		$D1,$H2,$H2		# h1 -> h2
-
-	vpaddq		$D4,$H0,$H0
-	vpsllq		\$2,$D4,$D4
-	vpaddq		$D4,$H0,$H0		# h4 -> h0
-
-	 vpand		$MASK,$T2,$T2		# 2
-	 vpsrlq		\$26,$T0,$T1
-
-	vpsrlq		\$26,$H2,$D2
-	vpand		$MASK,$H2,$H2
-	vpaddq		$D2,$H3,$H3		# h2 -> h3
-
-	 vpaddq		$T2,$H2,$H2		# modulo-scheduled
-	 vpsrlq		\$30,$T3,$T3
-
-	vpsrlq		\$26,$H0,$D0
-	vpand		$MASK,$H0,$H0
-	vpaddq		$D0,$H1,$H1		# h0 -> h1
-
-	 vpsrlq		\$40,$T4,$T4		# 4
-
-	vpsrlq		\$26,$H3,$D3
-	vpand		$MASK,$H3,$H3
-	vpaddq		$D3,$H4,$H4		# h3 -> h4
-
-	 vpand		$MASK,$T0,$T0		# 0
-	 vpand		$MASK,$T1,$T1		# 1
-	 vpand		$MASK,$T3,$T3		# 3
-	 vpor		32(%rcx),$T4,$T4	# padbit, yes, always
-
-	sub		\$64,$len
-	jnz		.Loop_avx2$suffix
-
-	.byte		0x66,0x90
-.Ltail_avx2$suffix:
-	################################################################
-	# while above multiplications were by r^4 in all lanes, in last
-	# iteration we multiply least significant lane by r^4 and most
-	# significant one by r, so copy of above except that references
-	# to the precomputed table are displaced by 4...
-
-	#vpaddq		$H2,$T2,$H2		# accumulate input
-	vpaddq		$H0,$T0,$H0
-	vmovdqu		`32*0+4`(%rsp),$T0	# r0^4
-	vpaddq		$H1,$T1,$H1
-	vmovdqu		`32*1+4`(%rsp),$T1	# r1^4
-	vpaddq		$H3,$T3,$H3
-	vmovdqu		`32*3+4`(%rsp),$T2	# r2^4
-	vpaddq		$H4,$T4,$H4
-	vmovdqu		`32*6+4-0x90`(%rax),$T3	# s3^4
-	vmovdqu		`32*8+4-0x90`(%rax),$S4	# s4^4
-
-	vpmuludq	$H2,$T0,$D2		# d2 = h2*r0
-	vpmuludq	$H2,$T1,$D3		# d3 = h2*r1
-	vpmuludq	$H2,$T2,$D4		# d4 = h2*r2
-	vpmuludq	$H2,$T3,$D0		# d0 = h2*s3
-	vpmuludq	$H2,$S4,$D1		# d1 = h2*s4
-
-	vpmuludq	$H0,$T1,$T4		# h0*r1
-	vpmuludq	$H1,$T1,$H2		# h1*r1
-	vpaddq		$T4,$D1,$D1		# d1 += h0*r1
-	vpaddq		$H2,$D2,$D2		# d2 += h1*r1
-	vpmuludq	$H3,$T1,$T4		# h3*r1
-	vpmuludq	`32*2+4`(%rsp),$H4,$H2	# h4*s1
-	vpaddq		$T4,$D4,$D4		# d4 += h3*r1
-	vpaddq		$H2,$D0,$D0		# d0 += h4*s1
-
-	vpmuludq	$H0,$T0,$T4		# h0*r0
-	vpmuludq	$H1,$T0,$H2		# h1*r0
-	vpaddq		$T4,$D0,$D0		# d0 += h0*r0
-	 vmovdqu	`32*4+4-0x90`(%rax),$T1	# s2
-	vpaddq		$H2,$D1,$D1		# d1 += h1*r0
-	vpmuludq	$H3,$T0,$T4		# h3*r0
-	vpmuludq	$H4,$T0,$H2		# h4*r0
-	vpaddq		$T4,$D3,$D3		# d3 += h3*r0
-	vpaddq		$H2,$D4,$D4		# d4 += h4*r0
-
-	vpmuludq	$H3,$T1,$T4		# h3*s2
-	vpmuludq	$H4,$T1,$H2		# h4*s2
-	vpaddq		$T4,$D0,$D0		# d0 += h3*s2
-	vpaddq		$H2,$D1,$D1		# d1 += h4*s2
-	 vmovdqu	`32*5+4-0x90`(%rax),$H2	# r3
-	vpmuludq	$H1,$T2,$T4		# h1*r2
-	vpmuludq	$H0,$T2,$T2		# h0*r2
-	vpaddq		$T4,$D3,$D3		# d3 += h1*r2
-	vpaddq		$T2,$D2,$D2		# d2 += h0*r2
-
-	vpmuludq	$H1,$H2,$T4		# h1*r3
-	vpmuludq	$H0,$H2,$H2		# h0*r3
-	vpaddq		$T4,$D4,$D4		# d4 += h1*r3
-	vpaddq		$H2,$D3,$D3		# d3 += h0*r3
-	vpmuludq	$H3,$T3,$T4		# h3*s3
-	vpmuludq	$H4,$T3,$H2		# h4*s3
-	vpaddq		$T4,$D1,$D1		# d1 += h3*s3
-	vpaddq		$H2,$D2,$D2		# d2 += h4*s3
-
-	vpmuludq	$H3,$S4,$H3		# h3*s4
-	vpmuludq	$H4,$S4,$H4		# h4*s4
-	vpaddq		$H3,$D2,$H2		# h2 = d2 + h3*r4
-	vpaddq		$H4,$D3,$H3		# h3 = d3 + h4*r4
-	vpmuludq	`32*7+4-0x90`(%rax),$H0,$H4		# h0*r4
-	vpmuludq	$H1,$S4,$H0		# h1*s4
-	vmovdqa		64(%rcx),$MASK		# .Lmask26
-	vpaddq		$H4,$D4,$H4		# h4 = d4 + h0*r4
-	vpaddq		$H0,$D0,$H0		# h0 = d0 + h1*s4
-
-	################################################################
-	# horizontal addition
-
-	vpsrldq		\$8,$D1,$T1
-	vpsrldq		\$8,$H2,$T2
-	vpsrldq		\$8,$H3,$T3
-	vpsrldq		\$8,$H4,$T4
-	vpsrldq		\$8,$H0,$T0
-	vpaddq		$T1,$D1,$D1
-	vpaddq		$T2,$H2,$H2
-	vpaddq		$T3,$H3,$H3
-	vpaddq		$T4,$H4,$H4
-	vpaddq		$T0,$H0,$H0
-
-	vpermq		\$0x2,$H3,$T3
-	vpermq		\$0x2,$H4,$T4
-	vpermq		\$0x2,$H0,$T0
-	vpermq		\$0x2,$D1,$T1
-	vpermq		\$0x2,$H2,$T2
-	vpaddq		$T3,$H3,$H3
-	vpaddq		$T4,$H4,$H4
-	vpaddq		$T0,$H0,$H0
-	vpaddq		$T1,$D1,$D1
-	vpaddq		$T2,$H2,$H2
-
-	################################################################
-	# lazy reduction
-
-	vpsrlq		\$26,$H3,$D3
-	vpand		$MASK,$H3,$H3
-	vpaddq		$D3,$H4,$H4		# h3 -> h4
-
-	vpsrlq		\$26,$H0,$D0
-	vpand		$MASK,$H0,$H0
-	vpaddq		$D0,$D1,$H1		# h0 -> h1
-
-	vpsrlq		\$26,$H4,$D4
-	vpand		$MASK,$H4,$H4
-
-	vpsrlq		\$26,$H1,$D1
-	vpand		$MASK,$H1,$H1
-	vpaddq		$D1,$H2,$H2		# h1 -> h2
-
-	vpaddq		$D4,$H0,$H0
-	vpsllq		\$2,$D4,$D4
-	vpaddq		$D4,$H0,$H0		# h4 -> h0
-
-	vpsrlq		\$26,$H2,$D2
-	vpand		$MASK,$H2,$H2
-	vpaddq		$D2,$H3,$H3		# h2 -> h3
-
-	vpsrlq		\$26,$H0,$D0
-	vpand		$MASK,$H0,$H0
-	vpaddq		$D0,$H1,$H1		# h0 -> h1
-
-	vpsrlq		\$26,$H3,$D3
-	vpand		$MASK,$H3,$H3
-	vpaddq		$D3,$H4,$H4		# h3 -> h4
-
-	vmovd		%x#$H0,`4*0-48-64`($ctx)# save partially reduced
-	vmovd		%x#$H1,`4*1-48-64`($ctx)
-	vmovd		%x#$H2,`4*2-48-64`($ctx)
-	vmovd		%x#$H3,`4*3-48-64`($ctx)
-	vmovd		%x#$H4,`4*4-48-64`($ctx)
-___
-$code.=<<___	if ($win64);
-	vmovdqa		-0xb0(%r10),%xmm6
-	vmovdqa		-0xa0(%r10),%xmm7
-	vmovdqa		-0x90(%r10),%xmm8
-	vmovdqa		-0x80(%r10),%xmm9
-	vmovdqa		-0x70(%r10),%xmm10
-	vmovdqa		-0x60(%r10),%xmm11
-	vmovdqa		-0x50(%r10),%xmm12
-	vmovdqa		-0x40(%r10),%xmm13
-	vmovdqa		-0x30(%r10),%xmm14
-	vmovdqa		-0x20(%r10),%xmm15
-	lea		-8(%r10),%rsp
-.Ldo_avx2_epilogue$suffix:
-___
-$code.=<<___	if (!$win64);
-	lea		-8(%r10),%rsp
-.cfi_def_cfa_register	%rsp
-___
-$code.=<<___;
-	vzeroupper
-	RET
-.cfi_endproc
-___
-if($avx > 2 && $avx512) {
-my ($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4) = map("%zmm$_",(16..24));
-my ($M0,$M1,$M2,$M3,$M4) = map("%zmm$_",(25..29));
-my $PADBIT="%zmm30";
-
-map(s/%y/%z/,($T4,$T0,$T1,$T2,$T3));		# switch to %zmm domain
-map(s/%y/%z/,($D0,$D1,$D2,$D3,$D4));
-map(s/%y/%z/,($H0,$H1,$H2,$H3,$H4));
-map(s/%y/%z/,($MASK));
-
-$code.=<<___;
-.cfi_startproc
-.Lblocks_avx512:
-	mov		\$15,%eax
-	kmovw		%eax,%k2
-___
-$code.=<<___	if (!$win64);
-	lea		8(%rsp),%r10
-.cfi_def_cfa_register	%r10
-	sub		\$0x128,%rsp
-___
-$code.=<<___	if ($win64);
-	lea		8(%rsp),%r10
-	sub		\$0x1c8,%rsp
-	vmovdqa		%xmm6,-0xb0(%r10)
-	vmovdqa		%xmm7,-0xa0(%r10)
-	vmovdqa		%xmm8,-0x90(%r10)
-	vmovdqa		%xmm9,-0x80(%r10)
-	vmovdqa		%xmm10,-0x70(%r10)
-	vmovdqa		%xmm11,-0x60(%r10)
-	vmovdqa		%xmm12,-0x50(%r10)
-	vmovdqa		%xmm13,-0x40(%r10)
-	vmovdqa		%xmm14,-0x30(%r10)
-	vmovdqa		%xmm15,-0x20(%r10)
-.Ldo_avx512_body:
-___
-$code.=<<___;
-	lea		.Lconst(%rip),%rcx
-	lea		48+64($ctx),$ctx	# size optimization
-	vmovdqa		96(%rcx),%y#$T2		# .Lpermd_avx2
-
-	# expand pre-calculated table
-	vmovdqu		`16*0-64`($ctx),%x#$D0	# will become expanded ${R0}
-	and		\$-512,%rsp
-	vmovdqu		`16*1-64`($ctx),%x#$D1	# will become ... ${R1}
-	mov		\$0x20,%rax
-	vmovdqu		`16*2-64`($ctx),%x#$T0	# ... ${S1}
-	vmovdqu		`16*3-64`($ctx),%x#$D2	# ... ${R2}
-	vmovdqu		`16*4-64`($ctx),%x#$T1	# ... ${S2}
-	vmovdqu		`16*5-64`($ctx),%x#$D3	# ... ${R3}
-	vmovdqu		`16*6-64`($ctx),%x#$T3	# ... ${S3}
-	vmovdqu		`16*7-64`($ctx),%x#$D4	# ... ${R4}
-	vmovdqu		`16*8-64`($ctx),%x#$T4	# ... ${S4}
-	vpermd		$D0,$T2,$R0		# 00003412 -> 14243444
-	vpbroadcastq	64(%rcx),$MASK		# .Lmask26
-	vpermd		$D1,$T2,$R1
-	vpermd		$T0,$T2,$S1
-	vpermd		$D2,$T2,$R2
-	vmovdqa64	$R0,0x00(%rsp){%k2}	# save in case $len%128 != 0
-	 vpsrlq		\$32,$R0,$T0		# 14243444 -> 01020304
-	vpermd		$T1,$T2,$S2
-	vmovdqu64	$R1,0x00(%rsp,%rax){%k2}
-	 vpsrlq		\$32,$R1,$T1
-	vpermd		$D3,$T2,$R3
-	vmovdqa64	$S1,0x40(%rsp){%k2}
-	vpermd		$T3,$T2,$S3
-	vpermd		$D4,$T2,$R4
-	vmovdqu64	$R2,0x40(%rsp,%rax){%k2}
-	vpermd		$T4,$T2,$S4
-	vmovdqa64	$S2,0x80(%rsp){%k2}
-	vmovdqu64	$R3,0x80(%rsp,%rax){%k2}
-	vmovdqa64	$S3,0xc0(%rsp){%k2}
-	vmovdqu64	$R4,0xc0(%rsp,%rax){%k2}
-	vmovdqa64	$S4,0x100(%rsp){%k2}
-
-	################################################################
-	# calculate 5th through 8th powers of the key
-	#
-	# d0 = r0'*r0 + r1'*5*r4 + r2'*5*r3 + r3'*5*r2 + r4'*5*r1
-	# d1 = r0'*r1 + r1'*r0   + r2'*5*r4 + r3'*5*r3 + r4'*5*r2
-	# d2 = r0'*r2 + r1'*r1   + r2'*r0   + r3'*5*r4 + r4'*5*r3
-	# d3 = r0'*r3 + r1'*r2   + r2'*r1   + r3'*r0   + r4'*5*r4
-	# d4 = r0'*r4 + r1'*r3   + r2'*r2   + r3'*r1   + r4'*r0
-
-	vpmuludq	$T0,$R0,$D0		# d0 = r0'*r0
-	vpmuludq	$T0,$R1,$D1		# d1 = r0'*r1
-	vpmuludq	$T0,$R2,$D2		# d2 = r0'*r2
-	vpmuludq	$T0,$R3,$D3		# d3 = r0'*r3
-	vpmuludq	$T0,$R4,$D4		# d4 = r0'*r4
-	 vpsrlq		\$32,$R2,$T2
-
-	vpmuludq	$T1,$S4,$M0
-	vpmuludq	$T1,$R0,$M1
-	vpmuludq	$T1,$R1,$M2
-	vpmuludq	$T1,$R2,$M3
-	vpmuludq	$T1,$R3,$M4
-	 vpsrlq		\$32,$R3,$T3
-	vpaddq		$M0,$D0,$D0		# d0 += r1'*5*r4
-	vpaddq		$M1,$D1,$D1		# d1 += r1'*r0
-	vpaddq		$M2,$D2,$D2		# d2 += r1'*r1
-	vpaddq		$M3,$D3,$D3		# d3 += r1'*r2
-	vpaddq		$M4,$D4,$D4		# d4 += r1'*r3
-
-	vpmuludq	$T2,$S3,$M0
-	vpmuludq	$T2,$S4,$M1
-	vpmuludq	$T2,$R1,$M3
-	vpmuludq	$T2,$R2,$M4
-	vpmuludq	$T2,$R0,$M2
-	 vpsrlq		\$32,$R4,$T4
-	vpaddq		$M0,$D0,$D0		# d0 += r2'*5*r3
-	vpaddq		$M1,$D1,$D1		# d1 += r2'*5*r4
-	vpaddq		$M3,$D3,$D3		# d3 += r2'*r1
-	vpaddq		$M4,$D4,$D4		# d4 += r2'*r2
-	vpaddq		$M2,$D2,$D2		# d2 += r2'*r0
-
-	vpmuludq	$T3,$S2,$M0
-	vpmuludq	$T3,$R0,$M3
-	vpmuludq	$T3,$R1,$M4
-	vpmuludq	$T3,$S3,$M1
-	vpmuludq	$T3,$S4,$M2
-	vpaddq		$M0,$D0,$D0		# d0 += r3'*5*r2
-	vpaddq		$M3,$D3,$D3		# d3 += r3'*r0
-	vpaddq		$M4,$D4,$D4		# d4 += r3'*r1
-	vpaddq		$M1,$D1,$D1		# d1 += r3'*5*r3
-	vpaddq		$M2,$D2,$D2		# d2 += r3'*5*r4
-
-	vpmuludq	$T4,$S4,$M3
-	vpmuludq	$T4,$R0,$M4
-	vpmuludq	$T4,$S1,$M0
-	vpmuludq	$T4,$S2,$M1
-	vpmuludq	$T4,$S3,$M2
-	vpaddq		$M3,$D3,$D3		# d3 += r2'*5*r4
-	vpaddq		$M4,$D4,$D4		# d4 += r2'*r0
-	vpaddq		$M0,$D0,$D0		# d0 += r2'*5*r1
-	vpaddq		$M1,$D1,$D1		# d1 += r2'*5*r2
-	vpaddq		$M2,$D2,$D2		# d2 += r2'*5*r3
-
-	################################################################
-	# load input
-	vmovdqu64	16*0($inp),%z#$T3
-	vmovdqu64	16*4($inp),%z#$T4
-	lea		16*8($inp),$inp
-
-	################################################################
-	# lazy reduction
-
-	vpsrlq		\$26,$D3,$M3
-	vpandq		$MASK,$D3,$D3
-	vpaddq		$M3,$D4,$D4		# d3 -> d4
-
-	vpsrlq		\$26,$D0,$M0
-	vpandq		$MASK,$D0,$D0
-	vpaddq		$M0,$D1,$D1		# d0 -> d1
-
-	vpsrlq		\$26,$D4,$M4
-	vpandq		$MASK,$D4,$D4
-
-	vpsrlq		\$26,$D1,$M1
-	vpandq		$MASK,$D1,$D1
-	vpaddq		$M1,$D2,$D2		# d1 -> d2
-
-	vpaddq		$M4,$D0,$D0
-	vpsllq		\$2,$M4,$M4
-	vpaddq		$M4,$D0,$D0		# d4 -> d0
-
-	vpsrlq		\$26,$D2,$M2
-	vpandq		$MASK,$D2,$D2
-	vpaddq		$M2,$D3,$D3		# d2 -> d3
-
-	vpsrlq		\$26,$D0,$M0
-	vpandq		$MASK,$D0,$D0
-	vpaddq		$M0,$D1,$D1		# d0 -> d1
-
-	vpsrlq		\$26,$D3,$M3
-	vpandq		$MASK,$D3,$D3
-	vpaddq		$M3,$D4,$D4		# d3 -> d4
-
-	################################################################
-	# at this point we have 14243444 in $R0-$S4 and 05060708 in
-	# $D0-$D4, ...
-
-	vpunpcklqdq	$T4,$T3,$T0	# transpose input
-	vpunpckhqdq	$T4,$T3,$T4
-
-	# ... since input 64-bit lanes are ordered as 73625140, we could
-	# "vperm" it to 76543210 (here and in each loop iteration), *or*
-	# we could just flow along, hence the goal for $R0-$S4 is
-	# 1858286838784888 ...
-
-	vmovdqa32	128(%rcx),$M0		# .Lpermd_avx512:
-	mov		\$0x7777,%eax
-	kmovw		%eax,%k1
-
-	vpermd		$R0,$M0,$R0		# 14243444 -> 1---2---3---4---
-	vpermd		$R1,$M0,$R1
-	vpermd		$R2,$M0,$R2
-	vpermd		$R3,$M0,$R3
-	vpermd		$R4,$M0,$R4
-
-	vpermd		$D0,$M0,${R0}{%k1}	# 05060708 -> 1858286838784888
-	vpermd		$D1,$M0,${R1}{%k1}
-	vpermd		$D2,$M0,${R2}{%k1}
-	vpermd		$D3,$M0,${R3}{%k1}
-	vpermd		$D4,$M0,${R4}{%k1}
-
-	vpslld		\$2,$R1,$S1		# *5
-	vpslld		\$2,$R2,$S2
-	vpslld		\$2,$R3,$S3
-	vpslld		\$2,$R4,$S4
-	vpaddd		$R1,$S1,$S1
-	vpaddd		$R2,$S2,$S2
-	vpaddd		$R3,$S3,$S3
-	vpaddd		$R4,$S4,$S4
-
-	vpbroadcastq	32(%rcx),$PADBIT	# .L129
-
-	vpsrlq		\$52,$T0,$T2		# splat input
-	vpsllq		\$12,$T4,$T3
-	vporq		$T3,$T2,$T2
-	vpsrlq		\$26,$T0,$T1
-	vpsrlq		\$14,$T4,$T3
-	vpsrlq		\$40,$T4,$T4		# 4
-	vpandq		$MASK,$T2,$T2		# 2
-	vpandq		$MASK,$T0,$T0		# 0
-	#vpandq		$MASK,$T1,$T1		# 1
-	#vpandq		$MASK,$T3,$T3		# 3
-	#vporq		$PADBIT,$T4,$T4		# padbit, yes, always
-
-	vpaddq		$H2,$T2,$H2		# accumulate input
-	sub		\$192,$len
-	jbe		.Ltail_avx512
-	jmp		.Loop_avx512
-
-.align	32
-.Loop_avx512:
-	################################################################
-	# ((inp[0]*r^8+inp[ 8])*r^8+inp[16])*r^8
-	# ((inp[1]*r^8+inp[ 9])*r^8+inp[17])*r^7
-	# ((inp[2]*r^8+inp[10])*r^8+inp[18])*r^6
-	# ((inp[3]*r^8+inp[11])*r^8+inp[19])*r^5
-	# ((inp[4]*r^8+inp[12])*r^8+inp[20])*r^4
-	# ((inp[5]*r^8+inp[13])*r^8+inp[21])*r^3
-	# ((inp[6]*r^8+inp[14])*r^8+inp[22])*r^2
-	# ((inp[7]*r^8+inp[15])*r^8+inp[23])*r^1
-	#   \________/\___________/
-	################################################################
-	#vpaddq		$H2,$T2,$H2		# accumulate input
-
-	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
-	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
-	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
-	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
-	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
-	#
-	# however, as h2 is "chronologically" first one available pull
-	# corresponding operations up, so it's
-	#
-	# d3 = h2*r1   + h0*r3 + h1*r2   + h3*r0 + h4*5*r4
-	# d4 = h2*r2   + h0*r4 + h1*r3   + h3*r1 + h4*r0
-	# d0 = h2*5*r3 + h0*r0 + h1*5*r4         + h3*5*r2 + h4*5*r1
-	# d1 = h2*5*r4 + h0*r1           + h1*r0 + h3*5*r3 + h4*5*r2
-	# d2 = h2*r0           + h0*r2   + h1*r1 + h3*5*r4 + h4*5*r3
-
-	vpmuludq	$H2,$R1,$D3		# d3 = h2*r1
-	 vpaddq		$H0,$T0,$H0
-	vpmuludq	$H2,$R2,$D4		# d4 = h2*r2
-	 vpandq		$MASK,$T1,$T1		# 1
-	vpmuludq	$H2,$S3,$D0		# d0 = h2*s3
-	 vpandq		$MASK,$T3,$T3		# 3
-	vpmuludq	$H2,$S4,$D1		# d1 = h2*s4
-	 vporq		$PADBIT,$T4,$T4		# padbit, yes, always
-	vpmuludq	$H2,$R0,$D2		# d2 = h2*r0
-	 vpaddq		$H1,$T1,$H1		# accumulate input
-	 vpaddq		$H3,$T3,$H3
-	 vpaddq		$H4,$T4,$H4
-
-	  vmovdqu64	16*0($inp),$T3		# load input
-	  vmovdqu64	16*4($inp),$T4
-	  lea		16*8($inp),$inp
-	vpmuludq	$H0,$R3,$M3
-	vpmuludq	$H0,$R4,$M4
-	vpmuludq	$H0,$R0,$M0
-	vpmuludq	$H0,$R1,$M1
-	vpaddq		$M3,$D3,$D3		# d3 += h0*r3
-	vpaddq		$M4,$D4,$D4		# d4 += h0*r4
-	vpaddq		$M0,$D0,$D0		# d0 += h0*r0
-	vpaddq		$M1,$D1,$D1		# d1 += h0*r1
-
-	vpmuludq	$H1,$R2,$M3
-	vpmuludq	$H1,$R3,$M4
-	vpmuludq	$H1,$S4,$M0
-	vpmuludq	$H0,$R2,$M2
-	vpaddq		$M3,$D3,$D3		# d3 += h1*r2
-	vpaddq		$M4,$D4,$D4		# d4 += h1*r3
-	vpaddq		$M0,$D0,$D0		# d0 += h1*s4
-	vpaddq		$M2,$D2,$D2		# d2 += h0*r2
-
-	  vpunpcklqdq	$T4,$T3,$T0		# transpose input
-	  vpunpckhqdq	$T4,$T3,$T4
-
-	vpmuludq	$H3,$R0,$M3
-	vpmuludq	$H3,$R1,$M4
-	vpmuludq	$H1,$R0,$M1
-	vpmuludq	$H1,$R1,$M2
-	vpaddq		$M3,$D3,$D3		# d3 += h3*r0
-	vpaddq		$M4,$D4,$D4		# d4 += h3*r1
-	vpaddq		$M1,$D1,$D1		# d1 += h1*r0
-	vpaddq		$M2,$D2,$D2		# d2 += h1*r1
-
-	vpmuludq	$H4,$S4,$M3
-	vpmuludq	$H4,$R0,$M4
-	vpmuludq	$H3,$S2,$M0
-	vpmuludq	$H3,$S3,$M1
-	vpaddq		$M3,$D3,$D3		# d3 += h4*s4
-	vpmuludq	$H3,$S4,$M2
-	vpaddq		$M4,$D4,$D4		# d4 += h4*r0
-	vpaddq		$M0,$D0,$D0		# d0 += h3*s2
-	vpaddq		$M1,$D1,$D1		# d1 += h3*s3
-	vpaddq		$M2,$D2,$D2		# d2 += h3*s4
-
-	vpmuludq	$H4,$S1,$M0
-	vpmuludq	$H4,$S2,$M1
-	vpmuludq	$H4,$S3,$M2
-	vpaddq		$M0,$D0,$H0		# h0 = d0 + h4*s1
-	vpaddq		$M1,$D1,$H1		# h1 = d2 + h4*s2
-	vpaddq		$M2,$D2,$H2		# h2 = d3 + h4*s3
-
-	################################################################
-	# lazy reduction (interleaved with input splat)
-
-	 vpsrlq		\$52,$T0,$T2		# splat input
-	 vpsllq		\$12,$T4,$T3
-
-	vpsrlq		\$26,$D3,$H3
-	vpandq		$MASK,$D3,$D3
-	vpaddq		$H3,$D4,$H4		# h3 -> h4
-
-	 vporq		$T3,$T2,$T2
-
-	vpsrlq		\$26,$H0,$D0
-	vpandq		$MASK,$H0,$H0
-	vpaddq		$D0,$H1,$H1		# h0 -> h1
-
-	 vpandq		$MASK,$T2,$T2		# 2
-
-	vpsrlq		\$26,$H4,$D4
-	vpandq		$MASK,$H4,$H4
-
-	vpsrlq		\$26,$H1,$D1
-	vpandq		$MASK,$H1,$H1
-	vpaddq		$D1,$H2,$H2		# h1 -> h2
-
-	vpaddq		$D4,$H0,$H0
-	vpsllq		\$2,$D4,$D4
-	vpaddq		$D4,$H0,$H0		# h4 -> h0
-
-	 vpaddq		$T2,$H2,$H2		# modulo-scheduled
-	 vpsrlq		\$26,$T0,$T1
-
-	vpsrlq		\$26,$H2,$D2
-	vpandq		$MASK,$H2,$H2
-	vpaddq		$D2,$D3,$H3		# h2 -> h3
-
-	 vpsrlq		\$14,$T4,$T3
-
-	vpsrlq		\$26,$H0,$D0
-	vpandq		$MASK,$H0,$H0
-	vpaddq		$D0,$H1,$H1		# h0 -> h1
-
-	 vpsrlq		\$40,$T4,$T4		# 4
-
-	vpsrlq		\$26,$H3,$D3
-	vpandq		$MASK,$H3,$H3
-	vpaddq		$D3,$H4,$H4		# h3 -> h4
-
-	 vpandq		$MASK,$T0,$T0		# 0
-	 #vpandq	$MASK,$T1,$T1		# 1
-	 #vpandq	$MASK,$T3,$T3		# 3
-	 #vporq		$PADBIT,$T4,$T4		# padbit, yes, always
-
-	sub		\$128,$len
-	ja		.Loop_avx512
-
-.Ltail_avx512:
-	################################################################
-	# while above multiplications were by r^8 in all lanes, in last
-	# iteration we multiply least significant lane by r^8 and most
-	# significant one by r, that's why table gets shifted...
-
-	vpsrlq		\$32,$R0,$R0		# 0105020603070408
-	vpsrlq		\$32,$R1,$R1
-	vpsrlq		\$32,$R2,$R2
-	vpsrlq		\$32,$S3,$S3
-	vpsrlq		\$32,$S4,$S4
-	vpsrlq		\$32,$R3,$R3
-	vpsrlq		\$32,$R4,$R4
-	vpsrlq		\$32,$S1,$S1
-	vpsrlq		\$32,$S2,$S2
-
-	################################################################
-	# load either next or last 64 byte of input
-	lea		($inp,$len),$inp
-
-	#vpaddq		$H2,$T2,$H2		# accumulate input
-	vpaddq		$H0,$T0,$H0
-
-	vpmuludq	$H2,$R1,$D3		# d3 = h2*r1
-	vpmuludq	$H2,$R2,$D4		# d4 = h2*r2
-	vpmuludq	$H2,$S3,$D0		# d0 = h2*s3
-	 vpandq		$MASK,$T1,$T1		# 1
-	vpmuludq	$H2,$S4,$D1		# d1 = h2*s4
-	 vpandq		$MASK,$T3,$T3		# 3
-	vpmuludq	$H2,$R0,$D2		# d2 = h2*r0
-	 vporq		$PADBIT,$T4,$T4		# padbit, yes, always
-	 vpaddq		$H1,$T1,$H1		# accumulate input
-	 vpaddq		$H3,$T3,$H3
-	 vpaddq		$H4,$T4,$H4
-
-	  vmovdqu	16*0($inp),%x#$T0
-	vpmuludq	$H0,$R3,$M3
-	vpmuludq	$H0,$R4,$M4
-	vpmuludq	$H0,$R0,$M0
-	vpmuludq	$H0,$R1,$M1
-	vpaddq		$M3,$D3,$D3		# d3 += h0*r3
-	vpaddq		$M4,$D4,$D4		# d4 += h0*r4
-	vpaddq		$M0,$D0,$D0		# d0 += h0*r0
-	vpaddq		$M1,$D1,$D1		# d1 += h0*r1
-
-	  vmovdqu	16*1($inp),%x#$T1
-	vpmuludq	$H1,$R2,$M3
-	vpmuludq	$H1,$R3,$M4
-	vpmuludq	$H1,$S4,$M0
-	vpmuludq	$H0,$R2,$M2
-	vpaddq		$M3,$D3,$D3		# d3 += h1*r2
-	vpaddq		$M4,$D4,$D4		# d4 += h1*r3
-	vpaddq		$M0,$D0,$D0		# d0 += h1*s4
-	vpaddq		$M2,$D2,$D2		# d2 += h0*r2
-
-	  vinserti128	\$1,16*2($inp),%y#$T0,%y#$T0
-	vpmuludq	$H3,$R0,$M3
-	vpmuludq	$H3,$R1,$M4
-	vpmuludq	$H1,$R0,$M1
-	vpmuludq	$H1,$R1,$M2
-	vpaddq		$M3,$D3,$D3		# d3 += h3*r0
-	vpaddq		$M4,$D4,$D4		# d4 += h3*r1
-	vpaddq		$M1,$D1,$D1		# d1 += h1*r0
-	vpaddq		$M2,$D2,$D2		# d2 += h1*r1
-
-	  vinserti128	\$1,16*3($inp),%y#$T1,%y#$T1
-	vpmuludq	$H4,$S4,$M3
-	vpmuludq	$H4,$R0,$M4
-	vpmuludq	$H3,$S2,$M0
-	vpmuludq	$H3,$S3,$M1
-	vpmuludq	$H3,$S4,$M2
-	vpaddq		$M3,$D3,$H3		# h3 = d3 + h4*s4
-	vpaddq		$M4,$D4,$D4		# d4 += h4*r0
-	vpaddq		$M0,$D0,$D0		# d0 += h3*s2
-	vpaddq		$M1,$D1,$D1		# d1 += h3*s3
-	vpaddq		$M2,$D2,$D2		# d2 += h3*s4
-
-	vpmuludq	$H4,$S1,$M0
-	vpmuludq	$H4,$S2,$M1
-	vpmuludq	$H4,$S3,$M2
-	vpaddq		$M0,$D0,$H0		# h0 = d0 + h4*s1
-	vpaddq		$M1,$D1,$H1		# h1 = d2 + h4*s2
-	vpaddq		$M2,$D2,$H2		# h2 = d3 + h4*s3
-
-	################################################################
-	# horizontal addition
-
-	mov		\$1,%eax
-	vpermq		\$0xb1,$H3,$D3
-	vpermq		\$0xb1,$D4,$H4
-	vpermq		\$0xb1,$H0,$D0
-	vpermq		\$0xb1,$H1,$D1
-	vpermq		\$0xb1,$H2,$D2
-	vpaddq		$D3,$H3,$H3
-	vpaddq		$D4,$H4,$H4
-	vpaddq		$D0,$H0,$H0
-	vpaddq		$D1,$H1,$H1
-	vpaddq		$D2,$H2,$H2
-
-	kmovw		%eax,%k3
-	vpermq		\$0x2,$H3,$D3
-	vpermq		\$0x2,$H4,$D4
-	vpermq		\$0x2,$H0,$D0
-	vpermq		\$0x2,$H1,$D1
-	vpermq		\$0x2,$H2,$D2
-	vpaddq		$D3,$H3,$H3
-	vpaddq		$D4,$H4,$H4
-	vpaddq		$D0,$H0,$H0
-	vpaddq		$D1,$H1,$H1
-	vpaddq		$D2,$H2,$H2
-
-	vextracti64x4	\$0x1,$H3,%y#$D3
-	vextracti64x4	\$0x1,$H4,%y#$D4
-	vextracti64x4	\$0x1,$H0,%y#$D0
-	vextracti64x4	\$0x1,$H1,%y#$D1
-	vextracti64x4	\$0x1,$H2,%y#$D2
-	vpaddq		$D3,$H3,${H3}{%k3}{z}	# keep single qword in case
-	vpaddq		$D4,$H4,${H4}{%k3}{z}	# it's passed to .Ltail_avx2
-	vpaddq		$D0,$H0,${H0}{%k3}{z}
-	vpaddq		$D1,$H1,${H1}{%k3}{z}
-	vpaddq		$D2,$H2,${H2}{%k3}{z}
-___
-map(s/%z/%y/,($T0,$T1,$T2,$T3,$T4, $PADBIT));
-map(s/%z/%y/,($H0,$H1,$H2,$H3,$H4, $D0,$D1,$D2,$D3,$D4, $MASK));
-$code.=<<___;
-	################################################################
-	# lazy reduction (interleaved with input splat)
-
-	vpsrlq		\$26,$H3,$D3
-	vpand		$MASK,$H3,$H3
-	 vpsrldq	\$6,$T0,$T2		# splat input
-	 vpsrldq	\$6,$T1,$T3
-	 vpunpckhqdq	$T1,$T0,$T4		# 4
-	vpaddq		$D3,$H4,$H4		# h3 -> h4
-
-	vpsrlq		\$26,$H0,$D0
-	vpand		$MASK,$H0,$H0
-	 vpunpcklqdq	$T3,$T2,$T2		# 2:3
-	 vpunpcklqdq	$T1,$T0,$T0		# 0:1
-	vpaddq		$D0,$H1,$H1		# h0 -> h1
-
-	vpsrlq		\$26,$H4,$D4
-	vpand		$MASK,$H4,$H4
-
-	vpsrlq		\$26,$H1,$D1
-	vpand		$MASK,$H1,$H1
-	 vpsrlq		\$30,$T2,$T3
-	 vpsrlq		\$4,$T2,$T2
-	vpaddq		$D1,$H2,$H2		# h1 -> h2
-
-	vpaddq		$D4,$H0,$H0
-	vpsllq		\$2,$D4,$D4
-	 vpsrlq		\$26,$T0,$T1
-	 vpsrlq		\$40,$T4,$T4		# 4
-	vpaddq		$D4,$H0,$H0		# h4 -> h0
-
-	vpsrlq		\$26,$H2,$D2
-	vpand		$MASK,$H2,$H2
-	 vpand		$MASK,$T2,$T2		# 2
-	 vpand		$MASK,$T0,$T0		# 0
-	vpaddq		$D2,$H3,$H3		# h2 -> h3
-
-	vpsrlq		\$26,$H0,$D0
-	vpand		$MASK,$H0,$H0
-	 vpaddq		$H2,$T2,$H2		# accumulate input for .Ltail_avx2
-	 vpand		$MASK,$T1,$T1		# 1
-	vpaddq		$D0,$H1,$H1		# h0 -> h1
-
-	vpsrlq		\$26,$H3,$D3
-	vpand		$MASK,$H3,$H3
-	 vpand		$MASK,$T3,$T3		# 3
-	 vpor		32(%rcx),$T4,$T4	# padbit, yes, always
-	vpaddq		$D3,$H4,$H4		# h3 -> h4
-
-	lea		0x90(%rsp),%rax		# size optimization for .Ltail_avx2
-	add		\$64,$len
-	jnz		.Ltail_avx2$suffix
-
-	vpsubq		$T2,$H2,$H2		# undo input accumulation
-	vmovd		%x#$H0,`4*0-48-64`($ctx)# save partially reduced
-	vmovd		%x#$H1,`4*1-48-64`($ctx)
-	vmovd		%x#$H2,`4*2-48-64`($ctx)
-	vmovd		%x#$H3,`4*3-48-64`($ctx)
-	vmovd		%x#$H4,`4*4-48-64`($ctx)
-	vzeroall
-___
-$code.=<<___	if ($win64);
-	movdqa		-0xb0(%r10),%xmm6
-	movdqa		-0xa0(%r10),%xmm7
-	movdqa		-0x90(%r10),%xmm8
-	movdqa		-0x80(%r10),%xmm9
-	movdqa		-0x70(%r10),%xmm10
-	movdqa		-0x60(%r10),%xmm11
-	movdqa		-0x50(%r10),%xmm12
-	movdqa		-0x40(%r10),%xmm13
-	movdqa		-0x30(%r10),%xmm14
-	movdqa		-0x20(%r10),%xmm15
-	lea		-8(%r10),%rsp
-.Ldo_avx512_epilogue:
-___
-$code.=<<___	if (!$win64);
-	lea		-8(%r10),%rsp
-.cfi_def_cfa_register	%rsp
-___
-$code.=<<___;
-	RET
-.cfi_endproc
-___
-
-}
-
-}
-
-&declare_function("poly1305_blocks_avx2", 32, 4);
-poly1305_blocks_avxN(0);
-&end_function("poly1305_blocks_avx2");
-
-#######################################################################
-if ($avx>2) {
-# On entry we have input length divisible by 64. But since inner loop
-# processes 128 bytes per iteration, cases when length is not divisible
-# by 128 are handled by passing tail 64 bytes to .Ltail_avx2. For this
-# reason stack layout is kept identical to poly1305_blocks_avx2. If not
-# for this tail, we wouldn't have to even allocate stack frame...
-
-if($kernel) {
-	$code .= "#ifdef CONFIG_AS_AVX512\n";
-}
-
-&declare_function("poly1305_blocks_avx512", 32, 4);
-poly1305_blocks_avxN(1);
-&end_function("poly1305_blocks_avx512");
-
-if ($kernel) {
-	$code .= "#endif\n";
-}
-
-if (!$kernel && $avx>3) {
-########################################################################
-# VPMADD52 version using 2^44 radix.
-#
-# One can argue that base 2^52 would be more natural. Well, even though
-# some operations would be more natural, one has to recognize couple of
-# things. Base 2^52 doesn't provide advantage over base 2^44 if you look
-# at amount of multiply-n-accumulate operations. Secondly, it makes it
-# impossible to pre-compute multiples of 5 [referred to as s[]/sN in
-# reference implementations], which means that more such operations
-# would have to be performed in inner loop, which in turn makes critical
-# path longer. In other words, even though base 2^44 reduction might
-# look less elegant, overall critical path is actually shorter...
-
-########################################################################
-# Layout of opaque area is following.
-#
-#	unsigned __int64 h[3];		# current hash value base 2^44
-#	unsigned __int64 s[2];		# key value*20 base 2^44
-#	unsigned __int64 r[3];		# key value base 2^44
-#	struct { unsigned __int64 r^1, r^3, r^2, r^4; } R[4];
-#					# r^n positions reflect
-#					# placement in register, not
-#					# memory, R[3] is R[1]*20
-
-$code.=<<___;
-.type	poly1305_init_base2_44,\@function,3
-.align	32
-poly1305_init_base2_44:
-	xor	%eax,%eax
-	mov	%rax,0($ctx)		# initialize hash value
-	mov	%rax,8($ctx)
-	mov	%rax,16($ctx)
-
-.Linit_base2_44:
-	lea	poly1305_blocks_vpmadd52(%rip),%r10
-	lea	poly1305_emit_base2_44(%rip),%r11
-
-	mov	\$0x0ffffffc0fffffff,%rax
-	mov	\$0x0ffffffc0ffffffc,%rcx
-	and	0($inp),%rax
-	mov	\$0x00000fffffffffff,%r8
-	and	8($inp),%rcx
-	mov	\$0x00000fffffffffff,%r9
-	and	%rax,%r8
-	shrd	\$44,%rcx,%rax
-	mov	%r8,40($ctx)		# r0
-	and	%r9,%rax
-	shr	\$24,%rcx
-	mov	%rax,48($ctx)		# r1
-	lea	(%rax,%rax,4),%rax	# *5
-	mov	%rcx,56($ctx)		# r2
-	shl	\$2,%rax		# magic <<2
-	lea	(%rcx,%rcx,4),%rcx	# *5
-	shl	\$2,%rcx		# magic <<2
-	mov	%rax,24($ctx)		# s1
-	mov	%rcx,32($ctx)		# s2
-	movq	\$-1,64($ctx)		# write impossible value
-___
-$code.=<<___	if ($flavour !~ /elf32/);
-	mov	%r10,0(%rdx)
-	mov	%r11,8(%rdx)
-___
-$code.=<<___	if ($flavour =~ /elf32/);
-	mov	%r10d,0(%rdx)
-	mov	%r11d,4(%rdx)
-___
-$code.=<<___;
-	mov	\$1,%eax
-	RET
-.size	poly1305_init_base2_44,.-poly1305_init_base2_44
-___
-{
-my ($H0,$H1,$H2,$r2r1r0,$r1r0s2,$r0s2s1,$Dlo,$Dhi) = map("%ymm$_",(0..5,16,17));
-my ($T0,$inp_permd,$inp_shift,$PAD) = map("%ymm$_",(18..21));
-my ($reduc_mask,$reduc_rght,$reduc_left) = map("%ymm$_",(22..25));
-
-$code.=<<___;
-.type	poly1305_blocks_vpmadd52,\@function,4
-.align	32
-poly1305_blocks_vpmadd52:
-	shr	\$4,$len
-	jz	.Lno_data_vpmadd52		# too short
-
-	shl	\$40,$padbit
-	mov	64($ctx),%r8			# peek on power of the key
-
-	# if powers of the key are not calculated yet, process up to 3
-	# blocks with this single-block subroutine, otherwise ensure that
-	# length is divisible by 2 blocks and pass the rest down to next
-	# subroutine...
-
-	mov	\$3,%rax
-	mov	\$1,%r10
-	cmp	\$4,$len			# is input long
-	cmovae	%r10,%rax
-	test	%r8,%r8				# is power value impossible?
-	cmovns	%r10,%rax
-
-	and	$len,%rax			# is input of favourable length?
-	jz	.Lblocks_vpmadd52_4x
-
-	sub		%rax,$len
-	mov		\$7,%r10d
-	mov		\$1,%r11d
-	kmovw		%r10d,%k7
-	lea		.L2_44_inp_permd(%rip),%r10
-	kmovw		%r11d,%k1
-
-	vmovq		$padbit,%x#$PAD
-	vmovdqa64	0(%r10),$inp_permd	# .L2_44_inp_permd
-	vmovdqa64	32(%r10),$inp_shift	# .L2_44_inp_shift
-	vpermq		\$0xcf,$PAD,$PAD
-	vmovdqa64	64(%r10),$reduc_mask	# .L2_44_mask
-
-	vmovdqu64	0($ctx),${Dlo}{%k7}{z}		# load hash value
-	vmovdqu64	40($ctx),${r2r1r0}{%k7}{z}	# load keys
-	vmovdqu64	32($ctx),${r1r0s2}{%k7}{z}
-	vmovdqu64	24($ctx),${r0s2s1}{%k7}{z}
-
-	vmovdqa64	96(%r10),$reduc_rght	# .L2_44_shift_rgt
-	vmovdqa64	128(%r10),$reduc_left	# .L2_44_shift_lft
-
-	jmp		.Loop_vpmadd52
-
-.align	32
-.Loop_vpmadd52:
-	vmovdqu32	0($inp),%x#$T0		# load input as ----3210
-	lea		16($inp),$inp
-
-	vpermd		$T0,$inp_permd,$T0	# ----3210 -> --322110
-	vpsrlvq		$inp_shift,$T0,$T0
-	vpandq		$reduc_mask,$T0,$T0
-	vporq		$PAD,$T0,$T0
-
-	vpaddq		$T0,$Dlo,$Dlo		# accumulate input
-
-	vpermq		\$0,$Dlo,${H0}{%k7}{z}	# smash hash value
-	vpermq		\$0b01010101,$Dlo,${H1}{%k7}{z}
-	vpermq		\$0b10101010,$Dlo,${H2}{%k7}{z}
-
-	vpxord		$Dlo,$Dlo,$Dlo
-	vpxord		$Dhi,$Dhi,$Dhi
-
-	vpmadd52luq	$r2r1r0,$H0,$Dlo
-	vpmadd52huq	$r2r1r0,$H0,$Dhi
-
-	vpmadd52luq	$r1r0s2,$H1,$Dlo
-	vpmadd52huq	$r1r0s2,$H1,$Dhi
-
-	vpmadd52luq	$r0s2s1,$H2,$Dlo
-	vpmadd52huq	$r0s2s1,$H2,$Dhi
-
-	vpsrlvq		$reduc_rght,$Dlo,$T0	# 0 in topmost qword
-	vpsllvq		$reduc_left,$Dhi,$Dhi	# 0 in topmost qword
-	vpandq		$reduc_mask,$Dlo,$Dlo
-
-	vpaddq		$T0,$Dhi,$Dhi
-
-	vpermq		\$0b10010011,$Dhi,$Dhi	# 0 in lowest qword
-
-	vpaddq		$Dhi,$Dlo,$Dlo		# note topmost qword :-)
-
-	vpsrlvq		$reduc_rght,$Dlo,$T0	# 0 in topmost word
-	vpandq		$reduc_mask,$Dlo,$Dlo
-
-	vpermq		\$0b10010011,$T0,$T0
-
-	vpaddq		$T0,$Dlo,$Dlo
-
-	vpermq		\$0b10010011,$Dlo,${T0}{%k1}{z}
-
-	vpaddq		$T0,$Dlo,$Dlo
-	vpsllq		\$2,$T0,$T0
-
-	vpaddq		$T0,$Dlo,$Dlo
-
-	dec		%rax			# len-=16
-	jnz		.Loop_vpmadd52
-
-	vmovdqu64	$Dlo,0($ctx){%k7}	# store hash value
-
-	test		$len,$len
-	jnz		.Lblocks_vpmadd52_4x
-
-.Lno_data_vpmadd52:
-	RET
-.size	poly1305_blocks_vpmadd52,.-poly1305_blocks_vpmadd52
-___
-}
-{
-########################################################################
-# As implied by its name 4x subroutine processes 4 blocks in parallel
-# (but handles even 4*n+2 blocks lengths). It takes up to 4th key power
-# and is handled in 256-bit %ymm registers.
-
-my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17));
-my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23));
-my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31));
-
-$code.=<<___;
-.type	poly1305_blocks_vpmadd52_4x,\@function,4
-.align	32
-poly1305_blocks_vpmadd52_4x:
-	shr	\$4,$len
-	jz	.Lno_data_vpmadd52_4x		# too short
-
-	shl	\$40,$padbit
-	mov	64($ctx),%r8			# peek on power of the key
-
-.Lblocks_vpmadd52_4x:
-	vpbroadcastq	$padbit,$PAD
-
-	vmovdqa64	.Lx_mask44(%rip),$mask44
-	mov		\$5,%eax
-	vmovdqa64	.Lx_mask42(%rip),$mask42
-	kmovw		%eax,%k1		# used in 2x path
-
-	test		%r8,%r8			# is power value impossible?
-	js		.Linit_vpmadd52		# if it is, then init R[4]
-
-	vmovq		0($ctx),%x#$H0		# load current hash value
-	vmovq		8($ctx),%x#$H1
-	vmovq		16($ctx),%x#$H2
-
-	test		\$3,$len		# is length 4*n+2?
-	jnz		.Lblocks_vpmadd52_2x_do
-
-.Lblocks_vpmadd52_4x_do:
-	vpbroadcastq	64($ctx),$R0		# load 4th power of the key
-	vpbroadcastq	96($ctx),$R1
-	vpbroadcastq	128($ctx),$R2
-	vpbroadcastq	160($ctx),$S1
-
-.Lblocks_vpmadd52_4x_key_loaded:
-	vpsllq		\$2,$R2,$S2		# S2 = R2*5*4
-	vpaddq		$R2,$S2,$S2
-	vpsllq		\$2,$S2,$S2
-
-	test		\$7,$len		# is len 8*n?
-	jz		.Lblocks_vpmadd52_8x
-
-	vmovdqu64	16*0($inp),$T2		# load data
-	vmovdqu64	16*2($inp),$T3
-	lea		16*4($inp),$inp
-
-	vpunpcklqdq	$T3,$T2,$T1		# transpose data
-	vpunpckhqdq	$T3,$T2,$T3
-
-	# at this point 64-bit lanes are ordered as 3-1-2-0
-
-	vpsrlq		\$24,$T3,$T2		# splat the data
-	vporq		$PAD,$T2,$T2
-	 vpaddq		$T2,$H2,$H2		# accumulate input
-	vpandq		$mask44,$T1,$T0
-	vpsrlq		\$44,$T1,$T1
-	vpsllq		\$20,$T3,$T3
-	vporq		$T3,$T1,$T1
-	vpandq		$mask44,$T1,$T1
-
-	sub		\$4,$len
-	jz		.Ltail_vpmadd52_4x
-	jmp		.Loop_vpmadd52_4x
-	ud2
-
-.align	32
-.Linit_vpmadd52:
-	vmovq		24($ctx),%x#$S1		# load key
-	vmovq		56($ctx),%x#$H2
-	vmovq		32($ctx),%x#$S2
-	vmovq		40($ctx),%x#$R0
-	vmovq		48($ctx),%x#$R1
-
-	vmovdqa		$R0,$H0
-	vmovdqa		$R1,$H1
-	vmovdqa		$H2,$R2
-
-	mov		\$2,%eax
-
-.Lmul_init_vpmadd52:
-	vpxorq		$D0lo,$D0lo,$D0lo
-	vpmadd52luq	$H2,$S1,$D0lo
-	vpxorq		$D0hi,$D0hi,$D0hi
-	vpmadd52huq	$H2,$S1,$D0hi
-	vpxorq		$D1lo,$D1lo,$D1lo
-	vpmadd52luq	$H2,$S2,$D1lo
-	vpxorq		$D1hi,$D1hi,$D1hi
-	vpmadd52huq	$H2,$S2,$D1hi
-	vpxorq		$D2lo,$D2lo,$D2lo
-	vpmadd52luq	$H2,$R0,$D2lo
-	vpxorq		$D2hi,$D2hi,$D2hi
-	vpmadd52huq	$H2,$R0,$D2hi
-
-	vpmadd52luq	$H0,$R0,$D0lo
-	vpmadd52huq	$H0,$R0,$D0hi
-	vpmadd52luq	$H0,$R1,$D1lo
-	vpmadd52huq	$H0,$R1,$D1hi
-	vpmadd52luq	$H0,$R2,$D2lo
-	vpmadd52huq	$H0,$R2,$D2hi
-
-	vpmadd52luq	$H1,$S2,$D0lo
-	vpmadd52huq	$H1,$S2,$D0hi
-	vpmadd52luq	$H1,$R0,$D1lo
-	vpmadd52huq	$H1,$R0,$D1hi
-	vpmadd52luq	$H1,$R1,$D2lo
-	vpmadd52huq	$H1,$R1,$D2hi
-
-	################################################################
-	# partial reduction
-	vpsrlq		\$44,$D0lo,$tmp
-	vpsllq		\$8,$D0hi,$D0hi
-	vpandq		$mask44,$D0lo,$H0
-	vpaddq		$tmp,$D0hi,$D0hi
-
-	vpaddq		$D0hi,$D1lo,$D1lo
-
-	vpsrlq		\$44,$D1lo,$tmp
-	vpsllq		\$8,$D1hi,$D1hi
-	vpandq		$mask44,$D1lo,$H1
-	vpaddq		$tmp,$D1hi,$D1hi
-
-	vpaddq		$D1hi,$D2lo,$D2lo
-
-	vpsrlq		\$42,$D2lo,$tmp
-	vpsllq		\$10,$D2hi,$D2hi
-	vpandq		$mask42,$D2lo,$H2
-	vpaddq		$tmp,$D2hi,$D2hi
-
-	vpaddq		$D2hi,$H0,$H0
-	vpsllq		\$2,$D2hi,$D2hi
-
-	vpaddq		$D2hi,$H0,$H0
-
-	vpsrlq		\$44,$H0,$tmp		# additional step
-	vpandq		$mask44,$H0,$H0
-
-	vpaddq		$tmp,$H1,$H1
-
-	dec		%eax
-	jz		.Ldone_init_vpmadd52
-
-	vpunpcklqdq	$R1,$H1,$R1		# 1,2
-	vpbroadcastq	%x#$H1,%x#$H1		# 2,2
-	vpunpcklqdq	$R2,$H2,$R2
-	vpbroadcastq	%x#$H2,%x#$H2
-	vpunpcklqdq	$R0,$H0,$R0
-	vpbroadcastq	%x#$H0,%x#$H0
-
-	vpsllq		\$2,$R1,$S1		# S1 = R1*5*4
-	vpsllq		\$2,$R2,$S2		# S2 = R2*5*4
-	vpaddq		$R1,$S1,$S1
-	vpaddq		$R2,$S2,$S2
-	vpsllq		\$2,$S1,$S1
-	vpsllq		\$2,$S2,$S2
-
-	jmp		.Lmul_init_vpmadd52
-	ud2
-
-.align	32
-.Ldone_init_vpmadd52:
-	vinserti128	\$1,%x#$R1,$H1,$R1	# 1,2,3,4
-	vinserti128	\$1,%x#$R2,$H2,$R2
-	vinserti128	\$1,%x#$R0,$H0,$R0
-
-	vpermq		\$0b11011000,$R1,$R1	# 1,3,2,4
-	vpermq		\$0b11011000,$R2,$R2
-	vpermq		\$0b11011000,$R0,$R0
-
-	vpsllq		\$2,$R1,$S1		# S1 = R1*5*4
-	vpaddq		$R1,$S1,$S1
-	vpsllq		\$2,$S1,$S1
-
-	vmovq		0($ctx),%x#$H0		# load current hash value
-	vmovq		8($ctx),%x#$H1
-	vmovq		16($ctx),%x#$H2
-
-	test		\$3,$len		# is length 4*n+2?
-	jnz		.Ldone_init_vpmadd52_2x
-
-	vmovdqu64	$R0,64($ctx)		# save key powers
-	vpbroadcastq	%x#$R0,$R0		# broadcast 4th power
-	vmovdqu64	$R1,96($ctx)
-	vpbroadcastq	%x#$R1,$R1
-	vmovdqu64	$R2,128($ctx)
-	vpbroadcastq	%x#$R2,$R2
-	vmovdqu64	$S1,160($ctx)
-	vpbroadcastq	%x#$S1,$S1
-
-	jmp		.Lblocks_vpmadd52_4x_key_loaded
-	ud2
-
-.align	32
-.Ldone_init_vpmadd52_2x:
-	vmovdqu64	$R0,64($ctx)		# save key powers
-	vpsrldq		\$8,$R0,$R0		# 0-1-0-2
-	vmovdqu64	$R1,96($ctx)
-	vpsrldq		\$8,$R1,$R1
-	vmovdqu64	$R2,128($ctx)
-	vpsrldq		\$8,$R2,$R2
-	vmovdqu64	$S1,160($ctx)
-	vpsrldq		\$8,$S1,$S1
-	jmp		.Lblocks_vpmadd52_2x_key_loaded
-	ud2
-
-.align	32
-.Lblocks_vpmadd52_2x_do:
-	vmovdqu64	128+8($ctx),${R2}{%k1}{z}# load 2nd and 1st key powers
-	vmovdqu64	160+8($ctx),${S1}{%k1}{z}
-	vmovdqu64	64+8($ctx),${R0}{%k1}{z}
-	vmovdqu64	96+8($ctx),${R1}{%k1}{z}
-
-.Lblocks_vpmadd52_2x_key_loaded:
-	vmovdqu64	16*0($inp),$T2		# load data
-	vpxorq		$T3,$T3,$T3
-	lea		16*2($inp),$inp
-
-	vpunpcklqdq	$T3,$T2,$T1		# transpose data
-	vpunpckhqdq	$T3,$T2,$T3
-
-	# at this point 64-bit lanes are ordered as x-1-x-0
-
-	vpsrlq		\$24,$T3,$T2		# splat the data
-	vporq		$PAD,$T2,$T2
-	 vpaddq		$T2,$H2,$H2		# accumulate input
-	vpandq		$mask44,$T1,$T0
-	vpsrlq		\$44,$T1,$T1
-	vpsllq		\$20,$T3,$T3
-	vporq		$T3,$T1,$T1
-	vpandq		$mask44,$T1,$T1
-
-	jmp		.Ltail_vpmadd52_2x
-	ud2
-
-.align	32
-.Loop_vpmadd52_4x:
-	#vpaddq		$T2,$H2,$H2		# accumulate input
-	vpaddq		$T0,$H0,$H0
-	vpaddq		$T1,$H1,$H1
-
-	vpxorq		$D0lo,$D0lo,$D0lo
-	vpmadd52luq	$H2,$S1,$D0lo
-	vpxorq		$D0hi,$D0hi,$D0hi
-	vpmadd52huq	$H2,$S1,$D0hi
-	vpxorq		$D1lo,$D1lo,$D1lo
-	vpmadd52luq	$H2,$S2,$D1lo
-	vpxorq		$D1hi,$D1hi,$D1hi
-	vpmadd52huq	$H2,$S2,$D1hi
-	vpxorq		$D2lo,$D2lo,$D2lo
-	vpmadd52luq	$H2,$R0,$D2lo
-	vpxorq		$D2hi,$D2hi,$D2hi
-	vpmadd52huq	$H2,$R0,$D2hi
-
-	 vmovdqu64	16*0($inp),$T2		# load data
-	 vmovdqu64	16*2($inp),$T3
-	 lea		16*4($inp),$inp
-	vpmadd52luq	$H0,$R0,$D0lo
-	vpmadd52huq	$H0,$R0,$D0hi
-	vpmadd52luq	$H0,$R1,$D1lo
-	vpmadd52huq	$H0,$R1,$D1hi
-	vpmadd52luq	$H0,$R2,$D2lo
-	vpmadd52huq	$H0,$R2,$D2hi
-
-	 vpunpcklqdq	$T3,$T2,$T1		# transpose data
-	 vpunpckhqdq	$T3,$T2,$T3
-	vpmadd52luq	$H1,$S2,$D0lo
-	vpmadd52huq	$H1,$S2,$D0hi
-	vpmadd52luq	$H1,$R0,$D1lo
-	vpmadd52huq	$H1,$R0,$D1hi
-	vpmadd52luq	$H1,$R1,$D2lo
-	vpmadd52huq	$H1,$R1,$D2hi
-
-	################################################################
-	# partial reduction (interleaved with data splat)
-	vpsrlq		\$44,$D0lo,$tmp
-	vpsllq		\$8,$D0hi,$D0hi
-	vpandq		$mask44,$D0lo,$H0
-	vpaddq		$tmp,$D0hi,$D0hi
-
-	 vpsrlq		\$24,$T3,$T2
-	 vporq		$PAD,$T2,$T2
-	vpaddq		$D0hi,$D1lo,$D1lo
-
-	vpsrlq		\$44,$D1lo,$tmp
-	vpsllq		\$8,$D1hi,$D1hi
-	vpandq		$mask44,$D1lo,$H1
-	vpaddq		$tmp,$D1hi,$D1hi
-
-	 vpandq		$mask44,$T1,$T0
-	 vpsrlq		\$44,$T1,$T1
-	 vpsllq		\$20,$T3,$T3
-	vpaddq		$D1hi,$D2lo,$D2lo
-
-	vpsrlq		\$42,$D2lo,$tmp
-	vpsllq		\$10,$D2hi,$D2hi
-	vpandq		$mask42,$D2lo,$H2
-	vpaddq		$tmp,$D2hi,$D2hi
-
-	  vpaddq	$T2,$H2,$H2		# accumulate input
-	vpaddq		$D2hi,$H0,$H0
-	vpsllq		\$2,$D2hi,$D2hi
-
-	vpaddq		$D2hi,$H0,$H0
-	 vporq		$T3,$T1,$T1
-	 vpandq		$mask44,$T1,$T1
-
-	vpsrlq		\$44,$H0,$tmp		# additional step
-	vpandq		$mask44,$H0,$H0
-
-	vpaddq		$tmp,$H1,$H1
-
-	sub		\$4,$len		# len-=64
-	jnz		.Loop_vpmadd52_4x
-
-.Ltail_vpmadd52_4x:
-	vmovdqu64	128($ctx),$R2		# load all key powers
-	vmovdqu64	160($ctx),$S1
-	vmovdqu64	64($ctx),$R0
-	vmovdqu64	96($ctx),$R1
-
-.Ltail_vpmadd52_2x:
-	vpsllq		\$2,$R2,$S2		# S2 = R2*5*4
-	vpaddq		$R2,$S2,$S2
-	vpsllq		\$2,$S2,$S2
-
-	#vpaddq		$T2,$H2,$H2		# accumulate input
-	vpaddq		$T0,$H0,$H0
-	vpaddq		$T1,$H1,$H1
-
-	vpxorq		$D0lo,$D0lo,$D0lo
-	vpmadd52luq	$H2,$S1,$D0lo
-	vpxorq		$D0hi,$D0hi,$D0hi
-	vpmadd52huq	$H2,$S1,$D0hi
-	vpxorq		$D1lo,$D1lo,$D1lo
-	vpmadd52luq	$H2,$S2,$D1lo
-	vpxorq		$D1hi,$D1hi,$D1hi
-	vpmadd52huq	$H2,$S2,$D1hi
-	vpxorq		$D2lo,$D2lo,$D2lo
-	vpmadd52luq	$H2,$R0,$D2lo
-	vpxorq		$D2hi,$D2hi,$D2hi
-	vpmadd52huq	$H2,$R0,$D2hi
-
-	vpmadd52luq	$H0,$R0,$D0lo
-	vpmadd52huq	$H0,$R0,$D0hi
-	vpmadd52luq	$H0,$R1,$D1lo
-	vpmadd52huq	$H0,$R1,$D1hi
-	vpmadd52luq	$H0,$R2,$D2lo
-	vpmadd52huq	$H0,$R2,$D2hi
-
-	vpmadd52luq	$H1,$S2,$D0lo
-	vpmadd52huq	$H1,$S2,$D0hi
-	vpmadd52luq	$H1,$R0,$D1lo
-	vpmadd52huq	$H1,$R0,$D1hi
-	vpmadd52luq	$H1,$R1,$D2lo
-	vpmadd52huq	$H1,$R1,$D2hi
-
-	################################################################
-	# horizontal addition
-
-	mov		\$1,%eax
-	kmovw		%eax,%k1
-	vpsrldq		\$8,$D0lo,$T0
-	vpsrldq		\$8,$D0hi,$H0
-	vpsrldq		\$8,$D1lo,$T1
-	vpsrldq		\$8,$D1hi,$H1
-	vpaddq		$T0,$D0lo,$D0lo
-	vpaddq		$H0,$D0hi,$D0hi
-	vpsrldq		\$8,$D2lo,$T2
-	vpsrldq		\$8,$D2hi,$H2
-	vpaddq		$T1,$D1lo,$D1lo
-	vpaddq		$H1,$D1hi,$D1hi
-	 vpermq		\$0x2,$D0lo,$T0
-	 vpermq		\$0x2,$D0hi,$H0
-	vpaddq		$T2,$D2lo,$D2lo
-	vpaddq		$H2,$D2hi,$D2hi
-
-	vpermq		\$0x2,$D1lo,$T1
-	vpermq		\$0x2,$D1hi,$H1
-	vpaddq		$T0,$D0lo,${D0lo}{%k1}{z}
-	vpaddq		$H0,$D0hi,${D0hi}{%k1}{z}
-	vpermq		\$0x2,$D2lo,$T2
-	vpermq		\$0x2,$D2hi,$H2
-	vpaddq		$T1,$D1lo,${D1lo}{%k1}{z}
-	vpaddq		$H1,$D1hi,${D1hi}{%k1}{z}
-	vpaddq		$T2,$D2lo,${D2lo}{%k1}{z}
-	vpaddq		$H2,$D2hi,${D2hi}{%k1}{z}
-
-	################################################################
-	# partial reduction
-	vpsrlq		\$44,$D0lo,$tmp
-	vpsllq		\$8,$D0hi,$D0hi
-	vpandq		$mask44,$D0lo,$H0
-	vpaddq		$tmp,$D0hi,$D0hi
-
-	vpaddq		$D0hi,$D1lo,$D1lo
-
-	vpsrlq		\$44,$D1lo,$tmp
-	vpsllq		\$8,$D1hi,$D1hi
-	vpandq		$mask44,$D1lo,$H1
-	vpaddq		$tmp,$D1hi,$D1hi
-
-	vpaddq		$D1hi,$D2lo,$D2lo
-
-	vpsrlq		\$42,$D2lo,$tmp
-	vpsllq		\$10,$D2hi,$D2hi
-	vpandq		$mask42,$D2lo,$H2
-	vpaddq		$tmp,$D2hi,$D2hi
-
-	vpaddq		$D2hi,$H0,$H0
-	vpsllq		\$2,$D2hi,$D2hi
-
-	vpaddq		$D2hi,$H0,$H0
-
-	vpsrlq		\$44,$H0,$tmp		# additional step
-	vpandq		$mask44,$H0,$H0
-
-	vpaddq		$tmp,$H1,$H1
-						# at this point $len is
-						# either 4*n+2 or 0...
-	sub		\$2,$len		# len-=32
-	ja		.Lblocks_vpmadd52_4x_do
-
-	vmovq		%x#$H0,0($ctx)
-	vmovq		%x#$H1,8($ctx)
-	vmovq		%x#$H2,16($ctx)
-	vzeroall
-
-.Lno_data_vpmadd52_4x:
-	RET
-.size	poly1305_blocks_vpmadd52_4x,.-poly1305_blocks_vpmadd52_4x
-___
-}
-{
-########################################################################
-# As implied by its name 8x subroutine processes 8 blocks in parallel...
-# This is intermediate version, as it's used only in cases when input
-# length is either 8*n, 8*n+1 or 8*n+2...
-
-my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17));
-my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23));
-my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31));
-my ($RR0,$RR1,$RR2,$SS1,$SS2) = map("%ymm$_",(6..10));
-
-$code.=<<___;
-.type	poly1305_blocks_vpmadd52_8x,\@function,4
-.align	32
-poly1305_blocks_vpmadd52_8x:
-	shr	\$4,$len
-	jz	.Lno_data_vpmadd52_8x		# too short
-
-	shl	\$40,$padbit
-	mov	64($ctx),%r8			# peek on power of the key
-
-	vmovdqa64	.Lx_mask44(%rip),$mask44
-	vmovdqa64	.Lx_mask42(%rip),$mask42
-
-	test	%r8,%r8				# is power value impossible?
-	js	.Linit_vpmadd52			# if it is, then init R[4]
-
-	vmovq	0($ctx),%x#$H0			# load current hash value
-	vmovq	8($ctx),%x#$H1
-	vmovq	16($ctx),%x#$H2
-
-.Lblocks_vpmadd52_8x:
-	################################################################
-	# fist we calculate more key powers
-
-	vmovdqu64	128($ctx),$R2		# load 1-3-2-4 powers
-	vmovdqu64	160($ctx),$S1
-	vmovdqu64	64($ctx),$R0
-	vmovdqu64	96($ctx),$R1
-
-	vpsllq		\$2,$R2,$S2		# S2 = R2*5*4
-	vpaddq		$R2,$S2,$S2
-	vpsllq		\$2,$S2,$S2
-
-	vpbroadcastq	%x#$R2,$RR2		# broadcast 4th power
-	vpbroadcastq	%x#$R0,$RR0
-	vpbroadcastq	%x#$R1,$RR1
-
-	vpxorq		$D0lo,$D0lo,$D0lo
-	vpmadd52luq	$RR2,$S1,$D0lo
-	vpxorq		$D0hi,$D0hi,$D0hi
-	vpmadd52huq	$RR2,$S1,$D0hi
-	vpxorq		$D1lo,$D1lo,$D1lo
-	vpmadd52luq	$RR2,$S2,$D1lo
-	vpxorq		$D1hi,$D1hi,$D1hi
-	vpmadd52huq	$RR2,$S2,$D1hi
-	vpxorq		$D2lo,$D2lo,$D2lo
-	vpmadd52luq	$RR2,$R0,$D2lo
-	vpxorq		$D2hi,$D2hi,$D2hi
-	vpmadd52huq	$RR2,$R0,$D2hi
-
-	vpmadd52luq	$RR0,$R0,$D0lo
-	vpmadd52huq	$RR0,$R0,$D0hi
-	vpmadd52luq	$RR0,$R1,$D1lo
-	vpmadd52huq	$RR0,$R1,$D1hi
-	vpmadd52luq	$RR0,$R2,$D2lo
-	vpmadd52huq	$RR0,$R2,$D2hi
-
-	vpmadd52luq	$RR1,$S2,$D0lo
-	vpmadd52huq	$RR1,$S2,$D0hi
-	vpmadd52luq	$RR1,$R0,$D1lo
-	vpmadd52huq	$RR1,$R0,$D1hi
-	vpmadd52luq	$RR1,$R1,$D2lo
-	vpmadd52huq	$RR1,$R1,$D2hi
-
-	################################################################
-	# partial reduction
-	vpsrlq		\$44,$D0lo,$tmp
-	vpsllq		\$8,$D0hi,$D0hi
-	vpandq		$mask44,$D0lo,$RR0
-	vpaddq		$tmp,$D0hi,$D0hi
-
-	vpaddq		$D0hi,$D1lo,$D1lo
-
-	vpsrlq		\$44,$D1lo,$tmp
-	vpsllq		\$8,$D1hi,$D1hi
-	vpandq		$mask44,$D1lo,$RR1
-	vpaddq		$tmp,$D1hi,$D1hi
-
-	vpaddq		$D1hi,$D2lo,$D2lo
-
-	vpsrlq		\$42,$D2lo,$tmp
-	vpsllq		\$10,$D2hi,$D2hi
-	vpandq		$mask42,$D2lo,$RR2
-	vpaddq		$tmp,$D2hi,$D2hi
-
-	vpaddq		$D2hi,$RR0,$RR0
-	vpsllq		\$2,$D2hi,$D2hi
-
-	vpaddq		$D2hi,$RR0,$RR0
-
-	vpsrlq		\$44,$RR0,$tmp		# additional step
-	vpandq		$mask44,$RR0,$RR0
-
-	vpaddq		$tmp,$RR1,$RR1
-
-	################################################################
-	# At this point Rx holds 1324 powers, RRx - 5768, and the goal
-	# is 15263748, which reflects how data is loaded...
-
-	vpunpcklqdq	$R2,$RR2,$T2		# 3748
-	vpunpckhqdq	$R2,$RR2,$R2		# 1526
-	vpunpcklqdq	$R0,$RR0,$T0
-	vpunpckhqdq	$R0,$RR0,$R0
-	vpunpcklqdq	$R1,$RR1,$T1
-	vpunpckhqdq	$R1,$RR1,$R1
-___
-######## switch to %zmm
-map(s/%y/%z/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2);
-map(s/%y/%z/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi);
-map(s/%y/%z/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD);
-map(s/%y/%z/, $RR0,$RR1,$RR2,$SS1,$SS2);
-
-$code.=<<___;
-	vshufi64x2	\$0x44,$R2,$T2,$RR2	# 15263748
-	vshufi64x2	\$0x44,$R0,$T0,$RR0
-	vshufi64x2	\$0x44,$R1,$T1,$RR1
-
-	vmovdqu64	16*0($inp),$T2		# load data
-	vmovdqu64	16*4($inp),$T3
-	lea		16*8($inp),$inp
-
-	vpsllq		\$2,$RR2,$SS2		# S2 = R2*5*4
-	vpsllq		\$2,$RR1,$SS1		# S1 = R1*5*4
-	vpaddq		$RR2,$SS2,$SS2
-	vpaddq		$RR1,$SS1,$SS1
-	vpsllq		\$2,$SS2,$SS2
-	vpsllq		\$2,$SS1,$SS1
-
-	vpbroadcastq	$padbit,$PAD
-	vpbroadcastq	%x#$mask44,$mask44
-	vpbroadcastq	%x#$mask42,$mask42
-
-	vpbroadcastq	%x#$SS1,$S1		# broadcast 8th power
-	vpbroadcastq	%x#$SS2,$S2
-	vpbroadcastq	%x#$RR0,$R0
-	vpbroadcastq	%x#$RR1,$R1
-	vpbroadcastq	%x#$RR2,$R2
-
-	vpunpcklqdq	$T3,$T2,$T1		# transpose data
-	vpunpckhqdq	$T3,$T2,$T3
-
-	# at this point 64-bit lanes are ordered as 73625140
-
-	vpsrlq		\$24,$T3,$T2		# splat the data
-	vporq		$PAD,$T2,$T2
-	 vpaddq		$T2,$H2,$H2		# accumulate input
-	vpandq		$mask44,$T1,$T0
-	vpsrlq		\$44,$T1,$T1
-	vpsllq		\$20,$T3,$T3
-	vporq		$T3,$T1,$T1
-	vpandq		$mask44,$T1,$T1
-
-	sub		\$8,$len
-	jz		.Ltail_vpmadd52_8x
-	jmp		.Loop_vpmadd52_8x
-
-.align	32
-.Loop_vpmadd52_8x:
-	#vpaddq		$T2,$H2,$H2		# accumulate input
-	vpaddq		$T0,$H0,$H0
-	vpaddq		$T1,$H1,$H1
-
-	vpxorq		$D0lo,$D0lo,$D0lo
-	vpmadd52luq	$H2,$S1,$D0lo
-	vpxorq		$D0hi,$D0hi,$D0hi
-	vpmadd52huq	$H2,$S1,$D0hi
-	vpxorq		$D1lo,$D1lo,$D1lo
-	vpmadd52luq	$H2,$S2,$D1lo
-	vpxorq		$D1hi,$D1hi,$D1hi
-	vpmadd52huq	$H2,$S2,$D1hi
-	vpxorq		$D2lo,$D2lo,$D2lo
-	vpmadd52luq	$H2,$R0,$D2lo
-	vpxorq		$D2hi,$D2hi,$D2hi
-	vpmadd52huq	$H2,$R0,$D2hi
-
-	 vmovdqu64	16*0($inp),$T2		# load data
-	 vmovdqu64	16*4($inp),$T3
-	 lea		16*8($inp),$inp
-	vpmadd52luq	$H0,$R0,$D0lo
-	vpmadd52huq	$H0,$R0,$D0hi
-	vpmadd52luq	$H0,$R1,$D1lo
-	vpmadd52huq	$H0,$R1,$D1hi
-	vpmadd52luq	$H0,$R2,$D2lo
-	vpmadd52huq	$H0,$R2,$D2hi
-
-	 vpunpcklqdq	$T3,$T2,$T1		# transpose data
-	 vpunpckhqdq	$T3,$T2,$T3
-	vpmadd52luq	$H1,$S2,$D0lo
-	vpmadd52huq	$H1,$S2,$D0hi
-	vpmadd52luq	$H1,$R0,$D1lo
-	vpmadd52huq	$H1,$R0,$D1hi
-	vpmadd52luq	$H1,$R1,$D2lo
-	vpmadd52huq	$H1,$R1,$D2hi
-
-	################################################################
-	# partial reduction (interleaved with data splat)
-	vpsrlq		\$44,$D0lo,$tmp
-	vpsllq		\$8,$D0hi,$D0hi
-	vpandq		$mask44,$D0lo,$H0
-	vpaddq		$tmp,$D0hi,$D0hi
-
-	 vpsrlq		\$24,$T3,$T2
-	 vporq		$PAD,$T2,$T2
-	vpaddq		$D0hi,$D1lo,$D1lo
-
-	vpsrlq		\$44,$D1lo,$tmp
-	vpsllq		\$8,$D1hi,$D1hi
-	vpandq		$mask44,$D1lo,$H1
-	vpaddq		$tmp,$D1hi,$D1hi
-
-	 vpandq		$mask44,$T1,$T0
-	 vpsrlq		\$44,$T1,$T1
-	 vpsllq		\$20,$T3,$T3
-	vpaddq		$D1hi,$D2lo,$D2lo
-
-	vpsrlq		\$42,$D2lo,$tmp
-	vpsllq		\$10,$D2hi,$D2hi
-	vpandq		$mask42,$D2lo,$H2
-	vpaddq		$tmp,$D2hi,$D2hi
-
-	  vpaddq	$T2,$H2,$H2		# accumulate input
-	vpaddq		$D2hi,$H0,$H0
-	vpsllq		\$2,$D2hi,$D2hi
-
-	vpaddq		$D2hi,$H0,$H0
-	 vporq		$T3,$T1,$T1
-	 vpandq		$mask44,$T1,$T1
-
-	vpsrlq		\$44,$H0,$tmp		# additional step
-	vpandq		$mask44,$H0,$H0
-
-	vpaddq		$tmp,$H1,$H1
-
-	sub		\$8,$len		# len-=128
-	jnz		.Loop_vpmadd52_8x
-
-.Ltail_vpmadd52_8x:
-	#vpaddq		$T2,$H2,$H2		# accumulate input
-	vpaddq		$T0,$H0,$H0
-	vpaddq		$T1,$H1,$H1
-
-	vpxorq		$D0lo,$D0lo,$D0lo
-	vpmadd52luq	$H2,$SS1,$D0lo
-	vpxorq		$D0hi,$D0hi,$D0hi
-	vpmadd52huq	$H2,$SS1,$D0hi
-	vpxorq		$D1lo,$D1lo,$D1lo
-	vpmadd52luq	$H2,$SS2,$D1lo
-	vpxorq		$D1hi,$D1hi,$D1hi
-	vpmadd52huq	$H2,$SS2,$D1hi
-	vpxorq		$D2lo,$D2lo,$D2lo
-	vpmadd52luq	$H2,$RR0,$D2lo
-	vpxorq		$D2hi,$D2hi,$D2hi
-	vpmadd52huq	$H2,$RR0,$D2hi
-
-	vpmadd52luq	$H0,$RR0,$D0lo
-	vpmadd52huq	$H0,$RR0,$D0hi
-	vpmadd52luq	$H0,$RR1,$D1lo
-	vpmadd52huq	$H0,$RR1,$D1hi
-	vpmadd52luq	$H0,$RR2,$D2lo
-	vpmadd52huq	$H0,$RR2,$D2hi
-
-	vpmadd52luq	$H1,$SS2,$D0lo
-	vpmadd52huq	$H1,$SS2,$D0hi
-	vpmadd52luq	$H1,$RR0,$D1lo
-	vpmadd52huq	$H1,$RR0,$D1hi
-	vpmadd52luq	$H1,$RR1,$D2lo
-	vpmadd52huq	$H1,$RR1,$D2hi
-
-	################################################################
-	# horizontal addition
-
-	mov		\$1,%eax
-	kmovw		%eax,%k1
-	vpsrldq		\$8,$D0lo,$T0
-	vpsrldq		\$8,$D0hi,$H0
-	vpsrldq		\$8,$D1lo,$T1
-	vpsrldq		\$8,$D1hi,$H1
-	vpaddq		$T0,$D0lo,$D0lo
-	vpaddq		$H0,$D0hi,$D0hi
-	vpsrldq		\$8,$D2lo,$T2
-	vpsrldq		\$8,$D2hi,$H2
-	vpaddq		$T1,$D1lo,$D1lo
-	vpaddq		$H1,$D1hi,$D1hi
-	 vpermq		\$0x2,$D0lo,$T0
-	 vpermq		\$0x2,$D0hi,$H0
-	vpaddq		$T2,$D2lo,$D2lo
-	vpaddq		$H2,$D2hi,$D2hi
-
-	vpermq		\$0x2,$D1lo,$T1
-	vpermq		\$0x2,$D1hi,$H1
-	vpaddq		$T0,$D0lo,$D0lo
-	vpaddq		$H0,$D0hi,$D0hi
-	vpermq		\$0x2,$D2lo,$T2
-	vpermq		\$0x2,$D2hi,$H2
-	vpaddq		$T1,$D1lo,$D1lo
-	vpaddq		$H1,$D1hi,$D1hi
-	 vextracti64x4	\$1,$D0lo,%y#$T0
-	 vextracti64x4	\$1,$D0hi,%y#$H0
-	vpaddq		$T2,$D2lo,$D2lo
-	vpaddq		$H2,$D2hi,$D2hi
-
-	vextracti64x4	\$1,$D1lo,%y#$T1
-	vextracti64x4	\$1,$D1hi,%y#$H1
-	vextracti64x4	\$1,$D2lo,%y#$T2
-	vextracti64x4	\$1,$D2hi,%y#$H2
-___
-######## switch back to %ymm
-map(s/%z/%y/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2);
-map(s/%z/%y/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi);
-map(s/%z/%y/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD);
-
-$code.=<<___;
-	vpaddq		$T0,$D0lo,${D0lo}{%k1}{z}
-	vpaddq		$H0,$D0hi,${D0hi}{%k1}{z}
-	vpaddq		$T1,$D1lo,${D1lo}{%k1}{z}
-	vpaddq		$H1,$D1hi,${D1hi}{%k1}{z}
-	vpaddq		$T2,$D2lo,${D2lo}{%k1}{z}
-	vpaddq		$H2,$D2hi,${D2hi}{%k1}{z}
-
-	################################################################
-	# partial reduction
-	vpsrlq		\$44,$D0lo,$tmp
-	vpsllq		\$8,$D0hi,$D0hi
-	vpandq		$mask44,$D0lo,$H0
-	vpaddq		$tmp,$D0hi,$D0hi
-
-	vpaddq		$D0hi,$D1lo,$D1lo
-
-	vpsrlq		\$44,$D1lo,$tmp
-	vpsllq		\$8,$D1hi,$D1hi
-	vpandq		$mask44,$D1lo,$H1
-	vpaddq		$tmp,$D1hi,$D1hi
-
-	vpaddq		$D1hi,$D2lo,$D2lo
-
-	vpsrlq		\$42,$D2lo,$tmp
-	vpsllq		\$10,$D2hi,$D2hi
-	vpandq		$mask42,$D2lo,$H2
-	vpaddq		$tmp,$D2hi,$D2hi
-
-	vpaddq		$D2hi,$H0,$H0
-	vpsllq		\$2,$D2hi,$D2hi
-
-	vpaddq		$D2hi,$H0,$H0
-
-	vpsrlq		\$44,$H0,$tmp		# additional step
-	vpandq		$mask44,$H0,$H0
-
-	vpaddq		$tmp,$H1,$H1
-
-	################################################################
-
-	vmovq		%x#$H0,0($ctx)
-	vmovq		%x#$H1,8($ctx)
-	vmovq		%x#$H2,16($ctx)
-	vzeroall
-
-.Lno_data_vpmadd52_8x:
-	RET
-.size	poly1305_blocks_vpmadd52_8x,.-poly1305_blocks_vpmadd52_8x
-___
-}
-$code.=<<___;
-.type	poly1305_emit_base2_44,\@function,3
-.align	32
-poly1305_emit_base2_44:
-	mov	0($ctx),%r8	# load hash value
-	mov	8($ctx),%r9
-	mov	16($ctx),%r10
-
-	mov	%r9,%rax
-	shr	\$20,%r9
-	shl	\$44,%rax
-	mov	%r10,%rcx
-	shr	\$40,%r10
-	shl	\$24,%rcx
-
-	add	%rax,%r8
-	adc	%rcx,%r9
-	adc	\$0,%r10
-
-	mov	%r8,%rax
-	add	\$5,%r8		# compare to modulus
-	mov	%r9,%rcx
-	adc	\$0,%r9
-	adc	\$0,%r10
-	shr	\$2,%r10	# did 130-bit value overflow?
-	cmovnz	%r8,%rax
-	cmovnz	%r9,%rcx
-
-	add	0($nonce),%rax	# accumulate nonce
-	adc	8($nonce),%rcx
-	mov	%rax,0($mac)	# write result
-	mov	%rcx,8($mac)
-
-	RET
-.size	poly1305_emit_base2_44,.-poly1305_emit_base2_44
-___
-}	}	}
-}
-
-if (!$kernel)
-{	# chacha20-poly1305 helpers
-my ($out,$inp,$otp,$len)=$win64 ? ("%rcx","%rdx","%r8", "%r9") :  # Win64 order
-                                  ("%rdi","%rsi","%rdx","%rcx");  # Unix order
-$code.=<<___;
-.globl	xor128_encrypt_n_pad
-.type	xor128_encrypt_n_pad,\@abi-omnipotent
-.align	16
-xor128_encrypt_n_pad:
-	sub	$otp,$inp
-	sub	$otp,$out
-	mov	$len,%r10		# put len aside
-	shr	\$4,$len		# len / 16
-	jz	.Ltail_enc
-	nop
-.Loop_enc_xmm:
-	movdqu	($inp,$otp),%xmm0
-	pxor	($otp),%xmm0
-	movdqu	%xmm0,($out,$otp)
-	movdqa	%xmm0,($otp)
-	lea	16($otp),$otp
-	dec	$len
-	jnz	.Loop_enc_xmm
-
-	and	\$15,%r10		# len % 16
-	jz	.Ldone_enc
-
-.Ltail_enc:
-	mov	\$16,$len
-	sub	%r10,$len
-	xor	%eax,%eax
-.Loop_enc_byte:
-	mov	($inp,$otp),%al
-	xor	($otp),%al
-	mov	%al,($out,$otp)
-	mov	%al,($otp)
-	lea	1($otp),$otp
-	dec	%r10
-	jnz	.Loop_enc_byte
-
-	xor	%eax,%eax
-.Loop_enc_pad:
-	mov	%al,($otp)
-	lea	1($otp),$otp
-	dec	$len
-	jnz	.Loop_enc_pad
-
-.Ldone_enc:
-	mov	$otp,%rax
-	RET
-.size	xor128_encrypt_n_pad,.-xor128_encrypt_n_pad
-
-.globl	xor128_decrypt_n_pad
-.type	xor128_decrypt_n_pad,\@abi-omnipotent
-.align	16
-xor128_decrypt_n_pad:
-	sub	$otp,$inp
-	sub	$otp,$out
-	mov	$len,%r10		# put len aside
-	shr	\$4,$len		# len / 16
-	jz	.Ltail_dec
-	nop
-.Loop_dec_xmm:
-	movdqu	($inp,$otp),%xmm0
-	movdqa	($otp),%xmm1
-	pxor	%xmm0,%xmm1
-	movdqu	%xmm1,($out,$otp)
-	movdqa	%xmm0,($otp)
-	lea	16($otp),$otp
-	dec	$len
-	jnz	.Loop_dec_xmm
-
-	pxor	%xmm1,%xmm1
-	and	\$15,%r10		# len % 16
-	jz	.Ldone_dec
-
-.Ltail_dec:
-	mov	\$16,$len
-	sub	%r10,$len
-	xor	%eax,%eax
-	xor	%r11d,%r11d
-.Loop_dec_byte:
-	mov	($inp,$otp),%r11b
-	mov	($otp),%al
-	xor	%r11b,%al
-	mov	%al,($out,$otp)
-	mov	%r11b,($otp)
-	lea	1($otp),$otp
-	dec	%r10
-	jnz	.Loop_dec_byte
-
-	xor	%eax,%eax
-.Loop_dec_pad:
-	mov	%al,($otp)
-	lea	1($otp),$otp
-	dec	$len
-	jnz	.Loop_dec_pad
-
-.Ldone_dec:
-	mov	$otp,%rax
-	RET
-.size	xor128_decrypt_n_pad,.-xor128_decrypt_n_pad
-___
-}
-
-# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
-#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
-if ($win64) {
-$rec="%rcx";
-$frame="%rdx";
-$context="%r8";
-$disp="%r9";
-
-$code.=<<___;
-.extern	__imp_RtlVirtualUnwind
-.type	se_handler,\@abi-omnipotent
-.align	16
-se_handler:
-	push	%rsi
-	push	%rdi
-	push	%rbx
-	push	%rbp
-	push	%r12
-	push	%r13
-	push	%r14
-	push	%r15
-	pushfq
-	sub	\$64,%rsp
-
-	mov	120($context),%rax	# pull context->Rax
-	mov	248($context),%rbx	# pull context->Rip
-
-	mov	8($disp),%rsi		# disp->ImageBase
-	mov	56($disp),%r11		# disp->HandlerData
-
-	mov	0(%r11),%r10d		# HandlerData[0]
-	lea	(%rsi,%r10),%r10	# prologue label
-	cmp	%r10,%rbx		# context->Rip<.Lprologue
-	jb	.Lcommon_seh_tail
-
-	mov	152($context),%rax	# pull context->Rsp
-
-	mov	4(%r11),%r10d		# HandlerData[1]
-	lea	(%rsi,%r10),%r10	# epilogue label
-	cmp	%r10,%rbx		# context->Rip>=.Lepilogue
-	jae	.Lcommon_seh_tail
-
-	lea	48(%rax),%rax
-
-	mov	-8(%rax),%rbx
-	mov	-16(%rax),%rbp
-	mov	-24(%rax),%r12
-	mov	-32(%rax),%r13
-	mov	-40(%rax),%r14
-	mov	-48(%rax),%r15
-	mov	%rbx,144($context)	# restore context->Rbx
-	mov	%rbp,160($context)	# restore context->Rbp
-	mov	%r12,216($context)	# restore context->R12
-	mov	%r13,224($context)	# restore context->R13
-	mov	%r14,232($context)	# restore context->R14
-	mov	%r15,240($context)	# restore context->R14
-
-	jmp	.Lcommon_seh_tail
-.size	se_handler,.-se_handler
-
-.type	avx_handler,\@abi-omnipotent
-.align	16
-avx_handler:
-	push	%rsi
-	push	%rdi
-	push	%rbx
-	push	%rbp
-	push	%r12
-	push	%r13
-	push	%r14
-	push	%r15
-	pushfq
-	sub	\$64,%rsp
-
-	mov	120($context),%rax	# pull context->Rax
-	mov	248($context),%rbx	# pull context->Rip
-
-	mov	8($disp),%rsi		# disp->ImageBase
-	mov	56($disp),%r11		# disp->HandlerData
-
-	mov	0(%r11),%r10d		# HandlerData[0]
-	lea	(%rsi,%r10),%r10	# prologue label
-	cmp	%r10,%rbx		# context->Rip<prologue label
-	jb	.Lcommon_seh_tail
-
-	mov	152($context),%rax	# pull context->Rsp
-
-	mov	4(%r11),%r10d		# HandlerData[1]
-	lea	(%rsi,%r10),%r10	# epilogue label
-	cmp	%r10,%rbx		# context->Rip>=epilogue label
-	jae	.Lcommon_seh_tail
-
-	mov	208($context),%rax	# pull context->R11
-
-	lea	0x50(%rax),%rsi
-	lea	0xf8(%rax),%rax
-	lea	512($context),%rdi	# &context.Xmm6
-	mov	\$20,%ecx
-	.long	0xa548f3fc		# cld; rep movsq
-
-.Lcommon_seh_tail:
-	mov	8(%rax),%rdi
-	mov	16(%rax),%rsi
-	mov	%rax,152($context)	# restore context->Rsp
-	mov	%rsi,168($context)	# restore context->Rsi
-	mov	%rdi,176($context)	# restore context->Rdi
-
-	mov	40($disp),%rdi		# disp->ContextRecord
-	mov	$context,%rsi		# context
-	mov	\$154,%ecx		# sizeof(CONTEXT)
-	.long	0xa548f3fc		# cld; rep movsq
-
-	mov	$disp,%rsi
-	xor	%ecx,%ecx		# arg1, UNW_FLAG_NHANDLER
-	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
-	mov	0(%rsi),%r8		# arg3, disp->ControlPc
-	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
-	mov	40(%rsi),%r10		# disp->ContextRecord
-	lea	56(%rsi),%r11		# &disp->HandlerData
-	lea	24(%rsi),%r12		# &disp->EstablisherFrame
-	mov	%r10,32(%rsp)		# arg5
-	mov	%r11,40(%rsp)		# arg6
-	mov	%r12,48(%rsp)		# arg7
-	mov	%rcx,56(%rsp)		# arg8, (NULL)
-	call	*__imp_RtlVirtualUnwind(%rip)
-
-	mov	\$1,%eax		# ExceptionContinueSearch
-	add	\$64,%rsp
-	popfq
-	pop	%r15
-	pop	%r14
-	pop	%r13
-	pop	%r12
-	pop	%rbp
-	pop	%rbx
-	pop	%rdi
-	pop	%rsi
-	RET
-.size	avx_handler,.-avx_handler
-
-.section	.pdata
-.align	4
-	.rva	.LSEH_begin_poly1305_init_x86_64
-	.rva	.LSEH_end_poly1305_init_x86_64
-	.rva	.LSEH_info_poly1305_init_x86_64
-
-	.rva	.LSEH_begin_poly1305_blocks_x86_64
-	.rva	.LSEH_end_poly1305_blocks_x86_64
-	.rva	.LSEH_info_poly1305_blocks_x86_64
-
-	.rva	.LSEH_begin_poly1305_emit_x86_64
-	.rva	.LSEH_end_poly1305_emit_x86_64
-	.rva	.LSEH_info_poly1305_emit_x86_64
-___
-$code.=<<___ if ($avx);
-	.rva	.LSEH_begin_poly1305_blocks_avx
-	.rva	.Lbase2_64_avx
-	.rva	.LSEH_info_poly1305_blocks_avx_1
-
-	.rva	.Lbase2_64_avx
-	.rva	.Leven_avx
-	.rva	.LSEH_info_poly1305_blocks_avx_2
-
-	.rva	.Leven_avx
-	.rva	.LSEH_end_poly1305_blocks_avx
-	.rva	.LSEH_info_poly1305_blocks_avx_3
-
-	.rva	.LSEH_begin_poly1305_emit_avx
-	.rva	.LSEH_end_poly1305_emit_avx
-	.rva	.LSEH_info_poly1305_emit_avx
-___
-$code.=<<___ if ($avx>1);
-	.rva	.LSEH_begin_poly1305_blocks_avx2
-	.rva	.Lbase2_64_avx2
-	.rva	.LSEH_info_poly1305_blocks_avx2_1
-
-	.rva	.Lbase2_64_avx2
-	.rva	.Leven_avx2
-	.rva	.LSEH_info_poly1305_blocks_avx2_2
-
-	.rva	.Leven_avx2
-	.rva	.LSEH_end_poly1305_blocks_avx2
-	.rva	.LSEH_info_poly1305_blocks_avx2_3
-___
-$code.=<<___ if ($avx>2);
-	.rva	.LSEH_begin_poly1305_blocks_avx512
-	.rva	.LSEH_end_poly1305_blocks_avx512
-	.rva	.LSEH_info_poly1305_blocks_avx512
-___
-$code.=<<___;
-.section	.xdata
-.align	8
-.LSEH_info_poly1305_init_x86_64:
-	.byte	9,0,0,0
-	.rva	se_handler
-	.rva	.LSEH_begin_poly1305_init_x86_64,.LSEH_begin_poly1305_init_x86_64
-
-.LSEH_info_poly1305_blocks_x86_64:
-	.byte	9,0,0,0
-	.rva	se_handler
-	.rva	.Lblocks_body,.Lblocks_epilogue
-
-.LSEH_info_poly1305_emit_x86_64:
-	.byte	9,0,0,0
-	.rva	se_handler
-	.rva	.LSEH_begin_poly1305_emit_x86_64,.LSEH_begin_poly1305_emit_x86_64
-___
-$code.=<<___ if ($avx);
-.LSEH_info_poly1305_blocks_avx_1:
-	.byte	9,0,0,0
-	.rva	se_handler
-	.rva	.Lblocks_avx_body,.Lblocks_avx_epilogue		# HandlerData[]
-
-.LSEH_info_poly1305_blocks_avx_2:
-	.byte	9,0,0,0
-	.rva	se_handler
-	.rva	.Lbase2_64_avx_body,.Lbase2_64_avx_epilogue	# HandlerData[]
-
-.LSEH_info_poly1305_blocks_avx_3:
-	.byte	9,0,0,0
-	.rva	avx_handler
-	.rva	.Ldo_avx_body,.Ldo_avx_epilogue			# HandlerData[]
-
-.LSEH_info_poly1305_emit_avx:
-	.byte	9,0,0,0
-	.rva	se_handler
-	.rva	.LSEH_begin_poly1305_emit_avx,.LSEH_begin_poly1305_emit_avx
-___
-$code.=<<___ if ($avx>1);
-.LSEH_info_poly1305_blocks_avx2_1:
-	.byte	9,0,0,0
-	.rva	se_handler
-	.rva	.Lblocks_avx2_body,.Lblocks_avx2_epilogue	# HandlerData[]
-
-.LSEH_info_poly1305_blocks_avx2_2:
-	.byte	9,0,0,0
-	.rva	se_handler
-	.rva	.Lbase2_64_avx2_body,.Lbase2_64_avx2_epilogue	# HandlerData[]
-
-.LSEH_info_poly1305_blocks_avx2_3:
-	.byte	9,0,0,0
-	.rva	avx_handler
-	.rva	.Ldo_avx2_body,.Ldo_avx2_epilogue		# HandlerData[]
-___
-$code.=<<___ if ($avx>2);
-.LSEH_info_poly1305_blocks_avx512:
-	.byte	9,0,0,0
-	.rva	avx_handler
-	.rva	.Ldo_avx512_body,.Ldo_avx512_epilogue		# HandlerData[]
-___
-}
-
-open SELF,$0;
-while(<SELF>) {
-	next if (/^#!/);
-	last if (!s/^#/\/\// and !/^$/);
-	print;
-}
-close SELF;
-
-foreach (split('\n',$code)) {
-	s/\`([^\`]*)\`/eval($1)/ge;
-	s/%r([a-z]+)#d/%e$1/g;
-	s/%r([0-9]+)#d/%r$1d/g;
-	s/%x#%[yz]/%x/g or s/%y#%z/%y/g or s/%z#%[yz]/%z/g;
-
-	if ($kernel) {
-		s/(^\.type.*),[0-9]+$/\1/;
-		s/(^\.type.*),\@abi-omnipotent+$/\1,\@function/;
-		next if /^\.cfi.*/;
-	}
-
-	print $_,"\n";
-}
-close STDOUT;
diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c
deleted file mode 100644
index 08ff4b489f7e..000000000000
--- a/arch/x86/crypto/poly1305_glue.c
+++ /dev/null
@@ -1,290 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0 OR MIT
-/*
- * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
- */
-
-#include <crypto/algapi.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/poly1305.h>
-#include <crypto/internal/simd.h>
-#include <linux/crypto.h>
-#include <linux/jump_label.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/sizes.h>
-#include <asm/cpu_device_id.h>
-#include <asm/simd.h>
-
-asmlinkage void poly1305_init_x86_64(void *ctx,
-				     const u8 key[POLY1305_BLOCK_SIZE]);
-asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp,
-				       const size_t len, const u32 padbit);
-asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_DIGEST_SIZE],
-				     const u32 nonce[4]);
-asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_DIGEST_SIZE],
-				  const u32 nonce[4]);
-asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len,
-				    const u32 padbit);
-asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len,
-				     const u32 padbit);
-asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp,
-				       const size_t len, const u32 padbit);
-
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx);
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2);
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx512);
-
-struct poly1305_arch_internal {
-	union {
-		struct {
-			u32 h[5];
-			u32 is_base2_26;
-		};
-		u64 hs[3];
-	};
-	u64 r[2];
-	u64 pad;
-	struct { u32 r2, r1, r4, r3; } rn[9];
-};
-
-/* The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit
- * the unfortunate situation of using AVX and then having to go back to scalar
- * -- because the user is silly and has called the update function from two
- * separate contexts -- then we need to convert back to the original base before
- * proceeding. It is possible to reason that the initial reduction below is
- * sufficient given the implementation invariants. However, for an avoidance of
- * doubt and because this is not performance critical, we do the full reduction
- * anyway. Z3 proof of below function: https://xn--4db.cc/ltPtHCKN/py
- */
-static void convert_to_base2_64(void *ctx)
-{
-	struct poly1305_arch_internal *state = ctx;
-	u32 cy;
-
-	if (!state->is_base2_26)
-		return;
-
-	cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy;
-	cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy;
-	cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy;
-	cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy;
-	state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0];
-	state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12);
-	state->hs[2] = state->h[4] >> 24;
-#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
-	cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL);
-	state->hs[2] &= 3;
-	state->hs[0] += cy;
-	state->hs[1] += (cy = ULT(state->hs[0], cy));
-	state->hs[2] += ULT(state->hs[1], cy);
-#undef ULT
-	state->is_base2_26 = 0;
-}
-
-static void poly1305_simd_init(void *ctx, const u8 key[POLY1305_BLOCK_SIZE])
-{
-	poly1305_init_x86_64(ctx, key);
-}
-
-static void poly1305_simd_blocks(void *ctx, const u8 *inp, size_t len,
-				 const u32 padbit)
-{
-	struct poly1305_arch_internal *state = ctx;
-
-	/* SIMD disables preemption, so relax after processing each page. */
-	BUILD_BUG_ON(SZ_4K < POLY1305_BLOCK_SIZE ||
-		     SZ_4K % POLY1305_BLOCK_SIZE);
-
-	if (!static_branch_likely(&poly1305_use_avx) ||
-	    (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) ||
-	    !crypto_simd_usable()) {
-		convert_to_base2_64(ctx);
-		poly1305_blocks_x86_64(ctx, inp, len, padbit);
-		return;
-	}
-
-	do {
-		const size_t bytes = min_t(size_t, len, SZ_4K);
-
-		kernel_fpu_begin();
-		if (IS_ENABLED(CONFIG_AS_AVX512) && static_branch_likely(&poly1305_use_avx512))
-			poly1305_blocks_avx512(ctx, inp, bytes, padbit);
-		else if (static_branch_likely(&poly1305_use_avx2))
-			poly1305_blocks_avx2(ctx, inp, bytes, padbit);
-		else
-			poly1305_blocks_avx(ctx, inp, bytes, padbit);
-		kernel_fpu_end();
-
-		len -= bytes;
-		inp += bytes;
-	} while (len);
-}
-
-static void poly1305_simd_emit(void *ctx, u8 mac[POLY1305_DIGEST_SIZE],
-			       const u32 nonce[4])
-{
-	if (!static_branch_likely(&poly1305_use_avx))
-		poly1305_emit_x86_64(ctx, mac, nonce);
-	else
-		poly1305_emit_avx(ctx, mac, nonce);
-}
-
-void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
-{
-	poly1305_simd_init(&dctx->h, key);
-	dctx->s[0] = get_unaligned_le32(&key[16]);
-	dctx->s[1] = get_unaligned_le32(&key[20]);
-	dctx->s[2] = get_unaligned_le32(&key[24]);
-	dctx->s[3] = get_unaligned_le32(&key[28]);
-	dctx->buflen = 0;
-	dctx->sset = true;
-}
-EXPORT_SYMBOL(poly1305_init_arch);
-
-static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx,
-					       const u8 *inp, unsigned int len)
-{
-	unsigned int acc = 0;
-	if (unlikely(!dctx->sset)) {
-		if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) {
-			poly1305_simd_init(&dctx->h, inp);
-			inp += POLY1305_BLOCK_SIZE;
-			len -= POLY1305_BLOCK_SIZE;
-			acc += POLY1305_BLOCK_SIZE;
-			dctx->rset = 1;
-		}
-		if (len >= POLY1305_BLOCK_SIZE) {
-			dctx->s[0] = get_unaligned_le32(&inp[0]);
-			dctx->s[1] = get_unaligned_le32(&inp[4]);
-			dctx->s[2] = get_unaligned_le32(&inp[8]);
-			dctx->s[3] = get_unaligned_le32(&inp[12]);
-			acc += POLY1305_BLOCK_SIZE;
-			dctx->sset = true;
-		}
-	}
-	return acc;
-}
-
-void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
-			  unsigned int srclen)
-{
-	unsigned int bytes, used;
-
-	if (unlikely(dctx->buflen)) {
-		bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);
-		memcpy(dctx->buf + dctx->buflen, src, bytes);
-		src += bytes;
-		srclen -= bytes;
-		dctx->buflen += bytes;
-
-		if (dctx->buflen == POLY1305_BLOCK_SIZE) {
-			if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, POLY1305_BLOCK_SIZE)))
-				poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1);
-			dctx->buflen = 0;
-		}
-	}
-
-	if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
-		bytes = round_down(srclen, POLY1305_BLOCK_SIZE);
-		srclen -= bytes;
-		used = crypto_poly1305_setdctxkey(dctx, src, bytes);
-		if (likely(bytes - used))
-			poly1305_simd_blocks(&dctx->h, src + used, bytes - used, 1);
-		src += bytes;
-	}
-
-	if (unlikely(srclen)) {
-		dctx->buflen = srclen;
-		memcpy(dctx->buf, src, srclen);
-	}
-}
-EXPORT_SYMBOL(poly1305_update_arch);
-
-void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
-{
-	if (unlikely(dctx->buflen)) {
-		dctx->buf[dctx->buflen++] = 1;
-		memset(dctx->buf + dctx->buflen, 0,
-		       POLY1305_BLOCK_SIZE - dctx->buflen);
-		poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
-	}
-
-	poly1305_simd_emit(&dctx->h, dst, dctx->s);
-	memzero_explicit(dctx, sizeof(*dctx));
-}
-EXPORT_SYMBOL(poly1305_final_arch);
-
-static int crypto_poly1305_init(struct shash_desc *desc)
-{
-	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
-
-	*dctx = (struct poly1305_desc_ctx){};
-	return 0;
-}
-
-static int crypto_poly1305_update(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen)
-{
-	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
-
-	poly1305_update_arch(dctx, src, srclen);
-	return 0;
-}
-
-static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst)
-{
-	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
-
-	if (unlikely(!dctx->sset))
-		return -ENOKEY;
-
-	poly1305_final_arch(dctx, dst);
-	return 0;
-}
-
-static struct shash_alg alg = {
-	.digestsize	= POLY1305_DIGEST_SIZE,
-	.init		= crypto_poly1305_init,
-	.update		= crypto_poly1305_update,
-	.final		= crypto_poly1305_final,
-	.descsize	= sizeof(struct poly1305_desc_ctx),
-	.base		= {
-		.cra_name		= "poly1305",
-		.cra_driver_name	= "poly1305-simd",
-		.cra_priority		= 300,
-		.cra_blocksize		= POLY1305_BLOCK_SIZE,
-		.cra_module		= THIS_MODULE,
-	},
-};
-
-static int __init poly1305_simd_mod_init(void)
-{
-	if (boot_cpu_has(X86_FEATURE_AVX) &&
-	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
-		static_branch_enable(&poly1305_use_avx);
-	if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) &&
-	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
-		static_branch_enable(&poly1305_use_avx2);
-	if (IS_ENABLED(CONFIG_AS_AVX512) && boot_cpu_has(X86_FEATURE_AVX) &&
-	    boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) &&
-	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_AVX512, NULL) &&
-	    /* Skylake downclocks unacceptably much when using zmm, but later generations are fast. */
-	    boot_cpu_data.x86_vfm != INTEL_SKYLAKE_X)
-		static_branch_enable(&poly1305_use_avx512);
-	return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? crypto_register_shash(&alg) : 0;
-}
-
-static void __exit poly1305_simd_mod_exit(void)
-{
-	if (IS_REACHABLE(CONFIG_CRYPTO_HASH))
-		crypto_unregister_shash(&alg);
-}
-
-module_init(poly1305_simd_mod_init);
-module_exit(poly1305_simd_mod_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
-MODULE_DESCRIPTION("Poly1305 authenticator");
-MODULE_ALIAS_CRYPTO("poly1305");
-MODULE_ALIAS_CRYPTO("poly1305-simd");
diff --git a/arch/x86/crypto/polyval-clmulni_asm.S b/arch/x86/crypto/polyval-clmulni_asm.S
deleted file mode 100644
index a6ebe4e7dd2b..000000000000
--- a/arch/x86/crypto/polyval-clmulni_asm.S
+++ /dev/null
@@ -1,321 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright 2021 Google LLC
- */
-/*
- * This is an efficient implementation of POLYVAL using intel PCLMULQDQ-NI
- * instructions. It works on 8 blocks at a time, by precomputing the first 8
- * keys powers h^8, ..., h^1 in the POLYVAL finite field. This precomputation
- * allows us to split finite field multiplication into two steps.
- *
- * In the first step, we consider h^i, m_i as normal polynomials of degree less
- * than 128. We then compute p(x) = h^8m_0 + ... + h^1m_7 where multiplication
- * is simply polynomial multiplication.
- *
- * In the second step, we compute the reduction of p(x) modulo the finite field
- * modulus g(x) = x^128 + x^127 + x^126 + x^121 + 1.
- *
- * This two step process is equivalent to computing h^8m_0 + ... + h^1m_7 where
- * multiplication is finite field multiplication. The advantage is that the
- * two-step process  only requires 1 finite field reduction for every 8
- * polynomial multiplications. Further parallelism is gained by interleaving the
- * multiplications and polynomial reductions.
- */
-
-#include <linux/linkage.h>
-#include <asm/frame.h>
-
-#define STRIDE_BLOCKS 8
-
-#define GSTAR %xmm7
-#define PL %xmm8
-#define PH %xmm9
-#define TMP_XMM %xmm11
-#define LO %xmm12
-#define HI %xmm13
-#define MI %xmm14
-#define SUM %xmm15
-
-#define KEY_POWERS %rdi
-#define MSG %rsi
-#define BLOCKS_LEFT %rdx
-#define ACCUMULATOR %rcx
-#define TMP %rax
-
-.section    .rodata.cst16.gstar, "aM", @progbits, 16
-.align 16
-
-.Lgstar:
-	.quad 0xc200000000000000, 0xc200000000000000
-
-.text
-
-/*
- * Performs schoolbook1_iteration on two lists of 128-bit polynomials of length
- * count pointed to by MSG and KEY_POWERS.
- */
-.macro schoolbook1 count
-	.set i, 0
-	.rept (\count)
-		schoolbook1_iteration i 0
-		.set i, (i +1)
-	.endr
-.endm
-
-/*
- * Computes the product of two 128-bit polynomials at the memory locations
- * specified by (MSG + 16*i) and (KEY_POWERS + 16*i) and XORs the components of
- * the 256-bit product into LO, MI, HI.
- *
- * Given:
- *   X = [X_1 : X_0]
- *   Y = [Y_1 : Y_0]
- *
- * We compute:
- *   LO += X_0 * Y_0
- *   MI += X_0 * Y_1 + X_1 * Y_0
- *   HI += X_1 * Y_1
- *
- * Later, the 256-bit result can be extracted as:
- *   [HI_1 : HI_0 + MI_1 : LO_1 + MI_0 : LO_0]
- * This step is done when computing the polynomial reduction for efficiency
- * reasons.
- *
- * If xor_sum == 1, then also XOR the value of SUM into m_0.  This avoids an
- * extra multiplication of SUM and h^8.
- */
-.macro schoolbook1_iteration i xor_sum
-	movups (16*\i)(MSG), %xmm0
-	.if (\i == 0 && \xor_sum == 1)
-		pxor SUM, %xmm0
-	.endif
-	vpclmulqdq $0x01, (16*\i)(KEY_POWERS), %xmm0, %xmm2
-	vpclmulqdq $0x00, (16*\i)(KEY_POWERS), %xmm0, %xmm1
-	vpclmulqdq $0x10, (16*\i)(KEY_POWERS), %xmm0, %xmm3
-	vpclmulqdq $0x11, (16*\i)(KEY_POWERS), %xmm0, %xmm4
-	vpxor %xmm2, MI, MI
-	vpxor %xmm1, LO, LO
-	vpxor %xmm4, HI, HI
-	vpxor %xmm3, MI, MI
-.endm
-
-/*
- * Performs the same computation as schoolbook1_iteration, except we expect the
- * arguments to already be loaded into xmm0 and xmm1 and we set the result
- * registers LO, MI, and HI directly rather than XOR'ing into them.
- */
-.macro schoolbook1_noload
-	vpclmulqdq $0x01, %xmm0, %xmm1, MI
-	vpclmulqdq $0x10, %xmm0, %xmm1, %xmm2
-	vpclmulqdq $0x00, %xmm0, %xmm1, LO
-	vpclmulqdq $0x11, %xmm0, %xmm1, HI
-	vpxor %xmm2, MI, MI
-.endm
-
-/*
- * Computes the 256-bit polynomial represented by LO, HI, MI. Stores
- * the result in PL, PH.
- *   [PH : PL] = [HI_1 : HI_0 + MI_1 : LO_1 + MI_0 : LO_0]
- */
-.macro schoolbook2
-	vpslldq $8, MI, PL
-	vpsrldq $8, MI, PH
-	pxor LO, PL
-	pxor HI, PH
-.endm
-
-/*
- * Computes the 128-bit reduction of PH : PL. Stores the result in dest.
- *
- * This macro computes p(x) mod g(x) where p(x) is in montgomery form and g(x) =
- * x^128 + x^127 + x^126 + x^121 + 1.
- *
- * We have a 256-bit polynomial PH : PL = P_3 : P_2 : P_1 : P_0 that is the
- * product of two 128-bit polynomials in Montgomery form.  We need to reduce it
- * mod g(x).  Also, since polynomials in Montgomery form have an "extra" factor
- * of x^128, this product has two extra factors of x^128.  To get it back into
- * Montgomery form, we need to remove one of these factors by dividing by x^128.
- *
- * To accomplish both of these goals, we add multiples of g(x) that cancel out
- * the low 128 bits P_1 : P_0, leaving just the high 128 bits. Since the low
- * bits are zero, the polynomial division by x^128 can be done by right shifting.
- *
- * Since the only nonzero term in the low 64 bits of g(x) is the constant term,
- * the multiple of g(x) needed to cancel out P_0 is P_0 * g(x).  The CPU can
- * only do 64x64 bit multiplications, so split P_0 * g(x) into x^128 * P_0 +
- * x^64 * g*(x) * P_0 + P_0, where g*(x) is bits 64-127 of g(x).  Adding this to
- * the original polynomial gives P_3 : P_2 + P_0 + T_1 : P_1 + T_0 : 0, where T
- * = T_1 : T_0 = g*(x) * P_0.  Thus, bits 0-63 got "folded" into bits 64-191.
- *
- * Repeating this same process on the next 64 bits "folds" bits 64-127 into bits
- * 128-255, giving the answer in bits 128-255. This time, we need to cancel P_1
- * + T_0 in bits 64-127. The multiple of g(x) required is (P_1 + T_0) * g(x) *
- * x^64. Adding this to our previous computation gives P_3 + P_1 + T_0 + V_1 :
- * P_2 + P_0 + T_1 + V_0 : 0 : 0, where V = V_1 : V_0 = g*(x) * (P_1 + T_0).
- *
- * So our final computation is:
- *   T = T_1 : T_0 = g*(x) * P_0
- *   V = V_1 : V_0 = g*(x) * (P_1 + T_0)
- *   p(x) / x^{128} mod g(x) = P_3 + P_1 + T_0 + V_1 : P_2 + P_0 + T_1 + V_0
- *
- * The implementation below saves a XOR instruction by computing P_1 + T_0 : P_0
- * + T_1 and XORing into dest, rather than separately XORing P_1 : P_0 and T_0 :
- * T_1 into dest.  This allows us to reuse P_1 + T_0 when computing V.
- */
-.macro montgomery_reduction dest
-	vpclmulqdq $0x00, PL, GSTAR, TMP_XMM	# TMP_XMM = T_1 : T_0 = P_0 * g*(x)
-	pshufd $0b01001110, TMP_XMM, TMP_XMM	# TMP_XMM = T_0 : T_1
-	pxor PL, TMP_XMM			# TMP_XMM = P_1 + T_0 : P_0 + T_1
-	pxor TMP_XMM, PH			# PH = P_3 + P_1 + T_0 : P_2 + P_0 + T_1
-	pclmulqdq $0x11, GSTAR, TMP_XMM		# TMP_XMM = V_1 : V_0 = V = [(P_1 + T_0) * g*(x)]
-	vpxor TMP_XMM, PH, \dest
-.endm
-
-/*
- * Compute schoolbook multiplication for 8 blocks
- * m_0h^8 + ... + m_7h^1
- *
- * If reduce is set, also computes the montgomery reduction of the
- * previous full_stride call and XORs with the first message block.
- * (m_0 + REDUCE(PL, PH))h^8 + ... + m_7h^1.
- * I.e., the first multiplication uses m_0 + REDUCE(PL, PH) instead of m_0.
- */
-.macro full_stride reduce
-	pxor LO, LO
-	pxor HI, HI
-	pxor MI, MI
-
-	schoolbook1_iteration 7 0
-	.if \reduce
-		vpclmulqdq $0x00, PL, GSTAR, TMP_XMM
-	.endif
-
-	schoolbook1_iteration 6 0
-	.if \reduce
-		pshufd $0b01001110, TMP_XMM, TMP_XMM
-	.endif
-
-	schoolbook1_iteration 5 0
-	.if \reduce
-		pxor PL, TMP_XMM
-	.endif
-
-	schoolbook1_iteration 4 0
-	.if \reduce
-		pxor TMP_XMM, PH
-	.endif
-
-	schoolbook1_iteration 3 0
-	.if \reduce
-		pclmulqdq $0x11, GSTAR, TMP_XMM
-	.endif
-
-	schoolbook1_iteration 2 0
-	.if \reduce
-		vpxor TMP_XMM, PH, SUM
-	.endif
-
-	schoolbook1_iteration 1 0
-
-	schoolbook1_iteration 0 1
-
-	addq $(8*16), MSG
-	schoolbook2
-.endm
-
-/*
- * Process BLOCKS_LEFT blocks, where 0 < BLOCKS_LEFT < STRIDE_BLOCKS
- */
-.macro partial_stride
-	mov BLOCKS_LEFT, TMP
-	shlq $4, TMP
-	addq $(16*STRIDE_BLOCKS), KEY_POWERS
-	subq TMP, KEY_POWERS
-
-	movups (MSG), %xmm0
-	pxor SUM, %xmm0
-	movaps (KEY_POWERS), %xmm1
-	schoolbook1_noload
-	dec BLOCKS_LEFT
-	addq $16, MSG
-	addq $16, KEY_POWERS
-
-	test $4, BLOCKS_LEFT
-	jz .Lpartial4BlocksDone
-	schoolbook1 4
-	addq $(4*16), MSG
-	addq $(4*16), KEY_POWERS
-.Lpartial4BlocksDone:
-	test $2, BLOCKS_LEFT
-	jz .Lpartial2BlocksDone
-	schoolbook1 2
-	addq $(2*16), MSG
-	addq $(2*16), KEY_POWERS
-.Lpartial2BlocksDone:
-	test $1, BLOCKS_LEFT
-	jz .LpartialDone
-	schoolbook1 1
-.LpartialDone:
-	schoolbook2
-	montgomery_reduction SUM
-.endm
-
-/*
- * Perform montgomery multiplication in GF(2^128) and store result in op1.
- *
- * Computes op1*op2*x^{-128} mod x^128 + x^127 + x^126 + x^121 + 1
- * If op1, op2 are in montgomery form, this computes the montgomery
- * form of op1*op2.
- *
- * void clmul_polyval_mul(u8 *op1, const u8 *op2);
- */
-SYM_FUNC_START(clmul_polyval_mul)
-	FRAME_BEGIN
-	vmovdqa .Lgstar(%rip), GSTAR
-	movups (%rdi), %xmm0
-	movups (%rsi), %xmm1
-	schoolbook1_noload
-	schoolbook2
-	montgomery_reduction SUM
-	movups SUM, (%rdi)
-	FRAME_END
-	RET
-SYM_FUNC_END(clmul_polyval_mul)
-
-/*
- * Perform polynomial evaluation as specified by POLYVAL.  This computes:
- *	h^n * accumulator + h^n * m_0 + ... + h^1 * m_{n-1}
- * where n=nblocks, h is the hash key, and m_i are the message blocks.
- *
- * rdi - pointer to precomputed key powers h^8 ... h^1
- * rsi - pointer to message blocks
- * rdx - number of blocks to hash
- * rcx - pointer to the accumulator
- *
- * void clmul_polyval_update(const struct polyval_tfm_ctx *keys,
- *	const u8 *in, size_t nblocks, u8 *accumulator);
- */
-SYM_FUNC_START(clmul_polyval_update)
-	FRAME_BEGIN
-	vmovdqa .Lgstar(%rip), GSTAR
-	movups (ACCUMULATOR), SUM
-	subq $STRIDE_BLOCKS, BLOCKS_LEFT
-	js .LstrideLoopExit
-	full_stride 0
-	subq $STRIDE_BLOCKS, BLOCKS_LEFT
-	js .LstrideLoopExitReduce
-.LstrideLoop:
-	full_stride 1
-	subq $STRIDE_BLOCKS, BLOCKS_LEFT
-	jns .LstrideLoop
-.LstrideLoopExitReduce:
-	montgomery_reduction SUM
-.LstrideLoopExit:
-	add $STRIDE_BLOCKS, BLOCKS_LEFT
-	jz .LskipPartial
-	partial_stride
-.LskipPartial:
-	movups SUM, (ACCUMULATOR)
-	FRAME_END
-	RET
-SYM_FUNC_END(clmul_polyval_update)
diff --git a/arch/x86/crypto/polyval-clmulni_glue.c b/arch/x86/crypto/polyval-clmulni_glue.c
deleted file mode 100644
index 8fa58b0f3cb3..000000000000
--- a/arch/x86/crypto/polyval-clmulni_glue.c
+++ /dev/null
@@ -1,212 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Glue code for POLYVAL using PCMULQDQ-NI
- *
- * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
- * Copyright (c) 2009 Intel Corp.
- *   Author: Huang Ying <ying.huang@intel.com>
- * Copyright 2021 Google LLC
- */
-
-/*
- * Glue code based on ghash-clmulni-intel_glue.c.
- *
- * This implementation of POLYVAL uses montgomery multiplication
- * accelerated by PCLMULQDQ-NI to implement the finite field
- * operations.
- */
-
-#include <crypto/algapi.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/polyval.h>
-#include <linux/crypto.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <asm/cpu_device_id.h>
-#include <asm/simd.h>
-
-#define POLYVAL_ALIGN	16
-#define POLYVAL_ALIGN_ATTR __aligned(POLYVAL_ALIGN)
-#define POLYVAL_ALIGN_EXTRA ((POLYVAL_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
-#define POLYVAL_CTX_SIZE (sizeof(struct polyval_tfm_ctx) + POLYVAL_ALIGN_EXTRA)
-#define NUM_KEY_POWERS	8
-
-struct polyval_tfm_ctx {
-	/*
-	 * These powers must be in the order h^8, ..., h^1.
-	 */
-	u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE] POLYVAL_ALIGN_ATTR;
-};
-
-struct polyval_desc_ctx {
-	u8 buffer[POLYVAL_BLOCK_SIZE];
-	u32 bytes;
-};
-
-asmlinkage void clmul_polyval_update(const struct polyval_tfm_ctx *keys,
-	const u8 *in, size_t nblocks, u8 *accumulator);
-asmlinkage void clmul_polyval_mul(u8 *op1, const u8 *op2);
-
-static inline struct polyval_tfm_ctx *polyval_tfm_ctx(struct crypto_shash *tfm)
-{
-	return PTR_ALIGN(crypto_shash_ctx(tfm), POLYVAL_ALIGN);
-}
-
-static void internal_polyval_update(const struct polyval_tfm_ctx *keys,
-	const u8 *in, size_t nblocks, u8 *accumulator)
-{
-	if (likely(crypto_simd_usable())) {
-		kernel_fpu_begin();
-		clmul_polyval_update(keys, in, nblocks, accumulator);
-		kernel_fpu_end();
-	} else {
-		polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in,
-			nblocks, accumulator);
-	}
-}
-
-static void internal_polyval_mul(u8 *op1, const u8 *op2)
-{
-	if (likely(crypto_simd_usable())) {
-		kernel_fpu_begin();
-		clmul_polyval_mul(op1, op2);
-		kernel_fpu_end();
-	} else {
-		polyval_mul_non4k(op1, op2);
-	}
-}
-
-static int polyval_x86_setkey(struct crypto_shash *tfm,
-			const u8 *key, unsigned int keylen)
-{
-	struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(tfm);
-	int i;
-
-	if (keylen != POLYVAL_BLOCK_SIZE)
-		return -EINVAL;
-
-	memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE);
-
-	for (i = NUM_KEY_POWERS-2; i >= 0; i--) {
-		memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE);
-		internal_polyval_mul(tctx->key_powers[i],
-				     tctx->key_powers[i+1]);
-	}
-
-	return 0;
-}
-
-static int polyval_x86_init(struct shash_desc *desc)
-{
-	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
-
-	memset(dctx, 0, sizeof(*dctx));
-
-	return 0;
-}
-
-static int polyval_x86_update(struct shash_desc *desc,
-			 const u8 *src, unsigned int srclen)
-{
-	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
-	const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm);
-	u8 *pos;
-	unsigned int nblocks;
-	unsigned int n;
-
-	if (dctx->bytes) {
-		n = min(srclen, dctx->bytes);
-		pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes;
-
-		dctx->bytes -= n;
-		srclen -= n;
-
-		while (n--)
-			*pos++ ^= *src++;
-
-		if (!dctx->bytes)
-			internal_polyval_mul(dctx->buffer,
-					    tctx->key_powers[NUM_KEY_POWERS-1]);
-	}
-
-	while (srclen >= POLYVAL_BLOCK_SIZE) {
-		/* Allow rescheduling every 4K bytes. */
-		nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE;
-		internal_polyval_update(tctx, src, nblocks, dctx->buffer);
-		srclen -= nblocks * POLYVAL_BLOCK_SIZE;
-		src += nblocks * POLYVAL_BLOCK_SIZE;
-	}
-
-	if (srclen) {
-		dctx->bytes = POLYVAL_BLOCK_SIZE - srclen;
-		pos = dctx->buffer;
-		while (srclen--)
-			*pos++ ^= *src++;
-	}
-
-	return 0;
-}
-
-static int polyval_x86_final(struct shash_desc *desc, u8 *dst)
-{
-	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
-	const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm);
-
-	if (dctx->bytes) {
-		internal_polyval_mul(dctx->buffer,
-				     tctx->key_powers[NUM_KEY_POWERS-1]);
-	}
-
-	memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE);
-
-	return 0;
-}
-
-static struct shash_alg polyval_alg = {
-	.digestsize	= POLYVAL_DIGEST_SIZE,
-	.init		= polyval_x86_init,
-	.update		= polyval_x86_update,
-	.final		= polyval_x86_final,
-	.setkey		= polyval_x86_setkey,
-	.descsize	= sizeof(struct polyval_desc_ctx),
-	.base		= {
-		.cra_name		= "polyval",
-		.cra_driver_name	= "polyval-clmulni",
-		.cra_priority		= 200,
-		.cra_blocksize		= POLYVAL_BLOCK_SIZE,
-		.cra_ctxsize		= POLYVAL_CTX_SIZE,
-		.cra_module		= THIS_MODULE,
-	},
-};
-
-__maybe_unused static const struct x86_cpu_id pcmul_cpu_id[] = {
-	X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL),
-	{}
-};
-MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);
-
-static int __init polyval_clmulni_mod_init(void)
-{
-	if (!x86_match_cpu(pcmul_cpu_id))
-		return -ENODEV;
-
-	if (!boot_cpu_has(X86_FEATURE_AVX))
-		return -ENODEV;
-
-	return crypto_register_shash(&polyval_alg);
-}
-
-static void __exit polyval_clmulni_mod_exit(void)
-{
-	crypto_unregister_shash(&polyval_alg);
-}
-
-module_init(polyval_clmulni_mod_init);
-module_exit(polyval_clmulni_mod_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("POLYVAL hash function accelerated by PCLMULQDQ-NI");
-MODULE_ALIAS_CRYPTO("polyval");
-MODULE_ALIAS_CRYPTO("polyval-clmulni");
diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
index 97e283621851..84e47f7f6188 100644
--- a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
@@ -9,6 +9,7 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 #include <asm/frame.h>
 #include "glue_helper-asm-avx.S"
 
@@ -656,7 +657,7 @@ SYM_FUNC_START_LOCAL(__serpent_dec_blk8_avx)
 	RET;
 SYM_FUNC_END(__serpent_dec_blk8_avx)
 
-SYM_FUNC_START(serpent_ecb_enc_8way_avx)
+SYM_TYPED_FUNC_START(serpent_ecb_enc_8way_avx)
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	%rsi: dst
@@ -674,7 +675,7 @@ SYM_FUNC_START(serpent_ecb_enc_8way_avx)
 	RET;
 SYM_FUNC_END(serpent_ecb_enc_8way_avx)
 
-SYM_FUNC_START(serpent_ecb_dec_8way_avx)
+SYM_TYPED_FUNC_START(serpent_ecb_dec_8way_avx)
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	%rsi: dst
@@ -692,7 +693,7 @@ SYM_FUNC_START(serpent_ecb_dec_8way_avx)
 	RET;
 SYM_FUNC_END(serpent_ecb_dec_8way_avx)
 
-SYM_FUNC_START(serpent_cbc_dec_8way_avx)
+SYM_TYPED_FUNC_START(serpent_cbc_dec_8way_avx)
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	%rsi: dst
diff --git a/arch/x86/crypto/serpent_avx2_glue.c b/arch/x86/crypto/serpent_avx2_glue.c
index 347e97f4b713..f5f2121b7956 100644
--- a/arch/x86/crypto/serpent_avx2_glue.c
+++ b/arch/x86/crypto/serpent_avx2_glue.c
@@ -10,7 +10,6 @@
 #include <linux/crypto.h>
 #include <linux/err.h>
 #include <crypto/algapi.h>
-#include <crypto/internal/simd.h>
 #include <crypto/serpent.h>
 
 #include "serpent-avx.h"
@@ -65,10 +64,9 @@ static int cbc_decrypt(struct skcipher_request *req)
 
 static struct skcipher_alg serpent_algs[] = {
 	{
-		.base.cra_name		= "__ecb(serpent)",
-		.base.cra_driver_name	= "__ecb-serpent-avx2",
+		.base.cra_name		= "ecb(serpent)",
+		.base.cra_driver_name	= "ecb-serpent-avx2",
 		.base.cra_priority	= 600,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -78,10 +76,9 @@ static struct skcipher_alg serpent_algs[] = {
 		.encrypt		= ecb_encrypt,
 		.decrypt		= ecb_decrypt,
 	}, {
-		.base.cra_name		= "__cbc(serpent)",
-		.base.cra_driver_name	= "__cbc-serpent-avx2",
+		.base.cra_name		= "cbc(serpent)",
+		.base.cra_driver_name	= "cbc-serpent-avx2",
 		.base.cra_priority	= 600,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -94,8 +91,6 @@ static struct skcipher_alg serpent_algs[] = {
 	},
 };
 
-static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
-
 static int __init serpent_avx2_init(void)
 {
 	const char *feature_name;
@@ -110,15 +105,13 @@ static int __init serpent_avx2_init(void)
 		return -ENODEV;
 	}
 
-	return simd_register_skciphers_compat(serpent_algs,
-					      ARRAY_SIZE(serpent_algs),
-					      serpent_simd_algs);
+	return crypto_register_skciphers(serpent_algs,
+					 ARRAY_SIZE(serpent_algs));
 }
 
 static void __exit serpent_avx2_fini(void)
 {
-	simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
-				  serpent_simd_algs);
+	crypto_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs));
 }
 
 module_init(serpent_avx2_init);
diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c
index 6c248e1ea4ef..9c8b3a335d5c 100644
--- a/arch/x86/crypto/serpent_avx_glue.c
+++ b/arch/x86/crypto/serpent_avx_glue.c
@@ -12,8 +12,8 @@
 #include <linux/types.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
+#include <linux/export.h>
 #include <crypto/algapi.h>
-#include <crypto/internal/simd.h>
 #include <crypto/serpent.h>
 
 #include "serpent-avx.h"
@@ -71,10 +71,9 @@ static int cbc_decrypt(struct skcipher_request *req)
 
 static struct skcipher_alg serpent_algs[] = {
 	{
-		.base.cra_name		= "__ecb(serpent)",
-		.base.cra_driver_name	= "__ecb-serpent-avx",
+		.base.cra_name		= "ecb(serpent)",
+		.base.cra_driver_name	= "ecb-serpent-avx",
 		.base.cra_priority	= 500,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -84,10 +83,9 @@ static struct skcipher_alg serpent_algs[] = {
 		.encrypt		= ecb_encrypt,
 		.decrypt		= ecb_decrypt,
 	}, {
-		.base.cra_name		= "__cbc(serpent)",
-		.base.cra_driver_name	= "__cbc-serpent-avx",
+		.base.cra_name		= "cbc(serpent)",
+		.base.cra_driver_name	= "cbc-serpent-avx",
 		.base.cra_priority	= 500,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -100,8 +98,6 @@ static struct skcipher_alg serpent_algs[] = {
 	},
 };
 
-static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
-
 static int __init serpent_init(void)
 {
 	const char *feature_name;
@@ -112,15 +108,13 @@ static int __init serpent_init(void)
 		return -ENODEV;
 	}
 
-	return simd_register_skciphers_compat(serpent_algs,
-					      ARRAY_SIZE(serpent_algs),
-					      serpent_simd_algs);
+	return crypto_register_skciphers(serpent_algs,
+					 ARRAY_SIZE(serpent_algs));
 }
 
 static void __exit serpent_exit(void)
 {
-	simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
-				  serpent_simd_algs);
+	crypto_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs));
 }
 
 module_init(serpent_init);
diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c
index d78f37e9b2cf..80ee17ec21b4 100644
--- a/arch/x86/crypto/serpent_sse2_glue.c
+++ b/arch/x86/crypto/serpent_sse2_glue.c
@@ -18,7 +18,6 @@
 #include <linux/err.h>
 #include <crypto/algapi.h>
 #include <crypto/b128ops.h>
-#include <crypto/internal/simd.h>
 #include <crypto/serpent.h>
 
 #include "serpent-sse2.h"
@@ -74,10 +73,9 @@ static int cbc_decrypt(struct skcipher_request *req)
 
 static struct skcipher_alg serpent_algs[] = {
 	{
-		.base.cra_name		= "__ecb(serpent)",
-		.base.cra_driver_name	= "__ecb-serpent-sse2",
+		.base.cra_name		= "ecb(serpent)",
+		.base.cra_driver_name	= "ecb-serpent-sse2",
 		.base.cra_priority	= 400,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -87,10 +85,9 @@ static struct skcipher_alg serpent_algs[] = {
 		.encrypt		= ecb_encrypt,
 		.decrypt		= ecb_decrypt,
 	}, {
-		.base.cra_name		= "__cbc(serpent)",
-		.base.cra_driver_name	= "__cbc-serpent-sse2",
+		.base.cra_name		= "cbc(serpent)",
+		.base.cra_driver_name	= "cbc-serpent-sse2",
 		.base.cra_priority	= 400,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -103,8 +100,6 @@ static struct skcipher_alg serpent_algs[] = {
 	},
 };
 
-static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
-
 static int __init serpent_sse2_init(void)
 {
 	if (!boot_cpu_has(X86_FEATURE_XMM2)) {
@@ -112,15 +107,13 @@ static int __init serpent_sse2_init(void)
 		return -ENODEV;
 	}
 
-	return simd_register_skciphers_compat(serpent_algs,
-					      ARRAY_SIZE(serpent_algs),
-					      serpent_simd_algs);
+	return crypto_register_skciphers(serpent_algs,
+					 ARRAY_SIZE(serpent_algs));
 }
 
 static void __exit serpent_sse2_exit(void)
 {
-	simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
-				  serpent_simd_algs);
+	crypto_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs));
 }
 
 module_init(serpent_sse2_init);
diff --git a/arch/x86/crypto/sha1_avx2_x86_64_asm.S b/arch/x86/crypto/sha1_avx2_x86_64_asm.S
deleted file mode 100644
index 4b49bdc95265..000000000000
--- a/arch/x86/crypto/sha1_avx2_x86_64_asm.S
+++ /dev/null
@@ -1,700 +0,0 @@
-/*
- *	Implement fast SHA-1 with AVX2 instructions. (x86_64)
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Contact Information:
- * Ilya Albrekht <ilya.albrekht@intel.com>
- * Maxim Locktyukhin <maxim.locktyukhin@intel.com>
- * Ronen Zohar <ronen.zohar@intel.com>
- * Chandramouli Narayanan <mouli@linux.intel.com>
- *
- * BSD LICENSE
- *
- * Copyright(c) 2014 Intel Corporation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/*
- * SHA-1 implementation with Intel(R) AVX2 instruction set extensions.
- *
- *This implementation is based on the previous SSSE3 release:
- *Visit http://software.intel.com/en-us/articles/
- *and refer to improving-the-performance-of-the-secure-hash-algorithm-1/
- *
- *Updates 20-byte SHA-1 record at start of 'state', from 'input', for
- *even number of 'blocks' consecutive 64-byte blocks.
- *
- *extern "C" void sha1_transform_avx2(
- *	struct sha1_state *state, const u8* input, int blocks );
- */
-
-#include <linux/linkage.h>
-
-#define	CTX	%rdi	/* arg1 */
-#define BUF	%rsi	/* arg2 */
-#define CNT	%rdx	/* arg3 */
-
-#define	REG_A	%ecx
-#define	REG_B	%esi
-#define	REG_C	%edi
-#define	REG_D	%eax
-#define	REG_E	%edx
-#define	REG_TB	%ebx
-#define	REG_TA	%r12d
-#define	REG_RA	%rcx
-#define	REG_RB	%rsi
-#define	REG_RC	%rdi
-#define	REG_RD	%rax
-#define	REG_RE	%rdx
-#define	REG_RTA	%r12
-#define	REG_RTB	%rbx
-#define	REG_T1	%r11d
-#define	xmm_mov	vmovups
-#define	avx2_zeroupper	vzeroupper
-#define	RND_F1	1
-#define	RND_F2	2
-#define	RND_F3	3
-
-.macro REGALLOC
-	.set A, REG_A
-	.set B, REG_B
-	.set C, REG_C
-	.set D, REG_D
-	.set E, REG_E
-	.set TB, REG_TB
-	.set TA, REG_TA
-
-	.set RA, REG_RA
-	.set RB, REG_RB
-	.set RC, REG_RC
-	.set RD, REG_RD
-	.set RE, REG_RE
-
-	.set RTA, REG_RTA
-	.set RTB, REG_RTB
-
-	.set T1, REG_T1
-.endm
-
-#define HASH_PTR	%r9
-#define BLOCKS_CTR	%r8
-#define BUFFER_PTR	%r10
-#define BUFFER_PTR2	%r13
-
-#define PRECALC_BUF	%r14
-#define WK_BUF		%r15
-
-#define W_TMP		%xmm0
-#define WY_TMP		%ymm0
-#define WY_TMP2		%ymm9
-
-# AVX2 variables
-#define WY0		%ymm3
-#define WY4		%ymm5
-#define WY08		%ymm7
-#define WY12		%ymm8
-#define WY16		%ymm12
-#define WY20		%ymm13
-#define WY24		%ymm14
-#define WY28		%ymm15
-
-#define YMM_SHUFB_BSWAP	%ymm10
-
-/*
- * Keep 2 iterations precalculated at a time:
- *    - 80 DWORDs per iteration * 2
- */
-#define W_SIZE		(80*2*2 +16)
-
-#define WK(t)	((((t) % 80) / 4)*32 + ( (t) % 4)*4 + ((t)/80)*16 )(WK_BUF)
-#define PRECALC_WK(t)	((t)*2*2)(PRECALC_BUF)
-
-
-.macro UPDATE_HASH  hash, val
-	add	\hash, \val
-	mov	\val, \hash
-.endm
-
-.macro PRECALC_RESET_WY
-	.set WY_00, WY0
-	.set WY_04, WY4
-	.set WY_08, WY08
-	.set WY_12, WY12
-	.set WY_16, WY16
-	.set WY_20, WY20
-	.set WY_24, WY24
-	.set WY_28, WY28
-	.set WY_32, WY_00
-.endm
-
-.macro PRECALC_ROTATE_WY
-	/* Rotate macros */
-	.set WY_32, WY_28
-	.set WY_28, WY_24
-	.set WY_24, WY_20
-	.set WY_20, WY_16
-	.set WY_16, WY_12
-	.set WY_12, WY_08
-	.set WY_08, WY_04
-	.set WY_04, WY_00
-	.set WY_00, WY_32
-
-	/* Define register aliases */
-	.set WY, WY_00
-	.set WY_minus_04, WY_04
-	.set WY_minus_08, WY_08
-	.set WY_minus_12, WY_12
-	.set WY_minus_16, WY_16
-	.set WY_minus_20, WY_20
-	.set WY_minus_24, WY_24
-	.set WY_minus_28, WY_28
-	.set WY_minus_32, WY
-.endm
-
-.macro PRECALC_00_15
-	.if (i == 0) # Initialize and rotate registers
-		PRECALC_RESET_WY
-		PRECALC_ROTATE_WY
-	.endif
-
-	/* message scheduling pre-compute for rounds 0-15 */
-	.if   ((i & 7) == 0)
-		/*
-		 * blended AVX2 and ALU instruction scheduling
-		 * 1 vector iteration per 8 rounds
-		 */
-		vmovdqu (i * 2)(BUFFER_PTR), W_TMP
-	.elseif ((i & 7) == 1)
-		vinsertf128 $1, ((i-1) * 2)(BUFFER_PTR2),\
-			 WY_TMP, WY_TMP
-	.elseif ((i & 7) == 2)
-		vpshufb YMM_SHUFB_BSWAP, WY_TMP, WY
-	.elseif ((i & 7) == 4)
-		vpaddd  K_XMM + K_XMM_AR(%rip), WY, WY_TMP
-	.elseif ((i & 7) == 7)
-		vmovdqu  WY_TMP, PRECALC_WK(i&~7)
-
-		PRECALC_ROTATE_WY
-	.endif
-.endm
-
-.macro PRECALC_16_31
-	/*
-	 * message scheduling pre-compute for rounds 16-31
-	 * calculating last 32 w[i] values in 8 XMM registers
-	 * pre-calculate K+w[i] values and store to mem
-	 * for later load by ALU add instruction
-	 *
-	 * "brute force" vectorization for rounds 16-31 only
-	 * due to w[i]->w[i-3] dependency
-	 */
-	.if   ((i & 7) == 0)
-		/*
-		 * blended AVX2 and ALU instruction scheduling
-		 * 1 vector iteration per 8 rounds
-		 */
-		/* w[i-14] */
-		vpalignr	$8, WY_minus_16, WY_minus_12, WY
-		vpsrldq	$4, WY_minus_04, WY_TMP               /* w[i-3] */
-	.elseif ((i & 7) == 1)
-		vpxor	WY_minus_08, WY, WY
-		vpxor	WY_minus_16, WY_TMP, WY_TMP
-	.elseif ((i & 7) == 2)
-		vpxor	WY_TMP, WY, WY
-		vpslldq	$12, WY, WY_TMP2
-	.elseif ((i & 7) == 3)
-		vpslld	$1, WY, WY_TMP
-		vpsrld	$31, WY, WY
-	.elseif ((i & 7) == 4)
-		vpor	WY, WY_TMP, WY_TMP
-		vpslld	$2, WY_TMP2, WY
-	.elseif ((i & 7) == 5)
-		vpsrld	$30, WY_TMP2, WY_TMP2
-		vpxor	WY, WY_TMP, WY_TMP
-	.elseif ((i & 7) == 7)
-		vpxor	WY_TMP2, WY_TMP, WY
-		vpaddd  K_XMM + K_XMM_AR(%rip), WY, WY_TMP
-		vmovdqu	WY_TMP, PRECALC_WK(i&~7)
-
-		PRECALC_ROTATE_WY
-	.endif
-.endm
-
-.macro PRECALC_32_79
-	/*
-	 * in SHA-1 specification:
-	 * w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
-	 * instead we do equal:
-	 * w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
-	 * allows more efficient vectorization
-	 * since w[i]=>w[i-3] dependency is broken
-	 */
-
-	.if   ((i & 7) == 0)
-	/*
-	 * blended AVX2 and ALU instruction scheduling
-	 * 1 vector iteration per 8 rounds
-	 */
-		vpalignr	$8, WY_minus_08, WY_minus_04, WY_TMP
-	.elseif ((i & 7) == 1)
-		/* W is W_minus_32 before xor */
-		vpxor	WY_minus_28, WY, WY
-	.elseif ((i & 7) == 2)
-		vpxor	WY_minus_16, WY_TMP, WY_TMP
-	.elseif ((i & 7) == 3)
-		vpxor	WY_TMP, WY, WY
-	.elseif ((i & 7) == 4)
-		vpslld	$2, WY, WY_TMP
-	.elseif ((i & 7) == 5)
-		vpsrld	$30, WY, WY
-		vpor	WY, WY_TMP, WY
-	.elseif ((i & 7) == 7)
-		vpaddd  K_XMM + K_XMM_AR(%rip), WY, WY_TMP
-		vmovdqu	WY_TMP, PRECALC_WK(i&~7)
-
-		PRECALC_ROTATE_WY
-	.endif
-.endm
-
-.macro PRECALC r, s
-	.set i, \r
-
-	.if (i < 40)
-		.set K_XMM, 32*0
-	.elseif (i < 80)
-		.set K_XMM, 32*1
-	.elseif (i < 120)
-		.set K_XMM, 32*2
-	.else
-		.set K_XMM, 32*3
-	.endif
-
-	.if (i<32)
-		PRECALC_00_15	\s
-	.elseif (i<64)
-		PRECALC_16_31	\s
-	.elseif (i < 160)
-		PRECALC_32_79	\s
-	.endif
-.endm
-
-.macro ROTATE_STATE
-	.set T_REG, E
-	.set E, D
-	.set D, C
-	.set C, B
-	.set B, TB
-	.set TB, A
-	.set A, T_REG
-
-	.set T_REG, RE
-	.set RE, RD
-	.set RD, RC
-	.set RC, RB
-	.set RB, RTB
-	.set RTB, RA
-	.set RA, T_REG
-.endm
-
-/* Macro relies on saved ROUND_Fx */
-
-.macro RND_FUN f, r
-	.if (\f == RND_F1)
-		ROUND_F1	\r
-	.elseif (\f == RND_F2)
-		ROUND_F2	\r
-	.elseif (\f == RND_F3)
-		ROUND_F3	\r
-	.endif
-.endm
-
-.macro RR r
-	.set round_id, (\r % 80)
-
-	.if (round_id == 0)        /* Precalculate F for first round */
-		.set ROUND_FUNC, RND_F1
-		mov	B, TB
-
-		rorx	$(32-30), B, B    /* b>>>2 */
-		andn	D, TB, T1
-		and	C, TB
-		xor	T1, TB
-	.endif
-
-	RND_FUN ROUND_FUNC, \r
-	ROTATE_STATE
-
-	.if   (round_id == 18)
-		.set ROUND_FUNC, RND_F2
-	.elseif (round_id == 38)
-		.set ROUND_FUNC, RND_F3
-	.elseif (round_id == 58)
-		.set ROUND_FUNC, RND_F2
-	.endif
-
-	.set round_id, ( (\r+1) % 80)
-
-	RND_FUN ROUND_FUNC, (\r+1)
-	ROTATE_STATE
-.endm
-
-.macro ROUND_F1 r
-	add	WK(\r), E
-
-	andn	C, A, T1			/* ~b&d */
-	lea	(RE,RTB), E		/* Add F from the previous round */
-
-	rorx	$(32-5), A, TA		/* T2 = A >>> 5 */
-	rorx	$(32-30),A, TB		/* b>>>2 for next round */
-
-	PRECALC	(\r)			/* msg scheduling for next 2 blocks */
-
-	/*
-	 * Calculate F for the next round
-	 * (b & c) ^ andn[b, d]
-	 */
-	and	B, A			/* b&c */
-	xor	T1, A			/* F1 = (b&c) ^ (~b&d) */
-
-	lea	(RE,RTA), E		/* E += A >>> 5 */
-.endm
-
-.macro ROUND_F2 r
-	add	WK(\r), E
-	lea	(RE,RTB), E		/* Add F from the previous round */
-
-	/* Calculate F for the next round */
-	rorx	$(32-5), A, TA		/* T2 = A >>> 5 */
-	.if ((round_id) < 79)
-		rorx	$(32-30), A, TB	/* b>>>2 for next round */
-	.endif
-	PRECALC	(\r)			/* msg scheduling for next 2 blocks */
-
-	.if ((round_id) < 79)
-		xor	B, A
-	.endif
-
-	add	TA, E			/* E += A >>> 5 */
-
-	.if ((round_id) < 79)
-		xor	C, A
-	.endif
-.endm
-
-.macro ROUND_F3 r
-	add	WK(\r), E
-	PRECALC	(\r)			/* msg scheduling for next 2 blocks */
-
-	lea	(RE,RTB), E		/* Add F from the previous round */
-
-	mov	B, T1
-	or	A, T1
-
-	rorx	$(32-5), A, TA		/* T2 = A >>> 5 */
-	rorx	$(32-30), A, TB		/* b>>>2 for next round */
-
-	/* Calculate F for the next round
-	 * (b and c) or (d and (b or c))
-	 */
-	and	C, T1
-	and	B, A
-	or	T1, A
-
-	add	TA, E			/* E += A >>> 5 */
-
-.endm
-
-/* Add constant only if (%2 > %3) condition met (uses RTA as temp)
- * %1 + %2 >= %3 ? %4 : 0
- */
-.macro ADD_IF_GE a, b, c, d
-	mov     \a, RTA
-	add     $\d, RTA
-	cmp     $\c, \b
-	cmovge  RTA, \a
-.endm
-
-/*
- * macro implements 80 rounds of SHA-1, for multiple blocks with s/w pipelining
- */
-.macro SHA1_PIPELINED_MAIN_BODY
-
-	REGALLOC
-
-	mov	(HASH_PTR), A
-	mov	4(HASH_PTR), B
-	mov	8(HASH_PTR), C
-	mov	12(HASH_PTR), D
-	mov	16(HASH_PTR), E
-
-	mov	%rsp, PRECALC_BUF
-	lea	(2*4*80+32)(%rsp), WK_BUF
-
-	# Precalc WK for first 2 blocks
-	ADD_IF_GE BUFFER_PTR2, BLOCKS_CTR, 2, 64
-	.set i, 0
-	.rept    160
-		PRECALC i
-		.set i, i + 1
-	.endr
-
-	/* Go to next block if needed */
-	ADD_IF_GE BUFFER_PTR, BLOCKS_CTR, 3, 128
-	ADD_IF_GE BUFFER_PTR2, BLOCKS_CTR, 4, 128
-	xchg	WK_BUF, PRECALC_BUF
-
-	.align 32
-.L_loop:
-	/*
-	 * code loops through more than one block
-	 * we use K_BASE value as a signal of a last block,
-	 * it is set below by: cmovae BUFFER_PTR, K_BASE
-	 */
-	test BLOCKS_CTR, BLOCKS_CTR
-	jnz .L_begin
-	.align 32
-	jmp	.L_end
-	.align 32
-.L_begin:
-
-	/*
-	 * Do first block
-	 * rounds: 0,2,4,6,8
-	 */
-	.set j, 0
-	.rept 5
-		RR	j
-		.set j, j+2
-	.endr
-
-	/*
-	 * rounds:
-	 * 10,12,14,16,18
-	 * 20,22,24,26,28
-	 * 30,32,34,36,38
-	 * 40,42,44,46,48
-	 * 50,52,54,56,58
-	 */
-	.rept 25
-		RR	j
-		.set j, j+2
-	.endr
-
-	/* Update Counter */
-	sub $1, BLOCKS_CTR
-	/* Move to the next block only if needed*/
-	ADD_IF_GE BUFFER_PTR, BLOCKS_CTR, 4, 128
-	/*
-	 * rounds
-	 * 60,62,64,66,68
-	 * 70,72,74,76,78
-	 */
-	.rept 10
-		RR	j
-		.set j, j+2
-	.endr
-
-	UPDATE_HASH	(HASH_PTR), A
-	UPDATE_HASH	4(HASH_PTR), TB
-	UPDATE_HASH	8(HASH_PTR), C
-	UPDATE_HASH	12(HASH_PTR), D
-	UPDATE_HASH	16(HASH_PTR), E
-
-	test	BLOCKS_CTR, BLOCKS_CTR
-	jz	.L_loop
-
-	mov	TB, B
-
-	/* Process second block */
-	/*
-	 * rounds
-	 *  0+80, 2+80, 4+80, 6+80, 8+80
-	 * 10+80,12+80,14+80,16+80,18+80
-	 */
-
-	.set j, 0
-	.rept 10
-		RR	j+80
-		.set j, j+2
-	.endr
-
-	/*
-	 * rounds
-	 * 20+80,22+80,24+80,26+80,28+80
-	 * 30+80,32+80,34+80,36+80,38+80
-	 */
-	.rept 10
-		RR	j+80
-		.set j, j+2
-	.endr
-
-	/*
-	 * rounds
-	 * 40+80,42+80,44+80,46+80,48+80
-	 * 50+80,52+80,54+80,56+80,58+80
-	 */
-	.rept 10
-		RR	j+80
-		.set j, j+2
-	.endr
-
-	/* update counter */
-	sub     $1, BLOCKS_CTR
-	/* Move to the next block only if needed*/
-	ADD_IF_GE BUFFER_PTR2, BLOCKS_CTR, 4, 128
-
-	/*
-	 * rounds
-	 * 60+80,62+80,64+80,66+80,68+80
-	 * 70+80,72+80,74+80,76+80,78+80
-	 */
-	.rept 10
-		RR	j+80
-		.set j, j+2
-	.endr
-
-	UPDATE_HASH	(HASH_PTR), A
-	UPDATE_HASH	4(HASH_PTR), TB
-	UPDATE_HASH	8(HASH_PTR), C
-	UPDATE_HASH	12(HASH_PTR), D
-	UPDATE_HASH	16(HASH_PTR), E
-
-	/* Reset state for AVX2 reg permutation */
-	mov	A, TA
-	mov	TB, A
-	mov	C, TB
-	mov	E, C
-	mov	D, B
-	mov	TA, D
-
-	REGALLOC
-
-	xchg	WK_BUF, PRECALC_BUF
-
-	jmp	.L_loop
-
-	.align 32
-.L_end:
-
-.endm
-/*
- * macro implements SHA-1 function's body for several 64-byte blocks
- * param: function's name
- */
-.macro SHA1_VECTOR_ASM  name
-	SYM_FUNC_START(\name)
-
-	push	%rbx
-	push	%r12
-	push	%r13
-	push	%r14
-	push	%r15
-
-	RESERVE_STACK  = (W_SIZE*4 + 8+24)
-
-	/* Align stack */
-	push	%rbp
-	mov	%rsp, %rbp
-	and	$~(0x20-1), %rsp
-	sub	$RESERVE_STACK, %rsp
-
-	avx2_zeroupper
-
-	/* Setup initial values */
-	mov	CTX, HASH_PTR
-	mov	BUF, BUFFER_PTR
-
-	mov	BUF, BUFFER_PTR2
-	mov	CNT, BLOCKS_CTR
-
-	xmm_mov	BSWAP_SHUFB_CTL(%rip), YMM_SHUFB_BSWAP
-
-	SHA1_PIPELINED_MAIN_BODY
-
-	avx2_zeroupper
-
-	mov	%rbp, %rsp
-	pop	%rbp
-
-	pop	%r15
-	pop	%r14
-	pop	%r13
-	pop	%r12
-	pop	%rbx
-
-	RET
-
-	SYM_FUNC_END(\name)
-.endm
-
-.section .rodata
-
-#define K1 0x5a827999
-#define K2 0x6ed9eba1
-#define K3 0x8f1bbcdc
-#define K4 0xca62c1d6
-
-.align 128
-K_XMM_AR:
-	.long K1, K1, K1, K1
-	.long K1, K1, K1, K1
-	.long K2, K2, K2, K2
-	.long K2, K2, K2, K2
-	.long K3, K3, K3, K3
-	.long K3, K3, K3, K3
-	.long K4, K4, K4, K4
-	.long K4, K4, K4, K4
-
-BSWAP_SHUFB_CTL:
-	.long 0x00010203
-	.long 0x04050607
-	.long 0x08090a0b
-	.long 0x0c0d0e0f
-	.long 0x00010203
-	.long 0x04050607
-	.long 0x08090a0b
-	.long 0x0c0d0e0f
-.text
-
-SHA1_VECTOR_ASM     sha1_transform_avx2
diff --git a/arch/x86/crypto/sha1_ni_asm.S b/arch/x86/crypto/sha1_ni_asm.S
deleted file mode 100644
index cade913d4882..000000000000
--- a/arch/x86/crypto/sha1_ni_asm.S
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Intel SHA Extensions optimized implementation of a SHA-1 update function
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Contact Information:
- * 	Sean Gulley <sean.m.gulley@intel.com>
- * 	Tim Chen <tim.c.chen@linux.intel.com>
- *
- * BSD LICENSE
- *
- * Copyright(c) 2015 Intel Corporation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 	* Redistributions of source code must retain the above copyright
- * 	  notice, this list of conditions and the following disclaimer.
- * 	* Redistributions in binary form must reproduce the above copyright
- * 	  notice, this list of conditions and the following disclaimer in
- * 	  the documentation and/or other materials provided with the
- * 	  distribution.
- * 	* Neither the name of Intel Corporation nor the names of its
- * 	  contributors may be used to endorse or promote products derived
- * 	  from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-#define DIGEST_PTR	%rdi	/* 1st arg */
-#define DATA_PTR	%rsi	/* 2nd arg */
-#define NUM_BLKS	%rdx	/* 3rd arg */
-
-/* gcc conversion */
-#define FRAME_SIZE	32	/* space for 2x16 bytes */
-
-#define ABCD		%xmm0
-#define E0		%xmm1	/* Need two E's b/c they ping pong */
-#define E1		%xmm2
-#define MSG0		%xmm3
-#define MSG1		%xmm4
-#define MSG2		%xmm5
-#define MSG3		%xmm6
-#define SHUF_MASK	%xmm7
-
-
-/*
- * Intel SHA Extensions optimized implementation of a SHA-1 update function
- *
- * The function takes a pointer to the current hash values, a pointer to the
- * input data, and a number of 64 byte blocks to process.  Once all blocks have
- * been processed, the digest pointer is  updated with the resulting hash value.
- * The function only processes complete blocks, there is no functionality to
- * store partial blocks. All message padding and hash value initialization must
- * be done outside the update function.
- *
- * The indented lines in the loop are instructions related to rounds processing.
- * The non-indented lines are instructions related to the message schedule.
- *
- * void sha1_ni_transform(uint32_t *digest, const void *data,
-		uint32_t numBlocks)
- * digest : pointer to digest
- * data: pointer to input data
- * numBlocks: Number of blocks to process
- */
-.text
-SYM_TYPED_FUNC_START(sha1_ni_transform)
-	push		%rbp
-	mov		%rsp, %rbp
-	sub		$FRAME_SIZE, %rsp
-	and		$~0xF, %rsp
-
-	shl		$6, NUM_BLKS		/* convert to bytes */
-	jz		.Ldone_hash
-	add		DATA_PTR, NUM_BLKS	/* pointer to end of data */
-
-	/* load initial hash values */
-	pinsrd		$3, 1*16(DIGEST_PTR), E0
-	movdqu		0*16(DIGEST_PTR), ABCD
-	pand		UPPER_WORD_MASK(%rip), E0
-	pshufd		$0x1B, ABCD, ABCD
-
-	movdqa		PSHUFFLE_BYTE_FLIP_MASK(%rip), SHUF_MASK
-
-.Lloop0:
-	/* Save hash values for addition after rounds */
-	movdqa		E0, (0*16)(%rsp)
-	movdqa		ABCD, (1*16)(%rsp)
-
-	/* Rounds 0-3 */
-	movdqu		0*16(DATA_PTR), MSG0
-	pshufb		SHUF_MASK, MSG0
-		paddd		MSG0, E0
-		movdqa		ABCD, E1
-		sha1rnds4	$0, E0, ABCD
-
-	/* Rounds 4-7 */
-	movdqu		1*16(DATA_PTR), MSG1
-	pshufb		SHUF_MASK, MSG1
-		sha1nexte	MSG1, E1
-		movdqa		ABCD, E0
-		sha1rnds4	$0, E1, ABCD
-	sha1msg1	MSG1, MSG0
-
-	/* Rounds 8-11 */
-	movdqu		2*16(DATA_PTR), MSG2
-	pshufb		SHUF_MASK, MSG2
-		sha1nexte	MSG2, E0
-		movdqa		ABCD, E1
-		sha1rnds4	$0, E0, ABCD
-	sha1msg1	MSG2, MSG1
-	pxor		MSG2, MSG0
-
-	/* Rounds 12-15 */
-	movdqu		3*16(DATA_PTR), MSG3
-	pshufb		SHUF_MASK, MSG3
-		sha1nexte	MSG3, E1
-		movdqa		ABCD, E0
-	sha1msg2	MSG3, MSG0
-		sha1rnds4	$0, E1, ABCD
-	sha1msg1	MSG3, MSG2
-	pxor		MSG3, MSG1
-
-	/* Rounds 16-19 */
-		sha1nexte	MSG0, E0
-		movdqa		ABCD, E1
-	sha1msg2	MSG0, MSG1
-		sha1rnds4	$0, E0, ABCD
-	sha1msg1	MSG0, MSG3
-	pxor		MSG0, MSG2
-
-	/* Rounds 20-23 */
-		sha1nexte	MSG1, E1
-		movdqa		ABCD, E0
-	sha1msg2	MSG1, MSG2
-		sha1rnds4	$1, E1, ABCD
-	sha1msg1	MSG1, MSG0
-	pxor		MSG1, MSG3
-
-	/* Rounds 24-27 */
-		sha1nexte	MSG2, E0
-		movdqa		ABCD, E1
-	sha1msg2	MSG2, MSG3
-		sha1rnds4	$1, E0, ABCD
-	sha1msg1	MSG2, MSG1
-	pxor		MSG2, MSG0
-
-	/* Rounds 28-31 */
-		sha1nexte	MSG3, E1
-		movdqa		ABCD, E0
-	sha1msg2	MSG3, MSG0
-		sha1rnds4	$1, E1, ABCD
-	sha1msg1	MSG3, MSG2
-	pxor		MSG3, MSG1
-
-	/* Rounds 32-35 */
-		sha1nexte	MSG0, E0
-		movdqa		ABCD, E1
-	sha1msg2	MSG0, MSG1
-		sha1rnds4	$1, E0, ABCD
-	sha1msg1	MSG0, MSG3
-	pxor		MSG0, MSG2
-
-	/* Rounds 36-39 */
-		sha1nexte	MSG1, E1
-		movdqa		ABCD, E0
-	sha1msg2	MSG1, MSG2
-		sha1rnds4	$1, E1, ABCD
-	sha1msg1	MSG1, MSG0
-	pxor		MSG1, MSG3
-
-	/* Rounds 40-43 */
-		sha1nexte	MSG2, E0
-		movdqa		ABCD, E1
-	sha1msg2	MSG2, MSG3
-		sha1rnds4	$2, E0, ABCD
-	sha1msg1	MSG2, MSG1
-	pxor		MSG2, MSG0
-
-	/* Rounds 44-47 */
-		sha1nexte	MSG3, E1
-		movdqa		ABCD, E0
-	sha1msg2	MSG3, MSG0
-		sha1rnds4	$2, E1, ABCD
-	sha1msg1	MSG3, MSG2
-	pxor		MSG3, MSG1
-
-	/* Rounds 48-51 */
-		sha1nexte	MSG0, E0
-		movdqa		ABCD, E1
-	sha1msg2	MSG0, MSG1
-		sha1rnds4	$2, E0, ABCD
-	sha1msg1	MSG0, MSG3
-	pxor		MSG0, MSG2
-
-	/* Rounds 52-55 */
-		sha1nexte	MSG1, E1
-		movdqa		ABCD, E0
-	sha1msg2	MSG1, MSG2
-		sha1rnds4	$2, E1, ABCD
-	sha1msg1	MSG1, MSG0
-	pxor		MSG1, MSG3
-
-	/* Rounds 56-59 */
-		sha1nexte	MSG2, E0
-		movdqa		ABCD, E1
-	sha1msg2	MSG2, MSG3
-		sha1rnds4	$2, E0, ABCD
-	sha1msg1	MSG2, MSG1
-	pxor		MSG2, MSG0
-
-	/* Rounds 60-63 */
-		sha1nexte	MSG3, E1
-		movdqa		ABCD, E0
-	sha1msg2	MSG3, MSG0
-		sha1rnds4	$3, E1, ABCD
-	sha1msg1	MSG3, MSG2
-	pxor		MSG3, MSG1
-
-	/* Rounds 64-67 */
-		sha1nexte	MSG0, E0
-		movdqa		ABCD, E1
-	sha1msg2	MSG0, MSG1
-		sha1rnds4	$3, E0, ABCD
-	sha1msg1	MSG0, MSG3
-	pxor		MSG0, MSG2
-
-	/* Rounds 68-71 */
-		sha1nexte	MSG1, E1
-		movdqa		ABCD, E0
-	sha1msg2	MSG1, MSG2
-		sha1rnds4	$3, E1, ABCD
-	pxor		MSG1, MSG3
-
-	/* Rounds 72-75 */
-		sha1nexte	MSG2, E0
-		movdqa		ABCD, E1
-	sha1msg2	MSG2, MSG3
-		sha1rnds4	$3, E0, ABCD
-
-	/* Rounds 76-79 */
-		sha1nexte	MSG3, E1
-		movdqa		ABCD, E0
-		sha1rnds4	$3, E1, ABCD
-
-	/* Add current hash values with previously saved */
-	sha1nexte	(0*16)(%rsp), E0
-	paddd		(1*16)(%rsp), ABCD
-
-	/* Increment data pointer and loop if more to process */
-	add		$64, DATA_PTR
-	cmp		NUM_BLKS, DATA_PTR
-	jne		.Lloop0
-
-	/* Write hash values back in the correct order */
-	pshufd		$0x1B, ABCD, ABCD
-	movdqu		ABCD, 0*16(DIGEST_PTR)
-	pextrd		$3, E0, 1*16(DIGEST_PTR)
-
-.Ldone_hash:
-	mov		%rbp, %rsp
-	pop		%rbp
-
-	RET
-SYM_FUNC_END(sha1_ni_transform)
-
-.section	.rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
-.align 16
-PSHUFFLE_BYTE_FLIP_MASK:
-	.octa 0x000102030405060708090a0b0c0d0e0f
-
-.section	.rodata.cst16.UPPER_WORD_MASK, "aM", @progbits, 16
-.align 16
-UPPER_WORD_MASK:
-	.octa 0xFFFFFFFF000000000000000000000000
diff --git a/arch/x86/crypto/sha1_ssse3_asm.S b/arch/x86/crypto/sha1_ssse3_asm.S
deleted file mode 100644
index f54988c80eb4..000000000000
--- a/arch/x86/crypto/sha1_ssse3_asm.S
+++ /dev/null
@@ -1,554 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * This is a SIMD SHA-1 implementation. It requires the Intel(R) Supplemental
- * SSE3 instruction set extensions introduced in Intel Core Microarchitecture
- * processors. CPUs supporting Intel(R) AVX extensions will get an additional
- * boost.
- *
- * This work was inspired by the vectorized implementation of Dean Gaudet.
- * Additional information on it can be found at:
- *    http://www.arctic.org/~dean/crypto/sha1.html
- *
- * It was improved upon with more efficient vectorization of the message
- * scheduling. This implementation has also been optimized for all current and
- * several future generations of Intel CPUs.
- *
- * See this article for more information about the implementation details:
- *   http://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1/
- *
- * Copyright (C) 2010, Intel Corp.
- *   Authors: Maxim Locktyukhin <maxim.locktyukhin@intel.com>
- *            Ronen Zohar <ronen.zohar@intel.com>
- *
- * Converted to AT&T syntax and adapted for inclusion in the Linux kernel:
- *   Author: Mathias Krause <minipli@googlemail.com>
- */
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-#define CTX	%rdi	// arg1
-#define BUF	%rsi	// arg2
-#define CNT	%rdx	// arg3
-
-#define REG_A	%ecx
-#define REG_B	%esi
-#define REG_C	%edi
-#define REG_D	%r12d
-#define REG_E	%edx
-
-#define REG_T1	%eax
-#define REG_T2	%ebx
-
-#define K_BASE		%r8
-#define HASH_PTR	%r9
-#define BUFFER_PTR	%r10
-#define BUFFER_END	%r11
-
-#define W_TMP1	%xmm0
-#define W_TMP2	%xmm9
-
-#define W0	%xmm1
-#define W4	%xmm2
-#define W8	%xmm3
-#define W12	%xmm4
-#define W16	%xmm5
-#define W20	%xmm6
-#define W24	%xmm7
-#define W28	%xmm8
-
-#define XMM_SHUFB_BSWAP	%xmm10
-
-/* we keep window of 64 w[i]+K pre-calculated values in a circular buffer */
-#define WK(t)	(((t) & 15) * 4)(%rsp)
-#define W_PRECALC_AHEAD	16
-
-/*
- * This macro implements the SHA-1 function's body for single 64-byte block
- * param: function's name
- */
-.macro SHA1_VECTOR_ASM  name
-	SYM_TYPED_FUNC_START(\name)
-
-	push	%rbx
-	push	%r12
-	push	%rbp
-	mov	%rsp, %rbp
-
-	sub	$64, %rsp		# allocate workspace
-	and	$~15, %rsp		# align stack
-
-	mov	CTX, HASH_PTR
-	mov	BUF, BUFFER_PTR
-
-	shl	$6, CNT			# multiply by 64
-	add	BUF, CNT
-	mov	CNT, BUFFER_END
-
-	lea	K_XMM_AR(%rip), K_BASE
-	xmm_mov	BSWAP_SHUFB_CTL(%rip), XMM_SHUFB_BSWAP
-
-	SHA1_PIPELINED_MAIN_BODY
-
-	# cleanup workspace
-	mov	$8, %ecx
-	mov	%rsp, %rdi
-	xor	%eax, %eax
-	rep stosq
-
-	mov	%rbp, %rsp		# deallocate workspace
-	pop	%rbp
-	pop	%r12
-	pop	%rbx
-	RET
-
-	SYM_FUNC_END(\name)
-.endm
-
-/*
- * This macro implements 80 rounds of SHA-1 for one 64-byte block
- */
-.macro SHA1_PIPELINED_MAIN_BODY
-	INIT_REGALLOC
-
-	mov	  (HASH_PTR), A
-	mov	 4(HASH_PTR), B
-	mov	 8(HASH_PTR), C
-	mov	12(HASH_PTR), D
-	mov	16(HASH_PTR), E
-
-  .set i, 0
-  .rept W_PRECALC_AHEAD
-	W_PRECALC i
-    .set i, (i+1)
-  .endr
-
-.align 4
-1:
-	RR F1,A,B,C,D,E,0
-	RR F1,D,E,A,B,C,2
-	RR F1,B,C,D,E,A,4
-	RR F1,E,A,B,C,D,6
-	RR F1,C,D,E,A,B,8
-
-	RR F1,A,B,C,D,E,10
-	RR F1,D,E,A,B,C,12
-	RR F1,B,C,D,E,A,14
-	RR F1,E,A,B,C,D,16
-	RR F1,C,D,E,A,B,18
-
-	RR F2,A,B,C,D,E,20
-	RR F2,D,E,A,B,C,22
-	RR F2,B,C,D,E,A,24
-	RR F2,E,A,B,C,D,26
-	RR F2,C,D,E,A,B,28
-
-	RR F2,A,B,C,D,E,30
-	RR F2,D,E,A,B,C,32
-	RR F2,B,C,D,E,A,34
-	RR F2,E,A,B,C,D,36
-	RR F2,C,D,E,A,B,38
-
-	RR F3,A,B,C,D,E,40
-	RR F3,D,E,A,B,C,42
-	RR F3,B,C,D,E,A,44
-	RR F3,E,A,B,C,D,46
-	RR F3,C,D,E,A,B,48
-
-	RR F3,A,B,C,D,E,50
-	RR F3,D,E,A,B,C,52
-	RR F3,B,C,D,E,A,54
-	RR F3,E,A,B,C,D,56
-	RR F3,C,D,E,A,B,58
-
-	add	$64, BUFFER_PTR		# move to the next 64-byte block
-	cmp	BUFFER_END, BUFFER_PTR	# if the current is the last one use
-	cmovae	K_BASE, BUFFER_PTR	# dummy source to avoid buffer overrun
-
-	RR F4,A,B,C,D,E,60
-	RR F4,D,E,A,B,C,62
-	RR F4,B,C,D,E,A,64
-	RR F4,E,A,B,C,D,66
-	RR F4,C,D,E,A,B,68
-
-	RR F4,A,B,C,D,E,70
-	RR F4,D,E,A,B,C,72
-	RR F4,B,C,D,E,A,74
-	RR F4,E,A,B,C,D,76
-	RR F4,C,D,E,A,B,78
-
-	UPDATE_HASH   (HASH_PTR), A
-	UPDATE_HASH  4(HASH_PTR), B
-	UPDATE_HASH  8(HASH_PTR), C
-	UPDATE_HASH 12(HASH_PTR), D
-	UPDATE_HASH 16(HASH_PTR), E
-
-	RESTORE_RENAMED_REGS
-	cmp	K_BASE, BUFFER_PTR	# K_BASE means, we reached the end
-	jne	1b
-.endm
-
-.macro INIT_REGALLOC
-  .set A, REG_A
-  .set B, REG_B
-  .set C, REG_C
-  .set D, REG_D
-  .set E, REG_E
-  .set T1, REG_T1
-  .set T2, REG_T2
-.endm
-
-.macro RESTORE_RENAMED_REGS
-	# order is important (REG_C is where it should be)
-	mov	B, REG_B
-	mov	D, REG_D
-	mov	A, REG_A
-	mov	E, REG_E
-.endm
-
-.macro SWAP_REG_NAMES  a, b
-  .set _T, \a
-  .set \a, \b
-  .set \b, _T
-.endm
-
-.macro F1  b, c, d
-	mov	\c, T1
-	SWAP_REG_NAMES \c, T1
-	xor	\d, T1
-	and	\b, T1
-	xor	\d, T1
-.endm
-
-.macro F2  b, c, d
-	mov	\d, T1
-	SWAP_REG_NAMES \d, T1
-	xor	\c, T1
-	xor	\b, T1
-.endm
-
-.macro F3  b, c ,d
-	mov	\c, T1
-	SWAP_REG_NAMES \c, T1
-	mov	\b, T2
-	or	\b, T1
-	and	\c, T2
-	and	\d, T1
-	or	T2, T1
-.endm
-
-.macro F4  b, c, d
-	F2 \b, \c, \d
-.endm
-
-.macro UPDATE_HASH  hash, val
-	add	\hash, \val
-	mov	\val, \hash
-.endm
-
-/*
- * RR does two rounds of SHA-1 back to back with W[] pre-calc
- *   t1 = F(b, c, d);   e += w(i)
- *   e += t1;           b <<= 30;   d  += w(i+1);
- *   t1 = F(a, b, c);
- *   d += t1;           a <<= 5;
- *   e += a;
- *   t1 = e;            a >>= 7;
- *   t1 <<= 5;
- *   d += t1;
- */
-.macro RR  F, a, b, c, d, e, round
-	add	WK(\round), \e
-	\F   \b, \c, \d		# t1 = F(b, c, d);
-	W_PRECALC (\round + W_PRECALC_AHEAD)
-	rol	$30, \b
-	add	T1, \e
-	add	WK(\round + 1), \d
-
-	\F   \a, \b, \c
-	W_PRECALC (\round + W_PRECALC_AHEAD + 1)
-	rol	$5, \a
-	add	\a, \e
-	add	T1, \d
-	ror	$7, \a		# (a <<r 5) >>r 7) => a <<r 30)
-
-	mov	\e, T1
-	SWAP_REG_NAMES \e, T1
-
-	rol	$5, T1
-	add	T1, \d
-
-	# write:  \a, \b
-	# rotate: \a<=\d, \b<=\e, \c<=\a, \d<=\b, \e<=\c
-.endm
-
-.macro W_PRECALC  r
-  .set i, \r
-
-  .if (i < 20)
-    .set K_XMM, 0
-  .elseif (i < 40)
-    .set K_XMM, 16
-  .elseif (i < 60)
-    .set K_XMM, 32
-  .elseif (i < 80)
-    .set K_XMM, 48
-  .endif
-
-  .if ((i < 16) || ((i >= 80) && (i < (80 + W_PRECALC_AHEAD))))
-    .set i, ((\r) % 80)	    # pre-compute for the next iteration
-    .if (i == 0)
-	W_PRECALC_RESET
-    .endif
-	W_PRECALC_00_15
-  .elseif (i<32)
-	W_PRECALC_16_31
-  .elseif (i < 80)   // rounds 32-79
-	W_PRECALC_32_79
-  .endif
-.endm
-
-.macro W_PRECALC_RESET
-  .set W,          W0
-  .set W_minus_04, W4
-  .set W_minus_08, W8
-  .set W_minus_12, W12
-  .set W_minus_16, W16
-  .set W_minus_20, W20
-  .set W_minus_24, W24
-  .set W_minus_28, W28
-  .set W_minus_32, W
-.endm
-
-.macro W_PRECALC_ROTATE
-  .set W_minus_32, W_minus_28
-  .set W_minus_28, W_minus_24
-  .set W_minus_24, W_minus_20
-  .set W_minus_20, W_minus_16
-  .set W_minus_16, W_minus_12
-  .set W_minus_12, W_minus_08
-  .set W_minus_08, W_minus_04
-  .set W_minus_04, W
-  .set W,          W_minus_32
-.endm
-
-.macro W_PRECALC_SSSE3
-
-.macro W_PRECALC_00_15
-	W_PRECALC_00_15_SSSE3
-.endm
-.macro W_PRECALC_16_31
-	W_PRECALC_16_31_SSSE3
-.endm
-.macro W_PRECALC_32_79
-	W_PRECALC_32_79_SSSE3
-.endm
-
-/* message scheduling pre-compute for rounds 0-15 */
-.macro W_PRECALC_00_15_SSSE3
-  .if ((i & 3) == 0)
-	movdqu	(i*4)(BUFFER_PTR), W_TMP1
-  .elseif ((i & 3) == 1)
-	pshufb	XMM_SHUFB_BSWAP, W_TMP1
-	movdqa	W_TMP1, W
-  .elseif ((i & 3) == 2)
-	paddd	(K_BASE), W_TMP1
-  .elseif ((i & 3) == 3)
-	movdqa  W_TMP1, WK(i&~3)
-	W_PRECALC_ROTATE
-  .endif
-.endm
-
-/* message scheduling pre-compute for rounds 16-31
- *
- * - calculating last 32 w[i] values in 8 XMM registers
- * - pre-calculate K+w[i] values and store to mem, for later load by ALU add
- *   instruction
- *
- * some "heavy-lifting" vectorization for rounds 16-31 due to w[i]->w[i-3]
- * dependency, but improves for 32-79
- */
-.macro W_PRECALC_16_31_SSSE3
-  # blended scheduling of vector and scalar instruction streams, one 4-wide
-  # vector iteration / 4 scalar rounds
-  .if ((i & 3) == 0)
-	movdqa	W_minus_12, W
-	palignr	$8, W_minus_16, W	# w[i-14]
-	movdqa	W_minus_04, W_TMP1
-	psrldq	$4, W_TMP1		# w[i-3]
-	pxor	W_minus_08, W
-  .elseif ((i & 3) == 1)
-	pxor	W_minus_16, W_TMP1
-	pxor	W_TMP1, W
-	movdqa	W, W_TMP2
-	movdqa	W, W_TMP1
-	pslldq	$12, W_TMP2
-  .elseif ((i & 3) == 2)
-	psrld	$31, W
-	pslld	$1, W_TMP1
-	por	W, W_TMP1
-	movdqa	W_TMP2, W
-	psrld	$30, W_TMP2
-	pslld	$2, W
-  .elseif ((i & 3) == 3)
-	pxor	W, W_TMP1
-	pxor	W_TMP2, W_TMP1
-	movdqa	W_TMP1, W
-	paddd	K_XMM(K_BASE), W_TMP1
-	movdqa	W_TMP1, WK(i&~3)
-	W_PRECALC_ROTATE
-  .endif
-.endm
-
-/* message scheduling pre-compute for rounds 32-79
- *
- * in SHA-1 specification: w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
- * instead we do equal:    w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
- * allows more efficient vectorization since w[i]=>w[i-3] dependency is broken
- */
-.macro W_PRECALC_32_79_SSSE3
-  .if ((i & 3) == 0)
-	movdqa	W_minus_04, W_TMP1
-	pxor	W_minus_28, W		# W is W_minus_32 before xor
-	palignr	$8, W_minus_08, W_TMP1
-  .elseif ((i & 3) == 1)
-	pxor	W_minus_16, W
-	pxor	W_TMP1, W
-	movdqa	W, W_TMP1
-  .elseif ((i & 3) == 2)
-	psrld	$30, W
-	pslld	$2, W_TMP1
-	por	W, W_TMP1
-  .elseif ((i & 3) == 3)
-	movdqa	W_TMP1, W
-	paddd	K_XMM(K_BASE), W_TMP1
-	movdqa	W_TMP1, WK(i&~3)
-	W_PRECALC_ROTATE
-  .endif
-.endm
-
-.endm		// W_PRECALC_SSSE3
-
-
-#define K1	0x5a827999
-#define K2	0x6ed9eba1
-#define K3	0x8f1bbcdc
-#define K4	0xca62c1d6
-
-.section .rodata
-.align 16
-
-K_XMM_AR:
-	.long K1, K1, K1, K1
-	.long K2, K2, K2, K2
-	.long K3, K3, K3, K3
-	.long K4, K4, K4, K4
-
-BSWAP_SHUFB_CTL:
-	.long 0x00010203
-	.long 0x04050607
-	.long 0x08090a0b
-	.long 0x0c0d0e0f
-
-
-.section .text
-
-W_PRECALC_SSSE3
-.macro xmm_mov a, b
-	movdqu	\a,\b
-.endm
-
-/*
- * SSSE3 optimized implementation:
- *
- * extern "C" void sha1_transform_ssse3(struct sha1_state *state,
- *					const u8 *data, int blocks);
- *
- * Note that struct sha1_state is assumed to begin with u32 state[5].
- */
-SHA1_VECTOR_ASM     sha1_transform_ssse3
-
-.macro W_PRECALC_AVX
-
-.purgem W_PRECALC_00_15
-.macro  W_PRECALC_00_15
-    W_PRECALC_00_15_AVX
-.endm
-.purgem W_PRECALC_16_31
-.macro  W_PRECALC_16_31
-    W_PRECALC_16_31_AVX
-.endm
-.purgem W_PRECALC_32_79
-.macro  W_PRECALC_32_79
-    W_PRECALC_32_79_AVX
-.endm
-
-.macro W_PRECALC_00_15_AVX
-  .if ((i & 3) == 0)
-	vmovdqu	(i*4)(BUFFER_PTR), W_TMP1
-  .elseif ((i & 3) == 1)
-	vpshufb	XMM_SHUFB_BSWAP, W_TMP1, W
-  .elseif ((i & 3) == 2)
-	vpaddd	(K_BASE), W, W_TMP1
-  .elseif ((i & 3) == 3)
-	vmovdqa	W_TMP1, WK(i&~3)
-	W_PRECALC_ROTATE
-  .endif
-.endm
-
-.macro W_PRECALC_16_31_AVX
-  .if ((i & 3) == 0)
-	vpalignr $8, W_minus_16, W_minus_12, W	# w[i-14]
-	vpsrldq	$4, W_minus_04, W_TMP1		# w[i-3]
-	vpxor	W_minus_08, W, W
-	vpxor	W_minus_16, W_TMP1, W_TMP1
-  .elseif ((i & 3) == 1)
-	vpxor	W_TMP1, W, W
-	vpslldq	$12, W, W_TMP2
-	vpslld	$1, W, W_TMP1
-  .elseif ((i & 3) == 2)
-	vpsrld	$31, W, W
-	vpor	W, W_TMP1, W_TMP1
-	vpslld	$2, W_TMP2, W
-	vpsrld	$30, W_TMP2, W_TMP2
-  .elseif ((i & 3) == 3)
-	vpxor	W, W_TMP1, W_TMP1
-	vpxor	W_TMP2, W_TMP1, W
-	vpaddd	K_XMM(K_BASE), W, W_TMP1
-	vmovdqu	W_TMP1, WK(i&~3)
-	W_PRECALC_ROTATE
-  .endif
-.endm
-
-.macro W_PRECALC_32_79_AVX
-  .if ((i & 3) == 0)
-	vpalignr $8, W_minus_08, W_minus_04, W_TMP1
-	vpxor	W_minus_28, W, W		# W is W_minus_32 before xor
-  .elseif ((i & 3) == 1)
-	vpxor	W_minus_16, W_TMP1, W_TMP1
-	vpxor	W_TMP1, W, W
-  .elseif ((i & 3) == 2)
-	vpslld	$2, W, W_TMP1
-	vpsrld	$30, W, W
-	vpor	W, W_TMP1, W
-  .elseif ((i & 3) == 3)
-	vpaddd	K_XMM(K_BASE), W, W_TMP1
-	vmovdqu	W_TMP1, WK(i&~3)
-	W_PRECALC_ROTATE
-  .endif
-.endm
-
-.endm    // W_PRECALC_AVX
-
-W_PRECALC_AVX
-.purgem xmm_mov
-.macro xmm_mov a, b
-	vmovdqu	\a,\b
-.endm
-
-
-/* AVX optimized implementation:
- *  extern "C" void sha1_transform_avx(struct sha1_state *state,
- *				       const u8 *data, int blocks);
- */
-SHA1_VECTOR_ASM     sha1_transform_avx
diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c
deleted file mode 100644
index ab8bc54f254d..000000000000
--- a/arch/x86/crypto/sha1_ssse3_glue.c
+++ /dev/null
@@ -1,365 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Cryptographic API.
- *
- * Glue code for the SHA1 Secure Hash Algorithm assembler implementations
- * using SSSE3, AVX, AVX2, and SHA-NI instructions.
- *
- * This file is based on sha1_generic.c
- *
- * Copyright (c) Alan Smithee.
- * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
- * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
- * Copyright (c) Mathias Krause <minipli@googlemail.com>
- * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
- */
-
-#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
-
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/mm.h>
-#include <linux/types.h>
-#include <crypto/sha1.h>
-#include <crypto/sha1_base.h>
-#include <asm/cpu_device_id.h>
-#include <asm/simd.h>
-
-static const struct x86_cpu_id module_cpu_ids[] = {
-#ifdef CONFIG_AS_SHA1_NI
-	X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL),
-#endif
-	X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL),
-	X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL),
-	X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL),
-	{}
-};
-MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids);
-
-static int sha1_update(struct shash_desc *desc, const u8 *data,
-			     unsigned int len, sha1_block_fn *sha1_xform)
-{
-	struct sha1_state *sctx = shash_desc_ctx(desc);
-
-	if (!crypto_simd_usable() ||
-	    (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
-		return crypto_sha1_update(desc, data, len);
-
-	/*
-	 * Make sure struct sha1_state begins directly with the SHA1
-	 * 160-bit internal state, as this is what the asm functions expect.
-	 */
-	BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);
-
-	kernel_fpu_begin();
-	sha1_base_do_update(desc, data, len, sha1_xform);
-	kernel_fpu_end();
-
-	return 0;
-}
-
-static int sha1_finup(struct shash_desc *desc, const u8 *data,
-		      unsigned int len, u8 *out, sha1_block_fn *sha1_xform)
-{
-	if (!crypto_simd_usable())
-		return crypto_sha1_finup(desc, data, len, out);
-
-	kernel_fpu_begin();
-	if (len)
-		sha1_base_do_update(desc, data, len, sha1_xform);
-	sha1_base_do_finalize(desc, sha1_xform);
-	kernel_fpu_end();
-
-	return sha1_base_finish(desc, out);
-}
-
-asmlinkage void sha1_transform_ssse3(struct sha1_state *state,
-				     const u8 *data, int blocks);
-
-static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data,
-			     unsigned int len)
-{
-	return sha1_update(desc, data, len, sha1_transform_ssse3);
-}
-
-static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data,
-			      unsigned int len, u8 *out)
-{
-	return sha1_finup(desc, data, len, out, sha1_transform_ssse3);
-}
-
-/* Add padding and return the message digest. */
-static int sha1_ssse3_final(struct shash_desc *desc, u8 *out)
-{
-	return sha1_ssse3_finup(desc, NULL, 0, out);
-}
-
-static struct shash_alg sha1_ssse3_alg = {
-	.digestsize	=	SHA1_DIGEST_SIZE,
-	.init		=	sha1_base_init,
-	.update		=	sha1_ssse3_update,
-	.final		=	sha1_ssse3_final,
-	.finup		=	sha1_ssse3_finup,
-	.descsize	=	sizeof(struct sha1_state),
-	.base		=	{
-		.cra_name	=	"sha1",
-		.cra_driver_name =	"sha1-ssse3",
-		.cra_priority	=	150,
-		.cra_blocksize	=	SHA1_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-};
-
-static int register_sha1_ssse3(void)
-{
-	if (boot_cpu_has(X86_FEATURE_SSSE3))
-		return crypto_register_shash(&sha1_ssse3_alg);
-	return 0;
-}
-
-static void unregister_sha1_ssse3(void)
-{
-	if (boot_cpu_has(X86_FEATURE_SSSE3))
-		crypto_unregister_shash(&sha1_ssse3_alg);
-}
-
-asmlinkage void sha1_transform_avx(struct sha1_state *state,
-				   const u8 *data, int blocks);
-
-static int sha1_avx_update(struct shash_desc *desc, const u8 *data,
-			     unsigned int len)
-{
-	return sha1_update(desc, data, len, sha1_transform_avx);
-}
-
-static int sha1_avx_finup(struct shash_desc *desc, const u8 *data,
-			      unsigned int len, u8 *out)
-{
-	return sha1_finup(desc, data, len, out, sha1_transform_avx);
-}
-
-static int sha1_avx_final(struct shash_desc *desc, u8 *out)
-{
-	return sha1_avx_finup(desc, NULL, 0, out);
-}
-
-static struct shash_alg sha1_avx_alg = {
-	.digestsize	=	SHA1_DIGEST_SIZE,
-	.init		=	sha1_base_init,
-	.update		=	sha1_avx_update,
-	.final		=	sha1_avx_final,
-	.finup		=	sha1_avx_finup,
-	.descsize	=	sizeof(struct sha1_state),
-	.base		=	{
-		.cra_name	=	"sha1",
-		.cra_driver_name =	"sha1-avx",
-		.cra_priority	=	160,
-		.cra_blocksize	=	SHA1_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-};
-
-static bool avx_usable(void)
-{
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
-		if (boot_cpu_has(X86_FEATURE_AVX))
-			pr_info("AVX detected but unusable.\n");
-		return false;
-	}
-
-	return true;
-}
-
-static int register_sha1_avx(void)
-{
-	if (avx_usable())
-		return crypto_register_shash(&sha1_avx_alg);
-	return 0;
-}
-
-static void unregister_sha1_avx(void)
-{
-	if (avx_usable())
-		crypto_unregister_shash(&sha1_avx_alg);
-}
-
-#define SHA1_AVX2_BLOCK_OPTSIZE	4	/* optimal 4*64 bytes of SHA1 blocks */
-
-asmlinkage void sha1_transform_avx2(struct sha1_state *state,
-				    const u8 *data, int blocks);
-
-static bool avx2_usable(void)
-{
-	if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2)
-		&& boot_cpu_has(X86_FEATURE_BMI1)
-		&& boot_cpu_has(X86_FEATURE_BMI2))
-		return true;
-
-	return false;
-}
-
-static void sha1_apply_transform_avx2(struct sha1_state *state,
-				      const u8 *data, int blocks)
-{
-	/* Select the optimal transform based on data block size */
-	if (blocks >= SHA1_AVX2_BLOCK_OPTSIZE)
-		sha1_transform_avx2(state, data, blocks);
-	else
-		sha1_transform_avx(state, data, blocks);
-}
-
-static int sha1_avx2_update(struct shash_desc *desc, const u8 *data,
-			     unsigned int len)
-{
-	return sha1_update(desc, data, len, sha1_apply_transform_avx2);
-}
-
-static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data,
-			      unsigned int len, u8 *out)
-{
-	return sha1_finup(desc, data, len, out, sha1_apply_transform_avx2);
-}
-
-static int sha1_avx2_final(struct shash_desc *desc, u8 *out)
-{
-	return sha1_avx2_finup(desc, NULL, 0, out);
-}
-
-static struct shash_alg sha1_avx2_alg = {
-	.digestsize	=	SHA1_DIGEST_SIZE,
-	.init		=	sha1_base_init,
-	.update		=	sha1_avx2_update,
-	.final		=	sha1_avx2_final,
-	.finup		=	sha1_avx2_finup,
-	.descsize	=	sizeof(struct sha1_state),
-	.base		=	{
-		.cra_name	=	"sha1",
-		.cra_driver_name =	"sha1-avx2",
-		.cra_priority	=	170,
-		.cra_blocksize	=	SHA1_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-};
-
-static int register_sha1_avx2(void)
-{
-	if (avx2_usable())
-		return crypto_register_shash(&sha1_avx2_alg);
-	return 0;
-}
-
-static void unregister_sha1_avx2(void)
-{
-	if (avx2_usable())
-		crypto_unregister_shash(&sha1_avx2_alg);
-}
-
-#ifdef CONFIG_AS_SHA1_NI
-asmlinkage void sha1_ni_transform(struct sha1_state *digest, const u8 *data,
-				  int rounds);
-
-static int sha1_ni_update(struct shash_desc *desc, const u8 *data,
-			     unsigned int len)
-{
-	return sha1_update(desc, data, len, sha1_ni_transform);
-}
-
-static int sha1_ni_finup(struct shash_desc *desc, const u8 *data,
-			      unsigned int len, u8 *out)
-{
-	return sha1_finup(desc, data, len, out, sha1_ni_transform);
-}
-
-static int sha1_ni_final(struct shash_desc *desc, u8 *out)
-{
-	return sha1_ni_finup(desc, NULL, 0, out);
-}
-
-static struct shash_alg sha1_ni_alg = {
-	.digestsize	=	SHA1_DIGEST_SIZE,
-	.init		=	sha1_base_init,
-	.update		=	sha1_ni_update,
-	.final		=	sha1_ni_final,
-	.finup		=	sha1_ni_finup,
-	.descsize	=	sizeof(struct sha1_state),
-	.base		=	{
-		.cra_name	=	"sha1",
-		.cra_driver_name =	"sha1-ni",
-		.cra_priority	=	250,
-		.cra_blocksize	=	SHA1_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-};
-
-static int register_sha1_ni(void)
-{
-	if (boot_cpu_has(X86_FEATURE_SHA_NI))
-		return crypto_register_shash(&sha1_ni_alg);
-	return 0;
-}
-
-static void unregister_sha1_ni(void)
-{
-	if (boot_cpu_has(X86_FEATURE_SHA_NI))
-		crypto_unregister_shash(&sha1_ni_alg);
-}
-
-#else
-static inline int register_sha1_ni(void) { return 0; }
-static inline void unregister_sha1_ni(void) { }
-#endif
-
-static int __init sha1_ssse3_mod_init(void)
-{
-	if (!x86_match_cpu(module_cpu_ids))
-		return -ENODEV;
-
-	if (register_sha1_ssse3())
-		goto fail;
-
-	if (register_sha1_avx()) {
-		unregister_sha1_ssse3();
-		goto fail;
-	}
-
-	if (register_sha1_avx2()) {
-		unregister_sha1_avx();
-		unregister_sha1_ssse3();
-		goto fail;
-	}
-
-	if (register_sha1_ni()) {
-		unregister_sha1_avx2();
-		unregister_sha1_avx();
-		unregister_sha1_ssse3();
-		goto fail;
-	}
-
-	return 0;
-fail:
-	return -ENODEV;
-}
-
-static void __exit sha1_ssse3_mod_fini(void)
-{
-	unregister_sha1_ni();
-	unregister_sha1_avx2();
-	unregister_sha1_avx();
-	unregister_sha1_ssse3();
-}
-
-module_init(sha1_ssse3_mod_init);
-module_exit(sha1_ssse3_mod_fini);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");
-
-MODULE_ALIAS_CRYPTO("sha1");
-MODULE_ALIAS_CRYPTO("sha1-ssse3");
-MODULE_ALIAS_CRYPTO("sha1-avx");
-MODULE_ALIAS_CRYPTO("sha1-avx2");
-#ifdef CONFIG_AS_SHA1_NI
-MODULE_ALIAS_CRYPTO("sha1-ni");
-#endif
diff --git a/arch/x86/crypto/sha256-avx-asm.S b/arch/x86/crypto/sha256-avx-asm.S
deleted file mode 100644
index 53de72bdd851..000000000000
--- a/arch/x86/crypto/sha256-avx-asm.S
+++ /dev/null
@@ -1,499 +0,0 @@
-########################################################################
-# Implement fast SHA-256 with AVX1 instructions. (x86_64)
-#
-# Copyright (C) 2013 Intel Corporation.
-#
-# Authors:
-#     James Guilford <james.guilford@intel.com>
-#     Kirk Yap <kirk.s.yap@intel.com>
-#     Tim Chen <tim.c.chen@linux.intel.com>
-#
-# This software is available to you under a choice of one of two
-# licenses.  You may choose to be licensed under the terms of the GNU
-# General Public License (GPL) Version 2, available from the file
-# COPYING in the main directory of this source tree, or the
-# OpenIB.org BSD license below:
-#
-#     Redistribution and use in source and binary forms, with or
-#     without modification, are permitted provided that the following
-#     conditions are met:
-#
-#      - Redistributions of source code must retain the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer.
-#
-#      - Redistributions in binary form must reproduce the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer in the documentation and/or other materials
-#        provided with the distribution.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-########################################################################
-#
-# This code is described in an Intel White-Paper:
-# "Fast SHA-256 Implementations on Intel Architecture Processors"
-#
-# To find it, surf to http://www.intel.com/p/en_US/embedded
-# and search for that title.
-#
-########################################################################
-# This code schedules 1 block at a time, with 4 lanes per block
-########################################################################
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-## assume buffers not aligned
-#define    VMOVDQ vmovdqu
-
-################################ Define Macros
-
-# addm [mem], reg
-# Add reg to mem using reg-mem add and store
-.macro addm p1 p2
-	add     \p1, \p2
-	mov     \p2, \p1
-.endm
-
-
-.macro MY_ROR p1 p2
-	shld    $(32-(\p1)), \p2, \p2
-.endm
-
-################################
-
-# COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask
-# Load xmm with mem and byte swap each dword
-.macro COPY_XMM_AND_BSWAP p1 p2 p3
-	VMOVDQ \p2, \p1
-	vpshufb \p3, \p1, \p1
-.endm
-
-################################
-
-X0 = %xmm4
-X1 = %xmm5
-X2 = %xmm6
-X3 = %xmm7
-
-XTMP0 = %xmm0
-XTMP1 = %xmm1
-XTMP2 = %xmm2
-XTMP3 = %xmm3
-XTMP4 = %xmm8
-XFER = %xmm9
-XTMP5 = %xmm11
-
-SHUF_00BA = %xmm10      # shuffle xBxA -> 00BA
-SHUF_DC00 = %xmm12      # shuffle xDxC -> DC00
-BYTE_FLIP_MASK = %xmm13
-
-NUM_BLKS = %rdx   # 3rd arg
-INP = %rsi        # 2nd arg
-CTX = %rdi        # 1st arg
-
-SRND = %rsi       # clobbers INP
-c = %ecx
-d = %r8d
-e = %edx
-TBL = %r12
-a = %eax
-b = %ebx
-
-f = %r9d
-g = %r10d
-h = %r11d
-
-y0 = %r13d
-y1 = %r14d
-y2 = %r15d
-
-
-_INP_END_SIZE = 8
-_INP_SIZE = 8
-_XFER_SIZE = 16
-_XMM_SAVE_SIZE = 0
-
-_INP_END = 0
-_INP            = _INP_END  + _INP_END_SIZE
-_XFER           = _INP      + _INP_SIZE
-_XMM_SAVE       = _XFER     + _XFER_SIZE
-STACK_SIZE      = _XMM_SAVE + _XMM_SAVE_SIZE
-
-# rotate_Xs
-# Rotate values of symbols X0...X3
-.macro rotate_Xs
-X_ = X0
-X0 = X1
-X1 = X2
-X2 = X3
-X3 = X_
-.endm
-
-# ROTATE_ARGS
-# Rotate values of symbols a...h
-.macro ROTATE_ARGS
-TMP_ = h
-h = g
-g = f
-f = e
-e = d
-d = c
-c = b
-b = a
-a = TMP_
-.endm
-
-.macro FOUR_ROUNDS_AND_SCHED
-	## compute s0 four at a time and s1 two at a time
-	## compute W[-16] + W[-7] 4 at a time
-
-	mov     e, y0			# y0 = e
-	MY_ROR  (25-11), y0             # y0 = e >> (25-11)
-	mov     a, y1                   # y1 = a
-	vpalignr $4, X2, X3, XTMP0      # XTMP0 = W[-7]
-	MY_ROR  (22-13), y1             # y1 = a >> (22-13)
-	xor     e, y0                   # y0 = e ^ (e >> (25-11))
-	mov     f, y2                   # y2 = f
-	MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
-	xor     a, y1                   # y1 = a ^ (a >> (22-13)
-	xor     g, y2                   # y2 = f^g
-	vpaddd  X0, XTMP0, XTMP0        # XTMP0 = W[-7] + W[-16]
-	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
-	and     e, y2                   # y2 = (f^g)&e
-	MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
-	## compute s0
-	vpalignr $4, X0, X1, XTMP1      # XTMP1 = W[-15]
-	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
-	MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
-	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
-	MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
-	add     y0, y2                  # y2 = S1 + CH
-	add     _XFER(%rsp), y2         # y2 = k + w + S1 + CH
-	mov     a, y0                   # y0 = a
-	add     y2, h                   # h = h + S1 + CH + k + w
-	mov     a, y2                   # y2 = a
-	vpsrld  $7, XTMP1, XTMP2
-	or      c, y0                   # y0 = a|c
-	add     h, d                    # d = d + h + S1 + CH + k + w
-	and     c, y2                   # y2 = a&c
-	vpslld  $(32-7), XTMP1, XTMP3
-	and     b, y0                   # y0 = (a|c)&b
-	add     y1, h                   # h = h + S1 + CH + k + w + S0
-	vpor    XTMP2, XTMP3, XTMP3     # XTMP1 = W[-15] MY_ROR 7
-	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
-	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
-	ROTATE_ARGS
-	mov     e, y0                   # y0 = e
-	mov     a, y1                   # y1 = a
-	MY_ROR  (25-11), y0             # y0 = e >> (25-11)
-	xor     e, y0                   # y0 = e ^ (e >> (25-11))
-	mov     f, y2                   # y2 = f
-	MY_ROR  (22-13), y1             # y1 = a >> (22-13)
-	vpsrld  $18, XTMP1, XTMP2       #
-	xor     a, y1                   # y1 = a ^ (a >> (22-13)
-	MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
-	xor     g, y2                   # y2 = f^g
-	vpsrld  $3, XTMP1, XTMP4        # XTMP4 = W[-15] >> 3
-	MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
-	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
-	and     e, y2                   # y2 = (f^g)&e
-	MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
-	vpslld  $(32-18), XTMP1, XTMP1
-	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
-	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
-	vpxor   XTMP1, XTMP3, XTMP3     #
-	add     y0, y2                  # y2 = S1 + CH
-	add     (1*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
-	MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
-	vpxor   XTMP2, XTMP3, XTMP3     # XTMP1 = W[-15] MY_ROR 7 ^ W[-15] MY_ROR
-	mov     a, y0                   # y0 = a
-	add     y2, h                   # h = h + S1 + CH + k + w
-	mov     a, y2                   # y2 = a
-	vpxor   XTMP4, XTMP3, XTMP1     # XTMP1 = s0
-	or      c, y0                   # y0 = a|c
-	add     h, d                    # d = d + h + S1 + CH + k + w
-	and     c, y2                   # y2 = a&c
-	## compute low s1
-	vpshufd $0b11111010, X3, XTMP2  # XTMP2 = W[-2] {BBAA}
-	and     b, y0                   # y0 = (a|c)&b
-	add     y1, h                   # h = h + S1 + CH + k + w + S0
-	vpaddd  XTMP1, XTMP0, XTMP0     # XTMP0 = W[-16] + W[-7] + s0
-	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
-	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
-	ROTATE_ARGS
-	mov     e, y0                   # y0 = e
-	mov     a, y1                   # y1 = a
-	MY_ROR  (25-11), y0             # y0 = e >> (25-11)
-	xor     e, y0                   # y0 = e ^ (e >> (25-11))
-	MY_ROR  (22-13), y1             # y1 = a >> (22-13)
-	mov     f, y2                   # y2 = f
-	xor     a, y1                   # y1 = a ^ (a >> (22-13)
-	MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
-	vpsrld  $10, XTMP2, XTMP4       # XTMP4 = W[-2] >> 10 {BBAA}
-	xor     g, y2                   # y2 = f^g
-	vpsrlq  $19, XTMP2, XTMP3       # XTMP3 = W[-2] MY_ROR 19 {xBxA}
-	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
-	and     e, y2                   # y2 = (f^g)&e
-	vpsrlq  $17, XTMP2, XTMP2       # XTMP2 = W[-2] MY_ROR 17 {xBxA}
-	MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
-	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
-	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
-	MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
-	vpxor   XTMP3, XTMP2, XTMP2     #
-	add     y0, y2                  # y2 = S1 + CH
-	MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
-	add     (2*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
-	vpxor   XTMP2, XTMP4, XTMP4     # XTMP4 = s1 {xBxA}
-	mov     a, y0                   # y0 = a
-	add     y2, h                   # h = h + S1 + CH + k + w
-	mov     a, y2                   # y2 = a
-	vpshufb SHUF_00BA, XTMP4, XTMP4 # XTMP4 = s1 {00BA}
-	or      c, y0                   # y0 = a|c
-	add     h, d                    # d = d + h + S1 + CH + k + w
-	and     c, y2                   # y2 = a&c
-	vpaddd  XTMP4, XTMP0, XTMP0     # XTMP0 = {..., ..., W[1], W[0]}
-	and     b, y0                   # y0 = (a|c)&b
-	add     y1, h                   # h = h + S1 + CH + k + w + S0
-	## compute high s1
-	vpshufd $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {DDCC}
-	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
-	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
-	ROTATE_ARGS
-	mov     e, y0                   # y0 = e
-	MY_ROR  (25-11), y0             # y0 = e >> (25-11)
-	mov     a, y1                   # y1 = a
-	MY_ROR  (22-13), y1             # y1 = a >> (22-13)
-	xor     e, y0                   # y0 = e ^ (e >> (25-11))
-	mov     f, y2                   # y2 = f
-	MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
-	vpsrld  $10, XTMP2, XTMP5       # XTMP5 = W[-2] >> 10 {DDCC}
-	xor     a, y1                   # y1 = a ^ (a >> (22-13)
-	xor     g, y2                   # y2 = f^g
-	vpsrlq  $19, XTMP2, XTMP3       # XTMP3 = W[-2] MY_ROR 19 {xDxC}
-	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
-	and     e, y2                   # y2 = (f^g)&e
-	MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
-	vpsrlq  $17, XTMP2, XTMP2       # XTMP2 = W[-2] MY_ROR 17 {xDxC}
-	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
-	MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
-	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
-	vpxor   XTMP3, XTMP2, XTMP2
-	MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
-	add     y0, y2                  # y2 = S1 + CH
-	add     (3*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
-	vpxor   XTMP2, XTMP5, XTMP5     # XTMP5 = s1 {xDxC}
-	mov     a, y0                   # y0 = a
-	add     y2, h                   # h = h + S1 + CH + k + w
-	mov     a, y2                   # y2 = a
-	vpshufb SHUF_DC00, XTMP5, XTMP5 # XTMP5 = s1 {DC00}
-	or      c, y0                   # y0 = a|c
-	add     h, d                    # d = d + h + S1 + CH + k + w
-	and     c, y2                   # y2 = a&c
-	vpaddd  XTMP0, XTMP5, X0        # X0 = {W[3], W[2], W[1], W[0]}
-	and     b, y0                   # y0 = (a|c)&b
-	add     y1, h                   # h = h + S1 + CH + k + w + S0
-	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
-	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
-	ROTATE_ARGS
-	rotate_Xs
-.endm
-
-## input is [rsp + _XFER + %1 * 4]
-.macro DO_ROUND round
-	mov	e, y0			# y0 = e
-        MY_ROR  (25-11), y0             # y0 = e >> (25-11)
-        mov     a, y1                   # y1 = a
-        xor     e, y0                   # y0 = e ^ (e >> (25-11))
-        MY_ROR  (22-13), y1             # y1 = a >> (22-13)
-        mov     f, y2                   # y2 = f
-        xor     a, y1                   # y1 = a ^ (a >> (22-13)
-        MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
-        xor     g, y2                   # y2 = f^g
-        xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
-        MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
-        and     e, y2                   # y2 = (f^g)&e
-        xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
-        MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
-        xor     g, y2                   # y2 = CH = ((f^g)&e)^g
-        add     y0, y2                  # y2 = S1 + CH
-        MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
-        offset = \round * 4 + _XFER     #
-        add     offset(%rsp), y2	# y2 = k + w + S1 + CH
-        mov     a, y0			# y0 = a
-        add     y2, h                   # h = h + S1 + CH + k + w
-        mov     a, y2                   # y2 = a
-        or      c, y0                   # y0 = a|c
-        add     h, d                    # d = d + h + S1 + CH + k + w
-        and     c, y2                   # y2 = a&c
-        and     b, y0                   # y0 = (a|c)&b
-        add     y1, h                   # h = h + S1 + CH + k + w + S0
-        or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
-        add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
-        ROTATE_ARGS
-.endm
-
-########################################################################
-## void sha256_transform_avx(state sha256_state *state, const u8 *data, int blocks)
-## arg 1 : pointer to state
-## arg 2 : pointer to input data
-## arg 3 : Num blocks
-########################################################################
-.text
-SYM_TYPED_FUNC_START(sha256_transform_avx)
-	pushq   %rbx
-	pushq   %r12
-	pushq   %r13
-	pushq   %r14
-	pushq   %r15
-	pushq	%rbp
-	movq	%rsp, %rbp
-
-	subq    $STACK_SIZE, %rsp	# allocate stack space
-	and	$~15, %rsp		# align stack pointer
-
-	shl     $6, NUM_BLKS		# convert to bytes
-	jz      .Ldone_hash
-	add     INP, NUM_BLKS		# pointer to end of data
-	mov     NUM_BLKS, _INP_END(%rsp)
-
-	## load initial digest
-	mov     4*0(CTX), a
-	mov     4*1(CTX), b
-	mov     4*2(CTX), c
-	mov     4*3(CTX), d
-	mov     4*4(CTX), e
-	mov     4*5(CTX), f
-	mov     4*6(CTX), g
-	mov     4*7(CTX), h
-
-	vmovdqa  PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
-	vmovdqa  _SHUF_00BA(%rip), SHUF_00BA
-	vmovdqa  _SHUF_DC00(%rip), SHUF_DC00
-.Lloop0:
-	lea     K256(%rip), TBL
-
-	## byte swap first 16 dwords
-	COPY_XMM_AND_BSWAP      X0, 0*16(INP), BYTE_FLIP_MASK
-	COPY_XMM_AND_BSWAP      X1, 1*16(INP), BYTE_FLIP_MASK
-	COPY_XMM_AND_BSWAP      X2, 2*16(INP), BYTE_FLIP_MASK
-	COPY_XMM_AND_BSWAP      X3, 3*16(INP), BYTE_FLIP_MASK
-
-	mov     INP, _INP(%rsp)
-
-	## schedule 48 input dwords, by doing 3 rounds of 16 each
-	mov     $3, SRND
-.align 16
-.Lloop1:
-	vpaddd  (TBL), X0, XFER
-	vmovdqa XFER, _XFER(%rsp)
-	FOUR_ROUNDS_AND_SCHED
-
-	vpaddd  1*16(TBL), X0, XFER
-	vmovdqa XFER, _XFER(%rsp)
-	FOUR_ROUNDS_AND_SCHED
-
-	vpaddd  2*16(TBL), X0, XFER
-	vmovdqa XFER, _XFER(%rsp)
-	FOUR_ROUNDS_AND_SCHED
-
-	vpaddd  3*16(TBL), X0, XFER
-	vmovdqa XFER, _XFER(%rsp)
-	add	$4*16, TBL
-	FOUR_ROUNDS_AND_SCHED
-
-	sub     $1, SRND
-	jne     .Lloop1
-
-	mov     $2, SRND
-.Lloop2:
-	vpaddd  (TBL), X0, XFER
-	vmovdqa XFER, _XFER(%rsp)
-	DO_ROUND        0
-	DO_ROUND        1
-	DO_ROUND        2
-	DO_ROUND        3
-
-	vpaddd  1*16(TBL), X1, XFER
-	vmovdqa XFER, _XFER(%rsp)
-	add     $2*16, TBL
-	DO_ROUND        0
-	DO_ROUND        1
-	DO_ROUND        2
-	DO_ROUND        3
-
-	vmovdqa X2, X0
-	vmovdqa X3, X1
-
-	sub     $1, SRND
-	jne     .Lloop2
-
-	addm    (4*0)(CTX),a
-	addm    (4*1)(CTX),b
-	addm    (4*2)(CTX),c
-	addm    (4*3)(CTX),d
-	addm    (4*4)(CTX),e
-	addm    (4*5)(CTX),f
-	addm    (4*6)(CTX),g
-	addm    (4*7)(CTX),h
-
-	mov     _INP(%rsp), INP
-	add     $64, INP
-	cmp     _INP_END(%rsp), INP
-	jne     .Lloop0
-
-.Ldone_hash:
-
-	mov	%rbp, %rsp
-	popq	%rbp
-	popq    %r15
-	popq    %r14
-	popq    %r13
-	popq	%r12
-	popq    %rbx
-	RET
-SYM_FUNC_END(sha256_transform_avx)
-
-.section	.rodata.cst256.K256, "aM", @progbits, 256
-.align 64
-K256:
-	.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
-	.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
-	.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
-	.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
-	.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
-	.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
-	.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
-	.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
-	.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
-	.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
-	.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
-	.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
-	.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
-	.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
-	.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
-	.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
-
-.section	.rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
-.align 16
-PSHUFFLE_BYTE_FLIP_MASK:
-	.octa 0x0c0d0e0f08090a0b0405060700010203
-
-.section	.rodata.cst16._SHUF_00BA, "aM", @progbits, 16
-.align 16
-# shuffle xBxA -> 00BA
-_SHUF_00BA:
-	.octa 0xFFFFFFFFFFFFFFFF0b0a090803020100
-
-.section	.rodata.cst16._SHUF_DC00, "aM", @progbits, 16
-.align 16
-# shuffle xDxC -> DC00
-_SHUF_DC00:
-	.octa 0x0b0a090803020100FFFFFFFFFFFFFFFF
diff --git a/arch/x86/crypto/sha256-avx2-asm.S b/arch/x86/crypto/sha256-avx2-asm.S
deleted file mode 100644
index 0bbec1c75cd0..000000000000
--- a/arch/x86/crypto/sha256-avx2-asm.S
+++ /dev/null
@@ -1,774 +0,0 @@
-########################################################################
-# Implement fast SHA-256 with AVX2 instructions. (x86_64)
-#
-# Copyright (C) 2013 Intel Corporation.
-#
-# Authors:
-#     James Guilford <james.guilford@intel.com>
-#     Kirk Yap <kirk.s.yap@intel.com>
-#     Tim Chen <tim.c.chen@linux.intel.com>
-#
-# This software is available to you under a choice of one of two
-# licenses.  You may choose to be licensed under the terms of the GNU
-# General Public License (GPL) Version 2, available from the file
-# COPYING in the main directory of this source tree, or the
-# OpenIB.org BSD license below:
-#
-#     Redistribution and use in source and binary forms, with or
-#     without modification, are permitted provided that the following
-#     conditions are met:
-#
-#      - Redistributions of source code must retain the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer.
-#
-#      - Redistributions in binary form must reproduce the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer in the documentation and/or other materials
-#        provided with the distribution.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-#
-########################################################################
-#
-# This code is described in an Intel White-Paper:
-# "Fast SHA-256 Implementations on Intel Architecture Processors"
-#
-# To find it, surf to http://www.intel.com/p/en_US/embedded
-# and search for that title.
-#
-########################################################################
-# This code schedules 2 blocks at a time, with 4 lanes per block
-########################################################################
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-## assume buffers not aligned
-#define	VMOVDQ vmovdqu
-
-################################ Define Macros
-
-# addm [mem], reg
-# Add reg to mem using reg-mem add and store
-.macro addm p1 p2
-	add	\p1, \p2
-	mov	\p2, \p1
-.endm
-
-################################
-
-X0 = %ymm4
-X1 = %ymm5
-X2 = %ymm6
-X3 = %ymm7
-
-# XMM versions of above
-XWORD0 = %xmm4
-XWORD1 = %xmm5
-XWORD2 = %xmm6
-XWORD3 = %xmm7
-
-XTMP0 = %ymm0
-XTMP1 = %ymm1
-XTMP2 = %ymm2
-XTMP3 = %ymm3
-XTMP4 = %ymm8
-XFER  = %ymm9
-XTMP5 = %ymm11
-
-SHUF_00BA =	%ymm10 # shuffle xBxA -> 00BA
-SHUF_DC00 =	%ymm12 # shuffle xDxC -> DC00
-BYTE_FLIP_MASK = %ymm13
-
-X_BYTE_FLIP_MASK = %xmm13 # XMM version of BYTE_FLIP_MASK
-
-NUM_BLKS = %rdx	# 3rd arg
-INP	= %rsi  # 2nd arg
-CTX	= %rdi	# 1st arg
-c	= %ecx
-d	= %r8d
-e       = %edx	# clobbers NUM_BLKS
-y3	= %esi	# clobbers INP
-
-SRND	= CTX	# SRND is same register as CTX
-
-a = %eax
-b = %ebx
-f = %r9d
-g = %r10d
-h = %r11d
-old_h = %r11d
-
-T1 = %r12d
-y0 = %r13d
-y1 = %r14d
-y2 = %r15d
-
-
-_XFER_SIZE	= 2*64*4	# 2 blocks, 64 rounds, 4 bytes/round
-_XMM_SAVE_SIZE	= 0
-_INP_END_SIZE	= 8
-_INP_SIZE	= 8
-_CTX_SIZE	= 8
-
-_XFER		= 0
-_XMM_SAVE	= _XFER     + _XFER_SIZE
-_INP_END	= _XMM_SAVE + _XMM_SAVE_SIZE
-_INP		= _INP_END  + _INP_END_SIZE
-_CTX		= _INP      + _INP_SIZE
-STACK_SIZE	= _CTX      + _CTX_SIZE
-
-# rotate_Xs
-# Rotate values of symbols X0...X3
-.macro rotate_Xs
-	X_ = X0
-	X0 = X1
-	X1 = X2
-	X2 = X3
-	X3 = X_
-.endm
-
-# ROTATE_ARGS
-# Rotate values of symbols a...h
-.macro ROTATE_ARGS
-	old_h = h
-	TMP_ = h
-	h = g
-	g = f
-	f = e
-	e = d
-	d = c
-	c = b
-	b = a
-	a = TMP_
-.endm
-
-.macro FOUR_ROUNDS_AND_SCHED disp
-################################### RND N + 0 ############################
-
-	mov	a, y3		# y3 = a                                # MAJA
-	rorx	$25, e, y0	# y0 = e >> 25				# S1A
-	rorx	$11, e, y1	# y1 = e >> 11				# S1B
-
-	addl	\disp(%rsp, SRND), h		# h = k + w + h         # --
-	or	c, y3		# y3 = a|c                              # MAJA
-	vpalignr $4, X2, X3, XTMP0 # XTMP0 = W[-7]
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$13, a, T1	# T1 = a >> 13				# S0B
-
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
-	xor	g, y2		# y2 = f^g                              # CH
-	vpaddd	X0, XTMP0, XTMP0 # XTMP0 = W[-7] + W[-16]# y1 = (e >> 6)# S1
-	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
-
-	and	e, y2		# y2 = (f^g)&e                          # CH
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
-	rorx	$22, a, y1	# y1 = a >> 22				# S0A
-	add	h, d		# d = k + w + h + d                     # --
-
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-	vpalignr $4, X0, X1, XTMP1	# XTMP1 = W[-15]
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
-	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
-
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-	vpsrld	$7, XTMP1, XTMP2
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	c, T1		# T1 = a&c                              # MAJB
-
-	add	y0, y2		# y2 = S1 + CH                          # --
-	vpslld	$(32-7), XTMP1, XTMP3
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-	vpor	XTMP2, XTMP3, XTMP3	# XTMP3 = W[-15] ror 7
-
-	vpsrld	$18, XTMP1, XTMP2
-	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-	add	y3, h		# h = t1 + S0 + MAJ                     # --
-
-
-	ROTATE_ARGS
-
-################################### RND N + 1 ############################
-
-	mov	a, y3		# y3 = a                                # MAJA
-	rorx	$25, e, y0	# y0 = e >> 25				# S1A
-	rorx	$11, e, y1	# y1 = e >> 11				# S1B
-	offset = \disp + 1*4
-	addl	offset(%rsp, SRND), h	# h = k + w + h         # --
-	or	c, y3		# y3 = a|c                              # MAJA
-
-
-	vpsrld	$3, XTMP1, XTMP4 # XTMP4 = W[-15] >> 3
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$13, a, T1	# T1 = a >> 13				# S0B
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
-	xor	g, y2		# y2 = f^g                              # CH
-
-
-	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
-	rorx	$22, a, y1	# y1 = a >> 22				# S0A
-	and	e, y2		# y2 = (f^g)&e                          # CH
-	add	h, d		# d = k + w + h + d                     # --
-
-	vpslld	$(32-18), XTMP1, XTMP1
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
-
-	vpxor	XTMP1, XTMP3, XTMP3
-	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-
-	vpxor	XTMP2, XTMP3, XTMP3	# XTMP3 = W[-15] ror 7 ^ W[-15] ror 18
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	c, T1		# T1 = a&c                              # MAJB
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-	vpxor	XTMP4, XTMP3, XTMP1	# XTMP1 = s0
-	vpshufd	$0b11111010, X3, XTMP2	# XTMP2 = W[-2] {BBAA}
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-
-	vpaddd	XTMP1, XTMP0, XTMP0	# XTMP0 = W[-16] + W[-7] + s0
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-	add	y3, h		# h = t1 + S0 + MAJ                     # --
-
-	vpsrld	$10, XTMP2, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA}
-
-
-	ROTATE_ARGS
-
-################################### RND N + 2 ############################
-
-	mov	a, y3		# y3 = a                                # MAJA
-	rorx	$25, e, y0	# y0 = e >> 25				# S1A
-	offset = \disp + 2*4
-	addl	offset(%rsp, SRND), h	# h = k + w + h         # --
-
-	vpsrlq	$19, XTMP2, XTMP3 # XTMP3 = W[-2] ror 19 {xBxA}
-	rorx	$11, e, y1	# y1 = e >> 11				# S1B
-	or	c, y3		# y3 = a|c                              # MAJA
-	mov	f, y2		# y2 = f                                # CH
-	xor	g, y2		# y2 = f^g                              # CH
-
-	rorx	$13, a, T1	# T1 = a >> 13				# S0B
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
-	vpsrlq	$17, XTMP2, XTMP2	# XTMP2 = W[-2] ror 17 {xBxA}
-	and	e, y2		# y2 = (f^g)&e                          # CH
-
-	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
-	vpxor	XTMP3, XTMP2, XTMP2
-	add	h, d		# d = k + w + h + d                     # --
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
-	rorx	$22, a, y1	# y1 = a >> 22				# S0A
-	vpxor	XTMP2, XTMP4, XTMP4	# XTMP4 = s1 {xBxA}
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-
-	vpshufb	SHUF_00BA, XTMP4, XTMP4	# XTMP4 = s1 {00BA}
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
-	rorx	$2, a ,T1	# T1 = (a >> 2)				# S0
-	vpaddd	XTMP4, XTMP0, XTMP0	# XTMP0 = {..., ..., W[1], W[0]}
-
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	c, T1		# T1 = a&c                              # MAJB
-	add	y0, y2		# y2 = S1 + CH                          # --
-	vpshufd	$0b01010000, XTMP0, XTMP2	# XTMP2 = W[-2] {DDCC}
-
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1,h		# h = k + w + h + S0                    # --
-	add	y2,d		# d = k + w + h + d + S1 + CH = d + t1  # --
-	add	y2,h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-
-	add	y3,h		# h = t1 + S0 + MAJ                     # --
-
-
-	ROTATE_ARGS
-
-################################### RND N + 3 ############################
-
-	mov	a, y3		# y3 = a                                # MAJA
-	rorx	$25, e, y0	# y0 = e >> 25				# S1A
-	rorx	$11, e, y1	# y1 = e >> 11				# S1B
-	offset = \disp + 3*4
-	addl	offset(%rsp, SRND), h	# h = k + w + h         # --
-	or	c, y3		# y3 = a|c                              # MAJA
-
-
-	vpsrld	$10, XTMP2, XTMP5	# XTMP5 = W[-2] >> 10 {DDCC}
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$13, a, T1	# T1 = a >> 13				# S0B
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
-	xor	g, y2		# y2 = f^g                              # CH
-
-
-	vpsrlq	$19, XTMP2, XTMP3	# XTMP3 = W[-2] ror 19 {xDxC}
-	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
-	and	e, y2		# y2 = (f^g)&e                          # CH
-	add	h, d		# d = k + w + h + d                     # --
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-
-	vpsrlq	$17, XTMP2, XTMP2	# XTMP2 = W[-2] ror 17 {xDxC}
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-
-	vpxor	XTMP3, XTMP2, XTMP2
-	rorx	$22, a, y1	# y1 = a >> 22				# S0A
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-	vpxor	XTMP2, XTMP5, XTMP5	# XTMP5 = s1 {xDxC}
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-
-	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
-	vpshufb	SHUF_DC00, XTMP5, XTMP5	# XTMP5 = s1 {DC00}
-
-	vpaddd	XTMP0, XTMP5, X0	# X0 = {W[3], W[2], W[1], W[0]}
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	c, T1		# T1 = a&c                              # MAJB
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-
-	add	y1, h		# h = k + w + h + S0                    # --
-	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-	add	y3, h		# h = t1 + S0 + MAJ                     # --
-
-	ROTATE_ARGS
-	rotate_Xs
-.endm
-
-.macro DO_4ROUNDS disp
-################################### RND N + 0 ###########################
-
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$25, e, y0	# y0 = e >> 25				# S1A
-	rorx	$11, e, y1	# y1 = e >> 11				# S1B
-	xor	g, y2		# y2 = f^g                              # CH
-
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
-	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
-	and	e, y2		# y2 = (f^g)&e                          # CH
-
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
-	rorx	$13, a, T1	# T1 = a >> 13				# S0B
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-	rorx	$22, a, y1	# y1 = a >> 22				# S0A
-	mov	a, y3		# y3 = a                                # MAJA
-
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
-	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
-	addl	\disp(%rsp, SRND), h		# h = k + w + h # --
-	or	c, y3		# y3 = a|c                              # MAJA
-
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-	and	c, T1		# T1 = a&c                              # MAJB
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-
-	add	h, d		# d = k + w + h + d                     # --
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-
-	ROTATE_ARGS
-
-################################### RND N + 1 ###########################
-
-	add	y2, old_h	# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$25, e, y0	# y0 = e >> 25				# S1A
-	rorx	$11, e, y1	# y1 = e >> 11				# S1B
-	xor	g, y2		# y2 = f^g                              # CH
-
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
-	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
-	and	e, y2		# y2 = (f^g)&e                          # CH
-	add	y3, old_h	# h = t1 + S0 + MAJ                     # --
-
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
-	rorx	$13, a, T1	# T1 = a >> 13				# S0B
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-	rorx	$22, a, y1	# y1 = a >> 22				# S0A
-	mov	a, y3		# y3 = a                                # MAJA
-
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
-	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
-	offset = 4*1 + \disp
-	addl	offset(%rsp, SRND), h		# h = k + w + h # --
-	or	c, y3		# y3 = a|c                              # MAJA
-
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-	and	c, T1		# T1 = a&c                              # MAJB
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-
-	add	h, d		# d = k + w + h + d                     # --
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-
-	ROTATE_ARGS
-
-################################### RND N + 2 ##############################
-
-	add	y2, old_h	# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$25, e, y0	# y0 = e >> 25				# S1A
-	rorx	$11, e, y1	# y1 = e >> 11				# S1B
-	xor	g, y2		# y2 = f^g                              # CH
-
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
-	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
-	and	e, y2		# y2 = (f^g)&e                          # CH
-	add	y3, old_h	# h = t1 + S0 + MAJ                     # --
-
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
-	rorx	$13, a, T1	# T1 = a >> 13				# S0B
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-	rorx	$22, a, y1	# y1 = a >> 22				# S0A
-	mov	a, y3		# y3 = a                                # MAJA
-
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
-	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
-	offset = 4*2 + \disp
-	addl	offset(%rsp, SRND), h		# h = k + w + h # --
-	or	c, y3		# y3 = a|c                              # MAJA
-
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-	and	c, T1		# T1 = a&c                              # MAJB
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-
-	add	h, d		# d = k + w + h + d                     # --
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-
-	ROTATE_ARGS
-
-################################### RND N + 3 ###########################
-
-	add	y2, old_h	# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$25, e, y0	# y0 = e >> 25				# S1A
-	rorx	$11, e, y1	# y1 = e >> 11				# S1B
-	xor	g, y2		# y2 = f^g                              # CH
-
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
-	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
-	and	e, y2		# y2 = (f^g)&e                          # CH
-	add	y3, old_h	# h = t1 + S0 + MAJ                     # --
-
-	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
-	rorx	$13, a, T1	# T1 = a >> 13				# S0B
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-	rorx	$22, a, y1	# y1 = a >> 22				# S0A
-	mov	a, y3		# y3 = a                                # MAJA
-
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
-	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
-	offset = 4*3 + \disp
-	addl	offset(%rsp, SRND), h		# h = k + w + h # --
-	or	c, y3		# y3 = a|c                              # MAJA
-
-	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-	and	c, T1		# T1 = a&c                              # MAJB
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-
-	add	h, d		# d = k + w + h + d                     # --
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-
-
-	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-
-	add	y3, h		# h = t1 + S0 + MAJ                     # --
-
-	ROTATE_ARGS
-
-.endm
-
-########################################################################
-## void sha256_transform_rorx(struct sha256_state *state, const u8 *data, int blocks)
-## arg 1 : pointer to state
-## arg 2 : pointer to input data
-## arg 3 : Num blocks
-########################################################################
-.text
-SYM_TYPED_FUNC_START(sha256_transform_rorx)
-	pushq	%rbx
-	pushq	%r12
-	pushq	%r13
-	pushq	%r14
-	pushq	%r15
-
-	push	%rbp
-	mov	%rsp, %rbp
-
-	subq	$STACK_SIZE, %rsp
-	and	$-32, %rsp	# align rsp to 32 byte boundary
-
-	shl	$6, NUM_BLKS	# convert to bytes
-	jz	.Ldone_hash
-	lea	-64(INP, NUM_BLKS), NUM_BLKS # pointer to last block
-	mov	NUM_BLKS, _INP_END(%rsp)
-
-	cmp	NUM_BLKS, INP
-	je	.Lonly_one_block
-
-	## load initial digest
-	mov	(CTX), a
-	mov	4*1(CTX), b
-	mov	4*2(CTX), c
-	mov	4*3(CTX), d
-	mov	4*4(CTX), e
-	mov	4*5(CTX), f
-	mov	4*6(CTX), g
-	mov	4*7(CTX), h
-
-	vmovdqa  PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
-	vmovdqa  _SHUF_00BA(%rip), SHUF_00BA
-	vmovdqa  _SHUF_DC00(%rip), SHUF_DC00
-
-	mov	CTX, _CTX(%rsp)
-
-.Lloop0:
-	## Load first 16 dwords from two blocks
-	VMOVDQ	0*32(INP),XTMP0
-	VMOVDQ	1*32(INP),XTMP1
-	VMOVDQ	2*32(INP),XTMP2
-	VMOVDQ	3*32(INP),XTMP3
-
-	## byte swap data
-	vpshufb	BYTE_FLIP_MASK, XTMP0, XTMP0
-	vpshufb	BYTE_FLIP_MASK, XTMP1, XTMP1
-	vpshufb	BYTE_FLIP_MASK, XTMP2, XTMP2
-	vpshufb	BYTE_FLIP_MASK, XTMP3, XTMP3
-
-	## transpose data into high/low halves
-	vperm2i128	$0x20, XTMP2, XTMP0, X0
-	vperm2i128	$0x31, XTMP2, XTMP0, X1
-	vperm2i128	$0x20, XTMP3, XTMP1, X2
-	vperm2i128	$0x31, XTMP3, XTMP1, X3
-
-.Llast_block_enter:
-	add	$64, INP
-	mov	INP, _INP(%rsp)
-
-	## schedule 48 input dwords, by doing 3 rounds of 12 each
-	xor	SRND, SRND
-
-.align 16
-.Lloop1:
-	leaq	K256+0*32(%rip), INP		## reuse INP as scratch reg
-	vpaddd	(INP, SRND), X0, XFER
-	vmovdqa XFER, 0*32+_XFER(%rsp, SRND)
-	FOUR_ROUNDS_AND_SCHED	(_XFER + 0*32)
-
-	leaq	K256+1*32(%rip), INP
-	vpaddd	(INP, SRND), X0, XFER
-	vmovdqa XFER, 1*32+_XFER(%rsp, SRND)
-	FOUR_ROUNDS_AND_SCHED	(_XFER + 1*32)
-
-	leaq	K256+2*32(%rip), INP
-	vpaddd	(INP, SRND), X0, XFER
-	vmovdqa XFER, 2*32+_XFER(%rsp, SRND)
-	FOUR_ROUNDS_AND_SCHED	(_XFER + 2*32)
-
-	leaq	K256+3*32(%rip), INP
-	vpaddd	(INP, SRND), X0, XFER
-	vmovdqa XFER, 3*32+_XFER(%rsp, SRND)
-	FOUR_ROUNDS_AND_SCHED	(_XFER + 3*32)
-
-	add	$4*32, SRND
-	cmp	$3*4*32, SRND
-	jb	.Lloop1
-
-.Lloop2:
-	## Do last 16 rounds with no scheduling
-	leaq	K256+0*32(%rip), INP
-	vpaddd	(INP, SRND), X0, XFER
-	vmovdqa XFER, 0*32+_XFER(%rsp, SRND)
-	DO_4ROUNDS	(_XFER + 0*32)
-
-	leaq	K256+1*32(%rip), INP
-	vpaddd	(INP, SRND), X1, XFER
-	vmovdqa XFER, 1*32+_XFER(%rsp, SRND)
-	DO_4ROUNDS	(_XFER + 1*32)
-	add	$2*32, SRND
-
-	vmovdqa	X2, X0
-	vmovdqa	X3, X1
-
-	cmp	$4*4*32, SRND
-	jb	.Lloop2
-
-	mov	_CTX(%rsp), CTX
-	mov	_INP(%rsp), INP
-
-	addm    (4*0)(CTX),a
-	addm    (4*1)(CTX),b
-	addm    (4*2)(CTX),c
-	addm    (4*3)(CTX),d
-	addm    (4*4)(CTX),e
-	addm    (4*5)(CTX),f
-	addm    (4*6)(CTX),g
-	addm    (4*7)(CTX),h
-
-	cmp	_INP_END(%rsp), INP
-	ja	.Ldone_hash
-
-	#### Do second block using previously scheduled results
-	xor	SRND, SRND
-.align 16
-.Lloop3:
-	DO_4ROUNDS	(_XFER + 0*32 + 16)
-	DO_4ROUNDS	(_XFER + 1*32 + 16)
-	add	$2*32, SRND
-	cmp	$4*4*32, SRND
-	jb	.Lloop3
-
-	mov	_CTX(%rsp), CTX
-	mov	_INP(%rsp), INP
-	add	$64, INP
-
-	addm    (4*0)(CTX),a
-	addm    (4*1)(CTX),b
-	addm    (4*2)(CTX),c
-	addm    (4*3)(CTX),d
-	addm    (4*4)(CTX),e
-	addm    (4*5)(CTX),f
-	addm    (4*6)(CTX),g
-	addm    (4*7)(CTX),h
-
-	cmp	_INP_END(%rsp), INP
-	jb	.Lloop0
-	ja	.Ldone_hash
-
-.Ldo_last_block:
-	VMOVDQ	0*16(INP),XWORD0
-	VMOVDQ	1*16(INP),XWORD1
-	VMOVDQ	2*16(INP),XWORD2
-	VMOVDQ	3*16(INP),XWORD3
-
-	vpshufb	X_BYTE_FLIP_MASK, XWORD0, XWORD0
-	vpshufb	X_BYTE_FLIP_MASK, XWORD1, XWORD1
-	vpshufb	X_BYTE_FLIP_MASK, XWORD2, XWORD2
-	vpshufb	X_BYTE_FLIP_MASK, XWORD3, XWORD3
-
-	jmp	.Llast_block_enter
-
-.Lonly_one_block:
-
-	## load initial digest
-	mov	(4*0)(CTX),a
-	mov	(4*1)(CTX),b
-	mov	(4*2)(CTX),c
-	mov	(4*3)(CTX),d
-	mov	(4*4)(CTX),e
-	mov	(4*5)(CTX),f
-	mov	(4*6)(CTX),g
-	mov	(4*7)(CTX),h
-
-	vmovdqa	PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
-	vmovdqa	_SHUF_00BA(%rip), SHUF_00BA
-	vmovdqa	_SHUF_DC00(%rip), SHUF_DC00
-
-	mov	CTX, _CTX(%rsp)
-	jmp	.Ldo_last_block
-
-.Ldone_hash:
-
-	mov	%rbp, %rsp
-	pop	%rbp
-
-	popq	%r15
-	popq	%r14
-	popq	%r13
-	popq	%r12
-	popq	%rbx
-	vzeroupper
-	RET
-SYM_FUNC_END(sha256_transform_rorx)
-
-.section	.rodata.cst512.K256, "aM", @progbits, 512
-.align 64
-K256:
-	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
-	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
-	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
-	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
-	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
-	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
-	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
-	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
-	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
-	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
-	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
-	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
-	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
-	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
-	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
-	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
-	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
-	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
-	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
-	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
-	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
-	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
-	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
-	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
-	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
-	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
-	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
-	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
-	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
-	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
-	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
-	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
-
-.section	.rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
-.align 32
-PSHUFFLE_BYTE_FLIP_MASK:
-	.octa 0x0c0d0e0f08090a0b0405060700010203,0x0c0d0e0f08090a0b0405060700010203
-
-# shuffle xBxA -> 00BA
-.section	.rodata.cst32._SHUF_00BA, "aM", @progbits, 32
-.align 32
-_SHUF_00BA:
-	.octa 0xFFFFFFFFFFFFFFFF0b0a090803020100,0xFFFFFFFFFFFFFFFF0b0a090803020100
-
-# shuffle xDxC -> DC00
-.section	.rodata.cst32._SHUF_DC00, "aM", @progbits, 32
-.align 32
-_SHUF_DC00:
-	.octa 0x0b0a090803020100FFFFFFFFFFFFFFFF,0x0b0a090803020100FFFFFFFFFFFFFFFF
diff --git a/arch/x86/crypto/sha256-ssse3-asm.S b/arch/x86/crypto/sha256-ssse3-asm.S
deleted file mode 100644
index 93264ee44543..000000000000
--- a/arch/x86/crypto/sha256-ssse3-asm.S
+++ /dev/null
@@ -1,513 +0,0 @@
-########################################################################
-# Implement fast SHA-256 with SSSE3 instructions. (x86_64)
-#
-# Copyright (C) 2013 Intel Corporation.
-#
-# Authors:
-#     James Guilford <james.guilford@intel.com>
-#     Kirk Yap <kirk.s.yap@intel.com>
-#     Tim Chen <tim.c.chen@linux.intel.com>
-#
-# This software is available to you under a choice of one of two
-# licenses.  You may choose to be licensed under the terms of the GNU
-# General Public License (GPL) Version 2, available from the file
-# COPYING in the main directory of this source tree, or the
-# OpenIB.org BSD license below:
-#
-#     Redistribution and use in source and binary forms, with or
-#     without modification, are permitted provided that the following
-#     conditions are met:
-#
-#      - Redistributions of source code must retain the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer.
-#
-#      - Redistributions in binary form must reproduce the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer in the documentation and/or other materials
-#        provided with the distribution.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-#
-########################################################################
-#
-# This code is described in an Intel White-Paper:
-# "Fast SHA-256 Implementations on Intel Architecture Processors"
-#
-# To find it, surf to http://www.intel.com/p/en_US/embedded
-# and search for that title.
-#
-########################################################################
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-## assume buffers not aligned
-#define    MOVDQ movdqu
-
-################################ Define Macros
-
-# addm [mem], reg
-# Add reg to mem using reg-mem add and store
-.macro addm p1 p2
-        add     \p1, \p2
-        mov     \p2, \p1
-.endm
-
-################################
-
-# COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask
-# Load xmm with mem and byte swap each dword
-.macro COPY_XMM_AND_BSWAP p1 p2 p3
-        MOVDQ \p2, \p1
-        pshufb \p3, \p1
-.endm
-
-################################
-
-X0 = %xmm4
-X1 = %xmm5
-X2 = %xmm6
-X3 = %xmm7
-
-XTMP0 = %xmm0
-XTMP1 = %xmm1
-XTMP2 = %xmm2
-XTMP3 = %xmm3
-XTMP4 = %xmm8
-XFER = %xmm9
-
-SHUF_00BA = %xmm10      # shuffle xBxA -> 00BA
-SHUF_DC00 = %xmm11      # shuffle xDxC -> DC00
-BYTE_FLIP_MASK = %xmm12
-
-NUM_BLKS = %rdx   # 3rd arg
-INP = %rsi        # 2nd arg
-CTX = %rdi        # 1st arg
-
-SRND = %rsi       # clobbers INP
-c = %ecx
-d = %r8d
-e = %edx
-TBL = %r12
-a = %eax
-b = %ebx
-
-f = %r9d
-g = %r10d
-h = %r11d
-
-y0 = %r13d
-y1 = %r14d
-y2 = %r15d
-
-
-
-_INP_END_SIZE = 8
-_INP_SIZE = 8
-_XFER_SIZE = 16
-_XMM_SAVE_SIZE = 0
-
-_INP_END = 0
-_INP            = _INP_END  + _INP_END_SIZE
-_XFER           = _INP      + _INP_SIZE
-_XMM_SAVE       = _XFER     + _XFER_SIZE
-STACK_SIZE      = _XMM_SAVE + _XMM_SAVE_SIZE
-
-# rotate_Xs
-# Rotate values of symbols X0...X3
-.macro rotate_Xs
-X_ = X0
-X0 = X1
-X1 = X2
-X2 = X3
-X3 = X_
-.endm
-
-# ROTATE_ARGS
-# Rotate values of symbols a...h
-.macro ROTATE_ARGS
-TMP_ = h
-h = g
-g = f
-f = e
-e = d
-d = c
-c = b
-b = a
-a = TMP_
-.endm
-
-.macro FOUR_ROUNDS_AND_SCHED
-	## compute s0 four at a time and s1 two at a time
-	## compute W[-16] + W[-7] 4 at a time
-	movdqa  X3, XTMP0
-	mov     e, y0			# y0 = e
-	ror     $(25-11), y0            # y0 = e >> (25-11)
-	mov     a, y1                   # y1 = a
-	palignr $4, X2, XTMP0           # XTMP0 = W[-7]
-	ror     $(22-13), y1            # y1 = a >> (22-13)
-	xor     e, y0                   # y0 = e ^ (e >> (25-11))
-	mov     f, y2                   # y2 = f
-	ror     $(11-6), y0             # y0 = (e >> (11-6)) ^ (e >> (25-6))
-	movdqa  X1, XTMP1
-	xor     a, y1                   # y1 = a ^ (a >> (22-13)
-	xor     g, y2                   # y2 = f^g
-	paddd   X0, XTMP0               # XTMP0 = W[-7] + W[-16]
-	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
-	and     e, y2                   # y2 = (f^g)&e
-	ror     $(13-2), y1             # y1 = (a >> (13-2)) ^ (a >> (22-2))
-	## compute s0
-	palignr $4, X0, XTMP1           # XTMP1 = W[-15]
-	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
-	ror     $6, y0                  # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
-	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
-	movdqa  XTMP1, XTMP2            # XTMP2 = W[-15]
-	ror     $2, y1                  # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
-	add     y0, y2                  # y2 = S1 + CH
-	add     _XFER(%rsp) , y2        # y2 = k + w + S1 + CH
-	movdqa  XTMP1, XTMP3            # XTMP3 = W[-15]
-	mov     a, y0                   # y0 = a
-	add     y2, h                   # h = h + S1 + CH + k + w
-	mov     a, y2                   # y2 = a
-	pslld   $(32-7), XTMP1          #
-	or      c, y0                   # y0 = a|c
-	add     h, d                    # d = d + h + S1 + CH + k + w
-	and     c, y2                   # y2 = a&c
-	psrld   $7, XTMP2               #
-	and     b, y0                   # y0 = (a|c)&b
-	add     y1, h                   # h = h + S1 + CH + k + w + S0
-	por     XTMP2, XTMP1            # XTMP1 = W[-15] ror 7
-	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
-	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
-					#
-	ROTATE_ARGS                     #
-	movdqa  XTMP3, XTMP2            # XTMP2 = W[-15]
-	mov     e, y0                   # y0 = e
-	mov     a, y1                   # y1 = a
-	movdqa  XTMP3, XTMP4            # XTMP4 = W[-15]
-	ror     $(25-11), y0            # y0 = e >> (25-11)
-	xor     e, y0                   # y0 = e ^ (e >> (25-11))
-	mov     f, y2                   # y2 = f
-	ror     $(22-13), y1            # y1 = a >> (22-13)
-	pslld   $(32-18), XTMP3         #
-	xor     a, y1                   # y1 = a ^ (a >> (22-13)
-	ror     $(11-6), y0             # y0 = (e >> (11-6)) ^ (e >> (25-6))
-	xor     g, y2                   # y2 = f^g
-	psrld   $18, XTMP2              #
-	ror     $(13-2), y1             # y1 = (a >> (13-2)) ^ (a >> (22-2))
-	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
-	and     e, y2                   # y2 = (f^g)&e
-	ror     $6, y0                  # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
-	pxor    XTMP3, XTMP1
-	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
-	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
-	psrld   $3, XTMP4               # XTMP4 = W[-15] >> 3
-	add     y0, y2                  # y2 = S1 + CH
-	add     (1*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
-	ror     $2, y1                  # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
-	pxor    XTMP2, XTMP1            # XTMP1 = W[-15] ror 7 ^ W[-15] ror 18
-	mov     a, y0                   # y0 = a
-	add     y2, h                   # h = h + S1 + CH + k + w
-	mov     a, y2                   # y2 = a
-	pxor    XTMP4, XTMP1            # XTMP1 = s0
-	or      c, y0                   # y0 = a|c
-	add     h, d                    # d = d + h + S1 + CH + k + w
-	and     c, y2                   # y2 = a&c
-	## compute low s1
-	pshufd  $0b11111010, X3, XTMP2   # XTMP2 = W[-2] {BBAA}
-	and     b, y0			# y0 = (a|c)&b
-	add     y1, h                   # h = h + S1 + CH + k + w + S0
-	paddd   XTMP1, XTMP0            # XTMP0 = W[-16] + W[-7] + s0
-	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
-	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
-
-	ROTATE_ARGS
-	movdqa  XTMP2, XTMP3            # XTMP3 = W[-2] {BBAA}
-	mov     e, y0                   # y0 = e
-	mov     a, y1                   # y1 = a
-	ror     $(25-11), y0            # y0 = e >> (25-11)
-	movdqa  XTMP2, XTMP4            # XTMP4 = W[-2] {BBAA}
-	xor     e, y0                   # y0 = e ^ (e >> (25-11))
-	ror     $(22-13), y1            # y1 = a >> (22-13)
-	mov     f, y2                   # y2 = f
-	xor     a, y1                   # y1 = a ^ (a >> (22-13)
-	ror     $(11-6), y0             # y0 = (e >> (11-6)) ^ (e >> (25-6))
-	psrlq   $17, XTMP2              # XTMP2 = W[-2] ror 17 {xBxA}
-	xor     g, y2                   # y2 = f^g
-	psrlq   $19, XTMP3              # XTMP3 = W[-2] ror 19 {xBxA}
-	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
-	and     e, y2                   # y2 = (f^g)&e
-	psrld   $10, XTMP4              # XTMP4 = W[-2] >> 10 {BBAA}
-	ror     $(13-2), y1             # y1 = (a >> (13-2)) ^ (a >> (22-2))
-	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
-	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
-	ror     $6, y0                  # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
-	pxor    XTMP3, XTMP2
-	add     y0, y2                  # y2 = S1 + CH
-	ror     $2, y1                  # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
-	add     (2*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
-	pxor    XTMP2, XTMP4            # XTMP4 = s1 {xBxA}
-	mov     a, y0                   # y0 = a
-	add     y2, h                   # h = h + S1 + CH + k + w
-	mov     a, y2                   # y2 = a
-	pshufb  SHUF_00BA, XTMP4        # XTMP4 = s1 {00BA}
-	or      c, y0                   # y0 = a|c
-	add     h, d                    # d = d + h + S1 + CH + k + w
-	and     c, y2                   # y2 = a&c
-	paddd   XTMP4, XTMP0            # XTMP0 = {..., ..., W[1], W[0]}
-	and     b, y0                   # y0 = (a|c)&b
-	add     y1, h                   # h = h + S1 + CH + k + w + S0
-	## compute high s1
-	pshufd  $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {BBAA}
-	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
-	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
-					#
-	ROTATE_ARGS                     #
-	movdqa  XTMP2, XTMP3            # XTMP3 = W[-2] {DDCC}
-	mov     e, y0                   # y0 = e
-	ror     $(25-11), y0            # y0 = e >> (25-11)
-	mov     a, y1                   # y1 = a
-	movdqa  XTMP2, X0               # X0    = W[-2] {DDCC}
-	ror     $(22-13), y1            # y1 = a >> (22-13)
-	xor     e, y0                   # y0 = e ^ (e >> (25-11))
-	mov     f, y2                   # y2 = f
-	ror     $(11-6), y0             # y0 = (e >> (11-6)) ^ (e >> (25-6))
-	psrlq   $17, XTMP2              # XTMP2 = W[-2] ror 17 {xDxC}
-	xor     a, y1                   # y1 = a ^ (a >> (22-13)
-	xor     g, y2                   # y2 = f^g
-	psrlq   $19, XTMP3              # XTMP3 = W[-2] ror 19 {xDxC}
-	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25
-	and     e, y2                   # y2 = (f^g)&e
-	ror     $(13-2), y1             # y1 = (a >> (13-2)) ^ (a >> (22-2))
-	psrld   $10, X0                 # X0 = W[-2] >> 10 {DDCC}
-	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22
-	ror     $6, y0                  # y0 = S1 = (e>>6) & (e>>11) ^ (e>>2
-	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
-	pxor    XTMP3, XTMP2            #
-	ror     $2, y1                  # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>2
-	add     y0, y2                  # y2 = S1 + CH
-	add     (3*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
-	pxor    XTMP2, X0               # X0 = s1 {xDxC}
-	mov     a, y0                   # y0 = a
-	add     y2, h                   # h = h + S1 + CH + k + w
-	mov     a, y2                   # y2 = a
-	pshufb  SHUF_DC00, X0           # X0 = s1 {DC00}
-	or      c, y0                   # y0 = a|c
-	add     h, d                    # d = d + h + S1 + CH + k + w
-	and     c, y2                   # y2 = a&c
-	paddd   XTMP0, X0               # X0 = {W[3], W[2], W[1], W[0]}
-	and     b, y0                   # y0 = (a|c)&b
-	add     y1, h                   # h = h + S1 + CH + k + w + S0
-	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
-	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
-
-	ROTATE_ARGS
-	rotate_Xs
-.endm
-
-## input is [rsp + _XFER + %1 * 4]
-.macro DO_ROUND round
-	mov     e, y0                 # y0 = e
-	ror     $(25-11), y0          # y0 = e >> (25-11)
-	mov     a, y1                 # y1 = a
-	xor     e, y0                 # y0 = e ^ (e >> (25-11))
-	ror     $(22-13), y1          # y1 = a >> (22-13)
-	mov     f, y2                 # y2 = f
-	xor     a, y1                 # y1 = a ^ (a >> (22-13)
-	ror     $(11-6), y0           # y0 = (e >> (11-6)) ^ (e >> (25-6))
-	xor     g, y2                 # y2 = f^g
-	xor     e, y0                 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
-	ror     $(13-2), y1           # y1 = (a >> (13-2)) ^ (a >> (22-2))
-	and     e, y2                 # y2 = (f^g)&e
-	xor     a, y1                 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
-	ror     $6, y0                # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
-	xor     g, y2                 # y2 = CH = ((f^g)&e)^g
-	add     y0, y2                # y2 = S1 + CH
-	ror     $2, y1                # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
-	offset = \round * 4 + _XFER
-	add     offset(%rsp), y2      # y2 = k + w + S1 + CH
-	mov     a, y0                 # y0 = a
-	add     y2, h                 # h = h + S1 + CH + k + w
-	mov     a, y2                 # y2 = a
-	or      c, y0                 # y0 = a|c
-	add     h, d                  # d = d + h + S1 + CH + k + w
-	and     c, y2                 # y2 = a&c
-	and     b, y0                 # y0 = (a|c)&b
-	add     y1, h                 # h = h + S1 + CH + k + w + S0
-	or      y2, y0		      # y0 = MAJ = (a|c)&b)|(a&c)
-	add     y0, h		      # h = h + S1 + CH + k + w + S0 + MAJ
-	ROTATE_ARGS
-.endm
-
-########################################################################
-## void sha256_transform_ssse3(struct sha256_state *state, const u8 *data,
-##			       int blocks);
-## arg 1 : pointer to state
-##	   (struct sha256_state is assumed to begin with u32 state[8])
-## arg 2 : pointer to input data
-## arg 3 : Num blocks
-########################################################################
-.text
-SYM_TYPED_FUNC_START(sha256_transform_ssse3)
-	pushq   %rbx
-	pushq   %r12
-	pushq   %r13
-	pushq   %r14
-	pushq   %r15
-	pushq   %rbp
-	mov	%rsp, %rbp
-
-	subq    $STACK_SIZE, %rsp
-	and	$~15, %rsp
-
-	shl     $6, NUM_BLKS		 # convert to bytes
-	jz      .Ldone_hash
-	add     INP, NUM_BLKS
-	mov     NUM_BLKS, _INP_END(%rsp) # pointer to end of data
-
-	## load initial digest
-	mov     4*0(CTX), a
-	mov     4*1(CTX), b
-	mov     4*2(CTX), c
-	mov     4*3(CTX), d
-	mov     4*4(CTX), e
-	mov     4*5(CTX), f
-	mov     4*6(CTX), g
-	mov     4*7(CTX), h
-
-	movdqa  PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
-	movdqa  _SHUF_00BA(%rip), SHUF_00BA
-	movdqa  _SHUF_DC00(%rip), SHUF_DC00
-
-.Lloop0:
-	lea     K256(%rip), TBL
-
-	## byte swap first 16 dwords
-	COPY_XMM_AND_BSWAP      X0, 0*16(INP), BYTE_FLIP_MASK
-	COPY_XMM_AND_BSWAP      X1, 1*16(INP), BYTE_FLIP_MASK
-	COPY_XMM_AND_BSWAP      X2, 2*16(INP), BYTE_FLIP_MASK
-	COPY_XMM_AND_BSWAP      X3, 3*16(INP), BYTE_FLIP_MASK
-
-	mov     INP, _INP(%rsp)
-
-	## schedule 48 input dwords, by doing 3 rounds of 16 each
-	mov     $3, SRND
-.align 16
-.Lloop1:
-	movdqa  (TBL), XFER
-	paddd   X0, XFER
-	movdqa  XFER, _XFER(%rsp)
-	FOUR_ROUNDS_AND_SCHED
-
-	movdqa  1*16(TBL), XFER
-	paddd   X0, XFER
-	movdqa  XFER, _XFER(%rsp)
-	FOUR_ROUNDS_AND_SCHED
-
-	movdqa  2*16(TBL), XFER
-	paddd   X0, XFER
-	movdqa  XFER, _XFER(%rsp)
-	FOUR_ROUNDS_AND_SCHED
-
-	movdqa  3*16(TBL), XFER
-	paddd   X0, XFER
-	movdqa  XFER, _XFER(%rsp)
-	add     $4*16, TBL
-	FOUR_ROUNDS_AND_SCHED
-
-	sub     $1, SRND
-	jne     .Lloop1
-
-	mov     $2, SRND
-.Lloop2:
-	paddd   (TBL), X0
-	movdqa  X0, _XFER(%rsp)
-	DO_ROUND        0
-	DO_ROUND        1
-	DO_ROUND        2
-	DO_ROUND        3
-	paddd   1*16(TBL), X1
-	movdqa  X1, _XFER(%rsp)
-	add     $2*16, TBL
-	DO_ROUND        0
-	DO_ROUND        1
-	DO_ROUND        2
-	DO_ROUND        3
-
-	movdqa  X2, X0
-	movdqa  X3, X1
-
-	sub     $1, SRND
-	jne     .Lloop2
-
-	addm    (4*0)(CTX),a
-	addm    (4*1)(CTX),b
-	addm    (4*2)(CTX),c
-	addm    (4*3)(CTX),d
-	addm    (4*4)(CTX),e
-	addm    (4*5)(CTX),f
-	addm    (4*6)(CTX),g
-	addm    (4*7)(CTX),h
-
-	mov     _INP(%rsp), INP
-	add     $64, INP
-	cmp     _INP_END(%rsp), INP
-	jne     .Lloop0
-
-.Ldone_hash:
-
-	mov	%rbp, %rsp
-	popq	%rbp
-	popq    %r15
-	popq    %r14
-	popq    %r13
-	popq    %r12
-	popq    %rbx
-
-	RET
-SYM_FUNC_END(sha256_transform_ssse3)
-
-.section	.rodata.cst256.K256, "aM", @progbits, 256
-.align 64
-K256:
-        .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
-        .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
-        .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
-        .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
-        .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
-        .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
-        .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
-        .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
-        .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
-        .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
-        .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
-        .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
-        .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
-        .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
-        .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
-        .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
-
-.section	.rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
-.align 16
-PSHUFFLE_BYTE_FLIP_MASK:
-	.octa 0x0c0d0e0f08090a0b0405060700010203
-
-.section	.rodata.cst16._SHUF_00BA, "aM", @progbits, 16
-.align 16
-# shuffle xBxA -> 00BA
-_SHUF_00BA:
-	.octa 0xFFFFFFFFFFFFFFFF0b0a090803020100
-
-.section	.rodata.cst16._SHUF_DC00, "aM", @progbits, 16
-.align 16
-# shuffle xDxC -> DC00
-_SHUF_DC00:
-	.octa 0x0b0a090803020100FFFFFFFFFFFFFFFF
diff --git a/arch/x86/crypto/sha256_ni_asm.S b/arch/x86/crypto/sha256_ni_asm.S
deleted file mode 100644
index d515a55a3bc1..000000000000
--- a/arch/x86/crypto/sha256_ni_asm.S
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Intel SHA Extensions optimized implementation of a SHA-256 update function
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Contact Information:
- * 	Sean Gulley <sean.m.gulley@intel.com>
- * 	Tim Chen <tim.c.chen@linux.intel.com>
- *
- * BSD LICENSE
- *
- * Copyright(c) 2015 Intel Corporation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 	* Redistributions of source code must retain the above copyright
- * 	  notice, this list of conditions and the following disclaimer.
- * 	* Redistributions in binary form must reproduce the above copyright
- * 	  notice, this list of conditions and the following disclaimer in
- * 	  the documentation and/or other materials provided with the
- * 	  distribution.
- * 	* Neither the name of Intel Corporation nor the names of its
- * 	  contributors may be used to endorse or promote products derived
- * 	  from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-#define DIGEST_PTR	%rdi	/* 1st arg */
-#define DATA_PTR	%rsi	/* 2nd arg */
-#define NUM_BLKS	%rdx	/* 3rd arg */
-
-#define SHA256CONSTANTS	%rax
-
-#define MSG		%xmm0  /* sha256rnds2 implicit operand */
-#define STATE0		%xmm1
-#define STATE1		%xmm2
-#define MSG0		%xmm3
-#define MSG1		%xmm4
-#define MSG2		%xmm5
-#define MSG3		%xmm6
-#define TMP		%xmm7
-
-#define SHUF_MASK	%xmm8
-
-#define ABEF_SAVE	%xmm9
-#define CDGH_SAVE	%xmm10
-
-.macro do_4rounds	i, m0, m1, m2, m3
-.if \i < 16
-	movdqu		\i*4(DATA_PTR), \m0
-	pshufb		SHUF_MASK, \m0
-.endif
-	movdqa		(\i-32)*4(SHA256CONSTANTS), MSG
-	paddd		\m0, MSG
-	sha256rnds2	STATE0, STATE1
-.if \i >= 12 && \i < 60
-	movdqa		\m0, TMP
-	palignr		$4, \m3, TMP
-	paddd		TMP, \m1
-	sha256msg2	\m0, \m1
-.endif
-	punpckhqdq	MSG, MSG
-	sha256rnds2	STATE1, STATE0
-.if \i >= 4 && \i < 52
-	sha256msg1	\m0, \m3
-.endif
-.endm
-
-/*
- * Intel SHA Extensions optimized implementation of a SHA-256 update function
- *
- * The function takes a pointer to the current hash values, a pointer to the
- * input data, and a number of 64 byte blocks to process.  Once all blocks have
- * been processed, the digest pointer is  updated with the resulting hash value.
- * The function only processes complete blocks, there is no functionality to
- * store partial blocks.  All message padding and hash value initialization must
- * be done outside the update function.
- *
- * void sha256_ni_transform(uint32_t *digest, const void *data,
-		uint32_t numBlocks);
- * digest : pointer to digest
- * data: pointer to input data
- * numBlocks: Number of blocks to process
- */
-
-.text
-SYM_TYPED_FUNC_START(sha256_ni_transform)
-
-	shl		$6, NUM_BLKS		/*  convert to bytes */
-	jz		.Ldone_hash
-	add		DATA_PTR, NUM_BLKS	/* pointer to end of data */
-
-	/*
-	 * load initial hash values
-	 * Need to reorder these appropriately
-	 * DCBA, HGFE -> ABEF, CDGH
-	 */
-	movdqu		0*16(DIGEST_PTR), STATE0	/* DCBA */
-	movdqu		1*16(DIGEST_PTR), STATE1	/* HGFE */
-
-	movdqa		STATE0, TMP
-	punpcklqdq	STATE1, STATE0			/* FEBA */
-	punpckhqdq	TMP, STATE1			/* DCHG */
-	pshufd		$0x1B, STATE0, STATE0		/* ABEF */
-	pshufd		$0xB1, STATE1, STATE1		/* CDGH */
-
-	movdqa		PSHUFFLE_BYTE_FLIP_MASK(%rip), SHUF_MASK
-	lea		K256+32*4(%rip), SHA256CONSTANTS
-
-.Lloop0:
-	/* Save hash values for addition after rounds */
-	movdqa		STATE0, ABEF_SAVE
-	movdqa		STATE1, CDGH_SAVE
-
-.irp i, 0, 16, 32, 48
-	do_4rounds	(\i + 0),  MSG0, MSG1, MSG2, MSG3
-	do_4rounds	(\i + 4),  MSG1, MSG2, MSG3, MSG0
-	do_4rounds	(\i + 8),  MSG2, MSG3, MSG0, MSG1
-	do_4rounds	(\i + 12), MSG3, MSG0, MSG1, MSG2
-.endr
-
-	/* Add current hash values with previously saved */
-	paddd		ABEF_SAVE, STATE0
-	paddd		CDGH_SAVE, STATE1
-
-	/* Increment data pointer and loop if more to process */
-	add		$64, DATA_PTR
-	cmp		NUM_BLKS, DATA_PTR
-	jne		.Lloop0
-
-	/* Write hash values back in the correct order */
-	movdqa		STATE0, TMP
-	punpcklqdq	STATE1, STATE0			/* GHEF */
-	punpckhqdq	TMP, STATE1			/* ABCD */
-	pshufd		$0xB1, STATE0, STATE0		/* HGFE */
-	pshufd		$0x1B, STATE1, STATE1		/* DCBA */
-
-	movdqu		STATE1, 0*16(DIGEST_PTR)
-	movdqu		STATE0, 1*16(DIGEST_PTR)
-
-.Ldone_hash:
-
-	RET
-SYM_FUNC_END(sha256_ni_transform)
-
-.section	.rodata.cst256.K256, "aM", @progbits, 256
-.align 64
-K256:
-	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
-	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
-	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
-	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
-	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
-	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
-	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
-	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
-	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
-	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
-	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
-	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
-	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
-	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
-	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
-	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
-
-.section	.rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
-.align 16
-PSHUFFLE_BYTE_FLIP_MASK:
-	.octa 0x0c0d0e0f08090a0b0405060700010203
diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c
deleted file mode 100644
index e04a43d9f7d5..000000000000
--- a/arch/x86/crypto/sha256_ssse3_glue.c
+++ /dev/null
@@ -1,467 +0,0 @@
-/*
- * Cryptographic API.
- *
- * Glue code for the SHA256 Secure Hash Algorithm assembler implementations
- * using SSSE3, AVX, AVX2, and SHA-NI instructions.
- *
- * This file is based on sha256_generic.c
- *
- * Copyright (C) 2013 Intel Corporation.
- *
- * Author:
- *     Tim Chen <tim.c.chen@linux.intel.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-
-#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
-
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/mm.h>
-#include <linux/types.h>
-#include <crypto/sha2.h>
-#include <crypto/sha256_base.h>
-#include <linux/string.h>
-#include <asm/cpu_device_id.h>
-#include <asm/simd.h>
-
-asmlinkage void sha256_transform_ssse3(struct sha256_state *state,
-				       const u8 *data, int blocks);
-
-static const struct x86_cpu_id module_cpu_ids[] = {
-#ifdef CONFIG_AS_SHA256_NI
-	X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL),
-#endif
-	X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL),
-	X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL),
-	X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL),
-	{}
-};
-MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids);
-
-static int _sha256_update(struct shash_desc *desc, const u8 *data,
-			  unsigned int len, sha256_block_fn *sha256_xform)
-{
-	struct sha256_state *sctx = shash_desc_ctx(desc);
-
-	if (!crypto_simd_usable() ||
-	    (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
-		return crypto_sha256_update(desc, data, len);
-
-	/*
-	 * Make sure struct sha256_state begins directly with the SHA256
-	 * 256-bit internal state, as this is what the asm functions expect.
-	 */
-	BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);
-
-	kernel_fpu_begin();
-	sha256_base_do_update(desc, data, len, sha256_xform);
-	kernel_fpu_end();
-
-	return 0;
-}
-
-static int sha256_finup(struct shash_desc *desc, const u8 *data,
-	      unsigned int len, u8 *out, sha256_block_fn *sha256_xform)
-{
-	if (!crypto_simd_usable())
-		return crypto_sha256_finup(desc, data, len, out);
-
-	kernel_fpu_begin();
-	if (len)
-		sha256_base_do_update(desc, data, len, sha256_xform);
-	sha256_base_do_finalize(desc, sha256_xform);
-	kernel_fpu_end();
-
-	return sha256_base_finish(desc, out);
-}
-
-static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
-			 unsigned int len)
-{
-	return _sha256_update(desc, data, len, sha256_transform_ssse3);
-}
-
-static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
-	      unsigned int len, u8 *out)
-{
-	return sha256_finup(desc, data, len, out, sha256_transform_ssse3);
-}
-
-/* Add padding and return the message digest. */
-static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
-{
-	return sha256_ssse3_finup(desc, NULL, 0, out);
-}
-
-static int sha256_ssse3_digest(struct shash_desc *desc, const u8 *data,
-	      unsigned int len, u8 *out)
-{
-	return sha256_base_init(desc) ?:
-	       sha256_ssse3_finup(desc, data, len, out);
-}
-
-static struct shash_alg sha256_ssse3_algs[] = { {
-	.digestsize	=	SHA256_DIGEST_SIZE,
-	.init		=	sha256_base_init,
-	.update		=	sha256_ssse3_update,
-	.final		=	sha256_ssse3_final,
-	.finup		=	sha256_ssse3_finup,
-	.digest		=	sha256_ssse3_digest,
-	.descsize	=	sizeof(struct sha256_state),
-	.base		=	{
-		.cra_name	=	"sha256",
-		.cra_driver_name =	"sha256-ssse3",
-		.cra_priority	=	150,
-		.cra_blocksize	=	SHA256_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-}, {
-	.digestsize	=	SHA224_DIGEST_SIZE,
-	.init		=	sha224_base_init,
-	.update		=	sha256_ssse3_update,
-	.final		=	sha256_ssse3_final,
-	.finup		=	sha256_ssse3_finup,
-	.descsize	=	sizeof(struct sha256_state),
-	.base		=	{
-		.cra_name	=	"sha224",
-		.cra_driver_name =	"sha224-ssse3",
-		.cra_priority	=	150,
-		.cra_blocksize	=	SHA224_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-} };
-
-static int register_sha256_ssse3(void)
-{
-	if (boot_cpu_has(X86_FEATURE_SSSE3))
-		return crypto_register_shashes(sha256_ssse3_algs,
-				ARRAY_SIZE(sha256_ssse3_algs));
-	return 0;
-}
-
-static void unregister_sha256_ssse3(void)
-{
-	if (boot_cpu_has(X86_FEATURE_SSSE3))
-		crypto_unregister_shashes(sha256_ssse3_algs,
-				ARRAY_SIZE(sha256_ssse3_algs));
-}
-
-asmlinkage void sha256_transform_avx(struct sha256_state *state,
-				     const u8 *data, int blocks);
-
-static int sha256_avx_update(struct shash_desc *desc, const u8 *data,
-			 unsigned int len)
-{
-	return _sha256_update(desc, data, len, sha256_transform_avx);
-}
-
-static int sha256_avx_finup(struct shash_desc *desc, const u8 *data,
-		      unsigned int len, u8 *out)
-{
-	return sha256_finup(desc, data, len, out, sha256_transform_avx);
-}
-
-static int sha256_avx_final(struct shash_desc *desc, u8 *out)
-{
-	return sha256_avx_finup(desc, NULL, 0, out);
-}
-
-static int sha256_avx_digest(struct shash_desc *desc, const u8 *data,
-		      unsigned int len, u8 *out)
-{
-	return sha256_base_init(desc) ?:
-	       sha256_avx_finup(desc, data, len, out);
-}
-
-static struct shash_alg sha256_avx_algs[] = { {
-	.digestsize	=	SHA256_DIGEST_SIZE,
-	.init		=	sha256_base_init,
-	.update		=	sha256_avx_update,
-	.final		=	sha256_avx_final,
-	.finup		=	sha256_avx_finup,
-	.digest		=	sha256_avx_digest,
-	.descsize	=	sizeof(struct sha256_state),
-	.base		=	{
-		.cra_name	=	"sha256",
-		.cra_driver_name =	"sha256-avx",
-		.cra_priority	=	160,
-		.cra_blocksize	=	SHA256_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-}, {
-	.digestsize	=	SHA224_DIGEST_SIZE,
-	.init		=	sha224_base_init,
-	.update		=	sha256_avx_update,
-	.final		=	sha256_avx_final,
-	.finup		=	sha256_avx_finup,
-	.descsize	=	sizeof(struct sha256_state),
-	.base		=	{
-		.cra_name	=	"sha224",
-		.cra_driver_name =	"sha224-avx",
-		.cra_priority	=	160,
-		.cra_blocksize	=	SHA224_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-} };
-
-static bool avx_usable(void)
-{
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
-		if (boot_cpu_has(X86_FEATURE_AVX))
-			pr_info("AVX detected but unusable.\n");
-		return false;
-	}
-
-	return true;
-}
-
-static int register_sha256_avx(void)
-{
-	if (avx_usable())
-		return crypto_register_shashes(sha256_avx_algs,
-				ARRAY_SIZE(sha256_avx_algs));
-	return 0;
-}
-
-static void unregister_sha256_avx(void)
-{
-	if (avx_usable())
-		crypto_unregister_shashes(sha256_avx_algs,
-				ARRAY_SIZE(sha256_avx_algs));
-}
-
-asmlinkage void sha256_transform_rorx(struct sha256_state *state,
-				      const u8 *data, int blocks);
-
-static int sha256_avx2_update(struct shash_desc *desc, const u8 *data,
-			 unsigned int len)
-{
-	return _sha256_update(desc, data, len, sha256_transform_rorx);
-}
-
-static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data,
-		      unsigned int len, u8 *out)
-{
-	return sha256_finup(desc, data, len, out, sha256_transform_rorx);
-}
-
-static int sha256_avx2_final(struct shash_desc *desc, u8 *out)
-{
-	return sha256_avx2_finup(desc, NULL, 0, out);
-}
-
-static int sha256_avx2_digest(struct shash_desc *desc, const u8 *data,
-		      unsigned int len, u8 *out)
-{
-	return sha256_base_init(desc) ?:
-	       sha256_avx2_finup(desc, data, len, out);
-}
-
-static struct shash_alg sha256_avx2_algs[] = { {
-	.digestsize	=	SHA256_DIGEST_SIZE,
-	.init		=	sha256_base_init,
-	.update		=	sha256_avx2_update,
-	.final		=	sha256_avx2_final,
-	.finup		=	sha256_avx2_finup,
-	.digest		=	sha256_avx2_digest,
-	.descsize	=	sizeof(struct sha256_state),
-	.base		=	{
-		.cra_name	=	"sha256",
-		.cra_driver_name =	"sha256-avx2",
-		.cra_priority	=	170,
-		.cra_blocksize	=	SHA256_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-}, {
-	.digestsize	=	SHA224_DIGEST_SIZE,
-	.init		=	sha224_base_init,
-	.update		=	sha256_avx2_update,
-	.final		=	sha256_avx2_final,
-	.finup		=	sha256_avx2_finup,
-	.descsize	=	sizeof(struct sha256_state),
-	.base		=	{
-		.cra_name	=	"sha224",
-		.cra_driver_name =	"sha224-avx2",
-		.cra_priority	=	170,
-		.cra_blocksize	=	SHA224_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-} };
-
-static bool avx2_usable(void)
-{
-	if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
-		    boot_cpu_has(X86_FEATURE_BMI2))
-		return true;
-
-	return false;
-}
-
-static int register_sha256_avx2(void)
-{
-	if (avx2_usable())
-		return crypto_register_shashes(sha256_avx2_algs,
-				ARRAY_SIZE(sha256_avx2_algs));
-	return 0;
-}
-
-static void unregister_sha256_avx2(void)
-{
-	if (avx2_usable())
-		crypto_unregister_shashes(sha256_avx2_algs,
-				ARRAY_SIZE(sha256_avx2_algs));
-}
-
-#ifdef CONFIG_AS_SHA256_NI
-asmlinkage void sha256_ni_transform(struct sha256_state *digest,
-				    const u8 *data, int rounds);
-
-static int sha256_ni_update(struct shash_desc *desc, const u8 *data,
-			 unsigned int len)
-{
-	return _sha256_update(desc, data, len, sha256_ni_transform);
-}
-
-static int sha256_ni_finup(struct shash_desc *desc, const u8 *data,
-		      unsigned int len, u8 *out)
-{
-	return sha256_finup(desc, data, len, out, sha256_ni_transform);
-}
-
-static int sha256_ni_final(struct shash_desc *desc, u8 *out)
-{
-	return sha256_ni_finup(desc, NULL, 0, out);
-}
-
-static int sha256_ni_digest(struct shash_desc *desc, const u8 *data,
-		      unsigned int len, u8 *out)
-{
-	return sha256_base_init(desc) ?:
-	       sha256_ni_finup(desc, data, len, out);
-}
-
-static struct shash_alg sha256_ni_algs[] = { {
-	.digestsize	=	SHA256_DIGEST_SIZE,
-	.init		=	sha256_base_init,
-	.update		=	sha256_ni_update,
-	.final		=	sha256_ni_final,
-	.finup		=	sha256_ni_finup,
-	.digest		=	sha256_ni_digest,
-	.descsize	=	sizeof(struct sha256_state),
-	.base		=	{
-		.cra_name	=	"sha256",
-		.cra_driver_name =	"sha256-ni",
-		.cra_priority	=	250,
-		.cra_blocksize	=	SHA256_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-}, {
-	.digestsize	=	SHA224_DIGEST_SIZE,
-	.init		=	sha224_base_init,
-	.update		=	sha256_ni_update,
-	.final		=	sha256_ni_final,
-	.finup		=	sha256_ni_finup,
-	.descsize	=	sizeof(struct sha256_state),
-	.base		=	{
-		.cra_name	=	"sha224",
-		.cra_driver_name =	"sha224-ni",
-		.cra_priority	=	250,
-		.cra_blocksize	=	SHA224_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-} };
-
-static int register_sha256_ni(void)
-{
-	if (boot_cpu_has(X86_FEATURE_SHA_NI))
-		return crypto_register_shashes(sha256_ni_algs,
-				ARRAY_SIZE(sha256_ni_algs));
-	return 0;
-}
-
-static void unregister_sha256_ni(void)
-{
-	if (boot_cpu_has(X86_FEATURE_SHA_NI))
-		crypto_unregister_shashes(sha256_ni_algs,
-				ARRAY_SIZE(sha256_ni_algs));
-}
-
-#else
-static inline int register_sha256_ni(void) { return 0; }
-static inline void unregister_sha256_ni(void) { }
-#endif
-
-static int __init sha256_ssse3_mod_init(void)
-{
-	if (!x86_match_cpu(module_cpu_ids))
-		return -ENODEV;
-
-	if (register_sha256_ssse3())
-		goto fail;
-
-	if (register_sha256_avx()) {
-		unregister_sha256_ssse3();
-		goto fail;
-	}
-
-	if (register_sha256_avx2()) {
-		unregister_sha256_avx();
-		unregister_sha256_ssse3();
-		goto fail;
-	}
-
-	if (register_sha256_ni()) {
-		unregister_sha256_avx2();
-		unregister_sha256_avx();
-		unregister_sha256_ssse3();
-		goto fail;
-	}
-
-	return 0;
-fail:
-	return -ENODEV;
-}
-
-static void __exit sha256_ssse3_mod_fini(void)
-{
-	unregister_sha256_ni();
-	unregister_sha256_avx2();
-	unregister_sha256_avx();
-	unregister_sha256_ssse3();
-}
-
-module_init(sha256_ssse3_mod_init);
-module_exit(sha256_ssse3_mod_fini);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
-
-MODULE_ALIAS_CRYPTO("sha256");
-MODULE_ALIAS_CRYPTO("sha256-ssse3");
-MODULE_ALIAS_CRYPTO("sha256-avx");
-MODULE_ALIAS_CRYPTO("sha256-avx2");
-MODULE_ALIAS_CRYPTO("sha224");
-MODULE_ALIAS_CRYPTO("sha224-ssse3");
-MODULE_ALIAS_CRYPTO("sha224-avx");
-MODULE_ALIAS_CRYPTO("sha224-avx2");
-#ifdef CONFIG_AS_SHA256_NI
-MODULE_ALIAS_CRYPTO("sha256-ni");
-MODULE_ALIAS_CRYPTO("sha224-ni");
-#endif
diff --git a/arch/x86/crypto/sha512-avx-asm.S b/arch/x86/crypto/sha512-avx-asm.S
deleted file mode 100644
index 5bfce4b045fd..000000000000
--- a/arch/x86/crypto/sha512-avx-asm.S
+++ /dev/null
@@ -1,423 +0,0 @@
-########################################################################
-# Implement fast SHA-512 with AVX instructions. (x86_64)
-#
-# Copyright (C) 2013 Intel Corporation.
-#
-# Authors:
-#     James Guilford <james.guilford@intel.com>
-#     Kirk Yap <kirk.s.yap@intel.com>
-#     David Cote <david.m.cote@intel.com>
-#     Tim Chen <tim.c.chen@linux.intel.com>
-#
-# This software is available to you under a choice of one of two
-# licenses.  You may choose to be licensed under the terms of the GNU
-# General Public License (GPL) Version 2, available from the file
-# COPYING in the main directory of this source tree, or the
-# OpenIB.org BSD license below:
-#
-#     Redistribution and use in source and binary forms, with or
-#     without modification, are permitted provided that the following
-#     conditions are met:
-#
-#      - Redistributions of source code must retain the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer.
-#
-#      - Redistributions in binary form must reproduce the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer in the documentation and/or other materials
-#        provided with the distribution.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-#
-########################################################################
-#
-# This code is described in an Intel White-Paper:
-# "Fast SHA-512 Implementations on Intel Architecture Processors"
-#
-# To find it, surf to http://www.intel.com/p/en_US/embedded
-# and search for that title.
-#
-########################################################################
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-.text
-
-# Virtual Registers
-# ARG1
-digest	= %rdi
-# ARG2
-msg	= %rsi
-# ARG3
-msglen	= %rdx
-T1	= %rcx
-T2	= %r8
-a_64	= %r9
-b_64	= %r10
-c_64	= %r11
-d_64	= %r12
-e_64	= %r13
-f_64	= %r14
-g_64	= %r15
-h_64	= %rbx
-tmp0	= %rax
-
-# Local variables (stack frame)
-
-# Message Schedule
-W_SIZE = 80*8
-# W[t] + K[t] | W[t+1] + K[t+1]
-WK_SIZE = 2*8
-
-frame_W = 0
-frame_WK = frame_W + W_SIZE
-frame_size = frame_WK + WK_SIZE
-
-# Useful QWORD "arrays" for simpler memory references
-# MSG, DIGEST, K_t, W_t are arrays
-# WK_2(t) points to 1 of 2 qwords at frame.WK depending on t being odd/even
-
-# Input message (arg1)
-#define MSG(i)    8*i(msg)
-
-# Output Digest (arg2)
-#define DIGEST(i) 8*i(digest)
-
-# SHA Constants (static mem)
-#define K_t(i)    8*i+K512(%rip)
-
-# Message Schedule (stack frame)
-#define W_t(i)    8*i+frame_W(%rsp)
-
-# W[t]+K[t] (stack frame)
-#define WK_2(i)   8*((i%2))+frame_WK(%rsp)
-
-.macro RotateState
-	# Rotate symbols a..h right
-	TMP   = h_64
-	h_64  = g_64
-	g_64  = f_64
-	f_64  = e_64
-	e_64  = d_64
-	d_64  = c_64
-	c_64  = b_64
-	b_64  = a_64
-	a_64  = TMP
-.endm
-
-.macro RORQ p1 p2
-	# shld is faster than ror on Sandybridge
-	shld	$(64-\p2), \p1, \p1
-.endm
-
-.macro SHA512_Round rnd
-	# Compute Round %%t
-	mov     f_64, T1          # T1 = f
-	mov     e_64, tmp0        # tmp = e
-	xor     g_64, T1          # T1 = f ^ g
-	RORQ    tmp0, 23   # 41    # tmp = e ror 23
-	and     e_64, T1          # T1 = (f ^ g) & e
-	xor     e_64, tmp0        # tmp = (e ror 23) ^ e
-	xor     g_64, T1          # T1 = ((f ^ g) & e) ^ g = CH(e,f,g)
-	idx = \rnd
-	add     WK_2(idx), T1     # W[t] + K[t] from message scheduler
-	RORQ    tmp0, 4   # 18    # tmp = ((e ror 23) ^ e) ror 4
-	xor     e_64, tmp0        # tmp = (((e ror 23) ^ e) ror 4) ^ e
-	mov     a_64, T2          # T2 = a
-	add     h_64, T1          # T1 = CH(e,f,g) + W[t] + K[t] + h
-	RORQ    tmp0, 14  # 14    # tmp = ((((e ror23)^e)ror4)^e)ror14 = S1(e)
-	add     tmp0, T1          # T1 = CH(e,f,g) + W[t] + K[t] + S1(e)
-	mov     a_64, tmp0        # tmp = a
-	xor     c_64, T2          # T2 = a ^ c
-	and     c_64, tmp0        # tmp = a & c
-	and     b_64, T2          # T2 = (a ^ c) & b
-	xor     tmp0, T2          # T2 = ((a ^ c) & b) ^ (a & c) = Maj(a,b,c)
-	mov     a_64, tmp0        # tmp = a
-	RORQ    tmp0, 5  # 39     # tmp = a ror 5
-	xor     a_64, tmp0        # tmp = (a ror 5) ^ a
-	add     T1, d_64          # e(next_state) = d + T1
-	RORQ    tmp0, 6  # 34     # tmp = ((a ror 5) ^ a) ror 6
-	xor     a_64, tmp0        # tmp = (((a ror 5) ^ a) ror 6) ^ a
-	lea     (T1, T2), h_64    # a(next_state) = T1 + Maj(a,b,c)
-	RORQ    tmp0, 28  # 28    # tmp = ((((a ror5)^a)ror6)^a)ror28 = S0(a)
-	add     tmp0, h_64        # a(next_state) = T1 + Maj(a,b,c) S0(a)
-	RotateState
-.endm
-
-.macro SHA512_2Sched_2Round_avx rnd
-	# Compute rounds t-2 and t-1
-	# Compute message schedule QWORDS t and t+1
-
-	#   Two rounds are computed based on the values for K[t-2]+W[t-2] and
-	# K[t-1]+W[t-1] which were previously stored at WK_2 by the message
-	# scheduler.
-	#   The two new schedule QWORDS are stored at [W_t(t)] and [W_t(t+1)].
-	# They are then added to their respective SHA512 constants at
-	# [K_t(t)] and [K_t(t+1)] and stored at dqword [WK_2(t)]
-	#   For brievity, the comments following vectored instructions only refer to
-	# the first of a pair of QWORDS.
-	# Eg. XMM4=W[t-2] really means XMM4={W[t-2]|W[t-1]}
-	#   The computation of the message schedule and the rounds are tightly
-	# stitched to take advantage of instruction-level parallelism.
-
-	idx = \rnd - 2
-	vmovdqa	W_t(idx), %xmm4		# XMM4 = W[t-2]
-	idx = \rnd - 15
-	vmovdqu	W_t(idx), %xmm5		# XMM5 = W[t-15]
-	mov	f_64, T1
-	vpsrlq	$61, %xmm4, %xmm0	# XMM0 = W[t-2]>>61
-	mov	e_64, tmp0
-	vpsrlq	$1, %xmm5, %xmm6	# XMM6 = W[t-15]>>1
-	xor	g_64, T1
-	RORQ	tmp0, 23 # 41
-	vpsrlq	$19, %xmm4, %xmm1	# XMM1 = W[t-2]>>19
-	and	e_64, T1
-	xor	e_64, tmp0
-	vpxor	%xmm1, %xmm0, %xmm0	# XMM0 = W[t-2]>>61 ^ W[t-2]>>19
-	xor	g_64, T1
-	idx = \rnd
-	add	WK_2(idx), T1#
-	vpsrlq	$8, %xmm5, %xmm7	# XMM7 = W[t-15]>>8
-	RORQ	tmp0, 4 # 18
-	vpsrlq	$6, %xmm4, %xmm2	# XMM2 = W[t-2]>>6
-	xor	e_64, tmp0
-	mov	a_64, T2
-	add	h_64, T1
-	vpxor	%xmm7, %xmm6, %xmm6	# XMM6 = W[t-15]>>1 ^ W[t-15]>>8
-	RORQ	tmp0, 14 # 14
-	add	tmp0, T1
-	vpsrlq	$7, %xmm5, %xmm8	# XMM8 = W[t-15]>>7
-	mov	a_64, tmp0
-	xor	c_64, T2
-	vpsllq	$(64-61), %xmm4, %xmm3  # XMM3 = W[t-2]<<3
-	and	c_64, tmp0
-	and	b_64, T2
-	vpxor	%xmm3, %xmm2, %xmm2	# XMM2 = W[t-2]>>6 ^ W[t-2]<<3
-	xor	tmp0, T2
-	mov	a_64, tmp0
-	vpsllq	$(64-1), %xmm5, %xmm9	# XMM9 = W[t-15]<<63
-	RORQ	tmp0, 5 # 39
-	vpxor	%xmm9, %xmm8, %xmm8	# XMM8 = W[t-15]>>7 ^ W[t-15]<<63
-	xor	a_64, tmp0
-	add	T1, d_64
-	RORQ	tmp0, 6 # 34
-	xor	a_64, tmp0
-	vpxor	%xmm8, %xmm6, %xmm6	# XMM6 = W[t-15]>>1 ^ W[t-15]>>8 ^
-					#  W[t-15]>>7 ^ W[t-15]<<63
-	lea	(T1, T2), h_64
-	RORQ	tmp0, 28 # 28
-	vpsllq	$(64-19), %xmm4, %xmm4  # XMM4 = W[t-2]<<25
-	add	tmp0, h_64
-	RotateState
-	vpxor	%xmm4, %xmm0, %xmm0     # XMM0 = W[t-2]>>61 ^ W[t-2]>>19 ^
-					#        W[t-2]<<25
-	mov	f_64, T1
-	vpxor	%xmm2, %xmm0, %xmm0     # XMM0 = s1(W[t-2])
-	mov	e_64, tmp0
-	xor	g_64, T1
-	idx = \rnd - 16
-	vpaddq	W_t(idx), %xmm0, %xmm0  # XMM0 = s1(W[t-2]) + W[t-16]
-	idx = \rnd - 7
-	vmovdqu	W_t(idx), %xmm1		# XMM1 = W[t-7]
-	RORQ	tmp0, 23 # 41
-	and	e_64, T1
-	xor	e_64, tmp0
-	xor	g_64, T1
-	vpsllq	$(64-8), %xmm5, %xmm5   # XMM5 = W[t-15]<<56
-	idx = \rnd + 1
-	add	WK_2(idx), T1
-	vpxor	%xmm5, %xmm6, %xmm6     # XMM6 = s0(W[t-15])
-	RORQ	tmp0, 4 # 18
-	vpaddq	%xmm6, %xmm0, %xmm0     # XMM0 = s1(W[t-2]) + W[t-16] + s0(W[t-15])
-	xor	e_64, tmp0
-	vpaddq	%xmm1, %xmm0, %xmm0     # XMM0 = W[t] = s1(W[t-2]) + W[t-7] +
-					#               s0(W[t-15]) + W[t-16]
-	mov	a_64, T2
-	add	h_64, T1
-	RORQ	tmp0, 14 # 14
-	add	tmp0, T1
-	idx = \rnd
-	vmovdqa	%xmm0, W_t(idx)		# Store W[t]
-	vpaddq	K_t(idx), %xmm0, %xmm0  # Compute W[t]+K[t]
-	vmovdqa	%xmm0, WK_2(idx)	# Store W[t]+K[t] for next rounds
-	mov	a_64, tmp0
-	xor	c_64, T2
-	and	c_64, tmp0
-	and	b_64, T2
-	xor	tmp0, T2
-	mov	a_64, tmp0
-	RORQ	tmp0, 5 # 39
-	xor	a_64, tmp0
-	add	T1, d_64
-	RORQ	tmp0, 6 # 34
-	xor	a_64, tmp0
-	lea	(T1, T2), h_64
-	RORQ	tmp0, 28 # 28
-	add	tmp0, h_64
-	RotateState
-.endm
-
-########################################################################
-# void sha512_transform_avx(sha512_state *state, const u8 *data, int blocks)
-# Purpose: Updates the SHA512 digest stored at "state" with the message
-# stored in "data".
-# The size of the message pointed to by "data" must be an integer multiple
-# of SHA512 message blocks.
-# "blocks" is the message length in SHA512 blocks
-########################################################################
-SYM_TYPED_FUNC_START(sha512_transform_avx)
-	test msglen, msglen
-	je .Lnowork
-
-	# Save GPRs
-	push	%rbx
-	push	%r12
-	push	%r13
-	push	%r14
-	push	%r15
-
-	# Allocate Stack Space
-	push	%rbp
-	mov	%rsp, %rbp
-	sub     $frame_size, %rsp
-	and	$~(0x20 - 1), %rsp
-
-.Lupdateblock:
-
-	# Load state variables
-	mov     DIGEST(0), a_64
-	mov     DIGEST(1), b_64
-	mov     DIGEST(2), c_64
-	mov     DIGEST(3), d_64
-	mov     DIGEST(4), e_64
-	mov     DIGEST(5), f_64
-	mov     DIGEST(6), g_64
-	mov     DIGEST(7), h_64
-
-	t = 0
-	.rept 80/2 + 1
-	# (80 rounds) / (2 rounds/iteration) + (1 iteration)
-	# +1 iteration because the scheduler leads hashing by 1 iteration
-		.if t < 2
-			# BSWAP 2 QWORDS
-			vmovdqa  XMM_QWORD_BSWAP(%rip), %xmm1
-			vmovdqu  MSG(t), %xmm0
-			vpshufb  %xmm1, %xmm0, %xmm0    # BSWAP
-			vmovdqa  %xmm0, W_t(t) # Store Scheduled Pair
-			vpaddq   K_t(t), %xmm0, %xmm0 # Compute W[t]+K[t]
-			vmovdqa  %xmm0, WK_2(t) # Store into WK for rounds
-		.elseif t < 16
-			# BSWAP 2 QWORDS# Compute 2 Rounds
-			vmovdqu  MSG(t), %xmm0
-			vpshufb  %xmm1, %xmm0, %xmm0    # BSWAP
-			SHA512_Round t-2    # Round t-2
-			vmovdqa  %xmm0, W_t(t) # Store Scheduled Pair
-			vpaddq   K_t(t), %xmm0, %xmm0 # Compute W[t]+K[t]
-			SHA512_Round t-1    # Round t-1
-			vmovdqa  %xmm0, WK_2(t)# Store W[t]+K[t] into WK
-		.elseif t < 79
-			# Schedule 2 QWORDS# Compute 2 Rounds
-			SHA512_2Sched_2Round_avx t
-		.else
-			# Compute 2 Rounds
-			SHA512_Round t-2
-			SHA512_Round t-1
-		.endif
-		t = t+2
-	.endr
-
-	# Update digest
-	add     a_64, DIGEST(0)
-	add     b_64, DIGEST(1)
-	add     c_64, DIGEST(2)
-	add     d_64, DIGEST(3)
-	add     e_64, DIGEST(4)
-	add     f_64, DIGEST(5)
-	add     g_64, DIGEST(6)
-	add     h_64, DIGEST(7)
-
-	# Advance to next message block
-	add     $16*8, msg
-	dec     msglen
-	jnz     .Lupdateblock
-
-	# Restore Stack Pointer
-	mov	%rbp, %rsp
-	pop	%rbp
-
-	# Restore GPRs
-	pop	%r15
-	pop	%r14
-	pop	%r13
-	pop	%r12
-	pop	%rbx
-
-.Lnowork:
-	RET
-SYM_FUNC_END(sha512_transform_avx)
-
-########################################################################
-### Binary Data
-
-.section	.rodata.cst16.XMM_QWORD_BSWAP, "aM", @progbits, 16
-.align 16
-# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
-XMM_QWORD_BSWAP:
-	.octa 0x08090a0b0c0d0e0f0001020304050607
-
-# Mergeable 640-byte rodata section. This allows linker to merge the table
-# with other, exactly the same 640-byte fragment of another rodata section
-# (if such section exists).
-.section	.rodata.cst640.K512, "aM", @progbits, 640
-.align 64
-# K[t] used in SHA512 hashing
-K512:
-	.quad 0x428a2f98d728ae22,0x7137449123ef65cd
-	.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
-	.quad 0x3956c25bf348b538,0x59f111f1b605d019
-	.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
-	.quad 0xd807aa98a3030242,0x12835b0145706fbe
-	.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
-	.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
-	.quad 0x9bdc06a725c71235,0xc19bf174cf692694
-	.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
-	.quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
-	.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
-	.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
-	.quad 0x983e5152ee66dfab,0xa831c66d2db43210
-	.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
-	.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
-	.quad 0x06ca6351e003826f,0x142929670a0e6e70
-	.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
-	.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
-	.quad 0x650a73548baf63de,0x766a0abb3c77b2a8
-	.quad 0x81c2c92e47edaee6,0x92722c851482353b
-	.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
-	.quad 0xc24b8b70d0f89791,0xc76c51a30654be30
-	.quad 0xd192e819d6ef5218,0xd69906245565a910
-	.quad 0xf40e35855771202a,0x106aa07032bbd1b8
-	.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
-	.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
-	.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
-	.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
-	.quad 0x748f82ee5defb2fc,0x78a5636f43172f60
-	.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
-	.quad 0x90befffa23631e28,0xa4506cebde82bde9
-	.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
-	.quad 0xca273eceea26619c,0xd186b8c721c0c207
-	.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
-	.quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
-	.quad 0x113f9804bef90dae,0x1b710b35131c471b
-	.quad 0x28db77f523047d84,0x32caab7b40c72493
-	.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
-	.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
-	.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
diff --git a/arch/x86/crypto/sha512-avx2-asm.S b/arch/x86/crypto/sha512-avx2-asm.S
deleted file mode 100644
index 24973f42c43f..000000000000
--- a/arch/x86/crypto/sha512-avx2-asm.S
+++ /dev/null
@@ -1,750 +0,0 @@
-########################################################################
-# Implement fast SHA-512 with AVX2 instructions. (x86_64)
-#
-# Copyright (C) 2013 Intel Corporation.
-#
-# Authors:
-#     James Guilford <james.guilford@intel.com>
-#     Kirk Yap <kirk.s.yap@intel.com>
-#     David Cote <david.m.cote@intel.com>
-#     Tim Chen <tim.c.chen@linux.intel.com>
-#
-# This software is available to you under a choice of one of two
-# licenses.  You may choose to be licensed under the terms of the GNU
-# General Public License (GPL) Version 2, available from the file
-# COPYING in the main directory of this source tree, or the
-# OpenIB.org BSD license below:
-#
-#     Redistribution and use in source and binary forms, with or
-#     without modification, are permitted provided that the following
-#     conditions are met:
-#
-#      - Redistributions of source code must retain the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer.
-#
-#      - Redistributions in binary form must reproduce the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer in the documentation and/or other materials
-#        provided with the distribution.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-#
-########################################################################
-#
-# This code is described in an Intel White-Paper:
-# "Fast SHA-512 Implementations on Intel Architecture Processors"
-#
-# To find it, surf to http://www.intel.com/p/en_US/embedded
-# and search for that title.
-#
-########################################################################
-# This code schedules 1 blocks at a time, with 4 lanes per block
-########################################################################
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-.text
-
-# Virtual Registers
-Y_0 = %ymm4
-Y_1 = %ymm5
-Y_2 = %ymm6
-Y_3 = %ymm7
-
-YTMP0 = %ymm0
-YTMP1 = %ymm1
-YTMP2 = %ymm2
-YTMP3 = %ymm3
-YTMP4 = %ymm8
-XFER  = YTMP0
-
-BYTE_FLIP_MASK  = %ymm9
-
-# 1st arg is %rdi, which is saved to the stack and accessed later via %r12
-CTX1        = %rdi
-CTX2        = %r12
-# 2nd arg
-INP         = %rsi
-# 3rd arg
-NUM_BLKS    = %rdx
-
-c           = %rcx
-d           = %r8
-e           = %rdx
-y3          = %rsi
-
-TBL   = %rdi # clobbers CTX1
-
-a     = %rax
-b     = %rbx
-
-f     = %r9
-g     = %r10
-h     = %r11
-old_h = %r11
-
-T1    = %r12 # clobbers CTX2
-y0    = %r13
-y1    = %r14
-y2    = %r15
-
-# Local variables (stack frame)
-XFER_SIZE = 4*8
-SRND_SIZE = 1*8
-INP_SIZE = 1*8
-INPEND_SIZE = 1*8
-CTX_SIZE = 1*8
-
-frame_XFER = 0
-frame_SRND = frame_XFER + XFER_SIZE
-frame_INP = frame_SRND + SRND_SIZE
-frame_INPEND = frame_INP + INP_SIZE
-frame_CTX = frame_INPEND + INPEND_SIZE
-frame_size = frame_CTX + CTX_SIZE
-
-## assume buffers not aligned
-#define	VMOVDQ vmovdqu
-
-# addm [mem], reg
-# Add reg to mem using reg-mem add and store
-.macro addm p1 p2
-	add	\p1, \p2
-	mov	\p2, \p1
-.endm
-
-
-# COPY_YMM_AND_BSWAP ymm, [mem], byte_flip_mask
-# Load ymm with mem and byte swap each dword
-.macro COPY_YMM_AND_BSWAP p1 p2 p3
-	VMOVDQ \p2, \p1
-	vpshufb \p3, \p1, \p1
-.endm
-# rotate_Ys
-# Rotate values of symbols Y0...Y3
-.macro rotate_Ys
-	Y_ = Y_0
-	Y_0 = Y_1
-	Y_1 = Y_2
-	Y_2 = Y_3
-	Y_3 = Y_
-.endm
-
-# RotateState
-.macro RotateState
-	# Rotate symbols a..h right
-	old_h  = h
-	TMP_   = h
-	h      = g
-	g      = f
-	f      = e
-	e      = d
-	d      = c
-	c      = b
-	b      = a
-	a      = TMP_
-.endm
-
-# macro MY_VPALIGNR	YDST, YSRC1, YSRC2, RVAL
-# YDST = {YSRC1, YSRC2} >> RVAL*8
-.macro MY_VPALIGNR YDST YSRC1 YSRC2 RVAL
-	vperm2f128      $0x3, \YSRC2, \YSRC1, \YDST     # YDST = {YS1_LO, YS2_HI}
-	vpalignr        $\RVAL, \YSRC2, \YDST, \YDST    # YDST = {YDS1, YS2} >> RVAL*8
-.endm
-
-.macro FOUR_ROUNDS_AND_SCHED
-################################### RND N + 0 #########################################
-
-	# Extract w[t-7]
-	MY_VPALIGNR	YTMP0, Y_3, Y_2, 8		# YTMP0 = W[-7]
-	# Calculate w[t-16] + w[t-7]
-	vpaddq		Y_0, YTMP0, YTMP0		# YTMP0 = W[-7] + W[-16]
-	# Extract w[t-15]
-	MY_VPALIGNR	YTMP1, Y_1, Y_0, 8		# YTMP1 = W[-15]
-
-	# Calculate sigma0
-
-	# Calculate w[t-15] ror 1
-	vpsrlq		$1, YTMP1, YTMP2
-	vpsllq		$(64-1), YTMP1, YTMP3
-	vpor		YTMP2, YTMP3, YTMP3		# YTMP3 = W[-15] ror 1
-	# Calculate w[t-15] shr 7
-	vpsrlq		$7, YTMP1, YTMP4		# YTMP4 = W[-15] >> 7
-
-	mov	a, y3		# y3 = a                                # MAJA
-	rorx	$41, e, y0	# y0 = e >> 41				# S1A
-	rorx	$18, e, y1	# y1 = e >> 18				# S1B
-	add	frame_XFER(%rsp),h		# h = k + w + h         # --
-	or	c, y3		# y3 = a|c                              # MAJA
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$34, a, T1	# T1 = a >> 34				# S0B
-
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18)		# S1
-	xor	g, y2		# y2 = f^g                              # CH
-	rorx	$14, e, y1	# y1 = (e >> 14)			# S1
-
-	and	e, y2		# y2 = (f^g)&e                          # CH
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18) ^ (e>>14)	# S1
-	rorx	$39, a, y1	# y1 = a >> 39				# S0A
-	add	h, d		# d = k + w + h + d                     # --
-
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34)		# S0
-	rorx	$28, a, T1	# T1 = (a >> 28)			# S0
-
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34) ^ (a>>28)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	c, T1		# T1 = a&c                              # MAJB
-
-	add	y0, y2		# y2 = S1 + CH                          # --
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-
-	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-	add	y3, h		# h = t1 + S0 + MAJ                     # --
-
-	RotateState
-
-################################### RND N + 1 #########################################
-
-	# Calculate w[t-15] ror 8
-	vpsrlq		$8, YTMP1, YTMP2
-	vpsllq		$(64-8), YTMP1, YTMP1
-	vpor		YTMP2, YTMP1, YTMP1		# YTMP1 = W[-15] ror 8
-	# XOR the three components
-	vpxor		YTMP4, YTMP3, YTMP3		# YTMP3 = W[-15] ror 1 ^ W[-15] >> 7
-	vpxor		YTMP1, YTMP3, YTMP1		# YTMP1 = s0
-
-
-	# Add three components, w[t-16], w[t-7] and sigma0
-	vpaddq		YTMP1, YTMP0, YTMP0		# YTMP0 = W[-16] + W[-7] + s0
-	# Move to appropriate lanes for calculating w[16] and w[17]
-	vperm2f128	$0x0, YTMP0, YTMP0, Y_0		# Y_0 = W[-16] + W[-7] + s0 {BABA}
-	# Move to appropriate lanes for calculating w[18] and w[19]
-	vpand		MASK_YMM_LO(%rip), YTMP0, YTMP0	# YTMP0 = W[-16] + W[-7] + s0 {DC00}
-
-	# Calculate w[16] and w[17] in both 128 bit lanes
-
-	# Calculate sigma1 for w[16] and w[17] on both 128 bit lanes
-	vperm2f128	$0x11, Y_3, Y_3, YTMP2		# YTMP2 = W[-2] {BABA}
-	vpsrlq		$6, YTMP2, YTMP4		# YTMP4 = W[-2] >> 6 {BABA}
-
-
-	mov	a, y3		# y3 = a                                # MAJA
-	rorx	$41, e, y0	# y0 = e >> 41				# S1A
-	rorx	$18, e, y1	# y1 = e >> 18				# S1B
-	add	1*8+frame_XFER(%rsp), h		# h = k + w + h         # --
-	or	c, y3		# y3 = a|c                              # MAJA
-
-
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$34, a, T1	# T1 = a >> 34				# S0B
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18)		# S1
-	xor	g, y2		# y2 = f^g                              # CH
-
-
-	rorx	$14, e, y1	# y1 = (e >> 14)			# S1
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18) ^ (e>>14)	# S1
-	rorx	$39, a, y1	# y1 = a >> 39				# S0A
-	and	e, y2		# y2 = (f^g)&e                          # CH
-	add	h, d		# d = k + w + h + d                     # --
-
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34)		# S0
-
-	rorx	$28, a, T1	# T1 = (a >> 28)			# S0
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34) ^ (a>>28)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	c, T1		# T1 = a&c                              # MAJB
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-	add	y3, h		# h = t1 + S0 + MAJ                     # --
-
-	RotateState
-
-
-################################### RND N + 2 #########################################
-
-	vpsrlq		$19, YTMP2, YTMP3		# YTMP3 = W[-2] >> 19 {BABA}
-	vpsllq		$(64-19), YTMP2, YTMP1		# YTMP1 = W[-2] << 19 {BABA}
-	vpor		YTMP1, YTMP3, YTMP3		# YTMP3 = W[-2] ror 19 {BABA}
-	vpxor		YTMP3, YTMP4, YTMP4		# YTMP4 = W[-2] ror 19 ^ W[-2] >> 6 {BABA}
-	vpsrlq		$61, YTMP2, YTMP3		# YTMP3 = W[-2] >> 61 {BABA}
-	vpsllq		$(64-61), YTMP2, YTMP1		# YTMP1 = W[-2] << 61 {BABA}
-	vpor		YTMP1, YTMP3, YTMP3		# YTMP3 = W[-2] ror 61 {BABA}
-	vpxor		YTMP3, YTMP4, YTMP4		# YTMP4 = s1 = (W[-2] ror 19) ^
-							#  (W[-2] ror 61) ^ (W[-2] >> 6) {BABA}
-
-	# Add sigma1 to the other compunents to get w[16] and w[17]
-	vpaddq		YTMP4, Y_0, Y_0			# Y_0 = {W[1], W[0], W[1], W[0]}
-
-	# Calculate sigma1 for w[18] and w[19] for upper 128 bit lane
-	vpsrlq		$6, Y_0, YTMP4			# YTMP4 = W[-2] >> 6 {DC--}
-
-	mov	a, y3		# y3 = a                                # MAJA
-	rorx	$41, e, y0	# y0 = e >> 41				# S1A
-	add	2*8+frame_XFER(%rsp), h		# h = k + w + h         # --
-
-	rorx	$18, e, y1	# y1 = e >> 18				# S1B
-	or	c, y3		# y3 = a|c                              # MAJA
-	mov	f, y2		# y2 = f                                # CH
-	xor	g, y2		# y2 = f^g                              # CH
-
-	rorx	$34, a, T1	# T1 = a >> 34				# S0B
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18)		# S1
-	and	e, y2		# y2 = (f^g)&e                          # CH
-
-	rorx	$14, e, y1	# y1 = (e >> 14)			# S1
-	add	h, d		# d = k + w + h + d                     # --
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18) ^ (e>>14)	# S1
-	rorx	$39, a, y1	# y1 = a >> 39				# S0A
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34)		# S0
-	rorx	$28, a, T1	# T1 = (a >> 28)			# S0
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34) ^ (a>>28)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	c, T1		# T1 = a&c                              # MAJB
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-
-	add	y3, h		# h = t1 + S0 + MAJ                     # --
-
-	RotateState
-
-################################### RND N + 3 #########################################
-
-	vpsrlq		$19, Y_0, YTMP3			# YTMP3 = W[-2] >> 19 {DC--}
-	vpsllq		$(64-19), Y_0, YTMP1		# YTMP1 = W[-2] << 19 {DC--}
-	vpor		YTMP1, YTMP3, YTMP3		# YTMP3 = W[-2] ror 19 {DC--}
-	vpxor		YTMP3, YTMP4, YTMP4		# YTMP4 = W[-2] ror 19 ^ W[-2] >> 6 {DC--}
-	vpsrlq		$61, Y_0, YTMP3			# YTMP3 = W[-2] >> 61 {DC--}
-	vpsllq		$(64-61), Y_0, YTMP1		# YTMP1 = W[-2] << 61 {DC--}
-	vpor		YTMP1, YTMP3, YTMP3		# YTMP3 = W[-2] ror 61 {DC--}
-	vpxor		YTMP3, YTMP4, YTMP4		# YTMP4 = s1 = (W[-2] ror 19) ^
-							#  (W[-2] ror 61) ^ (W[-2] >> 6) {DC--}
-
-	# Add the sigma0 + w[t-7] + w[t-16] for w[18] and w[19]
-	# to newly calculated sigma1 to get w[18] and w[19]
-	vpaddq		YTMP4, YTMP0, YTMP2		# YTMP2 = {W[3], W[2], --, --}
-
-	# Form w[19, w[18], w17], w[16]
-	vpblendd		$0xF0, YTMP2, Y_0, Y_0		# Y_0 = {W[3], W[2], W[1], W[0]}
-
-	mov	a, y3		# y3 = a                                # MAJA
-	rorx	$41, e, y0	# y0 = e >> 41				# S1A
-	rorx	$18, e, y1	# y1 = e >> 18				# S1B
-	add	3*8+frame_XFER(%rsp), h		# h = k + w + h         # --
-	or	c, y3		# y3 = a|c                              # MAJA
-
-
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$34, a, T1	# T1 = a >> 34				# S0B
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18)		# S1
-	xor	g, y2		# y2 = f^g                              # CH
-
-
-	rorx	$14, e, y1	# y1 = (e >> 14)			# S1
-	and	e, y2		# y2 = (f^g)&e                          # CH
-	add	h, d		# d = k + w + h + d                     # --
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18) ^ (e>>14)	# S1
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-
-	rorx	$39, a, y1	# y1 = a >> 39				# S0A
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34)		# S0
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-
-	rorx	$28, a, T1	# T1 = (a >> 28)			# S0
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34) ^ (a>>28)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	c, T1		# T1 = a&c                              # MAJB
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-
-	add	y1, h		# h = k + w + h + S0                    # --
-	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-	add	y3, h		# h = t1 + S0 + MAJ                     # --
-
-	RotateState
-
-	rotate_Ys
-.endm
-
-.macro DO_4ROUNDS
-
-################################### RND N + 0 #########################################
-
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$41, e, y0	# y0 = e >> 41				# S1A
-	rorx	$18, e, y1	# y1 = e >> 18				# S1B
-	xor	g, y2		# y2 = f^g                              # CH
-
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18)		# S1
-	rorx	$14, e, y1	# y1 = (e >> 14)			# S1
-	and	e, y2		# y2 = (f^g)&e                          # CH
-
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18) ^ (e>>14)	# S1
-	rorx	$34, a, T1	# T1 = a >> 34				# S0B
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-	rorx	$39, a, y1	# y1 = a >> 39				# S0A
-	mov	a, y3		# y3 = a                                # MAJA
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34)		# S0
-	rorx	$28, a, T1	# T1 = (a >> 28)			# S0
-	add	frame_XFER(%rsp), h		# h = k + w + h         # --
-	or	c, y3		# y3 = a|c                              # MAJA
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34) ^ (a>>28)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-	and	c, T1		# T1 = a&c                              # MAJB
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-	add	h, d		# d = k + w + h + d                     # --
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-
-	RotateState
-
-################################### RND N + 1 #########################################
-
-	add	y2, old_h	# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$41, e, y0	# y0 = e >> 41				# S1A
-	rorx	$18, e, y1	# y1 = e >> 18				# S1B
-	xor	g, y2		# y2 = f^g                              # CH
-
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18)		# S1
-	rorx	$14, e, y1	# y1 = (e >> 14)			# S1
-	and	e, y2		# y2 = (f^g)&e                          # CH
-	add	y3, old_h	# h = t1 + S0 + MAJ                     # --
-
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18) ^ (e>>14)	# S1
-	rorx	$34, a, T1	# T1 = a >> 34				# S0B
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-	rorx	$39, a, y1	# y1 = a >> 39				# S0A
-	mov	a, y3		# y3 = a                                # MAJA
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34)		# S0
-	rorx	$28, a, T1	# T1 = (a >> 28)			# S0
-	add	8*1+frame_XFER(%rsp), h		# h = k + w + h         # --
-	or	c, y3		# y3 = a|c                              # MAJA
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34) ^ (a>>28)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-	and	c, T1		# T1 = a&c                              # MAJB
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-	add	h, d		# d = k + w + h + d                     # --
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-
-	RotateState
-
-################################### RND N + 2 #########################################
-
-	add	y2, old_h	# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$41, e, y0	# y0 = e >> 41				# S1A
-	rorx	$18, e, y1	# y1 = e >> 18				# S1B
-	xor	g, y2		# y2 = f^g                              # CH
-
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18)		# S1
-	rorx	$14, e, y1	# y1 = (e >> 14)			# S1
-	and	e, y2		# y2 = (f^g)&e                          # CH
-	add	y3, old_h	# h = t1 + S0 + MAJ                     # --
-
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18) ^ (e>>14)	# S1
-	rorx	$34, a, T1	# T1 = a >> 34				# S0B
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-	rorx	$39, a, y1	# y1 = a >> 39				# S0A
-	mov	a, y3		# y3 = a                                # MAJA
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34)		# S0
-	rorx	$28, a, T1	# T1 = (a >> 28)			# S0
-	add	8*2+frame_XFER(%rsp), h		# h = k + w + h         # --
-	or	c, y3		# y3 = a|c                              # MAJA
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34) ^ (a>>28)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-	and	c, T1		# T1 = a&c                              # MAJB
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-	add	h, d		# d = k + w + h + d                     # --
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-
-	RotateState
-
-################################### RND N + 3 #########################################
-
-	add	y2, old_h	# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-	mov	f, y2		# y2 = f                                # CH
-	rorx	$41, e, y0	# y0 = e >> 41				# S1A
-	rorx	$18, e, y1	# y1 = e >> 18				# S1B
-	xor	g, y2		# y2 = f^g                              # CH
-
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18)		# S1
-	rorx	$14, e, y1	# y1 = (e >> 14)			# S1
-	and	e, y2		# y2 = (f^g)&e                          # CH
-	add	y3, old_h	# h = t1 + S0 + MAJ                     # --
-
-	xor	y1, y0		# y0 = (e>>41) ^ (e>>18) ^ (e>>14)	# S1
-	rorx	$34, a, T1	# T1 = a >> 34				# S0B
-	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
-	rorx	$39, a, y1	# y1 = a >> 39				# S0A
-	mov	a, y3		# y3 = a                                # MAJA
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34)		# S0
-	rorx	$28, a, T1	# T1 = (a >> 28)			# S0
-	add	8*3+frame_XFER(%rsp), h		# h = k + w + h         # --
-	or	c, y3		# y3 = a|c                              # MAJA
-
-	xor	T1, y1		# y1 = (a>>39) ^ (a>>34) ^ (a>>28)	# S0
-	mov	a, T1		# T1 = a                                # MAJB
-	and	b, y3		# y3 = (a|c)&b                          # MAJA
-	and	c, T1		# T1 = a&c                              # MAJB
-	add	y0, y2		# y2 = S1 + CH                          # --
-
-
-	add	h, d		# d = k + w + h + d                     # --
-	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
-	add	y1, h		# h = k + w + h + S0                    # --
-
-	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
-
-	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
-
-	add	y3, h		# h = t1 + S0 + MAJ                     # --
-
-	RotateState
-
-.endm
-
-########################################################################
-# void sha512_transform_rorx(sha512_state *state, const u8 *data, int blocks)
-# Purpose: Updates the SHA512 digest stored at "state" with the message
-# stored in "data".
-# The size of the message pointed to by "data" must be an integer multiple
-# of SHA512 message blocks.
-# "blocks" is the message length in SHA512 blocks
-########################################################################
-SYM_TYPED_FUNC_START(sha512_transform_rorx)
-	# Save GPRs
-	push	%rbx
-	push	%r12
-	push	%r13
-	push	%r14
-	push	%r15
-
-	# Allocate Stack Space
-	push	%rbp
-	mov	%rsp, %rbp
-	sub	$frame_size, %rsp
-	and	$~(0x20 - 1), %rsp
-
-	shl	$7, NUM_BLKS	# convert to bytes
-	jz	.Ldone_hash
-	add	INP, NUM_BLKS	# pointer to end of data
-	mov	NUM_BLKS, frame_INPEND(%rsp)
-
-	## load initial digest
-	mov	8*0(CTX1), a
-	mov	8*1(CTX1), b
-	mov	8*2(CTX1), c
-	mov	8*3(CTX1), d
-	mov	8*4(CTX1), e
-	mov	8*5(CTX1), f
-	mov	8*6(CTX1), g
-	mov	8*7(CTX1), h
-
-	# save %rdi (CTX) before it gets clobbered
-	mov	%rdi, frame_CTX(%rsp)
-
-	vmovdqa	PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
-
-.Lloop0:
-	lea	K512(%rip), TBL
-
-	## byte swap first 16 dwords
-	COPY_YMM_AND_BSWAP	Y_0, (INP), BYTE_FLIP_MASK
-	COPY_YMM_AND_BSWAP	Y_1, 1*32(INP), BYTE_FLIP_MASK
-	COPY_YMM_AND_BSWAP	Y_2, 2*32(INP), BYTE_FLIP_MASK
-	COPY_YMM_AND_BSWAP	Y_3, 3*32(INP), BYTE_FLIP_MASK
-
-	mov	INP, frame_INP(%rsp)
-
-	## schedule 64 input dwords, by doing 12 rounds of 4 each
-	movq	$4, frame_SRND(%rsp)
-
-.align 16
-.Lloop1:
-	vpaddq	(TBL), Y_0, XFER
-	vmovdqa XFER, frame_XFER(%rsp)
-	FOUR_ROUNDS_AND_SCHED
-
-	vpaddq	1*32(TBL), Y_0, XFER
-	vmovdqa XFER, frame_XFER(%rsp)
-	FOUR_ROUNDS_AND_SCHED
-
-	vpaddq	2*32(TBL), Y_0, XFER
-	vmovdqa XFER, frame_XFER(%rsp)
-	FOUR_ROUNDS_AND_SCHED
-
-	vpaddq	3*32(TBL), Y_0, XFER
-	vmovdqa XFER, frame_XFER(%rsp)
-	add	$(4*32), TBL
-	FOUR_ROUNDS_AND_SCHED
-
-	subq	$1, frame_SRND(%rsp)
-	jne	.Lloop1
-
-	movq	$2, frame_SRND(%rsp)
-.Lloop2:
-	vpaddq	(TBL), Y_0, XFER
-	vmovdqa XFER, frame_XFER(%rsp)
-	DO_4ROUNDS
-	vpaddq	1*32(TBL), Y_1, XFER
-	vmovdqa XFER, frame_XFER(%rsp)
-	add	$(2*32), TBL
-	DO_4ROUNDS
-
-	vmovdqa	Y_2, Y_0
-	vmovdqa	Y_3, Y_1
-
-	subq	$1, frame_SRND(%rsp)
-	jne	.Lloop2
-
-	mov	frame_CTX(%rsp), CTX2
-	addm	8*0(CTX2), a
-	addm	8*1(CTX2), b
-	addm	8*2(CTX2), c
-	addm	8*3(CTX2), d
-	addm	8*4(CTX2), e
-	addm	8*5(CTX2), f
-	addm	8*6(CTX2), g
-	addm	8*7(CTX2), h
-
-	mov	frame_INP(%rsp), INP
-	add	$128, INP
-	cmp	frame_INPEND(%rsp), INP
-	jne	.Lloop0
-
-.Ldone_hash:
-
-	# Restore Stack Pointer
-	mov	%rbp, %rsp
-	pop	%rbp
-
-	# Restore GPRs
-	pop	%r15
-	pop	%r14
-	pop	%r13
-	pop	%r12
-	pop	%rbx
-
-	vzeroupper
-	RET
-SYM_FUNC_END(sha512_transform_rorx)
-
-########################################################################
-### Binary Data
-
-
-# Mergeable 640-byte rodata section. This allows linker to merge the table
-# with other, exactly the same 640-byte fragment of another rodata section
-# (if such section exists).
-.section	.rodata.cst640.K512, "aM", @progbits, 640
-.align 64
-# K[t] used in SHA512 hashing
-K512:
-	.quad	0x428a2f98d728ae22,0x7137449123ef65cd
-	.quad	0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
-	.quad	0x3956c25bf348b538,0x59f111f1b605d019
-	.quad	0x923f82a4af194f9b,0xab1c5ed5da6d8118
-	.quad	0xd807aa98a3030242,0x12835b0145706fbe
-	.quad	0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
-	.quad	0x72be5d74f27b896f,0x80deb1fe3b1696b1
-	.quad	0x9bdc06a725c71235,0xc19bf174cf692694
-	.quad	0xe49b69c19ef14ad2,0xefbe4786384f25e3
-	.quad	0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
-	.quad	0x2de92c6f592b0275,0x4a7484aa6ea6e483
-	.quad	0x5cb0a9dcbd41fbd4,0x76f988da831153b5
-	.quad	0x983e5152ee66dfab,0xa831c66d2db43210
-	.quad	0xb00327c898fb213f,0xbf597fc7beef0ee4
-	.quad	0xc6e00bf33da88fc2,0xd5a79147930aa725
-	.quad	0x06ca6351e003826f,0x142929670a0e6e70
-	.quad	0x27b70a8546d22ffc,0x2e1b21385c26c926
-	.quad	0x4d2c6dfc5ac42aed,0x53380d139d95b3df
-	.quad	0x650a73548baf63de,0x766a0abb3c77b2a8
-	.quad	0x81c2c92e47edaee6,0x92722c851482353b
-	.quad	0xa2bfe8a14cf10364,0xa81a664bbc423001
-	.quad	0xc24b8b70d0f89791,0xc76c51a30654be30
-	.quad	0xd192e819d6ef5218,0xd69906245565a910
-	.quad	0xf40e35855771202a,0x106aa07032bbd1b8
-	.quad	0x19a4c116b8d2d0c8,0x1e376c085141ab53
-	.quad	0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
-	.quad	0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
-	.quad	0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
-	.quad	0x748f82ee5defb2fc,0x78a5636f43172f60
-	.quad	0x84c87814a1f0ab72,0x8cc702081a6439ec
-	.quad	0x90befffa23631e28,0xa4506cebde82bde9
-	.quad	0xbef9a3f7b2c67915,0xc67178f2e372532b
-	.quad	0xca273eceea26619c,0xd186b8c721c0c207
-	.quad	0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
-	.quad	0x06f067aa72176fba,0x0a637dc5a2c898a6
-	.quad	0x113f9804bef90dae,0x1b710b35131c471b
-	.quad	0x28db77f523047d84,0x32caab7b40c72493
-	.quad	0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
-	.quad	0x4cc5d4becb3e42b6,0x597f299cfc657e2a
-	.quad	0x5fcb6fab3ad6faec,0x6c44198c4a475817
-
-.section	.rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
-.align 32
-# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
-PSHUFFLE_BYTE_FLIP_MASK:
-	.octa 0x08090a0b0c0d0e0f0001020304050607
-	.octa 0x18191a1b1c1d1e1f1011121314151617
-
-.section	.rodata.cst32.MASK_YMM_LO, "aM", @progbits, 32
-.align 32
-MASK_YMM_LO:
-	.octa 0x00000000000000000000000000000000
-	.octa 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
diff --git a/arch/x86/crypto/sha512-ssse3-asm.S b/arch/x86/crypto/sha512-ssse3-asm.S
deleted file mode 100644
index 30a2c4777f9d..000000000000
--- a/arch/x86/crypto/sha512-ssse3-asm.S
+++ /dev/null
@@ -1,425 +0,0 @@
-########################################################################
-# Implement fast SHA-512 with SSSE3 instructions. (x86_64)
-#
-# Copyright (C) 2013 Intel Corporation.
-#
-# Authors:
-#     James Guilford <james.guilford@intel.com>
-#     Kirk Yap <kirk.s.yap@intel.com>
-#     David Cote <david.m.cote@intel.com>
-#     Tim Chen <tim.c.chen@linux.intel.com>
-#
-# This software is available to you under a choice of one of two
-# licenses.  You may choose to be licensed under the terms of the GNU
-# General Public License (GPL) Version 2, available from the file
-# COPYING in the main directory of this source tree, or the
-# OpenIB.org BSD license below:
-#
-#     Redistribution and use in source and binary forms, with or
-#     without modification, are permitted provided that the following
-#     conditions are met:
-#
-#      - Redistributions of source code must retain the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer.
-#
-#      - Redistributions in binary form must reproduce the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer in the documentation and/or other materials
-#        provided with the distribution.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-#
-########################################################################
-#
-# This code is described in an Intel White-Paper:
-# "Fast SHA-512 Implementations on Intel Architecture Processors"
-#
-# To find it, surf to http://www.intel.com/p/en_US/embedded
-# and search for that title.
-#
-########################################################################
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-.text
-
-# Virtual Registers
-# ARG1
-digest =	%rdi
-# ARG2
-msg =		%rsi
-# ARG3
-msglen =	%rdx
-T1 =		%rcx
-T2 =		%r8
-a_64 =		%r9
-b_64 =		%r10
-c_64 =		%r11
-d_64 =		%r12
-e_64 =		%r13
-f_64 =		%r14
-g_64 =		%r15
-h_64 =		%rbx
-tmp0 =		%rax
-
-# Local variables (stack frame)
-
-W_SIZE = 80*8
-WK_SIZE = 2*8
-
-frame_W = 0
-frame_WK = frame_W + W_SIZE
-frame_size = frame_WK + WK_SIZE
-
-# Useful QWORD "arrays" for simpler memory references
-# MSG, DIGEST, K_t, W_t are arrays
-# WK_2(t) points to 1 of 2 qwords at frame.WK depending on t being odd/even
-
-# Input message (arg1)
-#define MSG(i)    8*i(msg)
-
-# Output Digest (arg2)
-#define DIGEST(i) 8*i(digest)
-
-# SHA Constants (static mem)
-#define K_t(i)    8*i+K512(%rip)
-
-# Message Schedule (stack frame)
-#define W_t(i)    8*i+frame_W(%rsp)
-
-# W[t]+K[t] (stack frame)
-#define WK_2(i)   8*((i%2))+frame_WK(%rsp)
-
-.macro RotateState
-	# Rotate symbols a..h right
-	TMP   = h_64
-	h_64  = g_64
-	g_64  = f_64
-	f_64  = e_64
-	e_64  = d_64
-	d_64  = c_64
-	c_64  = b_64
-	b_64  = a_64
-	a_64  = TMP
-.endm
-
-.macro SHA512_Round rnd
-
-	# Compute Round %%t
-	mov	f_64, T1          # T1 = f
-	mov	e_64, tmp0        # tmp = e
-	xor	g_64, T1          # T1 = f ^ g
-	ror	$23, tmp0 # 41    # tmp = e ror 23
-	and	e_64, T1          # T1 = (f ^ g) & e
-	xor	e_64, tmp0        # tmp = (e ror 23) ^ e
-	xor	g_64, T1          # T1 = ((f ^ g) & e) ^ g = CH(e,f,g)
-	idx = \rnd
-	add	WK_2(idx), T1     # W[t] + K[t] from message scheduler
-	ror	$4, tmp0  # 18    # tmp = ((e ror 23) ^ e) ror 4
-	xor	e_64, tmp0        # tmp = (((e ror 23) ^ e) ror 4) ^ e
-	mov	a_64, T2          # T2 = a
-	add	h_64, T1          # T1 = CH(e,f,g) + W[t] + K[t] + h
-	ror	$14, tmp0 # 14    # tmp = ((((e ror23)^e)ror4)^e)ror14 = S1(e)
-	add	tmp0, T1          # T1 = CH(e,f,g) + W[t] + K[t] + S1(e)
-	mov	a_64, tmp0        # tmp = a
-	xor	c_64, T2          # T2 = a ^ c
-	and	c_64, tmp0        # tmp = a & c
-	and	b_64, T2          # T2 = (a ^ c) & b
-	xor	tmp0, T2          # T2 = ((a ^ c) & b) ^ (a & c) = Maj(a,b,c)
-	mov	a_64, tmp0        # tmp = a
-	ror	$5, tmp0 # 39     # tmp = a ror 5
-	xor	a_64, tmp0        # tmp = (a ror 5) ^ a
-	add	T1, d_64          # e(next_state) = d + T1
-	ror	$6, tmp0 # 34     # tmp = ((a ror 5) ^ a) ror 6
-	xor	a_64, tmp0        # tmp = (((a ror 5) ^ a) ror 6) ^ a
-	lea	(T1, T2), h_64    # a(next_state) = T1 + Maj(a,b,c)
-	ror	$28, tmp0 # 28    # tmp = ((((a ror5)^a)ror6)^a)ror28 = S0(a)
-	add	tmp0, h_64        # a(next_state) = T1 + Maj(a,b,c) S0(a)
-	RotateState
-.endm
-
-.macro SHA512_2Sched_2Round_sse rnd
-
-	# Compute rounds t-2 and t-1
-	# Compute message schedule QWORDS t and t+1
-
-	#   Two rounds are computed based on the values for K[t-2]+W[t-2] and
-	# K[t-1]+W[t-1] which were previously stored at WK_2 by the message
-	# scheduler.
-	#   The two new schedule QWORDS are stored at [W_t(%%t)] and [W_t(%%t+1)].
-	# They are then added to their respective SHA512 constants at
-	# [K_t(%%t)] and [K_t(%%t+1)] and stored at dqword [WK_2(%%t)]
-	#   For brievity, the comments following vectored instructions only refer to
-	# the first of a pair of QWORDS.
-	# Eg. XMM2=W[t-2] really means XMM2={W[t-2]|W[t-1]}
-	#   The computation of the message schedule and the rounds are tightly
-	# stitched to take advantage of instruction-level parallelism.
-	# For clarity, integer instructions (for the rounds calculation) are indented
-	# by one tab. Vectored instructions (for the message scheduler) are indented
-	# by two tabs.
-
-	mov	f_64, T1
-	idx = \rnd -2
-	movdqa	W_t(idx), %xmm2		    # XMM2 = W[t-2]
-	xor	g_64, T1
-	and	e_64, T1
-	movdqa	%xmm2, %xmm0	            # XMM0 = W[t-2]
-	xor	g_64, T1
-	idx = \rnd
-	add	WK_2(idx), T1
-	idx = \rnd - 15
-	movdqu	W_t(idx), %xmm5		    # XMM5 = W[t-15]
-	mov	e_64, tmp0
-	ror	$23, tmp0 # 41
-	movdqa	%xmm5, %xmm3	            # XMM3 = W[t-15]
-	xor	e_64, tmp0
-	ror	$4, tmp0 # 18
-	psrlq	$61-19, %xmm0		    # XMM0 = W[t-2] >> 42
-	xor	e_64, tmp0
-	ror	$14, tmp0 # 14
-	psrlq	$(8-7), %xmm3		    # XMM3 = W[t-15] >> 1
-	add	tmp0, T1
-	add	h_64, T1
-	pxor	%xmm2, %xmm0                # XMM0 = (W[t-2] >> 42) ^ W[t-2]
-	mov	a_64, T2
-	xor	c_64, T2
-	pxor	%xmm5, %xmm3                # XMM3 = (W[t-15] >> 1) ^ W[t-15]
-	and	b_64, T2
-	mov	a_64, tmp0
-	psrlq	$(19-6), %xmm0		    # XMM0 = ((W[t-2]>>42)^W[t-2])>>13
-	and	c_64, tmp0
-	xor	tmp0, T2
-	psrlq	$(7-1), %xmm3		    # XMM3 = ((W[t-15]>>1)^W[t-15])>>6
-	mov	a_64, tmp0
-	ror	$5, tmp0 # 39
-	pxor	%xmm2, %xmm0	            # XMM0 = (((W[t-2]>>42)^W[t-2])>>13)^W[t-2]
-	xor	a_64, tmp0
-	ror	$6, tmp0 # 34
-	pxor	%xmm5, %xmm3                # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]
-	xor	a_64, tmp0
-	ror	$28, tmp0 # 28
-	psrlq	$6, %xmm0                   # XMM0 = ((((W[t-2]>>42)^W[t-2])>>13)^W[t-2])>>6
-	add	tmp0, T2
-	add	T1, d_64
-	psrlq	$1, %xmm3                   # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]>>1
-	lea	(T1, T2), h_64
-	RotateState
-	movdqa	%xmm2, %xmm1	            # XMM1 = W[t-2]
-	mov	f_64, T1
-	xor	g_64, T1
-	movdqa	%xmm5, %xmm4		    # XMM4 = W[t-15]
-	and	e_64, T1
-	xor	g_64, T1
-	psllq	$(64-19)-(64-61) , %xmm1    # XMM1 = W[t-2] << 42
-	idx = \rnd + 1
-	add	WK_2(idx), T1
-	mov	e_64, tmp0
-	psllq	$(64-1)-(64-8), %xmm4	    # XMM4 = W[t-15] << 7
-	ror	$23, tmp0 # 41
-	xor	e_64, tmp0
-	pxor	%xmm2, %xmm1		    # XMM1 = (W[t-2] << 42)^W[t-2]
-	ror	$4, tmp0 # 18
-	xor	e_64, tmp0
-	pxor	%xmm5, %xmm4		    # XMM4 = (W[t-15]<<7)^W[t-15]
-	ror	$14, tmp0 # 14
-	add	tmp0, T1
-	psllq	$(64-61), %xmm1		    # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
-	add	h_64, T1
-	mov	a_64, T2
-	psllq	$(64-8), %xmm4		    # XMM4 = ((W[t-15]<<7)^W[t-15])<<56
-	xor	c_64, T2
-	and	b_64, T2
-	pxor	%xmm1, %xmm0		    # XMM0 = s1(W[t-2])
-	mov	a_64, tmp0
-	and	c_64, tmp0
-	idx = \rnd - 7
-	movdqu	W_t(idx), %xmm1		    # XMM1 = W[t-7]
-	xor	tmp0, T2
-	pxor	%xmm4, %xmm3                # XMM3 = s0(W[t-15])
-	mov	a_64, tmp0
-	paddq	%xmm3, %xmm0		    # XMM0 = s1(W[t-2]) + s0(W[t-15])
-	ror	$5, tmp0 # 39
-	idx =\rnd-16
-	paddq	W_t(idx), %xmm0		    # XMM0 = s1(W[t-2]) + s0(W[t-15]) + W[t-16]
-	xor	a_64, tmp0
-	paddq	%xmm1, %xmm0	            # XMM0 = s1(W[t-2]) + W[t-7] + s0(W[t-15]) + W[t-16]
-	ror	$6, tmp0 # 34
-	movdqa	%xmm0, W_t(\rnd)	    # Store scheduled qwords
-	xor	a_64, tmp0
-	paddq	K_t(\rnd), %xmm0	    # Compute W[t]+K[t]
-	ror	$28, tmp0 # 28
-	idx = \rnd
-	movdqa	%xmm0, WK_2(idx)	    # Store W[t]+K[t] for next rounds
-	add	tmp0, T2
-	add	T1, d_64
-	lea	(T1, T2), h_64
-	RotateState
-.endm
-
-########################################################################
-## void sha512_transform_ssse3(struct sha512_state *state, const u8 *data,
-##			       int blocks);
-# (struct sha512_state is assumed to begin with u64 state[8])
-# Purpose: Updates the SHA512 digest stored at "state" with the message
-# stored in "data".
-# The size of the message pointed to by "data" must be an integer multiple
-# of SHA512 message blocks.
-# "blocks" is the message length in SHA512 blocks.
-########################################################################
-SYM_TYPED_FUNC_START(sha512_transform_ssse3)
-
-	test msglen, msglen
-	je .Lnowork
-
-	# Save GPRs
-	push	%rbx
-	push	%r12
-	push	%r13
-	push	%r14
-	push	%r15
-
-	# Allocate Stack Space
-	push	%rbp
-	mov	%rsp, %rbp
-	sub	$frame_size, %rsp
-	and	$~(0x20 - 1), %rsp
-
-.Lupdateblock:
-
-# Load state variables
-	mov	DIGEST(0), a_64
-	mov	DIGEST(1), b_64
-	mov	DIGEST(2), c_64
-	mov	DIGEST(3), d_64
-	mov	DIGEST(4), e_64
-	mov	DIGEST(5), f_64
-	mov	DIGEST(6), g_64
-	mov	DIGEST(7), h_64
-
-	t = 0
-	.rept 80/2 + 1
-	# (80 rounds) / (2 rounds/iteration) + (1 iteration)
-	# +1 iteration because the scheduler leads hashing by 1 iteration
-		.if t < 2
-			# BSWAP 2 QWORDS
-			movdqa	XMM_QWORD_BSWAP(%rip), %xmm1
-			movdqu	MSG(t), %xmm0
-			pshufb	%xmm1, %xmm0	# BSWAP
-			movdqa	%xmm0, W_t(t)	# Store Scheduled Pair
-			paddq	K_t(t), %xmm0	# Compute W[t]+K[t]
-			movdqa	%xmm0, WK_2(t)	# Store into WK for rounds
-		.elseif t < 16
-			# BSWAP 2 QWORDS# Compute 2 Rounds
-			movdqu	MSG(t), %xmm0
-			pshufb	%xmm1, %xmm0	# BSWAP
-			SHA512_Round t-2	# Round t-2
-			movdqa	%xmm0, W_t(t)	# Store Scheduled Pair
-			paddq	K_t(t), %xmm0	# Compute W[t]+K[t]
-			SHA512_Round t-1	# Round t-1
-			movdqa	%xmm0, WK_2(t)	# Store W[t]+K[t] into WK
-		.elseif t < 79
-			# Schedule 2 QWORDS# Compute 2 Rounds
-			SHA512_2Sched_2Round_sse t
-		.else
-			# Compute 2 Rounds
-			SHA512_Round t-2
-			SHA512_Round t-1
-		.endif
-		t = t+2
-	.endr
-
-	# Update digest
-	add	a_64, DIGEST(0)
-	add	b_64, DIGEST(1)
-	add	c_64, DIGEST(2)
-	add	d_64, DIGEST(3)
-	add	e_64, DIGEST(4)
-	add	f_64, DIGEST(5)
-	add	g_64, DIGEST(6)
-	add	h_64, DIGEST(7)
-
-	# Advance to next message block
-	add	$16*8, msg
-	dec	msglen
-	jnz	.Lupdateblock
-
-	# Restore Stack Pointer
-	mov	%rbp, %rsp
-	pop	%rbp
-
-	# Restore GPRs
-	pop	%r15
-	pop	%r14
-	pop	%r13
-	pop	%r12
-	pop	%rbx
-
-.Lnowork:
-	RET
-SYM_FUNC_END(sha512_transform_ssse3)
-
-########################################################################
-### Binary Data
-
-.section	.rodata.cst16.XMM_QWORD_BSWAP, "aM", @progbits, 16
-.align 16
-# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
-XMM_QWORD_BSWAP:
-	.octa 0x08090a0b0c0d0e0f0001020304050607
-
-# Mergeable 640-byte rodata section. This allows linker to merge the table
-# with other, exactly the same 640-byte fragment of another rodata section
-# (if such section exists).
-.section	.rodata.cst640.K512, "aM", @progbits, 640
-.align 64
-# K[t] used in SHA512 hashing
-K512:
-	.quad 0x428a2f98d728ae22,0x7137449123ef65cd
-	.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
-	.quad 0x3956c25bf348b538,0x59f111f1b605d019
-	.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
-	.quad 0xd807aa98a3030242,0x12835b0145706fbe
-	.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
-	.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
-	.quad 0x9bdc06a725c71235,0xc19bf174cf692694
-	.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
-	.quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
-	.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
-	.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
-	.quad 0x983e5152ee66dfab,0xa831c66d2db43210
-	.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
-	.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
-	.quad 0x06ca6351e003826f,0x142929670a0e6e70
-	.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
-	.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
-	.quad 0x650a73548baf63de,0x766a0abb3c77b2a8
-	.quad 0x81c2c92e47edaee6,0x92722c851482353b
-	.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
-	.quad 0xc24b8b70d0f89791,0xc76c51a30654be30
-	.quad 0xd192e819d6ef5218,0xd69906245565a910
-	.quad 0xf40e35855771202a,0x106aa07032bbd1b8
-	.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
-	.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
-	.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
-	.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
-	.quad 0x748f82ee5defb2fc,0x78a5636f43172f60
-	.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
-	.quad 0x90befffa23631e28,0xa4506cebde82bde9
-	.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
-	.quad 0xca273eceea26619c,0xd186b8c721c0c207
-	.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
-	.quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
-	.quad 0x113f9804bef90dae,0x1b710b35131c471b
-	.quad 0x28db77f523047d84,0x32caab7b40c72493
-	.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
-	.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
-	.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c
deleted file mode 100644
index 6d3b85e53d0e..000000000000
--- a/arch/x86/crypto/sha512_ssse3_glue.c
+++ /dev/null
@@ -1,347 +0,0 @@
-/*
- * Cryptographic API.
- *
- * Glue code for the SHA512 Secure Hash Algorithm assembler
- * implementation using supplemental SSE3 / AVX / AVX2 instructions.
- *
- * This file is based on sha512_generic.c
- *
- * Copyright (C) 2013 Intel Corporation
- * Author: Tim Chen <tim.c.chen@linux.intel.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
-
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/mm.h>
-#include <linux/string.h>
-#include <linux/types.h>
-#include <crypto/sha2.h>
-#include <crypto/sha512_base.h>
-#include <asm/cpu_device_id.h>
-#include <asm/simd.h>
-
-asmlinkage void sha512_transform_ssse3(struct sha512_state *state,
-				       const u8 *data, int blocks);
-
-static int sha512_update(struct shash_desc *desc, const u8 *data,
-		       unsigned int len, sha512_block_fn *sha512_xform)
-{
-	struct sha512_state *sctx = shash_desc_ctx(desc);
-
-	if (!crypto_simd_usable() ||
-	    (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE)
-		return crypto_sha512_update(desc, data, len);
-
-	/*
-	 * Make sure struct sha512_state begins directly with the SHA512
-	 * 512-bit internal state, as this is what the asm functions expect.
-	 */
-	BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);
-
-	kernel_fpu_begin();
-	sha512_base_do_update(desc, data, len, sha512_xform);
-	kernel_fpu_end();
-
-	return 0;
-}
-
-static int sha512_finup(struct shash_desc *desc, const u8 *data,
-	      unsigned int len, u8 *out, sha512_block_fn *sha512_xform)
-{
-	if (!crypto_simd_usable())
-		return crypto_sha512_finup(desc, data, len, out);
-
-	kernel_fpu_begin();
-	if (len)
-		sha512_base_do_update(desc, data, len, sha512_xform);
-	sha512_base_do_finalize(desc, sha512_xform);
-	kernel_fpu_end();
-
-	return sha512_base_finish(desc, out);
-}
-
-static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data,
-		       unsigned int len)
-{
-	return sha512_update(desc, data, len, sha512_transform_ssse3);
-}
-
-static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data,
-	      unsigned int len, u8 *out)
-{
-	return sha512_finup(desc, data, len, out, sha512_transform_ssse3);
-}
-
-/* Add padding and return the message digest. */
-static int sha512_ssse3_final(struct shash_desc *desc, u8 *out)
-{
-	return sha512_ssse3_finup(desc, NULL, 0, out);
-}
-
-static struct shash_alg sha512_ssse3_algs[] = { {
-	.digestsize	=	SHA512_DIGEST_SIZE,
-	.init		=	sha512_base_init,
-	.update		=	sha512_ssse3_update,
-	.final		=	sha512_ssse3_final,
-	.finup		=	sha512_ssse3_finup,
-	.descsize	=	sizeof(struct sha512_state),
-	.base		=	{
-		.cra_name	=	"sha512",
-		.cra_driver_name =	"sha512-ssse3",
-		.cra_priority	=	150,
-		.cra_blocksize	=	SHA512_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-},  {
-	.digestsize	=	SHA384_DIGEST_SIZE,
-	.init		=	sha384_base_init,
-	.update		=	sha512_ssse3_update,
-	.final		=	sha512_ssse3_final,
-	.finup		=	sha512_ssse3_finup,
-	.descsize	=	sizeof(struct sha512_state),
-	.base		=	{
-		.cra_name	=	"sha384",
-		.cra_driver_name =	"sha384-ssse3",
-		.cra_priority	=	150,
-		.cra_blocksize	=	SHA384_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-} };
-
-static int register_sha512_ssse3(void)
-{
-	if (boot_cpu_has(X86_FEATURE_SSSE3))
-		return crypto_register_shashes(sha512_ssse3_algs,
-			ARRAY_SIZE(sha512_ssse3_algs));
-	return 0;
-}
-
-static void unregister_sha512_ssse3(void)
-{
-	if (boot_cpu_has(X86_FEATURE_SSSE3))
-		crypto_unregister_shashes(sha512_ssse3_algs,
-			ARRAY_SIZE(sha512_ssse3_algs));
-}
-
-asmlinkage void sha512_transform_avx(struct sha512_state *state,
-				     const u8 *data, int blocks);
-static bool avx_usable(void)
-{
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
-		if (boot_cpu_has(X86_FEATURE_AVX))
-			pr_info("AVX detected but unusable.\n");
-		return false;
-	}
-
-	return true;
-}
-
-static int sha512_avx_update(struct shash_desc *desc, const u8 *data,
-		       unsigned int len)
-{
-	return sha512_update(desc, data, len, sha512_transform_avx);
-}
-
-static int sha512_avx_finup(struct shash_desc *desc, const u8 *data,
-	      unsigned int len, u8 *out)
-{
-	return sha512_finup(desc, data, len, out, sha512_transform_avx);
-}
-
-/* Add padding and return the message digest. */
-static int sha512_avx_final(struct shash_desc *desc, u8 *out)
-{
-	return sha512_avx_finup(desc, NULL, 0, out);
-}
-
-static struct shash_alg sha512_avx_algs[] = { {
-	.digestsize	=	SHA512_DIGEST_SIZE,
-	.init		=	sha512_base_init,
-	.update		=	sha512_avx_update,
-	.final		=	sha512_avx_final,
-	.finup		=	sha512_avx_finup,
-	.descsize	=	sizeof(struct sha512_state),
-	.base		=	{
-		.cra_name	=	"sha512",
-		.cra_driver_name =	"sha512-avx",
-		.cra_priority	=	160,
-		.cra_blocksize	=	SHA512_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-},  {
-	.digestsize	=	SHA384_DIGEST_SIZE,
-	.init		=	sha384_base_init,
-	.update		=	sha512_avx_update,
-	.final		=	sha512_avx_final,
-	.finup		=	sha512_avx_finup,
-	.descsize	=	sizeof(struct sha512_state),
-	.base		=	{
-		.cra_name	=	"sha384",
-		.cra_driver_name =	"sha384-avx",
-		.cra_priority	=	160,
-		.cra_blocksize	=	SHA384_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-} };
-
-static int register_sha512_avx(void)
-{
-	if (avx_usable())
-		return crypto_register_shashes(sha512_avx_algs,
-			ARRAY_SIZE(sha512_avx_algs));
-	return 0;
-}
-
-static void unregister_sha512_avx(void)
-{
-	if (avx_usable())
-		crypto_unregister_shashes(sha512_avx_algs,
-			ARRAY_SIZE(sha512_avx_algs));
-}
-
-asmlinkage void sha512_transform_rorx(struct sha512_state *state,
-				      const u8 *data, int blocks);
-
-static int sha512_avx2_update(struct shash_desc *desc, const u8 *data,
-		       unsigned int len)
-{
-	return sha512_update(desc, data, len, sha512_transform_rorx);
-}
-
-static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data,
-	      unsigned int len, u8 *out)
-{
-	return sha512_finup(desc, data, len, out, sha512_transform_rorx);
-}
-
-/* Add padding and return the message digest. */
-static int sha512_avx2_final(struct shash_desc *desc, u8 *out)
-{
-	return sha512_avx2_finup(desc, NULL, 0, out);
-}
-
-static struct shash_alg sha512_avx2_algs[] = { {
-	.digestsize	=	SHA512_DIGEST_SIZE,
-	.init		=	sha512_base_init,
-	.update		=	sha512_avx2_update,
-	.final		=	sha512_avx2_final,
-	.finup		=	sha512_avx2_finup,
-	.descsize	=	sizeof(struct sha512_state),
-	.base		=	{
-		.cra_name	=	"sha512",
-		.cra_driver_name =	"sha512-avx2",
-		.cra_priority	=	170,
-		.cra_blocksize	=	SHA512_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-},  {
-	.digestsize	=	SHA384_DIGEST_SIZE,
-	.init		=	sha384_base_init,
-	.update		=	sha512_avx2_update,
-	.final		=	sha512_avx2_final,
-	.finup		=	sha512_avx2_finup,
-	.descsize	=	sizeof(struct sha512_state),
-	.base		=	{
-		.cra_name	=	"sha384",
-		.cra_driver_name =	"sha384-avx2",
-		.cra_priority	=	170,
-		.cra_blocksize	=	SHA384_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-} };
-
-static bool avx2_usable(void)
-{
-	if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
-		    boot_cpu_has(X86_FEATURE_BMI2))
-		return true;
-
-	return false;
-}
-
-static int register_sha512_avx2(void)
-{
-	if (avx2_usable())
-		return crypto_register_shashes(sha512_avx2_algs,
-			ARRAY_SIZE(sha512_avx2_algs));
-	return 0;
-}
-static const struct x86_cpu_id module_cpu_ids[] = {
-	X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL),
-	X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL),
-	X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL),
-	{}
-};
-MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids);
-
-static void unregister_sha512_avx2(void)
-{
-	if (avx2_usable())
-		crypto_unregister_shashes(sha512_avx2_algs,
-			ARRAY_SIZE(sha512_avx2_algs));
-}
-
-static int __init sha512_ssse3_mod_init(void)
-{
-	if (!x86_match_cpu(module_cpu_ids))
-		return -ENODEV;
-
-	if (register_sha512_ssse3())
-		goto fail;
-
-	if (register_sha512_avx()) {
-		unregister_sha512_ssse3();
-		goto fail;
-	}
-
-	if (register_sha512_avx2()) {
-		unregister_sha512_avx();
-		unregister_sha512_ssse3();
-		goto fail;
-	}
-
-	return 0;
-fail:
-	return -ENODEV;
-}
-
-static void __exit sha512_ssse3_mod_fini(void)
-{
-	unregister_sha512_avx2();
-	unregister_sha512_avx();
-	unregister_sha512_ssse3();
-}
-
-module_init(sha512_ssse3_mod_init);
-module_exit(sha512_ssse3_mod_fini);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated");
-
-MODULE_ALIAS_CRYPTO("sha512");
-MODULE_ALIAS_CRYPTO("sha512-ssse3");
-MODULE_ALIAS_CRYPTO("sha512-avx");
-MODULE_ALIAS_CRYPTO("sha512-avx2");
-MODULE_ALIAS_CRYPTO("sha384");
-MODULE_ALIAS_CRYPTO("sha384-ssse3");
-MODULE_ALIAS_CRYPTO("sha384-avx");
-MODULE_ALIAS_CRYPTO("sha384-avx2");
diff --git a/arch/x86/crypto/sm3_avx_glue.c b/arch/x86/crypto/sm3_avx_glue.c
index 661b6f22ffcd..6e8c42b9dc8e 100644
--- a/arch/x86/crypto/sm3_avx_glue.c
+++ b/arch/x86/crypto/sm3_avx_glue.c
@@ -10,12 +10,11 @@
 
 #include <crypto/internal/hash.h>
 #include <crypto/internal/simd.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/types.h>
 #include <crypto/sm3.h>
 #include <crypto/sm3_base.h>
-#include <asm/simd.h>
+#include <linux/cpufeature.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
 
 asmlinkage void sm3_transform_avx(struct sm3_state *state,
 			const u8 *data, int nblocks);
@@ -23,13 +22,7 @@ asmlinkage void sm3_transform_avx(struct sm3_state *state,
 static int sm3_avx_update(struct shash_desc *desc, const u8 *data,
 			 unsigned int len)
 {
-	struct sm3_state *sctx = shash_desc_ctx(desc);
-
-	if (!crypto_simd_usable() ||
-			(sctx->count % SM3_BLOCK_SIZE) + len < SM3_BLOCK_SIZE) {
-		sm3_update(sctx, data, len);
-		return 0;
-	}
+	int remain;
 
 	/*
 	 * Make sure struct sm3_state begins directly with the SM3
@@ -38,45 +31,17 @@ static int sm3_avx_update(struct shash_desc *desc, const u8 *data,
 	BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0);
 
 	kernel_fpu_begin();
-	sm3_base_do_update(desc, data, len, sm3_transform_avx);
+	remain = sm3_base_do_update_blocks(desc, data, len, sm3_transform_avx);
 	kernel_fpu_end();
-
-	return 0;
+	return remain;
 }
 
 static int sm3_avx_finup(struct shash_desc *desc, const u8 *data,
 		      unsigned int len, u8 *out)
 {
-	if (!crypto_simd_usable()) {
-		struct sm3_state *sctx = shash_desc_ctx(desc);
-
-		if (len)
-			sm3_update(sctx, data, len);
-
-		sm3_final(sctx, out);
-		return 0;
-	}
-
 	kernel_fpu_begin();
-	if (len)
-		sm3_base_do_update(desc, data, len, sm3_transform_avx);
-	sm3_base_do_finalize(desc, sm3_transform_avx);
+	sm3_base_do_finup(desc, data, len, sm3_transform_avx);
 	kernel_fpu_end();
-
-	return sm3_base_finish(desc, out);
-}
-
-static int sm3_avx_final(struct shash_desc *desc, u8 *out)
-{
-	if (!crypto_simd_usable()) {
-		sm3_final(shash_desc_ctx(desc), out);
-		return 0;
-	}
-
-	kernel_fpu_begin();
-	sm3_base_do_finalize(desc, sm3_transform_avx);
-	kernel_fpu_end();
-
 	return sm3_base_finish(desc, out);
 }
 
@@ -84,13 +49,14 @@ static struct shash_alg sm3_avx_alg = {
 	.digestsize	=	SM3_DIGEST_SIZE,
 	.init		=	sm3_base_init,
 	.update		=	sm3_avx_update,
-	.final		=	sm3_avx_final,
 	.finup		=	sm3_avx_finup,
-	.descsize	=	sizeof(struct sm3_state),
+	.descsize	=	SM3_STATE_SIZE,
 	.base		=	{
 		.cra_name	=	"sm3",
 		.cra_driver_name =	"sm3-avx",
 		.cra_priority	=	300,
+		.cra_flags	 =	CRYPTO_AHASH_ALG_BLOCK_ONLY |
+					CRYPTO_AHASH_ALG_FINUP_MAX,
 		.cra_blocksize	=	SM3_BLOCK_SIZE,
 		.cra_module	=	THIS_MODULE,
 	}
diff --git a/arch/x86/crypto/sm4_aesni_avx2_glue.c b/arch/x86/crypto/sm4_aesni_avx2_glue.c
index 1148fd4cd57f..fec0ab7a63dd 100644
--- a/arch/x86/crypto/sm4_aesni_avx2_glue.c
+++ b/arch/x86/crypto/sm4_aesni_avx2_glue.c
@@ -8,11 +8,10 @@
  * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
  */
 
+#include <asm/fpu/api.h>
 #include <linux/module.h>
 #include <linux/crypto.h>
 #include <linux/kernel.h>
-#include <asm/simd.h>
-#include <crypto/internal/simd.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/sm4.h>
 #include "sm4-avx.h"
@@ -48,10 +47,9 @@ static int ctr_crypt(struct skcipher_request *req)
 static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
 	{
 		.base = {
-			.cra_name		= "__ecb(sm4)",
-			.cra_driver_name	= "__ecb-sm4-aesni-avx2",
+			.cra_name		= "ecb(sm4)",
+			.cra_driver_name	= "ecb-sm4-aesni-avx2",
 			.cra_priority		= 500,
-			.cra_flags		= CRYPTO_ALG_INTERNAL,
 			.cra_blocksize		= SM4_BLOCK_SIZE,
 			.cra_ctxsize		= sizeof(struct sm4_ctx),
 			.cra_module		= THIS_MODULE,
@@ -64,10 +62,9 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
 		.decrypt	= sm4_avx_ecb_decrypt,
 	}, {
 		.base = {
-			.cra_name		= "__cbc(sm4)",
-			.cra_driver_name	= "__cbc-sm4-aesni-avx2",
+			.cra_name		= "cbc(sm4)",
+			.cra_driver_name	= "cbc-sm4-aesni-avx2",
 			.cra_priority		= 500,
-			.cra_flags		= CRYPTO_ALG_INTERNAL,
 			.cra_blocksize		= SM4_BLOCK_SIZE,
 			.cra_ctxsize		= sizeof(struct sm4_ctx),
 			.cra_module		= THIS_MODULE,
@@ -81,10 +78,9 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
 		.decrypt	= cbc_decrypt,
 	}, {
 		.base = {
-			.cra_name		= "__ctr(sm4)",
-			.cra_driver_name	= "__ctr-sm4-aesni-avx2",
+			.cra_name		= "ctr(sm4)",
+			.cra_driver_name	= "ctr-sm4-aesni-avx2",
 			.cra_priority		= 500,
-			.cra_flags		= CRYPTO_ALG_INTERNAL,
 			.cra_blocksize		= 1,
 			.cra_ctxsize		= sizeof(struct sm4_ctx),
 			.cra_module		= THIS_MODULE,
@@ -100,9 +96,6 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
 	}
 };
 
-static struct simd_skcipher_alg *
-simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)];
-
 static int __init sm4_init(void)
 {
 	const char *feature_name;
@@ -121,16 +114,14 @@ static int __init sm4_init(void)
 		return -ENODEV;
 	}
 
-	return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers,
-					ARRAY_SIZE(sm4_aesni_avx2_skciphers),
-					simd_sm4_aesni_avx2_skciphers);
+	return crypto_register_skciphers(sm4_aesni_avx2_skciphers,
+					 ARRAY_SIZE(sm4_aesni_avx2_skciphers));
 }
 
 static void __exit sm4_exit(void)
 {
-	simd_unregister_skciphers(sm4_aesni_avx2_skciphers,
-				ARRAY_SIZE(sm4_aesni_avx2_skciphers),
-				simd_sm4_aesni_avx2_skciphers);
+	crypto_unregister_skciphers(sm4_aesni_avx2_skciphers,
+				    ARRAY_SIZE(sm4_aesni_avx2_skciphers));
 }
 
 module_init(sm4_init);
diff --git a/arch/x86/crypto/sm4_aesni_avx_glue.c b/arch/x86/crypto/sm4_aesni_avx_glue.c
index 85b4ca78b47b..88caf418a06f 100644
--- a/arch/x86/crypto/sm4_aesni_avx_glue.c
+++ b/arch/x86/crypto/sm4_aesni_avx_glue.c
@@ -8,11 +8,11 @@
  * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
  */
 
+#include <asm/fpu/api.h>
 #include <linux/module.h>
 #include <linux/crypto.h>
+#include <linux/export.h>
 #include <linux/kernel.h>
-#include <asm/simd.h>
-#include <crypto/internal/simd.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/sm4.h>
 #include "sm4-avx.h"
@@ -263,10 +263,9 @@ static int ctr_crypt(struct skcipher_request *req)
 static struct skcipher_alg sm4_aesni_avx_skciphers[] = {
 	{
 		.base = {
-			.cra_name		= "__ecb(sm4)",
-			.cra_driver_name	= "__ecb-sm4-aesni-avx",
+			.cra_name		= "ecb(sm4)",
+			.cra_driver_name	= "ecb-sm4-aesni-avx",
 			.cra_priority		= 400,
-			.cra_flags		= CRYPTO_ALG_INTERNAL,
 			.cra_blocksize		= SM4_BLOCK_SIZE,
 			.cra_ctxsize		= sizeof(struct sm4_ctx),
 			.cra_module		= THIS_MODULE,
@@ -279,10 +278,9 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = {
 		.decrypt	= sm4_avx_ecb_decrypt,
 	}, {
 		.base = {
-			.cra_name		= "__cbc(sm4)",
-			.cra_driver_name	= "__cbc-sm4-aesni-avx",
+			.cra_name		= "cbc(sm4)",
+			.cra_driver_name	= "cbc-sm4-aesni-avx",
 			.cra_priority		= 400,
-			.cra_flags		= CRYPTO_ALG_INTERNAL,
 			.cra_blocksize		= SM4_BLOCK_SIZE,
 			.cra_ctxsize		= sizeof(struct sm4_ctx),
 			.cra_module		= THIS_MODULE,
@@ -296,10 +294,9 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = {
 		.decrypt	= cbc_decrypt,
 	}, {
 		.base = {
-			.cra_name		= "__ctr(sm4)",
-			.cra_driver_name	= "__ctr-sm4-aesni-avx",
+			.cra_name		= "ctr(sm4)",
+			.cra_driver_name	= "ctr-sm4-aesni-avx",
 			.cra_priority		= 400,
-			.cra_flags		= CRYPTO_ALG_INTERNAL,
 			.cra_blocksize		= 1,
 			.cra_ctxsize		= sizeof(struct sm4_ctx),
 			.cra_module		= THIS_MODULE,
@@ -315,9 +312,6 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = {
 	}
 };
 
-static struct simd_skcipher_alg *
-simd_sm4_aesni_avx_skciphers[ARRAY_SIZE(sm4_aesni_avx_skciphers)];
-
 static int __init sm4_init(void)
 {
 	const char *feature_name;
@@ -335,16 +329,14 @@ static int __init sm4_init(void)
 		return -ENODEV;
 	}
 
-	return simd_register_skciphers_compat(sm4_aesni_avx_skciphers,
-					ARRAY_SIZE(sm4_aesni_avx_skciphers),
-					simd_sm4_aesni_avx_skciphers);
+	return crypto_register_skciphers(sm4_aesni_avx_skciphers,
+					 ARRAY_SIZE(sm4_aesni_avx_skciphers));
 }
 
 static void __exit sm4_exit(void)
 {
-	simd_unregister_skciphers(sm4_aesni_avx_skciphers,
-					ARRAY_SIZE(sm4_aesni_avx_skciphers),
-					simd_sm4_aesni_avx_skciphers);
+	crypto_unregister_skciphers(sm4_aesni_avx_skciphers,
+				    ARRAY_SIZE(sm4_aesni_avx_skciphers));
 }
 
 module_init(sm4_init);
diff --git a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
index d2288bf38a8a..071e90e7f0d8 100644
--- a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
+++ b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
@@ -6,6 +6,7 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 
 .file "twofish-x86_64-asm-3way.S"
 .text
@@ -220,7 +221,7 @@
 	rorq $32,			RAB2; \
 	outunpack3(mov, RIO, 2, RAB, 2);
 
-SYM_FUNC_START(__twofish_enc_blk_3way)
+SYM_TYPED_FUNC_START(__twofish_enc_blk_3way)
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	%rsi: dst
@@ -269,7 +270,7 @@ SYM_FUNC_START(__twofish_enc_blk_3way)
 	RET;
 SYM_FUNC_END(__twofish_enc_blk_3way)
 
-SYM_FUNC_START(twofish_dec_blk_3way)
+SYM_TYPED_FUNC_START(twofish_dec_blk_3way)
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	%rsi: dst
diff --git a/arch/x86/crypto/twofish-x86_64-asm_64.S b/arch/x86/crypto/twofish-x86_64-asm_64.S
index 775af290cd19..e08b4ba07b93 100644
--- a/arch/x86/crypto/twofish-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-x86_64-asm_64.S
@@ -8,6 +8,7 @@
 .text
 
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 #include <asm/asm-offsets.h>
 
 #define a_offset	0
@@ -202,7 +203,7 @@
 	xor	%r8d,		d ## D;\
 	ror	$1,		d ## D;
 
-SYM_FUNC_START(twofish_enc_blk)
+SYM_TYPED_FUNC_START(twofish_enc_blk)
 	pushq    R1
 
 	/* %rdi contains the ctx address */
@@ -255,7 +256,7 @@ SYM_FUNC_START(twofish_enc_blk)
 	RET
 SYM_FUNC_END(twofish_enc_blk)
 
-SYM_FUNC_START(twofish_dec_blk)
+SYM_TYPED_FUNC_START(twofish_dec_blk)
 	pushq    R1
 
 	/* %rdi contains the ctx address */
diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c
index 3eb3440b477a..9e20db013750 100644
--- a/arch/x86/crypto/twofish_avx_glue.c
+++ b/arch/x86/crypto/twofish_avx_glue.c
@@ -13,7 +13,6 @@
 #include <linux/crypto.h>
 #include <linux/err.h>
 #include <crypto/algapi.h>
-#include <crypto/internal/simd.h>
 #include <crypto/twofish.h>
 
 #include "twofish.h"
@@ -74,10 +73,9 @@ static int cbc_decrypt(struct skcipher_request *req)
 
 static struct skcipher_alg twofish_algs[] = {
 	{
-		.base.cra_name		= "__ecb(twofish)",
-		.base.cra_driver_name	= "__ecb-twofish-avx",
+		.base.cra_name		= "ecb(twofish)",
+		.base.cra_driver_name	= "ecb-twofish-avx",
 		.base.cra_priority	= 400,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= TF_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct twofish_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -87,10 +85,9 @@ static struct skcipher_alg twofish_algs[] = {
 		.encrypt		= ecb_encrypt,
 		.decrypt		= ecb_decrypt,
 	}, {
-		.base.cra_name		= "__cbc(twofish)",
-		.base.cra_driver_name	= "__cbc-twofish-avx",
+		.base.cra_name		= "cbc(twofish)",
+		.base.cra_driver_name	= "cbc-twofish-avx",
 		.base.cra_priority	= 400,
-		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
 		.base.cra_blocksize	= TF_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct twofish_ctx),
 		.base.cra_module	= THIS_MODULE,
@@ -103,8 +100,6 @@ static struct skcipher_alg twofish_algs[] = {
 	},
 };
 
-static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];
-
 static int __init twofish_init(void)
 {
 	const char *feature_name;
@@ -114,15 +109,13 @@ static int __init twofish_init(void)
 		return -ENODEV;
 	}
 
-	return simd_register_skciphers_compat(twofish_algs,
-					      ARRAY_SIZE(twofish_algs),
-					      twofish_simd_algs);
+	return crypto_register_skciphers(twofish_algs,
+					 ARRAY_SIZE(twofish_algs));
 }
 
 static void __exit twofish_exit(void)
 {
-	simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
-				  twofish_simd_algs);
+	crypto_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs));
 }
 
 module_init(twofish_init);
diff --git a/arch/x86/crypto/twofish_glue.c b/arch/x86/crypto/twofish_glue.c
index 0614beece279..8e9906d36902 100644
--- a/arch/x86/crypto/twofish_glue.c
+++ b/arch/x86/crypto/twofish_glue.c
@@ -40,6 +40,7 @@
 
 #include <crypto/algapi.h>
 #include <crypto/twofish.h>
+#include <linux/export.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/types.h>
@@ -68,7 +69,6 @@ static struct crypto_alg alg = {
 	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize		=	TF_BLOCK_SIZE,
 	.cra_ctxsize		=	sizeof(struct twofish_ctx),
-	.cra_alignmask		=	0,
 	.cra_module		=	THIS_MODULE,
 	.cra_u			=	{
 		.cipher = {
diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c
index 1a1ecfa7f72a..8ad77725bf60 100644
--- a/arch/x86/crypto/twofish_glue_3way.c
+++ b/arch/x86/crypto/twofish_glue_3way.c
@@ -9,6 +9,7 @@
 #include <crypto/algapi.h>
 #include <crypto/twofish.h>
 #include <linux/crypto.h>
+#include <linux/export.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/types.h>
diff --git a/arch/x86/entry/Makefile b/arch/x86/entry/Makefile
index ce1cc1622385..72cae8e0ce85 100644
--- a/arch/x86/entry/Makefile
+++ b/arch/x86/entry/Makefile
@@ -7,12 +7,13 @@ KASAN_SANITIZE := n
 UBSAN_SANITIZE := n
 KCOV_INSTRUMENT := n
 
-CFLAGS_REMOVE_common.o		= $(CC_FLAGS_FTRACE)
+CFLAGS_REMOVE_syscall_32.o	= $(CC_FLAGS_FTRACE)
+CFLAGS_REMOVE_syscall_64.o	= $(CC_FLAGS_FTRACE)
 
-CFLAGS_common.o			+= -fno-stack-protector
+CFLAGS_syscall_32.o		+= -fno-stack-protector
+CFLAGS_syscall_64.o		+= -fno-stack-protector
 
 obj-y				:= entry.o entry_$(BITS).o syscall_$(BITS).o
-obj-y				+= common.o
 
 obj-y				+= vdso/
 obj-y				+= vsyscall/
@@ -23,4 +24,3 @@ CFLAGS_REMOVE_entry_fred.o	+= -pg $(CC_FLAGS_FTRACE)
 obj-$(CONFIG_X86_FRED)		+= entry_64_fred.o entry_fred.o
 
 obj-$(CONFIG_IA32_EMULATION)	+= entry_64_compat.o syscall_32.o
-obj-$(CONFIG_X86_X32_ABI)	+= syscall_x32.o
diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index ea81770629ee..77e2d920a640 100644
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -70,6 +70,8 @@ For 32-bit we have the following conventions - kernel is built with
 	pushq	%rsi		/* pt_regs->si */
 	movq	8(%rsp), %rsi	/* temporarily store the return address in %rsi */
 	movq	%rdi, 8(%rsp)	/* pt_regs->di (overwriting original return address) */
+	/* We just clobbered the return address - use the IRET frame for unwinding: */
+	UNWIND_HINT_IRET_REGS offset=3*8
 	.else
 	pushq   %rdi		/* pt_regs->di */
 	pushq   %rsi		/* pt_regs->si */
@@ -97,7 +99,7 @@ For 32-bit we have the following conventions - kernel is built with
 	.endif
 .endm
 
-.macro CLEAR_REGS clear_bp=1
+.macro CLEAR_REGS clear_callee=1
 	/*
 	 * Sanitize registers of values that a speculation attack might
 	 * otherwise want to exploit. The lower registers are likely clobbered
@@ -111,20 +113,19 @@ For 32-bit we have the following conventions - kernel is built with
 	xorl	%r9d,  %r9d	/* nospec r9  */
 	xorl	%r10d, %r10d	/* nospec r10 */
 	xorl	%r11d, %r11d	/* nospec r11 */
+	.if \clear_callee
 	xorl	%ebx,  %ebx	/* nospec rbx */
-	.if \clear_bp
 	xorl	%ebp,  %ebp	/* nospec rbp */
-	.endif
 	xorl	%r12d, %r12d	/* nospec r12 */
 	xorl	%r13d, %r13d	/* nospec r13 */
 	xorl	%r14d, %r14d	/* nospec r14 */
 	xorl	%r15d, %r15d	/* nospec r15 */
-
+	.endif
 .endm
 
-.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_bp=1 unwind_hint=1
+.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_callee=1 unwind_hint=1
 	PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret unwind_hint=\unwind_hint
-	CLEAR_REGS clear_bp=\clear_bp
+	CLEAR_REGS clear_callee=\clear_callee
 .endm
 
 .macro POP_REGS pop_rdi=1
@@ -367,7 +368,7 @@ For 32-bit we have the following conventions - kernel is built with
 .endm
 
 .macro STACKLEAK_ERASE_NOCLOBBER
-#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
+#ifdef CONFIG_KSTACK_ERASE
 	PUSH_AND_CLEAR_REGS
 	call stackleak_erase
 	POP_REGS
@@ -386,7 +387,7 @@ For 32-bit we have the following conventions - kernel is built with
 #endif /* !CONFIG_X86_64 */
 
 .macro STACKLEAK_ERASE
-#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
+#ifdef CONFIG_KSTACK_ERASE
 	call stackleak_erase
 #endif
 .endm
@@ -431,6 +432,7 @@ For 32-bit we have the following conventions - kernel is built with
 /* rdi:	arg1 ... normal C conventions. rax is saved/restored. */
 .macro THUNK name, func
 SYM_FUNC_START(\name)
+	ANNOTATE_NOENDBR
 	pushq %rbp
 	movq %rsp, %rbp
 
diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
deleted file mode 100644
index 94941c5a10ac..000000000000
--- a/arch/x86/entry/common.c
+++ /dev/null
@@ -1,523 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * common.c - C code for kernel entry and exit
- * Copyright (c) 2015 Andrew Lutomirski
- *
- * Based on asm and ptrace code by many authors.  The code here originated
- * in ptrace.c and signal.c.
- */
-
-#include <linux/kernel.h>
-#include <linux/sched.h>
-#include <linux/sched/task_stack.h>
-#include <linux/entry-common.h>
-#include <linux/mm.h>
-#include <linux/smp.h>
-#include <linux/errno.h>
-#include <linux/ptrace.h>
-#include <linux/export.h>
-#include <linux/nospec.h>
-#include <linux/syscalls.h>
-#include <linux/uaccess.h>
-#include <linux/init.h>
-
-#ifdef CONFIG_XEN_PV
-#include <xen/xen-ops.h>
-#include <xen/events.h>
-#endif
-
-#include <asm/apic.h>
-#include <asm/desc.h>
-#include <asm/traps.h>
-#include <asm/vdso.h>
-#include <asm/cpufeature.h>
-#include <asm/fpu/api.h>
-#include <asm/nospec-branch.h>
-#include <asm/io_bitmap.h>
-#include <asm/syscall.h>
-#include <asm/irq_stack.h>
-
-#ifdef CONFIG_X86_64
-
-static __always_inline bool do_syscall_x64(struct pt_regs *regs, int nr)
-{
-	/*
-	 * Convert negative numbers to very high and thus out of range
-	 * numbers for comparisons.
-	 */
-	unsigned int unr = nr;
-
-	if (likely(unr < NR_syscalls)) {
-		unr = array_index_nospec(unr, NR_syscalls);
-		regs->ax = x64_sys_call(regs, unr);
-		return true;
-	}
-	return false;
-}
-
-static __always_inline bool do_syscall_x32(struct pt_regs *regs, int nr)
-{
-	/*
-	 * Adjust the starting offset of the table, and convert numbers
-	 * < __X32_SYSCALL_BIT to very high and thus out of range
-	 * numbers for comparisons.
-	 */
-	unsigned int xnr = nr - __X32_SYSCALL_BIT;
-
-	if (IS_ENABLED(CONFIG_X86_X32_ABI) && likely(xnr < X32_NR_syscalls)) {
-		xnr = array_index_nospec(xnr, X32_NR_syscalls);
-		regs->ax = x32_sys_call(regs, xnr);
-		return true;
-	}
-	return false;
-}
-
-/* Returns true to return using SYSRET, or false to use IRET */
-__visible noinstr bool do_syscall_64(struct pt_regs *regs, int nr)
-{
-	add_random_kstack_offset();
-	nr = syscall_enter_from_user_mode(regs, nr);
-
-	instrumentation_begin();
-
-	if (!do_syscall_x64(regs, nr) && !do_syscall_x32(regs, nr) && nr != -1) {
-		/* Invalid system call, but still a system call. */
-		regs->ax = __x64_sys_ni_syscall(regs);
-	}
-
-	instrumentation_end();
-	syscall_exit_to_user_mode(regs);
-
-	/*
-	 * Check that the register state is valid for using SYSRET to exit
-	 * to userspace.  Otherwise use the slower but fully capable IRET
-	 * exit path.
-	 */
-
-	/* XEN PV guests always use the IRET path */
-	if (cpu_feature_enabled(X86_FEATURE_XENPV))
-		return false;
-
-	/* SYSRET requires RCX == RIP and R11 == EFLAGS */
-	if (unlikely(regs->cx != regs->ip || regs->r11 != regs->flags))
-		return false;
-
-	/* CS and SS must match the values set in MSR_STAR */
-	if (unlikely(regs->cs != __USER_CS || regs->ss != __USER_DS))
-		return false;
-
-	/*
-	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
-	 * in kernel space.  This essentially lets the user take over
-	 * the kernel, since userspace controls RSP.
-	 *
-	 * TASK_SIZE_MAX covers all user-accessible addresses other than
-	 * the deprecated vsyscall page.
-	 */
-	if (unlikely(regs->ip >= TASK_SIZE_MAX))
-		return false;
-
-	/*
-	 * SYSRET cannot restore RF.  It can restore TF, but unlike IRET,
-	 * restoring TF results in a trap from userspace immediately after
-	 * SYSRET.
-	 */
-	if (unlikely(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF)))
-		return false;
-
-	/* Use SYSRET to exit to userspace */
-	return true;
-}
-#endif
-
-#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
-static __always_inline int syscall_32_enter(struct pt_regs *regs)
-{
-	if (IS_ENABLED(CONFIG_IA32_EMULATION))
-		current_thread_info()->status |= TS_COMPAT;
-
-	return (int)regs->orig_ax;
-}
-
-#ifdef CONFIG_IA32_EMULATION
-bool __ia32_enabled __ro_after_init = !IS_ENABLED(CONFIG_IA32_EMULATION_DEFAULT_DISABLED);
-
-static int ia32_emulation_override_cmdline(char *arg)
-{
-	return kstrtobool(arg, &__ia32_enabled);
-}
-early_param("ia32_emulation", ia32_emulation_override_cmdline);
-#endif
-
-/*
- * Invoke a 32-bit syscall.  Called with IRQs on in CT_STATE_KERNEL.
- */
-static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs, int nr)
-{
-	/*
-	 * Convert negative numbers to very high and thus out of range
-	 * numbers for comparisons.
-	 */
-	unsigned int unr = nr;
-
-	if (likely(unr < IA32_NR_syscalls)) {
-		unr = array_index_nospec(unr, IA32_NR_syscalls);
-		regs->ax = ia32_sys_call(regs, unr);
-	} else if (nr != -1) {
-		regs->ax = __ia32_sys_ni_syscall(regs);
-	}
-}
-
-#ifdef CONFIG_IA32_EMULATION
-static __always_inline bool int80_is_external(void)
-{
-	const unsigned int offs = (0x80 / 32) * 0x10;
-	const u32 bit = BIT(0x80 % 32);
-
-	/* The local APIC on XENPV guests is fake */
-	if (cpu_feature_enabled(X86_FEATURE_XENPV))
-		return false;
-
-	/*
-	 * If vector 0x80 is set in the APIC ISR then this is an external
-	 * interrupt. Either from broken hardware or injected by a VMM.
-	 *
-	 * Note: In guest mode this is only valid for secure guests where
-	 * the secure module fully controls the vAPIC exposed to the guest.
-	 */
-	return apic_read(APIC_ISR + offs) & bit;
-}
-
-/**
- * do_int80_emulation - 32-bit legacy syscall C entry from asm
- *
- * This entry point can be used by 32-bit and 64-bit programs to perform
- * 32-bit system calls.  Instances of INT $0x80 can be found inline in
- * various programs and libraries.  It is also used by the vDSO's
- * __kernel_vsyscall fallback for hardware that doesn't support a faster
- * entry method.  Restarted 32-bit system calls also fall back to INT
- * $0x80 regardless of what instruction was originally used to do the
- * system call.
- *
- * This is considered a slow path.  It is not used by most libc
- * implementations on modern hardware except during process startup.
- *
- * The arguments for the INT $0x80 based syscall are on stack in the
- * pt_regs structure:
- *   eax:				system call number
- *   ebx, ecx, edx, esi, edi, ebp:	arg1 - arg 6
- */
-__visible noinstr void do_int80_emulation(struct pt_regs *regs)
-{
-	int nr;
-
-	/* Kernel does not use INT $0x80! */
-	if (unlikely(!user_mode(regs))) {
-		irqentry_enter(regs);
-		instrumentation_begin();
-		panic("Unexpected external interrupt 0x80\n");
-	}
-
-	/*
-	 * Establish kernel context for instrumentation, including for
-	 * int80_is_external() below which calls into the APIC driver.
-	 * Identical for soft and external interrupts.
-	 */
-	enter_from_user_mode(regs);
-
-	instrumentation_begin();
-	add_random_kstack_offset();
-
-	/* Validate that this is a soft interrupt to the extent possible */
-	if (unlikely(int80_is_external()))
-		panic("Unexpected external interrupt 0x80\n");
-
-	/*
-	 * The low level idtentry code pushed -1 into regs::orig_ax
-	 * and regs::ax contains the syscall number.
-	 *
-	 * User tracing code (ptrace or signal handlers) might assume
-	 * that the regs::orig_ax contains a 32-bit number on invoking
-	 * a 32-bit syscall.
-	 *
-	 * Establish the syscall convention by saving the 32bit truncated
-	 * syscall number in regs::orig_ax and by invalidating regs::ax.
-	 */
-	regs->orig_ax = regs->ax & GENMASK(31, 0);
-	regs->ax = -ENOSYS;
-
-	nr = syscall_32_enter(regs);
-
-	local_irq_enable();
-	nr = syscall_enter_from_user_mode_work(regs, nr);
-	do_syscall_32_irqs_on(regs, nr);
-
-	instrumentation_end();
-	syscall_exit_to_user_mode(regs);
-}
-
-#ifdef CONFIG_X86_FRED
-/*
- * A FRED-specific INT80 handler is warranted for the follwing reasons:
- *
- * 1) As INT instructions and hardware interrupts are separate event
- *    types, FRED does not preclude the use of vector 0x80 for external
- *    interrupts. As a result, the FRED setup code does not reserve
- *    vector 0x80 and calling int80_is_external() is not merely
- *    suboptimal but actively incorrect: it could cause a system call
- *    to be incorrectly ignored.
- *
- * 2) It is called only for handling vector 0x80 of event type
- *    EVENT_TYPE_SWINT and will never be called to handle any external
- *    interrupt (event type EVENT_TYPE_EXTINT).
- *
- * 3) FRED has separate entry flows depending on if the event came from
- *    user space or kernel space, and because the kernel does not use
- *    INT insns, the FRED kernel entry handler fred_entry_from_kernel()
- *    falls through to fred_bad_type() if the event type is
- *    EVENT_TYPE_SWINT, i.e., INT insns. So if the kernel is handling
- *    an INT insn, it can only be from a user level.
- *
- * 4) int80_emulation() does a CLEAR_BRANCH_HISTORY. While FRED will
- *    likely take a different approach if it is ever needed: it
- *    probably belongs in either fred_intx()/ fred_other() or
- *    asm_fred_entrypoint_user(), depending on if this ought to be done
- *    for all entries from userspace or only system
- *    calls.
- *
- * 5) INT $0x80 is the fast path for 32-bit system calls under FRED.
- */
-DEFINE_FREDENTRY_RAW(int80_emulation)
-{
-	int nr;
-
-	enter_from_user_mode(regs);
-
-	instrumentation_begin();
-	add_random_kstack_offset();
-
-	/*
-	 * FRED pushed 0 into regs::orig_ax and regs::ax contains the
-	 * syscall number.
-	 *
-	 * User tracing code (ptrace or signal handlers) might assume
-	 * that the regs::orig_ax contains a 32-bit number on invoking
-	 * a 32-bit syscall.
-	 *
-	 * Establish the syscall convention by saving the 32bit truncated
-	 * syscall number in regs::orig_ax and by invalidating regs::ax.
-	 */
-	regs->orig_ax = regs->ax & GENMASK(31, 0);
-	regs->ax = -ENOSYS;
-
-	nr = syscall_32_enter(regs);
-
-	local_irq_enable();
-	nr = syscall_enter_from_user_mode_work(regs, nr);
-	do_syscall_32_irqs_on(regs, nr);
-
-	instrumentation_end();
-	syscall_exit_to_user_mode(regs);
-}
-#endif
-#else /* CONFIG_IA32_EMULATION */
-
-/* Handles int $0x80 on a 32bit kernel */
-__visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
-{
-	int nr = syscall_32_enter(regs);
-
-	add_random_kstack_offset();
-	/*
-	 * Subtlety here: if ptrace pokes something larger than 2^31-1 into
-	 * orig_ax, the int return value truncates it. This matches
-	 * the semantics of syscall_get_nr().
-	 */
-	nr = syscall_enter_from_user_mode(regs, nr);
-	instrumentation_begin();
-
-	do_syscall_32_irqs_on(regs, nr);
-
-	instrumentation_end();
-	syscall_exit_to_user_mode(regs);
-}
-#endif /* !CONFIG_IA32_EMULATION */
-
-static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
-{
-	int nr = syscall_32_enter(regs);
-	int res;
-
-	add_random_kstack_offset();
-	/*
-	 * This cannot use syscall_enter_from_user_mode() as it has to
-	 * fetch EBP before invoking any of the syscall entry work
-	 * functions.
-	 */
-	syscall_enter_from_user_mode_prepare(regs);
-
-	instrumentation_begin();
-	/* Fetch EBP from where the vDSO stashed it. */
-	if (IS_ENABLED(CONFIG_X86_64)) {
-		/*
-		 * Micro-optimization: the pointer we're following is
-		 * explicitly 32 bits, so it can't be out of range.
-		 */
-		res = __get_user(*(u32 *)&regs->bp,
-			 (u32 __user __force *)(unsigned long)(u32)regs->sp);
-	} else {
-		res = get_user(*(u32 *)&regs->bp,
-		       (u32 __user __force *)(unsigned long)(u32)regs->sp);
-	}
-
-	if (res) {
-		/* User code screwed up. */
-		regs->ax = -EFAULT;
-
-		local_irq_disable();
-		instrumentation_end();
-		irqentry_exit_to_user_mode(regs);
-		return false;
-	}
-
-	nr = syscall_enter_from_user_mode_work(regs, nr);
-
-	/* Now this is just like a normal syscall. */
-	do_syscall_32_irqs_on(regs, nr);
-
-	instrumentation_end();
-	syscall_exit_to_user_mode(regs);
-	return true;
-}
-
-/* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */
-__visible noinstr bool do_fast_syscall_32(struct pt_regs *regs)
-{
-	/*
-	 * Called using the internal vDSO SYSENTER/SYSCALL32 calling
-	 * convention.  Adjust regs so it looks like we entered using int80.
-	 */
-	unsigned long landing_pad = (unsigned long)current->mm->context.vdso +
-					vdso_image_32.sym_int80_landing_pad;
-
-	/*
-	 * SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward
-	 * so that 'regs->ip -= 2' lands back on an int $0x80 instruction.
-	 * Fix it up.
-	 */
-	regs->ip = landing_pad;
-
-	/* Invoke the syscall. If it failed, keep it simple: use IRET. */
-	if (!__do_fast_syscall_32(regs))
-		return false;
-
-	/*
-	 * Check that the register state is valid for using SYSRETL/SYSEXIT
-	 * to exit to userspace.  Otherwise use the slower but fully capable
-	 * IRET exit path.
-	 */
-
-	/* XEN PV guests always use the IRET path */
-	if (cpu_feature_enabled(X86_FEATURE_XENPV))
-		return false;
-
-	/* EIP must point to the VDSO landing pad */
-	if (unlikely(regs->ip != landing_pad))
-		return false;
-
-	/* CS and SS must match the values set in MSR_STAR */
-	if (unlikely(regs->cs != __USER32_CS || regs->ss != __USER_DS))
-		return false;
-
-	/* If the TF, RF, or VM flags are set, use IRET */
-	if (unlikely(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF | X86_EFLAGS_VM)))
-		return false;
-
-	/* Use SYSRETL/SYSEXIT to exit to userspace */
-	return true;
-}
-
-/* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */
-__visible noinstr bool do_SYSENTER_32(struct pt_regs *regs)
-{
-	/* SYSENTER loses RSP, but the vDSO saved it in RBP. */
-	regs->sp = regs->bp;
-
-	/* SYSENTER clobbers EFLAGS.IF.  Assume it was set in usermode. */
-	regs->flags |= X86_EFLAGS_IF;
-
-	return do_fast_syscall_32(regs);
-}
-#endif
-
-SYSCALL_DEFINE0(ni_syscall)
-{
-	return -ENOSYS;
-}
-
-#ifdef CONFIG_XEN_PV
-#ifndef CONFIG_PREEMPTION
-/*
- * Some hypercalls issued by the toolstack can take many 10s of
- * seconds. Allow tasks running hypercalls via the privcmd driver to
- * be voluntarily preempted even if full kernel preemption is
- * disabled.
- *
- * Such preemptible hypercalls are bracketed by
- * xen_preemptible_hcall_begin() and xen_preemptible_hcall_end()
- * calls.
- */
-DEFINE_PER_CPU(bool, xen_in_preemptible_hcall);
-EXPORT_SYMBOL_GPL(xen_in_preemptible_hcall);
-
-/*
- * In case of scheduling the flag must be cleared and restored after
- * returning from schedule as the task might move to a different CPU.
- */
-static __always_inline bool get_and_clear_inhcall(void)
-{
-	bool inhcall = __this_cpu_read(xen_in_preemptible_hcall);
-
-	__this_cpu_write(xen_in_preemptible_hcall, false);
-	return inhcall;
-}
-
-static __always_inline void restore_inhcall(bool inhcall)
-{
-	__this_cpu_write(xen_in_preemptible_hcall, inhcall);
-}
-#else
-static __always_inline bool get_and_clear_inhcall(void) { return false; }
-static __always_inline void restore_inhcall(bool inhcall) { }
-#endif
-
-static void __xen_pv_evtchn_do_upcall(struct pt_regs *regs)
-{
-	struct pt_regs *old_regs = set_irq_regs(regs);
-
-	inc_irq_stat(irq_hv_callback_count);
-
-	xen_evtchn_do_upcall();
-
-	set_irq_regs(old_regs);
-}
-
-__visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs)
-{
-	irqentry_state_t state = irqentry_enter(regs);
-	bool inhcall;
-
-	instrumentation_begin();
-	run_sysvec_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs);
-
-	inhcall = get_and_clear_inhcall();
-	if (inhcall && !WARN_ON_ONCE(state.exit_rcu)) {
-		irqentry_exit_cond_resched();
-		instrumentation_end();
-		restore_inhcall(inhcall);
-	} else {
-		instrumentation_end();
-		irqentry_exit(regs, state);
-	}
-}
-#endif /* CONFIG_XEN_PV */
diff --git a/arch/x86/entry/entry.S b/arch/x86/entry/entry.S
index 324686bca368..6ba2b3adcef0 100644
--- a/arch/x86/entry/entry.S
+++ b/arch/x86/entry/entry.S
@@ -4,7 +4,9 @@
  */
 
 #include <linux/export.h>
+#include <linux/kvm_types.h>
 #include <linux/linkage.h>
+#include <linux/objtool.h>
 #include <asm/msr-index.h>
 #include <asm/unwind_hints.h>
 #include <asm/segment.h>
@@ -16,38 +18,60 @@
 
 .pushsection .noinstr.text, "ax"
 
-SYM_FUNC_START(entry_ibpb)
+/* Clobbers AX, CX, DX */
+SYM_FUNC_START(write_ibpb)
+	ANNOTATE_NOENDBR
 	movl	$MSR_IA32_PRED_CMD, %ecx
-	movl	$PRED_CMD_IBPB, %eax
+	movl	_ASM_RIP(x86_pred_cmd), %eax
 	xorl	%edx, %edx
 	wrmsr
 
 	/* Make sure IBPB clears return stack preductions too. */
 	FILL_RETURN_BUFFER %rax, RSB_CLEAR_LOOPS, X86_BUG_IBPB_NO_RET
 	RET
-SYM_FUNC_END(entry_ibpb)
-/* For KVM */
-EXPORT_SYMBOL_GPL(entry_ibpb);
+SYM_FUNC_END(write_ibpb)
+EXPORT_SYMBOL_FOR_KVM(write_ibpb);
+
+SYM_FUNC_START(__WARN_trap)
+	ANNOTATE_NOENDBR
+	ANNOTATE_REACHABLE
+	ud1 (%edx), %_ASM_ARG1
+	RET
+SYM_FUNC_END(__WARN_trap)
+EXPORT_SYMBOL(__WARN_trap)
 
 .popsection
 
 /*
  * Define the VERW operand that is disguised as entry code so that
- * it can be referenced with KPTI enabled. This ensure VERW can be
+ * it can be referenced with KPTI enabled. This ensures VERW can be
  * used late in exit-to-user path after page tables are switched.
  */
 .pushsection .entry.text, "ax"
 
 .align L1_CACHE_BYTES, 0xcc
-SYM_CODE_START_NOALIGN(mds_verw_sel)
+SYM_CODE_START_NOALIGN(x86_verw_sel)
 	UNWIND_HINT_UNDEFINED
 	ANNOTATE_NOENDBR
 	.word __KERNEL_DS
 .align L1_CACHE_BYTES, 0xcc
-SYM_CODE_END(mds_verw_sel);
-/* For KVM */
-EXPORT_SYMBOL_GPL(mds_verw_sel);
+SYM_CODE_END(x86_verw_sel);
+EXPORT_SYMBOL_FOR_KVM(x86_verw_sel);
 
 .popsection
 
 THUNK warn_thunk_thunk, __warn_thunk
+
+/*
+ * Clang's implementation of TLS stack cookies requires the variable in
+ * question to be a TLS variable. If the variable happens to be defined as an
+ * ordinary variable with external linkage in the same compilation unit (which
+ * amounts to the whole of vmlinux with LTO enabled), Clang will drop the
+ * segment register prefix from the references, resulting in broken code. Work
+ * around this by avoiding the symbol used in -mstack-protector-guard-symbol=
+ * entirely in the C code, and use an alias emitted by the linker script
+ * instead.
+ */
+#if defined(CONFIG_STACKPROTECTOR) && defined(CONFIG_SMP)
+EXPORT_SYMBOL(__ref_stack_chk_guard);
+#endif
diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 20be5758c2d2..92c0b4a94e0a 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -1153,7 +1153,7 @@ SYM_CODE_START(asm_exc_nmi)
 	 * is using the thread stack right now, so it's safe for us to use it.
 	 */
 	movl	%esp, %ebx
-	movl	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esp
+	movl	PER_CPU_VAR(cpu_current_top_of_stack), %esp
 	call	exc_nmi
 	movl	%ebx, %esp
 
@@ -1217,7 +1217,7 @@ SYM_CODE_START(rewind_stack_and_make_dead)
 	/* Prevent any naive code from trying to unwind to our caller. */
 	xorl	%ebp, %ebp
 
-	movl	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esi
+	movl	PER_CPU_VAR(cpu_current_top_of_stack), %esi
 	leal	-TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp
 
 	call	make_task_dead
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 1b5be07f8669..f9983a1907bf 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -19,6 +19,7 @@
  * - idtentry:		Define exception entry points.
  */
 #include <linux/export.h>
+#include <linux/kvm_types.h>
 #include <linux/linkage.h>
 #include <asm/segment.h>
 #include <asm/cache.h>
@@ -92,7 +93,7 @@ SYM_CODE_START(entry_SYSCALL_64)
 	/* tss.sp2 is scratch space. */
 	movq	%rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
-	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
+	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 
 SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
 	ANNOTATE_NOENDBR
@@ -175,6 +176,7 @@ SYM_CODE_END(entry_SYSCALL_64)
  */
 .pushsection .text, "ax"
 SYM_FUNC_START(__switch_to_asm)
+	ANNOTATE_NOENDBR
 	/*
 	 * Save callee-saved registers
 	 * This must match the order in inactive_task_frame
@@ -192,7 +194,7 @@ SYM_FUNC_START(__switch_to_asm)
 
 #ifdef CONFIG_STACKPROTECTOR
 	movq	TASK_stack_canary(%rsi), %rbx
-	movq	%rbx, PER_CPU_VAR(fixed_percpu_data + FIXED_stack_canary)
+	movq	%rbx, PER_CPU_VAR(__stack_chk_guard)
 #endif
 
 	/*
@@ -308,10 +310,9 @@ SYM_CODE_END(xen_error_entry)
 		movq	$-1, ORIG_RAX(%rsp)	/* no syscall to restart */
 	.endif
 
-	call	\cfunc
-
 	/* For some configurations \cfunc ends up being a noreturn. */
-	REACHABLE
+	ANNOTATE_REACHABLE
+	call	\cfunc
 
 	jmp	error_return
 .endm
@@ -529,10 +530,10 @@ SYM_CODE_START(\asmsym)
 	movq	%rsp, %rdi		/* pt_regs pointer into first argument */
 	movq	ORIG_RAX(%rsp), %rsi	/* get error code into 2nd argument*/
 	movq	$-1, ORIG_RAX(%rsp)	/* no syscall to restart */
-	call	\cfunc
 
 	/* For some configurations \cfunc ends up being a noreturn. */
-	REACHABLE
+	ANNOTATE_REACHABLE
+	call	\cfunc
 
 	jmp	paranoid_exit
 
@@ -743,6 +744,7 @@ _ASM_NOKPROBE(common_interrupt_return)
  * Is in entry.text as it shouldn't be instrumented.
  */
 SYM_FUNC_START(asm_load_gs_index)
+	ANNOTATE_NOENDBR
 	FRAME_BEGIN
 	swapgs
 .Lgs_change:
@@ -1167,7 +1169,7 @@ SYM_CODE_START(asm_exc_nmi)
 	FENCE_SWAPGS_USER_ENTRY
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
 	movq	%rsp, %rdx
-	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
+	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 	UNWIND_HINT_IRET_REGS base=%rdx offset=8
 	pushq	5*8(%rdx)	/* pt_regs->ss */
 	pushq	4*8(%rdx)	/* pt_regs->rsp */
@@ -1485,7 +1487,7 @@ SYM_CODE_START_NOALIGN(rewind_stack_and_make_dead)
 	/* Prevent any naive code from trying to unwind to our caller. */
 	xorl	%ebp, %ebp
 
-	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rax
+	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rax
 	leaq	-PTREGS_SIZE(%rax), %rsp
 	UNWIND_HINT_REGS
 
@@ -1524,9 +1526,12 @@ SYM_CODE_END(rewind_stack_and_make_dead)
  * ORC to unwind properly.
  *
  * The alignment is for performance and not for safety, and may be safely
- * refactored in the future if needed.
+ * refactored in the future if needed. The .skips are for safety, to ensure
+ * that all RETs are in the second half of a cacheline to mitigate Indirect
+ * Target Selection, rather than taking the slowpath via its_return_thunk.
  */
 SYM_FUNC_START(clear_bhb_loop)
+	ANNOTATE_NOENDBR
 	push	%rbp
 	mov	%rsp, %rbp
 	movl	$5, %ecx
@@ -1534,10 +1539,22 @@ SYM_FUNC_START(clear_bhb_loop)
 	call	1f
 	jmp	5f
 	.align 64, 0xcc
+	/*
+	 * Shift instructions so that the RET is in the upper half of the
+	 * cacheline and don't take the slowpath to its_return_thunk.
+	 */
+	.skip 32 - (.Lret1 - 1f), 0xcc
 	ANNOTATE_INTRA_FUNCTION_CALL
 1:	call	2f
-	RET
+.Lret1:	RET
 	.align 64, 0xcc
+	/*
+	 * As above shift instructions for RET at .Lret2 as well.
+	 *
+	 * This should be ideally be: .skip 32 - (.Lret2 - 2f), 0xcc
+	 * but some Clang versions (e.g. 18) don't like this.
+	 */
+	.skip 32 - 18, 0xcc
 2:	movl	$5, %eax
 3:	jmp	4f
 	nop
@@ -1545,10 +1562,10 @@ SYM_FUNC_START(clear_bhb_loop)
 	jnz	3b
 	sub	$1, %ecx
 	jnz	1b
-	RET
+.Lret2:	RET
 5:	lfence
 	pop	%rbp
 	RET
 SYM_FUNC_END(clear_bhb_loop)
-EXPORT_SYMBOL_GPL(clear_bhb_loop)
+EXPORT_SYMBOL_FOR_KVM(clear_bhb_loop)
 STACK_FRAME_NON_STANDARD(clear_bhb_loop)
diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S
index ed0a5f2dc129..a45e1125fc6c 100644
--- a/arch/x86/entry/entry_64_compat.S
+++ b/arch/x86/entry/entry_64_compat.S
@@ -57,7 +57,7 @@ SYM_CODE_START(entry_SYSENTER_compat)
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
 	popq	%rax
 
-	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
+	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 
 	/* Construct struct pt_regs on stack */
 	pushq	$__USER_DS		/* pt_regs->ss */
@@ -193,7 +193,7 @@ SYM_CODE_START(entry_SYSCALL_compat)
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
 
 	/* Switch to the kernel stack */
-	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
+	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 
 SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)
 	ANNOTATE_NOENDBR
diff --git a/arch/x86/entry/entry_64_fred.S b/arch/x86/entry/entry_64_fred.S
index a02bc6f3d2e6..894f7f16eb80 100644
--- a/arch/x86/entry/entry_64_fred.S
+++ b/arch/x86/entry/entry_64_fred.S
@@ -4,6 +4,7 @@
  */
 
 #include <linux/export.h>
+#include <linux/kvm_types.h>
 
 #include <asm/asm.h>
 #include <asm/fred.h>
@@ -16,7 +17,7 @@
 
 .macro FRED_ENTER
 	UNWIND_HINT_END_OF_STACK
-	ENDBR
+	ANNOTATE_NOENDBR
 	PUSH_AND_CLEAR_REGS
 	movq	%rsp, %rdi	/* %rdi -> pt_regs */
 .endm
@@ -58,6 +59,7 @@ SYM_CODE_END(asm_fred_entrypoint_kernel)
 
 #if IS_ENABLED(CONFIG_KVM_INTEL)
 SYM_FUNC_START(asm_fred_entry_from_kvm)
+	ANNOTATE_NOENDBR
 	push %rbp
 	mov %rsp, %rbp
 
@@ -96,8 +98,7 @@ SYM_FUNC_START(asm_fred_entry_from_kvm)
 	push %rdi			/* fred_ss handed in by the caller */
 	push %rbp
 	pushf
-	mov $__KERNEL_CS, %rax
-	push %rax
+	push $__KERNEL_CS
 
 	/*
 	 * Unlike the IDT event delivery, FRED _always_ pushes an error code
@@ -111,21 +112,40 @@ SYM_FUNC_START(asm_fred_entry_from_kvm)
 	push %rax				/* Return RIP */
 	push $0					/* Error code, 0 for IRQ/NMI */
 
-	PUSH_AND_CLEAR_REGS clear_bp=0 unwind_hint=0
+	PUSH_AND_CLEAR_REGS clear_callee=0 unwind_hint=0
+
 	movq %rsp, %rdi				/* %rdi -> pt_regs */
+	/*
+	 * At this point: {rdi, rsi, rdx, rcx, r8, r9}, {r10, r11}, {rax, rdx}
+	 * are clobbered, which corresponds to: arguments, extra caller-saved
+	 * and return. All registers a C function is allowed to clobber.
+	 *
+	 * Notably, the callee-saved registers: {rbx, r12, r13, r14, r15}
+	 * are untouched, with the exception of rbp, which carries the stack
+	 * frame and will be restored before exit.
+	 *
+	 * Further calling another C function will not alter this state.
+	 */
 	call __fred_entry_from_kvm		/* Call the C entry point */
-	POP_REGS
-	ERETS
-1:
+
 	/*
-	 * Objtool doesn't understand what ERETS does, this hint tells it that
-	 * yes, we'll reach here and with what stack state. A save/restore pair
-	 * isn't strictly needed, but it's the simplest form.
+	 * When FRED, use ERETS to potentially clear NMIs, otherwise simply
+	 * restore the stack pointer.
+	 */
+	ALTERNATIVE "nop; nop; mov %rbp, %rsp", \
+	            __stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \
+		    X86_FEATURE_FRED
+
+1:	/*
+	 * Objtool doesn't understand ERETS, and the cfi register state is
+	 * different from initial_func_cfi due to PUSH_REGS. Tell it the state
+	 * is similar to where UNWIND_HINT_SAVE is.
 	 */
 	UNWIND_HINT_RESTORE
+
 	pop %rbp
 	RET
 
 SYM_FUNC_END(asm_fred_entry_from_kvm)
-EXPORT_SYMBOL_GPL(asm_fred_entry_from_kvm);
+EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm);
 #endif
diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c
index f004a4dc74c2..94e626cc6a07 100644
--- a/arch/x86/entry/entry_fred.c
+++ b/arch/x86/entry/entry_fred.c
@@ -78,13 +78,13 @@ static noinstr void fred_intx(struct pt_regs *regs)
 static __always_inline void fred_other(struct pt_regs *regs)
 {
 	/* The compiler can fold these conditions into a single test */
-	if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.lm)) {
+	if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.l)) {
 		regs->orig_ax = regs->ax;
 		regs->ax = -ENOSYS;
 		do_syscall_64(regs, regs->orig_ax);
 		return;
 	} else if (ia32_enabled() &&
-		   likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.lm)) {
+		   likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.l)) {
 		regs->orig_ax = regs->ax;
 		regs->ax = -ENOSYS;
 		do_fast_syscall_32(regs);
diff --git a/arch/x86/entry/syscall_32.c b/arch/x86/entry/syscall_32.c
index 8cc9950d7104..a67a644d0cfe 100644
--- a/arch/x86/entry/syscall_32.c
+++ b/arch/x86/entry/syscall_32.c
@@ -1,10 +1,16 @@
-// SPDX-License-Identifier: GPL-2.0
-/* System call table for i386. */
+// SPDX-License-Identifier: GPL-2.0-only
+/* 32-bit system call dispatch */
 
 #include <linux/linkage.h>
 #include <linux/sys.h>
 #include <linux/cache.h>
 #include <linux/syscalls.h>
+#include <linux/entry-common.h>
+#include <linux/nospec.h>
+#include <linux/uaccess.h>
+#include <asm/apic.h>
+#include <asm/traps.h>
+#include <asm/cpufeature.h>
 #include <asm/syscall.h>
 
 #ifdef CONFIG_IA32_EMULATION
@@ -41,4 +47,325 @@ long ia32_sys_call(const struct pt_regs *regs, unsigned int nr)
 	#include <asm/syscalls_32.h>
 	default: return __ia32_sys_ni_syscall(regs);
 	}
-};
+}
+
+static __always_inline int syscall_32_enter(struct pt_regs *regs)
+{
+	if (IS_ENABLED(CONFIG_IA32_EMULATION))
+		current_thread_info()->status |= TS_COMPAT;
+
+	return (int)regs->orig_ax;
+}
+
+#ifdef CONFIG_IA32_EMULATION
+bool __ia32_enabled __ro_after_init = !IS_ENABLED(CONFIG_IA32_EMULATION_DEFAULT_DISABLED);
+
+static int __init ia32_emulation_override_cmdline(char *arg)
+{
+	return kstrtobool(arg, &__ia32_enabled);
+}
+early_param("ia32_emulation", ia32_emulation_override_cmdline);
+#endif
+
+/*
+ * Invoke a 32-bit syscall.  Called with IRQs on in CT_STATE_KERNEL.
+ */
+static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs, int nr)
+{
+	/*
+	 * Convert negative numbers to very high and thus out of range
+	 * numbers for comparisons.
+	 */
+	unsigned int unr = nr;
+
+	if (likely(unr < IA32_NR_syscalls)) {
+		unr = array_index_nospec(unr, IA32_NR_syscalls);
+		regs->ax = ia32_sys_call(regs, unr);
+	} else if (nr != -1) {
+		regs->ax = __ia32_sys_ni_syscall(regs);
+	}
+}
+
+#ifdef CONFIG_IA32_EMULATION
+static __always_inline bool int80_is_external(void)
+{
+	const unsigned int offs = (0x80 / 32) * 0x10;
+	const u32 bit = BIT(0x80 % 32);
+
+	/* The local APIC on XENPV guests is fake */
+	if (cpu_feature_enabled(X86_FEATURE_XENPV))
+		return false;
+
+	/*
+	 * If vector 0x80 is set in the APIC ISR then this is an external
+	 * interrupt. Either from broken hardware or injected by a VMM.
+	 *
+	 * Note: In guest mode this is only valid for secure guests where
+	 * the secure module fully controls the vAPIC exposed to the guest.
+	 */
+	return apic_read(APIC_ISR + offs) & bit;
+}
+
+/**
+ * do_int80_emulation - 32-bit legacy syscall C entry from asm
+ * @regs: syscall arguments in struct pt_args on the stack.
+ *
+ * This entry point can be used by 32-bit and 64-bit programs to perform
+ * 32-bit system calls.  Instances of INT $0x80 can be found inline in
+ * various programs and libraries.  It is also used by the vDSO's
+ * __kernel_vsyscall fallback for hardware that doesn't support a faster
+ * entry method.  Restarted 32-bit system calls also fall back to INT
+ * $0x80 regardless of what instruction was originally used to do the
+ * system call.
+ *
+ * This is considered a slow path.  It is not used by most libc
+ * implementations on modern hardware except during process startup.
+ *
+ * The arguments for the INT $0x80 based syscall are on stack in the
+ * pt_regs structure:
+ *   eax:				system call number
+ *   ebx, ecx, edx, esi, edi, ebp:	arg1 - arg 6
+ */
+__visible noinstr void do_int80_emulation(struct pt_regs *regs)
+{
+	int nr;
+
+	/* Kernel does not use INT $0x80! */
+	if (unlikely(!user_mode(regs))) {
+		irqentry_enter(regs);
+		instrumentation_begin();
+		panic("Unexpected external interrupt 0x80\n");
+	}
+
+	/*
+	 * Establish kernel context for instrumentation, including for
+	 * int80_is_external() below which calls into the APIC driver.
+	 * Identical for soft and external interrupts.
+	 */
+	enter_from_user_mode(regs);
+
+	instrumentation_begin();
+	add_random_kstack_offset();
+
+	/* Validate that this is a soft interrupt to the extent possible */
+	if (unlikely(int80_is_external()))
+		panic("Unexpected external interrupt 0x80\n");
+
+	/*
+	 * The low level idtentry code pushed -1 into regs::orig_ax
+	 * and regs::ax contains the syscall number.
+	 *
+	 * User tracing code (ptrace or signal handlers) might assume
+	 * that the regs::orig_ax contains a 32-bit number on invoking
+	 * a 32-bit syscall.
+	 *
+	 * Establish the syscall convention by saving the 32bit truncated
+	 * syscall number in regs::orig_ax and by invalidating regs::ax.
+	 */
+	regs->orig_ax = regs->ax & GENMASK(31, 0);
+	regs->ax = -ENOSYS;
+
+	nr = syscall_32_enter(regs);
+
+	local_irq_enable();
+	nr = syscall_enter_from_user_mode_work(regs, nr);
+	do_syscall_32_irqs_on(regs, nr);
+
+	instrumentation_end();
+	syscall_exit_to_user_mode(regs);
+}
+
+#ifdef CONFIG_X86_FRED
+/*
+ * A FRED-specific INT80 handler is warranted for the follwing reasons:
+ *
+ * 1) As INT instructions and hardware interrupts are separate event
+ *    types, FRED does not preclude the use of vector 0x80 for external
+ *    interrupts. As a result, the FRED setup code does not reserve
+ *    vector 0x80 and calling int80_is_external() is not merely
+ *    suboptimal but actively incorrect: it could cause a system call
+ *    to be incorrectly ignored.
+ *
+ * 2) It is called only for handling vector 0x80 of event type
+ *    EVENT_TYPE_SWINT and will never be called to handle any external
+ *    interrupt (event type EVENT_TYPE_EXTINT).
+ *
+ * 3) FRED has separate entry flows depending on if the event came from
+ *    user space or kernel space, and because the kernel does not use
+ *    INT insns, the FRED kernel entry handler fred_entry_from_kernel()
+ *    falls through to fred_bad_type() if the event type is
+ *    EVENT_TYPE_SWINT, i.e., INT insns. So if the kernel is handling
+ *    an INT insn, it can only be from a user level.
+ *
+ * 4) int80_emulation() does a CLEAR_BRANCH_HISTORY. While FRED will
+ *    likely take a different approach if it is ever needed: it
+ *    probably belongs in either fred_intx()/ fred_other() or
+ *    asm_fred_entrypoint_user(), depending on if this ought to be done
+ *    for all entries from userspace or only system
+ *    calls.
+ *
+ * 5) INT $0x80 is the fast path for 32-bit system calls under FRED.
+ */
+DEFINE_FREDENTRY_RAW(int80_emulation)
+{
+	int nr;
+
+	enter_from_user_mode(regs);
+
+	instrumentation_begin();
+	add_random_kstack_offset();
+
+	/*
+	 * FRED pushed 0 into regs::orig_ax and regs::ax contains the
+	 * syscall number.
+	 *
+	 * User tracing code (ptrace or signal handlers) might assume
+	 * that the regs::orig_ax contains a 32-bit number on invoking
+	 * a 32-bit syscall.
+	 *
+	 * Establish the syscall convention by saving the 32bit truncated
+	 * syscall number in regs::orig_ax and by invalidating regs::ax.
+	 */
+	regs->orig_ax = regs->ax & GENMASK(31, 0);
+	regs->ax = -ENOSYS;
+
+	nr = syscall_32_enter(regs);
+
+	local_irq_enable();
+	nr = syscall_enter_from_user_mode_work(regs, nr);
+	do_syscall_32_irqs_on(regs, nr);
+
+	instrumentation_end();
+	syscall_exit_to_user_mode(regs);
+}
+#endif /* CONFIG_X86_FRED */
+
+#else /* CONFIG_IA32_EMULATION */
+
+/* Handles int $0x80 on a 32bit kernel */
+__visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
+{
+	int nr = syscall_32_enter(regs);
+
+	add_random_kstack_offset();
+	/*
+	 * Subtlety here: if ptrace pokes something larger than 2^31-1 into
+	 * orig_ax, the int return value truncates it. This matches
+	 * the semantics of syscall_get_nr().
+	 */
+	nr = syscall_enter_from_user_mode(regs, nr);
+	instrumentation_begin();
+
+	do_syscall_32_irqs_on(regs, nr);
+
+	instrumentation_end();
+	syscall_exit_to_user_mode(regs);
+}
+#endif /* !CONFIG_IA32_EMULATION */
+
+static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
+{
+	int nr = syscall_32_enter(regs);
+	int res;
+
+	add_random_kstack_offset();
+	/*
+	 * This cannot use syscall_enter_from_user_mode() as it has to
+	 * fetch EBP before invoking any of the syscall entry work
+	 * functions.
+	 */
+	enter_from_user_mode(regs);
+
+	instrumentation_begin();
+	local_irq_enable();
+	/* Fetch EBP from where the vDSO stashed it. */
+	if (IS_ENABLED(CONFIG_X86_64)) {
+		/*
+		 * Micro-optimization: the pointer we're following is
+		 * explicitly 32 bits, so it can't be out of range.
+		 */
+		res = __get_user(*(u32 *)&regs->bp,
+			 (u32 __user __force *)(unsigned long)(u32)regs->sp);
+	} else {
+		res = get_user(*(u32 *)&regs->bp,
+		       (u32 __user __force *)(unsigned long)(u32)regs->sp);
+	}
+
+	if (res) {
+		/* User code screwed up. */
+		regs->ax = -EFAULT;
+
+		local_irq_disable();
+		instrumentation_end();
+		irqentry_exit_to_user_mode(regs);
+		return false;
+	}
+
+	nr = syscall_enter_from_user_mode_work(regs, nr);
+
+	/* Now this is just like a normal syscall. */
+	do_syscall_32_irqs_on(regs, nr);
+
+	instrumentation_end();
+	syscall_exit_to_user_mode(regs);
+	return true;
+}
+
+/* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */
+__visible noinstr bool do_fast_syscall_32(struct pt_regs *regs)
+{
+	/*
+	 * Called using the internal vDSO SYSENTER/SYSCALL32 calling
+	 * convention.  Adjust regs so it looks like we entered using int80.
+	 */
+	unsigned long landing_pad = (unsigned long)current->mm->context.vdso +
+					vdso_image_32.sym_int80_landing_pad;
+
+	/*
+	 * SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward
+	 * so that 'regs->ip -= 2' lands back on an int $0x80 instruction.
+	 * Fix it up.
+	 */
+	regs->ip = landing_pad;
+
+	/* Invoke the syscall. If it failed, keep it simple: use IRET. */
+	if (!__do_fast_syscall_32(regs))
+		return false;
+
+	/*
+	 * Check that the register state is valid for using SYSRETL/SYSEXIT
+	 * to exit to userspace.  Otherwise use the slower but fully capable
+	 * IRET exit path.
+	 */
+
+	/* XEN PV guests always use the IRET path */
+	if (cpu_feature_enabled(X86_FEATURE_XENPV))
+		return false;
+
+	/* EIP must point to the VDSO landing pad */
+	if (unlikely(regs->ip != landing_pad))
+		return false;
+
+	/* CS and SS must match the values set in MSR_STAR */
+	if (unlikely(regs->cs != __USER32_CS || regs->ss != __USER_DS))
+		return false;
+
+	/* If the TF, RF, or VM flags are set, use IRET */
+	if (unlikely(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF | X86_EFLAGS_VM)))
+		return false;
+
+	/* Use SYSRETL/SYSEXIT to exit to userspace */
+	return true;
+}
+
+/* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */
+__visible noinstr bool do_SYSENTER_32(struct pt_regs *regs)
+{
+	/* SYSENTER loses RSP, but the vDSO saved it in RBP. */
+	regs->sp = regs->bp;
+
+	/* SYSENTER clobbers EFLAGS.IF.  Assume it was set in usermode. */
+	regs->flags |= X86_EFLAGS_IF;
+
+	return do_fast_syscall_32(regs);
+}
diff --git a/arch/x86/entry/syscall_64.c b/arch/x86/entry/syscall_64.c
index ba8354424860..b6e68ea98b83 100644
--- a/arch/x86/entry/syscall_64.c
+++ b/arch/x86/entry/syscall_64.c
@@ -1,15 +1,20 @@
-// SPDX-License-Identifier: GPL-2.0
-/* System call table for x86-64. */
+// SPDX-License-Identifier: GPL-2.0-only
+/* 64-bit system call dispatch */
 
 #include <linux/linkage.h>
 #include <linux/sys.h>
 #include <linux/cache.h>
 #include <linux/syscalls.h>
+#include <linux/entry-common.h>
+#include <linux/nospec.h>
 #include <asm/syscall.h>
 
 #define __SYSCALL(nr, sym) extern long __x64_##sym(const struct pt_regs *);
 #define __SYSCALL_NORETURN(nr, sym) extern long __noreturn __x64_##sym(const struct pt_regs *);
 #include <asm/syscalls_64.h>
+#ifdef CONFIG_X86_X32_ABI
+#include <asm/syscalls_x32.h>
+#endif
 #undef  __SYSCALL
 
 #undef  __SYSCALL_NORETURN
@@ -33,4 +38,104 @@ long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
 	#include <asm/syscalls_64.h>
 	default: return __x64_sys_ni_syscall(regs);
 	}
-};
+}
+
+#ifdef CONFIG_X86_X32_ABI
+long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
+{
+	switch (nr) {
+	#include <asm/syscalls_x32.h>
+	default: return __x64_sys_ni_syscall(regs);
+	}
+}
+#endif
+
+static __always_inline bool do_syscall_x64(struct pt_regs *regs, int nr)
+{
+	/*
+	 * Convert negative numbers to very high and thus out of range
+	 * numbers for comparisons.
+	 */
+	unsigned int unr = nr;
+
+	if (likely(unr < NR_syscalls)) {
+		unr = array_index_nospec(unr, NR_syscalls);
+		regs->ax = x64_sys_call(regs, unr);
+		return true;
+	}
+	return false;
+}
+
+static __always_inline bool do_syscall_x32(struct pt_regs *regs, int nr)
+{
+	/*
+	 * Adjust the starting offset of the table, and convert numbers
+	 * < __X32_SYSCALL_BIT to very high and thus out of range
+	 * numbers for comparisons.
+	 */
+	unsigned int xnr = nr - __X32_SYSCALL_BIT;
+
+	if (IS_ENABLED(CONFIG_X86_X32_ABI) && likely(xnr < X32_NR_syscalls)) {
+		xnr = array_index_nospec(xnr, X32_NR_syscalls);
+		regs->ax = x32_sys_call(regs, xnr);
+		return true;
+	}
+	return false;
+}
+
+/* Returns true to return using SYSRET, or false to use IRET */
+__visible noinstr bool do_syscall_64(struct pt_regs *regs, int nr)
+{
+	add_random_kstack_offset();
+	nr = syscall_enter_from_user_mode(regs, nr);
+
+	instrumentation_begin();
+
+	if (!do_syscall_x64(regs, nr) && !do_syscall_x32(regs, nr) && nr != -1) {
+		/* Invalid system call, but still a system call. */
+		regs->ax = __x64_sys_ni_syscall(regs);
+	}
+
+	instrumentation_end();
+	syscall_exit_to_user_mode(regs);
+
+	/*
+	 * Check that the register state is valid for using SYSRET to exit
+	 * to userspace.  Otherwise use the slower but fully capable IRET
+	 * exit path.
+	 */
+
+	/* XEN PV guests always use the IRET path */
+	if (cpu_feature_enabled(X86_FEATURE_XENPV))
+		return false;
+
+	/* SYSRET requires RCX == RIP and R11 == EFLAGS */
+	if (unlikely(regs->cx != regs->ip || regs->r11 != regs->flags))
+		return false;
+
+	/* CS and SS must match the values set in MSR_STAR */
+	if (unlikely(regs->cs != __USER_CS || regs->ss != __USER_DS))
+		return false;
+
+	/*
+	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
+	 * in kernel space.  This essentially lets the user take over
+	 * the kernel, since userspace controls RSP.
+	 *
+	 * TASK_SIZE_MAX covers all user-accessible addresses other than
+	 * the deprecated vsyscall page.
+	 */
+	if (unlikely(regs->ip >= TASK_SIZE_MAX))
+		return false;
+
+	/*
+	 * SYSRET cannot restore RF.  It can restore TF, but unlike IRET,
+	 * restoring TF results in a trap from userspace immediately after
+	 * SYSRET.
+	 */
+	if (unlikely(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF)))
+		return false;
+
+	/* Use SYSRET to exit to userspace */
+	return true;
+}
diff --git a/arch/x86/entry/syscall_x32.c b/arch/x86/entry/syscall_x32.c
deleted file mode 100644
index fb77908f44f3..000000000000
--- a/arch/x86/entry/syscall_x32.c
+++ /dev/null
@@ -1,25 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* System call table for x32 ABI. */
-
-#include <linux/linkage.h>
-#include <linux/sys.h>
-#include <linux/cache.h>
-#include <linux/syscalls.h>
-#include <asm/syscall.h>
-
-#define __SYSCALL(nr, sym) extern long __x64_##sym(const struct pt_regs *);
-#define __SYSCALL_NORETURN(nr, sym) extern long __noreturn __x64_##sym(const struct pt_regs *);
-#include <asm/syscalls_x32.h>
-#undef  __SYSCALL
-
-#undef  __SYSCALL_NORETURN
-#define __SYSCALL_NORETURN __SYSCALL
-
-#define __SYSCALL(nr, sym) case nr: return __x64_##sym(regs);
-long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
-{
-	switch (nr) {
-	#include <asm/syscalls_x32.h>
-	default: return __x64_sys_ni_syscall(regs);
-	}
-};
diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl
index 534c74b14fab..e979a3eac7a3 100644
--- a/arch/x86/entry/syscalls/syscall_32.tbl
+++ b/arch/x86/entry/syscalls/syscall_32.tbl
@@ -396,7 +396,7 @@
 381	i386	pkey_alloc		sys_pkey_alloc
 382	i386	pkey_free		sys_pkey_free
 383	i386	statx			sys_statx
-384	i386	arch_prctl		sys_arch_prctl			compat_sys_arch_prctl
+384	i386	arch_prctl		sys_arch_prctl
 385	i386	io_pgetevents		sys_io_pgetevents_time32	compat_sys_io_pgetevents
 386	i386	rseq			sys_rseq
 393	i386	semget			sys_semget
@@ -468,3 +468,11 @@
 460	i386	lsm_set_self_attr	sys_lsm_set_self_attr
 461	i386	lsm_list_modules	sys_lsm_list_modules
 462	i386	mseal 			sys_mseal
+463	i386	setxattrat		sys_setxattrat
+464	i386	getxattrat		sys_getxattrat
+465	i386	listxattrat		sys_listxattrat
+466	i386	removexattrat		sys_removexattrat
+467	i386	open_tree_attr		sys_open_tree_attr
+468	i386	file_getattr		sys_file_getattr
+469	i386	file_setattr		sys_file_setattr
+470	i386	listns			sys_listns
diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl
index 7093ee21c0d1..8a4ac4841be6 100644
--- a/arch/x86/entry/syscalls/syscall_64.tbl
+++ b/arch/x86/entry/syscalls/syscall_64.tbl
@@ -345,6 +345,7 @@
 333	common	io_pgetevents		sys_io_pgetevents
 334	common	rseq			sys_rseq
 335	common	uretprobe		sys_uretprobe
+336	common	uprobe			sys_uprobe
 # don't use numbers 387 through 423, add new calls after the last
 # 'common' entry
 424	common	pidfd_send_signal	sys_pidfd_send_signal
@@ -386,6 +387,14 @@
 460	common	lsm_set_self_attr	sys_lsm_set_self_attr
 461	common	lsm_list_modules	sys_lsm_list_modules
 462 	common  mseal			sys_mseal
+463	common	setxattrat		sys_setxattrat
+464	common	getxattrat		sys_getxattrat
+465	common	listxattrat		sys_listxattrat
+466	common	removexattrat		sys_removexattrat
+467	common	open_tree_attr		sys_open_tree_attr
+468	common	file_getattr		sys_file_getattr
+469	common	file_setattr		sys_file_setattr
+470	common	listns			sys_listns
 
 #
 # Due to a historical design error, certain syscalls are numbered differently
diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile
index c9216ac4fb1e..f247f5f5cb44 100644
--- a/arch/x86/entry/vdso/Makefile
+++ b/arch/x86/entry/vdso/Makefile
@@ -4,7 +4,7 @@
 #
 
 # Include the generic Makefile to check the built vDSO:
-include $(srctree)/lib/vdso/Makefile
+include $(srctree)/lib/vdso/Makefile.include
 
 # Files to link into the vDSO:
 vobjs-y := vdso-note.o vclock_gettime.o vgetcpu.o vgetrandom.o vgetrandom-chacha.o
@@ -32,7 +32,7 @@ targets += $(foreach x, 64 x32 32, vdso-image-$(x).c vdso$(x).so vdso$(x).so.dbg
 
 CPPFLAGS_vdso.lds += -P -C
 
-VDSO_LDFLAGS_vdso.lds = -m elf_x86_64 -soname linux-vdso.so.1 --no-undefined \
+VDSO_LDFLAGS_vdso.lds = -m elf_x86_64 -soname linux-vdso.so.1 \
 			-z max-page-size=4096
 
 $(obj)/vdso64.so.dbg: $(obj)/vdso.lds $(vobjs) FORCE
@@ -62,7 +62,7 @@ ifneq ($(RETPOLINE_VDSO_CFLAGS),)
 endif
 endif
 
-$(vobjs): KBUILD_CFLAGS := $(filter-out $(PADDING_CFLAGS) $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)
+$(vobjs): KBUILD_CFLAGS := $(filter-out $(PADDING_CFLAGS) $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) $(RANDSTRUCT_CFLAGS) $(KSTACK_ERASE_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)
 $(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO
 
 #
@@ -123,6 +123,7 @@ KBUILD_CFLAGS_32 := $(filter-out -mcmodel=kernel,$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 := $(filter-out -fno-pic,$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 := $(filter-out -mfentry,$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 := $(filter-out $(RANDSTRUCT_CFLAGS),$(KBUILD_CFLAGS_32))
+KBUILD_CFLAGS_32 := $(filter-out $(KSTACK_ERASE_CFLAGS),$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 := $(filter-out $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS_32))
@@ -133,6 +134,7 @@ KBUILD_CFLAGS_32 += -fno-stack-protector
 KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls)
 KBUILD_CFLAGS_32 += -fno-omit-frame-pointer
 KBUILD_CFLAGS_32 += -DDISABLE_BRANCH_PROFILING
+KBUILD_CFLAGS_32 += -DBUILD_VDSO
 
 ifdef CONFIG_MITIGATION_RETPOLINE
 ifneq ($(RETPOLINE_VDSO_CFLAGS),)
@@ -151,10 +153,9 @@ $(obj)/vdso32.so.dbg: $(obj)/vdso32/vdso32.lds $(vobjs32) FORCE
 quiet_cmd_vdso = VDSO    $@
       cmd_vdso = $(LD) -o $@ \
 		       $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \
-		       -T $(filter %.lds,$^) $(filter %.o,$^) && \
-		 sh $(src)/checkundef.sh '$(NM)' '$@'
+		       -T $(filter %.lds,$^) $(filter %.o,$^)
 
-VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 \
+VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 --no-undefined \
 	$(call ld-option, --eh-frame-hdr) -Bsymbolic -z noexecstack
 
 quiet_cmd_vdso_and_check = VDSO    $@
diff --git a/arch/x86/entry/vdso/checkundef.sh b/arch/x86/entry/vdso/checkundef.sh
deleted file mode 100755
index 7ee90a9b549d..000000000000
--- a/arch/x86/entry/vdso/checkundef.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-nm="$1"
-file="$2"
-$nm "$file" | grep '^ *U' > /dev/null 2>&1
-if [ $? -eq 1 ]; then
-    exit 0
-else
-    echo "$file: undefined symbols found" >&2
-    exit 1
-fi
diff --git a/arch/x86/entry/vdso/extable.h b/arch/x86/entry/vdso/extable.h
index b56f6b012941..baba612b832c 100644
--- a/arch/x86/entry/vdso/extable.h
+++ b/arch/x86/entry/vdso/extable.h
@@ -7,7 +7,7 @@
  * vDSO uses a dedicated handler the addresses are relative to the overall
  * exception table, not each individual entry.
  */
-#ifdef __ASSEMBLY__
+#ifdef __ASSEMBLER__
 #define _ASM_VDSO_EXTABLE_HANDLE(from, to)	\
 	ASM_VDSO_EXTABLE_HANDLE from to
 
diff --git a/arch/x86/entry/vdso/vdso-layout.lds.S b/arch/x86/entry/vdso/vdso-layout.lds.S
index bafa73f09e92..ec1ac191a057 100644
--- a/arch/x86/entry/vdso/vdso-layout.lds.S
+++ b/arch/x86/entry/vdso/vdso-layout.lds.S
@@ -1,5 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #include <asm/vdso.h>
+#include <asm/vdso/vsyscall.h>
+#include <vdso/datapage.h>
 
 /*
  * Linker script for vDSO.  This is an ELF shared object prelinked to
@@ -16,23 +18,11 @@ SECTIONS
 	 * segment.
 	 */
 
-	vvar_start = . - 4 * PAGE_SIZE;
-	vvar_page  = vvar_start;
+	VDSO_VVAR_SYMS
 
-	/* Place all vvars at the offsets in asm/vvar.h. */
-#define EMIT_VVAR(name, offset) vvar_ ## name = vvar_page + offset;
-#include <asm/vvar.h>
-#undef EMIT_VVAR
-
-	pvclock_page = vvar_start + PAGE_SIZE;
-	hvclock_page = vvar_start + 2 * PAGE_SIZE;
-	timens_page  = vvar_start + 3 * PAGE_SIZE;
-
-#undef _ASM_X86_VVAR_H
-	/* Place all vvars in timens too at the offsets in asm/vvar.h. */
-#define EMIT_VVAR(name, offset) timens_ ## name = timens_page + offset;
-#include <asm/vvar.h>
-#undef EMIT_VVAR
+	vclock_pages = VDSO_VCLOCK_PAGES_START(vdso_u_data);
+	pvclock_page = vclock_pages + VDSO_PAGE_PVCLOCK_OFFSET * PAGE_SIZE;
+	hvclock_page = vclock_pages + VDSO_PAGE_HVCLOCK_OFFSET * PAGE_SIZE;
 
 	. = SIZEOF_HEADERS;
 
diff --git a/arch/x86/entry/vdso/vdso2c.c b/arch/x86/entry/vdso/vdso2c.c
index 90d15f2a7205..f84e8f8fa5fe 100644
--- a/arch/x86/entry/vdso/vdso2c.c
+++ b/arch/x86/entry/vdso/vdso2c.c
@@ -69,33 +69,12 @@
 
 const char *outfilename;
 
-/* Symbols that we need in vdso2c. */
-enum {
-	sym_vvar_start,
-	sym_vvar_page,
-	sym_pvclock_page,
-	sym_hvclock_page,
-	sym_timens_page,
-};
-
-const int special_pages[] = {
-	sym_vvar_page,
-	sym_pvclock_page,
-	sym_hvclock_page,
-	sym_timens_page,
-};
-
 struct vdso_sym {
 	const char *name;
 	bool export;
 };
 
 struct vdso_sym required_syms[] = {
-	[sym_vvar_start] = {"vvar_start", true},
-	[sym_vvar_page] = {"vvar_page", true},
-	[sym_pvclock_page] = {"pvclock_page", true},
-	[sym_hvclock_page] = {"hvclock_page", true},
-	[sym_timens_page] = {"timens_page", true},
 	{"VDSO32_NOTE_MASK", true},
 	{"__kernel_vsyscall", true},
 	{"__kernel_sigreturn", true},
diff --git a/arch/x86/entry/vdso/vdso2c.h b/arch/x86/entry/vdso/vdso2c.h
index 67b3e37576a6..78ed1c1f28b9 100644
--- a/arch/x86/entry/vdso/vdso2c.h
+++ b/arch/x86/entry/vdso/vdso2c.h
@@ -150,26 +150,6 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len,
 		}
 	}
 
-	/* Validate mapping addresses. */
-	for (i = 0; i < sizeof(special_pages) / sizeof(special_pages[0]); i++) {
-		INT_BITS symval = syms[special_pages[i]];
-
-		if (!symval)
-			continue;  /* The mapping isn't used; ignore it. */
-
-		if (symval % 4096)
-			fail("%s must be a multiple of 4096\n",
-			     required_syms[i].name);
-		if (symval + 4096 < syms[sym_vvar_start])
-			fail("%s underruns vvar_start\n",
-			     required_syms[i].name);
-		if (symval + 4096 > 0)
-			fail("%s is on the wrong side of the vdso text\n",
-			     required_syms[i].name);
-	}
-	if (syms[sym_vvar_start] % 4096)
-		fail("vvar_begin must be a multiple of 4096\n");
-
 	if (!image_name) {
 		fwrite(stripped_addr, stripped_len, 1, outfile);
 		return;
diff --git a/arch/x86/entry/vdso/vdso32-setup.c b/arch/x86/entry/vdso/vdso32-setup.c
index 76e4e74f35b5..8894013eea1d 100644
--- a/arch/x86/entry/vdso/vdso32-setup.c
+++ b/arch/x86/entry/vdso/vdso32-setup.c
@@ -51,15 +51,17 @@ __setup("vdso32=", vdso32_setup);
 __setup_param("vdso=", vdso_setup, vdso32_setup, 0);
 #endif
 
-#ifdef CONFIG_X86_64
 
 #ifdef CONFIG_SYSCTL
-/* Register vsyscall32 into the ABI table */
 #include <linux/sysctl.h>
 
-static struct ctl_table abi_table2[] = {
+static const struct ctl_table vdso_table[] = {
 	{
+#ifdef CONFIG_X86_64
 		.procname	= "vsyscall32",
+#else
+		.procname	= "vdso_enabled",
+#endif
 		.data		= &vdso32_enabled,
 		.maxlen		= sizeof(int),
 		.mode		= 0644,
@@ -71,10 +73,14 @@ static struct ctl_table abi_table2[] = {
 
 static __init int ia32_binfmt_init(void)
 {
-	register_sysctl("abi", abi_table2);
+#ifdef CONFIG_X86_64
+	/* Register vsyscall32 into the ABI table */
+	register_sysctl("abi", vdso_table);
+#else
+	register_sysctl_init("vm", vdso_table);
+#endif
 	return 0;
 }
 __initcall(ia32_binfmt_init);
 #endif /* CONFIG_SYSCTL */
 
-#endif	/* CONFIG_X86_64 */
diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c
index b8fed8b8b9cc..afe105b2f907 100644
--- a/arch/x86/entry/vdso/vma.c
+++ b/arch/x86/entry/vdso/vma.c
@@ -14,32 +14,20 @@
 #include <linux/elf.h>
 #include <linux/cpu.h>
 #include <linux/ptrace.h>
-#include <linux/time_namespace.h>
+#include <linux/vdso_datastore.h>
 
 #include <asm/pvclock.h>
 #include <asm/vgtod.h>
 #include <asm/proto.h>
 #include <asm/vdso.h>
-#include <asm/vvar.h>
 #include <asm/tlb.h>
 #include <asm/page.h>
 #include <asm/desc.h>
 #include <asm/cpufeature.h>
+#include <asm/vdso/vsyscall.h>
 #include <clocksource/hyperv_timer.h>
 
-#undef _ASM_X86_VVAR_H
-#define EMIT_VVAR(name, offset)	\
-	const size_t name ## _offset = offset;
-#include <asm/vvar.h>
-
-struct vdso_data *arch_get_vdso_data(void *vvar_page)
-{
-	return (struct vdso_data *)(vvar_page + _vdso_data_offset);
-}
-#undef EMIT_VVAR
-
-DEFINE_VVAR(struct vdso_data, _vdso_data);
-DEFINE_VVAR_SINGLE(struct vdso_rng_data, _vdso_rng_data);
+static_assert(VDSO_NR_PAGES + VDSO_NR_VCLOCK_PAGES == __VDSO_PAGES);
 
 unsigned int vclocks_used __read_mostly;
 
@@ -59,7 +47,6 @@ int __init init_vdso_image(const struct vdso_image *image)
 	return 0;
 }
 
-static const struct vm_special_mapping vvar_mapping;
 struct linux_binprm;
 
 static vm_fault_t vdso_fault(const struct vm_special_mapping *sm,
@@ -78,7 +65,6 @@ static vm_fault_t vdso_fault(const struct vm_special_mapping *sm,
 static void vdso_fix_landing(const struct vdso_image *image,
 		struct vm_area_struct *new_vma)
 {
-#if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
 	if (in_ia32_syscall() && image == &vdso_image_32) {
 		struct pt_regs *regs = current_pt_regs();
 		unsigned long vdso_land = image->sym_int80_landing_pad;
@@ -89,7 +75,6 @@ static void vdso_fix_landing(const struct vdso_image *image,
 		if (regs->ip == old_land_addr)
 			regs->ip = new_vma->vm_start + vdso_land;
 	}
-#endif
 }
 
 static int vdso_mremap(const struct vm_special_mapping *sm,
@@ -103,106 +88,32 @@ static int vdso_mremap(const struct vm_special_mapping *sm,
 	return 0;
 }
 
-#ifdef CONFIG_TIME_NS
-/*
- * The vvar page layout depends on whether a task belongs to the root or
- * non-root time namespace. Whenever a task changes its namespace, the VVAR
- * page tables are cleared and then they will re-faulted with a
- * corresponding layout.
- * See also the comment near timens_setup_vdso_data() for details.
- */
-int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
-{
-	struct mm_struct *mm = task->mm;
-	struct vm_area_struct *vma;
-	VMA_ITERATOR(vmi, mm, 0);
-
-	mmap_read_lock(mm);
-	for_each_vma(vmi, vma) {
-		if (vma_is_special_mapping(vma, &vvar_mapping))
-			zap_vma_pages(vma);
-	}
-	mmap_read_unlock(mm);
-
-	return 0;
-}
-#endif
-
-static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
-		      struct vm_area_struct *vma, struct vm_fault *vmf)
+static vm_fault_t vvar_vclock_fault(const struct vm_special_mapping *sm,
+				    struct vm_area_struct *vma, struct vm_fault *vmf)
 {
-	const struct vdso_image *image = vma->vm_mm->context.vdso_image;
-	unsigned long pfn;
-	long sym_offset;
-
-	if (!image)
-		return VM_FAULT_SIGBUS;
-
-	sym_offset = (long)(vmf->pgoff << PAGE_SHIFT) +
-		image->sym_vvar_start;
-
-	/*
-	 * Sanity check: a symbol offset of zero means that the page
-	 * does not exist for this vdso image, not that the page is at
-	 * offset zero relative to the text mapping.  This should be
-	 * impossible here, because sym_offset should only be zero for
-	 * the page past the end of the vvar mapping.
-	 */
-	if (sym_offset == 0)
-		return VM_FAULT_SIGBUS;
-
-	if (sym_offset == image->sym_vvar_page) {
-		struct page *timens_page = find_timens_vvar_page(vma);
-
-		pfn = __pa_symbol(&__vvar_page) >> PAGE_SHIFT;
-
-		/*
-		 * If a task belongs to a time namespace then a namespace
-		 * specific VVAR is mapped with the sym_vvar_page offset and
-		 * the real VVAR page is mapped with the sym_timens_page
-		 * offset.
-		 * See also the comment near timens_setup_vdso_data().
-		 */
-		if (timens_page) {
-			unsigned long addr;
-			vm_fault_t err;
-
-			/*
-			 * Optimization: inside time namespace pre-fault
-			 * VVAR page too. As on timens page there are only
-			 * offsets for clocks on VVAR, it'll be faulted
-			 * shortly by VDSO code.
-			 */
-			addr = vmf->address + (image->sym_timens_page - sym_offset);
-			err = vmf_insert_pfn(vma, addr, pfn);
-			if (unlikely(err & VM_FAULT_ERROR))
-				return err;
-
-			pfn = page_to_pfn(timens_page);
-		}
-
-		return vmf_insert_pfn(vma, vmf->address, pfn);
-	} else if (sym_offset == image->sym_pvclock_page) {
+	switch (vmf->pgoff) {
+#ifdef CONFIG_PARAVIRT_CLOCK
+	case VDSO_PAGE_PVCLOCK_OFFSET:
+	{
 		struct pvclock_vsyscall_time_info *pvti =
 			pvclock_get_pvti_cpu0_va();
-		if (pvti && vclock_was_used(VDSO_CLOCKMODE_PVCLOCK)) {
+
+		if (pvti && vclock_was_used(VDSO_CLOCKMODE_PVCLOCK))
 			return vmf_insert_pfn_prot(vma, vmf->address,
 					__pa(pvti) >> PAGE_SHIFT,
 					pgprot_decrypted(vma->vm_page_prot));
-		}
-	} else if (sym_offset == image->sym_hvclock_page) {
-		pfn = hv_get_tsc_pfn();
-
+		break;
+	}
+#endif /* CONFIG_PARAVIRT_CLOCK */
+#ifdef CONFIG_HYPERV_TIMER
+	case VDSO_PAGE_HVCLOCK_OFFSET:
+	{
+		unsigned long pfn = hv_get_tsc_pfn();
 		if (pfn && vclock_was_used(VDSO_CLOCKMODE_HVCLOCK))
 			return vmf_insert_pfn(vma, vmf->address, pfn);
-	} else if (sym_offset == image->sym_timens_page) {
-		struct page *timens_page = find_timens_vvar_page(vma);
-
-		if (!timens_page)
-			return VM_FAULT_SIGBUS;
-
-		pfn = __pa_symbol(&__vvar_page) >> PAGE_SHIFT;
-		return vmf_insert_pfn(vma, vmf->address, pfn);
+		break;
+	}
+#endif /* CONFIG_HYPERV_TIMER */
 	}
 
 	return VM_FAULT_SIGBUS;
@@ -213,9 +124,9 @@ static const struct vm_special_mapping vdso_mapping = {
 	.fault = vdso_fault,
 	.mremap = vdso_mremap,
 };
-static const struct vm_special_mapping vvar_mapping = {
-	.name = "[vvar]",
-	.fault = vvar_fault,
+static const struct vm_special_mapping vvar_vclock_mapping = {
+	.name = "[vvar_vclock]",
+	.fault = vvar_vclock_fault,
 };
 
 /*
@@ -234,13 +145,13 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr)
 		return -EINTR;
 
 	addr = get_unmapped_area(NULL, addr,
-				 image->size - image->sym_vvar_start, 0, 0);
+				 image->size + __VDSO_PAGES * PAGE_SIZE, 0, 0);
 	if (IS_ERR_VALUE(addr)) {
 		ret = addr;
 		goto up_fail;
 	}
 
-	text_start = addr - image->sym_vvar_start;
+	text_start = addr + __VDSO_PAGES * PAGE_SIZE;
 
 	/*
 	 * MAYWRITE to allow gdb to COW and set breakpoints
@@ -249,7 +160,8 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr)
 				       text_start,
 				       image->size,
 				       VM_READ|VM_EXEC|
-				       VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
+				       VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC|
+				       VM_SEALED_SYSMAP,
 				       &vdso_mapping);
 
 	if (IS_ERR(vma)) {
@@ -257,21 +169,30 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr)
 		goto up_fail;
 	}
 
+	vma = vdso_install_vvar_mapping(mm, addr);
+	if (IS_ERR(vma)) {
+		ret = PTR_ERR(vma);
+		do_munmap(mm, text_start, image->size, NULL);
+		goto up_fail;
+	}
+
 	vma = _install_special_mapping(mm,
-				       addr,
-				       -image->sym_vvar_start,
+				       VDSO_VCLOCK_PAGES_START(addr),
+				       VDSO_NR_VCLOCK_PAGES * PAGE_SIZE,
 				       VM_READ|VM_MAYREAD|VM_IO|VM_DONTDUMP|
-				       VM_PFNMAP,
-				       &vvar_mapping);
+				       VM_PFNMAP|VM_SEALED_SYSMAP,
+				       &vvar_vclock_mapping);
 
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
 		do_munmap(mm, text_start, image->size, NULL);
-	} else {
-		current->mm->context.vdso = (void __user *)text_start;
-		current->mm->context.vdso_image = image;
+		do_munmap(mm, addr, image->size, NULL);
+		goto up_fail;
 	}
 
+	current->mm->context.vdso = (void __user *)text_start;
+	current->mm->context.vdso_image = image;
+
 up_fail:
 	mmap_write_unlock(mm);
 	return ret;
@@ -293,7 +214,8 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr)
 	 */
 	for_each_vma(vmi, vma) {
 		if (vma_is_special_mapping(vma, &vdso_mapping) ||
-				vma_is_special_mapping(vma, &vvar_mapping)) {
+				vma_is_special_mapping(vma, &vdso_vvar_mapping) ||
+				vma_is_special_mapping(vma, &vvar_vclock_mapping)) {
 			mmap_write_unlock(mm);
 			return -EEXIST;
 		}
@@ -303,7 +225,6 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr)
 	return map_vdso(image, addr);
 }
 
-#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
 static int load_vdso32(void)
 {
 	if (vdso32_enabled != 1)  /* Other values all mean "disabled" */
@@ -311,45 +232,38 @@ static int load_vdso32(void)
 
 	return map_vdso(&vdso_image_32, 0);
 }
-#endif
 
-#ifdef CONFIG_X86_64
 int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 {
-	if (!vdso64_enabled)
-		return 0;
+	if (IS_ENABLED(CONFIG_X86_64)) {
+		if (!vdso64_enabled)
+			return 0;
+
+		return map_vdso(&vdso_image_64, 0);
+	}
 
-	return map_vdso(&vdso_image_64, 0);
+	return load_vdso32();
 }
 
 #ifdef CONFIG_COMPAT
 int compat_arch_setup_additional_pages(struct linux_binprm *bprm,
 				       int uses_interp, bool x32)
 {
-#ifdef CONFIG_X86_X32_ABI
-	if (x32) {
+	if (IS_ENABLED(CONFIG_X86_X32_ABI) && x32) {
 		if (!vdso64_enabled)
 			return 0;
 		return map_vdso(&vdso_image_x32, 0);
 	}
-#endif
-#ifdef CONFIG_IA32_EMULATION
-	return load_vdso32();
-#else
+
+	if (IS_ENABLED(CONFIG_IA32_EMULATION))
+		return load_vdso32();
+
 	return 0;
-#endif
-}
-#endif
-#else
-int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
-{
-	return load_vdso32();
 }
 #endif
 
 bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
 {
-#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
 	const struct vdso_image *image = current->mm->context.vdso_image;
 	unsigned long vdso = (unsigned long) current->mm->context.vdso;
 
@@ -358,7 +272,6 @@ bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
 		    regs->ip == vdso + image->sym_vdso32_rt_sigreturn_landing_pad)
 			return true;
 	}
-#endif
 	return false;
 }
 
diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c
index 2fb7d53cf333..6e6c0a740837 100644
--- a/arch/x86/entry/vsyscall/vsyscall_64.c
+++ b/arch/x86/entry/vsyscall/vsyscall_64.c
@@ -124,7 +124,12 @@ bool emulate_vsyscall(unsigned long error_code,
 	if ((error_code & (X86_PF_WRITE | X86_PF_USER)) != X86_PF_USER)
 		return false;
 
-	if (!(error_code & X86_PF_INSTR)) {
+	/*
+	 * Assume that faults at regs->ip are because of an
+	 * instruction fetch. Return early and avoid
+	 * emulation for faults during data accesses:
+	 */
+	if (address != regs->ip) {
 		/* Failed vsyscall read */
 		if (vsyscall_mode == EMULATE)
 			return false;
@@ -137,12 +142,18 @@ bool emulate_vsyscall(unsigned long error_code,
 	}
 
 	/*
+	 * X86_PF_INSTR is only set when NX is supported.  When
+	 * available, use it to double-check that the emulation code
+	 * is only being used for instruction fetches:
+	 */
+	if (cpu_feature_enabled(X86_FEATURE_NX))
+		WARN_ON_ONCE(!(error_code & X86_PF_INSTR));
+
+	/*
 	 * No point in checking CS -- the only way to get here is a user mode
 	 * trap to a high address, which means that we're in 64-bit user code.
 	 */
 
-	WARN_ON_ONCE(address != regs->ip);
-
 	if (vsyscall_mode == NONE) {
 		warn_bad_vsyscall(KERN_INFO, regs,
 				  "vsyscall attempted with vsyscall=none");
@@ -341,9 +352,7 @@ void __init set_vsyscall_pgtable_user_bits(pgd_t *root)
 	pgd = pgd_offset_pgd(root, VSYSCALL_ADDR);
 	set_pgd(pgd, __pgd(pgd_val(*pgd) | _PAGE_USER));
 	p4d = p4d_offset(pgd, VSYSCALL_ADDR);
-#if CONFIG_PGTABLE_LEVELS >= 5
 	set_p4d(p4d, __p4d(p4d_val(*p4d) | _PAGE_USER));
-#endif
 	pud = pud_offset(p4d, VSYSCALL_ADDR);
 	set_pud(pud, __pud(pud_val(*pud) | _PAGE_USER));
 	pmd = pmd_offset(pud, VSYSCALL_ADDR);
diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c
index 780acd3dff22..06f35a6b58a5 100644
--- a/arch/x86/events/amd/brs.c
+++ b/arch/x86/events/amd/brs.c
@@ -44,12 +44,12 @@ static inline unsigned int brs_to(int idx)
 static __always_inline void set_debug_extn_cfg(u64 val)
 {
 	/* bits[4:3] must always be set to 11b */
-	__wrmsr(MSR_AMD_DBG_EXTN_CFG, val | 3ULL << 3, val >> 32);
+	native_wrmsrq(MSR_AMD_DBG_EXTN_CFG, val | 3ULL << 3);
 }
 
 static __always_inline u64 get_debug_extn_cfg(void)
 {
-	return __rdmsr(MSR_AMD_DBG_EXTN_CFG);
+	return native_rdmsrq(MSR_AMD_DBG_EXTN_CFG);
 }
 
 static bool __init amd_brs_detect(void)
@@ -187,7 +187,7 @@ void amd_brs_reset(void)
 	/*
 	 * Mark first entry as poisoned
 	 */
-	wrmsrl(brs_to(0), BRS_POISON);
+	wrmsrq(brs_to(0), BRS_POISON);
 }
 
 int __init amd_brs_init(void)
@@ -325,7 +325,7 @@ void amd_brs_drain(void)
 		u32 brs_idx = tos - i;
 		u64 from, to;
 
-		rdmsrl(brs_to(brs_idx), to);
+		rdmsrq(brs_to(brs_idx), to);
 
 		/* Entry does not belong to us (as marked by kernel) */
 		if (to == BRS_POISON)
@@ -341,7 +341,7 @@ void amd_brs_drain(void)
 		if (!amd_brs_match_plm(event, to))
 			continue;
 
-		rdmsrl(brs_from(brs_idx), from);
+		rdmsrq(brs_from(brs_idx), from);
 
 		perf_clear_branch_entry_bitfields(br+nr);
 
@@ -371,7 +371,7 @@ static void amd_brs_poison_buffer(void)
 	idx = amd_brs_get_tos(&cfg);
 
 	/* Poison target of entry */
-	wrmsrl(brs_to(idx), BRS_POISON);
+	wrmsrq(brs_to(idx), BRS_POISON);
 }
 
 /*
@@ -381,7 +381,8 @@ static void amd_brs_poison_buffer(void)
  * On ctxswin, sched_in = true, called after the PMU has started
  * On ctxswout, sched_in = false, called before the PMU is stopped
  */
-void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
+void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx,
+			    struct task_struct *task, bool sched_in)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c
index 920e3a640cad..44656d2fb555 100644
--- a/arch/x86/events/amd/core.c
+++ b/arch/x86/events/amd/core.c
@@ -2,6 +2,7 @@
 #include <linux/perf_event.h>
 #include <linux/jump_label.h>
 #include <linux/export.h>
+#include <linux/kvm_types.h>
 #include <linux/types.h>
 #include <linux/init.h>
 #include <linux/slab.h>
@@ -9,6 +10,7 @@
 #include <linux/jiffies.h>
 #include <asm/apicdef.h>
 #include <asm/apic.h>
+#include <asm/msr.h>
 #include <asm/nmi.h>
 
 #include "../perf_event.h"
@@ -563,13 +565,13 @@ static void amd_pmu_cpu_reset(int cpu)
 		return;
 
 	/* Clear enable bits i.e. PerfCntrGlobalCtl.PerfCntrEn */
-	wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 0);
+	wrmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 0);
 
 	/*
 	 * Clear freeze and overflow bits i.e. PerfCntrGLobalStatus.LbrFreeze
 	 * and PerfCntrGLobalStatus.PerfCntrOvfl
 	 */
-	wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR,
+	wrmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR,
 	       GLOBAL_STATUS_LBRS_FROZEN | amd_pmu_global_cntr_mask);
 }
 
@@ -651,7 +653,7 @@ static void amd_pmu_cpu_dead(int cpu)
 
 static __always_inline void amd_pmu_set_global_ctl(u64 ctl)
 {
-	wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, ctl);
+	wrmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, ctl);
 }
 
 static inline u64 amd_pmu_get_global_status(void)
@@ -659,7 +661,7 @@ static inline u64 amd_pmu_get_global_status(void)
 	u64 status;
 
 	/* PerfCntrGlobalStatus is read-only */
-	rdmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, status);
+	rdmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, status);
 
 	return status;
 }
@@ -672,14 +674,14 @@ static inline void amd_pmu_ack_global_status(u64 status)
 	 * clears the same bit in PerfCntrGlobalStatus
 	 */
 
-	wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, status);
+	wrmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, status);
 }
 
 static bool amd_pmu_test_overflow_topbit(int idx)
 {
 	u64 counter;
 
-	rdmsrl(x86_pmu_event_addr(idx), counter);
+	rdmsrq(x86_pmu_event_addr(idx), counter);
 
 	return !(counter & BIT_ULL(x86_pmu.cntval_bits - 1));
 }
@@ -762,7 +764,12 @@ static void amd_pmu_enable_all(int added)
 		if (!test_bit(idx, cpuc->active_mask))
 			continue;
 
-		amd_pmu_enable_event(cpuc->events[idx]);
+		/*
+		 * FIXME: cpuc->events[idx] can become NULL in a subtle race
+		 * condition with NMI->throttle->x86_pmu_stop().
+		 */
+		if (cpuc->events[idx])
+			amd_pmu_enable_event(cpuc->events[idx]);
 	}
 }
 
@@ -943,11 +950,12 @@ static int amd_pmu_v2_snapshot_branch_stack(struct perf_branch_entry *entries, u
 static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	static atomic64_t status_warned = ATOMIC64_INIT(0);
+	u64 reserved, status, mask, new_bits, prev_bits;
 	struct perf_sample_data data;
 	struct hw_perf_event *hwc;
 	struct perf_event *event;
 	int handled = 0, idx;
-	u64 reserved, status, mask;
 	bool pmu_enabled;
 
 	/*
@@ -1000,11 +1008,9 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
 		if (!x86_perf_event_set_period(event))
 			continue;
 
-		if (has_branch_stack(event))
-			perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
+		perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
 
-		if (perf_event_overflow(event, &data, regs))
-			x86_pmu_stop(event, 0);
+		perf_event_overflow(event, &data, regs);
 	}
 
 	/*
@@ -1012,7 +1018,12 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
 	 * the corresponding PMCs are expected to be inactive according to the
 	 * active_mask
 	 */
-	WARN_ON(status > 0);
+	if (status > 0) {
+		prev_bits = atomic64_fetch_or(status, &status_warned);
+		// A new bit was set for the very first time.
+		new_bits = status & ~prev_bits;
+		WARN(new_bits, "New overflows for inactive PMCs: %llx\n", new_bits);
+	}
 
 	/* Clear overflow and freeze bits */
 	amd_pmu_ack_global_status(~status);
@@ -1564,7 +1575,7 @@ void amd_pmu_enable_virt(void)
 	/* Reload all events */
 	amd_pmu_reload_virt();
 }
-EXPORT_SYMBOL_GPL(amd_pmu_enable_virt);
+EXPORT_SYMBOL_FOR_KVM(amd_pmu_enable_virt);
 
 void amd_pmu_disable_virt(void)
 {
@@ -1581,4 +1592,4 @@ void amd_pmu_disable_virt(void)
 	/* Reload all events */
 	amd_pmu_reload_virt();
 }
-EXPORT_SYMBOL_GPL(amd_pmu_disable_virt);
+EXPORT_SYMBOL_FOR_KVM(amd_pmu_disable_virt);
diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c
index e91970b01d62..aca89f23d2e0 100644
--- a/arch/x86/events/amd/ibs.c
+++ b/arch/x86/events/amd/ibs.c
@@ -15,6 +15,7 @@
 #include <linux/sched/clock.h>
 
 #include <asm/apic.h>
+#include <asm/msr.h>
 
 #include "../perf_event.h"
 
@@ -26,11 +27,10 @@ static u32 ibs_caps;
 #include <linux/hardirq.h>
 
 #include <asm/nmi.h>
-#include <asm/amd-ibs.h>
-
-#define IBS_FETCH_CONFIG_MASK	(IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT)
-#define IBS_OP_CONFIG_MASK	IBS_OP_MAX_CNT
+#include <asm/amd/ibs.h>
 
+/* attr.config2 */
+#define IBS_SW_FILTER_MASK	1
 
 /*
  * IBS states:
@@ -87,6 +87,7 @@ struct perf_ibs {
 	u64				cnt_mask;
 	u64				enable_mask;
 	u64				valid_mask;
+	u16				min_period;
 	u64				max_period;
 	unsigned long			offset_mask[1];
 	int				offset_max;
@@ -268,11 +269,19 @@ static int validate_group(struct perf_event *event)
 	return 0;
 }
 
+static bool perf_ibs_ldlat_event(struct perf_ibs *perf_ibs,
+				 struct perf_event *event)
+{
+	return perf_ibs == &perf_ibs_op &&
+	       (ibs_caps & IBS_CAPS_OPLDLAT) &&
+	       (event->attr.config1 & 0xFFF);
+}
+
 static int perf_ibs_init(struct perf_event *event)
 {
 	struct hw_perf_event *hwc = &event->hw;
 	struct perf_ibs *perf_ibs;
-	u64 max_cnt, config;
+	u64 config;
 	int ret;
 
 	perf_ibs = get_ibs_pmu(event->attr.type);
@@ -290,6 +299,16 @@ static int perf_ibs_init(struct perf_event *event)
 	if (has_branch_stack(event))
 		return -EOPNOTSUPP;
 
+	/* handle exclude_{user,kernel} in the IRQ handler */
+	if (event->attr.exclude_host || event->attr.exclude_guest ||
+	    event->attr.exclude_idle)
+		return -EINVAL;
+
+	if (!(event->attr.config2 & IBS_SW_FILTER_MASK) &&
+	    (event->attr.exclude_kernel || event->attr.exclude_user ||
+	     event->attr.exclude_hv))
+		return -EINVAL;
+
 	ret = validate_group(event);
 	if (ret)
 		return ret;
@@ -298,25 +317,47 @@ static int perf_ibs_init(struct perf_event *event)
 		if (config & perf_ibs->cnt_mask)
 			/* raw max_cnt may not be set */
 			return -EINVAL;
-		if (!event->attr.sample_freq && hwc->sample_period & 0x0f)
-			/*
-			 * lower 4 bits can not be set in ibs max cnt,
-			 * but allowing it in case we adjust the
-			 * sample period to set a frequency.
-			 */
-			return -EINVAL;
-		hwc->sample_period &= ~0x0FULL;
-		if (!hwc->sample_period)
-			hwc->sample_period = 0x10;
+
+		if (event->attr.freq) {
+			hwc->sample_period = perf_ibs->min_period;
+		} else {
+			/* Silently mask off lower nibble. IBS hw mandates it. */
+			hwc->sample_period &= ~0x0FULL;
+			if (hwc->sample_period < perf_ibs->min_period)
+				return -EINVAL;
+		}
 	} else {
-		max_cnt = config & perf_ibs->cnt_mask;
+		u64 period = 0;
+
+		if (event->attr.freq)
+			return -EINVAL;
+
+		if (perf_ibs == &perf_ibs_op) {
+			period = (config & IBS_OP_MAX_CNT) << 4;
+			if (ibs_caps & IBS_CAPS_OPCNTEXT)
+				period |= config & IBS_OP_MAX_CNT_EXT_MASK;
+		} else {
+			period = (config & IBS_FETCH_MAX_CNT) << 4;
+		}
+
 		config &= ~perf_ibs->cnt_mask;
-		event->attr.sample_period = max_cnt << 4;
-		hwc->sample_period = event->attr.sample_period;
+		event->attr.sample_period = period;
+		hwc->sample_period = period;
+
+		if (hwc->sample_period < perf_ibs->min_period)
+			return -EINVAL;
 	}
 
-	if (!hwc->sample_period)
-		return -EINVAL;
+	if (perf_ibs_ldlat_event(perf_ibs, event)) {
+		u64 ldlat = event->attr.config1 & 0xFFF;
+
+		if (ldlat < 128 || ldlat > 2048)
+			return -EINVAL;
+		ldlat >>= 7;
+
+		config |= (ldlat - 1) << 59;
+		config |= IBS_OP_L3MISSONLY | IBS_OP_LDLAT_EN;
+	}
 
 	/*
 	 * If we modify hwc->sample_period, we also need to update
@@ -337,7 +378,8 @@ static int perf_ibs_set_period(struct perf_ibs *perf_ibs,
 	int overflow;
 
 	/* ignore lower 4 bits in min count: */
-	overflow = perf_event_set_period(hwc, 1<<4, perf_ibs->max_period, period);
+	overflow = perf_event_set_period(hwc, perf_ibs->min_period,
+					 perf_ibs->max_period, period);
 	local64_set(&hwc->prev_count, 0);
 
 	return overflow;
@@ -383,7 +425,7 @@ perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event,
 	 * prev count manually on overflow.
 	 */
 	while (!perf_event_try_update(event, count, 64)) {
-		rdmsrl(event->hw.config_base, *config);
+		rdmsrq(event->hw.config_base, *config);
 		count = perf_ibs->get_count(*config);
 	}
 }
@@ -394,9 +436,9 @@ static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs,
 	u64 tmp = hwc->config | config;
 
 	if (perf_ibs->fetch_count_reset_broken)
-		wrmsrl(hwc->config_base, tmp & ~perf_ibs->enable_mask);
+		wrmsrq(hwc->config_base, tmp & ~perf_ibs->enable_mask);
 
-	wrmsrl(hwc->config_base, tmp | perf_ibs->enable_mask);
+	wrmsrq(hwc->config_base, tmp | perf_ibs->enable_mask);
 }
 
 /*
@@ -411,9 +453,9 @@ static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs,
 {
 	config &= ~perf_ibs->cnt_mask;
 	if (boot_cpu_data.x86 == 0x10)
-		wrmsrl(hwc->config_base, config);
+		wrmsrq(hwc->config_base, config);
 	config &= ~perf_ibs->enable_mask;
-	wrmsrl(hwc->config_base, config);
+	wrmsrq(hwc->config_base, config);
 }
 
 /*
@@ -435,6 +477,9 @@ static void perf_ibs_start(struct perf_event *event, int flags)
 	WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
 	hwc->state = 0;
 
+	if (event->attr.freq && hwc->sample_period < perf_ibs->min_period)
+		hwc->sample_period = perf_ibs->min_period;
+
 	perf_ibs_set_period(perf_ibs, hwc, &period);
 	if (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_OPCNTEXT)) {
 		config |= period & IBS_OP_MAX_CNT_EXT_MASK;
@@ -469,7 +514,7 @@ static void perf_ibs_stop(struct perf_event *event, int flags)
 	if (!stopping && (hwc->state & PERF_HES_UPTODATE))
 		return;
 
-	rdmsrl(hwc->config_base, config);
+	rdmsrq(hwc->config_base, config);
 
 	if (stopping) {
 		/*
@@ -542,6 +587,28 @@ static void perf_ibs_del(struct perf_event *event, int flags)
 
 static void perf_ibs_read(struct perf_event *event) { }
 
+static int perf_ibs_check_period(struct perf_event *event, u64 value)
+{
+	struct perf_ibs *perf_ibs;
+	u64 low_nibble;
+
+	if (event->attr.freq)
+		return 0;
+
+	perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
+	low_nibble = value & 0xFULL;
+
+	/*
+	 * This contradicts with perf_ibs_init() which allows sample period
+	 * with lower nibble bits set but silently masks them off. Whereas
+	 * this returns error.
+	 */
+	if (low_nibble || value < perf_ibs->min_period)
+		return -EINVAL;
+
+	return 0;
+}
+
 /*
  * We need to initialize with empty group if all attributes in the
  * group are dynamic.
@@ -550,27 +617,20 @@ static struct attribute *attrs_empty[] = {
 	NULL,
 };
 
-static struct attribute_group empty_format_group = {
-	.name = "format",
-	.attrs = attrs_empty,
-};
-
 static struct attribute_group empty_caps_group = {
 	.name = "caps",
 	.attrs = attrs_empty,
 };
 
-static const struct attribute_group *empty_attr_groups[] = {
-	&empty_format_group,
-	&empty_caps_group,
-	NULL,
-};
-
 PMU_FORMAT_ATTR(rand_en,	"config:57");
 PMU_FORMAT_ATTR(cnt_ctl,	"config:19");
+PMU_FORMAT_ATTR(swfilt,		"config2:0");
 PMU_EVENT_ATTR_STRING(l3missonly, fetch_l3missonly, "config:59");
 PMU_EVENT_ATTR_STRING(l3missonly, op_l3missonly, "config:16");
+PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_format, "config1:0-11");
 PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1");
+PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_cap, "1");
+PMU_EVENT_ATTR_STRING(dtlb_pgsize, ibs_op_dtlb_pgsize_cap, "1");
 
 static umode_t
 zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i)
@@ -578,8 +638,21 @@ zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int
 	return ibs_caps & IBS_CAPS_ZEN4 ? attr->mode : 0;
 }
 
-static struct attribute *rand_en_attrs[] = {
+static umode_t
+ibs_op_ldlat_is_visible(struct kobject *kobj, struct attribute *attr, int i)
+{
+	return ibs_caps & IBS_CAPS_OPLDLAT ? attr->mode : 0;
+}
+
+static umode_t
+ibs_op_dtlb_pgsize_is_visible(struct kobject *kobj, struct attribute *attr, int i)
+{
+	return ibs_caps & IBS_CAPS_OPDTLBPGSIZE ? attr->mode : 0;
+}
+
+static struct attribute *fetch_attrs[] = {
 	&format_attr_rand_en.attr,
+	&format_attr_swfilt.attr,
 	NULL,
 };
 
@@ -593,9 +666,19 @@ static struct attribute *zen4_ibs_extensions_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group group_rand_en = {
+static struct attribute *ibs_op_ldlat_cap_attrs[] = {
+	&ibs_op_ldlat_cap.attr.attr,
+	NULL,
+};
+
+static struct attribute *ibs_op_dtlb_pgsize_cap_attrs[] = {
+	&ibs_op_dtlb_pgsize_cap.attr.attr,
+	NULL,
+};
+
+static struct attribute_group group_fetch_formats = {
 	.name = "format",
-	.attrs = rand_en_attrs,
+	.attrs = fetch_attrs,
 };
 
 static struct attribute_group group_fetch_l3missonly = {
@@ -610,8 +693,20 @@ static struct attribute_group group_zen4_ibs_extensions = {
 	.is_visible = zen4_ibs_extensions_is_visible,
 };
 
+static struct attribute_group group_ibs_op_ldlat_cap = {
+	.name = "caps",
+	.attrs = ibs_op_ldlat_cap_attrs,
+	.is_visible = ibs_op_ldlat_is_visible,
+};
+
+static struct attribute_group group_ibs_op_dtlb_pgsize_cap = {
+	.name = "caps",
+	.attrs = ibs_op_dtlb_pgsize_cap_attrs,
+	.is_visible = ibs_op_dtlb_pgsize_is_visible,
+};
+
 static const struct attribute_group *fetch_attr_groups[] = {
-	&group_rand_en,
+	&group_fetch_formats,
 	&empty_caps_group,
 	NULL,
 };
@@ -628,6 +723,11 @@ cnt_ctl_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 	return ibs_caps & IBS_CAPS_OPCNT ? attr->mode : 0;
 }
 
+static struct attribute *op_attrs[] = {
+	&format_attr_swfilt.attr,
+	NULL,
+};
+
 static struct attribute *cnt_ctl_attrs[] = {
 	&format_attr_cnt_ctl.attr,
 	NULL,
@@ -638,6 +738,16 @@ static struct attribute *op_l3missonly_attrs[] = {
 	NULL,
 };
 
+static struct attribute_group group_op_formats = {
+	.name = "format",
+	.attrs = op_attrs,
+};
+
+static struct attribute *ibs_op_ldlat_format_attrs[] = {
+	&ibs_op_ldlat_format.attr.attr,
+	NULL,
+};
+
 static struct attribute_group group_cnt_ctl = {
 	.name = "format",
 	.attrs = cnt_ctl_attrs,
@@ -650,10 +760,25 @@ static struct attribute_group group_op_l3missonly = {
 	.is_visible = zen4_ibs_extensions_is_visible,
 };
 
+static const struct attribute_group *op_attr_groups[] = {
+	&group_op_formats,
+	&empty_caps_group,
+	NULL,
+};
+
+static struct attribute_group group_ibs_op_ldlat_format = {
+	.name = "format",
+	.attrs = ibs_op_ldlat_format_attrs,
+	.is_visible = ibs_op_ldlat_is_visible,
+};
+
 static const struct attribute_group *op_attr_update[] = {
 	&group_cnt_ctl,
 	&group_op_l3missonly,
 	&group_zen4_ibs_extensions,
+	&group_ibs_op_ldlat_cap,
+	&group_ibs_op_ldlat_format,
+	&group_ibs_op_dtlb_pgsize_cap,
 	NULL,
 };
 
@@ -667,13 +792,14 @@ static struct perf_ibs perf_ibs_fetch = {
 		.start		= perf_ibs_start,
 		.stop		= perf_ibs_stop,
 		.read		= perf_ibs_read,
-		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
+		.check_period	= perf_ibs_check_period,
 	},
 	.msr			= MSR_AMD64_IBSFETCHCTL,
-	.config_mask		= IBS_FETCH_CONFIG_MASK,
+	.config_mask		= IBS_FETCH_MAX_CNT | IBS_FETCH_RAND_EN,
 	.cnt_mask		= IBS_FETCH_MAX_CNT,
 	.enable_mask		= IBS_FETCH_ENABLE,
 	.valid_mask		= IBS_FETCH_VAL,
+	.min_period		= 0x10,
 	.max_period		= IBS_FETCH_MAX_CNT << 4,
 	.offset_mask		= { MSR_AMD64_IBSFETCH_REG_MASK },
 	.offset_max		= MSR_AMD64_IBSFETCH_REG_COUNT,
@@ -691,14 +817,15 @@ static struct perf_ibs perf_ibs_op = {
 		.start		= perf_ibs_start,
 		.stop		= perf_ibs_stop,
 		.read		= perf_ibs_read,
-		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
+		.check_period	= perf_ibs_check_period,
 	},
 	.msr			= MSR_AMD64_IBSOPCTL,
-	.config_mask		= IBS_OP_CONFIG_MASK,
+	.config_mask		= IBS_OP_MAX_CNT,
 	.cnt_mask		= IBS_OP_MAX_CNT | IBS_OP_CUR_CNT |
 				  IBS_OP_CUR_CNT_RAND,
 	.enable_mask		= IBS_OP_ENABLE,
 	.valid_mask		= IBS_OP_VAL,
+	.min_period		= 0x90,
 	.max_period		= IBS_OP_MAX_CNT << 4,
 	.offset_mask		= { MSR_AMD64_IBSOP_REG_MASK },
 	.offset_max		= MSR_AMD64_IBSOP_REG_COUNT,
@@ -900,6 +1027,10 @@ static void perf_ibs_get_tlb_lvl(union ibs_op_data3 *op_data3,
 	if (!op_data3->dc_lin_addr_valid)
 		return;
 
+	if ((ibs_caps & IBS_CAPS_OPDTLBPGSIZE) &&
+	    !op_data3->dc_phy_addr_valid)
+		return;
+
 	if (!op_data3->dc_l1tlb_miss) {
 		data_src->mem_dtlb = PERF_MEM_TLB_L1 | PERF_MEM_TLB_HIT;
 		return;
@@ -924,6 +1055,8 @@ static void perf_ibs_get_mem_lock(union ibs_op_data3 *op_data3,
 		data_src->mem_lock = PERF_MEM_LOCK_LOCKED;
 }
 
+/* Be careful. Works only for contiguous MSRs. */
+#define ibs_fetch_msr_idx(msr)	(msr - MSR_AMD64_IBSFETCHCTL)
 #define ibs_op_msr_idx(msr)	(msr - MSR_AMD64_IBSOPCTL)
 
 static void perf_ibs_get_data_src(struct perf_ibs_data *ibs_data,
@@ -1004,21 +1137,92 @@ static void perf_ibs_parse_ld_st_data(__u64 sample_type,
 	}
 }
 
-static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, u64 sample_type,
+static bool perf_ibs_is_mem_sample_type(struct perf_ibs *perf_ibs,
+					struct perf_event *event)
+{
+	u64 sample_type = event->attr.sample_type;
+
+	return perf_ibs == &perf_ibs_op &&
+	       sample_type & (PERF_SAMPLE_DATA_SRC |
+			      PERF_SAMPLE_WEIGHT_TYPE |
+			      PERF_SAMPLE_ADDR |
+			      PERF_SAMPLE_PHYS_ADDR);
+}
+
+static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs,
+				   struct perf_event *event,
 				   int check_rip)
 {
-	if (sample_type & PERF_SAMPLE_RAW ||
-	    (perf_ibs == &perf_ibs_op &&
-	     (sample_type & PERF_SAMPLE_DATA_SRC ||
-	      sample_type & PERF_SAMPLE_WEIGHT_TYPE ||
-	      sample_type & PERF_SAMPLE_ADDR ||
-	      sample_type & PERF_SAMPLE_PHYS_ADDR)))
+	if (event->attr.sample_type & PERF_SAMPLE_RAW ||
+	    perf_ibs_is_mem_sample_type(perf_ibs, event) ||
+	    perf_ibs_ldlat_event(perf_ibs, event))
 		return perf_ibs->offset_max;
 	else if (check_rip)
 		return 3;
 	return 1;
 }
 
+static bool perf_ibs_is_kernel_data_addr(struct perf_event *event,
+					 struct perf_ibs_data *ibs_data)
+{
+	u64 sample_type_mask = PERF_SAMPLE_ADDR | PERF_SAMPLE_RAW;
+	union ibs_op_data3 op_data3;
+	u64 dc_lin_addr;
+
+	op_data3.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)];
+	dc_lin_addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCLINAD)];
+
+	return unlikely((event->attr.sample_type & sample_type_mask) &&
+			op_data3.dc_lin_addr_valid && kernel_ip(dc_lin_addr));
+}
+
+static bool perf_ibs_is_kernel_br_target(struct perf_event *event,
+					 struct perf_ibs_data *ibs_data,
+					 int br_target_idx)
+{
+	union ibs_op_data op_data;
+	u64 br_target;
+
+	op_data.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA)];
+	br_target = ibs_data->regs[br_target_idx];
+
+	return unlikely((event->attr.sample_type & PERF_SAMPLE_RAW) &&
+			op_data.op_brn_ret && kernel_ip(br_target));
+}
+
+static bool perf_ibs_swfilt_discard(struct perf_ibs *perf_ibs, struct perf_event *event,
+				    struct pt_regs *regs, struct perf_ibs_data *ibs_data,
+				    int br_target_idx)
+{
+	if (perf_exclude_event(event, regs))
+		return true;
+
+	if (perf_ibs != &perf_ibs_op || !event->attr.exclude_kernel)
+		return false;
+
+	if (perf_ibs_is_kernel_data_addr(event, ibs_data))
+		return true;
+
+	if (br_target_idx != -1 &&
+	    perf_ibs_is_kernel_br_target(event, ibs_data, br_target_idx))
+		return true;
+
+	return false;
+}
+
+static void perf_ibs_phyaddr_clear(struct perf_ibs *perf_ibs,
+				   struct perf_ibs_data *ibs_data)
+{
+	if (perf_ibs == &perf_ibs_op) {
+		ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)] &= ~(1ULL << 18);
+		ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCPHYSAD)] = 0;
+		return;
+	}
+
+	ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHCTL)] &= ~(1ULL << 52);
+	ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHPHYSAD)] = 0;
+}
+
 static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
 {
 	struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
@@ -1031,6 +1235,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
 	int offset, size, check_rip, offset_max, throttle = 0;
 	unsigned int msr;
 	u64 *buf, *config, period, new_config = 0;
+	int br_target_idx = -1;
 
 	if (!test_bit(IBS_STARTED, pcpu->state)) {
 fail:
@@ -1052,7 +1257,7 @@ fail:
 	hwc = &event->hw;
 	msr = hwc->config_base;
 	buf = ibs_data.regs;
-	rdmsrl(msr, *buf);
+	rdmsrq(msr, *buf);
 	if (!(*buf++ & perf_ibs->valid_mask))
 		goto fail;
 
@@ -1067,15 +1272,31 @@ fail:
 	offset = 1;
 	check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK));
 
-	offset_max = perf_ibs_get_offset_max(perf_ibs, event->attr.sample_type, check_rip);
+	offset_max = perf_ibs_get_offset_max(perf_ibs, event, check_rip);
 
 	do {
-		rdmsrl(msr + offset, *buf++);
+		rdmsrq(msr + offset, *buf++);
 		size++;
 		offset = find_next_bit(perf_ibs->offset_mask,
 				       perf_ibs->offset_max,
 				       offset + 1);
 	} while (offset < offset_max);
+
+	if (perf_ibs_ldlat_event(perf_ibs, event)) {
+		union ibs_op_data3 op_data3;
+
+		op_data3.val = ibs_data.regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)];
+		/*
+		 * Opening event is errored out if load latency threshold is
+		 * outside of [128, 2048] range. Since the event has reached
+		 * interrupt handler, we can safely assume the threshold is
+		 * within [128, 2048] range.
+		 */
+		if (!op_data3.ld_op || !op_data3.dc_miss ||
+		    op_data3.dc_miss_lat <= (event->attr.config1 & 0xFFF))
+			goto out;
+	}
+
 	/*
 	 * Read IbsBrTarget, IbsOpData4, and IbsExtdCtl separately
 	 * depending on their availability.
@@ -1084,16 +1305,17 @@ fail:
 	if (event->attr.sample_type & PERF_SAMPLE_RAW) {
 		if (perf_ibs == &perf_ibs_op) {
 			if (ibs_caps & IBS_CAPS_BRNTRGT) {
-				rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++);
+				rdmsrq(MSR_AMD64_IBSBRTARGET, *buf++);
+				br_target_idx = size;
 				size++;
 			}
 			if (ibs_caps & IBS_CAPS_OPDATA4) {
-				rdmsrl(MSR_AMD64_IBSOPDATA4, *buf++);
+				rdmsrq(MSR_AMD64_IBSOPDATA4, *buf++);
 				size++;
 			}
 		}
 		if (perf_ibs == &perf_ibs_fetch && (ibs_caps & IBS_CAPS_FETCHCTLEXTD)) {
-			rdmsrl(MSR_AMD64_ICIBSEXTDCTL, *buf++);
+			rdmsrq(MSR_AMD64_ICIBSEXTDCTL, *buf++);
 			size++;
 		}
 	}
@@ -1111,6 +1333,21 @@ fail:
 		regs.flags |= PERF_EFLAGS_EXACT;
 	}
 
+	if ((event->attr.config2 & IBS_SW_FILTER_MASK) &&
+	    perf_ibs_swfilt_discard(perf_ibs, event, &regs, &ibs_data, br_target_idx)) {
+		throttle = perf_event_account_interrupt(event);
+		goto out;
+	}
+	/*
+	 * Prevent leaking physical addresses to unprivileged users. Skip
+	 * PERF_SAMPLE_PHYS_ADDR check since generic code prevents it for
+	 * unprivileged users.
+	 */
+	if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
+	    perf_allow_kernel()) {
+		perf_ibs_phyaddr_clear(perf_ibs, &ibs_data);
+	}
+
 	if (event->attr.sample_type & PERF_SAMPLE_RAW) {
 		raw = (struct perf_raw_record){
 			.frag = {
@@ -1118,7 +1355,7 @@ fail:
 				.data = ibs_data.data,
 			},
 		};
-		perf_sample_save_raw_data(&data, &raw);
+		perf_sample_save_raw_data(&data, event, &raw);
 	}
 
 	if (perf_ibs == &perf_ibs_op)
@@ -1129,14 +1366,15 @@ fail:
 	 * recorded as part of interrupt regs. Thus we need to use rip from
 	 * interrupt regs while unwinding call stack.
 	 */
-	if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
-		perf_sample_save_callchain(&data, event, iregs);
+	perf_sample_save_callchain(&data, event, iregs);
 
 	throttle = perf_event_overflow(event, &data, &regs);
+
+	if (event->attr.freq && hwc->sample_period < perf_ibs->min_period)
+		hwc->sample_period = perf_ibs->min_period;
+
 out:
-	if (throttle) {
-		perf_ibs_stop(event, 0);
-	} else {
+	if (!throttle) {
 		if (perf_ibs == &perf_ibs_op) {
 			if (ibs_caps & IBS_CAPS_OPCNTEXT) {
 				new_config = period & IBS_OP_MAX_CNT_EXT_MASK;
@@ -1222,13 +1460,14 @@ static __init int perf_ibs_op_init(void)
 	if (ibs_caps & IBS_CAPS_OPCNTEXT) {
 		perf_ibs_op.max_period  |= IBS_OP_MAX_CNT_EXT_MASK;
 		perf_ibs_op.config_mask	|= IBS_OP_MAX_CNT_EXT_MASK;
-		perf_ibs_op.cnt_mask    |= IBS_OP_MAX_CNT_EXT_MASK;
+		perf_ibs_op.cnt_mask    |= (IBS_OP_MAX_CNT_EXT_MASK |
+					    IBS_OP_CUR_CNT_EXT_MASK);
 	}
 
 	if (ibs_caps & IBS_CAPS_ZEN4)
 		perf_ibs_op.config_mask |= IBS_OP_L3MISSONLY;
 
-	perf_ibs_op.pmu.attr_groups = empty_attr_groups;
+	perf_ibs_op.pmu.attr_groups = op_attr_groups;
 	perf_ibs_op.pmu.attr_update = op_attr_update;
 
 	return perf_ibs_pmu_init(&perf_ibs_op, "ibs_op");
@@ -1325,7 +1564,7 @@ static inline int ibs_eilvt_valid(void)
 
 	preempt_disable();
 
-	rdmsrl(MSR_AMD64_IBSCTL, val);
+	rdmsrq(MSR_AMD64_IBSCTL, val);
 	offset = val & IBSCTL_LVT_OFFSET_MASK;
 
 	if (!(val & IBSCTL_LVT_OFFSET_VALID)) {
@@ -1440,7 +1679,7 @@ static inline int get_ibs_lvt_offset(void)
 {
 	u64 val;
 
-	rdmsrl(MSR_AMD64_IBSCTL, val);
+	rdmsrq(MSR_AMD64_IBSCTL, val);
 	if (!(val & IBSCTL_LVT_OFFSET_VALID))
 		return -EINVAL;
 
@@ -1479,26 +1718,30 @@ static int x86_pmu_amd_ibs_starting_cpu(unsigned int cpu)
 
 #ifdef CONFIG_PM
 
-static int perf_ibs_suspend(void)
+static int perf_ibs_suspend(void *data)
 {
 	clear_APIC_ibs();
 	return 0;
 }
 
-static void perf_ibs_resume(void)
+static void perf_ibs_resume(void *data)
 {
 	ibs_eilvt_setup();
 	setup_APIC_ibs();
 }
 
-static struct syscore_ops perf_ibs_syscore_ops = {
+static const struct syscore_ops perf_ibs_syscore_ops = {
 	.resume		= perf_ibs_resume,
 	.suspend	= perf_ibs_suspend,
 };
 
+static struct syscore perf_ibs_syscore = {
+	.ops = &perf_ibs_syscore_ops,
+};
+
 static void perf_ibs_pm_init(void)
 {
-	register_syscore_ops(&perf_ibs_syscore_ops);
+	register_syscore(&perf_ibs_syscore);
 }
 
 #else
diff --git a/arch/x86/events/amd/iommu.c b/arch/x86/events/amd/iommu.c
index b15f7b950d2e..a721da9987dd 100644
--- a/arch/x86/events/amd/iommu.c
+++ b/arch/x86/events/amd/iommu.c
@@ -16,6 +16,8 @@
 #include <linux/slab.h>
 #include <linux/amd-iommu.h>
 
+#include <asm/msr.h>
+
 #include "../perf_event.h"
 #include "iommu.h"
 
@@ -30,7 +32,7 @@
 #define GET_DOMID_MASK(x)  (((x)->conf1 >> 16) & 0xFFFFULL)
 #define GET_PASID_MASK(x)  (((x)->conf1 >> 32) & 0xFFFFFULL)
 
-#define IOMMU_NAME_SIZE 16
+#define IOMMU_NAME_SIZE 24
 
 struct perf_amd_iommu {
 	struct list_head list;
diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c
index 19c7b76e21bc..d24da377df77 100644
--- a/arch/x86/events/amd/lbr.c
+++ b/arch/x86/events/amd/lbr.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/perf_event.h>
+#include <asm/msr.h>
 #include <asm/perf_event.h>
 
 #include "../perf_event.h"
@@ -61,19 +62,19 @@ struct branch_entry {
 
 static __always_inline void amd_pmu_lbr_set_from(unsigned int idx, u64 val)
 {
-	wrmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2, val);
+	wrmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2, val);
 }
 
 static __always_inline void amd_pmu_lbr_set_to(unsigned int idx, u64 val)
 {
-	wrmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val);
+	wrmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val);
 }
 
 static __always_inline u64 amd_pmu_lbr_get_from(unsigned int idx)
 {
 	u64 val;
 
-	rdmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2, val);
+	rdmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2, val);
 
 	return val;
 }
@@ -82,7 +83,7 @@ static __always_inline u64 amd_pmu_lbr_get_to(unsigned int idx)
 {
 	u64 val;
 
-	rdmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val);
+	rdmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val);
 
 	return val;
 }
@@ -333,7 +334,7 @@ void amd_pmu_lbr_reset(void)
 
 	cpuc->last_task_ctx = NULL;
 	cpuc->last_log_id = 0;
-	wrmsrl(MSR_AMD64_LBR_SELECT, 0);
+	wrmsrq(MSR_AMD64_LBR_SELECT, 0);
 }
 
 void amd_pmu_lbr_add(struct perf_event *event)
@@ -371,7 +372,8 @@ void amd_pmu_lbr_del(struct perf_event *event)
 	perf_sched_cb_dec(event->pmu);
 }
 
-void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
+void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
+			    struct task_struct *task, bool sched_in)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
@@ -395,16 +397,16 @@ void amd_pmu_lbr_enable_all(void)
 	/* Set hardware branch filter */
 	if (cpuc->lbr_select) {
 		lbr_select = cpuc->lbr_sel->config & LBR_SELECT_MASK;
-		wrmsrl(MSR_AMD64_LBR_SELECT, lbr_select);
+		wrmsrq(MSR_AMD64_LBR_SELECT, lbr_select);
 	}
 
 	if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) {
-		rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
-		wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
+		rdmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
+		wrmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
 	}
 
-	rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
-	wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg | DBG_EXTN_CFG_LBRV2EN);
+	rdmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
+	wrmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg | DBG_EXTN_CFG_LBRV2EN);
 }
 
 void amd_pmu_lbr_disable_all(void)
diff --git a/arch/x86/events/amd/power.c b/arch/x86/events/amd/power.c
index 37d5b380516e..dad42790cf7d 100644
--- a/arch/x86/events/amd/power.c
+++ b/arch/x86/events/amd/power.c
@@ -11,6 +11,7 @@
 #include <linux/slab.h>
 #include <linux/perf_event.h>
 #include <asm/cpu_device_id.h>
+#include <asm/msr.h>
 #include "../perf_event.h"
 
 /* Event code: LSB 8 bits, passed in attr->config any other bit is reserved. */
@@ -48,8 +49,8 @@ static void event_update(struct perf_event *event)
 
 	prev_pwr_acc = hwc->pwr_acc;
 	prev_ptsc = hwc->ptsc;
-	rdmsrl(MSR_F15H_CU_PWR_ACCUMULATOR, new_pwr_acc);
-	rdmsrl(MSR_F15H_PTSC, new_ptsc);
+	rdmsrq(MSR_F15H_CU_PWR_ACCUMULATOR, new_pwr_acc);
+	rdmsrq(MSR_F15H_PTSC, new_ptsc);
 
 	/*
 	 * Calculate the CU power consumption over a time period, the unit of
@@ -75,8 +76,8 @@ static void __pmu_event_start(struct perf_event *event)
 
 	event->hw.state = 0;
 
-	rdmsrl(MSR_F15H_PTSC, event->hw.ptsc);
-	rdmsrl(MSR_F15H_CU_PWR_ACCUMULATOR, event->hw.pwr_acc);
+	rdmsrq(MSR_F15H_PTSC, event->hw.ptsc);
+	rdmsrq(MSR_F15H_CU_PWR_ACCUMULATOR, event->hw.pwr_acc);
 }
 
 static void pmu_event_start(struct perf_event *event, int mode)
@@ -272,7 +273,7 @@ static int __init amd_power_pmu_init(void)
 
 	cpu_pwr_sample_ratio = cpuid_ecx(0x80000007);
 
-	if (rdmsrl_safe(MSR_F15H_CU_MAX_PWR_ACCUMULATOR, &max_cu_acc_power)) {
+	if (rdmsrq_safe(MSR_F15H_CU_MAX_PWR_ACCUMULATOR, &max_cu_acc_power)) {
 		pr_err("Failed to read max compute unit power accumulator MSR\n");
 		return -ENODEV;
 	}
diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c
index 0bfde2ea5cb8..e8b6af199c73 100644
--- a/arch/x86/events/amd/uncore.c
+++ b/arch/x86/events/amd/uncore.c
@@ -21,6 +21,7 @@
 #define NUM_COUNTERS_NB		4
 #define NUM_COUNTERS_L2		4
 #define NUM_COUNTERS_L3		6
+#define NUM_COUNTERS_MAX	64
 
 #define RDPMC_BASE_NB		6
 #define RDPMC_BASE_LLC		10
@@ -38,7 +39,10 @@ struct amd_uncore_ctx {
 	int refcnt;
 	int cpu;
 	struct perf_event **events;
-	struct hlist_node node;
+	unsigned long active_mask[BITS_TO_LONGS(NUM_COUNTERS_MAX)];
+	int nr_active;
+	struct hrtimer hrtimer;
+	u64 hrtimer_duration;
 };
 
 struct amd_uncore_pmu {
@@ -83,11 +87,51 @@ struct amd_uncore {
 
 static struct amd_uncore uncores[UNCORE_TYPE_MAX];
 
+/* Interval for hrtimer, defaults to 60000 milliseconds */
+static unsigned int update_interval = 60 * MSEC_PER_SEC;
+module_param(update_interval, uint, 0444);
+
 static struct amd_uncore_pmu *event_to_amd_uncore_pmu(struct perf_event *event)
 {
 	return container_of(event->pmu, struct amd_uncore_pmu, pmu);
 }
 
+static enum hrtimer_restart amd_uncore_hrtimer(struct hrtimer *hrtimer)
+{
+	struct amd_uncore_ctx *ctx;
+	struct perf_event *event;
+	int bit;
+
+	ctx = container_of(hrtimer, struct amd_uncore_ctx, hrtimer);
+
+	if (!ctx->nr_active || ctx->cpu != smp_processor_id())
+		return HRTIMER_NORESTART;
+
+	for_each_set_bit(bit, ctx->active_mask, NUM_COUNTERS_MAX) {
+		event = ctx->events[bit];
+		event->pmu->read(event);
+	}
+
+	hrtimer_forward_now(hrtimer, ns_to_ktime(ctx->hrtimer_duration));
+	return HRTIMER_RESTART;
+}
+
+static void amd_uncore_start_hrtimer(struct amd_uncore_ctx *ctx)
+{
+	hrtimer_start(&ctx->hrtimer, ns_to_ktime(ctx->hrtimer_duration),
+		      HRTIMER_MODE_REL_PINNED_HARD);
+}
+
+static void amd_uncore_cancel_hrtimer(struct amd_uncore_ctx *ctx)
+{
+	hrtimer_cancel(&ctx->hrtimer);
+}
+
+static void amd_uncore_init_hrtimer(struct amd_uncore_ctx *ctx)
+{
+	hrtimer_setup(&ctx->hrtimer, amd_uncore_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
+}
+
 static void amd_uncore_read(struct perf_event *event)
 {
 	struct hw_perf_event *hwc = &event->hw;
@@ -106,9 +150,9 @@ static void amd_uncore_read(struct perf_event *event)
 	 * read counts directly from the corresponding PERF_CTR.
 	 */
 	if (hwc->event_base_rdpmc < 0)
-		rdmsrl(hwc->event_base, new);
+		rdmsrq(hwc->event_base, new);
 	else
-		rdpmcl(hwc->event_base_rdpmc, new);
+		new = rdpmc(hwc->event_base_rdpmc);
 
 	local64_set(&hwc->prev_count, new);
 	delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
@@ -118,27 +162,40 @@ static void amd_uncore_read(struct perf_event *event)
 
 static void amd_uncore_start(struct perf_event *event, int flags)
 {
+	struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
+	struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
 	struct hw_perf_event *hwc = &event->hw;
 
+	if (!ctx->nr_active++)
+		amd_uncore_start_hrtimer(ctx);
+
 	if (flags & PERF_EF_RELOAD)
-		wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
+		wrmsrq(hwc->event_base, (u64)local64_read(&hwc->prev_count));
 
 	hwc->state = 0;
-	wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
+	__set_bit(hwc->idx, ctx->active_mask);
+	wrmsrq(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
 	perf_event_update_userpage(event);
 }
 
 static void amd_uncore_stop(struct perf_event *event, int flags)
 {
+	struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
+	struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
 	struct hw_perf_event *hwc = &event->hw;
 
-	wrmsrl(hwc->config_base, hwc->config);
+	wrmsrq(hwc->config_base, hwc->config);
 	hwc->state |= PERF_HES_STOPPED;
 
 	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
 		event->pmu->read(event);
 		hwc->state |= PERF_HES_UPTODATE;
 	}
+
+	if (!--ctx->nr_active)
+		amd_uncore_cancel_hrtimer(ctx);
+
+	__clear_bit(hwc->idx, ctx->active_mask);
 }
 
 static int amd_uncore_add(struct perf_event *event, int flags)
@@ -491,6 +548,9 @@ static int amd_uncore_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
 				goto fail;
 			}
 
+			amd_uncore_init_hrtimer(curr);
+			curr->hrtimer_duration = (u64)update_interval * NSEC_PER_MSEC;
+
 			cpumask_set_cpu(cpu, &pmu->active_mask);
 		}
 
@@ -880,16 +940,55 @@ static int amd_uncore_umc_event_init(struct perf_event *event)
 
 static void amd_uncore_umc_start(struct perf_event *event, int flags)
 {
+	struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
+	struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
 	struct hw_perf_event *hwc = &event->hw;
 
+	if (!ctx->nr_active++)
+		amd_uncore_start_hrtimer(ctx);
+
 	if (flags & PERF_EF_RELOAD)
-		wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
+		wrmsrq(hwc->event_base, (u64)local64_read(&hwc->prev_count));
 
 	hwc->state = 0;
-	wrmsrl(hwc->config_base, (hwc->config | AMD64_PERFMON_V2_ENABLE_UMC));
+	__set_bit(hwc->idx, ctx->active_mask);
+	wrmsrq(hwc->config_base, (hwc->config | AMD64_PERFMON_V2_ENABLE_UMC));
 	perf_event_update_userpage(event);
 }
 
+static void amd_uncore_umc_read(struct perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	u64 prev, new, shift;
+	s64 delta;
+
+	shift = COUNTER_SHIFT + 1;
+	prev = local64_read(&hwc->prev_count);
+
+	/*
+	 * UMC counters do not have RDPMC assignments. Read counts directly
+	 * from the corresponding PERF_CTR.
+	 */
+	rdmsrl(hwc->event_base, new);
+
+	/*
+	 * Unlike the other uncore counters, UMC counters saturate and set the
+	 * Overflow bit (bit 48) on overflow. Since they do not roll over,
+	 * proactively reset the corresponding PERF_CTR when bit 47 is set so
+	 * that the counter never gets a chance to saturate.
+	 */
+	if (new & BIT_ULL(63 - COUNTER_SHIFT)) {
+		wrmsrl(hwc->event_base, 0);
+		local64_set(&hwc->prev_count, 0);
+	} else {
+		local64_set(&hwc->prev_count, new);
+	}
+
+	delta = (new << shift) - (prev << shift);
+	delta >>= shift;
+	local64_add(delta, &event->count);
+}
+
 static
 void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
 {
@@ -916,7 +1015,8 @@ int amd_uncore_umc_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
 	u8 group_num_pmcs[UNCORE_GROUP_MAX] = { 0 };
 	union amd_uncore_info info;
 	struct amd_uncore_pmu *pmu;
-	int index = 0, gid, i;
+	int gid, i;
+	u16 index = 0;
 
 	if (pmu_version < 2)
 		return 0;
@@ -948,7 +1048,7 @@ int amd_uncore_umc_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
 	for_each_set_bit(gid, gmask, UNCORE_GROUP_MAX) {
 		for (i = 0; i < group_num_pmus[gid]; i++) {
 			pmu = &uncore->pmus[index];
-			snprintf(pmu->name, sizeof(pmu->name), "amd_umc_%d", index);
+			snprintf(pmu->name, sizeof(pmu->name), "amd_umc_%hu", index);
 			pmu->num_counters = group_num_pmcs[gid] / group_num_pmus[gid];
 			pmu->msr_base = MSR_F19H_UMC_PERF_CTL + i * pmu->num_counters * 2;
 			pmu->rdpmc_base = -1;
@@ -967,7 +1067,7 @@ int amd_uncore_umc_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
 				.del		= amd_uncore_del,
 				.start		= amd_uncore_umc_start,
 				.stop		= amd_uncore_stop,
-				.read		= amd_uncore_read,
+				.read		= amd_uncore_umc_read,
 				.capabilities	= PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
 				.module		= THIS_MODULE,
 			};
diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
index 65ab6460aed4..0c38a31d5fc7 100644
--- a/arch/x86/events/core.c
+++ b/arch/x86/events/core.c
@@ -20,6 +20,7 @@
 #include <linux/export.h>
 #include <linux/init.h>
 #include <linux/kdebug.h>
+#include <linux/kvm_types.h>
 #include <linux/sched/mm.h>
 #include <linux/sched/clock.h>
 #include <linux/uaccess.h>
@@ -32,6 +33,7 @@
 
 #include <asm/apic.h>
 #include <asm/stacktrace.h>
+#include <asm/msr.h>
 #include <asm/nmi.h>
 #include <asm/smp.h>
 #include <asm/alternative.h>
@@ -87,13 +89,19 @@ DEFINE_STATIC_CALL_NULL(x86_pmu_commit_scheduling, *x86_pmu.commit_scheduling);
 DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling,   *x86_pmu.stop_scheduling);
 
 DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task,    *x86_pmu.sched_task);
-DEFINE_STATIC_CALL_NULL(x86_pmu_swap_task_ctx, *x86_pmu.swap_task_ctx);
 
 DEFINE_STATIC_CALL_NULL(x86_pmu_drain_pebs,   *x86_pmu.drain_pebs);
 DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases);
 
 DEFINE_STATIC_CALL_NULL(x86_pmu_filter, *x86_pmu.filter);
 
+DEFINE_STATIC_CALL_NULL(x86_pmu_late_setup, *x86_pmu.late_setup);
+
+DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_enable, *x86_pmu.pebs_enable);
+DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_disable, *x86_pmu.pebs_disable);
+DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_enable_all, *x86_pmu.pebs_enable_all);
+DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_disable_all, *x86_pmu.pebs_disable_all);
+
 /*
  * This one is magic, it will get called even when PMU init fails (because
  * there is no PMU), in which case it should simply return NULL.
@@ -133,7 +141,7 @@ u64 x86_perf_event_update(struct perf_event *event)
 	 */
 	prev_raw_count = local64_read(&hwc->prev_count);
 	do {
-		rdpmcl(hwc->event_base_rdpmc, new_raw_count);
+		new_raw_count = rdpmc(hwc->event_base_rdpmc);
 	} while (!local64_try_cmpxchg(&hwc->prev_count,
 				      &prev_raw_count, new_raw_count));
 
@@ -268,7 +276,7 @@ bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask,
 	 */
 	for_each_set_bit(i, cntr_mask, X86_PMC_IDX_MAX) {
 		reg = x86_pmu_config_addr(i);
-		ret = rdmsrl_safe(reg, &val);
+		ret = rdmsrq_safe(reg, &val);
 		if (ret)
 			goto msr_fail;
 		if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
@@ -282,7 +290,7 @@ bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask,
 
 	if (*(u64 *)fixed_cntr_mask) {
 		reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
-		ret = rdmsrl_safe(reg, &val);
+		ret = rdmsrq_safe(reg, &val);
 		if (ret)
 			goto msr_fail;
 		for_each_set_bit(i, fixed_cntr_mask, X86_PMC_IDX_MAX) {
@@ -313,11 +321,11 @@ bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask,
 	 * (qemu/kvm) that don't trap on the MSR access and always return 0s.
 	 */
 	reg = x86_pmu_event_addr(reg_safe);
-	if (rdmsrl_safe(reg, &val))
+	if (rdmsrq_safe(reg, &val))
 		goto msr_fail;
 	val ^= 0xffffUL;
-	ret = wrmsrl_safe(reg, val);
-	ret |= rdmsrl_safe(reg, &val_new);
+	ret = wrmsrq_safe(reg, val);
+	ret |= rdmsrq_safe(reg, &val_new);
 	if (ret || val != val_new)
 		goto msr_fail;
 
@@ -547,14 +555,22 @@ static inline int precise_br_compat(struct perf_event *event)
 	return m == b;
 }
 
-int x86_pmu_max_precise(void)
+int x86_pmu_max_precise(struct pmu *pmu)
 {
 	int precise = 0;
 
-	/* Support for constant skid */
 	if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
-		precise++;
+		/* arch PEBS */
+		if (x86_pmu.arch_pebs) {
+			precise = 2;
+			if (hybrid(pmu, arch_pebs_cap).pdists)
+				precise++;
 
+			return precise;
+		}
+
+		/* legacy PEBS - support for constant skid */
+		precise++;
 		/* Support for IP fixup */
 		if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
 			precise++;
@@ -562,13 +578,14 @@ int x86_pmu_max_precise(void)
 		if (x86_pmu.pebs_prec_dist)
 			precise++;
 	}
+
 	return precise;
 }
 
 int x86_pmu_hw_config(struct perf_event *event)
 {
 	if (event->attr.precise_ip) {
-		int precise = x86_pmu_max_precise();
+		int precise = x86_pmu_max_precise(event->pmu);
 
 		if (event->attr.precise_ip > precise)
 			return -EOPNOTSUPP;
@@ -628,7 +645,7 @@ int x86_pmu_hw_config(struct perf_event *event)
 	if (event->attr.type == event->pmu->type)
 		event->hw.config |= x86_pmu_get_event_config(event);
 
-	if (event->attr.sample_period && x86_pmu.limit_period) {
+	if (is_sampling_event(event) && !event->attr.freq && x86_pmu.limit_period) {
 		s64 left = event->attr.sample_period;
 		x86_pmu.limit_period(event, &left);
 		if (left > event->attr.sample_period)
@@ -673,6 +690,7 @@ static int __x86_pmu_event_init(struct perf_event *event)
 	event->hw.idx = -1;
 	event->hw.last_cpu = -1;
 	event->hw.last_tag = ~0ULL;
+	event->hw.dyn_constraint = ~0ULL;
 
 	/* mark unused */
 	event->hw.extra_reg.idx = EXTRA_REG_NONE;
@@ -692,13 +710,13 @@ void x86_pmu_disable_all(void)
 
 		if (!test_bit(idx, cpuc->active_mask))
 			continue;
-		rdmsrl(x86_pmu_config_addr(idx), val);
+		rdmsrq(x86_pmu_config_addr(idx), val);
 		if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
 			continue;
 		val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
-		wrmsrl(x86_pmu_config_addr(idx), val);
+		wrmsrq(x86_pmu_config_addr(idx), val);
 		if (is_counter_pair(hwc))
-			wrmsrl(x86_pmu_config_addr(idx + 1), 0);
+			wrmsrq(x86_pmu_config_addr(idx + 1), 0);
 	}
 }
 
@@ -706,7 +724,7 @@ struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data)
 {
 	return static_call(x86_pmu_guest_get_msrs)(nr, data);
 }
-EXPORT_SYMBOL_GPL(perf_guest_get_msrs);
+EXPORT_SYMBOL_FOR_KVM(perf_guest_get_msrs);
 
 /*
  * There may be PMI landing after enabled=0. The PMI hitting could be before or
@@ -753,17 +771,18 @@ void x86_pmu_enable_all(int added)
 	}
 }
 
-static inline int is_x86_event(struct perf_event *event)
+int is_x86_event(struct perf_event *event)
 {
-	int i;
-
-	if (!is_hybrid())
-		return event->pmu == &pmu;
-
-	for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
-		if (event->pmu == &x86_pmu.hybrid_pmu[i].pmu)
-			return true;
-	}
+	/*
+	 * For a non-hybrid platforms, the type of X86 pmu is
+	 * always PERF_TYPE_RAW.
+	 * For a hybrid platform, the PERF_PMU_CAP_EXTENDED_HW_TYPE
+	 * is a unique capability for the X86 PMU.
+	 * Use them to detect a X86 event.
+	 */
+	if (event->pmu->type == PERF_TYPE_RAW ||
+	    event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE)
+		return true;
 
 	return false;
 }
@@ -1298,6 +1317,15 @@ static void x86_pmu_enable(struct pmu *pmu)
 
 	if (cpuc->n_added) {
 		int n_running = cpuc->n_events - cpuc->n_added;
+
+		/*
+		 * The late setup (after counters are scheduled)
+		 * is required for some cases, e.g., PEBS counters
+		 * snapshotting. Because an accurate counter index
+		 * is needed.
+		 */
+		static_call_cond(x86_pmu_late_setup)();
+
 		/*
 		 * apply assignment obtained either from
 		 * hw_perf_group_sched_in() or x86_pmu_enable()
@@ -1326,6 +1354,7 @@ static void x86_pmu_enable(struct pmu *pmu)
 				hwc->state |= PERF_HES_ARCH;
 
 			x86_pmu_stop(event, PERF_EF_UPDATE);
+			cpuc->events[hwc->idx] = NULL;
 		}
 
 		/*
@@ -1347,6 +1376,7 @@ static void x86_pmu_enable(struct pmu *pmu)
 			 * if cpuc->enabled = 0, then no wrmsr as
 			 * per x86_pmu_enable_event()
 			 */
+			cpuc->events[hwc->idx] = event;
 			x86_pmu_start(event, PERF_EF_RELOAD);
 		}
 		cpuc->n_added = 0;
@@ -1410,14 +1440,14 @@ int x86_perf_event_set_period(struct perf_event *event)
 	 */
 	local64_set(&hwc->prev_count, (u64)-left);
 
-	wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
+	wrmsrq(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
 
 	/*
 	 * Sign extend the Merge event counter's upper 16 bits since
 	 * we currently declare a 48-bit counter width
 	 */
 	if (is_counter_pair(hwc))
-		wrmsrl(x86_pmu_event_addr(idx + 1), 0xffff);
+		wrmsrq(x86_pmu_event_addr(idx + 1), 0xffff);
 
 	perf_event_update_userpage(event);
 
@@ -1513,7 +1543,6 @@ static void x86_pmu_start(struct perf_event *event, int flags)
 
 	event->hw.state = 0;
 
-	cpuc->events[idx] = event;
 	__set_bit(idx, cpuc->active_mask);
 	static_call(x86_pmu_enable)(event);
 	perf_event_update_userpage(event);
@@ -1540,10 +1569,10 @@ void perf_event_print_debug(void)
 		return;
 
 	if (x86_pmu.version >= 2) {
-		rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
-		rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
-		rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
-		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
+		rdmsrq(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
+		rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, status);
+		rdmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
+		rdmsrq(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
 
 		pr_info("\n");
 		pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
@@ -1551,19 +1580,19 @@ void perf_event_print_debug(void)
 		pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
 		pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
 		if (pebs_constraints) {
-			rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
+			rdmsrq(MSR_IA32_PEBS_ENABLE, pebs);
 			pr_info("CPU#%d: pebs:       %016llx\n", cpu, pebs);
 		}
 		if (x86_pmu.lbr_nr) {
-			rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
+			rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl);
 			pr_info("CPU#%d: debugctl:   %016llx\n", cpu, debugctl);
 		}
 	}
 	pr_info("CPU#%d: active:     %016llx\n", cpu, *(u64 *)cpuc->active_mask);
 
 	for_each_set_bit(idx, cntr_mask, X86_PMC_IDX_MAX) {
-		rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
-		rdmsrl(x86_pmu_event_addr(idx), pmc_count);
+		rdmsrq(x86_pmu_config_addr(idx), pmc_ctrl);
+		rdmsrq(x86_pmu_event_addr(idx), pmc_count);
 
 		prev_left = per_cpu(pmc_prev_left[idx], cpu);
 
@@ -1577,7 +1606,7 @@ void perf_event_print_debug(void)
 	for_each_set_bit(idx, fixed_cntr_mask, X86_PMC_IDX_MAX) {
 		if (fixed_counter_disabled(idx, cpuc->pmu))
 			continue;
-		rdmsrl(x86_pmu_fixed_ctr_addr(idx), pmc_count);
+		rdmsrq(x86_pmu_fixed_ctr_addr(idx), pmc_count);
 
 		pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
 			cpu, idx, pmc_count);
@@ -1592,7 +1621,6 @@ void x86_pmu_stop(struct perf_event *event, int flags)
 	if (test_bit(hwc->idx, cpuc->active_mask)) {
 		static_call(x86_pmu_disable)(event);
 		__clear_bit(hwc->idx, cpuc->active_mask);
-		cpuc->events[hwc->idx] = NULL;
 		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
 		hwc->state |= PERF_HES_STOPPED;
 	}
@@ -1630,6 +1658,7 @@ static void x86_pmu_del(struct perf_event *event, int flags)
 	 * Not a TXN, therefore cleanup properly.
 	 */
 	x86_pmu_stop(event, PERF_EF_UPDATE);
+	cpuc->events[event->hw.idx] = NULL;
 
 	for (i = 0; i < cpuc->n_events; i++) {
 		if (event == cpuc->event_list[i])
@@ -1673,6 +1702,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs)
 	struct cpu_hw_events *cpuc;
 	struct perf_event *event;
 	int idx, handled = 0;
+	u64 last_period;
 	u64 val;
 
 	cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -1692,6 +1722,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs)
 			continue;
 
 		event = cpuc->events[idx];
+		last_period = event->hw.last_period;
 
 		val = static_call(x86_pmu_update)(event);
 		if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
@@ -1705,13 +1736,11 @@ int x86_pmu_handle_irq(struct pt_regs *regs)
 		if (!static_call(x86_pmu_set_period)(event))
 			continue;
 
-		perf_sample_data_init(&data, 0, event->hw.last_period);
+		perf_sample_data_init(&data, 0, last_period);
 
-		if (has_branch_stack(event))
-			perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
+		perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
 
-		if (perf_event_overflow(event, &data, regs))
-			x86_pmu_stop(event, 0);
+		perf_event_overflow(event, &data, regs);
 	}
 
 	if (handled)
@@ -2029,13 +2058,19 @@ static void x86_pmu_static_call_update(void)
 	static_call_update(x86_pmu_stop_scheduling, x86_pmu.stop_scheduling);
 
 	static_call_update(x86_pmu_sched_task, x86_pmu.sched_task);
-	static_call_update(x86_pmu_swap_task_ctx, x86_pmu.swap_task_ctx);
 
 	static_call_update(x86_pmu_drain_pebs, x86_pmu.drain_pebs);
 	static_call_update(x86_pmu_pebs_aliases, x86_pmu.pebs_aliases);
 
 	static_call_update(x86_pmu_guest_get_msrs, x86_pmu.guest_get_msrs);
 	static_call_update(x86_pmu_filter, x86_pmu.filter);
+
+	static_call_update(x86_pmu_late_setup, x86_pmu.late_setup);
+
+	static_call_update(x86_pmu_pebs_enable, x86_pmu.pebs_enable);
+	static_call_update(x86_pmu_pebs_disable, x86_pmu.pebs_disable);
+	static_call_update(x86_pmu_pebs_enable_all, x86_pmu.pebs_enable_all);
+	static_call_update(x86_pmu_pebs_disable_all, x86_pmu.pebs_disable_all);
 }
 
 static void _x86_pmu_read(struct perf_event *event)
@@ -2045,13 +2080,15 @@ static void _x86_pmu_read(struct perf_event *event)
 
 void x86_pmu_show_pmu_cap(struct pmu *pmu)
 {
-	pr_info("... version:                %d\n",     x86_pmu.version);
-	pr_info("... bit width:              %d\n",     x86_pmu.cntval_bits);
-	pr_info("... generic registers:      %d\n",     x86_pmu_num_counters(pmu));
-	pr_info("... value mask:             %016Lx\n", x86_pmu.cntval_mask);
-	pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
-	pr_info("... fixed-purpose events:   %d\n",     x86_pmu_num_counters_fixed(pmu));
-	pr_info("... event mask:             %016Lx\n", hybrid(pmu, intel_ctrl));
+	pr_info("... version:                   %d\n", x86_pmu.version);
+	pr_info("... bit width:                 %d\n", x86_pmu.cntval_bits);
+	pr_info("... generic counters:          %d\n", x86_pmu_num_counters(pmu));
+	pr_info("... generic bitmap:            %016llx\n", hybrid(pmu, cntr_mask64));
+	pr_info("... fixed-purpose counters:    %d\n", x86_pmu_num_counters_fixed(pmu));
+	pr_info("... fixed-purpose bitmap:      %016llx\n", hybrid(pmu, fixed_cntr_mask64));
+	pr_info("... value mask:                %016llx\n", x86_pmu.cntval_mask);
+	pr_info("... max period:                %016llx\n", x86_pmu.max_period);
+	pr_info("... global_ctrl mask:          %016llx\n", hybrid(pmu, intel_ctrl));
 }
 
 static int __init init_hw_perf_events(void)
@@ -2486,9 +2523,9 @@ void perf_clear_dirty_counters(void)
 			if (!test_bit(i - INTEL_PMC_IDX_FIXED, hybrid(cpuc->pmu, fixed_cntr_mask)))
 				continue;
 
-			wrmsrl(x86_pmu_fixed_ctr_addr(i - INTEL_PMC_IDX_FIXED), 0);
+			wrmsrq(x86_pmu_fixed_ctr_addr(i - INTEL_PMC_IDX_FIXED), 0);
 		} else {
-			wrmsrl(x86_pmu_event_addr(i), 0);
+			wrmsrq(x86_pmu_event_addr(i), 0);
 		}
 	}
 
@@ -2603,7 +2640,9 @@ static ssize_t max_precise_show(struct device *cdev,
 				  struct device_attribute *attr,
 				  char *buf)
 {
-	return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());
+	struct pmu *pmu = dev_get_drvdata(cdev);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise(pmu));
 }
 
 static DEVICE_ATTR_RO(max_precise);
@@ -2626,15 +2665,10 @@ static const struct attribute_group *x86_pmu_attr_groups[] = {
 	NULL,
 };
 
-static void x86_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
-{
-	static_call_cond(x86_pmu_sched_task)(pmu_ctx, sched_in);
-}
-
-static void x86_pmu_swap_task_ctx(struct perf_event_pmu_context *prev_epc,
-				  struct perf_event_pmu_context *next_epc)
+static void x86_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx,
+			       struct task_struct *task, bool sched_in)
 {
-	static_call_cond(x86_pmu_swap_task_ctx)(prev_epc, next_epc);
+	static_call_cond(x86_pmu_sched_task)(pmu_ctx, task, sched_in);
 }
 
 void perf_check_microcode(void)
@@ -2701,7 +2735,6 @@ static struct pmu pmu = {
 
 	.event_idx		= x86_pmu_event_idx,
 	.sched_task		= x86_pmu_sched_task,
-	.swap_task_ctx		= x86_pmu_swap_task_ctx,
 	.check_period		= x86_pmu_check_period,
 
 	.aux_output_match	= x86_pmu_aux_output_match,
@@ -2769,13 +2802,13 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *re
 		return;
 	}
 
-	if (perf_callchain_store(entry, regs->ip))
-		return;
-
-	if (perf_hw_regs(regs))
+	if (perf_hw_regs(regs)) {
+		if (perf_callchain_store(entry, regs->ip))
+			return;
 		unwind_start(&state, current, regs, NULL);
-	else
+	} else {
 		unwind_start(&state, current, NULL, (void *)regs->sp);
+	}
 
 	for (; !unwind_done(&state); unwind_next_frame(&state)) {
 		addr = unwind_get_return_address(&state);
@@ -2799,8 +2832,15 @@ static unsigned long get_segment_base(unsigned int segment)
 #ifdef CONFIG_MODIFY_LDT_SYSCALL
 		struct ldt_struct *ldt;
 
+		/*
+		 * If we're not in a valid context with a real (not just lazy)
+		 * user mm, then don't even try.
+		 */
+		if (!nmi_uaccess_okay())
+			return 0;
+
 		/* IRQs are off, so this synchronizes with smp_store_release */
-		ldt = READ_ONCE(current->active_mm->context.ldt);
+		ldt = smp_load_acquire(&current->mm->context.ldt);
 		if (!ldt || idx >= ldt->nr_entries)
 			return 0;
 
@@ -2818,46 +2858,6 @@ static unsigned long get_segment_base(unsigned int segment)
 	return get_desc_base(desc);
 }
 
-#ifdef CONFIG_UPROBES
-/*
- * Heuristic-based check if uprobe is installed at the function entry.
- *
- * Under assumption of user code being compiled with frame pointers,
- * `push %rbp/%ebp` is a good indicator that we indeed are.
- *
- * Similarly, `endbr64` (assuming 64-bit mode) is also a common pattern.
- * If we get this wrong, captured stack trace might have one extra bogus
- * entry, but the rest of stack trace will still be meaningful.
- */
-static bool is_uprobe_at_func_entry(struct pt_regs *regs)
-{
-	struct arch_uprobe *auprobe;
-
-	if (!current->utask)
-		return false;
-
-	auprobe = current->utask->auprobe;
-	if (!auprobe)
-		return false;
-
-	/* push %rbp/%ebp */
-	if (auprobe->insn[0] == 0x55)
-		return true;
-
-	/* endbr64 (64-bit only) */
-	if (user_64bit_mode(regs) && is_endbr(*(u32 *)auprobe->insn))
-		return true;
-
-	return false;
-}
-
-#else
-static bool is_uprobe_at_func_entry(struct pt_regs *regs)
-{
-	return false;
-}
-#endif /* CONFIG_UPROBES */
-
 #ifdef CONFIG_IA32_EMULATION
 
 #include <linux/compat.h>
@@ -3003,35 +3003,57 @@ static unsigned long code_segment_base(struct pt_regs *regs)
 	return 0;
 }
 
-unsigned long perf_instruction_pointer(struct pt_regs *regs)
+unsigned long perf_arch_instruction_pointer(struct pt_regs *regs)
 {
-	if (perf_guest_state())
-		return perf_guest_get_ip();
-
 	return regs->ip + code_segment_base(regs);
 }
 
-unsigned long perf_misc_flags(struct pt_regs *regs)
+static unsigned long common_misc_flags(struct pt_regs *regs)
 {
-	unsigned int guest_state = perf_guest_state();
-	int misc = 0;
+	if (regs->flags & PERF_EFLAGS_EXACT)
+		return PERF_RECORD_MISC_EXACT_IP;
 
-	if (guest_state) {
-		if (guest_state & PERF_GUEST_USER)
-			misc |= PERF_RECORD_MISC_GUEST_USER;
-		else
-			misc |= PERF_RECORD_MISC_GUEST_KERNEL;
-	} else {
-		if (user_mode(regs))
-			misc |= PERF_RECORD_MISC_USER;
-		else
-			misc |= PERF_RECORD_MISC_KERNEL;
-	}
+	return 0;
+}
 
-	if (regs->flags & PERF_EFLAGS_EXACT)
-		misc |= PERF_RECORD_MISC_EXACT_IP;
+static unsigned long guest_misc_flags(struct pt_regs *regs)
+{
+	unsigned long guest_state = perf_guest_state();
+
+	if (!(guest_state & PERF_GUEST_ACTIVE))
+		return 0;
+
+	if (guest_state & PERF_GUEST_USER)
+		return PERF_RECORD_MISC_GUEST_USER;
+	else
+		return PERF_RECORD_MISC_GUEST_KERNEL;
+
+}
+
+static unsigned long host_misc_flags(struct pt_regs *regs)
+{
+	if (user_mode(regs))
+		return PERF_RECORD_MISC_USER;
+	else
+		return PERF_RECORD_MISC_KERNEL;
+}
+
+unsigned long perf_arch_guest_misc_flags(struct pt_regs *regs)
+{
+	unsigned long flags = common_misc_flags(regs);
+
+	flags |= guest_misc_flags(regs);
+
+	return flags;
+}
+
+unsigned long perf_arch_misc_flags(struct pt_regs *regs)
+{
+	unsigned long flags = common_misc_flags(regs);
+
+	flags |= host_misc_flags(regs);
 
-	return misc;
+	return flags;
 }
 
 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
@@ -3057,7 +3079,7 @@ void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
 	cap->events_mask_len	= x86_pmu.events_mask_len;
 	cap->pebs_ept		= x86_pmu.pebs_ept;
 }
-EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);
+EXPORT_SYMBOL_FOR_KVM(perf_get_x86_pmu_capability);
 
 u64 perf_get_hw_event_config(int hw_event)
 {
@@ -3068,4 +3090,4 @@ u64 perf_get_hw_event_config(int hw_event)
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(perf_get_hw_event_config);
+EXPORT_SYMBOL_FOR_KVM(perf_get_hw_event_config);
diff --git a/arch/x86/events/intel/bts.c b/arch/x86/events/intel/bts.c
index 8f78b0c900ef..cbac54cb3a9e 100644
--- a/arch/x86/events/intel/bts.c
+++ b/arch/x86/events/intel/bts.c
@@ -17,6 +17,7 @@
 
 #include <linux/sizes.h>
 #include <asm/perf_event.h>
+#include <asm/msr.h>
 
 #include "../perf_event.h"
 
@@ -36,7 +37,7 @@ enum {
 	BTS_STATE_ACTIVE,
 };
 
-static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
+static struct bts_ctx __percpu *bts_ctx;
 
 #define BTS_RECORD_SIZE		24
 #define BTS_SAFETY_MARGIN	4080
@@ -58,7 +59,7 @@ struct bts_buffer {
 	local_t		head;
 	unsigned long	end;
 	void		**data_pages;
-	struct bts_phys	buf[];
+	struct bts_phys	buf[] __counted_by(nr_bufs);
 };
 
 static struct pmu bts_pmu;
@@ -80,54 +81,54 @@ static void *
 bts_buffer_setup_aux(struct perf_event *event, void **pages,
 		     int nr_pages, bool overwrite)
 {
-	struct bts_buffer *buf;
+	struct bts_buffer *bb;
 	struct page *page;
 	int cpu = event->cpu;
 	int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
 	unsigned long offset;
 	size_t size = nr_pages << PAGE_SHIFT;
-	int pg, nbuf, pad;
+	int pg, nr_buf, pad;
 
 	/* count all the high order buffers */
-	for (pg = 0, nbuf = 0; pg < nr_pages;) {
+	for (pg = 0, nr_buf = 0; pg < nr_pages;) {
 		page = virt_to_page(pages[pg]);
 		pg += buf_nr_pages(page);
-		nbuf++;
+		nr_buf++;
 	}
 
 	/*
 	 * to avoid interrupts in overwrite mode, only allow one physical
 	 */
-	if (overwrite && nbuf > 1)
+	if (overwrite && nr_buf > 1)
 		return NULL;
 
-	buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
-	if (!buf)
+	bb = kzalloc_node(struct_size(bb, buf, nr_buf), GFP_KERNEL, node);
+	if (!bb)
 		return NULL;
 
-	buf->nr_pages = nr_pages;
-	buf->nr_bufs = nbuf;
-	buf->snapshot = overwrite;
-	buf->data_pages = pages;
-	buf->real_size = size - size % BTS_RECORD_SIZE;
+	bb->nr_pages = nr_pages;
+	bb->nr_bufs = nr_buf;
+	bb->snapshot = overwrite;
+	bb->data_pages = pages;
+	bb->real_size = size - size % BTS_RECORD_SIZE;
 
-	for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
+	for (pg = 0, nr_buf = 0, offset = 0, pad = 0; nr_buf < bb->nr_bufs; nr_buf++) {
 		unsigned int __nr_pages;
 
 		page = virt_to_page(pages[pg]);
 		__nr_pages = buf_nr_pages(page);
-		buf->buf[nbuf].page = page;
-		buf->buf[nbuf].offset = offset;
-		buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
-		buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
-		pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
-		buf->buf[nbuf].size -= pad;
+		bb->buf[nr_buf].page = page;
+		bb->buf[nr_buf].offset = offset;
+		bb->buf[nr_buf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
+		bb->buf[nr_buf].size = buf_size(page) - bb->buf[nr_buf].displacement;
+		pad = bb->buf[nr_buf].size % BTS_RECORD_SIZE;
+		bb->buf[nr_buf].size -= pad;
 
 		pg += __nr_pages;
 		offset += __nr_pages << PAGE_SHIFT;
 	}
 
-	return buf;
+	return bb;
 }
 
 static void bts_buffer_free_aux(void *data)
@@ -135,25 +136,25 @@ static void bts_buffer_free_aux(void *data)
 	kfree(data);
 }
 
-static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
+static unsigned long bts_buffer_offset(struct bts_buffer *bb, unsigned int idx)
 {
-	return buf->buf[idx].offset + buf->buf[idx].displacement;
+	return bb->buf[idx].offset + bb->buf[idx].displacement;
 }
 
 static void
-bts_config_buffer(struct bts_buffer *buf)
+bts_config_buffer(struct bts_buffer *bb)
 {
 	int cpu = raw_smp_processor_id();
 	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
-	struct bts_phys *phys = &buf->buf[buf->cur_buf];
+	struct bts_phys *phys = &bb->buf[bb->cur_buf];
 	unsigned long index, thresh = 0, end = phys->size;
 	struct page *page = phys->page;
 
-	index = local_read(&buf->head);
+	index = local_read(&bb->head);
 
-	if (!buf->snapshot) {
-		if (buf->end < phys->offset + buf_size(page))
-			end = buf->end - phys->offset - phys->displacement;
+	if (!bb->snapshot) {
+		if (bb->end < phys->offset + buf_size(page))
+			end = bb->end - phys->offset - phys->displacement;
 
 		index -= phys->offset + phys->displacement;
 
@@ -168,7 +169,7 @@ bts_config_buffer(struct bts_buffer *buf)
 	ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
 	ds->bts_index = ds->bts_buffer_base + index;
 	ds->bts_absolute_maximum = ds->bts_buffer_base + end;
-	ds->bts_interrupt_threshold = !buf->snapshot
+	ds->bts_interrupt_threshold = !bb->snapshot
 		? ds->bts_buffer_base + thresh
 		: ds->bts_absolute_maximum + BTS_RECORD_SIZE;
 }
@@ -184,16 +185,16 @@ static void bts_update(struct bts_ctx *bts)
 {
 	int cpu = raw_smp_processor_id();
 	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
-	struct bts_buffer *buf = perf_get_aux(&bts->handle);
+	struct bts_buffer *bb = perf_get_aux(&bts->handle);
 	unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
 
-	if (!buf)
+	if (!bb)
 		return;
 
-	head = index + bts_buffer_offset(buf, buf->cur_buf);
-	old = local_xchg(&buf->head, head);
+	head = index + bts_buffer_offset(bb, bb->cur_buf);
+	old = local_xchg(&bb->head, head);
 
-	if (!buf->snapshot) {
+	if (!bb->snapshot) {
 		if (old == head)
 			return;
 
@@ -205,9 +206,9 @@ static void bts_update(struct bts_ctx *bts)
 		 * old and head are always in the same physical buffer, so we
 		 * can subtract them to get the data size.
 		 */
-		local_add(head - old, &buf->data_size);
+		local_add(head - old, &bb->data_size);
 	} else {
-		local_set(&buf->data_size, head);
+		local_set(&bb->data_size, head);
 	}
 
 	/*
@@ -218,7 +219,7 @@ static void bts_update(struct bts_ctx *bts)
 }
 
 static int
-bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle);
+bts_buffer_reset(struct bts_buffer *bb, struct perf_output_handle *handle);
 
 /*
  * Ordering PMU callbacks wrt themselves and the PMI is done by means
@@ -231,18 +232,18 @@ bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle);
 
 static void __bts_event_start(struct perf_event *event)
 {
-	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
-	struct bts_buffer *buf = perf_get_aux(&bts->handle);
+	struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
+	struct bts_buffer *bb = perf_get_aux(&bts->handle);
 	u64 config = 0;
 
-	if (!buf->snapshot)
+	if (!bb->snapshot)
 		config |= ARCH_PERFMON_EVENTSEL_INT;
 	if (!event->attr.exclude_kernel)
 		config |= ARCH_PERFMON_EVENTSEL_OS;
 	if (!event->attr.exclude_user)
 		config |= ARCH_PERFMON_EVENTSEL_USR;
 
-	bts_config_buffer(buf);
+	bts_config_buffer(bb);
 
 	/*
 	 * local barrier to make sure that ds configuration made it
@@ -260,14 +261,14 @@ static void __bts_event_start(struct perf_event *event)
 static void bts_event_start(struct perf_event *event, int flags)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
-	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
-	struct bts_buffer *buf;
+	struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
+	struct bts_buffer *bb;
 
-	buf = perf_aux_output_begin(&bts->handle, event);
-	if (!buf)
+	bb = perf_aux_output_begin(&bts->handle, event);
+	if (!bb)
 		goto fail_stop;
 
-	if (bts_buffer_reset(buf, &bts->handle))
+	if (bts_buffer_reset(bb, &bts->handle))
 		goto fail_end_stop;
 
 	bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
@@ -290,7 +291,7 @@ fail_stop:
 
 static void __bts_event_stop(struct perf_event *event, int state)
 {
-	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
+	struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
 
 	/* ACTIVE -> INACTIVE(PMI)/STOPPED(->stop()) */
 	WRITE_ONCE(bts->state, state);
@@ -305,28 +306,28 @@ static void __bts_event_stop(struct perf_event *event, int state)
 static void bts_event_stop(struct perf_event *event, int flags)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
-	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
-	struct bts_buffer *buf = NULL;
+	struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
+	struct bts_buffer *bb = NULL;
 	int state = READ_ONCE(bts->state);
 
 	if (state == BTS_STATE_ACTIVE)
 		__bts_event_stop(event, BTS_STATE_STOPPED);
 
 	if (state != BTS_STATE_STOPPED)
-		buf = perf_get_aux(&bts->handle);
+		bb = perf_get_aux(&bts->handle);
 
 	event->hw.state |= PERF_HES_STOPPED;
 
 	if (flags & PERF_EF_UPDATE) {
 		bts_update(bts);
 
-		if (buf) {
-			if (buf->snapshot)
+		if (bb) {
+			if (bb->snapshot)
 				bts->handle.head =
-					local_xchg(&buf->data_size,
-						   buf->nr_pages << PAGE_SHIFT);
+					local_xchg(&bb->data_size,
+						   bb->nr_pages << PAGE_SHIFT);
 			perf_aux_output_end(&bts->handle,
-			                    local_xchg(&buf->data_size, 0));
+					    local_xchg(&bb->data_size, 0));
 		}
 
 		cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
@@ -338,9 +339,14 @@ static void bts_event_stop(struct perf_event *event, int flags)
 
 void intel_bts_enable_local(void)
 {
-	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
-	int state = READ_ONCE(bts->state);
+	struct bts_ctx *bts;
+	int state;
+
+	if (!bts_ctx)
+		return;
 
+	bts = this_cpu_ptr(bts_ctx);
+	state = READ_ONCE(bts->state);
 	/*
 	 * Here we transition from INACTIVE to ACTIVE;
 	 * if we instead are STOPPED from the interrupt handler,
@@ -358,7 +364,12 @@ void intel_bts_enable_local(void)
 
 void intel_bts_disable_local(void)
 {
-	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
+	struct bts_ctx *bts;
+
+	if (!bts_ctx)
+		return;
+
+	bts = this_cpu_ptr(bts_ctx);
 
 	/*
 	 * Here we transition from ACTIVE to INACTIVE;
@@ -372,19 +383,19 @@ void intel_bts_disable_local(void)
 }
 
 static int
-bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
+bts_buffer_reset(struct bts_buffer *bb, struct perf_output_handle *handle)
 {
 	unsigned long head, space, next_space, pad, gap, skip, wakeup;
 	unsigned int next_buf;
 	struct bts_phys *phys, *next_phys;
 	int ret;
 
-	if (buf->snapshot)
+	if (bb->snapshot)
 		return 0;
 
-	head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
+	head = handle->head & ((bb->nr_pages << PAGE_SHIFT) - 1);
 
-	phys = &buf->buf[buf->cur_buf];
+	phys = &bb->buf[bb->cur_buf];
 	space = phys->offset + phys->displacement + phys->size - head;
 	pad = space;
 	if (space > handle->size) {
@@ -393,10 +404,10 @@ bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
 	}
 	if (space <= BTS_SAFETY_MARGIN) {
 		/* See if next phys buffer has more space */
-		next_buf = buf->cur_buf + 1;
-		if (next_buf >= buf->nr_bufs)
+		next_buf = bb->cur_buf + 1;
+		if (next_buf >= bb->nr_bufs)
 			next_buf = 0;
-		next_phys = &buf->buf[next_buf];
+		next_phys = &bb->buf[next_buf];
 		gap = buf_size(phys->page) - phys->displacement - phys->size +
 		      next_phys->displacement;
 		skip = pad + gap;
@@ -421,8 +432,8 @@ bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
 				 * anymore, so we must not be racing with
 				 * bts_update().
 				 */
-				buf->cur_buf = next_buf;
-				local_set(&buf->head, head);
+				bb->cur_buf = next_buf;
+				local_set(&bb->head, head);
 			}
 		}
 	}
@@ -435,7 +446,7 @@ bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
 		space -= space % BTS_RECORD_SIZE;
 	}
 
-	buf->end = head + space;
+	bb->end = head + space;
 
 	/*
 	 * If we have no space, the lost notification would have been sent when
@@ -450,12 +461,17 @@ bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
 int intel_bts_interrupt(void)
 {
 	struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds;
-	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
-	struct perf_event *event = bts->handle.event;
-	struct bts_buffer *buf;
+	struct bts_ctx *bts;
+	struct perf_event *event;
+	struct bts_buffer *bb;
 	s64 old_head;
 	int err = -ENOSPC, handled = 0;
 
+	if (!bts_ctx)
+		return 0;
+
+	bts = this_cpu_ptr(bts_ctx);
+	event = bts->handle.event;
 	/*
 	 * The only surefire way of knowing if this NMI is ours is by checking
 	 * the write ptr against the PMI threshold.
@@ -470,8 +486,8 @@ int intel_bts_interrupt(void)
 	if (READ_ONCE(bts->state) == BTS_STATE_STOPPED)
 		return handled;
 
-	buf = perf_get_aux(&bts->handle);
-	if (!buf)
+	bb = perf_get_aux(&bts->handle);
+	if (!bb)
 		return handled;
 
 	/*
@@ -479,26 +495,26 @@ int intel_bts_interrupt(void)
 	 * there's no other way of telling, because the pointer will
 	 * keep moving
 	 */
-	if (buf->snapshot)
+	if (bb->snapshot)
 		return 0;
 
-	old_head = local_read(&buf->head);
+	old_head = local_read(&bb->head);
 	bts_update(bts);
 
 	/* no new data */
-	if (old_head == local_read(&buf->head))
+	if (old_head == local_read(&bb->head))
 		return handled;
 
-	perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0));
+	perf_aux_output_end(&bts->handle, local_xchg(&bb->data_size, 0));
 
-	buf = perf_aux_output_begin(&bts->handle, event);
-	if (buf)
-		err = bts_buffer_reset(buf, &bts->handle);
+	bb = perf_aux_output_begin(&bts->handle, event);
+	if (bb)
+		err = bts_buffer_reset(bb, &bts->handle);
 
 	if (err) {
 		WRITE_ONCE(bts->state, BTS_STATE_STOPPED);
 
-		if (buf) {
+		if (bb) {
 			/*
 			 * BTS_STATE_STOPPED should be visible before
 			 * cleared handle::event
@@ -518,7 +534,7 @@ static void bts_event_del(struct perf_event *event, int mode)
 
 static int bts_event_add(struct perf_event *event, int mode)
 {
-	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
+	struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct hw_perf_event *hwc = &event->hw;
 
@@ -559,7 +575,7 @@ static int bts_event_init(struct perf_event *event)
 	 * to the user in a zero-copy fashion.
 	 */
 	if (event->attr.exclude_kernel) {
-		ret = perf_allow_kernel(&event->attr);
+		ret = perf_allow_kernel();
 		if (ret)
 			return ret;
 	}
@@ -584,7 +600,11 @@ static void bts_event_read(struct perf_event *event)
 
 static __init int bts_init(void)
 {
-	if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
+	if (!boot_cpu_has(X86_FEATURE_DTES64))
+		return -ENODEV;
+
+	x86_pmu.bts = boot_cpu_has(X86_FEATURE_BTS);
+	if (!x86_pmu.bts)
 		return -ENODEV;
 
 	if (boot_cpu_has(X86_FEATURE_PTI)) {
@@ -605,6 +625,10 @@ static __init int bts_init(void)
 		return -ENODEV;
 	}
 
+	bts_ctx = alloc_percpu(struct bts_ctx);
+	if (!bts_ctx)
+		return -ENOMEM;
+
 	bts_pmu.capabilities	= PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE |
 				  PERF_PMU_CAP_EXCLUSIVE;
 	bts_pmu.task_ctx_nr	= perf_sw_context;
@@ -619,4 +643,4 @@ static __init int bts_init(void)
 
 	return perf_pmu_register(&bts_pmu, "intel_bts", -1);
 }
-arch_initcall(bts_init);
+early_initcall(bts_init);
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index d879478db3f5..853fe073bab3 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -23,6 +23,7 @@
 #include <asm/intel_pt.h>
 #include <asm/apic.h>
 #include <asm/cpu_device_id.h>
+#include <asm/msr.h>
 
 #include "../perf_event.h"
 
@@ -397,38 +398,42 @@ static struct event_constraint intel_lnc_event_constraints[] = {
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FETCH_LAT, 6),
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_MEM_BOUND, 7),
 
+	INTEL_EVENT_CONSTRAINT(0x20, 0xf),
+
+	INTEL_UEVENT_CONSTRAINT(0x012a, 0xf),
+	INTEL_UEVENT_CONSTRAINT(0x012b, 0xf),
 	INTEL_UEVENT_CONSTRAINT(0x0148, 0x4),
 	INTEL_UEVENT_CONSTRAINT(0x0175, 0x4),
 
 	INTEL_EVENT_CONSTRAINT(0x2e, 0x3ff),
 	INTEL_EVENT_CONSTRAINT(0x3c, 0x3ff),
-	/*
-	 * Generally event codes < 0x90 are restricted to counters 0-3.
-	 * The 0x2E and 0x3C are exception, which has no restriction.
-	 */
-	INTEL_EVENT_CONSTRAINT_RANGE(0x01, 0x8f, 0xf),
 
-	INTEL_UEVENT_CONSTRAINT(0x01a3, 0xf),
-	INTEL_UEVENT_CONSTRAINT(0x02a3, 0xf),
 	INTEL_UEVENT_CONSTRAINT(0x08a3, 0x4),
 	INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4),
 	INTEL_UEVENT_CONSTRAINT(0x04a4, 0x1),
 	INTEL_UEVENT_CONSTRAINT(0x08a4, 0x1),
 	INTEL_UEVENT_CONSTRAINT(0x10a4, 0x1),
 	INTEL_UEVENT_CONSTRAINT(0x01b1, 0x8),
+	INTEL_UEVENT_CONSTRAINT(0x01cd, 0x3fc),
 	INTEL_UEVENT_CONSTRAINT(0x02cd, 0x3),
-	INTEL_EVENT_CONSTRAINT(0xce, 0x1),
 
 	INTEL_EVENT_CONSTRAINT_RANGE(0xd0, 0xdf, 0xf),
-	/*
-	 * Generally event codes >= 0x90 are likely to have no restrictions.
-	 * The exception are defined as above.
-	 */
-	INTEL_EVENT_CONSTRAINT_RANGE(0x90, 0xfe, 0x3ff),
+
+	INTEL_UEVENT_CONSTRAINT(0x00e0, 0xf),
 
 	EVENT_CONSTRAINT_END
 };
 
+static struct extra_reg intel_lnc_extra_regs[] __read_mostly = {
+	INTEL_UEVENT_EXTRA_REG(0x012a, MSR_OFFCORE_RSP_0, 0xfffffffffffull, RSP_0),
+	INTEL_UEVENT_EXTRA_REG(0x012b, MSR_OFFCORE_RSP_1, 0xfffffffffffull, RSP_1),
+	INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
+	INTEL_UEVENT_EXTRA_REG(0x02c6, MSR_PEBS_FRONTEND, 0x9, FE),
+	INTEL_UEVENT_EXTRA_REG(0x03c6, MSR_PEBS_FRONTEND, 0x7fff1f, FE),
+	INTEL_UEVENT_EXTRA_REG(0x40ad, MSR_PEBS_FRONTEND, 0xf, FE),
+	INTEL_UEVENT_EXTRA_REG(0x04c2, MSR_PEBS_FRONTEND, 0x8, FE),
+	EVENT_EXTRA_END
+};
 
 EVENT_ATTR_STR(mem-loads,	mem_ld_nhm,	"event=0x0b,umask=0x10,ldlat=3");
 EVENT_ATTR_STR(mem-loads,	mem_ld_snb,	"event=0xcd,umask=0x1,ldlat=3");
@@ -2220,6 +2225,18 @@ static struct extra_reg intel_cmt_extra_regs[] __read_mostly = {
 	EVENT_EXTRA_END
 };
 
+EVENT_ATTR_STR(topdown-fe-bound,       td_fe_bound_skt,        "event=0x9c,umask=0x01");
+EVENT_ATTR_STR(topdown-retiring,       td_retiring_skt,        "event=0xc2,umask=0x02");
+EVENT_ATTR_STR(topdown-be-bound,       td_be_bound_skt,        "event=0xa4,umask=0x02");
+
+static struct attribute *skt_events_attrs[] = {
+	EVENT_PTR(td_fe_bound_skt),
+	EVENT_PTR(td_retiring_skt),
+	EVENT_PTR(td_bad_spec_cmt),
+	EVENT_PTR(td_be_bound_skt),
+	NULL,
+};
+
 #define KNL_OT_L2_HITE		BIT_ULL(19) /* Other Tile L2 Hit */
 #define KNL_OT_L2_HITF		BIT_ULL(20) /* Other Tile L2 Hit */
 #define KNL_MCDRAM_LOCAL	BIT_ULL(21)
@@ -2281,7 +2298,7 @@ static __always_inline void __intel_pmu_disable_all(bool bts)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
-	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
+	wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 
 	if (bts && test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
 		intel_pmu_disable_bts();
@@ -2290,7 +2307,7 @@ static __always_inline void __intel_pmu_disable_all(bool bts)
 static __always_inline void intel_pmu_disable_all(void)
 {
 	__intel_pmu_disable_all(true);
-	intel_pmu_pebs_disable_all();
+	static_call_cond(x86_pmu_pebs_disable_all)();
 	intel_pmu_lbr_disable_all();
 }
 
@@ -2302,11 +2319,11 @@ static void __intel_pmu_enable_all(int added, bool pmi)
 	intel_pmu_lbr_enable_all(pmi);
 
 	if (cpuc->fixed_ctrl_val != cpuc->active_fixed_ctrl_val) {
-		wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, cpuc->fixed_ctrl_val);
+		wrmsrq(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, cpuc->fixed_ctrl_val);
 		cpuc->active_fixed_ctrl_val = cpuc->fixed_ctrl_val;
 	}
 
-	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL,
+	wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL,
 	       intel_ctrl & ~cpuc->intel_ctrl_guest_mask);
 
 	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
@@ -2322,7 +2339,7 @@ static void __intel_pmu_enable_all(int added, bool pmi)
 
 static void intel_pmu_enable_all(int added)
 {
-	intel_pmu_pebs_enable_all();
+	static_call_cond(x86_pmu_pebs_enable_all)();
 	__intel_pmu_enable_all(added, false);
 }
 
@@ -2422,12 +2439,12 @@ static void intel_pmu_nhm_workaround(void)
 	}
 
 	for (i = 0; i < 4; i++) {
-		wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, nhm_magic[i]);
-		wrmsrl(MSR_ARCH_PERFMON_PERFCTR0 + i, 0x0);
+		wrmsrq(MSR_ARCH_PERFMON_EVENTSEL0 + i, nhm_magic[i]);
+		wrmsrq(MSR_ARCH_PERFMON_PERFCTR0 + i, 0x0);
 	}
 
-	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0xf);
-	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0x0);
+	wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, 0xf);
+	wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, 0x0);
 
 	for (i = 0; i < 4; i++) {
 		event = cpuc->events[i];
@@ -2437,7 +2454,7 @@ static void intel_pmu_nhm_workaround(void)
 			__x86_pmu_enable_event(&event->hw,
 					ARCH_PERFMON_EVENTSEL_ENABLE);
 		} else
-			wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, 0x0);
+			wrmsrq(MSR_ARCH_PERFMON_EVENTSEL0 + i, 0x0);
 	}
 }
 
@@ -2454,7 +2471,7 @@ static void intel_set_tfa(struct cpu_hw_events *cpuc, bool on)
 
 	if (cpuc->tfa_shadow != val) {
 		cpuc->tfa_shadow = val;
-		wrmsrl(MSR_TSX_FORCE_ABORT, val);
+		wrmsrq(MSR_TSX_FORCE_ABORT, val);
 	}
 }
 
@@ -2485,14 +2502,14 @@ static inline u64 intel_pmu_get_status(void)
 {
 	u64 status;
 
-	rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
+	rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, status);
 
 	return status;
 }
 
 static inline void intel_pmu_ack_status(u64 ack)
 {
-	wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
+	wrmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 }
 
 static inline bool event_is_checkpointed(struct perf_event *event)
@@ -2546,6 +2563,44 @@ static void intel_pmu_disable_fixed(struct perf_event *event)
 	cpuc->fixed_ctrl_val &= ~mask;
 }
 
+static inline void __intel_pmu_update_event_ext(int idx, u64 ext)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	u32 msr;
+
+	if (idx < INTEL_PMC_IDX_FIXED) {
+		msr = MSR_IA32_PMC_V6_GP0_CFG_C +
+		      x86_pmu.addr_offset(idx, false);
+	} else {
+		msr = MSR_IA32_PMC_V6_FX0_CFG_C +
+		      x86_pmu.addr_offset(idx - INTEL_PMC_IDX_FIXED, false);
+	}
+
+	cpuc->cfg_c_val[idx] = ext;
+	wrmsrq(msr, ext);
+}
+
+static void intel_pmu_disable_event_ext(struct perf_event *event)
+{
+	/*
+	 * Only clear CFG_C MSR for PEBS counter group events,
+	 * it avoids the HW counter's value to be added into
+	 * other PEBS records incorrectly after PEBS counter
+	 * group events are disabled.
+	 *
+	 * For other events, it's unnecessary to clear CFG_C MSRs
+	 * since CFG_C doesn't take effect if counter is in
+	 * disabled state. That helps to reduce the WRMSR overhead
+	 * in context switches.
+	 */
+	if (!is_pebs_counter_event_group(event))
+		return;
+
+	__intel_pmu_update_event_ext(event->hw.idx, 0);
+}
+
+DEFINE_STATIC_CALL_NULL(intel_pmu_disable_event_ext, intel_pmu_disable_event_ext);
+
 static void intel_pmu_disable_event(struct perf_event *event)
 {
 	struct hw_perf_event *hwc = &event->hw;
@@ -2554,9 +2609,12 @@ static void intel_pmu_disable_event(struct perf_event *event)
 	switch (idx) {
 	case 0 ... INTEL_PMC_IDX_FIXED - 1:
 		intel_clear_masks(event, idx);
+		static_call_cond(intel_pmu_disable_event_ext)(event);
 		x86_pmu_disable_event(event);
 		break;
 	case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:
+		static_call_cond(intel_pmu_disable_event_ext)(event);
+		fallthrough;
 	case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
 		intel_pmu_disable_fixed(event);
 		break;
@@ -2579,7 +2637,7 @@ static void intel_pmu_disable_event(struct perf_event *event)
 	 * so we don't trigger the event without PEBS bit set.
 	 */
 	if (unlikely(event->attr.precise_ip))
-		intel_pmu_pebs_disable(event);
+		static_call(x86_pmu_pebs_disable)(event);
 }
 
 static void intel_pmu_assign_event(struct perf_event *event, int idx)
@@ -2599,6 +2657,9 @@ static void intel_pmu_del_event(struct perf_event *event)
 		intel_pmu_lbr_del(event);
 	if (event->attr.precise_ip)
 		intel_pmu_pebs_del(event);
+	if (is_pebs_counter_event_group(event) ||
+	    is_acr_event_group(event))
+		this_cpu_ptr(&cpu_hw_events)->n_late_setup--;
 }
 
 static int icl_set_topdown_event_period(struct perf_event *event)
@@ -2615,15 +2676,15 @@ static int icl_set_topdown_event_period(struct perf_event *event)
 	 * Don't need to clear them again.
 	 */
 	if (left == x86_pmu.max_period) {
-		wrmsrl(MSR_CORE_PERF_FIXED_CTR3, 0);
-		wrmsrl(MSR_PERF_METRICS, 0);
+		wrmsrq(MSR_CORE_PERF_FIXED_CTR3, 0);
+		wrmsrq(MSR_PERF_METRICS, 0);
 		hwc->saved_slots = 0;
 		hwc->saved_metric = 0;
 	}
 
 	if ((hwc->saved_slots) && is_slots_event(event)) {
-		wrmsrl(MSR_CORE_PERF_FIXED_CTR3, hwc->saved_slots);
-		wrmsrl(MSR_PERF_METRICS, hwc->saved_metric);
+		wrmsrq(MSR_CORE_PERF_FIXED_CTR3, hwc->saved_slots);
+		wrmsrq(MSR_PERF_METRICS, hwc->saved_metric);
 	}
 
 	perf_event_update_userpage(event);
@@ -2710,7 +2771,7 @@ static void update_saved_topdown_regs(struct perf_event *event, u64 slots,
  * modify by a NMI. PMU has to be disabled before calling this function.
  */
 
-static u64 intel_update_topdown_event(struct perf_event *event, int metric_end)
+static u64 intel_update_topdown_event(struct perf_event *event, int metric_end, u64 *val)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct perf_event *other;
@@ -2718,13 +2779,24 @@ static u64 intel_update_topdown_event(struct perf_event *event, int metric_end)
 	bool reset = true;
 	int idx;
 
-	/* read Fixed counter 3 */
-	rdpmcl((3 | INTEL_PMC_FIXED_RDPMC_BASE), slots);
-	if (!slots)
-		return 0;
+	if (!val) {
+		/* read Fixed counter 3 */
+		slots = rdpmc(3 | INTEL_PMC_FIXED_RDPMC_BASE);
+		if (!slots)
+			return 0;
 
-	/* read PERF_METRICS */
-	rdpmcl(INTEL_PMC_FIXED_RDPMC_METRICS, metrics);
+		/* read PERF_METRICS */
+		metrics = rdpmc(INTEL_PMC_FIXED_RDPMC_METRICS);
+	} else {
+		slots = val[0];
+		metrics = val[1];
+		/*
+		 * Don't reset the PERF_METRICS and Fixed counter 3
+		 * for each PEBS record read. Utilize the RDPMC metrics
+		 * clear mode.
+		 */
+		reset = false;
+	}
 
 	for_each_set_bit(idx, cpuc->active_mask, metric_end + 1) {
 		if (!is_topdown_idx(idx))
@@ -2758,8 +2830,8 @@ static u64 intel_update_topdown_event(struct perf_event *event, int metric_end)
 
 	if (reset) {
 		/* The fixed counter 3 has to be written before the PERF_METRICS. */
-		wrmsrl(MSR_CORE_PERF_FIXED_CTR3, 0);
-		wrmsrl(MSR_PERF_METRICS, 0);
+		wrmsrq(MSR_CORE_PERF_FIXED_CTR3, 0);
+		wrmsrq(MSR_PERF_METRICS, 0);
 		if (event)
 			update_saved_topdown_regs(event, 0, 0, metric_end);
 	}
@@ -2767,44 +2839,55 @@ static u64 intel_update_topdown_event(struct perf_event *event, int metric_end)
 	return slots;
 }
 
-static u64 icl_update_topdown_event(struct perf_event *event)
+static u64 icl_update_topdown_event(struct perf_event *event, u64 *val)
 {
 	return intel_update_topdown_event(event, INTEL_PMC_IDX_METRIC_BASE +
-						 x86_pmu.num_topdown_events - 1);
+						 x86_pmu.num_topdown_events - 1,
+					  val);
 }
 
-DEFINE_STATIC_CALL(intel_pmu_update_topdown_event, x86_perf_event_update);
+DEFINE_STATIC_CALL(intel_pmu_update_topdown_event, intel_pmu_topdown_event_update);
 
-static void intel_pmu_read_topdown_event(struct perf_event *event)
+static void intel_pmu_read_event(struct perf_event *event)
 {
-	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD | PERF_X86_EVENT_TOPDOWN) ||
+	    is_pebs_counter_event_group(event)) {
+		struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+		bool pmu_enabled = cpuc->enabled;
 
-	/* Only need to call update_topdown_event() once for group read. */
-	if ((cpuc->txn_flags & PERF_PMU_TXN_READ) &&
-	    !is_slots_event(event))
-		return;
+		/* Only need to call update_topdown_event() once for group read. */
+		if (is_metric_event(event) && (cpuc->txn_flags & PERF_PMU_TXN_READ))
+			return;
 
-	perf_pmu_disable(event->pmu);
-	static_call(intel_pmu_update_topdown_event)(event);
-	perf_pmu_enable(event->pmu);
-}
+		cpuc->enabled = 0;
+		if (pmu_enabled)
+			intel_pmu_disable_all();
 
-static void intel_pmu_read_event(struct perf_event *event)
-{
-	if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
-		intel_pmu_auto_reload_read(event);
-	else if (is_topdown_count(event))
-		intel_pmu_read_topdown_event(event);
-	else
-		x86_perf_event_update(event);
+		/*
+		 * If the PEBS counters snapshotting is enabled,
+		 * the topdown event is available in PEBS records.
+		 */
+		if (is_topdown_count(event) && !is_pebs_counter_event_group(event))
+			static_call(intel_pmu_update_topdown_event)(event, NULL);
+		else
+			intel_pmu_drain_pebs_buffer();
+
+		cpuc->enabled = pmu_enabled;
+		if (pmu_enabled)
+			intel_pmu_enable_all(0);
+
+		return;
+	}
+
+	x86_perf_event_update(event);
 }
 
 static void intel_pmu_enable_fixed(struct perf_event *event)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct hw_perf_event *hwc = &event->hw;
-	u64 mask, bits = 0;
 	int idx = hwc->idx;
+	u64 bits = 0;
 
 	if (is_topdown_idx(idx)) {
 		struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -2816,6 +2899,9 @@ static void intel_pmu_enable_fixed(struct perf_event *event)
 			return;
 
 		idx = INTEL_PMC_IDX_FIXED_SLOTS;
+
+		if (event->attr.config1 & INTEL_TD_CFG_METRIC_CLEAR)
+			bits |= INTEL_FIXED_3_METRICS_CLEAR;
 	}
 
 	intel_set_masks(event, idx);
@@ -2840,17 +2926,134 @@ static void intel_pmu_enable_fixed(struct perf_event *event)
 
 	idx -= INTEL_PMC_IDX_FIXED;
 	bits = intel_fixed_bits_by_idx(idx, bits);
-	mask = intel_fixed_bits_by_idx(idx, INTEL_FIXED_BITS_MASK);
-
-	if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip) {
+	if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip)
 		bits |= intel_fixed_bits_by_idx(idx, ICL_FIXED_0_ADAPTIVE);
-		mask |= intel_fixed_bits_by_idx(idx, ICL_FIXED_0_ADAPTIVE);
-	}
 
-	cpuc->fixed_ctrl_val &= ~mask;
+	cpuc->fixed_ctrl_val &= ~intel_fixed_bits_by_idx(idx, INTEL_FIXED_BITS_MASK);
 	cpuc->fixed_ctrl_val |= bits;
 }
 
+static void intel_pmu_config_acr(int idx, u64 mask, u32 reload)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	int msr_b, msr_c;
+	int msr_offset;
+
+	if (!mask && !cpuc->acr_cfg_b[idx])
+		return;
+
+	if (idx < INTEL_PMC_IDX_FIXED) {
+		msr_b = MSR_IA32_PMC_V6_GP0_CFG_B;
+		msr_c = MSR_IA32_PMC_V6_GP0_CFG_C;
+		msr_offset = x86_pmu.addr_offset(idx, false);
+	} else {
+		msr_b = MSR_IA32_PMC_V6_FX0_CFG_B;
+		msr_c = MSR_IA32_PMC_V6_FX0_CFG_C;
+		msr_offset = x86_pmu.addr_offset(idx - INTEL_PMC_IDX_FIXED, false);
+	}
+
+	if (cpuc->acr_cfg_b[idx] != mask) {
+		wrmsrl(msr_b + msr_offset, mask);
+		cpuc->acr_cfg_b[idx] = mask;
+	}
+	/* Only need to update the reload value when there is a valid config value. */
+	if (mask && cpuc->acr_cfg_c[idx] != reload) {
+		wrmsrl(msr_c + msr_offset, reload);
+		cpuc->acr_cfg_c[idx] = reload;
+	}
+}
+
+static void intel_pmu_enable_acr(struct perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+
+	if (!is_acr_event_group(event) || !event->attr.config2) {
+		/*
+		 * The disable doesn't clear the ACR CFG register.
+		 * Check and clear the ACR CFG register.
+		 */
+		intel_pmu_config_acr(hwc->idx, 0, 0);
+		return;
+	}
+
+	intel_pmu_config_acr(hwc->idx, hwc->config1, -hwc->sample_period);
+}
+
+DEFINE_STATIC_CALL_NULL(intel_pmu_enable_acr_event, intel_pmu_enable_acr);
+
+static void intel_pmu_enable_event_ext(struct perf_event *event)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	struct hw_perf_event *hwc = &event->hw;
+	union arch_pebs_index old, new;
+	struct arch_pebs_cap cap;
+	u64 ext = 0;
+
+	cap = hybrid(cpuc->pmu, arch_pebs_cap);
+
+	if (event->attr.precise_ip) {
+		u64 pebs_data_cfg = intel_get_arch_pebs_data_config(event);
+
+		ext |= ARCH_PEBS_EN;
+		if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD)
+			ext |= (-hwc->sample_period) & ARCH_PEBS_RELOAD;
+
+		if (pebs_data_cfg && cap.caps) {
+			if (pebs_data_cfg & PEBS_DATACFG_MEMINFO)
+				ext |= ARCH_PEBS_AUX & cap.caps;
+
+			if (pebs_data_cfg & PEBS_DATACFG_GP)
+				ext |= ARCH_PEBS_GPR & cap.caps;
+
+			if (pebs_data_cfg & PEBS_DATACFG_XMMS)
+				ext |= ARCH_PEBS_VECR_XMM & cap.caps;
+
+			if (pebs_data_cfg & PEBS_DATACFG_LBRS)
+				ext |= ARCH_PEBS_LBR & cap.caps;
+
+			if (pebs_data_cfg &
+			    (PEBS_DATACFG_CNTR_MASK << PEBS_DATACFG_CNTR_SHIFT))
+				ext |= ARCH_PEBS_CNTR_GP & cap.caps;
+
+			if (pebs_data_cfg &
+			    (PEBS_DATACFG_FIX_MASK << PEBS_DATACFG_FIX_SHIFT))
+				ext |= ARCH_PEBS_CNTR_FIXED & cap.caps;
+
+			if (pebs_data_cfg & PEBS_DATACFG_METRICS)
+				ext |= ARCH_PEBS_CNTR_METRICS & cap.caps;
+		}
+
+		if (cpuc->n_pebs == cpuc->n_large_pebs)
+			new.thresh = ARCH_PEBS_THRESH_MULTI;
+		else
+			new.thresh = ARCH_PEBS_THRESH_SINGLE;
+
+		rdmsrq(MSR_IA32_PEBS_INDEX, old.whole);
+		if (new.thresh != old.thresh || !old.en) {
+			if (old.thresh == ARCH_PEBS_THRESH_MULTI && old.wr > 0) {
+				/*
+				 * Large PEBS was enabled.
+				 * Drain PEBS buffer before applying the single PEBS.
+				 */
+				intel_pmu_drain_pebs_buffer();
+			} else {
+				new.wr = 0;
+				new.full = 0;
+				new.en = 1;
+				wrmsrq(MSR_IA32_PEBS_INDEX, new.whole);
+			}
+		}
+	}
+
+	if (is_pebs_counter_event_group(event))
+		ext |= ARCH_PEBS_CNTR_ALLOW;
+
+	if (cpuc->cfg_c_val[hwc->idx] != ext)
+		__intel_pmu_update_event_ext(hwc->idx, ext);
+}
+
+DEFINE_STATIC_CALL_NULL(intel_pmu_enable_event_ext, intel_pmu_enable_event_ext);
+
 static void intel_pmu_enable_event(struct perf_event *event)
 {
 	u64 enable_mask = ARCH_PERFMON_EVENTSEL_ENABLE;
@@ -2858,16 +3061,21 @@ static void intel_pmu_enable_event(struct perf_event *event)
 	int idx = hwc->idx;
 
 	if (unlikely(event->attr.precise_ip))
-		intel_pmu_pebs_enable(event);
+		static_call(x86_pmu_pebs_enable)(event);
 
 	switch (idx) {
 	case 0 ... INTEL_PMC_IDX_FIXED - 1:
 		if (branch_sample_counters(event))
 			enable_mask |= ARCH_PERFMON_EVENTSEL_BR_CNTR;
 		intel_set_masks(event, idx);
+		static_call_cond(intel_pmu_enable_acr_event)(event);
+		static_call_cond(intel_pmu_enable_event_ext)(event);
 		__x86_pmu_enable_event(hwc, enable_mask);
 		break;
 	case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:
+		static_call_cond(intel_pmu_enable_acr_event)(event);
+		static_call_cond(intel_pmu_enable_event_ext)(event);
+		fallthrough;
 	case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
 		intel_pmu_enable_fixed(event);
 		break;
@@ -2885,12 +3093,52 @@ static void intel_pmu_enable_event(struct perf_event *event)
 	}
 }
 
+static void intel_pmu_acr_late_setup(struct cpu_hw_events *cpuc)
+{
+	struct perf_event *event, *leader;
+	int i, j, idx;
+
+	for (i = 0; i < cpuc->n_events; i++) {
+		leader = cpuc->event_list[i];
+		if (!is_acr_event_group(leader))
+			continue;
+
+		/* The ACR events must be contiguous. */
+		for (j = i; j < cpuc->n_events; j++) {
+			event = cpuc->event_list[j];
+			if (event->group_leader != leader->group_leader)
+				break;
+			for_each_set_bit(idx, (unsigned long *)&event->attr.config2, X86_PMC_IDX_MAX) {
+				if (i + idx >= cpuc->n_events ||
+				    !is_acr_event_group(cpuc->event_list[i + idx]))
+					return;
+				__set_bit(cpuc->assign[i + idx], (unsigned long *)&event->hw.config1);
+			}
+		}
+		i = j - 1;
+	}
+}
+
+void intel_pmu_late_setup(void)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+	if (!cpuc->n_late_setup)
+		return;
+
+	intel_pmu_pebs_late_setup(cpuc);
+	intel_pmu_acr_late_setup(cpuc);
+}
+
 static void intel_pmu_add_event(struct perf_event *event)
 {
 	if (event->attr.precise_ip)
 		intel_pmu_pebs_add(event);
 	if (intel_pmu_needs_branch_stack(event))
 		intel_pmu_lbr_add(event);
+	if (is_pebs_counter_event_group(event) ||
+	    is_acr_event_group(event))
+		this_cpu_ptr(&cpu_hw_events)->n_late_setup++;
 }
 
 /*
@@ -2908,7 +3156,7 @@ int intel_pmu_save_and_restart(struct perf_event *event)
 	 */
 	if (unlikely(event_is_checkpointed(event))) {
 		/* No race with NMIs because the counter should not be armed */
-		wrmsrl(event->hw.event_base, 0);
+		wrmsrq(event->hw.event_base, 0);
 		local64_set(&event->hw.prev_count, 0);
 	}
 	return static_call(x86_pmu_set_period)(event);
@@ -2925,7 +3173,7 @@ static int intel_pmu_set_period(struct perf_event *event)
 static u64 intel_pmu_update(struct perf_event *event)
 {
 	if (unlikely(is_topdown_count(event)))
-		return static_call(intel_pmu_update_topdown_event)(event);
+		return static_call(intel_pmu_update_topdown_event)(event, NULL);
 
 	return x86_perf_event_update(event);
 }
@@ -2947,13 +3195,13 @@ static void intel_pmu_reset(void)
 	pr_info("clearing PMU state on CPU#%d\n", smp_processor_id());
 
 	for_each_set_bit(idx, cntr_mask, INTEL_PMC_MAX_GENERIC) {
-		wrmsrl_safe(x86_pmu_config_addr(idx), 0ull);
-		wrmsrl_safe(x86_pmu_event_addr(idx),  0ull);
+		wrmsrq_safe(x86_pmu_config_addr(idx), 0ull);
+		wrmsrq_safe(x86_pmu_event_addr(idx),  0ull);
 	}
 	for_each_set_bit(idx, fixed_cntr_mask, INTEL_PMC_MAX_FIXED) {
 		if (fixed_counter_disabled(idx, cpuc->pmu))
 			continue;
-		wrmsrl_safe(x86_pmu_fixed_ctr_addr(idx), 0ull);
+		wrmsrq_safe(x86_pmu_fixed_ctr_addr(idx), 0ull);
 	}
 
 	if (ds)
@@ -2962,7 +3210,7 @@ static void intel_pmu_reset(void)
 	/* Ack all overflows and disable fixed counters */
 	if (x86_pmu.version >= 2) {
 		intel_pmu_ack_status(intel_pmu_get_status());
-		wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
+		wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 	}
 
 	/* Reset LBRs and LBR freezing */
@@ -3006,8 +3254,7 @@ static void x86_pmu_handle_guest_pebs(struct pt_regs *regs,
 			continue;
 
 		perf_sample_data_init(data, 0, event->hw.last_period);
-		if (perf_event_overflow(event, data, regs))
-			x86_pmu_stop(event, 0);
+		perf_event_overflow(event, data, regs);
 
 		/* Inject one fake event is enough. */
 		break;
@@ -3020,7 +3267,6 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	int bit;
 	int handled = 0;
-	u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl);
 
 	inc_irq_stat(apic_perf_irqs);
 
@@ -3063,8 +3309,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 
 		handled++;
 		x86_pmu_handle_guest_pebs(regs, &data);
-		x86_pmu.drain_pebs(regs, &data);
-		status &= intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI;
+		static_call(x86_pmu_drain_pebs)(regs, &data);
 
 		/*
 		 * PMI throttle may be triggered, which stops the PEBS event.
@@ -3074,7 +3319,29 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 		 * Update the MSR if pebs_enabled is changed.
 		 */
 		if (pebs_enabled != cpuc->pebs_enabled)
-			wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
+			wrmsrq(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
+
+		/*
+		 * Above PEBS handler (PEBS counters snapshotting) has updated fixed
+		 * counter 3 and perf metrics counts if they are in counter group,
+		 * unnecessary to update again.
+		 */
+		if (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS] &&
+		    is_pebs_counter_event_group(cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS]))
+			status &= ~GLOBAL_STATUS_PERF_METRICS_OVF_BIT;
+	}
+
+	/*
+	 * Arch PEBS sets bit 54 in the global status register
+	 */
+	if (__test_and_clear_bit(GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT,
+				 (unsigned long *)&status)) {
+		handled++;
+		static_call(x86_pmu_drain_pebs)(regs, &data);
+
+		if (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS] &&
+		    is_pebs_counter_event_group(cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS]))
+			status &= ~GLOBAL_STATUS_PERF_METRICS_OVF_BIT;
 	}
 
 	/*
@@ -3091,9 +3358,11 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 	 */
 	if (__test_and_clear_bit(GLOBAL_STATUS_PERF_METRICS_OVF_BIT, (unsigned long *)&status)) {
 		handled++;
-		static_call(intel_pmu_update_topdown_event)(NULL);
+		static_call(intel_pmu_update_topdown_event)(NULL, NULL);
 	}
 
+	status &= hybrid(cpuc->pmu, intel_ctrl);
+
 	/*
 	 * Checkpointed counters can lead to 'spurious' PMIs because the
 	 * rollback caused by the PMI will have cleared the overflow status
@@ -3103,22 +3372,45 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 
 	for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
 		struct perf_event *event = cpuc->events[bit];
+		u64 last_period;
 
 		handled++;
 
 		if (!test_bit(bit, cpuc->active_mask))
 			continue;
 
+		/*
+		 * There may be unprocessed PEBS records in the PEBS buffer,
+		 * which still stores the previous values.
+		 * Process those records first before handling the latest value.
+		 * For example,
+		 * A is a regular counter
+		 * B is a PEBS event which reads A
+		 * C is a PEBS event
+		 *
+		 * The following can happen:
+		 * B-assist			A=1
+		 * C				A=2
+		 * B-assist			A=3
+		 * A-overflow-PMI		A=4
+		 * C-assist-PMI (PEBS buffer)	A=5
+		 *
+		 * The PEBS buffer has to be drained before handling the A-PMI
+		 */
+		if (is_pebs_counter_event_group(event))
+			static_call(x86_pmu_drain_pebs)(regs, &data);
+
+		last_period = event->hw.last_period;
+
 		if (!intel_pmu_save_and_restart(event))
 			continue;
 
-		perf_sample_data_init(&data, 0, event->hw.last_period);
+		perf_sample_data_init(&data, 0, last_period);
 
 		if (has_branch_stack(event))
 			intel_pmu_lbr_save_brstack(&data, cpuc, event);
 
-		if (perf_event_overflow(event, &data, regs))
-			x86_pmu_stop(event, 0);
+		perf_event_overflow(event, &data, regs);
 	}
 
 	return handled;
@@ -3680,10 +3972,9 @@ intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
 	if (cpuc->excl_cntrs)
 		return intel_get_excl_constraints(cpuc, event, idx, c2);
 
-	/* Not all counters support the branch counter feature. */
-	if (branch_sample_counters(event)) {
+	if (event->hw.dyn_constraint != ~0ULL) {
 		c2 = dyn_constraint(cpuc, c2, idx);
-		c2->idxmsk64 &= x86_pmu.lbr_counters;
+		c2->idxmsk64 &= event->hw.dyn_constraint;
 		c2->weight = hweight64(c2->idxmsk64);
 	}
 
@@ -3867,7 +4158,9 @@ static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event)
 	if (!event->attr.exclude_kernel)
 		flags &= ~PERF_SAMPLE_REGS_USER;
 	if (event->attr.sample_regs_user & ~PEBS_GP_REGS)
-		flags &= ~(PERF_SAMPLE_REGS_USER | PERF_SAMPLE_REGS_INTR);
+		flags &= ~PERF_SAMPLE_REGS_USER;
+	if (event->attr.sample_regs_intr & ~PEBS_GP_REGS)
+		flags &= ~PERF_SAMPLE_REGS_INTR;
 	return flags;
 }
 
@@ -3945,6 +4238,132 @@ static inline bool intel_pmu_has_cap(struct perf_event *event, int idx)
 	return test_bit(idx, (unsigned long *)&intel_cap->capabilities);
 }
 
+static u64 intel_pmu_freq_start_period(struct perf_event *event)
+{
+	int type = event->attr.type;
+	u64 config, factor;
+	s64 start;
+
+	/*
+	 * The 127 is the lowest possible recommended SAV (sample after value)
+	 * for a 4000 freq (default freq), according to the event list JSON file.
+	 * Also, assume the workload is idle 50% time.
+	 */
+	factor = 64 * 4000;
+	if (type != PERF_TYPE_HARDWARE && type != PERF_TYPE_HW_CACHE)
+		goto end;
+
+	/*
+	 * The estimation of the start period in the freq mode is
+	 * based on the below assumption.
+	 *
+	 * For a cycles or an instructions event, 1GHZ of the
+	 * underlying platform, 1 IPC. The workload is idle 50% time.
+	 * The start period = 1,000,000,000 * 1 / freq / 2.
+	 *		    = 500,000,000 / freq
+	 *
+	 * Usually, the branch-related events occur less than the
+	 * instructions event. According to the Intel event list JSON
+	 * file, the SAV (sample after value) of a branch-related event
+	 * is usually 1/4 of an instruction event.
+	 * The start period of branch-related events = 125,000,000 / freq.
+	 *
+	 * The cache-related events occurs even less. The SAV is usually
+	 * 1/20 of an instruction event.
+	 * The start period of cache-related events = 25,000,000 / freq.
+	 */
+	config = event->attr.config & PERF_HW_EVENT_MASK;
+	if (type == PERF_TYPE_HARDWARE) {
+		switch (config) {
+		case PERF_COUNT_HW_CPU_CYCLES:
+		case PERF_COUNT_HW_INSTRUCTIONS:
+		case PERF_COUNT_HW_BUS_CYCLES:
+		case PERF_COUNT_HW_STALLED_CYCLES_FRONTEND:
+		case PERF_COUNT_HW_STALLED_CYCLES_BACKEND:
+		case PERF_COUNT_HW_REF_CPU_CYCLES:
+			factor = 500000000;
+			break;
+		case PERF_COUNT_HW_BRANCH_INSTRUCTIONS:
+		case PERF_COUNT_HW_BRANCH_MISSES:
+			factor = 125000000;
+			break;
+		case PERF_COUNT_HW_CACHE_REFERENCES:
+		case PERF_COUNT_HW_CACHE_MISSES:
+			factor = 25000000;
+			break;
+		default:
+			goto end;
+		}
+	}
+
+	if (type == PERF_TYPE_HW_CACHE)
+		factor = 25000000;
+end:
+	/*
+	 * Usually, a prime or a number with less factors (close to prime)
+	 * is chosen as an SAV, which makes it less likely that the sampling
+	 * period synchronizes with some periodic event in the workload.
+	 * Minus 1 to make it at least avoiding values near power of twos
+	 * for the default freq.
+	 */
+	start = DIV_ROUND_UP_ULL(factor, event->attr.sample_freq) - 1;
+
+	if (start > x86_pmu.max_period)
+		start = x86_pmu.max_period;
+
+	if (x86_pmu.limit_period)
+		x86_pmu.limit_period(event, &start);
+
+	return start;
+}
+
+static inline bool intel_pmu_has_acr(struct pmu *pmu)
+{
+	return !!hybrid(pmu, acr_cause_mask64);
+}
+
+static bool intel_pmu_is_acr_group(struct perf_event *event)
+{
+	/* The group leader has the ACR flag set */
+	if (is_acr_event_group(event))
+		return true;
+
+	/* The acr_mask is set */
+	if (event->attr.config2)
+		return true;
+
+	return false;
+}
+
+static inline bool intel_pmu_has_pebs_counter_group(struct pmu *pmu)
+{
+	u64 caps;
+
+	if (x86_pmu.intel_cap.pebs_format >= 6 && x86_pmu.intel_cap.pebs_baseline)
+		return true;
+
+	caps = hybrid(pmu, arch_pebs_cap).caps;
+	if (x86_pmu.arch_pebs && (caps & ARCH_PEBS_CNTR_MASK))
+		return true;
+
+	return false;
+}
+
+static inline void intel_pmu_set_acr_cntr_constr(struct perf_event *event,
+						 u64 *cause_mask, int *num)
+{
+	event->hw.dyn_constraint &= hybrid(event->pmu, acr_cntr_mask64);
+	*cause_mask |= event->attr.config2;
+	*num += 1;
+}
+
+static inline void intel_pmu_set_acr_caused_constr(struct perf_event *event,
+						   int idx, u64 cause_mask)
+{
+	if (test_bit(idx, (unsigned long *)&cause_mask))
+		event->hw.dyn_constraint &= hybrid(event->pmu, acr_cause_mask64);
+}
+
 static int intel_pmu_hw_config(struct perf_event *event)
 {
 	int ret = x86_pmu_hw_config(event);
@@ -3956,20 +4375,37 @@ static int intel_pmu_hw_config(struct perf_event *event)
 	if (ret)
 		return ret;
 
+	if (event->attr.freq && event->attr.sample_freq) {
+		event->hw.sample_period = intel_pmu_freq_start_period(event);
+		event->hw.last_period = event->hw.sample_period;
+		local64_set(&event->hw.period_left, event->hw.sample_period);
+	}
+
 	if (event->attr.precise_ip) {
+		struct arch_pebs_cap pebs_cap = hybrid(event->pmu, arch_pebs_cap);
+
 		if ((event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_FIXED_VLBR_EVENT)
 			return -EINVAL;
 
 		if (!(event->attr.freq || (event->attr.wakeup_events && !event->attr.watermark))) {
 			event->hw.flags |= PERF_X86_EVENT_AUTO_RELOAD;
-			if (!(event->attr.sample_type &
-			      ~intel_pmu_large_pebs_flags(event))) {
+			if (!(event->attr.sample_type & ~intel_pmu_large_pebs_flags(event)) &&
+			    !has_aux_action(event)) {
 				event->hw.flags |= PERF_X86_EVENT_LARGE_PEBS;
 				event->attach_state |= PERF_ATTACH_SCHED_CB;
 			}
 		}
 		if (x86_pmu.pebs_aliases)
 			x86_pmu.pebs_aliases(event);
+
+		if (x86_pmu.arch_pebs) {
+			u64 cntr_mask = hybrid(event->pmu, intel_ctrl) &
+						~GLOBAL_CTRL_EN_PERF_METRICS;
+			u64 pebs_mask = event->attr.precise_ip >= 3 ?
+						pebs_cap.pdists : pebs_cap.counters;
+			if (cntr_mask != pebs_mask)
+				event->hw.dyn_constraint &= pebs_mask;
+		}
 	}
 
 	if (needs_branch_stack(event)) {
@@ -4000,15 +4436,19 @@ static int intel_pmu_hw_config(struct perf_event *event)
 		leader = event->group_leader;
 		if (branch_sample_call_stack(leader))
 			return -EINVAL;
-		if (branch_sample_counters(leader))
+		if (branch_sample_counters(leader)) {
 			num++;
+			leader->hw.dyn_constraint &= x86_pmu.lbr_counters;
+		}
 		leader->hw.flags |= PERF_X86_EVENT_BRANCH_COUNTERS;
 
 		for_each_sibling_event(sibling, leader) {
 			if (branch_sample_call_stack(sibling))
 				return -EINVAL;
-			if (branch_sample_counters(sibling))
+			if (branch_sample_counters(sibling)) {
 				num++;
+				sibling->hw.dyn_constraint &= x86_pmu.lbr_counters;
+			}
 		}
 
 		if (num > fls(x86_pmu.lbr_counters))
@@ -4056,6 +4496,100 @@ static int intel_pmu_hw_config(struct perf_event *event)
 		event->hw.flags |= PERF_X86_EVENT_PEBS_VIA_PT;
 	}
 
+	if ((event->attr.sample_type & PERF_SAMPLE_READ) &&
+	    intel_pmu_has_pebs_counter_group(event->pmu) &&
+	    is_sampling_event(event) &&
+	    event->attr.precise_ip)
+		event->group_leader->hw.flags |= PERF_X86_EVENT_PEBS_CNTR;
+
+	if (intel_pmu_has_acr(event->pmu) && intel_pmu_is_acr_group(event)) {
+		struct perf_event *sibling, *leader = event->group_leader;
+		struct pmu *pmu = event->pmu;
+		bool has_sw_event = false;
+		int num = 0, idx = 0;
+		u64 cause_mask = 0;
+
+		/* Not support perf metrics */
+		if (is_metric_event(event))
+			return -EINVAL;
+
+		/* Not support freq mode */
+		if (event->attr.freq)
+			return -EINVAL;
+
+		/* PDist is not supported */
+		if (event->attr.config2 && event->attr.precise_ip > 2)
+			return -EINVAL;
+
+		/* The reload value cannot exceeds the max period */
+		if (event->attr.sample_period > x86_pmu.max_period)
+			return -EINVAL;
+		/*
+		 * The counter-constraints of each event cannot be finalized
+		 * unless the whole group is scanned. However, it's hard
+		 * to know whether the event is the last one of the group.
+		 * Recalculate the counter-constraints for each event when
+		 * adding a new event.
+		 *
+		 * The group is traversed twice, which may be optimized later.
+		 * In the first round,
+		 * - Find all events which do reload when other events
+		 *   overflow and set the corresponding counter-constraints
+		 * - Add all events, which can cause other events reload,
+		 *   in the cause_mask
+		 * - Error out if the number of events exceeds the HW limit
+		 * - The ACR events must be contiguous.
+		 *   Error out if there are non-X86 events between ACR events.
+		 *   This is not a HW limit, but a SW limit.
+		 *   With the assumption, the intel_pmu_acr_late_setup() can
+		 *   easily convert the event idx to counter idx without
+		 *   traversing the whole event list.
+		 */
+		if (!is_x86_event(leader))
+			return -EINVAL;
+
+		if (leader->attr.config2)
+			intel_pmu_set_acr_cntr_constr(leader, &cause_mask, &num);
+
+		if (leader->nr_siblings) {
+			for_each_sibling_event(sibling, leader) {
+				if (!is_x86_event(sibling)) {
+					has_sw_event = true;
+					continue;
+				}
+				if (!sibling->attr.config2)
+					continue;
+				if (has_sw_event)
+					return -EINVAL;
+				intel_pmu_set_acr_cntr_constr(sibling, &cause_mask, &num);
+			}
+		}
+		if (leader != event && event->attr.config2) {
+			if (has_sw_event)
+				return -EINVAL;
+			intel_pmu_set_acr_cntr_constr(event, &cause_mask, &num);
+		}
+
+		if (hweight64(cause_mask) > hweight64(hybrid(pmu, acr_cause_mask64)) ||
+		    num > hweight64(hybrid(event->pmu, acr_cntr_mask64)))
+			return -EINVAL;
+		/*
+		 * In the second round, apply the counter-constraints for
+		 * the events which can cause other events reload.
+		 */
+		intel_pmu_set_acr_caused_constr(leader, idx++, cause_mask);
+
+		if (leader->nr_siblings) {
+			for_each_sibling_event(sibling, leader)
+				intel_pmu_set_acr_caused_constr(sibling, idx++, cause_mask);
+		}
+
+		if (leader != event)
+			intel_pmu_set_acr_caused_constr(event, idx, cause_mask);
+
+		leader->hw.flags |= PERF_X86_EVENT_ACR;
+	}
+
 	if ((event->attr.type == PERF_TYPE_HARDWARE) ||
 	    (event->attr.type == PERF_TYPE_HW_CACHE))
 		return 0;
@@ -4071,7 +4605,12 @@ static int intel_pmu_hw_config(struct perf_event *event)
 	 * is used in a metrics group, it too cannot support sampling.
 	 */
 	if (intel_pmu_has_cap(event, PERF_CAP_METRICS_IDX) && is_topdown_event(event)) {
-		if (event->attr.config1 || event->attr.config2)
+		/* The metrics_clear can only be set for the slots event */
+		if (event->attr.config1 &&
+		    (!is_slots_event(event) || (event->attr.config1 & ~INTEL_TD_CFG_METRIC_CLEAR)))
+			return -EINVAL;
+
+		if (event->attr.config2)
 			return -EINVAL;
 
 		/*
@@ -4150,7 +4689,7 @@ static int intel_pmu_hw_config(struct perf_event *event)
 	if (x86_pmu.version < 3)
 		return -EINVAL;
 
-	ret = perf_allow_cpu(&event->attr);
+	ret = perf_allow_cpu();
 	if (ret)
 		return ret;
 
@@ -4198,7 +4737,7 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data)
 		.guest = intel_ctrl & ~cpuc->intel_ctrl_host_mask & ~pebs_mask,
 	};
 
-	if (!x86_pmu.pebs)
+	if (!x86_pmu.ds_pebs)
 		return arr;
 
 	/*
@@ -4239,7 +4778,7 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data)
 	arr[pebs_enable] = (struct perf_guest_switch_msr){
 		.msr = MSR_IA32_PEBS_ENABLE,
 		.host = cpuc->pebs_enabled & ~cpuc->intel_ctrl_guest_mask,
-		.guest = pebs_mask & ~cpuc->intel_ctrl_host_mask,
+		.guest = pebs_mask & ~cpuc->intel_ctrl_host_mask & kvm_pmu->pebs_enable,
 	};
 
 	if (arr[pebs_enable].host) {
@@ -4588,9 +5127,9 @@ static int adl_hw_config(struct perf_event *event)
 	return -EOPNOTSUPP;
 }
 
-static enum hybrid_cpu_type adl_get_hybrid_cpu_type(void)
+static enum intel_cpu_type adl_get_hybrid_cpu_type(void)
 {
-	return HYBRID_INTEL_CORE;
+	return INTEL_CPU_TYPE_CORE;
 }
 
 static inline bool erratum_hsw11(struct perf_event *event)
@@ -4599,6 +5138,28 @@ static inline bool erratum_hsw11(struct perf_event *event)
 		X86_CONFIG(.event=0xc0, .umask=0x01);
 }
 
+static struct event_constraint *
+arl_h_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
+			  struct perf_event *event)
+{
+	struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
+
+	if (pmu->pmu_type == hybrid_tiny)
+		return cmt_get_event_constraints(cpuc, idx, event);
+
+	return mtl_get_event_constraints(cpuc, idx, event);
+}
+
+static int arl_h_hw_config(struct perf_event *event)
+{
+	struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
+
+	if (pmu->pmu_type == hybrid_tiny)
+		return intel_pmu_hw_config(event);
+
+	return adl_hw_config(event);
+}
+
 /*
  * The HSW11 requires a period larger than 100 which is the same as the BDM11.
  * A minimum period of 128 is enforced as well for the INST_RETIRED.ALL.
@@ -4658,6 +5219,8 @@ PMU_FORMAT_ATTR(in_tx,  "config:32"	);
 PMU_FORMAT_ATTR(in_tx_cp, "config:33"	);
 PMU_FORMAT_ATTR(eq,	"config:36"	); /* v6 + */
 
+PMU_FORMAT_ATTR(metrics_clear,	"config1:0"); /* PERF_CAPABILITIES.RDPMC_METRICS_CLEAR */
+
 static ssize_t umask2_show(struct device *dev,
 			   struct device_attribute *attr,
 			   char *page)
@@ -4677,6 +5240,7 @@ static struct device_attribute format_attr_umask2  =
 static struct attribute *format_evtsel_ext_attrs[] = {
 	&format_attr_umask2.attr,
 	&format_attr_eq.attr,
+	&format_attr_metrics_clear.attr,
 	NULL
 };
 
@@ -4701,6 +5265,13 @@ evtsel_ext_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 	if (i == 1)
 		return (mask & ARCH_PERFMON_EVENTSEL_EQ) ? attr->mode : 0;
 
+	/* PERF_CAPABILITIES.RDPMC_METRICS_CLEAR */
+	if (i == 2) {
+		union perf_capabilities intel_cap = hybrid(dev_get_drvdata(dev), intel_cap);
+
+		return intel_cap.rdpmc_metrics_clear ? attr->mode : 0;
+	}
+
 	return 0;
 }
 
@@ -4764,7 +5335,7 @@ int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
 			goto err;
 	}
 
-	if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS | PMU_FL_TFA | PMU_FL_BR_CNTR)) {
+	if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS | PMU_FL_TFA | PMU_FL_DYN_CONSTRAINT)) {
 		size_t sz = X86_PMC_IDX_MAX * sizeof(struct event_constraint);
 
 		cpuc->constraint_list = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu));
@@ -4796,7 +5367,13 @@ err:
 
 static int intel_pmu_cpu_prepare(int cpu)
 {
-	return intel_cpuc_prepare(&per_cpu(cpu_hw_events, cpu), cpu);
+	int ret;
+
+	ret = intel_cpuc_prepare(&per_cpu(cpu_hw_events, cpu), cpu);
+	if (ret)
+		return ret;
+
+	return alloc_arch_pebs_buf_on_cpu(cpu);
 }
 
 static void flip_smm_bit(void *data)
@@ -4841,6 +5418,163 @@ static void intel_pmu_check_event_constraints(struct event_constraint *event_con
 					      u64 fixed_cntr_mask,
 					      u64 intel_ctrl);
 
+enum dyn_constr_type {
+	DYN_CONSTR_NONE,
+	DYN_CONSTR_BR_CNTR,
+	DYN_CONSTR_ACR_CNTR,
+	DYN_CONSTR_ACR_CAUSE,
+	DYN_CONSTR_PEBS,
+	DYN_CONSTR_PDIST,
+
+	DYN_CONSTR_MAX,
+};
+
+static const char * const dyn_constr_type_name[] = {
+	[DYN_CONSTR_NONE] = "a normal event",
+	[DYN_CONSTR_BR_CNTR] = "a branch counter logging event",
+	[DYN_CONSTR_ACR_CNTR] = "an auto-counter reload event",
+	[DYN_CONSTR_ACR_CAUSE] = "an auto-counter reload cause event",
+	[DYN_CONSTR_PEBS] = "a PEBS event",
+	[DYN_CONSTR_PDIST] = "a PEBS PDIST event",
+};
+
+static void __intel_pmu_check_dyn_constr(struct event_constraint *constr,
+					 enum dyn_constr_type type, u64 mask)
+{
+	struct event_constraint *c1, *c2;
+	int new_weight, check_weight;
+	u64 new_mask, check_mask;
+
+	for_each_event_constraint(c1, constr) {
+		new_mask = c1->idxmsk64 & mask;
+		new_weight = hweight64(new_mask);
+
+		/* ignore topdown perf metrics event */
+		if (c1->idxmsk64 & INTEL_PMC_MSK_TOPDOWN)
+			continue;
+
+		if (!new_weight && fls64(c1->idxmsk64) < INTEL_PMC_IDX_FIXED) {
+			pr_info("The event 0x%llx is not supported as %s.\n",
+				c1->code, dyn_constr_type_name[type]);
+		}
+
+		if (new_weight <= 1)
+			continue;
+
+		for_each_event_constraint(c2, c1 + 1) {
+			bool check_fail = false;
+
+			check_mask = c2->idxmsk64 & mask;
+			check_weight = hweight64(check_mask);
+
+			if (c2->idxmsk64 & INTEL_PMC_MSK_TOPDOWN ||
+			    !check_weight)
+				continue;
+
+			/* The same constraints or no overlap */
+			if (new_mask == check_mask ||
+			    (new_mask ^ check_mask) == (new_mask | check_mask))
+				continue;
+
+			/*
+			 * A scheduler issue may be triggered in the following cases.
+			 * - Two overlap constraints have the same weight.
+			 *   E.g., A constraints: 0x3, B constraints: 0x6
+			 *   event	counter		failure case
+			 *   B		PMC[2:1]	1
+			 *   A		PMC[1:0]	0
+			 *   A		PMC[1:0]	FAIL
+			 * - Two overlap constraints have different weight.
+			 *   The constraint has a low weight, but has high last bit.
+			 *   E.g., A constraints: 0x7, B constraints: 0xC
+			 *   event	counter		failure case
+			 *   B		PMC[3:2]	2
+			 *   A		PMC[2:0]	0
+			 *   A		PMC[2:0]	1
+			 *   A		PMC[2:0]	FAIL
+			 */
+			if (new_weight == check_weight) {
+				check_fail = true;
+			} else if (new_weight < check_weight) {
+				if ((new_mask | check_mask) != check_mask &&
+				    fls64(new_mask) > fls64(check_mask))
+					check_fail = true;
+			} else {
+				if ((new_mask | check_mask) != new_mask &&
+				    fls64(new_mask) < fls64(check_mask))
+					check_fail = true;
+			}
+
+			if (check_fail) {
+				pr_info("The two events 0x%llx and 0x%llx may not be "
+					"fully scheduled under some circumstances as "
+					"%s.\n",
+					c1->code, c2->code, dyn_constr_type_name[type]);
+			}
+		}
+	}
+}
+
+static void intel_pmu_check_dyn_constr(struct pmu *pmu,
+				       struct event_constraint *constr,
+				       u64 cntr_mask)
+{
+	enum dyn_constr_type i;
+	u64 mask;
+
+	for (i = DYN_CONSTR_NONE; i < DYN_CONSTR_MAX; i++) {
+		mask = 0;
+		switch (i) {
+		case DYN_CONSTR_NONE:
+			mask = cntr_mask;
+			break;
+		case DYN_CONSTR_BR_CNTR:
+			if (x86_pmu.flags & PMU_FL_BR_CNTR)
+				mask = x86_pmu.lbr_counters;
+			break;
+		case DYN_CONSTR_ACR_CNTR:
+			mask = hybrid(pmu, acr_cntr_mask64) & GENMASK_ULL(INTEL_PMC_MAX_GENERIC - 1, 0);
+			break;
+		case DYN_CONSTR_ACR_CAUSE:
+			if (hybrid(pmu, acr_cntr_mask64) == hybrid(pmu, acr_cause_mask64))
+				continue;
+			mask = hybrid(pmu, acr_cause_mask64) & GENMASK_ULL(INTEL_PMC_MAX_GENERIC - 1, 0);
+			break;
+		case DYN_CONSTR_PEBS:
+			if (x86_pmu.arch_pebs)
+				mask = hybrid(pmu, arch_pebs_cap).counters;
+			break;
+		case DYN_CONSTR_PDIST:
+			if (x86_pmu.arch_pebs)
+				mask = hybrid(pmu, arch_pebs_cap).pdists;
+			break;
+		default:
+			pr_warn("Unsupported dynamic constraint type %d\n", i);
+		}
+
+		if (mask)
+			__intel_pmu_check_dyn_constr(constr, i, mask);
+	}
+}
+
+static void intel_pmu_check_event_constraints_all(struct pmu *pmu)
+{
+	struct event_constraint *event_constraints = hybrid(pmu, event_constraints);
+	struct event_constraint *pebs_constraints = hybrid(pmu, pebs_constraints);
+	u64 cntr_mask = hybrid(pmu, cntr_mask64);
+	u64 fixed_cntr_mask = hybrid(pmu, fixed_cntr_mask64);
+	u64 intel_ctrl = hybrid(pmu, intel_ctrl);
+
+	intel_pmu_check_event_constraints(event_constraints, cntr_mask,
+					  fixed_cntr_mask, intel_ctrl);
+
+	if (event_constraints)
+		intel_pmu_check_dyn_constr(pmu, event_constraints, cntr_mask);
+
+	if (pebs_constraints)
+		intel_pmu_check_dyn_constr(pmu, pebs_constraints, cntr_mask);
+}
+
 static void intel_pmu_check_extra_regs(struct extra_reg *extra_regs);
 
 static inline bool intel_pmu_broken_perf_cap(void)
@@ -4853,27 +5587,94 @@ static inline bool intel_pmu_broken_perf_cap(void)
 	return false;
 }
 
-static void update_pmu_cap(struct x86_hybrid_pmu *pmu)
+static inline void __intel_update_pmu_caps(struct pmu *pmu)
+{
+	struct pmu *dest_pmu = pmu ? pmu : x86_get_pmu(smp_processor_id());
+
+	if (hybrid(pmu, arch_pebs_cap).caps & ARCH_PEBS_VECR_XMM)
+		dest_pmu->capabilities |= PERF_PMU_CAP_EXTENDED_REGS;
+}
+
+static inline void __intel_update_large_pebs_flags(struct pmu *pmu)
+{
+	u64 caps = hybrid(pmu, arch_pebs_cap).caps;
+
+	x86_pmu.large_pebs_flags |= PERF_SAMPLE_TIME;
+	if (caps & ARCH_PEBS_LBR)
+		x86_pmu.large_pebs_flags |= PERF_SAMPLE_BRANCH_STACK;
+	if (caps & ARCH_PEBS_CNTR_MASK)
+		x86_pmu.large_pebs_flags |= PERF_SAMPLE_READ;
+
+	if (!(caps & ARCH_PEBS_AUX))
+		x86_pmu.large_pebs_flags &= ~PERF_SAMPLE_DATA_SRC;
+	if (!(caps & ARCH_PEBS_GPR)) {
+		x86_pmu.large_pebs_flags &=
+			~(PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
+	}
+}
+
+#define counter_mask(_gp, _fixed) ((_gp) | ((u64)(_fixed) << INTEL_PMC_IDX_FIXED))
+
+static void update_pmu_cap(struct pmu *pmu)
 {
-	unsigned int sub_bitmaps, eax, ebx, ecx, edx;
+	unsigned int eax, ebx, ecx, edx;
+	union cpuid35_eax eax_0;
+	union cpuid35_ebx ebx_0;
+	u64 cntrs_mask = 0;
+	u64 pebs_mask = 0;
+	u64 pdists_mask = 0;
 
-	cpuid(ARCH_PERFMON_EXT_LEAF, &sub_bitmaps, &ebx, &ecx, &edx);
+	cpuid(ARCH_PERFMON_EXT_LEAF, &eax_0.full, &ebx_0.full, &ecx, &edx);
 
-	if (ebx & ARCH_PERFMON_EXT_UMASK2)
-		pmu->config_mask |= ARCH_PERFMON_EVENTSEL_UMASK2;
-	if (ebx & ARCH_PERFMON_EXT_EQ)
-		pmu->config_mask |= ARCH_PERFMON_EVENTSEL_EQ;
+	if (ebx_0.split.umask2)
+		hybrid(pmu, config_mask) |= ARCH_PERFMON_EVENTSEL_UMASK2;
+	if (ebx_0.split.eq)
+		hybrid(pmu, config_mask) |= ARCH_PERFMON_EVENTSEL_EQ;
 
-	if (sub_bitmaps & ARCH_PERFMON_NUM_COUNTER_LEAF_BIT) {
+	if (eax_0.split.cntr_subleaf) {
 		cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_NUM_COUNTER_LEAF,
 			    &eax, &ebx, &ecx, &edx);
-		pmu->cntr_mask64 = eax;
-		pmu->fixed_cntr_mask64 = ebx;
+		hybrid(pmu, cntr_mask64) = eax;
+		hybrid(pmu, fixed_cntr_mask64) = ebx;
+		cntrs_mask = counter_mask(eax, ebx);
+	}
+
+	if (eax_0.split.acr_subleaf) {
+		cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_ACR_LEAF,
+			    &eax, &ebx, &ecx, &edx);
+		/* The mask of the counters which can be reloaded */
+		hybrid(pmu, acr_cntr_mask64) = counter_mask(eax, ebx);
+		/* The mask of the counters which can cause a reload of reloadable counters */
+		hybrid(pmu, acr_cause_mask64) = counter_mask(ecx, edx);
+	}
+
+	/* Bits[5:4] should be set simultaneously if arch-PEBS is supported */
+	if (eax_0.split.pebs_caps_subleaf && eax_0.split.pebs_cnts_subleaf) {
+		cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_PEBS_CAP_LEAF,
+			    &eax, &ebx, &ecx, &edx);
+		hybrid(pmu, arch_pebs_cap).caps = (u64)ebx << 32;
+
+		cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_PEBS_COUNTER_LEAF,
+			    &eax, &ebx, &ecx, &edx);
+		pebs_mask   = counter_mask(eax, ecx);
+		pdists_mask = counter_mask(ebx, edx);
+		hybrid(pmu, arch_pebs_cap).counters = pebs_mask;
+		hybrid(pmu, arch_pebs_cap).pdists = pdists_mask;
+
+		if (WARN_ON((pebs_mask | pdists_mask) & ~cntrs_mask)) {
+			x86_pmu.arch_pebs = 0;
+		} else {
+			__intel_update_pmu_caps(pmu);
+			__intel_update_large_pebs_flags(pmu);
+		}
+	} else {
+		WARN_ON(x86_pmu.arch_pebs == 1);
+		x86_pmu.arch_pebs = 0;
 	}
 
 	if (!intel_pmu_broken_perf_cap()) {
 		/* Perf Metric (Bit 15) and PEBS via PT (Bit 16) are hybrid enumeration */
-		rdmsrl(MSR_IA32_PERF_CAPABILITIES, pmu->intel_cap.capabilities);
+		rdmsrq(MSR_IA32_PERF_CAPABILITIES, hybrid(pmu, intel_cap).capabilities);
 	}
 }
 
@@ -4887,26 +5688,19 @@ static void intel_pmu_check_hybrid_pmus(struct x86_hybrid_pmu *pmu)
 						0, x86_pmu_num_counters(&pmu->pmu), 0, 0);
 
 	if (pmu->intel_cap.perf_metrics)
-		pmu->intel_ctrl |= 1ULL << GLOBAL_CTRL_EN_PERF_METRICS;
+		pmu->intel_ctrl |= GLOBAL_CTRL_EN_PERF_METRICS;
 	else
-		pmu->intel_ctrl &= ~(1ULL << GLOBAL_CTRL_EN_PERF_METRICS);
+		pmu->intel_ctrl &= ~GLOBAL_CTRL_EN_PERF_METRICS;
 
-	if (pmu->intel_cap.pebs_output_pt_available)
-		pmu->pmu.capabilities |= PERF_PMU_CAP_AUX_OUTPUT;
-	else
-		pmu->pmu.capabilities &= ~PERF_PMU_CAP_AUX_OUTPUT;
-
-	intel_pmu_check_event_constraints(pmu->event_constraints,
-					  pmu->cntr_mask64,
-					  pmu->fixed_cntr_mask64,
-					  pmu->intel_ctrl);
+	intel_pmu_check_event_constraints_all(&pmu->pmu);
 
 	intel_pmu_check_extra_regs(pmu->extra_regs);
 }
 
 static struct x86_hybrid_pmu *find_hybrid_pmu_for_cpu(void)
 {
-	u8 cpu_type = get_this_hybrid_cpu_type();
+	struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
+	enum intel_cpu_type cpu_type = c->topo.intel_type;
 	int i;
 
 	/*
@@ -4915,7 +5709,7 @@ static struct x86_hybrid_pmu *find_hybrid_pmu_for_cpu(void)
 	 * on it. There should be a fixup function provided for these
 	 * troublesome CPUs (->get_hybrid_cpu_type).
 	 */
-	if (cpu_type == HYBRID_INTEL_NONE) {
+	if (cpu_type == INTEL_CPU_TYPE_UNKNOWN) {
 		if (x86_pmu.get_hybrid_cpu_type)
 			cpu_type = x86_pmu.get_hybrid_cpu_type();
 		else
@@ -4924,17 +5718,26 @@ static struct x86_hybrid_pmu *find_hybrid_pmu_for_cpu(void)
 
 	/*
 	 * This essentially just maps between the 'hybrid_cpu_type'
-	 * and 'hybrid_pmu_type' enums:
+	 * and 'hybrid_pmu_type' enums except for ARL-H processor
+	 * which needs to compare atom uarch native id since ARL-H
+	 * contains two different atom uarchs.
 	 */
 	for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
 		enum hybrid_pmu_type pmu_type = x86_pmu.hybrid_pmu[i].pmu_type;
+		u32 native_id;
 
-		if (cpu_type == HYBRID_INTEL_CORE &&
-		    pmu_type == hybrid_big)
-			return &x86_pmu.hybrid_pmu[i];
-		if (cpu_type == HYBRID_INTEL_ATOM &&
-		    pmu_type == hybrid_small)
+		if (cpu_type == INTEL_CPU_TYPE_CORE && pmu_type == hybrid_big)
 			return &x86_pmu.hybrid_pmu[i];
+		if (cpu_type == INTEL_CPU_TYPE_ATOM) {
+			if (x86_pmu.num_hybrid_pmus == 2 && pmu_type == hybrid_small)
+				return &x86_pmu.hybrid_pmu[i];
+
+			native_id = c->topo.intel_native_model_id;
+			if (native_id == INTEL_ATOM_SKT_NATIVE_ID && pmu_type == hybrid_small)
+				return &x86_pmu.hybrid_pmu[i];
+			if (native_id == INTEL_ATOM_CMT_NATIVE_ID && pmu_type == hybrid_tiny)
+				return &x86_pmu.hybrid_pmu[i];
+		}
 	}
 
 	return NULL;
@@ -4955,7 +5758,7 @@ static bool init_hybrid_pmu(int cpu)
 		goto end;
 
 	if (this_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))
-		update_pmu_cap(pmu);
+		update_pmu_cap(&pmu->pmu);
 
 	intel_pmu_check_hybrid_pmus(pmu);
 
@@ -4964,9 +5767,6 @@ static bool init_hybrid_pmu(int cpu)
 
 	pr_info("%s PMU driver: ", pmu->name);
 
-	if (pmu->intel_cap.pebs_output_pt_available)
-		pr_cont("PEBS-via-PT ");
-
 	pr_cont("\n");
 
 	x86_pmu_show_pmu_cap(&pmu->pmu);
@@ -4988,9 +5788,13 @@ static void intel_pmu_cpu_starting(int cpu)
 		return;
 
 	init_debug_store_on_cpu(cpu);
+	init_arch_pebs_on_cpu(cpu);
 	/*
-	 * Deal with CPUs that don't clear their LBRs on power-up.
+	 * Deal with CPUs that don't clear their LBRs on power-up, and that may
+	 * even boot with LBRs enabled.
 	 */
+	if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && x86_pmu.lbr_nr)
+		msr_clear_bit(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR_BIT);
 	intel_pmu_lbr_reset();
 
 	cpuc->lbr_sel = NULL;
@@ -5016,13 +5820,15 @@ static void intel_pmu_cpu_starting(int cpu)
 	if (!is_hybrid() && x86_pmu.intel_cap.perf_metrics) {
 		union perf_capabilities perf_cap;
 
-		rdmsrl(MSR_IA32_PERF_CAPABILITIES, perf_cap.capabilities);
+		rdmsrq(MSR_IA32_PERF_CAPABILITIES, perf_cap.capabilities);
 		if (!perf_cap.perf_metrics) {
 			x86_pmu.intel_cap.perf_metrics = 0;
-			x86_pmu.intel_ctrl &= ~(1ULL << GLOBAL_CTRL_EN_PERF_METRICS);
+			x86_pmu.intel_ctrl &= ~GLOBAL_CTRL_EN_PERF_METRICS;
 		}
 	}
 
+	__intel_update_pmu_caps(cpuc->pmu);
+
 	if (!cpuc->shared_regs)
 		return;
 
@@ -5082,6 +5888,7 @@ static void free_excl_cntrs(struct cpu_hw_events *cpuc)
 static void intel_pmu_cpu_dying(int cpu)
 {
 	fini_debug_store_on_cpu(cpu);
+	fini_arch_pebs_on_cpu(cpu);
 }
 
 void intel_cpuc_finish(struct cpu_hw_events *cpuc)
@@ -5102,6 +5909,7 @@ static void intel_pmu_cpu_dead(int cpu)
 {
 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
 
+	release_arch_pebs_buf_on_cpu(cpu);
 	intel_cpuc_finish(cpuc);
 
 	if (is_hybrid() && cpuc->pmu)
@@ -5109,16 +5917,10 @@ static void intel_pmu_cpu_dead(int cpu)
 }
 
 static void intel_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx,
-				 bool sched_in)
+				 struct task_struct *task, bool sched_in)
 {
 	intel_pmu_pebs_sched_task(pmu_ctx, sched_in);
-	intel_pmu_lbr_sched_task(pmu_ctx, sched_in);
-}
-
-static void intel_pmu_swap_task_ctx(struct perf_event_pmu_context *prev_epc,
-				    struct perf_event_pmu_context *next_epc)
-{
-	intel_pmu_lbr_swap_task_ctx(prev_epc, next_epc);
+	intel_pmu_lbr_sched_task(pmu_ctx, task, sched_in);
 }
 
 static int intel_pmu_check_period(struct perf_event *event, u64 value)
@@ -5289,7 +6091,6 @@ static __initconst const struct x86_pmu intel_pmu = {
 
 	.guest_get_msrs		= intel_guest_get_msrs,
 	.sched_task		= intel_pmu_sched_task,
-	.swap_task_ctx		= intel_pmu_swap_task_ctx,
 
 	.check_period		= intel_pmu_check_period,
 
@@ -5336,46 +6137,36 @@ static __init void intel_clovertown_quirk(void)
 	 * these chips.
 	 */
 	pr_warn("PEBS disabled due to CPU errata\n");
-	x86_pmu.pebs = 0;
+	x86_pmu.ds_pebs = 0;
 	x86_pmu.pebs_constraints = NULL;
 }
 
-static const struct x86_cpu_desc isolation_ucodes[] = {
-	INTEL_CPU_DESC(INTEL_HASWELL,		 3, 0x0000001f),
-	INTEL_CPU_DESC(INTEL_HASWELL_L,		 1, 0x0000001e),
-	INTEL_CPU_DESC(INTEL_HASWELL_G,		 1, 0x00000015),
-	INTEL_CPU_DESC(INTEL_HASWELL_X,		 2, 0x00000037),
-	INTEL_CPU_DESC(INTEL_HASWELL_X,		 4, 0x0000000a),
-	INTEL_CPU_DESC(INTEL_BROADWELL,		 4, 0x00000023),
-	INTEL_CPU_DESC(INTEL_BROADWELL_G,	 1, 0x00000014),
-	INTEL_CPU_DESC(INTEL_BROADWELL_D,	 2, 0x00000010),
-	INTEL_CPU_DESC(INTEL_BROADWELL_D,	 3, 0x07000009),
-	INTEL_CPU_DESC(INTEL_BROADWELL_D,	 4, 0x0f000009),
-	INTEL_CPU_DESC(INTEL_BROADWELL_D,	 5, 0x0e000002),
-	INTEL_CPU_DESC(INTEL_BROADWELL_X,	 1, 0x0b000014),
-	INTEL_CPU_DESC(INTEL_SKYLAKE_X,		 3, 0x00000021),
-	INTEL_CPU_DESC(INTEL_SKYLAKE_X,		 4, 0x00000000),
-	INTEL_CPU_DESC(INTEL_SKYLAKE_X,		 5, 0x00000000),
-	INTEL_CPU_DESC(INTEL_SKYLAKE_X,		 6, 0x00000000),
-	INTEL_CPU_DESC(INTEL_SKYLAKE_X,		 7, 0x00000000),
-	INTEL_CPU_DESC(INTEL_SKYLAKE_X,		11, 0x00000000),
-	INTEL_CPU_DESC(INTEL_SKYLAKE_L,		 3, 0x0000007c),
-	INTEL_CPU_DESC(INTEL_SKYLAKE,		 3, 0x0000007c),
-	INTEL_CPU_DESC(INTEL_KABYLAKE,		 9, 0x0000004e),
-	INTEL_CPU_DESC(INTEL_KABYLAKE_L,	 9, 0x0000004e),
-	INTEL_CPU_DESC(INTEL_KABYLAKE_L,	10, 0x0000004e),
-	INTEL_CPU_DESC(INTEL_KABYLAKE_L,	11, 0x0000004e),
-	INTEL_CPU_DESC(INTEL_KABYLAKE_L,	12, 0x0000004e),
-	INTEL_CPU_DESC(INTEL_KABYLAKE,		10, 0x0000004e),
-	INTEL_CPU_DESC(INTEL_KABYLAKE,		11, 0x0000004e),
-	INTEL_CPU_DESC(INTEL_KABYLAKE,		12, 0x0000004e),
-	INTEL_CPU_DESC(INTEL_KABYLAKE,		13, 0x0000004e),
+static const struct x86_cpu_id isolation_ucodes[] = {
+	X86_MATCH_VFM_STEPS(INTEL_HASWELL,	 3,  3, 0x0000001f),
+	X86_MATCH_VFM_STEPS(INTEL_HASWELL_L,	 1,  1, 0x0000001e),
+	X86_MATCH_VFM_STEPS(INTEL_HASWELL_G,	 1,  1, 0x00000015),
+	X86_MATCH_VFM_STEPS(INTEL_HASWELL_X,	 2,  2, 0x00000037),
+	X86_MATCH_VFM_STEPS(INTEL_HASWELL_X,	 4,  4, 0x0000000a),
+	X86_MATCH_VFM_STEPS(INTEL_BROADWELL,	 4,  4, 0x00000023),
+	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_G,	 1,  1, 0x00000014),
+	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D,	 2,  2, 0x00000010),
+	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D,	 3,  3, 0x07000009),
+	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D,	 4,  4, 0x0f000009),
+	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D,	 5,  5, 0x0e000002),
+	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_X,	 1,  1, 0x0b000014),
+	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X,	 3,  3, 0x00000021),
+	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X,	 4,  7, 0x00000000),
+	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X,	11, 11, 0x00000000),
+	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_L,	 3,  3, 0x0000007c),
+	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE,	 3,  3, 0x0000007c),
+	X86_MATCH_VFM_STEPS(INTEL_KABYLAKE,	 9, 13, 0x0000004e),
+	X86_MATCH_VFM_STEPS(INTEL_KABYLAKE_L,	 9, 12, 0x0000004e),
 	{}
 };
 
 static void intel_check_pebs_isolation(void)
 {
-	x86_pmu.pebs_no_isolation = !x86_cpu_has_min_microcode_rev(isolation_ucodes);
+	x86_pmu.pebs_no_isolation = !x86_match_min_microcode_rev(isolation_ucodes);
 }
 
 static __init void intel_pebs_isolation_quirk(void)
@@ -5385,16 +6176,16 @@ static __init void intel_pebs_isolation_quirk(void)
 	intel_check_pebs_isolation();
 }
 
-static const struct x86_cpu_desc pebs_ucodes[] = {
-	INTEL_CPU_DESC(INTEL_SANDYBRIDGE,	7, 0x00000028),
-	INTEL_CPU_DESC(INTEL_SANDYBRIDGE_X,	6, 0x00000618),
-	INTEL_CPU_DESC(INTEL_SANDYBRIDGE_X,	7, 0x0000070c),
+static const struct x86_cpu_id pebs_ucodes[] = {
+	X86_MATCH_VFM_STEPS(INTEL_SANDYBRIDGE,	7, 7, 0x00000028),
+	X86_MATCH_VFM_STEPS(INTEL_SANDYBRIDGE_X,	6, 6, 0x00000618),
+	X86_MATCH_VFM_STEPS(INTEL_SANDYBRIDGE_X,	7, 7, 0x0000070c),
 	{}
 };
 
 static bool intel_snb_pebs_broken(void)
 {
-	return !x86_cpu_has_min_microcode_rev(pebs_ucodes);
+	return !x86_match_min_microcode_rev(pebs_ucodes);
 }
 
 static void intel_snb_check_microcode(void)
@@ -5441,24 +6232,24 @@ static bool check_msr(unsigned long msr, u64 mask)
 	 * matches, this is needed to detect certain hardware emulators
 	 * (qemu/kvm) that don't trap on the MSR access and always return 0s.
 	 */
-	if (rdmsrl_safe(msr, &val_old))
+	if (rdmsrq_safe(msr, &val_old))
 		return false;
 
 	/*
-	 * Only change the bits which can be updated by wrmsrl.
+	 * Only change the bits which can be updated by wrmsrq.
 	 */
 	val_tmp = val_old ^ mask;
 
 	if (is_lbr_from(msr))
 		val_tmp = lbr_from_signext_quirk_wr(val_tmp);
 
-	if (wrmsrl_safe(msr, val_tmp) ||
-	    rdmsrl_safe(msr, &val_new))
+	if (wrmsrq_safe(msr, val_tmp) ||
+	    rdmsrq_safe(msr, &val_new))
 		return false;
 
 	/*
-	 * Quirk only affects validation in wrmsr(), so wrmsrl()'s value
-	 * should equal rdmsrl()'s even with the quirk.
+	 * Quirk only affects validation in wrmsr(), so wrmsrq()'s value
+	 * should equal rdmsrq()'s even with the quirk.
 	 */
 	if (val_new != val_tmp)
 		return false;
@@ -5469,7 +6260,7 @@ static bool check_msr(unsigned long msr, u64 mask)
 	/* Here it's sure that the MSR can be safely accessed.
 	 * Restore the old value and return.
 	 */
-	wrmsrl(msr, val_old);
+	wrmsrq(msr, val_old);
 
 	return true;
 }
@@ -5834,7 +6625,7 @@ tsx_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 static umode_t
 pebs_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 {
-	return x86_pmu.pebs ? attr->mode : 0;
+	return intel_pmu_has_pebs() ? attr->mode : 0;
 }
 
 static umode_t
@@ -5865,6 +6656,21 @@ td_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 	return attr->mode;
 }
 
+PMU_FORMAT_ATTR(acr_mask,	"config2:0-63");
+
+static struct attribute *format_acr_attrs[] = {
+	&format_attr_acr_mask.attr,
+	NULL
+};
+
+static umode_t
+acr_is_visible(struct kobject *kobj, struct attribute *attr, int i)
+{
+	struct device *dev = kobj_to_dev(kobj);
+
+	return intel_pmu_has_acr(dev_get_drvdata(dev)) ? attr->mode : 0;
+}
+
 static struct attribute_group group_events_td  = {
 	.name = "events",
 	.is_visible = td_is_visible,
@@ -5907,6 +6713,12 @@ static struct attribute_group group_format_evtsel_ext = {
 	.is_visible = evtsel_ext_is_visible,
 };
 
+static struct attribute_group group_format_acr = {
+	.name       = "format",
+	.attrs      = format_acr_attrs,
+	.is_visible = acr_is_visible,
+};
+
 static struct attribute_group group_default = {
 	.attrs      = intel_pmu_attrs,
 	.is_visible = default_is_visible,
@@ -5921,6 +6733,7 @@ static const struct attribute_group *attr_update[] = {
 	&group_format_extra,
 	&group_format_extra_skl,
 	&group_format_evtsel_ext,
+	&group_format_acr,
 	&group_default,
 	NULL,
 };
@@ -5965,6 +6778,37 @@ static struct attribute *lnl_hybrid_events_attrs[] = {
 	NULL
 };
 
+/* The event string must be in PMU IDX order. */
+EVENT_ATTR_STR_HYBRID(topdown-retiring,
+		      td_retiring_arl_h,
+		      "event=0xc2,umask=0x02;event=0x00,umask=0x80;event=0xc2,umask=0x0",
+		      hybrid_big_small_tiny);
+EVENT_ATTR_STR_HYBRID(topdown-bad-spec,
+		      td_bad_spec_arl_h,
+		      "event=0x73,umask=0x0;event=0x00,umask=0x81;event=0x73,umask=0x0",
+		      hybrid_big_small_tiny);
+EVENT_ATTR_STR_HYBRID(topdown-fe-bound,
+		      td_fe_bound_arl_h,
+		      "event=0x9c,umask=0x01;event=0x00,umask=0x82;event=0x71,umask=0x0",
+		      hybrid_big_small_tiny);
+EVENT_ATTR_STR_HYBRID(topdown-be-bound,
+		      td_be_bound_arl_h,
+		      "event=0xa4,umask=0x02;event=0x00,umask=0x83;event=0x74,umask=0x0",
+		      hybrid_big_small_tiny);
+
+static struct attribute *arl_h_hybrid_events_attrs[] = {
+	EVENT_PTR(slots_adl),
+	EVENT_PTR(td_retiring_arl_h),
+	EVENT_PTR(td_bad_spec_arl_h),
+	EVENT_PTR(td_fe_bound_arl_h),
+	EVENT_PTR(td_be_bound_arl_h),
+	EVENT_PTR(td_heavy_ops_adl),
+	EVENT_PTR(td_br_mis_adl),
+	EVENT_PTR(td_fetch_lat_adl),
+	EVENT_PTR(td_mem_bound_adl),
+	NULL,
+};
+
 /* Must be in IDX order */
 EVENT_ATTR_STR_HYBRID(mem-loads,     mem_ld_adl,     "event=0xd0,umask=0x5,ldlat=3;event=0xcd,umask=0x1,ldlat=3", hybrid_big_small);
 EVENT_ATTR_STR_HYBRID(mem-stores,    mem_st_adl,     "event=0xd0,umask=0x6;event=0xcd,umask=0x2",                 hybrid_big_small);
@@ -5983,6 +6827,21 @@ static struct attribute *mtl_hybrid_mem_attrs[] = {
 	NULL
 };
 
+EVENT_ATTR_STR_HYBRID(mem-loads,
+		      mem_ld_arl_h,
+		      "event=0xd0,umask=0x5,ldlat=3;event=0xcd,umask=0x1,ldlat=3;event=0xd0,umask=0x5,ldlat=3",
+		      hybrid_big_small_tiny);
+EVENT_ATTR_STR_HYBRID(mem-stores,
+		      mem_st_arl_h,
+		      "event=0xd0,umask=0x6;event=0xcd,umask=0x2;event=0xd0,umask=0x6",
+		      hybrid_big_small_tiny);
+
+static struct attribute *arl_h_hybrid_mem_attrs[] = {
+	EVENT_PTR(mem_ld_arl_h),
+	EVENT_PTR(mem_st_arl_h),
+	NULL,
+};
+
 EVENT_ATTR_STR_HYBRID(tx-start,          tx_start_adl,          "event=0xc9,umask=0x1",          hybrid_big);
 EVENT_ATTR_STR_HYBRID(tx-commit,         tx_commit_adl,         "event=0xc9,umask=0x2",          hybrid_big);
 EVENT_ATTR_STR_HYBRID(tx-abort,          tx_abort_adl,          "event=0xc9,umask=0x4",          hybrid_big);
@@ -6006,8 +6865,8 @@ static struct attribute *adl_hybrid_tsx_attrs[] = {
 
 FORMAT_ATTR_HYBRID(in_tx,       hybrid_big);
 FORMAT_ATTR_HYBRID(in_tx_cp,    hybrid_big);
-FORMAT_ATTR_HYBRID(offcore_rsp, hybrid_big_small);
-FORMAT_ATTR_HYBRID(ldlat,       hybrid_big_small);
+FORMAT_ATTR_HYBRID(offcore_rsp, hybrid_big_small_tiny);
+FORMAT_ATTR_HYBRID(ldlat,       hybrid_big_small_tiny);
 FORMAT_ATTR_HYBRID(frontend,    hybrid_big);
 
 #define ADL_HYBRID_RTM_FORMAT_ATTR	\
@@ -6030,7 +6889,7 @@ static struct attribute *adl_hybrid_extra_attr[] = {
 	NULL
 };
 
-FORMAT_ATTR_HYBRID(snoop_rsp,	hybrid_small);
+FORMAT_ATTR_HYBRID(snoop_rsp,	hybrid_small_tiny);
 
 static struct attribute *mtl_hybrid_extra_attr_rtm[] = {
 	ADL_HYBRID_RTM_FORMAT_ATTR,
@@ -6159,6 +7018,7 @@ static const struct attribute_group *hybrid_attr_update[] = {
 	&group_caps_lbr,
 	&hybrid_group_format_extra,
 	&group_format_evtsel_ext,
+	&group_format_acr,
 	&group_default,
 	&hybrid_group_cpus,
 	NULL,
@@ -6238,8 +7098,9 @@ static inline int intel_pmu_v6_addr_offset(int index, bool eventsel)
 }
 
 static const struct { enum hybrid_pmu_type id; char *name; } intel_hybrid_pmu_type_map[] __initconst = {
-	{ hybrid_small, "cpu_atom" },
-	{ hybrid_big, "cpu_core" },
+	{ hybrid_small,	"cpu_atom" },
+	{ hybrid_big,	"cpu_core" },
+	{ hybrid_tiny,	"cpu_lowpower" },
 };
 
 static __always_inline int intel_pmu_init_hybrid(enum hybrid_pmu_type pmus)
@@ -6272,13 +7133,11 @@ static __always_inline int intel_pmu_init_hybrid(enum hybrid_pmu_type pmus)
 							0, x86_pmu_num_counters(&pmu->pmu), 0, 0);
 
 		pmu->intel_cap.capabilities = x86_pmu.intel_cap.capabilities;
-		if (pmu->pmu_type & hybrid_small) {
+		if (pmu->pmu_type & hybrid_small_tiny) {
 			pmu->intel_cap.perf_metrics = 0;
-			pmu->intel_cap.pebs_output_pt_available = 1;
 			pmu->mid_ack = true;
 		} else if (pmu->pmu_type & hybrid_big) {
 			pmu->intel_cap.perf_metrics = 1;
-			pmu->intel_cap.pebs_output_pt_available = 0;
 			pmu->late_ack = true;
 		}
 	}
@@ -6344,7 +7203,7 @@ static __always_inline void intel_pmu_init_lnc(struct pmu *pmu)
 	intel_pmu_init_glc(pmu);
 	hybrid(pmu, event_constraints) = intel_lnc_event_constraints;
 	hybrid(pmu, pebs_constraints) = intel_lnc_pebs_event_constraints;
-	hybrid(pmu, extra_regs) = intel_rwc_extra_regs;
+	hybrid(pmu, extra_regs) = intel_lnc_extra_regs;
 }
 
 static __always_inline void intel_pmu_init_skt(struct pmu *pmu)
@@ -6352,6 +7211,7 @@ static __always_inline void intel_pmu_init_skt(struct pmu *pmu)
 	intel_pmu_init_grt(pmu);
 	hybrid(pmu, event_constraints) = intel_skt_event_constraints;
 	hybrid(pmu, extra_regs) = intel_cmt_extra_regs;
+	static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr);
 }
 
 __init int intel_pmu_init(void)
@@ -6370,15 +7230,21 @@ __init int intel_pmu_init(void)
 	char *name;
 	struct x86_hybrid_pmu *pmu;
 
+	/* Architectural Perfmon was introduced starting with Core "Yonah" */
 	if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
 		switch (boot_cpu_data.x86) {
-		case 0x6:
-			return p6_pmu_init();
-		case 0xb:
+		case  6:
+			if (boot_cpu_data.x86_vfm < INTEL_CORE_YONAH)
+				return p6_pmu_init();
+			break;
+		case 11:
 			return knc_pmu_init();
-		case 0xf:
+		case 15:
 			return p4_pmu_init();
 		}
+
+		pr_cont("unsupported CPU family %d model %d ",
+			boot_cpu_data.x86, boot_cpu_data.x86_model);
 		return -ENODEV;
 	}
 
@@ -6406,6 +7272,7 @@ __init int intel_pmu_init(void)
 
 	x86_pmu.pebs_events_mask	= intel_pmu_pebs_mask(x86_pmu.cntr_mask64);
 	x86_pmu.pebs_capable		= PEBS_COUNTER_MASK;
+	x86_pmu.config_mask		= X86_RAW_EVENT_MASK;
 
 	/*
 	 * Quirk: v2 perfmon does not report fixed-purpose events, so
@@ -6422,7 +7289,7 @@ __init int intel_pmu_init(void)
 	if (boot_cpu_has(X86_FEATURE_PDCM)) {
 		u64 capabilities;
 
-		rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities);
+		rdmsrq(MSR_IA32_PERF_CAPABILITIES, capabilities);
 		x86_pmu.intel_cap.capabilities = capabilities;
 	}
 
@@ -6434,7 +7301,7 @@ __init int intel_pmu_init(void)
 	if (boot_cpu_has(X86_FEATURE_ARCH_LBR))
 		intel_pmu_arch_lbr_init();
 
-	intel_ds_init();
+	intel_pebs_init();
 
 	x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */
 
@@ -6445,6 +7312,15 @@ __init int intel_pmu_init(void)
 	}
 
 	/*
+	 * Many features on and after V6 require dynamic constraint,
+	 * e.g., Arch PEBS, ACR.
+	 */
+	if (version >= 6) {
+		x86_pmu.flags |= PMU_FL_DYN_CONSTRAINT;
+		x86_pmu.late_setup = intel_pmu_late_setup;
+	}
+
+	/*
 	 * Install the hw-cache-events table:
 	 */
 	switch (boot_cpu_data.x86_vfm) {
@@ -6526,7 +7402,7 @@ __init int intel_pmu_init(void)
 	case INTEL_ATOM_SILVERMONT_D:
 	case INTEL_ATOM_SILVERMONT_MID:
 	case INTEL_ATOM_AIRMONT:
-	case INTEL_ATOM_AIRMONT_MID:
+	case INTEL_ATOM_SILVERMONT_MID2:
 		memcpy(hw_cache_event_ids, slm_hw_cache_event_ids,
 			sizeof(hw_cache_event_ids));
 		memcpy(hw_cache_extra_regs, slm_hw_cache_extra_regs,
@@ -6655,6 +7531,18 @@ __init int intel_pmu_init(void)
 		name = "crestmont";
 		break;
 
+	case INTEL_ATOM_DARKMONT_X:
+		intel_pmu_init_skt(NULL);
+		intel_pmu_pebs_data_source_cmt();
+		x86_pmu.pebs_latency_data = cmt_latency_data;
+		x86_pmu.get_event_constraints = cmt_get_event_constraints;
+		td_attr = skt_events_attrs;
+		mem_attr = grt_mem_attrs;
+		extra_attr = cmt_format_attr;
+		pr_cont("Darkmont events, ");
+		name = "darkmont";
+		break;
+
 	case INTEL_WESTMERE:
 	case INTEL_WESTMERE_EP:
 	case INTEL_WESTMERE_EX:
@@ -7057,6 +7945,7 @@ __init int intel_pmu_init(void)
 
 	case INTEL_METEORLAKE:
 	case INTEL_METEORLAKE_L:
+	case INTEL_ARROWLAKE_U:
 		intel_pmu_init_hybrid(hybrid_big_small);
 
 		x86_pmu.pebs_latency_data = cmt_latency_data;
@@ -7084,8 +7973,18 @@ __init int intel_pmu_init(void)
 		name = "meteorlake_hybrid";
 		break;
 
+	case INTEL_PANTHERLAKE_L:
+	case INTEL_WILDCATLAKE_L:
+		pr_cont("Pantherlake Hybrid events, ");
+		name = "pantherlake_hybrid";
+		goto lnl_common;
+
 	case INTEL_LUNARLAKE_M:
 	case INTEL_ARROWLAKE:
+		pr_cont("Lunarlake Hybrid events, ");
+		name = "lunarlake_hybrid";
+
+	lnl_common:
 		intel_pmu_init_hybrid(hybrid_big_small);
 
 		x86_pmu.pebs_latency_data = lnl_latency_data;
@@ -7107,8 +8006,37 @@ __init int intel_pmu_init(void)
 		intel_pmu_init_skt(&pmu->pmu);
 
 		intel_pmu_pebs_data_source_lnl();
-		pr_cont("Lunarlake Hybrid events, ");
-		name = "lunarlake_hybrid";
+		break;
+
+	case INTEL_ARROWLAKE_H:
+		intel_pmu_init_hybrid(hybrid_big_small_tiny);
+
+		x86_pmu.pebs_latency_data = arl_h_latency_data;
+		x86_pmu.get_event_constraints = arl_h_get_event_constraints;
+		x86_pmu.hw_config = arl_h_hw_config;
+
+		td_attr = arl_h_hybrid_events_attrs;
+		mem_attr = arl_h_hybrid_mem_attrs;
+		tsx_attr = adl_hybrid_tsx_attrs;
+		extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
+			mtl_hybrid_extra_attr_rtm : mtl_hybrid_extra_attr;
+
+		/* Initialize big core specific PerfMon capabilities. */
+		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
+		intel_pmu_init_lnc(&pmu->pmu);
+
+		/* Initialize Atom core specific PerfMon capabilities. */
+		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
+		intel_pmu_init_skt(&pmu->pmu);
+
+		/* Initialize Lower Power Atom specific PerfMon capabilities. */
+		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_TINY_IDX];
+		intel_pmu_init_grt(&pmu->pmu);
+		pmu->extra_regs = intel_cmt_extra_regs;
+
+		intel_pmu_pebs_data_source_arl_h();
+		pr_cont("ArrowLake-H Hybrid events, ");
+		name = "arrowlake_h_hybrid";
 		break;
 
 	default:
@@ -7165,6 +8093,26 @@ __init int intel_pmu_init(void)
 		x86_pmu.attr_update = hybrid_attr_update;
 	}
 
+	/*
+	 * The archPerfmonExt (0x23) includes an enhanced enumeration of
+	 * PMU architectural features with a per-core view. For non-hybrid,
+	 * each core has the same PMU capabilities. It's good enough to
+	 * update the x86_pmu from the booting CPU. For hybrid, the x86_pmu
+	 * is used to keep the common capabilities. Still keep the values
+	 * from the leaf 0xa. The core specific update will be done later
+	 * when a new type is online.
+	 */
+	if (!is_hybrid() && boot_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))
+		update_pmu_cap(NULL);
+
+	if (x86_pmu.arch_pebs) {
+		static_call_update(intel_pmu_disable_event_ext,
+				   intel_pmu_disable_event_ext);
+		static_call_update(intel_pmu_enable_event_ext,
+				   intel_pmu_enable_event_ext);
+		pr_cont("Architectural PEBS, ");
+	}
+
 	intel_pmu_check_counters_mask(&x86_pmu.cntr_mask64,
 				      &x86_pmu.fixed_cntr_mask64,
 				      &x86_pmu.intel_ctrl);
@@ -7173,10 +8121,8 @@ __init int intel_pmu_init(void)
 	if (x86_pmu.intel_cap.anythread_deprecated)
 		x86_pmu.format_attrs = intel_arch_formats_attr;
 
-	intel_pmu_check_event_constraints(x86_pmu.event_constraints,
-					  x86_pmu.cntr_mask64,
-					  x86_pmu.fixed_cntr_mask64,
-					  x86_pmu.intel_ctrl);
+	intel_pmu_check_event_constraints_all(NULL);
+
 	/*
 	 * Access LBR MSR may cause #GP under certain circumstances.
 	 * Check all LBR MSR here.
@@ -7225,7 +8171,7 @@ __init int intel_pmu_init(void)
 	}
 
 	if (!is_hybrid() && x86_pmu.intel_cap.perf_metrics)
-		x86_pmu.intel_ctrl |= 1ULL << GLOBAL_CTRL_EN_PERF_METRICS;
+		x86_pmu.intel_ctrl |= GLOBAL_CTRL_EN_PERF_METRICS;
 
 	if (x86_pmu.intel_cap.pebs_timing_info)
 		x86_pmu.flags |= PMU_FL_RETIRE_LATENCY;
diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c
index ae4ec16156bb..fa67fda6e45b 100644
--- a/arch/x86/events/intel/cstate.c
+++ b/arch/x86/events/intel/cstate.c
@@ -41,7 +41,7 @@
  *	MSR_CORE_C1_RES: CORE C1 Residency Counter
  *			 perf code: 0x00
  *			 Available model: SLM,AMT,GLM,CNL,ICX,TNT,ADL,RPL
- *					  MTL,SRF,GRR,ARL,LNL
+ *					  MTL,SRF,GRR,ARL,LNL,PTL
  *			 Scope: Core (each processor core has a MSR)
  *	MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter
  *			       perf code: 0x01
@@ -53,31 +53,32 @@
  *			       Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
  *						SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
  *						TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,
- *						GRR,ARL,LNL
+ *						GRR,ARL,LNL,PTL
  *			       Scope: Core
  *	MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter
  *			       perf code: 0x03
  *			       Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML,
- *						ICL,TGL,RKL,ADL,RPL,MTL,ARL,LNL
+ *						ICL,TGL,RKL,ADL,RPL,MTL,ARL,LNL,
+ *						PTL
  *			       Scope: Core
  *	MSR_PKG_C2_RESIDENCY:  Package C2 Residency Counter.
  *			       perf code: 0x00
  *			       Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL,
  *						KBL,CML,ICL,ICX,TGL,TNT,RKL,ADL,
- *						RPL,SPR,MTL,ARL,LNL,SRF
+ *						RPL,SPR,MTL,ARL,LNL,SRF,PTL
  *			       Scope: Package (physical package)
  *	MSR_PKG_C3_RESIDENCY:  Package C3 Residency Counter.
  *			       perf code: 0x01
  *			       Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL,
  *						GLM,CNL,KBL,CML,ICL,TGL,TNT,RKL,
- *						ADL,RPL,MTL,ARL,LNL
+ *						ADL,RPL,MTL,ARL
  *			       Scope: Package (physical package)
  *	MSR_PKG_C6_RESIDENCY:  Package C6 Residency Counter.
  *			       perf code: 0x02
  *			       Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
  *						SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
  *						TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,
- *						ARL,LNL
+ *						ARL,LNL,PTL
  *			       Scope: Package (physical package)
  *	MSR_PKG_C7_RESIDENCY:  Package C7 Residency Counter.
  *			       perf code: 0x03
@@ -96,7 +97,7 @@
  *	MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter.
  *			       perf code: 0x06
  *			       Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL,
- *						TNT,RKL,ADL,RPL,MTL,ARL,LNL
+ *						TNT,RKL,ADL,RPL,MTL,ARL,LNL,PTL
  *			       Scope: Package (physical package)
  *	MSR_MODULE_C6_RES_MS:  Module C6 Residency Counter.
  *			       perf code: 0x00
@@ -111,6 +112,7 @@
 #include <linux/nospec.h>
 #include <asm/cpu_device_id.h>
 #include <asm/intel-family.h>
+#include <asm/msr.h>
 #include "../perf_event.h"
 #include "../probe.h"
 
@@ -320,7 +322,7 @@ static inline u64 cstate_pmu_read_counter(struct perf_event *event)
 {
 	u64 val;
 
-	rdmsrl(event->hw.event_base, val);
+	rdmsrq(event->hw.event_base, val);
 	return val;
 }
 
@@ -521,7 +523,6 @@ static const struct cstate_model lnl_cstates __initconst = {
 				  BIT(PERF_CSTATE_CORE_C7_RES),
 
 	.pkg_events		= BIT(PERF_CSTATE_PKG_C2_RES) |
-				  BIT(PERF_CSTATE_PKG_C3_RES) |
 				  BIT(PERF_CSTATE_PKG_C6_RES) |
 				  BIT(PERF_CSTATE_PKG_C10_RES),
 };
@@ -627,6 +628,7 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = {
 	X86_MATCH_VFM(INTEL_ATOM_GRACEMONT,	&adl_cstates),
 	X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X,	&srf_cstates),
 	X86_MATCH_VFM(INTEL_ATOM_CRESTMONT,	&grr_cstates),
+	X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X,	&srf_cstates),
 
 	X86_MATCH_VFM(INTEL_ICELAKE_L,		&icl_cstates),
 	X86_MATCH_VFM(INTEL_ICELAKE,		&icl_cstates),
@@ -651,6 +653,7 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = {
 	X86_MATCH_VFM(INTEL_ARROWLAKE_H,	&adl_cstates),
 	X86_MATCH_VFM(INTEL_ARROWLAKE_U,	&adl_cstates),
 	X86_MATCH_VFM(INTEL_LUNARLAKE_M,	&lnl_cstates),
+	X86_MATCH_VFM(INTEL_PANTHERLAKE_L,	&lnl_cstates),
 	{ },
 };
 MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match);
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index fa5ea65de0d0..feb1c3cf63e4 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -10,6 +10,7 @@
 #include <asm/tlbflush.h>
 #include <asm/insn.h>
 #include <asm/io.h>
+#include <asm/msr.h>
 #include <asm/timer.h>
 
 #include "../perf_event.h"
@@ -177,6 +178,17 @@ void __init intel_pmu_pebs_data_source_mtl(void)
 	__intel_pmu_pebs_data_source_cmt(data_source);
 }
 
+void __init intel_pmu_pebs_data_source_arl_h(void)
+{
+	u64 *data_source;
+
+	intel_pmu_pebs_data_source_lnl();
+
+	data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_TINY_IDX].pebs_data_source;
+	memcpy(data_source, pebs_data_source, sizeof(pebs_data_source));
+	__intel_pmu_pebs_data_source_cmt(data_source);
+}
+
 void __init intel_pmu_pebs_data_source_cmt(void)
 {
 	__intel_pmu_pebs_data_source_cmt(pebs_data_source);
@@ -305,7 +317,8 @@ static u64 __grt_latency_data(struct perf_event *event, u64 status,
 {
 	u64 val;
 
-	WARN_ON_ONCE(hybrid_pmu(event->pmu)->pmu_type == hybrid_big);
+	WARN_ON_ONCE(is_hybrid() &&
+		     hybrid_pmu(event->pmu)->pmu_type == hybrid_big);
 
 	dse &= PERF_PEBS_DATA_SOURCE_GRT_MASK;
 	val = hybrid_var(event->pmu, pebs_data_source)[dse];
@@ -388,6 +401,16 @@ u64 lnl_latency_data(struct perf_event *event, u64 status)
 	return lnc_latency_data(event, status);
 }
 
+u64 arl_h_latency_data(struct perf_event *event, u64 status)
+{
+	struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
+
+	if (pmu->pmu_type == hybrid_tiny)
+		return cmt_latency_data(event, status);
+
+	return lnl_latency_data(event, status);
+}
+
 static u64 load_latency_data(struct perf_event *event, u64 status)
 {
 	union intel_x86_pebs_dse dse;
@@ -603,13 +626,18 @@ static int alloc_pebs_buffer(int cpu)
 	int max, node = cpu_to_node(cpu);
 	void *buffer, *insn_buff, *cea;
 
-	if (!x86_pmu.pebs)
+	if (!intel_pmu_has_pebs())
 		return 0;
 
 	buffer = dsalloc_pages(bsiz, GFP_KERNEL, cpu);
 	if (unlikely(!buffer))
 		return -ENOMEM;
 
+	if (x86_pmu.arch_pebs) {
+		hwev->pebs_vaddr = buffer;
+		return 0;
+	}
+
 	/*
 	 * HSW+ already provides us the eventing ip; no need to allocate this
 	 * buffer then.
@@ -622,7 +650,7 @@ static int alloc_pebs_buffer(int cpu)
 		}
 		per_cpu(insn_buffer, cpu) = insn_buff;
 	}
-	hwev->ds_pebs_vaddr = buffer;
+	hwev->pebs_vaddr = buffer;
 	/* Update the cpu entry area mapping */
 	cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;
 	ds->pebs_buffer_base = (unsigned long) cea;
@@ -638,17 +666,20 @@ static void release_pebs_buffer(int cpu)
 	struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
 	void *cea;
 
-	if (!x86_pmu.pebs)
+	if (!intel_pmu_has_pebs())
 		return;
 
-	kfree(per_cpu(insn_buffer, cpu));
-	per_cpu(insn_buffer, cpu) = NULL;
+	if (x86_pmu.ds_pebs) {
+		kfree(per_cpu(insn_buffer, cpu));
+		per_cpu(insn_buffer, cpu) = NULL;
 
-	/* Clear the fixmap */
-	cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;
-	ds_clear_cea(cea, x86_pmu.pebs_buffer_size);
-	dsfree_pages(hwev->ds_pebs_vaddr, x86_pmu.pebs_buffer_size);
-	hwev->ds_pebs_vaddr = NULL;
+		/* Clear the fixmap */
+		cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;
+		ds_clear_cea(cea, x86_pmu.pebs_buffer_size);
+	}
+
+	dsfree_pages(hwev->pebs_vaddr, x86_pmu.pebs_buffer_size);
+	hwev->pebs_vaddr = NULL;
 }
 
 static int alloc_bts_buffer(int cpu)
@@ -713,7 +744,7 @@ void release_ds_buffers(void)
 {
 	int cpu;
 
-	if (!x86_pmu.bts && !x86_pmu.pebs)
+	if (!x86_pmu.bts && !x86_pmu.ds_pebs)
 		return;
 
 	for_each_possible_cpu(cpu)
@@ -729,7 +760,8 @@ void release_ds_buffers(void)
 	}
 
 	for_each_possible_cpu(cpu) {
-		release_pebs_buffer(cpu);
+		if (x86_pmu.ds_pebs)
+			release_pebs_buffer(cpu);
 		release_bts_buffer(cpu);
 	}
 }
@@ -740,15 +772,17 @@ void reserve_ds_buffers(void)
 	int cpu;
 
 	x86_pmu.bts_active = 0;
-	x86_pmu.pebs_active = 0;
 
-	if (!x86_pmu.bts && !x86_pmu.pebs)
+	if (x86_pmu.ds_pebs)
+		x86_pmu.pebs_active = 0;
+
+	if (!x86_pmu.bts && !x86_pmu.ds_pebs)
 		return;
 
 	if (!x86_pmu.bts)
 		bts_err = 1;
 
-	if (!x86_pmu.pebs)
+	if (!x86_pmu.ds_pebs)
 		pebs_err = 1;
 
 	for_each_possible_cpu(cpu) {
@@ -760,7 +794,8 @@ void reserve_ds_buffers(void)
 		if (!bts_err && alloc_bts_buffer(cpu))
 			bts_err = 1;
 
-		if (!pebs_err && alloc_pebs_buffer(cpu))
+		if (x86_pmu.ds_pebs && !pebs_err &&
+		    alloc_pebs_buffer(cpu))
 			pebs_err = 1;
 
 		if (bts_err && pebs_err)
@@ -772,7 +807,7 @@ void reserve_ds_buffers(void)
 			release_bts_buffer(cpu);
 	}
 
-	if (pebs_err) {
+	if (x86_pmu.ds_pebs && pebs_err) {
 		for_each_possible_cpu(cpu)
 			release_pebs_buffer(cpu);
 	}
@@ -784,7 +819,7 @@ void reserve_ds_buffers(void)
 		if (x86_pmu.bts && !bts_err)
 			x86_pmu.bts_active = 1;
 
-		if (x86_pmu.pebs && !pebs_err)
+		if (x86_pmu.ds_pebs && !pebs_err)
 			x86_pmu.pebs_active = 1;
 
 		for_each_possible_cpu(cpu) {
@@ -797,6 +832,56 @@ void reserve_ds_buffers(void)
 	}
 }
 
+inline int alloc_arch_pebs_buf_on_cpu(int cpu)
+{
+	if (!x86_pmu.arch_pebs)
+		return 0;
+
+	return alloc_pebs_buffer(cpu);
+}
+
+inline void release_arch_pebs_buf_on_cpu(int cpu)
+{
+	if (!x86_pmu.arch_pebs)
+		return;
+
+	release_pebs_buffer(cpu);
+}
+
+void init_arch_pebs_on_cpu(int cpu)
+{
+	struct cpu_hw_events *cpuc = per_cpu_ptr(&cpu_hw_events, cpu);
+	u64 arch_pebs_base;
+
+	if (!x86_pmu.arch_pebs)
+		return;
+
+	if (!cpuc->pebs_vaddr) {
+		WARN(1, "Fail to allocate PEBS buffer on CPU %d\n", cpu);
+		x86_pmu.pebs_active = 0;
+		return;
+	}
+
+	/*
+	 * 4KB-aligned pointer of the output buffer
+	 * (__alloc_pages_node() return page aligned address)
+	 * Buffer Size = 4KB * 2^SIZE
+	 * contiguous physical buffer (__alloc_pages_node() with order)
+	 */
+	arch_pebs_base = virt_to_phys(cpuc->pebs_vaddr) | PEBS_BUFFER_SHIFT;
+	wrmsr_on_cpu(cpu, MSR_IA32_PEBS_BASE, (u32)arch_pebs_base,
+		     (u32)(arch_pebs_base >> 32));
+	x86_pmu.pebs_active = 1;
+}
+
+inline void fini_arch_pebs_on_cpu(int cpu)
+{
+	if (!x86_pmu.arch_pebs)
+		return;
+
+	wrmsr_on_cpu(cpu, MSR_IA32_PEBS_BASE, 0, 0);
+}
+
 /*
  * BTS
  */
@@ -932,11 +1017,11 @@ unlock:
 	return 1;
 }
 
-static inline void intel_pmu_drain_pebs_buffer(void)
+void intel_pmu_drain_pebs_buffer(void)
 {
 	struct perf_sample_data data;
 
-	x86_pmu.drain_pebs(NULL, &data);
+	static_call(x86_pmu_drain_pebs)(NULL, &data);
 }
 
 /*
@@ -1178,7 +1263,7 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = {
 	INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL),	/* INST_RETIRED.PREC_DIST */
 	INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),
 
-	INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0x3ff),
+	INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0x3fc),
 	INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3),
 	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf),	/* MEM_INST_RETIRED.STLB_MISS_LOADS */
 	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf),	/* MEM_INST_RETIRED.STLB_MISS_STORES */
@@ -1273,6 +1358,19 @@ static inline void pebs_update_threshold(struct cpu_hw_events *cpuc)
 	ds->pebs_interrupt_threshold = threshold;
 }
 
+#define PEBS_DATACFG_CNTRS(x)						\
+	((x >> PEBS_DATACFG_CNTR_SHIFT) & PEBS_DATACFG_CNTR_MASK)
+
+#define PEBS_DATACFG_CNTR_BIT(x)					\
+	(((1ULL << x) & PEBS_DATACFG_CNTR_MASK) << PEBS_DATACFG_CNTR_SHIFT)
+
+#define PEBS_DATACFG_FIX(x)						\
+	((x >> PEBS_DATACFG_FIX_SHIFT) & PEBS_DATACFG_FIX_MASK)
+
+#define PEBS_DATACFG_FIX_BIT(x)						\
+	(((1ULL << (x)) & PEBS_DATACFG_FIX_MASK)			\
+	 << PEBS_DATACFG_FIX_SHIFT)
+
 static void adaptive_pebs_record_size_update(void)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -1287,10 +1385,57 @@ static void adaptive_pebs_record_size_update(void)
 		sz += sizeof(struct pebs_xmm);
 	if (pebs_data_cfg & PEBS_DATACFG_LBRS)
 		sz += x86_pmu.lbr_nr * sizeof(struct lbr_entry);
+	if (pebs_data_cfg & (PEBS_DATACFG_METRICS | PEBS_DATACFG_CNTR)) {
+		sz += sizeof(struct pebs_cntr_header);
+
+		/* Metrics base and Metrics Data */
+		if (pebs_data_cfg & PEBS_DATACFG_METRICS)
+			sz += 2 * sizeof(u64);
+
+		if (pebs_data_cfg & PEBS_DATACFG_CNTR) {
+			sz += (hweight64(PEBS_DATACFG_CNTRS(pebs_data_cfg)) +
+			       hweight64(PEBS_DATACFG_FIX(pebs_data_cfg))) *
+			      sizeof(u64);
+		}
+	}
 
 	cpuc->pebs_record_size = sz;
 }
 
+static void __intel_pmu_pebs_update_cfg(struct perf_event *event,
+					int idx, u64 *pebs_data_cfg)
+{
+	if (is_metric_event(event)) {
+		*pebs_data_cfg |= PEBS_DATACFG_METRICS;
+		return;
+	}
+
+	*pebs_data_cfg |= PEBS_DATACFG_CNTR;
+
+	if (idx >= INTEL_PMC_IDX_FIXED)
+		*pebs_data_cfg |= PEBS_DATACFG_FIX_BIT(idx - INTEL_PMC_IDX_FIXED);
+	else
+		*pebs_data_cfg |= PEBS_DATACFG_CNTR_BIT(idx);
+}
+
+
+void intel_pmu_pebs_late_setup(struct cpu_hw_events *cpuc)
+{
+	struct perf_event *event;
+	u64 pebs_data_cfg = 0;
+	int i;
+
+	for (i = 0; i < cpuc->n_events; i++) {
+		event = cpuc->event_list[i];
+		if (!is_pebs_counter_event_group(event))
+			continue;
+		__intel_pmu_pebs_update_cfg(event, cpuc->assign[i], &pebs_data_cfg);
+	}
+
+	if (pebs_data_cfg & ~cpuc->pebs_data_cfg)
+		cpuc->pebs_data_cfg |= pebs_data_cfg | PEBS_UPDATE_DS_SW;
+}
+
 #define PERF_PEBS_MEMINFO_TYPE	(PERF_SAMPLE_ADDR | PERF_SAMPLE_DATA_SRC |   \
 				PERF_SAMPLE_PHYS_ADDR |			     \
 				PERF_SAMPLE_WEIGHT_TYPE |		     \
@@ -1317,8 +1462,10 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event)
 	 * + precise_ip < 2 for the non event IP
 	 * + For RTM TSX weight we need GPRs for the abort code.
 	 */
-	gprs = (sample_type & PERF_SAMPLE_REGS_INTR) &&
-	       (attr->sample_regs_intr & PEBS_GP_REGS);
+	gprs = ((sample_type & PERF_SAMPLE_REGS_INTR) &&
+		(attr->sample_regs_intr & PEBS_GP_REGS)) ||
+	       ((sample_type & PERF_SAMPLE_REGS_USER) &&
+		(attr->sample_regs_user & PEBS_GP_REGS));
 
 	tsx_weight = (sample_type & PERF_SAMPLE_WEIGHT_TYPE) &&
 		     ((attr->config & INTEL_ARCH_EVENT_MASK) ==
@@ -1382,6 +1529,25 @@ pebs_update_state(bool needed_cb, struct cpu_hw_events *cpuc,
 	}
 }
 
+u64 intel_get_arch_pebs_data_config(struct perf_event *event)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	u64 pebs_data_cfg = 0;
+	u64 cntr_mask;
+
+	if (WARN_ON(event->hw.idx < 0 || event->hw.idx >= X86_PMC_IDX_MAX))
+		return 0;
+
+	pebs_data_cfg |= pebs_update_adaptive_cfg(event);
+
+	cntr_mask = (PEBS_DATACFG_CNTR_MASK << PEBS_DATACFG_CNTR_SHIFT) |
+		    (PEBS_DATACFG_FIX_MASK << PEBS_DATACFG_FIX_SHIFT) |
+		    PEBS_DATACFG_CNTR | PEBS_DATACFG_METRICS;
+	pebs_data_cfg |= cpuc->pebs_data_cfg & cntr_mask;
+
+	return pebs_data_cfg;
+}
+
 void intel_pmu_pebs_add(struct perf_event *event)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -1433,7 +1599,7 @@ static void intel_pmu_pebs_via_pt_enable(struct perf_event *event)
 		else
 			value = ds->pebs_event_reset[MAX_PEBS_EVENTS + idx];
 	}
-	wrmsrl(base + idx, value);
+	wrmsrq(base + idx, value);
 }
 
 static inline void intel_pmu_drain_large_pebs(struct cpu_hw_events *cpuc)
@@ -1443,6 +1609,15 @@ static inline void intel_pmu_drain_large_pebs(struct cpu_hw_events *cpuc)
 		intel_pmu_drain_pebs_buffer();
 }
 
+static void __intel_pmu_pebs_enable(struct perf_event *event)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	struct hw_perf_event *hwc = &event->hw;
+
+	hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
+	cpuc->pebs_enabled |= 1ULL << hwc->idx;
+}
+
 void intel_pmu_pebs_enable(struct perf_event *event)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -1451,9 +1626,7 @@ void intel_pmu_pebs_enable(struct perf_event *event)
 	struct debug_store *ds = cpuc->ds;
 	unsigned int idx = hwc->idx;
 
-	hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
-
-	cpuc->pebs_enabled |= 1ULL << hwc->idx;
+	__intel_pmu_pebs_enable(event);
 
 	if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) && (x86_pmu.version < 5))
 		cpuc->pebs_enabled |= 1ULL << (hwc->idx + 32);
@@ -1468,9 +1641,9 @@ void intel_pmu_pebs_enable(struct perf_event *event)
 			 * hence we need to drain when changing said
 			 * size.
 			 */
-			intel_pmu_drain_large_pebs(cpuc);
+			intel_pmu_drain_pebs_buffer();
 			adaptive_pebs_record_size_update();
-			wrmsrl(MSR_PEBS_DATA_CFG, pebs_data_cfg);
+			wrmsrq(MSR_PEBS_DATA_CFG, pebs_data_cfg);
 			cpuc->active_pebs_data_cfg = pebs_data_cfg;
 		}
 	}
@@ -1515,14 +1688,22 @@ void intel_pmu_pebs_del(struct perf_event *event)
 	pebs_update_state(needed_cb, cpuc, event, false);
 }
 
-void intel_pmu_pebs_disable(struct perf_event *event)
+static void __intel_pmu_pebs_disable(struct perf_event *event)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct hw_perf_event *hwc = &event->hw;
 
 	intel_pmu_drain_large_pebs(cpuc);
-
 	cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
+	hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
+}
+
+void intel_pmu_pebs_disable(struct perf_event *event)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	struct hw_perf_event *hwc = &event->hw;
+
+	__intel_pmu_pebs_disable(event);
 
 	if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) &&
 	    (x86_pmu.version < 5))
@@ -1533,9 +1714,7 @@ void intel_pmu_pebs_disable(struct perf_event *event)
 	intel_pmu_pebs_via_pt_disable(event);
 
 	if (cpuc->enabled)
-		wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
-
-	hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
+		wrmsrq(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
 }
 
 void intel_pmu_pebs_enable_all(void)
@@ -1543,7 +1722,7 @@ void intel_pmu_pebs_enable_all(void)
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
 	if (cpuc->pebs_enabled)
-		wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
+		wrmsrq(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
 }
 
 void intel_pmu_pebs_disable_all(void)
@@ -1744,8 +1923,6 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event,
 
 	perf_sample_data_init(data, 0, event->hw.last_period);
 
-	data->period = event->hw.last_period;
-
 	/*
 	 * Use latency for weight (only avail with PEBS-LL)
 	 */
@@ -1768,8 +1945,7 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event,
 	 * previous PMI context or an (I)RET happened between the record and
 	 * PMI.
 	 */
-	if (sample_type & PERF_SAMPLE_CALLCHAIN)
-		perf_sample_save_callchain(data, event, iregs);
+	perf_sample_save_callchain(data, event, iregs);
 
 	/*
 	 * We use the interrupt regs as a base because the PEBS record does not
@@ -1868,8 +2044,7 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event,
 	if (x86_pmu.intel_cap.pebs_format >= 3)
 		setup_pebs_time(event, data, pebs->tsc);
 
-	if (has_branch_stack(event))
-		perf_sample_save_brstack(data, event, &cpuc->lbr_stack, NULL);
+	perf_sample_save_brstack(data, event, &cpuc->lbr_stack, NULL);
 }
 
 static void adaptive_pebs_save_regs(struct pt_regs *regs,
@@ -1895,27 +2070,187 @@ static void adaptive_pebs_save_regs(struct pt_regs *regs,
 #endif
 }
 
+static void intel_perf_event_update_pmc(struct perf_event *event, u64 pmc)
+{
+	int shift = 64 - x86_pmu.cntval_bits;
+	struct hw_perf_event *hwc;
+	u64 delta, prev_pmc;
+
+	/*
+	 * A recorded counter may not have an assigned event in the
+	 * following cases. The value should be dropped.
+	 * - An event is deleted. There is still an active PEBS event.
+	 *   The PEBS record doesn't shrink on pmu::del().
+	 *   If the counter of the deleted event once occurred in a PEBS
+	 *   record, PEBS still records the counter until the counter is
+	 *   reassigned.
+	 * - An event is stopped for some reason, e.g., throttled.
+	 *   During this period, another event is added and takes the
+	 *   counter of the stopped event. The stopped event is assigned
+	 *   to another new and uninitialized counter, since the
+	 *   x86_pmu_start(RELOAD) is not invoked for a stopped event.
+	 *   The PEBS__DATA_CFG is updated regardless of the event state.
+	 *   The uninitialized counter can be recorded in a PEBS record.
+	 *   But the cpuc->events[uninitialized_counter] is always NULL,
+	 *   because the event is stopped. The uninitialized value is
+	 *   safely dropped.
+	 */
+	if (!event)
+		return;
+
+	hwc = &event->hw;
+	prev_pmc = local64_read(&hwc->prev_count);
+
+	/* Only update the count when the PMU is disabled */
+	WARN_ON(this_cpu_read(cpu_hw_events.enabled));
+	local64_set(&hwc->prev_count, pmc);
+
+	delta = (pmc << shift) - (prev_pmc << shift);
+	delta >>= shift;
+
+	local64_add(delta, &event->count);
+	local64_sub(delta, &hwc->period_left);
+}
+
+static inline void __setup_pebs_counter_group(struct cpu_hw_events *cpuc,
+					      struct perf_event *event,
+					      struct pebs_cntr_header *cntr,
+					      void *next_record)
+{
+	int bit;
+
+	for_each_set_bit(bit, (unsigned long *)&cntr->cntr, INTEL_PMC_MAX_GENERIC) {
+		intel_perf_event_update_pmc(cpuc->events[bit], *(u64 *)next_record);
+		next_record += sizeof(u64);
+	}
+
+	for_each_set_bit(bit, (unsigned long *)&cntr->fixed, INTEL_PMC_MAX_FIXED) {
+		/* The slots event will be handled with perf_metric later */
+		if ((cntr->metrics == INTEL_CNTR_METRICS) &&
+		    (bit + INTEL_PMC_IDX_FIXED == INTEL_PMC_IDX_FIXED_SLOTS)) {
+			next_record += sizeof(u64);
+			continue;
+		}
+		intel_perf_event_update_pmc(cpuc->events[bit + INTEL_PMC_IDX_FIXED],
+					    *(u64 *)next_record);
+		next_record += sizeof(u64);
+	}
+
+	/* HW will reload the value right after the overflow. */
+	if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
+		local64_set(&event->hw.prev_count, (u64)-event->hw.sample_period);
+
+	if (cntr->metrics == INTEL_CNTR_METRICS) {
+		static_call(intel_pmu_update_topdown_event)
+			   (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS],
+			    (u64 *)next_record);
+		next_record += 2 * sizeof(u64);
+	}
+}
+
 #define PEBS_LATENCY_MASK			0xffff
-#define PEBS_CACHE_LATENCY_OFFSET		32
-#define PEBS_RETIRE_LATENCY_OFFSET		32
+
+static inline void __setup_perf_sample_data(struct perf_event *event,
+					    struct pt_regs *iregs,
+					    struct perf_sample_data *data)
+{
+	perf_sample_data_init(data, 0, event->hw.last_period);
+
+	/*
+	 * We must however always use iregs for the unwinder to stay sane; the
+	 * record BP,SP,IP can point into thin air when the record is from a
+	 * previous PMI context or an (I)RET happened between the record and
+	 * PMI.
+	 */
+	perf_sample_save_callchain(data, event, iregs);
+}
+
+static inline void __setup_pebs_basic_group(struct perf_event *event,
+					    struct pt_regs *regs,
+					    struct perf_sample_data *data,
+					    u64 sample_type, u64 ip,
+					    u64 tsc, u16 retire)
+{
+	/* The ip in basic is EventingIP */
+	set_linear_ip(regs, ip);
+	regs->flags = PERF_EFLAGS_EXACT;
+	setup_pebs_time(event, data, tsc);
+
+	if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT)
+		data->weight.var3_w = retire;
+}
+
+static inline void __setup_pebs_gpr_group(struct perf_event *event,
+					  struct pt_regs *regs,
+					  struct pebs_gprs *gprs,
+					  u64 sample_type)
+{
+	if (event->attr.precise_ip < 2) {
+		set_linear_ip(regs, gprs->ip);
+		regs->flags &= ~PERF_EFLAGS_EXACT;
+	}
+
+	if (sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER))
+		adaptive_pebs_save_regs(regs, gprs);
+}
+
+static inline void __setup_pebs_meminfo_group(struct perf_event *event,
+					      struct perf_sample_data *data,
+					      u64 sample_type, u64 latency,
+					      u16 instr_latency, u64 address,
+					      u64 aux, u64 tsx_tuning, u64 ax)
+{
+	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) {
+		u64 tsx_latency = intel_get_tsx_weight(tsx_tuning);
+
+		data->weight.var2_w = instr_latency;
+
+		/*
+		 * Although meminfo::latency is defined as a u64,
+		 * only the lower 32 bits include the valid data
+		 * in practice on Ice Lake and earlier platforms.
+		 */
+		if (sample_type & PERF_SAMPLE_WEIGHT)
+			data->weight.full = latency ?: tsx_latency;
+		else
+			data->weight.var1_dw = (u32)latency ?: tsx_latency;
+
+		data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;
+	}
+
+	if (sample_type & PERF_SAMPLE_DATA_SRC) {
+		data->data_src.val = get_data_src(event, aux);
+		data->sample_flags |= PERF_SAMPLE_DATA_SRC;
+	}
+
+	if (sample_type & PERF_SAMPLE_ADDR_TYPE) {
+		data->addr = address;
+		data->sample_flags |= PERF_SAMPLE_ADDR;
+	}
+
+	if (sample_type & PERF_SAMPLE_TRANSACTION) {
+		data->txn = intel_get_tsx_transaction(tsx_tuning, ax);
+		data->sample_flags |= PERF_SAMPLE_TRANSACTION;
+	}
+}
 
 /*
  * With adaptive PEBS the layout depends on what fields are configured.
  */
-
 static void setup_pebs_adaptive_sample_data(struct perf_event *event,
 					    struct pt_regs *iregs, void *__pebs,
 					    struct perf_sample_data *data,
 					    struct pt_regs *regs)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	u64 sample_type = event->attr.sample_type;
 	struct pebs_basic *basic = __pebs;
 	void *next_record = basic + 1;
-	u64 sample_type;
-	u64 format_size;
 	struct pebs_meminfo *meminfo = NULL;
 	struct pebs_gprs *gprs = NULL;
 	struct x86_perf_regs *perf_regs;
+	u64 format_group;
+	u16 retire;
 
 	if (basic == NULL)
 		return;
@@ -1923,110 +2258,193 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event,
 	perf_regs = container_of(regs, struct x86_perf_regs, regs);
 	perf_regs->xmm_regs = NULL;
 
-	sample_type = event->attr.sample_type;
-	format_size = basic->format_size;
-	perf_sample_data_init(data, 0, event->hw.last_period);
-	data->period = event->hw.last_period;
+	format_group = basic->format_group;
 
-	setup_pebs_time(event, data, basic->tsc);
-
-	/*
-	 * We must however always use iregs for the unwinder to stay sane; the
-	 * record BP,SP,IP can point into thin air when the record is from a
-	 * previous PMI context or an (I)RET happened between the record and
-	 * PMI.
-	 */
-	if (sample_type & PERF_SAMPLE_CALLCHAIN)
-		perf_sample_save_callchain(data, event, iregs);
+	__setup_perf_sample_data(event, iregs, data);
 
 	*regs = *iregs;
-	/* The ip in basic is EventingIP */
-	set_linear_ip(regs, basic->ip);
-	regs->flags = PERF_EFLAGS_EXACT;
 
-	if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT) {
-		if (x86_pmu.flags & PMU_FL_RETIRE_LATENCY)
-			data->weight.var3_w = format_size >> PEBS_RETIRE_LATENCY_OFFSET & PEBS_LATENCY_MASK;
-		else
-			data->weight.var3_w = 0;
-	}
+	/* basic group */
+	retire = x86_pmu.flags & PMU_FL_RETIRE_LATENCY ?
+			basic->retire_latency : 0;
+	__setup_pebs_basic_group(event, regs, data, sample_type,
+				 basic->ip, basic->tsc, retire);
 
 	/*
 	 * The record for MEMINFO is in front of GP
 	 * But PERF_SAMPLE_TRANSACTION needs gprs->ax.
 	 * Save the pointer here but process later.
 	 */
-	if (format_size & PEBS_DATACFG_MEMINFO) {
+	if (format_group & PEBS_DATACFG_MEMINFO) {
 		meminfo = next_record;
 		next_record = meminfo + 1;
 	}
 
-	if (format_size & PEBS_DATACFG_GP) {
+	if (format_group & PEBS_DATACFG_GP) {
 		gprs = next_record;
 		next_record = gprs + 1;
 
-		if (event->attr.precise_ip < 2) {
-			set_linear_ip(regs, gprs->ip);
-			regs->flags &= ~PERF_EFLAGS_EXACT;
-		}
+		__setup_pebs_gpr_group(event, regs, gprs, sample_type);
+	}
 
-		if (sample_type & PERF_SAMPLE_REGS_INTR)
-			adaptive_pebs_save_regs(regs, gprs);
+	if (format_group & PEBS_DATACFG_MEMINFO) {
+		u64 latency = x86_pmu.flags & PMU_FL_INSTR_LATENCY ?
+				meminfo->cache_latency : meminfo->mem_latency;
+		u64 instr_latency = x86_pmu.flags & PMU_FL_INSTR_LATENCY ?
+				meminfo->instr_latency : 0;
+		u64 ax = gprs ? gprs->ax : 0;
+
+		__setup_pebs_meminfo_group(event, data, sample_type, latency,
+					   instr_latency, meminfo->address,
+					   meminfo->aux, meminfo->tsx_tuning,
+					   ax);
 	}
 
-	if (format_size & PEBS_DATACFG_MEMINFO) {
-		if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) {
-			u64 weight = meminfo->latency;
+	if (format_group & PEBS_DATACFG_XMMS) {
+		struct pebs_xmm *xmm = next_record;
 
-			if (x86_pmu.flags & PMU_FL_INSTR_LATENCY) {
-				data->weight.var2_w = weight & PEBS_LATENCY_MASK;
-				weight >>= PEBS_CACHE_LATENCY_OFFSET;
-			}
+		next_record = xmm + 1;
+		perf_regs->xmm_regs = xmm->xmm;
+	}
 
-			/*
-			 * Although meminfo::latency is defined as a u64,
-			 * only the lower 32 bits include the valid data
-			 * in practice on Ice Lake and earlier platforms.
-			 */
-			if (sample_type & PERF_SAMPLE_WEIGHT) {
-				data->weight.full = weight ?:
-					intel_get_tsx_weight(meminfo->tsx_tuning);
-			} else {
-				data->weight.var1_dw = (u32)(weight & PEBS_LATENCY_MASK) ?:
-					intel_get_tsx_weight(meminfo->tsx_tuning);
-			}
-			data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;
-		}
+	if (format_group & PEBS_DATACFG_LBRS) {
+		struct lbr_entry *lbr = next_record;
+		int num_lbr = ((format_group >> PEBS_DATACFG_LBR_SHIFT)
+					& 0xff) + 1;
+		next_record = next_record + num_lbr * sizeof(struct lbr_entry);
 
-		if (sample_type & PERF_SAMPLE_DATA_SRC) {
-			data->data_src.val = get_data_src(event, meminfo->aux);
-			data->sample_flags |= PERF_SAMPLE_DATA_SRC;
+		if (has_branch_stack(event)) {
+			intel_pmu_store_pebs_lbrs(lbr);
+			intel_pmu_lbr_save_brstack(data, cpuc, event);
 		}
+	}
 
-		if (sample_type & PERF_SAMPLE_ADDR_TYPE) {
-			data->addr = meminfo->address;
-			data->sample_flags |= PERF_SAMPLE_ADDR;
-		}
+	if (format_group & (PEBS_DATACFG_CNTR | PEBS_DATACFG_METRICS)) {
+		struct pebs_cntr_header *cntr = next_record;
+		unsigned int nr;
 
-		if (sample_type & PERF_SAMPLE_TRANSACTION) {
-			data->txn = intel_get_tsx_transaction(meminfo->tsx_tuning,
-							  gprs ? gprs->ax : 0);
-			data->sample_flags |= PERF_SAMPLE_TRANSACTION;
+		next_record += sizeof(struct pebs_cntr_header);
+		/*
+		 * The PEBS_DATA_CFG is a global register, which is the
+		 * superset configuration for all PEBS events.
+		 * For the PEBS record of non-sample-read group, ignore
+		 * the counter snapshot fields.
+		 */
+		if (is_pebs_counter_event_group(event)) {
+			__setup_pebs_counter_group(cpuc, event, cntr, next_record);
+			data->sample_flags |= PERF_SAMPLE_READ;
 		}
+
+		nr = hweight32(cntr->cntr) + hweight32(cntr->fixed);
+		if (cntr->metrics == INTEL_CNTR_METRICS)
+			nr += 2;
+		next_record += nr * sizeof(u64);
 	}
 
-	if (format_size & PEBS_DATACFG_XMMS) {
-		struct pebs_xmm *xmm = next_record;
+	WARN_ONCE(next_record != __pebs + basic->format_size,
+			"PEBS record size %u, expected %llu, config %llx\n",
+			basic->format_size,
+			(u64)(next_record - __pebs),
+			format_group);
+}
 
-		next_record = xmm + 1;
+static inline bool arch_pebs_record_continued(struct arch_pebs_header *header)
+{
+	/* Continue bit or null PEBS record indicates fragment follows. */
+	return header->cont || !(header->format & GENMASK_ULL(63, 16));
+}
+
+static void setup_arch_pebs_sample_data(struct perf_event *event,
+					struct pt_regs *iregs,
+					void *__pebs,
+					struct perf_sample_data *data,
+					struct pt_regs *regs)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	u64 sample_type = event->attr.sample_type;
+	struct arch_pebs_header *header = NULL;
+	struct arch_pebs_aux *meminfo = NULL;
+	struct arch_pebs_gprs *gprs = NULL;
+	struct x86_perf_regs *perf_regs;
+	void *next_record;
+	void *at = __pebs;
+
+	if (at == NULL)
+		return;
+
+	perf_regs = container_of(regs, struct x86_perf_regs, regs);
+	perf_regs->xmm_regs = NULL;
+
+	__setup_perf_sample_data(event, iregs, data);
+
+	*regs = *iregs;
+
+again:
+	header = at;
+	next_record = at + sizeof(struct arch_pebs_header);
+	if (header->basic) {
+		struct arch_pebs_basic *basic = next_record;
+		u16 retire = 0;
+
+		next_record = basic + 1;
+
+		if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT)
+			retire = basic->valid ? basic->retire : 0;
+		__setup_pebs_basic_group(event, regs, data, sample_type,
+				 basic->ip, basic->tsc, retire);
+	}
+
+	/*
+	 * The record for MEMINFO is in front of GP
+	 * But PERF_SAMPLE_TRANSACTION needs gprs->ax.
+	 * Save the pointer here but process later.
+	 */
+	if (header->aux) {
+		meminfo = next_record;
+		next_record = meminfo + 1;
+	}
+
+	if (header->gpr) {
+		gprs = next_record;
+		next_record = gprs + 1;
+
+		__setup_pebs_gpr_group(event, regs,
+				       (struct pebs_gprs *)gprs,
+				       sample_type);
+	}
+
+	if (header->aux) {
+		u64 ax = gprs ? gprs->ax : 0;
+
+		__setup_pebs_meminfo_group(event, data, sample_type,
+					   meminfo->cache_latency,
+					   meminfo->instr_latency,
+					   meminfo->address, meminfo->aux,
+					   meminfo->tsx_tuning, ax);
+	}
+
+	if (header->xmm) {
+		struct pebs_xmm *xmm;
+
+		next_record += sizeof(struct arch_pebs_xer_header);
+
+		xmm = next_record;
 		perf_regs->xmm_regs = xmm->xmm;
+		next_record = xmm + 1;
 	}
 
-	if (format_size & PEBS_DATACFG_LBRS) {
-		struct lbr_entry *lbr = next_record;
-		int num_lbr = ((format_size >> PEBS_DATACFG_LBR_SHIFT)
-					& 0xff) + 1;
-		next_record = next_record + num_lbr * sizeof(struct lbr_entry);
+	if (header->lbr) {
+		struct arch_pebs_lbr_header *lbr_header = next_record;
+		struct lbr_entry *lbr;
+		int num_lbr;
+
+		next_record = lbr_header + 1;
+		lbr = next_record;
+
+		num_lbr = header->lbr == ARCH_PEBS_LBR_NUM_VAR ?
+				lbr_header->depth :
+				header->lbr * ARCH_PEBS_BASE_LBR_ENTRIES;
+		next_record += num_lbr * sizeof(struct lbr_entry);
 
 		if (has_branch_stack(event)) {
 			intel_pmu_store_pebs_lbrs(lbr);
@@ -2034,11 +2452,29 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event,
 		}
 	}
 
-	WARN_ONCE(next_record != __pebs + (format_size >> 48),
-			"PEBS record size %llu, expected %llu, config %llx\n",
-			format_size >> 48,
-			(u64)(next_record - __pebs),
-			basic->format_size);
+	if (header->cntr) {
+		struct arch_pebs_cntr_header *cntr = next_record;
+		unsigned int nr;
+
+		next_record += sizeof(struct arch_pebs_cntr_header);
+
+		if (is_pebs_counter_event_group(event)) {
+			__setup_pebs_counter_group(cpuc, event,
+				(struct pebs_cntr_header *)cntr, next_record);
+			data->sample_flags |= PERF_SAMPLE_READ;
+		}
+
+		nr = hweight32(cntr->cntr) + hweight32(cntr->fixed);
+		if (cntr->metrics == INTEL_CNTR_METRICS)
+			nr += 2;
+		next_record += nr * sizeof(u64);
+	}
+
+	/* Parse followed fragments if there are. */
+	if (arch_pebs_record_continued(header)) {
+		at = at + header->size;
+		goto again;
+	}
 }
 
 static inline void *
@@ -2079,15 +2515,6 @@ get_next_pebs_record_by_bit(void *base, void *top, int bit)
 	return NULL;
 }
 
-void intel_pmu_auto_reload_read(struct perf_event *event)
-{
-	WARN_ON(!(event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD));
-
-	perf_pmu_disable(event->pmu);
-	intel_pmu_drain_pebs_buffer();
-	perf_pmu_enable(event->pmu);
-}
-
 /*
  * Special variant of intel_pmu_save_and_restart() for auto-reload.
  */
@@ -2108,7 +2535,7 @@ intel_pmu_save_and_restart_reload(struct perf_event *event, int count)
 	WARN_ON(this_cpu_read(cpu_hw_events.enabled));
 
 	prev_raw_count = local64_read(&hwc->prev_count);
-	rdpmcl(hwc->event_base_rdpmc, new_raw_count);
+	new_raw_count = rdpmc(hwc->event_base_rdpmc);
 	local64_set(&hwc->prev_count, new_raw_count);
 
 	/*
@@ -2149,46 +2576,33 @@ intel_pmu_save_and_restart_reload(struct perf_event *event, int count)
 	return 0;
 }
 
+typedef void (*setup_fn)(struct perf_event *, struct pt_regs *, void *,
+			 struct perf_sample_data *, struct pt_regs *);
+
+static struct pt_regs dummy_iregs;
+
 static __always_inline void
 __intel_pmu_pebs_event(struct perf_event *event,
 		       struct pt_regs *iregs,
+		       struct pt_regs *regs,
 		       struct perf_sample_data *data,
-		       void *base, void *top,
-		       int bit, int count,
-		       void (*setup_sample)(struct perf_event *,
-					    struct pt_regs *,
-					    void *,
-					    struct perf_sample_data *,
-					    struct pt_regs *))
+		       void *at,
+		       setup_fn setup_sample)
 {
-	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
-	struct hw_perf_event *hwc = &event->hw;
-	struct x86_perf_regs perf_regs;
-	struct pt_regs *regs = &perf_regs.regs;
-	void *at = get_next_pebs_record_by_bit(base, top, bit);
-	static struct pt_regs dummy_iregs;
-
-	if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
-		/*
-		 * Now, auto-reload is only enabled in fixed period mode.
-		 * The reload value is always hwc->sample_period.
-		 * May need to change it, if auto-reload is enabled in
-		 * freq mode later.
-		 */
-		intel_pmu_save_and_restart_reload(event, count);
-	} else if (!intel_pmu_save_and_restart(event))
-		return;
-
-	if (!iregs)
-		iregs = &dummy_iregs;
+	setup_sample(event, iregs, at, data, regs);
+	perf_event_output(event, data, regs);
+}
 
-	while (count > 1) {
-		setup_sample(event, iregs, at, data, regs);
-		perf_event_output(event, data, regs);
-		at += cpuc->pebs_record_size;
-		at = get_next_pebs_record_by_bit(at, top, bit);
-		count--;
-	}
+static __always_inline void
+__intel_pmu_pebs_last_event(struct perf_event *event,
+			    struct pt_regs *iregs,
+			    struct pt_regs *regs,
+			    struct perf_sample_data *data,
+			    void *at,
+			    int count,
+			    setup_fn setup_sample)
+{
+	struct hw_perf_event *hwc = &event->hw;
 
 	setup_sample(event, iregs, at, data, regs);
 	if (iregs == &dummy_iregs) {
@@ -2204,11 +2618,73 @@ __intel_pmu_pebs_event(struct perf_event *event,
 		 * All but the last records are processed.
 		 * The last one is left to be able to call the overflow handler.
 		 */
-		if (perf_event_overflow(event, data, regs))
-			x86_pmu_stop(event, 0);
+		perf_event_overflow(event, data, regs);
+	}
+
+	if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
+		if ((is_pebs_counter_event_group(event))) {
+			/*
+			 * The value of each sample has been updated when setup
+			 * the corresponding sample data.
+			 */
+			perf_event_update_userpage(event);
+		} else {
+			/*
+			 * Now, auto-reload is only enabled in fixed period mode.
+			 * The reload value is always hwc->sample_period.
+			 * May need to change it, if auto-reload is enabled in
+			 * freq mode later.
+			 */
+			intel_pmu_save_and_restart_reload(event, count);
+		}
+	} else {
+		/*
+		 * For a non-precise event, it's possible the
+		 * counters-snapshotting records a positive value for the
+		 * overflowed event. Then the HW auto-reload mechanism
+		 * reset the counter to 0 immediately, because the
+		 * pebs_event_reset is cleared if the PERF_X86_EVENT_AUTO_RELOAD
+		 * is not set. The counter backwards may be observed in a
+		 * PMI handler.
+		 *
+		 * Since the event value has been updated when processing the
+		 * counters-snapshotting record, only needs to set the new
+		 * period for the counter.
+		 */
+		if (is_pebs_counter_event_group(event))
+			static_call(x86_pmu_set_period)(event);
+		else
+			intel_pmu_save_and_restart(event);
 	}
 }
 
+static __always_inline void
+__intel_pmu_pebs_events(struct perf_event *event,
+			struct pt_regs *iregs,
+			struct perf_sample_data *data,
+			void *base, void *top,
+			int bit, int count,
+			setup_fn setup_sample)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	struct x86_perf_regs perf_regs;
+	struct pt_regs *regs = &perf_regs.regs;
+	void *at = get_next_pebs_record_by_bit(base, top, bit);
+	int cnt = count;
+
+	if (!iregs)
+		iregs = &dummy_iregs;
+
+	while (cnt > 1) {
+		__intel_pmu_pebs_event(event, iregs, regs, data, at, setup_sample);
+		at += cpuc->pebs_record_size;
+		at = get_next_pebs_record_by_bit(at, top, bit);
+		cnt--;
+	}
+
+	__intel_pmu_pebs_last_event(event, iregs, regs, data, at, count, setup_sample);
+}
+
 static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -2243,12 +2719,13 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_
 		return;
 	}
 
-	__intel_pmu_pebs_event(event, iregs, data, at, top, 0, n,
-			       setup_pebs_fixed_sample_data);
+	__intel_pmu_pebs_events(event, iregs, data, at, top, 0, n,
+				setup_pebs_fixed_sample_data);
 }
 
-static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, int size)
+static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, u64 mask)
 {
+	u64 pebs_enabled = cpuc->pebs_enabled & mask;
 	struct perf_event *event;
 	int bit;
 
@@ -2259,7 +2736,7 @@ static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, int
 	 * It needs to call intel_pmu_save_and_restart_reload() to
 	 * update the event->count for this case.
 	 */
-	for_each_set_bit(bit, (unsigned long *)&cpuc->pebs_enabled, size) {
+	for_each_set_bit(bit, (unsigned long *)&pebs_enabled, X86_PMC_IDX_MAX) {
 		event = cpuc->events[bit];
 		if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
 			intel_pmu_save_and_restart_reload(event, 0);
@@ -2294,7 +2771,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d
 	}
 
 	if (unlikely(base >= top)) {
-		intel_pmu_pebs_event_update_no_drain(cpuc, size);
+		intel_pmu_pebs_event_update_no_drain(cpuc, mask);
 		return;
 	}
 
@@ -2370,26 +2847,79 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d
 		if (error[bit]) {
 			perf_log_lost_samples(event, error[bit]);
 
-			if (iregs && perf_event_account_interrupt(event))
-				x86_pmu_stop(event, 0);
+			if (iregs)
+				perf_event_account_interrupt(event);
 		}
 
 		if (counts[bit]) {
-			__intel_pmu_pebs_event(event, iregs, data, base,
-					       top, bit, counts[bit],
-					       setup_pebs_fixed_sample_data);
+			__intel_pmu_pebs_events(event, iregs, data, base,
+						top, bit, counts[bit],
+						setup_pebs_fixed_sample_data);
+		}
+	}
+}
+
+static __always_inline void
+__intel_pmu_handle_pebs_record(struct pt_regs *iregs,
+			       struct pt_regs *regs,
+			       struct perf_sample_data *data,
+			       void *at, u64 pebs_status,
+			       short *counts, void **last,
+			       setup_fn setup_sample)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	struct perf_event *event;
+	int bit;
+
+	for_each_set_bit(bit, (unsigned long *)&pebs_status, X86_PMC_IDX_MAX) {
+		event = cpuc->events[bit];
+
+		if (WARN_ON_ONCE(!event) ||
+		    WARN_ON_ONCE(!event->attr.precise_ip))
+			continue;
+
+		if (counts[bit]++) {
+			__intel_pmu_pebs_event(event, iregs, regs, data,
+					       last[bit], setup_sample);
 		}
+
+		last[bit] = at;
 	}
 }
 
+static __always_inline void
+__intel_pmu_handle_last_pebs_record(struct pt_regs *iregs,
+				    struct pt_regs *regs,
+				    struct perf_sample_data *data,
+				    u64 mask, short *counts, void **last,
+				    setup_fn setup_sample)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	struct perf_event *event;
+	int bit;
+
+	for_each_set_bit(bit, (unsigned long *)&mask, X86_PMC_IDX_MAX) {
+		if (!counts[bit])
+			continue;
+
+		event = cpuc->events[bit];
+
+		__intel_pmu_pebs_last_event(event, iregs, regs, data, last[bit],
+					    counts[bit], setup_sample);
+	}
+
+}
+
 static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data)
 {
 	short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
+	void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS];
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct debug_store *ds = cpuc->ds;
-	struct perf_event *event;
+	struct x86_perf_regs perf_regs;
+	struct pt_regs *regs = &perf_regs.regs;
+	struct pebs_basic *basic;
 	void *base, *at, *top;
-	int bit;
 	u64 mask;
 
 	if (!x86_pmu.pebs_active)
@@ -2402,44 +2932,132 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d
 
 	mask = hybrid(cpuc->pmu, pebs_events_mask) |
 	       (hybrid(cpuc->pmu, fixed_cntr_mask64) << INTEL_PMC_IDX_FIXED);
+	mask &= cpuc->pebs_enabled;
 
 	if (unlikely(base >= top)) {
-		intel_pmu_pebs_event_update_no_drain(cpuc, X86_PMC_IDX_MAX);
+		intel_pmu_pebs_event_update_no_drain(cpuc, mask);
 		return;
 	}
 
-	for (at = base; at < top; at += cpuc->pebs_record_size) {
+	if (!iregs)
+		iregs = &dummy_iregs;
+
+	/* Process all but the last event for each counter. */
+	for (at = base; at < top; at += basic->format_size) {
 		u64 pebs_status;
 
-		pebs_status = get_pebs_status(at) & cpuc->pebs_enabled;
-		pebs_status &= mask;
+		basic = at;
+		if (basic->format_size != cpuc->pebs_record_size)
+			continue;
 
-		for_each_set_bit(bit, (unsigned long *)&pebs_status, X86_PMC_IDX_MAX)
-			counts[bit]++;
+		pebs_status = mask & basic->applicable_counters;
+		__intel_pmu_handle_pebs_record(iregs, regs, data, at,
+					       pebs_status, counts, last,
+					       setup_pebs_adaptive_sample_data);
 	}
 
-	for_each_set_bit(bit, (unsigned long *)&mask, X86_PMC_IDX_MAX) {
-		if (counts[bit] == 0)
-			continue;
+	__intel_pmu_handle_last_pebs_record(iregs, regs, data, mask, counts, last,
+					    setup_pebs_adaptive_sample_data);
+}
 
-		event = cpuc->events[bit];
-		if (WARN_ON_ONCE(!event))
-			continue;
+static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
+				      struct perf_sample_data *data)
+{
+	short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
+	void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS];
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	union arch_pebs_index index;
+	struct x86_perf_regs perf_regs;
+	struct pt_regs *regs = &perf_regs.regs;
+	void *base, *at, *top;
+	u64 mask;
 
-		if (WARN_ON_ONCE(!event->attr.precise_ip))
+	rdmsrq(MSR_IA32_PEBS_INDEX, index.whole);
+
+	if (unlikely(!index.wr)) {
+		intel_pmu_pebs_event_update_no_drain(cpuc, X86_PMC_IDX_MAX);
+		return;
+	}
+
+	base = cpuc->pebs_vaddr;
+	top = cpuc->pebs_vaddr + (index.wr << ARCH_PEBS_INDEX_WR_SHIFT);
+
+	index.wr = 0;
+	index.full = 0;
+	index.en = 1;
+	if (cpuc->n_pebs == cpuc->n_large_pebs)
+		index.thresh = ARCH_PEBS_THRESH_MULTI;
+	else
+		index.thresh = ARCH_PEBS_THRESH_SINGLE;
+	wrmsrq(MSR_IA32_PEBS_INDEX, index.whole);
+
+	mask = hybrid(cpuc->pmu, arch_pebs_cap).counters & cpuc->pebs_enabled;
+
+	if (!iregs)
+		iregs = &dummy_iregs;
+
+	/* Process all but the last event for each counter. */
+	for (at = base; at < top;) {
+		struct arch_pebs_header *header;
+		struct arch_pebs_basic *basic;
+		u64 pebs_status;
+
+		header = at;
+
+		if (WARN_ON_ONCE(!header->size))
+			break;
+
+		/* 1st fragment or single record must have basic group */
+		if (!header->basic) {
+			at += header->size;
 			continue;
+		}
+
+		basic = at + sizeof(struct arch_pebs_header);
+		pebs_status = mask & basic->applicable_counters;
+		__intel_pmu_handle_pebs_record(iregs, regs, data, at,
+					       pebs_status, counts, last,
+					       setup_arch_pebs_sample_data);
+
+		/* Skip non-last fragments */
+		while (arch_pebs_record_continued(header)) {
+			if (!header->size)
+				break;
+			at += header->size;
+			header = at;
+		}
 
-		__intel_pmu_pebs_event(event, iregs, data, base,
-				       top, bit, counts[bit],
-				       setup_pebs_adaptive_sample_data);
+		/* Skip last fragment or the single record */
+		at += header->size;
 	}
+
+	__intel_pmu_handle_last_pebs_record(iregs, regs, data, mask,
+					    counts, last,
+					    setup_arch_pebs_sample_data);
+}
+
+static void __init intel_arch_pebs_init(void)
+{
+	/*
+	 * Current hybrid platforms always both support arch-PEBS or not
+	 * on all kinds of cores. So directly set x86_pmu.arch_pebs flag
+	 * if boot cpu supports arch-PEBS.
+	 */
+	x86_pmu.arch_pebs = 1;
+	x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
+	x86_pmu.drain_pebs = intel_pmu_drain_arch_pebs;
+	x86_pmu.pebs_capable = ~0ULL;
+	x86_pmu.flags |= PMU_FL_PEBS_ALL;
+
+	x86_pmu.pebs_enable = __intel_pmu_pebs_enable;
+	x86_pmu.pebs_disable = __intel_pmu_pebs_disable;
 }
 
 /*
- * BTS, PEBS probe and setup
+ * PEBS probe and setup
  */
 
-void __init intel_ds_init(void)
+static void __init intel_ds_pebs_init(void)
 {
 	/*
 	 * No support for 32bit formats
@@ -2447,13 +3065,12 @@ void __init intel_ds_init(void)
 	if (!boot_cpu_has(X86_FEATURE_DTES64))
 		return;
 
-	x86_pmu.bts  = boot_cpu_has(X86_FEATURE_BTS);
-	x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
+	x86_pmu.ds_pebs = boot_cpu_has(X86_FEATURE_PEBS);
 	x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
 	if (x86_pmu.version <= 4)
 		x86_pmu.pebs_no_isolation = 1;
 
-	if (x86_pmu.pebs) {
+	if (x86_pmu.ds_pebs) {
 		char pebs_type = x86_pmu.intel_cap.pebs_trap ?  '+' : '-';
 		char *pebs_qual = "";
 		int format = x86_pmu.intel_cap.pebs_format;
@@ -2461,6 +3078,11 @@ void __init intel_ds_init(void)
 		if (format < 4)
 			x86_pmu.intel_cap.pebs_baseline = 0;
 
+		x86_pmu.pebs_enable = intel_pmu_pebs_enable;
+		x86_pmu.pebs_disable = intel_pmu_pebs_disable;
+		x86_pmu.pebs_enable_all = intel_pmu_pebs_enable_all;
+		x86_pmu.pebs_disable_all = intel_pmu_pebs_disable_all;
+
 		switch (format) {
 		case 0:
 			pr_cont("PEBS fmt0%c, ", pebs_type);
@@ -2496,6 +3118,10 @@ void __init intel_ds_init(void)
 			x86_pmu.large_pebs_flags |= PERF_SAMPLE_TIME;
 			break;
 
+		case 6:
+			if (x86_pmu.intel_cap.pebs_baseline)
+				x86_pmu.large_pebs_flags |= PERF_SAMPLE_READ;
+			fallthrough;
 		case 5:
 			x86_pmu.pebs_ept = 1;
 			fallthrough;
@@ -2520,9 +3146,17 @@ void __init intel_ds_init(void)
 					  PERF_SAMPLE_REGS_USER |
 					  PERF_SAMPLE_REGS_INTR);
 			}
-			pr_cont("PEBS fmt4%c%s, ", pebs_type, pebs_qual);
+			pr_cont("PEBS fmt%d%c%s, ", format, pebs_type, pebs_qual);
 
-			if (!is_hybrid() && x86_pmu.intel_cap.pebs_output_pt_available) {
+			/*
+			 * The PEBS-via-PT is not supported on hybrid platforms,
+			 * because not all CPUs of a hybrid machine support it.
+			 * The global x86_pmu.intel_cap, which only contains the
+			 * common capabilities, is used to check the availability
+			 * of the feature. The per-PMU pebs_output_pt_available
+			 * in a hybrid machine should be ignored.
+			 */
+			if (x86_pmu.intel_cap.pebs_output_pt_available) {
 				pr_cont("PEBS-via-PT, ");
 				x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_AUX_OUTPUT;
 			}
@@ -2531,17 +3165,25 @@ void __init intel_ds_init(void)
 
 		default:
 			pr_cont("no PEBS fmt%d%c, ", format, pebs_type);
-			x86_pmu.pebs = 0;
+			x86_pmu.ds_pebs = 0;
 		}
 	}
 }
 
+void __init intel_pebs_init(void)
+{
+	if (x86_pmu.intel_cap.pebs_format == 0xf)
+		intel_arch_pebs_init();
+	else
+		intel_ds_pebs_init();
+}
+
 void perf_restore_debug_store(void)
 {
 	struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
 
-	if (!x86_pmu.bts && !x86_pmu.pebs)
+	if (!x86_pmu.bts && !x86_pmu.ds_pebs)
 		return;
 
-	wrmsrl(MSR_IA32_DS_AREA, (unsigned long)ds);
+	wrmsrq(MSR_IA32_DS_AREA, (unsigned long)ds);
 }
diff --git a/arch/x86/events/intel/knc.c b/arch/x86/events/intel/knc.c
index 034a1f6a457c..e614baf42926 100644
--- a/arch/x86/events/intel/knc.c
+++ b/arch/x86/events/intel/knc.c
@@ -5,6 +5,7 @@
 #include <linux/types.h>
 
 #include <asm/hardirq.h>
+#include <asm/msr.h>
 
 #include "../perf_event.h"
 
@@ -159,18 +160,18 @@ static void knc_pmu_disable_all(void)
 {
 	u64 val;
 
-	rdmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
+	rdmsrq(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
 	val &= ~(KNC_ENABLE_COUNTER0|KNC_ENABLE_COUNTER1);
-	wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
+	wrmsrq(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
 }
 
 static void knc_pmu_enable_all(int added)
 {
 	u64 val;
 
-	rdmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
+	rdmsrq(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
 	val |= (KNC_ENABLE_COUNTER0|KNC_ENABLE_COUNTER1);
-	wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
+	wrmsrq(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
 }
 
 static inline void
@@ -182,7 +183,7 @@ knc_pmu_disable_event(struct perf_event *event)
 	val = hwc->config;
 	val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
 
-	(void)wrmsrl_safe(hwc->config_base + hwc->idx, val);
+	(void)wrmsrq_safe(hwc->config_base + hwc->idx, val);
 }
 
 static void knc_pmu_enable_event(struct perf_event *event)
@@ -193,21 +194,21 @@ static void knc_pmu_enable_event(struct perf_event *event)
 	val = hwc->config;
 	val |= ARCH_PERFMON_EVENTSEL_ENABLE;
 
-	(void)wrmsrl_safe(hwc->config_base + hwc->idx, val);
+	(void)wrmsrq_safe(hwc->config_base + hwc->idx, val);
 }
 
 static inline u64 knc_pmu_get_status(void)
 {
 	u64 status;
 
-	rdmsrl(MSR_KNC_IA32_PERF_GLOBAL_STATUS, status);
+	rdmsrq(MSR_KNC_IA32_PERF_GLOBAL_STATUS, status);
 
 	return status;
 }
 
 static inline void knc_pmu_ack_status(u64 ack)
 {
-	wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_OVF_CONTROL, ack);
+	wrmsrq(MSR_KNC_IA32_PERF_GLOBAL_OVF_CONTROL, ack);
 }
 
 static int knc_pmu_handle_irq(struct pt_regs *regs)
@@ -241,19 +242,20 @@ again:
 
 	for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
 		struct perf_event *event = cpuc->events[bit];
+		u64 last_period;
 
 		handled++;
 
 		if (!test_bit(bit, cpuc->active_mask))
 			continue;
 
+		last_period = event->hw.last_period;
 		if (!intel_pmu_save_and_restart(event))
 			continue;
 
-		perf_sample_data_init(&data, 0, event->hw.last_period);
+		perf_sample_data_init(&data, 0, last_period);
 
-		if (perf_event_overflow(event, &data, regs))
-			x86_pmu_stop(event, 0);
+		perf_event_overflow(event, &data, regs);
 	}
 
 	/*
diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index dc641b50814e..72f2adcda7c6 100644
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
+#include <linux/kvm_types.h>
 #include <linux/perf_event.h>
 #include <linux/types.h>
 
@@ -137,9 +138,9 @@ static void __intel_pmu_lbr_enable(bool pmi)
 	if (cpuc->lbr_sel)
 		lbr_select = cpuc->lbr_sel->config & x86_pmu.lbr_sel_mask;
 	if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && !pmi && cpuc->lbr_sel)
-		wrmsrl(MSR_LBR_SELECT, lbr_select);
+		wrmsrq(MSR_LBR_SELECT, lbr_select);
 
-	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
+	rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl);
 	orig_debugctl = debugctl;
 
 	if (!static_cpu_has(X86_FEATURE_ARCH_LBR))
@@ -155,10 +156,10 @@ static void __intel_pmu_lbr_enable(bool pmi)
 		debugctl |= DEBUGCTLMSR_FREEZE_LBRS_ON_PMI;
 
 	if (orig_debugctl != debugctl)
-		wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
+		wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl);
 
 	if (static_cpu_has(X86_FEATURE_ARCH_LBR))
-		wrmsrl(MSR_ARCH_LBR_CTL, lbr_select | ARCH_LBR_CTL_LBREN);
+		wrmsrq(MSR_ARCH_LBR_CTL, lbr_select | ARCH_LBR_CTL_LBREN);
 }
 
 void intel_pmu_lbr_reset_32(void)
@@ -166,7 +167,7 @@ void intel_pmu_lbr_reset_32(void)
 	int i;
 
 	for (i = 0; i < x86_pmu.lbr_nr; i++)
-		wrmsrl(x86_pmu.lbr_from + i, 0);
+		wrmsrq(x86_pmu.lbr_from + i, 0);
 }
 
 void intel_pmu_lbr_reset_64(void)
@@ -174,17 +175,17 @@ void intel_pmu_lbr_reset_64(void)
 	int i;
 
 	for (i = 0; i < x86_pmu.lbr_nr; i++) {
-		wrmsrl(x86_pmu.lbr_from + i, 0);
-		wrmsrl(x86_pmu.lbr_to   + i, 0);
+		wrmsrq(x86_pmu.lbr_from + i, 0);
+		wrmsrq(x86_pmu.lbr_to   + i, 0);
 		if (x86_pmu.lbr_has_info)
-			wrmsrl(x86_pmu.lbr_info + i, 0);
+			wrmsrq(x86_pmu.lbr_info + i, 0);
 	}
 }
 
 static void intel_pmu_arch_lbr_reset(void)
 {
 	/* Write to ARCH_LBR_DEPTH MSR, all LBR entries are reset to 0 */
-	wrmsrl(MSR_ARCH_LBR_DEPTH, x86_pmu.lbr_nr);
+	wrmsrq(MSR_ARCH_LBR_DEPTH, x86_pmu.lbr_nr);
 }
 
 void intel_pmu_lbr_reset(void)
@@ -199,7 +200,7 @@ void intel_pmu_lbr_reset(void)
 	cpuc->last_task_ctx = NULL;
 	cpuc->last_log_id = 0;
 	if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && cpuc->lbr_select)
-		wrmsrl(MSR_LBR_SELECT, 0);
+		wrmsrq(MSR_LBR_SELECT, 0);
 }
 
 /*
@@ -209,7 +210,7 @@ static inline u64 intel_pmu_lbr_tos(void)
 {
 	u64 tos;
 
-	rdmsrl(x86_pmu.lbr_tos, tos);
+	rdmsrq(x86_pmu.lbr_tos, tos);
 	return tos;
 }
 
@@ -282,17 +283,17 @@ static u64 lbr_from_signext_quirk_rd(u64 val)
 static __always_inline void wrlbr_from(unsigned int idx, u64 val)
 {
 	val = lbr_from_signext_quirk_wr(val);
-	wrmsrl(x86_pmu.lbr_from + idx, val);
+	wrmsrq(x86_pmu.lbr_from + idx, val);
 }
 
 static __always_inline void wrlbr_to(unsigned int idx, u64 val)
 {
-	wrmsrl(x86_pmu.lbr_to + idx, val);
+	wrmsrq(x86_pmu.lbr_to + idx, val);
 }
 
 static __always_inline void wrlbr_info(unsigned int idx, u64 val)
 {
-	wrmsrl(x86_pmu.lbr_info + idx, val);
+	wrmsrq(x86_pmu.lbr_info + idx, val);
 }
 
 static __always_inline u64 rdlbr_from(unsigned int idx, struct lbr_entry *lbr)
@@ -302,7 +303,7 @@ static __always_inline u64 rdlbr_from(unsigned int idx, struct lbr_entry *lbr)
 	if (lbr)
 		return lbr->from;
 
-	rdmsrl(x86_pmu.lbr_from + idx, val);
+	rdmsrq(x86_pmu.lbr_from + idx, val);
 
 	return lbr_from_signext_quirk_rd(val);
 }
@@ -314,7 +315,7 @@ static __always_inline u64 rdlbr_to(unsigned int idx, struct lbr_entry *lbr)
 	if (lbr)
 		return lbr->to;
 
-	rdmsrl(x86_pmu.lbr_to + idx, val);
+	rdmsrq(x86_pmu.lbr_to + idx, val);
 
 	return val;
 }
@@ -326,7 +327,7 @@ static __always_inline u64 rdlbr_info(unsigned int idx, struct lbr_entry *lbr)
 	if (lbr)
 		return lbr->info;
 
-	rdmsrl(x86_pmu.lbr_info + idx, val);
+	rdmsrq(x86_pmu.lbr_info + idx, val);
 
 	return val;
 }
@@ -380,10 +381,10 @@ void intel_pmu_lbr_restore(void *ctx)
 			wrlbr_info(lbr_idx, 0);
 	}
 
-	wrmsrl(x86_pmu.lbr_tos, tos);
+	wrmsrq(x86_pmu.lbr_tos, tos);
 
 	if (cpuc->lbr_select)
-		wrmsrl(MSR_LBR_SELECT, task_ctx->lbr_sel);
+		wrmsrq(MSR_LBR_SELECT, task_ctx->lbr_sel);
 }
 
 static void intel_pmu_arch_lbr_restore(void *ctx)
@@ -422,11 +423,17 @@ static __always_inline bool lbr_is_reset_in_cstate(void *ctx)
 	return !rdlbr_from(((struct x86_perf_task_context *)ctx)->tos, NULL);
 }
 
+static inline bool has_lbr_callstack_users(void *ctx)
+{
+	return task_context_opt(ctx)->lbr_callstack_users ||
+	       x86_pmu.lbr_callstack_users;
+}
+
 static void __intel_pmu_lbr_restore(void *ctx)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
-	if (task_context_opt(ctx)->lbr_callstack_users == 0 ||
+	if (!has_lbr_callstack_users(ctx) ||
 	    task_context_opt(ctx)->lbr_stack_state == LBR_NONE) {
 		intel_pmu_lbr_reset();
 		return;
@@ -469,7 +476,7 @@ void intel_pmu_lbr_save(void *ctx)
 	task_ctx->tos = tos;
 
 	if (cpuc->lbr_select)
-		rdmsrl(MSR_LBR_SELECT, task_ctx->lbr_sel);
+		rdmsrq(MSR_LBR_SELECT, task_ctx->lbr_sel);
 }
 
 static void intel_pmu_arch_lbr_save(void *ctx)
@@ -503,7 +510,7 @@ static void __intel_pmu_lbr_save(void *ctx)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
-	if (task_context_opt(ctx)->lbr_callstack_users == 0) {
+	if (!has_lbr_callstack_users(ctx)) {
 		task_context_opt(ctx)->lbr_stack_state = LBR_NONE;
 		return;
 	}
@@ -516,32 +523,11 @@ static void __intel_pmu_lbr_save(void *ctx)
 	cpuc->last_log_id = ++task_context_opt(ctx)->log_id;
 }
 
-void intel_pmu_lbr_swap_task_ctx(struct perf_event_pmu_context *prev_epc,
-				 struct perf_event_pmu_context *next_epc)
-{
-	void *prev_ctx_data, *next_ctx_data;
-
-	swap(prev_epc->task_ctx_data, next_epc->task_ctx_data);
-
-	/*
-	 * Architecture specific synchronization makes sense in case
-	 * both prev_epc->task_ctx_data and next_epc->task_ctx_data
-	 * pointers are allocated.
-	 */
-
-	prev_ctx_data = next_epc->task_ctx_data;
-	next_ctx_data = prev_epc->task_ctx_data;
-
-	if (!prev_ctx_data || !next_ctx_data)
-		return;
-
-	swap(task_context_opt(prev_ctx_data)->lbr_callstack_users,
-	     task_context_opt(next_ctx_data)->lbr_callstack_users);
-}
-
-void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
+void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
+			      struct task_struct *task, bool sched_in)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	struct perf_ctx_data *ctx_data;
 	void *task_ctx;
 
 	if (!cpuc->lbr_users)
@@ -552,14 +538,18 @@ void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched
 	 * the task was scheduled out, restore the stack. Otherwise flush
 	 * the LBR stack.
 	 */
-	task_ctx = pmu_ctx ? pmu_ctx->task_ctx_data : NULL;
+	rcu_read_lock();
+	ctx_data = rcu_dereference(task->perf_ctx_data);
+	task_ctx = ctx_data ? ctx_data->data : NULL;
 	if (task_ctx) {
 		if (sched_in)
 			__intel_pmu_lbr_restore(task_ctx);
 		else
 			__intel_pmu_lbr_save(task_ctx);
+		rcu_read_unlock();
 		return;
 	}
+	rcu_read_unlock();
 
 	/*
 	 * Since a context switch can flip the address space and LBR entries
@@ -588,9 +578,19 @@ void intel_pmu_lbr_add(struct perf_event *event)
 
 	cpuc->br_sel = event->hw.branch_reg.reg;
 
-	if (branch_user_callstack(cpuc->br_sel) && event->pmu_ctx->task_ctx_data)
-		task_context_opt(event->pmu_ctx->task_ctx_data)->lbr_callstack_users++;
+	if (branch_user_callstack(cpuc->br_sel)) {
+		if (event->attach_state & PERF_ATTACH_TASK) {
+			struct task_struct *task = event->hw.target;
+			struct perf_ctx_data *ctx_data;
 
+			rcu_read_lock();
+			ctx_data = rcu_dereference(task->perf_ctx_data);
+			if (ctx_data)
+				task_context_opt(ctx_data->data)->lbr_callstack_users++;
+			rcu_read_unlock();
+		} else
+			x86_pmu.lbr_callstack_users++;
+	}
 	/*
 	 * Request pmu::sched_task() callback, which will fire inside the
 	 * regular perf event scheduling, so that call will:
@@ -664,9 +664,19 @@ void intel_pmu_lbr_del(struct perf_event *event)
 	if (!x86_pmu.lbr_nr)
 		return;
 
-	if (branch_user_callstack(cpuc->br_sel) &&
-	    event->pmu_ctx->task_ctx_data)
-		task_context_opt(event->pmu_ctx->task_ctx_data)->lbr_callstack_users--;
+	if (branch_user_callstack(cpuc->br_sel)) {
+		if (event->attach_state & PERF_ATTACH_TASK) {
+			struct task_struct *task = event->hw.target;
+			struct perf_ctx_data *ctx_data;
+
+			rcu_read_lock();
+			ctx_data = rcu_dereference(task->perf_ctx_data);
+			if (ctx_data)
+				task_context_opt(ctx_data->data)->lbr_callstack_users--;
+			rcu_read_unlock();
+		} else
+			x86_pmu.lbr_callstack_users--;
+	}
 
 	if (event->hw.flags & PERF_X86_EVENT_LBR_SELECT)
 		cpuc->lbr_select = 0;
@@ -743,7 +753,7 @@ void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
 			u64     lbr;
 		} msr_lastbranch;
 
-		rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
+		rdmsrq(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
 
 		perf_clear_branch_entry_bitfields(br);
 
@@ -1593,7 +1603,7 @@ void __init intel_pmu_arch_lbr_init(void)
 		goto clear_arch_lbr;
 
 	/* Apply the max depth of Arch LBR */
-	if (wrmsrl_safe(MSR_ARCH_LBR_DEPTH, lbr_nr))
+	if (wrmsrq_safe(MSR_ARCH_LBR_DEPTH, lbr_nr))
 		goto clear_arch_lbr;
 
 	x86_pmu.lbr_depth_mask = eax.split.lbr_depth_mask;
@@ -1609,7 +1619,7 @@ void __init intel_pmu_arch_lbr_init(void)
 	x86_pmu.lbr_nr = lbr_nr;
 
 	if (!!x86_pmu.lbr_counters)
-		x86_pmu.flags |= PMU_FL_BR_CNTR;
+		x86_pmu.flags |= PMU_FL_BR_CNTR | PMU_FL_DYN_CONSTRAINT;
 
 	if (x86_pmu.lbr_mispred)
 		static_branch_enable(&x86_lbr_mispred);
@@ -1696,7 +1706,7 @@ void x86_perf_get_lbr(struct x86_pmu_lbr *lbr)
 	lbr->info = x86_pmu.lbr_info;
 	lbr->has_callstack = x86_pmu_has_lbr_callstack();
 }
-EXPORT_SYMBOL_GPL(x86_perf_get_lbr);
+EXPORT_SYMBOL_FOR_KVM(x86_perf_get_lbr);
 
 struct event_constraint vlbr_constraint =
 	__EVENT_CONSTRAINT(INTEL_FIXED_VLBR_EVENT, (1ULL << INTEL_PMC_IDX_FIXED_VLBR),
diff --git a/arch/x86/events/intel/p4.c b/arch/x86/events/intel/p4.c
index 844bc4fc4724..e5fd7367e45d 100644
--- a/arch/x86/events/intel/p4.c
+++ b/arch/x86/events/intel/p4.c
@@ -10,8 +10,10 @@
 #include <linux/perf_event.h>
 
 #include <asm/perf_event_p4.h>
+#include <asm/cpu_device_id.h>
 #include <asm/hardirq.h>
 #include <asm/apic.h>
+#include <asm/msr.h>
 
 #include "../perf_event.h"
 
@@ -732,9 +734,9 @@ static bool p4_event_match_cpu_model(unsigned int event_idx)
 {
 	/* INSTR_COMPLETED event only exist for model 3, 4, 6 (Prescott) */
 	if (event_idx == P4_EVENT_INSTR_COMPLETED) {
-		if (boot_cpu_data.x86_model != 3 &&
-			boot_cpu_data.x86_model != 4 &&
-			boot_cpu_data.x86_model != 6)
+		if (boot_cpu_data.x86_vfm != INTEL_P4_PRESCOTT &&
+		    boot_cpu_data.x86_vfm != INTEL_P4_PRESCOTT_2M &&
+		    boot_cpu_data.x86_vfm != INTEL_P4_CEDARMILL)
 			return false;
 	}
 
@@ -776,7 +778,7 @@ static int p4_validate_raw_event(struct perf_event *event)
 	 * the user needs special permissions to be able to use it
 	 */
 	if (p4_ht_active() && p4_event_bind_map[v].shared) {
-		v = perf_allow_cpu(&event->attr);
+		v = perf_allow_cpu();
 		if (v)
 			return v;
 	}
@@ -858,9 +860,9 @@ static inline int p4_pmu_clear_cccr_ovf(struct hw_perf_event *hwc)
 	u64 v;
 
 	/* an official way for overflow indication */
-	rdmsrl(hwc->config_base, v);
+	rdmsrq(hwc->config_base, v);
 	if (v & P4_CCCR_OVF) {
-		wrmsrl(hwc->config_base, v & ~P4_CCCR_OVF);
+		wrmsrq(hwc->config_base, v & ~P4_CCCR_OVF);
 		return 1;
 	}
 
@@ -871,7 +873,7 @@ static inline int p4_pmu_clear_cccr_ovf(struct hw_perf_event *hwc)
 	 * the counter has reached zero value and continued counting before
 	 * real NMI signal was received:
 	 */
-	rdmsrl(hwc->event_base, v);
+	rdmsrq(hwc->event_base, v);
 	if (!(v & ARCH_P4_UNFLAGGED_BIT))
 		return 1;
 
@@ -896,8 +898,8 @@ static void p4_pmu_disable_pebs(void)
 	 * So at moment let leave metrics turned on forever -- it's
 	 * ok for now but need to be revisited!
 	 *
-	 * (void)wrmsrl_safe(MSR_IA32_PEBS_ENABLE, 0);
-	 * (void)wrmsrl_safe(MSR_P4_PEBS_MATRIX_VERT, 0);
+	 * (void)wrmsrq_safe(MSR_IA32_PEBS_ENABLE, 0);
+	 * (void)wrmsrq_safe(MSR_P4_PEBS_MATRIX_VERT, 0);
 	 */
 }
 
@@ -910,7 +912,7 @@ static inline void p4_pmu_disable_event(struct perf_event *event)
 	 * state we need to clear P4_CCCR_OVF, otherwise interrupt get
 	 * asserted again and again
 	 */
-	(void)wrmsrl_safe(hwc->config_base,
+	(void)wrmsrq_safe(hwc->config_base,
 		p4_config_unpack_cccr(hwc->config) & ~P4_CCCR_ENABLE & ~P4_CCCR_OVF & ~P4_CCCR_RESERVED);
 }
 
@@ -943,8 +945,8 @@ static void p4_pmu_enable_pebs(u64 config)
 
 	bind = &p4_pebs_bind_map[idx];
 
-	(void)wrmsrl_safe(MSR_IA32_PEBS_ENABLE,	(u64)bind->metric_pebs);
-	(void)wrmsrl_safe(MSR_P4_PEBS_MATRIX_VERT,	(u64)bind->metric_vert);
+	(void)wrmsrq_safe(MSR_IA32_PEBS_ENABLE,	(u64)bind->metric_pebs);
+	(void)wrmsrq_safe(MSR_P4_PEBS_MATRIX_VERT,	(u64)bind->metric_vert);
 }
 
 static void __p4_pmu_enable_event(struct perf_event *event)
@@ -978,8 +980,8 @@ static void __p4_pmu_enable_event(struct perf_event *event)
 	 */
 	p4_pmu_enable_pebs(hwc->config);
 
-	(void)wrmsrl_safe(escr_addr, escr_conf);
-	(void)wrmsrl_safe(hwc->config_base,
+	(void)wrmsrq_safe(escr_addr, escr_conf);
+	(void)wrmsrq_safe(hwc->config_base,
 				(cccr & ~P4_CCCR_RESERVED) | P4_CCCR_ENABLE);
 }
 
@@ -1023,7 +1025,7 @@ static int p4_pmu_set_period(struct perf_event *event)
 		 *
 		 * the former idea is taken from OProfile code
 		 */
-		wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
+		wrmsrq(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
 	}
 
 	return ret;
@@ -1071,8 +1073,7 @@ static int p4_pmu_handle_irq(struct pt_regs *regs)
 			continue;
 
 
-		if (perf_event_overflow(event, &data, regs))
-			x86_pmu_stop(event, 0);
+		perf_event_overflow(event, &data, regs);
 	}
 
 	if (handled)
@@ -1397,7 +1398,7 @@ __init int p4_pmu_init(void)
 	 */
 	for_each_set_bit(i, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) {
 		reg = x86_pmu_config_addr(i);
-		wrmsrl_safe(reg, 0ULL);
+		wrmsrq_safe(reg, 0ULL);
 	}
 
 	return 0;
diff --git a/arch/x86/events/intel/p6.c b/arch/x86/events/intel/p6.c
index a6cffb4f4ef5..6e41de355bd8 100644
--- a/arch/x86/events/intel/p6.c
+++ b/arch/x86/events/intel/p6.c
@@ -2,6 +2,9 @@
 #include <linux/perf_event.h>
 #include <linux/types.h>
 
+#include <asm/cpu_device_id.h>
+#include <asm/msr.h>
+
 #include "../perf_event.h"
 
 /*
@@ -140,9 +143,9 @@ static void p6_pmu_disable_all(void)
 	u64 val;
 
 	/* p6 only has one enable register */
-	rdmsrl(MSR_P6_EVNTSEL0, val);
+	rdmsrq(MSR_P6_EVNTSEL0, val);
 	val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
-	wrmsrl(MSR_P6_EVNTSEL0, val);
+	wrmsrq(MSR_P6_EVNTSEL0, val);
 }
 
 static void p6_pmu_enable_all(int added)
@@ -150,9 +153,9 @@ static void p6_pmu_enable_all(int added)
 	unsigned long val;
 
 	/* p6 only has one enable register */
-	rdmsrl(MSR_P6_EVNTSEL0, val);
+	rdmsrq(MSR_P6_EVNTSEL0, val);
 	val |= ARCH_PERFMON_EVENTSEL_ENABLE;
-	wrmsrl(MSR_P6_EVNTSEL0, val);
+	wrmsrq(MSR_P6_EVNTSEL0, val);
 }
 
 static inline void
@@ -161,7 +164,7 @@ p6_pmu_disable_event(struct perf_event *event)
 	struct hw_perf_event *hwc = &event->hw;
 	u64 val = P6_NOP_EVENT;
 
-	(void)wrmsrl_safe(hwc->config_base, val);
+	(void)wrmsrq_safe(hwc->config_base, val);
 }
 
 static void p6_pmu_enable_event(struct perf_event *event)
@@ -178,7 +181,7 @@ static void p6_pmu_enable_event(struct perf_event *event)
 	 * to actually enable the events.
 	 */
 
-	(void)wrmsrl_safe(hwc->config_base, val);
+	(void)wrmsrq_safe(hwc->config_base, val);
 }
 
 PMU_FORMAT_ATTR(event,	"config:0-7"	);
@@ -248,30 +251,8 @@ __init int p6_pmu_init(void)
 {
 	x86_pmu = p6_pmu;
 
-	switch (boot_cpu_data.x86_model) {
-	case  1: /* Pentium Pro */
+	if (boot_cpu_data.x86_vfm == INTEL_PENTIUM_PRO)
 		x86_add_quirk(p6_pmu_rdpmc_quirk);
-		break;
-
-	case  3: /* Pentium II - Klamath */
-	case  5: /* Pentium II - Deschutes */
-	case  6: /* Pentium II - Mendocino */
-		break;
-
-	case  7: /* Pentium III - Katmai */
-	case  8: /* Pentium III - Coppermine */
-	case 10: /* Pentium III Xeon */
-	case 11: /* Pentium III - Tualatin */
-		break;
-
-	case  9: /* Pentium M - Banias */
-	case 13: /* Pentium M - Dothan */
-		break;
-
-	default:
-		pr_cont("unsupported p6 CPU model %d ", boot_cpu_data.x86_model);
-		return -ENODEV;
-	}
 
 	memcpy(hw_cache_event_ids, p6_hw_cache_event_ids,
 		sizeof(hw_cache_event_ids));
diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c
index fd4670a6694e..44524a387c58 100644
--- a/arch/x86/events/intel/pt.c
+++ b/arch/x86/events/intel/pt.c
@@ -17,12 +17,15 @@
 #include <linux/limits.h>
 #include <linux/slab.h>
 #include <linux/device.h>
+#include <linux/kvm_types.h>
 
+#include <asm/cpuid/api.h>
 #include <asm/perf_event.h>
 #include <asm/insn.h>
 #include <asm/io.h>
 #include <asm/intel_pt.h>
 #include <asm/cpu_device_id.h>
+#include <asm/msr.h>
 
 #include "../perf_event.h"
 #include "pt.h"
@@ -80,13 +83,13 @@ u32 intel_pt_validate_cap(u32 *caps, enum pt_capabilities capability)
 
 	return (c & cd->mask) >> shift;
 }
-EXPORT_SYMBOL_GPL(intel_pt_validate_cap);
+EXPORT_SYMBOL_FOR_KVM(intel_pt_validate_cap);
 
 u32 intel_pt_validate_hw_cap(enum pt_capabilities cap)
 {
 	return intel_pt_validate_cap(pt_pmu.caps, cap);
 }
-EXPORT_SYMBOL_GPL(intel_pt_validate_hw_cap);
+EXPORT_SYMBOL_FOR_KVM(intel_pt_validate_hw_cap);
 
 static ssize_t pt_cap_show(struct device *cdev,
 			   struct device_attribute *attr,
@@ -193,7 +196,7 @@ static int __init pt_pmu_hw_init(void)
 	int ret;
 	long i;
 
-	rdmsrl(MSR_PLATFORM_INFO, reg);
+	rdmsrq(MSR_PLATFORM_INFO, reg);
 	pt_pmu.max_nonturbo_ratio = (reg & 0xff00) >> 8;
 
 	/*
@@ -201,10 +204,10 @@ static int __init pt_pmu_hw_init(void)
 	 * otherwise, zero for numerator stands for "not enumerated"
 	 * as per SDM
 	 */
-	if (boot_cpu_data.cpuid_level >= CPUID_TSC_LEAF) {
+	if (boot_cpu_data.cpuid_level >= CPUID_LEAF_TSC) {
 		u32 eax, ebx, ecx, edx;
 
-		cpuid(CPUID_TSC_LEAF, &eax, &ebx, &ecx, &edx);
+		cpuid(CPUID_LEAF_TSC, &eax, &ebx, &ecx, &edx);
 
 		pt_pmu.tsc_art_num = ebx;
 		pt_pmu.tsc_art_den = eax;
@@ -229,7 +232,7 @@ static int __init pt_pmu_hw_init(void)
 		 * "IA32_VMX_MISC[bit 14]" being 1 means PT can trace
 		 * post-VMXON.
 		 */
-		rdmsrl(MSR_IA32_VMX_MISC, reg);
+		rdmsrq(MSR_IA32_VMX_MISC, reg);
 		if (reg & BIT(14))
 			pt_pmu.vmx = true;
 	}
@@ -418,11 +421,14 @@ static void pt_config_start(struct perf_event *event)
 	struct pt *pt = this_cpu_ptr(&pt_ctx);
 	u64 ctl = event->hw.aux_config;
 
+	if (READ_ONCE(event->hw.aux_paused))
+		return;
+
 	ctl |= RTIT_CTL_TRACEEN;
 	if (READ_ONCE(pt->vmx_on))
 		perf_aux_output_flag(&pt->handle, PERF_AUX_FLAG_PARTIAL);
 	else
-		wrmsrl(MSR_IA32_RTIT_CTL, ctl);
+		wrmsrq(MSR_IA32_RTIT_CTL, ctl);
 
 	WRITE_ONCE(event->hw.aux_config, ctl);
 }
@@ -481,12 +487,12 @@ static u64 pt_config_filters(struct perf_event *event)
 
 		/* avoid redundant msr writes */
 		if (pt->filters.filter[range].msr_a != filter->msr_a) {
-			wrmsrl(pt_address_ranges[range].msr_a, filter->msr_a);
+			wrmsrq(pt_address_ranges[range].msr_a, filter->msr_a);
 			pt->filters.filter[range].msr_a = filter->msr_a;
 		}
 
 		if (pt->filters.filter[range].msr_b != filter->msr_b) {
-			wrmsrl(pt_address_ranges[range].msr_b, filter->msr_b);
+			wrmsrq(pt_address_ranges[range].msr_b, filter->msr_b);
 			pt->filters.filter[range].msr_b = filter->msr_b;
 		}
 
@@ -505,7 +511,7 @@ static void pt_config(struct perf_event *event)
 	/* First round: clear STATUS, in particular the PSB byte counter. */
 	if (!event->hw.aux_config) {
 		perf_event_itrace_started(event);
-		wrmsrl(MSR_IA32_RTIT_STATUS, 0);
+		wrmsrq(MSR_IA32_RTIT_STATUS, 0);
 	}
 
 	reg = pt_config_filters(event);
@@ -534,7 +540,24 @@ static void pt_config(struct perf_event *event)
 	reg |= (event->attr.config & PT_CONFIG_MASK);
 
 	event->hw.aux_config = reg;
+
+	/*
+	 * Allow resume before starting so as not to overwrite a value set by a
+	 * PMI.
+	 */
+	barrier();
+	WRITE_ONCE(pt->resume_allowed, 1);
+	/* Configuration is complete, it is now OK to handle an NMI */
+	barrier();
+	WRITE_ONCE(pt->handle_nmi, 1);
+	barrier();
 	pt_config_start(event);
+	barrier();
+	/*
+	 * Allow pause after starting so its pt_config_stop() doesn't race with
+	 * pt_config_start().
+	 */
+	WRITE_ONCE(pt->pause_allowed, 1);
 }
 
 static void pt_config_stop(struct perf_event *event)
@@ -548,7 +571,7 @@ static void pt_config_stop(struct perf_event *event)
 
 	ctl &= ~RTIT_CTL_TRACEEN;
 	if (!READ_ONCE(pt->vmx_on))
-		wrmsrl(MSR_IA32_RTIT_CTL, ctl);
+		wrmsrq(MSR_IA32_RTIT_CTL, ctl);
 
 	WRITE_ONCE(event->hw.aux_config, ctl);
 
@@ -637,13 +660,13 @@ static void pt_config_buffer(struct pt_buffer *buf)
 	reg = virt_to_phys(base);
 	if (pt->output_base != reg) {
 		pt->output_base = reg;
-		wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, reg);
+		wrmsrq(MSR_IA32_RTIT_OUTPUT_BASE, reg);
 	}
 
 	reg = 0x7f | (mask << 7) | ((u64)buf->output_off << 32);
 	if (pt->output_mask != reg) {
 		pt->output_mask = reg;
-		wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, reg);
+		wrmsrq(MSR_IA32_RTIT_OUTPUT_MASK, reg);
 	}
 }
 
@@ -828,11 +851,13 @@ static void pt_buffer_advance(struct pt_buffer *buf)
 	buf->cur_idx++;
 
 	if (buf->cur_idx == buf->cur->last) {
-		if (buf->cur == buf->last)
+		if (buf->cur == buf->last) {
 			buf->cur = buf->first;
-		else
+			buf->wrapped = true;
+		} else {
 			buf->cur = list_entry(buf->cur->list.next, struct topa,
 					      list);
+		}
 		buf->cur_idx = 0;
 	}
 }
@@ -846,8 +871,11 @@ static void pt_buffer_advance(struct pt_buffer *buf)
 static void pt_update_head(struct pt *pt)
 {
 	struct pt_buffer *buf = perf_get_aux(&pt->handle);
+	bool wrapped = buf->wrapped;
 	u64 topa_idx, base, old;
 
+	buf->wrapped = false;
+
 	if (buf->single) {
 		local_set(&buf->data_size, buf->output_off);
 		return;
@@ -865,7 +893,7 @@ static void pt_update_head(struct pt *pt)
 	} else {
 		old = (local64_xchg(&buf->head, base) &
 		       ((buf->nr_pages << PAGE_SHIFT) - 1));
-		if (base < old)
+		if (base < old || (base == old && wrapped))
 			base += buf->nr_pages << PAGE_SHIFT;
 
 		local_add(base - old, &buf->data_size);
@@ -900,7 +928,7 @@ static void pt_handle_status(struct pt *pt)
 	int advance = 0;
 	u64 status;
 
-	rdmsrl(MSR_IA32_RTIT_STATUS, status);
+	rdmsrq(MSR_IA32_RTIT_STATUS, status);
 
 	if (status & RTIT_STATUS_ERROR) {
 		pr_err_ratelimited("ToPA ERROR encountered, trying to recover\n");
@@ -944,7 +972,7 @@ static void pt_handle_status(struct pt *pt)
 	if (advance)
 		pt_buffer_advance(buf);
 
-	wrmsrl(MSR_IA32_RTIT_STATUS, status);
+	wrmsrq(MSR_IA32_RTIT_STATUS, status);
 }
 
 /**
@@ -959,12 +987,12 @@ static void pt_read_offset(struct pt_buffer *buf)
 	struct topa_page *tp;
 
 	if (!buf->single) {
-		rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, pt->output_base);
+		rdmsrq(MSR_IA32_RTIT_OUTPUT_BASE, pt->output_base);
 		tp = phys_to_virt(pt->output_base);
 		buf->cur = &tp->topa;
 	}
 
-	rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, pt->output_mask);
+	rdmsrq(MSR_IA32_RTIT_OUTPUT_MASK, pt->output_mask);
 	/* offset within current output region */
 	buf->output_off = pt->output_mask >> 32;
 	/* index of current output region within this table */
@@ -1511,6 +1539,7 @@ void intel_pt_interrupt(void)
 		buf = perf_aux_output_begin(&pt->handle, event);
 		if (!buf) {
 			event->hw.state = PERF_HES_STOPPED;
+			WRITE_ONCE(pt->resume_allowed, 0);
 			return;
 		}
 
@@ -1519,6 +1548,7 @@ void intel_pt_interrupt(void)
 		ret = pt_buffer_reset_markers(buf, &pt->handle);
 		if (ret) {
 			perf_aux_output_end(&pt->handle, 0);
+			WRITE_ONCE(pt->resume_allowed, 0);
 			return;
 		}
 
@@ -1557,11 +1587,11 @@ void intel_pt_handle_vmx(int on)
 
 	/* Turn PTs back on */
 	if (!on && event)
-		wrmsrl(MSR_IA32_RTIT_CTL, event->hw.aux_config);
+		wrmsrq(MSR_IA32_RTIT_CTL, event->hw.aux_config);
 
 	local_irq_restore(flags);
 }
-EXPORT_SYMBOL_GPL(intel_pt_handle_vmx);
+EXPORT_SYMBOL_FOR_KVM(intel_pt_handle_vmx);
 
 /*
  * PMU callbacks
@@ -1573,6 +1603,26 @@ static void pt_event_start(struct perf_event *event, int mode)
 	struct pt *pt = this_cpu_ptr(&pt_ctx);
 	struct pt_buffer *buf;
 
+	if (mode & PERF_EF_RESUME) {
+		if (READ_ONCE(pt->resume_allowed)) {
+			u64 status;
+
+			/*
+			 * Only if the trace is not active and the error and
+			 * stopped bits are clear, is it safe to start, but a
+			 * PMI might have just cleared these, so resume_allowed
+			 * must be checked again also.
+			 */
+			rdmsrq(MSR_IA32_RTIT_STATUS, status);
+			if (!(status & (RTIT_STATUS_TRIGGEREN |
+					RTIT_STATUS_ERROR |
+					RTIT_STATUS_STOPPED)) &&
+			   READ_ONCE(pt->resume_allowed))
+				pt_config_start(event);
+		}
+		return;
+	}
+
 	buf = perf_aux_output_begin(&pt->handle, event);
 	if (!buf)
 		goto fail_stop;
@@ -1583,7 +1633,6 @@ static void pt_event_start(struct perf_event *event, int mode)
 			goto fail_end_stop;
 	}
 
-	WRITE_ONCE(pt->handle_nmi, 1);
 	hwc->state = 0;
 
 	pt_config_buffer(buf);
@@ -1601,6 +1650,12 @@ static void pt_event_stop(struct perf_event *event, int mode)
 {
 	struct pt *pt = this_cpu_ptr(&pt_ctx);
 
+	if (mode & PERF_EF_PAUSE) {
+		if (READ_ONCE(pt->pause_allowed))
+			pt_config_stop(event);
+		return;
+	}
+
 	/*
 	 * Protect against the PMI racing with disabling wrmsr,
 	 * see comment in intel_pt_interrupt().
@@ -1608,6 +1663,15 @@ static void pt_event_stop(struct perf_event *event, int mode)
 	WRITE_ONCE(pt->handle_nmi, 0);
 	barrier();
 
+	/*
+	 * Prevent a resume from attempting to restart tracing, or a pause
+	 * during a subsequent start. Do this after clearing handle_nmi so that
+	 * pt_event_snapshot_aux() will not re-allow them.
+	 */
+	WRITE_ONCE(pt->pause_allowed, 0);
+	WRITE_ONCE(pt->resume_allowed, 0);
+	barrier();
+
 	pt_config_stop(event);
 
 	if (event->hw.state == PERF_HES_STOPPED)
@@ -1657,6 +1721,10 @@ static long pt_event_snapshot_aux(struct perf_event *event,
 	if (WARN_ON_ONCE(!buf->snapshot))
 		return 0;
 
+	/* Prevent pause/resume from attempting to start/stop tracing */
+	WRITE_ONCE(pt->pause_allowed, 0);
+	WRITE_ONCE(pt->resume_allowed, 0);
+	barrier();
 	/*
 	 * There is no PT interrupt in this mode, so stop the trace and it will
 	 * remain stopped while the buffer is copied.
@@ -1676,8 +1744,13 @@ static long pt_event_snapshot_aux(struct perf_event *event,
 	 * Here, handle_nmi tells us if the tracing was on.
 	 * If the tracing was on, restart it.
 	 */
-	if (READ_ONCE(pt->handle_nmi))
+	if (READ_ONCE(pt->handle_nmi)) {
+		WRITE_ONCE(pt->resume_allowed, 1);
+		barrier();
 		pt_config_start(event);
+		barrier();
+		WRITE_ONCE(pt->pause_allowed, 1);
+	}
 
 	return ret;
 }
@@ -1768,7 +1841,7 @@ static __init int pt_init(void)
 	for_each_online_cpu(cpu) {
 		u64 ctl;
 
-		ret = rdmsrl_safe_on_cpu(cpu, MSR_IA32_RTIT_CTL, &ctl);
+		ret = rdmsrq_safe_on_cpu(cpu, MSR_IA32_RTIT_CTL, &ctl);
 		if (!ret && (ctl & RTIT_CTL_TRACEEN))
 			prior_warn++;
 	}
@@ -1792,8 +1865,12 @@ static __init int pt_init(void)
 
 	if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries))
 		pt_pmu.pmu.capabilities = PERF_PMU_CAP_AUX_NO_SG;
+	else
+		pt_pmu.pmu.capabilities = PERF_PMU_CAP_AUX_PREFER_LARGE;
 
-	pt_pmu.pmu.capabilities	|= PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE;
+	pt_pmu.pmu.capabilities		|= PERF_PMU_CAP_EXCLUSIVE |
+					   PERF_PMU_CAP_ITRACE |
+					   PERF_PMU_CAP_AUX_PAUSE;
 	pt_pmu.pmu.attr_groups		 = pt_attr_groups;
 	pt_pmu.pmu.task_ctx_nr		 = perf_sw_context;
 	pt_pmu.pmu.event_init		 = pt_event_init;
diff --git a/arch/x86/events/intel/pt.h b/arch/x86/events/intel/pt.h
index f5e46c04c145..2ac36250b656 100644
--- a/arch/x86/events/intel/pt.h
+++ b/arch/x86/events/intel/pt.h
@@ -37,9 +37,6 @@ struct topa_entry {
 	u64	rsvd4	: 12;
 };
 
-/* TSC to Core Crystal Clock Ratio */
-#define CPUID_TSC_LEAF		0x15
-
 struct pt_pmu {
 	struct pmu		pmu;
 	u32			caps[PT_CPUID_REGS_NUM * PT_CPUID_LEAVES];
@@ -65,6 +62,7 @@ struct pt_pmu {
  * @head:	logical write offset inside the buffer
  * @snapshot:	if this is for a snapshot/overwrite counter
  * @single:	use Single Range Output instead of ToPA
+ * @wrapped:	buffer advance wrapped back to the first topa table
  * @stop_pos:	STOP topa entry index
  * @intr_pos:	INT topa entry index
  * @stop_te:	STOP topa entry pointer
@@ -82,6 +80,7 @@ struct pt_buffer {
 	local64_t		head;
 	bool			snapshot;
 	bool			single;
+	bool			wrapped;
 	long			stop_pos, intr_pos;
 	struct topa_entry	*stop_te, *intr_te;
 	void			**data_pages;
@@ -117,6 +116,8 @@ struct pt_filters {
  * @filters:		last configured filters
  * @handle_nmi:		do handle PT PMI on this cpu, there's an active event
  * @vmx_on:		1 if VMX is ON on this cpu
+ * @pause_allowed:	PERF_EF_PAUSE is allowed to stop tracing
+ * @resume_allowed:	PERF_EF_RESUME is allowed to start tracing
  * @output_base:	cached RTIT_OUTPUT_BASE MSR value
  * @output_mask:	cached RTIT_OUTPUT_MASK MSR value
  */
@@ -125,6 +126,8 @@ struct pt {
 	struct pt_filters	filters;
 	int			handle_nmi;
 	int			vmx_on;
+	int			pause_allowed;
+	int			resume_allowed;
 	u64			output_base;
 	u64			output_mask;
 };
diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c
index d98fac567684..e228e564b15e 100644
--- a/arch/x86/events/intel/uncore.c
+++ b/arch/x86/events/intel/uncore.c
@@ -3,6 +3,7 @@
 
 #include <asm/cpu_device_id.h>
 #include <asm/intel-family.h>
+#include <asm/msr.h>
 #include "uncore.h"
 #include "uncore_discovery.h"
 
@@ -150,7 +151,7 @@ u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *eve
 {
 	u64 count;
 
-	rdmsrl(event->hw.event_base, count);
+	rdmsrq(event->hw.event_base, count);
 
 	return count;
 }
@@ -305,17 +306,11 @@ static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
 {
 	struct intel_uncore_box *box;
 	struct perf_event *event;
-	unsigned long flags;
 	int bit;
 
 	box = container_of(hrtimer, struct intel_uncore_box, hrtimer);
 	if (!box->n_active || box->cpu != smp_processor_id())
 		return HRTIMER_NORESTART;
-	/*
-	 * disable local interrupt to prevent uncore_pmu_event_start/stop
-	 * to interrupt the update process
-	 */
-	local_irq_save(flags);
 
 	/*
 	 * handle boxes with an active event list as opposed to active
@@ -328,8 +323,6 @@ static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
 	for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX)
 		uncore_perf_event_update(box, box->events[bit]);
 
-	local_irq_restore(flags);
-
 	hrtimer_forward_now(hrtimer, ns_to_ktime(box->hrtimer_duration));
 	return HRTIMER_RESTART;
 }
@@ -337,7 +330,7 @@ static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
 void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
 {
 	hrtimer_start(&box->hrtimer, ns_to_ktime(box->hrtimer_duration),
-		      HRTIMER_MODE_REL_PINNED);
+		      HRTIMER_MODE_REL_PINNED_HARD);
 }
 
 void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
@@ -347,8 +340,7 @@ void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
 
 static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
 {
-	hrtimer_init(&box->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
-	box->hrtimer.function = uncore_pmu_hrtimer;
+	hrtimer_setup(&box->hrtimer, uncore_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
 }
 
 static struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type,
@@ -745,7 +737,7 @@ static int uncore_pmu_event_init(struct perf_event *event)
 
 	pmu = uncore_event_to_pmu(event);
 	/* no device found for this pmu */
-	if (pmu->func_id < 0)
+	if (!pmu->registered)
 		return -ENOENT;
 
 	/* Sampling not supported yet */
@@ -992,7 +984,7 @@ static void uncore_types_exit(struct intel_uncore_type **types)
 		uncore_type_exit(*types);
 }
 
-static int __init uncore_type_init(struct intel_uncore_type *type, bool setid)
+static int __init uncore_type_init(struct intel_uncore_type *type)
 {
 	struct intel_uncore_pmu *pmus;
 	size_t size;
@@ -1005,7 +997,6 @@ static int __init uncore_type_init(struct intel_uncore_type *type, bool setid)
 	size = uncore_max_dies() * sizeof(struct intel_uncore_box *);
 
 	for (i = 0; i < type->num_boxes; i++) {
-		pmus[i].func_id	= setid ? i : -1;
 		pmus[i].pmu_idx	= i;
 		pmus[i].type	= type;
 		pmus[i].boxes	= kzalloc(size, GFP_KERNEL);
@@ -1055,12 +1046,12 @@ err:
 }
 
 static int __init
-uncore_types_init(struct intel_uncore_type **types, bool setid)
+uncore_types_init(struct intel_uncore_type **types)
 {
 	int ret;
 
 	for (; *types; types++) {
-		ret = uncore_type_init(*types, setid);
+		ret = uncore_type_init(*types);
 		if (ret)
 			return ret;
 	}
@@ -1160,11 +1151,6 @@ static int uncore_pci_pmu_register(struct pci_dev *pdev,
 	if (!box)
 		return -ENOMEM;
 
-	if (pmu->func_id < 0)
-		pmu->func_id = pdev->devfn;
-	else
-		WARN_ON_ONCE(pmu->func_id != pdev->devfn);
-
 	atomic_inc(&box->refcnt);
 	box->dieid = die;
 	box->pci_dev = pdev;
@@ -1339,8 +1325,6 @@ static void uncore_pci_sub_driver_init(void)
 				continue;
 
 			pmu = &type->pmus[UNCORE_PCI_DEV_IDX(ids->driver_data)];
-			if (!pmu)
-				continue;
 
 			if (uncore_pci_get_dev_die_info(pci_sub_dev, &die))
 				continue;
@@ -1410,7 +1394,7 @@ static int __init uncore_pci_init(void)
 		goto err;
 	}
 
-	ret = uncore_types_init(uncore_pci_uncores, false);
+	ret = uncore_types_init(uncore_pci_uncores);
 	if (ret)
 		goto errtype;
 
@@ -1678,7 +1662,7 @@ static int __init uncore_cpu_init(void)
 {
 	int ret;
 
-	ret = uncore_types_init(uncore_msr_uncores, true);
+	ret = uncore_types_init(uncore_msr_uncores);
 	if (ret)
 		goto err;
 
@@ -1697,7 +1681,7 @@ static int __init uncore_mmio_init(void)
 	struct intel_uncore_type **types = uncore_mmio_uncores;
 	int ret;
 
-	ret = uncore_types_init(types, true);
+	ret = uncore_types_init(types);
 	if (ret)
 		goto err;
 
@@ -1821,6 +1805,12 @@ static const struct intel_uncore_init_fun lnl_uncore_init __initconst = {
 	.mmio_init = lnl_uncore_mmio_init,
 };
 
+static const struct intel_uncore_init_fun ptl_uncore_init __initconst = {
+	.cpu_init = ptl_uncore_cpu_init,
+	.mmio_init = ptl_uncore_mmio_init,
+	.use_discovery = true,
+};
+
 static const struct intel_uncore_init_fun icx_uncore_init __initconst = {
 	.cpu_init = icx_uncore_cpu_init,
 	.pci_init = icx_uncore_pci_init,
@@ -1902,6 +1892,8 @@ static const struct x86_cpu_id intel_uncore_match[] __initconst = {
 	X86_MATCH_VFM(INTEL_ARROWLAKE_U,	&mtl_uncore_init),
 	X86_MATCH_VFM(INTEL_ARROWLAKE_H,	&mtl_uncore_init),
 	X86_MATCH_VFM(INTEL_LUNARLAKE_M,	&lnl_uncore_init),
+	X86_MATCH_VFM(INTEL_PANTHERLAKE_L,	&ptl_uncore_init),
+	X86_MATCH_VFM(INTEL_WILDCATLAKE_L,	&ptl_uncore_init),
 	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X,	&spr_uncore_init),
 	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X,	&spr_uncore_init),
 	X86_MATCH_VFM(INTEL_GRANITERAPIDS_X,	&gnr_uncore_init),
@@ -1910,6 +1902,7 @@ static const struct x86_cpu_id intel_uncore_match[] __initconst = {
 	X86_MATCH_VFM(INTEL_ATOM_GRACEMONT,	&adl_uncore_init),
 	X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X,	&gnr_uncore_init),
 	X86_MATCH_VFM(INTEL_ATOM_CRESTMONT,	&gnr_uncore_init),
+	X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X,	&gnr_uncore_init),
 	{},
 };
 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_match);
diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h
index 79ff32e13dcc..d8815fff7588 100644
--- a/arch/x86/events/intel/uncore.h
+++ b/arch/x86/events/intel/uncore.h
@@ -125,7 +125,6 @@ struct intel_uncore_pmu {
 	struct pmu			pmu;
 	char				name[UNCORE_PMU_NAME_LEN];
 	int				pmu_idx;
-	int				func_id;
 	bool				registered;
 	atomic_t			activeboxes;
 	cpumask_t			cpu_mask;
@@ -613,10 +612,12 @@ void tgl_uncore_cpu_init(void);
 void adl_uncore_cpu_init(void);
 void lnl_uncore_cpu_init(void);
 void mtl_uncore_cpu_init(void);
+void ptl_uncore_cpu_init(void);
 void tgl_uncore_mmio_init(void);
 void tgl_l_uncore_mmio_init(void);
 void adl_uncore_mmio_init(void);
 void lnl_uncore_mmio_init(void);
+void ptl_uncore_mmio_init(void);
 int snb_pci2phy_map_init(int devid);
 
 /* uncore_snbep.c */
diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c
index 571e44b49691..7d57ce706feb 100644
--- a/arch/x86/events/intel/uncore_discovery.c
+++ b/arch/x86/events/intel/uncore_discovery.c
@@ -5,6 +5,7 @@
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <asm/msr.h>
 #include "uncore.h"
 #include "uncore_discovery.h"
 
@@ -273,32 +274,15 @@ uncore_ignore_unit(struct uncore_unit_discovery *unit, int *ignore)
 	return false;
 }
 
-static int parse_discovery_table(struct pci_dev *dev, int die,
-				 u32 bar_offset, bool *parsed,
-				 int *ignore)
+static int __parse_discovery_table(resource_size_t addr, int die,
+				   bool *parsed, int *ignore)
 {
 	struct uncore_global_discovery global;
 	struct uncore_unit_discovery unit;
 	void __iomem *io_addr;
-	resource_size_t addr;
 	unsigned long size;
-	u32 val;
 	int i;
 
-	pci_read_config_dword(dev, bar_offset, &val);
-
-	if (val & ~PCI_BASE_ADDRESS_MEM_MASK & ~PCI_BASE_ADDRESS_MEM_TYPE_64)
-		return -EINVAL;
-
-	addr = (resource_size_t)(val & PCI_BASE_ADDRESS_MEM_MASK);
-#ifdef CONFIG_PHYS_ADDR_T_64BIT
-	if ((val & PCI_BASE_ADDRESS_MEM_TYPE_MASK) == PCI_BASE_ADDRESS_MEM_TYPE_64) {
-		u32 val2;
-
-		pci_read_config_dword(dev, bar_offset + 4, &val2);
-		addr |= ((resource_size_t)val2) << 32;
-	}
-#endif
 	size = UNCORE_DISCOVERY_GLOBAL_MAP_SIZE;
 	io_addr = ioremap(addr, size);
 	if (!io_addr)
@@ -341,7 +325,32 @@ static int parse_discovery_table(struct pci_dev *dev, int die,
 	return 0;
 }
 
-bool intel_uncore_has_discovery_tables(int *ignore)
+static int parse_discovery_table(struct pci_dev *dev, int die,
+				 u32 bar_offset, bool *parsed,
+				 int *ignore)
+{
+	resource_size_t addr;
+	u32 val;
+
+	pci_read_config_dword(dev, bar_offset, &val);
+
+	if (val & ~PCI_BASE_ADDRESS_MEM_MASK & ~PCI_BASE_ADDRESS_MEM_TYPE_64)
+		return -EINVAL;
+
+	addr = (resource_size_t)(val & PCI_BASE_ADDRESS_MEM_MASK);
+#ifdef CONFIG_PHYS_ADDR_T_64BIT
+	if ((val & PCI_BASE_ADDRESS_MEM_TYPE_MASK) == PCI_BASE_ADDRESS_MEM_TYPE_64) {
+		u32 val2;
+
+		pci_read_config_dword(dev, bar_offset + 4, &val2);
+		addr |= ((resource_size_t)val2) << 32;
+	}
+#endif
+
+	return __parse_discovery_table(addr, die, parsed, ignore);
+}
+
+static bool intel_uncore_has_discovery_tables_pci(int *ignore)
 {
 	u32 device, val, entry_id, bar_offset;
 	int die, dvsec = 0, ret = true;
@@ -390,6 +399,45 @@ err:
 	return ret;
 }
 
+static bool intel_uncore_has_discovery_tables_msr(int *ignore)
+{
+	unsigned long *die_mask;
+	bool parsed = false;
+	int cpu, die;
+	u64 base;
+
+	die_mask = kcalloc(BITS_TO_LONGS(uncore_max_dies()),
+			   sizeof(unsigned long), GFP_KERNEL);
+	if (!die_mask)
+		return false;
+
+	cpus_read_lock();
+	for_each_online_cpu(cpu) {
+		die = topology_logical_die_id(cpu);
+		if (__test_and_set_bit(die, die_mask))
+			continue;
+
+		if (rdmsrq_safe_on_cpu(cpu, UNCORE_DISCOVERY_MSR, &base))
+			continue;
+
+		if (!base)
+			continue;
+
+		__parse_discovery_table(base, die, &parsed, ignore);
+	}
+
+	cpus_read_unlock();
+
+	kfree(die_mask);
+	return parsed;
+}
+
+bool intel_uncore_has_discovery_tables(int *ignore)
+{
+	return intel_uncore_has_discovery_tables_msr(ignore) ||
+	       intel_uncore_has_discovery_tables_pci(ignore);
+}
+
 void intel_uncore_clear_discovery_tables(void)
 {
 	struct intel_uncore_discovery_type *type, *next;
@@ -441,17 +489,17 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box)
 
 void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box)
 {
-	wrmsrl(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_INT);
+	wrmsrq(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_INT);
 }
 
 void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box)
 {
-	wrmsrl(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ);
+	wrmsrq(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ);
 }
 
 void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box)
 {
-	wrmsrl(intel_generic_uncore_box_ctl(box), 0);
+	wrmsrq(intel_generic_uncore_box_ctl(box), 0);
 }
 
 static void intel_generic_uncore_msr_enable_event(struct intel_uncore_box *box,
@@ -459,7 +507,7 @@ static void intel_generic_uncore_msr_enable_event(struct intel_uncore_box *box,
 {
 	struct hw_perf_event *hwc = &event->hw;
 
-	wrmsrl(hwc->config_base, hwc->config);
+	wrmsrq(hwc->config_base, hwc->config);
 }
 
 static void intel_generic_uncore_msr_disable_event(struct intel_uncore_box *box,
@@ -467,7 +515,7 @@ static void intel_generic_uncore_msr_disable_event(struct intel_uncore_box *box,
 {
 	struct hw_perf_event *hwc = &event->hw;
 
-	wrmsrl(hwc->config_base, 0);
+	wrmsrq(hwc->config_base, 0);
 }
 
 static struct intel_uncore_ops generic_uncore_msr_ops = {
@@ -603,7 +651,7 @@ void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box)
 	}
 
 	addr = unit->addr;
-	box->io_addr = ioremap(addr, UNCORE_GENERIC_MMIO_SIZE);
+	box->io_addr = ioremap(addr, type->mmio_map_size);
 	if (!box->io_addr) {
 		pr_warn("Uncore type %d box %d: ioremap error for 0x%llx.\n",
 			type->type_id, unit->id, (unsigned long long)addr);
diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h
index 0e94aa7db8e7..dff75c98e22f 100644
--- a/arch/x86/events/intel/uncore_discovery.h
+++ b/arch/x86/events/intel/uncore_discovery.h
@@ -1,5 +1,8 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 
+/* Store the full address of the global discovery table */
+#define UNCORE_DISCOVERY_MSR			0x201e
+
 /* Generic device ID of a discovery table device */
 #define UNCORE_DISCOVERY_TABLE_DEVICE		0x09a7
 /* Capability ID for a discovery table device */
@@ -168,3 +171,7 @@ bool intel_generic_uncore_assign_hw_event(struct perf_event *event,
 					  struct intel_uncore_box *box);
 void uncore_find_add_unit(struct intel_uncore_discovery_unit *node,
 			  struct rb_root *root, u16 *num_units);
+struct intel_uncore_type **
+uncore_get_uncores(enum uncore_access_type type_id, int num_extra,
+		   struct intel_uncore_type **extra, int max_num_types,
+		   struct intel_uncore_type **uncores);
diff --git a/arch/x86/events/intel/uncore_nhmex.c b/arch/x86/events/intel/uncore_nhmex.c
index 466833478e81..8962e7cb21e3 100644
--- a/arch/x86/events/intel/uncore_nhmex.c
+++ b/arch/x86/events/intel/uncore_nhmex.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Nehalem-EX/Westmere-EX uncore support */
 #include <asm/cpu_device_id.h>
+#include <asm/msr.h>
 #include "uncore.h"
 
 /* NHM-EX event control */
@@ -200,12 +201,12 @@ DEFINE_UNCORE_FORMAT_ATTR(mask, mask, "config2:0-63");
 
 static void nhmex_uncore_msr_init_box(struct intel_uncore_box *box)
 {
-	wrmsrl(NHMEX_U_MSR_PMON_GLOBAL_CTL, NHMEX_U_PMON_GLOBAL_EN_ALL);
+	wrmsrq(NHMEX_U_MSR_PMON_GLOBAL_CTL, NHMEX_U_PMON_GLOBAL_EN_ALL);
 }
 
 static void nhmex_uncore_msr_exit_box(struct intel_uncore_box *box)
 {
-	wrmsrl(NHMEX_U_MSR_PMON_GLOBAL_CTL, 0);
+	wrmsrq(NHMEX_U_MSR_PMON_GLOBAL_CTL, 0);
 }
 
 static void nhmex_uncore_msr_disable_box(struct intel_uncore_box *box)
@@ -214,12 +215,12 @@ static void nhmex_uncore_msr_disable_box(struct intel_uncore_box *box)
 	u64 config;
 
 	if (msr) {
-		rdmsrl(msr, config);
+		rdmsrq(msr, config);
 		config &= ~((1ULL << uncore_num_counters(box)) - 1);
 		/* WBox has a fixed counter */
 		if (uncore_msr_fixed_ctl(box))
 			config &= ~NHMEX_W_PMON_GLOBAL_FIXED_EN;
-		wrmsrl(msr, config);
+		wrmsrq(msr, config);
 	}
 }
 
@@ -229,18 +230,18 @@ static void nhmex_uncore_msr_enable_box(struct intel_uncore_box *box)
 	u64 config;
 
 	if (msr) {
-		rdmsrl(msr, config);
+		rdmsrq(msr, config);
 		config |= (1ULL << uncore_num_counters(box)) - 1;
 		/* WBox has a fixed counter */
 		if (uncore_msr_fixed_ctl(box))
 			config |= NHMEX_W_PMON_GLOBAL_FIXED_EN;
-		wrmsrl(msr, config);
+		wrmsrq(msr, config);
 	}
 }
 
 static void nhmex_uncore_msr_disable_event(struct intel_uncore_box *box, struct perf_event *event)
 {
-	wrmsrl(event->hw.config_base, 0);
+	wrmsrq(event->hw.config_base, 0);
 }
 
 static void nhmex_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
@@ -248,11 +249,11 @@ static void nhmex_uncore_msr_enable_event(struct intel_uncore_box *box, struct p
 	struct hw_perf_event *hwc = &event->hw;
 
 	if (hwc->idx == UNCORE_PMC_IDX_FIXED)
-		wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0);
+		wrmsrq(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0);
 	else if (box->pmu->type->event_mask & NHMEX_PMON_CTL_EN_BIT0)
-		wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22);
+		wrmsrq(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22);
 	else
-		wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0);
+		wrmsrq(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0);
 }
 
 #define NHMEX_UNCORE_OPS_COMMON_INIT()				\
@@ -382,10 +383,10 @@ static void nhmex_bbox_msr_enable_event(struct intel_uncore_box *box, struct per
 	struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
 
 	if (reg1->idx != EXTRA_REG_NONE) {
-		wrmsrl(reg1->reg, reg1->config);
-		wrmsrl(reg1->reg + 1, reg2->config);
+		wrmsrq(reg1->reg, reg1->config);
+		wrmsrq(reg1->reg + 1, reg2->config);
 	}
-	wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 |
+	wrmsrq(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 |
 		(hwc->config & NHMEX_B_PMON_CTL_EV_SEL_MASK));
 }
 
@@ -467,12 +468,12 @@ static void nhmex_sbox_msr_enable_event(struct intel_uncore_box *box, struct per
 	struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
 
 	if (reg1->idx != EXTRA_REG_NONE) {
-		wrmsrl(reg1->reg, 0);
-		wrmsrl(reg1->reg + 1, reg1->config);
-		wrmsrl(reg1->reg + 2, reg2->config);
-		wrmsrl(reg1->reg, NHMEX_S_PMON_MM_CFG_EN);
+		wrmsrq(reg1->reg, 0);
+		wrmsrq(reg1->reg + 1, reg1->config);
+		wrmsrq(reg1->reg + 2, reg2->config);
+		wrmsrq(reg1->reg, NHMEX_S_PMON_MM_CFG_EN);
 	}
-	wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22);
+	wrmsrq(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22);
 }
 
 static struct attribute *nhmex_uncore_sbox_formats_attr[] = {
@@ -842,25 +843,25 @@ static void nhmex_mbox_msr_enable_event(struct intel_uncore_box *box, struct per
 
 	idx = __BITS_VALUE(reg1->idx, 0, 8);
 	if (idx != 0xff)
-		wrmsrl(__BITS_VALUE(reg1->reg, 0, 16),
+		wrmsrq(__BITS_VALUE(reg1->reg, 0, 16),
 			nhmex_mbox_shared_reg_config(box, idx));
 	idx = __BITS_VALUE(reg1->idx, 1, 8);
 	if (idx != 0xff)
-		wrmsrl(__BITS_VALUE(reg1->reg, 1, 16),
+		wrmsrq(__BITS_VALUE(reg1->reg, 1, 16),
 			nhmex_mbox_shared_reg_config(box, idx));
 
 	if (reg2->idx != EXTRA_REG_NONE) {
-		wrmsrl(reg2->reg, 0);
+		wrmsrq(reg2->reg, 0);
 		if (reg2->config != ~0ULL) {
-			wrmsrl(reg2->reg + 1,
+			wrmsrq(reg2->reg + 1,
 				reg2->config & NHMEX_M_PMON_ADDR_MATCH_MASK);
-			wrmsrl(reg2->reg + 2, NHMEX_M_PMON_ADDR_MASK_MASK &
+			wrmsrq(reg2->reg + 2, NHMEX_M_PMON_ADDR_MASK_MASK &
 				(reg2->config >> NHMEX_M_PMON_ADDR_MASK_SHIFT));
-			wrmsrl(reg2->reg, NHMEX_M_PMON_MM_CFG_EN);
+			wrmsrq(reg2->reg, NHMEX_M_PMON_MM_CFG_EN);
 		}
 	}
 
-	wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0);
+	wrmsrq(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0);
 }
 
 DEFINE_UNCORE_FORMAT_ATTR(count_mode,		count_mode,	"config:2-3");
@@ -1121,31 +1122,31 @@ static void nhmex_rbox_msr_enable_event(struct intel_uncore_box *box, struct per
 
 	switch (idx % 6) {
 	case 0:
-		wrmsrl(NHMEX_R_MSR_PORTN_IPERF_CFG0(port), reg1->config);
+		wrmsrq(NHMEX_R_MSR_PORTN_IPERF_CFG0(port), reg1->config);
 		break;
 	case 1:
-		wrmsrl(NHMEX_R_MSR_PORTN_IPERF_CFG1(port), reg1->config);
+		wrmsrq(NHMEX_R_MSR_PORTN_IPERF_CFG1(port), reg1->config);
 		break;
 	case 2:
 	case 3:
-		wrmsrl(NHMEX_R_MSR_PORTN_QLX_CFG(port),
+		wrmsrq(NHMEX_R_MSR_PORTN_QLX_CFG(port),
 			uncore_shared_reg_config(box, 2 + (idx / 6) * 5));
 		break;
 	case 4:
-		wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(port),
+		wrmsrq(NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(port),
 			hwc->config >> 32);
-		wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MATCH(port), reg1->config);
-		wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MASK(port), reg2->config);
+		wrmsrq(NHMEX_R_MSR_PORTN_XBR_SET1_MATCH(port), reg1->config);
+		wrmsrq(NHMEX_R_MSR_PORTN_XBR_SET1_MASK(port), reg2->config);
 		break;
 	case 5:
-		wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(port),
+		wrmsrq(NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(port),
 			hwc->config >> 32);
-		wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MATCH(port), reg1->config);
-		wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MASK(port), reg2->config);
+		wrmsrq(NHMEX_R_MSR_PORTN_XBR_SET2_MATCH(port), reg1->config);
+		wrmsrq(NHMEX_R_MSR_PORTN_XBR_SET2_MASK(port), reg2->config);
 		break;
 	}
 
-	wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 |
+	wrmsrq(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 |
 		(hwc->config & NHMEX_R_PMON_CTL_EV_SEL_MASK));
 }
 
diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c
index 3934e1e4e3b1..807e582b8f17 100644
--- a/arch/x86/events/intel/uncore_snb.c
+++ b/arch/x86/events/intel/uncore_snb.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Nehalem/SandBridge/Haswell/Broadwell/Skylake uncore support */
+#include <asm/msr.h>
 #include "uncore.h"
 #include "uncore_discovery.h"
 
@@ -260,34 +261,34 @@ static void snb_uncore_msr_enable_event(struct intel_uncore_box *box, struct per
 	struct hw_perf_event *hwc = &event->hw;
 
 	if (hwc->idx < UNCORE_PMC_IDX_FIXED)
-		wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
+		wrmsrq(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
 	else
-		wrmsrl(hwc->config_base, SNB_UNC_CTL_EN);
+		wrmsrq(hwc->config_base, SNB_UNC_CTL_EN);
 }
 
 static void snb_uncore_msr_disable_event(struct intel_uncore_box *box, struct perf_event *event)
 {
-	wrmsrl(event->hw.config_base, 0);
+	wrmsrq(event->hw.config_base, 0);
 }
 
 static void snb_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0) {
-		wrmsrl(SNB_UNC_PERF_GLOBAL_CTL,
+		wrmsrq(SNB_UNC_PERF_GLOBAL_CTL,
 			SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL);
 	}
 }
 
 static void snb_uncore_msr_enable_box(struct intel_uncore_box *box)
 {
-	wrmsrl(SNB_UNC_PERF_GLOBAL_CTL,
+	wrmsrq(SNB_UNC_PERF_GLOBAL_CTL,
 		SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL);
 }
 
 static void snb_uncore_msr_exit_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0)
-		wrmsrl(SNB_UNC_PERF_GLOBAL_CTL, 0);
+		wrmsrq(SNB_UNC_PERF_GLOBAL_CTL, 0);
 }
 
 static struct uncore_event_desc snb_uncore_events[] = {
@@ -372,7 +373,7 @@ void snb_uncore_cpu_init(void)
 static void skl_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0) {
-		wrmsrl(SKL_UNC_PERF_GLOBAL_CTL,
+		wrmsrq(SKL_UNC_PERF_GLOBAL_CTL,
 			SNB_UNC_GLOBAL_CTL_EN | SKL_UNC_GLOBAL_CTL_CORE_ALL);
 	}
 
@@ -383,14 +384,14 @@ static void skl_uncore_msr_init_box(struct intel_uncore_box *box)
 
 static void skl_uncore_msr_enable_box(struct intel_uncore_box *box)
 {
-	wrmsrl(SKL_UNC_PERF_GLOBAL_CTL,
+	wrmsrq(SKL_UNC_PERF_GLOBAL_CTL,
 		SNB_UNC_GLOBAL_CTL_EN | SKL_UNC_GLOBAL_CTL_CORE_ALL);
 }
 
 static void skl_uncore_msr_exit_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0)
-		wrmsrl(SKL_UNC_PERF_GLOBAL_CTL, 0);
+		wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, 0);
 }
 
 static struct intel_uncore_ops skl_uncore_msr_ops = {
@@ -504,7 +505,7 @@ static int icl_get_cbox_num(void)
 {
 	u64 num_boxes;
 
-	rdmsrl(ICL_UNC_CBO_CONFIG, num_boxes);
+	rdmsrq(ICL_UNC_CBO_CONFIG, num_boxes);
 
 	return num_boxes & ICL_UNC_NUM_CBO_MASK;
 }
@@ -525,7 +526,7 @@ static struct intel_uncore_type *tgl_msr_uncores[] = {
 static void rkl_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0)
-		wrmsrl(SKL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN);
+		wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN);
 }
 
 void tgl_uncore_cpu_init(void)
@@ -541,24 +542,24 @@ void tgl_uncore_cpu_init(void)
 static void adl_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0)
-		wrmsrl(ADL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN);
+		wrmsrq(ADL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN);
 }
 
 static void adl_uncore_msr_enable_box(struct intel_uncore_box *box)
 {
-	wrmsrl(ADL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN);
+	wrmsrq(ADL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN);
 }
 
 static void adl_uncore_msr_disable_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0)
-		wrmsrl(ADL_UNC_PERF_GLOBAL_CTL, 0);
+		wrmsrq(ADL_UNC_PERF_GLOBAL_CTL, 0);
 }
 
 static void adl_uncore_msr_exit_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0)
-		wrmsrl(ADL_UNC_PERF_GLOBAL_CTL, 0);
+		wrmsrq(ADL_UNC_PERF_GLOBAL_CTL, 0);
 }
 
 static struct intel_uncore_ops adl_uncore_msr_ops = {
@@ -691,7 +692,7 @@ static struct intel_uncore_type mtl_uncore_hac_cbox = {
 
 static void mtl_uncore_msr_init_box(struct intel_uncore_box *box)
 {
-	wrmsrl(uncore_msr_box_ctl(box), SNB_UNC_GLOBAL_CTL_EN);
+	wrmsrq(uncore_msr_box_ctl(box), SNB_UNC_GLOBAL_CTL_EN);
 }
 
 static struct intel_uncore_ops mtl_uncore_msr_ops = {
@@ -758,7 +759,7 @@ static struct intel_uncore_type *lnl_msr_uncores[] = {
 static void lnl_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0)
-		wrmsrl(LNL_UNC_MSR_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN);
+		wrmsrq(LNL_UNC_MSR_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN);
 }
 
 static struct intel_uncore_ops lnl_uncore_msr_ops = {
@@ -910,7 +911,7 @@ static int snb_uncore_imc_event_init(struct perf_event *event)
 
 	pmu = uncore_event_to_pmu(event);
 	/* no device found for this pmu */
-	if (pmu->func_id < 0)
+	if (!pmu->registered)
 		return -ENOENT;
 
 	/* Sampling not supported yet */
@@ -1306,12 +1307,12 @@ int skl_uncore_pci_init(void)
 /* Nehalem uncore support */
 static void nhm_uncore_msr_disable_box(struct intel_uncore_box *box)
 {
-	wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, 0);
+	wrmsrq(NHM_UNC_PERF_GLOBAL_CTL, 0);
 }
 
 static void nhm_uncore_msr_enable_box(struct intel_uncore_box *box)
 {
-	wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, NHM_UNC_GLOBAL_CTL_EN_PC_ALL | NHM_UNC_GLOBAL_CTL_EN_FC);
+	wrmsrq(NHM_UNC_PERF_GLOBAL_CTL, NHM_UNC_GLOBAL_CTL_EN_PC_ALL | NHM_UNC_GLOBAL_CTL_EN_FC);
 }
 
 static void nhm_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
@@ -1319,9 +1320,9 @@ static void nhm_uncore_msr_enable_event(struct intel_uncore_box *box, struct per
 	struct hw_perf_event *hwc = &event->hw;
 
 	if (hwc->idx < UNCORE_PMC_IDX_FIXED)
-		wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
+		wrmsrq(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
 	else
-		wrmsrl(hwc->config_base, NHM_UNC_FIXED_CTR_CTL_EN);
+		wrmsrq(hwc->config_base, NHM_UNC_FIXED_CTR_CTL_EN);
 }
 
 static struct attribute *nhm_uncore_formats_attr[] = {
@@ -1854,3 +1855,82 @@ void lnl_uncore_mmio_init(void)
 }
 
 /* end of Lunar Lake MMIO uncore support */
+
+/* Panther Lake uncore support */
+
+#define UNCORE_PTL_MAX_NUM_UNCORE_TYPES		42
+#define UNCORE_PTL_TYPE_IMC			6
+#define UNCORE_PTL_TYPE_SNCU			34
+#define UNCORE_PTL_TYPE_HBO			41
+
+#define PTL_UNCORE_GLOBAL_CTL_OFFSET		0x380
+
+static struct intel_uncore_type ptl_uncore_imc = {
+	.name			= "imc",
+	.mmio_map_size		= 0xf00,
+};
+
+static void ptl_uncore_sncu_init_box(struct intel_uncore_box *box)
+{
+	intel_generic_uncore_mmio_init_box(box);
+
+	/* Clear the global freeze bit */
+	if (box->io_addr)
+		writel(0, box->io_addr + PTL_UNCORE_GLOBAL_CTL_OFFSET);
+}
+
+static struct intel_uncore_ops ptl_uncore_sncu_ops = {
+	.init_box		= ptl_uncore_sncu_init_box,
+	.exit_box		= uncore_mmio_exit_box,
+	.disable_box		= intel_generic_uncore_mmio_disable_box,
+	.enable_box		= intel_generic_uncore_mmio_enable_box,
+	.disable_event		= intel_generic_uncore_mmio_disable_event,
+	.enable_event		= intel_generic_uncore_mmio_enable_event,
+	.read_counter		= uncore_mmio_read_counter,
+};
+
+static struct intel_uncore_type ptl_uncore_sncu = {
+	.name			= "sncu",
+	.ops			= &ptl_uncore_sncu_ops,
+	.mmio_map_size		= 0xf00,
+};
+
+static struct intel_uncore_type ptl_uncore_hbo = {
+	.name			= "hbo",
+	.mmio_map_size		= 0xf00,
+};
+
+static struct intel_uncore_type *ptl_uncores[UNCORE_PTL_MAX_NUM_UNCORE_TYPES] = {
+	[UNCORE_PTL_TYPE_IMC] = &ptl_uncore_imc,
+	[UNCORE_PTL_TYPE_SNCU] = &ptl_uncore_sncu,
+	[UNCORE_PTL_TYPE_HBO] = &ptl_uncore_hbo,
+};
+
+#define UNCORE_PTL_MMIO_EXTRA_UNCORES		1
+
+static struct intel_uncore_type *ptl_mmio_extra_uncores[UNCORE_PTL_MMIO_EXTRA_UNCORES] = {
+	&adl_uncore_imc_free_running,
+};
+
+void ptl_uncore_mmio_init(void)
+{
+	uncore_mmio_uncores = uncore_get_uncores(UNCORE_ACCESS_MMIO,
+						 UNCORE_PTL_MMIO_EXTRA_UNCORES,
+						 ptl_mmio_extra_uncores,
+						 UNCORE_PTL_MAX_NUM_UNCORE_TYPES,
+						 ptl_uncores);
+}
+
+static struct intel_uncore_type *ptl_msr_uncores[] = {
+	&mtl_uncore_cbox,
+	NULL
+};
+
+void ptl_uncore_cpu_init(void)
+{
+	mtl_uncore_cbox.num_boxes = 6;
+	mtl_uncore_cbox.ops = &lnl_uncore_msr_ops;
+	uncore_msr_uncores = ptl_msr_uncores;
+}
+
+/* end of Panther Lake uncore support */
diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c
index ca98744343b8..e1f370b8d065 100644
--- a/arch/x86/events/intel/uncore_snbep.c
+++ b/arch/x86/events/intel/uncore_snbep.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /* SandyBridge-EP/IvyTown uncore support */
 #include <asm/cpu_device_id.h>
+#include <asm/msr.h>
 #include "uncore.h"
 #include "uncore_discovery.h"
 
@@ -618,9 +619,9 @@ static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box)
 
 	msr = uncore_msr_box_ctl(box);
 	if (msr) {
-		rdmsrl(msr, config);
+		rdmsrq(msr, config);
 		config |= SNBEP_PMON_BOX_CTL_FRZ;
-		wrmsrl(msr, config);
+		wrmsrq(msr, config);
 	}
 }
 
@@ -631,9 +632,9 @@ static void snbep_uncore_msr_enable_box(struct intel_uncore_box *box)
 
 	msr = uncore_msr_box_ctl(box);
 	if (msr) {
-		rdmsrl(msr, config);
+		rdmsrq(msr, config);
 		config &= ~SNBEP_PMON_BOX_CTL_FRZ;
-		wrmsrl(msr, config);
+		wrmsrq(msr, config);
 	}
 }
 
@@ -643,9 +644,9 @@ static void snbep_uncore_msr_enable_event(struct intel_uncore_box *box, struct p
 	struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 
 	if (reg1->idx != EXTRA_REG_NONE)
-		wrmsrl(reg1->reg, uncore_shared_reg_config(box, 0));
+		wrmsrq(reg1->reg, uncore_shared_reg_config(box, 0));
 
-	wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
+	wrmsrq(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 }
 
 static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box,
@@ -653,7 +654,7 @@ static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box,
 {
 	struct hw_perf_event *hwc = &event->hw;
 
-	wrmsrl(hwc->config_base, hwc->config);
+	wrmsrq(hwc->config_base, hwc->config);
 }
 
 static void snbep_uncore_msr_init_box(struct intel_uncore_box *box)
@@ -661,7 +662,7 @@ static void snbep_uncore_msr_init_box(struct intel_uncore_box *box)
 	unsigned msr = uncore_msr_box_ctl(box);
 
 	if (msr)
-		wrmsrl(msr, SNBEP_PMON_BOX_CTL_INT);
+		wrmsrq(msr, SNBEP_PMON_BOX_CTL_INT);
 }
 
 static struct attribute *snbep_uncore_formats_attr[] = {
@@ -1532,7 +1533,7 @@ static void ivbep_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	unsigned msr = uncore_msr_box_ctl(box);
 	if (msr)
-		wrmsrl(msr, IVBEP_PMON_BOX_CTL_INT);
+		wrmsrq(msr, IVBEP_PMON_BOX_CTL_INT);
 }
 
 static void ivbep_uncore_pci_init_box(struct intel_uncore_box *box)
@@ -1783,11 +1784,11 @@ static void ivbep_cbox_enable_event(struct intel_uncore_box *box, struct perf_ev
 
 	if (reg1->idx != EXTRA_REG_NONE) {
 		u64 filter = uncore_shared_reg_config(box, 0);
-		wrmsrl(reg1->reg, filter & 0xffffffff);
-		wrmsrl(reg1->reg + 6, filter >> 32);
+		wrmsrq(reg1->reg, filter & 0xffffffff);
+		wrmsrq(reg1->reg + 6, filter >> 32);
 	}
 
-	wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
+	wrmsrq(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 }
 
 static struct intel_uncore_ops ivbep_uncore_cbox_ops = {
@@ -2767,11 +2768,11 @@ static void hswep_cbox_enable_event(struct intel_uncore_box *box,
 
 	if (reg1->idx != EXTRA_REG_NONE) {
 		u64 filter = uncore_shared_reg_config(box, 0);
-		wrmsrl(reg1->reg, filter & 0xffffffff);
-		wrmsrl(reg1->reg + 1, filter >> 32);
+		wrmsrq(reg1->reg, filter & 0xffffffff);
+		wrmsrq(reg1->reg + 1, filter >> 32);
 	}
 
-	wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
+	wrmsrq(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 }
 
 static struct intel_uncore_ops hswep_uncore_cbox_ops = {
@@ -2816,7 +2817,7 @@ static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box)
 
 		for_each_set_bit(i, (unsigned long *)&init, 64) {
 			flags |= (1ULL << i);
-			wrmsrl(msr, flags);
+			wrmsrq(msr, flags);
 		}
 	}
 }
@@ -3708,7 +3709,7 @@ static void skx_iio_enable_event(struct intel_uncore_box *box,
 {
 	struct hw_perf_event *hwc = &event->hw;
 
-	wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
+	wrmsrq(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 }
 
 static struct intel_uncore_ops skx_uncore_iio_ops = {
@@ -3765,7 +3766,7 @@ static int skx_msr_cpu_bus_read(int cpu, u64 *topology)
 {
 	u64 msr_value;
 
-	if (rdmsrl_on_cpu(cpu, SKX_MSR_CPU_BUS_NUMBER, &msr_value) ||
+	if (rdmsrq_on_cpu(cpu, SKX_MSR_CPU_BUS_NUMBER, &msr_value) ||
 			!(msr_value & SKX_MSR_CPU_BUS_VALID_BIT))
 		return -ENXIO;
 
@@ -4655,9 +4656,9 @@ static void snr_cha_enable_event(struct intel_uncore_box *box,
 	struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 
 	if (reg1->idx != EXTRA_REG_NONE)
-		wrmsrl(reg1->reg, reg1->config);
+		wrmsrq(reg1->reg, reg1->config);
 
-	wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
+	wrmsrq(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 }
 
 static struct intel_uncore_ops snr_uncore_chabox_ops = {
@@ -4891,28 +4892,28 @@ static struct uncore_event_desc snr_uncore_iio_freerunning_events[] = {
 	INTEL_UNCORE_EVENT_DESC(ioclk,			"event=0xff,umask=0x10"),
 	/* Free-Running IIO BANDWIDTH IN Counters */
 	INTEL_UNCORE_EVENT_DESC(bw_in_port0,		"event=0xff,umask=0x20"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port0.scale,	"3.814697266e-6"),
+	INTEL_UNCORE_EVENT_DESC(bw_in_port0.scale,	"3.0517578125e-5"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port0.unit,	"MiB"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port1,		"event=0xff,umask=0x21"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port1.scale,	"3.814697266e-6"),
+	INTEL_UNCORE_EVENT_DESC(bw_in_port1.scale,	"3.0517578125e-5"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port1.unit,	"MiB"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port2,		"event=0xff,umask=0x22"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port2.scale,	"3.814697266e-6"),
+	INTEL_UNCORE_EVENT_DESC(bw_in_port2.scale,	"3.0517578125e-5"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port2.unit,	"MiB"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port3,		"event=0xff,umask=0x23"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port3.scale,	"3.814697266e-6"),
+	INTEL_UNCORE_EVENT_DESC(bw_in_port3.scale,	"3.0517578125e-5"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port3.unit,	"MiB"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port4,		"event=0xff,umask=0x24"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port4.scale,	"3.814697266e-6"),
+	INTEL_UNCORE_EVENT_DESC(bw_in_port4.scale,	"3.0517578125e-5"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port4.unit,	"MiB"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port5,		"event=0xff,umask=0x25"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port5.scale,	"3.814697266e-6"),
+	INTEL_UNCORE_EVENT_DESC(bw_in_port5.scale,	"3.0517578125e-5"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port5.unit,	"MiB"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port6,		"event=0xff,umask=0x26"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port6.scale,	"3.814697266e-6"),
+	INTEL_UNCORE_EVENT_DESC(bw_in_port6.scale,	"3.0517578125e-5"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port6.unit,	"MiB"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port7,		"event=0xff,umask=0x27"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port7.scale,	"3.814697266e-6"),
+	INTEL_UNCORE_EVENT_DESC(bw_in_port7.scale,	"3.0517578125e-5"),
 	INTEL_UNCORE_EVENT_DESC(bw_in_port7.unit,	"MiB"),
 	{ /* end: all zeroes */ },
 };
@@ -5485,37 +5486,6 @@ static struct freerunning_counters icx_iio_freerunning[] = {
 	[ICX_IIO_MSR_BW_IN]	= { 0xaa0, 0x1, 0x10, 8, 48, icx_iio_bw_freerunning_box_offsets },
 };
 
-static struct uncore_event_desc icx_uncore_iio_freerunning_events[] = {
-	/* Free-Running IIO CLOCKS Counter */
-	INTEL_UNCORE_EVENT_DESC(ioclk,			"event=0xff,umask=0x10"),
-	/* Free-Running IIO BANDWIDTH IN Counters */
-	INTEL_UNCORE_EVENT_DESC(bw_in_port0,		"event=0xff,umask=0x20"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port0.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port0.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port1,		"event=0xff,umask=0x21"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port1.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port1.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port2,		"event=0xff,umask=0x22"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port2.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port2.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port3,		"event=0xff,umask=0x23"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port3.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port3.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port4,		"event=0xff,umask=0x24"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port4.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port4.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port5,		"event=0xff,umask=0x25"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port5.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port5.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port6,		"event=0xff,umask=0x26"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port6.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port6.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port7,		"event=0xff,umask=0x27"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port7.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port7.unit,	"MiB"),
-	{ /* end: all zeroes */ },
-};
-
 static struct intel_uncore_type icx_uncore_iio_free_running = {
 	.name			= "iio_free_running",
 	.num_counters		= 9,
@@ -5523,7 +5493,7 @@ static struct intel_uncore_type icx_uncore_iio_free_running = {
 	.num_freerunning_types	= ICX_IIO_FREERUNNING_TYPE_MAX,
 	.freerunning		= icx_iio_freerunning,
 	.ops			= &skx_uncore_iio_freerunning_ops,
-	.event_descs		= icx_uncore_iio_freerunning_events,
+	.event_descs		= snr_uncore_iio_freerunning_events,
 	.format_group		= &skx_uncore_iio_freerunning_format_group,
 };
 
@@ -5913,9 +5883,9 @@ static void spr_uncore_msr_enable_event(struct intel_uncore_box *box,
 	struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 
 	if (reg1->idx != EXTRA_REG_NONE)
-		wrmsrl(reg1->reg, reg1->config);
+		wrmsrq(reg1->reg, reg1->config);
 
-	wrmsrl(hwc->config_base, hwc->config);
+	wrmsrq(hwc->config_base, hwc->config);
 }
 
 static void spr_uncore_msr_disable_event(struct intel_uncore_box *box,
@@ -5925,9 +5895,9 @@ static void spr_uncore_msr_disable_event(struct intel_uncore_box *box,
 	struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 
 	if (reg1->idx != EXTRA_REG_NONE)
-		wrmsrl(reg1->reg, 0);
+		wrmsrq(reg1->reg, 0);
 
-	wrmsrl(hwc->config_base, 0);
+	wrmsrq(hwc->config_base, 0);
 }
 
 static int spr_cha_hw_config(struct intel_uncore_box *box, struct perf_event *event)
@@ -6320,69 +6290,13 @@ static struct freerunning_counters spr_iio_freerunning[] = {
 	[SPR_IIO_MSR_BW_OUT]	= { 0x3808, 0x1, 0x10, 8, 48 },
 };
 
-static struct uncore_event_desc spr_uncore_iio_freerunning_events[] = {
-	/* Free-Running IIO CLOCKS Counter */
-	INTEL_UNCORE_EVENT_DESC(ioclk,			"event=0xff,umask=0x10"),
-	/* Free-Running IIO BANDWIDTH IN Counters */
-	INTEL_UNCORE_EVENT_DESC(bw_in_port0,		"event=0xff,umask=0x20"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port0.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port0.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port1,		"event=0xff,umask=0x21"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port1.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port1.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port2,		"event=0xff,umask=0x22"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port2.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port2.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port3,		"event=0xff,umask=0x23"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port3.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port3.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port4,		"event=0xff,umask=0x24"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port4.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port4.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port5,		"event=0xff,umask=0x25"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port5.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port5.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port6,		"event=0xff,umask=0x26"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port6.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port6.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port7,		"event=0xff,umask=0x27"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port7.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_in_port7.unit,	"MiB"),
-	/* Free-Running IIO BANDWIDTH OUT Counters */
-	INTEL_UNCORE_EVENT_DESC(bw_out_port0,		"event=0xff,umask=0x30"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port0.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port0.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port1,		"event=0xff,umask=0x31"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port1.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port1.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port2,		"event=0xff,umask=0x32"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port2.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port2.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port3,		"event=0xff,umask=0x33"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port3.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port3.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port4,		"event=0xff,umask=0x34"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port4.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port4.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port5,		"event=0xff,umask=0x35"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port5.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port5.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port6,		"event=0xff,umask=0x36"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port6.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port6.unit,	"MiB"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port7,		"event=0xff,umask=0x37"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port7.scale,	"3.814697266e-6"),
-	INTEL_UNCORE_EVENT_DESC(bw_out_port7.unit,	"MiB"),
-	{ /* end: all zeroes */ },
-};
-
 static struct intel_uncore_type spr_uncore_iio_free_running = {
 	.name			= "iio_free_running",
 	.num_counters		= 17,
 	.num_freerunning_types	= SPR_IIO_FREERUNNING_TYPE_MAX,
 	.freerunning		= spr_iio_freerunning,
 	.ops			= &skx_uncore_iio_freerunning_ops,
-	.event_descs		= spr_uncore_iio_freerunning_events,
+	.event_descs		= snr_uncore_iio_freerunning_events,
 	.format_group		= &skx_uncore_iio_freerunning_format_group,
 };
 
@@ -6495,9 +6409,11 @@ static void uncore_type_customized_copy(struct intel_uncore_type *to_type,
 		to_type->get_topology = from_type->get_topology;
 	if (from_type->cleanup_mapping)
 		to_type->cleanup_mapping = from_type->cleanup_mapping;
+	if (from_type->mmio_map_size)
+		to_type->mmio_map_size = from_type->mmio_map_size;
 }
 
-static struct intel_uncore_type **
+struct intel_uncore_type **
 uncore_get_uncores(enum uncore_access_type type_id, int num_extra,
 		   struct intel_uncore_type **extra, int max_num_types,
 		   struct intel_uncore_type **uncores)
@@ -6572,7 +6488,7 @@ void spr_uncore_cpu_init(void)
 		 * of UNCORE_SPR_CHA) is incorrect on some SPR variants because of a
 		 * firmware bug. Using the value from SPR_MSR_UNC_CBO_CONFIG to replace it.
 		 */
-		rdmsrl(SPR_MSR_UNC_CBO_CONFIG, num_cbo);
+		rdmsrq(SPR_MSR_UNC_CBO_CONFIG, num_cbo);
 		/*
 		 * The MSR doesn't work on the EMR XCC, but the firmware bug doesn't impact
 		 * the EMR XCC. Don't let the value from the MSR replace the existing value.
@@ -6684,17 +6600,8 @@ void spr_uncore_mmio_init(void)
 /* GNR uncore support */
 
 #define UNCORE_GNR_NUM_UNCORE_TYPES	23
-#define UNCORE_GNR_TYPE_15		15
-#define UNCORE_GNR_B2UPI		18
-#define UNCORE_GNR_TYPE_21		21
-#define UNCORE_GNR_TYPE_22		22
 
 int gnr_uncore_units_ignore[] = {
-	UNCORE_SPR_UPI,
-	UNCORE_GNR_TYPE_15,
-	UNCORE_GNR_B2UPI,
-	UNCORE_GNR_TYPE_21,
-	UNCORE_GNR_TYPE_22,
 	UNCORE_IGNORE_END
 };
 
@@ -6703,6 +6610,31 @@ static struct intel_uncore_type gnr_uncore_ubox = {
 	.attr_update		= uncore_alias_groups,
 };
 
+static struct intel_uncore_type gnr_uncore_pciex8 = {
+	SPR_UNCORE_PCI_COMMON_FORMAT(),
+	.name			= "pciex8",
+};
+
+static struct intel_uncore_type gnr_uncore_pciex16 = {
+	SPR_UNCORE_PCI_COMMON_FORMAT(),
+	.name			= "pciex16",
+};
+
+static struct intel_uncore_type gnr_uncore_upi = {
+	SPR_UNCORE_PCI_COMMON_FORMAT(),
+	.name			= "upi",
+};
+
+static struct intel_uncore_type gnr_uncore_b2upi = {
+	SPR_UNCORE_PCI_COMMON_FORMAT(),
+	.name			= "b2upi",
+};
+
+static struct intel_uncore_type gnr_uncore_b2hot = {
+	.name			= "b2hot",
+	.attr_update		= uncore_alias_groups,
+};
+
 static struct intel_uncore_type gnr_uncore_b2cmi = {
 	SPR_UNCORE_PCI_COMMON_FORMAT(),
 	.name			= "b2cmi",
@@ -6727,21 +6659,21 @@ static struct intel_uncore_type *gnr_uncores[UNCORE_GNR_NUM_UNCORE_TYPES] = {
 	&gnr_uncore_ubox,
 	&spr_uncore_imc,
 	NULL,
+	&gnr_uncore_upi,
 	NULL,
 	NULL,
 	NULL,
+	&spr_uncore_cxlcm,
+	&spr_uncore_cxldp,
 	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
+	&gnr_uncore_b2hot,
 	&gnr_uncore_b2cmi,
 	&gnr_uncore_b2cxl,
-	NULL,
+	&gnr_uncore_b2upi,
 	NULL,
 	&gnr_uncore_mdf_sbo,
-	NULL,
-	NULL,
+	&gnr_uncore_pciex16,
+	&gnr_uncore_pciex8,
 };
 
 static struct freerunning_counters gnr_iio_freerunning[] = {
diff --git a/arch/x86/events/msr.c b/arch/x86/events/msr.c
index 45b1866ff051..7f5007a4752a 100644
--- a/arch/x86/events/msr.c
+++ b/arch/x86/events/msr.c
@@ -3,6 +3,8 @@
 #include <linux/sysfs.h>
 #include <linux/nospec.h>
 #include <asm/cpu_device_id.h>
+#include <asm/msr.h>
+
 #include "probe.h"
 
 enum perf_msr_id {
@@ -231,7 +233,7 @@ static inline u64 msr_read_counter(struct perf_event *event)
 	u64 now;
 
 	if (event->hw.event_base)
-		rdmsrl(event->hw.event_base, now);
+		rdmsrq(event->hw.event_base, now);
 	else
 		now = rdtsc_ordered();
 
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index ac1182141bf6..3161ec0a3416 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -17,6 +17,7 @@
 #include <asm/fpu/xstate.h>
 #include <asm/intel_ds.h>
 #include <asm/cpu.h>
+#include <asm/msr.h>
 
 /* To enable MSR tracing please use the generic trace points. */
 
@@ -110,9 +111,26 @@ static inline bool is_topdown_event(struct perf_event *event)
 	return is_metric_event(event) || is_slots_event(event);
 }
 
+int is_x86_event(struct perf_event *event);
+
+static inline bool check_leader_group(struct perf_event *leader, int flags)
+{
+	return is_x86_event(leader) ? !!(leader->hw.flags & flags) : false;
+}
+
 static inline bool is_branch_counters_group(struct perf_event *event)
 {
-	return event->group_leader->hw.flags & PERF_X86_EVENT_BRANCH_COUNTERS;
+	return check_leader_group(event->group_leader, PERF_X86_EVENT_BRANCH_COUNTERS);
+}
+
+static inline bool is_pebs_counter_event_group(struct perf_event *event)
+{
+	return check_leader_group(event->group_leader, PERF_X86_EVENT_PEBS_CNTR);
+}
+
+static inline bool is_acr_event_group(struct perf_event *event)
+{
+	return check_leader_group(event->group_leader, PERF_X86_EVENT_ACR);
 }
 
 struct amd_nb {
@@ -256,6 +274,7 @@ struct cpu_hw_events {
 	struct event_constraint	*event_constraint[X86_PMC_IDX_MAX];
 
 	int			n_excl; /* the number of exclusive events */
+	int			n_late_setup; /* the num of events needs late setup */
 
 	unsigned int		txn_flags;
 	int			is_fake;
@@ -264,8 +283,9 @@ struct cpu_hw_events {
 	 * Intel DebugStore bits
 	 */
 	struct debug_store	*ds;
-	void			*ds_pebs_vaddr;
 	void			*ds_bts_vaddr;
+	/* DS based PEBS or arch-PEBS buffer address */
+	void			*pebs_vaddr;
 	u64			pebs_enabled;
 	int			n_pebs;
 	int			n_large_pebs;
@@ -281,6 +301,12 @@ struct cpu_hw_events {
 	u64			fixed_ctrl_val;
 	u64			active_fixed_ctrl_val;
 
+	/* Intel ACR configuration */
+	u64			acr_cfg_b[X86_PMC_IDX_MAX];
+	u64			acr_cfg_c[X86_PMC_IDX_MAX];
+	/* Cached CFG_C values */
+	u64			cfg_c_val[X86_PMC_IDX_MAX];
+
 	/*
 	 * Intel LBR bits
 	 */
@@ -624,6 +650,7 @@ union perf_capabilities {
 		u64	pebs_output_pt_available:1;
 		u64	pebs_timing_info:1;
 		u64	anythread_deprecated:1;
+		u64	rdpmc_metrics_clear:1;
 	};
 	u64	capabilities;
 };
@@ -668,24 +695,27 @@ enum {
 #define PERF_PEBS_DATA_SOURCE_GRT_MAX	0x10
 #define PERF_PEBS_DATA_SOURCE_GRT_MASK	(PERF_PEBS_DATA_SOURCE_GRT_MAX - 1)
 
-enum hybrid_cpu_type {
-	HYBRID_INTEL_NONE,
-	HYBRID_INTEL_ATOM	= 0x20,
-	HYBRID_INTEL_CORE	= 0x40,
-};
+#define X86_HYBRID_PMU_ATOM_IDX		0
+#define X86_HYBRID_PMU_CORE_IDX		1
+#define X86_HYBRID_PMU_TINY_IDX		2
 
 enum hybrid_pmu_type {
 	not_hybrid,
-	hybrid_small		= BIT(0),
-	hybrid_big		= BIT(1),
-
-	hybrid_big_small	= hybrid_big | hybrid_small, /* only used for matching */
+	hybrid_small		= BIT(X86_HYBRID_PMU_ATOM_IDX),
+	hybrid_big		= BIT(X86_HYBRID_PMU_CORE_IDX),
+	hybrid_tiny		= BIT(X86_HYBRID_PMU_TINY_IDX),
+
+	/* The belows are only used for matching */
+	hybrid_big_small	= hybrid_big   | hybrid_small,
+	hybrid_small_tiny	= hybrid_small | hybrid_tiny,
+	hybrid_big_small_tiny	= hybrid_big   | hybrid_small_tiny,
 };
 
-#define X86_HYBRID_PMU_ATOM_IDX		0
-#define X86_HYBRID_PMU_CORE_IDX		1
-
-#define X86_HYBRID_NUM_PMUS		2
+struct arch_pebs_cap {
+	u64 caps;
+	u64 counters;
+	u64 pdists;
+};
 
 struct x86_hybrid_pmu {
 	struct pmu			pmu;
@@ -704,6 +734,15 @@ struct x86_hybrid_pmu {
 			u64		fixed_cntr_mask64;
 			unsigned long	fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 	};
+
+	union {
+			u64		acr_cntr_mask64;
+			unsigned long	acr_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+	};
+	union {
+			u64		acr_cause_mask64;
+			unsigned long	acr_cause_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+	};
 	struct event_constraint		unconstrained;
 
 	u64				hw_cache_event_ids
@@ -722,6 +761,8 @@ struct x86_hybrid_pmu {
 					mid_ack		:1,
 					enabled_ack	:1;
 
+	struct arch_pebs_cap		arch_pebs_cap;
+
 	u64				pebs_data_source[PERF_PEBS_DATA_SOURCE_MAX];
 };
 
@@ -785,6 +826,11 @@ struct x86_pmu {
 	u64		(*update)(struct perf_event *event);
 	int		(*hw_config)(struct perf_event *event);
 	int		(*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
+	void		(*late_setup)(void);
+	void		(*pebs_enable)(struct perf_event *event);
+	void		(*pebs_disable)(struct perf_event *event);
+	void		(*pebs_enable_all)(void);
+	void		(*pebs_disable_all)(void);
 	unsigned	eventsel;
 	unsigned	perfctr;
 	unsigned	fixedctr;
@@ -801,6 +847,14 @@ struct x86_pmu {
 			u64		fixed_cntr_mask64;
 			unsigned long	fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 	};
+	union {
+			u64		acr_cntr_mask64;
+			unsigned long	acr_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+	};
+	union {
+			u64		acr_cause_mask64;
+			unsigned long	acr_cause_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+	};
 	int		cntval_bits;
 	u64		cntval_mask;
 	union {
@@ -854,7 +908,7 @@ struct x86_pmu {
 
 	void		(*check_microcode)(void);
 	void		(*sched_task)(struct perf_event_pmu_context *pmu_ctx,
-				      bool sched_in);
+				      struct task_struct *task, bool sched_in);
 
 	/*
 	 * Intel Arch Perfmon v2+
@@ -863,18 +917,19 @@ struct x86_pmu {
 	union perf_capabilities intel_cap;
 
 	/*
-	 * Intel DebugStore bits
+	 * Intel DebugStore and PEBS bits
 	 */
 	unsigned int	bts			:1,
 			bts_active		:1,
-			pebs			:1,
+			ds_pebs			:1,
 			pebs_active		:1,
 			pebs_broken		:1,
 			pebs_prec_dist		:1,
 			pebs_no_tlb		:1,
 			pebs_no_isolation	:1,
 			pebs_block		:1,
-			pebs_ept		:1;
+			pebs_ept		:1,
+			arch_pebs		:1;
 	int		pebs_record_size;
 	int		pebs_buffer_size;
 	u64		pebs_events_mask;
@@ -887,6 +942,11 @@ struct x86_pmu {
 	u64		pebs_capable;
 
 	/*
+	 * Intel Architectural PEBS
+	 */
+	struct arch_pebs_cap arch_pebs_cap;
+
+	/*
 	 * Intel LBR
 	 */
 	unsigned int	lbr_tos, lbr_from, lbr_to,
@@ -899,6 +959,7 @@ struct x86_pmu {
 		const int	*lbr_sel_map;	   /* lbr_select mappings */
 		int		*lbr_ctl_map;	   /* LBR_CTL mappings */
 	};
+	u64		lbr_callstack_users;	   /* lbr callstack system wide users */
 	bool		lbr_double_abort;	   /* duplicated lbr aborts */
 	bool		lbr_pt_coexist;		   /* (LBR|BTS) may coexist with PT */
 
@@ -937,14 +998,6 @@ struct x86_pmu {
 	int		num_topdown_events;
 
 	/*
-	 * perf task context (i.e. struct perf_event_pmu_context::task_ctx_data)
-	 * switch helper to bridge calls from perf/core to perf/x86.
-	 * See struct pmu::swap_task_ctx() usage for examples;
-	 */
-	void		(*swap_task_ctx)(struct perf_event_pmu_context *prev_epc,
-					 struct perf_event_pmu_context *next_epc);
-
-	/*
 	 * AMD bits
 	 */
 	unsigned int	amd_nb_constraints : 1;
@@ -979,7 +1032,7 @@ struct x86_pmu {
 	 */
 	int				num_hybrid_pmus;
 	struct x86_hybrid_pmu		*hybrid_pmu;
-	enum hybrid_cpu_type (*get_hybrid_cpu_type)	(void);
+	enum intel_cpu_type (*get_hybrid_cpu_type)	(void);
 };
 
 struct x86_perf_task_context_opt {
@@ -1045,6 +1098,7 @@ do {									\
 #define PMU_FL_MEM_LOADS_AUX	0x100 /* Require an auxiliary event for the complete memory info */
 #define PMU_FL_RETIRE_LATENCY	0x200 /* Support Retire Latency in PEBS */
 #define PMU_FL_BR_CNTR		0x400 /* Support branch counter logging */
+#define PMU_FL_DYN_CONSTRAINT	0x800 /* Needs dynamic constraint */
 
 #define EVENT_VAR(_id)  event_attr_##_id
 #define EVENT_PTR(_id) &event_attr_##_id.attr.attr
@@ -1092,6 +1146,12 @@ extern struct x86_pmu x86_pmu __read_mostly;
 
 DECLARE_STATIC_CALL(x86_pmu_set_period, *x86_pmu.set_period);
 DECLARE_STATIC_CALL(x86_pmu_update,     *x86_pmu.update);
+DECLARE_STATIC_CALL(x86_pmu_drain_pebs,	*x86_pmu.drain_pebs);
+DECLARE_STATIC_CALL(x86_pmu_late_setup,	*x86_pmu.late_setup);
+DECLARE_STATIC_CALL(x86_pmu_pebs_enable, *x86_pmu.pebs_enable);
+DECLARE_STATIC_CALL(x86_pmu_pebs_disable, *x86_pmu.pebs_disable);
+DECLARE_STATIC_CALL(x86_pmu_pebs_enable_all, *x86_pmu.pebs_enable_all);
+DECLARE_STATIC_CALL(x86_pmu_pebs_disable_all, *x86_pmu.pebs_disable_all);
 
 static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx)
 {
@@ -1133,6 +1193,12 @@ extern u64 __read_mostly hw_cache_extra_regs
 
 u64 x86_perf_event_update(struct perf_event *event);
 
+static inline u64 intel_pmu_topdown_event_update(struct perf_event *event, u64 *val)
+{
+	return x86_perf_event_update(event);
+}
+DECLARE_STATIC_CALL(intel_pmu_update_topdown_event, intel_pmu_topdown_event_update);
+
 static inline unsigned int x86_pmu_config_addr(int index)
 {
 	return x86_pmu.eventsel + (x86_pmu.addr_offset ?
@@ -1167,7 +1233,7 @@ int x86_reserve_hardware(void);
 
 void x86_release_hardware(void);
 
-int x86_pmu_max_precise(void);
+int x86_pmu_max_precise(struct pmu *pmu);
 
 void hw_perf_lbr_event_destroy(struct perf_event *event);
 
@@ -1193,16 +1259,16 @@ static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
 	u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
 
 	if (hwc->extra_reg.reg)
-		wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
+		wrmsrq(hwc->extra_reg.reg, hwc->extra_reg.config);
 
 	/*
 	 * Add enabled Merge event on next counter
 	 * if large increment event being enabled on this counter
 	 */
 	if (is_counter_pair(hwc))
-		wrmsrl(x86_pmu_config_addr(hwc->idx + 1), x86_pmu.perf_ctr_pair_en);
+		wrmsrq(x86_pmu_config_addr(hwc->idx + 1), x86_pmu.perf_ctr_pair_en);
 
-	wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
+	wrmsrq(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
 }
 
 void x86_pmu_enable_all(int added);
@@ -1218,10 +1284,10 @@ static inline void x86_pmu_disable_event(struct perf_event *event)
 	u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
 	struct hw_perf_event *hwc = &event->hw;
 
-	wrmsrl(hwc->config_base, hwc->config & ~disable_mask);
+	wrmsrq(hwc->config_base, hwc->config & ~disable_mask);
 
 	if (is_counter_pair(hwc))
-		wrmsrl(x86_pmu_config_addr(hwc->idx + 1), 0);
+		wrmsrq(x86_pmu_config_addr(hwc->idx + 1), 0);
 }
 
 void x86_pmu_enable_event(struct perf_event *event);
@@ -1379,7 +1445,8 @@ void amd_pmu_lbr_reset(void);
 void amd_pmu_lbr_read(void);
 void amd_pmu_lbr_add(struct perf_event *event);
 void amd_pmu_lbr_del(struct perf_event *event);
-void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
+void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
+			    struct task_struct *task, bool sched_in);
 void amd_pmu_lbr_enable_all(void);
 void amd_pmu_lbr_disable_all(void);
 int amd_pmu_lbr_hw_config(struct perf_event *event);
@@ -1388,12 +1455,12 @@ static __always_inline void __amd_pmu_lbr_disable(void)
 {
 	u64 dbg_ctl, dbg_extn_cfg;
 
-	rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
-	wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
+	rdmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
+	wrmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
 
 	if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) {
-		rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
-		wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
+		rdmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
+		wrmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
 	}
 }
 
@@ -1433,7 +1500,8 @@ static inline void amd_pmu_brs_del(struct perf_event *event)
 	perf_sched_cb_dec(event->pmu);
 }
 
-void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
+void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx,
+			    struct task_struct *task, bool sched_in);
 #else
 static inline int amd_brs_init(void)
 {
@@ -1458,7 +1526,8 @@ static inline void amd_pmu_brs_del(struct perf_event *event)
 {
 }
 
-static inline void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
+static inline void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx,
+					  struct task_struct *task, bool sched_in)
 {
 }
 
@@ -1523,21 +1592,21 @@ static inline bool intel_pmu_has_bts(struct perf_event *event)
 
 static __always_inline void __intel_pmu_pebs_disable_all(void)
 {
-	wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
+	wrmsrq(MSR_IA32_PEBS_ENABLE, 0);
 }
 
 static __always_inline void __intel_pmu_arch_lbr_disable(void)
 {
-	wrmsrl(MSR_ARCH_LBR_CTL, 0);
+	wrmsrq(MSR_ARCH_LBR_CTL, 0);
 }
 
 static __always_inline void __intel_pmu_lbr_disable(void)
 {
 	u64 debugctl;
 
-	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
+	rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl);
 	debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
-	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
+	wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl);
 }
 
 int intel_pmu_save_and_restart(struct perf_event *event);
@@ -1551,6 +1620,14 @@ extern void intel_cpuc_finish(struct cpu_hw_events *cpuc);
 
 int intel_pmu_init(void);
 
+int alloc_arch_pebs_buf_on_cpu(int cpu);
+
+void release_arch_pebs_buf_on_cpu(int cpu);
+
+void init_arch_pebs_on_cpu(int cpu);
+
+void fini_arch_pebs_on_cpu(int cpu);
+
 void init_debug_store_on_cpu(int cpu);
 
 void fini_debug_store_on_cpu(int cpu);
@@ -1572,12 +1649,16 @@ void intel_pmu_disable_bts(void);
 
 int intel_pmu_drain_bts_buffer(void);
 
+void intel_pmu_late_setup(void);
+
 u64 grt_latency_data(struct perf_event *event, u64 status);
 
 u64 cmt_latency_data(struct perf_event *event, u64 status);
 
 u64 lnl_latency_data(struct perf_event *event, u64 status);
 
+u64 arl_h_latency_data(struct perf_event *event, u64 status);
+
 extern struct event_constraint intel_core2_pebs_event_constraints[];
 
 extern struct event_constraint intel_atom_pebs_event_constraints[];
@@ -1626,20 +1707,20 @@ void intel_pmu_pebs_disable_all(void);
 
 void intel_pmu_pebs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
 
-void intel_pmu_auto_reload_read(struct perf_event *event);
+void intel_pmu_pebs_late_setup(struct cpu_hw_events *cpuc);
+
+void intel_pmu_drain_pebs_buffer(void);
 
 void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr);
 
-void intel_ds_init(void);
+void intel_pebs_init(void);
 
 void intel_pmu_lbr_save_brstack(struct perf_sample_data *data,
 				struct cpu_hw_events *cpuc,
 				struct perf_event *event);
 
-void intel_pmu_lbr_swap_task_ctx(struct perf_event_pmu_context *prev_epc,
-				 struct perf_event_pmu_context *next_epc);
-
-void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
+void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
+			      struct task_struct *task, bool sched_in);
 
 u64 lbr_from_signext_quirk_wr(u64 val);
 
@@ -1697,10 +1778,14 @@ void intel_pmu_pebs_data_source_grt(void);
 
 void intel_pmu_pebs_data_source_mtl(void);
 
+void intel_pmu_pebs_data_source_arl_h(void);
+
 void intel_pmu_pebs_data_source_cmt(void);
 
 void intel_pmu_pebs_data_source_lnl(void);
 
+u64 intel_get_arch_pebs_data_config(struct perf_event *event);
+
 int intel_pmu_setup_lbr_filter(struct perf_event *event);
 
 void intel_pt_interrupt(void);
@@ -1733,6 +1818,11 @@ static inline int intel_pmu_max_num_pebs(struct pmu *pmu)
 	return fls((u32)hybrid(pmu, pebs_events_mask));
 }
 
+static inline bool intel_pmu_has_pebs(void)
+{
+	return x86_pmu.ds_pebs || x86_pmu.arch_pebs;
+}
+
 #else /* CONFIG_CPU_SUP_INTEL */
 
 static inline void reserve_ds_buffers(void)
diff --git a/arch/x86/events/perf_event_flags.h b/arch/x86/events/perf_event_flags.h
index 6c977c19f2cd..70078334e4a3 100644
--- a/arch/x86/events/perf_event_flags.h
+++ b/arch/x86/events/perf_event_flags.h
@@ -2,23 +2,24 @@
 /*
  * struct hw_perf_event.flags flags
  */
-PERF_ARCH(PEBS_LDLAT,		0x00001) /* ld+ldlat data address sampling */
-PERF_ARCH(PEBS_ST,		0x00002) /* st data address sampling */
-PERF_ARCH(PEBS_ST_HSW,		0x00004) /* haswell style datala, store */
-PERF_ARCH(PEBS_LD_HSW,		0x00008) /* haswell style datala, load */
-PERF_ARCH(PEBS_NA_HSW,		0x00010) /* haswell style datala, unknown */
-PERF_ARCH(EXCL,			0x00020) /* HT exclusivity on counter */
-PERF_ARCH(DYNAMIC,		0x00040) /* dynamic alloc'd constraint */
-			/*	0x00080	*/
-PERF_ARCH(EXCL_ACCT,		0x00100) /* accounted EXCL event */
-PERF_ARCH(AUTO_RELOAD,		0x00200) /* use PEBS auto-reload */
-PERF_ARCH(LARGE_PEBS,		0x00400) /* use large PEBS */
-PERF_ARCH(PEBS_VIA_PT,		0x00800) /* use PT buffer for PEBS */
-PERF_ARCH(PAIR,			0x01000) /* Large Increment per Cycle */
-PERF_ARCH(LBR_SELECT,		0x02000) /* Save/Restore MSR_LBR_SELECT */
-PERF_ARCH(TOPDOWN,		0x04000) /* Count Topdown slots/metrics events */
-PERF_ARCH(PEBS_STLAT,		0x08000) /* st+stlat data address sampling */
-PERF_ARCH(AMD_BRS,		0x10000) /* AMD Branch Sampling */
-PERF_ARCH(PEBS_LAT_HYBRID,	0x20000) /* ld and st lat for hybrid */
-PERF_ARCH(NEEDS_BRANCH_STACK,	0x40000) /* require branch stack setup */
-PERF_ARCH(BRANCH_COUNTERS,	0x80000) /* logs the counters in the extra space of each branch */
+PERF_ARCH(PEBS_LDLAT,		0x0000001) /* ld+ldlat data address sampling */
+PERF_ARCH(PEBS_ST,		0x0000002) /* st data address sampling */
+PERF_ARCH(PEBS_ST_HSW,		0x0000004) /* haswell style datala, store */
+PERF_ARCH(PEBS_LD_HSW,		0x0000008) /* haswell style datala, load */
+PERF_ARCH(PEBS_NA_HSW,		0x0000010) /* haswell style datala, unknown */
+PERF_ARCH(EXCL,			0x0000020) /* HT exclusivity on counter */
+PERF_ARCH(DYNAMIC,		0x0000040) /* dynamic alloc'd constraint */
+PERF_ARCH(PEBS_CNTR,		0x0000080) /* PEBS counters snapshot */
+PERF_ARCH(EXCL_ACCT,		0x0000100) /* accounted EXCL event */
+PERF_ARCH(AUTO_RELOAD,		0x0000200) /* use PEBS auto-reload */
+PERF_ARCH(LARGE_PEBS,		0x0000400) /* use large PEBS */
+PERF_ARCH(PEBS_VIA_PT,		0x0000800) /* use PT buffer for PEBS */
+PERF_ARCH(PAIR,			0x0001000) /* Large Increment per Cycle */
+PERF_ARCH(LBR_SELECT,		0x0002000) /* Save/Restore MSR_LBR_SELECT */
+PERF_ARCH(TOPDOWN,		0x0004000) /* Count Topdown slots/metrics events */
+PERF_ARCH(PEBS_STLAT,		0x0008000) /* st+stlat data address sampling */
+PERF_ARCH(AMD_BRS,		0x0010000) /* AMD Branch Sampling */
+PERF_ARCH(PEBS_LAT_HYBRID,	0x0020000) /* ld and st lat for hybrid */
+PERF_ARCH(NEEDS_BRANCH_STACK,	0x0040000) /* require branch stack setup */
+PERF_ARCH(BRANCH_COUNTERS,	0x0080000) /* logs the counters in the extra space of each branch */
+PERF_ARCH(ACR,			0x0100000) /* Auto counter reload */
diff --git a/arch/x86/events/probe.c b/arch/x86/events/probe.c
index 600bf8d15c0c..bb719d0d3f0b 100644
--- a/arch/x86/events/probe.c
+++ b/arch/x86/events/probe.c
@@ -2,6 +2,8 @@
 #include <linux/export.h>
 #include <linux/types.h>
 #include <linux/bits.h>
+
+#include <asm/msr.h>
 #include "probe.h"
 
 static umode_t
@@ -43,7 +45,7 @@ perf_msr_probe(struct perf_msr *msr, int cnt, bool zero, void *data)
 			if (msr[bit].test && !msr[bit].test(bit, data))
 				continue;
 			/* Virt sucks; you cannot tell if a R/O MSR is present :/ */
-			if (rdmsrl_safe(msr[bit].msr, &val))
+			if (rdmsrq_safe(msr[bit].msr, &val))
 				continue;
 
 			mask = msr[bit].mask;
diff --git a/arch/x86/events/rapl.c b/arch/x86/events/rapl.c
index a481a939862e..defd86137f12 100644
--- a/arch/x86/events/rapl.c
+++ b/arch/x86/events/rapl.c
@@ -39,6 +39,10 @@
  *	  event: rapl_energy_psys
  *    perf code: 0x5
  *
+ *  core counter: consumption of a single physical core
+ *	  event: rapl_energy_core (power_core PMU)
+ *    perf code: 0x1
+ *
  * We manage those counters as free running (read-only). They may be
  * use simultaneously by other tools, such as turbostat.
  *
@@ -61,6 +65,7 @@
 #include <linux/nospec.h>
 #include <asm/cpu_device_id.h>
 #include <asm/intel-family.h>
+#include <asm/msr.h>
 #include "perf_event.h"
 #include "probe.h"
 
@@ -70,18 +75,22 @@ MODULE_LICENSE("GPL");
 /*
  * RAPL energy status counters
  */
-enum perf_rapl_events {
+enum perf_rapl_pkg_events {
 	PERF_RAPL_PP0 = 0,		/* all cores */
 	PERF_RAPL_PKG,			/* entire package */
 	PERF_RAPL_RAM,			/* DRAM */
 	PERF_RAPL_PP1,			/* gpu */
 	PERF_RAPL_PSYS,			/* psys */
 
-	PERF_RAPL_MAX,
-	NR_RAPL_DOMAINS = PERF_RAPL_MAX,
+	PERF_RAPL_PKG_EVENTS_MAX,
+	NR_RAPL_PKG_DOMAINS = PERF_RAPL_PKG_EVENTS_MAX,
 };
 
-static const char *const rapl_domain_names[NR_RAPL_DOMAINS] __initconst = {
+#define PERF_RAPL_CORE			0		/* single core */
+#define PERF_RAPL_CORE_EVENTS_MAX	1
+#define NR_RAPL_CORE_DOMAINS		PERF_RAPL_CORE_EVENTS_MAX
+
+static const char *const rapl_pkg_domain_names[NR_RAPL_PKG_DOMAINS] __initconst = {
 	"pp0-core",
 	"package",
 	"dram",
@@ -89,6 +98,8 @@ static const char *const rapl_domain_names[NR_RAPL_DOMAINS] __initconst = {
 	"psys",
 };
 
+static const char *const rapl_core_domain_name __initconst = "core";
+
 /*
  * event code: LSB 8 bits, passed in attr->config
  * any other bit is reserved
@@ -112,7 +123,7 @@ static struct perf_pmu_events_attr event_attr_##v = {				\
  *	     considered as either pkg-scope or die-scope, and we are considering
  *	     them as die-scope.
  */
-#define rapl_pmu_is_pkg_scope()				\
+#define rapl_pkg_pmu_is_pkg_scope()				\
 	(boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||	\
 	 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
 
@@ -129,7 +140,8 @@ struct rapl_pmu {
 struct rapl_pmus {
 	struct pmu		pmu;
 	unsigned int		nr_rapl_pmu;
-	struct rapl_pmu		*pmus[] __counted_by(nr_rapl_pmu);
+	unsigned int		cntr_mask;
+	struct rapl_pmu		*rapl_pmu[] __counted_by(nr_rapl_pmu);
 };
 
 enum rapl_unit_quirk {
@@ -139,67 +151,66 @@ enum rapl_unit_quirk {
 };
 
 struct rapl_model {
-	struct perf_msr *rapl_msrs;
-	unsigned long	events;
+	struct perf_msr *rapl_pkg_msrs;
+	struct perf_msr *rapl_core_msrs;
+	unsigned long	pkg_events;
+	unsigned long	core_events;
 	unsigned int	msr_power_unit;
 	enum rapl_unit_quirk	unit_quirk;
 };
 
  /* 1/2^hw_unit Joule */
-static int rapl_hw_unit[NR_RAPL_DOMAINS] __read_mostly;
-static struct rapl_pmus *rapl_pmus;
-static cpumask_t rapl_cpu_mask;
-static unsigned int rapl_cntr_mask;
+static int rapl_pkg_hw_unit[NR_RAPL_PKG_DOMAINS] __read_mostly;
+static int rapl_core_hw_unit __read_mostly;
+static struct rapl_pmus *rapl_pmus_pkg;
+static struct rapl_pmus *rapl_pmus_core;
 static u64 rapl_timer_ms;
-static struct perf_msr *rapl_msrs;
+static struct rapl_model *rapl_model;
 
 /*
- * Helper functions to get the correct topology macros according to the
+ * Helper function to get the correct topology id according to the
  * RAPL PMU scope.
  */
-static inline unsigned int get_rapl_pmu_idx(int cpu)
-{
-	return rapl_pmu_is_pkg_scope() ? topology_logical_package_id(cpu) :
-					 topology_logical_die_id(cpu);
-}
-
-static inline const struct cpumask *get_rapl_pmu_cpumask(int cpu)
+static inline unsigned int get_rapl_pmu_idx(int cpu, int scope)
 {
-	return rapl_pmu_is_pkg_scope() ? topology_core_cpumask(cpu) :
-					 topology_die_cpumask(cpu);
-}
-
-static inline struct rapl_pmu *cpu_to_rapl_pmu(unsigned int cpu)
-{
-	unsigned int rapl_pmu_idx = get_rapl_pmu_idx(cpu);
-
 	/*
-	 * The unsigned check also catches the '-1' return value for non
-	 * existent mappings in the topology map.
+	 * Returns unsigned int, which converts the '-1' return value
+	 * (for non-existent mappings in topology map) to UINT_MAX, so
+	 * the error check in the caller is simplified.
 	 */
-	return rapl_pmu_idx < rapl_pmus->nr_rapl_pmu ? rapl_pmus->pmus[rapl_pmu_idx] : NULL;
+	switch (scope) {
+	case PERF_PMU_SCOPE_PKG:
+		return topology_logical_package_id(cpu);
+	case PERF_PMU_SCOPE_DIE:
+		return topology_logical_die_id(cpu);
+	case PERF_PMU_SCOPE_CORE:
+		return topology_logical_core_id(cpu);
+	default:
+		return -EINVAL;
+	}
 }
 
 static inline u64 rapl_read_counter(struct perf_event *event)
 {
 	u64 raw;
-	rdmsrl(event->hw.event_base, raw);
+	rdmsrq(event->hw.event_base, raw);
 	return raw;
 }
 
-static inline u64 rapl_scale(u64 v, int cfg)
+static inline u64 rapl_scale(u64 v, struct perf_event *event)
 {
-	if (cfg > NR_RAPL_DOMAINS) {
-		pr_warn("Invalid domain %d, failed to scale data\n", cfg);
-		return v;
-	}
+	int hw_unit = rapl_pkg_hw_unit[event->hw.config - 1];
+
+	if (event->pmu->scope == PERF_PMU_SCOPE_CORE)
+		hw_unit = rapl_core_hw_unit;
+
 	/*
 	 * scale delta to smallest unit (1/2^32)
 	 * users must then scale back: count * 1/(1e9*2^32) to get Joules
 	 * or use ldexp(count, -32).
 	 * Watts = Joules/Time delta
 	 */
-	return v << (32 - rapl_hw_unit[cfg - 1]);
+	return v << (32 - hw_unit);
 }
 
 static u64 rapl_event_update(struct perf_event *event)
@@ -211,7 +222,7 @@ static u64 rapl_event_update(struct perf_event *event)
 
 	prev_raw_count = local64_read(&hwc->prev_count);
 	do {
-		rdmsrl(event->hw.event_base, new_raw_count);
+		rdmsrq(event->hw.event_base, new_raw_count);
 	} while (!local64_try_cmpxchg(&hwc->prev_count,
 				      &prev_raw_count, new_raw_count));
 
@@ -226,7 +237,7 @@ static u64 rapl_event_update(struct perf_event *event)
 	delta = (new_raw_count << shift) - (prev_raw_count << shift);
 	delta >>= shift;
 
-	sdelta = rapl_scale(delta, event->hw.config);
+	sdelta = rapl_scale(delta, event);
 
 	local64_add(sdelta, &event->count);
 
@@ -241,34 +252,33 @@ static void rapl_start_hrtimer(struct rapl_pmu *pmu)
 
 static enum hrtimer_restart rapl_hrtimer_handle(struct hrtimer *hrtimer)
 {
-	struct rapl_pmu *pmu = container_of(hrtimer, struct rapl_pmu, hrtimer);
+	struct rapl_pmu *rapl_pmu = container_of(hrtimer, struct rapl_pmu, hrtimer);
 	struct perf_event *event;
 	unsigned long flags;
 
-	if (!pmu->n_active)
+	if (!rapl_pmu->n_active)
 		return HRTIMER_NORESTART;
 
-	raw_spin_lock_irqsave(&pmu->lock, flags);
+	raw_spin_lock_irqsave(&rapl_pmu->lock, flags);
 
-	list_for_each_entry(event, &pmu->active_list, active_entry)
+	list_for_each_entry(event, &rapl_pmu->active_list, active_entry)
 		rapl_event_update(event);
 
-	raw_spin_unlock_irqrestore(&pmu->lock, flags);
+	raw_spin_unlock_irqrestore(&rapl_pmu->lock, flags);
 
-	hrtimer_forward_now(hrtimer, pmu->timer_interval);
+	hrtimer_forward_now(hrtimer, rapl_pmu->timer_interval);
 
 	return HRTIMER_RESTART;
 }
 
-static void rapl_hrtimer_init(struct rapl_pmu *pmu)
+static void rapl_hrtimer_init(struct rapl_pmu *rapl_pmu)
 {
-	struct hrtimer *hr = &pmu->hrtimer;
+	struct hrtimer *hr = &rapl_pmu->hrtimer;
 
-	hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
-	hr->function = rapl_hrtimer_handle;
+	hrtimer_setup(hr, rapl_hrtimer_handle, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 }
 
-static void __rapl_pmu_event_start(struct rapl_pmu *pmu,
+static void __rapl_pmu_event_start(struct rapl_pmu *rapl_pmu,
 				   struct perf_event *event)
 {
 	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
@@ -276,39 +286,39 @@ static void __rapl_pmu_event_start(struct rapl_pmu *pmu,
 
 	event->hw.state = 0;
 
-	list_add_tail(&event->active_entry, &pmu->active_list);
+	list_add_tail(&event->active_entry, &rapl_pmu->active_list);
 
 	local64_set(&event->hw.prev_count, rapl_read_counter(event));
 
-	pmu->n_active++;
-	if (pmu->n_active == 1)
-		rapl_start_hrtimer(pmu);
+	rapl_pmu->n_active++;
+	if (rapl_pmu->n_active == 1)
+		rapl_start_hrtimer(rapl_pmu);
 }
 
 static void rapl_pmu_event_start(struct perf_event *event, int mode)
 {
-	struct rapl_pmu *pmu = event->pmu_private;
+	struct rapl_pmu *rapl_pmu = event->pmu_private;
 	unsigned long flags;
 
-	raw_spin_lock_irqsave(&pmu->lock, flags);
-	__rapl_pmu_event_start(pmu, event);
-	raw_spin_unlock_irqrestore(&pmu->lock, flags);
+	raw_spin_lock_irqsave(&rapl_pmu->lock, flags);
+	__rapl_pmu_event_start(rapl_pmu, event);
+	raw_spin_unlock_irqrestore(&rapl_pmu->lock, flags);
 }
 
 static void rapl_pmu_event_stop(struct perf_event *event, int mode)
 {
-	struct rapl_pmu *pmu = event->pmu_private;
+	struct rapl_pmu *rapl_pmu = event->pmu_private;
 	struct hw_perf_event *hwc = &event->hw;
 	unsigned long flags;
 
-	raw_spin_lock_irqsave(&pmu->lock, flags);
+	raw_spin_lock_irqsave(&rapl_pmu->lock, flags);
 
 	/* mark event as deactivated and stopped */
 	if (!(hwc->state & PERF_HES_STOPPED)) {
-		WARN_ON_ONCE(pmu->n_active <= 0);
-		pmu->n_active--;
-		if (pmu->n_active == 0)
-			hrtimer_cancel(&pmu->hrtimer);
+		WARN_ON_ONCE(rapl_pmu->n_active <= 0);
+		rapl_pmu->n_active--;
+		if (rapl_pmu->n_active == 0)
+			hrtimer_cancel(&rapl_pmu->hrtimer);
 
 		list_del(&event->active_entry);
 
@@ -326,23 +336,23 @@ static void rapl_pmu_event_stop(struct perf_event *event, int mode)
 		hwc->state |= PERF_HES_UPTODATE;
 	}
 
-	raw_spin_unlock_irqrestore(&pmu->lock, flags);
+	raw_spin_unlock_irqrestore(&rapl_pmu->lock, flags);
 }
 
 static int rapl_pmu_event_add(struct perf_event *event, int mode)
 {
-	struct rapl_pmu *pmu = event->pmu_private;
+	struct rapl_pmu *rapl_pmu = event->pmu_private;
 	struct hw_perf_event *hwc = &event->hw;
 	unsigned long flags;
 
-	raw_spin_lock_irqsave(&pmu->lock, flags);
+	raw_spin_lock_irqsave(&rapl_pmu->lock, flags);
 
 	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
 
 	if (mode & PERF_EF_START)
-		__rapl_pmu_event_start(pmu, event);
+		__rapl_pmu_event_start(rapl_pmu, event);
 
-	raw_spin_unlock_irqrestore(&pmu->lock, flags);
+	raw_spin_unlock_irqrestore(&rapl_pmu->lock, flags);
 
 	return 0;
 }
@@ -355,13 +365,19 @@ static void rapl_pmu_event_del(struct perf_event *event, int flags)
 static int rapl_pmu_event_init(struct perf_event *event)
 {
 	u64 cfg = event->attr.config & RAPL_EVENT_MASK;
-	int bit, ret = 0;
-	struct rapl_pmu *pmu;
+	int bit, rapl_pmus_scope, ret = 0;
+	struct rapl_pmu *rapl_pmu;
+	unsigned int rapl_pmu_idx;
+	struct rapl_pmus *rapl_pmus;
 
 	/* only look at RAPL events */
-	if (event->attr.type != rapl_pmus->pmu.type)
+	if (event->attr.type != event->pmu->type)
 		return -ENOENT;
 
+	/* unsupported modes and filters */
+	if (event->attr.sample_period) /* no sampling */
+		return -EINVAL;
+
 	/* check only supported bits are set */
 	if (event->attr.config & ~RAPL_EVENT_MASK)
 		return -EINVAL;
@@ -369,29 +385,41 @@ static int rapl_pmu_event_init(struct perf_event *event)
 	if (event->cpu < 0)
 		return -EINVAL;
 
-	event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;
-
-	if (!cfg || cfg >= NR_RAPL_DOMAINS + 1)
+	rapl_pmus = container_of(event->pmu, struct rapl_pmus, pmu);
+	if (!rapl_pmus)
+		return -EINVAL;
+	rapl_pmus_scope = rapl_pmus->pmu.scope;
+
+	if (rapl_pmus_scope == PERF_PMU_SCOPE_PKG || rapl_pmus_scope == PERF_PMU_SCOPE_DIE) {
+		cfg = array_index_nospec((long)cfg, NR_RAPL_PKG_DOMAINS + 1);
+		if (!cfg || cfg >= NR_RAPL_PKG_DOMAINS + 1)
+			return -EINVAL;
+
+		bit = cfg - 1;
+		event->hw.event_base = rapl_model->rapl_pkg_msrs[bit].msr;
+	} else if (rapl_pmus_scope == PERF_PMU_SCOPE_CORE) {
+		cfg = array_index_nospec((long)cfg, NR_RAPL_CORE_DOMAINS + 1);
+		if (!cfg || cfg >= NR_RAPL_PKG_DOMAINS + 1)
+			return -EINVAL;
+
+		bit = cfg - 1;
+		event->hw.event_base = rapl_model->rapl_core_msrs[bit].msr;
+	} else
 		return -EINVAL;
-
-	cfg = array_index_nospec((long)cfg, NR_RAPL_DOMAINS + 1);
-	bit = cfg - 1;
 
 	/* check event supported */
-	if (!(rapl_cntr_mask & (1 << bit)))
+	if (!(rapl_pmus->cntr_mask & (1 << bit)))
 		return -EINVAL;
 
-	/* unsupported modes and filters */
-	if (event->attr.sample_period) /* no sampling */
+	rapl_pmu_idx = get_rapl_pmu_idx(event->cpu, rapl_pmus_scope);
+	if (rapl_pmu_idx >= rapl_pmus->nr_rapl_pmu)
 		return -EINVAL;
-
 	/* must be done before validate_group */
-	pmu = cpu_to_rapl_pmu(event->cpu);
-	if (!pmu)
+	rapl_pmu = rapl_pmus->rapl_pmu[rapl_pmu_idx];
+	if (!rapl_pmu)
 		return -EINVAL;
-	event->cpu = pmu->cpu;
-	event->pmu_private = pmu;
-	event->hw.event_base = rapl_msrs[bit].msr;
+
+	event->pmu_private = rapl_pmu;
 	event->hw.config = cfg;
 	event->hw.idx = bit;
 
@@ -403,34 +431,19 @@ static void rapl_pmu_event_read(struct perf_event *event)
 	rapl_event_update(event);
 }
 
-static ssize_t rapl_get_attr_cpumask(struct device *dev,
-				struct device_attribute *attr, char *buf)
-{
-	return cpumap_print_to_pagebuf(true, buf, &rapl_cpu_mask);
-}
-
-static DEVICE_ATTR(cpumask, S_IRUGO, rapl_get_attr_cpumask, NULL);
-
-static struct attribute *rapl_pmu_attrs[] = {
-	&dev_attr_cpumask.attr,
-	NULL,
-};
-
-static struct attribute_group rapl_pmu_attr_group = {
-	.attrs = rapl_pmu_attrs,
-};
-
 RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01");
 RAPL_EVENT_ATTR_STR(energy-pkg  ,   rapl_pkg, "event=0x02");
 RAPL_EVENT_ATTR_STR(energy-ram  ,   rapl_ram, "event=0x03");
 RAPL_EVENT_ATTR_STR(energy-gpu  ,   rapl_gpu, "event=0x04");
 RAPL_EVENT_ATTR_STR(energy-psys,   rapl_psys, "event=0x05");
+RAPL_EVENT_ATTR_STR(energy-core,   rapl_core, "event=0x01");
 
 RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_cores_unit, "Joules");
 RAPL_EVENT_ATTR_STR(energy-pkg.unit  ,   rapl_pkg_unit, "Joules");
 RAPL_EVENT_ATTR_STR(energy-ram.unit  ,   rapl_ram_unit, "Joules");
 RAPL_EVENT_ATTR_STR(energy-gpu.unit  ,   rapl_gpu_unit, "Joules");
 RAPL_EVENT_ATTR_STR(energy-psys.unit,   rapl_psys_unit, "Joules");
+RAPL_EVENT_ATTR_STR(energy-core.unit,   rapl_core_unit, "Joules");
 
 /*
  * we compute in 0.23 nJ increments regardless of MSR
@@ -440,6 +453,7 @@ RAPL_EVENT_ATTR_STR(energy-pkg.scale,     rapl_pkg_scale, "2.3283064365386962890
 RAPL_EVENT_ATTR_STR(energy-ram.scale,     rapl_ram_scale, "2.3283064365386962890625e-10");
 RAPL_EVENT_ATTR_STR(energy-gpu.scale,     rapl_gpu_scale, "2.3283064365386962890625e-10");
 RAPL_EVENT_ATTR_STR(energy-psys.scale,   rapl_psys_scale, "2.3283064365386962890625e-10");
+RAPL_EVENT_ATTR_STR(energy-core.scale,   rapl_core_scale, "2.3283064365386962890625e-10");
 
 /*
  * There are no default events, but we need to create
@@ -467,7 +481,12 @@ static struct attribute_group rapl_pmu_format_group = {
 };
 
 static const struct attribute_group *rapl_attr_groups[] = {
-	&rapl_pmu_attr_group,
+	&rapl_pmu_format_group,
+	&rapl_pmu_events_group,
+	NULL,
+};
+
+static const struct attribute_group *rapl_core_attr_groups[] = {
 	&rapl_pmu_format_group,
 	&rapl_pmu_events_group,
 	NULL,
@@ -533,6 +552,18 @@ static struct attribute_group rapl_events_psys_group = {
 	.attrs = rapl_events_psys,
 };
 
+static struct attribute *rapl_events_core[] = {
+	EVENT_PTR(rapl_core),
+	EVENT_PTR(rapl_core_unit),
+	EVENT_PTR(rapl_core_scale),
+	NULL,
+};
+
+static struct attribute_group rapl_events_core_group = {
+	.name  = "events",
+	.attrs = rapl_events_core,
+};
+
 static bool test_msr(int idx, void *data)
 {
 	return test_bit(idx, (unsigned long *) data);
@@ -558,11 +589,11 @@ static struct perf_msr intel_rapl_spr_msrs[] = {
 };
 
 /*
- * Force to PERF_RAPL_MAX size due to:
- * - perf_msr_probe(PERF_RAPL_MAX)
+ * Force to PERF_RAPL_PKG_EVENTS_MAX size due to:
+ * - perf_msr_probe(PERF_RAPL_PKG_EVENTS_MAX)
  * - want to use same event codes across both architectures
  */
-static struct perf_msr amd_rapl_msrs[] = {
+static struct perf_msr amd_rapl_pkg_msrs[] = {
 	[PERF_RAPL_PP0]  = { 0, &rapl_events_cores_group, NULL, false, 0 },
 	[PERF_RAPL_PKG]  = { MSR_AMD_PKG_ENERGY_STATUS,  &rapl_events_pkg_group,   test_msr, false, RAPL_MSR_MASK },
 	[PERF_RAPL_RAM]  = { 0, &rapl_events_ram_group,   NULL, false, 0 },
@@ -570,77 +601,25 @@ static struct perf_msr amd_rapl_msrs[] = {
 	[PERF_RAPL_PSYS] = { 0, &rapl_events_psys_group,  NULL, false, 0 },
 };
 
-static int rapl_cpu_offline(unsigned int cpu)
-{
-	struct rapl_pmu *pmu = cpu_to_rapl_pmu(cpu);
-	int target;
-
-	/* Check if exiting cpu is used for collecting rapl events */
-	if (!cpumask_test_and_clear_cpu(cpu, &rapl_cpu_mask))
-		return 0;
-
-	pmu->cpu = -1;
-	/* Find a new cpu to collect rapl events */
-	target = cpumask_any_but(get_rapl_pmu_cpumask(cpu), cpu);
-
-	/* Migrate rapl events to the new target */
-	if (target < nr_cpu_ids) {
-		cpumask_set_cpu(target, &rapl_cpu_mask);
-		pmu->cpu = target;
-		perf_pmu_migrate_context(pmu->pmu, cpu, target);
-	}
-	return 0;
-}
-
-static int rapl_cpu_online(unsigned int cpu)
-{
-	s32 rapl_pmu_idx = get_rapl_pmu_idx(cpu);
-	if (rapl_pmu_idx < 0) {
-		pr_err("topology_logical_(package/die)_id() returned a negative value");
-		return -EINVAL;
-	}
-	struct rapl_pmu *pmu = cpu_to_rapl_pmu(cpu);
-	int target;
-
-	if (!pmu) {
-		pmu = kzalloc_node(sizeof(*pmu), GFP_KERNEL, cpu_to_node(cpu));
-		if (!pmu)
-			return -ENOMEM;
-
-		raw_spin_lock_init(&pmu->lock);
-		INIT_LIST_HEAD(&pmu->active_list);
-		pmu->pmu = &rapl_pmus->pmu;
-		pmu->timer_interval = ms_to_ktime(rapl_timer_ms);
-		rapl_hrtimer_init(pmu);
-
-		rapl_pmus->pmus[rapl_pmu_idx] = pmu;
-	}
-
-	/*
-	 * Check if there is an online cpu in the package which collects rapl
-	 * events already.
-	 */
-	target = cpumask_any_and(&rapl_cpu_mask, get_rapl_pmu_cpumask(cpu));
-	if (target < nr_cpu_ids)
-		return 0;
-
-	cpumask_set_cpu(cpu, &rapl_cpu_mask);
-	pmu->cpu = cpu;
-	return 0;
-}
+static struct perf_msr amd_rapl_core_msrs[] = {
+	[PERF_RAPL_CORE] = { MSR_AMD_CORE_ENERGY_STATUS, &rapl_events_core_group,
+				 test_msr, false, RAPL_MSR_MASK },
+};
 
-static int rapl_check_hw_unit(struct rapl_model *rm)
+static int rapl_check_hw_unit(void)
 {
 	u64 msr_rapl_power_unit_bits;
 	int i;
 
-	/* protect rdmsrl() to handle virtualization */
-	if (rdmsrl_safe(rm->msr_power_unit, &msr_rapl_power_unit_bits))
+	/* protect rdmsrq() to handle virtualization */
+	if (rdmsrq_safe(rapl_model->msr_power_unit, &msr_rapl_power_unit_bits))
 		return -1;
-	for (i = 0; i < NR_RAPL_DOMAINS; i++)
-		rapl_hw_unit[i] = (msr_rapl_power_unit_bits >> 8) & 0x1FULL;
+	for (i = 0; i < NR_RAPL_PKG_DOMAINS; i++)
+		rapl_pkg_hw_unit[i] = (msr_rapl_power_unit_bits >> 8) & 0x1FULL;
 
-	switch (rm->unit_quirk) {
+	rapl_core_hw_unit = (msr_rapl_power_unit_bits >> 8) & 0x1FULL;
+
+	switch (rapl_model->unit_quirk) {
 	/*
 	 * DRAM domain on HSW server and KNL has fixed energy unit which can be
 	 * different than the unit from power unit MSR. See
@@ -648,17 +627,16 @@ static int rapl_check_hw_unit(struct rapl_model *rm)
 	 * of 2. Datasheet, September 2014, Reference Number: 330784-001 "
 	 */
 	case RAPL_UNIT_QUIRK_INTEL_HSW:
-		rapl_hw_unit[PERF_RAPL_RAM] = 16;
+		rapl_pkg_hw_unit[PERF_RAPL_RAM] = 16;
 		break;
 	/* SPR uses a fixed energy unit for Psys domain. */
 	case RAPL_UNIT_QUIRK_INTEL_SPR:
-		rapl_hw_unit[PERF_RAPL_PSYS] = 0;
+		rapl_pkg_hw_unit[PERF_RAPL_PSYS] = 0;
 		break;
 	default:
 		break;
 	}
 
-
 	/*
 	 * Calculate the timer rate:
 	 * Use reference of 200W for scaling the timeout to avoid counter
@@ -667,9 +645,9 @@ static int rapl_check_hw_unit(struct rapl_model *rm)
 	 * if hw unit is 32, then we use 2 ms 1/200/2
 	 */
 	rapl_timer_ms = 2;
-	if (rapl_hw_unit[0] < 32) {
+	if (rapl_pkg_hw_unit[0] < 32) {
 		rapl_timer_ms = (1000 / (2 * 100));
-		rapl_timer_ms *= (1ULL << (32 - rapl_hw_unit[0] - 1));
+		rapl_timer_ms *= (1ULL << (32 - rapl_pkg_hw_unit[0] - 1));
 	}
 	return 0;
 }
@@ -677,24 +655,32 @@ static int rapl_check_hw_unit(struct rapl_model *rm)
 static void __init rapl_advertise(void)
 {
 	int i;
+	int num_counters = hweight32(rapl_pmus_pkg->cntr_mask);
+
+	if (rapl_pmus_core)
+		num_counters += hweight32(rapl_pmus_core->cntr_mask);
 
 	pr_info("API unit is 2^-32 Joules, %d fixed counters, %llu ms ovfl timer\n",
-		hweight32(rapl_cntr_mask), rapl_timer_ms);
+		num_counters, rapl_timer_ms);
 
-	for (i = 0; i < NR_RAPL_DOMAINS; i++) {
-		if (rapl_cntr_mask & (1 << i)) {
+	for (i = 0; i < NR_RAPL_PKG_DOMAINS; i++) {
+		if (rapl_pmus_pkg->cntr_mask & (1 << i)) {
 			pr_info("hw unit of domain %s 2^-%d Joules\n",
-				rapl_domain_names[i], rapl_hw_unit[i]);
+				rapl_pkg_domain_names[i], rapl_pkg_hw_unit[i]);
 		}
 	}
+
+	if (rapl_pmus_core && (rapl_pmus_core->cntr_mask & (1 << PERF_RAPL_CORE)))
+		pr_info("hw unit of domain %s 2^-%d Joules\n",
+			rapl_core_domain_name, rapl_core_hw_unit);
 }
 
-static void cleanup_rapl_pmus(void)
+static void cleanup_rapl_pmus(struct rapl_pmus *rapl_pmus)
 {
 	int i;
 
 	for (i = 0; i < rapl_pmus->nr_rapl_pmu; i++)
-		kfree(rapl_pmus->pmus[i]);
+		kfree(rapl_pmus->rapl_pmu[i]);
 	kfree(rapl_pmus);
 }
 
@@ -707,17 +693,61 @@ static const struct attribute_group *rapl_attr_update[] = {
 	NULL,
 };
 
-static int __init init_rapl_pmus(void)
+static const struct attribute_group *rapl_core_attr_update[] = {
+	&rapl_events_core_group,
+	NULL,
+};
+
+static int __init init_rapl_pmu(struct rapl_pmus *rapl_pmus)
+{
+	struct rapl_pmu *rapl_pmu;
+	int idx;
+
+	for (idx = 0; idx < rapl_pmus->nr_rapl_pmu; idx++) {
+		rapl_pmu = kzalloc(sizeof(*rapl_pmu), GFP_KERNEL);
+		if (!rapl_pmu)
+			goto free;
+
+		raw_spin_lock_init(&rapl_pmu->lock);
+		INIT_LIST_HEAD(&rapl_pmu->active_list);
+		rapl_pmu->pmu = &rapl_pmus->pmu;
+		rapl_pmu->timer_interval = ms_to_ktime(rapl_timer_ms);
+		rapl_hrtimer_init(rapl_pmu);
+
+		rapl_pmus->rapl_pmu[idx] = rapl_pmu;
+	}
+
+	return 0;
+free:
+	for (; idx > 0; idx--)
+		kfree(rapl_pmus->rapl_pmu[idx - 1]);
+	return -ENOMEM;
+}
+
+static int __init init_rapl_pmus(struct rapl_pmus **rapl_pmus_ptr, int rapl_pmu_scope,
+				 const struct attribute_group **rapl_attr_groups,
+				 const struct attribute_group **rapl_attr_update)
 {
 	int nr_rapl_pmu = topology_max_packages();
+	struct rapl_pmus *rapl_pmus;
+	int ret;
 
-	if (!rapl_pmu_is_pkg_scope())
-		nr_rapl_pmu *= topology_max_dies_per_package();
+	/*
+	 * rapl_pmu_scope must be either PKG, DIE or CORE
+	 */
+	if (rapl_pmu_scope == PERF_PMU_SCOPE_DIE)
+		nr_rapl_pmu	*= topology_max_dies_per_package();
+	else if (rapl_pmu_scope == PERF_PMU_SCOPE_CORE)
+		nr_rapl_pmu	*= topology_num_cores_per_package();
+	else if (rapl_pmu_scope != PERF_PMU_SCOPE_PKG)
+		return -EINVAL;
 
-	rapl_pmus = kzalloc(struct_size(rapl_pmus, pmus, nr_rapl_pmu), GFP_KERNEL);
+	rapl_pmus = kzalloc(struct_size(rapl_pmus, rapl_pmu, nr_rapl_pmu), GFP_KERNEL);
 	if (!rapl_pmus)
 		return -ENOMEM;
 
+	*rapl_pmus_ptr = rapl_pmus;
+
 	rapl_pmus->nr_rapl_pmu		= nr_rapl_pmu;
 	rapl_pmus->pmu.attr_groups	= rapl_attr_groups;
 	rapl_pmus->pmu.attr_update	= rapl_attr_update;
@@ -728,77 +758,85 @@ static int __init init_rapl_pmus(void)
 	rapl_pmus->pmu.start		= rapl_pmu_event_start;
 	rapl_pmus->pmu.stop		= rapl_pmu_event_stop;
 	rapl_pmus->pmu.read		= rapl_pmu_event_read;
+	rapl_pmus->pmu.scope		= rapl_pmu_scope;
 	rapl_pmus->pmu.module		= THIS_MODULE;
 	rapl_pmus->pmu.capabilities	= PERF_PMU_CAP_NO_EXCLUDE;
-	return 0;
+
+	ret = init_rapl_pmu(rapl_pmus);
+	if (ret)
+		kfree(rapl_pmus);
+
+	return ret;
 }
 
 static struct rapl_model model_snb = {
-	.events		= BIT(PERF_RAPL_PP0) |
+	.pkg_events	= BIT(PERF_RAPL_PP0) |
 			  BIT(PERF_RAPL_PKG) |
 			  BIT(PERF_RAPL_PP1),
 	.msr_power_unit = MSR_RAPL_POWER_UNIT,
-	.rapl_msrs      = intel_rapl_msrs,
+	.rapl_pkg_msrs	= intel_rapl_msrs,
 };
 
 static struct rapl_model model_snbep = {
-	.events		= BIT(PERF_RAPL_PP0) |
+	.pkg_events	= BIT(PERF_RAPL_PP0) |
 			  BIT(PERF_RAPL_PKG) |
 			  BIT(PERF_RAPL_RAM),
 	.msr_power_unit = MSR_RAPL_POWER_UNIT,
-	.rapl_msrs      = intel_rapl_msrs,
+	.rapl_pkg_msrs	= intel_rapl_msrs,
 };
 
 static struct rapl_model model_hsw = {
-	.events		= BIT(PERF_RAPL_PP0) |
+	.pkg_events	= BIT(PERF_RAPL_PP0) |
 			  BIT(PERF_RAPL_PKG) |
 			  BIT(PERF_RAPL_RAM) |
 			  BIT(PERF_RAPL_PP1),
 	.msr_power_unit = MSR_RAPL_POWER_UNIT,
-	.rapl_msrs      = intel_rapl_msrs,
+	.rapl_pkg_msrs	= intel_rapl_msrs,
 };
 
 static struct rapl_model model_hsx = {
-	.events		= BIT(PERF_RAPL_PP0) |
+	.pkg_events	= BIT(PERF_RAPL_PP0) |
 			  BIT(PERF_RAPL_PKG) |
 			  BIT(PERF_RAPL_RAM),
 	.unit_quirk	= RAPL_UNIT_QUIRK_INTEL_HSW,
 	.msr_power_unit = MSR_RAPL_POWER_UNIT,
-	.rapl_msrs      = intel_rapl_msrs,
+	.rapl_pkg_msrs	= intel_rapl_msrs,
 };
 
 static struct rapl_model model_knl = {
-	.events		= BIT(PERF_RAPL_PKG) |
+	.pkg_events	= BIT(PERF_RAPL_PKG) |
 			  BIT(PERF_RAPL_RAM),
 	.unit_quirk	= RAPL_UNIT_QUIRK_INTEL_HSW,
 	.msr_power_unit = MSR_RAPL_POWER_UNIT,
-	.rapl_msrs      = intel_rapl_msrs,
+	.rapl_pkg_msrs	= intel_rapl_msrs,
 };
 
 static struct rapl_model model_skl = {
-	.events		= BIT(PERF_RAPL_PP0) |
+	.pkg_events	= BIT(PERF_RAPL_PP0) |
 			  BIT(PERF_RAPL_PKG) |
 			  BIT(PERF_RAPL_RAM) |
 			  BIT(PERF_RAPL_PP1) |
 			  BIT(PERF_RAPL_PSYS),
 	.msr_power_unit = MSR_RAPL_POWER_UNIT,
-	.rapl_msrs      = intel_rapl_msrs,
+	.rapl_pkg_msrs      = intel_rapl_msrs,
 };
 
 static struct rapl_model model_spr = {
-	.events		= BIT(PERF_RAPL_PP0) |
+	.pkg_events	= BIT(PERF_RAPL_PP0) |
 			  BIT(PERF_RAPL_PKG) |
 			  BIT(PERF_RAPL_RAM) |
 			  BIT(PERF_RAPL_PSYS),
 	.unit_quirk	= RAPL_UNIT_QUIRK_INTEL_SPR,
 	.msr_power_unit = MSR_RAPL_POWER_UNIT,
-	.rapl_msrs      = intel_rapl_spr_msrs,
+	.rapl_pkg_msrs	= intel_rapl_spr_msrs,
 };
 
 static struct rapl_model model_amd_hygon = {
-	.events		= BIT(PERF_RAPL_PKG),
+	.pkg_events	= BIT(PERF_RAPL_PKG),
+	.core_events	= BIT(PERF_RAPL_CORE),
 	.msr_power_unit = MSR_AMD_RAPL_POWER_UNIT,
-	.rapl_msrs      = amd_rapl_msrs,
+	.rapl_pkg_msrs	= amd_rapl_pkg_msrs,
+	.rapl_core_msrs	= amd_rapl_core_msrs,
 };
 
 static const struct x86_cpu_id rapl_model_match[] __initconst = {
@@ -846,6 +884,7 @@ static const struct x86_cpu_id rapl_model_match[] __initconst = {
 	X86_MATCH_VFM(INTEL_METEORLAKE_L,	&model_skl),
 	X86_MATCH_VFM(INTEL_ARROWLAKE_H,	&model_skl),
 	X86_MATCH_VFM(INTEL_ARROWLAKE,		&model_skl),
+	X86_MATCH_VFM(INTEL_ARROWLAKE_U,	&model_skl),
 	X86_MATCH_VFM(INTEL_LUNARLAKE_M,	&model_skl),
 	{},
 };
@@ -854,57 +893,73 @@ MODULE_DEVICE_TABLE(x86cpu, rapl_model_match);
 static int __init rapl_pmu_init(void)
 {
 	const struct x86_cpu_id *id;
-	struct rapl_model *rm;
+	int rapl_pkg_pmu_scope = PERF_PMU_SCOPE_DIE;
 	int ret;
 
+	if (rapl_pkg_pmu_is_pkg_scope())
+		rapl_pkg_pmu_scope = PERF_PMU_SCOPE_PKG;
+
 	id = x86_match_cpu(rapl_model_match);
 	if (!id)
 		return -ENODEV;
 
-	rm = (struct rapl_model *) id->driver_data;
-
-	rapl_msrs = rm->rapl_msrs;
+	rapl_model = (struct rapl_model *) id->driver_data;
 
-	rapl_cntr_mask = perf_msr_probe(rapl_msrs, PERF_RAPL_MAX,
-					false, (void *) &rm->events);
-
-	ret = rapl_check_hw_unit(rm);
+	ret = rapl_check_hw_unit();
 	if (ret)
 		return ret;
 
-	ret = init_rapl_pmus();
+	ret = init_rapl_pmus(&rapl_pmus_pkg, rapl_pkg_pmu_scope, rapl_attr_groups,
+			     rapl_attr_update);
 	if (ret)
 		return ret;
 
-	/*
-	 * Install callbacks. Core will call them for each online cpu.
-	 */
-	ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_RAPL_ONLINE,
-				"perf/x86/rapl:online",
-				rapl_cpu_online, rapl_cpu_offline);
+	rapl_pmus_pkg->cntr_mask = perf_msr_probe(rapl_model->rapl_pkg_msrs,
+						  PERF_RAPL_PKG_EVENTS_MAX, false,
+						  (void *) &rapl_model->pkg_events);
+
+	ret = perf_pmu_register(&rapl_pmus_pkg->pmu, "power", -1);
 	if (ret)
 		goto out;
 
-	ret = perf_pmu_register(&rapl_pmus->pmu, "power", -1);
-	if (ret)
-		goto out1;
+	if (rapl_model->core_events) {
+		ret = init_rapl_pmus(&rapl_pmus_core, PERF_PMU_SCOPE_CORE,
+				     rapl_core_attr_groups,
+				     rapl_core_attr_update);
+		if (ret) {
+			pr_warn("power-core PMU initialization failed (%d)\n", ret);
+			goto core_init_failed;
+		}
+
+		rapl_pmus_core->cntr_mask = perf_msr_probe(rapl_model->rapl_core_msrs,
+						     PERF_RAPL_CORE_EVENTS_MAX, false,
+						     (void *) &rapl_model->core_events);
+
+		ret = perf_pmu_register(&rapl_pmus_core->pmu, "power_core", -1);
+		if (ret) {
+			pr_warn("power-core PMU registration failed (%d)\n", ret);
+			cleanup_rapl_pmus(rapl_pmus_core);
+		}
+	}
 
+core_init_failed:
 	rapl_advertise();
 	return 0;
 
-out1:
-	cpuhp_remove_state(CPUHP_AP_PERF_X86_RAPL_ONLINE);
 out:
 	pr_warn("Initialization failed (%d), disabled\n", ret);
-	cleanup_rapl_pmus();
+	cleanup_rapl_pmus(rapl_pmus_pkg);
 	return ret;
 }
 module_init(rapl_pmu_init);
 
 static void __exit intel_rapl_exit(void)
 {
-	cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_RAPL_ONLINE);
-	perf_pmu_unregister(&rapl_pmus->pmu);
-	cleanup_rapl_pmus();
+	if (rapl_pmus_core) {
+		perf_pmu_unregister(&rapl_pmus_core->pmu);
+		cleanup_rapl_pmus(rapl_pmus_core);
+	}
+	perf_pmu_unregister(&rapl_pmus_pkg->pmu);
+	cleanup_rapl_pmus(rapl_pmus_pkg);
 }
 module_exit(intel_rapl_exit);
diff --git a/arch/x86/events/utils.c b/arch/x86/events/utils.c
index dab4ed199227..77fd00b3305e 100644
--- a/arch/x86/events/utils.c
+++ b/arch/x86/events/utils.c
@@ -2,6 +2,7 @@
 #include <asm/insn.h>
 #include <linux/mm.h>
 
+#include <asm/msr.h>
 #include "perf_event.h"
 
 static int decode_branch_type(struct insn *insn)
diff --git a/arch/x86/events/zhaoxin/core.c b/arch/x86/events/zhaoxin/core.c
index 2fd9b0cf9a5e..4bdfcf091200 100644
--- a/arch/x86/events/zhaoxin/core.c
+++ b/arch/x86/events/zhaoxin/core.c
@@ -15,6 +15,7 @@
 #include <asm/cpufeature.h>
 #include <asm/hardirq.h>
 #include <asm/apic.h>
+#include <asm/msr.h>
 
 #include "../perf_event.h"
 
@@ -254,26 +255,26 @@ static __initconst const u64 zxe_hw_cache_event_ids
 
 static void zhaoxin_pmu_disable_all(void)
 {
-	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
+	wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 }
 
 static void zhaoxin_pmu_enable_all(int added)
 {
-	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
+	wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
 }
 
 static inline u64 zhaoxin_pmu_get_status(void)
 {
 	u64 status;
 
-	rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
+	rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, status);
 
 	return status;
 }
 
 static inline void zhaoxin_pmu_ack_status(u64 ack)
 {
-	wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
+	wrmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 }
 
 static inline void zxc_pmu_ack_status(u64 ack)
@@ -293,9 +294,9 @@ static void zhaoxin_pmu_disable_fixed(struct hw_perf_event *hwc)
 
 	mask = 0xfULL << (idx * 4);
 
-	rdmsrl(hwc->config_base, ctrl_val);
+	rdmsrq(hwc->config_base, ctrl_val);
 	ctrl_val &= ~mask;
-	wrmsrl(hwc->config_base, ctrl_val);
+	wrmsrq(hwc->config_base, ctrl_val);
 }
 
 static void zhaoxin_pmu_disable_event(struct perf_event *event)
@@ -329,10 +330,10 @@ static void zhaoxin_pmu_enable_fixed(struct hw_perf_event *hwc)
 	bits <<= (idx * 4);
 	mask = 0xfULL << (idx * 4);
 
-	rdmsrl(hwc->config_base, ctrl_val);
+	rdmsrq(hwc->config_base, ctrl_val);
 	ctrl_val &= ~mask;
 	ctrl_val |= bits;
-	wrmsrl(hwc->config_base, ctrl_val);
+	wrmsrq(hwc->config_base, ctrl_val);
 }
 
 static void zhaoxin_pmu_enable_event(struct perf_event *event)
@@ -397,8 +398,7 @@ again:
 		if (!x86_perf_event_set_period(event))
 			continue;
 
-		if (perf_event_overflow(event, &data, regs))
-			x86_pmu_stop(event, 0);
+		perf_event_overflow(event, &data, regs);
 	}
 
 	/*
diff --git a/arch/x86/hyperv/Makefile b/arch/x86/hyperv/Makefile
index 3a1548054b48..56292102af62 100644
--- a/arch/x86/hyperv/Makefile
+++ b/arch/x86/hyperv/Makefile
@@ -1,8 +1,22 @@
 # SPDX-License-Identifier: GPL-2.0-only
 obj-y			:= hv_init.o mmu.o nested.o irqdomain.o ivm.o
-obj-$(CONFIG_X86_64)	+= hv_apic.o hv_proc.o
-obj-$(CONFIG_HYPERV_VTL_MODE)	+= hv_vtl.o
+obj-$(CONFIG_X86_64)	+= hv_apic.o
+obj-$(CONFIG_HYPERV_VTL_MODE)	+= hv_vtl.o mshv_vtl_asm.o
+
+$(obj)/mshv_vtl_asm.o: $(obj)/mshv-asm-offsets.h
+
+$(obj)/mshv-asm-offsets.h: $(obj)/mshv-asm-offsets.s FORCE
+	$(call filechk,offsets,__MSHV_ASM_OFFSETS_H__)
 
 ifdef CONFIG_X86_64
 obj-$(CONFIG_PARAVIRT_SPINLOCKS)	+= hv_spinlock.o
+
+ ifdef CONFIG_MSHV_ROOT
+  CFLAGS_REMOVE_hv_trampoline.o += -pg
+  CFLAGS_hv_trampoline.o        += -fno-stack-protector
+  obj-$(CONFIG_CRASH_DUMP)      += hv_crash.o hv_trampoline.o
+ endif
 endif
+
+targets += mshv-asm-offsets.s
+clean-files += mshv-asm-offsets.h
diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c
index 0569f579338b..a8de503def37 100644
--- a/arch/x86/hyperv/hv_apic.c
+++ b/arch/x86/hyperv/hv_apic.c
@@ -23,12 +23,12 @@
 #include <linux/vmalloc.h>
 #include <linux/mm.h>
 #include <linux/clockchips.h>
-#include <linux/hyperv.h>
 #include <linux/slab.h>
 #include <linux/cpuhotplug.h>
 #include <asm/hypervisor.h>
 #include <asm/mshyperv.h>
 #include <asm/apic.h>
+#include <asm/msr.h>
 
 #include <asm/trace/hyperv.h>
 
@@ -38,7 +38,7 @@ static u64 hv_apic_icr_read(void)
 {
 	u64 reg_val;
 
-	rdmsrl(HV_X64_MSR_ICR, reg_val);
+	rdmsrq(HV_X64_MSR_ICR, reg_val);
 	return reg_val;
 }
 
@@ -50,7 +50,12 @@ static void hv_apic_icr_write(u32 low, u32 id)
 	reg_val = reg_val << 32;
 	reg_val |= low;
 
-	wrmsrl(HV_X64_MSR_ICR, reg_val);
+	wrmsrq(HV_X64_MSR_ICR, reg_val);
+}
+
+void hv_enable_coco_interrupt(unsigned int cpu, unsigned int vector, bool set)
+{
+	apic_update_vector(cpu, vector, set);
 }
 
 static u32 hv_apic_read(u32 reg)
@@ -76,10 +81,10 @@ static void hv_apic_write(u32 reg, u32 val)
 {
 	switch (reg) {
 	case APIC_EOI:
-		wrmsr(HV_X64_MSR_EOI, val, 0);
+		wrmsrq(HV_X64_MSR_EOI, val);
 		break;
 	case APIC_TASKPRI:
-		wrmsr(HV_X64_MSR_TPR, val, 0);
+		wrmsrq(HV_X64_MSR_TPR, val);
 		break;
 	default:
 		native_apic_mem_write(reg, val);
@@ -93,7 +98,7 @@ static void hv_apic_eoi_write(void)
 	if (hvp && (xchg(&hvp->apic_assist, 0) & 0x1))
 		return;
 
-	wrmsr(HV_X64_MSR_EOI, APIC_EOI_ACK, 0);
+	wrmsrq(HV_X64_MSR_EOI, APIC_EOI_ACK);
 }
 
 static bool cpu_is_self(int cpu)
@@ -146,6 +151,11 @@ static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector,
 		ipi_arg->vp_set.format = HV_GENERIC_SET_ALL;
 	}
 
+	/*
+	 * For this hypercall, Hyper-V treats the valid_bank_mask field
+	 * of ipi_arg->vp_set as part of the fixed size input header.
+	 * So the variable input header size is equal to nr_bank.
+	 */
 	status = hv_do_rep_hypercall(HVCALL_SEND_IPI_EX, 0, nr_bank,
 				     ipi_arg, NULL);
 
@@ -288,6 +298,9 @@ static void hv_send_ipi_self(int vector)
 
 void __init hv_apic_init(void)
 {
+	if (cc_platform_has(CC_ATTR_SNP_SECURE_AVIC))
+		return;
+
 	if (ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) {
 		pr_info("Hyper-V: Using IPI hypercalls\n");
 		/*
diff --git a/arch/x86/hyperv/hv_crash.c b/arch/x86/hyperv/hv_crash.c
new file mode 100644
index 000000000000..c0e22921ace1
--- /dev/null
+++ b/arch/x86/hyperv/hv_crash.c
@@ -0,0 +1,642 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * X86 specific Hyper-V root partition kdump/crash support module
+ *
+ * Copyright (C) 2025, Microsoft, Inc.
+ *
+ * This module implements hypervisor RAM collection into vmcore for both
+ * cases of the hypervisor crash and Linux root crash. Hyper-V implements
+ * a disable hypercall with a 32bit protected mode ABI callback. This
+ * mechanism must be used to unlock hypervisor RAM. Since the hypervisor RAM
+ * is already mapped in Linux, it is automatically collected into Linux vmcore,
+ * and can be examined by the crash command (raw RAM dump) or windbg.
+ *
+ * At a high level:
+ *
+ *  Hypervisor Crash:
+ *    Upon crash, hypervisor goes into an emergency minimal dispatch loop, a
+ *    restrictive mode with very limited hypercall and MSR support. Each cpu
+ *    then injects NMIs into root vcpus. A shared page is used to check
+ *    by Linux in the NMI handler if the hypervisor has crashed. This shared
+ *    page is setup in hv_root_crash_init during boot.
+ *
+ *  Linux Crash:
+ *    In case of Linux crash, the callback hv_crash_stop_other_cpus will send
+ *    NMIs to all cpus, then proceed to the crash_nmi_callback where it waits
+ *    for all cpus to be in NMI.
+ *
+ *  NMI Handler (upon quorum):
+ *    Eventually, in both cases, all cpus will end up in the NMI handler.
+ *    Hyper-V requires the disable hypervisor must be done from the BSP. So
+ *    the BSP NMI handler saves current context, does some fixups and makes
+ *    the hypercall to disable the hypervisor, ie, devirtualize. Hypervisor
+ *    at that point will suspend all vcpus (except the BSP), unlock all its
+ *    RAM, and return to Linux at the 32bit mode entry RIP.
+ *
+ *  Linux 32bit entry trampoline will then restore long mode and call C
+ *  function here to restore context and continue execution to crash kexec.
+ */
+
+#include <linux/delay.h>
+#include <linux/kexec.h>
+#include <linux/crash_dump.h>
+#include <linux/panic.h>
+#include <asm/apic.h>
+#include <asm/desc.h>
+#include <asm/page.h>
+#include <asm/pgalloc.h>
+#include <asm/mshyperv.h>
+#include <asm/nmi.h>
+#include <asm/idtentry.h>
+#include <asm/reboot.h>
+#include <asm/intel_pt.h>
+
+bool hv_crash_enabled;
+EXPORT_SYMBOL_GPL(hv_crash_enabled);
+
+struct hv_crash_ctxt {
+	ulong rsp;
+	ulong cr0;
+	ulong cr2;
+	ulong cr4;
+	ulong cr8;
+
+	u16 cs;
+	u16 ss;
+	u16 ds;
+	u16 es;
+	u16 fs;
+	u16 gs;
+
+	u16 gdt_fill;
+	struct desc_ptr gdtr;
+	char idt_fill[6];
+	struct desc_ptr idtr;
+
+	u64 gsbase;
+	u64 efer;
+	u64 pat;
+};
+static struct hv_crash_ctxt hv_crash_ctxt;
+
+/* Shared hypervisor page that contains crash dump area we peek into.
+ * NB: windbg looks for "hv_cda" symbol so don't change it.
+ */
+static struct hv_crashdump_area *hv_cda;
+
+static u32 trampoline_pa, devirt_arg;
+static atomic_t crash_cpus_wait;
+static void *hv_crash_ptpgs[4];
+static bool hv_has_crashed, lx_has_crashed;
+
+static void __noreturn hv_panic_timeout_reboot(void)
+{
+	#define PANIC_TIMER_STEP 100
+
+	if (panic_timeout > 0) {
+		int i;
+
+		for (i = 0; i < panic_timeout * 1000; i += PANIC_TIMER_STEP)
+			mdelay(PANIC_TIMER_STEP);
+	}
+
+	if (panic_timeout)
+		native_wrmsrq(HV_X64_MSR_RESET, 1);    /* get hyp to reboot */
+
+	for (;;)
+		cpu_relax();
+}
+
+/* This cannot be inlined as it needs stack */
+static noinline __noclone void hv_crash_restore_tss(void)
+{
+	load_TR_desc();
+}
+
+/* This cannot be inlined as it needs stack */
+static noinline void hv_crash_clear_kernpt(void)
+{
+	pgd_t *pgd;
+	p4d_t *p4d;
+
+	/* Clear entry so it's not confusing to someone looking at the core */
+	pgd = pgd_offset_k(trampoline_pa);
+	p4d = p4d_offset(pgd, trampoline_pa);
+	native_p4d_clear(p4d);
+}
+
+/*
+ * This is the C entry point from the asm glue code after the disable hypercall.
+ * We enter here in IA32-e long mode, ie, full 64bit mode running on kernel
+ * page tables with our below 4G page identity mapped, but using a temporary
+ * GDT. ds/fs/gs/es are null. ss is not usable. bp is null. stack is not
+ * available. We restore kernel GDT, and rest of the context, and continue
+ * to kexec.
+ */
+static asmlinkage void __noreturn hv_crash_c_entry(void)
+{
+	struct hv_crash_ctxt *ctxt = &hv_crash_ctxt;
+
+	/* first thing, restore kernel gdt */
+	native_load_gdt(&ctxt->gdtr);
+
+	asm volatile("movw %%ax, %%ss" : : "a"(ctxt->ss));
+	asm volatile("movq %0, %%rsp" : : "m"(ctxt->rsp));
+
+	asm volatile("movw %%ax, %%ds" : : "a"(ctxt->ds));
+	asm volatile("movw %%ax, %%es" : : "a"(ctxt->es));
+	asm volatile("movw %%ax, %%fs" : : "a"(ctxt->fs));
+	asm volatile("movw %%ax, %%gs" : : "a"(ctxt->gs));
+
+	native_wrmsrq(MSR_IA32_CR_PAT, ctxt->pat);
+	asm volatile("movq %0, %%cr0" : : "r"(ctxt->cr0));
+
+	asm volatile("movq %0, %%cr8" : : "r"(ctxt->cr8));
+	asm volatile("movq %0, %%cr4" : : "r"(ctxt->cr4));
+	asm volatile("movq %0, %%cr2" : : "r"(ctxt->cr4));
+
+	native_load_idt(&ctxt->idtr);
+	native_wrmsrq(MSR_GS_BASE, ctxt->gsbase);
+	native_wrmsrq(MSR_EFER, ctxt->efer);
+
+	/* restore the original kernel CS now via far return */
+	asm volatile("movzwq %0, %%rax\n\t"
+		     "pushq %%rax\n\t"
+		     "pushq $1f\n\t"
+		     "lretq\n\t"
+		     "1:nop\n\t" : : "m"(ctxt->cs) : "rax");
+
+	/* We are in asmlinkage without stack frame, hence make C function
+	 * calls which will buy stack frames.
+	 */
+	hv_crash_restore_tss();
+	hv_crash_clear_kernpt();
+
+	/* we are now fully in devirtualized normal kernel mode */
+	__crash_kexec(NULL);
+
+	hv_panic_timeout_reboot();
+}
+/* Tell gcc we are using lretq long jump in the above function intentionally */
+STACK_FRAME_NON_STANDARD(hv_crash_c_entry);
+
+static void hv_mark_tss_not_busy(void)
+{
+	struct desc_struct *desc = get_current_gdt_rw();
+	tss_desc tss;
+
+	memcpy(&tss, &desc[GDT_ENTRY_TSS], sizeof(tss_desc));
+	tss.type = 0x9;        /* available 64-bit TSS. 0xB is busy TSS */
+	write_gdt_entry(desc, GDT_ENTRY_TSS, &tss, DESC_TSS);
+}
+
+/* Save essential context */
+static void hv_hvcrash_ctxt_save(void)
+{
+	struct hv_crash_ctxt *ctxt = &hv_crash_ctxt;
+
+	asm volatile("movq %%rsp,%0" : "=m"(ctxt->rsp));
+
+	ctxt->cr0 = native_read_cr0();
+	ctxt->cr4 = native_read_cr4();
+
+	asm volatile("movq %%cr2, %0" : "=a"(ctxt->cr2));
+	asm volatile("movq %%cr8, %0" : "=a"(ctxt->cr8));
+
+	asm volatile("movl %%cs, %%eax" : "=a"(ctxt->cs));
+	asm volatile("movl %%ss, %%eax" : "=a"(ctxt->ss));
+	asm volatile("movl %%ds, %%eax" : "=a"(ctxt->ds));
+	asm volatile("movl %%es, %%eax" : "=a"(ctxt->es));
+	asm volatile("movl %%fs, %%eax" : "=a"(ctxt->fs));
+	asm volatile("movl %%gs, %%eax" : "=a"(ctxt->gs));
+
+	native_store_gdt(&ctxt->gdtr);
+	store_idt(&ctxt->idtr);
+
+	ctxt->gsbase = __rdmsr(MSR_GS_BASE);
+	ctxt->efer = __rdmsr(MSR_EFER);
+	ctxt->pat = __rdmsr(MSR_IA32_CR_PAT);
+}
+
+/* Add trampoline page to the kernel pagetable for transition to kernel PT */
+static void hv_crash_fixup_kernpt(void)
+{
+	pgd_t *pgd;
+	p4d_t *p4d;
+
+	pgd = pgd_offset_k(trampoline_pa);
+	p4d = p4d_offset(pgd, trampoline_pa);
+
+	/* trampoline_pa is below 4G, so no pre-existing entry to clobber */
+	p4d_populate(&init_mm, p4d, (pud_t *)hv_crash_ptpgs[1]);
+	p4d->p4d = p4d->p4d & ~(_PAGE_NX);    /* enable execute */
+}
+
+/*
+ * Notify the hyp that Linux has crashed. This will cause the hyp to quiesce
+ * and suspend all guest VPs.
+ */
+static void hv_notify_prepare_hyp(void)
+{
+	u64 status;
+	struct hv_input_notify_partition_event *input;
+	struct hv_partition_event_root_crashdump_input *cda;
+
+	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
+	cda = &input->input.crashdump_input;
+	memset(input, 0, sizeof(*input));
+	input->event = HV_PARTITION_EVENT_ROOT_CRASHDUMP;
+
+	cda->crashdump_action = HV_CRASHDUMP_ENTRY;
+	status = hv_do_hypercall(HVCALL_NOTIFY_PARTITION_EVENT, input, NULL);
+	if (!hv_result_success(status))
+		return;
+
+	cda->crashdump_action = HV_CRASHDUMP_SUSPEND_ALL_VPS;
+	hv_do_hypercall(HVCALL_NOTIFY_PARTITION_EVENT, input, NULL);
+}
+
+/*
+ * Common function for all cpus before devirtualization.
+ *
+ * Hypervisor crash: all cpus get here in NMI context.
+ * Linux crash: the panicing cpu gets here at base level, all others in NMI
+ *		context. Note, panicing cpu may not be the BSP.
+ *
+ * The function is not inlined so it will show on the stack. It is named so
+ * because the crash cmd looks for certain well known function names on the
+ * stack before looking into the cpu saved note in the elf section, and
+ * that work is currently incomplete.
+ *
+ * Notes:
+ *  Hypervisor crash:
+ *    - the hypervisor is in a very restrictive mode at this point and any
+ *	vmexit it cannot handle would result in reboot. So, no mumbo jumbo,
+ *	just get to kexec as quickly as possible.
+ *
+ *  Devirtualization is supported from the BSP only at present.
+ */
+static noinline __noclone void crash_nmi_callback(struct pt_regs *regs)
+{
+	struct hv_input_disable_hyp_ex *input;
+	u64 status;
+	int msecs = 1000, ccpu = smp_processor_id();
+
+	if (ccpu == 0) {
+		/* crash_save_cpu() will be done in the kexec path */
+		cpu_emergency_stop_pt();	/* disable performance trace */
+		atomic_inc(&crash_cpus_wait);
+	} else {
+		crash_save_cpu(regs, ccpu);
+		cpu_emergency_stop_pt();	/* disable performance trace */
+		atomic_inc(&crash_cpus_wait);
+		for (;;)
+			cpu_relax();
+	}
+
+	while (atomic_read(&crash_cpus_wait) < num_online_cpus() && msecs--)
+		mdelay(1);
+
+	stop_nmi();
+	if (!hv_has_crashed)
+		hv_notify_prepare_hyp();
+
+	if (crashing_cpu == -1)
+		crashing_cpu = ccpu;		/* crash cmd uses this */
+
+	hv_hvcrash_ctxt_save();
+	hv_mark_tss_not_busy();
+	hv_crash_fixup_kernpt();
+
+	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
+	memset(input, 0, sizeof(*input));
+	input->rip = trampoline_pa;
+	input->arg = devirt_arg;
+
+	status = hv_do_hypercall(HVCALL_DISABLE_HYP_EX, input, NULL);
+
+	hv_panic_timeout_reboot();
+}
+
+
+static DEFINE_SPINLOCK(hv_crash_reboot_lk);
+
+/*
+ * Generic NMI callback handler: could be called without any crash also.
+ *   hv crash: hypervisor injects NMI's into all cpus
+ *   lx crash: panicing cpu sends NMI to all but self via crash_stop_other_cpus
+ */
+static int hv_crash_nmi_local(unsigned int cmd, struct pt_regs *regs)
+{
+	if (!hv_has_crashed && hv_cda && hv_cda->cda_valid)
+		hv_has_crashed = true;
+
+	if (!hv_has_crashed && !lx_has_crashed)
+		return NMI_DONE;	/* ignore the NMI */
+
+	if (hv_has_crashed && !kexec_crash_loaded()) {
+		if (spin_trylock(&hv_crash_reboot_lk))
+			hv_panic_timeout_reboot();
+		else
+			for (;;)
+				cpu_relax();
+	}
+
+	crash_nmi_callback(regs);
+
+	return NMI_DONE;
+}
+
+/*
+ * hv_crash_stop_other_cpus() == smp_ops.crash_stop_other_cpus
+ *
+ * On normal Linux panic, this is called twice: first from panic and then again
+ * from native_machine_crash_shutdown.
+ *
+ * In case of hyperv, 3 ways to get here:
+ *  1. hv crash (only BSP will get here):
+ *	BSP : NMI callback -> DisableHv -> hv_crash_asm32 -> hv_crash_c_entry
+ *		  -> __crash_kexec -> native_machine_crash_shutdown
+ *		  -> crash_smp_send_stop -> smp_ops.crash_stop_other_cpus
+ *  Linux panic:
+ *	2. panic cpu x: panic() -> crash_smp_send_stop
+ *				     -> smp_ops.crash_stop_other_cpus
+ *	3. BSP: native_machine_crash_shutdown -> crash_smp_send_stop
+ *
+ * NB: noclone and non standard stack because of call to crash_setup_regs().
+ */
+static void __noclone hv_crash_stop_other_cpus(void)
+{
+	static bool crash_stop_done;
+	struct pt_regs lregs;
+	int ccpu = smp_processor_id();
+
+	if (hv_has_crashed)
+		return;		/* all cpus already in NMI handler path */
+
+	if (!kexec_crash_loaded()) {
+		hv_notify_prepare_hyp();
+		hv_panic_timeout_reboot();	/* no return */
+	}
+
+	/* If the hv crashes also, we could come here again before cpus_stopped
+	 * is set in crash_smp_send_stop(). So use our own check.
+	 */
+	if (crash_stop_done)
+		return;
+	crash_stop_done = true;
+
+	/* Linux has crashed: hv is healthy, we can IPI safely */
+	lx_has_crashed = true;
+	wmb();			/* NMI handlers look at lx_has_crashed */
+
+	apic->send_IPI_allbutself(NMI_VECTOR);
+
+	if (crashing_cpu == -1)
+		crashing_cpu = ccpu;		/* crash cmd uses this */
+
+	/* crash_setup_regs() happens in kexec also, but for the kexec cpu which
+	 * is the BSP. We could be here on non-BSP cpu, collect regs if so.
+	 */
+	if (ccpu)
+		crash_setup_regs(&lregs, NULL);
+
+	crash_nmi_callback(&lregs);
+}
+STACK_FRAME_NON_STANDARD(hv_crash_stop_other_cpus);
+
+/* This GDT is accessed in IA32-e compat mode which uses 32bits addresses */
+struct hv_gdtreg_32 {
+	u16 fill;
+	u16 limit;
+	u32 address;
+} __packed;
+
+/* We need a CS with L bit to goto IA32-e long mode from 32bit compat mode */
+struct hv_crash_tramp_gdt {
+	u64 null;	/* index 0, selector 0, null selector */
+	u64 cs64;	/* index 1, selector 8, cs64 selector */
+} __packed;
+
+/* No stack, so jump via far ptr in memory to load the 64bit CS */
+struct hv_cs_jmptgt {
+	u32 address;
+	u16 csval;
+	u16 fill;
+} __packed;
+
+/* Linux use only, hypervisor doesn't look at this struct */
+struct hv_crash_tramp_data {
+	u64 tramp32_cr3;
+	u64 kernel_cr3;
+	struct hv_gdtreg_32 gdtr32;
+	struct hv_crash_tramp_gdt tramp_gdt;
+	struct hv_cs_jmptgt cs_jmptgt;
+	u64 c_entry_addr;
+} __packed;
+
+/*
+ * Setup a temporary gdt to allow the asm code to switch to the long mode.
+ * Since the asm code is relocated/copied to a below 4G page, it cannot use rip
+ * relative addressing, hence we must use trampoline_pa here. Also, save other
+ * info like jmp and C entry targets for same reasons.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+static int hv_crash_setup_trampdata(u64 trampoline_va)
+{
+	int size, offs;
+	void *dest;
+	struct hv_crash_tramp_data *tramp;
+
+	/* These must match exactly the ones in the corresponding asm file */
+	BUILD_BUG_ON(offsetof(struct hv_crash_tramp_data, tramp32_cr3) != 0);
+	BUILD_BUG_ON(offsetof(struct hv_crash_tramp_data, kernel_cr3) != 8);
+	BUILD_BUG_ON(offsetof(struct hv_crash_tramp_data, gdtr32.limit) != 18);
+	BUILD_BUG_ON(offsetof(struct hv_crash_tramp_data,
+						     cs_jmptgt.address) != 40);
+	BUILD_BUG_ON(offsetof(struct hv_crash_tramp_data, c_entry_addr) != 48);
+
+	/* hv_crash_asm_end is beyond last byte by 1 */
+	size = &hv_crash_asm_end - &hv_crash_asm32;
+	if (size + sizeof(struct hv_crash_tramp_data) > PAGE_SIZE) {
+		pr_err("%s: trampoline page overflow\n", __func__);
+		return -1;
+	}
+
+	dest = (void *)trampoline_va;
+	memcpy(dest, &hv_crash_asm32, size);
+
+	dest += size;
+	dest = (void *)round_up((ulong)dest, 16);
+	tramp = (struct hv_crash_tramp_data *)dest;
+
+	/* see MAX_ASID_AVAILABLE in tlb.c: "PCID 0 is reserved for use by
+	 * non-PCID-aware users". Build cr3 with pcid 0
+	 */
+	tramp->tramp32_cr3 = __sme_pa(hv_crash_ptpgs[0]);
+
+	/* Note, when restoring X86_CR4_PCIDE, cr3[11:0] must be zero */
+	tramp->kernel_cr3 = __sme_pa(init_mm.pgd);
+
+	tramp->gdtr32.limit = sizeof(struct hv_crash_tramp_gdt);
+	tramp->gdtr32.address = trampoline_pa +
+				   (ulong)&tramp->tramp_gdt - trampoline_va;
+
+	 /* base:0 limit:0xfffff type:b dpl:0 P:1 L:1 D:0 avl:0 G:1 */
+	tramp->tramp_gdt.cs64 = 0x00af9a000000ffff;
+
+	tramp->cs_jmptgt.csval = 0x8;
+	offs = (ulong)&hv_crash_asm64 - (ulong)&hv_crash_asm32;
+	tramp->cs_jmptgt.address = trampoline_pa + offs;
+
+	tramp->c_entry_addr = (u64)&hv_crash_c_entry;
+
+	devirt_arg = trampoline_pa + (ulong)dest - trampoline_va;
+
+	return 0;
+}
+
+/*
+ * Build 32bit trampoline page table for transition from protected mode
+ * non-paging to long-mode paging. This transition needs pagetables below 4G.
+ */
+static void hv_crash_build_tramp_pt(void)
+{
+	p4d_t *p4d;
+	pud_t *pud;
+	pmd_t *pmd;
+	pte_t *pte;
+	u64 pa, addr = trampoline_pa;
+
+	p4d = hv_crash_ptpgs[0] + pgd_index(addr) * sizeof(p4d);
+	pa = virt_to_phys(hv_crash_ptpgs[1]);
+	set_p4d(p4d, __p4d(_PAGE_TABLE | pa));
+	p4d->p4d &= ~(_PAGE_NX);	/* enable execute */
+
+	pud = hv_crash_ptpgs[1] + pud_index(addr) * sizeof(pud);
+	pa = virt_to_phys(hv_crash_ptpgs[2]);
+	set_pud(pud, __pud(_PAGE_TABLE | pa));
+
+	pmd = hv_crash_ptpgs[2] + pmd_index(addr) * sizeof(pmd);
+	pa = virt_to_phys(hv_crash_ptpgs[3]);
+	set_pmd(pmd, __pmd(_PAGE_TABLE | pa));
+
+	pte = hv_crash_ptpgs[3] + pte_index(addr) * sizeof(pte);
+	set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL_EXEC));
+}
+
+/*
+ * Setup trampoline for devirtualization:
+ *  - a page below 4G, ie 32bit addr containing asm glue code that hyp jmps to
+ *    in protected mode.
+ *  - 4 pages for a temporary page table that asm code uses to turn paging on
+ *  - a temporary gdt to use in the compat mode.
+ *
+ *  Returns: 0 on success
+ */
+static int hv_crash_trampoline_setup(void)
+{
+	int i, rc, order;
+	struct page *page;
+	u64 trampoline_va;
+	gfp_t flags32 = GFP_KERNEL | GFP_DMA32 | __GFP_ZERO;
+
+	/* page for 32bit trampoline assembly code + hv_crash_tramp_data */
+	page = alloc_page(flags32);
+	if (page == NULL) {
+		pr_err("%s: failed to alloc asm stub page\n", __func__);
+		return -1;
+	}
+
+	trampoline_va = (u64)page_to_virt(page);
+	trampoline_pa = (u32)page_to_phys(page);
+
+	order = 2;	   /* alloc 2^2 pages */
+	page = alloc_pages(flags32, order);
+	if (page == NULL) {
+		pr_err("%s: failed to alloc pt pages\n", __func__);
+		free_page(trampoline_va);
+		return -1;
+	}
+
+	for (i = 0; i < 4; i++, page++)
+		hv_crash_ptpgs[i] = page_to_virt(page);
+
+	hv_crash_build_tramp_pt();
+
+	rc = hv_crash_setup_trampdata(trampoline_va);
+	if (rc)
+		goto errout;
+
+	return 0;
+
+errout:
+	free_page(trampoline_va);
+	free_pages((ulong)hv_crash_ptpgs[0], order);
+
+	return rc;
+}
+
+/* Setup for kdump kexec to collect hypervisor RAM when running as root */
+void hv_root_crash_init(void)
+{
+	int rc;
+	struct hv_input_get_system_property *input;
+	struct hv_output_get_system_property *output;
+	unsigned long flags;
+	u64 status;
+	union hv_pfn_range cda_info;
+
+	if (pgtable_l5_enabled()) {
+		pr_err("Hyper-V: crash dump not yet supported on 5level PTs\n");
+		return;
+	}
+
+	rc = register_nmi_handler(NMI_LOCAL, hv_crash_nmi_local, NMI_FLAG_FIRST,
+				  "hv_crash_nmi");
+	if (rc) {
+		pr_err("Hyper-V: failed to register crash nmi handler\n");
+		return;
+	}
+
+	local_irq_save(flags);
+	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
+	output = *this_cpu_ptr(hyperv_pcpu_output_arg);
+
+	memset(input, 0, sizeof(*input));
+	input->property_id = HV_SYSTEM_PROPERTY_CRASHDUMPAREA;
+
+	status = hv_do_hypercall(HVCALL_GET_SYSTEM_PROPERTY, input, output);
+	cda_info.as_uint64 = output->hv_cda_info.as_uint64;
+	local_irq_restore(flags);
+
+	if (!hv_result_success(status)) {
+		pr_err("Hyper-V: %s: property:%d %s\n", __func__,
+		       input->property_id, hv_result_to_string(status));
+		goto err_out;
+	}
+
+	if (cda_info.base_pfn == 0) {
+		pr_err("Hyper-V: hypervisor crash dump area pfn is 0\n");
+		goto err_out;
+	}
+
+	hv_cda = phys_to_virt(cda_info.base_pfn << HV_HYP_PAGE_SHIFT);
+
+	rc = hv_crash_trampoline_setup();
+	if (rc)
+		goto err_out;
+
+	smp_ops.crash_stop_other_cpus = hv_crash_stop_other_cpus;
+
+	crash_kexec_post_notifiers = true;
+	hv_crash_enabled = true;
+	pr_info("Hyper-V: both linux and hypervisor kdump support enabled\n");
+
+	return;
+
+err_out:
+	unregister_nmi_handler(NMI_LOCAL, "hv_crash_nmi");
+	pr_err("Hyper-V: only linux root kdump support enabled\n");
+}
diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c
index 95eada2994e1..14de43f4bc6c 100644
--- a/arch/x86/hyperv/hv_init.c
+++ b/arch/x86/hyperv/hv_init.c
@@ -17,29 +17,64 @@
 #include <asm/desc.h>
 #include <asm/e820/api.h>
 #include <asm/sev.h>
-#include <asm/ibt.h>
 #include <asm/hypervisor.h>
-#include <asm/hyperv-tlfs.h>
+#include <hyperv/hvhdk.h>
 #include <asm/mshyperv.h>
+#include <asm/msr.h>
 #include <asm/idtentry.h>
 #include <asm/set_memory.h>
 #include <linux/kexec.h>
 #include <linux/version.h>
 #include <linux/vmalloc.h>
 #include <linux/mm.h>
-#include <linux/hyperv.h>
 #include <linux/slab.h>
 #include <linux/kernel.h>
 #include <linux/cpuhotplug.h>
 #include <linux/syscore_ops.h>
 #include <clocksource/hyperv_timer.h>
 #include <linux/highmem.h>
-
-u64 hv_current_partition_id = ~0ull;
-EXPORT_SYMBOL_GPL(hv_current_partition_id);
+#include <linux/export.h>
 
 void *hv_hypercall_pg;
+
+#ifdef CONFIG_X86_64
+static u64 __hv_hyperfail(u64 control, u64 param1, u64 param2)
+{
+	return U64_MAX;
+}
+
+DEFINE_STATIC_CALL(__hv_hypercall, __hv_hyperfail);
+
+u64 hv_std_hypercall(u64 control, u64 param1, u64 param2)
+{
+	u64 hv_status;
+
+	register u64 __r8 asm("r8") = param2;
+	asm volatile ("call " STATIC_CALL_TRAMP_STR(__hv_hypercall)
+		      : "=a" (hv_status), ASM_CALL_CONSTRAINT,
+		        "+c" (control), "+d" (param1), "+r" (__r8)
+		      : : "cc", "memory", "r9", "r10", "r11");
+
+	return hv_status;
+}
+
+typedef u64 (*hv_hypercall_f)(u64 control, u64 param1, u64 param2);
+
+static inline void hv_set_hypercall_pg(void *ptr)
+{
+	hv_hypercall_pg = ptr;
+
+	if (!ptr)
+		ptr = &__hv_hyperfail;
+	static_call_update(__hv_hypercall, (hv_hypercall_f)ptr);
+}
+#else
+static inline void hv_set_hypercall_pg(void *ptr)
+{
+	hv_hypercall_pg = ptr;
+}
 EXPORT_SYMBOL_GPL(hv_hypercall_pg);
+#endif
 
 union hv_ghcb * __percpu *hv_ghcb_pg;
 
@@ -66,7 +101,7 @@ static int hyperv_init_ghcb(void)
 	 * returned by MSR_AMD64_SEV_ES_GHCB is above shared
 	 * memory boundary and map it here.
 	 */
-	rdmsrl(MSR_AMD64_SEV_ES_GHCB, ghcb_gpa);
+	rdmsrq(MSR_AMD64_SEV_ES_GHCB, ghcb_gpa);
 
 	/* Mask out vTOM bit. ioremap_cache() maps decrypted */
 	ghcb_gpa &= ~ms_hyperv.shared_gpa_boundary;
@@ -94,12 +129,12 @@ static int hv_cpu_init(unsigned int cpu)
 		return 0;
 
 	hvp = &hv_vp_assist_page[cpu];
-	if (hv_root_partition) {
+	if (hv_root_partition()) {
 		/*
 		 * For root partition we get the hypervisor provided VP assist
 		 * page, instead of allocating a new page.
 		 */
-		rdmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
+		rdmsrq(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
 		*hvp = memremap(msr.pfn << HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT,
 				PAGE_SIZE, MEMREMAP_WB);
 	} else {
@@ -132,9 +167,13 @@ static int hv_cpu_init(unsigned int cpu)
 	}
 	if (!WARN_ON(!(*hvp))) {
 		msr.enable = 1;
-		wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
+		wrmsrq(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
 	}
 
+	/* Allow Hyper-V stimer vector to be injected from Hypervisor. */
+	if (ms_hyperv.misc_features & HV_STIMER_DIRECT_MODE_AVAILABLE)
+		apic_update_vector(cpu, HYPERV_STIMER0_VECTOR, true);
+
 	return hyperv_init_ghcb();
 }
 
@@ -144,7 +183,7 @@ static void hv_reenlightenment_notify(struct work_struct *dummy)
 {
 	struct hv_tsc_emulation_status emu_status;
 
-	rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
+	rdmsrq(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
 
 	/* Don't issue the callback if TSC accesses are not emulated */
 	if (hv_reenlightenment_cb && emu_status.inprogress)
@@ -157,11 +196,11 @@ void hyperv_stop_tsc_emulation(void)
 	u64 freq;
 	struct hv_tsc_emulation_status emu_status;
 
-	rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
+	rdmsrq(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
 	emu_status.inprogress = 0;
-	wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
+	wrmsrq(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
 
-	rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq);
+	rdmsrq(HV_X64_MSR_TSC_FREQUENCY, freq);
 	tsc_khz = div64_u64(freq, 1000);
 }
 EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation);
@@ -207,8 +246,8 @@ void set_hv_tscchange_cb(void (*cb)(void))
 
 	re_ctrl.target_vp = hv_vp_index[get_cpu()];
 
-	wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
-	wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl));
+	wrmsrq(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
+	wrmsrq(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl));
 
 	put_cpu();
 }
@@ -221,9 +260,9 @@ void clear_hv_tscchange_cb(void)
 	if (!hv_reenlightenment_available())
 		return;
 
-	rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl);
+	rdmsrq(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl);
 	re_ctrl.enabled = 0;
-	wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl);
+	wrmsrq(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl);
 
 	hv_reenlightenment_cb = NULL;
 }
@@ -242,11 +281,14 @@ static int hv_cpu_die(unsigned int cpu)
 		*ghcb_va = NULL;
 	}
 
+	if (ms_hyperv.misc_features & HV_STIMER_DIRECT_MODE_AVAILABLE)
+		apic_update_vector(cpu, HYPERV_STIMER0_VECTOR, false);
+
 	hv_common_cpu_die(cpu);
 
 	if (hv_vp_assist_page && hv_vp_assist_page[cpu]) {
 		union hv_vp_assist_msr_contents msr = { 0 };
-		if (hv_root_partition) {
+		if (hv_root_partition()) {
 			/*
 			 * For root partition the VP assist page is mapped to
 			 * hypervisor provided page, and thus we unmap the
@@ -255,16 +297,16 @@ static int hv_cpu_die(unsigned int cpu)
 			 */
 			memunmap(hv_vp_assist_page[cpu]);
 			hv_vp_assist_page[cpu] = NULL;
-			rdmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
+			rdmsrq(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
 			msr.enable = 0;
 		}
-		wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
+		wrmsrq(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
 	}
 
 	if (hv_reenlightenment_cb == NULL)
 		return 0;
 
-	rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
+	rdmsrq(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
 	if (re_ctrl.target_vp == hv_vp_index[cpu]) {
 		/*
 		 * Reassign reenlightenment notifications to some other online
@@ -278,7 +320,7 @@ static int hv_cpu_die(unsigned int cpu)
 		else
 			re_ctrl.enabled = 0;
 
-		wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
+		wrmsrq(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
 	}
 
 	return 0;
@@ -316,12 +358,12 @@ static int __init hv_pci_init(void)
 	return 1;
 }
 
-static int hv_suspend(void)
+static int hv_suspend(void *data)
 {
 	union hv_x64_msr_hypercall_contents hypercall_msr;
 	int ret;
 
-	if (hv_root_partition)
+	if (hv_root_partition())
 		return -EPERM;
 
 	/*
@@ -332,18 +374,18 @@ static int hv_suspend(void)
 	 * pointer is restored on resume.
 	 */
 	hv_hypercall_pg_saved = hv_hypercall_pg;
-	hv_hypercall_pg = NULL;
+	hv_set_hypercall_pg(NULL);
 
 	/* Disable the hypercall page in the hypervisor */
-	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
+	rdmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 	hypercall_msr.enable = 0;
-	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
+	wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 
 	ret = hv_cpu_die(0);
 	return ret;
 }
 
-static void hv_resume(void)
+static void hv_resume(void *data)
 {
 	union hv_x64_msr_hypercall_contents hypercall_msr;
 	int ret;
@@ -352,13 +394,13 @@ static void hv_resume(void)
 	WARN_ON(ret);
 
 	/* Re-enable the hypercall page */
-	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
+	rdmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 	hypercall_msr.enable = 1;
 	hypercall_msr.guest_physical_address =
 		vmalloc_to_pfn(hv_hypercall_pg_saved);
-	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
+	wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 
-	hv_hypercall_pg = hv_hypercall_pg_saved;
+	hv_set_hypercall_pg(hv_hypercall_pg_saved);
 	hv_hypercall_pg_saved = NULL;
 
 	/*
@@ -370,11 +412,15 @@ static void hv_resume(void)
 }
 
 /* Note: when the ops are called, only CPU0 is online and IRQs are disabled. */
-static struct syscore_ops hv_syscore_ops = {
+static const struct syscore_ops hv_syscore_ops = {
 	.suspend	= hv_suspend,
 	.resume		= hv_resume,
 };
 
+static struct syscore hv_syscore = {
+	.ops = &hv_syscore_ops,
+};
+
 static void (* __initdata old_setup_percpu_clockev)(void);
 
 static void __init hv_stimer_setup_percpu_clockev(void)
@@ -394,58 +440,6 @@ static void __init hv_stimer_setup_percpu_clockev(void)
 		old_setup_percpu_clockev();
 }
 
-static void __init hv_get_partition_id(void)
-{
-	struct hv_get_partition_id *output_page;
-	u64 status;
-	unsigned long flags;
-
-	local_irq_save(flags);
-	output_page = *this_cpu_ptr(hyperv_pcpu_output_arg);
-	status = hv_do_hypercall(HVCALL_GET_PARTITION_ID, NULL, output_page);
-	if (!hv_result_success(status)) {
-		/* No point in proceeding if this failed */
-		pr_err("Failed to get partition ID: %lld\n", status);
-		BUG();
-	}
-	hv_current_partition_id = output_page->partition_id;
-	local_irq_restore(flags);
-}
-
-#if IS_ENABLED(CONFIG_HYPERV_VTL_MODE)
-static u8 __init get_vtl(void)
-{
-	u64 control = HV_HYPERCALL_REP_COMP_1 | HVCALL_GET_VP_REGISTERS;
-	struct hv_get_vp_registers_input *input;
-	struct hv_get_vp_registers_output *output;
-	unsigned long flags;
-	u64 ret;
-
-	local_irq_save(flags);
-	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
-	output = (struct hv_get_vp_registers_output *)input;
-
-	memset(input, 0, struct_size(input, element, 1));
-	input->header.partitionid = HV_PARTITION_ID_SELF;
-	input->header.vpindex = HV_VP_INDEX_SELF;
-	input->header.inputvtl = 0;
-	input->element[0].name0 = HV_X64_REGISTER_VSM_VP_STATUS;
-
-	ret = hv_do_hypercall(control, input, output);
-	if (hv_result_success(ret)) {
-		ret = output->as64.low & HV_X64_VTL_MASK;
-	} else {
-		pr_err("Failed to get VTL(error: %lld) exiting...\n", ret);
-		BUG();
-	}
-
-	local_irq_restore(flags);
-	return ret;
-}
-#else
-static inline u8 get_vtl(void) { return 0; }
-#endif
-
 /*
  * This function is to be invoked early in the boot sequence after the
  * hypervisor has been detected.
@@ -473,7 +467,7 @@ void __init hyperv_init(void)
 	if (hv_isolation_type_tdx())
 		hv_vp_assist_page = NULL;
 	else
-		hv_vp_assist_page = kcalloc(num_possible_cpus(),
+		hv_vp_assist_page = kcalloc(nr_cpu_ids,
 					    sizeof(*hv_vp_assist_page),
 					    GFP_KERNEL);
 	if (!hv_vp_assist_page) {
@@ -521,7 +515,7 @@ void __init hyperv_init(void)
 	 * in such a VM and is only used in such a VM.
 	 */
 	guest_id = hv_generate_guest_id(LINUX_VERSION_CODE);
-	wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id);
+	wrmsrq(HV_X64_MSR_GUEST_OS_ID, guest_id);
 
 	/* With the paravisor, the VM must also write the ID via GHCB/GHCI */
 	hv_ivm_msr_write(HV_X64_MSR_GUEST_OS_ID, guest_id);
@@ -530,17 +524,17 @@ void __init hyperv_init(void)
 	if (hv_isolation_type_tdx() && !ms_hyperv.paravisor_present)
 		goto skip_hypercall_pg_init;
 
-	hv_hypercall_pg = __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START,
-			VMALLOC_END, GFP_KERNEL, PAGE_KERNEL_ROX,
+	hv_hypercall_pg = __vmalloc_node_range(PAGE_SIZE, 1, MODULES_VADDR,
+			MODULES_END, GFP_KERNEL, PAGE_KERNEL_ROX,
 			VM_FLUSH_RESET_PERMS, NUMA_NO_NODE,
 			__builtin_return_address(0));
 	if (hv_hypercall_pg == NULL)
 		goto clean_guest_os_id;
 
-	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
+	rdmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 	hypercall_msr.enable = 1;
 
-	if (hv_root_partition) {
+	if (hv_root_partition()) {
 		struct page *pg;
 		void *src;
 
@@ -554,7 +548,7 @@ void __init hyperv_init(void)
 		 * so it is populated with code, then copy the code to an
 		 * executable page.
 		 */
-		wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
+		wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 
 		pg = vmalloc_to_page(hv_hypercall_pg);
 		src = memremap(hypercall_msr.guest_physical_address << PAGE_SHIFT, PAGE_SIZE,
@@ -564,32 +558,16 @@ void __init hyperv_init(void)
 		memunmap(src);
 
 		hv_remap_tsc_clocksource();
+		hv_root_crash_init();
+		hv_sleep_notifiers_register();
 	} else {
 		hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg);
-		wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
+		wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 	}
 
-skip_hypercall_pg_init:
-	/*
-	 * Some versions of Hyper-V that provide IBT in guest VMs have a bug
-	 * in that there's no ENDBR64 instruction at the entry to the
-	 * hypercall page. Because hypercalls are invoked via an indirect call
-	 * to the hypercall page, all hypercall attempts fail when IBT is
-	 * enabled, and Linux panics. For such buggy versions, disable IBT.
-	 *
-	 * Fixed versions of Hyper-V always provide ENDBR64 on the hypercall
-	 * page, so if future Linux kernel versions enable IBT for 32-bit
-	 * builds, additional hypercall page hackery will be required here
-	 * to provide an ENDBR32.
-	 */
-#ifdef CONFIG_X86_KERNEL_IBT
-	if (cpu_feature_enabled(X86_FEATURE_IBT) &&
-	    *(u32 *)hv_hypercall_pg != gen_endbr()) {
-		setup_clear_cpu_cap(X86_FEATURE_IBT);
-		pr_warn("Disabling IBT because of Hyper-V bug\n");
-	}
-#endif
+	hv_set_hypercall_pg(hv_hypercall_pg);
 
+skip_hypercall_pg_init:
 	/*
 	 * hyperv_init() is called before LAPIC is initialized: see
 	 * apic_intr_mode_init() -> x86_platform.apic_post_init() and
@@ -604,19 +582,17 @@ skip_hypercall_pg_init:
 
 	x86_init.pci.arch_init = hv_pci_init;
 
-	register_syscore_ops(&hv_syscore_ops);
+	register_syscore(&hv_syscore);
 
-	if (cpuid_ebx(HYPERV_CPUID_FEATURES) & HV_ACCESS_PARTITION_ID)
+	if (ms_hyperv.priv_high & HV_ACCESS_PARTITION_ID)
 		hv_get_partition_id();
 
-	BUG_ON(hv_root_partition && hv_current_partition_id == ~0ull);
-
 #ifdef CONFIG_PCI_MSI
 	/*
 	 * If we're running as root, we want to create our own PCI MSI domain.
 	 * We can't set this in hv_pci_init because that would be too late.
 	 */
-	if (hv_root_partition)
+	if (hv_root_partition())
 		x86_init.irqs.create_pci_msi_domain = hv_create_pci_msi_domain;
 #endif
 
@@ -632,7 +608,7 @@ skip_hypercall_pg_init:
 	return;
 
 clean_guest_os_id:
-	wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0);
+	wrmsrq(HV_X64_MSR_GUEST_OS_ID, 0);
 	hv_ivm_msr_write(HV_X64_MSR_GUEST_OS_ID, 0);
 	cpuhp_remove_state(CPUHP_AP_HYPERV_ONLINE);
 free_ghcb_page:
@@ -653,7 +629,7 @@ void hyperv_cleanup(void)
 	union hv_reference_tsc_msr tsc_msr;
 
 	/* Reset our OS id */
-	wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0);
+	wrmsrq(HV_X64_MSR_GUEST_OS_ID, 0);
 	hv_ivm_msr_write(HV_X64_MSR_GUEST_OS_ID, 0);
 
 	/*
@@ -691,18 +667,18 @@ void hyperv_report_panic(struct pt_regs *regs, long err, bool in_die)
 		return;
 	panic_reported = true;
 
-	rdmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id);
+	rdmsrq(HV_X64_MSR_GUEST_OS_ID, guest_id);
 
-	wrmsrl(HV_X64_MSR_CRASH_P0, err);
-	wrmsrl(HV_X64_MSR_CRASH_P1, guest_id);
-	wrmsrl(HV_X64_MSR_CRASH_P2, regs->ip);
-	wrmsrl(HV_X64_MSR_CRASH_P3, regs->ax);
-	wrmsrl(HV_X64_MSR_CRASH_P4, regs->sp);
+	wrmsrq(HV_X64_MSR_CRASH_P0, err);
+	wrmsrq(HV_X64_MSR_CRASH_P1, guest_id);
+	wrmsrq(HV_X64_MSR_CRASH_P2, regs->ip);
+	wrmsrq(HV_X64_MSR_CRASH_P3, regs->ax);
+	wrmsrq(HV_X64_MSR_CRASH_P4, regs->sp);
 
 	/*
 	 * Let Hyper-V know there is crash data available
 	 */
-	wrmsrl(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY);
+	wrmsrq(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY);
 }
 EXPORT_SYMBOL_GPL(hyperv_report_panic);
 
@@ -725,8 +701,41 @@ bool hv_is_hyperv_initialized(void)
 	 * that the hypercall page is setup
 	 */
 	hypercall_msr.as_uint64 = 0;
-	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
+	rdmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 
 	return hypercall_msr.enable;
 }
 EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized);
+
+int hv_apicid_to_vp_index(u32 apic_id)
+{
+	u64 control;
+	u64 status;
+	unsigned long irq_flags;
+	struct hv_get_vp_from_apic_id_in *input;
+	u32 *output, ret;
+
+	local_irq_save(irq_flags);
+
+	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
+	memset(input, 0, sizeof(*input));
+	input->partition_id = HV_PARTITION_ID_SELF;
+	input->apic_ids[0] = apic_id;
+
+	output = *this_cpu_ptr(hyperv_pcpu_output_arg);
+
+	control = HV_HYPERCALL_REP_COMP_1 | HVCALL_GET_VP_INDEX_FROM_APIC_ID;
+	status = hv_do_hypercall(control, input, output);
+	ret = output[0];
+
+	local_irq_restore(irq_flags);
+
+	if (!hv_result_success(status)) {
+		pr_err("failed to get vp index from apic id %d, status %#llx\n",
+		       apic_id, status);
+		return -EINVAL;
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(hv_apicid_to_vp_index);
diff --git a/arch/x86/hyperv/hv_proc.c b/arch/x86/hyperv/hv_proc.c
deleted file mode 100644
index 3fa1f2ee7b0d..000000000000
--- a/arch/x86/hyperv/hv_proc.c
+++ /dev/null
@@ -1,199 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/types.h>
-#include <linux/vmalloc.h>
-#include <linux/mm.h>
-#include <linux/clockchips.h>
-#include <linux/hyperv.h>
-#include <linux/slab.h>
-#include <linux/cpuhotplug.h>
-#include <linux/minmax.h>
-#include <asm/hypervisor.h>
-#include <asm/mshyperv.h>
-#include <asm/apic.h>
-
-#include <asm/trace/hyperv.h>
-
-/*
- * See struct hv_deposit_memory. The first u64 is partition ID, the rest
- * are GPAs.
- */
-#define HV_DEPOSIT_MAX (HV_HYP_PAGE_SIZE / sizeof(u64) - 1)
-
-/* Deposits exact number of pages. Must be called with interrupts enabled.  */
-int hv_call_deposit_pages(int node, u64 partition_id, u32 num_pages)
-{
-	struct page **pages, *page;
-	int *counts;
-	int num_allocations;
-	int i, j, page_count;
-	int order;
-	u64 status;
-	int ret;
-	u64 base_pfn;
-	struct hv_deposit_memory *input_page;
-	unsigned long flags;
-
-	if (num_pages > HV_DEPOSIT_MAX)
-		return -E2BIG;
-	if (!num_pages)
-		return 0;
-
-	/* One buffer for page pointers and counts */
-	page = alloc_page(GFP_KERNEL);
-	if (!page)
-		return -ENOMEM;
-	pages = page_address(page);
-
-	counts = kcalloc(HV_DEPOSIT_MAX, sizeof(int), GFP_KERNEL);
-	if (!counts) {
-		free_page((unsigned long)pages);
-		return -ENOMEM;
-	}
-
-	/* Allocate all the pages before disabling interrupts */
-	i = 0;
-
-	while (num_pages) {
-		/* Find highest order we can actually allocate */
-		order = 31 - __builtin_clz(num_pages);
-
-		while (1) {
-			pages[i] = alloc_pages_node(node, GFP_KERNEL, order);
-			if (pages[i])
-				break;
-			if (!order) {
-				ret = -ENOMEM;
-				num_allocations = i;
-				goto err_free_allocations;
-			}
-			--order;
-		}
-
-		split_page(pages[i], order);
-		counts[i] = 1 << order;
-		num_pages -= counts[i];
-		i++;
-	}
-	num_allocations = i;
-
-	local_irq_save(flags);
-
-	input_page = *this_cpu_ptr(hyperv_pcpu_input_arg);
-
-	input_page->partition_id = partition_id;
-
-	/* Populate gpa_page_list - these will fit on the input page */
-	for (i = 0, page_count = 0; i < num_allocations; ++i) {
-		base_pfn = page_to_pfn(pages[i]);
-		for (j = 0; j < counts[i]; ++j, ++page_count)
-			input_page->gpa_page_list[page_count] = base_pfn + j;
-	}
-	status = hv_do_rep_hypercall(HVCALL_DEPOSIT_MEMORY,
-				     page_count, 0, input_page, NULL);
-	local_irq_restore(flags);
-	if (!hv_result_success(status)) {
-		pr_err("Failed to deposit pages: %lld\n", status);
-		ret = hv_result(status);
-		goto err_free_allocations;
-	}
-
-	ret = 0;
-	goto free_buf;
-
-err_free_allocations:
-	for (i = 0; i < num_allocations; ++i) {
-		base_pfn = page_to_pfn(pages[i]);
-		for (j = 0; j < counts[i]; ++j)
-			__free_page(pfn_to_page(base_pfn + j));
-	}
-
-free_buf:
-	free_page((unsigned long)pages);
-	kfree(counts);
-	return ret;
-}
-
-int hv_call_add_logical_proc(int node, u32 lp_index, u32 apic_id)
-{
-	struct hv_input_add_logical_processor *input;
-	struct hv_output_add_logical_processor *output;
-	u64 status;
-	unsigned long flags;
-	int ret = HV_STATUS_SUCCESS;
-
-	/*
-	 * When adding a logical processor, the hypervisor may return
-	 * HV_STATUS_INSUFFICIENT_MEMORY. When that happens, we deposit more
-	 * pages and retry.
-	 */
-	do {
-		local_irq_save(flags);
-
-		input = *this_cpu_ptr(hyperv_pcpu_input_arg);
-		/* We don't do anything with the output right now */
-		output = *this_cpu_ptr(hyperv_pcpu_output_arg);
-
-		input->lp_index = lp_index;
-		input->apic_id = apic_id;
-		input->proximity_domain_info = hv_numa_node_to_pxm_info(node);
-		status = hv_do_hypercall(HVCALL_ADD_LOGICAL_PROCESSOR,
-					 input, output);
-		local_irq_restore(flags);
-
-		if (hv_result(status) != HV_STATUS_INSUFFICIENT_MEMORY) {
-			if (!hv_result_success(status)) {
-				pr_err("%s: cpu %u apic ID %u, %lld\n", __func__,
-				       lp_index, apic_id, status);
-				ret = hv_result(status);
-			}
-			break;
-		}
-		ret = hv_call_deposit_pages(node, hv_current_partition_id, 1);
-	} while (!ret);
-
-	return ret;
-}
-
-int hv_call_create_vp(int node, u64 partition_id, u32 vp_index, u32 flags)
-{
-	struct hv_create_vp *input;
-	u64 status;
-	unsigned long irq_flags;
-	int ret = HV_STATUS_SUCCESS;
-
-	/* Root VPs don't seem to need pages deposited */
-	if (partition_id != hv_current_partition_id) {
-		/* The value 90 is empirically determined. It may change. */
-		ret = hv_call_deposit_pages(node, partition_id, 90);
-		if (ret)
-			return ret;
-	}
-
-	do {
-		local_irq_save(irq_flags);
-
-		input = *this_cpu_ptr(hyperv_pcpu_input_arg);
-
-		input->partition_id = partition_id;
-		input->vp_index = vp_index;
-		input->flags = flags;
-		input->subnode_type = HvSubnodeAny;
-		input->proximity_domain_info = hv_numa_node_to_pxm_info(node);
-		status = hv_do_hypercall(HVCALL_CREATE_VP, input, NULL);
-		local_irq_restore(irq_flags);
-
-		if (hv_result(status) != HV_STATUS_INSUFFICIENT_MEMORY) {
-			if (!hv_result_success(status)) {
-				pr_err("%s: vcpu %u, lp %u, %lld\n", __func__,
-				       vp_index, flags, status);
-				ret = hv_result(status);
-			}
-			break;
-		}
-		ret = hv_call_deposit_pages(node, partition_id, 1);
-
-	} while (!ret);
-
-	return ret;
-}
-
diff --git a/arch/x86/hyperv/hv_spinlock.c b/arch/x86/hyperv/hv_spinlock.c
index 151e851bef09..81b006601370 100644
--- a/arch/x86/hyperv/hv_spinlock.c
+++ b/arch/x86/hyperv/hv_spinlock.c
@@ -15,6 +15,7 @@
 #include <asm/mshyperv.h>
 #include <asm/paravirt.h>
 #include <asm/apic.h>
+#include <asm/msr.h>
 
 static bool hv_pvspin __initdata = true;
 
@@ -39,18 +40,18 @@ static void hv_qlock_wait(u8 *byte, u8 val)
 	 * To prevent a race against the unlock path it is required to
 	 * disable interrupts before accessing the HV_X64_MSR_GUEST_IDLE
 	 * MSR. Otherwise, if the IPI from hv_qlock_kick() arrives between
-	 * the lock value check and the rdmsrl() then the vCPU might be put
+	 * the lock value check and the rdmsrq() then the vCPU might be put
 	 * into 'idle' state by the hypervisor and kept in that state for
 	 * an unspecified amount of time.
 	 */
 	local_irq_save(flags);
 	/*
-	 * Only issue the rdmsrl() when the lock state has not changed.
+	 * Only issue the rdmsrq() when the lock state has not changed.
 	 */
 	if (READ_ONCE(*byte) == val) {
 		unsigned long msr_val;
 
-		rdmsrl(HV_X64_MSR_GUEST_IDLE, msr_val);
+		rdmsrq(HV_X64_MSR_GUEST_IDLE, msr_val);
 
 		(void)msr_val;
 	}
diff --git a/arch/x86/hyperv/hv_trampoline.S b/arch/x86/hyperv/hv_trampoline.S
new file mode 100644
index 000000000000..25f02ff12286
--- /dev/null
+++ b/arch/x86/hyperv/hv_trampoline.S
@@ -0,0 +1,101 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * X86 specific Hyper-V kdump/crash related code.
+ *
+ * Copyright (C) 2025, Microsoft, Inc.
+ *
+ */
+#include <linux/linkage.h>
+#include <asm/alternative.h>
+#include <asm/msr.h>
+#include <asm/processor-flags.h>
+#include <asm/nospec-branch.h>
+
+/*
+ * void noreturn hv_crash_asm32(arg1)
+ *    arg1 == edi == 32bit PA of struct hv_crash_tramp_data
+ *
+ * The hypervisor jumps here upon devirtualization in protected mode. This
+ * code gets copied to a page in the low 4G ie, 32bit space so it can run
+ * in the protected mode. Hence we cannot use any compile/link time offsets or
+ * addresses. It restores long mode via temporary gdt and page tables and
+ * eventually jumps to kernel code entry at HV_CRASHDATA_OFFS_C_entry.
+ *
+ * PreCondition (ie, Hypervisor call back ABI):
+ *  o CR0 is set to 0x0021: PE(prot mode) and NE are set, paging is disabled
+ *  o CR4 is set to 0x0
+ *  o IA32_EFER is set to 0x901 (SCE and NXE are set)
+ *  o EDI is set to the Arg passed to HVCALL_DISABLE_HYP_EX.
+ *  o CS, DS, ES, FS, GS are all initialized with a base of 0 and limit 0xFFFF
+ *  o IDTR, TR and GDTR are initialized with a base of 0 and limit of 0xFFFF
+ *  o LDTR is initialized as invalid (limit of 0)
+ *  o MSR PAT is power on default.
+ *  o Other state/registers are cleared. All TLBs flushed.
+ */
+
+#define HV_CRASHDATA_OFFS_TRAMPCR3    0x0    /*  0 */
+#define HV_CRASHDATA_OFFS_KERNCR3     0x8    /*  8 */
+#define HV_CRASHDATA_OFFS_GDTRLIMIT  0x12    /* 18 */
+#define HV_CRASHDATA_OFFS_CS_JMPTGT  0x28    /* 40 */
+#define HV_CRASHDATA_OFFS_C_entry    0x30    /* 48 */
+
+	.text
+	.code32
+
+SYM_CODE_START(hv_crash_asm32)
+	UNWIND_HINT_UNDEFINED
+	ENDBR
+	movl	$X86_CR4_PAE, %ecx
+	movl	%ecx, %cr4
+
+	movl %edi, %ebx
+	add $HV_CRASHDATA_OFFS_TRAMPCR3, %ebx
+	movl %cs:(%ebx), %eax
+	movl %eax, %cr3
+
+	/* Setup EFER for long mode now */
+	movl	$MSR_EFER, %ecx
+	rdmsr
+	btsl	$_EFER_LME, %eax
+	wrmsr
+
+	/* Turn paging on using the temp 32bit trampoline page table */
+	movl %cr0, %eax
+	orl $(X86_CR0_PG), %eax
+	movl %eax, %cr0
+
+	/* since kernel cr3 could be above 4G, we need to be in the long mode
+	 * before we can load 64bits of the kernel cr3. We use a temp gdt for
+	 * that with CS.L=1 and CS.D=0 */
+	mov %edi, %eax
+	add $HV_CRASHDATA_OFFS_GDTRLIMIT, %eax
+	lgdtl %cs:(%eax)
+
+	/* not done yet, restore CS now to switch to CS.L=1 */
+	mov %edi, %eax
+	add $HV_CRASHDATA_OFFS_CS_JMPTGT, %eax
+	ljmp %cs:*(%eax)
+SYM_CODE_END(hv_crash_asm32)
+
+	/* we now run in full 64bit IA32-e long mode, CS.L=1 and CS.D=0 */
+	.code64
+	.balign 8
+SYM_CODE_START(hv_crash_asm64)
+	UNWIND_HINT_UNDEFINED
+	ENDBR
+	/* restore kernel page tables so we can jump to kernel code */
+	mov %edi, %eax
+	add $HV_CRASHDATA_OFFS_KERNCR3, %eax
+	movq %cs:(%eax), %rbx
+	movq %rbx, %cr3
+
+	mov %edi, %eax
+	add $HV_CRASHDATA_OFFS_C_entry, %eax
+	movq %cs:(%eax), %rbx
+	ANNOTATE_RETPOLINE_SAFE
+	jmp *%rbx
+
+	int $3
+
+SYM_INNER_LABEL(hv_crash_asm_end, SYM_L_GLOBAL)
+SYM_CODE_END(hv_crash_asm64)
diff --git a/arch/x86/hyperv/hv_vtl.c b/arch/x86/hyperv/hv_vtl.c
index 04775346369c..c0edaed0efb3 100644
--- a/arch/x86/hyperv/hv_vtl.c
+++ b/arch/x86/hyperv/hv_vtl.c
@@ -9,10 +9,17 @@
 #include <asm/apic.h>
 #include <asm/boot.h>
 #include <asm/desc.h>
+#include <asm/fpu/api.h>
+#include <asm/fpu/types.h>
 #include <asm/i8259.h>
 #include <asm/mshyperv.h>
+#include <asm/msr.h>
 #include <asm/realmode.h>
+#include <asm/reboot.h>
+#include <asm/smap.h>
+#include <linux/export.h>
 #include <../kernel/smpboot.h>
+#include "../../kernel/fpu/legacy.h"
 
 extern struct boot_params boot_params;
 static struct real_mode_header hv_vtl_real_mode_header;
@@ -22,14 +29,50 @@ static bool __init hv_vtl_msi_ext_dest_id(void)
 	return true;
 }
 
+/*
+ * The `native_machine_emergency_restart` function from `reboot.c` writes
+ * to the physical address 0x472 to indicate the type of reboot for the
+ * firmware. We cannot have that in VSM as the memory composition might
+ * be more generic, and such write effectively corrupts the memory thus
+ * making diagnostics harder at the very least.
+ */
+static void  __noreturn hv_vtl_emergency_restart(void)
+{
+	/*
+	 * Cause a triple fault and the immediate reset. Here the code does not run
+	 * on the top of any firmware, whereby cannot reach out to its services.
+	 * The inifinite loop is for the improbable case that the triple fault does
+	 * not work and have to preserve the state intact for debugging.
+	 */
+	for (;;) {
+		idt_invalidate();
+		__asm__ __volatile__("int3");
+	}
+}
+
+/*
+ * The only way to restart in the VTL mode is to triple fault as the kernel runs
+ * as firmware.
+ */
+static void  __noreturn hv_vtl_restart(char __maybe_unused *cmd)
+{
+	hv_vtl_emergency_restart();
+}
+
 void __init hv_vtl_init_platform(void)
 {
-	pr_info("Linux runs in Hyper-V Virtual Trust Level\n");
+	/*
+	 * This function is a no-op if the VTL mode is not enabled.
+	 * If it is, this function runs if and only the kernel boots in
+	 * VTL2 which the x86 hv initialization path makes sure of.
+	 */
+	pr_info("Linux runs in Hyper-V Virtual Trust Level %d\n", ms_hyperv.vtl);
 
 	x86_platform.realmode_reserve = x86_init_noop;
 	x86_platform.realmode_init = x86_init_noop;
 	x86_init.irqs.pre_vector_init = x86_init_noop;
 	x86_init.timers.timer_init = x86_init_noop;
+	x86_init.resources.probe_roms = x86_init_noop;
 
 	/* Avoid searching for BIOS MP tables */
 	x86_init.mpparse.find_mptable = x86_init_noop;
@@ -117,11 +160,11 @@ static int hv_vtl_bringup_vcpu(u32 target_vp_index, int cpu, u64 eip_ignored)
 	input->vp_context.rip = rip;
 	input->vp_context.rsp = rsp;
 	input->vp_context.rflags = 0x0000000000000002;
-	input->vp_context.efer = __rdmsr(MSR_EFER);
+	input->vp_context.efer = native_rdmsrq(MSR_EFER);
 	input->vp_context.cr0 = native_read_cr0();
 	input->vp_context.cr3 = __native_read_cr3();
 	input->vp_context.cr4 = native_read_cr4();
-	input->vp_context.msr_cr_pat = __rdmsr(MSR_IA32_CR_PAT);
+	input->vp_context.msr_cr_pat = native_rdmsrq(MSR_IA32_CR_PAT);
 	input->vp_context.idtr.limit = idt_ptr.size;
 	input->vp_context.idtr.base = idt_ptr.address;
 	input->vp_context.gdtr.limit = gdt_ptr.size;
@@ -174,67 +217,30 @@ free_lock:
 	return ret;
 }
 
-static int hv_vtl_apicid_to_vp_id(u32 apic_id)
+static int hv_vtl_wakeup_secondary_cpu(u32 apicid, unsigned long start_eip, unsigned int cpu)
 {
-	u64 control;
-	u64 status;
-	unsigned long irq_flags;
-	struct hv_get_vp_from_apic_id_in *input;
-	u32 *output, ret;
-
-	local_irq_save(irq_flags);
-
-	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
-	memset(input, 0, sizeof(*input));
-	input->partition_id = HV_PARTITION_ID_SELF;
-	input->apic_ids[0] = apic_id;
-
-	output = (u32 *)input;
-
-	control = HV_HYPERCALL_REP_COMP_1 | HVCALL_GET_VP_ID_FROM_APIC_ID;
-	status = hv_do_hypercall(control, input, output);
-	ret = output[0];
-
-	local_irq_restore(irq_flags);
-
-	if (!hv_result_success(status)) {
-		pr_err("failed to get vp id from apic id %d, status %#llx\n",
-		       apic_id, status);
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-static int hv_vtl_wakeup_secondary_cpu(u32 apicid, unsigned long start_eip)
-{
-	int vp_id, cpu;
-
-	/* Find the logical CPU for the APIC ID */
-	for_each_present_cpu(cpu) {
-		if (arch_match_cpu_phys_id(cpu, apicid))
-			break;
-	}
-	if (cpu >= nr_cpu_ids)
-		return -EINVAL;
+	int vp_index;
 
 	pr_debug("Bringing up CPU with APIC ID %d in VTL2...\n", apicid);
-	vp_id = hv_vtl_apicid_to_vp_id(apicid);
+	vp_index = hv_apicid_to_vp_index(apicid);
 
-	if (vp_id < 0) {
+	if (vp_index < 0) {
 		pr_err("Couldn't find CPU with APIC ID %d\n", apicid);
 		return -EINVAL;
 	}
-	if (vp_id > ms_hyperv.max_vp_index) {
-		pr_err("Invalid CPU id %d for APIC ID %d\n", vp_id, apicid);
+	if (vp_index > ms_hyperv.max_vp_index) {
+		pr_err("Invalid CPU id %d for APIC ID %d\n", vp_index, apicid);
 		return -EINVAL;
 	}
 
-	return hv_vtl_bringup_vcpu(vp_id, cpu, start_eip);
+	return hv_vtl_bringup_vcpu(vp_index, cpu, start_eip);
 }
 
 int __init hv_vtl_early_init(void)
 {
+	machine_ops.emergency_restart = hv_vtl_emergency_restart;
+	machine_ops.restart = hv_vtl_restart;
+
 	/*
 	 * `boot_cpu_has` returns the runtime feature support,
 	 * and here is the earliest it can be used.
@@ -248,3 +254,28 @@ int __init hv_vtl_early_init(void)
 
 	return 0;
 }
+
+DEFINE_STATIC_CALL_NULL(__mshv_vtl_return_hypercall, void (*)(void));
+
+void mshv_vtl_return_call_init(u64 vtl_return_offset)
+{
+	static_call_update(__mshv_vtl_return_hypercall,
+			   (void *)((u8 *)hv_hypercall_pg + vtl_return_offset));
+}
+EXPORT_SYMBOL(mshv_vtl_return_call_init);
+
+void mshv_vtl_return_call(struct mshv_vtl_cpu_context *vtl0)
+{
+	struct hv_vp_assist_page *hvp;
+
+	hvp = hv_vp_assist_page[smp_processor_id()];
+	hvp->vtl_ret_x64rax = vtl0->rax;
+	hvp->vtl_ret_x64rcx = vtl0->rcx;
+
+	kernel_fpu_begin_mask(0);
+	fxrstor(&vtl0->fx_state);
+	__mshv_vtl_return_call(vtl0);
+	fxsave(&vtl0->fx_state);
+	kernel_fpu_end();
+}
+EXPORT_SYMBOL(mshv_vtl_return_call);
diff --git a/arch/x86/hyperv/irqdomain.c b/arch/x86/hyperv/irqdomain.c
index 3215a4a07408..c3ba12b1bc07 100644
--- a/arch/x86/hyperv/irqdomain.c
+++ b/arch/x86/hyperv/irqdomain.c
@@ -10,6 +10,8 @@
 
 #include <linux/pci.h>
 #include <linux/irq.h>
+#include <linux/export.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <asm/mshyperv.h>
 
 static int hv_map_interrupt(union hv_device_id device_id, bool level,
@@ -46,7 +48,7 @@ static int hv_map_interrupt(union hv_device_id device_id, bool level,
 	if (nr_bank < 0) {
 		local_irq_restore(flags);
 		pr_err("%s: unable to generate VP set\n", __func__);
-		return EINVAL;
+		return -EINVAL;
 	}
 	intr_desc->target.flags = HV_DEVICE_INTERRUPT_TARGET_PROCESSOR_SET;
 
@@ -64,9 +66,9 @@ static int hv_map_interrupt(union hv_device_id device_id, bool level,
 	local_irq_restore(flags);
 
 	if (!hv_result_success(status))
-		pr_err("%s: hypercall failed, status %lld\n", __func__, status);
+		hv_status_err(status, "\n");
 
-	return hv_result(status);
+	return hv_result_to_errno(status);
 }
 
 static int hv_unmap_interrupt(u64 id, struct hv_interrupt_entry *old_entry)
@@ -88,7 +90,10 @@ static int hv_unmap_interrupt(u64 id, struct hv_interrupt_entry *old_entry)
 	status = hv_do_hypercall(HVCALL_UNMAP_DEVICE_INTERRUPT, input, NULL);
 	local_irq_restore(flags);
 
-	return hv_result(status);
+	if (!hv_result_success(status))
+		hv_status_err(status, "\n");
+
+	return hv_result_to_errno(status);
 }
 
 #ifdef CONFIG_PCI_MSI
@@ -169,13 +174,34 @@ static union hv_device_id hv_build_pci_dev_id(struct pci_dev *dev)
 	return dev_id;
 }
 
-static int hv_map_msi_interrupt(struct pci_dev *dev, int cpu, int vector,
-				struct hv_interrupt_entry *entry)
+/**
+ * hv_map_msi_interrupt() - "Map" the MSI IRQ in the hypervisor.
+ * @data:      Describes the IRQ
+ * @out_entry: Hypervisor (MSI) interrupt entry (can be NULL)
+ *
+ * Map the IRQ in the hypervisor by issuing a MAP_DEVICE_INTERRUPT hypercall.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+int hv_map_msi_interrupt(struct irq_data *data,
+			 struct hv_interrupt_entry *out_entry)
 {
-	union hv_device_id device_id = hv_build_pci_dev_id(dev);
+	struct irq_cfg *cfg = irqd_cfg(data);
+	struct hv_interrupt_entry dummy;
+	union hv_device_id device_id;
+	struct msi_desc *msidesc;
+	struct pci_dev *dev;
+	int cpu;
 
-	return hv_map_interrupt(device_id, false, cpu, vector, entry);
+	msidesc = irq_data_get_msi_desc(data);
+	dev = msi_desc_to_pci_dev(msidesc);
+	device_id = hv_build_pci_dev_id(dev);
+	cpu = cpumask_first(irq_data_get_effective_affinity_mask(data));
+
+	return hv_map_interrupt(device_id, false, cpu, cfg->vector,
+				out_entry ? out_entry : &dummy);
 }
+EXPORT_SYMBOL_GPL(hv_map_msi_interrupt);
 
 static inline void entry_to_msi_msg(struct hv_interrupt_entry *entry, struct msi_msg *msg)
 {
@@ -188,13 +214,11 @@ static inline void entry_to_msi_msg(struct hv_interrupt_entry *entry, struct msi
 static int hv_unmap_msi_interrupt(struct pci_dev *dev, struct hv_interrupt_entry *old_entry);
 static void hv_irq_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 {
+	struct hv_interrupt_entry *stored_entry;
+	struct irq_cfg *cfg = irqd_cfg(data);
 	struct msi_desc *msidesc;
 	struct pci_dev *dev;
-	struct hv_interrupt_entry out_entry, *stored_entry;
-	struct irq_cfg *cfg = irqd_cfg(data);
-	const cpumask_t *affinity;
-	int cpu;
-	u64 status;
+	int ret;
 
 	msidesc = irq_data_get_msi_desc(data);
 	dev = msi_desc_to_pci_dev(msidesc);
@@ -204,9 +228,6 @@ static void hv_irq_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 		return;
 	}
 
-	affinity = irq_data_get_effective_affinity_mask(data);
-	cpu = cpumask_first_and(affinity, cpu_online_mask);
-
 	if (data->chip_data) {
 		/*
 		 * This interrupt is already mapped. Let's unmap first.
@@ -219,14 +240,12 @@ static void hv_irq_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 		stored_entry = data->chip_data;
 		data->chip_data = NULL;
 
-		status = hv_unmap_msi_interrupt(dev, stored_entry);
+		ret = hv_unmap_msi_interrupt(dev, stored_entry);
 
 		kfree(stored_entry);
 
-		if (status != HV_STATUS_SUCCESS) {
-			pr_debug("%s: failed to unmap, status %lld", __func__, status);
+		if (ret)
 			return;
-		}
 	}
 
 	stored_entry = kzalloc(sizeof(*stored_entry), GFP_ATOMIC);
@@ -235,15 +254,14 @@ static void hv_irq_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 		return;
 	}
 
-	status = hv_map_msi_interrupt(dev, cpu, cfg->vector, &out_entry);
-	if (status != HV_STATUS_SUCCESS) {
+	ret = hv_map_msi_interrupt(data, stored_entry);
+	if (ret) {
 		kfree(stored_entry);
 		return;
 	}
 
-	*stored_entry = out_entry;
 	data->chip_data = stored_entry;
-	entry_to_msi_msg(&out_entry, msg);
+	entry_to_msi_msg(data->chip_data, msg);
 
 	return;
 }
@@ -257,7 +275,6 @@ static void hv_teardown_msi_irq(struct pci_dev *dev, struct irq_data *irqd)
 {
 	struct hv_interrupt_entry old_entry;
 	struct msi_msg msg;
-	u64 status;
 
 	if (!irqd->chip_data) {
 		pr_debug("%s: no chip data\n!", __func__);
@@ -270,26 +287,7 @@ static void hv_teardown_msi_irq(struct pci_dev *dev, struct irq_data *irqd)
 	kfree(irqd->chip_data);
 	irqd->chip_data = NULL;
 
-	status = hv_unmap_msi_interrupt(dev, &old_entry);
-
-	if (status != HV_STATUS_SUCCESS)
-		pr_err("%s: hypercall failed, status %lld\n", __func__, status);
-}
-
-static void hv_msi_free_irq(struct irq_domain *domain,
-			    struct msi_domain_info *info, unsigned int virq)
-{
-	struct irq_data *irqd = irq_get_irq_data(virq);
-	struct msi_desc *desc;
-
-	if (!irqd)
-		return;
-
-	desc = irq_data_get_msi_desc(irqd);
-	if (!desc || !desc->irq || WARN_ON_ONCE(!dev_is_pci(desc->dev)))
-		return;
-
-	hv_teardown_msi_irq(to_pci_dev(desc->dev), irqd);
+	(void)hv_unmap_msi_interrupt(dev, &old_entry);
 }
 
 /*
@@ -298,37 +296,93 @@ static void hv_msi_free_irq(struct irq_domain *domain,
  */
 static struct irq_chip hv_pci_msi_controller = {
 	.name			= "HV-PCI-MSI",
-	.irq_unmask		= pci_msi_unmask_irq,
-	.irq_mask		= pci_msi_mask_irq,
 	.irq_ack		= irq_chip_ack_parent,
-	.irq_retrigger		= irq_chip_retrigger_hierarchy,
 	.irq_compose_msi_msg	= hv_irq_compose_msi_msg,
-	.irq_set_affinity	= msi_domain_set_affinity,
-	.flags			= IRQCHIP_SKIP_SET_WAKE,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
 };
 
-static struct msi_domain_ops pci_msi_domain_ops = {
-	.msi_free		= hv_msi_free_irq,
-	.msi_prepare		= pci_msi_prepare,
+static bool hv_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
+				 struct irq_domain *real_parent, struct msi_domain_info *info)
+{
+	struct irq_chip *chip = info->chip;
+
+	if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
+		return false;
+
+	chip->flags |= IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MOVE_DEFERRED;
+
+	info->ops->msi_prepare = pci_msi_prepare;
+
+	return true;
+}
+
+#define HV_MSI_FLAGS_SUPPORTED	(MSI_GENERIC_FLAGS_MASK | MSI_FLAG_PCI_MSIX)
+#define HV_MSI_FLAGS_REQUIRED	(MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS)
+
+static struct msi_parent_ops hv_msi_parent_ops = {
+	.supported_flags	= HV_MSI_FLAGS_SUPPORTED,
+	.required_flags		= HV_MSI_FLAGS_REQUIRED,
+	.bus_select_token	= DOMAIN_BUS_NEXUS,
+	.bus_select_mask	= MATCH_PCI_MSI,
+	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
+	.prefix			= "HV-",
+	.init_dev_msi_info	= hv_init_dev_msi_info,
 };
 
-static struct msi_domain_info hv_pci_msi_domain_info = {
-	.flags		= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-			  MSI_FLAG_PCI_MSIX,
-	.ops		= &pci_msi_domain_ops,
-	.chip		= &hv_pci_msi_controller,
-	.handler	= handle_edge_irq,
-	.handler_name	= "edge",
+static int hv_msi_domain_alloc(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs,
+			       void *arg)
+{
+	/*
+	 * TODO: The allocation bits of hv_irq_compose_msi_msg(), i.e. everything except
+	 * entry_to_msi_msg() should be in here.
+	 */
+
+	int ret;
+
+	ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, arg);
+	if (ret)
+		return ret;
+
+	for (int i = 0; i < nr_irqs; ++i) {
+		irq_domain_set_info(d, virq + i, 0, &hv_pci_msi_controller, NULL,
+				    handle_edge_irq, NULL, "edge");
+	}
+	return 0;
+}
+
+static void hv_msi_domain_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs)
+{
+	for (int i = 0; i < nr_irqs; ++i) {
+		struct irq_data *irqd = irq_domain_get_irq_data(d, virq);
+		struct msi_desc *desc;
+
+		desc = irq_data_get_msi_desc(irqd);
+		if (!desc || !desc->irq || WARN_ON_ONCE(!dev_is_pci(desc->dev)))
+			continue;
+
+		hv_teardown_msi_irq(to_pci_dev(desc->dev), irqd);
+	}
+	irq_domain_free_irqs_top(d, virq, nr_irqs);
+}
+
+static const struct irq_domain_ops hv_msi_domain_ops = {
+	.select	= msi_lib_irq_domain_select,
+	.alloc	= hv_msi_domain_alloc,
+	.free	= hv_msi_domain_free,
 };
 
 struct irq_domain * __init hv_create_pci_msi_domain(void)
 {
 	struct irq_domain *d = NULL;
-	struct fwnode_handle *fn;
 
-	fn = irq_domain_alloc_named_fwnode("HV-PCI-MSI");
-	if (fn)
-		d = pci_msi_create_irq_domain(fn, &hv_pci_msi_domain_info, x86_vector_domain);
+	struct irq_domain_info info = {
+		.fwnode		= irq_domain_alloc_named_fwnode("HV-PCI-MSI"),
+		.ops		= &hv_msi_domain_ops,
+		.parent		= x86_vector_domain,
+	};
+
+	if (info.fwnode)
+		d = msi_create_parent_irq_domain(&info, &hv_msi_parent_ops);
 
 	/* No point in going further if we can't get an irq domain */
 	BUG_ON(!d);
diff --git a/arch/x86/hyperv/ivm.c b/arch/x86/hyperv/ivm.c
index 60fc3ed72830..651771534cae 100644
--- a/arch/x86/hyperv/ivm.c
+++ b/arch/x86/hyperv/ivm.c
@@ -7,9 +7,10 @@
  */
 
 #include <linux/bitfield.h>
-#include <linux/hyperv.h>
 #include <linux/types.h>
 #include <linux/slab.h>
+#include <linux/cpu.h>
+#include <linux/export.h>
 #include <asm/svm.h>
 #include <asm/sev.h>
 #include <asm/io.h>
@@ -23,6 +24,7 @@
 #include <asm/realmode.h>
 #include <asm/e820/api.h>
 #include <asm/desc.h>
+#include <asm/msr.h>
 #include <uapi/asm/vmx.h>
 
 #ifdef CONFIG_AMD_MEM_ENCRYPT
@@ -111,12 +113,12 @@ u64 hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_size)
 
 static inline u64 rd_ghcb_msr(void)
 {
-	return __rdmsr(MSR_AMD64_SEV_ES_GHCB);
+	return native_rdmsrq(MSR_AMD64_SEV_ES_GHCB);
 }
 
 static inline void wr_ghcb_msr(u64 val)
 {
-	native_wrmsrl(MSR_AMD64_SEV_ES_GHCB, val);
+	native_wrmsrq(MSR_AMD64_SEV_ES_GHCB, val);
 }
 
 static enum es_result hv_ghcb_hv_call(struct ghcb *ghcb, u64 exit_code,
@@ -289,7 +291,7 @@ static void snp_cleanup_vmsa(struct sev_es_save_area *vmsa)
 		free_page((unsigned long)vmsa);
 }
 
-int hv_snp_boot_ap(u32 cpu, unsigned long start_ip)
+int hv_snp_boot_ap(u32 apic_id, unsigned long start_ip, unsigned int cpu)
 {
 	struct sev_es_save_area *vmsa = (struct sev_es_save_area *)
 		__get_free_page(GFP_KERNEL | __GFP_ZERO);
@@ -298,10 +300,16 @@ int hv_snp_boot_ap(u32 cpu, unsigned long start_ip)
 	u64 ret, retry = 5;
 	struct hv_enable_vp_vtl *start_vp_input;
 	unsigned long flags;
+	int vp_index;
 
 	if (!vmsa)
 		return -ENOMEM;
 
+	/* Find the Hyper-V VP index which might be not the same as APIC ID */
+	vp_index = hv_apicid_to_vp_index(apic_id);
+	if (vp_index < 0 || vp_index > ms_hyperv.max_vp_index)
+		return -EINVAL;
+
 	native_store_gdt(&gdtr);
 
 	vmsa->gdtr.base = gdtr.address;
@@ -339,7 +347,7 @@ int hv_snp_boot_ap(u32 cpu, unsigned long start_ip)
 	vmsa->sev_features = sev_status >> 2;
 
 	ret = snp_set_vmsa(vmsa, true);
-	if (!ret) {
+	if (ret) {
 		pr_err("RMPADJUST(%llx) failed: %llx\n", (u64)vmsa, ret);
 		free_page((u64)vmsa);
 		return ret;
@@ -349,7 +357,7 @@ int hv_snp_boot_ap(u32 cpu, unsigned long start_ip)
 	start_vp_input = (struct hv_enable_vp_vtl *)ap_start_input_arg;
 	memset(start_vp_input, 0, sizeof(*start_vp_input));
 	start_vp_input->partition_id = -1;
-	start_vp_input->vp_index = cpu;
+	start_vp_input->vp_index = vp_index;
 	start_vp_input->target_vtl.target_vtl = ms_hyperv.vtl;
 	*(u64 *)&start_vp_input->vp_context = __pa(vmsa) | 1;
 
@@ -377,9 +385,23 @@ int hv_snp_boot_ap(u32 cpu, unsigned long start_ip)
 	return ret;
 }
 
+u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2)
+{
+	u64 hv_status;
+
+	register u64 __r8 asm("r8") = param2;
+	asm volatile("vmmcall"
+		     : "=a" (hv_status), ASM_CALL_CONSTRAINT,
+		       "+c" (control), "+d" (param1), "+r" (__r8)
+		     : : "cc", "memory", "r9", "r10", "r11");
+
+	return hv_status;
+}
+
 #else
 static inline void hv_ghcb_msr_write(u64 msr, u64 value) {}
 static inline void hv_ghcb_msr_read(u64 msr, u64 *value) {}
+u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2) { return U64_MAX; }
 #endif /* CONFIG_AMD_MEM_ENCRYPT */
 
 #ifdef CONFIG_INTEL_TDX_GUEST
@@ -429,6 +451,7 @@ u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2)
 #else
 static inline void hv_tdx_msr_write(u64 msr, u64 value) {}
 static inline void hv_tdx_msr_read(u64 msr, u64 *value) {}
+u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2) { return U64_MAX; }
 #endif /* CONFIG_INTEL_TDX_GUEST */
 
 #if defined(CONFIG_AMD_MEM_ENCRYPT) || defined(CONFIG_INTEL_TDX_GUEST)
@@ -455,6 +478,195 @@ void hv_ivm_msr_read(u64 msr, u64 *value)
 }
 
 /*
+ * Keep track of the PFN regions which were shared with the host. The access
+ * must be revoked upon kexec/kdump (see hv_ivm_clear_host_access()).
+ */
+struct hv_enc_pfn_region {
+	struct list_head list;
+	u64 pfn;
+	int count;
+};
+
+static LIST_HEAD(hv_list_enc);
+static DEFINE_RAW_SPINLOCK(hv_list_enc_lock);
+
+static int hv_list_enc_add(const u64 *pfn_list, int count)
+{
+	struct hv_enc_pfn_region *ent;
+	unsigned long flags;
+	u64 pfn;
+	int i;
+
+	for (i = 0; i < count; i++) {
+		pfn = pfn_list[i];
+
+		raw_spin_lock_irqsave(&hv_list_enc_lock, flags);
+		/* Check if the PFN already exists in some region first */
+		list_for_each_entry(ent, &hv_list_enc, list) {
+			if ((ent->pfn <= pfn) && (ent->pfn + ent->count - 1 >= pfn))
+				/* Nothing to do - pfn is already in the list */
+				goto unlock_done;
+		}
+
+		/*
+		 * Check if the PFN is adjacent to an existing region. Growing
+		 * a region can make it adjacent to another one but merging is
+		 * not (yet) implemented for simplicity. A PFN cannot be added
+		 * to two regions to keep the logic in hv_list_enc_remove()
+		 * correct.
+		 */
+		list_for_each_entry(ent, &hv_list_enc, list) {
+			if (ent->pfn + ent->count == pfn) {
+				/* Grow existing region up */
+				ent->count++;
+				goto unlock_done;
+			} else if (pfn + 1 == ent->pfn) {
+				/* Grow existing region down */
+				ent->pfn--;
+				ent->count++;
+				goto unlock_done;
+			}
+		}
+		raw_spin_unlock_irqrestore(&hv_list_enc_lock, flags);
+
+		/* No adjacent region found -- create a new one */
+		ent = kzalloc(sizeof(struct hv_enc_pfn_region), GFP_KERNEL);
+		if (!ent)
+			return -ENOMEM;
+
+		ent->pfn = pfn;
+		ent->count = 1;
+
+		raw_spin_lock_irqsave(&hv_list_enc_lock, flags);
+		list_add(&ent->list, &hv_list_enc);
+
+unlock_done:
+		raw_spin_unlock_irqrestore(&hv_list_enc_lock, flags);
+	}
+
+	return 0;
+}
+
+static int hv_list_enc_remove(const u64 *pfn_list, int count)
+{
+	struct hv_enc_pfn_region *ent, *t;
+	struct hv_enc_pfn_region new_region;
+	unsigned long flags;
+	u64 pfn;
+	int i;
+
+	for (i = 0; i < count; i++) {
+		pfn = pfn_list[i];
+
+		raw_spin_lock_irqsave(&hv_list_enc_lock, flags);
+		list_for_each_entry_safe(ent, t, &hv_list_enc, list) {
+			if (pfn == ent->pfn + ent->count - 1) {
+				/* Removing tail pfn */
+				ent->count--;
+				if (!ent->count) {
+					list_del(&ent->list);
+					kfree(ent);
+				}
+				goto unlock_done;
+			} else if (pfn == ent->pfn) {
+				/* Removing head pfn */
+				ent->count--;
+				ent->pfn++;
+				if (!ent->count) {
+					list_del(&ent->list);
+					kfree(ent);
+				}
+				goto unlock_done;
+			} else if (pfn > ent->pfn && pfn < ent->pfn + ent->count - 1) {
+				/*
+				 * Removing a pfn in the middle. Cut off the tail
+				 * of the existing region and create a template for
+				 * the new one.
+				 */
+				new_region.pfn = pfn + 1;
+				new_region.count = ent->count - (pfn - ent->pfn + 1);
+				ent->count = pfn - ent->pfn;
+				goto unlock_split;
+			}
+
+		}
+unlock_done:
+		raw_spin_unlock_irqrestore(&hv_list_enc_lock, flags);
+		continue;
+
+unlock_split:
+		raw_spin_unlock_irqrestore(&hv_list_enc_lock, flags);
+
+		ent = kzalloc(sizeof(struct hv_enc_pfn_region), GFP_KERNEL);
+		if (!ent)
+			return -ENOMEM;
+
+		ent->pfn = new_region.pfn;
+		ent->count = new_region.count;
+
+		raw_spin_lock_irqsave(&hv_list_enc_lock, flags);
+		list_add(&ent->list, &hv_list_enc);
+		raw_spin_unlock_irqrestore(&hv_list_enc_lock, flags);
+	}
+
+	return 0;
+}
+
+/* Stop new private<->shared conversions */
+static void hv_vtom_kexec_begin(void)
+{
+	if (!IS_ENABLED(CONFIG_KEXEC_CORE))
+		return;
+
+	/*
+	 * Crash kernel reaches here with interrupts disabled: can't wait for
+	 * conversions to finish.
+	 *
+	 * If race happened, just report and proceed.
+	 */
+	if (!set_memory_enc_stop_conversion())
+		pr_warn("Failed to stop shared<->private conversions\n");
+}
+
+static void hv_vtom_kexec_finish(void)
+{
+	struct hv_gpa_range_for_visibility *input;
+	struct hv_enc_pfn_region *ent;
+	unsigned long flags;
+	u64 hv_status;
+	int cur, i;
+
+	local_irq_save(flags);
+	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
+
+	if (unlikely(!input))
+		goto out;
+
+	list_for_each_entry(ent, &hv_list_enc, list) {
+		for (i = 0, cur = 0; i < ent->count; i++) {
+			input->gpa_page_list[cur] = ent->pfn + i;
+			cur++;
+
+			if (cur == HV_MAX_MODIFY_GPA_REP_COUNT || i == ent->count - 1) {
+				input->partition_id = HV_PARTITION_ID_SELF;
+				input->host_visibility = VMBUS_PAGE_NOT_VISIBLE;
+				input->reserved0 = 0;
+				input->reserved1 = 0;
+				hv_status = hv_do_rep_hypercall(
+					HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY,
+					cur, 0, input, NULL);
+				WARN_ON_ONCE(!hv_result_success(hv_status));
+				cur = 0;
+			}
+		}
+
+	}
+
+out:
+	local_irq_restore(flags);
+}
+
+/*
  * hv_mark_gpa_visibility - Set pages visible to host via hvcall.
  *
  * In Isolation VM, all guest memory is encrypted from host and guest
@@ -465,9 +677,9 @@ static int hv_mark_gpa_visibility(u16 count, const u64 pfn[],
 			   enum hv_mem_host_visibility visibility)
 {
 	struct hv_gpa_range_for_visibility *input;
-	u16 pages_processed;
 	u64 hv_status;
 	unsigned long flags;
+	int ret;
 
 	/* no-op if partition isolation is not enabled */
 	if (!hv_is_isolation_supported())
@@ -479,6 +691,13 @@ static int hv_mark_gpa_visibility(u16 count, const u64 pfn[],
 		return -EINVAL;
 	}
 
+	if (visibility == VMBUS_PAGE_NOT_VISIBLE)
+		ret = hv_list_enc_remove(pfn, count);
+	else
+		ret = hv_list_enc_add(pfn, count);
+	if (ret)
+		return ret;
+
 	local_irq_save(flags);
 	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
 
@@ -494,13 +713,23 @@ static int hv_mark_gpa_visibility(u16 count, const u64 pfn[],
 	memcpy((void *)input->gpa_page_list, pfn, count * sizeof(*pfn));
 	hv_status = hv_do_rep_hypercall(
 			HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY, count,
-			0, input, &pages_processed);
+			0, input, NULL);
 	local_irq_restore(flags);
 
 	if (hv_result_success(hv_status))
 		return 0;
+
+	if (visibility == VMBUS_PAGE_NOT_VISIBLE)
+		ret = hv_list_enc_add(pfn, count);
 	else
-		return -EFAULT;
+		ret = hv_list_enc_remove(pfn, count);
+	/*
+	 * There's no good way to recover from -ENOMEM here, the accounting is
+	 * wrong either way.
+	 */
+	WARN_ON_ONCE(ret);
+
+	return -EFAULT;
 }
 
 /*
@@ -662,9 +891,11 @@ void __init hv_vtom_init(void)
 	x86_platform.guest.enc_tlb_flush_required = hv_vtom_tlb_flush_required;
 	x86_platform.guest.enc_status_change_prepare = hv_vtom_clear_present;
 	x86_platform.guest.enc_status_change_finish = hv_vtom_set_host_visibility;
+	x86_platform.guest.enc_kexec_begin = hv_vtom_kexec_begin;
+	x86_platform.guest.enc_kexec_finish = hv_vtom_kexec_finish;
 
 	/* Set WB as the default cache mode. */
-	mtrr_overwrite_state(NULL, 0, MTRR_TYPE_WRBACK);
+	guest_force_mtrr_state(NULL, 0, MTRR_TYPE_WRBACK);
 }
 
 #endif /* defined(CONFIG_AMD_MEM_ENCRYPT) || defined(CONFIG_INTEL_TDX_GUEST) */
diff --git a/arch/x86/hyperv/mmu.c b/arch/x86/hyperv/mmu.c
index 1cc113200ff5..cfcb60468b01 100644
--- a/arch/x86/hyperv/mmu.c
+++ b/arch/x86/hyperv/mmu.c
@@ -1,6 +1,5 @@
 #define pr_fmt(fmt)  "Hyper-V: " fmt
 
-#include <linux/hyperv.h>
 #include <linux/log2.h>
 #include <linux/slab.h>
 #include <linux/types.h>
@@ -206,6 +205,10 @@ static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
 	/*
 	 * We can flush not more than max_gvas with one hypercall. Flush the
 	 * whole address space if we were asked to do more.
+	 *
+	 * For these hypercalls, Hyper-V treats the valid_bank_mask field
+	 * of flush->hv_vp_set as part of the fixed size input header.
+	 * So the variable input header size is equal to nr_bank.
 	 */
 	max_gvas =
 		(PAGE_SIZE - sizeof(*flush) - nr_bank *
@@ -240,5 +243,4 @@ void hyperv_setup_mmu_ops(void)
 
 	pr_info("Using hypercall for remote TLB flush\n");
 	pv_ops.mmu.flush_tlb_multi = hyperv_flush_tlb_multi;
-	pv_ops.mmu.tlb_remove_table = tlb_remove_table;
 }
diff --git a/arch/x86/hyperv/mshv-asm-offsets.c b/arch/x86/hyperv/mshv-asm-offsets.c
new file mode 100644
index 000000000000..882c1db6df16
--- /dev/null
+++ b/arch/x86/hyperv/mshv-asm-offsets.c
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Generate definitions needed by assembly language modules.
+ * This code generates raw asm output which is post-processed to extract
+ * and format the required data.
+ *
+ * Copyright (c) 2025, Microsoft Corporation.
+ *
+ * Author:
+ *   Naman Jain <namjain@microsoft.com>
+ */
+#define COMPILE_OFFSETS
+
+#include <linux/kbuild.h>
+#include <asm/mshyperv.h>
+
+static void __used common(void)
+{
+	if (IS_ENABLED(CONFIG_HYPERV_VTL_MODE)) {
+		OFFSET(MSHV_VTL_CPU_CONTEXT_rax, mshv_vtl_cpu_context, rax);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_rcx, mshv_vtl_cpu_context, rcx);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_rdx, mshv_vtl_cpu_context, rdx);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_rbx, mshv_vtl_cpu_context, rbx);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_rbp, mshv_vtl_cpu_context, rbp);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_rsi, mshv_vtl_cpu_context, rsi);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_rdi, mshv_vtl_cpu_context, rdi);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_r8,  mshv_vtl_cpu_context, r8);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_r9,  mshv_vtl_cpu_context, r9);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_r10, mshv_vtl_cpu_context, r10);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_r11, mshv_vtl_cpu_context, r11);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_r12, mshv_vtl_cpu_context, r12);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_r13, mshv_vtl_cpu_context, r13);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_r14, mshv_vtl_cpu_context, r14);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_r15, mshv_vtl_cpu_context, r15);
+		OFFSET(MSHV_VTL_CPU_CONTEXT_cr2, mshv_vtl_cpu_context, cr2);
+	}
+}
diff --git a/arch/x86/hyperv/mshv_vtl_asm.S b/arch/x86/hyperv/mshv_vtl_asm.S
new file mode 100644
index 000000000000..f595eefad9ab
--- /dev/null
+++ b/arch/x86/hyperv/mshv_vtl_asm.S
@@ -0,0 +1,116 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Assembly level code for mshv_vtl VTL transition
+ *
+ * Copyright (c) 2025, Microsoft Corporation.
+ *
+ * Author:
+ *   Naman Jain <namjain@microsoft.com>
+ */
+
+#include <linux/linkage.h>
+#include <linux/static_call_types.h>
+#include <asm/asm.h>
+#include <asm/asm-offsets.h>
+#include <asm/frame.h>
+#include "mshv-asm-offsets.h"
+
+	.text
+	.section .noinstr.text, "ax"
+/*
+ * void __mshv_vtl_return_call(struct mshv_vtl_cpu_context *vtl0)
+ *
+ * This function is used to context switch between different Virtual Trust Levels.
+ * It is marked as 'noinstr' to prevent against instrumentation and debugging facilities.
+ * NMIs aren't a problem because the NMI handler saves/restores CR2 specifically to guard
+ * against #PFs in NMI context clobbering the guest state.
+ */
+SYM_FUNC_START(__mshv_vtl_return_call)
+	/* Push callee save registers */
+	pushq %rbp
+	mov %rsp, %rbp
+	pushq %r12
+	pushq %r13
+	pushq %r14
+	pushq %r15
+	pushq %rbx
+
+	/* register switch to VTL0 clobbers all registers except rax/rcx */
+	mov %_ASM_ARG1, %rax
+
+	/* grab rbx/rbp/rsi/rdi/r8-r15 */
+	mov MSHV_VTL_CPU_CONTEXT_rbx(%rax), %rbx
+	mov MSHV_VTL_CPU_CONTEXT_rbp(%rax), %rbp
+	mov MSHV_VTL_CPU_CONTEXT_rsi(%rax), %rsi
+	mov MSHV_VTL_CPU_CONTEXT_rdi(%rax), %rdi
+	mov MSHV_VTL_CPU_CONTEXT_r8(%rax), %r8
+	mov MSHV_VTL_CPU_CONTEXT_r9(%rax), %r9
+	mov MSHV_VTL_CPU_CONTEXT_r10(%rax), %r10
+	mov MSHV_VTL_CPU_CONTEXT_r11(%rax), %r11
+	mov MSHV_VTL_CPU_CONTEXT_r12(%rax), %r12
+	mov MSHV_VTL_CPU_CONTEXT_r13(%rax), %r13
+	mov MSHV_VTL_CPU_CONTEXT_r14(%rax), %r14
+	mov MSHV_VTL_CPU_CONTEXT_r15(%rax), %r15
+
+	mov MSHV_VTL_CPU_CONTEXT_cr2(%rax), %rdx
+	mov %rdx, %cr2
+	mov MSHV_VTL_CPU_CONTEXT_rdx(%rax), %rdx
+
+	/* stash host registers on stack */
+	pushq %rax
+	pushq %rcx
+
+	xor %ecx, %ecx
+
+	/* make a hypercall to switch VTL */
+	call STATIC_CALL_TRAMP_STR(__mshv_vtl_return_hypercall)
+
+	/* stash guest registers on stack, restore saved host copies */
+	pushq %rax
+	pushq %rcx
+	mov 16(%rsp), %rcx
+	mov 24(%rsp), %rax
+
+	mov %rdx, MSHV_VTL_CPU_CONTEXT_rdx(%rax)
+	mov %cr2, %rdx
+	mov %rdx, MSHV_VTL_CPU_CONTEXT_cr2(%rax)
+	pop MSHV_VTL_CPU_CONTEXT_rcx(%rax)
+	pop MSHV_VTL_CPU_CONTEXT_rax(%rax)
+	add $16, %rsp
+
+	/* save rbx/rbp/rsi/rdi/r8-r15 */
+	mov %rbx, MSHV_VTL_CPU_CONTEXT_rbx(%rax)
+	mov %rbp, MSHV_VTL_CPU_CONTEXT_rbp(%rax)
+	mov %rsi, MSHV_VTL_CPU_CONTEXT_rsi(%rax)
+	mov %rdi, MSHV_VTL_CPU_CONTEXT_rdi(%rax)
+	mov %r8,  MSHV_VTL_CPU_CONTEXT_r8(%rax)
+	mov %r9,  MSHV_VTL_CPU_CONTEXT_r9(%rax)
+	mov %r10, MSHV_VTL_CPU_CONTEXT_r10(%rax)
+	mov %r11, MSHV_VTL_CPU_CONTEXT_r11(%rax)
+	mov %r12, MSHV_VTL_CPU_CONTEXT_r12(%rax)
+	mov %r13, MSHV_VTL_CPU_CONTEXT_r13(%rax)
+	mov %r14, MSHV_VTL_CPU_CONTEXT_r14(%rax)
+	mov %r15, MSHV_VTL_CPU_CONTEXT_r15(%rax)
+
+	/* pop callee-save registers r12-r15, rbx */
+	pop %rbx
+	pop %r15
+	pop %r14
+	pop %r13
+	pop %r12
+
+	pop %rbp
+	RET
+SYM_FUNC_END(__mshv_vtl_return_call)
+/*
+ * Make sure that static_call_key symbol: __SCK____mshv_vtl_return_hypercall is accessible here.
+ * Below code is inspired from __ADDRESSABLE(sym) macro. Symbol name is kept simple, to avoid
+ * naming it something like "__UNIQUE_ID_addressable___SCK____mshv_vtl_return_hypercall_662.0"
+ * which would otherwise have been generated by the macro.
+ */
+	.section	.discard.addressable,"aw"
+	.align 8
+	.type	mshv_vtl_return_sym, @object
+	.size	mshv_vtl_return_sym, 8
+mshv_vtl_return_sym:
+	.quad	__SCK____mshv_vtl_return_hypercall
diff --git a/arch/x86/hyperv/nested.c b/arch/x86/hyperv/nested.c
index 9dc259fa322e..8ccbb7c4fc27 100644
--- a/arch/x86/hyperv/nested.c
+++ b/arch/x86/hyperv/nested.c
@@ -11,7 +11,8 @@
 
 
 #include <linux/types.h>
-#include <asm/hyperv-tlfs.h>
+#include <linux/export.h>
+#include <hyperv/hvhdk.h>
 #include <asm/mshyperv.h>
 #include <asm/tlbflush.h>
 
diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild
index 6c23d1661b17..4566000e15c4 100644
--- a/arch/x86/include/asm/Kbuild
+++ b/arch/x86/include/asm/Kbuild
@@ -8,7 +8,9 @@ generated-y += syscalls_x32.h
 generated-y += unistd_32_ia32.h
 generated-y += unistd_64_x32.h
 generated-y += xen-hypercalls.h
+generated-y += cpufeaturemasks.h
 
 generic-y += early_ioremap.h
+generic-y += fprobe.h
 generic-y += mcs_spinlock.h
 generic-y += mmzone.h
diff --git a/arch/x86/include/asm/acpi.h b/arch/x86/include/asm/acpi.h
index 5ab1a4598d00..a03aa6f999d1 100644
--- a/arch/x86/include/asm/acpi.h
+++ b/arch/x86/include/asm/acpi.h
@@ -158,13 +158,13 @@ static inline bool acpi_has_cpu_in_madt(void)
 }
 
 #define ACPI_HAVE_ARCH_SET_ROOT_POINTER
-static inline void acpi_arch_set_root_pointer(u64 addr)
+static __always_inline void acpi_arch_set_root_pointer(u64 addr)
 {
 	x86_init.acpi.set_root_pointer(addr);
 }
 
 #define ACPI_HAVE_ARCH_GET_ROOT_POINTER
-static inline u64 acpi_arch_get_root_pointer(void)
+static __always_inline u64 acpi_arch_get_root_pointer(void)
 {
 	return x86_init.acpi.get_root_pointer();
 }
diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h
index 1dd14381bcb6..fab11192c60a 100644
--- a/arch/x86/include/asm/acrn.h
+++ b/arch/x86/include/asm/acrn.h
@@ -25,7 +25,7 @@ void acrn_remove_intr_handler(void);
 static inline u32 acrn_cpuid_base(void)
 {
 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
-		return hypervisor_cpuid_base("ACRNACRNACRN", 0);
+		return cpuid_base_hypervisor("ACRNACRNACRN", 0);
 
 	return 0;
 }
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index ca9ae606aab9..03364510d5fe 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -4,7 +4,9 @@
 
 #include <linux/types.h>
 #include <linux/stringify.h>
+#include <linux/objtool.h>
 #include <asm/asm.h>
+#include <asm/bug.h>
 
 #define ALT_FLAGS_SHIFT		16
 
@@ -14,7 +16,7 @@
 #define ALT_DIRECT_CALL(feature) ((ALT_FLAG_DIRECT_CALL << ALT_FLAGS_SHIFT) | (feature))
 #define ALT_CALL_ALWAYS		ALT_DIRECT_CALL(X86_FEATURE_ALWAYS)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/stddef.h>
 
@@ -47,7 +49,7 @@
 		".popsection\n"				\
 		"671:"
 
-#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
+#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock "
 
 #else /* ! CONFIG_SMP */
 #define LOCK_PREFIX_HERE ""
@@ -55,16 +57,6 @@
 #endif
 
 /*
- * objtool annotation to ignore the alternatives and only consider the original
- * instruction(s).
- */
-#define ANNOTATE_IGNORE_ALTERNATIVE				\
-	"999:\n\t"						\
-	".pushsection .discard.ignore_alts\n\t"			\
-	".long 999b\n\t"					\
-	".popsection\n\t"
-
-/*
  * The patching flags are part of the upper bits of the @ft_flags parameter when
  * specifying them. The split is currently like this:
  *
@@ -91,6 +83,12 @@ struct alt_instr {
 
 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
 
+extern s32 __retpoline_sites[], __retpoline_sites_end[];
+extern s32 __return_sites[],	__return_sites_end[];
+extern s32 __cfi_sites[],	__cfi_sites_end[];
+extern s32 __ibt_endbr_seal[],	__ibt_endbr_seal_end[];
+extern s32 __smp_locks[],	__smp_locks_end[];
+
 /*
  * Debug flag that can be tested to see whether alternative
  * instructions were patched in already:
@@ -109,7 +107,6 @@ struct module;
 
 struct callthunk_sites {
 	s32				*call_start, *call_end;
-	struct alt_instr		*alt_start, *alt_end;
 };
 
 #ifdef CONFIG_CALL_THUNKS
@@ -134,6 +131,37 @@ static __always_inline int x86_call_depth_emit_accounting(u8 **pprog,
 }
 #endif
 
+#ifdef CONFIG_MITIGATION_ITS
+extern void its_init_mod(struct module *mod);
+extern void its_fini_mod(struct module *mod);
+extern void its_free_mod(struct module *mod);
+extern u8 *its_static_thunk(int reg);
+#else /* CONFIG_MITIGATION_ITS */
+static inline void its_init_mod(struct module *mod) { }
+static inline void its_fini_mod(struct module *mod) { }
+static inline void its_free_mod(struct module *mod) { }
+static inline u8 *its_static_thunk(int reg)
+{
+	WARN_ONCE(1, "ITS not compiled in");
+
+	return NULL;
+}
+#endif
+
+#if defined(CONFIG_MITIGATION_RETHUNK) && defined(CONFIG_OBJTOOL)
+extern bool cpu_wants_rethunk(void);
+extern bool cpu_wants_rethunk_at(void *addr);
+#else
+static __always_inline bool cpu_wants_rethunk(void)
+{
+	return false;
+}
+static __always_inline bool cpu_wants_rethunk_at(void *addr)
+{
+	return false;
+}
+#endif
+
 #ifdef CONFIG_SMP
 extern void alternatives_smp_module_add(struct module *mod, char *name,
 					void *locks, void *locks_end,
@@ -169,7 +197,8 @@ static inline int alternatives_text_reserved(void *start, void *end)
 	"773:\n"
 
 #define ALTINSTR_ENTRY(ft_flags)					      \
-	".pushsection .altinstructions,\"a\"\n"				      \
+	".pushsection .altinstructions, \"aM\", @progbits, "		      \
+		      __stringify(ALT_INSTR_SIZE) "\n"			      \
 	" .long 771b - .\n"				/* label           */ \
 	" .long 774f - .\n"				/* new instruction */ \
 	" .4byte " __stringify(ft_flags) "\n"		/* feature + flags */ \
@@ -179,6 +208,7 @@ static inline int alternatives_text_reserved(void *start, void *end)
 
 #define ALTINSTR_REPLACEMENT(newinstr)		/* replacement */	\
 	".pushsection .altinstr_replacement, \"ax\"\n"			\
+	ANNOTATE_DATA_SPECIAL "\n"					\
 	"# ALT: replacement\n"						\
 	"774:\n\t" newinstr "\n775:\n"					\
 	".popsection\n"
@@ -246,10 +276,12 @@ static inline int alternatives_text_reserved(void *start, void *end)
  * references: i.e., if used for a function, it would add the PLT
  * suffix.
  */
-#define alternative_call(oldfunc, newfunc, ft_flags, output, input...)			\
+#define alternative_call(oldfunc, newfunc, ft_flags, output, input, clobbers...)	\
 	asm_inline volatile(ALTERNATIVE("call %c[old]", "call %c[new]", ft_flags)	\
 		: ALT_OUTPUT_SP(output)							\
-		: [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
+		: [old] "i" (oldfunc), [new] "i" (newfunc)				\
+		  COMMA(input)								\
+		: clobbers)
 
 /*
  * Like alternative_call, but there are two features and respective functions.
@@ -258,24 +290,14 @@ static inline int alternatives_text_reserved(void *start, void *end)
  * Otherwise, old function is used.
  */
 #define alternative_call_2(oldfunc, newfunc1, ft_flags1, newfunc2, ft_flags2,		\
-			   output, input...)						\
+			   output, input, clobbers...)					\
 	asm_inline volatile(ALTERNATIVE_2("call %c[old]", "call %c[new1]", ft_flags1,	\
 		"call %c[new2]", ft_flags2)						\
 		: ALT_OUTPUT_SP(output)							\
 		: [old] "i" (oldfunc), [new1] "i" (newfunc1),				\
-		  [new2] "i" (newfunc2), ## input)
-
-/*
- * use this macro(s) if you need more than one output parameter
- * in alternative_io
- */
-#define ASM_OUTPUT2(a...) a
-
-/*
- * use this macro if you need clobbers but no inputs in
- * alternative_{input,io,call}()
- */
-#define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
+		  [new2] "i" (newfunc2)							\
+		  COMMA(input)								\
+		: clobbers)
 
 #define ALT_OUTPUT_SP(...) ASM_CALL_CONSTRAINT, ## __VA_ARGS__
 
@@ -295,7 +317,7 @@ static inline int alternatives_text_reserved(void *start, void *end)
 void BUG_func(void);
 void nop_func(void);
 
-#else /* __ASSEMBLY__ */
+#else /* __ASSEMBLER__ */
 
 #ifdef CONFIG_SMP
 	.macro LOCK_PREFIX
@@ -311,17 +333,6 @@ void nop_func(void);
 #endif
 
 /*
- * objtool annotation to ignore the alternatives and only consider the original
- * instruction(s).
- */
-.macro ANNOTATE_IGNORE_ALTERNATIVE
-	.Lannotate_\@:
-	.pushsection .discard.ignore_alts
-	.long .Lannotate_\@
-	.popsection
-.endm
-
-/*
  * Issue one struct alt_instr descriptor entry (need to put it into
  * the section .altinstructions, see below). This entry contains
  * enough information for the alternatives patching code to patch an
@@ -351,11 +362,12 @@ void nop_func(void);
 741:									\
 	.skip -(((744f-743f)-(741b-740b)) > 0) * ((744f-743f)-(741b-740b)),0x90	;\
 742:									\
-	.pushsection .altinstructions,"a" ;				\
+	.pushsection .altinstructions, "aM", @progbits, ALT_INSTR_SIZE ;\
 	altinstr_entry 740b,743f,flag,742b-740b,744f-743f ;		\
 	.popsection ;							\
 	.pushsection .altinstr_replacement,"ax"	;			\
 743:									\
+	ANNOTATE_DATA_SPECIAL ;						\
 	newinst	;							\
 744:									\
 	.popsection ;
@@ -364,11 +376,6 @@ void nop_func(void);
 	__ALTERNATIVE(\oldinstr, \newinstr, \ft_flags)
 .endm
 
-#define old_len			141b-140b
-#define new_len1		144f-143f
-#define new_len2		145f-144f
-#define new_len3		146f-145f
-
 /*
  * Same as ALTERNATIVE macro above but for two alternatives. If CPU
  * has @feature1, it replaces @oldinstr with @newinstr1. If CPU has
@@ -389,6 +396,6 @@ void nop_func(void);
 	ALTERNATIVE_2 oldinstr, newinstr_no, X86_FEATURE_ALWAYS,	\
 	newinstr_yes, ft_flags
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ASM_X86_ALTERNATIVE_H */
diff --git a/arch/x86/include/asm/amd_hsmp.h b/arch/x86/include/asm/amd/hsmp.h
index 03c2ce3edaf5..2137f62853ed 100644
--- a/arch/x86/include/asm/amd_hsmp.h
+++ b/arch/x86/include/asm/amd/hsmp.h
@@ -1,5 +1,4 @@
 /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-
 #ifndef _ASM_X86_AMD_HSMP_H_
 #define _ASM_X86_AMD_HSMP_H_
 
@@ -13,4 +12,5 @@ static inline int hsmp_send_message(struct hsmp_message *msg)
 	return -ENODEV;
 }
 #endif
+
 #endif /*_ASM_X86_AMD_HSMP_H_*/
diff --git a/arch/x86/include/asm/amd-ibs.h b/arch/x86/include/asm/amd/ibs.h
index cb2a5e113daa..3ee5903982c2 100644
--- a/arch/x86/include/asm/amd-ibs.h
+++ b/arch/x86/include/asm/amd/ibs.h
@@ -1,4 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_AMD_IBS_H
+#define _ASM_X86_AMD_IBS_H
+
 /*
  * From PPR Vol 1 for AMD Family 19h Model 01h B1
  * 55898 Rev 0.35 - Feb 5, 2021
@@ -64,7 +67,8 @@ union ibs_op_ctl {
 			opmaxcnt_ext:7,	/* 20-26: upper 7 bits of periodic op maximum count */
 			reserved0:5,	/* 27-31: reserved */
 			opcurcnt:27,	/* 32-58: periodic op counter current count */
-			reserved1:5;	/* 59-63: reserved */
+			ldlat_thrsh:4,	/* 59-62: Load Latency threshold */
+			ldlat_en:1;	/* 63: Load Latency enabled */
 	};
 };
 
@@ -150,3 +154,5 @@ struct perf_ibs_data {
 	};
 	u64		regs[MSR_AMD64_IBS_REG_COUNT_MAX];
 };
+
+#endif /* _ASM_X86_AMD_IBS_H */
diff --git a/arch/x86/include/asm/amd_nb.h b/arch/x86/include/asm/amd/nb.h
index 6f3b6aef47ba..ddb5108cf46c 100644
--- a/arch/x86/include/asm/amd_nb.h
+++ b/arch/x86/include/asm/amd/nb.h
@@ -4,7 +4,7 @@
 
 #include <linux/ioport.h>
 #include <linux/pci.h>
-#include <linux/refcount.h>
+#include <asm/amd/node.h>
 
 struct amd_nb_bus_dev_range {
 	u8 bus;
@@ -21,49 +21,15 @@ extern int amd_numa_init(void);
 extern int amd_get_subcaches(int);
 extern int amd_set_subcaches(int, unsigned long);
 
-int __must_check amd_smn_read(u16 node, u32 address, u32 *value);
-int __must_check amd_smn_write(u16 node, u32 address, u32 value);
-
 struct amd_l3_cache {
 	unsigned indices;
 	u8	 subcaches[4];
 };
 
-struct threshold_block {
-	unsigned int	 block;			/* Number within bank */
-	unsigned int	 bank;			/* MCA bank the block belongs to */
-	unsigned int	 cpu;			/* CPU which controls MCA bank */
-	u32		 address;		/* MSR address for the block */
-	u16		 interrupt_enable;	/* Enable/Disable APIC interrupt */
-	bool		 interrupt_capable;	/* Bank can generate an interrupt. */
-
-	u16		 threshold_limit;	/*
-						 * Value upon which threshold
-						 * interrupt is generated.
-						 */
-
-	struct kobject	 kobj;			/* sysfs object */
-	struct list_head miscj;			/*
-						 * List of threshold blocks
-						 * within a bank.
-						 */
-};
-
-struct threshold_bank {
-	struct kobject		*kobj;
-	struct threshold_block	*blocks;
-
-	/* initialized to the number of CPUs on the node sharing this bank */
-	refcount_t		cpus;
-	unsigned int		shared;
-};
-
 struct amd_northbridge {
-	struct pci_dev *root;
 	struct pci_dev *misc;
 	struct pci_dev *link;
 	struct amd_l3_cache l3_cache;
-	struct threshold_bank *bank4;
 };
 
 struct amd_northbridge_info {
@@ -82,23 +48,6 @@ u16 amd_nb_num(void);
 bool amd_nb_has_feature(unsigned int feature);
 struct amd_northbridge *node_to_amd_nb(int node);
 
-static inline u16 amd_pci_dev_to_node_id(struct pci_dev *pdev)
-{
-	struct pci_dev *misc;
-	int i;
-
-	for (i = 0; i != amd_nb_num(); i++) {
-		misc = node_to_amd_nb(i)->misc;
-
-		if (pci_domain_nr(misc->bus) == pci_domain_nr(pdev->bus) &&
-		    PCI_SLOT(misc->devfn) == PCI_SLOT(pdev->devfn))
-			return i;
-	}
-
-	WARN(1, "Unable to find AMD Northbridge id for %s\n", pci_name(pdev));
-	return 0;
-}
-
 static inline bool amd_gart_present(void)
 {
 	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
@@ -116,7 +65,10 @@ static inline bool amd_gart_present(void)
 
 #define amd_nb_num(x)		0
 #define amd_nb_has_feature(x)	false
-#define node_to_amd_nb(x)	NULL
+static inline struct amd_northbridge *node_to_amd_nb(int node)
+{
+	return NULL;
+}
 #define amd_gart_present(x)	false
 
 #endif
diff --git a/arch/x86/include/asm/amd/node.h b/arch/x86/include/asm/amd/node.h
new file mode 100644
index 000000000000..a672b8765fa8
--- /dev/null
+++ b/arch/x86/include/asm/amd/node.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * AMD Node helper functions and common defines
+ *
+ * Copyright (c) 2024, Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Author: Yazen Ghannam <Yazen.Ghannam@amd.com>
+ *
+ * Note:
+ * Items in this file may only be used in a single place.
+ * However, it's prudent to keep all AMD Node functionality
+ * in a unified place rather than spreading throughout the
+ * kernel.
+ */
+
+#ifndef _ASM_X86_AMD_NODE_H_
+#define _ASM_X86_AMD_NODE_H_
+
+#include <linux/pci.h>
+
+#define MAX_AMD_NUM_NODES	8
+#define AMD_NODE0_PCI_SLOT	0x18
+
+struct pci_dev *amd_node_get_func(u16 node, u8 func);
+
+static inline u16 amd_num_nodes(void)
+{
+	return topology_amd_nodes_per_pkg() * topology_max_packages();
+}
+
+#ifdef CONFIG_AMD_NODE
+int __must_check amd_smn_read(u16 node, u32 address, u32 *value);
+int __must_check amd_smn_write(u16 node, u32 address, u32 value);
+
+/* Should only be used by the HSMP driver. */
+int __must_check amd_smn_hsmp_rdwr(u16 node, u32 address, u32 *value, bool write);
+#else
+static inline int __must_check amd_smn_read(u16 node, u32 address, u32 *value) { return -ENODEV; }
+static inline int __must_check amd_smn_write(u16 node, u32 address, u32 value) { return -ENODEV; }
+
+static inline int __must_check amd_smn_hsmp_rdwr(u16 node, u32 address, u32 *value, bool write)
+{
+	return -ENODEV;
+}
+#endif /* CONFIG_AMD_NODE */
+
+/* helper for use with read_poll_timeout */
+static inline int smn_read_register(u32 reg)
+{
+	int data, rc;
+
+	rc = amd_smn_read(0, reg, &data);
+	if (rc)
+		return rc;
+
+	return data;
+}
+#endif /*_ASM_X86_AMD_NODE_H_*/
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index f21ff1932699..a26e66d66444 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -99,8 +99,8 @@ static inline void native_apic_mem_write(u32 reg, u32 v)
 	volatile u32 *addr = (volatile u32 *)(APIC_BASE + reg);
 
 	alternative_io("movl %0, %1", "xchgl %0, %1", X86_BUG_11AP,
-		       ASM_OUTPUT2("=r" (v), "=m" (*addr)),
-		       ASM_OUTPUT2("0" (v), "m" (*addr)));
+		       ASM_OUTPUT("=r" (v), "=m" (*addr)),
+		       ASM_INPUT("0" (v), "m" (*addr)));
 }
 
 static inline u32 native_apic_mem_read(u32 reg)
@@ -120,7 +120,7 @@ static inline bool apic_is_x2apic_enabled(void)
 {
 	u64 msr;
 
-	if (rdmsrl_safe(MSR_IA32_APICBASE, &msr))
+	if (rdmsrq_safe(MSR_IA32_APICBASE, &msr))
 		return false;
 	return msr & X2APIC_ENABLE;
 }
@@ -209,12 +209,12 @@ static inline void native_apic_msr_write(u32 reg, u32 v)
 	    reg == APIC_LVR)
 		return;
 
-	wrmsr(APIC_BASE_MSR + (reg >> 4), v, 0);
+	wrmsrq(APIC_BASE_MSR + (reg >> 4), v);
 }
 
 static inline void native_apic_msr_eoi(void)
 {
-	__wrmsr(APIC_BASE_MSR + (APIC_EOI >> 4), APIC_EOI_ACK, 0);
+	native_wrmsrq(APIC_BASE_MSR + (APIC_EOI >> 4), APIC_EOI_ACK);
 }
 
 static inline u32 native_apic_msr_read(u32 reg)
@@ -224,20 +224,20 @@ static inline u32 native_apic_msr_read(u32 reg)
 	if (reg == APIC_DFR)
 		return -1;
 
-	rdmsrl(APIC_BASE_MSR + (reg >> 4), msr);
+	rdmsrq(APIC_BASE_MSR + (reg >> 4), msr);
 	return (u32)msr;
 }
 
 static inline void native_x2apic_icr_write(u32 low, u32 id)
 {
-	wrmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), ((__u64) id) << 32 | low);
+	wrmsrq(APIC_BASE_MSR + (APIC_ICR >> 4), ((__u64) id) << 32 | low);
 }
 
 static inline u64 native_x2apic_icr_read(void)
 {
 	unsigned long val;
 
-	rdmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), val);
+	rdmsrq(APIC_BASE_MSR + (APIC_ICR >> 4), val);
 	return val;
 }
 
@@ -305,6 +305,8 @@ struct apic {
 
 	/* Probe, setup and smpboot functions */
 	int	(*probe)(void);
+	void	(*setup)(void);
+	void	(*teardown)(void);
 	int	(*acpi_madt_oem_check)(char *oem_id, char *oem_table_id);
 
 	void	(*init_apic_ldr)(void);
@@ -313,9 +315,11 @@ struct apic {
 	u32	(*get_apic_id)(u32 id);
 
 	/* wakeup_secondary_cpu */
-	int	(*wakeup_secondary_cpu)(u32 apicid, unsigned long start_eip);
+	int	(*wakeup_secondary_cpu)(u32 apicid, unsigned long start_eip, unsigned int cpu);
 	/* wakeup secondary CPU using 64-bit wakeup point */
-	int	(*wakeup_secondary_cpu_64)(u32 apicid, unsigned long start_eip);
+	int	(*wakeup_secondary_cpu_64)(u32 apicid, unsigned long start_eip, unsigned int cpu);
+
+	void	(*update_vector)(unsigned int cpu, unsigned int vector, bool set);
 
 	char	*name;
 };
@@ -333,8 +337,8 @@ struct apic_override {
 	void	(*send_IPI_self)(int vector);
 	u64	(*icr_read)(void);
 	void	(*icr_write)(u32 low, u32 high);
-	int	(*wakeup_secondary_cpu)(u32 apicid, unsigned long start_eip);
-	int	(*wakeup_secondary_cpu_64)(u32 apicid, unsigned long start_eip);
+	int	(*wakeup_secondary_cpu)(u32 apicid, unsigned long start_eip, unsigned int cpu);
+	int	(*wakeup_secondary_cpu_64)(u32 apicid, unsigned long start_eip, unsigned int cpu);
 };
 
 /*
@@ -470,6 +474,12 @@ static __always_inline bool apic_id_valid(u32 apic_id)
 	return apic_id <= apic->max_apic_id;
 }
 
+static __always_inline void apic_update_vector(unsigned int cpu, unsigned int vector, bool set)
+{
+	if (apic->update_vector)
+		apic->update_vector(cpu, vector, set);
+}
+
 #else /* CONFIG_X86_LOCAL_APIC */
 
 static inline u32 apic_read(u32 reg) { return 0; }
@@ -481,6 +491,7 @@ static inline void apic_wait_icr_idle(void) { }
 static inline u32 safe_apic_wait_icr_idle(void) { return 0; }
 static inline void apic_native_eoi(void) { WARN_ON_ONCE(1); }
 static inline void apic_setup_apic_calls(void) { }
+static inline void apic_update_vector(unsigned int cpu, unsigned int vector, bool set) { }
 
 #define apic_update_callback(_callback, _fn) do { } while (0)
 
@@ -488,11 +499,14 @@ static inline void apic_setup_apic_calls(void) { }
 
 extern void apic_ack_irq(struct irq_data *data);
 
+#define APIC_VECTOR_TO_BIT_NUMBER(v) ((unsigned int)(v) % 32)
+#define APIC_VECTOR_TO_REG_OFFSET(v) ((unsigned int)(v) / 32 * 0x10)
+
 static inline bool lapic_vector_set_in_irr(unsigned int vector)
 {
-	u32 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
+	u32 irr = apic_read(APIC_IRR + APIC_VECTOR_TO_REG_OFFSET(vector));
 
-	return !!(irr & (1U << (vector % 32)));
+	return !!(irr & (1U << APIC_VECTOR_TO_BIT_NUMBER(vector)));
 }
 
 static inline bool is_vector_pending(unsigned int vector)
@@ -500,6 +514,65 @@ static inline bool is_vector_pending(unsigned int vector)
 	return lapic_vector_set_in_irr(vector) || pi_pending_this_cpu(vector);
 }
 
+#define MAX_APIC_VECTOR			256
+#define APIC_VECTORS_PER_REG		32
+
+/*
+ * Vector states are maintained by APIC in 32-bit registers that are
+ * 16 bytes aligned. The status of each vector is kept in a single
+ * bit.
+ */
+static inline int apic_find_highest_vector(void *bitmap)
+{
+	int vec;
+	u32 *reg;
+
+	for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; vec >= 0; vec -= APIC_VECTORS_PER_REG) {
+		reg = bitmap + APIC_VECTOR_TO_REG_OFFSET(vec);
+		if (*reg)
+			return __fls(*reg) + vec;
+	}
+
+	return -1;
+}
+
+static inline u32 apic_get_reg(void *regs, int reg)
+{
+	return *((u32 *) (regs + reg));
+}
+
+static inline void apic_set_reg(void *regs, int reg, u32 val)
+{
+	*((u32 *) (regs + reg)) = val;
+}
+
+static __always_inline u64 apic_get_reg64(void *regs, int reg)
+{
+	BUILD_BUG_ON(reg != APIC_ICR);
+	return *((u64 *) (regs + reg));
+}
+
+static __always_inline void apic_set_reg64(void *regs, int reg, u64 val)
+{
+	BUILD_BUG_ON(reg != APIC_ICR);
+	*((u64 *) (regs + reg)) = val;
+}
+
+static inline void apic_clear_vector(int vec, void *bitmap)
+{
+	clear_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec));
+}
+
+static inline void apic_set_vector(int vec, void *bitmap)
+{
+	set_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec));
+}
+
+static inline int apic_test_vector(int vec, void *bitmap)
+{
+	return test_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec));
+}
+
 /*
  * Warm reset vector position:
  */
diff --git a/arch/x86/include/asm/apicdef.h b/arch/x86/include/asm/apicdef.h
index 094106b6a538..be39a543fbe5 100644
--- a/arch/x86/include/asm/apicdef.h
+++ b/arch/x86/include/asm/apicdef.h
@@ -135,6 +135,8 @@
 #define		APIC_TDR_DIV_128	0xA
 #define	APIC_EFEAT	0x400
 #define	APIC_ECTRL	0x410
+#define APIC_SEOI	0x420
+#define APIC_IER	0x480
 #define APIC_EILVTn(n)	(0x500 + 0x10 * n)
 #define		APIC_EILVT_NR_AMD_K8	1	/* # of extended interrupts */
 #define		APIC_EILVT_NR_AMD_10H	4
diff --git a/arch/x86/include/asm/arch_hweight.h b/arch/x86/include/asm/arch_hweight.h
index ba88edd0d58b..b5982b94bdba 100644
--- a/arch/x86/include/asm/arch_hweight.h
+++ b/arch/x86/include/asm/arch_hweight.h
@@ -16,9 +16,10 @@ static __always_inline unsigned int __arch_hweight32(unsigned int w)
 {
 	unsigned int res;
 
-	asm (ALTERNATIVE("call __sw_hweight32", "popcntl %1, %0", X86_FEATURE_POPCNT)
-			 : "="REG_OUT (res)
-			 : REG_IN (w));
+	asm_inline (ALTERNATIVE("call __sw_hweight32",
+				"popcntl %[val], %[cnt]", X86_FEATURE_POPCNT)
+			 : [cnt] "=" REG_OUT (res), ASM_CALL_CONSTRAINT
+			 : [val] REG_IN (w));
 
 	return res;
 }
@@ -44,9 +45,10 @@ static __always_inline unsigned long __arch_hweight64(__u64 w)
 {
 	unsigned long res;
 
-	asm (ALTERNATIVE("call __sw_hweight64", "popcntq %1, %0", X86_FEATURE_POPCNT)
-			 : "="REG_OUT (res)
-			 : REG_IN (w));
+	asm_inline (ALTERNATIVE("call __sw_hweight64",
+				"popcntq %[val], %[cnt]", X86_FEATURE_POPCNT)
+			 : [cnt] "=" REG_OUT (res), ASM_CALL_CONSTRAINT
+			 : [val] REG_IN (w));
 
 	return res;
 }
diff --git a/arch/x86/include/asm/archrandom.h b/arch/x86/include/asm/archrandom.h
index 02bae8e0758b..4c305305871b 100644
--- a/arch/x86/include/asm/archrandom.h
+++ b/arch/x86/include/asm/archrandom.h
@@ -23,8 +23,7 @@ static inline bool __must_check rdrand_long(unsigned long *v)
 	unsigned int retry = RDRAND_RETRY_LOOPS;
 	do {
 		asm volatile("rdrand %[out]"
-			     CC_SET(c)
-			     : CC_OUT(c) (ok), [out] "=r" (*v));
+			     : "=@ccc" (ok), [out] "=r" (*v));
 		if (ok)
 			return true;
 	} while (--retry);
@@ -35,8 +34,7 @@ static inline bool __must_check rdseed_long(unsigned long *v)
 {
 	bool ok;
 	asm volatile("rdseed %[out]"
-		     CC_SET(c)
-		     : CC_OUT(c) (ok), [out] "=r" (*v));
+		     : "=@ccc" (ok), [out] "=r" (*v));
 	return ok;
 }
 
diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h
index 25466c4d2134..11c6fecc3ad7 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -16,7 +16,10 @@
 #include <asm/gsseg.h>
 #include <asm/nospec-branch.h>
 
-#ifndef CONFIG_X86_CMPXCHG64
+#ifndef CONFIG_X86_CX8
 extern void cmpxchg8b_emu(void);
 #endif
 
+#ifdef CONFIG_STACKPROTECTOR
+extern unsigned long __ref_stack_chk_guard;
+#endif
diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
index 2bec0c89a95c..0e8c611bc9e2 100644
--- a/arch/x86/include/asm/asm.h
+++ b/arch/x86/include/asm/asm.h
@@ -2,7 +2,9 @@
 #ifndef _ASM_X86_ASM_H
 #define _ASM_X86_ASM_H
 
-#ifdef __ASSEMBLY__
+#include <linux/annotate.h>
+
+#ifdef __ASSEMBLER__
 # define __ASM_FORM(x, ...)		x,## __VA_ARGS__
 # define __ASM_FORM_RAW(x, ...)		x,## __VA_ARGS__
 # define __ASM_FORM_COMMA(x, ...)	x,## __VA_ARGS__,
@@ -113,45 +115,32 @@
 
 #endif
 
-#ifndef __ASSEMBLY__
-#ifndef __pic__
+#ifndef __ASSEMBLER__
 static __always_inline __pure void *rip_rel_ptr(void *p)
 {
 	asm("leaq %c1(%%rip), %0" : "=r"(p) : "i"(p));
 
 	return p;
 }
-#define RIP_REL_REF(var)	(*(typeof(&(var)))rip_rel_ptr(&(var)))
-#else
-#define RIP_REL_REF(var)	(var)
-#endif
-#endif
-
-/*
- * Macros to generate condition code outputs from inline assembly,
- * The output operand must be type "bool".
- */
-#ifdef __GCC_ASM_FLAG_OUTPUTS__
-# define CC_SET(c) "\n\t/* output condition code " #c "*/\n"
-# define CC_OUT(c) "=@cc" #c
-#else
-# define CC_SET(c) "\n\tset" #c " %[_cc_" #c "]\n"
-# define CC_OUT(c) [_cc_ ## c] "=qm"
 #endif
 
 #ifdef __KERNEL__
 
+#ifndef COMPILE_OFFSETS
+#include <asm/asm-offsets.h>
+#endif
+
 # include <asm/extable_fixup_types.h>
 
 /* Exception table entry */
-#ifdef __ASSEMBLY__
-
-# define _ASM_EXTABLE_TYPE(from, to, type)			\
-	.pushsection "__ex_table","a" ;				\
-	.balign 4 ;						\
-	.long (from) - . ;					\
-	.long (to) - . ;					\
-	.long type ;						\
+#ifdef __ASSEMBLER__
+
+# define _ASM_EXTABLE_TYPE(from, to, type)				\
+	.pushsection "__ex_table", "aM", @progbits, EXTABLE_SIZE ;	\
+	.balign 4 ;							\
+	.long (from) - . ;						\
+	.long (to) - . ;						\
+	.long type ;							\
 	.popsection
 
 # ifdef CONFIG_KPROBES
@@ -164,7 +153,7 @@ static __always_inline __pure void *rip_rel_ptr(void *p)
 #  define _ASM_NOKPROBE(entry)
 # endif
 
-#else /* ! __ASSEMBLY__ */
+#else /* ! __ASSEMBLER__ */
 
 # define DEFINE_EXTABLE_TYPE_REG \
 	".macro extable_type_reg type:req reg:req\n"						\
@@ -194,7 +183,8 @@ static __always_inline __pure void *rip_rel_ptr(void *p)
 	".purgem extable_type_reg\n"
 
 # define _ASM_EXTABLE_TYPE(from, to, type)			\
-	" .pushsection \"__ex_table\",\"a\"\n"			\
+	" .pushsection __ex_table, \"aM\", @progbits, "		\
+		       __stringify(EXTABLE_SIZE) "\n"		\
 	" .balign 4\n"						\
 	" .long (" #from ") - .\n"				\
 	" .long (" #to ") - .\n"				\
@@ -202,7 +192,8 @@ static __always_inline __pure void *rip_rel_ptr(void *p)
 	" .popsection\n"
 
 # define _ASM_EXTABLE_TYPE_REG(from, to, type, reg)				\
-	" .pushsection \"__ex_table\",\"a\"\n"					\
+	" .pushsection __ex_table, \"aM\", @progbits, "				\
+		       __stringify(EXTABLE_SIZE) "\n"				\
 	" .balign 4\n"								\
 	" .long (" #from ") - .\n"						\
 	" .long (" #to ") - .\n"						\
@@ -213,6 +204,17 @@ static __always_inline __pure void *rip_rel_ptr(void *p)
 
 /* For C file, we already have NOKPROBE_SYMBOL macro */
 
+/* Insert a comma if args are non-empty */
+#define COMMA(x...)		__COMMA(x)
+#define __COMMA(...)		, ##__VA_ARGS__
+
+/*
+ * Combine multiple asm inline constraint args into a single arg for passing to
+ * another macro.
+ */
+#define ASM_OUTPUT(x...)	x
+#define ASM_INPUT(x...)		x
+
 /*
  * This output constraint should be used for any inline asm which has a "call"
  * instruction.  Otherwise the asm may be inserted before the frame pointer
@@ -221,7 +223,7 @@ static __always_inline __pure void *rip_rel_ptr(void *p)
  */
 register unsigned long current_stack_pointer asm(_ASM_SP);
 #define ASM_CALL_CONSTRAINT "+r" (current_stack_pointer)
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #define _ASM_EXTABLE(from, to)					\
 	_ASM_EXTABLE_TYPE(from, to, EX_TYPE_DEFAULT)
@@ -232,5 +234,24 @@ register unsigned long current_stack_pointer asm(_ASM_SP);
 #define _ASM_EXTABLE_FAULT(from, to)				\
 	_ASM_EXTABLE_TYPE(from, to, EX_TYPE_FAULT)
 
+/*
+ * Both i386 and x86_64 returns 64-bit values in edx:eax for certain
+ * instructions, but GCC's "A" constraint has different meanings.
+ * For i386, "A" means exactly edx:eax, while for x86_64 it
+ * means rax *or* rdx.
+ *
+ * These helpers wrapping these semantic differences save one instruction
+ * clearing the high half of 'low':
+ */
+#ifdef CONFIG_X86_64
+# define EAX_EDX_DECLARE_ARGS(val, low, high)	unsigned long low, high
+# define EAX_EDX_VAL(val, low, high)		((low) | (high) << 32)
+# define EAX_EDX_RET(val, low, high)		"=a" (low), "=d" (high)
+#else
+# define EAX_EDX_DECLARE_ARGS(val, low, high)	u64 val
+# define EAX_EDX_VAL(val, low, high)		(val)
+# define EAX_EDX_RET(val, low, high)		"=A" (val)
+#endif
+
 #endif /* __KERNEL__ */
 #endif /* _ASM_X86_ASM_H */
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index 55b4d24356ea..75743f1dfd4e 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -30,14 +30,14 @@ static __always_inline void arch_atomic_set(atomic_t *v, int i)
 
 static __always_inline void arch_atomic_add(int i, atomic_t *v)
 {
-	asm volatile(LOCK_PREFIX "addl %1,%0"
+	asm_inline volatile(LOCK_PREFIX "addl %1, %0"
 		     : "+m" (v->counter)
 		     : "ir" (i) : "memory");
 }
 
 static __always_inline void arch_atomic_sub(int i, atomic_t *v)
 {
-	asm volatile(LOCK_PREFIX "subl %1,%0"
+	asm_inline volatile(LOCK_PREFIX "subl %1, %0"
 		     : "+m" (v->counter)
 		     : "ir" (i) : "memory");
 }
@@ -50,14 +50,14 @@ static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v)
 
 static __always_inline void arch_atomic_inc(atomic_t *v)
 {
-	asm volatile(LOCK_PREFIX "incl %0"
+	asm_inline volatile(LOCK_PREFIX "incl %0"
 		     : "+m" (v->counter) :: "memory");
 }
 #define arch_atomic_inc arch_atomic_inc
 
 static __always_inline void arch_atomic_dec(atomic_t *v)
 {
-	asm volatile(LOCK_PREFIX "decl %0"
+	asm_inline volatile(LOCK_PREFIX "decl %0"
 		     : "+m" (v->counter) :: "memory");
 }
 #define arch_atomic_dec arch_atomic_dec
@@ -116,7 +116,7 @@ static __always_inline int arch_atomic_xchg(atomic_t *v, int new)
 
 static __always_inline void arch_atomic_and(int i, atomic_t *v)
 {
-	asm volatile(LOCK_PREFIX "andl %1,%0"
+	asm_inline volatile(LOCK_PREFIX "andl %1, %0"
 			: "+m" (v->counter)
 			: "ir" (i)
 			: "memory");
@@ -134,7 +134,7 @@ static __always_inline int arch_atomic_fetch_and(int i, atomic_t *v)
 
 static __always_inline void arch_atomic_or(int i, atomic_t *v)
 {
-	asm volatile(LOCK_PREFIX "orl %1,%0"
+	asm_inline volatile(LOCK_PREFIX "orl %1, %0"
 			: "+m" (v->counter)
 			: "ir" (i)
 			: "memory");
@@ -152,7 +152,7 @@ static __always_inline int arch_atomic_fetch_or(int i, atomic_t *v)
 
 static __always_inline void arch_atomic_xor(int i, atomic_t *v)
 {
-	asm volatile(LOCK_PREFIX "xorl %1,%0"
+	asm_inline volatile(LOCK_PREFIX "xorl %1, %0"
 			: "+m" (v->counter)
 			: "ir" (i)
 			: "memory");
diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atomic64_32.h
index 1f650b4dde50..ab838205c1c6 100644
--- a/arch/x86/include/asm/atomic64_32.h
+++ b/arch/x86/include/asm/atomic64_32.h
@@ -48,16 +48,20 @@ static __always_inline s64 arch_atomic64_read_nonatomic(const atomic64_t *v)
 	ATOMIC64_EXPORT(atomic64_##sym)
 #endif
 
-#ifdef CONFIG_X86_CMPXCHG64
-#define __alternative_atomic64(f, g, out, in...) \
-	asm volatile("call %c[func]" \
-		     : out : [func] "i" (atomic64_##g##_cx8), ## in)
+#ifdef CONFIG_X86_CX8
+#define __alternative_atomic64(f, g, out, in, clobbers...)		\
+	asm volatile("call %c[func]"					\
+		     : ALT_OUTPUT_SP(out) \
+		     : [func] "i" (atomic64_##g##_cx8)			\
+		       COMMA(in)					\
+		     : clobbers)
 
 #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8)
 #else
-#define __alternative_atomic64(f, g, out, in...) \
-	alternative_call(atomic64_##f##_386, atomic64_##g##_cx8, \
-			 X86_FEATURE_CX8, ASM_OUTPUT2(out), ## in)
+#define __alternative_atomic64(f, g, out, in, clobbers...)		\
+	alternative_call(atomic64_##f##_386, atomic64_##g##_cx8,	\
+			 X86_FEATURE_CX8, ASM_OUTPUT(out),		\
+			 ASM_INPUT(in), clobbers)
 
 #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \
 	ATOMIC64_DECL_ONE(sym##_386)
@@ -68,8 +72,8 @@ ATOMIC64_DECL_ONE(inc_386);
 ATOMIC64_DECL_ONE(dec_386);
 #endif
 
-#define alternative_atomic64(f, out, in...) \
-	__alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in)
+#define alternative_atomic64(f, out, in, clobbers...) \
+	__alternative_atomic64(f, f, ASM_OUTPUT(out), ASM_INPUT(in), clobbers)
 
 ATOMIC64_DECL(read);
 ATOMIC64_DECL(set);
@@ -104,9 +108,10 @@ static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n)
 	s64 o;
 	unsigned high = (unsigned)(n >> 32);
 	unsigned low = (unsigned)n;
-	alternative_atomic64(xchg, "=&A" (o),
-			     "S" (v), "b" (low), "c" (high)
-			     : "memory");
+	alternative_atomic64(xchg,
+			     "=&A" (o),
+			     ASM_INPUT("S" (v), "b" (low), "c" (high)),
+			     "memory");
 	return o;
 }
 #define arch_atomic64_xchg arch_atomic64_xchg
@@ -115,23 +120,25 @@ static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
 {
 	unsigned high = (unsigned)(i >> 32);
 	unsigned low = (unsigned)i;
-	alternative_atomic64(set, /* no output */,
-			     "S" (v), "b" (low), "c" (high)
-			     : "eax", "edx", "memory");
+	alternative_atomic64(set,
+			     /* no output */,
+			     ASM_INPUT("S" (v), "b" (low), "c" (high)),
+			     "eax", "edx", "memory");
 }
 
 static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
 {
 	s64 r;
-	alternative_atomic64(read, "=&A" (r), "c" (v) : "memory");
+	alternative_atomic64(read, "=&A" (r), "c" (v), "memory");
 	return r;
 }
 
 static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
 {
 	alternative_atomic64(add_return,
-			     ASM_OUTPUT2("+A" (i), "+c" (v)),
-			     ASM_NO_INPUT_CLOBBER("memory"));
+			     ASM_OUTPUT("+A" (i), "+c" (v)),
+			     /* no input */,
+			     "memory");
 	return i;
 }
 #define arch_atomic64_add_return arch_atomic64_add_return
@@ -139,8 +146,9 @@ static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
 static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)
 {
 	alternative_atomic64(sub_return,
-			     ASM_OUTPUT2("+A" (i), "+c" (v)),
-			     ASM_NO_INPUT_CLOBBER("memory"));
+			     ASM_OUTPUT("+A" (i), "+c" (v)),
+			     /* no input */,
+			     "memory");
 	return i;
 }
 #define arch_atomic64_sub_return arch_atomic64_sub_return
@@ -148,8 +156,10 @@ static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)
 static __always_inline s64 arch_atomic64_inc_return(atomic64_t *v)
 {
 	s64 a;
-	alternative_atomic64(inc_return, "=&A" (a),
-			     "S" (v) : "memory", "ecx");
+	alternative_atomic64(inc_return,
+			     "=&A" (a),
+			     "S" (v),
+			     "memory", "ecx");
 	return a;
 }
 #define arch_atomic64_inc_return arch_atomic64_inc_return
@@ -157,8 +167,10 @@ static __always_inline s64 arch_atomic64_inc_return(atomic64_t *v)
 static __always_inline s64 arch_atomic64_dec_return(atomic64_t *v)
 {
 	s64 a;
-	alternative_atomic64(dec_return, "=&A" (a),
-			     "S" (v) : "memory", "ecx");
+	alternative_atomic64(dec_return,
+			     "=&A" (a),
+			     "S" (v),
+			     "memory", "ecx");
 	return a;
 }
 #define arch_atomic64_dec_return arch_atomic64_dec_return
@@ -166,28 +178,34 @@ static __always_inline s64 arch_atomic64_dec_return(atomic64_t *v)
 static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)
 {
 	__alternative_atomic64(add, add_return,
-			       ASM_OUTPUT2("+A" (i), "+c" (v)),
-			       ASM_NO_INPUT_CLOBBER("memory"));
+			       ASM_OUTPUT("+A" (i), "+c" (v)),
+			       /* no input */,
+			       "memory");
 }
 
 static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v)
 {
 	__alternative_atomic64(sub, sub_return,
-			       ASM_OUTPUT2("+A" (i), "+c" (v)),
-			       ASM_NO_INPUT_CLOBBER("memory"));
+			       ASM_OUTPUT("+A" (i), "+c" (v)),
+			       /* no input */,
+			       "memory");
 }
 
 static __always_inline void arch_atomic64_inc(atomic64_t *v)
 {
-	__alternative_atomic64(inc, inc_return, /* no output */,
-			       "S" (v) : "memory", "eax", "ecx", "edx");
+	__alternative_atomic64(inc, inc_return,
+			       /* no output */,
+			       "S" (v),
+			       "memory", "eax", "ecx", "edx");
 }
 #define arch_atomic64_inc arch_atomic64_inc
 
 static __always_inline void arch_atomic64_dec(atomic64_t *v)
 {
-	__alternative_atomic64(dec, dec_return, /* no output */,
-			       "S" (v) : "memory", "eax", "ecx", "edx");
+	__alternative_atomic64(dec, dec_return,
+			       /* no output */,
+			       "S" (v),
+			       "memory", "eax", "ecx", "edx");
 }
 #define arch_atomic64_dec arch_atomic64_dec
 
@@ -196,8 +214,9 @@ static __always_inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
 	unsigned low = (unsigned)u;
 	unsigned high = (unsigned)(u >> 32);
 	alternative_atomic64(add_unless,
-			     ASM_OUTPUT2("+A" (a), "+c" (low), "+D" (high)),
-			     "S" (v) : "memory");
+			     ASM_OUTPUT("+A" (a), "+c" (low), "+D" (high)),
+			     "S" (v),
+			     "memory");
 	return (int)a;
 }
 #define arch_atomic64_add_unless arch_atomic64_add_unless
@@ -205,8 +224,10 @@ static __always_inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
 static __always_inline int arch_atomic64_inc_not_zero(atomic64_t *v)
 {
 	int r;
-	alternative_atomic64(inc_not_zero, "=&a" (r),
-			     "S" (v) : "ecx", "edx", "memory");
+	alternative_atomic64(inc_not_zero,
+			     "=&a" (r),
+			     "S" (v),
+			     "ecx", "edx", "memory");
 	return r;
 }
 #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
@@ -214,8 +235,10 @@ static __always_inline int arch_atomic64_inc_not_zero(atomic64_t *v)
 static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
 {
 	s64 r;
-	alternative_atomic64(dec_if_positive, "=&A" (r),
-			     "S" (v) : "ecx", "memory");
+	alternative_atomic64(dec_if_positive,
+			     "=&A" (r),
+			     "S" (v),
+			     "ecx", "memory");
 	return r;
 }
 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h
index ae12acae5b06..87b496325b5b 100644
--- a/arch/x86/include/asm/atomic64_64.h
+++ b/arch/x86/include/asm/atomic64_64.h
@@ -22,14 +22,14 @@ static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
 
 static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)
 {
-	asm volatile(LOCK_PREFIX "addq %1,%0"
+	asm_inline volatile(LOCK_PREFIX "addq %1, %0"
 		     : "=m" (v->counter)
 		     : "er" (i), "m" (v->counter) : "memory");
 }
 
 static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v)
 {
-	asm volatile(LOCK_PREFIX "subq %1,%0"
+	asm_inline volatile(LOCK_PREFIX "subq %1, %0"
 		     : "=m" (v->counter)
 		     : "er" (i), "m" (v->counter) : "memory");
 }
@@ -42,7 +42,7 @@ static __always_inline bool arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
 
 static __always_inline void arch_atomic64_inc(atomic64_t *v)
 {
-	asm volatile(LOCK_PREFIX "incq %0"
+	asm_inline volatile(LOCK_PREFIX "incq %0"
 		     : "=m" (v->counter)
 		     : "m" (v->counter) : "memory");
 }
@@ -50,7 +50,7 @@ static __always_inline void arch_atomic64_inc(atomic64_t *v)
 
 static __always_inline void arch_atomic64_dec(atomic64_t *v)
 {
-	asm volatile(LOCK_PREFIX "decq %0"
+	asm_inline volatile(LOCK_PREFIX "decq %0"
 		     : "=m" (v->counter)
 		     : "m" (v->counter) : "memory");
 }
@@ -110,7 +110,7 @@ static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 new)
 
 static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v)
 {
-	asm volatile(LOCK_PREFIX "andq %1,%0"
+	asm_inline volatile(LOCK_PREFIX "andq %1, %0"
 			: "+m" (v->counter)
 			: "er" (i)
 			: "memory");
@@ -128,7 +128,7 @@ static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v)
 
 static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v)
 {
-	asm volatile(LOCK_PREFIX "orq %1,%0"
+	asm_inline volatile(LOCK_PREFIX "orq %1, %0"
 			: "+m" (v->counter)
 			: "er" (i)
 			: "memory");
@@ -146,7 +146,7 @@ static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v)
 
 static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v)
 {
-	asm volatile(LOCK_PREFIX "xorq %1,%0"
+	asm_inline volatile(LOCK_PREFIX "xorq %1, %0"
 			: "+m" (v->counter)
 			: "er" (i)
 			: "memory");
diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h
index 7b44b3c4cce1..db70832232d4 100644
--- a/arch/x86/include/asm/barrier.h
+++ b/arch/x86/include/asm/barrier.h
@@ -12,11 +12,11 @@
  */
 
 #ifdef CONFIG_X86_32
-#define mb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "mfence", \
+#define mb() asm volatile(ALTERNATIVE("lock addl $0,-4(%%esp)", "mfence", \
 				      X86_FEATURE_XMM2) ::: "memory", "cc")
-#define rmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "lfence", \
+#define rmb() asm volatile(ALTERNATIVE("lock addl $0,-4(%%esp)", "lfence", \
 				       X86_FEATURE_XMM2) ::: "memory", "cc")
-#define wmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "sfence", \
+#define wmb() asm volatile(ALTERNATIVE("lock addl $0,-4(%%esp)", "sfence", \
 				       X86_FEATURE_XMM2) ::: "memory", "cc")
 #else
 #define __mb()	asm volatile("mfence":::"memory")
@@ -50,7 +50,7 @@
 #define __dma_rmb()	barrier()
 #define __dma_wmb()	barrier()
 
-#define __smp_mb()	asm volatile("lock; addl $0,-4(%%" _ASM_SP ")" ::: "memory", "cc")
+#define __smp_mb()	asm volatile("lock addl $0,-4(%%" _ASM_SP ")" ::: "memory", "cc")
 
 #define __smp_rmb()	dma_rmb()
 #define __smp_wmb()	barrier()
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index b96d45944c59..c2ce213f2b9b 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -52,12 +52,12 @@ static __always_inline void
 arch_set_bit(long nr, volatile unsigned long *addr)
 {
 	if (__builtin_constant_p(nr)) {
-		asm volatile(LOCK_PREFIX "orb %b1,%0"
+		asm_inline volatile(LOCK_PREFIX "orb %b1,%0"
 			: CONST_MASK_ADDR(nr, addr)
 			: "iq" (CONST_MASK(nr))
 			: "memory");
 	} else {
-		asm volatile(LOCK_PREFIX __ASM_SIZE(bts) " %1,%0"
+		asm_inline volatile(LOCK_PREFIX __ASM_SIZE(bts) " %1,%0"
 			: : RLONG_ADDR(addr), "Ir" (nr) : "memory");
 	}
 }
@@ -72,11 +72,11 @@ static __always_inline void
 arch_clear_bit(long nr, volatile unsigned long *addr)
 {
 	if (__builtin_constant_p(nr)) {
-		asm volatile(LOCK_PREFIX "andb %b1,%0"
+		asm_inline volatile(LOCK_PREFIX "andb %b1,%0"
 			: CONST_MASK_ADDR(nr, addr)
 			: "iq" (~CONST_MASK(nr)));
 	} else {
-		asm volatile(LOCK_PREFIX __ASM_SIZE(btr) " %1,%0"
+		asm_inline volatile(LOCK_PREFIX __ASM_SIZE(btr) " %1,%0"
 			: : RLONG_ADDR(addr), "Ir" (nr) : "memory");
 	}
 }
@@ -98,9 +98,8 @@ static __always_inline bool arch_xor_unlock_is_negative_byte(unsigned long mask,
 		volatile unsigned long *addr)
 {
 	bool negative;
-	asm volatile(LOCK_PREFIX "xorb %2,%1"
-		CC_SET(s)
-		: CC_OUT(s) (negative), WBYTE_ADDR(addr)
+	asm_inline volatile(LOCK_PREFIX "xorb %2,%1"
+		: "=@ccs" (negative), WBYTE_ADDR(addr)
 		: "iq" ((char)mask) : "memory");
 	return negative;
 }
@@ -122,11 +121,11 @@ static __always_inline void
 arch_change_bit(long nr, volatile unsigned long *addr)
 {
 	if (__builtin_constant_p(nr)) {
-		asm volatile(LOCK_PREFIX "xorb %b1,%0"
+		asm_inline volatile(LOCK_PREFIX "xorb %b1,%0"
 			: CONST_MASK_ADDR(nr, addr)
 			: "iq" (CONST_MASK(nr)));
 	} else {
-		asm volatile(LOCK_PREFIX __ASM_SIZE(btc) " %1,%0"
+		asm_inline volatile(LOCK_PREFIX __ASM_SIZE(btc) " %1,%0"
 			: : RLONG_ADDR(addr), "Ir" (nr) : "memory");
 	}
 }
@@ -149,8 +148,7 @@ arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
 	bool oldbit;
 
 	asm(__ASM_SIZE(bts) " %2,%1"
-	    CC_SET(c)
-	    : CC_OUT(c) (oldbit)
+	    : "=@ccc" (oldbit)
 	    : ADDR, "Ir" (nr) : "memory");
 	return oldbit;
 }
@@ -175,8 +173,7 @@ arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 	bool oldbit;
 
 	asm volatile(__ASM_SIZE(btr) " %2,%1"
-		     CC_SET(c)
-		     : CC_OUT(c) (oldbit)
+		     : "=@ccc" (oldbit)
 		     : ADDR, "Ir" (nr) : "memory");
 	return oldbit;
 }
@@ -187,8 +184,7 @@ arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 	bool oldbit;
 
 	asm volatile(__ASM_SIZE(btc) " %2,%1"
-		     CC_SET(c)
-		     : CC_OUT(c) (oldbit)
+		     : "=@ccc" (oldbit)
 		     : ADDR, "Ir" (nr) : "memory");
 
 	return oldbit;
@@ -211,8 +207,7 @@ static __always_inline bool constant_test_bit_acquire(long nr, const volatile un
 	bool oldbit;
 
 	asm volatile("testb %2,%1"
-		     CC_SET(nz)
-		     : CC_OUT(nz) (oldbit)
+		     : "=@ccnz" (oldbit)
 		     : "m" (((unsigned char *)addr)[nr >> 3]),
 		       "i" (1 << (nr & 7))
 		     :"memory");
@@ -225,8 +220,7 @@ static __always_inline bool variable_test_bit(long nr, volatile const unsigned l
 	bool oldbit;
 
 	asm volatile(__ASM_SIZE(bt) " %2,%1"
-		     CC_SET(c)
-		     : CC_OUT(c) (oldbit)
+		     : "=@ccc" (oldbit)
 		     : "m" (*(unsigned long *)addr), "Ir" (nr) : "memory");
 
 	return oldbit;
@@ -246,9 +240,9 @@ arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
 					  variable_test_bit(nr, addr);
 }
 
-static __always_inline unsigned long variable__ffs(unsigned long word)
+static __always_inline __attribute_const__ unsigned long variable__ffs(unsigned long word)
 {
-	asm("rep; bsf %1,%0"
+	asm("tzcnt %1,%0"
 		: "=r" (word)
 		: ASM_INPUT_RM (word));
 	return word;
@@ -265,12 +259,9 @@ static __always_inline unsigned long variable__ffs(unsigned long word)
 	 (unsigned long)__builtin_ctzl(word) :	\
 	 variable__ffs(word))
 
-static __always_inline unsigned long variable_ffz(unsigned long word)
+static __always_inline __attribute_const__ unsigned long variable_ffz(unsigned long word)
 {
-	asm("rep; bsf %1,%0"
-		: "=r" (word)
-		: "r" (~word));
-	return word;
+	return variable__ffs(~word);
 }
 
 /**
@@ -290,7 +281,7 @@ static __always_inline unsigned long variable_ffz(unsigned long word)
  *
  * Undefined if no set bit exists, so code should check against 0 first.
  */
-static __always_inline unsigned long __fls(unsigned long word)
+static __always_inline __attribute_const__ unsigned long __fls(unsigned long word)
 {
 	if (__builtin_constant_p(word))
 		return BITS_PER_LONG - 1 - __builtin_clzl(word);
@@ -304,7 +295,7 @@ static __always_inline unsigned long __fls(unsigned long word)
 #undef ADDR
 
 #ifdef __KERNEL__
-static __always_inline int variable_ffs(int x)
+static __always_inline __attribute_const__ int variable_ffs(int x)
 {
 	int r;
 
@@ -358,7 +349,7 @@ static __always_inline int variable_ffs(int x)
  * set bit if value is nonzero. The last (most significant) bit is
  * at position 32.
  */
-static __always_inline int fls(unsigned int x)
+static __always_inline __attribute_const__ int fls(unsigned int x)
 {
 	int r;
 
@@ -403,7 +394,7 @@ static __always_inline int fls(unsigned int x)
  * at position 64.
  */
 #ifdef CONFIG_X86_64
-static __always_inline int fls64(__u64 x)
+static __always_inline __attribute_const__ int fls64(__u64 x)
 {
 	int bitpos = -1;
 
diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h
index 3e5b111e619d..f7b67cb73915 100644
--- a/arch/x86/include/asm/boot.h
+++ b/arch/x86/include/asm/boot.h
@@ -74,15 +74,27 @@
 # define BOOT_STACK_SIZE	0x1000
 #endif
 
-#ifndef __ASSEMBLY__
+#define TRAMPOLINE_32BIT_SIZE		(2 * PAGE_SIZE)
+
+#define TRAMPOLINE_32BIT_CODE_OFFSET	PAGE_SIZE
+#define TRAMPOLINE_32BIT_CODE_SIZE	0xA0
+
+#ifndef __ASSEMBLER__
 extern unsigned int output_len;
 extern const unsigned long kernel_text_size;
+extern const unsigned long kernel_inittext_offset;
+extern const unsigned long kernel_inittext_size;
 extern const unsigned long kernel_total_size;
 
 unsigned long decompress_kernel(unsigned char *outbuf, unsigned long virt_addr,
 				void (*error)(char *x));
 
 extern struct boot_params *boot_params_ptr;
+extern unsigned long *trampoline_32bit;
+extern const u16 trampoline_ljmp_imm_offset;
+
+void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
+
 #endif
 
 #endif /* _ASM_X86_BOOT_H */
diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h
index 806649c7f23d..d561a8443c13 100644
--- a/arch/x86/include/asm/bug.h
+++ b/arch/x86/include/asm/bug.h
@@ -5,72 +5,108 @@
 #include <linux/stringify.h>
 #include <linux/instrumentation.h>
 #include <linux/objtool.h>
+#include <asm/asm.h>
+
+#ifndef __ASSEMBLY__
+struct bug_entry;
+extern void __WARN_trap(struct bug_entry *bug, ...);
+#endif
 
 /*
  * Despite that some emulators terminate on UD2, we use it for WARN().
  */
-#define ASM_UD2		".byte 0x0f, 0x0b"
+#define ASM_UD2		_ASM_BYTES(0x0f, 0x0b)
 #define INSN_UD2	0x0b0f
 #define LEN_UD2		2
 
+#define ASM_UDB		_ASM_BYTES(0xd6)
+#define INSN_UDB	0xd6
+#define LEN_UDB		1
+
 /*
  * In clang we have UD1s reporting UBSAN failures on X86, 64 and 32bit.
  */
 #define INSN_ASOP		0x67
+#define INSN_LOCK		0xf0
 #define OPCODE_ESCAPE		0x0f
 #define SECOND_BYTE_OPCODE_UD1	0xb9
 #define SECOND_BYTE_OPCODE_UD2	0x0b
 
 #define BUG_NONE		0xffff
-#define BUG_UD1			0xfffe
-#define BUG_UD2			0xfffd
+#define BUG_UD2			0xfffe
+#define BUG_UD1			0xfffd
+#define BUG_UD1_UBSAN		0xfffc
+#define BUG_UD1_WARN		0xfffb
+#define BUG_UDB			0xffd6
+#define BUG_LOCK		0xfff0
 
 #ifdef CONFIG_GENERIC_BUG
 
-#ifdef CONFIG_X86_32
-# define __BUG_REL(val)	".long " __stringify(val)
+#ifdef CONFIG_DEBUG_BUGVERBOSE
+#define __BUG_ENTRY_VERBOSE(file, line)					\
+	"\t.long " file " - .\t# bug_entry::file\n"			\
+	"\t.word " line     "\t# bug_entry::line\n"
 #else
-# define __BUG_REL(val)	".long " __stringify(val) " - ."
+#define __BUG_ENTRY_VERBOSE(file, line)
 #endif
 
-#ifdef CONFIG_DEBUG_BUGVERBOSE
+#if defined(CONFIG_X86_64) || defined(CONFIG_DEBUG_BUGVERBOSE_DETAILED)
+#define HAVE_ARCH_BUG_FORMAT
+#define __BUG_ENTRY_FORMAT(format)					\
+	"\t.long " format " - .\t# bug_entry::format\n"
+#else
+#define __BUG_ENTRY_FORMAT(format)
+#endif
 
-#define _BUG_FLAGS(ins, flags, extra)					\
-do {									\
-	asm_inline volatile("1:\t" ins "\n"				\
-		     ".pushsection __bug_table,\"aw\"\n"		\
-		     "2:\t" __BUG_REL(1b) "\t# bug_entry::bug_addr\n"	\
-		     "\t"  __BUG_REL(%c0) "\t# bug_entry::file\n"	\
-		     "\t.word %c1"        "\t# bug_entry::line\n"	\
-		     "\t.word %c2"        "\t# bug_entry::flags\n"	\
-		     "\t.org 2b+%c3\n"					\
-		     ".popsection\n"					\
-		     extra						\
-		     : : "i" (__FILE__), "i" (__LINE__),		\
-			 "i" (flags),					\
-			 "i" (sizeof(struct bug_entry)));		\
-} while (0)
+#ifdef CONFIG_X86_64
+#define HAVE_ARCH_BUG_FORMAT_ARGS
+#endif
 
-#else /* !CONFIG_DEBUG_BUGVERBOSE */
+#define __BUG_ENTRY(format, file, line, flags)				\
+	"\t.long 1b - ."	"\t# bug_entry::bug_addr\n"		\
+	__BUG_ENTRY_FORMAT(format)					\
+	__BUG_ENTRY_VERBOSE(file, line)					\
+	"\t.word " flags	"\t# bug_entry::flags\n"
+
+#define _BUG_FLAGS_ASM(format, file, line, flags, size, extra)		\
+	".pushsection __bug_table,\"aw\"\n\t"				\
+	ANNOTATE_DATA_SPECIAL "\n\t"					\
+	"2:\n\t"							\
+	__BUG_ENTRY(format, file, line, flags)				\
+	"\t.org 2b + " size "\n"					\
+	".popsection\n"							\
+	extra
+
+#ifdef CONFIG_DEBUG_BUGVERBOSE_DETAILED
+#define WARN_CONDITION_STR(cond_str) cond_str
+#else
+#define WARN_CONDITION_STR(cond_str) ""
+#endif
 
-#define _BUG_FLAGS(ins, flags, extra)					\
+#define _BUG_FLAGS(cond_str, ins, flags, extra)				\
 do {									\
 	asm_inline volatile("1:\t" ins "\n"				\
-		     ".pushsection __bug_table,\"aw\"\n"		\
-		     "2:\t" __BUG_REL(1b) "\t# bug_entry::bug_addr\n"	\
-		     "\t.word %c0"        "\t# bug_entry::flags\n"	\
-		     "\t.org 2b+%c1\n"					\
-		     ".popsection\n"					\
-		     extra						\
-		     : : "i" (flags),					\
-			 "i" (sizeof(struct bug_entry)));		\
+			    _BUG_FLAGS_ASM("%c[fmt]", "%c[file]",	\
+					   "%c[line]", "%c[fl]",	\
+					   "%c[size]", extra)		\
+		   : : [fmt] "i" (WARN_CONDITION_STR(cond_str)),	\
+		       [file] "i" (__FILE__),				\
+		       [line] "i" (__LINE__),				\
+		       [fl] "i" (flags),				\
+		       [size] "i" (sizeof(struct bug_entry)));		\
 } while (0)
 
-#endif /* CONFIG_DEBUG_BUGVERBOSE */
+#define ARCH_WARN_ASM(file, line, flags, size)				\
+	".pushsection .rodata.str1.1, \"aMS\", @progbits, 1\n"		\
+	"99:\n"								\
+	"\t.string \"\"\n"						\
+	".popsection\n"							\
+	"1:\t " ASM_UD2 "\n"						\
+	_BUG_FLAGS_ASM("99b", file, line, flags, size, "")
 
 #else
 
-#define _BUG_FLAGS(ins, flags, extra)  asm volatile(ins)
+#define _BUG_FLAGS(cond_str, ins, flags, extra)  asm volatile(ins)
 
 #endif /* CONFIG_GENERIC_BUG */
 
@@ -78,7 +114,7 @@ do {									\
 #define BUG()							\
 do {								\
 	instrumentation_begin();				\
-	_BUG_FLAGS(ASM_UD2, 0, "");				\
+	_BUG_FLAGS("", ASM_UD2, 0, "");				\
 	__builtin_unreachable();				\
 } while (0)
 
@@ -88,14 +124,72 @@ do {								\
  * were to trigger, we'd rather wreck the machine in an attempt to get the
  * message out than not know about it.
  */
-#define __WARN_FLAGS(flags)					\
-do {								\
-	__auto_type __flags = BUGFLAG_WARNING|(flags);		\
-	instrumentation_begin();				\
-	_BUG_FLAGS(ASM_UD2, __flags, ASM_REACHABLE);		\
-	instrumentation_end();					\
+
+#define ARCH_WARN_REACHABLE	ANNOTATE_REACHABLE(1b)
+
+#define __WARN_FLAGS(cond_str, flags)					\
+do {									\
+	auto __flags = BUGFLAG_WARNING|(flags);				\
+	instrumentation_begin();					\
+	_BUG_FLAGS(cond_str, ASM_UD2, __flags, ARCH_WARN_REACHABLE);	\
+	instrumentation_end();						\
 } while (0)
 
+#ifdef HAVE_ARCH_BUG_FORMAT_ARGS
+
+#ifndef __ASSEMBLY__
+#include <linux/static_call_types.h>
+DECLARE_STATIC_CALL(WARN_trap, __WARN_trap);
+
+struct pt_regs;
+struct sysv_va_list { /* from AMD64 System V ABI */
+	unsigned int gp_offset;
+	unsigned int fp_offset;
+	void *overflow_arg_area;
+	void *reg_save_area;
+};
+struct arch_va_list {
+	unsigned long regs[6];
+	struct sysv_va_list args;
+};
+extern void *__warn_args(struct arch_va_list *args, struct pt_regs *regs);
+#endif /* __ASSEMBLY__ */
+
+#define __WARN_bug_entry(flags, format) ({				\
+	struct bug_entry *bug;						\
+	asm_inline volatile("lea (2f)(%%rip), %[addr]\n1:\n"		\
+			    _BUG_FLAGS_ASM("%c[fmt]", "%c[file]",	\
+					   "%c[line]", "%c[fl]",	\
+					   "%c[size]", "")		\
+		   : [addr] "=r" (bug)					\
+		   : [fmt] "i" (format),				\
+		     [file] "i" (__FILE__),				\
+		     [line] "i" (__LINE__),				\
+		     [fl] "i" (flags),					\
+		     [size] "i" (sizeof(struct bug_entry)));		\
+	bug; })
+
+#define __WARN_print_arg(flags, format, arg...)				\
+do {									\
+	int __flags = (flags) | BUGFLAG_WARNING | BUGFLAG_ARGS ;	\
+	static_call_mod(WARN_trap)(__WARN_bug_entry(__flags, format), ## arg); \
+	asm (""); /* inhibit tail-call optimization */			\
+} while (0)
+
+#define __WARN_printf(taint, fmt, arg...) \
+	__WARN_print_arg(BUGFLAG_TAINT(taint), fmt, ## arg)
+
+#define WARN_ONCE(cond, format, arg...) ({				\
+	int __ret_warn_on = !!(cond);					\
+	if (unlikely(__ret_warn_on)) {					\
+		__WARN_print_arg(BUGFLAG_ONCE|BUGFLAG_TAINT(TAINT_WARN),\
+				format, ## arg);			\
+	}								\
+	__ret_warn_on;							\
+})
+
+#endif /* HAVE_ARCH_BUG_FORMAT_ARGS */
+
 #include <asm-generic/bug.h>
 
 #endif /* _ASM_X86_BUG_H */
diff --git a/arch/x86/include/asm/ce4100.h b/arch/x86/include/asm/ce4100.h
index 2930f560d7f3..e1f965bb1e31 100644
--- a/arch/x86/include/asm/ce4100.h
+++ b/arch/x86/include/asm/ce4100.h
@@ -4,4 +4,10 @@
 
 int ce4100_pci_init(void);
 
+#ifdef CONFIG_SERIAL_8250
+void __init sdv_serial_fixup(void);
+#else
+static inline void sdv_serial_fixup(void) {};
+#endif
+
 #endif
diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h
index 31d19c815f99..c40b9ebc1fb4 100644
--- a/arch/x86/include/asm/cfi.h
+++ b/arch/x86/include/asm/cfi.h
@@ -71,12 +71,10 @@
  *
  * __cfi_foo:
  *   endbr64
- *   subl 0x12345678, %r10d
- *   jz   foo
- *   ud2
- *   nop
+ *   subl 0x12345678, %eax
+ *   jne.32,pn foo+3
  * foo:
- *   osp nop3			# was endbr64
+ *   nopl -42(%rax)		# was endbr64
  *   ... code here ...
  *   ret
  *
@@ -86,9 +84,9 @@
  * indirect caller:
  *   lea foo(%rip), %r11
  *   ...
- *   movl $0x12345678, %r10d
- *   subl $16, %r11
- *   nop4
+ *   movl $0x12345678, %eax
+ *   lea  -0x10(%r11), %r11
+ *   nop5
  *   call *%r11
  *
  */
@@ -101,13 +99,21 @@ enum cfi_mode {
 
 extern enum cfi_mode cfi_mode;
 
+#ifdef CONFIG_FINEIBT_BHI
+extern bool cfi_bhi;
+#else
+#define cfi_bhi (0)
+#endif
+
+typedef u8 bhi_thunk[32];
+extern bhi_thunk __bhi_args[];
+extern bhi_thunk __bhi_args_end[];
+
 struct pt_regs;
 
-#ifdef CONFIG_CFI_CLANG
+#ifdef CONFIG_CFI
 enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
 #define __bpfcall
-extern u32 cfi_bpf_hash;
-extern u32 cfi_bpf_subprog_hash;
 
 static inline int cfi_get_offset(void)
 {
@@ -125,19 +131,31 @@ static inline int cfi_get_offset(void)
 #define cfi_get_offset cfi_get_offset
 
 extern u32 cfi_get_func_hash(void *func);
+#define cfi_get_func_hash cfi_get_func_hash
+
+extern int cfi_get_func_arity(void *func);
+
+#ifdef CONFIG_FINEIBT
+extern bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type);
+#else
+static inline bool
+decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type)
+{
+	return false;
+}
+
+#endif
 
 #else
 static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
 {
 	return BUG_TRAP_TYPE_NONE;
 }
-#define cfi_bpf_hash 0U
-#define cfi_bpf_subprog_hash 0U
-static inline u32 cfi_get_func_hash(void *func)
+static inline int cfi_get_func_arity(void *func)
 {
 	return 0;
 }
-#endif /* CONFIG_CFI_CLANG */
+#endif /* CONFIG_CFI */
 
 #if HAS_KERNEL_IBT == 1
 #define CFI_NOSEAL(x)	asm(IBT_NOSEAL(__stringify(x)))
diff --git a/arch/x86/include/asm/cmpxchg.h b/arch/x86/include/asm/cmpxchg.h
index 5612648b0202..a88b06f1c35e 100644
--- a/arch/x86/include/asm/cmpxchg.h
+++ b/arch/x86/include/asm/cmpxchg.h
@@ -44,22 +44,22 @@ extern void __add_wrong_size(void)
 	        __typeof__ (*(ptr)) __ret = (arg);			\
 		switch (sizeof(*(ptr))) {				\
 		case __X86_CASE_B:					\
-			asm volatile (lock #op "b %b0, %1\n"		\
+			asm_inline volatile (lock #op "b %b0, %1"	\
 				      : "+q" (__ret), "+m" (*(ptr))	\
 				      : : "memory", "cc");		\
 			break;						\
 		case __X86_CASE_W:					\
-			asm volatile (lock #op "w %w0, %1\n"		\
+			asm_inline volatile (lock #op "w %w0, %1"	\
 				      : "+r" (__ret), "+m" (*(ptr))	\
 				      : : "memory", "cc");		\
 			break;						\
 		case __X86_CASE_L:					\
-			asm volatile (lock #op "l %0, %1\n"		\
+			asm_inline volatile (lock #op "l %0, %1"	\
 				      : "+r" (__ret), "+m" (*(ptr))	\
 				      : : "memory", "cc");		\
 			break;						\
 		case __X86_CASE_Q:					\
-			asm volatile (lock #op "q %q0, %1\n"		\
+			asm_inline volatile (lock #op "q %q0, %1"	\
 				      : "+r" (__ret), "+m" (*(ptr))	\
 				      : : "memory", "cc");		\
 			break;						\
@@ -91,7 +91,7 @@ extern void __add_wrong_size(void)
 	case __X86_CASE_B:						\
 	{								\
 		volatile u8 *__ptr = (volatile u8 *)(ptr);		\
-		asm volatile(lock "cmpxchgb %2,%1"			\
+		asm_inline volatile(lock "cmpxchgb %2, %1"		\
 			     : "=a" (__ret), "+m" (*__ptr)		\
 			     : "q" (__new), "0" (__old)			\
 			     : "memory");				\
@@ -100,7 +100,7 @@ extern void __add_wrong_size(void)
 	case __X86_CASE_W:						\
 	{								\
 		volatile u16 *__ptr = (volatile u16 *)(ptr);		\
-		asm volatile(lock "cmpxchgw %2,%1"			\
+		asm_inline volatile(lock "cmpxchgw %2, %1"		\
 			     : "=a" (__ret), "+m" (*__ptr)		\
 			     : "r" (__new), "0" (__old)			\
 			     : "memory");				\
@@ -109,7 +109,7 @@ extern void __add_wrong_size(void)
 	case __X86_CASE_L:						\
 	{								\
 		volatile u32 *__ptr = (volatile u32 *)(ptr);		\
-		asm volatile(lock "cmpxchgl %2,%1"			\
+		asm_inline volatile(lock "cmpxchgl %2, %1"		\
 			     : "=a" (__ret), "+m" (*__ptr)		\
 			     : "r" (__new), "0" (__old)			\
 			     : "memory");				\
@@ -118,7 +118,7 @@ extern void __add_wrong_size(void)
 	case __X86_CASE_Q:						\
 	{								\
 		volatile u64 *__ptr = (volatile u64 *)(ptr);		\
-		asm volatile(lock "cmpxchgq %2,%1"			\
+		asm_inline volatile(lock "cmpxchgq %2, %1"		\
 			     : "=a" (__ret), "+m" (*__ptr)		\
 			     : "r" (__new), "0" (__old)			\
 			     : "memory");				\
@@ -134,7 +134,7 @@ extern void __add_wrong_size(void)
 	__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
 
 #define __sync_cmpxchg(ptr, old, new, size)				\
-	__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
+	__raw_cmpxchg((ptr), (old), (new), (size), "lock ")
 
 #define __cmpxchg_local(ptr, old, new, size)				\
 	__raw_cmpxchg((ptr), (old), (new), (size), "")
@@ -165,9 +165,8 @@ extern void __add_wrong_size(void)
 	case __X86_CASE_B:						\
 	{								\
 		volatile u8 *__ptr = (volatile u8 *)(_ptr);		\
-		asm volatile(lock "cmpxchgb %[new], %[ptr]"		\
-			     CC_SET(z)					\
-			     : CC_OUT(z) (success),			\
+		asm_inline volatile(lock "cmpxchgb %[new], %[ptr]"	\
+			     : "=@ccz" (success),			\
 			       [ptr] "+m" (*__ptr),			\
 			       [old] "+a" (__old)			\
 			     : [new] "q" (__new)			\
@@ -177,9 +176,8 @@ extern void __add_wrong_size(void)
 	case __X86_CASE_W:						\
 	{								\
 		volatile u16 *__ptr = (volatile u16 *)(_ptr);		\
-		asm volatile(lock "cmpxchgw %[new], %[ptr]"		\
-			     CC_SET(z)					\
-			     : CC_OUT(z) (success),			\
+		asm_inline volatile(lock "cmpxchgw %[new], %[ptr]"	\
+			     : "=@ccz" (success),			\
 			       [ptr] "+m" (*__ptr),			\
 			       [old] "+a" (__old)			\
 			     : [new] "r" (__new)			\
@@ -189,9 +187,8 @@ extern void __add_wrong_size(void)
 	case __X86_CASE_L:						\
 	{								\
 		volatile u32 *__ptr = (volatile u32 *)(_ptr);		\
-		asm volatile(lock "cmpxchgl %[new], %[ptr]"		\
-			     CC_SET(z)					\
-			     : CC_OUT(z) (success),			\
+		asm_inline volatile(lock "cmpxchgl %[new], %[ptr]"	\
+			     : "=@ccz" (success),			\
 			       [ptr] "+m" (*__ptr),			\
 			       [old] "+a" (__old)			\
 			     : [new] "r" (__new)			\
@@ -201,9 +198,8 @@ extern void __add_wrong_size(void)
 	case __X86_CASE_Q:						\
 	{								\
 		volatile u64 *__ptr = (volatile u64 *)(_ptr);		\
-		asm volatile(lock "cmpxchgq %[new], %[ptr]"		\
-			     CC_SET(z)					\
-			     : CC_OUT(z) (success),			\
+		asm_inline volatile(lock "cmpxchgq %[new], %[ptr]"	\
+			     : "=@ccz" (success),			\
 			       [ptr] "+m" (*__ptr),			\
 			       [old] "+a" (__old)			\
 			     : [new] "r" (__new)			\
@@ -222,7 +218,7 @@ extern void __add_wrong_size(void)
 	__raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX)
 
 #define __sync_try_cmpxchg(ptr, pold, new, size)			\
-	__raw_try_cmpxchg((ptr), (pold), (new), (size), "lock; ")
+	__raw_try_cmpxchg((ptr), (pold), (new), (size), "lock ")
 
 #define __try_cmpxchg_local(ptr, pold, new, size)			\
 	__raw_try_cmpxchg((ptr), (pold), (new), (size), "")
diff --git a/arch/x86/include/asm/cmpxchg_32.h b/arch/x86/include/asm/cmpxchg_32.h
index 62cef2113ca7..1f80a62be969 100644
--- a/arch/x86/include/asm/cmpxchg_32.h
+++ b/arch/x86/include/asm/cmpxchg_32.h
@@ -19,7 +19,7 @@ union __u64_halves {
 	union __u64_halves o = { .full = (_old), },			\
 			   n = { .full = (_new), };			\
 									\
-	asm volatile(_lock "cmpxchg8b %[ptr]"				\
+	asm_inline volatile(_lock "cmpxchg8b %[ptr]"			\
 		     : [ptr] "+m" (*(_ptr)),				\
 		       "+a" (o.low), "+d" (o.high)			\
 		     : "b" (n.low), "c" (n.high)			\
@@ -45,9 +45,8 @@ static __always_inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new
 			   n = { .full = (_new), };			\
 	bool ret;							\
 									\
-	asm volatile(_lock "cmpxchg8b %[ptr]"				\
-		     CC_SET(e)						\
-		     : CC_OUT(e) (ret),					\
+	asm_inline volatile(_lock "cmpxchg8b %[ptr]"			\
+		     : "=@ccz" (ret),					\
 		       [ptr] "+m" (*(_ptr)),				\
 		       "+a" (o.low), "+d" (o.high)			\
 		     : "b" (n.low), "c" (n.high)			\
@@ -69,7 +68,7 @@ static __always_inline bool __try_cmpxchg64_local(volatile u64 *ptr, u64 *oldp,
 	return __arch_try_cmpxchg64(ptr, oldp, new,);
 }
 
-#ifdef CONFIG_X86_CMPXCHG64
+#ifdef CONFIG_X86_CX8
 
 #define arch_cmpxchg64 __cmpxchg64
 
@@ -91,19 +90,21 @@ static __always_inline bool __try_cmpxchg64_local(volatile u64 *ptr, u64 *oldp,
 	union __u64_halves o = { .full = (_old), },			\
 			   n = { .full = (_new), };			\
 									\
-	asm volatile(ALTERNATIVE(_lock_loc				\
-				 "call cmpxchg8b_emu",			\
-				 _lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \
-		     : "+a" (o.low), "+d" (o.high)			\
-		     : "b" (n.low), "c" (n.high), [ptr] "S" (_ptr)	\
-		     : "memory");					\
+	asm_inline volatile(						\
+		ALTERNATIVE(_lock_loc					\
+			    "call cmpxchg8b_emu",			\
+			    _lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8)	\
+		: ALT_OUTPUT_SP("+a" (o.low), "+d" (o.high))		\
+		: "b" (n.low), "c" (n.high),				\
+		  [ptr] "S" (_ptr)					\
+		: "memory");						\
 									\
 	o.full;								\
 })
 
 static __always_inline u64 arch_cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
 {
-	return __arch_cmpxchg64_emu(ptr, old, new, LOCK_PREFIX_HERE, "lock; ");
+	return __arch_cmpxchg64_emu(ptr, old, new, LOCK_PREFIX_HERE, "lock ");
 }
 #define arch_cmpxchg64 arch_cmpxchg64
 
@@ -119,14 +120,15 @@ static __always_inline u64 arch_cmpxchg64_local(volatile u64 *ptr, u64 old, u64
 			   n = { .full = (_new), };			\
 	bool ret;							\
 									\
-	asm volatile(ALTERNATIVE(_lock_loc				\
-				 "call cmpxchg8b_emu",			\
-				 _lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \
-		     CC_SET(e)						\
-		     : CC_OUT(e) (ret),					\
-		       "+a" (o.low), "+d" (o.high)			\
-		     : "b" (n.low), "c" (n.high), [ptr] "S" (_ptr)	\
-		     : "memory");					\
+	asm_inline volatile(						\
+		ALTERNATIVE(_lock_loc					\
+			    "call cmpxchg8b_emu",			\
+			    _lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \
+		: ALT_OUTPUT_SP("=@ccz" (ret),				\
+				"+a" (o.low), "+d" (o.high))		\
+		: "b" (n.low), "c" (n.high),				\
+		  [ptr] "S" (_ptr)					\
+		: "memory");						\
 									\
 	if (unlikely(!ret))						\
 		*(_oldp) = o.full;					\
@@ -136,7 +138,7 @@ static __always_inline u64 arch_cmpxchg64_local(volatile u64 *ptr, u64 old, u64
 
 static __always_inline bool arch_try_cmpxchg64(volatile u64 *ptr, u64 *oldp, u64 new)
 {
-	return __arch_try_cmpxchg64_emu(ptr, oldp, new, LOCK_PREFIX_HERE, "lock; ");
+	return __arch_try_cmpxchg64_emu(ptr, oldp, new, LOCK_PREFIX_HERE, "lock ");
 }
 #define arch_try_cmpxchg64 arch_try_cmpxchg64
 
diff --git a/arch/x86/include/asm/cmpxchg_64.h b/arch/x86/include/asm/cmpxchg_64.h
index 5e241306db26..5afea056fb20 100644
--- a/arch/x86/include/asm/cmpxchg_64.h
+++ b/arch/x86/include/asm/cmpxchg_64.h
@@ -38,7 +38,7 @@ union __u128_halves {
 	union __u128_halves o = { .full = (_old), },			\
 			    n = { .full = (_new), };			\
 									\
-	asm volatile(_lock "cmpxchg16b %[ptr]"				\
+	asm_inline volatile(_lock "cmpxchg16b %[ptr]"			\
 		     : [ptr] "+m" (*(_ptr)),				\
 		       "+a" (o.low), "+d" (o.high)			\
 		     : "b" (n.low), "c" (n.high)			\
@@ -65,9 +65,8 @@ static __always_inline u128 arch_cmpxchg128_local(volatile u128 *ptr, u128 old,
 			    n = { .full = (_new), };			\
 	bool ret;							\
 									\
-	asm volatile(_lock "cmpxchg16b %[ptr]"				\
-		     CC_SET(e)						\
-		     : CC_OUT(e) (ret),					\
+	asm_inline volatile(_lock "cmpxchg16b %[ptr]"			\
+		     : "=@ccz" (ret),					\
 		       [ptr] "+m" (*(_ptr)),				\
 		       "+a" (o.low), "+d" (o.high)			\
 		     : "b" (n.low), "c" (n.high)			\
diff --git a/arch/x86/include/asm/coco.h b/arch/x86/include/asm/coco.h
index aa6c8f8ca958..e1dbf8df1b69 100644
--- a/arch/x86/include/asm/coco.h
+++ b/arch/x86/include/asm/coco.h
@@ -15,9 +15,14 @@ enum cc_vendor {
 extern enum cc_vendor cc_vendor;
 extern u64 cc_mask;
 
+static inline u64 cc_get_mask(void)
+{
+	return cc_mask;
+}
+
 static inline void cc_set_mask(u64 mask)
 {
-	RIP_REL_REF(cc_mask) = mask;
+	cc_mask = mask;
 }
 
 u64 cc_mkenc(u64 val);
@@ -25,7 +30,10 @@ u64 cc_mkdec(u64 val);
 void cc_random_init(void);
 #else
 #define cc_vendor (CC_VENDOR_NONE)
-static const u64 cc_mask = 0;
+static inline u64 cc_get_mask(void)
+{
+	return 0;
+}
 
 static inline u64 cc_mkenc(u64 val)
 {
diff --git a/arch/x86/include/asm/cpu.h b/arch/x86/include/asm/cpu.h
index aa30fd8cad7f..ad235dda1ded 100644
--- a/arch/x86/include/asm/cpu.h
+++ b/arch/x86/include/asm/cpu.h
@@ -12,7 +12,6 @@
 #ifndef CONFIG_SMP
 #define cpu_physical_id(cpu)			boot_cpu_physical_apicid
 #define cpu_acpi_id(cpu)			0
-#define safe_smp_processor_id()			0
 #endif /* CONFIG_SMP */
 
 #ifdef CONFIG_HOTPLUG_CPU
@@ -26,12 +25,13 @@ int mwait_usable(const struct cpuinfo_x86 *);
 unsigned int x86_family(unsigned int sig);
 unsigned int x86_model(unsigned int sig);
 unsigned int x86_stepping(unsigned int sig);
-#ifdef CONFIG_CPU_SUP_INTEL
+#ifdef CONFIG_X86_BUS_LOCK_DETECT
 extern void __init sld_setup(struct cpuinfo_x86 *c);
 extern bool handle_user_split_lock(struct pt_regs *regs, long error_code);
 extern bool handle_guest_split_lock(unsigned long ip);
 extern void handle_bus_lock(struct pt_regs *regs);
-u8 get_this_hybrid_cpu_type(void);
+void split_lock_init(void);
+void bus_lock_init(void);
 #else
 static inline void __init sld_setup(struct cpuinfo_x86 *c) {}
 static inline bool handle_user_split_lock(struct pt_regs *regs, long error_code)
@@ -45,12 +45,10 @@ static inline bool handle_guest_split_lock(unsigned long ip)
 }
 
 static inline void handle_bus_lock(struct pt_regs *regs) {}
-
-static inline u8 get_this_hybrid_cpu_type(void)
-{
-	return 0;
-}
+static inline void split_lock_init(void) {}
+static inline void bus_lock_init(void) {}
 #endif
+
 #ifdef CONFIG_IA32_FEAT_CTL
 void init_ia32_feat_ctl(struct cpuinfo_x86 *c);
 #else
diff --git a/arch/x86/include/asm/cpu_device_id.h b/arch/x86/include/asm/cpu_device_id.h
index e4121d9aa9e1..6be777a06944 100644
--- a/arch/x86/include/asm/cpu_device_id.h
+++ b/arch/x86/include/asm/cpu_device_id.h
@@ -56,9 +56,8 @@
 /* x86_cpu_id::flags */
 #define X86_CPU_ID_FLAG_ENTRY_VALID	BIT(0)
 
-#define X86_STEPPINGS(mins, maxs)    GENMASK(maxs, mins)
 /**
- * X86_MATCH_VENDOR_FAM_MODEL_STEPPINGS_FEATURE - Base macro for CPU matching
+ * X86_MATCH_CPU -  Base macro for CPU matching
  * @_vendor:	The vendor name, e.g. INTEL, AMD, HYGON, ..., ANY
  *		The name is expanded to X86_VENDOR_@_vendor
  * @_family:	The family number or X86_FAMILY_ANY
@@ -75,47 +74,18 @@
  * into another macro at the usage site for good reasons, then please
  * start this local macro with X86_MATCH to allow easy grepping.
  */
-#define X86_MATCH_VENDOR_FAM_MODEL_STEPPINGS_FEATURE(_vendor, _family, _model, \
-						    _steppings, _feature, _data) { \
-	.vendor		= X86_VENDOR_##_vendor,				\
-	.family		= _family,					\
-	.model		= _model,					\
-	.steppings	= _steppings,					\
-	.feature	= _feature,					\
-	.flags		= X86_CPU_ID_FLAG_ENTRY_VALID,			\
-	.driver_data	= (unsigned long) _data				\
-}
-
-#define X86_MATCH_VENDORID_FAM_MODEL_STEPPINGS_FEATURE(_vendor, _family, _model, \
-						    _steppings, _feature, _data) { \
+#define X86_MATCH_CPU(_vendor, _family, _model, _steppings, _feature, _type, _data) { \
 	.vendor		= _vendor,					\
 	.family		= _family,					\
 	.model		= _model,					\
 	.steppings	= _steppings,					\
 	.feature	= _feature,					\
 	.flags		= X86_CPU_ID_FLAG_ENTRY_VALID,			\
+	.type		= _type,					\
 	.driver_data	= (unsigned long) _data				\
 }
 
 /**
- * X86_MATCH_VENDOR_FAM_MODEL_FEATURE - Macro for CPU matching
- * @_vendor:	The vendor name, e.g. INTEL, AMD, HYGON, ..., ANY
- *		The name is expanded to X86_VENDOR_@_vendor
- * @_family:	The family number or X86_FAMILY_ANY
- * @_model:	The model number, model constant or X86_MODEL_ANY
- * @_feature:	A X86_FEATURE bit or X86_FEATURE_ANY
- * @_data:	Driver specific data or NULL. The internal storage
- *		format is unsigned long. The supplied value, pointer
- *		etc. is casted to unsigned long internally.
- *
- * The steppings arguments of X86_MATCH_VENDOR_FAM_MODEL_STEPPINGS_FEATURE() is
- * set to wildcards.
- */
-#define X86_MATCH_VENDOR_FAM_MODEL_FEATURE(vendor, family, model, feature, data) \
-	X86_MATCH_VENDOR_FAM_MODEL_STEPPINGS_FEATURE(vendor, family, model, \
-						X86_STEPPING_ANY, feature, data)
-
-/**
  * X86_MATCH_VENDOR_FAM_FEATURE - Macro for matching vendor, family and CPU feature
  * @vendor:	The vendor name, e.g. INTEL, AMD, HYGON, ..., ANY
  *		The name is expanded to X86_VENDOR_@vendor
@@ -124,13 +94,10 @@
  * @data:	Driver specific data or NULL. The internal storage
  *		format is unsigned long. The supplied value, pointer
  *		etc. is casted to unsigned long internally.
- *
- * All other missing arguments of X86_MATCH_VENDOR_FAM_MODEL_FEATURE() are
- * set to wildcards.
  */
-#define X86_MATCH_VENDOR_FAM_FEATURE(vendor, family, feature, data)	\
-	X86_MATCH_VENDOR_FAM_MODEL_FEATURE(vendor, family,		\
-					   X86_MODEL_ANY, feature, data)
+#define X86_MATCH_VENDOR_FAM_FEATURE(vendor, family, feature, data)		\
+	X86_MATCH_CPU(X86_VENDOR_##vendor, family, X86_MODEL_ANY,		\
+		      X86_STEPPING_ANY, feature, X86_CPU_TYPE_ANY, data)
 
 /**
  * X86_MATCH_VENDOR_FEATURE - Macro for matching vendor and CPU feature
@@ -140,12 +107,10 @@
  * @data:	Driver specific data or NULL. The internal storage
  *		format is unsigned long. The supplied value, pointer
  *		etc. is casted to unsigned long internally.
- *
- * All other missing arguments of X86_MATCH_VENDOR_FAM_MODEL_FEATURE() are
- * set to wildcards.
  */
-#define X86_MATCH_VENDOR_FEATURE(vendor, feature, data)			\
-	X86_MATCH_VENDOR_FAM_FEATURE(vendor, X86_FAMILY_ANY, feature, data)
+#define X86_MATCH_VENDOR_FEATURE(vendor, feature, data)				\
+	X86_MATCH_CPU(X86_VENDOR_##vendor, X86_FAMILY_ANY, X86_MODEL_ANY,	\
+		      X86_STEPPING_ANY, feature, X86_CPU_TYPE_ANY, data)
 
 /**
  * X86_MATCH_FEATURE - Macro for matching a CPU feature
@@ -153,12 +118,10 @@
  * @data:	Driver specific data or NULL. The internal storage
  *		format is unsigned long. The supplied value, pointer
  *		etc. is casted to unsigned long internally.
- *
- * All other missing arguments of X86_MATCH_VENDOR_FAM_MODEL_FEATURE() are
- * set to wildcards.
  */
-#define X86_MATCH_FEATURE(feature, data)				\
-	X86_MATCH_VENDOR_FEATURE(ANY, feature, data)
+#define X86_MATCH_FEATURE(feature, data)					\
+	X86_MATCH_CPU(X86_VENDOR_ANY, X86_FAMILY_ANY, X86_MODEL_ANY,		\
+		      X86_STEPPING_ANY, feature, X86_CPU_TYPE_ANY, data)
 
 /**
  * X86_MATCH_VENDOR_FAM_MODEL - Match vendor, family and model
@@ -169,13 +132,10 @@
  * @data:	Driver specific data or NULL. The internal storage
  *		format is unsigned long. The supplied value, pointer
  *		etc. is casted to unsigned long internally.
- *
- * All other missing arguments of X86_MATCH_VENDOR_FAM_MODEL_FEATURE() are
- * set to wildcards.
  */
-#define X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, data)		\
-	X86_MATCH_VENDOR_FAM_MODEL_FEATURE(vendor, family, model,	\
-					   X86_FEATURE_ANY, data)
+#define X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, data)			\
+	X86_MATCH_CPU(X86_VENDOR_##vendor, family, model, X86_STEPPING_ANY,	\
+		      X86_FEATURE_ANY, X86_CPU_TYPE_ANY, data)
 
 /**
  * X86_MATCH_VENDOR_FAM - Match vendor and family
@@ -185,12 +145,10 @@
  * @data:	Driver specific data or NULL. The internal storage
  *		format is unsigned long. The supplied value, pointer
  *		etc. is casted to unsigned long internally.
- *
- * All other missing arguments to X86_MATCH_VENDOR_FAM_MODEL_FEATURE() are
- * set of wildcards.
  */
-#define X86_MATCH_VENDOR_FAM(vendor, family, data)			\
-	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, X86_MODEL_ANY, data)
+#define X86_MATCH_VENDOR_FAM(vendor, family, data)				\
+	X86_MATCH_CPU(X86_VENDOR_##vendor, family, X86_MODEL_ANY,		\
+		      X86_STEPPING_ANY, X86_FEATURE_ANY, X86_CPU_TYPE_ANY, data)
 
 /**
  * X86_MATCH_VFM - Match encoded vendor/family/model
@@ -198,32 +156,26 @@
  * @data:	Driver specific data or NULL. The internal storage
  *		format is unsigned long. The supplied value, pointer
  *		etc. is cast to unsigned long internally.
- *
- * Stepping and feature are set to wildcards
  */
-#define X86_MATCH_VFM(vfm, data)			\
-	X86_MATCH_VENDORID_FAM_MODEL_STEPPINGS_FEATURE(	\
-		VFM_VENDOR(vfm),			\
-		VFM_FAMILY(vfm),			\
-		VFM_MODEL(vfm),				\
-		X86_STEPPING_ANY, X86_FEATURE_ANY, data)
+#define X86_MATCH_VFM(vfm, data)						\
+	X86_MATCH_CPU(VFM_VENDOR(vfm), VFM_FAMILY(vfm),	VFM_MODEL(vfm),		\
+		      X86_STEPPING_ANY, X86_FEATURE_ANY, X86_CPU_TYPE_ANY, data)
 
+#define __X86_STEPPINGS(mins, maxs)    GENMASK(maxs, mins)
 /**
- * X86_MATCH_VFM_STEPPINGS - Match encoded vendor/family/model/stepping
+ * X86_MATCH_VFM_STEPS - Match encoded vendor/family/model and steppings
+ *			 range.
  * @vfm:	Encoded 8-bits each for vendor, family, model
- * @steppings:	Bitmask of steppings to match
+ * @min_step:	Lowest stepping number to match
+ * @max_step:	Highest stepping number to match
  * @data:	Driver specific data or NULL. The internal storage
  *		format is unsigned long. The supplied value, pointer
  *		etc. is cast to unsigned long internally.
- *
- * feature is set to wildcard
  */
-#define X86_MATCH_VFM_STEPPINGS(vfm, steppings, data)	\
-	X86_MATCH_VENDORID_FAM_MODEL_STEPPINGS_FEATURE(	\
-		VFM_VENDOR(vfm),			\
-		VFM_FAMILY(vfm),			\
-		VFM_MODEL(vfm),				\
-		steppings, X86_FEATURE_ANY, data)
+#define X86_MATCH_VFM_STEPS(vfm, min_step, max_step, data)			\
+	X86_MATCH_CPU(VFM_VENDOR(vfm), VFM_FAMILY(vfm), VFM_MODEL(vfm),		\
+		      __X86_STEPPINGS(min_step, max_step), X86_FEATURE_ANY,	\
+		      X86_CPU_TYPE_ANY, data)
 
 /**
  * X86_MATCH_VFM_FEATURE - Match encoded vendor/family/model/feature
@@ -232,51 +184,24 @@
  * @data:	Driver specific data or NULL. The internal storage
  *		format is unsigned long. The supplied value, pointer
  *		etc. is cast to unsigned long internally.
- *
- * Steppings is set to wildcard
  */
-#define X86_MATCH_VFM_FEATURE(vfm, feature, data)	\
-	X86_MATCH_VENDORID_FAM_MODEL_STEPPINGS_FEATURE(	\
-		VFM_VENDOR(vfm),			\
-		VFM_FAMILY(vfm),			\
-		VFM_MODEL(vfm),				\
-		X86_STEPPING_ANY, feature, data)
+#define X86_MATCH_VFM_FEATURE(vfm, feature, data)				\
+	X86_MATCH_CPU(VFM_VENDOR(vfm), VFM_FAMILY(vfm), VFM_MODEL(vfm),		\
+		      X86_STEPPING_ANY, feature, X86_CPU_TYPE_ANY, data)
 
-/*
- * Match specific microcode revisions.
- *
- * vendor/family/model/stepping must be all set.
- *
- * Only checks against the boot CPU.  When mixed-stepping configs are
- * valid for a CPU model, add a quirk for every valid stepping and
- * do the fine-tuning in the quirk handler.
+/**
+ * X86_MATCH_VFM_CPU_TYPE - Match encoded vendor/family/model/type
+ * @vfm:	Encoded 8-bits each for vendor, family, model
+ * @type:	CPU type e.g. P-core, E-core
+ * @data:	Driver specific data or NULL. The internal storage
+ *		format is unsigned long. The supplied value, pointer
+ *		etc. is cast to unsigned long internally.
  */
-
-struct x86_cpu_desc {
-	u8	x86_family;
-	u8	x86_vendor;
-	u8	x86_model;
-	u8	x86_stepping;
-	u32	x86_microcode_rev;
-};
-
-#define INTEL_CPU_DESC(vfm, stepping, revision) {		\
-	.x86_family		= VFM_FAMILY(vfm),		\
-	.x86_vendor		= VFM_VENDOR(vfm),		\
-	.x86_model		= VFM_MODEL(vfm),		\
-	.x86_stepping		= (stepping),			\
-	.x86_microcode_rev	= (revision),			\
-}
-
-#define AMD_CPU_DESC(fam, model, stepping, revision) {		\
-	.x86_family		= (fam),			\
-	.x86_vendor		= X86_VENDOR_AMD,		\
-	.x86_model		= (model),			\
-	.x86_stepping		= (stepping),			\
-	.x86_microcode_rev	= (revision),			\
-}
+#define X86_MATCH_VFM_CPU_TYPE(vfm, type, data)				\
+	X86_MATCH_CPU(VFM_VENDOR(vfm), VFM_FAMILY(vfm), VFM_MODEL(vfm),	\
+		      X86_STEPPING_ANY, X86_FEATURE_ANY, type, data)
 
 extern const struct x86_cpu_id *x86_match_cpu(const struct x86_cpu_id *match);
-extern bool x86_cpu_has_min_microcode_rev(const struct x86_cpu_desc *table);
+extern bool x86_match_min_microcode_rev(const struct x86_cpu_id *table);
 
 #endif /* _ASM_X86_CPU_DEVICE_ID */
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 0b9611da6c53..3ddc1d33399b 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -4,11 +4,12 @@
 
 #include <asm/processor.h>
 
-#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
+#if defined(__KERNEL__) && !defined(__ASSEMBLER__)
 
 #include <asm/asm.h>
 #include <linux/bitops.h>
 #include <asm/alternative.h>
+#include <asm/cpufeaturemasks.h>
 
 enum cpuid_leafs
 {
@@ -29,7 +30,7 @@ enum cpuid_leafs
 	CPUID_6_EAX,
 	CPUID_8000_000A_EDX,
 	CPUID_7_ECX,
-	CPUID_8000_0007_EBX,
+	CPUID_LNX_6,
 	CPUID_7_EDX,
 	CPUID_8000_001F_EAX,
 	CPUID_8000_0021_EAX,
@@ -37,92 +38,19 @@ enum cpuid_leafs
 	NR_CPUID_WORDS,
 };
 
-#define X86_CAP_FMT_NUM "%d:%d"
-#define x86_cap_flag_num(flag) ((flag) >> 5), ((flag) & 31)
-
 extern const char * const x86_cap_flags[NCAPINTS*32];
 extern const char * const x86_power_flags[32];
-#define X86_CAP_FMT "%s"
-#define x86_cap_flag(flag) x86_cap_flags[flag]
 
 /*
  * In order to save room, we index into this array by doing
  * X86_BUG_<name> - NCAPINTS*32.
  */
 extern const char * const x86_bug_flags[NBUGINTS*32];
+#define x86_bug_flag(flag) x86_bug_flags[flag]
 
 #define test_cpu_cap(c, bit)						\
 	 arch_test_bit(bit, (unsigned long *)((c)->x86_capability))
 
-/*
- * There are 32 bits/features in each mask word.  The high bits
- * (selected with (bit>>5) give us the word number and the low 5
- * bits give us the bit/feature number inside the word.
- * (1UL<<((bit)&31) gives us a mask for the feature_bit so we can
- * see if it is set in the mask word.
- */
-#define CHECK_BIT_IN_MASK_WORD(maskname, word, bit)	\
-	(((bit)>>5)==(word) && (1UL<<((bit)&31) & maskname##word ))
-
-/*
- * {REQUIRED,DISABLED}_MASK_CHECK below may seem duplicated with the
- * following BUILD_BUG_ON_ZERO() check but when NCAPINTS gets changed, all
- * header macros which use NCAPINTS need to be changed. The duplicated macro
- * use causes the compiler to issue errors for all headers so that all usage
- * sites can be corrected.
- */
-#define REQUIRED_MASK_BIT_SET(feature_bit)		\
-	 ( CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  0, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  1, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  2, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  3, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  4, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  5, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  6, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  7, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  8, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  9, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 10, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 11, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 12, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 13, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 14, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 15, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 16, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 17, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 18, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 19, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 20, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 21, feature_bit) ||	\
-	   REQUIRED_MASK_CHECK					  ||	\
-	   BUILD_BUG_ON_ZERO(NCAPINTS != 22))
-
-#define DISABLED_MASK_BIT_SET(feature_bit)				\
-	 ( CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  0, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  1, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  2, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  3, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  4, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  5, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  6, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  7, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  8, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  9, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 10, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 11, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 12, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 13, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 14, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 15, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 16, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 17, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 18, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 19, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 20, feature_bit) ||	\
-	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 21, feature_bit) ||	\
-	   DISABLED_MASK_CHECK					  ||	\
-	   BUILD_BUG_ON_ZERO(NCAPINTS != 22))
-
 #define cpu_has(c, bit)							\
 	(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :	\
 	 test_cpu_cap(c, bit))
@@ -132,11 +60,12 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 	 x86_this_cpu_test_bit(bit, cpu_info.x86_capability))
 
 /*
- * This macro is for detection of features which need kernel
- * infrastructure to be used.  It may *not* directly test the CPU
- * itself.  Use the cpu_has() family if you want true runtime
- * testing of CPU features, like in hypervisor code where you are
- * supporting a possible guest feature where host support for it
+ * This is the default CPU features testing macro to use in code.
+ *
+ * It is for detection of features which need kernel infrastructure to be
+ * used.  It may *not* directly test the CPU itself.  Use the cpu_has() family
+ * if you want true runtime testing of CPU features, like in hypervisor code
+ * where you are supporting a possible guest feature where host support for it
  * is not relevant.
  */
 #define cpu_feature_enabled(bit)	\
@@ -148,6 +77,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 
 extern void setup_clear_cpu_cap(unsigned int bit);
 extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit);
+void check_cpufeature_deps(struct cpuinfo_x86 *c);
 
 #define setup_force_cpu_cap(bit) do {			\
 							\
@@ -161,13 +91,6 @@ extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit);
 #define setup_force_cpu_bug(bit) setup_force_cpu_cap(bit)
 
 /*
- * Static testing of CPU features. Used the same as boot_cpu_has(). It
- * statically patches the target code for additional performance. Use
- * static_cpu_has() only in fast paths, where every cycle counts. Which
- * means that the boot_cpu_has() variant is already fast enough for the
- * majority of cases and you should stick to using it as it is generally
- * only two instructions: a RIP-relative MOV and a TEST.
- *
  * Do not use an "m" constraint for [cap_byte] here: gcc doesn't know
  * that this is only used on a fallback path and will sometimes cause
  * it to manifest the address of boot_cpu_data in a register, fouling
@@ -178,6 +101,7 @@ static __always_inline bool _static_cpu_has(u16 bit)
 	asm goto(ALTERNATIVE_TERNARY("jmp 6f", %c[feature], "", "jmp %l[t_no]")
 		".pushsection .altinstr_aux,\"ax\"\n"
 		"6:\n"
+		ANNOTATE_DATA_SPECIAL "\n"
 		" testb %[bitnum], %a[cap_byte]\n"
 		" jnz %l[t_yes]\n"
 		" jmp %l[t_no]\n"
@@ -214,5 +138,5 @@ t_no:
 #define CPU_FEATURE_TYPEVAL		boot_cpu_data.x86_vendor, boot_cpu_data.x86, \
 					boot_cpu_data.x86_model
 
-#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */
+#endif /* defined(__KERNEL__) && !defined(__ASSEMBLER__) */
 #endif /* _ASM_X86_CPUFEATURE_H */
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 913fd3a7bac6..c3b53beb1300 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -2,14 +2,6 @@
 #ifndef _ASM_X86_CPUFEATURES_H
 #define _ASM_X86_CPUFEATURES_H
 
-#ifndef _ASM_X86_REQUIRED_FEATURES_H
-#include <asm/required-features.h>
-#endif
-
-#ifndef _ASM_X86_DISABLED_FEATURES_H
-#include <asm/disabled-features.h>
-#endif
-
 /*
  * Defines x86 CPU feature bits
  */
@@ -83,8 +75,8 @@
 #define X86_FEATURE_CENTAUR_MCR		( 3*32+ 3) /* "centaur_mcr" Centaur MCRs (= MTRRs) */
 #define X86_FEATURE_K8			( 3*32+ 4) /* Opteron, Athlon64 */
 #define X86_FEATURE_ZEN5		( 3*32+ 5) /* CPU based on Zen5 microarchitecture */
-#define X86_FEATURE_P3			( 3*32+ 6) /* P3 */
-#define X86_FEATURE_P4			( 3*32+ 7) /* P4 */
+#define X86_FEATURE_ZEN6		( 3*32+ 6) /* CPU based on Zen6 microarchitecture */
+/* Free                                 ( 3*32+ 7) */
 #define X86_FEATURE_CONSTANT_TSC	( 3*32+ 8) /* "constant_tsc" TSC ticks at a constant rate */
 #define X86_FEATURE_UP			( 3*32+ 9) /* "up" SMP kernel running on UP */
 #define X86_FEATURE_ART			( 3*32+10) /* "art" Always running timer (ART) */
@@ -210,7 +202,6 @@
 #define X86_FEATURE_MBA			( 7*32+18) /* "mba" Memory Bandwidth Allocation */
 #define X86_FEATURE_RSB_CTXSW		( 7*32+19) /* Fill RSB on context switches */
 #define X86_FEATURE_PERFMON_V2		( 7*32+20) /* "perfmon_v2" AMD Performance Monitoring Version 2 */
-#define X86_FEATURE_USE_IBPB		( 7*32+21) /* Indirect Branch Prediction Barrier enabled */
 #define X86_FEATURE_USE_IBRS_FW		( 7*32+22) /* Use IBRS during runtime firmware calls */
 #define X86_FEATURE_SPEC_STORE_BYPASS_DISABLE	( 7*32+23) /* Disable Speculative Store Bypass. */
 #define X86_FEATURE_LS_CFG_SSBD		( 7*32+24)  /* AMD SSBD implementation via LS_CFG MSR */
@@ -227,6 +218,7 @@
 #define X86_FEATURE_FLEXPRIORITY	( 8*32+ 1) /* "flexpriority" Intel FlexPriority */
 #define X86_FEATURE_EPT			( 8*32+ 2) /* "ept" Intel Extended Page Table */
 #define X86_FEATURE_VPID		( 8*32+ 3) /* "vpid" Intel Virtual Processor ID */
+#define X86_FEATURE_COHERENCY_SFW_NO	( 8*32+ 4) /* SNP cache coherency software work around not needed */
 
 #define X86_FEATURE_VMMCALL		( 8*32+15) /* "vmmcall" Prefer VMMCALL to VMCALL */
 #define X86_FEATURE_XENPV		( 8*32+16) /* Xen paravirtual guest */
@@ -317,15 +309,19 @@
 #define X86_FEATURE_ZEN1		(11*32+31) /* CPU based on Zen1 microarchitecture */
 
 /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */
+#define X86_FEATURE_SHA512		(12*32+ 0) /* SHA512 instructions */
+#define X86_FEATURE_SM3			(12*32+ 1) /* SM3 instructions */
+#define X86_FEATURE_SM4			(12*32+ 2) /* SM4 instructions */
 #define X86_FEATURE_AVX_VNNI		(12*32+ 4) /* "avx_vnni" AVX VNNI instructions */
 #define X86_FEATURE_AVX512_BF16		(12*32+ 5) /* "avx512_bf16" AVX512 BFLOAT16 instructions */
+#define X86_FEATURE_LASS		(12*32+ 6) /* "lass" Linear Address Space Separation */
 #define X86_FEATURE_CMPCCXADD           (12*32+ 7) /* CMPccXADD instructions */
 #define X86_FEATURE_ARCH_PERFMON_EXT	(12*32+ 8) /* Intel Architectural PerfMon Extension */
 #define X86_FEATURE_FZRM		(12*32+10) /* Fast zero-length REP MOVSB */
 #define X86_FEATURE_FSRS		(12*32+11) /* Fast short REP STOSB */
 #define X86_FEATURE_FSRC		(12*32+12) /* Fast short REP {CMPSB,SCASB} */
 #define X86_FEATURE_FRED		(12*32+17) /* "fred" Flexible Return and Event Delivery */
-#define X86_FEATURE_LKGS		(12*32+18) /* Load "kernel" (userspace) GS */
+#define X86_FEATURE_LKGS		(12*32+18) /* Like MOV_GS except MSR_KERNEL_GS_BASE = GS.base */
 #define X86_FEATURE_WRMSRNS		(12*32+19) /* Non-serializing WRMSR */
 #define X86_FEATURE_AMX_FP16		(12*32+21) /* AMX fp16 Support */
 #define X86_FEATURE_AVX_IFMA            (12*32+23) /* Support for VPMADD52[H,L]UQ */
@@ -335,12 +331,15 @@
 #define X86_FEATURE_CLZERO		(13*32+ 0) /* "clzero" CLZERO instruction */
 #define X86_FEATURE_IRPERF		(13*32+ 1) /* "irperf" Instructions Retired Count */
 #define X86_FEATURE_XSAVEERPTR		(13*32+ 2) /* "xsaveerptr" Always save/restore FP error pointers */
+#define X86_FEATURE_INVLPGB		(13*32+ 3) /* INVLPGB and TLBSYNC instructions supported */
 #define X86_FEATURE_RDPRU		(13*32+ 4) /* "rdpru" Read processor register at user level */
 #define X86_FEATURE_WBNOINVD		(13*32+ 9) /* "wbnoinvd" WBNOINVD instruction */
 #define X86_FEATURE_AMD_IBPB		(13*32+12) /* Indirect Branch Prediction Barrier */
 #define X86_FEATURE_AMD_IBRS		(13*32+14) /* Indirect Branch Restricted Speculation */
 #define X86_FEATURE_AMD_STIBP		(13*32+15) /* Single Thread Indirect Branch Predictors */
 #define X86_FEATURE_AMD_STIBP_ALWAYS_ON	(13*32+17) /* Single Thread Indirect Branch Predictors always-on preferred */
+#define X86_FEATURE_AMD_IBRS_SAME_MODE	(13*32+19) /* Indirect Branch Restricted Speculation same mode protection*/
+#define X86_FEATURE_EFER_LMSLE_MBZ	(13*32+20) /* EFER.LMSLE must be zero */
 #define X86_FEATURE_AMD_PPIN		(13*32+23) /* "amd_ppin" Protected Processor Inventory Number */
 #define X86_FEATURE_AMD_SSBD		(13*32+24) /* Speculative Store Bypass Disable */
 #define X86_FEATURE_VIRT_SSBD		(13*32+25) /* "virt_ssbd" Virtualized Speculative Store Bypass Disable */
@@ -383,6 +382,8 @@
 #define X86_FEATURE_V_SPEC_CTRL		(15*32+20) /* "v_spec_ctrl" Virtual SPEC_CTRL */
 #define X86_FEATURE_VNMI		(15*32+25) /* "vnmi" Virtual NMI */
 #define X86_FEATURE_SVME_ADDR_CHK	(15*32+28) /* SVME addr check */
+#define X86_FEATURE_BUS_LOCK_THRESHOLD	(15*32+29) /* Bus lock threshold */
+#define X86_FEATURE_IDLE_HLT		(15*32+30) /* IDLE HLT intercept */
 
 /* Intel-defined CPU features, CPUID level 0x00000007:0 (ECX), word 16 */
 #define X86_FEATURE_AVX512VBMI		(16*32+ 1) /* "avx512vbmi" AVX512 Vector Bit Manipulation instructions*/
@@ -408,9 +409,12 @@
 #define X86_FEATURE_ENQCMD		(16*32+29) /* "enqcmd" ENQCMD and ENQCMDS instructions */
 #define X86_FEATURE_SGX_LC		(16*32+30) /* "sgx_lc" Software Guard Extensions Launch Control */
 
-/* AMD-defined CPU features, CPUID level 0x80000007 (EBX), word 17 */
+/*
+ * Linux-defined word for use with scattered/synthetic bits.
+ */
 #define X86_FEATURE_OVERFLOW_RECOV	(17*32+ 0) /* "overflow_recov" MCA overflow recovery support */
 #define X86_FEATURE_SUCCOR		(17*32+ 1) /* "succor" Uncorrectable error containment and recovery */
+
 #define X86_FEATURE_SMCA		(17*32+ 3) /* "smca" Scalable MCA */
 
 /* Intel-defined CPU features, CPUID level 0x00000007:0 (EDX), word 18 */
@@ -440,27 +444,42 @@
 #define X86_FEATURE_SPEC_CTRL_SSBD	(18*32+31) /* Speculative Store Bypass Disable */
 
 /* AMD-defined memory encryption features, CPUID level 0x8000001f (EAX), word 19 */
-#define X86_FEATURE_SME			(19*32+ 0) /* "sme" AMD Secure Memory Encryption */
-#define X86_FEATURE_SEV			(19*32+ 1) /* "sev" AMD Secure Encrypted Virtualization */
+#define X86_FEATURE_SME			(19*32+ 0) /* "sme" Secure Memory Encryption */
+#define X86_FEATURE_SEV			(19*32+ 1) /* "sev" Secure Encrypted Virtualization */
 #define X86_FEATURE_VM_PAGE_FLUSH	(19*32+ 2) /* VM Page Flush MSR is supported */
-#define X86_FEATURE_SEV_ES		(19*32+ 3) /* "sev_es" AMD Secure Encrypted Virtualization - Encrypted State */
-#define X86_FEATURE_SEV_SNP		(19*32+ 4) /* "sev_snp" AMD Secure Encrypted Virtualization - Secure Nested Paging */
+#define X86_FEATURE_SEV_ES		(19*32+ 3) /* "sev_es" Secure Encrypted Virtualization - Encrypted State */
+#define X86_FEATURE_SEV_SNP		(19*32+ 4) /* "sev_snp" Secure Encrypted Virtualization - Secure Nested Paging */
+#define X86_FEATURE_SNP_SECURE_TSC	(19*32+ 8) /* SEV-SNP Secure TSC */
 #define X86_FEATURE_V_TSC_AUX		(19*32+ 9) /* Virtual TSC_AUX */
-#define X86_FEATURE_SME_COHERENT	(19*32+10) /* AMD hardware-enforced cache coherency */
-#define X86_FEATURE_DEBUG_SWAP		(19*32+14) /* "debug_swap" AMD SEV-ES full debug state swap support */
+#define X86_FEATURE_SME_COHERENT	(19*32+10) /* hardware-enforced cache coherency */
+#define X86_FEATURE_DEBUG_SWAP		(19*32+14) /* "debug_swap" SEV-ES full debug state swap support */
+#define X86_FEATURE_RMPREAD		(19*32+21) /* RMPREAD instruction */
+#define X86_FEATURE_SEGMENTED_RMP	(19*32+23) /* Segmented RMP support */
+#define X86_FEATURE_ALLOWED_SEV_FEATURES (19*32+27) /* Allowed SEV Features */
 #define X86_FEATURE_SVSM		(19*32+28) /* "svsm" SVSM present */
+#define X86_FEATURE_HV_INUSE_WR_ALLOWED	(19*32+30) /* Allow Write to in-use hypervisor-owned pages */
 
 /* AMD-defined Extended Feature 2 EAX, CPUID level 0x80000021 (EAX), word 20 */
 #define X86_FEATURE_NO_NESTED_DATA_BP	(20*32+ 0) /* No Nested Data Breakpoints */
 #define X86_FEATURE_WRMSR_XX_BASE_NS	(20*32+ 1) /* WRMSR to {FS,GS,KERNEL_GS}_BASE is non-serializing */
 #define X86_FEATURE_LFENCE_RDTSC	(20*32+ 2) /* LFENCE always serializing / synchronizes RDTSC */
+#define X86_FEATURE_VERW_CLEAR		(20*32+ 5) /* The memory form of VERW mitigates TSA */
 #define X86_FEATURE_NULL_SEL_CLR_BASE	(20*32+ 6) /* Null Selector Clears Base */
+
 #define X86_FEATURE_AUTOIBRS		(20*32+ 8) /* Automatic IBRS */
 #define X86_FEATURE_NO_SMM_CTL_MSR	(20*32+ 9) /* SMM_CTL MSR is not present */
 
+#define X86_FEATURE_GP_ON_USER_CPUID	(20*32+17) /* User CPUID faulting */
+
+#define X86_FEATURE_PREFETCHI		(20*32+20) /* Prefetch Data/Instruction to Cache Level */
 #define X86_FEATURE_SBPB		(20*32+27) /* Selective Branch Prediction Barrier */
 #define X86_FEATURE_IBPB_BRTYPE		(20*32+28) /* MSR_PRED_CMD[IBPB] flushes all branch type predictions */
 #define X86_FEATURE_SRSO_NO		(20*32+29) /* CPU is not affected by SRSO */
+#define X86_FEATURE_SRSO_USER_KERNEL_NO	(20*32+30) /* CPU is not affected by SRSO across user/kernel boundaries */
+#define X86_FEATURE_SRSO_BP_SPEC_REDUCE	(20*32+31) /*
+						    * BP_CFG[BpSpecReduce] can be used to mitigate SRSO for VMs.
+						    * (SRSO_MSR_FIX in the official doc).
+						    */
 
 /*
  * Extended auxiliary flags: Linux defined - for features scattered in various
@@ -472,8 +491,28 @@
 #define X86_FEATURE_CLEAR_BHB_LOOP	(21*32+ 1) /* Clear branch history at syscall entry using SW loop */
 #define X86_FEATURE_BHI_CTRL		(21*32+ 2) /* BHI_DIS_S HW control available */
 #define X86_FEATURE_CLEAR_BHB_HW	(21*32+ 3) /* BHI_DIS_S HW control enabled */
-#define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* Clear branch history at vmexit using SW loop */
-#define X86_FEATURE_FAST_CPPC		(21*32 + 5) /* AMD Fast CPPC */
+#define X86_FEATURE_CLEAR_BHB_VMEXIT	(21*32+ 4) /* Clear branch history at vmexit using SW loop */
+#define X86_FEATURE_AMD_FAST_CPPC	(21*32+ 5) /* Fast CPPC */
+#define X86_FEATURE_AMD_HTR_CORES	(21*32+ 6) /* Heterogeneous Core Topology */
+#define X86_FEATURE_AMD_WORKLOAD_CLASS	(21*32+ 7) /* Workload Classification */
+#define X86_FEATURE_PREFER_YMM		(21*32+ 8) /* Avoid ZMM registers due to downclocking */
+#define X86_FEATURE_APX			(21*32+ 9) /* Advanced Performance Extensions */
+#define X86_FEATURE_INDIRECT_THUNK_ITS	(21*32+10) /* Use thunk for indirect branches in lower half of cacheline */
+#define X86_FEATURE_TSA_SQ_NO		(21*32+11) /* AMD CPU not vulnerable to TSA-SQ */
+#define X86_FEATURE_TSA_L1_NO		(21*32+12) /* AMD CPU not vulnerable to TSA-L1 */
+#define X86_FEATURE_CLEAR_CPU_BUF_VM	(21*32+13) /* Clear CPU buffers using VERW before VMRUN */
+#define X86_FEATURE_IBPB_EXIT_TO_USER	(21*32+14) /* Use IBPB on exit-to-userspace, see VMSCAPE bug */
+#define X86_FEATURE_ABMC		(21*32+15) /* Assignable Bandwidth Monitoring Counters */
+#define X86_FEATURE_MSR_IMM		(21*32+16) /* MSR immediate form instructions */
+#define X86_FEATURE_SGX_EUPDATESVN	(21*32+17) /* Support for ENCLS[EUPDATESVN] instruction */
+
+#define X86_FEATURE_SDCIAE		(21*32+18) /* L3 Smart Data Cache Injection Allocation Enforcement */
+#define X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO (21*32+19) /*
+						      * Clear CPU buffers before VM-Enter if the vCPU
+						      * can access host MMIO (ignored for all intents
+						      * and purposes if CLEAR_CPU_BUF_VM is set).
+						      */
+#define X86_FEATURE_X2AVIC_EXT		(21*32+20) /* AMD SVM x2AVIC support for 4k vCPUs */
 
 /*
  * BUG word(s)
@@ -512,7 +551,7 @@
 #define X86_BUG_ITLB_MULTIHIT		X86_BUG(23) /* "itlb_multihit" CPU may incur MCE during certain page attribute changes */
 #define X86_BUG_SRBDS			X86_BUG(24) /* "srbds" CPU may leak RNG bits if not mitigated */
 #define X86_BUG_MMIO_STALE_DATA		X86_BUG(25) /* "mmio_stale_data" CPU is affected by Processor MMIO Stale Data vulnerabilities */
-#define X86_BUG_MMIO_UNKNOWN		X86_BUG(26) /* "mmio_unknown" CPU is too old and its MMIO Stale Data status is unknown */
+/* unused, was #define X86_BUG_MMIO_UNKNOWN		X86_BUG(26) "mmio_unknown" CPU is too old and its MMIO Stale Data status is unknown */
 #define X86_BUG_RETBLEED		X86_BUG(27) /* "retbleed" CPU is affected by RETBleed */
 #define X86_BUG_EIBRS_PBRSB		X86_BUG(28) /* "eibrs_pbrsb" EIBRS is vulnerable to Post Barrier RSB Predictions */
 #define X86_BUG_SMT_RSB			X86_BUG(29) /* "smt_rsb" CPU is vulnerable to Cross-Thread Return Address Predictions */
@@ -520,9 +559,15 @@
 #define X86_BUG_TDX_PW_MCE		X86_BUG(31) /* "tdx_pw_mce" CPU may incur #MC if non-TD software does partial write to TDX private memory */
 
 /* BUG word 2 */
-#define X86_BUG_SRSO			X86_BUG(1*32 + 0) /* "srso" AMD SRSO bug */
-#define X86_BUG_DIV0			X86_BUG(1*32 + 1) /* "div0" AMD DIV0 speculation bug */
-#define X86_BUG_RFDS			X86_BUG(1*32 + 2) /* "rfds" CPU is vulnerable to Register File Data Sampling */
-#define X86_BUG_BHI			X86_BUG(1*32 + 3) /* "bhi" CPU is affected by Branch History Injection */
-#define X86_BUG_IBPB_NO_RET	   	X86_BUG(1*32 + 4) /* "ibpb_no_ret" IBPB omits return target predictions */
+#define X86_BUG_SRSO			X86_BUG( 1*32+ 0) /* "srso" AMD SRSO bug */
+#define X86_BUG_DIV0			X86_BUG( 1*32+ 1) /* "div0" AMD DIV0 speculation bug */
+#define X86_BUG_RFDS			X86_BUG( 1*32+ 2) /* "rfds" CPU is vulnerable to Register File Data Sampling */
+#define X86_BUG_BHI			X86_BUG( 1*32+ 3) /* "bhi" CPU is affected by Branch History Injection */
+#define X86_BUG_IBPB_NO_RET		X86_BUG( 1*32+ 4) /* "ibpb_no_ret" IBPB omits return target predictions */
+#define X86_BUG_SPECTRE_V2_USER		X86_BUG( 1*32+ 5) /* "spectre_v2_user" CPU is affected by Spectre variant 2 attack between user processes */
+#define X86_BUG_OLD_MICROCODE		X86_BUG( 1*32+ 6) /* "old_microcode" CPU has old microcode, it is surely vulnerable to something */
+#define X86_BUG_ITS			X86_BUG( 1*32+ 7) /* "its" CPU is affected by Indirect Target Selection */
+#define X86_BUG_ITS_NATIVE_ONLY		X86_BUG( 1*32+ 8) /* "its_native_only" CPU is affected by ITS, VMX is not affected */
+#define X86_BUG_TSA			X86_BUG( 1*32+ 9) /* "tsa" CPU is affected by Transient Scheduler Attacks */
+#define X86_BUG_VMSCAPE			X86_BUG( 1*32+10) /* "vmscape" CPU is affected by VMSCAPE attacks from guests */
 #endif /* _ASM_X86_CPUFEATURES_H */
diff --git a/arch/x86/include/asm/cpuid.h b/arch/x86/include/asm/cpuid.h
deleted file mode 100644
index ca4243318aad..000000000000
--- a/arch/x86/include/asm/cpuid.h
+++ /dev/null
@@ -1,213 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * CPUID-related helpers/definitions
- */
-
-#ifndef _ASM_X86_CPUID_H
-#define _ASM_X86_CPUID_H
-
-#include <asm/string.h>
-
-struct cpuid_regs {
-	u32 eax, ebx, ecx, edx;
-};
-
-enum cpuid_regs_idx {
-	CPUID_EAX = 0,
-	CPUID_EBX,
-	CPUID_ECX,
-	CPUID_EDX,
-};
-
-#ifdef CONFIG_X86_32
-extern int have_cpuid_p(void);
-#else
-static inline int have_cpuid_p(void)
-{
-	return 1;
-}
-#endif
-static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
-				unsigned int *ecx, unsigned int *edx)
-{
-	/* ecx is often an input as well as an output. */
-	asm volatile("cpuid"
-	    : "=a" (*eax),
-	      "=b" (*ebx),
-	      "=c" (*ecx),
-	      "=d" (*edx)
-	    : "0" (*eax), "2" (*ecx)
-	    : "memory");
-}
-
-#define native_cpuid_reg(reg)					\
-static inline unsigned int native_cpuid_##reg(unsigned int op)	\
-{								\
-	unsigned int eax = op, ebx, ecx = 0, edx;		\
-								\
-	native_cpuid(&eax, &ebx, &ecx, &edx);			\
-								\
-	return reg;						\
-}
-
-/*
- * Native CPUID functions returning a single datum.
- */
-native_cpuid_reg(eax)
-native_cpuid_reg(ebx)
-native_cpuid_reg(ecx)
-native_cpuid_reg(edx)
-
-#ifdef CONFIG_PARAVIRT_XXL
-#include <asm/paravirt.h>
-#else
-#define __cpuid			native_cpuid
-#endif
-
-/*
- * Generic CPUID function
- * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
- * resulting in stale register contents being returned.
- */
-static inline void cpuid(unsigned int op,
-			 unsigned int *eax, unsigned int *ebx,
-			 unsigned int *ecx, unsigned int *edx)
-{
-	*eax = op;
-	*ecx = 0;
-	__cpuid(eax, ebx, ecx, edx);
-}
-
-/* Some CPUID calls want 'count' to be placed in ecx */
-static inline void cpuid_count(unsigned int op, int count,
-			       unsigned int *eax, unsigned int *ebx,
-			       unsigned int *ecx, unsigned int *edx)
-{
-	*eax = op;
-	*ecx = count;
-	__cpuid(eax, ebx, ecx, edx);
-}
-
-/*
- * CPUID functions returning a single datum
- */
-static inline unsigned int cpuid_eax(unsigned int op)
-{
-	unsigned int eax, ebx, ecx, edx;
-
-	cpuid(op, &eax, &ebx, &ecx, &edx);
-
-	return eax;
-}
-
-static inline unsigned int cpuid_ebx(unsigned int op)
-{
-	unsigned int eax, ebx, ecx, edx;
-
-	cpuid(op, &eax, &ebx, &ecx, &edx);
-
-	return ebx;
-}
-
-static inline unsigned int cpuid_ecx(unsigned int op)
-{
-	unsigned int eax, ebx, ecx, edx;
-
-	cpuid(op, &eax, &ebx, &ecx, &edx);
-
-	return ecx;
-}
-
-static inline unsigned int cpuid_edx(unsigned int op)
-{
-	unsigned int eax, ebx, ecx, edx;
-
-	cpuid(op, &eax, &ebx, &ecx, &edx);
-
-	return edx;
-}
-
-static inline void __cpuid_read(unsigned int leaf, unsigned int subleaf, u32 *regs)
-{
-	regs[CPUID_EAX] = leaf;
-	regs[CPUID_ECX] = subleaf;
-	__cpuid(regs + CPUID_EAX, regs + CPUID_EBX, regs + CPUID_ECX, regs + CPUID_EDX);
-}
-
-#define cpuid_subleaf(leaf, subleaf, regs) {		\
-	static_assert(sizeof(*(regs)) == 16);		\
-	__cpuid_read(leaf, subleaf, (u32 *)(regs));	\
-}
-
-#define cpuid_leaf(leaf, regs) {			\
-	static_assert(sizeof(*(regs)) == 16);		\
-	__cpuid_read(leaf, 0, (u32 *)(regs));		\
-}
-
-static inline void __cpuid_read_reg(unsigned int leaf, unsigned int subleaf,
-				    enum cpuid_regs_idx regidx, u32 *reg)
-{
-	u32 regs[4];
-
-	__cpuid_read(leaf, subleaf, regs);
-	*reg = regs[regidx];
-}
-
-#define cpuid_subleaf_reg(leaf, subleaf, regidx, reg) {		\
-	static_assert(sizeof(*(reg)) == 4);			\
-	__cpuid_read_reg(leaf, subleaf, regidx, (u32 *)(reg));	\
-}
-
-#define cpuid_leaf_reg(leaf, regidx, reg) {			\
-	static_assert(sizeof(*(reg)) == 4);			\
-	__cpuid_read_reg(leaf, 0, regidx, (u32 *)(reg));	\
-}
-
-static __always_inline bool cpuid_function_is_indexed(u32 function)
-{
-	switch (function) {
-	case 4:
-	case 7:
-	case 0xb:
-	case 0xd:
-	case 0xf:
-	case 0x10:
-	case 0x12:
-	case 0x14:
-	case 0x17:
-	case 0x18:
-	case 0x1d:
-	case 0x1e:
-	case 0x1f:
-	case 0x24:
-	case 0x8000001d:
-		return true;
-	}
-
-	return false;
-}
-
-#define for_each_possible_hypervisor_cpuid_base(function) \
-	for (function = 0x40000000; function < 0x40010000; function += 0x100)
-
-static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves)
-{
-	uint32_t base, eax, signature[3];
-
-	for_each_possible_hypervisor_cpuid_base(base) {
-		cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);
-
-		/*
-		 * This must not compile to "call memcmp" because it's called
-		 * from PVH early boot code before instrumentation is set up
-		 * and memcmp() itself may be instrumented.
-		 */
-		if (!__builtin_memcmp(sig, signature, 12) &&
-		    (leaves == 0 || ((eax - base) >= leaves)))
-			return base;
-	}
-
-	return 0;
-}
-
-#endif /* _ASM_X86_CPUID_H */
diff --git a/arch/x86/include/asm/cpuid/api.h b/arch/x86/include/asm/cpuid/api.h
new file mode 100644
index 000000000000..44fa82e1267c
--- /dev/null
+++ b/arch/x86/include/asm/cpuid/api.h
@@ -0,0 +1,292 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_CPUID_API_H
+#define _ASM_X86_CPUID_API_H
+
+#include <asm/cpuid/types.h>
+
+#include <linux/build_bug.h>
+#include <linux/types.h>
+
+#include <asm/string.h>
+
+/*
+ * Raw CPUID accessors:
+ */
+
+#ifdef CONFIG_X86_32
+bool cpuid_feature(void);
+#else
+static inline bool cpuid_feature(void)
+{
+	return true;
+}
+#endif
+
+static inline void native_cpuid(u32 *eax, u32 *ebx,
+				u32 *ecx, u32 *edx)
+{
+	/* ecx is often an input as well as an output. */
+	asm volatile("cpuid"
+	    : "=a" (*eax),
+	      "=b" (*ebx),
+	      "=c" (*ecx),
+	      "=d" (*edx)
+	    : "0" (*eax), "2" (*ecx)
+	    : "memory");
+}
+
+#define NATIVE_CPUID_REG(reg)					\
+static inline u32 native_cpuid_##reg(u32 op)			\
+{								\
+	u32 eax = op, ebx, ecx = 0, edx;			\
+								\
+	native_cpuid(&eax, &ebx, &ecx, &edx);			\
+								\
+	return reg;						\
+}
+
+/*
+ * Native CPUID functions returning a single datum:
+ */
+NATIVE_CPUID_REG(eax)
+NATIVE_CPUID_REG(ebx)
+NATIVE_CPUID_REG(ecx)
+NATIVE_CPUID_REG(edx)
+
+#ifdef CONFIG_PARAVIRT_XXL
+# include <asm/paravirt.h>
+#else
+# define __cpuid native_cpuid
+#endif
+
+/*
+ * Generic CPUID function
+ *
+ * Clear ECX since some CPUs (Cyrix MII) do not set or clear ECX
+ * resulting in stale register contents being returned.
+ */
+static inline void cpuid(u32 op,
+			 u32 *eax, u32 *ebx,
+			 u32 *ecx, u32 *edx)
+{
+	*eax = op;
+	*ecx = 0;
+	__cpuid(eax, ebx, ecx, edx);
+}
+
+/* Some CPUID calls want 'count' to be placed in ECX */
+static inline void cpuid_count(u32 op, int count,
+			       u32 *eax, u32 *ebx,
+			       u32 *ecx, u32 *edx)
+{
+	*eax = op;
+	*ecx = count;
+	__cpuid(eax, ebx, ecx, edx);
+}
+
+/*
+ * CPUID functions returning a single datum:
+ */
+
+static inline u32 cpuid_eax(u32 op)
+{
+	u32 eax, ebx, ecx, edx;
+
+	cpuid(op, &eax, &ebx, &ecx, &edx);
+
+	return eax;
+}
+
+static inline u32 cpuid_ebx(u32 op)
+{
+	u32 eax, ebx, ecx, edx;
+
+	cpuid(op, &eax, &ebx, &ecx, &edx);
+
+	return ebx;
+}
+
+static inline u32 cpuid_ecx(u32 op)
+{
+	u32 eax, ebx, ecx, edx;
+
+	cpuid(op, &eax, &ebx, &ecx, &edx);
+
+	return ecx;
+}
+
+static inline u32 cpuid_edx(u32 op)
+{
+	u32 eax, ebx, ecx, edx;
+
+	cpuid(op, &eax, &ebx, &ecx, &edx);
+
+	return edx;
+}
+
+static inline void __cpuid_read(u32 leaf, u32 subleaf, u32 *regs)
+{
+	regs[CPUID_EAX] = leaf;
+	regs[CPUID_ECX] = subleaf;
+	__cpuid(regs + CPUID_EAX, regs + CPUID_EBX, regs + CPUID_ECX, regs + CPUID_EDX);
+}
+
+#define cpuid_subleaf(leaf, subleaf, regs) {		\
+	static_assert(sizeof(*(regs)) == 16);		\
+	__cpuid_read(leaf, subleaf, (u32 *)(regs));	\
+}
+
+#define cpuid_leaf(leaf, regs) {			\
+	static_assert(sizeof(*(regs)) == 16);		\
+	__cpuid_read(leaf, 0, (u32 *)(regs));		\
+}
+
+static inline void __cpuid_read_reg(u32 leaf, u32 subleaf,
+				    enum cpuid_regs_idx regidx, u32 *reg)
+{
+	u32 regs[4];
+
+	__cpuid_read(leaf, subleaf, regs);
+	*reg = regs[regidx];
+}
+
+#define cpuid_subleaf_reg(leaf, subleaf, regidx, reg) {		\
+	static_assert(sizeof(*(reg)) == 4);			\
+	__cpuid_read_reg(leaf, subleaf, regidx, (u32 *)(reg));	\
+}
+
+#define cpuid_leaf_reg(leaf, regidx, reg) {			\
+	static_assert(sizeof(*(reg)) == 4);			\
+	__cpuid_read_reg(leaf, 0, regidx, (u32 *)(reg));	\
+}
+
+/*
+ * Hypervisor-related APIs:
+ */
+
+static __always_inline bool cpuid_function_is_indexed(u32 function)
+{
+	switch (function) {
+	case 4:
+	case 7:
+	case 0xb:
+	case 0xd:
+	case 0xf:
+	case 0x10:
+	case 0x12:
+	case 0x14:
+	case 0x17:
+	case 0x18:
+	case 0x1d:
+	case 0x1e:
+	case 0x1f:
+	case 0x24:
+	case 0x8000001d:
+		return true;
+	}
+
+	return false;
+}
+
+#define for_each_possible_cpuid_base_hypervisor(function) \
+	for (function = 0x40000000; function < 0x40010000; function += 0x100)
+
+static inline u32 cpuid_base_hypervisor(const char *sig, u32 leaves)
+{
+	u32 base, eax, signature[3];
+
+	for_each_possible_cpuid_base_hypervisor(base) {
+		cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);
+
+		/*
+		 * This must not compile to "call memcmp" because it's called
+		 * from PVH early boot code before instrumentation is set up
+		 * and memcmp() itself may be instrumented.
+		 */
+		if (!__builtin_memcmp(sig, signature, 12) &&
+		    (leaves == 0 || ((eax - base) >= leaves)))
+			return base;
+	}
+
+	return 0;
+}
+
+/*
+ * CPUID(0x2) parsing:
+ */
+
+/**
+ * cpuid_leaf_0x2() - Return sanitized CPUID(0x2) register output
+ * @regs:	Output parameter
+ *
+ * Query CPUID(0x2) and store its output in @regs.  Force set any
+ * invalid 1-byte descriptor returned by the hardware to zero (the NULL
+ * cache/TLB descriptor) before returning it to the caller.
+ *
+ * Use for_each_cpuid_0x2_desc() to iterate over the register output in
+ * parsed form.
+ */
+static inline void cpuid_leaf_0x2(union leaf_0x2_regs *regs)
+{
+	cpuid_leaf(0x2, regs);
+
+	/*
+	 * All Intel CPUs must report an iteration count of 1.	In case
+	 * of bogus hardware, treat all returned descriptors as NULL.
+	 */
+	if (regs->desc[0] != 0x01) {
+		for (int i = 0; i < 4; i++)
+			regs->regv[i] = 0;
+		return;
+	}
+
+	/*
+	 * The most significant bit (MSB) of each register must be clear.
+	 * If a register is invalid, replace its descriptors with NULL.
+	 */
+	for (int i = 0; i < 4; i++) {
+		if (regs->reg[i].invalid)
+			regs->regv[i] = 0;
+	}
+}
+
+/**
+ * for_each_cpuid_0x2_desc() - Iterator for parsed CPUID(0x2) descriptors
+ * @_regs:	CPUID(0x2) register output, as returned by cpuid_leaf_0x2()
+ * @_ptr:	u8 pointer, for macro internal use only
+ * @_desc:	Pointer to the parsed CPUID(0x2) descriptor at each iteration
+ *
+ * Loop over the 1-byte descriptors in the passed CPUID(0x2) output registers
+ * @_regs.  Provide the parsed information for each descriptor through @_desc.
+ *
+ * To handle cache-specific descriptors, switch on @_desc->c_type.  For TLB
+ * descriptors, switch on @_desc->t_type.
+ *
+ * Example usage for cache descriptors::
+ *
+ *	const struct leaf_0x2_table *desc;
+ *	union leaf_0x2_regs regs;
+ *	u8 *ptr;
+ *
+ *	cpuid_leaf_0x2(&regs);
+ *	for_each_cpuid_0x2_desc(regs, ptr, desc) {
+ *		switch (desc->c_type) {
+ *			...
+ *		}
+ *	}
+ */
+#define for_each_cpuid_0x2_desc(_regs, _ptr, _desc)				\
+	for (_ptr = &(_regs).desc[1];						\
+	     _ptr < &(_regs).desc[16] && (_desc = &cpuid_0x2_table[*_ptr]);	\
+	     _ptr++)
+
+/*
+ * CPUID(0x80000006) parsing:
+ */
+
+static inline bool cpuid_amd_hygon_has_l3_cache(void)
+{
+	return cpuid_edx(0x80000006);
+}
+
+#endif /* _ASM_X86_CPUID_API_H */
diff --git a/arch/x86/include/asm/cpuid/types.h b/arch/x86/include/asm/cpuid/types.h
new file mode 100644
index 000000000000..8a00364b79de
--- /dev/null
+++ b/arch/x86/include/asm/cpuid/types.h
@@ -0,0 +1,127 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_CPUID_TYPES_H
+#define _ASM_X86_CPUID_TYPES_H
+
+#include <linux/build_bug.h>
+#include <linux/types.h>
+
+/*
+ * Types for raw CPUID access:
+ */
+
+struct cpuid_regs {
+	u32 eax;
+	u32 ebx;
+	u32 ecx;
+	u32 edx;
+};
+
+enum cpuid_regs_idx {
+	CPUID_EAX = 0,
+	CPUID_EBX,
+	CPUID_ECX,
+	CPUID_EDX,
+};
+
+#define CPUID_LEAF_MWAIT	0x05
+#define CPUID_LEAF_DCA		0x09
+#define CPUID_LEAF_XSTATE	0x0d
+#define CPUID_LEAF_TSC		0x15
+#define CPUID_LEAF_FREQ		0x16
+#define CPUID_LEAF_TILE		0x1d
+
+/*
+ * Types for CPUID(0x2) parsing:
+ */
+
+struct leaf_0x2_reg {
+		u32		: 31,
+			invalid	: 1;
+};
+
+union leaf_0x2_regs {
+	struct leaf_0x2_reg	reg[4];
+	u32			regv[4];
+	u8			desc[16];
+};
+
+/*
+ * Leaf 0x2 1-byte descriptors' cache types
+ * To be used for their mappings at cpuid_0x2_table[]
+ *
+ * Start at 1 since type 0 is reserved for HW byte descriptors which are
+ * not recognized by the kernel; i.e., those without an explicit mapping.
+ */
+enum _cache_table_type {
+	CACHE_L1_INST		= 1,
+	CACHE_L1_DATA,
+	CACHE_L2,
+	CACHE_L3
+	/* Adjust __TLB_TABLE_TYPE_BEGIN before adding more types */
+} __packed;
+#ifndef __CHECKER__
+static_assert(sizeof(enum _cache_table_type) == 1);
+#endif
+
+/*
+ * Ensure that leaf 0x2 cache and TLB type values do not intersect,
+ * since they share the same type field at struct cpuid_0x2_table.
+ */
+#define __TLB_TABLE_TYPE_BEGIN		(CACHE_L3 + 1)
+
+/*
+ * Leaf 0x2 1-byte descriptors' TLB types
+ * To be used for their mappings at cpuid_0x2_table[]
+ */
+enum _tlb_table_type {
+	TLB_INST_4K		= __TLB_TABLE_TYPE_BEGIN,
+	TLB_INST_4M,
+	TLB_INST_2M_4M,
+	TLB_INST_ALL,
+
+	TLB_DATA_4K,
+	TLB_DATA_4M,
+	TLB_DATA_2M_4M,
+	TLB_DATA_4K_4M,
+	TLB_DATA_1G,
+	TLB_DATA_1G_2M_4M,
+
+	TLB_DATA0_4K,
+	TLB_DATA0_4M,
+	TLB_DATA0_2M_4M,
+
+	STLB_4K,
+	STLB_4K_2M,
+} __packed;
+#ifndef __CHECKER__
+static_assert(sizeof(enum _tlb_table_type) == 1);
+#endif
+
+/*
+ * Combined parsing table for leaf 0x2 cache and TLB descriptors.
+ */
+
+struct leaf_0x2_table {
+	union {
+		enum _cache_table_type	c_type;
+		enum _tlb_table_type	t_type;
+	};
+	union {
+		short			c_size;
+		short			entries;
+	};
+};
+
+extern const struct leaf_0x2_table cpuid_0x2_table[256];
+
+/*
+ * All of leaf 0x2's one-byte TLB descriptors implies the same number of entries
+ * for their respective TLB types.  TLB descriptor 0x63 is an exception: it
+ * implies 4 dTLB entries for 1GB pages and 32 dTLB entries for 2MB or 4MB pages.
+ *
+ * Encode that descriptor's dTLB entry count for 2MB/4MB pages here, as the entry
+ * count for dTLB 1GB pages is already encoded at the cpuid_0x2_table[]'s mapping.
+ */
+#define TLB_0x63_2M_4M_ENTRIES		32
+
+#endif /* _ASM_X86_CPUID_TYPES_H */
diff --git a/arch/x86/include/asm/cpumask.h b/arch/x86/include/asm/cpumask.h
index 4acfd57de8f1..9df9e9cde670 100644
--- a/arch/x86/include/asm/cpumask.h
+++ b/arch/x86/include/asm/cpumask.h
@@ -1,7 +1,9 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #ifndef _ASM_X86_CPUMASK_H
 #define _ASM_X86_CPUMASK_H
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
+
+#include <linux/compiler.h>
 #include <linux/cpumask.h>
 
 extern void setup_cpu_local_masks(void);
@@ -34,5 +36,5 @@ static __always_inline void arch_cpumask_clear_cpu(int cpu, struct cpumask *dstp
 
 #define arch_cpu_is_offline(cpu)	unlikely(!arch_cpu_online(cpu))
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* _ASM_X86_CPUMASK_H */
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index bf5953883ec3..cc4a3f725b37 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -5,52 +5,28 @@
 #include <linux/build_bug.h>
 #include <linux/compiler.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/cache.h>
 #include <asm/percpu.h>
 
 struct task_struct;
 
-struct pcpu_hot {
-	union {
-		struct {
-			struct task_struct	*current_task;
-			int			preempt_count;
-			int			cpu_number;
-#ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING
-			u64			call_depth;
-#endif
-			unsigned long		top_of_stack;
-			void			*hardirq_stack_ptr;
-			u16			softirq_pending;
-#ifdef CONFIG_X86_64
-			bool			hardirq_stack_inuse;
-#else
-			void			*softirq_stack_ptr;
-#endif
-		};
-		u8	pad[64];
-	};
-};
-static_assert(sizeof(struct pcpu_hot) == 64);
-
-DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot);
-
-/* const-qualified alias to pcpu_hot, aliased by linker. */
-DECLARE_PER_CPU_ALIGNED(const struct pcpu_hot __percpu_seg_override,
-			const_pcpu_hot);
+DECLARE_PER_CPU_CACHE_HOT(struct task_struct *, current_task);
+/* const-qualified alias provided by the linker. */
+DECLARE_PER_CPU_CACHE_HOT(struct task_struct * const __percpu_seg_override,
+			  const_current_task);
 
 static __always_inline struct task_struct *get_current(void)
 {
 	if (IS_ENABLED(CONFIG_USE_X86_SEG_SUPPORT))
-		return this_cpu_read_const(const_pcpu_hot.current_task);
+		return this_cpu_read_const(const_current_task);
 
-	return this_cpu_read_stable(pcpu_hot.current_task);
+	return this_cpu_read_stable(current_task);
 }
 
 #define current get_current()
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ASM_X86_CURRENT_H */
diff --git a/arch/x86/include/asm/debugreg.h b/arch/x86/include/asm/debugreg.h
index fdbbbfec745a..a2c1f2d24b64 100644
--- a/arch/x86/include/asm/debugreg.h
+++ b/arch/x86/include/asm/debugreg.h
@@ -9,6 +9,14 @@
 #include <asm/cpufeature.h>
 #include <asm/msr.h>
 
+/*
+ * Define bits that are always set to 1 in DR7, only bit 10 is
+ * architecturally reserved to '1'.
+ *
+ * This is also the init/reset value for DR7.
+ */
+#define DR7_FIXED_1	0x00000400
+
 DECLARE_PER_CPU(unsigned long, cpu_dr7);
 
 #ifndef CONFIG_PARAVIRT_XXL
@@ -23,7 +31,7 @@ DECLARE_PER_CPU(unsigned long, cpu_dr7);
 
 static __always_inline unsigned long native_get_debugreg(int regno)
 {
-	unsigned long val = 0;	/* Damn you, gcc! */
+	unsigned long val;
 
 	switch (regno) {
 	case 0:
@@ -43,7 +51,7 @@ static __always_inline unsigned long native_get_debugreg(int regno)
 		break;
 	case 7:
 		/*
-		 * Apply __FORCE_ORDER to DR7 reads to forbid re-ordering them
+		 * Use "asm volatile" for DR7 reads to forbid re-ordering them
 		 * with other code.
 		 *
 		 * This is needed because a DR7 access can cause a #VC exception
@@ -55,7 +63,7 @@ static __always_inline unsigned long native_get_debugreg(int regno)
 		 * re-ordered to happen before the call to sev_es_ist_enter(),
 		 * causing stack recursion.
 		 */
-		asm volatile("mov %%db7, %0" : "=r" (val) : __FORCE_ORDER);
+		asm volatile("mov %%db7, %0" : "=r" (val));
 		break;
 	default:
 		BUG();
@@ -83,15 +91,15 @@ static __always_inline void native_set_debugreg(int regno, unsigned long value)
 		break;
 	case 7:
 		/*
-		 * Apply __FORCE_ORDER to DR7 writes to forbid re-ordering them
+		 * Use "asm volatile" for DR7 writes to forbid re-ordering them
 		 * with other code.
 		 *
 		 * While is didn't happen with a DR7 write (see the DR7 read
 		 * comment above which explains where it happened), add the
-		 * __FORCE_ORDER here too to avoid similar problems in the
+		 * "asm volatile" here too to avoid similar problems in the
 		 * future.
 		 */
-		asm volatile("mov %0, %%db7"	::"r" (value), __FORCE_ORDER);
+		asm volatile("mov %0, %%db7"	::"r" (value));
 		break;
 	default:
 		BUG();
@@ -100,8 +108,8 @@ static __always_inline void native_set_debugreg(int regno, unsigned long value)
 
 static inline void hw_breakpoint_disable(void)
 {
-	/* Zero the control register for HW Breakpoint */
-	set_debugreg(0UL, 7);
+	/* Reset the control register for HW Breakpoint */
+	set_debugreg(DR7_FIXED_1, 7);
 
 	/* Zero-out the individual HW breakpoint address registers */
 	set_debugreg(0UL, 0);
@@ -125,9 +133,12 @@ static __always_inline unsigned long local_db_save(void)
 		return 0;
 
 	get_debugreg(dr7, 7);
-	dr7 &= ~0x400; /* architecturally set bit */
+
+	/* Architecturally set bit */
+	dr7 &= ~DR7_FIXED_1;
 	if (dr7)
-		set_debugreg(0, 7);
+		set_debugreg(DR7_FIXED_1, 7);
+
 	/*
 	 * Ensure the compiler doesn't lower the above statements into
 	 * the critical section; disabling breakpoints late would not
@@ -169,7 +180,7 @@ static inline unsigned long get_debugctlmsr(void)
 	if (boot_cpu_data.x86 < 6)
 		return 0;
 #endif
-	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
+	rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
 
 	return debugctlmsr;
 }
@@ -180,7 +191,7 @@ static inline void update_debugctlmsr(unsigned long debugctlmsr)
 	if (boot_cpu_data.x86 < 6)
 		return;
 #endif
-	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
+	wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
 }
 
 #endif /* _ASM_X86_DEBUGREG_H */
diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h
index 62dc9f59ea76..ec95fe44fa3a 100644
--- a/arch/x86/include/asm/desc.h
+++ b/arch/x86/include/asm/desc.h
@@ -46,7 +46,6 @@ struct gdt_page {
 } __attribute__((aligned(PAGE_SIZE)));
 
 DECLARE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page);
-DECLARE_INIT_PER_CPU(gdt_page);
 
 /* Provide the original GDT */
 static inline struct desc_struct *get_cpu_gdt_rw(unsigned int cpu)
diff --git a/arch/x86/include/asm/desc_defs.h b/arch/x86/include/asm/desc_defs.h
index d440a65af8f3..7e6b9314758a 100644
--- a/arch/x86/include/asm/desc_defs.h
+++ b/arch/x86/include/asm/desc_defs.h
@@ -58,7 +58,7 @@
 
 #define DESC_USER		(_DESC_DPL(3))
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/types.h>
 
@@ -166,7 +166,7 @@ struct desc_ptr {
 	unsigned long address;
 } __attribute__((packed)) ;
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 /* Boot IDT definitions */
 #define	BOOT_IDT_ENTRIES	32
diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h
deleted file mode 100644
index c492bdc97b05..000000000000
--- a/arch/x86/include/asm/disabled-features.h
+++ /dev/null
@@ -1,161 +0,0 @@
-#ifndef _ASM_X86_DISABLED_FEATURES_H
-#define _ASM_X86_DISABLED_FEATURES_H
-
-/* These features, although they might be available in a CPU
- * will not be used because the compile options to support
- * them are not present.
- *
- * This code allows them to be checked and disabled at
- * compile time without an explicit #ifdef.  Use
- * cpu_feature_enabled().
- */
-
-#ifdef CONFIG_X86_UMIP
-# define DISABLE_UMIP	0
-#else
-# define DISABLE_UMIP	(1<<(X86_FEATURE_UMIP & 31))
-#endif
-
-#ifdef CONFIG_X86_64
-# define DISABLE_VME		(1<<(X86_FEATURE_VME & 31))
-# define DISABLE_K6_MTRR	(1<<(X86_FEATURE_K6_MTRR & 31))
-# define DISABLE_CYRIX_ARR	(1<<(X86_FEATURE_CYRIX_ARR & 31))
-# define DISABLE_CENTAUR_MCR	(1<<(X86_FEATURE_CENTAUR_MCR & 31))
-# define DISABLE_PCID		0
-#else
-# define DISABLE_VME		0
-# define DISABLE_K6_MTRR	0
-# define DISABLE_CYRIX_ARR	0
-# define DISABLE_CENTAUR_MCR	0
-# define DISABLE_PCID		(1<<(X86_FEATURE_PCID & 31))
-#endif /* CONFIG_X86_64 */
-
-#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
-# define DISABLE_PKU		0
-# define DISABLE_OSPKE		0
-#else
-# define DISABLE_PKU		(1<<(X86_FEATURE_PKU & 31))
-# define DISABLE_OSPKE		(1<<(X86_FEATURE_OSPKE & 31))
-#endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */
-
-#ifdef CONFIG_X86_5LEVEL
-# define DISABLE_LA57	0
-#else
-# define DISABLE_LA57	(1<<(X86_FEATURE_LA57 & 31))
-#endif
-
-#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
-# define DISABLE_PTI		0
-#else
-# define DISABLE_PTI		(1 << (X86_FEATURE_PTI & 31))
-#endif
-
-#ifdef CONFIG_MITIGATION_RETPOLINE
-# define DISABLE_RETPOLINE	0
-#else
-# define DISABLE_RETPOLINE	((1 << (X86_FEATURE_RETPOLINE & 31)) | \
-				 (1 << (X86_FEATURE_RETPOLINE_LFENCE & 31)))
-#endif
-
-#ifdef CONFIG_MITIGATION_RETHUNK
-# define DISABLE_RETHUNK	0
-#else
-# define DISABLE_RETHUNK	(1 << (X86_FEATURE_RETHUNK & 31))
-#endif
-
-#ifdef CONFIG_MITIGATION_UNRET_ENTRY
-# define DISABLE_UNRET		0
-#else
-# define DISABLE_UNRET		(1 << (X86_FEATURE_UNRET & 31))
-#endif
-
-#ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING
-# define DISABLE_CALL_DEPTH_TRACKING	0
-#else
-# define DISABLE_CALL_DEPTH_TRACKING	(1 << (X86_FEATURE_CALL_DEPTH & 31))
-#endif
-
-#ifdef CONFIG_ADDRESS_MASKING
-# define DISABLE_LAM		0
-#else
-# define DISABLE_LAM		(1 << (X86_FEATURE_LAM & 31))
-#endif
-
-#ifdef CONFIG_INTEL_IOMMU_SVM
-# define DISABLE_ENQCMD		0
-#else
-# define DISABLE_ENQCMD		(1 << (X86_FEATURE_ENQCMD & 31))
-#endif
-
-#ifdef CONFIG_X86_SGX
-# define DISABLE_SGX	0
-#else
-# define DISABLE_SGX	(1 << (X86_FEATURE_SGX & 31))
-#endif
-
-#ifdef CONFIG_XEN_PV
-# define DISABLE_XENPV		0
-#else
-# define DISABLE_XENPV		(1 << (X86_FEATURE_XENPV & 31))
-#endif
-
-#ifdef CONFIG_INTEL_TDX_GUEST
-# define DISABLE_TDX_GUEST	0
-#else
-# define DISABLE_TDX_GUEST	(1 << (X86_FEATURE_TDX_GUEST & 31))
-#endif
-
-#ifdef CONFIG_X86_USER_SHADOW_STACK
-#define DISABLE_USER_SHSTK	0
-#else
-#define DISABLE_USER_SHSTK	(1 << (X86_FEATURE_USER_SHSTK & 31))
-#endif
-
-#ifdef CONFIG_X86_KERNEL_IBT
-#define DISABLE_IBT	0
-#else
-#define DISABLE_IBT	(1 << (X86_FEATURE_IBT & 31))
-#endif
-
-#ifdef CONFIG_X86_FRED
-# define DISABLE_FRED	0
-#else
-# define DISABLE_FRED	(1 << (X86_FEATURE_FRED & 31))
-#endif
-
-#ifdef CONFIG_KVM_AMD_SEV
-#define DISABLE_SEV_SNP		0
-#else
-#define DISABLE_SEV_SNP		(1 << (X86_FEATURE_SEV_SNP & 31))
-#endif
-
-/*
- * Make sure to add features to the correct mask
- */
-#define DISABLED_MASK0	(DISABLE_VME)
-#define DISABLED_MASK1	0
-#define DISABLED_MASK2	0
-#define DISABLED_MASK3	(DISABLE_CYRIX_ARR|DISABLE_CENTAUR_MCR|DISABLE_K6_MTRR)
-#define DISABLED_MASK4	(DISABLE_PCID)
-#define DISABLED_MASK5	0
-#define DISABLED_MASK6	0
-#define DISABLED_MASK7	(DISABLE_PTI)
-#define DISABLED_MASK8	(DISABLE_XENPV|DISABLE_TDX_GUEST)
-#define DISABLED_MASK9	(DISABLE_SGX)
-#define DISABLED_MASK10	0
-#define DISABLED_MASK11	(DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET| \
-			 DISABLE_CALL_DEPTH_TRACKING|DISABLE_USER_SHSTK)
-#define DISABLED_MASK12	(DISABLE_FRED|DISABLE_LAM)
-#define DISABLED_MASK13	0
-#define DISABLED_MASK14	0
-#define DISABLED_MASK15	0
-#define DISABLED_MASK16	(DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP| \
-			 DISABLE_ENQCMD)
-#define DISABLED_MASK17	0
-#define DISABLED_MASK18	(DISABLE_IBT)
-#define DISABLED_MASK19	(DISABLE_SEV_SNP)
-#define DISABLED_MASK20	0
-#define DISABLED_MASK21	0
-#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 22)
-
-#endif /* _ASM_X86_DISABLED_FEATURES_H */
diff --git a/arch/x86/include/asm/div64.h b/arch/x86/include/asm/div64.h
index 9931e4c7d73f..30fd06ede751 100644
--- a/arch/x86/include/asm/div64.h
+++ b/arch/x86/include/asm/div64.h
@@ -60,6 +60,12 @@ static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
 }
 #define div_u64_rem	div_u64_rem
 
+/*
+ * gcc tends to zero extend 32bit values and do full 64bit maths.
+ * Define asm functions that avoid this.
+ * (clang generates better code for the C versions.)
+ */
+#ifndef __clang__
 static inline u64 mul_u32_u32(u32 a, u32 b)
 {
 	u32 high, low;
@@ -71,6 +77,19 @@ static inline u64 mul_u32_u32(u32 a, u32 b)
 }
 #define mul_u32_u32 mul_u32_u32
 
+static inline u64 add_u64_u32(u64 a, u32 b)
+{
+	u32 high = a >> 32, low = a;
+
+	asm ("addl %[b], %[low]; adcl $0, %[high]"
+		: [low] "+r" (low), [high] "+r" (high)
+		: [b] "rm" (b) );
+
+	return low | (u64)high << 32;
+}
+#define add_u64_u32 add_u64_u32
+#endif
+
 /*
  * __div64_32() is never called on x86, so prevent the
  * generic definition from getting built.
@@ -84,21 +103,25 @@ static inline u64 mul_u32_u32(u32 a, u32 b)
  * Will generate an #DE when the result doesn't fit u64, could fix with an
  * __ex_table[] entry when it becomes an issue.
  */
-static inline u64 mul_u64_u64_div_u64(u64 a, u64 mul, u64 div)
+static inline u64 mul_u64_add_u64_div_u64(u64 rax, u64 mul, u64 add, u64 div)
 {
-	u64 q;
+	u64 rdx;
+
+	asm ("mulq %[mul]" : "+a" (rax), "=d" (rdx) : [mul] "rm" (mul));
+
+	if (!statically_true(!add))
+		asm ("addq %[add], %[lo]; adcq $0, %[hi]" :
+			[lo] "+r" (rax), [hi] "+r" (rdx) : [add] "irm" (add));
 
-	asm ("mulq %2; divq %3" : "=a" (q)
-				: "a" (a), "rm" (mul), "rm" (div)
-				: "rdx");
+	asm ("divq %[div]" : "+a" (rax), "+d" (rdx) : [div] "rm" (div));
 
-	return q;
+	return rax;
 }
-#define mul_u64_u64_div_u64 mul_u64_u64_div_u64
+#define mul_u64_add_u64_div_u64 mul_u64_add_u64_div_u64
 
 static inline u64 mul_u64_u32_div(u64 a, u32 mul, u32 div)
 {
-	return mul_u64_u64_div_u64(a, mul, div);
+	return mul_u64_add_u64_div_u64(a, mul, 0, div);
 }
 #define mul_u64_u32_div	mul_u64_u32_div
 
diff --git a/arch/x86/include/asm/dwarf2.h b/arch/x86/include/asm/dwarf2.h
index 430fca13bb56..302e11b15da8 100644
--- a/arch/x86/include/asm/dwarf2.h
+++ b/arch/x86/include/asm/dwarf2.h
@@ -2,7 +2,7 @@
 #ifndef _ASM_X86_DWARF2_H
 #define _ASM_X86_DWARF2_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #warning "asm/dwarf2.h should be only included in pure assembly files"
 #endif
 
diff --git a/arch/x86/include/asm/e820/api.h b/arch/x86/include/asm/e820/api.h
index 2e74a7f0e935..c83645d5b2a8 100644
--- a/arch/x86/include/asm/e820/api.h
+++ b/arch/x86/include/asm/e820/api.h
@@ -29,7 +29,6 @@ extern unsigned long e820__end_of_low_ram_pfn(void);
 extern u64  e820__memblock_alloc_reserved(u64 size, u64 align);
 extern void e820__memblock_setup(void);
 
-extern void e820__reserve_setup_data(void);
 extern void e820__finish_early_params(void);
 extern void e820__reserve_resources(void);
 extern void e820__reserve_resources_late(void);
diff --git a/arch/x86/include/asm/e820/types.h b/arch/x86/include/asm/e820/types.h
index 314f75d886d0..80c4a7266629 100644
--- a/arch/x86/include/asm/e820/types.h
+++ b/arch/x86/include/asm/e820/types.h
@@ -35,15 +35,6 @@ enum e820_type {
 	 * marking it with the IORES_DESC_SOFT_RESERVED designation.
 	 */
 	E820_TYPE_SOFT_RESERVED	= 0xefffffff,
-
-	/*
-	 * Reserved RAM used by the kernel itself if
-	 * CONFIG_INTEL_TXT=y is enabled, memory of this type
-	 * will be included in the S3 integrity calculation
-	 * and so should not include any memory that the BIOS
-	 * might alter over the S3 transition:
-	 */
-	E820_TYPE_RESERVED_KERN	= 128,
 };
 
 /*
diff --git a/arch/x86/include/asm/edac.h b/arch/x86/include/asm/edac.h
index 426fc53ff803..dfbd1ebb9f10 100644
--- a/arch/x86/include/asm/edac.h
+++ b/arch/x86/include/asm/edac.h
@@ -13,7 +13,7 @@ static inline void edac_atomic_scrub(void *va, u32 size)
 	 * are interrupt, DMA and SMP safe.
 	 */
 	for (i = 0; i < size / 4; i++, virt_addr++)
-		asm volatile("lock; addl $0, %0"::"m" (*virt_addr));
+		asm volatile("lock addl $0, %0"::"m" (*virt_addr));
 }
 
 #endif /* _ASM_X86_EDAC_H */
diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index 521aad70e41b..f227a70ac91f 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -250,6 +250,9 @@ static inline u32 efi64_convert_status(efi_status_t status)
 #define __efi64_argmap_allocate_pool(type, size, buffer)		\
 	((type), (size), efi64_zero_upper(buffer))
 
+#define __efi64_argmap_locate_handle_buffer(type, proto, key, num, buf)	\
+	((type), (proto), (key), efi64_zero_upper(num), efi64_zero_upper(buf))
+
 #define __efi64_argmap_create_event(type, tpl, f, c, event)		\
 	((type), (tpl), (f), (c), efi64_zero_upper(event))
 
diff --git a/arch/x86/include/asm/elf.h b/arch/x86/include/asm/elf.h
index 1fb83d47711f..6c8fdc96be7e 100644
--- a/arch/x86/include/asm/elf.h
+++ b/arch/x86/include/asm/elf.h
@@ -54,8 +54,9 @@ typedef struct user_i387_struct elf_fpregset_t;
 #define R_X86_64_GLOB_DAT	6	/* Create GOT entry */
 #define R_X86_64_JUMP_SLOT	7	/* Create PLT entry */
 #define R_X86_64_RELATIVE	8	/* Adjust by program base */
-#define R_X86_64_GOTPCREL	9	/* 32 bit signed pc relative
-					   offset to GOT */
+#define R_X86_64_GOTPCREL	9	/* 32 bit signed pc relative offset to GOT */
+#define R_X86_64_GOTPCRELX	41
+#define R_X86_64_REX_GOTPCRELX	42
 #define R_X86_64_32		10	/* Direct 32 bit zero extended */
 #define R_X86_64_32S		11	/* Direct 32 bit sign extended */
 #define R_X86_64_16		12	/* Direct 16 bit zero extended */
@@ -75,12 +76,8 @@ typedef struct user_i387_struct elf_fpregset_t;
 
 #include <asm/vdso.h>
 
-#ifdef CONFIG_X86_64
 extern unsigned int vdso64_enabled;
-#endif
-#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
 extern unsigned int vdso32_enabled;
-#endif
 
 /*
  * This is used to ensure we don't load something for the wrong architecture.
diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h
index 77d20555e04d..ce3eb6d5fdf9 100644
--- a/arch/x86/include/asm/entry-common.h
+++ b/arch/x86/include/asm/entry-common.h
@@ -53,7 +53,6 @@ static inline void arch_exit_work(unsigned long ti_work)
 	if (unlikely(ti_work & _TIF_IO_BITMAP))
 		tss_update_io_bitmap();
 
-	fpregs_assert_state_consistent();
 	if (unlikely(ti_work & _TIF_NEED_FPU_LOAD))
 		switch_fpu_return();
 }
@@ -61,7 +60,9 @@ static inline void arch_exit_work(unsigned long ti_work)
 static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
 						  unsigned long ti_work)
 {
-	if (IS_ENABLED(CONFIG_X86_DEBUG_FPU) || unlikely(ti_work))
+	fpregs_assert_state_consistent();
+
+	if (unlikely(ti_work))
 		arch_exit_work(ti_work);
 
 	fred_update_rsp0();
@@ -92,6 +93,13 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
 	 * 8 (ia32) bits.
 	 */
 	choose_random_kstack_offset(rdtsc());
+
+	/* Avoid unnecessary reads of 'x86_ibpb_exit_to_user' */
+	if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER) &&
+	    this_cpu_read(x86_ibpb_exit_to_user)) {
+		indirect_branch_prediction_barrier();
+		this_cpu_write(x86_ibpb_exit_to_user, false);
+	}
 }
 #define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
 
diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h
index d0dcefb5cc59..4519c9f35ba0 100644
--- a/arch/x86/include/asm/fixmap.h
+++ b/arch/x86/include/asm/fixmap.h
@@ -31,7 +31,7 @@
 /* fixmap starts downwards from the 507th entry in level2_fixmap_pgt */
 #define FIXMAP_PMD_TOP	507
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/kernel.h>
 #include <asm/apicdef.h>
 #include <asm/page.h>
@@ -196,5 +196,5 @@ void __init *early_memremap_decrypted_wp(resource_size_t phys_addr,
 void __early_set_fixmap(enum fixed_addresses idx,
 			phys_addr_t phys, pgprot_t flags);
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 #endif /* _ASM_X86_FIXMAP_H */
diff --git a/arch/x86/include/asm/floppy.h b/arch/x86/include/asm/floppy.h
index 6ec3fc969ad5..e7a244051c62 100644
--- a/arch/x86/include/asm/floppy.h
+++ b/arch/x86/include/asm/floppy.h
@@ -10,6 +10,7 @@
 #ifndef _ASM_X86_FLOPPY_H
 #define _ASM_X86_FLOPPY_H
 
+#include <linux/sizes.h>
 #include <linux/vmalloc.h>
 
 /*
@@ -22,10 +23,7 @@
  */
 #define _CROSS_64KB(a, s, vdma)						\
 	(!(vdma) &&							\
-	 ((unsigned long)(a)/K_64 != ((unsigned long)(a) + (s) - 1) / K_64))
-
-#define CROSS_64KB(a, s) _CROSS_64KB(a, s, use_virtual_dma & 1)
-
+	 ((unsigned long)(a) / SZ_64K != ((unsigned long)(a) + (s) - 1) / SZ_64K))
 
 #define SW fd_routine[use_virtual_dma & 1]
 #define CSW fd_routine[can_use_virtual_dma & 1]
@@ -206,7 +204,7 @@ static int vdma_dma_setup(char *addr, unsigned long size, int mode, int io)
 static int hard_dma_setup(char *addr, unsigned long size, int mode, int io)
 {
 #ifdef FLOPPY_SANITY_CHECK
-	if (CROSS_64KB(addr, size)) {
+	if (_CROSS_64KB(addr, size, use_virtual_dma & 1)) {
 		printk("DMA crossing 64-K boundary %p-%p\n", addr, addr+size);
 		return -1;
 	}
diff --git a/arch/x86/include/asm/fpu/api.h b/arch/x86/include/asm/fpu/api.h
index f86ad3335529..cd6f194a912b 100644
--- a/arch/x86/include/asm/fpu/api.h
+++ b/arch/x86/include/asm/fpu/api.h
@@ -16,10 +16,9 @@
 
 /*
  * Use kernel_fpu_begin/end() if you intend to use FPU in kernel context. It
- * disables preemption so be careful if you intend to use it for long periods
- * of time.
- * If you intend to use the FPU in irq/softirq you need to check first with
- * irq_fpu_usable() if it is possible.
+ * disables preemption and softirq processing, so be careful if you intend to
+ * use it for long periods of time.  Kernel-mode FPU cannot be used in all
+ * contexts -- see irq_fpu_usable() for details.
  */
 
 /* Kernel FPU states to initialize in kernel_fpu_begin_mask() */
@@ -50,10 +49,10 @@ static inline void kernel_fpu_begin(void)
 }
 
 /*
- * Use fpregs_lock() while editing CPU's FPU registers or fpu->fpstate.
- * A context switch will (and softirq might) save CPU's FPU registers to
- * fpu->fpstate.regs and set TIF_NEED_FPU_LOAD leaving CPU's FPU registers in
- * a random state.
+ * Use fpregs_lock() while editing CPU's FPU registers or fpu->fpstate, or while
+ * using the FPU in kernel mode.  A context switch will (and softirq might) save
+ * CPU's FPU registers to fpu->fpstate.regs and set TIF_NEED_FPU_LOAD leaving
+ * CPU's FPU registers in a random state.
  *
  * local_bh_disable() protects against both preemption and soft interrupts
  * on !RT kernels.
@@ -63,8 +62,6 @@ static inline void kernel_fpu_begin(void)
  * preemptible. Disabling preemption is the right choice here as bottom
  * half processing is always in thread context on RT kernels so it
  * implicitly prevents bottom half processing as well.
- *
- * Disabling preemption also serializes against kernel_fpu_begin().
  */
 static inline void fpregs_lock(void)
 {
@@ -129,6 +126,7 @@ static inline void fpstate_init_soft(struct swregs_state *soft) {}
 #endif
 
 /* State tracking */
+DECLARE_PER_CPU(bool, kernel_fpu_allowed);
 DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
 
 /* Process cleanup */
@@ -139,7 +137,7 @@ static inline void fpstate_free(struct fpu *fpu) { }
 #endif
 
 /* fpstate-related functions which are exported to KVM */
-extern void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature);
+extern void fpstate_clear_xstate_component(struct fpstate *fpstate, unsigned int xfeature);
 
 extern u64 xstate_get_guest_group_perm(void);
 
diff --git a/arch/x86/include/asm/fpu/sched.h b/arch/x86/include/asm/fpu/sched.h
index c485f1944c5f..89004f4ca208 100644
--- a/arch/x86/include/asm/fpu/sched.h
+++ b/arch/x86/include/asm/fpu/sched.h
@@ -10,39 +10,33 @@
 #include <asm/trace/fpu.h>
 
 extern void save_fpregs_to_fpstate(struct fpu *fpu);
-extern void fpu__drop(struct fpu *fpu);
-extern int  fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal,
+extern void fpu__drop(struct task_struct *tsk);
+extern int  fpu_clone(struct task_struct *dst, u64 clone_flags, bool minimal,
 		      unsigned long shstk_addr);
 extern void fpu_flush_thread(void);
 
 /*
  * FPU state switching for scheduling.
  *
- * This is a two-stage process:
+ * switch_fpu() saves the old state and sets TIF_NEED_FPU_LOAD if
+ * TIF_NEED_FPU_LOAD is not set.  This is done within the context
+ * of the old process.
  *
- *  - switch_fpu_prepare() saves the old state.
- *    This is done within the context of the old process.
- *
- *  - switch_fpu_finish() sets TIF_NEED_FPU_LOAD; the floating point state
- *    will get loaded on return to userspace, or when the kernel needs it.
- *
- * If TIF_NEED_FPU_LOAD is cleared then the CPU's FPU registers
- * are saved in the current thread's FPU register state.
- *
- * If TIF_NEED_FPU_LOAD is set then CPU's FPU registers may not
- * hold current()'s FPU registers. It is required to load the
+ * Once TIF_NEED_FPU_LOAD is set, it is required to load the
  * registers before returning to userland or using the content
  * otherwise.
  *
  * The FPU context is only stored/restored for a user task and
  * PF_KTHREAD is used to distinguish between kernel and user threads.
  */
-static inline void switch_fpu_prepare(struct task_struct *old, int cpu)
+static inline void switch_fpu(struct task_struct *old, int cpu)
 {
-	if (cpu_feature_enabled(X86_FEATURE_FPU) &&
+	if (!test_tsk_thread_flag(old, TIF_NEED_FPU_LOAD) &&
+	    cpu_feature_enabled(X86_FEATURE_FPU) &&
 	    !(old->flags & (PF_KTHREAD | PF_USER_WORKER))) {
-		struct fpu *old_fpu = &old->thread.fpu;
+		struct fpu *old_fpu = x86_task_fpu(old);
 
+		set_tsk_thread_flag(old, TIF_NEED_FPU_LOAD);
 		save_fpregs_to_fpstate(old_fpu);
 		/*
 		 * The save operation preserved register state, so the
@@ -50,7 +44,7 @@ static inline void switch_fpu_prepare(struct task_struct *old, int cpu)
 		 * current CPU number in @old_fpu, so the next return
 		 * to user space can avoid the FPU register restore
 		 * when is returns on the same CPU and still owns the
-		 * context.
+		 * context. See fpregs_restore_userregs().
 		 */
 		old_fpu->last_cpu = cpu;
 
@@ -58,14 +52,4 @@ static inline void switch_fpu_prepare(struct task_struct *old, int cpu)
 	}
 }
 
-/*
- * Delay loading of the complete FPU state until the return to userland.
- * PKRU is handled separately.
- */
-static inline void switch_fpu_finish(struct task_struct *new)
-{
-	if (cpu_feature_enabled(X86_FEATURE_FPU))
-		set_tsk_thread_flag(new, TIF_NEED_FPU_LOAD);
-}
-
 #endif /* _ASM_X86_FPU_SCHED_H */
diff --git a/arch/x86/include/asm/fpu/types.h b/arch/x86/include/asm/fpu/types.h
index de16862bf230..93e99d2583d6 100644
--- a/arch/x86/include/asm/fpu/types.h
+++ b/arch/x86/include/asm/fpu/types.h
@@ -118,13 +118,14 @@ enum xfeature {
 	XFEATURE_PKRU,
 	XFEATURE_PASID,
 	XFEATURE_CET_USER,
-	XFEATURE_CET_KERNEL_UNUSED,
+	XFEATURE_CET_KERNEL,
 	XFEATURE_RSRVD_COMP_13,
 	XFEATURE_RSRVD_COMP_14,
 	XFEATURE_LBR,
 	XFEATURE_RSRVD_COMP_16,
 	XFEATURE_XTILE_CFG,
 	XFEATURE_XTILE_DATA,
+	XFEATURE_APX,
 
 	XFEATURE_MAX,
 };
@@ -141,10 +142,11 @@ enum xfeature {
 #define XFEATURE_MASK_PKRU		(1 << XFEATURE_PKRU)
 #define XFEATURE_MASK_PASID		(1 << XFEATURE_PASID)
 #define XFEATURE_MASK_CET_USER		(1 << XFEATURE_CET_USER)
-#define XFEATURE_MASK_CET_KERNEL	(1 << XFEATURE_CET_KERNEL_UNUSED)
+#define XFEATURE_MASK_CET_KERNEL	(1 << XFEATURE_CET_KERNEL)
 #define XFEATURE_MASK_LBR		(1 << XFEATURE_LBR)
 #define XFEATURE_MASK_XTILE_CFG		(1 << XFEATURE_XTILE_CFG)
 #define XFEATURE_MASK_XTILE_DATA	(1 << XFEATURE_XTILE_DATA)
+#define XFEATURE_MASK_APX		(1 << XFEATURE_APX)
 
 #define XFEATURE_MASK_FPSSE		(XFEATURE_MASK_FP | XFEATURE_MASK_SSE)
 #define XFEATURE_MASK_AVX512		(XFEATURE_MASK_OPMASK \
@@ -267,6 +269,16 @@ struct cet_user_state {
 };
 
 /*
+ * State component 12 is Control-flow Enforcement supervisor states.
+ * This state includes SSP pointers for privilege levels 0 through 2.
+ */
+struct cet_supervisor_state {
+	u64 pl0_ssp;
+	u64 pl1_ssp;
+	u64 pl2_ssp;
+} __packed;
+
+/*
  * State component 15: Architectural LBR configuration state.
  * The size of Arch LBR state depends on the number of LBRs (lbr_depth).
  */
@@ -304,6 +316,13 @@ struct xtile_data {
 } __packed;
 
 /*
+ * State component 19: 8B extended general purpose register.
+ */
+struct apx_state {
+	u64				egpr[16];
+} __packed;
+
+/*
  * State component 10 is supervisor state used for context-switching the
  * PASID state.
  */
@@ -407,9 +426,11 @@ struct fpu_state_perm {
 	/*
 	 * @__state_perm:
 	 *
-	 * This bitmap indicates the permission for state components, which
-	 * are available to a thread group. The permission prctl() sets the
-	 * enabled state bits in thread_group_leader()->thread.fpu.
+	 * This bitmap indicates the permission for state components
+	 * available to a thread group, including both user and supervisor
+	 * components and software-defined bits like FPU_GUEST_PERM_LOCKED.
+	 * The permission prctl() sets the enabled state bits in
+	 * thread_group_leader()->thread.fpu.
 	 *
 	 * All run time operations use the per thread information in the
 	 * currently active fpu.fpstate which contains the xfeature masks
@@ -525,13 +546,6 @@ struct fpu_guest {
 	u64				xfeatures;
 
 	/*
-	 * @perm:			xfeature bitmap of features which are
-	 *				permitted to be enabled for the guest
-	 *				vCPU.
-	 */
-	u64				perm;
-
-	/*
 	 * @xfd_err:			Save the guest value.
 	 */
 	u64				xfd_err;
@@ -548,6 +562,31 @@ struct fpu_guest {
 };
 
 /*
+ * FPU state configuration data for fpu_guest.
+ * Initialized at boot time. Read only after init.
+ */
+struct vcpu_fpu_config {
+	/*
+	 * @size:
+	 *
+	 * The default size of the register state buffer in guest FPUs.
+	 * Includes all supported features except independent managed
+	 * features and features which have to be requested by user space
+	 * before usage.
+	 */
+	unsigned int size;
+
+	/*
+	 * @features:
+	 *
+	 * The default supported features bitmap in guest FPUs. Does not
+	 * include independent managed features and features which have to
+	 * be requested by user space before usage.
+	 */
+	u64 features;
+};
+
+/*
  * FPU state configuration data. Initialized at boot time. Read only after init.
  */
 struct fpu_state_config {
@@ -563,8 +602,9 @@ struct fpu_state_config {
 	 * @default_size:
 	 *
 	 * The default size of the register state buffer. Includes all
-	 * supported features except independent managed features and
-	 * features which have to be requested by user space before usage.
+	 * supported features except independent managed features,
+	 * guest-only features and features which have to be requested by
+	 * user space before usage.
 	 */
 	unsigned int		default_size;
 
@@ -580,8 +620,8 @@ struct fpu_state_config {
 	 * @default_features:
 	 *
 	 * The default supported features bitmap. Does not include
-	 * independent managed features and features which have to
-	 * be requested by user space before usage.
+	 * independent managed features, guest-only features and features
+	 * which have to be requested by user space before usage.
 	 */
 	u64 default_features;
 	/*
@@ -602,5 +642,6 @@ struct fpu_state_config {
 
 /* FPU state configuration information */
 extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg;
+extern struct vcpu_fpu_config guest_default_cfg;
 
 #endif /* _ASM_X86_FPU_TYPES_H */
diff --git a/arch/x86/include/asm/fpu/xstate.h b/arch/x86/include/asm/fpu/xstate.h
index d4427b88ee12..7a7dc9d56027 100644
--- a/arch/x86/include/asm/fpu/xstate.h
+++ b/arch/x86/include/asm/fpu/xstate.h
@@ -12,10 +12,6 @@
 /* Bit 63 of XCR0 is reserved for future expansion */
 #define XFEATURE_MASK_EXTEND	(~(XFEATURE_MASK_FPSSE | (1ULL << 63)))
 
-#define XSTATE_CPUID		0x0000000d
-
-#define TILE_CPUID		0x0000001d
-
 #define FXSAVE_SIZE	512
 
 #define XSAVE_HDR_SIZE	    64
@@ -36,7 +32,8 @@
 				      XFEATURE_MASK_PKRU | \
 				      XFEATURE_MASK_BNDREGS | \
 				      XFEATURE_MASK_BNDCSR | \
-				      XFEATURE_MASK_XTILE)
+				      XFEATURE_MASK_XTILE | \
+				      XFEATURE_MASK_APX)
 
 /*
  * Features which are restored when returning to user space.
@@ -49,9 +46,13 @@
 /* Features which are dynamically enabled for a process on request */
 #define XFEATURE_MASK_USER_DYNAMIC	XFEATURE_MASK_XTILE_DATA
 
+/* Supervisor features which are enabled only in guest FPUs */
+#define XFEATURE_MASK_GUEST_SUPERVISOR	XFEATURE_MASK_CET_KERNEL
+
 /* All currently supported supervisor features */
 #define XFEATURE_MASK_SUPERVISOR_SUPPORTED (XFEATURE_MASK_PASID | \
-					    XFEATURE_MASK_CET_USER)
+					    XFEATURE_MASK_CET_USER | \
+					    XFEATURE_MASK_GUEST_SUPERVISOR)
 
 /*
  * A supervisor state component may not always contain valuable information,
@@ -78,8 +79,7 @@
  * Unsupported supervisor features. When a supervisor feature in this mask is
  * supported in the future, move it to the supported supervisor feature mask.
  */
-#define XFEATURE_MASK_SUPERVISOR_UNSUPPORTED (XFEATURE_MASK_PT | \
-					      XFEATURE_MASK_CET_KERNEL)
+#define XFEATURE_MASK_SUPERVISOR_UNSUPPORTED (XFEATURE_MASK_PT)
 
 /* All supervisor states including supported and unsupported states. */
 #define XFEATURE_MASK_SUPERVISOR_ALL (XFEATURE_MASK_SUPERVISOR_SUPPORTED | \
diff --git a/arch/x86/include/asm/frame.h b/arch/x86/include/asm/frame.h
index fb42659f6e98..0ab65073c1cc 100644
--- a/arch/x86/include/asm/frame.h
+++ b/arch/x86/include/asm/frame.h
@@ -11,7 +11,7 @@
 
 #ifdef CONFIG_FRAME_POINTER
 
-#ifdef __ASSEMBLY__
+#ifdef __ASSEMBLER__
 
 .macro FRAME_BEGIN
 	push %_ASM_BP
@@ -51,7 +51,7 @@
 .endm
 #endif /* CONFIG_X86_64 */
 
-#else /* !__ASSEMBLY__ */
+#else /* !__ASSEMBLER__ */
 
 #define FRAME_BEGIN				\
 	"push %" _ASM_BP "\n"			\
@@ -82,18 +82,18 @@ static inline unsigned long encode_frame_pointer(struct pt_regs *regs)
 
 #endif /* CONFIG_X86_64 */
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #define FRAME_OFFSET __ASM_SEL(4, 8)
 
 #else /* !CONFIG_FRAME_POINTER */
 
-#ifdef __ASSEMBLY__
+#ifdef __ASSEMBLER__
 
 .macro ENCODE_FRAME_POINTER ptregs_offset=0
 .endm
 
-#else /* !__ASSEMBLY */
+#else /* !__ASSEMBLER__ */
 
 #define ENCODE_FRAME_POINTER
 
diff --git a/arch/x86/include/asm/fred.h b/arch/x86/include/asm/fred.h
index 25ca00bd70e8..2bb65677c079 100644
--- a/arch/x86/include/asm/fred.h
+++ b/arch/x86/include/asm/fred.h
@@ -9,6 +9,7 @@
 #include <linux/const.h>
 
 #include <asm/asm.h>
+#include <asm/msr.h>
 #include <asm/trapnr.h>
 
 /*
@@ -32,7 +33,7 @@
 #define FRED_CONFIG_INT_STKLVL(l)	(_AT(unsigned long, l) << 9)
 #define FRED_CONFIG_ENTRYPOINT(p)	_AT(unsigned long, (p))
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #ifdef CONFIG_X86_FRED
 #include <linux/kernel.h>
@@ -78,7 +79,7 @@ static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int
 		.type   = type,
 		.vector = vector,
 		.nmi    = type == EVENT_TYPE_NMI,
-		.lm     = 1,
+		.l      = 1,
 	};
 
 	asm_fred_entry_from_kvm(ss);
@@ -113,6 +114,6 @@ static inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) {
 static inline void fred_sync_rsp0(unsigned long rsp0) { }
 static inline void fred_update_rsp0(void) { }
 #endif /* CONFIG_X86_FRED */
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #endif /* ASM_X86_FRED_H */
diff --git a/arch/x86/include/asm/fsgsbase.h b/arch/x86/include/asm/fsgsbase.h
index 9e7e8ca8e299..ab2547f97c2c 100644
--- a/arch/x86/include/asm/fsgsbase.h
+++ b/arch/x86/include/asm/fsgsbase.h
@@ -2,7 +2,7 @@
 #ifndef _ASM_FSGSBASE_H
 #define _ASM_FSGSBASE_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #ifdef CONFIG_X86_64
 
@@ -60,7 +60,7 @@ static inline unsigned long x86_fsbase_read_cpu(void)
 	if (boot_cpu_has(X86_FEATURE_FSGSBASE))
 		fsbase = rdfsbase();
 	else
-		rdmsrl(MSR_FS_BASE, fsbase);
+		rdmsrq(MSR_FS_BASE, fsbase);
 
 	return fsbase;
 }
@@ -70,7 +70,7 @@ static inline void x86_fsbase_write_cpu(unsigned long fsbase)
 	if (boot_cpu_has(X86_FEATURE_FSGSBASE))
 		wrfsbase(fsbase);
 	else
-		wrmsrl(MSR_FS_BASE, fsbase);
+		wrmsrq(MSR_FS_BASE, fsbase);
 }
 
 extern unsigned long x86_gsbase_read_cpu_inactive(void);
@@ -80,6 +80,6 @@ extern unsigned long x86_fsgsbase_read_task(struct task_struct *task,
 
 #endif /* CONFIG_X86_64 */
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ASM_FSGSBASE_H */
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index b4d719de2c84..b08c95872eed 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -22,7 +22,7 @@
 #define ARCH_SUPPORTS_FTRACE_OPS 1
 #endif
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 extern void __fentry__(void);
 
 static inline unsigned long ftrace_call_adjust(unsigned long addr)
@@ -34,38 +34,49 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr)
 	return addr;
 }
 
+static inline unsigned long arch_ftrace_get_symaddr(unsigned long fentry_ip)
+{
+	if (is_endbr((void*)(fentry_ip - ENDBR_INSN_SIZE)))
+		fentry_ip -= ENDBR_INSN_SIZE;
+
+	return fentry_ip;
+}
+#define ftrace_get_symaddr(fentry_ip)	arch_ftrace_get_symaddr(fentry_ip)
+
 #ifdef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
-struct ftrace_regs {
-	struct pt_regs		regs;
-};
+
+#include <linux/ftrace_regs.h>
 
 static __always_inline struct pt_regs *
 arch_ftrace_get_regs(struct ftrace_regs *fregs)
 {
 	/* Only when FL_SAVE_REGS is set, cs will be non zero */
-	if (!fregs->regs.cs)
+	if (!arch_ftrace_regs(fregs)->regs.cs)
 		return NULL;
-	return &fregs->regs;
+	return &arch_ftrace_regs(fregs)->regs;
 }
 
+#define arch_ftrace_partial_regs(regs) do {	\
+	regs->flags &= ~X86_EFLAGS_FIXED;	\
+	regs->cs = __KERNEL_CS;			\
+} while (0)
+
+#define arch_ftrace_fill_perf_regs(fregs, _regs) do {	\
+		(_regs)->ip = arch_ftrace_regs(fregs)->regs.ip;		\
+		(_regs)->sp = arch_ftrace_regs(fregs)->regs.sp;		\
+		(_regs)->cs = __KERNEL_CS;		\
+		(_regs)->flags = 0;			\
+	} while (0)
+
 #define ftrace_regs_set_instruction_pointer(fregs, _ip)	\
-	do { (fregs)->regs.ip = (_ip); } while (0)
-
-#define ftrace_regs_get_instruction_pointer(fregs) \
-	((fregs)->regs.ip)
-
-#define ftrace_regs_get_argument(fregs, n) \
-	regs_get_kernel_argument(&(fregs)->regs, n)
-#define ftrace_regs_get_stack_pointer(fregs) \
-	kernel_stack_pointer(&(fregs)->regs)
-#define ftrace_regs_return_value(fregs) \
-	regs_return_value(&(fregs)->regs)
-#define ftrace_regs_set_return_value(fregs, ret) \
-	regs_set_return_value(&(fregs)->regs, ret)
-#define ftrace_override_function_with_return(fregs) \
-	override_function_with_return(&(fregs)->regs)
-#define ftrace_regs_query_register_offset(name) \
-	regs_query_register_offset(name)
+	do { arch_ftrace_regs(fregs)->regs.ip = (_ip); } while (0)
+
+
+static __always_inline unsigned long
+ftrace_regs_get_return_address(struct ftrace_regs *fregs)
+{
+	return *(unsigned long *)ftrace_regs_get_stack_pointer(fregs);
+}
 
 struct ftrace_ops;
 #define ftrace_graph_func ftrace_graph_func
@@ -90,7 +101,7 @@ __arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr)
 	regs->orig_ax = addr;
 }
 #define arch_ftrace_set_direct_caller(fregs, addr) \
-	__arch_ftrace_set_direct_caller(&(fregs)->regs, addr)
+	__arch_ftrace_set_direct_caller(&arch_ftrace_regs(fregs)->regs, addr)
 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
 
 #ifdef CONFIG_DYNAMIC_FTRACE
@@ -100,11 +111,11 @@ struct dyn_arch_ftrace {
 };
 
 #endif /*  CONFIG_DYNAMIC_FTRACE */
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* CONFIG_FUNCTION_TRACER */
 
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 void prepare_ftrace_return(unsigned long ip, unsigned long *parent,
 			   unsigned long frame_pointer);
@@ -148,26 +159,6 @@ static inline bool arch_trace_is_compat_syscall(struct pt_regs *regs)
 }
 #endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_IA32_EMULATION */
 #endif /* !COMPILE_OFFSETS */
-#endif /* !__ASSEMBLY__ */
-
-#ifndef __ASSEMBLY__
-#ifdef CONFIG_FUNCTION_GRAPH_TRACER
-struct fgraph_ret_regs {
-	unsigned long ax;
-	unsigned long dx;
-	unsigned long bp;
-};
-
-static inline unsigned long fgraph_ret_regs_return_value(struct fgraph_ret_regs *ret_regs)
-{
-	return ret_regs->ax;
-}
-
-static inline unsigned long fgraph_ret_regs_frame_pointer(struct fgraph_ret_regs *ret_regs)
-{
-	return ret_regs->bp;
-}
-#endif /* ifdef CONFIG_FUNCTION_GRAPH_TRACER */
-#endif
+#endif /* !__ASSEMBLER__ */
 
 #endif /* _ASM_X86_FTRACE_H */
diff --git a/arch/x86/include/asm/futex.h b/arch/x86/include/asm/futex.h
index 99d345b686fa..fe5d9a10d900 100644
--- a/arch/x86/include/asm/futex.h
+++ b/arch/x86/include/asm/futex.h
@@ -46,36 +46,31 @@ do {								\
 } while(0)
 
 static __always_inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval,
-		u32 __user *uaddr)
+						       u32 __user *uaddr)
 {
-	if (!user_access_begin(uaddr, sizeof(u32)))
-		return -EFAULT;
-
-	switch (op) {
-	case FUTEX_OP_SET:
-		unsafe_atomic_op1("xchgl %0, %2", oval, uaddr, oparg, Efault);
-		break;
-	case FUTEX_OP_ADD:
-		unsafe_atomic_op1(LOCK_PREFIX "xaddl %0, %2", oval,
-				   uaddr, oparg, Efault);
-		break;
-	case FUTEX_OP_OR:
-		unsafe_atomic_op2("orl %4, %3", oval, uaddr, oparg, Efault);
-		break;
-	case FUTEX_OP_ANDN:
-		unsafe_atomic_op2("andl %4, %3", oval, uaddr, ~oparg, Efault);
-		break;
-	case FUTEX_OP_XOR:
-		unsafe_atomic_op2("xorl %4, %3", oval, uaddr, oparg, Efault);
-		break;
-	default:
-		user_access_end();
-		return -ENOSYS;
+	scoped_user_rw_access(uaddr, Efault) {
+		switch (op) {
+		case FUTEX_OP_SET:
+			unsafe_atomic_op1("xchgl %0, %2", oval, uaddr, oparg, Efault);
+			break;
+		case FUTEX_OP_ADD:
+			unsafe_atomic_op1(LOCK_PREFIX "xaddl %0, %2", oval, uaddr, oparg, Efault);
+			break;
+		case FUTEX_OP_OR:
+			unsafe_atomic_op2("orl %4, %3", oval, uaddr, oparg, Efault);
+			break;
+		case FUTEX_OP_ANDN:
+			unsafe_atomic_op2("andl %4, %3", oval, uaddr, ~oparg, Efault);
+			break;
+		case FUTEX_OP_XOR:
+			unsafe_atomic_op2("xorl %4, %3", oval, uaddr, oparg, Efault);
+			break;
+		default:
+			return -ENOSYS;
+		}
 	}
-	user_access_end();
 	return 0;
 Efault:
-	user_access_end();
 	return -EFAULT;
 }
 
@@ -84,19 +79,19 @@ static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
 {
 	int ret = 0;
 
-	if (!user_access_begin(uaddr, sizeof(u32)))
-		return -EFAULT;
-	asm volatile("\n"
-		"1:\t" LOCK_PREFIX "cmpxchgl %3, %2\n"
-		"2:\n"
-		_ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %0) \
-		: "+r" (ret), "=a" (oldval), "+m" (*uaddr)
-		: "r" (newval), "1" (oldval)
-		: "memory"
-	);
-	user_access_end();
-	*uval = oldval;
+	scoped_user_rw_access(uaddr, Efault) {
+		asm_inline volatile("\n"
+				    "1:\t" LOCK_PREFIX "cmpxchgl %3, %2\n"
+				    "2:\n"
+				    _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %0)
+				    : "+r" (ret), "=a" (oldval), "+m" (*uaddr)
+				    : "r" (newval), "1" (oldval)
+				    : "memory");
+		*uval = oldval;
+	}
 	return ret;
+Efault:
+	return -EFAULT;
 }
 
 #endif
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 6ffa8b75f4cd..6b6d472baa0b 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -3,10 +3,9 @@
 #define _ASM_X86_HARDIRQ_H
 
 #include <linux/threads.h>
-#include <asm/current.h>
 
 typedef struct {
-#if IS_ENABLED(CONFIG_KVM_INTEL)
+#if IS_ENABLED(CONFIG_CPU_MITIGATIONS) && IS_ENABLED(CONFIG_KVM_INTEL)
 	u8	     kvm_cpu_l1tf_flush_l1d;
 #endif
 	unsigned int __nmi_count;	/* arch dependent */
@@ -66,9 +65,10 @@ extern u64 arch_irq_stat_cpu(unsigned int cpu);
 extern u64 arch_irq_stat(void);
 #define arch_irq_stat		arch_irq_stat
 
-#define local_softirq_pending_ref       pcpu_hot.softirq_pending
+DECLARE_PER_CPU_CACHE_HOT(u16, __softirq_pending);
+#define local_softirq_pending_ref       __softirq_pending
 
-#if IS_ENABLED(CONFIG_KVM_INTEL)
+#if IS_ENABLED(CONFIG_CPU_MITIGATIONS) && IS_ENABLED(CONFIG_KVM_INTEL)
 /*
  * This function is called from noinstr interrupt contexts
  * and must be inlined to not get instrumentation.
diff --git a/arch/x86/include/asm/highmem.h b/arch/x86/include/asm/highmem.h
index 731ee7cc40a5..585bdadba47d 100644
--- a/arch/x86/include/asm/highmem.h
+++ b/arch/x86/include/asm/highmem.h
@@ -69,9 +69,6 @@ extern unsigned long highstart_pfn, highend_pfn;
 		arch_flush_lazy_mmu_mode();		\
 	} while (0)
 
-extern void add_highpages_with_active_regions(int nid, unsigned long start_pfn,
-					unsigned long end_pfn);
-
 #endif /* __KERNEL__ */
 
 #endif /* _ASM_X86_HIGHMEM_H */
diff --git a/arch/x86/include/asm/hpet.h b/arch/x86/include/asm/hpet.h
index ab9f3dd87c80..ab0c78855ecb 100644
--- a/arch/x86/include/asm/hpet.h
+++ b/arch/x86/include/asm/hpet.h
@@ -84,7 +84,6 @@ extern int hpet_set_rtc_irq_bit(unsigned long bit_mask);
 extern int hpet_set_alarm_time(unsigned char hrs, unsigned char min,
 			       unsigned char sec);
 extern int hpet_set_periodic_freq(unsigned long freq);
-extern int hpet_rtc_dropped_irq(void);
 extern int hpet_rtc_timer_init(void);
 extern irqreturn_t hpet_rtc_interrupt(int irq, void *dev_id);
 extern int hpet_register_irq_handler(rtc_irq_handler handler);
diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
index edebf1020e04..cbe19e669080 100644
--- a/arch/x86/include/asm/hw_irq.h
+++ b/arch/x86/include/asm/hw_irq.h
@@ -16,7 +16,7 @@
 
 #include <asm/irq_vectors.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/percpu.h>
 #include <linux/profile.h>
@@ -92,8 +92,6 @@ struct irq_cfg {
 
 extern struct irq_cfg *irq_cfg(unsigned int irq);
 extern struct irq_cfg *irqd_cfg(struct irq_data *irq_data);
-extern void lock_vector_lock(void);
-extern void unlock_vector_lock(void);
 #ifdef CONFIG_SMP
 extern void vector_schedule_cleanup(struct irq_cfg *);
 extern void irq_complete_move(struct irq_cfg *cfg);
@@ -101,12 +99,16 @@ extern void irq_complete_move(struct irq_cfg *cfg);
 static inline void vector_schedule_cleanup(struct irq_cfg *c) { }
 static inline void irq_complete_move(struct irq_cfg *c) { }
 #endif
-
 extern void apic_ack_edge(struct irq_data *data);
-#else	/*  CONFIG_IRQ_DOMAIN_HIERARCHY */
+#endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */
+
+#ifdef CONFIG_X86_LOCAL_APIC
+extern void lock_vector_lock(void);
+extern void unlock_vector_lock(void);
+#else
 static inline void lock_vector_lock(void) {}
 static inline void unlock_vector_lock(void) {}
-#endif	/* CONFIG_IRQ_DOMAIN_HIERARCHY */
+#endif
 
 /* Statistics */
 extern atomic_t irq_err_count;
@@ -128,6 +130,6 @@ extern char spurious_entries_start[];
 typedef struct irq_desc* vector_irq_t[NR_VECTORS];
 DECLARE_PER_CPU(vector_irq_t, vector_irq);
 
-#endif /* !ASSEMBLY_ */
+#endif /* !__ASSEMBLER__ */
 
 #endif /* _ASM_X86_HW_IRQ_H */
diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h
deleted file mode 100644
index 3787d26810c1..000000000000
--- a/arch/x86/include/asm/hyperv-tlfs.h
+++ /dev/null
@@ -1,811 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-
-/*
- * This file contains definitions from Hyper-V Hypervisor Top-Level Functional
- * Specification (TLFS):
- * https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/reference/tlfs
- */
-
-#ifndef _ASM_X86_HYPERV_TLFS_H
-#define _ASM_X86_HYPERV_TLFS_H
-
-#include <linux/types.h>
-#include <asm/page.h>
-/*
- * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
- * is set by CPUID(HvCpuIdFunctionVersionAndFeatures).
- */
-#define HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS	0x40000000
-#define HYPERV_CPUID_INTERFACE			0x40000001
-#define HYPERV_CPUID_VERSION			0x40000002
-#define HYPERV_CPUID_FEATURES			0x40000003
-#define HYPERV_CPUID_ENLIGHTMENT_INFO		0x40000004
-#define HYPERV_CPUID_IMPLEMENT_LIMITS		0x40000005
-#define HYPERV_CPUID_CPU_MANAGEMENT_FEATURES	0x40000007
-#define HYPERV_CPUID_NESTED_FEATURES		0x4000000A
-#define HYPERV_CPUID_ISOLATION_CONFIG		0x4000000C
-
-#define HYPERV_CPUID_VIRT_STACK_INTERFACE	0x40000081
-#define HYPERV_VS_INTERFACE_EAX_SIGNATURE	0x31235356  /* "VS#1" */
-
-#define HYPERV_CPUID_VIRT_STACK_PROPERTIES	0x40000082
-/* Support for the extended IOAPIC RTE format */
-#define HYPERV_VS_PROPERTIES_EAX_EXTENDED_IOAPIC_RTE	BIT(2)
-
-#define HYPERV_HYPERVISOR_PRESENT_BIT		0x80000000
-#define HYPERV_CPUID_MIN			0x40000005
-#define HYPERV_CPUID_MAX			0x4000ffff
-
-/*
- * Group D Features.  The bit assignments are custom to each architecture.
- * On x86/x64 these are HYPERV_CPUID_FEATURES.EDX bits.
- */
-/* The MWAIT instruction is available (per section MONITOR / MWAIT) */
-#define HV_X64_MWAIT_AVAILABLE				BIT(0)
-/* Guest debugging support is available */
-#define HV_X64_GUEST_DEBUGGING_AVAILABLE		BIT(1)
-/* Performance Monitor support is available*/
-#define HV_X64_PERF_MONITOR_AVAILABLE			BIT(2)
-/* Support for physical CPU dynamic partitioning events is available*/
-#define HV_X64_CPU_DYNAMIC_PARTITIONING_AVAILABLE	BIT(3)
-/*
- * Support for passing hypercall input parameter block via XMM
- * registers is available
- */
-#define HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE		BIT(4)
-/* Support for a virtual guest idle state is available */
-#define HV_X64_GUEST_IDLE_STATE_AVAILABLE		BIT(5)
-/* Frequency MSRs available */
-#define HV_FEATURE_FREQUENCY_MSRS_AVAILABLE		BIT(8)
-/* Crash MSR available */
-#define HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE		BIT(10)
-/* Support for debug MSRs available */
-#define HV_FEATURE_DEBUG_MSRS_AVAILABLE			BIT(11)
-/* Support for extended gva ranges for flush hypercalls available */
-#define HV_FEATURE_EXT_GVA_RANGES_FLUSH			BIT(14)
-/*
- * Support for returning hypercall output block via XMM
- * registers is available
- */
-#define HV_X64_HYPERCALL_XMM_OUTPUT_AVAILABLE		BIT(15)
-/* stimer Direct Mode is available */
-#define HV_STIMER_DIRECT_MODE_AVAILABLE			BIT(19)
-
-/*
- * Implementation recommendations. Indicates which behaviors the hypervisor
- * recommends the OS implement for optimal performance.
- * These are HYPERV_CPUID_ENLIGHTMENT_INFO.EAX bits.
- */
-/*
- * Recommend using hypercall for address space switches rather
- * than MOV to CR3 instruction
- */
-#define HV_X64_AS_SWITCH_RECOMMENDED			BIT(0)
-/* Recommend using hypercall for local TLB flushes rather
- * than INVLPG or MOV to CR3 instructions */
-#define HV_X64_LOCAL_TLB_FLUSH_RECOMMENDED		BIT(1)
-/*
- * Recommend using hypercall for remote TLB flushes rather
- * than inter-processor interrupts
- */
-#define HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED		BIT(2)
-/*
- * Recommend using MSRs for accessing APIC registers
- * EOI, ICR and TPR rather than their memory-mapped counterparts
- */
-#define HV_X64_APIC_ACCESS_RECOMMENDED			BIT(3)
-/* Recommend using the hypervisor-provided MSR to initiate a system RESET */
-#define HV_X64_SYSTEM_RESET_RECOMMENDED			BIT(4)
-/*
- * Recommend using relaxed timing for this partition. If used,
- * the VM should disable any watchdog timeouts that rely on the
- * timely delivery of external interrupts
- */
-#define HV_X64_RELAXED_TIMING_RECOMMENDED		BIT(5)
-
-/*
- * Recommend not using Auto End-Of-Interrupt feature
- */
-#define HV_DEPRECATING_AEOI_RECOMMENDED			BIT(9)
-
-/*
- * Recommend using cluster IPI hypercalls.
- */
-#define HV_X64_CLUSTER_IPI_RECOMMENDED			BIT(10)
-
-/* Recommend using the newer ExProcessorMasks interface */
-#define HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED		BIT(11)
-
-/* Indicates that the hypervisor is nested within a Hyper-V partition. */
-#define HV_X64_HYPERV_NESTED				BIT(12)
-
-/* Recommend using enlightened VMCS */
-#define HV_X64_ENLIGHTENED_VMCS_RECOMMENDED		BIT(14)
-
-/* Use hypercalls for MMIO config space access */
-#define HV_X64_USE_MMIO_HYPERCALLS			BIT(21)
-
-/*
- * CPU management features identification.
- * These are HYPERV_CPUID_CPU_MANAGEMENT_FEATURES.EAX bits.
- */
-#define HV_X64_START_LOGICAL_PROCESSOR			BIT(0)
-#define HV_X64_CREATE_ROOT_VIRTUAL_PROCESSOR		BIT(1)
-#define HV_X64_PERFORMANCE_COUNTER_SYNC			BIT(2)
-#define HV_X64_RESERVED_IDENTITY_BIT			BIT(31)
-
-/*
- * Virtual processor will never share a physical core with another virtual
- * processor, except for virtual processors that are reported as sibling SMT
- * threads.
- */
-#define HV_X64_NO_NONARCH_CORESHARING			BIT(18)
-
-/* Nested features. These are HYPERV_CPUID_NESTED_FEATURES.EAX bits. */
-#define HV_X64_NESTED_DIRECT_FLUSH			BIT(17)
-#define HV_X64_NESTED_GUEST_MAPPING_FLUSH		BIT(18)
-#define HV_X64_NESTED_MSR_BITMAP			BIT(19)
-
-/* Nested features #2. These are HYPERV_CPUID_NESTED_FEATURES.EBX bits. */
-#define HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL		BIT(0)
-
-/*
- * This is specific to AMD and specifies that enlightened TLB flush is
- * supported. If guest opts in to this feature, ASID invalidations only
- * flushes gva -> hpa mapping entries. To flush the TLB entries derived
- * from NPT, hypercalls should be used (HvFlushGuestPhysicalAddressSpace
- * or HvFlushGuestPhysicalAddressList).
- */
-#define HV_X64_NESTED_ENLIGHTENED_TLB			BIT(22)
-
-/* HYPERV_CPUID_ISOLATION_CONFIG.EAX bits. */
-#define HV_PARAVISOR_PRESENT				BIT(0)
-
-/* HYPERV_CPUID_ISOLATION_CONFIG.EBX bits. */
-#define HV_ISOLATION_TYPE				GENMASK(3, 0)
-#define HV_SHARED_GPA_BOUNDARY_ACTIVE			BIT(5)
-#define HV_SHARED_GPA_BOUNDARY_BITS			GENMASK(11, 6)
-
-enum hv_isolation_type {
-	HV_ISOLATION_TYPE_NONE	= 0,
-	HV_ISOLATION_TYPE_VBS	= 1,
-	HV_ISOLATION_TYPE_SNP	= 2,
-	HV_ISOLATION_TYPE_TDX	= 3
-};
-
-/* Hyper-V specific model specific registers (MSRs) */
-
-/* MSR used to identify the guest OS. */
-#define HV_X64_MSR_GUEST_OS_ID			0x40000000
-
-/* MSR used to setup pages used to communicate with the hypervisor. */
-#define HV_X64_MSR_HYPERCALL			0x40000001
-
-/* MSR used to provide vcpu index */
-#define HV_X64_MSR_VP_INDEX			0x40000002
-
-/* MSR used to reset the guest OS. */
-#define HV_X64_MSR_RESET			0x40000003
-
-/* MSR used to provide vcpu runtime in 100ns units */
-#define HV_X64_MSR_VP_RUNTIME			0x40000010
-
-/* MSR used to read the per-partition time reference counter */
-#define HV_X64_MSR_TIME_REF_COUNT		0x40000020
-
-/* A partition's reference time stamp counter (TSC) page */
-#define HV_X64_MSR_REFERENCE_TSC		0x40000021
-
-/* MSR used to retrieve the TSC frequency */
-#define HV_X64_MSR_TSC_FREQUENCY		0x40000022
-
-/* MSR used to retrieve the local APIC timer frequency */
-#define HV_X64_MSR_APIC_FREQUENCY		0x40000023
-
-/* Define the virtual APIC registers */
-#define HV_X64_MSR_EOI				0x40000070
-#define HV_X64_MSR_ICR				0x40000071
-#define HV_X64_MSR_TPR				0x40000072
-#define HV_X64_MSR_VP_ASSIST_PAGE		0x40000073
-
-/* Define synthetic interrupt controller model specific registers. */
-#define HV_X64_MSR_SCONTROL			0x40000080
-#define HV_X64_MSR_SVERSION			0x40000081
-#define HV_X64_MSR_SIEFP			0x40000082
-#define HV_X64_MSR_SIMP				0x40000083
-#define HV_X64_MSR_EOM				0x40000084
-#define HV_X64_MSR_SINT0			0x40000090
-#define HV_X64_MSR_SINT1			0x40000091
-#define HV_X64_MSR_SINT2			0x40000092
-#define HV_X64_MSR_SINT3			0x40000093
-#define HV_X64_MSR_SINT4			0x40000094
-#define HV_X64_MSR_SINT5			0x40000095
-#define HV_X64_MSR_SINT6			0x40000096
-#define HV_X64_MSR_SINT7			0x40000097
-#define HV_X64_MSR_SINT8			0x40000098
-#define HV_X64_MSR_SINT9			0x40000099
-#define HV_X64_MSR_SINT10			0x4000009A
-#define HV_X64_MSR_SINT11			0x4000009B
-#define HV_X64_MSR_SINT12			0x4000009C
-#define HV_X64_MSR_SINT13			0x4000009D
-#define HV_X64_MSR_SINT14			0x4000009E
-#define HV_X64_MSR_SINT15			0x4000009F
-
-/*
- * Define synthetic interrupt controller model specific registers for
- * nested hypervisor.
- */
-#define HV_X64_MSR_NESTED_SCONTROL		0x40001080
-#define HV_X64_MSR_NESTED_SVERSION		0x40001081
-#define HV_X64_MSR_NESTED_SIEFP			0x40001082
-#define HV_X64_MSR_NESTED_SIMP			0x40001083
-#define HV_X64_MSR_NESTED_EOM			0x40001084
-#define HV_X64_MSR_NESTED_SINT0			0x40001090
-
-/*
- * Synthetic Timer MSRs. Four timers per vcpu.
- */
-#define HV_X64_MSR_STIMER0_CONFIG		0x400000B0
-#define HV_X64_MSR_STIMER0_COUNT		0x400000B1
-#define HV_X64_MSR_STIMER1_CONFIG		0x400000B2
-#define HV_X64_MSR_STIMER1_COUNT		0x400000B3
-#define HV_X64_MSR_STIMER2_CONFIG		0x400000B4
-#define HV_X64_MSR_STIMER2_COUNT		0x400000B5
-#define HV_X64_MSR_STIMER3_CONFIG		0x400000B6
-#define HV_X64_MSR_STIMER3_COUNT		0x400000B7
-
-/* Hyper-V guest idle MSR */
-#define HV_X64_MSR_GUEST_IDLE			0x400000F0
-
-/* Hyper-V guest crash notification MSR's */
-#define HV_X64_MSR_CRASH_P0			0x40000100
-#define HV_X64_MSR_CRASH_P1			0x40000101
-#define HV_X64_MSR_CRASH_P2			0x40000102
-#define HV_X64_MSR_CRASH_P3			0x40000103
-#define HV_X64_MSR_CRASH_P4			0x40000104
-#define HV_X64_MSR_CRASH_CTL			0x40000105
-
-/* TSC emulation after migration */
-#define HV_X64_MSR_REENLIGHTENMENT_CONTROL	0x40000106
-#define HV_X64_MSR_TSC_EMULATION_CONTROL	0x40000107
-#define HV_X64_MSR_TSC_EMULATION_STATUS		0x40000108
-
-/* TSC invariant control */
-#define HV_X64_MSR_TSC_INVARIANT_CONTROL	0x40000118
-
-/* HV_X64_MSR_TSC_INVARIANT_CONTROL bits */
-#define HV_EXPOSE_INVARIANT_TSC		BIT_ULL(0)
-
-/*
- * To support arch-generic code calling hv_set/get_register:
- * - On x86, HV_MSR_ indicates an MSR accessed via rdmsrl/wrmsrl
- * - On ARM, HV_MSR_ indicates a VP register accessed via hypercall
- */
-#define HV_MSR_CRASH_P0		(HV_X64_MSR_CRASH_P0)
-#define HV_MSR_CRASH_P1		(HV_X64_MSR_CRASH_P1)
-#define HV_MSR_CRASH_P2		(HV_X64_MSR_CRASH_P2)
-#define HV_MSR_CRASH_P3		(HV_X64_MSR_CRASH_P3)
-#define HV_MSR_CRASH_P4		(HV_X64_MSR_CRASH_P4)
-#define HV_MSR_CRASH_CTL	(HV_X64_MSR_CRASH_CTL)
-
-#define HV_MSR_VP_INDEX		(HV_X64_MSR_VP_INDEX)
-#define HV_MSR_TIME_REF_COUNT	(HV_X64_MSR_TIME_REF_COUNT)
-#define HV_MSR_REFERENCE_TSC	(HV_X64_MSR_REFERENCE_TSC)
-
-#define HV_MSR_SINT0		(HV_X64_MSR_SINT0)
-#define HV_MSR_SVERSION		(HV_X64_MSR_SVERSION)
-#define HV_MSR_SCONTROL		(HV_X64_MSR_SCONTROL)
-#define HV_MSR_SIEFP		(HV_X64_MSR_SIEFP)
-#define HV_MSR_SIMP		(HV_X64_MSR_SIMP)
-#define HV_MSR_EOM		(HV_X64_MSR_EOM)
-
-#define HV_MSR_NESTED_SCONTROL	(HV_X64_MSR_NESTED_SCONTROL)
-#define HV_MSR_NESTED_SVERSION	(HV_X64_MSR_NESTED_SVERSION)
-#define HV_MSR_NESTED_SIEFP	(HV_X64_MSR_NESTED_SIEFP)
-#define HV_MSR_NESTED_SIMP	(HV_X64_MSR_NESTED_SIMP)
-#define HV_MSR_NESTED_EOM	(HV_X64_MSR_NESTED_EOM)
-#define HV_MSR_NESTED_SINT0	(HV_X64_MSR_NESTED_SINT0)
-
-#define HV_MSR_STIMER0_CONFIG	(HV_X64_MSR_STIMER0_CONFIG)
-#define HV_MSR_STIMER0_COUNT	(HV_X64_MSR_STIMER0_COUNT)
-
-/*
- * Registers are only accessible via HVCALL_GET_VP_REGISTERS hvcall and
- * there is not associated MSR address.
- */
-#define	HV_X64_REGISTER_VSM_VP_STATUS	0x000D0003
-#define	HV_X64_VTL_MASK			GENMASK(3, 0)
-
-/* Hyper-V memory host visibility */
-enum hv_mem_host_visibility {
-	VMBUS_PAGE_NOT_VISIBLE		= 0,
-	VMBUS_PAGE_VISIBLE_READ_ONLY	= 1,
-	VMBUS_PAGE_VISIBLE_READ_WRITE	= 3
-};
-
-/* HvCallModifySparseGpaPageHostVisibility hypercall */
-#define HV_MAX_MODIFY_GPA_REP_COUNT	((PAGE_SIZE / sizeof(u64)) - 2)
-struct hv_gpa_range_for_visibility {
-	u64 partition_id;
-	u32 host_visibility:2;
-	u32 reserved0:30;
-	u32 reserved1;
-	u64 gpa_page_list[HV_MAX_MODIFY_GPA_REP_COUNT];
-} __packed;
-
-/*
- * Declare the MSR used to setup pages used to communicate with the hypervisor.
- */
-union hv_x64_msr_hypercall_contents {
-	u64 as_uint64;
-	struct {
-		u64 enable:1;
-		u64 reserved:11;
-		u64 guest_physical_address:52;
-	} __packed;
-};
-
-union hv_vp_assist_msr_contents {
-	u64 as_uint64;
-	struct {
-		u64 enable:1;
-		u64 reserved:11;
-		u64 pfn:52;
-	} __packed;
-};
-
-struct hv_reenlightenment_control {
-	__u64 vector:8;
-	__u64 reserved1:8;
-	__u64 enabled:1;
-	__u64 reserved2:15;
-	__u64 target_vp:32;
-}  __packed;
-
-struct hv_tsc_emulation_control {
-	__u64 enabled:1;
-	__u64 reserved:63;
-} __packed;
-
-struct hv_tsc_emulation_status {
-	__u64 inprogress:1;
-	__u64 reserved:63;
-} __packed;
-
-#define HV_X64_MSR_HYPERCALL_ENABLE		0x00000001
-#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT	12
-#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK	\
-		(~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1))
-
-#define HV_X64_MSR_CRASH_PARAMS		\
-		(1 + (HV_X64_MSR_CRASH_P4 - HV_X64_MSR_CRASH_P0))
-
-#define HV_IPI_LOW_VECTOR	0x10
-#define HV_IPI_HIGH_VECTOR	0xff
-
-#define HV_X64_MSR_VP_ASSIST_PAGE_ENABLE	0x00000001
-#define HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT	12
-#define HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_MASK	\
-		(~((1ull << HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT) - 1))
-
-/* Hyper-V Enlightened VMCS version mask in nested features CPUID */
-#define HV_X64_ENLIGHTENED_VMCS_VERSION		0xff
-
-#define HV_X64_MSR_TSC_REFERENCE_ENABLE		0x00000001
-#define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT	12
-
-/* Number of XMM registers used in hypercall input/output */
-#define HV_HYPERCALL_MAX_XMM_REGISTERS		6
-
-struct hv_nested_enlightenments_control {
-	struct {
-		__u32 directhypercall:1;
-		__u32 reserved:31;
-	} features;
-	struct {
-		__u32 inter_partition_comm:1;
-		__u32 reserved:31;
-	} hypercallControls;
-} __packed;
-
-/* Define virtual processor assist page structure. */
-struct hv_vp_assist_page {
-	__u32 apic_assist;
-	__u32 reserved1;
-	__u32 vtl_entry_reason;
-	__u32 vtl_reserved;
-	__u64 vtl_ret_x64rax;
-	__u64 vtl_ret_x64rcx;
-	struct hv_nested_enlightenments_control nested_control;
-	__u8 enlighten_vmentry;
-	__u8 reserved2[7];
-	__u64 current_nested_vmcs;
-	__u8 synthetic_time_unhalted_timer_expired;
-	__u8 reserved3[7];
-	__u8 virtualization_fault_information[40];
-	__u8 reserved4[8];
-	__u8 intercept_message[256];
-	__u8 vtl_ret_actions[256];
-} __packed;
-
-struct hv_enlightened_vmcs {
-	u32 revision_id;
-	u32 abort;
-
-	u16 host_es_selector;
-	u16 host_cs_selector;
-	u16 host_ss_selector;
-	u16 host_ds_selector;
-	u16 host_fs_selector;
-	u16 host_gs_selector;
-	u16 host_tr_selector;
-
-	u16 padding16_1;
-
-	u64 host_ia32_pat;
-	u64 host_ia32_efer;
-
-	u64 host_cr0;
-	u64 host_cr3;
-	u64 host_cr4;
-
-	u64 host_ia32_sysenter_esp;
-	u64 host_ia32_sysenter_eip;
-	u64 host_rip;
-	u32 host_ia32_sysenter_cs;
-
-	u32 pin_based_vm_exec_control;
-	u32 vm_exit_controls;
-	u32 secondary_vm_exec_control;
-
-	u64 io_bitmap_a;
-	u64 io_bitmap_b;
-	u64 msr_bitmap;
-
-	u16 guest_es_selector;
-	u16 guest_cs_selector;
-	u16 guest_ss_selector;
-	u16 guest_ds_selector;
-	u16 guest_fs_selector;
-	u16 guest_gs_selector;
-	u16 guest_ldtr_selector;
-	u16 guest_tr_selector;
-
-	u32 guest_es_limit;
-	u32 guest_cs_limit;
-	u32 guest_ss_limit;
-	u32 guest_ds_limit;
-	u32 guest_fs_limit;
-	u32 guest_gs_limit;
-	u32 guest_ldtr_limit;
-	u32 guest_tr_limit;
-	u32 guest_gdtr_limit;
-	u32 guest_idtr_limit;
-
-	u32 guest_es_ar_bytes;
-	u32 guest_cs_ar_bytes;
-	u32 guest_ss_ar_bytes;
-	u32 guest_ds_ar_bytes;
-	u32 guest_fs_ar_bytes;
-	u32 guest_gs_ar_bytes;
-	u32 guest_ldtr_ar_bytes;
-	u32 guest_tr_ar_bytes;
-
-	u64 guest_es_base;
-	u64 guest_cs_base;
-	u64 guest_ss_base;
-	u64 guest_ds_base;
-	u64 guest_fs_base;
-	u64 guest_gs_base;
-	u64 guest_ldtr_base;
-	u64 guest_tr_base;
-	u64 guest_gdtr_base;
-	u64 guest_idtr_base;
-
-	u64 padding64_1[3];
-
-	u64 vm_exit_msr_store_addr;
-	u64 vm_exit_msr_load_addr;
-	u64 vm_entry_msr_load_addr;
-
-	u64 cr3_target_value0;
-	u64 cr3_target_value1;
-	u64 cr3_target_value2;
-	u64 cr3_target_value3;
-
-	u32 page_fault_error_code_mask;
-	u32 page_fault_error_code_match;
-
-	u32 cr3_target_count;
-	u32 vm_exit_msr_store_count;
-	u32 vm_exit_msr_load_count;
-	u32 vm_entry_msr_load_count;
-
-	u64 tsc_offset;
-	u64 virtual_apic_page_addr;
-	u64 vmcs_link_pointer;
-
-	u64 guest_ia32_debugctl;
-	u64 guest_ia32_pat;
-	u64 guest_ia32_efer;
-
-	u64 guest_pdptr0;
-	u64 guest_pdptr1;
-	u64 guest_pdptr2;
-	u64 guest_pdptr3;
-
-	u64 guest_pending_dbg_exceptions;
-	u64 guest_sysenter_esp;
-	u64 guest_sysenter_eip;
-
-	u32 guest_activity_state;
-	u32 guest_sysenter_cs;
-
-	u64 cr0_guest_host_mask;
-	u64 cr4_guest_host_mask;
-	u64 cr0_read_shadow;
-	u64 cr4_read_shadow;
-	u64 guest_cr0;
-	u64 guest_cr3;
-	u64 guest_cr4;
-	u64 guest_dr7;
-
-	u64 host_fs_base;
-	u64 host_gs_base;
-	u64 host_tr_base;
-	u64 host_gdtr_base;
-	u64 host_idtr_base;
-	u64 host_rsp;
-
-	u64 ept_pointer;
-
-	u16 virtual_processor_id;
-	u16 padding16_2[3];
-
-	u64 padding64_2[5];
-	u64 guest_physical_address;
-
-	u32 vm_instruction_error;
-	u32 vm_exit_reason;
-	u32 vm_exit_intr_info;
-	u32 vm_exit_intr_error_code;
-	u32 idt_vectoring_info_field;
-	u32 idt_vectoring_error_code;
-	u32 vm_exit_instruction_len;
-	u32 vmx_instruction_info;
-
-	u64 exit_qualification;
-	u64 exit_io_instruction_ecx;
-	u64 exit_io_instruction_esi;
-	u64 exit_io_instruction_edi;
-	u64 exit_io_instruction_eip;
-
-	u64 guest_linear_address;
-	u64 guest_rsp;
-	u64 guest_rflags;
-
-	u32 guest_interruptibility_info;
-	u32 cpu_based_vm_exec_control;
-	u32 exception_bitmap;
-	u32 vm_entry_controls;
-	u32 vm_entry_intr_info_field;
-	u32 vm_entry_exception_error_code;
-	u32 vm_entry_instruction_len;
-	u32 tpr_threshold;
-
-	u64 guest_rip;
-
-	u32 hv_clean_fields;
-	u32 padding32_1;
-	u32 hv_synthetic_controls;
-	struct {
-		u32 nested_flush_hypercall:1;
-		u32 msr_bitmap:1;
-		u32 reserved:30;
-	}  __packed hv_enlightenments_control;
-	u32 hv_vp_id;
-	u32 padding32_2;
-	u64 hv_vm_id;
-	u64 partition_assist_page;
-	u64 padding64_4[4];
-	u64 guest_bndcfgs;
-	u64 guest_ia32_perf_global_ctrl;
-	u64 guest_ia32_s_cet;
-	u64 guest_ssp;
-	u64 guest_ia32_int_ssp_table_addr;
-	u64 guest_ia32_lbr_ctl;
-	u64 padding64_5[2];
-	u64 xss_exit_bitmap;
-	u64 encls_exiting_bitmap;
-	u64 host_ia32_perf_global_ctrl;
-	u64 tsc_multiplier;
-	u64 host_ia32_s_cet;
-	u64 host_ssp;
-	u64 host_ia32_int_ssp_table_addr;
-	u64 padding64_6;
-} __packed;
-
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE			0
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP		BIT(0)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP		BIT(1)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2		BIT(2)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1		BIT(3)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC		BIT(4)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT		BIT(5)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY		BIT(6)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN		BIT(7)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR			BIT(8)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT		BIT(9)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC		BIT(10)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1		BIT(11)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2		BIT(12)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER		BIT(13)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1		BIT(14)
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_ENLIGHTENMENTSCONTROL	BIT(15)
-
-#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL			0xFFFF
-
-/*
- * Note, Hyper-V isn't actually stealing bit 28 from Intel, just abusing it by
- * pairing it with architecturally impossible exit reasons.  Bit 28 is set only
- * on SMI exits to a SMI transfer monitor (STM) and if and only if a MTF VM-Exit
- * is pending.  I.e. it will never be set by hardware for non-SMI exits (there
- * are only three), nor will it ever be set unless the VMM is an STM.
- */
-#define HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH		0x10000031
-
-/*
- * Hyper-V uses the software reserved 32 bytes in VMCB control area to expose
- * SVM enlightenments to guests.
- */
-struct hv_vmcb_enlightenments {
-	struct __packed hv_enlightenments_control {
-		u32 nested_flush_hypercall:1;
-		u32 msr_bitmap:1;
-		u32 enlightened_npt_tlb: 1;
-		u32 reserved:29;
-	} __packed hv_enlightenments_control;
-	u32 hv_vp_id;
-	u64 hv_vm_id;
-	u64 partition_assist_page;
-	u64 reserved;
-} __packed;
-
-/*
- * Hyper-V uses the software reserved clean bit in VMCB.
- */
-#define HV_VMCB_NESTED_ENLIGHTENMENTS		31
-
-/* Synthetic VM-Exit */
-#define HV_SVM_EXITCODE_ENL			0xf0000000
-#define HV_SVM_ENL_EXITCODE_TRAP_AFTER_FLUSH	(1)
-
-struct hv_partition_assist_pg {
-	u32 tlb_lock_count;
-};
-
-enum hv_interrupt_type {
-	HV_X64_INTERRUPT_TYPE_FIXED             = 0x0000,
-	HV_X64_INTERRUPT_TYPE_LOWESTPRIORITY    = 0x0001,
-	HV_X64_INTERRUPT_TYPE_SMI               = 0x0002,
-	HV_X64_INTERRUPT_TYPE_REMOTEREAD        = 0x0003,
-	HV_X64_INTERRUPT_TYPE_NMI               = 0x0004,
-	HV_X64_INTERRUPT_TYPE_INIT              = 0x0005,
-	HV_X64_INTERRUPT_TYPE_SIPI              = 0x0006,
-	HV_X64_INTERRUPT_TYPE_EXTINT            = 0x0007,
-	HV_X64_INTERRUPT_TYPE_LOCALINT0         = 0x0008,
-	HV_X64_INTERRUPT_TYPE_LOCALINT1         = 0x0009,
-	HV_X64_INTERRUPT_TYPE_MAXIMUM           = 0x000A,
-};
-
-union hv_msi_address_register {
-	u32 as_uint32;
-	struct {
-		u32 reserved1:2;
-		u32 destination_mode:1;
-		u32 redirection_hint:1;
-		u32 reserved2:8;
-		u32 destination_id:8;
-		u32 msi_base:12;
-	};
-} __packed;
-
-union hv_msi_data_register {
-	u32 as_uint32;
-	struct {
-		u32 vector:8;
-		u32 delivery_mode:3;
-		u32 reserved1:3;
-		u32 level_assert:1;
-		u32 trigger_mode:1;
-		u32 reserved2:16;
-	};
-} __packed;
-
-/* HvRetargetDeviceInterrupt hypercall */
-union hv_msi_entry {
-	u64 as_uint64;
-	struct {
-		union hv_msi_address_register address;
-		union hv_msi_data_register data;
-	} __packed;
-};
-
-struct hv_x64_segment_register {
-	u64 base;
-	u32 limit;
-	u16 selector;
-	union {
-		struct {
-			u16 segment_type : 4;
-			u16 non_system_segment : 1;
-			u16 descriptor_privilege_level : 2;
-			u16 present : 1;
-			u16 reserved : 4;
-			u16 available : 1;
-			u16 _long : 1;
-			u16 _default : 1;
-			u16 granularity : 1;
-		} __packed;
-		u16 attributes;
-	};
-} __packed;
-
-struct hv_x64_table_register {
-	u16 pad[3];
-	u16 limit;
-	u64 base;
-} __packed;
-
-struct hv_init_vp_context {
-	u64 rip;
-	u64 rsp;
-	u64 rflags;
-
-	struct hv_x64_segment_register cs;
-	struct hv_x64_segment_register ds;
-	struct hv_x64_segment_register es;
-	struct hv_x64_segment_register fs;
-	struct hv_x64_segment_register gs;
-	struct hv_x64_segment_register ss;
-	struct hv_x64_segment_register tr;
-	struct hv_x64_segment_register ldtr;
-
-	struct hv_x64_table_register idtr;
-	struct hv_x64_table_register gdtr;
-
-	u64 efer;
-	u64 cr0;
-	u64 cr3;
-	u64 cr4;
-	u64 msr_cr_pat;
-} __packed;
-
-union hv_input_vtl {
-	u8 as_uint8;
-	struct {
-		u8 target_vtl: 4;
-		u8 use_target_vtl: 1;
-		u8 reserved_z: 3;
-	};
-} __packed;
-
-struct hv_enable_vp_vtl {
-	u64				partition_id;
-	u32				vp_index;
-	union hv_input_vtl		target_vtl;
-	u8				mbz0;
-	u16				mbz1;
-	struct hv_init_vp_context	vp_context;
-} __packed;
-
-struct hv_get_vp_from_apic_id_in {
-	u64 partition_id;
-	union hv_input_vtl target_vtl;
-	u8 res[7];
-	u32 apic_ids[];
-} __packed;
-
-#include <asm-generic/hyperv-tlfs.h>
-
-#endif
diff --git a/arch/x86/include/asm/hypervisor.h b/arch/x86/include/asm/hypervisor.h
index e41cbf2ec41d..9ad86a7d13f6 100644
--- a/arch/x86/include/asm/hypervisor.h
+++ b/arch/x86/include/asm/hypervisor.h
@@ -30,6 +30,7 @@ enum x86_hypervisor_type {
 	X86_HYPER_KVM,
 	X86_HYPER_JAILHOUSE,
 	X86_HYPER_ACRN,
+	X86_HYPER_BHYVE,
 };
 
 #ifdef CONFIG_HYPERVISOR_GUEST
@@ -64,6 +65,7 @@ extern const struct hypervisor_x86 x86_hyper_xen_pv;
 extern const struct hypervisor_x86 x86_hyper_kvm;
 extern const struct hypervisor_x86 x86_hyper_jailhouse;
 extern const struct hypervisor_x86 x86_hyper_acrn;
+extern const struct hypervisor_x86 x86_hyper_bhyve;
 extern struct hypervisor_x86 x86_hyper_xen_hvm;
 
 extern bool nopv;
diff --git a/arch/x86/include/asm/ibt.h b/arch/x86/include/asm/ibt.h
index 1e59581d500c..5e45d6424722 100644
--- a/arch/x86/include/asm/ibt.h
+++ b/arch/x86/include/asm/ibt.h
@@ -21,7 +21,7 @@
 
 #define HAS_KERNEL_IBT	1
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #ifdef CONFIG_X86_64
 #define ASM_ENDBR	"endbr64\n\t"
@@ -41,7 +41,7 @@
 	_ASM_PTR fname "\n\t"				\
 	".popsection\n\t"
 
-static inline __attribute_const__ u32 gen_endbr(void)
+static __always_inline __attribute_const__ u32 gen_endbr(void)
 {
 	u32 endbr;
 
@@ -56,16 +56,16 @@ static inline __attribute_const__ u32 gen_endbr(void)
 	return endbr;
 }
 
-static inline __attribute_const__ u32 gen_endbr_poison(void)
+static __always_inline __attribute_const__ u32 gen_endbr_poison(void)
 {
 	/*
-	 * 4 byte NOP that isn't NOP4 (in fact it is OSP NOP3), such that it
-	 * will be unique to (former) ENDBR sites.
+	 * 4 byte NOP that isn't NOP4, such that it will be unique to (former)
+	 * ENDBR sites. Additionally it carries UDB as immediate.
 	 */
-	return 0x001f0f66; /* osp nopl (%rax) */
+	return 0xd6401f0f; /* nopl -42(%rax) */
 }
 
-static inline bool is_endbr(u32 val)
+static inline bool __is_endbr(u32 val)
 {
 	if (val == gen_endbr_poison())
 		return true;
@@ -74,10 +74,11 @@ static inline bool is_endbr(u32 val)
 	return val == gen_endbr();
 }
 
+extern __noendbr bool is_endbr(u32 *val);
 extern __noendbr u64 ibt_save(bool disable);
 extern __noendbr void ibt_restore(u64 save);
 
-#else /* __ASSEMBLY__ */
+#else /* __ASSEMBLER__ */
 
 #ifdef CONFIG_X86_64
 #define ENDBR	endbr64
@@ -85,29 +86,29 @@ extern __noendbr void ibt_restore(u64 save);
 #define ENDBR	endbr32
 #endif
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #else /* !IBT */
 
 #define HAS_KERNEL_IBT	0
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #define ASM_ENDBR
 #define IBT_NOSEAL(name)
 
 #define __noendbr
 
-static inline bool is_endbr(u32 val) { return false; }
+static inline bool is_endbr(u32 *val) { return false; }
 
 static inline u64 ibt_save(bool disable) { return 0; }
 static inline void ibt_restore(u64 save) { }
 
-#else /* __ASSEMBLY__ */
+#else /* __ASSEMBLER__ */
 
 #define ENDBR
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* CONFIG_X86_KERNEL_IBT */
 
diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index ad5c68f0509d..3218770670d3 100644
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -7,7 +7,7 @@
 
 #define IDT_ALIGN	(8 * (1 + HAS_KERNEL_IBT))
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/entry-common.h>
 #include <linux/hardirq.h>
 
@@ -393,7 +393,7 @@ static __always_inline void __##func(struct pt_regs *regs)
 
 /**
  * DEFINE_IDTENTRY_VC_KERNEL - Emit code for VMM communication handler
-			       when raised from kernel mode
+ *			       when raised from kernel mode
  * @func:	Function name of the entry point
  *
  * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
@@ -403,7 +403,7 @@ static __always_inline void __##func(struct pt_regs *regs)
 
 /**
  * DEFINE_IDTENTRY_VC_USER - Emit code for VMM communication handler
-			     when raised from user mode
+ *			     when raised from user mode
  * @func:	Function name of the entry point
  *
  * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
@@ -460,21 +460,16 @@ __visible noinstr void func(struct pt_regs *regs,			\
 #endif
 
 void idt_install_sysvec(unsigned int n, const void *function);
-
-#ifdef CONFIG_X86_FRED
 void fred_install_sysvec(unsigned int vector, const idtentry_t function);
-#else
-static inline void fred_install_sysvec(unsigned int vector, const idtentry_t function) { }
-#endif
 
 #define sysvec_install(vector, function) {				\
-	if (cpu_feature_enabled(X86_FEATURE_FRED))			\
+	if (IS_ENABLED(CONFIG_X86_FRED))				\
 		fred_install_sysvec(vector, function);			\
-	else								\
+	if (!cpu_feature_enabled(X86_FEATURE_FRED))			\
 		idt_install_sysvec(vector, asm_##function);		\
 }
 
-#else /* !__ASSEMBLY__ */
+#else /* !__ASSEMBLER__ */
 
 /*
  * The ASM variants for DECLARE_IDTENTRY*() which emit the ASM entry stubs.
@@ -579,7 +574,7 @@ SYM_CODE_START(spurious_entries_start)
 SYM_CODE_END(spurious_entries_start)
 #endif
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 /*
  * The actual entry points. Note that DECLARE_IDTENTRY*() serves two
diff --git a/arch/x86/include/asm/inat.h b/arch/x86/include/asm/inat.h
index 53e4015242b4..1b3060a3425c 100644
--- a/arch/x86/include/asm/inat.h
+++ b/arch/x86/include/asm/inat.h
@@ -37,6 +37,8 @@
 #define INAT_PFX_EVEX	15	/* EVEX prefix */
 /* x86-64 REX2 prefix */
 #define INAT_PFX_REX2	16	/* 0xD5 */
+/* AMD XOP prefix */
+#define INAT_PFX_XOP	17	/* 0x8F */
 
 #define INAT_LSTPFX_MAX	3
 #define INAT_LGCPFX_MAX	11
@@ -77,11 +79,13 @@
 #define INAT_MOFFSET	(1 << (INAT_FLAG_OFFS + 3))
 #define INAT_VARIANT	(1 << (INAT_FLAG_OFFS + 4))
 #define INAT_VEXOK	(1 << (INAT_FLAG_OFFS + 5))
+#define INAT_XOPOK	INAT_VEXOK
 #define INAT_VEXONLY	(1 << (INAT_FLAG_OFFS + 6))
 #define INAT_EVEXONLY	(1 << (INAT_FLAG_OFFS + 7))
 #define INAT_NO_REX2	(1 << (INAT_FLAG_OFFS + 8))
 #define INAT_REX2_VARIANT	(1 << (INAT_FLAG_OFFS + 9))
 #define INAT_EVEX_SCALABLE	(1 << (INAT_FLAG_OFFS + 10))
+#define INAT_INV64	(1 << (INAT_FLAG_OFFS + 11))
 /* Attribute making macros for attribute tables */
 #define INAT_MAKE_PREFIX(pfx)	(pfx << INAT_PFX_OFFS)
 #define INAT_MAKE_ESCAPE(esc)	(esc << INAT_ESC_OFFS)
@@ -110,6 +114,8 @@ extern insn_attr_t inat_get_group_attribute(insn_byte_t modrm,
 extern insn_attr_t inat_get_avx_attribute(insn_byte_t opcode,
 					  insn_byte_t vex_m,
 					  insn_byte_t vex_pp);
+extern insn_attr_t inat_get_xop_attribute(insn_byte_t opcode,
+					  insn_byte_t map_select);
 
 /* Attribute checking functions */
 static inline int inat_is_legacy_prefix(insn_attr_t attr)
@@ -163,6 +169,11 @@ static inline int inat_is_vex3_prefix(insn_attr_t attr)
 	return (attr & INAT_PFX_MASK) == INAT_PFX_VEX3;
 }
 
+static inline int inat_is_xop_prefix(insn_attr_t attr)
+{
+	return (attr & INAT_PFX_MASK) == INAT_PFX_XOP;
+}
+
 static inline int inat_is_escape(insn_attr_t attr)
 {
 	return attr & INAT_ESC_MASK;
@@ -228,6 +239,11 @@ static inline int inat_accept_vex(insn_attr_t attr)
 	return attr & INAT_VEXOK;
 }
 
+static inline int inat_accept_xop(insn_attr_t attr)
+{
+	return attr & INAT_XOPOK;
+}
+
 static inline int inat_must_vex(insn_attr_t attr)
 {
 	return attr & (INAT_VEXONLY | INAT_EVEXONLY);
@@ -242,4 +258,9 @@ static inline int inat_evex_scalable(insn_attr_t attr)
 {
 	return attr & INAT_EVEX_SCALABLE;
 }
+
+static inline int inat_is_invalid64(insn_attr_t attr)
+{
+	return attr & INAT_INV64;
+}
 #endif
diff --git a/arch/x86/include/asm/init.h b/arch/x86/include/asm/init.h
index 14d72727d7ee..01ccdd168df0 100644
--- a/arch/x86/include/asm/init.h
+++ b/arch/x86/include/asm/init.h
@@ -2,8 +2,6 @@
 #ifndef _ASM_X86_INIT_H
 #define _ASM_X86_INIT_H
 
-#define __head	__section(".head.text")
-
 struct x86_mapping_info {
 	void *(*alloc_pgt_page)(void *); /* allocate buf for page table */
 	void (*free_pgt_page)(void *, void *); /* free buf for page table */
diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h
index 54368a43abf6..4733e9064ee5 100644
--- a/arch/x86/include/asm/insn-eval.h
+++ b/arch/x86/include/asm/insn-eval.h
@@ -44,4 +44,6 @@ enum insn_mmio_type {
 
 enum insn_mmio_type insn_decode_mmio(struct insn *insn, int *bytes);
 
+bool insn_is_nop(struct insn *insn);
+
 #endif /* _ASM_X86_INSN_EVAL_H */
diff --git a/arch/x86/include/asm/insn.h b/arch/x86/include/asm/insn.h
index 7152ea809e6a..846d21c1a7f8 100644
--- a/arch/x86/include/asm/insn.h
+++ b/arch/x86/include/asm/insn.h
@@ -71,7 +71,10 @@ struct insn {
 					 * prefixes.bytes[3]: last prefix
 					 */
 	struct insn_field rex_prefix;	/* REX prefix */
-	struct insn_field vex_prefix;	/* VEX prefix */
+	union {
+		struct insn_field vex_prefix;	/* VEX prefix */
+		struct insn_field xop_prefix;	/* XOP prefix */
+	};
 	struct insn_field opcode;	/*
 					 * opcode.bytes[0]: opcode1
 					 * opcode.bytes[1]: opcode2
@@ -135,6 +138,17 @@ struct insn {
 #define X86_VEX_V(vex)	(((vex) & 0x78) >> 3)	/* VEX3 Byte2, VEX2 Byte1 */
 #define X86_VEX_P(vex)	((vex) & 0x03)		/* VEX3 Byte2, VEX2 Byte1 */
 #define X86_VEX_M_MAX	0x1f			/* VEX3.M Maximum value */
+/* XOP bit fields */
+#define X86_XOP_R(xop)	((xop) & 0x80)	/* XOP Byte2 */
+#define X86_XOP_X(xop)	((xop) & 0x40)	/* XOP Byte2 */
+#define X86_XOP_B(xop)	((xop) & 0x20)	/* XOP Byte2 */
+#define X86_XOP_M(xop)	((xop) & 0x1f)	/* XOP Byte2 */
+#define X86_XOP_W(xop)	((xop) & 0x80)	/* XOP Byte3 */
+#define X86_XOP_V(xop)	((xop) & 0x78)	/* XOP Byte3 */
+#define X86_XOP_L(xop)	((xop) & 0x04)	/* XOP Byte3 */
+#define X86_XOP_P(xop)	((xop) & 0x03)	/* XOP Byte3 */
+#define X86_XOP_M_MIN	0x08	/* Min of XOP.M */
+#define X86_XOP_M_MAX	0x1f	/* Max of XOP.M */
 
 extern void insn_init(struct insn *insn, const void *kaddr, int buf_len, int x86_64);
 extern int insn_get_prefixes(struct insn *insn);
@@ -178,7 +192,7 @@ static inline insn_byte_t insn_rex2_m_bit(struct insn *insn)
 	return X86_REX2_M(insn->rex_prefix.bytes[1]);
 }
 
-static inline int insn_is_avx(struct insn *insn)
+static inline int insn_is_avx_or_xop(struct insn *insn)
 {
 	if (!insn->prefixes.got)
 		insn_get_prefixes(insn);
@@ -192,6 +206,22 @@ static inline int insn_is_evex(struct insn *insn)
 	return (insn->vex_prefix.nbytes == 4);
 }
 
+/* If we already know this is AVX/XOP encoded */
+static inline int avx_insn_is_xop(struct insn *insn)
+{
+	insn_attr_t attr = inat_get_opcode_attribute(insn->vex_prefix.bytes[0]);
+
+	return inat_is_xop_prefix(attr);
+}
+
+static inline int insn_is_xop(struct insn *insn)
+{
+	if (!insn_is_avx_or_xop(insn))
+		return 0;
+
+	return avx_insn_is_xop(insn);
+}
+
 static inline int insn_has_emulate_prefix(struct insn *insn)
 {
 	return !!insn->emulate_prefix_size;
@@ -222,11 +252,26 @@ static inline insn_byte_t insn_vex_w_bit(struct insn *insn)
 	return X86_VEX_W(insn->vex_prefix.bytes[2]);
 }
 
+static inline insn_byte_t insn_xop_map_bits(struct insn *insn)
+{
+	if (insn->xop_prefix.nbytes < 3)	/* XOP is 3 bytes */
+		return 0;
+	return X86_XOP_M(insn->xop_prefix.bytes[1]);
+}
+
+static inline insn_byte_t insn_xop_p_bits(struct insn *insn)
+{
+	return X86_XOP_P(insn->vex_prefix.bytes[2]);
+}
+
 /* Get the last prefix id from last prefix or VEX prefix */
 static inline int insn_last_prefix_id(struct insn *insn)
 {
-	if (insn_is_avx(insn))
+	if (insn_is_avx_or_xop(insn)) {
+		if (avx_insn_is_xop(insn))
+			return insn_xop_p_bits(insn);
 		return insn_vex_p_bits(insn);	/* VEX_p is a SIMD prefix id */
+	}
 
 	if (insn->prefixes.bytes[3])
 		return inat_get_last_prefix_id(insn->prefixes.bytes[3]);
@@ -267,7 +312,6 @@ static inline int insn_offset_immediate(struct insn *insn)
 /**
  * for_each_insn_prefix() -- Iterate prefixes in the instruction
  * @insn: Pointer to struct insn.
- * @idx:  Index storage.
  * @prefix: Prefix byte.
  *
  * Iterate prefix bytes of given @insn. Each prefix byte is stored in @prefix
@@ -276,8 +320,8 @@ static inline int insn_offset_immediate(struct insn *insn)
  * Since prefixes.nbytes can be bigger than 4 if some prefixes
  * are repeated, it cannot be used for looping over the prefixes.
  */
-#define for_each_insn_prefix(insn, idx, prefix)	\
-	for (idx = 0; idx < ARRAY_SIZE(insn->prefixes.bytes) && (prefix = insn->prefixes.bytes[idx]) != 0; idx++)
+#define for_each_insn_prefix(insn, prefix)	\
+	for (int idx = 0; idx < ARRAY_SIZE(insn->prefixes.bytes) && (prefix = insn->prefixes.bytes[idx]) != 0; idx++)
 
 #define POP_SS_OPCODE 0x1f
 #define MOV_SREG_OPCODE 0x8e
diff --git a/arch/x86/include/asm/inst.h b/arch/x86/include/asm/inst.h
index 438ccd4f3cc4..e48a00b3311d 100644
--- a/arch/x86/include/asm/inst.h
+++ b/arch/x86/include/asm/inst.h
@@ -6,7 +6,7 @@
 #ifndef X86_ASM_INST_H
 #define X86_ASM_INST_H
 
-#ifdef __ASSEMBLY__
+#ifdef __ASSEMBLER__
 
 #define REG_NUM_INVALID		100
 
diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h
index 1a42f829667a..950bfd006905 100644
--- a/arch/x86/include/asm/intel-family.h
+++ b/arch/x86/include/asm/intel-family.h
@@ -45,7 +45,18 @@
 /* Wildcard match so X86_MATCH_VFM(ANY) works */
 #define INTEL_ANY			IFM(X86_FAMILY_ANY, X86_MODEL_ANY)
 
+/* Family 5 */
+#define INTEL_FAM5_START		IFM(5, 0x00) /* Notational marker, also P5 A-step */
+#define INTEL_PENTIUM_75		IFM(5, 0x02) /* P54C */
+#define INTEL_PENTIUM_MMX		IFM(5, 0x04) /* P55C */
+#define INTEL_QUARK_X1000		IFM(5, 0x09) /* Quark X1000 SoC */
+
+/* Family 6, 18, 19 */
 #define INTEL_PENTIUM_PRO		IFM(6, 0x01)
+#define INTEL_PENTIUM_II_KLAMATH	IFM(6, 0x03)
+#define INTEL_PENTIUM_III_DESCHUTES	IFM(6, 0x05)
+#define INTEL_PENTIUM_III_TUALATIN	IFM(6, 0x0B)
+#define INTEL_PENTIUM_M_DOTHAN		IFM(6, 0x0D)
 
 #define INTEL_CORE_YONAH		IFM(6, 0x0E)
 
@@ -110,11 +121,15 @@
 
 #define INTEL_SAPPHIRERAPIDS_X		IFM(6, 0x8F) /* Golden Cove */
 
-#define INTEL_EMERALDRAPIDS_X		IFM(6, 0xCF)
+#define INTEL_EMERALDRAPIDS_X		IFM(6, 0xCF) /* Raptor Cove */
 
-#define INTEL_GRANITERAPIDS_X		IFM(6, 0xAD)
+#define INTEL_GRANITERAPIDS_X		IFM(6, 0xAD) /* Redwood Cove */
 #define INTEL_GRANITERAPIDS_D		IFM(6, 0xAE)
 
+#define INTEL_DIAMONDRAPIDS_X		IFM(19, 0x01) /* Panther Cove */
+
+#define INTEL_BARTLETTLAKE		IFM(6, 0xD7) /* Raptor Cove */
+
 /* "Hybrid" Processors (P-Core/E-Core) */
 
 #define INTEL_LAKEFIELD			IFM(6, 0x8A) /* Sunny Cove / Tremont */
@@ -126,16 +141,21 @@
 #define INTEL_RAPTORLAKE_P		IFM(6, 0xBA)
 #define INTEL_RAPTORLAKE_S		IFM(6, 0xBF)
 
-#define INTEL_METEORLAKE		IFM(6, 0xAC)
+#define INTEL_METEORLAKE		IFM(6, 0xAC) /* Redwood Cove / Crestmont */
 #define INTEL_METEORLAKE_L		IFM(6, 0xAA)
 
-#define INTEL_ARROWLAKE_H		IFM(6, 0xC5)
+#define INTEL_ARROWLAKE_H		IFM(6, 0xC5) /* Lion Cove / Skymont */
 #define INTEL_ARROWLAKE			IFM(6, 0xC6)
 #define INTEL_ARROWLAKE_U		IFM(6, 0xB5)
 
-#define INTEL_LUNARLAKE_M		IFM(6, 0xBD)
+#define INTEL_LUNARLAKE_M		IFM(6, 0xBD) /* Lion Cove / Skymont */
+
+#define INTEL_PANTHERLAKE_L		IFM(6, 0xCC) /* Cougar Cove / Darkmont */
+
+#define INTEL_WILDCATLAKE_L		IFM(6, 0xD5)
 
-#define INTEL_PANTHERLAKE_L		IFM(6, 0xCC)
+#define INTEL_NOVALAKE			IFM(18, 0x01) /* Coyote Cove / Arctic Wolf */
+#define INTEL_NOVALAKE_L		IFM(18, 0x03) /* Coyote Cove / Arctic Wolf */
 
 /* "Small Core" Processors (Atom/E-Core) */
 
@@ -149,9 +169,9 @@
 #define INTEL_ATOM_SILVERMONT		IFM(6, 0x37) /* Bay Trail, Valleyview */
 #define INTEL_ATOM_SILVERMONT_D		IFM(6, 0x4D) /* Avaton, Rangely */
 #define INTEL_ATOM_SILVERMONT_MID	IFM(6, 0x4A) /* Merriefield */
+#define INTEL_ATOM_SILVERMONT_MID2	IFM(6, 0x5A) /* Anniedale */
 
 #define INTEL_ATOM_AIRMONT		IFM(6, 0x4C) /* Cherry Trail, Braswell */
-#define INTEL_ATOM_AIRMONT_MID		IFM(6, 0x5A) /* Moorefield */
 #define INTEL_ATOM_AIRMONT_NP		IFM(6, 0x75) /* Lightning Mountain */
 
 #define INTEL_ATOM_GOLDMONT		IFM(6, 0x5C) /* Apollo Lake */
@@ -176,11 +196,32 @@
 #define INTEL_XEON_PHI_KNL		IFM(6, 0x57) /* Knights Landing */
 #define INTEL_XEON_PHI_KNM		IFM(6, 0x85) /* Knights Mill */
 
-/* Family 5 */
-#define INTEL_FAM5_QUARK_X1000		0x09 /* Quark X1000 SoC */
-#define INTEL_QUARK_X1000		IFM(5, 0x09) /* Quark X1000 SoC */
+/* Notational marker denoting the last Family 6 model */
+#define INTEL_FAM6_LAST			IFM(6, 0xFF)
+
+/* Family 15 - NetBurst */
+#define INTEL_P4_WILLAMETTE		IFM(15, 0x01) /* Also Xeon Foster */
+#define INTEL_P4_PRESCOTT		IFM(15, 0x03)
+#define INTEL_P4_PRESCOTT_2M		IFM(15, 0x04)
+#define INTEL_P4_CEDARMILL		IFM(15, 0x06) /* Also Xeon Dempsey */
 
-/* Family 19 */
-#define INTEL_PANTHERCOVE_X		IFM(19, 0x01) /* Diamond Rapids */
+/*
+ * Intel CPU core types
+ *
+ * CPUID.1AH.EAX[31:0] uniquely identifies the microarchitecture
+ * of the core. Bits 31-24 indicates its core type (Core or Atom)
+ * and Bits [23:0] indicates the native model ID of the core.
+ * Core type and native model ID are defined in below enumerations.
+ */
+enum intel_cpu_type {
+	INTEL_CPU_TYPE_UNKNOWN,
+	INTEL_CPU_TYPE_ATOM = 0x20,
+	INTEL_CPU_TYPE_CORE = 0x40,
+};
+
+enum intel_native_id {
+	INTEL_ATOM_CMT_NATIVE_ID = 0x2,  /* Crestmont */
+	INTEL_ATOM_SKT_NATIVE_ID = 0x3,  /* Skymont */
+};
 
 #endif /* _ASM_X86_INTEL_FAMILY_H */
diff --git a/arch/x86/include/asm/intel_ds.h b/arch/x86/include/asm/intel_ds.h
index 5dbeac48a5b9..695f87efbeb8 100644
--- a/arch/x86/include/asm/intel_ds.h
+++ b/arch/x86/include/asm/intel_ds.h
@@ -4,7 +4,15 @@
 #include <linux/percpu-defs.h>
 
 #define BTS_BUFFER_SIZE		(PAGE_SIZE << 4)
-#define PEBS_BUFFER_SIZE	(PAGE_SIZE << 4)
+#define PEBS_BUFFER_SHIFT	4
+#define PEBS_BUFFER_SIZE	(PAGE_SIZE << PEBS_BUFFER_SHIFT)
+
+/*
+ * The largest PEBS record could consume a page, ensure
+ * a record at least can be written after triggering PMI.
+ */
+#define ARCH_PEBS_THRESH_MULTI	((PEBS_BUFFER_SIZE - PAGE_SIZE) >> PEBS_BUFFER_SHIFT)
+#define ARCH_PEBS_THRESH_SINGLE	1
 
 /* The maximal number of PEBS events: */
 #define MAX_PEBS_EVENTS_FMT4	8
diff --git a/arch/x86/include/asm/intel_punit_ipc.h b/arch/x86/include/asm/intel_punit_ipc.h
index ce16da719596..1f9b5d225912 100644
--- a/arch/x86/include/asm/intel_punit_ipc.h
+++ b/arch/x86/include/asm/intel_punit_ipc.h
@@ -80,17 +80,10 @@ typedef enum {
 
 #if IS_ENABLED(CONFIG_INTEL_PUNIT_IPC)
 
-int intel_punit_ipc_simple_command(int cmd, int para1, int para2);
 int intel_punit_ipc_command(u32 cmd, u32 para1, u32 para2, u32 *in, u32 *out);
 
 #else
 
-static inline int intel_punit_ipc_simple_command(int cmd,
-						  int para1, int para2)
-{
-	return -ENODEV;
-}
-
 static inline int intel_punit_ipc_command(u32 cmd, u32 para1, u32 para2,
 					  u32 *in, u32 *out)
 {
diff --git a/arch/x86/include/asm/intel_telemetry.h b/arch/x86/include/asm/intel_telemetry.h
index 43b7657febca..944637a4e6de 100644
--- a/arch/x86/include/asm/intel_telemetry.h
+++ b/arch/x86/include/asm/intel_telemetry.h
@@ -59,18 +59,6 @@ struct telemetry_plt_config {
 };
 
 struct telemetry_core_ops {
-	int (*get_sampling_period)(u8 *pss_min_period, u8 *pss_max_period,
-				   u8 *ioss_min_period, u8 *ioss_max_period);
-
-	int (*get_eventconfig)(struct telemetry_evtconfig *pss_evtconfig,
-			       struct telemetry_evtconfig *ioss_evtconfig,
-			       int pss_len, int ioss_len);
-
-	int (*update_events)(struct telemetry_evtconfig pss_evtconfig,
-			     struct telemetry_evtconfig ioss_evtconfig);
-
-	int (*set_sampling_period)(u8 pss_period, u8 ioss_period);
-
 	int (*get_trace_verbosity)(enum telemetry_unit telem_unit,
 				   u32 *verbosity);
 
@@ -84,11 +72,6 @@ struct telemetry_core_ops {
 	int (*read_eventlog)(enum telemetry_unit telem_unit,
 			     struct telemetry_evtlog *evtlog,
 			     int len, int log_all_evts);
-
-	int (*add_events)(u8 num_pss_evts, u8 num_ioss_evts,
-			  u32 *pss_evtmap, u32 *ioss_evtmap);
-
-	int (*reset_events)(void);
 };
 
 int telemetry_set_pltdata(const struct telemetry_core_ops *ops,
@@ -101,35 +84,15 @@ struct telemetry_plt_config *telemetry_get_pltdata(void);
 int telemetry_get_evtname(enum telemetry_unit telem_unit,
 			  const char **name, int len);
 
-int telemetry_update_events(struct telemetry_evtconfig pss_evtconfig,
-			    struct telemetry_evtconfig ioss_evtconfig);
-
-int telemetry_add_events(u8 num_pss_evts, u8 num_ioss_evts,
-			 u32 *pss_evtmap, u32 *ioss_evtmap);
-
-int telemetry_reset_events(void);
-
-int telemetry_get_eventconfig(struct telemetry_evtconfig *pss_config,
-			      struct telemetry_evtconfig *ioss_config,
-			      int pss_len, int ioss_len);
-
 int telemetry_read_events(enum telemetry_unit telem_unit,
 			  struct telemetry_evtlog *evtlog, int len);
 
-int telemetry_raw_read_events(enum telemetry_unit telem_unit,
-			      struct telemetry_evtlog *evtlog, int len);
-
 int telemetry_read_eventlog(enum telemetry_unit telem_unit,
 			    struct telemetry_evtlog *evtlog, int len);
 
 int telemetry_raw_read_eventlog(enum telemetry_unit telem_unit,
 				struct telemetry_evtlog *evtlog, int len);
 
-int telemetry_get_sampling_period(u8 *pss_min_period, u8 *pss_max_period,
-				  u8 *ioss_min_period, u8 *ioss_max_period);
-
-int telemetry_set_sampling_period(u8 pss_period, u8 ioss_period);
-
 int telemetry_set_trace_verbosity(enum telemetry_unit telem_unit,
 				  u32 verbosity);
 
diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index 1d60427379c9..ca309a3227c7 100644
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -152,11 +152,6 @@ static inline void *phys_to_virt(phys_addr_t address)
 #define phys_to_virt phys_to_virt
 
 /*
- * Change "struct page" to physical address.
- */
-#define page_to_phys(page)    ((dma_addr_t)page_to_pfn(page) << PAGE_SHIFT)
-
-/*
  * ISA I/O bus memory addresses are 1:1 with the physical address.
  * However, we truncate the address to unsigned int to avoid undesirable
  * promotions in legacy drivers.
@@ -175,11 +170,14 @@ extern void __iomem *ioremap_uc(resource_size_t offset, unsigned long size);
 #define ioremap_uc ioremap_uc
 extern void __iomem *ioremap_cache(resource_size_t offset, unsigned long size);
 #define ioremap_cache ioremap_cache
-extern void __iomem *ioremap_prot(resource_size_t offset, unsigned long size, unsigned long prot_val);
+extern void __iomem *ioremap_prot(resource_size_t offset, unsigned long size,  pgprot_t prot);
 #define ioremap_prot ioremap_prot
 extern void __iomem *ioremap_encrypted(resource_size_t phys_addr, unsigned long size);
 #define ioremap_encrypted ioremap_encrypted
 
+void *arch_memremap_wb(phys_addr_t phys_addr, size_t size, unsigned long flags);
+#define arch_memremap_wb arch_memremap_wb
+
 /**
  * ioremap     -   map bus memory into CPU space
  * @offset:    bus address of the memory
@@ -219,7 +217,7 @@ void memset_io(volatile void __iomem *, int, size_t);
 static inline void __iowrite32_copy(void __iomem *to, const void *from,
 				    size_t count)
 {
-	asm volatile("rep ; movsl"
+	asm volatile("rep movsl"
 		     : "=&c"(count), "=&D"(to), "=&S"(from)
 		     : "0"(count), "1"(to), "2"(from)
 		     : "memory");
@@ -284,7 +282,7 @@ static inline void outs##bwl(u16 port, const void *addr, unsigned long count) \
 			count--;					\
 		}							\
 	} else {							\
-		asm volatile("rep; outs" #bwl				\
+		asm volatile("rep outs" #bwl				\
 			     : "+S"(addr), "+c"(count)			\
 			     : "d"(port) : "memory");			\
 	}								\
@@ -300,7 +298,7 @@ static inline void ins##bwl(u16 port, void *addr, unsigned long count)	\
 			count--;					\
 		}							\
 	} else {							\
-		asm volatile("rep; ins" #bwl				\
+		asm volatile("rep ins" #bwl				\
 			     : "+D"(addr), "+c"(count)			\
 			     : "d"(port) : "memory");			\
 	}								\
diff --git a/arch/x86/include/asm/iosf_mbi.h b/arch/x86/include/asm/iosf_mbi.h
index af7541c11821..8ace6559d399 100644
--- a/arch/x86/include/asm/iosf_mbi.h
+++ b/arch/x86/include/asm/iosf_mbi.h
@@ -168,13 +168,6 @@ void iosf_mbi_unblock_punit_i2c_access(void);
 int iosf_mbi_register_pmic_bus_access_notifier(struct notifier_block *nb);
 
 /**
- * iosf_mbi_register_pmic_bus_access_notifier - Unregister PMIC bus notifier
- *
- * @nb: notifier_block to unregister
- */
-int iosf_mbi_unregister_pmic_bus_access_notifier(struct notifier_block *nb);
-
-/**
  * iosf_mbi_unregister_pmic_bus_access_notifier_unlocked - Unregister PMIC bus
  *                                                         notifier, unlocked
  *
diff --git a/arch/x86/include/asm/irq_remapping.h b/arch/x86/include/asm/irq_remapping.h
index 5036f13ab69f..5a0d42464d44 100644
--- a/arch/x86/include/asm/irq_remapping.h
+++ b/arch/x86/include/asm/irq_remapping.h
@@ -26,7 +26,22 @@ enum {
 	IRQ_REMAP_X2APIC_MODE,
 };
 
-struct vcpu_data {
+/*
+ * This is mainly used to communicate information back-and-forth
+ * between SVM and IOMMU for setting up and tearing down posted
+ * interrupt
+ */
+struct amd_iommu_pi_data {
+	u64 vapic_addr;		/* Physical address of the vCPU's vAPIC. */
+	u32 ga_tag;
+	u32 vector;		/* Guest vector of the interrupt */
+	int cpu;
+	bool ga_log_intr;
+	bool is_guest_mode;
+	void *ir_data;
+};
+
+struct intel_iommu_pi_data {
 	u64 pi_desc_addr;	/* Physical address of PI Descriptor */
 	u32 vector;		/* Guest vector of the interrupt */
 };
diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h
index b71ad173f877..8325b79f2ac6 100644
--- a/arch/x86/include/asm/irq_stack.h
+++ b/arch/x86/include/asm/irq_stack.h
@@ -100,8 +100,8 @@
 }
 
 #define ASM_CALL_ARG0							\
-	"call %c[__func]				\n"		\
-	ASM_REACHABLE
+	"1: call %c[__func]				\n"		\
+	ANNOTATE_REACHABLE(1b) "			\n"
 
 #define ASM_CALL_ARG1							\
 	"movq	%[arg1], %%rdi				\n"		\
@@ -116,7 +116,7 @@
 	ASM_CALL_ARG2
 
 #define call_on_irqstack(func, asm_call, argconstr...)			\
-	call_on_stack(__this_cpu_read(pcpu_hot.hardirq_stack_ptr),	\
+	call_on_stack(__this_cpu_read(hardirq_stack_ptr),		\
 		      func, asm_call, argconstr)
 
 /* Macros to assert type correctness for run_*_on_irqstack macros */
@@ -135,7 +135,7 @@
 	 * User mode entry and interrupt on the irq stack do not	\
 	 * switch stacks. If from user mode the task stack is empty.	\
 	 */								\
-	if (user_mode(regs) || __this_cpu_read(pcpu_hot.hardirq_stack_inuse)) { \
+	if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) {	\
 		irq_enter_rcu();					\
 		func(c_args);						\
 		irq_exit_rcu();						\
@@ -146,9 +146,9 @@
 		 * places. Invoke the stack switch macro with the call	\
 		 * sequence which matches the above direct invocation.	\
 		 */							\
-		__this_cpu_write(pcpu_hot.hardirq_stack_inuse, true);	\
+		__this_cpu_write(hardirq_stack_inuse, true);		\
 		call_on_irqstack(func, asm_call, constr);		\
-		__this_cpu_write(pcpu_hot.hardirq_stack_inuse, false);	\
+		__this_cpu_write(hardirq_stack_inuse, false);		\
 	}								\
 }
 
@@ -212,9 +212,9 @@
  */
 #define do_softirq_own_stack()						\
 {									\
-	__this_cpu_write(pcpu_hot.hardirq_stack_inuse, true);		\
+	__this_cpu_write(hardirq_stack_inuse, true);			\
 	call_on_irqstack(__do_softirq, ASM_CALL_ARG0);			\
-	__this_cpu_write(pcpu_hot.hardirq_stack_inuse, false);		\
+	__this_cpu_write(hardirq_stack_inuse, false);			\
 }
 
 #endif
diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h
index cf7fc2b8e3ce..b30e5474c18e 100644
--- a/arch/x86/include/asm/irqflags.h
+++ b/arch/x86/include/asm/irqflags.h
@@ -4,7 +4,7 @@
 
 #include <asm/processor-flags.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <asm/nospec-branch.h>
 
@@ -44,13 +44,13 @@ static __always_inline void native_irq_enable(void)
 
 static __always_inline void native_safe_halt(void)
 {
-	mds_idle_clear_cpu_buffers();
+	x86_idle_clear_cpu_buffers();
 	asm volatile("sti; hlt": : :"memory");
 }
 
 static __always_inline void native_halt(void)
 {
-	mds_idle_clear_cpu_buffers();
+	x86_idle_clear_cpu_buffers();
 	asm volatile("hlt": : :"memory");
 }
 
@@ -76,10 +76,32 @@ static __always_inline void native_local_irq_restore(unsigned long flags)
 
 #endif
 
+#ifndef CONFIG_PARAVIRT
+#ifndef __ASSEMBLY__
+/*
+ * Used in the idle loop; sti takes one instruction cycle
+ * to complete:
+ */
+static __always_inline void arch_safe_halt(void)
+{
+	native_safe_halt();
+}
+
+/*
+ * Used when interrupts are already enabled or to
+ * shutdown the processor:
+ */
+static __always_inline void halt(void)
+{
+	native_halt();
+}
+#endif /* __ASSEMBLY__ */
+#endif /* CONFIG_PARAVIRT */
+
 #ifdef CONFIG_PARAVIRT_XXL
 #include <asm/paravirt.h>
 #else
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/types.h>
 
 static __always_inline unsigned long arch_local_save_flags(void)
@@ -98,24 +120,6 @@ static __always_inline void arch_local_irq_enable(void)
 }
 
 /*
- * Used in the idle loop; sti takes one instruction cycle
- * to complete:
- */
-static __always_inline void arch_safe_halt(void)
-{
-	native_safe_halt();
-}
-
-/*
- * Used when interrupts are already enabled or to
- * shutdown the processor:
- */
-static __always_inline void halt(void)
-{
-	native_halt();
-}
-
-/*
  * For spinlocks, etc:
  */
 static __always_inline unsigned long arch_local_irq_save(void)
@@ -133,10 +137,10 @@ static __always_inline unsigned long arch_local_irq_save(void)
 
 #endif
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* CONFIG_PARAVIRT_XXL */
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 static __always_inline int arch_irqs_disabled_flags(unsigned long flags)
 {
 	return !(flags & X86_EFLAGS_IF);
@@ -154,6 +158,6 @@ static __always_inline void arch_local_irq_restore(unsigned long flags)
 	if (!arch_irqs_disabled_flags(flags))
 		arch_local_irq_enable();
 }
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #endif
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index cbbef32517f0..05b16299588d 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -7,40 +7,34 @@
 #include <asm/asm.h>
 #include <asm/nops.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/stringify.h>
 #include <linux/types.h>
 
-#define JUMP_TABLE_ENTRY				\
+#define JUMP_TABLE_ENTRY(key, label)			\
 	".pushsection __jump_table,  \"aw\" \n\t"	\
 	_ASM_ALIGN "\n\t"				\
+	ANNOTATE_DATA_SPECIAL "\n"			\
 	".long 1b - . \n\t"				\
-	".long %l[l_yes] - . \n\t"			\
-	_ASM_PTR "%c0 + %c1 - .\n\t"			\
+	".long " label " - . \n\t"			\
+	_ASM_PTR " " key " - . \n\t"			\
 	".popsection \n\t"
 
+/* This macro is also expanded on the Rust side. */
 #ifdef CONFIG_HAVE_JUMP_LABEL_HACK
-
-static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
-{
-	asm goto("1:"
-		"jmp %l[l_yes] # objtool NOPs this \n\t"
-		JUMP_TABLE_ENTRY
-		: :  "i" (key), "i" (2 | branch) : : l_yes);
-
-	return false;
-l_yes:
-	return true;
-}
-
+#define ARCH_STATIC_BRANCH_ASM(key, label)		\
+	"1: jmp " label " # objtool NOPs this \n\t"	\
+	JUMP_TABLE_ENTRY(key " + 2", label)
 #else /* !CONFIG_HAVE_JUMP_LABEL_HACK */
+#define ARCH_STATIC_BRANCH_ASM(key, label)		\
+	"1: .byte " __stringify(BYTES_NOP5) "\n\t"	\
+	JUMP_TABLE_ENTRY(key, label)
+#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */
 
 static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch)
 {
-	asm goto("1:"
-		".byte " __stringify(BYTES_NOP5) "\n\t"
-		JUMP_TABLE_ENTRY
+	asm goto(ARCH_STATIC_BRANCH_ASM("%c0 + %c1", "%l[l_yes]")
 		: :  "i" (key), "i" (branch) : : l_yes);
 
 	return false;
@@ -48,13 +42,11 @@ l_yes:
 	return true;
 }
 
-#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */
-
 static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch)
 {
 	asm goto("1:"
 		"jmp %l[l_yes]\n\t"
-		JUMP_TABLE_ENTRY
+		JUMP_TABLE_ENTRY("%c0 + %c1", "%l[l_yes]")
 		: :  "i" (key), "i" (branch) : : l_yes);
 
 	return false;
@@ -64,6 +56,6 @@ l_yes:
 
 extern int arch_jump_entry_size(struct jump_entry *entry);
 
-#endif	/* __ASSEMBLY__ */
+#endif	/* __ASSEMBLER__ */
 
 #endif
diff --git a/arch/x86/include/asm/kasan.h b/arch/x86/include/asm/kasan.h
index de75306b932e..d7e33c7f096b 100644
--- a/arch/x86/include/asm/kasan.h
+++ b/arch/x86/include/asm/kasan.h
@@ -23,7 +23,7 @@
 					(1ULL << (__VIRTUAL_MASK_SHIFT - \
 						  KASAN_SHADOW_SCALE_SHIFT)))
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #ifdef CONFIG_KASAN
 void __init kasan_early_init(void);
diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h
index ae5482a2f0ca..5cfb27f26583 100644
--- a/arch/x86/include/asm/kexec.h
+++ b/arch/x86/include/asm/kexec.h
@@ -9,20 +9,28 @@
 # define PA_SWAP_PAGE		3
 # define PAGES_NR		4
 #else
-# define PA_CONTROL_PAGE	0
-# define VA_CONTROL_PAGE	1
-# define PA_TABLE_PAGE		2
-# define PA_SWAP_PAGE		3
-# define PAGES_NR		4
+/* Size of each exception handler referenced by the IDT */
+# define KEXEC_DEBUG_EXC_HANDLER_SIZE	6 /* PUSHI, PUSHI, 2-byte JMP */
+#endif
+
+#ifdef CONFIG_X86_64
+
+#include <linux/bits.h>
+
+#define RELOC_KERNEL_PRESERVE_CONTEXT	BIT(0)
+#define RELOC_KERNEL_CACHE_INCOHERENT	BIT(1)
+
 #endif
 
+# define KEXEC_CONTROL_PAGE_SIZE	4096
 # define KEXEC_CONTROL_CODE_MAX_SIZE	2048
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/string.h>
 #include <linux/kernel.h>
 
+#include <asm/asm.h>
 #include <asm/page.h>
 #include <asm/ptrace.h>
 
@@ -43,7 +51,6 @@ struct kimage;
 /* Maximum address we can use for the control code buffer */
 # define KEXEC_CONTROL_MEMORY_LIMIT TASK_SIZE
 
-# define KEXEC_CONTROL_PAGE_SIZE	4096
 
 /* The native architecture */
 # define KEXEC_ARCH KEXEC_ARCH_386
@@ -58,11 +65,16 @@ struct kimage;
 /* Maximum address we can use for the control pages */
 # define KEXEC_CONTROL_MEMORY_LIMIT     (MAXMEM-1)
 
-/* Allocate one page for the pdp and the second for the code */
-# define KEXEC_CONTROL_PAGE_SIZE  (4096UL + 4096UL)
-
 /* The native architecture */
 # define KEXEC_ARCH KEXEC_ARCH_X86_64
+
+extern unsigned long kexec_va_control_page;
+extern unsigned long kexec_pa_table_page;
+extern unsigned long kexec_pa_swap_page;
+extern gate_desc kexec_debug_idt[];
+extern unsigned char kexec_debug_exc_vectors[];
+extern uint16_t kexec_debug_8250_port;
+extern unsigned long kexec_debug_8250_mmio32;
 #endif
 
 /*
@@ -76,61 +88,51 @@ static inline void crash_setup_regs(struct pt_regs *newregs,
 	if (oldregs) {
 		memcpy(newregs, oldregs, sizeof(*newregs));
 	} else {
+		asm volatile("mov %%" _ASM_BX ",%0" : "=m"(newregs->bx));
+		asm volatile("mov %%" _ASM_CX ",%0" : "=m"(newregs->cx));
+		asm volatile("mov %%" _ASM_DX ",%0" : "=m"(newregs->dx));
+		asm volatile("mov %%" _ASM_SI ",%0" : "=m"(newregs->si));
+		asm volatile("mov %%" _ASM_DI ",%0" : "=m"(newregs->di));
+		asm volatile("mov %%" _ASM_BP ",%0" : "=m"(newregs->bp));
+		asm volatile("mov %%" _ASM_AX ",%0" : "=m"(newregs->ax));
+		asm volatile("mov %%" _ASM_SP ",%0" : "=m"(newregs->sp));
+#ifdef CONFIG_X86_64
+		asm volatile("mov %%r8,%0" : "=m"(newregs->r8));
+		asm volatile("mov %%r9,%0" : "=m"(newregs->r9));
+		asm volatile("mov %%r10,%0" : "=m"(newregs->r10));
+		asm volatile("mov %%r11,%0" : "=m"(newregs->r11));
+		asm volatile("mov %%r12,%0" : "=m"(newregs->r12));
+		asm volatile("mov %%r13,%0" : "=m"(newregs->r13));
+		asm volatile("mov %%r14,%0" : "=m"(newregs->r14));
+		asm volatile("mov %%r15,%0" : "=m"(newregs->r15));
+#endif
+		asm volatile("mov %%ss,%k0" : "=a"(newregs->ss));
+		asm volatile("mov %%cs,%k0" : "=a"(newregs->cs));
 #ifdef CONFIG_X86_32
-		asm volatile("movl %%ebx,%0" : "=m"(newregs->bx));
-		asm volatile("movl %%ecx,%0" : "=m"(newregs->cx));
-		asm volatile("movl %%edx,%0" : "=m"(newregs->dx));
-		asm volatile("movl %%esi,%0" : "=m"(newregs->si));
-		asm volatile("movl %%edi,%0" : "=m"(newregs->di));
-		asm volatile("movl %%ebp,%0" : "=m"(newregs->bp));
-		asm volatile("movl %%eax,%0" : "=m"(newregs->ax));
-		asm volatile("movl %%esp,%0" : "=m"(newregs->sp));
-		asm volatile("movl %%ss, %%eax;" :"=a"(newregs->ss));
-		asm volatile("movl %%cs, %%eax;" :"=a"(newregs->cs));
-		asm volatile("movl %%ds, %%eax;" :"=a"(newregs->ds));
-		asm volatile("movl %%es, %%eax;" :"=a"(newregs->es));
-		asm volatile("pushfl; popl %0" :"=m"(newregs->flags));
-#else
-		asm volatile("movq %%rbx,%0" : "=m"(newregs->bx));
-		asm volatile("movq %%rcx,%0" : "=m"(newregs->cx));
-		asm volatile("movq %%rdx,%0" : "=m"(newregs->dx));
-		asm volatile("movq %%rsi,%0" : "=m"(newregs->si));
-		asm volatile("movq %%rdi,%0" : "=m"(newregs->di));
-		asm volatile("movq %%rbp,%0" : "=m"(newregs->bp));
-		asm volatile("movq %%rax,%0" : "=m"(newregs->ax));
-		asm volatile("movq %%rsp,%0" : "=m"(newregs->sp));
-		asm volatile("movq %%r8,%0" : "=m"(newregs->r8));
-		asm volatile("movq %%r9,%0" : "=m"(newregs->r9));
-		asm volatile("movq %%r10,%0" : "=m"(newregs->r10));
-		asm volatile("movq %%r11,%0" : "=m"(newregs->r11));
-		asm volatile("movq %%r12,%0" : "=m"(newregs->r12));
-		asm volatile("movq %%r13,%0" : "=m"(newregs->r13));
-		asm volatile("movq %%r14,%0" : "=m"(newregs->r14));
-		asm volatile("movq %%r15,%0" : "=m"(newregs->r15));
-		asm volatile("movl %%ss, %%eax;" :"=a"(newregs->ss));
-		asm volatile("movl %%cs, %%eax;" :"=a"(newregs->cs));
-		asm volatile("pushfq; popq %0" :"=m"(newregs->flags));
+		asm volatile("mov %%ds,%k0" : "=a"(newregs->ds));
+		asm volatile("mov %%es,%k0" : "=a"(newregs->es));
 #endif
+		asm volatile("pushf\n\t"
+			     "pop %0" : "=m"(newregs->flags));
 		newregs->ip = _THIS_IP_;
 	}
 }
 
 #ifdef CONFIG_X86_32
-asmlinkage unsigned long
-relocate_kernel(unsigned long indirection_page,
-		unsigned long control_page,
-		unsigned long start_address,
-		unsigned int has_pae,
-		unsigned int preserve_context);
+typedef asmlinkage unsigned long
+relocate_kernel_fn(unsigned long indirection_page,
+		   unsigned long control_page,
+		   unsigned long start_address,
+		   unsigned int has_pae,
+		   unsigned int preserve_context);
 #else
-unsigned long
-relocate_kernel(unsigned long indirection_page,
-		unsigned long page_list,
-		unsigned long start_address,
-		unsigned int preserve_context,
-		unsigned int host_mem_enc_active);
+typedef unsigned long
+relocate_kernel_fn(unsigned long indirection_page,
+		   unsigned long pa_control_page,
+		   unsigned long start_address,
+		   unsigned int flags);
 #endif
-
+extern relocate_kernel_fn relocate_kernel;
 #define ARCH_HAS_KIMAGE_ARCH
 
 #ifdef CONFIG_X86_32
@@ -145,6 +147,19 @@ struct kimage_arch {
 };
 #else
 struct kimage_arch {
+	/*
+	 * This is a kimage control page, as it must not overlap with either
+	 * source or destination address ranges.
+	 */
+	pgd_t *pgd;
+	/*
+	 * The virtual mapping of the control code page itself is used only
+	 * during the transition, while the current kernel's pages are all
+	 * in place. Thus the intermediate page table pages used to map it
+	 * are not control pages, but instead just normal pages obtained
+	 * with get_zeroed_page(). And have to be tracked (below) so that
+	 * they can be freed.
+	 */
 	p4d_t *p4d;
 	pud_t *pud;
 	pmd_t *pmd;
@@ -217,6 +232,6 @@ unsigned int arch_crash_get_elfcorehdr_size(void);
 #define crash_get_elfcorehdr_size arch_crash_get_elfcorehdr_size
 #endif
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ASM_X86_KEXEC_H */
diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index 861d080ed4c6..de709fb5bd76 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -21,6 +21,7 @@ KVM_X86_OP(has_emulated_msr)
 KVM_X86_OP(vcpu_after_set_cpuid)
 KVM_X86_OP(vm_init)
 KVM_X86_OP_OPTIONAL(vm_destroy)
+KVM_X86_OP_OPTIONAL(vm_pre_destroy)
 KVM_X86_OP_OPTIONAL_RET0(vcpu_precreate)
 KVM_X86_OP(vcpu_create)
 KVM_X86_OP(vcpu_free)
@@ -34,6 +35,7 @@ KVM_X86_OP(set_msr)
 KVM_X86_OP(get_segment_base)
 KVM_X86_OP(get_segment)
 KVM_X86_OP(get_cpl)
+KVM_X86_OP(get_cpl_no_cache)
 KVM_X86_OP(set_segment)
 KVM_X86_OP(get_cs_db_l_bits)
 KVM_X86_OP(is_valid_cr0)
@@ -82,7 +84,6 @@ KVM_X86_OP(enable_nmi_window)
 KVM_X86_OP(enable_irq_window)
 KVM_X86_OP_OPTIONAL(update_cr8_intercept)
 KVM_X86_OP(refresh_apicv_exec_ctrl)
-KVM_X86_OP_OPTIONAL(hwapic_irr_update)
 KVM_X86_OP_OPTIONAL(hwapic_isr_update)
 KVM_X86_OP_OPTIONAL(load_eoi_exitmap)
 KVM_X86_OP_OPTIONAL(set_virtual_apic_mode)
@@ -93,22 +94,28 @@ KVM_X86_OP_OPTIONAL_RET0(set_tss_addr)
 KVM_X86_OP_OPTIONAL_RET0(set_identity_map_addr)
 KVM_X86_OP_OPTIONAL_RET0(get_mt_mask)
 KVM_X86_OP(load_mmu_pgd)
+KVM_X86_OP_OPTIONAL(link_external_spt)
+KVM_X86_OP_OPTIONAL(set_external_spte)
+KVM_X86_OP_OPTIONAL(free_external_spt)
+KVM_X86_OP_OPTIONAL(remove_external_spte)
 KVM_X86_OP(has_wbinvd_exit)
 KVM_X86_OP(get_l2_tsc_offset)
 KVM_X86_OP(get_l2_tsc_multiplier)
 KVM_X86_OP(write_tsc_offset)
 KVM_X86_OP(write_tsc_multiplier)
 KVM_X86_OP(get_exit_info)
+KVM_X86_OP(get_entry_info)
 KVM_X86_OP(check_intercept)
 KVM_X86_OP(handle_exit_irqoff)
 KVM_X86_OP_OPTIONAL(update_cpu_dirty_logging)
 KVM_X86_OP_OPTIONAL(vcpu_blocking)
 KVM_X86_OP_OPTIONAL(vcpu_unblocking)
 KVM_X86_OP_OPTIONAL(pi_update_irte)
-KVM_X86_OP_OPTIONAL(pi_start_assignment)
+KVM_X86_OP_OPTIONAL(pi_start_bypass)
 KVM_X86_OP_OPTIONAL(apicv_pre_state_restore)
 KVM_X86_OP_OPTIONAL(apicv_post_state_restore)
 KVM_X86_OP_OPTIONAL_RET0(dy_apicv_has_pending_interrupt)
+KVM_X86_OP_OPTIONAL(protected_apic_has_interrupt)
 KVM_X86_OP_OPTIONAL(set_hv_timer)
 KVM_X86_OP_OPTIONAL(cancel_hv_timer)
 KVM_X86_OP(setup_mce)
@@ -120,6 +127,8 @@ KVM_X86_OP(enable_smi_window)
 #endif
 KVM_X86_OP_OPTIONAL(dev_get_attr)
 KVM_X86_OP_OPTIONAL(mem_enc_ioctl)
+KVM_X86_OP_OPTIONAL(vcpu_mem_enc_ioctl)
+KVM_X86_OP_OPTIONAL(vcpu_mem_enc_unlocked_ioctl)
 KVM_X86_OP_OPTIONAL(mem_enc_register_region)
 KVM_X86_OP_OPTIONAL(mem_enc_unregister_region)
 KVM_X86_OP_OPTIONAL(vm_copy_enc_context_from)
@@ -130,14 +139,14 @@ KVM_X86_OP(check_emulate_instruction)
 KVM_X86_OP(apic_init_signal_blocked)
 KVM_X86_OP_OPTIONAL(enable_l2_tlb_flush)
 KVM_X86_OP_OPTIONAL(migrate_timers)
-KVM_X86_OP(msr_filter_changed)
+KVM_X86_OP(recalc_intercepts)
 KVM_X86_OP(complete_emulated_msr)
 KVM_X86_OP(vcpu_deliver_sipi_vector)
 KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons);
 KVM_X86_OP_OPTIONAL(get_untagged_addr)
 KVM_X86_OP_OPTIONAL(alloc_apic_backing_page)
 KVM_X86_OP_OPTIONAL_RET0(gmem_prepare)
-KVM_X86_OP_OPTIONAL_RET0(private_max_mapping_level)
+KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level)
 KVM_X86_OP_OPTIONAL(gmem_invalidate)
 
 #undef KVM_X86_OP
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 6d9f763a7bb9..5a3bfa293e8b 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -24,19 +24,24 @@
 #include <linux/pvclock_gtod.h>
 #include <linux/clocksource.h>
 #include <linux/irqbypass.h>
-#include <linux/hyperv.h>
 #include <linux/kfifo.h>
+#include <linux/sched/vhost_task.h>
+#include <linux/call_once.h>
+#include <linux/atomic.h>
 
 #include <asm/apic.h>
 #include <asm/pvclock-abi.h>
+#include <asm/debugreg.h>
 #include <asm/desc.h>
 #include <asm/mtrr.h>
 #include <asm/msr-index.h>
+#include <asm/msr.h>
 #include <asm/asm.h>
+#include <asm/irq_remapping.h>
 #include <asm/kvm_page_track.h>
 #include <asm/kvm_vcpu_regs.h>
-#include <asm/hyperv-tlfs.h>
 #include <asm/reboot.h>
+#include <hyperv/hvhdk.h>
 
 #define __KVM_HAVE_ARCH_VCPU_DEBUGFS
 
@@ -115,14 +120,15 @@
 #define KVM_REQ_TLB_FLUSH_GUEST \
 	KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
 #define KVM_REQ_APF_READY		KVM_ARCH_REQ(28)
-#define KVM_REQ_MSR_FILTER_CHANGED	KVM_ARCH_REQ(29)
+#define KVM_REQ_RECALC_INTERCEPTS	KVM_ARCH_REQ(29)
 #define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \
 	KVM_ARCH_REQ_FLAGS(30, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
 #define KVM_REQ_MMU_FREE_OBSOLETE_ROOTS \
 	KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
 #define KVM_REQ_HV_TLB_FLUSH \
 	KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
-#define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE	KVM_ARCH_REQ(34)
+#define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \
+	KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT)
 
 #define CR0_RESERVED_BITS                                               \
 	(~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
@@ -136,7 +142,7 @@
 			  | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \
 			  | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \
 			  | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP \
-			  | X86_CR4_LAM_SUP))
+			  | X86_CR4_LAM_SUP | X86_CR4_CET))
 
 #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
 
@@ -244,7 +250,6 @@ enum x86_intercept_stage;
 #define DR7_BP_EN_MASK	0x000000ff
 #define DR7_GE		(1 << 9)
 #define DR7_GD		(1 << 13)
-#define DR7_FIXED_1	0x00000400
 #define DR7_VOLATILE	0xffff2bff
 
 #define KVM_GUESTDBG_VALID_MASK \
@@ -262,6 +267,7 @@ enum x86_intercept_stage;
 #define PFERR_RSVD_MASK		BIT(3)
 #define PFERR_FETCH_MASK	BIT(4)
 #define PFERR_PK_MASK		BIT(5)
+#define PFERR_SS_MASK		BIT(6)
 #define PFERR_SGX_MASK		BIT(15)
 #define PFERR_GUEST_RMP_MASK	BIT_ULL(31)
 #define PFERR_GUEST_FINAL_MASK	BIT_ULL(32)
@@ -292,6 +298,7 @@ enum x86_intercept_stage;
  */
 #define KVM_APIC_PV_EOI_PENDING	1
 
+struct kvm_kernel_irqfd;
 struct kvm_kernel_irq_routing_entry;
 
 /*
@@ -312,10 +319,11 @@ struct kvm_kernel_irq_routing_entry;
  * the number of unique SPs that can theoretically be created is 2^n, where n
  * is the number of bits that are used to compute the role.
  *
- * But, even though there are 19 bits in the mask below, not all combinations
+ * But, even though there are 20 bits in the mask below, not all combinations
  * of modes and flags are possible:
  *
- *   - invalid shadow pages are not accounted, so the bits are effectively 18
+ *   - invalid shadow pages are not accounted, mirror pages are not shadowed,
+ *     so the bits are effectively 18.
  *
  *   - quadrant will only be used if has_4_byte_gpte=1 (non-PAE paging);
  *     execonly and ad_disabled are only used for nested EPT which has
@@ -348,7 +356,8 @@ union kvm_mmu_page_role {
 		unsigned ad_disabled:1;
 		unsigned guest_mode:1;
 		unsigned passthrough:1;
-		unsigned :5;
+		unsigned is_mirror:1;
+		unsigned :4;
 
 		/*
 		 * This is left at the top of the word so that
@@ -402,11 +411,10 @@ union kvm_cpu_role {
 };
 
 struct kvm_rmap_head {
-	unsigned long val;
+	atomic_long_t val;
 };
 
 struct kvm_pio_request {
-	unsigned long linear_rip;
 	unsigned long count;
 	int in;
 	int port;
@@ -456,6 +464,7 @@ struct kvm_mmu {
 	int (*sync_spte)(struct kvm_vcpu *vcpu,
 			 struct kvm_mmu_page *sp, int i);
 	struct kvm_mmu_root_info root;
+	hpa_t mirror_root_hpa;
 	union kvm_cpu_role cpu_role;
 	union kvm_mmu_page_role root_role;
 
@@ -537,10 +546,10 @@ struct kvm_pmc {
 #define KVM_MAX_NR_GP_COUNTERS		KVM_MAX(KVM_MAX_NR_INTEL_GP_COUNTERS, \
 						KVM_MAX_NR_AMD_GP_COUNTERS)
 
-#define KVM_MAX_NR_INTEL_FIXED_COUTNERS	3
-#define KVM_MAX_NR_AMD_FIXED_COUTNERS	0
-#define KVM_MAX_NR_FIXED_COUNTERS	KVM_MAX(KVM_MAX_NR_INTEL_FIXED_COUTNERS, \
-						KVM_MAX_NR_AMD_FIXED_COUTNERS)
+#define KVM_MAX_NR_INTEL_FIXED_COUNTERS	3
+#define KVM_MAX_NR_AMD_FIXED_COUNTERS	0
+#define KVM_MAX_NR_FIXED_COUNTERS	KVM_MAX(KVM_MAX_NR_INTEL_FIXED_COUNTERS, \
+						KVM_MAX_NR_AMD_FIXED_COUNTERS)
 
 struct kvm_pmu {
 	u8 version;
@@ -571,6 +580,9 @@ struct kvm_pmu {
 	DECLARE_BITMAP(all_valid_pmc_idx, X86_PMC_IDX_MAX);
 	DECLARE_BITMAP(pmc_in_use, X86_PMC_IDX_MAX);
 
+	DECLARE_BITMAP(pmc_counting_instructions, X86_PMC_IDX_MAX);
+	DECLARE_BITMAP(pmc_counting_branches, X86_PMC_IDX_MAX);
+
 	u64 ds_area;
 	u64 pebs_enable;
 	u64 pebs_enable_rsvd;
@@ -602,8 +614,15 @@ struct kvm_pmu {
 struct kvm_pmu_ops;
 
 enum {
-	KVM_DEBUGREG_BP_ENABLED = 1,
-	KVM_DEBUGREG_WONT_EXIT = 2,
+	KVM_DEBUGREG_BP_ENABLED		= BIT(0),
+	KVM_DEBUGREG_WONT_EXIT		= BIT(1),
+	/*
+	 * Guest debug registers (DR0-3, DR6 and DR7) are saved/restored by
+	 * hardware on exit from or enter to guest. KVM needn't switch them.
+	 * DR0-3, DR6 and DR7 are set to their architectural INIT value on VM
+	 * exit, host values need to be restored.
+	 */
+	KVM_DEBUGREG_AUTO_SWITCH	= BIT(2),
 };
 
 struct kvm_mtrr {
@@ -686,8 +705,13 @@ struct kvm_vcpu_hv {
 
 	struct kvm_vcpu_hv_tlb_flush_fifo tlb_flush_fifo[HV_NR_TLB_FLUSH_FIFOS];
 
-	/* Preallocated buffer for handling hypercalls passing sparse vCPU set */
+	/*
+	 * Preallocated buffers for handling hypercalls that pass sparse vCPU
+	 * sets (for high vCPU counts, they're too large to comfortably fit on
+	 * the stack).
+	 */
 	u64 sparse_banks[HV_MAX_SPARSE_VCPU_BANKS];
+	DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS);
 
 	struct hv_vp_assist_page vp_assist_page;
 
@@ -738,6 +762,25 @@ struct kvm_queued_exception {
 	bool has_payload;
 };
 
+/*
+ * Hardware-defined CPUID leafs that are either scattered by the kernel or are
+ * unknown to the kernel, but need to be directly used by KVM.  Note, these
+ * word values conflict with the kernel's "bug" caps, but KVM doesn't use those.
+ */
+enum kvm_only_cpuid_leafs {
+	CPUID_12_EAX	 = NCAPINTS,
+	CPUID_7_1_EDX,
+	CPUID_8000_0007_EDX,
+	CPUID_8000_0022_EAX,
+	CPUID_7_2_EDX,
+	CPUID_24_0_EBX,
+	CPUID_8000_0021_ECX,
+	CPUID_7_1_ECX,
+	NR_KVM_CPU_CAPS,
+
+	NKVMCAPINTS = NR_KVM_CPU_CAPS - NCAPINTS,
+};
+
 struct kvm_vcpu_arch {
 	/*
 	 * rip and regs accesses must go through
@@ -759,6 +802,7 @@ struct kvm_vcpu_arch {
 	u32 pkru;
 	u32 hflags;
 	u64 efer;
+	u64 host_debugctl;
 	u64 apic_base;
 	struct kvm_lapic *apic;    /* kernel irqchip context */
 	bool load_eoi_exitmap_pending;
@@ -772,7 +816,6 @@ struct kvm_vcpu_arch {
 	bool at_instruction_boundary;
 	bool tpr_access_reporting;
 	bool xfd_no_write_intercept;
-	u64 ia32_xss;
 	u64 microcode_version;
 	u64 arch_capabilities;
 	u64 perf_capabilities;
@@ -812,6 +855,11 @@ struct kvm_vcpu_arch {
 	struct kvm_mmu_memory_cache mmu_shadow_page_cache;
 	struct kvm_mmu_memory_cache mmu_shadowed_info_cache;
 	struct kvm_mmu_memory_cache mmu_page_header_cache;
+	/*
+	 * This cache is to allocate external page table. E.g. private EPT used
+	 * by the TDX module.
+	 */
+	struct kvm_mmu_memory_cache mmu_external_spt_cache;
 
 	/*
 	 * QEMU userspace and the guest each have their own FPU state.
@@ -828,6 +876,8 @@ struct kvm_vcpu_arch {
 
 	u64 xcr0;
 	u64 guest_supported_xcr0;
+	u64 ia32_xss;
+	u64 guest_supported_xss;
 
 	struct kvm_pio_request pio;
 	void *pio_data;
@@ -853,27 +903,24 @@ struct kvm_vcpu_arch {
 
 	int cpuid_nent;
 	struct kvm_cpuid_entry2 *cpuid_entries;
-	struct kvm_hypervisor_cpuid kvm_cpuid;
+	bool cpuid_dynamic_bits_dirty;
 	bool is_amd_compatible;
 
 	/*
-	 * FIXME: Drop this macro and use KVM_NR_GOVERNED_FEATURES directly
-	 * when "struct kvm_vcpu_arch" is no longer defined in an
-	 * arch/x86/include/asm header.  The max is mostly arbitrary, i.e.
-	 * can be increased as necessary.
-	 */
-#define KVM_MAX_NR_GOVERNED_FEATURES BITS_PER_LONG
-
-	/*
-	 * Track whether or not the guest is allowed to use features that are
-	 * governed by KVM, where "governed" means KVM needs to manage state
-	 * and/or explicitly enable the feature in hardware.  Typically, but
-	 * not always, governed features can be used by the guest if and only
-	 * if both KVM and userspace want to expose the feature to the guest.
+	 * cpu_caps holds the effective guest capabilities, i.e. the features
+	 * the vCPU is allowed to use.  Typically, but not always, features can
+	 * be used by the guest if and only if both KVM and userspace want to
+	 * expose the feature to the guest.
+	 *
+	 * A common exception is for virtualization holes, i.e. when KVM can't
+	 * prevent the guest from using a feature, in which case the vCPU "has"
+	 * the feature regardless of what KVM or userspace desires.
+	 *
+	 * Note, features that don't require KVM involvement in any way are
+	 * NOT enforced/sanitized by KVM, i.e. are taken verbatim from the
+	 * guest CPUID provided by userspace.
 	 */
-	struct {
-		DECLARE_BITMAP(enabled, KVM_MAX_NR_GOVERNED_FEATURES);
-	} governed_features;
+	u32 cpu_caps[NR_KVM_CPU_CAPS];
 
 	u64 reserved_gpa_bits;
 	int maxphyaddr;
@@ -884,9 +931,12 @@ struct kvm_vcpu_arch {
 	bool emulate_regs_need_sync_to_vcpu;
 	bool emulate_regs_need_sync_from_vcpu;
 	int (*complete_userspace_io)(struct kvm_vcpu *vcpu);
+	unsigned long cui_linear_rip;
+	int cui_rdmsr_imm_reg;
 
 	gpa_t time;
-	struct pvclock_vcpu_time_info hv_clock;
+	s8  pvclock_tsc_shift;
+	u32 pvclock_tsc_mul;
 	unsigned int hw_tsc_khz;
 	struct gfn_to_pfn_cache pv_time;
 	/* set guest stopped flag in pvclock flags field */
@@ -974,8 +1024,8 @@ struct kvm_vcpu_arch {
 		u64 msr_int_val; /* MSR_KVM_ASYNC_PF_INT */
 		u16 vec;
 		u32 id;
-		bool send_user_only;
 		u32 host_apf_flags;
+		bool send_always;
 		bool delivery_as_pf_vmexit;
 		bool pageready_pending;
 	} apf;
@@ -1000,13 +1050,11 @@ struct kvm_vcpu_arch {
 
 	int pending_ioapic_eoi;
 	int pending_external_vector;
+	int highest_stale_pending_ioapic_eoi;
 
 	/* be preempted when it's in kernel-mode(cpl=0) */
 	bool preempted_in_kernel;
 
-	/* Flush the L1 Data cache for L1TF mitigation on VMENTER */
-	bool l1tf_flush_l1d;
-
 	/* Host CPU on which VM-entry was most recently attempted */
 	int last_vmentry_cpu;
 
@@ -1030,6 +1078,7 @@ struct kvm_vcpu_arch {
 
 	/* Protected Guests */
 	bool guest_state_protected;
+	bool guest_tsc_protected;
 
 	/*
 	 * Set when PDPTS were loaded directly by the userspace without
@@ -1166,6 +1215,8 @@ struct kvm_xen {
 	struct gfn_to_pfn_cache shinfo_cache;
 	struct idr evtchn_ports;
 	unsigned long poll_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)];
+
+	struct kvm_xen_hvm_config hvm_config;
 };
 #endif
 
@@ -1274,6 +1325,12 @@ enum kvm_apicv_inhibit {
 	 */
 	APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED,
 
+	/*
+	 * AVIC is disabled because the vCPU's APIC ID is beyond the max
+	 * supported by AVIC/x2AVIC, i.e. the vCPU is unaddressable.
+	 */
+	APICV_INHIBIT_REASON_PHYSICAL_ID_TOO_BIG,
+
 	NR_APICV_INHIBIT_REASONS,
 };
 
@@ -1292,21 +1349,10 @@ enum kvm_apicv_inhibit {
 	__APICV_INHIBIT_REASON(IRQWIN),			\
 	__APICV_INHIBIT_REASON(PIT_REINJ),		\
 	__APICV_INHIBIT_REASON(SEV),			\
-	__APICV_INHIBIT_REASON(LOGICAL_ID_ALIASED)
+	__APICV_INHIBIT_REASON(LOGICAL_ID_ALIASED),	\
+	__APICV_INHIBIT_REASON(PHYSICAL_ID_TOO_BIG)
 
-struct kvm_arch {
-	unsigned long n_used_mmu_pages;
-	unsigned long n_requested_mmu_pages;
-	unsigned long n_max_mmu_pages;
-	unsigned int indirect_shadow_pages;
-	u8 mmu_valid_gen;
-	u8 vm_type;
-	bool has_private_mem;
-	bool has_protected_state;
-	bool pre_fault_allowed;
-	struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES];
-	struct list_head active_mmu_pages;
-	struct list_head zapped_obsolete_pages;
+struct kvm_possible_nx_huge_pages {
 	/*
 	 * A list of kvm_mmu_page structs that, if zapped, could possibly be
 	 * replaced by an NX huge page.  A shadow page is on this list if its
@@ -1318,7 +1364,32 @@ struct kvm_arch {
 	 * guest attempts to execute from the region then KVM obviously can't
 	 * create an NX huge page (without hanging the guest).
 	 */
-	struct list_head possible_nx_huge_pages;
+	struct list_head pages;
+	u64 nr_pages;
+};
+
+enum kvm_mmu_type {
+	KVM_SHADOW_MMU,
+#ifdef CONFIG_X86_64
+	KVM_TDP_MMU,
+#endif
+	KVM_NR_MMU_TYPES,
+};
+
+struct kvm_arch {
+	unsigned long n_used_mmu_pages;
+	unsigned long n_requested_mmu_pages;
+	unsigned long n_max_mmu_pages;
+	unsigned int indirect_shadow_pages;
+	u8 mmu_valid_gen;
+	u8 vm_type;
+	bool has_private_mem;
+	bool has_protected_state;
+	bool has_protected_eoi;
+	bool pre_fault_allowed;
+	struct hlist_head *mmu_page_hash;
+	struct list_head active_mmu_pages;
+	struct kvm_possible_nx_huge_pages possible_nx_huge_pages[KVM_NR_MMU_TYPES];
 #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING
 	struct kvm_page_track_notifier_head track_notifier_head;
 #endif
@@ -1332,15 +1403,15 @@ struct kvm_arch {
 
 	u64 shadow_mmio_value;
 
-	struct iommu_domain *iommu_domain;
-	bool iommu_noncoherent;
 #define __KVM_HAVE_ARCH_NONCOHERENT_DMA
 	atomic_t noncoherent_dma_count;
-#define __KVM_HAVE_ARCH_ASSIGNED_DEVICE
-	atomic_t assigned_device_count;
+	unsigned long nr_possible_bypass_irqs;
+
+#ifdef CONFIG_KVM_IOAPIC
 	struct kvm_pic *vpic;
 	struct kvm_ioapic *vioapic;
 	struct kvm_pit *vpit;
+#endif
 	atomic_t vapics_in_nmi_mode;
 	struct mutex apic_map_lock;
 	struct kvm_apic_map __rcu *apic_map;
@@ -1355,12 +1426,8 @@ struct kvm_arch {
 
 	gpa_t wall_clock;
 
-	bool mwait_in_guest;
-	bool hlt_in_guest;
-	bool pause_in_guest;
-	bool cstate_in_guest;
+	u64 disabled_exits;
 
-	unsigned long irq_sources_bitmap;
 	s64 kvmclock_offset;
 
 	/*
@@ -1386,13 +1453,6 @@ struct kvm_arch {
 	bool use_master_clock;
 	u64 master_kernel_ns;
 	u64 master_cycle_now;
-	struct delayed_work kvmclock_update_work;
-	struct delayed_work kvmclock_sync_work;
-
-	struct kvm_xen_hvm_config xen_hvm_config;
-
-	/* reads protected by irq_srcu, writes by irq_lock */
-	struct hlist_head mask_notifier_list;
 
 #ifdef CONFIG_KVM_HYPERV
 	struct kvm_hv hyperv;
@@ -1416,6 +1476,7 @@ struct kvm_arch {
 	bool x2apic_format;
 	bool x2apic_broadcast_quirk_disabled;
 
+	bool has_mapped_host_mmio;
 	bool guest_can_read_msr_platform_info;
 	bool exception_payload_enabled;
 
@@ -1443,11 +1504,18 @@ struct kvm_arch {
 	bool sgx_provisioning_allowed;
 
 	struct kvm_x86_pmu_event_filter __rcu *pmu_event_filter;
-	struct task_struct *nx_huge_page_recovery_thread;
+	struct vhost_task *nx_huge_page_recovery_thread;
+	u64 nx_huge_page_last;
+	struct once nx_once;
 
 #ifdef CONFIG_X86_64
-	/* The number of TDP MMU pages across all roots. */
+#ifdef CONFIG_KVM_PROVE_MMU
+	/*
+	 * The number of TDP MMU pages across all roots.  Used only to sanity
+	 * check that KVM isn't leaking TDP MMU pages.
+	 */
 	atomic64_t tdp_mmu_pages;
+#endif
 
 	/*
 	 * List of struct kvm_mmu_pages being used as roots.
@@ -1455,6 +1523,7 @@ struct kvm_arch {
 	 * tdp_mmu_page set.
 	 *
 	 * For reads, this list is protected by:
+	 *	RCU alone or
 	 *	the MMU lock in read mode + RCU or
 	 *	the MMU lock in write mode
 	 *
@@ -1473,7 +1542,7 @@ struct kvm_arch {
 	 * is held in read mode:
 	 *  - tdp_mmu_roots (above)
 	 *  - the link field of kvm_mmu_page structs used by the TDP MMU
-	 *  - possible_nx_huge_pages;
+	 *  - possible_nx_huge_pages[KVM_TDP_MMU];
 	 *  - the possible_nx_huge_page_link field of kvm_mmu_page structs used
 	 *    by the TDP MMU
 	 * Because the lock is only taken within the MMU lock, strictly
@@ -1535,6 +1604,15 @@ struct kvm_arch {
 	 */
 #define SPLIT_DESC_CACHE_MIN_NR_OBJECTS (SPTE_ENT_PER_PAGE + 1)
 	struct kvm_mmu_memory_cache split_desc_cache;
+
+	gfn_t gfn_direct_bits;
+
+	/*
+	 * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A Zero
+	 * value indicates CPU dirty logging is unsupported or disabled in
+	 * current VM.
+	 */
+	int cpu_dirty_log_size;
 };
 
 struct kvm_vm_stat {
@@ -1622,6 +1700,12 @@ static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical)
 	return dest_mode_logical ? APIC_DEST_LOGICAL : APIC_DEST_PHYSICAL;
 }
 
+enum kvm_x86_run_flags {
+	KVM_RUN_FORCE_IMMEDIATE_EXIT	= BIT(0),
+	KVM_RUN_LOAD_GUEST_DR6		= BIT(1),
+	KVM_RUN_LOAD_DEBUGCTL		= BIT(2),
+};
+
 struct kvm_x86_ops {
 	const char *name;
 
@@ -1638,6 +1722,7 @@ struct kvm_x86_ops {
 	unsigned int vm_size;
 	int (*vm_init)(struct kvm *kvm);
 	void (*vm_destroy)(struct kvm *kvm);
+	void (*vm_pre_destroy)(struct kvm *kvm);
 
 	/* Create, but do not attach this VCPU */
 	int (*vcpu_precreate)(struct kvm *kvm);
@@ -1649,6 +1734,12 @@ struct kvm_x86_ops {
 	void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu);
 	void (*vcpu_put)(struct kvm_vcpu *vcpu);
 
+	/*
+	 * Mask of DEBUGCTL bits that are owned by the host, i.e. that need to
+	 * match the host's value even while the guest is active.
+	 */
+	const u64 HOST_OWNED_DEBUGCTL;
+
 	void (*update_exception_bitmap)(struct kvm_vcpu *vcpu);
 	int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
 	int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
@@ -1656,6 +1747,7 @@ struct kvm_x86_ops {
 	void (*get_segment)(struct kvm_vcpu *vcpu,
 			    struct kvm_segment *var, int seg);
 	int (*get_cpl)(struct kvm_vcpu *vcpu);
+	int (*get_cpl_no_cache)(struct kvm_vcpu *vcpu);
 	void (*set_segment)(struct kvm_vcpu *vcpu,
 			    struct kvm_segment *var, int seg);
 	void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l);
@@ -1700,7 +1792,7 @@ struct kvm_x86_ops {
 
 	int (*vcpu_pre_run)(struct kvm_vcpu *vcpu);
 	enum exit_fastpath_completion (*vcpu_run)(struct kvm_vcpu *vcpu,
-						  bool force_immediate_exit);
+						  u64 run_flags);
 	int (*handle_exit)(struct kvm_vcpu *vcpu,
 		enum exit_fastpath_completion exit_fastpath);
 	int (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
@@ -1732,8 +1824,7 @@ struct kvm_x86_ops {
 	const unsigned long required_apicv_inhibits;
 	bool allow_apicv_in_x2apic_without_x2apic_virtualization;
 	void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu);
-	void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr);
-	void (*hwapic_isr_update)(int isr);
+	void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr);
 	void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
 	void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu);
 	void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu);
@@ -1747,6 +1838,21 @@ struct kvm_x86_ops {
 	void (*load_mmu_pgd)(struct kvm_vcpu *vcpu, hpa_t root_hpa,
 			     int root_level);
 
+	/* Update external mapping with page table link. */
+	int (*link_external_spt)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
+				void *external_spt);
+	/* Update the external page table from spte getting set. */
+	int (*set_external_spte)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
+				 u64 mirror_spte);
+
+	/* Update external page tables for page table about to be freed. */
+	int (*free_external_spt)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
+				 void *external_spt);
+
+	/* Update external page table from spte getting removed, and flush TLB. */
+	void (*remove_external_spte)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
+				     u64 mirror_spte);
+
 	bool (*has_wbinvd_exit)(void);
 
 	u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu);
@@ -1755,12 +1861,15 @@ struct kvm_x86_ops {
 	void (*write_tsc_multiplier)(struct kvm_vcpu *vcpu);
 
 	/*
-	 * Retrieve somewhat arbitrary exit information.  Intended to
+	 * Retrieve somewhat arbitrary exit/entry information.  Intended to
 	 * be used only from within tracepoints or error paths.
 	 */
 	void (*get_exit_info)(struct kvm_vcpu *vcpu, u32 *reason,
 			      u64 *info1, u64 *info2,
-			      u32 *exit_int_info, u32 *exit_int_info_err_code);
+			      u32 *intr_info, u32 *error_code);
+
+	void (*get_entry_info)(struct kvm_vcpu *vcpu,
+			       u32 *intr_info, u32 *error_code);
 
 	int (*check_intercept)(struct kvm_vcpu *vcpu,
 			       struct x86_instruction_info *info,
@@ -1768,11 +1877,6 @@ struct kvm_x86_ops {
 			       struct x86_exception *exception);
 	void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu);
 
-	/*
-	 * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer.  A zero
-	 * value indicates CPU dirty logging is unsupported or disabled.
-	 */
-	int cpu_dirty_log_size;
 	void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu);
 
 	const struct kvm_x86_nested_ops *nested_ops;
@@ -1780,12 +1884,14 @@ struct kvm_x86_ops {
 	void (*vcpu_blocking)(struct kvm_vcpu *vcpu);
 	void (*vcpu_unblocking)(struct kvm_vcpu *vcpu);
 
-	int (*pi_update_irte)(struct kvm *kvm, unsigned int host_irq,
-			      uint32_t guest_irq, bool set);
-	void (*pi_start_assignment)(struct kvm *kvm);
+	int (*pi_update_irte)(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
+			      unsigned int host_irq, uint32_t guest_irq,
+			      struct kvm_vcpu *vcpu, u32 vector);
+	void (*pi_start_bypass)(struct kvm *kvm);
 	void (*apicv_pre_state_restore)(struct kvm_vcpu *vcpu);
 	void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu);
 	bool (*dy_apicv_has_pending_interrupt)(struct kvm_vcpu *vcpu);
+	bool (*protected_apic_has_interrupt)(struct kvm_vcpu *vcpu);
 
 	int (*set_hv_timer)(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
 			    bool *expired);
@@ -1802,6 +1908,8 @@ struct kvm_x86_ops {
 
 	int (*dev_get_attr)(u32 group, u64 attr, u64 *val);
 	int (*mem_enc_ioctl)(struct kvm *kvm, void __user *argp);
+	int (*vcpu_mem_enc_ioctl)(struct kvm_vcpu *vcpu, void __user *argp);
+	int (*vcpu_mem_enc_unlocked_ioctl)(struct kvm_vcpu *vcpu, void __user *argp);
 	int (*mem_enc_register_region)(struct kvm *kvm, struct kvm_enc_region *argp);
 	int (*mem_enc_unregister_region)(struct kvm *kvm, struct kvm_enc_region *argp);
 	int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
@@ -1817,7 +1925,7 @@ struct kvm_x86_ops {
 	int (*enable_l2_tlb_flush)(struct kvm_vcpu *vcpu);
 
 	void (*migrate_timers)(struct kvm_vcpu *vcpu);
-	void (*msr_filter_changed)(struct kvm_vcpu *vcpu);
+	void (*recalc_intercepts)(struct kvm_vcpu *vcpu);
 	int (*complete_emulated_msr)(struct kvm_vcpu *vcpu, int err);
 
 	void (*vcpu_deliver_sipi_vector)(struct kvm_vcpu *vcpu, u8 vector);
@@ -1831,7 +1939,7 @@ struct kvm_x86_ops {
 	void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu);
 	int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order);
 	void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end);
-	int (*private_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn);
+	int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private);
 };
 
 struct kvm_x86_nested_ops {
@@ -1875,6 +1983,8 @@ struct kvm_arch_async_pf {
 extern u32 __read_mostly kvm_nr_uret_msrs;
 extern bool __read_mostly allow_smaller_maxphyaddr;
 extern bool __read_mostly enable_apicv;
+extern bool __read_mostly enable_ipiv;
+extern bool __read_mostly enable_device_posted_irqs;
 extern struct kvm_x86_ops kvm_x86_ops;
 
 #define kvm_x86_call(func) static_call(kvm_x86_##func)
@@ -1892,7 +2002,7 @@ void kvm_x86_vendor_exit(void);
 #define __KVM_HAVE_ARCH_VM_ALLOC
 static inline struct kvm *kvm_arch_alloc_vm(void)
 {
-	return __vmalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+	return kvzalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT);
 }
 
 #define __KVM_HAVE_ARCH_VM_FREE
@@ -1937,7 +2047,7 @@ void kvm_mmu_vendor_module_exit(void);
 
 void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
 int kvm_mmu_create(struct kvm_vcpu *vcpu);
-void kvm_mmu_init_vm(struct kvm *kvm);
+int kvm_mmu_init_vm(struct kvm *kvm);
 void kvm_mmu_uninit_vm(struct kvm *kvm);
 
 void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
@@ -1955,8 +2065,8 @@ void kvm_mmu_try_split_huge_pages(struct kvm *kvm,
 				  const struct kvm_memory_slot *memslot,
 				  u64 start, u64 end,
 				  int target_level);
-void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
-				   const struct kvm_memory_slot *memslot);
+void kvm_mmu_recover_huge_pages(struct kvm *kvm,
+				const struct kvm_memory_slot *memslot);
 void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
 				   const struct kvm_memory_slot *memslot);
 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen);
@@ -1968,19 +2078,6 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3);
 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
 			  const void *val, int bytes);
 
-struct kvm_irq_mask_notifier {
-	void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked);
-	int irq;
-	struct hlist_node link;
-};
-
-void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
-				    struct kvm_irq_mask_notifier *kimn);
-void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
-				      struct kvm_irq_mask_notifier *kimn);
-void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
-			     bool mask);
-
 extern bool tdp_enabled;
 
 u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
@@ -2017,8 +2114,8 @@ u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
  *			VMware backdoor emulation handles select instructions
  *			and reinjects the #GP for all other cases.
  *
- * EMULTYPE_PF - Set when emulating MMIO by way of an intercepted #PF, in which
- *		 case the CR2/GPA value pass on the stack is valid.
+ * EMULTYPE_PF - Set when an intercepted #PF triggers the emulation, in which case
+ *		 the CR2/GPA value pass on the stack is valid.
  *
  * EMULTYPE_COMPLETE_USER_EXIT - Set when the emulator should update interruptibility
  *				 state and inject single-step #DBs after skipping
@@ -2042,6 +2139,11 @@ u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
  *			     the gfn, i.e. retrying the instruction will hit a
  *			     !PRESENT fault, which results in a new shadow page
  *			     and sends KVM back to square one.
+ *
+ * EMULTYPE_SKIP_SOFT_INT - Set in combination with EMULTYPE_SKIP to only skip
+ *                          an instruction if it could generate a given software
+ *                          interrupt, which must be encoded via
+ *                          EMULTYPE_SET_SOFT_INT_VECTOR().
  */
 #define EMULTYPE_NO_DECODE	    (1 << 0)
 #define EMULTYPE_TRAP_UD	    (1 << 1)
@@ -2052,6 +2154,15 @@ u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
 #define EMULTYPE_PF		    (1 << 6)
 #define EMULTYPE_COMPLETE_USER_EXIT (1 << 7)
 #define EMULTYPE_WRITE_PF_TO_SP	    (1 << 8)
+#define EMULTYPE_SKIP_SOFT_INT	    (1 << 9)
+
+#define EMULTYPE_SET_SOFT_INT_VECTOR(v)	((u32)((v) & 0xff) << 16)
+#define EMULTYPE_GET_SOFT_INT_VECTOR(e)	(((e) >> 16) & 0xff)
+
+static inline bool kvm_can_emulate_event_vectoring(int emul_type)
+{
+	return !(emul_type & EMULTYPE_PF);
+}
 
 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type);
 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
@@ -2060,15 +2171,21 @@ void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu,
 					  u64 *data, u8 ndata);
 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu);
 
+void kvm_prepare_event_vectoring_exit(struct kvm_vcpu *vcpu, gpa_t gpa);
+void kvm_prepare_unexpected_reason_exit(struct kvm_vcpu *vcpu, u64 exit_reason);
+
 void kvm_enable_efer_bits(u64);
 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer);
-int kvm_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 *data);
-int kvm_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 data);
-int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, bool host_initiated);
-int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data);
-int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data);
+int kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data);
+int kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data);
+int __kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data);
+int __kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data);
+int kvm_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data);
+int kvm_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data);
 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu);
+int kvm_emulate_rdmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg);
 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu);
+int kvm_emulate_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg);
 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu);
 int kvm_emulate_invd(struct kvm_vcpu *vcpu);
 int kvm_emulate_mwait(struct kvm_vcpu *vcpu);
@@ -2100,6 +2217,7 @@ int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
 unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr);
 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw);
+int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr);
 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu);
 
 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
@@ -2112,8 +2230,8 @@ int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu);
 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long payload);
-void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr);
-void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
+void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned int nr,
+			   bool has_error_code, u32 error_code);
 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
 void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
 				    struct x86_exception *fault);
@@ -2132,9 +2250,6 @@ static inline int __kvm_irq_line_state(unsigned long *irq_state,
 	return !!(*irq_state);
 }
 
-int kvm_pic_set_irq(struct kvm_pic *pic, int irq, int irq_source_id, int level);
-void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id);
-
 void kvm_inject_nmi(struct kvm_vcpu *vcpu);
 int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu);
 
@@ -2179,12 +2294,6 @@ static inline void kvm_clear_apicv_inhibit(struct kvm *kvm,
 	kvm_set_or_clear_apicv_inhibit(kvm, reason, false);
 }
 
-unsigned long __kvm_emulate_hypercall(struct kvm_vcpu *vcpu, unsigned long nr,
-				      unsigned long a0, unsigned long a1,
-				      unsigned long a2, unsigned long a3,
-				      int op_64_bit, int cpl);
-int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
-
 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
 		       void *insn, int insn_len);
 void kvm_mmu_print_sptes(struct kvm_vcpu *vcpu, gpa_t gpa, const char *msg);
@@ -2198,10 +2307,8 @@ void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
 		       int tdp_max_root_level, int tdp_huge_page_level);
 
 
-#ifdef CONFIG_KVM_PRIVATE_MEM
+#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
 #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem)
-#else
-#define kvm_arch_has_private_mem(kvm) false
 #endif
 
 #define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state)
@@ -2223,7 +2330,7 @@ static inline unsigned long read_msr(unsigned long msr)
 {
 	u64 value;
 
-	rdmsrl(msr, value);
+	rdmsrq(msr, value);
 	return value;
 }
 #endif
@@ -2277,6 +2384,7 @@ int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
 int kvm_add_user_return_msr(u32 msr);
 int kvm_find_user_return_msr(u32 msr);
 int kvm_set_user_return_msr(unsigned index, u64 val, u64 mask);
+u64 kvm_get_user_return_msr(unsigned int slot);
 
 static inline bool kvm_is_supported_user_return_msr(u32 msr)
 {
@@ -2313,12 +2421,6 @@ void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu);
 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu);
 
-bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq,
-			     struct kvm_vcpu **dest_vcpu);
-
-void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
-		     struct kvm_lapic_irq *irq);
-
 static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)
 {
 	/* We can only post Fixed and LowPrio IRQs */
@@ -2359,7 +2461,13 @@ int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages);
 	 KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT |	\
 	 KVM_X86_QUIRK_FIX_HYPERCALL_INSN |	\
 	 KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS |	\
-	 KVM_X86_QUIRK_SLOT_ZAP_ALL)
+	 KVM_X86_QUIRK_SLOT_ZAP_ALL |		\
+	 KVM_X86_QUIRK_STUFF_FEATURE_MSRS |	\
+	 KVM_X86_QUIRK_IGNORE_GUEST_PAT)
+
+#define KVM_X86_CONDITIONAL_QUIRKS		\
+	(KVM_X86_QUIRK_CD_NW_CLEARED |		\
+	 KVM_X86_QUIRK_IGNORE_GUEST_PAT)
 
 /*
  * KVM previously used a u32 field in kvm_run to indicate the hypercall was
@@ -2368,4 +2476,9 @@ int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages);
  */
 #define KVM_EXIT_HYPERCALL_MBZ		GENMASK_ULL(31, 1)
 
+static inline bool kvm_arch_has_irq_bypass(void)
+{
+	return enable_device_posted_irqs;
+}
+
 #endif /* _ASM_X86_KVM_HOST_H */
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index 57bc74e112f2..4a47c16e2df8 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -124,7 +124,6 @@ bool kvm_para_available(void);
 unsigned int kvm_arch_para_features(void);
 unsigned int kvm_arch_para_hints(void);
 void kvm_async_pf_task_wait_schedule(u32 token);
-void kvm_async_pf_task_wake(u32 token);
 u32 kvm_read_and_reset_apf_flags(void);
 bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token);
 
@@ -148,7 +147,6 @@ static inline void kvm_spinlock_init(void)
 
 #else /* CONFIG_KVM_GUEST */
 #define kvm_async_pf_task_wait_schedule(T) do {} while(0)
-#define kvm_async_pf_task_wake(T) do {} while(0)
 
 static inline bool kvm_para_available(void)
 {
diff --git a/arch/x86/include/asm/kvm_types.h b/arch/x86/include/asm/kvm_types.h
index 08f1b57d3b62..d7c704ed1be9 100644
--- a/arch/x86/include/asm/kvm_types.h
+++ b/arch/x86/include/asm/kvm_types.h
@@ -2,6 +2,21 @@
 #ifndef _ASM_X86_KVM_TYPES_H
 #define _ASM_X86_KVM_TYPES_H
 
+#if IS_MODULE(CONFIG_KVM_AMD) && IS_MODULE(CONFIG_KVM_INTEL)
+#define KVM_SUB_MODULES kvm-amd,kvm-intel
+#elif IS_MODULE(CONFIG_KVM_AMD)
+#define KVM_SUB_MODULES kvm-amd
+#elif IS_MODULE(CONFIG_KVM_INTEL)
+#define KVM_SUB_MODULES kvm-intel
+#else
+#undef KVM_SUB_MODULES
+/*
+ * Don't export symbols for KVM without vendor modules, as kvm.ko is built iff
+ * at least one vendor module is enabled.
+ */
+#define EXPORT_SYMBOL_FOR_KVM(symbol)
+#endif
+
 #define KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE 40
 
 #endif /* _ASM_X86_KVM_TYPES_H */
diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h
index dc31b13b87a0..9d38ae744a2e 100644
--- a/arch/x86/include/asm/linkage.h
+++ b/arch/x86/include/asm/linkage.h
@@ -38,7 +38,7 @@
 #define ASM_FUNC_ALIGN		__stringify(__FUNC_ALIGN)
 #define SYM_F_ALIGN		__FUNC_ALIGN
 
-#ifdef __ASSEMBLY__
+#ifdef __ASSEMBLER__
 
 #if defined(CONFIG_MITIGATION_RETHUNK) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO)
 #define RET	jmp __x86_return_thunk
@@ -50,7 +50,7 @@
 #endif
 #endif /* CONFIG_MITIGATION_RETPOLINE */
 
-#else /* __ASSEMBLY__ */
+#else /* __ASSEMBLER__ */
 
 #if defined(CONFIG_MITIGATION_RETHUNK) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO)
 #define ASM_RET	"jmp __x86_return_thunk\n\t"
@@ -62,7 +62,7 @@
 #endif
 #endif /* CONFIG_MITIGATION_RETPOLINE */
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 /*
  * Depending on -fpatchable-function-entry=N,N usage (CONFIG_CALL_PADDING) the
@@ -119,33 +119,37 @@
 
 /* SYM_FUNC_START -- use for global functions */
 #define SYM_FUNC_START(name)				\
-	SYM_START(name, SYM_L_GLOBAL, SYM_F_ALIGN)	\
-	ENDBR
+	SYM_START(name, SYM_L_GLOBAL, SYM_F_ALIGN)
 
 /* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */
 #define SYM_FUNC_START_NOALIGN(name)			\
-	SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE)	\
-	ENDBR
+	SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE)
 
 /* SYM_FUNC_START_LOCAL -- use for local functions */
 #define SYM_FUNC_START_LOCAL(name)			\
-	SYM_START(name, SYM_L_LOCAL, SYM_F_ALIGN)	\
-	ENDBR
+	SYM_START(name, SYM_L_LOCAL, SYM_F_ALIGN)
 
 /* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */
 #define SYM_FUNC_START_LOCAL_NOALIGN(name)		\
-	SYM_START(name, SYM_L_LOCAL, SYM_A_NONE)	\
-	ENDBR
+	SYM_START(name, SYM_L_LOCAL, SYM_A_NONE)
 
 /* SYM_FUNC_START_WEAK -- use for weak functions */
 #define SYM_FUNC_START_WEAK(name)			\
-	SYM_START(name, SYM_L_WEAK, SYM_F_ALIGN)	\
-	ENDBR
+	SYM_START(name, SYM_L_WEAK, SYM_F_ALIGN)
 
 /* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */
 #define SYM_FUNC_START_WEAK_NOALIGN(name)		\
-	SYM_START(name, SYM_L_WEAK, SYM_A_NONE)		\
-	ENDBR
+	SYM_START(name, SYM_L_WEAK, SYM_A_NONE)
+
+/*
+ * Expose 'sym' to the startup code in arch/x86/boot/startup/, by emitting an
+ * alias prefixed with __pi_
+ */
+#ifdef __ASSEMBLER__
+#define SYM_PIC_ALIAS(sym)	SYM_ALIAS(__pi_ ## sym, sym, SYM_L_GLOBAL)
+#else
+#define SYM_PIC_ALIAS(sym)	extern typeof(sym) __PASTE(__pi_, sym) __alias(sym)
+#endif
 
 #endif /* _ASM_X86_LINKAGE_H */
 
diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index 3b9970117a0f..2d98886de09a 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -48,6 +48,7 @@
 
 /* AMD-specific bits */
 #define MCI_STATUS_TCC		BIT_ULL(55)  /* Task context corrupt */
+#define MCI_STATUS_PADDRV	BIT_ULL(54)  /* Valid System Physical Address */
 #define MCI_STATUS_SYNDV	BIT_ULL(53)  /* synd reg. valid */
 #define MCI_STATUS_DEFERRED	BIT_ULL(44)  /* uncorrected error, deferred exception */
 #define MCI_STATUS_POISON	BIT_ULL(43)  /* access poisonous data */
@@ -61,6 +62,8 @@
  *  - TCC bit is present in MCx_STATUS.
  */
 #define MCI_CONFIG_MCAX		0x1
+#define MCI_CONFIG_FRUTEXT	BIT_ULL(9)
+#define MCI_CONFIG_PADDRV	BIT_ULL(11)
 #define MCI_IPID_MCATYPE	0xFFFF0000
 #define MCI_IPID_HWID		0xFFF
 
@@ -122,6 +125,9 @@
 #define MSR_AMD64_SMCA_MC0_DESTAT	0xc0002008
 #define MSR_AMD64_SMCA_MC0_DEADDR	0xc0002009
 #define MSR_AMD64_SMCA_MC0_MISC1	0xc000200a
+/* Registers MISC2 to MISC4 are at offsets B to D. */
+#define MSR_AMD64_SMCA_MC0_SYND1	0xc000200e
+#define MSR_AMD64_SMCA_MC0_SYND2	0xc000200f
 #define MSR_AMD64_SMCA_MCx_CTL(x)	(MSR_AMD64_SMCA_MC0_CTL + 0x10*(x))
 #define MSR_AMD64_SMCA_MCx_STATUS(x)	(MSR_AMD64_SMCA_MC0_STATUS + 0x10*(x))
 #define MSR_AMD64_SMCA_MCx_ADDR(x)	(MSR_AMD64_SMCA_MC0_ADDR + 0x10*(x))
@@ -132,6 +138,8 @@
 #define MSR_AMD64_SMCA_MCx_DESTAT(x)	(MSR_AMD64_SMCA_MC0_DESTAT + 0x10*(x))
 #define MSR_AMD64_SMCA_MCx_DEADDR(x)	(MSR_AMD64_SMCA_MC0_DEADDR + 0x10*(x))
 #define MSR_AMD64_SMCA_MCx_MISCy(x, y)	((MSR_AMD64_SMCA_MC0_MISC1 + y) + (0x10*(x)))
+#define MSR_AMD64_SMCA_MCx_SYND1(x)	(MSR_AMD64_SMCA_MC0_SYND1 + 0x10*(x))
+#define MSR_AMD64_SMCA_MCx_SYND2(x)	(MSR_AMD64_SMCA_MC0_SYND2 + 0x10*(x))
 
 #define XEC(x, mask)			(((x) >> 16) & mask)
 
@@ -160,6 +168,12 @@
 #define MCE_IN_KERNEL_COPYIN	BIT_ULL(7)
 
 /*
+ * Indicates that handler should check and clear Deferred error registers
+ * rather than common ones.
+ */
+#define MCE_CHECK_DFR_REGS	BIT_ULL(8)
+
+/*
  * This structure contains all data related to the MCE log.  Also
  * carries a signature to make it easier to find from external
  * debugging tools.  Each entry is only valid when its finished flag
@@ -187,6 +201,32 @@ enum mce_notifier_prios {
 	MCE_PRIO_HIGHEST = MCE_PRIO_CEC
 };
 
+/**
+ * struct mce_hw_err - Hardware Error Record.
+ * @m:		Machine Check record.
+ * @vendor:	Vendor-specific error information.
+ *
+ * Vendor-specific fields should not be added to struct mce. Instead, vendors
+ * should export their vendor-specific data through their structure in the
+ * vendor union below.
+ *
+ * AMD's vendor data is parsed by error decoding tools for supplemental error
+ * information. Thus, current offsets of existing fields must be maintained.
+ * Only add new fields at the end of AMD's vendor structure.
+ */
+struct mce_hw_err {
+	struct mce m;
+
+	union vendor_info {
+		struct {
+			u64 synd1;		/* MCA_SYND1 MSR */
+			u64 synd2;		/* MCA_SYND2 MSR */
+		} amd;
+	} vendor;
+};
+
+#define	to_mce_hw_err(mce) container_of(mce, struct mce_hw_err, m)
+
 struct notifier_block;
 extern void mce_register_decode_chain(struct notifier_block *nb);
 extern void mce_unregister_decode_chain(struct notifier_block *nb);
@@ -209,20 +249,22 @@ struct cper_ia_proc_ctx;
 
 #ifdef CONFIG_X86_MCE
 int mcheck_init(void);
+void mca_bsp_init(struct cpuinfo_x86 *c);
 void mcheck_cpu_init(struct cpuinfo_x86 *c);
 void mcheck_cpu_clear(struct cpuinfo_x86 *c);
 int apei_smca_report_x86_error(struct cper_ia_proc_ctx *ctx_info,
 			       u64 lapic_id);
 #else
 static inline int mcheck_init(void) { return 0; }
+static inline void mca_bsp_init(struct cpuinfo_x86 *c) {}
 static inline void mcheck_cpu_init(struct cpuinfo_x86 *c) {}
 static inline void mcheck_cpu_clear(struct cpuinfo_x86 *c) {}
 static inline int apei_smca_report_x86_error(struct cper_ia_proc_ctx *ctx_info,
 					     u64 lapic_id) { return -EINVAL; }
 #endif
 
-void mce_prep_record(struct mce *m);
-void mce_log(struct mce *m);
+void mce_prep_record(struct mce_hw_err *err);
+void mce_log(struct mce_hw_err *err);
 DECLARE_PER_CPU(struct device *, mce_device);
 
 /* Maximum number of MCA banks per CPU. */
@@ -244,7 +286,7 @@ static inline void cmci_rediscover(void) {}
 static inline void cmci_recheck(void) {}
 #endif
 
-int mce_available(struct cpuinfo_x86 *c);
+bool mce_available(struct cpuinfo_x86 *c);
 bool mce_is_memory_error(struct mce *m);
 bool mce_is_correctable(struct mce *m);
 bool mce_usable_address(struct mce *m);
@@ -258,19 +300,22 @@ DECLARE_PER_CPU(mce_banks_t, mce_poll_banks);
 enum mcp_flags {
 	MCP_TIMESTAMP	= BIT(0),	/* log time stamp */
 	MCP_UC		= BIT(1),	/* log uncorrected errors */
-	MCP_DONTLOG	= BIT(2),	/* only clear, don't log */
-	MCP_QUEUE_LOG	= BIT(3),	/* only queue to genpool */
+	MCP_QUEUE_LOG	= BIT(2),	/* only queue to genpool */
 };
 
 void machine_check_poll(enum mcp_flags flags, mce_banks_t *b);
 
-int mce_notify_irq(void);
-
 DECLARE_PER_CPU(struct mce, injectm);
 
 /* Disable CMCI/polling for MCA bank claimed by firmware */
 extern void mce_disable_bank(int bank);
 
+#ifdef CONFIG_X86_MCE_THRESHOLD
+void mce_save_apei_thr_limit(u32 thr_limit);
+#else
+static inline void mce_save_apei_thr_limit(u32 thr_limit) { }
+#endif /* CONFIG_X86_MCE_THRESHOLD */
+
 /*
  * Exception handler
  */
@@ -341,21 +386,13 @@ enum smca_bank_types {
 
 extern bool amd_mce_is_memory_error(struct mce *m);
 
-extern int mce_threshold_create_device(unsigned int cpu);
-extern int mce_threshold_remove_device(unsigned int cpu);
-
 void mce_amd_feature_init(struct cpuinfo_x86 *c);
 enum smca_bank_types smca_get_bank_type(unsigned int cpu, unsigned int bank);
 #else
-
-static inline int mce_threshold_create_device(unsigned int cpu)		{ return 0; };
-static inline int mce_threshold_remove_device(unsigned int cpu)		{ return 0; };
 static inline bool amd_mce_is_memory_error(struct mce *m)		{ return false; };
 static inline void mce_amd_feature_init(struct cpuinfo_x86 *c)		{ }
 #endif
 
-static inline void mce_hygon_feature_init(struct cpuinfo_x86 *c)	{ return mce_amd_feature_init(c); }
-
 unsigned long copy_mc_fragile_handle_tail(char *to, char *from, unsigned len);
 
 #endif /* _ASM_X86_MCE_H */
diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
index f922b682b9b4..ea6494628cb0 100644
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -10,7 +10,7 @@
 #ifndef __X86_MEM_ENCRYPT_H__
 #define __X86_MEM_ENCRYPT_H__
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/init.h>
 #include <linux/cc_platform.h>
@@ -61,7 +61,7 @@ void __init sev_es_init_vc_handling(void);
 
 static inline u64 sme_get_me_mask(void)
 {
-	return RIP_REL_REF(sme_me_mask);
+	return sme_me_mask;
 }
 
 #define __bss_decrypted __section(".bss..decrypted")
@@ -114,6 +114,6 @@ void add_encrypt_protection_map(void);
 
 extern char __start_bss_decrypted[], __end_bss_decrypted[], __start_bss_decrypted_unused[];
 
-#endif	/* __ASSEMBLY__ */
+#endif	/* __ASSEMBLER__ */
 
 #endif	/* __X86_MEM_ENCRYPT_H__ */
diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h
index 695e569159c1..8b41f26f003b 100644
--- a/arch/x86/include/asm/microcode.h
+++ b/arch/x86/include/asm/microcode.h
@@ -2,6 +2,8 @@
 #ifndef _ASM_X86_MICROCODE_H
 #define _ASM_X86_MICROCODE_H
 
+#include <asm/msr.h>
+
 struct cpu_signature {
 	unsigned int sig;
 	unsigned int pf;
@@ -17,10 +19,12 @@ struct ucode_cpu_info {
 void load_ucode_bsp(void);
 void load_ucode_ap(void);
 void microcode_bsp_resume(void);
+bool __init microcode_loader_disabled(void);
 #else
 static inline void load_ucode_bsp(void)	{ }
 static inline void load_ucode_ap(void) { }
 static inline void microcode_bsp_resume(void) { }
+static inline bool __init microcode_loader_disabled(void) { return false; }
 #endif
 
 extern unsigned long initrd_start_early;
@@ -61,7 +65,7 @@ static inline u32 intel_get_microcode_revision(void)
 {
 	u32 rev, dummy;
 
-	native_wrmsrl(MSR_IA32_UCODE_REV, 0);
+	native_wrmsrq(MSR_IA32_UCODE_REV, 0);
 
 	/* As documented in the SDM: Do a CPUID 1 here */
 	native_cpuid_eax(1);
diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h
index ce4677b8b735..0fe9c569d171 100644
--- a/arch/x86/include/asm/mmu.h
+++ b/arch/x86/include/asm/mmu.h
@@ -16,6 +16,8 @@
 #define MM_CONTEXT_LOCK_LAM		2
 /* Allow LAM and SVA coexisting */
 #define MM_CONTEXT_FORCE_TAGGED_SVA	3
+/* Tracks mm_cpumask */
+#define MM_CONTEXT_NOTRACK		4
 
 /*
  * x86 has arch-specific MMU state beyond what lives in mm_struct.
@@ -37,14 +39,14 @@ typedef struct {
 	 */
 	atomic64_t tlb_gen;
 
+	unsigned long next_trim_cpumask;
+
 #ifdef CONFIG_MODIFY_LDT_SYSCALL
 	struct rw_semaphore	ldt_usr_sem;
 	struct ldt_struct	*ldt;
 #endif
 
-#ifdef CONFIG_X86_64
 	unsigned long flags;
-#endif
 
 #ifdef CONFIG_ADDRESS_MASKING
 	/* Active LAM mode:  X86_CR3_LAM_U48 or X86_CR3_LAM_U57 or 0 (disabled) */
@@ -67,6 +69,18 @@ typedef struct {
 	u16 pkey_allocation_map;
 	s16 execute_only_pkey;
 #endif
+
+#ifdef CONFIG_BROADCAST_TLB_FLUSH
+	/*
+	 * The global ASID will be a non-zero value when the process has
+	 * the same ASID across all CPUs, allowing it to make use of
+	 * hardware-assisted remote TLB invalidation like AMD INVLPGB.
+	 */
+	u16 global_asid;
+
+	/* The process is transitioning to a new global ASID number. */
+	bool asid_transition;
+#endif
 } mm_context_t;
 
 #define INIT_MM_CONTEXT(mm)						\
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 2886cb668d7f..73bf3b1b44e8 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -2,7 +2,6 @@
 #ifndef _ASM_X86_MMU_CONTEXT_H
 #define _ASM_X86_MMU_CONTEXT_H
 
-#include <asm/desc.h>
 #include <linux/atomic.h>
 #include <linux/mm_types.h>
 #include <linux/pkeys.h>
@@ -13,6 +12,7 @@
 #include <asm/paravirt.h>
 #include <asm/debugreg.h>
 #include <asm/gsseg.h>
+#include <asm/desc.h>
 
 extern atomic64_t last_mm_ctx_id;
 
@@ -139,6 +139,11 @@ static inline void mm_reset_untag_mask(struct mm_struct *mm)
 #define enter_lazy_tlb enter_lazy_tlb
 extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk);
 
+#define mm_init_global_asid mm_init_global_asid
+extern void mm_init_global_asid(struct mm_struct *mm);
+
+extern void mm_free_global_asid(struct mm_struct *mm);
+
 /*
  * Init a new mm.  Used on mm copies, like at fork()
  * and on mm's that are brand-new, like at execve().
@@ -151,6 +156,7 @@ static inline int init_new_context(struct task_struct *tsk,
 
 	mm->context.ctx_id = atomic64_inc_return(&last_mm_ctx_id);
 	atomic64_set(&mm->context.tlb_gen, 0);
+	mm->context.next_trim_cpumask = jiffies + HZ;
 
 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
 	if (cpu_feature_enabled(X86_FEATURE_OSPKE)) {
@@ -160,6 +166,8 @@ static inline int init_new_context(struct task_struct *tsk,
 		mm->context.execute_only_pkey = -1;
 	}
 #endif
+
+	mm_init_global_asid(mm);
 	mm_reset_untag_mask(mm);
 	init_new_context_ldt(mm);
 	return 0;
@@ -169,6 +177,7 @@ static inline int init_new_context(struct task_struct *tsk,
 static inline void destroy_context(struct mm_struct *mm)
 {
 	destroy_context_ldt(mm);
+	mm_free_global_asid(mm);
 }
 
 extern void switch_mm(struct mm_struct *prev, struct mm_struct *next,
@@ -181,7 +190,7 @@ extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 #define activate_mm(prev, next)			\
 do {						\
 	paravirt_enter_mmap(next);		\
-	switch_mm((prev), (next), NULL);	\
+	switch_mm_irqs_off((prev), (next), NULL);	\
 } while (0);
 
 #ifdef CONFIG_X86_32
@@ -238,6 +247,16 @@ static inline bool is_64bit_mm(struct mm_struct *mm)
 }
 #endif
 
+static inline bool is_notrack_mm(struct mm_struct *mm)
+{
+	return test_bit(MM_CONTEXT_NOTRACK, &mm->context.flags);
+}
+
+static inline void set_notrack_mm(struct mm_struct *mm)
+{
+	set_bit(MM_CONTEXT_NOTRACK, &mm->context.flags);
+}
+
 /*
  * We only want to enforce protection keys on the current process
  * because we effectively have no access to PKRU for other
@@ -263,4 +282,7 @@ unsigned long __get_current_cr3_fast(void);
 
 #include <asm-generic/mmu_context.h>
 
+extern struct mm_struct *use_temporary_mm(struct mm_struct *temp_mm);
+extern void unuse_temporary_mm(struct mm_struct *prev_mm);
+
 #endif /* _ASM_X86_MMU_CONTEXT_H */
diff --git a/arch/x86/include/asm/module.h b/arch/x86/include/asm/module.h
index e988bac0a4a1..3c2de4ce3b10 100644
--- a/arch/x86/include/asm/module.h
+++ b/arch/x86/include/asm/module.h
@@ -5,12 +5,20 @@
 #include <asm-generic/module.h>
 #include <asm/orc_types.h>
 
+struct its_array {
+#ifdef CONFIG_MITIGATION_ITS
+	void **pages;
+	int num;
+#endif
+};
+
 struct mod_arch_specific {
 #ifdef CONFIG_UNWINDER_ORC
 	unsigned int num_orcs;
 	int *orc_unwind_ip;
 	struct orc_entry *orc_unwind;
 #endif
+	struct its_array its_pages;
 };
 
 #endif /* _ASM_X86_MODULE_H */
diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 5f0bc6a6d025..eef4c3a5ba28 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -6,10 +6,12 @@
 #include <linux/nmi.h>
 #include <linux/msi.h>
 #include <linux/io.h>
-#include <asm/hyperv-tlfs.h>
+#include <linux/static_call.h>
 #include <asm/nospec-branch.h>
 #include <asm/paravirt.h>
-#include <asm/mshyperv.h>
+#include <asm/msr.h>
+#include <hyperv/hvhdk.h>
+#include <asm/fpu/types.h>
 
 /*
  * Hyper-V always provides a single IO-APIC at this MMIO address.
@@ -39,18 +41,21 @@ static inline unsigned char hv_get_nmi_reason(void)
 	return 0;
 }
 
-#if IS_ENABLED(CONFIG_HYPERV)
-extern bool hyperv_paravisor_present;
+extern u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2);
+extern u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2);
+extern u64 hv_std_hypercall(u64 control, u64 param1, u64 param2);
 
+#if IS_ENABLED(CONFIG_HYPERV)
 extern void *hv_hypercall_pg;
 
-extern u64 hv_current_partition_id;
-
 extern union hv_ghcb * __percpu *hv_ghcb_pg;
 
 bool hv_isolation_type_snp(void);
 bool hv_isolation_type_tdx(void);
-u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2);
+
+#ifdef CONFIG_X86_64
+DECLARE_STATIC_CALL(hv_hypercall, hv_std_hypercall);
+#endif
 
 /*
  * DEFAULT INIT GPAT and SEGMENT LIMIT value in struct VMSA
@@ -59,10 +64,6 @@ u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2);
 #define HV_AP_INIT_GPAT_DEFAULT		0x0007040600070406ULL
 #define HV_AP_SEGMENT_LIMIT		0xffffffff
 
-int hv_call_deposit_pages(int node, u64 partition_id, u32 num_pages);
-int hv_call_add_logical_proc(int node, u32 lp_index, u32 acpi_id);
-int hv_call_create_vp(int node, u64 partition_id, u32 vp_index, u32 flags);
-
 /*
  * If the hypercall involves no input or output parameters, the hypervisor
  * ignores the corresponding GPA pointer.
@@ -71,37 +72,15 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void *output)
 {
 	u64 input_address = input ? virt_to_phys(input) : 0;
 	u64 output_address = output ? virt_to_phys(output) : 0;
-	u64 hv_status;
 
 #ifdef CONFIG_X86_64
-	if (hv_isolation_type_tdx() && !hyperv_paravisor_present)
-		return hv_tdx_hypercall(control, input_address, output_address);
-
-	if (hv_isolation_type_snp() && !hyperv_paravisor_present) {
-		__asm__ __volatile__("mov %4, %%r8\n"
-				     "vmmcall"
-				     : "=a" (hv_status), ASM_CALL_CONSTRAINT,
-				       "+c" (control), "+d" (input_address)
-				     :  "r" (output_address)
-				     : "cc", "memory", "r8", "r9", "r10", "r11");
-		return hv_status;
-	}
-
-	if (!hv_hypercall_pg)
-		return U64_MAX;
-
-	__asm__ __volatile__("mov %4, %%r8\n"
-			     CALL_NOSPEC
-			     : "=a" (hv_status), ASM_CALL_CONSTRAINT,
-			       "+c" (control), "+d" (input_address)
-			     :  "r" (output_address),
-				THUNK_TARGET(hv_hypercall_pg)
-			     : "cc", "memory", "r8", "r9", "r10", "r11");
+	return static_call_mod(hv_hypercall)(control, input_address, output_address);
 #else
 	u32 input_address_hi = upper_32_bits(input_address);
 	u32 input_address_lo = lower_32_bits(input_address);
 	u32 output_address_hi = upper_32_bits(output_address);
 	u32 output_address_lo = lower_32_bits(output_address);
+	u64 hv_status;
 
 	if (!hv_hypercall_pg)
 		return U64_MAX;
@@ -114,54 +93,30 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void *output)
 			       "D"(output_address_hi), "S"(output_address_lo),
 			       THUNK_TARGET(hv_hypercall_pg)
 			     : "cc", "memory");
-#endif /* !x86_64 */
 	return hv_status;
-}
-
-/* Hypercall to the L0 hypervisor */
-static inline u64 hv_do_nested_hypercall(u64 control, void *input, void *output)
-{
-	return hv_do_hypercall(control | HV_HYPERCALL_NESTED, input, output);
+#endif /* !x86_64 */
 }
 
 /* Fast hypercall with 8 bytes of input and no output */
 static inline u64 _hv_do_fast_hypercall8(u64 control, u64 input1)
 {
-	u64 hv_status;
-
 #ifdef CONFIG_X86_64
-	if (hv_isolation_type_tdx() && !hyperv_paravisor_present)
-		return hv_tdx_hypercall(control, input1, 0);
-
-	if (hv_isolation_type_snp() && !hyperv_paravisor_present) {
-		__asm__ __volatile__(
-				"vmmcall"
-				: "=a" (hv_status), ASM_CALL_CONSTRAINT,
-				"+c" (control), "+d" (input1)
-				:: "cc", "r8", "r9", "r10", "r11");
-	} else {
-		__asm__ __volatile__(CALL_NOSPEC
-				     : "=a" (hv_status), ASM_CALL_CONSTRAINT,
-				       "+c" (control), "+d" (input1)
-				     : THUNK_TARGET(hv_hypercall_pg)
-				     : "cc", "r8", "r9", "r10", "r11");
-	}
+	return static_call_mod(hv_hypercall)(control, input1, 0);
 #else
-	{
-		u32 input1_hi = upper_32_bits(input1);
-		u32 input1_lo = lower_32_bits(input1);
-
-		__asm__ __volatile__ (CALL_NOSPEC
-				      : "=A"(hv_status),
-					"+c"(input1_lo),
-					ASM_CALL_CONSTRAINT
-				      :	"A" (control),
-					"b" (input1_hi),
-					THUNK_TARGET(hv_hypercall_pg)
-				      : "cc", "edi", "esi");
-	}
+	u32 input1_hi = upper_32_bits(input1);
+	u32 input1_lo = lower_32_bits(input1);
+	u64 hv_status;
+
+	__asm__ __volatile__ (CALL_NOSPEC
+			      : "=A"(hv_status),
+			      "+c"(input1_lo),
+			      ASM_CALL_CONSTRAINT
+			      :	"A" (control),
+			      "b" (input1_hi),
+			      THUNK_TARGET(hv_hypercall_pg)
+			      : "cc", "edi", "esi");
+	return hv_status;
 #endif
-		return hv_status;
 }
 
 static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
@@ -171,55 +126,27 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
 	return _hv_do_fast_hypercall8(control, input1);
 }
 
-static inline u64 hv_do_fast_nested_hypercall8(u16 code, u64 input1)
-{
-	u64 control = (u64)code | HV_HYPERCALL_FAST_BIT | HV_HYPERCALL_NESTED;
-
-	return _hv_do_fast_hypercall8(control, input1);
-}
-
 /* Fast hypercall with 16 bytes of input */
 static inline u64 _hv_do_fast_hypercall16(u64 control, u64 input1, u64 input2)
 {
-	u64 hv_status;
-
 #ifdef CONFIG_X86_64
-	if (hv_isolation_type_tdx() && !hyperv_paravisor_present)
-		return hv_tdx_hypercall(control, input1, input2);
-
-	if (hv_isolation_type_snp() && !hyperv_paravisor_present) {
-		__asm__ __volatile__("mov %4, %%r8\n"
-				     "vmmcall"
-				     : "=a" (hv_status), ASM_CALL_CONSTRAINT,
-				       "+c" (control), "+d" (input1)
-				     : "r" (input2)
-				     : "cc", "r8", "r9", "r10", "r11");
-	} else {
-		__asm__ __volatile__("mov %4, %%r8\n"
-				     CALL_NOSPEC
-				     : "=a" (hv_status), ASM_CALL_CONSTRAINT,
-				       "+c" (control), "+d" (input1)
-				     : "r" (input2),
-				       THUNK_TARGET(hv_hypercall_pg)
-				     : "cc", "r8", "r9", "r10", "r11");
-	}
+	return static_call_mod(hv_hypercall)(control, input1, input2);
 #else
-	{
-		u32 input1_hi = upper_32_bits(input1);
-		u32 input1_lo = lower_32_bits(input1);
-		u32 input2_hi = upper_32_bits(input2);
-		u32 input2_lo = lower_32_bits(input2);
-
-		__asm__ __volatile__ (CALL_NOSPEC
-				      : "=A"(hv_status),
-					"+c"(input1_lo), ASM_CALL_CONSTRAINT
-				      :	"A" (control), "b" (input1_hi),
-					"D"(input2_hi), "S"(input2_lo),
-					THUNK_TARGET(hv_hypercall_pg)
-				      : "cc");
-	}
-#endif
+	u32 input1_hi = upper_32_bits(input1);
+	u32 input1_lo = lower_32_bits(input1);
+	u32 input2_hi = upper_32_bits(input2);
+	u32 input2_lo = lower_32_bits(input2);
+	u64 hv_status;
+
+	__asm__ __volatile__ (CALL_NOSPEC
+			      : "=A"(hv_status),
+			      "+c"(input1_lo), ASM_CALL_CONSTRAINT
+			      :	"A" (control), "b" (input1_hi),
+			      "D"(input2_hi), "S"(input2_lo),
+			      THUNK_TARGET(hv_hypercall_pg)
+			      : "cc");
 	return hv_status;
+#endif
 }
 
 static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)
@@ -229,13 +156,6 @@ static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)
 	return _hv_do_fast_hypercall16(control, input1, input2);
 }
 
-static inline u64 hv_do_fast_nested_hypercall16(u16 code, u64 input1, u64 input2)
-{
-	u64 control = (u64)code | HV_HYPERCALL_FAST_BIT | HV_HYPERCALL_NESTED;
-
-	return _hv_do_fast_hypercall16(control, input1, input2);
-}
-
 extern struct hv_vp_assist_page **hv_vp_assist_page;
 
 static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu)
@@ -257,6 +177,8 @@ int hyperv_flush_guest_mapping_range(u64 as,
 int hyperv_fill_flush_guest_mapping_list(
 		struct hv_guest_mapping_flush_list *flush,
 		u64 start_gfn, u64 end_gfn);
+void hv_sleep_notifiers_register(void);
+void hv_machine_power_off(void);
 
 #ifdef CONFIG_X86_64
 void hv_apic_init(void);
@@ -268,6 +190,8 @@ static inline void hv_apic_init(void) {}
 
 struct irq_domain *hv_create_pci_msi_domain(void);
 
+int hv_map_msi_interrupt(struct irq_data *data,
+			 struct hv_interrupt_entry *out_entry);
 int hv_map_ioapic_interrupt(int ioapic_id, bool level, int vcpu, int vector,
 		struct hv_interrupt_entry *entry);
 int hv_unmap_ioapic_interrupt(int ioapic_id, struct hv_interrupt_entry *entry);
@@ -275,11 +199,12 @@ int hv_unmap_ioapic_interrupt(int ioapic_id, struct hv_interrupt_entry *entry);
 #ifdef CONFIG_AMD_MEM_ENCRYPT
 bool hv_ghcb_negotiate_protocol(void);
 void __noreturn hv_ghcb_terminate(unsigned int set, unsigned int reason);
-int hv_snp_boot_ap(u32 cpu, unsigned long start_ip);
+int hv_snp_boot_ap(u32 apic_id, unsigned long start_ip, unsigned int cpu);
 #else
 static inline bool hv_ghcb_negotiate_protocol(void) { return false; }
 static inline void hv_ghcb_terminate(unsigned int set, unsigned int reason) {}
-static inline int hv_snp_boot_ap(u32 cpu, unsigned long start_ip) { return 0; }
+static inline int hv_snp_boot_ap(u32 apic_id, unsigned long start_ip,
+		unsigned int cpu) { return 0; }
 #endif
 
 #if defined(CONFIG_AMD_MEM_ENCRYPT) || defined(CONFIG_INTEL_TDX_GUEST)
@@ -311,8 +236,18 @@ void hv_set_non_nested_msr(unsigned int reg, u64 value);
 
 static __always_inline u64 hv_raw_get_msr(unsigned int reg)
 {
-	return __rdmsr(reg);
+	return native_rdmsrq(reg);
 }
+int hv_apicid_to_vp_index(u32 apic_id);
+
+#if IS_ENABLED(CONFIG_MSHV_ROOT) && IS_ENABLED(CONFIG_CRASH_DUMP)
+void hv_root_crash_init(void);
+void hv_crash_asm32(void);
+void hv_crash_asm64(void);
+void hv_crash_asm_end(void);
+#else   /* CONFIG_MSHV_ROOT && CONFIG_CRASH_DUMP */
+static inline void hv_root_crash_init(void) {}
+#endif  /* CONFIG_MSHV_ROOT && CONFIG_CRASH_DUMP */
 
 #else /* CONFIG_HYPERV */
 static inline void hyperv_init(void) {}
@@ -334,15 +269,49 @@ static inline void hv_set_msr(unsigned int reg, u64 value) { }
 static inline u64 hv_get_msr(unsigned int reg) { return 0; }
 static inline void hv_set_non_nested_msr(unsigned int reg, u64 value) { }
 static inline u64 hv_get_non_nested_msr(unsigned int reg) { return 0; }
+static inline int hv_apicid_to_vp_index(u32 apic_id) { return -EINVAL; }
 #endif /* CONFIG_HYPERV */
 
+struct mshv_vtl_cpu_context {
+	union {
+		struct {
+			u64 rax;
+			u64 rcx;
+			u64 rdx;
+			u64 rbx;
+			u64 cr2;
+			u64 rbp;
+			u64 rsi;
+			u64 rdi;
+			u64 r8;
+			u64 r9;
+			u64 r10;
+			u64 r11;
+			u64 r12;
+			u64 r13;
+			u64 r14;
+			u64 r15;
+		};
+		u64 gp_regs[16];
+	};
+
+	struct fxregs_state fx_state;
+};
 
 #ifdef CONFIG_HYPERV_VTL_MODE
 void __init hv_vtl_init_platform(void);
 int __init hv_vtl_early_init(void);
+void mshv_vtl_return_call(struct mshv_vtl_cpu_context *vtl0);
+void mshv_vtl_return_call_init(u64 vtl_return_offset);
+void mshv_vtl_return_hypercall(void);
+void __mshv_vtl_return_call(struct mshv_vtl_cpu_context *vtl0);
 #else
 static inline void __init hv_vtl_init_platform(void) {}
 static inline int __init hv_vtl_early_init(void) { return 0; }
+static inline void mshv_vtl_return_call(struct mshv_vtl_cpu_context *vtl0) {}
+static inline void mshv_vtl_return_call_init(u64 vtl_return_offset) {}
+static inline void mshv_vtl_return_hypercall(void) {}
+static inline void __mshv_vtl_return_call(struct mshv_vtl_cpu_context *vtl0) {}
 #endif
 
 #include <asm-generic/mshyperv.h>
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 3ae84c3b8e6d..3d0a0950d20a 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -25,6 +25,7 @@
 #define _EFER_SVME		12 /* Enable virtualization */
 #define _EFER_LMSLE		13 /* Long Mode Segment Limit Enable */
 #define _EFER_FFXSR		14 /* Enable Fast FXSAVE/FXRSTOR */
+#define _EFER_TCE		15 /* Enable Translation Cache Extensions */
 #define _EFER_AUTOIBRS		21 /* Enable Automatic IBRS */
 
 #define EFER_SCE		(1<<_EFER_SCE)
@@ -34,6 +35,7 @@
 #define EFER_SVME		(1<<_EFER_SVME)
 #define EFER_LMSLE		(1<<_EFER_LMSLE)
 #define EFER_FFXSR		(1<<_EFER_FFXSR)
+#define EFER_TCE		(1<<_EFER_TCE)
 #define EFER_AUTOIBRS		(1<<_EFER_AUTOIBRS)
 
 /*
@@ -164,6 +166,10 @@
 						 * Processor MMIO stale data
 						 * vulnerabilities.
 						 */
+#define ARCH_CAP_MCU_ENUM		BIT(16) /*
+						 * Indicates the presence of microcode update
+						 * feature enumeration and status information.
+						 */
 #define ARCH_CAP_FB_CLEAR		BIT(17)	/*
 						 * VERW clears CPU fill buffer
 						 * even on MDS_NO CPUs.
@@ -209,6 +215,14 @@
 						 * VERW clears CPU Register
 						 * File.
 						 */
+#define ARCH_CAP_ITS_NO			BIT_ULL(62) /*
+						     * Not susceptible to
+						     * Indirect Target Selection.
+						     * This bit is not set by
+						     * HW, but is synthesized by
+						     * VMMs for guests to know
+						     * their affected status.
+						     */
 
 #define MSR_IA32_FLUSH_CMD		0x0000010b
 #define L1D_FLUSH			BIT(0)	/*
@@ -305,12 +319,37 @@
 #define PERF_CAP_PT_IDX			16
 
 #define MSR_PEBS_LD_LAT_THRESHOLD	0x000003f6
-#define PERF_CAP_PEBS_TRAP             BIT_ULL(6)
-#define PERF_CAP_ARCH_REG              BIT_ULL(7)
-#define PERF_CAP_PEBS_FORMAT           0xf00
-#define PERF_CAP_PEBS_BASELINE         BIT_ULL(14)
-#define PERF_CAP_PEBS_MASK	(PERF_CAP_PEBS_TRAP | PERF_CAP_ARCH_REG | \
-				 PERF_CAP_PEBS_FORMAT | PERF_CAP_PEBS_BASELINE)
+
+#define PERF_CAP_LBR_FMT		0x3f
+#define PERF_CAP_PEBS_TRAP		BIT_ULL(6)
+#define PERF_CAP_ARCH_REG		BIT_ULL(7)
+#define PERF_CAP_PEBS_FORMAT		0xf00
+#define PERF_CAP_FW_WRITES		BIT_ULL(13)
+#define PERF_CAP_PEBS_BASELINE		BIT_ULL(14)
+#define PERF_CAP_PEBS_TIMING_INFO	BIT_ULL(17)
+#define PERF_CAP_PEBS_MASK		(PERF_CAP_PEBS_TRAP | PERF_CAP_ARCH_REG | \
+					 PERF_CAP_PEBS_FORMAT | PERF_CAP_PEBS_BASELINE | \
+					 PERF_CAP_PEBS_TIMING_INFO)
+
+/* Arch PEBS */
+#define MSR_IA32_PEBS_BASE		0x000003f4
+#define MSR_IA32_PEBS_INDEX		0x000003f5
+#define ARCH_PEBS_OFFSET_MASK		0x7fffff
+#define ARCH_PEBS_INDEX_WR_SHIFT	4
+
+#define ARCH_PEBS_RELOAD		0xffffffff
+#define ARCH_PEBS_CNTR_ALLOW		BIT_ULL(35)
+#define ARCH_PEBS_CNTR_GP		BIT_ULL(36)
+#define ARCH_PEBS_CNTR_FIXED		BIT_ULL(37)
+#define ARCH_PEBS_CNTR_METRICS		BIT_ULL(38)
+#define ARCH_PEBS_LBR_SHIFT		40
+#define ARCH_PEBS_LBR			(0x3ull << ARCH_PEBS_LBR_SHIFT)
+#define ARCH_PEBS_VECR_XMM		BIT_ULL(49)
+#define ARCH_PEBS_GPR			BIT_ULL(61)
+#define ARCH_PEBS_AUX			BIT_ULL(62)
+#define ARCH_PEBS_EN			BIT_ULL(63)
+#define ARCH_PEBS_CNTR_MASK		(ARCH_PEBS_CNTR_GP | ARCH_PEBS_CNTR_FIXED | \
+					 ARCH_PEBS_CNTR_METRICS)
 
 #define MSR_IA32_RTIT_CTL		0x00000570
 #define RTIT_CTL_TRACEEN		BIT(0)
@@ -395,7 +434,8 @@
 #define MSR_IA32_PASID_VALID		BIT_ULL(31)
 
 /* DEBUGCTLMSR bits (others vary by model): */
-#define DEBUGCTLMSR_LBR			(1UL <<  0) /* last branch recording */
+#define DEBUGCTLMSR_LBR_BIT		0	     /* last branch recording */
+#define DEBUGCTLMSR_LBR			(1UL <<  DEBUGCTLMSR_LBR_BIT)
 #define DEBUGCTLMSR_BTF_SHIFT		1
 #define DEBUGCTLMSR_BTF			(1UL <<  1) /* single-step on branches */
 #define DEBUGCTLMSR_BUS_LOCK_DETECT	(1UL <<  2)
@@ -408,6 +448,7 @@
 #define DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI	(1UL << 12)
 #define DEBUGCTLMSR_FREEZE_IN_SMM_BIT	14
 #define DEBUGCTLMSR_FREEZE_IN_SMM	(1UL << DEBUGCTLMSR_FREEZE_IN_SMM_BIT)
+#define DEBUGCTLMSR_RTM_DEBUG		BIT(15)
 
 #define MSR_PEBS_FRONTEND		0x000003f7
 
@@ -522,7 +563,7 @@
 #define MSR_HWP_CAPABILITIES		0x00000771
 #define MSR_HWP_REQUEST_PKG		0x00000772
 #define MSR_HWP_INTERRUPT		0x00000773
-#define MSR_HWP_REQUEST 		0x00000774
+#define MSR_HWP_REQUEST			0x00000774
 #define MSR_HWP_STATUS			0x00000777
 
 /* CPUID.6.EAX */
@@ -539,16 +580,16 @@
 #define HWP_LOWEST_PERF(x)		(((x) >> 24) & 0xff)
 
 /* IA32_HWP_REQUEST */
-#define HWP_MIN_PERF(x) 		(x & 0xff)
-#define HWP_MAX_PERF(x) 		((x & 0xff) << 8)
+#define HWP_MIN_PERF(x)			(x & 0xff)
+#define HWP_MAX_PERF(x)			((x & 0xff) << 8)
 #define HWP_DESIRED_PERF(x)		((x & 0xff) << 16)
-#define HWP_ENERGY_PERF_PREFERENCE(x)	(((unsigned long long) x & 0xff) << 24)
+#define HWP_ENERGY_PERF_PREFERENCE(x)	(((u64)x & 0xff) << 24)
 #define HWP_EPP_PERFORMANCE		0x00
 #define HWP_EPP_BALANCE_PERFORMANCE	0x80
 #define HWP_EPP_BALANCE_POWERSAVE	0xC0
 #define HWP_EPP_POWERSAVE		0xFF
-#define HWP_ACTIVITY_WINDOW(x)		((unsigned long long)(x & 0xff3) << 32)
-#define HWP_PACKAGE_CONTROL(x)		((unsigned long long)(x & 0x1) << 42)
+#define HWP_ACTIVITY_WINDOW(x)		((u64)(x & 0xff3) << 32)
+#define HWP_PACKAGE_CONTROL(x)		((u64)(x & 0x1) << 42)
 
 /* IA32_HWP_STATUS */
 #define HWP_GUARANTEED_CHANGE(x)	(x & 0x1)
@@ -591,7 +632,11 @@
 /* V6 PMON MSR range */
 #define MSR_IA32_PMC_V6_GP0_CTR		0x1900
 #define MSR_IA32_PMC_V6_GP0_CFG_A	0x1901
+#define MSR_IA32_PMC_V6_GP0_CFG_B	0x1902
+#define MSR_IA32_PMC_V6_GP0_CFG_C	0x1903
 #define MSR_IA32_PMC_V6_FX0_CTR		0x1980
+#define MSR_IA32_PMC_V6_FX0_CFG_B	0x1982
+#define MSR_IA32_PMC_V6_FX0_CFG_C	0x1983
 #define MSR_IA32_PMC_V6_STEP		4
 
 /* KeyID partitioning between MKTME and TDX */
@@ -608,11 +653,18 @@
 #define MSR_AMD_PERF_CTL		0xc0010062
 #define MSR_AMD_PERF_STATUS		0xc0010063
 #define MSR_AMD_PSTATE_DEF_BASE		0xc0010064
+#define MSR_AMD64_GUEST_TSC_FREQ	0xc0010134
 #define MSR_AMD64_OSVW_ID_LENGTH	0xc0010140
 #define MSR_AMD64_OSVW_STATUS		0xc0010141
 #define MSR_AMD_PPIN_CTL		0xc00102f0
 #define MSR_AMD_PPIN			0xc00102f1
+#define MSR_AMD64_CPUID_FN_7		0xc0011002
 #define MSR_AMD64_CPUID_FN_1		0xc0011004
+
+#define MSR_AMD64_CPUID_EXT_FEAT	0xc0011005
+#define MSR_AMD64_CPUID_EXT_FEAT_TOPOEXT_BIT	54
+#define MSR_AMD64_CPUID_EXT_FEAT_TOPOEXT	BIT_ULL(MSR_AMD64_CPUID_EXT_FEAT_TOPOEXT_BIT)
+
 #define MSR_AMD64_LS_CFG		0xc0011020
 #define MSR_AMD64_DC_CFG		0xc0011022
 #define MSR_AMD64_TW_CFG		0xc0011023
@@ -644,6 +696,7 @@
 #define MSR_AMD64_IBS_REG_COUNT_MAX	8 /* includes MSR_AMD64_IBSBRTARGET */
 #define MSR_AMD64_SVM_AVIC_DOORBELL	0xc001011b
 #define MSR_AMD64_VM_PAGE_FLUSH		0xc001011e
+#define MSR_AMD64_VIRT_SPEC_CTRL	0xc001011f
 #define MSR_AMD64_SEV_ES_GHCB		0xc0010130
 #define MSR_AMD64_SEV			0xc0010131
 #define MSR_AMD64_SEV_ENABLED_BIT	0
@@ -680,13 +733,21 @@
 #define MSR_AMD64_SNP_VMSA_REG_PROT	BIT_ULL(MSR_AMD64_SNP_VMSA_REG_PROT_BIT)
 #define MSR_AMD64_SNP_SMT_PROT_BIT	17
 #define MSR_AMD64_SNP_SMT_PROT		BIT_ULL(MSR_AMD64_SNP_SMT_PROT_BIT)
-#define MSR_AMD64_SNP_RESV_BIT		18
+#define MSR_AMD64_SNP_SECURE_AVIC_BIT	18
+#define MSR_AMD64_SNP_SECURE_AVIC	BIT_ULL(MSR_AMD64_SNP_SECURE_AVIC_BIT)
+#define MSR_AMD64_SNP_RESV_BIT		19
 #define MSR_AMD64_SNP_RESERVED_MASK	GENMASK_ULL(63, MSR_AMD64_SNP_RESV_BIT)
-
-#define MSR_AMD64_VIRT_SPEC_CTRL	0xc001011f
-
+#define MSR_AMD64_SAVIC_CONTROL		0xc0010138
+#define MSR_AMD64_SAVIC_EN_BIT		0
+#define MSR_AMD64_SAVIC_EN		BIT_ULL(MSR_AMD64_SAVIC_EN_BIT)
+#define MSR_AMD64_SAVIC_ALLOWEDNMI_BIT	1
+#define MSR_AMD64_SAVIC_ALLOWEDNMI	BIT_ULL(MSR_AMD64_SAVIC_ALLOWEDNMI_BIT)
 #define MSR_AMD64_RMP_BASE		0xc0010132
 #define MSR_AMD64_RMP_END		0xc0010133
+#define MSR_AMD64_RMP_CFG		0xc0010136
+#define MSR_AMD64_SEG_RMP_ENABLED_BIT	0
+#define MSR_AMD64_SEG_RMP_ENABLED	BIT_ULL(MSR_AMD64_SEG_RMP_ENABLED_BIT)
+#define MSR_AMD64_RMP_SEGMENT_SHIFT(x)	(((x) & GENMASK_ULL(13, 8)) >> 8)
 
 #define MSR_SVSM_CAA			0xc001f000
 
@@ -697,26 +758,35 @@
 #define MSR_AMD_CPPC_REQ		0xc00102b3
 #define MSR_AMD_CPPC_STATUS		0xc00102b4
 
-#define AMD_CPPC_LOWEST_PERF(x)		(((x) >> 0) & 0xff)
-#define AMD_CPPC_LOWNONLIN_PERF(x)	(((x) >> 8) & 0xff)
-#define AMD_CPPC_NOMINAL_PERF(x)	(((x) >> 16) & 0xff)
-#define AMD_CPPC_HIGHEST_PERF(x)	(((x) >> 24) & 0xff)
+/* Masks for use with MSR_AMD_CPPC_CAP1 */
+#define AMD_CPPC_LOWEST_PERF_MASK	GENMASK(7, 0)
+#define AMD_CPPC_LOWNONLIN_PERF_MASK	GENMASK(15, 8)
+#define AMD_CPPC_NOMINAL_PERF_MASK	GENMASK(23, 16)
+#define AMD_CPPC_HIGHEST_PERF_MASK	GENMASK(31, 24)
 
-#define AMD_CPPC_MAX_PERF(x)		(((x) & 0xff) << 0)
-#define AMD_CPPC_MIN_PERF(x)		(((x) & 0xff) << 8)
-#define AMD_CPPC_DES_PERF(x)		(((x) & 0xff) << 16)
-#define AMD_CPPC_ENERGY_PERF_PREF(x)	(((x) & 0xff) << 24)
+/* Masks for use with MSR_AMD_CPPC_REQ */
+#define AMD_CPPC_MAX_PERF_MASK		GENMASK(7, 0)
+#define AMD_CPPC_MIN_PERF_MASK		GENMASK(15, 8)
+#define AMD_CPPC_DES_PERF_MASK		GENMASK(23, 16)
+#define AMD_CPPC_EPP_PERF_MASK		GENMASK(31, 24)
 
 /* AMD Performance Counter Global Status and Control MSRs */
 #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS	0xc0000300
 #define MSR_AMD64_PERF_CNTR_GLOBAL_CTL		0xc0000301
 #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR	0xc0000302
+#define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET	0xc0000303
+
+/* AMD Hardware Feedback Support MSRs */
+#define MSR_AMD_WORKLOAD_CLASS_CONFIG		0xc0000500
+#define MSR_AMD_WORKLOAD_CLASS_ID		0xc0000501
+#define MSR_AMD_WORKLOAD_HRST			0xc0000502
 
 /* AMD Last Branch Record MSRs */
 #define MSR_AMD64_LBR_SELECT			0xc000010e
 
 /* Zen4 */
 #define MSR_ZEN4_BP_CFG                 0xc001102e
+#define MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT 4
 #define MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT 5
 
 /* Fam 19h MSRs */
@@ -809,6 +879,7 @@
 #define MSR_K7_HWCR_SMMLOCK		BIT_ULL(MSR_K7_HWCR_SMMLOCK_BIT)
 #define MSR_K7_HWCR_IRPERF_EN_BIT	30
 #define MSR_K7_HWCR_IRPERF_EN		BIT_ULL(MSR_K7_HWCR_IRPERF_EN_BIT)
+#define MSR_K7_HWCR_CPUID_USER_DIS_BIT	35
 #define MSR_K7_FID_VID_CTL		0xc0010041
 #define MSR_K7_FID_VID_STATUS		0xc0010042
 #define MSR_K7_HWCR_CPB_DIS_BIT		25
@@ -882,6 +953,10 @@
 #define MSR_IA32_APICBASE_BASE		(0xfffff<<12)
 
 #define MSR_IA32_UCODE_WRITE		0x00000079
+
+#define MSR_IA32_MCU_ENUMERATION	0x0000007b
+#define MCU_STAGING			BIT(4)
+
 #define MSR_IA32_UCODE_REV		0x0000008b
 
 /* Intel SGX Launch Enclave Public Key Hash MSRs */
@@ -1179,6 +1254,8 @@
 #define MSR_IA32_VMX_VMFUNC             0x00000491
 #define MSR_IA32_VMX_PROCBASED_CTLS3	0x00000492
 
+#define MSR_IA32_MCU_STAGING_MBOX_ADDR	0x000007a5
+
 /* Resctrl MSRs: */
 /* - Intel: */
 #define MSR_IA32_L3_QOS_CFG		0xc81
@@ -1194,6 +1271,8 @@
 /* - AMD: */
 #define MSR_IA32_MBA_BW_BASE		0xc0000200
 #define MSR_IA32_SMBA_BW_BASE		0xc0000280
+#define MSR_IA32_L3_QOS_ABMC_CFG	0xc00003fd
+#define MSR_IA32_L3_QOS_EXT_CFG		0xc00003ff
 #define MSR_IA32_EVT_CFG_BASE		0xc0000400
 
 /* AMD-V MSRs */
diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h
index 001853541f1e..9c2ea29e12a9 100644
--- a/arch/x86/include/asm/msr.h
+++ b/arch/x86/include/asm/msr.h
@@ -4,7 +4,7 @@
 
 #include "msr-index.h"
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <asm/asm.h>
 #include <asm/errno.h>
@@ -12,6 +12,7 @@
 #include <uapi/asm/msr.h>
 #include <asm/shared/msr.h>
 
+#include <linux/types.h>
 #include <linux/percpu.h>
 
 struct msr_info {
@@ -37,23 +38,6 @@ struct saved_msrs {
 };
 
 /*
- * both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A"
- * constraint has different meanings. For i386, "A" means exactly
- * edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead,
- * it means rax *or* rdx.
- */
-#ifdef CONFIG_X86_64
-/* Using 64-bit values saves one instruction clearing the high half of low */
-#define DECLARE_ARGS(val, low, high)	unsigned long low, high
-#define EAX_EDX_VAL(val, low, high)	((low) | (high) << 32)
-#define EAX_EDX_RET(val, low, high)	"=a" (low), "=d" (high)
-#else
-#define DECLARE_ARGS(val, low, high)	unsigned long long val
-#define EAX_EDX_VAL(val, low, high)	(val)
-#define EAX_EDX_RET(val, low, high)	"=A" (val)
-#endif
-
-/*
  * Be very careful with includes. This header is prone to include loops.
  */
 #include <asm/atomic.h>
@@ -63,13 +47,13 @@ struct saved_msrs {
 DECLARE_TRACEPOINT(read_msr);
 DECLARE_TRACEPOINT(write_msr);
 DECLARE_TRACEPOINT(rdpmc);
-extern void do_trace_write_msr(unsigned int msr, u64 val, int failed);
-extern void do_trace_read_msr(unsigned int msr, u64 val, int failed);
-extern void do_trace_rdpmc(unsigned int msr, u64 val, int failed);
+extern void do_trace_write_msr(u32 msr, u64 val, int failed);
+extern void do_trace_read_msr(u32 msr, u64 val, int failed);
+extern void do_trace_rdpmc(u32 msr, u64 val, int failed);
 #else
-static inline void do_trace_write_msr(unsigned int msr, u64 val, int failed) {}
-static inline void do_trace_read_msr(unsigned int msr, u64 val, int failed) {}
-static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {}
+static inline void do_trace_write_msr(u32 msr, u64 val, int failed) {}
+static inline void do_trace_read_msr(u32 msr, u64 val, int failed) {}
+static inline void do_trace_rdpmc(u32 msr, u64 val, int failed) {}
 #endif
 
 /*
@@ -79,9 +63,9 @@ static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {}
  * think of extending them - you will be slapped with a stinking trout or a frozen
  * shark will reach you, wherever you are! You've been warned.
  */
-static __always_inline unsigned long long __rdmsr(unsigned int msr)
+static __always_inline u64 __rdmsr(u32 msr)
 {
-	DECLARE_ARGS(val, low, high);
+	EAX_EDX_DECLARE_ARGS(val, low, high);
 
 	asm volatile("1: rdmsr\n"
 		     "2:\n"
@@ -91,12 +75,12 @@ static __always_inline unsigned long long __rdmsr(unsigned int msr)
 	return EAX_EDX_VAL(val, low, high);
 }
 
-static __always_inline void __wrmsr(unsigned int msr, u32 low, u32 high)
+static __always_inline void __wrmsrq(u32 msr, u64 val)
 {
 	asm volatile("1: wrmsr\n"
 		     "2:\n"
 		     _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR)
-		     : : "c" (msr), "a"(low), "d" (high) : "memory");
+		     : : "c" (msr), "a" ((u32)val), "d" ((u32)(val >> 32)) : "memory");
 }
 
 #define native_rdmsr(msr, val1, val2)			\
@@ -106,16 +90,20 @@ do {							\
 	(void)((val2) = (u32)(__val >> 32));		\
 } while (0)
 
+static __always_inline u64 native_rdmsrq(u32 msr)
+{
+	return __rdmsr(msr);
+}
+
 #define native_wrmsr(msr, low, high)			\
-	__wrmsr(msr, low, high)
+	__wrmsrq((msr), (u64)(high) << 32 | (low))
 
-#define native_wrmsrl(msr, val)				\
-	__wrmsr((msr), (u32)((u64)(val)),		\
-		       (u32)((u64)(val) >> 32))
+#define native_wrmsrq(msr, val)				\
+	__wrmsrq((msr), (val))
 
-static inline unsigned long long native_read_msr(unsigned int msr)
+static inline u64 native_read_msr(u32 msr)
 {
-	unsigned long long val;
+	u64 val;
 
 	val = __rdmsr(msr);
 
@@ -125,34 +113,35 @@ static inline unsigned long long native_read_msr(unsigned int msr)
 	return val;
 }
 
-static inline unsigned long long native_read_msr_safe(unsigned int msr,
-						      int *err)
+static inline int native_read_msr_safe(u32 msr, u64 *p)
 {
-	DECLARE_ARGS(val, low, high);
+	int err;
+	EAX_EDX_DECLARE_ARGS(val, low, high);
 
 	asm volatile("1: rdmsr ; xor %[err],%[err]\n"
 		     "2:\n\t"
 		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_RDMSR_SAFE, %[err])
-		     : [err] "=r" (*err), EAX_EDX_RET(val, low, high)
+		     : [err] "=r" (err), EAX_EDX_RET(val, low, high)
 		     : "c" (msr));
 	if (tracepoint_enabled(read_msr))
-		do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err);
-	return EAX_EDX_VAL(val, low, high);
+		do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), err);
+
+	*p = EAX_EDX_VAL(val, low, high);
+
+	return err;
 }
 
 /* Can be uninlined because referenced by paravirt */
-static inline void notrace
-native_write_msr(unsigned int msr, u32 low, u32 high)
+static inline void notrace native_write_msr(u32 msr, u64 val)
 {
-	__wrmsr(msr, low, high);
+	native_wrmsrq(msr, val);
 
 	if (tracepoint_enabled(write_msr))
-		do_trace_write_msr(msr, ((u64)high << 32 | low), 0);
+		do_trace_write_msr(msr, val, 0);
 }
 
 /* Can be uninlined because referenced by paravirt */
-static inline int notrace
-native_write_msr_safe(unsigned int msr, u32 low, u32 high)
+static inline int notrace native_write_msr_safe(u32 msr, u64 val)
 {
 	int err;
 
@@ -160,73 +149,19 @@ native_write_msr_safe(unsigned int msr, u32 low, u32 high)
 		     "2:\n\t"
 		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_WRMSR_SAFE, %[err])
 		     : [err] "=a" (err)
-		     : "c" (msr), "0" (low), "d" (high)
+		     : "c" (msr), "0" ((u32)val), "d" ((u32)(val >> 32))
 		     : "memory");
 	if (tracepoint_enabled(write_msr))
-		do_trace_write_msr(msr, ((u64)high << 32 | low), err);
+		do_trace_write_msr(msr, val, err);
 	return err;
 }
 
 extern int rdmsr_safe_regs(u32 regs[8]);
 extern int wrmsr_safe_regs(u32 regs[8]);
 
-/**
- * rdtsc() - returns the current TSC without ordering constraints
- *
- * rdtsc() returns the result of RDTSC as a 64-bit integer.  The
- * only ordering constraint it supplies is the ordering implied by
- * "asm volatile": it will put the RDTSC in the place you expect.  The
- * CPU can and will speculatively execute that RDTSC, though, so the
- * results can be non-monotonic if compared on different CPUs.
- */
-static __always_inline unsigned long long rdtsc(void)
+static inline u64 native_read_pmc(int counter)
 {
-	DECLARE_ARGS(val, low, high);
-
-	asm volatile("rdtsc" : EAX_EDX_RET(val, low, high));
-
-	return EAX_EDX_VAL(val, low, high);
-}
-
-/**
- * rdtsc_ordered() - read the current TSC in program order
- *
- * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
- * It is ordered like a load to a global in-memory counter.  It should
- * be impossible to observe non-monotonic rdtsc_unordered() behavior
- * across multiple CPUs as long as the TSC is synced.
- */
-static __always_inline unsigned long long rdtsc_ordered(void)
-{
-	DECLARE_ARGS(val, low, high);
-
-	/*
-	 * The RDTSC instruction is not ordered relative to memory
-	 * access.  The Intel SDM and the AMD APM are both vague on this
-	 * point, but empirically an RDTSC instruction can be
-	 * speculatively executed before prior loads.  An RDTSC
-	 * immediately after an appropriate barrier appears to be
-	 * ordered as a normal load, that is, it provides the same
-	 * ordering guarantees as reading from a global memory location
-	 * that some other imaginary CPU is updating continuously with a
-	 * time stamp.
-	 *
-	 * Thus, use the preferred barrier on the respective CPU, aiming for
-	 * RDTSCP as the default.
-	 */
-	asm volatile(ALTERNATIVE_2("rdtsc",
-				   "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,
-				   "rdtscp", X86_FEATURE_RDTSCP)
-			: EAX_EDX_RET(val, low, high)
-			/* RDTSCP clobbers ECX with MSR_TSC_AUX. */
-			:: "ecx");
-
-	return EAX_EDX_VAL(val, low, high);
-}
-
-static inline unsigned long long native_read_pmc(int counter)
-{
-	DECLARE_ARGS(val, low, high);
+	EAX_EDX_DECLARE_ARGS(val, low, high);
 
 	asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter));
 	if (tracepoint_enabled(rdpmc))
@@ -251,56 +186,49 @@ do {								\
 	(void)((high) = (u32)(__val >> 32));			\
 } while (0)
 
-static inline void wrmsr(unsigned int msr, u32 low, u32 high)
+static inline void wrmsr(u32 msr, u32 low, u32 high)
 {
-	native_write_msr(msr, low, high);
+	native_write_msr(msr, (u64)high << 32 | low);
 }
 
-#define rdmsrl(msr, val)			\
+#define rdmsrq(msr, val)			\
 	((val) = native_read_msr((msr)))
 
-static inline void wrmsrl(unsigned int msr, u64 val)
+static inline void wrmsrq(u32 msr, u64 val)
 {
-	native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32));
+	native_write_msr(msr, val);
 }
 
 /* wrmsr with exception handling */
-static inline int wrmsr_safe(unsigned int msr, u32 low, u32 high)
+static inline int wrmsrq_safe(u32 msr, u64 val)
 {
-	return native_write_msr_safe(msr, low, high);
+	return native_write_msr_safe(msr, val);
 }
 
 /* rdmsr with exception handling */
 #define rdmsr_safe(msr, low, high)				\
 ({								\
-	int __err;						\
-	u64 __val = native_read_msr_safe((msr), &__err);	\
+	u64 __val;						\
+	int __err = native_read_msr_safe((msr), &__val);	\
 	(*low) = (u32)__val;					\
 	(*high) = (u32)(__val >> 32);				\
 	__err;							\
 })
 
-static inline int rdmsrl_safe(unsigned int msr, unsigned long long *p)
+static inline int rdmsrq_safe(u32 msr, u64 *p)
 {
-	int err;
-
-	*p = native_read_msr_safe(msr, &err);
-	return err;
+	return native_read_msr_safe(msr, p);
 }
 
-#define rdpmc(counter, low, high)			\
-do {							\
-	u64 _l = native_read_pmc((counter));		\
-	(low)  = (u32)_l;				\
-	(high) = (u32)(_l >> 32);			\
-} while (0)
-
-#define rdpmcl(counter, val) ((val) = native_read_pmc(counter))
+static __always_inline u64 rdpmc(int counter)
+{
+	return native_read_pmc(counter);
+}
 
 #endif	/* !CONFIG_PARAVIRT_XXL */
 
 /* Instruction opcode for WRMSRNS supported in binutils >= 2.40 */
-#define WRMSRNS _ASM_BYTES(0x0f,0x01,0xc6)
+#define ASM_WRMSRNS _ASM_BYTES(0x0f,0x01,0xc6)
 
 /* Non-serializing WRMSR, when available.  Falls back to a serializing WRMSR. */
 static __always_inline void wrmsrns(u32 msr, u64 val)
@@ -309,17 +237,17 @@ static __always_inline void wrmsrns(u32 msr, u64 val)
 	 * WRMSR is 2 bytes.  WRMSRNS is 3 bytes.  Pad WRMSR with a redundant
 	 * DS prefix to avoid a trailing NOP.
 	 */
-	asm volatile("1: " ALTERNATIVE("ds wrmsr", WRMSRNS, X86_FEATURE_WRMSRNS)
+	asm volatile("1: " ALTERNATIVE("ds wrmsr", ASM_WRMSRNS, X86_FEATURE_WRMSRNS)
 		     "2: " _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR)
 		     : : "c" (msr), "a" ((u32)val), "d" ((u32)(val >> 32)));
 }
 
 /*
- * 64-bit version of wrmsr_safe():
+ * Dual u32 version of wrmsrq_safe():
  */
-static inline int wrmsrl_safe(u32 msr, u64 val)
+static inline int wrmsr_safe(u32 msr, u32 low, u32 high)
 {
-	return wrmsr_safe(msr, (u32)val,  (u32)(val >> 32));
+	return wrmsrq_safe(msr, (u64)high << 32 | low);
 }
 
 struct msr __percpu *msrs_alloc(void);
@@ -330,14 +258,14 @@ int msr_clear_bit(u32 msr, u8 bit);
 #ifdef CONFIG_SMP
 int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
 int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
-int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
-int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
+int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
+int wrmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
 void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs);
 void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs);
 int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
 int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
-int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
-int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
+int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
+int wrmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
 int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
 int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
 #else  /*  CONFIG_SMP  */
@@ -351,14 +279,14 @@ static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
 	wrmsr(msr_no, l, h);
 	return 0;
 }
-static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
+static inline int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
 {
-	rdmsrl(msr_no, *q);
+	rdmsrq(msr_no, *q);
 	return 0;
 }
-static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
+static inline int wrmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
 {
-	wrmsrl(msr_no, q);
+	wrmsrq(msr_no, q);
 	return 0;
 }
 static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no,
@@ -380,13 +308,13 @@ static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
 {
 	return wrmsr_safe(msr_no, l, h);
 }
-static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
+static inline int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
 {
-	return rdmsrl_safe(msr_no, q);
+	return rdmsrq_safe(msr_no, q);
 }
-static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
+static inline int wrmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
 {
-	return wrmsrl_safe(msr_no, q);
+	return wrmsrq_safe(msr_no, q);
 }
 static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
 {
@@ -397,5 +325,11 @@ static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
 	return wrmsr_safe_regs(regs);
 }
 #endif  /* CONFIG_SMP */
-#endif /* __ASSEMBLY__ */
+
+/* Compatibility wrappers: */
+#define rdmsrl(msr, val) rdmsrq(msr, val)
+#define wrmsrl(msr, val) wrmsrq(msr, val)
+#define rdmsrl_on_cpu(cpu, msr, q) rdmsrq_on_cpu(cpu, msr, q)
+
+#endif /* __ASSEMBLER__ */
 #endif /* _ASM_X86_MSR_H */
diff --git a/arch/x86/include/asm/mtrr.h b/arch/x86/include/asm/mtrr.h
index 4218248083d9..76b95bd1a405 100644
--- a/arch/x86/include/asm/mtrr.h
+++ b/arch/x86/include/asm/mtrr.h
@@ -1,21 +1,8 @@
+/* SPDX-License-Identifier: LGPL-2.0+ */
 /*  Generic MTRR (Memory Type Range Register) ioctls.
 
     Copyright (C) 1997-1999  Richard Gooch
 
-    This library is free software; you can redistribute it and/or
-    modify it under the terms of the GNU Library General Public
-    License as published by the Free Software Foundation; either
-    version 2 of the License, or (at your option) any later version.
-
-    This library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-    Library General Public License for more details.
-
-    You should have received a copy of the GNU Library General Public
-    License along with this library; if not, write to the Free
-    Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
     Richard Gooch may be reached by email at  rgooch@atnf.csiro.au
     The postal address is:
       Richard Gooch, c/o ATNF, P. O. Box 76, Epping, N.S.W., 2121, Australia.
@@ -58,8 +45,8 @@ struct mtrr_state_type {
  */
 # ifdef CONFIG_MTRR
 void mtrr_bp_init(void);
-void mtrr_overwrite_state(struct mtrr_var_range *var, unsigned int num_var,
-			  mtrr_type def_type);
+void guest_force_mtrr_state(struct mtrr_var_range *var, unsigned int num_var,
+			    mtrr_type def_type);
 extern u8 mtrr_type_lookup(u64 addr, u64 end, u8 *uniform);
 extern void mtrr_save_fixed_ranges(void *);
 extern void mtrr_save_state(void);
@@ -75,9 +62,9 @@ void mtrr_disable(void);
 void mtrr_enable(void);
 void mtrr_generic_set_state(void);
 #  else
-static inline void mtrr_overwrite_state(struct mtrr_var_range *var,
-					unsigned int num_var,
-					mtrr_type def_type)
+static inline void guest_force_mtrr_state(struct mtrr_var_range *var,
+					  unsigned int num_var,
+					  mtrr_type def_type)
 {
 }
 
diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
index 920426d691ce..e4815e15dc9a 100644
--- a/arch/x86/include/asm/mwait.h
+++ b/arch/x86/include/asm/mwait.h
@@ -15,7 +15,6 @@
 #define MWAIT_HINT2SUBSTATE(hint)	((hint) & MWAIT_CSTATE_MASK)
 #define MWAIT_C1_SUBSTATE_MASK  0xf0
 
-#define CPUID_MWAIT_LEAF		5
 #define CPUID5_ECX_EXTENSIONS_SUPPORTED 0x1
 #define CPUID5_ECX_INTERRUPT_BREAK	0x2
 
@@ -26,29 +25,27 @@
 #define TPAUSE_C01_STATE		1
 #define TPAUSE_C02_STATE		0
 
-static __always_inline void __monitor(const void *eax, unsigned long ecx,
-			     unsigned long edx)
+static __always_inline void __monitor(const void *eax, u32 ecx, u32 edx)
 {
-	/* "monitor %eax, %ecx, %edx;" */
-	asm volatile(".byte 0x0f, 0x01, 0xc8;"
-		     :: "a" (eax), "c" (ecx), "d"(edx));
+	/*
+	 * Use the instruction mnemonic with implicit operands, as the LLVM
+	 * assembler fails to assemble the mnemonic with explicit operands:
+	 */
+	asm volatile("monitor" :: "a" (eax), "c" (ecx), "d" (edx));
 }
 
-static __always_inline void __monitorx(const void *eax, unsigned long ecx,
-			      unsigned long edx)
+static __always_inline void __monitorx(const void *eax, u32 ecx, u32 edx)
 {
-	/* "monitorx %eax, %ecx, %edx;" */
-	asm volatile(".byte 0x0f, 0x01, 0xfa;"
-		     :: "a" (eax), "c" (ecx), "d"(edx));
+	asm volatile("monitorx" :: "a" (eax), "c" (ecx), "d"(edx));
 }
 
-static __always_inline void __mwait(unsigned long eax, unsigned long ecx)
+static __always_inline void __mwait(u32 eax, u32 ecx)
 {
-	mds_idle_clear_cpu_buffers();
-
-	/* "mwait %eax, %ecx;" */
-	asm volatile(".byte 0x0f, 0x01, 0xc9;"
-		     :: "a" (eax), "c" (ecx));
+	/*
+	 * Use the instruction mnemonic with implicit operands, as the LLVM
+	 * assembler fails to assemble the mnemonic with explicit operands:
+	 */
+	asm volatile("mwait" :: "a" (eax), "c" (ecx));
 }
 
 /*
@@ -77,14 +74,11 @@ static __always_inline void __mwait(unsigned long eax, unsigned long ecx)
  * EAX                     (logical) address to monitor
  * ECX                     #GP if not zero
  */
-static __always_inline void __mwaitx(unsigned long eax, unsigned long ebx,
-				     unsigned long ecx)
+static __always_inline void __mwaitx(u32 eax, u32 ebx, u32 ecx)
 {
-	/* No MDS buffer clear as this is AMD/HYGON only */
+	/* No need for TSA buffer clearing on AMD */
 
-	/* "mwaitx %eax, %ebx, %ecx;" */
-	asm volatile(".byte 0x0f, 0x01, 0xfb;"
-		     :: "a" (eax), "b" (ebx), "c" (ecx));
+	asm volatile("mwaitx" :: "a" (eax), "b" (ebx), "c" (ecx));
 }
 
 /*
@@ -96,12 +90,10 @@ static __always_inline void __mwaitx(unsigned long eax, unsigned long ebx,
  * executing mwait, it would otherwise go unnoticed and the next tick
  * would not be reprogrammed accordingly before mwait ever wakes up.
  */
-static __always_inline void __sti_mwait(unsigned long eax, unsigned long ecx)
+static __always_inline void __sti_mwait(u32 eax, u32 ecx)
 {
-	mds_idle_clear_cpu_buffers();
-	/* "mwait %eax, %ecx;" */
-	asm volatile("sti; .byte 0x0f, 0x01, 0xc9;"
-		     :: "a" (eax), "c" (ecx));
+
+	asm volatile("sti; mwait" :: "a" (eax), "c" (ecx));
 }
 
 /*
@@ -114,26 +106,31 @@ static __always_inline void __sti_mwait(unsigned long eax, unsigned long ecx)
  * New with Core Duo processors, MWAIT can take some hints based on CPU
  * capability.
  */
-static __always_inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
+static __always_inline void mwait_idle_with_hints(u32 eax, u32 ecx)
 {
+	if (need_resched())
+		return;
+
+	x86_idle_clear_cpu_buffers();
+
 	if (static_cpu_has_bug(X86_BUG_MONITOR) || !current_set_polling_and_test()) {
-		if (static_cpu_has_bug(X86_BUG_CLFLUSH_MONITOR)) {
-			mb();
-			clflush((void *)&current_thread_info()->flags);
-			mb();
-		}
+		const void *addr = &current_thread_info()->flags;
+
+		alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr));
+		__monitor(addr, 0, 0);
 
-		__monitor((void *)&current_thread_info()->flags, 0, 0);
+		if (need_resched())
+			goto out;
 
-		if (!need_resched()) {
-			if (ecx & 1) {
-				__mwait(eax, ecx);
-			} else {
-				__sti_mwait(eax, ecx);
-				raw_local_irq_disable();
-			}
+		if (ecx & 1) {
+			__mwait(eax, ecx);
+		} else {
+			__sti_mwait(eax, ecx);
+			raw_local_irq_disable();
 		}
 	}
+
+out:
 	current_clr_polling();
 }
 
@@ -145,16 +142,9 @@ static __always_inline void mwait_idle_with_hints(unsigned long eax, unsigned lo
  */
 static inline void __tpause(u32 ecx, u32 edx, u32 eax)
 {
-	/* "tpause %ecx, %edx, %eax;" */
-	#ifdef CONFIG_AS_TPAUSE
-	asm volatile("tpause %%ecx\n"
-		     :
-		     : "c"(ecx), "d"(edx), "a"(eax));
-	#else
-	asm volatile(".byte 0x66, 0x0f, 0xae, 0xf1\t\n"
-		     :
-		     : "c"(ecx), "d"(edx), "a"(eax));
-	#endif
+	/* "tpause %ecx" */
+	asm volatile(".byte 0x66, 0x0f, 0xae, 0xf1"
+		     :: "c" (ecx), "d" (edx), "a" (eax));
 }
 
 #endif /* _ASM_X86_MWAIT_H */
diff --git a/arch/x86/include/asm/nmi.h b/arch/x86/include/asm/nmi.h
index 41a0ebb699ec..79d88d12c8fb 100644
--- a/arch/x86/include/asm/nmi.h
+++ b/arch/x86/include/asm/nmi.h
@@ -14,12 +14,26 @@ extern void release_perfctr_nmi(unsigned int);
 extern int reserve_evntsel_nmi(unsigned int);
 extern void release_evntsel_nmi(unsigned int);
 
-extern int unknown_nmi_panic;
-
 #endif /* CONFIG_X86_LOCAL_APIC */
 
+extern int unknown_nmi_panic;
+extern int panic_on_unrecovered_nmi;
+extern int panic_on_io_nmi;
+
+/* NMI handler flags */
 #define NMI_FLAG_FIRST	1
 
+/**
+ * enum - NMI types.
+ * @NMI_LOCAL:    Local NMI, CPU-specific NMI generated by the Local APIC.
+ * @NMI_UNKNOWN:  Unknown NMI, the source of the NMI may not be identified.
+ * @NMI_SERR:     System Error NMI, typically triggered by PCI errors.
+ * @NMI_IO_CHECK: I/O Check NMI, related to I/O errors.
+ * @NMI_MAX:      Maximum value for NMI types.
+ *
+ * NMI types are used to categorize NMIs and to dispatch them to the
+ * appropriate handler.
+ */
 enum {
 	NMI_LOCAL=0,
 	NMI_UNKNOWN,
@@ -28,6 +42,7 @@ enum {
 	NMI_MAX
 };
 
+/* NMI handler return values */
 #define NMI_DONE	0
 #define NMI_HANDLED	1
 
@@ -41,6 +56,25 @@ struct nmiaction {
 	const char		*name;
 };
 
+/**
+ * register_nmi_handler - Register a handler for a specific NMI type
+ * @t:    NMI type (e.g. NMI_LOCAL)
+ * @fn:   The NMI handler
+ * @fg:   Flags associated with the NMI handler
+ * @n:    Name of the NMI handler
+ * @init: Optional __init* attributes for struct nmiaction
+ *
+ * Adds the provided handler to the list of handlers for the specified
+ * NMI type. Handlers flagged with NMI_FLAG_FIRST would be executed first.
+ *
+ * Sometimes the source of an NMI can't be reliably determined which
+ * results in an NMI being tagged as "unknown". Register an additional
+ * handler using the NMI type - NMI_UNKNOWN to handle such cases. The
+ * caller would get one last chance to assume responsibility for the
+ * NMI.
+ *
+ * Return: 0 on success, or an error code on failure.
+ */
 #define register_nmi_handler(t, fn, fg, n, init...)	\
 ({							\
 	static struct nmiaction init fn##_na = {	\
@@ -54,7 +88,18 @@ struct nmiaction {
 
 int __register_nmi_handler(unsigned int, struct nmiaction *);
 
-void unregister_nmi_handler(unsigned int, const char *);
+/**
+ * unregister_nmi_handler - Unregister a handler for a specific NMI type
+ * @type: NMI type (e.g. NMI_LOCAL)
+ * @name: Name of the NMI handler used during registration
+ *
+ * Removes the handler associated with the specified NMI type from the
+ * NMI handler list. The "name" is used as a lookup key to identify the
+ * handler.
+ */
+void unregister_nmi_handler(unsigned int type, const char *name);
+
+void set_emergency_nmi_handler(unsigned int type, nmi_handler_t handler);
 
 void stop_nmi(void);
 void restart_nmi(void);
diff --git a/arch/x86/include/asm/nops.h b/arch/x86/include/asm/nops.h
index 1c1b7550fa55..cd94221d8335 100644
--- a/arch/x86/include/asm/nops.h
+++ b/arch/x86/include/asm/nops.h
@@ -82,7 +82,7 @@
 #define ASM_NOP7 _ASM_BYTES(BYTES_NOP7)
 #define ASM_NOP8 _ASM_BYTES(BYTES_NOP8)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 extern const unsigned char * const x86_nops[];
 #endif
 
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 96b410b1d4e8..4f4b5e8a1574 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -12,7 +12,6 @@
 #include <asm/msr-index.h>
 #include <asm/unwind_hints.h>
 #include <asm/percpu.h>
-#include <asm/current.h>
 
 /*
  * Call depth tracking for Intel SKL CPUs to address the RSB underflow
@@ -78,21 +77,21 @@
 #include <asm/asm-offsets.h>
 
 #define CREDIT_CALL_DEPTH					\
-	movq	$-1, PER_CPU_VAR(pcpu_hot + X86_call_depth);
+	movq	$-1, PER_CPU_VAR(__x86_call_depth);
 
 #define RESET_CALL_DEPTH					\
 	xor	%eax, %eax;					\
 	bts	$63, %rax;					\
-	movq	%rax, PER_CPU_VAR(pcpu_hot + X86_call_depth);
+	movq	%rax, PER_CPU_VAR(__x86_call_depth);
 
 #define RESET_CALL_DEPTH_FROM_CALL				\
 	movb	$0xfc, %al;					\
 	shl	$56, %rax;					\
-	movq	%rax, PER_CPU_VAR(pcpu_hot + X86_call_depth);	\
+	movq	%rax, PER_CPU_VAR(__x86_call_depth);		\
 	CALL_THUNKS_DEBUG_INC_CALLS
 
 #define INCREMENT_CALL_DEPTH					\
-	sarq	$5, PER_CPU_VAR(pcpu_hot + X86_call_depth);	\
+	sarq	$5, PER_CPU_VAR(__x86_call_depth);		\
 	CALL_THUNKS_DEBUG_INC_CALLS
 
 #else
@@ -177,19 +176,7 @@
 	add	$(BITS_PER_LONG/8), %_ASM_SP;		\
 	lfence;
 
-#ifdef __ASSEMBLY__
-
-/*
- * This should be used immediately before an indirect jump/call. It tells
- * objtool the subsequent indirect jump/call is vouched safe for retpoline
- * builds.
- */
-.macro ANNOTATE_RETPOLINE_SAFE
-.Lhere_\@:
-	.pushsection .discard.retpoline_safe
-	.long .Lhere_\@
-	.popsection
-.endm
+#ifdef __ASSEMBLER__
 
 /*
  * (ab)use RETPOLINE_SAFE on RET to annotate away 'bare' RET instructions
@@ -210,9 +197,8 @@
 .endm
 
 /*
- * Equivalent to -mindirect-branch-cs-prefix; emit the 5 byte jmp/call
- * to the retpoline thunk with a CS prefix when the register requires
- * a RAX prefix byte to encode. Also see apply_retpolines().
+ * Emits a conditional CS prefix that is compatible with
+ * -mindirect-branch-cs-prefix.
  */
 .macro __CS_PREFIX reg:req
 	.irp rs,r8,r9,r10,r11,r12,r13,r14,r15
@@ -283,7 +269,7 @@
  * typically has NO_MELTDOWN).
  *
  * While retbleed_untrain_ret() doesn't clobber anything but requires stack,
- * entry_ibpb() will clobber AX, CX, DX.
+ * write_ibpb() will clobber AX, CX, DX.
  *
  * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point
  * where we have a stack but before any RET instruction.
@@ -293,7 +279,7 @@
 	VALIDATE_UNRET_END
 	CALL_UNTRAIN_RET
 	ALTERNATIVE_2 "",						\
-		      "call entry_ibpb", \ibpb_feature,			\
+		      "call write_ibpb", \ibpb_feature,			\
 		     __stringify(\call_depth_insns), X86_FEATURE_CALL_DEPTH
 #endif
 .endm
@@ -316,24 +302,32 @@
 .endm
 
 /*
- * Macro to execute VERW instruction that mitigate transient data sampling
- * attacks such as MDS. On affected systems a microcode update overloaded VERW
- * instruction to also clear the CPU buffers. VERW clobbers CFLAGS.ZF.
- *
+ * Macro to execute VERW insns that mitigate transient data sampling
+ * attacks such as MDS or TSA. On affected systems a microcode update
+ * overloaded VERW insns to also clear the CPU buffers. VERW clobbers
+ * CFLAGS.ZF.
  * Note: Only the memory operand variant of VERW clears the CPU buffers.
  */
-.macro CLEAR_CPU_BUFFERS
 #ifdef CONFIG_X86_64
-	ALTERNATIVE "", "verw mds_verw_sel(%rip)", X86_FEATURE_CLEAR_CPU_BUF
+#define VERW	verw x86_verw_sel(%rip)
 #else
-	/*
-	 * In 32bit mode, the memory operand must be a %cs reference. The data
-	 * segments may not be usable (vm86 mode), and the stack segment may not
-	 * be flat (ESPFIX32).
-	 */
-	ALTERNATIVE "", "verw %cs:mds_verw_sel", X86_FEATURE_CLEAR_CPU_BUF
+/*
+ * In 32bit mode, the memory operand must be a %cs reference. The data segments
+ * may not be usable (vm86 mode), and the stack segment may not be flat (ESPFIX32).
+ */
+#define VERW	verw %cs:x86_verw_sel
 #endif
-.endm
+
+/*
+ * Provide a stringified VERW macro for simple usage, and a non-stringified
+ * VERW macro for use in more elaborate sequences, e.g. to encode a conditional
+ * VERW within an ALTERNATIVE.
+ */
+#define __CLEAR_CPU_BUFFERS	__stringify(VERW)
+
+/* If necessary, emit VERW on exit-to-userspace to clear CPU buffers. */
+#define CLEAR_CPU_BUFFERS \
+	ALTERNATIVE "", __CLEAR_CPU_BUFFERS, X86_FEATURE_CLEAR_CPU_BUF
 
 #ifdef CONFIG_X86_64
 .macro CLEAR_BRANCH_HISTORY
@@ -341,25 +335,23 @@
 .endm
 
 .macro CLEAR_BRANCH_HISTORY_VMEXIT
-	ALTERNATIVE "", "call clear_bhb_loop", X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT
+	ALTERNATIVE "", "call clear_bhb_loop", X86_FEATURE_CLEAR_BHB_VMEXIT
 .endm
 #else
 #define CLEAR_BRANCH_HISTORY
 #define CLEAR_BRANCH_HISTORY_VMEXIT
 #endif
 
-#else /* __ASSEMBLY__ */
+#else /* __ASSEMBLER__ */
 
-#define ANNOTATE_RETPOLINE_SAFE					\
-	"999:\n\t"						\
-	".pushsection .discard.retpoline_safe\n\t"		\
-	".long 999b\n\t"					\
-	".popsection\n\t"
+#define ITS_THUNK_SIZE	64
 
 typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];
+typedef u8 its_thunk_t[ITS_THUNK_SIZE];
 extern retpoline_thunk_t __x86_indirect_thunk_array[];
 extern retpoline_thunk_t __x86_indirect_call_thunk_array[];
 extern retpoline_thunk_t __x86_indirect_jump_thunk_array[];
+extern its_thunk_t	 __x86_indirect_its_thunk_array[];
 
 #ifdef CONFIG_MITIGATION_RETHUNK
 extern void __x86_return_thunk(void);
@@ -383,12 +375,18 @@ static inline void srso_return_thunk(void) {}
 static inline void srso_alias_return_thunk(void) {}
 #endif
 
+#ifdef CONFIG_MITIGATION_ITS
+extern void its_return_thunk(void);
+#else
+static inline void its_return_thunk(void) {}
+#endif
+
 extern void retbleed_return_thunk(void);
 extern void srso_return_thunk(void);
 extern void srso_alias_return_thunk(void);
 
 extern void entry_untrain_ret(void);
-extern void entry_ibpb(void);
+extern void write_ibpb(void);
 
 #ifdef CONFIG_X86_64
 extern void clear_bhb_loop(void);
@@ -406,6 +404,8 @@ extern void call_depth_return_thunk(void);
 		    __stringify(INCREMENT_CALL_DEPTH),		\
 		    X86_FEATURE_CALL_DEPTH)
 
+DECLARE_PER_CPU_CACHE_HOT(u64, __x86_call_depth);
+
 #ifdef CONFIG_CALL_THUNKS_DEBUG
 DECLARE_PER_CPU(u64, __x86_call_count);
 DECLARE_PER_CPU(u64, __x86_ret_count);
@@ -439,19 +439,22 @@ static inline void call_depth_return_thunk(void) {}
 #ifdef CONFIG_X86_64
 
 /*
+ * Emits a conditional CS prefix that is compatible with
+ * -mindirect-branch-cs-prefix.
+ */
+#define __CS_PREFIX(reg)				\
+	".irp rs,r8,r9,r10,r11,r12,r13,r14,r15\n"	\
+	".ifc \\rs," reg "\n"				\
+	".byte 0x2e\n"					\
+	".endif\n"					\
+	".endr\n"
+
+/*
  * Inline asm uses the %V modifier which is only in newer GCC
  * which is ensured when CONFIG_MITIGATION_RETPOLINE is defined.
  */
-# define CALL_NOSPEC						\
-	ALTERNATIVE_2(						\
-	ANNOTATE_RETPOLINE_SAFE					\
-	"call *%[thunk_target]\n",				\
-	"call __x86_indirect_thunk_%V[thunk_target]\n",		\
-	X86_FEATURE_RETPOLINE,					\
-	"lfence;\n"						\
-	ANNOTATE_RETPOLINE_SAFE					\
-	"call *%[thunk_target]\n",				\
-	X86_FEATURE_RETPOLINE_LFENCE)
+#define CALL_NOSPEC	__CS_PREFIX("%V[thunk_target]")	\
+			"call __x86_indirect_thunk_%V[thunk_target]\n"
 
 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
 
@@ -463,7 +466,7 @@ static inline void call_depth_return_thunk(void) {}
  */
 # define CALL_NOSPEC						\
 	ALTERNATIVE_2(						\
-	ANNOTATE_RETPOLINE_SAFE					\
+	ANNOTATE_RETPOLINE_SAFE "\n"				\
 	"call *%[thunk_target]\n",				\
 	"       jmp    904f;\n"					\
 	"       .align 16\n"					\
@@ -479,7 +482,7 @@ static inline void call_depth_return_thunk(void) {}
 	"904:	call   901b;\n",				\
 	X86_FEATURE_RETPOLINE,					\
 	"lfence;\n"						\
-	ANNOTATE_RETPOLINE_SAFE					\
+	ANNOTATE_RETPOLINE_SAFE "\n"				\
 	"call *%[thunk_target]\n",				\
 	X86_FEATURE_RETPOLINE_LFENCE)
 
@@ -513,6 +516,7 @@ enum spectre_v2_user_mitigation {
 /* The Speculative Store Bypass disable variants */
 enum ssb_mitigation {
 	SPEC_STORE_BYPASS_NONE,
+	SPEC_STORE_BYPASS_AUTO,
 	SPEC_STORE_BYPASS_DISABLE,
 	SPEC_STORE_BYPASS_PRCTL,
 	SPEC_STORE_BYPASS_SECCOMP,
@@ -529,11 +533,13 @@ void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
 		: "memory");
 }
 
-extern u64 x86_pred_cmd;
+DECLARE_PER_CPU(bool, x86_ibpb_exit_to_user);
 
 static inline void indirect_branch_prediction_barrier(void)
 {
-	alternative_msr_write(MSR_IA32_PRED_CMD, x86_pred_cmd, X86_FEATURE_USE_IBPB);
+	asm_inline volatile(ALTERNATIVE("", "call write_ibpb", X86_FEATURE_IBPB)
+			    : ASM_CALL_CONSTRAINT
+			    :: "rax", "rcx", "rdx", "memory");
 }
 
 /* The Intel SPEC CTRL MSR base value cache */
@@ -570,24 +576,24 @@ DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp);
 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 
-DECLARE_STATIC_KEY_FALSE(mds_idle_clear);
+DECLARE_STATIC_KEY_FALSE(switch_vcpu_ibpb);
 
-DECLARE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
+DECLARE_STATIC_KEY_FALSE(cpu_buf_idle_clear);
 
-DECLARE_STATIC_KEY_FALSE(mmio_stale_data_clear);
+DECLARE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
 
-extern u16 mds_verw_sel;
+extern u16 x86_verw_sel;
 
 #include <asm/segment.h>
 
 /**
- * mds_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability
+ * x86_clear_cpu_buffers - Buffer clearing support for different x86 CPU vulns
  *
  * This uses the otherwise unused and obsolete VERW instruction in
  * combination with microcode which triggers a CPU buffer flush when the
  * instruction is executed.
  */
-static __always_inline void mds_clear_cpu_buffers(void)
+static __always_inline void x86_clear_cpu_buffers(void)
 {
 	static const u16 ds = __KERNEL_DS;
 
@@ -604,16 +610,17 @@ static __always_inline void mds_clear_cpu_buffers(void)
 }
 
 /**
- * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability
+ * x86_idle_clear_cpu_buffers - Buffer clearing support in idle for the MDS
+ * and TSA vulnerabilities.
  *
  * Clear CPU buffers if the corresponding static key is enabled
  */
-static __always_inline void mds_idle_clear_cpu_buffers(void)
+static __always_inline void x86_idle_clear_cpu_buffers(void)
 {
-	if (static_branch_likely(&mds_idle_clear))
-		mds_clear_cpu_buffers();
+	if (static_branch_likely(&cpu_buf_idle_clear))
+		x86_clear_cpu_buffers();
 }
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
diff --git a/arch/x86/include/asm/numa.h b/arch/x86/include/asm/numa.h
index 5469d7a7c40f..53ba39ce010c 100644
--- a/arch/x86/include/asm/numa.h
+++ b/arch/x86/include/asm/numa.h
@@ -41,10 +41,6 @@ static inline int numa_cpu_node(int cpu)
 }
 #endif	/* CONFIG_NUMA */
 
-#ifdef CONFIG_X86_32
-# include <asm/numa_32.h>
-#endif
-
 #ifdef CONFIG_NUMA
 extern void numa_set_node(int cpu, int node);
 extern void numa_clear_node(int cpu);
diff --git a/arch/x86/include/asm/numa_32.h b/arch/x86/include/asm/numa_32.h
deleted file mode 100644
index 9c8e9e85be77..000000000000
--- a/arch/x86/include/asm/numa_32.h
+++ /dev/null
@@ -1,13 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_X86_NUMA_32_H
-#define _ASM_X86_NUMA_32_H
-
-#ifdef CONFIG_HIGHMEM
-extern void set_highmem_pages_init(void);
-#else
-static inline void set_highmem_pages_init(void)
-{
-}
-#endif
-
-#endif /* _ASM_X86_NUMA_32_H */
diff --git a/arch/x86/include/asm/orc_types.h b/arch/x86/include/asm/orc_types.h
index 46d7e06763c9..e0125afa53fb 100644
--- a/arch/x86/include/asm/orc_types.h
+++ b/arch/x86/include/asm/orc_types.h
@@ -45,7 +45,7 @@
 #define ORC_TYPE_REGS			3
 #define ORC_TYPE_REGS_PARTIAL		4
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <asm/byteorder.h>
 
 /*
@@ -73,6 +73,6 @@ struct orc_entry {
 #endif
 } __packed;
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ORC_TYPES_H */
diff --git a/arch/x86/include/asm/page.h b/arch/x86/include/asm/page.h
index 1b93ff80b43b..9265f2fca99a 100644
--- a/arch/x86/include/asm/page.h
+++ b/arch/x86/include/asm/page.h
@@ -14,7 +14,7 @@
 #include <asm/page_32.h>
 #endif	/* CONFIG_X86_64 */
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 struct page;
 
@@ -35,7 +35,7 @@ static inline void copy_user_page(void *to, void *from, unsigned long vaddr,
 }
 
 #define vma_alloc_zeroed_movable_folio(vma, vaddr) \
-	vma_alloc_folio(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, 0, vma, vaddr, false)
+	vma_alloc_folio(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, 0, vma, vaddr)
 
 #ifndef __pa
 #define __pa(x)		__phys_addr((unsigned long)(x))
@@ -84,7 +84,7 @@ static __always_inline u64 __is_canonical_address(u64 vaddr, u8 vaddr_bits)
 	return __canonical_address(vaddr, vaddr_bits) == vaddr;
 }
 
-#endif	/* __ASSEMBLY__ */
+#endif	/* __ASSEMBLER__ */
 
 #include <asm-generic/memory_model.h>
 #include <asm-generic/getorder.h>
diff --git a/arch/x86/include/asm/page_32.h b/arch/x86/include/asm/page_32.h
index 580d71aca65a..0c623706cb7e 100644
--- a/arch/x86/include/asm/page_32.h
+++ b/arch/x86/include/asm/page_32.h
@@ -4,7 +4,7 @@
 
 #include <asm/page_32_types.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #define __phys_addr_nodebug(x)	((x) - PAGE_OFFSET)
 #ifdef CONFIG_DEBUG_VIRTUAL
@@ -26,6 +26,6 @@ static inline void copy_page(void *to, void *from)
 {
 	memcpy(to, from, PAGE_SIZE);
 }
-#endif	/* !__ASSEMBLY__ */
+#endif	/* !__ASSEMBLER__ */
 
 #endif /* _ASM_X86_PAGE_32_H */
diff --git a/arch/x86/include/asm/page_32_types.h b/arch/x86/include/asm/page_32_types.h
index faf9cc1c14bb..623f1e9f493e 100644
--- a/arch/x86/include/asm/page_32_types.h
+++ b/arch/x86/include/asm/page_32_types.h
@@ -11,8 +11,8 @@
  * a virtual address space of one gigabyte, which limits the
  * amount of physical memory you can use to about 950MB.
  *
- * If you want more physical memory than this then see the CONFIG_HIGHMEM4G
- * and CONFIG_HIGHMEM64G options in the kernel configuration.
+ * If you want more physical memory than this then see the CONFIG_VMSPLIT_2G
+ * and CONFIG_HIGHMEM4G options in the kernel configuration.
  */
 #define __PAGE_OFFSET_BASE	_AC(CONFIG_PAGE_OFFSET, UL)
 #define __PAGE_OFFSET		__PAGE_OFFSET_BASE
@@ -63,7 +63,7 @@
  */
 #define KERNEL_IMAGE_SIZE	(512 * 1024 * 1024)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 /*
  * This much address space is reserved for vmalloc() and iomap()
@@ -73,8 +73,7 @@ extern unsigned int __VMALLOC_RESERVE;
 extern int sysctl_legacy_va_layout;
 
 extern void find_low_pfn_range(void);
-extern void setup_bootmem_allocator(void);
 
-#endif	/* !__ASSEMBLY__ */
+#endif	/* !__ASSEMBLER__ */
 
 #endif /* _ASM_X86_PAGE_32_DEFS_H */
diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index f3d257c45225..2f0e47be79a4 100644
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -4,11 +4,12 @@
 
 #include <asm/page_64_types.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <asm/cpufeatures.h>
 #include <asm/alternative.h>
 
 #include <linux/kmsan-checks.h>
+#include <linux/mmdebug.h>
 
 /* duplicated to the one in bootmem.h */
 extern unsigned long max_pfn;
@@ -17,7 +18,7 @@ extern unsigned long phys_base;
 extern unsigned long page_offset_base;
 extern unsigned long vmalloc_base;
 extern unsigned long vmemmap_base;
-extern unsigned long physmem_end;
+extern unsigned long direct_map_physmem_end;
 
 static __always_inline unsigned long __phys_addr_nodebug(unsigned long x)
 {
@@ -31,18 +32,28 @@ static __always_inline unsigned long __phys_addr_nodebug(unsigned long x)
 
 #ifdef CONFIG_DEBUG_VIRTUAL
 extern unsigned long __phys_addr(unsigned long);
-extern unsigned long __phys_addr_symbol(unsigned long);
 #else
 #define __phys_addr(x)		__phys_addr_nodebug(x)
-#define __phys_addr_symbol(x) \
-	((unsigned long)(x) - __START_KERNEL_map + phys_base)
 #endif
 
+static inline unsigned long __phys_addr_symbol(unsigned long x)
+{
+	unsigned long y = x - __START_KERNEL_map;
+
+	/* only check upper bounds since lower bounds will trigger carry */
+	VIRTUAL_BUG_ON(y >= KERNEL_IMAGE_SIZE);
+
+	return y + phys_base;
+}
+
 #define __phys_reloc_hide(x)	(x)
 
 void clear_page_orig(void *page);
 void clear_page_rep(void *page);
 void clear_page_erms(void *page);
+KCFI_REFERENCE(clear_page_orig);
+KCFI_REFERENCE(clear_page_rep);
+KCFI_REFERENCE(clear_page_erms);
 
 static inline void clear_page(void *page)
 {
@@ -55,13 +66,13 @@ static inline void clear_page(void *page)
 			   clear_page_rep, X86_FEATURE_REP_GOOD,
 			   clear_page_erms, X86_FEATURE_ERMS,
 			   "=D" (page),
-			   "D" (page)
-			   : "cc", "memory", "rax", "rcx");
+			   "D" (page),
+			   "cc", "memory", "rax", "rcx");
 }
 
 void copy_page(void *to, void *from);
+KCFI_REFERENCE(copy_page);
 
-#ifdef CONFIG_X86_5LEVEL
 /*
  * User space process size.  This is the first address outside the user range.
  * There are a few constraints that determine this:
@@ -92,9 +103,8 @@ static __always_inline unsigned long task_size_max(void)
 
 	return ret;
 }
-#endif	/* CONFIG_X86_5LEVEL */
 
-#endif	/* !__ASSEMBLY__ */
+#endif	/* !__ASSEMBLER__ */
 
 #ifdef CONFIG_X86_VSYSCALL_EMULATION
 # define __HAVE_ARCH_GATE_AREA 1
diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h
index 06ef25411d62..7400dab373fe 100644
--- a/arch/x86/include/asm/page_64_types.h
+++ b/arch/x86/include/asm/page_64_types.h
@@ -2,7 +2,7 @@
 #ifndef _ASM_X86_PAGE_64_DEFS_H
 #define _ASM_X86_PAGE_64_DEFS_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <asm/kaslr.h>
 #endif
 
@@ -41,25 +41,14 @@
 #define __PAGE_OFFSET_BASE_L5	_AC(0xff11000000000000, UL)
 #define __PAGE_OFFSET_BASE_L4	_AC(0xffff888000000000, UL)
 
-#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
 #define __PAGE_OFFSET           page_offset_base
-#else
-#define __PAGE_OFFSET           __PAGE_OFFSET_BASE_L4
-#endif /* CONFIG_DYNAMIC_MEMORY_LAYOUT */
 
 #define __START_KERNEL_map	_AC(0xffffffff80000000, UL)
 
 /* See Documentation/arch/x86/x86_64/mm.rst for a description of the memory map. */
 
 #define __PHYSICAL_MASK_SHIFT	52
-
-#ifdef CONFIG_X86_5LEVEL
 #define __VIRTUAL_MASK_SHIFT	(pgtable_l5_enabled() ? 56 : 47)
-/* See task_size_max() in <asm/page_64.h> */
-#else
-#define __VIRTUAL_MASK_SHIFT	47
-#define task_size_max()		((_AC(1,UL) << __VIRTUAL_MASK_SHIFT) - PAGE_SIZE)
-#endif
 
 #define TASK_SIZE_MAX		task_size_max()
 #define DEFAULT_MAP_WINDOW	((1UL << 47) - PAGE_SIZE)
diff --git a/arch/x86/include/asm/page_types.h b/arch/x86/include/asm/page_types.h
index 52f1b4ff0cc1..018a8d906ca3 100644
--- a/arch/x86/include/asm/page_types.h
+++ b/arch/x86/include/asm/page_types.h
@@ -6,10 +6,7 @@
 #include <linux/types.h>
 #include <linux/mem_encrypt.h>
 
-/* PAGE_SHIFT determines the page size */
-#define PAGE_SHIFT		CONFIG_PAGE_SHIFT
-#define PAGE_SIZE		(_AC(1,UL) << PAGE_SHIFT)
-#define PAGE_MASK		(~(PAGE_SIZE-1))
+#include <vdso/page.h>
 
 #define __VIRTUAL_MASK		((1UL << __VIRTUAL_MASK_SHIFT) - 1)
 
@@ -32,9 +29,7 @@
 #define VM_DATA_DEFAULT_FLAGS	VM_DATA_FLAGS_TSK_EXEC
 
 /* Physical address where kernel should be loaded. */
-#define LOAD_PHYSICAL_ADDR ((CONFIG_PHYSICAL_START \
-				+ (CONFIG_PHYSICAL_ALIGN - 1)) \
-				& ~(CONFIG_PHYSICAL_ALIGN - 1))
+#define LOAD_PHYSICAL_ADDR	__ALIGN_KERNEL_MASK(CONFIG_PHYSICAL_START, CONFIG_PHYSICAL_ALIGN - 1)
 
 #define __START_KERNEL		(__START_KERNEL_map + LOAD_PHYSICAL_ADDR)
 
@@ -46,7 +41,7 @@
 #define IOREMAP_MAX_ORDER       (PMD_SHIFT)
 #endif	/* CONFIG_X86_64 */
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK
 extern phys_addr_t physical_mask;
@@ -69,6 +64,6 @@ bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn);
 
 extern void initmem_init(void);
 
-#endif	/* !__ASSEMBLY__ */
+#endif	/* !__ASSEMBLER__ */
 
 #endif	/* _ASM_X86_PAGE_DEFS_H */
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index d4eb9e1d61b8..b5e59a7ba0d0 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -6,7 +6,7 @@
 
 #include <asm/paravirt_types.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 struct mm_struct;
 #endif
 
@@ -15,7 +15,7 @@ struct mm_struct;
 #include <asm/asm.h>
 #include <asm/nospec-branch.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/bug.h>
 #include <linux/types.h>
 #include <linux/cpumask.h>
@@ -91,11 +91,6 @@ static inline void __flush_tlb_multi(const struct cpumask *cpumask,
 	PVOP_VCALL2(mmu.flush_tlb_multi, cpumask, info);
 }
 
-static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table)
-{
-	PVOP_VCALL2(mmu.tlb_remove_table, tlb, table);
-}
-
 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
 {
 	PVOP_VCALL1(mmu.exit_mmap, mm);
@@ -107,6 +102,16 @@ static inline void notify_page_enc_status_changed(unsigned long pfn,
 	PVOP_VCALL3(mmu.notify_page_enc_status_changed, pfn, npages, enc);
 }
 
+static __always_inline void arch_safe_halt(void)
+{
+	PVOP_VCALL0(irq.safe_halt);
+}
+
+static inline void halt(void)
+{
+	PVOP_VCALL0(irq.halt);
+}
+
 #ifdef CONFIG_PARAVIRT_XXL
 static inline void load_sp0(unsigned long sp0)
 {
@@ -170,43 +175,24 @@ static inline void __write_cr4(unsigned long x)
 	PVOP_VCALL1(cpu.write_cr4, x);
 }
 
-static __always_inline void arch_safe_halt(void)
-{
-	PVOP_VCALL0(irq.safe_halt);
-}
-
-static inline void halt(void)
-{
-	PVOP_VCALL0(irq.halt);
-}
-
-extern noinstr void pv_native_wbinvd(void);
-
-static __always_inline void wbinvd(void)
-{
-	PVOP_ALT_VCALL0(cpu.wbinvd, "wbinvd", ALT_NOT_XEN);
-}
-
-static inline u64 paravirt_read_msr(unsigned msr)
+static inline u64 paravirt_read_msr(u32 msr)
 {
 	return PVOP_CALL1(u64, cpu.read_msr, msr);
 }
 
-static inline void paravirt_write_msr(unsigned msr,
-				      unsigned low, unsigned high)
+static inline void paravirt_write_msr(u32 msr, u64 val)
 {
-	PVOP_VCALL3(cpu.write_msr, msr, low, high);
+	PVOP_VCALL2(cpu.write_msr, msr, val);
 }
 
-static inline u64 paravirt_read_msr_safe(unsigned msr, int *err)
+static inline int paravirt_read_msr_safe(u32 msr, u64 *val)
 {
-	return PVOP_CALL2(u64, cpu.read_msr_safe, msr, err);
+	return PVOP_CALL2(int, cpu.read_msr_safe, msr, val);
 }
 
-static inline int paravirt_write_msr_safe(unsigned msr,
-					  unsigned low, unsigned high)
+static inline int paravirt_write_msr_safe(u32 msr, u64 val)
 {
-	return PVOP_CALL3(int, cpu.write_msr_safe, msr, low, high);
+	return PVOP_CALL2(int, cpu.write_msr_safe, msr, val);
 }
 
 #define rdmsr(msr, val1, val2)			\
@@ -216,55 +202,46 @@ do {						\
 	val2 = _l >> 32;			\
 } while (0)
 
-#define wrmsr(msr, val1, val2)			\
-do {						\
-	paravirt_write_msr(msr, val1, val2);	\
-} while (0)
+static __always_inline void wrmsr(u32 msr, u32 low, u32 high)
+{
+	paravirt_write_msr(msr, (u64)high << 32 | low);
+}
 
-#define rdmsrl(msr, val)			\
+#define rdmsrq(msr, val)			\
 do {						\
 	val = paravirt_read_msr(msr);		\
 } while (0)
 
-static inline void wrmsrl(unsigned msr, u64 val)
+static inline void wrmsrq(u32 msr, u64 val)
 {
-	wrmsr(msr, (u32)val, (u32)(val>>32));
+	paravirt_write_msr(msr, val);
 }
 
-#define wrmsr_safe(msr, a, b)	paravirt_write_msr_safe(msr, a, b)
+static inline int wrmsrq_safe(u32 msr, u64 val)
+{
+	return paravirt_write_msr_safe(msr, val);
+}
 
 /* rdmsr with exception handling */
 #define rdmsr_safe(msr, a, b)				\
 ({							\
-	int _err;					\
-	u64 _l = paravirt_read_msr_safe(msr, &_err);	\
+	u64 _l;						\
+	int _err = paravirt_read_msr_safe((msr), &_l);	\
 	(*a) = (u32)_l;					\
-	(*b) = _l >> 32;				\
+	(*b) = (u32)(_l >> 32);				\
 	_err;						\
 })
 
-static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
+static __always_inline int rdmsrq_safe(u32 msr, u64 *p)
 {
-	int err;
-
-	*p = paravirt_read_msr_safe(msr, &err);
-	return err;
+	return paravirt_read_msr_safe(msr, p);
 }
 
-static inline unsigned long long paravirt_read_pmc(int counter)
+static __always_inline u64 rdpmc(int counter)
 {
 	return PVOP_CALL1(u64, cpu.read_pmc, counter);
 }
 
-#define rdpmc(counter, low, high)		\
-do {						\
-	u64 _l = paravirt_read_pmc(counter);	\
-	low = (u32)_l;				\
-	high = _l >> 32;			\
-} while (0)
-
-#define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter))
-
 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
 {
 	PVOP_VCALL2(cpu.alloc_ldt, ldt, entries);
@@ -486,8 +463,6 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
 	PVOP_VCALL2(mmu.set_p4d, p4dp, val);
 }
 
-#if CONFIG_PGTABLE_LEVELS >= 5
-
 static inline p4d_t __p4d(p4dval_t val)
 {
 	p4dval_t ret = PVOP_ALT_CALLEE1(p4dval_t, mmu.make_p4d, val,
@@ -519,8 +494,6 @@ static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd)
 		set_pgd(pgdp, native_make_pgd(0));			\
 } while (0)
 
-#endif  /* CONFIG_PGTABLE_LEVELS == 5 */
-
 static inline void p4d_clear(p4d_t *p4dp)
 {
 	set_p4d(p4dp, native_make_p4d(0));
@@ -727,7 +700,7 @@ static __always_inline unsigned long arch_local_irq_save(void)
 extern void default_banner(void);
 void native_pv_lock_init(void) __init;
 
-#else  /* __ASSEMBLY__ */
+#else  /* __ASSEMBLER__ */
 
 #ifdef CONFIG_X86_64
 #ifdef CONFIG_PARAVIRT_XXL
@@ -747,18 +720,18 @@ void native_pv_lock_init(void) __init;
 #endif /* CONFIG_PARAVIRT_XXL */
 #endif	/* CONFIG_X86_64 */
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #else  /* CONFIG_PARAVIRT */
 # define default_banner x86_init_noop
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 static inline void native_pv_lock_init(void)
 {
 }
 #endif
 #endif /* !CONFIG_PARAVIRT */
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #ifndef CONFIG_PARAVIRT_XXL
 static inline void paravirt_enter_mmap(struct mm_struct *mm)
 {
@@ -776,5 +749,5 @@ static inline void paravirt_set_cap(void)
 {
 }
 #endif
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* _ASM_X86_PARAVIRT_H */
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 8d4fbe1be489..3502939415ad 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -4,7 +4,7 @@
 
 #ifdef CONFIG_PARAVIRT
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/types.h>
 
 #include <asm/desc_defs.h>
@@ -86,22 +86,20 @@ struct pv_cpu_ops {
 	void (*update_io_bitmap)(void);
 #endif
 
-	void (*wbinvd)(void);
-
 	/* cpuid emulation, mostly so that caps bits can be disabled */
 	void (*cpuid)(unsigned int *eax, unsigned int *ebx,
 		      unsigned int *ecx, unsigned int *edx);
 
 	/* Unsafe MSR operations.  These will warn or panic on failure. */
-	u64 (*read_msr)(unsigned int msr);
-	void (*write_msr)(unsigned int msr, unsigned low, unsigned high);
+	u64 (*read_msr)(u32 msr);
+	void (*write_msr)(u32 msr, u64 val);
 
 	/*
 	 * Safe MSR operations.
-	 * read sets err to 0 or -EIO.  write returns 0 or -EIO.
+	 * Returns 0 or -EIO.
 	 */
-	u64 (*read_msr_safe)(unsigned int msr, int *err);
-	int (*write_msr_safe)(unsigned int msr, unsigned low, unsigned high);
+	int (*read_msr_safe)(u32 msr, u64 *val);
+	int (*write_msr_safe)(u32 msr, u64 val);
 
 	u64 (*read_pmc)(int counter);
 
@@ -122,10 +120,9 @@ struct pv_irq_ops {
 	struct paravirt_callee_save save_fl;
 	struct paravirt_callee_save irq_disable;
 	struct paravirt_callee_save irq_enable;
-
+#endif
 	void (*safe_halt)(void);
 	void (*halt)(void);
-#endif
 } __no_randomize_layout;
 
 struct pv_mmu_ops {
@@ -136,8 +133,6 @@ struct pv_mmu_ops {
 	void (*flush_tlb_multi)(const struct cpumask *cpus,
 				const struct flush_tlb_info *info);
 
-	void (*tlb_remove_table)(struct mmu_gather *tlb, void *table);
-
 	/* Hook for intercepting the destruction of an mm_struct. */
 	void (*exit_mmap)(struct mm_struct *mm);
 	void (*notify_page_enc_status_changed)(unsigned long pfn, int npages, bool enc);
@@ -194,12 +189,10 @@ struct pv_mmu_ops {
 
 	void (*set_p4d)(p4d_t *p4dp, p4d_t p4dval);
 
-#if CONFIG_PGTABLE_LEVELS >= 5
 	struct paravirt_callee_save p4d_val;
 	struct paravirt_callee_save make_p4d;
 
 	void (*set_pgd)(pgd_t *pgdp, pgd_t pgdval);
-#endif	/* CONFIG_PGTABLE_LEVELS >= 5 */
 
 	struct pv_lazy_ops lazy_mode;
 
@@ -244,11 +237,19 @@ extern struct paravirt_patch_template pv_ops;
 
 #define paravirt_ptr(op)	[paravirt_opptr] "m" (pv_ops.op)
 
-int paravirt_disable_iospace(void);
-
-/* This generates an indirect call based on the operation type number. */
+/*
+ * This generates an indirect call based on the operation type number.
+ *
+ * Since alternatives run after enabling CET/IBT -- the latter setting/clearing
+ * capabilities and the former requiring all capabilities being finalized --
+ * these indirect calls are subject to IBT and the paravirt stubs should have
+ * ENDBR on.
+ *
+ * OTOH since this is effectively a __nocfi indirect call, the paravirt stubs
+ * don't need to bother with CFI prefixes.
+ */
 #define PARAVIRT_CALL					\
-	ANNOTATE_RETPOLINE_SAFE				\
+	ANNOTATE_RETPOLINE_SAFE "\n\t"			\
 	"call *%[paravirt_opptr];"
 
 /*
@@ -521,7 +522,7 @@ unsigned long pv_native_read_cr2(void);
 
 #define paravirt_nop	((void *)nop_func)
 
-#endif	/* __ASSEMBLY__ */
+#endif	/* __ASSEMBLER__ */
 
 #define ALT_NOT_XEN	ALT_NOT(X86_FEATURE_XENPV)
 
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index c55a79d5feae..725d0eff7acd 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -10,7 +10,7 @@
 # define __percpu_rel
 #endif
 
-#ifdef __ASSEMBLY__
+#ifdef __ASSEMBLER__
 
 #ifdef CONFIG_SMP
 # define __percpu		%__percpu_seg:
@@ -20,20 +20,18 @@
 
 #define PER_CPU_VAR(var)	__percpu(var)__percpu_rel
 
-#ifdef CONFIG_X86_64_SMP
-# define INIT_PER_CPU_VAR(var)  init_per_cpu__##var
-#else
-# define INIT_PER_CPU_VAR(var)  var
-#endif
-
 #else /* !__ASSEMBLY__: */
 
+#include <linux/args.h>
+#include <linux/bits.h>
 #include <linux/build_bug.h>
 #include <linux/stringify.h>
 #include <asm/asm.h>
 
 #ifdef CONFIG_SMP
 
+#define __force_percpu_prefix	"%%"__stringify(__percpu_seg)":"
+
 #ifdef CONFIG_CC_HAS_NAMED_AS
 
 #ifdef __CHECKER__
@@ -41,28 +39,23 @@
 # define __seg_fs		__attribute__((address_space(__seg_fs)))
 #endif
 
-#ifdef CONFIG_X86_64
-# define __percpu_seg_override	__seg_gs
-#else
-# define __percpu_seg_override	__seg_fs
-#endif
-
-#define __percpu_prefix		""
+#define __percpu_prefix
+#define __percpu_seg_override	CONCATENATE(__seg_, __percpu_seg)
 
 #else /* !CONFIG_CC_HAS_NAMED_AS: */
 
+#define __percpu_prefix		__force_percpu_prefix
 #define __percpu_seg_override
-#define __percpu_prefix		"%%"__stringify(__percpu_seg)":"
 
 #endif /* CONFIG_CC_HAS_NAMED_AS */
 
-#define __force_percpu_prefix	"%%"__stringify(__percpu_seg)":"
-#define __my_cpu_offset		this_cpu_read(this_cpu_off)
-
 /*
  * Compared to the generic __my_cpu_offset version, the following
  * saves one instruction and avoids clobbering a temp register.
- *
+ */
+#define __my_cpu_offset		this_cpu_read(this_cpu_off)
+
+/*
  * arch_raw_cpu_ptr should not be used in 32-bit VDSO for a 64-bit
  * kernel, because games are played with CONFIG_X86_64 there and
  * sizeof(this_cpu_off) becames 4.
@@ -73,46 +66,43 @@
 	unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off);	\
 									\
 	tcp_ptr__ += (__force unsigned long)(_ptr);			\
-	(typeof(*(_ptr)) __kernel __force *)tcp_ptr__;			\
+	(TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__;		\
 })
 #else
-#define arch_raw_cpu_ptr(_ptr) ({ BUILD_BUG(); (typeof(_ptr))0; })
+#define arch_raw_cpu_ptr(_ptr)						\
+({									\
+	BUILD_BUG();							\
+	(TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)0;			\
+})
 #endif
 
 #define PER_CPU_VAR(var)	%__percpu_seg:(var)__percpu_rel
 
 #else /* !CONFIG_SMP: */
 
+#define __force_percpu_prefix
+#define __percpu_prefix
 #define __percpu_seg_override
-#define __percpu_prefix		""
-#define __force_percpu_prefix	""
 
 #define PER_CPU_VAR(var)	(var)__percpu_rel
 
 #endif /* CONFIG_SMP */
 
-#define __my_cpu_type(var)	typeof(var) __percpu_seg_override
-#define __my_cpu_ptr(ptr)	(__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)
-#define __my_cpu_var(var)	(*__my_cpu_ptr(&(var)))
-#define __percpu_arg(x)		__percpu_prefix "%" #x
-#define __force_percpu_arg(x)	__force_percpu_prefix "%" #x
-
-/*
- * Initialized pointers to per-CPU variables needed for the boot
- * processor need to use these macros to get the proper address
- * offset from __per_cpu_load on SMP.
- *
- * There also must be an entry in vmlinux_64.lds.S
- */
-#define DECLARE_INIT_PER_CPU(var) \
-       extern typeof(var) init_per_cpu_var(var)
+#if defined(CONFIG_USE_X86_SEG_SUPPORT) && defined(USE_TYPEOF_UNQUAL)
+# define __my_cpu_type(var)	typeof(var)
+# define __my_cpu_ptr(ptr)	(ptr)
+# define __my_cpu_var(var)	(var)
 
-#ifdef CONFIG_X86_64_SMP
-# define init_per_cpu_var(var)  init_per_cpu__##var
+# define __percpu_qual		__percpu_seg_override
 #else
-# define init_per_cpu_var(var)  var
+# define __my_cpu_type(var)	typeof(var) __percpu_seg_override
+# define __my_cpu_ptr(ptr)	(__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)
+# define __my_cpu_var(var)	(*__my_cpu_ptr(&(var)))
 #endif
 
+#define __force_percpu_arg(x)	__force_percpu_prefix "%" #x
+#define __percpu_arg(x)		__percpu_prefix "%" #x
+
 /*
  * For arch-specific code, we can use direct single-insn ops (they
  * don't give an lvalue though).
@@ -128,15 +118,10 @@
 #define __pcpu_cast_4(val)	((u32)(((unsigned long) val) & 0xffffffff))
 #define __pcpu_cast_8(val)	((u64)(val))
 
-#define __pcpu_op1_1(op, dst)	op "b " dst
-#define __pcpu_op1_2(op, dst)	op "w " dst
-#define __pcpu_op1_4(op, dst)	op "l " dst
-#define __pcpu_op1_8(op, dst)	op "q " dst
-
-#define __pcpu_op2_1(op, src, dst) op "b " src ", " dst
-#define __pcpu_op2_2(op, src, dst) op "w " src ", " dst
-#define __pcpu_op2_4(op, src, dst) op "l " src ", " dst
-#define __pcpu_op2_8(op, src, dst) op "q " src ", " dst
+#define __pcpu_op_1(op)		op "b "
+#define __pcpu_op_2(op)		op "w "
+#define __pcpu_op_4(op)		op "l "
+#define __pcpu_op_8(op)		op "q "
 
 #define __pcpu_reg_1(mod, x)	mod "q" (x)
 #define __pcpu_reg_2(mod, x)	mod "r" (x)
@@ -168,7 +153,8 @@ do {									\
 ({									\
 	__pcpu_type_##size pfo_val__;					\
 									\
-	asm qual (__pcpu_op2_##size("mov", __percpu_arg([var]), "%[val]") \
+	asm qual (__pcpu_op_##size("mov")				\
+		  __percpu_arg([var]) ", %[val]"			\
 	    : [val] __pcpu_reg_##size("=", pfo_val__)			\
 	    : [var] "m" (__my_cpu_var(_var)));				\
 									\
@@ -180,11 +166,12 @@ do {									\
 	__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val);	\
 									\
 	if (0) {		                                        \
-		typeof(_var) pto_tmp__;					\
+		TYPEOF_UNQUAL(_var) pto_tmp__;				\
 		pto_tmp__ = (_val);					\
 		(void)pto_tmp__;					\
 	}								\
-	asm qual(__pcpu_op2_##size("mov", "%[val]", __percpu_arg([var])) \
+	asm qual (__pcpu_op_##size("mov") "%[val], "			\
+		  __percpu_arg([var])					\
 	    : [var] "=m" (__my_cpu_var(_var))				\
 	    : [val] __pcpu_reg_imm_##size(pto_val__));			\
 } while (0)
@@ -201,7 +188,8 @@ do {									\
 ({									\
 	__pcpu_type_##size pfo_val__;					\
 									\
-	asm(__pcpu_op2_##size("mov", __force_percpu_arg(a[var]), "%[val]") \
+	asm(__pcpu_op_##size("mov")					\
+	    __force_percpu_arg(a[var]) ", %[val]"			\
 	    : [val] __pcpu_reg_##size("=", pfo_val__)			\
 	    : [var] "i" (&(_var)));					\
 									\
@@ -210,7 +198,7 @@ do {									\
 
 #define percpu_unary_op(size, qual, op, _var)				\
 ({									\
-	asm qual (__pcpu_op1_##size(op, __percpu_arg([var]))		\
+	asm qual (__pcpu_op_##size(op) __percpu_arg([var])		\
 	    : [var] "+m" (__my_cpu_var(_var)));				\
 })
 
@@ -219,11 +207,11 @@ do {									\
 	__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val);	\
 									\
 	if (0) {		                                        \
-		typeof(_var) pto_tmp__;					\
+		TYPEOF_UNQUAL(_var) pto_tmp__;				\
 		pto_tmp__ = (_val);					\
 		(void)pto_tmp__;					\
 	}								\
-	asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var]))	\
+	asm qual (__pcpu_op_##size(op) "%[val], " __percpu_arg([var])	\
 	    : [var] "+m" (__my_cpu_var(_var))				\
 	    : [val] __pcpu_reg_imm_##size(pto_val__));			\
 } while (0)
@@ -234,12 +222,13 @@ do {									\
  */
 #define percpu_add_op(size, qual, var, val)				\
 do {									\
-	const int pao_ID__ = (__builtin_constant_p(val) &&		\
-			      ((val) == 1 || (val) == -1)) ?		\
-				(int)(val) : 0;				\
+	const int pao_ID__ =						\
+		(__builtin_constant_p(val) &&				\
+			((val) == 1 ||					\
+			 (val) == (typeof(val))-1)) ? (int)(val) : 0;	\
 									\
 	if (0) {							\
-		typeof(var) pao_tmp__;					\
+		TYPEOF_UNQUAL(var) pao_tmp__;				\
 		pao_tmp__ = (val);					\
 		(void)pao_tmp__;					\
 	}								\
@@ -258,8 +247,8 @@ do {									\
 ({									\
 	__pcpu_type_##size paro_tmp__ = __pcpu_cast_##size(_val);	\
 									\
-	asm qual (__pcpu_op2_##size("xadd", "%[tmp]",			\
-				     __percpu_arg([var]))		\
+	asm qual (__pcpu_op_##size("xadd") "%[tmp], "			\
+		  __percpu_arg([var])					\
 		  : [tmp] __pcpu_reg_##size("+", paro_tmp__),		\
 		    [var] "+m" (__my_cpu_var(_var))			\
 		  : : "memory");					\
@@ -272,7 +261,7 @@ do {									\
  */
 #define raw_percpu_xchg_op(_var, _nval)					\
 ({									\
-	typeof(_var) pxo_old__ = raw_cpu_read(_var);			\
+	TYPEOF_UNQUAL(_var) pxo_old__ = raw_cpu_read(_var);		\
 									\
 	raw_cpu_write(_var, _nval);					\
 									\
@@ -286,7 +275,7 @@ do {									\
  */
 #define this_percpu_xchg_op(_var, _nval)				\
 ({									\
-	typeof(_var) pxo_old__ = this_cpu_read(_var);			\
+	TYPEOF_UNQUAL(_var) pxo_old__ = this_cpu_read(_var);		\
 									\
 	do { } while (!this_cpu_try_cmpxchg(_var, &pxo_old__, _nval));	\
 									\
@@ -302,8 +291,8 @@ do {									\
 	__pcpu_type_##size pco_old__ = __pcpu_cast_##size(_oval);	\
 	__pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval);	\
 									\
-	asm qual (__pcpu_op2_##size("cmpxchg", "%[nval]",		\
-				    __percpu_arg([var]))		\
+	asm qual (__pcpu_op_##size("cmpxchg") "%[nval], "		\
+		  __percpu_arg([var])					\
 		  : [oval] "+a" (pco_old__),				\
 		    [var] "+m" (__my_cpu_var(_var))			\
 		  : [nval] __pcpu_reg_##size(, pco_new__)		\
@@ -319,10 +308,9 @@ do {									\
 	__pcpu_type_##size pco_old__ = *pco_oval__;			\
 	__pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval);	\
 									\
-	asm qual (__pcpu_op2_##size("cmpxchg", "%[nval]",		\
-				    __percpu_arg([var]))		\
-		  CC_SET(z)						\
-		  : CC_OUT(z) (success),				\
+	asm qual (__pcpu_op_##size("cmpxchg") "%[nval], "		\
+		  __percpu_arg([var])					\
+		  : "=@ccz" (success),					\
 		    [oval] "+a" (pco_old__),				\
 		    [var] "+m" (__my_cpu_var(_var))			\
 		  : [nval] __pcpu_reg_##size(, pco_new__)		\
@@ -347,15 +335,14 @@ do {									\
 	old__.var = _oval;						\
 	new__.var = _nval;						\
 									\
-	asm qual (ALTERNATIVE("call this_cpu_cmpxchg8b_emu",		\
-			      "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
-		  : [var] "+m" (__my_cpu_var(_var)),			\
-		    "+a" (old__.low),					\
-		    "+d" (old__.high)					\
-		  : "b" (new__.low),					\
-		    "c" (new__.high),					\
-		    "S" (&(_var))					\
-		  : "memory");						\
+	asm_inline qual (						\
+		ALTERNATIVE("call this_cpu_cmpxchg8b_emu",		\
+			    "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
+		: ALT_OUTPUT_SP([var] "+m" (__my_cpu_var(_var)),	\
+				"+a" (old__.low), "+d" (old__.high))	\
+		: "b" (new__.low), "c" (new__.high),			\
+		  "S" (&(_var))						\
+		: "memory");						\
 									\
 	old__.var;							\
 })
@@ -377,17 +364,15 @@ do {									\
 	old__.var = *_oval;						\
 	new__.var = _nval;						\
 									\
-	asm qual (ALTERNATIVE("call this_cpu_cmpxchg8b_emu",		\
-			      "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
-		  CC_SET(z)						\
-		  : CC_OUT(z) (success),				\
-		    [var] "+m" (__my_cpu_var(_var)),			\
-		    "+a" (old__.low),					\
-		    "+d" (old__.high)					\
-		  : "b" (new__.low),					\
-		    "c" (new__.high),					\
-		    "S" (&(_var))					\
-		  : "memory");						\
+	asm_inline qual (						\
+		ALTERNATIVE("call this_cpu_cmpxchg8b_emu",		\
+			    "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
+		: ALT_OUTPUT_SP("=@ccz" (success),			\
+				[var] "+m" (__my_cpu_var(_var)),	\
+				"+a" (old__.low), "+d" (old__.high))	\
+		: "b" (new__.low), "c" (new__.high),			\
+		  "S" (&(_var))						\
+		: "memory");						\
 	if (unlikely(!success))						\
 		*_oval = old__.var;					\
 									\
@@ -418,15 +403,14 @@ do {									\
 	old__.var = _oval;						\
 	new__.var = _nval;						\
 									\
-	asm qual (ALTERNATIVE("call this_cpu_cmpxchg16b_emu",		\
-			      "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
-		  : [var] "+m" (__my_cpu_var(_var)),			\
-		    "+a" (old__.low),					\
-		    "+d" (old__.high)					\
-		  : "b" (new__.low),					\
-		    "c" (new__.high),					\
-		    "S" (&(_var))					\
-		  : "memory");						\
+	asm_inline qual (						\
+		ALTERNATIVE("call this_cpu_cmpxchg16b_emu",		\
+			    "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
+		: ALT_OUTPUT_SP([var] "+m" (__my_cpu_var(_var)),	\
+				"+a" (old__.low), "+d" (old__.high))	\
+		: "b" (new__.low), "c" (new__.high),			\
+		  "S" (&(_var))						\
+		: "memory");						\
 									\
 	old__.var;							\
 })
@@ -448,19 +432,18 @@ do {									\
 	old__.var = *_oval;						\
 	new__.var = _nval;						\
 									\
-	asm qual (ALTERNATIVE("call this_cpu_cmpxchg16b_emu",		\
-			      "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
-		  CC_SET(z)						\
-		  : CC_OUT(z) (success),				\
-		    [var] "+m" (__my_cpu_var(_var)),			\
-		    "+a" (old__.low),					\
-		    "+d" (old__.high)					\
-		  : "b" (new__.low),					\
-		    "c" (new__.high),					\
-		    "S" (&(_var))					\
-		  : "memory");						\
+	asm_inline qual (						\
+		ALTERNATIVE("call this_cpu_cmpxchg16b_emu",		\
+			    "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
+		: ALT_OUTPUT_SP("=@ccz" (success),			\
+				[var] "+m" (__my_cpu_var(_var)),	\
+				"+a" (old__.low), "+d" (old__.high))	\
+		: "b" (new__.low), "c" (new__.high),			\
+		  "S" (&(_var))						\
+		: "memory");						\
 	if (unlikely(!success))						\
 		*_oval = old__.var;					\
+									\
 	likely(success);						\
 })
 
@@ -581,7 +564,7 @@ do {									\
  * it is accessed while this_cpu_read_stable() allows the value to be cached.
  * this_cpu_read_stable() is more efficient and can be used if its value
  * is guaranteed to be valid across CPUs.  The current users include
- * pcpu_hot.current_task and pcpu_hot.top_of_stack, both of which are
+ * current_task and cpu_current_top_of_stack, both of which are
  * actually per-thread variables implemented as per-CPU variables and
  * thus stable for the duration of the respective task.
  */
@@ -590,9 +573,9 @@ do {									\
 #define x86_this_cpu_constant_test_bit(_nr, _var)			\
 ({									\
 	unsigned long __percpu *addr__ =				\
-		(unsigned long __percpu *)&(_var) + ((_nr) / BITS_PER_LONG); \
+		(unsigned long __percpu *)&(_var) + BIT_WORD(_nr);	\
 									\
-	!!((1UL << ((_nr) % BITS_PER_LONG)) & raw_cpu_read(*addr__));	\
+	!!(BIT_MASK(_nr) & raw_cpu_read(*addr__));			\
 })
 
 #define x86_this_cpu_variable_test_bit(_nr, _var)			\
@@ -600,8 +583,7 @@ do {									\
 	bool oldbit;							\
 									\
 	asm volatile("btl %[nr], " __percpu_arg([var])			\
-		     CC_SET(c)						\
-		     : CC_OUT(c) (oldbit)				\
+		     : "=@ccc" (oldbit)					\
 		     : [var] "m" (__my_cpu_var(_var)),			\
 		       [nr] "rI" (_nr));				\
 	oldbit;								\
@@ -616,9 +598,9 @@ do {									\
 #include <asm-generic/percpu.h>
 
 /* We can use this directly for local CPU (faster). */
-DECLARE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off);
+DECLARE_PER_CPU_CACHE_HOT(unsigned long, this_cpu_off);
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #ifdef CONFIG_SMP
 
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index 91b73571412f..7276ba70c88a 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -35,18 +35,23 @@
 #define ARCH_PERFMON_EVENTSEL_EQ			(1ULL << 36)
 #define ARCH_PERFMON_EVENTSEL_UMASK2			(0xFFULL << 40)
 
-#define INTEL_FIXED_BITS_MASK				0xFULL
 #define INTEL_FIXED_BITS_STRIDE			4
 #define INTEL_FIXED_0_KERNEL				(1ULL << 0)
 #define INTEL_FIXED_0_USER				(1ULL << 1)
 #define INTEL_FIXED_0_ANYTHREAD			(1ULL << 2)
 #define INTEL_FIXED_0_ENABLE_PMI			(1ULL << 3)
+#define INTEL_FIXED_3_METRICS_CLEAR			(1ULL << 2)
 
 #define HSW_IN_TX					(1ULL << 32)
 #define HSW_IN_TX_CHECKPOINTED				(1ULL << 33)
 #define ICL_EVENTSEL_ADAPTIVE				(1ULL << 34)
 #define ICL_FIXED_0_ADAPTIVE				(1ULL << 32)
 
+#define INTEL_FIXED_BITS_MASK					\
+	(INTEL_FIXED_0_KERNEL | INTEL_FIXED_0_USER |		\
+	 INTEL_FIXED_0_ANYTHREAD | INTEL_FIXED_0_ENABLE_PMI |	\
+	 ICL_FIXED_0_ADAPTIVE)
+
 #define intel_fixed_bits_by_idx(_idx, _bits)			\
 	((_bits) << ((_idx) * INTEL_FIXED_BITS_STRIDE))
 
@@ -136,10 +141,16 @@
 #define ARCH_PERFMON_EVENTS_COUNT			7
 
 #define PEBS_DATACFG_MEMINFO	BIT_ULL(0)
-#define PEBS_DATACFG_GP	BIT_ULL(1)
+#define PEBS_DATACFG_GP		BIT_ULL(1)
 #define PEBS_DATACFG_XMMS	BIT_ULL(2)
 #define PEBS_DATACFG_LBRS	BIT_ULL(3)
+#define PEBS_DATACFG_CNTR	BIT_ULL(4)
+#define PEBS_DATACFG_METRICS	BIT_ULL(5)
 #define PEBS_DATACFG_LBR_SHIFT	24
+#define PEBS_DATACFG_CNTR_SHIFT	32
+#define PEBS_DATACFG_CNTR_MASK	GENMASK_ULL(15, 0)
+#define PEBS_DATACFG_FIX_SHIFT	48
+#define PEBS_DATACFG_FIX_MASK	GENMASK_ULL(7, 0)
 
 /* Steal the highest bit of pebs_data_cfg for SW usage */
 #define PEBS_UPDATE_DS_SW	BIT_ULL(63)
@@ -187,10 +198,38 @@ union cpuid10_edx {
  * detection/enumeration details:
  */
 #define ARCH_PERFMON_EXT_LEAF			0x00000023
-#define ARCH_PERFMON_EXT_UMASK2			0x1
-#define ARCH_PERFMON_EXT_EQ			0x2
-#define ARCH_PERFMON_NUM_COUNTER_LEAF_BIT	0x1
 #define ARCH_PERFMON_NUM_COUNTER_LEAF		0x1
+#define ARCH_PERFMON_ACR_LEAF			0x2
+#define ARCH_PERFMON_PEBS_CAP_LEAF		0x4
+#define ARCH_PERFMON_PEBS_COUNTER_LEAF		0x5
+
+union cpuid35_eax {
+	struct {
+		unsigned int	leaf0:1;
+		/* Counters Sub-Leaf */
+		unsigned int    cntr_subleaf:1;
+		/* Auto Counter Reload Sub-Leaf */
+		unsigned int    acr_subleaf:1;
+		/* Events Sub-Leaf */
+		unsigned int    events_subleaf:1;
+		/* arch-PEBS Sub-Leaves */
+		unsigned int	pebs_caps_subleaf:1;
+		unsigned int	pebs_cnts_subleaf:1;
+		unsigned int	reserved:26;
+	} split;
+	unsigned int            full;
+};
+
+union cpuid35_ebx {
+	struct {
+		/* UnitMask2 Supported */
+		unsigned int    umask2:1;
+		/* EQ-bit Supported */
+		unsigned int    eq:1;
+		unsigned int	reserved:30;
+	} split;
+	unsigned int            full;
+};
 
 /*
  * Intel Architectural LBR CPUID detection/enumeration details:
@@ -372,6 +411,9 @@ static inline bool use_fixed_pseudo_encoding(u64 code)
 #define INTEL_TD_METRIC_MAX			INTEL_TD_METRIC_MEM_BOUND
 #define INTEL_TD_METRIC_NUM			8
 
+#define INTEL_TD_CFG_METRIC_CLEAR_BIT		0
+#define INTEL_TD_CFG_METRIC_CLEAR		BIT_ULL(INTEL_TD_CFG_METRIC_CLEAR_BIT)
+
 static inline bool is_metric_idx(int idx)
 {
 	return (unsigned)(idx - INTEL_PMC_IDX_METRIC_BASE) < INTEL_TD_METRIC_NUM;
@@ -395,9 +437,11 @@ static inline bool is_topdown_idx(int idx)
 #define GLOBAL_STATUS_LBRS_FROZEN		BIT_ULL(GLOBAL_STATUS_LBRS_FROZEN_BIT)
 #define GLOBAL_STATUS_TRACE_TOPAPMI_BIT		55
 #define GLOBAL_STATUS_TRACE_TOPAPMI		BIT_ULL(GLOBAL_STATUS_TRACE_TOPAPMI_BIT)
+#define GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT	54
+#define GLOBAL_STATUS_ARCH_PEBS_THRESHOLD	BIT_ULL(GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT)
 #define GLOBAL_STATUS_PERF_METRICS_OVF_BIT	48
 
-#define GLOBAL_CTRL_EN_PERF_METRICS		48
+#define GLOBAL_CTRL_EN_PERF_METRICS		BIT_ULL(48)
 /*
  * We model guest LBR event tracing as another fixed-mode PMC like BTS.
  *
@@ -422,7 +466,9 @@ static inline bool is_topdown_idx(int idx)
  */
 
 struct pebs_basic {
-	u64 format_size;
+	u64 format_group:32,
+	    retire_latency:16,
+	    format_size:16;
 	u64 ip;
 	u64 applicable_counters;
 	u64 tsc;
@@ -431,7 +477,17 @@ struct pebs_basic {
 struct pebs_meminfo {
 	u64 address;
 	u64 aux;
-	u64 latency;
+	union {
+		/* pre Alder Lake */
+		u64 mem_latency;
+		/* Alder Lake and later */
+		struct {
+			u64 instr_latency:16;
+			u64 pad2:16;
+			u64 cache_latency:16;
+			u64 pad3:16;
+		};
+	};
 	u64 tsx_tuning;
 };
 
@@ -444,6 +500,116 @@ struct pebs_xmm {
 	u64 xmm[16*2];	/* two entries for each register */
 };
 
+struct pebs_cntr_header {
+	u32 cntr;
+	u32 fixed;
+	u32 metrics;
+	u32 reserved;
+};
+
+#define INTEL_CNTR_METRICS		0x3
+
+/*
+ * Arch PEBS
+ */
+union arch_pebs_index {
+	struct {
+		u64 rsvd:4,
+		    wr:23,
+		    rsvd2:4,
+		    full:1,
+		    en:1,
+		    rsvd3:3,
+		    thresh:23,
+		    rsvd4:5;
+	};
+	u64 whole;
+};
+
+struct arch_pebs_header {
+	union {
+		u64 format;
+		struct {
+			u64 size:16,	/* Record size */
+			    rsvd:14,
+			    mode:1,	/* 64BIT_MODE */
+			    cont:1,
+			    rsvd2:3,
+			    cntr:5,
+			    lbr:2,
+			    rsvd3:7,
+			    xmm:1,
+			    ymmh:1,
+			    rsvd4:2,
+			    opmask:1,
+			    zmmh:1,
+			    h16zmm:1,
+			    rsvd5:5,
+			    gpr:1,
+			    aux:1,
+			    basic:1;
+		};
+	};
+	u64 rsvd6;
+};
+
+struct arch_pebs_basic {
+	u64 ip;
+	u64 applicable_counters;
+	u64 tsc;
+	u64 retire	:16,	/* Retire Latency */
+	    valid	:1,
+	    rsvd	:47;
+	u64 rsvd2;
+	u64 rsvd3;
+};
+
+struct arch_pebs_aux {
+	u64 address;
+	u64 rsvd;
+	u64 rsvd2;
+	u64 rsvd3;
+	u64 rsvd4;
+	u64 aux;
+	u64 instr_latency	:16,
+	    pad2		:16,
+	    cache_latency	:16,
+	    pad3		:16;
+	u64 tsx_tuning;
+};
+
+struct arch_pebs_gprs {
+	u64 flags, ip, ax, cx, dx, bx, sp, bp, si, di;
+	u64 r8, r9, r10, r11, r12, r13, r14, r15, ssp;
+	u64 rsvd;
+};
+
+struct arch_pebs_xer_header {
+	u64 xstate;
+	u64 rsvd;
+};
+
+#define ARCH_PEBS_LBR_NAN		0x0
+#define ARCH_PEBS_LBR_NUM_8		0x1
+#define ARCH_PEBS_LBR_NUM_16		0x2
+#define ARCH_PEBS_LBR_NUM_VAR		0x3
+#define ARCH_PEBS_BASE_LBR_ENTRIES	8
+struct arch_pebs_lbr_header {
+	u64 rsvd;
+	u64 ctl;
+	u64 depth;
+	u64 ler_from;
+	u64 ler_to;
+	u64 ler_info;
+};
+
+struct arch_pebs_cntr_header {
+	u32 cntr;
+	u32 fixed;
+	u32 metrics;
+	u32 reserved;
+};
+
 /*
  * AMD Extended Performance Monitoring and Debug cpuid feature detection
  */
@@ -471,6 +637,8 @@ struct pebs_xmm {
 #define IBS_CAPS_FETCHCTLEXTD		(1U<<9)
 #define IBS_CAPS_OPDATA4		(1U<<10)
 #define IBS_CAPS_ZEN4			(1U<<11)
+#define IBS_CAPS_OPLDLAT		(1U<<12)
+#define IBS_CAPS_OPDTLBPGSIZE		(1U<<19)
 
 #define IBS_CAPS_DEFAULT		(IBS_CAPS_AVAIL		\
 					 | IBS_CAPS_FETCHSAM	\
@@ -496,8 +664,11 @@ struct pebs_xmm {
  * The lower 7 bits of the current count are random bits
  * preloaded by hardware and ignored in software
  */
+#define IBS_OP_LDLAT_EN		(1ULL<<63)
+#define IBS_OP_LDLAT_THRSH	(0xFULL<<59)
 #define IBS_OP_CUR_CNT		(0xFFF80ULL<<32)
 #define IBS_OP_CUR_CNT_RAND	(0x0007FULL<<32)
+#define IBS_OP_CUR_CNT_EXT_MASK	(0x7FULL<<52)
 #define IBS_OP_CNT_CTL		(1ULL<<19)
 #define IBS_OP_VAL		(1ULL<<18)
 #define IBS_OP_ENABLE		(1ULL<<17)
@@ -536,15 +707,17 @@ struct x86_perf_regs {
 	u64		*xmm_regs;
 };
 
-extern unsigned long perf_instruction_pointer(struct pt_regs *regs);
-extern unsigned long perf_misc_flags(struct pt_regs *regs);
-#define perf_misc_flags(regs)	perf_misc_flags(regs)
+extern unsigned long perf_arch_instruction_pointer(struct pt_regs *regs);
+extern unsigned long perf_arch_misc_flags(struct pt_regs *regs);
+extern unsigned long perf_arch_guest_misc_flags(struct pt_regs *regs);
+#define perf_arch_misc_flags(regs)	perf_arch_misc_flags(regs)
+#define perf_arch_guest_misc_flags(regs)	perf_arch_guest_misc_flags(regs)
 
 #include <asm/stacktrace.h>
 
 /*
- * We abuse bit 3 from flags to pass exact information, see perf_misc_flags
- * and the comment with PERF_EFLAGS_EXACT.
+ * We abuse bit 3 from flags to pass exact information, see
+ * perf_arch_misc_flags() and the comment with PERF_EFLAGS_EXACT.
  */
 #define perf_arch_fetch_caller_regs(regs, __ip)		{	\
 	(regs)->ip = (__ip);					\
diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h
index dcd836b59beb..c88691b15f3c 100644
--- a/arch/x86/include/asm/pgalloc.h
+++ b/arch/x86/include/asm/pgalloc.h
@@ -6,6 +6,8 @@
 #include <linux/mm.h>		/* for struct page */
 #include <linux/pagemap.h>
 
+#include <asm/cpufeature.h>
+
 #define __HAVE_ARCH_PTE_ALLOC_ONE
 #define __HAVE_ARCH_PGD_FREE
 #include <asm-generic/pgalloc.h>
@@ -30,20 +32,16 @@ static inline void paravirt_release_p4d(unsigned long pfn) {}
 #endif
 
 /*
- * Flags to use when allocating a user page table page.
+ * In case of Page Table Isolation active, we acquire two PGDs instead of one.
+ * Being order-1, it is both 8k in size and 8k-aligned.  That lets us just
+ * flip bit 12 in a pointer to swap between the two 4k halves.
  */
-extern gfp_t __userpte_alloc_gfp;
-
-#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
-/*
- * Instead of one PGD, we acquire two PGDs.  Being order-1, it is
- * both 8k in size and 8k-aligned.  That lets us just flip bit 12
- * in a pointer to swap between the two 4k halves.
- */
-#define PGD_ALLOCATION_ORDER 1
-#else
-#define PGD_ALLOCATION_ORDER 0
-#endif
+static inline unsigned int pgd_allocation_order(void)
+{
+	if (cpu_feature_enabled(X86_FEATURE_PTI))
+		return 1;
+	return 0;
+}
 
 /*
  * Allocate and free page tables.
@@ -147,24 +145,6 @@ static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4
 	set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
 }
 
-static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
-{
-	gfp_t gfp = GFP_KERNEL_ACCOUNT;
-
-	if (mm == &init_mm)
-		gfp &= ~__GFP_ACCOUNT;
-	return (p4d_t *)get_zeroed_page(gfp);
-}
-
-static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
-{
-	if (!pgtable_l5_enabled())
-		return;
-
-	BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));
-	free_page((unsigned long)p4d);
-}
-
 extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d);
 
 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
diff --git a/arch/x86/include/asm/pgtable-2level_types.h b/arch/x86/include/asm/pgtable-2level_types.h
index 7f6ccff0ba72..54690bd4ddbe 100644
--- a/arch/x86/include/asm/pgtable-2level_types.h
+++ b/arch/x86/include/asm/pgtable-2level_types.h
@@ -2,7 +2,7 @@
 #ifndef _ASM_X86_PGTABLE_2LEVEL_DEFS_H
 #define _ASM_X86_PGTABLE_2LEVEL_DEFS_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/types.h>
 
 typedef unsigned long	pteval_t;
@@ -16,24 +16,22 @@ typedef union {
 	pteval_t pte;
 	pteval_t pte_low;
 } pte_t;
-#endif	/* !__ASSEMBLY__ */
-
-#define SHARED_KERNEL_PMD	0
+#endif	/* !__ASSEMBLER__ */
 
 #define ARCH_PAGE_TABLE_SYNC_MASK	PGTBL_PMD_MODIFIED
 
 /*
- * traditional i386 two-level paging structure:
+ * Traditional i386 two-level paging structure:
  */
 
 #define PGDIR_SHIFT	22
 #define PTRS_PER_PGD	1024
 
-
 /*
- * the i386 is two-level, so we don't really have any
- * PMD directory physically.
+ * The i386 is two-level, so we don't really have any
+ * PMD directory physically:
  */
+#define PTRS_PER_PMD	1
 
 #define PTRS_PER_PTE	1024
 
diff --git a/arch/x86/include/asm/pgtable-3level_types.h b/arch/x86/include/asm/pgtable-3level_types.h
index 80911349519e..580b09bf6a45 100644
--- a/arch/x86/include/asm/pgtable-3level_types.h
+++ b/arch/x86/include/asm/pgtable-3level_types.h
@@ -2,7 +2,7 @@
 #ifndef _ASM_X86_PGTABLE_3LEVEL_DEFS_H
 #define _ASM_X86_PGTABLE_3LEVEL_DEFS_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/types.h>
 
 typedef u64	pteval_t;
@@ -25,11 +25,9 @@ typedef union {
 	};
 	pmdval_t pmd;
 } pmd_t;
-#endif	/* !__ASSEMBLY__ */
+#endif	/* !__ASSEMBLER__ */
 
-#define SHARED_KERNEL_PMD	(!static_cpu_has(X86_FEATURE_PTI))
-
-#define ARCH_PAGE_TABLE_SYNC_MASK	(SHARED_KERNEL_PMD ? 0 : PGTBL_PMD_MODIFIED)
+#define ARCH_PAGE_TABLE_SYNC_MASK	PGTBL_PMD_MODIFIED
 
 /*
  * PGDIR_SHIFT determines what a top-level page table entry can map
diff --git a/arch/x86/include/asm/pgtable-invert.h b/arch/x86/include/asm/pgtable-invert.h
index a0c1525f1b6f..e12e52ae8083 100644
--- a/arch/x86/include/asm/pgtable-invert.h
+++ b/arch/x86/include/asm/pgtable-invert.h
@@ -2,7 +2,7 @@
 #ifndef _ASM_PGTABLE_INVERT_H
 #define _ASM_PGTABLE_INVERT_H 1
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 /*
  * A clear pte value is special, and doesn't get inverted.
@@ -36,6 +36,6 @@ static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask)
 	return val;
 }
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 4c2d080d26b4..e33df3da6980 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -15,7 +15,7 @@
 		     cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS)))	\
 	 : (prot))
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/spinlock.h>
 #include <asm/x86_init.h>
 #include <asm/pkru.h>
@@ -292,13 +292,6 @@ static inline unsigned long pgd_pfn(pgd_t pgd)
 	return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT;
 }
 
-#define p4d_leaf p4d_leaf
-static inline bool p4d_leaf(p4d_t p4d)
-{
-	/* No 512 GiB pages yet */
-	return 0;
-}
-
 #define pte_page(pte)	pfn_to_page(pte_pfn(pte))
 
 #define pmd_leaf pmd_leaf
@@ -308,16 +301,15 @@ static inline bool pmd_leaf(pmd_t pte)
 }
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-/* NOTE: when predicate huge page, consider also pmd_devmap, or use pmd_leaf */
 static inline int pmd_trans_huge(pmd_t pmd)
 {
-	return (pmd_val(pmd) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
+	return (pmd_val(pmd) & _PAGE_PSE) == _PAGE_PSE;
 }
 
 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
 static inline int pud_trans_huge(pud_t pud)
 {
-	return (pud_val(pud) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
+	return (pud_val(pud) & _PAGE_PSE) == _PAGE_PSE;
 }
 #endif
 
@@ -327,24 +319,6 @@ static inline int has_transparent_hugepage(void)
 	return boot_cpu_has(X86_FEATURE_PSE);
 }
 
-#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP
-static inline int pmd_devmap(pmd_t pmd)
-{
-	return !!(pmd_val(pmd) & _PAGE_DEVMAP);
-}
-
-#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
-static inline int pud_devmap(pud_t pud)
-{
-	return !!(pud_val(pud) & _PAGE_DEVMAP);
-}
-#else
-static inline int pud_devmap(pud_t pud)
-{
-	return 0;
-}
-#endif
-
 #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
 static inline bool pmd_special(pmd_t pmd)
 {
@@ -368,12 +342,6 @@ static inline pud_t pud_mkspecial(pud_t pud)
 	return pud_set_flags(pud, _PAGE_SPECIAL);
 }
 #endif	/* CONFIG_ARCH_SUPPORTS_PUD_PFNMAP */
-
-static inline int pgd_devmap(pgd_t pgd)
-{
-	return 0;
-}
-#endif
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
 static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
@@ -534,11 +502,6 @@ static inline pte_t pte_mkspecial(pte_t pte)
 	return pte_set_flags(pte, _PAGE_SPECIAL);
 }
 
-static inline pte_t pte_mkdevmap(pte_t pte)
-{
-	return pte_set_flags(pte, _PAGE_SPECIAL|_PAGE_DEVMAP);
-}
-
 /* See comments above mksaveddirty_shift() */
 static inline pmd_t pmd_mksaveddirty(pmd_t pmd)
 {
@@ -610,11 +573,6 @@ static inline pmd_t pmd_mkwrite_shstk(pmd_t pmd)
 	return pmd_set_flags(pmd, _PAGE_DIRTY);
 }
 
-static inline pmd_t pmd_mkdevmap(pmd_t pmd)
-{
-	return pmd_set_flags(pmd, _PAGE_DEVMAP);
-}
-
 static inline pmd_t pmd_mkhuge(pmd_t pmd)
 {
 	return pmd_set_flags(pmd, _PAGE_PSE);
@@ -680,11 +638,6 @@ static inline pud_t pud_mkdirty(pud_t pud)
 	return pud_mksaveddirty(pud);
 }
 
-static inline pud_t pud_mkdevmap(pud_t pud)
-{
-	return pud_set_flags(pud, _PAGE_DEVMAP);
-}
-
 static inline pud_t pud_mkhuge(pud_t pud)
 {
 	return pud_set_flags(pud, _PAGE_PSE);
@@ -784,6 +737,9 @@ static inline pgprotval_t check_pgprot(pgprot_t pgprot)
 static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
 {
 	phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
+	/* This bit combination is used to mark shadow stacks */
+	WARN_ON_ONCE((pgprot_val(pgprot) & (_PAGE_DIRTY | _PAGE_RW)) ==
+			_PAGE_DIRTY);
 	pfn ^= protnone_mask(pgprot_val(pgprot));
 	pfn &= PTE_PFN_MASK;
 	return __pte(pfn | check_pgprot(pgprot));
@@ -973,7 +929,7 @@ static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
 }
 #endif  /* CONFIG_MITIGATION_PAGE_TABLE_ISOLATION */
 
-#endif	/* __ASSEMBLY__ */
+#endif	/* __ASSEMBLER__ */
 
 
 #ifdef CONFIG_X86_32
@@ -982,7 +938,7 @@ static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
 # include <asm/pgtable_64.h>
 #endif
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/mm_types.h>
 #include <linux/mmdebug.h>
 #include <linux/log2.h>
@@ -1012,13 +968,6 @@ static inline int pte_present(pte_t a)
 	return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
 }
 
-#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP
-static inline int pte_devmap(pte_t a)
-{
-	return (pte_flags(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP;
-}
-#endif
-
 #define pte_accessible pte_accessible
 static inline bool pte_accessible(struct mm_struct *mm, pte_t a)
 {
@@ -1080,22 +1029,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
  */
 #define pmd_page(pmd)	pfn_to_page(pmd_pfn(pmd))
 
-/*
- * Conversion functions: convert a page and protection to a page entry,
- * and a page entry and page directory to the page they refer to.
- *
- * (Currently stuck as a macro because of indirect forward reference
- * to linux/mm.h:page_to_nid())
- */
-#define mk_pte(page, pgprot)						  \
-({									  \
-	pgprot_t __pgprot = pgprot;					  \
-									  \
-	WARN_ON_ONCE((pgprot_val(__pgprot) & (_PAGE_DIRTY | _PAGE_RW)) == \
-		    _PAGE_DIRTY);					  \
-	pfn_pte(page_to_pfn(page), __pgprot);				  \
-})
-
 static inline int pmd_bad(pmd_t pmd)
 {
 	return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) !=
@@ -1233,12 +1166,12 @@ static inline int pgd_none(pgd_t pgd)
 }
 #endif	/* CONFIG_PGTABLE_LEVELS > 4 */
 
-#endif	/* __ASSEMBLY__ */
+#endif	/* __ASSEMBLER__ */
 
 #define KERNEL_PGD_BOUNDARY	pgd_index(PAGE_OFFSET)
 #define KERNEL_PGD_PTRS		(PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 extern int direct_gbpages;
 void init_mem_mapping(void);
@@ -1360,8 +1293,6 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm,
 
 #define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0)
 
-#define mk_pmd(page, pgprot)   pfn_pmd(page_to_pfn(page), (pgprot))
-
 #define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
 extern int pmdp_set_access_flags(struct vm_area_struct *vma,
 				 unsigned long address, pmd_t *pmdp,
@@ -1472,9 +1403,6 @@ static inline bool pgdp_maps_userspace(void *__ptr)
 	return (((ptr & ~PAGE_MASK) / sizeof(pgd_t)) < PGD_KERNEL_START);
 }
 
-#define pgd_leaf	pgd_leaf
-static inline bool pgd_leaf(pgd_t pgd) { return false; }
-
 #ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
 /*
  * All top-level MITIGATION_PAGE_TABLE_ISOLATION page tables are order-1 pages
@@ -1586,7 +1514,7 @@ static inline pte_t pte_swp_mkexclusive(pte_t pte)
 	return pte_set_flags(pte, _PAGE_SWP_EXCLUSIVE);
 }
 
-static inline int pte_swp_exclusive(pte_t pte)
+static inline bool pte_swp_exclusive(pte_t pte)
 {
 	return pte_flags(pte) & _PAGE_SWP_EXCLUSIVE;
 }
@@ -1775,6 +1703,43 @@ bool arch_is_platform_page(u64 paddr);
 #define arch_is_platform_page arch_is_platform_page
 #endif
 
-#endif	/* __ASSEMBLY__ */
+/*
+ * Use set_p*_safe(), and elide TLB flushing, when confident that *no*
+ * TLB flush will be required as a result of the "set". For example, use
+ * in scenarios where it is known ahead of time that the routine is
+ * setting non-present entries, or re-setting an existing entry to the
+ * same value. Otherwise, use the typical "set" helpers and flush the
+ * TLB.
+ */
+#define set_pte_safe(ptep, pte) \
+({ \
+	WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \
+	set_pte(ptep, pte); \
+})
+
+#define set_pmd_safe(pmdp, pmd) \
+({ \
+	WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \
+	set_pmd(pmdp, pmd); \
+})
+
+#define set_pud_safe(pudp, pud) \
+({ \
+	WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \
+	set_pud(pudp, pud); \
+})
+
+#define set_p4d_safe(p4dp, p4d) \
+({ \
+	WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \
+	set_p4d(p4dp, p4d); \
+})
+
+#define set_pgd_safe(pgdp, pgd) \
+({ \
+	WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \
+	set_pgd(pgdp, pgd); \
+})
+#endif	/* __ASSEMBLER__ */
 
 #endif /* _ASM_X86_PGTABLE_H */
diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h
index 7d4ad8907297..b612cc57a4d3 100644
--- a/arch/x86/include/asm/pgtable_32.h
+++ b/arch/x86/include/asm/pgtable_32.h
@@ -13,7 +13,7 @@
  * This file contains the functions and defines necessary to modify and use
  * the i386 page table tree.
  */
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <asm/processor.h>
 #include <linux/threads.h>
 #include <asm/paravirt.h>
@@ -45,7 +45,7 @@ do {						\
 	flush_tlb_one_kernel((vaddr));		\
 } while (0)
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 /*
  * This is used to calculate the .brk reservation for initial pagetables.
diff --git a/arch/x86/include/asm/pgtable_32_areas.h b/arch/x86/include/asm/pgtable_32_areas.h
index b6355416a15a..921148b42967 100644
--- a/arch/x86/include/asm/pgtable_32_areas.h
+++ b/arch/x86/include/asm/pgtable_32_areas.h
@@ -13,7 +13,7 @@
  */
 #define VMALLOC_OFFSET	(8 * 1024 * 1024)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 extern bool __vmalloc_start_set; /* set once high_memory is set */
 #endif
 
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index d1426b64c1b9..f06e5d6a2747 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -5,7 +5,7 @@
 #include <linux/const.h>
 #include <asm/pgtable_64_types.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 /*
  * This file contains the functions and defines necessary to modify and use
@@ -41,11 +41,9 @@ static inline void sync_initial_page_table(void) { }
 	pr_err("%s:%d: bad pud %p(%016lx)\n",		\
 	       __FILE__, __LINE__, &(e), pud_val(e))
 
-#if CONFIG_PGTABLE_LEVELS >= 5
 #define p4d_ERROR(e)					\
 	pr_err("%s:%d: bad p4d %p(%016lx)\n",		\
 	       __FILE__, __LINE__, &(e), p4d_val(e))
-#endif
 
 #define pgd_ERROR(e)					\
 	pr_err("%s:%d: bad pgd %p(%016lx)\n",		\
@@ -270,7 +268,7 @@ static inline bool gup_fast_permitted(unsigned long start, unsigned long end)
 
 #include <asm/pgtable-invert.h>
 
-#else /* __ASSEMBLY__ */
+#else /* __ASSEMBLER__ */
 
 #define l4_index(x)	(((x) >> 39) & 511)
 #define pud_index(x)	(((x) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
@@ -291,5 +289,5 @@ L3_START_KERNEL = pud_index(__START_KERNEL_map)
 	i = i + 1 ;					\
 	.endr
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* _ASM_X86_PGTABLE_64_H */
diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h
index a98e53491a4e..7eb61ef6a185 100644
--- a/arch/x86/include/asm/pgtable_64_types.h
+++ b/arch/x86/include/asm/pgtable_64_types.h
@@ -4,7 +4,7 @@
 
 #include <asm/sparsemem.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/types.h>
 #include <asm/kaslr.h>
 
@@ -23,7 +23,6 @@ typedef struct { pmdval_t pmd; } pmd_t;
 
 extern unsigned int __pgtable_l5_enabled;
 
-#ifdef CONFIG_X86_5LEVEL
 #ifdef USE_EARLY_PGTABLE_L5
 /*
  * cpu_feature_enabled() is not available in early boot code.
@@ -37,18 +36,13 @@ static inline bool pgtable_l5_enabled(void)
 #define pgtable_l5_enabled() cpu_feature_enabled(X86_FEATURE_LA57)
 #endif /* USE_EARLY_PGTABLE_L5 */
 
-#else
-#define pgtable_l5_enabled() 0
-#endif /* CONFIG_X86_5LEVEL */
+#define ARCH_PAGE_TABLE_SYNC_MASK \
+	(pgtable_l5_enabled() ? PGTBL_PGD_MODIFIED : PGTBL_P4D_MODIFIED)
 
 extern unsigned int pgdir_shift;
 extern unsigned int ptrs_per_p4d;
 
-#endif	/* !__ASSEMBLY__ */
-
-#define SHARED_KERNEL_PMD	0
-
-#ifdef CONFIG_X86_5LEVEL
+#endif	/* !__ASSEMBLER__ */
 
 /*
  * PGDIR_SHIFT determines what a top-level page table entry can map
@@ -67,17 +61,6 @@ extern unsigned int ptrs_per_p4d;
 
 #define MAX_POSSIBLE_PHYSMEM_BITS	52
 
-#else /* CONFIG_X86_5LEVEL */
-
-/*
- * PGDIR_SHIFT determines what a top-level page table entry can map
- */
-#define PGDIR_SHIFT		39
-#define PTRS_PER_PGD		512
-#define MAX_PTRS_PER_P4D	1
-
-#endif /* CONFIG_X86_5LEVEL */
-
 /*
  * 3rd level page
  */
@@ -130,18 +113,12 @@ extern unsigned int ptrs_per_p4d;
 #define __VMEMMAP_BASE_L4	0xffffea0000000000UL
 #define __VMEMMAP_BASE_L5	0xffd4000000000000UL
 
-#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
 # define VMALLOC_START		vmalloc_base
 # define VMALLOC_SIZE_TB	(pgtable_l5_enabled() ? VMALLOC_SIZE_TB_L5 : VMALLOC_SIZE_TB_L4)
 # define VMEMMAP_START		vmemmap_base
-#else
-# define VMALLOC_START		__VMALLOC_BASE_L4
-# define VMALLOC_SIZE_TB	VMALLOC_SIZE_TB_L4
-# define VMEMMAP_START		__VMEMMAP_BASE_L4
-#endif /* CONFIG_DYNAMIC_MEMORY_LAYOUT */
 
 #ifdef CONFIG_RANDOMIZE_MEMORY
-# define PHYSMEM_END		physmem_end
+# define DIRECT_MAP_PHYSMEM_END	direct_map_physmem_end
 #endif
 
 /*
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index 6f82e75b6149..2ec250ba467e 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -33,13 +33,15 @@
 #define _PAGE_BIT_CPA_TEST	_PAGE_BIT_SOFTW1
 #define _PAGE_BIT_UFFD_WP	_PAGE_BIT_SOFTW2 /* userfaultfd wrprotected */
 #define _PAGE_BIT_SOFT_DIRTY	_PAGE_BIT_SOFTW3 /* software dirty tracking */
-#define _PAGE_BIT_DEVMAP	_PAGE_BIT_SOFTW4
+#define _PAGE_BIT_KERNEL_4K	_PAGE_BIT_SOFTW3 /* page must not be converted to large */
 
 #ifdef CONFIG_X86_64
-#define _PAGE_BIT_SAVED_DIRTY	_PAGE_BIT_SOFTW5 /* Saved Dirty bit */
+#define _PAGE_BIT_SAVED_DIRTY	_PAGE_BIT_SOFTW5 /* Saved Dirty bit (leaf) */
+#define _PAGE_BIT_NOPTISHADOW	_PAGE_BIT_SOFTW5 /* No PTI shadow (root PGD) */
 #else
 /* Shared with _PAGE_BIT_UFFD_WP which is not supported on 32 bit */
-#define _PAGE_BIT_SAVED_DIRTY	_PAGE_BIT_SOFTW2 /* Saved Dirty bit */
+#define _PAGE_BIT_SAVED_DIRTY	_PAGE_BIT_SOFTW2 /* Saved Dirty bit (leaf) */
+#define _PAGE_BIT_NOPTISHADOW	_PAGE_BIT_SOFTW2 /* No PTI shadow (root PGD) */
 #endif
 
 /* If _PAGE_BIT_PRESENT is clear, we use these: */
@@ -62,6 +64,7 @@
 #define _PAGE_PAT_LARGE (_AT(pteval_t, 1) << _PAGE_BIT_PAT_LARGE)
 #define _PAGE_SPECIAL	(_AT(pteval_t, 1) << _PAGE_BIT_SPECIAL)
 #define _PAGE_CPA_TEST	(_AT(pteval_t, 1) << _PAGE_BIT_CPA_TEST)
+#define _PAGE_KERNEL_4K	(_AT(pteval_t, 1) << _PAGE_BIT_KERNEL_4K)
 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
 #define _PAGE_PKEY_BIT0	(_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT0)
 #define _PAGE_PKEY_BIT1	(_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT1)
@@ -117,11 +120,9 @@
 
 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 #define _PAGE_NX	(_AT(pteval_t, 1) << _PAGE_BIT_NX)
-#define _PAGE_DEVMAP	(_AT(u64, 1) << _PAGE_BIT_DEVMAP)
 #define _PAGE_SOFTW4	(_AT(pteval_t, 1) << _PAGE_BIT_SOFTW4)
 #else
 #define _PAGE_NX	(_AT(pteval_t, 0))
-#define _PAGE_DEVMAP	(_AT(pteval_t, 0))
 #define _PAGE_SOFTW4	(_AT(pteval_t, 0))
 #endif
 
@@ -139,6 +140,8 @@
 
 #define _PAGE_PROTNONE	(_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE)
 
+#define _PAGE_NOPTISHADOW (_AT(pteval_t, 1) << _PAGE_BIT_NOPTISHADOW)
+
 /*
  * Set of bits not changed in pte_modify.  The pte's
  * protection key is treated like _PAGE_RW, for
@@ -148,7 +151,7 @@
 #define _COMMON_PAGE_CHG_MASK	(PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT |	\
 				 _PAGE_SPECIAL | _PAGE_ACCESSED |	\
 				 _PAGE_DIRTY_BITS | _PAGE_SOFT_DIRTY |	\
-				 _PAGE_DEVMAP | _PAGE_CC | _PAGE_UFFD_WP)
+				 _PAGE_CC | _PAGE_UFFD_WP)
 #define _PAGE_CHG_MASK	(_COMMON_PAGE_CHG_MASK | _PAGE_PAT)
 #define _HPAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PAT_LARGE)
 
@@ -160,7 +163,7 @@
  * to have the WB mode at index 0 (all bits clear). This is the default
  * right now and likely would break too much if changed.
  */
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 enum page_cache_mode {
 	_PAGE_CACHE_MODE_WB       = 0,
 	_PAGE_CACHE_MODE_WC       = 1,
@@ -173,7 +176,7 @@ enum page_cache_mode {
 };
 #endif
 
-#define _PAGE_CC		(_AT(pteval_t, cc_mask))
+#define _PAGE_CC		(_AT(pteval_t, cc_get_mask()))
 #define _PAGE_ENC		(_AT(pteval_t, sme_me_mask))
 
 #define _PAGE_CACHE_MASK	(_PAGE_PWT | _PAGE_PCD | _PAGE_PAT)
@@ -208,9 +211,6 @@ enum page_cache_mode {
 #define PAGE_READONLY	     __pg(__PP|   0|_USR|___A|__NX|   0|   0|   0)
 #define PAGE_READONLY_EXEC   __pg(__PP|   0|_USR|___A|   0|   0|   0|   0)
 
-#define __PAGE_KERNEL		 (__PP|__RW|   0|___A|__NX|___D|   0|___G)
-#define __PAGE_KERNEL_EXEC	 (__PP|__RW|   0|___A|   0|___D|   0|___G)
-
 /*
  * Page tables needs to have Write=1 in order for any lower PTEs to be
  * writable. This includes shadow stack memory (Write=0, Dirty=1)
@@ -235,7 +235,7 @@ enum page_cache_mode {
 #define __PAGE_KERNEL_IO_NOCACHE	__PAGE_KERNEL_NOCACHE
 
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #define __PAGE_KERNEL_ENC	(__PAGE_KERNEL    | _ENC)
 #define __PAGE_KERNEL_ENC_WP	(__PAGE_KERNEL_WP | _ENC)
@@ -258,7 +258,7 @@ enum page_cache_mode {
 #define PAGE_KERNEL_IO		__pgprot_mask(__PAGE_KERNEL_IO)
 #define PAGE_KERNEL_IO_NOCACHE	__pgprot_mask(__PAGE_KERNEL_IO_NOCACHE)
 
-#endif	/* __ASSEMBLY__ */
+#endif	/* __ASSEMBLER__ */
 
 /*
  * early identity mapping  pte attrib macros.
@@ -277,7 +277,7 @@ enum page_cache_mode {
 # include <asm/pgtable_64_types.h>
 #endif
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/types.h>
 
@@ -576,6 +576,6 @@ extern int __init kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn,
 					  unsigned long page_flags);
 extern int __init kernel_unmap_pages_in_pgd(pgd_t *pgd, unsigned long address,
 					    unsigned long numpages);
-#endif	/* !__ASSEMBLY__ */
+#endif	/* !__ASSEMBLER__ */
 
 #endif /* _ASM_X86_PGTABLE_DEFS_H */
diff --git a/arch/x86/include/asm/posted_intr.h b/arch/x86/include/asm/posted_intr.h
index de788b400fba..a5f761fbf45b 100644
--- a/arch/x86/include/asm/posted_intr.h
+++ b/arch/x86/include/asm/posted_intr.h
@@ -1,19 +1,24 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #ifndef _X86_POSTED_INTR_H
 #define _X86_POSTED_INTR_H
+
+#include <asm/cmpxchg.h>
+#include <asm/rwonce.h>
 #include <asm/irq_vectors.h>
 
+#include <linux/bitmap.h>
+
 #define POSTED_INTR_ON  0
 #define POSTED_INTR_SN  1
 
 #define PID_TABLE_ENTRY_VALID 1
 
+#define NR_PIR_VECTORS	256
+#define NR_PIR_WORDS	(NR_PIR_VECTORS / BITS_PER_LONG)
+
 /* Posted-Interrupt Descriptor */
 struct pi_desc {
-	union {
-		u32 pir[8];     /* Posted interrupt requested */
-		u64 pir64[4];
-	};
+	unsigned long pir[NR_PIR_WORDS];     /* Posted interrupt requested */
 	union {
 		struct {
 			u16	notifications; /* Suppress and outstanding bits */
@@ -26,6 +31,65 @@ struct pi_desc {
 	u32 rsvd[6];
 } __aligned(64);
 
+/*
+ * De-multiplexing posted interrupts is on the performance path, the code
+ * below is written to optimize the cache performance based on the following
+ * considerations:
+ * 1.Posted interrupt descriptor (PID) fits in a cache line that is frequently
+ *   accessed by both CPU and IOMMU.
+ * 2.During software processing of posted interrupts, the CPU needs to do
+ *   natural width read and xchg for checking and clearing posted interrupt
+ *   request (PIR), a 256 bit field within the PID.
+ * 3.On the other side, the IOMMU does atomic swaps of the entire PID cache
+ *   line when posting interrupts and setting control bits.
+ * 4.The CPU can access the cache line a magnitude faster than the IOMMU.
+ * 5.Each time the IOMMU does interrupt posting to the PIR will evict the PID
+ *   cache line. The cache line states after each operation are as follows,
+ *   assuming a 64-bit kernel:
+ *   CPU		IOMMU			PID Cache line state
+ *   ---------------------------------------------------------------
+ *...read64					exclusive
+ *...lock xchg64				modified
+ *...			post/atomic swap	invalid
+ *...-------------------------------------------------------------
+ *
+ * To reduce L1 data cache miss, it is important to avoid contention with
+ * IOMMU's interrupt posting/atomic swap. Therefore, a copy of PIR is used
+ * when processing posted interrupts in software, e.g. to dispatch interrupt
+ * handlers for posted MSIs, or to move interrupts from the PIR to the vIRR
+ * in KVM.
+ *
+ * In addition, the code is trying to keep the cache line state consistent
+ * as much as possible. e.g. when making a copy and clearing the PIR
+ * (assuming non-zero PIR bits are present in the entire PIR), it does:
+ *		read, read, read, read, xchg, xchg, xchg, xchg
+ * instead of:
+ *		read, xchg, read, xchg, read, xchg, read, xchg
+ */
+static __always_inline bool pi_harvest_pir(unsigned long *pir,
+					   unsigned long *pir_vals)
+{
+	unsigned long pending = 0;
+	int i;
+
+	for (i = 0; i < NR_PIR_WORDS; i++) {
+		pir_vals[i] = READ_ONCE(pir[i]);
+		pending |= pir_vals[i];
+	}
+
+	if (!pending)
+		return false;
+
+	for (i = 0; i < NR_PIR_WORDS; i++) {
+		if (!pir_vals[i])
+			continue;
+
+		pir_vals[i] = arch_xchg(&pir[i], 0);
+	}
+
+	return true;
+}
+
 static inline bool pi_test_and_set_on(struct pi_desc *pi_desc)
 {
 	return test_and_set_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control);
@@ -43,12 +107,12 @@ static inline bool pi_test_and_clear_sn(struct pi_desc *pi_desc)
 
 static inline bool pi_test_and_set_pir(int vector, struct pi_desc *pi_desc)
 {
-	return test_and_set_bit(vector, (unsigned long *)pi_desc->pir);
+	return test_and_set_bit(vector, pi_desc->pir);
 }
 
 static inline bool pi_is_pir_empty(struct pi_desc *pi_desc)
 {
-	return bitmap_empty((unsigned long *)pi_desc->pir, NR_VECTORS);
+	return bitmap_empty(pi_desc->pir, NR_VECTORS);
 }
 
 static inline void pi_set_sn(struct pi_desc *pi_desc)
@@ -81,6 +145,11 @@ static inline bool pi_test_sn(struct pi_desc *pi_desc)
 	return test_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control);
 }
 
+static inline bool pi_test_pir(int vector, struct pi_desc *pi_desc)
+{
+	return test_bit(vector, (unsigned long *)pi_desc->pir);
+}
+
 /* Non-atomic helpers */
 static inline void __pi_set_sn(struct pi_desc *pi_desc)
 {
@@ -105,7 +174,7 @@ static inline bool pi_pending_this_cpu(unsigned int vector)
 	if (WARN_ON_ONCE(vector > NR_VECTORS || vector < FIRST_EXTERNAL_VECTOR))
 		return false;
 
-	return test_bit(vector, (unsigned long *)pid->pir);
+	return test_bit(vector, pid->pir);
 }
 
 extern void intel_posted_msi_init(void);
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 919909d8cb77..578441db09f0 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -4,10 +4,11 @@
 
 #include <asm/rmwcc.h>
 #include <asm/percpu.h>
-#include <asm/current.h>
 
 #include <linux/static_call_types.h>
 
+DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count);
+
 /* We use the MSB mostly because its available */
 #define PREEMPT_NEED_RESCHED	0x80000000
 
@@ -23,18 +24,18 @@
  */
 static __always_inline int preempt_count(void)
 {
-	return raw_cpu_read_4(pcpu_hot.preempt_count) & ~PREEMPT_NEED_RESCHED;
+	return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;
 }
 
 static __always_inline void preempt_count_set(int pc)
 {
 	int old, new;
 
-	old = raw_cpu_read_4(pcpu_hot.preempt_count);
+	old = raw_cpu_read_4(__preempt_count);
 	do {
 		new = (old & PREEMPT_NEED_RESCHED) |
 			(pc & ~PREEMPT_NEED_RESCHED);
-	} while (!raw_cpu_try_cmpxchg_4(pcpu_hot.preempt_count, &old, new));
+	} while (!raw_cpu_try_cmpxchg_4(__preempt_count, &old, new));
 }
 
 /*
@@ -43,7 +44,7 @@ static __always_inline void preempt_count_set(int pc)
 #define init_task_preempt_count(p) do { } while (0)
 
 #define init_idle_preempt_count(p, cpu) do { \
-	per_cpu(pcpu_hot.preempt_count, (cpu)) = PREEMPT_DISABLED; \
+	per_cpu(__preempt_count, (cpu)) = PREEMPT_DISABLED; \
 } while (0)
 
 /*
@@ -57,17 +58,17 @@ static __always_inline void preempt_count_set(int pc)
 
 static __always_inline void set_preempt_need_resched(void)
 {
-	raw_cpu_and_4(pcpu_hot.preempt_count, ~PREEMPT_NEED_RESCHED);
+	raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);
 }
 
 static __always_inline void clear_preempt_need_resched(void)
 {
-	raw_cpu_or_4(pcpu_hot.preempt_count, PREEMPT_NEED_RESCHED);
+	raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);
 }
 
 static __always_inline bool test_preempt_need_resched(void)
 {
-	return !(raw_cpu_read_4(pcpu_hot.preempt_count) & PREEMPT_NEED_RESCHED);
+	return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);
 }
 
 /*
@@ -76,12 +77,12 @@ static __always_inline bool test_preempt_need_resched(void)
 
 static __always_inline void __preempt_count_add(int val)
 {
-	raw_cpu_add_4(pcpu_hot.preempt_count, val);
+	raw_cpu_add_4(__preempt_count, val);
 }
 
 static __always_inline void __preempt_count_sub(int val)
 {
-	raw_cpu_add_4(pcpu_hot.preempt_count, -val);
+	raw_cpu_add_4(__preempt_count, -val);
 }
 
 /*
@@ -91,7 +92,7 @@ static __always_inline void __preempt_count_sub(int val)
  */
 static __always_inline bool __preempt_count_dec_and_test(void)
 {
-	return GEN_UNARY_RMWcc("decl", __my_cpu_var(pcpu_hot.preempt_count), e,
+	return GEN_UNARY_RMWcc("decl", __my_cpu_var(__preempt_count), e,
 			       __percpu_arg([var]));
 }
 
@@ -100,7 +101,7 @@ static __always_inline bool __preempt_count_dec_and_test(void)
  */
 static __always_inline bool should_resched(int preempt_offset)
 {
-	return unlikely(raw_cpu_read_4(pcpu_hot.preempt_count) == preempt_offset);
+	return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
 }
 
 #ifdef CONFIG_PREEMPTION
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 4a686f0e5dbf..a24c7805acdb 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -16,7 +16,7 @@ struct vm86;
 #include <uapi/asm/sigcontext.h>
 #include <asm/current.h>
 #include <asm/cpufeatures.h>
-#include <asm/cpuid.h>
+#include <asm/cpuid/api.h>
 #include <asm/page.h>
 #include <asm/pgtable_types.h>
 #include <asm/percpu.h>
@@ -60,18 +60,13 @@ struct vm86;
 # define ARCH_MIN_MMSTRUCT_ALIGN	0
 #endif
 
-enum tlb_infos {
-	ENTRIES,
-	NR_INFO
-};
-
-extern u16 __read_mostly tlb_lli_4k[NR_INFO];
-extern u16 __read_mostly tlb_lli_2m[NR_INFO];
-extern u16 __read_mostly tlb_lli_4m[NR_INFO];
-extern u16 __read_mostly tlb_lld_4k[NR_INFO];
-extern u16 __read_mostly tlb_lld_2m[NR_INFO];
-extern u16 __read_mostly tlb_lld_4m[NR_INFO];
-extern u16 __read_mostly tlb_lld_1g[NR_INFO];
+extern u16 __read_mostly tlb_lli_4k;
+extern u16 __read_mostly tlb_lli_2m;
+extern u16 __read_mostly tlb_lli_4m;
+extern u16 __read_mostly tlb_lld_4k;
+extern u16 __read_mostly tlb_lld_2m;
+extern u16 __read_mostly tlb_lld_4m;
+extern u16 __read_mostly tlb_lld_1g;
 
 /*
  * CPU type and hardware bug flags. Kept separately for each CPU.
@@ -98,6 +93,7 @@ struct cpuinfo_topology {
 	// Logical ID mappings
 	u32			logical_pkg_id;
 	u32			logical_die_id;
+	u32			logical_core_id;
 
 	// AMD Node ID and Nodes per Package info
 	u32			amd_node_id;
@@ -105,6 +101,24 @@ struct cpuinfo_topology {
 	// Cache level topology IDs
 	u32			llc_id;
 	u32			l2c_id;
+
+	// Hardware defined CPU-type
+	union {
+		u32		cpu_type;
+		struct {
+			// CPUID.1A.EAX[23-0]
+			u32	intel_native_model_id	:24;
+			// CPUID.1A.EAX[31-24]
+			u32	intel_type		:8;
+		};
+		struct {
+			// CPUID 0x80000026.EBX
+			u32	amd_num_processors	:16,
+				amd_power_eff_ranking	:8,
+				amd_native_model_id	:4,
+				amd_type		:4;
+		};
+	};
 };
 
 struct cpuinfo_x86 {
@@ -212,8 +226,10 @@ static inline unsigned long long l1tf_pfn_limit(void)
 	return BIT_ULL(boot_cpu_data.x86_cache_bits - 1 - PAGE_SHIFT);
 }
 
+void init_cpu_devs(void);
+void get_cpu_vendor(struct cpuinfo_x86 *c);
 extern void early_cpu_init(void);
-extern void identify_secondary_cpu(struct cpuinfo_x86 *);
+extern void identify_secondary_cpu(unsigned int cpu);
 extern void print_cpu_info(struct cpuinfo_x86 *);
 void print_cpu_msr(struct cpuinfo_x86 *);
 
@@ -399,37 +415,33 @@ struct irq_stack {
 	char		stack[IRQ_STACK_SIZE];
 } __aligned(IRQ_STACK_SIZE);
 
+DECLARE_PER_CPU_CACHE_HOT(struct irq_stack *, hardirq_stack_ptr);
 #ifdef CONFIG_X86_64
-struct fixed_percpu_data {
-	/*
-	 * GCC hardcodes the stack canary as %gs:40.  Since the
-	 * irq_stack is the object at %gs:0, we reserve the bottom
-	 * 48 bytes of the irq stack for the canary.
-	 *
-	 * Once we are willing to require -mstack-protector-guard-symbol=
-	 * support for x86_64 stackprotector, we can get rid of this.
-	 */
-	char		gs_base[40];
-	unsigned long	stack_canary;
-};
+DECLARE_PER_CPU_CACHE_HOT(bool, hardirq_stack_inuse);
+#else
+DECLARE_PER_CPU_CACHE_HOT(struct irq_stack *, softirq_stack_ptr);
+#endif
 
-DECLARE_PER_CPU_FIRST(struct fixed_percpu_data, fixed_percpu_data) __visible;
-DECLARE_INIT_PER_CPU(fixed_percpu_data);
+DECLARE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack);
+/* const-qualified alias provided by the linker. */
+DECLARE_PER_CPU_CACHE_HOT(const unsigned long __percpu_seg_override,
+			  const_cpu_current_top_of_stack);
 
+#ifdef CONFIG_X86_64
 static inline unsigned long cpu_kernelmode_gs_base(int cpu)
 {
-	return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
+#ifdef CONFIG_SMP
+	return per_cpu_offset(cpu);
+#else
+	return 0;
+#endif
 }
 
 extern asmlinkage void entry_SYSCALL32_ignore(void);
 
 /* Save actual FS/GS selectors and bases to current->thread */
 void current_save_fsgs(void);
-#else	/* X86_64 */
-#ifdef CONFIG_STACKPROTECTOR
-DECLARE_PER_CPU(unsigned long, __stack_chk_guard);
-#endif
-#endif	/* !X86_64 */
+#endif	/* X86_64 */
 
 struct perf_event;
 
@@ -502,15 +514,14 @@ struct thread_struct {
 
 	struct thread_shstk	shstk;
 #endif
-
-	/* Floating point and extended processor state */
-	struct fpu		fpu;
-	/*
-	 * WARNING: 'fpu' is dynamically-sized.  It *MUST* be at
-	 * the end.
-	 */
 };
 
+#ifdef CONFIG_X86_DEBUG_FPU
+extern struct fpu *x86_task_fpu(struct task_struct *task);
+#else
+# define x86_task_fpu(task)	((struct fpu *)((void *)(task) + sizeof(*(task))))
+#endif
+
 extern void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size);
 
 static inline void arch_thread_struct_whitelist(unsigned long *offset,
@@ -540,9 +551,9 @@ static __always_inline unsigned long current_top_of_stack(void)
 	 *  entry trampoline.
 	 */
 	if (IS_ENABLED(CONFIG_USE_X86_SEG_SUPPORT))
-		return this_cpu_read_const(const_pcpu_hot.top_of_stack);
+		return this_cpu_read_const(const_cpu_current_top_of_stack);
 
-	return this_cpu_read_stable(pcpu_hot.top_of_stack);
+	return this_cpu_read_stable(cpu_current_top_of_stack);
 }
 
 static __always_inline bool on_thread_stack(void)
@@ -647,8 +658,6 @@ static __always_inline void prefetchw(const void *x)
 	.sysenter_cs		= __KERNEL_CS,				  \
 }
 
-#define KSTK_ESP(task)		(task_pt_regs(task)->sp)
-
 #else
 extern unsigned long __top_init_kernel_stack[];
 
@@ -656,8 +665,6 @@ extern unsigned long __top_init_kernel_stack[];
 	.sp	= (unsigned long)&__top_init_kernel_stack,		\
 }
 
-extern unsigned long KSTK_ESP(struct task_struct *task);
-
 #endif /* CONFIG_X86_64 */
 
 extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
@@ -671,6 +678,7 @@ extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
 #define TASK_UNMAPPED_BASE		__TASK_UNMAPPED_BASE(TASK_SIZE_LOW)
 
 #define KSTK_EIP(task)		(task_pt_regs(task)->ip)
+#define KSTK_ESP(task)		(task_pt_regs(task)->sp)
 
 /* Get/set a process' ability to use the timestamp counter instruction */
 #define GET_TSC_CTL(adr)	get_tsc_mode((adr))
@@ -723,8 +731,11 @@ void __noreturn stop_this_cpu(void *dummy);
 void microcode_check(struct cpuinfo_x86 *prev_info);
 void store_cpu_caps(struct cpuinfo_x86 *info);
 
+DECLARE_PER_CPU(bool, cache_state_incoherent);
+
 enum l1tf_mitigations {
 	L1TF_MITIGATION_OFF,
+	L1TF_MITIGATION_AUTO,
 	L1TF_MITIGATION_FLUSH_NOWARN,
 	L1TF_MITIGATION_FLUSH,
 	L1TF_MITIGATION_FLUSH_NOSMT,
@@ -736,6 +747,7 @@ extern enum l1tf_mitigations l1tf_mitigation;
 
 enum mds_mitigations {
 	MDS_MITIGATION_OFF,
+	MDS_MITIGATION_AUTO,
 	MDS_MITIGATION_FULL,
 	MDS_MITIGATION_VMWERV,
 };
diff --git a/arch/x86/include/asm/prom.h b/arch/x86/include/asm/prom.h
index 365798cb4408..5d0dbab85264 100644
--- a/arch/x86/include/asm/prom.h
+++ b/arch/x86/include/asm/prom.h
@@ -8,7 +8,7 @@
 
 #ifndef _ASM_X86_PROM_H
 #define _ASM_X86_PROM_H
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/of.h>
 #include <linux/types.h>
@@ -33,5 +33,5 @@ static inline void x86_flattree_get_config(void) { }
 
 extern char cmd_line[COMMAND_LINE_SIZE];
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif
diff --git a/arch/x86/include/asm/proto.h b/arch/x86/include/asm/proto.h
index 484f4f0131a5..05224a695872 100644
--- a/arch/x86/include/asm/proto.h
+++ b/arch/x86/include/asm/proto.h
@@ -15,7 +15,6 @@ void entry_SYSCALL_64(void);
 void entry_SYSCALL_64_safe_stack(void);
 void entry_SYSRETQ_unsafe_stack(void);
 void entry_SYSRETQ_end(void);
-long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2);
 #endif
 
 #ifdef CONFIG_X86_32
@@ -41,6 +40,6 @@ void x86_configure_nx(void);
 
 extern int reboot_force;
 
-long do_arch_prctl_common(int option, unsigned long arg2);
+long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2);
 
 #endif /* _ASM_X86_PROTO_H */
diff --git a/arch/x86/include/asm/pti.h b/arch/x86/include/asm/pti.h
index ab167c96b9ab..88d0a1ab1f77 100644
--- a/arch/x86/include/asm/pti.h
+++ b/arch/x86/include/asm/pti.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #ifndef _ASM_X86_PTI_H
 #define _ASM_X86_PTI_H
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
 extern void pti_init(void);
@@ -11,5 +11,5 @@ extern void pti_finalize(void);
 static inline void pti_check_boottime_disable(void) { }
 #endif
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* _ASM_X86_PTI_H */
diff --git a/arch/x86/include/asm/ptrace.h b/arch/x86/include/asm/ptrace.h
index 5a83fbd9bc0b..35d062a2e304 100644
--- a/arch/x86/include/asm/ptrace.h
+++ b/arch/x86/include/asm/ptrace.h
@@ -6,7 +6,7 @@
 #include <asm/page_types.h>
 #include <uapi/asm/ptrace.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #ifdef __i386__
 
 struct pt_regs {
@@ -84,8 +84,8 @@ struct fred_ss {
 			:  4,
 		/* Event was incident to enclave execution */
 		enclave	:  1,
-		/* CPU was in long mode */
-		lm	:  1,
+		/* CPU was in 64-bit mode */
+		l	:  1,
 		/*
 		 * Nested exception during FRED delivery, not set
 		 * for #DF.
@@ -187,12 +187,12 @@ convert_ip_to_linear(struct task_struct *child, struct pt_regs *regs);
 extern void send_sigtrap(struct pt_regs *regs, int error_code, int si_code);
 
 
-static inline unsigned long regs_return_value(struct pt_regs *regs)
+static __always_inline unsigned long regs_return_value(struct pt_regs *regs)
 {
 	return regs->ax;
 }
 
-static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc)
+static __always_inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc)
 {
 	regs->ax = rc;
 }
@@ -277,34 +277,34 @@ static __always_inline bool ip_within_syscall_gap(struct pt_regs *regs)
 }
 #endif
 
-static inline unsigned long kernel_stack_pointer(struct pt_regs *regs)
+static __always_inline unsigned long kernel_stack_pointer(struct pt_regs *regs)
 {
 	return regs->sp;
 }
 
-static inline unsigned long instruction_pointer(struct pt_regs *regs)
+static __always_inline unsigned long instruction_pointer(struct pt_regs *regs)
 {
 	return regs->ip;
 }
 
-static inline void instruction_pointer_set(struct pt_regs *regs,
-		unsigned long val)
+static __always_inline
+void instruction_pointer_set(struct pt_regs *regs, unsigned long val)
 {
 	regs->ip = val;
 }
 
-static inline unsigned long frame_pointer(struct pt_regs *regs)
+static __always_inline unsigned long frame_pointer(struct pt_regs *regs)
 {
 	return regs->bp;
 }
 
-static inline unsigned long user_stack_pointer(struct pt_regs *regs)
+static __always_inline unsigned long user_stack_pointer(struct pt_regs *regs)
 {
 	return regs->sp;
 }
 
-static inline void user_stack_pointer_set(struct pt_regs *regs,
-		unsigned long val)
+static __always_inline
+void user_stack_pointer_set(struct pt_regs *regs, unsigned long val)
 {
 	regs->sp = val;
 }
@@ -469,5 +469,5 @@ extern int do_set_thread_area(struct task_struct *p, int idx,
 # define do_set_thread_area_64(p, s, t)	(0)
 #endif
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 #endif /* _ASM_X86_PTRACE_H */
diff --git a/arch/x86/include/asm/purgatory.h b/arch/x86/include/asm/purgatory.h
index 5528e9325049..2fee5e9f1ccc 100644
--- a/arch/x86/include/asm/purgatory.h
+++ b/arch/x86/include/asm/purgatory.h
@@ -2,10 +2,10 @@
 #ifndef _ASM_X86_PURGATORY_H
 #define _ASM_X86_PURGATORY_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/purgatory.h>
 
 extern void purgatory(void);
-#endif	/* __ASSEMBLY__ */
+#endif	/* __ASSEMBLER__ */
 
 #endif /* _ASM_PURGATORY_H */
diff --git a/arch/x86/include/asm/pvclock-abi.h b/arch/x86/include/asm/pvclock-abi.h
index 1436226efe3e..b9fece5fc96d 100644
--- a/arch/x86/include/asm/pvclock-abi.h
+++ b/arch/x86/include/asm/pvclock-abi.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #ifndef _ASM_X86_PVCLOCK_ABI_H
 #define _ASM_X86_PVCLOCK_ABI_H
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 /*
  * These structs MUST NOT be changed.
@@ -44,5 +44,5 @@ struct pvclock_wall_clock {
 #define PVCLOCK_GUEST_STOPPED	(1 << 1)
 /* PVCLOCK_COUNTS_FROM_ZERO broke ABI and can't be used anymore. */
 #define PVCLOCK_COUNTS_FROM_ZERO (1 << 2)
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* _ASM_X86_PVCLOCK_ABI_H */
diff --git a/arch/x86/include/asm/realmode.h b/arch/x86/include/asm/realmode.h
index 87e5482acd0d..e406a1e92c63 100644
--- a/arch/x86/include/asm/realmode.h
+++ b/arch/x86/include/asm/realmode.h
@@ -9,7 +9,7 @@
 #define TH_FLAGS_SME_ACTIVE_BIT		0
 #define TH_FLAGS_SME_ACTIVE		BIT(TH_FLAGS_SME_ACTIVE_BIT)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/types.h>
 #include <asm/io.h>
@@ -78,7 +78,7 @@ extern unsigned char secondary_startup_64[];
 extern unsigned char secondary_startup_64_no_verify[];
 #endif
 
-static inline size_t real_mode_size_needed(void)
+static __always_inline size_t real_mode_size_needed(void)
 {
 	if (real_mode_header)
 		return 0;	/* already allocated. */
@@ -95,6 +95,6 @@ void reserve_real_mode(void);
 void load_trampoline_pgtable(void);
 void init_real_mode(void);
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ARCH_X86_REALMODE_H */
diff --git a/arch/x86/include/asm/required-features.h b/arch/x86/include/asm/required-features.h
deleted file mode 100644
index e9187ddd3d1f..000000000000
--- a/arch/x86/include/asm/required-features.h
+++ /dev/null
@@ -1,105 +0,0 @@
-#ifndef _ASM_X86_REQUIRED_FEATURES_H
-#define _ASM_X86_REQUIRED_FEATURES_H
-
-/* Define minimum CPUID feature set for kernel These bits are checked
-   really early to actually display a visible error message before the
-   kernel dies.  Make sure to assign features to the proper mask!
-
-   Some requirements that are not in CPUID yet are also in the
-   CONFIG_X86_MINIMUM_CPU_FAMILY which is checked too.
-
-   The real information is in arch/x86/Kconfig.cpu, this just converts
-   the CONFIGs into a bitmask */
-
-#ifndef CONFIG_MATH_EMULATION
-# define NEED_FPU	(1<<(X86_FEATURE_FPU & 31))
-#else
-# define NEED_FPU	0
-#endif
-
-#if defined(CONFIG_X86_PAE) || defined(CONFIG_X86_64)
-# define NEED_PAE	(1<<(X86_FEATURE_PAE & 31))
-#else
-# define NEED_PAE	0
-#endif
-
-#ifdef CONFIG_X86_CMPXCHG64
-# define NEED_CX8	(1<<(X86_FEATURE_CX8 & 31))
-#else
-# define NEED_CX8	0
-#endif
-
-#if defined(CONFIG_X86_CMOV) || defined(CONFIG_X86_64)
-# define NEED_CMOV	(1<<(X86_FEATURE_CMOV & 31))
-#else
-# define NEED_CMOV	0
-#endif
-
-# define NEED_3DNOW	0
-
-#if defined(CONFIG_X86_P6_NOP) || defined(CONFIG_X86_64)
-# define NEED_NOPL	(1<<(X86_FEATURE_NOPL & 31))
-#else
-# define NEED_NOPL	0
-#endif
-
-#ifdef CONFIG_MATOM
-# define NEED_MOVBE	(1<<(X86_FEATURE_MOVBE & 31))
-#else
-# define NEED_MOVBE	0
-#endif
-
-#ifdef CONFIG_X86_64
-#ifdef CONFIG_PARAVIRT_XXL
-/* Paravirtualized systems may not have PSE or PGE available */
-#define NEED_PSE	0
-#define NEED_PGE	0
-#else
-#define NEED_PSE	(1<<(X86_FEATURE_PSE) & 31)
-#define NEED_PGE	(1<<(X86_FEATURE_PGE) & 31)
-#endif
-#define NEED_MSR	(1<<(X86_FEATURE_MSR & 31))
-#define NEED_FXSR	(1<<(X86_FEATURE_FXSR & 31))
-#define NEED_XMM	(1<<(X86_FEATURE_XMM & 31))
-#define NEED_XMM2	(1<<(X86_FEATURE_XMM2 & 31))
-#define NEED_LM		(1<<(X86_FEATURE_LM & 31))
-#else
-#define NEED_PSE	0
-#define NEED_MSR	0
-#define NEED_PGE	0
-#define NEED_FXSR	0
-#define NEED_XMM	0
-#define NEED_XMM2	0
-#define NEED_LM		0
-#endif
-
-#define REQUIRED_MASK0	(NEED_FPU|NEED_PSE|NEED_MSR|NEED_PAE|\
-			 NEED_CX8|NEED_PGE|NEED_FXSR|NEED_CMOV|\
-			 NEED_XMM|NEED_XMM2)
-#define SSE_MASK	(NEED_XMM|NEED_XMM2)
-
-#define REQUIRED_MASK1	(NEED_LM|NEED_3DNOW)
-
-#define REQUIRED_MASK2	0
-#define REQUIRED_MASK3	(NEED_NOPL)
-#define REQUIRED_MASK4	(NEED_MOVBE)
-#define REQUIRED_MASK5	0
-#define REQUIRED_MASK6	0
-#define REQUIRED_MASK7	0
-#define REQUIRED_MASK8	0
-#define REQUIRED_MASK9	0
-#define REQUIRED_MASK10	0
-#define REQUIRED_MASK11	0
-#define REQUIRED_MASK12	0
-#define REQUIRED_MASK13	0
-#define REQUIRED_MASK14	0
-#define REQUIRED_MASK15	0
-#define REQUIRED_MASK16	0
-#define REQUIRED_MASK17	0
-#define REQUIRED_MASK18	0
-#define REQUIRED_MASK19	0
-#define REQUIRED_MASK20	0
-#define REQUIRED_MASK21	0
-#define REQUIRED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 22)
-
-#endif /* _ASM_X86_REQUIRED_FEATURES_H */
diff --git a/arch/x86/include/asm/resctrl.h b/arch/x86/include/asm/resctrl.h
index 8b1b6ce1e51b..575f8408a9e7 100644
--- a/arch/x86/include/asm/resctrl.h
+++ b/arch/x86/include/asm/resctrl.h
@@ -4,8 +4,12 @@
 
 #ifdef CONFIG_X86_CPU_RESCTRL
 
-#include <linux/sched.h>
 #include <linux/jump_label.h>
+#include <linux/percpu.h>
+#include <linux/resctrl_types.h>
+#include <linux/sched.h>
+
+#include <asm/msr.h>
 
 /*
  * This value can never be a valid CLOSID, and is used when mapping a
@@ -96,8 +100,8 @@ static inline void resctrl_arch_disable_mon(void)
 static inline void __resctrl_sched_in(struct task_struct *tsk)
 {
 	struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state);
-	u32 closid = state->default_closid;
-	u32 rmid = state->default_rmid;
+	u32 closid = READ_ONCE(state->default_closid);
+	u32 rmid = READ_ONCE(state->default_rmid);
 	u32 tmp;
 
 	/*
@@ -132,6 +136,13 @@ static inline unsigned int resctrl_arch_round_mon_val(unsigned int val)
 	return val * scale;
 }
 
+static inline void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid,
+							    u32 rmid)
+{
+	WRITE_ONCE(per_cpu(pqr_state.default_closid, cpu), closid);
+	WRITE_ONCE(per_cpu(pqr_state.default_rmid, cpu), rmid);
+}
+
 static inline void resctrl_arch_set_closid_rmid(struct task_struct *tsk,
 						u32 closid, u32 rmid)
 {
@@ -150,7 +161,7 @@ static inline bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 ignored,
 	return READ_ONCE(tsk->rmid) == rmid;
 }
 
-static inline void resctrl_sched_in(struct task_struct *tsk)
+static inline void resctrl_arch_sched_in(struct task_struct *tsk)
 {
 	if (static_branch_likely(&rdt_enable_key))
 		__resctrl_sched_in(tsk);
@@ -169,20 +180,22 @@ static inline u32 resctrl_arch_rmid_idx_encode(u32 ignored, u32 rmid)
 
 /* x86 can always read an rmid, nothing needs allocating */
 struct rdt_resource;
-static inline void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, int evtid)
+static inline void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r,
+					       enum resctrl_event_id evtid)
 {
 	might_sleep();
 	return NULL;
-};
+}
 
-static inline void resctrl_arch_mon_ctx_free(struct rdt_resource *r, int evtid,
-					     void *ctx) { };
+static inline void resctrl_arch_mon_ctx_free(struct rdt_resource *r,
+					     enum resctrl_event_id evtid,
+					     void *ctx) { }
 
 void resctrl_cpu_detect(struct cpuinfo_x86 *c);
 
 #else
 
-static inline void resctrl_sched_in(struct task_struct *tsk) {}
+static inline void resctrl_arch_sched_in(struct task_struct *tsk) {}
 static inline void resctrl_cpu_detect(struct cpuinfo_x86 *c) {}
 
 #endif /* CONFIG_X86_CPU_RESCTRL */
diff --git a/arch/x86/include/asm/rmwcc.h b/arch/x86/include/asm/rmwcc.h
index 363266cbcada..54c8fc430684 100644
--- a/arch/x86/include/asm/rmwcc.h
+++ b/arch/x86/include/asm/rmwcc.h
@@ -6,37 +6,15 @@
 
 #define __CLOBBERS_MEM(clb...)	"memory", ## clb
 
-#ifndef __GCC_ASM_FLAG_OUTPUTS__
-
-/* Use asm goto */
-
-#define __GEN_RMWcc(fullop, _var, cc, clobbers, ...)			\
-({									\
-	bool c = false;							\
-	asm goto (fullop "; j" #cc " %l[cc_label]"		\
-			: : [var] "m" (_var), ## __VA_ARGS__		\
-			: clobbers : cc_label);				\
-	if (0) {							\
-cc_label:	c = true;						\
-	}								\
-	c;								\
-})
-
-#else /* defined(__GCC_ASM_FLAG_OUTPUTS__) */
-
-/* Use flags output or a set instruction */
-
 #define __GEN_RMWcc(fullop, _var, cc, clobbers, ...)			\
 ({									\
 	bool c;								\
-	asm volatile (fullop CC_SET(cc)					\
-			: [var] "+m" (_var), CC_OUT(cc) (c)		\
+	asm_inline volatile (fullop					\
+			: [var] "+m" (_var), "=@cc" #cc (c)		\
 			: __VA_ARGS__ : clobbers);			\
 	c;								\
 })
 
-#endif /* defined(__GCC_ASM_FLAG_OUTPUTS__) */
-
 #define GEN_UNARY_RMWcc_4(op, var, cc, arg0)				\
 	__GEN_RMWcc(op " " arg0, var, cc, __CLOBBERS_MEM())
 
diff --git a/arch/x86/include/asm/rqspinlock.h b/arch/x86/include/asm/rqspinlock.h
new file mode 100644
index 000000000000..24a885449ee6
--- /dev/null
+++ b/arch/x86/include/asm/rqspinlock.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_RQSPINLOCK_H
+#define _ASM_X86_RQSPINLOCK_H
+
+#include <asm/paravirt.h>
+
+#ifdef CONFIG_PARAVIRT
+DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key);
+
+#define resilient_virt_spin_lock_enabled resilient_virt_spin_lock_enabled
+static __always_inline bool resilient_virt_spin_lock_enabled(void)
+{
+       return static_branch_likely(&virt_spin_lock_key);
+}
+
+#ifdef CONFIG_QUEUED_SPINLOCKS
+typedef struct qspinlock rqspinlock_t;
+#else
+typedef struct rqspinlock rqspinlock_t;
+#endif
+extern int resilient_tas_spin_lock(rqspinlock_t *lock);
+
+#define resilient_virt_spin_lock resilient_virt_spin_lock
+static inline int resilient_virt_spin_lock(rqspinlock_t *lock)
+{
+	return resilient_tas_spin_lock(lock);
+}
+
+#endif /* CONFIG_PARAVIRT */
+
+#include <asm-generic/rqspinlock.h>
+
+#endif /* _ASM_X86_RQSPINLOCK_H */
diff --git a/arch/x86/include/asm/runtime-const.h b/arch/x86/include/asm/runtime-const.h
index 24e3a53ca255..e5a13dc8816e 100644
--- a/arch/x86/include/asm/runtime-const.h
+++ b/arch/x86/include/asm/runtime-const.h
@@ -2,11 +2,27 @@
 #ifndef _ASM_RUNTIME_CONST_H
 #define _ASM_RUNTIME_CONST_H
 
+#ifdef MODULE
+  #error "Cannot use runtime-const infrastructure from modules"
+#endif
+
+#ifdef __ASSEMBLY__
+
+.macro RUNTIME_CONST_PTR sym reg
+	movq	$0x0123456789abcdef, %\reg
+	1:
+	.pushsection runtime_ptr_\sym, "a"
+	.long	1b - 8 - .
+	.popsection
+.endm
+
+#else /* __ASSEMBLY__ */
+
 #define runtime_const_ptr(sym) ({				\
 	typeof(sym) __ret;					\
 	asm_inline("mov %1,%0\n1:\n"				\
 		".pushsection runtime_ptr_" #sym ",\"a\"\n\t"	\
-		".long 1b - %c2 - .\n\t"			\
+		".long 1b - %c2 - .\n"				\
 		".popsection"					\
 		:"=r" (__ret)					\
 		:"i" ((unsigned long)0x0123456789abcdefull),	\
@@ -20,7 +36,7 @@
 	typeof(0u+(val)) __ret = (val);				\
 	asm_inline("shrl $12,%k0\n1:\n"				\
 		".pushsection runtime_shift_" #sym ",\"a\"\n\t"	\
-		".long 1b - 1 - .\n\t"				\
+		".long 1b - 1 - .\n"				\
 		".popsection"					\
 		:"+r" (__ret));					\
 	__ret; })
@@ -58,4 +74,5 @@ static inline void runtime_const_fixup(void (*fn)(void *, unsigned long),
 	}
 }
 
+#endif /* __ASSEMBLY__ */
 #endif
diff --git a/arch/x86/include/asm/sections.h b/arch/x86/include/asm/sections.h
index 3fa87e5e11ab..30e8ee7006f9 100644
--- a/arch/x86/include/asm/sections.h
+++ b/arch/x86/include/asm/sections.h
@@ -5,6 +5,7 @@
 #include <asm-generic/sections.h>
 #include <asm/extable.h>
 
+extern char __relocate_kernel_start[], __relocate_kernel_end[];
 extern char __brk_base[], __brk_limit[];
 extern char __end_rodata_aligned[];
 
diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h
index 9d6411c65920..f59ae7186940 100644
--- a/arch/x86/include/asm/segment.h
+++ b/arch/x86/include/asm/segment.h
@@ -233,7 +233,7 @@
 #define VDSO_CPUNODE_BITS		12
 #define VDSO_CPUNODE_MASK		0xfff
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 /* Helper functions to store/load CPU and node numbers */
 
@@ -244,7 +244,7 @@ static inline unsigned long vdso_encode_cpunode(int cpu, unsigned long node)
 
 static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node)
 {
-	unsigned int p;
+	unsigned long p;
 
 	/*
 	 * Load CPU and node number from the GDT.  LSL is faster than RDTSCP
@@ -254,10 +254,10 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node)
 	 *
 	 * If RDPID is available, use it.
 	 */
-	alternative_io ("lsl %[seg],%[p]",
-			".byte 0xf3,0x0f,0xc7,0xf8", /* RDPID %eax/rax */
+	alternative_io ("lsl %[seg],%k[p]",
+			"rdpid %[p]",
 			X86_FEATURE_RDPID,
-			[p] "=a" (p), [seg] "r" (__CPUNODE_SEG));
+			[p] "=r" (p), [seg] "r" (__CPUNODE_SEG));
 
 	if (cpu)
 		*cpu = (p & VDSO_CPUNODE_MASK);
@@ -265,7 +265,7 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node)
 		*node = (p >> VDSO_CPUNODE_BITS);
 }
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #ifdef __KERNEL__
 
@@ -286,7 +286,7 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node)
  */
 #define XEN_EARLY_IDT_HANDLER_SIZE (8 + ENDBR_INSN_SIZE)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 extern const char early_idt_handler_array[NUM_EXCEPTION_VECTORS][EARLY_IDT_HANDLER_SIZE];
 extern void early_ignore_irq(void);
@@ -350,7 +350,7 @@ static inline void __loadsegment_fs(unsigned short value)
 #define savesegment(seg, value)				\
 	asm("mov %%" #seg ",%0":"=r" (value) : : "memory")
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 #endif /* __KERNEL__ */
 
 #endif /* _ASM_X86_SEGMENT_H */
diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h
index 4b2abce2e3e7..61f56cdaccb5 100644
--- a/arch/x86/include/asm/set_memory.h
+++ b/arch/x86/include/asm/set_memory.h
@@ -2,9 +2,9 @@
 #ifndef _ASM_X86_SET_MEMORY_H
 #define _ASM_X86_SET_MEMORY_H
 
-#include <linux/mm.h>
 #include <asm/page.h>
 #include <asm-generic/set_memory.h>
+#include <asm/pgtable.h>
 
 #define set_memory_rox set_memory_rox
 int set_memory_rox(unsigned long addr, int numpages);
@@ -89,6 +89,7 @@ int set_pages_rw(struct page *page, int numpages);
 
 int set_direct_map_invalid_noflush(struct page *page);
 int set_direct_map_default_noflush(struct page *page);
+int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid);
 bool kernel_page_present(struct page *page);
 
 extern int kernel_set_to_readonly;
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index 0667b2a88614..914eb32581c7 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -27,7 +27,7 @@
 #define OLD_CL_ADDRESS		0x020	/* Relative to real mode data */
 #define NEW_CL_POINTER		0x228	/* Relative to real mode data */
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/cache.h>
 
 #include <asm/bootparam.h>
@@ -46,11 +46,14 @@ void setup_bios_corruption_check(void);
 void early_platform_quirks(void);
 
 extern unsigned long saved_video_mode;
+extern unsigned long acpi_realmode_flags;
 
 extern void reserve_standard_io_resources(void);
 extern void i386_reserve_resources(void);
-extern unsigned long __startup_64(unsigned long physaddr, struct boot_params *bp);
+extern unsigned long __startup_64(unsigned long p2v_offset, struct boot_params *bp);
 extern void startup_64_setup_gdt_idt(void);
+extern void startup_64_load_idt(void *vc_handler);
+extern void __pi_startup_64_load_idt(void *vc_handler);
 extern void early_setup_idt(void);
 extern void __init do_early_exception(struct pt_regs *regs, int trapnr);
 
@@ -66,6 +69,8 @@ extern void x86_ce4100_early_setup(void);
 static inline void x86_ce4100_early_setup(void) { }
 #endif
 
+#include <linux/kexec_handover.h>
+
 #ifndef _SETUP
 
 #include <asm/espfix.h>
@@ -141,7 +146,7 @@ extern bool builtin_cmdline_added __ro_after_init;
 #define builtin_cmdline_added 0
 #endif
 
-#else  /* __ASSEMBLY */
+#else  /* __ASSEMBLER__ */
 
 .macro __RESERVE_BRK name, size
 	.pushsection .bss..brk, "aw"
@@ -153,6 +158,6 @@ SYM_DATA_END(__brk_\name)
 
 #define RESERVE_BRK(name, size) __RESERVE_BRK name, size
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ASM_X86_SETUP_H */
diff --git a/arch/x86/include/asm/setup_data.h b/arch/x86/include/asm/setup_data.h
index 77c51111a893..7bb16f843c93 100644
--- a/arch/x86/include/asm/setup_data.h
+++ b/arch/x86/include/asm/setup_data.h
@@ -4,7 +4,7 @@
 
 #include <uapi/asm/setup_data.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 struct pci_setup_rom {
 	struct setup_data data;
@@ -27,6 +27,6 @@ struct efi_setup_data {
 	u64 reserved[8];
 };
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ASM_X86_SETUP_DATA_H */
diff --git a/arch/x86/include/asm/sev-common.h b/arch/x86/include/asm/sev-common.h
index 98726c2b04f8..01a6e4dbe423 100644
--- a/arch/x86/include/asm/sev-common.h
+++ b/arch/x86/include/asm/sev-common.h
@@ -116,7 +116,7 @@ enum psc_op {
 #define GHCB_MSR_VMPL_REQ		0x016
 #define GHCB_MSR_VMPL_REQ_LEVEL(v)			\
 	/* GHCBData[39:32] */				\
-	(((u64)(v) & GENMASK_ULL(7, 0) << 32) |		\
+	((((u64)(v) & GENMASK_ULL(7, 0)) << 32) |	\
 	/* GHCBDdata[11:0] */				\
 	GHCB_MSR_VMPL_REQ)
 
@@ -206,12 +206,23 @@ struct snp_psc_desc {
 #define GHCB_TERM_NO_SVSM		7	/* SVSM is not advertised in the secrets page */
 #define GHCB_TERM_SVSM_VMPL0		8	/* SVSM is present but has set VMPL to 0 */
 #define GHCB_TERM_SVSM_CAA		9	/* SVSM is present but CAA is not page aligned */
+#define GHCB_TERM_SECURE_TSC		10	/* Secure TSC initialization failed */
+#define GHCB_TERM_SVSM_CA_REMAP_FAIL	11	/* SVSM is present but CA could not be remapped */
+#define GHCB_TERM_SAVIC_FAIL		12	/* Secure AVIC-specific failure */
 
 #define GHCB_RESP_CODE(v)		((v) & GHCB_MSR_INFO_MASK)
 
 /*
- * Error codes related to GHCB input that can be communicated back to the guest
- * by setting the lower 32-bits of the GHCB SW_EXITINFO1 field to 2.
+ * GHCB-defined return codes that are communicated back to the guest via
+ * SW_EXITINFO1.
+ */
+#define GHCB_HV_RESP_NO_ACTION		0
+#define GHCB_HV_RESP_ISSUE_EXCEPTION	1
+#define GHCB_HV_RESP_MALFORMED_INPUT	2
+
+/*
+ * GHCB-defined sub-error codes for malformed input (see above) that are
+ * communicated back to the guest via SW_EXITINFO2[31:0].
  */
 #define GHCB_ERR_NOT_REGISTERED		1
 #define GHCB_ERR_INVALID_USAGE		2
@@ -220,4 +231,31 @@ struct snp_psc_desc {
 #define GHCB_ERR_INVALID_INPUT		5
 #define GHCB_ERR_INVALID_EVENT		6
 
+struct sev_config {
+	__u64 debug		: 1,
+
+	      /*
+	       * Indicates when the per-CPU GHCB has been created and registered
+	       * and thus can be used by the BSP instead of the early boot GHCB.
+	       *
+	       * For APs, the per-CPU GHCB is created before they are started
+	       * and registered upon startup, so this flag can be used globally
+	       * for the BSP and APs.
+	       */
+	      ghcbs_initialized	: 1,
+
+	      /*
+	       * Indicates when the per-CPU SVSM CA is to be used instead of the
+	       * boot SVSM CA.
+	       *
+	       * For APs, the per-CPU SVSM CA is created as part of the AP
+	       * bringup, so this flag can be used globally for the BSP and APs.
+	       */
+	      use_cas		: 1,
+
+	      __reserved	: 61;
+};
+
+extern struct sev_config sev_cfg;
+
 #endif
diff --git a/arch/x86/include/asm/sev-internal.h b/arch/x86/include/asm/sev-internal.h
new file mode 100644
index 000000000000..c58c47c68ab6
--- /dev/null
+++ b/arch/x86/include/asm/sev-internal.h
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#define DR7_RESET_VALUE        0x400
+
+extern u64 sev_hv_features;
+extern u64 sev_secrets_pa;
+
+/* #VC handler runtime per-CPU data */
+struct sev_es_runtime_data {
+	struct ghcb ghcb_page;
+
+	/*
+	 * Reserve one page per CPU as backup storage for the unencrypted GHCB.
+	 * It is needed when an NMI happens while the #VC handler uses the real
+	 * GHCB, and the NMI handler itself is causing another #VC exception. In
+	 * that case the GHCB content of the first handler needs to be backed up
+	 * and restored.
+	 */
+	struct ghcb backup_ghcb;
+
+	/*
+	 * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions.
+	 * There is no need for it to be atomic, because nothing is written to
+	 * the GHCB between the read and the write of ghcb_active. So it is safe
+	 * to use it when a nested #VC exception happens before the write.
+	 *
+	 * This is necessary for example in the #VC->NMI->#VC case when the NMI
+	 * happens while the first #VC handler uses the GHCB. When the NMI code
+	 * raises a second #VC handler it might overwrite the contents of the
+	 * GHCB written by the first handler. To avoid this the content of the
+	 * GHCB is saved and restored when the GHCB is detected to be in use
+	 * already.
+	 */
+	bool ghcb_active;
+	bool backup_ghcb_active;
+
+	/*
+	 * Cached DR7 value - write it on DR7 writes and return it on reads.
+	 * That value will never make it to the real hardware DR7 as debugging
+	 * is currently unsupported in SEV-ES guests.
+	 */
+	unsigned long dr7;
+};
+
+struct ghcb_state {
+	struct ghcb *ghcb;
+};
+
+extern struct svsm_ca boot_svsm_ca_page;
+
+struct ghcb *__sev_get_ghcb(struct ghcb_state *state);
+void __sev_put_ghcb(struct ghcb_state *state);
+
+DECLARE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
+DECLARE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
+
+void early_set_pages_state(unsigned long vaddr, unsigned long paddr,
+			   unsigned long npages, const struct psc_desc *desc);
+
+DECLARE_PER_CPU(struct svsm_ca *, svsm_caa);
+DECLARE_PER_CPU(u64, svsm_caa_pa);
+
+extern u64 boot_svsm_caa_pa;
+
+enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt);
+void vc_forward_exception(struct es_em_ctxt *ctxt);
+
+static inline u64 sev_es_rd_ghcb_msr(void)
+{
+	return native_rdmsrq(MSR_AMD64_SEV_ES_GHCB);
+}
+
+static __always_inline void sev_es_wr_ghcb_msr(u64 val)
+{
+	u32 low, high;
+
+	low  = (u32)(val);
+	high = (u32)(val >> 32);
+
+	native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high);
+}
+
+enum es_result sev_es_ghcb_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt, bool write);
+
+u64 get_hv_features(void);
+
+const struct snp_cpuid_table *snp_cpuid_get_table(void);
diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
index ee34ab00a8d6..0e6c0940100f 100644
--- a/arch/x86/include/asm/sev.h
+++ b/arch/x86/include/asm/sev.h
@@ -14,6 +14,8 @@
 #include <asm/insn.h>
 #include <asm/sev-common.h>
 #include <asm/coco.h>
+#include <asm/set_memory.h>
+#include <asm/svm.h>
 
 #define GHCB_PROTOCOL_MIN	1ULL
 #define GHCB_PROTOCOL_MAX	2ULL
@@ -82,6 +84,36 @@ extern void vc_no_ghcb(void);
 extern void vc_boot_ghcb(void);
 extern bool handle_vc_boot_ghcb(struct pt_regs *regs);
 
+/*
+ * Individual entries of the SNP CPUID table, as defined by the SNP
+ * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
+ */
+struct snp_cpuid_fn {
+	u32 eax_in;
+	u32 ecx_in;
+	u64 xcr0_in;
+	u64 xss_in;
+	u32 eax;
+	u32 ebx;
+	u32 ecx;
+	u32 edx;
+	u64 __reserved;
+} __packed;
+
+/*
+ * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
+ * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
+ * of 64 entries per CPUID table.
+ */
+#define SNP_CPUID_COUNT_MAX 64
+
+struct snp_cpuid_table {
+	u32 count;
+	u32 __reserved1;
+	u64 __reserved2;
+	struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
+} __packed;
+
 /* PVALIDATE return codes */
 #define PVALIDATE_FAIL_SIZEMISMATCH	6
 
@@ -120,6 +152,12 @@ struct snp_req_data {
 };
 
 #define MAX_AUTHTAG_LEN		32
+#define AUTHTAG_LEN		16
+#define AAD_LEN			48
+#define MSG_HDR_VER		1
+
+#define SNP_REQ_MAX_RETRY_DURATION      (60*HZ)
+#define SNP_REQ_RETRY_DELAY             (2*HZ)
 
 /* See SNP spec SNP_GUEST_REQUEST section for the structure */
 enum msg_type {
@@ -139,6 +177,9 @@ enum msg_type {
 	SNP_MSG_VMRK_REQ,
 	SNP_MSG_VMRK_RSP,
 
+	SNP_MSG_TSC_INFO_REQ = 17,
+	SNP_MSG_TSC_INFO_RSP,
+
 	SNP_MSG_TYPE_MAX
 };
 
@@ -167,8 +208,48 @@ struct snp_guest_msg {
 	u8 payload[PAGE_SIZE - sizeof(struct snp_guest_msg_hdr)];
 } __packed;
 
-struct sev_guest_platform_data {
-	u64 secrets_gpa;
+#define SNP_TSC_INFO_REQ_SZ	128
+
+struct snp_tsc_info_req {
+	u8 rsvd[SNP_TSC_INFO_REQ_SZ];
+} __packed;
+
+struct snp_tsc_info_resp {
+	u32 status;
+	u32 rsvd1;
+	u64 tsc_scale;
+	u64 tsc_offset;
+	u32 tsc_factor;
+	u8 rsvd2[100];
+} __packed;
+
+/*
+ * Obtain the mean TSC frequency by decreasing the nominal TSC frequency with
+ * TSC_FACTOR as documented in the SNP Firmware ABI specification:
+ *
+ * GUEST_TSC_FREQ * (1 - (TSC_FACTOR * 0.00001))
+ *
+ * which is equivalent to:
+ *
+ * GUEST_TSC_FREQ -= (GUEST_TSC_FREQ * TSC_FACTOR) / 100000;
+ */
+#define SNP_SCALE_TSC_FREQ(freq, factor) ((freq) - (freq) * (factor) / 100000)
+
+struct snp_guest_req {
+	void *req_buf;
+	size_t req_sz;
+
+	void *resp_buf;
+	size_t resp_sz;
+
+	u64 exit_code;
+	u64 exitinfo2;
+	unsigned int vmpck_id;
+	u8 msg_version;
+	u8 msg_type;
+
+	struct snp_req_data input;
+	void *certs_data;
 };
 
 /*
@@ -214,10 +295,32 @@ struct snp_secrets_page {
 	u8 svsm_guest_vmpl;
 	u8 rsvd3[3];
 
+	/* The percentage decrease from nominal to mean TSC frequency. */
+	u32 tsc_factor;
+
 	/* Remainder of page */
-	u8 rsvd4[3744];
+	u8 rsvd4[3740];
 } __packed;
 
+struct snp_msg_desc {
+	/* request and response are in unencrypted memory */
+	struct snp_guest_msg *request, *response;
+
+	/*
+	 * Avoid information leakage by double-buffering shared messages
+	 * in fields that are in regular encrypted memory.
+	 */
+	struct snp_guest_msg secret_request, secret_response;
+
+	struct snp_secrets_page *secrets;
+
+	struct aesgcm_ctx *ctx;
+
+	u32 *os_area_msg_seqno;
+	u8 *vmpck;
+	int vmpck_id;
+};
+
 /*
  * The SVSM Calling Area (CA) related structures.
  */
@@ -285,6 +388,22 @@ struct svsm_attest_call {
 	u8 rsvd[4];
 };
 
+/* PTE descriptor used for the prepare_pte_enc() operations. */
+struct pte_enc_desc {
+	pte_t *kpte;
+	int pte_level;
+	bool encrypt;
+	/* pfn of the kpte above */
+	unsigned long pfn;
+	/* physical address of @pfn */
+	unsigned long pa;
+	/* virtual address of @pfn */
+	void *va;
+	/* memory covered by the pte */
+	unsigned long size;
+	pgprot_t new_pgprot;
+};
+
 /*
  * SVSM protocol structure
  */
@@ -312,6 +431,10 @@ struct svsm_call {
 #define SVSM_ATTEST_SERVICES		0
 #define SVSM_ATTEST_SINGLE_SERVICE	1
 
+#define SVSM_VTPM_CALL(x)		((2ULL << 32) | (x))
+#define SVSM_VTPM_QUERY			0
+#define SVSM_VTPM_CMD			1
+
 #ifdef CONFIG_AMD_MEM_ENCRYPT
 
 extern u8 snp_vmpl;
@@ -338,7 +461,7 @@ static __always_inline void sev_es_nmi_complete(void)
 	    cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
 		__sev_es_nmi_complete();
 }
-extern int __init sev_es_efi_map_ghcbs(pgd_t *pgd);
+extern int __init sev_es_efi_map_ghcbs_cas(pgd_t *pgd);
 extern void sev_enable(struct boot_params *bp);
 
 /*
@@ -368,8 +491,7 @@ static inline int pvalidate(unsigned long vaddr, bool rmp_psize, bool validate)
 
 	/* "pvalidate" mnemonic support in binutils 2.36 and newer */
 	asm volatile(".byte 0xF2, 0x0F, 0x01, 0xFF\n\t"
-		     CC_SET(c)
-		     : CC_OUT(c) (no_rmpupdate), "=a"(rc)
+		     : "=@ccc"(no_rmpupdate), "=a"(rc)
 		     : "a"(vaddr), "c"(rmp_psize), "d"(validate)
 		     : "memory", "cc");
 
@@ -379,9 +501,8 @@ static inline int pvalidate(unsigned long vaddr, bool rmp_psize, bool validate)
 	return rc;
 }
 
-struct snp_guest_request_ioctl;
-
 void setup_ghcb(void);
+void snp_register_ghcb_early(unsigned long paddr);
 void early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr,
 				  unsigned long npages);
 void early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr,
@@ -390,15 +511,91 @@ void snp_set_memory_shared(unsigned long vaddr, unsigned long npages);
 void snp_set_memory_private(unsigned long vaddr, unsigned long npages);
 void snp_set_wakeup_secondary_cpu(void);
 bool snp_init(struct boot_params *bp);
-void __noreturn snp_abort(void);
 void snp_dmi_setup(void);
-int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct snp_guest_request_ioctl *rio);
 int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call, struct svsm_attest_call *input);
 void snp_accept_memory(phys_addr_t start, phys_addr_t end);
 u64 snp_get_unsupported_features(u64 status);
 u64 sev_get_status(void);
 void sev_show_status(void);
-void snp_update_svsm_ca(void);
+int prepare_pte_enc(struct pte_enc_desc *d);
+void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot);
+void snp_kexec_finish(void);
+void snp_kexec_begin(void);
+
+int snp_msg_init(struct snp_msg_desc *mdesc, int vmpck_id);
+struct snp_msg_desc *snp_msg_alloc(void);
+void snp_msg_free(struct snp_msg_desc *mdesc);
+int snp_send_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req);
+
+int snp_svsm_vtpm_send_command(u8 *buffer);
+
+void __init snp_secure_tsc_prepare(void);
+void __init snp_secure_tsc_init(void);
+enum es_result savic_register_gpa(u64 gpa);
+enum es_result savic_unregister_gpa(u64 *gpa);
+u64 savic_ghcb_msr_read(u32 reg);
+void savic_ghcb_msr_write(u32 reg, u64 value);
+
+static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
+{
+	ghcb->save.sw_exit_code = 0;
+	__builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
+}
+
+/* I/O parameters for CPUID-related helpers */
+struct cpuid_leaf {
+	u32 fn;
+	u32 subfn;
+	u32 eax;
+	u32 ebx;
+	u32 ecx;
+	u32 edx;
+};
+
+int svsm_perform_msr_protocol(struct svsm_call *call);
+int __pi_svsm_perform_msr_protocol(struct svsm_call *call);
+int snp_cpuid(void (*cpuid_fn)(void *ctx, struct cpuid_leaf *leaf),
+	      void *ctx, struct cpuid_leaf *leaf);
+
+void svsm_issue_call(struct svsm_call *call, u8 *pending);
+int svsm_process_result_codes(struct svsm_call *call);
+
+void __noreturn sev_es_terminate(unsigned int set, unsigned int reason);
+enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
+				   struct es_em_ctxt *ctxt,
+				   u64 exit_code, u64 exit_info_1,
+				   u64 exit_info_2);
+
+bool sev_es_negotiate_protocol(void);
+bool sev_es_check_cpu_features(void);
+
+extern u16 ghcb_version;
+extern struct ghcb *boot_ghcb;
+extern bool sev_snp_needs_sfw;
+
+struct psc_desc {
+	enum psc_op op;
+	struct svsm_ca *ca;
+	u64 caa_pa;
+};
+
+static inline void sev_evict_cache(void *va, int npages)
+{
+	volatile u8 val __always_unused;
+	u8 *bytes = va;
+	int page_idx;
+
+	/*
+	 * For SEV guests, a read from the first/last cache-lines of a 4K page
+	 * using the guest key is sufficient to cause a flush of all cache-lines
+	 * associated with that 4K page without incurring all the overhead of a
+	 * full CLFLUSH sequence.
+	 */
+	for (page_idx = 0; page_idx < npages; page_idx++) {
+		val = bytes[page_idx * PAGE_SIZE];
+		val = bytes[page_idx * PAGE_SIZE + PAGE_SIZE - 1];
+	}
+}
 
 #else	/* !CONFIG_AMD_MEM_ENCRYPT */
 
@@ -407,7 +604,7 @@ static inline void sev_es_ist_enter(struct pt_regs *regs) { }
 static inline void sev_es_ist_exit(void) { }
 static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) { return 0; }
 static inline void sev_es_nmi_complete(void) { }
-static inline int sev_es_efi_map_ghcbs(pgd_t *pgd) { return 0; }
+static inline int sev_es_efi_map_ghcbs_cas(pgd_t *pgd) { return 0; }
 static inline void sev_enable(struct boot_params *bp) { }
 static inline int pvalidate(unsigned long vaddr, bool rmp_psize, bool validate) { return 0; }
 static inline int rmpadjust(unsigned long vaddr, bool rmp_psize, unsigned long attrs) { return 0; }
@@ -420,12 +617,7 @@ static inline void snp_set_memory_shared(unsigned long vaddr, unsigned long npag
 static inline void snp_set_memory_private(unsigned long vaddr, unsigned long npages) { }
 static inline void snp_set_wakeup_secondary_cpu(void) { }
 static inline bool snp_init(struct boot_params *bp) { return false; }
-static inline void snp_abort(void) { }
 static inline void snp_dmi_setup(void) { }
-static inline int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct snp_guest_request_ioctl *rio)
-{
-	return -ENOTTY;
-}
 static inline int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call, struct svsm_attest_call *input)
 {
 	return -ENOTTY;
@@ -434,22 +626,44 @@ static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { }
 static inline u64 snp_get_unsupported_features(u64 status) { return 0; }
 static inline u64 sev_get_status(void) { return 0; }
 static inline void sev_show_status(void) { }
-static inline void snp_update_svsm_ca(void) { }
+static inline int prepare_pte_enc(struct pte_enc_desc *d) { return 0; }
+static inline void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot) { }
+static inline void snp_kexec_finish(void) { }
+static inline void snp_kexec_begin(void) { }
+static inline int snp_msg_init(struct snp_msg_desc *mdesc, int vmpck_id) { return -1; }
+static inline struct snp_msg_desc *snp_msg_alloc(void) { return NULL; }
+static inline void snp_msg_free(struct snp_msg_desc *mdesc) { }
+static inline int snp_send_guest_request(struct snp_msg_desc *mdesc,
+					 struct snp_guest_req *req) { return -ENODEV; }
+static inline int snp_svsm_vtpm_send_command(u8 *buffer) { return -ENODEV; }
+static inline void __init snp_secure_tsc_prepare(void) { }
+static inline void __init snp_secure_tsc_init(void) { }
+static inline void sev_evict_cache(void *va, int npages) {}
+static inline enum es_result savic_register_gpa(u64 gpa) { return ES_UNSUPPORTED; }
+static inline enum es_result savic_unregister_gpa(u64 *gpa) { return ES_UNSUPPORTED; }
+static inline void savic_ghcb_msr_write(u32 reg, u64 value) { }
+static inline u64 savic_ghcb_msr_read(u32 reg) { return 0; }
 
 #endif	/* CONFIG_AMD_MEM_ENCRYPT */
 
 #ifdef CONFIG_KVM_AMD_SEV
 bool snp_probe_rmptable_info(void);
+int snp_rmptable_init(void);
 int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level);
 void snp_dump_hva_rmpentry(unsigned long address);
 int psmash(u64 pfn);
 int rmp_make_private(u64 pfn, u64 gpa, enum pg_level level, u32 asid, bool immutable);
 int rmp_make_shared(u64 pfn, enum pg_level level);
-void snp_leak_pages(u64 pfn, unsigned int npages);
+void __snp_leak_pages(u64 pfn, unsigned int npages, bool dump_rmp);
 void kdump_sev_callback(void);
 void snp_fixup_e820_tables(void);
+static inline void snp_leak_pages(u64 pfn, unsigned int pages)
+{
+	__snp_leak_pages(pfn, pages, true);
+}
 #else
 static inline bool snp_probe_rmptable_info(void) { return false; }
+static inline int snp_rmptable_init(void) { return -ENOSYS; }
 static inline int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level) { return -ENODEV; }
 static inline void snp_dump_hva_rmpentry(unsigned long address) {}
 static inline int psmash(u64 pfn) { return -ENODEV; }
@@ -459,6 +673,7 @@ static inline int rmp_make_private(u64 pfn, u64 gpa, enum pg_level level, u32 as
 	return -ENODEV;
 }
 static inline int rmp_make_shared(u64 pfn, enum pg_level level) { return -ENODEV; }
+static inline void __snp_leak_pages(u64 pfn, unsigned int npages, bool dump_rmp) {}
 static inline void snp_leak_pages(u64 pfn, unsigned int npages) {}
 static inline void kdump_sev_callback(void) { }
 static inline void snp_fixup_e820_tables(void) {}
diff --git a/arch/x86/include/asm/sgx.h b/arch/x86/include/asm/sgx.h
index 6a0069761508..04958459a7ca 100644
--- a/arch/x86/include/asm/sgx.h
+++ b/arch/x86/include/asm/sgx.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/**
+/*
  * Copyright(c) 2016-20 Intel Corporation.
  *
  * Intel Software Guard Extensions (SGX) support.
@@ -28,21 +28,22 @@
 #define SGX_CPUID_EPC_MASK	GENMASK(3, 0)
 
 enum sgx_encls_function {
-	ECREATE	= 0x00,
-	EADD	= 0x01,
-	EINIT	= 0x02,
-	EREMOVE	= 0x03,
-	EDGBRD	= 0x04,
-	EDGBWR	= 0x05,
-	EEXTEND	= 0x06,
-	ELDU	= 0x08,
-	EBLOCK	= 0x09,
-	EPA	= 0x0A,
-	EWB	= 0x0B,
-	ETRACK	= 0x0C,
-	EAUG	= 0x0D,
-	EMODPR	= 0x0E,
-	EMODT	= 0x0F,
+	ECREATE		= 0x00,
+	EADD		= 0x01,
+	EINIT		= 0x02,
+	EREMOVE		= 0x03,
+	EDGBRD		= 0x04,
+	EDGBWR		= 0x05,
+	EEXTEND		= 0x06,
+	ELDU		= 0x08,
+	EBLOCK		= 0x09,
+	EPA		= 0x0A,
+	EWB		= 0x0B,
+	ETRACK		= 0x0C,
+	EAUG		= 0x0D,
+	EMODPR		= 0x0E,
+	EMODT		= 0x0F,
+	EUPDATESVN	= 0x18,
 };
 
 /**
@@ -65,15 +66,19 @@ enum sgx_encls_function {
 
 /**
  * enum sgx_return_code - The return code type for ENCLS, ENCLU and ENCLV
- * %SGX_EPC_PAGE_CONFLICT:	Page is being written by other ENCLS function.
- * %SGX_NOT_TRACKED:		Previous ETRACK's shootdown sequence has not
+ * @SGX_EPC_PAGE_CONFLICT:	Page is being written by other ENCLS function.
+ * @SGX_NOT_TRACKED:		Previous ETRACK's shootdown sequence has not
  *				been completed yet.
- * %SGX_CHILD_PRESENT		SECS has child pages present in the EPC.
- * %SGX_INVALID_EINITTOKEN:	EINITTOKEN is invalid and enclave signer's
+ * @SGX_CHILD_PRESENT:		SECS has child pages present in the EPC.
+ * @SGX_INVALID_EINITTOKEN:	EINITTOKEN is invalid and enclave signer's
  *				public key does not match IA32_SGXLEPUBKEYHASH.
- * %SGX_PAGE_NOT_MODIFIABLE:	The EPC page cannot be modified because it
+ * @SGX_PAGE_NOT_MODIFIABLE:	The EPC page cannot be modified because it
  *				is in the PENDING or MODIFIED state.
- * %SGX_UNMASKED_EVENT:		An unmasked event, e.g. INTR, was received
+ * @SGX_INSUFFICIENT_ENTROPY:	Insufficient entropy in RNG.
+ * @SGX_NO_UPDATE:		EUPDATESVN could not update the CPUSVN because the
+ *				current SVN was not newer than CPUSVN. This is the most
+ *				common error code returned by EUPDATESVN.
+ * @SGX_UNMASKED_EVENT:		An unmasked event, e.g. INTR, was received
  */
 enum sgx_return_code {
 	SGX_EPC_PAGE_CONFLICT		= 7,
@@ -81,6 +86,8 @@ enum sgx_return_code {
 	SGX_CHILD_PRESENT		= 13,
 	SGX_INVALID_EINITTOKEN		= 16,
 	SGX_PAGE_NOT_MODIFIABLE		= 20,
+	SGX_INSUFFICIENT_ENTROPY	= 29,
+	SGX_NO_UPDATE			= 31,
 	SGX_UNMASKED_EVENT		= 128,
 };
 
@@ -89,7 +96,7 @@ enum sgx_return_code {
 
 /**
  * enum sgx_miscselect - additional information to an SSA frame
- * %SGX_MISC_EXINFO:	Report #PF or #GP to the SSA frame.
+ * @SGX_MISC_EXINFO:	Report #PF or #GP to the SSA frame.
  *
  * Save State Area (SSA) is a stack inside the enclave used to store processor
  * state when an exception or interrupt occurs. This enum defines additional
@@ -105,17 +112,17 @@ enum sgx_miscselect {
 #define SGX_SSA_MISC_EXINFO_SIZE	16
 
 /**
- * enum sgx_attributes - the attributes field in &struct sgx_secs
- * %SGX_ATTR_INIT:		Enclave can be entered (is initialized).
- * %SGX_ATTR_DEBUG:		Allow ENCLS(EDBGRD) and ENCLS(EDBGWR).
- * %SGX_ATTR_MODE64BIT:		Tell that this a 64-bit enclave.
- * %SGX_ATTR_PROVISIONKEY:      Allow to use provisioning keys for remote
+ * enum sgx_attribute - the attributes field in &struct sgx_secs
+ * @SGX_ATTR_INIT:		Enclave can be entered (is initialized).
+ * @SGX_ATTR_DEBUG:		Allow ENCLS(EDBGRD) and ENCLS(EDBGWR).
+ * @SGX_ATTR_MODE64BIT:		Tell that this a 64-bit enclave.
+ * @SGX_ATTR_PROVISIONKEY:      Allow to use provisioning keys for remote
  *				attestation.
- * %SGX_ATTR_KSS:		Allow to use key separation and sharing (KSS).
- * %SGX_ATTR_EINITTOKENKEY:	Allow to use token signing key that is used to
+ * @SGX_ATTR_KSS:		Allow to use key separation and sharing (KSS).
+ * @SGX_ATTR_EINITTOKENKEY:	Allow to use token signing key that is used to
  *				sign cryptographic tokens that can be passed to
  *				EINIT as an authorization to run an enclave.
- * %SGX_ATTR_ASYNC_EXIT_NOTIFY:	Allow enclaves to be notified after an
+ * @SGX_ATTR_ASYNC_EXIT_NOTIFY:	Allow enclaves to be notified after an
  *				asynchronous exit has occurred.
  */
 enum sgx_attribute {
@@ -188,7 +195,7 @@ struct sgx_secs {
 
 /**
  * enum sgx_tcs_flags - execution flags for TCS
- * %SGX_TCS_DBGOPTIN:	If enabled allows single-stepping and breakpoints
+ * @SGX_TCS_DBGOPTIN:	If enabled allows single-stepping and breakpoints
  *			inside an enclave. It is cleared by EADD but can
  *			be set later with EDBGWR.
  */
@@ -253,11 +260,11 @@ struct sgx_pageinfo {
 
 /**
  * enum sgx_page_type - bits in the SECINFO flags defining the page type
- * %SGX_PAGE_TYPE_SECS:	a SECS page
- * %SGX_PAGE_TYPE_TCS:	a TCS page
- * %SGX_PAGE_TYPE_REG:	a regular page
- * %SGX_PAGE_TYPE_VA:	a VA page
- * %SGX_PAGE_TYPE_TRIM:	a page in trimmed state
+ * @SGX_PAGE_TYPE_SECS:	a SECS page
+ * @SGX_PAGE_TYPE_TCS:	a TCS page
+ * @SGX_PAGE_TYPE_REG:	a regular page
+ * @SGX_PAGE_TYPE_VA:	a VA page
+ * @SGX_PAGE_TYPE_TRIM:	a page in trimmed state
  *
  * Make sure when making changes to this enum that its values can still fit
  * in the bitfield within &struct sgx_encl_page
@@ -275,14 +282,14 @@ enum sgx_page_type {
 
 /**
  * enum sgx_secinfo_flags - the flags field in &struct sgx_secinfo
- * %SGX_SECINFO_R:	allow read
- * %SGX_SECINFO_W:	allow write
- * %SGX_SECINFO_X:	allow execution
- * %SGX_SECINFO_SECS:	a SECS page
- * %SGX_SECINFO_TCS:	a TCS page
- * %SGX_SECINFO_REG:	a regular page
- * %SGX_SECINFO_VA:	a VA page
- * %SGX_SECINFO_TRIM:	a page in trimmed state
+ * @SGX_SECINFO_R:	allow read
+ * @SGX_SECINFO_W:	allow write
+ * @SGX_SECINFO_X:	allow execution
+ * @SGX_SECINFO_SECS:	a SECS page
+ * @SGX_SECINFO_TCS:	a TCS page
+ * @SGX_SECINFO_REG:	a regular page
+ * @SGX_SECINFO_VA:	a VA page
+ * @SGX_SECINFO_TRIM:	a page in trimmed state
  */
 enum sgx_secinfo_flags {
 	SGX_SECINFO_R			= BIT(0),
diff --git a/arch/x86/include/asm/shared/msr.h b/arch/x86/include/asm/shared/msr.h
index 1e6ec10b3a15..a20b1c08c99f 100644
--- a/arch/x86/include/asm/shared/msr.h
+++ b/arch/x86/include/asm/shared/msr.h
@@ -12,4 +12,19 @@ struct msr {
 	};
 };
 
+/*
+ * The kernel proper already defines rdmsr()/wrmsr(), but they are not for the
+ * boot kernel since they rely on tracepoint/exception handling infrastructure
+ * that's not available here.
+ */
+static inline void raw_rdmsr(unsigned int reg, struct msr *m)
+{
+	asm volatile("rdmsr" : "=a" (m->l), "=d" (m->h) : "c" (reg));
+}
+
+static inline void raw_wrmsr(unsigned int reg, const struct msr *m)
+{
+	asm volatile("wrmsr" : : "c" (reg), "a"(m->l), "d" (m->h) : "memory");
+}
+
 #endif /* _ASM_X86_SHARED_MSR_H */
diff --git a/arch/x86/include/asm/shared/tdx.h b/arch/x86/include/asm/shared/tdx.h
index fdfd41511b02..8bc074c8d7c6 100644
--- a/arch/x86/include/asm/shared/tdx.h
+++ b/arch/x86/include/asm/shared/tdx.h
@@ -13,20 +13,75 @@
 /* TDX module Call Leaf IDs */
 #define TDG_VP_VMCALL			0
 #define TDG_VP_INFO			1
+#define TDG_MR_RTMR_EXTEND		2
 #define TDG_VP_VEINFO_GET		3
 #define TDG_MR_REPORT			4
 #define TDG_MEM_PAGE_ACCEPT		6
+#define TDG_VM_RD			7
 #define TDG_VM_WR			8
 
-/* TDCS fields. To be used by TDG.VM.WR and TDG.VM.RD module calls */
+/* TDX attributes */
+#define TDX_ATTR_DEBUG_BIT		0
+#define TDX_ATTR_DEBUG			BIT_ULL(TDX_ATTR_DEBUG_BIT)
+#define TDX_ATTR_HGS_PLUS_PROF_BIT	4
+#define TDX_ATTR_HGS_PLUS_PROF		BIT_ULL(TDX_ATTR_HGS_PLUS_PROF_BIT)
+#define TDX_ATTR_PERF_PROF_BIT		5
+#define TDX_ATTR_PERF_PROF		BIT_ULL(TDX_ATTR_PERF_PROF_BIT)
+#define TDX_ATTR_PMT_PROF_BIT		6
+#define TDX_ATTR_PMT_PROF		BIT_ULL(TDX_ATTR_PMT_PROF_BIT)
+#define TDX_ATTR_ICSSD_BIT		16
+#define TDX_ATTR_ICSSD			BIT_ULL(TDX_ATTR_ICSSD_BIT)
+#define TDX_ATTR_LASS_BIT		27
+#define TDX_ATTR_LASS			BIT_ULL(TDX_ATTR_LASS_BIT)
+#define TDX_ATTR_SEPT_VE_DISABLE_BIT	28
+#define TDX_ATTR_SEPT_VE_DISABLE	BIT_ULL(TDX_ATTR_SEPT_VE_DISABLE_BIT)
+#define TDX_ATTR_MIGRTABLE_BIT		29
+#define TDX_ATTR_MIGRTABLE		BIT_ULL(TDX_ATTR_MIGRTABLE_BIT)
+#define TDX_ATTR_PKS_BIT		30
+#define TDX_ATTR_PKS			BIT_ULL(TDX_ATTR_PKS_BIT)
+#define TDX_ATTR_KL_BIT			31
+#define TDX_ATTR_KL			BIT_ULL(TDX_ATTR_KL_BIT)
+#define TDX_ATTR_TPA_BIT		62
+#define TDX_ATTR_TPA			BIT_ULL(TDX_ATTR_TPA_BIT)
+#define TDX_ATTR_PERFMON_BIT		63
+#define TDX_ATTR_PERFMON		BIT_ULL(TDX_ATTR_PERFMON_BIT)
+
+/* TDX TD-Scope Metadata. To be used by TDG.VM.WR and TDG.VM.RD */
+#define TDCS_CONFIG_FLAGS		0x1110000300000016
+#define TDCS_TD_CTLS			0x1110000300000017
 #define TDCS_NOTIFY_ENABLES		0x9100000000000010
+#define TDCS_TOPOLOGY_ENUM_CONFIGURED	0x9100000000000019
+
+/* TDCS_CONFIG_FLAGS bits */
+#define TDCS_CONFIG_FLEXIBLE_PENDING_VE	BIT_ULL(1)
+
+/* TDCS_TD_CTLS bits */
+#define TD_CTLS_PENDING_VE_DISABLE_BIT	0
+#define TD_CTLS_PENDING_VE_DISABLE	BIT_ULL(TD_CTLS_PENDING_VE_DISABLE_BIT)
+#define TD_CTLS_ENUM_TOPOLOGY_BIT	1
+#define TD_CTLS_ENUM_TOPOLOGY		BIT_ULL(TD_CTLS_ENUM_TOPOLOGY_BIT)
+#define TD_CTLS_VIRT_CPUID2_BIT		2
+#define TD_CTLS_VIRT_CPUID2		BIT_ULL(TD_CTLS_VIRT_CPUID2_BIT)
+#define TD_CTLS_REDUCE_VE_BIT		3
+#define TD_CTLS_REDUCE_VE		BIT_ULL(TD_CTLS_REDUCE_VE_BIT)
+#define TD_CTLS_LOCK_BIT		63
+#define TD_CTLS_LOCK			BIT_ULL(TD_CTLS_LOCK_BIT)
 
 /* TDX hypercall Leaf IDs */
+#define TDVMCALL_GET_TD_VM_CALL_INFO	0x10000
 #define TDVMCALL_MAP_GPA		0x10001
 #define TDVMCALL_GET_QUOTE		0x10002
 #define TDVMCALL_REPORT_FATAL_ERROR	0x10003
+#define TDVMCALL_SETUP_EVENT_NOTIFY_INTERRUPT 0x10004ULL
 
-#define TDVMCALL_STATUS_RETRY		1
+/*
+ * TDG.VP.VMCALL Status Codes (returned in R10)
+ */
+#define TDVMCALL_STATUS_SUCCESS		0x0000000000000000ULL
+#define TDVMCALL_STATUS_RETRY		0x0000000000000001ULL
+#define TDVMCALL_STATUS_INVALID_OPERAND	0x8000000000000000ULL
+#define TDVMCALL_STATUS_ALIGN_ERROR	0x8000000000000002ULL
+#define TDVMCALL_STATUS_SUBFUNC_UNSUPPORTED	0x8000000000000003ULL
 
 /*
  * Bitmasks of exposed registers (with VMM).
@@ -61,7 +116,7 @@
 #define TDX_PS_1G	2
 #define TDX_PS_NR	(TDX_PS_1G + 1)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/compiler_attributes.h>
 
@@ -132,5 +187,5 @@ static __always_inline u64 hcall_func(u64 exit_reason)
         return exit_reason;
 }
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 #endif /* _ASM_X86_SHARED_TDX_H */
diff --git a/arch/x86/include/asm/shstk.h b/arch/x86/include/asm/shstk.h
index 4cb77e004615..fc7dcec58fd4 100644
--- a/arch/x86/include/asm/shstk.h
+++ b/arch/x86/include/asm/shstk.h
@@ -2,7 +2,7 @@
 #ifndef _ASM_X86_SHSTK_H
 #define _ASM_X86_SHSTK_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/types.h>
 
 struct task_struct;
@@ -16,27 +16,31 @@ struct thread_shstk {
 
 long shstk_prctl(struct task_struct *task, int option, unsigned long arg2);
 void reset_thread_features(void);
-unsigned long shstk_alloc_thread_stack(struct task_struct *p, unsigned long clone_flags,
+unsigned long shstk_alloc_thread_stack(struct task_struct *p, u64 clone_flags,
 				       unsigned long stack_size);
 void shstk_free(struct task_struct *p);
 int setup_signal_shadow_stack(struct ksignal *ksig);
 int restore_signal_shadow_stack(void);
 int shstk_update_last_frame(unsigned long val);
 bool shstk_is_enabled(void);
+int shstk_pop(u64 *val);
+int shstk_push(u64 val);
 #else
 static inline long shstk_prctl(struct task_struct *task, int option,
 			       unsigned long arg2) { return -EINVAL; }
 static inline void reset_thread_features(void) {}
 static inline unsigned long shstk_alloc_thread_stack(struct task_struct *p,
-						     unsigned long clone_flags,
+						     u64 clone_flags,
 						     unsigned long stack_size) { return 0; }
 static inline void shstk_free(struct task_struct *p) {}
 static inline int setup_signal_shadow_stack(struct ksignal *ksig) { return 0; }
 static inline int restore_signal_shadow_stack(void) { return 0; }
 static inline int shstk_update_last_frame(unsigned long val) { return 0; }
 static inline bool shstk_is_enabled(void) { return false; }
+static inline int shstk_pop(u64 *val) { return -ENOTSUPP; }
+static inline int shstk_push(u64 val) { return -ENOTSUPP; }
 #endif /* CONFIG_X86_USER_SHADOW_STACK */
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ASM_X86_SHSTK_H */
diff --git a/arch/x86/include/asm/sighandling.h b/arch/x86/include/asm/sighandling.h
index e770c4fc47f4..8727c7e21dd1 100644
--- a/arch/x86/include/asm/sighandling.h
+++ b/arch/x86/include/asm/sighandling.h
@@ -24,4 +24,26 @@ int ia32_setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs);
 int x64_setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs);
 int x32_setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs);
 
+/*
+ * To prevent immediate repeat of single step trap on return from SIGTRAP
+ * handler if the trap flag (TF) is set without an external debugger attached,
+ * clear the software event flag in the augmented SS, ensuring no single-step
+ * trap is pending upon ERETU completion.
+ *
+ * Note, this function should be called in sigreturn() before the original
+ * state is restored to make sure the TF is read from the entry frame.
+ */
+static __always_inline void prevent_single_step_upon_eretu(struct pt_regs *regs)
+{
+	/*
+	 * If the trap flag (TF) is set, i.e., the sigreturn() SYSCALL instruction
+	 * is being single-stepped, do not clear the software event flag in the
+	 * augmented SS, thus a debugger won't skip over the following instruction.
+	 */
+#ifdef CONFIG_X86_FRED
+	if (!(regs->flags & X86_EFLAGS_TF))
+		regs->fred_ss.swevent = 0;
+#endif
+}
+
 #endif /* _ASM_X86_SIGHANDLING_H */
diff --git a/arch/x86/include/asm/signal.h b/arch/x86/include/asm/signal.h
index 4a4043ca6493..5c03aaa89014 100644
--- a/arch/x86/include/asm/signal.h
+++ b/arch/x86/include/asm/signal.h
@@ -2,7 +2,7 @@
 #ifndef _ASM_X86_SIGNAL_H
 #define _ASM_X86_SIGNAL_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/linkage.h>
 
 /* Most things should be clean enough to redefine this at will, if care
@@ -28,9 +28,9 @@ typedef struct {
 #define SA_IA32_ABI	0x02000000u
 #define SA_X32_ABI	0x01000000u
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #include <uapi/asm/signal.h>
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #define __ARCH_HAS_SA_RESTORER
 
@@ -83,8 +83,7 @@ static inline int __const_sigismember(sigset_t *set, int _sig)
 static inline int __gen_sigismember(sigset_t *set, int _sig)
 {
 	bool ret;
-	asm("btl %2,%1" CC_SET(c)
-	    : CC_OUT(c) (ret) : "m"(*set), "Ir"(_sig-1));
+	asm("btl %2,%1" : "=@ccc"(ret) : "m"(*set), "Ir"(_sig-1));
 	return ret;
 }
 
@@ -101,5 +100,5 @@ struct pt_regs;
 
 #endif /* !__i386__ */
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* _ASM_X86_SIGNAL_H */
diff --git a/arch/x86/include/asm/simd.h b/arch/x86/include/asm/simd.h
index a341c878e977..b8027b63cd7a 100644
--- a/arch/x86/include/asm/simd.h
+++ b/arch/x86/include/asm/simd.h
@@ -1,6 +1,10 @@
 /* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SIMD_H
+#define _ASM_SIMD_H
 
 #include <asm/fpu/api.h>
+#include <linux/compiler_attributes.h>
+#include <linux/types.h>
 
 /*
  * may_use_simd - whether it is allowable at this time to issue SIMD
@@ -10,3 +14,5 @@ static __must_check inline bool may_use_simd(void)
 {
 	return irq_fpu_usable();
 }
+
+#endif	/* _ASM_SIMD_H */
diff --git a/arch/x86/include/asm/smap.h b/arch/x86/include/asm/smap.h
index bab490379c65..977bef14a0ab 100644
--- a/arch/x86/include/asm/smap.h
+++ b/arch/x86/include/asm/smap.h
@@ -13,30 +13,63 @@
 #include <asm/cpufeatures.h>
 #include <asm/alternative.h>
 
-/* "Raw" instruction opcodes */
-#define __ASM_CLAC	".byte 0x0f,0x01,0xca"
-#define __ASM_STAC	".byte 0x0f,0x01,0xcb"
-
-#ifdef __ASSEMBLY__
+#ifdef __ASSEMBLER__
 
 #define ASM_CLAC \
-	ALTERNATIVE "", __ASM_CLAC, X86_FEATURE_SMAP
+	ALTERNATIVE "", "clac", X86_FEATURE_SMAP
 
 #define ASM_STAC \
-	ALTERNATIVE "", __ASM_STAC, X86_FEATURE_SMAP
+	ALTERNATIVE "", "stac", X86_FEATURE_SMAP
+
+#else /* __ASSEMBLER__ */
 
-#else /* __ASSEMBLY__ */
+/*
+ * The CLAC/STAC instructions toggle the enforcement of
+ * X86_FEATURE_SMAP along with X86_FEATURE_LASS.
+ *
+ * SMAP enforcement is based on the _PAGE_BIT_USER bit in the page
+ * tables. The kernel is not allowed to touch pages with that bit set
+ * unless the AC bit is set.
+ *
+ * Use stac()/clac() when accessing userspace (_PAGE_USER) mappings,
+ * regardless of location.
+ *
+ * Note: a barrier is implicit in alternative().
+ */
 
 static __always_inline void clac(void)
 {
-	/* Note: a barrier is implicit in alternative() */
-	alternative("", __ASM_CLAC, X86_FEATURE_SMAP);
+	alternative("", "clac", X86_FEATURE_SMAP);
 }
 
 static __always_inline void stac(void)
 {
-	/* Note: a barrier is implicit in alternative() */
-	alternative("", __ASM_STAC, X86_FEATURE_SMAP);
+	alternative("", "stac", X86_FEATURE_SMAP);
+}
+
+/*
+ * LASS enforcement is based on bit 63 of the virtual address. The
+ * kernel is not allowed to touch memory in the lower half of the
+ * virtual address space.
+ *
+ * Use lass_stac()/lass_clac() to toggle the AC bit for kernel data
+ * accesses (!_PAGE_USER) that are blocked by LASS, but not by SMAP.
+ *
+ * Even with the AC bit set, LASS will continue to block instruction
+ * fetches from the user half of the address space. To allow those,
+ * clear CR4.LASS to disable the LASS mechanism entirely.
+ *
+ * Note: a barrier is implicit in alternative().
+ */
+
+static __always_inline void lass_clac(void)
+{
+	alternative("", "clac", X86_FEATURE_LASS);
+}
+
+static __always_inline void lass_stac(void)
+{
+	alternative("", "stac", X86_FEATURE_LASS);
 }
 
 static __always_inline unsigned long smap_save(void)
@@ -44,7 +77,8 @@ static __always_inline unsigned long smap_save(void)
 	unsigned long flags;
 
 	asm volatile ("# smap_save\n\t"
-		      ALTERNATIVE("", "pushf; pop %0; " __ASM_CLAC "\n\t",
+		      ALTERNATIVE(ANNOTATE_IGNORE_ALTERNATIVE "\n\t"
+				  "", "pushf; pop %0; clac",
 				  X86_FEATURE_SMAP)
 		      : "=rm" (flags) : : "memory", "cc");
 
@@ -54,17 +88,23 @@ static __always_inline unsigned long smap_save(void)
 static __always_inline void smap_restore(unsigned long flags)
 {
 	asm volatile ("# smap_restore\n\t"
-		      ALTERNATIVE("", "push %0; popf\n\t",
+		      ALTERNATIVE(ANNOTATE_IGNORE_ALTERNATIVE "\n\t"
+				  "", "push %0; popf",
 				  X86_FEATURE_SMAP)
 		      : : "g" (flags) : "memory", "cc");
 }
 
 /* These macros can be used in asm() statements */
 #define ASM_CLAC \
-	ALTERNATIVE("", __ASM_CLAC, X86_FEATURE_SMAP)
+	ALTERNATIVE("", "clac", X86_FEATURE_SMAP)
 #define ASM_STAC \
-	ALTERNATIVE("", __ASM_STAC, X86_FEATURE_SMAP)
+	ALTERNATIVE("", "stac", X86_FEATURE_SMAP)
+
+#define ASM_CLAC_UNSAFE \
+	ALTERNATIVE("", ANNOTATE_IGNORE_ALTERNATIVE "\n\t" "clac", X86_FEATURE_SMAP)
+#define ASM_STAC_UNSAFE \
+	ALTERNATIVE("", ANNOTATE_IGNORE_ALTERNATIVE "\n\t" "stac", X86_FEATURE_SMAP)
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ASM_X86_SMAP_H */
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index ca073f40698f..84951572ab81 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -1,12 +1,13 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #ifndef _ASM_X86_SMP_H
 #define _ASM_X86_SMP_H
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/cpumask.h>
+#include <linux/thread_info.h>
 
 #include <asm/cpumask.h>
-#include <asm/current.h>
-#include <asm/thread_info.h>
+
+DECLARE_PER_CPU_CACHE_HOT(int, cpu_number);
 
 DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
 DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
@@ -108,19 +109,21 @@ int common_cpu_up(unsigned int cpunum, struct task_struct *tidle);
 int native_kick_ap(unsigned int cpu, struct task_struct *tidle);
 int native_cpu_disable(void);
 void __noreturn hlt_play_dead(void);
-void native_play_dead(void);
+void __noreturn native_play_dead(void);
 void play_dead_common(void);
 void wbinvd_on_cpu(int cpu);
-int wbinvd_on_all_cpus(void);
+void wbinvd_on_all_cpus(void);
+void wbinvd_on_cpus_mask(struct cpumask *cpus);
+void wbnoinvd_on_all_cpus(void);
+void wbnoinvd_on_cpus_mask(struct cpumask *cpus);
 
 void smp_kick_mwait_play_dead(void);
+void __noreturn mwait_play_dead(unsigned int eax_hint);
 
 void native_smp_send_reschedule(int cpu);
 void native_send_call_func_ipi(const struct cpumask *mask);
 void native_send_call_func_single_ipi(int cpu);
 
-void smp_store_cpu_info(int id);
-
 asmlinkage __visible void smp_reboot_interrupt(void);
 __visible void smp_reschedule_interrupt(struct pt_regs *regs);
 __visible void smp_call_function_interrupt(struct pt_regs *regs);
@@ -133,14 +136,8 @@ __visible void smp_call_function_single_interrupt(struct pt_regs *r);
  * This function is needed by all SMP systems. It must _always_ be valid
  * from the initial startup.
  */
-#define raw_smp_processor_id()  this_cpu_read(pcpu_hot.cpu_number)
-#define __smp_processor_id() __this_cpu_read(pcpu_hot.cpu_number)
-
-#ifdef CONFIG_X86_32
-extern int safe_smp_processor_id(void);
-#else
-# define safe_smp_processor_id()	smp_processor_id()
-#endif
+#define raw_smp_processor_id()  this_cpu_read(cpu_number)
+#define __smp_processor_id() __this_cpu_read(cpu_number)
 
 static inline struct cpumask *cpu_llc_shared_mask(int cpu)
 {
@@ -154,16 +151,32 @@ static inline struct cpumask *cpu_l2c_shared_mask(int cpu)
 
 #else /* !CONFIG_SMP */
 #define wbinvd_on_cpu(cpu)     wbinvd()
-static inline int wbinvd_on_all_cpus(void)
+static inline void wbinvd_on_all_cpus(void)
+{
+	wbinvd();
+}
+
+static inline void wbinvd_on_cpus_mask(struct cpumask *cpus)
 {
 	wbinvd();
-	return 0;
+}
+
+static inline void wbnoinvd_on_all_cpus(void)
+{
+	wbnoinvd();
+}
+
+static inline void wbnoinvd_on_cpus_mask(struct cpumask *cpus)
+{
+	wbnoinvd();
 }
 
 static inline struct cpumask *cpu_llc_shared_mask(int cpu)
 {
 	return (struct cpumask *)cpumask_of(0);
 }
+
+static inline void __noreturn mwait_play_dead(unsigned int eax_hint) { BUG(); }
 #endif /* CONFIG_SMP */
 
 #ifdef CONFIG_DEBUG_NMI_SELFTEST
@@ -175,7 +188,7 @@ extern void nmi_selftest(void);
 extern unsigned int smpboot_control;
 extern unsigned long apic_mmio_base;
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 /* Control bits for startup_64 */
 #define STARTUP_READ_APICID	0x80000000
diff --git a/arch/x86/include/asm/spec-ctrl.h b/arch/x86/include/asm/spec-ctrl.h
index 658b690b2ccb..00b7e0398210 100644
--- a/arch/x86/include/asm/spec-ctrl.h
+++ b/arch/x86/include/asm/spec-ctrl.h
@@ -84,7 +84,7 @@ static inline u64 ssbd_tif_to_amd_ls_cfg(u64 tifn)
 static __always_inline void __update_spec_ctrl(u64 val)
 {
 	__this_cpu_write(x86_spec_ctrl_current, val);
-	native_wrmsrl(MSR_IA32_SPEC_CTRL, val);
+	native_wrmsrq(MSR_IA32_SPEC_CTRL, val);
 }
 
 #ifdef CONFIG_SMP
diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h
index aec6e2d3aa1d..46aa2c9c1bda 100644
--- a/arch/x86/include/asm/special_insns.h
+++ b/arch/x86/include/asm/special_insns.h
@@ -10,30 +10,19 @@
 #include <linux/irqflags.h>
 #include <linux/jump_label.h>
 
-/*
- * The compiler should not reorder volatile asm statements with respect to each
- * other: they should execute in program order. However GCC 4.9.x and 5.x have
- * a bug (which was fixed in 8.1, 7.3 and 6.5) where they might reorder
- * volatile asm. The write functions are not affected since they have memory
- * clobbers preventing reordering. To prevent reads from being reordered with
- * respect to writes, use a dummy memory operand.
- */
-
-#define __FORCE_ORDER "m"(*(unsigned int *)0x1000UL)
-
 void native_write_cr0(unsigned long val);
 
 static inline unsigned long native_read_cr0(void)
 {
 	unsigned long val;
-	asm volatile("mov %%cr0,%0\n\t" : "=r" (val) : __FORCE_ORDER);
+	asm volatile("mov %%cr0,%0" : "=r" (val));
 	return val;
 }
 
 static __always_inline unsigned long native_read_cr2(void)
 {
 	unsigned long val;
-	asm volatile("mov %%cr2,%0\n\t" : "=r" (val) : __FORCE_ORDER);
+	asm volatile("mov %%cr2,%0" : "=r" (val));
 	return val;
 }
 
@@ -42,14 +31,14 @@ static __always_inline void native_write_cr2(unsigned long val)
 	asm volatile("mov %0,%%cr2": : "r" (val) : "memory");
 }
 
-static inline unsigned long __native_read_cr3(void)
+static __always_inline unsigned long __native_read_cr3(void)
 {
 	unsigned long val;
-	asm volatile("mov %%cr3,%0\n\t" : "=r" (val) : __FORCE_ORDER);
+	asm volatile("mov %%cr3,%0" : "=r" (val));
 	return val;
 }
 
-static inline void native_write_cr3(unsigned long val)
+static __always_inline void native_write_cr3(unsigned long val)
 {
 	asm volatile("mov %0,%%cr3": : "r" (val) : "memory");
 }
@@ -66,10 +55,10 @@ static inline unsigned long native_read_cr4(void)
 	asm volatile("1: mov %%cr4, %0\n"
 		     "2:\n"
 		     _ASM_EXTABLE(1b, 2b)
-		     : "=r" (val) : "0" (0), __FORCE_ORDER);
+		     : "=r" (val) : "0" (0));
 #else
 	/* CR4 always exists on x86_64. */
-	asm volatile("mov %%cr4,%0\n\t" : "=r" (val) : __FORCE_ORDER);
+	asm volatile("mov %%cr4,%0" : "=r" (val));
 #endif
 	return val;
 }
@@ -86,9 +75,7 @@ static inline u32 rdpkru(void)
 	 * "rdpkru" instruction.  Places PKRU contents in to EAX,
 	 * clears EDX and requires that ecx=0.
 	 */
-	asm volatile(".byte 0x0f,0x01,0xee\n\t"
-		     : "=a" (pkru), "=d" (edx)
-		     : "c" (ecx));
+	asm volatile("rdpkru" : "=a" (pkru), "=d" (edx) : "c" (ecx));
 	return pkru;
 }
 
@@ -100,8 +87,7 @@ static inline void wrpkru(u32 pkru)
 	 * "wrpkru" instruction.  Loads contents in EAX to PKRU,
 	 * requires that ecx = edx = 0.
 	 */
-	asm volatile(".byte 0x0f,0x01,0xef\n\t"
-		     : : "a" (pkru), "c"(ecx), "d"(edx));
+	asm volatile("wrpkru" : : "a" (pkru), "c"(ecx), "d"(edx));
 }
 
 #else
@@ -115,9 +101,36 @@ static inline void wrpkru(u32 pkru)
 }
 #endif
 
-static __always_inline void native_wbinvd(void)
+/*
+ * Write back all modified lines in all levels of cache associated with this
+ * logical processor to main memory, and then invalidate all caches.  Depending
+ * on the micro-architecture, WBINVD (and WBNOINVD below) may or may not affect
+ * lower level caches associated with another logical processor that shares any
+ * level of this processor's cache hierarchy.
+ */
+static __always_inline void wbinvd(void)
+{
+	asm volatile("wbinvd" : : : "memory");
+}
+
+/* Instruction encoding provided for binutils backwards compatibility. */
+#define ASM_WBNOINVD _ASM_BYTES(0xf3,0x0f,0x09)
+
+/*
+ * Write back all modified lines in all levels of cache associated with this
+ * logical processor to main memory, but do NOT explicitly invalidate caches,
+ * i.e. leave all/most cache lines in the hierarchy in non-modified state.
+ */
+static __always_inline void wbnoinvd(void)
 {
-	asm volatile("wbinvd": : :"memory");
+	/*
+	 * Explicitly encode WBINVD if X86_FEATURE_WBNOINVD is unavailable even
+	 * though WBNOINVD is backwards compatible (it's simply WBINVD with an
+	 * ignored REP prefix), to guarantee that WBNOINVD isn't used if it
+	 * needs to be avoided for any reason.  For all supported usage in the
+	 * kernel, WBINVD is functionally a superset of WBNOINVD.
+	 */
+	alternative("wbinvd", ASM_WBNOINVD, X86_FEATURE_WBNOINVD);
 }
 
 static inline unsigned long __read_cr4(void)
@@ -167,12 +180,6 @@ static inline void __write_cr4(unsigned long x)
 {
 	native_write_cr4(x);
 }
-
-static __always_inline void wbinvd(void)
-{
-	native_wbinvd();
-}
-
 #endif /* CONFIG_PARAVIRT_XXL */
 
 static __always_inline void clflush(volatile void *__p)
@@ -182,9 +189,8 @@ static __always_inline void clflush(volatile void *__p)
 
 static inline void clflushopt(volatile void *__p)
 {
-	alternative_io(".byte 0x3e; clflush %0",
-		       ".byte 0x66; clflush %0",
-		       X86_FEATURE_CLFLUSHOPT,
+	alternative_io("ds clflush %0",
+		       "clflushopt %0", X86_FEATURE_CLFLUSHOPT,
 		       "+m" (*(volatile char __force *)__p));
 }
 
@@ -192,14 +198,11 @@ static inline void clwb(volatile void *__p)
 {
 	volatile struct { char x[64]; } *p = __p;
 
-	asm volatile(ALTERNATIVE_2(
-		".byte 0x3e; clflush (%[pax])",
-		".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */
-		X86_FEATURE_CLFLUSHOPT,
-		".byte 0x66, 0x0f, 0xae, 0x30",  /* clwb (%%rax) */
-		X86_FEATURE_CLWB)
-		: [p] "+m" (*p)
-		: [pax] "a" (p));
+	asm_inline volatile(ALTERNATIVE_2(
+		"ds clflush %0",
+		"clflushopt %0", X86_FEATURE_CLFLUSHOPT,
+		"clwb %0", X86_FEATURE_CLWB)
+		: "+m" (*p));
 }
 
 #ifdef CONFIG_X86_USER_SHADOW_STACK
@@ -217,7 +220,7 @@ fail:
 
 #define nop() asm volatile ("nop")
 
-static inline void serialize(void)
+static __always_inline void serialize(void)
 {
 	/* Instruction opcode for SERIALIZE; supported in binutils >= 2.35. */
 	asm volatile(".byte 0xf, 0x1, 0xe8" ::: "memory");
@@ -281,8 +284,7 @@ static inline int enqcmds(void __iomem *dst, const void *src)
 	 * See movdir64b()'s comment on operand specification.
 	 */
 	asm volatile(".byte 0xf3, 0x0f, 0x38, 0xf8, 0x02, 0x66, 0x90"
-		     CC_SET(z)
-		     : CC_OUT(z) (zf), "+m" (*__dst)
+		     : "=@ccz" (zf), "+m" (*__dst)
 		     : "m" (*__src), "a" (__dst), "d" (__src));
 
 	/* Submission failure is indicated via EFLAGS.ZF=1 */
diff --git a/arch/x86/include/asm/sta2x11.h b/arch/x86/include/asm/sta2x11.h
deleted file mode 100644
index e0975e9c4f47..000000000000
--- a/arch/x86/include/asm/sta2x11.h
+++ /dev/null
@@ -1,13 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Header file for STMicroelectronics ConneXt (STA2X11) IOHub
- */
-#ifndef __ASM_STA2X11_H
-#define __ASM_STA2X11_H
-
-#include <linux/pci.h>
-
-/* This needs to be called from the MFD to configure its sub-devices */
-struct sta2x11_instance *sta2x11_get_instance(struct pci_dev *pdev);
-
-#endif /* __ASM_STA2X11_H */
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 00473a650f51..cd761b14eb02 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -2,26 +2,10 @@
 /*
  * GCC stack protector support.
  *
- * Stack protector works by putting predefined pattern at the start of
+ * Stack protector works by putting a predefined pattern at the start of
  * the stack frame and verifying that it hasn't been overwritten when
- * returning from the function.  The pattern is called stack canary
- * and unfortunately gcc historically required it to be at a fixed offset
- * from the percpu segment base.  On x86_64, the offset is 40 bytes.
- *
- * The same segment is shared by percpu area and stack canary.  On
- * x86_64, percpu symbols are zero based and %gs (64-bit) points to the
- * base of percpu area.  The first occupant of the percpu area is always
- * fixed_percpu_data which contains stack_canary at the appropriate
- * offset.  On x86_32, the stack canary is just a regular percpu
- * variable.
- *
- * Putting percpu data in %fs on 32-bit is a minor optimization compared to
- * using %gs.  Since 32-bit userspace normally has %fs == 0, we are likely
- * to load 0 into %fs on exit to usermode, whereas with percpu data in
- * %gs, we are likely to load a non-null %gs on return to user mode.
- *
- * Once we are willing to require GCC 8.1 or better for 64-bit stackprotector
- * support, we can remove some of this complexity.
+ * returning from the function.  The pattern is called the stack canary
+ * and is a unique value for each task.
  */
 
 #ifndef _ASM_STACKPROTECTOR_H
@@ -36,6 +20,8 @@
 
 #include <linux/sched.h>
 
+DECLARE_PER_CPU_CACHE_HOT(unsigned long, __stack_chk_guard);
+
 /*
  * Initialize the stackprotector canary value.
  *
@@ -51,25 +37,13 @@ static __always_inline void boot_init_stack_canary(void)
 {
 	unsigned long canary = get_random_canary();
 
-#ifdef CONFIG_X86_64
-	BUILD_BUG_ON(offsetof(struct fixed_percpu_data, stack_canary) != 40);
-#endif
-
 	current->stack_canary = canary;
-#ifdef CONFIG_X86_64
-	this_cpu_write(fixed_percpu_data.stack_canary, canary);
-#else
 	this_cpu_write(__stack_chk_guard, canary);
-#endif
 }
 
 static inline void cpu_init_stack_canary(int cpu, struct task_struct *idle)
 {
-#ifdef CONFIG_X86_64
-	per_cpu(fixed_percpu_data.stack_canary, cpu) = idle->stack_canary;
-#else
 	per_cpu(__stack_chk_guard, cpu) = idle->stack_canary;
-#endif
 }
 
 #else	/* STACKPROTECTOR */
diff --git a/arch/x86/include/asm/static_call.h b/arch/x86/include/asm/static_call.h
index 125c407e2abe..4cd725a8fe91 100644
--- a/arch/x86/include/asm/static_call.h
+++ b/arch/x86/include/asm/static_call.h
@@ -36,7 +36,7 @@
 	    ".align 4						\n"	\
 	    ".globl " STATIC_CALL_TRAMP_STR(name) "		\n"	\
 	    STATIC_CALL_TRAMP_STR(name) ":			\n"	\
-	    ANNOTATE_NOENDBR						\
+	    ANNOTATE_NOENDBR "					\n"	\
 	    insns "						\n"	\
 	    ".byte 0x0f, 0xb9, 0xcc				\n"	\
 	    ".type " STATIC_CALL_TRAMP_STR(name) ", @function	\n"	\
@@ -65,4 +65,19 @@
 
 extern bool __static_call_fixup(void *tramp, u8 op, void *dest);
 
+extern void __static_call_update_early(void *tramp, void *func);
+
+#define static_call_update_early(name, _func)				\
+({									\
+	typeof(&STATIC_CALL_TRAMP(name)) __F = (_func);			\
+	if (static_call_initialized) {					\
+		__static_call_update(&STATIC_CALL_KEY(name),		\
+				     STATIC_CALL_TRAMP_ADDR(name), __F);\
+	} else {							\
+		WRITE_ONCE(STATIC_CALL_KEY(name).func, _func);		\
+		__static_call_update_early(STATIC_CALL_TRAMP_ADDR(name),\
+					   __F);			\
+	}								\
+})
+
 #endif /* _ASM_STATIC_CALL_H */
diff --git a/arch/x86/include/asm/string.h b/arch/x86/include/asm/string.h
index c3c2c1914d65..9cb5aae7fba9 100644
--- a/arch/x86/include/asm/string.h
+++ b/arch/x86/include/asm/string.h
@@ -1,6 +1,32 @@
 /* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_STRING_H
+#define _ASM_X86_STRING_H
+
 #ifdef CONFIG_X86_32
 # include <asm/string_32.h>
 #else
 # include <asm/string_64.h>
 #endif
+
+static __always_inline void *__inline_memcpy(void *to, const void *from, size_t len)
+{
+	void *ret = to;
+
+	asm volatile("rep movsb"
+		     : "+D" (to), "+S" (from), "+c" (len)
+		     : : "memory");
+	return ret;
+}
+
+static __always_inline void *__inline_memset(void *s, int v, size_t n)
+{
+	void *ret = s;
+
+	asm volatile("rep stosb"
+		     : "+D" (s), "+c" (n)
+		     : "a" ((uint8_t)v)
+		     : "memory");
+	return ret;
+}
+
+#endif /* _ASM_X86_STRING_H */
diff --git a/arch/x86/include/asm/string_32.h b/arch/x86/include/asm/string_32.h
index 32c0d981a82a..e9cce169bb4c 100644
--- a/arch/x86/include/asm/string_32.h
+++ b/arch/x86/include/asm/string_32.h
@@ -33,11 +33,11 @@ extern size_t strlen(const char *s);
 static __always_inline void *__memcpy(void *to, const void *from, size_t n)
 {
 	int d0, d1, d2;
-	asm volatile("rep ; movsl\n\t"
+	asm volatile("rep movsl\n\t"
 		     "movl %4,%%ecx\n\t"
 		     "andl $3,%%ecx\n\t"
 		     "jz 1f\n\t"
-		     "rep ; movsb\n\t"
+		     "rep movsb\n\t"
 		     "1:"
 		     : "=&c" (d0), "=&D" (d1), "=&S" (d2)
 		     : "0" (n / 4), "g" (n), "1" ((long)to), "2" ((long)from)
@@ -89,7 +89,7 @@ static __always_inline void *__constant_memcpy(void *to, const void *from,
 	if (n >= 5 * 4) {
 		/* large block: use rep prefix */
 		int ecx;
-		asm volatile("rep ; movsl"
+		asm volatile("rep movsl"
 			     : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
 			     : "0" (n / 4), "1" (edi), "2" (esi)
 			     : "memory"
@@ -165,8 +165,7 @@ extern void *memchr(const void *cs, int c, size_t count);
 static inline void *__memset_generic(void *s, char c, size_t count)
 {
 	int d0, d1;
-	asm volatile("rep\n\t"
-		     "stosb"
+	asm volatile("rep stosb"
 		     : "=&c" (d0), "=&D" (d1)
 		     : "a" (c), "1" (s), "0" (count)
 		     : "memory");
@@ -199,8 +198,7 @@ extern void *memset(void *, int, size_t);
 static inline void *memset16(uint16_t *s, uint16_t v, size_t n)
 {
 	int d0, d1;
-	asm volatile("rep\n\t"
-		     "stosw"
+	asm volatile("rep stosw"
 		     : "=&c" (d0), "=&D" (d1)
 		     : "a" (v), "1" (s), "0" (n)
 		     : "memory");
@@ -211,8 +209,7 @@ static inline void *memset16(uint16_t *s, uint16_t v, size_t n)
 static inline void *memset32(uint32_t *s, uint32_t v, size_t n)
 {
 	int d0, d1;
-	asm volatile("rep\n\t"
-		     "stosl"
+	asm volatile("rep stosl"
 		     : "=&c" (d0), "=&D" (d1)
 		     : "a" (v), "1" (s), "0" (n)
 		     : "memory");
diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h
index 9d0b324eab21..4635616863f5 100644
--- a/arch/x86/include/asm/string_64.h
+++ b/arch/x86/include/asm/string_64.h
@@ -21,6 +21,7 @@ extern void *__memcpy(void *to, const void *from, size_t len);
 #define __HAVE_ARCH_MEMSET
 void *memset(void *s, int c, size_t n);
 void *__memset(void *s, int c, size_t n);
+KCFI_REFERENCE(__memset);
 
 /*
  * KMSAN needs to instrument as much code as possible. Use C versions of
@@ -30,7 +31,7 @@ void *__memset(void *s, int c, size_t n);
 #define __HAVE_ARCH_MEMSET16
 static inline void *memset16(uint16_t *s, uint16_t v, size_t n)
 {
-	const __auto_type s0 = s;
+	const auto s0 = s;
 	asm volatile (
 		"rep stosw"
 		: "+D" (s), "+c" (n)
@@ -43,7 +44,7 @@ static inline void *memset16(uint16_t *s, uint16_t v, size_t n)
 #define __HAVE_ARCH_MEMSET32
 static inline void *memset32(uint32_t *s, uint32_t v, size_t n)
 {
-	const __auto_type s0 = s;
+	const auto s0 = s;
 	asm volatile (
 		"rep stosl"
 		: "+D" (s), "+c" (n)
@@ -56,7 +57,7 @@ static inline void *memset32(uint32_t *s, uint32_t v, size_t n)
 #define __HAVE_ARCH_MEMSET64
 static inline void *memset64(uint64_t *s, uint64_t v, size_t n)
 {
-	const __auto_type s0 = s;
+	const auto s0 = s;
 	asm volatile (
 		"rep stosq"
 		: "+D" (s), "+c" (n)
@@ -70,6 +71,7 @@ static inline void *memset64(uint64_t *s, uint64_t v, size_t n)
 #define __HAVE_ARCH_MEMMOVE
 void *memmove(void *dest, const void *src, size_t count);
 void *__memmove(void *dest, const void *src, size_t count);
+KCFI_REFERENCE(__memmove);
 
 int memcmp(const void *cs, const void *ct, size_t count);
 size_t strlen(const char *s);
diff --git a/arch/x86/include/asm/suspend_32.h b/arch/x86/include/asm/suspend_32.h
index d8416b3bf832..e8e5aab06255 100644
--- a/arch/x86/include/asm/suspend_32.h
+++ b/arch/x86/include/asm/suspend_32.h
@@ -9,6 +9,7 @@
 
 #include <asm/desc.h>
 #include <asm/fpu/api.h>
+#include <asm/msr.h>
 
 /* image of the saved processor state */
 struct saved_context {
diff --git a/arch/x86/include/asm/suspend_64.h b/arch/x86/include/asm/suspend_64.h
index 54df06687d83..b512f9665f78 100644
--- a/arch/x86/include/asm/suspend_64.h
+++ b/arch/x86/include/asm/suspend_64.h
@@ -9,6 +9,7 @@
 
 #include <asm/desc.h>
 #include <asm/fpu/api.h>
+#include <asm/msr.h>
 
 /*
  * Image of the saved processor state, used by the low level ACPI suspend to
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h
index 2b59b9951c90..56aa99503dc4 100644
--- a/arch/x86/include/asm/svm.h
+++ b/arch/x86/include/asm/svm.h
@@ -5,7 +5,7 @@
 #include <uapi/asm/svm.h>
 #include <uapi/asm/kvm.h>
 
-#include <asm/hyperv-tlfs.h>
+#include <hyperv/hvhdk.h>
 
 /*
  * 32-bit intercept words in the VMCB Control Area, starting
@@ -116,6 +116,8 @@ enum {
 	INTERCEPT_INVPCID,
 	INTERCEPT_MCOMMIT,
 	INTERCEPT_TLBSYNC,
+	INTERCEPT_BUSLOCK,
+	INTERCEPT_IDLE_HLT = 166,
 };
 
 
@@ -158,7 +160,12 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
 	u64 avic_physical_id;	/* Offset 0xf8 */
 	u8 reserved_7[8];
 	u64 vmsa_pa;		/* Used for an SEV-ES guest */
-	u8 reserved_8[720];
+	u8 reserved_8[16];
+	u16 bus_lock_counter;		/* Offset 0x120 */
+	u8 reserved_9[22];
+	u64 allowed_sev_features;	/* Offset 0x138 */
+	u64 guest_sev_features;		/* Offset 0x140 */
+	u8 reserved_10[664];
 	/*
 	 * Offset 0x3e0, 32 bytes reserved
 	 * for use by hypervisor/software.
@@ -245,16 +252,21 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
 #define AVIC_LOGICAL_ID_ENTRY_VALID_BIT			31
 #define AVIC_LOGICAL_ID_ENTRY_VALID_MASK		(1 << 31)
 
+/*
+ * GA_LOG_INTR is a synthetic flag that's never propagated to hardware-visible
+ * tables.  GA_LOG_INTR is set if the vCPU needs device posted IRQs to generate
+ * GA log interrupts to wake the vCPU (because it's blocking or about to block).
+ */
+#define AVIC_PHYSICAL_ID_ENTRY_GA_LOG_INTR		BIT_ULL(61)
+
 #define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK	GENMASK_ULL(11, 0)
-#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK	(0xFFFFFFFFFFULL << 12)
+#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK	GENMASK_ULL(51, 12)
 #define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK		(1ULL << 62)
 #define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK		(1ULL << 63)
 #define AVIC_PHYSICAL_ID_TABLE_SIZE_MASK		(0xFFULL)
 
 #define AVIC_DOORBELL_PHYSICAL_ID_MASK			GENMASK_ULL(11, 0)
 
-#define VMCB_AVIC_APIC_BAR_MASK				0xFFFFFFFFFF000ULL
-
 #define AVIC_UNACCEL_ACCESS_WRITE_MASK		1
 #define AVIC_UNACCEL_ACCESS_OFFSET_MASK		0xFF0
 #define AVIC_UNACCEL_ACCESS_VECTOR_MASK		0xFFFFFFFF
@@ -267,7 +279,7 @@ enum avic_ipi_failure_cause {
 	AVIC_IPI_FAILURE_INVALID_IPI_VECTOR,
 };
 
-#define AVIC_PHYSICAL_MAX_INDEX_MASK	GENMASK_ULL(8, 0)
+#define AVIC_PHYSICAL_MAX_INDEX_MASK	GENMASK_ULL(11, 0)
 
 /*
  * For AVIC, the max index allowed for physical APIC ID table is 0xfe (254), as
@@ -277,22 +289,22 @@ enum avic_ipi_failure_cause {
 
 /*
  * For x2AVIC, the max index allowed for physical APIC ID table is 0x1ff (511).
+ * With X86_FEATURE_X2AVIC_EXT, the max index is increased to 0xfff (4095).
  */
 #define X2AVIC_MAX_PHYSICAL_ID		0x1FFUL
+#define X2AVIC_4K_MAX_PHYSICAL_ID	0xFFFUL
 
 static_assert((AVIC_MAX_PHYSICAL_ID & AVIC_PHYSICAL_MAX_INDEX_MASK) == AVIC_MAX_PHYSICAL_ID);
 static_assert((X2AVIC_MAX_PHYSICAL_ID & AVIC_PHYSICAL_MAX_INDEX_MASK) == X2AVIC_MAX_PHYSICAL_ID);
-
-#define AVIC_HPA_MASK	~((0xFFFULL << 52) | 0xFFF)
+static_assert((X2AVIC_4K_MAX_PHYSICAL_ID & AVIC_PHYSICAL_MAX_INDEX_MASK) == X2AVIC_4K_MAX_PHYSICAL_ID);
 
 #define SVM_SEV_FEAT_SNP_ACTIVE				BIT(0)
 #define SVM_SEV_FEAT_RESTRICTED_INJECTION		BIT(3)
 #define SVM_SEV_FEAT_ALTERNATE_INJECTION		BIT(4)
 #define SVM_SEV_FEAT_DEBUG_SWAP				BIT(5)
+#define SVM_SEV_FEAT_SECURE_TSC				BIT(9)
 
-#define SVM_SEV_FEAT_INT_INJ_MODES		\
-	(SVM_SEV_FEAT_RESTRICTED_INJECTION |	\
-	 SVM_SEV_FEAT_ALTERNATE_INJECTION)
+#define VMCB_ALLOWED_SEV_FEATURES_VALID			BIT_ULL(63)
 
 struct vmcb_seg {
 	u16 selector;
@@ -417,7 +429,9 @@ struct sev_es_save_area {
 	u8 reserved_0x298[80];
 	u32 pkru;
 	u32 tsc_aux;
-	u8 reserved_0x2f0[24];
+	u64 tsc_scale;
+	u64 tsc_offset;
+	u8 reserved_0x300[8];
 	u64 rcx;
 	u64 rdx;
 	u64 rbx;
@@ -564,7 +578,7 @@ static inline void __unused_size_checks(void)
 	BUILD_BUG_RESERVED_OFFSET(sev_es_save_area, 0x1c0);
 	BUILD_BUG_RESERVED_OFFSET(sev_es_save_area, 0x248);
 	BUILD_BUG_RESERVED_OFFSET(sev_es_save_area, 0x298);
-	BUILD_BUG_RESERVED_OFFSET(sev_es_save_area, 0x2f0);
+	BUILD_BUG_RESERVED_OFFSET(sev_es_save_area, 0x300);
 	BUILD_BUG_RESERVED_OFFSET(sev_es_save_area, 0x320);
 	BUILD_BUG_RESERVED_OFFSET(sev_es_save_area, 0x380);
 	BUILD_BUG_RESERVED_OFFSET(sev_es_save_area, 0x3f0);
@@ -690,5 +704,6 @@ DEFINE_GHCB_ACCESSORS(sw_exit_info_1)
 DEFINE_GHCB_ACCESSORS(sw_exit_info_2)
 DEFINE_GHCB_ACCESSORS(sw_scratch)
 DEFINE_GHCB_ACCESSORS(xcr0)
+DEFINE_GHCB_ACCESSORS(xss)
 
 #endif
diff --git a/arch/x86/include/asm/switch_to.h b/arch/x86/include/asm/switch_to.h
index 75248546403d..499b1c15cc8b 100644
--- a/arch/x86/include/asm/switch_to.h
+++ b/arch/x86/include/asm/switch_to.h
@@ -52,6 +52,8 @@ do {									\
 } while (0)
 
 #ifdef CONFIG_X86_32
+#include <asm/msr.h>
+
 static inline void refresh_sysenter_cs(struct thread_struct *thread)
 {
 	/* Only happens when SEP is enabled, no need to test "SEP"arately: */
@@ -59,7 +61,7 @@ static inline void refresh_sysenter_cs(struct thread_struct *thread)
 		return;
 
 	this_cpu_write(cpu_tss_rw.x86_tss.ss1, thread->sysenter_cs);
-	wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0);
+	wrmsrq(MSR_IA32_SYSENTER_CS, thread->sysenter_cs);
 }
 #endif
 
diff --git a/arch/x86/include/asm/sync_bitops.h b/arch/x86/include/asm/sync_bitops.h
index 6d8d6bc183b7..cd21a0405ac5 100644
--- a/arch/x86/include/asm/sync_bitops.h
+++ b/arch/x86/include/asm/sync_bitops.h
@@ -31,7 +31,7 @@
  */
 static inline void sync_set_bit(long nr, volatile unsigned long *addr)
 {
-	asm volatile("lock; " __ASM_SIZE(bts) " %1,%0"
+	asm volatile("lock " __ASM_SIZE(bts) " %1,%0"
 		     : "+m" (ADDR)
 		     : "Ir" (nr)
 		     : "memory");
@@ -49,7 +49,7 @@ static inline void sync_set_bit(long nr, volatile unsigned long *addr)
  */
 static inline void sync_clear_bit(long nr, volatile unsigned long *addr)
 {
-	asm volatile("lock; " __ASM_SIZE(btr) " %1,%0"
+	asm volatile("lock " __ASM_SIZE(btr) " %1,%0"
 		     : "+m" (ADDR)
 		     : "Ir" (nr)
 		     : "memory");
@@ -66,7 +66,7 @@ static inline void sync_clear_bit(long nr, volatile unsigned long *addr)
  */
 static inline void sync_change_bit(long nr, volatile unsigned long *addr)
 {
-	asm volatile("lock; " __ASM_SIZE(btc) " %1,%0"
+	asm volatile("lock " __ASM_SIZE(btc) " %1,%0"
 		     : "+m" (ADDR)
 		     : "Ir" (nr)
 		     : "memory");
@@ -82,7 +82,7 @@ static inline void sync_change_bit(long nr, volatile unsigned long *addr)
  */
 static inline bool sync_test_and_set_bit(long nr, volatile unsigned long *addr)
 {
-	return GEN_BINARY_RMWcc("lock; " __ASM_SIZE(bts), *addr, c, "Ir", nr);
+	return GEN_BINARY_RMWcc("lock " __ASM_SIZE(bts), *addr, c, "Ir", nr);
 }
 
 /**
@@ -95,7 +95,7 @@ static inline bool sync_test_and_set_bit(long nr, volatile unsigned long *addr)
  */
 static inline int sync_test_and_clear_bit(long nr, volatile unsigned long *addr)
 {
-	return GEN_BINARY_RMWcc("lock; " __ASM_SIZE(btr), *addr, c, "Ir", nr);
+	return GEN_BINARY_RMWcc("lock " __ASM_SIZE(btr), *addr, c, "Ir", nr);
 }
 
 /**
@@ -108,7 +108,7 @@ static inline int sync_test_and_clear_bit(long nr, volatile unsigned long *addr)
  */
 static inline int sync_test_and_change_bit(long nr, volatile unsigned long *addr)
 {
-	return GEN_BINARY_RMWcc("lock; " __ASM_SIZE(btc), *addr, c, "Ir", nr);
+	return GEN_BINARY_RMWcc("lock " __ASM_SIZE(btc), *addr, c, "Ir", nr);
 }
 
 #define sync_test_bit(nr, addr) test_bit(nr, addr)
diff --git a/arch/x86/include/asm/sync_core.h b/arch/x86/include/asm/sync_core.h
index ab7382f92aff..96bda43538ee 100644
--- a/arch/x86/include/asm/sync_core.h
+++ b/arch/x86/include/asm/sync_core.h
@@ -8,7 +8,7 @@
 #include <asm/special_insns.h>
 
 #ifdef CONFIG_X86_32
-static inline void iret_to_self(void)
+static __always_inline void iret_to_self(void)
 {
 	asm volatile (
 		"pushfl\n\t"
@@ -19,7 +19,7 @@ static inline void iret_to_self(void)
 		: ASM_CALL_CONSTRAINT : : "memory");
 }
 #else
-static inline void iret_to_self(void)
+static __always_inline void iret_to_self(void)
 {
 	unsigned int tmp;
 
@@ -55,7 +55,7 @@ static inline void iret_to_self(void)
  * Like all of Linux's memory ordering operations, this is a
  * compiler barrier as well.
  */
-static inline void sync_core(void)
+static __always_inline void sync_core(void)
 {
 	/*
 	 * The SERIALIZE instruction is the most straightforward way to
diff --git a/arch/x86/include/asm/syscall.h b/arch/x86/include/asm/syscall.h
index 7c488ff0c764..c10dbb74cd00 100644
--- a/arch/x86/include/asm/syscall.h
+++ b/arch/x86/include/asm/syscall.h
@@ -38,6 +38,13 @@ static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs)
 	return regs->orig_ax;
 }
 
+static inline void syscall_set_nr(struct task_struct *task,
+				  struct pt_regs *regs,
+				  int nr)
+{
+	regs->orig_ax = nr;
+}
+
 static inline void syscall_rollback(struct task_struct *task,
 				    struct pt_regs *regs)
 {
@@ -90,6 +97,18 @@ static inline void syscall_get_arguments(struct task_struct *task,
 	args[5] = regs->bp;
 }
 
+static inline void syscall_set_arguments(struct task_struct *task,
+					 struct pt_regs *regs,
+					 const unsigned long *args)
+{
+	regs->bx = args[0];
+	regs->cx = args[1];
+	regs->dx = args[2];
+	regs->si = args[3];
+	regs->di = args[4];
+	regs->bp = args[5];
+}
+
 static inline int syscall_get_arch(struct task_struct *task)
 {
 	return AUDIT_ARCH_I386;
@@ -121,6 +140,30 @@ static inline void syscall_get_arguments(struct task_struct *task,
 	}
 }
 
+static inline void syscall_set_arguments(struct task_struct *task,
+					 struct pt_regs *regs,
+					 const unsigned long *args)
+{
+# ifdef CONFIG_IA32_EMULATION
+	if (task->thread_info.status & TS_COMPAT) {
+		regs->bx = *args++;
+		regs->cx = *args++;
+		regs->dx = *args++;
+		regs->si = *args++;
+		regs->di = *args++;
+		regs->bp = *args;
+	} else
+# endif
+	{
+		regs->di = *args++;
+		regs->si = *args++;
+		regs->dx = *args++;
+		regs->r10 = *args++;
+		regs->r8 = *args++;
+		regs->r9 = *args;
+	}
+}
+
 static inline int syscall_get_arch(struct task_struct *task)
 {
 	/* x32 tasks should be considered AUDIT_ARCH_X86_64. */
diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h
index eba178996d84..6b338d7f01b7 100644
--- a/arch/x86/include/asm/tdx.h
+++ b/arch/x86/include/asm/tdx.h
@@ -5,6 +5,7 @@
 
 #include <linux/init.h>
 #include <linux/bits.h>
+#include <linux/mmzone.h>
 
 #include <asm/errno.h>
 #include <asm/ptrace.h>
@@ -18,6 +19,7 @@
  * TDX module.
  */
 #define TDX_ERROR			_BITUL(63)
+#define TDX_NON_RECOVERABLE		_BITUL(62)
 #define TDX_SW_ERROR			(TDX_ERROR | GENMASK_ULL(47, 40))
 #define TDX_SEAMCALL_VMFAILINVALID	(TDX_SW_ERROR | _UL(0xFFFF0000))
 
@@ -30,9 +32,11 @@
 #define TDX_SUCCESS		0ULL
 #define TDX_RND_NO_ENTROPY	0x8000020300000000ULL
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <uapi/asm/mce.h>
+#include <asm/tdx_global_metadata.h>
+#include <linux/pgtable.h>
 
 /*
  * Used by the #VE exception handler to gather the #VE exception
@@ -58,18 +62,23 @@ void tdx_get_ve_info(struct ve_info *ve);
 
 bool tdx_handle_virt_exception(struct pt_regs *regs, struct ve_info *ve);
 
-void tdx_safe_halt(void);
+void tdx_halt(void);
 
 bool tdx_early_handle_ve(struct pt_regs *regs);
 
 int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport);
 
+int tdx_mcall_extend_rtmr(u8 index, u8 *data);
+
 u64 tdx_hcall_get_quote(u8 *buf, size_t size);
 
+void __init tdx_dump_attributes(u64 td_attr);
+void __init tdx_dump_td_ctls(u64 td_ctls);
+
 #else
 
 static inline void tdx_early_init(void) { };
-static inline void tdx_safe_halt(void) { };
+static inline void tdx_halt(void) { };
 
 static inline bool tdx_early_handle_ve(struct pt_regs *regs) { return false; }
 
@@ -93,18 +102,41 @@ u64 __seamcall_ret(u64 fn, struct tdx_module_args *args);
 u64 __seamcall_saved_ret(u64 fn, struct tdx_module_args *args);
 void tdx_init(void);
 
+#include <linux/preempt.h>
 #include <asm/archrandom.h>
+#include <asm/processor.h>
 
 typedef u64 (*sc_func_t)(u64 fn, struct tdx_module_args *args);
 
-static inline u64 sc_retry(sc_func_t func, u64 fn,
+static __always_inline u64 __seamcall_dirty_cache(sc_func_t func, u64 fn,
+						  struct tdx_module_args *args)
+{
+	lockdep_assert_preemption_disabled();
+
+	/*
+	 * SEAMCALLs are made to the TDX module and can generate dirty
+	 * cachelines of TDX private memory.  Mark cache state incoherent
+	 * so that the cache can be flushed during kexec.
+	 *
+	 * This needs to be done before actually making the SEAMCALL,
+	 * because kexec-ing CPU could send NMI to stop remote CPUs,
+	 * in which case even disabling IRQ won't help here.
+	 */
+	this_cpu_write(cache_state_incoherent, true);
+
+	return func(fn, args);
+}
+
+static __always_inline u64 sc_retry(sc_func_t func, u64 fn,
 			   struct tdx_module_args *args)
 {
 	int retry = RDRAND_RETRY_LOOPS;
 	u64 ret;
 
 	do {
-		ret = func(fn, args);
+		preempt_disable();
+		ret = __seamcall_dirty_cache(func, fn, args);
+		preempt_enable();
 	} while (ret == TDX_RND_NO_ENTROPY && --retry);
 
 	return ret;
@@ -116,12 +148,93 @@ static inline u64 sc_retry(sc_func_t func, u64 fn,
 int tdx_cpu_enable(void);
 int tdx_enable(void);
 const char *tdx_dump_mce_info(struct mce *m);
+const struct tdx_sys_info *tdx_get_sysinfo(void);
+
+int tdx_guest_keyid_alloc(void);
+u32 tdx_get_nr_guest_keyids(void);
+void tdx_guest_keyid_free(unsigned int keyid);
+
+void tdx_quirk_reset_page(struct page *page);
+
+struct tdx_td {
+	/* TD root structure: */
+	struct page *tdr_page;
+
+	int tdcs_nr_pages;
+	/* TD control structure: */
+	struct page **tdcs_pages;
+
+	/* Size of `tdcx_pages` in struct tdx_vp */
+	int tdcx_nr_pages;
+};
+
+struct tdx_vp {
+	/* TDVP root page */
+	struct page *tdvpr_page;
+	/* precalculated page_to_phys(tdvpr_page) for use in noinstr code */
+	phys_addr_t tdvpr_pa;
+
+	/* TD vCPU control structure: */
+	struct page **tdcx_pages;
+};
+
+static inline u64 mk_keyed_paddr(u16 hkid, struct page *page)
+{
+	u64 ret;
+
+	ret = page_to_phys(page);
+	/* KeyID bits are just above the physical address bits: */
+	ret |= (u64)hkid << boot_cpu_data.x86_phys_bits;
+
+	return ret;
+}
+
+static inline int pg_level_to_tdx_sept_level(enum pg_level level)
+{
+        WARN_ON_ONCE(level == PG_LEVEL_NONE);
+        return level - 1;
+}
+
+u64 tdh_vp_enter(struct tdx_vp *vp, struct tdx_module_args *args);
+u64 tdh_mng_addcx(struct tdx_td *td, struct page *tdcs_page);
+u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, struct page *page, struct page *source, u64 *ext_err1, u64 *ext_err2);
+u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2);
+u64 tdh_vp_addcx(struct tdx_vp *vp, struct page *tdcx_page);
+u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2);
+u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, int level, u64 *ext_err1, u64 *ext_err2);
+u64 tdh_mng_key_config(struct tdx_td *td);
+u64 tdh_mng_create(struct tdx_td *td, u16 hkid);
+u64 tdh_vp_create(struct tdx_td *td, struct tdx_vp *vp);
+u64 tdh_mng_rd(struct tdx_td *td, u64 field, u64 *data);
+u64 tdh_mr_extend(struct tdx_td *td, u64 gpa, u64 *ext_err1, u64 *ext_err2);
+u64 tdh_mr_finalize(struct tdx_td *td);
+u64 tdh_vp_flush(struct tdx_vp *vp);
+u64 tdh_mng_vpflushdone(struct tdx_td *td);
+u64 tdh_mng_key_freeid(struct tdx_td *td);
+u64 tdh_mng_init(struct tdx_td *td, u64 td_params, u64 *extended_err);
+u64 tdh_vp_init(struct tdx_vp *vp, u64 initial_rcx, u32 x2apicid);
+u64 tdh_vp_rd(struct tdx_vp *vp, u64 field, u64 *data);
+u64 tdh_vp_wr(struct tdx_vp *vp, u64 field, u64 data, u64 mask);
+u64 tdh_phymem_page_reclaim(struct page *page, u64 *tdx_pt, u64 *tdx_owner, u64 *tdx_size);
+u64 tdh_mem_track(struct tdx_td *tdr);
+u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, u64 level, u64 *ext_err1, u64 *ext_err2);
+u64 tdh_phymem_cache_wb(bool resume);
+u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td);
+u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page);
 #else
 static inline void tdx_init(void) { }
 static inline int tdx_cpu_enable(void) { return -ENODEV; }
 static inline int tdx_enable(void)  { return -ENODEV; }
+static inline u32 tdx_get_nr_guest_keyids(void) { return 0; }
 static inline const char *tdx_dump_mce_info(struct mce *m) { return NULL; }
+static inline const struct tdx_sys_info *tdx_get_sysinfo(void) { return NULL; }
 #endif	/* CONFIG_INTEL_TDX_HOST */
 
-#endif /* !__ASSEMBLY__ */
+#ifdef CONFIG_KEXEC_CORE
+void tdx_cpu_flush_cache_for_kexec(void);
+#else
+static inline void tdx_cpu_flush_cache_for_kexec(void) { }
+#endif
+
+#endif /* !__ASSEMBLER__ */
 #endif /* _ASM_X86_TDX_H */
diff --git a/arch/x86/include/asm/tdx_global_metadata.h b/arch/x86/include/asm/tdx_global_metadata.h
new file mode 100644
index 000000000000..060a2ad744bf
--- /dev/null
+++ b/arch/x86/include/asm/tdx_global_metadata.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Automatically generated TDX global metadata structures. */
+#ifndef _X86_VIRT_TDX_AUTO_GENERATED_TDX_GLOBAL_METADATA_H
+#define _X86_VIRT_TDX_AUTO_GENERATED_TDX_GLOBAL_METADATA_H
+
+#include <linux/types.h>
+
+struct tdx_sys_info_features {
+	u64 tdx_features0;
+};
+
+struct tdx_sys_info_tdmr {
+	u16 max_tdmrs;
+	u16 max_reserved_per_tdmr;
+	u16 pamt_4k_entry_size;
+	u16 pamt_2m_entry_size;
+	u16 pamt_1g_entry_size;
+};
+
+struct tdx_sys_info_td_ctrl {
+	u16 tdr_base_size;
+	u16 tdcs_base_size;
+	u16 tdvps_base_size;
+};
+
+struct tdx_sys_info_td_conf {
+	u64 attributes_fixed0;
+	u64 attributes_fixed1;
+	u64 xfam_fixed0;
+	u64 xfam_fixed1;
+	u16 num_cpuid_config;
+	u16 max_vcpus_per_td;
+	u64 cpuid_config_leaves[128];
+	u64 cpuid_config_values[128][2];
+};
+
+struct tdx_sys_info {
+	struct tdx_sys_info_features features;
+	struct tdx_sys_info_tdmr tdmr;
+	struct tdx_sys_info_td_ctrl td_ctrl;
+	struct tdx_sys_info_td_conf td_conf;
+};
+
+#endif
diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h
index 6259f1937fe7..f2d142a0a862 100644
--- a/arch/x86/include/asm/text-patching.h
+++ b/arch/x86/include/asm/text-patching.h
@@ -11,11 +11,11 @@
  * JUMP_LABEL_NOP_SIZE/RELATIVEJUMP_SIZE, which are 5.
  * Raise it if needed.
  */
-#define POKE_MAX_OPCODE_SIZE	5
+#define TEXT_POKE_MAX_OPCODE_SIZE	5
 
 extern void text_poke_early(void *addr, const void *opcode, size_t len);
 
-extern void apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len);
+extern void text_poke_apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len);
 
 /*
  * Clear and restore the kernel write-protection flag on the local CPU.
@@ -32,16 +32,17 @@ extern void apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u
  * an inconsistent instruction while you patch.
  */
 extern void *text_poke(void *addr, const void *opcode, size_t len);
-extern void text_poke_sync(void);
+extern void smp_text_poke_sync_each_cpu(void);
 extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len);
 extern void *text_poke_copy(void *addr, const void *opcode, size_t len);
+#define text_poke_copy text_poke_copy
 extern void *text_poke_copy_locked(void *addr, const void *opcode, size_t len, bool core_ok);
 extern void *text_poke_set(void *addr, int c, size_t len);
-extern int poke_int3_handler(struct pt_regs *regs);
-extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate);
+extern int smp_text_poke_int3_handler(struct pt_regs *regs);
+extern void smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate);
 
-extern void text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate);
-extern void text_poke_finish(void);
+extern void smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate);
+extern void smp_text_poke_batch_finish(void);
 
 #define INT3_INSN_SIZE		1
 #define INT3_INSN_OPCODE	0xCC
@@ -81,7 +82,7 @@ static __always_inline int text_opcode_size(u8 opcode)
 }
 
 union text_poke_insn {
-	u8 text[POKE_MAX_OPCODE_SIZE];
+	u8 text[TEXT_POKE_MAX_OPCODE_SIZE];
 	struct {
 		u8 opcode;
 		s32 disp;
@@ -127,8 +128,8 @@ void *text_gen_insn(u8 opcode, const void *addr, const void *dest)
 }
 
 extern int after_bootmem;
-extern __ro_after_init struct mm_struct *poking_mm;
-extern __ro_after_init unsigned long poking_addr;
+extern __ro_after_init struct mm_struct *text_poke_mm;
+extern __ro_after_init unsigned long text_poke_mm_addr;
 
 #ifndef CONFIG_UML_X86
 static __always_inline
@@ -141,13 +142,14 @@ static __always_inline
 void int3_emulate_push(struct pt_regs *regs, unsigned long val)
 {
 	/*
-	 * The int3 handler in entry_64.S adds a gap between the
+	 * The INT3 handler in entry_64.S adds a gap between the
 	 * stack where the break point happened, and the saving of
 	 * pt_regs. We can extend the original stack because of
-	 * this gap. See the idtentry macro's create_gap option.
+	 * this gap. See the idtentry macro's X86_TRAP_BP logic.
 	 *
-	 * Similarly entry_32.S will have a gap on the stack for (any) hardware
-	 * exception and pt_regs; see FIXUP_FRAME.
+	 * Similarly, entry_32.S will have a gap on the stack for
+	 * (any) hardware exception and pt_regs; see the
+	 * FIXUP_FRAME macro.
 	 */
 	regs->sp -= sizeof(unsigned long);
 	*(unsigned long *)regs->sp = val;
@@ -176,9 +178,9 @@ void int3_emulate_ret(struct pt_regs *regs)
 }
 
 static __always_inline
-void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp)
+bool __emulate_cc(unsigned long flags, u8 cc)
 {
-	static const unsigned long jcc_mask[6] = {
+	static const unsigned long cc_mask[6] = {
 		[0] = X86_EFLAGS_OF,
 		[1] = X86_EFLAGS_CF,
 		[2] = X86_EFLAGS_ZF,
@@ -191,15 +193,21 @@ void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned lo
 	bool match;
 
 	if (cc < 0xc) {
-		match = regs->flags & jcc_mask[cc >> 1];
+		match = flags & cc_mask[cc >> 1];
 	} else {
-		match = ((regs->flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^
-			((regs->flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT);
+		match = ((flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^
+			((flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT);
 		if (cc >= 0xe)
-			match = match || (regs->flags & X86_EFLAGS_ZF);
+			match = match || (flags & X86_EFLAGS_ZF);
 	}
 
-	if ((match && !invert) || (!match && invert))
+	return (match && !invert) || (!match && invert);
+}
+
+static __always_inline
+void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp)
+{
+	if (__emulate_cc(regs->flags, cc))
 		ip += disp;
 
 	int3_emulate_jmp(regs, ip);
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 12da7dfd5ef1..e71e0e8362ed 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -54,7 +54,7 @@
  * - this struct should fit entirely inside of one cache line
  * - this struct shares the supervisor stack pages
  */
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 struct task_struct;
 #include <asm/cpufeature.h>
 #include <linux/atomic.h>
@@ -73,61 +73,49 @@ struct thread_info {
 	.flags		= 0,			\
 }
 
-#else /* !__ASSEMBLY__ */
+#else /* !__ASSEMBLER__ */
 
 #include <asm/asm-offsets.h>
 
 #endif
 
 /*
- * thread information flags
- * - these are process state flags that various assembly files
- *   may need to access
+ * Tell the generic TIF infrastructure which bits x86 supports
  */
-#define TIF_NOTIFY_RESUME	1	/* callback before returning to user */
-#define TIF_SIGPENDING		2	/* signal pending */
-#define TIF_NEED_RESCHED	3	/* rescheduling necessary */
-#define TIF_SINGLESTEP		4	/* reenable singlestep on user return*/
-#define TIF_SSBD		5	/* Speculative store bypass disable */
-#define TIF_SPEC_IB		9	/* Indirect branch speculation mitigation */
-#define TIF_SPEC_L1D_FLUSH	10	/* Flush L1D on mm switches (processes) */
-#define TIF_USER_RETURN_NOTIFY	11	/* notify kernel of userspace return */
-#define TIF_UPROBE		12	/* breakpointed or singlestepping */
-#define TIF_PATCH_PENDING	13	/* pending live patching update */
-#define TIF_NEED_FPU_LOAD	14	/* load FPU on return to userspace */
-#define TIF_NOCPUID		15	/* CPUID is not accessible in userland */
-#define TIF_NOTSC		16	/* TSC is not accessible in userland */
-#define TIF_NOTIFY_SIGNAL	17	/* signal notifications exist */
-#define TIF_MEMDIE		20	/* is terminating due to OOM killer */
-#define TIF_POLLING_NRFLAG	21	/* idle is polling for TIF_NEED_RESCHED */
+#define HAVE_TIF_NEED_RESCHED_LAZY
+#define HAVE_TIF_POLLING_NRFLAG
+#define HAVE_TIF_SINGLESTEP
+
+#include <asm-generic/thread_info_tif.h>
+
+/* Architecture specific TIF space starts at 16 */
+#define TIF_SSBD		16	/* Speculative store bypass disable */
+#define TIF_SPEC_IB		17	/* Indirect branch speculation mitigation */
+#define TIF_SPEC_L1D_FLUSH	18	/* Flush L1D on mm switches (processes) */
+#define TIF_NEED_FPU_LOAD	19	/* load FPU on return to userspace */
+#define TIF_NOCPUID		20	/* CPUID is not accessible in userland */
+#define TIF_NOTSC		21	/* TSC is not accessible in userland */
 #define TIF_IO_BITMAP		22	/* uses I/O bitmap */
 #define TIF_SPEC_FORCE_UPDATE	23	/* Force speculation MSR update in context switch */
 #define TIF_FORCED_TF		24	/* true if TF in eflags artificially */
-#define TIF_BLOCKSTEP		25	/* set when we want DEBUGCTLMSR_BTF */
+#define TIF_SINGLESTEP		25	/* reenable singlestep on user return*/
+#define TIF_BLOCKSTEP		26	/* set when we want DEBUGCTLMSR_BTF */
 #define TIF_LAZY_MMU_UPDATES	27	/* task is updating the mmu lazily */
-#define TIF_ADDR32		29	/* 32-bit address space on 64 bits */
-
-#define _TIF_NOTIFY_RESUME	(1 << TIF_NOTIFY_RESUME)
-#define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)
-#define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)
-#define _TIF_SINGLESTEP		(1 << TIF_SINGLESTEP)
-#define _TIF_SSBD		(1 << TIF_SSBD)
-#define _TIF_SPEC_IB		(1 << TIF_SPEC_IB)
-#define _TIF_SPEC_L1D_FLUSH	(1 << TIF_SPEC_L1D_FLUSH)
-#define _TIF_USER_RETURN_NOTIFY	(1 << TIF_USER_RETURN_NOTIFY)
-#define _TIF_UPROBE		(1 << TIF_UPROBE)
-#define _TIF_PATCH_PENDING	(1 << TIF_PATCH_PENDING)
-#define _TIF_NEED_FPU_LOAD	(1 << TIF_NEED_FPU_LOAD)
-#define _TIF_NOCPUID		(1 << TIF_NOCPUID)
-#define _TIF_NOTSC		(1 << TIF_NOTSC)
-#define _TIF_NOTIFY_SIGNAL	(1 << TIF_NOTIFY_SIGNAL)
-#define _TIF_POLLING_NRFLAG	(1 << TIF_POLLING_NRFLAG)
-#define _TIF_IO_BITMAP		(1 << TIF_IO_BITMAP)
-#define _TIF_SPEC_FORCE_UPDATE	(1 << TIF_SPEC_FORCE_UPDATE)
-#define _TIF_FORCED_TF		(1 << TIF_FORCED_TF)
-#define _TIF_BLOCKSTEP		(1 << TIF_BLOCKSTEP)
-#define _TIF_LAZY_MMU_UPDATES	(1 << TIF_LAZY_MMU_UPDATES)
-#define _TIF_ADDR32		(1 << TIF_ADDR32)
+#define TIF_ADDR32		28	/* 32-bit address space on 64 bits */
+
+#define _TIF_SSBD		BIT(TIF_SSBD)
+#define _TIF_SPEC_IB		BIT(TIF_SPEC_IB)
+#define _TIF_SPEC_L1D_FLUSH	BIT(TIF_SPEC_L1D_FLUSH)
+#define _TIF_NEED_FPU_LOAD	BIT(TIF_NEED_FPU_LOAD)
+#define _TIF_NOCPUID		BIT(TIF_NOCPUID)
+#define _TIF_NOTSC		BIT(TIF_NOTSC)
+#define _TIF_IO_BITMAP		BIT(TIF_IO_BITMAP)
+#define _TIF_SPEC_FORCE_UPDATE	BIT(TIF_SPEC_FORCE_UPDATE)
+#define _TIF_FORCED_TF		BIT(TIF_FORCED_TF)
+#define _TIF_BLOCKSTEP		BIT(TIF_BLOCKSTEP)
+#define _TIF_SINGLESTEP		BIT(TIF_SINGLESTEP)
+#define _TIF_LAZY_MMU_UPDATES	BIT(TIF_LAZY_MMU_UPDATES)
+#define _TIF_ADDR32		BIT(TIF_ADDR32)
 
 /* flags to check in __switch_to() */
 #define _TIF_WORK_CTXSW_BASE					\
@@ -159,7 +147,7 @@ struct thread_info {
  *
  * preempt_count needs to be 1 initially, until the scheduler is functional.
  */
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 /*
  * Walks up the stack frames to make sure that the specified object is
@@ -211,7 +199,7 @@ static inline int arch_within_stack_frames(const void * const stack,
 #endif
 }
 
-#endif  /* !__ASSEMBLY__ */
+#endif  /* !__ASSEMBLER__ */
 
 /*
  * Thread-synchronous status.
@@ -222,7 +210,7 @@ static inline int arch_within_stack_frames(const void * const stack,
  */
 #define TS_COMPAT		0x0002	/* 32bit syscall active (64BIT)*/
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #ifdef CONFIG_COMPAT
 #define TS_I386_REGS_POKED	0x0004	/* regs poked by 32-bit ptracer */
 
@@ -240,6 +228,6 @@ static inline int arch_within_stack_frames(const void * const stack,
 
 extern void arch_setup_new_exec(void);
 #define arch_setup_new_exec arch_setup_new_exec
-#endif	/* !__ASSEMBLY__ */
+#endif	/* !__ASSEMBLER__ */
 
 #endif /* _ASM_X86_THREAD_INFO_H */
diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h
index 7365dd4acffb..23baf8c9b34c 100644
--- a/arch/x86/include/asm/timer.h
+++ b/arch/x86/include/asm/timer.h
@@ -6,8 +6,6 @@
 #include <linux/interrupt.h>
 #include <linux/math64.h>
 
-#define TICK_SIZE (tick_nsec / 1000)
-
 unsigned long long native_sched_clock(void);
 extern void recalibrate_cpu_khz(void);
 
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 580636cdc257..866ea78ba156 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -6,6 +6,9 @@
 static inline void tlb_flush(struct mmu_gather *tlb);
 
 #include <asm-generic/tlb.h>
+#include <linux/kernel.h>
+#include <vdso/bits.h>
+#include <vdso/page.h>
 
 static inline void tlb_flush(struct mmu_gather *tlb)
 {
@@ -20,18 +23,144 @@ static inline void tlb_flush(struct mmu_gather *tlb)
 	flush_tlb_mm_range(tlb->mm, start, end, stride_shift, tlb->freed_tables);
 }
 
+static inline void invlpg(unsigned long addr)
+{
+	asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
+}
+
+enum addr_stride {
+	PTE_STRIDE = 0,
+	PMD_STRIDE = 1
+};
+
+/*
+ * INVLPGB can be targeted by virtual address, PCID, ASID, or any combination
+ * of the three. For example:
+ * - FLAG_VA | FLAG_INCLUDE_GLOBAL: invalidate all TLB entries at the address
+ * - FLAG_PCID:			    invalidate all TLB entries matching the PCID
+ *
+ * The first is used to invalidate (kernel) mappings at a particular
+ * address across all processes.
+ *
+ * The latter invalidates all TLB entries matching a PCID.
+ */
+#define INVLPGB_FLAG_VA			BIT(0)
+#define INVLPGB_FLAG_PCID		BIT(1)
+#define INVLPGB_FLAG_ASID		BIT(2)
+#define INVLPGB_FLAG_INCLUDE_GLOBAL	BIT(3)
+#define INVLPGB_FLAG_FINAL_ONLY		BIT(4)
+#define INVLPGB_FLAG_INCLUDE_NESTED	BIT(5)
+
+/* The implied mode when all bits are clear: */
+#define INVLPGB_MODE_ALL_NONGLOBALS	0UL
+
+#ifdef CONFIG_BROADCAST_TLB_FLUSH
 /*
- * While x86 architecture in general requires an IPI to perform TLB
- * shootdown, enablement code for several hypervisors overrides
- * .flush_tlb_others hook in pv_mmu_ops and implements it by issuing
- * a hypercall. To keep software pagetable walkers safe in this case we
- * switch to RCU based table free (MMU_GATHER_RCU_TABLE_FREE). See the comment
- * below 'ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE' in include/asm-generic/tlb.h
- * for more details.
+ * INVLPGB does broadcast TLB invalidation across all the CPUs in the system.
+ *
+ * The INVLPGB instruction is weakly ordered, and a batch of invalidations can
+ * be done in a parallel fashion.
+ *
+ * The instruction takes the number of extra pages to invalidate, beyond the
+ * first page, while __invlpgb gets the more human readable number of pages to
+ * invalidate.
+ *
+ * The bits in rax[0:2] determine respectively which components of the address
+ * (VA, PCID, ASID) get compared when flushing. If neither bits are set, *any*
+ * address in the specified range matches.
+ *
+ * Since it is desired to only flush TLB entries for the ASID that is executing
+ * the instruction (a host/hypervisor or a guest), the ASID valid bit should
+ * always be set. On a host/hypervisor, the hardware will use the ASID value
+ * specified in EDX[15:0] (which should be 0). On a guest, the hardware will
+ * use the actual ASID value of the guest.
+ *
+ * TLBSYNC is used to ensure that pending INVLPGB invalidations initiated from
+ * this CPU have completed.
  */
-static inline void __tlb_remove_table(void *table)
+static inline void __invlpgb(unsigned long asid, unsigned long pcid,
+			     unsigned long addr, u16 nr_pages,
+			     enum addr_stride stride, u8 flags)
+{
+	u64 rax = addr | flags | INVLPGB_FLAG_ASID;
+	u32 ecx = (stride << 31) | (nr_pages - 1);
+	u32 edx = (pcid << 16) | asid;
+
+	/* The low bits in rax are for flags. Verify addr is clean. */
+	VM_WARN_ON_ONCE(addr & ~PAGE_MASK);
+
+	/* INVLPGB; supported in binutils >= 2.36. */
+	asm volatile(".byte 0x0f, 0x01, 0xfe" :: "a" (rax), "c" (ecx), "d" (edx));
+}
+
+static inline void __invlpgb_all(unsigned long asid, unsigned long pcid, u8 flags)
+{
+	__invlpgb(asid, pcid, 0, 1, 0, flags);
+}
+
+static inline void __tlbsync(void)
 {
-	free_page_and_swap_cache(table);
+	/*
+	 * TLBSYNC waits for INVLPGB instructions originating on the same CPU
+	 * to have completed. Print a warning if the task has been migrated,
+	 * and might not be waiting on all the INVLPGBs issued during this TLB
+	 * invalidation sequence.
+	 */
+	cant_migrate();
+
+	/* TLBSYNC: supported in binutils >= 0.36. */
+	asm volatile(".byte 0x0f, 0x01, 0xff" ::: "memory");
+}
+#else
+/* Some compilers (I'm looking at you clang!) simply can't do DCE */
+static inline void __invlpgb(unsigned long asid, unsigned long pcid,
+			     unsigned long addr, u16 nr_pages,
+			     enum addr_stride s, u8 flags) { }
+static inline void __invlpgb_all(unsigned long asid, unsigned long pcid, u8 flags) { }
+static inline void __tlbsync(void) { }
+#endif
+
+static inline void invlpgb_flush_user_nr_nosync(unsigned long pcid,
+						unsigned long addr,
+						u16 nr, bool stride)
+{
+	enum addr_stride str = stride ? PMD_STRIDE : PTE_STRIDE;
+	u8 flags = INVLPGB_FLAG_PCID | INVLPGB_FLAG_VA;
+
+	__invlpgb(0, pcid, addr, nr, str, flags);
+}
+
+/* Flush all mappings for a given PCID, not including globals. */
+static inline void invlpgb_flush_single_pcid_nosync(unsigned long pcid)
+{
+	__invlpgb_all(0, pcid, INVLPGB_FLAG_PCID);
 }
 
+/* Flush all mappings, including globals, for all PCIDs. */
+static inline void invlpgb_flush_all(void)
+{
+	/*
+	 * TLBSYNC at the end needs to make sure all flushes done on the
+	 * current CPU have been executed system-wide. Therefore, make
+	 * sure nothing gets migrated in-between but disable preemption
+	 * as it is cheaper.
+	 */
+	guard(preempt)();
+	__invlpgb_all(0, 0, INVLPGB_FLAG_INCLUDE_GLOBAL);
+	__tlbsync();
+}
+
+/* Flush addr, including globals, for all PCIDs. */
+static inline void invlpgb_flush_addr_nosync(unsigned long addr, u16 nr)
+{
+	__invlpgb(0, 0, addr, nr, PTE_STRIDE, INVLPGB_FLAG_INCLUDE_GLOBAL);
+}
+
+/* Flush all mappings for all PCIDs except globals. */
+static inline void invlpgb_flush_all_nonglobals(void)
+{
+	guard(preempt)();
+	__invlpgb_all(0, 0, INVLPGB_MODE_ALL_NONGLOBALS);
+	__tlbsync();
+}
 #endif /* _ASM_X86_TLB_H */
diff --git a/arch/x86/include/asm/tlbbatch.h b/arch/x86/include/asm/tlbbatch.h
index 1ad56eb3e8a8..80aaf64ff25f 100644
--- a/arch/x86/include/asm/tlbbatch.h
+++ b/arch/x86/include/asm/tlbbatch.h
@@ -10,6 +10,11 @@ struct arch_tlbflush_unmap_batch {
 	 * the PFNs being flushed..
 	 */
 	struct cpumask cpumask;
+	/*
+	 * Set if pages were unmapped from any MM, even one that does not
+	 * have active CPUs in its cpumask.
+	 */
+	bool unmapped_pages;
 };
 
 #endif /* _ARCH_X86_TLBBATCH_H */
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 69e79fff41b8..00daedfefc1b 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -6,6 +6,7 @@
 #include <linux/mmu_notifier.h>
 #include <linux/sched.h>
 
+#include <asm/barrier.h>
 #include <asm/processor.h>
 #include <asm/cpufeature.h>
 #include <asm/special_insns.h>
@@ -183,6 +184,9 @@ static inline void cr4_init_shadow(void)
 extern unsigned long mmu_cr4_features;
 extern u32 *trampoline_cr4_features;
 
+/* How many pages can be invalidated with one INVLPGB. */
+extern u16 invlpgb_count_max;
+
 extern void initialize_tlbstate_and_flush(void);
 
 /*
@@ -222,6 +226,7 @@ struct flush_tlb_info {
 	unsigned int		initiating_cpu;
 	u8			stride_shift;
 	u8			freed_tables;
+	u8			trim_cpumask;
 };
 
 void flush_tlb_local(void);
@@ -230,6 +235,71 @@ void flush_tlb_one_kernel(unsigned long addr);
 void flush_tlb_multi(const struct cpumask *cpumask,
 		      const struct flush_tlb_info *info);
 
+static inline bool is_dyn_asid(u16 asid)
+{
+	return asid < TLB_NR_DYN_ASIDS;
+}
+
+static inline bool is_global_asid(u16 asid)
+{
+	return !is_dyn_asid(asid);
+}
+
+#ifdef CONFIG_BROADCAST_TLB_FLUSH
+static inline u16 mm_global_asid(struct mm_struct *mm)
+{
+	u16 asid;
+
+	if (!cpu_feature_enabled(X86_FEATURE_INVLPGB))
+		return 0;
+
+	asid = smp_load_acquire(&mm->context.global_asid);
+
+	/* mm->context.global_asid is either 0, or a global ASID */
+	VM_WARN_ON_ONCE(asid && is_dyn_asid(asid));
+
+	return asid;
+}
+
+static inline void mm_init_global_asid(struct mm_struct *mm)
+{
+	if (cpu_feature_enabled(X86_FEATURE_INVLPGB)) {
+		mm->context.global_asid = 0;
+		mm->context.asid_transition = false;
+	}
+}
+
+static inline void mm_assign_global_asid(struct mm_struct *mm, u16 asid)
+{
+	/*
+	 * Notably flush_tlb_mm_range() -> broadcast_tlb_flush() ->
+	 * finish_asid_transition() needs to observe asid_transition = true
+	 * once it observes global_asid.
+	 */
+	mm->context.asid_transition = true;
+	smp_store_release(&mm->context.global_asid, asid);
+}
+
+static inline void mm_clear_asid_transition(struct mm_struct *mm)
+{
+	WRITE_ONCE(mm->context.asid_transition, false);
+}
+
+static inline bool mm_in_asid_transition(struct mm_struct *mm)
+{
+	if (!cpu_feature_enabled(X86_FEATURE_INVLPGB))
+		return false;
+
+	return mm && READ_ONCE(mm->context.asid_transition);
+}
+#else
+static inline u16 mm_global_asid(struct mm_struct *mm) { return 0; }
+static inline void mm_init_global_asid(struct mm_struct *mm) { }
+static inline void mm_assign_global_asid(struct mm_struct *mm, u16 asid) { }
+static inline void mm_clear_asid_transition(struct mm_struct *mm) { }
+static inline bool mm_in_asid_transition(struct mm_struct *mm) { return false; }
+#endif /* CONFIG_BROADCAST_TLB_FLUSH */
+
 #ifdef CONFIG_PARAVIRT
 #include <asm/paravirt.h>
 #endif
@@ -241,7 +311,7 @@ void flush_tlb_multi(const struct cpumask *cpumask,
 	flush_tlb_mm_range((vma)->vm_mm, start, end,			\
 			   ((vma)->vm_flags & VM_HUGETLB)		\
 				? huge_page_shift(hstate_vma(vma))	\
-				: PAGE_SHIFT, false)
+				: PAGE_SHIFT, true)
 
 extern void flush_tlb_all(void);
 extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
@@ -278,19 +348,14 @@ static inline u64 inc_mm_tlb_gen(struct mm_struct *mm)
 }
 
 static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,
-					     struct mm_struct *mm,
-					     unsigned long uaddr)
+		struct mm_struct *mm, unsigned long start, unsigned long end)
 {
 	inc_mm_tlb_gen(mm);
 	cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm));
+	batch->unmapped_pages = true;
 	mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL);
 }
 
-static inline void arch_flush_tlb_batched_pending(struct mm_struct *mm)
-{
-	flush_tlb_mm(mm);
-}
-
 extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch);
 
 static inline bool pte_flags_need_flush(unsigned long oldflags,
diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index aef70336d624..1fadf0cf520c 100644
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -114,6 +114,12 @@ enum x86_topology_domains {
 	TOPO_MAX_DOMAIN,
 };
 
+enum x86_topology_cpu_type {
+	TOPO_CPU_TYPE_PERFORMANCE,
+	TOPO_CPU_TYPE_EFFICIENCY,
+	TOPO_CPU_TYPE_UNKNOWN,
+};
+
 struct x86_topology_system {
 	unsigned int	dom_shifts[TOPO_MAX_DOMAIN];
 	unsigned int	dom_size[TOPO_MAX_DOMAIN];
@@ -137,6 +143,7 @@ extern const struct cpumask *cpu_clustergroup_mask(int cpu);
 #define topology_logical_package_id(cpu)	(cpu_data(cpu).topo.logical_pkg_id)
 #define topology_physical_package_id(cpu)	(cpu_data(cpu).topo.pkg_id)
 #define topology_logical_die_id(cpu)		(cpu_data(cpu).topo.logical_die_id)
+#define topology_logical_core_id(cpu)		(cpu_data(cpu).topo.logical_core_id)
 #define topology_die_id(cpu)			(cpu_data(cpu).topo.die_id)
 #define topology_core_id(cpu)			(cpu_data(cpu).topo.core_id)
 #define topology_ppin(cpu)			(cpu_data(cpu).ppin)
@@ -149,6 +156,9 @@ extern unsigned int __max_threads_per_core;
 extern unsigned int __num_threads_per_package;
 extern unsigned int __num_cores_per_package;
 
+const char *get_topology_cpu_type_name(struct cpuinfo_x86 *c);
+enum x86_topology_cpu_type get_topology_cpu_type(struct cpuinfo_x86 *c);
+
 static inline unsigned int topology_max_packages(void)
 {
 	return __max_logical_packages;
@@ -208,6 +218,12 @@ static inline unsigned int topology_amd_nodes_per_pkg(void)
 	return __amd_nodes_per_pkg;
 }
 
+#else /* CONFIG_SMP */
+static inline int topology_phys_to_logical_pkg(unsigned int pkg) { return 0; }
+static inline int topology_max_smt_threads(void) { return 1; }
+static inline unsigned int topology_amd_nodes_per_pkg(void) { return 1; }
+#endif /* !CONFIG_SMP */
+
 extern struct cpumask __cpu_primary_thread_mask;
 #define cpu_primary_thread_mask ((const struct cpumask *)&__cpu_primary_thread_mask)
 
@@ -219,13 +235,17 @@ static inline bool topology_is_primary_thread(unsigned int cpu)
 {
 	return cpumask_test_cpu(cpu, cpu_primary_thread_mask);
 }
+#define topology_is_primary_thread topology_is_primary_thread
 
-#else /* CONFIG_SMP */
-static inline int topology_phys_to_logical_pkg(unsigned int pkg) { return 0; }
-static inline int topology_max_smt_threads(void) { return 1; }
-static inline bool topology_is_primary_thread(unsigned int cpu) { return true; }
-static inline unsigned int topology_amd_nodes_per_pkg(void) { return 1; }
-#endif /* !CONFIG_SMP */
+int topology_get_primary_thread(unsigned int cpu);
+
+static inline bool topology_is_core_online(unsigned int cpu)
+{
+	int pcpu = topology_get_primary_thread(cpu);
+
+	return pcpu >= 0 ? cpu_online(pcpu) : false;
+}
+#define topology_is_core_online topology_is_core_online
 
 static inline void arch_fix_phys_package_id(int num, u32 slot)
 {
@@ -241,7 +261,7 @@ extern bool x86_topology_update;
 #include <asm/percpu.h>
 
 DECLARE_PER_CPU_READ_MOSTLY(int, sched_core_priority);
-extern unsigned int __read_mostly sysctl_sched_itmt_enabled;
+extern bool __read_mostly sysctl_sched_itmt_enabled;
 
 /* Interface to set priority of a cpu */
 void sched_set_itmt_core_prio(int prio, int core_cpu);
@@ -254,7 +274,7 @@ void sched_clear_itmt_support(void);
 
 #else /* CONFIG_SCHED_MC_PRIO */
 
-#define sysctl_sched_itmt_enabled	0
+#define sysctl_sched_itmt_enabled	false
 static inline void sched_set_itmt_core_prio(int prio, int core_cpu)
 {
 }
@@ -305,9 +325,6 @@ static inline void freq_invariance_set_perf_ratio(u64 ratio, bool turbo_disabled
 extern void arch_scale_freq_tick(void);
 #define arch_scale_freq_tick arch_scale_freq_tick
 
-#ifdef CONFIG_ACPI_CPPC_LIB
-void init_freq_invariance_cppc(void);
-#define arch_init_invariance_cppc init_freq_invariance_cppc
-#endif
+extern int arch_sched_node_distance(int from, int to);
 
 #endif /* _ASM_X86_TOPOLOGY_H */
diff --git a/arch/x86/include/asm/trace/common.h b/arch/x86/include/asm/trace/common.h
deleted file mode 100644
index f0f9bcdb74d9..000000000000
--- a/arch/x86/include/asm/trace/common.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _ASM_TRACE_COMMON_H
-#define _ASM_TRACE_COMMON_H
-
-#ifdef CONFIG_TRACING
-DECLARE_STATIC_KEY_FALSE(trace_pagefault_key);
-#define trace_pagefault_enabled()			\
-	static_branch_unlikely(&trace_pagefault_key)
-#else
-static inline bool trace_pagefault_enabled(void) { return false; }
-#endif
-
-#endif
diff --git a/arch/x86/include/asm/trace/exceptions.h b/arch/x86/include/asm/trace/exceptions.h
deleted file mode 100644
index 6b1e87194809..000000000000
--- a/arch/x86/include/asm/trace/exceptions.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#undef TRACE_SYSTEM
-#define TRACE_SYSTEM exceptions
-
-#if !defined(_TRACE_PAGE_FAULT_H) || defined(TRACE_HEADER_MULTI_READ)
-#define _TRACE_PAGE_FAULT_H
-
-#include <linux/tracepoint.h>
-#include <asm/trace/common.h>
-
-extern int trace_pagefault_reg(void);
-extern void trace_pagefault_unreg(void);
-
-DECLARE_EVENT_CLASS(x86_exceptions,
-
-	TP_PROTO(unsigned long address, struct pt_regs *regs,
-		 unsigned long error_code),
-
-	TP_ARGS(address, regs, error_code),
-
-	TP_STRUCT__entry(
-		__field(		unsigned long, address	)
-		__field(		unsigned long, ip	)
-		__field(		unsigned long, error_code )
-	),
-
-	TP_fast_assign(
-		__entry->address = address;
-		__entry->ip = regs->ip;
-		__entry->error_code = error_code;
-	),
-
-	TP_printk("address=%ps ip=%ps error_code=0x%lx",
-		  (void *)__entry->address, (void *)__entry->ip,
-		  __entry->error_code) );
-
-#define DEFINE_PAGE_FAULT_EVENT(name)				\
-DEFINE_EVENT_FN(x86_exceptions, name,				\
-	TP_PROTO(unsigned long address,	struct pt_regs *regs,	\
-		 unsigned long error_code),			\
-	TP_ARGS(address, regs, error_code),			\
-	trace_pagefault_reg, trace_pagefault_unreg);
-
-DEFINE_PAGE_FAULT_EVENT(page_fault_user);
-DEFINE_PAGE_FAULT_EVENT(page_fault_kernel);
-
-#undef TRACE_INCLUDE_PATH
-#undef TRACE_INCLUDE_FILE
-#define TRACE_INCLUDE_PATH .
-#define TRACE_INCLUDE_FILE exceptions
-#endif /*  _TRACE_PAGE_FAULT_H */
-
-/* This part must be outside protection */
-#include <trace/define_trace.h>
diff --git a/arch/x86/include/asm/trace/fpu.h b/arch/x86/include/asm/trace/fpu.h
index 4645a6334063..721b408d9a67 100644
--- a/arch/x86/include/asm/trace/fpu.h
+++ b/arch/x86/include/asm/trace/fpu.h
@@ -44,16 +44,6 @@ DEFINE_EVENT(x86_fpu, x86_fpu_after_save,
 	TP_ARGS(fpu)
 );
 
-DEFINE_EVENT(x86_fpu, x86_fpu_before_restore,
-	TP_PROTO(struct fpu *fpu),
-	TP_ARGS(fpu)
-);
-
-DEFINE_EVENT(x86_fpu, x86_fpu_after_restore,
-	TP_PROTO(struct fpu *fpu),
-	TP_ARGS(fpu)
-);
-
 DEFINE_EVENT(x86_fpu, x86_fpu_regs_activated,
 	TP_PROTO(struct fpu *fpu),
 	TP_ARGS(fpu)
@@ -64,21 +54,11 @@ DEFINE_EVENT(x86_fpu, x86_fpu_regs_deactivated,
 	TP_ARGS(fpu)
 );
 
-DEFINE_EVENT(x86_fpu, x86_fpu_init_state,
-	TP_PROTO(struct fpu *fpu),
-	TP_ARGS(fpu)
-);
-
 DEFINE_EVENT(x86_fpu, x86_fpu_dropped,
 	TP_PROTO(struct fpu *fpu),
 	TP_ARGS(fpu)
 );
 
-DEFINE_EVENT(x86_fpu, x86_fpu_copy_src,
-	TP_PROTO(struct fpu *fpu),
-	TP_ARGS(fpu)
-);
-
 DEFINE_EVENT(x86_fpu, x86_fpu_copy_dst,
 	TP_PROTO(struct fpu *fpu),
 	TP_ARGS(fpu)
diff --git a/arch/x86/include/asm/trace/irq_vectors.h b/arch/x86/include/asm/trace/irq_vectors.h
index 88e7f0f3bf62..7408bebdfde0 100644
--- a/arch/x86/include/asm/trace/irq_vectors.h
+++ b/arch/x86/include/asm/trace/irq_vectors.h
@@ -6,7 +6,6 @@
 #define _TRACE_IRQ_VECTORS_H
 
 #include <linux/tracepoint.h>
-#include <asm/trace/common.h>
 
 #ifdef CONFIG_X86_LOCAL_APIC
 
diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index 1f1deaecd364..869b88061801 100644
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -35,8 +35,6 @@ static inline int get_si_code(unsigned long condition)
 		return TRAP_BRKPT;
 }
 
-extern int panic_on_unrecovered_nmi;
-
 void math_emulate(struct math_emu_info *);
 
 bool fault_in_kernel_space(unsigned long address);
diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
index 94408a784c8e..4f7f09f50552 100644
--- a/arch/x86/include/asm/tsc.h
+++ b/arch/x86/include/asm/tsc.h
@@ -5,10 +5,65 @@
 #ifndef _ASM_X86_TSC_H
 #define _ASM_X86_TSC_H
 
+#include <asm/asm.h>
 #include <asm/cpufeature.h>
 #include <asm/processor.h>
 #include <asm/msr.h>
 
+/**
+ * rdtsc() - returns the current TSC without ordering constraints
+ *
+ * rdtsc() returns the result of RDTSC as a 64-bit integer.  The
+ * only ordering constraint it supplies is the ordering implied by
+ * "asm volatile": it will put the RDTSC in the place you expect.  The
+ * CPU can and will speculatively execute that RDTSC, though, so the
+ * results can be non-monotonic if compared on different CPUs.
+ */
+static __always_inline u64 rdtsc(void)
+{
+	EAX_EDX_DECLARE_ARGS(val, low, high);
+
+	asm volatile("rdtsc" : EAX_EDX_RET(val, low, high));
+
+	return EAX_EDX_VAL(val, low, high);
+}
+
+/**
+ * rdtsc_ordered() - read the current TSC in program order
+ *
+ * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
+ * It is ordered like a load to a global in-memory counter.  It should
+ * be impossible to observe non-monotonic rdtsc_unordered() behavior
+ * across multiple CPUs as long as the TSC is synced.
+ */
+static __always_inline u64 rdtsc_ordered(void)
+{
+	EAX_EDX_DECLARE_ARGS(val, low, high);
+
+	/*
+	 * The RDTSC instruction is not ordered relative to memory
+	 * access.  The Intel SDM and the AMD APM are both vague on this
+	 * point, but empirically an RDTSC instruction can be
+	 * speculatively executed before prior loads.  An RDTSC
+	 * immediately after an appropriate barrier appears to be
+	 * ordered as a normal load, that is, it provides the same
+	 * ordering guarantees as reading from a global memory location
+	 * that some other imaginary CPU is updating continuously with a
+	 * time stamp.
+	 *
+	 * Thus, use the preferred barrier on the respective CPU, aiming for
+	 * RDTSCP as the default.
+	 */
+	asm volatile(ALTERNATIVE_2("rdtsc",
+				   "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,
+				   "rdtscp", X86_FEATURE_RDTSCP)
+			: EAX_EDX_RET(val, low, high)
+			/* RDTSCP clobbers ECX with MSR_TSC_AUX. */
+			:: "ecx");
+
+	return EAX_EDX_VAL(val, low, high);
+}
+
 /*
  * Standard way to access the cycle counter.
  */
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 3a7755c1a441..367297b188c3 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -378,7 +378,7 @@ do {									\
 	asm_goto_output("\n"						\
 		     "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
 		     _ASM_EXTABLE_UA(1b, %l[label])			\
-		     : CC_OUT(z) (success),				\
+		     : "=@ccz" (success),				\
 		       [ptr] "+m" (*_ptr),				\
 		       [old] "+a" (__old)				\
 		     : [new] ltype (__new)				\
@@ -397,7 +397,7 @@ do {									\
 	asm_goto_output("\n"						\
 		     "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n"		\
 		     _ASM_EXTABLE_UA(1b, %l[label])			\
-		     : CC_OUT(z) (success),				\
+		     : "=@ccz" (success),				\
 		       "+A" (__old),					\
 		       [ptr] "+m" (*_ptr)				\
 		     : "b" ((u32)__new),				\
@@ -417,11 +417,10 @@ do {									\
 	__typeof__(*(_ptr)) __new = (_new);				\
 	asm volatile("\n"						\
 		     "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
-		     CC_SET(z)						\
 		     "2:\n"						\
 		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG,	\
 					   %[errout])			\
-		     : CC_OUT(z) (success),				\
+		     : "=@ccz" (success),				\
 		       [errout] "+r" (__err),				\
 		       [ptr] "+m" (*_ptr),				\
 		       [old] "+a" (__old)				\
@@ -529,18 +528,18 @@ static __must_check __always_inline bool user_access_begin(const void __user *pt
 #define user_access_save()	smap_save()
 #define user_access_restore(x)	smap_restore(x)
 
-#define unsafe_put_user(x, ptr, label)	\
+#define arch_unsafe_put_user(x, ptr, label)	\
 	__put_user_size((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), label)
 
 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
-#define unsafe_get_user(x, ptr, err_label)					\
+#define arch_unsafe_get_user(x, ptr, err_label)					\
 do {										\
 	__inttype(*(ptr)) __gu_val;						\
 	__get_user_size(__gu_val, (ptr), sizeof(*(ptr)), err_label);		\
 	(x) = (__force __typeof__(*(ptr)))__gu_val;				\
 } while (0)
 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
-#define unsafe_get_user(x, ptr, err_label)					\
+#define arch_unsafe_get_user(x, ptr, err_label)					\
 do {										\
 	int __gu_err;								\
 	__inttype(*(ptr)) __gu_val;						\
@@ -619,11 +618,11 @@ do {									\
 } while (0)
 
 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
-#define __get_kernel_nofault(dst, src, type, err_label)			\
+#define arch_get_kernel_nofault(dst, src, type, err_label)		\
 	__get_user_size(*((type *)(dst)), (__force type __user *)(src),	\
 			sizeof(type), err_label)
 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
-#define __get_kernel_nofault(dst, src, type, err_label)			\
+#define arch_get_kernel_nofault(dst, src, type, err_label)			\
 do {									\
 	int __kr_err;							\
 									\
@@ -634,7 +633,7 @@ do {									\
 } while (0)
 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 
-#define __put_kernel_nofault(dst, src, type, err_label)			\
+#define arch_put_kernel_nofault(dst, src, type, err_label)		\
 	__put_user_size(*((type *)(src)), (__force type __user *)(dst),	\
 			sizeof(type), err_label)
 
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index afce8ee5d7b7..915124011c27 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -13,14 +13,21 @@
 #include <asm/page.h>
 #include <asm/percpu.h>
 
+#ifdef MODULE
+  #define runtime_const_ptr(sym) (sym)
+#else
+  #include <asm/runtime-const.h>
+#endif
+extern unsigned long USER_PTR_MAX;
+
 #ifdef CONFIG_ADDRESS_MASKING
 /*
  * Mask out tag bits from the address.
  */
 static inline unsigned long __untagged_addr(unsigned long addr)
 {
-	asm (ALTERNATIVE("",
-			 "and " __percpu_arg([mask]) ", %[addr]", X86_FEATURE_LAM)
+	asm_inline (ALTERNATIVE("", "and " __percpu_arg([mask]) ", %[addr]",
+				X86_FEATURE_LAM)
 	     : [addr] "+r" (addr)
 	     : [mask] "m" (__my_cpu_var(tlbstate_untag_mask)));
 
@@ -46,21 +53,26 @@ static inline unsigned long __untagged_addr_remote(struct mm_struct *mm,
 
 #endif
 
-/*
- * The virtual address space space is logically divided into a kernel
- * half and a user half.  When cast to a signed type, user pointers
- * are positive and kernel pointers are negative.
- */
-#define valid_user_address(x) ((__force long)(x) >= 0)
+#define valid_user_address(x) \
+	likely((__force unsigned long)(x) <= runtime_const_ptr(USER_PTR_MAX))
 
 /*
  * Masking the user address is an alternative to a conditional
  * user_access_begin that can avoid the fencing. This only works
  * for dense accesses starting at the address.
  */
-#define mask_user_address(x) ((typeof(x))((long)(x)|((long)(x)>>63)))
+static inline void __user *mask_user_address(const void __user *ptr)
+{
+	void __user *ret;
+	asm("cmp %1,%0\n\t"
+	    "cmova %1,%0"
+		:"=r" (ret)
+		:"r" (runtime_const_ptr(USER_PTR_MAX)),
+		 "0" (ptr));
+	return ret;
+}
 #define masked_user_access_begin(x) ({				\
-	__auto_type __masked_ptr = (x);				\
+	auto __masked_ptr = (x);				\
 	__masked_ptr = mask_user_address(__masked_ptr);		\
 	__uaccess_begin(); __masked_ptr; })
 
@@ -69,23 +81,16 @@ static inline unsigned long __untagged_addr_remote(struct mm_struct *mm,
  * arbitrary values in those bits rather then masking them off.
  *
  * Enforce two rules:
- * 1. 'ptr' must be in the user half of the address space
+ * 1. 'ptr' must be in the user part of the address space
  * 2. 'ptr+size' must not overflow into kernel addresses
  *
- * Note that addresses around the sign change are not valid addresses,
- * and will GP-fault even with LAM enabled if the sign bit is set (see
- * "CR3.LAM_SUP" that can narrow the canonicality check if we ever
- * enable it, but not remove it entirely).
- *
- * So the "overflow into kernel addresses" does not imply some sudden
- * exact boundary at the sign bit, and we can allow a lot of slop on the
- * size check.
+ * Note that we always have at least one guard page between the
+ * max user address and the non-canonical gap, allowing us to
+ * ignore small sizes entirely.
  *
  * In fact, we could probably remove the size check entirely, since
  * any kernel accesses will be in increasing address order starting
- * at 'ptr', and even if the end might be in kernel space, we'll
- * hit the GP faults for non-canonical accesses before we ever get
- * there.
+ * at 'ptr'.
  *
  * That's a separate optimization, for now just handle the small
  * constant case.
diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h
index 85cc57cb6539..8f4579c5a6f8 100644
--- a/arch/x86/include/asm/unwind_hints.h
+++ b/arch/x86/include/asm/unwind_hints.h
@@ -5,7 +5,7 @@
 
 #include "orc_types.h"
 
-#ifdef __ASSEMBLY__
+#ifdef __ASSEMBLER__
 
 .macro UNWIND_HINT_END_OF_STACK
 	UNWIND_HINT type=UNWIND_HINT_TYPE_END_OF_STACK
@@ -88,6 +88,6 @@
 #define UNWIND_HINT_RESTORE \
 	UNWIND_HINT(UNWIND_HINT_TYPE_RESTORE, 0, 0, 0)
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ASM_X86_UNWIND_HINTS_H */
diff --git a/arch/x86/include/asm/unwind_user.h b/arch/x86/include/asm/unwind_user.h
new file mode 100644
index 000000000000..12064284bc4e
--- /dev/null
+++ b/arch/x86/include/asm/unwind_user.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_UNWIND_USER_H
+#define _ASM_X86_UNWIND_USER_H
+
+#ifdef CONFIG_HAVE_UNWIND_USER_FP
+
+#include <asm/ptrace.h>
+#include <asm/uprobes.h>
+
+#define ARCH_INIT_USER_FP_FRAME(ws)			\
+	.cfa_off	=  2*(ws),			\
+	.ra_off		= -1*(ws),			\
+	.fp_off		= -2*(ws),			\
+	.use_fp		= true,
+
+#define ARCH_INIT_USER_FP_ENTRY_FRAME(ws)		\
+	.cfa_off	=  1*(ws),			\
+	.ra_off		= -1*(ws),			\
+	.fp_off		= 0,				\
+	.use_fp		= false,
+
+static inline int unwind_user_word_size(struct pt_regs *regs)
+{
+	/* We can't unwind VM86 stacks */
+	if (regs->flags & X86_VM_MASK)
+		return 0;
+#ifdef CONFIG_X86_64
+	if (!user_64bit_mode(regs))
+		return sizeof(int);
+#endif
+	return sizeof(long);
+}
+
+static inline bool unwind_user_at_function_start(struct pt_regs *regs)
+{
+	return is_uprobe_at_func_entry(regs);
+}
+
+#endif /* CONFIG_HAVE_UNWIND_USER_FP */
+
+#endif /* _ASM_X86_UNWIND_USER_H */
diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h
index 678fb546f0a7..362210c79998 100644
--- a/arch/x86/include/asm/uprobes.h
+++ b/arch/x86/include/asm/uprobes.h
@@ -20,6 +20,11 @@ typedef u8 uprobe_opcode_t;
 #define UPROBE_SWBP_INSN		0xcc
 #define UPROBE_SWBP_INSN_SIZE		   1
 
+enum {
+	ARCH_UPROBE_FLAG_CAN_OPTIMIZE   = 0,
+	ARCH_UPROBE_FLAG_OPTIMIZE_FAIL  = 1,
+};
+
 struct uprobe_xol_ops;
 
 struct arch_uprobe {
@@ -45,6 +50,8 @@ struct arch_uprobe {
 			u8	ilen;
 		}			push;
 	};
+
+	unsigned long flags;
 };
 
 struct arch_uprobe_task {
@@ -55,4 +62,13 @@ struct arch_uprobe_task {
 	unsigned int			saved_tf;
 };
 
+#ifdef CONFIG_UPROBES
+extern bool is_uprobe_at_func_entry(struct pt_regs *regs);
+#else
+static bool is_uprobe_at_func_entry(struct pt_regs *regs)
+{
+	return false;
+}
+#endif /* CONFIG_UPROBES */
+
 #endif	/* _ASM_UPROBES_H */
diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h
index d7f6592b74a9..b7253ef3205a 100644
--- a/arch/x86/include/asm/vdso.h
+++ b/arch/x86/include/asm/vdso.h
@@ -18,12 +18,6 @@ struct vdso_image {
 	unsigned long extable_base, extable_len;
 	const void *extable;
 
-	long sym_vvar_start;  /* Negative offset to the vvar area */
-
-	long sym_vvar_page;
-	long sym_pvclock_page;
-	long sym_hvclock_page;
-	long sym_timens_page;
 	long sym_VDSO32_NOTE_MASK;
 	long sym___kernel_sigreturn;
 	long sym___kernel_rt_sigreturn;
@@ -33,17 +27,9 @@ struct vdso_image {
 	long sym_vdso32_rt_sigreturn_landing_pad;
 };
 
-#ifdef CONFIG_X86_64
 extern const struct vdso_image vdso_image_64;
-#endif
-
-#ifdef CONFIG_X86_X32_ABI
 extern const struct vdso_image vdso_image_x32;
-#endif
-
-#if defined CONFIG_X86_32 || defined CONFIG_COMPAT
 extern const struct vdso_image vdso_image_32;
-#endif
 
 extern int __init init_vdso_image(const struct vdso_image *image);
 
diff --git a/arch/x86/include/asm/vdso/getrandom.h b/arch/x86/include/asm/vdso/getrandom.h
index ff5334ad32a0..ff1c11b9fa27 100644
--- a/arch/x86/include/asm/vdso/getrandom.h
+++ b/arch/x86/include/asm/vdso/getrandom.h
@@ -5,10 +5,9 @@
 #ifndef __ASM_VDSO_GETRANDOM_H
 #define __ASM_VDSO_GETRANDOM_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <asm/unistd.h>
-#include <asm/vvar.h>
 
 /**
  * getrandom_syscall - Invoke the getrandom() syscall.
@@ -28,15 +27,6 @@ static __always_inline ssize_t getrandom_syscall(void *buffer, size_t len, unsig
 	return ret;
 }
 
-#define __vdso_rng_data (VVAR(_vdso_rng_data))
-
-static __always_inline const struct vdso_rng_data *__arch_get_vdso_rng_data(void)
-{
-	if (IS_ENABLED(CONFIG_TIME_NS) && __vdso_data->clock_mode == VDSO_CLOCKMODE_TIMENS)
-		return (void *)&__vdso_rng_data + ((void *)&__timens_vdso_data - (void *)&__vdso_data);
-	return &__vdso_rng_data;
-}
-
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #endif /* __ASM_VDSO_GETRANDOM_H */
diff --git a/arch/x86/include/asm/vdso/gettimeofday.h b/arch/x86/include/asm/vdso/gettimeofday.h
index b2d2df026f6e..73b2e7ee8f0f 100644
--- a/arch/x86/include/asm/vdso/gettimeofday.h
+++ b/arch/x86/include/asm/vdso/gettimeofday.h
@@ -10,19 +10,15 @@
 #ifndef __ASM_VDSO_GETTIMEOFDAY_H
 #define __ASM_VDSO_GETTIMEOFDAY_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <uapi/linux/time.h>
 #include <asm/vgtod.h>
-#include <asm/vvar.h>
 #include <asm/unistd.h>
 #include <asm/msr.h>
 #include <asm/pvclock.h>
 #include <clocksource/hyperv_timer.h>
 
-#define __vdso_data (VVAR(_vdso_data))
-#define __timens_vdso_data (TIMENS(_vdso_data))
-
 #define VDSO_HAS_TIME 1
 
 #define VDSO_HAS_CLOCK_GETRES 1
@@ -57,14 +53,6 @@ extern struct ms_hyperv_tsc_page hvclock_page
 	__attribute__((visibility("hidden")));
 #endif
 
-#ifdef CONFIG_TIME_NS
-static __always_inline
-const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
-{
-	return __timens_vdso_data;
-}
-#endif
-
 #ifndef BUILD_VDSO32
 
 static __always_inline
@@ -248,7 +236,7 @@ static u64 vread_hvclock(void)
 #endif
 
 static inline u64 __arch_get_hw_counter(s32 clock_mode,
-					const struct vdso_data *vd)
+					const struct vdso_time_data *vd)
 {
 	if (likely(clock_mode == VDSO_CLOCKMODE_TSC))
 		return (u64)rdtsc_ordered() & S64_MAX;
@@ -273,12 +261,7 @@ static inline u64 __arch_get_hw_counter(s32 clock_mode,
 	return U64_MAX;
 }
 
-static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
-{
-	return __vdso_data;
-}
-
-static inline bool arch_vdso_clocksource_ok(const struct vdso_data *vd)
+static inline bool arch_vdso_clocksource_ok(const struct vdso_clock *vc)
 {
 	return true;
 }
@@ -317,37 +300,37 @@ static inline bool arch_vdso_cycles_ok(u64 cycles)
  * declares everything with the MSB/Sign-bit set as invalid. Therefore the
  * effective mask is S64_MAX.
  */
-static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base)
+static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base)
 {
-	u64 delta = cycles - vd->cycle_last;
+	u64 delta = cycles - vc->cycle_last;
 
 	/*
 	 * Negative motion and deltas which can cause multiplication
 	 * overflow require special treatment. This check covers both as
-	 * negative motion is guaranteed to be greater than @vd::max_cycles
+	 * negative motion is guaranteed to be greater than @vc::max_cycles
 	 * due to unsigned comparison.
 	 *
 	 * Due to the MSB/Sign-bit being used as invalid marker (see
 	 * arch_vdso_cycles_ok() above), the effective mask is S64_MAX, but that
 	 * case is also unlikely and will also take the unlikely path here.
 	 */
-	if (unlikely(delta > vd->max_cycles)) {
+	if (unlikely(delta > vc->max_cycles)) {
 		/*
 		 * Due to the above mentioned TSC wobbles, filter out
 		 * negative motion.  Per the above masking, the effective
 		 * sign bit is now bit 62.
 		 */
 		if (delta & (1ULL << 62))
-			return base >> vd->shift;
+			return base >> vc->shift;
 
 		/* Handle multiplication overflow gracefully */
-		return mul_u64_u32_add_u64_shr(delta & S64_MAX, vd->mult, base, vd->shift);
+		return mul_u64_u32_add_u64_shr(delta & S64_MAX, vc->mult, base, vc->shift);
 	}
 
-	return ((delta * vd->mult) + base) >> vd->shift;
+	return ((delta * vc->mult) + base) >> vc->shift;
 }
 #define vdso_calc_ns vdso_calc_ns
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #endif /* __ASM_VDSO_GETTIMEOFDAY_H */
diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h
index 2cbce97d29ea..7000aeb59aa2 100644
--- a/arch/x86/include/asm/vdso/processor.h
+++ b/arch/x86/include/asm/vdso/processor.h
@@ -5,23 +5,23 @@
 #ifndef __ASM_VDSO_PROCESSOR_H
 #define __ASM_VDSO_PROCESSOR_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
-/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
-static __always_inline void rep_nop(void)
+/* PAUSE is a good thing to insert into busy-wait loops. */
+static __always_inline void native_pause(void)
 {
-	asm volatile("rep; nop" ::: "memory");
+	asm volatile("pause" ::: "memory");
 }
 
 static __always_inline void cpu_relax(void)
 {
-	rep_nop();
+	native_pause();
 }
 
 struct getcpu_cache;
 
 notrace long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused);
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* __ASM_VDSO_PROCESSOR_H */
diff --git a/arch/x86/include/asm/vdso/vsyscall.h b/arch/x86/include/asm/vdso/vsyscall.h
index 67fedf1698b5..4aa311a923f2 100644
--- a/arch/x86/include/asm/vdso/vsyscall.h
+++ b/arch/x86/include/asm/vdso/vsyscall.h
@@ -2,33 +2,21 @@
 #ifndef __ASM_VDSO_VSYSCALL_H
 #define __ASM_VDSO_VSYSCALL_H
 
-#ifndef __ASSEMBLY__
+#define __VDSO_PAGES	6
 
-#include <linux/timekeeper_internal.h>
-#include <vdso/datapage.h>
-#include <asm/vgtod.h>
-#include <asm/vvar.h>
+#define VDSO_NR_VCLOCK_PAGES	2
+#define VDSO_VCLOCK_PAGES_START(_b)	((_b) + (__VDSO_PAGES - VDSO_NR_VCLOCK_PAGES) * PAGE_SIZE)
+#define VDSO_PAGE_PVCLOCK_OFFSET	0
+#define VDSO_PAGE_HVCLOCK_OFFSET	1
 
-/*
- * Update the vDSO data page to keep in sync with kernel timekeeping.
- */
-static __always_inline
-struct vdso_data *__x86_get_k_vdso_data(void)
-{
-	return _vdso_data;
-}
-#define __arch_get_k_vdso_data __x86_get_k_vdso_data
+#ifndef __ASSEMBLER__
 
-static __always_inline
-struct vdso_rng_data *__x86_get_k_vdso_rng_data(void)
-{
-	return &_vdso_rng_data;
-}
-#define __arch_get_k_vdso_rng_data __x86_get_k_vdso_rng_data
+#include <vdso/datapage.h>
+#include <asm/vgtod.h>
 
 /* The asm-generic header needs to be included after the definitions above */
 #include <asm-generic/vdso/vsyscall.h>
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #endif /* __ASM_VDSO_VSYSCALL_H */
diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h
index 75884d2cdec3..5d471253c755 100644
--- a/arch/x86/include/asm/vermagic.h
+++ b/arch/x86/include/asm/vermagic.h
@@ -15,8 +15,6 @@
 #define MODULE_PROC_FAMILY "586TSC "
 #elif defined CONFIG_M586MMX
 #define MODULE_PROC_FAMILY "586MMX "
-#elif defined CONFIG_MCORE2
-#define MODULE_PROC_FAMILY "CORE2 "
 #elif defined CONFIG_MATOM
 #define MODULE_PROC_FAMILY "ATOM "
 #elif defined CONFIG_M686
@@ -33,8 +31,6 @@
 #define MODULE_PROC_FAMILY "K6 "
 #elif defined CONFIG_MK7
 #define MODULE_PROC_FAMILY "K7 "
-#elif defined CONFIG_MK8
-#define MODULE_PROC_FAMILY "K8 "
 #elif defined CONFIG_MELAN
 #define MODULE_PROC_FAMILY "ELAN "
 #elif defined CONFIG_MCRUSOE
diff --git a/arch/x86/include/asm/video.h b/arch/x86/include/asm/video.h
index 0950c9535fae..08ec328203ef 100644
--- a/arch/x86/include/asm/video.h
+++ b/arch/x86/include/asm/video.h
@@ -13,8 +13,10 @@ pgprot_t pgprot_framebuffer(pgprot_t prot,
 			    unsigned long offset);
 #define pgprot_framebuffer pgprot_framebuffer
 
+#ifdef CONFIG_VIDEO
 bool video_is_primary_device(struct device *dev);
 #define video_is_primary_device video_is_primary_device
+#endif
 
 #include <asm-generic/video.h>
 
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index f7fd4369b821..c85c50019523 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -106,6 +106,7 @@
 #define VM_EXIT_CLEAR_BNDCFGS                   0x00800000
 #define VM_EXIT_PT_CONCEAL_PIP			0x01000000
 #define VM_EXIT_CLEAR_IA32_RTIT_CTL		0x02000000
+#define VM_EXIT_LOAD_CET_STATE                  0x10000000
 
 #define VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR	0x00036dff
 
@@ -119,6 +120,7 @@
 #define VM_ENTRY_LOAD_BNDCFGS                   0x00010000
 #define VM_ENTRY_PT_CONCEAL_PIP			0x00020000
 #define VM_ENTRY_LOAD_IA32_RTIT_CTL		0x00040000
+#define VM_ENTRY_LOAD_CET_STATE                 0x00100000
 
 #define VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR	0x000011ff
 
@@ -132,6 +134,7 @@
 #define VMX_BASIC_DUAL_MONITOR_TREATMENT	BIT_ULL(49)
 #define VMX_BASIC_INOUT				BIT_ULL(54)
 #define VMX_BASIC_TRUE_CTLS			BIT_ULL(55)
+#define VMX_BASIC_NO_HW_ERROR_CODE_CC		BIT_ULL(56)
 
 static inline u32 vmx_basic_vmcs_revision_id(u64 vmx_basic)
 {
@@ -256,6 +259,7 @@ enum vmcs_field {
 	TSC_MULTIPLIER_HIGH             = 0x00002033,
 	TERTIARY_VM_EXEC_CONTROL	= 0x00002034,
 	TERTIARY_VM_EXEC_CONTROL_HIGH	= 0x00002035,
+	SHARED_EPT_POINTER		= 0x0000203C,
 	PID_POINTER_TABLE		= 0x00002042,
 	PID_POINTER_TABLE_HIGH		= 0x00002043,
 	GUEST_PHYSICAL_ADDRESS          = 0x00002400,
@@ -368,6 +372,9 @@ enum vmcs_field {
 	GUEST_PENDING_DBG_EXCEPTIONS    = 0x00006822,
 	GUEST_SYSENTER_ESP              = 0x00006824,
 	GUEST_SYSENTER_EIP              = 0x00006826,
+	GUEST_S_CET                     = 0x00006828,
+	GUEST_SSP                       = 0x0000682a,
+	GUEST_INTR_SSP_TABLE            = 0x0000682c,
 	HOST_CR0                        = 0x00006c00,
 	HOST_CR3                        = 0x00006c02,
 	HOST_CR4                        = 0x00006c04,
@@ -380,6 +387,9 @@ enum vmcs_field {
 	HOST_IA32_SYSENTER_EIP          = 0x00006c12,
 	HOST_RSP                        = 0x00006c14,
 	HOST_RIP                        = 0x00006c16,
+	HOST_S_CET                      = 0x00006c18,
+	HOST_SSP                        = 0x00006c1a,
+	HOST_INTR_SSP_TABLE             = 0x00006c1c
 };
 
 /*
@@ -580,18 +590,23 @@ enum vm_entry_failure_code {
 /*
  * Exit Qualifications for EPT Violations
  */
-#define EPT_VIOLATION_ACC_READ_BIT	0
-#define EPT_VIOLATION_ACC_WRITE_BIT	1
-#define EPT_VIOLATION_ACC_INSTR_BIT	2
-#define EPT_VIOLATION_RWX_SHIFT		3
-#define EPT_VIOLATION_GVA_IS_VALID_BIT	7
-#define EPT_VIOLATION_GVA_TRANSLATED_BIT 8
-#define EPT_VIOLATION_ACC_READ		(1 << EPT_VIOLATION_ACC_READ_BIT)
-#define EPT_VIOLATION_ACC_WRITE		(1 << EPT_VIOLATION_ACC_WRITE_BIT)
-#define EPT_VIOLATION_ACC_INSTR		(1 << EPT_VIOLATION_ACC_INSTR_BIT)
-#define EPT_VIOLATION_RWX_MASK		(VMX_EPT_RWX_MASK << EPT_VIOLATION_RWX_SHIFT)
-#define EPT_VIOLATION_GVA_IS_VALID	(1 << EPT_VIOLATION_GVA_IS_VALID_BIT)
-#define EPT_VIOLATION_GVA_TRANSLATED	(1 << EPT_VIOLATION_GVA_TRANSLATED_BIT)
+#define EPT_VIOLATION_ACC_READ		BIT(0)
+#define EPT_VIOLATION_ACC_WRITE		BIT(1)
+#define EPT_VIOLATION_ACC_INSTR		BIT(2)
+#define EPT_VIOLATION_PROT_READ		BIT(3)
+#define EPT_VIOLATION_PROT_WRITE	BIT(4)
+#define EPT_VIOLATION_PROT_EXEC		BIT(5)
+#define EPT_VIOLATION_EXEC_FOR_RING3_LIN BIT(6)
+#define EPT_VIOLATION_PROT_MASK		(EPT_VIOLATION_PROT_READ  | \
+					 EPT_VIOLATION_PROT_WRITE | \
+					 EPT_VIOLATION_PROT_EXEC)
+#define EPT_VIOLATION_GVA_IS_VALID	BIT(7)
+#define EPT_VIOLATION_GVA_TRANSLATED	BIT(8)
+
+#define EPT_VIOLATION_RWX_TO_PROT(__epte) (((__epte) & VMX_EPT_RWX_MASK) << 3)
+
+static_assert(EPT_VIOLATION_RWX_TO_PROT(VMX_EPT_RWX_MASK) ==
+	      (EPT_VIOLATION_PROT_READ | EPT_VIOLATION_PROT_WRITE | EPT_VIOLATION_PROT_EXEC));
 
 /*
  * Exit Qualifications for NOTIFY VM EXIT
diff --git a/arch/x86/include/asm/vvar.h b/arch/x86/include/asm/vvar.h
deleted file mode 100644
index 9d9af37f7cab..000000000000
--- a/arch/x86/include/asm/vvar.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * vvar.h: Shared vDSO/kernel variable declarations
- * Copyright (c) 2011 Andy Lutomirski
- *
- * A handful of variables are accessible (read-only) from userspace
- * code in the vsyscall page and the vdso.  They are declared here.
- * Some other file must define them with DEFINE_VVAR.
- *
- * In normal kernel code, they are used like any other variable.
- * In user code, they are accessed through the VVAR macro.
- *
- * These variables live in a page of kernel data that has an extra RO
- * mapping for userspace.  Each variable needs a unique offset within
- * that page; specify that offset with the DECLARE_VVAR macro.  (If
- * you mess up, the linker will catch it.)
- */
-
-#ifndef _ASM_X86_VVAR_H
-#define _ASM_X86_VVAR_H
-
-#ifdef EMIT_VVAR
-/*
- * EMIT_VVAR() is used by the kernel linker script to put vvars in the
- * right place. Also, it's used by kernel code to import offsets values.
- */
-#define DECLARE_VVAR(offset, type, name) \
-	EMIT_VVAR(name, offset)
-#define DECLARE_VVAR_SINGLE(offset, type, name) \
-	EMIT_VVAR(name, offset)
-
-#else
-
-extern char __vvar_page;
-
-#define DECLARE_VVAR(offset, type, name)				\
-	extern type vvar_ ## name[CS_BASES]				\
-	__attribute__((visibility("hidden")));				\
-	extern type timens_ ## name[CS_BASES]				\
-	__attribute__((visibility("hidden")));				\
-
-#define DECLARE_VVAR_SINGLE(offset, type, name)				\
-	extern type vvar_ ## name					\
-	__attribute__((visibility("hidden")));				\
-
-#define VVAR(name) (vvar_ ## name)
-#define TIMENS(name) (timens_ ## name)
-
-#define DEFINE_VVAR(type, name)						\
-	type name[CS_BASES]						\
-	__attribute__((section(".vvar_" #name), aligned(16))) __visible
-
-#define DEFINE_VVAR_SINGLE(type, name)					\
-	type name							\
-	__attribute__((section(".vvar_" #name), aligned(16))) __visible
-
-#endif
-
-/* DECLARE_VVAR(offset, type, name) */
-
-DECLARE_VVAR(128, struct vdso_data, _vdso_data)
-
-#if !defined(_SINGLE_DATA)
-#define _SINGLE_DATA
-DECLARE_VVAR_SINGLE(640, struct vdso_rng_data, _vdso_rng_data)
-#endif
-
-#undef DECLARE_VVAR
-#undef DECLARE_VVAR_SINGLE
-
-#endif
diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
index 213cf5379a5a..6c8a6ead84f6 100644
--- a/arch/x86/include/asm/x86_init.h
+++ b/arch/x86/include/asm/x86_init.h
@@ -79,7 +79,7 @@ struct x86_init_paging {
 
 /**
  * struct x86_init_timers - platform specific timer setup
- * @setup_perpcu_clockev:	set up the per cpu clock event device for the
+ * @setup_percpu_clockev:	set up the per cpu clock event device for the
  *				boot cpu
  * @timer_init:			initialize the platform timer (default PIT/HPET)
  * @wallclock_init:		init the wallclock device
@@ -132,7 +132,7 @@ struct x86_hyper_init {
 
 /**
  * struct x86_init_acpi - x86 ACPI init functions
- * @set_root_poitner:		set RSDP address
+ * @set_root_pointer:		set RSDP address
  * @get_root_pointer:		get RSDP address
  * @reduced_hw_early_init:	hardware reduced platform early init
  */
@@ -145,14 +145,14 @@ struct x86_init_acpi {
 /**
  * struct x86_guest - Functions used by misc guest incarnations like SEV, TDX, etc.
  *
- * @enc_status_change_prepare	Notify HV before the encryption status of a range is changed
- * @enc_status_change_finish	Notify HV after the encryption status of a range is changed
- * @enc_tlb_flush_required	Returns true if a TLB flush is needed before changing page encryption status
- * @enc_cache_flush_required	Returns true if a cache flush is needed before changing page encryption status
- * @enc_kexec_begin		Begin the two-step process of converting shared memory back
+ * @enc_status_change_prepare:	Notify HV before the encryption status of a range is changed
+ * @enc_status_change_finish:	Notify HV after the encryption status of a range is changed
+ * @enc_tlb_flush_required:	Returns true if a TLB flush is needed before changing page encryption status
+ * @enc_cache_flush_required:	Returns true if a cache flush is needed before changing page encryption status
+ * @enc_kexec_begin:		Begin the two-step process of converting shared memory back
  *				to private. It stops the new conversions from being started
  *				and waits in-flight conversions to finish, if possible.
- * @enc_kexec_finish		Finish the two-step process of converting shared memory to
+ * @enc_kexec_finish:		Finish the two-step process of converting shared memory to
  *				private. All memory is private after the call when
  *				the function returns.
  *				It is called on only one CPU while the others are shut down
@@ -229,7 +229,7 @@ struct x86_legacy_devices {
  *	given platform/subarch.
  * @X86_LEGACY_I8042_FIRMWARE_ABSENT: firmware reports that the controller
  *	is absent.
- * @X86_LEGACY_i8042_EXPECTED_PRESENT: the controller is likely to be
+ * @X86_LEGACY_I8042_EXPECTED_PRESENT: the controller is likely to be
  *	present, the i8042 driver should probe for controller existence.
  */
 enum x86_legacy_i8042_state {
@@ -244,6 +244,8 @@ enum x86_legacy_i8042_state {
  * @i8042: indicated if we expect the device to have i8042 controller
  *	present.
  * @rtc: this device has a CMOS real-time clock present
+ * @warm_reset: 1 if platform allows warm reset, else 0
+ * @no_vga: 1 if (FADT.boot_flags & ACPI_FADT_NO_VGA) is set, else 0
  * @reserve_bios_regions: boot code will search for the EBDA address and the
  * 	start of the 640k - 1M BIOS region.  If false, the platform must
  * 	ensure that its memory map correctly reserves sub-1MB regions as needed.
@@ -290,8 +292,10 @@ struct x86_hyper_runtime {
  * @calibrate_tsc:		calibrate TSC, if different from CPU
  * @get_wallclock:		get time from HW clock like RTC etc.
  * @set_wallclock:		set time back to HW clock
- * @is_untracked_pat_range	exclude from PAT logic
- * @nmi_init			enable NMI on cpus
+ * @iommu_shutdown:		set by an IOMMU driver for shutdown if necessary
+ * @is_untracked_pat_range:	exclude from PAT logic
+ * @nmi_init:			enable NMI on cpus
+ * @get_nmi_reason:		get the reason an NMI was received
  * @save_sched_clock_state:	save state for sched_clock() on suspend
  * @restore_sched_clock_state:	restore state for sched_clock() on resume
  * @apic_post_init:		adjust apic if needed
@@ -306,6 +310,7 @@ struct x86_hyper_runtime {
  * @realmode_reserve:		reserve memory for realmode trampoline
  * @realmode_init:		initialize realmode trampoline
  * @hyper:			x86 hypervisor specific runtime callbacks
+ * @guest:			guest incarnations callbacks
  */
 struct x86_platform_ops {
 	unsigned long (*calibrate_cpu)(void);
diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h
index a2dd24947eb8..a16d4631547c 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -39,9 +39,11 @@
 #include <linux/string.h>
 #include <linux/types.h>
 #include <linux/pgtable.h>
+#include <linux/instrumentation.h>
 
 #include <trace/events/xen.h>
 
+#include <asm/alternative.h>
 #include <asm/page.h>
 #include <asm/smap.h>
 #include <asm/nospec-branch.h>
@@ -86,11 +88,21 @@ struct xen_dm_op_buf;
  * there aren't more than 5 arguments...)
  */
 
-extern struct { char _entry[32]; } hypercall_page[];
+void xen_hypercall_func(void);
+DECLARE_STATIC_CALL(xen_hypercall, xen_hypercall_func);
 
-#define __HYPERCALL		"call hypercall_page+%c[offset]"
-#define __HYPERCALL_ENTRY(x)						\
-	[offset] "i" (__HYPERVISOR_##x * sizeof(hypercall_page[0]))
+#ifdef MODULE
+#define __ADDRESSABLE_xen_hypercall
+#else
+#define __ADDRESSABLE_xen_hypercall \
+	__stringify(.global STATIC_CALL_KEY(xen_hypercall);)
+#endif
+
+#define __HYPERCALL					\
+	__ADDRESSABLE_xen_hypercall			\
+	__stringify(call STATIC_CALL_TRAMP(xen_hypercall))
+
+#define __HYPERCALL_ENTRY(x)	"a" (x)
 
 #ifdef CONFIG_X86_32
 #define __HYPERCALL_RETREG	"eax"
@@ -148,7 +160,7 @@ extern struct { char _entry[32]; } hypercall_page[];
 	__HYPERCALL_0ARG();						\
 	asm volatile (__HYPERCALL					\
 		      : __HYPERCALL_0PARAM				\
-		      : __HYPERCALL_ENTRY(name)				\
+		      : __HYPERCALL_ENTRY(__HYPERVISOR_ ## name)	\
 		      : __HYPERCALL_CLOBBER0);				\
 	(type)__res;							\
 })
@@ -159,7 +171,7 @@ extern struct { char _entry[32]; } hypercall_page[];
 	__HYPERCALL_1ARG(a1);						\
 	asm volatile (__HYPERCALL					\
 		      : __HYPERCALL_1PARAM				\
-		      : __HYPERCALL_ENTRY(name)				\
+		      : __HYPERCALL_ENTRY(__HYPERVISOR_ ## name)	\
 		      : __HYPERCALL_CLOBBER1);				\
 	(type)__res;							\
 })
@@ -170,7 +182,7 @@ extern struct { char _entry[32]; } hypercall_page[];
 	__HYPERCALL_2ARG(a1, a2);					\
 	asm volatile (__HYPERCALL					\
 		      : __HYPERCALL_2PARAM				\
-		      : __HYPERCALL_ENTRY(name)				\
+		      : __HYPERCALL_ENTRY(__HYPERVISOR_ ## name)	\
 		      : __HYPERCALL_CLOBBER2);				\
 	(type)__res;							\
 })
@@ -181,7 +193,7 @@ extern struct { char _entry[32]; } hypercall_page[];
 	__HYPERCALL_3ARG(a1, a2, a3);					\
 	asm volatile (__HYPERCALL					\
 		      : __HYPERCALL_3PARAM				\
-		      : __HYPERCALL_ENTRY(name)				\
+		      : __HYPERCALL_ENTRY(__HYPERVISOR_ ## name)	\
 		      : __HYPERCALL_CLOBBER3);				\
 	(type)__res;							\
 })
@@ -192,7 +204,7 @@ extern struct { char _entry[32]; } hypercall_page[];
 	__HYPERCALL_4ARG(a1, a2, a3, a4);				\
 	asm volatile (__HYPERCALL					\
 		      : __HYPERCALL_4PARAM				\
-		      : __HYPERCALL_ENTRY(name)				\
+		      : __HYPERCALL_ENTRY(__HYPERVISOR_ ## name)	\
 		      : __HYPERCALL_CLOBBER4);				\
 	(type)__res;							\
 })
@@ -206,12 +218,9 @@ xen_single_call(unsigned int call,
 	__HYPERCALL_DECLS;
 	__HYPERCALL_5ARG(a1, a2, a3, a4, a5);
 
-	if (call >= PAGE_SIZE / sizeof(hypercall_page[0]))
-		return -EINVAL;
-
-	asm volatile(CALL_NOSPEC
+	asm volatile(__HYPERCALL
 		     : __HYPERCALL_5PARAM
-		     : [thunk_target] "a" (&hypercall_page[call])
+		     : __HYPERCALL_ENTRY(call)
 		     : __HYPERCALL_CLOBBER5);
 
 	return (long)__res;
@@ -223,14 +232,12 @@ static __always_inline void __xen_stac(void)
 	 * Suppress objtool seeing the STAC/CLAC and getting confused about it
 	 * calling random code with AC=1.
 	 */
-	asm volatile(ANNOTATE_IGNORE_ALTERNATIVE
-		     ASM_STAC ::: "memory", "flags");
+	asm volatile(ASM_STAC_UNSAFE ::: "memory", "flags");
 }
 
 static __always_inline void __xen_clac(void)
 {
-	asm volatile(ANNOTATE_IGNORE_ALTERNATIVE
-		     ASM_CLAC ::: "memory", "flags");
+	asm volatile(ASM_CLAC_UNSAFE ::: "memory", "flags");
 }
 
 static inline long
diff --git a/arch/x86/include/asm/xen/hypervisor.h b/arch/x86/include/asm/xen/hypervisor.h
index a9088250770f..c2fc7869b996 100644
--- a/arch/x86/include/asm/xen/hypervisor.h
+++ b/arch/x86/include/asm/xen/hypervisor.h
@@ -43,7 +43,7 @@ extern struct start_info *xen_start_info;
 
 static inline uint32_t xen_cpuid_base(void)
 {
-	return hypervisor_cpuid_base(XEN_SIGNATURE, 2);
+	return cpuid_base_hypervisor(XEN_SIGNATURE, 2);
 }
 
 struct pci_dev;
@@ -72,18 +72,10 @@ enum xen_lazy_mode {
 };
 
 DECLARE_PER_CPU(enum xen_lazy_mode, xen_lazy_mode);
-DECLARE_PER_CPU(unsigned int, xen_lazy_nesting);
 
 static inline void enter_lazy(enum xen_lazy_mode mode)
 {
-	enum xen_lazy_mode old_mode = this_cpu_read(xen_lazy_mode);
-
-	if (mode == old_mode) {
-		this_cpu_inc(xen_lazy_nesting);
-		return;
-	}
-
-	BUG_ON(old_mode != XEN_LAZY_NONE);
+	BUG_ON(this_cpu_read(xen_lazy_mode) != XEN_LAZY_NONE);
 
 	this_cpu_write(xen_lazy_mode, mode);
 }
@@ -92,10 +84,7 @@ static inline void leave_lazy(enum xen_lazy_mode mode)
 {
 	BUG_ON(this_cpu_read(xen_lazy_mode) != mode);
 
-	if (this_cpu_read(xen_lazy_nesting) == 0)
-		this_cpu_write(xen_lazy_mode, XEN_LAZY_NONE);
-	else
-		this_cpu_dec(xen_lazy_nesting);
+	this_cpu_write(xen_lazy_mode, XEN_LAZY_NONE);
 }
 
 enum xen_lazy_mode xen_get_lazy_mode(void);
diff --git a/arch/x86/include/asm/xen/interface.h b/arch/x86/include/asm/xen/interface.h
index baca0b00ef76..a078a2b0f032 100644
--- a/arch/x86/include/asm/xen/interface.h
+++ b/arch/x86/include/asm/xen/interface.h
@@ -72,7 +72,7 @@
 #endif
 #endif
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 /* Explicitly size integers that represent pfns in the public interface
  * with Xen so that on ARM we can have one ABI that works for 32 and 64
  * bit guests. */
@@ -137,7 +137,7 @@ DEFINE_GUEST_HANDLE(xen_ulong_t);
 #define TI_SET_DPL(_ti, _dpl)	((_ti)->flags |= (_dpl))
 #define TI_SET_IF(_ti, _if)	((_ti)->flags |= ((!!(_if))<<2))
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 struct trap_info {
     uint8_t       vector;  /* exception vector                              */
     uint8_t       flags;   /* 0-3: privilege level; 4: clear event enable?  */
@@ -186,7 +186,7 @@ struct arch_shared_info {
 	uint32_t wc_sec_hi;
 #endif
 };
-#endif	/* !__ASSEMBLY__ */
+#endif	/* !__ASSEMBLER__ */
 
 #ifdef CONFIG_X86_32
 #include <asm/xen/interface_32.h>
@@ -196,7 +196,7 @@ struct arch_shared_info {
 
 #include <asm/pvclock-abi.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 /*
  * The following is all CPU context. Note that the fpu_ctxt block is filled
  * in by FXSAVE if the CPU has feature FXSR; otherwise FSAVE is used.
@@ -376,7 +376,7 @@ struct xen_pmu_arch {
 	} c;
 };
 
-#endif	/* !__ASSEMBLY__ */
+#endif	/* !__ASSEMBLER__ */
 
 /*
  * Prefix forces emulation of some non-trapping instructions.
diff --git a/arch/x86/include/asm/xen/interface_32.h b/arch/x86/include/asm/xen/interface_32.h
index dc40578abded..74d9768a9cf7 100644
--- a/arch/x86/include/asm/xen/interface_32.h
+++ b/arch/x86/include/asm/xen/interface_32.h
@@ -44,7 +44,7 @@
  */
 #define __HYPERVISOR_VIRT_START 0xF5800000
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 struct cpu_user_regs {
     uint32_t ebx;
@@ -85,7 +85,7 @@ typedef struct xen_callback xen_callback_t;
 
 #define XEN_CALLBACK(__cs, __eip)				\
 	((struct xen_callback){ .cs = (__cs), .eip = (unsigned long)(__eip) })
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 
 /*
diff --git a/arch/x86/include/asm/xen/interface_64.h b/arch/x86/include/asm/xen/interface_64.h
index c10f279aae93..38a19edb81a3 100644
--- a/arch/x86/include/asm/xen/interface_64.h
+++ b/arch/x86/include/asm/xen/interface_64.h
@@ -77,7 +77,7 @@
 #define VGCF_in_syscall  (1<<_VGCF_in_syscall)
 #define VGCF_IN_SYSCALL  VGCF_in_syscall
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 struct iret_context {
     /* Top of stack (%rsp at point of hypercall). */
@@ -143,7 +143,7 @@ typedef unsigned long xen_callback_t;
 #define XEN_CALLBACK(__cs, __rip)				\
 	((unsigned long)(__rip))
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 
 #endif /* _ASM_X86_XEN_INTERFACE_64_H */
diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h
index 85e63d58c074..59f642a94b9d 100644
--- a/arch/x86/include/asm/xen/page.h
+++ b/arch/x86/include/asm/xen/page.h
@@ -12,9 +12,9 @@
 #include <asm/extable.h>
 #include <asm/page.h>
 
+#include <xen/xen.h>
 #include <xen/interface/xen.h>
 #include <xen/interface/grant_table.h>
-#include <xen/features.h>
 
 /* Xen machine address */
 typedef struct xmaddr {
@@ -162,7 +162,7 @@ static inline unsigned long pfn_to_mfn(unsigned long pfn)
 	 * pfn_to_mfn. This will have to be removed when we figured
 	 * out which call.
 	 */
-	if (xen_feature(XENFEAT_auto_translated_physmap))
+	if (!xen_pv_domain())
 		return pfn;
 
 	mfn = __pfn_to_mfn(pfn);
@@ -175,7 +175,7 @@ static inline unsigned long pfn_to_mfn(unsigned long pfn)
 
 static inline int phys_to_machine_mapping_valid(unsigned long pfn)
 {
-	if (xen_feature(XENFEAT_auto_translated_physmap))
+	if (!xen_pv_domain())
 		return 1;
 
 	return __pfn_to_mfn(pfn) != INVALID_P2M_ENTRY;
@@ -210,7 +210,7 @@ static inline unsigned long mfn_to_pfn(unsigned long mfn)
 	 * gfn_to_pfn. This will have to be removed when we figure
 	 * out which call.
 	 */
-	if (xen_feature(XENFEAT_auto_translated_physmap))
+	if (!xen_pv_domain())
 		return mfn;
 
 	pfn = mfn_to_pfn_no_overrides(mfn);
@@ -242,7 +242,7 @@ static inline xpaddr_t machine_to_phys(xmaddr_t machine)
 /* Pseudo-physical <-> Guest conversion */
 static inline unsigned long pfn_to_gfn(unsigned long pfn)
 {
-	if (xen_feature(XENFEAT_auto_translated_physmap))
+	if (!xen_pv_domain())
 		return pfn;
 	else
 		return pfn_to_mfn(pfn);
@@ -250,7 +250,7 @@ static inline unsigned long pfn_to_gfn(unsigned long pfn)
 
 static inline unsigned long gfn_to_pfn(unsigned long gfn)
 {
-	if (xen_feature(XENFEAT_auto_translated_physmap))
+	if (!xen_pv_domain())
 		return gfn;
 	else
 		return mfn_to_pfn(gfn);
@@ -284,7 +284,7 @@ static inline unsigned long bfn_to_local_pfn(unsigned long mfn)
 {
 	unsigned long pfn;
 
-	if (xen_feature(XENFEAT_auto_translated_physmap))
+	if (!xen_pv_domain())
 		return mfn;
 
 	pfn = mfn_to_pfn(mfn);
diff --git a/arch/x86/include/uapi/asm/amd_hsmp.h b/arch/x86/include/uapi/asm/amd_hsmp.h
index e5d182c7373c..92d8f256d096 100644
--- a/arch/x86/include/uapi/asm/amd_hsmp.h
+++ b/arch/x86/include/uapi/asm/amd_hsmp.h
@@ -50,6 +50,12 @@ enum hsmp_message_ids {
 	HSMP_GET_METRIC_TABLE_VER,	/* 23h Get metrics table version */
 	HSMP_GET_METRIC_TABLE,		/* 24h Get metrics table */
 	HSMP_GET_METRIC_TABLE_DRAM_ADDR,/* 25h Get metrics table dram address */
+	HSMP_SET_XGMI_PSTATE_RANGE,	/* 26h Set xGMI P-state range */
+	HSMP_CPU_RAIL_ISO_FREQ_POLICY,	/* 27h Get/Set Cpu Iso frequency policy */
+	HSMP_DFC_ENABLE_CTRL,		/* 28h Enable/Disable DF C-state */
+	HSMP_GET_RAPL_UNITS = 0x30,	/* 30h Get scaling factor for energy */
+	HSMP_GET_RAPL_CORE_COUNTER,	/* 31h Get core energy counter value */
+	HSMP_GET_RAPL_PACKAGE_COUNTER,	/* 32h Get package energy counter value */
 	HSMP_MSG_ID_MAX,
 };
 
@@ -65,6 +71,7 @@ enum hsmp_msg_type {
 	HSMP_RSVD = -1,
 	HSMP_SET  = 0,
 	HSMP_GET  = 1,
+	HSMP_SET_GET	= 2,
 };
 
 enum hsmp_proto_versions {
@@ -72,7 +79,8 @@ enum hsmp_proto_versions {
 	HSMP_PROTO_VER3,
 	HSMP_PROTO_VER4,
 	HSMP_PROTO_VER5,
-	HSMP_PROTO_VER6
+	HSMP_PROTO_VER6,
+	HSMP_PROTO_VER7
 };
 
 struct hsmp_msg_desc {
@@ -88,7 +96,8 @@ struct hsmp_msg_desc {
  *
  * Not supported messages would return -ENOMSG.
  */
-static const struct hsmp_msg_desc hsmp_msg_desc_table[] = {
+static const struct hsmp_msg_desc hsmp_msg_desc_table[]
+				__attribute__((unused)) = {
 	/* RESERVED */
 	{0, 0, HSMP_RSVD},
 
@@ -299,7 +308,7 @@ static const struct hsmp_msg_desc hsmp_msg_desc_table[] = {
 	 * HSMP_SET_POWER_MODE, num_args = 1, response_sz = 0
 	 * input: args[0] = power efficiency mode[2:0]
 	 */
-	{1, 0, HSMP_SET},
+	{1, 1, HSMP_SET_GET},
 
 	/*
 	 * HSMP_SET_PSTATE_MAX_MIN, num_args = 1, response_sz = 0
@@ -324,6 +333,58 @@ static const struct hsmp_msg_desc hsmp_msg_desc_table[] = {
 	 * output: args[1] = upper 32 bits of the address
 	 */
 	{0, 2, HSMP_GET},
+
+	/*
+	 * HSMP_SET_XGMI_PSTATE_RANGE, num_args = 1, response_sz = 0
+	 * input: args[0] = min xGMI p-state[15:8] + max xGMI p-state[7:0]
+	 */
+	{1, 0, HSMP_SET},
+
+	/*
+	 * HSMP_CPU_RAIL_ISO_FREQ_POLICY, num_args = 1, response_sz = 1
+	 * input: args[0] = set/get policy[31] +
+	 * disable/enable independent control[0]
+	 * output: args[0] = current policy[0]
+	 */
+	{1, 1, HSMP_SET_GET},
+
+	/*
+	 * HSMP_DFC_ENABLE_CTRL, num_args = 1, response_sz = 1
+	 * input: args[0] = set/get policy[31] + enable/disable DFC[0]
+	 * output: args[0] = current policy[0]
+	 */
+	{1, 1, HSMP_SET_GET},
+
+	/* RESERVED(0x29-0x2f) */
+	{0, 0, HSMP_RSVD},
+	{0, 0, HSMP_RSVD},
+	{0, 0, HSMP_RSVD},
+	{0, 0, HSMP_RSVD},
+	{0, 0, HSMP_RSVD},
+	{0, 0, HSMP_RSVD},
+	{0, 0, HSMP_RSVD},
+
+	/*
+	 * HSMP_GET_RAPL_UNITS, response_sz = 1
+	 * output: args[0] = tu value[19:16] + esu value[12:8]
+	 */
+	{0, 1, HSMP_GET},
+
+	/*
+	 * HSMP_GET_RAPL_CORE_COUNTER, num_args = 1, response_sz = 1
+	 * input: args[0] = apic id[15:0]
+	 * output: args[0] = lower 32 bits of energy
+	 * output: args[1] = upper 32 bits of energy
+	 */
+	{1, 2, HSMP_GET},
+
+	/*
+	 * HSMP_GET_RAPL_PACKAGE_COUNTER, num_args = 0, response_sz = 1
+	 * output: args[0] = lower 32 bits of energy
+	 * output: args[1] = upper 32 bits of energy
+	 */
+	{0, 2, HSMP_GET},
+
 };
 
 /* Metrics table (supported only with proto version 6) */
diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h
index 9b82eebd7add..dafbf581c515 100644
--- a/arch/x86/include/uapi/asm/bootparam.h
+++ b/arch/x86/include/uapi/asm/bootparam.h
@@ -26,7 +26,7 @@
 #define XLF_5LEVEL_ENABLED		(1<<6)
 #define XLF_MEM_ENCRYPTION		(1<<7)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/types.h>
 #include <linux/screen_info.h>
@@ -210,6 +210,6 @@ enum x86_hardware_subarch {
 	X86_NR_SUBARCHS,
 };
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _ASM_X86_BOOTPARAM_H */
diff --git a/arch/x86/include/uapi/asm/debugreg.h b/arch/x86/include/uapi/asm/debugreg.h
index 0007ba077c0c..41da492dfb01 100644
--- a/arch/x86/include/uapi/asm/debugreg.h
+++ b/arch/x86/include/uapi/asm/debugreg.h
@@ -15,7 +15,26 @@
    which debugging register was responsible for the trap.  The other bits
    are either reserved or not of interest to us. */
 
-/* Define reserved bits in DR6 which are always set to 1 */
+/*
+ * Define bits in DR6 which are set to 1 by default.
+ *
+ * This is also the DR6 architectural value following Power-up, Reset or INIT.
+ *
+ * Note, with the introduction of Bus Lock Detection (BLD) and Restricted
+ * Transactional Memory (RTM), the DR6 register has been modified:
+ *
+ * 1) BLD flag (bit 11) is no longer reserved to 1 if the CPU supports
+ *    Bus Lock Detection.  The assertion of a bus lock could clear it.
+ *
+ * 2) RTM flag (bit 16) is no longer reserved to 1 if the CPU supports
+ *    restricted transactional memory.  #DB occurred inside an RTM region
+ *    could clear it.
+ *
+ * Apparently, DR6.BLD and DR6.RTM are active low bits.
+ *
+ * As a result, DR6_RESERVED is an incorrect name now, but it is kept for
+ * compatibility.
+ */
 #define DR6_RESERVED	(0xFFFF0FF0)
 
 #define DR_TRAP0	(0x1)		/* db0 */
diff --git a/arch/x86/include/uapi/asm/e820.h b/arch/x86/include/uapi/asm/e820.h
index 2f491efe3a12..55bc66867156 100644
--- a/arch/x86/include/uapi/asm/e820.h
+++ b/arch/x86/include/uapi/asm/e820.h
@@ -54,7 +54,7 @@
  */
 #define E820_RESERVED_KERN        128
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/types.h>
 struct e820entry {
 	__u64 addr;	/* start of memory segment */
@@ -76,7 +76,7 @@ struct e820map {
 #define BIOS_ROM_BASE		0xffe00000
 #define BIOS_ROM_END		0xffffffff
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 
 #endif /* _UAPI_ASM_X86_E820_H */
diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h
index a8debbf2f702..7ceff6583652 100644
--- a/arch/x86/include/uapi/asm/kvm.h
+++ b/arch/x86/include/uapi/asm/kvm.h
@@ -35,6 +35,11 @@
 #define MC_VECTOR 18
 #define XM_VECTOR 19
 #define VE_VECTOR 20
+#define CP_VECTOR 21
+
+#define HV_VECTOR 28
+#define VC_VECTOR 29
+#define SX_VECTOR 30
 
 /* Select x86 specific features in <linux/kvm.h> */
 #define __KVM_HAVE_PIT
@@ -411,6 +416,35 @@ struct kvm_xcrs {
 	__u64 padding[16];
 };
 
+#define KVM_X86_REG_TYPE_MSR		2
+#define KVM_X86_REG_TYPE_KVM		3
+
+#define KVM_X86_KVM_REG_SIZE(reg)						\
+({										\
+	reg == KVM_REG_GUEST_SSP ? KVM_REG_SIZE_U64 : 0;			\
+})
+
+#define KVM_X86_REG_TYPE_SIZE(type, reg)					\
+({										\
+	__u64 type_size = (__u64)type << 32;					\
+										\
+	type_size |= type == KVM_X86_REG_TYPE_MSR ? KVM_REG_SIZE_U64 :		\
+		     type == KVM_X86_REG_TYPE_KVM ? KVM_X86_KVM_REG_SIZE(reg) :	\
+		     0;								\
+	type_size;								\
+})
+
+#define KVM_X86_REG_ID(type, index)				\
+	(KVM_REG_X86 | KVM_X86_REG_TYPE_SIZE(type, index) | index)
+
+#define KVM_X86_REG_MSR(index)					\
+	KVM_X86_REG_ID(KVM_X86_REG_TYPE_MSR, index)
+#define KVM_X86_REG_KVM(index)					\
+	KVM_X86_REG_ID(KVM_X86_REG_TYPE_KVM, index)
+
+/* KVM-defined registers starting from 0 */
+#define KVM_REG_GUEST_SSP	0
+
 #define KVM_SYNC_X86_REGS      (1UL << 0)
 #define KVM_SYNC_X86_SREGS     (1UL << 1)
 #define KVM_SYNC_X86_EVENTS    (1UL << 2)
@@ -440,6 +474,8 @@ struct kvm_sync_regs {
 #define KVM_X86_QUIRK_FIX_HYPERCALL_INSN	(1 << 5)
 #define KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS	(1 << 6)
 #define KVM_X86_QUIRK_SLOT_ZAP_ALL		(1 << 7)
+#define KVM_X86_QUIRK_STUFF_FEATURE_MSRS	(1 << 8)
+#define KVM_X86_QUIRK_IGNORE_GUEST_PAT		(1 << 9)
 
 #define KVM_STATE_NESTED_FORMAT_VMX	0
 #define KVM_STATE_NESTED_FORMAT_SVM	1
@@ -466,6 +502,7 @@ struct kvm_sync_regs {
 /* vendor-specific groups and attributes for system fd */
 #define KVM_X86_GRP_SEV			1
 #  define KVM_X86_SEV_VMSA_FEATURES	0
+#  define KVM_X86_SNP_POLICY_BITS	1
 
 struct kvm_vmx_nested_state_data {
 	__u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];
@@ -558,6 +595,9 @@ struct kvm_x86_mce {
 #define KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE	(1 << 7)
 #define KVM_XEN_HVM_CONFIG_SHARED_INFO_HVA	(1 << 8)
 
+#define KVM_XEN_MSR_MIN_INDEX			0x40000000u
+#define KVM_XEN_MSR_MAX_INDEX			0x4fffffffu
+
 struct kvm_xen_hvm_config {
 	__u32 flags;
 	__u32 msr;
@@ -840,6 +880,7 @@ struct kvm_sev_snp_launch_start {
 };
 
 /* Kept in sync with firmware values for simplicity. */
+#define KVM_SEV_PAGE_TYPE_INVALID		0x0
 #define KVM_SEV_SNP_PAGE_TYPE_NORMAL		0x1
 #define KVM_SEV_SNP_PAGE_TYPE_ZERO		0x3
 #define KVM_SEV_SNP_PAGE_TYPE_UNMEASURED	0x4
@@ -924,5 +965,82 @@ struct kvm_hyperv_eventfd {
 #define KVM_X86_SEV_VM		2
 #define KVM_X86_SEV_ES_VM	3
 #define KVM_X86_SNP_VM		4
+#define KVM_X86_TDX_VM		5
+
+/* Trust Domain eXtension sub-ioctl() commands. */
+enum kvm_tdx_cmd_id {
+	KVM_TDX_CAPABILITIES = 0,
+	KVM_TDX_INIT_VM,
+	KVM_TDX_INIT_VCPU,
+	KVM_TDX_INIT_MEM_REGION,
+	KVM_TDX_FINALIZE_VM,
+	KVM_TDX_GET_CPUID,
+
+	KVM_TDX_CMD_NR_MAX,
+};
+
+struct kvm_tdx_cmd {
+	/* enum kvm_tdx_cmd_id */
+	__u32 id;
+	/* flags for sub-commend. If sub-command doesn't use this, set zero. */
+	__u32 flags;
+	/*
+	 * data for each sub-command. An immediate or a pointer to the actual
+	 * data in process virtual address.  If sub-command doesn't use it,
+	 * set zero.
+	 */
+	__u64 data;
+	/*
+	 * Auxiliary error code.  The sub-command may return TDX SEAMCALL
+	 * status code in addition to -Exxx.
+	 */
+	__u64 hw_error;
+};
+
+struct kvm_tdx_capabilities {
+	__u64 supported_attrs;
+	__u64 supported_xfam;
+
+	__u64 kernel_tdvmcallinfo_1_r11;
+	__u64 user_tdvmcallinfo_1_r11;
+	__u64 kernel_tdvmcallinfo_1_r12;
+	__u64 user_tdvmcallinfo_1_r12;
+
+	__u64 reserved[250];
+
+	/* Configurable CPUID bits for userspace */
+	struct kvm_cpuid2 cpuid;
+};
+
+struct kvm_tdx_init_vm {
+	__u64 attributes;
+	__u64 xfam;
+	__u64 mrconfigid[6];	/* sha384 digest */
+	__u64 mrowner[6];	/* sha384 digest */
+	__u64 mrownerconfig[6];	/* sha384 digest */
+
+	/* The total space for TD_PARAMS before the CPUIDs is 256 bytes */
+	__u64 reserved[12];
+
+	/*
+	 * Call KVM_TDX_INIT_VM before vcpu creation, thus before
+	 * KVM_SET_CPUID2.
+	 * This configuration supersedes KVM_SET_CPUID2s for VCPUs because the
+	 * TDX module directly virtualizes those CPUIDs without VMM.  The user
+	 * space VMM, e.g. qemu, should make KVM_SET_CPUID2 consistent with
+	 * those values.  If it doesn't, KVM may have wrong idea of vCPUIDs of
+	 * the guest, and KVM may wrongly emulate CPUIDs or MSRs that the TDX
+	 * module doesn't virtualize.
+	 */
+	struct kvm_cpuid2 cpuid;
+};
+
+#define KVM_TDX_MEASURE_MEMORY_REGION   _BITULL(0)
+
+struct kvm_tdx_init_mem_region {
+	__u64 source_addr;
+	__u64 gpa;
+	__u64 nr_pages;
+};
 
 #endif /* _ASM_X86_KVM_H */
diff --git a/arch/x86/include/uapi/asm/ldt.h b/arch/x86/include/uapi/asm/ldt.h
index d62ac5db093b..a82c039d8e6a 100644
--- a/arch/x86/include/uapi/asm/ldt.h
+++ b/arch/x86/include/uapi/asm/ldt.h
@@ -12,7 +12,7 @@
 /* The size of each LDT entry. */
 #define LDT_ENTRY_SIZE	8
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 /*
  * Note on 64bit base and limit is ignored and you cannot set DS/ES/CS
  * not to the default values if you still want to do syscalls. This
@@ -44,5 +44,5 @@ struct user_desc {
 #define MODIFY_LDT_CONTENTS_STACK	1
 #define MODIFY_LDT_CONTENTS_CODE	2
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 #endif /* _ASM_X86_LDT_H */
diff --git a/arch/x86/include/uapi/asm/mce.h b/arch/x86/include/uapi/asm/mce.h
index db9adc081c5a..cb6b48a7c22b 100644
--- a/arch/x86/include/uapi/asm/mce.h
+++ b/arch/x86/include/uapi/asm/mce.h
@@ -8,7 +8,8 @@
 /*
  * Fields are zero when not available. Also, this struct is shared with
  * userspace mcelog and thus must keep existing fields at current offsets.
- * Only add new fields to the end of the structure
+ * Only add new, shared fields to the end of the structure.
+ * Do not add vendor-specific fields.
  */
 struct mce {
 	__u64 status;		/* Bank's MCi_STATUS MSR */
diff --git a/arch/x86/include/uapi/asm/mman.h b/arch/x86/include/uapi/asm/mman.h
index 46cdc941f958..ac1e6277212b 100644
--- a/arch/x86/include/uapi/asm/mman.h
+++ b/arch/x86/include/uapi/asm/mman.h
@@ -5,9 +5,6 @@
 #define MAP_32BIT	0x40		/* only give out 32bit addresses */
 #define MAP_ABOVE4G	0x80		/* only map above 4GB */
 
-/* Flags for map_shadow_stack(2) */
-#define SHADOW_STACK_SET_TOKEN	(1ULL << 0)	/* Set up a restore token in the shadow stack */
-
 #include <asm-generic/mman.h>
 
 #endif /* _ASM_X86_MMAN_H */
diff --git a/arch/x86/include/uapi/asm/msr.h b/arch/x86/include/uapi/asm/msr.h
index e7516b402a00..4b8917ca28fe 100644
--- a/arch/x86/include/uapi/asm/msr.h
+++ b/arch/x86/include/uapi/asm/msr.h
@@ -2,7 +2,7 @@
 #ifndef _UAPI_ASM_X86_MSR_H
 #define _UAPI_ASM_X86_MSR_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/types.h>
 #include <linux/ioctl.h>
@@ -10,5 +10,5 @@
 #define X86_IOC_RDMSR_REGS	_IOWR('c', 0xA0, __u32[8])
 #define X86_IOC_WRMSR_REGS	_IOWR('c', 0xA1, __u32[8])
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* _UAPI_ASM_X86_MSR_H */
diff --git a/arch/x86/include/uapi/asm/processor-flags.h b/arch/x86/include/uapi/asm/processor-flags.h
index f1a4adc78272..81d0c8bf1137 100644
--- a/arch/x86/include/uapi/asm/processor-flags.h
+++ b/arch/x86/include/uapi/asm/processor-flags.h
@@ -136,6 +136,8 @@
 #define X86_CR4_PKE		_BITUL(X86_CR4_PKE_BIT)
 #define X86_CR4_CET_BIT		23 /* enable Control-flow Enforcement Technology */
 #define X86_CR4_CET		_BITUL(X86_CR4_CET_BIT)
+#define X86_CR4_LASS_BIT	27 /* enable Linear Address Space Separation support */
+#define X86_CR4_LASS		_BITUL(X86_CR4_LASS_BIT)
 #define X86_CR4_LAM_SUP_BIT	28 /* LAM for supervisor pointers */
 #define X86_CR4_LAM_SUP		_BITUL(X86_CR4_LAM_SUP_BIT)
 
diff --git a/arch/x86/include/uapi/asm/ptrace-abi.h b/arch/x86/include/uapi/asm/ptrace-abi.h
index 16074b9c93bb..5823584dea13 100644
--- a/arch/x86/include/uapi/asm/ptrace-abi.h
+++ b/arch/x86/include/uapi/asm/ptrace-abi.h
@@ -25,7 +25,7 @@
 
 #else /* __i386__ */
 
-#if defined(__ASSEMBLY__) || defined(__FRAME_OFFSETS)
+#if defined(__ASSEMBLER__) || defined(__FRAME_OFFSETS)
 /*
  * C ABI says these regs are callee-preserved. They aren't saved on kernel entry
  * unless syscall needs a complete, fully filled "struct pt_regs".
@@ -57,7 +57,7 @@
 #define EFLAGS 144
 #define RSP 152
 #define SS 160
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 /* top of stack page */
 #define FRAME_SIZE 168
@@ -87,7 +87,7 @@
 
 #define PTRACE_SINGLEBLOCK	33	/* resume execution until next branch */
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/types.h>
 #endif
 
diff --git a/arch/x86/include/uapi/asm/ptrace.h b/arch/x86/include/uapi/asm/ptrace.h
index 85165c0edafc..e0b5b4f6226b 100644
--- a/arch/x86/include/uapi/asm/ptrace.h
+++ b/arch/x86/include/uapi/asm/ptrace.h
@@ -7,7 +7,7 @@
 #include <asm/processor-flags.h>
 
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #ifdef __i386__
 /* this struct defines the way the registers are stored on the
@@ -81,6 +81,6 @@ struct pt_regs {
 
 
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #endif /* _UAPI_ASM_X86_PTRACE_H */
diff --git a/arch/x86/include/uapi/asm/setup_data.h b/arch/x86/include/uapi/asm/setup_data.h
index b111b0c18544..2671c4e1b3a0 100644
--- a/arch/x86/include/uapi/asm/setup_data.h
+++ b/arch/x86/include/uapi/asm/setup_data.h
@@ -13,12 +13,13 @@
 #define SETUP_CC_BLOB			7
 #define SETUP_IMA			8
 #define SETUP_RNG_SEED			9
-#define SETUP_ENUM_MAX			SETUP_RNG_SEED
+#define SETUP_KEXEC_KHO			10
+#define SETUP_ENUM_MAX			SETUP_KEXEC_KHO
 
 #define SETUP_INDIRECT			(1<<31)
 #define SETUP_TYPE_MAX			(SETUP_ENUM_MAX | SETUP_INDIRECT)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/types.h>
 
@@ -78,6 +79,16 @@ struct ima_setup_data {
 	__u64 size;
 } __attribute__((packed));
 
-#endif /* __ASSEMBLY__ */
+/*
+ * Locations of kexec handover metadata
+ */
+struct kho_data {
+	__u64 fdt_addr;
+	__u64 fdt_size;
+	__u64 scratch_addr;
+	__u64 scratch_size;
+} __attribute__((packed));
+
+#endif /* __ASSEMBLER__ */
 
 #endif /* _UAPI_ASM_X86_SETUP_DATA_H */
diff --git a/arch/x86/include/uapi/asm/sgx.h b/arch/x86/include/uapi/asm/sgx.h
index 2dd35bbdc822..3c4d52072189 100644
--- a/arch/x86/include/uapi/asm/sgx.h
+++ b/arch/x86/include/uapi/asm/sgx.h
@@ -10,7 +10,7 @@
 
 /**
  * enum sgx_page_flags - page control flags
- * %SGX_PAGE_MEASURE:	Measure the page contents with a sequence of
+ * @SGX_PAGE_MEASURE:	Measure the page contents with a sequence of
  *			ENCLS[EEXTEND] operations.
  */
 enum sgx_page_flags {
@@ -143,6 +143,12 @@ struct sgx_enclave_run;
 /**
  * typedef sgx_enclave_user_handler_t - Exit handler function accepted by
  *					__vdso_sgx_enter_enclave()
+ * @rdi:	RDI at the time of EEXIT, undefined on AEX
+ * @rsi:	RSI at the time of EEXIT, undefined on AEX
+ * @rdx:	RDX at the time of EEXIT, undefined on AEX
+ * @rsp:	RSP (untrusted) at the time of EEXIT or AEX
+ * @r8:		R8 at the time of EEXIT, undefined on AEX
+ * @r9:		R9 at the time of EEXIT, undefined on AEX
  * @run:	The run instance given by the caller
  *
  * The register parameters contain the snapshot of their values at enclave
@@ -166,7 +172,7 @@ typedef int (*sgx_enclave_user_handler_t)(long rdi, long rsi, long rdx,
  * @exception_addr:		The address that triggered the exception
  * @user_handler:		User provided callback run on exception
  * @user_data:			Data passed to the user handler
- * @reserved			Reserved for future extensions
+ * @reserved:			Reserved for future extensions
  *
  * If @user_handler is provided, the handler will be invoked on all return paths
  * of the normal flow.  The user handler may transfer control, e.g. via a
diff --git a/arch/x86/include/uapi/asm/signal.h b/arch/x86/include/uapi/asm/signal.h
index f777346450ec..1067efabf18b 100644
--- a/arch/x86/include/uapi/asm/signal.h
+++ b/arch/x86/include/uapi/asm/signal.h
@@ -2,7 +2,7 @@
 #ifndef _UAPI_ASM_X86_SIGNAL_H
 #define _UAPI_ASM_X86_SIGNAL_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/types.h>
 #include <linux/compiler.h>
 
@@ -16,7 +16,7 @@ struct siginfo;
 typedef unsigned long sigset_t;
 
 #endif /* __KERNEL__ */
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 
 #define SIGHUP		 1
@@ -68,7 +68,7 @@ typedef unsigned long sigset_t;
 
 #include <asm-generic/signal-defs.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 
 # ifndef __KERNEL__
@@ -106,6 +106,6 @@ typedef struct sigaltstack {
 	__kernel_size_t ss_size;
 } stack_t;
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _UAPI_ASM_X86_SIGNAL_H */
diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h
index 1814b413fd57..650e3256ea7d 100644
--- a/arch/x86/include/uapi/asm/svm.h
+++ b/arch/x86/include/uapi/asm/svm.h
@@ -95,6 +95,8 @@
 #define SVM_EXIT_CR14_WRITE_TRAP		0x09e
 #define SVM_EXIT_CR15_WRITE_TRAP		0x09f
 #define SVM_EXIT_INVPCID       0x0a2
+#define SVM_EXIT_BUS_LOCK			0x0a5
+#define SVM_EXIT_IDLE_HLT      0x0a6
 #define SVM_EXIT_NPF           0x400
 #define SVM_EXIT_AVIC_INCOMPLETE_IPI		0x401
 #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS	0x402
@@ -116,6 +118,10 @@
 #define SVM_VMGEXIT_AP_CREATE			1
 #define SVM_VMGEXIT_AP_DESTROY			2
 #define SVM_VMGEXIT_SNP_RUN_VMPL		0x80000018
+#define SVM_VMGEXIT_SAVIC			0x8000001a
+#define SVM_VMGEXIT_SAVIC_REGISTER_GPA		0
+#define SVM_VMGEXIT_SAVIC_UNREGISTER_GPA	1
+#define SVM_VMGEXIT_SAVIC_SELF_GPA		~0ULL
 #define SVM_VMGEXIT_HV_FEATURES			0x8000fffd
 #define SVM_VMGEXIT_TERM_REQUEST		0x8000fffe
 #define SVM_VMGEXIT_TERM_REASON(reason_set, reason_code)	\
@@ -224,6 +230,8 @@
 	{ SVM_EXIT_CR4_WRITE_TRAP,	"write_cr4_trap" }, \
 	{ SVM_EXIT_CR8_WRITE_TRAP,	"write_cr8_trap" }, \
 	{ SVM_EXIT_INVPCID,     "invpcid" }, \
+	{ SVM_EXIT_BUS_LOCK,     "buslock" }, \
+	{ SVM_EXIT_IDLE_HLT,     "idle-halt" }, \
 	{ SVM_EXIT_NPF,         "npf" }, \
 	{ SVM_EXIT_AVIC_INCOMPLETE_IPI,		"avic_incomplete_ipi" }, \
 	{ SVM_EXIT_AVIC_UNACCELERATED_ACCESS,   "avic_unaccelerated_access" }, \
diff --git a/arch/x86/include/uapi/asm/vmx.h b/arch/x86/include/uapi/asm/vmx.h
index a5faf6d88f1b..1baa86dfe029 100644
--- a/arch/x86/include/uapi/asm/vmx.h
+++ b/arch/x86/include/uapi/asm/vmx.h
@@ -34,6 +34,7 @@
 #define EXIT_REASON_TRIPLE_FAULT        2
 #define EXIT_REASON_INIT_SIGNAL			3
 #define EXIT_REASON_SIPI_SIGNAL         4
+#define EXIT_REASON_OTHER_SMI           6
 
 #define EXIT_REASON_INTERRUPT_WINDOW    7
 #define EXIT_REASON_NMI_WINDOW          8
@@ -92,6 +93,10 @@
 #define EXIT_REASON_TPAUSE              68
 #define EXIT_REASON_BUS_LOCK            74
 #define EXIT_REASON_NOTIFY              75
+#define EXIT_REASON_SEAMCALL            76
+#define EXIT_REASON_TDCALL              77
+#define EXIT_REASON_MSR_READ_IMM        84
+#define EXIT_REASON_MSR_WRITE_IMM       85
 
 #define VMX_EXIT_REASONS \
 	{ EXIT_REASON_EXCEPTION_NMI,         "EXCEPTION_NMI" }, \
@@ -155,7 +160,10 @@
 	{ EXIT_REASON_UMWAIT,                "UMWAIT" }, \
 	{ EXIT_REASON_TPAUSE,                "TPAUSE" }, \
 	{ EXIT_REASON_BUS_LOCK,              "BUS_LOCK" }, \
-	{ EXIT_REASON_NOTIFY,                "NOTIFY" }
+	{ EXIT_REASON_NOTIFY,                "NOTIFY" }, \
+	{ EXIT_REASON_TDCALL,                "TDCALL" }, \
+	{ EXIT_REASON_MSR_READ_IMM,          "MSR_READ_IMM" }, \
+	{ EXIT_REASON_MSR_WRITE_IMM,         "MSR_WRITE_IMM" }
 
 #define VMX_EXIT_REASON_FLAGS \
 	{ VMX_EXIT_REASONS_FAILED_VMENTRY,	"FAILED_VMENTRY" }
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index f7918980667a..bc184dd38d99 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -3,7 +3,7 @@
 # Makefile for the linux kernel.
 #
 
-extra-y	+= vmlinux.lds
+always-$(KBUILD_BUILTIN)	+= vmlinux.lds
 
 CPPFLAGS_vmlinux.lds += -U$(UTS_MACHINE)
 
@@ -44,6 +44,8 @@ KCOV_INSTRUMENT_unwind_orc.o				:= n
 KCOV_INSTRUMENT_unwind_frame.o				:= n
 KCOV_INSTRUMENT_unwind_guess.o				:= n
 
+CFLAGS_head32.o := -fno-stack-protector
+CFLAGS_head64.o := -fno-stack-protector
 CFLAGS_irq.o := -I $(src)/../include/asm/trace
 
 obj-y			+= head_$(BITS).o
@@ -119,6 +121,7 @@ obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
 obj-$(CONFIG_HPET_TIMER) 	+= hpet.o
 
 obj-$(CONFIG_AMD_NB)		+= amd_nb.o
+obj-$(CONFIG_AMD_NODE)		+= amd_node.o
 obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o
 
 obj-$(CONFIG_KVM_GUEST)		+= kvm.o kvmclock.o
@@ -138,7 +141,6 @@ obj-$(CONFIG_OF)			+= devicetree.o
 obj-$(CONFIG_UPROBES)			+= uprobes.o
 
 obj-$(CONFIG_PERF_EVENTS)		+= perf_regs.o
-obj-$(CONFIG_TRACING)			+= tracepoint.o
 obj-$(CONFIG_SCHED_MC_PRIO)		+= itmt.o
 obj-$(CONFIG_X86_UMIP)			+= umip.o
 
@@ -146,7 +148,7 @@ obj-$(CONFIG_UNWINDER_ORC)		+= unwind_orc.o
 obj-$(CONFIG_UNWINDER_FRAME_POINTER)	+= unwind_frame.o
 obj-$(CONFIG_UNWINDER_GUESS)		+= unwind_guess.o
 
-obj-$(CONFIG_CFI_CLANG)			+= cfi.o
+obj-$(CONFIG_CFI)			+= cfi.o
 
 obj-$(CONFIG_CALL_THUNKS)		+= callthunks.o
 
diff --git a/arch/x86/kernel/acpi/apei.c b/arch/x86/kernel/acpi/apei.c
index 0916f00a992e..e21419e686eb 100644
--- a/arch/x86/kernel/acpi/apei.c
+++ b/arch/x86/kernel/acpi/apei.c
@@ -19,6 +19,8 @@ int arch_apei_enable_cmcff(struct acpi_hest_header *hest_hdr, void *data)
 	if (!cmc->enabled)
 		return 0;
 
+	mce_save_apei_thr_limit(cmc->notify.error_threshold_value);
+
 	/*
 	 * We expect HEST to provide a list of MC banks that report errors
 	 * in firmware first mode. Otherwise, return non-zero value to
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index 4efecac49863..9fa321a95eb3 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -23,6 +23,8 @@
 #include <linux/serial_core.h>
 #include <linux/pgtable.h>
 
+#include <xen/xen.h>
+
 #include <asm/e820/api.h>
 #include <asm/irqdomain.h>
 #include <asm/pci_x86.h>
@@ -227,6 +229,28 @@ acpi_parse_x2apic(union acpi_subtable_headers *header, const unsigned long end)
 }
 
 static int __init
+acpi_check_lapic(union acpi_subtable_headers *header, const unsigned long end)
+{
+	struct acpi_madt_local_apic *processor = NULL;
+
+	processor = (struct acpi_madt_local_apic *)header;
+
+	if (BAD_MADT_ENTRY(processor, end))
+		return -EINVAL;
+
+	/* Ignore invalid ID */
+	if (processor->id == 0xff)
+		return 0;
+
+	/* Ignore processors that can not be onlined */
+	if (!acpi_is_processor_usable(processor->lapic_flags))
+		return 0;
+
+	has_lapic_cpus = true;
+	return 0;
+}
+
+static int __init
 acpi_parse_lapic(union acpi_subtable_headers * header, const unsigned long end)
 {
 	struct acpi_madt_local_apic *processor = NULL;
@@ -257,7 +281,6 @@ acpi_parse_lapic(union acpi_subtable_headers * header, const unsigned long end)
 			       processor->processor_id, /* ACPI ID */
 			       processor->lapic_flags & ACPI_MADT_ENABLED);
 
-	has_lapic_cpus = true;
 	return 0;
 }
 
@@ -911,11 +934,8 @@ static int __init acpi_parse_hpet(struct acpi_table_header *table)
 	 * the resource tree during the lateinit timeframe.
 	 */
 #define HPET_RESOURCE_NAME_SIZE 9
-	hpet_res = memblock_alloc(sizeof(*hpet_res) + HPET_RESOURCE_NAME_SIZE,
+	hpet_res = memblock_alloc_or_panic(sizeof(*hpet_res) + HPET_RESOURCE_NAME_SIZE,
 				  SMP_CACHE_BYTES);
-	if (!hpet_res)
-		panic("%s: Failed to allocate %zu bytes\n", __func__,
-		      sizeof(*hpet_res) + HPET_RESOURCE_NAME_SIZE);
 
 	hpet_res->name = (void *)&hpet_res[1];
 	hpet_res->flags = IORESOURCE_MEM;
@@ -1029,6 +1049,8 @@ static int __init early_acpi_parse_madt_lapic_addr_ovr(void)
 static int __init acpi_parse_madt_lapic_entries(void)
 {
 	int count, x2count = 0;
+	struct acpi_subtable_proc madt_proc[2];
+	int ret;
 
 	if (!boot_cpu_has(X86_FEATURE_APIC))
 		return -ENODEV;
@@ -1037,10 +1059,27 @@ static int __init acpi_parse_madt_lapic_entries(void)
 				      acpi_parse_sapic, MAX_LOCAL_APIC);
 
 	if (!count) {
-		count = acpi_table_parse_madt(ACPI_MADT_TYPE_LOCAL_APIC,
-					acpi_parse_lapic, MAX_LOCAL_APIC);
-		x2count = acpi_table_parse_madt(ACPI_MADT_TYPE_LOCAL_X2APIC,
-					acpi_parse_x2apic, MAX_LOCAL_APIC);
+		/* Check if there are valid LAPIC entries */
+		acpi_table_parse_madt(ACPI_MADT_TYPE_LOCAL_APIC, acpi_check_lapic, MAX_LOCAL_APIC);
+
+		/*
+		 * Enumerate the APIC IDs in the order that they appear in the
+		 * MADT, no matter LAPIC entry or x2APIC entry is used.
+		 */
+		memset(madt_proc, 0, sizeof(madt_proc));
+		madt_proc[0].id = ACPI_MADT_TYPE_LOCAL_APIC;
+		madt_proc[0].handler = acpi_parse_lapic;
+		madt_proc[1].id = ACPI_MADT_TYPE_LOCAL_X2APIC;
+		madt_proc[1].handler = acpi_parse_x2apic;
+		ret = acpi_table_parse_entries_array(ACPI_SIG_MADT,
+				sizeof(struct acpi_table_madt),
+				madt_proc, ARRAY_SIZE(madt_proc), MAX_LOCAL_APIC);
+		if (ret < 0) {
+			pr_err("Error parsing LAPIC/X2APIC entries\n");
+			return ret;
+		}
+		count = madt_proc[0].count;
+		x2count = madt_proc[1].count;
 	}
 	if (!count && !x2count) {
 		pr_err("No LAPIC entries present\n");
@@ -1171,7 +1210,8 @@ static int __init acpi_parse_madt_ioapic_entries(void)
 	}
 
 	count = acpi_table_parse_madt(ACPI_MADT_TYPE_INTERRUPT_OVERRIDE,
-				      acpi_parse_int_src_ovr, nr_irqs);
+				      acpi_parse_int_src_ovr,
+				      irq_get_nr_irqs());
 	if (count < 0) {
 		pr_err("Error parsing interrupt source overrides entry\n");
 		/* TBD: Cleanup to allow fallback to MPS */
@@ -1191,7 +1231,8 @@ static int __init acpi_parse_madt_ioapic_entries(void)
 	mp_config_acpi_legacy_irqs();
 
 	count = acpi_table_parse_madt(ACPI_MADT_TYPE_NMI_SOURCE,
-				      acpi_parse_nmi_src, nr_irqs);
+				      acpi_parse_nmi_src,
+				      irq_get_nr_irqs());
 	if (count < 0) {
 		pr_err("Error parsing NMI SRC entry\n");
 		/* TBD: Cleanup to allow fallback to MPS */
@@ -1690,6 +1731,15 @@ int __init acpi_mps_check(void)
 {
 #if defined(CONFIG_X86_LOCAL_APIC) && !defined(CONFIG_X86_MPPARSE)
 /* mptable code is not built-in*/
+
+	/*
+	 * Xen disables ACPI in PV DomU guests but it still emulates APIC and
+	 * supports SMP. Returning early here ensures that APIC is not disabled
+	 * unnecessarily and the guest is not limited to a single vCPU.
+	 */
+	if (xen_pv_domain() && !xen_initial_domain())
+		return 0;
+
 	if (acpi_disabled || acpi_noirq) {
 		pr_warn("MPS support code is not built-in, using acpi=off or acpi=noirq or pci=noacpi may have problem\n");
 		return 1;
diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c
index 956984054bf3..d7c8ef1e354d 100644
--- a/arch/x86/kernel/acpi/cppc.c
+++ b/arch/x86/kernel/acpi/cppc.c
@@ -4,6 +4,8 @@
  * Copyright (c) 2016, Intel Corporation.
  */
 
+#include <linux/bitfield.h>
+
 #include <acpi/cppc_acpi.h>
 #include <asm/msr.h>
 #include <asm/processor.h>
@@ -47,7 +49,7 @@ int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val)
 {
 	int err;
 
-	err = rdmsrl_safe_on_cpu(cpunum, reg->address, val);
+	err = rdmsrq_safe_on_cpu(cpunum, reg->address, val);
 	if (!err) {
 		u64 mask = GENMASK_ULL(reg->bit_offset + reg->bit_width - 1,
 				       reg->bit_offset);
@@ -63,7 +65,7 @@ int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val)
 	u64 rd_val;
 	int err;
 
-	err = rdmsrl_safe_on_cpu(cpunum, reg->address, &rd_val);
+	err = rdmsrq_safe_on_cpu(cpunum, reg->address, &rd_val);
 	if (!err) {
 		u64 mask = GENMASK_ULL(reg->bit_offset + reg->bit_width - 1,
 				       reg->bit_offset);
@@ -72,7 +74,7 @@ int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val)
 		val &= mask;
 		rd_val &= ~mask;
 		rd_val |= val;
-		err = wrmsrl_safe_on_cpu(cpunum, reg->address, rd_val);
+		err = wrmsrq_safe_on_cpu(cpunum, reg->address, rd_val);
 	}
 	return err;
 }
@@ -110,7 +112,7 @@ static void amd_set_max_freq_ratio(void)
 
 static DEFINE_MUTEX(freq_invariance_lock);
 
-void init_freq_invariance_cppc(void)
+static inline void init_freq_invariance_cppc(void)
 {
 	static bool init_done;
 
@@ -127,6 +129,11 @@ void init_freq_invariance_cppc(void)
 	mutex_unlock(&freq_invariance_lock);
 }
 
+void acpi_processor_init_invariance_cppc(void)
+{
+	init_freq_invariance_cppc();
+}
+
 /*
  * Get the highest performance register value.
  * @cpu: CPU from which to get highest performance.
@@ -140,11 +147,11 @@ int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf)
 	int ret;
 
 	if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
-		ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &val);
+		ret = rdmsrq_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &val);
 		if (ret)
 			goto out;
 
-		val = AMD_CPPC_HIGHEST_PERF(val);
+		val = FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, val);
 	} else {
 		ret = cppc_get_highest_perf(cpu, &val);
 		if (ret)
@@ -189,7 +196,7 @@ int amd_detect_prefcore(bool *detected)
 		break;
 	}
 
-	for_each_present_cpu(cpu) {
+	for_each_online_cpu(cpu) {
 		u32 tmp;
 		int ret;
 
@@ -234,8 +241,10 @@ EXPORT_SYMBOL_GPL(amd_detect_prefcore);
  */
 int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator)
 {
+	enum x86_topology_cpu_type core_type = get_topology_cpu_type(&cpu_data(cpu));
 	bool prefcore;
 	int ret;
+	u32 tmp;
 
 	ret = amd_detect_prefcore(&prefcore);
 	if (ret)
@@ -261,6 +270,27 @@ int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator)
 			break;
 		}
 	}
+
+	/* detect if running on heterogeneous design */
+	if (cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) {
+		switch (core_type) {
+		case TOPO_CPU_TYPE_UNKNOWN:
+			pr_warn("Undefined core type found for cpu %d\n", cpu);
+			break;
+		case TOPO_CPU_TYPE_PERFORMANCE:
+			/* use the max scale for performance cores */
+			*numerator = CPPC_HIGHEST_PERF_PERFORMANCE;
+			return 0;
+		case TOPO_CPU_TYPE_EFFICIENCY:
+			/* use the highest perf value for efficiency cores */
+			ret = amd_get_highest_perf(cpu, &tmp);
+			if (ret)
+				return ret;
+			*numerator = tmp;
+			return 0;
+		}
+	}
+
 	*numerator = CPPC_HIGHEST_PERF_PREFCORE;
 
 	return 0;
diff --git a/arch/x86/kernel/acpi/cstate.c b/arch/x86/kernel/acpi/cstate.c
index f3ffd0a3a012..0281703da5e2 100644
--- a/arch/x86/kernel/acpi/cstate.c
+++ b/arch/x86/kernel/acpi/cstate.c
@@ -13,8 +13,11 @@
 #include <linux/sched.h>
 
 #include <acpi/processor.h>
+#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/mwait.h>
 #include <asm/special_insns.h>
+#include <asm/smp.h>
 
 /*
  * Initialize bm_flags based on the CPU cache properties
@@ -46,12 +49,11 @@ void acpi_processor_power_init_bm_check(struct acpi_processor_flags *flags,
 	/*
 	 * On all recent Intel platforms, ARB_DISABLE is a nop.
 	 * So, set bm_control to zero to indicate that ARB_DISABLE
-	 * is not required while entering C3 type state on
-	 * P4, Core and beyond CPUs
+	 * is not required while entering C3 type state.
 	 */
 	if (c->x86_vendor == X86_VENDOR_INTEL &&
-	    (c->x86 > 0xf || (c->x86 == 6 && c->x86_model >= 0x0f)))
-			flags->bm_control = 0;
+	    (c->x86 > 15 || (c->x86_vfm >= INTEL_CORE2_MEROM && c->x86_vfm <= INTEL_FAM6_LAST)))
+		flags->bm_control = 0;
 
 	if (c->x86_vendor == X86_VENDOR_CENTAUR) {
 		if (c->x86 > 6 || (c->x86 == 6 && c->x86_model == 0x0f &&
@@ -87,7 +89,7 @@ void acpi_processor_power_init_bm_check(struct acpi_processor_flags *flags,
 		 */
 		flags->bm_control = 0;
 	}
-	if (c->x86_vendor == X86_VENDOR_AMD && c->x86 >= 0x17) {
+	if (cpu_feature_enabled(X86_FEATURE_ZEN)) {
 		/*
 		 * For all AMD Zen or newer CPUs that support C3, caches
 		 * should not be flushed by software while entering C3
@@ -128,7 +130,7 @@ static long acpi_processor_ffh_cstate_probe_cpu(void *_cx)
 	unsigned int cstate_type; /* C-state type and not ACPI C-state type */
 	unsigned int num_cstate_subtype;
 
-	cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
+	cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &edx);
 
 	/* Check whether this particular cx_type (in CST) is supported or not */
 	cstate_type = (((cx->address >> MWAIT_SUBSTATE_SIZE) &
@@ -172,7 +174,7 @@ int acpi_processor_ffh_cstate_probe(unsigned int cpu,
 	struct cpuinfo_x86 *c = &cpu_data(cpu);
 	long retval;
 
-	if (!cpu_cstate_entry || c->cpuid_level < CPUID_MWAIT_LEAF)
+	if (!cpu_cstate_entry || c->cpuid_level < CPUID_LEAF_MWAIT)
 		return -1;
 
 	if (reg->bit_offset != NATIVE_CSTATE_BEYOND_HALT)
@@ -204,6 +206,16 @@ int acpi_processor_ffh_cstate_probe(unsigned int cpu,
 }
 EXPORT_SYMBOL_GPL(acpi_processor_ffh_cstate_probe);
 
+void __noreturn acpi_processor_ffh_play_dead(struct acpi_processor_cx *cx)
+{
+	unsigned int cpu = smp_processor_id();
+	struct cstate_entry *percpu_entry;
+
+	percpu_entry = per_cpu_ptr(cpu_cstate_entry, cpu);
+	mwait_play_dead(percpu_entry->states[cx->index].eax);
+}
+EXPORT_SYMBOL_GPL(acpi_processor_ffh_play_dead);
+
 void __cpuidle acpi_processor_ffh_cstate_enter(struct acpi_processor_cx *cx)
 {
 	unsigned int cpu = smp_processor_id();
diff --git a/arch/x86/kernel/acpi/madt_playdead.S b/arch/x86/kernel/acpi/madt_playdead.S
index 4e498d28cdc8..aefb9cb583ad 100644
--- a/arch/x86/kernel/acpi/madt_playdead.S
+++ b/arch/x86/kernel/acpi/madt_playdead.S
@@ -14,6 +14,7 @@
  * rsi: PGD of the identity mapping
  */
 SYM_FUNC_START(asm_acpi_mp_play_dead)
+	ANNOTATE_NOENDBR
 	/* Turn off global entries. Following CR3 write will flush them. */
 	movq	%cr4, %rdx
 	andq	$~(X86_CR4_PGE), %rdx
diff --git a/arch/x86/kernel/acpi/madt_wakeup.c b/arch/x86/kernel/acpi/madt_wakeup.c
index d5ef6215583b..6d7603511f52 100644
--- a/arch/x86/kernel/acpi/madt_wakeup.c
+++ b/arch/x86/kernel/acpi/madt_wakeup.c
@@ -70,58 +70,6 @@ static void __init free_pgt_page(void *pgt, void *dummy)
 	return memblock_free(pgt, PAGE_SIZE);
 }
 
-/*
- * Make sure asm_acpi_mp_play_dead() is present in the identity mapping at
- * the same place as in the kernel page tables. asm_acpi_mp_play_dead() switches
- * to the identity mapping and the function has be present at the same spot in
- * the virtual address space before and after switching page tables.
- */
-static int __init init_transition_pgtable(pgd_t *pgd)
-{
-	pgprot_t prot = PAGE_KERNEL_EXEC_NOENC;
-	unsigned long vaddr, paddr;
-	p4d_t *p4d;
-	pud_t *pud;
-	pmd_t *pmd;
-	pte_t *pte;
-
-	vaddr = (unsigned long)asm_acpi_mp_play_dead;
-	pgd += pgd_index(vaddr);
-	if (!pgd_present(*pgd)) {
-		p4d = (p4d_t *)alloc_pgt_page(NULL);
-		if (!p4d)
-			return -ENOMEM;
-		set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE));
-	}
-	p4d = p4d_offset(pgd, vaddr);
-	if (!p4d_present(*p4d)) {
-		pud = (pud_t *)alloc_pgt_page(NULL);
-		if (!pud)
-			return -ENOMEM;
-		set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
-	}
-	pud = pud_offset(p4d, vaddr);
-	if (!pud_present(*pud)) {
-		pmd = (pmd_t *)alloc_pgt_page(NULL);
-		if (!pmd)
-			return -ENOMEM;
-		set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
-	}
-	pmd = pmd_offset(pud, vaddr);
-	if (!pmd_present(*pmd)) {
-		pte = (pte_t *)alloc_pgt_page(NULL);
-		if (!pte)
-			return -ENOMEM;
-		set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
-	}
-	pte = pte_offset_kernel(pmd, vaddr);
-
-	paddr = __pa(vaddr);
-	set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, prot));
-
-	return 0;
-}
-
 static int __init acpi_mp_setup_reset(u64 reset_vector)
 {
 	struct x86_mapping_info info = {
@@ -130,6 +78,7 @@ static int __init acpi_mp_setup_reset(u64 reset_vector)
 		.page_flag      = __PAGE_KERNEL_LARGE_EXEC,
 		.kernpg_flag    = _KERNPG_TABLE_NOENC,
 	};
+	unsigned long mstart, mend;
 	pgd_t *pgd;
 
 	pgd = alloc_pgt_page(NULL);
@@ -137,8 +86,6 @@ static int __init acpi_mp_setup_reset(u64 reset_vector)
 		return -ENOMEM;
 
 	for (int i = 0; i < nr_pfn_mapped; i++) {
-		unsigned long mstart, mend;
-
 		mstart = pfn_mapped[i].start << PAGE_SHIFT;
 		mend   = pfn_mapped[i].end << PAGE_SHIFT;
 		if (kernel_ident_mapping_init(&info, pgd, mstart, mend)) {
@@ -147,14 +94,24 @@ static int __init acpi_mp_setup_reset(u64 reset_vector)
 		}
 	}
 
-	if (kernel_ident_mapping_init(&info, pgd,
-				      PAGE_ALIGN_DOWN(reset_vector),
-				      PAGE_ALIGN(reset_vector + 1))) {
+	mstart = PAGE_ALIGN_DOWN(reset_vector);
+	mend = mstart + PAGE_SIZE;
+	if (kernel_ident_mapping_init(&info, pgd, mstart, mend)) {
 		kernel_ident_mapping_free(&info, pgd);
 		return -ENOMEM;
 	}
 
-	if (init_transition_pgtable(pgd)) {
+	/*
+	 * Make sure asm_acpi_mp_play_dead() is present in the identity mapping
+	 * at the same place as in the kernel page tables.
+	 * asm_acpi_mp_play_dead() switches to the identity mapping and the
+	 * function must be present at the same spot in the virtual address space
+	 * before and after switching page tables.
+	 */
+	info.offset = __START_KERNEL_map - phys_base;
+	mstart = PAGE_ALIGN_DOWN(__pa(asm_acpi_mp_play_dead));
+	mend = mstart + PAGE_SIZE;
+	if (kernel_ident_mapping_init(&info, pgd, mstart, mend)) {
 		kernel_ident_mapping_free(&info, pgd);
 		return -ENOMEM;
 	}
@@ -169,7 +126,7 @@ static int __init acpi_mp_setup_reset(u64 reset_vector)
 	return 0;
 }
 
-static int acpi_wakeup_cpu(u32 apicid, unsigned long start_ip)
+static int acpi_wakeup_cpu(u32 apicid, unsigned long start_ip, unsigned int cpu)
 {
 	if (!acpi_mp_wake_mailbox_paddr) {
 		pr_warn_once("No MADT mailbox: cannot bringup secondary CPUs. Booting with kexec?\n");
diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index 6dfecb27b846..91fa262f0e30 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -16,6 +16,7 @@
 #include <asm/cacheflush.h>
 #include <asm/realmode.h>
 #include <asm/hypervisor.h>
+#include <asm/msr.h>
 #include <asm/smp.h>
 
 #include <linux/ftrace.h>
diff --git a/arch/x86/kernel/acpi/wakeup_64.S b/arch/x86/kernel/acpi/wakeup_64.S
index 94ff83f3d3fe..04f561f75e99 100644
--- a/arch/x86/kernel/acpi/wakeup_64.S
+++ b/arch/x86/kernel/acpi/wakeup_64.S
@@ -17,6 +17,7 @@
 	 * Hooray, we are in Long 64-bit mode (but still running in low memory)
 	 */
 SYM_FUNC_START(wakeup_long64)
+	ANNOTATE_NOENDBR
 	movq	saved_magic(%rip), %rax
 	movq	$0x123456789abcdef0, %rdx
 	cmpq	%rdx, %rax
@@ -87,6 +88,7 @@ SYM_FUNC_START(do_suspend_lowlevel)
 
 	.align 4
 .Lresume_point:
+	ANNOTATE_NOENDBR
 	/* We don't restore %rax, it must be 0 anyway */
 	movq	$saved_context, %rax
 	movq	saved_context_cr4(%rax), %rbx
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index d17518ca19b8..28518371d8bf 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -1,36 +1,18 @@
 // SPDX-License-Identifier: GPL-2.0-only
 #define pr_fmt(fmt) "SMP alternatives: " fmt
 
-#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/mmu_context.h>
 #include <linux/perf_event.h>
-#include <linux/mutex.h>
-#include <linux/list.h>
-#include <linux/stringify.h>
-#include <linux/highmem.h>
-#include <linux/mm.h>
 #include <linux/vmalloc.h>
 #include <linux/memory.h>
-#include <linux/stop_machine.h>
-#include <linux/slab.h>
-#include <linux/kdebug.h>
-#include <linux/kprobes.h>
-#include <linux/mmu_context.h>
-#include <linux/bsearch.h>
-#include <linux/sync_core.h>
+#include <linux/execmem.h>
+
 #include <asm/text-patching.h>
-#include <asm/alternative.h>
-#include <asm/sections.h>
-#include <asm/mce.h>
-#include <asm/nmi.h>
-#include <asm/cacheflush.h>
-#include <asm/tlbflush.h>
 #include <asm/insn.h>
-#include <asm/io.h>
-#include <asm/fixmap.h>
-#include <asm/paravirt.h>
-#include <asm/asm-prototypes.h>
-#include <asm/cfi.h>
+#include <asm/insn-eval.h>
+#include <asm/ibt.h>
+#include <asm/set_memory.h>
+#include <asm/nmi.h>
 
 int __read_mostly alternatives_patched;
 
@@ -124,6 +106,199 @@ const unsigned char * const x86_nops[ASM_NOP_MAX+1] =
 #endif
 };
 
+#ifdef CONFIG_FINEIBT
+static bool cfi_paranoid __ro_after_init;
+#endif
+
+#ifdef CONFIG_MITIGATION_ITS
+
+#ifdef CONFIG_MODULES
+static struct module *its_mod;
+#endif
+static void *its_page;
+static unsigned int its_offset;
+struct its_array its_pages;
+
+static void *__its_alloc(struct its_array *pages)
+{
+	void *page __free(execmem) = execmem_alloc_rw(EXECMEM_MODULE_TEXT, PAGE_SIZE);
+	if (!page)
+		return NULL;
+
+	void *tmp = krealloc(pages->pages, (pages->num+1) * sizeof(void *),
+			     GFP_KERNEL);
+	if (!tmp)
+		return NULL;
+
+	pages->pages = tmp;
+	pages->pages[pages->num++] = page;
+
+	return no_free_ptr(page);
+}
+
+/* Initialize a thunk with the "jmp *reg; int3" instructions. */
+static void *its_init_thunk(void *thunk, int reg)
+{
+	u8 *bytes = thunk;
+	int offset = 0;
+	int i = 0;
+
+#ifdef CONFIG_FINEIBT
+	if (cfi_paranoid) {
+		/*
+		 * When ITS uses indirect branch thunk the fineibt_paranoid
+		 * caller sequence doesn't fit in the caller site. So put the
+		 * remaining part of the sequence (UDB + JNE) into the ITS
+		 * thunk.
+		 */
+		bytes[i++] = 0xd6; /* UDB */
+		bytes[i++] = 0x75; /* JNE */
+		bytes[i++] = 0xfd;
+
+		offset = 1;
+	}
+#endif
+
+	if (reg >= 8) {
+		bytes[i++] = 0x41; /* REX.B prefix */
+		reg -= 8;
+	}
+	bytes[i++] = 0xff;
+	bytes[i++] = 0xe0 + reg; /* JMP *reg */
+	bytes[i++] = 0xcc;
+
+	return thunk + offset;
+}
+
+static void its_pages_protect(struct its_array *pages)
+{
+	for (int i = 0; i < pages->num; i++) {
+		void *page = pages->pages[i];
+		execmem_restore_rox(page, PAGE_SIZE);
+	}
+}
+
+static void its_fini_core(void)
+{
+	if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX))
+		its_pages_protect(&its_pages);
+	kfree(its_pages.pages);
+}
+
+#ifdef CONFIG_MODULES
+void its_init_mod(struct module *mod)
+{
+	if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
+		return;
+
+	mutex_lock(&text_mutex);
+	its_mod = mod;
+	its_page = NULL;
+}
+
+void its_fini_mod(struct module *mod)
+{
+	if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
+		return;
+
+	WARN_ON_ONCE(its_mod != mod);
+
+	its_mod = NULL;
+	its_page = NULL;
+	mutex_unlock(&text_mutex);
+
+	if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
+		its_pages_protect(&mod->arch.its_pages);
+}
+
+void its_free_mod(struct module *mod)
+{
+	if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
+		return;
+
+	for (int i = 0; i < mod->arch.its_pages.num; i++) {
+		void *page = mod->arch.its_pages.pages[i];
+		execmem_free(page);
+	}
+	kfree(mod->arch.its_pages.pages);
+}
+#endif /* CONFIG_MODULES */
+
+static void *its_alloc(void)
+{
+	struct its_array *pages = &its_pages;
+	void *page;
+
+#ifdef CONFIG_MODULES
+	if (its_mod)
+		pages = &its_mod->arch.its_pages;
+#endif
+
+	page = __its_alloc(pages);
+	if (!page)
+		return NULL;
+
+	if (pages == &its_pages)
+		set_memory_x((unsigned long)page, 1);
+
+	return page;
+}
+
+static void *its_allocate_thunk(int reg)
+{
+	int size = 3 + (reg / 8);
+	void *thunk;
+
+#ifdef CONFIG_FINEIBT
+	/*
+	 * The ITS thunk contains an indirect jump and an int3 instruction so
+	 * its size is 3 or 4 bytes depending on the register used. If CFI
+	 * paranoid is used then 3 extra bytes are added in the ITS thunk to
+	 * complete the fineibt_paranoid caller sequence.
+	 */
+	if (cfi_paranoid)
+		size += 3;
+#endif
+
+	if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) {
+		its_page = its_alloc();
+		if (!its_page) {
+			pr_err("ITS page allocation failed\n");
+			return NULL;
+		}
+		memset(its_page, INT3_INSN_OPCODE, PAGE_SIZE);
+		its_offset = 32;
+	}
+
+	/*
+	 * If the indirect branch instruction will be in the lower half
+	 * of a cacheline, then update the offset to reach the upper half.
+	 */
+	if ((its_offset + size - 1) % 64 < 32)
+		its_offset = ((its_offset - 1) | 0x3F) + 33;
+
+	thunk = its_page + its_offset;
+	its_offset += size;
+
+	return its_init_thunk(thunk, reg);
+}
+
+u8 *its_static_thunk(int reg)
+{
+	u8 *thunk = __x86_indirect_its_thunk_array[reg];
+
+#ifdef CONFIG_FINEIBT
+	/* Paranoid thunk starts 2 bytes before */
+	if (cfi_paranoid)
+		return thunk - 2;
+#endif
+	return thunk;
+}
+
+#else
+static inline void its_fini_core(void) {}
+#endif /* CONFIG_MITIGATION_ITS */
+
 /*
  * Nomenclature for variable names to simplify and clarify this code and ease
  * any potential staring at it:
@@ -171,32 +346,6 @@ static void add_nop(u8 *buf, unsigned int len)
 		*buf = INT3_INSN_OPCODE;
 }
 
-extern s32 __retpoline_sites[], __retpoline_sites_end[];
-extern s32 __return_sites[], __return_sites_end[];
-extern s32 __cfi_sites[], __cfi_sites_end[];
-extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[];
-extern s32 __smp_locks[], __smp_locks_end[];
-void text_poke_early(void *addr, const void *opcode, size_t len);
-
-/*
- * Matches NOP and NOPL, not any of the other possible NOPs.
- */
-static bool insn_is_nop(struct insn *insn)
-{
-	/* Anything NOP, but no REP NOP */
-	if (insn->opcode.bytes[0] == 0x90 &&
-	    (!insn->prefixes.nbytes || insn->prefixes.bytes[0] != 0xF3))
-		return true;
-
-	/* NOPL */
-	if (insn->opcode.bytes[0] == 0x0F && insn->opcode.bytes[1] == 0x1F)
-		return true;
-
-	/* TODO: more nops */
-
-	return false;
-}
-
 /*
  * Find the offset of the first non-NOP instruction starting at @offset
  * but no further than @len.
@@ -369,7 +518,7 @@ static void __apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen,
 	}
 }
 
-void apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len)
+void text_poke_apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len)
 {
 	__apply_relocation(buf, instr, instrlen, repl, repl_len);
 	optimize_nops(instr, buf, instrlen);
@@ -392,7 +541,7 @@ EXPORT_SYMBOL(BUG_func);
  * Rewrite the "call BUG_func" replacement to point to the target of the
  * indirect pv_ops call "call *disp(%ip)".
  */
-static int alt_replace_call(u8 *instr, u8 *insn_buff, struct alt_instr *a)
+static unsigned int alt_replace_call(u8 *instr, u8 *insn_buff, struct alt_instr *a)
 {
 	void *target, *bug = &BUG_func;
 	s32 disp;
@@ -457,7 +606,7 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
 	DPRINTK(ALT, "alt table %px, -> %px", start, end);
 
 	/*
-	 * In the case CONFIG_X86_5LEVEL=y, KASAN_SHADOW_START is defined using
+	 * KASAN_SHADOW_START is defined using
 	 * cpu_feature_enabled(X86_FEATURE_LA57) and is therefore patched here.
 	 * During the process, KASAN becomes confused seeing partial LA57
 	 * conversion and triggers a false-positive out-of-bound report.
@@ -476,7 +625,7 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
 	 * order.
 	 */
 	for (a = start; a < end; a++) {
-		int insn_buff_sz = 0;
+		unsigned int insn_buff_sz = 0;
 
 		/*
 		 * In case of nested ALTERNATIVE()s the outer alternative might
@@ -516,16 +665,13 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
 		memcpy(insn_buff, replacement, a->replacementlen);
 		insn_buff_sz = a->replacementlen;
 
-		if (a->flags & ALT_FLAG_DIRECT_CALL) {
+		if (a->flags & ALT_FLAG_DIRECT_CALL)
 			insn_buff_sz = alt_replace_call(instr, insn_buff, a);
-			if (insn_buff_sz < 0)
-				continue;
-		}
 
 		for (; insn_buff_sz < a->instrlen; insn_buff_sz++)
 			insn_buff[insn_buff_sz] = 0x90;
 
-		apply_relocation(insn_buff, instr, a->instrlen, replacement, a->replacementlen);
+		text_poke_apply_relocation(insn_buff, instr, a->instrlen, replacement, a->replacementlen);
 
 		DUMP_BYTES(ALT, instr, a->instrlen, "%px:   old_insn: ", instr);
 		DUMP_BYTES(ALT, replacement, a->replacementlen, "%px:   rpl_insn: ", replacement);
@@ -546,20 +692,33 @@ static inline bool is_jcc32(struct insn *insn)
 #if defined(CONFIG_MITIGATION_RETPOLINE) && defined(CONFIG_OBJTOOL)
 
 /*
- * CALL/JMP *%\reg
+ * [CS]{,3} CALL/JMP *%\reg [INT3]*
  */
-static int emit_indirect(int op, int reg, u8 *bytes)
+static int emit_indirect(int op, int reg, u8 *bytes, int len)
 {
+	int cs = 0, bp = 0;
 	int i = 0;
 	u8 modrm;
 
+	/*
+	 * Set @len to the excess bytes after writing the instruction.
+	 */
+	len -= 2 + (reg >= 8);
+	WARN_ON_ONCE(len < 0);
+
 	switch (op) {
 	case CALL_INSN_OPCODE:
 		modrm = 0x10; /* Reg = 2; CALL r/m */
+		/*
+		 * Additional NOP is better than prefix decode penalty.
+		 */
+		if (len <= 3)
+			cs = len;
 		break;
 
 	case JMP32_INSN_OPCODE:
 		modrm = 0x20; /* Reg = 4; JMP r/m */
+		bp = len;
 		break;
 
 	default:
@@ -567,6 +726,9 @@ static int emit_indirect(int op, int reg, u8 *bytes)
 		return -1;
 	}
 
+	while (cs--)
+		bytes[i++] = 0x2e; /* CS-prefix */
+
 	if (reg >= 8) {
 		bytes[i++] = 0x41; /* REX.B prefix */
 		reg -= 8;
@@ -578,10 +740,14 @@ static int emit_indirect(int op, int reg, u8 *bytes)
 	bytes[i++] = 0xff; /* opcode */
 	bytes[i++] = modrm;
 
+	while (bp--)
+		bytes[i++] = 0xcc; /* INT3 */
+
 	return i;
 }
 
-static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)
+static int __emit_trampoline(void *addr, struct insn *insn, u8 *bytes,
+			     void *call_dest, void *jmp_dest)
 {
 	u8 op = insn->opcode.bytes[0];
 	int i = 0;
@@ -602,7 +768,7 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8
 	switch (op) {
 	case CALL_INSN_OPCODE:
 		__text_gen_insn(bytes+i, op, addr+i,
-				__x86_indirect_call_thunk_array[reg],
+				call_dest,
 				CALL_INSN_SIZE);
 		i += CALL_INSN_SIZE;
 		break;
@@ -610,7 +776,7 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8
 	case JMP32_INSN_OPCODE:
 clang_jcc:
 		__text_gen_insn(bytes+i, op, addr+i,
-				__x86_indirect_jump_thunk_array[reg],
+				jmp_dest,
 				JMP32_INSN_SIZE);
 		i += JMP32_INSN_SIZE;
 		break;
@@ -625,6 +791,48 @@ clang_jcc:
 	return i;
 }
 
+static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)
+{
+	return __emit_trampoline(addr, insn, bytes,
+				 __x86_indirect_call_thunk_array[reg],
+				 __x86_indirect_jump_thunk_array[reg]);
+}
+
+#ifdef CONFIG_MITIGATION_ITS
+static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes)
+{
+	u8 *thunk = __x86_indirect_its_thunk_array[reg];
+	u8 *tmp = its_allocate_thunk(reg);
+
+	if (tmp)
+		thunk = tmp;
+
+	return __emit_trampoline(addr, insn, bytes, thunk, thunk);
+}
+
+/* Check if an indirect branch is at ITS-unsafe address */
+static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg)
+{
+	if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
+		return false;
+
+	/* Indirect branch opcode is 2 or 3 bytes depending on reg */
+	addr += 1 + reg / 8;
+
+	/* Lower-half of the cacheline? */
+	return !(addr & 0x20);
+}
+#else /* CONFIG_MITIGATION_ITS */
+
+#ifdef CONFIG_FINEIBT
+static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg)
+{
+	return false;
+}
+#endif
+
+#endif /* CONFIG_MITIGATION_ITS */
+
 /*
  * Rewrite the compiler generated retpoline thunk calls.
  *
@@ -699,20 +907,20 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes)
 		bytes[i++] = 0xe8; /* LFENCE */
 	}
 
-	ret = emit_indirect(op, reg, bytes + i);
+#ifdef CONFIG_MITIGATION_ITS
+	/*
+	 * Check if the address of last byte of emitted-indirect is in
+	 * lower-half of the cacheline. Such branches need ITS mitigation.
+	 */
+	if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + i, reg))
+		return emit_its_trampoline(addr, insn, reg, bytes);
+#endif
+
+	ret = emit_indirect(op, reg, bytes + i, insn->length - i);
 	if (ret < 0)
 		return ret;
 	i += ret;
 
-	/*
-	 * The compiler is supposed to EMIT an INT3 after every unconditional
-	 * JMP instruction due to AMD BTC. However, if the compiler is too old
-	 * or MITIGATION_SLS isn't enabled, we still need an INT3 after
-	 * indirect JMPs even on Intel.
-	 */
-	if (op == JMP32_INSN_OPCODE && i < insn->length)
-		bytes[i++] = INT3_INSN_OPCODE;
-
 	for (; i < insn->length;)
 		bytes[i++] = BYTES_NOP1;
 
@@ -732,6 +940,7 @@ void __init_or_module noinline apply_retpolines(s32 *start, s32 *end)
 		int len, ret;
 		u8 bytes[16];
 		u8 op1, op2;
+		u8 *dest;
 
 		ret = insn_decode_kernel(&insn, addr);
 		if (WARN_ON_ONCE(ret < 0))
@@ -741,8 +950,19 @@ void __init_or_module noinline apply_retpolines(s32 *start, s32 *end)
 		op2 = insn.opcode.bytes[1];
 
 		switch (op1) {
+		case 0x70 ... 0x7f:	/* Jcc.d8 */
+			/* See cfi_paranoid. */
+			WARN_ON_ONCE(cfi_mode != CFI_FINEIBT);
+			continue;
+
 		case CALL_INSN_OPCODE:
 		case JMP32_INSN_OPCODE:
+			/* Check for cfi_paranoid + ITS */
+			dest = addr + insn.length + insn.immediate.value;
+			if (dest[-1] == 0xd6 && (dest[0] & 0xf0) == 0x70) {
+				WARN_ON_ONCE(cfi_mode != CFI_FINEIBT);
+				continue;
+			}
 			break;
 
 		case 0x0f: /* escape */
@@ -770,6 +990,21 @@ void __init_or_module noinline apply_retpolines(s32 *start, s32 *end)
 
 #ifdef CONFIG_MITIGATION_RETHUNK
 
+bool cpu_wants_rethunk(void)
+{
+	return cpu_feature_enabled(X86_FEATURE_RETHUNK);
+}
+
+bool cpu_wants_rethunk_at(void *addr)
+{
+	if (!cpu_feature_enabled(X86_FEATURE_RETHUNK))
+		return false;
+	if (x86_return_thunk != its_return_thunk)
+		return true;
+
+	return !((unsigned long)addr & 0x20);
+}
+
 /*
  * Rewrite the compiler generated return thunk tail-calls.
  *
@@ -786,7 +1021,7 @@ static int patch_return(void *addr, struct insn *insn, u8 *bytes)
 	int i = 0;
 
 	/* Patch the custom return thunks... */
-	if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) {
+	if (cpu_wants_rethunk_at(addr)) {
 		i = JMP32_INSN_SIZE;
 		__text_gen_insn(bytes, JMP32_INSN_OPCODE, addr, x86_return_thunk, i);
 	} else {
@@ -803,7 +1038,7 @@ void __init_or_module noinline apply_returns(s32 *start, s32 *end)
 {
 	s32 *s;
 
-	if (cpu_feature_enabled(X86_FEATURE_RETHUNK))
+	if (cpu_wants_rethunk())
 		static_call_force_reinit();
 
 	for (s = start; s < end; s++) {
@@ -839,32 +1074,53 @@ void __init_or_module noinline apply_returns(s32 *start, s32 *end)
 		}
 	}
 }
-#else
+#else /* !CONFIG_MITIGATION_RETHUNK: */
 void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }
-#endif /* CONFIG_MITIGATION_RETHUNK */
+#endif /* !CONFIG_MITIGATION_RETHUNK */
 
 #else /* !CONFIG_MITIGATION_RETPOLINE || !CONFIG_OBJTOOL */
 
 void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) { }
 void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }
 
-#endif /* CONFIG_MITIGATION_RETPOLINE && CONFIG_OBJTOOL */
+#endif /* !CONFIG_MITIGATION_RETPOLINE || !CONFIG_OBJTOOL */
 
 #ifdef CONFIG_X86_KERNEL_IBT
 
-static void poison_cfi(void *addr);
+__noendbr bool is_endbr(u32 *val)
+{
+	u32 endbr;
+
+	__get_kernel_nofault(&endbr, val, u32, Efault);
+	return __is_endbr(endbr);
+
+Efault:
+	return false;
+}
+
+#ifdef CONFIG_FINEIBT
 
-static void __init_or_module poison_endbr(void *addr, bool warn)
+static __noendbr bool exact_endbr(u32 *val)
 {
-	u32 endbr, poison = gen_endbr_poison();
+	u32 endbr;
 
-	if (WARN_ON_ONCE(get_kernel_nofault(endbr, addr)))
-		return;
+	__get_kernel_nofault(&endbr, val, u32, Efault);
+	return endbr == gen_endbr();
 
-	if (!is_endbr(endbr)) {
-		WARN_ON_ONCE(warn);
+Efault:
+	return false;
+}
+
+#endif
+
+static void poison_cfi(void *addr);
+
+static void __init_or_module poison_endbr(void *addr)
+{
+	u32 poison = gen_endbr_poison();
+
+	if (WARN_ON_ONCE(!is_endbr(addr)))
 		return;
-	}
 
 	DPRINTK(ENDBR, "ENDBR at: %pS (%px)", addr, addr);
 
@@ -889,70 +1145,34 @@ void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end)
 	for (s = start; s < end; s++) {
 		void *addr = (void *)s + *s;
 
-		poison_endbr(addr, true);
+		poison_endbr(addr);
 		if (IS_ENABLED(CONFIG_FINEIBT))
 			poison_cfi(addr - 16);
 	}
 }
 
-#else
+#else /* !CONFIG_X86_KERNEL_IBT: */
 
 void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { }
 
-#endif /* CONFIG_X86_KERNEL_IBT */
+#endif /* !CONFIG_X86_KERNEL_IBT */
 
 #ifdef CONFIG_CFI_AUTO_DEFAULT
-#define __CFI_DEFAULT	CFI_AUTO
-#elif defined(CONFIG_CFI_CLANG)
-#define __CFI_DEFAULT	CFI_KCFI
+# define __CFI_DEFAULT CFI_AUTO
+#elif defined(CONFIG_CFI)
+# define __CFI_DEFAULT CFI_KCFI
 #else
-#define __CFI_DEFAULT	CFI_OFF
+# define __CFI_DEFAULT CFI_OFF
 #endif
 
 enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT;
+static bool cfi_debug __ro_after_init;
 
-#ifdef CONFIG_CFI_CLANG
-struct bpf_insn;
-
-/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */
-extern unsigned int __bpf_prog_runX(const void *ctx,
-				    const struct bpf_insn *insn);
-
-/*
- * Force a reference to the external symbol so the compiler generates
- * __kcfi_typid.
- */
-__ADDRESSABLE(__bpf_prog_runX);
-
-/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */
-asm (
-"	.pushsection	.data..ro_after_init,\"aw\",@progbits	\n"
-"	.type	cfi_bpf_hash,@object				\n"
-"	.globl	cfi_bpf_hash					\n"
-"	.p2align	2, 0x0					\n"
-"cfi_bpf_hash:							\n"
-"	.long	__kcfi_typeid___bpf_prog_runX			\n"
-"	.size	cfi_bpf_hash, 4					\n"
-"	.popsection						\n"
-);
-
-/* Must match bpf_callback_t */
-extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64);
-
-__ADDRESSABLE(__bpf_callback_fn);
-
-/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */
-asm (
-"	.pushsection	.data..ro_after_init,\"aw\",@progbits	\n"
-"	.type	cfi_bpf_subprog_hash,@object			\n"
-"	.globl	cfi_bpf_subprog_hash				\n"
-"	.p2align	2, 0x0					\n"
-"cfi_bpf_subprog_hash:						\n"
-"	.long	__kcfi_typeid___bpf_callback_fn			\n"
-"	.size	cfi_bpf_subprog_hash, 4				\n"
-"	.popsection						\n"
-);
+#ifdef CONFIG_FINEIBT_BHI
+bool cfi_bhi __ro_after_init = false;
+#endif
 
+#ifdef CONFIG_CFI
 u32 cfi_get_func_hash(void *func)
 {
 	u32 hash;
@@ -974,6 +1194,21 @@ u32 cfi_get_func_hash(void *func)
 
 	return hash;
 }
+
+int cfi_get_func_arity(void *func)
+{
+	bhi_thunk *target;
+	s32 disp;
+
+	if (cfi_mode != CFI_FINEIBT && !cfi_bhi)
+		return 0;
+
+	if (get_kernel_nofault(disp, func - 4))
+		return 0;
+
+	target = func + disp;
+	return target - __bhi_args;
+}
 #endif
 
 #ifdef CONFIG_FINEIBT
@@ -988,7 +1223,7 @@ static u32  cfi_seed __ro_after_init;
 static u32 cfi_rehash(u32 hash)
 {
 	hash ^= cfi_seed;
-	while (unlikely(is_endbr(hash) || is_endbr(-hash))) {
+	while (unlikely(__is_endbr(hash) || __is_endbr(-hash))) {
 		bool lsb = hash & 1;
 		hash >>= 1;
 		if (lsb)
@@ -1014,14 +1249,35 @@ static __init int cfi_parse_cmdline(char *str)
 		} else if (!strcmp(str, "off")) {
 			cfi_mode = CFI_OFF;
 			cfi_rand = false;
+		} else if (!strcmp(str, "debug")) {
+			cfi_debug = true;
 		} else if (!strcmp(str, "kcfi")) {
 			cfi_mode = CFI_KCFI;
 		} else if (!strcmp(str, "fineibt")) {
 			cfi_mode = CFI_FINEIBT;
 		} else if (!strcmp(str, "norand")) {
 			cfi_rand = false;
+		} else if (!strcmp(str, "warn")) {
+			pr_alert("CFI: mismatch non-fatal!\n");
+			cfi_warn = true;
+		} else if (!strcmp(str, "paranoid")) {
+			if (cfi_mode == CFI_FINEIBT) {
+				cfi_paranoid = true;
+			} else {
+				pr_err("CFI: ignoring paranoid; depends on fineibt.\n");
+			}
+		} else if (!strcmp(str, "bhi")) {
+#ifdef CONFIG_FINEIBT_BHI
+			if (cfi_mode == CFI_FINEIBT) {
+				cfi_bhi = true;
+			} else {
+				pr_err("CFI: ignoring bhi; depends on fineibt.\n");
+			}
+#else
+			pr_err("CFI: ignoring bhi; depends on FINEIBT_BHI=y.\n");
+#endif
 		} else {
-			pr_err("Ignoring unknown cfi option (%s).", str);
+			pr_err("CFI: Ignoring unknown option (%s).", str);
 		}
 
 		str = next;
@@ -1036,10 +1292,10 @@ early_param("cfi", cfi_parse_cmdline);
  *
  * __cfi_\func:					__cfi_\func:
  *	movl   $0x12345678,%eax		// 5	     endbr64			// 4
- *	nop					     subl   $0x12345678,%r10d   // 7
- *	nop					     jz     1f			// 2
- *	nop					     ud2			// 2
- *	nop					1:   nop			// 1
+ *	nop					     subl   $0x12345678,%eax    // 5
+ *	nop					     jne.d32,pn \func+3		// 7
+ *	nop
+ *	nop
  *	nop
  *	nop
  *	nop
@@ -1047,39 +1303,68 @@ early_param("cfi", cfi_parse_cmdline);
  *	nop
  *	nop
  *	nop
+ * \func:					\func:
+ *	endbr64					     nopl -42(%rax)
  *
  *
  * caller:					caller:
- *	movl	$(-0x12345678),%r10d	 // 6	     movl   $0x12345678,%r10d	// 6
- *	addl	$-15(%r11),%r10d	 // 4	     sub    $16,%r11		// 4
- *	je	1f			 // 2	     nop4			// 4
+ *	movl	$(-0x12345678),%r10d	 // 6	     movl   $0x12345678,%eax	// 5
+ *	addl	$-15(%r11),%r10d	 // 4	     lea    -0x10(%r11),%r11	// 4
+ *	je	1f			 // 2	     nop5			// 5
  *	ud2				 // 2
- * 1:	call	__x86_indirect_thunk_r11 // 5	     call   *%r11; nop2;	// 5
+ * 1:	cs call	__x86_indirect_thunk_r11 // 6	     call   *%r11; nop3;	// 6
  *
+ *
+ * Notably, the FineIBT sequences are crafted such that branches are presumed
+ * non-taken. This is based on Agner Fog's optimization manual, which states:
+ *
+ *  "Make conditional jumps most often not taken: The efficiency and throughput
+ *   for not-taken branches is better than for taken branches on most
+ *   processors. Therefore, it is good to place the most frequent branch first"
  */
 
-asm(	".pushsection .rodata			\n"
-	"fineibt_preamble_start:		\n"
-	"	endbr64				\n"
-	"	subl	$0x12345678, %r10d	\n"
-	"	je	fineibt_preamble_end	\n"
-	"	ud2				\n"
-	"	nop				\n"
-	"fineibt_preamble_end:			\n"
+/*
+ * <fineibt_preamble_start>:
+ *  0:   f3 0f 1e fa             endbr64
+ *  4:   2d 78 56 34 12          sub    $0x12345678, %eax
+ *  9:   2e 0f 85 03 00 00 00    jne,pn 13 <fineibt_preamble_start+0x13>
+ * 10:   0f 1f 40 d6             nopl   -0x2a(%rax)
+ *
+ * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
+ * UDB on x86_64 and raises #UD.
+ */
+asm(	".pushsection .rodata				\n"
+	"fineibt_preamble_start:			\n"
+	"	endbr64					\n"
+	"	subl	$0x12345678, %eax		\n"
+	"fineibt_preamble_bhi:				\n"
+	"	cs jne.d32 fineibt_preamble_start+0x13	\n"
+	"#fineibt_func:					\n"
+	"	nopl	-42(%rax)			\n"
+	"fineibt_preamble_end:				\n"
 	".popsection\n"
 );
 
 extern u8 fineibt_preamble_start[];
+extern u8 fineibt_preamble_bhi[];
 extern u8 fineibt_preamble_end[];
 
 #define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
-#define fineibt_preamble_hash 7
+#define fineibt_preamble_bhi  (fineibt_preamble_bhi - fineibt_preamble_start)
+#define fineibt_preamble_ud   0x13
+#define fineibt_preamble_hash 5
 
+/*
+ * <fineibt_caller_start>:
+ *  0:   b8 78 56 34 12          mov    $0x12345678, %eax
+ *  5:   4d 8d 5b f0             lea    -0x10(%r11), %r11
+ *  9:   0f 1f 44 00 00          nopl   0x0(%rax,%rax,1)
+ */
 asm(	".pushsection .rodata			\n"
 	"fineibt_caller_start:			\n"
-	"	movl	$0x12345678, %r10d	\n"
-	"	sub	$16, %r11		\n"
-	ASM_NOP4
+	"	movl	$0x12345678, %eax	\n"
+	"	lea	-0x10(%r11), %r11	\n"
+	ASM_NOP5
 	"fineibt_caller_end:			\n"
 	".popsection				\n"
 );
@@ -1088,17 +1373,66 @@ extern u8 fineibt_caller_start[];
 extern u8 fineibt_caller_end[];
 
 #define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start)
-#define fineibt_caller_hash 2
+#define fineibt_caller_hash 1
 
 #define fineibt_caller_jmp (fineibt_caller_size - 2)
 
-static u32 decode_preamble_hash(void *addr)
+/*
+ * Since FineIBT does hash validation on the callee side it is prone to
+ * circumvention attacks where a 'naked' ENDBR instruction exists that
+ * is not part of the fineibt_preamble sequence.
+ *
+ * Notably the x86 entry points must be ENDBR and equally cannot be
+ * fineibt_preamble.
+ *
+ * The fineibt_paranoid caller sequence adds additional caller side
+ * hash validation. This stops such circumvention attacks dead, but at the cost
+ * of adding a load.
+ *
+ * <fineibt_paranoid_start>:
+ *  0:   b8 78 56 34 12          mov    $0x12345678, %eax
+ *  5:   41 3b 43 f5             cmp    -0x11(%r11), %eax
+ *  9:   2e 4d 8d 5b <f0>        cs lea -0x10(%r11), %r11
+ *  e:   75 fd                   jne    d <fineibt_paranoid_start+0xd>
+ * 10:   41 ff d3                call   *%r11
+ * 13:   90                      nop
+ *
+ * Notably LEA does not modify flags and can be reordered with the CMP,
+ * avoiding a dependency. Again, using a non-taken (backwards) branch
+ * for the failure case, abusing LEA's immediate 0xf0 as LOCK prefix for the
+ * Jcc.d8, causing #UD.
+ */
+asm(	".pushsection .rodata				\n"
+	"fineibt_paranoid_start:			\n"
+	"	mov	$0x12345678, %eax		\n"
+	"	cmpl	-11(%r11), %eax			\n"
+	"	cs lea	-0x10(%r11), %r11		\n"
+	"#fineibt_caller_size:                          \n"
+	"	jne	fineibt_paranoid_start+0xd	\n"
+	"fineibt_paranoid_ind:				\n"
+	"	cs call	*%r11				\n"
+	"fineibt_paranoid_end:				\n"
+	".popsection					\n"
+);
+
+extern u8 fineibt_paranoid_start[];
+extern u8 fineibt_paranoid_ind[];
+extern u8 fineibt_paranoid_end[];
+
+#define fineibt_paranoid_size (fineibt_paranoid_end - fineibt_paranoid_start)
+#define fineibt_paranoid_ind  (fineibt_paranoid_ind - fineibt_paranoid_start)
+#define fineibt_paranoid_ud   0xd
+
+static u32 decode_preamble_hash(void *addr, int *reg)
 {
 	u8 *p = addr;
 
-	/* b8 78 56 34 12          mov    $0x12345678,%eax */
-	if (p[0] == 0xb8)
+	/* b8+reg 78 56 34 12          movl    $0x12345678,\reg */
+	if (p[0] >= 0xb8 && p[0] < 0xc0) {
+		if (reg)
+			*reg = p[0] - 0xb8;
 		return *(u32 *)(addr + 1);
+	}
 
 	return 0; /* invalid hash value */
 }
@@ -1107,11 +1441,11 @@ static u32 decode_caller_hash(void *addr)
 {
 	u8 *p = addr;
 
-	/* 41 ba 78 56 34 12       mov    $0x12345678,%r10d */
+	/* 41 ba 88 a9 cb ed       mov    $(-0x12345678),%r10d */
 	if (p[0] == 0x41 && p[1] == 0xba)
 		return -*(u32 *)(addr + 2);
 
-	/* e8 0c 78 56 34 12	   jmp.d8  +12 */
+	/* e8 0c 88 a9 cb ed	   jmp.d8  +12 */
 	if (p[0] == JMP8_INSN_OPCODE && p[1] == fineibt_caller_jmp)
 		return -*(u32 *)(addr + 2);
 
@@ -1176,7 +1510,7 @@ static int cfi_rand_preamble(s32 *start, s32 *end)
 		void *addr = (void *)s + *s;
 		u32 hash;
 
-		hash = decode_preamble_hash(addr);
+		hash = decode_preamble_hash(addr, NULL);
 		if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",
 			 addr, addr, 5, addr))
 			return -EINVAL;
@@ -1188,15 +1522,87 @@ static int cfi_rand_preamble(s32 *start, s32 *end)
 	return 0;
 }
 
+/*
+ * Inline the bhi-arity 1 case:
+ *
+ * __cfi_foo:
+ *  0: f3 0f 1e fa             endbr64
+ *  4: 2d 78 56 34 12          sub    $0x12345678, %eax
+ *  9: 49 0f 45 fa             cmovne %rax, %rdi
+ *  d: 2e 75 03                jne,pn    foo+0x3
+ *
+ * foo:
+ * 10: 0f 1f 40 <d6>           nopl -42(%rax)
+ *
+ * Notably, this scheme is incompatible with permissive CFI
+ * because the CMOVcc is unconditional and RDI will have been
+ * clobbered.
+ */
+asm(	".pushsection .rodata				\n"
+	"fineibt_bhi1_start:				\n"
+	"	cmovne %rax, %rdi			\n"
+	"	cs jne fineibt_bhi1_func + 0x3		\n"
+	"fineibt_bhi1_func:				\n"
+	"	nopl -42(%rax)				\n"
+	"fineibt_bhi1_end:				\n"
+	".popsection					\n"
+);
+
+extern u8 fineibt_bhi1_start[];
+extern u8 fineibt_bhi1_end[];
+
+#define fineibt_bhi1_size (fineibt_bhi1_end - fineibt_bhi1_start)
+
+static void cfi_fineibt_bhi_preamble(void *addr, int arity)
+{
+	u8 bytes[MAX_INSN_SIZE];
+
+	if (!arity)
+		return;
+
+	if (!cfi_warn && arity == 1) {
+		text_poke_early(addr + fineibt_preamble_bhi,
+				fineibt_bhi1_start, fineibt_bhi1_size);
+		return;
+	}
+
+	/*
+	 * Replace the bytes at fineibt_preamble_bhi with a CALL instruction
+	 * that lines up exactly with the end of the preamble, such that the
+	 * return address will be foo+0.
+	 *
+	 * __cfi_foo:
+	 *  0: f3 0f 1e fa             endbr64
+	 *  4: 2d 78 56 34 12          sub    $0x12345678, %eax
+	 *  9: 2e 2e e8 DD DD DD DD    cs cs call __bhi_args[arity]
+	 */
+	bytes[0] = 0x2e;
+	bytes[1] = 0x2e;
+	__text_gen_insn(bytes + 2, CALL_INSN_OPCODE,
+			addr + fineibt_preamble_bhi + 2,
+			__bhi_args[arity], CALL_INSN_SIZE);
+
+	text_poke_early(addr + fineibt_preamble_bhi, bytes, 7);
+}
+
 static int cfi_rewrite_preamble(s32 *start, s32 *end)
 {
 	s32 *s;
 
 	for (s = start; s < end; s++) {
 		void *addr = (void *)s + *s;
+		int arity;
 		u32 hash;
 
-		hash = decode_preamble_hash(addr);
+		/*
+		 * When the function doesn't start with ENDBR the compiler will
+		 * have determined there are no indirect calls to it and we
+		 * don't need no CFI either.
+		 */
+		if (!is_endbr(addr + 16))
+			continue;
+
+		hash = decode_preamble_hash(addr, &arity);
 		if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",
 			 addr, addr, 5, addr))
 			return -EINVAL;
@@ -1204,6 +1610,13 @@ static int cfi_rewrite_preamble(s32 *start, s32 *end)
 		text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size);
 		WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) != 0x12345678);
 		text_poke_early(addr + fineibt_preamble_hash, &hash, 4);
+
+		WARN_ONCE(!IS_ENABLED(CONFIG_FINEIBT_BHI) && arity,
+			  "kCFI preamble has wrong register at: %pS %*ph\n",
+			  addr, 5, addr);
+
+		if (cfi_bhi)
+			cfi_fineibt_bhi_preamble(addr, arity);
 	}
 
 	return 0;
@@ -1216,7 +1629,10 @@ static void cfi_rewrite_endbr(s32 *start, s32 *end)
 	for (s = start; s < end; s++) {
 		void *addr = (void *)s + *s;
 
-		poison_endbr(addr+16, false);
+		if (!exact_endbr(addr + 16))
+			continue;
+
+		poison_endbr(addr + 16);
 	}
 }
 
@@ -1240,40 +1656,101 @@ static int cfi_rand_callers(s32 *start, s32 *end)
 	return 0;
 }
 
+static int emit_paranoid_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes)
+{
+	u8 *thunk = (void *)__x86_indirect_its_thunk_array[reg] - 2;
+
+#ifdef CONFIG_MITIGATION_ITS
+	u8 *tmp = its_allocate_thunk(reg);
+	if (tmp)
+		thunk = tmp;
+#endif
+
+	return __emit_trampoline(addr, insn, bytes, thunk, thunk);
+}
+
 static int cfi_rewrite_callers(s32 *start, s32 *end)
 {
 	s32 *s;
 
 	for (s = start; s < end; s++) {
 		void *addr = (void *)s + *s;
+		struct insn insn;
+		u8 bytes[20];
 		u32 hash;
+		int ret;
+		u8 op;
 
 		addr -= fineibt_caller_size;
 		hash = decode_caller_hash(addr);
-		if (hash) {
+		if (!hash)
+			continue;
+
+		if (!cfi_paranoid) {
 			text_poke_early(addr, fineibt_caller_start, fineibt_caller_size);
 			WARN_ON(*(u32 *)(addr + fineibt_caller_hash) != 0x12345678);
 			text_poke_early(addr + fineibt_caller_hash, &hash, 4);
+			/* rely on apply_retpolines() */
+			continue;
 		}
-		/* rely on apply_retpolines() */
+
+		/* cfi_paranoid */
+		ret = insn_decode_kernel(&insn, addr + fineibt_caller_size);
+		if (WARN_ON_ONCE(ret < 0))
+			continue;
+
+		op = insn.opcode.bytes[0];
+		if (op != CALL_INSN_OPCODE && op != JMP32_INSN_OPCODE) {
+			WARN_ON_ONCE(1);
+			continue;
+		}
+
+		memcpy(bytes, fineibt_paranoid_start, fineibt_paranoid_size);
+		memcpy(bytes + fineibt_caller_hash, &hash, 4);
+
+		if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + fineibt_paranoid_ind, 11)) {
+			emit_paranoid_trampoline(addr + fineibt_caller_size,
+						 &insn, 11, bytes + fineibt_caller_size);
+		} else {
+			int len = fineibt_paranoid_size - fineibt_paranoid_ind;
+			ret = emit_indirect(op, 11, bytes + fineibt_paranoid_ind, len);
+			if (WARN_ON_ONCE(ret != len))
+				continue;
+		}
+
+		text_poke_early(addr, bytes, fineibt_paranoid_size);
 	}
 
 	return 0;
 }
 
+#define pr_cfi_debug(X...) if (cfi_debug) pr_info(X)
+
+#define FINEIBT_WARN(_f, _v) \
+	WARN_ONCE((_f) != (_v), "FineIBT: " #_f " %ld != %d\n", _f, _v)
+
 static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
 			    s32 *start_cfi, s32 *end_cfi, bool builtin)
 {
 	int ret;
 
-	if (WARN_ONCE(fineibt_preamble_size != 16,
-		      "FineIBT preamble wrong size: %ld", fineibt_preamble_size))
+	if (FINEIBT_WARN(fineibt_preamble_size, 20)			||
+	    FINEIBT_WARN(fineibt_preamble_bhi + fineibt_bhi1_size, 20)	||
+	    FINEIBT_WARN(fineibt_caller_size, 14)			||
+	    FINEIBT_WARN(fineibt_paranoid_size, 20))
 		return;
 
 	if (cfi_mode == CFI_AUTO) {
 		cfi_mode = CFI_KCFI;
-		if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
+		if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT)) {
+			/*
+			 * FRED has much saner context on exception entry and
+			 * is less easy to take advantage of.
+			 */
+			if (!cpu_feature_enabled(X86_FEATURE_FRED))
+				cfi_paranoid = true;
 			cfi_mode = CFI_FINEIBT;
+		}
 	}
 
 	/*
@@ -1281,6 +1758,7 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
 	 * rewrite them. This disables all CFI. If this succeeds but any of the
 	 * later stages fails, we're without CFI.
 	 */
+	pr_cfi_debug("CFI: disabling all indirect call checking\n");
 	ret = cfi_disable_callers(start_retpoline, end_retpoline);
 	if (ret)
 		goto err;
@@ -1291,47 +1769,60 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
 			cfi_bpf_hash = cfi_rehash(cfi_bpf_hash);
 			cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash);
 		}
+		pr_cfi_debug("CFI: cfi_seed: 0x%08x\n", cfi_seed);
 
+		pr_cfi_debug("CFI: rehashing all preambles\n");
 		ret = cfi_rand_preamble(start_cfi, end_cfi);
 		if (ret)
 			goto err;
 
+		pr_cfi_debug("CFI: rehashing all indirect calls\n");
 		ret = cfi_rand_callers(start_retpoline, end_retpoline);
 		if (ret)
 			goto err;
+	} else {
+		pr_cfi_debug("CFI: rehashing disabled\n");
 	}
 
 	switch (cfi_mode) {
 	case CFI_OFF:
 		if (builtin)
-			pr_info("Disabling CFI\n");
+			pr_info("CFI: disabled\n");
 		return;
 
 	case CFI_KCFI:
+		pr_cfi_debug("CFI: re-enabling all indirect call checking\n");
 		ret = cfi_enable_callers(start_retpoline, end_retpoline);
 		if (ret)
 			goto err;
 
 		if (builtin)
-			pr_info("Using kCFI\n");
+			pr_info("CFI: Using %sretpoline kCFI\n",
+				cfi_rand ? "rehashed " : "");
 		return;
 
 	case CFI_FINEIBT:
+		pr_cfi_debug("CFI: adding FineIBT to all preambles\n");
 		/* place the FineIBT preamble at func()-16 */
 		ret = cfi_rewrite_preamble(start_cfi, end_cfi);
 		if (ret)
 			goto err;
 
 		/* rewrite the callers to target func()-16 */
+		pr_cfi_debug("CFI: rewriting indirect call sites to use FineIBT\n");
 		ret = cfi_rewrite_callers(start_retpoline, end_retpoline);
 		if (ret)
 			goto err;
 
 		/* now that nobody targets func()+0, remove ENDBR there */
+		pr_cfi_debug("CFI: removing old endbr insns\n");
 		cfi_rewrite_endbr(start_cfi, end_cfi);
 
-		if (builtin)
-			pr_info("Using FineIBT CFI\n");
+		if (builtin) {
+			pr_info("Using %sFineIBT%s CFI\n",
+				cfi_paranoid ? "paranoid " : "",
+				cfi_bhi ? "+BHI" : "");
+		}
 		return;
 
 	default:
@@ -1349,22 +1840,42 @@ static inline void poison_hash(void *addr)
 
 static void poison_cfi(void *addr)
 {
+	/*
+	 * Compilers manage to be inconsistent with ENDBR vs __cfi prefixes,
+	 * some (static) functions for which they can determine the address
+	 * is never taken do not get a __cfi prefix, but *DO* get an ENDBR.
+	 *
+	 * As such, these functions will get sealed, but we need to be careful
+	 * to not unconditionally scribble the previous function.
+	 */
 	switch (cfi_mode) {
 	case CFI_FINEIBT:
 		/*
+		 * FineIBT prefix should start with an ENDBR.
+		 */
+		if (!is_endbr(addr))
+			break;
+
+		/*
 		 * __cfi_\func:
-		 *	osp nopl (%rax)
-		 *	subl	$0, %r10d
-		 *	jz	1f
-		 *	ud2
-		 * 1:	nop
+		 *	nopl	-42(%rax)
+		 *	sub	$0, %eax
+		 *	jne	\func+3
+		 * \func:
+		 *	nopl	-42(%rax)
 		 */
-		poison_endbr(addr, false);
+		poison_endbr(addr);
 		poison_hash(addr + fineibt_preamble_hash);
 		break;
 
 	case CFI_KCFI:
 		/*
+		 * kCFI prefix should start with a valid hash.
+		 */
+		if (!decode_preamble_hash(addr, NULL))
+			break;
+
+		/*
 		 * __cfi_\func:
 		 *	movl	$0, %eax
 		 *	.skip	11, 0x90
@@ -1377,18 +1888,168 @@ static void poison_cfi(void *addr)
 	}
 }
 
-#else
+#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
+
+/*
+ * When regs->ip points to a 0xD6 byte in the FineIBT preamble,
+ * return true and fill out target and type.
+ *
+ * We check the preamble by checking for the ENDBR instruction relative to the
+ * UDB instruction.
+ */
+static bool decode_fineibt_preamble(struct pt_regs *regs, unsigned long *target, u32 *type)
+{
+	unsigned long addr = regs->ip - fineibt_preamble_ud;
+	u32 hash;
+
+	if (!exact_endbr((void *)addr))
+		return false;
+
+	*target = addr + fineibt_prefix_size;
+
+	__get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault);
+	*type = (u32)regs->ax + hash;
+
+	/*
+	 * Since regs->ip points to the middle of an instruction; it cannot
+	 * continue with the normal fixup.
+	 */
+	regs->ip = *target;
+
+	return true;
+
+Efault:
+	return false;
+}
+
+/*
+ * regs->ip points to one of the UD2 in __bhi_args[].
+ */
+static bool decode_fineibt_bhi(struct pt_regs *regs, unsigned long *target, u32 *type)
+{
+	unsigned long addr;
+	u32 hash;
+
+	if (!cfi_bhi)
+		return false;
+
+	if (regs->ip < (unsigned long)__bhi_args ||
+	    regs->ip >= (unsigned long)__bhi_args_end)
+		return false;
+
+	/*
+	 * Fetch the return address from the stack, this points to the
+	 * FineIBT preamble. Since the CALL instruction is in the 5 last
+	 * bytes of the preamble, the return address is in fact the target
+	 * address.
+	 */
+	__get_kernel_nofault(&addr, regs->sp, unsigned long, Efault);
+	*target = addr;
+
+	addr -= fineibt_prefix_size;
+	if (!exact_endbr((void *)addr))
+		return false;
+
+	__get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault);
+	*type = (u32)regs->ax + hash;
+
+	/*
+	 * The UD2 sites are constructed with a RET immediately following,
+	 * as such the non-fatal case can use the regular fixup.
+	 */
+	return true;
+
+Efault:
+	return false;
+}
+
+static bool is_paranoid_thunk(unsigned long addr)
+{
+	u32 thunk;
+
+	__get_kernel_nofault(&thunk, (u32 *)addr, u32, Efault);
+	return (thunk & 0x00FFFFFF) == 0xfd75d6;
+
+Efault:
+	return false;
+}
+
+/*
+ * regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_start[]
+ * sequence, or to UDB + Jcc.d8 for cfi_paranoid + ITS thunk.
+ */
+static bool decode_fineibt_paranoid(struct pt_regs *regs, unsigned long *target, u32 *type)
+{
+	unsigned long addr = regs->ip - fineibt_paranoid_ud;
+
+	if (!cfi_paranoid)
+		return false;
+
+	if (is_cfi_trap(addr + fineibt_caller_size - LEN_UD2)) {
+		*target = regs->r11 + fineibt_prefix_size;
+		*type = regs->ax;
+
+		/*
+		 * Since the trapping instruction is the exact, but LOCK prefixed,
+		 * Jcc.d8 that got us here, the normal fixup will work.
+		 */
+		return true;
+	}
+
+	/*
+	 * The cfi_paranoid + ITS thunk combination results in:
+	 *
+	 *  0:   b8 78 56 34 12          mov    $0x12345678, %eax
+	 *  5:   41 3b 43 f7             cmp    -11(%r11), %eax
+	 *  a:   2e 3d 8d 5b f0          cs lea -0x10(%r11), %r11
+	 *  e:   2e e8 XX XX XX XX	 cs call __x86_indirect_paranoid_thunk_r11
+	 *
+	 * Where the paranoid_thunk looks like:
+	 *
+	 *  1d:  <d6>                    udb
+	 *  __x86_indirect_paranoid_thunk_r11:
+	 *  1e:  75 fd                   jne 1d
+	 *  __x86_indirect_its_thunk_r11:
+	 *  20:  41 ff eb                jmp *%r11
+	 *  23:  cc                      int3
+	 *
+	 */
+	if (is_paranoid_thunk(regs->ip)) {
+		*target = regs->r11 + fineibt_prefix_size;
+		*type = regs->ax;
+
+		regs->ip = *target;
+		return true;
+	}
+
+	return false;
+}
+
+bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type)
+{
+	if (decode_fineibt_paranoid(regs, target, type))
+		return true;
+
+	if (decode_fineibt_bhi(regs, target, type))
+		return true;
+
+	return decode_fineibt_preamble(regs, target, type);
+}
+
+#else /* !CONFIG_FINEIBT: */
 
 static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
 			    s32 *start_cfi, s32 *end_cfi, bool builtin)
 {
+	if (IS_ENABLED(CONFIG_CFI) && builtin)
+		pr_info("CFI: Using standard kCFI\n");
 }
 
 #ifdef CONFIG_X86_KERNEL_IBT
 static void poison_cfi(void *addr) { }
 #endif
 
-#endif
+#endif /* !CONFIG_FINEIBT */
 
 void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
 		   s32 *start_cfi, s32 *end_cfi)
@@ -1562,21 +2223,34 @@ int alternatives_text_reserved(void *start, void *end)
  * See entry_{32,64}.S for more details.
  */
 
-/*
- * We define the int3_magic() function in assembly to control the calling
- * convention such that we can 'call' it from assembly.
- */
+extern void int3_selftest_asm(unsigned int *ptr);
 
-extern void int3_magic(unsigned int *ptr); /* defined in asm */
+asm (
+"	.pushsection	.init.text, \"ax\", @progbits\n"
+"	.type		int3_selftest_asm, @function\n"
+"int3_selftest_asm:\n"
+	ANNOTATE_NOENDBR "\n"
+	/*
+	 * INT3 padded with NOP to CALL_INSN_SIZE. The INT3 triggers an
+	 * exception, then the int3_exception_nb notifier emulates a call to
+	 * int3_selftest_callee().
+	 */
+"	int3; nop; nop; nop; nop\n"
+	ASM_RET
+"	.size		int3_selftest_asm, . - int3_selftest_asm\n"
+"	.popsection\n"
+);
+
+extern void int3_selftest_callee(unsigned int *ptr);
 
 asm (
 "	.pushsection	.init.text, \"ax\", @progbits\n"
-"	.type		int3_magic, @function\n"
-"int3_magic:\n"
-	ANNOTATE_NOENDBR
-"	movl	$1, (%" _ASM_ARG1 ")\n"
+"	.type		int3_selftest_callee, @function\n"
+"int3_selftest_callee:\n"
+	ANNOTATE_NOENDBR "\n"
+"	movl	$0x1234, (%" _ASM_ARG1 ")\n"
 	ASM_RET
-"	.size		int3_magic, .-int3_magic\n"
+"	.size		int3_selftest_callee, . - int3_selftest_callee\n"
 "	.popsection\n"
 );
 
@@ -1585,7 +2259,7 @@ extern void int3_selftest_ip(void); /* defined in asm below */
 static int __init
 int3_exception_notify(struct notifier_block *self, unsigned long val, void *data)
 {
-	unsigned long selftest = (unsigned long)&int3_selftest_ip;
+	unsigned long selftest = (unsigned long)&int3_selftest_asm;
 	struct die_args *args = data;
 	struct pt_regs *regs = args->regs;
 
@@ -1600,7 +2274,7 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *data
 	if (regs->ip - INT3_INSN_SIZE != selftest)
 		return NOTIFY_DONE;
 
-	int3_emulate_call(regs, (unsigned long)&int3_magic);
+	int3_emulate_call(regs, (unsigned long)&int3_selftest_callee);
 	return NOTIFY_STOP;
 }
 
@@ -1616,19 +2290,11 @@ static noinline void __init int3_selftest(void)
 	BUG_ON(register_die_notifier(&int3_exception_nb));
 
 	/*
-	 * Basically: int3_magic(&val); but really complicated :-)
-	 *
-	 * INT3 padded with NOP to CALL_INSN_SIZE. The int3_exception_nb
-	 * notifier above will emulate CALL for us.
+	 * Basically: int3_selftest_callee(&val); but really complicated :-)
 	 */
-	asm volatile ("int3_selftest_ip:\n\t"
-		      ANNOTATE_NOENDBR
-		      "    int3; nop; nop; nop; nop\n\t"
-		      : ASM_CALL_CONSTRAINT
-		      : __ASM_SEL_RAW(a, D) (&val)
-		      : "memory");
+	int3_selftest_asm(&val);
 
-	BUG_ON(val != 1);
+	BUG_ON(val != 0x1234);
 
 	unregister_die_notifier(&int3_exception_nb);
 }
@@ -1644,7 +2310,7 @@ __visible noinline void __init __alt_reloc_selftest(void *arg)
 static noinline void __init alt_reloc_selftest(void)
 {
 	/*
-	 * Tests apply_relocation().
+	 * Tests text_poke_apply_relocation().
 	 *
 	 * This has a relative immediate (CALL) in a place other than the first
 	 * instruction and additionally on x86_64 we get a RIP-relative LEA:
@@ -1665,6 +2331,8 @@ static noinline void __init alt_reloc_selftest(void)
 
 void __init alternative_instructions(void)
 {
+	u64 ibt;
+
 	int3_selftest();
 
 	/*
@@ -1691,8 +2359,12 @@ void __init alternative_instructions(void)
 	 */
 	paravirt_set_cap();
 
+	/* Keep CET-IBT disabled until caller/callee are patched */
+	ibt = ibt_save(/*disable*/ true);
+
 	__apply_fineibt(__retpoline_sites, __retpoline_sites_end,
 			__cfi_sites, __cfi_sites_end, true);
+	cfi_debug = false;
 
 	/*
 	 * Rewrite the retpolines, must be done before alternatives since
@@ -1701,19 +2373,23 @@ void __init alternative_instructions(void)
 	apply_retpolines(__retpoline_sites, __retpoline_sites_end);
 	apply_returns(__return_sites, __return_sites_end);
 
-	apply_alternatives(__alt_instructions, __alt_instructions_end);
+	its_fini_core();
 
 	/*
-	 * Now all calls are established. Apply the call thunks if
-	 * required.
+	 * Adjust all CALL instructions to point to func()-10, including
+	 * those in .altinstr_replacement.
 	 */
 	callthunks_patch_builtin_calls();
 
+	apply_alternatives(__alt_instructions, __alt_instructions_end);
+
 	/*
 	 * Seal all functions that do not have their address taken.
 	 */
 	apply_seal_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end);
 
+	ibt_restore(ibt);
+
 #ifdef CONFIG_SMP
 	/* Patch to UP if other cpus not imminent. */
 	if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) {
@@ -1774,83 +2450,33 @@ void __init_or_module text_poke_early(void *addr, const void *opcode,
 	}
 }
 
-typedef struct {
-	struct mm_struct *mm;
-} temp_mm_state_t;
+__ro_after_init struct mm_struct *text_poke_mm;
+__ro_after_init unsigned long text_poke_mm_addr;
 
 /*
- * Using a temporary mm allows to set temporary mappings that are not accessible
- * by other CPUs. Such mappings are needed to perform sensitive memory writes
- * that override the kernel memory protections (e.g., W^X), without exposing the
- * temporary page-table mappings that are required for these write operations to
- * other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the
- * mapping is torn down.
+ * Text poking creates and uses a mapping in the lower half of the
+ * address space. Relax LASS enforcement when accessing the poking
+ * address.
  *
- * Context: The temporary mm needs to be used exclusively by a single core. To
- *          harden security IRQs must be disabled while the temporary mm is
- *          loaded, thereby preventing interrupt handler bugs from overriding
- *          the kernel memory protection.
+ * objtool enforces a strict policy of "no function calls within AC=1
+ * regions". Adhere to the policy by using inline versions of
+ * memcpy()/memset() that will never result in a function call.
  */
-static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm)
-{
-	temp_mm_state_t temp_state;
-
-	lockdep_assert_irqs_disabled();
-
-	/*
-	 * Make sure not to be in TLB lazy mode, as otherwise we'll end up
-	 * with a stale address space WITHOUT being in lazy mode after
-	 * restoring the previous mm.
-	 */
-	if (this_cpu_read(cpu_tlbstate_shared.is_lazy))
-		leave_mm();
-
-	temp_state.mm = this_cpu_read(cpu_tlbstate.loaded_mm);
-	switch_mm_irqs_off(NULL, mm, current);
-
-	/*
-	 * If breakpoints are enabled, disable them while the temporary mm is
-	 * used. Userspace might set up watchpoints on addresses that are used
-	 * in the temporary mm, which would lead to wrong signals being sent or
-	 * crashes.
-	 *
-	 * Note that breakpoints are not disabled selectively, which also causes
-	 * kernel breakpoints (e.g., perf's) to be disabled. This might be
-	 * undesirable, but still seems reasonable as the code that runs in the
-	 * temporary mm should be short.
-	 */
-	if (hw_breakpoint_active())
-		hw_breakpoint_disable();
-
-	return temp_state;
-}
-
-static inline void unuse_temporary_mm(temp_mm_state_t prev_state)
-{
-	lockdep_assert_irqs_disabled();
-	switch_mm_irqs_off(NULL, prev_state.mm, current);
-
-	/*
-	 * Restore the breakpoints if they were disabled before the temporary mm
-	 * was loaded.
-	 */
-	if (hw_breakpoint_active())
-		hw_breakpoint_restore();
-}
-
-__ro_after_init struct mm_struct *poking_mm;
-__ro_after_init unsigned long poking_addr;
 
 static void text_poke_memcpy(void *dst, const void *src, size_t len)
 {
-	memcpy(dst, src, len);
+	lass_stac();
+	__inline_memcpy(dst, src, len);
+	lass_clac();
 }
 
 static void text_poke_memset(void *dst, const void *src, size_t len)
 {
 	int c = *(const int *)src;
 
-	memset(dst, c, len);
+	lass_stac();
+	__inline_memset(dst, c, len);
+	lass_clac();
 }
 
 typedef void text_poke_f(void *dst, const void *src, size_t len);
@@ -1859,7 +2485,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l
 {
 	bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE;
 	struct page *pages[2] = {NULL};
-	temp_mm_state_t prev;
+	struct mm_struct *prev_mm;
 	unsigned long flags;
 	pte_t pte, *ptep;
 	spinlock_t *ptl;
@@ -1896,7 +2522,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l
 	/*
 	 * The lock is not really needed, but this allows to avoid open-coding.
 	 */
-	ptep = get_locked_pte(poking_mm, poking_addr, &ptl);
+	ptep = get_locked_pte(text_poke_mm, text_poke_mm_addr, &ptl);
 
 	/*
 	 * This must not fail; preallocated in poking_init().
@@ -1906,21 +2532,21 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l
 	local_irq_save(flags);
 
 	pte = mk_pte(pages[0], pgprot);
-	set_pte_at(poking_mm, poking_addr, ptep, pte);
+	set_pte_at(text_poke_mm, text_poke_mm_addr, ptep, pte);
 
 	if (cross_page_boundary) {
 		pte = mk_pte(pages[1], pgprot);
-		set_pte_at(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte);
+		set_pte_at(text_poke_mm, text_poke_mm_addr + PAGE_SIZE, ptep + 1, pte);
 	}
 
 	/*
 	 * Loading the temporary mm behaves as a compiler barrier, which
 	 * guarantees that the PTE will be set at the time memcpy() is done.
 	 */
-	prev = use_temporary_mm(poking_mm);
+	prev_mm = use_temporary_mm(text_poke_mm);
 
 	kasan_disable_current();
-	func((u8 *)poking_addr + offset_in_page(addr), src, len);
+	func((u8 *)text_poke_mm_addr + offset_in_page(addr), src, len);
 	kasan_enable_current();
 
 	/*
@@ -1929,22 +2555,22 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l
 	 */
 	barrier();
 
-	pte_clear(poking_mm, poking_addr, ptep);
+	pte_clear(text_poke_mm, text_poke_mm_addr, ptep);
 	if (cross_page_boundary)
-		pte_clear(poking_mm, poking_addr + PAGE_SIZE, ptep + 1);
+		pte_clear(text_poke_mm, text_poke_mm_addr + PAGE_SIZE, ptep + 1);
 
 	/*
 	 * Loading the previous page-table hierarchy requires a serializing
 	 * instruction that already allows the core to see the updated version.
 	 * Xen-PV is assumed to serialize execution in a similar manner.
 	 */
-	unuse_temporary_mm(prev);
+	unuse_temporary_mm(prev_mm);
 
 	/*
 	 * Flushing the TLB might involve IPIs, which would require enabled
 	 * IRQs, but not if the mm is not used, as it is in this point.
 	 */
-	flush_tlb_mm_range(poking_mm, poking_addr, poking_addr +
+	flush_tlb_mm_range(text_poke_mm, text_poke_mm_addr, text_poke_mm_addr +
 			   (cross_page_boundary ? 2 : 1) * PAGE_SIZE,
 			   PAGE_SHIFT, false);
 
@@ -2080,7 +2706,7 @@ static void do_sync_core(void *info)
 	sync_core();
 }
 
-void text_poke_sync(void)
+void smp_text_poke_sync_each_cpu(void)
 {
 	on_each_cpu(do_sync_core, NULL, 1);
 }
@@ -2090,64 +2716,66 @@ void text_poke_sync(void)
  * this thing. When len == 6 everything is prefixed with 0x0f and we map
  * opcode to Jcc.d8, using len to distinguish.
  */
-struct text_poke_loc {
+struct smp_text_poke_loc {
 	/* addr := _stext + rel_addr */
 	s32 rel_addr;
 	s32 disp;
 	u8 len;
 	u8 opcode;
-	const u8 text[POKE_MAX_OPCODE_SIZE];
-	/* see text_poke_bp_batch() */
+	const u8 text[TEXT_POKE_MAX_OPCODE_SIZE];
+	/* see smp_text_poke_batch_finish() */
 	u8 old;
 };
 
-struct bp_patching_desc {
-	struct text_poke_loc *vec;
+#define TEXT_POKE_ARRAY_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc))
+
+static struct smp_text_poke_array {
+	struct smp_text_poke_loc vec[TEXT_POKE_ARRAY_MAX];
 	int nr_entries;
-	atomic_t refs;
-};
+} text_poke_array;
 
-static struct bp_patching_desc bp_desc;
+static DEFINE_PER_CPU(atomic_t, text_poke_array_refs);
 
-static __always_inline
-struct bp_patching_desc *try_get_desc(void)
+/*
+ * These four __always_inline annotations imply noinstr, necessary
+ * due to smp_text_poke_int3_handler() being noinstr:
+ */
+
+static __always_inline bool try_get_text_poke_array(void)
 {
-	struct bp_patching_desc *desc = &bp_desc;
+	atomic_t *refs = this_cpu_ptr(&text_poke_array_refs);
 
-	if (!raw_atomic_inc_not_zero(&desc->refs))
-		return NULL;
+	if (!raw_atomic_inc_not_zero(refs))
+		return false;
 
-	return desc;
+	return true;
 }
 
-static __always_inline void put_desc(void)
+static __always_inline void put_text_poke_array(void)
 {
-	struct bp_patching_desc *desc = &bp_desc;
+	atomic_t *refs = this_cpu_ptr(&text_poke_array_refs);
 
 	smp_mb__before_atomic();
-	raw_atomic_dec(&desc->refs);
+	raw_atomic_dec(refs);
 }
 
-static __always_inline void *text_poke_addr(struct text_poke_loc *tp)
+static __always_inline void *text_poke_addr(const struct smp_text_poke_loc *tpl)
 {
-	return _stext + tp->rel_addr;
+	return _stext + tpl->rel_addr;
 }
 
-static __always_inline int patch_cmp(const void *key, const void *elt)
+static __always_inline int patch_cmp(const void *tpl_a, const void *tpl_b)
 {
-	struct text_poke_loc *tp = (struct text_poke_loc *) elt;
-
-	if (key < text_poke_addr(tp))
+	if (tpl_a < text_poke_addr(tpl_b))
 		return -1;
-	if (key > text_poke_addr(tp))
+	if (tpl_a > text_poke_addr(tpl_b))
 		return 1;
 	return 0;
 }
 
-noinstr int poke_int3_handler(struct pt_regs *regs)
+noinstr int smp_text_poke_int3_handler(struct pt_regs *regs)
 {
-	struct bp_patching_desc *desc;
-	struct text_poke_loc *tp;
+	struct smp_text_poke_loc *tpl;
 	int ret = 0;
 	void *ip;
 
@@ -2156,41 +2784,40 @@ noinstr int poke_int3_handler(struct pt_regs *regs)
 
 	/*
 	 * Having observed our INT3 instruction, we now must observe
-	 * bp_desc with non-zero refcount:
+	 * text_poke_array with non-zero refcount:
 	 *
-	 *	bp_desc.refs = 1		INT3
-	 *	WMB				RMB
-	 *	write INT3			if (bp_desc.refs != 0)
+	 *	text_poke_array_refs = 1		INT3
+	 *	WMB			RMB
+	 *	write INT3		if (text_poke_array_refs != 0)
 	 */
 	smp_rmb();
 
-	desc = try_get_desc();
-	if (!desc)
+	if (!try_get_text_poke_array())
 		return 0;
 
 	/*
-	 * Discount the INT3. See text_poke_bp_batch().
+	 * Discount the INT3. See smp_text_poke_batch_finish().
 	 */
 	ip = (void *) regs->ip - INT3_INSN_SIZE;
 
 	/*
 	 * Skip the binary search if there is a single member in the vector.
 	 */
-	if (unlikely(desc->nr_entries > 1)) {
-		tp = __inline_bsearch(ip, desc->vec, desc->nr_entries,
-				      sizeof(struct text_poke_loc),
+	if (unlikely(text_poke_array.nr_entries > 1)) {
+		tpl = __inline_bsearch(ip, text_poke_array.vec, text_poke_array.nr_entries,
+				      sizeof(struct smp_text_poke_loc),
 				      patch_cmp);
-		if (!tp)
+		if (!tpl)
 			goto out_put;
 	} else {
-		tp = desc->vec;
-		if (text_poke_addr(tp) != ip)
+		tpl = text_poke_array.vec;
+		if (text_poke_addr(tpl) != ip)
 			goto out_put;
 	}
 
-	ip += tp->len;
+	ip += tpl->len;
 
-	switch (tp->opcode) {
+	switch (tpl->opcode) {
 	case INT3_INSN_OPCODE:
 		/*
 		 * Someone poked an explicit INT3, they'll want to handle it,
@@ -2203,16 +2830,16 @@ noinstr int poke_int3_handler(struct pt_regs *regs)
 		break;
 
 	case CALL_INSN_OPCODE:
-		int3_emulate_call(regs, (long)ip + tp->disp);
+		int3_emulate_call(regs, (long)ip + tpl->disp);
 		break;
 
 	case JMP32_INSN_OPCODE:
 	case JMP8_INSN_OPCODE:
-		int3_emulate_jmp(regs, (long)ip + tp->disp);
+		int3_emulate_jmp(regs, (long)ip + tpl->disp);
 		break;
 
 	case 0x70 ... 0x7f: /* Jcc */
-		int3_emulate_jcc(regs, tp->opcode & 0xf, (long)ip, tp->disp);
+		int3_emulate_jcc(regs, tpl->opcode & 0xf, (long)ip, tpl->disp);
 		break;
 
 	default:
@@ -2222,51 +2849,50 @@ noinstr int poke_int3_handler(struct pt_regs *regs)
 	ret = 1;
 
 out_put:
-	put_desc();
+	put_text_poke_array();
 	return ret;
 }
 
-#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc))
-static struct text_poke_loc tp_vec[TP_VEC_MAX];
-static int tp_vec_nr;
-
 /**
- * text_poke_bp_batch() -- update instructions on live kernel on SMP
- * @tp:			vector of instructions to patch
- * @nr_entries:		number of entries in the vector
+ * smp_text_poke_batch_finish() -- update instructions on live kernel on SMP
  *
- * Modify multi-byte instruction by using int3 breakpoint on SMP.
- * We completely avoid stop_machine() here, and achieve the
- * synchronization using int3 breakpoint.
+ * Input state:
+ *  text_poke_array.vec: vector of instructions to patch
+ *  text_poke_array.nr_entries: number of entries in the vector
+ *
+ * Modify multi-byte instructions by using INT3 breakpoints on SMP.
+ * We completely avoid using stop_machine() here, and achieve the
+ * synchronization using INT3 breakpoints and SMP cross-calls.
  *
  * The way it is done:
  *	- For each entry in the vector:
- *		- add a int3 trap to the address that will be patched
- *	- sync cores
+ *		- add an INT3 trap to the address that will be patched
+ *	- SMP sync all CPUs
  *	- For each entry in the vector:
  *		- update all but the first byte of the patched range
- *	- sync cores
+ *	- SMP sync all CPUs
  *	- For each entry in the vector:
- *		- replace the first byte (int3) by the first byte of
+ *		- replace the first byte (INT3) by the first byte of the
  *		  replacing opcode
- *	- sync cores
+ *	- SMP sync all CPUs
  */
-static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries)
+void smp_text_poke_batch_finish(void)
 {
 	unsigned char int3 = INT3_INSN_OPCODE;
 	unsigned int i;
 	int do_sync;
 
-	lockdep_assert_held(&text_mutex);
+	if (!text_poke_array.nr_entries)
+		return;
 
-	bp_desc.vec = tp;
-	bp_desc.nr_entries = nr_entries;
+	lockdep_assert_held(&text_mutex);
 
 	/*
-	 * Corresponds to the implicit memory barrier in try_get_desc() to
-	 * ensure reading a non-zero refcount provides up to date bp_desc data.
+	 * Corresponds to the implicit memory barrier in try_get_text_poke_array() to
+	 * ensure reading a non-zero refcount provides up to date text_poke_array data.
 	 */
-	atomic_set_release(&bp_desc.refs, 1);
+	for_each_possible_cpu(i)
+		atomic_set_release(per_cpu_ptr(&text_poke_array_refs, i), 1);
 
 	/*
 	 * Function tracing can enable thousands of places that need to be
@@ -2279,33 +2905,33 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries
 	cond_resched();
 
 	/*
-	 * Corresponding read barrier in int3 notifier for making sure the
-	 * nr_entries and handler are correctly ordered wrt. patching.
+	 * Corresponding read barrier in INT3 notifier for making sure the
+	 * text_poke_array.nr_entries and handler are correctly ordered wrt. patching.
 	 */
 	smp_wmb();
 
 	/*
-	 * First step: add a int3 trap to the address that will be patched.
+	 * First step: add a INT3 trap to the address that will be patched.
 	 */
-	for (i = 0; i < nr_entries; i++) {
-		tp[i].old = *(u8 *)text_poke_addr(&tp[i]);
-		text_poke(text_poke_addr(&tp[i]), &int3, INT3_INSN_SIZE);
+	for (i = 0; i < text_poke_array.nr_entries; i++) {
+		text_poke_array.vec[i].old = *(u8 *)text_poke_addr(&text_poke_array.vec[i]);
+		text_poke(text_poke_addr(&text_poke_array.vec[i]), &int3, INT3_INSN_SIZE);
 	}
 
-	text_poke_sync();
+	smp_text_poke_sync_each_cpu();
 
 	/*
 	 * Second step: update all but the first byte of the patched range.
 	 */
-	for (do_sync = 0, i = 0; i < nr_entries; i++) {
-		u8 old[POKE_MAX_OPCODE_SIZE+1] = { tp[i].old, };
-		u8 _new[POKE_MAX_OPCODE_SIZE+1];
-		const u8 *new = tp[i].text;
-		int len = tp[i].len;
+	for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) {
+		u8 old[TEXT_POKE_MAX_OPCODE_SIZE+1] = { text_poke_array.vec[i].old, };
+		u8 _new[TEXT_POKE_MAX_OPCODE_SIZE+1];
+		const u8 *new = text_poke_array.vec[i].text;
+		int len = text_poke_array.vec[i].len;
 
 		if (len - INT3_INSN_SIZE > 0) {
 			memcpy(old + INT3_INSN_SIZE,
-			       text_poke_addr(&tp[i]) + INT3_INSN_SIZE,
+			       text_poke_addr(&text_poke_array.vec[i]) + INT3_INSN_SIZE,
 			       len - INT3_INSN_SIZE);
 
 			if (len == 6) {
@@ -2314,7 +2940,7 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries
 				new = _new;
 			}
 
-			text_poke(text_poke_addr(&tp[i]) + INT3_INSN_SIZE,
+			text_poke(text_poke_addr(&text_poke_array.vec[i]) + INT3_INSN_SIZE,
 				  new + INT3_INSN_SIZE,
 				  len - INT3_INSN_SIZE);
 
@@ -2345,7 +2971,7 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries
 		 * The old instruction is recorded so that the event can be
 		 * processed forwards or backwards.
 		 */
-		perf_event_text_poke(text_poke_addr(&tp[i]), old, len, new, len);
+		perf_event_text_poke(text_poke_addr(&text_poke_array.vec[i]), old, len, new, len);
 	}
 
 	if (do_sync) {
@@ -2354,63 +2980,79 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries
 		 * not necessary and we'd be safe even without it. But
 		 * better safe than sorry (plus there's not only Intel).
 		 */
-		text_poke_sync();
+		smp_text_poke_sync_each_cpu();
 	}
 
 	/*
-	 * Third step: replace the first byte (int3) by the first byte of
+	 * Third step: replace the first byte (INT3) by the first byte of the
 	 * replacing opcode.
 	 */
-	for (do_sync = 0, i = 0; i < nr_entries; i++) {
-		u8 byte = tp[i].text[0];
+	for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) {
+		u8 byte = text_poke_array.vec[i].text[0];
 
-		if (tp[i].len == 6)
+		if (text_poke_array.vec[i].len == 6)
 			byte = 0x0f;
 
 		if (byte == INT3_INSN_OPCODE)
 			continue;
 
-		text_poke(text_poke_addr(&tp[i]), &byte, INT3_INSN_SIZE);
+		text_poke(text_poke_addr(&text_poke_array.vec[i]), &byte, INT3_INSN_SIZE);
 		do_sync++;
 	}
 
 	if (do_sync)
-		text_poke_sync();
+		smp_text_poke_sync_each_cpu();
 
 	/*
 	 * Remove and wait for refs to be zero.
+	 *
+	 * Notably, if after step-3 above the INT3 got removed, then the
+	 * smp_text_poke_sync_each_cpu() will have serialized against any running INT3
+	 * handlers and the below spin-wait will not happen.
+	 *
+	 * IOW. unless the replacement instruction is INT3, this case goes
+	 * unused.
 	 */
-	if (!atomic_dec_and_test(&bp_desc.refs))
-		atomic_cond_read_acquire(&bp_desc.refs, !VAL);
+	for_each_possible_cpu(i) {
+		atomic_t *refs = per_cpu_ptr(&text_poke_array_refs, i);
+
+		if (unlikely(!atomic_dec_and_test(refs)))
+			atomic_cond_read_acquire(refs, !VAL);
+	}
+
+	/* They are all completed: */
+	text_poke_array.nr_entries = 0;
 }
 
-static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
-			       const void *opcode, size_t len, const void *emulate)
+static void __smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate)
 {
+	struct smp_text_poke_loc *tpl;
 	struct insn insn;
 	int ret, i = 0;
 
+	tpl = &text_poke_array.vec[text_poke_array.nr_entries++];
+
 	if (len == 6)
 		i = 1;
-	memcpy((void *)tp->text, opcode+i, len-i);
+	memcpy((void *)tpl->text, opcode+i, len-i);
 	if (!emulate)
 		emulate = opcode;
 
 	ret = insn_decode_kernel(&insn, emulate);
 	BUG_ON(ret < 0);
 
-	tp->rel_addr = addr - (void *)_stext;
-	tp->len = len;
-	tp->opcode = insn.opcode.bytes[0];
+	tpl->rel_addr = addr - (void *)_stext;
+	tpl->len = len;
+	tpl->opcode = insn.opcode.bytes[0];
 
 	if (is_jcc32(&insn)) {
 		/*
 		 * Map Jcc.d32 onto Jcc.d8 and use len to distinguish.
 		 */
-		tp->opcode = insn.opcode.bytes[1] - 0x10;
+		tpl->opcode = insn.opcode.bytes[1] - 0x10;
 	}
 
-	switch (tp->opcode) {
+	switch (tpl->opcode) {
 	case RET_INSN_OPCODE:
 	case JMP32_INSN_OPCODE:
 	case JMP8_INSN_OPCODE:
@@ -2419,14 +3061,14 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
 		 * next instruction can be padded with INT3.
 		 */
 		for (i = insn.length; i < len; i++)
-			BUG_ON(tp->text[i] != INT3_INSN_OPCODE);
+			BUG_ON(tpl->text[i] != INT3_INSN_OPCODE);
 		break;
 
 	default:
 		BUG_ON(len != insn.length);
 	}
 
-	switch (tp->opcode) {
+	switch (tpl->opcode) {
 	case INT3_INSN_OPCODE:
 	case RET_INSN_OPCODE:
 		break;
@@ -2435,21 +3077,21 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
 	case JMP32_INSN_OPCODE:
 	case JMP8_INSN_OPCODE:
 	case 0x70 ... 0x7f: /* Jcc */
-		tp->disp = insn.immediate.value;
+		tpl->disp = insn.immediate.value;
 		break;
 
 	default: /* assume NOP */
 		switch (len) {
 		case 2: /* NOP2 -- emulate as JMP8+0 */
 			BUG_ON(memcmp(emulate, x86_nops[len], len));
-			tp->opcode = JMP8_INSN_OPCODE;
-			tp->disp = 0;
+			tpl->opcode = JMP8_INSN_OPCODE;
+			tpl->disp = 0;
 			break;
 
 		case 5: /* NOP5 -- emulate as JMP32+0 */
 			BUG_ON(memcmp(emulate, x86_nops[len], len));
-			tp->opcode = JMP32_INSN_OPCODE;
-			tp->disp = 0;
+			tpl->opcode = JMP32_INSN_OPCODE;
+			tpl->disp = 0;
 			break;
 
 		default: /* unknown instruction */
@@ -2460,51 +3102,50 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
 }
 
 /*
- * We hard rely on the tp_vec being ordered; ensure this is so by flushing
+ * We hard rely on the text_poke_array.vec being ordered; ensure this is so by flushing
  * early if needed.
  */
-static bool tp_order_fail(void *addr)
+static bool text_poke_addr_ordered(void *addr)
 {
-	struct text_poke_loc *tp;
-
-	if (!tp_vec_nr)
-		return false;
+	WARN_ON_ONCE(!addr);
 
-	if (!addr) /* force */
+	if (!text_poke_array.nr_entries)
 		return true;
 
-	tp = &tp_vec[tp_vec_nr - 1];
-	if ((unsigned long)text_poke_addr(tp) > (unsigned long)addr)
-		return true;
-
-	return false;
-}
-
-static void text_poke_flush(void *addr)
-{
-	if (tp_vec_nr == TP_VEC_MAX || tp_order_fail(addr)) {
-		text_poke_bp_batch(tp_vec, tp_vec_nr);
-		tp_vec_nr = 0;
-	}
-}
+	/*
+	 * If the last current entry's address is higher than the
+	 * new entry's address we'd like to add, then ordering
+	 * is violated and we must first flush all pending patching
+	 * requests:
+	 */
+	if (text_poke_addr(text_poke_array.vec + text_poke_array.nr_entries-1) > addr)
+		return false;
 
-void text_poke_finish(void)
-{
-	text_poke_flush(NULL);
+	return true;
 }
 
-void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate)
+/**
+ * smp_text_poke_batch_add() -- update instruction on live kernel on SMP, batched
+ * @addr:	address to patch
+ * @opcode:	opcode of new instruction
+ * @len:	length to copy
+ * @emulate:	instruction to be emulated
+ *
+ * Add a new instruction to the current queue of to-be-patched instructions
+ * the kernel maintains. The patching request will not be executed immediately,
+ * but becomes part of an array of patching requests, optimized for batched
+ * execution. All pending patching requests will be executed on the next
+ * smp_text_poke_batch_finish() call.
+ */
+void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate)
 {
-	struct text_poke_loc *tp;
-
-	text_poke_flush(addr);
-
-	tp = &tp_vec[tp_vec_nr++];
-	text_poke_loc_init(tp, addr, opcode, len, emulate);
+	if (text_poke_array.nr_entries == TEXT_POKE_ARRAY_MAX || !text_poke_addr_ordered(addr))
+		smp_text_poke_batch_finish();
+	__smp_text_poke_batch_add(addr, opcode, len, emulate);
 }
 
 /**
- * text_poke_bp() -- update instructions on live kernel on SMP
+ * smp_text_poke_single() -- update instruction on live kernel on SMP immediately
  * @addr:	address to patch
  * @opcode:	opcode of new instruction
  * @len:	length to copy
@@ -2512,12 +3153,11 @@ void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const voi
  *
  * Update a single instruction with the vector in the stack, avoiding
  * dynamically allocated memory. This function should be used when it is
- * not possible to allocate memory.
+ * not possible to allocate memory for a vector. The single instruction
+ * is patched in immediately.
  */
-void __ref text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate)
+void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate)
 {
-	struct text_poke_loc tp;
-
-	text_poke_loc_init(&tp, addr, opcode, len, emulate);
-	text_poke_bp_batch(&tp, 1);
+	smp_text_poke_batch_add(addr, opcode, len, emulate);
+	smp_text_poke_batch_finish();
 }
diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c
index c884deca839b..e8000a56732e 100644
--- a/arch/x86/kernel/amd_gart_64.c
+++ b/arch/x86/kernel/amd_gart_64.c
@@ -39,7 +39,7 @@
 #include <asm/gart.h>
 #include <asm/set_memory.h>
 #include <asm/dma.h>
-#include <asm/amd_nb.h>
+#include <asm/amd/nb.h>
 #include <asm/x86_init.h>
 
 static unsigned long iommu_bus_base;	/* GART remapping area (physical) */
@@ -222,13 +222,14 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
 }
 
 /* Map a single area into the IOMMU */
-static dma_addr_t gart_map_page(struct device *dev, struct page *page,
-				unsigned long offset, size_t size,
-				enum dma_data_direction dir,
+static dma_addr_t gart_map_phys(struct device *dev, phys_addr_t paddr,
+				size_t size, enum dma_data_direction dir,
 				unsigned long attrs)
 {
 	unsigned long bus;
-	phys_addr_t paddr = page_to_phys(page) + offset;
+
+	if (unlikely(attrs & DMA_ATTR_MMIO))
+		return DMA_MAPPING_ERROR;
 
 	if (!need_iommu(dev, paddr, size))
 		return paddr;
@@ -242,7 +243,7 @@ static dma_addr_t gart_map_page(struct device *dev, struct page *page,
 /*
  * Free a DMA mapping.
  */
-static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr,
+static void gart_unmap_phys(struct device *dev, dma_addr_t dma_addr,
 			    size_t size, enum dma_data_direction dir,
 			    unsigned long attrs)
 {
@@ -282,7 +283,7 @@ static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
 	for_each_sg(sg, s, nents, i) {
 		if (!s->dma_length || !s->length)
 			break;
-		gart_unmap_page(dev, s->dma_address, s->dma_length, dir, 0);
+		gart_unmap_phys(dev, s->dma_address, s->dma_length, dir, 0);
 	}
 }
 
@@ -487,7 +488,7 @@ static void
 gart_free_coherent(struct device *dev, size_t size, void *vaddr,
 		   dma_addr_t dma_addr, unsigned long attrs)
 {
-	gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0);
+	gart_unmap_phys(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0);
 	dma_direct_free(dev, size, vaddr, dma_addr, attrs);
 }
 
@@ -591,7 +592,7 @@ static void gart_fixup_northbridges(void)
 	}
 }
 
-static void gart_resume(void)
+static void gart_resume(void *data)
 {
 	pr_info("PCI-DMA: Resuming GART IOMMU\n");
 
@@ -600,11 +601,15 @@ static void gart_resume(void)
 	enable_gart_translations();
 }
 
-static struct syscore_ops gart_syscore_ops = {
+static const struct syscore_ops gart_syscore_ops = {
 	.resume		= gart_resume,
 
 };
 
+static struct syscore gart_syscore = {
+	.ops = &gart_syscore_ops,
+};
+
 /*
  * Private Northbridge GATT initialization in case we cannot use the
  * AGP driver for some reason.
@@ -650,7 +655,7 @@ static __init int init_amd_gatt(struct agp_kern_info *info)
 
 	agp_gatt_table = gatt;
 
-	register_syscore_ops(&gart_syscore_ops);
+	register_syscore(&gart_syscore);
 
 	flush_gart();
 
@@ -668,8 +673,8 @@ static __init int init_amd_gatt(struct agp_kern_info *info)
 static const struct dma_map_ops gart_dma_ops = {
 	.map_sg				= gart_map_sg,
 	.unmap_sg			= gart_unmap_sg,
-	.map_page			= gart_map_page,
-	.unmap_page			= gart_unmap_page,
+	.map_phys			= gart_map_phys,
+	.unmap_phys			= gart_unmap_phys,
 	.alloc				= gart_alloc_coherent,
 	.free				= gart_free_coherent,
 	.mmap				= dma_common_mmap,
diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c
index 9fe9972d2071..c1acead6227a 100644
--- a/arch/x86/kernel/amd_nb.c
+++ b/arch/x86/kernel/amd_nb.c
@@ -13,67 +13,11 @@
 #include <linux/export.h>
 #include <linux/spinlock.h>
 #include <linux/pci_ids.h>
-#include <asm/amd_nb.h>
-
-#define PCI_DEVICE_ID_AMD_17H_ROOT		0x1450
-#define PCI_DEVICE_ID_AMD_17H_M10H_ROOT		0x15d0
-#define PCI_DEVICE_ID_AMD_17H_M30H_ROOT		0x1480
-#define PCI_DEVICE_ID_AMD_17H_M60H_ROOT		0x1630
-#define PCI_DEVICE_ID_AMD_17H_MA0H_ROOT		0x14b5
-#define PCI_DEVICE_ID_AMD_19H_M10H_ROOT		0x14a4
-#define PCI_DEVICE_ID_AMD_19H_M40H_ROOT		0x14b5
-#define PCI_DEVICE_ID_AMD_19H_M60H_ROOT		0x14d8
-#define PCI_DEVICE_ID_AMD_19H_M70H_ROOT		0x14e8
-#define PCI_DEVICE_ID_AMD_1AH_M00H_ROOT		0x153a
-#define PCI_DEVICE_ID_AMD_1AH_M20H_ROOT		0x1507
-#define PCI_DEVICE_ID_AMD_1AH_M60H_ROOT		0x1122
-#define PCI_DEVICE_ID_AMD_MI200_ROOT		0x14bb
-#define PCI_DEVICE_ID_AMD_MI300_ROOT		0x14f8
-
-#define PCI_DEVICE_ID_AMD_17H_DF_F4		0x1464
-#define PCI_DEVICE_ID_AMD_17H_M10H_DF_F4	0x15ec
-#define PCI_DEVICE_ID_AMD_17H_M30H_DF_F4	0x1494
-#define PCI_DEVICE_ID_AMD_17H_M60H_DF_F4	0x144c
-#define PCI_DEVICE_ID_AMD_17H_M70H_DF_F4	0x1444
-#define PCI_DEVICE_ID_AMD_17H_MA0H_DF_F4	0x1728
-#define PCI_DEVICE_ID_AMD_19H_DF_F4		0x1654
-#define PCI_DEVICE_ID_AMD_19H_M10H_DF_F4	0x14b1
-#define PCI_DEVICE_ID_AMD_19H_M40H_DF_F4	0x167d
-#define PCI_DEVICE_ID_AMD_19H_M50H_DF_F4	0x166e
-#define PCI_DEVICE_ID_AMD_19H_M60H_DF_F4	0x14e4
-#define PCI_DEVICE_ID_AMD_19H_M70H_DF_F4	0x14f4
-#define PCI_DEVICE_ID_AMD_19H_M78H_DF_F4	0x12fc
-#define PCI_DEVICE_ID_AMD_1AH_M00H_DF_F4	0x12c4
-#define PCI_DEVICE_ID_AMD_1AH_M20H_DF_F4	0x16fc
-#define PCI_DEVICE_ID_AMD_1AH_M60H_DF_F4	0x124c
-#define PCI_DEVICE_ID_AMD_1AH_M70H_DF_F4	0x12bc
-#define PCI_DEVICE_ID_AMD_MI200_DF_F4		0x14d4
-#define PCI_DEVICE_ID_AMD_MI300_DF_F4		0x152c
-
-/* Protect the PCI config register pairs used for SMN. */
-static DEFINE_MUTEX(smn_mutex);
 
-static u32 *flush_words;
+#include <asm/amd/nb.h>
+#include <asm/cpuid/api.h>
 
-static const struct pci_device_id amd_root_ids[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M60H_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_MA0H_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M10H_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M40H_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M60H_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M70H_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M00H_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M20H_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M60H_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_MI200_ROOT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_MI300_ROOT) },
-	{}
-};
-
-#define PCI_DEVICE_ID_AMD_CNB17H_F4     0x1704
+static u32 *flush_words;
 
 static const struct pci_device_id amd_nb_misc_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB_MISC) },
@@ -84,70 +28,6 @@ static const struct pci_device_id amd_nb_misc_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M60H_NB_F3) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_NB_F3) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_M30H_NB_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M60H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_MA0H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CNB17H_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M10H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M40H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M50H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M60H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M70H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M78H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M00H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M20H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M60H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M70H_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_MI200_DF_F3) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_MI300_DF_F3) },
-	{}
-};
-
-static const struct pci_device_id amd_nb_link_ids[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_NB_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M30H_NB_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M60H_NB_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_NB_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_M30H_NB_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M60H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_MA0H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M10H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M40H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M50H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M60H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M70H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M78H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CNB17H_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M00H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M20H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M60H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M70H_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_MI200_DF_F4) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_MI300_DF_F4) },
-	{}
-};
-
-static const struct pci_device_id hygon_root_ids[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_HYGON, PCI_DEVICE_ID_AMD_17H_ROOT) },
-	{}
-};
-
-static const struct pci_device_id hygon_nb_misc_ids[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_HYGON, PCI_DEVICE_ID_AMD_17H_DF_F3) },
-	{}
-};
-
-static const struct pci_device_id hygon_nb_link_ids[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_HYGON, PCI_DEVICE_ID_AMD_17H_DF_F4) },
 	{}
 };
 
@@ -178,185 +58,42 @@ struct amd_northbridge *node_to_amd_nb(int node)
 }
 EXPORT_SYMBOL_GPL(node_to_amd_nb);
 
-static struct pci_dev *next_northbridge(struct pci_dev *dev,
-					const struct pci_device_id *ids)
-{
-	do {
-		dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev);
-		if (!dev)
-			break;
-	} while (!pci_match_id(ids, dev));
-	return dev;
-}
-
-/*
- * SMN accesses may fail in ways that are difficult to detect here in the called
- * functions amd_smn_read() and amd_smn_write(). Therefore, callers must do
- * their own checking based on what behavior they expect.
- *
- * For SMN reads, the returned value may be zero if the register is Read-as-Zero.
- * Or it may be a "PCI Error Response", e.g. all 0xFFs. The "PCI Error Response"
- * can be checked here, and a proper error code can be returned.
- *
- * But the Read-as-Zero response cannot be verified here. A value of 0 may be
- * correct in some cases, so callers must check that this correct is for the
- * register/fields they need.
- *
- * For SMN writes, success can be determined through a "write and read back"
- * However, this is not robust when done here.
- *
- * Possible issues:
- *
- * 1) Bits that are "Write-1-to-Clear". In this case, the read value should
- *    *not* match the write value.
- *
- * 2) Bits that are "Read-as-Zero"/"Writes-Ignored". This information cannot be
- *    known here.
- *
- * 3) Bits that are "Reserved / Set to 1". Ditto above.
- *
- * Callers of amd_smn_write() should do the "write and read back" check
- * themselves, if needed.
- *
- * For #1, they can see if their target bits got cleared.
- *
- * For #2 and #3, they can check if their target bits got set as intended.
- *
- * This matches what is done for RDMSR/WRMSR. As long as there's no #GP, then
- * the operation is considered a success, and the caller does their own
- * checking.
- */
-static int __amd_smn_rw(u16 node, u32 address, u32 *value, bool write)
-{
-	struct pci_dev *root;
-	int err = -ENODEV;
-
-	if (node >= amd_northbridges.num)
-		goto out;
-
-	root = node_to_amd_nb(node)->root;
-	if (!root)
-		goto out;
-
-	mutex_lock(&smn_mutex);
-
-	err = pci_write_config_dword(root, 0x60, address);
-	if (err) {
-		pr_warn("Error programming SMN address 0x%x.\n", address);
-		goto out_unlock;
-	}
-
-	err = (write ? pci_write_config_dword(root, 0x64, *value)
-		     : pci_read_config_dword(root, 0x64, value));
-
-out_unlock:
-	mutex_unlock(&smn_mutex);
-
-out:
-	return err;
-}
-
-int __must_check amd_smn_read(u16 node, u32 address, u32 *value)
-{
-	int err = __amd_smn_rw(node, address, value, false);
-
-	if (PCI_POSSIBLE_ERROR(*value)) {
-		err = -ENODEV;
-		*value = 0;
-	}
-
-	return err;
-}
-EXPORT_SYMBOL_GPL(amd_smn_read);
-
-int __must_check amd_smn_write(u16 node, u32 address, u32 value)
-{
-	return __amd_smn_rw(node, address, &value, true);
-}
-EXPORT_SYMBOL_GPL(amd_smn_write);
-
-
 static int amd_cache_northbridges(void)
 {
-	const struct pci_device_id *misc_ids = amd_nb_misc_ids;
-	const struct pci_device_id *link_ids = amd_nb_link_ids;
-	const struct pci_device_id *root_ids = amd_root_ids;
-	struct pci_dev *root, *misc, *link;
 	struct amd_northbridge *nb;
-	u16 roots_per_misc = 0;
-	u16 misc_count = 0;
-	u16 root_count = 0;
-	u16 i, j;
+	u16 i;
 
 	if (amd_northbridges.num)
 		return 0;
 
-	if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
-		root_ids = hygon_root_ids;
-		misc_ids = hygon_nb_misc_ids;
-		link_ids = hygon_nb_link_ids;
-	}
-
-	misc = NULL;
-	while ((misc = next_northbridge(misc, misc_ids)))
-		misc_count++;
+	amd_northbridges.num = amd_num_nodes();
 
-	if (!misc_count)
-		return -ENODEV;
-
-	root = NULL;
-	while ((root = next_northbridge(root, root_ids)))
-		root_count++;
-
-	if (root_count) {
-		roots_per_misc = root_count / misc_count;
-
-		/*
-		 * There should be _exactly_ N roots for each DF/SMN
-		 * interface.
-		 */
-		if (!roots_per_misc || (root_count % roots_per_misc)) {
-			pr_info("Unsupported AMD DF/PCI configuration found\n");
-			return -ENODEV;
-		}
-	}
-
-	nb = kcalloc(misc_count, sizeof(struct amd_northbridge), GFP_KERNEL);
+	nb = kcalloc(amd_northbridges.num, sizeof(struct amd_northbridge), GFP_KERNEL);
 	if (!nb)
 		return -ENOMEM;
 
 	amd_northbridges.nb = nb;
-	amd_northbridges.num = misc_count;
 
-	link = misc = root = NULL;
 	for (i = 0; i < amd_northbridges.num; i++) {
-		node_to_amd_nb(i)->root = root =
-			next_northbridge(root, root_ids);
-		node_to_amd_nb(i)->misc = misc =
-			next_northbridge(misc, misc_ids);
-		node_to_amd_nb(i)->link = link =
-			next_northbridge(link, link_ids);
+		node_to_amd_nb(i)->misc = amd_node_get_func(i, 3);
 
 		/*
-		 * If there are more PCI root devices than data fabric/
-		 * system management network interfaces, then the (N)
-		 * PCI roots per DF/SMN interface are functionally the
-		 * same (for DF/SMN access) and N-1 are redundant.  N-1
-		 * PCI roots should be skipped per DF/SMN interface so
-		 * the following DF/SMN interfaces get mapped to
-		 * correct PCI roots.
+		 * Each Northbridge must have a 'misc' device.
+		 * If not, then uninitialize everything.
 		 */
-		for (j = 1; j < roots_per_misc; j++)
-			root = next_northbridge(root, root_ids);
+		if (!node_to_amd_nb(i)->misc) {
+			amd_northbridges.num = 0;
+			kfree(nb);
+			return -ENODEV;
+		}
+
+		node_to_amd_nb(i)->link = amd_node_get_func(i, 4);
 	}
 
 	if (amd_gart_present())
 		amd_northbridges.flags |= AMD_NB_GART;
 
-	/*
-	 * Check for L3 cache presence.
-	 */
-	if (!cpuid_edx(0x80000006))
+	if (!cpuid_amd_hygon_has_l3_cache())
 		return 0;
 
 	/*
@@ -385,7 +122,6 @@ static int amd_cache_northbridges(void)
  */
 bool __init early_is_amd_nb(u32 device)
 {
-	const struct pci_device_id *misc_ids = amd_nb_misc_ids;
 	const struct pci_device_id *id;
 	u32 vendor = device & 0xffff;
 
@@ -393,11 +129,11 @@ bool __init early_is_amd_nb(u32 device)
 	    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
 		return false;
 
-	if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
-		misc_ids = hygon_nb_misc_ids;
+	if (cpu_feature_enabled(X86_FEATURE_ZEN))
+		return false;
 
 	device >>= 16;
-	for (id = misc_ids; id->vendor; id++)
+	for (id = amd_nb_misc_ids; id->vendor; id++)
 		if (vendor == id->vendor && device == id->device)
 			return true;
 	return false;
@@ -405,7 +141,6 @@ bool __init early_is_amd_nb(u32 device)
 
 struct resource *amd_get_mmconfig_range(struct resource *res)
 {
-	u32 address;
 	u64 base, msr;
 	unsigned int segn_busn_bits;
 
@@ -413,13 +148,11 @@ struct resource *amd_get_mmconfig_range(struct resource *res)
 	    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
 		return NULL;
 
-	/* assume all cpus from fam10h have mmconfig */
-	if (boot_cpu_data.x86 < 0x10)
+	/* Assume CPUs from Fam10h have mmconfig, although not all VMs do */
+	if (boot_cpu_data.x86 < 0x10 ||
+	    rdmsrq_safe(MSR_FAM10H_MMIO_CONF_BASE, &msr))
 		return NULL;
 
-	address = MSR_FAM10H_MMIO_CONF_BASE;
-	rdmsrl(address, msr);
-
 	/* mmconfig is not enabled */
 	if (!(msr & FAM10H_MMIO_CONF_ENABLE))
 		return NULL;
@@ -582,6 +315,10 @@ static __init void fix_erratum_688(void)
 
 static __init int init_amd_nbs(void)
 {
+	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
+	    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
+		return 0;
+
 	amd_cache_northbridges();
 	amd_cache_gart();
 
diff --git a/arch/x86/kernel/amd_node.c b/arch/x86/kernel/amd_node.c
new file mode 100644
index 000000000000..3d0a4768d603
--- /dev/null
+++ b/arch/x86/kernel/amd_node.c
@@ -0,0 +1,316 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * AMD Node helper functions and common defines
+ *
+ * Copyright (c) 2024, Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Author: Yazen Ghannam <Yazen.Ghannam@amd.com>
+ */
+
+#include <linux/debugfs.h>
+#include <asm/amd/node.h>
+
+/*
+ * AMD Nodes are a physical collection of I/O devices within an SoC. There can be one
+ * or more nodes per package.
+ *
+ * The nodes are software-visible through PCI config space. All nodes are enumerated
+ * on segment 0 bus 0. The device (slot) numbers range from 0x18 to 0x1F (maximum 8
+ * nodes) with 0x18 corresponding to node 0, 0x19 to node 1, etc. Each node can be a
+ * multi-function device.
+ *
+ * On legacy systems, these node devices represent integrated Northbridge functionality.
+ * On Zen-based systems, these node devices represent Data Fabric functionality.
+ *
+ * See "Configuration Space Accesses" section in BKDGs or
+ * "Processor x86 Core" -> "Configuration Space" section in PPRs.
+ */
+struct pci_dev *amd_node_get_func(u16 node, u8 func)
+{
+	if (node >= MAX_AMD_NUM_NODES)
+		return NULL;
+
+	return pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(AMD_NODE0_PCI_SLOT + node, func));
+}
+
+static struct pci_dev **amd_roots;
+
+/* Protect the PCI config register pairs used for SMN. */
+static DEFINE_MUTEX(smn_mutex);
+static bool smn_exclusive;
+
+#define SMN_INDEX_OFFSET	0x60
+#define SMN_DATA_OFFSET		0x64
+
+#define HSMP_INDEX_OFFSET	0xc4
+#define HSMP_DATA_OFFSET	0xc8
+
+/*
+ * SMN accesses may fail in ways that are difficult to detect here in the called
+ * functions amd_smn_read() and amd_smn_write(). Therefore, callers must do
+ * their own checking based on what behavior they expect.
+ *
+ * For SMN reads, the returned value may be zero if the register is Read-as-Zero.
+ * Or it may be a "PCI Error Response", e.g. all 0xFFs. The "PCI Error Response"
+ * can be checked here, and a proper error code can be returned.
+ *
+ * But the Read-as-Zero response cannot be verified here. A value of 0 may be
+ * correct in some cases, so callers must check that this correct is for the
+ * register/fields they need.
+ *
+ * For SMN writes, success can be determined through a "write and read back"
+ * However, this is not robust when done here.
+ *
+ * Possible issues:
+ *
+ * 1) Bits that are "Write-1-to-Clear". In this case, the read value should
+ *    *not* match the write value.
+ *
+ * 2) Bits that are "Read-as-Zero"/"Writes-Ignored". This information cannot be
+ *    known here.
+ *
+ * 3) Bits that are "Reserved / Set to 1". Ditto above.
+ *
+ * Callers of amd_smn_write() should do the "write and read back" check
+ * themselves, if needed.
+ *
+ * For #1, they can see if their target bits got cleared.
+ *
+ * For #2 and #3, they can check if their target bits got set as intended.
+ *
+ * This matches what is done for RDMSR/WRMSR. As long as there's no #GP, then
+ * the operation is considered a success, and the caller does their own
+ * checking.
+ */
+static int __amd_smn_rw(u8 i_off, u8 d_off, u16 node, u32 address, u32 *value, bool write)
+{
+	struct pci_dev *root;
+	int err = -ENODEV;
+
+	if (node >= amd_num_nodes())
+		return err;
+
+	root = amd_roots[node];
+	if (!root)
+		return err;
+
+	if (!smn_exclusive)
+		return err;
+
+	guard(mutex)(&smn_mutex);
+
+	err = pci_write_config_dword(root, i_off, address);
+	if (err) {
+		pr_warn("Error programming SMN address 0x%x.\n", address);
+		return pcibios_err_to_errno(err);
+	}
+
+	err = (write ? pci_write_config_dword(root, d_off, *value)
+		     : pci_read_config_dword(root, d_off, value));
+
+	return pcibios_err_to_errno(err);
+}
+
+int __must_check amd_smn_read(u16 node, u32 address, u32 *value)
+{
+	int err = __amd_smn_rw(SMN_INDEX_OFFSET, SMN_DATA_OFFSET, node, address, value, false);
+
+	if (PCI_POSSIBLE_ERROR(*value)) {
+		err = -ENODEV;
+		*value = 0;
+	}
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(amd_smn_read);
+
+int __must_check amd_smn_write(u16 node, u32 address, u32 value)
+{
+	return __amd_smn_rw(SMN_INDEX_OFFSET, SMN_DATA_OFFSET, node, address, &value, true);
+}
+EXPORT_SYMBOL_GPL(amd_smn_write);
+
+int __must_check amd_smn_hsmp_rdwr(u16 node, u32 address, u32 *value, bool write)
+{
+	return __amd_smn_rw(HSMP_INDEX_OFFSET, HSMP_DATA_OFFSET, node, address, value, write);
+}
+EXPORT_SYMBOL_GPL(amd_smn_hsmp_rdwr);
+
+static struct dentry *debugfs_dir;
+static u16 debug_node;
+static u32 debug_address;
+
+static ssize_t smn_node_write(struct file *file, const char __user *userbuf,
+			      size_t count, loff_t *ppos)
+{
+	u16 node;
+	int ret;
+
+	ret = kstrtou16_from_user(userbuf, count, 0, &node);
+	if (ret)
+		return ret;
+
+	if (node >= amd_num_nodes())
+		return -ENODEV;
+
+	debug_node = node;
+	return count;
+}
+
+static int smn_node_show(struct seq_file *m, void *v)
+{
+	seq_printf(m, "0x%08x\n", debug_node);
+	return 0;
+}
+
+static ssize_t smn_address_write(struct file *file, const char __user *userbuf,
+				 size_t count, loff_t *ppos)
+{
+	int ret;
+
+	ret = kstrtouint_from_user(userbuf, count, 0, &debug_address);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static int smn_address_show(struct seq_file *m, void *v)
+{
+	seq_printf(m, "0x%08x\n", debug_address);
+	return 0;
+}
+
+static int smn_value_show(struct seq_file *m, void *v)
+{
+	u32 val;
+	int ret;
+
+	ret = amd_smn_read(debug_node, debug_address, &val);
+	if (ret)
+		return ret;
+
+	seq_printf(m, "0x%08x\n", val);
+	return 0;
+}
+
+static ssize_t smn_value_write(struct file *file, const char __user *userbuf,
+			       size_t count, loff_t *ppos)
+{
+	u32 val;
+	int ret;
+
+	ret = kstrtouint_from_user(userbuf, count, 0, &val);
+	if (ret)
+		return ret;
+
+	add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
+
+	ret = amd_smn_write(debug_node, debug_address, val);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+DEFINE_SHOW_STORE_ATTRIBUTE(smn_node);
+DEFINE_SHOW_STORE_ATTRIBUTE(smn_address);
+DEFINE_SHOW_STORE_ATTRIBUTE(smn_value);
+
+static struct pci_dev *get_next_root(struct pci_dev *root)
+{
+	while ((root = pci_get_class(PCI_CLASS_BRIDGE_HOST << 8, root))) {
+		/* Root device is Device 0 Function 0. */
+		if (root->devfn)
+			continue;
+
+		if (root->vendor != PCI_VENDOR_ID_AMD &&
+		    root->vendor != PCI_VENDOR_ID_HYGON)
+			continue;
+
+		break;
+	}
+
+	return root;
+}
+
+static bool enable_dfs;
+
+static int __init amd_smn_enable_dfs(char *str)
+{
+	enable_dfs = true;
+	return 1;
+}
+__setup("amd_smn_debugfs_enable", amd_smn_enable_dfs);
+
+static int __init amd_smn_init(void)
+{
+	u16 count, num_roots, roots_per_node, node, num_nodes;
+	struct pci_dev *root;
+
+	if (!cpu_feature_enabled(X86_FEATURE_ZEN))
+		return 0;
+
+	guard(mutex)(&smn_mutex);
+
+	if (amd_roots)
+		return 0;
+
+	num_roots = 0;
+	root = NULL;
+	while ((root = get_next_root(root))) {
+		pci_dbg(root, "Reserving PCI config space\n");
+
+		/*
+		 * There are a few SMN index/data pairs and other registers
+		 * that shouldn't be accessed by user space. So reserve the
+		 * entire PCI config space for simplicity rather than covering
+		 * specific registers piecemeal.
+		 */
+		if (!pci_request_config_region_exclusive(root, 0, PCI_CFG_SPACE_SIZE, NULL)) {
+			pci_err(root, "Failed to reserve config space\n");
+			return -EEXIST;
+		}
+
+		num_roots++;
+	}
+
+	pr_debug("Found %d AMD root devices\n", num_roots);
+
+	if (!num_roots)
+		return -ENODEV;
+
+	num_nodes = amd_num_nodes();
+	amd_roots = kcalloc(num_nodes, sizeof(*amd_roots), GFP_KERNEL);
+	if (!amd_roots)
+		return -ENOMEM;
+
+	roots_per_node = num_roots / num_nodes;
+
+	count = 0;
+	node = 0;
+	root = NULL;
+	while (node < num_nodes && (root = get_next_root(root))) {
+		/* Use one root for each node and skip the rest. */
+		if (count++ % roots_per_node)
+			continue;
+
+		pci_dbg(root, "is root for AMD node %u\n", node);
+		amd_roots[node++] = root;
+	}
+
+	if (enable_dfs) {
+		debugfs_dir = debugfs_create_dir("amd_smn", arch_debugfs_dir);
+
+		debugfs_create_file("node",	0600, debugfs_dir, NULL, &smn_node_fops);
+		debugfs_create_file("address",	0600, debugfs_dir, NULL, &smn_address_fops);
+		debugfs_create_file("value",	0600, debugfs_dir, NULL, &smn_value_fops);
+	}
+
+	smn_exclusive = true;
+
+	return 0;
+}
+
+fs_initcall(amd_smn_init);
diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c
index 89c0c8a3fc7e..769321185a08 100644
--- a/arch/x86/kernel/aperture_64.c
+++ b/arch/x86/kernel/aperture_64.c
@@ -29,7 +29,7 @@
 #include <asm/gart.h>
 #include <asm/pci-direct.h>
 #include <asm/dma.h>
-#include <asm/amd_nb.h>
+#include <asm/amd/nb.h>
 #include <asm/x86_init.h>
 #include <linux/crash_dump.h>
 
diff --git a/arch/x86/kernel/apic/Makefile b/arch/x86/kernel/apic/Makefile
index 3bf0487cf3b7..581db89477f9 100644
--- a/arch/x86/kernel/apic/Makefile
+++ b/arch/x86/kernel/apic/Makefile
@@ -18,13 +18,11 @@ ifeq ($(CONFIG_X86_64),y)
 # APIC probe will depend on the listing order here
 obj-$(CONFIG_X86_NUMACHIP)	+= apic_numachip.o
 obj-$(CONFIG_X86_UV)		+= x2apic_uv_x.o
+obj-$(CONFIG_AMD_SECURE_AVIC)	+= x2apic_savic.o
 obj-$(CONFIG_X86_X2APIC)	+= x2apic_phys.o
 obj-$(CONFIG_X86_X2APIC)	+= x2apic_cluster.o
 obj-y				+= apic_flat_64.o
 endif
 
-# APIC probe will depend on the listing order here
-obj-$(CONFIG_X86_BIGSMP)	+= bigsmp_32.o
-
 # For 32bit, probe_32 need to be listed last
 obj-$(CONFIG_X86_LOCAL_APIC)	+= probe_$(BITS).o
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index c5fb28e6451a..d93f87f29d03 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -36,6 +36,7 @@
 #include <linux/dmi.h>
 #include <linux/smp.h>
 #include <linux/mm.h>
+#include <linux/kvm_types.h>
 
 #include <xen/xen.h>
 
@@ -59,6 +60,7 @@
 #include <asm/time.h>
 #include <asm/smp.h>
 #include <asm/mce.h>
+#include <asm/msr.h>
 #include <asm/tsc.h>
 #include <asm/hypervisor.h>
 #include <asm/cpu_device_id.h>
@@ -172,6 +174,7 @@ static struct resource lapic_resource = {
 	.flags = IORESOURCE_MEM | IORESOURCE_BUSY,
 };
 
+/* Measured in ticks per HZ. */
 unsigned int lapic_timer_period = 0;
 
 static void apic_pm_activate(void);
@@ -425,7 +428,7 @@ static int lapic_next_deadline(unsigned long delta,
 	weak_wrmsr_fence();
 
 	tsc = rdtsc();
-	wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
+	wrmsrq(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
 	return 0;
 }
 
@@ -449,7 +452,7 @@ static int lapic_timer_shutdown(struct clock_event_device *evt)
 	 * the timer _and_ zero the counter registers:
 	 */
 	if (v & APIC_LVT_TIMER_TSCDEADLINE)
-		wrmsrl(MSR_IA32_TSC_DEADLINE, 0);
+		wrmsrq(MSR_IA32_TSC_DEADLINE, 0);
 	else
 		apic_write(APIC_TMICT, 0);
 
@@ -509,19 +512,19 @@ static struct clock_event_device lapic_clockevent = {
 static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
 
 static const struct x86_cpu_id deadline_match[] __initconst = {
-	X86_MATCH_VFM_STEPPINGS(INTEL_HASWELL_X, X86_STEPPINGS(0x2, 0x2), 0x3a), /* EP */
-	X86_MATCH_VFM_STEPPINGS(INTEL_HASWELL_X, X86_STEPPINGS(0x4, 0x4), 0x0f), /* EX */
+	X86_MATCH_VFM_STEPS(INTEL_HASWELL_X,   0x2, 0x2, 0x3a), /* EP */
+	X86_MATCH_VFM_STEPS(INTEL_HASWELL_X,   0x4, 0x4, 0x0f), /* EX */
 
 	X86_MATCH_VFM(INTEL_BROADWELL_X,	0x0b000020),
 
-	X86_MATCH_VFM_STEPPINGS(INTEL_BROADWELL_D, X86_STEPPINGS(0x2, 0x2), 0x00000011),
-	X86_MATCH_VFM_STEPPINGS(INTEL_BROADWELL_D, X86_STEPPINGS(0x3, 0x3), 0x0700000e),
-	X86_MATCH_VFM_STEPPINGS(INTEL_BROADWELL_D, X86_STEPPINGS(0x4, 0x4), 0x0f00000c),
-	X86_MATCH_VFM_STEPPINGS(INTEL_BROADWELL_D, X86_STEPPINGS(0x5, 0x5), 0x0e000003),
+	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x2, 0x2, 0x00000011),
+	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x3, 0x3, 0x0700000e),
+	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x4, 0x4, 0x0f00000c),
+	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x5, 0x5, 0x0e000003),
 
-	X86_MATCH_VFM_STEPPINGS(INTEL_SKYLAKE_X, X86_STEPPINGS(0x3, 0x3), 0x01000136),
-	X86_MATCH_VFM_STEPPINGS(INTEL_SKYLAKE_X, X86_STEPPINGS(0x4, 0x4), 0x02000014),
-	X86_MATCH_VFM_STEPPINGS(INTEL_SKYLAKE_X, X86_STEPPINGS(0x5, 0xf), 0),
+	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X,   0x3, 0x3, 0x01000136),
+	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X,   0x4, 0x4, 0x02000014),
+	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X,   0x5, 0xf, 0),
 
 	X86_MATCH_VFM(INTEL_HASWELL,		0x22),
 	X86_MATCH_VFM(INTEL_HASWELL_L,		0x20),
@@ -591,6 +594,8 @@ static void setup_APIC_timer(void)
 						0xF, ~0UL);
 	} else
 		clockevents_register_device(levt);
+
+	apic_update_vector(smp_processor_id(), LOCAL_TIMER_VECTOR, true);
 }
 
 /*
@@ -789,6 +794,7 @@ static int __init calibrate_APIC_clock(void)
 {
 	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
 	u64 tsc_perj = 0, tsc_start = 0;
+	long delta_tsc_khz, bus_khz;
 	unsigned long jif_start;
 	unsigned long deltaj;
 	long delta, deltatsc;
@@ -891,14 +897,15 @@ static int __init calibrate_APIC_clock(void)
 	apic_pr_verbose("..... calibration result: %u\n", lapic_timer_period);
 
 	if (boot_cpu_has(X86_FEATURE_TSC)) {
-		apic_pr_verbose("..... CPU clock speed is %ld.%04ld MHz.\n",
-				(deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
-				(deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
+		delta_tsc_khz = (deltatsc * HZ) / (1000 * LAPIC_CAL_LOOPS);
+
+		apic_pr_verbose("..... CPU clock speed is %ld.%03ld MHz.\n",
+				delta_tsc_khz / 1000, delta_tsc_khz % 1000);
 	}
 
-	apic_pr_verbose("..... host bus clock speed is %u.%04u MHz.\n",
-			lapic_timer_period / (1000000 / HZ),
-			lapic_timer_period % (1000000 / HZ));
+	bus_khz = (long)lapic_timer_period * HZ / 1000;
+	apic_pr_verbose("..... host bus clock speed is %ld.%03ld MHz.\n",
+			bus_khz / 1000, bus_khz % 1000);
 
 	/*
 	 * Do a sanity check on the APIC calibration result
@@ -1167,6 +1174,9 @@ void disable_local_APIC(void)
 	if (!apic_accessible())
 		return;
 
+	if (apic->teardown)
+		apic->teardown();
+
 	apic_soft_disable();
 
 #ifdef CONFIG_X86_32
@@ -1371,8 +1381,6 @@ void __init apic_intr_mode_init(void)
 
 	x86_64_probe_apic();
 
-	x86_32_install_bigsmp();
-
 	if (x86_platform.apic_post_init)
 		x86_platform.apic_post_init();
 
@@ -1429,63 +1437,61 @@ union apic_ir {
 	u32		regs[APIC_IR_REGS];
 };
 
-static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr)
+static bool apic_check_and_eoi_isr(union apic_ir *isr)
 {
 	int i, bit;
 
-	/* Read the IRRs */
-	for (i = 0; i < APIC_IR_REGS; i++)
-		irr->regs[i] = apic_read(APIC_IRR + i * 0x10);
-
 	/* Read the ISRs */
 	for (i = 0; i < APIC_IR_REGS; i++)
 		isr->regs[i] = apic_read(APIC_ISR + i * 0x10);
 
+	/* If the ISR map empty, nothing to do here. */
+	if (bitmap_empty(isr->map, APIC_IR_BITS))
+		return true;
+
 	/*
-	 * If the ISR map is not empty. ACK the APIC and run another round
-	 * to verify whether a pending IRR has been unblocked and turned
-	 * into a ISR.
+	 * There can be multiple ISR bits set when a high priority
+	 * interrupt preempted a lower priority one. Issue an EOI for each
+	 * set bit. The priority traversal order does not matter as there
+	 * can't be new ISR bits raised at this point. What matters is that
+	 * an EOI is issued for each ISR bit.
 	 */
-	if (!bitmap_empty(isr->map, APIC_IR_BITS)) {
-		/*
-		 * There can be multiple ISR bits set when a high priority
-		 * interrupt preempted a lower priority one. Issue an ACK
-		 * per set bit.
-		 */
-		for_each_set_bit(bit, isr->map, APIC_IR_BITS)
-			apic_eoi();
-		return true;
-	}
+	for_each_set_bit(bit, isr->map, APIC_IR_BITS)
+		apic_eoi();
 
-	return !bitmap_empty(irr->map, APIC_IR_BITS);
+	/* Reread the ISRs, they should be empty now */
+	for (i = 0; i < APIC_IR_REGS; i++)
+		isr->regs[i] = apic_read(APIC_ISR + i * 0x10);
+
+	return bitmap_empty(isr->map, APIC_IR_BITS);
 }
 
 /*
- * After a crash, we no longer service the interrupts and a pending
- * interrupt from previous kernel might still have ISR bit set.
+ * If a CPU services an interrupt and crashes before issuing EOI to the
+ * local APIC, the corresponding ISR bit is still set when the crashing CPU
+ * jumps into a crash kernel. Read the ISR and issue an EOI for each set
+ * bit to acknowledge it as otherwise these slots would be locked forever
+ * waiting for an EOI.
  *
- * Most probably by now the CPU has serviced that pending interrupt and it
- * might not have done the apic_eoi() because it thought, interrupt
- * came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear
- * the ISR bit and cpu thinks it has already serviced the interrupt. Hence
- * a vector might get locked. It was noticed for timer irq (vector
- * 0x31). Issue an extra EOI to clear ISR.
+ * If there are pending bits in the IRR, then they won't be converted into
+ * ISR bits as the CPU has interrupts disabled. They will be delivered once
+ * the CPU enables interrupts and there is nothing which can prevent that.
  *
- * If there are pending IRR bits they turn into ISR bits after a higher
- * priority ISR bit has been acked.
+ * In the worst case this results in spurious interrupt warnings.
  */
-static void apic_pending_intr_clear(void)
+static void apic_clear_isr(void)
 {
-	union apic_ir irr, isr;
+	union apic_ir ir;
 	unsigned int i;
 
-	/* 512 loops are way oversized and give the APIC a chance to obey. */
-	for (i = 0; i < 512; i++) {
-		if (!apic_check_and_ack(&irr, &isr))
-			return;
-	}
-	/* Dump the IRR/ISR content if that failed */
-	pr_warn("APIC: Stale IRR: %256pb ISR: %256pb\n", irr.map, isr.map);
+	if (!apic_check_and_eoi_isr(&ir))
+		pr_warn("APIC: Stale ISR: %256pb\n", ir.map);
+
+	for (i = 0; i < APIC_IR_REGS; i++)
+		ir.regs[i] = apic_read(APIC_IRR + i * 0x10);
+
+	if (!bitmap_empty(ir.map, APIC_IR_BITS))
+		pr_warn("APIC: Stale IRR: %256pb\n", ir.map);
 }
 
 /**
@@ -1504,6 +1510,9 @@ static void setup_local_APIC(void)
 		return;
 	}
 
+	if (apic->setup)
+		apic->setup();
+
 	/*
 	 * If this comes from kexec/kcrash the APIC might be enabled in
 	 * SPIV. Soft disable it before doing further initialization.
@@ -1542,8 +1551,7 @@ static void setup_local_APIC(void)
 	value |= 0x10;
 	apic_write(APIC_TASKPRI, value);
 
-	/* Clear eventually stale ISR/IRR bits */
-	apic_pending_intr_clear();
+	apic_clear_isr();
 
 	/*
 	 * Now that we are all set up, enable the APIC
@@ -1674,7 +1682,6 @@ static __init void apic_read_boot_cpu_id(bool x2apic)
 		boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
 	}
 	topology_register_boot_apic(boot_cpu_physical_apicid);
-	x86_32_probe_bigsmp_early();
 }
 
 #ifdef CONFIG_X86_X2APIC
@@ -1697,7 +1704,7 @@ static bool x2apic_hw_locked(void)
 
 	x86_arch_cap_msr = x86_read_arch_cap_msr();
 	if (x86_arch_cap_msr & ARCH_CAP_XAPIC_DISABLE) {
-		rdmsrl(MSR_IA32_XAPIC_DISABLE_STATUS, msr);
+		rdmsrq(MSR_IA32_XAPIC_DISABLE_STATUS, msr);
 		return (msr & LEGACY_XAPIC_DISABLED);
 	}
 	return false;
@@ -1710,12 +1717,12 @@ static void __x2apic_disable(void)
 	if (!boot_cpu_has(X86_FEATURE_APIC))
 		return;
 
-	rdmsrl(MSR_IA32_APICBASE, msr);
+	rdmsrq(MSR_IA32_APICBASE, msr);
 	if (!(msr & X2APIC_ENABLE))
 		return;
 	/* Disable xapic and x2apic first and then reenable xapic mode */
-	wrmsrl(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
-	wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE);
+	wrmsrq(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
+	wrmsrq(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE);
 	printk_once(KERN_INFO "x2apic disabled\n");
 }
 
@@ -1723,10 +1730,10 @@ static void __x2apic_enable(void)
 {
 	u64 msr;
 
-	rdmsrl(MSR_IA32_APICBASE, msr);
+	rdmsrq(MSR_IA32_APICBASE, msr);
 	if (msr & X2APIC_ENABLE)
 		return;
-	wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE);
+	wrmsrq(MSR_IA32_APICBASE, msr | X2APIC_ENABLE);
 	printk_once(KERN_INFO "x2apic enabled\n");
 }
 
@@ -2014,8 +2021,8 @@ static bool __init detect_init_APIC(void)
 	case X86_VENDOR_HYGON:
 		break;
 	case X86_VENDOR_INTEL:
-		if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
-		    (boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC)))
+		if ((boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC)) ||
+		    boot_cpu_data.x86_vfm >= INTEL_PENTIUM_PRO)
 			break;
 		goto no_apic;
 	default:
@@ -2313,7 +2320,7 @@ u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid)
 		dest |= msg->arch_addr_hi.destid_8_31 << 8;
 	return dest;
 }
-EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid);
+EXPORT_SYMBOL_FOR_KVM(x86_msi_msg_get_destid);
 
 static void __init apic_bsp_up_setup(void)
 {
@@ -2378,7 +2385,7 @@ static struct {
 	unsigned int apic_cmci;
 } apic_pm_state;
 
-static int lapic_suspend(void)
+static int lapic_suspend(void *data)
 {
 	unsigned long flags;
 	int maxlvt;
@@ -2426,7 +2433,7 @@ static int lapic_suspend(void)
 	return 0;
 }
 
-static void lapic_resume(void)
+static void lapic_resume(void *data)
 {
 	unsigned int l, h;
 	unsigned long flags;
@@ -2501,11 +2508,15 @@ static void lapic_resume(void)
  * are needed on every CPU up until machine_halt/restart/poweroff.
  */
 
-static struct syscore_ops lapic_syscore_ops = {
+static const struct syscore_ops lapic_syscore_ops = {
 	.resume		= lapic_resume,
 	.suspend	= lapic_suspend,
 };
 
+static struct syscore lapic_syscore = {
+	.ops = &lapic_syscore_ops,
+};
+
 static void apic_pm_activate(void)
 {
 	apic_pm_state.active = 1;
@@ -2515,7 +2526,7 @@ static int __init init_lapic_sysfs(void)
 {
 	/* XXX: remove suspend/resume procs if !apic_pm_state.active? */
 	if (boot_cpu_has(X86_FEATURE_APIC))
-		register_syscore_ops(&lapic_syscore_ops);
+		register_syscore(&lapic_syscore);
 
 	return 0;
 }
@@ -2582,19 +2593,12 @@ int apic_is_clustered_box(void)
 /*
  * APIC command line parameters
  */
-static int __init setup_disableapic(char *arg)
+static int __init setup_nolapic(char *arg)
 {
 	apic_is_disabled = true;
 	setup_clear_cpu_cap(X86_FEATURE_APIC);
 	return 0;
 }
-early_param("disableapic", setup_disableapic);
-
-/* same as disableapic, for compatibility */
-static int __init setup_nolapic(char *arg)
-{
-	return setup_disableapic(arg);
-}
 early_param("nolapic", setup_nolapic);
 
 static int __init parse_lapic_timer_c2_ok(char *arg)
diff --git a/arch/x86/kernel/apic/apic_common.c b/arch/x86/kernel/apic/apic_common.c
index 9ef3be866832..2ed3b5c88c7f 100644
--- a/arch/x86/kernel/apic/apic_common.c
+++ b/arch/x86/kernel/apic/apic_common.c
@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: GPL-2.0
  */
 #include <linux/irq.h>
+#include <linux/kvm_types.h>
 #include <asm/apic.h>
 
 #include "local.h"
@@ -25,7 +26,7 @@ u32 default_cpu_present_to_apicid(int mps_cpu)
 	else
 		return BAD_APICID;
 }
-EXPORT_SYMBOL_GPL(default_cpu_present_to_apicid);
+EXPORT_SYMBOL_FOR_KVM(default_cpu_present_to_apicid);
 
 /*
  * Set up the logical destination ID when the APIC operates in logical
diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c
index b5bb7a2e8340..58abb941c45b 100644
--- a/arch/x86/kernel/apic/apic_noop.c
+++ b/arch/x86/kernel/apic/apic_noop.c
@@ -27,7 +27,13 @@ static void noop_send_IPI_allbutself(int vector) { }
 static void noop_send_IPI_all(int vector) { }
 static void noop_send_IPI_self(int vector) { }
 static void noop_apic_icr_write(u32 low, u32 id) { }
-static int noop_wakeup_secondary_cpu(u32 apicid, unsigned long start_eip) { return -1; }
+
+static int noop_wakeup_secondary_cpu(u32 apicid, unsigned long start_eip,
+	unsigned int cpu)
+{
+	return -1;
+}
+
 static u64 noop_apic_icr_read(void) { return 0; }
 static u32 noop_get_apic_id(u32 apicid) { return 0; }
 static void noop_apic_eoi(void) { }
diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index 16410f087b7a..5c5be2d58242 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -14,6 +14,7 @@
 #include <linux/init.h>
 #include <linux/pgtable.h>
 
+#include <asm/msr.h>
 #include <asm/numachip/numachip.h>
 #include <asm/numachip/numachip_csr.h>
 
@@ -31,7 +32,7 @@ static u32 numachip1_get_apic_id(u32 x)
 	unsigned int id = (x >> 24) & 0xff;
 
 	if (static_cpu_has(X86_FEATURE_NODEID_MSR)) {
-		rdmsrl(MSR_FAM10H_NODE_ID, value);
+		rdmsrq(MSR_FAM10H_NODE_ID, value);
 		id |= (value << 2) & 0xff00;
 	}
 
@@ -42,7 +43,7 @@ static u32 numachip2_get_apic_id(u32 x)
 {
 	u64 mcfg;
 
-	rdmsrl(MSR_FAM10H_MMIO_CONF_BASE, mcfg);
+	rdmsrq(MSR_FAM10H_MMIO_CONF_BASE, mcfg);
 	return ((mcfg >> (28 - 8)) & 0xfff00) | (x >> 24);
 }
 
@@ -56,7 +57,7 @@ static void numachip2_apic_icr_write(int apicid, unsigned int val)
 	numachip2_write32_lcsr(NUMACHIP2_APIC_ICR, (apicid << 12) | val);
 }
 
-static int numachip_wakeup_secondary(u32 phys_apicid, unsigned long start_rip)
+static int numachip_wakeup_secondary(u32 phys_apicid, unsigned long start_rip, unsigned int cpu)
 {
 	numachip_apic_icr_write(phys_apicid, APIC_DM_INIT);
 	numachip_apic_icr_write(phys_apicid, APIC_DM_STARTUP |
@@ -150,7 +151,7 @@ static void fixup_cpu_id(struct cpuinfo_x86 *c, int node)
 
 	/* Account for nodes per socket in multi-core-module processors */
 	if (boot_cpu_has(X86_FEATURE_NODEID_MSR)) {
-		rdmsrl(MSR_FAM10H_NODE_ID, val);
+		rdmsrq(MSR_FAM10H_NODE_ID, val);
 		nodes = ((val >> 3) & 7) + 1;
 	}
 
diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c
deleted file mode 100644
index 9285d500d5b4..000000000000
--- a/arch/x86/kernel/apic/bigsmp_32.c
+++ /dev/null
@@ -1,105 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * APIC driver for "bigsmp" xAPIC machines with more than 8 virtual CPUs.
- *
- * Drives the local APIC in "clustered mode".
- */
-#include <linux/cpumask.h>
-#include <linux/dmi.h>
-#include <linux/smp.h>
-
-#include <asm/apic.h>
-#include <asm/io_apic.h>
-
-#include "local.h"
-
-static u32 bigsmp_get_apic_id(u32 x)
-{
-	return (x >> 24) & 0xFF;
-}
-
-static void bigsmp_send_IPI_allbutself(int vector)
-{
-	default_send_IPI_mask_allbutself_phys(cpu_online_mask, vector);
-}
-
-static void bigsmp_send_IPI_all(int vector)
-{
-	default_send_IPI_mask_sequence_phys(cpu_online_mask, vector);
-}
-
-static int dmi_bigsmp; /* can be set by dmi scanners */
-
-static int hp_ht_bigsmp(const struct dmi_system_id *d)
-{
-	printk(KERN_NOTICE "%s detected: force use of apic=bigsmp\n", d->ident);
-	dmi_bigsmp = 1;
-
-	return 0;
-}
-
-
-static const struct dmi_system_id bigsmp_dmi_table[] = {
-	{ hp_ht_bigsmp, "HP ProLiant DL760 G2",
-		{	DMI_MATCH(DMI_BIOS_VENDOR, "HP"),
-			DMI_MATCH(DMI_BIOS_VERSION, "P44-"),
-		}
-	},
-
-	{ hp_ht_bigsmp, "HP ProLiant DL740",
-		{	DMI_MATCH(DMI_BIOS_VENDOR, "HP"),
-			DMI_MATCH(DMI_BIOS_VERSION, "P47-"),
-		}
-	},
-	{ } /* NULL entry stops DMI scanning */
-};
-
-static int probe_bigsmp(void)
-{
-	return dmi_check_system(bigsmp_dmi_table);
-}
-
-static struct apic apic_bigsmp __ro_after_init = {
-
-	.name				= "bigsmp",
-	.probe				= probe_bigsmp,
-
-	.dest_mode_logical		= false,
-
-	.disable_esr			= 1,
-
-	.cpu_present_to_apicid		= default_cpu_present_to_apicid,
-
-	.max_apic_id			= 0xFE,
-	.get_apic_id			= bigsmp_get_apic_id,
-
-	.calc_dest_apicid		= apic_default_calc_apicid,
-
-	.send_IPI			= default_send_IPI_single_phys,
-	.send_IPI_mask			= default_send_IPI_mask_sequence_phys,
-	.send_IPI_mask_allbutself	= NULL,
-	.send_IPI_allbutself		= bigsmp_send_IPI_allbutself,
-	.send_IPI_all			= bigsmp_send_IPI_all,
-	.send_IPI_self			= default_send_IPI_self,
-
-	.read				= native_apic_mem_read,
-	.write				= native_apic_mem_write,
-	.eoi				= native_apic_mem_eoi,
-	.icr_read			= native_apic_icr_read,
-	.icr_write			= native_apic_icr_write,
-	.wait_icr_idle			= apic_mem_wait_icr_idle,
-	.safe_wait_icr_idle		= apic_mem_wait_icr_idle_timeout,
-};
-
-bool __init apic_bigsmp_possible(bool cmdline_override)
-{
-	return apic == &apic_bigsmp || !cmdline_override;
-}
-
-void __init apic_bigsmp_force(void)
-{
-	if (apic != &apic_bigsmp)
-		apic_install_driver(&apic_bigsmp);
-}
-
-apic_driver(apic_bigsmp);
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 1029ea4ac8ba..28f934f05a85 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -1165,7 +1165,7 @@ static void io_apic_print_entries(unsigned int apic, unsigned int nr_entries)
 				 (entry.ir_index_15 << 15) | entry.ir_index_0_14, entry.ir_zero);
 		} else {
 			apic_dbg("%s, %s, D(%02X%02X), M(%1d)\n", buf,
-				 entry.dest_mode_logical ? "logical " : "physic	al",
+				 entry.dest_mode_logical ? "logical " : "physical",
 				 entry.virt_destid_8_14, entry.destid_0_7, entry.delivery_mode);
 		}
 	}
@@ -1486,7 +1486,7 @@ static void __init delay_with_tsc(void)
 	 * 1 GHz == 40 jiffies
 	 */
 	do {
-		rep_nop();
+		native_pause();
 		now = rdtsc();
 	} while ((now - start) < 40000000000ULL / HZ &&	time_before_eq(jiffies, end));
 }
@@ -1861,7 +1861,7 @@ static struct irq_chip ioapic_chip __read_mostly = {
 	.irq_set_affinity	= ioapic_set_affinity,
 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
 	.irq_get_irqchip_state	= ioapic_irq_get_chip_state,
-	.flags			= IRQCHIP_SKIP_SET_WAKE |
+	.flags			= IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MOVE_DEFERRED |
 				  IRQCHIP_AFFINITY_PRE_STARTUP,
 };
 
@@ -2225,7 +2225,7 @@ static int mp_irqdomain_create(int ioapic)
 
 	/* Handle device tree enumerated APICs proper */
 	if (cfg->dev) {
-		fn = of_node_to_fwnode(cfg->dev);
+		fn = of_fwnode_handle(cfg->dev);
 	} else {
 		fn = irq_domain_alloc_named_id_fwnode("IO-APIC", mpc_ioapic_id(ioapic));
 		if (!fn)
@@ -2308,7 +2308,12 @@ static void resume_ioapic_id(int ioapic_idx)
 	}
 }
 
-static void ioapic_resume(void)
+static int ioapic_suspend(void *data)
+{
+	return save_ioapic_entries();
+}
+
+static void ioapic_resume(void *data)
 {
 	int ioapic_idx;
 
@@ -2318,14 +2323,18 @@ static void ioapic_resume(void)
 	restore_ioapic_entries();
 }
 
-static struct syscore_ops ioapic_syscore_ops = {
-	.suspend	= save_ioapic_entries,
+static const struct syscore_ops ioapic_syscore_ops = {
+	.suspend	= ioapic_suspend,
 	.resume		= ioapic_resume,
 };
 
+static struct syscore ioapic_syscore = {
+	.ops = &ioapic_syscore_ops,
+};
+
 static int __init ioapic_init_ops(void)
 {
-	register_syscore_ops(&ioapic_syscore_ops);
+	register_syscore(&ioapic_syscore);
 
 	return 0;
 }
@@ -2503,9 +2512,7 @@ static struct resource * __init ioapic_setup_resources(void)
 	n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
 	n *= nr_ioapics;
 
-	mem = memblock_alloc(n, SMP_CACHE_BYTES);
-	if (!mem)
-		panic("%s: Failed to allocate %lu bytes\n", __func__, n);
+	mem = memblock_alloc_or_panic(n, SMP_CACHE_BYTES);
 	res = (void *)mem;
 
 	mem += sizeof(struct resource) * nr_ioapics;
@@ -2564,11 +2571,8 @@ void __init io_apic_init_mappings(void)
 #ifdef CONFIG_X86_32
 fake_ioapic_page:
 #endif
-			ioapic_phys = (unsigned long)memblock_alloc(PAGE_SIZE,
+			ioapic_phys = (unsigned long)memblock_alloc_or_panic(PAGE_SIZE,
 								    PAGE_SIZE);
-			if (!ioapic_phys)
-				panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
-				      __func__, PAGE_SIZE, PAGE_SIZE);
 			ioapic_phys = __pa(ioapic_phys);
 		}
 		io_apic_set_fixmap(idx, ioapic_phys);
@@ -2869,7 +2873,7 @@ int mp_irqdomain_alloc(struct irq_domain *domain, unsigned int virq,
 
 	ioapic = mp_irqdomain_ioapic_idx(domain);
 	pin = info->ioapic.pin;
-	if (irq_find_mapping(domain, (irq_hw_number_t)pin) > 0)
+	if (irq_resolve_mapping(domain, (irq_hw_number_t)pin))
 		return -EEXIST;
 
 	data = kzalloc(sizeof(*data), GFP_KERNEL);
diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c
index 5da693d633b7..98a57cb4aa86 100644
--- a/arch/x86/kernel/apic/ipi.c
+++ b/arch/x86/kernel/apic/ipi.c
@@ -3,6 +3,7 @@
 #include <linux/cpumask.h>
 #include <linux/delay.h>
 #include <linux/smp.h>
+#include <linux/string_choices.h>
 
 #include <asm/io_apic.h>
 
@@ -23,7 +24,7 @@ __setup("no_ipi_broadcast=", apic_ipi_shorthand);
 static int __init print_ipi_mode(void)
 {
 	pr_info("IPI shorthand broadcast: %s\n",
-		apic_ipi_shorthand_off ? "disabled" : "enabled");
+		str_disabled_enabled(apic_ipi_shorthand_off));
 	return 0;
 }
 late_initcall(print_ipi_mode);
@@ -287,34 +288,4 @@ void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
 	__default_send_IPI_dest_field(mask, vector, APIC_DEST_LOGICAL);
 	local_irq_restore(flags);
 }
-
-#ifdef CONFIG_SMP
-static int convert_apicid_to_cpu(u32 apic_id)
-{
-	int i;
-
-	for_each_possible_cpu(i) {
-		if (per_cpu(x86_cpu_to_apicid, i) == apic_id)
-			return i;
-	}
-	return -1;
-}
-
-int safe_smp_processor_id(void)
-{
-	u32 apicid;
-	int cpuid;
-
-	if (!boot_cpu_has(X86_FEATURE_APIC))
-		return 0;
-
-	apicid = read_apic_id();
-	if (apicid == BAD_APICID)
-		return 0;
-
-	cpuid = convert_apicid_to_cpu(apicid);
-
-	return cpuid >= 0 ? cpuid : 0;
-}
-#endif
 #endif
diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h
index 842fe28496be..bdcf609eb283 100644
--- a/arch/x86/kernel/apic/local.h
+++ b/arch/x86/kernel/apic/local.h
@@ -65,17 +65,4 @@ void default_send_IPI_self(int vector);
 void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, int vector);
 void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask, int vector);
 void default_send_IPI_mask_logical(const struct cpumask *mask, int vector);
-void x86_32_probe_bigsmp_early(void);
-void x86_32_install_bigsmp(void);
-#else
-static inline void x86_32_probe_bigsmp_early(void) { }
-static inline void x86_32_install_bigsmp(void) { }
-#endif
-
-#ifdef CONFIG_X86_BIGSMP
-bool apic_bigsmp_possible(bool cmdline_selected);
-void apic_bigsmp_force(void);
-#else
-static inline bool apic_bigsmp_possible(bool cmdline_selected) { return false; };
-static inline void apic_bigsmp_force(void) { }
 #endif
diff --git a/arch/x86/kernel/apic/msi.c b/arch/x86/kernel/apic/msi.c
index 340769242dea..66bc5d3e79db 100644
--- a/arch/x86/kernel/apic/msi.c
+++ b/arch/x86/kernel/apic/msi.c
@@ -214,6 +214,7 @@ static bool x86_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
 		if (WARN_ON_ONCE(domain != real_parent))
 			return false;
 		info->chip->irq_set_affinity = msi_set_affinity;
+		info->chip->flags |= IRQCHIP_MOVE_DEFERRED;
 		break;
 	case DOMAIN_BUS_DMAR:
 	case DOMAIN_BUS_AMDVI:
@@ -315,7 +316,7 @@ static struct irq_chip dmar_msi_controller = {
 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
 	.irq_compose_msi_msg	= dmar_msi_compose_msg,
 	.irq_write_msi_msg	= dmar_msi_write_msg,
-	.flags			= IRQCHIP_SKIP_SET_WAKE |
+	.flags			= IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MOVE_DEFERRED |
 				  IRQCHIP_AFFINITY_PRE_STARTUP,
 };
 
diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c
index f75ee345c02d..87bc9e7ca5d6 100644
--- a/arch/x86/kernel/apic/probe_32.c
+++ b/arch/x86/kernel/apic/probe_32.c
@@ -93,35 +93,6 @@ static int __init parse_apic(char *arg)
 }
 early_param("apic", parse_apic);
 
-void __init x86_32_probe_bigsmp_early(void)
-{
-	if (nr_cpu_ids <= 8 || xen_pv_domain())
-		return;
-
-	if (IS_ENABLED(CONFIG_X86_BIGSMP)) {
-		switch (boot_cpu_data.x86_vendor) {
-		case X86_VENDOR_INTEL:
-			if (!APIC_XAPIC(boot_cpu_apic_version))
-				break;
-			/* P4 and above */
-			fallthrough;
-		case X86_VENDOR_HYGON:
-		case X86_VENDOR_AMD:
-			if (apic_bigsmp_possible(cmdline_apic))
-				return;
-			break;
-		}
-	}
-	pr_info("Limiting to 8 possible CPUs\n");
-	set_nr_cpu_ids(8);
-}
-
-void __init x86_32_install_bigsmp(void)
-{
-	if (nr_cpu_ids > 8 && !xen_pv_domain())
-		apic_bigsmp_force();
-}
-
 void __init x86_32_probe_apic(void)
 {
 	if (!cmdline_apic) {
diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c
index 557318145038..bddc54465399 100644
--- a/arch/x86/kernel/apic/vector.c
+++ b/arch/x86/kernel/apic/vector.c
@@ -134,13 +134,20 @@ static void apic_update_irq_cfg(struct irq_data *irqd, unsigned int vector,
 
 	apicd->hw_irq_cfg.vector = vector;
 	apicd->hw_irq_cfg.dest_apicid = apic->calc_dest_apicid(cpu);
+
+	apic_update_vector(cpu, vector, true);
+
 	irq_data_update_effective_affinity(irqd, cpumask_of(cpu));
-	trace_vector_config(irqd->irq, vector, cpu,
-			    apicd->hw_irq_cfg.dest_apicid);
+	trace_vector_config(irqd->irq, vector, cpu, apicd->hw_irq_cfg.dest_apicid);
 }
 
-static void apic_update_vector(struct irq_data *irqd, unsigned int newvec,
-			       unsigned int newcpu)
+static void apic_free_vector(unsigned int cpu, unsigned int vector, bool managed)
+{
+	apic_update_vector(cpu, vector, false);
+	irq_matrix_free(vector_matrix, cpu, vector, managed);
+}
+
+static void chip_data_update(struct irq_data *irqd, unsigned int newvec, unsigned int newcpu)
 {
 	struct apic_chip_data *apicd = apic_chip_data(irqd);
 	struct irq_desc *desc = irq_data_to_desc(irqd);
@@ -174,8 +181,7 @@ static void apic_update_vector(struct irq_data *irqd, unsigned int newvec,
 		apicd->prev_cpu = apicd->cpu;
 		WARN_ON_ONCE(apicd->cpu == newcpu);
 	} else {
-		irq_matrix_free(vector_matrix, apicd->cpu, apicd->vector,
-				managed);
+		apic_free_vector(apicd->cpu, apicd->vector, managed);
 	}
 
 setnew:
@@ -183,6 +189,7 @@ setnew:
 	apicd->cpu = newcpu;
 	BUG_ON(!IS_ERR_OR_NULL(per_cpu(vector_irq, newcpu)[newvec]));
 	per_cpu(vector_irq, newcpu)[newvec] = desc;
+	apic_update_irq_cfg(irqd, newvec, newcpu);
 }
 
 static void vector_assign_managed_shutdown(struct irq_data *irqd)
@@ -260,8 +267,7 @@ assign_vector_locked(struct irq_data *irqd, const struct cpumask *dest)
 	trace_vector_alloc(irqd->irq, vector, resvd, vector);
 	if (vector < 0)
 		return vector;
-	apic_update_vector(irqd, vector, cpu);
-	apic_update_irq_cfg(irqd, vector, cpu);
+	chip_data_update(irqd, vector, cpu);
 
 	return 0;
 }
@@ -337,8 +343,8 @@ assign_managed_vector(struct irq_data *irqd, const struct cpumask *dest)
 	trace_vector_alloc_managed(irqd->irq, vector, vector);
 	if (vector < 0)
 		return vector;
-	apic_update_vector(irqd, vector, cpu);
-	apic_update_irq_cfg(irqd, vector, cpu);
+	chip_data_update(irqd, vector, cpu);
+
 	return 0;
 }
 
@@ -357,7 +363,7 @@ static void clear_irq_vector(struct irq_data *irqd)
 			   apicd->prev_cpu);
 
 	per_cpu(vector_irq, apicd->cpu)[vector] = VECTOR_SHUTDOWN;
-	irq_matrix_free(vector_matrix, apicd->cpu, vector, managed);
+	apic_free_vector(apicd->cpu, vector, managed);
 	apicd->vector = 0;
 
 	/* Clean up move in progress */
@@ -366,7 +372,7 @@ static void clear_irq_vector(struct irq_data *irqd)
 		return;
 
 	per_cpu(vector_irq, apicd->prev_cpu)[vector] = VECTOR_SHUTDOWN;
-	irq_matrix_free(vector_matrix, apicd->prev_cpu, vector, managed);
+	apic_free_vector(apicd->prev_cpu, vector, managed);
 	apicd->prev_vector = 0;
 	apicd->move_in_progress = 0;
 	hlist_del_init(&apicd->clist);
@@ -712,8 +718,8 @@ int __init arch_probe_nr_irqs(void)
 {
 	int nr;
 
-	if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
-		nr_irqs = NR_VECTORS * nr_cpu_ids;
+	if (irq_get_nr_irqs() > NR_VECTORS * nr_cpu_ids)
+		irq_set_nr_irqs(NR_VECTORS * nr_cpu_ids);
 
 	nr = (gsi_top + nr_legacy_irqs()) + 8 * nr_cpu_ids;
 #if defined(CONFIG_PCI_MSI)
@@ -725,8 +731,8 @@ int __init arch_probe_nr_irqs(void)
 	else
 		nr += gsi_top * 16;
 #endif
-	if (nr < nr_irqs)
-		nr_irqs = nr;
+	if (nr < irq_get_nr_irqs())
+		irq_set_nr_irqs(nr);
 
 	/*
 	 * We don't know if PIC is present at this point so we need to do
@@ -799,7 +805,7 @@ int __init arch_early_irq_init(void)
 	x86_vector_domain = irq_domain_create_tree(fn, &x86_vector_domain_ops,
 						   NULL);
 	BUG_ON(x86_vector_domain == NULL);
-	irq_set_default_host(x86_vector_domain);
+	irq_set_default_domain(x86_vector_domain);
 
 	BUG_ON(!alloc_cpumask_var(&vector_searchmask, GFP_KERNEL));
 
@@ -864,7 +870,7 @@ void lapic_offline(void)
 	__vector_cleanup(cl, false);
 
 	irq_matrix_offline(vector_matrix);
-	WARN_ON_ONCE(try_to_del_timer_sync(&cl->timer) < 0);
+	WARN_ON_ONCE(timer_delete_sync_try(&cl->timer) < 0);
 	WARN_ON_ONCE(!hlist_empty(&cl->head));
 
 	unlock_vector_lock();
@@ -888,8 +894,109 @@ static int apic_set_affinity(struct irq_data *irqd,
 	return err ? err : IRQ_SET_MASK_OK;
 }
 
+static void free_moved_vector(struct apic_chip_data *apicd)
+{
+	unsigned int vector = apicd->prev_vector;
+	unsigned int cpu = apicd->prev_cpu;
+	bool managed = apicd->is_managed;
+
+	/*
+	 * Managed interrupts are usually not migrated away
+	 * from an online CPU, but CPU isolation 'managed_irq'
+	 * can make that happen.
+	 * 1) Activation does not take the isolation into account
+	 *    to keep the code simple
+	 * 2) Migration away from an isolated CPU can happen when
+	 *    a non-isolated CPU which is in the calculated
+	 *    affinity mask comes online.
+	 */
+	trace_vector_free_moved(apicd->irq, cpu, vector, managed);
+	apic_free_vector(cpu, vector, managed);
+	per_cpu(vector_irq, cpu)[vector] = VECTOR_UNUSED;
+	hlist_del_init(&apicd->clist);
+	apicd->prev_vector = 0;
+	apicd->move_in_progress = 0;
+}
+
+/*
+ * Called from fixup_irqs() with @desc->lock held and interrupts disabled.
+ */
+static void apic_force_complete_move(struct irq_data *irqd)
+{
+	unsigned int cpu = smp_processor_id();
+	struct apic_chip_data *apicd;
+	unsigned int vector;
+
+	guard(raw_spinlock)(&vector_lock);
+	apicd = apic_chip_data(irqd);
+	if (!apicd)
+		return;
+
+	/*
+	 * If prev_vector is empty or the descriptor is neither currently
+	 * nor previously on the outgoing CPU no action required.
+	 */
+	vector = apicd->prev_vector;
+	if (!vector || (apicd->cpu != cpu && apicd->prev_cpu != cpu))
+		return;
+
+	/*
+	 * This is tricky. If the cleanup of the old vector has not been
+	 * done yet, then the following setaffinity call will fail with
+	 * -EBUSY. This can leave the interrupt in a stale state.
+	 *
+	 * All CPUs are stuck in stop machine with interrupts disabled so
+	 * calling __irq_complete_move() would be completely pointless.
+	 *
+	 * 1) The interrupt is in move_in_progress state. That means that we
+	 *    have not seen an interrupt since the io_apic was reprogrammed to
+	 *    the new vector.
+	 *
+	 * 2) The interrupt has fired on the new vector, but the cleanup IPIs
+	 *    have not been processed yet.
+	 */
+	if (apicd->move_in_progress) {
+		/*
+		 * In theory there is a race:
+		 *
+		 * set_ioapic(new_vector) <-- Interrupt is raised before update
+		 *			      is effective, i.e. it's raised on
+		 *			      the old vector.
+		 *
+		 * So if the target cpu cannot handle that interrupt before
+		 * the old vector is cleaned up, we get a spurious interrupt
+		 * and in the worst case the ioapic irq line becomes stale.
+		 *
+		 * But in case of cpu hotplug this should be a non issue
+		 * because if the affinity update happens right before all
+		 * cpus rendezvous in stop machine, there is no way that the
+		 * interrupt can be blocked on the target cpu because all cpus
+		 * loops first with interrupts enabled in stop machine, so the
+		 * old vector is not yet cleaned up when the interrupt fires.
+		 *
+		 * So the only way to run into this issue is if the delivery
+		 * of the interrupt on the apic/system bus would be delayed
+		 * beyond the point where the target cpu disables interrupts
+		 * in stop machine. I doubt that it can happen, but at least
+		 * there is a theoretical chance. Virtualization might be
+		 * able to expose this, but AFAICT the IOAPIC emulation is not
+		 * as stupid as the real hardware.
+		 *
+		 * Anyway, there is nothing we can do about that at this point
+		 * w/o refactoring the whole fixup_irq() business completely.
+		 * We print at least the irq number and the old vector number,
+		 * so we have the necessary information when a problem in that
+		 * area arises.
+		 */
+		pr_warn("IRQ fixup: irq %d move in progress, old vector %d\n",
+			irqd->irq, vector);
+	}
+	free_moved_vector(apicd);
+}
+
 #else
-# define apic_set_affinity	NULL
+# define apic_set_affinity		NULL
+# define apic_force_complete_move	NULL
 #endif
 
 static int apic_retrigger_irq(struct irq_data *irqd)
@@ -923,39 +1030,16 @@ static void x86_vector_msi_compose_msg(struct irq_data *data,
 }
 
 static struct irq_chip lapic_controller = {
-	.name			= "APIC",
-	.irq_ack		= apic_ack_edge,
-	.irq_set_affinity	= apic_set_affinity,
-	.irq_compose_msi_msg	= x86_vector_msi_compose_msg,
-	.irq_retrigger		= apic_retrigger_irq,
+	.name				= "APIC",
+	.irq_ack			= apic_ack_edge,
+	.irq_set_affinity		= apic_set_affinity,
+	.irq_compose_msi_msg		= x86_vector_msi_compose_msg,
+	.irq_force_complete_move	= apic_force_complete_move,
+	.irq_retrigger			= apic_retrigger_irq,
 };
 
 #ifdef CONFIG_SMP
 
-static void free_moved_vector(struct apic_chip_data *apicd)
-{
-	unsigned int vector = apicd->prev_vector;
-	unsigned int cpu = apicd->prev_cpu;
-	bool managed = apicd->is_managed;
-
-	/*
-	 * Managed interrupts are usually not migrated away
-	 * from an online CPU, but CPU isolation 'managed_irq'
-	 * can make that happen.
-	 * 1) Activation does not take the isolation into account
-	 *    to keep the code simple
-	 * 2) Migration away from an isolated CPU can happen when
-	 *    a non-isolated CPU which is in the calculated
-	 *    affinity mask comes online.
-	 */
-	trace_vector_free_moved(apicd->irq, cpu, vector, managed);
-	irq_matrix_free(vector_matrix, cpu, vector, managed);
-	per_cpu(vector_irq, cpu)[vector] = VECTOR_UNUSED;
-	hlist_del_init(&apicd->clist);
-	apicd->prev_vector = 0;
-	apicd->move_in_progress = 0;
-}
-
 static void __vector_cleanup(struct vector_cleanup *cl, bool check_irr)
 {
 	struct apic_chip_data *apicd;
@@ -1068,99 +1152,6 @@ void irq_complete_move(struct irq_cfg *cfg)
 		__vector_schedule_cleanup(apicd);
 }
 
-/*
- * Called from fixup_irqs() with @desc->lock held and interrupts disabled.
- */
-void irq_force_complete_move(struct irq_desc *desc)
-{
-	unsigned int cpu = smp_processor_id();
-	struct apic_chip_data *apicd;
-	struct irq_data *irqd;
-	unsigned int vector;
-
-	/*
-	 * The function is called for all descriptors regardless of which
-	 * irqdomain they belong to. For example if an IRQ is provided by
-	 * an irq_chip as part of a GPIO driver, the chip data for that
-	 * descriptor is specific to the irq_chip in question.
-	 *
-	 * Check first that the chip_data is what we expect
-	 * (apic_chip_data) before touching it any further.
-	 */
-	irqd = irq_domain_get_irq_data(x86_vector_domain,
-				       irq_desc_get_irq(desc));
-	if (!irqd)
-		return;
-
-	raw_spin_lock(&vector_lock);
-	apicd = apic_chip_data(irqd);
-	if (!apicd)
-		goto unlock;
-
-	/*
-	 * If prev_vector is empty or the descriptor is neither currently
-	 * nor previously on the outgoing CPU no action required.
-	 */
-	vector = apicd->prev_vector;
-	if (!vector || (apicd->cpu != cpu && apicd->prev_cpu != cpu))
-		goto unlock;
-
-	/*
-	 * This is tricky. If the cleanup of the old vector has not been
-	 * done yet, then the following setaffinity call will fail with
-	 * -EBUSY. This can leave the interrupt in a stale state.
-	 *
-	 * All CPUs are stuck in stop machine with interrupts disabled so
-	 * calling __irq_complete_move() would be completely pointless.
-	 *
-	 * 1) The interrupt is in move_in_progress state. That means that we
-	 *    have not seen an interrupt since the io_apic was reprogrammed to
-	 *    the new vector.
-	 *
-	 * 2) The interrupt has fired on the new vector, but the cleanup IPIs
-	 *    have not been processed yet.
-	 */
-	if (apicd->move_in_progress) {
-		/*
-		 * In theory there is a race:
-		 *
-		 * set_ioapic(new_vector) <-- Interrupt is raised before update
-		 *			      is effective, i.e. it's raised on
-		 *			      the old vector.
-		 *
-		 * So if the target cpu cannot handle that interrupt before
-		 * the old vector is cleaned up, we get a spurious interrupt
-		 * and in the worst case the ioapic irq line becomes stale.
-		 *
-		 * But in case of cpu hotplug this should be a non issue
-		 * because if the affinity update happens right before all
-		 * cpus rendezvous in stop machine, there is no way that the
-		 * interrupt can be blocked on the target cpu because all cpus
-		 * loops first with interrupts enabled in stop machine, so the
-		 * old vector is not yet cleaned up when the interrupt fires.
-		 *
-		 * So the only way to run into this issue is if the delivery
-		 * of the interrupt on the apic/system bus would be delayed
-		 * beyond the point where the target cpu disables interrupts
-		 * in stop machine. I doubt that it can happen, but at least
-		 * there is a theoretical chance. Virtualization might be
-		 * able to expose this, but AFAICT the IOAPIC emulation is not
-		 * as stupid as the real hardware.
-		 *
-		 * Anyway, there is nothing we can do about that at this point
-		 * w/o refactoring the whole fixup_irq() business completely.
-		 * We print at least the irq number and the old vector number,
-		 * so we have the necessary information when a problem in that
-		 * area arises.
-		 */
-		pr_warn("IRQ fixup: irq %d move in progress, old vector %d\n",
-			irqd->irq, vector);
-	}
-	free_moved_vector(apicd);
-unlock:
-	raw_spin_unlock(&vector_lock);
-}
-
 #ifdef CONFIG_HOTPLUG_CPU
 /*
  * Note, this is not accurate accounting, but at least good enough to
diff --git a/arch/x86/kernel/apic/x2apic_savic.c b/arch/x86/kernel/apic/x2apic_savic.c
new file mode 100644
index 000000000000..dbc5678bc3b6
--- /dev/null
+++ b/arch/x86/kernel/apic/x2apic_savic.c
@@ -0,0 +1,428 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AMD Secure AVIC Support (SEV-SNP Guests)
+ *
+ * Copyright (C) 2024 Advanced Micro Devices, Inc.
+ *
+ * Author: Neeraj Upadhyay <Neeraj.Upadhyay@amd.com>
+ */
+
+#include <linux/cc_platform.h>
+#include <linux/cpumask.h>
+#include <linux/percpu-defs.h>
+#include <linux/align.h>
+
+#include <asm/apic.h>
+#include <asm/sev.h>
+
+#include "local.h"
+
+struct secure_avic_page {
+	u8 regs[PAGE_SIZE];
+} __aligned(PAGE_SIZE);
+
+static struct secure_avic_page __percpu *savic_page __ro_after_init;
+
+static int savic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
+{
+	return x2apic_enabled() && cc_platform_has(CC_ATTR_SNP_SECURE_AVIC);
+}
+
+static inline void *get_reg_bitmap(unsigned int cpu, unsigned int offset)
+{
+	return &per_cpu_ptr(savic_page, cpu)->regs[offset];
+}
+
+static inline void update_vector(unsigned int cpu, unsigned int offset,
+				 unsigned int vector, bool set)
+{
+	void *bitmap = get_reg_bitmap(cpu, offset);
+
+	if (set)
+		apic_set_vector(vector, bitmap);
+	else
+		apic_clear_vector(vector, bitmap);
+}
+
+#define SAVIC_ALLOWED_IRR	0x204
+
+/*
+ * When Secure AVIC is enabled, RDMSR/WRMSR of the APIC registers
+ * result in #VC exception (for non-accelerated register accesses)
+ * with VMEXIT_AVIC_NOACCEL error code. The #VC exception handler
+ * can read/write the x2APIC register in the guest APIC backing page.
+ *
+ * Since doing this would increase the latency of accessing x2APIC
+ * registers, instead of doing RDMSR/WRMSR based accesses and
+ * handling the APIC register reads/writes in the #VC exception handler,
+ * the read() and write() callbacks directly read/write the APIC register
+ * from/to the vCPU's APIC backing page.
+ */
+static u32 savic_read(u32 reg)
+{
+	void *ap = this_cpu_ptr(savic_page);
+
+	switch (reg) {
+	case APIC_LVTT:
+	case APIC_TMICT:
+	case APIC_TMCCT:
+	case APIC_TDCR:
+	case APIC_LVTTHMR:
+	case APIC_LVTPC:
+	case APIC_LVT0:
+	case APIC_LVT1:
+	case APIC_LVTERR:
+		return savic_ghcb_msr_read(reg);
+	case APIC_ID:
+	case APIC_LVR:
+	case APIC_TASKPRI:
+	case APIC_ARBPRI:
+	case APIC_PROCPRI:
+	case APIC_LDR:
+	case APIC_SPIV:
+	case APIC_ESR:
+	case APIC_EFEAT:
+	case APIC_ECTRL:
+	case APIC_SEOI:
+	case APIC_IER:
+	case APIC_EILVTn(0) ... APIC_EILVTn(3):
+		return apic_get_reg(ap, reg);
+	case APIC_ICR:
+		return (u32)apic_get_reg64(ap, reg);
+	case APIC_ISR ... APIC_ISR + 0x70:
+	case APIC_TMR ... APIC_TMR + 0x70:
+		if (WARN_ONCE(!IS_ALIGNED(reg, 16),
+			      "APIC register read offset 0x%x not aligned at 16 bytes", reg))
+			return 0;
+		return apic_get_reg(ap, reg);
+	/* IRR and ALLOWED_IRR offset range */
+	case APIC_IRR ... APIC_IRR + 0x74:
+		/*
+		 * Valid APIC_IRR/SAVIC_ALLOWED_IRR registers are at 16 bytes strides from
+		 * their respective base offset. APIC_IRRs are in the range
+		 *
+		 * (0x200, 0x210,  ..., 0x270)
+		 *
+		 * while the SAVIC_ALLOWED_IRR range starts 4 bytes later, in the range
+		 *
+		 * (0x204, 0x214, ..., 0x274).
+		 *
+		 * Filter out everything else.
+		 */
+		if (WARN_ONCE(!(IS_ALIGNED(reg, 16) ||
+				IS_ALIGNED(reg - 4, 16)),
+			      "Misaligned APIC_IRR/ALLOWED_IRR APIC register read offset 0x%x", reg))
+			return 0;
+		return apic_get_reg(ap, reg);
+	default:
+		pr_err("Error reading unknown Secure AVIC reg offset 0x%x\n", reg);
+		return 0;
+	}
+}
+
+#define SAVIC_NMI_REQ		0x278
+
+/*
+ * On WRMSR to APIC_SELF_IPI register by the guest, Secure AVIC hardware
+ * updates the APIC_IRR in the APIC backing page of the vCPU. In addition,
+ * hardware evaluates the new APIC_IRR update for interrupt injection to
+ * the vCPU. So, self IPIs are hardware-accelerated.
+ */
+static inline void self_ipi_reg_write(unsigned int vector)
+{
+	native_apic_msr_write(APIC_SELF_IPI, vector);
+}
+
+static void send_ipi_dest(unsigned int cpu, unsigned int vector, bool nmi)
+{
+	if (nmi)
+		apic_set_reg(per_cpu_ptr(savic_page, cpu), SAVIC_NMI_REQ, 1);
+	else
+		update_vector(cpu, APIC_IRR, vector, true);
+}
+
+static void send_ipi_allbut(unsigned int vector, bool nmi)
+{
+	unsigned int cpu, src_cpu;
+
+	guard(irqsave)();
+
+	src_cpu = raw_smp_processor_id();
+
+	for_each_cpu(cpu, cpu_online_mask) {
+		if (cpu == src_cpu)
+			continue;
+		send_ipi_dest(cpu, vector, nmi);
+	}
+}
+
+static inline void self_ipi(unsigned int vector, bool nmi)
+{
+	u32 icr_low = APIC_SELF_IPI | vector;
+
+	if (nmi)
+		icr_low |= APIC_DM_NMI;
+
+	native_x2apic_icr_write(icr_low, 0);
+}
+
+static void savic_icr_write(u32 icr_low, u32 icr_high)
+{
+	unsigned int dsh, vector;
+	u64 icr_data;
+	bool nmi;
+
+	dsh = icr_low & APIC_DEST_ALLBUT;
+	vector = icr_low & APIC_VECTOR_MASK;
+	nmi = ((icr_low & APIC_DM_FIXED_MASK) == APIC_DM_NMI);
+
+	switch (dsh) {
+	case APIC_DEST_SELF:
+		self_ipi(vector, nmi);
+		break;
+	case APIC_DEST_ALLINC:
+		self_ipi(vector, nmi);
+		fallthrough;
+	case APIC_DEST_ALLBUT:
+		send_ipi_allbut(vector, nmi);
+		break;
+	default:
+		send_ipi_dest(icr_high, vector, nmi);
+		break;
+	}
+
+	icr_data = ((u64)icr_high) << 32 | icr_low;
+	if (dsh != APIC_DEST_SELF)
+		savic_ghcb_msr_write(APIC_ICR, icr_data);
+	apic_set_reg64(this_cpu_ptr(savic_page), APIC_ICR, icr_data);
+}
+
+static void savic_write(u32 reg, u32 data)
+{
+	void *ap = this_cpu_ptr(savic_page);
+
+	switch (reg) {
+	case APIC_LVTT:
+	case APIC_TMICT:
+	case APIC_TDCR:
+	case APIC_LVT0:
+	case APIC_LVT1:
+	case APIC_LVTTHMR:
+	case APIC_LVTPC:
+	case APIC_LVTERR:
+		savic_ghcb_msr_write(reg, data);
+		break;
+	case APIC_TASKPRI:
+	case APIC_EOI:
+	case APIC_SPIV:
+	case SAVIC_NMI_REQ:
+	case APIC_ESR:
+	case APIC_ECTRL:
+	case APIC_SEOI:
+	case APIC_IER:
+	case APIC_EILVTn(0) ... APIC_EILVTn(3):
+		apic_set_reg(ap, reg, data);
+		break;
+	case APIC_ICR:
+		savic_icr_write(data, 0);
+		break;
+	case APIC_SELF_IPI:
+		self_ipi_reg_write(data);
+		break;
+	/* ALLOWED_IRR offsets are writable */
+	case SAVIC_ALLOWED_IRR ... SAVIC_ALLOWED_IRR + 0x70:
+		if (IS_ALIGNED(reg - 4, 16)) {
+			apic_set_reg(ap, reg, data);
+			break;
+		}
+		fallthrough;
+	default:
+		pr_err("Error writing unknown Secure AVIC reg offset 0x%x\n", reg);
+	}
+}
+
+static void send_ipi(u32 dest, unsigned int vector, unsigned int dsh)
+{
+	unsigned int icr_low;
+
+	icr_low = __prepare_ICR(dsh, vector, APIC_DEST_PHYSICAL);
+	savic_icr_write(icr_low, dest);
+}
+
+static void savic_send_ipi(int cpu, int vector)
+{
+	u32 dest = per_cpu(x86_cpu_to_apicid, cpu);
+
+	send_ipi(dest, vector, 0);
+}
+
+static void send_ipi_mask(const struct cpumask *mask, unsigned int vector, bool excl_self)
+{
+	unsigned int cpu, this_cpu;
+
+	guard(irqsave)();
+
+	this_cpu = raw_smp_processor_id();
+
+	for_each_cpu(cpu, mask) {
+		if (excl_self && cpu == this_cpu)
+			continue;
+		send_ipi(per_cpu(x86_cpu_to_apicid, cpu), vector, 0);
+	}
+}
+
+static void savic_send_ipi_mask(const struct cpumask *mask, int vector)
+{
+	send_ipi_mask(mask, vector, false);
+}
+
+static void savic_send_ipi_mask_allbutself(const struct cpumask *mask, int vector)
+{
+	send_ipi_mask(mask, vector, true);
+}
+
+static void savic_send_ipi_allbutself(int vector)
+{
+	send_ipi(0, vector, APIC_DEST_ALLBUT);
+}
+
+static void savic_send_ipi_all(int vector)
+{
+	send_ipi(0, vector, APIC_DEST_ALLINC);
+}
+
+static void savic_send_ipi_self(int vector)
+{
+	self_ipi_reg_write(vector);
+}
+
+static void savic_update_vector(unsigned int cpu, unsigned int vector, bool set)
+{
+	update_vector(cpu, SAVIC_ALLOWED_IRR, vector, set);
+}
+
+static void savic_eoi(void)
+{
+	unsigned int cpu;
+	int vec;
+
+	cpu = raw_smp_processor_id();
+	vec = apic_find_highest_vector(get_reg_bitmap(cpu, APIC_ISR));
+	if (WARN_ONCE(vec == -1, "EOI write while no active interrupt in APIC_ISR"))
+		return;
+
+	/* Is level-triggered interrupt? */
+	if (apic_test_vector(vec, get_reg_bitmap(cpu, APIC_TMR))) {
+		update_vector(cpu, APIC_ISR, vec, false);
+		/*
+		 * Propagate the EOI write to the hypervisor for level-triggered
+		 * interrupts. Return to the guest from GHCB protocol event takes
+		 * care of re-evaluating interrupt state.
+		 */
+		savic_ghcb_msr_write(APIC_EOI, 0);
+	} else {
+		/*
+		 * Hardware clears APIC_ISR and re-evaluates the interrupt state
+		 * to determine if there is any pending interrupt which can be
+		 * delivered to CPU.
+		 */
+		native_apic_msr_eoi();
+	}
+}
+
+static void savic_teardown(void)
+{
+	/* Disable Secure AVIC */
+	native_wrmsrq(MSR_AMD64_SAVIC_CONTROL, 0);
+	savic_unregister_gpa(NULL);
+}
+
+static void savic_setup(void)
+{
+	void *ap = this_cpu_ptr(savic_page);
+	enum es_result res;
+	unsigned long gpa;
+
+	/*
+	 * Before Secure AVIC is enabled, APIC MSR reads are intercepted.
+	 * APIC_ID MSR read returns the value from the hypervisor.
+	 */
+	apic_set_reg(ap, APIC_ID, native_apic_msr_read(APIC_ID));
+
+	gpa = __pa(ap);
+
+	/*
+	 * The NPT entry for a vCPU's APIC backing page must always be
+	 * present when the vCPU is running in order for Secure AVIC to
+	 * function. A VMEXIT_BUSY is returned on VMRUN and the vCPU cannot
+	 * be resumed if the NPT entry for the APIC backing page is not
+	 * present. Notify GPA of the vCPU's APIC backing page to the
+	 * hypervisor by calling savic_register_gpa(). Before executing
+	 * VMRUN, the hypervisor makes use of this information to make sure
+	 * the APIC backing page is mapped in NPT.
+	 */
+	res = savic_register_gpa(gpa);
+	if (res != ES_OK)
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL);
+
+	native_wrmsrq(MSR_AMD64_SAVIC_CONTROL,
+		      gpa | MSR_AMD64_SAVIC_EN | MSR_AMD64_SAVIC_ALLOWEDNMI);
+}
+
+static int savic_probe(void)
+{
+	if (!cc_platform_has(CC_ATTR_SNP_SECURE_AVIC))
+		return 0;
+
+	if (!x2apic_mode) {
+		pr_err("Secure AVIC enabled in non x2APIC mode\n");
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL);
+		/* unreachable */
+	}
+
+	savic_page = alloc_percpu(struct secure_avic_page);
+	if (!savic_page)
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL);
+
+	return 1;
+}
+
+static struct apic apic_x2apic_savic __ro_after_init = {
+
+	.name				= "secure avic x2apic",
+	.probe				= savic_probe,
+	.acpi_madt_oem_check		= savic_acpi_madt_oem_check,
+	.setup				= savic_setup,
+	.teardown			= savic_teardown,
+
+	.dest_mode_logical		= false,
+
+	.disable_esr			= 0,
+
+	.cpu_present_to_apicid		= default_cpu_present_to_apicid,
+
+	.max_apic_id			= UINT_MAX,
+	.x2apic_set_max_apicid		= true,
+	.get_apic_id			= x2apic_get_apic_id,
+
+	.calc_dest_apicid		= apic_default_calc_apicid,
+
+	.send_IPI			= savic_send_ipi,
+	.send_IPI_mask			= savic_send_ipi_mask,
+	.send_IPI_mask_allbutself	= savic_send_ipi_mask_allbutself,
+	.send_IPI_allbutself		= savic_send_ipi_allbutself,
+	.send_IPI_all			= savic_send_ipi_all,
+	.send_IPI_self			= savic_send_ipi_self,
+
+	.nmi_to_offline_cpu		= true,
+
+	.read				= savic_read,
+	.write				= savic_write,
+	.eoi				= savic_eoi,
+	.icr_read			= native_x2apic_icr_read,
+	.icr_write			= savic_icr_write,
+
+	.update_vector			= savic_update_vector,
+};
+
+apic_driver(apic_x2apic_savic);
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index 7fef504ca508..15209f220e1f 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -667,7 +667,7 @@ static __init void build_uv_gr_table(void)
 	}
 }
 
-static int uv_wakeup_secondary(u32 phys_apicid, unsigned long start_rip)
+static int uv_wakeup_secondary(u32 phys_apicid, unsigned long start_rip, unsigned int cpu)
 {
 	unsigned long val;
 	int pnode;
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index a98020bf31bb..25fcde525c68 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -33,6 +33,14 @@
 
 static void __used common(void)
 {
+	OFFSET(CPUINFO_x86, cpuinfo_x86, x86);
+	OFFSET(CPUINFO_x86_vendor, cpuinfo_x86, x86_vendor);
+	OFFSET(CPUINFO_x86_model, cpuinfo_x86, x86_model);
+	OFFSET(CPUINFO_x86_stepping, cpuinfo_x86, x86_stepping);
+	OFFSET(CPUINFO_cpuid_level, cpuinfo_x86, cpuid_level);
+	OFFSET(CPUINFO_x86_capability, cpuinfo_x86, x86_capability);
+	OFFSET(CPUINFO_x86_vendor_id, cpuinfo_x86, x86_vendor_id);
+
 	BLANK();
 	OFFSET(TASK_threadsp, task_struct, thread.sp);
 #ifdef CONFIG_STACKPROTECTOR
@@ -94,6 +102,7 @@ static void __used common(void)
 
 	BLANK();
 	DEFINE(PTREGS_SIZE, sizeof(struct pt_regs));
+	OFFSET(C_PTREGS_SIZE, pt_regs, orig_ax);
 
 	/* TLB state for the entry code */
 	OFFSET(TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask);
@@ -107,11 +116,6 @@ static void __used common(void)
 	OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
 	OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);
 	OFFSET(TSS_sp2, tss_struct, x86_tss.sp2);
-	OFFSET(X86_top_of_stack, pcpu_hot, top_of_stack);
-	OFFSET(X86_current_task, pcpu_hot, current_task);
-#ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING
-	OFFSET(X86_call_depth, pcpu_hot, call_depth);
-#endif
 #if IS_ENABLED(CONFIG_CRYPTO_ARIA_AESNI_AVX_X86_64)
 	/* Offset for fields in aria_ctx */
 	BLANK();
@@ -120,4 +124,7 @@ static void __used common(void)
 	OFFSET(ARIA_CTX_rounds, aria_ctx, rounds);
 #endif
 
+	BLANK();
+	DEFINE(ALT_INSTR_SIZE,	sizeof(struct alt_instr));
+	DEFINE(EXTABLE_SIZE,	sizeof(struct exception_table_entry));
 }
diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
index 2b411cd00a4e..e0a292db97b2 100644
--- a/arch/x86/kernel/asm-offsets_32.c
+++ b/arch/x86/kernel/asm-offsets_32.c
@@ -12,15 +12,6 @@ void foo(void);
 
 void foo(void)
 {
-	OFFSET(CPUINFO_x86, cpuinfo_x86, x86);
-	OFFSET(CPUINFO_x86_vendor, cpuinfo_x86, x86_vendor);
-	OFFSET(CPUINFO_x86_model, cpuinfo_x86, x86_model);
-	OFFSET(CPUINFO_x86_stepping, cpuinfo_x86, x86_stepping);
-	OFFSET(CPUINFO_cpuid_level, cpuinfo_x86, cpuid_level);
-	OFFSET(CPUINFO_x86_capability, cpuinfo_x86, x86_capability);
-	OFFSET(CPUINFO_x86_vendor_id, cpuinfo_x86, x86_vendor_id);
-	BLANK();
-
 	OFFSET(PT_EBX, pt_regs, bx);
 	OFFSET(PT_ECX, pt_regs, cx);
 	OFFSET(PT_EDX, pt_regs, dx);
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index bb65371ea9df..590b6cd0eac0 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -54,11 +54,5 @@ int main(void)
 	BLANK();
 #undef ENTRY
 
-	BLANK();
-
-#ifdef CONFIG_STACKPROTECTOR
-	OFFSET(FIXED_stack_canary, fixed_percpu_data, stack_canary);
-	BLANK();
-#endif
 	return 0;
 }
diff --git a/arch/x86/kernel/bootflag.c b/arch/x86/kernel/bootflag.c
index 3fed7ae58b60..73274d76ce16 100644
--- a/arch/x86/kernel/bootflag.c
+++ b/arch/x86/kernel/bootflag.c
@@ -8,6 +8,7 @@
 #include <linux/string.h>
 #include <linux/spinlock.h>
 #include <linux/acpi.h>
+#include <linux/bitops.h>
 #include <asm/io.h>
 
 #include <linux/mc146818rtc.h>
@@ -20,27 +21,13 @@
 
 int sbf_port __initdata = -1;	/* set via acpi_boot_init() */
 
-static int __init parity(u8 v)
-{
-	int x = 0;
-	int i;
-
-	for (i = 0; i < 8; i++) {
-		x ^= (v & 1);
-		v >>= 1;
-	}
-
-	return x;
-}
-
 static void __init sbf_write(u8 v)
 {
 	unsigned long flags;
 
 	if (sbf_port != -1) {
-		v &= ~SBF_PARITY;
-		if (!parity(v))
-			v |= SBF_PARITY;
+		if (!parity8(v))
+			v ^= SBF_PARITY;
 
 		printk(KERN_INFO "Simple Boot Flag at 0x%x set to 0x%x\n",
 			sbf_port, v);
@@ -66,14 +53,14 @@ static u8 __init sbf_read(void)
 	return v;
 }
 
-static int __init sbf_value_valid(u8 v)
+static bool __init sbf_value_valid(u8 v)
 {
 	if (v & SBF_RESERVED)		/* Reserved bits */
-		return 0;
-	if (!parity(v))
-		return 0;
+		return false;
+	if (!parity8(v))
+		return false;
 
-	return 1;
+	return true;
 }
 
 static int __init sbf_init(void)
diff --git a/arch/x86/kernel/callthunks.c b/arch/x86/kernel/callthunks.c
index 465647456753..a951333c5995 100644
--- a/arch/x86/kernel/callthunks.c
+++ b/arch/x86/kernel/callthunks.c
@@ -98,11 +98,10 @@ static inline bool within_module_coretext(void *addr)
 #ifdef CONFIG_MODULES
 	struct module *mod;
 
-	preempt_disable();
+	guard(rcu)();
 	mod = __module_address((unsigned long)addr);
 	if (mod && within_module_core((unsigned long)addr, mod))
 		ret = true;
-	preempt_enable();
 #endif
 	return ret;
 }
@@ -139,14 +138,15 @@ static bool skip_addr(void *dest)
 		return true;
 #endif
 #ifdef CONFIG_KEXEC_CORE
+# ifdef CONFIG_X86_64
+	if (dest >= (void *)__relocate_kernel_start &&
+	    dest < (void *)__relocate_kernel_end)
+		return true;
+# else
 	if (dest >= (void *)relocate_kernel &&
 	    dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
 		return true;
-#endif
-#ifdef CONFIG_XEN
-	if (dest >= (void *)hypercall_page &&
-	    dest < (void*)hypercall_page + PAGE_SIZE)
-		return true;
+# endif
 #endif
 	return false;
 }
@@ -185,7 +185,7 @@ static void *patch_dest(void *dest, bool direct)
 	u8 *pad = dest - tsize;
 
 	memcpy(insn_buff, skl_call_thunk_template, tsize);
-	apply_relocation(insn_buff, pad, tsize, skl_call_thunk_template, tsize);
+	text_poke_apply_relocation(insn_buff, pad, tsize, skl_call_thunk_template, tsize);
 
 	/* Already patched? */
 	if (!bcmp(pad, insn_buff, tsize))
@@ -239,21 +239,10 @@ patch_call_sites(s32 *start, s32 *end, const struct core_text *ct)
 }
 
 static __init_or_module void
-patch_alt_call_sites(struct alt_instr *start, struct alt_instr *end,
-		     const struct core_text *ct)
-{
-	struct alt_instr *a;
-
-	for (a = start; a < end; a++)
-		patch_call((void *)&a->instr_offset + a->instr_offset, ct);
-}
-
-static __init_or_module void
 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)
 {
 	prdbg("Patching call sites %s\n", ct->name);
 	patch_call_sites(cs->call_start, cs->call_end, ct);
-	patch_alt_call_sites(cs->alt_start, cs->alt_end, ct);
 	prdbg("Patching call sites done%s\n", ct->name);
 }
 
@@ -262,8 +251,6 @@ void __init callthunks_patch_builtin_calls(void)
 	struct callthunk_sites cs = {
 		.call_start	= __call_sites,
 		.call_end	= __call_sites_end,
-		.alt_start	= __alt_instructions,
-		.alt_end	= __alt_instructions_end
 	};
 
 	if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
@@ -307,7 +294,7 @@ static bool is_callthunk(void *addr)
 	pad = (void *)(dest - tmpl_size);
 
 	memcpy(insn_buff, skl_call_thunk_template, tmpl_size);
-	apply_relocation(insn_buff, pad, tmpl_size, skl_call_thunk_template, tmpl_size);
+	text_poke_apply_relocation(insn_buff, pad, tmpl_size, skl_call_thunk_template, tmpl_size);
 
 	return !bcmp(pad, insn_buff, tmpl_size);
 }
@@ -325,7 +312,7 @@ int x86_call_depth_emit_accounting(u8 **pprog, void *func, void *ip)
 		return 0;
 
 	memcpy(insn_buff, skl_call_thunk_template, tmpl_size);
-	apply_relocation(insn_buff, ip, tmpl_size, skl_call_thunk_template, tmpl_size);
+	text_poke_apply_relocation(insn_buff, ip, tmpl_size, skl_call_thunk_template, tmpl_size);
 
 	memcpy(*pprog, insn_buff, tmpl_size);
 	*pprog += tmpl_size;
diff --git a/arch/x86/kernel/cet.c b/arch/x86/kernel/cet.c
index d2c732a34e5d..99444409c026 100644
--- a/arch/x86/kernel/cet.c
+++ b/arch/x86/kernel/cet.c
@@ -2,6 +2,7 @@
 
 #include <linux/ptrace.h>
 #include <asm/bugs.h>
+#include <asm/msr.h>
 #include <asm/traps.h>
 
 enum cp_error_code {
@@ -55,7 +56,7 @@ static void do_user_cp_fault(struct pt_regs *regs, unsigned long error_code)
 	 * will be whatever is live in userspace. So read the SSP before enabling
 	 * interrupts so locking the fpregs to do it later is not required.
 	 */
-	rdmsrl(MSR_IA32_PL3_SSP, ssp);
+	rdmsrq(MSR_IA32_PL3_SSP, ssp);
 
 	cond_local_irq_enable(regs);
 
@@ -81,6 +82,34 @@ static void do_user_cp_fault(struct pt_regs *regs, unsigned long error_code)
 
 static __ro_after_init bool ibt_fatal = true;
 
+/*
+ * By definition, all missing-ENDBRANCH #CPs are a result of WFE && !ENDBR.
+ *
+ * For the kernel IBT no ENDBR selftest where #CPs are deliberately triggered,
+ * the WFE state of the interrupted context needs to be cleared to let execution
+ * continue.  Otherwise when the CPU resumes from the instruction that just
+ * caused the previous #CP, another missing-ENDBRANCH #CP is raised and the CPU
+ * enters a dead loop.
+ *
+ * This is not a problem with IDT because it doesn't preserve WFE and IRET doesn't
+ * set WFE.  But FRED provides space on the entry stack (in an expanded CS area)
+ * to save and restore the WFE state, thus the WFE state is no longer clobbered,
+ * so software must clear it.
+ */
+static void ibt_clear_fred_wfe(struct pt_regs *regs)
+{
+	/*
+	 * No need to do any FRED checks.
+	 *
+	 * For IDT event delivery, the high-order 48 bits of CS are pushed
+	 * as 0s into the stack, and later IRET ignores these bits.
+	 *
+	 * For FRED, a test to check if fred_cs.wfe is set would be dropped
+	 * by compilers.
+	 */
+	regs->fred_cs.wfe = 0;
+}
+
 static void do_kernel_cp_fault(struct pt_regs *regs, unsigned long error_code)
 {
 	if ((error_code & CP_EC) != CP_ENDBR) {
@@ -90,6 +119,7 @@ static void do_kernel_cp_fault(struct pt_regs *regs, unsigned long error_code)
 
 	if (unlikely(regs->ip == (unsigned long)&ibt_selftest_noendbr)) {
 		regs->ax = 0;
+		ibt_clear_fred_wfe(regs);
 		return;
 	}
 
@@ -97,6 +127,7 @@ static void do_kernel_cp_fault(struct pt_regs *regs, unsigned long error_code)
 	if (!ibt_fatal) {
 		printk(KERN_DEFAULT CUT_HERE);
 		__warn(__FILE__, __LINE__, (void *)regs->ip, TAINT_WARN, regs, NULL);
+		ibt_clear_fred_wfe(regs);
 		return;
 	}
 	BUG();
diff --git a/arch/x86/kernel/cfi.c b/arch/x86/kernel/cfi.c
index e6bf78fac146..638eb5c933e0 100644
--- a/arch/x86/kernel/cfi.c
+++ b/arch/x86/kernel/cfi.c
@@ -27,7 +27,7 @@ static bool decode_cfi_insn(struct pt_regs *regs, unsigned long *target,
 	 * for indirect call checks:
 	 *
 	 *   movl    -<id>, %r10d       ; 6 bytes
-	 *   addl    -4(%reg), %r10d    ; 4 bytes
+	 *   addl    -<pos>(%reg), %r10d; 4 bytes
 	 *   je      .Ltmp1             ; 2 bytes
 	 *   ud2                        ; <- regs->ip
 	 *   .Ltmp1:
@@ -67,16 +67,30 @@ static bool decode_cfi_insn(struct pt_regs *regs, unsigned long *target,
  */
 enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
 {
-v>+#define CAN3_DIV	47
-+#define CAN3_GATE	48
-+#define SDH0_MUX	49
-+#define SDH0_GATE	50
-+#define SDH1_MUX	51
-+#define SDH1_GATE	52
-+#define NAND_GATE	53
-+#define USBD_GATE	54
-+#define USBH_GATE	55
-+#define HUSBH0_GATE	56
-+#define HUSBH1_GATE	57
-+#define GFX_MUX		58
-+#define GFX_GATE	59
-+#define VC8K_GATE	60
-+#define DCU_MUX		61
-+#define DCU_GATE	62
-+#define DCUP_DIV	63
-+#define EMAC0_GATE	64
-+#define EMAC1_GATE	65
-+#define CCAP0_MUX	66
-+#define CCAP0_DIV	67
-+#define CCAP0_GATE	68
-+#define CCAP1_MUX	69
-+#define CCAP1_DIV	70
-+#define CCAP1_GATE	71
-+#define PDMA0_GATE	72
-+#define PDMA1_GATE	73
-+#define PDMA2_GATE	74
-+#define PDMA3_GATE	75
-+#define WH0_GATE	76
-+#define WH1_GATE	77
-+#define HWS_GATE	78
-+#define EBI_GATE	79
-+#define SRAM0_GATE	80
-+#define SRAM1_GATE	81
-+#define ROM_GATE	82
-+#define TRA_GATE	83
-+#define DBG_MUX		84
-+#define DBG_GATE	85
-+#define CKO_MUX		86
-+#define CKO_DIV		87
-+#define CKO_GATE	88
-+#define GTMR_GATE	89
-+#define GPA_GATE	90
-+#define GPB_GATE	91
-+#define GPC_GATE	92
-+#define GPD_GATE	93
-+#define GPE_GATE	94
-+#define GPF_GATE	95
-+#define GPG_GATE	96
-+#define GPH_GATE	97
-+#define GPI_GATE	98
-+#define GPJ_GATE	99
-+#define GPK_GATE	100
-+#define GPL_GATE	101
-+#define GPM_GATE	102
-+#define GPN_GATE	103
-+/* APB peripheral clocks */
-+#define TMR0_MUX	104
-+#define TMR0_GATE	105
-+#define TMR1_MUX	106
-+#define TMR1_GATE	107
-+#define TMR2_MUX	108
-+#define TMR2_GATE	109
-+#define TMR3_MUX	110
-+#define TMR3_GATE	111
-+#define TMR4_MUX	112
-+#define TMR4_GATE	113
-+#define TMR5_MUX	114
-+#define TMR5_GATE	115
-+#define TMR6_MUX	116
-+#define TMR6_GATE	117
-+#define TMR7_MUX	118
-+#define TMR7_GATE	119
-+#define TMR8_MUX	120
-+#define TMR8_GATE	121
-+#define TMR9_MUX	122
-+#define TMR9_GATE	123
-+#define TMR10_MUX	124
-+#define TMR10_GATE	125
-+#define TMR11_MUX	126
-+#define TMR11_GATE	127
-+#define UART0_MUX	128
-+#define UART0_DIV	129
-+#define UART0_GATE	130
-+#define UART1_MUX	131
-+#define UART1_DIV	132
-+#define UART1_GATE	133
-+#define UART2_MUX	134
-+#define UART2_DIV	135
-+#define UART2_GATE	136
-+#define UART3_MUX	137
-+#define UART3_DIV	138
-+#define UART3_GATE	139
-+#define UART4_MUX	140
-+#define UART4_DIV	141
-+#define UART4_GATE	142
-+#define UART5_MUX	143
-+#define UART5_DIV	144
-+#define UART5_GATE	145
-+#define UART6_MUX	146
-+#define UART6_DIV	147
-+#define UART6_GATE	148
-+#define UART7_MUX	149
-+#define UART7_DIV	150
-+#define UART7_GATE	151
-+#define UART8_MUX	152
-+#define UART8_DIV	153
-+#define UART8_GATE	154
-+#define UART9_MUX	155
-+#define UART9_DIV	156
-+#define UART9_GATE	157
-+#define UART10_MUX	158
-+#define UART10_DIV	159
-+#define UART10_GATE	160
-+#define UART11_MUX	161
-+#define UART11_DIV	162
-+#define UART11_GATE	163
-+#define UART12_MUX	164
-+#define UART12_DIV	165
-+#define UART12_GATE	166
-+#define UART13_MUX	167
-+#define UART13_DIV	168
-+#define UART13_GATE	169
-+#define UART14_MUX	170
-+#define UART14_DIV	171
-+#define UART14_GATE	172
-+#define UART15_MUX	173
-+#define UART15_DIV	174
-+#define UART15_GATE	175
-+#define UART16_MUX	176
-+#define UART16_DIV	177
-+#define UART16_GATE	178
-+#define RTC_GATE	179
-+#define DDR_GATE	180
-+#define KPI_MUX		181
-+#define KPI_DIV		182
-+#define KPI_GATE	183
-+#define I2C0_GATE	184
-+#define I2C1_GATE	185
-+#define I2C2_GATE	186
-+#define I2C3_GATE	187
-+#define I2C4_GATE	188
-+#define I2C5_GATE	189
-+#define QSPI0_MUX	190
-+#define QSPI0_GATE	191
-+#define QSPI1_MUX	192
-+#define QSPI1_GATE	193
-+#define SMC0_MUX	194
-+#define SMC0_DIV	195
-+#define SMC0_GATE	196
-+#define SMC1_MUX	197
-+#define SMC1_DIV	198
-+#define SMC1_GATE	199
-+#define WDT0_MUX	200
-+#define WDT0_GATE	201
-+#define WDT1_MUX	202
-+#define WDT1_GATE	203
-+#define WDT2_MUX	204
-+#define WDT2_GATE	205
-+#define WWDT0_MUX	206
-+#define WWDT1_MUX	207
-+#define WWDT2_MUX	208
-+#define EPWM0_GATE	209
-+#define EPWM1_GATE	210
-+#define EPWM2_GATE	211
-+#define I2S0_MUX	212
-+#define I2S0_GATE	213
-+#define I2S1_MUX	214
-+#define I2S1_GATE	215
-+#define SSMCC_GATE	216
-+#define SSPCC_GATE	217
-+#define SPI0_MUX	218
-+#define SPI0_GATE	219
-+#define SPI1_MUX	220
-+#define SPI1_GATE	221
-+#define SPI2_MUX	222
-+#define SPI2_GATE	223
-+#define SPI3_MUX	224
-+#define SPI3_GATE	225
-+#define ECAP0_GATE	226
-+#define ECAP1_GATE	227
-+#define ECAP2_GATE	228
-+#define QEI0_GATE	229
-+#define QEI1_GATE	230
-+#define QEI2_GATE	231
-+#define ADC_DIV		232
-+#define ADC_GATE	233
-+#define EADC_DIV	234
-+#define EADC_GATE	235
-+#define	CLK_MAX_IDX	236
-+
-+#endif /* __DT_BINDINGS_CLOCK_NUVOTON_MA35D1_CLK_H */
-diff --git a/include/dt-bindings/clock/nvidia,tegra264.h b/include/dt-bindings/clock/nvidia,tegra264.h
-new file mode 100644
-index 000000000000..0fc2ad5e6cef
--- /dev/null
-+++ b/include/dt-bindings/clock/nvidia,tegra264.h
-@@ -0,0 +1,466 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/* Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. */
-+
-+#ifndef DT_BINDINGS_CLOCK_NVIDIA_TEGRA264_H
-+#define DT_BINDINGS_CLOCK_NVIDIA_TEGRA264_H
-+
-+#define TEGRA264_CLK_OSC				1
-+#define TEGRA264_CLK_CLK_S				2
-+#define TEGRA264_CLK_JTAG_REG				3
-+#define TEGRA264_CLK_SPLL				4
-+#define TEGRA264_CLK_SPLL_OUT0				5
-+#define TEGRA264_CLK_SPLL_OUT1				6
-+#define TEGRA264_CLK_SPLL_OUT2				7
-+#define TEGRA264_CLK_SPLL_OUT3				8
-+#define TEGRA264_CLK_SPLL_OUT4				9
-+#define TEGRA264_CLK_SPLL_OUT5				10
-+#define TEGRA264_CLK_SPLL_OUT6				11
-+#define TEGRA264_CLK_SPLL_OUT7				12
-+#define TEGRA264_CLK_AON_I2C				13
-+#define TEGRA264_CLK_HOST1X				14
-+#define TEGRA264_CLK_ISP				15
-+#define TEGRA264_CLK_ISP1				16
-+#define TEGRA264_CLK_ISP_ROOT				17
-+#define TEGRA264_CLK_NAFLL_PVA0_CORE			18
-+#define TEGRA264_CLK_NAFLL_PVA0_VPS			19
-+#define TEGRA264_CLK_NVCSI				20
-+#define TEGRA264_CLK_NVCSILP				21
-+#define TEGRA264_CLK_PLLP_OUT0				22
-+#define TEGRA264_CLK_PVA0_CPU_AXI			23
-+#define TEGRA264_CLK_PVA0_VPS				24
-+#define TEGRA264_CLK_PWM10				25
-+#define TEGRA264_CLK_PWM2				26
-+#define TEGRA264_CLK_PWM3				27
-+#define TEGRA264_CLK_PWM4				28
-+#define TEGRA264_CLK_PWM5				29
-+#define TEGRA264_CLK_PWM9				30
-+#define TEGRA264_CLK_QSPI0				31
-+#define TEGRA264_CLK_QSPI0_2X_PM			32
-+#define TEGRA264_CLK_RCE1_CPU				33
-+#define TEGRA264_CLK_RCE1_NIC				34
-+#define TEGRA264_CLK_RCE_CPU				35
-+#define TEGRA264_CLK_RCE_NIC				36
-+#define TEGRA264_CLK_SE					37
-+#define TEGRA264_CLK_SEU1				38
-+#define TEGRA264_CLK_SEU2				39
-+#define TEGRA264_CLK_SEU3				40
-+#define TEGRA264_CLK_SE_ROOT				41
-+#define TEGRA264_CLK_SPI1				42
-+#define TEGRA264_CLK_SPI2				43
-+#define TEGRA264_CLK_SPI3				44
-+#define TEGRA264_CLK_SPI4				45
-+#define TEGRA264_CLK_SPI5				46
-+#define TEGRA264_CLK_TOP_I2C				47
-+#define TEGRA264_CLK_TSEC				48
-+#define TEGRA264_CLK_TSEC_PKA				49
-+#define TEGRA264_CLK_UART0				50
-+#define TEGRA264_CLK_UART10				51
-+#define TEGRA264_CLK_UART11				52
-+#define TEGRA264_CLK_UART4				53
-+#define TEGRA264_CLK_UART5				54
-+#define TEGRA264_CLK_UART8				55
-+#define TEGRA264_CLK_UART9				56
-+#define TEGRA264_CLK_VI					57
-+#define TEGRA264_CLK_VI1				58
-+#define TEGRA264_CLK_VIC				59
-+#define TEGRA264_CLK_VI_ROOT				60
-+#define TEGRA264_CLK_DISPPLL				61
-+#define TEGRA264_CLK_SPPLL0				62
-+#define TEGRA264_CLK_SPPLL0_CLKOUT1A			63
-+#define TEGRA264_CLK_SPPLL0_CLKOUT2A			64
-+#define TEGRA264_CLK_SPPLL1				65
-+#define TEGRA264_CLK_VPLL0				66
-+#define TEGRA264_CLK_VPLL1				67
-+#define TEGRA264_CLK_VPLL2				68
-+#define TEGRA264_CLK_VPLL3				69
-+#define TEGRA264_CLK_VPLL4				70
-+#define TEGRA264_CLK_VPLL5				71
-+#define TEGRA264_CLK_VPLL6				72
-+#define TEGRA264_CLK_VPLL7				73
-+#define TEGRA264_CLK_RG0_DIV				74
-+#define TEGRA264_CLK_RG1_DIV				75
-+#define TEGRA264_CLK_RG2_DIV				76
-+#define TEGRA264_CLK_RG3_DIV				77
-+#define TEGRA264_CLK_RG4_DIV				78
-+#define TEGRA264_CLK_RG5_DIV				79
-+#define TEGRA264_CLK_RG6_DIV				80
-+#define TEGRA264_CLK_RG7_DIV				81
-+#define TEGRA264_CLK_RG0				82
-+#define TEGRA264_CLK_RG1				83
-+#define TEGRA264_CLK_RG2				84
-+#define TEGRA264_CLK_RG3				85
-+#define TEGRA264_CLK_RG4				86
-+#define TEGRA264_CLK_RG5				87
-+#define TEGRA264_CLK_RG6				88
-+#define TEGRA264_CLK_RG7				89
-+#define TEGRA264_CLK_DISP				90
-+#define TEGRA264_CLK_DSC				91
-+#define TEGRA264_CLK_DSC_ROOT				92
-+#define TEGRA264_CLK_HUB				93
-+#define TEGRA264_CLK_VPLLX_SOR0_MUXED			94
-+#define TEGRA264_CLK_VPLLX_SOR1_MUXED			95
-+#define TEGRA264_CLK_VPLLX_SOR2_MUXED			96
-+#define TEGRA264_CLK_VPLLX_SOR3_MUXED			97
-+#define TEGRA264_CLK_LINKA_SYM				98
-+#define TEGRA264_CLK_LINKB_SYM				99
-+#define TEGRA264_CLK_LINKC_SYM				100
-+#define TEGRA264_CLK_LINKD_SYM				101
-+#define TEGRA264_CLK_PRE_SOR0				102
-+#define TEGRA264_CLK_PRE_SOR1				103
-+#define TEGRA264_CLK_PRE_SOR2				104
-+#define TEGRA264_CLK_PRE_SOR3				105
-+#define TEGRA264_CLK_SOR0_PLL_REF			106
-+#define TEGRA264_CLK_SOR1_PLL_REF			107
-+#define TEGRA264_CLK_SOR2_PLL_REF			108
-+#define TEGRA264_CLK_SOR3_PLL_REF			109
-+#define TEGRA264_CLK_SOR0_PAD				110
-+#define TEGRA264_CLK_SOR1_PAD				111
-+#define TEGRA264_CLK_SOR2_PAD				112
-+#define TEGRA264_CLK_SOR3_PAD				113
-+#define TEGRA264_CLK_SOR0_REF				114
-+#define TEGRA264_CLK_SOR1_REF				115
-+#define TEGRA264_CLK_SOR2_REF				116
-+#define TEGRA264_CLK_SOR3_REF				117
-+#define TEGRA264_CLK_SOR0_DIV				118
-+#define TEGRA264_CLK_SOR1_DIV				119
-+#define TEGRA264_CLK_SOR2_DIV				120
-+#define TEGRA264_CLK_SOR3_DIV				121
-+#define TEGRA264_CLK_SOR0				122
-+#define TEGRA264_CLK_SOR1				123
-+#define TEGRA264_CLK_SOR2				124
-+#define TEGRA264_CLK_SOR3				125
-+#define TEGRA264_CLK_SF0_SOR				126
-+#define TEGRA264_CLK_SF1_SOR				127
-+#define TEGRA264_CLK_SF2_SOR				128
-+#define TEGRA264_CLK_SF3_SOR				129
-+#define TEGRA264_CLK_SF4_SOR				130
-+#define TEGRA264_CLK_SF5_SOR				131
-+#define TEGRA264_CLK_SF6_SOR				132
-+#define TEGRA264_CLK_SF7_SOR				133
-+#define TEGRA264_CLK_SF0				134
-+#define TEGRA264_CLK_SF1				135
-+#define TEGRA264_CLK_SF2				136
-+#define TEGRA264_CLK_SF3				137
-+#define TEGRA264_CLK_SF4				138
-+#define TEGRA264_CLK_SF5				139
-+#define TEGRA264_CLK_SF6				140
-+#define TEGRA264_CLK_SF7				141
-+#define TEGRA264_CLK_MAUD				142
-+#define TEGRA264_CLK_AZA_2XBIT				143
-+#define TEGRA264_CLK_DCE_CPU				144
-+#define TEGRA264_CLK_DCE_NIC				145
-+#define TEGRA264_CLK_PLLC4				146
-+#define TEGRA264_CLK_PLLC4_OUT0				147
-+#define TEGRA264_CLK_PLLC4_OUT1				148
-+#define TEGRA264_CLK_PLLC4_MUXED			149
-+#define TEGRA264_CLK_SDMMC1				150
-+#define TEGRA264_CLK_SDMMC_LEGACY_TM			151
-+#define TEGRA264_CLK_PLLC0				152
-+#define TEGRA264_CLK_NAFLL_BPMP				153
-+#define TEGRA264_CLK_PLLP_OUT_PDIV			154
-+#define TEGRA264_CLK_DISP_ROOT				155
-+#define TEGRA264_CLK_ADSP				156
-+#define TEGRA264_CLK_PLLA				157
-+#define TEGRA264_CLK_PLLA1				158
-+#define TEGRA264_CLK_PLLA1_OUT1				159
-+#define TEGRA264_CLK_PLLAON				160
-+#define TEGRA264_CLK_PLLAON_APE				161
-+#define TEGRA264_CLK_PLLA_OUT0				162
-+#define TEGRA264_CLK_AHUB				163
-+#define TEGRA264_CLK_APE				164
-+#define TEGRA264_CLK_I2S1_SCLK_IN			165
-+#define TEGRA264_CLK_I2S2_SCLK_IN			166
-+#define TEGRA264_CLK_I2S3_SCLK_IN			167
-+#define TEGRA264_CLK_I2S4_SCLK_IN			168
-+#define TEGRA264_CLK_I2S5_SCLK_IN			169
-+#define TEGRA264_CLK_I2S6_SCLK_IN			170
-+#define TEGRA264_CLK_I2S7_SCLK_IN			171
-+#define TEGRA264_CLK_I2S8_SCLK_IN			172
-+#define TEGRA264_CLK_I2S9_SCLK_IN			173
-+#define TEGRA264_CLK_I2S1_AUDIO_SYNC			174
-+#define TEGRA264_CLK_I2S2_AUDIO_SYNC			175
-+#define TEGRA264_CLK_I2S3_AUDIO_SYNC			176
-+#define TEGRA264_CLK_I2S4_AUDIO_SYNC			177
-+#define TEGRA264_CLK_I2S5_AUDIO_SYNC			178
-+#define TEGRA264_CLK_I2S6_AUDIO_SYNC			179
-+#define TEGRA264_CLK_I2S7_AUDIO_SYNC			180
-+#define TEGRA264_CLK_I2S8_AUDIO_SYNC			181
-+#define TEGRA264_CLK_DMIC1_AUDIO_SYNC			182
-+#define TEGRA264_CLK_DSPK1_AUDIO_SYNC			183
-+#define TEGRA264_CLK_I2S1				184
-+#define TEGRA264_CLK_I2S2				185
-+#define TEGRA264_CLK_I2S3				186
-+#define TEGRA264_CLK_I2S4				187
-+#define TEGRA264_CLK_I2S5				188
-+#define TEGRA264_CLK_I2S6				189
-+#define TEGRA264_CLK_I2S7				190
-+#define TEGRA264_CLK_I2S8				191
-+#define TEGRA264_CLK_I2S9				192
-+#define TEGRA264_CLK_DMIC1				193
-+#define TEGRA264_CLK_DMIC5				194
-+#define TEGRA264_CLK_DSPK1				195
-+#define TEGRA264_CLK_AON_CPU				196
-+#define TEGRA264_CLK_AON_NIC				197
-+#define TEGRA264_CLK_BPMP				198
-+#define TEGRA264_CLK_AXI_CBB				199
-+#define TEGRA264_CLK_FUSE				200
-+#define TEGRA264_CLK_TSENSE				201
-+#define TEGRA264_CLK_CSITE				202
-+#define TEGRA264_CLK_HCSITE				203
-+#define TEGRA264_CLK_DBGAPB				204
-+#define TEGRA264_CLK_LA					205
-+#define TEGRA264_CLK_PLLREFGP				206
-+#define TEGRA264_CLK_PLLE0				207
-+#define TEGRA264_CLK_UPHY0_PLL0_XDIG			208
-+#define TEGRA264_CLK_EQOS_APP				209
-+#define TEGRA264_CLK_EQOS_MAC				210
-+#define TEGRA264_CLK_EQOS_MACSEC			211
-+#define TEGRA264_CLK_EQOS_TX_PCS			212
-+#define TEGRA264_CLK_MGBES_PTP_REF			213
-+#define TEGRA264_CLK_MGBE0_UPHY1_PLL_XDIG		214
-+#define TEGRA264_CLK_MGBE0_TX_PCS			215
-+#define TEGRA264_CLK_MGBE0_MAC				216
-+#define TEGRA264_CLK_MGBE0_MACSEC			217
-+#define TEGRA264_CLK_MGBE0_APP				218
-+#define TEGRA264_CLK_MGBE1_UPHY1_PLL_XDIG		219
-+#define TEGRA264_CLK_MGBE1_TX_PCS			220
-+#define TEGRA264_CLK_MGBE1_MAC				221
-+#define TEGRA264_CLK_MGBE1_MACSEC			222
-+#define TEGRA264_CLK_MGBE1_APP				223
-+#define TEGRA264_CLK_MGBE2_UPHY1_PLL_XDIG		224
-+#define TEGRA264_CLK_MGBE2_TX_PCS			225
-+#define TEGRA264_CLK_MGBE2_MAC				226
-+#define TEGRA264_CLK_MGBE2_MACSEC			227
-+#define TEGRA264_CLK_MGBE2_APP				228
-+#define TEGRA264_CLK_MGBE3_UPHY1_PLL_XDIG		229
-+#define TEGRA264_CLK_MGBE3_TX_PCS			230
-+#define TEGRA264_CLK_MGBE3_MAC				231
-+#define TEGRA264_CLK_MGBE3_MACSEC			232
-+#define TEGRA264_CLK_MGBE3_APP				233
-+#define TEGRA264_CLK_PLLREFUFS				234
-+#define TEGRA264_CLK_PLLREFUFS_CLKOUT624		235
-+#define TEGRA264_CLK_PLLREFUFS_REFCLKOUT		236
-+#define TEGRA264_CLK_PLLREFUFS_UFSDEV_REFCLKOUT		237
-+#define TEGRA264_CLK_UFSHC_CG_SYS			238
-+#define TEGRA264_CLK_MPHY_L0_RX_LS_BIT_DIV		239
-+#define TEGRA264_CLK_MPHY_L0_RX_LS_BIT			240
-+#define TEGRA264_CLK_MPHY_L0_RX_LS_SYMB_DIV		241
-+#define TEGRA264_CLK_MPHY_L0_RX_HS_SYMB_DIV		242
-+#define TEGRA264_CLK_MPHY_L0_RX_SYMB			243
-+#define TEGRA264_CLK_MPHY_L0_UPHY_TX_FIFO		244
-+#define TEGRA264_CLK_MPHY_L0_TX_LS_3XBIT_DIV		245
-+#define TEGRA264_CLK_MPHY_L0_TX_LS_SYMB_DIV		246
-+#define TEGRA264_CLK_UPHY0_PLL4_XDIG			247
-+#define TEGRA264_CLK_MPHY_L0_TX_HS_SYMB_DIV		248
-+#define TEGRA264_CLK_MPHY_L0_TX_SYMB			249
-+#define TEGRA264_CLK_MPHY_L0_TX_LS_3XBIT		250
-+#define TEGRA264_CLK_MPHY_L0_RX_ANA			251
-+#define TEGRA264_CLK_MPHY_L1_RX_ANA			252
-+#define TEGRA264_CLK_MPHY_TX_1MHZ_REF			253
-+#define TEGRA264_CLK_MPHY_CORE_PLL_FIXED		254
-+#define TEGRA264_CLK_MPHY_IOBIST			255
-+#define TEGRA264_CLK_UFSHC_CG_SYS_DIV			256
-+#define TEGRA264_CLK_XUSB1_CORE				257
-+#define TEGRA264_CLK_XUSB1_FALCON			258
-+#define TEGRA264_CLK_XUSB1_FS				259
-+#define TEGRA264_CLK_XUSB1_SS				260
-+#define TEGRA264_CLK_UPHY0_USB_P0_RX_CORE		261
-+#define TEGRA264_CLK_UPHY0_USB_P1_RX_CORE		262
-+#define TEGRA264_CLK_UPHY0_USB_P2_RX_CORE		263
-+#define TEGRA264_CLK_UPHY0_USB_P3_RX_CORE		264
-+#define TEGRA264_CLK_XUSB1_CLK480M_NVWRAP_CORE		265
-+#define TEGRA264_CLK_XUSB1_CORE_HOST			266
-+#define TEGRA264_CLK_XUSB1_CORE_DEV			267
-+#define TEGRA264_CLK_XUSB1_CORE_SUPERSPEED		268
-+#define TEGRA264_CLK_XUSB1_FALCON_HOST			269
-+#define TEGRA264_CLK_XUSB1_FALCON_SUPERSPEED		270
-+#define TEGRA264_CLK_XUSB1_FS_HOST			271
-+#define TEGRA264_CLK_XUSB1_FS_DEV			272
-+#define TEGRA264_CLK_XUSB1_HS_HSICP			273
-+#define TEGRA264_CLK_XUSB1_SS_DEV			274
-+#define TEGRA264_CLK_XUSB1_SS_SUPERSPEED		275
-+#define TEGRA264_CLK_AON_TOUCH				276
-+#define TEGRA264_CLK_AUD_MCLK				277
-+#define TEGRA264_CLK_EXTPERIPH1				278
-+#define TEGRA264_CLK_EXTPERIPH2				279
-+#define TEGRA264_CLK_EXTPERIPH3				280
-+#define TEGRA264_CLK_EXTPERIPH4				281
-+#define TEGRA264_CLK_JTAG_REG_UNGATED			282
-+#define TEGRA264_CLK_IST_BUS				283
-+#define TEGRA264_CLK_IST_BUS_RIST_MCC			284
-+#define TEGRA264_CLK_MATHS_SEC_RIST			285
-+#define TEGRA264_CLK_NAFLL_IST				286
-+#define TEGRA264_CLK_RIST_ROOT				287
-+#define TEGRA264_CLK_IST_CONTROLLER_RIST		288
-+#define TEGRA264_CLK_MSS_ENCRYPT			289
-+#define TEGRA264_CLK_EMC				290
-+#define TEGRA264_CLK_SPPLL0_CLKOUT100			291
-+#define TEGRA264_CLK_SPPLL0_CLKOUT270			292
-+#define TEGRA264_CLK_SPPLL1_CLKOUT100			293
-+#define TEGRA264_CLK_SPPLL1_CLKOUT270			294
-+#define TEGRA264_CLK_DP_LINKA_REF			295
-+#define TEGRA264_CLK_DP_LINKB_REF			296
-+#define TEGRA264_CLK_DP_LINKC_REF			297
-+#define TEGRA264_CLK_DP_LINKD_REF			298
-+#define TEGRA264_CLK_PLLNVCSI				299
-+#define TEGRA264_CLK_PLLBPMPCAM				300
-+#define TEGRA264_CLK_UTMI_PLL1				301
-+#define TEGRA264_CLK_UTMI_PLL1_CLKOUT48			302
-+#define TEGRA264_CLK_UTMI_PLL1_CLKOUT60			303
-+#define TEGRA264_CLK_UTMI_PLL1_CLKOUT480		304
-+#define TEGRA264_CLK_NAFLL_ISP				305
-+#define TEGRA264_CLK_NAFLL_RCE				306
-+#define TEGRA264_CLK_NAFLL_RCE1				307
-+#define TEGRA264_CLK_NAFLL_SE				308
-+#define TEGRA264_CLK_NAFLL_VI				309
-+#define TEGRA264_CLK_NAFLL_VIC				310
-+#define TEGRA264_CLK_NAFLL_DCE				311
-+#define TEGRA264_CLK_NAFLL_TSEC				312
-+#define TEGRA264_CLK_NAFLL_CPAIR0			313
-+#define TEGRA264_CLK_NAFLL_CPAIR1			314
-+#define TEGRA264_CLK_NAFLL_CPAIR2			315
-+#define TEGRA264_CLK_NAFLL_CPAIR3			316
-+#define TEGRA264_CLK_NAFLL_CPAIR4			317
-+#define TEGRA264_CLK_NAFLL_CPAIR5			318
-+#define TEGRA264_CLK_NAFLL_CPAIR6			319
-+#define TEGRA264_CLK_NAFLL_GPU_SYS			320
-+#define TEGRA264_CLK_NAFLL_GPU_NVD			321
-+#define TEGRA264_CLK_NAFLL_GPU_UPROC			322
-+#define TEGRA264_CLK_NAFLL_GPU_GPC0			323
-+#define TEGRA264_CLK_NAFLL_GPU_GPC1			324
-+#define TEGRA264_CLK_NAFLL_GPU_GPC2			325
-+#define TEGRA264_CLK_SOR_LINKA_INPUT			326
-+#define TEGRA264_CLK_SOR_LINKB_INPUT			327
-+#define TEGRA264_CLK_SOR_LINKC_INPUT			328
-+#define TEGRA264_CLK_SOR_LINKD_INPUT			329
-+#define TEGRA264_CLK_SOR_LINKA_AFIFO			330
-+#define TEGRA264_CLK_SOR_LINKB_AFIFO			331
-+#define TEGRA264_CLK_SOR_LINKC_AFIFO			332
-+#define TEGRA264_CLK_SOR_LINKD_AFIFO			333
-+#define TEGRA264_CLK_I2S1_PAD_M				334
-+#define TEGRA264_CLK_I2S2_PAD_M				335
-+#define TEGRA264_CLK_I2S3_PAD_M				336
-+#define TEGRA264_CLK_I2S4_PAD_M				337
-+#define TEGRA264_CLK_I2S5_PAD_M				338
-+#define TEGRA264_CLK_I2S6_PAD_M				339
-+#define TEGRA264_CLK_I2S7_PAD_M				340
-+#define TEGRA264_CLK_I2S8_PAD_M				341
-+#define TEGRA264_CLK_I2S9_PAD_M				342
-+#define TEGRA264_CLK_BPMP_NIC				343
-+#define TEGRA264_CLK_CLK1M				344
-+#define TEGRA264_CLK_RDET				345
-+#define TEGRA264_CLK_ADC_SOC_REF			346
-+#define TEGRA264_CLK_UPHY0_PLL0_TXREF			347
-+#define TEGRA264_CLK_EQOS_TX				348
-+#define TEGRA264_CLK_EQOS_TX_M				349
-+#define TEGRA264_CLK_EQOS_RX_PCS_IN			350
-+#define TEGRA264_CLK_EQOS_RX_PCS_M			351
-+#define TEGRA264_CLK_EQOS_RX_IN				352
-+#define TEGRA264_CLK_EQOS_RX				353
-+#define TEGRA264_CLK_EQOS_RX_M				354
-+#define TEGRA264_CLK_MGBE0_UPHY1_PLL_TXREF		355
-+#define TEGRA264_CLK_MGBE0_TX				356
-+#define TEGRA264_CLK_MGBE0_TX_M				357
-+#define TEGRA264_CLK_MGBE0_RX_PCS_IN			358
-+#define TEGRA264_CLK_MGBE0_RX_PCS_M			359
-+#define TEGRA264_CLK_MGBE0_RX_IN			360
-+#define TEGRA264_CLK_MGBE0_RX_M				361
-+#define TEGRA264_CLK_MGBE1_UPHY1_PLL_TXREF		362
-+#define TEGRA264_CLK_MGBE1_TX				363
-+#define TEGRA264_CLK_MGBE1_TX_M				364
-+#define TEGRA264_CLK_MGBE1_RX_PCS_IN			365
-+#define TEGRA264_CLK_MGBE1_RX_PCS_M			366
-+#define TEGRA264_CLK_MGBE1_RX_IN			367
-+#define TEGRA264_CLK_MGBE1_RX_M				368
-+#define TEGRA264_CLK_MGBE2_UPHY1_PLL_TXREF		369
-+#define TEGRA264_CLK_MGBE2_TX				370
-+#define TEGRA264_CLK_MGBE2_TX_M				371
-+#define TEGRA264_CLK_MGBE2_RX_PCS_IN			372
-+#define TEGRA264_CLK_MGBE2_RX_PCS_M			373
-+#define TEGRA264_CLK_MGBE2_RX_IN			374
-+#define TEGRA264_CLK_MGBE2_RX_M				375
-+#define TEGRA264_CLK_MGBE3_UPHY1_PLL_TXREF		376
-+#define TEGRA264_CLK_MGBE3_TX				377
-+#define TEGRA264_CLK_MGBE3_TX_M				378
-+#define TEGRA264_CLK_MGBE3_RX_PCS_IN			379
-+#define TEGRA264_CLK_MGBE3_RX_PCS_M			380
-+#define TEGRA264_CLK_MGBE3_RX_IN			381
-+#define TEGRA264_CLK_MGBE3_RX_M				382
-+#define TEGRA264_CLK_UPHY0_USB_P0_TX_CORE		383
-+#define TEGRA264_CLK_UPHY0_USB_P1_TX_CORE		384
-+#define TEGRA264_CLK_UPHY0_USB_P2_TX_CORE		385
-+#define TEGRA264_CLK_UPHY0_USB_P3_TX_CORE		386
-+#define TEGRA264_CLK_UPHY0_USB_P0_TX			387
-+#define TEGRA264_CLK_UPHY0_USB_P1_TX			388
-+#define TEGRA264_CLK_UPHY0_USB_P2_TX			389
-+#define TEGRA264_CLK_UPHY0_USB_P3_TX			390
-+#define TEGRA264_CLK_UPHY0_USB_P0_RX_IN			391
-+#define TEGRA264_CLK_UPHY0_USB_P1_RX_IN			392
-+#define TEGRA264_CLK_UPHY0_USB_P2_RX_IN			393
-+#define TEGRA264_CLK_UPHY0_USB_P3_RX_IN			394
-+#define TEGRA264_CLK_UPHY0_USB_P0_RX_M			395
-+#define TEGRA264_CLK_UPHY0_USB_P1_RX_M			396
-+#define TEGRA264_CLK_UPHY0_USB_P2_RX_M			397
-+#define TEGRA264_CLK_UPHY0_USB_P3_RX_M			398
-+#define TEGRA264_CLK_UPHY0_LANE0_TX_M			399
-+#define TEGRA264_CLK_PCIE_C1_XCLK_NOBG_M		400
-+#define TEGRA264_CLK_PCIE_C2_XCLK_NOBG_M		401
-+#define TEGRA264_CLK_PCIE_C3_XCLK_NOBG_M		402
-+#define TEGRA264_CLK_PCIE_C4_XCLK_NOBG_M		403
-+#define TEGRA264_CLK_PCIE_C5_XCLK_NOBG_M		404
-+#define TEGRA264_CLK_PCIE_C1_L0_RX_M			405
-+#define TEGRA264_CLK_PCIE_C1_L1_RX_M			406
-+#define TEGRA264_CLK_PCIE_C1_L2_RX_M			407
-+#define TEGRA264_CLK_PCIE_C1_L3_RX_M			408
-+#define TEGRA264_CLK_PCIE_C2_L0_RX_M			409
-+#define TEGRA264_CLK_PCIE_C2_L1_RX_M			410
-+#define TEGRA264_CLK_PCIE_C2_L2_RX_M			411
-+#define TEGRA264_CLK_PCIE_C2_L3_RX_M			412
-+#define TEGRA264_CLK_PCIE_C3_L0_RX_M			413
-+#define TEGRA264_CLK_PCIE_C3_L1_RX_M			414
-+#define TEGRA264_CLK_PCIE_C4_L0_RX_M			415
-+#define TEGRA264_CLK_PCIE_C4_L1_RX_M			416
-+#define TEGRA264_CLK_PCIE_C4_L2_RX_M			417
-+#define TEGRA264_CLK_PCIE_C4_L3_RX_M			418
-+#define TEGRA264_CLK_PCIE_C4_L4_RX_M			419
-+#define TEGRA264_CLK_PCIE_C4_L5_RX_M			420
-+#define TEGRA264_CLK_PCIE_C4_L6_RX_M			421
-+#define TEGRA264_CLK_PCIE_C4_L7_RX_M			422
-+#define TEGRA264_CLK_PCIE_C5_L0_RX_M			423
-+#define TEGRA264_CLK_PCIE_C5_L1_RX_M			424
-+#define TEGRA264_CLK_PCIE_C5_L2_RX_M			425
-+#define TEGRA264_CLK_PCIE_C5_L3_RX_M			426
-+#define TEGRA264_CLK_MPHY_L0_RX_PWM_BIT_M		427
-+#define TEGRA264_CLK_MPHY_L1_RX_PWM_BIT_M		428
-+#define TEGRA264_CLK_DBB_UPHY0				429
-+#define TEGRA264_CLK_UPHY0_UXL_CORE			430
-+#define TEGRA264_CLK_ISC_CPU_ROOT			431
-+#define TEGRA264_CLK_ISC_NIC				432
-+#define TEGRA264_CLK_CTC_TXCLK0_M			433
-+#define TEGRA264_CLK_CTC_TXCLK1_M			434
-+#define TEGRA264_CLK_CTC_RXCLK0_M			435
-+#define TEGRA264_CLK_CTC_RXCLK1_M			436
-+#define TEGRA264_CLK_PLLREFGP_OUT			437
-+#define TEGRA264_CLK_PLLREFGP_OUT1			438
-+#define TEGRA264_CLK_GPU_SYS				439
-+#define TEGRA264_CLK_GPU_NVD				440
-+#define TEGRA264_CLK_GPU_UPROC				441
-+#define TEGRA264_CLK_GPU_GPC0				442
-+#define TEGRA264_CLK_GPU_GPC1				443
-+#define TEGRA264_CLK_GPU_GPC2				444
-+#define TEGRA264_CLK_PLLX				445
-+#define TEGRA264_CLK_APE_SOUNDWIRE_MSRC0		446
-+#define TEGRA264_CLK_APE_SOUNDWIRE_DATA_EN_SHAPER	447
-+#define TEGRA264_CLK_AO_SOUNDWIRE_MSRC0			448
-+#define TEGRA264_CLK_AO_SOUNDWIRE_DATA_EN_SHAPER	449
-+#define TEGRA264_CLK_MGBE0_TX_SER			459
-+#define TEGRA264_CLK_MGBE1_TX_SER			460
-+#define TEGRA264_CLK_MGBE2_TX_SER			461
-+#define TEGRA264_CLK_MGBE3_TX_SER			462
-+#define TEGRA264_CLK_MGBE0_RX_SER			463
-+#define TEGRA264_CLK_MGBE1_RX_SER			464
-+#define TEGRA264_CLK_MGBE2_RX_SER			465
-+#define TEGRA264_CLK_MGBE3_RX_SER			466
-+#define TEGRA264_CLK_DPAUX				467
-+
-+#endif /* DT_BINDINGS_CLOCK_NVIDIA_TEGRA264_H */
-diff --git a/include/dt-bindings/clock/nxp,imx94-clock.h b/include/dt-bindings/clock/nxp,imx94-clock.h
-new file mode 100644
-index 000000000000..c4ba13352b99
--- /dev/null
-+++ b/include/dt-bindings/clock/nxp,imx94-clock.h
-@@ -0,0 +1,13 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright 2025 NXP
-+ */
-+
-+#ifndef __DT_BINDINGS_CLOCK_IMX94_H
-+#define __DT_BINDINGS_CLOCK_IMX94_H
-+
-+#define IMX94_CLK_DISPMIX_CLK_SEL	0
-+
-+#define IMX94_CLK_DISPMIX_LVDS_CLK_GATE	0
-+
-+#endif /* __DT_BINDINGS_CLOCK_IMX94_H */
-diff --git a/include/dt-bindings/clock/nxp,imx95-clock.h b/include/dt-bindings/clock/nxp,imx95-clock.h
-new file mode 100644
-index 000000000000..b7a713a9ac8c
--- /dev/null
-+++ b/include/dt-bindings/clock/nxp,imx95-clock.h
-@@ -0,0 +1,31 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
-+/*
-+ * Copyright 2024 NXP
-+ */
-+
-+#ifndef __DT_BINDINGS_CLOCK_IMX95_H
-+#define __DT_BINDINGS_CLOCK_IMX95_H
-+
-+#define IMX95_CLK_VPUBLK_WAVE			0
-+#define IMX95_CLK_VPUBLK_JPEG_ENC		1
-+#define IMX95_CLK_VPUBLK_JPEG_DEC		2
-+
-+#define IMX95_CLK_CAMBLK_CSI2_FOR0		0
-+#define IMX95_CLK_CAMBLK_CSI2_FOR1		1
-+#define IMX95_CLK_CAMBLK_ISP_AXI		2
-+#define IMX95_CLK_CAMBLK_ISP_PIXEL		3
-+#define IMX95_CLK_CAMBLK_ISP			4
-+
-+#define IMX95_CLK_DISPMIX_LVDS_PHY_DIV		0
-+#define IMX95_CLK_DISPMIX_LVDS_CH0_GATE		1
-+#define IMX95_CLK_DISPMIX_LVDS_CH1_GATE		2
-+#define IMX95_CLK_DISPMIX_PIX_DI0_GATE		3
-+#define IMX95_CLK_DISPMIX_PIX_DI1_GATE		4
-+
-+#define IMX95_CLK_DISPMIX_ENG0_SEL		0
-+#define IMX95_CLK_DISPMIX_ENG1_SEL		1
-+
-+#define IMX95_CLK_NETCMIX_ENETC0_RMII		0
-+#define IMX95_CLK_NETCMIX_ENETC1_RMII		1
-+
-+#endif	/* __DT_BINDINGS_CLOCK_IMX95_H */
-diff --git a/include/dt-bindings/clock/px30-cru.h b/include/dt-bindings/clock/px30-cru.h
-index 5b1416fcde6f..a2abf1995c34 100644
--- a/include/dt-bindings/clock/px30-cru.h
-+++ b/include/dt-bindings/clock/px30-cru.h
-@@ -175,8 +175,6 @@
- #define PCLK_CIF		352
- #define PCLK_OTP_PHY		353
- 
-#define CLK_NR_CLKS		(PCLK_OTP_PHY + 1)
-
- /* pmu-clocks indices */
- 
- #define PLL_GPLL		1
-@@ -195,8 +193,6 @@
- #define PCLK_GPIO0_PMU		20
- #define PCLK_UART0_PMU		21
- 
-#define CLKPMU_NR_CLKS		(PCLK_UART0_PMU + 1)
-
- /* soft-reset indices */
- #define SRST_CORE0_PO		0
- #define SRST_CORE1_PO		1
-diff --git a/include/dt-bindings/clock/qcom,apss-ipq.h b/include/dt-bindings/clock/qcom,apss-ipq.h
-index 77b6e05492e2..0bb41e5efdef 100644
--- a/include/dt-bindings/clock/qcom,apss-ipq.h
-+++ b/include/dt-bindings/clock/qcom,apss-ipq.h
-@@ -8,5 +8,11 @@
- 
- #define APCS_ALIAS0_CLK_SRC			0
- #define APCS_ALIAS0_CORE_CLK			1
-+#define APSS_PLL_EARLY				2
-+#define APSS_SILVER_CLK_SRC			3
-+#define APSS_SILVER_CORE_CLK			4
-+#define L3_PLL					5
-+#define L3_CLK_SRC				6
-+#define L3_CORE_CLK				7
- 
- #endif
-diff --git a/include/dt-bindings/clock/qcom,dispcc-qcm2290.h b/include/dt-bindings/clock/qcom,dispcc-qcm2290.h
-index 1db513d6b3ee..cb687949be41 100644
--- a/include/dt-bindings/clock/qcom,dispcc-qcm2290.h
-+++ b/include/dt-bindings/clock/qcom,dispcc-qcm2290.h
-@@ -29,6 +29,10 @@
- #define DISP_CC_XO_CLK				19
- #define DISP_CC_XO_CLK_SRC			20
- 
-+/* GDSCs */
- #define MDSS_GDSC				0
- 
-+/* Resets */
-+#define DISP_CC_MDSS_CORE_BCR			0
-+
- #endif
-diff --git a/include/dt-bindings/clock/qcom,dispcc-sc7280.h b/include/dt-bindings/clock/qcom,dispcc-sc7280.h
-index a4a692c20acf..9f113f346be8 100644
--- a/include/dt-bindings/clock/qcom,dispcc-sc7280.h
-+++ b/include/dt-bindings/clock/qcom,dispcc-sc7280.h
-@@ -52,4 +52,8 @@
- /* DISP_CC power domains */
- #define DISP_CC_MDSS_CORE_GDSC				0
- 
-+/* DISPCC resets */
-+#define DISP_CC_MDSS_CORE_BCR				0
-+#define DISP_CC_MDSS_RSCC_BCR				1
-+
- #endif
-diff --git a/include/dt-bindings/clock/qcom,dispcc-sc8280xp.h b/include/dt-bindings/clock/qcom,dispcc-sc8280xp.h
-new file mode 100644
-index 000000000000..2831c61fa979
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,dispcc-sc8280xp.h
-@@ -0,0 +1,100 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_SC8280XP_H
-+#define _DT_BINDINGS_CLK_QCOM_DISP_CC_SC8280XP_H
-+
-+/* DISPCC clocks */
-+#define DISP_CC_PLL0					0
-+#define DISP_CC_PLL1					1
-+#define DISP_CC_PLL1_OUT_EVEN				2
-+#define DISP_CC_PLL2					3
-+#define DISP_CC_MDSS_AHB1_CLK				4
-+#define DISP_CC_MDSS_AHB_CLK				5
-+#define DISP_CC_MDSS_AHB_CLK_SRC			6
-+#define DISP_CC_MDSS_BYTE0_CLK				7
-+#define DISP_CC_MDSS_BYTE0_CLK_SRC			8
-+#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC			9
-+#define DISP_CC_MDSS_BYTE0_INTF_CLK			10
-+#define DISP_CC_MDSS_BYTE1_CLK				11
-+#define DISP_CC_MDSS_BYTE1_CLK_SRC			12
-+#define DISP_CC_MDSS_BYTE1_DIV_CLK_SRC			13
-+#define DISP_CC_MDSS_BYTE1_INTF_CLK			14
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK			15
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK_SRC			16
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK			17
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK_SRC			18
-+#define DISP_CC_MDSS_DPTX0_LINK_DIV_CLK_SRC		19
-+#define DISP_CC_MDSS_DPTX0_LINK_INTF_CLK		20
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK			21
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC		22
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK			23
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC		24
-+#define DISP_CC_MDSS_DPTX0_USB_ROUTER_LINK_INTF_CLK	25
-+#define DISP_CC_MDSS_DPTX1_AUX_CLK			26
-+#define DISP_CC_MDSS_DPTX1_AUX_CLK_SRC			27
-+#define DISP_CC_MDSS_DPTX1_LINK_CLK			28
-+#define DISP_CC_MDSS_DPTX1_LINK_CLK_SRC			29
-+#define DISP_CC_MDSS_DPTX1_LINK_DIV_CLK_SRC		30
-+#define DISP_CC_MDSS_DPTX1_LINK_INTF_CLK		31
-+#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK			32
-+#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK_SRC		33
-+#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK			34
-+#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK_SRC		35
-+#define DISP_CC_MDSS_DPTX1_USB_ROUTER_LINK_INTF_CLK	36
-+#define DISP_CC_MDSS_DPTX2_AUX_CLK			37
-+#define DISP_CC_MDSS_DPTX2_AUX_CLK_SRC			38
-+#define DISP_CC_MDSS_DPTX2_LINK_CLK			39
-+#define DISP_CC_MDSS_DPTX2_LINK_CLK_SRC			40
-+#define DISP_CC_MDSS_DPTX2_LINK_DIV_CLK_SRC		41
-+#define DISP_CC_MDSS_DPTX2_LINK_INTF_CLK		42
-+#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK			43
-+#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK_SRC		44
-+#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK			45
-+#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK_SRC		46
-+#define DISP_CC_MDSS_DPTX3_AUX_CLK			47
-+#define DISP_CC_MDSS_DPTX3_AUX_CLK_SRC			48
-+#define DISP_CC_MDSS_DPTX3_LINK_CLK			49
-+#define DISP_CC_MDSS_DPTX3_LINK_CLK_SRC			50
-+#define DISP_CC_MDSS_DPTX3_LINK_DIV_CLK_SRC		51
-+#define DISP_CC_MDSS_DPTX3_LINK_INTF_CLK		52
-+#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK			53
-+#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK_SRC		54
-+#define DISP_CC_MDSS_ESC0_CLK				55
-+#define DISP_CC_MDSS_ESC0_CLK_SRC			56
-+#define DISP_CC_MDSS_ESC1_CLK				57
-+#define DISP_CC_MDSS_ESC1_CLK_SRC			58
-+#define DISP_CC_MDSS_MDP1_CLK				59
-+#define DISP_CC_MDSS_MDP_CLK				60
-+#define DISP_CC_MDSS_MDP_CLK_SRC			61
-+#define DISP_CC_MDSS_MDP_LUT1_CLK			62
-+#define DISP_CC_MDSS_MDP_LUT_CLK			63
-+#define DISP_CC_MDSS_NON_GDSC_AHB_CLK			64
-+#define DISP_CC_MDSS_PCLK0_CLK				65
-+#define DISP_CC_MDSS_PCLK0_CLK_SRC			66
-+#define DISP_CC_MDSS_PCLK1_CLK				67
-+#define DISP_CC_MDSS_PCLK1_CLK_SRC			68
-+#define DISP_CC_MDSS_ROT1_CLK				69
-+#define DISP_CC_MDSS_ROT_CLK				70
-+#define DISP_CC_MDSS_ROT_CLK_SRC			71
-+#define DISP_CC_MDSS_RSCC_AHB_CLK			72
-+#define DISP_CC_MDSS_RSCC_VSYNC_CLK			73
-+#define DISP_CC_MDSS_VSYNC1_CLK				74
-+#define DISP_CC_MDSS_VSYNC_CLK				75
-+#define DISP_CC_MDSS_VSYNC_CLK_SRC			76
-+#define DISP_CC_SLEEP_CLK				77
-+#define DISP_CC_SLEEP_CLK_SRC				78
-+#define DISP_CC_XO_CLK					79
-+#define DISP_CC_XO_CLK_SRC				80
-+
-+/* DISPCC resets */
-+#define DISP_CC_MDSS_CORE_BCR				0
-+#define DISP_CC_MDSS_RSCC_BCR				1
-+
-+/* DISPCC GDSCs */
-+#define MDSS_GDSC					0
-+#define MDSS_INT2_GDSC					1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,dispcc-sm6350.h b/include/dt-bindings/clock/qcom,dispcc-sm6350.h
-index cb54aae2723e..61426a80e620 100644
--- a/include/dt-bindings/clock/qcom,dispcc-sm6350.h
-+++ b/include/dt-bindings/clock/qcom,dispcc-sm6350.h
-@@ -42,6 +42,10 @@
- #define DISP_CC_SLEEP_CLK			31
- #define DISP_CC_XO_CLK				32
- 
-+/* Resets */
-+#define DISP_CC_MDSS_CORE_BCR			0
-+#define DISP_CC_MDSS_RSCC_BCR			1
-+
- /* GDSCs */
- #define MDSS_GDSC				0
- 
-diff --git a/include/dt-bindings/clock/qcom,dispcc-sm8250.h b/include/dt-bindings/clock/qcom,dispcc-sm8250.h
-index ce001cbbc27f..767fdb27e514 100644
--- a/include/dt-bindings/clock/qcom,dispcc-sm8250.h
-+++ b/include/dt-bindings/clock/qcom,dispcc-sm8250.h
-@@ -64,6 +64,7 @@
- #define DISP_CC_MDSS_EDP_LINK_INTF_CLK		54
- #define DISP_CC_MDSS_EDP_PIXEL_CLK		55
- #define DISP_CC_MDSS_EDP_PIXEL_CLK_SRC		56
-+#define DISP_CC_MDSS_EDP_LINK_DIV_CLK_SRC	57
- 
- /* DISP_CC Reset */
- #define DISP_CC_MDSS_CORE_BCR			0
-diff --git a/include/dt-bindings/clock/qcom,dsi-phy-28nm.h b/include/dt-bindings/clock/qcom,dsi-phy-28nm.h
-new file mode 100644
-index 000000000000..ab94d58377a1
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,dsi-phy-28nm.h
-@@ -0,0 +1,9 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_DSI_PHY_28NM_H
-+#define _DT_BINDINGS_CLK_QCOM_DSI_PHY_28NM_H
-+
-+#define DSI_BYTE_PLL_CLK		0
-+#define DSI_PIXEL_PLL_CLK		1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,gcc-apq8084.h b/include/dt-bindings/clock/qcom,gcc-apq8084.h
-index 7f657cf8cc8a..a985248d6332 100644
--- a/include/dt-bindings/clock/qcom,gcc-apq8084.h
-+++ b/include/dt-bindings/clock/qcom,gcc-apq8084.h
-@@ -339,6 +339,7 @@
- #define GCC_PCIE_1_MSTR_AXI_CLK				330
- #define GCC_PCIE_1_PIPE_CLK				331
- #define GCC_PCIE_1_SLV_AXI_CLK				332
-+#define GCC_MMSS_GPLL0_CLK_SRC				333
- 
- /* gdscs */
- #define USB_HS_HSIC_GDSC				0
-diff --git a/include/dt-bindings/clock/qcom,gcc-ipq4019.h b/include/dt-bindings/clock/qcom,gcc-ipq4019.h
-index 7e8a7be6dcda..fa0587857547 100644
--- a/include/dt-bindings/clock/qcom,gcc-ipq4019.h
-+++ b/include/dt-bindings/clock/qcom,gcc-ipq4019.h
-@@ -165,5 +165,11 @@
- #define GCC_QDSS_BCR					69
- #define GCC_MPM_BCR					70
- #define GCC_SPDM_BCR					71
-+#define ESS_MAC1_ARES					72
-+#define ESS_MAC2_ARES					73
-+#define ESS_MAC3_ARES					74
-+#define ESS_MAC4_ARES					75
-+#define ESS_MAC5_ARES					76
-+#define ESS_PSGMII_ARES					77
- 
- #endif
-diff --git a/include/dt-bindings/clock/qcom,gcc-ipq5018.h b/include/dt-bindings/clock/qcom,gcc-ipq5018.h
-new file mode 100644
-index 000000000000..f3de2fdfeea1
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,gcc-ipq5018.h
-@@ -0,0 +1,183 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLOCK_IPQ_GCC_5018_H
-+#define _DT_BINDINGS_CLOCK_IPQ_GCC_5018_H
-+
-+#define GPLL0_MAIN					0
-+#define GPLL0						1
-+#define GPLL2_MAIN					2
-+#define GPLL2						3
-+#define GPLL4_MAIN					4
-+#define GPLL4						5
-+#define UBI32_PLL_MAIN					6
-+#define UBI32_PLL					7
-+#define ADSS_PWM_CLK_SRC				8
-+#define BLSP1_QUP1_I2C_APPS_CLK_SRC			9
-+#define BLSP1_QUP1_SPI_APPS_CLK_SRC			10
-+#define BLSP1_QUP2_I2C_APPS_CLK_SRC			11
-+#define BLSP1_QUP2_SPI_APPS_CLK_SRC			12
-+#define BLSP1_QUP3_I2C_APPS_CLK_SRC			13
-+#define BLSP1_QUP3_SPI_APPS_CLK_SRC			14
-+#define BLSP1_UART1_APPS_CLK_SRC			15
-+#define BLSP1_UART2_APPS_CLK_SRC			16
-+#define CRYPTO_CLK_SRC					17
-+#define GCC_ADSS_PWM_CLK				18
-+#define GCC_BLSP1_AHB_CLK				19
-+#define GCC_BLSP1_QUP1_I2C_APPS_CLK			20
-+#define GCC_BLSP1_QUP1_SPI_APPS_CLK			21
-+#define GCC_BLSP1_QUP2_I2C_APPS_CLK			22
-+#define GCC_BLSP1_QUP2_SPI_APPS_CLK			23
-+#define GCC_BLSP1_QUP3_I2C_APPS_CLK			24
-+#define GCC_BLSP1_QUP3_SPI_APPS_CLK			25
-+#define GCC_BLSP1_UART1_APPS_CLK			26
-+#define GCC_BLSP1_UART2_APPS_CLK			27
-+#define GCC_BTSS_LPO_CLK				28
-+#define GCC_CMN_BLK_AHB_CLK				29
-+#define GCC_CMN_BLK_SYS_CLK				30
-+#define GCC_CRYPTO_AHB_CLK				31
-+#define GCC_CRYPTO_AXI_CLK				32
-+#define GCC_CRYPTO_CLK					33
-+#define GCC_CRYPTO_PPE_CLK				34
-+#define GCC_DCC_CLK					35
-+#define GCC_GEPHY_RX_CLK				36
-+#define GCC_GEPHY_TX_CLK				37
-+#define GCC_GMAC0_CFG_CLK				38
-+#define GCC_GMAC0_PTP_CLK				39
-+#define GCC_GMAC0_RX_CLK				40
-+#define GCC_GMAC0_SYS_CLK				41
-+#define GCC_GMAC0_TX_CLK				42
-+#define GCC_GMAC1_CFG_CLK				43
-+#define GCC_GMAC1_PTP_CLK				44
-+#define GCC_GMAC1_RX_CLK				45
-+#define GCC_GMAC1_SYS_CLK				46
-+#define GCC_GMAC1_TX_CLK				47
-+#define GCC_GP1_CLK					48
-+#define GCC_GP2_CLK					49
-+#define GCC_GP3_CLK					50
-+#define GCC_LPASS_CORE_AXIM_CLK				51
-+#define GCC_LPASS_SWAY_CLK				52
-+#define GCC_MDIO0_AHB_CLK				53
-+#define GCC_MDIO1_AHB_CLK				54
-+#define GCC_PCIE0_AHB_CLK				55
-+#define GCC_PCIE0_AUX_CLK				56
-+#define GCC_PCIE0_AXI_M_CLK				57
-+#define GCC_PCIE0_AXI_S_BRIDGE_CLK			58
-+#define GCC_PCIE0_AXI_S_CLK				59
-+#define GCC_PCIE0_PIPE_CLK				60
-+#define GCC_PCIE1_AHB_CLK				61
-+#define GCC_PCIE1_AUX_CLK				62
-+#define GCC_PCIE1_AXI_M_CLK				63
-+#define GCC_PCIE1_AXI_S_BRIDGE_CLK			64
-+#define GCC_PCIE1_AXI_S_CLK				65
-+#define GCC_PCIE1_PIPE_CLK				66
-+#define GCC_PRNG_AHB_CLK				67
-+#define GCC_Q6_AXIM_CLK					68
-+#define GCC_Q6_AXIM2_CLK				69
-+#define GCC_Q6_AXIS_CLK					70
-+#define GCC_Q6_AHB_CLK					71
-+#define GCC_Q6_AHB_S_CLK				72
-+#define GCC_Q6_TSCTR_1TO2_CLK				73
-+#define GCC_Q6SS_ATBM_CLK				74
-+#define GCC_Q6SS_PCLKDBG_CLK				75
-+#define GCC_Q6SS_TRIG_CLK				76
-+#define GCC_QDSS_AT_CLK					77
-+#define GCC_QDSS_CFG_AHB_CLK				78
-+#define GCC_QDSS_DAP_AHB_CLK				79
-+#define GCC_QDSS_DAP_CLK				80
-+#define GCC_QDSS_ETR_USB_CLK				81
-+#define GCC_QDSS_EUD_AT_CLK				82
-+#define GCC_QDSS_STM_CLK				83
-+#define GCC_QDSS_TRACECLKIN_CLK				84
-+#define GCC_QDSS_TSCTR_DIV8_CLK				85
-+#define GCC_QPIC_AHB_CLK				86
-+#define GCC_QPIC_CLK					87
-+#define GCC_QPIC_IO_MACRO_CLK				88
-+#define GCC_SDCC1_AHB_CLK				89
-+#define GCC_SDCC1_APPS_CLK				90
-+#define GCC_SLEEP_CLK_SRC				91
-+#define GCC_SNOC_GMAC0_AHB_CLK				92
-+#define GCC_SNOC_GMAC0_AXI_CLK				93
-+#define GCC_SNOC_GMAC1_AHB_CLK				94
-+#define GCC_SNOC_GMAC1_AXI_CLK				95
-+#define GCC_SNOC_LPASS_AXIM_CLK				96
-+#define GCC_SNOC_LPASS_SWAY_CLK				97
-+#define GCC_SNOC_UBI0_AXI_CLK				98
-+#define GCC_SYS_NOC_PCIE0_AXI_CLK			99
-+#define GCC_SYS_NOC_PCIE1_AXI_CLK			100
-+#define GCC_SYS_NOC_QDSS_STM_AXI_CLK			101
-+#define GCC_SYS_NOC_USB0_AXI_CLK			102
-+#define GCC_SYS_NOC_WCSS_AHB_CLK			103
-+#define GCC_UBI0_AXI_CLK				104
-+#define GCC_UBI0_CFG_CLK				105
-+#define GCC_UBI0_CORE_CLK				106
-+#define GCC_UBI0_DBG_CLK				107
-+#define GCC_UBI0_NC_AXI_CLK				108
-+#define GCC_UBI0_UTCM_CLK				109
-+#define GCC_UNIPHY_AHB_CLK				110
-+#define GCC_UNIPHY_RX_CLK				111
-+#define GCC_UNIPHY_SYS_CLK				112
-+#define GCC_UNIPHY_TX_CLK				113
-+#define GCC_USB0_AUX_CLK				114
-+#define GCC_USB0_EUD_AT_CLK				115
-+#define GCC_USB0_LFPS_CLK				116
-+#define GCC_USB0_MASTER_CLK				117
-+#define GCC_USB0_MOCK_UTMI_CLK				118
-+#define GCC_USB0_PHY_CFG_AHB_CLK			119
-+#define GCC_USB0_SLEEP_CLK				120
-+#define GCC_WCSS_ACMT_CLK				121
-+#define GCC_WCSS_AHB_S_CLK				122
-+#define GCC_WCSS_AXI_M_CLK				123
-+#define GCC_WCSS_AXI_S_CLK				124
-+#define GCC_WCSS_DBG_IFC_APB_BDG_CLK			125
-+#define GCC_WCSS_DBG_IFC_APB_CLK			126
-+#define GCC_WCSS_DBG_IFC_ATB_BDG_CLK			127
-+#define GCC_WCSS_DBG_IFC_ATB_CLK			128
-+#define GCC_WCSS_DBG_IFC_DAPBUS_BDG_CLK			129
-+#define GCC_WCSS_DBG_IFC_DAPBUS_CLK			130
-+#define GCC_WCSS_DBG_IFC_NTS_BDG_CLK			131
-+#define GCC_WCSS_DBG_IFC_NTS_CLK			132
-+#define GCC_WCSS_ECAHB_CLK				133
-+#define GCC_XO_CLK					134
-+#define GCC_XO_CLK_SRC					135
-+#define GMAC0_RX_CLK_SRC				136
-+#define GMAC0_TX_CLK_SRC				137
-+#define GMAC1_RX_CLK_SRC				138
-+#define GMAC1_TX_CLK_SRC				139
-+#define GMAC_CLK_SRC					140
-+#define GP1_CLK_SRC					141
-+#define GP2_CLK_SRC					142
-+#define GP3_CLK_SRC					143
-+#define LPASS_AXIM_CLK_SRC				144
-+#define LPASS_SWAY_CLK_SRC				145
-+#define PCIE0_AUX_CLK_SRC				146
-+#define PCIE0_AXI_CLK_SRC				147
-+#define PCIE1_AUX_CLK_SRC				148
-+#define PCIE1_AXI_CLK_SRC				149
-+#define PCNOC_BFDCD_CLK_SRC				150
-+#define Q6_AXI_CLK_SRC					151
-+#define QDSS_AT_CLK_SRC					152
-+#define QDSS_STM_CLK_SRC				153
-+#define QDSS_TSCTR_CLK_SRC				154
-+#define QDSS_TRACECLKIN_CLK_SRC				155
-+#define QPIC_IO_MACRO_CLK_SRC				156
-+#define SDCC1_APPS_CLK_SRC				157
-+#define SYSTEM_NOC_BFDCD_CLK_SRC			158
-+#define UBI0_AXI_CLK_SRC				159
-+#define UBI0_CORE_CLK_SRC				160
-+#define USB0_AUX_CLK_SRC				161
-+#define USB0_LFPS_CLK_SRC				162
-+#define USB0_MASTER_CLK_SRC				163
-+#define USB0_MOCK_UTMI_CLK_SRC				164
-+#define WCSS_AHB_CLK_SRC				165
-+#define PCIE0_PIPE_CLK_SRC				166
-+#define PCIE1_PIPE_CLK_SRC				167
-+#define USB0_PIPE_CLK_SRC				168
-+#define GCC_USB0_PIPE_CLK				169
-+#define GMAC0_RX_DIV_CLK_SRC				170
-+#define GMAC0_TX_DIV_CLK_SRC				171
-+#define GMAC1_RX_DIV_CLK_SRC				172
-+#define GMAC1_TX_DIV_CLK_SRC				173
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,gcc-ipq8074.h b/include/dt-bindings/clock/qcom,gcc-ipq8074.h
-index e4991d303708..f9ea55811104 100644
--- a/include/dt-bindings/clock/qcom,gcc-ipq8074.h
-+++ b/include/dt-bindings/clock/qcom,gcc-ipq8074.h
-@@ -367,6 +367,20 @@
- #define GCC_PCIE1_AHB_ARES			129
- #define GCC_PCIE1_AXI_MASTER_STICKY_ARES	130
- #define GCC_PCIE0_AXI_SLAVE_STICKY_ARES		131
-+#define GCC_PPE_FULL_RESET			132
-+#define GCC_UNIPHY0_SOFT_RESET			133
-+#define GCC_UNIPHY0_XPCS_RESET			134
-+#define GCC_UNIPHY1_SOFT_RESET			135
-+#define GCC_UNIPHY1_XPCS_RESET			136
-+#define GCC_UNIPHY2_SOFT_RESET			137
-+#define GCC_UNIPHY2_XPCS_RESET			138
-+#define GCC_EDMA_HW_RESET			139
-+#define GCC_NSSPORT1_RESET			140
-+#define GCC_NSSPORT2_RESET			141
-+#define GCC_NSSPORT3_RESET			142
-+#define GCC_NSSPORT4_RESET			143
-+#define GCC_NSSPORT5_RESET			144
-+#define GCC_NSSPORT6_RESET			145
- 
- #define USB0_GDSC				0
- #define USB1_GDSC				1
-diff --git a/include/dt-bindings/clock/qcom,gcc-msm8909.h b/include/dt-bindings/clock/qcom,gcc-msm8909.h
-new file mode 100644
-index 000000000000..4394ba003425
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,gcc-msm8909.h
-@@ -0,0 +1,218 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2022 Kernkonzept GmbH.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_8909_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_8909_H
-+
-+/* PLLs */
-+#define GPLL0_EARLY				0
-+#define GPLL0					1
-+#define GPLL1					2
-+#define GPLL1_VOTE				3
-+#define GPLL2_EARLY				4
-+#define GPLL2					5
-+#define BIMC_PLL_EARLY				6
-+#define BIMC_PLL				7
-+
-+/* RCGs */
-+#define APSS_AHB_CLK_SRC			8
-+#define BIMC_DDR_CLK_SRC			9
-+#define BIMC_GPU_CLK_SRC			10
-+#define BLSP1_QUP1_I2C_APPS_CLK_SRC		11
-+#define BLSP1_QUP1_SPI_APPS_CLK_SRC		12
-+#define BLSP1_QUP2_I2C_APPS_CLK_SRC		13
-+#define BLSP1_QUP2_SPI_APPS_CLK_SRC		14
-+#define BLSP1_QUP3_I2C_APPS_CLK_SRC		15
-+#define BLSP1_QUP3_SPI_APPS_CLK_SRC		16
-+#define BLSP1_QUP4_I2C_APPS_CLK_SRC		17
-+#define BLSP1_QUP4_SPI_APPS_CLK_SRC		18
-+#define BLSP1_QUP5_I2C_APPS_CLK_SRC		19
-+#define BLSP1_QUP5_SPI_APPS_CLK_SRC		20
-+#define BLSP1_QUP6_I2C_APPS_CLK_SRC		21
-+#define BLSP1_QUP6_SPI_APPS_CLK_SRC		22
-+#define BLSP1_UART1_APPS_CLK_SRC		23
-+#define BLSP1_UART2_APPS_CLK_SRC		24
-+#define BYTE0_CLK_SRC				25
-+#define CAMSS_GP0_CLK_SRC			26
-+#define CAMSS_GP1_CLK_SRC			27
-+#define CAMSS_TOP_AHB_CLK_SRC			28
-+#define CODEC_DIGCODEC_CLK_SRC			29
-+#define CRYPTO_CLK_SRC				30
-+#define CSI0_CLK_SRC				31
-+#define CSI0PHYTIMER_CLK_SRC			32
-+#define CSI1_CLK_SRC				33
-+#define ESC0_CLK_SRC				34
-+#define GFX3D_CLK_SRC				35
-+#define GP1_CLK_SRC				36
-+#define GP2_CLK_SRC				37
-+#define GP3_CLK_SRC				38
-+#define MCLK0_CLK_SRC				39
-+#define MCLK1_CLK_SRC				40
-+#define MDP_CLK_SRC				41
-+#define PCLK0_CLK_SRC				42
-+#define PCNOC_BFDCD_CLK_SRC			43
-+#define PDM2_CLK_SRC				44
-+#define SDCC1_APPS_CLK_SRC			45
-+#define SDCC2_APPS_CLK_SRC			46
-+#define SYSTEM_NOC_BFDCD_CLK_SRC		47
-+#define ULTAUDIO_AHBFABRIC_CLK_SRC		48
-+#define ULTAUDIO_LPAIF_AUX_I2S_CLK_SRC		49
-+#define ULTAUDIO_LPAIF_PRI_I2S_CLK_SRC		50
-+#define ULTAUDIO_LPAIF_SEC_I2S_CLK_SRC		51
-+#define ULTAUDIO_XO_CLK_SRC			52
-+#define USB_HS_SYSTEM_CLK_SRC			53
-+#define VCODEC0_CLK_SRC				54
-+#define VFE0_CLK_SRC				55
-+#define VSYNC_CLK_SRC				56
-+
-+/* Voteable Clocks */
-+#define GCC_APSS_TCU_CLK			57
-+#define GCC_BLSP1_AHB_CLK			58
-+#define GCC_BLSP1_SLEEP_CLK			59
-+#define GCC_BOOT_ROM_AHB_CLK			60
-+#define GCC_CRYPTO_CLK				61
-+#define GCC_CRYPTO_AHB_CLK			62
-+#define GCC_CRYPTO_AXI_CLK			63
-+#define GCC_GFX_TBU_CLK				64
-+#define GCC_GFX_TCU_CLK				65
-+#define GCC_GTCU_AHB_CLK			66
-+#define GCC_MDP_TBU_CLK				67
-+#define GCC_PRNG_AHB_CLK			68
-+#define GCC_SMMU_CFG_CLK			69
-+#define GCC_VENUS_TBU_CLK			70
-+#define GCC_VFE_TBU_CLK				71
-+
-+/* Branches */
-+#define GCC_BIMC_GFX_CLK			72
-+#define GCC_BIMC_GPU_CLK			73
-+#define GCC_BLSP1_QUP1_I2C_APPS_CLK		74
-+#define GCC_BLSP1_QUP1_SPI_APPS_CLK		75
-+#define GCC_BLSP1_QUP2_I2C_APPS_CLK		76
-+#define GCC_BLSP1_QUP2_SPI_APPS_CLK		77
-+#define GCC_BLSP1_QUP3_I2C_APPS_CLK		78
-+#define GCC_BLSP1_QUP3_SPI_APPS_CLK		79
-+#define GCC_BLSP1_QUP4_I2C_APPS_CLK		80
-+#define GCC_BLSP1_QUP4_SPI_APPS_CLK		81
-+#define GCC_BLSP1_QUP5_I2C_APPS_CLK		82
-+#define GCC_BLSP1_QUP5_SPI_APPS_CLK		83
-+#define GCC_BLSP1_QUP6_I2C_APPS_CLK		84
-+#define GCC_BLSP1_QUP6_SPI_APPS_CLK		85
-+#define GCC_BLSP1_UART1_APPS_CLK		86
-+#define GCC_BLSP1_UART2_APPS_CLK		87
-+#define GCC_CAMSS_AHB_CLK			88
-+#define GCC_CAMSS_CSI0_CLK			89
-+#define GCC_CAMSS_CSI0_AHB_CLK			90
-+#define GCC_CAMSS_CSI0PHY_CLK			91
-+#define GCC_CAMSS_CSI0PHYTIMER_CLK		92
-+#define GCC_CAMSS_CSI0PIX_CLK			93
-+#define GCC_CAMSS_CSI0RDI_CLK			94
-+#define GCC_CAMSS_CSI1_CLK			95
-+#define GCC_CAMSS_CSI1_AHB_CLK			96
-+#define GCC_CAMSS_CSI1PHY_CLK			97
-+#define GCC_CAMSS_CSI1PIX_CLK			98
-+#define GCC_CAMSS_CSI1RDI_CLK			99
-+#define GCC_CAMSS_CSI_VFE0_CLK			100
-+#define GCC_CAMSS_GP0_CLK			101
-+#define GCC_CAMSS_GP1_CLK			102
-+#define GCC_CAMSS_ISPIF_AHB_CLK			103
-+#define GCC_CAMSS_MCLK0_CLK			104
-+#define GCC_CAMSS_MCLK1_CLK			105
-+#define GCC_CAMSS_TOP_AHB_CLK			106
-+#define GCC_CAMSS_VFE0_CLK			107
-+#define GCC_CAMSS_VFE_AHB_CLK			108
-+#define GCC_CAMSS_VFE_AXI_CLK			109
-+#define GCC_CODEC_DIGCODEC_CLK			110
-+#define GCC_GP1_CLK				111
-+#define GCC_GP2_CLK				112
-+#define GCC_GP3_CLK				113
-+#define GCC_MDSS_AHB_CLK			114
-+#define GCC_MDSS_AXI_CLK			115
-+#define GCC_MDSS_BYTE0_CLK			116
-+#define GCC_MDSS_ESC0_CLK			117
-+#define GCC_MDSS_MDP_CLK			118
-+#define GCC_MDSS_PCLK0_CLK			119
-+#define GCC_MDSS_VSYNC_CLK			120
-+#define GCC_MSS_CFG_AHB_CLK			121
-+#define GCC_MSS_Q6_BIMC_AXI_CLK			122
-+#define GCC_OXILI_AHB_CLK			123
-+#define GCC_OXILI_GFX3D_CLK			124
-+#define GCC_PDM2_CLK				125
-+#define GCC_PDM_AHB_CLK				126
-+#define GCC_SDCC1_AHB_CLK			127
-+#define GCC_SDCC1_APPS_CLK			128
-+#define GCC_SDCC2_AHB_CLK			129
-+#define GCC_SDCC2_APPS_CLK			130
-+#define GCC_ULTAUDIO_AHBFABRIC_IXFABRIC_CLK	131
-+#define GCC_ULTAUDIO_AHBFABRIC_IXFABRIC_LPM_CLK	132
-+#define GCC_ULTAUDIO_AVSYNC_XO_CLK		133
-+#define GCC_ULTAUDIO_LPAIF_AUX_I2S_CLK		134
-+#define GCC_ULTAUDIO_LPAIF_PRI_I2S_CLK		135
-+#define GCC_ULTAUDIO_LPAIF_SEC_I2S_CLK		136
-+#define GCC_ULTAUDIO_PCNOC_MPORT_CLK		137
-+#define GCC_ULTAUDIO_PCNOC_SWAY_CLK		138
-+#define GCC_ULTAUDIO_STC_XO_CLK			139
-+#define GCC_USB2A_PHY_SLEEP_CLK			140
-+#define GCC_USB_HS_AHB_CLK			141
-+#define GCC_USB_HS_PHY_CFG_AHB_CLK		142
-+#define GCC_USB_HS_SYSTEM_CLK			143
-+#define GCC_VENUS0_AHB_CLK			144
-+#define GCC_VENUS0_AXI_CLK			145
-+#define GCC_VENUS0_CORE0_VCODEC0_CLK		146
-+#define GCC_VENUS0_VCODEC0_CLK			147
-+
-+/* Resets */
-+#define GCC_AUDIO_CORE_BCR			0
-+#define GCC_BLSP1_BCR				1
-+#define GCC_BLSP1_QUP1_BCR			2
-+#define GCC_BLSP1_QUP2_BCR			3
-+#define GCC_BLSP1_QUP3_BCR			4
-+#define GCC_BLSP1_QUP4_BCR			5
-+#define GCC_BLSP1_QUP5_BCR			6
-+#define GCC_BLSP1_QUP6_BCR			7
-+#define GCC_BLSP1_UART1_BCR			8
-+#define GCC_BLSP1_UART2_BCR			9
-+#define GCC_CAMSS_CSI0_BCR			10
-+#define GCC_CAMSS_CSI0PHY_BCR			11
-+#define GCC_CAMSS_CSI0PIX_BCR			12
-+#define GCC_CAMSS_CSI0RDI_BCR			13
-+#define GCC_CAMSS_CSI1_BCR			14
-+#define GCC_CAMSS_CSI1PHY_BCR			15
-+#define GCC_CAMSS_CSI1PIX_BCR			16
-+#define GCC_CAMSS_CSI1RDI_BCR			17
-+#define GCC_CAMSS_CSI_VFE0_BCR			18
-+#define GCC_CAMSS_GP0_BCR			19
-+#define GCC_CAMSS_GP1_BCR			20
-+#define GCC_CAMSS_ISPIF_BCR			21
-+#define GCC_CAMSS_MCLK0_BCR			22
-+#define GCC_CAMSS_MCLK1_BCR			23
-+#define GCC_CAMSS_PHY0_BCR			24
-+#define GCC_CAMSS_TOP_BCR			25
-+#define GCC_CAMSS_TOP_AHB_BCR			26
-+#define GCC_CAMSS_VFE_BCR			27
-+#define GCC_CRYPTO_BCR				28
-+#define GCC_MDSS_BCR				29
-+#define GCC_OXILI_BCR				30
-+#define GCC_PDM_BCR				31
-+#define GCC_PRNG_BCR				32
-+#define GCC_QUSB2_PHY_BCR			33
-+#define GCC_SDCC1_BCR				34
-+#define GCC_SDCC2_BCR				35
-+#define GCC_ULT_AUDIO_BCR			36
-+#define GCC_USB2A_PHY_BCR			37
-+#define GCC_USB2_HS_PHY_ONLY_BCR		38
-+#define GCC_USB_HS_BCR				39
-+#define GCC_VENUS0_BCR				40
-+
-+/* Subsystem Restart */
-+#define GCC_MSS_RESTART				41
-+
-+/* Power Domains */
-+#define MDSS_GDSC				0
-+#define OXILI_GDSC				1
-+#define VENUS_GDSC				2
-+#define VENUS_CORE0_GDSC			3
-+#define VFE_GDSC				4
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,gcc-msm8917.h b/include/dt-bindings/clock/qcom,gcc-msm8917.h
-new file mode 100644
-index 000000000000..4e3897b3669d
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,gcc-msm8917.h
-@@ -0,0 +1,210 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_CLK_MSM_GCC_8917_H
-+#define _DT_BINDINGS_CLK_MSM_GCC_8917_H
-+
-+/* Clocks */
-+#define APSS_AHB_CLK_SRC			0
-+#define BLSP1_QUP2_I2C_APPS_CLK_SRC		1
-+#define BLSP1_QUP2_SPI_APPS_CLK_SRC		2
-+#define BLSP1_QUP3_I2C_APPS_CLK_SRC		3
-+#define BLSP1_QUP3_SPI_APPS_CLK_SRC		4
-+#define BLSP1_QUP4_I2C_APPS_CLK_SRC		5
-+#define BLSP1_QUP4_SPI_APPS_CLK_SRC		6
-+#define BLSP1_UART1_APPS_CLK_SRC		7
-+#define BLSP1_UART2_APPS_CLK_SRC		8
-+#define BLSP2_QUP1_I2C_APPS_CLK_SRC		9
-+#define BLSP2_QUP1_SPI_APPS_CLK_SRC		10
-+#define BLSP2_QUP2_I2C_APPS_CLK_SRC		11
-+#define BLSP2_QUP2_SPI_APPS_CLK_SRC		12
-+#define BLSP2_QUP3_I2C_APPS_CLK_SRC		13
-+#define BLSP2_QUP3_SPI_APPS_CLK_SRC		14
-+#define BLSP2_UART1_APPS_CLK_SRC		15
-+#define BLSP2_UART2_APPS_CLK_SRC		16
-+#define BYTE0_CLK_SRC				17
-+#define CAMSS_GP0_CLK_SRC			18
-+#define CAMSS_GP1_CLK_SRC			19
-+#define CAMSS_TOP_AHB_CLK_SRC			20
-+#define CCI_CLK_SRC				21
-+#define CPP_CLK_SRC				22
-+#define CRYPTO_CLK_SRC				23
-+#define CSI0PHYTIMER_CLK_SRC			24
-+#define CSI0_CLK_SRC				25
-+#define CSI1PHYTIMER_CLK_SRC			26
-+#define CSI1_CLK_SRC				27
-+#define CSI2_CLK_SRC				28
-+#define ESC0_CLK_SRC				29
-+#define GCC_APSS_TCU_CLK			30
-+#define GCC_BIMC_GFX_CLK			31
-+#define GCC_BIMC_GPU_CLK			32
-+#define GCC_BLSP1_AHB_CLK			33
-+#define GCC_BLSP1_QUP2_I2C_APPS_CLK		34
-+#define GCC_BLSP1_QUP2_SPI_APPS_CLK		35
-+#define GCC_BLSP1_QUP3_I2C_APPS_CLK		36
-+#define GCC_BLSP1_QUP3_SPI_APPS_CLK		37
-+#define GCC_BLSP1_QUP4_I2C_APPS_CLK		38
-+#define GCC_BLSP1_QUP4_SPI_APPS_CLK		39
-+#define GCC_BLSP1_UART1_APPS_CLK		40
-+#define GCC_BLSP1_UART2_APPS_CLK		41
-+#define GCC_BLSP2_AHB_CLK			42
-+#define GCC_BLSP2_QUP1_I2C_APPS_CLK		43
-+#define GCC_BLSP2_QUP1_SPI_APPS_CLK		44
-+#define GCC_BLSP2_QUP2_I2C_APPS_CLK		45
-+#define GCC_BLSP2_QUP2_SPI_APPS_CLK		46
-+#define GCC_BLSP2_QUP3_I2C_APPS_CLK		47
-+#define GCC_BLSP2_QUP3_SPI_APPS_CLK		48
-+#define GCC_BLSP2_UART1_APPS_CLK		49
-+#define GCC_BLSP2_UART2_APPS_CLK		50
-+#define GCC_BOOT_ROM_AHB_CLK			51
-+#define GCC_CAMSS_AHB_CLK			52
-+#define GCC_CAMSS_CCI_AHB_CLK			53
-+#define GCC_CAMSS_CCI_CLK			54
-+#define GCC_CAMSS_CPP_AHB_CLK			55
-+#define GCC_CAMSS_CPP_CLK			56
-+#define GCC_CAMSS_CSI0PHYTIMER_CLK		57
-+#define GCC_CAMSS_CSI0PHY_CLK			58
-+#define GCC_CAMSS_CSI0PIX_CLK			59
-+#define GCC_CAMSS_CSI0RDI_CLK			60
-+#define GCC_CAMSS_CSI0_AHB_CLK			61
-+#define GCC_CAMSS_CSI0_CLK			62
-+#define GCC_CAMSS_CSI1PHYTIMER_CLK		63
-+#define GCC_CAMSS_CSI1PHY_CLK			64
-+#define GCC_CAMSS_CSI1PIX_CLK			65
-+#define GCC_CAMSS_CSI1RDI_CLK			66
-+#define GCC_CAMSS_CSI1_AHB_CLK			67
-+#define GCC_CAMSS_CSI1_CLK			68
-+#define GCC_CAMSS_CSI2PHY_CLK			69
-+#define GCC_CAMSS_CSI2PIX_CLK			70
-+#define GCC_CAMSS_CSI2RDI_CLK			71
-+#define GCC_CAMSS_CSI2_AHB_CLK			72
-+#define GCC_CAMSS_CSI2_CLK			73
-+#define GCC_CAMSS_CSI_VFE0_CLK			74
-+#define GCC_CAMSS_CSI_VFE1_CLK			75
-+#define GCC_CAMSS_GP0_CLK			76
-+#define GCC_CAMSS_GP1_CLK			77
-+#define GCC_CAMSS_ISPIF_AHB_CLK			78
-+#define GCC_CAMSS_JPEG0_CLK			79
-+#define GCC_CAMSS_JPEG_AHB_CLK			80
-+#define GCC_CAMSS_JPEG_AXI_CLK			81
-+#define GCC_CAMSS_MCLK0_CLK			82
-+#define GCC_CAMSS_MCLK1_CLK			83
-+#define GCC_CAMSS_MCLK2_CLK			84
-+#define GCC_CAMSS_MICRO_AHB_CLK			85
-+#define GCC_CAMSS_TOP_AHB_CLK			86
-+#define GCC_CAMSS_VFE0_AHB_CLK			87
-+#define GCC_CAMSS_VFE0_AXI_CLK			88
-+#define GCC_CAMSS_VFE0_CLK			89
-+#define GCC_CAMSS_VFE1_AHB_CLK			90
-+#define GCC_CAMSS_VFE1_AXI_CLK			91
-+#define GCC_CAMSS_VFE1_CLK			92
-+#define GCC_CPP_TBU_CLK				93
-+#define GCC_CRYPTO_AHB_CLK			94
-+#define GCC_CRYPTO_AXI_CLK			95
-+#define GCC_CRYPTO_CLK				96
-+#define GCC_DCC_CLK				97
-+#define GCC_GFX_TBU_CLK				98
-+#define GCC_GFX_TCU_CLK				99
-+#define GCC_GP1_CLK				100
-+#define GCC_GP2_CLK				101
-+#define GCC_GP3_CLK				102
-+#define GCC_GTCU_AHB_CLK			103
-+#define GCC_JPEG_TBU_CLK			104
-+#define GCC_MDP_TBU_CLK				105
-+#define GCC_MDSS_AHB_CLK			106
-+#define GCC_MDSS_AXI_CLK			107
-+#define GCC_MDSS_BYTE0_CLK			108
-+#define GCC_MDSS_ESC0_CLK			109
-+#define GCC_MDSS_MDP_CLK			110
-+#define GCC_MDSS_PCLK0_CLK			111
-+#define GCC_MDSS_VSYNC_CLK			112
-+#define GCC_MSS_CFG_AHB_CLK			113
-+#define GCC_MSS_Q6_BIMC_AXI_CLK			114
-+#define GCC_OXILI_AHB_CLK			115
-+#define GCC_OXILI_GFX3D_CLK			116
-+#define GCC_PDM2_CLK				117
-+#define GCC_PDM_AHB_CLK				118
-+#define GCC_PRNG_AHB_CLK			119
-+#define GCC_QDSS_DAP_CLK			120
-+#define GCC_SDCC1_AHB_CLK			121
-+#define GCC_SDCC1_APPS_CLK			122
-+#define GCC_SDCC1_ICE_CORE_CLK			123
-+#define GCC_SDCC2_AHB_CLK			124
-+#define GCC_SDCC2_APPS_CLK			125
-+#define GCC_SMMU_CFG_CLK			126
-+#define GCC_USB2A_PHY_SLEEP_CLK			127
-+#define GCC_USB_HS_AHB_CLK			128
-+#define GCC_USB_HS_PHY_CFG_AHB_CLK		129
-+#define GCC_USB_HS_SYSTEM_CLK			130
-+#define GCC_VENUS0_AHB_CLK			131
-+#define GCC_VENUS0_AXI_CLK			132
-+#define GCC_VENUS0_CORE0_VCODEC0_CLK		133
-+#define GCC_VENUS0_VCODEC0_CLK			134
-+#define GCC_VENUS_TBU_CLK			135
-+#define GCC_VFE1_TBU_CLK			136
-+#define GCC_VFE_TBU_CLK				137
-+#define GFX3D_CLK_SRC				138
-+#define GP1_CLK_SRC				139
-+#define GP2_CLK_SRC				140
-+#define GP3_CLK_SRC				141
-+#define GPLL0					142
-+#define GPLL0_EARLY				143
-+#define GPLL3					144
-+#define GPLL3_EARLY				145
-+#define GPLL4					146
-+#define GPLL4_EARLY				147
-+#define GPLL6					148
-+#define GPLL6_EARLY				149
-+#define JPEG0_CLK_SRC				150
-+#define MCLK0_CLK_SRC				151
-+#define MCLK1_CLK_SRC				152
-+#define MCLK2_CLK_SRC				153
-+#define MDP_CLK_SRC				154
-+#define PCLK0_CLK_SRC				155
-+#define PDM2_CLK_SRC				156
-+#define SDCC1_APPS_CLK_SRC			157
-+#define SDCC1_ICE_CORE_CLK_SRC			158
-+#define SDCC2_APPS_CLK_SRC			159
-+#define USB_HS_SYSTEM_CLK_SRC			160
-+#define VCODEC0_CLK_SRC				161
-+#define VFE0_CLK_SRC				162
-+#define VFE1_CLK_SRC				163
-+#define VSYNC_CLK_SRC				164
-+#define GPLL0_SLEEP_CLK_SRC			165
-+/* Addtional MSM8937-specific clocks */
-+#define MSM8937_BLSP1_QUP1_I2C_APPS_CLK_SRC		166
-+#define MSM8937_BLSP1_QUP1_SPI_APPS_CLK_SRC		167
-+#define MSM8937_BLSP2_QUP4_I2C_APPS_CLK_SRC		168
-+#define MSM8937_BLSP2_QUP4_SPI_APPS_CLK_SRC		169
-+#define MSM8937_BYTE1_CLK_SRC				170
-+#define MSM8937_ESC1_CLK_SRC				171
-+#define MSM8937_PCLK1_CLK_SRC				172
-+#define MSM8937_GCC_BLSP1_QUP1_I2C_APPS_CLK		173
-+#define MSM8937_GCC_BLSP1_QUP1_SPI_APPS_CLK		174
-+#define MSM8937_GCC_BLSP2_QUP4_I2C_APPS_CLK		175
-+#define MSM8937_GCC_BLSP2_QUP4_SPI_APPS_CLK		176
-+#define MSM8937_GCC_MDSS_BYTE1_CLK			177
-+#define MSM8937_GCC_MDSS_ESC1_CLK			178
-+#define MSM8937_GCC_MDSS_PCLK1_CLK			179
-+#define MSM8937_GCC_OXILI_AON_CLK			180
-+#define MSM8937_GCC_OXILI_TIMER_CLK			181
-+
-+/* GCC block resets */
-+#define GCC_CAMSS_MICRO_BCR			0
-+#define GCC_MSS_BCR				1
-+#define GCC_QUSB2_PHY_BCR			2
-+#define GCC_USB_HS_BCR				3
-+#define GCC_USB2_HS_PHY_ONLY_BCR		4
-+
-+/* GDSCs */
-+#define CPP_GDSC				0
-+#define JPEG_GDSC				1
-+#define MDSS_GDSC				2
-+#define OXILI_GX_GDSC				3
-+#define VENUS_CORE0_GDSC			4
-+#define VENUS_GDSC				5
-+#define VFE0_GDSC				6
-+#define VFE1_GDSC				7
-+/* Additional MSM8937-specific GDSCs */
-+#define MSM8937_OXILI_CX_GDSC				8
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,gcc-msm8939.h b/include/dt-bindings/clock/qcom,gcc-msm8939.h
-index 2d545ed0d35a..9a9bc55b49af 100644
--- a/include/dt-bindings/clock/qcom,gcc-msm8939.h
-+++ b/include/dt-bindings/clock/qcom,gcc-msm8939.h
-@@ -193,6 +193,12 @@
- #define GCC_VENUS0_CORE1_VCODEC0_CLK		184
- #define GCC_OXILI_TIMER_CLK			185
- #define SYSTEM_MM_NOC_BFDCD_CLK_SRC		186
-+#define CSI2_CLK_SRC				187
-+#define GCC_CAMSS_CSI2_AHB_CLK			188
-+#define GCC_CAMSS_CSI2_CLK			189
-+#define GCC_CAMSS_CSI2PHY_CLK			190
-+#define GCC_CAMSS_CSI2PIX_CLK			191
-+#define GCC_CAMSS_CSI2RDI_CLK			192
- 
- /* Indexes for GDSCs */
- #define BIMC_GDSC				0
-diff --git a/include/dt-bindings/clock/qcom,gcc-msm8953.h b/include/dt-bindings/clock/qcom,gcc-msm8953.h
-index 783162da6148..13b4a62877e5 100644
--- a/include/dt-bindings/clock/qcom,gcc-msm8953.h
-+++ b/include/dt-bindings/clock/qcom,gcc-msm8953.h
-@@ -218,6 +218,10 @@
- #define GCC_USB3PHY_PHY_BCR			3
- #define GCC_USB3_PHY_BCR			4
- #define GCC_USB_30_BCR				5
-+#define GCC_MDSS_BCR				6
-+#define GCC_CRYPTO_BCR				7
-+#define GCC_SDCC1_BCR				8
-+#define GCC_SDCC2_BCR				9
- 
- /* GDSCs */
- #define CPP_GDSC				0
-diff --git a/include/dt-bindings/clock/qcom,gcc-msm8998.h b/include/dt-bindings/clock/qcom,gcc-msm8998.h
-index 1badb4f9c58f..5b0dde080900 100644
--- a/include/dt-bindings/clock/qcom,gcc-msm8998.h
-+++ b/include/dt-bindings/clock/qcom,gcc-msm8998.h
-@@ -190,10 +190,18 @@
- #define AGGRE2_SNOC_NORTH_AXI					181
- #define SSC_XO							182
- #define SSC_CNOC_AHBS_CLK					183
-+#define GCC_MMSS_GPLL0_DIV_CLK					184
-+#define GCC_GPU_GPLL0_DIV_CLK					185
-+#define GCC_GPU_GPLL0_CLK					186
-+#define HLOS1_VOTE_LPASS_CORE_SMMU_CLK				187
-+#define HLOS1_VOTE_LPASS_ADSP_SMMU_CLK				188
-+#define GCC_MSS_Q6_BIMC_AXI_CLK					189
- 
- #define PCIE_0_GDSC						0
- #define UFS_GDSC						1
- #define USB_30_GDSC						2
-+#define LPASS_ADSP_GDSC						3
-+#define LPASS_CORE_GDSC						4
- 
- #define GCC_BLSP1_QUP1_BCR					0
- #define GCC_BLSP1_QUP2_BCR					1
-diff --git a/include/dt-bindings/clock/qcom,gcc-qcs404.h b/include/dt-bindings/clock/qcom,gcc-qcs404.h
-index bc3051543347..126a51898571 100644
--- a/include/dt-bindings/clock/qcom,gcc-qcs404.h
-+++ b/include/dt-bindings/clock/qcom,gcc-qcs404.h
-@@ -177,4 +177,8 @@
- #define GCC_PCIE_0_PIPE_ARES				21
- #define GCC_WDSP_RESTART				22
- 
-+/* Indexes for GDSCs */
-+#define MDSS_GDSC				0
-+#define OXILI_GDSC				1
-+
- #endif
-diff --git a/include/dt-bindings/clock/qcom,gcc-sc8180x.h b/include/dt-bindings/clock/qcom,gcc-sc8180x.h
-index e893415ae13d..b9d8438a15ff 100644
--- a/include/dt-bindings/clock/qcom,gcc-sc8180x.h
-+++ b/include/dt-bindings/clock/qcom,gcc-sc8180x.h
-@@ -246,6 +246,19 @@
- #define GCC_PCIE_3_CLKREF_CLK					236
- #define GCC_USB3_PRIM_CLKREF_CLK				237
- #define GCC_USB3_SEC_CLKREF_CLK					238
-+#define GCC_UFS_MEM_CLKREF_EN					239
-+#define GCC_UFS_CARD_CLKREF_EN					240
-+#define GPLL9							241
-+#define GCC_CAMERA_AHB_CLK					242
-+#define GCC_CAMERA_XO_CLK					243
-+#define GCC_CPUSS_DVM_BUS_CLK					244
-+#define GCC_CPUSS_GNOC_CLK					245
-+#define GCC_DISP_AHB_CLK					246
-+#define GCC_DISP_XO_CLK						247
-+#define GCC_GPU_CFG_AHB_CLK					248
-+#define GCC_NPU_CFG_AHB_CLK					249
-+#define GCC_VIDEO_AHB_CLK					250
-+#define GCC_VIDEO_XO_CLK					251
- 
- #define GCC_EMAC_BCR						0
- #define GCC_GPU_BCR						1
-@@ -292,6 +305,10 @@
- #define GCC_VIDEO_AXI0_CLK_BCR					42
- #define GCC_VIDEO_AXI1_CLK_BCR					43
- #define GCC_USB3_DP_PHY_SEC_BCR					44
-+#define GCC_USB3_UNIPHY_MP0_BCR					45
-+#define GCC_USB3_UNIPHY_MP1_BCR					46
-+#define GCC_USB3UNIPHY_PHY_MP0_BCR				47
-+#define GCC_USB3UNIPHY_PHY_MP1_BCR				48
- 
- /* GCC GDSCRs */
- #define EMAC_GDSC						0
-diff --git a/include/dt-bindings/clock/qcom,gcc-sc8280xp.h b/include/dt-bindings/clock/qcom,gcc-sc8280xp.h
-index cb2fb638825c..845491591784 100644
--- a/include/dt-bindings/clock/qcom,gcc-sc8280xp.h
-+++ b/include/dt-bindings/clock/qcom,gcc-sc8280xp.h
-@@ -492,5 +492,17 @@
- #define USB30_MP_GDSC					9
- #define USB30_PRIM_GDSC					10
- #define USB30_SEC_GDSC					11
-+#define EMAC_0_GDSC					12
-+#define EMAC_1_GDSC					13
-+#define USB4_1_GDSC					14
-+#define USB4_GDSC					15
-+#define HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC		16
-+#define HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC		17
-+#define HLOS1_VOTE_MMNOC_MMU_TBU_SF0_GDSC		18
-+#define HLOS1_VOTE_MMNOC_MMU_TBU_SF1_GDSC		19
-+#define HLOS1_VOTE_TURING_MMU_TBU0_GDSC			20
-+#define HLOS1_VOTE_TURING_MMU_TBU1_GDSC			21
-+#define HLOS1_VOTE_TURING_MMU_TBU2_GDSC			22
-+#define HLOS1_VOTE_TURING_MMU_TBU3_GDSC			23
- 
- #endif
-diff --git a/include/dt-bindings/clock/qcom,gcc-sdm660.h b/include/dt-bindings/clock/qcom,gcc-sdm660.h
-index df8a6f3d367e..f19018b742f5 100644
--- a/include/dt-bindings/clock/qcom,gcc-sdm660.h
-+++ b/include/dt-bindings/clock/qcom,gcc-sdm660.h
-@@ -138,10 +138,16 @@
- #define GCC_UFS_UNIPRO_CORE_HW_CTL_CLK		128
- #define GCC_RX0_USB2_CLKREF_CLK			129
- #define GCC_RX1_USB2_CLKREF_CLK			130
-+#define GCC_HLOS1_VOTE_LPASS_ADSP_SMMU_CLK	131
-+#define GCC_HLOS1_VOTE_TURING_ADSP_SMMU_CLK	132
-+#define GCC_HLOS2_VOTE_TURING_ADSP_SMMU_CLK	133
- 
- #define PCIE_0_GDSC	0
- #define UFS_GDSC	1
- #define USB_30_GDSC	2
-+#define HLOS1_VOTE_TURING_ADSP_GDSC 3
-+#define HLOS2_VOTE_TURING_ADSP_GDSC 4
-+#define HLOS1_VOTE_LPASS_ADSP_GDSC 5
- 
- #define GCC_QUSB2PHY_PRIM_BCR		0
- #define GCC_QUSB2PHY_SEC_BCR		1
-@@ -153,5 +159,7 @@
- #define GCC_USB_30_BCR			7
- #define GCC_USB_PHY_CFG_AHB2PHY_BCR	8
- #define GCC_MSS_RESTART			9
-+#define GCC_SDCC1_BCR			10
-+#define GCC_SDCC2_BCR			11
- 
- #endif
-diff --git a/include/dt-bindings/clock/qcom,gcc-sdm845.h b/include/dt-bindings/clock/qcom,gcc-sdm845.h
-index 968fa65b9c42..d78b899263a2 100644
--- a/include/dt-bindings/clock/qcom,gcc-sdm845.h
-+++ b/include/dt-bindings/clock/qcom,gcc-sdm845.h
-@@ -199,6 +199,7 @@
- #define GCC_QSPI_CNOC_PERIPH_AHB_CLK				189
- #define GCC_LPASS_Q6_AXI_CLK					190
- #define GCC_LPASS_SWAY_CLK					191
-+#define GPLL6							192
- 
- /* GCC Resets */
- #define GCC_MMSS_BCR						0
-diff --git a/include/dt-bindings/clock/qcom,gcc-sm8150.h b/include/dt-bindings/clock/qcom,gcc-sm8150.h
-index dfefd5e8bf6e..921a33f24d33 100644
--- a/include/dt-bindings/clock/qcom,gcc-sm8150.h
-+++ b/include/dt-bindings/clock/qcom,gcc-sm8150.h
-@@ -239,6 +239,9 @@
- #define GCC_USB30_PRIM_BCR					26
- #define GCC_USB30_SEC_BCR					27
- #define GCC_USB_PHY_CFG_AHB2PHY_BCR				28
-+#define GCC_VIDEO_AXIC_CLK_BCR					29
-+#define GCC_VIDEO_AXI0_CLK_BCR					30
-+#define GCC_VIDEO_AXI1_CLK_BCR					31
- 
- /* GCC GDSCRs */
- #define PCIE_0_GDSC						0
-diff --git a/include/dt-bindings/clock/qcom,gcc-sm8350.h b/include/dt-bindings/clock/qcom,gcc-sm8350.h
-index f6be3da5f781..529c1b8b0417 100644
--- a/include/dt-bindings/clock/qcom,gcc-sm8350.h
-+++ b/include/dt-bindings/clock/qcom,gcc-sm8350.h
-@@ -8,7 +8,6 @@
- #define _DT_BINDINGS_CLK_QCOM_GCC_SM8350_H
- 
- /* GCC HW clocks */
-#define CORE_BI_PLL_TEST_SE					0
- #define PCIE_0_PIPE_CLK						1
- #define PCIE_1_PIPE_CLK						2
- #define UFS_CARD_RX_SYMBOL_0_CLK				3
-diff --git a/include/dt-bindings/clock/qcom,gcc-sm8450.h b/include/dt-bindings/clock/qcom,gcc-sm8450.h
-index cf1469312c4c..7320e63c3a2f 100644
--- a/include/dt-bindings/clock/qcom,gcc-sm8450.h
-+++ b/include/dt-bindings/clock/qcom,gcc-sm8450.h
-@@ -8,7 +8,6 @@
- #define _DT_BINDINGS_CLK_QCOM_GCC_SM8450_H
- 
- /* GCC HW clocks */
-#define CORE_BI_PLL_TEST_SE					0
- #define PCIE_0_PIPE_CLK						1
- #define PCIE_1_PHY_AUX_CLK					2
- #define PCIE_1_PIPE_CLK						3
-@@ -195,6 +194,9 @@
- #define GCC_VIDEO_AXI0_CLK					182
- #define GCC_VIDEO_AXI1_CLK					183
- #define GCC_VIDEO_XO_CLK					184
-+/* Additional SM8475-specific clocks */
-+#define SM8475_GCC_GPLL2					185
-+#define SM8475_GCC_GPLL3					186
- 
- /* GCC resets */
- #define GCC_CAMERA_BCR						0
-diff --git a/include/dt-bindings/clock/qcom,glymur-dispcc.h b/include/dt-bindings/clock/qcom,glymur-dispcc.h
-new file mode 100644
-index 000000000000..a845d76defe2
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,glymur-dispcc.h
-@@ -0,0 +1,114 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2025, Qualcomm Technologies, Inc. and/or its subsidiaries.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_GLYMUR_H
-+#define _DT_BINDINGS_CLK_QCOM_DISP_CC_GLYMUR_H
-+
-+/* DISP_CC clocks */
-+#define DISP_CC_ESYNC0_CLK					0
-+#define DISP_CC_ESYNC0_CLK_SRC					1
-+#define DISP_CC_ESYNC1_CLK					2
-+#define DISP_CC_ESYNC1_CLK_SRC					3
-+#define DISP_CC_MDSS_ACCU_SHIFT_CLK				4
-+#define DISP_CC_MDSS_AHB1_CLK					5
-+#define DISP_CC_MDSS_AHB_CLK					6
-+#define DISP_CC_MDSS_AHB_CLK_SRC				7
-+#define DISP_CC_MDSS_BYTE0_CLK					8
-+#define DISP_CC_MDSS_BYTE0_CLK_SRC				9
-+#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC				10
-+#define DISP_CC_MDSS_BYTE0_INTF_CLK				11
-+#define DISP_CC_MDSS_BYTE1_CLK					12
-+#define DISP_CC_MDSS_BYTE1_CLK_SRC				13
-+#define DISP_CC_MDSS_BYTE1_DIV_CLK_SRC				14
-+#define DISP_CC_MDSS_BYTE1_INTF_CLK				15
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK				16
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK_SRC				17
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK				18
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK_SRC				19
-+#define DISP_CC_MDSS_DPTX0_LINK_DIV_CLK_SRC			20
-+#define DISP_CC_MDSS_DPTX0_LINK_DPIN_CLK			21
-+#define DISP_CC_MDSS_DPTX0_LINK_DPIN_DIV_CLK_SRC		22
-+#define DISP_CC_MDSS_DPTX0_LINK_INTF_CLK			23
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK				24
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC			25
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK				26
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC			27
-+#define DISP_CC_MDSS_DPTX0_USB_ROUTER_LINK_INTF_CLK		28
-+#define DISP_CC_MDSS_DPTX1_AUX_CLK				29
-+#define DISP_CC_MDSS_DPTX1_AUX_CLK_SRC				30
-+#define DISP_CC_MDSS_DPTX1_LINK_CLK				31
-+#define DISP_CC_MDSS_DPTX1_LINK_CLK_SRC				32
-+#define DISP_CC_MDSS_DPTX1_LINK_DIV_CLK_SRC			33
-+#define DISP_CC_MDSS_DPTX1_LINK_DPIN_CLK			34
-+#define DISP_CC_MDSS_DPTX1_LINK_DPIN_DIV_CLK_SRC		35
-+#define DISP_CC_MDSS_DPTX1_LINK_INTF_CLK			36
-+#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK				37
-+#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK_SRC			38
-+#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK				39
-+#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK_SRC			40
-+#define DISP_CC_MDSS_DPTX1_USB_ROUTER_LINK_INTF_CLK		41
-+#define DISP_CC_MDSS_DPTX2_AUX_CLK				42
-+#define DISP_CC_MDSS_DPTX2_AUX_CLK_SRC				43
-+#define DISP_CC_MDSS_DPTX2_LINK_CLK				44
-+#define DISP_CC_MDSS_DPTX2_LINK_CLK_SRC				45
-+#define DISP_CC_MDSS_DPTX2_LINK_DIV_CLK_SRC			46
-+#define DISP_CC_MDSS_DPTX2_LINK_DPIN_CLK			47
-+#define DISP_CC_MDSS_DPTX2_LINK_DPIN_DIV_CLK_SRC		48
-+#define DISP_CC_MDSS_DPTX2_LINK_INTF_CLK			49
-+#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK				50
-+#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK_SRC			51
-+#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK				52
-+#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK_SRC			53
-+#define DISP_CC_MDSS_DPTX2_USB_ROUTER_LINK_INTF_CLK		54
-+#define DISP_CC_MDSS_DPTX3_AUX_CLK				55
-+#define DISP_CC_MDSS_DPTX3_AUX_CLK_SRC				56
-+#define DISP_CC_MDSS_DPTX3_LINK_CLK				57
-+#define DISP_CC_MDSS_DPTX3_LINK_CLK_SRC				58
-+#define DISP_CC_MDSS_DPTX3_LINK_DIV_CLK_SRC			59
-+#define DISP_CC_MDSS_DPTX3_LINK_DPIN_CLK			60
-+#define DISP_CC_MDSS_DPTX3_LINK_DPIN_DIV_CLK_SRC		61
-+#define DISP_CC_MDSS_DPTX3_LINK_INTF_CLK			62
-+#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK				63
-+#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK_SRC			64
-+#define DISP_CC_MDSS_ESC0_CLK					65
-+#define DISP_CC_MDSS_ESC0_CLK_SRC				66
-+#define DISP_CC_MDSS_ESC1_CLK					67
-+#define DISP_CC_MDSS_ESC1_CLK_SRC				68
-+#define DISP_CC_MDSS_MDP1_CLK					69
-+#define DISP_CC_MDSS_MDP_CLK					70
-+#define DISP_CC_MDSS_MDP_CLK_SRC				71
-+#define DISP_CC_MDSS_MDP_LUT1_CLK				72
-+#define DISP_CC_MDSS_MDP_LUT_CLK				73
-+#define DISP_CC_MDSS_NON_GDSC_AHB_CLK				74
-+#define DISP_CC_MDSS_PCLK0_CLK					75
-+#define DISP_CC_MDSS_PCLK0_CLK_SRC				76
-+#define DISP_CC_MDSS_PCLK1_CLK					77
-+#define DISP_CC_MDSS_PCLK1_CLK_SRC				78
-+#define DISP_CC_MDSS_PCLK2_CLK					79
-+#define DISP_CC_MDSS_PCLK2_CLK_SRC				80
-+#define DISP_CC_MDSS_RSCC_AHB_CLK				81
-+#define DISP_CC_MDSS_RSCC_VSYNC_CLK				82
-+#define DISP_CC_MDSS_VSYNC1_CLK					83
-+#define DISP_CC_MDSS_VSYNC_CLK					84
-+#define DISP_CC_MDSS_VSYNC_CLK_SRC				85
-+#define DISP_CC_OSC_CLK						86
-+#define DISP_CC_OSC_CLK_SRC					87
-+#define DISP_CC_PLL0						88
-+#define DISP_CC_PLL1						89
-+#define DISP_CC_SLEEP_CLK					90
-+#define DISP_CC_SLEEP_CLK_SRC					91
-+#define DISP_CC_XO_CLK						92
-+#define DISP_CC_XO_CLK_SRC					93
-+
-+/* DISP_CC power domains */
-+#define DISP_CC_MDSS_CORE_GDSC					0
-+#define DISP_CC_MDSS_CORE_INT2_GDSC				1
-+
-+/* DISP_CC resets */
-+#define DISP_CC_MDSS_CORE_BCR					0
-+#define DISP_CC_MDSS_CORE_INT2_BCR				1
-+#define DISP_CC_MDSS_RSCC_BCR					2
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,glymur-gcc.h b/include/dt-bindings/clock/qcom,glymur-gcc.h
-new file mode 100644
-index 000000000000..10c12b8c51c3
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,glymur-gcc.h
-@@ -0,0 +1,578 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_GLYMUR_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_GLYMUR_H
-+
-+/* GCC clocks */
-+#define GCC_GPLL0						0
-+#define GCC_GPLL0_OUT_EVEN					1
-+#define GCC_GPLL1						2
-+#define GCC_GPLL14						3
-+#define GCC_GPLL14_OUT_EVEN					4
-+#define GCC_GPLL4						5
-+#define GCC_GPLL5						6
-+#define GCC_GPLL7						7
-+#define GCC_GPLL8						8
-+#define GCC_GPLL9						9
-+#define GCC_AGGRE_NOC_PCIE_3A_WEST_SF_AXI_CLK			10
-+#define GCC_AGGRE_NOC_PCIE_3B_WEST_SF_AXI_CLK			11
-+#define GCC_AGGRE_NOC_PCIE_4_WEST_SF_AXI_CLK			12
-+#define GCC_AGGRE_NOC_PCIE_5_EAST_SF_AXI_CLK			13
-+#define GCC_AGGRE_NOC_PCIE_6_WEST_SF_AXI_CLK			14
-+#define GCC_AGGRE_UFS_PHY_AXI_CLK				15
-+#define GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK			16
-+#define GCC_AGGRE_USB2_PRIM_AXI_CLK				17
-+#define GCC_AGGRE_USB3_MP_AXI_CLK				18
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK				19
-+#define GCC_AGGRE_USB3_SEC_AXI_CLK				20
-+#define GCC_AGGRE_USB3_TERT_AXI_CLK				21
-+#define GCC_AGGRE_USB4_0_AXI_CLK				22
-+#define GCC_AGGRE_USB4_1_AXI_CLK				23
-+#define GCC_AGGRE_USB4_2_AXI_CLK				24
-+#define GCC_AV1E_AHB_CLK					25
-+#define GCC_AV1E_AXI_CLK					26
-+#define GCC_AV1E_XO_CLK						27
-+#define GCC_BOOT_ROM_AHB_CLK					28
-+#define GCC_CAMERA_AHB_CLK					29
-+#define GCC_CAMERA_HF_AXI_CLK					30
-+#define GCC_CAMERA_SF_AXI_CLK					31
-+#define GCC_CAMERA_XO_CLK					32
-+#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK				33
-+#define GCC_CFG_NOC_PCIE_ANOC_SOUTH_AHB_CLK			34
-+#define GCC_CFG_NOC_USB2_PRIM_AXI_CLK				35
-+#define GCC_CFG_NOC_USB3_MP_AXI_CLK				36
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				37
-+#define GCC_CFG_NOC_USB3_SEC_AXI_CLK				38
-+#define GCC_CFG_NOC_USB3_TERT_AXI_CLK				39
-+#define GCC_CFG_NOC_USB_ANOC_AHB_CLK				40
-+#define GCC_CFG_NOC_USB_ANOC_SOUTH_AHB_CLK			41
-+#define GCC_DISP_AHB_CLK					42
-+#define GCC_DISP_HF_AXI_CLK					43
-+#define GCC_EVA_AHB_CLK						44
-+#define GCC_EVA_AXI0_CLK					45
-+#define GCC_EVA_AXI0C_CLK					46
-+#define GCC_EVA_XO_CLK						47
-+#define GCC_GP1_CLK						48
-+#define GCC_GP1_CLK_SRC						49
-+#define GCC_GP2_CLK						50
-+#define GCC_GP2_CLK_SRC						51
-+#define GCC_GP3_CLK						52
-+#define GCC_GP3_CLK_SRC						53
-+#define GCC_GPU_CFG_AHB_CLK					54
-+#define GCC_GPU_GEMNOC_GFX_CLK					55
-+#define GCC_GPU_GPLL0_CLK_SRC					56
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC				57
-+#define GCC_PCIE_0_AUX_CLK					58
-+#define GCC_PCIE_0_AUX_CLK_SRC					59
-+#define GCC_PCIE_0_CFG_AHB_CLK					60
-+#define GCC_PCIE_0_MSTR_AXI_CLK					61
-+#define GCC_PCIE_0_PHY_RCHNG_CLK				62
-+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC				63
-+#define GCC_PCIE_0_PIPE_CLK					64
-+#define GCC_PCIE_0_SLV_AXI_CLK					65
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				66
-+#define GCC_PCIE_1_AUX_CLK					67
-+#define GCC_PCIE_1_AUX_CLK_SRC					68
-+#define GCC_PCIE_1_CFG_AHB_CLK					69
-+#define GCC_PCIE_1_MSTR_AXI_CLK					70
-+#define GCC_PCIE_1_PHY_RCHNG_CLK				71
-+#define GCC_PCIE_1_PHY_RCHNG_CLK_SRC				72
-+#define GCC_PCIE_1_PIPE_CLK					73
-+#define GCC_PCIE_1_SLV_AXI_CLK					74
-+#define GCC_PCIE_1_SLV_Q2A_AXI_CLK				75
-+#define GCC_PCIE_2_AUX_CLK					76
-+#define GCC_PCIE_2_AUX_CLK_SRC					77
-+#define GCC_PCIE_2_CFG_AHB_CLK					78
-+#define GCC_PCIE_2_MSTR_AXI_CLK					79
-+#define GCC_PCIE_2_PHY_RCHNG_CLK				80
-+#define GCC_PCIE_2_PHY_RCHNG_CLK_SRC				81
-+#define GCC_PCIE_2_PIPE_CLK					82
-+#define GCC_PCIE_2_SLV_AXI_CLK					83
-+#define GCC_PCIE_2_SLV_Q2A_AXI_CLK				84
-+#define GCC_PCIE_3A_AUX_CLK					85
-+#define GCC_PCIE_3A_AUX_CLK_SRC					86
-+#define GCC_PCIE_3A_CFG_AHB_CLK					87
-+#define GCC_PCIE_3A_MSTR_AXI_CLK				88
-+#define GCC_PCIE_3A_PHY_RCHNG_CLK				89
-+#define GCC_PCIE_3A_PHY_RCHNG_CLK_SRC				90
-+#define GCC_PCIE_3A_PIPE_CLK					91
-+#define GCC_PCIE_3A_PIPE_CLK_SRC				92
-+#define GCC_PCIE_3A_SLV_AXI_CLK					93
-+#define GCC_PCIE_3A_SLV_Q2A_AXI_CLK				94
-+#define GCC_PCIE_3B_AUX_CLK					95
-+#define GCC_PCIE_3B_AUX_CLK_SRC					96
-+#define GCC_PCIE_3B_CFG_AHB_CLK					97
-+#define GCC_PCIE_3B_MSTR_AXI_CLK				98
-+#define GCC_PCIE_3B_PHY_RCHNG_CLK				99
-+#define GCC_PCIE_3B_PHY_RCHNG_CLK_SRC				100
-+#define GCC_PCIE_3B_PIPE_CLK					101
-+#define GCC_PCIE_3B_PIPE_CLK_SRC				102
-+#define GCC_PCIE_3B_PIPE_DIV2_CLK				103
-+#define GCC_PCIE_3B_PIPE_DIV_CLK_SRC				104
-+#define GCC_PCIE_3B_SLV_AXI_CLK					105
-+#define GCC_PCIE_3B_SLV_Q2A_AXI_CLK				106
-+#define GCC_PCIE_4_AUX_CLK					107
-+#define GCC_PCIE_4_AUX_CLK_SRC					108
-+#define GCC_PCIE_4_CFG_AHB_CLK					109
-+#define GCC_PCIE_4_MSTR_AXI_CLK					110
-+#define GCC_PCIE_4_PHY_RCHNG_CLK				111
-+#define GCC_PCIE_4_PHY_RCHNG_CLK_SRC				112
-+#define GCC_PCIE_4_PIPE_CLK					113
-+#define GCC_PCIE_4_PIPE_CLK_SRC					114
-+#define GCC_PCIE_4_PIPE_DIV2_CLK				115
-+#define GCC_PCIE_4_PIPE_DIV_CLK_SRC				116
-+#define GCC_PCIE_4_SLV_AXI_CLK					117
-+#define GCC_PCIE_4_SLV_Q2A_AXI_CLK				118
-+#define GCC_PCIE_5_AUX_CLK					119
-+#define GCC_PCIE_5_AUX_CLK_SRC					120
-+#define GCC_PCIE_5_CFG_AHB_CLK					121
-+#define GCC_PCIE_5_MSTR_AXI_CLK					122
-+#define GCC_PCIE_5_PHY_RCHNG_CLK				123
-+#define GCC_PCIE_5_PHY_RCHNG_CLK_SRC				124
-+#define GCC_PCIE_5_PIPE_CLK					125
-+#define GCC_PCIE_5_PIPE_CLK_SRC					126
-+#define GCC_PCIE_5_PIPE_DIV2_CLK				127
-+#define GCC_PCIE_5_PIPE_DIV_CLK_SRC				128
-+#define GCC_PCIE_5_SLV_AXI_CLK					129
-+#define GCC_PCIE_5_SLV_Q2A_AXI_CLK				130
-+#define GCC_PCIE_6_AUX_CLK					131
-+#define GCC_PCIE_6_AUX_CLK_SRC					132
-+#define GCC_PCIE_6_CFG_AHB_CLK					133
-+#define GCC_PCIE_6_MSTR_AXI_CLK					134
-+#define GCC_PCIE_6_PHY_RCHNG_CLK				135
-+#define GCC_PCIE_6_PHY_RCHNG_CLK_SRC				136
-+#define GCC_PCIE_6_PIPE_CLK					137
-+#define GCC_PCIE_6_PIPE_CLK_SRC					138
-+#define GCC_PCIE_6_PIPE_DIV2_CLK				139
-+#define GCC_PCIE_6_PIPE_DIV_CLK_SRC				140
-+#define GCC_PCIE_6_SLV_AXI_CLK					141
-+#define GCC_PCIE_6_SLV_Q2A_AXI_CLK				142
-+#define GCC_PCIE_NOC_PWRCTL_CLK					143
-+#define GCC_PCIE_NOC_QOSGEN_EXTREF_CLK				144
-+#define GCC_PCIE_NOC_SF_CENTER_CLK				145
-+#define GCC_PCIE_NOC_SLAVE_SF_EAST_CLK				146
-+#define GCC_PCIE_NOC_SLAVE_SF_WEST_CLK				147
-+#define GCC_PCIE_NOC_TSCTR_CLK					148
-+#define GCC_PCIE_PHY_3A_AUX_CLK					149
-+#define GCC_PCIE_PHY_3A_AUX_CLK_SRC				150
-+#define GCC_PCIE_PHY_3B_AUX_CLK					151
-+#define GCC_PCIE_PHY_3B_AUX_CLK_SRC				152
-+#define GCC_PCIE_PHY_4_AUX_CLK					153
-+#define GCC_PCIE_PHY_4_AUX_CLK_SRC				154
-+#define GCC_PCIE_PHY_5_AUX_CLK					155
-+#define GCC_PCIE_PHY_5_AUX_CLK_SRC				156
-+#define GCC_PCIE_PHY_6_AUX_CLK					157
-+#define GCC_PCIE_PHY_6_AUX_CLK_SRC				158
-+#define GCC_PCIE_RSCC_CFG_AHB_CLK				159
-+#define GCC_PCIE_RSCC_XO_CLK					160
-+#define GCC_PDM2_CLK						161
-+#define GCC_PDM2_CLK_SRC					162
-+#define GCC_PDM_AHB_CLK						163
-+#define GCC_PDM_XO4_CLK						164
-+#define GCC_QMIP_AV1E_AHB_CLK					165
-+#define GCC_QMIP_CAMERA_CMD_AHB_CLK				166
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				167
-+#define GCC_QMIP_CAMERA_RT_AHB_CLK				168
-+#define GCC_QMIP_GPU_AHB_CLK					169
-+#define GCC_QMIP_PCIE_3A_AHB_CLK				170
-+#define GCC_QMIP_PCIE_3B_AHB_CLK				171
-+#define GCC_QMIP_PCIE_4_AHB_CLK					172
-+#define GCC_QMIP_PCIE_5_AHB_CLK					173
-+#define GCC_QMIP_PCIE_6_AHB_CLK					174
-+#define GCC_QMIP_VIDEO_CV_CPU_AHB_CLK				175
-+#define GCC_QMIP_VIDEO_CVP_AHB_CLK				176
-+#define GCC_QMIP_VIDEO_V_CPU_AHB_CLK				177
-+#define GCC_QMIP_VIDEO_VCODEC1_AHB_CLK				178
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				179
-+#define GCC_QUPV3_OOB_CORE_2X_CLK				180
-+#define GCC_QUPV3_OOB_CORE_CLK					181
-+#define GCC_QUPV3_OOB_M_AHB_CLK					182
-+#define GCC_QUPV3_OOB_QSPI_S0_CLK				183
-+#define GCC_QUPV3_OOB_QSPI_S0_CLK_SRC				184
-+#define GCC_QUPV3_OOB_QSPI_S1_CLK				185
-+#define GCC_QUPV3_OOB_QSPI_S1_CLK_SRC				186
-+#define GCC_QUPV3_OOB_S0_CLK					187
-+#define GCC_QUPV3_OOB_S0_CLK_SRC				188
-+#define GCC_QUPV3_OOB_S1_CLK					189
-+#define GCC_QUPV3_OOB_S1_CLK_SRC				190
-+#define GCC_QUPV3_OOB_S_AHB_CLK					191
-+#define GCC_QUPV3_OOB_TCXO_CLK					192
-+#define GCC_QUPV3_WRAP0_CORE_2X_CLK				193
-+#define GCC_QUPV3_WRAP0_CORE_CLK				194
-+#define GCC_QUPV3_WRAP0_QSPI_S2_CLK				195
-+#define GCC_QUPV3_WRAP0_QSPI_S2_CLK_SRC				196
-+#define GCC_QUPV3_WRAP0_QSPI_S3_CLK				197
-+#define GCC_QUPV3_WRAP0_QSPI_S3_CLK_SRC				198
-+#define GCC_QUPV3_WRAP0_QSPI_S6_CLK				199
-+#define GCC_QUPV3_WRAP0_QSPI_S6_CLK_SRC				200
-+#define GCC_QUPV3_WRAP0_S0_CLK					201
-+#define GCC_QUPV3_WRAP0_S0_CLK_SRC				202
-+#define GCC_QUPV3_WRAP0_S1_CLK					203
-+#define GCC_QUPV3_WRAP0_S1_CLK_SRC				204
-+#define GCC_QUPV3_WRAP0_S2_CLK					205
-+#define GCC_QUPV3_WRAP0_S2_CLK_SRC				206
-+#define GCC_QUPV3_WRAP0_S3_CLK					207
-+#define GCC_QUPV3_WRAP0_S3_CLK_SRC				208
-+#define GCC_QUPV3_WRAP0_S4_CLK					209
-+#define GCC_QUPV3_WRAP0_S4_CLK_SRC				210
-+#define GCC_QUPV3_WRAP0_S5_CLK					211
-+#define GCC_QUPV3_WRAP0_S5_CLK_SRC				212
-+#define GCC_QUPV3_WRAP0_S6_CLK					213
-+#define GCC_QUPV3_WRAP0_S6_CLK_SRC				214
-+#define GCC_QUPV3_WRAP0_S7_CLK					215
-+#define GCC_QUPV3_WRAP0_S7_CLK_SRC				216
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				217
-+#define GCC_QUPV3_WRAP1_CORE_CLK				218
-+#define GCC_QUPV3_WRAP1_QSPI_S2_CLK				219
-+#define GCC_QUPV3_WRAP1_QSPI_S2_CLK_SRC				220
-+#define GCC_QUPV3_WRAP1_QSPI_S3_CLK				221
-+#define GCC_QUPV3_WRAP1_QSPI_S3_CLK_SRC				222
-+#define GCC_QUPV3_WRAP1_QSPI_S6_CLK				223
-+#define GCC_QUPV3_WRAP1_QSPI_S6_CLK_SRC				224
-+#define GCC_QUPV3_WRAP1_S0_CLK					225
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				226
-+#define GCC_QUPV3_WRAP1_S1_CLK					227
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				228
-+#define GCC_QUPV3_WRAP1_S2_CLK					229
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				230
-+#define GCC_QUPV3_WRAP1_S3_CLK					231
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				232
-+#define GCC_QUPV3_WRAP1_S4_CLK					233
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				234
-+#define GCC_QUPV3_WRAP1_S5_CLK					235
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC				236
-+#define GCC_QUPV3_WRAP1_S6_CLK					237
-+#define GCC_QUPV3_WRAP1_S6_CLK_SRC				238
-+#define GCC_QUPV3_WRAP1_S7_CLK					239
-+#define GCC_QUPV3_WRAP1_S7_CLK_SRC				240
-+#define GCC_QUPV3_WRAP2_CORE_2X_CLK				241
-+#define GCC_QUPV3_WRAP2_CORE_CLK				242
-+#define GCC_QUPV3_WRAP2_QSPI_S2_CLK				243
-+#define GCC_QUPV3_WRAP2_QSPI_S2_CLK_SRC				244
-+#define GCC_QUPV3_WRAP2_QSPI_S3_CLK				245
-+#define GCC_QUPV3_WRAP2_QSPI_S3_CLK_SRC				246
-+#define GCC_QUPV3_WRAP2_QSPI_S6_CLK				247
-+#define GCC_QUPV3_WRAP2_QSPI_S6_CLK_SRC				248
-+#define GCC_QUPV3_WRAP2_S0_CLK					249
-+#define GCC_QUPV3_WRAP2_S0_CLK_SRC				250
-+#define GCC_QUPV3_WRAP2_S1_CLK					251
-+#define GCC_QUPV3_WRAP2_S1_CLK_SRC				252
-+#define GCC_QUPV3_WRAP2_S2_CLK					253
-+#define GCC_QUPV3_WRAP2_S2_CLK_SRC				254
-+#define GCC_QUPV3_WRAP2_S3_CLK					255
-+#define GCC_QUPV3_WRAP2_S3_CLK_SRC				256
-+#define GCC_QUPV3_WRAP2_S4_CLK					257
-+#define GCC_QUPV3_WRAP2_S4_CLK_SRC				258
-+#define GCC_QUPV3_WRAP2_S5_CLK					259
-+#define GCC_QUPV3_WRAP2_S5_CLK_SRC				260
-+#define GCC_QUPV3_WRAP2_S6_CLK					261
-+#define GCC_QUPV3_WRAP2_S6_CLK_SRC				262
-+#define GCC_QUPV3_WRAP2_S7_CLK					263
-+#define GCC_QUPV3_WRAP2_S7_CLK_SRC				264
-+#define GCC_QUPV3_WRAP_0_M_AHB_CLK				265
-+#define GCC_QUPV3_WRAP_0_S_AHB_CLK				266
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK				267
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				268
-+#define GCC_QUPV3_WRAP_2_M_AHB_CLK				269
-+#define GCC_QUPV3_WRAP_2_S_AHB_CLK				270
-+#define GCC_SDCC2_AHB_CLK					271
-+#define GCC_SDCC2_APPS_CLK					272
-+#define GCC_SDCC2_APPS_CLK_SRC					273
-+#define GCC_SDCC4_AHB_CLK					274
-+#define GCC_SDCC4_APPS_CLK					275
-+#define GCC_SDCC4_APPS_CLK_SRC					276
-+#define GCC_UFS_PHY_AHB_CLK					277
-+#define GCC_UFS_PHY_AXI_CLK					278
-+#define GCC_UFS_PHY_AXI_CLK_SRC					279
-+#define GCC_UFS_PHY_AXI_HW_CTL_CLK				280
-+#define GCC_UFS_PHY_ICE_CORE_CLK				281
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC				282
-+#define GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK				283
-+#define GCC_UFS_PHY_PHY_AUX_CLK					284
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC				285
-+#define GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK				286
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK				287
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC				288
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK				289
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC				290
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK				291
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC				292
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK				293
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC				294
-+#define GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK			295
-+#define GCC_USB20_MASTER_CLK					296
-+#define GCC_USB20_MASTER_CLK_SRC				297
-+#define GCC_USB20_MOCK_UTMI_CLK					298
-+#define GCC_USB20_MOCK_UTMI_CLK_SRC				299
-+#define GCC_USB20_MOCK_UTMI_POSTDIV_CLK_SRC			300
-+#define GCC_USB20_SLEEP_CLK					301
-+#define GCC_USB30_MP_MASTER_CLK					302
-+#define GCC_USB30_MP_MASTER_CLK_SRC				303
-+#define GCC_USB30_MP_MOCK_UTMI_CLK				304
-+#define GCC_USB30_MP_MOCK_UTMI_CLK_SRC				305
-+#define GCC_USB30_MP_MOCK_UTMI_POSTDIV_CLK_SRC			306
-+#define GCC_USB30_MP_SLEEP_CLK					307
-+#define GCC_USB30_PRIM_MASTER_CLK				308
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC				309
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				310
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			311
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		312
-+#define GCC_USB30_PRIM_SLEEP_CLK				313
-+#define GCC_USB30_SEC_MASTER_CLK				314
-+#define GCC_USB30_SEC_MASTER_CLK_SRC				315
-+#define GCC_USB30_SEC_MOCK_UTMI_CLK				316
-+#define GCC_USB30_SEC_MOCK_UTMI_CLK_SRC				317
-+#define GCC_USB30_SEC_MOCK_UTMI_POSTDIV_CLK_SRC			318
-+#define GCC_USB30_SEC_SLEEP_CLK					319
-+#define GCC_USB30_TERT_MASTER_CLK				320
-+#define GCC_USB30_TERT_MASTER_CLK_SRC				321
-+#define GCC_USB30_TERT_MOCK_UTMI_CLK				322
-+#define GCC_USB30_TERT_MOCK_UTMI_CLK_SRC			323
-+#define GCC_USB30_TERT_MOCK_UTMI_POSTDIV_CLK_SRC		324
-+#define GCC_USB30_TERT_SLEEP_CLK				325
-+#define GCC_USB34_PRIM_PHY_PIPE_CLK_SRC				326
-+#define GCC_USB34_SEC_PHY_PIPE_CLK_SRC				327
-+#define GCC_USB34_TERT_PHY_PIPE_CLK_SRC				328
-+#define GCC_USB3_MP_PHY_AUX_CLK					329
-+#define GCC_USB3_MP_PHY_AUX_CLK_SRC				330
-+#define GCC_USB3_MP_PHY_COM_AUX_CLK				331
-+#define GCC_USB3_MP_PHY_PIPE_0_CLK				332
-+#define GCC_USB3_MP_PHY_PIPE_0_CLK_SRC				333
-+#define GCC_USB3_MP_PHY_PIPE_1_CLK				334
-+#define GCC_USB3_MP_PHY_PIPE_1_CLK_SRC				335
-+#define GCC_USB3_PRIM_PHY_AUX_CLK				336
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				337
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				338
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK				339
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				340
-+#define GCC_USB3_SEC_PHY_AUX_CLK				341
-+#define GCC_USB3_SEC_PHY_AUX_CLK_SRC				342
-+#define GCC_USB3_SEC_PHY_COM_AUX_CLK				343
-+#define GCC_USB3_SEC_PHY_PIPE_CLK				344
-+#define GCC_USB3_SEC_PHY_PIPE_CLK_SRC				345
-+#define GCC_USB3_TERT_PHY_AUX_CLK				346
-+#define GCC_USB3_TERT_PHY_AUX_CLK_SRC				347
-+#define GCC_USB3_TERT_PHY_COM_AUX_CLK				348
-+#define GCC_USB3_TERT_PHY_PIPE_CLK				349
-+#define GCC_USB3_TERT_PHY_PIPE_CLK_SRC				350
-+#define GCC_USB4_0_CFG_AHB_CLK					351
-+#define GCC_USB4_0_DP0_CLK					352
-+#define GCC_USB4_0_DP1_CLK					353
-+#define GCC_USB4_0_MASTER_CLK					354
-+#define GCC_USB4_0_MASTER_CLK_SRC				355
-+#define GCC_USB4_0_PHY_DP0_CLK_SRC				356
-+#define GCC_USB4_0_PHY_DP0_GMUX_CLK_SRC				357
-+#define GCC_USB4_0_PHY_DP1_CLK_SRC				358
-+#define GCC_USB4_0_PHY_DP1_GMUX_CLK_SRC				359
-+#define GCC_USB4_0_PHY_P2RR2P_PIPE_CLK				360
-+#define GCC_USB4_0_PHY_P2RR2P_PIPE_CLK_SRC			361
-+#define GCC_USB4_0_PHY_PCIE_PIPE_CLK				362
-+#define GCC_USB4_0_PHY_PCIE_PIPE_CLK_SRC			363
-+#define GCC_USB4_0_PHY_PCIE_PIPE_MUX_CLK_SRC			364
-+#define GCC_USB4_0_PHY_PCIE_PIPEGMUX_CLK_SRC			365
-+#define GCC_USB4_0_PHY_PIPEGMUX_CLK_SRC				366
-+#define GCC_USB4_0_PHY_RX0_CLK					367
-+#define GCC_USB4_0_PHY_RX0_CLK_SRC				368
-+#define GCC_USB4_0_PHY_RX1_CLK					369
-+#define GCC_USB4_0_PHY_RX1_CLK_SRC				370
-+#define GCC_USB4_0_PHY_SYS_CLK_SRC				371
-+#define GCC_USB4_0_PHY_SYS_PIPEGMUX_CLK_SRC			372
-+#define GCC_USB4_0_PHY_USB_PIPE_CLK				373
-+#define GCC_USB4_0_SB_IF_CLK					374
-+#define GCC_USB4_0_SB_IF_CLK_SRC				375
-+#define GCC_USB4_0_SYS_CLK					376
-+#define GCC_USB4_0_TMU_CLK					377
-+#define GCC_USB4_0_TMU_CLK_SRC					378
-+#define GCC_USB4_0_UC_HRR_CLK					379
-+#define GCC_USB4_1_CFG_AHB_CLK					380
-+#define GCC_USB4_1_DP0_CLK					381
-+#define GCC_USB4_1_DP1_CLK					382
-+#define GCC_USB4_1_MASTER_CLK					383
-+#define GCC_USB4_1_MASTER_CLK_SRC				384
-+#define GCC_USB4_1_PHY_DP0_CLK_SRC				385
-+#define GCC_USB4_1_PHY_DP0_GMUX_2_CLK_SRC			386
-+#define GCC_USB4_1_PHY_DP1_CLK_SRC				387
-+#define GCC_USB4_1_PHY_DP1_GMUX_2_CLK_SRC			388
-+#define GCC_USB4_1_PHY_P2RR2P_PIPE_CLK				389
-+#define GCC_USB4_1_PHY_P2RR2P_PIPE_CLK_SRC			390
-+#define GCC_USB4_1_PHY_PCIE_PIPE_CLK				391
-+#define GCC_USB4_1_PHY_PCIE_PIPE_CLK_SRC			392
-+#define GCC_USB4_1_PHY_PCIE_PIPE_MUX_CLK_SRC			393
-+#define GCC_USB4_1_PHY_PCIE_PIPEGMUX_CLK_SRC			394
-+#define GCC_USB4_1_PHY_PIPEGMUX_CLK_SRC				395
-+#define GCC_USB4_1_PHY_PLL_PIPE_CLK_SRC				396
-+#define GCC_USB4_1_PHY_RX0_CLK					397
-+#define GCC_USB4_1_PHY_RX0_CLK_SRC				398
-+#define GCC_USB4_1_PHY_RX1_CLK					399
-+#define GCC_USB4_1_PHY_RX1_CLK_SRC				400
-+#define GCC_USB4_1_PHY_SYS_CLK_SRC				401
-+#define GCC_USB4_1_PHY_SYS_PIPEGMUX_CLK_SRC			402
-+#define GCC_USB4_1_PHY_USB_PIPE_CLK				403
-+#define GCC_USB4_1_SB_IF_CLK					404
-+#define GCC_USB4_1_SB_IF_CLK_SRC				405
-+#define GCC_USB4_1_SYS_CLK					406
-+#define GCC_USB4_1_TMU_CLK					407
-+#define GCC_USB4_1_TMU_CLK_SRC					408
-+#define GCC_USB4_1_UC_HRR_CLK					409
-+#define GCC_USB4_2_CFG_AHB_CLK					410
-+#define GCC_USB4_2_DP0_CLK					411
-+#define GCC_USB4_2_DP1_CLK					412
-+#define GCC_USB4_2_MASTER_CLK					413
-+#define GCC_USB4_2_MASTER_CLK_SRC				414
-+#define GCC_USB4_2_PHY_DP0_CLK_SRC				415
-+#define GCC_USB4_2_PHY_DP0_GMUX_CLK_SRC				416
-+#define GCC_USB4_2_PHY_DP1_CLK_SRC				417
-+#define GCC_USB4_2_PHY_DP1_GMUX_CLK_SRC				418
-+#define GCC_USB4_2_PHY_P2RR2P_PIPE_CLK				419
-+#define GCC_USB4_2_PHY_P2RR2P_PIPE_CLK_SRC			420
-+#define GCC_USB4_2_PHY_PCIE_PIPE_CLK				421
-+#define GCC_USB4_2_PHY_PCIE_PIPE_CLK_SRC			422
-+#define GCC_USB4_2_PHY_PCIE_PIPE_MUX_CLK_SRC			423
-+#define GCC_USB4_2_PHY_PCIE_PIPEGMUX_CLK_SRC			424
-+#define GCC_USB4_2_PHY_PIPEGMUX_CLK_SRC				425
-+#define GCC_USB4_2_PHY_RX0_CLK					426
-+#define GCC_USB4_2_PHY_RX0_CLK_SRC				427
-+#define GCC_USB4_2_PHY_RX1_CLK					428
-+#define GCC_USB4_2_PHY_RX1_CLK_SRC				429
-+#define GCC_USB4_2_PHY_SYS_CLK_SRC				430
-+#define GCC_USB4_2_PHY_SYS_PIPEGMUX_CLK_SRC			431
-+#define GCC_USB4_2_PHY_USB_PIPE_CLK				432
-+#define GCC_USB4_2_SB_IF_CLK					433
-+#define GCC_USB4_2_SB_IF_CLK_SRC				434
-+#define GCC_USB4_2_SYS_CLK					435
-+#define GCC_USB4_2_TMU_CLK					436
-+#define GCC_USB4_2_TMU_CLK_SRC					437
-+#define GCC_USB4_2_UC_HRR_CLK					438
-+#define GCC_VIDEO_AHB_CLK					439
-+#define GCC_VIDEO_AXI0_CLK					440
-+#define GCC_VIDEO_AXI0C_CLK					441
-+#define GCC_VIDEO_AXI1_CLK					442
-+#define GCC_VIDEO_XO_CLK					443
-+
-+/* GCC power domains */
-+#define GCC_PCIE_0_TUNNEL_GDSC					0
-+#define GCC_PCIE_1_TUNNEL_GDSC					1
-+#define GCC_PCIE_2_TUNNEL_GDSC					2
-+#define GCC_PCIE_3A_GDSC					3
-+#define GCC_PCIE_3A_PHY_GDSC					4
-+#define GCC_PCIE_3B_GDSC					5
-+#define GCC_PCIE_3B_PHY_GDSC					6
-+#define GCC_PCIE_4_GDSC						7
-+#define GCC_PCIE_4_PHY_GDSC					8
-+#define GCC_PCIE_5_GDSC						9
-+#define GCC_PCIE_5_PHY_GDSC					10
-+#define GCC_PCIE_6_GDSC						11
-+#define GCC_PCIE_6_PHY_GDSC					12
-+#define GCC_UFS_PHY_GDSC					13
-+#define GCC_USB20_PRIM_GDSC					14
-+#define GCC_USB30_MP_GDSC					15
-+#define GCC_USB30_PRIM_GDSC					16
-+#define GCC_USB30_SEC_GDSC					17
-+#define GCC_USB30_TERT_GDSC					18
-+#define GCC_USB3_MP_SS0_PHY_GDSC				19
-+#define GCC_USB3_MP_SS1_PHY_GDSC				20
-+#define GCC_USB4_0_GDSC						21
-+#define GCC_USB4_1_GDSC						22
-+#define GCC_USB4_2_GDSC						23
-+#define GCC_USB_0_PHY_GDSC					24
-+#define GCC_USB_1_PHY_GDSC					25
-+#define GCC_USB_2_PHY_GDSC					26
-+
-+/* GCC resets */
-+#define GCC_AV1E_BCR						0
-+#define GCC_CAMERA_BCR						1
-+#define GCC_DISPLAY_BCR						2
-+#define GCC_EVA_BCR						3
-+#define GCC_GPU_BCR						4
-+#define GCC_PCIE_0_LINK_DOWN_BCR				5
-+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR				6
-+#define GCC_PCIE_0_PHY_BCR					7
-+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR			8
-+#define GCC_PCIE_0_TUNNEL_BCR					9
-+#define GCC_PCIE_1_LINK_DOWN_BCR				10
-+#define GCC_PCIE_1_NOCSR_COM_PHY_BCR				11
-+#define GCC_PCIE_1_PHY_BCR					12
-+#define GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR			13
-+#define GCC_PCIE_1_TUNNEL_BCR					14
-+#define GCC_PCIE_2_LINK_DOWN_BCR				15
-+#define GCC_PCIE_2_NOCSR_COM_PHY_BCR				16
-+#define GCC_PCIE_2_PHY_BCR					17
-+#define GCC_PCIE_2_PHY_NOCSR_COM_PHY_BCR			18
-+#define GCC_PCIE_2_TUNNEL_BCR					19
-+#define GCC_PCIE_3A_BCR						20
-+#define GCC_PCIE_3A_LINK_DOWN_BCR				21
-+#define GCC_PCIE_3A_NOCSR_COM_PHY_BCR				22
-+#define GCC_PCIE_3A_PHY_BCR					23
-+#define GCC_PCIE_3A_PHY_NOCSR_COM_PHY_BCR			24
-+#define GCC_PCIE_3B_BCR						25
-+#define GCC_PCIE_3B_LINK_DOWN_BCR				26
-+#define GCC_PCIE_3B_NOCSR_COM_PHY_BCR				27
-+#define GCC_PCIE_3B_PHY_BCR					28
-+#define GCC_PCIE_3B_PHY_NOCSR_COM_PHY_BCR			29
-+#define GCC_PCIE_4_BCR						30
-+#define GCC_PCIE_4_LINK_DOWN_BCR				31
-+#define GCC_PCIE_4_NOCSR_COM_PHY_BCR				32
-+#define GCC_PCIE_4_PHY_BCR					33
-+#define GCC_PCIE_4_PHY_NOCSR_COM_PHY_BCR			34
-+#define GCC_PCIE_5_BCR						35
-+#define GCC_PCIE_5_LINK_DOWN_BCR				36
-+#define GCC_PCIE_5_NOCSR_COM_PHY_BCR				37
-+#define GCC_PCIE_5_PHY_BCR					38
-+#define GCC_PCIE_5_PHY_NOCSR_COM_PHY_BCR			39
-+#define GCC_PCIE_6_BCR						40
-+#define GCC_PCIE_6_LINK_DOWN_BCR				41
-+#define GCC_PCIE_6_NOCSR_COM_PHY_BCR				42
-+#define GCC_PCIE_6_PHY_BCR					43
-+#define GCC_PCIE_6_PHY_NOCSR_COM_PHY_BCR			44
-+#define GCC_PCIE_NOC_BCR					45
-+#define GCC_PCIE_PHY_BCR					46
-+#define GCC_PCIE_PHY_CFG_AHB_BCR				47
-+#define GCC_PCIE_PHY_COM_BCR					48
-+#define GCC_PCIE_RSCC_BCR					49
-+#define GCC_PDM_BCR						50
-+#define GCC_QUPV3_WRAPPER_0_BCR					51
-+#define GCC_QUPV3_WRAPPER_1_BCR					52
-+#define GCC_QUPV3_WRAPPER_2_BCR					53
-+#define GCC_QUPV3_WRAPPER_OOB_BCR				54
-+#define GCC_QUSB2PHY_HS0_MP_BCR					55
-+#define GCC_QUSB2PHY_HS1_MP_BCR					56
-+#define GCC_QUSB2PHY_PRIM_BCR					57
-+#define GCC_QUSB2PHY_SEC_BCR					58
-+#define GCC_QUSB2PHY_TERT_BCR					59
-+#define GCC_QUSB2PHY_USB20_HS_BCR				60
-+#define GCC_SDCC2_BCR						61
-+#define GCC_SDCC4_BCR						62
-+#define GCC_TCSR_PCIE_BCR					63
-+#define GCC_UFS_PHY_BCR						64
-+#define GCC_USB20_PRIM_BCR					65
-+#define GCC_USB30_MP_BCR					66
-+#define GCC_USB30_PRIM_BCR					67
-+#define GCC_USB30_SEC_BCR					68
-+#define GCC_USB30_TERT_BCR					69
-+#define GCC_USB3_MP_SS0_PHY_BCR					70
-+#define GCC_USB3_MP_SS1_PHY_BCR					71
-+#define GCC_USB3_PHY_PRIM_BCR					72
-+#define GCC_USB3_PHY_SEC_BCR					73
-+#define GCC_USB3_PHY_TERT_BCR					74
-+#define GCC_USB3_UNIPHY_MP0_BCR					75
-+#define GCC_USB3_UNIPHY_MP1_BCR					76
-+#define GCC_USB3PHY_PHY_PRIM_BCR				77
-+#define GCC_USB3PHY_PHY_SEC_BCR					78
-+#define GCC_USB3PHY_PHY_TERT_BCR				79
-+#define GCC_USB3UNIPHY_PHY_MP0_BCR				80
-+#define GCC_USB3UNIPHY_PHY_MP1_BCR				81
-+#define GCC_USB4_0_BCR						82
-+#define GCC_USB4_0_DP0_PHY_PRIM_BCR				83
-+#define GCC_USB4_1_BCR						84
-+#define GCC_USB4_2_BCR						85
-+#define GCC_USB_0_PHY_BCR					86
-+#define GCC_USB_1_PHY_BCR					87
-+#define GCC_USB_2_PHY_BCR					88
-+#define GCC_VIDEO_AXI0_CLK_ARES					89
-+#define GCC_VIDEO_AXI1_CLK_ARES					90
-+#define GCC_VIDEO_BCR						91
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,glymur-tcsr.h b/include/dt-bindings/clock/qcom,glymur-tcsr.h
-new file mode 100644
-index 000000000000..72614226b113
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,glymur-tcsr.h
-@@ -0,0 +1,24 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_TCSR_CC_GLYMUR_H
-+#define _DT_BINDINGS_CLK_QCOM_TCSR_CC_GLYMUR_H
-+
-+/* TCSR_CC clocks */
-+#define TCSR_EDP_CLKREF_EN					0
-+#define TCSR_PCIE_1_CLKREF_EN					1
-+#define TCSR_PCIE_2_CLKREF_EN					2
-+#define TCSR_PCIE_3_CLKREF_EN					3
-+#define TCSR_PCIE_4_CLKREF_EN					4
-+#define TCSR_USB2_1_CLKREF_EN					5
-+#define TCSR_USB2_2_CLKREF_EN					6
-+#define TCSR_USB2_3_CLKREF_EN					7
-+#define TCSR_USB2_4_CLKREF_EN					8
-+#define TCSR_USB3_0_CLKREF_EN					9
-+#define TCSR_USB3_1_CLKREF_EN					10
-+#define TCSR_USB4_1_CLKREF_EN					11
-+#define TCSR_USB4_2_CLKREF_EN					12
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,gpucc-sc8280xp.h b/include/dt-bindings/clock/qcom,gpucc-sc8280xp.h
-new file mode 100644
-index 000000000000..bb7da46333b0
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,gpucc-sc8280xp.h
-@@ -0,0 +1,35 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_SC8280XP_H
-+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_SC8280XP_H
-+
-+/* GPU_CC clocks */
-+#define GPU_CC_PLL0						0
-+#define GPU_CC_PLL1						1
-+#define GPU_CC_AHB_CLK						2
-+#define GPU_CC_CB_CLK						3
-+#define GPU_CC_CRC_AHB_CLK					4
-+#define GPU_CC_CX_GMU_CLK					5
-+#define GPU_CC_CX_SNOC_DVM_CLK					6
-+#define GPU_CC_CXO_AON_CLK					7
-+#define GPU_CC_CXO_CLK						8
-+#define GPU_CC_FREQ_MEASURE_CLK					9
-+#define GPU_CC_GMU_CLK_SRC					10
-+#define GPU_CC_GX_GMU_CLK					11
-+#define GPU_CC_GX_VSENSE_CLK					12
-+#define GPU_CC_HUB_AHB_DIV_CLK_SRC				13
-+#define GPU_CC_HUB_AON_CLK					14
-+#define GPU_CC_HUB_CLK_SRC					15
-+#define GPU_CC_HUB_CX_INT_CLK					16
-+#define GPU_CC_HUB_CX_INT_DIV_CLK_SRC				17
-+#define GPU_CC_SLEEP_CLK					18
-+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK				19
-+
-+/* GPU_CC power domains */
-+#define GPU_CC_CX_GDSC				0
-+#define GPU_CC_GX_GDSC				1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,ipq-cmn-pll.h b/include/dt-bindings/clock/qcom,ipq-cmn-pll.h
-new file mode 100644
-index 000000000000..936e92b3b62c
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,ipq-cmn-pll.h
-@@ -0,0 +1,22 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_IPQ_CMN_PLL_H
-+#define _DT_BINDINGS_CLK_QCOM_IPQ_CMN_PLL_H
-+
-+/* CMN PLL core clock. */
-+#define CMN_PLL_CLK			0
-+
-+/* The output clocks from CMN PLL of IPQ9574. */
-+#define XO_24MHZ_CLK			1
-+#define SLEEP_32KHZ_CLK			2
-+#define PCS_31P25MHZ_CLK		3
-+#define NSS_1200MHZ_CLK			4
-+#define PPE_353MHZ_CLK			5
-+#define ETH0_50MHZ_CLK			6
-+#define ETH1_50MHZ_CLK			7
-+#define ETH2_50MHZ_CLK			8
-+#define ETH_25MHZ_CLK			9
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,ipq5018-cmn-pll.h b/include/dt-bindings/clock/qcom,ipq5018-cmn-pll.h
-new file mode 100644
-index 000000000000..586d1c9b33b3
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,ipq5018-cmn-pll.h
-@@ -0,0 +1,16 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_IPQ5018_CMN_PLL_H
-+#define _DT_BINDINGS_CLK_QCOM_IPQ5018_CMN_PLL_H
-+
-+/* CMN PLL core clock. */
-+#define IPQ5018_CMN_PLL_CLK			0
-+
-+/* The output clocks from CMN PLL of IPQ5018. */
-+#define IPQ5018_XO_24MHZ_CLK			1
-+#define IPQ5018_SLEEP_32KHZ_CLK			2
-+#define IPQ5018_ETH_50MHZ_CLK			3
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,ipq5332-gcc.h b/include/dt-bindings/clock/qcom,ipq5332-gcc.h
-new file mode 100644
-index 000000000000..da9b507c30bf
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,ipq5332-gcc.h
-@@ -0,0 +1,336 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_IPQ5332_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_IPQ5332_H
-+
-+#define GPLL0_MAIN					0
-+#define GPLL0						1
-+#define GPLL2_MAIN					2
-+#define GPLL2						3
-+#define GPLL4_MAIN					4
-+#define GPLL4						5
-+#define GCC_ADSS_PWM_CLK				6
-+#define GCC_ADSS_PWM_CLK_SRC				7
-+#define GCC_AHB_CLK					8
-+#define GCC_APSS_AXI_CLK_SRC				9
-+#define GCC_BLSP1_AHB_CLK				10
-+#define GCC_BLSP1_QUP1_I2C_APPS_CLK			11
-+#define GCC_BLSP1_QUP1_SPI_APPS_CLK			12
-+#define GCC_BLSP1_QUP1_SPI_APPS_CLK_SRC			13
-+#define GCC_BLSP1_QUP2_I2C_APPS_CLK			14
-+#define GCC_BLSP1_QUP2_SPI_APPS_CLK			15
-+#define GCC_BLSP1_QUP2_SPI_APPS_CLK_SRC			16
-+#define GCC_BLSP1_QUP3_I2C_APPS_CLK			17
-+#define GCC_BLSP1_QUP3_SPI_APPS_CLK			18
-+#define GCC_BLSP1_QUP3_SPI_APPS_CLK_SRC			19
-+#define GCC_BLSP1_SLEEP_CLK				20
-+#define GCC_BLSP1_UART1_APPS_CLK			21
-+#define GCC_BLSP1_UART1_APPS_CLK_SRC			22
-+#define GCC_BLSP1_UART2_APPS_CLK			23
-+#define GCC_BLSP1_UART2_APPS_CLK_SRC			24
-+#define GCC_BLSP1_UART3_APPS_CLK			25
-+#define GCC_BLSP1_UART3_APPS_CLK_SRC			26
-+#define GCC_CE_AHB_CLK					27
-+#define GCC_CE_AXI_CLK					28
-+#define GCC_CE_PCNOC_AHB_CLK				29
-+#define GCC_CMN_12GPLL_AHB_CLK				30
-+#define GCC_CMN_12GPLL_APU_CLK				31
-+#define GCC_CMN_12GPLL_SYS_CLK				32
-+#define GCC_GP1_CLK					33
-+#define GCC_GP1_CLK_SRC					34
-+#define GCC_GP2_CLK					35
-+#define GCC_GP2_CLK_SRC					36
-+#define GCC_LPASS_CORE_AXIM_CLK				37
-+#define GCC_LPASS_SWAY_CLK				38
-+#define GCC_LPASS_SWAY_CLK_SRC				39
-+#define GCC_MDIO_AHB_CLK				40
-+#define GCC_MDIO_SLAVE_AHB_CLK				41
-+#define GCC_MEM_NOC_Q6_AXI_CLK				42
-+#define GCC_MEM_NOC_TS_CLK				43
-+#define GCC_NSS_TS_CLK					44
-+#define GCC_NSS_TS_CLK_SRC				45
-+#define GCC_NSSCC_CLK					46
-+#define GCC_NSSCFG_CLK					47
-+#define GCC_NSSNOC_ATB_CLK				48
-+#define GCC_NSSNOC_NSSCC_CLK				49
-+#define GCC_NSSNOC_QOSGEN_REF_CLK			50
-+#define GCC_NSSNOC_SNOC_1_CLK				51
-+#define GCC_NSSNOC_SNOC_CLK				52
-+#define GCC_NSSNOC_TIMEOUT_REF_CLK			53
-+#define GCC_NSSNOC_XO_DCD_CLK				54
-+#define GCC_PCIE3X1_0_AHB_CLK				55
-+#define GCC_PCIE3X1_0_AUX_CLK				56
-+#define GCC_PCIE3X1_0_AXI_CLK_SRC			57
-+#define GCC_PCIE3X1_0_AXI_M_CLK				58
-+#define GCC_PCIE3X1_0_AXI_S_BRIDGE_CLK			59
-+#define GCC_PCIE3X1_0_AXI_S_CLK				60
-+#define GCC_PCIE3X1_0_PIPE_CLK				61
-+#define GCC_PCIE3X1_0_RCHG_CLK				62
-+#define GCC_PCIE3X1_0_RCHG_CLK_SRC			63
-+#define GCC_PCIE3X1_1_AHB_CLK				64
-+#define GCC_PCIE3X1_1_AUX_CLK				65
-+#define GCC_PCIE3X1_1_AXI_CLK_SRC			66
-+#define GCC_PCIE3X1_1_AXI_M_CLK				67
-+#define GCC_PCIE3X1_1_AXI_S_BRIDGE_CLK			68
-+#define GCC_PCIE3X1_1_AXI_S_CLK				69
-+#define GCC_PCIE3X1_1_PIPE_CLK				70
-+#define GCC_PCIE3X1_1_RCHG_CLK				71
-+#define GCC_PCIE3X1_1_RCHG_CLK_SRC			72
-+#define GCC_PCIE3X1_PHY_AHB_CLK				73
-+#define GCC_PCIE3X2_AHB_CLK				74
-+#define GCC_PCIE3X2_AUX_CLK				75
-+#define GCC_PCIE3X2_AXI_M_CLK				76
-+#define GCC_PCIE3X2_AXI_M_CLK_SRC			77
-+#define GCC_PCIE3X2_AXI_S_BRIDGE_CLK			78
-+#define GCC_PCIE3X2_AXI_S_CLK				79
-+#define GCC_PCIE3X2_AXI_S_CLK_SRC			80
-+#define GCC_PCIE3X2_PHY_AHB_CLK				81
-+#define GCC_PCIE3X2_PIPE_CLK				82
-+#define GCC_PCIE3X2_RCHG_CLK				83
-+#define GCC_PCIE3X2_RCHG_CLK_SRC			84
-+#define GCC_PCIE_AUX_CLK_SRC				85
-+#define GCC_PCNOC_AT_CLK				86
-+#define GCC_PCNOC_BFDCD_CLK_SRC				87
-+#define GCC_PCNOC_LPASS_CLK				88
-+#define GCC_PRNG_AHB_CLK				89
-+#define GCC_Q6_AXIM_CLK_SRC				93
-+#define GCC_QDSS_AT_CLK					99
-+#define GCC_QDSS_AT_CLK_SRC				100
-+#define GCC_QDSS_CFG_AHB_CLK				101
-+#define GCC_QDSS_DAP_AHB_CLK				102
-+#define GCC_QDSS_DAP_CLK				103
-+#define GCC_QDSS_DAP_DIV_CLK_SRC			104
-+#define GCC_QDSS_ETR_USB_CLK				105
-+#define GCC_QDSS_EUD_AT_CLK				106
-+#define GCC_QDSS_TSCTR_CLK_SRC				107
-+#define GCC_QPIC_AHB_CLK				108
-+#define GCC_QPIC_CLK					109
-+#define GCC_QPIC_IO_MACRO_CLK				110
-+#define GCC_QPIC_IO_MACRO_CLK_SRC			111
-+#define GCC_QPIC_SLEEP_CLK				112
-+#define GCC_SDCC1_AHB_CLK				113
-+#define GCC_SDCC1_APPS_CLK				114
-+#define GCC_SDCC1_APPS_CLK_SRC				115
-+#define GCC_SLEEP_CLK_SRC				116
-+#define GCC_SNOC_LPASS_CFG_CLK				117
-+#define GCC_SNOC_NSSNOC_1_CLK				118
-+#define GCC_SNOC_NSSNOC_CLK				119
-+#define GCC_SNOC_PCIE3_1LANE_1_M_CLK			120
-+#define GCC_SNOC_PCIE3_1LANE_1_S_CLK			121
-+#define GCC_SNOC_PCIE3_1LANE_M_CLK			122
-+#define GCC_SNOC_PCIE3_1LANE_S_CLK			123
-+#define GCC_SNOC_PCIE3_2LANE_M_CLK			124
-+#define GCC_SNOC_PCIE3_2LANE_S_CLK			125
-+#define GCC_SNOC_USB_CLK				126
-+#define GCC_SYS_NOC_AT_CLK				127
-+#define GCC_SYSTEM_NOC_BFDCD_CLK_SRC			129
-+#define GCC_UNIPHY0_AHB_CLK				130
-+#define GCC_UNIPHY0_SYS_CLK				131
-+#define GCC_UNIPHY1_AHB_CLK				132
-+#define GCC_UNIPHY1_SYS_CLK				133
-+#define GCC_UNIPHY_SYS_CLK_SRC				134
-+#define GCC_USB0_AUX_CLK				135
-+#define GCC_USB0_AUX_CLK_SRC				136
-+#define GCC_USB0_EUD_AT_CLK				137
-+#define GCC_USB0_LFPS_CLK				138
-+#define GCC_USB0_LFPS_CLK_SRC				139
-+#define GCC_USB0_MASTER_CLK				140
-+#define GCC_USB0_MASTER_CLK_SRC				141
-+#define GCC_USB0_MOCK_UTMI_CLK				142
-+#define GCC_USB0_MOCK_UTMI_CLK_SRC			143
-+#define GCC_USB0_MOCK_UTMI_DIV_CLK_SRC			144
-+#define GCC_USB0_PHY_CFG_AHB_CLK			145
-+#define GCC_USB0_PIPE_CLK				146
-+#define GCC_USB0_SLEEP_CLK				147
-+#define GCC_WCSS_AHB_CLK_SRC				148
-+#define GCC_XO_CLK					160
-+#define GCC_XO_CLK_SRC					161
-+#define GCC_XO_DIV4_CLK					162
-+#define GCC_IM_SLEEP_CLK				163
-+#define GCC_NSSNOC_PCNOC_1_CLK				164
-+#define GCC_MEM_NOC_AHB_CLK				165
-+#define GCC_MEM_NOC_APSS_AXI_CLK			166
-+#define GCC_SNOC_QOSGEN_EXTREF_DIV_CLK_SRC		167
-+#define GCC_MEM_NOC_QOSGEN_EXTREF_CLK			168
-+#define GCC_PCIE3X2_PIPE_CLK_SRC			169
-+#define GCC_PCIE3X1_0_PIPE_CLK_SRC			170
-+#define GCC_PCIE3X1_1_PIPE_CLK_SRC			171
-+#define GCC_USB0_PIPE_CLK_SRC				172
-+
-+#define GCC_ADSS_BCR					0
-+#define GCC_ADSS_PWM_CLK_ARES				1
-+#define GCC_AHB_CLK_ARES				2
-+#define GCC_APC0_VOLTAGE_DROOP_DETECTOR_BCR		3
-+#define GCC_APC0_VOLTAGE_DROOP_DETECTOR_GPLL0_CLK_ARES	4
-+#define GCC_APSS_AHB_CLK_ARES				5
-+#define GCC_APSS_AXI_CLK_ARES				6
-+#define GCC_BLSP1_AHB_CLK_ARES				7
-+#define GCC_BLSP1_BCR					8
-+#define GCC_BLSP1_QUP1_BCR				9
-+#define GCC_BLSP1_QUP1_I2C_APPS_CLK_ARES		10
-+#define GCC_BLSP1_QUP1_SPI_APPS_CLK_ARES		11
-+#define GCC_BLSP1_QUP2_BCR				12
-+#define GCC_BLSP1_QUP2_I2C_APPS_CLK_ARES		13
-+#define GCC_BLSP1_QUP2_SPI_APPS_CLK_ARES		14
-+#define GCC_BLSP1_QUP3_BCR				15
-+#define GCC_BLSP1_QUP3_I2C_APPS_CLK_ARES		16
-+#define GCC_BLSP1_QUP3_SPI_APPS_CLK_ARES		17
-+#define GCC_BLSP1_SLEEP_CLK_ARES			18
-+#define GCC_BLSP1_UART1_APPS_CLK_ARES			19
-+#define GCC_BLSP1_UART1_BCR				20
-+#define GCC_BLSP1_UART2_APPS_CLK_ARES			21
-+#define GCC_BLSP1_UART2_BCR				22
-+#define GCC_BLSP1_UART3_APPS_CLK_ARES			23
-+#define GCC_BLSP1_UART3_BCR				24
-+#define GCC_CE_BCR					25
-+#define GCC_CMN_BLK_BCR					26
-+#define GCC_CMN_LDO0_BCR				27
-+#define GCC_CMN_LDO1_BCR				28
-+#define GCC_DCC_BCR					29
-+#define GCC_GP1_CLK_ARES				30
-+#define GCC_GP2_CLK_ARES				31
-+#define GCC_LPASS_BCR					32
-+#define GCC_LPASS_CORE_AXIM_CLK_ARES			33
-+#define GCC_LPASS_SWAY_CLK_ARES				34
-+#define GCC_MDIOM_BCR					35
-+#define GCC_MDIOS_BCR					36
-+#define GCC_NSS_BCR					37
-+#define GCC_NSS_TS_CLK_ARES				38
-+#define GCC_NSSCC_CLK_ARES				39
-+#define GCC_NSSCFG_CLK_ARES				40
-+#define GCC_NSSNOC_ATB_CLK_ARES				41
-+#define GCC_NSSNOC_NSSCC_CLK_ARES			42
-+#define GCC_NSSNOC_QOSGEN_REF_CLK_ARES			43
-+#define GCC_NSSNOC_SNOC_1_CLK_ARES			44
-+#define GCC_NSSNOC_SNOC_CLK_ARES			45
-+#define GCC_NSSNOC_TIMEOUT_REF_CLK_ARES			46
-+#define GCC_NSSNOC_XO_DCD_CLK_ARES			47
-+#define GCC_PCIE3X1_0_AHB_CLK_ARES			48
-+#define GCC_PCIE3X1_0_AUX_CLK_ARES			49
-+#define GCC_PCIE3X1_0_AXI_M_CLK_ARES			50
-+#define GCC_PCIE3X1_0_AXI_S_BRIDGE_CLK_ARES		51
-+#define GCC_PCIE3X1_0_AXI_S_CLK_ARES			52
-+#define GCC_PCIE3X1_0_BCR				53
-+#define GCC_PCIE3X1_0_LINK_DOWN_BCR			54
-+#define GCC_PCIE3X1_0_PHY_BCR				55
-+#define GCC_PCIE3X1_0_PHY_PHY_BCR			56
-+#define GCC_PCIE3X1_1_AHB_CLK_ARES			57
-+#define GCC_PCIE3X1_1_AUX_CLK_ARES			58
-+#define GCC_PCIE3X1_1_AXI_M_CLK_ARES			59
-+#define GCC_PCIE3X1_1_AXI_S_BRIDGE_CLK_ARES		60
-+#define GCC_PCIE3X1_1_AXI_S_CLK_ARES			61
-+#define GCC_PCIE3X1_1_BCR				62
-+#define GCC_PCIE3X1_1_LINK_DOWN_BCR			63
-+#define GCC_PCIE3X1_1_PHY_BCR				64
-+#define GCC_PCIE3X1_1_PHY_PHY_BCR			65
-+#define GCC_PCIE3X1_PHY_AHB_CLK_ARES			66
-+#define GCC_PCIE3X2_AHB_CLK_ARES			67
-+#define GCC_PCIE3X2_AUX_CLK_ARES			68
-+#define GCC_PCIE3X2_AXI_M_CLK_ARES			69
-+#define GCC_PCIE3X2_AXI_S_BRIDGE_CLK_ARES		70
-+#define GCC_PCIE3X2_AXI_S_CLK_ARES			71
-+#define GCC_PCIE3X2_BCR					72
-+#define GCC_PCIE3X2_LINK_DOWN_BCR			73
-+#define GCC_PCIE3X2_PHY_AHB_CLK_ARES			74
-+#define GCC_PCIE3X2_PHY_BCR				75
-+#define GCC_PCIE3X2PHY_PHY_BCR				76
-+#define GCC_PCNOC_BCR					77
-+#define GCC_PCNOC_LPASS_CLK_ARES			78
-+#define GCC_PRNG_AHB_CLK_ARES				79
-+#define GCC_PRNG_BCR					80
-+#define GCC_Q6_AHB_CLK_ARES				81
-+#define GCC_Q6_AHB_S_CLK_ARES				82
-+#define GCC_Q6_AXIM_CLK_ARES				83
-+#define GCC_Q6_AXIS_CLK_ARES				84
-+#define GCC_Q6_TSCTR_1TO2_CLK_ARES			85
-+#define GCC_Q6SS_ATBM_CLK_ARES				86
-+#define GCC_Q6SS_PCLKDBG_CLK_ARES			87
-+#define GCC_Q6SS_TRIG_CLK_ARES				88
-+#define GCC_QDSS_APB2JTAG_CLK_ARES			89
-+#define GCC_QDSS_AT_CLK_ARES				90
-+#define GCC_QDSS_BCR					91
-+#define GCC_QDSS_CFG_AHB_CLK_ARES			92
-+#define GCC_QDSS_DAP_AHB_CLK_ARES			93
-+#define GCC_QDSS_DAP_CLK_ARES				94
-+#define GCC_QDSS_ETR_USB_CLK_ARES			95
-+#define GCC_QDSS_EUD_AT_CLK_ARES			96
-+#define GCC_QDSS_STM_CLK_ARES				97
-+#define GCC_QDSS_TRACECLKIN_CLK_ARES			98
-+#define GCC_QDSS_TS_CLK_ARES				99
-+#define GCC_QDSS_TSCTR_DIV16_CLK_ARES			100
-+#define GCC_QDSS_TSCTR_DIV2_CLK_ARES			101
-+#define GCC_QDSS_TSCTR_DIV3_CLK_ARES			102
-+#define GCC_QDSS_TSCTR_DIV4_CLK_ARES			103
-+#define GCC_QDSS_TSCTR_DIV8_CLK_ARES			104
-+#define GCC_QPIC_AHB_CLK_ARES				105
-+#define GCC_QPIC_CLK_ARES				106
-+#define GCC_QPIC_BCR					107
-+#define GCC_QPIC_IO_MACRO_CLK_ARES			108
-+#define GCC_QPIC_SLEEP_CLK_ARES				109
-+#define GCC_QUSB2_0_PHY_BCR				110
-+#define GCC_SDCC1_AHB_CLK_ARES				111
-+#define GCC_SDCC1_APPS_CLK_ARES				112
-+#define GCC_SDCC_BCR					113
-+#define GCC_SNOC_BCR					114
-+#define GCC_SNOC_LPASS_CFG_CLK_ARES			115
-+#define GCC_SNOC_NSSNOC_1_CLK_ARES			116
-+#define GCC_SNOC_NSSNOC_CLK_ARES			117
-+#define GCC_SYS_NOC_QDSS_STM_AXI_CLK_ARES		118
-+#define GCC_SYS_NOC_WCSS_AHB_CLK_ARES			119
-+#define GCC_UNIPHY0_AHB_CLK_ARES			120
-+#define GCC_UNIPHY0_BCR					121
-+#define GCC_UNIPHY0_SYS_CLK_ARES			122
-+#define GCC_UNIPHY1_AHB_CLK_ARES			123
-+#define GCC_UNIPHY1_BCR					124
-+#define GCC_UNIPHY1_SYS_CLK_ARES			125
-+#define GCC_USB0_AUX_CLK_ARES				126
-+#define GCC_USB0_EUD_AT_CLK_ARES			127
-+#define GCC_USB0_LFPS_CLK_ARES				128
-+#define GCC_USB0_MASTER_CLK_ARES			129
-+#define GCC_USB0_MOCK_UTMI_CLK_ARES			130
-+#define GCC_USB0_PHY_BCR				131
-+#define GCC_USB0_PHY_CFG_AHB_CLK_ARES			132
-+#define GCC_USB0_SLEEP_CLK_ARES				133
-+#define GCC_USB3PHY_0_PHY_BCR				134
-+#define GCC_USB_BCR					135
-+#define GCC_WCSS_AXIM_CLK_ARES				136
-+#define GCC_WCSS_AXIS_CLK_ARES				137
-+#define GCC_WCSS_BCR					138
-+#define GCC_WCSS_DBG_IFC_APB_BDG_CLK_ARES		139
-+#define GCC_WCSS_DBG_IFC_APB_CLK_ARES			140
-+#define GCC_WCSS_DBG_IFC_ATB_BDG_CLK_ARES		141
-+#define GCC_WCSS_DBG_IFC_ATB_CLK_ARES			142
-+#define GCC_WCSS_DBG_IFC_NTS_BDG_CLK_ARES		143
-+#define GCC_WCSS_DBG_IFC_NTS_CLK_ARES			144
-+#define GCC_WCSS_ECAHB_CLK_ARES				145
-+#define GCC_WCSS_MST_ASYNC_BDG_CLK_ARES			146
-+#define GCC_WCSS_Q6_BCR					147
-+#define GCC_WCSS_SLV_ASYNC_BDG_CLK_ARES			148
-+#define GCC_XO_CLK_ARES					149
-+#define GCC_XO_DIV4_CLK_ARES				150
-+#define GCC_Q6SS_DBG_ARES				151
-+#define GCC_WCSS_DBG_BDG_ARES				152
-+#define GCC_WCSS_DBG_ARES				153
-+#define GCC_WCSS_AXI_S_ARES				154
-+#define GCC_WCSS_AXI_M_ARES				155
-+#define GCC_WCSSAON_ARES				156
-+#define GCC_PCIE3X2_PIPE_ARES				157
-+#define GCC_PCIE3X2_CORE_STICKY_ARES			158
-+#define GCC_PCIE3X2_AXI_S_STICKY_ARES			159
-+#define GCC_PCIE3X2_AXI_M_STICKY_ARES			160
-+#define GCC_PCIE3X1_0_PIPE_ARES				161
-+#define GCC_PCIE3X1_0_CORE_STICKY_ARES			162
-+#define GCC_PCIE3X1_0_AXI_S_STICKY_ARES			163
-+#define GCC_PCIE3X1_0_AXI_M_STICKY_ARES			164
-+#define GCC_PCIE3X1_1_PIPE_ARES				165
-+#define GCC_PCIE3X1_1_CORE_STICKY_ARES			166
-+#define GCC_PCIE3X1_1_AXI_S_STICKY_ARES			167
-+#define GCC_PCIE3X1_1_AXI_M_STICKY_ARES			168
-+#define GCC_IM_SLEEP_CLK_ARES				169
-+#define GCC_NSSNOC_PCNOC_1_CLK_ARES			170
-+#define GCC_UNIPHY0_XPCS_ARES				171
-+#define GCC_UNIPHY1_XPCS_ARES				172
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,ipq5424-cmn-pll.h b/include/dt-bindings/clock/qcom,ipq5424-cmn-pll.h
-new file mode 100644
-index 000000000000..f643c2668c04
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,ipq5424-cmn-pll.h
-@@ -0,0 +1,22 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_IPQ5424_CMN_PLL_H
-+#define _DT_BINDINGS_CLK_QCOM_IPQ5424_CMN_PLL_H
-+
-+/* CMN PLL core clock. */
-+#define IPQ5424_CMN_PLL_CLK			0
-+
-+/* The output clocks from CMN PLL of IPQ5424. */
-+#define IPQ5424_XO_24MHZ_CLK			1
-+#define IPQ5424_SLEEP_32KHZ_CLK			2
-+#define IPQ5424_PCS_31P25MHZ_CLK		3
-+#define IPQ5424_NSS_300MHZ_CLK			4
-+#define IPQ5424_PPE_375MHZ_CLK			5
-+#define IPQ5424_ETH0_50MHZ_CLK			6
-+#define IPQ5424_ETH1_50MHZ_CLK			7
-+#define IPQ5424_ETH2_50MHZ_CLK			8
-+#define IPQ5424_ETH_25MHZ_CLK			9
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,ipq5424-gcc.h b/include/dt-bindings/clock/qcom,ipq5424-gcc.h
-new file mode 100644
-index 000000000000..3ae33a0fa002
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,ipq5424-gcc.h
-@@ -0,0 +1,157 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2018,2020 The Linux Foundation. All rights reserved.
-+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLOCK_IPQ_GCC_IPQ5424_H
-+#define _DT_BINDINGS_CLOCK_IPQ_GCC_IPQ5424_H
-+
-+#define GPLL0					0
-+#define GPLL4					1
-+#define GPLL2					2
-+#define GPLL2_OUT_MAIN                          3
-+#define GCC_SLEEP_CLK_SRC			4
-+#define GCC_USB0_EUD_AT_CLK			6
-+#define GCC_PCIE0_AXI_M_CLK_SRC			7
-+#define GCC_PCIE0_AXI_M_CLK			8
-+#define GCC_PCIE1_AXI_M_CLK_SRC			9
-+#define GCC_PCIE1_AXI_M_CLK			10
-+#define GCC_PCIE2_AXI_M_CLK_SRC			11
-+#define GCC_PCIE2_AXI_M_CLK			12
-+#define GCC_PCIE3_AXI_M_CLK_SRC			13
-+#define GCC_PCIE3_AXI_M_CLK			14
-+#define GCC_PCIE0_AXI_S_CLK_SRC			15
-+#define GCC_PCIE0_AXI_S_BRIDGE_CLK		16
-+#define GCC_PCIE0_AXI_S_CLK			17
-+#define GCC_PCIE1_AXI_S_CLK_SRC			18
-+#define GCC_PCIE1_AXI_S_BRIDGE_CLK		19
-+#define GCC_PCIE1_AXI_S_CLK			20
-+#define GCC_PCIE2_AXI_S_CLK_SRC			21
-+#define GCC_PCIE2_AXI_S_BRIDGE_CLK		22
-+#define GCC_PCIE2_AXI_S_CLK			23
-+#define GCC_PCIE3_AXI_S_CLK_SRC			24
-+#define GCC_PCIE3_AXI_S_BRIDGE_CLK		25
-+#define GCC_PCIE3_AXI_S_CLK			26
-+#define GCC_PCIE0_PIPE_CLK_SRC			27
-+#define GCC_PCIE0_PIPE_CLK			28
-+#define GCC_PCIE1_PIPE_CLK_SRC			29
-+#define GCC_PCIE1_PIPE_CLK			30
-+#define GCC_PCIE2_PIPE_CLK_SRC			31
-+#define GCC_PCIE2_PIPE_CLK			32
-+#define GCC_PCIE3_PIPE_CLK_SRC			33
-+#define GCC_PCIE3_PIPE_CLK			34
-+#define GCC_PCIE_AUX_CLK_SRC			35
-+#define GCC_PCIE0_AUX_CLK			36
-+#define GCC_PCIE1_AUX_CLK			37
-+#define GCC_PCIE2_AUX_CLK			38
-+#define GCC_PCIE3_AUX_CLK			39
-+#define GCC_PCIE0_AHB_CLK			40
-+#define GCC_PCIE1_AHB_CLK			41
-+#define GCC_PCIE2_AHB_CLK			42
-+#define GCC_PCIE3_AHB_CLK			43
-+#define GCC_USB0_AUX_CLK_SRC			44
-+#define GCC_USB0_AUX_CLK			45
-+#define GCC_USB0_MASTER_CLK			46
-+#define GCC_USB0_MOCK_UTMI_CLK_SRC		47
-+#define GCC_USB0_MOCK_UTMI_DIV_CLK_SRC		48
-+#define GCC_USB0_MOCK_UTMI_CLK			49
-+#define GCC_USB0_PIPE_CLK_SRC			50
-+#define GCC_USB0_PIPE_CLK			51
-+#define GCC_USB0_PHY_CFG_AHB_CLK		52
-+#define GCC_USB0_SLEEP_CLK			53
-+#define GCC_SDCC1_APPS_CLK_SRC			54
-+#define GCC_SDCC1_APPS_CLK			55
-+#define GCC_SDCC1_ICE_CORE_CLK_SRC		56
-+#define GCC_SDCC1_ICE_CORE_CLK			57
-+#define GCC_SDCC1_AHB_CLK			58
-+#define GCC_PCNOC_BFDCD_CLK_SRC			59
-+#define GCC_NSSCFG_CLK				60
-+#define GCC_NSSNOC_NSSCC_CLK			61
-+#define GCC_NSSCC_CLK				62
-+#define GCC_NSSNOC_PCNOC_1_CLK			63
-+#define GCC_QPIC_AHB_CLK			64
-+#define GCC_QPIC_CLK				65
-+#define GCC_MDIO_AHB_CLK			66
-+#define GCC_PRNG_AHB_CLK			67
-+#define GCC_UNIPHY0_AHB_CLK			68
-+#define GCC_UNIPHY1_AHB_CLK			69
-+#define GCC_UNIPHY2_AHB_CLK			70
-+#define GCC_CMN_12GPLL_AHB_CLK			71
-+#define GCC_SYSTEM_NOC_BFDCD_CLK_SRC		72
-+#define GCC_NSSNOC_SNOC_CLK			73
-+#define GCC_NSSNOC_SNOC_1_CLK			74
-+#define GCC_WCSS_AHB_CLK_SRC			75
-+#define GCC_QDSS_AT_CLK_SRC			76
-+#define GCC_NSSNOC_ATB_CLK			77
-+#define GCC_QDSS_AT_CLK				78
-+#define GCC_QDSS_TSCTR_CLK_SRC			79
-+#define GCC_NSS_TS_CLK				80
-+#define GCC_QPIC_IO_MACRO_CLK_SRC		81
-+#define GCC_QPIC_IO_MACRO_CLK			82
-+#define GCC_LPASS_AXIM_CLK_SRC			83
-+#define GCC_LPASS_CORE_AXIM_CLK			84
-+#define GCC_LPASS_SWAY_CLK_SRC			85
-+#define GCC_LPASS_SWAY_CLK			86
-+#define GCC_CNOC_LPASS_CFG_CLK                  87
-+#define GCC_SNOC_LPASS_CLK                      88
-+#define GCC_ADSS_PWM_CLK_SRC			89
-+#define GCC_ADSS_PWM_CLK			90
-+#define GCC_XO_CLK_SRC				91
-+#define GCC_NSSNOC_XO_DCD_CLK			92
-+#define GCC_NSSNOC_QOSGEN_REF_CLK		93
-+#define GCC_NSSNOC_TIMEOUT_REF_CLK		94
-+#define GCC_UNIPHY0_SYS_CLK			95
-+#define GCC_UNIPHY1_SYS_CLK			96
-+#define GCC_UNIPHY2_SYS_CLK			97
-+#define GCC_CMN_12GPLL_SYS_CLK			98
-+#define GCC_UNIPHY_SYS_CLK_SRC			99
-+#define GCC_NSS_TS_CLK_SRC			100
-+#define GCC_ANOC_PCIE0_1LANE_M_CLK		101
-+#define GCC_ANOC_PCIE1_1LANE_M_CLK		102
-+#define GCC_ANOC_PCIE2_2LANE_M_CLK		103
-+#define GCC_ANOC_PCIE3_2LANE_M_CLK		104
-+#define GCC_CNOC_PCIE0_1LANE_S_CLK		105
-+#define GCC_CNOC_PCIE1_1LANE_S_CLK		106
-+#define GCC_CNOC_PCIE2_2LANE_S_CLK		107
-+#define GCC_CNOC_PCIE3_2LANE_S_CLK		108
-+#define GCC_CNOC_USB_CLK			109
-+#define GCC_CNOC_WCSS_AHB_CLK			110
-+#define GCC_QUPV3_AHB_MST_CLK			111
-+#define GCC_QUPV3_AHB_SLV_CLK			112
-+#define GCC_QUPV3_I2C0_CLK			113
-+#define GCC_QUPV3_I2C1_CLK			114
-+#define GCC_QUPV3_SPI0_CLK			115
-+#define GCC_QUPV3_SPI1_CLK			116
-+#define GCC_QUPV3_UART0_CLK			117
-+#define GCC_QUPV3_UART1_CLK			118
-+#define GCC_QPIC_CLK_SRC			119
-+#define GCC_QUPV3_I2C0_CLK_SRC			120
-+#define GCC_QUPV3_I2C1_CLK_SRC			121
-+#define GCC_QUPV3_I2C0_DIV_CLK_SRC              122
-+#define GCC_QUPV3_I2C1_DIV_CLK_SRC              123
-+#define GCC_QUPV3_SPI0_CLK_SRC			124
-+#define GCC_QUPV3_SPI1_CLK_SRC			125
-+#define GCC_QUPV3_UART0_CLK_SRC			126
-+#define GCC_QUPV3_UART1_CLK_SRC			127
-+#define GCC_USB1_MASTER_CLK			128
-+#define GCC_USB1_MOCK_UTMI_CLK_SRC		129
-+#define GCC_USB1_MOCK_UTMI_DIV_CLK_SRC		130
-+#define GCC_USB1_MOCK_UTMI_CLK			131
-+#define GCC_USB1_SLEEP_CLK			132
-+#define GCC_USB1_PHY_CFG_AHB_CLK		133
-+#define GCC_USB0_MASTER_CLK_SRC			134
-+#define GCC_QDSS_DAP_CLK			135
-+#define GCC_PCIE0_RCHNG_CLK_SRC			136
-+#define GCC_PCIE0_RCHNG_CLK			137
-+#define GCC_PCIE1_RCHNG_CLK_SRC			138
-+#define GCC_PCIE1_RCHNG_CLK			139
-+#define GCC_PCIE2_RCHNG_CLK_SRC			140
-+#define GCC_PCIE2_RCHNG_CLK			141
-+#define GCC_PCIE3_RCHNG_CLK_SRC			142
-+#define GCC_PCIE3_RCHNG_CLK			143
-+#define GCC_IM_SLEEP_CLK			144
-+#define GCC_XO_CLK				145
-+#define GPLL0_OUT_AUX				146
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,ipq5424-nsscc.h b/include/dt-bindings/clock/qcom,ipq5424-nsscc.h
-new file mode 100644
-index 000000000000..eeae0dc38042
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,ipq5424-nsscc.h
-@@ -0,0 +1,65 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLOCK_QCOM_IPQ5424_NSSCC_H
-+#define _DT_BINDINGS_CLOCK_QCOM_IPQ5424_NSSCC_H
-+
-+/* NSS_CC clocks */
-+#define NSS_CC_CE_APB_CLK					0
-+#define NSS_CC_CE_AXI_CLK					1
-+#define NSS_CC_CE_CLK_SRC					2
-+#define NSS_CC_CFG_CLK_SRC					3
-+#define NSS_CC_DEBUG_CLK					4
-+#define NSS_CC_EIP_BFDCD_CLK_SRC				5
-+#define NSS_CC_EIP_CLK						6
-+#define NSS_CC_NSS_CSR_CLK					7
-+#define NSS_CC_NSSNOC_CE_APB_CLK				8
-+#define NSS_CC_NSSNOC_CE_AXI_CLK				9
-+#define NSS_CC_NSSNOC_EIP_CLK					10
-+#define NSS_CC_NSSNOC_NSS_CSR_CLK				11
-+#define NSS_CC_NSSNOC_PPE_CFG_CLK				12
-+#define NSS_CC_NSSNOC_PPE_CLK					13
-+#define NSS_CC_PORT1_MAC_CLK					14
-+#define NSS_CC_PORT1_RX_CLK					15
-+#define NSS_CC_PORT1_RX_CLK_SRC					16
-+#define NSS_CC_PORT1_RX_DIV_CLK_SRC				17
-+#define NSS_CC_PORT1_TX_CLK					18
-+#define NSS_CC_PORT1_TX_CLK_SRC					19
-+#define NSS_CC_PORT1_TX_DIV_CLK_SRC				20
-+#define NSS_CC_PORT2_MAC_CLK					21
-+#define NSS_CC_PORT2_RX_CLK					22
-+#define NSS_CC_PORT2_RX_CLK_SRC					23
-+#define NSS_CC_PORT2_RX_DIV_CLK_SRC				24
-+#define NSS_CC_PORT2_TX_CLK					25
-+#define NSS_CC_PORT2_TX_CLK_SRC					26
-+#define NSS_CC_PORT2_TX_DIV_CLK_SRC				27
-+#define NSS_CC_PORT3_MAC_CLK					28
-+#define NSS_CC_PORT3_RX_CLK					29
-+#define NSS_CC_PORT3_RX_CLK_SRC					30
-+#define NSS_CC_PORT3_RX_DIV_CLK_SRC				31
-+#define NSS_CC_PORT3_TX_CLK					32
-+#define NSS_CC_PORT3_TX_CLK_SRC					33
-+#define NSS_CC_PORT3_TX_DIV_CLK_SRC				34
-+#define NSS_CC_PPE_CLK_SRC					35
-+#define NSS_CC_PPE_EDMA_CFG_CLK					36
-+#define NSS_CC_PPE_EDMA_CLK					37
-+#define NSS_CC_PPE_SWITCH_BTQ_CLK				38
-+#define NSS_CC_PPE_SWITCH_CFG_CLK				39
-+#define NSS_CC_PPE_SWITCH_CLK					40
-+#define NSS_CC_PPE_SWITCH_IPE_CLK				41
-+#define NSS_CC_UNIPHY_PORT1_RX_CLK				42
-+#define NSS_CC_UNIPHY_PORT1_TX_CLK				43
-+#define NSS_CC_UNIPHY_PORT2_RX_CLK				44
-+#define NSS_CC_UNIPHY_PORT2_TX_CLK				45
-+#define NSS_CC_UNIPHY_PORT3_RX_CLK				46
-+#define NSS_CC_UNIPHY_PORT3_TX_CLK				47
-+#define NSS_CC_XGMAC0_PTP_REF_CLK				48
-+#define NSS_CC_XGMAC0_PTP_REF_DIV_CLK_SRC			49
-+#define NSS_CC_XGMAC1_PTP_REF_CLK				50
-+#define NSS_CC_XGMAC1_PTP_REF_DIV_CLK_SRC			51
-+#define NSS_CC_XGMAC2_PTP_REF_CLK				52
-+#define NSS_CC_XGMAC2_PTP_REF_DIV_CLK_SRC			53
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,ipq9574-gcc.h b/include/dt-bindings/clock/qcom,ipq9574-gcc.h
-new file mode 100644
-index 000000000000..0e7c319897f3
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,ipq9574-gcc.h
-@@ -0,0 +1,206 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2018-2023 The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLOCK_IPQ_GCC_9574_H
-+#define _DT_BINDINGS_CLOCK_IPQ_GCC_9574_H
-+
-+#define GPLL0_MAIN					0
-+#define GPLL0						1
-+#define GPLL2_MAIN					2
-+#define GPLL2						3
-+#define GPLL4_MAIN					4
-+#define GPLL4						5
-+#define GCC_SLEEP_CLK_SRC				6
-+#define APSS_AHB_CLK_SRC				7
-+#define APSS_AXI_CLK_SRC				8
-+#define BLSP1_QUP1_I2C_APPS_CLK_SRC			9
-+#define BLSP1_QUP1_SPI_APPS_CLK_SRC			10
-+#define BLSP1_QUP2_I2C_APPS_CLK_SRC			11
-+#define BLSP1_QUP2_SPI_APPS_CLK_SRC			12
-+#define BLSP1_QUP3_I2C_APPS_CLK_SRC			13
-+#define BLSP1_QUP3_SPI_APPS_CLK_SRC			14
-+#define BLSP1_QUP4_I2C_APPS_CLK_SRC			15
-+#define BLSP1_QUP4_SPI_APPS_CLK_SRC			16
-+#define BLSP1_QUP5_I2C_APPS_CLK_SRC			17
-+#define BLSP1_QUP5_SPI_APPS_CLK_SRC			18
-+#define BLSP1_QUP6_I2C_APPS_CLK_SRC			19
-+#define BLSP1_QUP6_SPI_APPS_CLK_SRC			20
-+#define BLSP1_UART1_APPS_CLK_SRC			21
-+#define BLSP1_UART2_APPS_CLK_SRC			22
-+#define BLSP1_UART3_APPS_CLK_SRC			23
-+#define BLSP1_UART4_APPS_CLK_SRC			24
-+#define BLSP1_UART5_APPS_CLK_SRC			25
-+#define BLSP1_UART6_APPS_CLK_SRC			26
-+#define GCC_APSS_AHB_CLK				27
-+#define GCC_APSS_AXI_CLK				28
-+#define GCC_BLSP1_QUP1_I2C_APPS_CLK			29
-+#define GCC_BLSP1_QUP1_SPI_APPS_CLK			30
-+#define GCC_BLSP1_QUP2_I2C_APPS_CLK			31
-+#define GCC_BLSP1_QUP2_SPI_APPS_CLK			32
-+#define GCC_BLSP1_QUP3_I2C_APPS_CLK			33
-+#define GCC_BLSP1_QUP3_SPI_APPS_CLK			34
-+#define GCC_BLSP1_QUP4_I2C_APPS_CLK			35
-+#define GCC_BLSP1_QUP4_SPI_APPS_CLK			36
-+#define GCC_BLSP1_QUP5_I2C_APPS_CLK			37
-+#define GCC_BLSP1_QUP5_SPI_APPS_CLK			38
-+#define GCC_BLSP1_QUP6_I2C_APPS_CLK			39
-+#define GCC_BLSP1_QUP6_SPI_APPS_CLK			40
-+#define GCC_BLSP1_UART1_APPS_CLK			41
-+#define GCC_BLSP1_UART2_APPS_CLK			42
-+#define GCC_BLSP1_UART3_APPS_CLK			43
-+#define GCC_BLSP1_UART4_APPS_CLK			44
-+#define GCC_BLSP1_UART5_APPS_CLK			45
-+#define GCC_BLSP1_UART6_APPS_CLK			46
-+#define PCIE0_AXI_M_CLK_SRC				47
-+#define GCC_PCIE0_AXI_M_CLK				48
-+#define PCIE1_AXI_M_CLK_SRC				49
-+#define GCC_PCIE1_AXI_M_CLK				50
-+#define PCIE2_AXI_M_CLK_SRC				51
-+#define GCC_PCIE2_AXI_M_CLK				52
-+#define PCIE3_AXI_M_CLK_SRC				53
-+#define GCC_PCIE3_AXI_M_CLK				54
-+#define PCIE0_AXI_S_CLK_SRC				55
-+#define GCC_PCIE0_AXI_S_BRIDGE_CLK			56
-+#define GCC_PCIE0_AXI_S_CLK				57
-+#define PCIE1_AXI_S_CLK_SRC				58
-+#define GCC_PCIE1_AXI_S_BRIDGE_CLK			59
-+#define GCC_PCIE1_AXI_S_CLK				60
-+#define PCIE2_AXI_S_CLK_SRC				61
-+#define GCC_PCIE2_AXI_S_BRIDGE_CLK			62
-+#define GCC_PCIE2_AXI_S_CLK				63
-+#define PCIE3_AXI_S_CLK_SRC				64
-+#define GCC_PCIE3_AXI_S_BRIDGE_CLK			65
-+#define GCC_PCIE3_AXI_S_CLK				66
-+#define PCIE0_PIPE_CLK_SRC				67
-+#define PCIE1_PIPE_CLK_SRC				68
-+#define PCIE2_PIPE_CLK_SRC				69
-+#define PCIE3_PIPE_CLK_SRC				70
-+#define PCIE_AUX_CLK_SRC				71
-+#define GCC_PCIE0_AUX_CLK				72
-+#define GCC_PCIE1_AUX_CLK				73
-+#define GCC_PCIE2_AUX_CLK				74
-+#define GCC_PCIE3_AUX_CLK				75
-+#define PCIE0_RCHNG_CLK_SRC				76
-+#define GCC_PCIE0_RCHNG_CLK				77
-+#define PCIE1_RCHNG_CLK_SRC				78
-+#define GCC_PCIE1_RCHNG_CLK				79
-+#define PCIE2_RCHNG_CLK_SRC				80
-+#define GCC_PCIE2_RCHNG_CLK				81
-+#define PCIE3_RCHNG_CLK_SRC				82
-+#define GCC_PCIE3_RCHNG_CLK				83
-+#define GCC_PCIE0_AHB_CLK				84
-+#define GCC_PCIE1_AHB_CLK				85
-+#define GCC_PCIE2_AHB_CLK				86
-+#define GCC_PCIE3_AHB_CLK				87
-+#define USB0_AUX_CLK_SRC				88
-+#define GCC_USB0_AUX_CLK				89
-+#define USB0_MASTER_CLK_SRC				90
-+#define GCC_USB0_MASTER_CLK				91
-+#define GCC_SNOC_USB_CLK				92
-+#define GCC_ANOC_USB_AXI_CLK				93
-+#define USB0_MOCK_UTMI_CLK_SRC				94
-+#define USB0_MOCK_UTMI_DIV_CLK_SRC			95
-+#define GCC_USB0_MOCK_UTMI_CLK				96
-+#define USB0_PIPE_CLK_SRC				97
-+#define GCC_USB0_PHY_CFG_AHB_CLK			98
-+#define SDCC1_APPS_CLK_SRC				99
-+#define GCC_SDCC1_APPS_CLK				100
-+#define SDCC1_ICE_CORE_CLK_SRC				101
-+#define GCC_SDCC1_ICE_CORE_CLK				102
-+#define GCC_SDCC1_AHB_CLK				103
-+#define PCNOC_BFDCD_CLK_SRC				104
-+#define GCC_NSSCFG_CLK					105
-+#define GCC_NSSNOC_NSSCC_CLK				106
-+#define GCC_NSSCC_CLK					107
-+#define GCC_NSSNOC_PCNOC_1_CLK				108
-+#define GCC_QDSS_DAP_AHB_CLK				109
-+#define GCC_QDSS_CFG_AHB_CLK				110
-+#define GCC_QPIC_AHB_CLK				111
-+#define GCC_QPIC_CLK					112
-+#define GCC_BLSP1_AHB_CLK				113
-+#define GCC_MDIO_AHB_CLK				114
-+#define GCC_PRNG_AHB_CLK				115
-+#define GCC_UNIPHY0_AHB_CLK				116
-+#define GCC_UNIPHY1_AHB_CLK				117
-+#define GCC_UNIPHY2_AHB_CLK				118
-+#define GCC_CMN_12GPLL_AHB_CLK				119
-+#define GCC_CMN_12GPLL_APU_CLK				120
-+#define SYSTEM_NOC_BFDCD_CLK_SRC			121
-+#define GCC_NSSNOC_SNOC_CLK				122
-+#define GCC_NSSNOC_SNOC_1_CLK				123
-+#define GCC_QDSS_ETR_USB_CLK				124
-+#define WCSS_AHB_CLK_SRC				125
-+#define WCSS_AXI_M_CLK_SRC				131
-+#define QDSS_AT_CLK_SRC					133
-+#define GCC_NSSNOC_ATB_CLK				136
-+#define GCC_QDSS_AT_CLK					137
-+#define GCC_SYS_NOC_AT_CLK				138
-+#define GCC_PCNOC_AT_CLK				139
-+#define GCC_USB0_EUD_AT_CLK				140
-+#define GCC_QDSS_EUD_AT_CLK				141
-+#define QDSS_STM_CLK_SRC				142
-+#define GCC_QDSS_STM_CLK				143
-+#define GCC_SYS_NOC_QDSS_STM_AXI_CLK			144
-+#define QDSS_TRACECLKIN_CLK_SRC				145
-+#define GCC_QDSS_TRACECLKIN_CLK				146
-+#define QDSS_TSCTR_CLK_SRC				147
-+#define GCC_QDSS_TSCTR_DIV2_CLK				150
-+#define GCC_QDSS_TS_CLK					151
-+#define GCC_QDSS_TSCTR_DIV4_CLK				152
-+#define GCC_NSS_TS_CLK					153
-+#define GCC_QDSS_TSCTR_DIV8_CLK				154
-+#define GCC_QDSS_TSCTR_DIV16_CLK			155
-+#define GCC_QDSS_DAP_CLK				160
-+#define GCC_QDSS_APB2JTAG_CLK				161
-+#define GCC_QDSS_TSCTR_DIV3_CLK				162
-+#define QPIC_IO_MACRO_CLK_SRC				163
-+#define GCC_QPIC_IO_MACRO_CLK                           164
-+#define Q6_AXI_CLK_SRC					165
-+#define Q6_AXIM2_CLK_SRC				169
-+#define NSSNOC_MEMNOC_BFDCD_CLK_SRC			170
-+#define GCC_NSSNOC_MEMNOC_CLK				171
-+#define GCC_NSSNOC_MEM_NOC_1_CLK			172
-+#define GCC_NSS_TBU_CLK					173
-+#define GCC_MEM_NOC_NSSNOC_CLK				174
-+#define LPASS_AXIM_CLK_SRC				175
-+#define LPASS_SWAY_CLK_SRC				176
-+#define ADSS_PWM_CLK_SRC				177
-+#define GCC_ADSS_PWM_CLK				178
-+#define GP1_CLK_SRC					179
-+#define GP2_CLK_SRC					180
-+#define GP3_CLK_SRC					181
-+#define DDRSS_SMS_SLOW_CLK_SRC				182
-+#define GCC_XO_CLK_SRC					183
-+#define GCC_XO_CLK					184
-+#define GCC_NSSNOC_QOSGEN_REF_CLK			185
-+#define GCC_NSSNOC_TIMEOUT_REF_CLK			186
-+#define GCC_XO_DIV4_CLK					187
-+#define GCC_UNIPHY0_SYS_CLK				188
-+#define GCC_UNIPHY1_SYS_CLK				189
-+#define GCC_UNIPHY2_SYS_CLK				190
-+#define GCC_CMN_12GPLL_SYS_CLK				191
-+#define GCC_NSSNOC_XO_DCD_CLK				192
-+#define UNIPHY_SYS_CLK_SRC				194
-+#define NSS_TS_CLK_SRC					195
-+#define GCC_ANOC_PCIE0_1LANE_M_CLK			196
-+#define GCC_ANOC_PCIE1_1LANE_M_CLK			197
-+#define GCC_ANOC_PCIE2_2LANE_M_CLK			198
-+#define GCC_ANOC_PCIE3_2LANE_M_CLK			199
-+#define GCC_SNOC_PCIE0_1LANE_S_CLK			200
-+#define GCC_SNOC_PCIE1_1LANE_S_CLK			201
-+#define GCC_SNOC_PCIE2_2LANE_S_CLK			202
-+#define GCC_SNOC_PCIE3_2LANE_S_CLK			203
-+#define GCC_CRYPTO_CLK_SRC				204
-+#define GCC_CRYPTO_CLK					205
-+#define GCC_CRYPTO_AXI_CLK				206
-+#define GCC_CRYPTO_AHB_CLK				207
-+#define GCC_USB0_PIPE_CLK				208
-+#define GCC_USB0_SLEEP_CLK				209
-+#define GCC_PCIE0_PIPE_CLK				210
-+#define GCC_PCIE1_PIPE_CLK				211
-+#define GCC_PCIE2_PIPE_CLK				212
-+#define GCC_PCIE3_PIPE_CLK				213
-+#define GPLL0_OUT_AUX					214
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,ipq9574-nsscc.h b/include/dt-bindings/clock/qcom,ipq9574-nsscc.h
-new file mode 100644
-index 000000000000..21a16dc0e64c
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,ipq9574-nsscc.h
-@@ -0,0 +1,152 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, 2025 The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLOCK_IPQ_NSSCC_9574_H
-+#define _DT_BINDINGS_CLOCK_IPQ_NSSCC_9574_H
-+
-+#define NSS_CC_CE_APB_CLK					0
-+#define NSS_CC_CE_AXI_CLK					1
-+#define NSS_CC_CE_CLK_SRC					2
-+#define NSS_CC_CFG_CLK_SRC					3
-+#define NSS_CC_CLC_AXI_CLK					4
-+#define NSS_CC_CLC_CLK_SRC					5
-+#define NSS_CC_CRYPTO_CLK					6
-+#define NSS_CC_CRYPTO_CLK_SRC					7
-+#define NSS_CC_CRYPTO_PPE_CLK					8
-+#define NSS_CC_HAQ_AHB_CLK					9
-+#define NSS_CC_HAQ_AXI_CLK					10
-+#define NSS_CC_HAQ_CLK_SRC					11
-+#define NSS_CC_IMEM_AHB_CLK					12
-+#define NSS_CC_IMEM_CLK_SRC					13
-+#define NSS_CC_IMEM_QSB_CLK					14
-+#define NSS_CC_INT_CFG_CLK_SRC					15
-+#define NSS_CC_NSS_CSR_CLK					16
-+#define NSS_CC_NSSNOC_CE_APB_CLK				17
-+#define NSS_CC_NSSNOC_CE_AXI_CLK				18
-+#define NSS_CC_NSSNOC_CLC_AXI_CLK				19
-+#define NSS_CC_NSSNOC_CRYPTO_CLK				20
-+#define NSS_CC_NSSNOC_HAQ_AHB_CLK				21
-+#define NSS_CC_NSSNOC_HAQ_AXI_CLK				22
-+#define NSS_CC_NSSNOC_IMEM_AHB_CLK				23
-+#define NSS_CC_NSSNOC_IMEM_QSB_CLK				24
-+#define NSS_CC_NSSNOC_NSS_CSR_CLK				25
-+#define NSS_CC_NSSNOC_PPE_CFG_CLK				26
-+#define NSS_CC_NSSNOC_PPE_CLK					27
-+#define NSS_CC_NSSNOC_UBI32_AHB0_CLK				28
-+#define NSS_CC_NSSNOC_UBI32_AXI0_CLK				29
-+#define NSS_CC_NSSNOC_UBI32_INT0_AHB_CLK			30
-+#define NSS_CC_NSSNOC_UBI32_NC_AXI0_1_CLK			31
-+#define NSS_CC_NSSNOC_UBI32_NC_AXI0_CLK				32
-+#define NSS_CC_PORT1_MAC_CLK					33
-+#define NSS_CC_PORT1_RX_CLK					34
-+#define NSS_CC_PORT1_RX_CLK_SRC					35
-+#define NSS_CC_PORT1_RX_DIV_CLK_SRC				36
-+#define NSS_CC_PORT1_TX_CLK					37
-+#define NSS_CC_PORT1_TX_CLK_SRC					38
-+#define NSS_CC_PORT1_TX_DIV_CLK_SRC				39
-+#define NSS_CC_PORT2_MAC_CLK					40
-+#define NSS_CC_PORT2_RX_CLK					41
-+#define NSS_CC_PORT2_RX_CLK_SRC					42
-+#define NSS_CC_PORT2_RX_DIV_CLK_SRC				43
-+#define NSS_CC_PORT2_TX_CLK					44
-+#define NSS_CC_PORT2_TX_CLK_SRC					45
-+#define NSS_CC_PORT2_TX_DIV_CLK_SRC				46
-+#define NSS_CC_PORT3_MAC_CLK					47
-+#define NSS_CC_PORT3_RX_CLK					48
-+#define NSS_CC_PORT3_RX_CLK_SRC					49
-+#define NSS_CC_PORT3_RX_DIV_CLK_SRC				50
-+#define NSS_CC_PORT3_TX_CLK					51
-+#define NSS_CC_PORT3_TX_CLK_SRC					52
-+#define NSS_CC_PORT3_TX_DIV_CLK_SRC				53
-+#define NSS_CC_PORT4_MAC_CLK					54
-+#define NSS_CC_PORT4_RX_CLK					55
-+#define NSS_CC_PORT4_RX_CLK_SRC					56
-+#define NSS_CC_PORT4_RX_DIV_CLK_SRC				57
-+#define NSS_CC_PORT4_TX_CLK					58
-+#define NSS_CC_PORT4_TX_CLK_SRC					59
-+#define NSS_CC_PORT4_TX_DIV_CLK_SRC				60
-+#define NSS_CC_PORT5_MAC_CLK					61
-+#define NSS_CC_PORT5_RX_CLK					62
-+#define NSS_CC_PORT5_RX_CLK_SRC					63
-+#define NSS_CC_PORT5_RX_DIV_CLK_SRC				64
-+#define NSS_CC_PORT5_TX_CLK					65
-+#define NSS_CC_PORT5_TX_CLK_SRC					66
-+#define NSS_CC_PORT5_TX_DIV_CLK_SRC				67
-+#define NSS_CC_PORT6_MAC_CLK					68
-+#define NSS_CC_PORT6_RX_CLK					69
-+#define NSS_CC_PORT6_RX_CLK_SRC					70
-+#define NSS_CC_PORT6_RX_DIV_CLK_SRC				71
-+#define NSS_CC_PORT6_TX_CLK					72
-+#define NSS_CC_PORT6_TX_CLK_SRC					73
-+#define NSS_CC_PORT6_TX_DIV_CLK_SRC				74
-+#define NSS_CC_PPE_CLK_SRC					75
-+#define NSS_CC_PPE_EDMA_CFG_CLK					76
-+#define NSS_CC_PPE_EDMA_CLK					77
-+#define NSS_CC_PPE_SWITCH_BTQ_CLK				78
-+#define NSS_CC_PPE_SWITCH_CFG_CLK				79
-+#define NSS_CC_PPE_SWITCH_CLK					80
-+#define NSS_CC_PPE_SWITCH_IPE_CLK				81
-+#define NSS_CC_UBI0_CLK_SRC					82
-+#define NSS_CC_UBI0_DIV_CLK_SRC					83
-+#define NSS_CC_UBI1_CLK_SRC					84
-+#define NSS_CC_UBI1_DIV_CLK_SRC					85
-+#define NSS_CC_UBI2_CLK_SRC					86
-+#define NSS_CC_UBI2_DIV_CLK_SRC					87
-+#define NSS_CC_UBI32_AHB0_CLK					88
-+#define NSS_CC_UBI32_AHB1_CLK					89
-+#define NSS_CC_UBI32_AHB2_CLK					90
-+#define NSS_CC_UBI32_AHB3_CLK					91
-+#define NSS_CC_UBI32_AXI0_CLK					92
-+#define NSS_CC_UBI32_AXI1_CLK					93
-+#define NSS_CC_UBI32_AXI2_CLK					94
-+#define NSS_CC_UBI32_AXI3_CLK					95
-+#define NSS_CC_UBI32_CORE0_CLK					96
-+#define NSS_CC_UBI32_CORE1_CLK					97
-+#define NSS_CC_UBI32_CORE2_CLK					98
-+#define NSS_CC_UBI32_CORE3_CLK					99
-+#define NSS_CC_UBI32_INTR0_AHB_CLK				100
-+#define NSS_CC_UBI32_INTR1_AHB_CLK				101
-+#define NSS_CC_UBI32_INTR2_AHB_CLK				102
-+#define NSS_CC_UBI32_INTR3_AHB_CLK				103
-+#define NSS_CC_UBI32_NC_AXI0_CLK				104
-+#define NSS_CC_UBI32_NC_AXI1_CLK				105
-+#define NSS_CC_UBI32_NC_AXI2_CLK				106
-+#define NSS_CC_UBI32_NC_AXI3_CLK				107
-+#define NSS_CC_UBI32_UTCM0_CLK					108
-+#define NSS_CC_UBI32_UTCM1_CLK					109
-+#define NSS_CC_UBI32_UTCM2_CLK					110
-+#define NSS_CC_UBI32_UTCM3_CLK					111
-+#define NSS_CC_UBI3_CLK_SRC					112
-+#define NSS_CC_UBI3_DIV_CLK_SRC					113
-+#define NSS_CC_UBI_AXI_CLK_SRC					114
-+#define NSS_CC_UBI_NC_AXI_BFDCD_CLK_SRC				115
-+#define NSS_CC_UNIPHY_PORT1_RX_CLK				116
-+#define NSS_CC_UNIPHY_PORT1_TX_CLK				117
-+#define NSS_CC_UNIPHY_PORT2_RX_CLK				118
-+#define NSS_CC_UNIPHY_PORT2_TX_CLK				119
-+#define NSS_CC_UNIPHY_PORT3_RX_CLK				120
-+#define NSS_CC_UNIPHY_PORT3_TX_CLK				121
-+#define NSS_CC_UNIPHY_PORT4_RX_CLK				122
-+#define NSS_CC_UNIPHY_PORT4_TX_CLK				123
-+#define NSS_CC_UNIPHY_PORT5_RX_CLK				124
-+#define NSS_CC_UNIPHY_PORT5_TX_CLK				125
-+#define NSS_CC_UNIPHY_PORT6_RX_CLK				126
-+#define NSS_CC_UNIPHY_PORT6_TX_CLK				127
-+#define NSS_CC_XGMAC0_PTP_REF_CLK				128
-+#define NSS_CC_XGMAC0_PTP_REF_DIV_CLK_SRC			129
-+#define NSS_CC_XGMAC1_PTP_REF_CLK				130
-+#define NSS_CC_XGMAC1_PTP_REF_DIV_CLK_SRC			131
-+#define NSS_CC_XGMAC2_PTP_REF_CLK				132
-+#define NSS_CC_XGMAC2_PTP_REF_DIV_CLK_SRC			133
-+#define NSS_CC_XGMAC3_PTP_REF_CLK				134
-+#define NSS_CC_XGMAC3_PTP_REF_DIV_CLK_SRC			135
-+#define NSS_CC_XGMAC4_PTP_REF_CLK				136
-+#define NSS_CC_XGMAC4_PTP_REF_DIV_CLK_SRC			137
-+#define NSS_CC_XGMAC5_PTP_REF_CLK				138
-+#define NSS_CC_XGMAC5_PTP_REF_DIV_CLK_SRC			139
-+#define UBI32_PLL						140
-+#define UBI32_PLL_MAIN						141
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,kaanapali-gcc.h b/include/dt-bindings/clock/qcom,kaanapali-gcc.h
-new file mode 100644
-index 000000000000..890e48709f09
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,kaanapali-gcc.h
-@@ -0,0 +1,241 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_KAANAPALI_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_KAANAPALI_H
-+
-+/* GCC clocks */
-+#define GCC_AGGRE_NOC_PCIE_AXI_CLK				0
-+#define GCC_AGGRE_UFS_PHY_AXI_CLK				1
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK				2
-+#define GCC_BOOT_ROM_AHB_CLK					3
-+#define GCC_CAM_BIST_MCLK_AHB_CLK				4
-+#define GCC_CAMERA_AHB_CLK					5
-+#define GCC_CAMERA_HF_AXI_CLK					6
-+#define GCC_CAMERA_SF_AXI_CLK					7
-+#define GCC_CAMERA_XO_CLK					8
-+#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK				9
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				10
-+#define GCC_CNOC_PCIE_SF_AXI_CLK				11
-+#define GCC_DDRSS_PCIE_SF_QTB_CLK				12
-+#define GCC_QMIP_CAMERA_CMD_AHB_CLK				13
-+#define GCC_DISP_HF_AXI_CLK					14
-+#define GCC_DISP_SF_AXI_CLK					15
-+#define GCC_EVA_AHB_CLK						16
-+#define GCC_EVA_AXI0_CLK					17
-+#define GCC_EVA_AXI0C_CLK					18
-+#define GCC_EVA_XO_CLK						19
-+#define GCC_GP1_CLK						20
-+#define GCC_GP1_CLK_SRC						21
-+#define GCC_GP2_CLK						22
-+#define GCC_GP2_CLK_SRC						23
-+#define GCC_GP3_CLK						24
-+#define GCC_GP3_CLK_SRC						25
-+#define GCC_GPLL0						26
-+#define GCC_GPLL0_OUT_EVEN					27
-+#define GCC_GPLL1						28
-+#define GCC_GPLL4						29
-+#define GCC_GPLL7						30
-+#define GCC_GPLL9						31
-+#define GCC_GPU_CFG_AHB_CLK					32
-+#define GCC_GPU_GEMNOC_GFX_CLK					33
-+#define GCC_GPU_GPLL0_CLK_SRC					34
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC				35
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				36
-+#define GCC_QMIP_GPU_AHB_CLK					37
-+#define GCC_PCIE_0_AUX_CLK					38
-+#define GCC_PCIE_0_AUX_CLK_SRC					39
-+#define GCC_PCIE_0_CFG_AHB_CLK					40
-+#define GCC_PCIE_0_MSTR_AXI_CLK					41
-+#define GCC_PCIE_0_PHY_AUX_CLK					42
-+#define GCC_PCIE_0_PHY_AUX_CLK_SRC				43
-+#define GCC_PCIE_0_PHY_RCHNG_CLK				44
-+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC				45
-+#define GCC_PCIE_0_PIPE_CLK					46
-+#define GCC_PCIE_0_PIPE_CLK_SRC					47
-+#define GCC_PCIE_0_SLV_AXI_CLK					48
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				49
-+#define GCC_PCIE_RSCC_CFG_AHB_CLK				50
-+#define GCC_PCIE_RSCC_XO_CLK					51
-+#define GCC_PDM2_CLK						52
-+#define GCC_PDM2_CLK_SRC					53
-+#define GCC_PDM_AHB_CLK						54
-+#define GCC_PDM_XO4_CLK						55
-+#define GCC_QUPV3_I2C_CORE_CLK					56
-+#define GCC_QUPV3_I2C_S0_CLK					57
-+#define GCC_QUPV3_I2C_S0_CLK_SRC				58
-+#define GCC_QUPV3_I2C_S1_CLK					59
-+#define GCC_QUPV3_I2C_S1_CLK_SRC				60
-+#define GCC_QUPV3_I2C_S2_CLK					61
-+#define GCC_QUPV3_I2C_S2_CLK_SRC				62
-+#define GCC_QUPV3_I2C_S3_CLK					63
-+#define GCC_QUPV3_I2C_S3_CLK_SRC				64
-+#define GCC_QUPV3_I2C_S4_CLK					65
-+#define GCC_QUPV3_I2C_S4_CLK_SRC				66
-+#define GCC_QUPV3_I2C_S_AHB_CLK					67
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				68
-+#define GCC_QUPV3_WRAP1_CORE_CLK				69
-+#define GCC_QUPV3_WRAP1_QSPI_REF_CLK				70
-+#define GCC_QUPV3_WRAP1_QSPI_REF_CLK_SRC			71
-+#define GCC_QUPV3_WRAP1_S0_CLK					72
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				73
-+#define GCC_QUPV3_WRAP1_S1_CLK					74
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				75
-+#define GCC_QUPV3_WRAP1_S2_CLK					76
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				77
-+#define GCC_QUPV3_WRAP1_S3_CLK					78
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				79
-+#define GCC_QUPV3_WRAP1_S4_CLK					80
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				81
-+#define GCC_QUPV3_WRAP1_S5_CLK					82
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC				83
-+#define GCC_QUPV3_WRAP1_S6_CLK					84
-+#define GCC_QUPV3_WRAP1_S6_CLK_SRC				85
-+#define GCC_QUPV3_WRAP1_S7_CLK					86
-+#define GCC_QUPV3_WRAP1_S7_CLK_SRC				87
-+#define GCC_QUPV3_WRAP2_CORE_2X_CLK				88
-+#define GCC_QUPV3_WRAP2_CORE_CLK				89
-+#define GCC_QUPV3_WRAP2_S0_CLK					90
-+#define GCC_QUPV3_WRAP2_S0_CLK_SRC				91
-+#define GCC_QUPV3_WRAP2_S1_CLK					92
-+#define GCC_QUPV3_WRAP2_S1_CLK_SRC				93
-+#define GCC_QUPV3_WRAP2_S2_CLK					94
-+#define GCC_QUPV3_WRAP2_S2_CLK_SRC				95
-+#define GCC_QUPV3_WRAP2_S3_CLK					96
-+#define GCC_QUPV3_WRAP2_S3_CLK_SRC				97
-+#define GCC_QUPV3_WRAP2_S4_CLK					98
-+#define GCC_QUPV3_WRAP2_S4_CLK_SRC				99
-+#define GCC_QUPV3_WRAP3_CORE_2X_CLK				100
-+#define GCC_QUPV3_WRAP3_CORE_CLK				101
-+#define GCC_QUPV3_WRAP3_IBI_CTRL_0_CLK_SRC			102
-+#define GCC_QUPV3_WRAP3_IBI_CTRL_1_CLK				103
-+#define GCC_QUPV3_WRAP3_IBI_CTRL_2_CLK				104
-+#define GCC_QUPV3_WRAP3_S0_CLK					105
-+#define GCC_QUPV3_WRAP3_S0_CLK_SRC				106
-+#define GCC_QUPV3_WRAP3_S1_CLK					107
-+#define GCC_QUPV3_WRAP3_S1_CLK_SRC				108
-+#define GCC_QUPV3_WRAP3_S2_CLK					109
-+#define GCC_QUPV3_WRAP3_S2_CLK_SRC				110
-+#define GCC_QUPV3_WRAP3_S3_CLK					111
-+#define GCC_QUPV3_WRAP3_S3_CLK_SRC				112
-+#define GCC_QUPV3_WRAP3_S4_CLK					113
-+#define GCC_QUPV3_WRAP3_S4_CLK_SRC				114
-+#define GCC_QUPV3_WRAP3_S5_CLK					115
-+#define GCC_QUPV3_WRAP3_S5_CLK_SRC				116
-+#define GCC_QUPV3_WRAP4_CORE_2X_CLK				117
-+#define GCC_QUPV3_WRAP4_CORE_CLK				118
-+#define GCC_QUPV3_WRAP4_S0_CLK					119
-+#define GCC_QUPV3_WRAP4_S0_CLK_SRC				120
-+#define GCC_QUPV3_WRAP4_S1_CLK					121
-+#define GCC_QUPV3_WRAP4_S1_CLK_SRC				122
-+#define GCC_QUPV3_WRAP4_S2_CLK					123
-+#define GCC_QUPV3_WRAP4_S2_CLK_SRC				124
-+#define GCC_QUPV3_WRAP4_S3_CLK					125
-+#define GCC_QUPV3_WRAP4_S3_CLK_SRC				126
-+#define GCC_QUPV3_WRAP4_S4_CLK					127
-+#define GCC_QUPV3_WRAP4_S4_CLK_SRC				128
-+#define GCC_QUPV3_WRAP_1_M_AXI_CLK				129
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				130
-+#define GCC_QUPV3_WRAP_2_M_AHB_CLK				131
-+#define GCC_QUPV3_WRAP_2_S_AHB_CLK				132
-+#define GCC_QUPV3_WRAP_3_IBI_1_AHB_CLK				133
-+#define GCC_QUPV3_WRAP_3_IBI_2_AHB_CLK				134
-+#define GCC_QUPV3_WRAP_3_M_AHB_CLK				135
-+#define GCC_QUPV3_WRAP_3_S_AHB_CLK				136
-+#define GCC_QUPV3_WRAP_4_M_AHB_CLK				137
-+#define GCC_QUPV3_WRAP_4_S_AHB_CLK				138
-+#define GCC_SDCC2_AHB_CLK					139
-+#define GCC_SDCC2_APPS_CLK					140
-+#define GCC_SDCC2_APPS_CLK_SRC					141
-+#define GCC_SDCC4_AHB_CLK					142
-+#define GCC_SDCC4_APPS_CLK					143
-+#define GCC_SDCC4_APPS_CLK_SRC					144
-+#define GCC_UFS_PHY_AHB_CLK					145
-+#define GCC_UFS_PHY_AXI_CLK					146
-+#define GCC_UFS_PHY_AXI_CLK_SRC					147
-+#define GCC_UFS_PHY_ICE_CORE_CLK				148
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC				149
-+#define GCC_UFS_PHY_PHY_AUX_CLK					150
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC				151
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK				152
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC				153
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK				154
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC				155
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK				156
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC				157
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK				158
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC				159
-+#define GCC_USB30_PRIM_MASTER_CLK				160
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC				161
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				162
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			163
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		164
-+#define GCC_USB30_PRIM_SLEEP_CLK				165
-+#define GCC_USB3_PRIM_PHY_AUX_CLK				166
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				167
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				168
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK				169
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				170
-+#define GCC_VIDEO_AHB_CLK					171
-+#define GCC_VIDEO_AXI0_CLK					172
-+#define GCC_VIDEO_AXI1_CLK					173
-+#define GCC_VIDEO_XO_CLK					174
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				175
-+#define GCC_QMIP_CAMERA_RT_AHB_CLK				176
-+#define GCC_QMIP_DISP_DCP_SF_AHB_CLK				177
-+#define GCC_QMIP_PCIE_AHB_CLK					178
-+#define GCC_QMIP_VIDEO_CV_CPU_AHB_CLK				179
-+#define GCC_QMIP_VIDEO_CVP_AHB_CLK				180
-+#define GCC_QMIP_VIDEO_V_CPU_AHB_CLK				181
-+#define GCC_DISP_AHB_CLK					182
-+
-+/* GCC power domains */
-+#define GCC_PCIE_0_GDSC						0
-+#define GCC_PCIE_0_PHY_GDSC					1
-+#define GCC_UFS_MEM_PHY_GDSC					2
-+#define GCC_UFS_PHY_GDSC					3
-+#define GCC_USB30_PRIM_GDSC					4
-+#define GCC_USB3_PHY_GDSC					5
-+
-+/* GCC resets */
-+#define GCC_CAMERA_BCR						0
-+#define GCC_DISPLAY_BCR						1
-+#define GCC_EVA_AXI0_CLK_ARES					2
-+#define GCC_EVA_AXI0C_CLK_ARES					3
-+#define GCC_EVA_BCR						4
-+#define GCC_GPU_BCR						5
-+#define GCC_PCIE_0_BCR						6
-+#define GCC_PCIE_0_LINK_DOWN_BCR				7
-+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR				8
-+#define GCC_PCIE_0_PHY_BCR					9
-+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR			10
-+#define GCC_PCIE_PHY_BCR					11
-+#define GCC_PCIE_PHY_CFG_AHB_BCR				12
-+#define GCC_PCIE_PHY_COM_BCR					13
-+#define GCC_PCIE_RSCC_BCR					14
-+#define GCC_PDM_BCR						15
-+#define GCC_QUPV3_WRAPPER_1_BCR					16
-+#define GCC_QUPV3_WRAPPER_2_BCR					17
-+#define GCC_QUPV3_WRAPPER_3_BCR					18
-+#define GCC_QUPV3_WRAPPER_4_BCR					19
-+#define GCC_QUPV3_WRAPPER_I2C_BCR				20
-+#define GCC_QUSB2PHY_PRIM_BCR					21
-+#define GCC_QUSB2PHY_SEC_BCR					22
-+#define GCC_SDCC2_BCR						23
-+#define GCC_SDCC4_BCR						24
-+#define GCC_UFS_PHY_BCR						25
-+#define GCC_USB30_PRIM_BCR					26
-+#define GCC_USB3_DP_PHY_PRIM_BCR				27
-+#define GCC_USB3_DP_PHY_SEC_BCR					28
-+#define GCC_USB3_PHY_PRIM_BCR					29
-+#define GCC_USB3_PHY_SEC_BCR					30
-+#define GCC_USB3PHY_PHY_PRIM_BCR				31
-+#define GCC_USB3PHY_PHY_SEC_BCR					32
-+#define GCC_VIDEO_AXI0_CLK_ARES					33
-+#define GCC_VIDEO_AXI1_CLK_ARES					34
-+#define GCC_VIDEO_BCR						35
-+#define GCC_VIDEO_XO_CLK_ARES					36
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,lcc-ipq806x.h b/include/dt-bindings/clock/qcom,lcc-ipq806x.h
-index 25b92bbf0ab4..e0fb4acf4ba8 100644
--- a/include/dt-bindings/clock/qcom,lcc-ipq806x.h
-+++ b/include/dt-bindings/clock/qcom,lcc-ipq806x.h
-@@ -19,4 +19,6 @@
- #define SPDIF_CLK			10
- #define AHBIX_CLK			11
- 
-+#define LCC_PCM_RESET			0
-+
- #endif
-diff --git a/include/dt-bindings/clock/qcom,lcc-mdm9615.h b/include/dt-bindings/clock/qcom,lcc-mdm9615.h
-deleted file mode 100644
-index 299338ee1d88..000000000000
--- a/include/dt-bindings/clock/qcom,lcc-mdm9615.h
-+++ /dev/null
-@@ -1,44 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2014, The Linux Foundation. All rights reserved.
- * Copyright (c) BayLibre, SAS.
- * Author : Neil Armstrong <narmstrong@baylibre.com>
- */
-
-#ifndef _DT_BINDINGS_CLK_LCC_MDM9615_H
-#define _DT_BINDINGS_CLK_LCC_MDM9615_H
-
-#define PLL4				0
-#define MI2S_OSR_SRC			1
-#define MI2S_OSR_CLK			2
-#define MI2S_DIV_CLK			3
-#define MI2S_BIT_DIV_CLK		4
-#define MI2S_BIT_CLK			5
-#define PCM_SRC				6
-#define PCM_CLK_OUT			7
-#define PCM_CLK				8
-#define SLIMBUS_SRC			9
-#define AUDIO_SLIMBUS_CLK		10
-#define SPS_SLIMBUS_CLK			11
-#define CODEC_I2S_MIC_OSR_SRC		12
-#define CODEC_I2S_MIC_OSR_CLK		13
-#define CODEC_I2S_MIC_DIV_CLK		14
-#define CODEC_I2S_MIC_BIT_DIV_CLK	15
-#define CODEC_I2S_MIC_BIT_CLK		16
-#define SPARE_I2S_MIC_OSR_SRC		17
-#define SPARE_I2S_MIC_OSR_CLK		18
-#define SPARE_I2S_MIC_DIV_CLK		19
-#define SPARE_I2S_MIC_BIT_DIV_CLK	20
-#define SPARE_I2S_MIC_BIT_CLK		21
-#define CODEC_I2S_SPKR_OSR_SRC		22
-#define CODEC_I2S_SPKR_OSR_CLK		23
-#define CODEC_I2S_SPKR_DIV_CLK		24
-#define CODEC_I2S_SPKR_BIT_DIV_CLK	25
-#define CODEC_I2S_SPKR_BIT_CLK		26
-#define SPARE_I2S_SPKR_OSR_SRC		27
-#define SPARE_I2S_SPKR_OSR_CLK		28
-#define SPARE_I2S_SPKR_DIV_CLK		29
-#define SPARE_I2S_SPKR_BIT_DIV_CLK	30
-#define SPARE_I2S_SPKR_BIT_CLK		31
-
-#endif
-diff --git a/include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h b/include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h
-index 20ef2ea673f3..22dcd47d4513 100644
--- a/include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h
-+++ b/include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h
-@@ -24,6 +24,11 @@
- #define LPASS_AUDIO_CC_RX_MCLK_CLK			14
- #define LPASS_AUDIO_CC_RX_MCLK_CLK_SRC			15
- 
-+/* LPASS AUDIO CC CSR */
-+#define LPASS_AUDIO_SWR_RX_CGCR				0
-+#define LPASS_AUDIO_SWR_TX_CGCR				1
-+#define LPASS_AUDIO_SWR_WSA_CGCR			2
-+
- /* LPASS_AON_CC clocks */
- #define LPASS_AON_CC_PLL				0
- #define LPASS_AON_CC_PLL_OUT_EVEN			1
-diff --git a/include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h b/include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h
-index 28ed2a07aacc..0324c69ce968 100644
--- a/include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h
-+++ b/include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h
-@@ -19,6 +19,8 @@
- #define LPASS_CORE_CC_LPM_CORE_CLK			9
- #define LPASS_CORE_CC_LPM_MEM0_CORE_CLK			10
- #define LPASS_CORE_CC_SYSNOC_MPORT_CORE_CLK		11
-+#define LPASS_CORE_CC_EXT_MCLK0_CLK			12
-+#define LPASS_CORE_CC_EXT_MCLK0_CLK_SRC			13
- 
- /* LPASS_CORE_CC power domains */
- #define LPASS_CORE_CC_LPASS_CORE_HM_GDSC		0
-diff --git a/include/dt-bindings/clock/qcom,milos-camcc.h b/include/dt-bindings/clock/qcom,milos-camcc.h
-new file mode 100644
-index 000000000000..21925dca9a20
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,milos-camcc.h
-@@ -0,0 +1,131 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2025, Luca Weiss <luca.weiss@fairphone.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_MILOS_H
-+#define _DT_BINDINGS_CLK_QCOM_CAM_CC_MILOS_H
-+
-+/* CAM_CC clocks */
-+#define CAM_CC_PLL0						0
-+#define CAM_CC_PLL0_OUT_EVEN					1
-+#define CAM_CC_PLL0_OUT_ODD					2
-+#define CAM_CC_PLL1						3
-+#define CAM_CC_PLL1_OUT_EVEN					4
-+#define CAM_CC_PLL2						5
-+#define CAM_CC_PLL2_OUT_EVEN					6
-+#define CAM_CC_PLL3						7
-+#define CAM_CC_PLL3_OUT_EVEN					8
-+#define CAM_CC_PLL4						9
-+#define CAM_CC_PLL4_OUT_EVEN					10
-+#define CAM_CC_PLL5						11
-+#define CAM_CC_PLL5_OUT_EVEN					12
-+#define CAM_CC_PLL6						13
-+#define CAM_CC_PLL6_OUT_EVEN					14
-+#define CAM_CC_BPS_AHB_CLK					15
-+#define CAM_CC_BPS_AREG_CLK					16
-+#define CAM_CC_BPS_CLK						17
-+#define CAM_CC_BPS_CLK_SRC					18
-+#define CAM_CC_CAMNOC_ATB_CLK					19
-+#define CAM_CC_CAMNOC_AXI_CLK_SRC				20
-+#define CAM_CC_CAMNOC_AXI_HF_CLK				21
-+#define CAM_CC_CAMNOC_AXI_SF_CLK				22
-+#define CAM_CC_CAMNOC_NRT_AXI_CLK				23
-+#define CAM_CC_CAMNOC_RT_AXI_CLK				24
-+#define CAM_CC_CCI_0_CLK					25
-+#define CAM_CC_CCI_0_CLK_SRC					26
-+#define CAM_CC_CCI_1_CLK					27
-+#define CAM_CC_CCI_1_CLK_SRC					28
-+#define CAM_CC_CORE_AHB_CLK					29
-+#define CAM_CC_CPAS_AHB_CLK					30
-+#define CAM_CC_CPHY_RX_CLK_SRC					31
-+#define CAM_CC_CRE_AHB_CLK					32
-+#define CAM_CC_CRE_CLK						33
-+#define CAM_CC_CRE_CLK_SRC					34
-+#define CAM_CC_CSI0PHYTIMER_CLK					35
-+#define CAM_CC_CSI0PHYTIMER_CLK_SRC				36
-+#define CAM_CC_CSI1PHYTIMER_CLK					37
-+#define CAM_CC_CSI1PHYTIMER_CLK_SRC				38
-+#define CAM_CC_CSI2PHYTIMER_CLK					39
-+#define CAM_CC_CSI2PHYTIMER_CLK_SRC				40
-+#define CAM_CC_CSI3PHYTIMER_CLK					41
-+#define CAM_CC_CSI3PHYTIMER_CLK_SRC				42
-+#define CAM_CC_CSIPHY0_CLK					43
-+#define CAM_CC_CSIPHY1_CLK					44
-+#define CAM_CC_CSIPHY2_CLK					45
-+#define CAM_CC_CSIPHY3_CLK					46
-+#define CAM_CC_FAST_AHB_CLK_SRC					47
-+#define CAM_CC_GDSC_CLK						48
-+#define CAM_CC_ICP_ATB_CLK					49
-+#define CAM_CC_ICP_CLK						50
-+#define CAM_CC_ICP_CLK_SRC					51
-+#define CAM_CC_ICP_CTI_CLK					52
-+#define CAM_CC_ICP_TS_CLK					53
-+#define CAM_CC_MCLK0_CLK					54
-+#define CAM_CC_MCLK0_CLK_SRC					55
-+#define CAM_CC_MCLK1_CLK					56
-+#define CAM_CC_MCLK1_CLK_SRC					57
-+#define CAM_CC_MCLK2_CLK					58
-+#define CAM_CC_MCLK2_CLK_SRC					59
-+#define CAM_CC_MCLK3_CLK					60
-+#define CAM_CC_MCLK3_CLK_SRC					61
-+#define CAM_CC_MCLK4_CLK					62
-+#define CAM_CC_MCLK4_CLK_SRC					63
-+#define CAM_CC_OPE_0_AHB_CLK					64
-+#define CAM_CC_OPE_0_AREG_CLK					65
-+#define CAM_CC_OPE_0_CLK					66
-+#define CAM_CC_OPE_0_CLK_SRC					67
-+#define CAM_CC_SLEEP_CLK					68
-+#define CAM_CC_SLEEP_CLK_SRC					69
-+#define CAM_CC_SLOW_AHB_CLK_SRC					70
-+#define CAM_CC_SOC_AHB_CLK					71
-+#define CAM_CC_SYS_TMR_CLK					72
-+#define CAM_CC_TFE_0_AHB_CLK					73
-+#define CAM_CC_TFE_0_CLK					74
-+#define CAM_CC_TFE_0_CLK_SRC					75
-+#define CAM_CC_TFE_0_CPHY_RX_CLK				76
-+#define CAM_CC_TFE_0_CSID_CLK					77
-+#define CAM_CC_TFE_0_CSID_CLK_SRC				78
-+#define CAM_CC_TFE_1_AHB_CLK					79
-+#define CAM_CC_TFE_1_CLK					80
-+#define CAM_CC_TFE_1_CLK_SRC					81
-+#define CAM_CC_TFE_1_CPHY_RX_CLK				82
-+#define CAM_CC_TFE_1_CSID_CLK					83
-+#define CAM_CC_TFE_1_CSID_CLK_SRC				84
-+#define CAM_CC_TFE_2_AHB_CLK					85
-+#define CAM_CC_TFE_2_CLK					86
-+#define CAM_CC_TFE_2_CLK_SRC					87
-+#define CAM_CC_TFE_2_CPHY_RX_CLK				88
-+#define CAM_CC_TFE_2_CSID_CLK					89
-+#define CAM_CC_TFE_2_CSID_CLK_SRC				90
-+#define CAM_CC_TOP_SHIFT_CLK					91
-+#define CAM_CC_XO_CLK_SRC					92
-+
-+/* CAM_CC resets */
-+#define CAM_CC_BPS_BCR						0
-+#define CAM_CC_CAMNOC_BCR					1
-+#define CAM_CC_CAMSS_TOP_BCR					2
-+#define CAM_CC_CCI_0_BCR					3
-+#define CAM_CC_CCI_1_BCR					4
-+#define CAM_CC_CPAS_BCR						5
-+#define CAM_CC_CRE_BCR						6
-+#define CAM_CC_CSI0PHY_BCR					7
-+#define CAM_CC_CSI1PHY_BCR					8
-+#define CAM_CC_CSI2PHY_BCR					9
-+#define CAM_CC_CSI3PHY_BCR					10
-+#define CAM_CC_ICP_BCR						11
-+#define CAM_CC_MCLK0_BCR					12
-+#define CAM_CC_MCLK1_BCR					13
-+#define CAM_CC_MCLK2_BCR					14
-+#define CAM_CC_MCLK3_BCR					15
-+#define CAM_CC_MCLK4_BCR					16
-+#define CAM_CC_OPE_0_BCR					17
-+#define CAM_CC_TFE_0_BCR					18
-+#define CAM_CC_TFE_1_BCR					19
-+#define CAM_CC_TFE_2_BCR					20
-+
-+/* CAM_CC power domains */
-+#define CAM_CC_CAMSS_TOP_GDSC					0
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,milos-dispcc.h b/include/dt-bindings/clock/qcom,milos-dispcc.h
-new file mode 100644
-index 000000000000..c70f23f32f0a
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,milos-dispcc.h
-@@ -0,0 +1,61 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2025, Luca Weiss <luca.weiss@fairphone.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_MILOS_H
-+#define _DT_BINDINGS_CLK_QCOM_DISP_CC_MILOS_H
-+
-+/* DISP_CC clocks */
-+#define DISP_CC_PLL0						0
-+#define DISP_CC_MDSS_ACCU_CLK					1
-+#define DISP_CC_MDSS_AHB1_CLK					2
-+#define DISP_CC_MDSS_AHB_CLK					3
-+#define DISP_CC_MDSS_AHB_CLK_SRC				4
-+#define DISP_CC_MDSS_BYTE0_CLK					5
-+#define DISP_CC_MDSS_BYTE0_CLK_SRC				6
-+#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC				7
-+#define DISP_CC_MDSS_BYTE0_INTF_CLK				8
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK				9
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK_SRC				10
-+#define DISP_CC_MDSS_DPTX0_CRYPTO_CLK				11
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK				12
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK_SRC				13
-+#define DISP_CC_MDSS_DPTX0_LINK_DIV_CLK_SRC			14
-+#define DISP_CC_MDSS_DPTX0_LINK_INTF_CLK			15
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK				16
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC			17
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK				18
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC			19
-+#define DISP_CC_MDSS_DPTX0_USB_ROUTER_LINK_INTF_CLK		20
-+#define DISP_CC_MDSS_ESC0_CLK					21
-+#define DISP_CC_MDSS_ESC0_CLK_SRC				22
-+#define DISP_CC_MDSS_MDP1_CLK					23
-+#define DISP_CC_MDSS_MDP_CLK					24
-+#define DISP_CC_MDSS_MDP_CLK_SRC				25
-+#define DISP_CC_MDSS_MDP_LUT1_CLK				26
-+#define DISP_CC_MDSS_MDP_LUT_CLK				27
-+#define DISP_CC_MDSS_NON_GDSC_AHB_CLK				28
-+#define DISP_CC_MDSS_PCLK0_CLK					29
-+#define DISP_CC_MDSS_PCLK0_CLK_SRC				30
-+#define DISP_CC_MDSS_RSCC_AHB_CLK				31
-+#define DISP_CC_MDSS_RSCC_VSYNC_CLK				32
-+#define DISP_CC_MDSS_VSYNC1_CLK					33
-+#define DISP_CC_MDSS_VSYNC_CLK					34
-+#define DISP_CC_MDSS_VSYNC_CLK_SRC				35
-+#define DISP_CC_SLEEP_CLK					36
-+#define DISP_CC_SLEEP_CLK_SRC					37
-+#define DISP_CC_XO_CLK						38
-+#define DISP_CC_XO_CLK_SRC					39
-+
-+/* DISP_CC resets */
-+#define DISP_CC_MDSS_CORE_BCR					0
-+#define DISP_CC_MDSS_CORE_INT2_BCR				1
-+#define DISP_CC_MDSS_RSCC_BCR					2
-+
-+/* DISP_CC power domains */
-+#define DISP_CC_MDSS_CORE_GDSC					0
-+#define DISP_CC_MDSS_CORE_INT2_GDSC				1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,milos-gcc.h b/include/dt-bindings/clock/qcom,milos-gcc.h
-new file mode 100644
-index 000000000000..a530ca39e1ef
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,milos-gcc.h
-@@ -0,0 +1,210 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2025, Luca Weiss <luca.weiss@fairphone.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_MILOS_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_MILOS_H
-+
-+/* GCC clocks */
-+#define GCC_GPLL0						0
-+#define GCC_GPLL0_OUT_EVEN					1
-+#define GCC_GPLL2						2
-+#define GCC_GPLL4						3
-+#define GCC_GPLL6						4
-+#define GCC_GPLL7						5
-+#define GCC_GPLL9						6
-+#define GCC_AGGRE_NOC_PCIE_AXI_CLK				7
-+#define GCC_AGGRE_UFS_PHY_AXI_CLK				8
-+#define GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK			9
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK				10
-+#define GCC_BOOT_ROM_AHB_CLK					11
-+#define GCC_CAMERA_AHB_CLK					12
-+#define GCC_CAMERA_HF_AXI_CLK					13
-+#define GCC_CAMERA_HF_XO_CLK					14
-+#define GCC_CAMERA_SF_AXI_CLK					15
-+#define GCC_CAMERA_SF_XO_CLK					16
-+#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK				17
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				18
-+#define GCC_CNOC_PCIE_SF_AXI_CLK				19
-+#define GCC_DDRSS_GPU_AXI_CLK					20
-+#define GCC_DDRSS_PCIE_SF_QTB_CLK				21
-+#define GCC_DISP_AHB_CLK					22
-+#define GCC_DISP_GPLL0_DIV_CLK_SRC				23
-+#define GCC_DISP_HF_AXI_CLK					24
-+#define GCC_DISP_XO_CLK						25
-+#define GCC_GP1_CLK						26
-+#define GCC_GP1_CLK_SRC						27
-+#define GCC_GP2_CLK						28
-+#define GCC_GP2_CLK_SRC						29
-+#define GCC_GP3_CLK						30
-+#define GCC_GP3_CLK_SRC						31
-+#define GCC_GPU_CFG_AHB_CLK					32
-+#define GCC_GPU_GPLL0_CLK_SRC					33
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC				34
-+#define GCC_GPU_MEMNOC_GFX_CLK					35
-+#define GCC_GPU_SNOC_DVM_GFX_CLK				36
-+#define GCC_PCIE_0_AUX_CLK					37
-+#define GCC_PCIE_0_AUX_CLK_SRC					38
-+#define GCC_PCIE_0_CFG_AHB_CLK					39
-+#define GCC_PCIE_0_MSTR_AXI_CLK					40
-+#define GCC_PCIE_0_PHY_RCHNG_CLK				41
-+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC				42
-+#define GCC_PCIE_0_PIPE_CLK					43
-+#define GCC_PCIE_0_PIPE_CLK_SRC					44
-+#define GCC_PCIE_0_PIPE_DIV2_CLK				45
-+#define GCC_PCIE_0_PIPE_DIV2_CLK_SRC				46
-+#define GCC_PCIE_0_SLV_AXI_CLK					47
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				48
-+#define GCC_PCIE_1_AUX_CLK					49
-+#define GCC_PCIE_1_AUX_CLK_SRC					50
-+#define GCC_PCIE_1_CFG_AHB_CLK					51
-+#define GCC_PCIE_1_MSTR_AXI_CLK					52
-+#define GCC_PCIE_1_PHY_RCHNG_CLK				53
-+#define GCC_PCIE_1_PHY_RCHNG_CLK_SRC				54
-+#define GCC_PCIE_1_PIPE_CLK					55
-+#define GCC_PCIE_1_PIPE_CLK_SRC					56
-+#define GCC_PCIE_1_PIPE_DIV2_CLK				57
-+#define GCC_PCIE_1_PIPE_DIV2_CLK_SRC				58
-+#define GCC_PCIE_1_SLV_AXI_CLK					59
-+#define GCC_PCIE_1_SLV_Q2A_AXI_CLK				60
-+#define GCC_PCIE_RSCC_CFG_AHB_CLK				61
-+#define GCC_PCIE_RSCC_XO_CLK					62
-+#define GCC_PDM2_CLK						63
-+#define GCC_PDM2_CLK_SRC					64
-+#define GCC_PDM_AHB_CLK						65
-+#define GCC_PDM_XO4_CLK						66
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				67
-+#define GCC_QMIP_CAMERA_RT_AHB_CLK				68
-+#define GCC_QMIP_DISP_AHB_CLK					69
-+#define GCC_QMIP_GPU_AHB_CLK					70
-+#define GCC_QMIP_PCIE_AHB_CLK					71
-+#define GCC_QMIP_VIDEO_CV_CPU_AHB_CLK				72
-+#define GCC_QMIP_VIDEO_CVP_AHB_CLK				73
-+#define GCC_QMIP_VIDEO_V_CPU_AHB_CLK				74
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				75
-+#define GCC_QUPV3_WRAP0_CORE_2X_CLK				76
-+#define GCC_QUPV3_WRAP0_CORE_CLK				77
-+#define GCC_QUPV3_WRAP0_QSPI_REF_CLK				78
-+#define GCC_QUPV3_WRAP0_QSPI_REF_CLK_SRC			79
-+#define GCC_QUPV3_WRAP0_S0_CLK					80
-+#define GCC_QUPV3_WRAP0_S0_CLK_SRC				81
-+#define GCC_QUPV3_WRAP0_S1_CLK					82
-+#define GCC_QUPV3_WRAP0_S1_CLK_SRC				83
-+#define GCC_QUPV3_WRAP0_S2_CLK					84
-+#define GCC_QUPV3_WRAP0_S2_CLK_SRC				85
-+#define GCC_QUPV3_WRAP0_S3_CLK					86
-+#define GCC_QUPV3_WRAP0_S3_CLK_SRC				87
-+#define GCC_QUPV3_WRAP0_S4_CLK					88
-+#define GCC_QUPV3_WRAP0_S4_CLK_SRC				89
-+#define GCC_QUPV3_WRAP0_S5_CLK					90
-+#define GCC_QUPV3_WRAP0_S5_CLK_SRC				91
-+#define GCC_QUPV3_WRAP0_S6_CLK					92
-+#define GCC_QUPV3_WRAP0_S6_CLK_SRC				93
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				94
-+#define GCC_QUPV3_WRAP1_CORE_CLK				95
-+#define GCC_QUPV3_WRAP1_QSPI_REF_CLK				96
-+#define GCC_QUPV3_WRAP1_QSPI_REF_CLK_SRC			97
-+#define GCC_QUPV3_WRAP1_S0_CLK					98
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				99
-+#define GCC_QUPV3_WRAP1_S1_CLK					100
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				101
-+#define GCC_QUPV3_WRAP1_S2_CLK					102
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				103
-+#define GCC_QUPV3_WRAP1_S3_CLK					104
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				105
-+#define GCC_QUPV3_WRAP1_S4_CLK					106
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				107
-+#define GCC_QUPV3_WRAP1_S5_CLK					108
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC				109
-+#define GCC_QUPV3_WRAP1_S6_CLK					110
-+#define GCC_QUPV3_WRAP1_S6_CLK_SRC				111
-+#define GCC_QUPV3_WRAP_0_M_AHB_CLK				112
-+#define GCC_QUPV3_WRAP_0_S_AHB_CLK				113
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK				114
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				115
-+#define GCC_SDCC1_AHB_CLK					116
-+#define GCC_SDCC1_APPS_CLK					117
-+#define GCC_SDCC1_APPS_CLK_SRC					118
-+#define GCC_SDCC1_ICE_CORE_CLK					119
-+#define GCC_SDCC1_ICE_CORE_CLK_SRC				120
-+#define GCC_SDCC2_AHB_CLK					121
-+#define GCC_SDCC2_APPS_CLK					122
-+#define GCC_SDCC2_APPS_CLK_SRC					123
-+#define GCC_UFS_PHY_AHB_CLK					124
-+#define GCC_UFS_PHY_AXI_CLK					125
-+#define GCC_UFS_PHY_AXI_CLK_SRC					126
-+#define GCC_UFS_PHY_AXI_HW_CTL_CLK				127
-+#define GCC_UFS_PHY_ICE_CORE_CLK				128
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC				129
-+#define GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK				130
-+#define GCC_UFS_PHY_PHY_AUX_CLK					131
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC				132
-+#define GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK				133
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK				134
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC				135
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK				136
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC				137
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK				138
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC				139
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK				140
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC				141
-+#define GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK			142
-+#define GCC_USB30_PRIM_ATB_CLK					143
-+#define GCC_USB30_PRIM_MASTER_CLK				144
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC				145
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				146
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			147
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		148
-+#define GCC_USB30_PRIM_SLEEP_CLK				149
-+#define GCC_USB3_PRIM_PHY_AUX_CLK				150
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				151
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				152
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK				153
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				154
-+#define GCC_VIDEO_AHB_CLK					155
-+#define GCC_VIDEO_AXI0_CLK					156
-+#define GCC_VIDEO_XO_CLK					157
-+
-+/* GCC resets */
-+#define GCC_CAMERA_BCR						0
-+#define GCC_DISPLAY_BCR						1
-+#define GCC_GPU_BCR						2
-+#define GCC_PCIE_0_BCR						3
-+#define GCC_PCIE_0_LINK_DOWN_BCR				4
-+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR				5
-+#define GCC_PCIE_0_PHY_BCR					6
-+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR			7
-+#define GCC_PCIE_1_BCR						8
-+#define GCC_PCIE_1_LINK_DOWN_BCR				9
-+#define GCC_PCIE_1_NOCSR_COM_PHY_BCR				10
-+#define GCC_PCIE_1_PHY_BCR					11
-+#define GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR			12
-+#define GCC_PCIE_RSCC_BCR					13
-+#define GCC_PDM_BCR						14
-+#define GCC_QUPV3_WRAPPER_0_BCR					15
-+#define GCC_QUPV3_WRAPPER_1_BCR					16
-+#define GCC_QUSB2PHY_PRIM_BCR					17
-+#define GCC_QUSB2PHY_SEC_BCR					18
-+#define GCC_SDCC1_BCR						19
-+#define GCC_SDCC2_BCR						20
-+#define GCC_UFS_PHY_BCR						21
-+#define GCC_USB30_PRIM_BCR					22
-+#define GCC_USB3_DP_PHY_PRIM_BCR				23
-+#define GCC_USB3_PHY_PRIM_BCR					24
-+#define GCC_USB3PHY_PHY_PRIM_BCR				25
-+#define GCC_VIDEO_AXI0_CLK_ARES					26
-+#define GCC_VIDEO_BCR						27
-+
-+/* GCC power domains */
-+#define PCIE_0_GDSC						0
-+#define PCIE_0_PHY_GDSC						1
-+#define PCIE_1_GDSC						2
-+#define PCIE_1_PHY_GDSC						3
-+#define UFS_PHY_GDSC						4
-+#define UFS_MEM_PHY_GDSC					5
-+#define USB30_PRIM_GDSC						6
-+#define USB3_PHY_GDSC						7
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,milos-gpucc.h b/include/dt-bindings/clock/qcom,milos-gpucc.h
-new file mode 100644
-index 000000000000..6ff1925d409f
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,milos-gpucc.h
-@@ -0,0 +1,56 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2025, Luca Weiss <luca.weiss@fairphone.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_MILOS_H
-+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_MILOS_H
-+
-+/* GPU_CC clocks */
-+#define GPU_CC_PLL0						0
-+#define GPU_CC_PLL0_OUT_EVEN					1
-+#define GPU_CC_AHB_CLK						2
-+#define GPU_CC_CB_CLK						3
-+#define GPU_CC_CX_ACCU_SHIFT_CLK				4
-+#define GPU_CC_CX_FF_CLK					5
-+#define GPU_CC_CX_GMU_CLK					6
-+#define GPU_CC_CXO_AON_CLK					7
-+#define GPU_CC_CXO_CLK						8
-+#define GPU_CC_DEMET_CLK					9
-+#define GPU_CC_DEMET_DIV_CLK_SRC				10
-+#define GPU_CC_DPM_CLK						11
-+#define GPU_CC_FF_CLK_SRC					12
-+#define GPU_CC_FREQ_MEASURE_CLK					13
-+#define GPU_CC_GMU_CLK_SRC					14
-+#define GPU_CC_GX_ACCU_SHIFT_CLK				15
-+#define GPU_CC_GX_ACD_AHB_FF_CLK				16
-+#define GPU_CC_GX_AHB_FF_CLK					17
-+#define GPU_CC_GX_GMU_CLK					18
-+#define GPU_CC_GX_RCG_AHB_FF_CLK				19
-+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK				20
-+#define GPU_CC_HUB_AON_CLK					21
-+#define GPU_CC_HUB_CLK_SRC					22
-+#define GPU_CC_HUB_CX_INT_CLK					23
-+#define GPU_CC_HUB_DIV_CLK_SRC					24
-+#define GPU_CC_MEMNOC_GFX_CLK					25
-+#define GPU_CC_RSCC_HUB_AON_CLK					26
-+#define GPU_CC_RSCC_XO_AON_CLK					27
-+#define GPU_CC_SLEEP_CLK					28
-+#define GPU_CC_XO_CLK_SRC					29
-+#define GPU_CC_XO_DIV_CLK_SRC					30
-+
-+/* GPU_CC resets */
-+#define GPU_CC_CB_BCR						0
-+#define GPU_CC_CX_BCR						1
-+#define GPU_CC_FAST_HUB_BCR					2
-+#define GPU_CC_FF_BCR						3
-+#define GPU_CC_GMU_BCR						4
-+#define GPU_CC_GX_BCR						5
-+#define GPU_CC_RBCPR_BCR					6
-+#define GPU_CC_XO_BCR						7
-+
-+/* GPU_CC power domains */
-+#define GPU_CC_CX_GDSC						0
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,milos-videocc.h b/include/dt-bindings/clock/qcom,milos-videocc.h
-new file mode 100644
-index 000000000000..3544db81ffae
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,milos-videocc.h
-@@ -0,0 +1,36 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2025, Luca Weiss <luca.weiss@fairphone.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_MILOS_H
-+#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_MILOS_H
-+
-+/* VIDEO_CC clocks */
-+#define VIDEO_CC_PLL0						0
-+#define VIDEO_CC_AHB_CLK					1
-+#define VIDEO_CC_AHB_CLK_SRC					2
-+#define VIDEO_CC_MVS0_CLK					3
-+#define VIDEO_CC_MVS0_CLK_SRC					4
-+#define VIDEO_CC_MVS0_DIV_CLK_SRC				5
-+#define VIDEO_CC_MVS0_SHIFT_CLK					6
-+#define VIDEO_CC_MVS0C_CLK					7
-+#define VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC				8
-+#define VIDEO_CC_MVS0C_SHIFT_CLK				9
-+#define VIDEO_CC_SLEEP_CLK					10
-+#define VIDEO_CC_SLEEP_CLK_SRC					11
-+#define VIDEO_CC_XO_CLK						12
-+#define VIDEO_CC_XO_CLK_SRC					13
-+
-+/* VIDEO_CC resets */
-+#define VIDEO_CC_INTERFACE_BCR					0
-+#define VIDEO_CC_MVS0_BCR					1
-+#define VIDEO_CC_MVS0C_CLK_ARES					2
-+#define VIDEO_CC_MVS0C_BCR					3
-+
-+/* VIDEO_CC power domains */
-+#define VIDEO_CC_MVS0_GDSC					0
-+#define VIDEO_CC_MVS0C_GDSC					1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,mmcc-msm8960.h b/include/dt-bindings/clock/qcom,mmcc-msm8960.h
-index 81714fc859c5..717431d735c1 100644
--- a/include/dt-bindings/clock/qcom,mmcc-msm8960.h
-+++ b/include/dt-bindings/clock/qcom,mmcc-msm8960.h
-@@ -133,5 +133,7 @@
- #define VCAP_CLK					124
- #define VCAP_NPL_CLK					125
- #define PLL15						126
-+#define DSI2_PIXEL_LVDS_SRC				127
-+#define LVDS_CLK					128
- 
- #endif
-diff --git a/include/dt-bindings/clock/qcom,mmcc-msm8974.h b/include/dt-bindings/clock/qcom,mmcc-msm8974.h
-index a62cb0629a7a..743ee60632eb 100644
--- a/include/dt-bindings/clock/qcom,mmcc-msm8974.h
-+++ b/include/dt-bindings/clock/qcom,mmcc-msm8974.h
-@@ -121,7 +121,6 @@
- #define MMSS_MMSSNOC_BTO_AHB_CLK			112
- #define MMSS_MMSSNOC_AXI_CLK				113
- #define MMSS_S0_AXI_CLK					114
-#define OCMEMCX_AHB_CLK					115
- #define OCMEMCX_OCMEMNOC_CLK				116
- #define OXILI_OCMEMGX_CLK				117
- #define OCMEMNOC_CLK					118
-diff --git a/include/dt-bindings/clock/qcom,mmcc-sdm660.h b/include/dt-bindings/clock/qcom,mmcc-sdm660.h
-index f9dbc21cb5c7..ee2a89dae72d 100644
--- a/include/dt-bindings/clock/qcom,mmcc-sdm660.h
-+++ b/include/dt-bindings/clock/qcom,mmcc-sdm660.h
-@@ -157,6 +157,7 @@
- #define BIMC_SMMU_GDSC							7
- 
- #define CAMSS_MICRO_BCR				 0
-+#define MDSS_BCR				1
- 
- #endif
- 
-diff --git a/include/dt-bindings/clock/qcom,qca8k-nsscc.h b/include/dt-bindings/clock/qcom,qca8k-nsscc.h
-new file mode 100644
-index 000000000000..0ac3e4c69a1a
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,qca8k-nsscc.h
-@@ -0,0 +1,101 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_QCA8K_NSS_CC_H
-+#define _DT_BINDINGS_CLK_QCOM_QCA8K_NSS_CC_H
-+
-+#define NSS_CC_SWITCH_CORE_CLK_SRC				0
-+#define NSS_CC_SWITCH_CORE_CLK					1
-+#define NSS_CC_APB_BRIDGE_CLK					2
-+#define NSS_CC_MAC0_TX_CLK_SRC					3
-+#define NSS_CC_MAC0_TX_DIV_CLK_SRC				4
-+#define NSS_CC_MAC0_TX_CLK					5
-+#define NSS_CC_MAC0_TX_SRDS1_CLK				6
-+#define NSS_CC_MAC0_RX_CLK_SRC					7
-+#define NSS_CC_MAC0_RX_DIV_CLK_SRC				8
-+#define NSS_CC_MAC0_RX_CLK					9
-+#define NSS_CC_MAC0_RX_SRDS1_CLK				10
-+#define NSS_CC_MAC1_TX_CLK_SRC					11
-+#define NSS_CC_MAC1_TX_DIV_CLK_SRC				12
-+#define NSS_CC_MAC1_SRDS1_CH0_XGMII_RX_DIV_CLK_SRC		13
-+#define NSS_CC_MAC1_SRDS1_CH0_RX_CLK				14
-+#define NSS_CC_MAC1_TX_CLK					15
-+#define NSS_CC_MAC1_GEPHY0_TX_CLK				16
-+#define NSS_CC_MAC1_SRDS1_CH0_XGMII_RX_CLK			17
-+#define NSS_CC_MAC1_RX_CLK_SRC					18
-+#define NSS_CC_MAC1_RX_DIV_CLK_SRC				19
-+#define NSS_CC_MAC1_SRDS1_CH0_XGMII_TX_DIV_CLK_SRC		20
-+#define NSS_CC_MAC1_SRDS1_CH0_TX_CLK				21
-+#define NSS_CC_MAC1_RX_CLK					22
-+#define NSS_CC_MAC1_GEPHY0_RX_CLK				23
-+#define NSS_CC_MAC1_SRDS1_CH0_XGMII_TX_CLK			24
-+#define NSS_CC_MAC2_TX_CLK_SRC					25
-+#define NSS_CC_MAC2_TX_DIV_CLK_SRC				26
-+#define NSS_CC_MAC2_SRDS1_CH1_XGMII_RX_DIV_CLK_SRC		27
-+#define NSS_CC_MAC2_SRDS1_CH1_RX_CLK				28
-+#define NSS_CC_MAC2_TX_CLK					29
-+#define NSS_CC_MAC2_GEPHY1_TX_CLK				30
-+#define NSS_CC_MAC2_SRDS1_CH1_XGMII_RX_CLK			31
-+#define NSS_CC_MAC2_RX_CLK_SRC					32
-+#define NSS_CC_MAC2_RX_DIV_CLK_SRC				33
-+#define NSS_CC_MAC2_SRDS1_CH1_XGMII_TX_DIV_CLK_SRC		34
-+#define NSS_CC_MAC2_SRDS1_CH1_TX_CLK				35
-+#define NSS_CC_MAC2_RX_CLK					36
-+#define NSS_CC_MAC2_GEPHY1_RX_CLK				37
-+#define NSS_CC_MAC2_SRDS1_CH1_XGMII_TX_CLK			38
-+#define NSS_CC_MAC3_TX_CLK_SRC					39
-+#define NSS_CC_MAC3_TX_DIV_CLK_SRC				40
-+#define NSS_CC_MAC3_SRDS1_CH2_XGMII_RX_DIV_CLK_SRC		41
-+#define NSS_CC_MAC3_SRDS1_CH2_RX_CLK				42
-+#define NSS_CC_MAC3_TX_CLK					43
-+#define NSS_CC_MAC3_GEPHY2_TX_CLK				44
-+#define NSS_CC_MAC3_SRDS1_CH2_XGMII_RX_CLK			45
-+#define NSS_CC_MAC3_RX_CLK_SRC					46
-+#define NSS_CC_MAC3_RX_DIV_CLK_SRC				47
-+#define NSS_CC_MAC3_SRDS1_CH2_XGMII_TX_DIV_CLK_SRC		48
-+#define NSS_CC_MAC3_SRDS1_CH2_TX_CLK				49
-+#define NSS_CC_MAC3_RX_CLK					50
-+#define NSS_CC_MAC3_GEPHY2_RX_CLK				51
-+#define NSS_CC_MAC3_SRDS1_CH2_XGMII_TX_CLK			52
-+#define NSS_CC_MAC4_TX_CLK_SRC					53
-+#define NSS_CC_MAC4_TX_DIV_CLK_SRC				54
-+#define NSS_CC_MAC4_SRDS1_CH3_XGMII_RX_DIV_CLK_SRC		55
-+#define NSS_CC_MAC4_SRDS1_CH3_RX_CLK				56
-+#define NSS_CC_MAC4_TX_CLK					57
-+#define NSS_CC_MAC4_GEPHY3_TX_CLK				58
-+#define NSS_CC_MAC4_SRDS1_CH3_XGMII_RX_CLK			59
-+#define NSS_CC_MAC4_RX_CLK_SRC					60
-+#define NSS_CC_MAC4_RX_DIV_CLK_SRC				61
-+#define NSS_CC_MAC4_SRDS1_CH3_XGMII_TX_DIV_CLK_SRC		62
-+#define NSS_CC_MAC4_SRDS1_CH3_TX_CLK				63
-+#define NSS_CC_MAC4_RX_CLK					64
-+#define NSS_CC_MAC4_GEPHY3_RX_CLK				65
-+#define NSS_CC_MAC4_SRDS1_CH3_XGMII_TX_CLK			66
-+#define NSS_CC_MAC5_TX_CLK_SRC					67
-+#define NSS_CC_MAC5_TX_DIV_CLK_SRC				68
-+#define NSS_CC_MAC5_TX_SRDS0_CLK				69
-+#define NSS_CC_MAC5_TX_CLK					70
-+#define NSS_CC_MAC5_RX_CLK_SRC					71
-+#define NSS_CC_MAC5_RX_DIV_CLK_SRC				72
-+#define NSS_CC_MAC5_RX_SRDS0_CLK				73
-+#define NSS_CC_MAC5_RX_CLK					74
-+#define NSS_CC_MAC5_TX_SRDS0_CLK_SRC				75
-+#define NSS_CC_MAC5_RX_SRDS0_CLK_SRC				76
-+#define NSS_CC_AHB_CLK_SRC					77
-+#define NSS_CC_AHB_CLK						78
-+#define NSS_CC_SEC_CTRL_AHB_CLK					79
-+#define NSS_CC_TLMM_CLK						80
-+#define NSS_CC_TLMM_AHB_CLK					81
-+#define NSS_CC_CNOC_AHB_CLK					82
-+#define NSS_CC_MDIO_AHB_CLK					83
-+#define NSS_CC_MDIO_MASTER_AHB_CLK				84
-+#define NSS_CC_SYS_CLK_SRC					85
-+#define NSS_CC_SRDS0_SYS_CLK					86
-+#define NSS_CC_SRDS1_SYS_CLK					87
-+#define NSS_CC_GEPHY0_SYS_CLK					88
-+#define NSS_CC_GEPHY1_SYS_CLK					89
-+#define NSS_CC_GEPHY2_SYS_CLK					90
-+#define NSS_CC_GEPHY3_SYS_CLK					91
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,qcm2290-gpucc.h b/include/dt-bindings/clock/qcom,qcm2290-gpucc.h
-new file mode 100644
-index 000000000000..7c76dd05278f
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,qcm2290-gpucc.h
-@@ -0,0 +1,32 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2024, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_QCM2290_H
-+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_QCM2290_H
-+
-+/* GPU_CC clocks */
-+#define GPU_CC_AHB_CLK			0
-+#define GPU_CC_CRC_AHB_CLK		1
-+#define GPU_CC_CX_GFX3D_CLK		2
-+#define GPU_CC_CX_GMU_CLK		3
-+#define GPU_CC_CX_SNOC_DVM_CLK		4
-+#define GPU_CC_CXO_AON_CLK		5
-+#define GPU_CC_CXO_CLK			6
-+#define GPU_CC_GMU_CLK_SRC		7
-+#define GPU_CC_GX_GFX3D_CLK		8
-+#define GPU_CC_GX_GFX3D_CLK_SRC		9
-+#define GPU_CC_PLL0			10
-+#define GPU_CC_SLEEP_CLK		11
-+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK	12
-+
-+/* Resets */
-+#define GPU_GX_BCR			0
-+
-+/* GDSCs */
-+#define GPU_CX_GDSC			0
-+#define GPU_GX_GDSC			1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,qcs615-camcc.h b/include/dt-bindings/clock/qcom,qcs615-camcc.h
-new file mode 100644
-index 000000000000..aec57dddc067
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,qcs615-camcc.h
-@@ -0,0 +1,110 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_QCS615_H
-+#define _DT_BINDINGS_CLK_QCOM_CAM_CC_QCS615_H
-+
-+/* CAM_CC clocks */
-+#define CAM_CC_BPS_AHB_CLK					0
-+#define CAM_CC_BPS_AREG_CLK					1
-+#define CAM_CC_BPS_AXI_CLK					2
-+#define CAM_CC_BPS_CLK						3
-+#define CAM_CC_BPS_CLK_SRC					4
-+#define CAM_CC_CAMNOC_ATB_CLK					5
-+#define CAM_CC_CAMNOC_AXI_CLK					6
-+#define CAM_CC_CCI_CLK						7
-+#define CAM_CC_CCI_CLK_SRC					8
-+#define CAM_CC_CORE_AHB_CLK					9
-+#define CAM_CC_CPAS_AHB_CLK					10
-+#define CAM_CC_CPHY_RX_CLK_SRC					11
-+#define CAM_CC_CSI0PHYTIMER_CLK					12
-+#define CAM_CC_CSI0PHYTIMER_CLK_SRC				13
-+#define CAM_CC_CSI1PHYTIMER_CLK					14
-+#define CAM_CC_CSI1PHYTIMER_CLK_SRC				15
-+#define CAM_CC_CSI2PHYTIMER_CLK					16
-+#define CAM_CC_CSI2PHYTIMER_CLK_SRC				17
-+#define CAM_CC_CSIPHY0_CLK					18
-+#define CAM_CC_CSIPHY1_CLK					19
-+#define CAM_CC_CSIPHY2_CLK					20
-+#define CAM_CC_FAST_AHB_CLK_SRC					21
-+#define CAM_CC_ICP_ATB_CLK					22
-+#define CAM_CC_ICP_CLK						23
-+#define CAM_CC_ICP_CLK_SRC					24
-+#define CAM_CC_ICP_CTI_CLK					25
-+#define CAM_CC_ICP_TS_CLK					26
-+#define CAM_CC_IFE_0_AXI_CLK					27
-+#define CAM_CC_IFE_0_CLK					28
-+#define CAM_CC_IFE_0_CLK_SRC					29
-+#define CAM_CC_IFE_0_CPHY_RX_CLK				30
-+#define CAM_CC_IFE_0_CSID_CLK					31
-+#define CAM_CC_IFE_0_CSID_CLK_SRC				32
-+#define CAM_CC_IFE_0_DSP_CLK					33
-+#define CAM_CC_IFE_1_AXI_CLK					34
-+#define CAM_CC_IFE_1_CLK					35
-+#define CAM_CC_IFE_1_CLK_SRC					36
-+#define CAM_CC_IFE_1_CPHY_RX_CLK				37
-+#define CAM_CC_IFE_1_CSID_CLK					38
-+#define CAM_CC_IFE_1_CSID_CLK_SRC				39
-+#define CAM_CC_IFE_1_DSP_CLK					40
-+#define CAM_CC_IFE_LITE_CLK					41
-+#define CAM_CC_IFE_LITE_CLK_SRC					42
-+#define CAM_CC_IFE_LITE_CPHY_RX_CLK				43
-+#define CAM_CC_IFE_LITE_CSID_CLK				44
-+#define CAM_CC_IFE_LITE_CSID_CLK_SRC				45
-+#define CAM_CC_IPE_0_AHB_CLK					46
-+#define CAM_CC_IPE_0_AREG_CLK					47
-+#define CAM_CC_IPE_0_AXI_CLK					48
-+#define CAM_CC_IPE_0_CLK					49
-+#define CAM_CC_IPE_0_CLK_SRC					50
-+#define CAM_CC_JPEG_CLK						51
-+#define CAM_CC_JPEG_CLK_SRC					52
-+#define CAM_CC_LRME_CLK						53
-+#define CAM_CC_LRME_CLK_SRC					54
-+#define CAM_CC_MCLK0_CLK					55
-+#define CAM_CC_MCLK0_CLK_SRC					56
-+#define CAM_CC_MCLK1_CLK					57
-+#define CAM_CC_MCLK1_CLK_SRC					58
-+#define CAM_CC_MCLK2_CLK					59
-+#define CAM_CC_MCLK2_CLK_SRC					60
-+#define CAM_CC_MCLK3_CLK					61
-+#define CAM_CC_MCLK3_CLK_SRC					62
-+#define CAM_CC_PLL0						63
-+#define CAM_CC_PLL1						64
-+#define CAM_CC_PLL2						65
-+#define CAM_CC_PLL2_OUT_AUX2					66
-+#define CAM_CC_PLL3						67
-+#define CAM_CC_SLOW_AHB_CLK_SRC					68
-+#define CAM_CC_SOC_AHB_CLK					69
-+#define CAM_CC_SYS_TMR_CLK					70
-+
-+/* CAM_CC power domains */
-+#define BPS_GDSC						0
-+#define IFE_0_GDSC						1
-+#define IFE_1_GDSC						2
-+#define IPE_0_GDSC						3
-+#define TITAN_TOP_GDSC						4
-+
-+/* CAM_CC resets */
-+#define CAM_CC_BPS_BCR						0
-+#define CAM_CC_CAMNOC_BCR					1
-+#define CAM_CC_CCI_BCR						2
-+#define CAM_CC_CPAS_BCR						3
-+#define CAM_CC_CSI0PHY_BCR					4
-+#define CAM_CC_CSI1PHY_BCR					5
-+#define CAM_CC_CSI2PHY_BCR					6
-+#define CAM_CC_ICP_BCR						7
-+#define CAM_CC_IFE_0_BCR					8
-+#define CAM_CC_IFE_1_BCR					9
-+#define CAM_CC_IFE_LITE_BCR					10
-+#define CAM_CC_IPE_0_BCR					11
-+#define CAM_CC_JPEG_BCR						12
-+#define CAM_CC_LRME_BCR						13
-+#define CAM_CC_MCLK0_BCR					14
-+#define CAM_CC_MCLK1_BCR					15
-+#define CAM_CC_MCLK2_BCR					16
-+#define CAM_CC_MCLK3_BCR					17
-+#define CAM_CC_TITAN_TOP_BCR					18
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,qcs615-dispcc.h b/include/dt-bindings/clock/qcom,qcs615-dispcc.h
-new file mode 100644
-index 000000000000..9a29945c5762
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,qcs615-dispcc.h
-@@ -0,0 +1,52 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_QCS615_H
-+#define _DT_BINDINGS_CLK_QCOM_DISP_CC_QCS615_H
-+
-+/* DISP_CC clocks */
-+#define DISP_CC_MDSS_AHB_CLK					0
-+#define DISP_CC_MDSS_AHB_CLK_SRC				1
-+#define DISP_CC_MDSS_BYTE0_CLK					2
-+#define DISP_CC_MDSS_BYTE0_CLK_SRC				3
-+#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC				4
-+#define DISP_CC_MDSS_BYTE0_INTF_CLK				5
-+#define DISP_CC_MDSS_DP_AUX_CLK					6
-+#define DISP_CC_MDSS_DP_AUX_CLK_SRC				7
-+#define DISP_CC_MDSS_DP_CRYPTO_CLK				8
-+#define DISP_CC_MDSS_DP_CRYPTO_CLK_SRC				9
-+#define DISP_CC_MDSS_DP_LINK_CLK				10
-+#define DISP_CC_MDSS_DP_LINK_CLK_SRC				11
-+#define DISP_CC_MDSS_DP_LINK_DIV_CLK_SRC			12
-+#define DISP_CC_MDSS_DP_LINK_INTF_CLK				13
-+#define DISP_CC_MDSS_DP_PIXEL1_CLK				14
-+#define DISP_CC_MDSS_DP_PIXEL1_CLK_SRC				15
-+#define DISP_CC_MDSS_DP_PIXEL_CLK				16
-+#define DISP_CC_MDSS_DP_PIXEL_CLK_SRC				17
-+#define DISP_CC_MDSS_ESC0_CLK					18
-+#define DISP_CC_MDSS_ESC0_CLK_SRC				19
-+#define DISP_CC_MDSS_MDP_CLK					20
-+#define DISP_CC_MDSS_MDP_CLK_SRC				21
-+#define DISP_CC_MDSS_MDP_LUT_CLK				22
-+#define DISP_CC_MDSS_NON_GDSC_AHB_CLK				23
-+#define DISP_CC_MDSS_PCLK0_CLK					24
-+#define DISP_CC_MDSS_PCLK0_CLK_SRC				25
-+#define DISP_CC_MDSS_ROT_CLK					26
-+#define DISP_CC_MDSS_ROT_CLK_SRC				27
-+#define DISP_CC_MDSS_RSCC_AHB_CLK				28
-+#define DISP_CC_MDSS_RSCC_VSYNC_CLK				29
-+#define DISP_CC_MDSS_VSYNC_CLK					30
-+#define DISP_CC_MDSS_VSYNC_CLK_SRC				31
-+#define DISP_CC_PLL0						32
-+#define DISP_CC_XO_CLK						33
-+
-+/* DISP_CC power domains */
-+#define MDSS_CORE_GDSC						0
-+
-+/* DISP_CC resets */
-+#define DISP_CC_MDSS_CORE_BCR					0
-+#define DISP_CC_MDSS_RSCC_BCR					1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,qcs615-gcc.h b/include/dt-bindings/clock/qcom,qcs615-gcc.h
-new file mode 100644
-index 000000000000..9704091636b8
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,qcs615-gcc.h
-@@ -0,0 +1,211 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_QCS615_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_QCS615_H
-+
-+/* GCC clocks */
-+#define GPLL0_OUT_AUX2_DIV					0
-+#define GPLL3_OUT_AUX2_DIV					1
-+#define GPLL0							2
-+#define GPLL3							3
-+#define GPLL4							4
-+#define GPLL6							5
-+#define GPLL6_OUT_MAIN						6
-+#define GPLL7							7
-+#define GPLL8							8
-+#define GPLL8_OUT_MAIN						9
-+#define GCC_AGGRE_UFS_PHY_AXI_CLK				10
-+#define GCC_AGGRE_USB2_SEC_AXI_CLK				11
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK				12
-+#define GCC_AHB2PHY_EAST_CLK					13
-+#define GCC_AHB2PHY_WEST_CLK					14
-+#define GCC_BOOT_ROM_AHB_CLK					15
-+#define GCC_CAMERA_AHB_CLK					16
-+#define GCC_CAMERA_HF_AXI_CLK					17
-+#define GCC_CAMERA_XO_CLK					18
-+#define GCC_CE1_AHB_CLK						19
-+#define GCC_CE1_AXI_CLK						20
-+#define GCC_CE1_CLK						21
-+#define GCC_CFG_NOC_USB2_SEC_AXI_CLK				22
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				23
-+#define GCC_CPUSS_AHB_CLK					24
-+#define GCC_CPUSS_AHB_CLK_SRC					25
-+#define GCC_CPUSS_GNOC_CLK					26
-+#define GCC_DDRSS_GPU_AXI_CLK					27
-+#define GCC_DISP_AHB_CLK					28
-+#define GCC_DISP_GPLL0_DIV_CLK_SRC				29
-+#define GCC_DISP_HF_AXI_CLK					30
-+#define GCC_DISP_XO_CLK						31
-+#define GCC_EMAC_AXI_CLK					32
-+#define GCC_EMAC_PTP_CLK					33
-+#define GCC_EMAC_PTP_CLK_SRC					34
-+#define GCC_EMAC_RGMII_CLK					35
-+#define GCC_EMAC_RGMII_CLK_SRC					36
-+#define GCC_EMAC_SLV_AHB_CLK					37
-+#define GCC_GP1_CLK						38
-+#define GCC_GP1_CLK_SRC						39
-+#define GCC_GP2_CLK						40
-+#define GCC_GP2_CLK_SRC						41
-+#define GCC_GP3_CLK						42
-+#define GCC_GP3_CLK_SRC						43
-+#define GCC_GPU_CFG_AHB_CLK					44
-+#define GCC_GPU_GPLL0_CLK_SRC					45
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC				46
-+#define GCC_GPU_IREF_CLK					47
-+#define GCC_GPU_MEMNOC_GFX_CLK					48
-+#define GCC_GPU_SNOC_DVM_GFX_CLK				49
-+#define GCC_PCIE0_PHY_REFGEN_CLK				50
-+#define GCC_PCIE_0_AUX_CLK					51
-+#define GCC_PCIE_0_AUX_CLK_SRC					52
-+#define GCC_PCIE_0_CFG_AHB_CLK					53
-+#define GCC_PCIE_0_CLKREF_CLK					54
-+#define GCC_PCIE_0_MSTR_AXI_CLK					55
-+#define GCC_PCIE_0_PIPE_CLK					56
-+#define GCC_PCIE_0_SLV_AXI_CLK					57
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				58
-+#define GCC_PCIE_PHY_AUX_CLK					59
-+#define GCC_PCIE_PHY_REFGEN_CLK_SRC				60
-+#define GCC_PDM2_CLK						61
-+#define GCC_PDM2_CLK_SRC					62
-+#define GCC_PDM_AHB_CLK						63
-+#define GCC_PDM_XO4_CLK						64
-+#define GCC_PRNG_AHB_CLK					65
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				66
-+#define GCC_QMIP_DISP_AHB_CLK					67
-+#define GCC_QMIP_PCIE_AHB_CLK					68
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				69
-+#define GCC_QSPI_CNOC_PERIPH_AHB_CLK				70
-+#define GCC_QSPI_CORE_CLK					71
-+#define GCC_QSPI_CORE_CLK_SRC					72
-+#define GCC_QUPV3_WRAP0_CORE_2X_CLK				73
-+#define GCC_QUPV3_WRAP0_CORE_CLK				74
-+#define GCC_QUPV3_WRAP0_S0_CLK					75
-+#define GCC_QUPV3_WRAP0_S0_CLK_SRC				76
-+#define GCC_QUPV3_WRAP0_S1_CLK					77
-+#define GCC_QUPV3_WRAP0_S1_CLK_SRC				78
-+#define GCC_QUPV3_WRAP0_S2_CLK					79
-+#define GCC_QUPV3_WRAP0_S2_CLK_SRC				80
-+#define GCC_QUPV3_WRAP0_S3_CLK					81
-+#define GCC_QUPV3_WRAP0_S3_CLK_SRC				82
-+#define GCC_QUPV3_WRAP0_S4_CLK					83
-+#define GCC_QUPV3_WRAP0_S4_CLK_SRC				84
-+#define GCC_QUPV3_WRAP0_S5_CLK					85
-+#define GCC_QUPV3_WRAP0_S5_CLK_SRC				86
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				87
-+#define GCC_QUPV3_WRAP1_CORE_CLK				88
-+#define GCC_QUPV3_WRAP1_S0_CLK					89
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				90
-+#define GCC_QUPV3_WRAP1_S1_CLK					91
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				92
-+#define GCC_QUPV3_WRAP1_S2_CLK					93
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				94
-+#define GCC_QUPV3_WRAP1_S3_CLK					95
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				96
-+#define GCC_QUPV3_WRAP1_S4_CLK					97
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				98
-+#define GCC_QUPV3_WRAP1_S5_CLK					99
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC				100
-+#define GCC_QUPV3_WRAP_0_M_AHB_CLK				101
-+#define GCC_QUPV3_WRAP_0_S_AHB_CLK				102
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK				103
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				104
-+#define GCC_RX1_USB2_CLKREF_CLK					105
-+#define GCC_RX3_USB2_CLKREF_CLK					106
-+#define GCC_SDCC1_AHB_CLK					107
-+#define GCC_SDCC1_APPS_CLK					108
-+#define GCC_SDCC1_APPS_CLK_SRC					109
-+#define GCC_SDCC1_ICE_CORE_CLK					110
-+#define GCC_SDCC1_ICE_CORE_CLK_SRC				111
-+#define GCC_SDCC2_AHB_CLK					112
-+#define GCC_SDCC2_APPS_CLK					113
-+#define GCC_SDCC2_APPS_CLK_SRC					114
-+#define GCC_SDR_CORE_CLK					115
-+#define GCC_SDR_CSR_HCLK					116
-+#define GCC_SDR_PRI_MI2S_CLK					117
-+#define GCC_SDR_SEC_MI2S_CLK					118
-+#define GCC_SDR_WR0_MEM_CLK					119
-+#define GCC_SDR_WR1_MEM_CLK					120
-+#define GCC_SDR_WR2_MEM_CLK					121
-+#define GCC_SYS_NOC_CPUSS_AHB_CLK				122
-+#define GCC_UFS_CARD_CLKREF_CLK					123
-+#define GCC_UFS_MEM_CLKREF_CLK					124
-+#define GCC_UFS_PHY_AHB_CLK					125
-+#define GCC_UFS_PHY_AXI_CLK					126
-+#define GCC_UFS_PHY_AXI_CLK_SRC					127
-+#define GCC_UFS_PHY_ICE_CORE_CLK				128
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC				129
-+#define GCC_UFS_PHY_PHY_AUX_CLK					130
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC				131
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK				132
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK				133
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK				134
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC				135
-+#define GCC_USB20_SEC_MASTER_CLK				136
-+#define GCC_USB20_SEC_MASTER_CLK_SRC				137
-+#define GCC_USB20_SEC_MOCK_UTMI_CLK				138
-+#define GCC_USB20_SEC_MOCK_UTMI_CLK_SRC				139
-+#define GCC_USB20_SEC_SLEEP_CLK					140
-+#define GCC_USB2_PRIM_CLKREF_CLK				141
-+#define GCC_USB2_SEC_CLKREF_CLK					142
-+#define GCC_USB2_SEC_PHY_AUX_CLK				143
-+#define GCC_USB2_SEC_PHY_AUX_CLK_SRC				144
-+#define GCC_USB2_SEC_PHY_COM_AUX_CLK				145
-+#define GCC_USB2_SEC_PHY_PIPE_CLK				146
-+#define GCC_USB30_PRIM_MASTER_CLK				147
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC				148
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				149
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			150
-+#define GCC_USB30_PRIM_SLEEP_CLK				151
-+#define GCC_USB3_PRIM_CLKREF_CLK				152
-+#define GCC_USB3_PRIM_PHY_AUX_CLK				153
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				154
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				155
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK				156
-+#define GCC_USB3_SEC_CLKREF_CLK					157
-+#define GCC_VIDEO_AHB_CLK					158
-+#define GCC_VIDEO_AXI0_CLK					159
-+#define GCC_VIDEO_XO_CLK					160
-+#define GCC_VSENSOR_CLK_SRC					161
-+#define GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK			162
-+#define GCC_UFS_PHY_AXI_HW_CTL_CLK				163
-+#define GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK				164
-+#define GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK				165
-+#define GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK			166
-+
-+/* GCC Resets */
-+#define GCC_EMAC_BCR                                            0
-+#define GCC_QUSB2PHY_PRIM_BCR                                   1
-+#define GCC_QUSB2PHY_SEC_BCR                                    2
-+#define GCC_USB30_PRIM_BCR                                      3
-+#define GCC_USB2_PHY_SEC_BCR                                    4
-+#define GCC_USB3_DP_PHY_SEC_BCR                                 5
-+#define GCC_USB3PHY_PHY_SEC_BCR                                 6
-+#define GCC_PCIE_0_BCR                                          7
-+#define GCC_PCIE_0_PHY_BCR                                      8
-+#define GCC_PCIE_PHY_BCR                                        9
-+#define GCC_PCIE_PHY_COM_BCR                                    10
-+#define GCC_UFS_PHY_BCR                                         11
-+#define GCC_USB20_SEC_BCR                                       12
-+#define GCC_USB3_PHY_PRIM_SP0_BCR                               13
-+#define GCC_USB3PHY_PHY_PRIM_SP0_BCR                            14
-+#define GCC_SDCC1_BCR                                           15
-+#define GCC_SDCC2_BCR                                           16
-+
-+/* GCC power domains */
-+#define EMAC_GDSC						0
-+#define PCIE_0_GDSC						1
-+#define UFS_PHY_GDSC						2
-+#define USB20_SEC_GDSC						3
-+#define USB30_PRIM_GDSC						4
-+#define HLOS1_VOTE_AGGRE_NOC_MMU_AUDIO_TBU_GDSC			5
-+#define HLOS1_VOTE_AGGRE_NOC_MMU_TBU1_GDSC			6
-+#define HLOS1_VOTE_AGGRE_NOC_MMU_TBU2_GDSC			7
-+#define HLOS1_VOTE_AGGRE_NOC_MMU_PCIE_TBU_GDSC			8
-+#define HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC			9
-+#define HLOS1_VOTE_MMNOC_MMU_TBU_SF_GDSC			10
-+#define HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC			11
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,qcs615-gpucc.h b/include/dt-bindings/clock/qcom,qcs615-gpucc.h
-new file mode 100644
-index 000000000000..6d8394b90d59
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,qcs615-gpucc.h
-@@ -0,0 +1,39 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_QCS615_H
-+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_QCS615_H
-+
-+/* GPU_CC clocks */
-+#define CRC_DIV_PLL0						0
-+#define CRC_DIV_PLL1						1
-+#define GPU_CC_PLL0						2
-+#define GPU_CC_PLL1						3
-+#define GPU_CC_CRC_AHB_CLK					4
-+#define GPU_CC_CX_GFX3D_CLK					5
-+#define GPU_CC_CX_GFX3D_SLV_CLK					6
-+#define GPU_CC_CX_GMU_CLK					7
-+#define GPU_CC_CX_SNOC_DVM_CLK					8
-+#define GPU_CC_CXO_AON_CLK					9
-+#define GPU_CC_CXO_CLK						10
-+#define GPU_CC_GMU_CLK_SRC					11
-+#define GPU_CC_GX_GFX3D_CLK					12
-+#define GPU_CC_GX_GFX3D_CLK_SRC					13
-+#define GPU_CC_GX_GMU_CLK					14
-+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK				15
-+#define GPU_CC_SLEEP_CLK					16
-+
-+/* GPU_CC power domains */
-+#define CX_GDSC							0
-+#define GX_GDSC							1
-+
-+/* GPU_CC resets */
-+#define GPU_CC_CX_BCR						0
-+#define GPU_CC_GFX3D_AON_BCR					1
-+#define GPU_CC_GMU_BCR						2
-+#define GPU_CC_GX_BCR						3
-+#define GPU_CC_XO_BCR						4
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,qcs615-videocc.h b/include/dt-bindings/clock/qcom,qcs615-videocc.h
-new file mode 100644
-index 000000000000..0ca3efb21103
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,qcs615-videocc.h
-@@ -0,0 +1,30 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_QCS615_H
-+#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_QCS615_H
-+
-+/* VIDEO_CC clocks */
-+#define VIDEO_CC_SLEEP_CLK					0
-+#define VIDEO_CC_SLEEP_CLK_SRC					1
-+#define VIDEO_CC_VCODEC0_AXI_CLK				2
-+#define VIDEO_CC_VCODEC0_CORE_CLK				3
-+#define VIDEO_CC_VENUS_AHB_CLK					4
-+#define VIDEO_CC_VENUS_CLK_SRC					5
-+#define VIDEO_CC_VENUS_CTL_AXI_CLK				6
-+#define VIDEO_CC_VENUS_CTL_CORE_CLK				7
-+#define VIDEO_CC_XO_CLK						8
-+#define VIDEO_PLL0						9
-+
-+/* VIDEO_CC power domains */
-+#define VCODEC0_GDSC						0
-+#define VENUS_GDSC						1
-+
-+/* VIDEO_CC resets */
-+#define VIDEO_CC_INTERFACE_BCR					0
-+#define VIDEO_CC_VCODEC0_BCR					1
-+#define VIDEO_CC_VENUS_BCR					2
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,qcs8300-camcc.h b/include/dt-bindings/clock/qcom,qcs8300-camcc.h
-new file mode 100644
-index 000000000000..fc535c847859
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,qcs8300-camcc.h
-@@ -0,0 +1,16 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_QCS8300_CAM_CC_H
-+#define _DT_BINDINGS_CLK_QCOM_QCS8300_CAM_CC_H
-+
-+#include "qcom,sa8775p-camcc.h"
-+
-+/* QCS8300 introduces below new clocks compared to SA8775P */
-+
-+/* CAM_CC clocks */
-+#define CAM_CC_TITAN_TOP_ACCU_SHIFT_CLK				86
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,qcs8300-gcc.h b/include/dt-bindings/clock/qcom,qcs8300-gcc.h
-new file mode 100644
-index 000000000000..a0083b1d2126
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,qcs8300-gcc.h
-@@ -0,0 +1,234 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_QCS8300_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_QCS8300_H
-+
-+/* GCC clocks */
-+#define GCC_GPLL0						0
-+#define GCC_GPLL0_OUT_EVEN					1
-+#define GCC_GPLL1						2
-+#define GCC_GPLL4						3
-+#define GCC_GPLL7						4
-+#define GCC_GPLL9						5
-+#define GCC_AGGRE_NOC_QUPV3_AXI_CLK				6
-+#define GCC_AGGRE_UFS_PHY_AXI_CLK				7
-+#define GCC_AGGRE_USB2_PRIM_AXI_CLK				8
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK				9
-+#define GCC_AHB2PHY0_CLK					10
-+#define GCC_AHB2PHY2_CLK					11
-+#define GCC_AHB2PHY3_CLK					12
-+#define GCC_BOOT_ROM_AHB_CLK					13
-+#define GCC_CAMERA_AHB_CLK					14
-+#define GCC_CAMERA_HF_AXI_CLK					15
-+#define GCC_CAMERA_SF_AXI_CLK					16
-+#define GCC_CAMERA_THROTTLE_XO_CLK				17
-+#define GCC_CAMERA_XO_CLK					18
-+#define GCC_CFG_NOC_USB2_PRIM_AXI_CLK				19
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				20
-+#define GCC_DDRSS_GPU_AXI_CLK					21
-+#define GCC_DISP_AHB_CLK					22
-+#define GCC_DISP_HF_AXI_CLK					23
-+#define GCC_DISP_XO_CLK						24
-+#define GCC_EDP_REF_CLKREF_EN					25
-+#define GCC_EMAC0_AXI_CLK					26
-+#define GCC_EMAC0_PHY_AUX_CLK					27
-+#define GCC_EMAC0_PHY_AUX_CLK_SRC				28
-+#define GCC_EMAC0_PTP_CLK					29
-+#define GCC_EMAC0_PTP_CLK_SRC					30
-+#define GCC_EMAC0_RGMII_CLK					31
-+#define GCC_EMAC0_RGMII_CLK_SRC					32
-+#define GCC_EMAC0_SLV_AHB_CLK					33
-+#define GCC_GP1_CLK						34
-+#define GCC_GP1_CLK_SRC						35
-+#define GCC_GP2_CLK						36
-+#define GCC_GP2_CLK_SRC						37
-+#define GCC_GP3_CLK						38
-+#define GCC_GP3_CLK_SRC						39
-+#define GCC_GP4_CLK						40
-+#define GCC_GP4_CLK_SRC						41
-+#define GCC_GP5_CLK						42
-+#define GCC_GP5_CLK_SRC						43
-+#define GCC_GPU_CFG_AHB_CLK					44
-+#define GCC_GPU_GPLL0_CLK_SRC					45
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC				46
-+#define GCC_GPU_MEMNOC_GFX_CENTER_PIPELINE_CLK			47
-+#define GCC_GPU_MEMNOC_GFX_CLK					48
-+#define GCC_GPU_SNOC_DVM_GFX_CLK				49
-+#define GCC_GPU_TCU_THROTTLE_AHB_CLK				50
-+#define GCC_GPU_TCU_THROTTLE_CLK				51
-+#define GCC_PCIE_0_AUX_CLK					52
-+#define GCC_PCIE_0_AUX_CLK_SRC					53
-+#define GCC_PCIE_0_CFG_AHB_CLK					54
-+#define GCC_PCIE_0_MSTR_AXI_CLK					55
-+#define GCC_PCIE_0_PHY_AUX_CLK					56
-+#define GCC_PCIE_0_PHY_AUX_CLK_SRC				57
-+#define GCC_PCIE_0_PHY_RCHNG_CLK				58
-+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC				59
-+#define GCC_PCIE_0_PIPE_CLK					60
-+#define GCC_PCIE_0_PIPE_CLK_SRC					61
-+#define GCC_PCIE_0_PIPE_DIV_CLK_SRC				62
-+#define GCC_PCIE_0_PIPEDIV2_CLK					63
-+#define GCC_PCIE_0_SLV_AXI_CLK					64
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				65
-+#define GCC_PCIE_1_AUX_CLK					66
-+#define GCC_PCIE_1_AUX_CLK_SRC					67
-+#define GCC_PCIE_1_CFG_AHB_CLK					68
-+#define GCC_PCIE_1_MSTR_AXI_CLK					69
-+#define GCC_PCIE_1_PHY_AUX_CLK					70
-+#define GCC_PCIE_1_PHY_AUX_CLK_SRC				71
-+#define GCC_PCIE_1_PHY_RCHNG_CLK				72
-+#define GCC_PCIE_1_PHY_RCHNG_CLK_SRC				73
-+#define GCC_PCIE_1_PIPE_CLK					74
-+#define GCC_PCIE_1_PIPE_CLK_SRC					75
-+#define GCC_PCIE_1_PIPE_DIV_CLK_SRC				76
-+#define GCC_PCIE_1_PIPEDIV2_CLK					77
-+#define GCC_PCIE_1_SLV_AXI_CLK					78
-+#define GCC_PCIE_1_SLV_Q2A_AXI_CLK				79
-+#define GCC_PCIE_CLKREF_EN					80
-+#define GCC_PCIE_THROTTLE_CFG_CLK				81
-+#define GCC_PDM2_CLK						82
-+#define GCC_PDM2_CLK_SRC					83
-+#define GCC_PDM_AHB_CLK						84
-+#define GCC_PDM_XO4_CLK						85
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				86
-+#define GCC_QMIP_CAMERA_RT_AHB_CLK				87
-+#define GCC_QMIP_DISP_AHB_CLK					88
-+#define GCC_QMIP_DISP_ROT_AHB_CLK				89
-+#define GCC_QMIP_VIDEO_CVP_AHB_CLK				90
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				91
-+#define GCC_QMIP_VIDEO_VCPU_AHB_CLK				92
-+#define GCC_QUPV3_WRAP0_CORE_2X_CLK				93
-+#define GCC_QUPV3_WRAP0_CORE_CLK				94
-+#define GCC_QUPV3_WRAP0_S0_CLK					95
-+#define GCC_QUPV3_WRAP0_S0_CLK_SRC				96
-+#define GCC_QUPV3_WRAP0_S1_CLK					97
-+#define GCC_QUPV3_WRAP0_S1_CLK_SRC				98
-+#define GCC_QUPV3_WRAP0_S2_CLK					99
-+#define GCC_QUPV3_WRAP0_S2_CLK_SRC				100
-+#define GCC_QUPV3_WRAP0_S3_CLK					101
-+#define GCC_QUPV3_WRAP0_S3_CLK_SRC				102
-+#define GCC_QUPV3_WRAP0_S4_CLK					103
-+#define GCC_QUPV3_WRAP0_S4_CLK_SRC				104
-+#define GCC_QUPV3_WRAP0_S5_CLK					105
-+#define GCC_QUPV3_WRAP0_S5_CLK_SRC				106
-+#define GCC_QUPV3_WRAP0_S6_CLK					107
-+#define GCC_QUPV3_WRAP0_S6_CLK_SRC				108
-+#define GCC_QUPV3_WRAP0_S7_CLK					109
-+#define GCC_QUPV3_WRAP0_S7_CLK_SRC				110
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				111
-+#define GCC_QUPV3_WRAP1_CORE_CLK				112
-+#define GCC_QUPV3_WRAP1_S0_CLK					113
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				114
-+#define GCC_QUPV3_WRAP1_S1_CLK					115
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				116
-+#define GCC_QUPV3_WRAP1_S2_CLK					117
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				118
-+#define GCC_QUPV3_WRAP1_S3_CLK					119
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				120
-+#define GCC_QUPV3_WRAP1_S4_CLK					121
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				122
-+#define GCC_QUPV3_WRAP1_S5_CLK					123
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC				124
-+#define GCC_QUPV3_WRAP1_S6_CLK					125
-+#define GCC_QUPV3_WRAP1_S6_CLK_SRC				126
-+#define GCC_QUPV3_WRAP1_S7_CLK					127
-+#define GCC_QUPV3_WRAP1_S7_CLK_SRC				128
-+#define GCC_QUPV3_WRAP3_CORE_2X_CLK				129
-+#define GCC_QUPV3_WRAP3_CORE_CLK				130
-+#define GCC_QUPV3_WRAP3_QSPI_CLK				131
-+#define GCC_QUPV3_WRAP3_S0_CLK					132
-+#define GCC_QUPV3_WRAP3_S0_CLK_SRC				133
-+#define GCC_QUPV3_WRAP3_S0_DIV_CLK_SRC				134
-+#define GCC_QUPV3_WRAP_0_M_AHB_CLK				135
-+#define GCC_QUPV3_WRAP_0_S_AHB_CLK				136
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK				137
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				138
-+#define GCC_QUPV3_WRAP_3_M_AHB_CLK				139
-+#define GCC_QUPV3_WRAP_3_S_AHB_CLK				140
-+#define GCC_SDCC1_AHB_CLK					141
-+#define GCC_SDCC1_APPS_CLK					142
-+#define GCC_SDCC1_APPS_CLK_SRC					143
-+#define GCC_SDCC1_ICE_CORE_CLK					144
-+#define GCC_SDCC1_ICE_CORE_CLK_SRC				145
-+#define GCC_SGMI_CLKREF_EN					146
-+#define GCC_UFS_PHY_AHB_CLK					147
-+#define GCC_UFS_PHY_AXI_CLK					148
-+#define GCC_UFS_PHY_AXI_CLK_SRC					149
-+#define GCC_UFS_PHY_ICE_CORE_CLK				150
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC				151
-+#define GCC_UFS_PHY_PHY_AUX_CLK					152
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC				153
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK				154
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC				155
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK				156
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC				157
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK				158
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC				159
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK				160
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC				161
-+#define GCC_USB20_MASTER_CLK					162
-+#define GCC_USB20_MASTER_CLK_SRC				163
-+#define GCC_USB20_MOCK_UTMI_CLK					164
-+#define GCC_USB20_MOCK_UTMI_CLK_SRC				165
-+#define GCC_USB20_MOCK_UTMI_POSTDIV_CLK_SRC			166
-+#define GCC_USB20_SLEEP_CLK					167
-+#define GCC_USB30_PRIM_MASTER_CLK				168
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC				169
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				170
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			171
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		172
-+#define GCC_USB30_PRIM_SLEEP_CLK				173
-+#define GCC_USB3_PRIM_PHY_AUX_CLK				174
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				175
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				176
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK				177
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				178
-+#define GCC_USB_CLKREF_EN					179
-+#define GCC_VIDEO_AHB_CLK					180
-+#define GCC_VIDEO_AXI0_CLK					181
-+#define GCC_VIDEO_AXI1_CLK					182
-+#define GCC_VIDEO_XO_CLK					183
-+
-+/* GCC power domains */
-+#define GCC_EMAC0_GDSC						0
-+#define GCC_PCIE_0_GDSC						1
-+#define GCC_PCIE_1_GDSC						2
-+#define GCC_UFS_PHY_GDSC					3
-+#define GCC_USB20_PRIM_GDSC					4
-+#define GCC_USB30_PRIM_GDSC					5
-+
-+/* GCC resets */
-+#define GCC_EMAC0_BCR						0
-+#define GCC_PCIE_0_BCR						1
-+#define GCC_PCIE_0_LINK_DOWN_BCR				2
-+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR				3
-+#define GCC_PCIE_0_PHY_BCR					4
-+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR			5
-+#define GCC_PCIE_1_BCR						6
-+#define GCC_PCIE_1_LINK_DOWN_BCR				7
-+#define GCC_PCIE_1_NOCSR_COM_PHY_BCR				8
-+#define GCC_PCIE_1_PHY_BCR					9
-+#define GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR			10
-+#define GCC_SDCC1_BCR						11
-+#define GCC_UFS_PHY_BCR						12
-+#define GCC_USB20_PRIM_BCR					13
-+#define GCC_USB2_PHY_PRIM_BCR					14
-+#define GCC_USB2_PHY_SEC_BCR					15
-+#define GCC_USB30_PRIM_BCR					16
-+#define GCC_USB3_DP_PHY_PRIM_BCR				17
-+#define GCC_USB3_PHY_PRIM_BCR					18
-+#define GCC_USB3_PHY_TERT_BCR					19
-+#define GCC_USB3_UNIPHY_MP0_BCR					20
-+#define GCC_USB3_UNIPHY_MP1_BCR					21
-+#define GCC_USB3PHY_PHY_PRIM_BCR				22
-+#define GCC_USB3UNIPHY_PHY_MP0_BCR				23
-+#define GCC_USB3UNIPHY_PHY_MP1_BCR				24
-+#define GCC_USB_PHY_CFG_AHB2PHY_BCR				25
-+#define GCC_VIDEO_BCR						26
-+#define GCC_VIDEO_AXI0_CLK_ARES					27
-+#define GCC_VIDEO_AXI1_CLK_ARES					28
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,qcs8300-gpucc.h b/include/dt-bindings/clock/qcom,qcs8300-gpucc.h
-new file mode 100644
-index 000000000000..afa187467b4c
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,qcs8300-gpucc.h
-@@ -0,0 +1,17 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPUCC_QCS8300_H
-+#define _DT_BINDINGS_CLK_QCOM_GPUCC_QCS8300_H
-+
-+#include "qcom,sa8775p-gpucc.h"
-+
-+/* QCS8300 introduces below new clocks compared to SA8775P */
-+
-+/* GPU_CC clocks */
-+#define GPU_CC_CX_ACCU_SHIFT_CLK		23
-+#define GPU_CC_GX_ACCU_SHIFT_CLK		24
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,qdu1000-ecpricc.h b/include/dt-bindings/clock/qcom,qdu1000-ecpricc.h
-new file mode 100644
-index 000000000000..731e404a2ce6
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,qdu1000-ecpricc.h
-@@ -0,0 +1,147 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_ECPRI_CC_QDU1000_H
-+#define _DT_BINDINGS_CLK_QCOM_ECPRI_CC_QDU1000_H
-+
-+/* ECPRI_CC clocks */
-+#define ECPRI_CC_PLL0						0
-+#define ECPRI_CC_PLL1						1
-+#define ECPRI_CC_ECPRI_CG_CLK					2
-+#define ECPRI_CC_ECPRI_CLK_SRC					3
-+#define ECPRI_CC_ECPRI_DMA_CLK					4
-+#define ECPRI_CC_ECPRI_DMA_CLK_SRC				5
-+#define ECPRI_CC_ECPRI_DMA_NOC_CLK				6
-+#define ECPRI_CC_ECPRI_FAST_CLK					7
-+#define ECPRI_CC_ECPRI_FAST_CLK_SRC				8
-+#define ECPRI_CC_ECPRI_FAST_DIV2_CLK				9
-+#define ECPRI_CC_ECPRI_FAST_DIV2_CLK_SRC			10
-+#define ECPRI_CC_ECPRI_FAST_DIV2_NOC_CLK			11
-+#define ECPRI_CC_ECPRI_FR_CLK					12
-+#define ECPRI_CC_ECPRI_ORAN_CLK_SRC				13
-+#define ECPRI_CC_ECPRI_ORAN_DIV2_CLK				14
-+#define ECPRI_CC_ETH_100G_C2C0_HM_FF_CLK_SRC			15
-+#define ECPRI_CC_ETH_100G_C2C0_UDP_FIFO_CLK			16
-+#define ECPRI_CC_ETH_100G_C2C1_UDP_FIFO_CLK			17
-+#define ECPRI_CC_ETH_100G_C2C_0_HM_FF_0_CLK			18
-+#define ECPRI_CC_ETH_100G_C2C_0_HM_FF_1_CLK			19
-+#define ECPRI_CC_ETH_100G_C2C_HM_FF_0_DIV_CLK_SRC		20
-+#define ECPRI_CC_ETH_100G_C2C_HM_FF_1_DIV_CLK_SRC		21
-+#define ECPRI_CC_ETH_100G_C2C_HM_MACSEC_CLK			22
-+#define ECPRI_CC_ETH_100G_C2C_HM_MACSEC_CLK_SRC			23
-+#define ECPRI_CC_ETH_100G_DBG_C2C_HM_FF_0_CLK			24
-+#define ECPRI_CC_ETH_100G_DBG_C2C_HM_FF_0_DIV_CLK_SRC		25
-+#define ECPRI_CC_ETH_100G_DBG_C2C_HM_FF_1_CLK			26
-+#define ECPRI_CC_ETH_100G_DBG_C2C_HM_FF_1_DIV_CLK_SRC		27
-+#define ECPRI_CC_ETH_100G_DBG_C2C_HM_FF_CLK_SRC			28
-+#define ECPRI_CC_ETH_100G_DBG_C2C_UDP_FIFO_CLK			29
-+#define ECPRI_CC_ETH_100G_FH0_HM_FF_CLK_SRC			30
-+#define ECPRI_CC_ETH_100G_FH0_MACSEC_CLK_SRC			31
-+#define ECPRI_CC_ETH_100G_FH1_HM_FF_CLK_SRC			32
-+#define ECPRI_CC_ETH_100G_FH1_MACSEC_CLK_SRC			33
-+#define ECPRI_CC_ETH_100G_FH2_HM_FF_CLK_SRC			34
-+#define ECPRI_CC_ETH_100G_FH2_MACSEC_CLK_SRC			35
-+#define ECPRI_CC_ETH_100G_FH_0_HM_FF_0_CLK			36
-+#define ECPRI_CC_ETH_100G_FH_0_HM_FF_0_DIV_CLK_SRC		37
-+#define ECPRI_CC_ETH_100G_FH_0_HM_FF_1_CLK			38
-+#define ECPRI_CC_ETH_100G_FH_0_HM_FF_1_DIV_CLK_SRC		39
-+#define ECPRI_CC_ETH_100G_FH_0_HM_FF_2_CLK			40
-+#define ECPRI_CC_ETH_100G_FH_0_HM_FF_2_DIV_CLK_SRC		41
-+#define ECPRI_CC_ETH_100G_FH_0_HM_FF_3_CLK			42
-+#define ECPRI_CC_ETH_100G_FH_0_HM_FF_3_DIV_CLK_SRC		43
-+#define ECPRI_CC_ETH_100G_FH_0_UDP_FIFO_CLK			44
-+#define ECPRI_CC_ETH_100G_FH_1_HM_FF_0_CLK			45
-+#define ECPRI_CC_ETH_100G_FH_1_HM_FF_0_DIV_CLK_SRC		46
-+#define ECPRI_CC_ETH_100G_FH_1_HM_FF_1_CLK			47
-+#define ECPRI_CC_ETH_100G_FH_1_HM_FF_1_DIV_CLK_SRC		48
-+#define ECPRI_CC_ETH_100G_FH_1_HM_FF_2_CLK			49
-+#define ECPRI_CC_ETH_100G_FH_1_HM_FF_2_DIV_CLK_SRC		50
-+#define ECPRI_CC_ETH_100G_FH_1_HM_FF_3_CLK			51
-+#define ECPRI_CC_ETH_100G_FH_1_HM_FF_3_DIV_CLK_SRC		52
-+#define ECPRI_CC_ETH_100G_FH_1_UDP_FIFO_CLK			53
-+#define ECPRI_CC_ETH_100G_FH_2_HM_FF_0_CLK			54
-+#define ECPRI_CC_ETH_100G_FH_2_HM_FF_0_DIV_CLK_SRC		55
-+#define ECPRI_CC_ETH_100G_FH_2_HM_FF_1_CLK			56
-+#define ECPRI_CC_ETH_100G_FH_2_HM_FF_1_DIV_CLK_SRC		57
-+#define ECPRI_CC_ETH_100G_FH_2_HM_FF_2_CLK			58
-+#define ECPRI_CC_ETH_100G_FH_2_HM_FF_2_DIV_CLK_SRC		59
-+#define ECPRI_CC_ETH_100G_FH_2_HM_FF_3_CLK			60
-+#define ECPRI_CC_ETH_100G_FH_2_HM_FF_3_DIV_CLK_SRC		61
-+#define ECPRI_CC_ETH_100G_FH_2_UDP_FIFO_CLK			62
-+#define ECPRI_CC_ETH_100G_FH_MACSEC_0_CLK			63
-+#define ECPRI_CC_ETH_100G_FH_MACSEC_1_CLK			64
-+#define ECPRI_CC_ETH_100G_FH_MACSEC_2_CLK			65
-+#define ECPRI_CC_ETH_100G_MAC_C2C_HM_REF_CLK			66
-+#define ECPRI_CC_ETH_100G_MAC_C2C_HM_REF_CLK_SRC		67
-+#define ECPRI_CC_ETH_100G_MAC_DBG_C2C_HM_REF_CLK		68
-+#define ECPRI_CC_ETH_100G_MAC_DBG_C2C_HM_REF_CLK_SRC		69
-+#define ECPRI_CC_ETH_100G_MAC_FH0_HM_REF_CLK			70
-+#define ECPRI_CC_ETH_100G_MAC_FH0_HM_REF_CLK_SRC		71
-+#define ECPRI_CC_ETH_100G_MAC_FH1_HM_REF_CLK			72
-+#define ECPRI_CC_ETH_100G_MAC_FH1_HM_REF_CLK_SRC		73
-+#define ECPRI_CC_ETH_100G_MAC_FH2_HM_REF_CLK			74
-+#define ECPRI_CC_ETH_100G_MAC_FH2_HM_REF_CLK_SRC		75
-+#define ECPRI_CC_ETH_DBG_NFAPI_AXI_CLK				76
-+#define ECPRI_CC_ETH_DBG_NOC_AXI_CLK				77
-+#define ECPRI_CC_ETH_PHY_0_OCK_SRAM_CLK				78
-+#define ECPRI_CC_ETH_PHY_1_OCK_SRAM_CLK				79
-+#define ECPRI_CC_ETH_PHY_2_OCK_SRAM_CLK				80
-+#define ECPRI_CC_ETH_PHY_3_OCK_SRAM_CLK				81
-+#define ECPRI_CC_ETH_PHY_4_OCK_SRAM_CLK				82
-+#define ECPRI_CC_MSS_EMAC_CLK					83
-+#define ECPRI_CC_MSS_EMAC_CLK_SRC				84
-+#define ECPRI_CC_MSS_ORAN_CLK					85
-+#define ECPRI_CC_PHY0_LANE0_RX_CLK				86
-+#define ECPRI_CC_PHY0_LANE0_TX_CLK				87
-+#define ECPRI_CC_PHY0_LANE1_RX_CLK				88
-+#define ECPRI_CC_PHY0_LANE1_TX_CLK				89
-+#define ECPRI_CC_PHY0_LANE2_RX_CLK				90
-+#define ECPRI_CC_PHY0_LANE2_TX_CLK				91
-+#define ECPRI_CC_PHY0_LANE3_RX_CLK				92
-+#define ECPRI_CC_PHY0_LANE3_TX_CLK				93
-+#define ECPRI_CC_PHY1_LANE0_RX_CLK				94
-+#define ECPRI_CC_PHY1_LANE0_TX_CLK				95
-+#define ECPRI_CC_PHY1_LANE1_RX_CLK				96
-+#define ECPRI_CC_PHY1_LANE1_TX_CLK				97
-+#define ECPRI_CC_PHY1_LANE2_RX_CLK				98
-+#define ECPRI_CC_PHY1_LANE2_TX_CLK				99
-+#define ECPRI_CC_PHY1_LANE3_RX_CLK				100
-+#define ECPRI_CC_PHY1_LANE3_TX_CLK				101
-+#define ECPRI_CC_PHY2_LANE0_RX_CLK				102
-+#define ECPRI_CC_PHY2_LANE0_TX_CLK				103
-+#define ECPRI_CC_PHY2_LANE1_RX_CLK				104
-+#define ECPRI_CC_PHY2_LANE1_TX_CLK				105
-+#define ECPRI_CC_PHY2_LANE2_RX_CLK				106
-+#define ECPRI_CC_PHY2_LANE2_TX_CLK				107
-+#define ECPRI_CC_PHY2_LANE3_RX_CLK				108
-+#define ECPRI_CC_PHY2_LANE3_TX_CLK				109
-+#define ECPRI_CC_PHY3_LANE0_RX_CLK				110
-+#define ECPRI_CC_PHY3_LANE0_TX_CLK				111
-+#define ECPRI_CC_PHY3_LANE1_RX_CLK				112
-+#define ECPRI_CC_PHY3_LANE1_TX_CLK				113
-+#define ECPRI_CC_PHY3_LANE2_RX_CLK				114
-+#define ECPRI_CC_PHY3_LANE2_TX_CLK				115
-+#define ECPRI_CC_PHY3_LANE3_RX_CLK				116
-+#define ECPRI_CC_PHY3_LANE3_TX_CLK				117
-+#define ECPRI_CC_PHY4_LANE0_RX_CLK				118
-+#define ECPRI_CC_PHY4_LANE0_TX_CLK				119
-+#define ECPRI_CC_PHY4_LANE1_RX_CLK				120
-+#define ECPRI_CC_PHY4_LANE1_TX_CLK				121
-+#define ECPRI_CC_PHY4_LANE2_RX_CLK				122
-+#define ECPRI_CC_PHY4_LANE2_TX_CLK				123
-+#define ECPRI_CC_PHY4_LANE3_RX_CLK				124
-+#define ECPRI_CC_PHY4_LANE3_TX_CLK				125
-+
-+/* ECPRI_CC resets */
-+#define ECPRI_CC_CLK_CTL_TOP_ECPRI_CC_ECPRI_SS_BCR		0
-+#define ECPRI_CC_CLK_CTL_TOP_ECPRI_CC_ETH_C2C_BCR		1
-+#define ECPRI_CC_CLK_CTL_TOP_ECPRI_CC_ETH_FH0_BCR		2
-+#define ECPRI_CC_CLK_CTL_TOP_ECPRI_CC_ETH_FH1_BCR		3
-+#define ECPRI_CC_CLK_CTL_TOP_ECPRI_CC_ETH_FH2_BCR		4
-+#define ECPRI_CC_CLK_CTL_TOP_ECPRI_CC_ETH_WRAPPER_TOP_BCR	5
-+#define ECPRI_CC_CLK_CTL_TOP_ECPRI_CC_MODEM_BCR			6
-+#define ECPRI_CC_CLK_CTL_TOP_ECPRI_CC_NOC_BCR			7
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,qdu1000-gcc.h b/include/dt-bindings/clock/qcom,qdu1000-gcc.h
-new file mode 100644
-index 000000000000..2fd36cbfddbb
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,qdu1000-gcc.h
-@@ -0,0 +1,177 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (c) 2021-2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_QDU1000_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_QDU1000_H
-+
-+/* GCC clocks */
-+#define GCC_GPLL0					0
-+#define GCC_GPLL0_OUT_EVEN				1
-+#define GCC_GPLL1					2
-+#define GCC_GPLL2					3
-+#define GCC_GPLL2_OUT_EVEN				4
-+#define GCC_GPLL3					5
-+#define GCC_GPLL4					6
-+#define GCC_GPLL5					7
-+#define GCC_GPLL5_OUT_EVEN				8
-+#define GCC_GPLL6					9
-+#define GCC_GPLL7					10
-+#define GCC_GPLL8					11
-+#define GCC_AGGRE_NOC_ECPRI_DMA_CLK			12
-+#define GCC_AGGRE_NOC_ECPRI_DMA_CLK_SRC			13
-+#define GCC_AGGRE_NOC_ECPRI_GSI_CLK_SRC			14
-+#define GCC_BOOT_ROM_AHB_CLK				15
-+#define GCC_CFG_NOC_ECPRI_CC_AHB_CLK			16
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK			17
-+#define GCC_DDRSS_ECPRI_DMA_CLK				18
-+#define GCC_ECPRI_AHB_CLK				19
-+#define GCC_ECPRI_CC_GPLL0_CLK_SRC			20
-+#define GCC_ECPRI_CC_GPLL1_EVEN_CLK_SRC			21
-+#define GCC_ECPRI_CC_GPLL2_EVEN_CLK_SRC			22
-+#define GCC_ECPRI_CC_GPLL3_CLK_SRC			23
-+#define GCC_ECPRI_CC_GPLL4_CLK_SRC			24
-+#define GCC_ECPRI_CC_GPLL5_EVEN_CLK_SRC			25
-+#define GCC_ECPRI_XO_CLK				26
-+#define GCC_ETH_DBG_SNOC_AXI_CLK			27
-+#define GCC_GEMNOC_PCIE_QX_CLK				28
-+#define GCC_GP1_CLK					29
-+#define GCC_GP1_CLK_SRC					30
-+#define GCC_GP2_CLK					31
-+#define GCC_GP2_CLK_SRC					32
-+#define GCC_GP3_CLK					33
-+#define GCC_GP3_CLK_SRC					34
-+#define GCC_PCIE_0_AUX_CLK				35
-+#define GCC_PCIE_0_AUX_CLK_SRC				36
-+#define GCC_PCIE_0_CFG_AHB_CLK				37
-+#define GCC_PCIE_0_CLKREF_EN				38
-+#define GCC_PCIE_0_MSTR_AXI_CLK				39
-+#define GCC_PCIE_0_PHY_AUX_CLK				40
-+#define GCC_PCIE_0_PHY_RCHNG_CLK			41
-+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC			42
-+#define GCC_PCIE_0_PIPE_CLK				43
-+#define GCC_PCIE_0_SLV_AXI_CLK				44
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK			45
-+#define GCC_PDM2_CLK					46
-+#define GCC_PDM2_CLK_SRC				47
-+#define GCC_PDM_AHB_CLK					48
-+#define GCC_PDM_XO4_CLK					49
-+#define GCC_QMIP_ANOC_PCIE_CLK				50
-+#define GCC_QMIP_ECPRI_DMA0_CLK				51
-+#define GCC_QMIP_ECPRI_DMA1_CLK				52
-+#define GCC_QMIP_ECPRI_GSI_CLK				53
-+#define GCC_QUPV3_WRAP0_CORE_2X_CLK			54
-+#define GCC_QUPV3_WRAP0_CORE_CLK			55
-+#define GCC_QUPV3_WRAP0_S0_CLK				56
-+#define GCC_QUPV3_WRAP0_S0_CLK_SRC			57
-+#define GCC_QUPV3_WRAP0_S1_CLK				58
-+#define GCC_QUPV3_WRAP0_S1_CLK_SRC			59
-+#define GCC_QUPV3_WRAP0_S2_CLK				60
-+#define GCC_QUPV3_WRAP0_S2_CLK_SRC			61
-+#define GCC_QUPV3_WRAP0_S3_CLK				62
-+#define GCC_QUPV3_WRAP0_S3_CLK_SRC			63
-+#define GCC_QUPV3_WRAP0_S4_CLK				64
-+#define GCC_QUPV3_WRAP0_S4_CLK_SRC			65
-+#define GCC_QUPV3_WRAP0_S5_CLK				66
-+#define GCC_QUPV3_WRAP0_S5_CLK_SRC			67
-+#define GCC_QUPV3_WRAP0_S6_CLK				68
-+#define GCC_QUPV3_WRAP0_S6_CLK_SRC			69
-+#define GCC_QUPV3_WRAP0_S7_CLK				70
-+#define GCC_QUPV3_WRAP0_S7_CLK_SRC			71
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK			72
-+#define GCC_QUPV3_WRAP1_CORE_CLK			73
-+#define GCC_QUPV3_WRAP1_S0_CLK				74
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC			75
-+#define GCC_QUPV3_WRAP1_S1_CLK				76
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC			77
-+#define GCC_QUPV3_WRAP1_S2_CLK				78
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC			79
-+#define GCC_QUPV3_WRAP1_S3_CLK				80
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC			81
-+#define GCC_QUPV3_WRAP1_S4_CLK				82
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC			83
-+#define GCC_QUPV3_WRAP1_S5_CLK				84
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC			85
-+#define GCC_QUPV3_WRAP1_S6_CLK				86
-+#define GCC_QUPV3_WRAP1_S6_CLK_SRC			87
-+#define GCC_QUPV3_WRAP1_S7_CLK				88
-+#define GCC_QUPV3_WRAP1_S7_CLK_SRC			89
-+#define GCC_QUPV3_WRAP_0_M_AHB_CLK			90
-+#define GCC_QUPV3_WRAP_0_S_AHB_CLK			91
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK			92
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK			93
-+#define GCC_SDCC5_AHB_CLK				94
-+#define GCC_SDCC5_APPS_CLK				95
-+#define GCC_SDCC5_APPS_CLK_SRC				96
-+#define GCC_SDCC5_ICE_CORE_CLK				97
-+#define GCC_SDCC5_ICE_CORE_CLK_SRC			98
-+#define GCC_SNOC_CNOC_GEMNOC_PCIE_QX_CLK		99
-+#define GCC_SNOC_CNOC_GEMNOC_PCIE_SOUTH_QX_CLK		100
-+#define GCC_SNOC_CNOC_PCIE_QX_CLK			101
-+#define GCC_SNOC_PCIE_SF_CENTER_QX_CLK			102
-+#define GCC_SNOC_PCIE_SF_SOUTH_QX_CLK			103
-+#define GCC_TSC_CFG_AHB_CLK				104
-+#define GCC_TSC_CLK_SRC					105
-+#define GCC_TSC_CNTR_CLK				106
-+#define GCC_TSC_ETU_CLK					107
-+#define GCC_USB2_CLKREF_EN				108
-+#define GCC_USB30_PRIM_MASTER_CLK			109
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC			110
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK			111
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC		112
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC	113
-+#define GCC_USB30_PRIM_SLEEP_CLK			114
-+#define GCC_USB3_PRIM_PHY_AUX_CLK			115
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC			116
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK			117
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK			118
-+#define GCC_SM_BUS_AHB_CLK				119
-+#define GCC_SM_BUS_XO_CLK				120
-+#define GCC_SM_BUS_XO_CLK_SRC				121
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC			122
-+#define GCC_ETH_100G_C2C_HM_APB_CLK			123
-+#define GCC_ETH_100G_FH_HM_APB_0_CLK			124
-+#define GCC_ETH_100G_FH_HM_APB_1_CLK			125
-+#define GCC_ETH_100G_FH_HM_APB_2_CLK			126
-+#define GCC_ETH_DBG_C2C_HM_APB_CLK			127
-+#define GCC_AGGRE_NOC_ECPRI_GSI_CLK			128
-+#define GCC_PCIE_0_PIPE_CLK_SRC				129
-+#define GCC_PCIE_0_PHY_AUX_CLK_SRC			130
-+#define GCC_GPLL1_OUT_EVEN				131
-+#define GCC_DDRSS_ECPRI_GSI_CLK				132
-+
-+/* GCC resets */
-+#define GCC_ECPRI_CC_BCR				0
-+#define GCC_ECPRI_SS_BCR				1
-+#define GCC_ETH_WRAPPER_BCR				2
-+#define GCC_PCIE_0_BCR					3
-+#define GCC_PCIE_0_LINK_DOWN_BCR			4
-+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR			5
-+#define GCC_PCIE_0_PHY_BCR				6
-+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR		7
-+#define GCC_PCIE_PHY_CFG_AHB_BCR			8
-+#define GCC_PCIE_PHY_COM_BCR				9
-+#define GCC_PDM_BCR					10
-+#define GCC_QUPV3_WRAPPER_0_BCR				11
-+#define GCC_QUPV3_WRAPPER_1_BCR				12
-+#define GCC_QUSB2PHY_PRIM_BCR				13
-+#define GCC_QUSB2PHY_SEC_BCR				14
-+#define GCC_SDCC5_BCR					15
-+#define GCC_TCSR_PCIE_BCR				16
-+#define GCC_TSC_BCR					17
-+#define GCC_USB30_PRIM_BCR				18
-+#define GCC_USB3_DP_PHY_PRIM_BCR			19
-+#define GCC_USB3_DP_PHY_SEC_BCR				20
-+#define GCC_USB3_PHY_PRIM_BCR				21
-+#define GCC_USB3_PHY_SEC_BCR				22
-+#define GCC_USB3PHY_PHY_PRIM_BCR			23
-+#define GCC_USB3PHY_PHY_SEC_BCR				24
-+#define GCC_USB_PHY_CFG_AHB2PHY_BCR			25
-+
-+/* GCC power domains */
-+#define PCIE_0_GDSC					0
-+#define PCIE_0_PHY_GDSC					1
-+#define USB30_PRIM_GDSC					2
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,rpmcc.h b/include/dt-bindings/clock/qcom,rpmcc.h
-index 015db95303d1..1477a75e7f6d 100644
--- a/include/dt-bindings/clock/qcom,rpmcc.h
-+++ b/include/dt-bindings/clock/qcom,rpmcc.h
-@@ -167,5 +167,12 @@
- #define RPM_SMD_CPUSS_GNOC_A_CLK		121
- #define RPM_SMD_MSS_CFG_AHB_CLK		122
- #define RPM_SMD_MSS_CFG_AHB_A_CLK		123
-+#define RPM_SMD_BIMC_FREQ_LOG			124
-+#define RPM_SMD_LN_BB_CLK_PIN			125
-+#define RPM_SMD_LN_BB_A_CLK_PIN			126
-+#define RPM_SMD_BB_CLK3				127
-+#define RPM_SMD_BB_CLK3_A			128
-+#define RPM_SMD_BB_CLK3_PIN			129
-+#define RPM_SMD_BB_CLK3_A_PIN			130
- 
- #endif
-diff --git a/include/dt-bindings/clock/qcom,sa8775p-camcc.h b/include/dt-bindings/clock/qcom,sa8775p-camcc.h
-new file mode 100644
-index 000000000000..38531acd699f
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sa8775p-camcc.h
-@@ -0,0 +1,108 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_SA8775P_CAM_CC_H
-+#define _DT_BINDINGS_CLK_QCOM_SA8775P_CAM_CC_H
-+
-+/* CAM_CC clocks */
-+#define CAM_CC_CAMNOC_AXI_CLK					0
-+#define CAM_CC_CAMNOC_AXI_CLK_SRC				1
-+#define CAM_CC_CAMNOC_DCD_XO_CLK				2
-+#define CAM_CC_CAMNOC_XO_CLK					3
-+#define CAM_CC_CCI_0_CLK					4
-+#define CAM_CC_CCI_0_CLK_SRC					5
-+#define CAM_CC_CCI_1_CLK					6
-+#define CAM_CC_CCI_1_CLK_SRC					7
-+#define CAM_CC_CCI_2_CLK					8
-+#define CAM_CC_CCI_2_CLK_SRC					9
-+#define CAM_CC_CCI_3_CLK					10
-+#define CAM_CC_CCI_3_CLK_SRC					11
-+#define CAM_CC_CORE_AHB_CLK					12
-+#define CAM_CC_CPAS_AHB_CLK					13
-+#define CAM_CC_CPAS_FAST_AHB_CLK				14
-+#define CAM_CC_CPAS_IFE_0_CLK					15
-+#define CAM_CC_CPAS_IFE_1_CLK					16
-+#define CAM_CC_CPAS_IFE_LITE_CLK				17
-+#define CAM_CC_CPAS_IPE_CLK					18
-+#define CAM_CC_CPAS_SFE_LITE_0_CLK				19
-+#define CAM_CC_CPAS_SFE_LITE_1_CLK				20
-+#define CAM_CC_CPHY_RX_CLK_SRC					21
-+#define CAM_CC_CSI0PHYTIMER_CLK					22
-+#define CAM_CC_CSI0PHYTIMER_CLK_SRC				23
-+#define CAM_CC_CSI1PHYTIMER_CLK					24
-+#define CAM_CC_CSI1PHYTIMER_CLK_SRC				25
-+#define CAM_CC_CSI2PHYTIMER_CLK					26
-+#define CAM_CC_CSI2PHYTIMER_CLK_SRC				27
-+#define CAM_CC_CSI3PHYTIMER_CLK					28
-+#define CAM_CC_CSI3PHYTIMER_CLK_SRC				29
-+#define CAM_CC_CSID_CLK						30
-+#define CAM_CC_CSID_CLK_SRC					31
-+#define CAM_CC_CSID_CSIPHY_RX_CLK				32
-+#define CAM_CC_CSIPHY0_CLK					33
-+#define CAM_CC_CSIPHY1_CLK					34
-+#define CAM_CC_CSIPHY2_CLK					35
-+#define CAM_CC_CSIPHY3_CLK					36
-+#define CAM_CC_FAST_AHB_CLK_SRC					37
-+#define CAM_CC_GDSC_CLK						38
-+#define CAM_CC_ICP_AHB_CLK					39
-+#define CAM_CC_ICP_CLK						40
-+#define CAM_CC_ICP_CLK_SRC					41
-+#define CAM_CC_IFE_0_CLK					42
-+#define CAM_CC_IFE_0_CLK_SRC					43
-+#define CAM_CC_IFE_0_FAST_AHB_CLK				44
-+#define CAM_CC_IFE_1_CLK					45
-+#define CAM_CC_IFE_1_CLK_SRC					46
-+#define CAM_CC_IFE_1_FAST_AHB_CLK				47
-+#define CAM_CC_IFE_LITE_AHB_CLK					48
-+#define CAM_CC_IFE_LITE_CLK					49
-+#define CAM_CC_IFE_LITE_CLK_SRC					50
-+#define CAM_CC_IFE_LITE_CPHY_RX_CLK				51
-+#define CAM_CC_IFE_LITE_CSID_CLK				52
-+#define CAM_CC_IFE_LITE_CSID_CLK_SRC				53
-+#define CAM_CC_IPE_AHB_CLK					54
-+#define CAM_CC_IPE_CLK						55
-+#define CAM_CC_IPE_CLK_SRC					56
-+#define CAM_CC_IPE_FAST_AHB_CLK					57
-+#define CAM_CC_MCLK0_CLK					58
-+#define CAM_CC_MCLK0_CLK_SRC					59
-+#define CAM_CC_MCLK1_CLK					60
-+#define CAM_CC_MCLK1_CLK_SRC					61
-+#define CAM_CC_MCLK2_CLK					62
-+#define CAM_CC_MCLK2_CLK_SRC					63
-+#define CAM_CC_MCLK3_CLK					64
-+#define CAM_CC_MCLK3_CLK_SRC					65
-+#define CAM_CC_PLL0						66
-+#define CAM_CC_PLL0_OUT_EVEN					67
-+#define CAM_CC_PLL0_OUT_ODD					68
-+#define CAM_CC_PLL2						69
-+#define CAM_CC_PLL3						70
-+#define CAM_CC_PLL3_OUT_EVEN					71
-+#define CAM_CC_PLL4						72
-+#define CAM_CC_PLL4_OUT_EVEN					73
-+#define CAM_CC_PLL5						74
-+#define CAM_CC_PLL5_OUT_EVEN					75
-+#define CAM_CC_SFE_LITE_0_CLK					76
-+#define CAM_CC_SFE_LITE_0_FAST_AHB_CLK				77
-+#define CAM_CC_SFE_LITE_1_CLK					78
-+#define CAM_CC_SFE_LITE_1_FAST_AHB_CLK				79
-+#define CAM_CC_SLEEP_CLK					80
-+#define CAM_CC_SLEEP_CLK_SRC					81
-+#define CAM_CC_SLOW_AHB_CLK_SRC					82
-+#define CAM_CC_SM_OBS_CLK					83
-+#define CAM_CC_XO_CLK_SRC					84
-+#define CAM_CC_QDSS_DEBUG_XO_CLK				85
-+
-+/* CAM_CC power domains */
-+#define CAM_CC_TITAN_TOP_GDSC					0
-+
-+/* CAM_CC resets */
-+#define CAM_CC_ICP_BCR						0
-+#define CAM_CC_IFE_0_BCR					1
-+#define CAM_CC_IFE_1_BCR					2
-+#define CAM_CC_IPE_0_BCR					3
-+#define CAM_CC_SFE_LITE_0_BCR					4
-+#define CAM_CC_SFE_LITE_1_BCR					5
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sa8775p-dispcc.h b/include/dt-bindings/clock/qcom,sa8775p-dispcc.h
-new file mode 100644
-index 000000000000..e2049e510658
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sa8775p-dispcc.h
-@@ -0,0 +1,87 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_SA8775P_DISP_CC_H
-+#define _DT_BINDINGS_CLK_QCOM_SA8775P_DISP_CC_H
-+
-+/* DISP_CC_0/1 clocks */
-+#define MDSS_DISP_CC_MDSS_AHB1_CLK				0
-+#define MDSS_DISP_CC_MDSS_AHB_CLK				1
-+#define MDSS_DISP_CC_MDSS_AHB_CLK_SRC				2
-+#define MDSS_DISP_CC_MDSS_BYTE0_CLK				3
-+#define MDSS_DISP_CC_MDSS_BYTE0_CLK_SRC				4
-+#define MDSS_DISP_CC_MDSS_BYTE0_DIV_CLK_SRC			5
-+#define MDSS_DISP_CC_MDSS_BYTE0_INTF_CLK			6
-+#define MDSS_DISP_CC_MDSS_BYTE1_CLK				7
-+#define MDSS_DISP_CC_MDSS_BYTE1_CLK_SRC				8
-+#define MDSS_DISP_CC_MDSS_BYTE1_DIV_CLK_SRC			9
-+#define MDSS_DISP_CC_MDSS_BYTE1_INTF_CLK			10
-+#define MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK				11
-+#define MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK_SRC			12
-+#define MDSS_DISP_CC_MDSS_DPTX0_CRYPTO_CLK			13
-+#define MDSS_DISP_CC_MDSS_DPTX0_CRYPTO_CLK_SRC			14
-+#define MDSS_DISP_CC_MDSS_DPTX0_LINK_CLK			15
-+#define MDSS_DISP_CC_MDSS_DPTX0_LINK_CLK_SRC			16
-+#define MDSS_DISP_CC_MDSS_DPTX0_LINK_DIV_CLK_SRC		17
-+#define MDSS_DISP_CC_MDSS_DPTX0_LINK_INTF_CLK			18
-+#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL0_CLK			19
-+#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC			20
-+#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL1_CLK			21
-+#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC			22
-+#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL2_CLK			23
-+#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL2_CLK_SRC			24
-+#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL3_CLK			25
-+#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL3_CLK_SRC			26
-+#define MDSS_DISP_CC_MDSS_DPTX0_USB_ROUTER_LINK_INTF_CLK	27
-+#define MDSS_DISP_CC_MDSS_DPTX1_AUX_CLK				28
-+#define MDSS_DISP_CC_MDSS_DPTX1_AUX_CLK_SRC			29
-+#define MDSS_DISP_CC_MDSS_DPTX1_CRYPTO_CLK			30
-+#define MDSS_DISP_CC_MDSS_DPTX1_CRYPTO_CLK_SRC			31
-+#define MDSS_DISP_CC_MDSS_DPTX1_LINK_CLK			32
-+#define MDSS_DISP_CC_MDSS_DPTX1_LINK_CLK_SRC			33
-+#define MDSS_DISP_CC_MDSS_DPTX1_LINK_DIV_CLK_SRC		34
-+#define MDSS_DISP_CC_MDSS_DPTX1_LINK_INTF_CLK			35
-+#define MDSS_DISP_CC_MDSS_DPTX1_PIXEL0_CLK			36
-+#define MDSS_DISP_CC_MDSS_DPTX1_PIXEL0_CLK_SRC			37
-+#define MDSS_DISP_CC_MDSS_DPTX1_PIXEL1_CLK			38
-+#define MDSS_DISP_CC_MDSS_DPTX1_PIXEL1_CLK_SRC			39
-+#define MDSS_DISP_CC_MDSS_DPTX1_USB_ROUTER_LINK_INTF_CLK	40
-+#define MDSS_DISP_CC_MDSS_ESC0_CLK				41
-+#define MDSS_DISP_CC_MDSS_ESC0_CLK_SRC				42
-+#define MDSS_DISP_CC_MDSS_ESC1_CLK				43
-+#define MDSS_DISP_CC_MDSS_ESC1_CLK_SRC				44
-+#define MDSS_DISP_CC_MDSS_MDP1_CLK				45
-+#define MDSS_DISP_CC_MDSS_MDP_CLK				46
-+#define MDSS_DISP_CC_MDSS_MDP_CLK_SRC				47
-+#define MDSS_DISP_CC_MDSS_MDP_LUT1_CLK				48
-+#define MDSS_DISP_CC_MDSS_MDP_LUT_CLK				49
-+#define MDSS_DISP_CC_MDSS_NON_GDSC_AHB_CLK			50
-+#define MDSS_DISP_CC_MDSS_PCLK0_CLK				51
-+#define MDSS_DISP_CC_MDSS_PCLK0_CLK_SRC				52
-+#define MDSS_DISP_CC_MDSS_PCLK1_CLK				53
-+#define MDSS_DISP_CC_MDSS_PCLK1_CLK_SRC				54
-+#define MDSS_DISP_CC_MDSS_PLL_LOCK_MONITOR_CLK			55
-+#define MDSS_DISP_CC_MDSS_RSCC_AHB_CLK				56
-+#define MDSS_DISP_CC_MDSS_RSCC_VSYNC_CLK			57
-+#define MDSS_DISP_CC_MDSS_VSYNC1_CLK				58
-+#define MDSS_DISP_CC_MDSS_VSYNC_CLK				59
-+#define MDSS_DISP_CC_MDSS_VSYNC_CLK_SRC				60
-+#define MDSS_DISP_CC_PLL0					61
-+#define MDSS_DISP_CC_PLL1					62
-+#define MDSS_DISP_CC_SLEEP_CLK					63
-+#define MDSS_DISP_CC_SLEEP_CLK_SRC				64
-+#define MDSS_DISP_CC_SM_OBS_CLK					65
-+#define MDSS_DISP_CC_XO_CLK					66
-+#define MDSS_DISP_CC_XO_CLK_SRC					67
-+
-+/* DISP_CC_0/1 power domains */
-+#define MDSS_DISP_CC_MDSS_CORE_GDSC				0
-+#define MDSS_DISP_CC_MDSS_CORE_INT2_GDSC			1
-+
-+/* DISP_CC_0/1 resets */
-+#define MDSS_DISP_CC_MDSS_CORE_BCR				0
-+#define MDSS_DISP_CC_MDSS_RSCC_BCR				1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sa8775p-gcc.h b/include/dt-bindings/clock/qcom,sa8775p-gcc.h
-new file mode 100644
-index 000000000000..01f54234963d
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sa8775p-gcc.h
-@@ -0,0 +1,320 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_SA8775P_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_SA8775P_H
-+
-+/* GCC clocks */
-+#define GCC_GPLL0					0
-+#define GCC_GPLL0_OUT_EVEN				1
-+#define GCC_GPLL1					2
-+#define GCC_GPLL4					3
-+#define GCC_GPLL5					4
-+#define GCC_GPLL7					5
-+#define GCC_GPLL9					6
-+#define GCC_AGGRE_NOC_QUPV3_AXI_CLK			7
-+#define GCC_AGGRE_UFS_CARD_AXI_CLK			8
-+#define GCC_AGGRE_UFS_PHY_AXI_CLK			9
-+#define GCC_AGGRE_USB2_PRIM_AXI_CLK			10
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK			11
-+#define GCC_AGGRE_USB3_SEC_AXI_CLK			12
-+#define GCC_AHB2PHY0_CLK				13
-+#define GCC_AHB2PHY2_CLK				14
-+#define GCC_AHB2PHY3_CLK				15
-+#define GCC_BOOT_ROM_AHB_CLK				16
-+#define GCC_CAMERA_AHB_CLK				17
-+#define GCC_CAMERA_HF_AXI_CLK				18
-+#define GCC_CAMERA_SF_AXI_CLK				19
-+#define GCC_CAMERA_THROTTLE_XO_CLK			20
-+#define GCC_CAMERA_XO_CLK				21
-+#define GCC_CFG_NOC_USB2_PRIM_AXI_CLK			22
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK			23
-+#define GCC_CFG_NOC_USB3_SEC_AXI_CLK			24
-+#define GCC_DDRSS_GPU_AXI_CLK				25
-+#define GCC_DISP1_AHB_CLK				26
-+#define GCC_DISP1_HF_AXI_CLK				27
-+#define GCC_DISP1_XO_CLK				28
-+#define GCC_DISP_AHB_CLK				29
-+#define GCC_DISP_HF_AXI_CLK				30
-+#define GCC_DISP_XO_CLK					31
-+#define GCC_EDP_REF_CLKREF_EN				32
-+#define GCC_EMAC0_AXI_CLK				33
-+#define GCC_EMAC0_PHY_AUX_CLK				34
-+#define GCC_EMAC0_PHY_AUX_CLK_SRC			35
-+#define GCC_EMAC0_PTP_CLK				36
-+#define GCC_EMAC0_PTP_CLK_SRC				37
-+#define GCC_EMAC0_RGMII_CLK				38
-+#define GCC_EMAC0_RGMII_CLK_SRC				39
-+#define GCC_EMAC0_SLV_AHB_CLK				40
-+#define GCC_EMAC1_AXI_CLK				41
-+#define GCC_EMAC1_PHY_AUX_CLK				42
-+#define GCC_EMAC1_PHY_AUX_CLK_SRC			43
-+#define GCC_EMAC1_PTP_CLK				44
-+#define GCC_EMAC1_PTP_CLK_SRC				45
-+#define GCC_EMAC1_RGMII_CLK				46
-+#define GCC_EMAC1_RGMII_CLK_SRC				47
-+#define GCC_EMAC1_SLV_AHB_CLK				48
-+#define GCC_GP1_CLK					49
-+#define GCC_GP1_CLK_SRC					50
-+#define GCC_GP2_CLK					51
-+#define GCC_GP2_CLK_SRC					52
-+#define GCC_GP3_CLK					53
-+#define GCC_GP3_CLK_SRC					54
-+#define GCC_GP4_CLK					55
-+#define GCC_GP4_CLK_SRC					56
-+#define GCC_GP5_CLK					57
-+#define GCC_GP5_CLK_SRC					58
-+#define GCC_GPU_CFG_AHB_CLK				59
-+#define GCC_GPU_GPLL0_CLK_SRC				60
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC			61
-+#define GCC_GPU_MEMNOC_GFX_CLK				62
-+#define GCC_GPU_SNOC_DVM_GFX_CLK			63
-+#define GCC_GPU_TCU_THROTTLE_AHB_CLK			64
-+#define GCC_GPU_TCU_THROTTLE_CLK			65
-+#define GCC_PCIE_0_AUX_CLK				66
-+#define GCC_PCIE_0_AUX_CLK_SRC				67
-+#define GCC_PCIE_0_CFG_AHB_CLK				68
-+#define GCC_PCIE_0_MSTR_AXI_CLK				69
-+#define GCC_PCIE_0_PHY_AUX_CLK				70
-+#define GCC_PCIE_0_PHY_AUX_CLK_SRC			71
-+#define GCC_PCIE_0_PHY_RCHNG_CLK			72
-+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC			73
-+#define GCC_PCIE_0_PIPE_CLK				74
-+#define GCC_PCIE_0_PIPE_CLK_SRC				75
-+#define GCC_PCIE_0_PIPE_DIV_CLK_SRC			76
-+#define GCC_PCIE_0_PIPEDIV2_CLK				77
-+#define GCC_PCIE_0_SLV_AXI_CLK				78
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK			79
-+#define GCC_PCIE_1_AUX_CLK				80
-+#define GCC_PCIE_1_AUX_CLK_SRC				81
-+#define GCC_PCIE_1_CFG_AHB_CLK				82
-+#define GCC_PCIE_1_MSTR_AXI_CLK				83
-+#define GCC_PCIE_1_PHY_AUX_CLK				84
-+#define GCC_PCIE_1_PHY_AUX_CLK_SRC			85
-+#define GCC_PCIE_1_PHY_RCHNG_CLK			86
-+#define GCC_PCIE_1_PHY_RCHNG_CLK_SRC			87
-+#define GCC_PCIE_1_PIPE_CLK				88
-+#define GCC_PCIE_1_PIPE_CLK_SRC				89
-+#define GCC_PCIE_1_PIPE_DIV_CLK_SRC			90
-+#define GCC_PCIE_1_PIPEDIV2_CLK				91
-+#define GCC_PCIE_1_SLV_AXI_CLK				92
-+#define GCC_PCIE_1_SLV_Q2A_AXI_CLK			93
-+#define GCC_PCIE_CLKREF_EN				94
-+#define GCC_PCIE_THROTTLE_CFG_CLK			95
-+#define GCC_PDM2_CLK					96
-+#define GCC_PDM2_CLK_SRC				97
-+#define GCC_PDM_AHB_CLK					98
-+#define GCC_PDM_XO4_CLK					99
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK			100
-+#define GCC_QMIP_CAMERA_RT_AHB_CLK			101
-+#define GCC_QMIP_DISP1_AHB_CLK				102
-+#define GCC_QMIP_DISP1_ROT_AHB_CLK			103
-+#define GCC_QMIP_DISP_AHB_CLK				104
-+#define GCC_QMIP_DISP_ROT_AHB_CLK			105
-+#define GCC_QMIP_VIDEO_CVP_AHB_CLK			106
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK			107
-+#define GCC_QMIP_VIDEO_VCPU_AHB_CLK			108
-+#define GCC_QUPV3_WRAP0_CORE_2X_CLK			109
-+#define GCC_QUPV3_WRAP0_CORE_CLK			110
-+#define GCC_QUPV3_WRAP0_S0_CLK				111
-+#define GCC_QUPV3_WRAP0_S0_CLK_SRC			112
-+#define GCC_QUPV3_WRAP0_S1_CLK				113
-+#define GCC_QUPV3_WRAP0_S1_CLK_SRC			114
-+#define GCC_QUPV3_WRAP0_S2_CLK				115
-+#define GCC_QUPV3_WRAP0_S2_CLK_SRC			116
-+#define GCC_QUPV3_WRAP0_S3_CLK				117
-+#define GCC_QUPV3_WRAP0_S3_CLK_SRC			118
-+#define GCC_QUPV3_WRAP0_S4_CLK				119
-+#define GCC_QUPV3_WRAP0_S4_CLK_SRC			120
-+#define GCC_QUPV3_WRAP0_S5_CLK				121
-+#define GCC_QUPV3_WRAP0_S5_CLK_SRC			122
-+#define GCC_QUPV3_WRAP0_S6_CLK				123
-+#define GCC_QUPV3_WRAP0_S6_CLK_SRC			124
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK			125
-+#define GCC_QUPV3_WRAP1_CORE_CLK			126
-+#define GCC_QUPV3_WRAP1_S0_CLK				127
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC			128
-+#define GCC_QUPV3_WRAP1_S1_CLK				129
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC			130
-+#define GCC_QUPV3_WRAP1_S2_CLK				131
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC			132
-+#define GCC_QUPV3_WRAP1_S3_CLK				133
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC			134
-+#define GCC_QUPV3_WRAP1_S4_CLK				135
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC			136
-+#define GCC_QUPV3_WRAP1_S5_CLK				137
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC			138
-+#define GCC_QUPV3_WRAP1_S6_CLK				139
-+#define GCC_QUPV3_WRAP1_S6_CLK_SRC			140
-+#define GCC_QUPV3_WRAP2_CORE_2X_CLK			141
-+#define GCC_QUPV3_WRAP2_CORE_CLK			142
-+#define GCC_QUPV3_WRAP2_S0_CLK				143
-+#define GCC_QUPV3_WRAP2_S0_CLK_SRC			144
-+#define GCC_QUPV3_WRAP2_S1_CLK				145
-+#define GCC_QUPV3_WRAP2_S1_CLK_SRC			146
-+#define GCC_QUPV3_WRAP2_S2_CLK				147
-+#define GCC_QUPV3_WRAP2_S2_CLK_SRC			148
-+#define GCC_QUPV3_WRAP2_S3_CLK				149
-+#define GCC_QUPV3_WRAP2_S3_CLK_SRC			150
-+#define GCC_QUPV3_WRAP2_S4_CLK				151
-+#define GCC_QUPV3_WRAP2_S4_CLK_SRC			152
-+#define GCC_QUPV3_WRAP2_S5_CLK				153
-+#define GCC_QUPV3_WRAP2_S5_CLK_SRC			154
-+#define GCC_QUPV3_WRAP2_S6_CLK				155
-+#define GCC_QUPV3_WRAP2_S6_CLK_SRC			156
-+#define GCC_QUPV3_WRAP3_CORE_2X_CLK			157
-+#define GCC_QUPV3_WRAP3_CORE_CLK			158
-+#define GCC_QUPV3_WRAP3_QSPI_CLK			159
-+#define GCC_QUPV3_WRAP3_S0_CLK				160
-+#define GCC_QUPV3_WRAP3_S0_CLK_SRC			161
-+#define GCC_QUPV3_WRAP3_S0_DIV_CLK_SRC			162
-+#define GCC_QUPV3_WRAP_0_M_AHB_CLK			163
-+#define GCC_QUPV3_WRAP_0_S_AHB_CLK			164
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK			165
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK			166
-+#define GCC_QUPV3_WRAP_2_M_AHB_CLK			167
-+#define GCC_QUPV3_WRAP_2_S_AHB_CLK			168
-+#define GCC_QUPV3_WRAP_3_M_AHB_CLK			169
-+#define GCC_QUPV3_WRAP_3_S_AHB_CLK			170
-+#define GCC_SDCC1_AHB_CLK				171
-+#define GCC_SDCC1_APPS_CLK				172
-+#define GCC_SDCC1_APPS_CLK_SRC				173
-+#define GCC_SDCC1_ICE_CORE_CLK				174
-+#define GCC_SDCC1_ICE_CORE_CLK_SRC			175
-+#define GCC_SGMI_CLKREF_EN				176
-+#define GCC_TSCSS_AHB_CLK				177
-+#define GCC_TSCSS_CNTR_CLK_SRC				178
-+#define GCC_TSCSS_ETU_CLK				179
-+#define GCC_TSCSS_GLOBAL_CNTR_CLK			180
-+#define GCC_UFS_CARD_AHB_CLK				181
-+#define GCC_UFS_CARD_AXI_CLK				182
-+#define GCC_UFS_CARD_AXI_CLK_SRC			183
-+#define GCC_UFS_CARD_ICE_CORE_CLK			184
-+#define GCC_UFS_CARD_ICE_CORE_CLK_SRC			185
-+#define GCC_UFS_CARD_PHY_AUX_CLK			186
-+#define GCC_UFS_CARD_PHY_AUX_CLK_SRC			187
-+#define GCC_UFS_CARD_RX_SYMBOL_0_CLK			188
-+#define GCC_UFS_CARD_RX_SYMBOL_0_CLK_SRC		189
-+#define GCC_UFS_CARD_RX_SYMBOL_1_CLK			190
-+#define GCC_UFS_CARD_RX_SYMBOL_1_CLK_SRC		191
-+#define GCC_UFS_CARD_TX_SYMBOL_0_CLK			192
-+#define GCC_UFS_CARD_TX_SYMBOL_0_CLK_SRC		193
-+#define GCC_UFS_CARD_UNIPRO_CORE_CLK			194
-+#define GCC_UFS_CARD_UNIPRO_CORE_CLK_SRC		195
-+#define GCC_UFS_PHY_AHB_CLK				196
-+#define GCC_UFS_PHY_AXI_CLK				197
-+#define GCC_UFS_PHY_AXI_CLK_SRC				198
-+#define GCC_UFS_PHY_ICE_CORE_CLK			199
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC			200
-+#define GCC_UFS_PHY_PHY_AUX_CLK				201
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC			202
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK			203
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC			204
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK			205
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC			206
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK			207
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC			208
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK			209
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC			210
-+#define GCC_USB20_MASTER_CLK				211
-+#define GCC_USB20_MASTER_CLK_SRC			212
-+#define GCC_USB20_MOCK_UTMI_CLK				213
-+#define GCC_USB20_MOCK_UTMI_CLK_SRC			214
-+#define GCC_USB20_MOCK_UTMI_POSTDIV_CLK_SRC		215
-+#define GCC_USB20_SLEEP_CLK				216
-+#define GCC_USB30_PRIM_MASTER_CLK			217
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC			218
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK			219
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC		220
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC	221
-+#define GCC_USB30_PRIM_SLEEP_CLK			222
-+#define GCC_USB30_SEC_MASTER_CLK			223
-+#define GCC_USB30_SEC_MASTER_CLK_SRC			224
-+#define GCC_USB30_SEC_MOCK_UTMI_CLK			225
-+#define GCC_USB30_SEC_MOCK_UTMI_CLK_SRC			226
-+#define GCC_USB30_SEC_MOCK_UTMI_POSTDIV_CLK_SRC		227
-+#define GCC_USB30_SEC_SLEEP_CLK				228
-+#define GCC_USB3_PRIM_PHY_AUX_CLK			229
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC			230
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK			231
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK			232
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC			233
-+#define GCC_USB3_SEC_PHY_AUX_CLK			234
-+#define GCC_USB3_SEC_PHY_AUX_CLK_SRC			235
-+#define GCC_USB3_SEC_PHY_COM_AUX_CLK			236
-+#define GCC_USB3_SEC_PHY_PIPE_CLK			237
-+#define GCC_USB3_SEC_PHY_PIPE_CLK_SRC			238
-+#define GCC_USB_CLKREF_EN				239
-+#define GCC_VIDEO_AHB_CLK				240
-+#define GCC_VIDEO_AXI0_CLK				241
-+#define GCC_VIDEO_AXI1_CLK				242
-+#define GCC_VIDEO_XO_CLK				243
-+#define GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK		244
-+#define GCC_UFS_PHY_AXI_HW_CTL_CLK			245
-+#define GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK			246
-+#define GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK			247
-+#define GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK		248
-+
-+/* GCC resets */
-+#define GCC_CAMERA_BCR					0
-+#define GCC_DISPLAY1_BCR				1
-+#define GCC_DISPLAY_BCR					2
-+#define GCC_EMAC0_BCR					3
-+#define GCC_EMAC1_BCR					4
-+#define GCC_GPU_BCR					5
-+#define GCC_MMSS_BCR					6
-+#define GCC_PCIE_0_BCR					7
-+#define GCC_PCIE_0_LINK_DOWN_BCR			8
-+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR			9
-+#define GCC_PCIE_0_PHY_BCR				10
-+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR		11
-+#define GCC_PCIE_1_BCR					12
-+#define GCC_PCIE_1_LINK_DOWN_BCR			13
-+#define GCC_PCIE_1_NOCSR_COM_PHY_BCR			14
-+#define GCC_PCIE_1_PHY_BCR				15
-+#define GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR		16
-+#define GCC_PDM_BCR					17
-+#define GCC_QUPV3_WRAPPER_0_BCR				18
-+#define GCC_QUPV3_WRAPPER_1_BCR				19
-+#define GCC_QUPV3_WRAPPER_2_BCR				20
-+#define GCC_QUPV3_WRAPPER_3_BCR				21
-+#define GCC_SDCC1_BCR					22
-+#define GCC_TSCSS_BCR					23
-+#define GCC_UFS_CARD_BCR				24
-+#define GCC_UFS_PHY_BCR					25
-+#define GCC_USB20_PRIM_BCR				26
-+#define GCC_USB2_PHY_PRIM_BCR				27
-+#define GCC_USB2_PHY_SEC_BCR				28
-+#define GCC_USB30_PRIM_BCR				29
-+#define GCC_USB30_SEC_BCR				30
-+#define GCC_USB3_DP_PHY_PRIM_BCR			31
-+#define GCC_USB3_DP_PHY_SEC_BCR				32
-+#define GCC_USB3_PHY_PRIM_BCR				33
-+#define GCC_USB3_PHY_SEC_BCR				34
-+#define GCC_USB3_PHY_TERT_BCR				35
-+#define GCC_USB3_UNIPHY_MP0_BCR				36
-+#define GCC_USB3_UNIPHY_MP1_BCR				37
-+#define GCC_USB3PHY_PHY_PRIM_BCR			38
-+#define GCC_USB3PHY_PHY_SEC_BCR				39
-+#define GCC_USB3UNIPHY_PHY_MP0_BCR			40
-+#define GCC_USB3UNIPHY_PHY_MP1_BCR			41
-+#define GCC_USB_PHY_CFG_AHB2PHY_BCR			42
-+#define GCC_VIDEO_BCR					43
-+#define GCC_VIDEO_AXI0_CLK_ARES				44
-+#define GCC_VIDEO_AXI1_CLK_ARES				45
-+
-+/* GCC GDSCs */
-+#define PCIE_0_GDSC					0
-+#define PCIE_1_GDSC					1
-+#define UFS_CARD_GDSC					2
-+#define UFS_PHY_GDSC					3
-+#define USB20_PRIM_GDSC					4
-+#define USB30_PRIM_GDSC					5
-+#define USB30_SEC_GDSC					6
-+#define EMAC0_GDSC					7
-+#define EMAC1_GDSC					8
-+
-+#endif /* _DT_BINDINGS_CLK_QCOM_GCC_SA8775P_H */
-diff --git a/include/dt-bindings/clock/qcom,sa8775p-gpucc.h b/include/dt-bindings/clock/qcom,sa8775p-gpucc.h
-new file mode 100644
-index 000000000000..a5fd784b1ea2
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sa8775p-gpucc.h
-@@ -0,0 +1,50 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPUCC_SA8775P_H
-+#define _DT_BINDINGS_CLK_QCOM_GPUCC_SA8775P_H
-+
-+/* GPU_CC clocks */
-+#define GPU_CC_PLL0				0
-+#define GPU_CC_PLL1				1
-+#define GPU_CC_AHB_CLK				2
-+#define GPU_CC_CB_CLK				3
-+#define GPU_CC_CRC_AHB_CLK			4
-+#define GPU_CC_CX_FF_CLK			5
-+#define GPU_CC_CX_GMU_CLK			6
-+#define GPU_CC_CX_SNOC_DVM_CLK			7
-+#define GPU_CC_CXO_AON_CLK			8
-+#define GPU_CC_CXO_CLK				9
-+#define GPU_CC_DEMET_CLK			10
-+#define GPU_CC_DEMET_DIV_CLK_SRC		11
-+#define GPU_CC_FF_CLK_SRC			12
-+#define GPU_CC_GMU_CLK_SRC			13
-+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK		14
-+#define GPU_CC_HUB_AHB_DIV_CLK_SRC		15
-+#define GPU_CC_HUB_AON_CLK			16
-+#define GPU_CC_HUB_CLK_SRC			17
-+#define GPU_CC_HUB_CX_INT_CLK			18
-+#define GPU_CC_HUB_CX_INT_DIV_CLK_SRC		19
-+#define GPU_CC_MEMNOC_GFX_CLK			20
-+#define GPU_CC_SLEEP_CLK			21
-+#define GPU_CC_XO_CLK_SRC			22
-+
-+/* GPU_CC resets */
-+#define GPUCC_GPU_CC_ACD_BCR			0
-+#define GPUCC_GPU_CC_CB_BCR			1
-+#define GPUCC_GPU_CC_CX_BCR			2
-+#define GPUCC_GPU_CC_FAST_HUB_BCR		3
-+#define GPUCC_GPU_CC_FF_BCR			4
-+#define GPUCC_GPU_CC_GFX3D_AON_BCR		5
-+#define GPUCC_GPU_CC_GMU_BCR			6
-+#define GPUCC_GPU_CC_GX_BCR			7
-+#define GPUCC_GPU_CC_XO_BCR			8
-+
-+/* GPU_CC power domains */
-+#define GPU_CC_CX_GDSC				0
-+#define GPU_CC_GX_GDSC				1
-+
-+#endif /* _DT_BINDINGS_CLK_QCOM_GPUCC_SA8775P_H */
-diff --git a/include/dt-bindings/clock/qcom,sa8775p-videocc.h b/include/dt-bindings/clock/qcom,sa8775p-videocc.h
-new file mode 100644
-index 000000000000..e6325f68c317
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sa8775p-videocc.h
-@@ -0,0 +1,47 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_SA8775P_VIDEO_CC_H
-+#define _DT_BINDINGS_CLK_QCOM_SA8775P_VIDEO_CC_H
-+
-+/* VIDEO_CC clocks */
-+#define VIDEO_CC_AHB_CLK					0
-+#define VIDEO_CC_AHB_CLK_SRC					1
-+#define VIDEO_CC_MVS0_CLK					2
-+#define VIDEO_CC_MVS0_CLK_SRC					3
-+#define VIDEO_CC_MVS0_DIV_CLK_SRC				4
-+#define VIDEO_CC_MVS0C_CLK					5
-+#define VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC				6
-+#define VIDEO_CC_MVS1_CLK					7
-+#define VIDEO_CC_MVS1_CLK_SRC					8
-+#define VIDEO_CC_MVS1_DIV_CLK_SRC				9
-+#define VIDEO_CC_MVS1C_CLK					10
-+#define VIDEO_CC_MVS1C_DIV2_DIV_CLK_SRC				11
-+#define VIDEO_CC_PLL_LOCK_MONITOR_CLK				12
-+#define VIDEO_CC_SLEEP_CLK					13
-+#define VIDEO_CC_SLEEP_CLK_SRC					14
-+#define VIDEO_CC_SM_DIV_CLK_SRC					15
-+#define VIDEO_CC_SM_OBS_CLK					16
-+#define VIDEO_CC_XO_CLK						17
-+#define VIDEO_CC_XO_CLK_SRC					18
-+#define VIDEO_PLL0						19
-+#define VIDEO_PLL1						20
-+
-+/* VIDEO_CC power domains */
-+#define VIDEO_CC_MVS0C_GDSC					0
-+#define VIDEO_CC_MVS0_GDSC					1
-+#define VIDEO_CC_MVS1C_GDSC					2
-+#define VIDEO_CC_MVS1_GDSC					3
-+
-+/* VIDEO_CC resets */
-+#define VIDEO_CC_INTERFACE_BCR					0
-+#define VIDEO_CC_MVS0_BCR					1
-+#define VIDEO_CC_MVS0C_CLK_ARES					2
-+#define VIDEO_CC_MVS0C_BCR					3
-+#define VIDEO_CC_MVS1_BCR					4
-+#define VIDEO_CC_MVS1C_CLK_ARES					5
-+#define VIDEO_CC_MVS1C_BCR					6
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sar2130p-gcc.h b/include/dt-bindings/clock/qcom,sar2130p-gcc.h
-new file mode 100644
-index 000000000000..69d2dd2538a6
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sar2130p-gcc.h
-@@ -0,0 +1,185 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (c) 2021-2022, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_SAR2130P_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_SAR2130P_H
-+
-+/* GCC clocks */
-+#define GCC_GPLL0						0
-+#define GCC_GPLL0_OUT_EVEN					1
-+#define GCC_GPLL1						2
-+#define GCC_GPLL9						3
-+#define GCC_GPLL9_OUT_EVEN					4
-+#define GCC_AGGRE_NOC_PCIE_1_AXI_CLK				5
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK				6
-+#define GCC_BOOT_ROM_AHB_CLK					7
-+#define GCC_CAMERA_AHB_CLK					8
-+#define GCC_CAMERA_HF_AXI_CLK					9
-+#define GCC_CAMERA_SF_AXI_CLK					10
-+#define GCC_CAMERA_XO_CLK					11
-+#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK				12
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				13
-+#define GCC_DDRSS_GPU_AXI_CLK					14
-+#define GCC_DDRSS_PCIE_SF_CLK					15
-+#define GCC_DISP_AHB_CLK					16
-+#define GCC_DISP_HF_AXI_CLK					17
-+#define GCC_GP1_CLK						18
-+#define GCC_GP1_CLK_SRC						19
-+#define GCC_GP2_CLK						20
-+#define GCC_GP2_CLK_SRC						21
-+#define GCC_GP3_CLK						22
-+#define GCC_GP3_CLK_SRC						23
-+#define GCC_GPU_CFG_AHB_CLK					24
-+#define GCC_GPU_GPLL0_CLK_SRC					25
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC				26
-+#define GCC_GPU_MEMNOC_GFX_CLK					27
-+#define GCC_GPU_SNOC_DVM_GFX_CLK				28
-+#define GCC_IRIS_SS_HF_AXI1_CLK					29
-+#define GCC_IRIS_SS_SPD_AXI1_CLK				30
-+#define GCC_PCIE_0_AUX_CLK					31
-+#define GCC_PCIE_0_AUX_CLK_SRC					32
-+#define GCC_PCIE_0_CFG_AHB_CLK					33
-+#define GCC_PCIE_0_MSTR_AXI_CLK					34
-+#define GCC_PCIE_0_PHY_RCHNG_CLK				35
-+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC				36
-+#define GCC_PCIE_0_PIPE_CLK					37
-+#define GCC_PCIE_0_PIPE_CLK_SRC					38
-+#define GCC_PCIE_0_SLV_AXI_CLK					39
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				40
-+#define GCC_PCIE_1_AUX_CLK					41
-+#define GCC_PCIE_1_AUX_CLK_SRC					42
-+#define GCC_PCIE_1_CFG_AHB_CLK					43
-+#define GCC_PCIE_1_MSTR_AXI_CLK					44
-+#define GCC_PCIE_1_PHY_RCHNG_CLK				45
-+#define GCC_PCIE_1_PHY_RCHNG_CLK_SRC				46
-+#define GCC_PCIE_1_PIPE_CLK					47
-+#define GCC_PCIE_1_PIPE_CLK_SRC					48
-+#define GCC_PCIE_1_SLV_AXI_CLK					49
-+#define GCC_PCIE_1_SLV_Q2A_AXI_CLK				50
-+#define GCC_PDM2_CLK						51
-+#define GCC_PDM2_CLK_SRC					52
-+#define GCC_PDM_AHB_CLK						53
-+#define GCC_PDM_XO4_CLK						54
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				55
-+#define GCC_QMIP_CAMERA_RT_AHB_CLK				56
-+#define GCC_QMIP_GPU_AHB_CLK					57
-+#define GCC_QMIP_PCIE_AHB_CLK					58
-+#define GCC_QMIP_VIDEO_CV_CPU_AHB_CLK				59
-+#define GCC_QMIP_VIDEO_CVP_AHB_CLK				60
-+#define GCC_QMIP_VIDEO_LSR_AHB_CLK				61
-+#define GCC_QMIP_VIDEO_V_CPU_AHB_CLK				62
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				63
-+#define GCC_QUPV3_WRAP0_CORE_2X_CLK				64
-+#define GCC_QUPV3_WRAP0_CORE_CLK				65
-+#define GCC_QUPV3_WRAP0_S0_CLK					66
-+#define GCC_QUPV3_WRAP0_S0_CLK_SRC				67
-+#define GCC_QUPV3_WRAP0_S1_CLK					68
-+#define GCC_QUPV3_WRAP0_S1_CLK_SRC				69
-+#define GCC_QUPV3_WRAP0_S2_CLK					70
-+#define GCC_QUPV3_WRAP0_S2_CLK_SRC				71
-+#define GCC_QUPV3_WRAP0_S3_CLK					72
-+#define GCC_QUPV3_WRAP0_S3_CLK_SRC				73
-+#define GCC_QUPV3_WRAP0_S4_CLK					74
-+#define GCC_QUPV3_WRAP0_S4_CLK_SRC				75
-+#define GCC_QUPV3_WRAP0_S5_CLK					76
-+#define GCC_QUPV3_WRAP0_S5_CLK_SRC				77
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				78
-+#define GCC_QUPV3_WRAP1_CORE_CLK				79
-+#define GCC_QUPV3_WRAP1_S0_CLK					80
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				81
-+#define GCC_QUPV3_WRAP1_S1_CLK					82
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				83
-+#define GCC_QUPV3_WRAP1_S2_CLK					84
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				85
-+#define GCC_QUPV3_WRAP1_S3_CLK					86
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				87
-+#define GCC_QUPV3_WRAP1_S4_CLK					88
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				89
-+#define GCC_QUPV3_WRAP1_S5_CLK					90
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC				91
-+#define GCC_QUPV3_WRAP_0_M_AHB_CLK				92
-+#define GCC_QUPV3_WRAP_0_S_AHB_CLK				93
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK				94
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				95
-+#define GCC_SDCC1_AHB_CLK					96
-+#define GCC_SDCC1_APPS_CLK					97
-+#define GCC_SDCC1_APPS_CLK_SRC					98
-+#define GCC_SDCC1_ICE_CORE_CLK					99
-+#define GCC_SDCC1_ICE_CORE_CLK_SRC				100
-+#define GCC_USB30_PRIM_MASTER_CLK				101
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC				102
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				103
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			104
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		105
-+#define GCC_USB30_PRIM_SLEEP_CLK				106
-+#define GCC_USB3_PRIM_PHY_AUX_CLK				107
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				108
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				109
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK				110
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				111
-+#define GCC_VIDEO_AHB_CLK					112
-+#define GCC_VIDEO_AXI0_CLK					113
-+#define GCC_VIDEO_AXI1_CLK					114
-+#define GCC_VIDEO_XO_CLK					115
-+#define GCC_GPLL4						116
-+#define GCC_GPLL5						117
-+#define GCC_GPLL7						118
-+#define GCC_DDRSS_SPAD_CLK					119
-+#define GCC_DDRSS_SPAD_CLK_SRC					120
-+#define GCC_VIDEO_AXI0_SREG					121
-+#define GCC_VIDEO_AXI1_SREG					122
-+#define GCC_IRIS_SS_HF_AXI1_SREG				123
-+#define GCC_IRIS_SS_SPD_AXI1_SREG				124
-+
-+/* GCC resets */
-+#define GCC_CAMERA_BCR						0
-+#define GCC_DISPLAY_BCR						1
-+#define GCC_GPU_BCR						2
-+#define GCC_PCIE_0_BCR						3
-+#define GCC_PCIE_0_LINK_DOWN_BCR				4
-+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR				5
-+#define GCC_PCIE_0_PHY_BCR					6
-+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR			7
-+#define GCC_PCIE_1_BCR						8
-+#define GCC_PCIE_1_LINK_DOWN_BCR				9
-+#define GCC_PCIE_1_NOCSR_COM_PHY_BCR				10
-+#define GCC_PCIE_1_PHY_BCR					11
-+#define GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR			12
-+#define GCC_PCIE_PHY_BCR					13
-+#define GCC_PCIE_PHY_CFG_AHB_BCR				14
-+#define GCC_PCIE_PHY_COM_BCR					15
-+#define GCC_PDM_BCR						16
-+#define GCC_QUPV3_WRAPPER_0_BCR					17
-+#define GCC_QUPV3_WRAPPER_1_BCR					18
-+#define GCC_QUSB2PHY_PRIM_BCR					19
-+#define GCC_QUSB2PHY_SEC_BCR					20
-+#define GCC_SDCC1_BCR						21
-+#define GCC_USB30_PRIM_BCR					22
-+#define GCC_USB3_DP_PHY_PRIM_BCR				23
-+#define GCC_USB3_DP_PHY_SEC_BCR					24
-+#define GCC_USB3_PHY_PRIM_BCR					25
-+#define GCC_USB3_PHY_SEC_BCR					26
-+#define GCC_USB3PHY_PHY_PRIM_BCR				27
-+#define GCC_USB3PHY_PHY_SEC_BCR					28
-+#define GCC_VIDEO_AXI0_CLK_ARES					29
-+#define GCC_VIDEO_AXI1_CLK_ARES					30
-+#define GCC_VIDEO_BCR						31
-+#define GCC_IRIS_SS_HF_AXI_CLK_ARES				32
-+#define GCC_IRIS_SS_SPD_AXI_CLK_ARES				33
-+#define GCC_DDRSS_SPAD_CLK_ARES					34
-+
-+/* GCC power domains */
-+#define PCIE_0_GDSC						0
-+#define PCIE_0_PHY_GDSC						1
-+#define PCIE_1_GDSC						2
-+#define PCIE_1_PHY_GDSC						3
-+#define USB30_PRIM_GDSC						4
-+#define USB3_PHY_GDSC						5
-+#define HLOS1_VOTE_MM_SNOC_MMU_TBU_HF0_GDSC			6
-+#define HLOS1_VOTE_MM_SNOC_MMU_TBU_SF0_GDSC			7
-+#define HLOS1_VOTE_TURING_MMU_TBU0_GDSC				8
-+#define HLOS1_VOTE_TURING_MMU_TBU1_GDSC				9
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sar2130p-gpucc.h b/include/dt-bindings/clock/qcom,sar2130p-gpucc.h
-new file mode 100644
-index 000000000000..a2204369110a
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sar2130p-gpucc.h
-@@ -0,0 +1,33 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved
-+ * Copyright (c) 2024, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_SAR2130P_H
-+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_SAR2130P_H
-+
-+/* GPU_CC clocks */
-+#define GPU_CC_AHB_CLK				0
-+#define GPU_CC_CRC_AHB_CLK			1
-+#define GPU_CC_CX_FF_CLK			2
-+#define GPU_CC_CX_GMU_CLK			3
-+#define GPU_CC_CXO_AON_CLK			4
-+#define GPU_CC_CXO_CLK				5
-+#define GPU_CC_FF_CLK_SRC			6
-+#define GPU_CC_GMU_CLK_SRC			7
-+#define GPU_CC_GX_GMU_CLK			8
-+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK		9
-+#define GPU_CC_HUB_AON_CLK			10
-+#define GPU_CC_HUB_CLK_SRC			11
-+#define GPU_CC_HUB_CX_INT_CLK			12
-+#define GPU_CC_MEMNOC_GFX_CLK			13
-+#define GPU_CC_PLL0				14
-+#define GPU_CC_PLL1				15
-+#define GPU_CC_SLEEP_CLK			16
-+
-+/* GDSCs */
-+#define GPU_GX_GDSC				0
-+#define GPU_CX_GDSC				1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sc8180x-camcc.h b/include/dt-bindings/clock/qcom,sc8180x-camcc.h
-new file mode 100644
-index 000000000000..3e57b80f65e8
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sc8180x-camcc.h
-@@ -0,0 +1,181 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2025, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_SC8180X_H
-+#define _DT_BINDINGS_CLK_QCOM_CAM_CC_SC8180X_H
-+
-+/* CAM_CC clocks */
-+#define CAM_CC_BPS_AHB_CLK					0
-+#define CAM_CC_BPS_AREG_CLK					1
-+#define CAM_CC_BPS_AXI_CLK					2
-+#define CAM_CC_BPS_CLK						3
-+#define CAM_CC_BPS_CLK_SRC					4
-+#define CAM_CC_CAMNOC_AXI_CLK					5
-+#define CAM_CC_CAMNOC_AXI_CLK_SRC				6
-+#define CAM_CC_CAMNOC_DCD_XO_CLK				7
-+#define CAM_CC_CCI_0_CLK					8
-+#define CAM_CC_CCI_0_CLK_SRC					9
-+#define CAM_CC_CCI_1_CLK					10
-+#define CAM_CC_CCI_1_CLK_SRC					11
-+#define CAM_CC_CCI_2_CLK					12
-+#define CAM_CC_CCI_2_CLK_SRC					13
-+#define CAM_CC_CCI_3_CLK					14
-+#define CAM_CC_CCI_3_CLK_SRC					15
-+#define CAM_CC_CORE_AHB_CLK					16
-+#define CAM_CC_CPAS_AHB_CLK					17
-+#define CAM_CC_CPHY_RX_CLK_SRC					18
-+#define CAM_CC_CSI0PHYTIMER_CLK					19
-+#define CAM_CC_CSI0PHYTIMER_CLK_SRC				20
-+#define CAM_CC_CSI1PHYTIMER_CLK					21
-+#define CAM_CC_CSI1PHYTIMER_CLK_SRC				22
-+#define CAM_CC_CSI2PHYTIMER_CLK					23
-+#define CAM_CC_CSI2PHYTIMER_CLK_SRC				24
-+#define CAM_CC_CSI3PHYTIMER_CLK					25
-+#define CAM_CC_CSI3PHYTIMER_CLK_SRC				26
-+#define CAM_CC_CSIPHY0_CLK					27
-+#define CAM_CC_CSIPHY1_CLK					28
-+#define CAM_CC_CSIPHY2_CLK					29
-+#define CAM_CC_CSIPHY3_CLK					30
-+#define CAM_CC_FAST_AHB_CLK_SRC					31
-+#define CAM_CC_FD_CORE_CLK					32
-+#define CAM_CC_FD_CORE_CLK_SRC					33
-+#define CAM_CC_FD_CORE_UAR_CLK					34
-+#define CAM_CC_ICP_AHB_CLK					35
-+#define CAM_CC_ICP_CLK						36
-+#define CAM_CC_ICP_CLK_SRC					37
-+#define CAM_CC_IFE_0_AXI_CLK					38
-+#define CAM_CC_IFE_0_CLK					39
-+#define CAM_CC_IFE_0_CLK_SRC					40
-+#define CAM_CC_IFE_0_CPHY_RX_CLK				41
-+#define CAM_CC_IFE_0_CSID_CLK					42
-+#define CAM_CC_IFE_0_CSID_CLK_SRC				43
-+#define CAM_CC_IFE_0_DSP_CLK					44
-+#define CAM_CC_IFE_1_AXI_CLK					45
-+#define CAM_CC_IFE_1_CLK					46
-+#define CAM_CC_IFE_1_CLK_SRC					47
-+#define CAM_CC_IFE_1_CPHY_RX_CLK				48
-+#define CAM_CC_IFE_1_CSID_CLK					49
-+#define CAM_CC_IFE_1_CSID_CLK_SRC				50
-+#define CAM_CC_IFE_1_DSP_CLK					51
-+#define CAM_CC_IFE_2_AXI_CLK					52
-+#define CAM_CC_IFE_2_CLK					53
-+#define CAM_CC_IFE_2_CLK_SRC					54
-+#define CAM_CC_IFE_2_CPHY_RX_CLK				55
-+#define CAM_CC_IFE_2_CSID_CLK					56
-+#define CAM_CC_IFE_2_CSID_CLK_SRC				57
-+#define CAM_CC_IFE_2_DSP_CLK					58
-+#define CAM_CC_IFE_3_AXI_CLK					59
-+#define CAM_CC_IFE_3_CLK					60
-+#define CAM_CC_IFE_3_CLK_SRC					61
-+#define CAM_CC_IFE_3_CPHY_RX_CLK				62
-+#define CAM_CC_IFE_3_CSID_CLK					63
-+#define CAM_CC_IFE_3_CSID_CLK_SRC				64
-+#define CAM_CC_IFE_3_DSP_CLK					65
-+#define CAM_CC_IFE_LITE_0_CLK					66
-+#define CAM_CC_IFE_LITE_0_CLK_SRC				67
-+#define CAM_CC_IFE_LITE_0_CPHY_RX_CLK				68
-+#define CAM_CC_IFE_LITE_0_CSID_CLK				69
-+#define CAM_CC_IFE_LITE_0_CSID_CLK_SRC				70
-+#define CAM_CC_IFE_LITE_1_CLK					71
-+#define CAM_CC_IFE_LITE_1_CLK_SRC				72
-+#define CAM_CC_IFE_LITE_1_CPHY_RX_CLK				73
-+#define CAM_CC_IFE_LITE_1_CSID_CLK				74
-+#define CAM_CC_IFE_LITE_1_CSID_CLK_SRC				75
-+#define CAM_CC_IFE_LITE_2_CLK					76
-+#define CAM_CC_IFE_LITE_2_CLK_SRC				77
-+#define CAM_CC_IFE_LITE_2_CPHY_RX_CLK				78
-+#define CAM_CC_IFE_LITE_2_CSID_CLK				79
-+#define CAM_CC_IFE_LITE_2_CSID_CLK_SRC				80
-+#define CAM_CC_IFE_LITE_3_CLK					81
-+#define CAM_CC_IFE_LITE_3_CLK_SRC				82
-+#define CAM_CC_IFE_LITE_3_CPHY_RX_CLK				83
-+#define CAM_CC_IFE_LITE_3_CSID_CLK				84
-+#define CAM_CC_IFE_LITE_3_CSID_CLK_SRC				85
-+#define CAM_CC_IPE_0_AHB_CLK					86
-+#define CAM_CC_IPE_0_AREG_CLK					87
-+#define CAM_CC_IPE_0_AXI_CLK					88
-+#define CAM_CC_IPE_0_CLK					89
-+#define CAM_CC_IPE_0_CLK_SRC					90
-+#define CAM_CC_IPE_1_AHB_CLK					91
-+#define CAM_CC_IPE_1_AREG_CLK					92
-+#define CAM_CC_IPE_1_AXI_CLK					93
-+#define CAM_CC_IPE_1_CLK					94
-+#define CAM_CC_JPEG_CLK						95
-+#define CAM_CC_JPEG_CLK_SRC					96
-+#define CAM_CC_LRME_CLK						97
-+#define CAM_CC_LRME_CLK_SRC					98
-+#define CAM_CC_MCLK0_CLK					99
-+#define CAM_CC_MCLK0_CLK_SRC					100
-+#define CAM_CC_MCLK1_CLK					101
-+#define CAM_CC_MCLK1_CLK_SRC					102
-+#define CAM_CC_MCLK2_CLK					103
-+#define CAM_CC_MCLK2_CLK_SRC					104
-+#define CAM_CC_MCLK3_CLK					105
-+#define CAM_CC_MCLK3_CLK_SRC					106
-+#define CAM_CC_MCLK4_CLK					107
-+#define CAM_CC_MCLK4_CLK_SRC					108
-+#define CAM_CC_MCLK5_CLK					109
-+#define CAM_CC_MCLK5_CLK_SRC					110
-+#define CAM_CC_MCLK6_CLK					111
-+#define CAM_CC_MCLK6_CLK_SRC					112
-+#define CAM_CC_MCLK7_CLK					113
-+#define CAM_CC_MCLK7_CLK_SRC					114
-+#define CAM_CC_PLL0						115
-+#define CAM_CC_PLL0_OUT_EVEN					116
-+#define CAM_CC_PLL0_OUT_ODD					117
-+#define CAM_CC_PLL1						118
-+#define CAM_CC_PLL2						119
-+#define CAM_CC_PLL2_OUT_MAIN					120
-+#define CAM_CC_PLL3						121
-+#define CAM_CC_PLL4						122
-+#define CAM_CC_PLL5						123
-+#define CAM_CC_PLL6						124
-+#define CAM_CC_SLOW_AHB_CLK_SRC					125
-+#define CAM_CC_XO_CLK_SRC					126
-+
-+
-+/* CAM_CC power domains */
-+#define BPS_GDSC						0
-+#define IFE_0_GDSC						1
-+#define IFE_1_GDSC						2
-+#define IFE_2_GDSC						3
-+#define IFE_3_GDSC						4
-+#define IPE_0_GDSC						5
-+#define IPE_1_GDSC						6
-+#define TITAN_TOP_GDSC						7
-+
-+/* CAM_CC resets */
-+#define CAM_CC_BPS_BCR						0
-+#define CAM_CC_CAMNOC_BCR					1
-+#define CAM_CC_CCI_BCR						2
-+#define CAM_CC_CPAS_BCR						3
-+#define CAM_CC_CSI0PHY_BCR					4
-+#define CAM_CC_CSI1PHY_BCR					5
-+#define CAM_CC_CSI2PHY_BCR					6
-+#define CAM_CC_CSI3PHY_BCR					7
-+#define CAM_CC_FD_BCR						8
-+#define CAM_CC_ICP_BCR						9
-+#define CAM_CC_IFE_0_BCR					10
-+#define CAM_CC_IFE_1_BCR					11
-+#define CAM_CC_IFE_2_BCR					12
-+#define CAM_CC_IFE_3_BCR					13
-+#define CAM_CC_IFE_LITE_0_BCR					14
-+#define CAM_CC_IFE_LITE_1_BCR					15
-+#define CAM_CC_IFE_LITE_2_BCR					16
-+#define CAM_CC_IFE_LITE_3_BCR					17
-+#define CAM_CC_IPE_0_BCR					18
-+#define CAM_CC_IPE_1_BCR					19
-+#define CAM_CC_JPEG_BCR						20
-+#define CAM_CC_LRME_BCR						21
-+#define CAM_CC_MCLK0_BCR					22
-+#define CAM_CC_MCLK1_BCR					23
-+#define CAM_CC_MCLK2_BCR					24
-+#define CAM_CC_MCLK3_BCR					25
-+#define CAM_CC_MCLK4_BCR					26
-+#define CAM_CC_MCLK5_BCR					27
-+#define CAM_CC_MCLK6_BCR					28
-+#define CAM_CC_MCLK7_BCR					29
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sc8280xp-camcc.h b/include/dt-bindings/clock/qcom,sc8280xp-camcc.h
-new file mode 100644
-index 000000000000..ea5ec73c8c6a
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sc8280xp-camcc.h
-@@ -0,0 +1,179 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Linaro Ltd.
-+ */
-+
-+#ifndef __DT_BINDINGS_CLK_QCOM_CAMCC_SC8280XP_H__
-+#define __DT_BINDINGS_CLK_QCOM_CAMCC_SC8280XP_H__
-+
-+/* CAMCC clocks */
-+#define CAMCC_PLL0					0
-+#define CAMCC_PLL0_OUT_EVEN				1
-+#define CAMCC_PLL0_OUT_ODD				2
-+#define CAMCC_PLL1					3
-+#define CAMCC_PLL1_OUT_EVEN				4
-+#define CAMCC_PLL2					5
-+#define CAMCC_PLL3					6
-+#define CAMCC_PLL3_OUT_EVEN				7
-+#define CAMCC_PLL4					8
-+#define CAMCC_PLL4_OUT_EVEN				9
-+#define CAMCC_PLL5					10
-+#define CAMCC_PLL5_OUT_EVEN				11
-+#define CAMCC_PLL6					12
-+#define CAMCC_PLL6_OUT_EVEN				13
-+#define CAMCC_PLL7					14
-+#define CAMCC_PLL7_OUT_EVEN				15
-+#define CAMCC_PLL7_OUT_ODD				16
-+#define CAMCC_BPS_AHB_CLK				17
-+#define CAMCC_BPS_AREG_CLK				18
-+#define CAMCC_BPS_AXI_CLK				19
-+#define CAMCC_BPS_CLK					20
-+#define CAMCC_BPS_CLK_SRC				21
-+#define CAMCC_CAMNOC_AXI_CLK				22
-+#define CAMCC_CAMNOC_AXI_CLK_SRC			23
-+#define CAMCC_CAMNOC_DCD_XO_CLK				24
-+#define CAMCC_CCI_0_CLK					25
-+#define CAMCC_CCI_0_CLK_SRC				26
-+#define CAMCC_CCI_1_CLK					27
-+#define CAMCC_CCI_1_CLK_SRC				28
-+#define CAMCC_CCI_2_CLK					29
-+#define CAMCC_CCI_2_CLK_SRC				30
-+#define CAMCC_CCI_3_CLK					31
-+#define CAMCC_CCI_3_CLK_SRC				32
-+#define CAMCC_CORE_AHB_CLK				33
-+#define CAMCC_CPAS_AHB_CLK				34
-+#define CAMCC_CPHY_RX_CLK_SRC				35
-+#define CAMCC_CSI0PHYTIMER_CLK				36
-+#define CAMCC_CSI0PHYTIMER_CLK_SRC			37
-+#define CAMCC_CSI1PHYTIMER_CLK				38
-+#define CAMCC_CSI1PHYTIMER_CLK_SRC			39
-+#define CAMCC_CSI2PHYTIMER_CLK				40
-+#define CAMCC_CSI2PHYTIMER_CLK_SRC			41
-+#define CAMCC_CSI3PHYTIMER_CLK				42
-+#define CAMCC_CSI3PHYTIMER_CLK_SRC			43
-+#define CAMCC_CSIPHY0_CLK				44
-+#define CAMCC_CSIPHY1_CLK				45
-+#define CAMCC_CSIPHY2_CLK				46
-+#define CAMCC_CSIPHY3_CLK				47
-+#define CAMCC_FAST_AHB_CLK_SRC				48
-+#define CAMCC_GDSC_CLK					49
-+#define CAMCC_ICP_AHB_CLK				50
-+#define CAMCC_ICP_CLK					51
-+#define CAMCC_ICP_CLK_SRC				52
-+#define CAMCC_IFE_0_AXI_CLK				53
-+#define CAMCC_IFE_0_CLK					54
-+#define CAMCC_IFE_0_CLK_SRC				55
-+#define CAMCC_IFE_0_CPHY_RX_CLK				56
-+#define CAMCC_IFE_0_CSID_CLK				57
-+#define CAMCC_IFE_0_CSID_CLK_SRC			58
-+#define CAMCC_IFE_0_DSP_CLK				59
-+#define CAMCC_IFE_1_AXI_CLK				60
-+#define CAMCC_IFE_1_CLK					61
-+#define CAMCC_IFE_1_CLK_SRC				62
-+#define CAMCC_IFE_1_CPHY_RX_CLK				63
-+#define CAMCC_IFE_1_CSID_CLK				64
-+#define CAMCC_IFE_1_CSID_CLK_SRC			65
-+#define CAMCC_IFE_1_DSP_CLK				66
-+#define CAMCC_IFE_2_AXI_CLK				67
-+#define CAMCC_IFE_2_CLK					68
-+#define CAMCC_IFE_2_CLK_SRC				69
-+#define CAMCC_IFE_2_CPHY_RX_CLK				70
-+#define CAMCC_IFE_2_CSID_CLK				71
-+#define CAMCC_IFE_2_CSID_CLK_SRC			72
-+#define CAMCC_IFE_2_DSP_CLK				73
-+#define CAMCC_IFE_3_AXI_CLK				74
-+#define CAMCC_IFE_3_CLK					75
-+#define CAMCC_IFE_3_CLK_SRC				76
-+#define CAMCC_IFE_3_CPHY_RX_CLK				77
-+#define CAMCC_IFE_3_CSID_CLK				78
-+#define CAMCC_IFE_3_CSID_CLK_SRC			79
-+#define CAMCC_IFE_3_DSP_CLK				80
-+#define CAMCC_IFE_LITE_0_CLK				81
-+#define CAMCC_IFE_LITE_0_CLK_SRC			82
-+#define CAMCC_IFE_LITE_0_CPHY_RX_CLK			83
-+#define CAMCC_IFE_LITE_0_CSID_CLK			84
-+#define CAMCC_IFE_LITE_0_CSID_CLK_SRC			85
-+#define CAMCC_IFE_LITE_1_CLK				86
-+#define CAMCC_IFE_LITE_1_CLK_SRC			87
-+#define CAMCC_IFE_LITE_1_CPHY_RX_CLK			88
-+#define CAMCC_IFE_LITE_1_CSID_CLK			89
-+#define CAMCC_IFE_LITE_1_CSID_CLK_SRC			90
-+#define CAMCC_IFE_LITE_2_CLK				91
-+#define CAMCC_IFE_LITE_2_CLK_SRC			92
-+#define CAMCC_IFE_LITE_2_CPHY_RX_CLK			93
-+#define CAMCC_IFE_LITE_2_CSID_CLK			94
-+#define CAMCC_IFE_LITE_2_CSID_CLK_SRC			95
-+#define CAMCC_IFE_LITE_3_CLK				96
-+#define CAMCC_IFE_LITE_3_CLK_SRC			97
-+#define CAMCC_IFE_LITE_3_CPHY_RX_CLK			98
-+#define CAMCC_IFE_LITE_3_CSID_CLK			99
-+#define CAMCC_IFE_LITE_3_CSID_CLK_SRC			100
-+#define CAMCC_IPE_0_AHB_CLK				101
-+#define CAMCC_IPE_0_AREG_CLK				102
-+#define CAMCC_IPE_0_AXI_CLK				103
-+#define CAMCC_IPE_0_CLK					104
-+#define CAMCC_IPE_0_CLK_SRC				105
-+#define CAMCC_IPE_1_AHB_CLK				106
-+#define CAMCC_IPE_1_AREG_CLK				107
-+#define CAMCC_IPE_1_AXI_CLK				108
-+#define CAMCC_IPE_1_CLK					109
-+#define CAMCC_JPEG_CLK					110
-+#define CAMCC_JPEG_CLK_SRC				111
-+#define CAMCC_LRME_CLK					112
-+#define CAMCC_LRME_CLK_SRC				113
-+#define CAMCC_MCLK0_CLK					114
-+#define CAMCC_MCLK0_CLK_SRC				115
-+#define CAMCC_MCLK1_CLK					116
-+#define CAMCC_MCLK1_CLK_SRC				117
-+#define CAMCC_MCLK2_CLK					118
-+#define CAMCC_MCLK2_CLK_SRC				119
-+#define CAMCC_MCLK3_CLK					120
-+#define CAMCC_MCLK3_CLK_SRC				121
-+#define CAMCC_MCLK4_CLK					122
-+#define CAMCC_MCLK4_CLK_SRC				123
-+#define CAMCC_MCLK5_CLK					124
-+#define CAMCC_MCLK5_CLK_SRC				125
-+#define CAMCC_MCLK6_CLK					126
-+#define CAMCC_MCLK6_CLK_SRC				127
-+#define CAMCC_MCLK7_CLK					128
-+#define CAMCC_MCLK7_CLK_SRC				129
-+#define CAMCC_SLEEP_CLK					130
-+#define CAMCC_SLEEP_CLK_SRC				131
-+#define CAMCC_SLOW_AHB_CLK_SRC				132
-+#define CAMCC_XO_CLK_SRC				133
-+
-+/* CAMCC resets */
-+#define CAMCC_BPS_BCR					0
-+#define CAMCC_CAMNOC_BCR				1
-+#define CAMCC_CCI_BCR					2
-+#define CAMCC_CPAS_BCR					3
-+#define CAMCC_CSI0PHY_BCR				4
-+#define CAMCC_CSI1PHY_BCR				5
-+#define CAMCC_CSI2PHY_BCR				6
-+#define CAMCC_CSI3PHY_BCR				7
-+#define CAMCC_ICP_BCR					8
-+#define CAMCC_IFE_0_BCR					9
-+#define CAMCC_IFE_1_BCR					10
-+#define CAMCC_IFE_2_BCR					11
-+#define CAMCC_IFE_3_BCR					12
-+#define CAMCC_IFE_LITE_0_BCR				13
-+#define CAMCC_IFE_LITE_1_BCR				14
-+#define CAMCC_IFE_LITE_2_BCR				15
-+#define CAMCC_IFE_LITE_3_BCR				16
-+#define CAMCC_IPE_0_BCR					17
-+#define CAMCC_IPE_1_BCR					18
-+#define CAMCC_JPEG_BCR					19
-+#define CAMCC_LRME_BCR					20
-+
-+/* CAMCC GDSCRs */
-+#define BPS_GDSC					0
-+#define IFE_0_GDSC					1
-+#define IFE_1_GDSC					2
-+#define IFE_2_GDSC					3
-+#define IFE_3_GDSC					4
-+#define IPE_0_GDSC					5
-+#define IPE_1_GDSC					6
-+#define TITAN_TOP_GDSC					7
-+
-+#endif /* __DT_BINDINGS_CLK_QCOM_CAMCC_SC8280XP_H__ */
-diff --git a/include/dt-bindings/clock/qcom,sc8280xp-lpasscc.h b/include/dt-bindings/clock/qcom,sc8280xp-lpasscc.h
-new file mode 100644
-index 000000000000..d190d57fc81a
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sc8280xp-lpasscc.h
-@@ -0,0 +1,17 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Linaro Ltd.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_LPASSCC_SC8280XP_H
-+#define _DT_BINDINGS_CLK_QCOM_LPASSCC_SC8280XP_H
-+
-+/* LPASS AUDIO CC CSR */
-+#define LPASS_AUDIO_SWR_RX_CGCR				0
-+#define LPASS_AUDIO_SWR_WSA_CGCR			1
-+#define LPASS_AUDIO_SWR_WSA2_CGCR			2
-+
-+/* LPASS TCSR */
-+#define LPASS_AUDIO_SWR_TX_CGCR				0
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sdx75-gcc.h b/include/dt-bindings/clock/qcom,sdx75-gcc.h
-new file mode 100644
-index 000000000000..a470e8c4fd41
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sdx75-gcc.h
-@@ -0,0 +1,193 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_SDX75_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_SDX75_H
-+
-+/* GCC clocks */
-+#define GPLL0							0
-+#define GPLL0_OUT_EVEN						1
-+#define GPLL4							2
-+#define GPLL5							3
-+#define GPLL6							4
-+#define GPLL8							5
-+#define GCC_AHB_PCIE_LINK_CLK					6
-+#define GCC_BOOT_ROM_AHB_CLK					7
-+#define GCC_EEE_EMAC0_CLK					8
-+#define GCC_EEE_EMAC0_CLK_SRC					9
-+#define GCC_EEE_EMAC1_CLK					10
-+#define GCC_EEE_EMAC1_CLK_SRC					11
-+#define GCC_EMAC0_AXI_CLK					12
-+#define GCC_EMAC0_CC_SGMIIPHY_RX_CLK				13
-+#define GCC_EMAC0_CC_SGMIIPHY_RX_CLK_SRC			14
-+#define GCC_EMAC0_CC_SGMIIPHY_TX_CLK				15
-+#define GCC_EMAC0_CC_SGMIIPHY_TX_CLK_SRC			16
-+#define GCC_EMAC0_PHY_AUX_CLK					17
-+#define GCC_EMAC0_PHY_AUX_CLK_SRC				18
-+#define GCC_EMAC0_PTP_CLK					19
-+#define GCC_EMAC0_PTP_CLK_SRC					20
-+#define GCC_EMAC0_RGMII_CLK					21
-+#define GCC_EMAC0_RGMII_CLK_SRC					22
-+#define GCC_EMAC0_RPCS_RX_CLK					23
-+#define GCC_EMAC0_RPCS_TX_CLK					24
-+#define GCC_EMAC0_SGMIIPHY_MAC_RCLK_SRC				25
-+#define GCC_EMAC0_SGMIIPHY_MAC_TCLK_SRC				26
-+#define GCC_EMAC0_SLV_AHB_CLK					27
-+#define GCC_EMAC0_XGXS_RX_CLK					28
-+#define GCC_EMAC0_XGXS_TX_CLK					29
-+#define GCC_EMAC1_AXI_CLK					30
-+#define GCC_EMAC1_CC_SGMIIPHY_RX_CLK				31
-+#define GCC_EMAC1_CC_SGMIIPHY_RX_CLK_SRC			32
-+#define GCC_EMAC1_CC_SGMIIPHY_TX_CLK				33
-+#define GCC_EMAC1_CC_SGMIIPHY_TX_CLK_SRC			34
-+#define GCC_EMAC1_PHY_AUX_CLK					35
-+#define GCC_EMAC1_PHY_AUX_CLK_SRC				36
-+#define GCC_EMAC1_PTP_CLK					37
-+#define GCC_EMAC1_PTP_CLK_SRC					38
-+#define GCC_EMAC1_RGMII_CLK					39
-+#define GCC_EMAC1_RGMII_CLK_SRC					40
-+#define GCC_EMAC1_RPCS_RX_CLK					41
-+#define GCC_EMAC1_RPCS_TX_CLK					42
-+#define GCC_EMAC1_SGMIIPHY_MAC_RCLK_SRC				43
-+#define GCC_EMAC1_SGMIIPHY_MAC_TCLK_SRC				44
-+#define GCC_EMAC1_SLV_AHB_CLK					45
-+#define GCC_EMAC1_XGXS_RX_CLK					46
-+#define GCC_EMAC1_XGXS_TX_CLK					47
-+#define GCC_EMAC_0_CLKREF_EN					48
-+#define GCC_EMAC_1_CLKREF_EN					49
-+#define GCC_GP1_CLK						50
-+#define GCC_GP1_CLK_SRC						51
-+#define GCC_GP2_CLK						52
-+#define GCC_GP2_CLK_SRC						53
-+#define GCC_GP3_CLK						54
-+#define GCC_GP3_CLK_SRC						55
-+#define GCC_PCIE_0_CLKREF_EN					56
-+#define GCC_PCIE_1_AUX_CLK					57
-+#define GCC_PCIE_1_AUX_PHY_CLK_SRC				58
-+#define GCC_PCIE_1_CFG_AHB_CLK					59
-+#define GCC_PCIE_1_CLKREF_EN					60
-+#define GCC_PCIE_1_MSTR_AXI_CLK					61
-+#define GCC_PCIE_1_PHY_RCHNG_CLK				62
-+#define GCC_PCIE_1_PHY_RCHNG_CLK_SRC				63
-+#define GCC_PCIE_1_PIPE_CLK					64
-+#define GCC_PCIE_1_PIPE_CLK_SRC					65
-+#define GCC_PCIE_1_PIPE_DIV2_CLK				66
-+#define GCC_PCIE_1_PIPE_DIV2_CLK_SRC				67
-+#define GCC_PCIE_1_SLV_AXI_CLK					68
-+#define GCC_PCIE_1_SLV_Q2A_AXI_CLK				69
-+#define GCC_PCIE_2_AUX_CLK					70
-+#define GCC_PCIE_2_AUX_PHY_CLK_SRC				71
-+#define GCC_PCIE_2_CFG_AHB_CLK					72
-+#define GCC_PCIE_2_CLKREF_EN					73
-+#define GCC_PCIE_2_MSTR_AXI_CLK					74
-+#define GCC_PCIE_2_PHY_RCHNG_CLK				75
-+#define GCC_PCIE_2_PHY_RCHNG_CLK_SRC				76
-+#define GCC_PCIE_2_PIPE_CLK					77
-+#define GCC_PCIE_2_PIPE_CLK_SRC					78
-+#define GCC_PCIE_2_PIPE_DIV2_CLK				79
-+#define GCC_PCIE_2_PIPE_DIV2_CLK_SRC				80
-+#define GCC_PCIE_2_SLV_AXI_CLK					81
-+#define GCC_PCIE_2_SLV_Q2A_AXI_CLK				82
-+#define GCC_PCIE_AUX_CLK					83
-+#define GCC_PCIE_AUX_CLK_SRC					84
-+#define GCC_PCIE_AUX_PHY_CLK_SRC				85
-+#define GCC_PCIE_CFG_AHB_CLK					86
-+#define GCC_PCIE_MSTR_AXI_CLK					87
-+#define GCC_PCIE_PIPE_CLK					88
-+#define GCC_PCIE_PIPE_CLK_SRC					89
-+#define GCC_PCIE_RCHNG_PHY_CLK					90
-+#define GCC_PCIE_RCHNG_PHY_CLK_SRC				91
-+#define GCC_PCIE_SLEEP_CLK					92
-+#define GCC_PCIE_SLV_AXI_CLK					93
-+#define GCC_PCIE_SLV_Q2A_AXI_CLK				94
-+#define GCC_PDM2_CLK						95
-+#define GCC_PDM2_CLK_SRC					96
-+#define GCC_PDM_AHB_CLK						97
-+#define GCC_PDM_XO4_CLK						98
-+#define GCC_QUPV3_WRAP0_CORE_2X_CLK				99
-+#define GCC_QUPV3_WRAP0_CORE_CLK				100
-+#define GCC_QUPV3_WRAP0_S0_CLK					101
-+#define GCC_QUPV3_WRAP0_S0_CLK_SRC				102
-+#define GCC_QUPV3_WRAP0_S1_CLK					103
-+#define GCC_QUPV3_WRAP0_S1_CLK_SRC				104
-+#define GCC_QUPV3_WRAP0_S2_CLK					105
-+#define GCC_QUPV3_WRAP0_S2_CLK_SRC				106
-+#define GCC_QUPV3_WRAP0_S3_CLK					107
-+#define GCC_QUPV3_WRAP0_S3_CLK_SRC				108
-+#define GCC_QUPV3_WRAP0_S4_CLK					109
-+#define GCC_QUPV3_WRAP0_S4_CLK_SRC				110
-+#define GCC_QUPV3_WRAP0_S5_CLK					111
-+#define GCC_QUPV3_WRAP0_S5_CLK_SRC				112
-+#define GCC_QUPV3_WRAP0_S6_CLK					113
-+#define GCC_QUPV3_WRAP0_S6_CLK_SRC				114
-+#define GCC_QUPV3_WRAP0_S7_CLK					115
-+#define GCC_QUPV3_WRAP0_S7_CLK_SRC				116
-+#define GCC_QUPV3_WRAP0_S8_CLK					117
-+#define GCC_QUPV3_WRAP0_S8_CLK_SRC				118
-+#define GCC_QUPV3_WRAP_0_M_AHB_CLK				119
-+#define GCC_QUPV3_WRAP_0_S_AHB_CLK				120
-+#define GCC_SDCC1_AHB_CLK					121
-+#define GCC_SDCC1_APPS_CLK					122
-+#define GCC_SDCC1_APPS_CLK_SRC					123
-+#define GCC_SDCC2_AHB_CLK					124
-+#define GCC_SDCC2_APPS_CLK					125
-+#define GCC_SDCC2_APPS_CLK_SRC					126
-+#define GCC_USB2_CLKREF_EN					127
-+#define GCC_USB30_MASTER_CLK					128
-+#define GCC_USB30_MASTER_CLK_SRC				129
-+#define GCC_USB30_MOCK_UTMI_CLK					130
-+#define GCC_USB30_MOCK_UTMI_CLK_SRC				131
-+#define GCC_USB30_MOCK_UTMI_POSTDIV_CLK_SRC			132
-+#define GCC_USB30_MSTR_AXI_CLK					133
-+#define GCC_USB30_SLEEP_CLK					134
-+#define GCC_USB30_SLV_AHB_CLK					135
-+#define GCC_USB3_PHY_AUX_CLK					136
-+#define GCC_USB3_PHY_AUX_CLK_SRC				137
-+#define GCC_USB3_PHY_PIPE_CLK					138
-+#define GCC_USB3_PHY_PIPE_CLK_SRC				139
-+#define GCC_USB3_PRIM_CLKREF_EN					140
-+#define GCC_USB_PHY_CFG_AHB2PHY_CLK				141
-+#define GCC_XO_PCIE_LINK_CLK					142
-+
-+/* GCC power domains */
-+#define GCC_EMAC0_GDSC						0
-+#define GCC_EMAC1_GDSC						1
-+#define GCC_PCIE_1_GDSC						2
-+#define GCC_PCIE_1_PHY_GDSC					3
-+#define GCC_PCIE_2_GDSC						4
-+#define GCC_PCIE_2_PHY_GDSC					5
-+#define GCC_PCIE_GDSC						6
-+#define GCC_PCIE_PHY_GDSC					7
-+#define GCC_USB30_GDSC						8
-+#define GCC_USB3_PHY_GDSC					9
-+
-+/* GCC resets */
-+#define GCC_EMAC0_BCR						0
-+#define GCC_EMAC1_BCR						1
-+#define GCC_EMMC_BCR						2
-+#define GCC_PCIE_1_BCR						3
-+#define GCC_PCIE_1_LINK_DOWN_BCR				4
-+#define GCC_PCIE_1_NOCSR_COM_PHY_BCR				5
-+#define GCC_PCIE_1_PHY_BCR					6
-+#define GCC_PCIE_2_BCR						7
-+#define GCC_PCIE_2_LINK_DOWN_BCR				8
-+#define GCC_PCIE_2_NOCSR_COM_PHY_BCR				9
-+#define GCC_PCIE_2_PHY_BCR					10
-+#define GCC_PCIE_BCR						11
-+#define GCC_PCIE_LINK_DOWN_BCR					12
-+#define GCC_PCIE_NOCSR_COM_PHY_BCR				13
-+#define GCC_PCIE_PHY_BCR					14
-+#define GCC_PCIE_PHY_CFG_AHB_BCR				15
-+#define GCC_PCIE_PHY_COM_BCR					16
-+#define GCC_PCIE_PHY_NOCSR_COM_PHY_BCR				17
-+#define GCC_QUSB2PHY_BCR					18
-+#define GCC_TCSR_PCIE_BCR					19
-+#define GCC_USB30_BCR						20
-+#define GCC_USB3_PHY_BCR					21
-+#define GCC_USB3PHY_PHY_BCR					22
-+#define GCC_USB_PHY_CFG_AHB2PHY_BCR				23
-+#define GCC_EMAC0_RGMII_CLK_ARES				24
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm4450-camcc.h b/include/dt-bindings/clock/qcom,sm4450-camcc.h
-new file mode 100644
-index 000000000000..bf077951bf1c
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm4450-camcc.h
-@@ -0,0 +1,106 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_SM4450_H
-+#define _DT_BINDINGS_CLK_QCOM_CAM_CC_SM4450_H
-+
-+/* CAM_CC clocks */
-+#define CAM_CC_BPS_AHB_CLK					0
-+#define CAM_CC_BPS_AREG_CLK					1
-+#define CAM_CC_BPS_CLK						2
-+#define CAM_CC_BPS_CLK_SRC					3
-+#define CAM_CC_CAMNOC_ATB_CLK					4
-+#define CAM_CC_CAMNOC_AXI_CLK					5
-+#define CAM_CC_CAMNOC_AXI_CLK_SRC				6
-+#define CAM_CC_CAMNOC_AXI_HF_CLK				7
-+#define CAM_CC_CAMNOC_AXI_SF_CLK				8
-+#define CAM_CC_CCI_0_CLK					9
-+#define CAM_CC_CCI_0_CLK_SRC					10
-+#define CAM_CC_CCI_1_CLK					11
-+#define CAM_CC_CCI_1_CLK_SRC					12
-+#define CAM_CC_CORE_AHB_CLK					13
-+#define CAM_CC_CPAS_AHB_CLK					14
-+#define CAM_CC_CPHY_RX_CLK_SRC					15
-+#define CAM_CC_CRE_AHB_CLK					16
-+#define CAM_CC_CRE_CLK						17
-+#define CAM_CC_CRE_CLK_SRC					18
-+#define CAM_CC_CSI0PHYTIMER_CLK					19
-+#define CAM_CC_CSI0PHYTIMER_CLK_SRC				20
-+#define CAM_CC_CSI1PHYTIMER_CLK					21
-+#define CAM_CC_CSI1PHYTIMER_CLK_SRC				22
-+#define CAM_CC_CSI2PHYTIMER_CLK					23
-+#define CAM_CC_CSI2PHYTIMER_CLK_SRC				24
-+#define CAM_CC_CSIPHY0_CLK					25
-+#define CAM_CC_CSIPHY1_CLK					26
-+#define CAM_CC_CSIPHY2_CLK					27
-+#define CAM_CC_FAST_AHB_CLK_SRC					28
-+#define CAM_CC_ICP_ATB_CLK					29
-+#define CAM_CC_ICP_CLK						30
-+#define CAM_CC_ICP_CLK_SRC					31
-+#define CAM_CC_ICP_CTI_CLK					32
-+#define CAM_CC_ICP_TS_CLK					33
-+#define CAM_CC_MCLK0_CLK					34
-+#define CAM_CC_MCLK0_CLK_SRC					35
-+#define CAM_CC_MCLK1_CLK					36
-+#define CAM_CC_MCLK1_CLK_SRC					37
-+#define CAM_CC_MCLK2_CLK					38
-+#define CAM_CC_MCLK2_CLK_SRC					39
-+#define CAM_CC_MCLK3_CLK					40
-+#define CAM_CC_MCLK3_CLK_SRC					41
-+#define CAM_CC_OPE_0_AHB_CLK					42
-+#define CAM_CC_OPE_0_AREG_CLK					43
-+#define CAM_CC_OPE_0_CLK					44
-+#define CAM_CC_OPE_0_CLK_SRC					45
-+#define CAM_CC_PLL0						46
-+#define CAM_CC_PLL0_OUT_EVEN					47
-+#define CAM_CC_PLL0_OUT_ODD					48
-+#define CAM_CC_PLL1						49
-+#define CAM_CC_PLL1_OUT_EVEN					50
-+#define CAM_CC_PLL2						51
-+#define CAM_CC_PLL2_OUT_EVEN					52
-+#define CAM_CC_PLL3						53
-+#define CAM_CC_PLL3_OUT_EVEN					54
-+#define CAM_CC_PLL4						55
-+#define CAM_CC_PLL4_OUT_EVEN					56
-+#define CAM_CC_SLOW_AHB_CLK_SRC					57
-+#define CAM_CC_SOC_AHB_CLK					58
-+#define CAM_CC_SYS_TMR_CLK					59
-+#define CAM_CC_TFE_0_AHB_CLK					60
-+#define CAM_CC_TFE_0_CLK					61
-+#define CAM_CC_TFE_0_CLK_SRC					62
-+#define CAM_CC_TFE_0_CPHY_RX_CLK				63
-+#define CAM_CC_TFE_0_CSID_CLK					64
-+#define CAM_CC_TFE_0_CSID_CLK_SRC				65
-+#define CAM_CC_TFE_1_AHB_CLK					66
-+#define CAM_CC_TFE_1_CLK					67
-+#define CAM_CC_TFE_1_CLK_SRC					68
-+#define CAM_CC_TFE_1_CPHY_RX_CLK				69
-+#define CAM_CC_TFE_1_CSID_CLK					70
-+#define CAM_CC_TFE_1_CSID_CLK_SRC				71
-+
-+/* CAM_CC power domains */
-+#define CAM_CC_CAMSS_TOP_GDSC					0
-+
-+/* CAM_CC resets */
-+#define CAM_CC_BPS_BCR						0
-+#define CAM_CC_CAMNOC_BCR					1
-+#define CAM_CC_CAMSS_TOP_BCR					2
-+#define CAM_CC_CCI_0_BCR					3
-+#define CAM_CC_CCI_1_BCR					4
-+#define CAM_CC_CPAS_BCR						5
-+#define CAM_CC_CRE_BCR						6
-+#define CAM_CC_CSI0PHY_BCR					7
-+#define CAM_CC_CSI1PHY_BCR					8
-+#define CAM_CC_CSI2PHY_BCR					9
-+#define CAM_CC_ICP_BCR						10
-+#define CAM_CC_MCLK0_BCR					11
-+#define CAM_CC_MCLK1_BCR					12
-+#define CAM_CC_MCLK2_BCR					13
-+#define CAM_CC_MCLK3_BCR					14
-+#define CAM_CC_OPE_0_BCR					15
-+#define CAM_CC_TFE_0_BCR					16
-+#define CAM_CC_TFE_1_BCR					17
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm4450-dispcc.h b/include/dt-bindings/clock/qcom,sm4450-dispcc.h
-new file mode 100644
-index 000000000000..ca6f2ef90157
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm4450-dispcc.h
-@@ -0,0 +1,51 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_SM4450_H
-+#define _DT_BINDINGS_CLK_QCOM_DISP_CC_SM4450_H
-+
-+/* DISP_CC clocks */
-+#define DISP_CC_MDSS_AHB1_CLK					0
-+#define DISP_CC_MDSS_AHB_CLK					1
-+#define DISP_CC_MDSS_AHB_CLK_SRC				2
-+#define DISP_CC_MDSS_BYTE0_CLK					3
-+#define DISP_CC_MDSS_BYTE0_CLK_SRC				4
-+#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC				5
-+#define DISP_CC_MDSS_BYTE0_INTF_CLK				6
-+#define DISP_CC_MDSS_ESC0_CLK					7
-+#define DISP_CC_MDSS_ESC0_CLK_SRC				8
-+#define DISP_CC_MDSS_MDP1_CLK					9
-+#define DISP_CC_MDSS_MDP_CLK					10
-+#define DISP_CC_MDSS_MDP_CLK_SRC				11
-+#define DISP_CC_MDSS_MDP_LUT1_CLK				12
-+#define DISP_CC_MDSS_MDP_LUT_CLK				13
-+#define DISP_CC_MDSS_NON_GDSC_AHB_CLK				14
-+#define DISP_CC_MDSS_PCLK0_CLK					15
-+#define DISP_CC_MDSS_PCLK0_CLK_SRC				16
-+#define DISP_CC_MDSS_ROT1_CLK					17
-+#define DISP_CC_MDSS_ROT_CLK					18
-+#define DISP_CC_MDSS_ROT_CLK_SRC				19
-+#define DISP_CC_MDSS_RSCC_AHB_CLK				20
-+#define DISP_CC_MDSS_RSCC_VSYNC_CLK				21
-+#define DISP_CC_MDSS_VSYNC1_CLK					22
-+#define DISP_CC_MDSS_VSYNC_CLK					23
-+#define DISP_CC_MDSS_VSYNC_CLK_SRC				24
-+#define DISP_CC_PLL0						25
-+#define DISP_CC_PLL1						26
-+#define DISP_CC_SLEEP_CLK					27
-+#define DISP_CC_SLEEP_CLK_SRC					28
-+#define DISP_CC_XO_CLK						29
-+#define DISP_CC_XO_CLK_SRC					30
-+
-+/* DISP_CC power domains */
-+#define DISP_CC_MDSS_CORE_GDSC					0
-+#define DISP_CC_MDSS_CORE_INT2_GDSC				1
-+
-+/* DISP_CC resets */
-+#define DISP_CC_MDSS_CORE_BCR					0
-+#define DISP_CC_MDSS_CORE_INT2_BCR				1
-+#define DISP_CC_MDSS_RSCC_BCR					2
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm4450-gcc.h b/include/dt-bindings/clock/qcom,sm4450-gcc.h
-new file mode 100644
-index 000000000000..c18e47a86f40
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm4450-gcc.h
-@@ -0,0 +1,197 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_SM4450_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_SM4450_H
-+
-+/* GCC clocks */
-+#define GCC_AGGRE_NOC_PCIE_0_AXI_CLK				0
-+#define GCC_AGGRE_UFS_PHY_AXI_CLK				1
-+#define GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK			2
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK				3
-+#define GCC_BOOT_ROM_AHB_CLK					4
-+#define GCC_CAMERA_AHB_CLK					5
-+#define GCC_CAMERA_HF_AXI_CLK					6
-+#define GCC_CAMERA_SF_AXI_CLK					7
-+#define GCC_CAMERA_SLEEP_CLK					8
-+#define GCC_CAMERA_XO_CLK					9
-+#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK				10
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				11
-+#define GCC_DDRSS_GPU_AXI_CLK					12
-+#define GCC_DDRSS_PCIE_SF_TBU_CLK				13
-+#define GCC_DISP_AHB_CLK					14
-+#define GCC_DISP_HF_AXI_CLK					15
-+#define GCC_DISP_XO_CLK						16
-+#define GCC_EUSB3_0_CLKREF_EN					17
-+#define GCC_GP1_CLK						18
-+#define GCC_GP1_CLK_SRC						19
-+#define GCC_GP2_CLK						20
-+#define GCC_GP2_CLK_SRC						21
-+#define GCC_GP3_CLK						22
-+#define GCC_GP3_CLK_SRC						23
-+#define GCC_GPLL0						24
-+#define GCC_GPLL0_OUT_EVEN					25
-+#define GCC_GPLL0_OUT_ODD					26
-+#define GCC_GPLL1						27
-+#define GCC_GPLL3						28
-+#define GCC_GPLL4						29
-+#define GCC_GPLL9						30
-+#define GCC_GPLL10						31
-+#define GCC_GPU_CFG_AHB_CLK					32
-+#define GCC_GPU_GPLL0_CLK_SRC					33
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC				34
-+#define GCC_GPU_MEMNOC_GFX_CLK					35
-+#define GCC_GPU_SNOC_DVM_GFX_CLK				36
-+#define GCC_HLOS1_VOTE_AGGRE_NOC_MMU_AUDIO_TBU_CLK		37
-+#define GCC_HLOS1_VOTE_AGGRE_NOC_MMU_PCIE_TBU_CLK		38
-+#define GCC_HLOS1_VOTE_AGGRE_NOC_MMU_TBU1_CLK			39
-+#define GCC_HLOS1_VOTE_AGGRE_NOC_MMU_TBU2_CLK			40
-+#define GCC_HLOS1_VOTE_MMNOC_MMU_TBU_HF0_CLK			41
-+#define GCC_HLOS1_VOTE_MMNOC_MMU_TBU_HF1_CLK			42
-+#define GCC_HLOS1_VOTE_MMNOC_MMU_TBU_SF0_CLK			43
-+#define GCC_HLOS1_VOTE_MMU_TCU_CLK				44
-+#define GCC_PCIE_0_AUX_CLK					45
-+#define GCC_PCIE_0_AUX_CLK_SRC					46
-+#define GCC_PCIE_0_CFG_AHB_CLK					47
-+#define GCC_PCIE_0_CLKREF_EN					48
-+#define GCC_PCIE_0_MSTR_AXI_CLK					49
-+#define GCC_PCIE_0_PHY_RCHNG_CLK				50
-+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC				51
-+#define GCC_PCIE_0_PIPE_CLK					52
-+#define GCC_PCIE_0_PIPE_CLK_SRC					53
-+#define GCC_PCIE_0_PIPE_DIV2_CLK				54
-+#define GCC_PCIE_0_PIPE_DIV2_CLK_SRC				55
-+#define GCC_PCIE_0_SLV_AXI_CLK					56
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				57
-+#define GCC_PDM2_CLK						58
-+#define GCC_PDM2_CLK_SRC					59
-+#define GCC_PDM_AHB_CLK						60
-+#define GCC_PDM_XO4_CLK						61
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				62
-+#define GCC_QMIP_CAMERA_RT_AHB_CLK				63
-+#define GCC_QMIP_DISP_AHB_CLK					64
-+#define GCC_QMIP_GPU_AHB_CLK					65
-+#define GCC_QMIP_PCIE_AHB_CLK					66
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				67
-+#define GCC_QUPV3_WRAP0_CORE_2X_CLK				68
-+#define GCC_QUPV3_WRAP0_CORE_CLK				69
-+#define GCC_QUPV3_WRAP0_S0_CLK					70
-+#define GCC_QUPV3_WRAP0_S0_CLK_SRC				71
-+#define GCC_QUPV3_WRAP0_S1_CLK					72
-+#define GCC_QUPV3_WRAP0_S1_CLK_SRC				73
-+#define GCC_QUPV3_WRAP0_S2_CLK					74
-+#define GCC_QUPV3_WRAP0_S2_CLK_SRC				75
-+#define GCC_QUPV3_WRAP0_S3_CLK					76
-+#define GCC_QUPV3_WRAP0_S3_CLK_SRC				77
-+#define GCC_QUPV3_WRAP0_S4_CLK					78
-+#define GCC_QUPV3_WRAP0_S4_CLK_SRC				79
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				80
-+#define GCC_QUPV3_WRAP1_CORE_CLK				81
-+#define GCC_QUPV3_WRAP1_S0_CLK					82
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				83
-+#define GCC_QUPV3_WRAP1_S1_CLK					84
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				85
-+#define GCC_QUPV3_WRAP1_S2_CLK					86
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				87
-+#define GCC_QUPV3_WRAP1_S3_CLK					88
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				89
-+#define GCC_QUPV3_WRAP1_S4_CLK					90
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				91
-+#define GCC_QUPV3_WRAP_0_M_AHB_CLK				92
-+#define GCC_QUPV3_WRAP_0_S_AHB_CLK				93
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK				94
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				95
-+#define GCC_SDCC1_AHB_CLK					96
-+#define GCC_SDCC1_APPS_CLK					97
-+#define GCC_SDCC1_APPS_CLK_SRC					98
-+#define GCC_SDCC1_ICE_CORE_CLK					99
-+#define GCC_SDCC1_ICE_CORE_CLK_SRC				100
-+#define GCC_SDCC2_AHB_CLK					101
-+#define GCC_SDCC2_APPS_CLK					102
-+#define GCC_SDCC2_APPS_CLK_SRC					103
-+#define GCC_UFS_0_CLKREF_EN					104
-+#define GCC_UFS_PAD_CLKREF_EN					105
-+#define GCC_UFS_PHY_AHB_CLK					106
-+#define GCC_UFS_PHY_AXI_CLK					107
-+#define GCC_UFS_PHY_AXI_CLK_SRC					108
-+#define GCC_UFS_PHY_AXI_HW_CTL_CLK				109
-+#define GCC_UFS_PHY_ICE_CORE_CLK				110
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC				111
-+#define GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK				112
-+#define GCC_UFS_PHY_PHY_AUX_CLK					113
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC				114
-+#define GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK				115
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK				116
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC				117
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK				118
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC				119
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK				120
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC				121
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK				122
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC				123
-+#define GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK			124
-+#define GCC_USB30_PRIM_MASTER_CLK				125
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC				126
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				127
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			128
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		129
-+#define GCC_USB30_PRIM_SLEEP_CLK				130
-+#define GCC_USB3_0_CLKREF_EN					131
-+#define GCC_USB3_PRIM_PHY_AUX_CLK				132
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				133
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				134
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK				135
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				136
-+#define GCC_VCODEC0_AXI_CLK					137
-+#define GCC_VENUS_CTL_AXI_CLK					138
-+#define GCC_VIDEO_AHB_CLK					139
-+#define GCC_VIDEO_THROTTLE_CORE_CLK				140
-+#define GCC_VIDEO_VCODEC0_SYS_CLK				141
-+#define GCC_VIDEO_VENUS_CLK_SRC					142
-+#define GCC_VIDEO_VENUS_CTL_CLK					143
-+#define GCC_VIDEO_XO_CLK					144
-+
-+/* GCC power domains */
-+#define GCC_PCIE_0_GDSC						0
-+#define GCC_UFS_PHY_GDSC					1
-+#define GCC_USB30_PRIM_GDSC					2
-+#define GCC_VCODEC0_GDSC					3
-+#define GCC_VENUS_GDSC						4
-+
-+/* GCC resets */
-+#define GCC_CAMERA_BCR						0
-+#define GCC_DISPLAY_BCR						1
-+#define GCC_GPU_BCR						2
-+#define GCC_PCIE_0_BCR						3
-+#define GCC_PCIE_0_LINK_DOWN_BCR				4
-+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR				5
-+#define GCC_PCIE_0_PHY_BCR					6
-+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR			7
-+#define GCC_PCIE_PHY_BCR					8
-+#define GCC_PCIE_PHY_CFG_AHB_BCR				9
-+#define GCC_PCIE_PHY_COM_BCR					10
-+#define GCC_PDM_BCR						11
-+#define GCC_QUPV3_WRAPPER_0_BCR					12
-+#define GCC_QUPV3_WRAPPER_1_BCR					13
-+#define GCC_QUSB2PHY_PRIM_BCR					14
-+#define GCC_QUSB2PHY_SEC_BCR					15
-+#define GCC_SDCC1_BCR						16
-+#define GCC_SDCC2_BCR						17
-+#define GCC_UFS_PHY_BCR						18
-+#define GCC_USB30_PRIM_BCR					19
-+#define GCC_USB3_DP_PHY_PRIM_BCR				20
-+#define GCC_USB3_DP_PHY_SEC_BCR					21
-+#define GCC_USB3_PHY_PRIM_BCR					22
-+#define GCC_USB3_PHY_SEC_BCR					23
-+#define GCC_USB3PHY_PHY_PRIM_BCR				24
-+#define GCC_USB3PHY_PHY_SEC_BCR					25
-+#define GCC_VCODEC0_BCR						26
-+#define GCC_VENUS_BCR						27
-+#define GCC_VIDEO_BCR						28
-+#define GCC_VIDEO_VENUS_BCR					29
-+#define GCC_VENUS_CTL_AXI_CLK_ARES				30
-+#define GCC_VIDEO_VENUS_CTL_CLK_ARES				31
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm4450-gpucc.h b/include/dt-bindings/clock/qcom,sm4450-gpucc.h
-new file mode 100644
-index 000000000000..304f83e5f645
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm4450-gpucc.h
-@@ -0,0 +1,62 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_SM4450_H
-+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_SM4450_H
-+
-+/* GPU_CC clocks */
-+#define GPU_CC_AHB_CLK						0
-+#define GPU_CC_CB_CLK						1
-+#define GPU_CC_CRC_AHB_CLK					2
-+#define GPU_CC_CX_FF_CLK					3
-+#define GPU_CC_CX_GFX3D_CLK					4
-+#define GPU_CC_CX_GFX3D_SLV_CLK					5
-+#define GPU_CC_CX_GMU_CLK					6
-+#define GPU_CC_CX_SNOC_DVM_CLK					7
-+#define GPU_CC_CXO_AON_CLK					8
-+#define GPU_CC_CXO_CLK						9
-+#define GPU_CC_DEMET_CLK					10
-+#define GPU_CC_DEMET_DIV_CLK_SRC				11
-+#define GPU_CC_FF_CLK_SRC					12
-+#define GPU_CC_FREQ_MEASURE_CLK					13
-+#define GPU_CC_GMU_CLK_SRC					14
-+#define GPU_CC_GX_CXO_CLK					15
-+#define GPU_CC_GX_FF_CLK					16
-+#define GPU_CC_GX_GFX3D_CLK					17
-+#define GPU_CC_GX_GFX3D_CLK_SRC					18
-+#define GPU_CC_GX_GFX3D_RDVM_CLK				19
-+#define GPU_CC_GX_GMU_CLK					20
-+#define GPU_CC_GX_VSENSE_CLK					21
-+#define GPU_CC_HUB_AHB_DIV_CLK_SRC				22
-+#define GPU_CC_HUB_AON_CLK					23
-+#define GPU_CC_HUB_CLK_SRC					24
-+#define GPU_CC_HUB_CX_INT_CLK					25
-+#define GPU_CC_HUB_CX_INT_DIV_CLK_SRC				26
-+#define GPU_CC_MEMNOC_GFX_CLK					27
-+#define GPU_CC_MND1X_0_GFX3D_CLK				28
-+#define GPU_CC_PLL0						29
-+#define GPU_CC_PLL1						30
-+#define GPU_CC_SLEEP_CLK					31
-+#define GPU_CC_XO_CLK_SRC					32
-+#define GPU_CC_XO_DIV_CLK_SRC					33
-+
-+/* GPU_CC power domains */
-+#define GPU_CC_CX_GDSC						0
-+#define GPU_CC_GX_GDSC						1
-+
-+/* GPU_CC resets */
-+#define GPU_CC_ACD_BCR						0
-+#define GPU_CC_CB_BCR						1
-+#define GPU_CC_CX_BCR						2
-+#define GPU_CC_FAST_HUB_BCR					3
-+#define GPU_CC_FF_BCR						4
-+#define GPU_CC_GFX3D_AON_BCR					5
-+#define GPU_CC_GMU_BCR						6
-+#define GPU_CC_GX_BCR						7
-+#define GPU_CC_XO_BCR						8
-+#define GPU_CC_GX_ACD_IROOT_BCR					9
-+#define GPU_CC_RBCPR_BCR					10
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm6115-dispcc.h b/include/dt-bindings/clock/qcom,sm6115-dispcc.h
-new file mode 100644
-index 000000000000..d1a6c45b5029
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm6115-dispcc.h
-@@ -0,0 +1,36 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_SM6115_H
-+#define _DT_BINDINGS_CLK_QCOM_DISP_CC_SM6115_H
-+
-+/* DISP_CC clocks */
-+#define DISP_CC_PLL0			0
-+#define DISP_CC_PLL0_OUT_MAIN		1
-+#define DISP_CC_MDSS_AHB_CLK		2
-+#define DISP_CC_MDSS_AHB_CLK_SRC	3
-+#define DISP_CC_MDSS_BYTE0_CLK		4
-+#define DISP_CC_MDSS_BYTE0_CLK_SRC	5
-+#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC	6
-+#define DISP_CC_MDSS_BYTE0_INTF_CLK	7
-+#define DISP_CC_MDSS_ESC0_CLK		8
-+#define DISP_CC_MDSS_ESC0_CLK_SRC	9
-+#define DISP_CC_MDSS_MDP_CLK		10
-+#define DISP_CC_MDSS_MDP_CLK_SRC	11
-+#define DISP_CC_MDSS_MDP_LUT_CLK	12
-+#define DISP_CC_MDSS_NON_GDSC_AHB_CLK	13
-+#define DISP_CC_MDSS_PCLK0_CLK		14
-+#define DISP_CC_MDSS_PCLK0_CLK_SRC	15
-+#define DISP_CC_MDSS_ROT_CLK		16
-+#define DISP_CC_MDSS_ROT_CLK_SRC	17
-+#define DISP_CC_MDSS_VSYNC_CLK		18
-+#define DISP_CC_MDSS_VSYNC_CLK_SRC	19
-+#define DISP_CC_SLEEP_CLK		20
-+#define DISP_CC_SLEEP_CLK_SRC		21
-+
-+/* DISP_CC GDSCR */
-+#define MDSS_GDSC			0
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm6115-gpucc.h b/include/dt-bindings/clock/qcom,sm6115-gpucc.h
-new file mode 100644
-index 000000000000..945f21a7d745
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm6115-gpucc.h
-@@ -0,0 +1,36 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_SM6115_H
-+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_SM6115_H
-+
-+/* GPU_CC clocks */
-+#define GPU_CC_PLL0			0
-+#define GPU_CC_PLL0_OUT_AUX2		1
-+#define GPU_CC_PLL1			2
-+#define GPU_CC_PLL1_OUT_AUX		3
-+#define GPU_CC_AHB_CLK			4
-+#define GPU_CC_CRC_AHB_CLK		5
-+#define GPU_CC_CX_GFX3D_CLK		6
-+#define GPU_CC_CX_GMU_CLK		7
-+#define GPU_CC_CX_SNOC_DVM_CLK		8
-+#define GPU_CC_CXO_AON_CLK		9
-+#define GPU_CC_CXO_CLK			10
-+#define GPU_CC_GMU_CLK_SRC		11
-+#define GPU_CC_GX_CXO_CLK		12
-+#define GPU_CC_GX_GFX3D_CLK		13
-+#define GPU_CC_GX_GFX3D_CLK_SRC		14
-+#define GPU_CC_SLEEP_CLK		15
-+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK	16
-+
-+/* Resets */
-+#define GPU_GX_BCR			0
-+
-+/* GDSCs */
-+#define GPU_CX_GDSC			0
-+#define GPU_GX_GDSC			1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm6115-lpasscc.h b/include/dt-bindings/clock/qcom,sm6115-lpasscc.h
-new file mode 100644
-index 000000000000..799274517c9a
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm6115-lpasscc.h
-@@ -0,0 +1,15 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Linaro Ltd.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_LPASSCC_SM6115_H
-+#define _DT_BINDINGS_CLK_QCOM_LPASSCC_SM6115_H
-+
-+/* LPASS CC */
-+#define LPASS_SWR_TX_CONFIG_CGCR		0
-+
-+/* LPASS_AUDIO CC */
-+#define LPASS_AUDIO_SWR_RX_CGCR			0
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm6125-gpucc.h b/include/dt-bindings/clock/qcom,sm6125-gpucc.h
-new file mode 100644
-index 000000000000..ce5bd920f2c4
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm6125-gpucc.h
-@@ -0,0 +1,31 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_SM6125_H
-+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_SM6125_H
-+
-+/* Clocks */
-+#define GPU_CC_PLL0_OUT_AUX2			0
-+#define GPU_CC_PLL1_OUT_AUX2			1
-+#define GPU_CC_CRC_AHB_CLK			2
-+#define GPU_CC_CX_APB_CLK			3
-+#define GPU_CC_CX_GFX3D_CLK			4
-+#define GPU_CC_CX_GMU_CLK			5
-+#define GPU_CC_CX_SNOC_DVM_CLK			6
-+#define GPU_CC_CXO_AON_CLK			7
-+#define GPU_CC_CXO_CLK				8
-+#define GPU_CC_GMU_CLK_SRC			9
-+#define GPU_CC_SLEEP_CLK			10
-+#define GPU_CC_GX_GFX3D_CLK			11
-+#define GPU_CC_GX_GFX3D_CLK_SRC			12
-+#define GPU_CC_AHB_CLK				13
-+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK		14
-+
-+/* GDSCs */
-+#define GPU_CX_GDSC				0
-+#define GPU_GX_GDSC				1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm6350-camcc.h b/include/dt-bindings/clock/qcom,sm6350-camcc.h
-new file mode 100644
-index 000000000000..c6bcdc8fd485
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm6350-camcc.h
-@@ -0,0 +1,109 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2022, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_CAMCC_SM6350_H
-+#define _DT_BINDINGS_CLK_QCOM_CAMCC_SM6350_H
-+
-+/* CAMCC clocks */
-+#define CAMCC_PLL2_OUT_EARLY		0
-+#define CAMCC_PLL0			1
-+#define CAMCC_PLL0_OUT_EVEN		2
-+#define CAMCC_PLL1			3
-+#define CAMCC_PLL1_OUT_EVEN		4
-+#define CAMCC_PLL2			5
-+#define CAMCC_PLL2_OUT_MAIN		6
-+#define CAMCC_PLL3			7
-+#define CAMCC_BPS_AHB_CLK		8
-+#define CAMCC_BPS_AREG_CLK		9
-+#define CAMCC_BPS_AXI_CLK		10
-+#define CAMCC_BPS_CLK			11
-+#define CAMCC_BPS_CLK_SRC		12
-+#define CAMCC_CAMNOC_ATB_CLK		13
-+#define CAMCC_CAMNOC_AXI_CLK		14
-+#define CAMCC_CCI_0_CLK		15
-+#define CAMCC_CCI_0_CLK_SRC		16
-+#define CAMCC_CCI_1_CLK		17
-+#define CAMCC_CCI_1_CLK_SRC		18
-+#define CAMCC_CORE_AHB_CLK		19
-+#define CAMCC_CPAS_AHB_CLK		20
-+#define CAMCC_CPHY_RX_CLK_SRC		21
-+#define CAMCC_CSI0PHYTIMER_CLK		22
-+#define CAMCC_CSI0PHYTIMER_CLK_SRC	23
-+#define CAMCC_CSI1PHYTIMER_CLK		24
-+#define CAMCC_CSI1PHYTIMER_CLK_SRC	25
-+#define CAMCC_CSI2PHYTIMER_CLK		26
-+#define CAMCC_CSI2PHYTIMER_CLK_SRC	27
-+#define CAMCC_CSI3PHYTIMER_CLK		28
-+#define CAMCC_CSI3PHYTIMER_CLK_SRC	29
-+#define CAMCC_CSIPHY0_CLK		30
-+#define CAMCC_CSIPHY1_CLK		31
-+#define CAMCC_CSIPHY2_CLK		32
-+#define CAMCC_CSIPHY3_CLK		33
-+#define CAMCC_FAST_AHB_CLK_SRC		34
-+#define CAMCC_ICP_APB_CLK		35
-+#define CAMCC_ICP_ATB_CLK		36
-+#define CAMCC_ICP_CLK			37
-+#define CAMCC_ICP_CLK_SRC		38
-+#define CAMCC_ICP_CTI_CLK		39
-+#define CAMCC_ICP_TS_CLK		40
-+#define CAMCC_IFE_0_AXI_CLK		41
-+#define CAMCC_IFE_0_CLK		42
-+#define CAMCC_IFE_0_CLK_SRC		43
-+#define CAMCC_IFE_0_CPHY_RX_CLK	44
-+#define CAMCC_IFE_0_CSID_CLK		45
-+#define CAMCC_IFE_0_CSID_CLK_SRC	46
-+#define CAMCC_IFE_0_DSP_CLK		47
-+#define CAMCC_IFE_1_AXI_CLK		48
-+#define CAMCC_IFE_1_CLK		49
-+#define CAMCC_IFE_1_CLK_SRC		50
-+#define CAMCC_IFE_1_CPHY_RX_CLK	51
-+#define CAMCC_IFE_1_CSID_CLK		52
-+#define CAMCC_IFE_1_CSID_CLK_SRC	53
-+#define CAMCC_IFE_1_DSP_CLK		54
-+#define CAMCC_IFE_2_AXI_CLK		55
-+#define CAMCC_IFE_2_CLK		56
-+#define CAMCC_IFE_2_CLK_SRC		57
-+#define CAMCC_IFE_2_CPHY_RX_CLK	58
-+#define CAMCC_IFE_2_CSID_CLK		59
-+#define CAMCC_IFE_2_CSID_CLK_SRC	60
-+#define CAMCC_IFE_2_DSP_CLK		61
-+#define CAMCC_IFE_LITE_CLK		62
-+#define CAMCC_IFE_LITE_CLK_SRC		63
-+#define CAMCC_IFE_LITE_CPHY_RX_CLK	64
-+#define CAMCC_IFE_LITE_CSID_CLK	65
-+#define CAMCC_IFE_LITE_CSID_CLK_SRC	66
-+#define CAMCC_IPE_0_AHB_CLK		67
-+#define CAMCC_IPE_0_AREG_CLK		68
-+#define CAMCC_IPE_0_AXI_CLK		69
-+#define CAMCC_IPE_0_CLK		70
-+#define CAMCC_IPE_0_CLK_SRC		71
-+#define CAMCC_JPEG_CLK			72
-+#define CAMCC_JPEG_CLK_SRC		73
-+#define CAMCC_LRME_CLK			74
-+#define CAMCC_LRME_CLK_SRC		75
-+#define CAMCC_MCLK0_CLK		76
-+#define CAMCC_MCLK0_CLK_SRC		77
-+#define CAMCC_MCLK1_CLK		78
-+#define CAMCC_MCLK1_CLK_SRC		79
-+#define CAMCC_MCLK2_CLK		80
-+#define CAMCC_MCLK2_CLK_SRC		81
-+#define CAMCC_MCLK3_CLK		82
-+#define CAMCC_MCLK3_CLK_SRC		83
-+#define CAMCC_MCLK4_CLK		84
-+#define CAMCC_MCLK4_CLK_SRC		85
-+#define CAMCC_SLOW_AHB_CLK_SRC		86
-+#define CAMCC_SOC_AHB_CLK		87
-+#define CAMCC_SYS_TMR_CLK		88
-+
-+/* GDSCs */
-+#define BPS_GDSC			0
-+#define IPE_0_GDSC			1
-+#define IFE_0_GDSC			2
-+#define IFE_1_GDSC			3
-+#define IFE_2_GDSC			4
-+#define TITAN_TOP_GDSC			5
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm6350-videocc.h b/include/dt-bindings/clock/qcom,sm6350-videocc.h
-new file mode 100644
-index 000000000000..2af7f91fa023
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm6350-videocc.h
-@@ -0,0 +1,27 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2021, Konrad Dybcio <konrad.dybcio@somainline.org>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_SM6350_H
-+#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_SM6350_H
-+
-+/* VIDEO_CC clocks */
-+#define VIDEO_PLL0			0
-+#define VIDEO_PLL0_OUT_EVEN             1
-+#define VIDEO_CC_IRIS_AHB_CLK		2
-+#define VIDEO_CC_IRIS_CLK_SRC		3
-+#define VIDEO_CC_MVS0_AXI_CLK		4
-+#define VIDEO_CC_MVS0_CORE_CLK		5
-+#define VIDEO_CC_MVSC_CORE_CLK		6
-+#define VIDEO_CC_MVSC_CTL_AXI_CLK	7
-+#define VIDEO_CC_SLEEP_CLK		8
-+#define VIDEO_CC_SLEEP_CLK_SRC		9
-+#define VIDEO_CC_VENUS_AHB_CLK		10
-+
-+/* GDSCs */
-+#define MVSC_GDSC			0
-+#define MVS0_GDSC			1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm6375-dispcc.h b/include/dt-bindings/clock/qcom,sm6375-dispcc.h
-new file mode 100644
-index 000000000000..1cb0bed004bd
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm6375-dispcc.h
-@@ -0,0 +1,42 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2022, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_SM6375_H
-+#define _DT_BINDINGS_CLK_QCOM_DISP_CC_SM6375_H
-+
-+/* Clocks */
-+#define DISP_CC_PLL0					0
-+#define DISP_CC_MDSS_AHB_CLK				1
-+#define DISP_CC_MDSS_AHB_CLK_SRC			2
-+#define DISP_CC_MDSS_BYTE0_CLK				3
-+#define DISP_CC_MDSS_BYTE0_CLK_SRC			4
-+#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC			5
-+#define DISP_CC_MDSS_BYTE0_INTF_CLK			6
-+#define DISP_CC_MDSS_ESC0_CLK				7
-+#define DISP_CC_MDSS_ESC0_CLK_SRC			8
-+#define DISP_CC_MDSS_MDP_CLK				9
-+#define DISP_CC_MDSS_MDP_CLK_SRC			10
-+#define DISP_CC_MDSS_MDP_LUT_CLK			11
-+#define DISP_CC_MDSS_NON_GDSC_AHB_CLK			12
-+#define DISP_CC_MDSS_PCLK0_CLK				13
-+#define DISP_CC_MDSS_PCLK0_CLK_SRC			14
-+#define DISP_CC_MDSS_ROT_CLK				15
-+#define DISP_CC_MDSS_ROT_CLK_SRC			16
-+#define DISP_CC_MDSS_RSCC_AHB_CLK			17
-+#define DISP_CC_MDSS_RSCC_VSYNC_CLK			18
-+#define DISP_CC_MDSS_VSYNC_CLK				19
-+#define DISP_CC_MDSS_VSYNC_CLK_SRC			20
-+#define DISP_CC_SLEEP_CLK				21
-+#define DISP_CC_XO_CLK					22
-+
-+/* Resets */
-+#define DISP_CC_MDSS_CORE_BCR				0
-+#define DISP_CC_MDSS_RSCC_BCR				1
-+
-+/* GDSCs */
-+#define MDSS_GDSC					0
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm6375-gcc.h b/include/dt-bindings/clock/qcom,sm6375-gcc.h
-new file mode 100644
-index 000000000000..1e9801e1cedf
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm6375-gcc.h
-@@ -0,0 +1,234 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2022, Konrad Dybcio <konrad.dybcio@somainline.org>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_SM6375_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_SM6375_H
-+
-+/* Clocks */
-+#define GPLL0						0
-+#define GPLL0_OUT_EVEN					1
-+#define GPLL0_OUT_ODD					2
-+#define GPLL1						3
-+#define GPLL10						4
-+#define GPLL11						5
-+#define GPLL3						6
-+#define GPLL3_OUT_EVEN					7
-+#define GPLL4						8
-+#define GPLL5						9
-+#define GPLL6						10
-+#define GPLL6_OUT_EVEN					11
-+#define GPLL7						12
-+#define GPLL8						13
-+#define GPLL8_OUT_EVEN					14
-+#define GPLL9						15
-+#define GPLL9_OUT_MAIN					16
-+#define GCC_AHB2PHY_CSI_CLK				17
-+#define GCC_AHB2PHY_USB_CLK				18
-+#define GCC_BIMC_GPU_AXI_CLK				19
-+#define GCC_BOOT_ROM_AHB_CLK				20
-+#define GCC_CAM_THROTTLE_NRT_CLK			21
-+#define GCC_CAM_THROTTLE_RT_CLK				22
-+#define GCC_CAMERA_AHB_CLK				23
-+#define GCC_CAMERA_XO_CLK				24
-+#define GCC_CAMSS_AXI_CLK				25
-+#define GCC_CAMSS_AXI_CLK_SRC				26
-+#define GCC_CAMSS_CAMNOC_ATB_CLK			27
-+#define GCC_CAMSS_CAMNOC_NTS_XO_CLK			28
-+#define GCC_CAMSS_CCI_0_CLK				29
-+#define GCC_CAMSS_CCI_0_CLK_SRC				30
-+#define GCC_CAMSS_CCI_1_CLK				31
-+#define GCC_CAMSS_CCI_1_CLK_SRC				32
-+#define GCC_CAMSS_CPHY_0_CLK				33
-+#define GCC_CAMSS_CPHY_1_CLK				34
-+#define GCC_CAMSS_CPHY_2_CLK				35
-+#define GCC_CAMSS_CPHY_3_CLK				36
-+#define GCC_CAMSS_CSI0PHYTIMER_CLK			37
-+#define GCC_CAMSS_CSI0PHYTIMER_CLK_SRC			38
-+#define GCC_CAMSS_CSI1PHYTIMER_CLK			39
-+#define GCC_CAMSS_CSI1PHYTIMER_CLK_SRC			40
-+#define GCC_CAMSS_CSI2PHYTIMER_CLK			41
-+#define GCC_CAMSS_CSI2PHYTIMER_CLK_SRC			42
-+#define GCC_CAMSS_CSI3PHYTIMER_CLK			43
-+#define GCC_CAMSS_CSI3PHYTIMER_CLK_SRC			44
-+#define GCC_CAMSS_MCLK0_CLK				45
-+#define GCC_CAMSS_MCLK0_CLK_SRC				46
-+#define GCC_CAMSS_MCLK1_CLK				47
-+#define GCC_CAMSS_MCLK1_CLK_SRC				48
-+#define GCC_CAMSS_MCLK2_CLK				49
-+#define GCC_CAMSS_MCLK2_CLK_SRC				50
-+#define GCC_CAMSS_MCLK3_CLK				51
-+#define GCC_CAMSS_MCLK3_CLK_SRC				52
-+#define GCC_CAMSS_MCLK4_CLK				53
-+#define GCC_CAMSS_MCLK4_CLK_SRC				54
-+#define GCC_CAMSS_NRT_AXI_CLK				55
-+#define GCC_CAMSS_OPE_AHB_CLK				56
-+#define GCC_CAMSS_OPE_AHB_CLK_SRC			57
-+#define GCC_CAMSS_OPE_CLK				58
-+#define GCC_CAMSS_OPE_CLK_SRC				59
-+#define GCC_CAMSS_RT_AXI_CLK				60
-+#define GCC_CAMSS_TFE_0_CLK				61
-+#define GCC_CAMSS_TFE_0_CLK_SRC				62
-+#define GCC_CAMSS_TFE_0_CPHY_RX_CLK			63
-+#define GCC_CAMSS_TFE_0_CSID_CLK			64
-+#define GCC_CAMSS_TFE_0_CSID_CLK_SRC			65
-+#define GCC_CAMSS_TFE_1_CLK				66
-+#define GCC_CAMSS_TFE_1_CLK_SRC				67
-+#define GCC_CAMSS_TFE_1_CPHY_RX_CLK			68
-+#define GCC_CAMSS_TFE_1_CSID_CLK			69
-+#define GCC_CAMSS_TFE_1_CSID_CLK_SRC			70
-+#define GCC_CAMSS_TFE_2_CLK				71
-+#define GCC_CAMSS_TFE_2_CLK_SRC				72
-+#define GCC_CAMSS_TFE_2_CPHY_RX_CLK			73
-+#define GCC_CAMSS_TFE_2_CSID_CLK			74
-+#define GCC_CAMSS_TFE_2_CSID_CLK_SRC			75
-+#define GCC_CAMSS_TFE_CPHY_RX_CLK_SRC			76
-+#define GCC_CAMSS_TOP_AHB_CLK				77
-+#define GCC_CAMSS_TOP_AHB_CLK_SRC			78
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK			79
-+#define GCC_CPUSS_AHB_CLK_SRC				80
-+#define GCC_CPUSS_AHB_POSTDIV_CLK_SRC			81
-+#define GCC_CPUSS_GNOC_CLK				82
-+#define GCC_DISP_AHB_CLK				83
-+#define GCC_DISP_GPLL0_CLK_SRC				84
-+#define GCC_DISP_GPLL0_DIV_CLK_SRC			85
-+#define GCC_DISP_HF_AXI_CLK				86
-+#define GCC_DISP_SLEEP_CLK				87
-+#define GCC_DISP_THROTTLE_CORE_CLK			88
-+#define GCC_DISP_XO_CLK					89
-+#define GCC_GP1_CLK					90
-+#define GCC_GP1_CLK_SRC					91
-+#define GCC_GP2_CLK					92
-+#define GCC_GP2_CLK_SRC					93
-+#define GCC_GP3_CLK					94
-+#define GCC_GP3_CLK_SRC					95
-+#define GCC_GPU_CFG_AHB_CLK				96
-+#define GCC_GPU_GPLL0_CLK_SRC				97
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC			98
-+#define GCC_GPU_MEMNOC_GFX_CLK				99
-+#define GCC_GPU_SNOC_DVM_GFX_CLK			100
-+#define GCC_GPU_THROTTLE_CORE_CLK			101
-+#define GCC_PDM2_CLK					102
-+#define GCC_PDM2_CLK_SRC				103
-+#define GCC_PDM_AHB_CLK					104
-+#define GCC_PDM_XO4_CLK					105
-+#define GCC_PRNG_AHB_CLK				106
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK			107
-+#define GCC_QMIP_CAMERA_RT_AHB_CLK			108
-+#define GCC_QMIP_DISP_AHB_CLK				109
-+#define GCC_QMIP_GPU_CFG_AHB_CLK			110
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK			111
-+#define GCC_QUPV3_WRAP0_CORE_2X_CLK			112
-+#define GCC_QUPV3_WRAP0_CORE_CLK			113
-+#define GCC_QUPV3_WRAP0_S0_CLK				114
-+#define GCC_QUPV3_WRAP0_S0_CLK_SRC			115
-+#define GCC_QUPV3_WRAP0_S1_CLK				116
-+#define GCC_QUPV3_WRAP0_S1_CLK_SRC			117
-+#define GCC_QUPV3_WRAP0_S2_CLK				118
-+#define GCC_QUPV3_WRAP0_S2_CLK_SRC			119
-+#define GCC_QUPV3_WRAP0_S3_CLK				120
-+#define GCC_QUPV3_WRAP0_S3_CLK_SRC			121
-+#define GCC_QUPV3_WRAP0_S4_CLK				122
-+#define GCC_QUPV3_WRAP0_S4_CLK_SRC			123
-+#define GCC_QUPV3_WRAP0_S5_CLK				124
-+#define GCC_QUPV3_WRAP0_S5_CLK_SRC			125
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK			126
-+#define GCC_QUPV3_WRAP1_CORE_CLK			127
-+#define GCC_QUPV3_WRAP1_S0_CLK				128
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC			129
-+#define GCC_QUPV3_WRAP1_S1_CLK				130
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC			131
-+#define GCC_QUPV3_WRAP1_S2_CLK				132
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC			133
-+#define GCC_QUPV3_WRAP1_S3_CLK				134
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC			135
-+#define GCC_QUPV3_WRAP1_S4_CLK				136
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC			137
-+#define GCC_QUPV3_WRAP1_S5_CLK				138
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC			139
-+#define GCC_QUPV3_WRAP_0_M_AHB_CLK			140
-+#define GCC_QUPV3_WRAP_0_S_AHB_CLK			141
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK			142
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK			143
-+#define GCC_RX5_PCIE_CLKREF_EN_CLK			144
-+#define GCC_SDCC1_AHB_CLK				145
-+#define GCC_SDCC1_APPS_CLK				146
-+#define GCC_SDCC1_APPS_CLK_SRC				147
-+#define GCC_SDCC1_ICE_CORE_CLK				148
-+#define GCC_SDCC1_ICE_CORE_CLK_SRC			149
-+#define GCC_SDCC2_AHB_CLK				150
-+#define GCC_SDCC2_APPS_CLK				151
-+#define GCC_SDCC2_APPS_CLK_SRC				152
-+#define GCC_SYS_NOC_CPUSS_AHB_CLK			153
-+#define GCC_SYS_NOC_UFS_PHY_AXI_CLK			154
-+#define GCC_SYS_NOC_USB3_PRIM_AXI_CLK			155
-+#define GCC_UFS_MEM_CLKREF_CLK				156
-+#define GCC_UFS_PHY_AHB_CLK				157
-+#define GCC_UFS_PHY_AXI_CLK				158
-+#define GCC_UFS_PHY_AXI_CLK_SRC				159
-+#define GCC_UFS_PHY_ICE_CORE_CLK			160
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC			161
-+#define GCC_UFS_PHY_PHY_AUX_CLK				162
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC			163
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK			164
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK			165
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK			166
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC			167
-+#define GCC_USB30_PRIM_MASTER_CLK			168
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC			169
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK			170
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC		171
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC	172
-+#define GCC_USB30_PRIM_SLEEP_CLK			173
-+#define GCC_USB3_PRIM_CLKREF_CLK			174
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC			175
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK			176
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK			177
-+#define GCC_VCODEC0_AXI_CLK				178
-+#define GCC_VENUS_AHB_CLK				179
-+#define GCC_VENUS_CTL_AXI_CLK				180
-+#define GCC_VIDEO_AHB_CLK				181
-+#define GCC_VIDEO_AXI0_CLK				182
-+#define GCC_VIDEO_THROTTLE_CORE_CLK			183
-+#define GCC_VIDEO_VCODEC0_SYS_CLK			184
-+#define GCC_VIDEO_VENUS_CLK_SRC				185
-+#define GCC_VIDEO_VENUS_CTL_CLK				186
-+#define GCC_VIDEO_XO_CLK				187
-+
-+/* Resets */
-+#define GCC_CAMSS_OPE_BCR				0
-+#define GCC_CAMSS_TFE_BCR				1
-+#define GCC_CAMSS_TOP_BCR				2
-+#define GCC_GPU_BCR					3
-+#define GCC_MMSS_BCR					4
-+#define GCC_PDM_BCR					5
-+#define GCC_PRNG_BCR					6
-+#define GCC_QUPV3_WRAPPER_0_BCR				7
-+#define GCC_QUPV3_WRAPPER_1_BCR				8
-+#define GCC_QUSB2PHY_PRIM_BCR				9
-+#define GCC_QUSB2PHY_SEC_BCR				10
-+#define GCC_SDCC1_BCR					11
-+#define GCC_SDCC2_BCR					12
-+#define GCC_UFS_PHY_BCR					13
-+#define GCC_USB30_PRIM_BCR				14
-+#define GCC_USB_PHY_CFG_AHB2PHY_BCR			15
-+#define GCC_VCODEC0_BCR					16
-+#define GCC_VENUS_BCR					17
-+#define GCC_VIDEO_INTERFACE_BCR				18
-+#define GCC_USB3_DP_PHY_PRIM_BCR			19
-+#define GCC_USB3_PHY_PRIM_SP0_BCR			20
-+
-+/* GDSCs */
-+#define USB30_PRIM_GDSC					0
-+#define UFS_PHY_GDSC					1
-+#define CAMSS_TOP_GDSC					2
-+#define VENUS_GDSC					3
-+#define VCODEC0_GDSC					4
-+#define HLOS1_VOTE_MM_SNOC_MMU_TBU_NRT_GDSC		5
-+#define HLOS1_VOTE_MM_SNOC_MMU_TBU_RT_GDSC		6
-+#define HLOS1_VOTE_TURING_MMU_TBU0_GDSC			7
-+#define HLOS1_VOTE_TURING_MMU_TBU1_GDSC			8
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm6375-gpucc.h b/include/dt-bindings/clock/qcom,sm6375-gpucc.h
-new file mode 100644
-index 000000000000..0887ac03825e
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm6375-gpucc.h
-@@ -0,0 +1,36 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_BLAIR_H
-+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_BLAIR_H
-+
-+/* GPU CC clocks */
-+#define GPU_CC_PLL0					0
-+#define GPU_CC_PLL1					1
-+#define GPU_CC_AHB_CLK					2
-+#define GPU_CC_CX_GFX3D_CLK				3
-+#define GPU_CC_CX_GFX3D_SLV_CLK				4
-+#define GPU_CC_CX_GMU_CLK				5
-+#define GPU_CC_CX_SNOC_DVM_CLK				6
-+#define GPU_CC_CXO_AON_CLK				7
-+#define GPU_CC_CXO_CLK					8
-+#define GPU_CC_GMU_CLK_SRC				9
-+#define GPU_CC_GX_CXO_CLK				10
-+#define GPU_CC_GX_GFX3D_CLK				11
-+#define GPU_CC_GX_GFX3D_CLK_SRC				12
-+#define GPU_CC_GX_GMU_CLK				13
-+#define GPU_CC_SLEEP_CLK				14
-+
-+/* GDSCs */
-+#define GPU_CX_GDSC					0
-+#define GPU_GX_GDSC					1
-+
-+/* Resets */
-+#define GPU_GX_BCR					0
-+#define GPU_ACD_BCR					1
-+#define GPU_GX_ACD_MISC_BCR				2
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm7150-camcc.h b/include/dt-bindings/clock/qcom,sm7150-camcc.h
-new file mode 100644
-index 000000000000..ce73ef0fe95d
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm7150-camcc.h
-@@ -0,0 +1,113 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2024, Danila Tikhonov <danila@jiaxyga.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_CAMCC_SM7150_H
-+#define _DT_BINDINGS_CLK_QCOM_CAMCC_SM7150_H
-+
-+/* Hardware clocks */
-+#define CAMCC_PLL0_OUT_EVEN					0
-+#define CAMCC_PLL0_OUT_ODD					1
-+#define CAMCC_PLL1_OUT_EVEN					2
-+#define CAMCC_PLL2_OUT_EARLY					3
-+#define CAMCC_PLL3_OUT_EVEN					4
-+#define CAMCC_PLL4_OUT_EVEN					5
-+
-+/* CAMCC clock registers */
-+#define CAMCC_PLL0						6
-+#define CAMCC_PLL1						7
-+#define CAMCC_PLL2						8
-+#define CAMCC_PLL2_OUT_AUX					9
-+#define CAMCC_PLL2_OUT_MAIN					10
-+#define CAMCC_PLL3						11
-+#define CAMCC_PLL4						12
-+#define CAMCC_BPS_AHB_CLK					13
-+#define CAMCC_BPS_AREG_CLK					14
-+#define CAMCC_BPS_AXI_CLK					15
-+#define CAMCC_BPS_CLK						16
-+#define CAMCC_BPS_CLK_SRC					17
-+#define CAMCC_CAMNOC_AXI_CLK					18
-+#define CAMCC_CAMNOC_AXI_CLK_SRC				19
-+#define CAMCC_CAMNOC_DCD_XO_CLK					20
-+#define CAMCC_CCI_0_CLK						21
-+#define CAMCC_CCI_0_CLK_SRC					22
-+#define CAMCC_CCI_1_CLK						23
-+#define CAMCC_CCI_1_CLK_SRC					24
-+#define CAMCC_CORE_AHB_CLK					25
-+#define CAMCC_CPAS_AHB_CLK					26
-+#define CAMCC_CPHY_RX_CLK_SRC					27
-+#define CAMCC_CSI0PHYTIMER_CLK					28
-+#define CAMCC_CSI0PHYTIMER_CLK_SRC				29
-+#define CAMCC_CSI1PHYTIMER_CLK					30
-+#define CAMCC_CSI1PHYTIMER_CLK_SRC				31
-+#define CAMCC_CSI2PHYTIMER_CLK					32
-+#define CAMCC_CSI2PHYTIMER_CLK_SRC				33
-+#define CAMCC_CSI3PHYTIMER_CLK					34
-+#define CAMCC_CSI3PHYTIMER_CLK_SRC				35
-+#define CAMCC_CSIPHY0_CLK					36
-+#define CAMCC_CSIPHY1_CLK					37
-+#define CAMCC_CSIPHY2_CLK					38
-+#define CAMCC_CSIPHY3_CLK					39
-+#define CAMCC_FAST_AHB_CLK_SRC					40
-+#define CAMCC_FD_CORE_CLK					41
-+#define CAMCC_FD_CORE_CLK_SRC					42
-+#define CAMCC_FD_CORE_UAR_CLK					43
-+#define CAMCC_ICP_AHB_CLK					44
-+#define CAMCC_ICP_CLK						45
-+#define CAMCC_ICP_CLK_SRC					46
-+#define CAMCC_IFE_0_AXI_CLK					47
-+#define CAMCC_IFE_0_CLK						48
-+#define CAMCC_IFE_0_CLK_SRC					49
-+#define CAMCC_IFE_0_CPHY_RX_CLK					50
-+#define CAMCC_IFE_0_CSID_CLK					51
-+#define CAMCC_IFE_0_CSID_CLK_SRC				52
-+#define CAMCC_IFE_0_DSP_CLK					53
-+#define CAMCC_IFE_1_AXI_CLK					54
-+#define CAMCC_IFE_1_CLK						55
-+#define CAMCC_IFE_1_CLK_SRC					56
-+#define CAMCC_IFE_1_CPHY_RX_CLK					57
-+#define CAMCC_IFE_1_CSID_CLK					58
-+#define CAMCC_IFE_1_CSID_CLK_SRC				59
-+#define CAMCC_IFE_1_DSP_CLK					60
-+#define CAMCC_IFE_LITE_CLK					61
-+#define CAMCC_IFE_LITE_CLK_SRC					62
-+#define CAMCC_IFE_LITE_CPHY_RX_CLK				63
-+#define CAMCC_IFE_LITE_CSID_CLK					64
-+#define CAMCC_IFE_LITE_CSID_CLK_SRC				65
-+#define CAMCC_IPE_0_AHB_CLK					66
-+#define CAMCC_IPE_0_AREG_CLK					67
-+#define CAMCC_IPE_0_AXI_CLK					68
-+#define CAMCC_IPE_0_CLK						69
-+#define CAMCC_IPE_0_CLK_SRC					70
-+#define CAMCC_IPE_1_AHB_CLK					71
-+#define CAMCC_IPE_1_AREG_CLK					72
-+#define CAMCC_IPE_1_AXI_CLK					73
-+#define CAMCC_IPE_1_CLK						74
-+#define CAMCC_JPEG_CLK						75
-+#define CAMCC_JPEG_CLK_SRC					76
-+#define CAMCC_LRME_CLK						77
-+#define CAMCC_LRME_CLK_SRC					78
-+#define CAMCC_MCLK0_CLK						79
-+#define CAMCC_MCLK0_CLK_SRC					80
-+#define CAMCC_MCLK1_CLK						81
-+#define CAMCC_MCLK1_CLK_SRC					82
-+#define CAMCC_MCLK2_CLK						83
-+#define CAMCC_MCLK2_CLK_SRC					84
-+#define CAMCC_MCLK3_CLK						85
-+#define CAMCC_MCLK3_CLK_SRC					86
-+#define CAMCC_SLEEP_CLK						87
-+#define CAMCC_SLEEP_CLK_SRC					88
-+#define CAMCC_SLOW_AHB_CLK_SRC					89
-+#define CAMCC_XO_CLK_SRC					90
-+
-+/* CAMCC GDSCRs */
-+#define BPS_GDSC						0
-+#define IFE_0_GDSC						1
-+#define IFE_1_GDSC						2
-+#define IPE_0_GDSC						3
-+#define IPE_1_GDSC						4
-+#define TITAN_TOP_GDSC						5
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm7150-dispcc.h b/include/dt-bindings/clock/qcom,sm7150-dispcc.h
-new file mode 100644
-index 000000000000..1e4e6432d506
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm7150-dispcc.h
-@@ -0,0 +1,62 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2024, Danila Tikhonov <danila@jiaxyga.com>
-+ * Copyright (c) 2024, David Wronek <david@mainlining.org>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_DISPCC_SM7150_H
-+#define _DT_BINDINGS_CLK_QCOM_DISPCC_SM7150_H
-+
-+/* DISPCC clock registers */
-+#define DISPCC_PLL0				0
-+#define DISPCC_MDSS_AHB_CLK			1
-+#define DISPCC_MDSS_AHB_CLK_SRC			2
-+#define DISPCC_MDSS_BYTE0_CLK			3
-+#define DISPCC_MDSS_BYTE0_CLK_SRC		4
-+#define DISPCC_MDSS_BYTE0_DIV_CLK_SRC		5
-+#define DISPCC_MDSS_BYTE0_INTF_CLK		6
-+#define DISPCC_MDSS_BYTE1_CLK			7
-+#define DISPCC_MDSS_BYTE1_CLK_SRC		8
-+#define DISPCC_MDSS_BYTE1_DIV_CLK_SRC		9
-+#define DISPCC_MDSS_BYTE1_INTF_CLK		10
-+#define DISPCC_MDSS_DP_AUX_CLK			11
-+#define DISPCC_MDSS_DP_AUX_CLK_SRC		12
-+#define DISPCC_MDSS_DP_CRYPTO_CLK		13
-+#define DISPCC_MDSS_DP_CRYPTO_CLK_SRC		14
-+#define DISPCC_MDSS_DP_LINK_CLK			15
-+#define DISPCC_MDSS_DP_LINK_CLK_SRC		16
-+#define DISPCC_MDSS_DP_LINK_INTF_CLK		17
-+#define DISPCC_MDSS_DP_PIXEL1_CLK		18
-+#define DISPCC_MDSS_DP_PIXEL1_CLK_SRC		19
-+#define DISPCC_MDSS_DP_PIXEL_CLK		20
-+#define DISPCC_MDSS_DP_PIXEL_CLK_SRC		21
-+#define DISPCC_MDSS_ESC0_CLK			22
-+#define DISPCC_MDSS_ESC0_CLK_SRC		23
-+#define DISPCC_MDSS_ESC1_CLK			24
-+#define DISPCC_MDSS_ESC1_CLK_SRC		25
-+#define DISPCC_MDSS_MDP_CLK			26
-+#define DISPCC_MDSS_MDP_CLK_SRC			27
-+#define DISPCC_MDSS_MDP_LUT_CLK			28
-+#define DISPCC_MDSS_NON_GDSC_AHB_CLK		29
-+#define DISPCC_MDSS_PCLK0_CLK			30
-+#define DISPCC_MDSS_PCLK0_CLK_SRC		31
-+#define DISPCC_MDSS_PCLK1_CLK			32
-+#define DISPCC_MDSS_PCLK1_CLK_SRC		33
-+#define DISPCC_MDSS_ROT_CLK			34
-+#define DISPCC_MDSS_ROT_CLK_SRC			35
-+#define DISPCC_MDSS_RSCC_AHB_CLK		36
-+#define DISPCC_MDSS_RSCC_VSYNC_CLK		37
-+#define DISPCC_MDSS_VSYNC_CLK			38
-+#define DISPCC_MDSS_VSYNC_CLK_SRC		39
-+#define DISPCC_XO_CLK_SRC			40
-+#define DISPCC_SLEEP_CLK			41
-+#define DISPCC_SLEEP_CLK_SRC			42
-+
-+/* DISPCC resets */
-+#define DISPCC_MDSS_CORE_BCR			0
-+
-+/* DISPCC GDSCR */
-+#define MDSS_GDSC				0
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm7150-gcc.h b/include/dt-bindings/clock/qcom,sm7150-gcc.h
-new file mode 100644
-index 000000000000..7719ffc86139
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm7150-gcc.h
-@@ -0,0 +1,186 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Danila Tikhonov <danila@jiaxyga.com>
-+ * Copyright (c) 2023, David Wronek <davidwronek@gmail.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_SM7150_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_SM7150_H
-+
-+/* GCC clock registers */
-+#define GCC_GPLL0_MAIN_DIV_CDIV				0
-+#define GPLL0						1
-+#define GPLL0_OUT_EVEN					2
-+#define GPLL6						3
-+#define GPLL7						4
-+#define GCC_AGGRE_NOC_PCIE_TBU_CLK			5
-+#define GCC_AGGRE_UFS_PHY_AXI_CLK			6
-+#define GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK		7
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK			8
-+#define GCC_APC_VS_CLK					9
-+#define GCC_BOOT_ROM_AHB_CLK				10
-+#define GCC_CAMERA_HF_AXI_CLK				11
-+#define GCC_CAMERA_SF_AXI_CLK				12
-+#define GCC_CE1_AHB_CLK					13
-+#define GCC_CE1_AXI_CLK					14
-+#define GCC_CE1_CLK					15
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK			16
-+#define GCC_CPUSS_AHB_CLK				17
-+#define GCC_CPUSS_AHB_CLK_SRC				18
-+#define GCC_CPUSS_RBCPR_CLK				19
-+#define GCC_CPUSS_RBCPR_CLK_SRC				20
-+#define GCC_DDRSS_GPU_AXI_CLK				21
-+#define GCC_DISP_GPLL0_CLK_SRC				22
-+#define GCC_DISP_GPLL0_DIV_CLK_SRC			23
-+#define GCC_DISP_HF_AXI_CLK				24
-+#define GCC_DISP_SF_AXI_CLK				25
-+#define GCC_GP1_CLK					26
-+#define GCC_GP1_CLK_SRC					27
-+#define GCC_GP2_CLK					28
-+#define GCC_GP2_CLK_SRC					29
-+#define GCC_GP3_CLK					30
-+#define GCC_GP3_CLK_SRC					31
-+#define GCC_GPU_GPLL0_CLK_SRC				32
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC			33
-+#define GCC_GPU_MEMNOC_GFX_CLK				34
-+#define GCC_GPU_SNOC_DVM_GFX_CLK			35
-+#define GCC_GPU_VS_CLK					36
-+#define GCC_NPU_AXI_CLK					37
-+#define GCC_NPU_CFG_AHB_CLK				38
-+#define GCC_NPU_GPLL0_CLK_SRC				39
-+#define GCC_NPU_GPLL0_DIV_CLK_SRC			40
-+#define GCC_PCIE_0_AUX_CLK				41
-+#define GCC_PCIE_0_AUX_CLK_SRC				42
-+#define GCC_PCIE_0_CFG_AHB_CLK				43
-+#define GCC_PCIE_0_CLKREF_CLK				44
-+#define GCC_PCIE_0_MSTR_AXI_CLK				45
-+#define GCC_PCIE_0_PIPE_CLK				46
-+#define GCC_PCIE_0_SLV_AXI_CLK				47
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK			48
-+#define GCC_PCIE_PHY_AUX_CLK				49
-+#define GCC_PCIE_PHY_REFGEN_CLK				50
-+#define GCC_PCIE_PHY_REFGEN_CLK_SRC			51
-+#define GCC_PDM2_CLK					52
-+#define GCC_PDM2_CLK_SRC				53
-+#define GCC_PDM_AHB_CLK					54
-+#define GCC_PDM_XO4_CLK					55
-+#define GCC_PRNG_AHB_CLK				56
-+#define GCC_QUPV3_WRAP0_CORE_2X_CLK			57
-+#define GCC_QUPV3_WRAP0_CORE_CLK			58
-+#define GCC_QUPV3_WRAP0_S0_CLK				59
-+#define GCC_QUPV3_WRAP0_S0_CLK_SRC			60
-+#define GCC_QUPV3_WRAP0_S1_CLK				61
-+#define GCC_QUPV3_WRAP0_S1_CLK_SRC			62
-+#define GCC_QUPV3_WRAP0_S2_CLK				63
-+#define GCC_QUPV3_WRAP0_S2_CLK_SRC			64
-+#define GCC_QUPV3_WRAP0_S3_CLK				65
-+#define GCC_QUPV3_WRAP0_S3_CLK_SRC			66
-+#define GCC_QUPV3_WRAP0_S4_CLK				67
-+#define GCC_QUPV3_WRAP0_S4_CLK_SRC			68
-+#define GCC_QUPV3_WRAP0_S5_CLK				69
-+#define GCC_QUPV3_WRAP0_S5_CLK_SRC			70
-+#define GCC_QUPV3_WRAP0_S6_CLK				71
-+#define GCC_QUPV3_WRAP0_S6_CLK_SRC			72
-+#define GCC_QUPV3_WRAP0_S7_CLK				73
-+#define GCC_QUPV3_WRAP0_S7_CLK_SRC			74
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK			75
-+#define GCC_QUPV3_WRAP1_CORE_CLK			76
-+#define GCC_QUPV3_WRAP1_S0_CLK				77
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC			78
-+#define GCC_QUPV3_WRAP1_S1_CLK				79
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC			80
-+#define GCC_QUPV3_WRAP1_S2_CLK				81
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC			82
-+#define GCC_QUPV3_WRAP1_S3_CLK				83
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC			84
-+#define GCC_QUPV3_WRAP1_S4_CLK				85
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC			86
-+#define GCC_QUPV3_WRAP1_S5_CLK				87
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC			88
-+#define GCC_QUPV3_WRAP1_S6_CLK				89
-+#define GCC_QUPV3_WRAP1_S6_CLK_SRC			90
-+#define GCC_QUPV3_WRAP1_S7_CLK				91
-+#define GCC_QUPV3_WRAP1_S7_CLK_SRC			92
-+#define GCC_QUPV3_WRAP_0_M_AHB_CLK			93
-+#define GCC_QUPV3_WRAP_0_S_AHB_CLK			94
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK			95
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK			96
-+#define GCC_SDCC1_AHB_CLK				97
-+#define GCC_SDCC1_APPS_CLK				98
-+#define GCC_SDCC1_APPS_CLK_SRC				99
-+#define GCC_SDCC1_ICE_CORE_CLK				100
-+#define GCC_SDCC1_ICE_CORE_CLK_SRC			101
-+#define GCC_SDCC2_AHB_CLK				102
-+#define GCC_SDCC2_APPS_CLK				103
-+#define GCC_SDCC2_APPS_CLK_SRC				104
-+#define GCC_SDCC4_AHB_CLK				105
-+#define GCC_SDCC4_APPS_CLK				106
-+#define GCC_SDCC4_APPS_CLK_SRC				107
-+#define GCC_SYS_NOC_CPUSS_AHB_CLK			108
-+#define GCC_TSIF_AHB_CLK				109
-+#define GCC_TSIF_INACTIVITY_TIMERS_CLK			110
-+#define GCC_TSIF_REF_CLK				111
-+#define GCC_TSIF_REF_CLK_SRC				112
-+#define GCC_UFS_MEM_CLKREF_CLK				113
-+#define GCC_UFS_PHY_AHB_CLK				114
-+#define GCC_UFS_PHY_AXI_CLK				115
-+#define GCC_UFS_PHY_AXI_CLK_SRC				116
-+#define GCC_UFS_PHY_AXI_HW_CTL_CLK			117
-+#define GCC_UFS_PHY_ICE_CORE_CLK			118
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC			119
-+#define GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK			120
-+#define GCC_UFS_PHY_PHY_AUX_CLK				121
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC			122
-+#define GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK			123
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK			124
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK			125
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK			126
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC			127
-+#define GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK		128
-+#define GCC_USB30_PRIM_MASTER_CLK			129
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC			130
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK			131
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC		132
-+#define GCC_USB30_PRIM_SLEEP_CLK			133
-+#define GCC_USB3_PRIM_CLKREF_CLK			134
-+#define GCC_USB3_PRIM_PHY_AUX_CLK			135
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC			136
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK			137
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK			138
-+#define GCC_USB_PHY_CFG_AHB2PHY_CLK			139
-+#define GCC_VDDA_VS_CLK					140
-+#define GCC_VDDCX_VS_CLK				141
-+#define GCC_VDDMX_VS_CLK				142
-+#define GCC_VIDEO_AXI_CLK				143
-+#define GCC_VS_CTRL_AHB_CLK				144
-+#define GCC_VS_CTRL_CLK					145
-+#define GCC_VS_CTRL_CLK_SRC				146
-+#define GCC_VSENSOR_CLK_SRC				147
-+
-+/* GCC Resets */
-+#define GCC_PCIE_0_BCR					0
-+#define GCC_PCIE_PHY_BCR				1
-+#define GCC_PCIE_PHY_COM_BCR				2
-+#define GCC_UFS_PHY_BCR					3
-+#define GCC_USB30_PRIM_BCR				4
-+#define GCC_USB3_DP_PHY_PRIM_BCR			5
-+#define GCC_USB3_DP_PHY_SEC_BCR				6
-+#define GCC_USB3_PHY_PRIM_BCR				7
-+#define GCC_USB3_PHY_SEC_BCR				8
-+#define GCC_QUSB2PHY_PRIM_BCR				9
-+#define GCC_VIDEO_AXI_CLK_BCR				10
-+
-+/* GCC GDSCRs */
-+#define PCIE_0_GDSC					0
-+#define UFS_PHY_GDSC					1
-+#define USB30_PRIM_GDSC					2
-+#define HLOS1_VOTE_AGGRE_NOC_MMU_AUDIO_TBU_GDSC		3
-+#define HLOS1_VOTE_AGGRE_NOC_MMU_PCIE_TBU_GDSC		4
-+#define HLOS1_VOTE_AGGRE_NOC_MMU_TBU1_GDSC		5
-+#define HLOS1_VOTE_AGGRE_NOC_MMU_TBU2_GDSC		6
-+#define HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC		7
-+#define HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC		8
-+#define HLOS1_VOTE_MMNOC_MMU_TBU_SF_GDSC		9
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm7150-videocc.h b/include/dt-bindings/clock/qcom,sm7150-videocc.h
-new file mode 100644
-index 000000000000..d86e0fbb159a
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm7150-videocc.h
-@@ -0,0 +1,28 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2024, Danila Tikhonov <danila@jiaxyga.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_VIDEOCC_SM7150_H
-+#define _DT_BINDINGS_CLK_QCOM_VIDEOCC_SM7150_H
-+
-+#define VIDEOCC_PLL0			0
-+#define VIDEOCC_IRIS_AHB_CLK		1
-+#define VIDEOCC_IRIS_CLK_SRC		2
-+#define VIDEOCC_MVS0_AXI_CLK		3
-+#define VIDEOCC_MVS0_CORE_CLK		4
-+#define VIDEOCC_MVS1_AXI_CLK		5
-+#define VIDEOCC_MVS1_CORE_CLK		6
-+#define VIDEOCC_MVSC_CORE_CLK		7
-+#define VIDEOCC_MVSC_CTL_AXI_CLK	8
-+#define VIDEOCC_VENUS_AHB_CLK		9
-+#define VIDEOCC_XO_CLK			10
-+#define VIDEOCC_XO_CLK_SRC		11
-+
-+/* VIDEOCC GDSCRs */
-+#define VENUS_GDSC			0
-+#define VCODEC0_GDSC			1
-+#define VCODEC1_GDSC			2
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8150-camcc.h b/include/dt-bindings/clock/qcom,sm8150-camcc.h
-new file mode 100644
-index 000000000000..5444035efa93
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8150-camcc.h
-@@ -0,0 +1,135 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_SM8150_H
-+#define _DT_BINDINGS_CLK_QCOM_CAM_CC_SM8150_H
-+
-+/* CAM_CC clocks */
-+#define CAM_CC_PLL0					0
-+#define CAM_CC_PLL0_OUT_EVEN				1
-+#define CAM_CC_PLL0_OUT_ODD				2
-+#define CAM_CC_PLL1					3
-+#define CAM_CC_PLL1_OUT_EVEN				4
-+#define CAM_CC_PLL2					5
-+#define CAM_CC_PLL2_OUT_MAIN				6
-+#define CAM_CC_PLL3					7
-+#define CAM_CC_PLL3_OUT_EVEN				8
-+#define CAM_CC_PLL4					9
-+#define CAM_CC_PLL4_OUT_EVEN				10
-+#define CAM_CC_BPS_AHB_CLK				11
-+#define CAM_CC_BPS_AREG_CLK				12
-+#define CAM_CC_BPS_AXI_CLK				13
-+#define CAM_CC_BPS_CLK					14
-+#define CAM_CC_BPS_CLK_SRC				15
-+#define CAM_CC_CAMNOC_AXI_CLK				16
-+#define CAM_CC_CAMNOC_AXI_CLK_SRC			17
-+#define CAM_CC_CAMNOC_DCD_XO_CLK			18
-+#define CAM_CC_CCI_0_CLK				19
-+#define CAM_CC_CCI_0_CLK_SRC				20
-+#define CAM_CC_CCI_1_CLK				21
-+#define CAM_CC_CCI_1_CLK_SRC				22
-+#define CAM_CC_CORE_AHB_CLK				23
-+#define CAM_CC_CPAS_AHB_CLK				24
-+#define CAM_CC_CPHY_RX_CLK_SRC				25
-+#define CAM_CC_CSI0PHYTIMER_CLK				26
-+#define CAM_CC_CSI0PHYTIMER_CLK_SRC			27
-+#define CAM_CC_CSI1PHYTIMER_CLK				28
-+#define CAM_CC_CSI1PHYTIMER_CLK_SRC			29
-+#define CAM_CC_CSI2PHYTIMER_CLK				30
-+#define CAM_CC_CSI2PHYTIMER_CLK_SRC			31
-+#define CAM_CC_CSI3PHYTIMER_CLK				32
-+#define CAM_CC_CSI3PHYTIMER_CLK_SRC			33
-+#define CAM_CC_CSIPHY0_CLK				34
-+#define CAM_CC_CSIPHY1_CLK				35
-+#define CAM_CC_CSIPHY2_CLK				36
-+#define CAM_CC_CSIPHY3_CLK				37
-+#define CAM_CC_FAST_AHB_CLK_SRC				38
-+#define CAM_CC_FD_CORE_CLK				39
-+#define CAM_CC_FD_CORE_CLK_SRC				40
-+#define CAM_CC_FD_CORE_UAR_CLK				41
-+#define CAM_CC_GDSC_CLK					42
-+#define CAM_CC_ICP_AHB_CLK				43
-+#define CAM_CC_ICP_CLK					44
-+#define CAM_CC_ICP_CLK_SRC				45
-+#define CAM_CC_IFE_0_AXI_CLK				46
-+#define CAM_CC_IFE_0_CLK				47
-+#define CAM_CC_IFE_0_CLK_SRC				48
-+#define CAM_CC_IFE_0_CPHY_RX_CLK			49
-+#define CAM_CC_IFE_0_CSID_CLK				50
-+#define CAM_CC_IFE_0_CSID_CLK_SRC			51
-+#define CAM_CC_IFE_0_DSP_CLK				52
-+#define CAM_CC_IFE_1_AXI_CLK				53
-+#define CAM_CC_IFE_1_CLK				54
-+#define CAM_CC_IFE_1_CLK_SRC				55
-+#define CAM_CC_IFE_1_CPHY_RX_CLK			56
-+#define CAM_CC_IFE_1_CSID_CLK				57
-+#define CAM_CC_IFE_1_CSID_CLK_SRC			58
-+#define CAM_CC_IFE_1_DSP_CLK				59
-+#define CAM_CC_IFE_LITE_0_CLK				60
-+#define CAM_CC_IFE_LITE_0_CLK_SRC			61
-+#define CAM_CC_IFE_LITE_0_CPHY_RX_CLK			62
-+#define CAM_CC_IFE_LITE_0_CSID_CLK			63
-+#define CAM_CC_IFE_LITE_0_CSID_CLK_SRC			64
-+#define CAM_CC_IFE_LITE_1_CLK				65
-+#define CAM_CC_IFE_LITE_1_CLK_SRC			66
-+#define CAM_CC_IFE_LITE_1_CPHY_RX_CLK			67
-+#define CAM_CC_IFE_LITE_1_CSID_CLK			68
-+#define CAM_CC_IFE_LITE_1_CSID_CLK_SRC			69
-+#define CAM_CC_IPE_0_AHB_CLK				70
-+#define CAM_CC_IPE_0_AREG_CLK				71
-+#define CAM_CC_IPE_0_AXI_CLK				72
-+#define CAM_CC_IPE_0_CLK				73
-+#define CAM_CC_IPE_0_CLK_SRC				74
-+#define CAM_CC_IPE_1_AHB_CLK				75
-+#define CAM_CC_IPE_1_AREG_CLK				76
-+#define CAM_CC_IPE_1_AXI_CLK				77
-+#define CAM_CC_IPE_1_CLK				78
-+#define CAM_CC_JPEG_CLK					79
-+#define CAM_CC_JPEG_CLK_SRC				80
-+#define CAM_CC_LRME_CLK					81
-+#define CAM_CC_LRME_CLK_SRC				82
-+#define CAM_CC_MCLK0_CLK				83
-+#define CAM_CC_MCLK0_CLK_SRC				84
-+#define CAM_CC_MCLK1_CLK				85
-+#define CAM_CC_MCLK1_CLK_SRC				86
-+#define CAM_CC_MCLK2_CLK				87
-+#define CAM_CC_MCLK2_CLK_SRC				88
-+#define CAM_CC_MCLK3_CLK				89
-+#define CAM_CC_MCLK3_CLK_SRC				90
-+#define CAM_CC_SLOW_AHB_CLK_SRC				91
-+
-+/* CAM_CC power domains */
-+#define TITAN_TOP_GDSC					0
-+#define BPS_GDSC					1
-+#define IFE_0_GDSC					2
-+#define IFE_1_GDSC					3
-+#define IPE_0_GDSC					4
-+#define IPE_1_GDSC					5
-+
-+/* CAM_CC resets */
-+#define CAM_CC_BPS_BCR					0
-+#define CAM_CC_CAMNOC_BCR				1
-+#define CAM_CC_CCI_BCR					2
-+#define CAM_CC_CPAS_BCR					3
-+#define CAM_CC_CSI0PHY_BCR				4
-+#define CAM_CC_CSI1PHY_BCR				5
-+#define CAM_CC_CSI2PHY_BCR				6
-+#define CAM_CC_CSI3PHY_BCR				7
-+#define CAM_CC_FD_BCR					8
-+#define CAM_CC_ICP_BCR					9
-+#define CAM_CC_IFE_0_BCR				10
-+#define CAM_CC_IFE_1_BCR				11
-+#define CAM_CC_IFE_LITE_0_BCR				12
-+#define CAM_CC_IFE_LITE_1_BCR				13
-+#define CAM_CC_IPE_0_BCR				14
-+#define CAM_CC_IPE_1_BCR				15
-+#define CAM_CC_JPEG_BCR					16
-+#define CAM_CC_LRME_BCR					17
-+#define CAM_CC_MCLK0_BCR				18
-+#define CAM_CC_MCLK1_BCR				19
-+#define CAM_CC_MCLK2_BCR				20
-+#define CAM_CC_MCLK3_BCR				21
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8350-videocc.h b/include/dt-bindings/clock/qcom,sm8350-videocc.h
-new file mode 100644
-index 000000000000..b6945a448676
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8350-videocc.h
-@@ -0,0 +1,35 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2019, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_SM8350_H
-+#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_SM8350_H
-+
-+/* Clocks */
-+#define VIDEO_CC_AHB_CLK_SRC					0
-+#define VIDEO_CC_MVS0_CLK					1
-+#define VIDEO_CC_MVS0_CLK_SRC					2
-+#define VIDEO_CC_MVS0_DIV_CLK_SRC				3
-+#define VIDEO_CC_MVS0C_CLK					4
-+#define VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC				5
-+#define VIDEO_CC_MVS1_CLK					6
-+#define VIDEO_CC_MVS1_CLK_SRC					7
-+#define VIDEO_CC_MVS1_DIV2_CLK					8
-+#define VIDEO_CC_MVS1_DIV_CLK_SRC				9
-+#define VIDEO_CC_MVS1C_CLK					10
-+#define VIDEO_CC_MVS1C_DIV2_DIV_CLK_SRC				11
-+#define VIDEO_CC_SLEEP_CLK					12
-+#define VIDEO_CC_SLEEP_CLK_SRC					13
-+#define VIDEO_CC_XO_CLK_SRC					14
-+#define VIDEO_PLL0						15
-+#define VIDEO_PLL1						16
-+
-+/* GDSCs */
-+#define MVS0C_GDSC						0
-+#define MVS1C_GDSC						1
-+#define MVS0_GDSC						2
-+#define MVS1_GDSC						3
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8450-dispcc.h b/include/dt-bindings/clock/qcom,sm8450-dispcc.h
-new file mode 100644
-index 000000000000..fd16ca894971
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8450-dispcc.h
-@@ -0,0 +1,103 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_SM8450_H
-+#define _DT_BINDINGS_CLK_QCOM_DISP_CC_SM8450_H
-+
-+/* DISP_CC clocks */
-+#define DISP_CC_MDSS_AHB1_CLK					0
-+#define DISP_CC_MDSS_AHB_CLK					1
-+#define DISP_CC_MDSS_AHB_CLK_SRC				2
-+#define DISP_CC_MDSS_BYTE0_CLK					3
-+#define DISP_CC_MDSS_BYTE0_CLK_SRC				4
-+#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC				5
-+#define DISP_CC_MDSS_BYTE0_INTF_CLK				6
-+#define DISP_CC_MDSS_BYTE1_CLK					7
-+#define DISP_CC_MDSS_BYTE1_CLK_SRC				8
-+#define DISP_CC_MDSS_BYTE1_DIV_CLK_SRC				9
-+#define DISP_CC_MDSS_BYTE1_INTF_CLK				10
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK				11
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK_SRC				12
-+#define DISP_CC_MDSS_DPTX0_CRYPTO_CLK				13
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK				14
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK_SRC				15
-+#define DISP_CC_MDSS_DPTX0_LINK_DIV_CLK_SRC			16
-+#define DISP_CC_MDSS_DPTX0_LINK_INTF_CLK			17
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK				18
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC			19
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK				20
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC			21
-+#define DISP_CC_MDSS_DPTX0_USB_ROUTER_LINK_INTF_CLK		22
-+#define DISP_CC_MDSS_DPTX1_AUX_CLK				23
-+#define DISP_CC_MDSS_DPTX1_AUX_CLK_SRC				24
-+#define DISP_CC_MDSS_DPTX1_CRYPTO_CLK				25
-+#define DISP_CC_MDSS_DPTX1_LINK_CLK				26
-+#define DISP_CC_MDSS_DPTX1_LINK_CLK_SRC				27
-+#define DISP_CC_MDSS_DPTX1_LINK_DIV_CLK_SRC			28
-+#define DISP_CC_MDSS_DPTX1_LINK_INTF_CLK			29
-+#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK				30
-+#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK_SRC			31
-+#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK				32
-+#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK_SRC			33
-+#define DISP_CC_MDSS_DPTX1_USB_ROUTER_LINK_INTF_CLK		34
-+#define DISP_CC_MDSS_DPTX2_AUX_CLK				35
-+#define DISP_CC_MDSS_DPTX2_AUX_CLK_SRC				36
-+#define DISP_CC_MDSS_DPTX2_CRYPTO_CLK				37
-+#define DISP_CC_MDSS_DPTX2_LINK_CLK				38
-+#define DISP_CC_MDSS_DPTX2_LINK_CLK_SRC				39
-+#define DISP_CC_MDSS_DPTX2_LINK_DIV_CLK_SRC			40
-+#define DISP_CC_MDSS_DPTX2_LINK_INTF_CLK			41
-+#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK				42
-+#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK_SRC			43
-+#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK				44
-+#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK_SRC			45
-+#define DISP_CC_MDSS_DPTX3_AUX_CLK				46
-+#define DISP_CC_MDSS_DPTX3_AUX_CLK_SRC				47
-+#define DISP_CC_MDSS_DPTX3_CRYPTO_CLK				48
-+#define DISP_CC_MDSS_DPTX3_LINK_CLK				49
-+#define DISP_CC_MDSS_DPTX3_LINK_CLK_SRC				50
-+#define DISP_CC_MDSS_DPTX3_LINK_DIV_CLK_SRC			51
-+#define DISP_CC_MDSS_DPTX3_LINK_INTF_CLK			52
-+#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK				53
-+#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK_SRC			54
-+#define DISP_CC_MDSS_ESC0_CLK					55
-+#define DISP_CC_MDSS_ESC0_CLK_SRC				56
-+#define DISP_CC_MDSS_ESC1_CLK					57
-+#define DISP_CC_MDSS_ESC1_CLK_SRC				58
-+#define DISP_CC_MDSS_MDP1_CLK					59
-+#define DISP_CC_MDSS_MDP_CLK					60
-+#define DISP_CC_MDSS_MDP_CLK_SRC				61
-+#define DISP_CC_MDSS_MDP_LUT1_CLK				62
-+#define DISP_CC_MDSS_MDP_LUT_CLK				63
-+#define DISP_CC_MDSS_NON_GDSC_AHB_CLK				64
-+#define DISP_CC_MDSS_PCLK0_CLK					65
-+#define DISP_CC_MDSS_PCLK0_CLK_SRC				66
-+#define DISP_CC_MDSS_PCLK1_CLK					67
-+#define DISP_CC_MDSS_PCLK1_CLK_SRC				68
-+#define DISP_CC_MDSS_ROT1_CLK					69
-+#define DISP_CC_MDSS_ROT_CLK					70
-+#define DISP_CC_MDSS_ROT_CLK_SRC				71
-+#define DISP_CC_MDSS_RSCC_AHB_CLK				72
-+#define DISP_CC_MDSS_RSCC_VSYNC_CLK				73
-+#define DISP_CC_MDSS_VSYNC1_CLK					74
-+#define DISP_CC_MDSS_VSYNC_CLK					75
-+#define DISP_CC_MDSS_VSYNC_CLK_SRC				76
-+#define DISP_CC_PLL0						77
-+#define DISP_CC_PLL1						78
-+#define DISP_CC_SLEEP_CLK					79
-+#define DISP_CC_SLEEP_CLK_SRC					80
-+#define DISP_CC_XO_CLK						81
-+#define DISP_CC_XO_CLK_SRC					82
-+
-+/* DISP_CC resets */
-+#define DISP_CC_MDSS_CORE_BCR					0
-+#define DISP_CC_MDSS_CORE_INT2_BCR				1
-+#define DISP_CC_MDSS_RSCC_BCR					2
-+
-+/* DISP_CC GDSCR */
-+#define MDSS_GDSC				0
-+#define MDSS_INT2_GDSC				1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8450-gpucc.h b/include/dt-bindings/clock/qcom,sm8450-gpucc.h
-new file mode 100644
-index 000000000000..712b171503d6
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8450-gpucc.h
-@@ -0,0 +1,48 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2020, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_SM8450_H
-+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_SM8450_H
-+
-+/* Clocks */
-+#define GPU_CC_AHB_CLK				0
-+#define GPU_CC_CRC_AHB_CLK			1
-+#define GPU_CC_CX_APB_CLK			2
-+#define GPU_CC_CX_FF_CLK			3
-+#define GPU_CC_CX_GMU_CLK			4
-+#define GPU_CC_CX_SNOC_DVM_CLK			5
-+#define GPU_CC_CXO_AON_CLK			6
-+#define GPU_CC_CXO_CLK				7
-+#define GPU_CC_DEMET_CLK			8
-+#define GPU_CC_DEMET_DIV_CLK_SRC		9
-+#define GPU_CC_FF_CLK_SRC			10
-+#define GPU_CC_FREQ_MEASURE_CLK			11
-+#define GPU_CC_GMU_CLK_SRC			12
-+#define GPU_CC_GX_FF_CLK			13
-+#define GPU_CC_GX_GFX3D_CLK			14
-+#define GPU_CC_GX_GFX3D_RDVM_CLK		15
-+#define GPU_CC_GX_GMU_CLK			16
-+#define GPU_CC_GX_VSENSE_CLK			17
-+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK		18
-+#define GPU_CC_HUB_AHB_DIV_CLK_SRC		19
-+#define GPU_CC_HUB_AON_CLK			20
-+#define GPU_CC_HUB_CLK_SRC			21
-+#define GPU_CC_HUB_CX_INT_CLK			22
-+#define GPU_CC_HUB_CX_INT_DIV_CLK_SRC		23
-+#define GPU_CC_MEMNOC_GFX_CLK			24
-+#define GPU_CC_MND1X_0_GFX3D_CLK		25
-+#define GPU_CC_MND1X_1_GFX3D_CLK		26
-+#define GPU_CC_PLL0				27
-+#define GPU_CC_PLL1				28
-+#define GPU_CC_SLEEP_CLK			29
-+#define GPU_CC_XO_CLK_SRC			30
-+#define GPU_CC_XO_DIV_CLK_SRC			31
-+
-+/* GDSCs */
-+#define GPU_GX_GDSC				0
-+#define GPU_CX_GDSC				1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8450-videocc.h b/include/dt-bindings/clock/qcom,sm8450-videocc.h
-new file mode 100644
-index 000000000000..9d795adfe4eb
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8450-videocc.h
-@@ -0,0 +1,38 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_SM8450_H
-+#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_SM8450_H
-+
-+/* VIDEO_CC clocks */
-+#define VIDEO_CC_MVS0_CLK					0
-+#define VIDEO_CC_MVS0_CLK_SRC					1
-+#define VIDEO_CC_MVS0_DIV_CLK_SRC				2
-+#define VIDEO_CC_MVS0C_CLK					3
-+#define VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC				4
-+#define VIDEO_CC_MVS1_CLK					5
-+#define VIDEO_CC_MVS1_CLK_SRC					6
-+#define VIDEO_CC_MVS1_DIV_CLK_SRC				7
-+#define VIDEO_CC_MVS1C_CLK					8
-+#define VIDEO_CC_MVS1C_DIV2_DIV_CLK_SRC				9
-+#define VIDEO_CC_PLL0						10
-+#define VIDEO_CC_PLL1						11
-+
-+/* VIDEO_CC power domains */
-+#define VIDEO_CC_MVS0C_GDSC					0
-+#define VIDEO_CC_MVS0_GDSC					1
-+#define VIDEO_CC_MVS1C_GDSC					2
-+#define VIDEO_CC_MVS1_GDSC					3
-+
-+/* VIDEO_CC resets */
-+#define CVP_VIDEO_CC_INTERFACE_BCR				0
-+#define CVP_VIDEO_CC_MVS0_BCR					1
-+#define CVP_VIDEO_CC_MVS0C_BCR					2
-+#define CVP_VIDEO_CC_MVS1_BCR					3
-+#define CVP_VIDEO_CC_MVS1C_BCR					4
-+#define VIDEO_CC_MVS0C_CLK_ARES					5
-+#define VIDEO_CC_MVS1C_CLK_ARES					6
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8550-camcc.h b/include/dt-bindings/clock/qcom,sm8550-camcc.h
-new file mode 100644
-index 000000000000..a2a256691c2b
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8550-camcc.h
-@@ -0,0 +1,187 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_SM8550_H
-+#define _DT_BINDINGS_CLK_QCOM_CAM_CC_SM8550_H
-+
-+/* CAM_CC clocks */
-+#define CAM_CC_BPS_AHB_CLK					0
-+#define CAM_CC_BPS_CLK						1
-+#define CAM_CC_BPS_CLK_SRC					2
-+#define CAM_CC_BPS_FAST_AHB_CLK					3
-+#define CAM_CC_CAMNOC_AXI_CLK					4
-+#define CAM_CC_CAMNOC_AXI_CLK_SRC				5
-+#define CAM_CC_CAMNOC_DCD_XO_CLK				6
-+#define CAM_CC_CAMNOC_XO_CLK					7
-+#define CAM_CC_CCI_0_CLK					8
-+#define CAM_CC_CCI_0_CLK_SRC					9
-+#define CAM_CC_CCI_1_CLK					10
-+#define CAM_CC_CCI_1_CLK_SRC					11
-+#define CAM_CC_CCI_2_CLK					12
-+#define CAM_CC_CCI_2_CLK_SRC					13
-+#define CAM_CC_CORE_AHB_CLK					14
-+#define CAM_CC_CPAS_AHB_CLK					15
-+#define CAM_CC_CPAS_BPS_CLK					16
-+#define CAM_CC_CPAS_CRE_CLK					17
-+#define CAM_CC_CPAS_FAST_AHB_CLK				18
-+#define CAM_CC_CPAS_IFE_0_CLK					19
-+#define CAM_CC_CPAS_IFE_1_CLK					20
-+#define CAM_CC_CPAS_IFE_2_CLK					21
-+#define CAM_CC_CPAS_IFE_LITE_CLK				22
-+#define CAM_CC_CPAS_IPE_NPS_CLK					23
-+#define CAM_CC_CPAS_SBI_CLK					24
-+#define CAM_CC_CPAS_SFE_0_CLK					25
-+#define CAM_CC_CPAS_SFE_1_CLK					26
-+#define CAM_CC_CPHY_RX_CLK_SRC					27
-+#define CAM_CC_CRE_AHB_CLK					28
-+#define CAM_CC_CRE_CLK						29
-+#define CAM_CC_CRE_CLK_SRC					30
-+#define CAM_CC_CSI0PHYTIMER_CLK					31
-+#define CAM_CC_CSI0PHYTIMER_CLK_SRC				32
-+#define CAM_CC_CSI1PHYTIMER_CLK					33
-+#define CAM_CC_CSI1PHYTIMER_CLK_SRC				34
-+#define CAM_CC_CSI2PHYTIMER_CLK					35
-+#define CAM_CC_CSI2PHYTIMER_CLK_SRC				36
-+#define CAM_CC_CSI3PHYTIMER_CLK					37
-+#define CAM_CC_CSI3PHYTIMER_CLK_SRC				38
-+#define CAM_CC_CSI4PHYTIMER_CLK					39
-+#define CAM_CC_CSI4PHYTIMER_CLK_SRC				40
-+#define CAM_CC_CSI5PHYTIMER_CLK					41
-+#define CAM_CC_CSI5PHYTIMER_CLK_SRC				42
-+#define CAM_CC_CSI6PHYTIMER_CLK					43
-+#define CAM_CC_CSI6PHYTIMER_CLK_SRC				44
-+#define CAM_CC_CSI7PHYTIMER_CLK					45
-+#define CAM_CC_CSI7PHYTIMER_CLK_SRC				46
-+#define CAM_CC_CSID_CLK						47
-+#define CAM_CC_CSID_CLK_SRC					48
-+#define CAM_CC_CSID_CSIPHY_RX_CLK				49
-+#define CAM_CC_CSIPHY0_CLK					50
-+#define CAM_CC_CSIPHY1_CLK					51
-+#define CAM_CC_CSIPHY2_CLK					52
-+#define CAM_CC_CSIPHY3_CLK					53
-+#define CAM_CC_CSIPHY4_CLK					54
-+#define CAM_CC_CSIPHY5_CLK					55
-+#define CAM_CC_CSIPHY6_CLK					56
-+#define CAM_CC_CSIPHY7_CLK					57
-+#define CAM_CC_DRV_AHB_CLK					58
-+#define CAM_CC_DRV_XO_CLK					59
-+#define CAM_CC_FAST_AHB_CLK_SRC					60
-+#define CAM_CC_GDSC_CLK						61
-+#define CAM_CC_ICP_AHB_CLK					62
-+#define CAM_CC_ICP_CLK						63
-+#define CAM_CC_ICP_CLK_SRC					64
-+#define CAM_CC_IFE_0_CLK					65
-+#define CAM_CC_IFE_0_CLK_SRC					66
-+#define CAM_CC_IFE_0_DSP_CLK					67
-+#define CAM_CC_IFE_0_DSP_CLK_SRC				68
-+#define CAM_CC_IFE_0_FAST_AHB_CLK				69
-+#define CAM_CC_IFE_1_CLK					70
-+#define CAM_CC_IFE_1_CLK_SRC					71
-+#define CAM_CC_IFE_1_DSP_CLK					72
-+#define CAM_CC_IFE_1_DSP_CLK_SRC				73
-+#define CAM_CC_IFE_1_FAST_AHB_CLK				74
-+#define CAM_CC_IFE_2_CLK					75
-+#define CAM_CC_IFE_2_CLK_SRC					76
-+#define CAM_CC_IFE_2_DSP_CLK					77
-+#define CAM_CC_IFE_2_DSP_CLK_SRC				78
-+#define CAM_CC_IFE_2_FAST_AHB_CLK				79
-+#define CAM_CC_IFE_LITE_AHB_CLK					80
-+#define CAM_CC_IFE_LITE_CLK					81
-+#define CAM_CC_IFE_LITE_CLK_SRC					82
-+#define CAM_CC_IFE_LITE_CPHY_RX_CLK				83
-+#define CAM_CC_IFE_LITE_CSID_CLK				84
-+#define CAM_CC_IFE_LITE_CSID_CLK_SRC				85
-+#define CAM_CC_IPE_NPS_AHB_CLK					86
-+#define CAM_CC_IPE_NPS_CLK					87
-+#define CAM_CC_IPE_NPS_CLK_SRC					88
-+#define CAM_CC_IPE_NPS_FAST_AHB_CLK				89
-+#define CAM_CC_IPE_PPS_CLK					90
-+#define CAM_CC_IPE_PPS_FAST_AHB_CLK				91
-+#define CAM_CC_JPEG_1_CLK					92
-+#define CAM_CC_JPEG_CLK						93
-+#define CAM_CC_JPEG_CLK_SRC					94
-+#define CAM_CC_MCLK0_CLK					95
-+#define CAM_CC_MCLK0_CLK_SRC					96
-+#define CAM_CC_MCLK1_CLK					97
-+#define CAM_CC_MCLK1_CLK_SRC					98
-+#define CAM_CC_MCLK2_CLK					99
-+#define CAM_CC_MCLK2_CLK_SRC					100
-+#define CAM_CC_MCLK3_CLK					101
-+#define CAM_CC_MCLK3_CLK_SRC					102
-+#define CAM_CC_MCLK4_CLK					103
-+#define CAM_CC_MCLK4_CLK_SRC					104
-+#define CAM_CC_MCLK5_CLK					105
-+#define CAM_CC_MCLK5_CLK_SRC					106
-+#define CAM_CC_MCLK6_CLK					107
-+#define CAM_CC_MCLK6_CLK_SRC					108
-+#define CAM_CC_MCLK7_CLK					109
-+#define CAM_CC_MCLK7_CLK_SRC					110
-+#define CAM_CC_PLL0						111
-+#define CAM_CC_PLL0_OUT_EVEN					112
-+#define CAM_CC_PLL0_OUT_ODD					113
-+#define CAM_CC_PLL1						114
-+#define CAM_CC_PLL1_OUT_EVEN					115
-+#define CAM_CC_PLL2						116
-+#define CAM_CC_PLL3						117
-+#define CAM_CC_PLL3_OUT_EVEN					118
-+#define CAM_CC_PLL4						119
-+#define CAM_CC_PLL4_OUT_EVEN					120
-+#define CAM_CC_PLL5						121
-+#define CAM_CC_PLL5_OUT_EVEN					122
-+#define CAM_CC_PLL6						123
-+#define CAM_CC_PLL6_OUT_EVEN					124
-+#define CAM_CC_PLL7						125
-+#define CAM_CC_PLL7_OUT_EVEN					126
-+#define CAM_CC_PLL8						127
-+#define CAM_CC_PLL8_OUT_EVEN					128
-+#define CAM_CC_PLL9						129
-+#define CAM_CC_PLL9_OUT_EVEN					130
-+#define CAM_CC_PLL10						131
-+#define CAM_CC_PLL10_OUT_EVEN					132
-+#define CAM_CC_PLL11						133
-+#define CAM_CC_PLL11_OUT_EVEN					134
-+#define CAM_CC_PLL12						135
-+#define CAM_CC_PLL12_OUT_EVEN					136
-+#define CAM_CC_QDSS_DEBUG_CLK					137
-+#define CAM_CC_QDSS_DEBUG_CLK_SRC				138
-+#define CAM_CC_QDSS_DEBUG_XO_CLK				139
-+#define CAM_CC_SBI_CLK						140
-+#define CAM_CC_SBI_FAST_AHB_CLK					141
-+#define CAM_CC_SFE_0_CLK					142
-+#define CAM_CC_SFE_0_CLK_SRC					143
-+#define CAM_CC_SFE_0_FAST_AHB_CLK				144
-+#define CAM_CC_SFE_1_CLK					145
-+#define CAM_CC_SFE_1_CLK_SRC					146
-+#define CAM_CC_SFE_1_FAST_AHB_CLK				147
-+#define CAM_CC_SLEEP_CLK					148
-+#define CAM_CC_SLEEP_CLK_SRC					149
-+#define CAM_CC_SLOW_AHB_CLK_SRC					150
-+#define CAM_CC_XO_CLK_SRC					151
-+
-+/* CAM_CC power domains */
-+#define CAM_CC_BPS_GDSC						0
-+#define CAM_CC_IFE_0_GDSC					1
-+#define CAM_CC_IFE_1_GDSC					2
-+#define CAM_CC_IFE_2_GDSC					3
-+#define CAM_CC_IPE_0_GDSC					4
-+#define CAM_CC_SBI_GDSC						5
-+#define CAM_CC_SFE_0_GDSC					6
-+#define CAM_CC_SFE_1_GDSC					7
-+#define CAM_CC_TITAN_TOP_GDSC					8
-+
-+/* CAM_CC resets */
-+#define CAM_CC_BPS_BCR						0
-+#define CAM_CC_DRV_BCR						1
-+#define CAM_CC_ICP_BCR						2
-+#define CAM_CC_IFE_0_BCR					3
-+#define CAM_CC_IFE_1_BCR					4
-+#define CAM_CC_IFE_2_BCR					5
-+#define CAM_CC_IPE_0_BCR					6
-+#define CAM_CC_QDSS_DEBUG_BCR					7
-+#define CAM_CC_SBI_BCR						8
-+#define CAM_CC_SFE_0_BCR					9
-+#define CAM_CC_SFE_1_BCR					10
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8550-dispcc.h b/include/dt-bindings/clock/qcom,sm8550-dispcc.h
-new file mode 100644
-index 000000000000..ed3094c694e0
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8550-dispcc.h
-@@ -0,0 +1,101 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_SM8550_DISP_CC_H
-+#define _DT_BINDINGS_CLK_QCOM_SM8550_DISP_CC_H
-+
-+/* DISP_CC clocks */
-+#define DISP_CC_MDSS_ACCU_CLK					0
-+#define DISP_CC_MDSS_AHB1_CLK					1
-+#define DISP_CC_MDSS_AHB_CLK					2
-+#define DISP_CC_MDSS_AHB_CLK_SRC				3
-+#define DISP_CC_MDSS_BYTE0_CLK					4
-+#define DISP_CC_MDSS_BYTE0_CLK_SRC				5
-+#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC				6
-+#define DISP_CC_MDSS_BYTE0_INTF_CLK				7
-+#define DISP_CC_MDSS_BYTE1_CLK					8
-+#define DISP_CC_MDSS_BYTE1_CLK_SRC				9
-+#define DISP_CC_MDSS_BYTE1_DIV_CLK_SRC				10
-+#define DISP_CC_MDSS_BYTE1_INTF_CLK				11
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK				12
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK_SRC				13
-+#define DISP_CC_MDSS_DPTX0_CRYPTO_CLK				14
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK				15
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK_SRC				16
-+#define DISP_CC_MDSS_DPTX0_LINK_DIV_CLK_SRC			17
-+#define DISP_CC_MDSS_DPTX0_LINK_INTF_CLK			18
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK				19
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC			20
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK				21
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC			22
-+#define DISP_CC_MDSS_DPTX0_USB_ROUTER_LINK_INTF_CLK		23
-+#define DISP_CC_MDSS_DPTX1_AUX_CLK				24
-+#define DISP_CC_MDSS_DPTX1_AUX_CLK_SRC				25
-+#define DISP_CC_MDSS_DPTX1_CRYPTO_CLK				26
-+#define DISP_CC_MDSS_DPTX1_LINK_CLK				27
-+#define DISP_CC_MDSS_DPTX1_LINK_CLK_SRC				28
-+#define DISP_CC_MDSS_DPTX1_LINK_DIV_CLK_SRC			29
-+#define DISP_CC_MDSS_DPTX1_LINK_INTF_CLK			30
-+#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK				31
-+#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK_SRC			32
-+#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK				33
-+#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK_SRC			34
-+#define DISP_CC_MDSS_DPTX1_USB_ROUTER_LINK_INTF_CLK		35
-+#define DISP_CC_MDSS_DPTX2_AUX_CLK				36
-+#define DISP_CC_MDSS_DPTX2_AUX_CLK_SRC				37
-+#define DISP_CC_MDSS_DPTX2_CRYPTO_CLK				38
-+#define DISP_CC_MDSS_DPTX2_LINK_CLK				39
-+#define DISP_CC_MDSS_DPTX2_LINK_CLK_SRC				40
-+#define DISP_CC_MDSS_DPTX2_LINK_DIV_CLK_SRC			41
-+#define DISP_CC_MDSS_DPTX2_LINK_INTF_CLK			42
-+#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK				43
-+#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK_SRC			44
-+#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK				45
-+#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK_SRC			46
-+#define DISP_CC_MDSS_DPTX3_AUX_CLK				47
-+#define DISP_CC_MDSS_DPTX3_AUX_CLK_SRC				48
-+#define DISP_CC_MDSS_DPTX3_CRYPTO_CLK				49
-+#define DISP_CC_MDSS_DPTX3_LINK_CLK				50
-+#define DISP_CC_MDSS_DPTX3_LINK_CLK_SRC				51
-+#define DISP_CC_MDSS_DPTX3_LINK_DIV_CLK_SRC			52
-+#define DISP_CC_MDSS_DPTX3_LINK_INTF_CLK			53
-+#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK				54
-+#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK_SRC			55
-+#define DISP_CC_MDSS_ESC0_CLK					56
-+#define DISP_CC_MDSS_ESC0_CLK_SRC				57
-+#define DISP_CC_MDSS_ESC1_CLK					58
-+#define DISP_CC_MDSS_ESC1_CLK_SRC				59
-+#define DISP_CC_MDSS_MDP1_CLK					60
-+#define DISP_CC_MDSS_MDP_CLK					61
-+#define DISP_CC_MDSS_MDP_CLK_SRC				62
-+#define DISP_CC_MDSS_MDP_LUT1_CLK				63
-+#define DISP_CC_MDSS_MDP_LUT_CLK				64
-+#define DISP_CC_MDSS_NON_GDSC_AHB_CLK				65
-+#define DISP_CC_MDSS_PCLK0_CLK					66
-+#define DISP_CC_MDSS_PCLK0_CLK_SRC				67
-+#define DISP_CC_MDSS_PCLK1_CLK					68
-+#define DISP_CC_MDSS_PCLK1_CLK_SRC				69
-+#define DISP_CC_MDSS_RSCC_AHB_CLK				70
-+#define DISP_CC_MDSS_RSCC_VSYNC_CLK				71
-+#define DISP_CC_MDSS_VSYNC1_CLK					72
-+#define DISP_CC_MDSS_VSYNC_CLK					73
-+#define DISP_CC_MDSS_VSYNC_CLK_SRC				74
-+#define DISP_CC_PLL0						75
-+#define DISP_CC_PLL1						76
-+#define DISP_CC_SLEEP_CLK					77
-+#define DISP_CC_SLEEP_CLK_SRC					78
-+#define DISP_CC_XO_CLK						79
-+#define DISP_CC_XO_CLK_SRC					80
-+
-+/* DISP_CC resets */
-+#define DISP_CC_MDSS_CORE_BCR					0
-+#define DISP_CC_MDSS_CORE_INT2_BCR				1
-+#define DISP_CC_MDSS_RSCC_BCR					2
-+
-+/* DISP_CC GDSCR */
-+#define MDSS_GDSC						0
-+#define MDSS_INT2_GDSC						1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8550-gcc.h b/include/dt-bindings/clock/qcom,sm8550-gcc.h
-new file mode 100644
-index 000000000000..3bf6f2b75c99
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8550-gcc.h
-@@ -0,0 +1,231 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2022, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_SM8550_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_SM8550_H
-+
-+/* GCC clocks */
-+#define GCC_AGGRE_NOC_PCIE_AXI_CLK				0
-+#define GCC_AGGRE_UFS_PHY_AXI_CLK				1
-+#define GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK			2
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK				3
-+#define GCC_AHB2PHY_0_CLK					4
-+#define GCC_BOOT_ROM_AHB_CLK					5
-+#define GCC_CAMERA_AHB_CLK					6
-+#define GCC_CAMERA_HF_AXI_CLK					7
-+#define GCC_CAMERA_SF_AXI_CLK					8
-+#define GCC_CAMERA_XO_CLK					9
-+#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK				10
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				11
-+#define GCC_CNOC_PCIE_SF_AXI_CLK				12
-+#define GCC_DDRSS_GPU_AXI_CLK					13
-+#define GCC_DDRSS_PCIE_SF_QTB_CLK				14
-+#define GCC_DISP_AHB_CLK					15
-+#define GCC_DISP_HF_AXI_CLK					16
-+#define GCC_DISP_XO_CLK						17
-+#define GCC_GP1_CLK						18
-+#define GCC_GP1_CLK_SRC						19
-+#define GCC_GP2_CLK						20
-+#define GCC_GP2_CLK_SRC						21
-+#define GCC_GP3_CLK						22
-+#define GCC_GP3_CLK_SRC						23
-+#define GCC_GPLL0						24
-+#define GCC_GPLL0_OUT_EVEN					25
-+#define GCC_GPLL4						26
-+#define GCC_GPLL7						27
-+#define GCC_GPLL9						28
-+#define GCC_GPU_CFG_AHB_CLK					29
-+#define GCC_GPU_GPLL0_CLK_SRC					30
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC				31
-+#define GCC_GPU_MEMNOC_GFX_CLK					32
-+#define GCC_GPU_SNOC_DVM_GFX_CLK				33
-+#define GCC_PCIE_0_AUX_CLK					34
-+#define GCC_PCIE_0_AUX_CLK_SRC					35
-+#define GCC_PCIE_0_CFG_AHB_CLK					36
-+#define GCC_PCIE_0_MSTR_AXI_CLK					37
-+#define GCC_PCIE_0_PHY_RCHNG_CLK				38
-+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC				39
-+#define GCC_PCIE_0_PIPE_CLK					40
-+#define GCC_PCIE_0_PIPE_CLK_SRC					41
-+#define GCC_PCIE_0_SLV_AXI_CLK					42
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				43
-+#define GCC_PCIE_1_AUX_CLK					44
-+#define GCC_PCIE_1_AUX_CLK_SRC					45
-+#define GCC_PCIE_1_CFG_AHB_CLK					46
-+#define GCC_PCIE_1_MSTR_AXI_CLK					47
-+#define GCC_PCIE_1_PHY_AUX_CLK					48
-+#define GCC_PCIE_1_PHY_AUX_CLK_SRC				49
-+#define GCC_PCIE_1_PHY_RCHNG_CLK				50
-+#define GCC_PCIE_1_PHY_RCHNG_CLK_SRC				51
-+#define GCC_PCIE_1_PIPE_CLK					52
-+#define GCC_PCIE_1_PIPE_CLK_SRC					53
-+#define GCC_PCIE_1_SLV_AXI_CLK					54
-+#define GCC_PCIE_1_SLV_Q2A_AXI_CLK				55
-+#define GCC_PDM2_CLK						56
-+#define GCC_PDM2_CLK_SRC					57
-+#define GCC_PDM_AHB_CLK						58
-+#define GCC_PDM_XO4_CLK						59
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				60
-+#define GCC_QMIP_CAMERA_RT_AHB_CLK				61
-+#define GCC_QMIP_DISP_AHB_CLK					62
-+#define GCC_QMIP_GPU_AHB_CLK					63
-+#define GCC_QMIP_PCIE_AHB_CLK					64
-+#define GCC_QMIP_VIDEO_CV_CPU_AHB_CLK				65
-+#define GCC_QMIP_VIDEO_CVP_AHB_CLK				66
-+#define GCC_QMIP_VIDEO_V_CPU_AHB_CLK				67
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				68
-+#define GCC_QUPV3_I2C_CORE_CLK					69
-+#define GCC_QUPV3_I2C_S0_CLK					70
-+#define GCC_QUPV3_I2C_S0_CLK_SRC				71
-+#define GCC_QUPV3_I2C_S1_CLK					72
-+#define GCC_QUPV3_I2C_S1_CLK_SRC				73
-+#define GCC_QUPV3_I2C_S2_CLK					74
-+#define GCC_QUPV3_I2C_S2_CLK_SRC				75
-+#define GCC_QUPV3_I2C_S3_CLK					76
-+#define GCC_QUPV3_I2C_S3_CLK_SRC				77
-+#define GCC_QUPV3_I2C_S4_CLK					78
-+#define GCC_QUPV3_I2C_S4_CLK_SRC				79
-+#define GCC_QUPV3_I2C_S5_CLK					80
-+#define GCC_QUPV3_I2C_S5_CLK_SRC				81
-+#define GCC_QUPV3_I2C_S6_CLK					82
-+#define GCC_QUPV3_I2C_S6_CLK_SRC				83
-+#define GCC_QUPV3_I2C_S7_CLK					84
-+#define GCC_QUPV3_I2C_S7_CLK_SRC				85
-+#define GCC_QUPV3_I2C_S8_CLK					86
-+#define GCC_QUPV3_I2C_S8_CLK_SRC				87
-+#define GCC_QUPV3_I2C_S9_CLK					88
-+#define GCC_QUPV3_I2C_S9_CLK_SRC				89
-+#define GCC_QUPV3_I2C_S_AHB_CLK					90
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				91
-+#define GCC_QUPV3_WRAP1_CORE_CLK				92
-+#define GCC_QUPV3_WRAP1_S0_CLK					93
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				94
-+#define GCC_QUPV3_WRAP1_S1_CLK					95
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				96
-+#define GCC_QUPV3_WRAP1_S2_CLK					97
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				98
-+#define GCC_QUPV3_WRAP1_S3_CLK					99
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				100
-+#define GCC_QUPV3_WRAP1_S4_CLK					101
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				102
-+#define GCC_QUPV3_WRAP1_S5_CLK					103
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC				104
-+#define GCC_QUPV3_WRAP1_S6_CLK					105
-+#define GCC_QUPV3_WRAP1_S6_CLK_SRC				106
-+#define GCC_QUPV3_WRAP1_S7_CLK					107
-+#define GCC_QUPV3_WRAP1_S7_CLK_SRC				108
-+#define GCC_QUPV3_WRAP2_CORE_2X_CLK				109
-+#define GCC_QUPV3_WRAP2_CORE_CLK				110
-+#define GCC_QUPV3_WRAP2_S0_CLK					111
-+#define GCC_QUPV3_WRAP2_S0_CLK_SRC				112
-+#define GCC_QUPV3_WRAP2_S1_CLK					113
-+#define GCC_QUPV3_WRAP2_S1_CLK_SRC				114
-+#define GCC_QUPV3_WRAP2_S2_CLK					115
-+#define GCC_QUPV3_WRAP2_S2_CLK_SRC				116
-+#define GCC_QUPV3_WRAP2_S3_CLK					117
-+#define GCC_QUPV3_WRAP2_S3_CLK_SRC				118
-+#define GCC_QUPV3_WRAP2_S4_CLK					119
-+#define GCC_QUPV3_WRAP2_S4_CLK_SRC				120
-+#define GCC_QUPV3_WRAP2_S5_CLK					121
-+#define GCC_QUPV3_WRAP2_S5_CLK_SRC				122
-+#define GCC_QUPV3_WRAP2_S6_CLK					123
-+#define GCC_QUPV3_WRAP2_S6_CLK_SRC				124
-+#define GCC_QUPV3_WRAP2_S7_CLK					125
-+#define GCC_QUPV3_WRAP2_S7_CLK_SRC				126
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK				127
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				128
-+#define GCC_QUPV3_WRAP_2_M_AHB_CLK				129
-+#define GCC_QUPV3_WRAP_2_S_AHB_CLK				130
-+#define GCC_SDCC2_AHB_CLK					131
-+#define GCC_SDCC2_APPS_CLK					132
-+#define GCC_SDCC2_APPS_CLK_SRC					133
-+#define GCC_SDCC4_AHB_CLK					134
-+#define GCC_SDCC4_APPS_CLK					135
-+#define GCC_SDCC4_APPS_CLK_SRC					136
-+#define GCC_UFS_PHY_AHB_CLK					137
-+#define GCC_UFS_PHY_AXI_CLK					138
-+#define GCC_UFS_PHY_AXI_CLK_SRC					139
-+#define GCC_UFS_PHY_AXI_HW_CTL_CLK				140
-+#define GCC_UFS_PHY_ICE_CORE_CLK				141
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC				142
-+#define GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK				143
-+#define GCC_UFS_PHY_PHY_AUX_CLK					144
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC				145
-+#define GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK				146
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK				147
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC				148
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK				149
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC				150
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK				151
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC				152
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK				153
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC				154
-+#define GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK			155
-+#define GCC_USB30_PRIM_MASTER_CLK				156
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC				157
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				158
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			159
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		160
-+#define GCC_USB30_PRIM_SLEEP_CLK				161
-+#define GCC_USB3_PRIM_PHY_AUX_CLK				162
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				163
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				164
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK				165
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				166
-+#define GCC_VIDEO_AHB_CLK					167
-+#define GCC_VIDEO_AXI0_CLK					168
-+#define GCC_VIDEO_AXI1_CLK					169
-+#define GCC_VIDEO_XO_CLK					170
-+
-+/* GCC resets */
-+#define GCC_CAMERA_BCR						0
-+#define GCC_DISPLAY_BCR						1
-+#define GCC_GPU_BCR						2
-+#define GCC_PCIE_0_BCR						3
-+#define GCC_PCIE_0_LINK_DOWN_BCR				4
-+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR				5
-+#define GCC_PCIE_0_PHY_BCR					6
-+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR			7
-+#define GCC_PCIE_1_BCR						8
-+#define GCC_PCIE_1_LINK_DOWN_BCR				9
-+#define GCC_PCIE_1_NOCSR_COM_PHY_BCR				10
-+#define GCC_PCIE_1_PHY_BCR					11
-+#define GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR			12
-+#define GCC_PCIE_PHY_BCR					13
-+#define GCC_PCIE_PHY_CFG_AHB_BCR				14
-+#define GCC_PCIE_PHY_COM_BCR					15
-+#define GCC_PDM_BCR						16
-+#define GCC_QUPV3_WRAPPER_1_BCR					17
-+#define GCC_QUPV3_WRAPPER_2_BCR					18
-+#define GCC_QUPV3_WRAPPER_I2C_BCR				19
-+#define GCC_QUSB2PHY_PRIM_BCR					20
-+#define GCC_QUSB2PHY_SEC_BCR					21
-+#define GCC_SDCC2_BCR						22
-+#define GCC_SDCC4_BCR						23
-+#define GCC_UFS_PHY_BCR						24
-+#define GCC_USB30_PRIM_BCR					25
-+#define GCC_USB3_DP_PHY_PRIM_BCR				26
-+#define GCC_USB3_DP_PHY_SEC_BCR					27
-+#define GCC_USB3_PHY_PRIM_BCR					28
-+#define GCC_USB3_PHY_SEC_BCR					29
-+#define GCC_USB3PHY_PHY_PRIM_BCR				30
-+#define GCC_USB3PHY_PHY_SEC_BCR					31
-+#define GCC_USB_PHY_CFG_AHB2PHY_BCR				32
-+#define GCC_VIDEO_AXI0_CLK_ARES					33
-+#define GCC_VIDEO_AXI1_CLK_ARES					34
-+#define GCC_VIDEO_BCR						35
-+
-+/* GCC power domains */
-+#define PCIE_0_GDSC						0
-+#define PCIE_0_PHY_GDSC						1
-+#define PCIE_1_GDSC						2
-+#define PCIE_1_PHY_GDSC						3
-+#define UFS_PHY_GDSC						4
-+#define UFS_MEM_PHY_GDSC					5
-+#define USB30_PRIM_GDSC						6
-+#define USB3_PHY_GDSC						7
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8550-gpucc.h b/include/dt-bindings/clock/qcom,sm8550-gpucc.h
-new file mode 100644
-index 000000000000..a6760547a3ab
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8550-gpucc.h
-@@ -0,0 +1,48 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_SM8550_H
-+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_SM8550_H
-+
-+/* GPU_CC clocks */
-+#define GPU_CC_AHB_CLK						0
-+#define GPU_CC_CRC_AHB_CLK					1
-+#define GPU_CC_CX_FF_CLK					2
-+#define GPU_CC_CX_GMU_CLK					3
-+#define GPU_CC_CXO_AON_CLK					4
-+#define GPU_CC_CXO_CLK						5
-+#define GPU_CC_DEMET_CLK					6
-+#define GPU_CC_DEMET_DIV_CLK_SRC				7
-+#define GPU_CC_FF_CLK_SRC					8
-+#define GPU_CC_FREQ_MEASURE_CLK					9
-+#define GPU_CC_GMU_CLK_SRC					10
-+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK				11
-+#define GPU_CC_HUB_AON_CLK					12
-+#define GPU_CC_HUB_CLK_SRC					13
-+#define GPU_CC_HUB_CX_INT_CLK					14
-+#define GPU_CC_MEMNOC_GFX_CLK					15
-+#define GPU_CC_MND1X_0_GFX3D_CLK				16
-+#define GPU_CC_MND1X_1_GFX3D_CLK				17
-+#define GPU_CC_PLL0						18
-+#define GPU_CC_PLL1						19
-+#define GPU_CC_SLEEP_CLK					20
-+#define GPU_CC_XO_CLK_SRC					21
-+#define GPU_CC_XO_DIV_CLK_SRC					22
-+
-+/* GPU_CC power domains */
-+#define GPU_CC_CX_GDSC						0
-+#define GPU_CC_GX_GDSC						1
-+
-+/* GPU_CC resets */
-+#define GPUCC_GPU_CC_ACD_BCR					0
-+#define GPUCC_GPU_CC_CX_BCR					1
-+#define GPUCC_GPU_CC_FAST_HUB_BCR				2
-+#define GPUCC_GPU_CC_FF_BCR					3
-+#define GPUCC_GPU_CC_GFX3D_AON_BCR				4
-+#define GPUCC_GPU_CC_GMU_BCR					5
-+#define GPUCC_GPU_CC_GX_BCR					6
-+#define GPUCC_GPU_CC_XO_BCR					7
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8550-tcsr.h b/include/dt-bindings/clock/qcom,sm8550-tcsr.h
-new file mode 100644
-index 000000000000..091cb76f953a
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8550-tcsr.h
-@@ -0,0 +1,18 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2022, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_TCSR_CC_SM8550_H
-+#define _DT_BINDINGS_CLK_QCOM_TCSR_CC_SM8550_H
-+
-+/* TCSR CC clocks */
-+#define TCSR_PCIE_0_CLKREF_EN					0
-+#define TCSR_PCIE_1_CLKREF_EN					1
-+#define TCSR_UFS_CLKREF_EN					2
-+#define TCSR_UFS_PAD_CLKREF_EN					3
-+#define TCSR_USB2_CLKREF_EN					4
-+#define TCSR_USB3_CLKREF_EN					5
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8650-camcc.h b/include/dt-bindings/clock/qcom,sm8650-camcc.h
-new file mode 100644
-index 000000000000..df73bf35f4bf
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8650-camcc.h
-@@ -0,0 +1,195 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_SM8650_H
-+#define _DT_BINDINGS_CLK_QCOM_CAM_CC_SM8650_H
-+
-+/* CAM_CC clocks */
-+#define CAM_CC_BPS_AHB_CLK					0
-+#define CAM_CC_BPS_CLK						1
-+#define CAM_CC_BPS_CLK_SRC					2
-+#define CAM_CC_BPS_FAST_AHB_CLK					3
-+#define CAM_CC_BPS_SHIFT_CLK					4
-+#define CAM_CC_CAMNOC_AXI_NRT_CLK				5
-+#define CAM_CC_CAMNOC_AXI_RT_CLK				6
-+#define CAM_CC_CAMNOC_AXI_RT_CLK_SRC				7
-+#define CAM_CC_CAMNOC_DCD_XO_CLK				8
-+#define CAM_CC_CAMNOC_XO_CLK					9
-+#define CAM_CC_CCI_0_CLK					10
-+#define CAM_CC_CCI_0_CLK_SRC					11
-+#define CAM_CC_CCI_1_CLK					12
-+#define CAM_CC_CCI_1_CLK_SRC					13
-+#define CAM_CC_CCI_2_CLK					14
-+#define CAM_CC_CCI_2_CLK_SRC					15
-+#define CAM_CC_CORE_AHB_CLK					16
-+#define CAM_CC_CPAS_AHB_CLK					17
-+#define CAM_CC_CPAS_BPS_CLK					18
-+#define CAM_CC_CPAS_CRE_CLK					19
-+#define CAM_CC_CPAS_FAST_AHB_CLK				20
-+#define CAM_CC_CPAS_IFE_0_CLK					21
-+#define CAM_CC_CPAS_IFE_1_CLK					22
-+#define CAM_CC_CPAS_IFE_2_CLK					23
-+#define CAM_CC_CPAS_IFE_LITE_CLK				24
-+#define CAM_CC_CPAS_IPE_NPS_CLK					25
-+#define CAM_CC_CPAS_SBI_CLK					26
-+#define CAM_CC_CPAS_SFE_0_CLK					27
-+#define CAM_CC_CPAS_SFE_1_CLK					28
-+#define CAM_CC_CPAS_SFE_2_CLK					29
-+#define CAM_CC_CPHY_RX_CLK_SRC					30
-+#define CAM_CC_CRE_AHB_CLK					31
-+#define CAM_CC_CRE_CLK						32
-+#define CAM_CC_CRE_CLK_SRC					33
-+#define CAM_CC_CSI0PHYTIMER_CLK					34
-+#define CAM_CC_CSI0PHYTIMER_CLK_SRC				35
-+#define CAM_CC_CSI1PHYTIMER_CLK					36
-+#define CAM_CC_CSI1PHYTIMER_CLK_SRC				37
-+#define CAM_CC_CSI2PHYTIMER_CLK					38
-+#define CAM_CC_CSI2PHYTIMER_CLK_SRC				39
-+#define CAM_CC_CSI3PHYTIMER_CLK					40
-+#define CAM_CC_CSI3PHYTIMER_CLK_SRC				41
-+#define CAM_CC_CSI4PHYTIMER_CLK					42
-+#define CAM_CC_CSI4PHYTIMER_CLK_SRC				43
-+#define CAM_CC_CSI5PHYTIMER_CLK					44
-+#define CAM_CC_CSI5PHYTIMER_CLK_SRC				45
-+#define CAM_CC_CSI6PHYTIMER_CLK					46
-+#define CAM_CC_CSI6PHYTIMER_CLK_SRC				47
-+#define CAM_CC_CSI7PHYTIMER_CLK					48
-+#define CAM_CC_CSI7PHYTIMER_CLK_SRC				49
-+#define CAM_CC_CSID_CLK						50
-+#define CAM_CC_CSID_CLK_SRC					51
-+#define CAM_CC_CSID_CSIPHY_RX_CLK				52
-+#define CAM_CC_CSIPHY0_CLK					53
-+#define CAM_CC_CSIPHY1_CLK					54
-+#define CAM_CC_CSIPHY2_CLK					55
-+#define CAM_CC_CSIPHY3_CLK					56
-+#define CAM_CC_CSIPHY4_CLK					57
-+#define CAM_CC_CSIPHY5_CLK					58
-+#define CAM_CC_CSIPHY6_CLK					59
-+#define CAM_CC_CSIPHY7_CLK					60
-+#define CAM_CC_DRV_AHB_CLK					61
-+#define CAM_CC_DRV_XO_CLK					62
-+#define CAM_CC_FAST_AHB_CLK_SRC					63
-+#define CAM_CC_GDSC_CLK						64
-+#define CAM_CC_ICP_AHB_CLK					65
-+#define CAM_CC_ICP_CLK						66
-+#define CAM_CC_ICP_CLK_SRC					67
-+#define CAM_CC_IFE_0_CLK					68
-+#define CAM_CC_IFE_0_CLK_SRC					69
-+#define CAM_CC_IFE_0_FAST_AHB_CLK				70
-+#define CAM_CC_IFE_0_SHIFT_CLK					71
-+#define CAM_CC_IFE_1_CLK					72
-+#define CAM_CC_IFE_1_CLK_SRC					73
-+#define CAM_CC_IFE_1_FAST_AHB_CLK				74
-+#define CAM_CC_IFE_1_SHIFT_CLK					75
-+#define CAM_CC_IFE_2_CLK					76
-+#define CAM_CC_IFE_2_CLK_SRC					77
-+#define CAM_CC_IFE_2_FAST_AHB_CLK				78
-+#define CAM_CC_IFE_2_SHIFT_CLK					79
-+#define CAM_CC_IFE_LITE_AHB_CLK					80
-+#define CAM_CC_IFE_LITE_CLK					81
-+#define CAM_CC_IFE_LITE_CLK_SRC					82
-+#define CAM_CC_IFE_LITE_CPHY_RX_CLK				83
-+#define CAM_CC_IFE_LITE_CSID_CLK				84
-+#define CAM_CC_IFE_LITE_CSID_CLK_SRC				85
-+#define CAM_CC_IPE_NPS_AHB_CLK					86
-+#define CAM_CC_IPE_NPS_CLK					87
-+#define CAM_CC_IPE_NPS_CLK_SRC					88
-+#define CAM_CC_IPE_NPS_FAST_AHB_CLK				89
-+#define CAM_CC_IPE_PPS_CLK					90
-+#define CAM_CC_IPE_PPS_FAST_AHB_CLK				91
-+#define CAM_CC_IPE_SHIFT_CLK					92
-+#define CAM_CC_JPEG_1_CLK					93
-+#define CAM_CC_JPEG_CLK						94
-+#define CAM_CC_JPEG_CLK_SRC					95
-+#define CAM_CC_MCLK0_CLK					96
-+#define CAM_CC_MCLK0_CLK_SRC					97
-+#define CAM_CC_MCLK1_CLK					98
-+#define CAM_CC_MCLK1_CLK_SRC					99
-+#define CAM_CC_MCLK2_CLK					100
-+#define CAM_CC_MCLK2_CLK_SRC					101
-+#define CAM_CC_MCLK3_CLK					102
-+#define CAM_CC_MCLK3_CLK_SRC					103
-+#define CAM_CC_MCLK4_CLK					104
-+#define CAM_CC_MCLK4_CLK_SRC					105
-+#define CAM_CC_MCLK5_CLK					106
-+#define CAM_CC_MCLK5_CLK_SRC					107
-+#define CAM_CC_MCLK6_CLK					108
-+#define CAM_CC_MCLK6_CLK_SRC					109
-+#define CAM_CC_MCLK7_CLK					110
-+#define CAM_CC_MCLK7_CLK_SRC					111
-+#define CAM_CC_PLL0						112
-+#define CAM_CC_PLL0_OUT_EVEN					113
-+#define CAM_CC_PLL0_OUT_ODD					114
-+#define CAM_CC_PLL1						115
-+#define CAM_CC_PLL1_OUT_EVEN					116
-+#define CAM_CC_PLL2						117
-+#define CAM_CC_PLL3						118
-+#define CAM_CC_PLL3_OUT_EVEN					119
-+#define CAM_CC_PLL4						120
-+#define CAM_CC_PLL4_OUT_EVEN					121
-+#define CAM_CC_PLL5						122
-+#define CAM_CC_PLL5_OUT_EVEN					123
-+#define CAM_CC_PLL6						124
-+#define CAM_CC_PLL6_OUT_EVEN					125
-+#define CAM_CC_PLL7						126
-+#define CAM_CC_PLL7_OUT_EVEN					127
-+#define CAM_CC_PLL8						128
-+#define CAM_CC_PLL8_OUT_EVEN					129
-+#define CAM_CC_PLL9						130
-+#define CAM_CC_PLL9_OUT_EVEN					131
-+#define CAM_CC_PLL9_OUT_ODD					132
-+#define CAM_CC_PLL10						133
-+#define CAM_CC_PLL10_OUT_EVEN					134
-+#define CAM_CC_QDSS_DEBUG_CLK					135
-+#define CAM_CC_QDSS_DEBUG_CLK_SRC				136
-+#define CAM_CC_QDSS_DEBUG_XO_CLK				137
-+#define CAM_CC_SBI_CLK						138
-+#define CAM_CC_SBI_FAST_AHB_CLK					139
-+#define CAM_CC_SBI_SHIFT_CLK					140
-+#define CAM_CC_SFE_0_CLK					141
-+#define CAM_CC_SFE_0_CLK_SRC					142
-+#define CAM_CC_SFE_0_FAST_AHB_CLK				143
-+#define CAM_CC_SFE_0_SHIFT_CLK					144
-+#define CAM_CC_SFE_1_CLK					145
-+#define CAM_CC_SFE_1_CLK_SRC					146
-+#define CAM_CC_SFE_1_FAST_AHB_CLK				147
-+#define CAM_CC_SFE_1_SHIFT_CLK					148
-+#define CAM_CC_SFE_2_CLK					149
-+#define CAM_CC_SFE_2_CLK_SRC					150
-+#define CAM_CC_SFE_2_FAST_AHB_CLK				151
-+#define CAM_CC_SFE_2_SHIFT_CLK					152
-+#define CAM_CC_SLEEP_CLK					153
-+#define CAM_CC_SLEEP_CLK_SRC					154
-+#define CAM_CC_SLOW_AHB_CLK_SRC					155
-+#define CAM_CC_TITAN_TOP_SHIFT_CLK				156
-+#define CAM_CC_XO_CLK_SRC					157
-+
-+/* CAM_CC power domains */
-+#define CAM_CC_TITAN_TOP_GDSC					0
-+#define CAM_CC_BPS_GDSC						1
-+#define CAM_CC_IFE_0_GDSC					2
-+#define CAM_CC_IFE_1_GDSC					3
-+#define CAM_CC_IFE_2_GDSC					4
-+#define CAM_CC_IPE_0_GDSC					5
-+#define CAM_CC_SBI_GDSC						6
-+#define CAM_CC_SFE_0_GDSC					7
-+#define CAM_CC_SFE_1_GDSC					8
-+#define CAM_CC_SFE_2_GDSC					9
-+
-+/* CAM_CC resets */
-+#define CAM_CC_BPS_BCR						0
-+#define CAM_CC_DRV_BCR						1
-+#define CAM_CC_ICP_BCR						2
-+#define CAM_CC_IFE_0_BCR					3
-+#define CAM_CC_IFE_1_BCR					4
-+#define CAM_CC_IFE_2_BCR					5
-+#define CAM_CC_IPE_0_BCR					6
-+#define CAM_CC_QDSS_DEBUG_BCR					7
-+#define CAM_CC_SBI_BCR						8
-+#define CAM_CC_SFE_0_BCR					9
-+#define CAM_CC_SFE_1_BCR					10
-+#define CAM_CC_SFE_2_BCR					11
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8650-dispcc.h b/include/dt-bindings/clock/qcom,sm8650-dispcc.h
-new file mode 120000
-index 000000000000..c0a291188f28
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8650-dispcc.h
-@@ -0,0 +1 @@
-+qcom,sm8550-dispcc.h
-\ No newline at end of file
-diff --git a/include/dt-bindings/clock/qcom,sm8650-gcc.h b/include/dt-bindings/clock/qcom,sm8650-gcc.h
-new file mode 100644
-index 000000000000..0c543ba46079
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8650-gcc.h
-@@ -0,0 +1,254 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_SM8650_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_SM8650_H
-+
-+/* GCC clocks */
-+#define GCC_AGGRE_NOC_PCIE_AXI_CLK				0
-+#define GCC_AGGRE_UFS_PHY_AXI_CLK				1
-+#define GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK			2
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK				3
-+#define GCC_BOOT_ROM_AHB_CLK					4
-+#define GCC_CAMERA_AHB_CLK					5
-+#define GCC_CAMERA_HF_AXI_CLK					6
-+#define GCC_CAMERA_SF_AXI_CLK					7
-+#define GCC_CAMERA_XO_CLK					8
-+#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK				9
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				10
-+#define GCC_CNOC_PCIE_SF_AXI_CLK				11
-+#define GCC_DDRSS_GPU_AXI_CLK					12
-+#define GCC_DDRSS_PCIE_SF_QTB_CLK				13
-+#define GCC_DISP_AHB_CLK					14
-+#define GCC_DISP_HF_AXI_CLK					15
-+#define GCC_DISP_XO_CLK						16
-+#define GCC_GP1_CLK						17
-+#define GCC_GP1_CLK_SRC						18
-+#define GCC_GP2_CLK						19
-+#define GCC_GP2_CLK_SRC						20
-+#define GCC_GP3_CLK						21
-+#define GCC_GP3_CLK_SRC						22
-+#define GCC_GPLL0						23
-+#define GCC_GPLL0_OUT_EVEN					24
-+#define GCC_GPLL1						25
-+#define GCC_GPLL3						26
-+#define GCC_GPLL4						27
-+#define GCC_GPLL6						28
-+#define GCC_GPLL7						29
-+#define GCC_GPLL9						30
-+#define GCC_GPU_CFG_AHB_CLK					31
-+#define GCC_GPU_GPLL0_CLK_SRC					32
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC				33
-+#define GCC_GPU_MEMNOC_GFX_CLK					34
-+#define GCC_GPU_SNOC_DVM_GFX_CLK				35
-+#define GCC_PCIE_0_AUX_CLK					36
-+#define GCC_PCIE_0_AUX_CLK_SRC					37
-+#define GCC_PCIE_0_CFG_AHB_CLK					38
-+#define GCC_PCIE_0_MSTR_AXI_CLK					39
-+#define GCC_PCIE_0_PHY_RCHNG_CLK				40
-+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC				41
-+#define GCC_PCIE_0_PIPE_CLK					42
-+#define GCC_PCIE_0_PIPE_CLK_SRC					43
-+#define GCC_PCIE_0_SLV_AXI_CLK					44
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				45
-+#define GCC_PCIE_1_AUX_CLK					46
-+#define GCC_PCIE_1_AUX_CLK_SRC					47
-+#define GCC_PCIE_1_CFG_AHB_CLK					48
-+#define GCC_PCIE_1_MSTR_AXI_CLK					49
-+#define GCC_PCIE_1_PHY_AUX_CLK					50
-+#define GCC_PCIE_1_PHY_AUX_CLK_SRC				51
-+#define GCC_PCIE_1_PHY_RCHNG_CLK				52
-+#define GCC_PCIE_1_PHY_RCHNG_CLK_SRC				53
-+#define GCC_PCIE_1_PIPE_CLK					54
-+#define GCC_PCIE_1_PIPE_CLK_SRC					55
-+#define GCC_PCIE_1_SLV_AXI_CLK					56
-+#define GCC_PCIE_1_SLV_Q2A_AXI_CLK				57
-+#define GCC_PDM2_CLK						58
-+#define GCC_PDM2_CLK_SRC					59
-+#define GCC_PDM_AHB_CLK						60
-+#define GCC_PDM_XO4_CLK						61
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				62
-+#define GCC_QMIP_CAMERA_RT_AHB_CLK				63
-+#define GCC_QMIP_DISP_AHB_CLK					64
-+#define GCC_QMIP_GPU_AHB_CLK					65
-+#define GCC_QMIP_PCIE_AHB_CLK					66
-+#define GCC_QMIP_VIDEO_CV_CPU_AHB_CLK				67
-+#define GCC_QMIP_VIDEO_CVP_AHB_CLK				68
-+#define GCC_QMIP_VIDEO_V_CPU_AHB_CLK				69
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				70
-+#define GCC_QUPV3_I2C_CORE_CLK					71
-+#define GCC_QUPV3_I2C_S0_CLK					72
-+#define GCC_QUPV3_I2C_S0_CLK_SRC				73
-+#define GCC_QUPV3_I2C_S1_CLK					74
-+#define GCC_QUPV3_I2C_S1_CLK_SRC				75
-+#define GCC_QUPV3_I2C_S2_CLK					76
-+#define GCC_QUPV3_I2C_S2_CLK_SRC				77
-+#define GCC_QUPV3_I2C_S3_CLK					78
-+#define GCC_QUPV3_I2C_S3_CLK_SRC				79
-+#define GCC_QUPV3_I2C_S4_CLK					80
-+#define GCC_QUPV3_I2C_S4_CLK_SRC				81
-+#define GCC_QUPV3_I2C_S5_CLK					82
-+#define GCC_QUPV3_I2C_S5_CLK_SRC				83
-+#define GCC_QUPV3_I2C_S6_CLK					84
-+#define GCC_QUPV3_I2C_S6_CLK_SRC				85
-+#define GCC_QUPV3_I2C_S7_CLK					86
-+#define GCC_QUPV3_I2C_S7_CLK_SRC				87
-+#define GCC_QUPV3_I2C_S8_CLK					88
-+#define GCC_QUPV3_I2C_S8_CLK_SRC				89
-+#define GCC_QUPV3_I2C_S9_CLK					90
-+#define GCC_QUPV3_I2C_S9_CLK_SRC				91
-+#define GCC_QUPV3_I2C_S_AHB_CLK					92
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				93
-+#define GCC_QUPV3_WRAP1_CORE_CLK				94
-+#define GCC_QUPV3_WRAP1_QSPI_REF_CLK				95
-+#define GCC_QUPV3_WRAP1_QSPI_REF_CLK_SRC			96
-+#define GCC_QUPV3_WRAP1_S0_CLK					97
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				98
-+#define GCC_QUPV3_WRAP1_S1_CLK					99
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				100
-+#define GCC_QUPV3_WRAP1_S2_CLK					101
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				102
-+#define GCC_QUPV3_WRAP1_S3_CLK					103
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				104
-+#define GCC_QUPV3_WRAP1_S4_CLK					105
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				106
-+#define GCC_QUPV3_WRAP1_S5_CLK					107
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC				108
-+#define GCC_QUPV3_WRAP1_S6_CLK					109
-+#define GCC_QUPV3_WRAP1_S6_CLK_SRC				110
-+#define GCC_QUPV3_WRAP1_S7_CLK					111
-+#define GCC_QUPV3_WRAP1_S7_CLK_SRC				112
-+#define GCC_QUPV3_WRAP2_CORE_2X_CLK				113
-+#define GCC_QUPV3_WRAP2_CORE_CLK				114
-+#define GCC_QUPV3_WRAP2_IBI_CTRL_0_CLK_SRC			115
-+#define GCC_QUPV3_WRAP2_IBI_CTRL_2_CLK				116
-+#define GCC_QUPV3_WRAP2_IBI_CTRL_3_CLK				117
-+#define GCC_QUPV3_WRAP2_S0_CLK					118
-+#define GCC_QUPV3_WRAP2_S0_CLK_SRC				119
-+#define GCC_QUPV3_WRAP2_S1_CLK					120
-+#define GCC_QUPV3_WRAP2_S1_CLK_SRC				121
-+#define GCC_QUPV3_WRAP2_S2_CLK					122
-+#define GCC_QUPV3_WRAP2_S2_CLK_SRC				123
-+#define GCC_QUPV3_WRAP2_S3_CLK					124
-+#define GCC_QUPV3_WRAP2_S3_CLK_SRC				125
-+#define GCC_QUPV3_WRAP2_S4_CLK					126
-+#define GCC_QUPV3_WRAP2_S4_CLK_SRC				127
-+#define GCC_QUPV3_WRAP2_S5_CLK					128
-+#define GCC_QUPV3_WRAP2_S5_CLK_SRC				129
-+#define GCC_QUPV3_WRAP2_S6_CLK					130
-+#define GCC_QUPV3_WRAP2_S6_CLK_SRC				131
-+#define GCC_QUPV3_WRAP2_S7_CLK					132
-+#define GCC_QUPV3_WRAP2_S7_CLK_SRC				133
-+#define GCC_QUPV3_WRAP3_CORE_2X_CLK				134
-+#define GCC_QUPV3_WRAP3_CORE_CLK				135
-+#define GCC_QUPV3_WRAP3_QSPI_REF_CLK				136
-+#define GCC_QUPV3_WRAP3_QSPI_REF_CLK_SRC			137
-+#define GCC_QUPV3_WRAP3_S0_CLK					138
-+#define GCC_QUPV3_WRAP3_S0_CLK_SRC				139
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK				140
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				141
-+#define GCC_QUPV3_WRAP_2_IBI_2_AHB_CLK				142
-+#define GCC_QUPV3_WRAP_2_IBI_3_AHB_CLK				143
-+#define GCC_QUPV3_WRAP_2_M_AHB_CLK				144
-+#define GCC_QUPV3_WRAP_2_S_AHB_CLK				145
-+#define GCC_QUPV3_WRAP_3_M_AHB_CLK				146
-+#define GCC_QUPV3_WRAP_3_S_AHB_CLK				147
-+#define GCC_SDCC2_AHB_CLK					148
-+#define GCC_SDCC2_APPS_CLK					149
-+#define GCC_SDCC2_APPS_CLK_SRC					150
-+#define GCC_SDCC4_AHB_CLK					151
-+#define GCC_SDCC4_APPS_CLK					152
-+#define GCC_SDCC4_APPS_CLK_SRC					153
-+#define GCC_UFS_PHY_AHB_CLK					154
-+#define GCC_UFS_PHY_AXI_CLK					155
-+#define GCC_UFS_PHY_AXI_CLK_SRC					156
-+#define GCC_UFS_PHY_AXI_HW_CTL_CLK				157
-+#define GCC_UFS_PHY_ICE_CORE_CLK				158
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC				159
-+#define GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK				160
-+#define GCC_UFS_PHY_PHY_AUX_CLK					161
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC				162
-+#define GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK				163
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK				164
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC				165
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK				166
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC				167
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK				168
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC				169
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK				170
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC				171
-+#define GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK			172
-+#define GCC_USB30_PRIM_MASTER_CLK				173
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC				174
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				175
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			176
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		177
-+#define GCC_USB30_PRIM_SLEEP_CLK				178
-+#define GCC_USB3_PRIM_PHY_AUX_CLK				179
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				180
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				181
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK				182
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				183
-+#define GCC_VIDEO_AHB_CLK					184
-+#define GCC_VIDEO_AXI0_CLK					185
-+#define GCC_VIDEO_AXI1_CLK					186
-+#define GCC_VIDEO_XO_CLK					187
-+#define GCC_GPLL0_AO						188
-+#define GCC_GPLL0_OUT_EVEN_AO					189
-+#define GCC_GPLL1_AO						190
-+#define GCC_GPLL3_AO						191
-+#define GCC_GPLL4_AO						192
-+#define GCC_GPLL6_AO						193
-+
-+/* GCC resets */
-+#define GCC_CAMERA_BCR						0
-+#define GCC_DISPLAY_BCR						1
-+#define GCC_GPU_BCR						2
-+#define GCC_PCIE_0_BCR						3
-+#define GCC_PCIE_0_LINK_DOWN_BCR				4
-+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR				5
-+#define GCC_PCIE_0_PHY_BCR					6
-+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR			7
-+#define GCC_PCIE_1_BCR						8
-+#define GCC_PCIE_1_LINK_DOWN_BCR				9
-+#define GCC_PCIE_1_NOCSR_COM_PHY_BCR				10
-+#define GCC_PCIE_1_PHY_BCR					11
-+#define GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR			12
-+#define GCC_PCIE_PHY_BCR					13
-+#define GCC_PCIE_PHY_CFG_AHB_BCR				14
-+#define GCC_PCIE_PHY_COM_BCR					15
-+#define GCC_PDM_BCR						16
-+#define GCC_QUPV3_WRAPPER_1_BCR					17
-+#define GCC_QUPV3_WRAPPER_2_BCR					18
-+#define GCC_QUPV3_WRAPPER_3_BCR					19
-+#define GCC_QUPV3_WRAPPER_I2C_BCR				20
-+#define GCC_QUSB2PHY_PRIM_BCR					21
-+#define GCC_QUSB2PHY_SEC_BCR					22
-+#define GCC_SDCC2_BCR						23
-+#define GCC_SDCC4_BCR						24
-+#define GCC_UFS_PHY_BCR						25
-+#define GCC_USB30_PRIM_BCR					26
-+#define GCC_USB3_DP_PHY_PRIM_BCR				27
-+#define GCC_USB3_DP_PHY_SEC_BCR					28
-+#define GCC_USB3_PHY_PRIM_BCR					29
-+#define GCC_USB3_PHY_SEC_BCR					30
-+#define GCC_USB3PHY_PHY_PRIM_BCR				31
-+#define GCC_USB3PHY_PHY_SEC_BCR					32
-+#define GCC_VIDEO_AXI0_CLK_ARES					33
-+#define GCC_VIDEO_AXI1_CLK_ARES					34
-+#define GCC_VIDEO_BCR						35
-+
-+/* GCC power domains */
-+#define PCIE_0_GDSC						0
-+#define PCIE_0_PHY_GDSC						1
-+#define PCIE_1_GDSC						2
-+#define PCIE_1_PHY_GDSC						3
-+#define UFS_PHY_GDSC						4
-+#define UFS_MEM_PHY_GDSC					5
-+#define USB30_PRIM_GDSC						6
-+#define USB3_PHY_GDSC						7
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8650-gpucc.h b/include/dt-bindings/clock/qcom,sm8650-gpucc.h
-new file mode 100644
-index 000000000000..d0dc457cfe75
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8650-gpucc.h
-@@ -0,0 +1,43 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_SM8650_H
-+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_SM8650_H
-+
-+/* GPU_CC clocks */
-+#define GPU_CC_AHB_CLK				0
-+#define GPU_CC_CRC_AHB_CLK			1
-+#define GPU_CC_CX_ACCU_SHIFT_CLK		2
-+#define GPU_CC_CX_FF_CLK			3
-+#define GPU_CC_CX_GMU_CLK			4
-+#define GPU_CC_CXO_AON_CLK			5
-+#define GPU_CC_CXO_CLK				6
-+#define GPU_CC_DEMET_CLK			7
-+#define GPU_CC_DPM_CLK				8
-+#define GPU_CC_FF_CLK_SRC			9
-+#define GPU_CC_FREQ_MEASURE_CLK			10
-+#define GPU_CC_GMU_CLK_SRC			11
-+#define GPU_CC_GX_ACCU_SHIFT_CLK		12
-+#define GPU_CC_GX_FF_CLK			13
-+#define GPU_CC_GX_GFX3D_CLK			14
-+#define GPU_CC_GX_GFX3D_RDVM_CLK		15
-+#define GPU_CC_GX_GMU_CLK			16
-+#define GPU_CC_GX_VSENSE_CLK			17
-+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK		18
-+#define GPU_CC_HUB_AON_CLK			19
-+#define GPU_CC_HUB_CLK_SRC			20
-+#define GPU_CC_HUB_CX_INT_CLK			21
-+#define GPU_CC_HUB_DIV_CLK_SRC			22
-+#define GPU_CC_MEMNOC_GFX_CLK			23
-+#define GPU_CC_PLL0				24
-+#define GPU_CC_PLL1				25
-+#define GPU_CC_SLEEP_CLK			26
-+
-+/* GDSCs */
-+#define GPU_GX_GDSC				0
-+#define GPU_CX_GDSC				1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8650-tcsr.h b/include/dt-bindings/clock/qcom,sm8650-tcsr.h
-new file mode 100644
-index 000000000000..b2c72d492f1f
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8650-tcsr.h
-@@ -0,0 +1,18 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_TCSR_CC_SM8650_H
-+#define _DT_BINDINGS_CLK_QCOM_TCSR_CC_SM8650_H
-+
-+/* TCSR CC clocks */
-+#define TCSR_PCIE_0_CLKREF_EN					0
-+#define TCSR_PCIE_1_CLKREF_EN					1
-+#define TCSR_UFS_CLKREF_EN					2
-+#define TCSR_UFS_PAD_CLKREF_EN					3
-+#define TCSR_USB2_CLKREF_EN					4
-+#define TCSR_USB3_CLKREF_EN					5
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8650-videocc.h b/include/dt-bindings/clock/qcom,sm8650-videocc.h
-new file mode 100644
-index 000000000000..4e3c2d87280f
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8650-videocc.h
-@@ -0,0 +1,23 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_SM8650_H
-+#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_SM8650_H
-+
-+#include "qcom,sm8450-videocc.h"
-+
-+/* SM8650 introduces below new clocks and resets compared to SM8450 */
-+
-+/* VIDEO_CC clocks */
-+#define VIDEO_CC_MVS0_SHIFT_CLK					12
-+#define VIDEO_CC_MVS0C_SHIFT_CLK				13
-+#define VIDEO_CC_MVS1_SHIFT_CLK					14
-+#define VIDEO_CC_MVS1C_SHIFT_CLK				15
-+#define VIDEO_CC_XO_CLK_SRC					16
-+
-+/* VIDEO_CC resets */
-+#define VIDEO_CC_XO_CLK_ARES					7
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8750-dispcc.h b/include/dt-bindings/clock/qcom,sm8750-dispcc.h
-new file mode 100644
-index 000000000000..dafb5069c96a
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8750-dispcc.h
-@@ -0,0 +1,112 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2024, Linaro Ltd.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_SM8750_DISP_CC_H
-+#define _DT_BINDINGS_CLK_QCOM_SM8750_DISP_CC_H
-+
-+/* DISP_CC clocks */
-+#define DISP_CC_ESYNC0_CLK					0
-+#define DISP_CC_ESYNC0_CLK_SRC					1
-+#define DISP_CC_ESYNC1_CLK					2
-+#define DISP_CC_ESYNC1_CLK_SRC					3
-+#define DISP_CC_MDSS_ACCU_SHIFT_CLK				4
-+#define DISP_CC_MDSS_AHB1_CLK					5
-+#define DISP_CC_MDSS_AHB_CLK					6
-+#define DISP_CC_MDSS_AHB_CLK_SRC				7
-+#define DISP_CC_MDSS_BYTE0_CLK					8
-+#define DISP_CC_MDSS_BYTE0_CLK_SRC				9
-+#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC				10
-+#define DISP_CC_MDSS_BYTE0_INTF_CLK				11
-+#define DISP_CC_MDSS_BYTE1_CLK					12
-+#define DISP_CC_MDSS_BYTE1_CLK_SRC				13
-+#define DISP_CC_MDSS_BYTE1_DIV_CLK_SRC				14
-+#define DISP_CC_MDSS_BYTE1_INTF_CLK				15
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK				16
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK_SRC				17
-+#define DISP_CC_MDSS_DPTX0_CRYPTO_CLK				18
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK				19
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK_SRC				20
-+#define DISP_CC_MDSS_DPTX0_LINK_DIV_CLK_SRC			21
-+#define DISP_CC_MDSS_DPTX0_LINK_INTF_CLK			22
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK				23
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC			24
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK				25
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC			26
-+#define DISP_CC_MDSS_DPTX0_USB_ROUTER_LINK_INTF_CLK		27
-+#define DISP_CC_MDSS_DPTX1_AUX_CLK				28
-+#define DISP_CC_MDSS_DPTX1_AUX_CLK_SRC				29
-+#define DISP_CC_MDSS_DPTX1_CRYPTO_CLK				30
-+#define DISP_CC_MDSS_DPTX1_LINK_CLK				31
-+#define DISP_CC_MDSS_DPTX1_LINK_CLK_SRC				32
-+#define DISP_CC_MDSS_DPTX1_LINK_DIV_CLK_SRC			33
-+#define DISP_CC_MDSS_DPTX1_LINK_INTF_CLK			34
-+#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK				35
-+#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK_SRC			36
-+#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK				37
-+#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK_SRC			38
-+#define DISP_CC_MDSS_DPTX1_USB_ROUTER_LINK_INTF_CLK		39
-+#define DISP_CC_MDSS_DPTX2_AUX_CLK				40
-+#define DISP_CC_MDSS_DPTX2_AUX_CLK_SRC				41
-+#define DISP_CC_MDSS_DPTX2_CRYPTO_CLK				42
-+#define DISP_CC_MDSS_DPTX2_LINK_CLK				43
-+#define DISP_CC_MDSS_DPTX2_LINK_CLK_SRC				44
-+#define DISP_CC_MDSS_DPTX2_LINK_DIV_CLK_SRC			45
-+#define DISP_CC_MDSS_DPTX2_LINK_INTF_CLK			46
-+#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK				47
-+#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK_SRC			48
-+#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK				49
-+#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK_SRC			50
-+#define DISP_CC_MDSS_DPTX3_AUX_CLK				51
-+#define DISP_CC_MDSS_DPTX3_AUX_CLK_SRC				52
-+#define DISP_CC_MDSS_DPTX3_CRYPTO_CLK				53
-+#define DISP_CC_MDSS_DPTX3_LINK_CLK				54
-+#define DISP_CC_MDSS_DPTX3_LINK_CLK_SRC				55
-+#define DISP_CC_MDSS_DPTX3_LINK_DIV_CLK_SRC			56
-+#define DISP_CC_MDSS_DPTX3_LINK_INTF_CLK			57
-+#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK				58
-+#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK_SRC			59
-+#define DISP_CC_MDSS_ESC0_CLK					60
-+#define DISP_CC_MDSS_ESC0_CLK_SRC				61
-+#define DISP_CC_MDSS_ESC1_CLK					62
-+#define DISP_CC_MDSS_ESC1_CLK_SRC				63
-+#define DISP_CC_MDSS_MDP1_CLK					64
-+#define DISP_CC_MDSS_MDP_CLK					65
-+#define DISP_CC_MDSS_MDP_CLK_SRC				66
-+#define DISP_CC_MDSS_MDP_LUT1_CLK				67
-+#define DISP_CC_MDSS_MDP_LUT_CLK				68
-+#define DISP_CC_MDSS_NON_GDSC_AHB_CLK				69
-+#define DISP_CC_MDSS_PCLK0_CLK					70
-+#define DISP_CC_MDSS_PCLK0_CLK_SRC				71
-+#define DISP_CC_MDSS_PCLK1_CLK					72
-+#define DISP_CC_MDSS_PCLK1_CLK_SRC				73
-+#define DISP_CC_MDSS_PCLK2_CLK					74
-+#define DISP_CC_MDSS_PCLK2_CLK_SRC				75
-+#define DISP_CC_MDSS_RSCC_AHB_CLK				76
-+#define DISP_CC_MDSS_RSCC_VSYNC_CLK				77
-+#define DISP_CC_MDSS_VSYNC1_CLK					78
-+#define DISP_CC_MDSS_VSYNC_CLK					79
-+#define DISP_CC_MDSS_VSYNC_CLK_SRC				80
-+#define DISP_CC_OSC_CLK						81
-+#define DISP_CC_OSC_CLK_SRC					82
-+#define DISP_CC_PLL0						83
-+#define DISP_CC_PLL1						84
-+#define DISP_CC_PLL2						85
-+#define DISP_CC_SLEEP_CLK					86
-+#define DISP_CC_SLEEP_CLK_SRC					87
-+#define DISP_CC_XO_CLK						88
-+#define DISP_CC_XO_CLK_SRC					89
-+
-+/* DISP_CC resets */
-+#define DISP_CC_MDSS_CORE_BCR					0
-+#define DISP_CC_MDSS_CORE_INT2_BCR				1
-+#define DISP_CC_MDSS_RSCC_BCR					2
-+
-+/* DISP_CC GDSCR */
-+#define MDSS_GDSC						0
-+#define MDSS_INT2_GDSC						1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8750-gcc.h b/include/dt-bindings/clock/qcom,sm8750-gcc.h
-new file mode 100644
-index 000000000000..e234595d7f42
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8750-gcc.h
-@@ -0,0 +1,226 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_SM8750_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_SM8750_H
-+
-+/* GCC clocks */
-+#define GCC_AGGRE_NOC_PCIE_AXI_CLK				0
-+#define GCC_AGGRE_UFS_PHY_AXI_CLK				1
-+#define GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK			2
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK				3
-+#define GCC_BOOT_ROM_AHB_CLK					4
-+#define GCC_CAM_BIST_MCLK_AHB_CLK				5
-+#define GCC_CAMERA_AHB_CLK					6
-+#define GCC_CAMERA_HF_AXI_CLK					7
-+#define GCC_CAMERA_SF_AXI_CLK					8
-+#define GCC_CAMERA_XO_CLK					9
-+#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK				10
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				11
-+#define GCC_CNOC_PCIE_SF_AXI_CLK				12
-+#define GCC_DDRSS_GPU_AXI_CLK					13
-+#define GCC_DDRSS_PCIE_SF_QTB_CLK				14
-+#define GCC_DISP_AHB_CLK					15
-+#define GCC_DISP_HF_AXI_CLK					16
-+#define GCC_EVA_AHB_CLK						17
-+#define GCC_EVA_AXI0_CLK					18
-+#define GCC_EVA_AXI0C_CLK					19
-+#define GCC_EVA_XO_CLK						20
-+#define GCC_GP1_CLK						21
-+#define GCC_GP1_CLK_SRC						22
-+#define GCC_GP2_CLK						23
-+#define GCC_GP2_CLK_SRC						24
-+#define GCC_GP3_CLK						25
-+#define GCC_GP3_CLK_SRC						26
-+#define GCC_GPLL0						27
-+#define GCC_GPLL0_OUT_EVEN					28
-+#define GCC_GPLL1						29
-+#define GCC_GPLL4						30
-+#define GCC_GPLL7						31
-+#define GCC_GPLL9						32
-+#define GCC_GPU_CFG_AHB_CLK					33
-+#define GCC_GPU_GEMNOC_GFX_CLK					34
-+#define GCC_GPU_GPLL0_CLK_SRC					35
-+#define GCC_GPU_GPLL0_DIV_CLK_SRC				36
-+#define GCC_PCIE_0_AUX_CLK					37
-+#define GCC_PCIE_0_AUX_CLK_SRC					38
-+#define GCC_PCIE_0_CFG_AHB_CLK					39
-+#define GCC_PCIE_0_MSTR_AXI_CLK					40
-+#define GCC_PCIE_0_PHY_RCHNG_CLK				41
-+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC				42
-+#define GCC_PCIE_0_PIPE_CLK					43
-+#define GCC_PCIE_0_PIPE_CLK_SRC					44
-+#define GCC_PCIE_0_SLV_AXI_CLK					45
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				46
-+#define GCC_PCIE_RSCC_CFG_AHB_CLK				47
-+#define GCC_PCIE_RSCC_XO_CLK					48
-+#define GCC_PDM2_CLK						49
-+#define GCC_PDM2_CLK_SRC					50
-+#define GCC_PDM_AHB_CLK						51
-+#define GCC_PDM_XO4_CLK						52
-+#define GCC_QMIP_CAMERA_CMD_AHB_CLK				53
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				54
-+#define GCC_QMIP_CAMERA_RT_AHB_CLK				55
-+#define GCC_QMIP_GPU_AHB_CLK					56
-+#define GCC_QMIP_PCIE_AHB_CLK					57
-+#define GCC_QMIP_VIDEO_CV_CPU_AHB_CLK				58
-+#define GCC_QMIP_VIDEO_CVP_AHB_CLK				59
-+#define GCC_QMIP_VIDEO_V_CPU_AHB_CLK				60
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				61
-+#define GCC_QUPV3_I2C_CORE_CLK					62
-+#define GCC_QUPV3_I2C_S0_CLK					63
-+#define GCC_QUPV3_I2C_S0_CLK_SRC				64
-+#define GCC_QUPV3_I2C_S1_CLK					65
-+#define GCC_QUPV3_I2C_S1_CLK_SRC				66
-+#define GCC_QUPV3_I2C_S2_CLK					67
-+#define GCC_QUPV3_I2C_S2_CLK_SRC				68
-+#define GCC_QUPV3_I2C_S3_CLK					69
-+#define GCC_QUPV3_I2C_S3_CLK_SRC				70
-+#define GCC_QUPV3_I2C_S4_CLK					71
-+#define GCC_QUPV3_I2C_S4_CLK_SRC				72
-+#define GCC_QUPV3_I2C_S5_CLK					73
-+#define GCC_QUPV3_I2C_S5_CLK_SRC				74
-+#define GCC_QUPV3_I2C_S6_CLK					75
-+#define GCC_QUPV3_I2C_S6_CLK_SRC				76
-+#define GCC_QUPV3_I2C_S7_CLK					77
-+#define GCC_QUPV3_I2C_S7_CLK_SRC				78
-+#define GCC_QUPV3_I2C_S8_CLK					79
-+#define GCC_QUPV3_I2C_S8_CLK_SRC				80
-+#define GCC_QUPV3_I2C_S9_CLK					81
-+#define GCC_QUPV3_I2C_S9_CLK_SRC				82
-+#define GCC_QUPV3_I2C_S_AHB_CLK					83
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				84
-+#define GCC_QUPV3_WRAP1_CORE_CLK				85
-+#define GCC_QUPV3_WRAP1_QSPI_REF_CLK				86
-+#define GCC_QUPV3_WRAP1_QSPI_REF_CLK_SRC			87
-+#define GCC_QUPV3_WRAP1_S0_CLK					88
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				89
-+#define GCC_QUPV3_WRAP1_S1_CLK					90
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				91
-+#define GCC_QUPV3_WRAP1_S2_CLK					92
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				93
-+#define GCC_QUPV3_WRAP1_S3_CLK					94
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				95
-+#define GCC_QUPV3_WRAP1_S4_CLK					96
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				97
-+#define GCC_QUPV3_WRAP1_S5_CLK					98
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC				99
-+#define GCC_QUPV3_WRAP1_S6_CLK					100
-+#define GCC_QUPV3_WRAP1_S6_CLK_SRC				101
-+#define GCC_QUPV3_WRAP1_S7_CLK					102
-+#define GCC_QUPV3_WRAP1_S7_CLK_SRC				103
-+#define GCC_QUPV3_WRAP2_CORE_2X_CLK				104
-+#define GCC_QUPV3_WRAP2_CORE_CLK				105
-+#define GCC_QUPV3_WRAP2_IBI_CTRL_0_CLK_SRC			106
-+#define GCC_QUPV3_WRAP2_IBI_CTRL_2_CLK				107
-+#define GCC_QUPV3_WRAP2_IBI_CTRL_3_CLK				108
-+#define GCC_QUPV3_WRAP2_S0_CLK					109
-+#define GCC_QUPV3_WRAP2_S0_CLK_SRC				110
-+#define GCC_QUPV3_WRAP2_S1_CLK					111
-+#define GCC_QUPV3_WRAP2_S1_CLK_SRC				112
-+#define GCC_QUPV3_WRAP2_S2_CLK					113
-+#define GCC_QUPV3_WRAP2_S2_CLK_SRC				114
-+#define GCC_QUPV3_WRAP2_S3_CLK					115
-+#define GCC_QUPV3_WRAP2_S3_CLK_SRC				116
-+#define GCC_QUPV3_WRAP2_S4_CLK					117
-+#define GCC_QUPV3_WRAP2_S4_CLK_SRC				118
-+#define GCC_QUPV3_WRAP2_S5_CLK					119
-+#define GCC_QUPV3_WRAP2_S5_CLK_SRC				120
-+#define GCC_QUPV3_WRAP2_S6_CLK					121
-+#define GCC_QUPV3_WRAP2_S6_CLK_SRC				122
-+#define GCC_QUPV3_WRAP2_S7_CLK					123
-+#define GCC_QUPV3_WRAP2_S7_CLK_SRC				124
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK				125
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				126
-+#define GCC_QUPV3_WRAP_2_IBI_2_AHB_CLK				127
-+#define GCC_QUPV3_WRAP_2_IBI_3_AHB_CLK				128
-+#define GCC_QUPV3_WRAP_2_M_AHB_CLK				129
-+#define GCC_QUPV3_WRAP_2_S_AHB_CLK				130
-+#define GCC_SDCC2_AHB_CLK					131
-+#define GCC_SDCC2_APPS_CLK					132
-+#define GCC_SDCC2_APPS_CLK_SRC					133
-+#define GCC_SDCC4_AHB_CLK					134
-+#define GCC_SDCC4_APPS_CLK					135
-+#define GCC_SDCC4_APPS_CLK_SRC					136
-+#define GCC_UFS_PHY_AHB_CLK					137
-+#define GCC_UFS_PHY_AXI_CLK					138
-+#define GCC_UFS_PHY_AXI_CLK_SRC					139
-+#define GCC_UFS_PHY_AXI_HW_CTL_CLK				140
-+#define GCC_UFS_PHY_ICE_CORE_CLK				141
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC				142
-+#define GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK				143
-+#define GCC_UFS_PHY_PHY_AUX_CLK					144
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC				145
-+#define GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK				146
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK				147
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC				148
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK				149
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC				150
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK				151
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC				152
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK				153
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC				154
-+#define GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK			155
-+#define GCC_USB30_PRIM_MASTER_CLK				156
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC				157
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				158
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			159
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		160
-+#define GCC_USB30_PRIM_SLEEP_CLK				161
-+#define GCC_USB3_PRIM_PHY_AUX_CLK				162
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				163
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				164
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK				165
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				166
-+#define GCC_VIDEO_AHB_CLK					167
-+#define GCC_VIDEO_AXI0_CLK					168
-+#define GCC_VIDEO_AXI1_CLK					169
-+#define GCC_VIDEO_XO_CLK					170
-+
-+/* GCC power domains */
-+#define GCC_PCIE_0_GDSC						0
-+#define GCC_PCIE_0_PHY_GDSC					1
-+#define GCC_UFS_MEM_PHY_GDSC					2
-+#define GCC_UFS_PHY_GDSC					3
-+#define GCC_USB30_PRIM_GDSC					4
-+#define GCC_USB3_PHY_GDSC					5
-+
-+/* GCC resets */
-+#define GCC_CAMERA_BCR						0
-+#define GCC_DISPLAY_BCR						1
-+#define GCC_EVA_BCR						2
-+#define GCC_GPU_BCR						3
-+#define GCC_PCIE_0_BCR						4
-+#define GCC_PCIE_0_LINK_DOWN_BCR				5
-+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR				6
-+#define GCC_PCIE_0_PHY_BCR					7
-+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR			8
-+#define GCC_PCIE_PHY_BCR					9
-+#define GCC_PCIE_PHY_CFG_AHB_BCR				10
-+#define GCC_PCIE_PHY_COM_BCR					11
-+#define GCC_PCIE_RSCC_BCR					12
-+#define GCC_PDM_BCR						13
-+#define GCC_QUPV3_WRAPPER_1_BCR					14
-+#define GCC_QUPV3_WRAPPER_2_BCR					15
-+#define GCC_QUPV3_WRAPPER_I2C_BCR				16
-+#define GCC_QUSB2PHY_PRIM_BCR					17
-+#define GCC_QUSB2PHY_SEC_BCR					18
-+#define GCC_SDCC2_BCR						19
-+#define GCC_SDCC4_BCR						20
-+#define GCC_UFS_PHY_BCR						21
-+#define GCC_USB30_PRIM_BCR					22
-+#define GCC_USB3_DP_PHY_PRIM_BCR				23
-+#define GCC_USB3_DP_PHY_SEC_BCR					24
-+#define GCC_USB3_PHY_PRIM_BCR					25
-+#define GCC_USB3_PHY_SEC_BCR					26
-+#define GCC_USB3PHY_PHY_PRIM_BCR				27
-+#define GCC_USB3PHY_PHY_SEC_BCR					28
-+#define GCC_VIDEO_AXI0_CLK_ARES					29
-+#define GCC_VIDEO_AXI1_CLK_ARES					30
-+#define GCC_VIDEO_BCR						31
-+#define GCC_EVA_AXI0_CLK_ARES					32
-+#define GCC_EVA_AXI0C_CLK_ARES					33
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8750-tcsr.h b/include/dt-bindings/clock/qcom,sm8750-tcsr.h
-new file mode 100644
-index 000000000000..1c502ac7c7f4
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8750-tcsr.h
-@@ -0,0 +1,15 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_TCSR_CC_SM8750_H
-+#define _DT_BINDINGS_CLK_QCOM_TCSR_CC_SM8750_H
-+
-+/* TCSR_CC clocks */
-+#define TCSR_PCIE_0_CLKREF_EN				0
-+#define TCSR_UFS_CLKREF_EN				1
-+#define TCSR_USB2_CLKREF_EN				2
-+#define TCSR_USB3_CLKREF_EN				3
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,sm8750-videocc.h b/include/dt-bindings/clock/qcom,sm8750-videocc.h
-new file mode 100644
-index 000000000000..f3bfa2ba5160
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,sm8750-videocc.h
-@@ -0,0 +1,40 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_SM8750_H
-+#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_SM8750_H
-+
-+/* VIDEO_CC clocks */
-+#define VIDEO_CC_AHB_CLK					0
-+#define VIDEO_CC_AHB_CLK_SRC					1
-+#define VIDEO_CC_MVS0_CLK					2
-+#define VIDEO_CC_MVS0_CLK_SRC					3
-+#define VIDEO_CC_MVS0_DIV_CLK_SRC				4
-+#define VIDEO_CC_MVS0_FREERUN_CLK				5
-+#define VIDEO_CC_MVS0_SHIFT_CLK					6
-+#define VIDEO_CC_MVS0C_CLK					7
-+#define VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC				8
-+#define VIDEO_CC_MVS0C_FREERUN_CLK				9
-+#define VIDEO_CC_MVS0C_SHIFT_CLK				10
-+#define VIDEO_CC_PLL0						11
-+#define VIDEO_CC_SLEEP_CLK					12
-+#define VIDEO_CC_SLEEP_CLK_SRC					13
-+#define VIDEO_CC_XO_CLK						14
-+#define VIDEO_CC_XO_CLK_SRC					15
-+
-+/* VIDEO_CC power domains */
-+#define VIDEO_CC_MVS0_GDSC					0
-+#define VIDEO_CC_MVS0C_GDSC					1
-+
-+/* VIDEO_CC resets */
-+#define VIDEO_CC_INTERFACE_BCR					0
-+#define VIDEO_CC_MVS0_BCR					1
-+#define VIDEO_CC_MVS0C_CLK_ARES					2
-+#define VIDEO_CC_MVS0C_BCR					3
-+#define VIDEO_CC_MVS0_FREERUN_CLK_ARES				4
-+#define VIDEO_CC_MVS0C_FREERUN_CLK_ARES				5
-+#define VIDEO_CC_XO_CLK_ARES					6
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,videocc-sm8150.h b/include/dt-bindings/clock/qcom,videocc-sm8150.h
-index e24ee840cfdb..c557b78dc572 100644
--- a/include/dt-bindings/clock/qcom,videocc-sm8150.h
-+++ b/include/dt-bindings/clock/qcom,videocc-sm8150.h
-@@ -16,6 +16,10 @@
- 
- /* VIDEO_CC Resets */
- #define VIDEO_CC_MVSC_CORE_CLK_BCR	0
-+#define VIDEO_CC_INTERFACE_BCR		1
-+#define VIDEO_CC_MVS0_BCR		2
-+#define VIDEO_CC_MVS1_BCR		3
-+#define VIDEO_CC_MVSC_BCR		4
- 
- /* VIDEO_CC GDSCRs */
- #define VENUS_GDSC			0
-diff --git a/include/dt-bindings/clock/qcom,x1e80100-camcc.h b/include/dt-bindings/clock/qcom,x1e80100-camcc.h
-new file mode 100644
-index 000000000000..d72fdfb06a7c
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,x1e80100-camcc.h
-@@ -0,0 +1,135 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_X1E80100_H
-+#define _DT_BINDINGS_CLK_QCOM_CAM_CC_X1E80100_H
-+
-+/* CAM_CC clocks */
-+#define CAM_CC_BPS_AHB_CLK					0
-+#define CAM_CC_BPS_CLK						1
-+#define CAM_CC_BPS_CLK_SRC					2
-+#define CAM_CC_BPS_FAST_AHB_CLK					3
-+#define CAM_CC_CAMNOC_AXI_NRT_CLK				4
-+#define CAM_CC_CAMNOC_AXI_RT_CLK				5
-+#define CAM_CC_CAMNOC_AXI_RT_CLK_SRC				6
-+#define CAM_CC_CAMNOC_DCD_XO_CLK				7
-+#define CAM_CC_CAMNOC_XO_CLK					8
-+#define CAM_CC_CCI_0_CLK					9
-+#define CAM_CC_CCI_0_CLK_SRC					10
-+#define CAM_CC_CCI_1_CLK					11
-+#define CAM_CC_CCI_1_CLK_SRC					12
-+#define CAM_CC_CORE_AHB_CLK					13
-+#define CAM_CC_CPAS_AHB_CLK					14
-+#define CAM_CC_CPAS_BPS_CLK					15
-+#define CAM_CC_CPAS_FAST_AHB_CLK				16
-+#define CAM_CC_CPAS_IFE_0_CLK					17
-+#define CAM_CC_CPAS_IFE_1_CLK					18
-+#define CAM_CC_CPAS_IFE_LITE_CLK				19
-+#define CAM_CC_CPAS_IPE_NPS_CLK					20
-+#define CAM_CC_CPAS_SFE_0_CLK					21
-+#define CAM_CC_CPHY_RX_CLK_SRC					22
-+#define CAM_CC_CSI0PHYTIMER_CLK					23
-+#define CAM_CC_CSI0PHYTIMER_CLK_SRC				24
-+#define CAM_CC_CSI1PHYTIMER_CLK					25
-+#define CAM_CC_CSI1PHYTIMER_CLK_SRC				26
-+#define CAM_CC_CSI2PHYTIMER_CLK					27
-+#define CAM_CC_CSI2PHYTIMER_CLK_SRC				28
-+#define CAM_CC_CSI3PHYTIMER_CLK					29
-+#define CAM_CC_CSI3PHYTIMER_CLK_SRC				30
-+#define CAM_CC_CSI4PHYTIMER_CLK					31
-+#define CAM_CC_CSI4PHYTIMER_CLK_SRC				32
-+#define CAM_CC_CSI5PHYTIMER_CLK					33
-+#define CAM_CC_CSI5PHYTIMER_CLK_SRC				34
-+#define CAM_CC_CSID_CLK						35
-+#define CAM_CC_CSID_CLK_SRC					36
-+#define CAM_CC_CSID_CSIPHY_RX_CLK				37
-+#define CAM_CC_CSIPHY0_CLK					38
-+#define CAM_CC_CSIPHY1_CLK					39
-+#define CAM_CC_CSIPHY2_CLK					40
-+#define CAM_CC_CSIPHY3_CLK					41
-+#define CAM_CC_CSIPHY4_CLK					42
-+#define CAM_CC_CSIPHY5_CLK					43
-+#define CAM_CC_FAST_AHB_CLK_SRC					44
-+#define CAM_CC_GDSC_CLK						45
-+#define CAM_CC_ICP_AHB_CLK					46
-+#define CAM_CC_ICP_CLK						47
-+#define CAM_CC_ICP_CLK_SRC					48
-+#define CAM_CC_IFE_0_CLK					49
-+#define CAM_CC_IFE_0_CLK_SRC					50
-+#define CAM_CC_IFE_0_DSP_CLK					51
-+#define CAM_CC_IFE_0_FAST_AHB_CLK				52
-+#define CAM_CC_IFE_1_CLK					53
-+#define CAM_CC_IFE_1_CLK_SRC					54
-+#define CAM_CC_IFE_1_DSP_CLK					55
-+#define CAM_CC_IFE_1_FAST_AHB_CLK				56
-+#define CAM_CC_IFE_LITE_AHB_CLK					57
-+#define CAM_CC_IFE_LITE_CLK					58
-+#define CAM_CC_IFE_LITE_CLK_SRC					59
-+#define CAM_CC_IFE_LITE_CPHY_RX_CLK				60
-+#define CAM_CC_IFE_LITE_CSID_CLK				61
-+#define CAM_CC_IFE_LITE_CSID_CLK_SRC				62
-+#define CAM_CC_IPE_NPS_AHB_CLK					63
-+#define CAM_CC_IPE_NPS_CLK					64
-+#define CAM_CC_IPE_NPS_CLK_SRC					65
-+#define CAM_CC_IPE_NPS_FAST_AHB_CLK				66
-+#define CAM_CC_IPE_PPS_CLK					67
-+#define CAM_CC_IPE_PPS_FAST_AHB_CLK				68
-+#define CAM_CC_JPEG_CLK						69
-+#define CAM_CC_JPEG_CLK_SRC					70
-+#define CAM_CC_MCLK0_CLK					71
-+#define CAM_CC_MCLK0_CLK_SRC					72
-+#define CAM_CC_MCLK1_CLK					73
-+#define CAM_CC_MCLK1_CLK_SRC					74
-+#define CAM_CC_MCLK2_CLK					75
-+#define CAM_CC_MCLK2_CLK_SRC					76
-+#define CAM_CC_MCLK3_CLK					77
-+#define CAM_CC_MCLK3_CLK_SRC					78
-+#define CAM_CC_MCLK4_CLK					79
-+#define CAM_CC_MCLK4_CLK_SRC					80
-+#define CAM_CC_MCLK5_CLK					81
-+#define CAM_CC_MCLK5_CLK_SRC					82
-+#define CAM_CC_MCLK6_CLK					83
-+#define CAM_CC_MCLK6_CLK_SRC					84
-+#define CAM_CC_MCLK7_CLK					85
-+#define CAM_CC_MCLK7_CLK_SRC					86
-+#define CAM_CC_PLL0						87
-+#define CAM_CC_PLL0_OUT_EVEN					88
-+#define CAM_CC_PLL0_OUT_ODD					89
-+#define CAM_CC_PLL1						90
-+#define CAM_CC_PLL1_OUT_EVEN					91
-+#define CAM_CC_PLL2						92
-+#define CAM_CC_PLL3						93
-+#define CAM_CC_PLL3_OUT_EVEN					94
-+#define CAM_CC_PLL4						95
-+#define CAM_CC_PLL4_OUT_EVEN					96
-+#define CAM_CC_PLL6						97
-+#define CAM_CC_PLL6_OUT_EVEN					98
-+#define CAM_CC_PLL8						99
-+#define CAM_CC_PLL8_OUT_EVEN					100
-+#define CAM_CC_SFE_0_CLK					101
-+#define CAM_CC_SFE_0_CLK_SRC					102
-+#define CAM_CC_SFE_0_FAST_AHB_CLK				103
-+#define CAM_CC_SLEEP_CLK					104
-+#define CAM_CC_SLEEP_CLK_SRC					105
-+#define CAM_CC_SLOW_AHB_CLK_SRC					106
-+#define CAM_CC_XO_CLK_SRC					107
-+
-+/* CAM_CC power domains */
-+#define CAM_CC_BPS_GDSC						0
-+#define CAM_CC_IFE_0_GDSC					1
-+#define CAM_CC_IFE_1_GDSC					2
-+#define CAM_CC_IPE_0_GDSC					3
-+#define CAM_CC_SFE_0_GDSC					4
-+#define CAM_CC_TITAN_TOP_GDSC					5
-+
-+/* CAM_CC resets */
-+#define CAM_CC_BPS_BCR						0
-+#define CAM_CC_ICP_BCR						1
-+#define CAM_CC_IFE_0_BCR					2
-+#define CAM_CC_IFE_1_BCR					3
-+#define CAM_CC_IPE_0_BCR					4
-+#define CAM_CC_SFE_0_BCR					5
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,x1e80100-dispcc.h b/include/dt-bindings/clock/qcom,x1e80100-dispcc.h
-new file mode 100644
-index 000000000000..49b3a9e5ce4a
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,x1e80100-dispcc.h
-@@ -0,0 +1,101 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_X1E80100_DISP_CC_H
-+#define _DT_BINDINGS_CLK_QCOM_X1E80100_DISP_CC_H
-+
-+/* DISP_CC clocks */
-+#define DISP_CC_MDSS_ACCU_CLK					0
-+#define DISP_CC_MDSS_AHB1_CLK					1
-+#define DISP_CC_MDSS_AHB_CLK					2
-+#define DISP_CC_MDSS_AHB_CLK_SRC				3
-+#define DISP_CC_MDSS_BYTE0_CLK					4
-+#define DISP_CC_MDSS_BYTE0_CLK_SRC				5
-+#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC				6
-+#define DISP_CC_MDSS_BYTE0_INTF_CLK				7
-+#define DISP_CC_MDSS_BYTE1_CLK					8
-+#define DISP_CC_MDSS_BYTE1_CLK_SRC				9
-+#define DISP_CC_MDSS_BYTE1_DIV_CLK_SRC				10
-+#define DISP_CC_MDSS_BYTE1_INTF_CLK				11
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK				12
-+#define DISP_CC_MDSS_DPTX0_AUX_CLK_SRC				13
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK				14
-+#define DISP_CC_MDSS_DPTX0_LINK_CLK_SRC				15
-+#define DISP_CC_MDSS_DPTX0_LINK_DIV_CLK_SRC			16
-+#define DISP_CC_MDSS_DPTX0_LINK_INTF_CLK			17
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK				18
-+#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC			19
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK				20
-+#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC			21
-+#define DISP_CC_MDSS_DPTX0_USB_ROUTER_LINK_INTF_CLK		22
-+#define DISP_CC_MDSS_DPTX1_AUX_CLK				23
-+#define DISP_CC_MDSS_DPTX1_AUX_CLK_SRC				24
-+#define DISP_CC_MDSS_DPTX1_LINK_CLK				25
-+#define DISP_CC_MDSS_DPTX1_LINK_CLK_SRC				26
-+#define DISP_CC_MDSS_DPTX1_LINK_DIV_CLK_SRC			27
-+#define DISP_CC_MDSS_DPTX1_LINK_INTF_CLK			28
-+#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK				29
-+#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK_SRC			30
-+#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK				31
-+#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK_SRC			32
-+#define DISP_CC_MDSS_DPTX1_USB_ROUTER_LINK_INTF_CLK		33
-+#define DISP_CC_MDSS_DPTX2_AUX_CLK				34
-+#define DISP_CC_MDSS_DPTX2_AUX_CLK_SRC				35
-+#define DISP_CC_MDSS_DPTX2_LINK_CLK				36
-+#define DISP_CC_MDSS_DPTX2_LINK_CLK_SRC				37
-+#define DISP_CC_MDSS_DPTX2_LINK_DIV_CLK_SRC			38
-+#define DISP_CC_MDSS_DPTX2_LINK_INTF_CLK			39
-+#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK				40
-+#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK_SRC			41
-+#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK				42
-+#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK_SRC			43
-+#define DISP_CC_MDSS_DPTX2_USB_ROUTER_LINK_INTF_CLK		44
-+#define DISP_CC_MDSS_DPTX3_AUX_CLK				45
-+#define DISP_CC_MDSS_DPTX3_AUX_CLK_SRC				46
-+#define DISP_CC_MDSS_DPTX3_LINK_CLK				47
-+#define DISP_CC_MDSS_DPTX3_LINK_CLK_SRC				48
-+#define DISP_CC_MDSS_DPTX3_LINK_DIV_CLK_SRC			49
-+#define DISP_CC_MDSS_DPTX3_LINK_INTF_CLK			50
-+#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK				51
-+#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK_SRC			52
-+#define DISP_CC_MDSS_ESC0_CLK					53
-+#define DISP_CC_MDSS_ESC0_CLK_SRC				54
-+#define DISP_CC_MDSS_ESC1_CLK					55
-+#define DISP_CC_MDSS_ESC1_CLK_SRC				56
-+#define DISP_CC_MDSS_MDP1_CLK					57
-+#define DISP_CC_MDSS_MDP_CLK					58
-+#define DISP_CC_MDSS_MDP_CLK_SRC				59
-+#define DISP_CC_MDSS_MDP_LUT1_CLK				60
-+#define DISP_CC_MDSS_MDP_LUT_CLK				61
-+#define DISP_CC_MDSS_NON_GDSC_AHB_CLK				62
-+#define DISP_CC_MDSS_PCLK0_CLK					63
-+#define DISP_CC_MDSS_PCLK0_CLK_SRC				64
-+#define DISP_CC_MDSS_PCLK1_CLK					65
-+#define DISP_CC_MDSS_PCLK1_CLK_SRC				66
-+#define DISP_CC_MDSS_RSCC_AHB_CLK				67
-+#define DISP_CC_MDSS_RSCC_VSYNC_CLK				68
-+#define DISP_CC_MDSS_VSYNC1_CLK					69
-+#define DISP_CC_MDSS_VSYNC_CLK					70
-+#define DISP_CC_MDSS_VSYNC_CLK_SRC				71
-+#define DISP_CC_PLL0						72
-+#define DISP_CC_PLL1						73
-+#define DISP_CC_SLEEP_CLK					74
-+#define DISP_CC_SLEEP_CLK_SRC					75
-+#define DISP_CC_XO_CLK						76
-+#define DISP_CC_XO_CLK_SRC					77
-+
-+/* DISP_CC resets */
-+#define DISP_CC_MDSS_CORE_BCR					0
-+#define DISP_CC_MDSS_CORE_INT2_BCR				1
-+#define DISP_CC_MDSS_RSCC_BCR					2
-+#define DISP_CC_MDSS_DPTX0_USB_ROUTER_LINK_INTF_CLK_ARES	3
-+#define DISP_CC_MDSS_DPTX1_USB_ROUTER_LINK_INTF_CLK_ARES	4
-+#define DISP_CC_MDSS_DPTX2_USB_ROUTER_LINK_INTF_CLK_ARES	5
-+
-+/* DISP_CC GDSCR */
-+#define MDSS_GDSC						0
-+#define MDSS_INT2_GDSC						1
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,x1e80100-gcc.h b/include/dt-bindings/clock/qcom,x1e80100-gcc.h
-new file mode 100644
-index 000000000000..62aa12425592
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,x1e80100-gcc.h
-@@ -0,0 +1,548 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_X1E80100_H
-+#define _DT_BINDINGS_CLK_QCOM_GCC_X1E80100_H
-+
-+/* GCC clocks */
-+#define GCC_AGGRE_NOC_USB_NORTH_AXI_CLK				0
-+#define GCC_AGGRE_NOC_USB_SOUTH_AXI_CLK				1
-+#define GCC_AGGRE_UFS_PHY_AXI_CLK				2
-+#define GCC_AGGRE_USB2_PRIM_AXI_CLK				3
-+#define GCC_AGGRE_USB3_MP_AXI_CLK				4
-+#define GCC_AGGRE_USB3_PRIM_AXI_CLK				5
-+#define GCC_AGGRE_USB3_SEC_AXI_CLK				6
-+#define GCC_AGGRE_USB3_TERT_AXI_CLK				7
-+#define GCC_AGGRE_USB4_0_AXI_CLK				8
-+#define GCC_AGGRE_USB4_1_AXI_CLK				9
-+#define GCC_AGGRE_USB4_2_AXI_CLK				10
-+#define GCC_AGGRE_USB_NOC_AXI_CLK				11
-+#define GCC_AV1E_AHB_CLK					12
-+#define GCC_AV1E_AXI_CLK					13
-+#define GCC_AV1E_XO_CLK						14
-+#define GCC_BOOT_ROM_AHB_CLK					15
-+#define GCC_CAMERA_AHB_CLK					16
-+#define GCC_CAMERA_HF_AXI_CLK					17
-+#define GCC_CAMERA_SF_AXI_CLK					18
-+#define GCC_CAMERA_XO_CLK					19
-+#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK				20
-+#define GCC_CFG_NOC_PCIE_ANOC_NORTH_AHB_CLK			21
-+#define GCC_CFG_NOC_PCIE_ANOC_SOUTH_AHB_CLK			22
-+#define GCC_CFG_NOC_USB2_PRIM_AXI_CLK				23
-+#define GCC_CFG_NOC_USB3_MP_AXI_CLK				24
-+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				25
-+#define GCC_CFG_NOC_USB3_SEC_AXI_CLK				26
-+#define GCC_CFG_NOC_USB3_TERT_AXI_CLK				27
-+#define GCC_CFG_NOC_USB_ANOC_AHB_CLK				28
-+#define GCC_CFG_NOC_USB_ANOC_NORTH_AHB_CLK			29
-+#define GCC_CFG_NOC_USB_ANOC_SOUTH_AHB_CLK			30
-+#define GCC_CNOC_PCIE1_TUNNEL_CLK				31
-+#define GCC_CNOC_PCIE2_TUNNEL_CLK				32
-+#define GCC_CNOC_PCIE_NORTH_SF_AXI_CLK				33
-+#define GCC_CNOC_PCIE_SOUTH_SF_AXI_CLK				34
-+#define GCC_CNOC_PCIE_TUNNEL_CLK				35
-+#define GCC_DDRSS_GPU_AXI_CLK					36
-+#define GCC_DISP_AHB_CLK					37
-+#define GCC_DISP_HF_AXI_CLK					38
-+#define GCC_DISP_XO_CLK						39
-+#define GCC_GP1_CLK						40
-+#define GCC_GP1_CLK_SRC						41
-+#define GCC_GP2_CLK						42
-+#define GCC_GP2_CLK_SRC						43
-+#define GCC_GP3_CLK						44
-+#define GCC_GP3_CLK_SRC						45
-+#define GCC_GPLL0						46
-+#define GCC_GPLL0_OUT_EVEN					47
-+#define GCC_GPLL4						48
-+#define GCC_GPLL7						49
-+#define GCC_GPLL8						50
-+#define GCC_GPLL9						51
-+#define GCC_GPU_CFG_AHB_CLK					52
-+#define GCC_GPU_GPLL0_CPH_CLK_SRC				53
-+#define GCC_GPU_GPLL0_DIV_CPH_CLK_SRC				54
-+#define GCC_GPU_MEMNOC_GFX_CLK					55
-+#define GCC_GPU_SNOC_DVM_GFX_CLK				56
-+#define GCC_PCIE0_PHY_RCHNG_CLK					57
-+#define GCC_PCIE1_PHY_RCHNG_CLK					58
-+#define GCC_PCIE2_PHY_RCHNG_CLK					59
-+#define GCC_PCIE_0_AUX_CLK					60
-+#define GCC_PCIE_0_AUX_CLK_SRC					61
-+#define GCC_PCIE_0_CFG_AHB_CLK					62
-+#define GCC_PCIE_0_MSTR_AXI_CLK					63
-+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC				64
-+#define GCC_PCIE_0_PIPE_CLK					65
-+#define GCC_PCIE_0_SLV_AXI_CLK					66
-+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				67
-+#define GCC_PCIE_1_AUX_CLK					68
-+#define GCC_PCIE_1_AUX_CLK_SRC					69
-+#define GCC_PCIE_1_CFG_AHB_CLK					70
-+#define GCC_PCIE_1_MSTR_AXI_CLK					71
-+#define GCC_PCIE_1_PHY_RCHNG_CLK_SRC				72
-+#define GCC_PCIE_1_PIPE_CLK					73
-+#define GCC_PCIE_1_SLV_AXI_CLK					74
-+#define GCC_PCIE_1_SLV_Q2A_AXI_CLK				75
-+#define GCC_PCIE_2_AUX_CLK					76
-+#define GCC_PCIE_2_AUX_CLK_SRC					77
-+#define GCC_PCIE_2_CFG_AHB_CLK					78
-+#define GCC_PCIE_2_MSTR_AXI_CLK					79
-+#define GCC_PCIE_2_PHY_RCHNG_CLK_SRC				80
-+#define GCC_PCIE_2_PIPE_CLK					81
-+#define GCC_PCIE_2_SLV_AXI_CLK					82
-+#define GCC_PCIE_2_SLV_Q2A_AXI_CLK				83
-+#define GCC_PCIE_3_AUX_CLK					84
-+#define GCC_PCIE_3_AUX_CLK_SRC					85
-+#define GCC_PCIE_3_CFG_AHB_CLK					86
-+#define GCC_PCIE_3_MSTR_AXI_CLK					87
-+#define GCC_PCIE_3_PHY_AUX_CLK					88
-+#define GCC_PCIE_3_PHY_RCHNG_CLK				89
-+#define GCC_PCIE_3_PHY_RCHNG_CLK_SRC				90
-+#define GCC_PCIE_3_PIPE_CLK					91
-+#define GCC_PCIE_3_PIPE_DIV_CLK_SRC				92
-+#define GCC_PCIE_3_PIPEDIV2_CLK					93
-+#define GCC_PCIE_3_SLV_AXI_CLK					94
-+#define GCC_PCIE_3_SLV_Q2A_AXI_CLK				95
-+#define GCC_PCIE_4_AUX_CLK					96
-+#define GCC_PCIE_4_AUX_CLK_SRC					97
-+#define GCC_PCIE_4_CFG_AHB_CLK					98
-+#define GCC_PCIE_4_MSTR_AXI_CLK					99
-+#define GCC_PCIE_4_PHY_RCHNG_CLK				100
-+#define GCC_PCIE_4_PHY_RCHNG_CLK_SRC				101
-+#define GCC_PCIE_4_PIPE_CLK					102
-+#define GCC_PCIE_4_PIPE_DIV_CLK_SRC				103
-+#define GCC_PCIE_4_PIPEDIV2_CLK					104
-+#define GCC_PCIE_4_SLV_AXI_CLK					105
-+#define GCC_PCIE_4_SLV_Q2A_AXI_CLK				106
-+#define GCC_PCIE_5_AUX_CLK					107
-+#define GCC_PCIE_5_AUX_CLK_SRC					108
-+#define GCC_PCIE_5_CFG_AHB_CLK					109
-+#define GCC_PCIE_5_MSTR_AXI_CLK					110
-+#define GCC_PCIE_5_PHY_RCHNG_CLK				111
-+#define GCC_PCIE_5_PHY_RCHNG_CLK_SRC				112
-+#define GCC_PCIE_5_PIPE_CLK					113
-+#define GCC_PCIE_5_PIPE_DIV_CLK_SRC				114
-+#define GCC_PCIE_5_PIPEDIV2_CLK					115
-+#define GCC_PCIE_5_SLV_AXI_CLK					116
-+#define GCC_PCIE_5_SLV_Q2A_AXI_CLK				117
-+#define GCC_PCIE_6A_AUX_CLK					118
-+#define GCC_PCIE_6A_AUX_CLK_SRC					119
-+#define GCC_PCIE_6A_CFG_AHB_CLK					120
-+#define GCC_PCIE_6A_MSTR_AXI_CLK				121
-+#define GCC_PCIE_6A_PHY_AUX_CLK					122
-+#define GCC_PCIE_6A_PHY_RCHNG_CLK				123
-+#define GCC_PCIE_6A_PHY_RCHNG_CLK_SRC				124
-+#define GCC_PCIE_6A_PIPE_CLK					125
-+#define GCC_PCIE_6A_PIPE_DIV_CLK_SRC				126
-+#define GCC_PCIE_6A_PIPEDIV2_CLK				127
-+#define GCC_PCIE_6A_SLV_AXI_CLK					128
-+#define GCC_PCIE_6A_SLV_Q2A_AXI_CLK				129
-+#define GCC_PCIE_6B_AUX_CLK					130
-+#define GCC_PCIE_6B_AUX_CLK_SRC					131
-+#define GCC_PCIE_6B_CFG_AHB_CLK					132
-+#define GCC_PCIE_6B_MSTR_AXI_CLK				133
-+#define GCC_PCIE_6B_PHY_AUX_CLK					134
-+#define GCC_PCIE_6B_PHY_RCHNG_CLK				135
-+#define GCC_PCIE_6B_PHY_RCHNG_CLK_SRC				136
-+#define GCC_PCIE_6B_PIPE_CLK					137
-+#define GCC_PCIE_6B_PIPE_DIV_CLK_SRC				138
-+#define GCC_PCIE_6B_PIPEDIV2_CLK				139
-+#define GCC_PCIE_6B_SLV_AXI_CLK					140
-+#define GCC_PCIE_6B_SLV_Q2A_AXI_CLK				141
-+#define GCC_PCIE_RSCC_AHB_CLK					142
-+#define GCC_PCIE_RSCC_XO_CLK					143
-+#define GCC_PCIE_RSCC_XO_CLK_SRC				144
-+#define GCC_PDM2_CLK						145
-+#define GCC_PDM2_CLK_SRC					146
-+#define GCC_PDM_AHB_CLK						147
-+#define GCC_PDM_XO4_CLK						148
-+#define GCC_QMIP_AV1E_AHB_CLK					149
-+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				150
-+#define GCC_QMIP_CAMERA_RT_AHB_CLK				151
-+#define GCC_QMIP_DISP_AHB_CLK					152
-+#define GCC_QMIP_GPU_AHB_CLK					153
-+#define GCC_QMIP_VIDEO_CV_CPU_AHB_CLK				154
-+#define GCC_QMIP_VIDEO_CVP_AHB_CLK				155
-+#define GCC_QMIP_VIDEO_V_CPU_AHB_CLK				156
-+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				157
-+#define GCC_QUPV3_WRAP0_CORE_2X_CLK				158
-+#define GCC_QUPV3_WRAP0_CORE_CLK				159
-+#define GCC_QUPV3_WRAP0_QSPI_S2_CLK				160
-+#define GCC_QUPV3_WRAP0_QSPI_S3_CLK				161
-+#define GCC_QUPV3_WRAP0_S0_CLK					162
-+#define GCC_QUPV3_WRAP0_S0_CLK_SRC				163
-+#define GCC_QUPV3_WRAP0_S1_CLK					164
-+#define GCC_QUPV3_WRAP0_S1_CLK_SRC				165
-+#define GCC_QUPV3_WRAP0_S2_CLK					166
-+#define GCC_QUPV3_WRAP0_S2_CLK_SRC				167
-+#define GCC_QUPV3_WRAP0_S2_DIV_CLK_SRC				168
-+#define GCC_QUPV3_WRAP0_S3_CLK					169
-+#define GCC_QUPV3_WRAP0_S3_CLK_SRC				170
-+#define GCC_QUPV3_WRAP0_S3_DIV_CLK_SRC				171
-+#define GCC_QUPV3_WRAP0_S4_CLK					172
-+#define GCC_QUPV3_WRAP0_S4_CLK_SRC				173
-+#define GCC_QUPV3_WRAP0_S5_CLK					174
-+#define GCC_QUPV3_WRAP0_S5_CLK_SRC				175
-+#define GCC_QUPV3_WRAP0_S6_CLK					176
-+#define GCC_QUPV3_WRAP0_S6_CLK_SRC				177
-+#define GCC_QUPV3_WRAP0_S7_CLK					178
-+#define GCC_QUPV3_WRAP0_S7_CLK_SRC				179
-+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				180
-+#define GCC_QUPV3_WRAP1_CORE_CLK				181
-+#define GCC_QUPV3_WRAP1_QSPI_S2_CLK				182
-+#define GCC_QUPV3_WRAP1_QSPI_S3_CLK				183
-+#define GCC_QUPV3_WRAP1_S0_CLK					184
-+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				185
-+#define GCC_QUPV3_WRAP1_S1_CLK					186
-+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				187
-+#define GCC_QUPV3_WRAP1_S2_CLK					188
-+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				189
-+#define GCC_QUPV3_WRAP1_S2_DIV_CLK_SRC				190
-+#define GCC_QUPV3_WRAP1_S3_CLK					191
-+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				192
-+#define GCC_QUPV3_WRAP1_S3_DIV_CLK_SRC				193
-+#define GCC_QUPV3_WRAP1_S4_CLK					194
-+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				195
-+#define GCC_QUPV3_WRAP1_S5_CLK					196
-+#define GCC_QUPV3_WRAP1_S5_CLK_SRC				197
-+#define GCC_QUPV3_WRAP1_S6_CLK					198
-+#define GCC_QUPV3_WRAP1_S6_CLK_SRC				199
-+#define GCC_QUPV3_WRAP1_S7_CLK					200
-+#define GCC_QUPV3_WRAP1_S7_CLK_SRC				201
-+#define GCC_QUPV3_WRAP2_CORE_2X_CLK				202
-+#define GCC_QUPV3_WRAP2_CORE_CLK				203
-+#define GCC_QUPV3_WRAP2_QSPI_S2_CLK				204
-+#define GCC_QUPV3_WRAP2_QSPI_S3_CLK				205
-+#define GCC_QUPV3_WRAP2_S0_CLK					206
-+#define GCC_QUPV3_WRAP2_S0_CLK_SRC				207
-+#define GCC_QUPV3_WRAP2_S1_CLK					208
-+#define GCC_QUPV3_WRAP2_S1_CLK_SRC				209
-+#define GCC_QUPV3_WRAP2_S2_CLK					210
-+#define GCC_QUPV3_WRAP2_S2_CLK_SRC				211
-+#define GCC_QUPV3_WRAP2_S2_DIV_CLK_SRC				212
-+#define GCC_QUPV3_WRAP2_S3_CLK					213
-+#define GCC_QUPV3_WRAP2_S3_CLK_SRC				214
-+#define GCC_QUPV3_WRAP2_S3_DIV_CLK_SRC				215
-+#define GCC_QUPV3_WRAP2_S4_CLK					216
-+#define GCC_QUPV3_WRAP2_S4_CLK_SRC				217
-+#define GCC_QUPV3_WRAP2_S5_CLK					218
-+#define GCC_QUPV3_WRAP2_S5_CLK_SRC				219
-+#define GCC_QUPV3_WRAP2_S6_CLK					220
-+#define GCC_QUPV3_WRAP2_S6_CLK_SRC				221
-+#define GCC_QUPV3_WRAP2_S7_CLK					222
-+#define GCC_QUPV3_WRAP2_S7_CLK_SRC				223
-+#define GCC_QUPV3_WRAP_0_M_AHB_CLK				224
-+#define GCC_QUPV3_WRAP_0_S_AHB_CLK				225
-+#define GCC_QUPV3_WRAP_1_M_AHB_CLK				226
-+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				227
-+#define GCC_QUPV3_WRAP_2_M_AHB_CLK				228
-+#define GCC_QUPV3_WRAP_2_S_AHB_CLK				229
-+#define GCC_SDCC2_AHB_CLK					230
-+#define GCC_SDCC2_APPS_CLK					231
-+#define GCC_SDCC2_APPS_CLK_SRC					232
-+#define GCC_SDCC4_AHB_CLK					233
-+#define GCC_SDCC4_APPS_CLK					234
-+#define GCC_SDCC4_APPS_CLK_SRC					235
-+#define GCC_SYS_NOC_USB_AXI_CLK					236
-+#define GCC_UFS_PHY_AHB_CLK					237
-+#define GCC_UFS_PHY_AXI_CLK					238
-+#define GCC_UFS_PHY_AXI_CLK_SRC					239
-+#define GCC_UFS_PHY_ICE_CORE_CLK				240
-+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC				241
-+#define GCC_UFS_PHY_PHY_AUX_CLK					242
-+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC				243
-+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK				244
-+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK				245
-+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK				246
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK				247
-+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC				248
-+#define GCC_USB20_MASTER_CLK					249
-+#define GCC_USB20_MASTER_CLK_SRC				250
-+#define GCC_USB20_MOCK_UTMI_CLK					251
-+#define GCC_USB20_MOCK_UTMI_CLK_SRC				252
-+#define GCC_USB20_MOCK_UTMI_POSTDIV_CLK_SRC			253
-+#define GCC_USB20_SLEEP_CLK					254
-+#define GCC_USB30_MP_MASTER_CLK					255
-+#define GCC_USB30_MP_MASTER_CLK_SRC				256
-+#define GCC_USB30_MP_MOCK_UTMI_CLK				257
-+#define GCC_USB30_MP_MOCK_UTMI_CLK_SRC				258
-+#define GCC_USB30_MP_MOCK_UTMI_POSTDIV_CLK_SRC			259
-+#define GCC_USB30_MP_SLEEP_CLK					260
-+#define GCC_USB30_PRIM_MASTER_CLK				261
-+#define GCC_USB30_PRIM_MASTER_CLK_SRC				262
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				263
-+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			264
-+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		265
-+#define GCC_USB30_PRIM_SLEEP_CLK				266
-+#define GCC_USB30_SEC_MASTER_CLK				267
-+#define GCC_USB30_SEC_MASTER_CLK_SRC				268
-+#define GCC_USB30_SEC_MOCK_UTMI_CLK				269
-+#define GCC_USB30_SEC_MOCK_UTMI_CLK_SRC				270
-+#define GCC_USB30_SEC_MOCK_UTMI_POSTDIV_CLK_SRC			271
-+#define GCC_USB30_SEC_SLEEP_CLK					272
-+#define GCC_USB30_TERT_MASTER_CLK				273
-+#define GCC_USB30_TERT_MASTER_CLK_SRC				274
-+#define GCC_USB30_TERT_MOCK_UTMI_CLK				275
-+#define GCC_USB30_TERT_MOCK_UTMI_CLK_SRC			276
-+#define GCC_USB30_TERT_MOCK_UTMI_POSTDIV_CLK_SRC		277
-+#define GCC_USB30_TERT_SLEEP_CLK				278
-+#define GCC_USB3_MP_PHY_AUX_CLK					279
-+#define GCC_USB3_MP_PHY_AUX_CLK_SRC				280
-+#define GCC_USB3_MP_PHY_COM_AUX_CLK				281
-+#define GCC_USB3_MP_PHY_PIPE_0_CLK				282
-+#define GCC_USB3_MP_PHY_PIPE_1_CLK				283
-+#define GCC_USB3_PRIM_PHY_AUX_CLK				284
-+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				285
-+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				286
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK				287
-+#define GCC_USB3_SEC_PHY_AUX_CLK				288
-+#define GCC_USB3_SEC_PHY_AUX_CLK_SRC				289
-+#define GCC_USB3_SEC_PHY_COM_AUX_CLK				290
-+#define GCC_USB3_SEC_PHY_PIPE_CLK				291
-+#define GCC_USB3_TERT_PHY_AUX_CLK				292
-+#define GCC_USB3_TERT_PHY_AUX_CLK_SRC				293
-+#define GCC_USB3_TERT_PHY_COM_AUX_CLK				294
-+#define GCC_USB3_TERT_PHY_PIPE_CLK				295
-+#define GCC_USB4_0_CFG_AHB_CLK					296
-+#define GCC_USB4_0_DP0_CLK					297
-+#define GCC_USB4_0_DP1_CLK					298
-+#define GCC_USB4_0_MASTER_CLK					299
-+#define GCC_USB4_0_MASTER_CLK_SRC				300
-+#define GCC_USB4_0_PHY_P2RR2P_PIPE_CLK				301
-+#define GCC_USB4_0_PHY_PCIE_PIPE_CLK				302
-+#define GCC_USB4_0_PHY_PCIE_PIPE_CLK_SRC			303
-+#define GCC_USB4_0_PHY_RX0_CLK					304
-+#define GCC_USB4_0_PHY_RX1_CLK					305
-+#define GCC_USB4_0_PHY_USB_PIPE_CLK				306
-+#define GCC_USB4_0_SB_IF_CLK					307
-+#define GCC_USB4_0_SB_IF_CLK_SRC				308
-+#define GCC_USB4_0_SYS_CLK					309
-+#define GCC_USB4_0_TMU_CLK					310
-+#define GCC_USB4_0_TMU_CLK_SRC					311
-+#define GCC_USB4_1_CFG_AHB_CLK					312
-+#define GCC_USB4_1_DP0_CLK					313
-+#define GCC_USB4_1_DP1_CLK					314
-+#define GCC_USB4_1_MASTER_CLK					315
-+#define GCC_USB4_1_MASTER_CLK_SRC				316
-+#define GCC_USB4_1_PHY_P2RR2P_PIPE_CLK				317
-+#define GCC_USB4_1_PHY_PCIE_PIPE_CLK				318
-+#define GCC_USB4_1_PHY_PCIE_PIPE_CLK_SRC			319
-+#define GCC_USB4_1_PHY_RX0_CLK					320
-+#define GCC_USB4_1_PHY_RX1_CLK					321
-+#define GCC_USB4_1_PHY_USB_PIPE_CLK				322
-+#define GCC_USB4_1_SB_IF_CLK					323
-+#define GCC_USB4_1_SB_IF_CLK_SRC				324
-+#define GCC_USB4_1_SYS_CLK					325
-+#define GCC_USB4_1_TMU_CLK					326
-+#define GCC_USB4_1_TMU_CLK_SRC					327
-+#define GCC_USB4_2_CFG_AHB_CLK					328
-+#define GCC_USB4_2_DP0_CLK					329
-+#define GCC_USB4_2_DP1_CLK					330
-+#define GCC_USB4_2_MASTER_CLK					331
-+#define GCC_USB4_2_MASTER_CLK_SRC				332
-+#define GCC_USB4_2_PHY_P2RR2P_PIPE_CLK				333
-+#define GCC_USB4_2_PHY_PCIE_PIPE_CLK				334
-+#define GCC_USB4_2_PHY_PCIE_PIPE_CLK_SRC			335
-+#define GCC_USB4_2_PHY_RX0_CLK					336
-+#define GCC_USB4_2_PHY_RX1_CLK					337
-+#define GCC_USB4_2_PHY_USB_PIPE_CLK				338
-+#define GCC_USB4_2_SB_IF_CLK					339
-+#define GCC_USB4_2_SB_IF_CLK_SRC				340
-+#define GCC_USB4_2_SYS_CLK					341
-+#define GCC_USB4_2_TMU_CLK					342
-+#define GCC_USB4_2_TMU_CLK_SRC					343
-+#define GCC_VIDEO_AHB_CLK					344
-+#define GCC_VIDEO_AXI0_CLK					345
-+#define GCC_VIDEO_AXI1_CLK					346
-+#define GCC_VIDEO_XO_CLK					347
-+#define GCC_PCIE_3_PIPE_CLK_SRC					348
-+#define GCC_PCIE_4_PIPE_CLK_SRC					349
-+#define GCC_PCIE_5_PIPE_CLK_SRC					350
-+#define GCC_PCIE_6A_PIPE_CLK_SRC				351
-+#define GCC_PCIE_6B_PIPE_CLK_SRC				352
-+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				353
-+#define GCC_USB3_SEC_PHY_PIPE_CLK_SRC				354
-+#define GCC_USB3_TERT_PHY_PIPE_CLK_SRC				355
-+#define GCC_USB34_PRIM_PHY_PIPE_CLK_SRC				356
-+#define GCC_USB34_SEC_PHY_PIPE_CLK_SRC				357
-+#define GCC_USB34_TERT_PHY_PIPE_CLK_SRC				358
-+#define GCC_USB4_0_PHY_DP0_CLK_SRC				359
-+#define GCC_USB4_0_PHY_DP1_CLK_SRC				360
-+#define GCC_USB4_0_PHY_P2RR2P_PIPE_CLK_SRC			361
-+#define GCC_USB4_0_PHY_PCIE_PIPE_MUX_CLK_SRC			362
-+#define GCC_USB4_0_PHY_RX0_CLK_SRC				363
-+#define GCC_USB4_0_PHY_RX1_CLK_SRC				364
-+#define GCC_USB4_0_PHY_SYS_CLK_SRC				365
-+#define GCC_USB4_1_PHY_DP0_CLK_SRC				366
-+#define GCC_USB4_1_PHY_DP1_CLK_SRC				367
-+#define GCC_USB4_1_PHY_P2RR2P_PIPE_CLK_SRC			368
-+#define GCC_USB4_1_PHY_PCIE_PIPE_MUX_CLK_SRC			369
-+#define GCC_USB4_1_PHY_RX0_CLK_SRC				370
-+#define GCC_USB4_1_PHY_RX1_CLK_SRC				371
-+#define GCC_USB4_1_PHY_SYS_CLK_SRC				372
-+#define GCC_USB4_2_PHY_DP0_CLK_SRC				373
-+#define GCC_USB4_2_PHY_DP1_CLK_SRC				374
-+#define GCC_USB4_2_PHY_P2RR2P_PIPE_CLK_SRC			375
-+#define GCC_USB4_2_PHY_PCIE_PIPE_MUX_CLK_SRC			376
-+#define GCC_USB4_2_PHY_RX0_CLK_SRC				377
-+#define GCC_USB4_2_PHY_RX1_CLK_SRC				378
-+#define GCC_USB4_2_PHY_SYS_CLK_SRC				379
-+
-+/* GCC power domains */
-+#define GCC_PCIE_0_TUNNEL_GDSC					0
-+#define GCC_PCIE_1_TUNNEL_GDSC					1
-+#define GCC_PCIE_2_TUNNEL_GDSC					2
-+#define GCC_PCIE_3_GDSC						3
-+#define GCC_PCIE_3_PHY_GDSC					4
-+#define GCC_PCIE_4_GDSC						5
-+#define GCC_PCIE_4_PHY_GDSC					6
-+#define GCC_PCIE_5_GDSC						7
-+#define GCC_PCIE_5_PHY_GDSC					8
-+#define GCC_PCIE_6_PHY_GDSC					9
-+#define GCC_PCIE_6A_GDSC					10
-+#define GCC_PCIE_6B_GDSC					11
-+#define GCC_UFS_MEM_PHY_GDSC					12
-+#define GCC_UFS_PHY_GDSC					13
-+#define GCC_USB20_PRIM_GDSC					14
-+#define GCC_USB30_MP_GDSC					15
-+#define GCC_USB30_PRIM_GDSC					16
-+#define GCC_USB30_SEC_GDSC					17
-+#define GCC_USB30_TERT_GDSC					18
-+#define GCC_USB3_MP_SS0_PHY_GDSC				19
-+#define GCC_USB3_MP_SS1_PHY_GDSC				20
-+#define GCC_USB4_0_GDSC						21
-+#define GCC_USB4_1_GDSC						22
-+#define GCC_USB4_2_GDSC						23
-+#define GCC_USB_0_PHY_GDSC					24
-+#define GCC_USB_1_PHY_GDSC					25
-+#define GCC_USB_2_PHY_GDSC					26
-+
-+/* GCC resets */
-+#define GCC_AV1E_BCR						0
-+#define GCC_CAMERA_BCR						1
-+#define GCC_DISPLAY_BCR						2
-+#define GCC_GPU_BCR						3
-+#define GCC_PCIE_0_LINK_DOWN_BCR				4
-+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR				5
-+#define GCC_PCIE_0_PHY_BCR					6
-+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR			7
-+#define GCC_PCIE_0_TUNNEL_BCR					8
-+#define GCC_PCIE_1_LINK_DOWN_BCR				9
-+#define GCC_PCIE_1_NOCSR_COM_PHY_BCR				10
-+#define GCC_PCIE_1_PHY_BCR					11
-+#define GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR			12
-+#define GCC_PCIE_1_TUNNEL_BCR					13
-+#define GCC_PCIE_2_LINK_DOWN_BCR				14
-+#define GCC_PCIE_2_NOCSR_COM_PHY_BCR				15
-+#define GCC_PCIE_2_PHY_BCR					16
-+#define GCC_PCIE_2_PHY_NOCSR_COM_PHY_BCR			17
-+#define GCC_PCIE_2_TUNNEL_BCR					18
-+#define GCC_PCIE_3_BCR						19
-+#define GCC_PCIE_3_LINK_DOWN_BCR				20
-+#define GCC_PCIE_3_NOCSR_COM_PHY_BCR				21
-+#define GCC_PCIE_3_PHY_BCR					22
-+#define GCC_PCIE_3_PHY_NOCSR_COM_PHY_BCR			23
-+#define GCC_PCIE_4_BCR						24
-+#define GCC_PCIE_4_LINK_DOWN_BCR				25
-+#define GCC_PCIE_4_NOCSR_COM_PHY_BCR				26
-+#define GCC_PCIE_4_PHY_BCR					27
-+#define GCC_PCIE_4_PHY_NOCSR_COM_PHY_BCR			28
-+#define GCC_PCIE_5_BCR						29
-+#define GCC_PCIE_5_LINK_DOWN_BCR				30
-+#define GCC_PCIE_5_NOCSR_COM_PHY_BCR				31
-+#define GCC_PCIE_5_PHY_BCR					32
-+#define GCC_PCIE_5_PHY_NOCSR_COM_PHY_BCR			33
-+#define GCC_PCIE_6A_BCR						34
-+#define GCC_PCIE_6A_LINK_DOWN_BCR				35
-+#define GCC_PCIE_6A_NOCSR_COM_PHY_BCR				36
-+#define GCC_PCIE_6A_PHY_BCR					37
-+#define GCC_PCIE_6A_PHY_NOCSR_COM_PHY_BCR			38
-+#define GCC_PCIE_6B_BCR						39
-+#define GCC_PCIE_6B_LINK_DOWN_BCR				40
-+#define GCC_PCIE_6B_NOCSR_COM_PHY_BCR				41
-+#define GCC_PCIE_6B_PHY_BCR					42
-+#define GCC_PCIE_6B_PHY_NOCSR_COM_PHY_BCR			43
-+#define GCC_PCIE_PHY_BCR					44
-+#define GCC_PCIE_PHY_CFG_AHB_BCR				45
-+#define GCC_PCIE_PHY_COM_BCR					46
-+#define GCC_PCIE_RSCC_BCR					47
-+#define GCC_PDM_BCR						48
-+#define GCC_QUPV3_WRAPPER_0_BCR					49
-+#define GCC_QUPV3_WRAPPER_1_BCR					50
-+#define GCC_QUPV3_WRAPPER_2_BCR					51
-+#define GCC_QUSB2PHY_HS0_MP_BCR					52
-+#define GCC_QUSB2PHY_HS1_MP_BCR					53
-+#define GCC_QUSB2PHY_PRIM_BCR					54
-+#define GCC_QUSB2PHY_SEC_BCR					55
-+#define GCC_QUSB2PHY_TERT_BCR					56
-+#define GCC_QUSB2PHY_USB20_HS_BCR				57
-+#define GCC_SDCC2_BCR						58
-+#define GCC_SDCC4_BCR						59
-+#define GCC_UFS_PHY_BCR						60
-+#define GCC_USB20_PRIM_BCR					61
-+#define GCC_USB30_MP_BCR					62
-+#define GCC_USB30_PRIM_BCR					63
-+#define GCC_USB30_SEC_BCR					64
-+#define GCC_USB30_TERT_BCR					65
-+#define GCC_USB3_MP_SS0_PHY_BCR					66
-+#define GCC_USB3_MP_SS1_PHY_BCR					67
-+#define GCC_USB3_PHY_PRIM_BCR					68
-+#define GCC_USB3_PHY_SEC_BCR					69
-+#define GCC_USB3_PHY_TERT_BCR					70
-+#define GCC_USB3_UNIPHY_MP0_BCR					71
-+#define GCC_USB3_UNIPHY_MP1_BCR					72
-+#define GCC_USB3PHY_PHY_PRIM_BCR				73
-+#define GCC_USB3PHY_PHY_SEC_BCR					74
-+#define GCC_USB3PHY_PHY_TERT_BCR				75
-+#define GCC_USB3UNIPHY_PHY_MP0_BCR				76
-+#define GCC_USB3UNIPHY_PHY_MP1_BCR				77
-+#define GCC_USB4_0_BCR						78
-+#define GCC_USB4_0_DP0_PHY_PRIM_BCR				79
-+#define GCC_USB4_1_DP0_PHY_SEC_BCR				80
-+#define GCC_USB4_2_DP0_PHY_TERT_BCR				81
-+#define GCC_USB4_1_BCR						82
-+#define GCC_USB4_2_BCR						83
-+#define GCC_USB_0_PHY_BCR					84
-+#define GCC_USB_1_PHY_BCR					85
-+#define GCC_USB_2_PHY_BCR					86
-+#define GCC_VIDEO_BCR						87
-+#define GCC_VIDEO_AXI0_CLK_ARES					88
-+#define GCC_VIDEO_AXI1_CLK_ARES					89
-+#define GCC_USB4_0_MISC_USB4_SYS_BCR				90
-+#define GCC_USB4_0_MISC_RX_CLK_0_BCR				91
-+#define GCC_USB4_0_MISC_RX_CLK_1_BCR				92
-+#define GCC_USB4_0_MISC_USB_PIPE_BCR				93
-+#define GCC_USB4_0_MISC_PCIE_PIPE_BCR				94
-+#define GCC_USB4_0_MISC_TMU_BCR					95
-+#define GCC_USB4_0_MISC_SB_IF_BCR				96
-+#define GCC_USB4_0_MISC_HIA_MSTR_BCR				97
-+#define GCC_USB4_0_MISC_AHB_BCR					98
-+#define GCC_USB4_0_MISC_DP0_MAX_PCLK_BCR			99
-+#define GCC_USB4_0_MISC_DP1_MAX_PCLK_BCR			100
-+#define GCC_USB4_1_MISC_USB4_SYS_BCR				101
-+#define GCC_USB4_1_MISC_RX_CLK_0_BCR				102
-+#define GCC_USB4_1_MISC_RX_CLK_1_BCR				103
-+#define GCC_USB4_1_MISC_USB_PIPE_BCR				104
-+#define GCC_USB4_1_MISC_PCIE_PIPE_BCR				105
-+#define GCC_USB4_1_MISC_TMU_BCR					106
-+#define GCC_USB4_1_MISC_SB_IF_BCR				107
-+#define GCC_USB4_1_MISC_HIA_MSTR_BCR				108
-+#define GCC_USB4_1_MISC_AHB_BCR					109
-+#define GCC_USB4_1_MISC_DP0_MAX_PCLK_BCR			110
-+#define GCC_USB4_1_MISC_DP1_MAX_PCLK_BCR			111
-+#define GCC_USB4_2_MISC_USB4_SYS_BCR				112
-+#define GCC_USB4_2_MISC_RX_CLK_0_BCR				113
-+#define GCC_USB4_2_MISC_RX_CLK_1_BCR				114
-+#define GCC_USB4_2_MISC_USB_PIPE_BCR				115
-+#define GCC_USB4_2_MISC_PCIE_PIPE_BCR				116
-+#define GCC_USB4_2_MISC_TMU_BCR					117
-+#define GCC_USB4_2_MISC_SB_IF_BCR				118
-+#define GCC_USB4_2_MISC_HIA_MSTR_BCR				119
-+#define GCC_USB4_2_MISC_AHB_BCR					120
-+#define GCC_USB4_2_MISC_DP0_MAX_PCLK_BCR			121
-+#define GCC_USB4_2_MISC_DP1_MAX_PCLK_BCR			122
-+#define GCC_USB4PHY_PHY_PRIM_BCR				123
-+#define GCC_USB4PHY_PHY_SEC_BCR					124
-+#define GCC_USB4PHY_PHY_TERT_BCR				125
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,x1e80100-gpucc.h b/include/dt-bindings/clock/qcom,x1e80100-gpucc.h
-new file mode 100644
-index 000000000000..27b8f50541fd
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,x1e80100-gpucc.h
-@@ -0,0 +1,54 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_X1E80100_GPU_CC_H
-+#define _DT_BINDINGS_CLK_QCOM_X1E80100_GPU_CC_H
-+
-+/* GPU_CC clocks */
-+#define GPU_CC_AHB_CLK						0
-+#define GPU_CC_CB_CLK						1
-+#define GPU_CC_CRC_AHB_CLK					2
-+#define GPU_CC_CX_FF_CLK					3
-+#define GPU_CC_CX_GMU_CLK					4
-+#define GPU_CC_CXO_AON_CLK					5
-+#define GPU_CC_CXO_CLK						6
-+#define GPU_CC_DEMET_CLK					7
-+#define GPU_CC_DEMET_DIV_CLK_SRC				8
-+#define GPU_CC_FF_CLK_SRC					9
-+#define GPU_CC_FREQ_MEASURE_CLK					10
-+#define GPU_CC_GMU_CLK_SRC					11
-+#define GPU_CC_GX_GMU_CLK					12
-+#define GPU_CC_GX_VSENSE_CLK					13
-+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK				14
-+#define GPU_CC_HUB_AON_CLK					15
-+#define GPU_CC_HUB_CLK_SRC					16
-+#define GPU_CC_HUB_CX_INT_CLK					17
-+#define GPU_CC_MEMNOC_GFX_CLK					18
-+#define GPU_CC_MND1X_0_GFX3D_CLK				19
-+#define GPU_CC_MND1X_1_GFX3D_CLK				20
-+#define GPU_CC_PLL0						21
-+#define GPU_CC_PLL1						22
-+#define GPU_CC_SLEEP_CLK					23
-+#define GPU_CC_XO_CLK_SRC					24
-+#define GPU_CC_XO_DIV_CLK_SRC					25
-+#define GPU_CC_CX_ACCU_SHIFT_CLK				26
-+#define GPU_CC_GX_ACCU_SHIFT_CLK				27
-+
-+/* GDSCs */
-+#define GPU_CX_GDSC						0
-+#define GPU_GX_GDSC						1
-+
-+/* GPU_CC resets */
-+#define GPU_CC_ACD_BCR                                          0
-+#define GPU_CC_CB_BCR                                           1
-+#define GPU_CC_CX_BCR                                           2
-+#define GPU_CC_FAST_HUB_BCR                                     3
-+#define GPU_CC_FF_BCR                                           4
-+#define GPU_CC_GFX3D_AON_BCR                                    5
-+#define GPU_CC_GMU_BCR                                          6
-+#define GPU_CC_GX_BCR                                           7
-+#define GPU_CC_XO_BCR                                           8
-+
-+#endif
-diff --git a/include/dt-bindings/clock/qcom,x1e80100-tcsr.h b/include/dt-bindings/clock/qcom,x1e80100-tcsr.h
-new file mode 100644
-index 000000000000..bae2c4654ee2
--- /dev/null
-+++ b/include/dt-bindings/clock/qcom,x1e80100-tcsr.h
-@@ -0,0 +1,23 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_QCOM_X1E80100_TCSR_CC_H
-+#define _DT_BINDINGS_CLK_QCOM_X1E80100_TCSR_CC_H
-+
-+/* TCSR CC clocks */
-+#define TCSR_PCIE_2L_4_CLKREF_EN				0
-+#define TCSR_PCIE_2L_5_CLKREF_EN				1
-+#define TCSR_PCIE_8L_CLKREF_EN					2
-+#define TCSR_USB3_MP0_CLKREF_EN					3
-+#define TCSR_USB3_MP1_CLKREF_EN					4
-+#define TCSR_USB2_1_CLKREF_EN					5
-+#define TCSR_UFS_PHY_CLKREF_EN					6
-+#define TCSR_USB4_1_CLKREF_EN					7
-+#define TCSR_USB4_2_CLKREF_EN					8
-+#define TCSR_USB2_2_CLKREF_EN					9
-+#define TCSR_PCIE_4L_CLKREF_EN					10
-+#define TCSR_EDP_CLKREF_EN					11
-+
-+#endif
-diff --git a/include/dt-bindings/clock/r8a73a4-clock.h b/include/dt-bindings/clock/r8a73a4-clock.h
-index 1ec4827b8091..655440a3e7c6 100644
--- a/include/dt-bindings/clock/r8a73a4-clock.h
-+++ b/include/dt-bindings/clock/r8a73a4-clock.h
-@@ -24,6 +24,10 @@
- #define R8A73A4_CLK_ZS		14
- #define R8A73A4_CLK_HP		15
- 
-+/* MSTP1 */
-+#define R8A73A4_CLK_TMU0	25
-+#define R8A73A4_CLK_TMU3	21
-+
- /* MSTP2 */
- #define R8A73A4_CLK_DMAC	18
- #define R8A73A4_CLK_SCIFB3	17
-diff --git a/include/dt-bindings/clock/r8a7779-clock.h b/include/dt-bindings/clock/r8a7779-clock.h
-index f0549234b7d8..e39acdc6499c 100644
--- a/include/dt-bindings/clock/r8a7779-clock.h
-+++ b/include/dt-bindings/clock/r8a7779-clock.h
-@@ -19,6 +19,7 @@
- #define R8A7779_CLK_OUT		7
- 
- /* MSTP 0 */
-+#define R8A7779_CLK_PWM		5
- #define R8A7779_CLK_HSPI	7
- #define R8A7779_CLK_TMU2	14
- #define R8A7779_CLK_TMU1	15
-@@ -56,5 +57,4 @@
- #define R8A7779_CLK_MMC1	30
- #define R8A7779_CLK_MMC0	31
- 
-
- #endif /* __DT_BINDINGS_CLOCK_R8A7779_H__ */
-diff --git a/include/dt-bindings/clock/r8a7790-clock.h b/include/dt-bindings/clock/r8a7790-clock.h
-deleted file mode 100644
-index c92ff1e60223..000000000000
--- a/include/dt-bindings/clock/r8a7790-clock.h
-+++ /dev/null
-@@ -1,158 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright 2013 Ideas On Board SPRL
- */
-
-#ifndef __DT_BINDINGS_CLOCK_R8A7790_H__
-#define __DT_BINDINGS_CLOCK_R8A7790_H__
-
-/* CPG */
-#define R8A7790_CLK_MAIN		0
-#define R8A7790_CLK_PLL0		1
-#define R8A7790_CLK_PLL1		2
-#define R8A7790_CLK_PLL3		3
-#define R8A7790_CLK_LB			4
-#define R8A7790_CLK_QSPI		5
-#define R8A7790_CLK_SDH			6
-#define R8A7790_CLK_SD0			7
-#define R8A7790_CLK_SD1			8
-#define R8A7790_CLK_Z			9
-#define R8A7790_CLK_RCAN		10
-#define R8A7790_CLK_ADSP		11
-
-/* MSTP0 */
-#define R8A7790_CLK_MSIOF0		0
-
-/* MSTP1 */
-#define R8A7790_CLK_VCP1		0
-#define R8A7790_CLK_VCP0		1
-#define R8A7790_CLK_VPC1		2
-#define R8A7790_CLK_VPC0		3
-#define R8A7790_CLK_JPU			6
-#define R8A7790_CLK_SSP1		9
-#define R8A7790_CLK_TMU1		11
-#define R8A7790_CLK_3DG			12
-#define R8A7790_CLK_2DDMAC		15
-#define R8A7790_CLK_FDP1_2		17
-#define R8A7790_CLK_FDP1_1		18
-#define R8A7790_CLK_FDP1_0		19
-#define R8A7790_CLK_TMU3		21
-#define R8A7790_CLK_TMU2		22
-#define R8A7790_CLK_CMT0		24
-#define R8A7790_CLK_TMU0		25
-#define R8A7790_CLK_VSP1_DU1		27
-#define R8A7790_CLK_VSP1_DU0		28
-#define R8A7790_CLK_VSP1_R		30
-#define R8A7790_CLK_VSP1_S		31
-
-/* MSTP2 */
-#define R8A7790_CLK_SCIFA2		2
-#define R8A7790_CLK_SCIFA1		3
-#define R8A7790_CLK_SCIFA0		4
-#define R8A7790_CLK_MSIOF2		5
-#define R8A7790_CLK_SCIFB0		6
-#define R8A7790_CLK_SCIFB1		7
-#define R8A7790_CLK_MSIOF1		8
-#define R8A7790_CLK_MSIOF3		15
-#define R8A7790_CLK_SCIFB2		16
-#define R8A7790_CLK_SYS_DMAC1		18
-#define R8A7790_CLK_SYS_DMAC0		19
-
-/* MSTP3 */
-#define R8A7790_CLK_IIC2		0
-#define R8A7790_CLK_TPU0		4
-#define R8A7790_CLK_MMCIF1		5
-#define R8A7790_CLK_SCIF2		10
-#define R8A7790_CLK_SDHI3		11
-#define R8A7790_CLK_SDHI2		12
-#define R8A7790_CLK_SDHI1		13
-#define R8A7790_CLK_SDHI0		14
-#define R8A7790_CLK_MMCIF0		15
-#define R8A7790_CLK_IIC0		18
-#define R8A7790_CLK_PCIEC		19
-#define R8A7790_CLK_IIC1		23
-#define R8A7790_CLK_SSUSB		28
-#define R8A7790_CLK_CMT1		29
-#define R8A7790_CLK_USBDMAC0		30
-#define R8A7790_CLK_USBDMAC1		31
-
-/* MSTP4 */
-#define R8A7790_CLK_IRQC		7
-#define R8A7790_CLK_INTC_SYS		8
-
-/* MSTP5 */
-#define R8A7790_CLK_AUDIO_DMAC1		1
-#define R8A7790_CLK_AUDIO_DMAC0		2
-#define R8A7790_CLK_ADSP_MOD		6
-#define R8A7790_CLK_THERMAL		22
-#define R8A7790_CLK_PWM			23
-
-/* MSTP7 */
-#define R8A7790_CLK_EHCI		3
-#define R8A7790_CLK_HSUSB		4
-#define R8A7790_CLK_HSCIF1		16
-#define R8A7790_CLK_HSCIF0		17
-#define R8A7790_CLK_SCIF1		20
-#define R8A7790_CLK_SCIF0		21
-#define R8A7790_CLK_DU2			22
-#define R8A7790_CLK_DU1			23
-#define R8A7790_CLK_DU0			24
-#define R8A7790_CLK_LVDS1		25
-#define R8A7790_CLK_LVDS0		26
-
-/* MSTP8 */
-#define R8A7790_CLK_MLB			2
-#define R8A7790_CLK_VIN3		8
-#define R8A7790_CLK_VIN2		9
-#define R8A7790_CLK_VIN1		10
-#define R8A7790_CLK_VIN0		11
-#define R8A7790_CLK_ETHERAVB		12
-#define R8A7790_CLK_ETHER		13
-#define R8A7790_CLK_SATA1		14
-#define R8A7790_CLK_SATA0		15
-
-/* MSTP9 */
-#define R8A7790_CLK_GPIO5		7
-#define R8A7790_CLK_GPIO4		8
-#define R8A7790_CLK_GPIO3		9
-#define R8A7790_CLK_GPIO2		10
-#define R8A7790_CLK_GPIO1		11
-#define R8A7790_CLK_GPIO0		12
-#define R8A7790_CLK_RCAN1		15
-#define R8A7790_CLK_RCAN0		16
-#define R8A7790_CLK_QSPI_MOD		17
-#define R8A7790_CLK_IICDVFS		26
-#define R8A7790_CLK_I2C3		28
-#define R8A7790_CLK_I2C2		29
-#define R8A7790_CLK_I2C1		30
-#define R8A7790_CLK_I2C0		31
-
-/* MSTP10 */
-#define R8A7790_CLK_SSI_ALL		5
-#define R8A7790_CLK_SSI9		6
-#define R8A7790_CLK_SSI8		7
-#define R8A7790_CLK_SSI7		8
-#define R8A7790_CLK_SSI6		9
-#define R8A7790_CLK_SSI5		10
-#define R8A7790_CLK_SSI4		11
-#define R8A7790_CLK_SSI3		12
-#define R8A7790_CLK_SSI2		13
-#define R8A7790_CLK_SSI1		14
-#define R8A7790_CLK_SSI0		15
-#define R8A7790_CLK_SCU_ALL		17
-#define R8A7790_CLK_SCU_DVC1		18
-#define R8A7790_CLK_SCU_DVC0		19
-#define R8A7790_CLK_SCU_CTU1_MIX1	20
-#define R8A7790_CLK_SCU_CTU0_MIX0	21
-#define R8A7790_CLK_SCU_SRC9		22
-#define R8A7790_CLK_SCU_SRC8		23
-#define R8A7790_CLK_SCU_SRC7		24
-#define R8A7790_CLK_SCU_SRC6		25
-#define R8A7790_CLK_SCU_SRC5		26
-#define R8A7790_CLK_SCU_SRC4		27
-#define R8A7790_CLK_SCU_SRC3		28
-#define R8A7790_CLK_SCU_SRC2		29
-#define R8A7790_CLK_SCU_SRC1		30
-#define R8A7790_CLK_SCU_SRC0		31
-
-#endif /* __DT_BINDINGS_CLOCK_R8A7790_H__ */
-diff --git a/include/dt-bindings/clock/r8a7791-clock.h b/include/dt-bindings/clock/r8a7791-clock.h
-deleted file mode 100644
-index bb4f18b1b3d5..000000000000
--- a/include/dt-bindings/clock/r8a7791-clock.h
-+++ /dev/null
-@@ -1,161 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright 2013 Ideas On Board SPRL
- */
-
-#ifndef __DT_BINDINGS_CLOCK_R8A7791_H__
-#define __DT_BINDINGS_CLOCK_R8A7791_H__
-
-/* CPG */
-#define R8A7791_CLK_MAIN		0
-#define R8A7791_CLK_PLL0		1
-#define R8A7791_CLK_PLL1		2
-#define R8A7791_CLK_PLL3		3
-#define R8A7791_CLK_LB			4
-#define R8A7791_CLK_QSPI		5
-#define R8A7791_CLK_SDH			6
-#define R8A7791_CLK_SD0			7
-#define R8A7791_CLK_Z			8
-#define R8A7791_CLK_RCAN		9
-#define R8A7791_CLK_ADSP		10
-
-/* MSTP0 */
-#define R8A7791_CLK_MSIOF0		0
-
-/* MSTP1 */
-#define R8A7791_CLK_VCP0		1
-#define R8A7791_CLK_VPC0		3
-#define R8A7791_CLK_JPU			6
-#define R8A7791_CLK_SSP1		9
-#define R8A7791_CLK_TMU1		11
-#define R8A7791_CLK_3DG			12
-#define R8A7791_CLK_2DDMAC		15
-#define R8A7791_CLK_FDP1_1		18
-#define R8A7791_CLK_FDP1_0		19
-#define R8A7791_CLK_TMU3		21
-#define R8A7791_CLK_TMU2		22
-#define R8A7791_CLK_CMT0		24
-#define R8A7791_CLK_TMU0		25
-#define R8A7791_CLK_VSP1_DU1		27
-#define R8A7791_CLK_VSP1_DU0		28
-#define R8A7791_CLK_VSP1_S		31
-
-/* MSTP2 */
-#define R8A7791_CLK_SCIFA2		2
-#define R8A7791_CLK_SCIFA1		3
-#define R8A7791_CLK_SCIFA0		4
-#define R8A7791_CLK_MSIOF2		5
-#define R8A7791_CLK_SCIFB0		6
-#define R8A7791_CLK_SCIFB1		7
-#define R8A7791_CLK_MSIOF1		8
-#define R8A7791_CLK_SCIFB2		16
-#define R8A7791_CLK_SYS_DMAC1		18
-#define R8A7791_CLK_SYS_DMAC0		19
-
-/* MSTP3 */
-#define R8A7791_CLK_TPU0		4
-#define R8A7791_CLK_SDHI2		11
-#define R8A7791_CLK_SDHI1		12
-#define R8A7791_CLK_SDHI0		14
-#define R8A7791_CLK_MMCIF0		15
-#define R8A7791_CLK_IIC0		18
-#define R8A7791_CLK_PCIEC		19
-#define R8A7791_CLK_IIC1		23
-#define R8A7791_CLK_SSUSB		28
-#define R8A7791_CLK_CMT1		29
-#define R8A7791_CLK_USBDMAC0		30
-#define R8A7791_CLK_USBDMAC1		31
-
-/* MSTP4 */
-#define R8A7791_CLK_IRQC		7
-#define R8A7791_CLK_INTC_SYS		8
-
-/* MSTP5 */
-#define R8A7791_CLK_AUDIO_DMAC1		1
-#define R8A7791_CLK_AUDIO_DMAC0		2
-#define R8A7791_CLK_ADSP_MOD		6
-#define R8A7791_CLK_THERMAL		22
-#define R8A7791_CLK_PWM			23
-
-/* MSTP7 */
-#define R8A7791_CLK_EHCI		3
-#define R8A7791_CLK_HSUSB		4
-#define R8A7791_CLK_HSCIF2		13
-#define R8A7791_CLK_SCIF5		14
-#define R8A7791_CLK_SCIF4		15
-#define R8A7791_CLK_HSCIF1		16
-#define R8A7791_CLK_HSCIF0		17
-#define R8A7791_CLK_SCIF3		18
-#define R8A7791_CLK_SCIF2		19
-#define R8A7791_CLK_SCIF1		20
-#define R8A7791_CLK_SCIF0		21
-#define R8A7791_CLK_DU1			23
-#define R8A7791_CLK_DU0			24
-#define R8A7791_CLK_LVDS0		26
-
-/* MSTP8 */
-#define R8A7791_CLK_IPMMU_SGX		0
-#define R8A7791_CLK_MLB			2
-#define R8A7791_CLK_VIN2		9
-#define R8A7791_CLK_VIN1		10
-#define R8A7791_CLK_VIN0		11
-#define R8A7791_CLK_ETHERAVB		12
-#define R8A7791_CLK_ETHER		13
-#define R8A7791_CLK_SATA1		14
-#define R8A7791_CLK_SATA0		15
-
-/* MSTP9 */
-#define R8A7791_CLK_GYROADC		1
-#define R8A7791_CLK_GPIO7		4
-#define R8A7791_CLK_GPIO6		5
-#define R8A7791_CLK_GPIO5		7
-#define R8A7791_CLK_GPIO4		8
-#define R8A7791_CLK_GPIO3		9
-#define R8A7791_CLK_GPIO2		10
-#define R8A7791_CLK_GPIO1		11
-#define R8A7791_CLK_GPIO0		12
-#define R8A7791_CLK_RCAN1		15
-#define R8A7791_CLK_RCAN0		16
-#define R8A7791_CLK_QSPI_MOD		17
-#define R8A7791_CLK_I2C5		25
-#define R8A7791_CLK_IICDVFS		26
-#define R8A7791_CLK_I2C4		27
-#define R8A7791_CLK_I2C3		28
-#define R8A7791_CLK_I2C2		29
-#define R8A7791_CLK_I2C1		30
-#define R8A7791_CLK_I2C0		31
-
-/* MSTP10 */
-#define R8A7791_CLK_SSI_ALL		5
-#define R8A7791_CLK_SSI9		6
-#define R8A7791_CLK_SSI8		7
-#define R8A7791_CLK_SSI7		8
-#define R8A7791_CLK_SSI6		9
-#define R8A7791_CLK_SSI5		10
-#define R8A7791_CLK_SSI4		11
-#define R8A7791_CLK_SSI3		12
-#define R8A7791_CLK_SSI2		13
-#define R8A7791_CLK_SSI1		14
-#define R8A7791_CLK_SSI0		15
-#define R8A7791_CLK_SCU_ALL		17
-#define R8A7791_CLK_SCU_DVC1		18
-#define R8A7791_CLK_SCU_DVC0		19
-#define R8A7791_CLK_SCU_CTU1_MIX1	20
-#define R8A7791_CLK_SCU_CTU0_MIX0	21
-#define R8A7791_CLK_SCU_SRC9		22
-#define R8A7791_CLK_SCU_SRC8		23
-#define R8A7791_CLK_SCU_SRC7		24
-#define R8A7791_CLK_SCU_SRC6		25
-#define R8A7791_CLK_SCU_SRC5		26
-#define R8A7791_CLK_SCU_SRC4		27
-#define R8A7791_CLK_SCU_SRC3		28
-#define R8A7791_CLK_SCU_SRC2		29
-#define R8A7791_CLK_SCU_SRC1		30
-#define R8A7791_CLK_SCU_SRC0		31
-
-/* MSTP11 */
-#define R8A7791_CLK_SCIFA3		6
-#define R8A7791_CLK_SCIFA4		7
-#define R8A7791_CLK_SCIFA5		8
-
-#endif /* __DT_BINDINGS_CLOCK_R8A7791_H__ */
-diff --git a/include/dt-bindings/clock/r8a7792-clock.h b/include/dt-bindings/clock/r8a7792-clock.h
-deleted file mode 100644
-index 2948d9ce3a14..000000000000
--- a/include/dt-bindings/clock/r8a7792-clock.h
-+++ /dev/null
-@@ -1,98 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2016 Cogent Embedded, Inc.
- */
-
-#ifndef __DT_BINDINGS_CLOCK_R8A7792_H__
-#define __DT_BINDINGS_CLOCK_R8A7792_H__
-
-/* CPG */
-#define R8A7792_CLK_MAIN		0
-#define R8A7792_CLK_PLL0		1
-#define R8A7792_CLK_PLL1		2
-#define R8A7792_CLK_PLL3		3
-#define R8A7792_CLK_LB			4
-#define R8A7792_CLK_QSPI		5
-
-/* MSTP0 */
-#define R8A7792_CLK_MSIOF0		0
-
-/* MSTP1 */
-#define R8A7792_CLK_JPU			6
-#define R8A7792_CLK_TMU1		11
-#define R8A7792_CLK_TMU3		21
-#define R8A7792_CLK_TMU2		22
-#define R8A7792_CLK_CMT0		24
-#define R8A7792_CLK_TMU0		25
-#define R8A7792_CLK_VSP1DU1		27
-#define R8A7792_CLK_VSP1DU0		28
-#define R8A7792_CLK_VSP1_SY		31
-
-/* MSTP2 */
-#define R8A7792_CLK_MSIOF1		8
-#define R8A7792_CLK_SYS_DMAC1		18
-#define R8A7792_CLK_SYS_DMAC0		19
-
-/* MSTP3 */
-#define R8A7792_CLK_TPU0		4
-#define R8A7792_CLK_SDHI0		14
-#define R8A7792_CLK_CMT1		29
-
-/* MSTP4 */
-#define R8A7792_CLK_IRQC		7
-#define R8A7792_CLK_INTC_SYS		8
-
-/* MSTP5 */
-#define R8A7792_CLK_AUDIO_DMAC0		2
-#define R8A7792_CLK_THERMAL		22
-#define R8A7792_CLK_PWM			23
-
-/* MSTP7 */
-#define R8A7792_CLK_HSCIF1		16
-#define R8A7792_CLK_HSCIF0		17
-#define R8A7792_CLK_SCIF3		18
-#define R8A7792_CLK_SCIF2		19
-#define R8A7792_CLK_SCIF1		20
-#define R8A7792_CLK_SCIF0		21
-#define R8A7792_CLK_DU1			23
-#define R8A7792_CLK_DU0			24
-
-/* MSTP8 */
-#define R8A7792_CLK_VIN5		4
-#define R8A7792_CLK_VIN4		5
-#define R8A7792_CLK_VIN3		8
-#define R8A7792_CLK_VIN2		9
-#define R8A7792_CLK_VIN1		10
-#define R8A7792_CLK_VIN0		11
-#define R8A7792_CLK_ETHERAVB		12
-
-/* MSTP9 */
-#define R8A7792_CLK_GPIO7		4
-#define R8A7792_CLK_GPIO6		5
-#define R8A7792_CLK_GPIO5		7
-#define R8A7792_CLK_GPIO4		8
-#define R8A7792_CLK_GPIO3		9
-#define R8A7792_CLK_GPIO2		10
-#define R8A7792_CLK_GPIO1		11
-#define R8A7792_CLK_GPIO0		12
-#define R8A7792_CLK_GPIO11		13
-#define R8A7792_CLK_GPIO10		14
-#define R8A7792_CLK_CAN1		15
-#define R8A7792_CLK_CAN0		16
-#define R8A7792_CLK_QSPI_MOD		17
-#define R8A7792_CLK_GPIO9		19
-#define R8A7792_CLK_GPIO8		21
-#define R8A7792_CLK_I2C5		25
-#define R8A7792_CLK_IICDVFS		26
-#define R8A7792_CLK_I2C4		27
-#define R8A7792_CLK_I2C3		28
-#define R8A7792_CLK_I2C2		29
-#define R8A7792_CLK_I2C1		30
-#define R8A7792_CLK_I2C0		31
-
-/* MSTP10 */
-#define R8A7792_CLK_SSI_ALL		5
-#define R8A7792_CLK_SSI4		11
-#define R8A7792_CLK_SSI3		12
-
-#endif /* __DT_BINDINGS_CLOCK_R8A7792_H__ */
-diff --git a/include/dt-bindings/clock/r8a7793-clock.h b/include/dt-bindings/clock/r8a7793-clock.h
-deleted file mode 100644
-index 49c66d8ed178..000000000000
--- a/include/dt-bindings/clock/r8a7793-clock.h
-+++ /dev/null
-@@ -1,159 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0
- *
- * r8a7793 clock definition
- *
- * Copyright (C) 2014  Renesas Electronics Corporation
- */
-
-#ifndef __DT_BINDINGS_CLOCK_R8A7793_H__
-#define __DT_BINDINGS_CLOCK_R8A7793_H__
-
-/* CPG */
-#define R8A7793_CLK_MAIN		0
-#define R8A7793_CLK_PLL0		1
-#define R8A7793_CLK_PLL1		2
-#define R8A7793_CLK_PLL3		3
-#define R8A7793_CLK_LB			4
-#define R8A7793_CLK_QSPI		5
-#define R8A7793_CLK_SDH			6
-#define R8A7793_CLK_SD0			7
-#define R8A7793_CLK_Z			8
-#define R8A7793_CLK_RCAN		9
-#define R8A7793_CLK_ADSP		10
-
-/* MSTP0 */
-#define R8A7793_CLK_MSIOF0		0
-
-/* MSTP1 */
-#define R8A7793_CLK_VCP0		1
-#define R8A7793_CLK_VPC0		3
-#define R8A7793_CLK_SSP1		9
-#define R8A7793_CLK_TMU1		11
-#define R8A7793_CLK_3DG			12
-#define R8A7793_CLK_2DDMAC		15
-#define R8A7793_CLK_FDP1_1		18
-#define R8A7793_CLK_FDP1_0		19
-#define R8A7793_CLK_TMU3		21
-#define R8A7793_CLK_TMU2		22
-#define R8A7793_CLK_CMT0		24
-#define R8A7793_CLK_TMU0		25
-#define R8A7793_CLK_VSP1_DU1		27
-#define R8A7793_CLK_VSP1_DU0		28
-#define R8A7793_CLK_VSP1_S		31
-
-/* MSTP2 */
-#define R8A7793_CLK_SCIFA2		2
-#define R8A7793_CLK_SCIFA1		3
-#define R8A7793_CLK_SCIFA0		4
-#define R8A7793_CLK_MSIOF2		5
-#define R8A7793_CLK_SCIFB0		6
-#define R8A7793_CLK_SCIFB1		7
-#define R8A7793_CLK_MSIOF1		8
-#define R8A7793_CLK_SCIFB2		16
-#define R8A7793_CLK_SYS_DMAC1		18
-#define R8A7793_CLK_SYS_DMAC0		19
-
-/* MSTP3 */
-#define R8A7793_CLK_TPU0		4
-#define R8A7793_CLK_SDHI2		11
-#define R8A7793_CLK_SDHI1		12
-#define R8A7793_CLK_SDHI0		14
-#define R8A7793_CLK_MMCIF0		15
-#define R8A7793_CLK_IIC0		18
-#define R8A7793_CLK_PCIEC		19
-#define R8A7793_CLK_IIC1		23
-#define R8A7793_CLK_SSUSB		28
-#define R8A7793_CLK_CMT1		29
-#define R8A7793_CLK_USBDMAC0		30
-#define R8A7793_CLK_USBDMAC1		31
-
-/* MSTP4 */
-#define R8A7793_CLK_IRQC		7
-#define R8A7793_CLK_INTC_SYS		8
-
-/* MSTP5 */
-#define R8A7793_CLK_AUDIO_DMAC1		1
-#define R8A7793_CLK_AUDIO_DMAC0		2
-#define R8A7793_CLK_ADSP_MOD		6
-#define R8A7793_CLK_THERMAL		22
-#define R8A7793_CLK_PWM			23
-
-/* MSTP7 */
-#define R8A7793_CLK_EHCI		3
-#define R8A7793_CLK_HSUSB		4
-#define R8A7793_CLK_HSCIF2		13
-#define R8A7793_CLK_SCIF5		14
-#define R8A7793_CLK_SCIF4		15
-#define R8A7793_CLK_HSCIF1		16
-#define R8A7793_CLK_HSCIF0		17
-#define R8A7793_CLK_SCIF3		18
-#define R8A7793_CLK_SCIF2		19
-#define R8A7793_CLK_SCIF1		20
-#define R8A7793_CLK_SCIF0		21
-#define R8A7793_CLK_DU1			23
-#define R8A7793_CLK_DU0			24
-#define R8A7793_CLK_LVDS0		26
-
-/* MSTP8 */
-#define R8A7793_CLK_IPMMU_SGX		0
-#define R8A7793_CLK_VIN2		9
-#define R8A7793_CLK_VIN1		10
-#define R8A7793_CLK_VIN0		11
-#define R8A7793_CLK_ETHER		13
-#define R8A7793_CLK_SATA1		14
-#define R8A7793_CLK_SATA0		15
-
-/* MSTP9 */
-#define R8A7793_CLK_GPIO7		4
-#define R8A7793_CLK_GPIO6		5
-#define R8A7793_CLK_GPIO5		7
-#define R8A7793_CLK_GPIO4		8
-#define R8A7793_CLK_GPIO3		9
-#define R8A7793_CLK_GPIO2		10
-#define R8A7793_CLK_GPIO1		11
-#define R8A7793_CLK_GPIO0		12
-#define R8A7793_CLK_RCAN1		15
-#define R8A7793_CLK_RCAN0		16
-#define R8A7793_CLK_QSPI_MOD		17
-#define R8A7793_CLK_I2C5		25
-#define R8A7793_CLK_IICDVFS		26
-#define R8A7793_CLK_I2C4		27
-#define R8A7793_CLK_I2C3		28
-#define R8A7793_CLK_I2C2		29
-#define R8A7793_CLK_I2C1		30
-#define R8A7793_CLK_I2C0		31
-
-/* MSTP10 */
-#define R8A7793_CLK_SSI_ALL		5
-#define R8A7793_CLK_SSI9		6
-#define R8A7793_CLK_SSI8		7
-#define R8A7793_CLK_SSI7		8
-#define R8A7793_CLK_SSI6		9
-#define R8A7793_CLK_SSI5		10
-#define R8A7793_CLK_SSI4		11
-#define R8A7793_CLK_SSI3		12
-#define R8A7793_CLK_SSI2		13
-#define R8A7793_CLK_SSI1		14
-#define R8A7793_CLK_SSI0		15
-#define R8A7793_CLK_SCU_ALL		17
-#define R8A7793_CLK_SCU_DVC1		18
-#define R8A7793_CLK_SCU_DVC0		19
-#define R8A7793_CLK_SCU_CTU1_MIX1	20
-#define R8A7793_CLK_SCU_CTU0_MIX0	21
-#define R8A7793_CLK_SCU_SRC9		22
-#define R8A7793_CLK_SCU_SRC8		23
-#define R8A7793_CLK_SCU_SRC7		24
-#define R8A7793_CLK_SCU_SRC6		25
-#define R8A7793_CLK_SCU_SRC5		26
-#define R8A7793_CLK_SCU_SRC4		27
-#define R8A7793_CLK_SCU_SRC3		28
-#define R8A7793_CLK_SCU_SRC2		29
-#define R8A7793_CLK_SCU_SRC1		30
-#define R8A7793_CLK_SCU_SRC0		31
-
-/* MSTP11 */
-#define R8A7793_CLK_SCIFA3		6
-#define R8A7793_CLK_SCIFA4		7
-#define R8A7793_CLK_SCIFA5		8
-
-#endif /* __DT_BINDINGS_CLOCK_R8A7793_H__ */
-diff --git a/include/dt-bindings/clock/r8a7794-clock.h b/include/dt-bindings/clock/r8a7794-clock.h
-deleted file mode 100644
-index 649f005782d0..000000000000
--- a/include/dt-bindings/clock/r8a7794-clock.h
-+++ /dev/null
-@@ -1,137 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+
- *
- * Copyright (C) 2014 Renesas Electronics Corporation
- * Copyright 2013 Ideas On Board SPRL
- */
-
-#ifndef __DT_BINDINGS_CLOCK_R8A7794_H__
-#define __DT_BINDINGS_CLOCK_R8A7794_H__
-
-/* CPG */
-#define R8A7794_CLK_MAIN		0
-#define R8A7794_CLK_PLL0		1
-#define R8A7794_CLK_PLL1		2
-#define R8A7794_CLK_PLL3		3
-#define R8A7794_CLK_LB			4
-#define R8A7794_CLK_QSPI		5
-#define R8A7794_CLK_SDH			6
-#define R8A7794_CLK_SD0			7
-#define R8A7794_CLK_RCAN		8
-
-/* MSTP0 */
-#define R8A7794_CLK_MSIOF0		0
-
-/* MSTP1 */
-#define R8A7794_CLK_VCP0		1
-#define R8A7794_CLK_VPC0		3
-#define R8A7794_CLK_TMU1		11
-#define R8A7794_CLK_3DG			12
-#define R8A7794_CLK_2DDMAC		15
-#define R8A7794_CLK_FDP1_0		19
-#define R8A7794_CLK_TMU3		21
-#define R8A7794_CLK_TMU2		22
-#define R8A7794_CLK_CMT0		24
-#define R8A7794_CLK_TMU0		25
-#define R8A7794_CLK_VSP1_DU0		28
-#define R8A7794_CLK_VSP1_S		31
-
-/* MSTP2 */
-#define R8A7794_CLK_SCIFA2		2
-#define R8A7794_CLK_SCIFA1		3
-#define R8A7794_CLK_SCIFA0		4
-#define R8A7794_CLK_MSIOF2		5
-#define R8A7794_CLK_SCIFB0		6
-#define R8A7794_CLK_SCIFB1		7
-#define R8A7794_CLK_MSIOF1		8
-#define R8A7794_CLK_SCIFB2		16
-#define R8A7794_CLK_SYS_DMAC1		18
-#define R8A7794_CLK_SYS_DMAC0		19
-
-/* MSTP3 */
-#define R8A7794_CLK_SDHI2		11
-#define R8A7794_CLK_SDHI1		12
-#define R8A7794_CLK_SDHI0		14
-#define R8A7794_CLK_MMCIF0		15
-#define R8A7794_CLK_IIC0		18
-#define R8A7794_CLK_IIC1		23
-#define R8A7794_CLK_CMT1		29
-#define R8A7794_CLK_USBDMAC0		30
-#define R8A7794_CLK_USBDMAC1		31
-
-/* MSTP4 */
-#define R8A7794_CLK_IRQC		7
-#define R8A7794_CLK_INTC_SYS		8
-
-/* MSTP5 */
-#define R8A7794_CLK_AUDIO_DMAC0		2
-#define R8A7794_CLK_PWM			23
-
-/* MSTP7 */
-#define R8A7794_CLK_EHCI		3
-#define R8A7794_CLK_HSUSB		4
-#define R8A7794_CLK_HSCIF2		13
-#define R8A7794_CLK_SCIF5		14
-#define R8A7794_CLK_SCIF4		15
-#define R8A7794_CLK_HSCIF1		16
-#define R8A7794_CLK_HSCIF0		17
-#define R8A7794_CLK_SCIF3		18
-#define R8A7794_CLK_SCIF2		19
-#define R8A7794_CLK_SCIF1		20
-#define R8A7794_CLK_SCIF0		21
-#define R8A7794_CLK_DU1			23
-#define R8A7794_CLK_DU0			24
-
-/* MSTP8 */
-#define R8A7794_CLK_VIN1		10
-#define R8A7794_CLK_VIN0		11
-#define R8A7794_CLK_ETHERAVB		12
-#define R8A7794_CLK_ETHER		13
-
-/* MSTP9 */
-#define R8A7794_CLK_GPIO6		5
-#define R8A7794_CLK_GPIO5		7
-#define R8A7794_CLK_GPIO4		8
-#define R8A7794_CLK_GPIO3		9
-#define R8A7794_CLK_GPIO2		10
-#define R8A7794_CLK_GPIO1		11
-#define R8A7794_CLK_GPIO0		12
-#define R8A7794_CLK_RCAN1		15
-#define R8A7794_CLK_RCAN0		16
-#define R8A7794_CLK_QSPI_MOD		17
-#define R8A7794_CLK_I2C5		25
-#define R8A7794_CLK_I2C4		27
-#define R8A7794_CLK_I2C3		28
-#define R8A7794_CLK_I2C2		29
-#define R8A7794_CLK_I2C1		30
-#define R8A7794_CLK_I2C0		31
-
-/* MSTP10 */
-#define R8A7794_CLK_SSI_ALL		5
-#define R8A7794_CLK_SSI9		6
-#define R8A7794_CLK_SSI8		7
-#define R8A7794_CLK_SSI7		8
-#define R8A7794_CLK_SSI6		9
-#define R8A7794_CLK_SSI5		10
-#define R8A7794_CLK_SSI4		11
-#define R8A7794_CLK_SSI3		12
-#define R8A7794_CLK_SSI2		13
-#define R8A7794_CLK_SSI1		14
-#define R8A7794_CLK_SSI0		15
-#define R8A7794_CLK_SCU_ALL		17
-#define R8A7794_CLK_SCU_DVC1		18
-#define R8A7794_CLK_SCU_DVC0		19
-#define R8A7794_CLK_SCU_CTU1_MIX1	20
-#define R8A7794_CLK_SCU_CTU0_MIX0	21
-#define R8A7794_CLK_SCU_SRC6		25
-#define R8A7794_CLK_SCU_SRC5		26
-#define R8A7794_CLK_SCU_SRC4		27
-#define R8A7794_CLK_SCU_SRC3		28
-#define R8A7794_CLK_SCU_SRC2		29
-#define R8A7794_CLK_SCU_SRC1		30
-
-/* MSTP11 */
-#define R8A7794_CLK_SCIFA3		6
-#define R8A7794_CLK_SCIFA4		7
-#define R8A7794_CLK_SCIFA5		8
-
-#endif /* __DT_BINDINGS_CLOCK_R8A7794_H__ */
-diff --git a/include/dt-bindings/clock/r8a779a0-cpg-mssr.h b/include/dt-bindings/clock/r8a779a0-cpg-mssr.h
-index f1d737ca7ca1..124a6b8856df 100644
--- a/include/dt-bindings/clock/r8a779a0-cpg-mssr.h
-+++ b/include/dt-bindings/clock/r8a779a0-cpg-mssr.h
-@@ -51,5 +51,6 @@
- #define R8A779A0_CLK_CBFUSA		40
- #define R8A779A0_CLK_R			41
- #define R8A779A0_CLK_OSC		42
-+#define R8A779A0_CLK_ZG			43
- 
- #endif /* __DT_BINDINGS_CLOCK_R8A779A0_CPG_MSSR_H__ */
-diff --git a/include/dt-bindings/clock/r8a779f0-cpg-mssr.h b/include/dt-bindings/clock/r8a779f0-cpg-mssr.h
-index f2ae1c6a82dd..c34be5624954 100644
--- a/include/dt-bindings/clock/r8a779f0-cpg-mssr.h
-+++ b/include/dt-bindings/clock/r8a779f0-cpg-mssr.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0 or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
- /*
-  * Copyright (C) 2021 Renesas Electronics Corp.
-  */
-diff --git a/include/dt-bindings/clock/r8a779g0-cpg-mssr.h b/include/dt-bindings/clock/r8a779g0-cpg-mssr.h
-index 754c54a6eb06..7850cdc62e28 100644
--- a/include/dt-bindings/clock/r8a779g0-cpg-mssr.h
-+++ b/include/dt-bindings/clock/r8a779g0-cpg-mssr.h
-@@ -86,5 +86,6 @@
- #define R8A779G0_CLK_CPEX		74
- #define R8A779G0_CLK_CBFUSA		75
- #define R8A779G0_CLK_R			76
-+#define R8A779G0_CLK_CP			77
- 
- #endif /* __DT_BINDINGS_CLOCK_R8A779G0_CPG_MSSR_H__ */
-diff --git a/include/dt-bindings/clock/r9a07g043-cpg.h b/include/dt-bindings/clock/r9a07g043-cpg.h
-index 77cde8effdc7..e1f65f1928cf 100644
--- a/include/dt-bindings/clock/r9a07g043-cpg.h
-+++ b/include/dt-bindings/clock/r9a07g043-cpg.h
-@@ -16,15 +16,15 @@
- #define R9A07G043_CLK_SD0		5
- #define R9A07G043_CLK_SD1		6
- #define R9A07G043_CLK_M0		7
-#define R9A07G043_CLK_M2		8
-#define R9A07G043_CLK_M3		9
-+#define R9A07G043_CLK_M2		8	/* RZ/G2UL Only */
-+#define R9A07G043_CLK_M3		9	/* RZ/G2UL Only */
- #define R9A07G043_CLK_HP		10
- #define R9A07G043_CLK_TSU		11
- #define R9A07G043_CLK_ZT		12
- #define R9A07G043_CLK_P0		13
- #define R9A07G043_CLK_P1		14
- #define R9A07G043_CLK_P2		15
-#define R9A07G043_CLK_AT		16
-+#define R9A07G043_CLK_AT		16	/* RZ/G2UL Only */
- #define R9A07G043_OSCCLK		17
- #define R9A07G043_CLK_P0_DIV2		18
- 
-@@ -200,5 +200,4 @@
- #define R9A07G043_AX45MP_CORE0_RESETN	78	/* RZ/Five Only */
- #define R9A07G043_IAX45_RESETN		79	/* RZ/Five Only */
- 
-
- #endif /* __DT_BINDINGS_CLOCK_R9A07G043_CPG_H__ */
-diff --git a/include/dt-bindings/clock/r9a08g045-cpg.h b/include/dt-bindings/clock/r9a08g045-cpg.h
-new file mode 100644
-index 000000000000..410725b778a8
--- /dev/null
-+++ b/include/dt-bindings/clock/r9a08g045-cpg.h
-@@ -0,0 +1,242 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-+ *
-+ * Copyright (C) 2023 Renesas Electronics Corp.
-+ */
-+#ifndef __DT_BINDINGS_CLOCK_R9A08G045_CPG_H__
-+#define __DT_BINDINGS_CLOCK_R9A08G045_CPG_H__
-+
-+#include <dt-bindings/clock/renesas-cpg-mssr.h>
-+
-+/* R9A08G045 CPG Core Clocks */
-+#define R9A08G045_CLK_I			0
-+#define R9A08G045_CLK_I2		1
-+#define R9A08G045_CLK_I3		2
-+#define R9A08G045_CLK_S0		3
-+#define R9A08G045_CLK_SPI0		4
-+#define R9A08G045_CLK_SPI1		5
-+#define R9A08G045_CLK_SD0		6
-+#define R9A08G045_CLK_SD1		7
-+#define R9A08G045_CLK_SD2		8
-+#define R9A08G045_CLK_M0		9
-+#define R9A08G045_CLK_HP		10
-+#define R9A08G045_CLK_TSU		11
-+#define R9A08G045_CLK_ZT		12
-+#define R9A08G045_CLK_P0		13
-+#define R9A08G045_CLK_P1		14
-+#define R9A08G045_CLK_P2		15
-+#define R9A08G045_CLK_P3		16
-+#define R9A08G045_CLK_P4		17
-+#define R9A08G045_CLK_P5		18
-+#define R9A08G045_CLK_AT		19
-+#define R9A08G045_CLK_OC0		20
-+#define R9A08G045_CLK_OC1		21
-+#define R9A08G045_OSCCLK		22
-+#define R9A08G045_OSCCLK2		23
-+#define R9A08G045_SWD			24
-+
-+/* R9A08G045 Module Clocks */
-+#define R9A08G045_OCTA_ACLK		0
-+#define R9A08G045_OCTA_MCLK		1
-+#define R9A08G045_CA55_SCLK		2
-+#define R9A08G045_CA55_PCLK		3
-+#define R9A08G045_CA55_ATCLK		4
-+#define R9A08G045_CA55_GICCLK		5
-+#define R9A08G045_CA55_PERICLK		6
-+#define R9A08G045_CA55_ACLK		7
-+#define R9A08G045_CA55_TSCLK		8
-+#define R9A08G045_SRAM_ACPU_ACLK0	9
-+#define R9A08G045_SRAM_ACPU_ACLK1	10
-+#define R9A08G045_SRAM_ACPU_ACLK2	11
-+#define R9A08G045_GIC600_GICCLK		12
-+#define R9A08G045_IA55_CLK		13
-+#define R9A08G045_IA55_PCLK		14
-+#define R9A08G045_MHU_PCLK		15
-+#define R9A08G045_SYC_CNT_CLK		16
-+#define R9A08G045_DMAC_ACLK		17
-+#define R9A08G045_DMAC_PCLK		18
-+#define R9A08G045_OSTM0_PCLK		19
-+#define R9A08G045_OSTM1_PCLK		20
-+#define R9A08G045_OSTM2_PCLK		21
-+#define R9A08G045_OSTM3_PCLK		22
-+#define R9A08G045_OSTM4_PCLK		23
-+#define R9A08G045_OSTM5_PCLK		24
-+#define R9A08G045_OSTM6_PCLK		25
-+#define R9A08G045_OSTM7_PCLK		26
-+#define R9A08G045_MTU_X_MCK_MTU3	27
-+#define R9A08G045_POE3_CLKM_POE		28
-+#define R9A08G045_GPT_PCLK		29
-+#define R9A08G045_POEG_A_CLKP		30
-+#define R9A08G045_POEG_B_CLKP		31
-+#define R9A08G045_POEG_C_CLKP		32
-+#define R9A08G045_POEG_D_CLKP		33
-+#define R9A08G045_WDT0_PCLK		34
-+#define R9A08G045_WDT0_CLK		35
-+#define R9A08G045_WDT1_PCLK		36
-+#define R9A08G045_WDT1_CLK		37
-+#define R9A08G045_WDT2_PCLK		38
-+#define R9A08G045_WDT2_CLK		39
-+#define R9A08G045_SPI_HCLK		40
-+#define R9A08G045_SPI_ACLK		41
-+#define R9A08G045_SPI_CLK		42
-+#define R9A08G045_SPI_CLKX2		43
-+#define R9A08G045_SDHI0_IMCLK		44
-+#define R9A08G045_SDHI0_IMCLK2		45
-+#define R9A08G045_SDHI0_CLK_HS		46
-+#define R9A08G045_SDHI0_ACLK		47
-+#define R9A08G045_SDHI1_IMCLK		48
-+#define R9A08G045_SDHI1_IMCLK2		49
-+#define R9A08G045_SDHI1_CLK_HS		50
-+#define R9A08G045_SDHI1_ACLK		51
-+#define R9A08G045_SDHI2_IMCLK		52
-+#define R9A08G045_SDHI2_IMCLK2		53
-+#define R9A08G045_SDHI2_CLK_HS		54
-+#define R9A08G045_SDHI2_ACLK		55
-+#define R9A08G045_SSI0_PCLK2		56
-+#define R9A08G045_SSI0_PCLK_SFR		57
-+#define R9A08G045_SSI1_PCLK2		58
-+#define R9A08G045_SSI1_PCLK_SFR		59
-+#define R9A08G045_SSI2_PCLK2		60
-+#define R9A08G045_SSI2_PCLK_SFR		61
-+#define R9A08G045_SSI3_PCLK2		62
-+#define R9A08G045_SSI3_PCLK_SFR		63
-+#define R9A08G045_SRC_CLKP		64
-+#define R9A08G045_USB_U2H0_HCLK		65
-+#define R9A08G045_USB_U2H1_HCLK		66
-+#define R9A08G045_USB_U2P_EXR_CPUCLK	67
-+#define R9A08G045_USB_PCLK		68
-+#define R9A08G045_ETH0_CLK_AXI		69
-+#define R9A08G045_ETH0_CLK_CHI		70
-+#define R9A08G045_ETH0_REFCLK		71
-+#define R9A08G045_ETH1_CLK_AXI		72
-+#define R9A08G045_ETH1_CLK_CHI		73
-+#define R9A08G045_ETH1_REFCLK		74
-+#define R9A08G045_I2C0_PCLK		75
-+#define R9A08G045_I2C1_PCLK		76
-+#define R9A08G045_I2C2_PCLK		77
-+#define R9A08G045_I2C3_PCLK		78
-+#define R9A08G045_SCIF0_CLK_PCK		79
-+#define R9A08G045_SCIF1_CLK_PCK		80
-+#define R9A08G045_SCIF2_CLK_PCK		81
-+#define R9A08G045_SCIF3_CLK_PCK		82
-+#define R9A08G045_SCIF4_CLK_PCK		83
-+#define R9A08G045_SCIF5_CLK_PCK		84
-+#define R9A08G045_SCI0_CLKP		85
-+#define R9A08G045_SCI1_CLKP		86
-+#define R9A08G045_IRDA_CLKP		87
-+#define R9A08G045_RSPI0_CLKB		88
-+#define R9A08G045_RSPI1_CLKB		89
-+#define R9A08G045_RSPI2_CLKB		90
-+#define R9A08G045_RSPI3_CLKB		91
-+#define R9A08G045_RSPI4_CLKB		92
-+#define R9A08G045_CANFD_PCLK		93
-+#define R9A08G045_CANFD_CLK_RAM		94
-+#define R9A08G045_GPIO_HCLK		95
-+#define R9A08G045_ADC_ADCLK		96
-+#define R9A08G045_ADC_PCLK		97
-+#define R9A08G045_TSU_PCLK		98
-+#define R9A08G045_PDM_PCLK		99
-+#define R9A08G045_PDM_CCLK		100
-+#define R9A08G045_PCI_ACLK		101
-+#define R9A08G045_PCI_CLKL1PM		102
-+#define R9A08G045_SPDIF_PCLK		103
-+#define R9A08G045_I3C_PCLK		104
-+#define R9A08G045_I3C_TCLK		105
-+#define R9A08G045_VBAT_BCLK		106
-+
-+/* R9A08G045 Resets */
-+#define R9A08G045_CA55_RST_1_0		0
-+#define R9A08G045_CA55_RST_3_0		1
-+#define R9A08G045_CA55_RST_4		2
-+#define R9A08G045_CA55_RST_5		3
-+#define R9A08G045_CA55_RST_6		4
-+#define R9A08G045_CA55_RST_7		5
-+#define R9A08G045_CA55_RST_8		6
-+#define R9A08G045_CA55_RST_9		7
-+#define R9A08G045_CA55_RST_10		8
-+#define R9A08G045_CA55_RST_11		9
-+#define R9A08G045_CA55_RST_12		10
-+#define R9A08G045_SRAM_ACPU_ARESETN0	11
-+#define R9A08G045_SRAM_ACPU_ARESETN1	12
-+#define R9A08G045_SRAM_ACPU_ARESETN2	13
-+#define R9A08G045_GIC600_GICRESET_N	14
-+#define R9A08G045_GIC600_DBG_GICRESET_N	15
-+#define R9A08G045_IA55_RESETN		16
-+#define R9A08G045_MHU_RESETN		17
-+#define R9A08G045_DMAC_ARESETN		18
-+#define R9A08G045_DMAC_RST_ASYNC	19
-+#define R9A08G045_SYC_RESETN		20
-+#define R9A08G045_OSTM0_PRESETZ		21
-+#define R9A08G045_OSTM1_PRESETZ		22
-+#define R9A08G045_OSTM2_PRESETZ		23
-+#define R9A08G045_OSTM3_PRESETZ		24
-+#define R9A08G045_OSTM4_PRESETZ		25
-+#define R9A08G045_OSTM5_PRESETZ		26
-+#define R9A08G045_OSTM6_PRESETZ		27
-+#define R9A08G045_OSTM7_PRESETZ		28
-+#define R9A08G045_MTU_X_PRESET_MTU3	29
-+#define R9A08G045_POE3_RST_M_REG	30
-+#define R9A08G045_GPT_RST_C		31
-+#define R9A08G045_POEG_A_RST		32
-+#define R9A08G045_POEG_B_RST		33
-+#define R9A08G045_POEG_C_RST		34
-+#define R9A08G045_POEG_D_RST		35
-+#define R9A08G045_WDT0_PRESETN		36
-+#define R9A08G045_WDT1_PRESETN		37
-+#define R9A08G045_WDT2_PRESETN		38
-+#define R9A08G045_SPI_HRESETN		39
-+#define R9A08G045_SPI_ARESETN		40
-+#define R9A08G045_SDHI0_IXRST		41
-+#define R9A08G045_SDHI1_IXRST		42
-+#define R9A08G045_SDHI2_IXRST		43
-+#define R9A08G045_SSI0_RST_M2_REG	44
-+#define R9A08G045_SSI1_RST_M2_REG	45
-+#define R9A08G045_SSI2_RST_M2_REG	46
-+#define R9A08G045_SSI3_RST_M2_REG	47
-+#define R9A08G045_SRC_RST		48
-+#define R9A08G045_USB_U2H0_HRESETN	49
-+#define R9A08G045_USB_U2H1_HRESETN	50
-+#define R9A08G045_USB_U2P_EXL_SYSRST	51
-+#define R9A08G045_USB_PRESETN		52
-+#define R9A08G045_ETH0_RST_HW_N		53
-+#define R9A08G045_ETH1_RST_HW_N		54
-+#define R9A08G045_I2C0_MRST		55
-+#define R9A08G045_I2C1_MRST		56
-+#define R9A08G045_I2C2_MRST		57
-+#define R9A08G045_I2C3_MRST		58
-+#define R9A08G045_SCIF0_RST_SYSTEM_N	59
-+#define R9A08G045_SCIF1_RST_SYSTEM_N	60
-+#define R9A08G045_SCIF2_RST_SYSTEM_N	61
-+#define R9A08G045_SCIF3_RST_SYSTEM_N	62
-+#define R9A08G045_SCIF4_RST_SYSTEM_N	63
-+#define R9A08G045_SCIF5_RST_SYSTEM_N	64
-+#define R9A08G045_SCI0_RST		65
-+#define R9A08G045_SCI1_RST		66
-+#define R9A08G045_IRDA_RST		67
-+#define R9A08G045_RSPI0_RST		68
-+#define R9A08G045_RSPI1_RST		69
-+#define R9A08G045_RSPI2_RST		70
-+#define R9A08G045_RSPI3_RST		71
-+#define R9A08G045_RSPI4_RST		72
-+#define R9A08G045_CANFD_RSTP_N		73
-+#define R9A08G045_CANFD_RSTC_N		74
-+#define R9A08G045_GPIO_RSTN		75
-+#define R9A08G045_GPIO_PORT_RESETN	76
-+#define R9A08G045_GPIO_SPARE_RESETN	77
-+#define R9A08G045_ADC_PRESETN		78
-+#define R9A08G045_ADC_ADRST_N		79
-+#define R9A08G045_TSU_PRESETN		80
-+#define R9A08G045_OCTA_ARESETN		81
-+#define R9A08G045_PDM0_PRESETNT		82
-+#define R9A08G045_PCI_ARESETN		83
-+#define R9A08G045_PCI_RST_B		84
-+#define R9A08G045_PCI_RST_GP_B		85
-+#define R9A08G045_PCI_RST_PS_B		86
-+#define R9A08G045_PCI_RST_RSM_B		87
-+#define R9A08G045_PCI_RST_CFG_B		88
-+#define R9A08G045_PCI_RST_LOAD_B	89
-+#define R9A08G045_SPDIF_RST		90
-+#define R9A08G045_I3C_TRESETN		91
-+#define R9A08G045_I3C_PRESETN		92
-+#define R9A08G045_VBAT_BRESETN		93
-+
-+#endif /* __DT_BINDINGS_CLOCK_R9A08G045_CPG_H__ */
-diff --git a/include/dt-bindings/clock/raspberrypi,rp1-clocks.h b/include/dt-bindings/clock/raspberrypi,rp1-clocks.h
-new file mode 100644
-index 000000000000..7915fb8197bf
--- /dev/null
-+++ b/include/dt-bindings/clock/raspberrypi,rp1-clocks.h
-@@ -0,0 +1,65 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2021 Raspberry Pi Ltd.
-+ */
-+
-+#ifndef __DT_BINDINGS_CLOCK_RASPBERRYPI_RP1
-+#define __DT_BINDINGS_CLOCK_RASPBERRYPI_RP1
-+
-+#define RP1_PLL_SYS_CORE		0
-+#define RP1_PLL_AUDIO_CORE		1
-+#define RP1_PLL_VIDEO_CORE		2
-+
-+#define RP1_PLL_SYS			3
-+#define RP1_PLL_AUDIO			4
-+#define RP1_PLL_VIDEO			5
-+
-+#define RP1_PLL_SYS_PRI_PH		6
-+#define RP1_PLL_SYS_SEC_PH		7
-+#define RP1_PLL_AUDIO_PRI_PH		8
-+
-+#define RP1_PLL_SYS_SEC			9
-+#define RP1_PLL_AUDIO_SEC		10
-+#define RP1_PLL_VIDEO_SEC		11
-+
-+#define RP1_CLK_SYS			12
-+#define RP1_CLK_SLOW_SYS		13
-+#define RP1_CLK_DMA			14
-+#define RP1_CLK_UART			15
-+#define RP1_CLK_ETH			16
-+#define RP1_CLK_PWM0			17
-+#define RP1_CLK_PWM1			18
-+#define RP1_CLK_AUDIO_IN		19
-+#define RP1_CLK_AUDIO_OUT		20
-+#define RP1_CLK_I2S			21
-+#define RP1_CLK_MIPI0_CFG		22
-+#define RP1_CLK_MIPI1_CFG		23
-+#define RP1_CLK_PCIE_AUX		24
-+#define RP1_CLK_USBH0_MICROFRAME	25
-+#define RP1_CLK_USBH1_MICROFRAME	26
-+#define RP1_CLK_USBH0_SUSPEND		27
-+#define RP1_CLK_USBH1_SUSPEND		28
-+#define RP1_CLK_ETH_TSU			29
-+#define RP1_CLK_ADC			30
-+#define RP1_CLK_SDIO_TIMER		31
-+#define RP1_CLK_SDIO_ALT_SRC		32
-+#define RP1_CLK_GP0			33
-+#define RP1_CLK_GP1			34
-+#define RP1_CLK_GP2			35
-+#define RP1_CLK_GP3			36
-+#define RP1_CLK_GP4			37
-+#define RP1_CLK_GP5			38
-+#define RP1_CLK_VEC			39
-+#define RP1_CLK_DPI			40
-+#define RP1_CLK_MIPI0_DPI		41
-+#define RP1_CLK_MIPI1_DPI		42
-+
-+/* Extra PLL output channels - RP1B0 only */
-+#define RP1_PLL_VIDEO_PRI_PH		43
-+#define RP1_PLL_AUDIO_TERN		44
-+
-+/* MIPI clocks managed by the DSI driver */
-+#define RP1_CLK_MIPI0_DSI_BYTECLOCK	45
-+#define RP1_CLK_MIPI1_DSI_BYTECLOCK	46
-+
-+#endif
-diff --git a/include/dt-bindings/clock/renesas,r8a779h0-cpg-mssr.h b/include/dt-bindings/clock/renesas,r8a779h0-cpg-mssr.h
-new file mode 100644
-index 000000000000..7ab6cfbaf901
--- /dev/null
-+++ b/include/dt-bindings/clock/renesas,r8a779h0-cpg-mssr.h
-@@ -0,0 +1,96 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2023 Renesas Electronics Corp.
-+ */
-+#ifndef __DT_BINDINGS_CLOCK_RENESAS_R8A779H0_CPG_MSSR_H__
-+#define __DT_BINDINGS_CLOCK_RENESAS_R8A779H0_CPG_MSSR_H__
-+
-+#include <dt-bindings/clock/renesas-cpg-mssr.h>
-+
-+/* r8a779h0 CPG Core Clocks */
-+
-+#define R8A779H0_CLK_ZX			0
-+#define R8A779H0_CLK_ZD			1
-+#define R8A779H0_CLK_ZS			2
-+#define R8A779H0_CLK_ZT			3
-+#define R8A779H0_CLK_ZTR		4
-+#define R8A779H0_CLK_S0D2		5
-+#define R8A779H0_CLK_S0D3		6
-+#define R8A779H0_CLK_S0D4		7
-+#define R8A779H0_CLK_S0D1_VIO		8
-+#define R8A779H0_CLK_S0D2_VIO		9
-+#define R8A779H0_CLK_S0D4_VIO		10
-+#define R8A779H0_CLK_S0D8_VIO		11
-+#define R8A779H0_CLK_VIOBUSD1		12
-+#define R8A779H0_CLK_VIOBUSD2		13
-+#define R8A779H0_CLK_S0D1_VC		14
-+#define R8A779H0_CLK_S0D2_VC		15
-+#define R8A779H0_CLK_S0D4_VC		16
-+#define R8A779H0_CLK_VCBUSD1		17
-+#define R8A779H0_CLK_VCBUSD2		18
-+#define R8A779H0_CLK_S0D2_MM		19
-+#define R8A779H0_CLK_S0D4_MM		20
-+#define R8A779H0_CLK_S0D2_U3DG		21
-+#define R8A779H0_CLK_S0D4_U3DG		22
-+#define R8A779H0_CLK_S0D2_RT		23
-+#define R8A779H0_CLK_S0D3_RT		24
-+#define R8A779H0_CLK_S0D4_RT		25
-+#define R8A779H0_CLK_S0D6_RT		26
-+#define R8A779H0_CLK_S0D2_PER		27
-+#define R8A779H0_CLK_S0D3_PER		28
-+#define R8A779H0_CLK_S0D4_PER		29
-+#define R8A779H0_CLK_S0D6_PER		30
-+#define R8A779H0_CLK_S0D12_PER		31
-+#define R8A779H0_CLK_S0D24_PER		32
-+#define R8A779H0_CLK_S0D1_HSC		33
-+#define R8A779H0_CLK_S0D2_HSC		34
-+#define R8A779H0_CLK_S0D4_HSC		35
-+#define R8A779H0_CLK_S0D8_HSC		36
-+#define R8A779H0_CLK_SVD1_IR		37
-+#define R8A779H0_CLK_SVD2_IR		38
-+#define R8A779H0_CLK_IMPAD1		39
-+#define R8A779H0_CLK_IMPAD4		40
-+#define R8A779H0_CLK_IMPB		41
-+#define R8A779H0_CLK_SVD1_VIP		42
-+#define R8A779H0_CLK_SVD2_VIP		43
-+#define R8A779H0_CLK_CL			44
-+#define R8A779H0_CLK_CL16M		45
-+#define R8A779H0_CLK_CL16M_MM		46
-+#define R8A779H0_CLK_CL16M_RT		47
-+#define R8A779H0_CLK_CL16M_PER		48
-+#define R8A779H0_CLK_CL16M_HSC		49
-+#define R8A779H0_CLK_ZC0		50
-+#define R8A779H0_CLK_ZC1		51
-+#define R8A779H0_CLK_ZC2		52
-+#define R8A779H0_CLK_ZC3		53
-+#define R8A779H0_CLK_ZB3		54
-+#define R8A779H0_CLK_ZB3D2		55
-+#define R8A779H0_CLK_ZB3D4		56
-+#define R8A779H0_CLK_ZG			57
-+#define R8A779H0_CLK_SD0H		58
-+#define R8A779H0_CLK_SD0		59
-+#define R8A779H0_CLK_RPC		60
-+#define R8A779H0_CLK_RPCD2		61
-+#define R8A779H0_CLK_MSO		62
-+#define R8A779H0_CLK_CANFD		63
-+#define R8A779H0_CLK_CSI		64
-+#define R8A779H0_CLK_FRAY		65
-+#define R8A779H0_CLK_IPC		66
-+#define R8A779H0_CLK_SASYNCRT		67
-+#define R8A779H0_CLK_SASYNCPERD1	68
-+#define R8A779H0_CLK_SASYNCPERD2	69
-+#define R8A779H0_CLK_SASYNCPERD4	70
-+#define R8A779H0_CLK_DSIEXT		71
-+#define R8A779H0_CLK_DSIREF		72
-+#define R8A779H0_CLK_ADGH		73
-+#define R8A779H0_CLK_OSC		74
-+#define R8A779H0_CLK_ZR0		75
-+#define R8A779H0_CLK_ZR1		76
-+#define R8A779H0_CLK_ZR2		77
-+#define R8A779H0_CLK_RGMII		78
-+#define R8A779H0_CLK_CPEX		79
-+#define R8A779H0_CLK_CP			80
-+#define R8A779H0_CLK_CBFUSA		81
-+#define R8A779H0_CLK_R			82
-+
-+#endif /* __DT_BINDINGS_CLOCK_RENESAS_R8A779H0_CPG_MSSR_H__ */
-diff --git a/include/dt-bindings/clock/renesas,r9a08g045-vbattb.h b/include/dt-bindings/clock/renesas,r9a08g045-vbattb.h
-new file mode 100644
-index 000000000000..4cc8fc34b23c
--- /dev/null
-+++ b/include/dt-bindings/clock/renesas,r9a08g045-vbattb.h
-@@ -0,0 +1,13 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-+ *
-+ * Copyright (C) 2024 Renesas Electronics Corp.
-+ */
-+#ifndef __DT_BINDINGS_CLOCK_RENESAS_R9A08G045_VBATTB_H__
-+#define __DT_BINDINGS_CLOCK_RENESAS_R9A08G045_VBATTB_H__
-+
-+#define VBATTB_XC		0
-+#define VBATTB_XBYP		1
-+#define VBATTB_MUX		2
-+#define VBATTB_VBATTCLK		3
-+
-+#endif /* __DT_BINDINGS_CLOCK_RENESAS_R9A08G045_VBATTB_H__ */
-diff --git a/include/dt-bindings/clock/renesas,r9a09g047-cpg.h b/include/dt-bindings/clock/renesas,r9a09g047-cpg.h
-new file mode 100644
-index 000000000000..dab24740de3c
--- /dev/null
-+++ b/include/dt-bindings/clock/renesas,r9a09g047-cpg.h
-@@ -0,0 +1,28 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-+ *
-+ * Copyright (C) 2024 Renesas Electronics Corp.
-+ */
-+#ifndef __DT_BINDINGS_CLOCK_RENESAS_R9A09G047_CPG_H__
-+#define __DT_BINDINGS_CLOCK_RENESAS_R9A09G047_CPG_H__
-+
-+#include <dt-bindings/clock/renesas-cpg-mssr.h>
-+
-+/* Core Clock list */
-+#define R9A09G047_SYS_0_PCLK			0
-+#define R9A09G047_CA55_0_CORECLK0		1
-+#define R9A09G047_CA55_0_CORECLK1		2
-+#define R9A09G047_CA55_0_CORECLK2		3
-+#define R9A09G047_CA55_0_CORECLK3		4
-+#define R9A09G047_CA55_0_PERIPHCLK		5
-+#define R9A09G047_CM33_CLK0			6
-+#define R9A09G047_CST_0_SWCLKTCK		7
-+#define R9A09G047_IOTOP_0_SHCLK			8
-+#define R9A09G047_SPI_CLK_SPI			9
-+#define R9A09G047_GBETH_0_CLK_PTP_REF_I		10
-+#define R9A09G047_GBETH_1_CLK_PTP_REF_I		11
-+#define R9A09G047_USB3_0_REF_ALT_CLK_P		12
-+#define R9A09G047_USB3_0_CLKCORE		13
-+#define R9A09G047_USB2_0_CLK_CORE0		14
-+#define R9A09G047_USB2_0_CLK_CORE1		15
-+
-+#endif /* __DT_BINDINGS_CLOCK_RENESAS_R9A09G047_CPG_H__ */
-diff --git a/include/dt-bindings/clock/renesas,r9a09g056-cpg.h b/include/dt-bindings/clock/renesas,r9a09g056-cpg.h
-new file mode 100644
-index 000000000000..234dcf4f0f91
--- /dev/null
-+++ b/include/dt-bindings/clock/renesas,r9a09g056-cpg.h
-@@ -0,0 +1,27 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-+ *
-+ * Copyright (C) 2025 Renesas Electronics Corp.
-+ */
-+#ifndef __DT_BINDINGS_CLOCK_RENESAS_R9A09G056_CPG_H__
-+#define __DT_BINDINGS_CLOCK_RENESAS_R9A09G056_CPG_H__
-+
-+#include <dt-bindings/clock/renesas-cpg-mssr.h>
-+
-+/* Core Clock list */
-+#define R9A09G056_SYS_0_PCLK			0
-+#define R9A09G056_CA55_0_CORE_CLK0		1
-+#define R9A09G056_CA55_0_CORE_CLK1		2
-+#define R9A09G056_CA55_0_CORE_CLK2		3
-+#define R9A09G056_CA55_0_CORE_CLK3		4
-+#define R9A09G056_CA55_0_PERIPHCLK		5
-+#define R9A09G056_CM33_CLK0			6
-+#define R9A09G056_CST_0_SWCLKTCK		7
-+#define R9A09G056_IOTOP_0_SHCLK			8
-+#define R9A09G056_USB2_0_CLK_CORE0		9
-+#define R9A09G056_GBETH_0_CLK_PTP_REF_I		10
-+#define R9A09G056_GBETH_1_CLK_PTP_REF_I		11
-+#define R9A09G056_SPI_CLK_SPI			12
-+#define R9A09G056_USB3_0_REF_ALT_CLK_P		13
-+#define R9A09G056_USB3_0_CLKCORE		14
-+
-+#endif /* __DT_BINDINGS_CLOCK_RENESAS_R9A09G056_CPG_H__ */
-diff --git a/include/dt-bindings/clock/renesas,r9a09g057-cpg.h b/include/dt-bindings/clock/renesas,r9a09g057-cpg.h
-new file mode 100644
-index 000000000000..f91d7f72922a
--- /dev/null
-+++ b/include/dt-bindings/clock/renesas,r9a09g057-cpg.h
-@@ -0,0 +1,30 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-+ *
-+ * Copyright (C) 2024 Renesas Electronics Corp.
-+ */
-+#ifndef __DT_BINDINGS_CLOCK_RENESAS_R9A09G057_CPG_H__
-+#define __DT_BINDINGS_CLOCK_RENESAS_R9A09G057_CPG_H__
-+
-+#include <dt-bindings/clock/renesas-cpg-mssr.h>
-+
-+/* Core Clock list */
-+#define R9A09G057_SYS_0_PCLK			0
-+#define R9A09G057_CA55_0_CORE_CLK0		1
-+#define R9A09G057_CA55_0_CORE_CLK1		2
-+#define R9A09G057_CA55_0_CORE_CLK2		3
-+#define R9A09G057_CA55_0_CORE_CLK3		4
-+#define R9A09G057_CA55_0_PERIPHCLK		5
-+#define R9A09G057_CM33_CLK0			6
-+#define R9A09G057_CST_0_SWCLKTCK		7
-+#define R9A09G057_IOTOP_0_SHCLK			8
-+#define R9A09G057_USB2_0_CLK_CORE0		9
-+#define R9A09G057_USB2_0_CLK_CORE1		10
-+#define R9A09G057_GBETH_0_CLK_PTP_REF_I		11
-+#define R9A09G057_GBETH_1_CLK_PTP_REF_I		12
-+#define R9A09G057_SPI_CLK_SPI			13
-+#define R9A09G057_USB3_0_REF_ALT_CLK_P		14
-+#define R9A09G057_USB3_0_CLKCORE		15
-+#define R9A09G057_USB3_1_REF_ALT_CLK_P		16
-+#define R9A09G057_USB3_1_CLKCORE		17
-+
-+#endif /* __DT_BINDINGS_CLOCK_RENESAS_R9A09G057_CPG_H__ */
-diff --git a/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h b/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h
-new file mode 100644
-index 000000000000..2a805e06487b
--- /dev/null
-+++ b/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h
-@@ -0,0 +1,35 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-+ *
-+ * Copyright (C) 2025 Renesas Electronics Corp.
-+ */
-+
-+#ifndef __DT_BINDINGS_CLOCK_RENESAS_R9A09G077_CPG_H__
-+#define __DT_BINDINGS_CLOCK_RENESAS_R9A09G077_CPG_H__
-+
-+#include <dt-bindings/clock/renesas-cpg-mssr.h>
-+
-+/* R9A09G077 CPG Core Clocks */
-+#define R9A09G077_CLK_CA55C0		0
-+#define R9A09G077_CLK_CA55C1		1
-+#define R9A09G077_CLK_CA55C2		2
-+#define R9A09G077_CLK_CA55C3		3
-+#define R9A09G077_CLK_CA55S		4
-+#define R9A09G077_CLK_CR52_CPU0		5
-+#define R9A09G077_CLK_CR52_CPU1		6
-+#define R9A09G077_CLK_CKIO		7
-+#define R9A09G077_CLK_PCLKAH		8
-+#define R9A09G077_CLK_PCLKAM		9
-+#define R9A09G077_CLK_PCLKAL		10
-+#define R9A09G077_CLK_PCLKGPTL		11
-+#define R9A09G077_CLK_PCLKH		12
-+#define R9A09G077_CLK_PCLKM		13
-+#define R9A09G077_CLK_PCLKL		14
-+#define R9A09G077_SDHI_CLKHS		15
-+#define R9A09G077_USB_CLK		16
-+#define R9A09G077_ETCLKA		17
-+#define R9A09G077_ETCLKB		18
-+#define R9A09G077_ETCLKC		19
-+#define R9A09G077_ETCLKD		20
-+#define R9A09G077_ETCLKE		21
-+
-+#endif /* __DT_BINDINGS_CLOCK_RENESAS_R9A09G077_CPG_H__ */
-diff --git a/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h b/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h
-new file mode 100644
-index 000000000000..09da0ad33be6
--- /dev/null
-+++ b/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h
-@@ -0,0 +1,35 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-+ *
-+ * Copyright (C) 2025 Renesas Electronics Corp.
-+ */
-+
-+#ifndef __DT_BINDINGS_CLOCK_RENESAS_R9A09G087_CPG_H__
-+#define __DT_BINDINGS_CLOCK_RENESAS_R9A09G087_CPG_H__
-+
-+#include <dt-bindings/clock/renesas-cpg-mssr.h>
-+
-+/* R9A09G087 CPG Core Clocks */
-+#define R9A09G087_CLK_CA55C0		0
-+#define R9A09G087_CLK_CA55C1		1
-+#define R9A09G087_CLK_CA55C2		2
-+#define R9A09G087_CLK_CA55C3		3
-+#define R9A09G087_CLK_CA55S		4
-+#define R9A09G087_CLK_CR52_CPU0		5
-+#define R9A09G087_CLK_CR52_CPU1		6
-+#define R9A09G087_CLK_CKIO		7
-+#define R9A09G087_CLK_PCLKAH		8
-+#define R9A09G087_CLK_PCLKAM		9
-+#define R9A09G087_CLK_PCLKAL		10
-+#define R9A09G087_CLK_PCLKGPTL		11
-+#define R9A09G087_CLK_PCLKH		12
-+#define R9A09G087_CLK_PCLKM		13
-+#define R9A09G087_CLK_PCLKL		14
-+#define R9A09G087_SDHI_CLKHS		15
-+#define R9A09G087_USB_CLK		16
-+#define R9A09G087_ETCLKA		17
-+#define R9A09G087_ETCLKB		18
-+#define R9A09G087_ETCLKC		19
-+#define R9A09G087_ETCLKD		20
-+#define R9A09G087_ETCLKE		21
-+
-+#endif /* __DT_BINDINGS_CLOCK_RENESAS_R9A09G087_CPG_H__ */
-diff --git a/include/dt-bindings/clock/rk3036-cru.h b/include/dt-bindings/clock/rk3036-cru.h
-index a96a9870ad59..5cbc0e2b08ff 100644
--- a/include/dt-bindings/clock/rk3036-cru.h
-+++ b/include/dt-bindings/clock/rk3036-cru.h
-@@ -47,6 +47,7 @@
- #define SCLK_MACREF		152
- #define SCLK_MACPLL		153
- #define SCLK_SFC		160
-+#define SCLK_USB480M		161
- 
- /* aclk gates */
- #define ACLK_DMAC2		194
-@@ -94,8 +95,6 @@
- #define HCLK_CPU		477
- #define HCLK_PERI		478
- 
-#define CLK_NR_CLKS		(HCLK_PERI + 1)
-
- /* soft-reset indices */
- #define SRST_CORE0		0
- #define SRST_CORE1		1
-diff --git a/include/dt-bindings/clock/rk3128-cru.h b/include/dt-bindings/clock/rk3128-cru.h
-index 6a47825dac5d..b609fcf96508 100644
--- a/include/dt-bindings/clock/rk3128-cru.h
-+++ b/include/dt-bindings/clock/rk3128-cru.h
-@@ -116,6 +116,7 @@
- #define PCLK_GMAC		367
- #define PCLK_PMU_PRE		368
- #define PCLK_SIM_CARD		369
-+#define PCLK_MIPIPHY		370
- 
- /* hclk gates */
- #define HCLK_SPDIF		440
-@@ -143,8 +144,7 @@
- #define HCLK_TSP		475
- #define HCLK_CRYPTO		476
- #define HCLK_PERI		478
-
-#define CLK_NR_CLKS		(HCLK_PERI + 1)
-+#define HCLK_SFC		479
- 
- /* soft-reset indices */
- #define SRST_CORE0_PO		0
-diff --git a/include/dt-bindings/clock/rk3188-cru-common.h b/include/dt-bindings/clock/rk3188-cru-common.h
-index afad90680fce..dd988cc9d582 100644
--- a/include/dt-bindings/clock/rk3188-cru-common.h
-+++ b/include/dt-bindings/clock/rk3188-cru-common.h
-@@ -103,6 +103,8 @@
- #define PCLK_PERI		351
- #define PCLK_DDRUPCTL		352
- #define PCLK_PUBL		353
-+#define PCLK_CIF0		354
-+#define PCLK_CIF1		355
- 
- /* hclk gates */
- #define HCLK_SDMMC		448
-@@ -132,8 +134,6 @@
- #define HCLK_VDPU		472
- #define HCLK_HDMI		473
- 
-#define CLK_NR_CLKS		(HCLK_HDMI + 1)
-
- /* soft-reset indices */
- #define SRST_MCORE		2
- #define SRST_CORE0		3
-diff --git a/include/dt-bindings/clock/rk3228-cru.h b/include/dt-bindings/clock/rk3228-cru.h
-index de550ea56eeb..138b6ce514dd 100644
--- a/include/dt-bindings/clock/rk3228-cru.h
-+++ b/include/dt-bindings/clock/rk3228-cru.h
-@@ -146,8 +146,6 @@
- #define HCLK_S_CRYPTO		477
- #define HCLK_PERI		478
- 
-#define CLK_NR_CLKS		(HCLK_PERI + 1)
-
- /* soft-reset indices */
- #define SRST_CORE0_PO		0
- #define SRST_CORE1_PO		1
-diff --git a/include/dt-bindings/clock/rk3288-cru.h b/include/dt-bindings/clock/rk3288-cru.h
-index 33819acbfc56..c6034b01b050 100644
--- a/include/dt-bindings/clock/rk3288-cru.h
-+++ b/include/dt-bindings/clock/rk3288-cru.h
-@@ -195,8 +195,6 @@
- #define HCLK_CPU		477
- #define HCLK_PERI		478
- 
-#define CLK_NR_CLKS		(HCLK_PERI + 1)
-
- /* soft-reset indices */
- #define SRST_CORE0		0
- #define SRST_CORE1		1
-diff --git a/include/dt-bindings/clock/rk3308-cru.h b/include/dt-bindings/clock/rk3308-cru.h
-index d97840f9ee2e..ce4cd72b9d3d 100644
--- a/include/dt-bindings/clock/rk3308-cru.h
-+++ b/include/dt-bindings/clock/rk3308-cru.h
-@@ -212,8 +212,6 @@
- #define PCLK_CAN		233
- #define PCLK_OWIRE		234
- 
-#define CLK_NR_CLKS		(PCLK_OWIRE + 1)
-
- /* soft-reset indices */
- 
- /* cru_softrst_con0 */
-diff --git a/include/dt-bindings/clock/rk3328-cru.h b/include/dt-bindings/clock/rk3328-cru.h
-index 555b4ff660ae..8885a2e98c65 100644
--- a/include/dt-bindings/clock/rk3328-cru.h
-+++ b/include/dt-bindings/clock/rk3328-cru.h
-@@ -201,8 +201,6 @@
- #define HCLK_RGA		340
- #define HCLK_HDCP		341
- 
-#define CLK_NR_CLKS		(HCLK_HDCP + 1)
-
- /* soft-reset indices */
- #define SRST_CORE0_PO		0
- #define SRST_CORE1_PO		1
-diff --git a/include/dt-bindings/clock/rk3368-cru.h b/include/dt-bindings/clock/rk3368-cru.h
-index 83c72a163fd3..b951e2906948 100644
--- a/include/dt-bindings/clock/rk3368-cru.h
-+++ b/include/dt-bindings/clock/rk3368-cru.h
-@@ -72,6 +72,7 @@
- #define SCLK_SFC		126
- #define SCLK_MAC		127
- #define SCLK_MACREF_OUT		128
-+#define SCLK_MIPIDSI_24M	129
- #define SCLK_TIMER10		133
- #define SCLK_TIMER11		134
- #define SCLK_TIMER12		135
-@@ -182,8 +183,6 @@
- #define HCLK_BUS		477
- #define HCLK_PERI		478
- 
-#define CLK_NR_CLKS		(HCLK_PERI + 1)
-
- /* soft-reset indices */
- #define SRST_CORE_B0		0
- #define SRST_CORE_B1		1
-diff --git a/include/dt-bindings/clock/rk3399-cru.h b/include/dt-bindings/clock/rk3399-cru.h
-index 44e0a319f077..4c90c7703a83 100644
--- a/include/dt-bindings/clock/rk3399-cru.h
-+++ b/include/dt-bindings/clock/rk3399-cru.h
-@@ -335,8 +335,6 @@
- #define HCLK_SDIO_NOC			495
- #define HCLK_SDIOAUDIO_NOC		496
- 
-#define CLK_NR_CLKS			(HCLK_SDIOAUDIO_NOC + 1)
-
- /* pmu-clocks indices */
- 
- #define PLL_PPLL			1
-@@ -378,8 +376,6 @@
- #define PCLK_INTR_ARB_PMU		49
- #define HCLK_NOC_PMU			50
- 
-#define CLKPMU_NR_CLKS			(HCLK_NOC_PMU + 1)
-
- /* soft-reset indices */
- 
- /* cru_softrst_con0 */
-@@ -547,8 +543,8 @@
- #define SRST_H_PERILP0			171
- #define SRST_H_PERILP0_NOC		172
- #define SRST_ROM			173
-#define SRST_CRYPTO_S			174
-#define SRST_CRYPTO_M			175
-+#define SRST_CRYPTO0_S			174
-+#define SRST_CRYPTO0_M			175
- 
- /* cru_softrst_con11 */
- #define SRST_P_DCF			176
-@@ -556,7 +552,7 @@
- #define SRST_CM0S			178
- #define SRST_CM0S_DBG			179
- #define SRST_CM0S_PO			180
-#define SRST_CRYPTO			181
-+#define SRST_CRYPTO0			181
- #define SRST_P_PERILP1_SGRF		182
- #define SRST_P_PERILP1_GRF		183
- #define SRST_CRYPTO1_S			184
-diff --git a/include/dt-bindings/clock/rk3568-cru.h b/include/dt-bindings/clock/rk3568-cru.h
-index d29890865150..1e0aef8a645d 100644
--- a/include/dt-bindings/clock/rk3568-cru.h
-+++ b/include/dt-bindings/clock/rk3568-cru.h
-@@ -78,6 +78,7 @@
- #define CPLL_333M		9
- #define ARMCLK			10
- #define USB480M			11
-+#define USB480M_PHY		12
- #define ACLK_CORE_NIU2BUS	18
- #define CLK_CORE_PVTM		19
- #define CLK_CORE_PVTM_CORE	20
-@@ -482,7 +483,11 @@
- 
- #define PCLK_CORE_PVTM		450
- 
-#define CLK_NR_CLKS		(PCLK_CORE_PVTM + 1)
-+/* scmi-clocks indices */
-+
-+#define SCMI_CLK_CPU		0
-+#define SCMI_CLK_GPU		1
-+#define SCMI_CLK_NPU		2
- 
- /* pmu soft-reset indices */
- /* pmucru_softrst_con0 */
-diff --git a/include/dt-bindings/clock/rockchip,rk3506-cru.h b/include/dt-bindings/clock/rockchip,rk3506-cru.h
-new file mode 100644
-index 000000000000..71d7dda23cc9
--- /dev/null
-+++ b/include/dt-bindings/clock/rockchip,rk3506-cru.h
-@@ -0,0 +1,285 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023-2025 Rockchip Electronics Co., Ltd.
-+ * Author: Finley Xiao <finley.xiao@rock-chips.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_ROCKCHIP_RK3506_H
-+#define _DT_BINDINGS_CLK_ROCKCHIP_RK3506_H
-+
-+/* cru plls */
-+#define PLL_GPLL			0
-+#define PLL_V0PLL			1
-+#define PLL_V1PLL			2
-+
-+/* cru-clocks indices */
-+#define ARMCLK				3
-+#define CLK_DDR				4
-+#define XIN24M_GATE			5
-+#define CLK_GPLL_GATE			6
-+#define CLK_V0PLL_GATE			7
-+#define CLK_V1PLL_GATE			8
-+#define CLK_GPLL_DIV			9
-+#define CLK_GPLL_DIV_100M		10
-+#define CLK_V0PLL_DIV			11
-+#define CLK_V1PLL_DIV			12
-+#define CLK_INT_VOICE_MATRIX0		13
-+#define CLK_INT_VOICE_MATRIX1		14
-+#define CLK_INT_VOICE_MATRIX2		15
-+#define CLK_FRAC_UART_MATRIX0_MUX	16
-+#define CLK_FRAC_UART_MATRIX1_MUX	17
-+#define CLK_FRAC_VOICE_MATRIX0_MUX	18
-+#define CLK_FRAC_VOICE_MATRIX1_MUX	19
-+#define CLK_FRAC_COMMON_MATRIX0_MUX	20
-+#define CLK_FRAC_COMMON_MATRIX1_MUX	21
-+#define CLK_FRAC_COMMON_MATRIX2_MUX	22
-+#define CLK_FRAC_UART_MATRIX0		23
-+#define CLK_FRAC_UART_MATRIX1		24
-+#define CLK_FRAC_VOICE_MATRIX0		25
-+#define CLK_FRAC_VOICE_MATRIX1		26
-+#define CLK_FRAC_COMMON_MATRIX0		27
-+#define CLK_FRAC_COMMON_MATRIX1		28
-+#define CLK_FRAC_COMMON_MATRIX2		29
-+#define CLK_REF_USBPHY_TOP		30
-+#define CLK_REF_DPHY_TOP		31
-+#define ACLK_CORE_ROOT			32
-+#define PCLK_CORE_ROOT			33
-+#define PCLK_DBG			34
-+#define PCLK_CORE_GRF			35
-+#define PCLK_CORE_CRU			36
-+#define CLK_CORE_EMA_DETECT		37
-+#define CLK_REF_PVTPLL_CORE		38
-+#define PCLK_GPIO1			39
-+#define DBCLK_GPIO1			40
-+#define ACLK_CORE_PERI_ROOT		41
-+#define HCLK_CORE_PERI_ROOT		42
-+#define PCLK_CORE_PERI_ROOT		43
-+#define CLK_DSMC			44
-+#define ACLK_DSMC			45
-+#define PCLK_DSMC			46
-+#define CLK_FLEXBUS_TX			47
-+#define CLK_FLEXBUS_RX			48
-+#define ACLK_FLEXBUS			49
-+#define HCLK_FLEXBUS			50
-+#define ACLK_DSMC_SLV			51
-+#define HCLK_DSMC_SLV			52
-+#define ACLK_BUS_ROOT			53
-+#define HCLK_BUS_ROOT			54
-+#define PCLK_BUS_ROOT			55
-+#define ACLK_SYSRAM			56
-+#define HCLK_SYSRAM			57
-+#define ACLK_DMAC0			58
-+#define ACLK_DMAC1			59
-+#define HCLK_M0				60
-+#define PCLK_BUS_GRF			61
-+#define PCLK_TIMER			62
-+#define CLK_TIMER0_CH0			63
-+#define CLK_TIMER0_CH1			64
-+#define CLK_TIMER0_CH2			65
-+#define CLK_TIMER0_CH3			66
-+#define CLK_TIMER0_CH4			67
-+#define CLK_TIMER0_CH5			68
-+#define PCLK_WDT0			69
-+#define TCLK_WDT0			70
-+#define PCLK_WDT1			71
-+#define TCLK_WDT1			72
-+#define PCLK_MAILBOX			73
-+#define PCLK_INTMUX			74
-+#define PCLK_SPINLOCK			75
-+#define PCLK_DDRC			76
-+#define HCLK_DDRPHY			77
-+#define PCLK_DDRMON			78
-+#define CLK_DDRMON_OSC			79
-+#define PCLK_STDBY			80
-+#define HCLK_USBOTG0			81
-+#define HCLK_USBOTG0_PMU		82
-+#define CLK_USBOTG0_ADP			83
-+#define HCLK_USBOTG1			84
-+#define HCLK_USBOTG1_PMU		85
-+#define CLK_USBOTG1_ADP			86
-+#define PCLK_USBPHY			87
-+#define ACLK_DMA2DDR			88
-+#define PCLK_DMA2DDR			89
-+#define STCLK_M0			90
-+#define CLK_DDRPHY			91
-+#define CLK_DDRC_SRC			92
-+#define ACLK_DDRC_0			93
-+#define ACLK_DDRC_1			94
-+#define CLK_DDRC			95
-+#define CLK_DDRMON			96
-+#define HCLK_LSPERI_ROOT		97
-+#define PCLK_LSPERI_ROOT		98
-+#define PCLK_UART0			99
-+#define PCLK_UART1			100
-+#define PCLK_UART2			101
-+#define PCLK_UART3			102
-+#define PCLK_UART4			103
-+#define SCLK_UART0			104
-+#define SCLK_UART1			105
-+#define SCLK_UART2			106
-+#define SCLK_UART3			107
-+#define SCLK_UART4			108
-+#define PCLK_I2C0			109
-+#define CLK_I2C0			110
-+#define PCLK_I2C1			111
-+#define CLK_I2C1			112
-+#define PCLK_I2C2			113
-+#define CLK_I2C2			114
-+#define PCLK_PWM1			115
-+#define CLK_PWM1			116
-+#define CLK_OSC_PWM1			117
-+#define CLK_RC_PWM1			118
-+#define CLK_FREQ_PWM1			119
-+#define CLK_COUNTER_PWM1		120
-+#define PCLK_SPI0			121
-+#define CLK_SPI0			122
-+#define PCLK_SPI1			123
-+#define CLK_SPI1			124
-+#define PCLK_GPIO2			125
-+#define DBCLK_GPIO2			126
-+#define PCLK_GPIO3			127
-+#define DBCLK_GPIO3			128
-+#define PCLK_GPIO4			129
-+#define DBCLK_GPIO4			130
-+#define HCLK_CAN0			131
-+#define CLK_CAN0			132
-+#define HCLK_CAN1			133
-+#define CLK_CAN1			134
-+#define HCLK_PDM			135
-+#define MCLK_PDM			136
-+#define CLKOUT_PDM			137
-+#define MCLK_SPDIFTX			138
-+#define HCLK_SPDIFTX			139
-+#define HCLK_SPDIFRX			140
-+#define MCLK_SPDIFRX			141
-+#define MCLK_SAI0			142
-+#define HCLK_SAI0			143
-+#define MCLK_OUT_SAI0			144
-+#define MCLK_SAI1			145
-+#define HCLK_SAI1			146
-+#define MCLK_OUT_SAI1			147
-+#define HCLK_ASRC0			148
-+#define CLK_ASRC0			149
-+#define HCLK_ASRC1			150
-+#define CLK_ASRC1			151
-+#define PCLK_CRU			152
-+#define PCLK_PMU_ROOT			153
-+#define MCLK_ASRC0			154
-+#define MCLK_ASRC1			155
-+#define MCLK_ASRC2			156
-+#define MCLK_ASRC3			157
-+#define LRCK_ASRC0_SRC			158
-+#define LRCK_ASRC0_DST			159
-+#define LRCK_ASRC1_SRC			160
-+#define LRCK_ASRC1_DST			161
-+#define ACLK_HSPERI_ROOT		162
-+#define HCLK_HSPERI_ROOT		163
-+#define PCLK_HSPERI_ROOT		164
-+#define CCLK_SRC_SDMMC			165
-+#define HCLK_SDMMC			166
-+#define HCLK_FSPI			167
-+#define SCLK_FSPI			168
-+#define PCLK_SPI2			169
-+#define ACLK_MAC0			170
-+#define ACLK_MAC1			171
-+#define PCLK_MAC0			172
-+#define PCLK_MAC1			173
-+#define CLK_MAC_ROOT			174
-+#define CLK_MAC0			175
-+#define CLK_MAC1			176
-+#define MCLK_SAI2			177
-+#define HCLK_SAI2			178
-+#define MCLK_OUT_SAI2			179
-+#define MCLK_SAI3_SRC			180
-+#define HCLK_SAI3			181
-+#define MCLK_SAI3			182
-+#define MCLK_OUT_SAI3			183
-+#define MCLK_SAI4_SRC			184
-+#define HCLK_SAI4			185
-+#define MCLK_SAI4			186
-+#define HCLK_DSM			187
-+#define MCLK_DSM			188
-+#define PCLK_AUDIO_ADC			189
-+#define MCLK_AUDIO_ADC			190
-+#define MCLK_AUDIO_ADC_DIV4		191
-+#define PCLK_SARADC			192
-+#define CLK_SARADC			193
-+#define PCLK_OTPC_NS			194
-+#define CLK_SBPI_OTPC_NS		195
-+#define CLK_USER_OTPC_NS		196
-+#define PCLK_UART5			197
-+#define SCLK_UART5			198
-+#define PCLK_GPIO234_IOC		199
-+#define CLK_MAC_PTP_ROOT		200
-+#define CLK_MAC0_PTP			201
-+#define CLK_MAC1_PTP			202
-+#define CLK_SPI2			203
-+#define ACLK_VIO_ROOT			204
-+#define HCLK_VIO_ROOT			205
-+#define PCLK_VIO_ROOT			206
-+#define HCLK_RGA			207
-+#define ACLK_RGA			208
-+#define CLK_CORE_RGA			209
-+#define ACLK_VOP			210
-+#define HCLK_VOP			211
-+#define DCLK_VOP			212
-+#define PCLK_DPHY			213
-+#define PCLK_DSI_HOST			214
-+#define PCLK_TSADC			215
-+#define CLK_TSADC			216
-+#define CLK_TSADC_TSEN			217
-+#define PCLK_GPIO1_IOC			218
-+#define PCLK_OTPC_S			219
-+#define CLK_SBPI_OTPC_S			220
-+#define CLK_USER_OTPC_S			221
-+#define PCLK_OTP_MASK			222
-+#define PCLK_KEYREADER			223
-+#define HCLK_BOOTROM			224
-+#define PCLK_DDR_SERVICE		225
-+#define HCLK_CRYPTO_S			226
-+#define HCLK_KEYLAD			227
-+#define CLK_CORE_CRYPTO			228
-+#define CLK_PKA_CRYPTO			229
-+#define CLK_CORE_CRYPTO_S		230
-+#define CLK_PKA_CRYPTO_S		231
-+#define ACLK_CRYPTO_S			232
-+#define HCLK_RNG_S			233
-+#define CLK_CORE_CRYPTO_NS		234
-+#define CLK_PKA_CRYPTO_NS		235
-+#define ACLK_CRYPTO_NS			236
-+#define HCLK_CRYPTO_NS			237
-+#define HCLK_RNG			238
-+#define CLK_PMU				239
-+#define PCLK_PMU			240
-+#define CLK_PMU_32K			241
-+#define PCLK_PMU_CRU			242
-+#define PCLK_PMU_GRF			243
-+#define PCLK_GPIO0_IOC			244
-+#define PCLK_GPIO0			245
-+#define DBCLK_GPIO0			246
-+#define PCLK_GPIO1_SHADOW		247
-+#define DBCLK_GPIO1_SHADOW		248
-+#define PCLK_PMU_HP_TIMER		249
-+#define CLK_PMU_HP_TIMER		250
-+#define CLK_PMU_HP_TIMER_32K		251
-+#define PCLK_PWM0			252
-+#define CLK_PWM0			253
-+#define CLK_OSC_PWM0			254
-+#define CLK_RC_PWM0			255
-+#define CLK_MAC_OUT			256
-+#define CLK_REF_OUT0			257
-+#define CLK_REF_OUT1			258
-+#define CLK_32K_FRAC			259
-+#define CLK_32K_RC			260
-+#define CLK_32K				261
-+#define CLK_32K_PMU			262
-+#define PCLK_TOUCH_KEY			263
-+#define CLK_TOUCH_KEY			264
-+#define CLK_REF_PHY_PLL			265
-+#define CLK_REF_PHY_PMU_MUX		266
-+#define CLK_WIFI_OUT			267
-+#define CLK_V0PLL_REF			268
-+#define CLK_V1PLL_REF			269
-+#define CLK_32K_FRAC_MUX		270
-+
-+#endif
-diff --git a/include/dt-bindings/clock/rockchip,rk3528-cru.h b/include/dt-bindings/clock/rockchip,rk3528-cru.h
-new file mode 100644
-index 000000000000..0245a53fc334
--- /dev/null
-+++ b/include/dt-bindings/clock/rockchip,rk3528-cru.h
-@@ -0,0 +1,459 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (c) 2022 Rockchip Electronics Co. Ltd.
-+ * Copyright (c) 2024 Yao Zi <ziyao@disroot.org>
-+ * Author: Joseph Chen <chenjh@rock-chips.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_ROCKCHIP_RK3528_H
-+#define _DT_BINDINGS_CLK_ROCKCHIP_RK3528_H
-+
-+/* cru-clocks indices */
-+#define PLL_APLL			0
-+#define PLL_CPLL			1
-+#define PLL_GPLL			2
-+#define PLL_PPLL			3
-+#define PLL_DPLL			4
-+#define ARMCLK				5
-+#define XIN_OSC0_HALF			6
-+#define CLK_MATRIX_50M_SRC		7
-+#define CLK_MATRIX_100M_SRC		8
-+#define CLK_MATRIX_150M_SRC		9
-+#define CLK_MATRIX_200M_SRC		10
-+#define CLK_MATRIX_250M_SRC		11
-+#define CLK_MATRIX_300M_SRC		12
-+#define CLK_MATRIX_339M_SRC		13
-+#define CLK_MATRIX_400M_SRC		14
-+#define CLK_MATRIX_500M_SRC		15
-+#define CLK_MATRIX_600M_SRC		16
-+#define CLK_UART0_SRC			17
-+#define CLK_UART0_FRAC			18
-+#define SCLK_UART0			19
-+#define CLK_UART1_SRC			20
-+#define CLK_UART1_FRAC			21
-+#define SCLK_UART1			22
-+#define CLK_UART2_SRC			23
-+#define CLK_UART2_FRAC			24
-+#define SCLK_UART2			25
-+#define CLK_UART3_SRC			26
-+#define CLK_UART3_FRAC			27
-+#define SCLK_UART3			28
-+#define CLK_UART4_SRC			29
-+#define CLK_UART4_FRAC			30
-+#define SCLK_UART4			31
-+#define CLK_UART5_SRC			32
-+#define CLK_UART5_FRAC			33
-+#define SCLK_UART5			34
-+#define CLK_UART6_SRC			35
-+#define CLK_UART6_FRAC			36
-+#define SCLK_UART6			37
-+#define CLK_UART7_SRC			38
-+#define CLK_UART7_FRAC			39
-+#define SCLK_UART7			40
-+#define CLK_I2S0_2CH_SRC		41
-+#define CLK_I2S0_2CH_FRAC		42
-+#define MCLK_I2S0_2CH_SAI_SRC		43
-+#define CLK_I2S3_8CH_SRC		44
-+#define CLK_I2S3_8CH_FRAC		45
-+#define MCLK_I2S3_8CH_SAI_SRC		46
-+#define CLK_I2S1_8CH_SRC		47
-+#define CLK_I2S1_8CH_FRAC		48
-+#define MCLK_I2S1_8CH_SAI_SRC		49
-+#define CLK_I2S2_2CH_SRC		50
-+#define CLK_I2S2_2CH_FRAC		51
-+#define MCLK_I2S2_2CH_SAI_SRC		52
-+#define CLK_SPDIF_SRC			53
-+#define CLK_SPDIF_FRAC			54
-+#define MCLK_SPDIF_SRC			55
-+#define DCLK_VOP_SRC0			56
-+#define DCLK_VOP_SRC1			57
-+#define CLK_HSM				58
-+#define CLK_CORE_SRC_ACS		59
-+#define CLK_CORE_SRC_PVTMUX		60
-+#define CLK_CORE_SRC			61
-+#define CLK_CORE			62
-+#define ACLK_M_CORE_BIU			63
-+#define CLK_CORE_PVTPLL_SRC		64
-+#define PCLK_DBG			65
-+#define SWCLKTCK			66
-+#define CLK_SCANHS_CORE			67
-+#define CLK_SCANHS_ACLKM_CORE		68
-+#define CLK_SCANHS_PCLK_DBG		69
-+#define CLK_SCANHS_PCLK_CPU_BIU		70
-+#define PCLK_CPU_ROOT			71
-+#define PCLK_CORE_GRF			72
-+#define PCLK_DAPLITE_BIU		73
-+#define PCLK_CPU_BIU			74
-+#define CLK_REF_PVTPLL_CORE		75
-+#define ACLK_BUS_VOPGL_ROOT		76
-+#define ACLK_BUS_VOPGL_BIU		77
-+#define ACLK_BUS_H_ROOT			78
-+#define ACLK_BUS_H_BIU			79
-+#define ACLK_BUS_ROOT			80
-+#define HCLK_BUS_ROOT			81
-+#define PCLK_BUS_ROOT			82
-+#define ACLK_BUS_M_ROOT			83
-+#define ACLK_SYSMEM_BIU			84
-+#define CLK_TIMER_ROOT			85
-+#define ACLK_BUS_BIU			86
-+#define HCLK_BUS_BIU			87
-+#define PCLK_BUS_BIU			88
-+#define PCLK_DFT2APB			89
-+#define PCLK_BUS_GRF			90
-+#define ACLK_BUS_M_BIU			91
-+#define ACLK_GIC			92
-+#define ACLK_SPINLOCK			93
-+#define ACLK_DMAC			94
-+#define PCLK_TIMER			95
-+#define CLK_TIMER0			96
-+#define CLK_TIMER1			97
-+#define CLK_TIMER2			98
-+#define CLK_TIMER3			99
-+#define CLK_TIMER4			100
-+#define CLK_TIMER5			101
-+#define PCLK_JDBCK_DAP			102
-+#define CLK_JDBCK_DAP			103
-+#define PCLK_WDT_NS			104
-+#define TCLK_WDT_NS			105
-+#define HCLK_TRNG_NS			106
-+#define PCLK_UART0			107
-+#define PCLK_DMA2DDR			108
-+#define ACLK_DMA2DDR			109
-+#define PCLK_PWM0			110
-+#define CLK_PWM0			111
-+#define CLK_CAPTURE_PWM0		112
-+#define PCLK_PWM1			113
-+#define CLK_PWM1			114
-+#define CLK_CAPTURE_PWM1		115
-+#define PCLK_SCR			116
-+#define ACLK_DCF			117
-+#define PCLK_INTMUX			118
-+#define CLK_PPLL_I			119
-+#define CLK_PPLL_MUX			120
-+#define CLK_PPLL_100M_MATRIX		121
-+#define CLK_PPLL_50M_MATRIX		122
-+#define CLK_REF_PCIE_INNER_PHY		123
-+#define CLK_REF_PCIE_100M_PHY		124
-+#define ACLK_VPU_L_ROOT			125
-+#define CLK_GMAC1_VPU_25M		126
-+#define CLK_PPLL_125M_MATRIX		127
-+#define ACLK_VPU_ROOT			128
-+#define HCLK_VPU_ROOT			129
-+#define PCLK_VPU_ROOT			130
-+#define ACLK_VPU_BIU			131
-+#define HCLK_VPU_BIU			132
-+#define PCLK_VPU_BIU			133
-+#define ACLK_VPU			134
-+#define HCLK_VPU			135
-+#define PCLK_CRU_PCIE			136
-+#define PCLK_VPU_GRF			137
-+#define HCLK_SFC			138
-+#define SCLK_SFC			139
-+#define CCLK_SRC_EMMC			140
-+#define HCLK_EMMC			141
-+#define ACLK_EMMC			142
-+#define BCLK_EMMC			143
-+#define TCLK_EMMC			144
-+#define PCLK_GPIO1			145
-+#define DBCLK_GPIO1			146
-+#define ACLK_VPU_L_BIU			147
-+#define PCLK_VPU_IOC			148
-+#define HCLK_SAI_I2S0			149
-+#define MCLK_SAI_I2S0			150
-+#define HCLK_SAI_I2S2			151
-+#define MCLK_SAI_I2S2			152
-+#define PCLK_ACODEC			153
-+#define MCLK_ACODEC_TX			154
-+#define PCLK_GPIO3			155
-+#define DBCLK_GPIO3			156
-+#define PCLK_SPI1			157
-+#define CLK_SPI1			158
-+#define SCLK_IN_SPI1			159
-+#define PCLK_UART2			160
-+#define PCLK_UART5			161
-+#define PCLK_UART6			162
-+#define PCLK_UART7			163
-+#define PCLK_I2C3			164
-+#define CLK_I2C3			165
-+#define PCLK_I2C5			166
-+#define CLK_I2C5			167
-+#define PCLK_I2C6			168
-+#define CLK_I2C6			169
-+#define ACLK_MAC_VPU			170
-+#define PCLK_MAC_VPU			171
-+#define CLK_GMAC1_RMII_VPU		172
-+#define CLK_GMAC1_SRC_VPU		173
-+#define PCLK_PCIE			174
-+#define CLK_PCIE_AUX			175
-+#define ACLK_PCIE			176
-+#define HCLK_PCIE_SLV			177
-+#define HCLK_PCIE_DBI			178
-+#define PCLK_PCIE_PHY			179
-+#define PCLK_PIPE_GRF			180
-+#define CLK_PIPE_USB3OTG_COMBO		181
-+#define CLK_UTMI_USB3OTG		182
-+#define CLK_PCIE_PIPE_PHY		183
-+#define CCLK_SRC_SDIO0			184
-+#define HCLK_SDIO0			185
-+#define CCLK_SRC_SDIO1			186
-+#define HCLK_SDIO1			187
-+#define CLK_TS_0			188
-+#define CLK_TS_1			189
-+#define PCLK_CAN2			190
-+#define CLK_CAN2			191
-+#define PCLK_CAN3			192
-+#define CLK_CAN3			193
-+#define PCLK_SARADC			194
-+#define CLK_SARADC			195
-+#define PCLK_TSADC			196
-+#define CLK_TSADC			197
-+#define CLK_TSADC_TSEN			198
-+#define ACLK_USB3OTG			199
-+#define CLK_REF_USB3OTG			200
-+#define CLK_SUSPEND_USB3OTG		201
-+#define ACLK_GPU_ROOT			202
-+#define PCLK_GPU_ROOT			203
-+#define ACLK_GPU_BIU			204
-+#define PCLK_GPU_BIU			205
-+#define ACLK_GPU			206
-+#define CLK_GPU_PVTPLL_SRC		207
-+#define ACLK_GPU_MALI			208
-+#define HCLK_RKVENC_ROOT		209
-+#define ACLK_RKVENC_ROOT		210
-+#define PCLK_RKVENC_ROOT		211
-+#define HCLK_RKVENC_BIU			212
-+#define ACLK_RKVENC_BIU			213
-+#define PCLK_RKVENC_BIU			214
-+#define HCLK_RKVENC			215
-+#define ACLK_RKVENC			216
-+#define CLK_CORE_RKVENC			217
-+#define HCLK_SAI_I2S1			218
-+#define MCLK_SAI_I2S1			219
-+#define PCLK_I2C1			220
-+#define CLK_I2C1			221
-+#define PCLK_I2C0			222
-+#define CLK_I2C0			223
-+#define CLK_UART_JTAG			224
-+#define PCLK_SPI0			225
-+#define CLK_SPI0			226
-+#define SCLK_IN_SPI0			227
-+#define PCLK_GPIO4			228
-+#define DBCLK_GPIO4			229
-+#define PCLK_RKVENC_IOC			230
-+#define HCLK_SPDIF			231
-+#define MCLK_SPDIF			232
-+#define HCLK_PDM			233
-+#define MCLK_PDM			234
-+#define PCLK_UART1			235
-+#define PCLK_UART3			236
-+#define PCLK_RKVENC_GRF			237
-+#define PCLK_CAN0			238
-+#define CLK_CAN0			239
-+#define PCLK_CAN1			240
-+#define CLK_CAN1			241
-+#define ACLK_VO_ROOT			242
-+#define HCLK_VO_ROOT			243
-+#define PCLK_VO_ROOT			244
-+#define ACLK_VO_BIU			245
-+#define HCLK_VO_BIU			246
-+#define PCLK_VO_BIU			247
-+#define HCLK_RGA2E			248
-+#define ACLK_RGA2E			249
-+#define CLK_CORE_RGA2E			250
-+#define HCLK_VDPP			251
-+#define ACLK_VDPP			252
-+#define CLK_CORE_VDPP			253
-+#define PCLK_VO_GRF			254
-+#define PCLK_CRU			255
-+#define ACLK_VOP_ROOT			256
-+#define ACLK_VOP_BIU			257
-+#define HCLK_VOP			258
-+#define DCLK_VOP0			259
-+#define DCLK_VOP1			260
-+#define ACLK_VOP			261
-+#define PCLK_HDMI			262
-+#define CLK_SFR_HDMI			263
-+#define CLK_CEC_HDMI			264
-+#define CLK_SPDIF_HDMI			265
-+#define CLK_HDMIPHY_TMDSSRC		266
-+#define CLK_HDMIPHY_PREP		267
-+#define PCLK_HDMIPHY			268
-+#define HCLK_HDCP_KEY			269
-+#define ACLK_HDCP			270
-+#define HCLK_HDCP			271
-+#define PCLK_HDCP			272
-+#define HCLK_CVBS			273
-+#define DCLK_CVBS			274
-+#define DCLK_4X_CVBS			275
-+#define ACLK_JPEG_DECODER		276
-+#define HCLK_JPEG_DECODER		277
-+#define ACLK_VO_L_ROOT			278
-+#define ACLK_VO_L_BIU			279
-+#define ACLK_MAC_VO			280
-+#define PCLK_MAC_VO			281
-+#define CLK_GMAC0_SRC			282
-+#define CLK_GMAC0_RMII_50M		283
-+#define CLK_GMAC0_TX			284
-+#define CLK_GMAC0_RX			285
-+#define ACLK_JPEG_ROOT			286
-+#define ACLK_JPEG_BIU			287
-+#define HCLK_SAI_I2S3			288
-+#define MCLK_SAI_I2S3			289
-+#define CLK_MACPHY			290
-+#define PCLK_VCDCPHY			291
-+#define PCLK_GPIO2			292
-+#define DBCLK_GPIO2			293
-+#define PCLK_VO_IOC			294
-+#define CCLK_SRC_SDMMC0			295
-+#define HCLK_SDMMC0			296
-+#define PCLK_OTPC_NS			297
-+#define CLK_SBPI_OTPC_NS		298
-+#define CLK_USER_OTPC_NS		299
-+#define CLK_HDMIHDP0			300
-+#define HCLK_USBHOST			301
-+#define HCLK_USBHOST_ARB		302
-+#define CLK_USBHOST_OHCI		303
-+#define CLK_USBHOST_UTMI		304
-+#define PCLK_UART4			305
-+#define PCLK_I2C4			306
-+#define CLK_I2C4			307
-+#define PCLK_I2C7			308
-+#define CLK_I2C7			309
-+#define PCLK_USBPHY			310
-+#define CLK_REF_USBPHY			311
-+#define HCLK_RKVDEC_ROOT		312
-+#define ACLK_RKVDEC_ROOT_NDFT		313
-+#define PCLK_DDRPHY_CRU			314
-+#define HCLK_RKVDEC_BIU			315
-+#define ACLK_RKVDEC_BIU			316
-+#define ACLK_RKVDEC			317
-+#define HCLK_RKVDEC			318
-+#define CLK_HEVC_CA_RKVDEC		319
-+#define ACLK_RKVDEC_PVTMUX_ROOT		320
-+#define CLK_RKVDEC_PVTPLL_SRC		321
-+#define PCLK_DDR_ROOT			322
-+#define PCLK_DDR_BIU			323
-+#define PCLK_DDRC			324
-+#define PCLK_DDRMON			325
-+#define CLK_TIMER_DDRMON		326
-+#define PCLK_MSCH_BIU			327
-+#define PCLK_DDR_GRF			328
-+#define PCLK_DDR_HWLP			329
-+#define PCLK_DDRPHY			330
-+#define CLK_MSCH_BIU			331
-+#define ACLK_DDR_UPCTL			332
-+#define CLK_DDR_UPCTL			333
-+#define CLK_DDRMON			334
-+#define ACLK_DDR_SCRAMBLE		335
-+#define ACLK_SPLIT			336
-+#define CLK_DDRC_SRC			337
-+#define CLK_DDR_PHY			338
-+#define PCLK_OTPC_S			339
-+#define CLK_SBPI_OTPC_S			340
-+#define CLK_USER_OTPC_S			341
-+#define PCLK_KEYREADER			342
-+#define PCLK_BUS_SGRF			343
-+#define PCLK_STIMER			344
-+#define CLK_STIMER0			345
-+#define CLK_STIMER1			346
-+#define PCLK_WDT_S			347
-+#define TCLK_WDT_S			348
-+#define HCLK_TRNG_S			349
-+#define HCLK_BOOTROM			350
-+#define PCLK_DCF			351
-+#define ACLK_SYSMEM			352
-+#define HCLK_TSP			353
-+#define ACLK_TSP			354
-+#define CLK_CORE_TSP			355
-+#define CLK_OTPC_ARB			356
-+#define PCLK_OTP_MASK			357
-+#define CLK_PMC_OTP			358
-+#define PCLK_PMU_ROOT			359
-+#define HCLK_PMU_ROOT			360
-+#define PCLK_I2C2			361
-+#define CLK_I2C2			362
-+#define HCLK_PMU_BIU			363
-+#define PCLK_PMU_BIU			364
-+#define FCLK_MCU			365
-+#define RTC_CLK_MCU			366
-+#define PCLK_OSCCHK			367
-+#define CLK_PMU_MCU_JTAG		368
-+#define PCLK_PMU			369
-+#define PCLK_GPIO0			370
-+#define DBCLK_GPIO0			371
-+#define XIN_OSC0_DIV			372
-+#define CLK_DEEPSLOW			373
-+#define CLK_DDR_FAIL_SAFE		374
-+#define PCLK_PMU_HP_TIMER		375
-+#define CLK_PMU_HP_TIMER		376
-+#define CLK_PMU_32K_HP_TIMER		377
-+#define PCLK_PMU_IOC			378
-+#define PCLK_PMU_CRU			379
-+#define PCLK_PMU_GRF			380
-+#define PCLK_PMU_WDT			381
-+#define TCLK_PMU_WDT			382
-+#define PCLK_PMU_MAILBOX		383
-+#define PCLK_SCRKEYGEN			384
-+#define CLK_SCRKEYGEN			385
-+#define CLK_PVTM_OSCCHK			386
-+#define CLK_REFOUT			387
-+#define CLK_PVTM_PMU			388
-+#define PCLK_PVTM_PMU			389
-+#define PCLK_PMU_SGRF			390
-+#define HCLK_PMU_SRAM			391
-+#define CLK_UART0			392
-+#define CLK_UART1			393
-+#define CLK_UART2			394
-+#define CLK_UART3			395
-+#define CLK_UART4			396
-+#define CLK_UART5			397
-+#define CLK_UART6			398
-+#define CLK_UART7			399
-+#define MCLK_I2S0_2CH_SAI_SRC_PRE	400
-+#define MCLK_I2S1_8CH_SAI_SRC_PRE	401
-+#define MCLK_I2S2_2CH_SAI_SRC_PRE	402
-+#define MCLK_I2S3_8CH_SAI_SRC_PRE	403
-+#define MCLK_SDPDIF_SRC_PRE		404
-+#define SCLK_SDMMC_DRV			405
-+#define SCLK_SDMMC_SAMPLE		406
-+#define SCLK_SDIO0_DRV			407
-+#define SCLK_SDIO0_SAMPLE		408
-+#define SCLK_SDIO1_DRV			409
-+#define SCLK_SDIO1_SAMPLE		410
-+
-+/* scmi-clocks indices */
-+#define SCMI_PCLK_KEYREADER		0
-+#define SCMI_HCLK_KLAD			1
-+#define SCMI_PCLK_KLAD			2
-+#define SCMI_HCLK_TRNG_S		3
-+#define SCMI_HCLK_CRYPTO_S		4
-+#define SCMI_PCLK_WDT_S			5
-+#define SCMI_TCLK_WDT_S			6
-+#define SCMI_PCLK_STIMER		7
-+#define SCMI_CLK_STIMER0		8
-+#define SCMI_CLK_STIMER1		9
-+#define SCMI_PCLK_OTP_MASK		10
-+#define SCMI_PCLK_OTPC_S		11
-+#define SCMI_CLK_SBPI_OTPC_S		12
-+#define SCMI_CLK_USER_OTPC_S		13
-+#define SCMI_CLK_PMC_OTP		14
-+#define SCMI_CLK_OTPC_ARB		15
-+#define SCMI_CLK_CORE_TSP		16
-+#define SCMI_ACLK_TSP			17
-+#define SCMI_HCLK_TSP			18
-+#define SCMI_PCLK_DCF			19
-+#define SCMI_CLK_DDR			20
-+#define SCMI_CLK_CPU			21
-+#define SCMI_CLK_GPU			22
-+#define SCMI_CORE_CRYPTO		23
-+#define SCMI_ACLK_CRYPTO		24
-+#define SCMI_PKA_CRYPTO			25
-+#define SCMI_HCLK_CRYPTO		26
-+#define SCMI_CORE_CRYPTO_S		27
-+#define SCMI_ACLK_CRYPTO_S		28
-+#define SCMI_PKA_CRYPTO_S		29
-+#define SCMI_CORE_KLAD			30
-+#define SCMI_ACLK_KLAD			31
-+#define SCMI_HCLK_TRNG			32
-+
-+#endif // _DT_BINDINGS_CLK_ROCKCHIP_RK3528_H
-diff --git a/include/dt-bindings/clock/rockchip,rk3562-cru.h b/include/dt-bindings/clock/rockchip,rk3562-cru.h
-new file mode 100644
-index 000000000000..a5b0b153209c
--- /dev/null
-+++ b/include/dt-bindings/clock/rockchip,rk3562-cru.h
-@@ -0,0 +1,379 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022-2025 Rockchip Electronics Co., Ltd.
-+ * Author: Finley Xiao <finley.xiao@rock-chips.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_ROCKCHIP_RK3562_H
-+#define _DT_BINDINGS_CLK_ROCKCHIP_RK3562_H
-+
-+/* cru-clocks indices */
-+
-+/* cru plls */
-+#define PLL_DMPLL0			0
-+#define PLL_APLL			1
-+#define PLL_GPLL			2
-+#define PLL_VPLL			3
-+#define PLL_HPLL			4
-+#define PLL_CPLL			5
-+#define PLL_DPLL			6
-+#define PLL_DMPLL1			7
-+
-+/* cru clocks */
-+#define ARMCLK				8
-+#define CLK_GPU				9
-+#define ACLK_RKNN			10
-+#define CLK_DDR				11
-+#define CLK_MATRIX_50M_SRC		12
-+#define CLK_MATRIX_100M_SRC		13
-+#define CLK_MATRIX_125M_SRC		14
-+#define CLK_MATRIX_200M_SRC		15
-+#define CLK_MATRIX_300M_SRC		16
-+#define ACLK_TOP			17
-+#define ACLK_TOP_VIO			18
-+#define CLK_CAM0_OUT2IO			19
-+#define CLK_CAM1_OUT2IO			20
-+#define CLK_CAM2_OUT2IO			21
-+#define CLK_CAM3_OUT2IO			22
-+#define ACLK_BUS			23
-+#define HCLK_BUS			24
-+#define PCLK_BUS			25
-+#define PCLK_I2C1			26
-+#define PCLK_I2C2			27
-+#define PCLK_I2C3			28
-+#define PCLK_I2C4			29
-+#define PCLK_I2C5			30
-+#define CLK_I2C				31
-+#define CLK_I2C1			32
-+#define CLK_I2C2			33
-+#define CLK_I2C3			34
-+#define CLK_I2C4			35
-+#define CLK_I2C5			36
-+#define DCLK_BUS_GPIO			37
-+#define DCLK_BUS_GPIO3			38
-+#define DCLK_BUS_GPIO4			39
-+#define PCLK_TIMER			40
-+#define CLK_TIMER0			41
-+#define CLK_TIMER1			42
-+#define CLK_TIMER2			43
-+#define CLK_TIMER3			44
-+#define CLK_TIMER4			45
-+#define CLK_TIMER5			46
-+#define PCLK_STIMER			47
-+#define CLK_STIMER0			48
-+#define CLK_STIMER1			49
-+#define PCLK_WDTNS			50
-+#define CLK_WDTNS			51
-+#define PCLK_GRF			52
-+#define PCLK_SGRF			53
-+#define PCLK_MAILBOX			54
-+#define PCLK_INTC			55
-+#define ACLK_BUS_GIC400			56
-+#define ACLK_BUS_SPINLOCK		57
-+#define ACLK_DCF			58
-+#define PCLK_DCF			59
-+#define FCLK_BUS_CM0_CORE		60
-+#define CLK_BUS_CM0_RTC			61
-+#define HCLK_ICACHE			62
-+#define HCLK_DCACHE			63
-+#define PCLK_TSADC			64
-+#define CLK_TSADC			65
-+#define CLK_TSADC_TSEN			66
-+#define PCLK_DFT2APB			67
-+#define CLK_SARADC_VCCIO156		68
-+#define PCLK_GMAC			69
-+#define ACLK_GMAC			70
-+#define CLK_GMAC_125M_CRU_I		71
-+#define CLK_GMAC_50M_CRU_I		72
-+#define CLK_GMAC_50M_O			73
-+#define CLK_GMAC_ETH_OUT2IO		74
-+#define PCLK_APB2ASB_VCCIO156		75
-+#define PCLK_TO_VCCIO156		76
-+#define PCLK_DSIPHY			77
-+#define PCLK_DSITX			78
-+#define PCLK_CPU_EMA_DET		79
-+#define PCLK_HASH			80
-+#define PCLK_TOPCRU			81
-+#define PCLK_ASB2APB_VCCIO156		82
-+#define PCLK_IOC_VCCIO156		83
-+#define PCLK_GPIO3_VCCIO156		84
-+#define PCLK_GPIO4_VCCIO156		85
-+#define PCLK_SARADC_VCCIO156		86
-+#define PCLK_MAC100			87
-+#define ACLK_MAC100			89
-+#define CLK_MAC100_50M_MATRIX		90
-+#define HCLK_CORE			91
-+#define PCLK_DDR			92
-+#define CLK_MSCH_BRG_BIU		93
-+#define PCLK_DDR_HWLP			94
-+#define PCLK_DDR_UPCTL			95
-+#define PCLK_DDR_PHY			96
-+#define PCLK_DDR_DFICTL			97
-+#define PCLK_DDR_DMA2DDR		98
-+#define PCLK_DDR_MON			99
-+#define TMCLK_DDR_MON			100
-+#define PCLK_DDR_GRF			101
-+#define PCLK_DDR_CRU			102
-+#define PCLK_SUBDDR_CRU			103
-+#define CLK_GPU_PRE			104
-+#define ACLK_GPU_PRE			105
-+#define CLK_GPU_BRG			107
-+#define CLK_NPU_PRE			108
-+#define HCLK_NPU_PRE			109
-+#define HCLK_RKNN			111
-+#define ACLK_PERI			112
-+#define HCLK_PERI			113
-+#define PCLK_PERI			114
-+#define PCLK_PERICRU			115
-+#define HCLK_SAI0			116
-+#define CLK_SAI0_SRC			117
-+#define CLK_SAI0_FRAC			118
-+#define CLK_SAI0			119
-+#define MCLK_SAI0			120
-+#define MCLK_SAI0_OUT2IO		121
-+#define HCLK_SAI1			122
-+#define CLK_SAI1_SRC			123
-+#define CLK_SAI1_FRAC			124
-+#define CLK_SAI1			125
-+#define MCLK_SAI1			126
-+#define MCLK_SAI1_OUT2IO		127
-+#define HCLK_SAI2			128
-+#define CLK_SAI2_SRC			129
-+#define CLK_SAI2_FRAC			130
-+#define CLK_SAI2			131
-+#define MCLK_SAI2			132
-+#define MCLK_SAI2_OUT2IO		133
-+#define HCLK_DSM			134
-+#define CLK_DSM				135
-+#define HCLK_PDM			136
-+#define MCLK_PDM			137
-+#define HCLK_SPDIF			138
-+#define CLK_SPDIF_SRC			139
-+#define CLK_SPDIF_FRAC			140
-+#define CLK_SPDIF			141
-+#define MCLK_SPDIF			142
-+#define HCLK_SDMMC0			143
-+#define CCLK_SDMMC0			144
-+#define HCLK_SDMMC1			145
-+#define CCLK_SDMMC1			146
-+#define SCLK_SDMMC0_DRV			147
-+#define SCLK_SDMMC0_SAMPLE		148
-+#define SCLK_SDMMC1_DRV			149
-+#define SCLK_SDMMC1_SAMPLE		150
-+#define HCLK_EMMC			151
-+#define ACLK_EMMC			152
-+#define CCLK_EMMC			153
-+#define BCLK_EMMC			154
-+#define TMCLK_EMMC			155
-+#define SCLK_SFC			156
-+#define HCLK_SFC			157
-+#define HCLK_USB2HOST			158
-+#define HCLK_USB2HOST_ARB		159
-+#define PCLK_SPI1			160
-+#define CLK_SPI1			161
-+#define SCLK_IN_SPI1			162
-+#define PCLK_SPI2			163
-+#define CLK_SPI2			164
-+#define SCLK_IN_SPI2			165
-+#define PCLK_UART1			166
-+#define PCLK_UART2			167
-+#define PCLK_UART3			168
-+#define PCLK_UART4			169
-+#define PCLK_UART5			170
-+#define PCLK_UART6			171
-+#define PCLK_UART7			172
-+#define PCLK_UART8			173
-+#define PCLK_UART9			174
-+#define CLK_UART1_SRC			175
-+#define CLK_UART1_FRAC			176
-+#define CLK_UART1			177
-+#define SCLK_UART1			178
-+#define CLK_UART2_SRC			179
-+#define CLK_UART2_FRAC			180
-+#define CLK_UART2			181
-+#define SCLK_UART2			182
-+#define CLK_UART3_SRC			183
-+#define CLK_UART3_FRAC			184
-+#define CLK_UART3			185
-+#define SCLK_UART3			186
-+#define CLK_UART4_SRC			187
-+#define CLK_UART4_FRAC			188
-+#define CLK_UART4			189
-+#define SCLK_UART4			190
-+#define CLK_UART5_SRC			191
-+#define CLK_UART5_FRAC			192
-+#define CLK_UART5			193
-+#define SCLK_UART5			194
-+#define CLK_UART6_SRC			195
-+#define CLK_UART6_FRAC			196
-+#define CLK_UART6			197
-+#define SCLK_UART6			198
-+#define CLK_UART7_SRC			199
-+#define CLK_UART7_FRAC			200
-+#define CLK_UART7			201
-+#define SCLK_UART7			202
-+#define CLK_UART8_SRC			203
-+#define CLK_UART8_FRAC			204
-+#define CLK_UART8			205
-+#define SCLK_UART8			206
-+#define CLK_UART9_SRC			207
-+#define CLK_UART9_FRAC			208
-+#define CLK_UART9			209
-+#define SCLK_UART9			210
-+#define PCLK_PWM1_PERI			211
-+#define CLK_PWM1_PERI			212
-+#define CLK_CAPTURE_PWM1_PERI		213
-+#define PCLK_PWM2_PERI			214
-+#define CLK_PWM2_PERI			215
-+#define CLK_CAPTURE_PWM2_PERI		216
-+#define PCLK_PWM3_PERI			217
-+#define CLK_PWM3_PERI			218
-+#define CLK_CAPTURE_PWM3_PERI		219
-+#define PCLK_CAN0			220
-+#define CLK_CAN0			221
-+#define PCLK_CAN1			222
-+#define CLK_CAN1			223
-+#define ACLK_CRYPTO			224
-+#define HCLK_CRYPTO			225
-+#define PCLK_CRYPTO			226
-+#define CLK_CORE_CRYPTO			227
-+#define CLK_PKA_CRYPTO			228
-+#define HCLK_KLAD			229
-+#define PCLK_KEY_READER			230
-+#define HCLK_RK_RNG_NS			231
-+#define HCLK_RK_RNG_S			232
-+#define HCLK_TRNG_NS			233
-+#define HCLK_TRNG_S			234
-+#define HCLK_CRYPTO_S			235
-+#define PCLK_PERI_WDT			236
-+#define TCLK_PERI_WDT			237
-+#define ACLK_SYSMEM			238
-+#define HCLK_BOOTROM			239
-+#define PCLK_PERI_GRF			240
-+#define ACLK_DMAC			241
-+#define ACLK_RKDMAC			242
-+#define PCLK_OTPC_NS			243
-+#define CLK_SBPI_OTPC_NS		244
-+#define CLK_USER_OTPC_NS		245
-+#define PCLK_OTPC_S			246
-+#define CLK_SBPI_OTPC_S			247
-+#define CLK_USER_OTPC_S			248
-+#define CLK_OTPC_ARB			249
-+#define PCLK_OTPPHY			250
-+#define PCLK_USB2PHY			251
-+#define PCLK_PIPEPHY			252
-+#define PCLK_SARADC			253
-+#define CLK_SARADC			254
-+#define PCLK_IOC_VCCIO234		255
-+#define PCLK_PERI_GPIO1			256
-+#define PCLK_PERI_GPIO2			257
-+#define DCLK_PERI_GPIO			258
-+#define DCLK_PERI_GPIO1			259
-+#define DCLK_PERI_GPIO2			260
-+#define ACLK_PHP			261
-+#define PCLK_PHP			262
-+#define ACLK_PCIE20_MST			263
-+#define ACLK_PCIE20_SLV			264
-+#define ACLK_PCIE20_DBI			265
-+#define PCLK_PCIE20			266
-+#define CLK_PCIE20_AUX			267
-+#define ACLK_USB3OTG			268
-+#define CLK_USB3OTG_SUSPEND		269
-+#define CLK_USB3OTG_REF			270
-+#define CLK_PIPEPHY_REF_FUNC		271
-+#define CLK_200M_PMU			272
-+#define CLK_RTC_32K			273
-+#define CLK_RTC32K_FRAC			274
-+#define BUSCLK_PDPMU0			275
-+#define PCLK_PMU0_CRU			276
-+#define PCLK_PMU0_PMU			277
-+#define CLK_PMU0_PMU			278
-+#define PCLK_PMU0_HP_TIMER		279
-+#define CLK_PMU0_HP_TIMER		280
-+#define CLK_PMU0_32K_HP_TIMER		281
-+#define PCLK_PMU0_PVTM			282
-+#define CLK_PMU0_PVTM			283
-+#define PCLK_IOC_PMUIO			284
-+#define PCLK_PMU0_GPIO0			285
-+#define DBCLK_PMU0_GPIO0		286
-+#define PCLK_PMU0_GRF			287
-+#define PCLK_PMU0_SGRF			288
-+#define CLK_DDR_FAIL_SAFE		289
-+#define PCLK_PMU0_SCRKEYGEN		290
-+#define PCLK_PMU1_CRU			291
-+#define HCLK_PMU1_MEM			292
-+#define PCLK_PMU0_I2C0			293
-+#define CLK_PMU0_I2C0			294
-+#define PCLK_PMU1_UART0			295
-+#define CLK_PMU1_UART0_SRC		296
-+#define CLK_PMU1_UART0_FRAC		297
-+#define CLK_PMU1_UART0			298
-+#define SCLK_PMU1_UART0			299
-+#define PCLK_PMU1_SPI0			300
-+#define CLK_PMU1_SPI0			301
-+#define SCLK_IN_PMU1_SPI0		302
-+#define PCLK_PMU1_PWM0			303
-+#define CLK_PMU1_PWM0			304
-+#define CLK_CAPTURE_PMU1_PWM0		305
-+#define CLK_PMU1_WIFI			306
-+#define FCLK_PMU1_CM0_CORE		307
-+#define CLK_PMU1_CM0_RTC		308
-+#define PCLK_PMU1_WDTNS			309
-+#define CLK_PMU1_WDTNS			310
-+#define PCLK_PMU1_MAILBOX		311
-+#define CLK_PIPEPHY_DIV			312
-+#define CLK_PIPEPHY_XIN24M		313
-+#define CLK_PIPEPHY_REF			314
-+#define CLK_24M_SSCSRC			315
-+#define CLK_USB2PHY_XIN24M		316
-+#define CLK_USB2PHY_REF			317
-+#define CLK_MIPIDSIPHY_XIN24M		318
-+#define CLK_MIPIDSIPHY_REF		319
-+#define ACLK_RGA_PRE			320
-+#define HCLK_RGA_PRE			321
-+#define ACLK_RGA			322
-+#define HCLK_RGA			323
-+#define CLK_RGA_CORE			324
-+#define ACLK_JDEC			325
-+#define HCLK_JDEC			326
-+#define ACLK_VDPU_PRE			327
-+#define CLK_RKVDEC_HEVC_CA		328
-+#define HCLK_VDPU_PRE			329
-+#define ACLK_RKVDEC			330
-+#define HCLK_RKVDEC			331
-+#define CLK_RKVENC_CORE			332
-+#define ACLK_VEPU_PRE			333
-+#define HCLK_VEPU_PRE			334
-+#define ACLK_RKVENC			335
-+#define HCLK_RKVENC			336
-+#define ACLK_VI				337
-+#define HCLK_VI				338
-+#define PCLK_VI				339
-+#define ACLK_ISP			340
-+#define HCLK_ISP			341
-+#define CLK_ISP				342
-+#define ACLK_VICAP			343
-+#define HCLK_VICAP			344
-+#define DCLK_VICAP			345
-+#define CSIRX0_CLK_DATA			346
-+#define CSIRX1_CLK_DATA			347
-+#define CSIRX2_CLK_DATA			348
-+#define CSIRX3_CLK_DATA			349
-+#define PCLK_CSIHOST0			350
-+#define PCLK_CSIHOST1			351
-+#define PCLK_CSIHOST2			352
-+#define PCLK_CSIHOST3			353
-+#define PCLK_CSIPHY0			354
-+#define PCLK_CSIPHY1			355
-+#define ACLK_VO_PRE			356
-+#define HCLK_VO_PRE			357
-+#define ACLK_VOP			358
-+#define HCLK_VOP			359
-+#define DCLK_VOP			360
-+#define DCLK_VOP1			361
-+#define ACLK_CRYPTO_S			362
-+#define PCLK_CRYPTO_S			363
-+#define CLK_CORE_CRYPTO_S		364
-+#define CLK_PKA_CRYPTO_S		365
-+
-+#endif
-diff --git a/include/dt-bindings/clock/rockchip,rk3576-cru.h b/include/dt-bindings/clock/rockchip,rk3576-cru.h
-new file mode 100644
-index 000000000000..ded5ce42e62a
--- /dev/null
-+++ b/include/dt-bindings/clock/rockchip,rk3576-cru.h
-@@ -0,0 +1,607 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
-+/*
-+ * Copyright (c) 2023 Rockchip Electronics Co. Ltd.
-+ * Copyright (c) 2024 Collabora Ltd.
-+ *
-+ * Author: Elaine Zhang <zhangqing@rock-chips.com>
-+ * Author: Detlev Casanova <detlev.casanova@collabora.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_ROCKCHIP_RK3576_H
-+#define _DT_BINDINGS_CLK_ROCKCHIP_RK3576_H
-+
-+/* cru-clocks indices */
-+
-+/* cru plls */
-+#define PLL_BPLL			0
-+#define PLL_LPLL			1
-+#define PLL_VPLL			2
-+#define PLL_AUPLL			3
-+#define PLL_CPLL			4
-+#define PLL_GPLL			5
-+#define PLL_PPLL			6
-+#define ARMCLK_L			7
-+#define ARMCLK_B			8
-+
-+/* cru clocks */
-+#define CLK_CPLL_DIV20			9
-+#define CLK_CPLL_DIV10			10
-+#define CLK_GPLL_DIV8			11
-+#define CLK_GPLL_DIV6			12
-+#define CLK_CPLL_DIV4			13
-+#define CLK_GPLL_DIV4			14
-+#define CLK_SPLL_DIV2			15
-+#define CLK_GPLL_DIV3			16
-+#define CLK_CPLL_DIV2			17
-+#define CLK_GPLL_DIV2			18
-+#define CLK_SPLL_DIV1			19
-+#define PCLK_TOP_ROOT			20
-+#define ACLK_TOP			21
-+#define HCLK_TOP			22
-+#define CLK_AUDIO_FRAC_0		23
-+#define CLK_AUDIO_FRAC_1		24
-+#define CLK_AUDIO_FRAC_2		25
-+#define CLK_AUDIO_FRAC_3		26
-+#define CLK_UART_FRAC_0			27
-+#define CLK_UART_FRAC_1			28
-+#define CLK_UART_FRAC_2			29
-+#define CLK_UART1_SRC_TOP		30
-+#define CLK_AUDIO_INT_0			31
-+#define CLK_AUDIO_INT_1			32
-+#define CLK_AUDIO_INT_2			33
-+#define CLK_PDM0_SRC_TOP		34
-+#define CLK_PDM1_OUT			35
-+#define CLK_GMAC0_125M_SRC		36
-+#define CLK_GMAC1_125M_SRC		37
-+#define LCLK_ASRC_SRC_0			38
-+#define LCLK_ASRC_SRC_1			39
-+#define REF_CLK0_OUT_PLL		40
-+#define REF_CLK1_OUT_PLL		41
-+#define REF_CLK2_OUT_PLL		42
-+#define REFCLKO25M_GMAC0_OUT		43
-+#define REFCLKO25M_GMAC1_OUT		44
-+#define CLK_CIFOUT_OUT			45
-+#define CLK_GMAC0_RMII_CRU		46
-+#define CLK_GMAC1_RMII_CRU		47
-+#define CLK_OTPC_AUTO_RD_G		48
-+#define CLK_OTP_PHY_G			49
-+#define CLK_MIPI_CAMERAOUT_M0		50
-+#define CLK_MIPI_CAMERAOUT_M1		51
-+#define CLK_MIPI_CAMERAOUT_M2		52
-+#define MCLK_PDM0_SRC_TOP		53
-+#define HCLK_AUDIO_ROOT			54
-+#define HCLK_ASRC_2CH_0			55
-+#define HCLK_ASRC_2CH_1			56
-+#define HCLK_ASRC_4CH_0			57
-+#define HCLK_ASRC_4CH_1			58
-+#define CLK_ASRC_2CH_0			59
-+#define CLK_ASRC_2CH_1			60
-+#define CLK_ASRC_4CH_0			61
-+#define CLK_ASRC_4CH_1			62
-+#define MCLK_SAI0_8CH_SRC		63
-+#define MCLK_SAI0_8CH			64
-+#define HCLK_SAI0_8CH			65
-+#define HCLK_SPDIF_RX0			66
-+#define MCLK_SPDIF_RX0			67
-+#define HCLK_SPDIF_RX1			68
-+#define MCLK_SPDIF_RX1			69
-+#define MCLK_SAI1_8CH_SRC		70
-+#define MCLK_SAI1_8CH			71
-+#define HCLK_SAI1_8CH			72
-+#define MCLK_SAI2_2CH_SRC		73
-+#define MCLK_SAI2_2CH			74
-+#define HCLK_SAI2_2CH			75
-+#define MCLK_SAI3_2CH_SRC		76
-+#define MCLK_SAI3_2CH			77
-+#define HCLK_SAI3_2CH			78
-+#define MCLK_SAI4_2CH_SRC		79
-+#define MCLK_SAI4_2CH			80
-+#define HCLK_SAI4_2CH			81
-+#define HCLK_ACDCDIG_DSM		82
-+#define MCLK_ACDCDIG_DSM		83
-+#define CLK_PDM1			84
-+#define HCLK_PDM1			85
-+#define MCLK_PDM1			86
-+#define HCLK_SPDIF_TX0			87
-+#define MCLK_SPDIF_TX0			88
-+#define HCLK_SPDIF_TX1			89
-+#define MCLK_SPDIF_TX1			90
-+#define CLK_SAI1_MCLKOUT		91
-+#define CLK_SAI2_MCLKOUT		92
-+#define CLK_SAI3_MCLKOUT		93
-+#define CLK_SAI4_MCLKOUT		94
-+#define CLK_SAI0_MCLKOUT		95
-+#define HCLK_BUS_ROOT			96
-+#define PCLK_BUS_ROOT			97
-+#define ACLK_BUS_ROOT			98
-+#define HCLK_CAN0			99
-+#define CLK_CAN0			100
-+#define HCLK_CAN1			101
-+#define CLK_CAN1			102
-+#define CLK_KEY_SHIFT			103
-+#define PCLK_I2C1			104
-+#define PCLK_I2C2			105
-+#define PCLK_I2C3			106
-+#define PCLK_I2C4			107
-+#define PCLK_I2C5			108
-+#define PCLK_I2C6			109
-+#define PCLK_I2C7			110
-+#define PCLK_I2C8			111
-+#define PCLK_I2C9			112
-+#define PCLK_WDT_BUSMCU			113
-+#define TCLK_WDT_BUSMCU			114
-+#define ACLK_GIC			115
-+#define CLK_I2C1			116
-+#define CLK_I2C2			117
-+#define CLK_I2C3			118
-+#define CLK_I2C4			119
-+#define CLK_I2C5			120
-+#define CLK_I2C6			121
-+#define CLK_I2C7			122
-+#define CLK_I2C8			123
-+#define CLK_I2C9			124
-+#define PCLK_SARADC			125
-+#define CLK_SARADC			126
-+#define PCLK_TSADC			127
-+#define CLK_TSADC			128
-+#define PCLK_UART0			129
-+#define PCLK_UART2			130
-+#define PCLK_UART3			131
-+#define PCLK_UART4			132
-+#define PCLK_UART5			133
-+#define PCLK_UART6			134
-+#define PCLK_UART7			135
-+#define PCLK_UART8			136
-+#define PCLK_UART9			137
-+#define PCLK_UART10			138
-+#define PCLK_UART11			139
-+#define SCLK_UART0			140
-+#define SCLK_UART2			141
-+#define SCLK_UART3			142
-+#define SCLK_UART4			143
-+#define SCLK_UART5			144
-+#define SCLK_UART6			145
-+#define SCLK_UART7			146
-+#define SCLK_UART8			147
-+#define SCLK_UART9			148
-+#define SCLK_UART10			149
-+#define SCLK_UART11			150
-+#define PCLK_SPI0			151
-+#define PCLK_SPI1			152
-+#define PCLK_SPI2			153
-+#define PCLK_SPI3			154
-+#define PCLK_SPI4			155
-+#define CLK_SPI0			156
-+#define CLK_SPI1			157
-+#define CLK_SPI2			158
-+#define CLK_SPI3			159
-+#define CLK_SPI4			160
-+#define PCLK_WDT0			161
-+#define TCLK_WDT0			162
-+#define PCLK_PWM1			163
-+#define CLK_PWM1			164
-+#define CLK_OSC_PWM1			165
-+#define CLK_RC_PWM1			166
-+#define PCLK_BUSTIMER0			167
-+#define PCLK_BUSTIMER1			168
-+#define CLK_TIMER0_ROOT			169
-+#define CLK_TIMER0			170
-+#define CLK_TIMER1			171
-+#define CLK_TIMER2			172
-+#define CLK_TIMER3			173
-+#define CLK_TIMER4			174
-+#define CLK_TIMER5			175
-+#define PCLK_MAILBOX0			176
-+#define PCLK_GPIO1			177
-+#define DBCLK_GPIO1			178
-+#define PCLK_GPIO2			179
-+#define DBCLK_GPIO2			180
-+#define PCLK_GPIO3			181
-+#define DBCLK_GPIO3			182
-+#define PCLK_GPIO4			183
-+#define DBCLK_GPIO4			184
-+#define ACLK_DECOM			185
-+#define PCLK_DECOM			186
-+#define DCLK_DECOM			187
-+#define CLK_TIMER1_ROOT			188
-+#define CLK_TIMER6			189
-+#define CLK_TIMER7			190
-+#define CLK_TIMER8			191
-+#define CLK_TIMER9			192
-+#define CLK_TIMER10			193
-+#define CLK_TIMER11			194
-+#define ACLK_DMAC0			195
-+#define ACLK_DMAC1			196
-+#define ACLK_DMAC2			197
-+#define ACLK_SPINLOCK			198
-+#define HCLK_I3C0			199
-+#define HCLK_I3C1			200
-+#define HCLK_BUS_CM0_ROOT		201
-+#define FCLK_BUS_CM0_CORE		202
-+#define CLK_BUS_CM0_RTC			203
-+#define PCLK_PMU2			204
-+#define PCLK_PWM2			205
-+#define CLK_PWM2			206
-+#define CLK_RC_PWM2			207
-+#define CLK_OSC_PWM2			208
-+#define CLK_FREQ_PWM1			209
-+#define CLK_COUNTER_PWM1		210
-+#define SAI_SCLKIN_FREQ			211
-+#define SAI_SCLKIN_COUNTER		212
-+#define CLK_I3C0			213
-+#define CLK_I3C1			214
-+#define PCLK_CSIDPHY1			215
-+#define PCLK_DDR_ROOT			216
-+#define PCLK_DDR_MON_CH0		217
-+#define TMCLK_DDR_MON_CH0		218
-+#define ACLK_DDR_ROOT			219
-+#define HCLK_DDR_ROOT			220
-+#define FCLK_DDR_CM0_CORE		221
-+#define CLK_DDR_TIMER_ROOT		222
-+#define CLK_DDR_TIMER0			223
-+#define CLK_DDR_TIMER1			224
-+#define TCLK_WDT_DDR			225
-+#define PCLK_WDT			226
-+#define PCLK_TIMER			227
-+#define CLK_DDR_CM0_RTC			228
-+#define ACLK_RKNN0			229
-+#define ACLK_RKNN1			230
-+#define HCLK_RKNN_ROOT			231
-+#define CLK_RKNN_DSU0			232
-+#define PCLK_NPUTOP_ROOT		233
-+#define PCLK_NPU_TIMER			234
-+#define CLK_NPUTIMER_ROOT		235
-+#define CLK_NPUTIMER0			236
-+#define CLK_NPUTIMER1			237
-+#define PCLK_NPU_WDT			238
-+#define TCLK_NPU_WDT			239
-+#define ACLK_RKNN_CBUF			240
-+#define HCLK_NPU_CM0_ROOT		241
-+#define FCLK_NPU_CM0_CORE		242
-+#define CLK_NPU_CM0_RTC			243
-+#define HCLK_RKNN_CBUF			244
-+#define HCLK_NVM_ROOT			245
-+#define ACLK_NVM_ROOT			246
-+#define SCLK_FSPI_X2			247
-+#define HCLK_FSPI			248
-+#define CCLK_SRC_EMMC			249
-+#define HCLK_EMMC			250
-+#define ACLK_EMMC			251
-+#define BCLK_EMMC			252
-+#define TCLK_EMMC			253
-+#define PCLK_PHP_ROOT			254
-+#define ACLK_PHP_ROOT			255
-+#define PCLK_PCIE0			256
-+#define CLK_PCIE0_AUX			257
-+#define ACLK_PCIE0_MST			258
-+#define ACLK_PCIE0_SLV			259
-+#define ACLK_PCIE0_DBI			260
-+#define ACLK_USB3OTG1			261
-+#define CLK_REF_USB3OTG1		262
-+#define CLK_SUSPEND_USB3OTG1		263
-+#define ACLK_MMU0			264
-+#define ACLK_SLV_MMU0			265
-+#define ACLK_MMU1			266
-+#define ACLK_SLV_MMU1			267
-+#define PCLK_PCIE1			268
-+#define CLK_PCIE1_AUX			269
-+#define ACLK_PCIE1_MST			270
-+#define ACLK_PCIE1_SLV			271
-+#define ACLK_PCIE1_DBI			272
-+#define CLK_RXOOB0			273
-+#define CLK_RXOOB1			274
-+#define CLK_PMALIVE0			275
-+#define CLK_PMALIVE1			276
-+#define ACLK_SATA0			277
-+#define ACLK_SATA1			278
-+#define CLK_USB3OTG1_PIPE_PCLK		279
-+#define CLK_USB3OTG1_UTMI		280
-+#define CLK_USB3OTG0_PIPE_PCLK		281
-+#define CLK_USB3OTG0_UTMI		282
-+#define HCLK_SDGMAC_ROOT		283
-+#define ACLK_SDGMAC_ROOT		284
-+#define PCLK_SDGMAC_ROOT		285
-+#define ACLK_GMAC0			286
-+#define ACLK_GMAC1			287
-+#define PCLK_GMAC0			288
-+#define PCLK_GMAC1			289
-+#define CCLK_SRC_SDIO			290
-+#define HCLK_SDIO			291
-+#define CLK_GMAC1_PTP_REF		292
-+#define CLK_GMAC0_PTP_REF		293
-+#define CLK_GMAC1_PTP_REF_SRC		294
-+#define CLK_GMAC0_PTP_REF_SRC		295
-+#define CCLK_SRC_SDMMC0			296
-+#define HCLK_SDMMC0			297
-+#define SCLK_FSPI1_X2			298
-+#define HCLK_FSPI1			299
-+#define ACLK_DSMC_ROOT			300
-+#define ACLK_DSMC			301
-+#define PCLK_DSMC			302
-+#define CLK_DSMC_SYS			303
-+#define HCLK_HSGPIO			304
-+#define CLK_HSGPIO_TX			305
-+#define CLK_HSGPIO_RX			306
-+#define ACLK_HSGPIO			307
-+#define PCLK_PHPPHY_ROOT		308
-+#define PCLK_PCIE2_COMBOPHY0		309
-+#define PCLK_PCIE2_COMBOPHY1		310
-+#define CLK_PCIE_100M_SRC		311
-+#define CLK_PCIE_100M_NDUTY_SRC		312
-+#define CLK_REF_PCIE0_PHY		313
-+#define CLK_REF_PCIE1_PHY		314
-+#define CLK_REF_MPHY_26M		315
-+#define HCLK_RKVDEC_ROOT		316
-+#define ACLK_RKVDEC_ROOT		317
-+#define HCLK_RKVDEC			318
-+#define CLK_RKVDEC_HEVC_CA		319
-+#define CLK_RKVDEC_CORE			320
-+#define ACLK_UFS_ROOT			321
-+#define ACLK_USB_ROOT			322
-+#define PCLK_USB_ROOT			323
-+#define ACLK_USB3OTG0			324
-+#define CLK_REF_USB3OTG0		325
-+#define CLK_SUSPEND_USB3OTG0		326
-+#define ACLK_MMU2			327
-+#define ACLK_SLV_MMU2			328
-+#define ACLK_UFS_SYS			329
-+#define ACLK_VPU_ROOT			330
-+#define ACLK_VPU_MID_ROOT		331
-+#define HCLK_VPU_ROOT			332
-+#define ACLK_JPEG_ROOT			333
-+#define ACLK_VPU_LOW_ROOT		334
-+#define HCLK_RGA2E_0			335
-+#define ACLK_RGA2E_0			336
-+#define CLK_CORE_RGA2E_0		337
-+#define ACLK_JPEG			338
-+#define HCLK_JPEG			339
-+#define HCLK_VDPP			340
-+#define ACLK_VDPP			341
-+#define CLK_CORE_VDPP			342
-+#define HCLK_RGA2E_1			343
-+#define ACLK_RGA2E_1			344
-+#define CLK_CORE_RGA2E_1		345
-+#define DCLK_EBC_FRAC_SRC		346
-+#define HCLK_EBC			347
-+#define ACLK_EBC			348
-+#define DCLK_EBC			349
-+#define HCLK_VEPU0_ROOT			350
-+#define ACLK_VEPU0_ROOT			351
-+#define HCLK_VEPU0			352
-+#define ACLK_VEPU0			353
-+#define CLK_VEPU0_CORE			354
-+#define ACLK_VI_ROOT			355
-+#define HCLK_VI_ROOT			356
-+#define PCLK_VI_ROOT			357
-+#define DCLK_VICAP			358
-+#define ACLK_VICAP			359
-+#define HCLK_VICAP			360
-+#define CLK_ISP_CORE			361
-+#define CLK_ISP_CORE_MARVIN		362
-+#define CLK_ISP_CORE_VICAP		363
-+#define ACLK_ISP			364
-+#define HCLK_ISP			365
-+#define ACLK_VPSS			366
-+#define HCLK_VPSS			367
-+#define CLK_CORE_VPSS			368
-+#define PCLK_CSI_HOST_0			369
-+#define PCLK_CSI_HOST_1			370
-+#define PCLK_CSI_HOST_2			371
-+#define PCLK_CSI_HOST_3			372
-+#define PCLK_CSI_HOST_4			373
-+#define ICLK_CSIHOST01			374
-+#define ICLK_CSIHOST0			375
-+#define CLK_ISP_PVTPLL_SRC		376
-+#define ACLK_VI_ROOT_INTER		377
-+#define CLK_VICAP_I0CLK			378
-+#define CLK_VICAP_I1CLK			379
-+#define CLK_VICAP_I2CLK			380
-+#define CLK_VICAP_I3CLK			381
-+#define CLK_VICAP_I4CLK			382
-+#define ACLK_VOP_ROOT			383
-+#define HCLK_VOP_ROOT			384
-+#define PCLK_VOP_ROOT			385
-+#define HCLK_VOP			386
-+#define ACLK_VOP			387
-+#define DCLK_VP0_SRC			388
-+#define DCLK_VP1_SRC			389
-+#define DCLK_VP2_SRC			390
-+#define DCLK_VP0			391
-+#define DCLK_VP1			392
-+#define DCLK_VP2			393
-+#define PCLK_VOPGRF			394
-+#define ACLK_VO0_ROOT			395
-+#define HCLK_VO0_ROOT			396
-+#define PCLK_VO0_ROOT			397
-+#define PCLK_VO0_GRF			398
-+#define ACLK_HDCP0			399
-+#define HCLK_HDCP0			400
-+#define PCLK_HDCP0			401
-+#define CLK_TRNG0_SKP			402
-+#define PCLK_DSIHOST0			403
-+#define CLK_DSIHOST0			404
-+#define PCLK_HDMITX0			405
-+#define CLK_HDMITX0_EARC		406
-+#define CLK_HDMITX0_REF			407
-+#define PCLK_EDP0			408
-+#define CLK_EDP0_24M			409
-+#define CLK_EDP0_200M			410
-+#define MCLK_SAI5_8CH_SRC		411
-+#define MCLK_SAI5_8CH			412
-+#define HCLK_SAI5_8CH			413
-+#define MCLK_SAI6_8CH_SRC		414
-+#define MCLK_SAI6_8CH			415
-+#define HCLK_SAI6_8CH			416
-+#define HCLK_SPDIF_TX2			417
-+#define MCLK_SPDIF_TX2			418
-+#define HCLK_SPDIF_RX2			419
-+#define MCLK_SPDIF_RX2			420
-+#define HCLK_SAI8_8CH			421
-+#define MCLK_SAI8_8CH_SRC		422
-+#define MCLK_SAI8_8CH			423
-+#define ACLK_VO1_ROOT			424
-+#define HCLK_VO1_ROOT			425
-+#define PCLK_VO1_ROOT			426
-+#define MCLK_SAI7_8CH_SRC		427
-+#define MCLK_SAI7_8CH			428
-+#define HCLK_SAI7_8CH			429
-+#define HCLK_SPDIF_TX3			430
-+#define HCLK_SPDIF_TX4			431
-+#define HCLK_SPDIF_TX5			432
-+#define MCLK_SPDIF_TX3			433
-+#define CLK_AUX16MHZ_0			434
-+#define ACLK_DP0			435
-+#define PCLK_DP0			436
-+#define PCLK_VO1_GRF			437
-+#define ACLK_HDCP1			438
-+#define HCLK_HDCP1			439
-+#define PCLK_HDCP1			440
-+#define CLK_TRNG1_SKP			441
-+#define HCLK_SAI9_8CH			442
-+#define MCLK_SAI9_8CH_SRC		443
-+#define MCLK_SAI9_8CH			444
-+#define MCLK_SPDIF_TX4			445
-+#define MCLK_SPDIF_TX5			446
-+#define CLK_GPU_SRC_PRE			447
-+#define CLK_GPU				448
-+#define PCLK_GPU_ROOT			449
-+#define ACLK_CENTER_ROOT		450
-+#define ACLK_CENTER_LOW_ROOT		451
-+#define HCLK_CENTER_ROOT		452
-+#define PCLK_CENTER_ROOT		453
-+#define ACLK_DMA2DDR			454
-+#define ACLK_DDR_SHAREMEM		455
-+#define PCLK_DMA2DDR			456
-+#define PCLK_SHAREMEM			457
-+#define HCLK_VEPU1_ROOT			458
-+#define ACLK_VEPU1_ROOT			459
-+#define HCLK_VEPU1			460
-+#define ACLK_VEPU1			461
-+#define CLK_VEPU1_CORE			462
-+#define CLK_JDBCK_DAP			463
-+#define PCLK_MIPI_DCPHY			464
-+#define CLK_32K_USB2DEBUG		465
-+#define PCLK_CSIDPHY			466
-+#define PCLK_USBDPPHY			467
-+#define CLK_PMUPHY_REF_SRC		468
-+#define CLK_USBDP_COMBO_PHY_IMMORTAL	469
-+#define CLK_HDMITXHDP			470
-+#define PCLK_MPHY			471
-+#define CLK_REF_OSC_MPHY		472
-+#define CLK_REF_UFS_CLKOUT		473
-+#define HCLK_PMU1_ROOT			474
-+#define HCLK_PMU_CM0_ROOT		475
-+#define CLK_200M_PMU_SRC		476
-+#define CLK_100M_PMU_SRC		477
-+#define CLK_50M_PMU_SRC			478
-+#define FCLK_PMU_CM0_CORE		479
-+#define CLK_PMU_CM0_RTC			480
-+#define PCLK_PMU1			481
-+#define CLK_PMU1			482
-+#define PCLK_PMU1WDT			483
-+#define TCLK_PMU1WDT			484
-+#define PCLK_PMUTIMER			485
-+#define CLK_PMUTIMER_ROOT		486
-+#define CLK_PMUTIMER0			487
-+#define CLK_PMUTIMER1			488
-+#define PCLK_PMU1PWM			489
-+#define CLK_PMU1PWM			490
-+#define CLK_PMU1PWM_OSC			491
-+#define PCLK_PMUPHY_ROOT		492
-+#define PCLK_I2C0			493
-+#define CLK_I2C0			494
-+#define SCLK_UART1			495
-+#define PCLK_UART1			496
-+#define CLK_PMU1PWM_RC			497
-+#define CLK_PDM0			498
-+#define HCLK_PDM0			499
-+#define MCLK_PDM0			500
-+#define HCLK_VAD			501
-+#define CLK_OSCCHK_PVTM			502
-+#define CLK_PDM0_OUT			503
-+#define CLK_HPTIMER_SRC			504
-+#define PCLK_PMU0_ROOT			505
-+#define PCLK_PMU0			506
-+#define PCLK_GPIO0			507
-+#define DBCLK_GPIO0			508
-+#define CLK_OSC0_PMU1			509
-+#define PCLK_PMU1_ROOT			510
-+#define XIN_OSC0_DIV			511
-+#define ACLK_USB			512
-+#define ACLK_UFS			513
-+#define ACLK_SDGMAC			514
-+#define HCLK_SDGMAC			515
-+#define PCLK_SDGMAC			516
-+#define HCLK_VO1			517
-+#define HCLK_VO0			518
-+#define PCLK_CCI_ROOT			519
-+#define ACLK_CCI_ROOT			520
-+#define HCLK_VO0VOP_CHANNEL		521
-+#define ACLK_VO0VOP_CHANNEL		522
-+#define ACLK_TOP_MID			523
-+#define ACLK_SECURE_HIGH		524
-+#define CLK_USBPHY_REF_SRC		525
-+#define CLK_PHY_REF_SRC			526
-+#define CLK_CPLL_REF_SRC		527
-+#define CLK_AUPLL_REF_SRC		528
-+#define PCLK_SECURE_NS			529
-+#define HCLK_SECURE_NS			530
-+#define ACLK_SECURE_NS			531
-+#define PCLK_OTPC_NS			532
-+#define HCLK_CRYPTO_NS			533
-+#define HCLK_TRNG_NS			534
-+#define CLK_OTPC_NS			535
-+#define SCLK_DSU			536
-+#define SCLK_DDR			537
-+#define ACLK_CRYPTO_NS			538
-+#define CLK_PKA_CRYPTO_NS		539
-+#define ACLK_RKVDEC_ROOT_BAK		540
-+#define CLK_AUDIO_FRAC_0_SRC		541
-+#define CLK_AUDIO_FRAC_1_SRC		542
-+#define CLK_AUDIO_FRAC_2_SRC		543
-+#define CLK_AUDIO_FRAC_3_SRC		544
-+#define PCLK_HDPTX_APB			545
-+
-+/* secure clk */
-+#define CLK_STIMER0_ROOT		546
-+#define CLK_STIMER1_ROOT		547
-+#define PCLK_SECURE_S			548
-+#define HCLK_SECURE_S			549
-+#define ACLK_SECURE_S			550
-+#define CLK_PKA_CRYPTO_S		551
-+#define HCLK_VO1_S			552
-+#define PCLK_VO1_S			553
-+#define HCLK_VO0_S			554
-+#define PCLK_VO0_S			555
-+#define PCLK_KLAD			556
-+#define HCLK_CRYPTO_S			557
-+#define HCLK_KLAD			558
-+#define ACLK_CRYPTO_S			559
-+#define HCLK_TRNG_S			560
-+#define PCLK_OTPC_S			561
-+#define CLK_OTPC_S			562
-+#define PCLK_WDT_S			563
-+#define TCLK_WDT_S			564
-+#define PCLK_HDCP0_TRNG			565
-+#define PCLK_HDCP1_TRNG			566
-+#define HCLK_HDCP_KEY0			567
-+#define HCLK_HDCP_KEY1			568
-+#define PCLK_EDP_S			569
-+#define ACLK_KLAD			570
-+
-+/* SCMI clocks, use these when changing clocks through SCMI */
-+#define SCMI_ARMCLK_L			10
-+#define SCMI_ARMCLK_B			11
-+#define SCMI_CLK_GPU			456
-+
-+/* IOC-controlled output clocks */
-+#define CLK_SAI0_MCLKOUT_TO_IO		571
-+#define CLK_SAI1_MCLKOUT_TO_IO		572
-+#define CLK_SAI2_MCLKOUT_TO_IO		573
-+#define CLK_SAI3_MCLKOUT_TO_IO		574
-+#define CLK_SAI4_MCLKOUT_TO_IO		575
-+#define CLK_SAI4_MCLKOUT_TO_IO		575
-+#define CLK_FSPI0_TO_IO			576
-+#define CLK_FSPI1_TO_IO			577
-+
-+#endif
-diff --git a/include/dt-bindings/clock/rockchip,rk3588-cru.h b/include/dt-bindings/clock/rockchip,rk3588-cru.h
-new file mode 100644
-index 000000000000..0c7d3ca2d5bc
--- /dev/null
-+++ b/include/dt-bindings/clock/rockchip,rk3588-cru.h
-@@ -0,0 +1,765 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
-+/*
-+ * Copyright (c) 2021 Rockchip Electronics Co. Ltd.
-+ * Copyright (c) 2022 Collabora Ltd.
-+ *
-+ * Author: Elaine Zhang <zhangqing@rock-chips.com>
-+ * Author: Sebastian Reichel <sebastian.reichel@collabora.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_ROCKCHIP_RK3588_H
-+#define _DT_BINDINGS_CLK_ROCKCHIP_RK3588_H
-+
-+/* cru-clocks indices */
-+
-+#define PLL_B0PLL			0
-+#define PLL_B1PLL			1
-+#define PLL_LPLL			2
-+#define PLL_V0PLL			3
-+#define PLL_AUPLL			4
-+#define PLL_CPLL			5
-+#define PLL_GPLL			6
-+#define PLL_NPLL			7
-+#define PLL_PPLL			8
-+#define ARMCLK_L			9
-+#define ARMCLK_B01			10
-+#define ARMCLK_B23			11
-+#define PCLK_BIGCORE0_ROOT		12
-+#define PCLK_BIGCORE0_PVTM		13
-+#define PCLK_BIGCORE1_ROOT		14
-+#define PCLK_BIGCORE1_PVTM		15
-+#define PCLK_DSU_S_ROOT			16
-+#define PCLK_DSU_ROOT			17
-+#define PCLK_DSU_NS_ROOT		18
-+#define PCLK_LITCORE_PVTM		19
-+#define PCLK_DBG			20
-+#define PCLK_DSU			21
-+#define PCLK_S_DAPLITE			22
-+#define PCLK_M_DAPLITE			23
-+#define MBIST_MCLK_PDM1			24
-+#define MBIST_CLK_ACDCDIG		25
-+#define HCLK_I2S2_2CH			26
-+#define HCLK_I2S3_2CH			27
-+#define CLK_I2S2_2CH_SRC		28
-+#define CLK_I2S2_2CH_FRAC		29
-+#define CLK_I2S2_2CH			30
-+#define MCLK_I2S2_2CH			31
-+#define I2S2_2CH_MCLKOUT		32
-+#define CLK_DAC_ACDCDIG			33
-+#define CLK_I2S3_2CH_SRC		34
-+#define CLK_I2S3_2CH_FRAC		35
-+#define CLK_I2S3_2CH			36
-+#define MCLK_I2S3_2CH			37
-+#define I2S3_2CH_MCLKOUT		38
-+#define PCLK_ACDCDIG			39
-+#define HCLK_I2S0_8CH			40
-+#define CLK_I2S0_8CH_TX_SRC		41
-+#define CLK_I2S0_8CH_TX_FRAC		42
-+#define MCLK_I2S0_8CH_TX		43
-+#define CLK_I2S0_8CH_TX			44
-+#define CLK_I2S0_8CH_RX_SRC		45
-+#define CLK_I2S0_8CH_RX_FRAC		46
-+#define MCLK_I2S0_8CH_RX		47
-+#define CLK_I2S0_8CH_RX			48
-+#define I2S0_8CH_MCLKOUT		49
-+#define HCLK_PDM1			50
-+#define MCLK_PDM1			51
-+#define HCLK_AUDIO_ROOT			52
-+#define PCLK_AUDIO_ROOT			53
-+#define HCLK_SPDIF0			54
-+#define CLK_SPDIF0_SRC			55
-+#define CLK_SPDIF0_FRAC			56
-+#define MCLK_SPDIF0			57
-+#define CLK_SPDIF0			58
-+#define CLK_SPDIF1			59
-+#define HCLK_SPDIF1			60
-+#define CLK_SPDIF1_SRC			61
-+#define CLK_SPDIF1_FRAC			62
-+#define MCLK_SPDIF1			63
-+#define ACLK_AV1_ROOT			64
-+#define ACLK_AV1			65
-+#define PCLK_AV1_ROOT			66
-+#define PCLK_AV1			67
-+#define PCLK_MAILBOX0			68
-+#define PCLK_MAILBOX1			69
-+#define PCLK_MAILBOX2			70
-+#define PCLK_PMU2			71
-+#define PCLK_PMUCM0_INTMUX		72
-+#define PCLK_DDRCM0_INTMUX		73
-+#define PCLK_TOP			74
-+#define PCLK_PWM1			75
-+#define CLK_PWM1			76
-+#define CLK_PWM1_CAPTURE		77
-+#define PCLK_PWM2			78
-+#define CLK_PWM2			79
-+#define CLK_PWM2_CAPTURE		80
-+#define PCLK_PWM3			81
-+#define CLK_PWM3			82
-+#define CLK_PWM3_CAPTURE		83
-+#define PCLK_BUSTIMER0			84
-+#define PCLK_BUSTIMER1			85
-+#define CLK_BUS_TIMER_ROOT		86
-+#define CLK_BUSTIMER0			87
-+#define CLK_BUSTIMER1			88
-+#define CLK_BUSTIMER2			89
-+#define CLK_BUSTIMER3			90
-+#define CLK_BUSTIMER4			91
-+#define CLK_BUSTIMER5			92
-+#define CLK_BUSTIMER6			93
-+#define CLK_BUSTIMER7			94
-+#define CLK_BUSTIMER8			95
-+#define CLK_BUSTIMER9			96
-+#define CLK_BUSTIMER10			97
-+#define CLK_BUSTIMER11			98
-+#define PCLK_WDT0			99
-+#define TCLK_WDT0			100
-+#define PCLK_CAN0			101
-+#define CLK_CAN0			102
-+#define PCLK_CAN1			103
-+#define CLK_CAN1			104
-+#define PCLK_CAN2			105
-+#define CLK_CAN2			106
-+#define ACLK_DECOM			107
-+#define PCLK_DECOM			108
-+#define DCLK_DECOM			109
-+#define ACLK_DMAC0			110
-+#define ACLK_DMAC1			111
-+#define ACLK_DMAC2			112
-+#define ACLK_BUS_ROOT			113
-+#define ACLK_GIC			114
-+#define PCLK_GPIO1			115
-+#define DBCLK_GPIO1			116
-+#define PCLK_GPIO2			117
-+#define DBCLK_GPIO2			118
-+#define PCLK_GPIO3			119
-+#define DBCLK_GPIO3			120
-+#define PCLK_GPIO4			121
-+#define DBCLK_GPIO4			122
-+#define PCLK_I2C1			123
-+#define PCLK_I2C2			124
-+#define PCLK_I2C3			125
-+#define PCLK_I2C4			126
-+#define PCLK_I2C5			127
-+#define PCLK_I2C6			128
-+#define PCLK_I2C7			129
-+#define PCLK_I2C8			130
-+#define CLK_I2C1			131
-+#define CLK_I2C2			132
-+#define CLK_I2C3			133
-+#define CLK_I2C4			134
-+#define CLK_I2C5			135
-+#define CLK_I2C6			136
-+#define CLK_I2C7			137
-+#define CLK_I2C8			138
-+#define PCLK_OTPC_NS			139
-+#define CLK_OTPC_NS			140
-+#define CLK_OTPC_ARB			141
-+#define CLK_OTPC_AUTO_RD_G		142
-+#define CLK_OTP_PHY_G			143
-+#define PCLK_SARADC			144
-+#define CLK_SARADC			145
-+#define PCLK_SPI0			146
-+#define PCLK_SPI1			147
-+#define PCLK_SPI2			148
-+#define PCLK_SPI3			149
-+#define PCLK_SPI4			150
-+#define CLK_SPI0			151
-+#define CLK_SPI1			152
-+#define CLK_SPI2			153
-+#define CLK_SPI3			154
-+#define CLK_SPI4			155
-+#define ACLK_SPINLOCK			156
-+#define PCLK_TSADC			157
-+#define CLK_TSADC			158
-+#define PCLK_UART1			159
-+#define PCLK_UART2			160
-+#define PCLK_UART3			161
-+#define PCLK_UART4			162
-+#define PCLK_UART5			163
-+#define PCLK_UART6			164
-+#define PCLK_UART7			165
-+#define PCLK_UART8			166
-+#define PCLK_UART9			167
-+#define CLK_UART1_SRC			168
-+#define CLK_UART1_FRAC			169
-+#define CLK_UART1			170
-+#define SCLK_UART1			171
-+#define CLK_UART2_SRC			172
-+#define CLK_UART2_FRAC			173
-+#define CLK_UART2			174
-+#define SCLK_UART2			175
-+#define CLK_UART3_SRC			176
-+#define CLK_UART3_FRAC			177
-+#define CLK_UART3			178
-+#define SCLK_UART3			179
-+#define CLK_UART4_SRC			180
-+#define CLK_UART4_FRAC			181
-+#define CLK_UART4			182
-+#define SCLK_UART4			183
-+#define CLK_UART5_SRC			184
-+#define CLK_UART5_FRAC			185
-+#define CLK_UART5			186
-+#define SCLK_UART5			187
-+#define CLK_UART6_SRC			188
-+#define CLK_UART6_FRAC			189
-+#define CLK_UART6			190
-+#define SCLK_UART6			191
-+#define CLK_UART7_SRC			192
-+#define CLK_UART7_FRAC			193
-+#define CLK_UART7			194
-+#define SCLK_UART7			195
-+#define CLK_UART8_SRC			196
-+#define CLK_UART8_FRAC			197
-+#define CLK_UART8			198
-+#define SCLK_UART8			199
-+#define CLK_UART9_SRC			200
-+#define CLK_UART9_FRAC			201
-+#define CLK_UART9			202
-+#define SCLK_UART9			203
-+#define ACLK_CENTER_ROOT		204
-+#define ACLK_CENTER_LOW_ROOT		205
-+#define HCLK_CENTER_ROOT		206
-+#define PCLK_CENTER_ROOT		207
-+#define ACLK_DMA2DDR			208
-+#define ACLK_DDR_SHAREMEM		209
-+#define ACLK_CENTER_S200_ROOT		210
-+#define ACLK_CENTER_S400_ROOT		211
-+#define FCLK_DDR_CM0_CORE		212
-+#define CLK_DDR_TIMER_ROOT		213
-+#define CLK_DDR_TIMER0			214
-+#define CLK_DDR_TIMER1			215
-+#define TCLK_WDT_DDR			216
-+#define CLK_DDR_CM0_RTC			217
-+#define PCLK_WDT			218
-+#define PCLK_TIMER			219
-+#define PCLK_DMA2DDR			220
-+#define PCLK_SHAREMEM			221
-+#define CLK_50M_SRC			222
-+#define CLK_100M_SRC			223
-+#define CLK_150M_SRC			224
-+#define CLK_200M_SRC			225
-+#define CLK_250M_SRC			226
-+#define CLK_300M_SRC			227
-+#define CLK_350M_SRC			228
-+#define CLK_400M_SRC			229
-+#define CLK_450M_SRC			230
-+#define CLK_500M_SRC			231
-+#define CLK_600M_SRC			232
-+#define CLK_650M_SRC			233
-+#define CLK_700M_SRC			234
-+#define CLK_800M_SRC			235
-+#define CLK_1000M_SRC			236
-+#define CLK_1200M_SRC			237
-+#define ACLK_TOP_M300_ROOT		238
-+#define ACLK_TOP_M500_ROOT		239
-+#define ACLK_TOP_M400_ROOT		240
-+#define ACLK_TOP_S200_ROOT		241
-+#define ACLK_TOP_S400_ROOT		242
-+#define CLK_MIPI_CAMARAOUT_M0		243
-+#define CLK_MIPI_CAMARAOUT_M1		244
-+#define CLK_MIPI_CAMARAOUT_M2		245
-+#define CLK_MIPI_CAMARAOUT_M3		246
-+#define CLK_MIPI_CAMARAOUT_M4		247
-+#define MCLK_GMAC0_OUT			248
-+#define REFCLKO25M_ETH0_OUT		249
-+#define REFCLKO25M_ETH1_OUT		250
-+#define CLK_CIFOUT_OUT			251
-+#define PCLK_MIPI_DCPHY0		252
-+#define PCLK_MIPI_DCPHY1		253
-+#define PCLK_CSIPHY0			254
-+#define PCLK_CSIPHY1			255
-+#define ACLK_TOP_ROOT			256
-+#define PCLK_TOP_ROOT			257
-+#define ACLK_LOW_TOP_ROOT		258
-+#define PCLK_CRU			259
-+#define PCLK_GPU_ROOT			260
-+#define CLK_GPU_SRC			261
-+#define CLK_GPU				262
-+#define CLK_GPU_COREGROUP		263
-+#define CLK_GPU_STACKS			264
-+#define PCLK_GPU_PVTM			265
-+#define CLK_GPU_PVTM			266
-+#define CLK_CORE_GPU_PVTM		267
-+#define PCLK_GPU_GRF			268
-+#define ACLK_ISP1_ROOT			269
-+#define HCLK_ISP1_ROOT			270
-+#define CLK_ISP1_CORE			271
-+#define CLK_ISP1_CORE_MARVIN		272
-+#define CLK_ISP1_CORE_VICAP		273
-+#define ACLK_ISP1			274
-+#define HCLK_ISP1			275
-+#define ACLK_NPU1			276
-+#define HCLK_NPU1			277
-+#define ACLK_NPU2			278
-+#define HCLK_NPU2			279
-+#define HCLK_NPU_CM0_ROOT		280
-+#define FCLK_NPU_CM0_CORE		281
-+#define CLK_NPU_CM0_RTC			282
-+#define PCLK_NPU_PVTM			283
-+#define PCLK_NPU_GRF			284
-+#define CLK_NPU_PVTM			285
-+#define CLK_CORE_NPU_PVTM		286
-+#define ACLK_NPU0			287
-+#define HCLK_NPU0			288
-+#define HCLK_NPU_ROOT			289
-+#define CLK_NPU_DSU0			290
-+#define PCLK_NPU_ROOT			291
-+#define PCLK_NPU_TIMER			292
-+#define CLK_NPUTIMER_ROOT		293
-+#define CLK_NPUTIMER0			294
-+#define CLK_NPUTIMER1			295
-+#define PCLK_NPU_WDT			296
-+#define TCLK_NPU_WDT			297
-+#define HCLK_EMMC			298
-+#define ACLK_EMMC			299
-+#define CCLK_EMMC			300
-+#define BCLK_EMMC			301
-+#define TMCLK_EMMC			302
-+#define SCLK_SFC			303
-+#define HCLK_SFC			304
-+#define HCLK_SFC_XIP			305
-+#define HCLK_NVM_ROOT			306
-+#define ACLK_NVM_ROOT			307
-+#define CLK_GMAC0_PTP_REF		308
-+#define CLK_GMAC1_PTP_REF		309
-+#define CLK_GMAC_125M			310
-+#define CLK_GMAC_50M			311
-+#define ACLK_PHP_GIC_ITS		312
-+#define ACLK_MMU_PCIE			313
-+#define ACLK_MMU_PHP			314
-+#define ACLK_PCIE_4L_DBI		315
-+#define ACLK_PCIE_2L_DBI		316
-+#define ACLK_PCIE_1L0_DBI		317
-+#define ACLK_PCIE_1L1_DBI		318
-+#define ACLK_PCIE_1L2_DBI		319
-+#define ACLK_PCIE_4L_MSTR		320
-+#define ACLK_PCIE_2L_MSTR		321
-+#define ACLK_PCIE_1L0_MSTR		322
-+#define ACLK_PCIE_1L1_MSTR		323
-+#define ACLK_PCIE_1L2_MSTR		324
-+#define ACLK_PCIE_4L_SLV		325
-+#define ACLK_PCIE_2L_SLV		326
-+#define ACLK_PCIE_1L0_SLV		327
-+#define ACLK_PCIE_1L1_SLV		328
-+#define ACLK_PCIE_1L2_SLV		329
-+#define PCLK_PCIE_4L			330
-+#define PCLK_PCIE_2L			331
-+#define PCLK_PCIE_1L0			332
-+#define PCLK_PCIE_1L1			333
-+#define PCLK_PCIE_1L2			334
-+#define CLK_PCIE_AUX0			335
-+#define CLK_PCIE_AUX1			336
-+#define CLK_PCIE_AUX2			337
-+#define CLK_PCIE_AUX3			338
-+#define CLK_PCIE_AUX4			339
-+#define CLK_PIPEPHY0_REF		340
-+#define CLK_PIPEPHY1_REF		341
-+#define CLK_PIPEPHY2_REF		342
-+#define PCLK_PHP_ROOT			343
-+#define PCLK_GMAC0			344
-+#define PCLK_GMAC1			345
-+#define ACLK_PCIE_ROOT			346
-+#define ACLK_PHP_ROOT			347
-+#define ACLK_PCIE_BRIDGE		348
-+#define ACLK_GMAC0			349
-+#define ACLK_GMAC1			350
-+#define CLK_PMALIVE0			351
-+#define CLK_PMALIVE1			352
-+#define CLK_PMALIVE2			353
-+#define ACLK_SATA0			354
-+#define ACLK_SATA1			355
-+#define ACLK_SATA2			356
-+#define CLK_RXOOB0			357
-+#define CLK_RXOOB1			358
-+#define CLK_RXOOB2			359
-+#define ACLK_USB3OTG2			360
-+#define SUSPEND_CLK_USB3OTG2		361
-+#define REF_CLK_USB3OTG2		362
-+#define CLK_UTMI_OTG2			363
-+#define CLK_PIPEPHY0_PIPE_G		364
-+#define CLK_PIPEPHY1_PIPE_G		365
-+#define CLK_PIPEPHY2_PIPE_G		366
-+#define CLK_PIPEPHY0_PIPE_ASIC_G	367
-+#define CLK_PIPEPHY1_PIPE_ASIC_G	368
-+#define CLK_PIPEPHY2_PIPE_ASIC_G	369
-+#define CLK_PIPEPHY2_PIPE_U3_G		370
-+#define CLK_PCIE1L2_PIPE		371
-+#define CLK_PCIE4L_PIPE			372
-+#define CLK_PCIE2L_PIPE			373
-+#define PCLK_PCIE_COMBO_PIPE_PHY0	374
-+#define PCLK_PCIE_COMBO_PIPE_PHY1	375
-+#define PCLK_PCIE_COMBO_PIPE_PHY2	376
-+#define PCLK_PCIE_COMBO_PIPE_PHY	377
-+#define HCLK_RGA3_1			378
-+#define ACLK_RGA3_1			379
-+#define CLK_RGA3_1_CORE			380
-+#define ACLK_RGA3_ROOT			381
-+#define HCLK_RGA3_ROOT			382
-+#define ACLK_RKVDEC_CCU			383
-+#define HCLK_RKVDEC0			384
-+#define ACLK_RKVDEC0			385
-+#define CLK_RKVDEC0_CA			386
-+#define CLK_RKVDEC0_HEVC_CA		387
-+#define CLK_RKVDEC0_CORE		388
-+#define HCLK_RKVDEC1			389
-+#define ACLK_RKVDEC1			390
-+#define CLK_RKVDEC1_CA			391
-+#define CLK_RKVDEC1_HEVC_CA		392
-+#define CLK_RKVDEC1_CORE		393
-+#define HCLK_SDIO			394
-+#define CCLK_SRC_SDIO			395
-+#define ACLK_USB_ROOT			396
-+#define HCLK_USB_ROOT			397
-+#define HCLK_HOST0			398
-+#define HCLK_HOST_ARB0			399
-+#define HCLK_HOST1			400
-+#define HCLK_HOST_ARB1			401
-+#define ACLK_USB3OTG0			402
-+#define SUSPEND_CLK_USB3OTG0		403
-+#define REF_CLK_USB3OTG0		404
-+#define ACLK_USB3OTG1			405
-+#define SUSPEND_CLK_USB3OTG1		406
-+#define REF_CLK_USB3OTG1		407
-+#define UTMI_OHCI_CLK48_HOST0		408
-+#define UTMI_OHCI_CLK48_HOST1		409
-+#define HCLK_IEP2P0			410
-+#define ACLK_IEP2P0			411
-+#define CLK_IEP2P0_CORE			412
-+#define ACLK_JPEG_ENCODER0		413
-+#define HCLK_JPEG_ENCODER0		414
-+#define ACLK_JPEG_ENCODER1		415
-+#define HCLK_JPEG_ENCODER1		416
-+#define ACLK_JPEG_ENCODER2		417
-+#define HCLK_JPEG_ENCODER2		418
-+#define ACLK_JPEG_ENCODER3		419
-+#define HCLK_JPEG_ENCODER3		420
-+#define ACLK_JPEG_DECODER		421
-+#define HCLK_JPEG_DECODER		422
-+#define HCLK_RGA2			423
-+#define ACLK_RGA2			424
-+#define CLK_RGA2_CORE			425
-+#define HCLK_RGA3_0			426
-+#define ACLK_RGA3_0			427
-+#define CLK_RGA3_0_CORE			428
-+#define ACLK_VDPU_ROOT			429
-+#define ACLK_VDPU_LOW_ROOT		430
-+#define HCLK_VDPU_ROOT			431
-+#define ACLK_JPEG_DECODER_ROOT		432
-+#define ACLK_VPU			433
-+#define HCLK_VPU			434
-+#define HCLK_RKVENC0_ROOT		435
-+#define ACLK_RKVENC0_ROOT		436
-+#define HCLK_RKVENC0			437
-+#define ACLK_RKVENC0			438
-+#define CLK_RKVENC0_CORE		439
-+#define HCLK_RKVENC1_ROOT		440
-+#define ACLK_RKVENC1_ROOT		441
-+#define HCLK_RKVENC1			442
-+#define ACLK_RKVENC1			443
-+#define CLK_RKVENC1_CORE		444
-+#define ICLK_CSIHOST01			445
-+#define ICLK_CSIHOST0			446
-+#define ICLK_CSIHOST1			447
-+#define PCLK_CSI_HOST_0			448
-+#define PCLK_CSI_HOST_1			449
-+#define PCLK_CSI_HOST_2			450
-+#define PCLK_CSI_HOST_3			451
-+#define PCLK_CSI_HOST_4			452
-+#define PCLK_CSI_HOST_5			453
-+#define ACLK_FISHEYE0			454
-+#define HCLK_FISHEYE0			455
-+#define CLK_FISHEYE0_CORE		456
-+#define ACLK_FISHEYE1			457
-+#define HCLK_FISHEYE1			458
-+#define CLK_FISHEYE1_CORE		459
-+#define CLK_ISP0_CORE			460
-+#define CLK_ISP0_CORE_MARVIN		461
-+#define CLK_ISP0_CORE_VICAP		462
-+#define ACLK_ISP0			463
-+#define HCLK_ISP0			464
-+#define ACLK_VI_ROOT			465
-+#define HCLK_VI_ROOT			466
-+#define PCLK_VI_ROOT			467
-+#define DCLK_VICAP			468
-+#define ACLK_VICAP			469
-+#define HCLK_VICAP			470
-+#define PCLK_DP0			471
-+#define PCLK_DP1			472
-+#define PCLK_S_DP0			473
-+#define PCLK_S_DP1			474
-+#define CLK_DP0				475
-+#define CLK_DP1				476
-+#define HCLK_HDCP_KEY0			477
-+#define ACLK_HDCP0			478
-+#define HCLK_HDCP0			479
-+#define PCLK_HDCP0			480
-+#define HCLK_I2S4_8CH			481
-+#define ACLK_TRNG0			482
-+#define PCLK_TRNG0			483
-+#define ACLK_VO0_ROOT			484
-+#define HCLK_VO0_ROOT			485
-+#define HCLK_VO0_S_ROOT			486
-+#define PCLK_VO0_ROOT			487
-+#define PCLK_VO0_S_ROOT			488
-+#define PCLK_VO0GRF			489
-+#define CLK_I2S4_8CH_TX_SRC		490
-+#define CLK_I2S4_8CH_TX_FRAC		491
-+#define MCLK_I2S4_8CH_TX		492
-+#define CLK_I2S4_8CH_TX			493
-+#define HCLK_I2S8_8CH			494
-+#define CLK_I2S8_8CH_TX_SRC		495
-+#define CLK_I2S8_8CH_TX_FRAC		496
-+#define MCLK_I2S8_8CH_TX		497
-+#define CLK_I2S8_8CH_TX			498
-+#define HCLK_SPDIF2_DP0			499
-+#define CLK_SPDIF2_DP0_SRC		500
-+#define CLK_SPDIF2_DP0_FRAC		501
-+#define MCLK_SPDIF2_DP0			502
-+#define CLK_SPDIF2_DP0			503
-+#define MCLK_SPDIF2			504
-+#define HCLK_SPDIF5_DP1			505
-+#define CLK_SPDIF5_DP1_SRC		506
-+#define CLK_SPDIF5_DP1_FRAC		507
-+#define MCLK_SPDIF5_DP1			508
-+#define CLK_SPDIF5_DP1			509
-+#define MCLK_SPDIF5			510
-+#define PCLK_EDP0			511
-+#define CLK_EDP0_24M			512
-+#define CLK_EDP0_200M			513
-+#define PCLK_EDP1			514
-+#define CLK_EDP1_24M			515
-+#define CLK_EDP1_200M			516
-+#define HCLK_HDCP_KEY1			517
-+#define ACLK_HDCP1			518
-+#define HCLK_HDCP1			519
-+#define PCLK_HDCP1			520
-+#define ACLK_HDMIRX			521
-+#define PCLK_HDMIRX			522
-+#define CLK_HDMIRX_REF			523
-+#define CLK_HDMIRX_AUD_SRC		524
-+#define CLK_HDMIRX_AUD_FRAC		525
-+#define CLK_HDMIRX_AUD			526
-+#define CLK_HDMIRX_AUD_P_MUX		527
-+#define PCLK_HDMITX0			528
-+#define CLK_HDMITX0_EARC		529
-+#define CLK_HDMITX0_REF			530
-+#define PCLK_HDMITX1			531
-+#define CLK_HDMITX1_EARC		532
-+#define CLK_HDMITX1_REF			533
-+#define CLK_HDMITRX_REFSRC		534
-+#define ACLK_TRNG1			535
-+#define PCLK_TRNG1			536
-+#define ACLK_HDCP1_ROOT			537
-+#define ACLK_HDMIRX_ROOT		538
-+#define HCLK_VO1_ROOT			539
-+#define HCLK_VO1_S_ROOT			540
-+#define PCLK_VO1_ROOT			541
-+#define PCLK_VO1_S_ROOT			542
-+#define PCLK_S_EDP0			543
-+#define PCLK_S_EDP1			544
-+#define PCLK_S_HDMIRX			545
-+#define HCLK_I2S10_8CH			546
-+#define CLK_I2S10_8CH_RX_SRC		547
-+#define CLK_I2S10_8CH_RX_FRAC		548
-+#define CLK_I2S10_8CH_RX		549
-+#define MCLK_I2S10_8CH_RX		550
-+#define HCLK_I2S7_8CH			551
-+#define CLK_I2S7_8CH_RX_SRC		552
-+#define CLK_I2S7_8CH_RX_FRAC		553
-+#define CLK_I2S7_8CH_RX			554
-+#define MCLK_I2S7_8CH_RX		555
-+#define HCLK_I2S9_8CH			556
-+#define CLK_I2S9_8CH_RX_SRC		557
-+#define CLK_I2S9_8CH_RX_FRAC		558
-+#define CLK_I2S9_8CH_RX			559
-+#define MCLK_I2S9_8CH_RX		560
-+#define CLK_I2S5_8CH_TX_SRC		561
-+#define CLK_I2S5_8CH_TX_FRAC		562
-+#define CLK_I2S5_8CH_TX			563
-+#define MCLK_I2S5_8CH_TX		564
-+#define HCLK_I2S5_8CH			565
-+#define CLK_I2S6_8CH_TX_SRC		566
-+#define CLK_I2S6_8CH_TX_FRAC		567
-+#define CLK_I2S6_8CH_TX			568
-+#define MCLK_I2S6_8CH_TX		569
-+#define CLK_I2S6_8CH_RX_SRC		570
-+#define CLK_I2S6_8CH_RX_FRAC		571
-+#define CLK_I2S6_8CH_RX			572
-+#define MCLK_I2S6_8CH_RX		573
-+#define I2S6_8CH_MCLKOUT		574
-+#define HCLK_I2S6_8CH			575
-+#define HCLK_SPDIF3			576
-+#define CLK_SPDIF3_SRC			577
-+#define CLK_SPDIF3_FRAC			578
-+#define CLK_SPDIF3			579
-+#define MCLK_SPDIF3			580
-+#define HCLK_SPDIF4			581
-+#define CLK_SPDIF4_SRC			582
-+#define CLK_SPDIF4_FRAC			583
-+#define CLK_SPDIF4			584
-+#define MCLK_SPDIF4			585
-+#define HCLK_SPDIFRX0			586
-+#define MCLK_SPDIFRX0			587
-+#define HCLK_SPDIFRX1			588
-+#define MCLK_SPDIFRX1			589
-+#define HCLK_SPDIFRX2			590
-+#define MCLK_SPDIFRX2			591
-+#define ACLK_VO1USB_TOP_ROOT		592
-+#define HCLK_VO1USB_TOP_ROOT		593
-+#define CLK_HDMIHDP0			594
-+#define CLK_HDMIHDP1			595
-+#define PCLK_HDPTX0			596
-+#define PCLK_HDPTX1			597
-+#define PCLK_USBDPPHY0			598
-+#define PCLK_USBDPPHY1			599
-+#define ACLK_VOP_ROOT			600
-+#define ACLK_VOP_LOW_ROOT		601
-+#define HCLK_VOP_ROOT			602
-+#define PCLK_VOP_ROOT			603
-+#define HCLK_VOP			604
-+#define ACLK_VOP			605
-+#define DCLK_VOP0_SRC			606
-+#define DCLK_VOP1_SRC			607
-+#define DCLK_VOP2_SRC			608
-+#define DCLK_VOP0			609
-+#define DCLK_VOP1			610
-+#define DCLK_VOP2			611
-+#define DCLK_VOP3			612
-+#define PCLK_DSIHOST0			613
-+#define PCLK_DSIHOST1			614
-+#define CLK_DSIHOST0			615
-+#define CLK_DSIHOST1			616
-+#define CLK_VOP_PMU			617
-+#define ACLK_VOP_DOBY			618
-+#define ACLK_VOP_SUB_SRC		619
-+#define CLK_USBDP_PHY0_IMMORTAL		620
-+#define CLK_USBDP_PHY1_IMMORTAL		621
-+#define CLK_PMU0			622
-+#define PCLK_PMU0			623
-+#define PCLK_PMU0IOC			624
-+#define PCLK_GPIO0			625
-+#define DBCLK_GPIO0			626
-+#define PCLK_I2C0			627
-+#define CLK_I2C0			628
-+#define HCLK_I2S1_8CH			629
-+#define CLK_I2S1_8CH_TX_SRC		630
-+#define CLK_I2S1_8CH_TX_FRAC		631
-+#define CLK_I2S1_8CH_TX			632
-+#define MCLK_I2S1_8CH_TX		633
-+#define CLK_I2S1_8CH_RX_SRC		634
-+#define CLK_I2S1_8CH_RX_FRAC		635
-+#define CLK_I2S1_8CH_RX			636
-+#define MCLK_I2S1_8CH_RX		637
-+#define I2S1_8CH_MCLKOUT		638
-+#define CLK_PMU1_50M_SRC		639
-+#define CLK_PMU1_100M_SRC		640
-+#define CLK_PMU1_200M_SRC		641
-+#define CLK_PMU1_300M_SRC		642
-+#define CLK_PMU1_400M_SRC		643
-+#define HCLK_PMU1_ROOT			644
-+#define PCLK_PMU1_ROOT			645
-+#define PCLK_PMU0_ROOT			646
-+#define HCLK_PMU_CM0_ROOT		647
-+#define PCLK_PMU1			648
-+#define CLK_DDR_FAIL_SAFE		649
-+#define CLK_PMU1			650
-+#define HCLK_PDM0			651
-+#define MCLK_PDM0			652
-+#define HCLK_VAD			653
-+#define FCLK_PMU_CM0_CORE		654
-+#define CLK_PMU_CM0_RTC			655
-+#define PCLK_PMU1_IOC			656
-+#define PCLK_PMU1PWM			657
-+#define CLK_PMU1PWM			658
-+#define CLK_PMU1PWM_CAPTURE		659
-+#define PCLK_PMU1TIMER			660
-+#define CLK_PMU1TIMER_ROOT		661
-+#define CLK_PMU1TIMER0			662
-+#define CLK_PMU1TIMER1			663
-+#define CLK_UART0_SRC			664
-+#define CLK_UART0_FRAC			665
-+#define CLK_UART0			666
-+#define SCLK_UART0			667
-+#define PCLK_UART0			668
-+#define PCLK_PMU1WDT			669
-+#define TCLK_PMU1WDT			670
-+#define CLK_CR_PARA			671
-+#define CLK_USB2PHY_HDPTXRXPHY_REF	672
-+#define CLK_USBDPPHY_MIPIDCPPHY_REF	673
-+#define CLK_REF_PIPE_PHY0_OSC_SRC	674
-+#define CLK_REF_PIPE_PHY1_OSC_SRC	675
-+#define CLK_REF_PIPE_PHY2_OSC_SRC	676
-+#define CLK_REF_PIPE_PHY0_PLL_SRC	677
-+#define CLK_REF_PIPE_PHY1_PLL_SRC	678
-+#define CLK_REF_PIPE_PHY2_PLL_SRC	679
-+#define CLK_REF_PIPE_PHY0		680
-+#define CLK_REF_PIPE_PHY1		681
-+#define CLK_REF_PIPE_PHY2		682
-+#define SCLK_SDIO_DRV			683
-+#define SCLK_SDIO_SAMPLE		684
-+#define SCLK_SDMMC_DRV			685
-+#define SCLK_SDMMC_SAMPLE		686
-+#define CLK_PCIE1L0_PIPE		687
-+#define CLK_PCIE1L1_PIPE		688
-+#define CLK_BIGCORE0_PVTM		689
-+#define CLK_CORE_BIGCORE0_PVTM		690
-+#define CLK_BIGCORE1_PVTM		691
-+#define CLK_CORE_BIGCORE1_PVTM		692
-+#define CLK_LITCORE_PVTM		693
-+#define CLK_CORE_LITCORE_PVTM		694
-+#define CLK_AUX16M_0			695
-+#define CLK_AUX16M_1			696
-+#define CLK_PHY0_REF_ALT_P		697
-+#define CLK_PHY0_REF_ALT_M		698
-+#define CLK_PHY1_REF_ALT_P		699
-+#define CLK_PHY1_REF_ALT_M		700
-+#define ACLK_ISP1_PRE			701
-+#define HCLK_ISP1_PRE			702
-+#define HCLK_NVM			703
-+#define ACLK_USB			704
-+#define HCLK_USB			705
-+#define ACLK_JPEG_DECODER_PRE		706
-+#define ACLK_VDPU_LOW_PRE		707
-+#define ACLK_RKVENC1_PRE		708
-+#define HCLK_RKVENC1_PRE		709
-+#define HCLK_RKVDEC0_PRE		710
-+#define ACLK_RKVDEC0_PRE		711
-+#define HCLK_RKVDEC1_PRE		712
-+#define ACLK_RKVDEC1_PRE		713
-+#define ACLK_HDCP0_PRE			714
-+#define HCLK_VO0			715
-+#define ACLK_HDCP1_PRE			716
-+#define HCLK_VO1			717
-+#define ACLK_AV1_PRE			718
-+#define PCLK_AV1_PRE			719
-+#define HCLK_SDIO_PRE			720
-+#define PCLK_VO1GRF			721
-+
-+/* scmi-clocks indices */
-+
-+#define SCMI_CLK_CPUL			0
-+#define SCMI_CLK_DSU			1
-+#define SCMI_CLK_CPUB01			2
-+#define SCMI_CLK_CPUB23			3
-+#define SCMI_CLK_DDR			4
-+#define SCMI_CLK_GPU			5
-+#define SCMI_CLK_NPU			6
-+#define SCMI_CLK_SBUS			7
-+#define SCMI_PCLK_SBUS			8
-+#define SCMI_CCLK_SD			9
-+#define SCMI_DCLK_SD			10
-+#define SCMI_ACLK_SECURE_NS		11
-+#define SCMI_HCLK_SECURE_NS		12
-+#define SCMI_TCLK_WDT			13
-+#define SCMI_KEYLADDER_CORE		14
-+#define SCMI_KEYLADDER_RNG		15
-+#define SCMI_ACLK_SECURE_S		16
-+#define SCMI_HCLK_SECURE_S		17
-+#define SCMI_PCLK_SECURE_S		18
-+#define SCMI_CRYPTO_RNG			19
-+#define SCMI_CRYPTO_CORE		20
-+#define SCMI_CRYPTO_PKA			21
-+#define SCMI_SPLL			22
-+#define SCMI_HCLK_SD			23
-+
-+#endif
-diff --git a/include/dt-bindings/clock/rockchip,rv1126-cru.h b/include/dt-bindings/clock/rockchip,rv1126-cru.h
-new file mode 100644
-index 000000000000..e89a3a5a4a34
--- /dev/null
-+++ b/include/dt-bindings/clock/rockchip,rv1126-cru.h
-@@ -0,0 +1,632 @@
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
-+/*
-+ * Copyright (c) 2019 Rockchip Electronics Co. Ltd.
-+ * Author: Finley Xiao <finley.xiao@rock-chips.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_ROCKCHIP_RV1126_H
-+#define _DT_BINDINGS_CLK_ROCKCHIP_RV1126_H
-+
-+/* pmucru-clocks indices */
-+
-+/* pll clocks */
-+#define PLL_GPLL		1
-+
-+/* sclk (special clocks) */
-+#define CLK_OSC0_DIV32K		2
-+#define CLK_RTC32K		3
-+#define CLK_WIFI_DIV		4
-+#define CLK_WIFI_OSC0		5
-+#define CLK_WIFI		6
-+#define CLK_PMU			7
-+#define SCLK_UART1_DIV		8
-+#define SCLK_UART1_FRACDIV	9
-+#define SCLK_UART1_MUX		10
-+#define SCLK_UART1		11
-+#define CLK_I2C0		12
-+#define CLK_I2C2		13
-+#define CLK_CAPTURE_PWM0	14
-+#define CLK_PWM0		15
-+#define CLK_CAPTURE_PWM1	16
-+#define CLK_PWM1		17
-+#define CLK_SPI0		18
-+#define DBCLK_GPIO0		19
-+#define CLK_PMUPVTM		20
-+#define CLK_CORE_PMUPVTM	21
-+#define CLK_REF12M		22
-+#define CLK_USBPHY_OTG_REF	23
-+#define CLK_USBPHY_HOST_REF	24
-+#define CLK_REF24M		25
-+#define CLK_MIPIDSIPHY_REF	26
-+
-+/* pclk */
-+#define PCLK_PDPMU		30
-+#define PCLK_PMU		31
-+#define PCLK_UART1		32
-+#define PCLK_I2C0		33
-+#define PCLK_I2C2		34
-+#define PCLK_PWM0		35
-+#define PCLK_PWM1		36
-+#define PCLK_SPI0		37
-+#define PCLK_GPIO0		38
-+#define PCLK_PMUSGRF		39
-+#define PCLK_PMUGRF		40
-+#define PCLK_PMUCRU		41
-+#define PCLK_CHIPVEROTP		42
-+#define PCLK_PDPMU_NIU		43
-+#define PCLK_PMUPVTM		44
-+#define PCLK_SCRKEYGEN		45
-+
-+#define CLKPMU_NR_CLKS		(PCLK_SCRKEYGEN + 1)
-+
-+/* cru-clocks indices */
-+
-+/* pll clocks */
-+#define PLL_APLL		1
-+#define PLL_DPLL		2
-+#define PLL_CPLL		3
-+#define PLL_HPLL		4
-+
-+/* sclk (special clocks) */
-+#define ARMCLK			5
-+#define USB480M			6
-+#define CLK_CORE_CPUPVTM	7
-+#define CLK_CPUPVTM		8
-+#define CLK_SCR1		9
-+#define CLK_SCR1_CORE		10
-+#define CLK_SCR1_RTC		11
-+#define CLK_SCR1_JTAG		12
-+#define SCLK_UART0_DIV		13
-+#define SCLK_UART0_FRAC		14
-+#define SCLK_UART0_MUX		15
-+#define SCLK_UART0		16
-+#define SCLK_UART2_DIV		17
-+#define SCLK_UART2_FRAC		18
-+#define SCLK_UART2_MUX		19
-+#define SCLK_UART2		20
-+#define SCLK_UART3_DIV		21
-+#define SCLK_UART3_FRAC		22
-+#define SCLK_UART3_MUX		23
-+#define SCLK_UART3		24
-+#define SCLK_UART4_DIV		25
-+#define SCLK_UART4_FRAC		26
-+#define SCLK_UART4_MUX		27
-+#define SCLK_UART4		28
-+#define SCLK_UART5_DIV		29
-+#define SCLK_UART5_FRAC		30
-+#define SCLK_UART5_MUX		31
-+#define SCLK_UART5		32
-+#define CLK_I2C1		33
-+#define CLK_I2C3		34
-+#define CLK_I2C4		35
-+#define CLK_I2C5		36
-+#define CLK_SPI1		37
-+#define CLK_CAPTURE_PWM2	38
-+#define CLK_PWM2		39
-+#define DBCLK_GPIO1		40
-+#define DBCLK_GPIO2		41
-+#define DBCLK_GPIO3		42
-+#define DBCLK_GPIO4		43
-+#define CLK_SARADC		44
-+#define CLK_TIMER0		45
-+#define CLK_TIMER1		46
-+#define CLK_TIMER2		47
-+#define CLK_TIMER3		48
-+#define CLK_TIMER4		49
-+#define CLK_TIMER5		50
-+#define CLK_CAN			51
-+#define CLK_NPU_TSADC		52
-+#define CLK_NPU_TSADCPHY	53
-+#define CLK_CPU_TSADC		54
-+#define CLK_CPU_TSADCPHY	55
-+#define CLK_CRYPTO_CORE		56
-+#define CLK_CRYPTO_PKA		57
-+#define MCLK_I2S0_TX_DIV	58
-+#define MCLK_I2S0_TX_FRACDIV	59
-+#define MCLK_I2S0_TX_MUX	60
-+#define MCLK_I2S0_TX		61
-+#define MCLK_I2S0_RX_DIV	62
-+#define MCLK_I2S0_RX_FRACDIV	63
-+#define MCLK_I2S0_RX_MUX	64
-+#define MCLK_I2S0_RX		65
-+#define MCLK_I2S0_TX_OUT2IO	66
-+#define MCLK_I2S0_RX_OUT2IO	67
-+#define MCLK_I2S1_DIV		68
-+#define MCLK_I2S1_FRACDIV	69
-+#define MCLK_I2S1_MUX		70
-+#define MCLK_I2S1		71
-+#define MCLK_I2S1_OUT2IO	72
-+#define MCLK_I2S2_DIV		73
-+#define MCLK_I2S2_FRACDIV	74
-+#define MCLK_I2S2_MUX		75
-+#define MCLK_I2S2		76
-+#define MCLK_I2S2_OUT2IO	77
-+#define MCLK_PDM		78
-+#define SCLK_ADUPWM_DIV		79
-+#define SCLK_AUDPWM_FRACDIV	80
-+#define SCLK_AUDPWM_MUX		81
-+#define	SCLK_AUDPWM		82
-+#define CLK_ACDCDIG_ADC		83
-+#define CLK_ACDCDIG_DAC		84
-+#define CLK_ACDCDIG_I2C		85
-+#define CLK_VENC_CORE		86
-+#define CLK_VDEC_CORE		87
-+#define CLK_VDEC_CA		88
-+#define CLK_VDEC_HEVC_CA	89
-+#define CLK_RGA_CORE		90
-+#define CLK_IEP_CORE		91
-+#define CLK_ISP_DIV		92
-+#define CLK_ISP_NP5		93
-+#define CLK_ISP_NUX		94
-+#define CLK_ISP			95
-+#define CLK_CIF_OUT_DIV		96
-+#define CLK_CIF_OUT_FRACDIV	97
-+#define CLK_CIF_OUT_MUX		98
-+#define CLK_CIF_OUT		99
-+#define CLK_MIPICSI_OUT_DIV	100
-+#define CLK_MIPICSI_OUT_FRACDIV	101
-+#define CLK_MIPICSI_OUT_MUX	102
-+#define CLK_MIPICSI_OUT		103
-+#define CLK_ISPP_DIV		104
-+#define CLK_ISPP_NP5		105
-+#define CLK_ISPP_NUX		106
-+#define CLK_ISPP		107
-+#define CLK_SDMMC		108
-+#define SCLK_SDMMC_DRV		109
-+#define SCLK_SDMMC_SAMPLE	110
-+#define CLK_SDIO		111
-+#define SCLK_SDIO_DRV		112
-+#define SCLK_SDIO_SAMPLE	113
-+#define CLK_EMMC		114
-+#define SCLK_EMMC_DRV		115
-+#define SCLK_EMMC_SAMPLE	116
-+#define CLK_NANDC		117
-+#define SCLK_SFC		118
-+#define CLK_USBHOST_UTMI_OHCI	119
-+#define CLK_USBOTG_REF		120
-+#define CLK_GMAC_DIV		121
-+#define CLK_GMAC_RGMII_M0	122
-+#define CLK_GMAC_SRC_M0		123
-+#define CLK_GMAC_RGMII_M1	124
-+#define CLK_GMAC_SRC_M1		125
-+#define CLK_GMAC_SRC		126
-+#define CLK_GMAC_REF		127
-+#define CLK_GMAC_TX_SRC		128
-+#define CLK_GMAC_TX_DIV5	129
-+#define CLK_GMAC_TX_DIV50	130
-+#define RGMII_MODE_CLK		131
-+#define CLK_GMAC_RX_SRC		132
-+#define CLK_GMAC_RX_DIV2	133
-+#define CLK_GMAC_RX_DIV20	134
-+#define RMII_MODE_CLK		135
-+#define CLK_GMAC_TX_RX		136
-+#define CLK_GMAC_PTPREF		137
-+#define CLK_GMAC_ETHERNET_OUT	138
-+#define CLK_DDRPHY		139
-+#define CLK_DDR_MON		140
-+#define TMCLK_DDR_MON		141
-+#define CLK_NPU_DIV		142
-+#define CLK_NPU_NP5		143
-+#define CLK_CORE_NPU		144
-+#define CLK_CORE_NPUPVTM	145
-+#define CLK_NPUPVTM		146
-+#define SCLK_DDRCLK		147
-+#define CLK_OTP			148
-+
-+/* dclk */
-+#define DCLK_DECOM		150
-+#define DCLK_VOP_DIV		151
-+#define DCLK_VOP_FRACDIV	152
-+#define DCLK_VOP_MUX		153
-+#define DCLK_VOP		154
-+#define DCLK_CIF		155
-+#define DCLK_CIFLITE		156
-+
-+/* aclk */
-+#define ACLK_PDBUS		160
-+#define ACLK_DMAC		161
-+#define ACLK_DCF		162
-+#define ACLK_SPINLOCK		163
-+#define ACLK_DECOM		164
-+#define ACLK_PDCRYPTO		165
-+#define ACLK_CRYPTO		166
-+#define ACLK_PDVEPU		167
-+#define ACLK_VENC		168
-+#define ACLK_PDVDEC		169
-+#define ACLK_PDJPEG		170
-+#define ACLK_VDEC		171
-+#define ACLK_JPEG		172
-+#define ACLK_PDVO		173
-+#define ACLK_RGA		174
-+#define ACLK_VOP		175
-+#define ACLK_IEP		176
-+#define ACLK_PDVI_DIV		177
-+#define ACLK_PDVI_NP5		178
-+#define ACLK_PDVI		179
-+#define ACLK_ISP		180
-+#define ACLK_CIF		181
-+#define ACLK_CIFLITE		182
-+#define ACLK_PDISPP_DIV		183
-+#define ACLK_PDISPP_NP5		184
-+#define ACLK_PDISPP		185
-+#define ACLK_ISPP		186
-+#define ACLK_PDPHP		187
-+#define ACLK_PDUSB		188
-+#define ACLK_USBOTG		189
-+#define ACLK_PDGMAC		190
-+#define ACLK_GMAC		191
-+#define ACLK_PDNPU_DIV		192
-+#define ACLK_PDNPU_NP5		193
-+#define ACLK_PDNPU		194
-+#define ACLK_NPU		195
-+
-+/* hclk */
-+#define HCLK_PDCORE_NIU		200
-+#define HCLK_PDUSB		201
-+#define HCLK_PDCRYPTO		202
-+#define HCLK_CRYPTO		203
-+#define HCLK_PDAUDIO		204
-+#define HCLK_I2S0		205
-+#define HCLK_I2S1		206
-+#define HCLK_I2S2		207
-+#define HCLK_PDM		208
-+#define HCLK_AUDPWM		209
-+#define HCLK_PDVEPU		210
-+#define HCLK_VENC		211
-+#define HCLK_PDVDEC		212
-+#define HCLK_PDJPEG		213
-+#define HCLK_VDEC		214
-+#define HCLK_JPEG		215
-+#define HCLK_PDVO		216
-+#define HCLK_RGA		217
-+#define HCLK_VOP		218
-+#define HCLK_IEP		219
-+#define HCLK_PDVI		220
-+#define HCLK_ISP		221
-+#define HCLK_CIF		222
-+#define HCLK_CIFLITE		223
-+#define HCLK_PDISPP		224
-+#define HCLK_ISPP		225
-+#define HCLK_PDPHP		226
-+#define HCLK_PDSDMMC		227
-+#define HCLK_SDMMC		228
-+#define HCLK_PDSDIO		229
-+#define HCLK_SDIO		230
-+#define HCLK_PDNVM		231
-+#define HCLK_EMMC		232
-+#define HCLK_NANDC		233
-+#define HCLK_SFC		234
-+#define HCLK_SFCXIP		235
-+#define HCLK_PDBUS		236
-+#define HCLK_USBHOST		237
-+#define HCLK_USBHOST_ARB	238
-+#define HCLK_PDNPU		239
-+#define HCLK_NPU		240
-+
-+/* pclk */
-+#define PCLK_CPUPVTM		245
-+#define PCLK_PDBUS		246
-+#define PCLK_DCF		247
-+#define PCLK_WDT		248
-+#define PCLK_MAILBOX		249
-+#define PCLK_UART0		250
-+#define PCLK_UART2		251
-+#define PCLK_UART3		252
-+#define PCLK_UART4		253
-+#define PCLK_UART5		254
-+#define PCLK_I2C1		255
-+#define PCLK_I2C3		256
-+#define PCLK_I2C4		257
-+#define PCLK_I2C5		258
-+#define PCLK_SPI1		259
-+#define PCLK_PWM2		261
-+#define PCLK_GPIO1		262
-+#define PCLK_GPIO2		263
-+#define PCLK_GPIO3		264
-+#define PCLK_GPIO4		265
-+#define PCLK_SARADC		266
-+#define PCLK_TIMER		267
-+#define PCLK_DECOM		268
-+#define PCLK_CAN		269
-+#define PCLK_NPU_TSADC		270
-+#define PCLK_CPU_TSADC		271
-+#define PCLK_ACDCDIG		272
-+#define PCLK_PDVO		273
-+#define PCLK_DSIHOST		274
-+#define PCLK_PDVI		275
-+#define PCLK_CSIHOST		276
-+#define PCLK_PDGMAC		277
-+#define PCLK_GMAC		278
-+#define PCLK_PDDDR		279
-+#define PCLK_DDR_MON		280
-+#define PCLK_PDNPU		281
-+#define PCLK_NPUPVTM		282
-+#define PCLK_PDTOP		283
-+#define PCLK_TOPCRU		284
-+#define PCLK_TOPGRF		285
-+#define PCLK_CPUEMADET		286
-+#define PCLK_DDRPHY		287
-+#define PCLK_DSIPHY		289
-+#define PCLK_CSIPHY0		290
-+#define PCLK_CSIPHY1		291
-+#define PCLK_USBPHY_HOST	292
-+#define PCLK_USBPHY_OTG		293
-+#define PCLK_OTP		294
-+
-+#define CLK_NR_CLKS		(PCLK_OTP + 1)
-+
-+/* pmu soft-reset indices */
-+
-+/* pmu_cru_softrst_con0 */
-+#define SRST_PDPMU_NIU_P	0
-+#define SRST_PMU_SGRF_P		1
-+#define SRST_PMU_SGRF_REMAP_P	2
-+#define SRST_I2C0_P		3
-+#define SRST_I2C0		4
-+#define SRST_I2C2_P		7
-+#define SRST_I2C2		8
-+#define SRST_UART1_P		9
-+#define SRST_UART1		10
-+#define SRST_PWM0_P		11
-+#define SRST_PWM0		12
-+#define SRST_PWM1_P		13
-+#define SRST_PWM1		14
-+#define SRST_DDR_FAIL_SAFE	15
-+
-+/* pmu_cru_softrst_con1 */
-+#define SRST_GPIO0_P		17
-+#define SRST_GPIO0_DB		18
-+#define SRST_SPI0_P		19
-+#define SRST_SPI0		20
-+#define SRST_PMUGRF_P		21
-+#define SRST_CHIPVEROTP_P	22
-+#define SRST_PMUPVTM		24
-+#define SRST_PMUPVTM_P		25
-+#define SRST_PMUCRU_P		30
-+
-+/* soft-reset indices */
-+
-+/* cru_softrst_con0 */
-+#define SRST_CORE0_PO		0
-+#define SRST_CORE1_PO		1
-+#define SRST_CORE2_PO		2
-+#define SRST_CORE3_PO		3
-+#define SRST_CORE0		4
-+#define SRST_CORE1		5
-+#define SRST_CORE2		6
-+#define SRST_CORE3		7
-+#define SRST_CORE0_DBG		8
-+#define SRST_CORE1_DBG		9
-+#define SRST_CORE2_DBG		10
-+#define SRST_CORE3_DBG		11
-+#define SRST_NL2		12
-+#define SRST_CORE_NIU_A		13
-+#define SRST_DBG_DAPLITE_P	14
-+#define SRST_DAPLITE_P		15
-+
-+/* cru_softrst_con1 */
-+#define SRST_PDBUS_NIU1_A	16
-+#define SRST_PDBUS_NIU1_H	17
-+#define SRST_PDBUS_NIU1_P	18
-+#define SRST_PDBUS_NIU2_A	19
-+#define SRST_PDBUS_NIU2_H	20
-+#define SRST_PDBUS_NIU3_A	21
-+#define SRST_PDBUS_NIU3_H	22
-+#define SRST_PDBUS_HOLD_NIU1_A	23
-+#define SRST_DBG_NIU_P		24
-+#define SRST_PDCORE_NIIU_H	25
-+#define SRST_MUC_NIU		26
-+#define SRST_DCF_A		29
-+#define SRST_DCF_P		30
-+#define SRST_SYSTEM_SRAM_A	31
-+
-+/* cru_softrst_con2 */
-+#define SRST_I2C1_P		32
-+#define SRST_I2C1		33
-+#define SRST_I2C3_P		34
-+#define SRST_I2C3		35
-+#define SRST_I2C4_P		36
-+#define SRST_I2C4		37
-+#define SRST_I2C5_P		38
-+#define SRST_I2C5		39
-+#define SRST_SPI1_P		40
-+#define SRST_SPI1		41
-+#define SRST_MCU_CORE		42
-+#define SRST_PWM2_P		44
-+#define SRST_PWM2		45
-+#define SRST_SPINLOCK_A		46
-+
-+/* cru_softrst_con3 */
-+#define SRST_UART0_P		48
-+#define SRST_UART0		49
-+#define SRST_UART2_P		50
-+#define SRST_UART2		51
-+#define SRST_UART3_P		52
-+#define SRST_UART3		53
-+#define SRST_UART4_P		54
-+#define SRST_UART4		55
-+#define SRST_UART5_P		56
-+#define SRST_UART5		57
-+#define SRST_WDT_P		58
-+#define SRST_SARADC_P		59
-+#define SRST_GRF_P		61
-+#define SRST_TIMER_P		62
-+#define SRST_MAILBOX_P		63
-+
-+/* cru_softrst_con4 */
-+#define SRST_TIMER0		64
-+#define SRST_TIMER1		65
-+#define SRST_TIMER2		66
-+#define SRST_TIMER3		67
-+#define SRST_TIMER4		68
-+#define SRST_TIMER5		69
-+#define SRST_INTMUX_P		70
-+#define SRST_GPIO1_P		72
-+#define SRST_GPIO1_DB		73
-+#define SRST_GPIO2_P		74
-+#define SRST_GPIO2_DB		75
-+#define SRST_GPIO3_P		76
-+#define SRST_GPIO3_DB		77
-+#define SRST_GPIO4_P		78
-+#define SRST_GPIO4_DB		79
-+
-+/* cru_softrst_con5 */
-+#define SRST_CAN_P		80
-+#define SRST_CAN		81
-+#define SRST_DECOM_A		85
-+#define SRST_DECOM_P		86
-+#define SRST_DECOM_D		87
-+#define SRST_PDCRYPTO_NIU_A	88
-+#define SRST_PDCRYPTO_NIU_H	89
-+#define SRST_CRYPTO_A		90
-+#define SRST_CRYPTO_H		91
-+#define SRST_CRYPTO_CORE	92
-+#define SRST_CRYPTO_PKA		93
-+#define SRST_SGRF_P		95
-+
-+/* cru_softrst_con6 */
-+#define SRST_PDAUDIO_NIU_H	96
-+#define SRST_PDAUDIO_NIU_P	97
-+#define SRST_I2S0_H		98
-+#define SRST_I2S0_TX_M		99
-+#define SRST_I2S0_RX_M		100
-+#define SRST_I2S1_H		101
-+#define SRST_I2S1_M		102
-+#define SRST_I2S2_H		103
-+#define SRST_I2S2_M		104
-+#define SRST_PDM_H		105
-+#define SRST_PDM_M		106
-+#define SRST_AUDPWM_H		107
-+#define SRST_AUDPWM		108
-+#define SRST_ACDCDIG_P		109
-+#define SRST_ACDCDIG		110
-+
-+/* cru_softrst_con7 */
-+#define SRST_PDVEPU_NIU_A	112
-+#define SRST_PDVEPU_NIU_H	113
-+#define SRST_VENC_A		114
-+#define SRST_VENC_H		115
-+#define SRST_VENC_CORE		116
-+#define SRST_PDVDEC_NIU_A	117
-+#define SRST_PDVDEC_NIU_H	118
-+#define SRST_VDEC_A		119
-+#define SRST_VDEC_H		120
-+#define SRST_VDEC_CORE		121
-+#define SRST_VDEC_CA		122
-+#define SRST_VDEC_HEVC_CA	123
-+#define SRST_PDJPEG_NIU_A	124
-+#define SRST_PDJPEG_NIU_H	125
-+#define SRST_JPEG_A		126
-+#define SRST_JPEG_H		127
-+
-+/* cru_softrst_con8 */
-+#define SRST_PDVO_NIU_A		128
-+#define SRST_PDVO_NIU_H		129
-+#define SRST_PDVO_NIU_P		130
-+#define SRST_RGA_A		131
-+#define SRST_RGA_H		132
-+#define SRST_RGA_CORE		133
-+#define SRST_VOP_A		134
-+#define SRST_VOP_H		135
-+#define SRST_VOP_D		136
-+#define SRST_TXBYTEHS_DSIHOST	137
-+#define SRST_DSIHOST_P		138
-+#define SRST_IEP_A		139
-+#define SRST_IEP_H		140
-+#define SRST_IEP_CORE		141
-+#define SRST_ISP_RX_P		142
-+
-+/* cru_softrst_con9 */
-+#define SRST_PDVI_NIU_A		144
-+#define SRST_PDVI_NIU_H		145
-+#define SRST_PDVI_NIU_P		146
-+#define SRST_ISP		147
-+#define SRST_CIF_A		148
-+#define SRST_CIF_H		149
-+#define SRST_CIF_D		150
-+#define SRST_CIF_P		151
-+#define SRST_CIF_I		152
-+#define SRST_CIF_RX_P		153
-+#define SRST_PDISPP_NIU_A	154
-+#define SRST_PDISPP_NIU_H	155
-+#define SRST_ISPP_A		156
-+#define SRST_ISPP_H		157
-+#define SRST_ISPP		158
-+#define SRST_CSIHOST_P		159
-+
-+/* cru_softrst_con10 */
-+#define SRST_PDPHPMID_NIU_A	160
-+#define SRST_PDPHPMID_NIU_H	161
-+#define SRST_PDNVM_NIU_H	163
-+#define SRST_SDMMC_H		164
-+#define SRST_SDIO_H		165
-+#define SRST_EMMC_H		166
-+#define SRST_SFC_H		167
-+#define SRST_SFCXIP_H		168
-+#define SRST_SFC		169
-+#define SRST_NANDC_H		170
-+#define SRST_NANDC		171
-+#define SRST_PDSDMMC_H		173
-+#define SRST_PDSDIO_H		174
-+
-+/* cru_softrst_con11 */
-+#define SRST_PDUSB_NIU_A	176
-+#define SRST_PDUSB_NIU_H	177
-+#define SRST_USBHOST_H		178
-+#define SRST_USBHOST_ARB_H	179
-+#define SRST_USBHOST_UTMI	180
-+#define SRST_USBOTG_A		181
-+#define SRST_USBPHY_OTG_P	182
-+#define SRST_USBPHY_HOST_P	183
-+#define SRST_USBPHYPOR_OTG	184
-+#define SRST_USBPHYPOR_HOST	185
-+#define SRST_PDGMAC_NIU_A	188
-+#define SRST_PDGMAC_NIU_P	189
-+#define SRST_GMAC_A		190
-+
-+/* cru_softrst_con12 */
-+#define SRST_DDR_DFICTL_P	193
-+#define SRST_DDR_MON_P		194
-+#define SRST_DDR_STANDBY_P	195
-+#define SRST_DDR_GRF_P		196
-+#define SRST_DDR_MSCH_P		197
-+#define SRST_DDR_SPLIT_A	198
-+#define SRST_DDR_MSCH		199
-+#define SRST_DDR_DFICTL		202
-+#define SRST_DDR_STANDBY	203
-+#define SRST_NPUMCU_NIU		205
-+#define SRST_DDRPHY_P		206
-+#define SRST_DDRPHY		207
-+
-+/* cru_softrst_con13 */
-+#define SRST_PDNPU_NIU_A	208
-+#define SRST_PDNPU_NIU_H	209
-+#define SRST_PDNPU_NIU_P	210
-+#define SRST_NPU_A		211
-+#define SRST_NPU_H		212
-+#define SRST_NPU		213
-+#define SRST_NPUPVTM_P		214
-+#define SRST_NPUPVTM		215
-+#define SRST_NPU_TSADC_P	216
-+#define SRST_NPU_TSADC		217
-+#define SRST_NPU_TSADCPHY	218
-+#define SRST_CIFLITE_A		220
-+#define SRST_CIFLITE_H		221
-+#define SRST_CIFLITE_D		222
-+#define SRST_CIFLITE_RX_P	223
-+
-+/* cru_softrst_con14 */
-+#define SRST_TOPNIU_P		224
-+#define SRST_TOPCRU_P		225
-+#define SRST_TOPGRF_P		226
-+#define SRST_CPUEMADET_P	227
-+#define SRST_CSIPHY0_P		228
-+#define SRST_CSIPHY1_P		229
-+#define SRST_DSIPHY_P		230
-+#define SRST_CPU_TSADC_P	232
-+#define SRST_CPU_TSADC		233
-+#define SRST_CPU_TSADCPHY	234
-+#define SRST_CPUPVTM_P		235
-+#define SRST_CPUPVTM		236
-+
-+#endif
-diff --git a/include/dt-bindings/clock/rockchip,rv1126b-cru.h b/include/dt-bindings/clock/rockchip,rv1126b-cru.h
-new file mode 100644
-index 000000000000..721d50a1419f
--- /dev/null
-+++ b/include/dt-bindings/clock/rockchip,rv1126b-cru.h
-@@ -0,0 +1,392 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (c) 2025 Rockchip Electronics Co., Ltd.
-+ * Author: Elaine Zhang <zhangqing@rock-chips.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_ROCKCHIP_RV1126B_H
-+#define _DT_BINDINGS_CLK_ROCKCHIP_RV1126B_H
-+
-+/* pll clocks */
-+#define PLL_GPLL				0
-+#define PLL_CPLL				1
-+#define PLL_AUPLL				2
-+#define ARMCLK					3
-+#define SCLK_DDR				4
-+
-+/* clk (clocks) */
-+#define CLK_CPLL_DIV20				5
-+#define CLK_CPLL_DIV10				6
-+#define CLK_CPLL_DIV8				7
-+#define CLK_GPLL_DIV8				8
-+#define CLK_GPLL_DIV6				9
-+#define CLK_GPLL_DIV4				10
-+#define CLK_CPLL_DIV3				11
-+#define CLK_GPLL_DIV3				12
-+#define CLK_CPLL_DIV2				13
-+#define CLK_GPLL_DIV2				14
-+#define CLK_CM_FRAC0				15
-+#define CLK_CM_FRAC1				16
-+#define CLK_CM_FRAC2				17
-+#define CLK_UART_FRAC0				18
-+#define CLK_UART_FRAC1				19
-+#define CLK_AUDIO_FRAC0				20
-+#define CLK_AUDIO_FRAC1				21
-+#define CLK_AUDIO_INT0				22
-+#define CLK_AUDIO_INT1				23
-+#define SCLK_UART0_SRC				24
-+#define SCLK_UART1				25
-+#define SCLK_UART2				26
-+#define SCLK_UART3				27
-+#define SCLK_UART4				28
-+#define SCLK_UART5				29
-+#define SCLK_UART6				30
-+#define SCLK_UART7				31
-+#define MCLK_SAI0				32
-+#define MCLK_SAI1				33
-+#define MCLK_SAI2				34
-+#define MCLK_PDM				35
-+#define CLKOUT_PDM				36
-+#define MCLK_ASRC0				37
-+#define MCLK_ASRC1				38
-+#define MCLK_ASRC2				39
-+#define MCLK_ASRC3				40
-+#define CLK_ASRC0				41
-+#define CLK_ASRC1				42
-+#define CLK_CORE_PLL				43
-+#define CLK_NPU_PLL				44
-+#define CLK_VEPU_PLL				45
-+#define CLK_ISP_PLL				46
-+#define CLK_AISP_PLL				47
-+#define CLK_SARADC0_SRC				48
-+#define CLK_SARADC1_SRC				49
-+#define CLK_SARADC2_SRC				50
-+#define HCLK_NPU_ROOT				51
-+#define PCLK_NPU_ROOT				52
-+#define ACLK_VEPU_ROOT				53
-+#define HCLK_VEPU_ROOT				54
-+#define PCLK_VEPU_ROOT				55
-+#define CLK_CORE_RGA_SRC			56
-+#define ACLK_GMAC_ROOT				57
-+#define ACLK_VI_ROOT				58
-+#define HCLK_VI_ROOT				59
-+#define PCLK_VI_ROOT				60
-+#define DCLK_VICAP_ROOT				61
-+#define CLK_SYS_DSMC_ROOT			62
-+#define ACLK_VDO_ROOT				63
-+#define ACLK_RKVDEC_ROOT			64
-+#define HCLK_VDO_ROOT				65
-+#define PCLK_VDO_ROOT				66
-+#define DCLK_OOC_SRC				67
-+#define DCLK_VOP				68
-+#define DCLK_DECOM_SRC				69
-+#define PCLK_DDR_ROOT				70
-+#define ACLK_SYSMEM_SRC				71
-+#define ACLK_TOP_ROOT				72
-+#define ACLK_BUS_ROOT				73
-+#define HCLK_BUS_ROOT				74
-+#define PCLK_BUS_ROOT				75
-+#define CCLK_SDMMC0				76
-+#define CCLK_SDMMC1				77
-+#define CCLK_EMMC				78
-+#define SCLK_2X_FSPI0				79
-+#define CLK_GMAC_PTP_REF_SRC			80
-+#define CLK_GMAC_125M				81
-+#define CLK_TIMER_ROOT				82
-+#define TCLK_WDT_NS_SRC				83
-+#define TCLK_WDT_S_SRC				84
-+#define TCLK_WDT_HPMCU				85
-+#define CLK_CAN0				86
-+#define CLK_CAN1				87
-+#define PCLK_PERI_ROOT				88
-+#define ACLK_PERI_ROOT				89
-+#define CLK_I2C_BUS_SRC				90
-+#define CLK_SPI0				91
-+#define CLK_SPI1				92
-+#define BUSCLK_PMU_SRC				93
-+#define CLK_PWM0				94
-+#define CLK_PWM2				95
-+#define CLK_PWM3				96
-+#define CLK_PKA_RKCE_SRC			97
-+#define ACLK_RKCE_SRC				98
-+#define ACLK_VCP_ROOT				99
-+#define HCLK_VCP_ROOT				100
-+#define PCLK_VCP_ROOT				101
-+#define CLK_CORE_FEC_SRC			102
-+#define CLK_CORE_AVSP_SRC			103
-+#define CLK_50M_GMAC_IOBUF_VI			104
-+#define PCLK_TOP_ROOT				105
-+#define CLK_MIPI0_OUT2IO			106
-+#define CLK_MIPI1_OUT2IO			107
-+#define CLK_MIPI2_OUT2IO			108
-+#define CLK_MIPI3_OUT2IO			109
-+#define CLK_CIF_OUT2IO				110
-+#define CLK_MAC_OUT2IO				111
-+#define MCLK_SAI0_OUT2IO			112
-+#define MCLK_SAI1_OUT2IO			113
-+#define MCLK_SAI2_OUT2IO			114
-+#define CLK_CM_FRAC0_SRC			115
-+#define CLK_CM_FRAC1_SRC			116
-+#define CLK_CM_FRAC2_SRC			117
-+#define CLK_UART_FRAC0_SRC			118
-+#define CLK_UART_FRAC1_SRC			119
-+#define CLK_AUDIO_FRAC0_SRC			120
-+#define CLK_AUDIO_FRAC1_SRC			121
-+#define ACLK_NPU_ROOT				122
-+#define HCLK_RKNN				123
-+#define ACLK_RKNN				124
-+#define PCLK_GPIO3				125
-+#define DBCLK_GPIO3				126
-+#define PCLK_IOC_VCCIO3				127
-+#define PCLK_SARADC0				128
-+#define CLK_SARADC0				129
-+#define HCLK_SDMMC1				130
-+#define HCLK_VEPU				131
-+#define ACLK_VEPU				132
-+#define CLK_CORE_VEPU				133
-+#define HCLK_FEC				134
-+#define ACLK_FEC				135
-+#define CLK_CORE_FEC				136
-+#define HCLK_AVSP				137
-+#define ACLK_AVSP				138
-+#define BUSCLK_PMU1_ROOT			139
-+#define HCLK_AISP				140
-+#define ACLK_AISP				141
-+#define CLK_CORE_AISP				142
-+#define CLK_CORE_ISP_ROOT			143
-+#define PCLK_DSMC				144
-+#define ACLK_DSMC				145
-+#define HCLK_CAN0				146
-+#define HCLK_CAN1				147
-+#define PCLK_GPIO2				148
-+#define DBCLK_GPIO2				149
-+#define PCLK_GPIO4				150
-+#define DBCLK_GPIO4				151
-+#define PCLK_GPIO5				152
-+#define DBCLK_GPIO5				153
-+#define PCLK_GPIO6				154
-+#define DBCLK_GPIO6				155
-+#define PCLK_GPIO7				156
-+#define DBCLK_GPIO7				157
-+#define PCLK_IOC_VCCIO2				158
-+#define PCLK_IOC_VCCIO4				159
-+#define PCLK_IOC_VCCIO5				160
-+#define PCLK_IOC_VCCIO6				161
-+#define PCLK_IOC_VCCIO7				162
-+#define HCLK_ISP				163
-+#define ACLK_ISP				164
-+#define CLK_CORE_ISP				165
-+#define HCLK_VICAP				166
-+#define ACLK_VICAP				167
-+#define DCLK_VICAP				168
-+#define ISP0CLK_VICAP				169
-+#define HCLK_VPSS				170
-+#define ACLK_VPSS				171
-+#define CLK_CORE_VPSS				172
-+#define PCLK_CSI2HOST0				173
-+#define DCLK_CSI2HOST0				174
-+#define PCLK_CSI2HOST1				175
-+#define DCLK_CSI2HOST1				176
-+#define PCLK_CSI2HOST2				177
-+#define DCLK_CSI2HOST2				178
-+#define PCLK_CSI2HOST3				179
-+#define DCLK_CSI2HOST3				180
-+#define HCLK_SDMMC0				181
-+#define ACLK_GMAC				182
-+#define PCLK_GMAC				183
-+#define CLK_GMAC_PTP_REF			184
-+#define PCLK_CSIPHY0				185
-+#define PCLK_CSIPHY1				186
-+#define PCLK_MACPHY				187
-+#define PCLK_SARADC1				188
-+#define CLK_SARADC1				189
-+#define PCLK_SARADC2				190
-+#define CLK_SARADC2				191
-+#define ACLK_RKVDEC				192
-+#define HCLK_RKVDEC				193
-+#define CLK_HEVC_CA_RKVDEC			194
-+#define ACLK_VOP				195
-+#define HCLK_VOP				196
-+#define HCLK_RKJPEG				197
-+#define ACLK_RKJPEG				198
-+#define ACLK_RKMMU_DECOM			199
-+#define HCLK_RKMMU_DECOM			200
-+#define DCLK_DECOM				201
-+#define ACLK_DECOM				202
-+#define PCLK_DECOM				203
-+#define PCLK_MIPI_DSI				204
-+#define PCLK_DSIPHY				205
-+#define ACLK_OOC				206
-+#define ACLK_SYSMEM				207
-+#define PCLK_DDRC				208
-+#define PCLK_DDRMON				209
-+#define CLK_TIMER_DDRMON			210
-+#define PCLK_DFICTRL				211
-+#define PCLK_DDRPHY				212
-+#define PCLK_DMA2DDR				213
-+#define CLK_RCOSC_SRC				214
-+#define BUSCLK_PMU_MUX				215
-+#define BUSCLK_PMU_ROOT				216
-+#define PCLK_PMU				217
-+#define CLK_XIN_RC_DIV				218
-+#define CLK_32K					219
-+#define PCLK_PMU_GPIO0				220
-+#define DBCLK_PMU_GPIO0				221
-+#define PCLK_PMU_HP_TIMER			222
-+#define CLK_PMU_HP_TIMER			223
-+#define CLK_PMU_32K_HP_TIMER			224
-+#define PCLK_PWM1				225
-+#define CLK_PWM1				226
-+#define CLK_OSC_PWM1				227
-+#define CLK_RC_PWM1				228
-+#define CLK_FREQ_PWM1				229
-+#define CLK_COUNTER_PWM1			230
-+#define PCLK_I2C2				231
-+#define CLK_I2C2				232
-+#define PCLK_UART0				233
-+#define SCLK_UART0				234
-+#define PCLK_RCOSC_CTRL				235
-+#define CLK_OSC_RCOSC_CTRL			236
-+#define CLK_REF_RCOSC_CTRL			237
-+#define PCLK_IOC_PMUIO0				238
-+#define CLK_REFOUT				239
-+#define CLK_PREROLL				240
-+#define CLK_PREROLL_32K				241
-+#define HCLK_PMU_SRAM				242
-+#define PCLK_WDT_LPMCU				243
-+#define TCLK_WDT_LPMCU				244
-+#define CLK_LPMCU				245
-+#define CLK_LPMCU_RTC				246
-+#define PCLK_LPMCU_MAILBOX			247
-+#define HCLK_OOC				248
-+#define PCLK_SPI2AHB				249
-+#define HCLK_SPI2AHB				250
-+#define HCLK_FSPI1				251
-+#define HCLK_XIP_FSPI1				252
-+#define SCLK_1X_FSPI1				253
-+#define PCLK_IOC_PMUIO1				254
-+#define PCLK_AUDIO_ADC_PMU			255
-+#define MCLK_AUDIO_ADC_PMU			256
-+#define MCLK_AUDIO_ADC_DIV4_PMU			257
-+#define MCLK_LPSAI				258
-+#define ACLK_GIC400				259
-+#define PCLK_WDT_NS				260
-+#define TCLK_WDT_NS				261
-+#define PCLK_WDT_HPMCU				262
-+#define HCLK_CACHE				263
-+#define PCLK_HPMCU_MAILBOX			264
-+#define PCLK_HPMCU_INTMUX			265
-+#define CLK_HPMCU				266
-+#define CLK_HPMCU_RTC				267
-+#define PCLK_RKDMA				268
-+#define ACLK_RKDMA				269
-+#define PCLK_DCF				270
-+#define ACLK_DCF				271
-+#define HCLK_RGA				272
-+#define ACLK_RGA				273
-+#define CLK_CORE_RGA				274
-+#define PCLK_TIMER				275
-+#define CLK_TIMER0				276
-+#define CLK_TIMER1				277
-+#define CLK_TIMER2				278
-+#define CLK_TIMER3				279
-+#define CLK_TIMER4				280
-+#define CLK_TIMER5				281
-+#define PCLK_I2C0				282
-+#define CLK_I2C0				283
-+#define PCLK_I2C1				284
-+#define CLK_I2C1				285
-+#define PCLK_I2C3				286
-+#define CLK_I2C3				287
-+#define PCLK_I2C4				288
-+#define CLK_I2C4				289
-+#define PCLK_I2C5				290
-+#define CLK_I2C5				291
-+#define PCLK_SPI0				292
-+#define PCLK_SPI1				293
-+#define PCLK_PWM0				294
-+#define CLK_OSC_PWM0				295
-+#define CLK_RC_PWM0				296
-+#define PCLK_PWM2				297
-+#define CLK_OSC_PWM2				298
-+#define CLK_RC_PWM2				299
-+#define PCLK_PWM3				300
-+#define CLK_OSC_PWM3				301
-+#define CLK_RC_PWM3				302
-+#define PCLK_UART1				303
-+#define PCLK_UART2				304
-+#define PCLK_UART3				305
-+#define PCLK_UART4				306
-+#define PCLK_UART5				307
-+#define PCLK_UART6				308
-+#define PCLK_UART7				309
-+#define PCLK_TSADC				310
-+#define CLK_TSADC				311
-+#define HCLK_SAI0				312
-+#define HCLK_SAI1				313
-+#define HCLK_SAI2				314
-+#define HCLK_RKDSM				315
-+#define MCLK_RKDSM				316
-+#define HCLK_PDM				317
-+#define HCLK_ASRC0				318
-+#define HCLK_ASRC1				319
-+#define PCLK_AUDIO_ADC_BUS			320
-+#define MCLK_AUDIO_ADC_BUS			321
-+#define MCLK_AUDIO_ADC_DIV4_BUS			322
-+#define PCLK_RKCE				323
-+#define HCLK_NS_RKCE				324
-+#define PCLK_OTPC_NS				325
-+#define CLK_SBPI_OTPC_NS			326
-+#define CLK_USER_OTPC_NS			327
-+#define CLK_OTPC_ARB				328
-+#define PCLK_OTP_MASK				329
-+#define CLK_TSADC_PHYCTRL			330
-+#define LRCK_SRC_ASRC0				331
-+#define LRCK_DST_ASRC0				332
-+#define LRCK_SRC_ASRC1				333
-+#define LRCK_DST_ASRC1				334
-+#define PCLK_KEY_READER				335
-+#define ACLK_NSRKCE				336
-+#define CLK_PKA_NSRKCE				337
-+#define PCLK_RTC_ROOT				338
-+#define PCLK_GPIO1				339
-+#define DBCLK_GPIO1				340
-+#define PCLK_IOC_VCCIO1				341
-+#define ACLK_USB3OTG				342
-+#define CLK_REF_USB3OTG				343
-+#define CLK_SUSPEND_USB3OTG			344
-+#define HCLK_USB2HOST				345
-+#define HCLK_ARB_USB2HOST			346
-+#define PCLK_RTC_TEST				347
-+#define HCLK_EMMC				348
-+#define HCLK_FSPI0				349
-+#define HCLK_XIP_FSPI0				350
-+#define PCLK_PIPEPHY				351
-+#define PCLK_USB2PHY				352
-+#define CLK_REF_PIPEPHY_CPLL_SRC		353
-+#define CLK_REF_PIPEPHY				354
-+#define HCLK_VPSL				355
-+#define ACLK_VPSL				356
-+#define CLK_CORE_VPSL				357
-+#define CLK_MACPHY				358
-+#define HCLK_RKRNG_NS				359
-+#define HCLK_RKRNG_S_NS				360
-+#define CLK_AISP_PLL_SRC			361
-+
-+/* secure clks */
-+#define CLK_USER_OTPC_S				362
-+#define CLK_SBPI_OTPC_S				363
-+#define PCLK_OTPC_S				364
-+#define PCLK_KEY_READER_S			365
-+#define HCLK_KL_RKCE_S				366
-+#define HCLK_RKCE_S				367
-+#define PCLK_WDT_S				368
-+#define TCLK_WDT_S				369
-+#define CLK_STIMER0				370
-+#define CLK_STIMER1				371
-+#define PLK_STIMER				372
-+#define HCLK_RKRNG_S				373
-+#define CLK_PKA_RKCE_S				374
-+#define ACLK_RKCE_S				375
-+
-+#endif
-diff --git a/include/dt-bindings/clock/s3c2410.h b/include/dt-bindings/clock/s3c2410.h
-deleted file mode 100644
-index 0fb65c3f2f59..000000000000
--- a/include/dt-bindings/clock/s3c2410.h
-+++ /dev/null
-@@ -1,59 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
- *
- * Device Tree binding constants clock controllers of Samsung S3C2410 and later.
- */
-
-#ifndef _DT_BINDINGS_CLOCK_SAMSUNG_S3C2410_CLOCK_H
-#define _DT_BINDINGS_CLOCK_SAMSUNG_S3C2410_CLOCK_H
-
-/*
- * Let each exported clock get a unique index, which is used on DT-enabled
- * platforms to lookup the clock from a clock specifier. These indices are
- * therefore considered an ABI and so must not be changed. This implies
- * that new clocks should be added either in free spaces between clock groups
- * or at the end.
- */
-
-/* Core clocks. */
-
-/* id 1 is reserved */
-#define MPLL			2
-#define UPLL			3
-#define FCLK			4
-#define HCLK			5
-#define PCLK			6
-#define UCLK			7
-#define ARMCLK			8
-
-/* pclk-gates */
-#define PCLK_UART0		16
-#define PCLK_UART1		17
-#define PCLK_UART2		18
-#define PCLK_I2C		19
-#define PCLK_SDI		20
-#define PCLK_SPI		21
-#define PCLK_ADC		22
-#define PCLK_AC97		23
-#define PCLK_I2S		24
-#define PCLK_PWM		25
-#define PCLK_RTC		26
-#define PCLK_GPIO		27
-
-
-/* hclk-gates */
-#define HCLK_LCD		32
-#define HCLK_USBH		33
-#define HCLK_USBD		34
-#define HCLK_NAND		35
-#define HCLK_CAM		36
-
-
-#define CAMIF			40
-
-
-/* Total number of clocks. */
-#define NR_CLKS			(CAMIF + 1)
-
-#endif /* _DT_BINDINGS_CLOCK_SAMSUNG_S3C2443_CLOCK_H */
-diff --git a/include/dt-bindings/clock/s3c2412.h b/include/dt-bindings/clock/s3c2412.h
-deleted file mode 100644
-index b4656156cc0f..000000000000
--- a/include/dt-bindings/clock/s3c2412.h
-+++ /dev/null
-@@ -1,70 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
- *
- * Device Tree binding constants clock controllers of Samsung S3C2412.
- */
-
-#ifndef _DT_BINDINGS_CLOCK_SAMSUNG_S3C2412_CLOCK_H
-#define _DT_BINDINGS_CLOCK_SAMSUNG_S3C2412_CLOCK_H
-
-/*
- * Let each exported clock get a unique index, which is used on DT-enabled
- * platforms to lookup the clock from a clock specifier. These indices are
- * therefore considered an ABI and so must not be changed. This implies
- * that new clocks should be added either in free spaces between clock groups
- * or at the end.
- */
-
-/* Core clocks. */
-
-/* id 1 is reserved */
-#define MPLL			2
-#define UPLL			3
-#define MDIVCLK			4
-#define MSYSCLK			5
-#define USYSCLK			6
-#define HCLK			7
-#define PCLK			8
-#define ARMDIV			9
-#define ARMCLK			10
-
-
-/* Special clocks */
-#define SCLK_CAM		16
-#define SCLK_UART		17
-#define SCLK_I2S		18
-#define SCLK_USBD		19
-#define SCLK_USBH		20
-
-/* pclk-gates */
-#define PCLK_WDT		32
-#define PCLK_SPI		33
-#define PCLK_I2S		34
-#define PCLK_I2C		35
-#define PCLK_ADC		36
-#define PCLK_RTC		37
-#define PCLK_GPIO		38
-#define PCLK_UART2		39
-#define PCLK_UART1		40
-#define PCLK_UART0		41
-#define PCLK_SDI		42
-#define PCLK_PWM		43
-#define PCLK_USBD		44
-
-/* hclk-gates */
-#define HCLK_HALF		48
-#define HCLK_X2			49
-#define HCLK_SDRAM		50
-#define HCLK_USBH		51
-#define HCLK_LCD		52
-#define HCLK_NAND		53
-#define HCLK_DMA3		54
-#define HCLK_DMA2		55
-#define HCLK_DMA1		56
-#define HCLK_DMA0		57
-
-/* Total number of clocks. */
-#define NR_CLKS			(HCLK_DMA0 + 1)
-
-#endif /* _DT_BINDINGS_CLOCK_SAMSUNG_S3C2412_CLOCK_H */
-diff --git a/include/dt-bindings/clock/s3c2443.h b/include/dt-bindings/clock/s3c2443.h
-deleted file mode 100644
-index a9d2f105d536..000000000000
--- a/include/dt-bindings/clock/s3c2443.h
-+++ /dev/null
-@@ -1,91 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
- *
- * Device Tree binding constants clock controllers of Samsung S3C2443 and later.
- */
-
-#ifndef _DT_BINDINGS_CLOCK_SAMSUNG_S3C2443_CLOCK_H
-#define _DT_BINDINGS_CLOCK_SAMSUNG_S3C2443_CLOCK_H
-
-/*
- * Let each exported clock get a unique index, which is used on DT-enabled
- * platforms to lookup the clock from a clock specifier. These indices are
- * therefore considered an ABI and so must not be changed. This implies
- * that new clocks should be added either in free spaces between clock groups
- * or at the end.
- */
-
-/* Core clocks. */
-#define MSYSCLK			1
-#define ESYSCLK			2
-#define ARMDIV			3
-#define ARMCLK			4
-#define HCLK			5
-#define PCLK			6
-#define MPLL			7
-#define EPLL			8
-
-/* Special clocks */
-#define SCLK_HSSPI0		16
-#define SCLK_FIMD		17
-#define SCLK_I2S0		18
-#define SCLK_I2S1		19
-#define SCLK_HSMMC1		20
-#define SCLK_HSMMC_EXT		21
-#define SCLK_CAM		22
-#define SCLK_UART		23
-#define SCLK_USBH		24
-
-/* Muxes */
-#define MUX_HSSPI0		32
-#define MUX_HSSPI1		33
-#define MUX_HSMMC0		34
-#define MUX_HSMMC1		35
-
-/* hclk-gates */
-#define HCLK_DMA0		48
-#define HCLK_DMA1		49
-#define HCLK_DMA2		50
-#define HCLK_DMA3		51
-#define HCLK_DMA4		52
-#define HCLK_DMA5		53
-#define HCLK_DMA6		54
-#define HCLK_DMA7		55
-#define HCLK_CAM		56
-#define HCLK_LCD		57
-#define HCLK_USBH		58
-#define HCLK_USBD		59
-#define HCLK_IROM		60
-#define HCLK_HSMMC0		61
-#define HCLK_HSMMC1		62
-#define HCLK_CFC		63
-#define HCLK_SSMC		64
-#define HCLK_DRAM		65
-#define HCLK_2D			66
-
-/* pclk-gates */
-#define PCLK_UART0		72
-#define PCLK_UART1		73
-#define PCLK_UART2		74
-#define PCLK_UART3		75
-#define PCLK_I2C0		76
-#define PCLK_SDI		77
-#define PCLK_SPI0		78
-#define PCLK_ADC		79
-#define PCLK_AC97		80
-#define PCLK_I2S0		81
-#define PCLK_PWM		82
-#define PCLK_WDT		83
-#define PCLK_RTC		84
-#define PCLK_GPIO		85
-#define PCLK_SPI1		86
-#define PCLK_CHIPID		87
-#define PCLK_I2C1		88
-#define PCLK_I2S1		89
-#define PCLK_PCM		90
-
-/* Total number of clocks. */
-#define NR_CLKS			(PCLK_PCM + 1)
-
-#endif /* _DT_BINDINGS_CLOCK_SAMSUNG_S3C2443_CLOCK_H */
-diff --git a/include/dt-bindings/clock/samsung,exynos2200-cmu.h b/include/dt-bindings/clock/samsung,exynos2200-cmu.h
-new file mode 100644
-index 000000000000..310552be0c8c
--- /dev/null
-+++ b/include/dt-bindings/clock/samsung,exynos2200-cmu.h
-@@ -0,0 +1,431 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2025 Ivaylo Ivanov <ivo.ivanov.ivanov1@gmail.com>
-+ * Author: Ivaylo Ivanov <ivo.ivanov.ivanov1@gmail.com>
-+ *
-+ * Device Tree binding constants for Exynos2200 clock controller.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLOCK_EXYNOS2200_H
-+#define _DT_BINDINGS_CLOCK_EXYNOS2200_H
-+
-+/* CMU_TOP */
-+#define CLK_FOUT_SHARED0_PLL			1
-+#define CLK_FOUT_SHARED1_PLL			2
-+#define CLK_FOUT_SHARED2_PLL			3
-+#define CLK_FOUT_SHARED3_PLL			4
-+#define CLK_FOUT_SHARED4_PLL			5
-+#define CLK_FOUT_MMC_PLL			6
-+#define CLK_FOUT_SHARED_MIF_PLL			7
-+
-+#define CLK_MOUT_CMU_CP_MPLL_CLK_D2_USER	8
-+#define CLK_MOUT_CMU_CP_MPLL_CLK_USER		9
-+#define CLK_MOUT_CMU_AUD_AUDIF0			10
-+#define CLK_MOUT_CMU_AUD_AUDIF1			11
-+#define CLK_MOUT_CMU_AUD_CPU			12
-+#define CLK_MOUT_CMU_CPUCL0_DBG_NOC		13
-+#define CLK_MOUT_CMU_CPUCL0_SWITCH		14
-+#define CLK_MOUT_CMU_CPUCL1_SWITCH		15
-+#define CLK_MOUT_CMU_CPUCL2_SWITCH		16
-+#define CLK_MOUT_CMU_DNC_NOC			17
-+#define CLK_MOUT_CMU_DPUB_NOC			18
-+#define CLK_MOUT_CMU_DPUF_NOC			19
-+#define CLK_MOUT_CMU_DSP_NOC			20
-+#define CLK_MOUT_CMU_DSU_SWITCH			21
-+#define CLK_MOUT_CMU_G3D_SWITCH			22
-+#define CLK_MOUT_CMU_GNPU_NOC			23
-+#define CLK_MOUT_CMU_UFS_MMC_CARD		24
-+#define CLK_MOUT_CMU_M2M_NOC			25
-+#define CLK_MOUT_CMU_NOCL0_NOC			26
-+#define CLK_MOUT_CMU_NOCL1A_NOC			27
-+#define CLK_MOUT_CMU_NOCL1B_NOC0		28
-+#define CLK_MOUT_CMU_NOCL1C_NOC			29
-+#define CLK_MOUT_CMU_SDMA_NOC			30
-+#define CLK_MOUT_CMU_CP_HISPEEDY_CLK		31
-+#define CLK_MOUT_CMU_CP_SHARED0_CLK		32
-+#define CLK_MOUT_CMU_CP_SHARED2_CLK		33
-+#define CLK_MOUT_CMU_MUX_ALIVE_NOC		34
-+#define CLK_MOUT_CMU_MUX_AUD_AUDIF0		35
-+#define CLK_MOUT_CMU_MUX_AUD_AUDIF1		36
-+#define CLK_MOUT_CMU_MUX_AUD_CPU		37
-+#define CLK_MOUT_CMU_MUX_AUD_NOC		38
-+#define CLK_MOUT_CMU_MUX_BRP_NOC		39
-+#define CLK_MOUT_CMU_MUX_CIS_CLK0		40
-+#define CLK_MOUT_CMU_MUX_CIS_CLK1		41
-+#define CLK_MOUT_CMU_MUX_CIS_CLK2		42
-+#define CLK_MOUT_CMU_MUX_CIS_CLK3		43
-+#define CLK_MOUT_CMU_MUX_CIS_CLK4		44
-+#define CLK_MOUT_CMU_MUX_CIS_CLK5		45
-+#define CLK_MOUT_CMU_MUX_CIS_CLK6		46
-+#define CLK_MOUT_CMU_MUX_CIS_CLK7		47
-+#define CLK_MOUT_CMU_MUX_CMU_BOOST		48
-+#define CLK_MOUT_CMU_MUX_CMU_BOOST_CAM		49
-+#define CLK_MOUT_CMU_MUX_CMU_BOOST_CPU		50
-+#define CLK_MOUT_CMU_MUX_CMU_BOOST_MIF		51
-+#define CLK_MOUT_CMU_MUX_CPUCL0_DBG_NOC		52
-+#define CLK_MOUT_CMU_MUX_CPUCL0_NOCP		53
-+#define CLK_MOUT_CMU_MUX_CPUCL0_SWITCH		54
-+#define CLK_MOUT_CMU_MUX_CPUCL1_SWITCH		55
-+#define CLK_MOUT_CMU_MUX_CPUCL2_SWITCH		56
-+#define CLK_MOUT_CMU_MUX_CSIS_DCPHY		57
-+#define CLK_MOUT_CMU_MUX_CSIS_NOC		58
-+#define CLK_MOUT_CMU_MUX_CSIS_OIS_MCU		59
-+#define CLK_MOUT_CMU_MUX_CSTAT_NOC		60
-+#define CLK_MOUT_CMU_MUX_DNC_NOC		61
-+#define CLK_MOUT_CMU_MUX_DPUB			62
-+#define CLK_MOUT_CMU_MUX_DPUB_ALT		63
-+#define CLK_MOUT_CMU_MUX_DPUB_DSIM		64
-+#define CLK_MOUT_CMU_MUX_DPUF			65
-+#define CLK_MOUT_CMU_MUX_DPUF_ALT		66
-+#define CLK_MOUT_CMU_MUX_DSP_NOC		67
-+#define CLK_MOUT_CMU_MUX_DSU_SWITCH		68
-+#define CLK_MOUT_CMU_MUX_G3D_NOCP		69
-+#define CLK_MOUT_CMU_MUX_G3D_SWITCH		70
-+#define CLK_MOUT_CMU_MUX_GNPU_NOC		71
-+#define CLK_MOUT_CMU_MUX_HSI0_DPGTC		72
-+#define CLK_MOUT_CMU_MUX_HSI0_DPOSC		73
-+#define CLK_MOUT_CMU_MUX_HSI0_NOC		74
-+#define CLK_MOUT_CMU_MUX_HSI0_USB32DRD		75
-+#define CLK_MOUT_CMU_MUX_UFS_MMC_CARD		76
-+#define CLK_MOUT_CMU_MUX_HSI1_NOC		77
-+#define CLK_MOUT_CMU_MUX_HSI1_PCIE		78
-+#define CLK_MOUT_CMU_MUX_UFS_UFS_EMBD		79
-+#define CLK_MOUT_CMU_MUX_LME_LME		80
-+#define CLK_MOUT_CMU_MUX_LME_NOC		81
-+#define CLK_MOUT_CMU_MUX_M2M_NOC		82
-+#define CLK_MOUT_CMU_MUX_MCSC_MCSC		83
-+#define CLK_MOUT_CMU_MUX_MCSC_NOC		84
-+#define CLK_MOUT_CMU_MUX_MFC0_MFC0		85
-+#define CLK_MOUT_CMU_MUX_MFC0_WFD		86
-+#define CLK_MOUT_CMU_MUX_MFC1_MFC1		87
-+#define CLK_MOUT_CMU_MUX_MIF_NOCP		88
-+#define CLK_MOUT_CMU_MUX_MIF_SWITCH		89
-+#define CLK_MOUT_CMU_MUX_NOCL0_NOC		90
-+#define CLK_MOUT_CMU_MUX_NOCL1A_NOC		91
-+#define CLK_MOUT_CMU_MUX_NOCL1B_NOC0		92
-+#define CLK_MOUT_CMU_MUX_NOCL1B_NOC1		93
-+#define CLK_MOUT_CMU_MUX_NOCL1C_NOC		94
-+#define CLK_MOUT_CMU_MUX_PERIC0_IP0		95
-+#define CLK_MOUT_CMU_MUX_PERIC0_IP1		96
-+#define CLK_MOUT_CMU_MUX_PERIC0_NOC		97
-+#define CLK_MOUT_CMU_MUX_PERIC1_IP0		98
-+#define CLK_MOUT_CMU_MUX_PERIC1_IP1		99
-+#define CLK_MOUT_CMU_MUX_PERIC1_NOC		100
-+#define CLK_MOUT_CMU_MUX_PERIC2_IP0		101
-+#define CLK_MOUT_CMU_MUX_PERIC2_IP1		102
-+#define CLK_MOUT_CMU_MUX_PERIC2_NOC		103
-+#define CLK_MOUT_CMU_MUX_PERIS_GIC		104
-+#define CLK_MOUT_CMU_MUX_PERIS_NOC		105
-+#define CLK_MOUT_CMU_MUX_SDMA_NOC		106
-+#define CLK_MOUT_CMU_MUX_SSP_NOC		107
-+#define CLK_MOUT_CMU_MUX_VTS_DMIC		108
-+#define CLK_MOUT_CMU_MUX_YUVP_NOC		109
-+#define CLK_MOUT_CMU_MUX_CMU_CMUREF		110
-+#define CLK_MOUT_CMU_MUX_CP_HISPEEDY_CLK	111
-+#define CLK_MOUT_CMU_MUX_CP_SHARED0_CLK		112
-+#define CLK_MOUT_CMU_MUX_CP_SHARED1_CLK		113
-+#define CLK_MOUT_CMU_MUX_CP_SHARED2_CLK		114
-+#define CLK_MOUT_CMU_M2M_FRC			115
-+#define CLK_MOUT_CMU_MCSC_MCSC			116
-+#define CLK_MOUT_CMU_MCSC_NOC			117
-+#define CLK_MOUT_CMU_MUX_M2M_FRC		118
-+#define CLK_MOUT_CMU_MUX_UFS_NOC		119
-+
-+#define CLK_DOUT_CMU_ALIVE_NOC			120
-+#define CLK_DOUT_CMU_AUD_NOC			121
-+#define CLK_DOUT_CMU_BRP_NOC			122
-+#define CLK_DOUT_CMU_CMU_BOOST			123
-+#define CLK_DOUT_CMU_CMU_BOOST_CAM		124
-+#define CLK_DOUT_CMU_CMU_BOOST_CPU		125
-+#define CLK_DOUT_CMU_CMU_BOOST_MIF		126
-+#define CLK_DOUT_CMU_CPUCL0_NOCP		127
-+#define CLK_DOUT_CMU_CSIS_DCPHY			128
-+#define CLK_DOUT_CMU_CSIS_NOC			129
-+#define CLK_DOUT_CMU_CSIS_OIS_MCU		130
-+#define CLK_DOUT_CMU_CSTAT_NOC			131
-+#define CLK_DOUT_CMU_DPUB_DSIM			132
-+#define CLK_DOUT_CMU_LME_LME			133
-+#define CLK_DOUT_CMU_G3D_NOCP			134
-+#define CLK_DOUT_CMU_HSI0_DPGTC			135
-+#define CLK_DOUT_CMU_HSI0_DPOSC			136
-+#define CLK_DOUT_CMU_HSI0_NOC			137
-+#define CLK_DOUT_CMU_HSI0_USB32DRD		138
-+#define CLK_DOUT_CMU_HSI1_NOC			139
-+#define CLK_DOUT_CMU_HSI1_PCIE			140
-+#define CLK_DOUT_CMU_UFS_UFS_EMBD		141
-+#define CLK_DOUT_CMU_LME_NOC			142
-+#define CLK_DOUT_CMU_MFC0_MFC0			143
-+#define CLK_DOUT_CMU_MFC0_WFD			144
-+#define CLK_DOUT_CMU_MFC1_MFC1			145
-+#define CLK_DOUT_CMU_MIF_NOCP			146
-+#define CLK_DOUT_CMU_NOCL1B_NOC1		147
-+#define CLK_DOUT_CMU_PERIC0_IP0			148
-+#define CLK_DOUT_CMU_PERIC0_IP1			149
-+#define CLK_DOUT_CMU_PERIC0_NOC			150
-+#define CLK_DOUT_CMU_PERIC1_IP0			151
-+#define CLK_DOUT_CMU_PERIC1_IP1			152
-+#define CLK_DOUT_CMU_PERIC1_NOC			153
-+#define CLK_DOUT_CMU_PERIC2_IP0			154
-+#define CLK_DOUT_CMU_PERIC2_IP1			155
-+#define CLK_DOUT_CMU_PERIC2_NOC			156
-+#define CLK_DOUT_CMU_PERIS_GIC			157
-+#define CLK_DOUT_CMU_PERIS_NOC			158
-+#define CLK_DOUT_CMU_SSP_NOC			159
-+#define CLK_DOUT_CMU_VTS_DMIC			160
-+#define CLK_DOUT_CMU_YUVP_NOC			161
-+#define CLK_DOUT_CMU_CP_SHARED1_CLK		162
-+#define CLK_DOUT_CMU_DIV_AUD_AUDIF0		163
-+#define CLK_DOUT_CMU_DIV_AUD_AUDIF0_SM		164
-+#define CLK_DOUT_CMU_DIV_AUD_AUDIF1		165
-+#define CLK_DOUT_CMU_DIV_AUD_AUDIF1_SM		166
-+#define CLK_DOUT_CMU_DIV_AUD_CPU		167
-+#define CLK_DOUT_CMU_DIV_AUD_CPU_SM		168
-+#define CLK_DOUT_CMU_DIV_CIS_CLK0		169
-+#define CLK_DOUT_CMU_DIV_CIS_CLK1		170
-+#define CLK_DOUT_CMU_DIV_CIS_CLK2		171
-+#define CLK_DOUT_CMU_DIV_CIS_CLK3		172
-+#define CLK_DOUT_CMU_DIV_CIS_CLK4		173
-+#define CLK_DOUT_CMU_DIV_CIS_CLK5		174
-+#define CLK_DOUT_CMU_DIV_CIS_CLK6		175
-+#define CLK_DOUT_CMU_DIV_CIS_CLK7		176
-+#define CLK_DOUT_CMU_DIV_CPUCL0_DBG_NOC		177
-+#define CLK_DOUT_CMU_DIV_CPUCL0_DBG_NOC_SM	178
-+#define CLK_DOUT_CMU_DIV_CPUCL0_SWITCH		179
-+#define CLK_DOUT_CMU_DIV_CPUCL0_SWITCH_SM	180
-+#define CLK_DOUT_CMU_DIV_CPUCL1_SWITCH		181
-+#define CLK_DOUT_CMU_DIV_CPUCL1_SWITCH_SM	182
-+#define CLK_DOUT_CMU_DIV_CPUCL2_SWITCH		183
-+#define CLK_DOUT_CMU_DIV_CPUCL2_SWITCH_SM	184
-+#define CLK_DOUT_CMU_DIV_DNC_NOC		185
-+#define CLK_DOUT_CMU_DIV_DNC_NOC_SM		186
-+#define CLK_DOUT_CMU_DIV_DPUB			187
-+#define CLK_DOUT_CMU_DIV_DPUB_ALT		188
-+#define CLK_DOUT_CMU_DIV_DPUF			189
-+#define CLK_DOUT_CMU_DIV_DPUF_ALT		190
-+#define CLK_DOUT_CMU_DIV_DSP_NOC		191
-+#define CLK_DOUT_CMU_DIV_DSP_NOC_SM		192
-+#define CLK_DOUT_CMU_DIV_DSU_SWITCH		193
-+#define CLK_DOUT_CMU_DIV_DSU_SWITCH_SM		194
-+#define CLK_DOUT_CMU_DIV_G3D_SWITCH		195
-+#define CLK_DOUT_CMU_DIV_G3D_SWITCH_SM		196
-+#define CLK_DOUT_CMU_DIV_GNPU_NOC		197
-+#define CLK_DOUT_CMU_DIV_GNPU_NOC_SM		198
-+#define CLK_DOUT_CMU_DIV_UFS_MMC_CARD		199
-+#define CLK_DOUT_CMU_DIV_UFS_MMC_CARD_SM	200
-+#define CLK_DOUT_CMU_DIV_M2M_NOC		201
-+#define CLK_DOUT_CMU_DIV_M2M_NOC_SM		202
-+#define CLK_DOUT_CMU_DIV_NOCL0_NOC		203
-+#define CLK_DOUT_CMU_DIV_NOCL0_NOC_SM		204
-+#define CLK_DOUT_CMU_DIV_NOCL1A_NOC		205
-+#define CLK_DOUT_CMU_DIV_NOCL1A_NOC_SM		206
-+#define CLK_DOUT_CMU_DIV_NOCL1B_NOC0		207
-+#define CLK_DOUT_CMU_DIV_NOCL1B_NOC0_SM		208
-+#define CLK_DOUT_CMU_DIV_NOCL1C_NOC		209
-+#define CLK_DOUT_CMU_DIV_NOCL1C_NOC_SM		210
-+#define CLK_DOUT_CMU_DIV_SDMA_NOC		211
-+#define CLK_DOUT_CMU_DIV_SDMA_NOC_SM		212
-+#define CLK_DOUT_CMU_DIV_CP_HISPEEDY_CLK	213
-+#define CLK_DOUT_CMU_DIV_CP_HISPEEDY_CLK_SM	214
-+#define CLK_DOUT_CMU_DIV_CP_SHARED0_CLK		215
-+#define CLK_DOUT_CMU_DIV_CP_SHARED0_CLK_SM	216
-+#define CLK_DOUT_CMU_DIV_CP_SHARED2_CLK		217
-+#define CLK_DOUT_CMU_DIV_CP_SHARED2_CLK_SM	218
-+#define CLK_DOUT_CMU_UFS_NOC		219
-+#define CLK_DOUT_CMU_DIV_M2M_FRC		220
-+#define CLK_DOUT_CMU_DIV_M2M_FRC_SM		221
-+#define CLK_DOUT_CMU_DIV_MCSC_MCSC		222
-+#define CLK_DOUT_CMU_DIV_MCSC_MCSC_SM		223
-+#define CLK_DOUT_CMU_DIV_MCSC_NOC		224
-+#define CLK_DOUT_CMU_DIV_MCSC_NOC_SM		225
-+#define CLK_DOUT_SHARED0_DIV1			226
-+#define CLK_DOUT_SHARED0_DIV2			227
-+#define CLK_DOUT_SHARED0_DIV4			228
-+#define CLK_DOUT_SHARED1_DIV1			229
-+#define CLK_DOUT_SHARED1_DIV2			230
-+#define CLK_DOUT_SHARED1_DIV4			231
-+#define CLK_DOUT_SHARED2_DIV1			232
-+#define CLK_DOUT_SHARED2_DIV2			233
-+#define CLK_DOUT_SHARED2_DIV4			234
-+#define CLK_DOUT_SHARED3_DIV1			235
-+#define CLK_DOUT_SHARED3_DIV2			236
-+#define CLK_DOUT_SHARED3_DIV4			237
-+#define CLK_DOUT_SHARED4_DIV1			238
-+#define CLK_DOUT_SHARED4_DIV2			239
-+#define CLK_DOUT_SHARED4_DIV4			240
-+#define CLK_DOUT_SHARED_MIF_DIV1		241
-+#define CLK_DOUT_SHARED_MIF_DIV2		242
-+#define CLK_DOUT_SHARED_MIF_DIV4		243
-+#define CLK_DOUT_TCXO_DIV3			244
-+#define CLK_DOUT_TCXO_DIV4			245
-+
-+/* CMU_ALIVE */
-+#define CLK_MOUT_ALIVE_NOC_USER			1
-+#define CLK_MOUT_ALIVE_RCO_SPMI_USER		2
-+#define CLK_MOUT_RCO_ALIVE_USER			3
-+#define CLK_MOUT_ALIVE_CHUB_PERI		4
-+#define CLK_MOUT_ALIVE_CMGP_NOC			5
-+#define CLK_MOUT_ALIVE_CMGP_PERI		6
-+#define CLK_MOUT_ALIVE_DBGCORE_NOC		7
-+#define CLK_MOUT_ALIVE_DNC_NOC			8
-+#define CLK_MOUT_ALIVE_CHUBVTS_NOC		9
-+#define CLK_MOUT_ALIVE_GNPU_NOC			10
-+#define CLK_MOUT_ALIVE_GNSS_NOC			11
-+#define CLK_MOUT_ALIVE_SDMA_NOC			12
-+#define CLK_MOUT_ALIVE_UFD_NOC			13
-+#define CLK_MOUT_ALIVE_DBGCORE_UART		14
-+#define CLK_MOUT_ALIVE_NOC			15
-+#define CLK_MOUT_ALIVE_PMU_SUB			16
-+#define CLK_MOUT_ALIVE_SPMI			17
-+#define CLK_MOUT_ALIVE_TIMER			18
-+#define CLK_MOUT_ALIVE_CSIS_NOC			19
-+#define CLK_MOUT_ALIVE_DSP_NOC			20
-+
-+#define CLK_DOUT_ALIVE_CHUB_PERI		21
-+#define CLK_DOUT_ALIVE_CMGP_NOC			22
-+#define CLK_DOUT_ALIVE_CMGP_PERI		23
-+#define CLK_DOUT_ALIVE_DBGCORE_NOC		24
-+#define CLK_DOUT_ALIVE_DNC_NOC			25
-+#define CLK_DOUT_ALIVE_CHUBVTS_NOC		26
-+#define CLK_DOUT_ALIVE_GNPU_NOC			27
-+#define CLK_DOUT_ALIVE_SDMA_NOC			28
-+#define CLK_DOUT_ALIVE_UFD_NOC			29
-+#define CLK_DOUT_ALIVE_DBGCORE_UART		30
-+#define CLK_DOUT_ALIVE_NOC			31
-+#define CLK_DOUT_ALIVE_PMU_SUB			32
-+#define CLK_DOUT_ALIVE_SPMI			33
-+#define CLK_DOUT_ALIVE_CSIS_NOC			34
-+#define CLK_DOUT_ALIVE_DSP_NOC			35
-+
-+/* CMU_PERIS */
-+#define CLK_MOUT_PERIS_GIC_USER			1
-+#define CLK_MOUT_PERIS_NOC_USER			2
-+#define CLK_MOUT_PERIS_GIC			3
-+
-+#define CLK_DOUT_PERIS_OTP			4
-+#define CLK_DOUT_PERIS_DDD_CTRL			5
-+
-+/* CMU_CMGP */
-+#define CLK_MOUT_CMGP_CLKALIVE_NOC_USER		1
-+#define CLK_MOUT_CMGP_CLKALIVE_PERI_USER	2
-+#define CLK_MOUT_CMGP_I2C			3
-+#define CLK_MOUT_CMGP_SPI_I2C0			4
-+#define CLK_MOUT_CMGP_SPI_I2C1			5
-+#define CLK_MOUT_CMGP_SPI_MS_CTRL		6
-+#define CLK_MOUT_CMGP_USI0			7
-+#define CLK_MOUT_CMGP_USI1			8
-+#define CLK_MOUT_CMGP_USI2			9
-+#define CLK_MOUT_CMGP_USI3			10
-+#define CLK_MOUT_CMGP_USI4			11
-+#define CLK_MOUT_CMGP_USI5			12
-+#define CLK_MOUT_CMGP_USI6			13
-+
-+#define CLK_DOUT_CMGP_I2C			14
-+#define CLK_DOUT_CMGP_SPI_I2C0			15
-+#define CLK_DOUT_CMGP_SPI_I2C1			16
-+#define CLK_DOUT_CMGP_SPI_MS_CTRL		17
-+#define CLK_DOUT_CMGP_USI0			18
-+#define CLK_DOUT_CMGP_USI1			19
-+#define CLK_DOUT_CMGP_USI2			20
-+#define CLK_DOUT_CMGP_USI3			21
-+#define CLK_DOUT_CMGP_USI4			22
-+#define CLK_DOUT_CMGP_USI5			23
-+#define CLK_DOUT_CMGP_USI6			24
-+
-+/* CMU_HSI0 */
-+#define CLK_MOUT_CLKCMU_HSI0_DPGTC_USER		1
-+#define CLK_MOUT_CLKCMU_HSI0_DPOSC_USER		2
-+#define CLK_MOUT_CLKCMU_HSI0_NOC_USER		3
-+#define CLK_MOUT_CLKCMU_HSI0_USB32DRD_USER	4
-+#define CLK_MOUT_HSI0_NOC			5
-+#define CLK_MOUT_HSI0_RTCCLK			6
-+#define CLK_MOUT_HSI0_USB32DRD			7
-+
-+#define CLK_DOUT_DIV_CLK_HSI0_EUSB		8
-+
-+/* CMU_PERIC0 */
-+#define CLK_MOUT_PERIC0_IP0_USER		1
-+#define CLK_MOUT_PERIC0_IP1_USER		2
-+#define CLK_MOUT_PERIC0_NOC_USER		3
-+#define CLK_MOUT_PERIC0_I2C			4
-+#define CLK_MOUT_PERIC0_USI04			5
-+
-+#define CLK_DOUT_PERIC0_I2C			6
-+#define CLK_DOUT_PERIC0_USI04			7
-+
-+/* CMU_PERIC1 */
-+#define CLK_MOUT_PERIC1_IP0_USER		1
-+#define CLK_MOUT_PERIC1_IP1_USER		2
-+#define CLK_MOUT_PERIC1_NOC_USER		3
-+#define CLK_MOUT_PERIC1_I2C			4
-+#define CLK_MOUT_PERIC1_SPI_MS_CTRL		5
-+#define CLK_MOUT_PERIC1_UART_BT			6
-+#define CLK_MOUT_PERIC1_USI07			7
-+#define CLK_MOUT_PERIC1_USI07_SPI_I2C		8
-+#define CLK_MOUT_PERIC1_USI08			9
-+#define CLK_MOUT_PERIC1_USI08_SPI_I2C		10
-+#define CLK_MOUT_PERIC1_USI09			11
-+#define CLK_MOUT_PERIC1_USI10			12
-+
-+#define CLK_DOUT_PERIC1_I2C			13
-+#define CLK_DOUT_PERIC1_SPI_MS_CTRL		14
-+#define CLK_DOUT_PERIC1_UART_BT			15
-+#define CLK_DOUT_PERIC1_USI07			16
-+#define CLK_DOUT_PERIC1_USI07_SPI_I2C		17
-+#define CLK_DOUT_PERIC1_USI08			18
-+#define CLK_DOUT_PERIC1_USI08_SPI_I2C		19
-+#define CLK_DOUT_PERIC1_USI09			20
-+#define CLK_DOUT_PERIC1_USI10			21
-+
-+/* CMU_PERIC2 */
-+#define CLK_MOUT_PERIC2_IP0_USER		1
-+#define CLK_MOUT_PERIC2_IP1_USER		2
-+#define CLK_MOUT_PERIC2_NOC_USER		3
-+#define CLK_MOUT_PERIC2_I2C			4
-+#define CLK_MOUT_PERIC2_SPI_MS_CTRL		5
-+#define CLK_MOUT_PERIC2_UART_DBG		6
-+#define CLK_MOUT_PERIC2_USI00			7
-+#define CLK_MOUT_PERIC2_USI00_SPI_I2C		8
-+#define CLK_MOUT_PERIC2_USI01			9
-+#define CLK_MOUT_PERIC2_USI01_SPI_I2C		10
-+#define CLK_MOUT_PERIC2_USI02			11
-+#define CLK_MOUT_PERIC2_USI03			12
-+#define CLK_MOUT_PERIC2_USI05			13
-+#define CLK_MOUT_PERIC2_USI06			14
-+#define CLK_MOUT_PERIC2_USI11			15
-+
-+#define CLK_DOUT_PERIC2_I2C			16
-+#define CLK_DOUT_PERIC2_SPI_MS_CTRL		17
-+#define CLK_DOUT_PERIC2_UART_DBG		18
-+#define CLK_DOUT_PERIC2_USI00			19
-+#define CLK_DOUT_PERIC2_USI00_SPI_I2C		20
-+#define CLK_DOUT_PERIC2_USI01			21
-+#define CLK_DOUT_PERIC2_USI01_SPI_I2C		22
-+#define CLK_DOUT_PERIC2_USI02			23
-+#define CLK_DOUT_PERIC2_USI03			24
-+#define CLK_DOUT_PERIC2_USI05			25
-+#define CLK_DOUT_PERIC2_USI06			26
-+#define CLK_DOUT_PERIC2_USI11			27
-+
-+/* CMU_UFS */
-+#define CLK_MOUT_UFS_MMC_CARD_USER		1
-+#define CLK_MOUT_UFS_NOC_USER			2
-+#define CLK_MOUT_UFS_UFS_EMBD_USER		3
-+
-+/* CMU_VTS */
-+#define CLK_MOUT_CLKALIVE_VTS_NOC_USER		1
-+#define CLK_MOUT_CLKALIVE_VTS_RCO_USER		2
-+#define CLK_MOUT_CLKCMU_VTS_DMIC_USER		3
-+#define CLK_MOUT_CLKVTS_AUD_DMIC1		4
-+#define CLK_MOUT_CLKVTS_NOC			5
-+#define CLK_MOUT_CLKVTS_DMIC_PAD		6
-+
-+#define CLK_DOUT_CLKVTS_AUD_DMIC0		7
-+#define CLK_DOUT_CLKVTS_AUD_DMIC1		8
-+#define CLK_DOUT_CLKVTS_CPU			9
-+#define CLK_DOUT_CLKVTS_DMIC_IF			10
-+#define CLK_DOUT_CLKVTS_DMIC_IF_DIV2		11
-+#define CLK_DOUT_CLKVTS_NOC			12
-+#define CLK_DOUT_CLKVTS_SERIAL_LIF		13
-+#define CLK_DOUT_CLKVTS_SERIAL_LIF_CORE		14
-+
-+#endif
-diff --git a/include/dt-bindings/clock/samsung,exynos7870-cmu.h b/include/dt-bindings/clock/samsung,exynos7870-cmu.h
-new file mode 100644
-index 000000000000..57d04bbe342d
--- /dev/null
-+++ b/include/dt-bindings/clock/samsung,exynos7870-cmu.h
-@@ -0,0 +1,324 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2015 Samsung Electronics Co., Ltd.
-+ * Author: Kaustabh Chakraborty <kauschluss@disroot.org>
-+ *
-+ * Device Tree binding constants for Exynos7870 clock controller.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLOCK_EXYNOS7870_H
-+#define _DT_BINDINGS_CLOCK_EXYNOS7870_H
-+
-+/* CMU_MIF */
-+#define CLK_DOUT_MIF_APB				1
-+#define CLK_DOUT_MIF_BUSD				2
-+#define CLK_DOUT_MIF_CMU_DISPAUD_BUS			3
-+#define CLK_DOUT_MIF_CMU_DISPAUD_DECON_ECLK		4
-+#define CLK_DOUT_MIF_CMU_DISPAUD_DECON_VCLK		5
-+#define CLK_DOUT_MIF_CMU_FSYS_BUS			6
-+#define CLK_DOUT_MIF_CMU_FSYS_MMC0			7
-+#define CLK_DOUT_MIF_CMU_FSYS_MMC1			8
-+#define CLK_DOUT_MIF_CMU_FSYS_MMC2			9
-+#define CLK_DOUT_MIF_CMU_FSYS_USB20DRD_REFCLK		10
-+#define CLK_DOUT_MIF_CMU_G3D_SWITCH			11
-+#define CLK_DOUT_MIF_CMU_ISP_CAM			12
-+#define CLK_DOUT_MIF_CMU_ISP_ISP			13
-+#define CLK_DOUT_MIF_CMU_ISP_SENSOR0			14
-+#define CLK_DOUT_MIF_CMU_ISP_SENSOR1			15
-+#define CLK_DOUT_MIF_CMU_ISP_SENSOR2			16
-+#define CLK_DOUT_MIF_CMU_ISP_VRA			17
-+#define CLK_DOUT_MIF_CMU_MFCMSCL_MFC			18
-+#define CLK_DOUT_MIF_CMU_MFCMSCL_MSCL			19
-+#define CLK_DOUT_MIF_CMU_PERI_BUS			20
-+#define CLK_DOUT_MIF_CMU_PERI_SPI0			21
-+#define CLK_DOUT_MIF_CMU_PERI_SPI1			22
-+#define CLK_DOUT_MIF_CMU_PERI_SPI2			23
-+#define CLK_DOUT_MIF_CMU_PERI_SPI3			24
-+#define CLK_DOUT_MIF_CMU_PERI_SPI4			25
-+#define CLK_DOUT_MIF_CMU_PERI_UART0			26
-+#define CLK_DOUT_MIF_CMU_PERI_UART1			27
-+#define CLK_DOUT_MIF_CMU_PERI_UART2			28
-+#define CLK_DOUT_MIF_HSI2C				29
-+#define CLK_FOUT_MIF_BUS_PLL				30
-+#define CLK_FOUT_MIF_MEDIA_PLL				31
-+#define CLK_FOUT_MIF_MEM_PLL				32
-+#define CLK_GOUT_MIF_CMU_DISPAUD_BUS			33
-+#define CLK_GOUT_MIF_CMU_DISPAUD_DECON_ECLK		34
-+#define CLK_GOUT_MIF_CMU_DISPAUD_DECON_VCLK		35
-+#define CLK_GOUT_MIF_CMU_FSYS_BUS			36
-+#define CLK_GOUT_MIF_CMU_FSYS_MMC0			37
-+#define CLK_GOUT_MIF_CMU_FSYS_MMC1			38
-+#define CLK_GOUT_MIF_CMU_FSYS_MMC2			39
-+#define CLK_GOUT_MIF_CMU_FSYS_USB20DRD_REFCLK		40
-+#define CLK_GOUT_MIF_CMU_G3D_SWITCH			41
-+#define CLK_GOUT_MIF_CMU_ISP_CAM			42
-+#define CLK_GOUT_MIF_CMU_ISP_ISP			43
-+#define CLK_GOUT_MIF_CMU_ISP_SENSOR0			44
-+#define CLK_GOUT_MIF_CMU_ISP_SENSOR1			45
-+#define CLK_GOUT_MIF_CMU_ISP_SENSOR2			46
-+#define CLK_GOUT_MIF_CMU_ISP_VRA			47
-+#define CLK_GOUT_MIF_CMU_MFCMSCL_MFC			48
-+#define CLK_GOUT_MIF_CMU_MFCMSCL_MSCL			49
-+#define CLK_GOUT_MIF_CMU_PERI_BUS			50
-+#define CLK_GOUT_MIF_CMU_PERI_SPI0			51
-+#define CLK_GOUT_MIF_CMU_PERI_SPI1			52
-+#define CLK_GOUT_MIF_CMU_PERI_SPI2			53
-+#define CLK_GOUT_MIF_CMU_PERI_SPI3			54
-+#define CLK_GOUT_MIF_CMU_PERI_SPI4			55
-+#define CLK_GOUT_MIF_CMU_PERI_UART0			56
-+#define CLK_GOUT_MIF_CMU_PERI_UART1			57
-+#define CLK_GOUT_MIF_CMU_PERI_UART2			58
-+#define CLK_GOUT_MIF_CP_PCLK_HSI2C			59
-+#define CLK_GOUT_MIF_CP_PCLK_HSI2C_BAT_0		60
-+#define CLK_GOUT_MIF_CP_PCLK_HSI2C_BAT_1		61
-+#define CLK_GOUT_MIF_HSI2C_AP_PCLKM			62
-+#define CLK_GOUT_MIF_HSI2C_AP_PCLKS			63
-+#define CLK_GOUT_MIF_HSI2C_CP_PCLKM			64
-+#define CLK_GOUT_MIF_HSI2C_CP_PCLKS			65
-+#define CLK_GOUT_MIF_HSI2C_IPCLK			66
-+#define CLK_GOUT_MIF_HSI2C_ITCLK			67
-+#define CLK_GOUT_MIF_MUX_BUSD				68
-+#define CLK_GOUT_MIF_MUX_BUS_PLL			69
-+#define CLK_GOUT_MIF_MUX_BUS_PLL_CON			70
-+#define CLK_GOUT_MIF_MUX_CMU_DISPAUD_BUS		71
-+#define CLK_GOUT_MIF_MUX_CMU_DISPAUD_DECON_ECLK		72
-+#define CLK_GOUT_MIF_MUX_CMU_DISPAUD_DECON_VCLK		73
-+#define CLK_GOUT_MIF_MUX_CMU_FSYS_BUS			74
-+#define CLK_GOUT_MIF_MUX_CMU_FSYS_MMC0			75
-+#define CLK_GOUT_MIF_MUX_CMU_FSYS_MMC1			76
-+#define CLK_GOUT_MIF_MUX_CMU_FSYS_MMC2			77
-+#define CLK_GOUT_MIF_MUX_CMU_FSYS_USB20DRD_REFCLK	78
-+#define CLK_GOUT_MIF_MUX_CMU_ISP_CAM			79
-+#define CLK_GOUT_MIF_MUX_CMU_ISP_ISP			80
-+#define CLK_GOUT_MIF_MUX_CMU_ISP_SENSOR0		81
-+#define CLK_GOUT_MIF_MUX_CMU_ISP_SENSOR1		82
-+#define CLK_GOUT_MIF_MUX_CMU_ISP_SENSOR2		83
-+#define CLK_GOUT_MIF_MUX_CMU_ISP_VRA			84
-+#define CLK_GOUT_MIF_MUX_CMU_MFCMSCL_MFC		85
-+#define CLK_GOUT_MIF_MUX_CMU_MFCMSCL_MSCL		86
-+#define CLK_GOUT_MIF_MUX_CMU_PERI_BUS			87
-+#define CLK_GOUT_MIF_MUX_CMU_PERI_SPI0			88
-+#define CLK_GOUT_MIF_MUX_CMU_PERI_SPI1			89
-+#define CLK_GOUT_MIF_MUX_CMU_PERI_SPI2			90
-+#define CLK_GOUT_MIF_MUX_CMU_PERI_SPI3			91
-+#define CLK_GOUT_MIF_MUX_CMU_PERI_SPI4			92
-+#define CLK_GOUT_MIF_MUX_CMU_PERI_UART0			93
-+#define CLK_GOUT_MIF_MUX_CMU_PERI_UART1			94
-+#define CLK_GOUT_MIF_MUX_CMU_PERI_UART2			95
-+#define CLK_GOUT_MIF_MUX_MEDIA_PLL			96
-+#define CLK_GOUT_MIF_MUX_MEDIA_PLL_CON			97
-+#define CLK_GOUT_MIF_MUX_MEM_PLL			98
-+#define CLK_GOUT_MIF_MUX_MEM_PLL_CON			99
-+#define CLK_GOUT_MIF_WRAP_ADC_IF_OSC_SYS		100
-+#define CLK_GOUT_MIF_WRAP_ADC_IF_PCLK_S0		101
-+#define CLK_GOUT_MIF_WRAP_ADC_IF_PCLK_S1		102
-+#define CLK_MOUT_MIF_BUSD				103
-+#define CLK_MOUT_MIF_CMU_DISPAUD_BUS			104
-+#define CLK_MOUT_MIF_CMU_DISPAUD_DECON_ECLK		105
-+#define CLK_MOUT_MIF_CMU_DISPAUD_DECON_VCLK		106
-+#define CLK_MOUT_MIF_CMU_FSYS_BUS			107
-+#define CLK_MOUT_MIF_CMU_FSYS_MMC0			108
-+#define CLK_MOUT_MIF_CMU_FSYS_MMC1			109
-+#define CLK_MOUT_MIF_CMU_FSYS_MMC2			110
-+#define CLK_MOUT_MIF_CMU_FSYS_USB20DRD_REFCLK		111
-+#define CLK_MOUT_MIF_CMU_ISP_CAM			112
-+#define CLK_MOUT_MIF_CMU_ISP_ISP			113
-+#define CLK_MOUT_MIF_CMU_ISP_SENSOR0			114
-+#define CLK_MOUT_MIF_CMU_ISP_SENSOR1			115
-+#define CLK_MOUT_MIF_CMU_ISP_SENSOR2			116
-+#define CLK_MOUT_MIF_CMU_ISP_VRA			117
-+#define CLK_MOUT_MIF_CMU_MFCMSCL_MFC			118
-+#define CLK_MOUT_MIF_CMU_MFCMSCL_MSCL			119
-+#define CLK_MOUT_MIF_CMU_PERI_BUS			120
-+#define CLK_MOUT_MIF_CMU_PERI_SPI0			121
-+#define CLK_MOUT_MIF_CMU_PERI_SPI1			122
-+#define CLK_MOUT_MIF_CMU_PERI_SPI2			123
-+#define CLK_MOUT_MIF_CMU_PERI_SPI3			124
-+#define CLK_MOUT_MIF_CMU_PERI_SPI4			125
-+#define CLK_MOUT_MIF_CMU_PERI_UART0			126
-+#define CLK_MOUT_MIF_CMU_PERI_UART1			127
-+#define CLK_MOUT_MIF_CMU_PERI_UART2			128
-+#define MIF_NR_CLK					129
-+
-+/* CMU_DISPAUD */
-+#define CLK_DOUT_DISPAUD_APB					1
-+#define CLK_DOUT_DISPAUD_DECON_ECLK				2
-+#define CLK_DOUT_DISPAUD_DECON_VCLK				3
-+#define CLK_DOUT_DISPAUD_MI2S					4
-+#define CLK_DOUT_DISPAUD_MIXER					5
-+#define CLK_FOUT_DISPAUD_AUD_PLL				6
-+#define CLK_FOUT_DISPAUD_PLL					7
-+#define CLK_GOUT_DISPAUD_APB_AUD				8
-+#define CLK_GOUT_DISPAUD_APB_AUD_AMP				9
-+#define CLK_GOUT_DISPAUD_APB_DISP				10
-+#define CLK_GOUT_DISPAUD_BUS					11
-+#define CLK_GOUT_DISPAUD_BUS_DISP				12
-+#define CLK_GOUT_DISPAUD_BUS_PPMU				13
-+#define CLK_GOUT_DISPAUD_CON_AUD_I2S_BCLK_BT_IN			14
-+#define CLK_GOUT_DISPAUD_CON_AUD_I2S_BCLK_FM_IN			15
-+#define CLK_GOUT_DISPAUD_CON_CP2AUD_BCK				16
-+#define CLK_GOUT_DISPAUD_CON_EXT2AUD_BCK_GPIO_I2S		17
-+#define CLK_GOUT_DISPAUD_DECON_ECLK				18
-+#define CLK_GOUT_DISPAUD_DECON_VCLK				19
-+#define CLK_GOUT_DISPAUD_MI2S_AMP_I2SCODCLKI			20
-+#define CLK_GOUT_DISPAUD_MI2S_AUD_I2SCODCLKI			21
-+#define CLK_GOUT_DISPAUD_MIXER_AUD_SYSCLK			22
-+#define CLK_GOUT_DISPAUD_MUX_AUD_PLL				23
-+#define CLK_GOUT_DISPAUD_MUX_AUD_PLL_CON			24
-+#define CLK_GOUT_DISPAUD_MUX_BUS_USER				25
-+#define CLK_GOUT_DISPAUD_MUX_DECON_ECLK				26
-+#define CLK_GOUT_DISPAUD_MUX_DECON_ECLK_USER			27
-+#define CLK_GOUT_DISPAUD_MUX_DECON_VCLK				28
-+#define CLK_GOUT_DISPAUD_MUX_DECON_VCLK_USER			29
-+#define CLK_GOUT_DISPAUD_MUX_MI2S				30
-+#define CLK_GOUT_DISPAUD_MUX_MIPIPHY_RXCLKESC0_USER		31
-+#define CLK_GOUT_DISPAUD_MUX_MIPIPHY_RXCLKESC0_USER_CON		32
-+#define CLK_GOUT_DISPAUD_MUX_MIPIPHY_TXBYTECLKHS_USER		33
-+#define CLK_GOUT_DISPAUD_MUX_MIPIPHY_TXBYTECLKHS_USER_CON	34
-+#define CLK_GOUT_DISPAUD_MUX_PLL				35
-+#define CLK_GOUT_DISPAUD_MUX_PLL_CON				36
-+#define CLK_MOUT_DISPAUD_BUS_USER				37
-+#define CLK_MOUT_DISPAUD_DECON_ECLK				38
-+#define CLK_MOUT_DISPAUD_DECON_ECLK_USER			39
-+#define CLK_MOUT_DISPAUD_DECON_VCLK				40
-+#define CLK_MOUT_DISPAUD_DECON_VCLK_USER			41
-+#define CLK_MOUT_DISPAUD_MI2S					42
-+#define DISPAUD_NR_CLK						43
-+
-+/* CMU_FSYS */
-+#define CLK_FOUT_FSYS_USB_PLL				1
-+#define CLK_GOUT_FSYS_BUSP3_HCLK			2
-+#define CLK_GOUT_FSYS_MMC0_ACLK				3
-+#define CLK_GOUT_FSYS_MMC1_ACLK				4
-+#define CLK_GOUT_FSYS_MMC2_ACLK				5
-+#define CLK_GOUT_FSYS_MUX_USB20DRD_PHYCLOCK_USER	6
-+#define CLK_GOUT_FSYS_MUX_USB20DRD_PHYCLOCK_USER_CON	7
-+#define CLK_GOUT_FSYS_MUX_USB_PLL			8
-+#define CLK_GOUT_FSYS_MUX_USB_PLL_CON			9
-+#define CLK_GOUT_FSYS_PDMA0_ACLK_PDMA0			10
-+#define CLK_GOUT_FSYS_PPMU_ACLK				11
-+#define CLK_GOUT_FSYS_PPMU_PCLK				12
-+#define CLK_GOUT_FSYS_SROMC_HCLK			13
-+#define CLK_GOUT_FSYS_UPSIZER_BUS1_ACLK			14
-+#define CLK_GOUT_FSYS_USB20DRD_ACLK_HSDRD		15
-+#define CLK_GOUT_FSYS_USB20DRD_HCLK_USB20_CTRL		16
-+#define CLK_GOUT_FSYS_USB20DRD_HSDRD_REF_CLK		17
-+#define FSYS_NR_CLK					18
-+
-+/* CMU_G3D */
-+#define CLK_DOUT_G3D_APB		1
-+#define CLK_DOUT_G3D_BUS		2
-+#define CLK_FOUT_G3D_PLL		3
-+#define CLK_GOUT_G3D_ASYNCS_D0_CLK	4
-+#define CLK_GOUT_G3D_ASYNC_PCLKM	5
-+#define CLK_GOUT_G3D_CLK		6
-+#define CLK_GOUT_G3D_MUX		7
-+#define CLK_GOUT_G3D_MUX_PLL		8
-+#define CLK_GOUT_G3D_MUX_PLL_CON	9
-+#define CLK_GOUT_G3D_MUX_SWITCH_USER	10
-+#define CLK_GOUT_G3D_PPMU_ACLK		11
-+#define CLK_GOUT_G3D_PPMU_PCLK		12
-+#define CLK_GOUT_G3D_QE_ACLK		13
-+#define CLK_GOUT_G3D_QE_PCLK		14
-+#define CLK_GOUT_G3D_SYSREG_PCLK	15
-+#define CLK_MOUT_G3D			16
-+#define CLK_MOUT_G3D_SWITCH_USER	17
-+#define G3D_NR_CLK			18
-+
-+/* CMU_ISP */
-+#define CLK_DOUT_ISP_APB				1
-+#define CLK_DOUT_ISP_CAM_HALF				2
-+#define CLK_FOUT_ISP_PLL				3
-+#define CLK_GOUT_ISP_CAM				4
-+#define CLK_GOUT_ISP_CAM_HALF				5
-+#define CLK_GOUT_ISP_ISPD				6
-+#define CLK_GOUT_ISP_ISPD_PPMU				7
-+#define CLK_GOUT_ISP_MUX_CAM				8
-+#define CLK_GOUT_ISP_MUX_CAM_USER			9
-+#define CLK_GOUT_ISP_MUX_ISP				10
-+#define CLK_GOUT_ISP_MUX_ISPD				11
-+#define CLK_GOUT_ISP_MUX_PLL				12
-+#define CLK_GOUT_ISP_MUX_PLL_CON			13
-+#define CLK_GOUT_ISP_MUX_RXBYTECLKHS0_SENSOR0_USER	14
-+#define CLK_GOUT_ISP_MUX_RXBYTECLKHS0_SENSOR0_USER_CON	15
-+#define CLK_GOUT_ISP_MUX_RXBYTECLKHS0_SENSOR1_USER	16
-+#define CLK_GOUT_ISP_MUX_RXBYTECLKHS0_SENSOR1_USER_CON	17
-+#define CLK_GOUT_ISP_MUX_USER				18
-+#define CLK_GOUT_ISP_MUX_VRA				19
-+#define CLK_GOUT_ISP_MUX_VRA_USER			20
-+#define CLK_GOUT_ISP_VRA				21
-+#define CLK_MOUT_ISP_CAM				22
-+#define CLK_MOUT_ISP_CAM_USER				23
-+#define CLK_MOUT_ISP_ISP				24
-+#define CLK_MOUT_ISP_ISPD				25
-+#define CLK_MOUT_ISP_USER				26
-+#define CLK_MOUT_ISP_VRA				27
-+#define CLK_MOUT_ISP_VRA_USER				28
-+#define ISP_NR_CLK					29
-+
-+/* CMU_MFCMSCL */
-+#define CLK_DOUT_MFCMSCL_APB		1
-+#define CLK_GOUT_MFCMSCL_MFC		2
-+#define CLK_GOUT_MFCMSCL_MSCL		3
-+#define CLK_GOUT_MFCMSCL_MSCL_BI	4
-+#define CLK_GOUT_MFCMSCL_MSCL_D		5
-+#define CLK_GOUT_MFCMSCL_MSCL_JPEG	6
-+#define CLK_GOUT_MFCMSCL_MSCL_POLY	7
-+#define CLK_GOUT_MFCMSCL_MSCL_PPMU	8
-+#define CLK_GOUT_MFCMSCL_MUX_MFC_USER	9
-+#define CLK_GOUT_MFCMSCL_MUX_MSCL_USER	10
-+#define CLK_MOUT_MFCMSCL_MFC_USER	11
-+#define CLK_MOUT_MFCMSCL_MSCL_USER	12
-+#define MFCMSCL_NR_CLK			13
-+
-+/* CMU_PERI */
-+#define CLK_GOUT_PERI_BUSP1_PERIC0_HCLK		1
-+#define CLK_GOUT_PERI_GPIO2_PCLK		2
-+#define CLK_GOUT_PERI_GPIO5_PCLK		3
-+#define CLK_GOUT_PERI_GPIO6_PCLK		4
-+#define CLK_GOUT_PERI_GPIO7_PCLK		5
-+#define CLK_GOUT_PERI_HSI2C1_IPCLK		6
-+#define CLK_GOUT_PERI_HSI2C2_IPCLK		7
-+#define CLK_GOUT_PERI_HSI2C3_IPCLK		8
-+#define CLK_GOUT_PERI_HSI2C4_IPCLK		9
-+#define CLK_GOUT_PERI_HSI2C5_IPCLK		10
-+#define CLK_GOUT_PERI_HSI2C6_IPCLK		11
-+#define CLK_GOUT_PERI_I2C0_PCLK			12
-+#define CLK_GOUT_PERI_I2C1_PCLK			13
-+#define CLK_GOUT_PERI_I2C2_PCLK			14
-+#define CLK_GOUT_PERI_I2C3_PCLK			15
-+#define CLK_GOUT_PERI_I2C4_PCLK			16
-+#define CLK_GOUT_PERI_I2C5_PCLK			17
-+#define CLK_GOUT_PERI_I2C6_PCLK			18
-+#define CLK_GOUT_PERI_I2C7_PCLK			19
-+#define CLK_GOUT_PERI_I2C8_PCLK			20
-+#define CLK_GOUT_PERI_MCT_PCLK			21
-+#define CLK_GOUT_PERI_PWM_MOTOR_OSCCLK		22
-+#define CLK_GOUT_PERI_PWM_MOTOR_PCLK_S0		23
-+#define CLK_GOUT_PERI_SFRIF_TMU_CPUCL0_PCLK	24
-+#define CLK_GOUT_PERI_SFRIF_TMU_CPUCL1_PCLK	25
-+#define CLK_GOUT_PERI_SFRIF_TMU_PCLK		26
-+#define CLK_GOUT_PERI_SPI0_PCLK			27
-+#define CLK_GOUT_PERI_SPI0_SPI_EXT_CLK		28
-+#define CLK_GOUT_PERI_SPI1_PCLK			29
-+#define CLK_GOUT_PERI_SPI1_SPI_EXT_CLK		30
-+#define CLK_GOUT_PERI_SPI2_PCLK			31
-+#define CLK_GOUT_PERI_SPI2_SPI_EXT_CLK		32
-+#define CLK_GOUT_PERI_SPI3_PCLK			33
-+#define CLK_GOUT_PERI_SPI3_SPI_EXT_CLK		34
-+#define CLK_GOUT_PERI_SPI4_PCLK			35
-+#define CLK_GOUT_PERI_SPI4_SPI_EXT_CLK		36
-+#define CLK_GOUT_PERI_TMU_CLK			37
-+#define CLK_GOUT_PERI_TMU_CPUCL0_CLK		38
-+#define CLK_GOUT_PERI_TMU_CPUCL1_CLK		39
-+#define CLK_GOUT_PERI_UART0_EXT_UCLK		40
-+#define CLK_GOUT_PERI_UART0_PCLK		41
-+#define CLK_GOUT_PERI_UART1_EXT_UCLK		42
-+#define CLK_GOUT_PERI_UART1_PCLK		43
-+#define CLK_GOUT_PERI_UART2_EXT_UCLK		44
-+#define CLK_GOUT_PERI_UART2_PCLK		45
-+#define CLK_GOUT_PERI_WDT_CPUCL0_PCLK		46
-+#define CLK_GOUT_PERI_WDT_CPUCL1_PCLK		47
-+#define PERI_NR_CLK				48
-+
-+#endif /* _DT_BINDINGS_CLOCK_EXYNOS7870_H */
-diff --git a/include/dt-bindings/clock/samsung,exynos8895.h b/include/dt-bindings/clock/samsung,exynos8895.h
-new file mode 100644
-index 000000000000..27998c53f929
--- /dev/null
-+++ b/include/dt-bindings/clock/samsung,exynos8895.h
-@@ -0,0 +1,453 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2024 Ivaylo Ivanov <ivo.ivanov.ivanov1@gmail.com>
-+ * Author: Ivaylo Ivanov <ivo.ivanov.ivanov1@gmail.com>
-+ *
-+ * Device Tree binding constants for Exynos8895 clock controller.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLOCK_EXYNOS8895_H
-+#define _DT_BINDINGS_CLOCK_EXYNOS8895_H
-+
-+/* CMU_TOP */
-+#define CLK_FOUT_SHARED0_PLL			1
-+#define CLK_FOUT_SHARED1_PLL			2
-+#define CLK_FOUT_SHARED2_PLL			3
-+#define CLK_FOUT_SHARED3_PLL			4
-+#define CLK_FOUT_SHARED4_PLL			5
-+#define CLK_MOUT_PLL_SHARED0			6
-+#define CLK_MOUT_PLL_SHARED1			7
-+#define CLK_MOUT_PLL_SHARED2			8
-+#define CLK_MOUT_PLL_SHARED3			9
-+#define CLK_MOUT_PLL_SHARED4			10
-+#define CLK_MOUT_CP2AP_MIF_CLK_USER		11
-+#define CLK_MOUT_CMU_ABOX_CPUABOX		12
-+#define CLK_MOUT_CMU_APM_BUS			13
-+#define CLK_MOUT_CMU_BUS1_BUS			14
-+#define CLK_MOUT_CMU_BUSC_BUS			15
-+#define CLK_MOUT_CMU_BUSC_BUSPHSI2C		16
-+#define CLK_MOUT_CMU_CAM_BUS			17
-+#define CLK_MOUT_CMU_CAM_TPU0			18
-+#define CLK_MOUT_CMU_CAM_TPU1			19
-+#define CLK_MOUT_CMU_CAM_VRA			20
-+#define CLK_MOUT_CMU_CIS_CLK0			21
-+#define CLK_MOUT_CMU_CIS_CLK1			22
-+#define CLK_MOUT_CMU_CIS_CLK2			23
-+#define CLK_MOUT_CMU_CIS_CLK3			24
-+#define CLK_MOUT_CMU_CORE_BUS			25
-+#define CLK_MOUT_CMU_CPUCL0_SWITCH		26
-+#define CLK_MOUT_CMU_CPUCL1_SWITCH		27
-+#define CLK_MOUT_CMU_DBG_BUS			28
-+#define CLK_MOUT_CMU_DCAM_BUS			29
-+#define CLK_MOUT_CMU_DCAM_IMGD			30
-+#define CLK_MOUT_CMU_DPU_BUS			31
-+#define CLK_MOUT_CMU_DROOPDETECTOR		32
-+#define CLK_MOUT_CMU_DSP_BUS			33
-+#define CLK_MOUT_CMU_FSYS0_BUS			34
-+#define CLK_MOUT_CMU_FSYS0_DPGTC		35
-+#define CLK_MOUT_CMU_FSYS0_MMC_EMBD		36
-+#define CLK_MOUT_CMU_FSYS0_UFS_EMBD		37
-+#define CLK_MOUT_CMU_FSYS0_USBDRD30		38
-+#define CLK_MOUT_CMU_FSYS1_BUS			39
-+#define CLK_MOUT_CMU_FSYS1_MMC_CARD		40
-+#define CLK_MOUT_CMU_FSYS1_PCIE			41
-+#define CLK_MOUT_CMU_FSYS1_UFS_CARD		42
-+#define CLK_MOUT_CMU_G2D_G2D			43
-+#define CLK_MOUT_CMU_G2D_JPEG			44
-+#define CLK_MOUT_CMU_HPM			45
-+#define CLK_MOUT_CMU_IMEM_BUS			46
-+#define CLK_MOUT_CMU_ISPHQ_BUS			47
-+#define CLK_MOUT_CMU_ISPLP_BUS			48
-+#define CLK_MOUT_CMU_IVA_BUS			49
-+#define CLK_MOUT_CMU_MFC_BUS			50
-+#define CLK_MOUT_CMU_MIF_SWITCH			51
-+#define CLK_MOUT_CMU_PERIC0_BUS			52
-+#define CLK_MOUT_CMU_PERIC0_UART_DBG		53
-+#define CLK_MOUT_CMU_PERIC0_USI00		54
-+#define CLK_MOUT_CMU_PERIC0_USI01		55
-+#define CLK_MOUT_CMU_PERIC0_USI02		56
-+#define CLK_MOUT_CMU_PERIC0_USI03		57
-+#define CLK_MOUT_CMU_PERIC1_BUS			58
-+#define CLK_MOUT_CMU_PERIC1_SPEEDY2		59
-+#define CLK_MOUT_CMU_PERIC1_SPI_CAM0		60
-+#define CLK_MOUT_CMU_PERIC1_SPI_CAM1		61
-+#define CLK_MOUT_CMU_PERIC1_UART_BT		62
-+#define CLK_MOUT_CMU_PERIC1_USI04		63
-+#define CLK_MOUT_CMU_PERIC1_USI05		64
-+#define CLK_MOUT_CMU_PERIC1_USI06		65
-+#define CLK_MOUT_CMU_PERIC1_USI07		66
-+#define CLK_MOUT_CMU_PERIC1_USI08		67
-+#define CLK_MOUT_CMU_PERIC1_USI09		68
-+#define CLK_MOUT_CMU_PERIC1_USI10		69
-+#define CLK_MOUT_CMU_PERIC1_USI11		70
-+#define CLK_MOUT_CMU_PERIC1_USI12		71
-+#define CLK_MOUT_CMU_PERIC1_USI13		72
-+#define CLK_MOUT_CMU_PERIS_BUS			73
-+#define CLK_MOUT_CMU_SRDZ_BUS			74
-+#define CLK_MOUT_CMU_SRDZ_IMGD			75
-+#define CLK_MOUT_CMU_VPU_BUS			76
-+#define CLK_DOUT_CMU_ABOX_CPUABOX		77
-+#define CLK_DOUT_CMU_APM_BUS			78
-+#define CLK_DOUT_CMU_BUS1_BUS			79
-+#define CLK_DOUT_CMU_BUSC_BUS			80
-+#define CLK_DOUT_CMU_BUSC_BUSPHSI2C		81
-+#define CLK_DOUT_CMU_CAM_BUS			82
-+#define CLK_DOUT_CMU_CAM_TPU0			83
-+#define CLK_DOUT_CMU_CAM_TPU1			84
-+#define CLK_DOUT_CMU_CAM_VRA			85
-+#define CLK_DOUT_CMU_CIS_CLK0			86
-+#define CLK_DOUT_CMU_CIS_CLK1			87
-+#define CLK_DOUT_CMU_CIS_CLK2			88
-+#define CLK_DOUT_CMU_CIS_CLK3			89
-+#define CLK_DOUT_CMU_CORE_BUS			90
-+#define CLK_DOUT_CMU_CPUCL0_SWITCH		91
-+#define CLK_DOUT_CMU_CPUCL1_SWITCH		92
-+#define CLK_DOUT_CMU_DBG_BUS			93
-+#define CLK_DOUT_CMU_DCAM_BUS			94
-+#define CLK_DOUT_CMU_DCAM_IMGD			95
-+#define CLK_DOUT_CMU_DPU_BUS			96
-+#define CLK_DOUT_CMU_DSP_BUS			97
-+#define CLK_DOUT_CMU_FSYS0_BUS			98
-+#define CLK_DOUT_CMU_FSYS0_DPGTC		99
-+#define CLK_DOUT_CMU_FSYS0_MMC_EMBD		100
-+#define CLK_DOUT_CMU_FSYS0_UFS_EMBD		101
-+#define CLK_DOUT_CMU_FSYS0_USBDRD30		102
-+#define CLK_DOUT_CMU_FSYS1_BUS			103
-+#define CLK_DOUT_CMU_FSYS1_MMC_CARD		104
-+#define CLK_DOUT_CMU_FSYS1_UFS_CARD		105
-+#define CLK_DOUT_CMU_G2D_G2D			106
-+#define CLK_DOUT_CMU_G2D_JPEG			107
-+#define CLK_DOUT_CMU_G3D_SWITCH			108
-+#define CLK_DOUT_CMU_HPM			109
-+#define CLK_DOUT_CMU_IMEM_BUS			110
-+#define CLK_DOUT_CMU_ISPHQ_BUS			111
-+#define CLK_DOUT_CMU_ISPLP_BUS			112
-+#define CLK_DOUT_CMU_IVA_BUS			113
-+#define CLK_DOUT_CMU_MFC_BUS			114
-+#define CLK_DOUT_CMU_MODEM_SHARED0		115
-+#define CLK_DOUT_CMU_MODEM_SHARED1		116
-+#define CLK_DOUT_CMU_PERIC0_BUS			117
-+#define CLK_DOUT_CMU_PERIC0_UART_DBG		118
-+#define CLK_DOUT_CMU_PERIC0_USI00		119
-+#define CLK_DOUT_CMU_PERIC0_USI01		120
-+#define CLK_DOUT_CMU_PERIC0_USI02		121
-+#define CLK_DOUT_CMU_PERIC0_USI03		122
-+#define CLK_DOUT_CMU_PERIC1_BUS			123
-+#define CLK_DOUT_CMU_PERIC1_SPEEDY2		124
-+#define CLK_DOUT_CMU_PERIC1_SPI_CAM0		125
-+#define CLK_DOUT_CMU_PERIC1_SPI_CAM1		126
-+#define CLK_DOUT_CMU_PERIC1_UART_BT		127
-+#define CLK_DOUT_CMU_PERIC1_USI04		128
-+#define CLK_DOUT_CMU_PERIC1_USI05		129
-+#define CLK_DOUT_CMU_PERIC1_USI06		130
-+#define CLK_DOUT_CMU_PERIC1_USI07		131
-+#define CLK_DOUT_CMU_PERIC1_USI08		132
-+#define CLK_DOUT_CMU_PERIC1_USI09		133
-+#define CLK_DOUT_CMU_PERIC1_USI10		134
-+#define CLK_DOUT_CMU_PERIC1_USI11		135
-+#define CLK_DOUT_CMU_PERIC1_USI12		136
-+#define CLK_DOUT_CMU_PERIC1_USI13		137
-+#define CLK_DOUT_CMU_PERIS_BUS			138
-+#define CLK_DOUT_CMU_SRDZ_BUS			139
-+#define CLK_DOUT_CMU_SRDZ_IMGD			140
-+#define CLK_DOUT_CMU_VPU_BUS			141
-+#define CLK_DOUT_CMU_SHARED0_DIV2		142
-+#define CLK_DOUT_CMU_SHARED0_DIV4		143
-+#define CLK_DOUT_CMU_SHARED1_DIV2		144
-+#define CLK_DOUT_CMU_SHARED1_DIV4		145
-+#define CLK_DOUT_CMU_SHARED2_DIV2		146
-+#define CLK_DOUT_CMU_SHARED3_DIV2		147
-+#define CLK_DOUT_CMU_SHARED4_DIV2		148
-+#define CLK_DOUT_CMU_FSYS1_PCIE			149
-+#define CLK_DOUT_CMU_CP2AP_MIF_CLK_DIV2		150
-+#define CLK_DOUT_CMU_CMU_OTP			151
-+#define CLK_GOUT_CMU_DROOPDETECTOR		152
-+#define CLK_GOUT_CMU_MIF_SWITCH			153
-+#define CLK_GOUT_CMU_ABOX_CPUABOX		154
-+#define CLK_GOUT_CMU_APM_BUS			155
-+#define CLK_GOUT_CMU_BUS1_BUS			156
-+#define CLK_GOUT_CMU_BUSC_BUS			157
-+#define CLK_GOUT_CMU_BUSC_BUSPHSI2C		158
-+#define CLK_GOUT_CMU_CAM_BUS			159
-+#define CLK_GOUT_CMU_CAM_TPU0			160
-+#define CLK_GOUT_CMU_CAM_TPU1			161
-+#define CLK_GOUT_CMU_CAM_VRA			162
-+#define CLK_GOUT_CMU_CIS_CLK0			163
-+#define CLK_GOUT_CMU_CIS_CLK1			164
-+#define CLK_GOUT_CMU_CIS_CLK2			165
-+#define CLK_GOUT_CMU_CIS_CLK3			166
-+#define CLK_GOUT_CMU_CORE_BUS			167
-+#define CLK_GOUT_CMU_CPUCL0_SWITCH		168
-+#define CLK_GOUT_CMU_CPUCL1_SWITCH		169
-+#define CLK_GOUT_CMU_DBG_BUS			170
-+#define CLK_GOUT_CMU_DCAM_BUS			171
-+#define CLK_GOUT_CMU_DCAM_IMGD			172
-+#define CLK_GOUT_CMU_DPU_BUS			173
-+#define CLK_GOUT_CMU_DSP_BUS			174
-+#define CLK_GOUT_CMU_FSYS0_BUS			175
-+#define CLK_GOUT_CMU_FSYS0_DPGTC		176
-+#define CLK_GOUT_CMU_FSYS0_MMC_EMBD		177
-+#define CLK_GOUT_CMU_FSYS0_UFS_EMBD		178
-+#define CLK_GOUT_CMU_FSYS0_USBDRD30		179
-+#define CLK_GOUT_CMU_FSYS1_BUS			180
-+#define CLK_GOUT_CMU_FSYS1_MMC_CARD		181
-+#define CLK_GOUT_CMU_FSYS1_PCIE			182
-+#define CLK_GOUT_CMU_FSYS1_UFS_CARD		183
-+#define CLK_GOUT_CMU_G2D_G2D			184
-+#define CLK_GOUT_CMU_G2D_JPEG			185
-+#define CLK_GOUT_CMU_G3D_SWITCH			186
-+#define CLK_GOUT_CMU_HPM			187
-+#define CLK_GOUT_CMU_IMEM_BUS			188
-+#define CLK_GOUT_CMU_ISPHQ_BUS			189
-+#define CLK_GOUT_CMU_ISPLP_BUS			190
-+#define CLK_GOUT_CMU_IVA_BUS			191
-+#define CLK_GOUT_CMU_MFC_BUS			192
-+#define CLK_GOUT_CMU_MODEM_SHARED0		193
-+#define CLK_GOUT_CMU_MODEM_SHARED1		194
-+#define CLK_GOUT_CMU_PERIC0_BUS			195
-+#define CLK_GOUT_CMU_PERIC0_UART_DBG		196
-+#define CLK_GOUT_CMU_PERIC0_USI00		197
-+#define CLK_GOUT_CMU_PERIC0_USI01		198
-+#define CLK_GOUT_CMU_PERIC0_USI02		199
-+#define CLK_GOUT_CMU_PERIC0_USI03		200
-+#define CLK_GOUT_CMU_PERIC1_BUS			201
-+#define CLK_GOUT_CMU_PERIC1_SPEEDY2		202
-+#define CLK_GOUT_CMU_PERIC1_SPI_CAM0		203
-+#define CLK_GOUT_CMU_PERIC1_SPI_CAM1		204
-+#define CLK_GOUT_CMU_PERIC1_UART_BT		205
-+#define CLK_GOUT_CMU_PERIC1_USI04		206
-+#define CLK_GOUT_CMU_PERIC1_USI05		207
-+#define CLK_GOUT_CMU_PERIC1_USI06		208
-+#define CLK_GOUT_CMU_PERIC1_USI07		209
-+#define CLK_GOUT_CMU_PERIC1_USI08		210
-+#define CLK_GOUT_CMU_PERIC1_USI09		211
-+#define CLK_GOUT_CMU_PERIC1_USI10		212
-+#define CLK_GOUT_CMU_PERIC1_USI11		213
-+#define CLK_GOUT_CMU_PERIC1_USI12		214
-+#define CLK_GOUT_CMU_PERIC1_USI13		215
-+#define CLK_GOUT_CMU_PERIS_BUS			216
-+#define CLK_GOUT_CMU_SRDZ_BUS			217
-+#define CLK_GOUT_CMU_SRDZ_IMGD			218
-+#define CLK_GOUT_CMU_VPU_BUS			219
-+
-+/* CMU_PERIS */
-+#define CLK_MOUT_PERIS_BUS_USER				1
-+#define CLK_MOUT_PERIS_GIC				2
-+#define CLK_GOUT_PERIS_CMU_PERIS_PCLK			3
-+#define CLK_GOUT_PERIS_AD_AXI_P_PERIS_ACLKM		4
-+#define CLK_GOUT_PERIS_AD_AXI_P_PERIS_ACLKS		5
-+#define CLK_GOUT_PERIS_AXI2APB_PERISP0_ACLK		6
-+#define CLK_GOUT_PERIS_AXI2APB_PERISP1_ACLK		7
-+#define CLK_GOUT_PERIS_BUSIF_TMU_PCLK			8
-+#define CLK_GOUT_PERIS_GIC_CLK				9
-+#define CLK_GOUT_PERIS_LHM_AXI_P_PERIS_I_CLK		10
-+#define CLK_GOUT_PERIS_MCT_PCLK				11
-+#define CLK_GOUT_PERIS_OTP_CON_BIRA_PCLK		12
-+#define CLK_GOUT_PERIS_OTP_CON_TOP_PCLK			13
-+#define CLK_GOUT_PERIS_PMU_PERIS_PCLK			14
-+#define CLK_GOUT_PERIS_RSTNSYNC_CLK_PERIS_BUSP_CLK	15
-+#define CLK_GOUT_PERIS_RSTNSYNC_CLK_PERIS_GIC_CLK	16
-+#define CLK_GOUT_PERIS_SYSREG_PERIS_PCLK		17
-+#define CLK_GOUT_PERIS_TZPC00_PCLK			18
-+#define CLK_GOUT_PERIS_TZPC01_PCLK			19
-+#define CLK_GOUT_PERIS_TZPC02_PCLK			20
-+#define CLK_GOUT_PERIS_TZPC03_PCLK			21
-+#define CLK_GOUT_PERIS_TZPC04_PCLK			22
-+#define CLK_GOUT_PERIS_TZPC05_PCLK			23
-+#define CLK_GOUT_PERIS_TZPC06_PCLK			24
-+#define CLK_GOUT_PERIS_TZPC07_PCLK			25
-+#define CLK_GOUT_PERIS_TZPC08_PCLK			26
-+#define CLK_GOUT_PERIS_TZPC09_PCLK			27
-+#define CLK_GOUT_PERIS_TZPC10_PCLK			28
-+#define CLK_GOUT_PERIS_TZPC11_PCLK			29
-+#define CLK_GOUT_PERIS_TZPC12_PCLK			30
-+#define CLK_GOUT_PERIS_TZPC13_PCLK			31
-+#define CLK_GOUT_PERIS_TZPC14_PCLK			32
-+#define CLK_GOUT_PERIS_TZPC15_PCLK			33
-+#define CLK_GOUT_PERIS_WDT_CLUSTER0_PCLK		34
-+#define CLK_GOUT_PERIS_WDT_CLUSTER1_PCLK		35
-+#define CLK_GOUT_PERIS_XIU_P_PERIS_ACLK			36
-+
-+/* CMU_FSYS0 */
-+#define CLK_MOUT_FSYS0_BUS_USER				1
-+#define CLK_MOUT_FSYS0_DPGTC_USER			2
-+#define CLK_MOUT_FSYS0_MMC_EMBD_USER			3
-+#define CLK_MOUT_FSYS0_UFS_EMBD_USER			4
-+#define CLK_MOUT_FSYS0_USBDRD30_USER			5
-+#define CLK_GOUT_FSYS0_FSYS0_CMU_FSYS0_PCLK		6
-+#define CLK_GOUT_FSYS0_AHBBR_FSYS0_HCLK			7
-+#define CLK_GOUT_FSYS0_AXI2AHB_FSYS0_ACLK		8
-+#define CLK_GOUT_FSYS0_AXI2AHB_USB_FSYS0_ACLK		9
-+#define CLK_GOUT_FSYS0_AXI2APB_FSYS0_ACLK		10
-+#define CLK_GOUT_FSYS0_BTM_FSYS0_I_ACLK			11
-+#define CLK_GOUT_FSYS0_BTM_FSYS0_I_PCLK			12
-+#define CLK_GOUT_FSYS0_DP_LINK_I_GTC_EXT_CLK		13
-+#define CLK_GOUT_FSYS0_DP_LINK_I_PCLK			14
-+#define CLK_GOUT_FSYS0_ETR_MIU_I_ACLK			15
-+#define CLK_GOUT_FSYS0_ETR_MIU_I_PCLK			16
-+#define CLK_GOUT_FSYS0_GPIO_FSYS0_PCLK			17
-+#define CLK_GOUT_FSYS0_LHM_AXI_D_USBTV_I_CLK		18
-+#define CLK_GOUT_FSYS0_LHM_AXI_G_ETR_I_CLK		19
-+#define CLK_GOUT_FSYS0_LHM_AXI_P_FSYS0_I_CLK		20
-+#define CLK_GOUT_FSYS0_LHS_ACEL_D_FSYS0_I_CLK		21
-+#define CLK_GOUT_FSYS0_MMC_EMBD_I_ACLK			22
-+#define CLK_GOUT_FSYS0_MMC_EMBD_SDCLKIN			23
-+#define CLK_GOUT_FSYS0_PMU_FSYS0_PCLK			24
-+#define CLK_GOUT_FSYS0_BCM_FSYS0_ACLK			25
-+#define CLK_GOUT_FSYS0_BCM_FSYS0_PCLK			26
-+#define CLK_GOUT_FSYS0_RSTNSYNC_CLK_FSYS0_BUS_CLK	27
-+#define CLK_GOUT_FSYS0_SYSREG_FSYS0_PCLK		28
-+#define CLK_GOUT_FSYS0_UFS_EMBD_I_ACLK			29
-+#define CLK_GOUT_FSYS0_UFS_EMBD_I_CLK_UNIPRO		30
-+#define CLK_GOUT_FSYS0_UFS_EMBD_I_FMP_CLK		31
-+#define CLK_GOUT_FSYS0_USBTV_I_USB30DRD_ACLK		32
-+#define CLK_GOUT_FSYS0_USBTV_I_USB30DRD_REF_CLK		33
-+#define CLK_GOUT_FSYS0_USBTV_I_USB30DRD_SUSPEND_CLK	34
-+#define CLK_GOUT_FSYS0_USBTV_I_USBTVH_AHB_CLK		35
-+#define CLK_GOUT_FSYS0_USBTV_I_USBTVH_CORE_CLK		36
-+#define CLK_GOUT_FSYS0_USBTV_I_USBTVH_XIU_CLK		37
-+#define CLK_GOUT_FSYS0_US_D_FSYS0_USB_ACLK		38
-+#define CLK_GOUT_FSYS0_XIU_D_FSYS0_ACLK			39
-+#define CLK_GOUT_FSYS0_XIU_D_FSYS0_USB_ACLK		40
-+#define CLK_GOUT_FSYS0_XIU_P_FSYS0_ACLK			41
-+
-+/* CMU_FSYS1 */
-+#define CLK_MOUT_FSYS1_BUS_USER								1
-+#define CLK_MOUT_FSYS1_MMC_CARD_USER							2
-+#define CLK_MOUT_FSYS1_PCIE_USER							3
-+#define CLK_MOUT_FSYS1_UFS_CARD_USER							4
-+#define CLK_GOUT_FSYS1_PCIE_PHY_REF_CLK_IN						5
-+#define CLK_GOUT_FSYS1_ADM_AHB_SSS_HCLKM						6
-+#define CLK_GOUT_FSYS1_AHBBR_FSYS1_HCLK							7
-+#define CLK_GOUT_FSYS1_AXI2AHB_FSYS1_ACLK						8
-+#define CLK_GOUT_FSYS1_AXI2APB_FSYS1P0_ACLK						9
-+#define CLK_GOUT_FSYS1_AXI2APB_FSYS1P1_ACLK						10
-+#define CLK_GOUT_FSYS1_BTM_FSYS1_I_ACLK							11
-+#define CLK_GOUT_FSYS1_BTM_FSYS1_I_PCLK							12
-+#define CLK_GOUT_FSYS1_FSYS1_CMU_FSYS1_PCLK						13
-+#define CLK_GOUT_FSYS1_GPIO_FSYS1_PCLK							14
-+#define CLK_GOUT_FSYS1_LHM_AXI_P_FSYS1_I_CLK						15
-+#define CLK_GOUT_FSYS1_LHS_ACEL_D_FSYS1_I_CLK						16
-+#define CLK_GOUT_FSYS1_MMC_CARD_I_ACLK							17
-+#define CLK_GOUT_FSYS1_MMC_CARD_SDCLKIN							18
-+#define CLK_GOUT_FSYS1_PCIE_DBI_ACLK_0							19
-+#define CLK_GOUT_FSYS1_PCIE_DBI_ACLK_1							20
-+#define CLK_GOUT_FSYS1_PCIE_IEEE1500_WRAPPER_FOR_PCIE_PHY_LC_X2_INST_0_I_SCL_APB_PCLK	21
-+#define CLK_GOUT_FSYS1_PCIE_MSTR_ACLK_0							22
-+#define CLK_GOUT_FSYS1_PCIE_MSTR_ACLK_1							23
-+#define CLK_GOUT_FSYS1_PCIE_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK			24
-+#define CLK_GOUT_FSYS1_PCIE_PCIE_SUB_CTRL_INST_1_I_DRIVER_APB_CLK			25
-+#define CLK_GOUT_FSYS1_PCIE_PIPE2_DIGITAL_X2_WRAP_INST_0_I_APB_PCLK_SCL			26
-+#define CLK_GOUT_FSYS1_PCIE_SLV_ACLK_0							27
-+#define CLK_GOUT_FSYS1_PCIE_SLV_ACLK_1							28
-+#define CLK_GOUT_FSYS1_PMU_FSYS1_PCLK							29
-+#define CLK_GOUT_FSYS1_BCM_FSYS1_ACLK							30
-+#define CLK_GOUT_FSYS1_BCM_FSYS1_PCLK							31
-+#define CLK_GOUT_FSYS1_RSTNSYNC_CLK_FSYS1_BUS_CLK					32
-+#define CLK_GOUT_FSYS1_RTIC_I_ACLK							33
-+#define CLK_GOUT_FSYS1_RTIC_I_PCLK							34
-+#define CLK_GOUT_FSYS1_SSS_I_ACLK							35
-+#define CLK_GOUT_FSYS1_SSS_I_PCLK							36
-+#define CLK_GOUT_FSYS1_SYSREG_FSYS1_PCLK						37
-+#define CLK_GOUT_FSYS1_TOE_WIFI0_I_CLK							38
-+#define CLK_GOUT_FSYS1_TOE_WIFI1_I_CLK							39
-+#define CLK_GOUT_FSYS1_UFS_CARD_I_ACLK							40
-+#define CLK_GOUT_FSYS1_UFS_CARD_I_CLK_UNIPRO						41
-+#define CLK_GOUT_FSYS1_UFS_CARD_I_FMP_CLK						42
-+#define CLK_GOUT_FSYS1_XIU_D_FSYS1_ACLK							43
-+#define CLK_GOUT_FSYS1_XIU_P_FSYS1_ACLK							44
-+
-+/* CMU_PERIC0 */
-+#define CLK_MOUT_PERIC0_BUS_USER			1
-+#define CLK_MOUT_PERIC0_UART_DBG_USER			2
-+#define CLK_MOUT_PERIC0_USI00_USER			3
-+#define CLK_MOUT_PERIC0_USI01_USER			4
-+#define CLK_MOUT_PERIC0_USI02_USER			5
-+#define CLK_MOUT_PERIC0_USI03_USER			6
-+#define CLK_GOUT_PERIC0_PERIC0_CMU_PERIC0_PCLK		7
-+#define CLK_GOUT_PERIC0_AXI2APB_PERIC0_ACLK		8
-+#define CLK_GOUT_PERIC0_GPIO_PERIC0_PCLK		9
-+#define CLK_GOUT_PERIC0_LHM_AXI_P_PERIC0_I_CLK		10
-+#define CLK_GOUT_PERIC0_PMU_PERIC0_PCLK			11
-+#define CLK_GOUT_PERIC0_PWM_I_PCLK_S0			12
-+#define CLK_GOUT_PERIC0_RSTNSYNC_CLK_PERIC0_BUSP_CLK	13
-+#define CLK_GOUT_PERIC0_SPEEDY2_TSP_CLK			14
-+#define CLK_GOUT_PERIC0_SYSREG_PERIC0_PCLK		15
-+#define CLK_GOUT_PERIC0_UART_DBG_EXT_UCLK		16
-+#define CLK_GOUT_PERIC0_UART_DBG_PCLK			17
-+#define CLK_GOUT_PERIC0_USI00_I_PCLK			18
-+#define CLK_GOUT_PERIC0_USI00_I_SCLK_USI		19
-+#define CLK_GOUT_PERIC0_USI01_I_PCLK			20
-+#define CLK_GOUT_PERIC0_USI01_I_SCLK_USI		21
-+#define CLK_GOUT_PERIC0_USI02_I_PCLK			22
-+#define CLK_GOUT_PERIC0_USI02_I_SCLK_USI		23
-+#define CLK_GOUT_PERIC0_USI03_I_PCLK			24
-+#define CLK_GOUT_PERIC0_USI03_I_SCLK_USI		25
-+
-+/* CMU_PERIC1 */
-+#define CLK_MOUT_PERIC1_BUS_USER			1
-+#define CLK_MOUT_PERIC1_SPEEDY2_USER			2
-+#define CLK_MOUT_PERIC1_SPI_CAM0_USER			3
-+#define CLK_MOUT_PERIC1_SPI_CAM1_USER			4
-+#define CLK_MOUT_PERIC1_UART_BT_USER			5
-+#define CLK_MOUT_PERIC1_USI04_USER			6
-+#define CLK_MOUT_PERIC1_USI05_USER			7
-+#define CLK_MOUT_PERIC1_USI06_USER			8
-+#define CLK_MOUT_PERIC1_USI07_USER			9
-+#define CLK_MOUT_PERIC1_USI08_USER			10
-+#define CLK_MOUT_PERIC1_USI09_USER			11
-+#define CLK_MOUT_PERIC1_USI10_USER			12
-+#define CLK_MOUT_PERIC1_USI11_USER			13
-+#define CLK_MOUT_PERIC1_USI12_USER			14
-+#define CLK_MOUT_PERIC1_USI13_USER			15
-+#define CLK_GOUT_PERIC1_PERIC1_CMU_PERIC1_PCLK		16
-+#define CLK_GOUT_PERIC1_RSTNSYNC_CLK_PERIC1_SPEEDY2_CLK	17
-+#define CLK_GOUT_PERIC1_AXI2APB_PERIC1P0_ACLK		18
-+#define CLK_GOUT_PERIC1_AXI2APB_PERIC1P1_ACLK		19
-+#define CLK_GOUT_PERIC1_AXI2APB_PERIC1P2_ACLK		20
-+#define CLK_GOUT_PERIC1_GPIO_PERIC1_PCLK		21
-+#define CLK_GOUT_PERIC1_HSI2C_CAM0_IPCLK		22
-+#define CLK_GOUT_PERIC1_HSI2C_CAM1_IPCLK		23
-+#define CLK_GOUT_PERIC1_HSI2C_CAM2_IPCLK		24
-+#define CLK_GOUT_PERIC1_HSI2C_CAM3_IPCLK		25
-+#define CLK_GOUT_PERIC1_LHM_AXI_P_PERIC1_I_CLK		26
-+#define CLK_GOUT_PERIC1_PMU_PERIC1_PCLK			27
-+#define CLK_GOUT_PERIC1_RSTNSYNC_CLK_PERIC1_BUSP_CLK	28
-+#define CLK_GOUT_PERIC1_SPEEDY2_DDI1_CLK		29
-+#define CLK_GOUT_PERIC1_SPEEDY2_DDI1_SCLK		30
-+#define CLK_GOUT_PERIC1_SPEEDY2_DDI2_CLK		31
-+#define CLK_GOUT_PERIC1_SPEEDY2_DDI2_SCLK		32
-+#define CLK_GOUT_PERIC1_SPEEDY2_DDI_CLK			33
-+#define CLK_GOUT_PERIC1_SPEEDY2_DDI_SCLK		34
-+#define CLK_GOUT_PERIC1_SPEEDY2_TSP1_CLK		35
-+#define CLK_GOUT_PERIC1_SPEEDY2_TSP2_CLK		36
-+#define CLK_GOUT_PERIC1_SPI_CAM0_PCLK			37
-+#define CLK_GOUT_PERIC1_SPI_CAM0_SPI_EXT_CLK		38
-+#define CLK_GOUT_PERIC1_SPI_CAM1_PCLK			39
-+#define CLK_GOUT_PERIC1_SPI_CAM1_SPI_EXT_CLK		40
-+#define CLK_GOUT_PERIC1_SYSREG_PERIC1_PCLK		41
-+#define CLK_GOUT_PERIC1_UART_BT_EXT_UCLK		42
-+#define CLK_GOUT_PERIC1_UART_BT_PCLK			43
-+#define CLK_GOUT_PERIC1_USI04_I_PCLK			44
-+#define CLK_GOUT_PERIC1_USI04_I_SCLK_USI		45
-+#define CLK_GOUT_PERIC1_USI05_I_PCLK			46
-+#define CLK_GOUT_PERIC1_USI05_I_SCLK_USI		47
-+#define CLK_GOUT_PERIC1_USI06_I_PCLK			48
-+#define CLK_GOUT_PERIC1_USI06_I_SCLK_USI		49
-+#define CLK_GOUT_PERIC1_USI07_I_PCLK			50
-+#define CLK_GOUT_PERIC1_USI07_I_SCLK_USI		51
-+#define CLK_GOUT_PERIC1_USI08_I_PCLK			52
-+#define CLK_GOUT_PERIC1_USI08_I_SCLK_USI		53
-+#define CLK_GOUT_PERIC1_USI09_I_PCLK			54
-+#define CLK_GOUT_PERIC1_USI09_I_SCLK_USI		55
-+#define CLK_GOUT_PERIC1_USI10_I_PCLK			56
-+#define CLK_GOUT_PERIC1_USI10_I_SCLK_USI		57
-+#define CLK_GOUT_PERIC1_USI11_I_PCLK			58
-+#define CLK_GOUT_PERIC1_USI11_I_SCLK_USI		59
-+#define CLK_GOUT_PERIC1_USI12_I_PCLK			60
-+#define CLK_GOUT_PERIC1_USI12_I_SCLK_USI		61
-+#define CLK_GOUT_PERIC1_USI13_I_PCLK			62
-+#define CLK_GOUT_PERIC1_USI13_I_SCLK_USI		63
-+#define CLK_GOUT_PERIC1_XIU_P_PERIC1_ACLK		64
-+
-+#endif /* _DT_BINDINGS_CLOCK_EXYNOS8895_H */
-diff --git a/include/dt-bindings/clock/samsung,exynos990.h b/include/dt-bindings/clock/samsung,exynos990.h
-new file mode 100644
-index 000000000000..47540307cb52
--- /dev/null
-+++ b/include/dt-bindings/clock/samsung,exynos990.h
-@@ -0,0 +1,438 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2024 Igor Belwon <igor.belwon@mentallysanemainliners.org>
-+ *
-+ * Device Tree binding constants for Exynos990 clock controller.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLOCK_EXYNOS_990_H
-+#define _DT_BINDINGS_CLOCK_EXYNOS_990_H
-+
-+/* CMU_TOP */
-+#define CLK_FOUT_SHARED0_PLL		1
-+#define CLK_FOUT_SHARED1_PLL		2
-+#define CLK_FOUT_SHARED2_PLL		3
-+#define CLK_FOUT_SHARED3_PLL		4
-+#define CLK_FOUT_SHARED4_PLL		5
-+#define CLK_FOUT_G3D_PLL		6
-+#define CLK_FOUT_MMC_PLL		7
-+#define CLK_MOUT_PLL_SHARED0		8
-+#define CLK_MOUT_PLL_SHARED1		9
-+#define CLK_MOUT_PLL_SHARED2		10
-+#define CLK_MOUT_PLL_SHARED3		11
-+#define CLK_MOUT_PLL_SHARED4		12
-+#define CLK_MOUT_PLL_MMC		13
-+#define CLK_MOUT_PLL_G3D		14
-+#define CLK_MOUT_CMU_APM_BUS		15
-+#define CLK_MOUT_CMU_AUD_CPU		16
-+#define CLK_MOUT_CMU_BUS0_BUS		17
-+#define CLK_MOUT_CMU_BUS1_BUS		18
-+#define CLK_MOUT_CMU_BUS1_SSS		19
-+#define CLK_MOUT_CMU_CIS_CLK0		20
-+#define CLK_MOUT_CMU_CIS_CLK1		21
-+#define CLK_MOUT_CMU_CIS_CLK2		22
-+#define CLK_MOUT_CMU_CIS_CLK3		23
-+#define CLK_MOUT_CMU_CIS_CLK4		24
-+#define CLK_MOUT_CMU_CIS_CLK5		25
-+#define CLK_MOUT_CMU_CMU_BOOST		26
-+#define CLK_MOUT_CMU_CORE_BUS		27
-+#define CLK_MOUT_CMU_CPUCL0_DBG_BUS	28
-+#define CLK_MOUT_CMU_CPUCL0_SWITCH	29
-+#define CLK_MOUT_CMU_CPUCL1_SWITCH	30
-+#define CLK_MOUT_CMU_CPUCL2_BUSP	31
-+#define CLK_MOUT_CMU_CPUCL2_SWITCH	32
-+#define CLK_MOUT_CMU_CSIS_BUS		33
-+#define CLK_MOUT_CMU_CSIS_OIS_MCU	34
-+#define CLK_MOUT_CMU_DNC_BUS		35
-+#define CLK_MOUT_CMU_DNC_BUSM		36
-+#define CLK_MOUT_CMU_DNS_BUS		37
-+#define CLK_MOUT_CMU_DPU		38
-+#define CLK_MOUT_CMU_DPU_ALT		39
-+#define CLK_MOUT_CMU_DSP_BUS		40
-+#define CLK_MOUT_CMU_G2D_G2D		41
-+#define CLK_MOUT_CMU_G2D_MSCL		42
-+#define CLK_MOUT_CMU_HPM		43
-+#define CLK_MOUT_CMU_HSI0_BUS		44
-+#define CLK_MOUT_CMU_HSI0_DPGTC		45
-+#define CLK_MOUT_CMU_HSI0_USB31DRD	46
-+#define CLK_MOUT_CMU_HSI0_USBDP_DEBUG	47
-+#define CLK_MOUT_CMU_HSI1_BUS		48
-+#define CLK_MOUT_CMU_HSI1_MMC_CARD	49
-+#define CLK_MOUT_CMU_HSI1_PCIE		50
-+#define CLK_MOUT_CMU_HSI1_UFS_CARD	51
-+#define CLK_MOUT_CMU_HSI1_UFS_EMBD	52
-+#define CLK_MOUT_CMU_HSI2_BUS		53
-+#define CLK_MOUT_CMU_HSI2_PCIE		54
-+#define CLK_MOUT_CMU_IPP_BUS		55
-+#define CLK_MOUT_CMU_ITP_BUS		56
-+#define CLK_MOUT_CMU_MCSC_BUS		57
-+#define CLK_MOUT_CMU_MCSC_GDC		58
-+#define CLK_MOUT_CMU_CMU_BOOST_CPU	59
-+#define CLK_MOUT_CMU_MFC0_MFC0		60
-+#define CLK_MOUT_CMU_MFC0_WFD		61
-+#define CLK_MOUT_CMU_MIF_BUSP		62
-+#define CLK_MOUT_CMU_MIF_SWITCH		63
-+#define CLK_MOUT_CMU_NPU_BUS		64
-+#define CLK_MOUT_CMU_PERIC0_BUS		65
-+#define CLK_MOUT_CMU_PERIC0_IP		66
-+#define CLK_MOUT_CMU_PERIC1_BUS		67
-+#define CLK_MOUT_CMU_PERIC1_IP		68
-+#define CLK_MOUT_CMU_PERIS_BUS		69
-+#define CLK_MOUT_CMU_SSP_BUS		70
-+#define CLK_MOUT_CMU_TNR_BUS		71
-+#define CLK_MOUT_CMU_VRA_BUS		72
-+#define CLK_DOUT_CMU_APM_BUS		73
-+#define CLK_DOUT_CMU_AUD_CPU		74
-+#define CLK_DOUT_CMU_BUS0_BUS		75
-+#define CLK_DOUT_CMU_BUS1_BUS		76
-+#define CLK_DOUT_CMU_BUS1_SSS		77
-+#define CLK_DOUT_CMU_CIS_CLK0		78
-+#define CLK_DOUT_CMU_CIS_CLK1		79
-+#define CLK_DOUT_CMU_CIS_CLK2		80
-+#define CLK_DOUT_CMU_CIS_CLK3		81
-+#define CLK_DOUT_CMU_CIS_CLK4		82
-+#define CLK_DOUT_CMU_CIS_CLK5		83
-+#define CLK_DOUT_CMU_CMU_BOOST		84
-+#define CLK_DOUT_CMU_CORE_BUS		85
-+#define CLK_DOUT_CMU_CPUCL0_DBG_BUS	86
-+#define CLK_DOUT_CMU_CPUCL0_SWITCH	87
-+#define CLK_DOUT_CMU_CPUCL1_SWITCH	88
-+#define CLK_DOUT_CMU_CPUCL2_BUSP	89
-+#define CLK_DOUT_CMU_CPUCL2_SWITCH	90
-+#define CLK_DOUT_CMU_CSIS_BUS		91
-+#define CLK_DOUT_CMU_CSIS_OIS_MCU	92
-+#define CLK_DOUT_CMU_DNC_BUS		93
-+#define CLK_DOUT_CMU_DNC_BUSM		94
-+#define CLK_DOUT_CMU_DNS_BUS		95
-+#define CLK_DOUT_CMU_DSP_BUS		96
-+#define CLK_DOUT_CMU_G2D_G2D		97
-+#define CLK_DOUT_CMU_G2D_MSCL		98
-+#define CLK_DOUT_CMU_G3D_SWITCH		99
-+#define CLK_DOUT_CMU_HPM		100
-+#define CLK_DOUT_CMU_HSI0_BUS		101
-+#define CLK_DOUT_CMU_HSI0_DPGTC		102
-+#define CLK_DOUT_CMU_HSI0_USB31DRD	103
-+#define CLK_DOUT_CMU_HSI0_USBDP_DEBUG	104
-+#define CLK_DOUT_CMU_HSI1_BUS		105
-+#define CLK_DOUT_CMU_HSI1_MMC_CARD	106
-+#define CLK_DOUT_CMU_HSI1_PCIE		107
-+#define CLK_DOUT_CMU_HSI1_UFS_CARD	108
-+#define CLK_DOUT_CMU_HSI1_UFS_EMBD	109
-+#define CLK_DOUT_CMU_HSI2_BUS		110
-+#define CLK_DOUT_CMU_HSI2_PCIE		111
-+#define CLK_DOUT_CMU_IPP_BUS		112
-+#define CLK_DOUT_CMU_ITP_BUS		113
-+#define CLK_DOUT_CMU_MCSC_BUS		114
-+#define CLK_DOUT_CMU_MCSC_GDC		115
-+#define CLK_DOUT_CMU_CMU_BOOST_CPU	116
-+#define CLK_DOUT_CMU_MFC0_MFC0		117
-+#define CLK_DOUT_CMU_MFC0_WFD		118
-+#define CLK_DOUT_CMU_MIF_BUSP		119
-+#define CLK_DOUT_CMU_NPU_BUS		120
-+#define CLK_DOUT_CMU_OTP		121
-+#define CLK_DOUT_CMU_PERIC0_BUS		122
-+#define CLK_DOUT_CMU_PERIC0_IP		123
-+#define CLK_DOUT_CMU_PERIC1_BUS		124
-+#define CLK_DOUT_CMU_PERIC1_IP		125
-+#define CLK_DOUT_CMU_PERIS_BUS		126
-+#define CLK_DOUT_CMU_SSP_BUS		127
-+#define CLK_DOUT_CMU_TNR_BUS		128
-+#define CLK_DOUT_CMU_VRA_BUS		129
-+#define CLK_DOUT_CMU_DPU		130
-+#define CLK_DOUT_CMU_DPU_ALT		131
-+#define CLK_DOUT_CMU_SHARED0_DIV2	132
-+#define CLK_DOUT_CMU_SHARED0_DIV3	133
-+#define CLK_DOUT_CMU_SHARED0_DIV4	134
-+#define CLK_DOUT_CMU_SHARED1_DIV2	135
-+#define CLK_DOUT_CMU_SHARED1_DIV3	136
-+#define CLK_DOUT_CMU_SHARED1_DIV4	137
-+#define CLK_DOUT_CMU_SHARED2_DIV2	138
-+#define CLK_DOUT_CMU_SHARED4_DIV2	139
-+#define CLK_DOUT_CMU_SHARED4_DIV3	140
-+#define CLK_DOUT_CMU_SHARED4_DIV4	141
-+#define CLK_GOUT_CMU_G3D_BUS		142
-+#define CLK_GOUT_CMU_MIF_SWITCH		143
-+#define CLK_GOUT_CMU_APM_BUS		144
-+#define CLK_GOUT_CMU_AUD_CPU		145
-+#define CLK_GOUT_CMU_BUS0_BUS		146
-+#define CLK_GOUT_CMU_BUS1_BUS		147
-+#define CLK_GOUT_CMU_BUS1_SSS		148
-+#define CLK_GOUT_CMU_CIS_CLK0		149
-+#define CLK_GOUT_CMU_CIS_CLK1		150
-+#define CLK_GOUT_CMU_CIS_CLK2		151
-+#define CLK_GOUT_CMU_CIS_CLK3		152
-+#define CLK_GOUT_CMU_CIS_CLK4		153
-+#define CLK_GOUT_CMU_CIS_CLK5		154
-+#define CLK_GOUT_CMU_CORE_BUS		155
-+#define CLK_GOUT_CMU_CPUCL0_DBG_BUS	156
-+#define CLK_GOUT_CMU_CPUCL0_SWITCH	157
-+#define CLK_GOUT_CMU_CPUCL1_SWITCH	158
-+#define CLK_GOUT_CMU_CPUCL2_BUSP	159
-+#define CLK_GOUT_CMU_CPUCL2_SWITCH	160
-+#define CLK_GOUT_CMU_CSIS_BUS		161
-+#define CLK_GOUT_CMU_CSIS_OIS_MCU	162
-+#define CLK_GOUT_CMU_DNC_BUS		163
-+#define CLK_GOUT_CMU_DNC_BUSM		164
-+#define CLK_GOUT_CMU_DNS_BUS		165
-+#define CLK_GOUT_CMU_DPU		166
-+#define CLK_GOUT_CMU_DPU_BUS		167
-+#define CLK_GOUT_CMU_DSP_BUS		168
-+#define CLK_GOUT_CMU_G2D_G2D		169
-+#define CLK_GOUT_CMU_G2D_MSCL		170
-+#define CLK_GOUT_CMU_G3D_SWITCH		171
-+#define CLK_GOUT_CMU_HPM		172
-+#define CLK_GOUT_CMU_HSI0_BUS		173
-+#define CLK_GOUT_CMU_HSI0_DPGTC		174
-+#define CLK_GOUT_CMU_HSI0_USB31DRD	175
-+#define CLK_GOUT_CMU_HSI0_USBDP_DEBUG	176
-+#define CLK_GOUT_CMU_HSI1_BUS		177
-+#define CLK_GOUT_CMU_HSI1_MMC_CARD	178
-+#define CLK_GOUT_CMU_HSI1_PCIE		179
-+#define CLK_GOUT_CMU_HSI1_UFS_CARD	180
-+#define CLK_GOUT_CMU_HSI1_UFS_EMBD	181
-+#define CLK_GOUT_CMU_HSI2_BUS		182
-+#define CLK_GOUT_CMU_HSI2_PCIE		183
-+#define CLK_GOUT_CMU_IPP_BUS		184
-+#define CLK_GOUT_CMU_ITP_BUS		185
-+#define CLK_GOUT_CMU_MCSC_BUS		186
-+#define CLK_GOUT_CMU_MCSC_GDC		187
-+#define CLK_GOUT_CMU_MFC0_MFC0		188
-+#define CLK_GOUT_CMU_MFC0_WFD		189
-+#define CLK_GOUT_CMU_MIF_BUSP		190
-+#define CLK_GOUT_CMU_NPU_BUS		191
-+#define CLK_GOUT_CMU_PERIC0_BUS		192
-+#define CLK_GOUT_CMU_PERIC0_IP		193
-+#define CLK_GOUT_CMU_PERIC1_BUS		194
-+#define CLK_GOUT_CMU_PERIC1_IP		195
-+#define CLK_GOUT_CMU_PERIS_BUS		196
-+#define CLK_GOUT_CMU_SSP_BUS		197
-+#define CLK_GOUT_CMU_TNR_BUS		198
-+#define CLK_GOUT_CMU_VRA_BUS		199
-+#define CLK_MOUT_CMU_CMUREF		200
-+#define CLK_MOUT_CMU_DPU_BUS		201
-+#define CLK_MOUT_CMU_CLK_CMUREF		202
-+#define CLK_DOUT_CMU_CLK_CMUREF		203
-+
-+/* CMU_HSI0 */
-+#define CLK_MOUT_HSI0_BUS_USER				1
-+#define CLK_MOUT_HSI0_USB31DRD_USER			2
-+#define CLK_MOUT_HSI0_USBDP_DEBUG_USER			3
-+#define CLK_MOUT_HSI0_DPGTC_USER			4
-+#define CLK_GOUT_HSI0_DP_LINK_DP_GTC_CLK		5
-+#define CLK_GOUT_HSI0_DP_LINK_PCLK			6
-+#define CLK_GOUT_HSI0_D_TZPC_HSI0_PCLK			7
-+#define CLK_GOUT_HSI0_LHM_AXI_P_HSI0_CLK		8
-+#define CLK_GOUT_HSI0_PPMU_HSI0_BUS1_ACLK		9
-+#define CLK_GOUT_HSI0_PPMU_HSI0_BUS1_PCLK		10
-+#define CLK_GOUT_HSI0_CLK_HSI0_BUS_CLK			11
-+#define CLK_GOUT_HSI0_SYSMMU_USB_CLK_S2			12
-+#define CLK_GOUT_HSI0_SYSREG_HSI0_PCLK			13
-+#define CLK_GOUT_HSI0_USB31DRD_ACLK_PHYCTRL		14
-+#define CLK_GOUT_HSI0_USB31DRD_BUS_CLK_EARLY		15
-+#define CLK_GOUT_HSI0_USB31DRD_USB31DRD_REF_CLK_40	16
-+#define CLK_GOUT_HSI0_USB31DRD_USBDPPHY_REF_SOC_PLL	17
-+#define CLK_GOUT_HSI0_USB31DRD_USBDPPHY_SCL_APB		18
-+#define CLK_GOUT_HSI0_USB31DRD_USBPCS_APB_CLK		19
-+#define CLK_GOUT_HSI0_VGEN_LITE_HSI0_CLK		20
-+#define CLK_GOUT_HSI0_CMU_HSI0_PCLK			21
-+#define CLK_GOUT_HSI0_XIU_D_HSI0_ACLK			22
-+#define CLK_GOUT_HSI0_LHS_ACEL_D_HSI0_CLK		23
-+
-+/* CMU_PERIC0 */
-+#define CLK_MOUT_PERIC0_BUS_USER		1
-+#define CLK_MOUT_PERIC0_UART_DBG		2
-+#define CLK_MOUT_PERIC0_USI00_USI_USER		3
-+#define CLK_MOUT_PERIC0_USI01_USI_USER		4
-+#define CLK_MOUT_PERIC0_USI02_USI_USER		5
-+#define CLK_MOUT_PERIC0_USI03_USI_USER		6
-+#define CLK_MOUT_PERIC0_USI04_USI_USER		7
-+#define CLK_MOUT_PERIC0_USI05_USI_USER		8
-+#define CLK_MOUT_PERIC0_USI13_USI_USER		9
-+#define CLK_MOUT_PERIC0_USI14_USI_USER		10
-+#define CLK_MOUT_PERIC0_USI15_USI_USER		11
-+#define CLK_MOUT_PERIC0_USI_I2C_USER		12
-+#define CLK_DOUT_PERIC0_UART_DBG		13
-+#define CLK_DOUT_PERIC0_USI00_USI		14
-+#define CLK_DOUT_PERIC0_USI01_USI		15
-+#define CLK_DOUT_PERIC0_USI02_USI		16
-+#define CLK_DOUT_PERIC0_USI03_USI		17
-+#define CLK_DOUT_PERIC0_USI04_USI		18
-+#define CLK_DOUT_PERIC0_USI05_USI		19
-+#define CLK_DOUT_PERIC0_USI13_USI		20
-+#define CLK_DOUT_PERIC0_USI14_USI		21
-+#define CLK_DOUT_PERIC0_USI15_USI		22
-+#define CLK_DOUT_PERIC0_USI_I2C			23
-+#define CLK_GOUT_PERIC0_CMU_PCLK		24
-+#define CLK_GOUT_PERIC0_OSCCLK_CLK		25
-+#define CLK_GOUT_PERIC0_D_TZPC_PCLK		26
-+#define CLK_GOUT_PERIC0_GPIO_PCLK		27
-+#define CLK_GOUT_PERIC0_LHM_AXI_P_CLK		28
-+#define CLK_GOUT_PERIC0_TOP0_IPCLK_10		29
-+#define CLK_GOUT_PERIC0_TOP0_IPCLK_11		30
-+#define CLK_GOUT_PERIC0_TOP0_IPCLK_12		31
-+#define CLK_GOUT_PERIC0_TOP0_IPCLK_13		32
-+#define CLK_GOUT_PERIC0_TOP0_IPCLK_14		33
-+#define CLK_GOUT_PERIC0_TOP0_IPCLK_15		34
-+#define CLK_GOUT_PERIC0_TOP0_IPCLK_4		35
-+#define CLK_GOUT_PERIC0_TOP0_IPCLK_5		36
-+#define CLK_GOUT_PERIC0_TOP0_IPCLK_6		37
-+#define CLK_GOUT_PERIC0_TOP0_IPCLK_7		38
-+#define CLK_GOUT_PERIC0_TOP0_IPCLK_8		39
-+#define CLK_GOUT_PERIC0_TOP0_IPCLK_9		40
-+#define CLK_GOUT_PERIC0_TOP0_PCLK_10		41
-+#define CLK_GOUT_PERIC0_TOP0_PCLK_11		42
-+#define CLK_GOUT_PERIC0_TOP0_PCLK_12		43
-+#define CLK_GOUT_PERIC0_TOP0_PCLK_13		44
-+#define CLK_GOUT_PERIC0_TOP0_PCLK_14		45
-+#define CLK_GOUT_PERIC0_TOP0_PCLK_15		46
-+#define CLK_GOUT_PERIC0_TOP0_PCLK_4		47
-+#define CLK_GOUT_PERIC0_TOP0_PCLK_5		48
-+#define CLK_GOUT_PERIC0_TOP0_PCLK_6		49
-+#define CLK_GOUT_PERIC0_TOP0_PCLK_7		50
-+#define CLK_GOUT_PERIC0_TOP0_PCLK_8		51
-+#define CLK_GOUT_PERIC0_TOP0_PCLK_9		52
-+#define CLK_GOUT_PERIC0_TOP1_IPCLK_0		53
-+#define CLK_GOUT_PERIC0_TOP1_IPCLK_3		54
-+#define CLK_GOUT_PERIC0_TOP1_IPCLK_4		55
-+#define CLK_GOUT_PERIC0_TOP1_IPCLK_5		56
-+#define CLK_GOUT_PERIC0_TOP1_IPCLK_6		57
-+#define CLK_GOUT_PERIC0_TOP1_IPCLK_7		58
-+#define CLK_GOUT_PERIC0_TOP1_IPCLK_8		59
-+#define CLK_GOUT_PERIC0_TOP1_PCLK_0		60
-+#define CLK_GOUT_PERIC0_TOP1_PCLK_15		61
-+#define CLK_GOUT_PERIC0_TOP1_PCLK_3		62
-+#define CLK_GOUT_PERIC0_TOP1_PCLK_4		63
-+#define CLK_GOUT_PERIC0_TOP1_PCLK_5		64
-+#define CLK_GOUT_PERIC0_TOP1_PCLK_6		65
-+#define CLK_GOUT_PERIC0_TOP1_PCLK_7		66
-+#define CLK_GOUT_PERIC0_TOP1_PCLK_8		67
-+#define CLK_GOUT_PERIC0_BUSP_CLK		68
-+#define CLK_GOUT_PERIC0_UART_DBG_CLK		69
-+#define CLK_GOUT_PERIC0_USI00_USI_CLK		70
-+#define CLK_GOUT_PERIC0_USI01_USI_CLK		71
-+#define CLK_GOUT_PERIC0_USI02_USI_CLK		72
-+#define CLK_GOUT_PERIC0_USI03_USI_CLK		73
-+#define CLK_GOUT_PERIC0_USI04_USI_CLK		74
-+#define CLK_GOUT_PERIC0_USI05_USI_CLK		75
-+#define CLK_GOUT_PERIC0_USI13_USI_CLK		76
-+#define CLK_GOUT_PERIC0_USI14_USI_CLK		77
-+#define CLK_GOUT_PERIC0_USI15_USI_CLK		78
-+#define CLK_GOUT_PERIC0_USI_I2C_CLK		79
-+#define CLK_GOUT_PERIC0_SYSREG_PCLK		80
-+
-+/* CMU_PERIC1 */
-+#define CLK_MOUT_PERIC1_BUS_USER		1
-+#define CLK_MOUT_PERIC1_UART_BT_USER		2
-+#define CLK_MOUT_PERIC1_USI06_USI_USER		3
-+#define CLK_MOUT_PERIC1_USI07_USI_USER		4
-+#define CLK_MOUT_PERIC1_USI08_USI_USER		5
-+#define CLK_MOUT_PERIC1_USI09_USI_USER		6
-+#define CLK_MOUT_PERIC1_USI10_USI_USER		7
-+#define CLK_MOUT_PERIC1_USI11_USI_USER		8
-+#define CLK_MOUT_PERIC1_USI12_USI_USER		9
-+#define CLK_MOUT_PERIC1_USI18_USI_USER		10
-+#define CLK_MOUT_PERIC1_USI16_USI_USER		11
-+#define CLK_MOUT_PERIC1_USI17_USI_USER		12
-+#define CLK_MOUT_PERIC1_USI_I2C_USER            13
-+#define CLK_DOUT_PERIC1_UART_BT			14
-+#define CLK_DOUT_PERIC1_USI06_USI		15
-+#define CLK_DOUT_PERIC1_USI07_USI		16
-+#define CLK_DOUT_PERIC1_USI08_USI		17
-+#define CLK_DOUT_PERIC1_USI18_USI		18
-+#define CLK_DOUT_PERIC1_USI12_USI		19
-+#define CLK_DOUT_PERIC1_USI09_USI		20
-+#define CLK_DOUT_PERIC1_USI10_USI		21
-+#define CLK_DOUT_PERIC1_USI11_USI		22
-+#define CLK_DOUT_PERIC1_USI16_USI		23
-+#define CLK_DOUT_PERIC1_USI17_USI		24
-+#define CLK_DOUT_PERIC1_USI_I2C			25
-+#define CLK_GOUT_PERIC1_CMU_PCLK		26
-+#define CLK_GOUT_PERIC1_UART_BT_CLK		27
-+#define CLK_GOUT_PERIC1_USI12_USI_CLK		28
-+#define CLK_GOUT_PERIC1_USI18_USI_CLK		29
-+#define CLK_GOUT_PERIC1_D_TZPC_PCLK		30
-+#define CLK_GOUT_PERIC1_GPIO_PCLK		31
-+#define CLK_GOUT_PERIC1_LHM_AXI_P_CSIS_CLK	32
-+#define CLK_GOUT_PERIC1_LHM_AXI_P_CLK		33
-+#define CLK_GOUT_PERIC1_TOP0_IPCLK_10		34
-+#define CLK_GOUT_PERIC1_TOP0_IPCLK_11		35
-+#define CLK_GOUT_PERIC1_TOP0_IPCLK_12		36
-+#define CLK_GOUT_PERIC1_TOP0_IPCLK_13		37
-+#define CLK_GOUT_PERIC1_TOP0_IPCLK_14		38
-+#define CLK_GOUT_PERIC1_TOP0_IPCLK_15		39
-+#define CLK_GOUT_PERIC1_TOP0_IPCLK_4		40
-+#define CLK_GOUT_PERIC1_TOP0_PCLK_10		41
-+#define CLK_GOUT_PERIC1_TOP0_PCLK_11		42
-+#define CLK_GOUT_PERIC1_TOP0_PCLK_12		43
-+#define CLK_GOUT_PERIC1_TOP0_PCLK_13		44
-+#define CLK_GOUT_PERIC1_TOP0_PCLK_14		45
-+#define CLK_GOUT_PERIC1_TOP0_PCLK_15		46
-+#define CLK_GOUT_PERIC1_TOP0_PCLK_4		47
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_0		48
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_1		49
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_10		50
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_12		51
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_13		52
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_14		53
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_15		54
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_2		55
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_3		56
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_4		57
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_5		58
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_6		59
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_7		60
-+#define CLK_GOUT_PERIC1_TOP1_IPCLK_9		61
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_0		62
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_1		63
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_10		64
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_12		65
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_13		66
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_14		67
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_15		68
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_2		69
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_3		70
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_4		71
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_5		72
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_6		73
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_7		74
-+#define CLK_GOUT_PERIC1_TOP1_PCLK_9		75
-+#define CLK_GOUT_PERIC1_BUSP_CLK		76
-+#define CLK_GOUT_PERIC1_OSCCLK_CLK		77
-+#define CLK_GOUT_PERIC1_USI06_USI_CLK		78
-+#define CLK_GOUT_PERIC1_USI07_USI_CLK		79
-+#define CLK_GOUT_PERIC1_USI08_USI_CLK		80
-+#define CLK_GOUT_PERIC1_USI09_USI_CLK		81
-+#define CLK_GOUT_PERIC1_USI10_USI_CLK		82
-+#define CLK_GOUT_PERIC1_USI11_USI_CLK		83
-+#define CLK_GOUT_PERIC1_USI16_USI_CLK		84
-+#define CLK_GOUT_PERIC1_USI17_USI_CLK		85
-+#define CLK_GOUT_PERIC1_USI_I2C_CLK		86
-+#define CLK_GOUT_PERIC1_SYSREG_PCLK		87
-+#define CLK_GOUT_PERIC1_USI16_I3C_PCLK		88
-+#define CLK_GOUT_PERIC1_USI16_I3C_SCLK		89
-+#define CLK_GOUT_PERIC1_USI17_I3C_PCLK		90
-+#define CLK_GOUT_PERIC1_USI17_I3C_SCLK		91
-+#define CLK_GOUT_PERIC1_XIU_P_ACLK		92
-+
-+/* CMU_PERIS */
-+#define CLK_MOUT_PERIS_BUS_USER			1
-+#define CLK_MOUT_PERIS_CLK_PERIS_GIC		2
-+#define CLK_GOUT_PERIS_SYSREG_PERIS_PCLK	3
-+#define CLK_GOUT_PERIS_WDT_CLUSTER2_PCLK	4
-+#define CLK_GOUT_PERIS_WDT_CLUSTER0_PCLK	5
-+#define CLK_CLK_PERIS_PERIS_CMU_PERIS_PCLK	6
-+#define CLK_GOUT_PERIS_CLK_PERIS_BUSP_CLK	7
-+#define CLK_GOUT_PERIS_CLK_PERIS_OSCCLK_CLK	8
-+#define CLK_GOUT_PERIS_CLK_PERIS_GIC_CLK	9
-+#define CLK_GOUT_PERIS_AD_AXI_P_PERIS_ACLKM	10
-+#define CLK_GOUT_PERIS_OTP_CON_BIRA_PCLK	11
-+#define CLK_GOUT_PERIS_GIC_CLK			12
-+#define CLK_GOUT_PERIS_LHM_AXI_P_PERIS_CLK	13
-+#define CLK_GOUT_PERIS_MCT_PCLK			14
-+#define CLK_GOUT_PERIS_OTP_CON_TOP_PCLK		15
-+#define CLK_GOUT_PERIS_D_TZPC_PERIS_PCLK	16
-+#define CLK_GOUT_PERIS_TMU_TOP_PCLK		17
-+#define CLK_GOUT_PERIS_OTP_CON_BIRA_OSCCLK	18
-+#define CLK_GOUT_PERIS_OTP_CON_TOP_OSCCLK	19
-+
-+#endif
-diff --git a/include/dt-bindings/clock/samsung,exynosautov9.h b/include/dt-bindings/clock/samsung,exynosautov9.h
-index ea9f91b4eb1a..ce8fb8f7d718 100644
--- a/include/dt-bindings/clock/samsung,exynosautov9.h
-+++ b/include/dt-bindings/clock/samsung,exynosautov9.h
-@@ -166,16 +166,12 @@
- #define GOUT_CLKCMU_PERIC1_IP		248
- #define GOUT_CLKCMU_PERIS_BUS		249
- 
-#define TOP_NR_CLK			250
-
- /* CMU_BUSMC */
- #define CLK_MOUT_BUSMC_BUS_USER		1
- #define CLK_DOUT_BUSMC_BUSP		2
- #define CLK_GOUT_BUSMC_PDMA0_PCLK	3
- #define CLK_GOUT_BUSMC_SPDMA_PCLK	4
- 
-#define BUSMC_NR_CLK			5
-
- /* CMU_CORE */
- #define CLK_MOUT_CORE_BUS_USER		1
- #define CLK_DOUT_CORE_BUSP		2
-@@ -183,7 +179,80 @@
- #define CLK_GOUT_CORE_CCI_PCLK		4
- #define CLK_GOUT_CORE_CMU_CORE_PCLK	5
- 
-#define CORE_NR_CLK			6
-+/* CMU_DPUM */
-+#define CLK_MOUT_DPUM_BUS_USER		1
-+#define CLK_DOUT_DPUM_BUSP		2
-+#define CLK_GOUT_DPUM_ACLK_DECON	3
-+#define CLK_GOUT_DPUM_ACLK_DMA		4
-+#define CLK_GOUT_DPUM_ACLK_DPP		5
-+#define CLK_GOUT_DPUM_SYSMMU_D0_CLK	6
-+#define CLK_GOUT_DPUM_SYSMMU_D1_CLK	7
-+#define CLK_GOUT_DPUM_SYSMMU_D2_CLK	8
-+#define CLK_GOUT_DPUM_SYSMMU_D3_CLK	9
-+
-+/* CMU_FSYS0 */
-+#define CLK_MOUT_FSYS0_BUS_USER		1
-+#define CLK_MOUT_FSYS0_PCIE_USER	2
-+#define CLK_GOUT_FSYS0_BUS_PCLK		3
-+
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_REFCLK		4
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_REFCLK		5
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_DBI_ACLK	6
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_MSTR_ACLK	7
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_SLV_ACLK	8
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_DBI_ACLK	9
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_MSTR_ACLK	10
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_SLV_ACLK	11
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_PIPE_CLK	12
-+#define CLK_GOUT_FSYS0_PCIE_GEN3A_2L0_CLK		13
-+#define CLK_GOUT_FSYS0_PCIE_GEN3B_2L0_CLK		14
-+
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_REFCLK		15
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_REFCLK		16
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_DBI_ACLK	17
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_MSTR_ACLK	18
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_SLV_ACLK	19
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_DBI_ACLK	20
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_MSTR_ACLK	21
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_SLV_ACLK	22
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_PIPE_CLK	23
-+#define CLK_GOUT_FSYS0_PCIE_GEN3A_2L1_CLK		24
-+#define CLK_GOUT_FSYS0_PCIE_GEN3B_2L1_CLK		25
-+
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_REFCLK		26
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_REFCLK		27
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_DBI_ACLK		28
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_MSTR_ACLK	29
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_SLV_ACLK		30
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_DBI_ACLK		31
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_MSTR_ACLK	32
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_SLV_ACLK		33
-+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_PIPE_CLK		34
-+#define CLK_GOUT_FSYS0_PCIE_GEN3A_4L_CLK		35
-+#define CLK_GOUT_FSYS0_PCIE_GEN3B_4L_CLK		36
-+
-+/* CMU_FSYS1 */
-+#define FOUT_MMC_PLL				1
-+
-+#define CLK_MOUT_FSYS1_BUS_USER			2
-+#define CLK_MOUT_FSYS1_MMC_PLL			3
-+#define CLK_MOUT_FSYS1_MMC_CARD_USER		4
-+#define CLK_MOUT_FSYS1_USBDRD_USER		5
-+#define CLK_MOUT_FSYS1_MMC_CARD			6
-+
-+#define CLK_DOUT_FSYS1_MMC_CARD			7
-+
-+#define CLK_GOUT_FSYS1_PCLK			8
-+#define CLK_GOUT_FSYS1_MMC_CARD_SDCLKIN		9
-+#define CLK_GOUT_FSYS1_MMC_CARD_ACLK		10
-+#define CLK_GOUT_FSYS1_USB20DRD_0_REFCLK	11
-+#define CLK_GOUT_FSYS1_USB20DRD_1_REFCLK	12
-+#define CLK_GOUT_FSYS1_USB30DRD_0_REFCLK	13
-+#define CLK_GOUT_FSYS1_USB30DRD_1_REFCLK	14
-+#define CLK_GOUT_FSYS1_USB20_0_ACLK		15
-+#define CLK_GOUT_FSYS1_USB20_1_ACLK		16
-+#define CLK_GOUT_FSYS1_USB30_0_ACLK		17
-+#define CLK_GOUT_FSYS1_USB30_1_ACLK		18
- 
- /* CMU_FSYS2 */
- #define CLK_MOUT_FSYS2_BUS_USER		1
-@@ -194,8 +263,6 @@
- #define CLK_GOUT_FSYS2_UFS_EMBD1_ACLK	6
- #define CLK_GOUT_FSYS2_UFS_EMBD1_UNIPRO	7
- 
-#define FSYS2_NR_CLK			8
-
- /* CMU_PERIC0 */
- #define CLK_MOUT_PERIC0_BUS_USER	1
- #define CLK_MOUT_PERIC0_IP_USER		2
-@@ -226,21 +293,19 @@
- #define CLK_GOUT_PERIC0_IPCLK_8		28
- #define CLK_GOUT_PERIC0_IPCLK_9		29
- #define CLK_GOUT_PERIC0_IPCLK_10	30
-#define CLK_GOUT_PERIC0_IPCLK_11	30
-#define CLK_GOUT_PERIC0_PCLK_0		31
-#define CLK_GOUT_PERIC0_PCLK_1		32
-#define CLK_GOUT_PERIC0_PCLK_2		33
-#define CLK_GOUT_PERIC0_PCLK_3		34
-#define CLK_GOUT_PERIC0_PCLK_4		35
-#define CLK_GOUT_PERIC0_PCLK_5		36
-#define CLK_GOUT_PERIC0_PCLK_6		37
-#define CLK_GOUT_PERIC0_PCLK_7		38
-#define CLK_GOUT_PERIC0_PCLK_8		39
-#define CLK_GOUT_PERIC0_PCLK_9		40
-#define CLK_GOUT_PERIC0_PCLK_10		41
-#define CLK_GOUT_PERIC0_PCLK_11		42
-
-#define PERIC0_NR_CLK			43
-+#define CLK_GOUT_PERIC0_IPCLK_11	31
-+#define CLK_GOUT_PERIC0_PCLK_0		32
-+#define CLK_GOUT_PERIC0_PCLK_1		33
-+#define CLK_GOUT_PERIC0_PCLK_2		34
-+#define CLK_GOUT_PERIC0_PCLK_3		35
-+#define CLK_GOUT_PERIC0_PCLK_4		36
-+#define CLK_GOUT_PERIC0_PCLK_5		37
-+#define CLK_GOUT_PERIC0_PCLK_6		38
-+#define CLK_GOUT_PERIC0_PCLK_7		39
-+#define CLK_GOUT_PERIC0_PCLK_8		40
-+#define CLK_GOUT_PERIC0_PCLK_9		41
-+#define CLK_GOUT_PERIC0_PCLK_10		42
-+#define CLK_GOUT_PERIC0_PCLK_11		43
- 
- /* CMU_PERIC1 */
- #define CLK_MOUT_PERIC1_BUS_USER	1
-@@ -272,21 +337,19 @@
- #define CLK_GOUT_PERIC1_IPCLK_8		28
- #define CLK_GOUT_PERIC1_IPCLK_9		29
- #define CLK_GOUT_PERIC1_IPCLK_10	30
-#define CLK_GOUT_PERIC1_IPCLK_11	30
-#define CLK_GOUT_PERIC1_PCLK_0		31
-#define CLK_GOUT_PERIC1_PCLK_1		32
-#define CLK_GOUT_PERIC1_PCLK_2		33
-#define CLK_GOUT_PERIC1_PCLK_3		34
-#define CLK_GOUT_PERIC1_PCLK_4		35
-#define CLK_GOUT_PERIC1_PCLK_5		36
-#define CLK_GOUT_PERIC1_PCLK_6		37
-#define CLK_GOUT_PERIC1_PCLK_7		38
-#define CLK_GOUT_PERIC1_PCLK_8		39
-#define CLK_GOUT_PERIC1_PCLK_9		40
-#define CLK_GOUT_PERIC1_PCLK_10		41
-#define CLK_GOUT_PERIC1_PCLK_11		42
-
-#define PERIC1_NR_CLK			43
-+#define CLK_GOUT_PERIC1_IPCLK_11	31
-+#define CLK_GOUT_PERIC1_PCLK_0		32
-+#define CLK_GOUT_PERIC1_PCLK_1		33
-+#define CLK_GOUT_PERIC1_PCLK_2		34
-+#define CLK_GOUT_PERIC1_PCLK_3		35
-+#define CLK_GOUT_PERIC1_PCLK_4		36
-+#define CLK_GOUT_PERIC1_PCLK_5		37
-+#define CLK_GOUT_PERIC1_PCLK_6		38
-+#define CLK_GOUT_PERIC1_PCLK_7		39
-+#define CLK_GOUT_PERIC1_PCLK_8		40
-+#define CLK_GOUT_PERIC1_PCLK_9		41
-+#define CLK_GOUT_PERIC1_PCLK_10		42
-+#define CLK_GOUT_PERIC1_PCLK_11		43
- 
- /* CMU_PERIS */
- #define CLK_MOUT_PERIS_BUS_USER		1
-@@ -294,6 +357,4 @@
- #define CLK_GOUT_WDT_CLUSTER0		3
- #define CLK_GOUT_WDT_CLUSTER1		4
- 
-#define PERIS_NR_CLK			5
-
- #endif /* _DT_BINDINGS_CLOCK_EXYNOSAUTOV9_H */
-diff --git a/include/dt-bindings/clock/samsung,exynosautov920.h b/include/dt-bindings/clock/samsung,exynosautov920.h
-new file mode 100644
-index 000000000000..970d05167fc6
--- /dev/null
-+++ b/include/dt-bindings/clock/samsung,exynosautov920.h
-@@ -0,0 +1,308 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024 Samsung Electronics Co., Ltd.
-+ * Author: Sunyeal Hong <sunyeal.hong@samsung.com>
-+ *
-+ * Device Tree binding constants for ExynosAuto v920 clock controller.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLOCK_EXYNOSAUTOV920_H
-+#define _DT_BINDINGS_CLOCK_EXYNOSAUTOV920_H
-+
-+/* CMU_TOP */
-+#define FOUT_SHARED0_PLL		1
-+#define FOUT_SHARED1_PLL		2
-+#define FOUT_SHARED2_PLL		3
-+#define FOUT_SHARED3_PLL		4
-+#define FOUT_SHARED4_PLL		5
-+#define FOUT_SHARED5_PLL		6
-+#define FOUT_MMC_PLL			7
-+
-+/* MUX in CMU_TOP */
-+#define MOUT_SHARED0_PLL		8
-+#define MOUT_SHARED1_PLL		9
-+#define MOUT_SHARED2_PLL		10
-+#define MOUT_SHARED3_PLL		11
-+#define MOUT_SHARED4_PLL		12
-+#define MOUT_SHARED5_PLL		13
-+#define MOUT_MMC_PLL			14
-+#define MOUT_CLKCMU_CMU_BOOST		15
-+#define MOUT_CLKCMU_CMU_CMUREF		16
-+#define MOUT_CLKCMU_ACC_NOC		17
-+#define MOUT_CLKCMU_ACC_ORB		18
-+#define MOUT_CLKCMU_APM_NOC		19
-+#define MOUT_CLKCMU_AUD_CPU		20
-+#define MOUT_CLKCMU_AUD_NOC		21
-+#define MOUT_CLKCMU_CPUCL0_SWITCH	22
-+#define MOUT_CLKCMU_CPUCL0_CLUSTER	23
-+#define MOUT_CLKCMU_CPUCL0_DBG		24
-+#define MOUT_CLKCMU_CPUCL1_SWITCH	25
-+#define MOUT_CLKCMU_CPUCL1_CLUSTER	26
-+#define MOUT_CLKCMU_CPUCL2_SWITCH	27
-+#define MOUT_CLKCMU_CPUCL2_CLUSTER	28
-+#define MOUT_CLKCMU_DNC_NOC		29
-+#define MOUT_CLKCMU_DPTX_NOC		30
-+#define MOUT_CLKCMU_DPTX_DPGTC		31
-+#define MOUT_CLKCMU_DPTX_DPOSC		32
-+#define MOUT_CLKCMU_DPUB_NOC		33
-+#define MOUT_CLKCMU_DPUB_DSIM		34
-+#define MOUT_CLKCMU_DPUF0_NOC		35
-+#define MOUT_CLKCMU_DPUF1_NOC		36
-+#define MOUT_CLKCMU_DPUF2_NOC		37
-+#define MOUT_CLKCMU_DSP_NOC		38
-+#define MOUT_CLKCMU_G3D_SWITCH		39
-+#define MOUT_CLKCMU_G3D_NOCP		40
-+#define MOUT_CLKCMU_GNPU_NOC		41
-+#define MOUT_CLKCMU_HSI0_NOC		42
-+#define MOUT_CLKCMU_HSI1_NOC		43
-+#define MOUT_CLKCMU_HSI1_USBDRD		44
-+#define MOUT_CLKCMU_HSI1_MMC_CARD	45
-+#define MOUT_CLKCMU_HSI2_NOC		46
-+#define MOUT_CLKCMU_HSI2_NOC_UFS	47
-+#define MOUT_CLKCMU_HSI2_UFS_EMBD	48
-+#define MOUT_CLKCMU_HSI2_ETHERNET	49
-+#define MOUT_CLKCMU_ISP_NOC		50
-+#define MOUT_CLKCMU_M2M_NOC		51
-+#define MOUT_CLKCMU_M2M_JPEG		52
-+#define MOUT_CLKCMU_MFC_MFC		53
-+#define MOUT_CLKCMU_MFC_WFD		54
-+#define MOUT_CLKCMU_MFD_NOC		55
-+#define MOUT_CLKCMU_MIF_SWITCH		56
-+#define MOUT_CLKCMU_MIF_NOCP		57
-+#define MOUT_CLKCMU_MISC_NOC		58
-+#define MOUT_CLKCMU_NOCL0_NOC		59
-+#define MOUT_CLKCMU_NOCL1_NOC		60
-+#define MOUT_CLKCMU_NOCL2_NOC		61
-+#define MOUT_CLKCMU_PERIC0_NOC		62
-+#define MOUT_CLKCMU_PERIC0_IP		63
-+#define MOUT_CLKCMU_PERIC1_NOC		64
-+#define MOUT_CLKCMU_PERIC1_IP		65
-+#define MOUT_CLKCMU_SDMA_NOC		66
-+#define MOUT_CLKCMU_SNW_NOC		67
-+#define MOUT_CLKCMU_SSP_NOC		68
-+#define MOUT_CLKCMU_TAA_NOC		69
-+
-+/* DIV in CMU_TOP */
-+#define DOUT_SHARED0_DIV1		70
-+#define DOUT_SHARED0_DIV2		71
-+#define DOUT_SHARED0_DIV3		72
-+#define DOUT_SHARED0_DIV4		73
-+#define DOUT_SHARED1_DIV1		74
-+#define DOUT_SHARED1_DIV2		75
-+#define DOUT_SHARED1_DIV3		76
-+#define DOUT_SHARED1_DIV4		77
-+#define DOUT_SHARED2_DIV1		78
-+#define DOUT_SHARED2_DIV2		79
-+#define DOUT_SHARED2_DIV3		80
-+#define DOUT_SHARED2_DIV4		81
-+#define DOUT_SHARED3_DIV1		82
-+#define DOUT_SHARED3_DIV2		83
-+#define DOUT_SHARED3_DIV3		84
-+#define DOUT_SHARED3_DIV4		85
-+#define DOUT_SHARED4_DIV1		86
-+#define DOUT_SHARED4_DIV2		87
-+#define DOUT_SHARED4_DIV3		88
-+#define DOUT_SHARED4_DIV4		89
-+#define DOUT_SHARED5_DIV1		90
-+#define DOUT_SHARED5_DIV2		91
-+#define DOUT_SHARED5_DIV3		92
-+#define DOUT_SHARED5_DIV4		93
-+#define DOUT_CLKCMU_CMU_BOOST		94
-+#define DOUT_CLKCMU_ACC_NOC		95
-+#define DOUT_CLKCMU_ACC_ORB		96
-+#define DOUT_CLKCMU_APM_NOC		97
-+#define DOUT_CLKCMU_AUD_CPU		98
-+#define DOUT_CLKCMU_AUD_NOC		99
-+#define DOUT_CLKCMU_CPUCL0_SWITCH	100
-+#define DOUT_CLKCMU_CPUCL0_CLUSTER	101
-+#define DOUT_CLKCMU_CPUCL0_DBG		102
-+#define DOUT_CLKCMU_CPUCL1_SWITCH	103
-+#define DOUT_CLKCMU_CPUCL1_CLUSTER	104
-+#define DOUT_CLKCMU_CPUCL2_SWITCH	105
-+#define DOUT_CLKCMU_CPUCL2_CLUSTER	106
-+#define DOUT_CLKCMU_DNC_NOC		107
-+#define DOUT_CLKCMU_DPTX_NOC		108
-+#define DOUT_CLKCMU_DPTX_DPGTC		109
-+#define DOUT_CLKCMU_DPTX_DPOSC		110
-+#define DOUT_CLKCMU_DPUB_NOC		111
-+#define DOUT_CLKCMU_DPUB_DSIM		112
-+#define DOUT_CLKCMU_DPUF0_NOC		113
-+#define DOUT_CLKCMU_DPUF1_NOC		114
-+#define DOUT_CLKCMU_DPUF2_NOC		115
-+#define DOUT_CLKCMU_DSP_NOC		116
-+#define DOUT_CLKCMU_G3D_SWITCH		117
-+#define DOUT_CLKCMU_G3D_NOCP		118
-+#define DOUT_CLKCMU_GNPU_NOC		119
-+#define DOUT_CLKCMU_HSI0_NOC		120
-+#define DOUT_CLKCMU_HSI1_NOC		121
-+#define DOUT_CLKCMU_HSI1_USBDRD		122
-+#define DOUT_CLKCMU_HSI1_MMC_CARD	123
-+#define DOUT_CLKCMU_HSI2_NOC		124
-+#define DOUT_CLKCMU_HSI2_NOC_UFS	125
-+#define DOUT_CLKCMU_HSI2_UFS_EMBD	126
-+#define DOUT_CLKCMU_HSI2_ETHERNET	127
-+#define DOUT_CLKCMU_ISP_NOC		128
-+#define DOUT_CLKCMU_M2M_NOC		129
-+#define DOUT_CLKCMU_M2M_JPEG		130
-+#define DOUT_CLKCMU_MFC_MFC		131
-+#define DOUT_CLKCMU_MFC_WFD		132
-+#define DOUT_CLKCMU_MFD_NOC		133
-+#define DOUT_CLKCMU_MIF_NOCP		134
-+#define DOUT_CLKCMU_MISC_NOC		135
-+#define DOUT_CLKCMU_NOCL0_NOC		136
-+#define DOUT_CLKCMU_NOCL1_NOC		137
-+#define DOUT_CLKCMU_NOCL2_NOC		138
-+#define DOUT_CLKCMU_PERIC0_NOC		139
-+#define DOUT_CLKCMU_PERIC0_IP		140
-+#define DOUT_CLKCMU_PERIC1_NOC		141
-+#define DOUT_CLKCMU_PERIC1_IP		142
-+#define DOUT_CLKCMU_SDMA_NOC		143
-+#define DOUT_CLKCMU_SNW_NOC		144
-+#define DOUT_CLKCMU_SSP_NOC		145
-+#define DOUT_CLKCMU_TAA_NOC		146
-+#define DOUT_TCXO_DIV2			147
-+
-+/* CMU_CPUCL0 */
-+#define CLK_FOUT_CPUCL0_PLL		1
-+
-+#define CLK_MOUT_PLL_CPUCL0		2
-+#define CLK_MOUT_CPUCL0_CLUSTER_USER	3
-+#define CLK_MOUT_CPUCL0_DBG_USER	4
-+#define CLK_MOUT_CPUCL0_SWITCH_USER	5
-+#define CLK_MOUT_CPUCL0_CLUSTER		6
-+#define CLK_MOUT_CPUCL0_CORE		7
-+
-+#define CLK_DOUT_CLUSTER0_ACLK		8
-+#define CLK_DOUT_CLUSTER0_ATCLK		9
-+#define CLK_DOUT_CLUSTER0_MPCLK		10
-+#define CLK_DOUT_CLUSTER0_PCLK		11
-+#define CLK_DOUT_CLUSTER0_PERIPHCLK	12
-+#define CLK_DOUT_CPUCL0_DBG_NOC		13
-+#define CLK_DOUT_CPUCL0_DBG_PCLKDBG	14
-+#define CLK_DOUT_CPUCL0_NOCP		15
-+
-+/* CMU_CPUCL1 */
-+#define CLK_FOUT_CPUCL1_PLL		1
-+
-+#define CLK_MOUT_PLL_CPUCL1		2
-+#define CLK_MOUT_CPUCL1_CLUSTER_USER	3
-+#define CLK_MOUT_CPUCL1_SWITCH_USER	4
-+#define CLK_MOUT_CPUCL1_CLUSTER		5
-+#define CLK_MOUT_CPUCL1_CORE		6
-+
-+#define CLK_DOUT_CLUSTER1_ACLK		7
-+#define CLK_DOUT_CLUSTER1_ATCLK		8
-+#define CLK_DOUT_CLUSTER1_MPCLK		9
-+#define CLK_DOUT_CLUSTER1_PCLK		10
-+#define CLK_DOUT_CLUSTER1_PERIPHCLK	11
-+#define CLK_DOUT_CPUCL1_NOCP		12
-+
-+/* CMU_CPUCL2 */
-+#define CLK_FOUT_CPUCL2_PLL		1
-+
-+#define CLK_MOUT_PLL_CPUCL2		2
-+#define CLK_MOUT_CPUCL2_CLUSTER_USER	3
-+#define CLK_MOUT_CPUCL2_SWITCH_USER	4
-+#define CLK_MOUT_CPUCL2_CLUSTER		5
-+#define CLK_MOUT_CPUCL2_CORE		6
-+
-+#define CLK_DOUT_CLUSTER2_ACLK		7
-+#define CLK_DOUT_CLUSTER2_ATCLK		8
-+#define CLK_DOUT_CLUSTER2_MPCLK		9
-+#define CLK_DOUT_CLUSTER2_PCLK		10
-+#define CLK_DOUT_CLUSTER2_PERIPHCLK	11
-+#define CLK_DOUT_CPUCL2_NOCP		12
-+
-+/* CMU_PERIC0 */
-+#define CLK_MOUT_PERIC0_IP_USER		1
-+#define CLK_MOUT_PERIC0_NOC_USER	2
-+#define CLK_MOUT_PERIC0_USI00_USI	3
-+#define CLK_MOUT_PERIC0_USI01_USI	4
-+#define CLK_MOUT_PERIC0_USI02_USI	5
-+#define CLK_MOUT_PERIC0_USI03_USI	6
-+#define CLK_MOUT_PERIC0_USI04_USI	7
-+#define CLK_MOUT_PERIC0_USI05_USI	8
-+#define CLK_MOUT_PERIC0_USI06_USI	9
-+#define CLK_MOUT_PERIC0_USI07_USI	10
-+#define CLK_MOUT_PERIC0_USI08_USI	11
-+#define CLK_MOUT_PERIC0_USI_I2C		12
-+#define CLK_MOUT_PERIC0_I3C		13
-+
-+#define CLK_DOUT_PERIC0_USI00_USI	14
-+#define CLK_DOUT_PERIC0_USI01_USI	15
-+#define CLK_DOUT_PERIC0_USI02_USI	16
-+#define CLK_DOUT_PERIC0_USI03_USI	17
-+#define CLK_DOUT_PERIC0_USI04_USI	18
-+#define CLK_DOUT_PERIC0_USI05_USI	19
-+#define CLK_DOUT_PERIC0_USI06_USI	20
-+#define CLK_DOUT_PERIC0_USI07_USI	21
-+#define CLK_DOUT_PERIC0_USI08_USI	22
-+#define CLK_DOUT_PERIC0_USI_I2C		23
-+#define CLK_DOUT_PERIC0_I3C		24
-+
-+/* CMU_PERIC1 */
-+#define CLK_MOUT_PERIC1_IP_USER		1
-+#define CLK_MOUT_PERIC1_NOC_USER	2
-+#define CLK_MOUT_PERIC1_USI09_USI	3
-+#define CLK_MOUT_PERIC1_USI10_USI	4
-+#define CLK_MOUT_PERIC1_USI11_USI	5
-+#define CLK_MOUT_PERIC1_USI12_USI	6
-+#define CLK_MOUT_PERIC1_USI13_USI	7
-+#define CLK_MOUT_PERIC1_USI14_USI	8
-+#define CLK_MOUT_PERIC1_USI15_USI	9
-+#define CLK_MOUT_PERIC1_USI16_USI	10
-+#define CLK_MOUT_PERIC1_USI17_USI	11
-+#define CLK_MOUT_PERIC1_USI_I2C		12
-+#define CLK_MOUT_PERIC1_I3C		13
-+
-+#define CLK_DOUT_PERIC1_USI09_USI	14
-+#define CLK_DOUT_PERIC1_USI10_USI	15
-+#define CLK_DOUT_PERIC1_USI11_USI	16
-+#define CLK_DOUT_PERIC1_USI12_USI	17
-+#define CLK_DOUT_PERIC1_USI13_USI	18
-+#define CLK_DOUT_PERIC1_USI14_USI	19
-+#define CLK_DOUT_PERIC1_USI15_USI	20
-+#define CLK_DOUT_PERIC1_USI16_USI	21
-+#define CLK_DOUT_PERIC1_USI17_USI	22
-+#define CLK_DOUT_PERIC1_USI_I2C		23
-+#define CLK_DOUT_PERIC1_I3C		24
-+
-+/* CMU_MISC */
-+#define CLK_MOUT_MISC_NOC_USER		1
-+#define CLK_MOUT_MISC_GIC		2
-+
-+#define CLK_DOUT_MISC_OTP		3
-+#define CLK_DOUT_MISC_NOCP		4
-+#define CLK_DOUT_MISC_OSC_DIV2		5
-+
-+/* CMU_HSI0 */
-+#define CLK_MOUT_HSI0_NOC_USER		1
-+
-+#define CLK_DOUT_HSI0_PCIE_APB		2
-+
-+/* CMU_HSI1 */
-+#define CLK_MOUT_HSI1_MMC_CARD_USER	1
-+#define CLK_MOUT_HSI1_NOC_USER		2
-+#define CLK_MOUT_HSI1_USBDRD_USER	3
-+#define CLK_MOUT_HSI1_USBDRD		4
-+
-+/* CMU_HSI2 */
-+#define FOUT_PLL_ETH                    1
-+#define CLK_MOUT_HSI2_NOC_UFS_USER      2
-+#define CLK_MOUT_HSI2_UFS_EMBD_USER     3
-+#define CLK_MOUT_HSI2_ETHERNET          4
-+#define CLK_MOUT_HSI2_ETHERNET_USER     5
-+#define CLK_DOUT_HSI2_ETHERNET          6
-+#define CLK_DOUT_HSI2_ETHERNET_PTP      7
-+
-+/* CMU_M2M */
-+#define CLK_MOUT_M2M_JPEG_USER          1
-+#define CLK_MOUT_M2M_NOC_USER           2
-+#define CLK_DOUT_M2M_NOCP               3
-+
-+/* CMU_MFC */
-+#define CLK_MOUT_MFC_MFC_USER           1
-+#define CLK_MOUT_MFC_WFD_USER           2
-+#define CLK_DOUT_MFC_NOCP               3
-+
-+#endif /* _DT_BINDINGS_CLOCK_EXYNOSAUTOV920_H */
-diff --git a/include/dt-bindings/clock/sophgo,cv1800.h b/include/dt-bindings/clock/sophgo,cv1800.h
-new file mode 100644
-index 000000000000..cfbeca25a650
--- /dev/null
-+++ b/include/dt-bindings/clock/sophgo,cv1800.h
-@@ -0,0 +1,176 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2023 Sophgo Ltd.
-+ */
-+
-+#ifndef __DT_BINDINGS_SOPHGO_CV1800_CLK_H__
-+#define __DT_BINDINGS_SOPHGO_CV1800_CLK_H__
-+
-+#define CLK_MPLL			0
-+#define CLK_TPLL			1
-+#define CLK_FPLL			2
-+#define CLK_MIPIMPLL			3
-+#define CLK_A0PLL			4
-+#define CLK_DISPPLL			5
-+#define CLK_CAM0PLL			6
-+#define CLK_CAM1PLL			7
-+
-+#define CLK_MIPIMPLL_D3			8
-+#define CLK_CAM0PLL_D2			9
-+#define CLK_CAM0PLL_D3			10
-+
-+#define CLK_TPU				11
-+#define CLK_TPU_FAB			12
-+#define CLK_AHB_ROM			13
-+#define CLK_DDR_AXI_REG			14
-+#define CLK_RTC_25M			15
-+#define CLK_SRC_RTC_SYS_0		16
-+#define CLK_TEMPSEN			17
-+#define CLK_SARADC			18
-+#define CLK_EFUSE			19
-+#define CLK_APB_EFUSE			20
-+#define CLK_DEBUG			21
-+#define CLK_AP_DEBUG			22
-+#define CLK_XTAL_MISC			23
-+#define CLK_AXI4_EMMC			24
-+#define CLK_EMMC			25
-+#define CLK_EMMC_100K			26
-+#define CLK_AXI4_SD0			27
-+#define CLK_SD0				28
-+#define CLK_SD0_100K			29
-+#define CLK_AXI4_SD1			30
-+#define CLK_SD1				31
-+#define CLK_SD1_100K			32
-+#define CLK_SPI_NAND			33
-+#define CLK_ETH0_500M			34
-+#define CLK_AXI4_ETH0			35
-+#define CLK_ETH1_500M			36
-+#define CLK_AXI4_ETH1			37
-+#define CLK_APB_GPIO			38
-+#define CLK_APB_GPIO_INTR		39
-+#define CLK_GPIO_DB			40
-+#define CLK_AHB_SF			41
-+#define CLK_AHB_SF1			42
-+#define CLK_A24M			43
-+#define CLK_AUDSRC			44
-+#define CLK_APB_AUDSRC			45
-+#define CLK_SDMA_AXI			46
-+#define CLK_SDMA_AUD0			47
-+#define CLK_SDMA_AUD1			48
-+#define CLK_SDMA_AUD2			49
-+#define CLK_SDMA_AUD3			50
-+#define CLK_I2C				51
-+#define CLK_APB_I2C			52
-+#define CLK_APB_I2C0			53
-+#define CLK_APB_I2C1			54
-+#define CLK_APB_I2C2			55
-+#define CLK_APB_I2C3			56
-+#define CLK_APB_I2C4			57
-+#define CLK_APB_WDT			58
-+#define CLK_PWM_SRC			59
-+#define CLK_PWM				60
-+#define CLK_SPI				61
-+#define CLK_APB_SPI0			62
-+#define CLK_APB_SPI1			63
-+#define CLK_APB_SPI2			64
-+#define CLK_APB_SPI3			65
-+#define CLK_1M				66
-+#define CLK_CAM0_200			67
-+#define CLK_PM				68
-+#define CLK_TIMER0			69
-+#define CLK_TIMER1			70
-+#define CLK_TIMER2			71
-+#define CLK_TIMER3			72
-+#define CLK_TIMER4			73
-+#define CLK_TIMER5			74
-+#define CLK_TIMER6			75
-+#define CLK_TIMER7			76
-+#define CLK_UART0			77
-+#define CLK_APB_UART0			78
-+#define CLK_UART1			79
-+#define CLK_APB_UART1			80
-+#define CLK_UART2			81
-+#define CLK_APB_UART2			82
-+#define CLK_UART3			83
-+#define CLK_APB_UART3			84
-+#define CLK_UART4			85
-+#define CLK_APB_UART4			86
-+#define CLK_APB_I2S0			87
-+#define CLK_APB_I2S1			88
-+#define CLK_APB_I2S2			89
-+#define CLK_APB_I2S3			90
-+#define CLK_AXI4_USB			91
-+#define CLK_APB_USB			92
-+#define CLK_USB_125M			93
-+#define CLK_USB_33K			94
-+#define CLK_USB_12M			95
-+#define CLK_AXI4			96
-+#define CLK_AXI6			97
-+#define CLK_DSI_ESC			98
-+#define CLK_AXI_VIP			99
-+#define CLK_SRC_VIP_SYS_0		100
-+#define CLK_SRC_VIP_SYS_1		101
-+#define CLK_SRC_VIP_SYS_2		102
-+#define CLK_SRC_VIP_SYS_3		103
-+#define CLK_SRC_VIP_SYS_4		104
-+#define CLK_CSI_BE_VIP			105
-+#define CLK_CSI_MAC0_VIP		106
-+#define CLK_CSI_MAC1_VIP		107
-+#define CLK_CSI_MAC2_VIP		108
-+#define CLK_CSI0_RX_VIP			109
-+#define CLK_CSI1_RX_VIP			110
-+#define CLK_ISP_TOP_VIP			111
-+#define CLK_IMG_D_VIP			112
-+#define CLK_IMG_V_VIP			113
-+#define CLK_SC_TOP_VIP			114
-+#define CLK_SC_D_VIP			115
-+#define CLK_SC_V1_VIP			116
-+#define CLK_SC_V2_VIP			117
-+#define CLK_SC_V3_VIP			118
-+#define CLK_DWA_VIP			119
-+#define CLK_BT_VIP			120
-+#define CLK_DISP_VIP			121
-+#define CLK_DSI_MAC_VIP			122
-+#define CLK_LVDS0_VIP			123
-+#define CLK_LVDS1_VIP			124
-+#define CLK_PAD_VI_VIP			125
-+#define CLK_PAD_VI1_VIP			126
-+#define CLK_PAD_VI2_VIP			127
-+#define CLK_CFG_REG_VIP			128
-+#define CLK_VIP_IP0			129
-+#define CLK_VIP_IP1			130
-+#define CLK_VIP_IP2			131
-+#define CLK_VIP_IP3			132
-+#define CLK_IVE_VIP			133
-+#define CLK_RAW_VIP			134
-+#define CLK_OSDC_VIP			135
-+#define CLK_CAM0_VIP			136
-+#define CLK_AXI_VIDEO_CODEC		137
-+#define CLK_VC_SRC0			138
-+#define CLK_VC_SRC1			139
-+#define CLK_VC_SRC2			140
-+#define CLK_H264C			141
-+#define CLK_APB_H264C			142
-+#define CLK_H265C			143
-+#define CLK_APB_H265C			144
-+#define CLK_JPEG			145
-+#define CLK_APB_JPEG			146
-+#define CLK_CAM0			147
-+#define CLK_CAM1			148
-+#define CLK_WGN				149
-+#define CLK_WGN0			150
-+#define CLK_WGN1			151
-+#define CLK_WGN2			152
-+#define CLK_KEYSCAN			153
-+#define CLK_CFG_REG_VC			154
-+#define CLK_C906_0			155
-+#define CLK_C906_1			156
-+#define CLK_A53				157
-+#define CLK_CPU_AXI0			158
-+#define CLK_CPU_GIC			159
-+#define CLK_XTAL_AP			160
-+
-+// Only for CV181x
-+#define CLK_DISP_SRC_VIP		161
-+
-+#endif /* __DT_BINDINGS_SOPHGO_CV1800_CLK_H__ */
-diff --git a/include/dt-bindings/clock/sophgo,sg2042-clkgen.h b/include/dt-bindings/clock/sophgo,sg2042-clkgen.h
-new file mode 100644
-index 000000000000..84f7857317a2
--- /dev/null
-+++ b/include/dt-bindings/clock/sophgo,sg2042-clkgen.h
-@@ -0,0 +1,111 @@
-+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2023 Sophgo Technology Inc. All rights reserved.
-+ */
-+
-+#ifndef __DT_BINDINGS_SOPHGO_SG2042_CLKGEN_H__
-+#define __DT_BINDINGS_SOPHGO_SG2042_CLKGEN_H__
-+
-+#define DIV_CLK_MPLL_RP_CPU_NORMAL_0	0
-+#define DIV_CLK_MPLL_AXI_DDR_0		1
-+#define DIV_CLK_FPLL_DDR01_1		2
-+#define DIV_CLK_FPLL_DDR23_1		3
-+#define DIV_CLK_FPLL_RP_CPU_NORMAL_1	4
-+#define DIV_CLK_FPLL_50M_A53		5
-+#define DIV_CLK_FPLL_TOP_RP_CMN_DIV2	6
-+#define DIV_CLK_FPLL_UART_500M		7
-+#define DIV_CLK_FPLL_AHB_LPC		8
-+#define DIV_CLK_FPLL_EFUSE		9
-+#define DIV_CLK_FPLL_TX_ETH0		10
-+#define DIV_CLK_FPLL_PTP_REF_I_ETH0	11
-+#define DIV_CLK_FPLL_REF_ETH0		12
-+#define DIV_CLK_FPLL_EMMC		13
-+#define DIV_CLK_FPLL_SD			14
-+#define DIV_CLK_FPLL_TOP_AXI0		15
-+#define DIV_CLK_FPLL_TOP_AXI_HSPERI	16
-+#define DIV_CLK_FPLL_AXI_DDR_1		17
-+#define DIV_CLK_FPLL_DIV_TIMER1		18
-+#define DIV_CLK_FPLL_DIV_TIMER2		19
-+#define DIV_CLK_FPLL_DIV_TIMER3		20
-+#define DIV_CLK_FPLL_DIV_TIMER4		21
-+#define DIV_CLK_FPLL_DIV_TIMER5		22
-+#define DIV_CLK_FPLL_DIV_TIMER6		23
-+#define DIV_CLK_FPLL_DIV_TIMER7		24
-+#define DIV_CLK_FPLL_DIV_TIMER8		25
-+#define DIV_CLK_FPLL_100K_EMMC		26
-+#define DIV_CLK_FPLL_100K_SD		27
-+#define DIV_CLK_FPLL_GPIO_DB		28
-+#define DIV_CLK_DPLL0_DDR01_0		29
-+#define DIV_CLK_DPLL1_DDR23_0		30
-+
-+#define GATE_CLK_RP_CPU_NORMAL_DIV0	31
-+#define GATE_CLK_AXI_DDR_DIV0		32
-+
-+#define GATE_CLK_RP_CPU_NORMAL_DIV1	33
-+#define GATE_CLK_A53_50M		34
-+#define GATE_CLK_TOP_RP_CMN_DIV2	35
-+#define GATE_CLK_HSDMA			36
-+#define GATE_CLK_EMMC_100M		37
-+#define GATE_CLK_SD_100M		38
-+#define GATE_CLK_TX_ETH0		39
-+#define GATE_CLK_PTP_REF_I_ETH0		40
-+#define GATE_CLK_REF_ETH0		41
-+#define GATE_CLK_UART_500M		42
-+#define GATE_CLK_EFUSE			43
-+
-+#define GATE_CLK_AHB_LPC		44
-+#define GATE_CLK_AHB_ROM		45
-+#define GATE_CLK_AHB_SF			46
-+
-+#define GATE_CLK_APB_UART		47
-+#define GATE_CLK_APB_TIMER		48
-+#define GATE_CLK_APB_EFUSE		49
-+#define GATE_CLK_APB_GPIO		50
-+#define GATE_CLK_APB_GPIO_INTR		51
-+#define GATE_CLK_APB_SPI		52
-+#define GATE_CLK_APB_I2C		53
-+#define GATE_CLK_APB_WDT		54
-+#define GATE_CLK_APB_PWM		55
-+#define GATE_CLK_APB_RTC		56
-+
-+#define GATE_CLK_AXI_PCIE0		57
-+#define GATE_CLK_AXI_PCIE1		58
-+#define GATE_CLK_SYSDMA_AXI		59
-+#define GATE_CLK_AXI_DBG_I2C		60
-+#define GATE_CLK_AXI_SRAM		61
-+#define GATE_CLK_AXI_ETH0		62
-+#define GATE_CLK_AXI_EMMC		63
-+#define GATE_CLK_AXI_SD			64
-+#define GATE_CLK_TOP_AXI0		65
-+#define GATE_CLK_TOP_AXI_HSPERI		66
-+
-+#define GATE_CLK_TIMER1			67
-+#define GATE_CLK_TIMER2			68
-+#define GATE_CLK_TIMER3			69
-+#define GATE_CLK_TIMER4			70
-+#define GATE_CLK_TIMER5			71
-+#define GATE_CLK_TIMER6			72
-+#define GATE_CLK_TIMER7			73
-+#define GATE_CLK_TIMER8			74
-+#define GATE_CLK_100K_EMMC		75
-+#define GATE_CLK_100K_SD		76
-+#define GATE_CLK_GPIO_DB		77
-+
-+#define GATE_CLK_AXI_DDR_DIV1		78
-+#define GATE_CLK_DDR01_DIV1		79
-+#define GATE_CLK_DDR23_DIV1		80
-+
-+#define GATE_CLK_DDR01_DIV0		81
-+#define GATE_CLK_DDR23_DIV0		82
-+
-+#define GATE_CLK_DDR01			83
-+#define GATE_CLK_DDR23			84
-+#define GATE_CLK_RP_CPU_NORMAL		85
-+#define GATE_CLK_AXI_DDR		86
-+
-+#define MUX_CLK_DDR01			87
-+#define MUX_CLK_DDR23			88
-+#define MUX_CLK_RP_CPU_NORMAL		89
-+#define MUX_CLK_AXI_DDR			90
-+
-+#endif /* __DT_BINDINGS_SOPHGO_SG2042_CLKGEN_H__ */
-diff --git a/include/dt-bindings/clock/sophgo,sg2042-pll.h b/include/dt-bindings/clock/sophgo,sg2042-pll.h
-new file mode 100644
-index 000000000000..2d519b3bf51c
--- /dev/null
-+++ b/include/dt-bindings/clock/sophgo,sg2042-pll.h
-@@ -0,0 +1,14 @@
-+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2023 Sophgo Technology Inc. All rights reserved.
-+ */
-+
-+#ifndef __DT_BINDINGS_SOPHGO_SG2042_PLL_H__
-+#define __DT_BINDINGS_SOPHGO_SG2042_PLL_H__
-+
-+#define MPLL_CLK			0
-+#define FPLL_CLK			1
-+#define DPLL0_CLK			2
-+#define DPLL1_CLK			3
-+
-+#endif /* __DT_BINDINGS_SOPHGO_SG2042_PLL_H__ */
-diff --git a/include/dt-bindings/clock/sophgo,sg2042-rpgate.h b/include/dt-bindings/clock/sophgo,sg2042-rpgate.h
-new file mode 100644
-index 000000000000..8b4522d5f559
--- /dev/null
-+++ b/include/dt-bindings/clock/sophgo,sg2042-rpgate.h
-@@ -0,0 +1,58 @@
-+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2023 Sophgo Technology Inc. All rights reserved.
-+ */
-+
-+#ifndef __DT_BINDINGS_SOPHGO_SG2042_RPGATE_H__
-+#define __DT_BINDINGS_SOPHGO_SG2042_RPGATE_H__
-+
-+#define GATE_CLK_RXU0			0
-+#define GATE_CLK_RXU1			1
-+#define GATE_CLK_RXU2			2
-+#define GATE_CLK_RXU3			3
-+#define GATE_CLK_RXU4			4
-+#define GATE_CLK_RXU5			5
-+#define GATE_CLK_RXU6			6
-+#define GATE_CLK_RXU7			7
-+#define GATE_CLK_RXU8			8
-+#define GATE_CLK_RXU9			9
-+#define GATE_CLK_RXU10			10
-+#define GATE_CLK_RXU11			11
-+#define GATE_CLK_RXU12			12
-+#define GATE_CLK_RXU13			13
-+#define GATE_CLK_RXU14			14
-+#define GATE_CLK_RXU15			15
-+#define GATE_CLK_RXU16			16
-+#define GATE_CLK_RXU17			17
-+#define GATE_CLK_RXU18			18
-+#define GATE_CLK_RXU19			19
-+#define GATE_CLK_RXU20			20
-+#define GATE_CLK_RXU21			21
-+#define GATE_CLK_RXU22			22
-+#define GATE_CLK_RXU23			23
-+#define GATE_CLK_RXU24			24
-+#define GATE_CLK_RXU25			25
-+#define GATE_CLK_RXU26			26
-+#define GATE_CLK_RXU27			27
-+#define GATE_CLK_RXU28			28
-+#define GATE_CLK_RXU29			29
-+#define GATE_CLK_RXU30			30
-+#define GATE_CLK_RXU31			31
-+#define GATE_CLK_MP0			32
-+#define GATE_CLK_MP1			33
-+#define GATE_CLK_MP2			34
-+#define GATE_CLK_MP3			35
-+#define GATE_CLK_MP4			36
-+#define GATE_CLK_MP5			37
-+#define GATE_CLK_MP6			38
-+#define GATE_CLK_MP7			39
-+#define GATE_CLK_MP8			40
-+#define GATE_CLK_MP9			41
-+#define GATE_CLK_MP10			42
-+#define GATE_CLK_MP11			43
-+#define GATE_CLK_MP12			44
-+#define GATE_CLK_MP13			45
-+#define GATE_CLK_MP14			46
-+#define GATE_CLK_MP15			47
-+
-+#endif /* __DT_BINDINGS_SOPHGO_SG2042_RPGATE_H__ */
-diff --git a/include/dt-bindings/clock/sophgo,sg2044-clk.h b/include/dt-bindings/clock/sophgo,sg2044-clk.h
-new file mode 100644
-index 000000000000..d9adca42548e
--- /dev/null
-+++ b/include/dt-bindings/clock/sophgo,sg2044-clk.h
-@@ -0,0 +1,153 @@
-+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2024 Inochi Amaoto <inochiama@gmail.com>
-+ */
-+
-+#ifndef __DT_BINDINGS_SOPHGO_SG2044_CLK_H__
-+#define __DT_BINDINGS_SOPHGO_SG2044_CLK_H__
-+
-+#define CLK_DIV_AP_SYS_FIXED		0
-+#define CLK_DIV_AP_SYS_MAIN		1
-+#define CLK_DIV_RP_SYS_FIXED		2
-+#define CLK_DIV_RP_SYS_MAIN		3
-+#define CLK_DIV_TPU_SYS_FIXED		4
-+#define CLK_DIV_TPU_SYS_MAIN		5
-+#define CLK_DIV_NOC_SYS_FIXED		6
-+#define CLK_DIV_NOC_SYS_MAIN		7
-+#define CLK_DIV_VC_SRC0_FIXED		8
-+#define CLK_DIV_VC_SRC0_MAIN		9
-+#define CLK_DIV_VC_SRC1_FIXED		10
-+#define CLK_DIV_VC_SRC1_MAIN		11
-+#define CLK_DIV_CXP_MAC_FIXED		12
-+#define CLK_DIV_CXP_MAC_MAIN		13
-+#define CLK_DIV_DDR0_FIXED		14
-+#define CLK_DIV_DDR0_MAIN		15
-+#define CLK_DIV_DDR1_FIXED		16
-+#define CLK_DIV_DDR1_MAIN		17
-+#define CLK_DIV_DDR2_FIXED		18
-+#define CLK_DIV_DDR2_MAIN		19
-+#define CLK_DIV_DDR3_FIXED		20
-+#define CLK_DIV_DDR3_MAIN		21
-+#define CLK_DIV_DDR4_FIXED		22
-+#define CLK_DIV_DDR4_MAIN		23
-+#define CLK_DIV_DDR5_FIXED		24
-+#define CLK_DIV_DDR5_MAIN		25
-+#define CLK_DIV_DDR6_FIXED		26
-+#define CLK_DIV_DDR6_MAIN		27
-+#define CLK_DIV_DDR7_FIXED		28
-+#define CLK_DIV_DDR7_MAIN		29
-+#define CLK_DIV_TOP_50M			30
-+#define CLK_DIV_TOP_AXI0		31
-+#define CLK_DIV_TOP_AXI_HSPERI		32
-+#define CLK_DIV_TIMER0			33
-+#define CLK_DIV_TIMER1			34
-+#define CLK_DIV_TIMER2			35
-+#define CLK_DIV_TIMER3			36
-+#define CLK_DIV_TIMER4			37
-+#define CLK_DIV_TIMER5			38
-+#define CLK_DIV_TIMER6			39
-+#define CLK_DIV_TIMER7			40
-+#define CLK_DIV_CXP_TEST_PHY		41
-+#define CLK_DIV_CXP_TEST_ETH_PHY	42
-+#define CLK_DIV_C2C0_TEST_PHY		43
-+#define CLK_DIV_C2C1_TEST_PHY		44
-+#define CLK_DIV_PCIE_1G			45
-+#define CLK_DIV_UART_500M		46
-+#define CLK_DIV_GPIO_DB			47
-+#define CLK_DIV_SD			48
-+#define CLK_DIV_SD_100K			49
-+#define CLK_DIV_EMMC			50
-+#define CLK_DIV_EMMC_100K		51
-+#define CLK_DIV_EFUSE			52
-+#define CLK_DIV_TX_ETH0			53
-+#define CLK_DIV_PTP_REF_I_ETH0		54
-+#define CLK_DIV_REF_ETH0		55
-+#define CLK_DIV_PKA			56
-+#define CLK_MUX_DDR0			57
-+#define CLK_MUX_DDR1			58
-+#define CLK_MUX_DDR2			59
-+#define CLK_MUX_DDR3			60
-+#define CLK_MUX_DDR4			61
-+#define CLK_MUX_DDR5			62
-+#define CLK_MUX_DDR6			63
-+#define CLK_MUX_DDR7			64
-+#define CLK_MUX_NOC_SYS			65
-+#define CLK_MUX_TPU_SYS			66
-+#define CLK_MUX_RP_SYS			67
-+#define CLK_MUX_AP_SYS			68
-+#define CLK_MUX_VC_SRC0			69
-+#define CLK_MUX_VC_SRC1			70
-+#define CLK_MUX_CXP_MAC			71
-+#define CLK_GATE_AP_SYS			72
-+#define CLK_GATE_RP_SYS			73
-+#define CLK_GATE_TPU_SYS		74
-+#define CLK_GATE_NOC_SYS		75
-+#define CLK_GATE_VC_SRC0		76
-+#define CLK_GATE_VC_SRC1		77
-+#define CLK_GATE_DDR0			78
-+#define CLK_GATE_DDR1			79
-+#define CLK_GATE_DDR2			80
-+#define CLK_GATE_DDR3			81
-+#define CLK_GATE_DDR4			82
-+#define CLK_GATE_DDR5			83
-+#define CLK_GATE_DDR6			84
-+#define CLK_GATE_DDR7			85
-+#define CLK_GATE_TOP_50M		86
-+#define CLK_GATE_SC_RX			87
-+#define CLK_GATE_SC_RX_X0Y1		88
-+#define CLK_GATE_TOP_AXI0		89
-+#define CLK_GATE_INTC0			90
-+#define CLK_GATE_INTC1			91
-+#define CLK_GATE_INTC2			92
-+#define CLK_GATE_INTC3			93
-+#define CLK_GATE_MAILBOX0		94
-+#define CLK_GATE_MAILBOX1		95
-+#define CLK_GATE_MAILBOX2		96
-+#define CLK_GATE_MAILBOX3		97
-+#define CLK_GATE_TOP_AXI_HSPERI		98
-+#define CLK_GATE_APB_TIMER		99
-+#define CLK_GATE_TIMER0			100
-+#define CLK_GATE_TIMER1			101
-+#define CLK_GATE_TIMER2			102
-+#define CLK_GATE_TIMER3			103
-+#define CLK_GATE_TIMER4			104
-+#define CLK_GATE_TIMER5			105
-+#define CLK_GATE_TIMER6			106
-+#define CLK_GATE_TIMER7			107
-+#define CLK_GATE_CXP_CFG		108
-+#define CLK_GATE_CXP_MAC		109
-+#define CLK_GATE_CXP_TEST_PHY		110
-+#define CLK_GATE_CXP_TEST_ETH_PHY	111
-+#define CLK_GATE_PCIE_1G		112
-+#define CLK_GATE_C2C0_TEST_PHY		113
-+#define CLK_GATE_C2C1_TEST_PHY		114
-+#define CLK_GATE_UART_500M		115
-+#define CLK_GATE_APB_UART		116
-+#define CLK_GATE_APB_SPI		117
-+#define CLK_GATE_AHB_SPIFMC		118
-+#define CLK_GATE_APB_I2C		119
-+#define CLK_GATE_AXI_DBG_I2C		120
-+#define CLK_GATE_GPIO_DB		121
-+#define CLK_GATE_APB_GPIO_INTR		122
-+#define CLK_GATE_APB_GPIO		123
-+#define CLK_GATE_SD			124
-+#define CLK_GATE_AXI_SD			125
-+#define CLK_GATE_SD_100K		126
-+#define CLK_GATE_EMMC			127
-+#define CLK_GATE_AXI_EMMC		128
-+#define CLK_GATE_EMMC_100K		129
-+#define CLK_GATE_EFUSE			130
-+#define CLK_GATE_APB_EFUSE		131
-+#define CLK_GATE_SYSDMA_AXI		132
-+#define CLK_GATE_TX_ETH0		133
-+#define CLK_GATE_AXI_ETH0		134
-+#define CLK_GATE_PTP_REF_I_ETH0		135
-+#define CLK_GATE_REF_ETH0		136
-+#define CLK_GATE_APB_RTC		137
-+#define CLK_GATE_APB_PWM		138
-+#define CLK_GATE_APB_WDT		139
-+#define CLK_GATE_AXI_SRAM		140
-+#define CLK_GATE_AHB_ROM		141
-+#define CLK_GATE_PKA			142
-+
-+#endif /* __DT_BINDINGS_SOPHGO_SG2044_CLK_H__ */
-diff --git a/include/dt-bindings/clock/sophgo,sg2044-pll.h b/include/dt-bindings/clock/sophgo,sg2044-pll.h
-new file mode 100644
-index 000000000000..817d45e700cc
--- /dev/null
-+++ b/include/dt-bindings/clock/sophgo,sg2044-pll.h
-@@ -0,0 +1,27 @@
-+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2024 Inochi Amaoto <inochiama@gmail.com>
-+ */
-+
-+#ifndef __DT_BINDINGS_SOPHGO_SG2044_PLL_H__
-+#define __DT_BINDINGS_SOPHGO_SG2044_PLL_H__
-+
-+#define CLK_FPLL0			0
-+#define CLK_FPLL1			1
-+#define CLK_FPLL2			2
-+#define CLK_DPLL0			3
-+#define CLK_DPLL1			4
-+#define CLK_DPLL2			5
-+#define CLK_DPLL3			6
-+#define CLK_DPLL4			7
-+#define CLK_DPLL5			8
-+#define CLK_DPLL6			9
-+#define CLK_DPLL7			10
-+#define CLK_MPLL0			11
-+#define CLK_MPLL1			12
-+#define CLK_MPLL2			13
-+#define CLK_MPLL3			14
-+#define CLK_MPLL4			15
-+#define CLK_MPLL5			16
-+
-+#endif /* __DT_BINDINGS_SOPHGO_SG2044_PLL_H__ */
-diff --git a/include/dt-bindings/clock/spacemit,k1-syscon.h b/include/dt-bindings/clock/spacemit,k1-syscon.h
-new file mode 100644
-index 000000000000..0f8b59d6753c
--- /dev/null
-+++ b/include/dt-bindings/clock/spacemit,k1-syscon.h
-@@ -0,0 +1,394 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2024-2025 Haylen Chu <heylenay@outlook.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_SPACEMIT_CCU_H_
-+#define _DT_BINDINGS_SPACEMIT_CCU_H_
-+
-+/* APBS (PLL) clocks */
-+#define CLK_PLL1		0
-+#define CLK_PLL2		1
-+#define CLK_PLL3		2
-+#define CLK_PLL1_D2		3
-+#define CLK_PLL1_D3		4
-+#define CLK_PLL1_D4		5
-+#define CLK_PLL1_D5		6
-+#define CLK_PLL1_D6		7
-+#define CLK_PLL1_D7		8
-+#define CLK_PLL1_D8		9
-+#define CLK_PLL1_D11		10
-+#define CLK_PLL1_D13		11
-+#define CLK_PLL1_D23		12
-+#define CLK_PLL1_D64		13
-+#define CLK_PLL1_D10_AUD	14
-+#define CLK_PLL1_D100_AUD	15
-+#define CLK_PLL2_D1		16
-+#define CLK_PLL2_D2		17
-+#define CLK_PLL2_D3		18
-+#define CLK_PLL2_D4		19
-+#define CLK_PLL2_D5		20
-+#define CLK_PLL2_D6		21
-+#define CLK_PLL2_D7		22
-+#define CLK_PLL2_D8		23
-+#define CLK_PLL3_D1		24
-+#define CLK_PLL3_D2		25
-+#define CLK_PLL3_D3		26
-+#define CLK_PLL3_D4		27
-+#define CLK_PLL3_D5		28
-+#define CLK_PLL3_D6		29
-+#define CLK_PLL3_D7		30
-+#define CLK_PLL3_D8		31
-+#define CLK_PLL3_80		32
-+#define CLK_PLL3_40		33
-+#define CLK_PLL3_20		34
-+
-+/* MPMU clocks */
-+#define CLK_PLL1_307P2		0
-+#define CLK_PLL1_76P8		1
-+#define CLK_PLL1_61P44		2
-+#define CLK_PLL1_153P6		3
-+#define CLK_PLL1_102P4		4
-+#define CLK_PLL1_51P2		5
-+#define CLK_PLL1_51P2_AP	6
-+#define CLK_PLL1_57P6		7
-+#define CLK_PLL1_25P6		8
-+#define CLK_PLL1_12P8		9
-+#define CLK_PLL1_12P8_WDT	10
-+#define CLK_PLL1_6P4		11
-+#define CLK_PLL1_3P2		12
-+#define CLK_PLL1_1P6		13
-+#define CLK_PLL1_0P8		14
-+#define CLK_PLL1_409P6		15
-+#define CLK_PLL1_204P8		16
-+#define CLK_PLL1_491		17
-+#define CLK_PLL1_245P76		18
-+#define CLK_PLL1_614		19
-+#define CLK_PLL1_47P26		20
-+#define CLK_PLL1_31P5		21
-+#define CLK_PLL1_819		22
-+#define CLK_PLL1_1228		23
-+#define CLK_SLOW_UART		24
-+#define CLK_SLOW_UART1		25
-+#define CLK_SLOW_UART2		26
-+#define CLK_WDT			27
-+#define CLK_RIPC		28
-+#define CLK_I2S_SYSCLK		29
-+#define CLK_I2S_BCLK		30
-+#define CLK_APB			31
-+#define CLK_WDT_BUS		32
-+#define CLK_I2S_153P6		33
-+#define CLK_I2S_153P6_BASE	34
-+#define CLK_I2S_SYSCLK_SRC	35
-+#define CLK_I2S_BCLK_FACTOR	36
-+
-+/* MPMU resets */
-+#define RESET_WDT		0
-+
-+/* APBC clocks */
-+#define CLK_UART0		0
-+#define CLK_UART2		1
-+#define CLK_UART3		2
-+#define CLK_UART4		3
-+#define CLK_UART5		4
-+#define CLK_UART6		5
-+#define CLK_UART7		6
-+#define CLK_UART8		7
-+#define CLK_UART9		8
-+#define CLK_GPIO		9
-+#define CLK_PWM0		10
-+#define CLK_PWM1		11
-+#define CLK_PWM2		12
-+#define CLK_PWM3		13
-+#define CLK_PWM4		14
-+#define CLK_PWM5		15
-+#define CLK_PWM6		16
-+#define CLK_PWM7		17
-+#define CLK_PWM8		18
-+#define CLK_PWM9		19
-+#define CLK_PWM10		20
-+#define CLK_PWM11		21
-+#define CLK_PWM12		22
-+#define CLK_PWM13		23
-+#define CLK_PWM14		24
-+#define CLK_PWM15		25
-+#define CLK_PWM16		26
-+#define CLK_PWM17		27
-+#define CLK_PWM18		28
-+#define CLK_PWM19		29
-+#define CLK_SSP3		30
-+#define CLK_RTC			31
-+#define CLK_TWSI0		32
-+#define CLK_TWSI1		33
-+#define CLK_TWSI2		34
-+#define CLK_TWSI4		35
-+#define CLK_TWSI5		36
-+#define CLK_TWSI6		37
-+#define CLK_TWSI7		38
-+#define CLK_TWSI8		39
-+#define CLK_TIMERS1		40
-+#define CLK_TIMERS2		41
-+#define CLK_AIB			42
-+#define CLK_ONEWIRE		43
-+#define CLK_SSPA0		44
-+#define CLK_SSPA1		45
-+#define CLK_DRO			46
-+#define CLK_IR			47
-+#define CLK_TSEN		48
-+#define CLK_IPC_AP2AUD		49
-+#define CLK_CAN0		50
-+#define CLK_CAN0_BUS		51
-+#define CLK_UART0_BUS		52
-+#define CLK_UART2_BUS		53
-+#define CLK_UART3_BUS		54
-+#define CLK_UART4_BUS		55
-+#define CLK_UART5_BUS		56
-+#define CLK_UART6_BUS		57
-+#define CLK_UART7_BUS		58
-+#define CLK_UART8_BUS		59
-+#define CLK_UART9_BUS		60
-+#define CLK_GPIO_BUS		61
-+#define CLK_PWM0_BUS		62
-+#define CLK_PWM1_BUS		63
-+#define CLK_PWM2_BUS		64
-+#define CLK_PWM3_BUS		65
-+#define CLK_PWM4_BUS		66
-+#define CLK_PWM5_BUS		67
-+#define CLK_PWM6_BUS		68
-+#define CLK_PWM7_BUS		69
-+#define CLK_PWM8_BUS		70
-+#define CLK_PWM9_BUS		71
-+#define CLK_PWM10_BUS		72
-+#define CLK_PWM11_BUS		73
-+#define CLK_PWM12_BUS		74
-+#define CLK_PWM13_BUS		75
-+#define CLK_PWM14_BUS		76
-+#define CLK_PWM15_BUS		77
-+#define CLK_PWM16_BUS		78
-+#define CLK_PWM17_BUS		79
-+#define CLK_PWM18_BUS		80
-+#define CLK_PWM19_BUS		81
-+#define CLK_SSP3_BUS		82
-+#define CLK_RTC_BUS		83
-+#define CLK_TWSI0_BUS		84
-+#define CLK_TWSI1_BUS		85
-+#define CLK_TWSI2_BUS		86
-+#define CLK_TWSI4_BUS		87
-+#define CLK_TWSI5_BUS		88
-+#define CLK_TWSI6_BUS		89
-+#define CLK_TWSI7_BUS		90
-+#define CLK_TWSI8_BUS		91
-+#define CLK_TIMERS1_BUS		92
-+#define CLK_TIMERS2_BUS		93
-+#define CLK_AIB_BUS		94
-+#define CLK_ONEWIRE_BUS		95
-+#define CLK_SSPA0_BUS		96
-+#define CLK_SSPA1_BUS		97
-+#define CLK_TSEN_BUS		98
-+#define CLK_IPC_AP2AUD_BUS	99
-+#define CLK_SSPA0_I2S_BCLK	100
-+#define CLK_SSPA1_I2S_BCLK	101
-+
-+/* APBC resets */
-+#define RESET_UART0		0
-+#define RESET_UART2		1
-+#define RESET_UART3		2
-+#define RESET_UART4		3
-+#define RESET_UART5		4
-+#define RESET_UART6		5
-+#define RESET_UART7		6
-+#define RESET_UART8		7
-+#define RESET_UART9		8
-+#define RESET_GPIO		9
-+#define RESET_PWM0		10
-+#define RESET_PWM1		11
-+#define RESET_PWM2		12
-+#define RESET_PWM3		13
-+#define RESET_PWM4		14
-+#define RESET_PWM5		15
-+#define RESET_PWM6		16
-+#define RESET_PWM7		17
-+#define RESET_PWM8		18
-+#define RESET_PWM9		19
-+#define RESET_PWM10		20
-+#define RESET_PWM11		21
-+#define RESET_PWM12		22
-+#define RESET_PWM13		23
-+#define RESET_PWM14		24
-+#define RESET_PWM15		25
-+#define RESET_PWM16		26
-+#define RESET_PWM17		27
-+#define RESET_PWM18		28
-+#define RESET_PWM19		29
-+#define RESET_SSP3		30
-+#define RESET_RTC		31
-+#define RESET_TWSI0		32
-+#define RESET_TWSI1		33
-+#define RESET_TWSI2		34
-+#define RESET_TWSI4		35
-+#define RESET_TWSI5		36
-+#define RESET_TWSI6		37
-+#define RESET_TWSI7		38
-+#define RESET_TWSI8		39
-+#define RESET_TIMERS1		40
-+#define RESET_TIMERS2		41
-+#define RESET_AIB		42
-+#define RESET_ONEWIRE		43
-+#define RESET_SSPA0		44
-+#define RESET_SSPA1		45
-+#define RESET_DRO		46
-+#define RESET_IR		47
-+#define RESET_TSEN		48
-+#define RESET_IPC_AP2AUD	49
-+#define RESET_CAN0		50
-+
-+/* APMU clocks */
-+#define CLK_CCI550		0
-+#define CLK_CPU_C0_HI		1
-+#define CLK_CPU_C0_CORE		2
-+#define CLK_CPU_C0_ACE		3
-+#define CLK_CPU_C0_TCM		4
-+#define CLK_CPU_C1_HI		5
-+#define CLK_CPU_C1_CORE		6
-+#define CLK_CPU_C1_ACE		7
-+#define CLK_CCIC_4X		8
-+#define CLK_CCIC1PHY		9
-+#define CLK_SDH_AXI		10
-+#define CLK_SDH0		11
-+#define CLK_SDH1		12
-+#define CLK_SDH2		13
-+#define CLK_USB_P1		14
-+#define CLK_USB_AXI		15
-+#define CLK_USB30		16
-+#define CLK_QSPI		17
-+#define CLK_QSPI_BUS		18
-+#define CLK_DMA			19
-+#define CLK_AES			20
-+#define CLK_VPU			21
-+#define CLK_GPU			22
-+#define CLK_EMMC		23
-+#define CLK_EMMC_X		24
-+#define CLK_AUDIO		25
-+#define CLK_HDMI		26
-+#define CLK_PMUA_ACLK		27
-+#define CLK_PCIE0_MASTER	28
-+#define CLK_PCIE0_SLAVE		29
-+#define CLK_PCIE0_DBI		30
-+#define CLK_PCIE1_MASTER	31
-+#define CLK_PCIE1_SLAVE		32
-+#define CLK_PCIE1_DBI		33
-+#define CLK_PCIE2_MASTER	34
-+#define CLK_PCIE2_SLAVE		35
-+#define CLK_PCIE2_DBI		36
-+#define CLK_EMAC0_BUS		37
-+#define CLK_EMAC0_PTP		38
-+#define CLK_EMAC1_BUS		39
-+#define CLK_EMAC1_PTP		40
-+#define CLK_JPG			41
-+#define CLK_CCIC2PHY		42
-+#define CLK_CCIC3PHY		43
-+#define CLK_CSI			44
-+#define CLK_CAMM0		45
-+#define CLK_CAMM1		46
-+#define CLK_CAMM2		47
-+#define CLK_ISP_CPP		48
-+#define CLK_ISP_BUS		49
-+#define CLK_ISP			50
-+#define CLK_DPU_MCLK		51
-+#define CLK_DPU_ESC		52
-+#define CLK_DPU_BIT		53
-+#define CLK_DPU_PXCLK		54
-+#define CLK_DPU_HCLK		55
-+#define CLK_DPU_SPI		56
-+#define CLK_DPU_SPI_HBUS	57
-+#define CLK_DPU_SPIBUS		58
-+#define CLK_DPU_SPI_ACLK	59
-+#define CLK_V2D			60
-+#define CLK_EMMC_BUS		61
-+
-+/* APMU resets */
-+#define RESET_CCIC_4X		0
-+#define RESET_CCIC1_PHY		1
-+#define RESET_SDH_AXI		2
-+#define RESET_SDH0		3
-+#define RESET_SDH1		4
-+#define RESET_SDH2		5
-+#define RESET_USBP1_AXI		6
-+#define RESET_USB_AXI		7
-+#define RESET_USB30_AHB		8
-+#define RESET_USB30_VCC		9
-+#define RESET_USB30_PHY		10
-+#define RESET_QSPI		11
-+#define RESET_QSPI_BUS		12
-+#define RESET_DMA		13
-+#define RESET_AES		14
-+#define RESET_VPU		15
-+#define RESET_GPU		16
-+#define RESET_EMMC		17
-+#define RESET_EMMC_X		18
-+#define RESET_AUDIO_SYS		19
-+#define RESET_AUDIO_MCU		20
-+#define RESET_AUDIO_APMU	21
-+#define RESET_HDMI		22
-+#define RESET_PCIE0_MASTER	23
-+#define RESET_PCIE0_SLAVE	24
-+#define RESET_PCIE0_DBI		25
-+#define RESET_PCIE0_GLOBAL	26
-+#define RESET_PCIE1_MASTER	27
-+#define RESET_PCIE1_SLAVE	28
-+#define RESET_PCIE1_DBI		29
-+#define RESET_PCIE1_GLOBAL	30
-+#define RESET_PCIE2_MASTER	31
-+#define RESET_PCIE2_SLAVE	32
-+#define RESET_PCIE2_DBI		33
-+#define RESET_PCIE2_GLOBAL	34
-+#define RESET_EMAC0		35
-+#define RESET_EMAC1		36
-+#define RESET_JPG		37
-+#define RESET_CCIC2PHY		38
-+#define RESET_CCIC3PHY		39
-+#define RESET_CSI		40
-+#define RESET_ISP_CPP		41
-+#define RESET_ISP_BUS		42
-+#define RESET_ISP		43
-+#define RESET_ISP_CI		44
-+#define RESET_DPU_MCLK		45
-+#define RESET_DPU_ESC		46
-+#define RESET_DPU_HCLK		47
-+#define RESET_DPU_SPIBUS	48
-+#define RESET_DPU_SPI_HBUS	49
-+#define RESET_V2D		50
-+#define RESET_MIPI		51
-+#define RESET_MC		52
-+
-+/*	RCPU resets	*/
-+#define RESET_RCPU_SSP0		0
-+#define RESET_RCPU_I2C0		1
-+#define RESET_RCPU_UART1	2
-+#define RESET_RCPU_IR		3
-+#define RESET_RCPU_CAN		4
-+#define RESET_RCPU_UART0	5
-+#define RESET_RCPU_HDMI_AUDIO	6
-+
-+/*	RCPU2 resets	*/
-+#define RESET_RCPU2_PWM0	0
-+#define RESET_RCPU2_PWM1	1
-+#define RESET_RCPU2_PWM2	2
-+#define RESET_RCPU2_PWM3	3
-+#define RESET_RCPU2_PWM4	4
-+#define RESET_RCPU2_PWM5	5
-+#define RESET_RCPU2_PWM6	6
-+#define RESET_RCPU2_PWM7	7
-+#define RESET_RCPU2_PWM8	8
-+#define RESET_RCPU2_PWM9	9
-+
-+/*	APBC2 resets	*/
-+#define RESET_APBC2_UART1	0
-+#define RESET_APBC2_SSP2	1
-+#define RESET_APBC2_TWSI3	2
-+#define RESET_APBC2_RTC		3
-+#define RESET_APBC2_TIMERS0	4
-+#define RESET_APBC2_KPC		5
-+#define RESET_APBC2_GPIO	6
-+
-+#endif /* _DT_BINDINGS_SPACEMIT_CCU_H_ */
-diff --git a/include/dt-bindings/clock/st,stm32mp21-rcc.h b/include/dt-bindings/clock/st,stm32mp21-rcc.h
-new file mode 100644
-index 000000000000..054b785f2796
--- /dev/null
-+++ b/include/dt-bindings/clock/st,stm32mp21-rcc.h
-@@ -0,0 +1,426 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (C) STMicroelectronics 2025 - All Rights Reserved
-+ * Author: Gabriel Fernandez <gabriel.fernandez@foss.st.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_STM32MP21_CLKS_H_
-+#define _DT_BINDINGS_STM32MP21_CLKS_H_
-+
-+/* INTERNAL/EXTERNAL OSCILLATORS */
-+#define HSI_CK			0
-+#define HSE_CK			1
-+#define MSI_CK			2
-+#define LSI_CK			3
-+#define LSE_CK			4
-+#define I2S_CK			5
-+#define RTC_CK			6
-+#define SPDIF_CK_SYMB		7
-+
-+/* PLL CLOCKS */
-+#define PLL1_CK			8
-+#define PLL2_CK			9
-+#define PLL4_CK			10
-+#define PLL5_CK			11
-+#define PLL6_CK			12
-+#define PLL7_CK			13
-+#define PLL8_CK			14
-+
-+#define CK_CPU1			15
-+
-+/* APB DIV CLOCKS */
-+#define CK_ICN_APB1		16
-+#define CK_ICN_APB2		17
-+#define CK_ICN_APB3		18
-+#define CK_ICN_APB4		19
-+#define CK_ICN_APB5		20
-+#define CK_ICN_APBDBG		21
-+
-+/* GLOBAL TIMER */
-+#define TIMG1_CK		22
-+#define TIMG2_CK		23
-+
-+/* FLEXGEN CLOCKS */
-+#define CK_ICN_HS_MCU		24
-+#define CK_ICN_SDMMC		25
-+#define CK_ICN_DDR		26
-+#define CK_ICN_DISPLAY		27
-+#define CK_ICN_HSL		28
-+#define CK_ICN_NIC		29
-+#define CK_ICN_VID		30
-+#define CK_FLEXGEN_07		31
-+#define CK_FLEXGEN_08		32
-+#define CK_FLEXGEN_09		33
-+#define CK_FLEXGEN_10		34
-+#define CK_FLEXGEN_11		35
-+#define CK_FLEXGEN_12		36
-+#define CK_FLEXGEN_13		37
-+#define CK_FLEXGEN_14		38
-+#define CK_FLEXGEN_15		39
-+#define CK_FLEXGEN_16		40
-+#define CK_FLEXGEN_17		41
-+#define CK_FLEXGEN_18		42
-+#define CK_FLEXGEN_19		43
-+#define CK_FLEXGEN_20		44
-+#define CK_FLEXGEN_21		45
-+#define CK_FLEXGEN_22		46
-+#define CK_FLEXGEN_23		47
-+#define CK_FLEXGEN_24		48
-+#define CK_FLEXGEN_25		49
-+#define CK_FLEXGEN_26		50
-+#define CK_FLEXGEN_27		51
-+#define CK_FLEXGEN_28		52
-+#define CK_FLEXGEN_29		53
-+#define CK_FLEXGEN_30		54
-+#define CK_FLEXGEN_31		55
-+#define CK_FLEXGEN_32		56
-+#define CK_FLEXGEN_33		57
-+#define CK_FLEXGEN_34		58
-+#define CK_FLEXGEN_35		59
-+#define CK_FLEXGEN_36		60
-+#define CK_FLEXGEN_37		61
-+#define CK_FLEXGEN_38		62
-+#define CK_FLEXGEN_39		63
-+#define CK_FLEXGEN_40		64
-+#define CK_FLEXGEN_41		65
-+#define CK_FLEXGEN_42		66
-+#define CK_FLEXGEN_43		67
-+#define CK_FLEXGEN_44		68
-+#define CK_FLEXGEN_45		69
-+#define CK_FLEXGEN_46		70
-+#define CK_FLEXGEN_47		71
-+#define CK_FLEXGEN_48		72
-+#define CK_FLEXGEN_49		73
-+#define CK_FLEXGEN_50		74
-+#define CK_FLEXGEN_51		75
-+#define CK_FLEXGEN_52		76
-+#define CK_FLEXGEN_53		77
-+#define CK_FLEXGEN_54		78
-+#define CK_FLEXGEN_55		79
-+#define CK_FLEXGEN_56		80
-+#define CK_FLEXGEN_57		81
-+#define CK_FLEXGEN_58		82
-+#define CK_FLEXGEN_59		83
-+#define CK_FLEXGEN_60		84
-+#define CK_FLEXGEN_61		85
-+#define CK_FLEXGEN_62		86
-+#define CK_FLEXGEN_63		87
-+
-+/* LOW SPEED MCU CLOCK */
-+#define CK_ICN_LS_MCU		88
-+
-+#define CK_BUS_STM		89
-+#define CK_BUS_FMC		90
-+#define CK_BUS_ETH1		91
-+#define CK_BUS_ETH2		92
-+#define CK_BUS_DDRPHYC		93
-+#define CK_BUS_SYSCPU1		94
-+#define CK_BUS_HPDMA1		95
-+#define CK_BUS_HPDMA2		96
-+#define CK_BUS_HPDMA3		97
-+#define CK_BUS_ADC1		98
-+#define CK_BUS_ADC2		99
-+#define CK_BUS_IPCC1		100
-+#define CK_BUS_DCMIPSSI		101
-+#define CK_BUS_CRC		102
-+#define CK_BUS_MDF1		103
-+#define CK_BUS_BKPSRAM		104
-+#define CK_BUS_HASH1		105
-+#define CK_BUS_HASH2		106
-+#define CK_BUS_RNG1		107
-+#define CK_BUS_RNG2		108
-+#define CK_BUS_CRYP1		109
-+#define CK_BUS_CRYP2		110
-+#define CK_BUS_SAES		111
-+#define CK_BUS_PKA		112
-+#define CK_BUS_GPIOA		113
-+#define CK_BUS_GPIOB		114
-+#define CK_BUS_GPIOC		115
-+#define CK_BUS_GPIOD		116
-+#define CK_BUS_GPIOE		117
-+#define CK_BUS_GPIOF		118
-+#define CK_BUS_GPIOG		119
-+#define CK_BUS_GPIOH		120
-+#define CK_BUS_GPIOI		121
-+#define CK_BUS_GPIOZ		122
-+#define CK_BUS_RTC		124
-+#define CK_BUS_LPUART1		125
-+#define CK_BUS_LPTIM3		126
-+#define CK_BUS_LPTIM4		127
-+#define CK_BUS_LPTIM5		128
-+#define CK_BUS_TIM2		129
-+#define CK_BUS_TIM3		130
-+#define CK_BUS_TIM4		131
-+#define CK_BUS_TIM5		132
-+#define CK_BUS_TIM6		133
-+#define CK_BUS_TIM7		134
-+#define CK_BUS_TIM10		135
-+#define CK_BUS_TIM11		136
-+#define CK_BUS_TIM12		137
-+#define CK_BUS_TIM13		138
-+#define CK_BUS_TIM14		139
-+#define CK_BUS_LPTIM1		140
-+#define CK_BUS_LPTIM2		141
-+#define CK_BUS_SPI2		142
-+#define CK_BUS_SPI3		143
-+#define CK_BUS_SPDIFRX		144
-+#define CK_BUS_USART2		145
-+#define CK_BUS_USART3		146
-+#define CK_BUS_UART4		147
-+#define CK_BUS_UART5		148
-+#define CK_BUS_I2C1		149
-+#define CK_BUS_I2C2		150
-+#define CK_BUS_I2C3		151
-+#define CK_BUS_I3C1		152
-+#define CK_BUS_I3C2		153
-+#define CK_BUS_I3C3		154
-+#define CK_BUS_TIM1		155
-+#define CK_BUS_TIM8		156
-+#define CK_BUS_TIM15		157
-+#define CK_BUS_TIM16		158
-+#define CK_BUS_TIM17		159
-+#define CK_BUS_SAI1		160
-+#define CK_BUS_SAI2		161
-+#define CK_BUS_SAI3		162
-+#define CK_BUS_SAI4		163
-+#define CK_BUS_USART1		164
-+#define CK_BUS_USART6		165
-+#define CK_BUS_UART7		166
-+#define CK_BUS_FDCAN		167
-+#define CK_BUS_SPI1		168
-+#define CK_BUS_SPI4		169
-+#define CK_BUS_SPI5		170
-+#define CK_BUS_SPI6		171
-+#define CK_BUS_BSEC		172
-+#define CK_BUS_IWDG1		173
-+#define CK_BUS_IWDG2		174
-+#define CK_BUS_IWDG3		175
-+#define CK_BUS_IWDG4		176
-+#define CK_BUS_WWDG1		177
-+#define CK_BUS_VREF		178
-+#define CK_BUS_DTS		179
-+#define CK_BUS_SERC		180
-+#define CK_BUS_HDP		181
-+#define CK_BUS_DDRPERFM		182
-+#define CK_BUS_OTG		183
-+#define CK_BUS_LTDC		184
-+#define CK_BUS_CSI		185
-+#define CK_BUS_DCMIPP		186
-+#define CK_BUS_DDRC		187
-+#define CK_BUS_DDRCFG		188
-+#define CK_BUS_STGEN		189
-+#define CK_SYSDBG		190
-+#define CK_KER_TIM2		191
-+#define CK_KER_TIM3		192
-+#define CK_KER_TIM4		193
-+#define CK_KER_TIM5		194
-+#define CK_KER_TIM6		195
-+#define CK_KER_TIM7		196
-+#define CK_KER_TIM10		197
-+#define CK_KER_TIM11		198
-+#define CK_KER_TIM12		199
-+#define CK_KER_TIM13		200
-+#define CK_KER_TIM14		201
-+#define CK_KER_TIM1		202
-+#define CK_KER_TIM8		203
-+#define CK_KER_TIM15		204
-+#define CK_KER_TIM16		205
-+#define CK_KER_TIM17		206
-+#define CK_BUS_SYSRAM		207
-+#define CK_BUS_RETRAM		208
-+#define CK_BUS_OSPI1		209
-+#define CK_BUS_OTFD1		210
-+#define CK_BUS_SRAM1		211
-+#define CK_BUS_SDMMC1		212
-+#define CK_BUS_SDMMC2		213
-+#define CK_BUS_SDMMC3		214
-+#define CK_BUS_DDR		215
-+#define CK_BUS_RISAF4		216
-+#define CK_BUS_USBHOHCI		217
-+#define CK_BUS_USBHEHCI		218
-+#define CK_KER_LPTIM1		219
-+#define CK_KER_LPTIM2		220
-+#define CK_KER_USART2		221
-+#define CK_KER_UART4		222
-+#define CK_KER_USART3		223
-+#define CK_KER_UART5		224
-+#define CK_KER_SPI2		225
-+#define CK_KER_SPI3		226
-+#define CK_KER_SPDIFRX		227
-+#define CK_KER_I2C1		228
-+#define CK_KER_I2C2		229
-+#define CK_KER_I3C1		230
-+#define CK_KER_I3C2		231
-+#define CK_KER_I2C3		232
-+#define CK_KER_I3C3		233
-+#define CK_KER_SPI1		234
-+#define CK_KER_SPI4		235
-+#define CK_KER_SPI5		236
-+#define CK_KER_SPI6		237
-+#define CK_KER_USART1		238
-+#define CK_KER_USART6		239
-+#define CK_KER_UART7		240
-+#define CK_KER_MDF1		241
-+#define CK_KER_SAI1		242
-+#define CK_KER_SAI2		243
-+#define CK_KER_SAI3		244
-+#define CK_KER_SAI4		245
-+#define CK_KER_FDCAN		246
-+#define CK_KER_CSI		247
-+#define CK_KER_CSITXESC		248
-+#define CK_KER_CSIPHY		249
-+#define CK_KER_STGEN		250
-+#define CK_KER_USB2PHY2EN	251
-+#define CK_KER_LPUART1		252
-+#define CK_KER_LPTIM3		253
-+#define CK_KER_LPTIM4		254
-+#define CK_KER_LPTIM5		255
-+#define CK_KER_TSDBG		256
-+#define CK_KER_TPIU		257
-+#define CK_BUS_ETR		258
-+#define CK_BUS_SYSATB		259
-+#define CK_KER_ADC1		260
-+#define CK_KER_ADC2		261
-+#define CK_KER_OSPI1		262
-+#define CK_KER_FMC		263
-+#define CK_KER_SDMMC1		264
-+#define CK_KER_SDMMC2		265
-+#define CK_KER_SDMMC3		266
-+#define CK_KER_ETH1		267
-+#define CK_KER_ETH2		268
-+#define CK_KER_ETH1PTP		269
-+#define CK_KER_ETH2PTP		270
-+#define CK_KER_USB2PHY1		271
-+#define CK_KER_USB2PHY2		272
-+#define CK_MCO1			273
-+#define CK_MCO2			274
-+#define CK_KER_DTS		275
-+#define CK_ETH1_RX		276
-+#define CK_ETH1_TX		277
-+#define CK_ETH1_MAC		278
-+#define CK_ETH2_RX		279
-+#define CK_ETH2_TX		280
-+#define CK_ETH2_MAC		281
-+#define CK_ETH1_STP		282
-+#define CK_ETH2_STP		283
-+#define CK_KER_LTDC		284
-+#define HSE_DIV2_CK		285
-+#define CK_DBGMCU		286
-+#define CK_DAP			287
-+#define CK_KER_ETR		288
-+#define CK_KER_STM		289
-+
-+#define CK_SCMI_ICN_HS_MCU	0
-+#define CK_SCMI_ICN_SDMMC	1
-+#define CK_SCMI_ICN_DDR		2
-+#define CK_SCMI_ICN_DISPLAY	3
-+#define CK_SCMI_ICN_HSL		4
-+#define CK_SCMI_ICN_NIC		5
-+#define CK_SCMI_FLEXGEN_07	7
-+#define CK_SCMI_FLEXGEN_08	8
-+#define CK_SCMI_FLEXGEN_09	9
-+#define CK_SCMI_FLEXGEN_10	10
-+#define CK_SCMI_FLEXGEN_11	11
-+#define CK_SCMI_FLEXGEN_12	12
-+#define CK_SCMI_FLEXGEN_13	13
-+#define CK_SCMI_FLEXGEN_14	14
-+#define CK_SCMI_FLEXGEN_15	15
-+#define CK_SCMI_FLEXGEN_16	16
-+#define CK_SCMI_FLEXGEN_17	17
-+#define CK_SCMI_FLEXGEN_18	18
-+#define CK_SCMI_FLEXGEN_19	19
-+#define CK_SCMI_FLEXGEN_20	20
-+#define CK_SCMI_FLEXGEN_21	21
-+#define CK_SCMI_FLEXGEN_22	22
-+#define CK_SCMI_FLEXGEN_23	23
-+#define CK_SCMI_FLEXGEN_24	24
-+#define CK_SCMI_FLEXGEN_25	25
-+#define CK_SCMI_FLEXGEN_26	26
-+#define CK_SCMI_FLEXGEN_27	27
-+#define CK_SCMI_FLEXGEN_28	28
-+#define CK_SCMI_FLEXGEN_29	29
-+#define CK_SCMI_FLEXGEN_30	30
-+#define CK_SCMI_FLEXGEN_31	31
-+#define CK_SCMI_FLEXGEN_32	32
-+#define CK_SCMI_FLEXGEN_33	33
-+#define CK_SCMI_FLEXGEN_34	34
-+#define CK_SCMI_FLEXGEN_35	35
-+#define CK_SCMI_FLEXGEN_36	36
-+#define CK_SCMI_FLEXGEN_37	37
-+#define CK_SCMI_FLEXGEN_38	38
-+#define CK_SCMI_FLEXGEN_39	39
-+#define CK_SCMI_FLEXGEN_40	40
-+#define CK_SCMI_FLEXGEN_41	41
-+#define CK_SCMI_FLEXGEN_42	42
-+#define CK_SCMI_FLEXGEN_43	43
-+#define CK_SCMI_FLEXGEN_44	44
-+#define CK_SCMI_FLEXGEN_45	45
-+#define CK_SCMI_FLEXGEN_46	46
-+#define CK_SCMI_FLEXGEN_47	47
-+#define CK_SCMI_FLEXGEN_48	48
-+#define CK_SCMI_FLEXGEN_49	49
-+#define CK_SCMI_FLEXGEN_50	50
-+#define CK_SCMI_FLEXGEN_51	51
-+#define CK_SCMI_FLEXGEN_52	52
-+#define CK_SCMI_FLEXGEN_53	53
-+#define CK_SCMI_FLEXGEN_54	54
-+#define CK_SCMI_FLEXGEN_55	55
-+#define CK_SCMI_FLEXGEN_56	56
-+#define CK_SCMI_FLEXGEN_57	57
-+#define CK_SCMI_FLEXGEN_58	58
-+#define CK_SCMI_FLEXGEN_59	59
-+#define CK_SCMI_FLEXGEN_60	60
-+#define CK_SCMI_FLEXGEN_61	61
-+#define CK_SCMI_FLEXGEN_62	62
-+#define CK_SCMI_FLEXGEN_63	63
-+#define CK_SCMI_ICN_LS_MCU	64
-+#define CK_SCMI_HSE		65
-+#define CK_SCMI_LSE		66
-+#define CK_SCMI_HSI		67
-+#define CK_SCMI_LSI		68
-+#define CK_SCMI_MSI		69
-+#define CK_SCMI_HSE_DIV2	70
-+#define CK_SCMI_CPU1		71
-+#define CK_SCMI_SYSCPU1		72
-+#define CK_SCMI_PLL2		73
-+#define CK_SCMI_RTC		74
-+#define CK_SCMI_RTCCK		75
-+#define CK_SCMI_ICN_APB1	76
-+#define CK_SCMI_ICN_APB2	77
-+#define CK_SCMI_ICN_APB3	78
-+#define CK_SCMI_ICN_APB4	79
-+#define CK_SCMI_ICN_APB5	80
-+#define CK_SCMI_ICN_APBDBG	81
-+#define CK_SCMI_TIMG1		82
-+#define CK_SCMI_TIMG2		83
-+#define CK_SCMI_BKPSRAM		84
-+#define CK_SCMI_BSEC		85
-+#define CK_SCMI_BUS_ETR		86
-+#define CK_SCMI_FMC		87
-+#define CK_SCMI_GPIOA		88
-+#define CK_SCMI_GPIOB		89
-+#define CK_SCMI_GPIOC		90
-+#define CK_SCMI_GPIOD		91
-+#define CK_SCMI_GPIOE		92
-+#define CK_SCMI_GPIOF		93
-+#define CK_SCMI_GPIOG		94
-+#define CK_SCMI_GPIOH		95
-+#define CK_SCMI_GPIOI		96
-+#define CK_SCMI_GPIOZ		97
-+#define CK_SCMI_HPDMA1		98
-+#define CK_SCMI_HPDMA2		99
-+#define CK_SCMI_HPDMA3		100
-+#define CK_SCMI_IPCC1		101
-+#define CK_SCMI_RETRAM		102
-+#define CK_SCMI_SRAM1		103
-+#define CK_SCMI_SYSRAM		104
-+#define CK_SCMI_OSPI1		105
-+#define CK_SCMI_TPIU		106
-+#define CK_SCMI_SYSDBG		107
-+#define CK_SCMI_SYSATB		108
-+#define CK_SCMI_TSDBG		109
-+#define CK_SCMI_BUS_STM		110
-+#define CK_SCMI_KER_STM		111
-+#define CK_SCMI_KER_ETR		112
-+
-+#endif /* _DT_BINDINGS_STM32MP21_CLKS_H_ */
-diff --git a/include/dt-bindings/clock/st,stm32mp25-rcc.h b/include/dt-bindings/clock/st,stm32mp25-rcc.h
-new file mode 100644
-index 000000000000..b6cf05ad4be6
--- /dev/null
-+++ b/include/dt-bindings/clock/st,stm32mp25-rcc.h
-@@ -0,0 +1,492 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (C) STMicroelectronics 2023 - All Rights Reserved
-+ * Author: Gabriel Fernandez <gabriel.fernandez@foss.st.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_STM32MP25_CLKS_H_
-+#define _DT_BINDINGS_STM32MP25_CLKS_H_
-+
-+/* INTERNAL/EXTERNAL OSCILLATORS */
-+#define HSI_CK			0
-+#define HSE_CK			1
-+#define MSI_CK			2
-+#define LSI_CK			3
-+#define LSE_CK			4
-+#define I2S_CK			5
-+#define RTC_CK			6
-+#define SPDIF_CK_SYMB		7
-+
-+/* PLL CLOCKS */
-+#define PLL1_CK			8
-+#define PLL2_CK			9
-+#define PLL3_CK			10
-+#define PLL4_CK			11
-+#define PLL5_CK			12
-+#define PLL6_CK			13
-+#define PLL7_CK			14
-+#define PLL8_CK			15
-+
-+#define CK_CPU1			16
-+
-+/* APB DIV CLOCKS */
-+#define CK_ICN_APB1		17
-+#define CK_ICN_APB2		18
-+#define CK_ICN_APB3		19
-+#define CK_ICN_APB4		20
-+#define CK_ICN_APBDBG		21
-+
-+/* GLOBAL TIMER */
-+#define TIMG1_CK		22
-+#define TIMG2_CK		23
-+
-+/* FLEXGEN CLOCKS */
-+#define CK_ICN_HS_MCU		24
-+#define CK_ICN_SDMMC		25
-+#define CK_ICN_DDR		26
-+#define CK_ICN_DISPLAY		27
-+#define CK_ICN_HSL		28
-+#define CK_ICN_NIC		29
-+#define CK_ICN_VID		30
-+#define CK_FLEXGEN_07		31
-+#define CK_FLEXGEN_08		32
-+#define CK_FLEXGEN_09		33
-+#define CK_FLEXGEN_10		34
-+#define CK_FLEXGEN_11		35
-+#define CK_FLEXGEN_12		36
-+#define CK_FLEXGEN_13		37
-+#define CK_FLEXGEN_14		38
-+#define CK_FLEXGEN_15		39
-+#define CK_FLEXGEN_16		40
-+#define CK_FLEXGEN_17		41
-+#define CK_FLEXGEN_18		42
-+#define CK_FLEXGEN_19		43
-+#define CK_FLEXGEN_20		44
-+#define CK_FLEXGEN_21		45
-+#define CK_FLEXGEN_22		46
-+#define CK_FLEXGEN_23		47
-+#define CK_FLEXGEN_24		48
-+#define CK_FLEXGEN_25		49
-+#define CK_FLEXGEN_26		50
-+#define CK_FLEXGEN_27		51
-+#define CK_FLEXGEN_28		52
-+#define CK_FLEXGEN_29		53
-+#define CK_FLEXGEN_30		54
-+#define CK_FLEXGEN_31		55
-+#define CK_FLEXGEN_32		56
-+#define CK_FLEXGEN_33		57
-+#define CK_FLEXGEN_34		58
-+#define CK_FLEXGEN_35		59
-+#define CK_FLEXGEN_36		60
-+#define CK_FLEXGEN_37		61
-+#define CK_FLEXGEN_38		62
-+#define CK_FLEXGEN_39		63
-+#define CK_FLEXGEN_40		64
-+#define CK_FLEXGEN_41		65
-+#define CK_FLEXGEN_42		66
-+#define CK_FLEXGEN_43		67
-+#define CK_FLEXGEN_44		68
-+#define CK_FLEXGEN_45		69
-+#define CK_FLEXGEN_46		70
-+#define CK_FLEXGEN_47		71
-+#define CK_FLEXGEN_48		72
-+#define CK_FLEXGEN_49		73
-+#define CK_FLEXGEN_50		74
-+#define CK_FLEXGEN_51		75
-+#define CK_FLEXGEN_52		76
-+#define CK_FLEXGEN_53		77
-+#define CK_FLEXGEN_54		78
-+#define CK_FLEXGEN_55		79
-+#define CK_FLEXGEN_56		80
-+#define CK_FLEXGEN_57		81
-+#define CK_FLEXGEN_58		82
-+#define CK_FLEXGEN_59		83
-+#define CK_FLEXGEN_60		84
-+#define CK_FLEXGEN_61		85
-+#define CK_FLEXGEN_62		86
-+#define CK_FLEXGEN_63		87
-+
-+/* LOW SPEED MCU CLOCK */
-+#define CK_ICN_LS_MCU		88
-+
-+#define CK_BUS_STM500		89
-+#define CK_BUS_FMC		90
-+#define CK_BUS_GPU		91
-+#define CK_BUS_ETH1		92
-+#define CK_BUS_ETH2		93
-+#define CK_BUS_PCIE		94
-+#define CK_BUS_DDRPHYC		95
-+#define CK_BUS_SYSCPU1		96
-+#define CK_BUS_ETHSW		97
-+#define CK_BUS_HPDMA1		98
-+#define CK_BUS_HPDMA2		99
-+#define CK_BUS_HPDMA3		100
-+#define CK_BUS_ADC12		101
-+#define CK_BUS_ADC3		102
-+#define CK_BUS_IPCC1		103
-+#define CK_BUS_CCI		104
-+#define CK_BUS_CRC		105
-+#define CK_BUS_MDF1		106
-+#define CK_BUS_OSPIIOM		107
-+#define CK_BUS_BKPSRAM		108
-+#define CK_BUS_HASH		109
-+#define CK_BUS_RNG		110
-+#define CK_BUS_CRYP1		111
-+#define CK_BUS_CRYP2		112
-+#define CK_BUS_SAES		113
-+#define CK_BUS_PKA		114
-+#define CK_BUS_GPIOA		115
-+#define CK_BUS_GPIOB		116
-+#define CK_BUS_GPIOC		117
-+#define CK_BUS_GPIOD		118
-+#define CK_BUS_GPIOE		119
-+#define CK_BUS_GPIOF		120
-+#define CK_BUS_GPIOG		121
-+#define CK_BUS_GPIOH		122
-+#define CK_BUS_GPIOI		123
-+#define CK_BUS_GPIOJ		124
-+#define CK_BUS_GPIOK		125
-+#define CK_BUS_LPSRAM1		126
-+#define CK_BUS_LPSRAM2		127
-+#define CK_BUS_LPSRAM3		128
-+#define CK_BUS_GPIOZ		129
-+#define CK_BUS_LPDMA		130
-+#define CK_BUS_HSEM		131
-+#define CK_BUS_IPCC2		132
-+#define CK_BUS_RTC		133
-+#define CK_BUS_SPI8		134
-+#define CK_BUS_LPUART1		135
-+#define CK_BUS_I2C8		136
-+#define CK_BUS_LPTIM3		137
-+#define CK_BUS_LPTIM4		138
-+#define CK_BUS_LPTIM5		139
-+#define CK_BUS_IWDG5		140
-+#define CK_BUS_WWDG2		141
-+#define CK_BUS_I3C4		142
-+#define CK_BUS_TIM2		143
-+#define CK_BUS_TIM3		144
-+#define CK_BUS_TIM4		145
-+#define CK_BUS_TIM5		146
-+#define CK_BUS_TIM6		147
-+#define CK_BUS_TIM7		148
-+#define CK_BUS_TIM10		149
-+#define CK_BUS_TIM11		150
-+#define CK_BUS_TIM12		151
-+#define CK_BUS_TIM13		152
-+#define CK_BUS_TIM14		153
-+#define CK_BUS_LPTIM1		154
-+#define CK_BUS_LPTIM2		155
-+#define CK_BUS_SPI2		156
-+#define CK_BUS_SPI3		157
-+#define CK_BUS_SPDIFRX		158
-+#define CK_BUS_USART2		159
-+#define CK_BUS_USART3		160
-+#define CK_BUS_UART4		161
-+#define CK_BUS_UART5		162
-+#define CK_BUS_I2C1		163
-+#define CK_BUS_I2C2		164
-+#define CK_BUS_I2C3		165
-+#define CK_BUS_I2C4		166
-+#define CK_BUS_I2C5		167
-+#define CK_BUS_I2C6		168
-+#define CK_BUS_I2C7		169
-+#define CK_BUS_I3C1		170
-+#define CK_BUS_I3C2		171
-+#define CK_BUS_I3C3		172
-+#define CK_BUS_TIM1		173
-+#define CK_BUS_TIM8		174
-+#define CK_BUS_TIM15		175
-+#define CK_BUS_TIM16		176
-+#define CK_BUS_TIM17		177
-+#define CK_BUS_TIM20		178
-+#define CK_BUS_SAI1		179
-+#define CK_BUS_SAI2		180
-+#define CK_BUS_SAI3		181
-+#define CK_BUS_SAI4		182
-+#define CK_BUS_USART1		183
-+#define CK_BUS_USART6		184
-+#define CK_BUS_UART7		185
-+#define CK_BUS_UART8		186
-+#define CK_BUS_UART9		187
-+#define CK_BUS_FDCAN		188
-+#define CK_BUS_SPI1		189
-+#define CK_BUS_SPI4		190
-+#define CK_BUS_SPI5		191
-+#define CK_BUS_SPI6		192
-+#define CK_BUS_SPI7		193
-+#define CK_BUS_BSEC		194
-+#define CK_BUS_IWDG1		195
-+#define CK_BUS_IWDG2		196
-+#define CK_BUS_IWDG3		197
-+#define CK_BUS_IWDG4		198
-+#define CK_BUS_WWDG1		199
-+#define CK_BUS_VREF		200
-+#define CK_BUS_DTS		201
-+#define CK_BUS_SERC		202
-+#define CK_BUS_HDP		203
-+#define CK_BUS_IS2M		204
-+#define CK_BUS_DSI		205
-+#define CK_BUS_LTDC		206
-+#define CK_BUS_CSI		207
-+#define CK_BUS_DCMIPP		208
-+#define CK_BUS_DDRC		209
-+#define CK_BUS_DDRCFG		210
-+#define CK_BUS_GICV2M		211
-+#define CK_BUS_USBTC		212
-+#define CK_BUS_USB3PCIEPHY	214
-+#define CK_BUS_STGEN		215
-+#define CK_BUS_VDEC		216
-+#define CK_BUS_VENC		217
-+#define CK_SYSDBG		218
-+#define CK_KER_TIM2		219
-+#define CK_KER_TIM3		220
-+#define CK_KER_TIM4		221
-+#define CK_KER_TIM5		222
-+#define CK_KER_TIM6		223
-+#define CK_KER_TIM7		224
-+#define CK_KER_TIM10		225
-+#define CK_KER_TIM11		226
-+#define CK_KER_TIM12		227
-+#define CK_KER_TIM13		228
-+#define CK_KER_TIM14		229
-+#define CK_KER_TIM1		230
-+#define CK_KER_TIM8		231
-+#define CK_KER_TIM15		232
-+#define CK_KER_TIM16		233
-+#define CK_KER_TIM17		234
-+#define CK_KER_TIM20		235
-+#define CK_BUS_SYSRAM		236
-+#define CK_BUS_VDERAM		237
-+#define CK_BUS_RETRAM		238
-+#define CK_BUS_OSPI1		239
-+#define CK_BUS_OSPI2		240
-+#define CK_BUS_OTFD1		241
-+#define CK_BUS_OTFD2		242
-+#define CK_BUS_SRAM1		243
-+#define CK_BUS_SRAM2		244
-+#define CK_BUS_SDMMC1		245
-+#define CK_BUS_SDMMC2		246
-+#define CK_BUS_SDMMC3		247
-+#define CK_BUS_DDR		248
-+#define CK_BUS_RISAF4		249
-+#define CK_BUS_USB2OHCI		250
-+#define CK_BUS_USB2EHCI		251
-+#define CK_BUS_USB3DR		252
-+#define CK_KER_LPTIM1		253
-+#define CK_KER_LPTIM2		254
-+#define CK_KER_USART2		255
-+#define CK_KER_UART4		256
-+#define CK_KER_USART3		257
-+#define CK_KER_UART5		258
-+#define CK_KER_SPI2		259
-+#define CK_KER_SPI3		260
-+#define CK_KER_SPDIFRX		261
-+#define CK_KER_I2C1		262
-+#define CK_KER_I2C2		263
-+#define CK_KER_I3C1		264
-+#define CK_KER_I3C2		265
-+#define CK_KER_I2C3		266
-+#define CK_KER_I2C5		267
-+#define CK_KER_I3C3		268
-+#define CK_KER_I2C4		269
-+#define CK_KER_I2C6		270
-+#define CK_KER_I2C7		271
-+#define CK_KER_SPI1		272
-+#define CK_KER_SPI4		273
-+#define CK_KER_SPI5		274
-+#define CK_KER_SPI6		275
-+#define CK_KER_SPI7		276
-+#define CK_KER_USART1		277
-+#define CK_KER_USART6		278
-+#define CK_KER_UART7		279
-+#define CK_KER_UART8		280
-+#define CK_KER_UART9		281
-+#define CK_KER_MDF1		282
-+#define CK_KER_SAI1		283
-+#define CK_KER_SAI2		284
-+#define CK_KER_SAI3		285
-+#define CK_KER_SAI4		286
-+#define CK_KER_FDCAN		287
-+#define CK_KER_DSIBLANE		288
-+#define CK_KER_DSIPHY		289
-+#define CK_KER_CSI		290
-+#define CK_KER_CSITXESC		291
-+#define CK_KER_CSIPHY		292
-+#define CK_KER_LVDSPHY		293
-+#define CK_KER_STGEN		294
-+#define CK_KER_USB3PCIEPHY	295
-+#define CK_KER_USB2PHY2EN	296
-+#define CK_KER_I3C4		297
-+#define CK_KER_SPI8		298
-+#define CK_KER_I2C8		299
-+#define CK_KER_LPUART1		300
-+#define CK_KER_LPTIM3		301
-+#define CK_KER_LPTIM4		302
-+#define CK_KER_LPTIM5		303
-+#define CK_KER_TSDBG		304
-+#define CK_KER_TPIU		305
-+#define CK_BUS_ETR		306
-+#define CK_BUS_SYSATB		307
-+#define CK_KER_ADC12		308
-+#define CK_KER_ADC3		309
-+#define CK_KER_OSPI1		310
-+#define CK_KER_OSPI2		311
-+#define CK_KER_FMC		312
-+#define CK_KER_SDMMC1		313
-+#define CK_KER_SDMMC2		314
-+#define CK_KER_SDMMC3		315
-+#define CK_KER_ETH1		316
-+#define CK_KER_ETH2		317
-+#define CK_KER_ETH1PTP		318
-+#define CK_KER_ETH2PTP		319
-+#define CK_KER_USB2PHY1		320
-+#define CK_KER_USB2PHY2		321
-+#define CK_KER_ETHSW		322
-+#define CK_KER_ETHSWREF		323
-+#define CK_MCO1			324
-+#define CK_MCO2			325
-+#define CK_KER_DTS		326
-+#define CK_ETH1_RX		327
-+#define CK_ETH1_TX		328
-+#define CK_ETH1_MAC		329
-+#define CK_ETH2_RX		330
-+#define CK_ETH2_TX		331
-+#define CK_ETH2_MAC		332
-+#define CK_ETH1_STP		333
-+#define CK_ETH2_STP		334
-+#define CK_KER_USBTC		335
-+#define CK_BUS_ADF1		336
-+#define CK_KER_ADF1		337
-+#define CK_BUS_LVDS		338
-+#define CK_KER_LTDC		339
-+#define CK_KER_GPU		340
-+#define CK_BUS_ETHSWACMCFG	341
-+#define CK_BUS_ETHSWACMMSG	342
-+#define HSE_DIV2_CK		343
-+
-+#define STM32MP25_LAST_CLK	344
-+
-+#define CK_SCMI_ICN_HS_MCU	0
-+#define CK_SCMI_ICN_SDMMC	1
-+#define CK_SCMI_ICN_DDR		2
-+#define CK_SCMI_ICN_DISPLAY	3
-+#define CK_SCMI_ICN_HSL		4
-+#define CK_SCMI_ICN_NIC		5
-+#define CK_SCMI_ICN_VID		6
-+#define CK_SCMI_FLEXGEN_07	7
-+#define CK_SCMI_FLEXGEN_08	8
-+#define CK_SCMI_FLEXGEN_09	9
-+#define CK_SCMI_FLEXGEN_10	10
-+#define CK_SCMI_FLEXGEN_11	11
-+#define CK_SCMI_FLEXGEN_12	12
-+#define CK_SCMI_FLEXGEN_13	13
-+#define CK_SCMI_FLEXGEN_14	14
-+#define CK_SCMI_FLEXGEN_15	15
-+#define CK_SCMI_FLEXGEN_16	16
-+#define CK_SCMI_FLEXGEN_17	17
-+#define CK_SCMI_FLEXGEN_18	18
-+#define CK_SCMI_FLEXGEN_19	19
-+#define CK_SCMI_FLEXGEN_20	20
-+#define CK_SCMI_FLEXGEN_21	21
-+#define CK_SCMI_FLEXGEN_22	22
-+#define CK_SCMI_FLEXGEN_23	23
-+#define CK_SCMI_FLEXGEN_24	24
-+#define CK_SCMI_FLEXGEN_25	25
-+#define CK_SCMI_FLEXGEN_26	26
-+#define CK_SCMI_FLEXGEN_27	27
-+#define CK_SCMI_FLEXGEN_28	28
-+#define CK_SCMI_FLEXGEN_29	29
-+#define CK_SCMI_FLEXGEN_30	30
-+#define CK_SCMI_FLEXGEN_31	31
-+#define CK_SCMI_FLEXGEN_32	32
-+#define CK_SCMI_FLEXGEN_33	33
-+#define CK_SCMI_FLEXGEN_34	34
-+#define CK_SCMI_FLEXGEN_35	35
-+#define CK_SCMI_FLEXGEN_36	36
-+#define CK_SCMI_FLEXGEN_37	37
-+#define CK_SCMI_FLEXGEN_38	38
-+#define CK_SCMI_FLEXGEN_39	39
-+#define CK_SCMI_FLEXGEN_40	40
-+#define CK_SCMI_FLEXGEN_41	41
-+#define CK_SCMI_FLEXGEN_42	42
-+#define CK_SCMI_FLEXGEN_43	43
-+#define CK_SCMI_FLEXGEN_44	44
-+#define CK_SCMI_FLEXGEN_45	45
-+#define CK_SCMI_FLEXGEN_46	46
-+#define CK_SCMI_FLEXGEN_47	47
-+#define CK_SCMI_FLEXGEN_48	48
-+#define CK_SCMI_FLEXGEN_49	49
-+#define CK_SCMI_FLEXGEN_50	50
-+#define CK_SCMI_FLEXGEN_51	51
-+#define CK_SCMI_FLEXGEN_52	52
-+#define CK_SCMI_FLEXGEN_53	53
-+#define CK_SCMI_FLEXGEN_54	54
-+#define CK_SCMI_FLEXGEN_55	55
-+#define CK_SCMI_FLEXGEN_56	56
-+#define CK_SCMI_FLEXGEN_57	57
-+#define CK_SCMI_FLEXGEN_58	58
-+#define CK_SCMI_FLEXGEN_59	59
-+#define CK_SCMI_FLEXGEN_60	60
-+#define CK_SCMI_FLEXGEN_61	61
-+#define CK_SCMI_FLEXGEN_62	62
-+#define CK_SCMI_FLEXGEN_63	63
-+#define CK_SCMI_ICN_LS_MCU	64
-+#define CK_SCMI_HSE		65
-+#define CK_SCMI_LSE		66
-+#define CK_SCMI_HSI		67
-+#define CK_SCMI_LSI		68
-+#define CK_SCMI_MSI		69
-+#define CK_SCMI_HSE_DIV2	70
-+#define CK_SCMI_CPU1		71
-+#define CK_SCMI_SYSCPU1		72
-+#define CK_SCMI_PLL2		73
-+#define CK_SCMI_PLL3		74
-+#define CK_SCMI_RTC		75
-+#define CK_SCMI_RTCCK		76
-+#define CK_SCMI_ICN_APB1	77
-+#define CK_SCMI_ICN_APB2	78
-+#define CK_SCMI_ICN_APB3	79
-+#define CK_SCMI_ICN_APB4	80
-+#define CK_SCMI_ICN_APBDBG	81
-+#define CK_SCMI_TIMG1		82
-+#define CK_SCMI_TIMG2		83
-+#define CK_SCMI_BKPSRAM		84
-+#define CK_SCMI_BSEC		85
-+#define CK_SCMI_ETR		87
-+#define CK_SCMI_FMC		88
-+#define CK_SCMI_GPIOA		89
-+#define CK_SCMI_GPIOB		90
-+#define CK_SCMI_GPIOC		91
-+#define CK_SCMI_GPIOD		92
-+#define CK_SCMI_GPIOE		93
-+#define CK_SCMI_GPIOF		94
-+#define CK_SCMI_GPIOG		95
-+#define CK_SCMI_GPIOH		96
-+#define CK_SCMI_GPIOI		97
-+#define CK_SCMI_GPIOJ		98
-+#define CK_SCMI_GPIOK		99
-+#define CK_SCMI_GPIOZ		100
-+#define CK_SCMI_HPDMA1		101
-+#define CK_SCMI_HPDMA2		102
-+#define CK_SCMI_HPDMA3		103
-+#define CK_SCMI_HSEM		104
-+#define CK_SCMI_IPCC1		105
-+#define CK_SCMI_IPCC2		106
-+#define CK_SCMI_LPDMA		107
-+#define CK_SCMI_RETRAM		108
-+#define CK_SCMI_SRAM1		109
-+#define CK_SCMI_SRAM2		110
-+#define CK_SCMI_LPSRAM1		111
-+#define CK_SCMI_LPSRAM2		112
-+#define CK_SCMI_LPSRAM3		113
-+#define CK_SCMI_VDERAM		114
-+#define CK_SCMI_SYSRAM		115
-+#define CK_SCMI_OSPI1		116
-+#define CK_SCMI_OSPI2		117
-+#define CK_SCMI_TPIU		118
-+#define CK_SCMI_SYSDBG		119
-+#define CK_SCMI_SYSATB		120
-+#define CK_SCMI_TSDBG		121
-+#define CK_SCMI_STM500		122
-+
-+#endif /* _DT_BINDINGS_STM32MP25_CLKS_H_ */
-diff --git a/include/dt-bindings/clock/starfive,jh7110-crg.h b/include/dt-bindings/clock/starfive,jh7110-crg.h
-new file mode 100644
-index 000000000000..467ccab3bfaa
--- /dev/null
-+++ b/include/dt-bindings/clock/starfive,jh7110-crg.h
-@@ -0,0 +1,301 @@
-+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
-+/*
-+ * Copyright 2022 Emil Renner Berthing <kernel@esmil.dk>
-+ * Copyright 2022 StarFive Technology Co., Ltd.
-+ */
-+
-+#ifndef __DT_BINDINGS_CLOCK_STARFIVE_JH7110_CRG_H__
-+#define __DT_BINDINGS_CLOCK_STARFIVE_JH7110_CRG_H__
-+
-+/* PLL clocks */
-+#define JH7110_PLLCLK_PLL0_OUT			0
-+#define JH7110_PLLCLK_PLL1_OUT			1
-+#define JH7110_PLLCLK_PLL2_OUT			2
-+#define JH7110_PLLCLK_END			3
-+
-+/* SYSCRG clocks */
-+#define JH7110_SYSCLK_CPU_ROOT			0
-+#define JH7110_SYSCLK_CPU_CORE			1
-+#define JH7110_SYSCLK_CPU_BUS			2
-+#define JH7110_SYSCLK_GPU_ROOT			3
-+#define JH7110_SYSCLK_PERH_ROOT			4
-+#define JH7110_SYSCLK_BUS_ROOT			5
-+#define JH7110_SYSCLK_NOCSTG_BUS		6
-+#define JH7110_SYSCLK_AXI_CFG0			7
-+#define JH7110_SYSCLK_STG_AXIAHB		8
-+#define JH7110_SYSCLK_AHB0			9
-+#define JH7110_SYSCLK_AHB1			10
-+#define JH7110_SYSCLK_APB_BUS			11
-+#define JH7110_SYSCLK_APB0			12
-+#define JH7110_SYSCLK_PLL0_DIV2			13
-+#define JH7110_SYSCLK_PLL1_DIV2			14
-+#define JH7110_SYSCLK_PLL2_DIV2			15
-+#define JH7110_SYSCLK_AUDIO_ROOT		16
-+#define JH7110_SYSCLK_MCLK_INNER		17
-+#define JH7110_SYSCLK_MCLK			18
-+#define JH7110_SYSCLK_MCLK_OUT			19
-+#define JH7110_SYSCLK_ISP_2X			20
-+#define JH7110_SYSCLK_ISP_AXI			21
-+#define JH7110_SYSCLK_GCLK0			22
-+#define JH7110_SYSCLK_GCLK1			23
-+#define JH7110_SYSCLK_GCLK2			24
-+#define JH7110_SYSCLK_CORE			25
-+#define JH7110_SYSCLK_CORE1			26
-+#define JH7110_SYSCLK_CORE2			27
-+#define JH7110_SYSCLK_CORE3			28
-+#define JH7110_SYSCLK_CORE4			29
-+#define JH7110_SYSCLK_DEBUG			30
-+#define JH7110_SYSCLK_RTC_TOGGLE		31
-+#define JH7110_SYSCLK_TRACE0			32
-+#define JH7110_SYSCLK_TRACE1			33
-+#define JH7110_SYSCLK_TRACE2			34
-+#define JH7110_SYSCLK_TRACE3			35
-+#define JH7110_SYSCLK_TRACE4			36
-+#define JH7110_SYSCLK_TRACE_COM			37
-+#define JH7110_SYSCLK_NOC_BUS_CPU_AXI		38
-+#define JH7110_SYSCLK_NOC_BUS_AXICFG0_AXI	39
-+#define JH7110_SYSCLK_OSC_DIV2			40
-+#define JH7110_SYSCLK_PLL1_DIV4			41
-+#define JH7110_SYSCLK_PLL1_DIV8			42
-+#define JH7110_SYSCLK_DDR_BUS			43
-+#define JH7110_SYSCLK_DDR_AXI			44
-+#define JH7110_SYSCLK_GPU_CORE			45
-+#define JH7110_SYSCLK_GPU_CORE_CLK		46
-+#define JH7110_SYSCLK_GPU_SYS_CLK		47
-+#define JH7110_SYSCLK_GPU_APB			48
-+#define JH7110_SYSCLK_GPU_RTC_TOGGLE		49
-+#define JH7110_SYSCLK_NOC_BUS_GPU_AXI		50
-+#define JH7110_SYSCLK_ISP_TOP_CORE		51
-+#define JH7110_SYSCLK_ISP_TOP_AXI		52
-+#define JH7110_SYSCLK_NOC_BUS_ISP_AXI		53
-+#define JH7110_SYSCLK_HIFI4_CORE		54
-+#define JH7110_SYSCLK_HIFI4_AXI			55
-+#define JH7110_SYSCLK_AXI_CFG1_MAIN		56
-+#define JH7110_SYSCLK_AXI_CFG1_AHB		57
-+#define JH7110_SYSCLK_VOUT_SRC			58
-+#define JH7110_SYSCLK_VOUT_AXI			59
-+#define JH7110_SYSCLK_NOC_BUS_DISP_AXI		60
-+#define JH7110_SYSCLK_VOUT_TOP_AHB		61
-+#define JH7110_SYSCLK_VOUT_TOP_AXI		62
-+#define JH7110_SYSCLK_VOUT_TOP_HDMITX0_MCLK	63
-+#define JH7110_SYSCLK_VOUT_TOP_MIPIPHY_REF	64
-+#define JH7110_SYSCLK_JPEGC_AXI			65
-+#define JH7110_SYSCLK_CODAJ12_AXI		66
-+#define JH7110_SYSCLK_CODAJ12_CORE		67
-+#define JH7110_SYSCLK_CODAJ12_APB		68
-+#define JH7110_SYSCLK_VDEC_AXI			69
-+#define JH7110_SYSCLK_WAVE511_AXI		70
-+#define JH7110_SYSCLK_WAVE511_BPU		71
-+#define JH7110_SYSCLK_WAVE511_VCE		72
-+#define JH7110_SYSCLK_WAVE511_APB		73
-+#define JH7110_SYSCLK_VDEC_JPG			74
-+#define JH7110_SYSCLK_VDEC_MAIN			75
-+#define JH7110_SYSCLK_NOC_BUS_VDEC_AXI		76
-+#define JH7110_SYSCLK_VENC_AXI			77
-+#define JH7110_SYSCLK_WAVE420L_AXI		78
-+#define JH7110_SYSCLK_WAVE420L_BPU		79
-+#define JH7110_SYSCLK_WAVE420L_VCE		80
-+#define JH7110_SYSCLK_WAVE420L_APB		81
-+#define JH7110_SYSCLK_NOC_BUS_VENC_AXI		82
-+#define JH7110_SYSCLK_AXI_CFG0_MAIN_DIV		83
-+#define JH7110_SYSCLK_AXI_CFG0_MAIN		84
-+#define JH7110_SYSCLK_AXI_CFG0_HIFI4		85
-+#define JH7110_SYSCLK_AXIMEM2_AXI		86
-+#define JH7110_SYSCLK_QSPI_AHB			87
-+#define JH7110_SYSCLK_QSPI_APB			88
-+#define JH7110_SYSCLK_QSPI_REF_SRC		89
-+#define JH7110_SYSCLK_QSPI_REF			90
-+#define JH7110_SYSCLK_SDIO0_AHB			91
-+#define JH7110_SYSCLK_SDIO1_AHB			92
-+#define JH7110_SYSCLK_SDIO0_SDCARD		93
-+#define JH7110_SYSCLK_SDIO1_SDCARD		94
-+#define JH7110_SYSCLK_USB_125M			95
-+#define JH7110_SYSCLK_NOC_BUS_STG_AXI		96
-+#define JH7110_SYSCLK_GMAC1_AHB			97
-+#define JH7110_SYSCLK_GMAC1_AXI			98
-+#define JH7110_SYSCLK_GMAC_SRC			99
-+#define JH7110_SYSCLK_GMAC1_GTXCLK		100
-+#define JH7110_SYSCLK_GMAC1_RMII_RTX		101
-+#define JH7110_SYSCLK_GMAC1_PTP			102
-+#define JH7110_SYSCLK_GMAC1_RX			103
-+#define JH7110_SYSCLK_GMAC1_RX_INV		104
-+#define JH7110_SYSCLK_GMAC1_TX			105
-+#define JH7110_SYSCLK_GMAC1_TX_INV		106
-+#define JH7110_SYSCLK_GMAC1_GTXC		107
-+#define JH7110_SYSCLK_GMAC0_GTXCLK		108
-+#define JH7110_SYSCLK_GMAC0_PTP			109
-+#define JH7110_SYSCLK_GMAC_PHY			110
-+#define JH7110_SYSCLK_GMAC0_GTXC		111
-+#define JH7110_SYSCLK_IOMUX_APB			112
-+#define JH7110_SYSCLK_MAILBOX_APB		113
-+#define JH7110_SYSCLK_INT_CTRL_APB		114
-+#define JH7110_SYSCLK_CAN0_APB			115
-+#define JH7110_SYSCLK_CAN0_TIMER		116
-+#define JH7110_SYSCLK_CAN0_CAN			117
-+#define JH7110_SYSCLK_CAN1_APB			118
-+#define JH7110_SYSCLK_CAN1_TIMER		119
-+#define JH7110_SYSCLK_CAN1_CAN			120
-+#define JH7110_SYSCLK_PWM_APB			121
-+#define JH7110_SYSCLK_WDT_APB			122
-+#define JH7110_SYSCLK_WDT_CORE			123
-+#define JH7110_SYSCLK_TIMER_APB			124
-+#define JH7110_SYSCLK_TIMER0			125
-+#define JH7110_SYSCLK_TIMER1			126
-+#define JH7110_SYSCLK_TIMER2			127
-+#define JH7110_SYSCLK_TIMER3			128
-+#define JH7110_SYSCLK_TEMP_APB			129
-+#define JH7110_SYSCLK_TEMP_CORE			130
-+#define JH7110_SYSCLK_SPI0_APB			131
-+#define JH7110_SYSCLK_SPI1_APB			132
-+#define JH7110_SYSCLK_SPI2_APB			133
-+#define JH7110_SYSCLK_SPI3_APB			134
-+#define JH7110_SYSCLK_SPI4_APB			135
-+#define JH7110_SYSCLK_SPI5_APB			136
-+#define JH7110_SYSCLK_SPI6_APB			137
-+#define JH7110_SYSCLK_I2C0_APB			138
-+#define JH7110_SYSCLK_I2C1_APB			139
-+#define JH7110_SYSCLK_I2C2_APB			140
-+#define JH7110_SYSCLK_I2C3_APB			141
-+#define JH7110_SYSCLK_I2C4_APB			142
-+#define JH7110_SYSCLK_I2C5_APB			143
-+#define JH7110_SYSCLK_I2C6_APB			144
-+#define JH7110_SYSCLK_UART0_APB			145
-+#define JH7110_SYSCLK_UART0_CORE		146
-+#define JH7110_SYSCLK_UART1_APB			147
-+#define JH7110_SYSCLK_UART1_CORE		148
-+#define JH7110_SYSCLK_UART2_APB			149
-+#define JH7110_SYSCLK_UART2_CORE		150
-+#define JH7110_SYSCLK_UART3_APB			151
-+#define JH7110_SYSCLK_UART3_CORE		152
-+#define JH7110_SYSCLK_UART4_APB			153
-+#define JH7110_SYSCLK_UART4_CORE		154
-+#define JH7110_SYSCLK_UART5_APB			155
-+#define JH7110_SYSCLK_UART5_CORE		156
-+#define JH7110_SYSCLK_PWMDAC_APB		157
-+#define JH7110_SYSCLK_PWMDAC_CORE		158
-+#define JH7110_SYSCLK_SPDIF_APB			159
-+#define JH7110_SYSCLK_SPDIF_CORE		160
-+#define JH7110_SYSCLK_I2STX0_APB		161
-+#define JH7110_SYSCLK_I2STX0_BCLK_MST		162
-+#define JH7110_SYSCLK_I2STX0_BCLK_MST_INV	163
-+#define JH7110_SYSCLK_I2STX0_LRCK_MST		164
-+#define JH7110_SYSCLK_I2STX0_BCLK		165
-+#define JH7110_SYSCLK_I2STX0_BCLK_INV		166
-+#define JH7110_SYSCLK_I2STX0_LRCK		167
-+#define JH7110_SYSCLK_I2STX1_APB		168
-+#define JH7110_SYSCLK_I2STX1_BCLK_MST		169
-+#define JH7110_SYSCLK_I2STX1_BCLK_MST_INV	170
-+#define JH7110_SYSCLK_I2STX1_LRCK_MST		171
-+#define JH7110_SYSCLK_I2STX1_BCLK		172
-+#define JH7110_SYSCLK_I2STX1_BCLK_INV		173
-+#define JH7110_SYSCLK_I2STX1_LRCK		174
-+#define JH7110_SYSCLK_I2SRX_APB			175
-+#define JH7110_SYSCLK_I2SRX_BCLK_MST		176
-+#define JH7110_SYSCLK_I2SRX_BCLK_MST_INV	177
-+#define JH7110_SYSCLK_I2SRX_LRCK_MST		178
-+#define JH7110_SYSCLK_I2SRX_BCLK		179
-+#define JH7110_SYSCLK_I2SRX_BCLK_INV		180
-+#define JH7110_SYSCLK_I2SRX_LRCK		181
-+#define JH7110_SYSCLK_PDM_DMIC			182
-+#define JH7110_SYSCLK_PDM_APB			183
-+#define JH7110_SYSCLK_TDM_AHB			184
-+#define JH7110_SYSCLK_TDM_APB			185
-+#define JH7110_SYSCLK_TDM_INTERNAL		186
-+#define JH7110_SYSCLK_TDM_TDM			187
-+#define JH7110_SYSCLK_TDM_TDM_INV		188
-+#define JH7110_SYSCLK_JTAG_CERTIFICATION_TRNG	189
-+
-+#define JH7110_SYSCLK_END			190
-+
-+/* AONCRG clocks */
-+#define JH7110_AONCLK_OSC_DIV4			0
-+#define JH7110_AONCLK_APB_FUNC			1
-+#define JH7110_AONCLK_GMAC0_AHB			2
-+#define JH7110_AONCLK_GMAC0_AXI			3
-+#define JH7110_AONCLK_GMAC0_RMII_RTX		4
-+#define JH7110_AONCLK_GMAC0_TX			5
-+#define JH7110_AONCLK_GMAC0_TX_INV		6
-+#define JH7110_AONCLK_GMAC0_RX			7
-+#define JH7110_AONCLK_GMAC0_RX_INV		8
-+#define JH7110_AONCLK_OTPC_APB			9
-+#define JH7110_AONCLK_RTC_APB			10
-+#define JH7110_AONCLK_RTC_INTERNAL		11
-+#define JH7110_AONCLK_RTC_32K			12
-+#define JH7110_AONCLK_RTC_CAL			13
-+
-+#define JH7110_AONCLK_END			14
-+
-+/* STGCRG clocks */
-+#define JH7110_STGCLK_HIFI4_CLK_CORE		0
-+#define JH7110_STGCLK_USB0_APB			1
-+#define JH7110_STGCLK_USB0_UTMI_APB		2
-+#define JH7110_STGCLK_USB0_AXI			3
-+#define JH7110_STGCLK_USB0_LPM			4
-+#define JH7110_STGCLK_USB0_STB			5
-+#define JH7110_STGCLK_USB0_APP_125		6
-+#define JH7110_STGCLK_USB0_REFCLK		7
-+#define JH7110_STGCLK_PCIE0_AXI_MST0		8
-+#define JH7110_STGCLK_PCIE0_APB			9
-+#define JH7110_STGCLK_PCIE0_TL			10
-+#define JH7110_STGCLK_PCIE1_AXI_MST0		11
-+#define JH7110_STGCLK_PCIE1_APB			12
-+#define JH7110_STGCLK_PCIE1_TL			13
-+#define JH7110_STGCLK_PCIE_SLV_MAIN		14
-+#define JH7110_STGCLK_SEC_AHB			15
-+#define JH7110_STGCLK_SEC_MISC_AHB		16
-+#define JH7110_STGCLK_GRP0_MAIN			17
-+#define JH7110_STGCLK_GRP0_BUS			18
-+#define JH7110_STGCLK_GRP0_STG			19
-+#define JH7110_STGCLK_GRP1_MAIN			20
-+#define JH7110_STGCLK_GRP1_BUS			21
-+#define JH7110_STGCLK_GRP1_STG			22
-+#define JH7110_STGCLK_GRP1_HIFI			23
-+#define JH7110_STGCLK_E2_RTC			24
-+#define JH7110_STGCLK_E2_CORE			25
-+#define JH7110_STGCLK_E2_DBG			26
-+#define JH7110_STGCLK_DMA1P_AXI			27
-+#define JH7110_STGCLK_DMA1P_AHB			28
-+
-+#define JH7110_STGCLK_END			29
-+
-+/* ISPCRG clocks */
-+#define JH7110_ISPCLK_DOM4_APB_FUNC		0
-+#define JH7110_ISPCLK_MIPI_RX0_PXL		1
-+#define JH7110_ISPCLK_DVP_INV			2
-+#define JH7110_ISPCLK_M31DPHY_CFG_IN		3
-+#define JH7110_ISPCLK_M31DPHY_REF_IN		4
-+#define JH7110_ISPCLK_M31DPHY_TX_ESC_LAN0	5
-+#define JH7110_ISPCLK_VIN_APB			6
-+#define JH7110_ISPCLK_VIN_SYS			7
-+#define JH7110_ISPCLK_VIN_PIXEL_IF0		8
-+#define JH7110_ISPCLK_VIN_PIXEL_IF1		9
-+#define JH7110_ISPCLK_VIN_PIXEL_IF2		10
-+#define JH7110_ISPCLK_VIN_PIXEL_IF3		11
-+#define JH7110_ISPCLK_VIN_P_AXI_WR		12
-+#define JH7110_ISPCLK_ISPV2_TOP_WRAPPER_C	13
-+
-+#define JH7110_ISPCLK_END			14
-+
-+/* VOUTCRG clocks */
-+#define JH7110_VOUTCLK_APB			0
-+#define JH7110_VOUTCLK_DC8200_PIX		1
-+#define JH7110_VOUTCLK_DSI_SYS			2
-+#define JH7110_VOUTCLK_TX_ESC			3
-+#define JH7110_VOUTCLK_DC8200_AXI		4
-+#define JH7110_VOUTCLK_DC8200_CORE		5
-+#define JH7110_VOUTCLK_DC8200_AHB		6
-+#define JH7110_VOUTCLK_DC8200_PIX0		7
-+#define JH7110_VOUTCLK_DC8200_PIX1		8
-+#define JH7110_VOUTCLK_DOM_VOUT_TOP_LCD		9
-+#define JH7110_VOUTCLK_DSITX_APB		10
-+#define JH7110_VOUTCLK_DSITX_SYS		11
-+#define JH7110_VOUTCLK_DSITX_DPI		12
-+#define JH7110_VOUTCLK_DSITX_TXESC		13
-+#define JH7110_VOUTCLK_MIPITX_DPHY_TXESC	14
-+#define JH7110_VOUTCLK_HDMI_TX_MCLK		15
-+#define JH7110_VOUTCLK_HDMI_TX_BCLK		16
-+#define JH7110_VOUTCLK_HDMI_TX_SYS		17
-+
-+#define JH7110_VOUTCLK_END			18
-+
-+#endif /* __DT_BINDINGS_CLOCK_STARFIVE_JH7110_CRG_H__ */
-diff --git a/include/dt-bindings/clock/stih416-clks.h b/include/dt-bindings/clock/stih416-clks.h
-deleted file mode 100644
-index 74302278024e..000000000000
--- a/include/dt-bindings/clock/stih416-clks.h
-+++ /dev/null
-@@ -1,17 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * This header provides constants clk index STMicroelectronics
- * STiH416 SoC.
- */
-#ifndef _CLK_STIH416
-#define _CLK_STIH416
-
-/* CLOCKGEN A0 */
-#define CLK_ICN_REG		0
-#define CLK_ETH1_PHY		4
-
-/* CLOCKGEN A1 */
-#define CLK_ICN_IF_2		0
-#define CLK_GMAC0_PHY		3
-
-#endif
-diff --git a/include/dt-bindings/clock/stm32fx-clock.h b/include/dt-bindings/clock/stm32fx-clock.h
-index e5dad050d518..b6ff9c68cb3f 100644
--- a/include/dt-bindings/clock/stm32fx-clock.h
-+++ b/include/dt-bindings/clock/stm32fx-clock.h
-@@ -10,7 +10,7 @@
-  * List of clocks which are not derived from system clock (SYSCLOCK)
-  *
-  * The index of these clocks is the secondary index of DT bindings
- * (see Documentation/devicetree/bindings/clock/st,stm32-rcc.txt)
-+ * (see Documentation/devicetree/bindings/clock/st,stm32-rcc.yaml)
-  *
-  * e.g:
- 	<assigned-clocks = <&rcc 1 CLK_LSE>;
-diff --git a/include/dt-bindings/clock/stm32h7-clks.h b/include/dt-bindings/clock/stm32h7-clks.h
-index 6637272b3242..330b39c2c303 100644
--- a/include/dt-bindings/clock/stm32h7-clks.h
-+++ b/include/dt-bindings/clock/stm32h7-clks.h
-@@ -126,8 +126,8 @@
- #define ADC3_CK 128
- #define DSI_CK 129
- #define LTDC_CK 130
-#define USART8_CK 131
-#define USART7_CK 132
-+#define UART8_CK 131
-+#define UART7_CK 132
- #define HDMICEC_CK 133
- #define I2C3_CK 134
- #define I2C2_CK 135
-diff --git a/include/dt-bindings/clock/stm32mp1-clks.h b/include/dt-bindings/clock/stm32mp1-clks.h
-index 25e8cfd43459..0a5324bcdbda 100644
--- a/include/dt-bindings/clock/stm32mp1-clks.h
-+++ b/include/dt-bindings/clock/stm32mp1-clks.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */
-+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
- /*
-  * Copyright (C) STMicroelectronics 2018 - All Rights Reserved
-  * Author: Gabriel Fernandez <gabriel.fernandez@st.com> for STMicroelectronics.
-diff --git a/include/dt-bindings/clock/stm32mp13-clks.h b/include/dt-bindings/clock/stm32mp13-clks.h
-index 02befd25edce..0bd7b54c65ff 100644
--- a/include/dt-bindings/clock/stm32mp13-clks.h
-+++ b/include/dt-bindings/clock/stm32mp13-clks.h
-@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0+ or BSD-3-Clause */
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */
- /*
-  * Copyright (C) STMicroelectronics 2020 - All Rights Reserved
- * Author: Gabriel Fernandez <gabriel.fernandez@st.com> for STMicroelectronics.
-+ * Author: Gabriel Fernandez <gabriel.fernandez@foss.st.com> for STMicroelectronics.
-  */
- 
- #ifndef _DT_BINDINGS_STM32MP13_CLKS_H_
-@@ -64,7 +64,7 @@
- #define CK_MCO1		38
- #define CK_MCO2		39
- 
-/*  IP clocks */
-+/* IP clocks */
- #define SYSCFG		40
- #define VREF		41
- #define DTS		42
-diff --git a/include/dt-bindings/clock/sun20i-d1-ccu.h b/include/dt-bindings/clock/sun20i-d1-ccu.h
-index e3ac53315e1a..fdbfb404f92a 100644
--- a/include/dt-bindings/clock/sun20i-d1-ccu.h
-+++ b/include/dt-bindings/clock/sun20i-d1-ccu.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (C) 2020 huangzhenwei@allwinnertech.com
-  * Copyright (C) 2021 Samuel Holland <samuel@sholland.org>
-@@ -152,5 +152,7 @@
- #define CLK_FANOUT0		142
- #define CLK_FANOUT1		143
- #define CLK_FANOUT2		144
-+#define CLK_BUS_CAN0		145
-+#define CLK_BUS_CAN1		146
- 
- #endif /* _DT_BINDINGS_CLK_SUN20I_D1_CCU_H_ */
-diff --git a/include/dt-bindings/clock/sun20i-d1-r-ccu.h b/include/dt-bindings/clock/sun20i-d1-r-ccu.h
-index 4c2697fd32b0..f95c170711e5 100644
--- a/include/dt-bindings/clock/sun20i-d1-r-ccu.h
-+++ b/include/dt-bindings/clock/sun20i-d1-r-ccu.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (C) 2021 Samuel Holland <samuel@sholland.org>
-  */
-diff --git a/include/dt-bindings/clock/sun50i-a100-ccu.h b/include/dt-bindings/clock/sun50i-a100-ccu.h
-index 28dc36e1a232..06a2031d466b 100644
--- a/include/dt-bindings/clock/sun50i-a100-ccu.h
-+++ b/include/dt-bindings/clock/sun50i-a100-ccu.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (c) 2020 Yangtao Li <frank@allwinnertech.com>
-  */
-diff --git a/include/dt-bindings/clock/sun50i-a64-ccu.h b/include/dt-bindings/clock/sun50i-a64-ccu.h
-index 175892189e9d..4f220ea7a23c 100644
--- a/include/dt-bindings/clock/sun50i-a64-ccu.h
-+++ b/include/dt-bindings/clock/sun50i-a64-ccu.h
-@@ -44,7 +44,9 @@
- #define _DT_BINDINGS_CLK_SUN50I_A64_H_
- 
- #define CLK_PLL_VIDEO0		7
-+#define CLK_PLL_VIDEO0_2X	8
- #define CLK_PLL_PERIPH0		11
-+#define CLK_PLL_MIPI		17
- 
- #define CLK_CPUX		21
- #define CLK_BUS_MIPI_DSI	28
-diff --git a/include/dt-bindings/clock/sun50i-h6-ccu.h b/include/dt-bindings/clock/sun50i-h6-ccu.h
-index a1545cd60e75..ef9123d81937 100644
--- a/include/dt-bindings/clock/sun50i-h6-ccu.h
-+++ b/include/dt-bindings/clock/sun50i-h6-ccu.h
-@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: (GPL-2.0+ or MIT)
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (C) 2017 Icenowy Zheng <icenowy@aosc.io>
-  */
-diff --git a/include/dt-bindings/clock/sun50i-h616-ccu.h b/include/dt-bindings/clock/sun50i-h616-ccu.h
-index 1191aca53ac6..6889405f9fec 100644
--- a/include/dt-bindings/clock/sun50i-h616-ccu.h
-+++ b/include/dt-bindings/clock/sun50i-h616-ccu.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (C) 2020 Arm Ltd.
-  */
-@@ -112,5 +112,10 @@
- #define CLK_HDCP		126
- #define CLK_BUS_HDCP		127
- #define CLK_PLL_SYSTEM_32K	128
-+#define CLK_BUS_GPADC		129
-+#define CLK_TCON_LCD0		130
-+#define CLK_BUS_TCON_LCD0	131
-+#define CLK_TCON_LCD1		132
-+#define CLK_BUS_TCON_LCD1	133
- 
- #endif /* _DT_BINDINGS_CLK_SUN50I_H616_H_ */
-diff --git a/include/dt-bindings/clock/sun55i-a523-ccu.h b/include/dt-bindings/clock/sun55i-a523-ccu.h
-new file mode 100644
-index 000000000000..54808fcfd556
--- /dev/null
-+++ b/include/dt-bindings/clock/sun55i-a523-ccu.h
-@@ -0,0 +1,190 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (C) 2024 Arm Ltd.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_SUN55I_A523_CCU_H_
-+#define _DT_BINDINGS_CLK_SUN55I_A523_CCU_H_
-+
-+#define CLK_PLL_DDR0		0
-+#define CLK_PLL_PERIPH0_4X	1
-+#define CLK_PLL_PERIPH0_2X	2
-+#define CLK_PLL_PERIPH0_800M	3
-+#define CLK_PLL_PERIPH0_480M	4
-+#define CLK_PLL_PERIPH0_600M	5
-+#define CLK_PLL_PERIPH0_400M	6
-+#define CLK_PLL_PERIPH0_300M	7
-+#define CLK_PLL_PERIPH0_200M	8
-+#define CLK_PLL_PERIPH0_160M	9
-+#define CLK_PLL_PERIPH0_150M	10
-+#define CLK_PLL_PERIPH1_4X	11
-+#define CLK_PLL_PERIPH1_2X	12
-+#define CLK_PLL_PERIPH1_800M	13
-+#define CLK_PLL_PERIPH1_480M	14
-+#define CLK_PLL_PERIPH1_600M	15
-+#define CLK_PLL_PERIPH1_400M	16
-+#define CLK_PLL_PERIPH1_300M	17
-+#define CLK_PLL_PERIPH1_200M	18
-+#define CLK_PLL_PERIPH1_160M	19
-+#define CLK_PLL_PERIPH1_150M	20
-+#define CLK_PLL_GPU		21
-+#define CLK_PLL_VIDEO0_8X	22
-+#define CLK_PLL_VIDEO0_4X	23
-+#define CLK_PLL_VIDEO0_3X	24
-+#define CLK_PLL_VIDEO1_8X	25
-+#define CLK_PLL_VIDEO1_4X	26
-+#define CLK_PLL_VIDEO1_3X	27
-+#define CLK_PLL_VIDEO2_8X	28
-+#define CLK_PLL_VIDEO2_4X	29
-+#define CLK_PLL_VIDEO2_3X	30
-+#define CLK_PLL_VIDEO3_8X	31
-+#define CLK_PLL_VIDEO3_4X	32
-+#define CLK_PLL_VIDEO3_3X	33
-+#define CLK_PLL_VE		34
-+#define CLK_PLL_AUDIO0_4X	35
-+#define CLK_PLL_AUDIO0_2X	36
-+#define CLK_PLL_AUDIO0		37
-+#define CLK_PLL_NPU_4X		38
-+#define CLK_PLL_NPU_2X		39
-+#define CLK_PLL_NPU		40
-+#define CLK_AHB			41
-+#define CLK_APB0		42
-+#define CLK_APB1		43
-+#define CLK_MBUS		44
-+#define CLK_DE			45
-+#define CLK_BUS_DE		46
-+#define CLK_DI			47
-+#define CLK_BUS_DI		48
-+#define CLK_G2D			49
-+#define CLK_BUS_G2D		50
-+#define CLK_GPU			51
-+#define CLK_BUS_GPU		52
-+#define CLK_CE			53
-+#define CLK_BUS_CE		54
-+#define CLK_BUS_CE_SYS		55
-+#define CLK_VE			56
-+#define CLK_BUS_VE		57
-+#define CLK_BUS_DMA		58
-+#define CLK_BUS_MSGBOX		59
-+#define CLK_BUS_SPINLOCK	60
-+#define CLK_HSTIMER0		61
-+#define CLK_HSTIMER1		62
-+#define CLK_HSTIMER2		63
-+#define CLK_HSTIMER3		64
-+#define CLK_HSTIMER4		65
-+#define CLK_HSTIMER5		66
-+#define CLK_BUS_HSTIMER		67
-+#define CLK_BUS_DBG		68
-+#define CLK_BUS_PWM0		69
-+#define CLK_BUS_PWM1		70
-+#define CLK_IOMMU		71
-+#define CLK_BUS_IOMMU		72
-+#define CLK_DRAM		73
-+#define CLK_MBUS_DMA		74
-+#define CLK_MBUS_VE		75
-+#define CLK_MBUS_CE		76
-+#define CLK_MBUS_CSI		77
-+#define CLK_MBUS_ISP		78
-+#define CLK_MBUS_EMAC1		79
-+#define CLK_BUS_DRAM		80
-+#define CLK_NAND0		81
-+#define CLK_NAND1		82
-+#define CLK_BUS_NAND		83
-+#define CLK_MMC0		84
-+#define CLK_MMC1		85
-+#define CLK_MMC2		86
-+#define CLK_BUS_SYSDAP		87
-+#define CLK_BUS_MMC0		88
-+#define CLK_BUS_MMC1		89
-+#define CLK_BUS_MMC2		90
-+#define CLK_BUS_UART0		91
-+#define CLK_BUS_UART1		92
-+#define CLK_BUS_UART2		93
-+#define CLK_BUS_UART3		94
-+#define CLK_BUS_UART4		95
-+#define CLK_BUS_UART5		96
-+#define CLK_BUS_UART6		97
-+#define CLK_BUS_UART7		98
-+#define CLK_BUS_I2C0		99
-+#define CLK_BUS_I2C1		100
-+#define CLK_BUS_I2C2		101
-+#define CLK_BUS_I2C3		102
-+#define CLK_BUS_I2C4		103
-+#define CLK_BUS_I2C5		104
-+#define CLK_BUS_CAN		105
-+#define CLK_SPI0		106
-+#define CLK_SPI1		107
-+#define CLK_SPI2		108
-+#define CLK_SPIFC		109
-+#define CLK_BUS_SPI0		110
-+#define CLK_BUS_SPI1		111
-+#define CLK_BUS_SPI2		112
-+#define CLK_BUS_SPIFC		113
-+#define CLK_EMAC0_25M		114
-+#define CLK_EMAC1_25M		115
-+#define CLK_BUS_EMAC0		116
-+#define CLK_BUS_EMAC1		117
-+#define CLK_IR_RX		118
-+#define CLK_BUS_IR_RX		119
-+#define CLK_IR_TX		120
-+#define CLK_BUS_IR_TX		121
-+#define CLK_GPADC0		122
-+#define CLK_GPADC1		123
-+#define CLK_BUS_GPADC0		124
-+#define CLK_BUS_GPADC1		125
-+#define CLK_BUS_THS		126
-+#define CLK_USB_OHCI0		127
-+#define CLK_USB_OHCI1		128
-+#define CLK_BUS_OHCI0		129
-+#define CLK_BUS_OHCI1		130
-+#define CLK_BUS_EHCI0		131
-+#define CLK_BUS_EHCI1		132
-+#define CLK_BUS_OTG		133
-+#define CLK_BUS_LRADC		134
-+#define CLK_PCIE_AUX		135
-+#define CLK_BUS_DISPLAY0_TOP	136
-+#define CLK_BUS_DISPLAY1_TOP	137
-+#define CLK_HDMI_24M		138
-+#define CLK_HDMI_CEC_32K	139
-+#define CLK_HDMI_CEC		140
-+#define CLK_BUS_HDMI		141
-+#define CLK_MIPI_DSI0		142
-+#define CLK_MIPI_DSI1		143
-+#define CLK_BUS_MIPI_DSI0	144
-+#define CLK_BUS_MIPI_DSI1	145
-+#define CLK_TCON_LCD0		146
-+#define CLK_TCON_LCD1		147
-+#define CLK_TCON_LCD2		148
-+#define CLK_COMBOPHY_DSI0	149
-+#define CLK_COMBOPHY_DSI1	150
-+#define CLK_BUS_TCON_LCD0	151
-+#define CLK_BUS_TCON_LCD1	152
-+#define CLK_BUS_TCON_LCD2	153
-+#define CLK_TCON_TV0		154
-+#define CLK_TCON_TV1		155
-+#define CLK_BUS_TCON_TV0	156
-+#define CLK_BUS_TCON_TV1	157
-+#define CLK_EDP			158
-+#define CLK_BUS_EDP		159
-+#define CLK_LEDC		160
-+#define CLK_BUS_LEDC		161
-+#define CLK_CSI_TOP		162
-+#define CLK_CSI_MCLK0		163
-+#define CLK_CSI_MCLK1		164
-+#define CLK_CSI_MCLK2		165
-+#define CLK_CSI_MCLK3		166
-+#define CLK_BUS_CSI		167
-+#define CLK_ISP			168
-+#define CLK_DSP			169
-+#define CLK_FANOUT_24M		170
-+#define CLK_FANOUT_12M		171
-+#define CLK_FANOUT_16M		172
-+#define CLK_FANOUT_25M		173
-+#define CLK_FANOUT_27M		174
-+#define CLK_FANOUT_PCLK		175
-+#define CLK_FANOUT0		176
-+#define CLK_FANOUT1		177
-+#define CLK_FANOUT2		178
-+#define CLK_NPU			179
-+
-+#endif /* _DT_BINDINGS_CLK_SUN55I_A523_CCU_H_ */
-diff --git a/include/dt-bindings/clock/sun55i-a523-mcu-ccu.h b/include/dt-bindings/clock/sun55i-a523-mcu-ccu.h
-new file mode 100644
-index 000000000000..6efc6bc7e11a
--- /dev/null
-+++ b/include/dt-bindings/clock/sun55i-a523-mcu-ccu.h
-@@ -0,0 +1,54 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (C) 2025 Chen-Yu Tsai <wens@csie.org>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_SUN55I_A523_MCU_CCU_H_
-+#define _DT_BINDINGS_CLK_SUN55I_A523_MCU_CCU_H_
-+
-+#define CLK_MCU_PLL_AUDIO1	0
-+#define CLK_MCU_PLL_AUDIO1_DIV2	1
-+#define CLK_MCU_PLL_AUDIO1_DIV5	2
-+#define CLK_MCU_AUDIO_OUT	3
-+#define CLK_MCU_DSP		4
-+#define CLK_MCU_I2S0		5
-+#define CLK_MCU_I2S1		6
-+#define CLK_MCU_I2S2		7
-+#define CLK_MCU_I2S3		8
-+#define CLK_MCU_I2S3_ASRC	9
-+#define CLK_BUS_MCU_I2S0	10
-+#define CLK_BUS_MCU_I2S1	11
-+#define CLK_BUS_MCU_I2S2	12
-+#define CLK_BUS_MCU_I2S3	13
-+#define CLK_MCU_SPDIF_TX	14
-+#define CLK_MCU_SPDIF_RX	15
-+#define CLK_BUS_MCU_SPDIF	16
-+#define CLK_MCU_DMIC		17
-+#define CLK_BUS_MCU_DMIC	18
-+#define CLK_MCU_AUDIO_CODEC_DAC	19
-+#define CLK_MCU_AUDIO_CODEC_ADC	20
-+#define CLK_BUS_MCU_AUDIO_CODEC	21
-+#define CLK_BUS_MCU_DSP_MSGBOX	22
-+#define CLK_BUS_MCU_DSP_CFG	23
-+#define CLK_BUS_MCU_NPU_HCLK	24
-+#define CLK_BUS_MCU_NPU_ACLK	25
-+#define CLK_MCU_TIMER0		26
-+#define CLK_MCU_TIMER1		27
-+#define CLK_MCU_TIMER2		28
-+#define CLK_MCU_TIMER3		29
-+#define CLK_MCU_TIMER4		30
-+#define CLK_MCU_TIMER5		31
-+#define CLK_BUS_MCU_TIMER	32
-+#define CLK_BUS_MCU_DMA		33
-+#define CLK_MCU_TZMA0		34
-+#define CLK_MCU_TZMA1		35
-+#define CLK_BUS_MCU_PUBSRAM	36
-+#define CLK_MCU_MBUS_DMA	37
-+#define CLK_MCU_MBUS		38
-+#define CLK_MCU_RISCV		39
-+#define CLK_BUS_MCU_RISCV_CFG	40
-+#define CLK_BUS_MCU_RISCV_MSGBOX	41
-+#define CLK_MCU_PWM0		42
-+#define CLK_BUS_MCU_PWM0	43
-+
-+#endif /* _DT_BINDINGS_CLK_SUN55I_A523_MCU_CCU_H_ */
-diff --git a/include/dt-bindings/clock/sun55i-a523-r-ccu.h b/include/dt-bindings/clock/sun55i-a523-r-ccu.h
-new file mode 100644
-index 000000000000..365647499b9a
--- /dev/null
-+++ b/include/dt-bindings/clock/sun55i-a523-r-ccu.h
-@@ -0,0 +1,37 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (C) 2024 Arm Ltd.
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_SUN55I_A523_R_CCU_H_
-+#define _DT_BINDINGS_CLK_SUN55I_A523_R_CCU_H_
-+
-+#define CLK_R_AHB		0
-+#define CLK_R_APB0		1
-+#define CLK_R_APB1		2
-+#define CLK_R_TIMER0		3
-+#define CLK_R_TIMER1		4
-+#define CLK_R_TIMER2		5
-+#define CLK_BUS_R_TIMER		6
-+#define CLK_BUS_R_TWD		7
-+#define CLK_R_PWMCTRL		8
-+#define CLK_BUS_R_PWMCTRL	9
-+#define CLK_R_SPI		10
-+#define CLK_BUS_R_SPI		11
-+#define CLK_BUS_R_SPINLOCK	12
-+#define CLK_BUS_R_MSGBOX	13
-+#define CLK_BUS_R_UART0		14
-+#define CLK_BUS_R_UART1		15
-+#define CLK_BUS_R_I2C0		16
-+#define CLK_BUS_R_I2C1		17
-+#define CLK_BUS_R_I2C2		18
-+#define CLK_BUS_R_PPU0		19
-+#define CLK_BUS_R_PPU1		20
-+#define CLK_BUS_R_CPU_BIST	21
-+#define CLK_R_IR_RX		22
-+#define CLK_BUS_R_IR_RX		23
-+#define CLK_BUS_R_DMA		24
-+#define CLK_BUS_R_RTC		25
-+#define CLK_BUS_R_CPUCFG	26
-+
-+#endif /* _DT_BINDINGS_CLK_SUN55I_A523_R_CCU_H_ */
-diff --git a/include/dt-bindings/clock/sun6i-rtc.h b/include/dt-bindings/clock/sun6i-rtc.h
-index c845493e4d37..3bd3aa3d57ce 100644
--- a/include/dt-bindings/clock/sun6i-rtc.h
-+++ b/include/dt-bindings/clock/sun6i-rtc.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- 
- #ifndef _DT_BINDINGS_CLK_SUN6I_RTC_H_
- #define _DT_BINDINGS_CLK_SUN6I_RTC_H_
-diff --git a/include/dt-bindings/clock/sun8i-v3s-ccu.h b/include/dt-bindings/clock/sun8i-v3s-ccu.h
-index 014ac6123d17..c4055629c9f9 100644
--- a/include/dt-bindings/clock/sun8i-v3s-ccu.h
-+++ b/include/dt-bindings/clock/sun8i-v3s-ccu.h
-@@ -96,7 +96,7 @@
- #define CLK_TCON0		64
- #define CLK_CSI_MISC		65
- #define CLK_CSI0_MCLK		66
-#define CLK_CSI1_SCLK		67
-+#define CLK_CSI_SCLK		67
- #define CLK_CSI1_MCLK		68
- #define CLK_VE			69
- #define CLK_AC_DIG		70
-diff --git a/include/dt-bindings/clock/suniv-ccu-f1c100s.h b/include/dt-bindings/clock/suniv-ccu-f1c100s.h
-index f5ac155c9c70..d7570765f424 100644
--- a/include/dt-bindings/clock/suniv-ccu-f1c100s.h
-+++ b/include/dt-bindings/clock/suniv-ccu-f1c100s.h
-@@ -67,4 +67,6 @@
- #define CLK_CODEC		65
- #define CLK_AVS			66
- 
-+#define CLK_IR			67
-+
- #endif
-diff --git a/include/dt-bindings/clock/tegra234-clock.h b/include/dt-bindings/clock/tegra234-clock.h
-index 173364a93381..c360455d02ee 100644
--- a/include/dt-bindings/clock/tegra234-clock.h
-+++ b/include/dt-bindings/clock/tegra234-clock.h
-@@ -9,6 +9,12 @@
-  * @defgroup bpmp_clock_ids Clock ID's
-  * @{
-  */
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_ACTMON */
-+#define TEGRA234_CLK_ACTMON			1U
-+/** @brief output of gate CLK_ENB_ADSP */
-+#define TEGRA234_CLK_ADSP			2U
-+/** @brief output of gate CLK_ENB_ADSPNEON */
-+#define TEGRA234_CLK_ADSPNEON			3U
- /** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_AHUB */
- #define TEGRA234_CLK_AHUB			4U
- /** @brief output of gate CLK_ENB_APB2APE */
-@@ -17,6 +23,18 @@
- #define TEGRA234_CLK_APE			6U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_AUD_MCLK */
- #define TEGRA234_CLK_AUD_MCLK			7U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_AXI_CBB */
-+#define TEGRA234_CLK_AXI_CBB			8U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_CAN1 */
-+#define TEGRA234_CLK_CAN1			9U
-+/** @brief output of gate CLK_ENB_CAN1_HOST */
-+#define TEGRA234_CLK_CAN1_HOST			10U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_CAN2 */
-+#define TEGRA234_CLK_CAN2			11U
-+/** @brief output of gate CLK_ENB_CAN2_HOST */
-+#define TEGRA234_CLK_CAN2_HOST			12U
-+/** @brief output of divider CLK_RST_CONTROLLER_CLK_M_DIVIDE */
-+#define TEGRA234_CLK_CLK_M			14U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DMIC1 */
- #define TEGRA234_CLK_DMIC1			15U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DMIC2 */
-@@ -25,6 +43,28 @@
- #define TEGRA234_CLK_DMIC3			17U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DMIC4 */
- #define TEGRA234_CLK_DMIC4			18U
-+/** @brief output of gate CLK_ENB_DPAUX */
-+#define TEGRA234_CLK_DPAUX			19U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVJPG1 */
-+#define TEGRA234_CLK_NVJPG1			20U
-+/**
-+ * @brief output of mux controlled by CLK_RST_CONTROLLER_ACLK_BURST_POLICY
-+ * divided by the divider controlled by ACLK_CLK_DIVISOR in
-+ * CLK_RST_CONTROLLER_SUPER_ACLK_DIVIDER
-+ */
-+#define TEGRA234_CLK_ACLK			21U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_MSS_ENCRYPT switch divider output */
-+#define TEGRA234_CLK_MSS_ENCRYPT		22U
-+/** @brief clock recovered from EAVB input */
-+#define TEGRA234_CLK_EQOS_RX_INPUT		23U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_AON_APB switch divider output */
-+#define TEGRA234_CLK_AON_APB			25U
-+/** @brief CLK_RST_CONTROLLER_AON_NIC_RATE divider output */
-+#define TEGRA234_CLK_AON_NIC			26U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_AON_CPU_NIC switch divider output */
-+#define TEGRA234_CLK_AON_CPU_NIC		27U
-+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLA1_BASE for use by audio clocks */
-+#define TEGRA234_CLK_PLLA1			28U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DSPK1 */
- #define TEGRA234_CLK_DSPK1			29U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DSPK2 */
-@@ -38,10 +78,33 @@
-  * throughput and memory controller power.
-  */
- #define TEGRA234_CLK_EMC			31U
-/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_HOST1X */
-#define TEGRA234_CLK_HOST1X                     46U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_EQOS_AXI_CLK_0 divider gated output */
-+#define TEGRA234_CLK_EQOS_AXI			32U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_EQOS_PTP_REF_CLK_0 divider gated output */
-+#define TEGRA234_CLK_EQOS_PTP_REF		33U
-+/** @brief output of gate CLK_ENB_EQOS_RX */
-+#define TEGRA234_CLK_EQOS_RX			34U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_EQOS_TX_CLK divider gated output */
-+#define TEGRA234_CLK_EQOS_TX			35U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_EXTPERIPH1 */
-+#define TEGRA234_CLK_EXTPERIPH1			36U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_EXTPERIPH2 */
-+#define TEGRA234_CLK_EXTPERIPH2			37U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_EXTPERIPH3 */
-+#define TEGRA234_CLK_EXTPERIPH3			38U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_EXTPERIPH4 */
-+#define TEGRA234_CLK_EXTPERIPH4			39U
- /** @brief output of gate CLK_ENB_FUSE */
- #define TEGRA234_CLK_FUSE			40U
-+/** @brief output of GPU GPC0 clkGen (in 1x mode same rate as GPC0 MUX2 out) */
-+#define TEGRA234_CLK_GPC0CLK			41U
-+/** @brief TODO */
-+#define TEGRA234_CLK_GPU_PWR			42U
-+/** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_HDA2CODEC_2X */
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_HOST1X */
-+#define TEGRA234_CLK_HOST1X			46U
-+/** @brief xusb_hs_hsicp_clk */
-+#define TEGRA234_CLK_XUSB_HS_HSICP		47U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C1 */
- #define TEGRA234_CLK_I2C1			48U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C2 */
-@@ -82,10 +145,66 @@
- #define TEGRA234_CLK_I2S6			66U
- /** @brief clock recovered from I2S6 input */
- #define TEGRA234_CLK_I2S6_SYNC_INPUT		67U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_ISP */
-+#define TEGRA234_CLK_ISP			69U
-+/** @brief Monitored branch of EQOS_RX clock */
-+#define TEGRA234_CLK_EQOS_RX_M			70U
-+/** @brief CLK_RST_CONTROLLER_MAUDCLK_OUT_SWITCH_DIVIDER switch divider output (maudclk) */
-+#define TEGRA234_CLK_MAUD			71U
-+/** @brief output of gate CLK_ENB_MIPI_CAL */
-+#define TEGRA234_CLK_MIPI_CAL			72U
-+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_MPHY_CORE_PLL_FIXED */
-+#define TEGRA234_CLK_MPHY_CORE_PLL_FIXED	73U
-+/** @brief output of gate CLK_ENB_MPHY_L0_RX_ANA */
-+#define TEGRA234_CLK_MPHY_L0_RX_ANA		74U
-+/** @brief output of gate CLK_ENB_MPHY_L0_RX_LS_BIT */
-+#define TEGRA234_CLK_MPHY_L0_RX_LS_BIT		75U
-+/** @brief output of gate CLK_ENB_MPHY_L0_RX_SYMB */
-+#define TEGRA234_CLK_MPHY_L0_RX_SYMB		76U
-+/** @brief output of gate CLK_ENB_MPHY_L0_TX_LS_3XBIT */
-+#define TEGRA234_CLK_MPHY_L0_TX_LS_3XBIT	77U
-+/** @brief output of gate CLK_ENB_MPHY_L0_TX_SYMB */
-+#define TEGRA234_CLK_MPHY_L0_TX_SYMB		78U
-+/** @brief output of gate CLK_ENB_MPHY_L1_RX_ANA */
-+#define TEGRA234_CLK_MPHY_L1_RX_ANA		79U
-+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_MPHY_TX_1MHZ_REF */
-+#define TEGRA234_CLK_MPHY_TX_1MHZ_REF		80U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVCSI */
-+#define TEGRA234_CLK_NVCSI			81U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVCSILP */
-+#define TEGRA234_CLK_NVCSILP			82U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVDEC */
-+#define TEGRA234_CLK_NVDEC			83U
-+/** @brief CLK_RST_CONTROLLER_HUBCLK_OUT_SWITCH_DIVIDER switch divider output (hubclk) */
-+#define TEGRA234_CLK_HUB			84U
-+/** @brief CLK_RST_CONTROLLER_DISPCLK_SWITCH_DIVIDER switch divider output (dispclk) */
-+#define TEGRA234_CLK_DISP			85U
-+/** @brief RG_CLK_CTRL__0_DIV divider output (nvdisplay_p0_clk) */
-+#define TEGRA234_CLK_NVDISPLAY_P0		86U
-+/** @brief RG_CLK_CTRL__1_DIV divider output (nvdisplay_p1_clk) */
-+#define TEGRA234_CLK_NVDISPLAY_P1		87U
-+/** @brief DSC_CLK (DISPCLK ÷ 3) */
-+#define TEGRA234_CLK_DSC			88U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVENC */
-+#define TEGRA234_CLK_NVENC			89U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVJPG */
-+#define TEGRA234_CLK_NVJPG			90U
-+/** @brief input from Tegra's XTAL_IN */
-+#define TEGRA234_CLK_OSC			91U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_AON_TOUCH switch divider output */
-+#define TEGRA234_CLK_AON_TOUCH			92U
- /** PLL controlled by CLK_RST_CONTROLLER_PLLA_BASE for use by audio clocks */
- #define TEGRA234_CLK_PLLA			93U
-+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLAON_BASE for use by IP blocks in the AON domain */
-+#define TEGRA234_CLK_PLLAON			94U
-+/** Fixed 100MHz PLL for PCIe, SATA and superspeed USB */
-+#define TEGRA234_CLK_PLLE			100U
-+/** @brief PLLP vco output */
-+#define TEGRA234_CLK_PLLP			101U
- /** @brief PLLP clk output */
- #define TEGRA234_CLK_PLLP_OUT0			102U
-+/** Fixed frequency 960MHz PLL for USB and EAVB */
-+#define TEGRA234_CLK_UTMIP_PLL			103U
- /** @brief output of the divider CLK_RST_CONTROLLER_PLLA_OUT */
- #define TEGRA234_CLK_PLLA_OUT0			104U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_PWM1 */
-@@ -104,8 +223,50 @@
- #define TEGRA234_CLK_PWM7			111U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_PWM8 */
- #define TEGRA234_CLK_PWM8			112U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_RCE_CPU_NIC output */
-+#define TEGRA234_CLK_RCE_CPU_NIC		113U
-+/** @brief CLK_RST_CONTROLLER_RCE_NIC_RATE divider output */
-+#define TEGRA234_CLK_RCE_NIC			114U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_AON_I2C_SLOW switch divider output */
-+#define TEGRA234_CLK_AON_I2C_SLOW		117U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SCE_CPU_NIC */
-+#define TEGRA234_CLK_SCE_CPU_NIC		118U
-+/** @brief output of divider CLK_RST_CONTROLLER_SCE_NIC_RATE */
-+#define TEGRA234_CLK_SCE_NIC			119U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SDMMC1 */
-+#define TEGRA234_CLK_SDMMC1			120U
-+/** @brief Logical clk for setting the UPHY PLL3 rate */
-+#define TEGRA234_CLK_UPHY_PLL3			121U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SDMMC4 */
- #define TEGRA234_CLK_SDMMC4			123U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_SE switch divider gated output */
-+#define TEGRA234_CLK_SE				124U
-+/** @brief VPLL select for sor0_ref clk driven by disp_2clk_sor0_head_sel signal */
-+#define TEGRA234_CLK_SOR0_PLL_REF		125U
-+/** @brief Output of mux controlled by disp_2clk_sor0_pll_ref_clk_safe signal (sor0_ref_clk) */
-+#define TEGRA234_CLK_SOR0_REF			126U
-+/** @brief VPLL select for sor1_ref clk driven by disp_2clk_sor0_head_sel signal */
-+#define TEGRA234_CLK_SOR1_PLL_REF		127U
-+/** @brief SOR_PLL_REF_CLK_CTRL__0_DIV divider output */
-+#define TEGRA234_CLK_PRE_SOR0_REF		128U
-+/** @brief Output of mux controlled by disp_2clk_sor1_pll_ref_clk_safe signal (sor1_ref_clk) */
-+#define TEGRA234_CLK_SOR1_REF			129U
-+/** @brief SOR_PLL_REF_CLK_CTRL__1_DIV divider output */
-+#define TEGRA234_CLK_PRE_SOR1_REF		130U
-+/** @brief output of gate CLK_ENB_SOR_SAFE */
-+#define TEGRA234_CLK_SOR_SAFE			131U
-+/** @brief SOR_CLK_CTRL__0_DIV divider output */
-+#define TEGRA234_CLK_SOR0_DIV			132U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DMIC5 */
-+#define TEGRA234_CLK_DMIC5			134U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SPI1 */
-+#define TEGRA234_CLK_SPI1			135U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SPI2 */
-+#define TEGRA234_CLK_SPI2			136U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SPI3 */
-+#define TEGRA234_CLK_SPI3			137U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C_SLOW */
-+#define TEGRA234_CLK_I2C_SLOW			138U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_DMIC1 */
- #define TEGRA234_CLK_SYNC_DMIC1			139U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_DMIC2 */
-@@ -130,28 +291,132 @@
- #define TEGRA234_CLK_SYNC_I2S5			149U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_I2S6 */
- #define TEGRA234_CLK_SYNC_I2S6			150U
-+/** @brief controls MPHY_FORCE_LS_MODE upon enable & disable */
-+#define TEGRA234_CLK_MPHY_FORCE_LS_MODE		151U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_TACH0 */
-+#define TEGRA234_CLK_TACH0			152U
-+/** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_TSEC */
-+#define TEGRA234_CLK_TSEC			153U
-+/** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_PKA */
-+#define TEGRA234_CLK_TSEC_PKA			154U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTA */
- #define TEGRA234_CLK_UARTA			155U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTB */
-+#define TEGRA234_CLK_UARTB			156U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTC */
-+#define TEGRA234_CLK_UARTC			157U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTD */
-+#define TEGRA234_CLK_UARTD			158U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTE */
-+#define TEGRA234_CLK_UARTE			159U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTF */
-+#define TEGRA234_CLK_UARTF			160U
- /** @brief output of gate CLK_ENB_PEX1_CORE_6 */
- #define TEGRA234_CLK_PEX1_C6_CORE		161U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UART_FST_MIPI_CAL */
-+#define TEGRA234_CLK_UART_FST_MIPI_CAL		162U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UFSDEV_REF */
-+#define TEGRA234_CLK_UFSDEV_REF			163U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UFSHC_CG_SYS */
-+#define TEGRA234_CLK_UFSHC			164U
-+/** @brief output of gate CLK_ENB_USB2_TRK */
-+#define TEGRA234_CLK_USB2_TRK			165U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_VI */
-+#define TEGRA234_CLK_VI				166U
- /** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_VIC */
-#define TEGRA234_CLK_VIC                        167U
-+#define TEGRA234_CLK_VIC			167U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_CSITE switch divider output */
-+#define TEGRA234_CLK_CSITE			168U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_IST switch divider output */
-+#define TEGRA234_CLK_IST			169U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_IST_JTAG_REG_CLK_SEL */
-+#define TEGRA234_CLK_JTAG_INTFC_PRE_CG		170U
- /** @brief output of gate CLK_ENB_PEX2_CORE_7 */
- #define TEGRA234_CLK_PEX2_C7_CORE		171U
- /** @brief output of gate CLK_ENB_PEX2_CORE_8 */
- #define TEGRA234_CLK_PEX2_C8_CORE		172U
- /** @brief output of gate CLK_ENB_PEX2_CORE_9 */
- #define TEGRA234_CLK_PEX2_C9_CORE		173U
-+/** @brief dla0_falcon_clk */
-+#define TEGRA234_CLK_DLA0_FALCON		174U
-+/** @brief dla0_core_clk */
-+#define TEGRA234_CLK_DLA0_CORE			175U
-+/** @brief dla1_falcon_clk */
-+#define TEGRA234_CLK_DLA1_FALCON		176U
-+/** @brief dla1_core_clk */
-+#define TEGRA234_CLK_DLA1_CORE			177U
-+/** @brief Output of mux controlled by disp_2clk_sor0_clk_safe signal (sor0_clk) */
-+#define TEGRA234_CLK_SOR0			178U
-+/** @brief Output of mux controlled by disp_2clk_sor1_clk_safe signal (sor1_clk) */
-+#define TEGRA234_CLK_SOR1			179U
-+/** @brief DP macro feedback clock (same as LINKA_SYM CLKOUT) */
-+#define TEGRA234_CLK_SOR_PAD_INPUT		180U
-+/** @brief Output of mux controlled by disp_2clk_h0_dsi_sel signal in sf0_clk path */
-+#define TEGRA234_CLK_PRE_SF0			181U
-+/** @brief Output of mux controlled by disp_2clk_sf0_clk_safe signal (sf0_clk) */
-+#define TEGRA234_CLK_SF0			182U
-+/** @brief Output of mux controlled by disp_2clk_sf1_clk_safe signal (sf1_clk) */
-+#define TEGRA234_CLK_SF1			183U
-+/** @brief CLKOUT_AB output from DSI BRICK A (dsi_clkout_ab) */
-+#define TEGRA234_CLK_DSI_PAD_INPUT		184U
- /** @brief output of gate CLK_ENB_PEX2_CORE_10 */
- #define TEGRA234_CLK_PEX2_C10_CORE		187U
-/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_QSPI0 switch divider output */
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_UARTI switch divider output (uarti_r_clk) */
-+#define TEGRA234_CLK_UARTI			188U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_UARTJ switch divider output (uartj_r_clk) */
-+#define TEGRA234_CLK_UARTJ			189U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_UARTH switch divider output */
-+#define TEGRA234_CLK_UARTH			190U
-+/** @brief ungated version of fuse clk */
-+#define TEGRA234_CLK_FUSE_SERIAL		191U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_QSPI0 switch divider output (qspi0_2x_pm_clk) */
- #define TEGRA234_CLK_QSPI0_2X_PM		192U
-/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_QSPI1 switch divider output */
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_QSPI1 switch divider output (qspi1_2x_pm_clk) */
- #define TEGRA234_CLK_QSPI1_2X_PM		193U
-/** @brief output of the divider QSPI_CLK_DIV2_SEL in CLK_RST_CONTROLLER_CLK_SOURCE_QSPI0 */
-+/** @brief output of the divider QSPI_CLK_DIV2_SEL in CLK_RST_CONTROLLER_CLK_SOURCE_QSPI0 (qspi0_pm_clk) */
- #define TEGRA234_CLK_QSPI0_PM			194U
-/** @brief output of the divider QSPI_CLK_DIV2_SEL in CLK_RST_CONTROLLER_CLK_SOURCE_QSPI1 */
-+/** @brief output of the divider QSPI_CLK_DIV2_SEL in CLK_RST_CONTROLLER_CLK_SOURCE_QSPI1 (qspi1_pm_clk) */
- #define TEGRA234_CLK_QSPI1_PM			195U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_VI_CONST switch divider output */
-+#define TEGRA234_CLK_VI_CONST			196U
-+/** @brief NAFLL clock source for BPMP */
-+#define TEGRA234_CLK_NAFLL_BPMP			197U
-+/** @brief NAFLL clock source for SCE */
-+#define TEGRA234_CLK_NAFLL_SCE			198U
-+/** @brief NAFLL clock source for NVDEC */
-+#define TEGRA234_CLK_NAFLL_NVDEC		199U
-+/** @brief NAFLL clock source for NVJPG */
-+#define TEGRA234_CLK_NAFLL_NVJPG		200U
-+/** @brief NAFLL clock source for TSEC */
-+#define TEGRA234_CLK_NAFLL_TSEC			201U
-+/** @brief NAFLL clock source for VI */
-+#define TEGRA234_CLK_NAFLL_VI			203U
-+/** @brief NAFLL clock source for SE */
-+#define TEGRA234_CLK_NAFLL_SE			204U
-+/** @brief NAFLL clock source for NVENC */
-+#define TEGRA234_CLK_NAFLL_NVENC		205U
-+/** @brief NAFLL clock source for ISP */
-+#define TEGRA234_CLK_NAFLL_ISP			206U
-+/** @brief NAFLL clock source for VIC */
-+#define TEGRA234_CLK_NAFLL_VIC			207U
-+/** @brief NAFLL clock source for AXICBB */
-+#define TEGRA234_CLK_NAFLL_AXICBB		209U
-+/** @brief NAFLL clock source for NVJPG1 */
-+#define TEGRA234_CLK_NAFLL_NVJPG1		210U
-+/** @brief NAFLL clock source for PVA core */
-+#define TEGRA234_CLK_NAFLL_PVA0_CORE		211U
-+/** @brief NAFLL clock source for PVA VPS */
-+#define TEGRA234_CLK_NAFLL_PVA0_VPS		212U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_DBGAPB_0 switch divider output (dbgapb_clk) */
-+#define TEGRA234_CLK_DBGAPB			213U
-+/** @brief NAFLL clock source for RCE */
-+#define TEGRA234_CLK_NAFLL_RCE			214U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_LA switch divider output (la_r_clk) */
-+#define TEGRA234_CLK_LA				215U
-+/** @brief output of the divider CLK_RST_CONTROLLER_PLLP_OUTD */
-+#define TEGRA234_CLK_PLLP_OUT_JTAG		216U
-+/** @brief AXI_CBB branch sharing gate control with SDMMC4 */
-+#define TEGRA234_CLK_SDMMC4_AXICIF		217U
- /** @brief CLK_RST_CONTROLLER_CLK_SOURCE_SDMMC_LEGACY_TM switch divider output */
- #define TEGRA234_CLK_SDMMC_LEGACY_TM		219U
- /** @brief output of gate CLK_ENB_PEX0_CORE_0 */
-@@ -166,8 +431,42 @@
- #define TEGRA234_CLK_PEX0_C4_CORE		224U
- /** @brief output of gate CLK_ENB_PEX1_CORE_5 */
- #define TEGRA234_CLK_PEX1_C5_CORE		225U
-+/** @brief Monitored branch of PEX0_C0_CORE clock */
-+#define TEGRA234_CLK_PEX0_C0_CORE_M		229U
-+/** @brief Monitored branch of PEX0_C1_CORE clock */
-+#define TEGRA234_CLK_PEX0_C1_CORE_M		230U
-+/** @brief Monitored branch of PEX0_C2_CORE clock */
-+#define TEGRA234_CLK_PEX0_C2_CORE_M		231U
-+/** @brief Monitored branch of PEX0_C3_CORE clock */
-+#define TEGRA234_CLK_PEX0_C3_CORE_M		232U
-+/** @brief Monitored branch of PEX0_C4_CORE clock */
-+#define TEGRA234_CLK_PEX0_C4_CORE_M		233U
-+/** @brief Monitored branch of PEX1_C5_CORE clock */
-+#define TEGRA234_CLK_PEX1_C5_CORE_M		234U
-+/** @brief Monitored branch of PEX1_C6_CORE clock */
-+#define TEGRA234_CLK_PEX1_C6_CORE_M		235U
-+/** @brief output of GPU GPC1 clkGen (in 1x mode same rate as GPC1 MUX2 out) */
-+#define TEGRA234_CLK_GPC1CLK			236U
- /** @brief PLL controlled by CLK_RST_CONTROLLER_PLLC4_BASE */
- #define TEGRA234_CLK_PLLC4			237U
-+/** @brief PLLC4 VCO followed by DIV3 path */
-+#define TEGRA234_CLK_PLLC4_OUT1			239U
-+/** @brief PLLC4 VCO followed by DIV5 path */
-+#define TEGRA234_CLK_PLLC4_OUT2			240U
-+/** @brief output of the mux controlled by PLLC4_CLK_SEL */
-+#define TEGRA234_CLK_PLLC4_MUXED		241U
-+/** @brief PLLC4 VCO followed by DIV2 path */
-+#define TEGRA234_CLK_PLLC4_VCO_DIV2		242U
-+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLNVHS_BASE */
-+#define TEGRA234_CLK_PLLNVHS			243U
-+/** @brief Monitored branch of PEX2_C7_CORE clock */
-+#define TEGRA234_CLK_PEX2_C7_CORE_M		244U
-+/** @brief Monitored branch of PEX2_C8_CORE clock */
-+#define TEGRA234_CLK_PEX2_C8_CORE_M		245U
-+/** @brief Monitored branch of PEX2_C9_CORE clock */
-+#define TEGRA234_CLK_PEX2_C9_CORE_M		246U
-+/** @brief Monitored branch of PEX2_C10_CORE clock */
-+#define TEGRA234_CLK_PEX2_C10_CORE_M		247U
- /** @brief RX clock recovered from MGBE0 lane input */
- #define TEGRA234_CLK_MGBE0_RX_INPUT		248U
- /** @brief RX clock recovered from MGBE1 lane input */
-@@ -176,8 +475,185 @@
- #define TEGRA234_CLK_MGBE2_RX_INPUT		250U
- /** @brief RX clock recovered from MGBE3 lane input */
- #define TEGRA234_CLK_MGBE3_RX_INPUT		251U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_PEX_SATA_USB_RX_BYP switch divider output */
-+#define TEGRA234_CLK_PEX_SATA_USB_RX_BYP	254U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_PEX_USB_PAD_PLL0_MGMT switch divider output */
-+#define TEGRA234_CLK_PEX_USB_PAD_PLL0_MGMT	255U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_PEX_USB_PAD_PLL1_MGMT switch divider output */
-+#define TEGRA234_CLK_PEX_USB_PAD_PLL1_MGMT	256U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_PEX_USB_PAD_PLL2_MGMT switch divider output */
-+#define TEGRA234_CLK_PEX_USB_PAD_PLL2_MGMT	257U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_PEX_USB_PAD_PLL3_MGMT switch divider output */
-+#define TEGRA234_CLK_PEX_USB_PAD_PLL3_MGMT	258U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_NVHS_RX_BYP switch divider output */
-+#define TEGRA234_CLK_NVHS_RX_BYP_REF		263U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_NVHS_PLL0_MGMT switch divider output */
-+#define TEGRA234_CLK_NVHS_PLL0_MGMT		264U
-+/** @brief xusb_core_dev_clk */
-+#define TEGRA234_CLK_XUSB_CORE_DEV		265U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_XUSB_CORE_HOST switch divider output  */
-+#define TEGRA234_CLK_XUSB_CORE_MUX		266U
-+/** @brief xusb_core_host_clk */
-+#define TEGRA234_CLK_XUSB_CORE_HOST		267U
-+/** @brief xusb_core_superspeed_clk */
-+#define TEGRA234_CLK_XUSB_CORE_SS		268U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_XUSB_FALCON switch divider output */
-+#define TEGRA234_CLK_XUSB_FALCON		269U
-+/** @brief xusb_falcon_host_clk */
-+#define TEGRA234_CLK_XUSB_FALCON_HOST		270U
-+/** @brief xusb_falcon_superspeed_clk */
-+#define TEGRA234_CLK_XUSB_FALCON_SS		271U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_XUSB_FS switch divider output */
-+#define TEGRA234_CLK_XUSB_FS			272U
-+/** @brief xusb_fs_host_clk */
-+#define TEGRA234_CLK_XUSB_FS_HOST		273U
-+/** @brief xusb_fs_dev_clk */
-+#define TEGRA234_CLK_XUSB_FS_DEV		274U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_XUSB_SS switch divider output */
-+#define TEGRA234_CLK_XUSB_SS			275U
-+/** @brief xusb_ss_dev_clk */
-+#define TEGRA234_CLK_XUSB_SS_DEV		276U
-+/** @brief xusb_ss_superspeed_clk */
-+#define TEGRA234_CLK_XUSB_SS_SUPERSPEED		277U
-+/** @brief NAFLL clock source for CPU cluster 0 */
-+#define TEGRA234_CLK_NAFLL_CLUSTER0		280U /* TODO: remove */
-+#define TEGRA234_CLK_NAFLL_CLUSTER0_CORE	280U
-+/** @brief NAFLL clock source for CPU cluster 1 */
-+#define TEGRA234_CLK_NAFLL_CLUSTER1		281U /* TODO: remove */
-+#define TEGRA234_CLK_NAFLL_CLUSTER1_CORE	281U
-+/** @brief NAFLL clock source for CPU cluster 2 */
-+#define TEGRA234_CLK_NAFLL_CLUSTER2		282U /* TODO: remove */
-+#define TEGRA234_CLK_NAFLL_CLUSTER2_CORE	282U
-+/** @brief CLK_RST_CONTROLLER_CAN1_CORE_RATE divider output */
-+#define TEGRA234_CLK_CAN1_CORE			284U
-+/** @brief CLK_RST_CONTROLLER_CAN2_CORE_RATE divider outputt */
-+#define TEGRA234_CLK_CAN2_CORE			285U
-+/** @brief CLK_RST_CONTROLLER_PLLA1_OUT1 switch divider output */
-+#define TEGRA234_CLK_PLLA1_OUT1			286U
-+/** @brief NVHS PLL hardware power sequencer (overrides 'manual' programming of PLL) */
-+#define TEGRA234_CLK_PLLNVHS_HPS		287U
-+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLREFE_BASE */
-+#define TEGRA234_CLK_PLLREFE_VCOOUT		288U
- /** @brief 32K input clock provided by PMIC */
- #define TEGRA234_CLK_CLK_32K			289U
-+/** @brief Fixed 48MHz clock divided down from utmipll */
-+#define TEGRA234_CLK_UTMIPLL_CLKOUT48		291U
-+/** @brief Fixed 480MHz clock divided down from utmipll */
-+#define TEGRA234_CLK_UTMIPLL_CLKOUT480		292U
-+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLNVCSI_BASE  */
-+#define TEGRA234_CLK_PLLNVCSI			294U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_PVA0_CPU_AXI switch divider output */
-+#define TEGRA234_CLK_PVA0_CPU_AXI		295U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_PVA0_VPS switch divider output */
-+#define TEGRA234_CLK_PVA0_VPS			297U
-+/** @brief DLA0_CORE_NAFLL */
-+#define TEGRA234_CLK_NAFLL_DLA0_CORE		299U
-+/** @brief DLA0_FALCON_NAFLL */
-+#define TEGRA234_CLK_NAFLL_DLA0_FALCON		300U
-+/** @brief DLA1_CORE_NAFLL */
-+#define TEGRA234_CLK_NAFLL_DLA1_CORE		301U
-+/** @brief DLA1_FALCON_NAFLL */
-+#define TEGRA234_CLK_NAFLL_DLA1_FALCON		302U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_AON_UART_FST_MIPI_CAL */
-+#define TEGRA234_CLK_AON_UART_FST_MIPI_CAL	303U
-+/** @brief GPU system clock */
-+#define TEGRA234_CLK_GPUSYS			304U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C5 */
-+#define TEGRA234_CLK_I2C5			305U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_SE switch divider free running clk */
-+#define TEGRA234_CLK_FR_SE			306U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_BPMP_CPU_NIC switch divider output */
-+#define TEGRA234_CLK_BPMP_CPU_NIC		307U
-+/** @brief output of gate CLK_ENB_BPMP_CPU */
-+#define TEGRA234_CLK_BPMP_CPU			308U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_TSC switch divider output */
-+#define TEGRA234_CLK_TSC			309U
-+/** @brief output of mem pll A sync mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_EMC */
-+#define TEGRA234_CLK_EMCSA_MPLL			310U
-+/** @brief output of mem pll B sync mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_EMCSB */
-+#define TEGRA234_CLK_EMCSB_MPLL			311U
-+/** @brief output of mem pll C sync mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_EMCSC */
-+#define TEGRA234_CLK_EMCSC_MPLL			312U
-+/** @brief output of mem pll D sync mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_EMCSD */
-+#define TEGRA234_CLK_EMCSD_MPLL			313U
-+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLC_BASE */
-+#define TEGRA234_CLK_PLLC			314U
-+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLC2_BASE */
-+#define TEGRA234_CLK_PLLC2			315U
-+/** @brief CLK_RST_CONTROLLER_TSC_HS_SUPER_CLK_DIVIDER skip divider output */
-+#define TEGRA234_CLK_TSC_REF			317U
-+/** @brief Dummy clock to ensure minimum SoC voltage for fuse burning */
-+#define TEGRA234_CLK_FUSE_BURN			318U
-+/** @brief GBE PLL */
-+#define TEGRA234_CLK_PLLGBE			319U
-+/** @brief GBE PLL hardware power sequencer */
-+#define TEGRA234_CLK_PLLGBE_HPS			320U
-+/** @brief output of EMC CDB side A fixed (DIV4)  divider */
-+#define TEGRA234_CLK_EMCSA_EMC			321U
-+/** @brief output of EMC CDB side B fixed (DIV4)  divider */
-+#define TEGRA234_CLK_EMCSB_EMC			322U
-+/** @brief output of EMC CDB side C fixed (DIV4)  divider */
-+#define TEGRA234_CLK_EMCSC_EMC			323U
-+/** @brief output of EMC CDB side D fixed (DIV4)  divider */
-+#define TEGRA234_CLK_EMCSD_EMC			324U
-+/** @brief PLLE hardware power sequencer (overrides 'manual' programming of PLL) */
-+#define TEGRA234_CLK_PLLE_HPS			326U
-+/** @brief CLK_ENB_PLLREFE_OUT gate output */
-+#define TEGRA234_CLK_PLLREFE_VCOOUT_GATED	327U
-+/** @brief TEGRA234_CLK_SOR_SAFE clk source (PLLP_OUT0 divided by 17) */
-+#define TEGRA234_CLK_PLLP_DIV17			328U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_SOC_THERM switch divider output */
-+#define TEGRA234_CLK_SOC_THERM			329U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_TSENSE switch divider output */
-+#define TEGRA234_CLK_TSENSE			330U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_SEU1 switch divider free running clk */
-+#define TEGRA234_CLK_FR_SEU1			331U
-+/** @brief NAFLL clock source for OFA */
-+#define TEGRA234_CLK_NAFLL_OFA			333U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_OFA switch divider output */
-+#define TEGRA234_CLK_OFA			334U
-+/** @brief NAFLL clock source for SEU1 */
-+#define TEGRA234_CLK_NAFLL_SEU1			335U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_SEU1 switch divider gated output */
-+#define TEGRA234_CLK_SEU1			336U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SPI4 */
-+#define TEGRA234_CLK_SPI4			337U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SPI5 */
-+#define TEGRA234_CLK_SPI5			338U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DCE_CPU_NIC */
-+#define TEGRA234_CLK_DCE_CPU_NIC		339U
-+/** @brief output of divider CLK_RST_CONTROLLER_DCE_NIC_RATE */
-+#define TEGRA234_CLK_DCE_NIC			340U
-+/** @brief NAFLL clock source for DCE */
-+#define TEGRA234_CLK_NAFLL_DCE			341U
-+/** @brief Monitored branch of MPHY_L0_RX_ANA clock */
-+#define TEGRA234_CLK_MPHY_L0_RX_ANA_M		342U
-+/** @brief Monitored branch of MPHY_L1_RX_ANA clock */
-+#define TEGRA234_CLK_MPHY_L1_RX_ANA_M		343U
-+/** @brief ungated version of TX symbol clock after fixed 1/2 divider */
-+#define TEGRA234_CLK_MPHY_L0_TX_PRE_SYMB	344U
-+/** @brief output of divider CLK_RST_CONTROLLER_CLK_SOURCE_MPHY_L0_TX_LS_SYMB */
-+#define TEGRA234_CLK_MPHY_L0_TX_LS_SYMB_DIV	345U
-+/** @brief output of gate CLK_ENB_MPHY_L0_TX_2X_SYMB */
-+#define TEGRA234_CLK_MPHY_L0_TX_2X_SYMB		346U
-+/** @brief output of SW_MPHY_L0_TX_HS_SYMB divider in CLK_RST_CONTROLLER_MPHY_L0_TX_CLK_CTRL_0 */
-+#define TEGRA234_CLK_MPHY_L0_TX_HS_SYMB_DIV	347U
-+/** @brief output of SW_MPHY_L0_TX_LS_3XBIT divider in CLK_RST_CONTROLLER_MPHY_L0_TX_CLK_CTRL_0 */
-+#define TEGRA234_CLK_MPHY_L0_TX_LS_3XBIT_DIV	348U
-+/** @brief LS/HS divider mux SW_MPHY_L0_TX_LS_HS_SEL in CLK_RST_CONTROLLER_MPHY_L0_TX_CLK_CTRL_0 */
-+#define TEGRA234_CLK_MPHY_L0_TX_MUX_SYMB_DIV	349U
-+/** @brief Monitored branch of MPHY_L0_TX_SYMB clock */
-+#define TEGRA234_CLK_MPHY_L0_TX_SYMB_M		350U
-+/** @brief output of divider CLK_RST_CONTROLLER_CLK_SOURCE_MPHY_L0_RX_LS_SYMB */
-+#define TEGRA234_CLK_MPHY_L0_RX_LS_SYMB_DIV	351U
-+/** @brief output of SW_MPHY_L0_RX_HS_SYMB divider in CLK_RST_CONTROLLER_MPHY_L0_RX_CLK_CTRL_0 */
-+#define TEGRA234_CLK_MPHY_L0_RX_HS_SYMB_DIV	352U
-+/** @brief output of SW_MPHY_L0_RX_LS_BIT divider in  CLK_RST_CONTROLLER_MPHY_L0_RX_CLK_CTRL_0 */
-+#define TEGRA234_CLK_MPHY_L0_RX_LS_BIT_DIV	353U
-+/** @brief LS/HS divider mux SW_MPHY_L0_RX_LS_HS_SEL in CLK_RST_CONTROLLER_MPHY_L0_RX_CLK_CTRL_0 */
-+#define TEGRA234_CLK_MPHY_L0_RX_MUX_SYMB_DIV	354U
-+/** @brief Monitored branch of MPHY_L0_RX_SYMB clock */
-+#define TEGRA234_CLK_MPHY_L0_RX_SYMB_M		355U
- /** @brief Monitored branch of MBGE0 RX input clock */
- #define TEGRA234_CLK_MGBE0_RX_INPUT_M		357U
- /** @brief Monitored branch of MBGE1 RX input clock */
-@@ -194,6 +670,14 @@
- #define TEGRA234_CLK_MGBE2_RX_PCS_M		363U
- /** @brief Monitored branch of MGBE3 RX PCS mux output */
- #define TEGRA234_CLK_MGBE3_RX_PCS_M		364U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_TACH1 */
-+#define TEGRA234_CLK_TACH1			365U
-+/** @brief GBE_UPHY_MGBES_APP_CLK switch divider gated output */
-+#define TEGRA234_CLK_MGBES_APP			366U
-+/** @brief Logical clk for setting GBE UPHY PLL2 TX_REF rate */
-+#define TEGRA234_CLK_UPHY_GBE_PLL2_TX_REF	367U
-+/** @brief Logical clk for setting GBE UPHY PLL2 XDIG rate */
-+#define TEGRA234_CLK_UPHY_GBE_PLL2_XDIG		368U
- /** @brief RX PCS clock recovered from MGBE0 lane input */
- #define TEGRA234_CLK_MGBE0_RX_PCS_INPUT		369U
- /** @brief RX PCS clock recovered from MGBE1 lane input */
-@@ -230,6 +714,8 @@
- #define TEGRA234_CLK_MGBE1_MAC_DIVIDER		385U
- /** @brief GBE_UPHY_MGBE1_MAC_CLK gate output */
- #define TEGRA234_CLK_MGBE1_MAC			386U
-+/** @brief GBE_UPHY_MGBE1_MACSEC_CLK gate output */
-+#define TEGRA234_CLK_MGBE1_MACSEC		387U
- /** @brief GBE_UPHY_MGBE1_EEE_PCS_CLK gate output */
- #define TEGRA234_CLK_MGBE1_EEE_PCS		388U
- /** @brief GBE_UPHY_MGBE1_APP_CLK gate output */
-@@ -246,6 +732,8 @@
- #define TEGRA234_CLK_MGBE2_MAC_DIVIDER		394U
- /** @brief GBE_UPHY_MGBE2_MAC_CLK gate output */
- #define TEGRA234_CLK_MGBE2_MAC			395U
-+/** @brief GBE_UPHY_MGBE2_MACSEC_CLK gate output */
-+#define TEGRA234_CLK_MGBE2_MACSEC		396U
- /** @brief GBE_UPHY_MGBE2_EEE_PCS_CLK gate output */
- #define TEGRA234_CLK_MGBE2_EEE_PCS		397U
- /** @brief GBE_UPHY_MGBE2_APP_CLK gate output */
-@@ -270,9 +758,146 @@
- #define TEGRA234_CLK_MGBE3_APP			407U
- /** @brief GBE_UPHY_MGBE3_PTP_REF_CLK divider gated output */
- #define TEGRA234_CLK_MGBE3_PTP_REF		408U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_GBE_RX_BYP switch divider output */
-+#define TEGRA234_CLK_GBE_RX_BYP_REF		409U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_GBE_PLL0_MGMT switch divider output */
-+#define TEGRA234_CLK_GBE_PLL0_MGMT		410U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_GBE_PLL1_MGMT switch divider output */
-+#define TEGRA234_CLK_GBE_PLL1_MGMT		411U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_GBE_PLL2_MGMT switch divider output */
-+#define TEGRA234_CLK_GBE_PLL2_MGMT		412U
-+/** @brief output of gate CLK_ENB_EQOS_MACSEC_RX */
-+#define TEGRA234_CLK_EQOS_MACSEC_RX		413U
-+/** @brief output of gate CLK_ENB_EQOS_MACSEC_TX */
-+#define TEGRA234_CLK_EQOS_MACSEC_TX		414U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_EQOS_TX_CLK divider ungated output */
-+#define TEGRA234_CLK_EQOS_TX_DIVIDER		415U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_NVHS_PLL1_MGMT switch divider output */
-+#define TEGRA234_CLK_NVHS_PLL1_MGMT		416U
-+/** @brief CLK_RST_CONTROLLER_CLK_SOURCE_EMCHUB mux output */
-+#define TEGRA234_CLK_EMCHUB			417U
-+/** @brief clock recovered from I2S7 input */
-+#define TEGRA234_CLK_I2S7_SYNC_INPUT		418U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_I2S7 */
-+#define TEGRA234_CLK_SYNC_I2S7			419U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2S7 */
-+#define TEGRA234_CLK_I2S7			420U
-+/** @brief Monitored output of I2S7 pad macro mux */
-+#define TEGRA234_CLK_I2S7_PAD_M			421U
-+/** @brief clock recovered from I2S8 input */
-+#define TEGRA234_CLK_I2S8_SYNC_INPUT		422U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_I2S8 */
-+#define TEGRA234_CLK_SYNC_I2S8			423U
-+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2S8 */
-+#define TEGRA234_CLK_I2S8			424U
-+/** @brief Monitored output of I2S8 pad macro mux */
-+#define TEGRA234_CLK_I2S8_PAD_M			425U
-+/** @brief NAFLL clock source for GPU GPC0 */
-+#define TEGRA234_CLK_NAFLL_GPC0			426U
-+/** @brief NAFLL clock source for GPU GPC1 */
-+#define TEGRA234_CLK_NAFLL_GPC1			427U
-+/** @brief NAFLL clock source for GPU SYSCLK */
-+#define TEGRA234_CLK_NAFLL_GPUSYS		428U
-+/** @brief NAFLL clock source for CPU cluster 0 DSUCLK */
-+#define TEGRA234_CLK_NAFLL_DSU0			429U /* TODO: remove */
-+#define TEGRA234_CLK_NAFLL_CLUSTER0_DSU		429U
-+/** @brief NAFLL clock source for CPU cluster 1 DSUCLK */
-+#define TEGRA234_CLK_NAFLL_DSU1			430U /* TODO: remove */
-+#define TEGRA234_CLK_NAFLL_CLUSTER1_DSU		430U
-+/** @brief NAFLL clock source for CPU cluster 2 DSUCLK */
-+#define TEGRA234_CLK_NAFLL_DSU2			431U /* TODO: remove */
-+#define TEGRA234_CLK_NAFLL_CLUSTER2_DSU		431U
-+/** @brief output of gate CLK_ENB_SCE_CPU */
-+#define TEGRA234_CLK_SCE_CPU			432U
-+/** @brief output of gate CLK_ENB_RCE_CPU */
-+#define TEGRA234_CLK_RCE_CPU			433U
-+/** @brief output of gate CLK_ENB_DCE_CPU */
-+#define TEGRA234_CLK_DCE_CPU			434U
-+/** @brief DSIPLL VCO output */
-+#define TEGRA234_CLK_DSIPLL_VCO			435U
-+/** @brief DSIPLL SYNC_CLKOUTP/N differential output */
-+#define TEGRA234_CLK_DSIPLL_CLKOUTPN		436U
-+/** @brief DSIPLL SYNC_CLKOUTA output */
-+#define TEGRA234_CLK_DSIPLL_CLKOUTA		437U
-+/** @brief SPPLL0 VCO output */
-+#define TEGRA234_CLK_SPPLL0_VCO			438U
-+/** @brief SPPLL0 SYNC_CLKOUTP/N differential output */
-+#define TEGRA234_CLK_SPPLL0_CLKOUTPN		439U
-+/** @brief SPPLL0 SYNC_CLKOUTA output */
-+#define TEGRA234_CLK_SPPLL0_CLKOUTA		440U
-+/** @brief SPPLL0 SYNC_CLKOUTB output */
-+#define TEGRA234_CLK_SPPLL0_CLKOUTB		441U
-+/** @brief SPPLL0 CLKOUT_DIVBY10 output */
-+#define TEGRA234_CLK_SPPLL0_DIV10		442U
-+/** @brief SPPLL0 CLKOUT_DIVBY25 output */
-+#define TEGRA234_CLK_SPPLL0_DIV25		443U
-+/** @brief SPPLL0 CLKOUT_DIVBY27P/N differential output */
-+#define TEGRA234_CLK_SPPLL0_DIV27PN		444U
-+/** @brief SPPLL1 VCO output */
-+#define TEGRA234_CLK_SPPLL1_VCO			445U
-+/** @brief SPPLL1 SYNC_CLKOUTP/N differential output */
-+#define TEGRA234_CLK_SPPLL1_CLKOUTPN		446U
-+/** @brief SPPLL1 CLKOUT_DIVBY27P/N differential output */
-+#define TEGRA234_CLK_SPPLL1_DIV27PN		447U
-+/** @brief VPLL0 reference clock */
-+#define TEGRA234_CLK_VPLL0_REF			448U
-+/** @brief VPLL0 */
-+#define TEGRA234_CLK_VPLL0			449U
-+/** @brief VPLL1 */
-+#define TEGRA234_CLK_VPLL1			450U
-+/** @brief NVDISPLAY_P0_CLK reference select */
-+#define TEGRA234_CLK_NVDISPLAY_P0_REF		451U
-+/** @brief RG0_PCLK */
-+#define TEGRA234_CLK_RG0			452U
-+/** @brief RG1_PCLK */
-+#define TEGRA234_CLK_RG1			453U
-+/** @brief DISPPLL output */
-+#define TEGRA234_CLK_DISPPLL			454U
-+/** @brief DISPHUBPLL output */
-+#define TEGRA234_CLK_DISPHUBPLL			455U
-+/** @brief CLK_RST_CONTROLLER_DSI_LP_SWITCH_DIVIDER switch divider output (dsi_lp_clk) */
-+#define TEGRA234_CLK_DSI_LP			456U
- /** @brief CLK_RST_CONTROLLER_AZA2XBITCLK_OUT_SWITCH_DIVIDER switch divider output (aza_2xbitclk) */
- #define TEGRA234_CLK_AZA_2XBIT			457U
- /** @brief aza_2xbitclk / 2 (aza_bitclk) */
- #define TEGRA234_CLK_AZA_BIT			458U
-+/** @brief SWITCH_DSI_CORE_PIXEL_MISC_DSI_CORE_CLK_SRC switch output (dsi_core_clk) */
-+#define TEGRA234_CLK_DSI_CORE			459U
-+/** @brief Output of mux controlled by pkt_wr_fifo_signal from dsi (dsi_pixel_clk) */
-+#define TEGRA234_CLK_DSI_PIXEL			460U
-+/** @brief Output of mux controlled by disp_2clk_sor0_dp_sel (pre_sor0_clk) */
-+#define TEGRA234_CLK_PRE_SOR0			461U
-+/** @brief Output of mux controlled by disp_2clk_sor1_dp_sel (pre_sor1_clk) */
-+#define TEGRA234_CLK_PRE_SOR1			462U
-+/** @brief CLK_RST_CONTROLLER_LINK_REFCLK_CFG__0 output */
-+#define TEGRA234_CLK_DP_LINK_REF		463U
-+/** @brief Link clock input from DP macro brick PLL */
-+#define TEGRA234_CLK_SOR_LINKA_INPUT		464U
-+/** @brief SOR AFIFO clock outut */
-+#define TEGRA234_CLK_SOR_LINKA_AFIFO		465U
-+/** @brief Monitored branch of linka_afifo_clk */
-+#define TEGRA234_CLK_SOR_LINKA_AFIFO_M		466U
-+/** @brief Monitored branch of rg0_pclk */
-+#define TEGRA234_CLK_RG0_M			467U
-+/** @brief Monitored branch of rg1_pclk */
-+#define TEGRA234_CLK_RG1_M			468U
-+/** @brief Monitored branch of sor0_clk */
-+#define TEGRA234_CLK_SOR0_M			469U
-+/** @brief Monitored branch of sor1_clk */
-+#define TEGRA234_CLK_SOR1_M			470U
-+/** @brief EMC PLLHUB output */
-+#define TEGRA234_CLK_PLLHUB			471U
-+/** @brief output of fixed (DIV2) MC HUB divider */
-+#define TEGRA234_CLK_MCHUB			472U
-+/** @brief output of divider controlled by EMC side A MC_EMC_SAFE_SAME_FREQ */
-+#define TEGRA234_CLK_EMCSA_MC			473U
-+/** @brief output of divider controlled by EMC side B MC_EMC_SAFE_SAME_FREQ */
-+#define TEGRA234_CLK_EMCSB_MC			474U
-+/** @brief output of divider controlled by EMC side C MC_EMC_SAFE_SAME_FREQ */
-+#define TEGRA234_CLK_EMCSC_MC			475U
-+/** @brief output of divider controlled by EMC side D MC_EMC_SAFE_SAME_FREQ */
-+#define TEGRA234_CLK_EMCSD_MC			476U
-+
-+/** @} */
- 
- #endif
-diff --git a/include/dt-bindings/clock/tegra30-car.h b/include/dt-bindings/clock/tegra30-car.h
-index f193663e6f28..763b81f80908 100644
--- a/include/dt-bindings/clock/tegra30-car.h
-+++ b/include/dt-bindings/clock/tegra30-car.h
-@@ -271,6 +271,7 @@
- #define TEGRA30_CLK_AUDIO3_MUX 306
- #define TEGRA30_CLK_AUDIO4_MUX 307
- #define TEGRA30_CLK_SPDIF_MUX 308
-#define TEGRA30_CLK_CLK_MAX 309
-+#define TEGRA30_CLK_CSIA_PAD 309
-+#define TEGRA30_CLK_CSIB_PAD 310
- 
- #endif	/* _DT_BINDINGS_CLOCK_TEGRA30_CAR_H */
-diff --git a/include/dt-bindings/clock/thead,th1520-clk-ap.h b/include/dt-bindings/clock/thead,th1520-clk-ap.h
-new file mode 100644
-index 000000000000..09a9aa7b3ab1
--- /dev/null
-+++ b/include/dt-bindings/clock/thead,th1520-clk-ap.h
-@@ -0,0 +1,130 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2023 Vivo Communication Technology Co. Ltd.
-+ * Authors: Yangtao Li <frank.li@vivo.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_CLK_TH1520_H_
-+#define _DT_BINDINGS_CLK_TH1520_H_
-+
-+#define CLK_CPU_PLL0		0
-+#define CLK_CPU_PLL1		1
-+#define CLK_GMAC_PLL		2
-+#define CLK_VIDEO_PLL		3
-+#define CLK_DPU0_PLL		4
-+#define CLK_DPU1_PLL		5
-+#define CLK_TEE_PLL		6
-+#define CLK_C910_I0		7
-+#define CLK_C910		8
-+#define CLK_BROM		9
-+#define CLK_BMU			10
-+#define CLK_AHB2_CPUSYS_HCLK	11
-+#define CLK_APB3_CPUSYS_PCLK	12
-+#define CLK_AXI4_CPUSYS2_ACLK	13
-+#define CLK_AON2CPU_A2X		14
-+#define CLK_X2X_CPUSYS		15
-+#define CLK_AXI_ACLK		16
-+#define CLK_CPU2AON_X2H		17
-+#define CLK_PERI_AHB_HCLK	18
-+#define CLK_CPU2PERI_X2H	19
-+#define CLK_PERI_APB_PCLK	20
-+#define CLK_PERI2APB_PCLK	21
-+#define CLK_PERISYS_APB1_HCLK	22
-+#define CLK_PERISYS_APB2_HCLK	23
-+#define CLK_PERISYS_APB3_HCLK	24
-+#define CLK_PERISYS_APB4_HCLK	25
-+#define CLK_OSC12M		26
-+#define CLK_OUT1		27
-+#define CLK_OUT2		28
-+#define CLK_OUT3		29
-+#define CLK_OUT4		30
-+#define CLK_APB_PCLK		31
-+#define CLK_NPU			32
-+#define CLK_NPU_AXI		33
-+#define CLK_VI			34
-+#define CLK_VI_AHB		35
-+#define CLK_VO_AXI		36
-+#define CLK_VP_APB		37
-+#define CLK_VP_AXI		38
-+#define CLK_CPU2VP		39
-+#define CLK_VENC		40
-+#define CLK_DPU0		41
-+#define CLK_DPU1		42
-+#define CLK_EMMC_SDIO		43
-+#define CLK_GMAC1		44
-+#define CLK_PADCTRL1		45
-+#define CLK_DSMART		46
-+#define CLK_PADCTRL0		47
-+#define CLK_GMAC_AXI		48
-+#define CLK_GPIO3		49
-+#define CLK_GMAC0		50
-+#define CLK_PWM			51
-+#define CLK_QSPI0		52
-+#define CLK_QSPI1		53
-+#define CLK_SPI			54
-+#define CLK_UART0_PCLK		55
-+#define CLK_UART1_PCLK		56
-+#define CLK_UART2_PCLK		57
-+#define CLK_UART3_PCLK		58
-+#define CLK_UART4_PCLK		59
-+#define CLK_UART5_PCLK		60
-+#define CLK_GPIO0		61
-+#define CLK_GPIO1		62
-+#define CLK_GPIO2		63
-+#define CLK_I2C0		64
-+#define CLK_I2C1		65
-+#define CLK_I2C2		66
-+#define CLK_I2C3		67
-+#define CLK_I2C4		68
-+#define CLK_I2C5		69
-+#define CLK_SPINLOCK		70
-+#define CLK_DMA			71
-+#define CLK_MBOX0		72
-+#define CLK_MBOX1		73
-+#define CLK_MBOX2		74
-+#define CLK_MBOX3		75
-+#define CLK_WDT0		76
-+#define CLK_WDT1		77
-+#define CLK_TIMER0		78
-+#define CLK_TIMER1		79
-+#define CLK_SRAM0		80
-+#define CLK_SRAM1		81
-+#define CLK_SRAM2		82
-+#define CLK_SRAM3		83
-+#define CLK_PLL_GMAC_100M	84
-+#define CLK_UART_SCLK		85
-+
-+/* VO clocks */
-+#define CLK_AXI4_VO_ACLK		0
-+#define CLK_GPU_MEM			1
-+#define CLK_GPU_CORE			2
-+#define CLK_GPU_CFG_ACLK		3
-+#define CLK_DPU_PIXELCLK0		4
-+#define CLK_DPU_PIXELCLK1		5
-+#define CLK_DPU_HCLK			6
-+#define CLK_DPU_ACLK			7
-+#define CLK_DPU_CCLK			8
-+#define CLK_HDMI_SFR			9
-+#define CLK_HDMI_PCLK			10
-+#define CLK_HDMI_CEC			11
-+#define CLK_MIPI_DSI0_PCLK		12
-+#define CLK_MIPI_DSI1_PCLK		13
-+#define CLK_MIPI_DSI0_CFG		14
-+#define CLK_MIPI_DSI1_CFG		15
-+#define CLK_MIPI_DSI0_REFCLK		16
-+#define CLK_MIPI_DSI1_REFCLK		17
-+#define CLK_HDMI_I2S			18
-+#define CLK_X2H_DPU1_ACLK		19
-+#define CLK_X2H_DPU_ACLK		20
-+#define CLK_AXI4_VO_PCLK		21
-+#define CLK_IOPMP_VOSYS_DPU_PCLK	22
-+#define CLK_IOPMP_VOSYS_DPU1_PCLK	23
-+#define CLK_IOPMP_VOSYS_GPU_PCLK	24
-+#define CLK_IOPMP_DPU1_ACLK		25
-+#define CLK_IOPMP_DPU_ACLK		26
-+#define CLK_IOPMP_GPU_ACLK		27
-+#define CLK_MIPIDSI0_PIXCLK		28
-+#define CLK_MIPIDSI1_PIXCLK		29
-+#define CLK_HDMI_PIXCLK			30
-+
-+#endif
-diff --git a/include/dt-bindings/clock/toshiba,tmpv770x.h b/include/dt-bindings/clock/toshiba,tmpv770x.h
-index 5fce713001fd..a36c89266686 100644
--- a/include/dt-bindings/clock/toshiba,tmpv770x.h
-+++ b/include/dt-bindings/clock/toshiba,tmpv770x.h
-@@ -11,7 +11,6 @@
- #define TMPV770X_PLL_PIDDRCPLL		4
- #define TMPV770X_PLL_PIVOIFPLL		5
- #define TMPV770X_PLL_PIIMGERPLL		6
-#define TMPV770X_NR_PLL		7
- 
- /* Clocks */
- #define TMPV770X_CLK_PIPLL1_DIV1	0
-@@ -141,7 +140,9 @@
- #define TMPV770X_CLK_PIREFCLK		124
- #define TMPV770X_CLK_SBUS		125
- #define TMPV770X_CLK_BUSLCK		126
-#define TMPV770X_NR_CLK			127
-+#define TMPV770X_CLK_VIIFBS1_L2ISP	127
-+#define TMPV770X_CLK_VIIFBS1_L1ISP	128
-+#define TMPV770X_CLK_VIIFBS1_PROC	129
- 
- /* Reset */
- #define TMPV770X_RESET_PIETHER_2P5M	0
-@@ -176,6 +177,13 @@
- #define TMPV770X_RESET_PIPCMIF		29
- #define TMPV770X_RESET_PICKMON		30
- #define TMPV770X_RESET_SBUSCLK		31
-#define TMPV770X_NR_RESET		32
-+#define TMPV770X_RESET_VIIFBS0		32
-+#define TMPV770X_RESET_VIIFBS0_APB	33
-+#define TMPV770X_RESET_VIIFBS0_L2ISP	34
-+#define TMPV770X_RESET_VIIFBS0_L1ISP	35
-+#define TMPV770X_RESET_VIIFBS1		36
-+#define TMPV770X_RESET_VIIFBS1_APB	37
-+#define TMPV770X_RESET_VIIFBS1_L2ISP	38
-+#define TMPV770X_RESET_VIIFBS1_L1ISP	39
- 
- #endif /*_DT_BINDINGS_CLOCK_TOSHIBA_TMPV770X_H_ */
-diff --git a/include/dt-bindings/clk/versaclock.h b/include/dt-bindings/clock/versaclock.h
-index c6a6a0946564..c6a6a0946564 100644
--- a/include/dt-bindings/clk/versaclock.h
-+++ b/include/dt-bindings/clock/versaclock.h
-diff --git a/include/dt-bindings/clock/xlnx-zynqmp-clk.h b/include/dt-bindings/clock/xlnx-zynqmp-clk.h
-index cdc4c0b9a374..f0f7ddd3dcbd 100644
--- a/include/dt-bindings/clock/xlnx-zynqmp-clk.h
-+++ b/include/dt-bindings/clock/xlnx-zynqmp-clk.h
-@@ -9,6 +9,13 @@
- #ifndef _DT_BINDINGS_CLK_ZYNQMP_H
- #define _DT_BINDINGS_CLK_ZYNQMP_H
- 
-+/*
-+ * These bindings are deprecated, because they do not match the actual
-+ * concept of bindings but rather contain pure firmware values.
-+ * Instead include the header in the DTS source directory.
-+ */
-+#warning "These bindings are deprecated. Instead use the header in the DTS source directory."
-+
- #define IOPLL			0
- #define RPLL			1
- #define APLL			2
-diff --git a/include/dt-bindings/display/sdtv-standards.h b/include/dt-bindings/display/sdtv-standards.h
-index fbc1a3db2ea7..8249a2b47b79 100644
--- a/include/dt-bindings/display/sdtv-standards.h
-+++ b/include/dt-bindings/display/sdtv-standards.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0-only or X11 */
-+/* SPDX-License-Identifier: GPL-2.0-only OR X11 */
- /*
-  * Copyright 2019 Pengutronix, Marco Felsch <kernel@pengutronix.de>
-  */
-diff --git a/include/dt-bindings/dma/fsl-edma.h b/include/dt-bindings/dma/fsl-edma.h
-new file mode 100644
-index 000000000000..fd11478cfe9c
--- /dev/null
-+++ b/include/dt-bindings/dma/fsl-edma.h
-@@ -0,0 +1,21 @@
-+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
-+
-+#ifndef _FSL_EDMA_DT_BINDING_H_
-+#define _FSL_EDMA_DT_BINDING_H_
-+
-+/* Receive Channel */
-+#define FSL_EDMA_RX		0x1
-+
-+/* iMX8 audio remote DMA */
-+#define FSL_EDMA_REMOTE		0x2
-+
-+/* FIFO is continue memory region */
-+#define FSL_EDMA_MULTI_FIFO	0x4
-+
-+/* Channel need stick to even channel */
-+#define FSL_EDMA_EVEN_CH	0x8
-+
-+/* Channel need stick to odd channel */
-+#define FSL_EDMA_ODD_CH		0x10
-+
-+#endif
-diff --git a/include/dt-bindings/firmware/imx/rsrc.h b/include/dt-bindings/firmware/imx/rsrc.h
-index 43885056557c..1a8c025d77b8 100644
--- a/include/dt-bindings/firmware/imx/rsrc.h
-+++ b/include/dt-bindings/firmware/imx/rsrc.h
-@@ -13,34 +13,38 @@
-  * never be changed or removed (only added to at the end of the list).
-  */
- 
-#define IMX_SC_R_A53			0
-#define IMX_SC_R_A53_0			1
-#define IMX_SC_R_A53_1			2
-#define IMX_SC_R_A53_2			3
-#define IMX_SC_R_A53_3			4
-#define IMX_SC_R_A72			5
-#define IMX_SC_R_A72_0			6
-#define IMX_SC_R_A72_1			7
-#define IMX_SC_R_A72_2			8
-#define IMX_SC_R_A72_3			9
-+#define IMX_SC_R_AP_0			0
-+#define IMX_SC_R_AP_0_0			1
-+#define IMX_SC_R_AP_0_1			2
-+#define IMX_SC_R_AP_0_2			3
-+#define IMX_SC_R_AP_0_3			4
-+#define IMX_SC_R_AP_1			5
-+#define IMX_SC_R_AP_1_0			6
-+#define IMX_SC_R_AP_1_1			7
-+#define IMX_SC_R_AP_1_2			8
-+#define IMX_SC_R_AP_1_3			9
- #define IMX_SC_R_CCI			10
- #define IMX_SC_R_DB			11
- #define IMX_SC_R_DRC_0			12
- #define IMX_SC_R_DRC_1			13
- #define IMX_SC_R_GIC_SMMU		14
-#define IMX_SC_R_IRQSTR_M4_0		15
-#define IMX_SC_R_IRQSTR_M4_1		16
-#define IMX_SC_R_SMMU			17
-#define IMX_SC_R_GIC			18
-+#define IMX_SC_R_IRQSTR_MCU_0		15
-+#define IMX_SC_R_IRQSTR_MCU_1		16
-+#define IMX_SC_R_SMMU_0			17
-+#define IMX_SC_R_GIC_0			18
- #define IMX_SC_R_DC_0_BLIT0		19
- #define IMX_SC_R_DC_0_BLIT1		20
- #define IMX_SC_R_DC_0_BLIT2		21
- #define IMX_SC_R_DC_0_BLIT_OUT		22
-#define IMX_SC_R_PERF			23
-+#define IMX_SC_R_PERF_0			23
-+#define IMX_SC_R_USB_1_PHY		24
- #define IMX_SC_R_DC_0_WARP		25
-+#define IMX_SC_R_V2X_MU_0		26
-+#define IMX_SC_R_V2X_MU_1		27
- #define IMX_SC_R_DC_0_VIDEO0		28
- #define IMX_SC_R_DC_0_VIDEO1		29
- #define IMX_SC_R_DC_0_FRAC0		30
-+#define IMX_SC_R_V2X_MU_2		31
- #define IMX_SC_R_DC_0			32
- #define IMX_SC_R_GPU_2_PID0		33
- #define IMX_SC_R_DC_0_PLL_0		34
-@@ -49,11 +53,17 @@
- #define IMX_SC_R_DC_1_BLIT1		37
- #define IMX_SC_R_DC_1_BLIT2		38
- #define IMX_SC_R_DC_1_BLIT_OUT		39
-+#define IMX_SC_R_V2X_MU_3		40
-+#define IMX_SC_R_V2X_MU_4		41
- #define IMX_SC_R_DC_1_WARP		42
-+#define IMX_SC_R_STM			43
-+#define IMX_SC_R_SECVIO			44
- #define IMX_SC_R_DC_1_VIDEO0		45
- #define IMX_SC_R_DC_1_VIDEO1		46
- #define IMX_SC_R_DC_1_FRAC0		47
-+#define IMX_SC_R_V2X			48
- #define IMX_SC_R_DC_1			49
-+#define IMX_SC_R_UNUSED14		50
- #define IMX_SC_R_DC_1_PLL_0		51
- #define IMX_SC_R_DC_1_PLL_1		52
- #define IMX_SC_R_SPI_0			53
-@@ -144,10 +154,10 @@
- #define IMX_SC_R_DMA_1_CH29		137
- #define IMX_SC_R_DMA_1_CH30		138
- #define IMX_SC_R_DMA_1_CH31		139
-#define IMX_SC_R_UNUSED1		140
-#define IMX_SC_R_UNUSED2		141
-#define IMX_SC_R_UNUSED3		142
-#define IMX_SC_R_UNUSED4		143
-+#define IMX_SC_R_V2X_PID0		140
-+#define IMX_SC_R_V2X_PID1		141
-+#define IMX_SC_R_V2X_PID2		142
-+#define IMX_SC_R_V2X_PID3		143
- #define IMX_SC_R_GPU_0_PID0		144
- #define IMX_SC_R_GPU_0_PID1		145
- #define IMX_SC_R_GPU_0_PID2		146
-@@ -176,7 +186,7 @@
- #define IMX_SC_R_PCIE_B			169
- #define IMX_SC_R_SATA_0			170
- #define IMX_SC_R_SERDES_1		171
-#define IMX_SC_R_HSIO_GPIO		172
-+#define IMX_SC_R_HSIO_GPIO_0		172
- #define IMX_SC_R_MATCH_15		173
- #define IMX_SC_R_MATCH_16		174
- #define IMX_SC_R_MATCH_17		175
-@@ -243,15 +253,15 @@
- #define IMX_SC_R_ROM_0			236
- #define IMX_SC_R_FSPI_0			237
- #define IMX_SC_R_FSPI_1			238
-#define IMX_SC_R_IEE			239
-#define IMX_SC_R_IEE_R0			240
-#define IMX_SC_R_IEE_R1			241
-#define IMX_SC_R_IEE_R2			242
-#define IMX_SC_R_IEE_R3			243
-#define IMX_SC_R_IEE_R4			244
-#define IMX_SC_R_IEE_R5			245
-#define IMX_SC_R_IEE_R6			246
-#define IMX_SC_R_IEE_R7			247
-+#define IMX_SC_R_IEE_0			239
-+#define IMX_SC_R_IEE_0_R0		240
-+#define IMX_SC_R_IEE_0_R1		241
-+#define IMX_SC_R_IEE_0_R2		242
-+#define IMX_SC_R_IEE_0_R3		243
-+#define IMX_SC_R_IEE_0_R4		244
-+#define IMX_SC_R_IEE_0_R5		245
-+#define IMX_SC_R_IEE_0_R6		246
-+#define IMX_SC_R_IEE_0_R7		247
- #define IMX_SC_R_SDHC_0			248
- #define IMX_SC_R_SDHC_1			249
- #define IMX_SC_R_SDHC_2			250
-@@ -282,46 +292,50 @@
- #define IMX_SC_R_LVDS_2_PWM_0		275
- #define IMX_SC_R_LVDS_2_I2C_0		276
- #define IMX_SC_R_LVDS_2_I2C_1		277
-#define IMX_SC_R_M4_0_PID0		278
-#define IMX_SC_R_M4_0_PID1		279
-#define IMX_SC_R_M4_0_PID2		280
-#define IMX_SC_R_M4_0_PID3		281
-#define IMX_SC_R_M4_0_PID4		282
-#define IMX_SC_R_M4_0_RGPIO		283
-#define IMX_SC_R_M4_0_SEMA42		284
-#define IMX_SC_R_M4_0_TPM		285
-#define IMX_SC_R_M4_0_PIT		286
-#define IMX_SC_R_M4_0_UART		287
-#define IMX_SC_R_M4_0_I2C		288
-#define IMX_SC_R_M4_0_INTMUX		289
-#define IMX_SC_R_M4_0_MU_0B		292
-#define IMX_SC_R_M4_0_MU_0A0		293
-#define IMX_SC_R_M4_0_MU_0A1		294
-#define IMX_SC_R_M4_0_MU_0A2		295
-#define IMX_SC_R_M4_0_MU_0A3		296
-#define IMX_SC_R_M4_0_MU_1A		297
-#define IMX_SC_R_M4_1_PID0		298
-#define IMX_SC_R_M4_1_PID1		299
-#define IMX_SC_R_M4_1_PID2		300
-#define IMX_SC_R_M4_1_PID3		301
-#define IMX_SC_R_M4_1_PID4		302
-#define IMX_SC_R_M4_1_RGPIO		303
-#define IMX_SC_R_M4_1_SEMA42		304
-#define IMX_SC_R_M4_1_TPM		305
-#define IMX_SC_R_M4_1_PIT		306
-#define IMX_SC_R_M4_1_UART		307
-#define IMX_SC_R_M4_1_I2C		308
-#define IMX_SC_R_M4_1_INTMUX		309
-#define IMX_SC_R_M4_1_MU_0B		312
-#define IMX_SC_R_M4_1_MU_0A0		313
-#define IMX_SC_R_M4_1_MU_0A1		314
-#define IMX_SC_R_M4_1_MU_0A2		315
-#define IMX_SC_R_M4_1_MU_0A3		316
-#define IMX_SC_R_M4_1_MU_1A		317
-+#define IMX_SC_R_MCU_0_PID0		278
-+#define IMX_SC_R_MCU_0_PID1		279
-+#define IMX_SC_R_MCU_0_PID2		280
-+#define IMX_SC_R_MCU_0_PID3		281
-+#define IMX_SC_R_MCU_0_PID4		282
-+#define IMX_SC_R_MCU_0_RGPIO		283
-+#define IMX_SC_R_MCU_0_SEMA42		284
-+#define IMX_SC_R_MCU_0_TPM		285
-+#define IMX_SC_R_MCU_0_PIT		286
-+#define IMX_SC_R_MCU_0_UART		287
-+#define IMX_SC_R_MCU_0_I2C		288
-+#define IMX_SC_R_MCU_0_INTMUX		289
-+#define IMX_SC_R_ENET_0_A0		290
-+#define IMX_SC_R_ENET_0_A1		291
-+#define IMX_SC_R_MCU_0_MU_0B		292
-+#define IMX_SC_R_MCU_0_MU_0A0		293
-+#define IMX_SC_R_MCU_0_MU_0A1		294
-+#define IMX_SC_R_MCU_0_MU_0A2		295
-+#define IMX_SC_R_MCU_0_MU_0A3		296
-+#define IMX_SC_R_MCU_0_MU_1A		297
-+#define IMX_SC_R_MCU_1_PID0		298
-+#define IMX_SC_R_MCU_1_PID1		299
-+#define IMX_SC_R_MCU_1_PID2		300
-+#define IMX_SC_R_MCU_1_PID3		301
-+#define IMX_SC_R_MCU_1_PID4		302
-+#define IMX_SC_R_MCU_1_RGPIO		303
-+#define IMX_SC_R_MCU_1_SEMA42		304
-+#define IMX_SC_R_MCU_1_TPM		305
-+#define IMX_SC_R_MCU_1_PIT		306
-+#define IMX_SC_R_MCU_1_UART		307
-+#define IMX_SC_R_MCU_1_I2C		308
-+#define IMX_SC_R_MCU_1_INTMUX		309
-+#define IMX_SC_R_UNUSED17		310
-+#define IMX_SC_R_UNUSED18		311
-+#define IMX_SC_R_MCU_1_MU_0B		312
-+#define IMX_SC_R_MCU_1_MU_0A0		313
-+#define IMX_SC_R_MCU_1_MU_0A1		314
-+#define IMX_SC_R_MCU_1_MU_0A2		315
-+#define IMX_SC_R_MCU_1_MU_0A3		316
-+#define IMX_SC_R_MCU_1_MU_1A		317
- #define IMX_SC_R_SAI_0			318
- #define IMX_SC_R_SAI_1			319
- #define IMX_SC_R_SAI_2			320
-#define IMX_SC_R_IRQSTR_SCU2		321
-+#define IMX_SC_R_IRQSTR_AP_0		321
- #define IMX_SC_R_IRQSTR_DSP		322
- #define IMX_SC_R_ELCDIF_PLL		323
- #define IMX_SC_R_OCRAM			324
-@@ -366,33 +380,33 @@
- #define IMX_SC_R_VPU_PID5		363
- #define IMX_SC_R_VPU_PID6		364
- #define IMX_SC_R_VPU_PID7		365
-#define IMX_SC_R_VPU_UART		366
-#define IMX_SC_R_VPUCORE		367
-#define IMX_SC_R_VPUCORE_0		368
-#define IMX_SC_R_VPUCORE_1		369
-#define IMX_SC_R_VPUCORE_2		370
-#define IMX_SC_R_VPUCORE_3		371
-+#define IMX_SC_R_ENET_0_A2		366
-+#define IMX_SC_R_ENET_1_A0		367
-+#define IMX_SC_R_ENET_1_A1		368
-+#define IMX_SC_R_ENET_1_A2		369
-+#define IMX_SC_R_ENET_1_A3		370
-+#define IMX_SC_R_ENET_1_A4		371
- #define IMX_SC_R_DMA_4_CH0		372
- #define IMX_SC_R_DMA_4_CH1		373
- #define IMX_SC_R_DMA_4_CH2		374
- #define IMX_SC_R_DMA_4_CH3		375
- #define IMX_SC_R_DMA_4_CH4		376
-#define IMX_SC_R_ISI_CH0		377
-#define IMX_SC_R_ISI_CH1		378
-#define IMX_SC_R_ISI_CH2		379
-#define IMX_SC_R_ISI_CH3		380
-#define IMX_SC_R_ISI_CH4		381
-#define IMX_SC_R_ISI_CH5		382
-#define IMX_SC_R_ISI_CH6		383
-#define IMX_SC_R_ISI_CH7		384
-#define IMX_SC_R_MJPEG_DEC_S0		385
-#define IMX_SC_R_MJPEG_DEC_S1		386
-#define IMX_SC_R_MJPEG_DEC_S2		387
-#define IMX_SC_R_MJPEG_DEC_S3		388
-#define IMX_SC_R_MJPEG_ENC_S0		389
-#define IMX_SC_R_MJPEG_ENC_S1		390
-#define IMX_SC_R_MJPEG_ENC_S2		391
-#define IMX_SC_R_MJPEG_ENC_S3		392
-+#define IMX_SC_R_ISI_0_CH0		377
-+#define IMX_SC_R_ISI_0_CH1		378
-+#define IMX_SC_R_ISI_0_CH2		379
-+#define IMX_SC_R_ISI_0_CH3		380
-+#define IMX_SC_R_ISI_0_CH4		381
-+#define IMX_SC_R_ISI_0_CH5		382
-+#define IMX_SC_R_ISI_0_CH6		383
-+#define IMX_SC_R_ISI_0_CH7		384
-+#define IMX_SC_R_MJPEG_0_DEC_S0		385
-+#define IMX_SC_R_MJPEG_0_DEC_S1		386
-+#define IMX_SC_R_MJPEG_0_DEC_S2		387
-+#define IMX_SC_R_MJPEG_0_DEC_S3		388
-+#define IMX_SC_R_MJPEG_0_ENC_S0		389
-+#define IMX_SC_R_MJPEG_0_ENC_S1		390
-+#define IMX_SC_R_MJPEG_0_ENC_S2		391
-+#define IMX_SC_R_MJPEG_0_ENC_S3		392
- #define IMX_SC_R_MIPI_0			393
- #define IMX_SC_R_MIPI_0_PWM_0		394
- #define IMX_SC_R_MIPI_0_I2C_0		395
-@@ -507,11 +521,11 @@
- #define IMX_SC_R_SECO_MU_3		504
- #define IMX_SC_R_SECO_MU_4		505
- #define IMX_SC_R_HDMI_RX_PWM_0		506
-#define IMX_SC_R_A35			507
-#define IMX_SC_R_A35_0			508
-#define IMX_SC_R_A35_1			509
-#define IMX_SC_R_A35_2			510
-#define IMX_SC_R_A35_3			511
-+#define IMX_SC_R_AP_2			507
-+#define IMX_SC_R_AP_2_0			508
-+#define IMX_SC_R_AP_2_1			509
-+#define IMX_SC_R_AP_2_2			510
-+#define IMX_SC_R_AP_2_3			511
- #define IMX_SC_R_DSP			512
- #define IMX_SC_R_DSP_RAM		513
- #define IMX_SC_R_CAAM_JR1_OUT		514
-@@ -532,8 +546,8 @@
- #define IMX_SC_R_BOARD_R5		529
- #define IMX_SC_R_BOARD_R6		530
- #define IMX_SC_R_BOARD_R7		531
-#define IMX_SC_R_MJPEG_DEC_MP		532
-#define IMX_SC_R_MJPEG_ENC_MP		533
-+#define IMX_SC_R_MJPEG_0_DEC_MP		532
-+#define IMX_SC_R_MJPEG_0_ENC_MP		533
- #define IMX_SC_R_VPU_TS_0		534
- #define IMX_SC_R_VPU_MU_0		535
- #define IMX_SC_R_VPU_MU_1		536
-@@ -566,6 +580,105 @@
- #define IMX_SC_PM_CLK_BYPASS		4	/* Bypass clock */
- 
- /*
-+ * Compatibility defines for sc_rsrc_t
-+ */
-+#define IMX_SC_R_A35			IMX_SC_R_AP_2
-+#define IMX_SC_R_A35_0			IMX_SC_R_AP_2_0
-+#define IMX_SC_R_A35_1			IMX_SC_R_AP_2_1
-+#define IMX_SC_R_A35_2			IMX_SC_R_AP_2_2
-+#define IMX_SC_R_A35_3			IMX_SC_R_AP_2_3
-+#define IMX_SC_R_A53			IMX_SC_R_AP_0
-+#define IMX_SC_R_A53_0			IMX_SC_R_AP_0_0
-+#define IMX_SC_R_A53_1			IMX_SC_R_AP_0_1
-+#define IMX_SC_R_A53_2			IMX_SC_R_AP_0_2
-+#define IMX_SC_R_A53_3			IMX_SC_R_AP_0_3
-+#define IMX_SC_R_A72			IMX_SC_R_AP_1
-+#define IMX_SC_R_A72_0			IMX_SC_R_AP_1_0
-+#define IMX_SC_R_A72_1			IMX_SC_R_AP_1_1
-+#define IMX_SC_R_A72_2			IMX_SC_R_AP_1_2
-+#define IMX_SC_R_A72_3			IMX_SC_R_AP_1_3
-+#define IMX_SC_R_GIC			IMX_SC_R_GIC_0
-+#define IMX_SC_R_HSIO_GPIO		IMX_SC_R_HSIO_GPIO_0
-+#define IMX_SC_R_IEE			IMX_SC_R_IEE_0
-+#define IMX_SC_R_IEE_R0			IMX_SC_R_IEE_0_R0
-+#define IMX_SC_R_IEE_R1			IMX_SC_R_IEE_0_R1
-+#define IMX_SC_R_IEE_R2			IMX_SC_R_IEE_0_R2
-+#define IMX_SC_R_IEE_R3			IMX_SC_R_IEE_0_R3
-+#define IMX_SC_R_IEE_R4			IMX_SC_R_IEE_0_R4
-+#define IMX_SC_R_IEE_R5			IMX_SC_R_IEE_0_R5
-+#define IMX_SC_R_IEE_R6			IMX_SC_R_IEE_0_R6
-+#define IMX_SC_R_IEE_R7			IMX_SC_R_IEE_0_R7
-+#define IMX_SC_R_IRQSTR_M4_0		IMX_SC_R_IRQSTR_MCU_0
-+#define IMX_SC_R_IRQSTR_M4_1		IMX_SC_R_IRQSTR_MCU_1
-+#define IMX_SC_R_IRQSTR_SCU2		IMX_SC_R_IRQSTR_AP_0
-+#define IMX_SC_R_ISI_CH0		IMX_SC_R_ISI_0_CH0
-+#define IMX_SC_R_ISI_CH1		IMX_SC_R_ISI_0_CH1
-+#define IMX_SC_R_ISI_CH2		IMX_SC_R_ISI_0_CH2
-+#define IMX_SC_R_ISI_CH3		IMX_SC_R_ISI_0_CH3
-+#define IMX_SC_R_ISI_CH4		IMX_SC_R_ISI_0_CH4
-+#define IMX_SC_R_ISI_CH5		IMX_SC_R_ISI_0_CH5
-+#define IMX_SC_R_ISI_CH6		IMX_SC_R_ISI_0_CH6
-+#define IMX_SC_R_ISI_CH7		IMX_SC_R_ISI_0_CH7
-+#define IMX_SC_R_M4_0_I2C		IMX_SC_R_MCU_0_I2C
-+#define IMX_SC_R_M4_0_INTMUX		IMX_SC_R_MCU_0_INTMUX
-+#define IMX_SC_R_M4_0_MU_0A0		IMX_SC_R_MCU_0_MU_0A0
-+#define IMX_SC_R_M4_0_MU_0A1		IMX_SC_R_MCU_0_MU_0A1
-+#define IMX_SC_R_M4_0_MU_0A2		IMX_SC_R_MCU_0_MU_0A2
-+#define IMX_SC_R_M4_0_MU_0A3		IMX_SC_R_MCU_0_MU_0A3
-+#define IMX_SC_R_M4_0_MU_0B		IMX_SC_R_MCU_0_MU_0B
-+#define IMX_SC_R_M4_0_MU_1A		IMX_SC_R_MCU_0_MU_1A
-+#define IMX_SC_R_M4_0_PID0		IMX_SC_R_MCU_0_PID0
-+#define IMX_SC_R_M4_0_PID1		IMX_SC_R_MCU_0_PID1
-+#define IMX_SC_R_M4_0_PID2		IMX_SC_R_MCU_0_PID2
-+#define IMX_SC_R_M4_0_PID3		IMX_SC_R_MCU_0_PID3
-+#define IMX_SC_R_M4_0_PID4		IMX_SC_R_MCU_0_PID4
-+#define IMX_SC_R_M4_0_PIT		IMX_SC_R_MCU_0_PIT
-+#define IMX_SC_R_M4_0_RGPIO		IMX_SC_R_MCU_0_RGPIO
-+#define IMX_SC_R_M4_0_SEMA42		IMX_SC_R_MCU_0_SEMA42
-+#define IMX_SC_R_M4_0_TPM		IMX_SC_R_MCU_0_TPM
-+#define IMX_SC_R_M4_0_UART		IMX_SC_R_MCU_0_UART
-+#define IMX_SC_R_M4_1_I2C		IMX_SC_R_MCU_1_I2C
-+#define IMX_SC_R_M4_1_INTMUX		IMX_SC_R_MCU_1_INTMUX
-+#define IMX_SC_R_M4_1_MU_0A0		IMX_SC_R_MCU_1_MU_0A0
-+#define IMX_SC_R_M4_1_MU_0A1		IMX_SC_R_MCU_1_MU_0A1
-+#define IMX_SC_R_M4_1_MU_0A2		IMX_SC_R_MCU_1_MU_0A2
-+#define IMX_SC_R_M4_1_MU_0A3		IMX_SC_R_MCU_1_MU_0A3
-+#define IMX_SC_R_M4_1_MU_0B		IMX_SC_R_MCU_1_MU_0B
-+#define IMX_SC_R_M4_1_MU_1A		IMX_SC_R_MCU_1_MU_1A
-+#define IMX_SC_R_M4_1_PID0		IMX_SC_R_MCU_1_PID0
-+#define IMX_SC_R_M4_1_PID1		IMX_SC_R_MCU_1_PID1
-+#define IMX_SC_R_M4_1_PID2		IMX_SC_R_MCU_1_PID2
-+#define IMX_SC_R_M4_1_PID3		IMX_SC_R_MCU_1_PID3
-+#define IMX_SC_R_M4_1_PID4		IMX_SC_R_MCU_1_PID4
-+#define IMX_SC_R_M4_1_PIT		IMX_SC_R_MCU_1_PIT
-+#define IMX_SC_R_M4_1_RGPIO		IMX_SC_R_MCU_1_RGPIO
-+#define IMX_SC_R_M4_1_SEMA42		IMX_SC_R_MCU_1_SEMA42
-+#define IMX_SC_R_M4_1_TPM		IMX_SC_R_MCU_1_TPM
-+#define IMX_SC_R_M4_1_UART		IMX_SC_R_MCU_1_UART
-+#define IMX_SC_R_MJPEG_DEC_MP		IMX_SC_R_MJPEG_0_DEC_MP
-+#define IMX_SC_R_MJPEG_DEC_S0		IMX_SC_R_MJPEG_0_DEC_S0
-+#define IMX_SC_R_MJPEG_DEC_S1		IMX_SC_R_MJPEG_0_DEC_S1
-+#define IMX_SC_R_MJPEG_DEC_S2		IMX_SC_R_MJPEG_0_DEC_S2
-+#define IMX_SC_R_MJPEG_DEC_S3		IMX_SC_R_MJPEG_0_DEC_S3
-+#define IMX_SC_R_MJPEG_ENC_MP		IMX_SC_R_MJPEG_0_ENC_MP
-+#define IMX_SC_R_MJPEG_ENC_S0		IMX_SC_R_MJPEG_0_ENC_S0
-+#define IMX_SC_R_MJPEG_ENC_S1		IMX_SC_R_MJPEG_0_ENC_S1
-+#define IMX_SC_R_MJPEG_ENC_S2		IMX_SC_R_MJPEG_0_ENC_S2
-+#define IMX_SC_R_MJPEG_ENC_S3		IMX_SC_R_MJPEG_0_ENC_S3
-+#define IMX_SC_R_PERF			IMX_SC_R_PERF_0
-+#define IMX_SC_R_SMMU			IMX_SC_R_SMMU_0
-+#define IMX_SC_R_VPU_UART		IMX_SC_R_ENET_0_A2
-+#define IMX_SC_R_VPUCORE		IMX_SC_R_ENET_1_A0
-+#define IMX_SC_R_VPUCORE_0		IMX_SC_R_ENET_1_A1
-+#define IMX_SC_R_VPUCORE_1		IMX_SC_R_ENET_1_A2
-+#define IMX_SC_R_VPUCORE_2		IMX_SC_R_ENET_1_A3
-+#define IMX_SC_R_VPUCORE_3		IMX_SC_R_ENET_1_A4
-+#define IMX_SC_R_UNUSED1		IMX_SC_R_V2X_PID0
-+#define IMX_SC_R_UNUSED2		IMX_SC_R_V2X_PID1
-+#define IMX_SC_R_UNUSED3		IMX_SC_R_V2X_PID2
-+#define IMX_SC_R_UNUSED4		IMX_SC_R_V2X_PID3
-+
-+/*
-  * Defines for SC CONTROL
-  */
- #define IMX_SC_C_TEMP				0
-@@ -630,6 +743,10 @@
- #define IMX_SC_C_INTF_SEL			59
- #define IMX_SC_C_RXC_DLY			60
- #define IMX_SC_C_TIMER_SEL			61
-#define IMX_SC_C_LAST				62
-+#define IMX_SC_C_MISC0				62
-+#define IMX_SC_C_MISC1				63
-+#define IMX_SC_C_MISC2				64
-+#define IMX_SC_C_MISC3				65
-+#define IMX_SC_C_LAST				66
- 
- #endif /* __DT_BINDINGS_RSCRC_IMX_H */
-diff --git a/include/dt-bindings/firmware/qcom,scm.h b/include/dt-bindings/firmware/qcom,scm.h
-new file mode 100644
-index 000000000000..6de8b08e1e79
--- /dev/null
-+++ b/include/dt-bindings/firmware/qcom,scm.h
-@@ -0,0 +1,39 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (c) 2010-2015, 2018-2019 The Linux Foundation. All rights reserved.
-+ * Copyright (C) 2015 Linaro Ltd.
-+ * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_FIRMWARE_QCOM_SCM_H
-+#define _DT_BINDINGS_FIRMWARE_QCOM_SCM_H
-+
-+#define QCOM_SCM_VMID_TZ		0x1
-+#define QCOM_SCM_VMID_HLOS		0x3
-+#define QCOM_SCM_VMID_SSC_Q6		0x5
-+#define QCOM_SCM_VMID_ADSP_Q6		0x6
-+#define QCOM_SCM_VMID_CP_TOUCH		0x8
-+#define QCOM_SCM_VMID_CP_BITSTREAM	0x9
-+#define QCOM_SCM_VMID_CP_PIXEL		0xA
-+#define QCOM_SCM_VMID_CP_NON_PIXEL	0xB
-+#define QCOM_SCM_VMID_CP_CAMERA		0xD
-+#define QCOM_SCM_VMID_HLOS_FREE		0xE
-+#define QCOM_SCM_VMID_MSS_MSA		0xF
-+#define QCOM_SCM_VMID_MSS_NONMSA	0x10
-+#define QCOM_SCM_VMID_CP_SEC_DISPLAY	0x11
-+#define QCOM_SCM_VMID_CP_APP		0x12
-+#define QCOM_SCM_VMID_LPASS		0x16
-+#define QCOM_SCM_VMID_WLAN		0x18
-+#define QCOM_SCM_VMID_WLAN_CE		0x19
-+#define QCOM_SCM_VMID_CP_SPSS_SP	0x1A
-+#define QCOM_SCM_VMID_CP_CAMERA_PREVIEW 0x1D
-+#define QCOM_SCM_VMID_CDSP		0x1E
-+#define QCOM_SCM_VMID_CP_SPSS_SP_SHARED 0x22
-+#define QCOM_SCM_VMID_CP_SPSS_HLOS_SHARED 0x24
-+#define QCOM_SCM_VMID_ADSP_HEAP		0x25
-+#define QCOM_SCM_VMID_CP_CDSP		0x2A
-+#define QCOM_SCM_VMID_NAV		0x2B
-+#define QCOM_SCM_VMID_TVM		0x2D
-+#define QCOM_SCM_VMID_OEMVM		0x31
-+
-+#endif
-diff --git a/include/dt-bindings/gce/mediatek,mt6795-gce.h b/include/dt-bindings/gce/mediatek,mt6795-gce.h
-new file mode 100644
-index 000000000000..97d5ba2d2b44
--- /dev/null
-+++ b/include/dt-bindings/gce/mediatek,mt6795-gce.h
-@@ -0,0 +1,123 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023 Collabora Ltd.
-+ * Author: AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
-+ */
-+#ifndef _DT_BINDINGS_GCE_MT6795_H
-+#define _DT_BINDINGS_GCE_MT6795_H
-+
-+/* GCE HW thread priority */
-+#define CMDQ_THR_PRIO_LOWEST			0
-+#define CMDQ_THR_PRIO_NORMAL			1
-+#define CMDQ_THR_PRIO_NORMAL_2			2
-+#define CMDQ_THR_PRIO_MEDIUM			3
-+#define CMDQ_THR_PRIO_MEDIUM_2			4
-+#define CMDQ_THR_PRIO_HIGH			5
-+#define CMDQ_THR_PRIO_HIGHER			6
-+#define CMDQ_THR_PRIO_HIGHEST			7
-+
-+/* GCE SUBSYS */
-+#define SUBSYS_1300XXXX				0
-+#define SUBSYS_1400XXXX				1
-+#define SUBSYS_1401XXXX				2
-+#define SUBSYS_1402XXXX				3
-+#define SUBSYS_1500XXXX				4
-+#define SUBSYS_1600XXXX				5
-+#define SUBSYS_1700XXXX				6
-+#define SUBSYS_1800XXXX				7
-+#define SUBSYS_1000XXXX				8
-+#define SUBSYS_1001XXXX				9
-+#define SUBSYS_1002XXXX				10
-+#define SUBSYS_1003XXXX				11
-+#define SUBSYS_1004XXXX				12
-+#define SUBSYS_1005XXXX				13
-+#define SUBSYS_1020XXXX				14
-+#define SUBSYS_1021XXXX				15
-+#define SUBSYS_1120XXXX				16
-+#define SUBSYS_1121XXXX				17
-+#define SUBSYS_1122XXXX				18
-+#define SUBSYS_1123XXXX				19
-+#define SUBSYS_1124XXXX				20
-+#define SUBSYS_1125XXXX				21
-+#define SUBSYS_1126XXXX				22
-+
-+/* GCE HW EVENT */
-+#define CMDQ_EVENT_MDP_RDMA0_SOF		0
-+#define CMDQ_EVENT_MDP_RDMA1_SOF		1
-+#define CMDQ_EVENT_MDP_DSI0_TE_SOF		2
-+#define CMDQ_EVENT_MDP_DSI1_TE_SOF		3
-+#define CMDQ_EVENT_MDP_MVW_SOF			4
-+#define CMDQ_EVENT_MDP_TDSHP0_SOF		5
-+#define CMDQ_EVENT_MDP_TDSHP1_SOF		6
-+#define CMDQ_EVENT_MDP_WDMA_SOF			7
-+#define CMDQ_EVENT_MDP_WROT0_SOF		8
-+#define CMDQ_EVENT_MDP_WROT1_SOF		9
-+#define CMDQ_EVENT_MDP_CROP_SOF			10
-+#define CMDQ_EVENT_DISP_OVL0_SOF		11
-+#define CMDQ_EVENT_DISP_OVL1_SOF		12
-+#define CMDQ_EVENT_DISP_RDMA0_SOF		13
-+#define CMDQ_EVENT_DISP_RDMA1_SOF		14
-+#define CMDQ_EVENT_DISP_RDMA2_SOF		15
-+#define CMDQ_EVENT_DISP_WDMA0_SOF		16
-+#define CMDQ_EVENT_DISP_WDMA1_SOF		17
-+#define CMDQ_EVENT_DISP_COLOR0_SOF		18
-+#define CMDQ_EVENT_DISP_COLOR1_SOF		19
-+#define CMDQ_EVENT_DISP_AAL_SOF			20
-+#define CMDQ_EVENT_DISP_GAMMA_SOF		21
-+#define CMDQ_EVENT_DISP_UFOE_SOF		22
-+#define CMDQ_EVENT_DISP_PWM0_SOF		23
-+#define CMDQ_EVENT_DISP_PWM1_SOF		24
-+#define CMDQ_EVENT_DISP_OD_SOF			25
-+#define CMDQ_EVENT_MDP_RDMA0_EOF		26
-+#define CMDQ_EVENT_MDP_RDMA1_EOF		27
-+#define CMDQ_EVENT_MDP_RSZ0_EOF			28
-+#define CMDQ_EVENT_MDP_RSZ1_EOF			29
-+#define CMDQ_EVENT_MDP_RSZ2_EOF			30
-+#define CMDQ_EVENT_MDP_TDSHP0_EOF		31
-+#define CMDQ_EVENT_MDP_TDSHP1_EOF		32
-+#define CMDQ_EVENT_MDP_WDMA_EOF			33
-+#define CMDQ_EVENT_MDP_WROT0_WRITE_EOF		34
-+#define CMDQ_EVENT_MDP_WROT0_READ_EOF		35
-+#define CMDQ_EVENT_MDP_WROT1_WRITE_EOF		36
-+#define CMDQ_EVENT_MDP_WROT1_READ_EOF		37
-+#define CMDQ_EVENT_MDP_CROP_EOF			38
-+#define CMDQ_EVENT_DISP_OVL0_EOF		39
-+#define CMDQ_EVENT_DISP_OVL1_EOF		40
-+#define CMDQ_EVENT_DISP_RDMA0_EOF		41
-+#define CMDQ_EVENT_DISP_RDMA1_EOF		42
-+#define CMDQ_EVENT_DISP_RDMA2_EOF		43
-+#define CMDQ_EVENT_DISP_WDMA0_EOF		44
-+#define CMDQ_EVENT_DISP_WDMA1_EOF		45
-+#define CMDQ_EVENT_DISP_COLOR0_EOF		46
-+#define CMDQ_EVENT_DISP_COLOR1_EOF		47
-+#define CMDQ_EVENT_DISP_AAL_EOF			48
-+#define CMDQ_EVENT_DISP_GAMMA_EOF		49
-+#define CMDQ_EVENT_DISP_UFOE_EOF		50
-+#define CMDQ_EVENT_DISP_DPI0_EOF		51
-+#define CMDQ_EVENT_MUTEX0_STREAM_EOF		52
-+#define CMDQ_EVENT_MUTEX1_STREAM_EOF		53
-+#define CMDQ_EVENT_MUTEX2_STREAM_EOF		54
-+#define CMDQ_EVENT_MUTEX3_STREAM_EOF		55
-+#define CMDQ_EVENT_MUTEX4_STREAM_EOF		56
-+#define CMDQ_EVENT_MUTEX5_STREAM_EOF		57
-+#define CMDQ_EVENT_MUTEX6_STREAM_EOF		58
-+#define CMDQ_EVENT_MUTEX7_STREAM_EOF		59
-+#define CMDQ_EVENT_MUTEX8_STREAM_EOF		60
-+#define CMDQ_EVENT_MUTEX9_STREAM_EOF		61
-+#define CMDQ_EVENT_DISP_RDMA0_UNDERRUN		62
-+#define CMDQ_EVENT_DISP_RDMA1_UNDERRUN		63
-+#define CMDQ_EVENT_DISP_RDMA2_UNDERRUN		64
-+#define CMDQ_EVENT_ISP_PASS2_2_EOF		129
-+#define CMDQ_EVENT_ISP_PASS2_1_EOF		130
-+#define CMDQ_EVENT_ISP_PASS2_0_EOF		131
-+#define CMDQ_EVENT_ISP_PASS1_1_EOF		132
-+#define CMDQ_EVENT_ISP_PASS1_0_EOF		133
-+#define CMDQ_EVENT_CAMSV_2_PASS1_EOF		134
-+#define CMDQ_EVENT_CAMSV_1_PASS1_EOF		135
-+#define CMDQ_EVENT_SENINF_CAM1_2_3_FIFO_FULL	136
-+#define CMDQ_EVENT_SENINF_CAM0_FIFO_FULL	137
-+#define CMDQ_EVENT_JPGENC_PASS2_EOF		257
-+#define CMDQ_EVENT_JPGENC_PASS1_EOF		258
-+#define CMDQ_EVENT_JPGDEC_EOF			259
-+
-+#endif
-diff --git a/include/dt-bindings/gpio/amlogic,t7-periphs-pinctrl.h b/include/dt-bindings/gpio/amlogic,t7-periphs-pinctrl.h
-new file mode 100644
-index 000000000000..4e16d31a71c9
--- /dev/null
-+++ b/include/dt-bindings/gpio/amlogic,t7-periphs-pinctrl.h
-@@ -0,0 +1,179 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (c) 2023 Amlogic, Inc. All rights reserved.
-+ * Author: Huqiang Qin <huqiang.qin@amlogic.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_AMLOGIC_T7_GPIO_H
-+#define _DT_BINDINGS_AMLOGIC_T7_GPIO_H
-+
-+#define GPIOB_0		0
-+#define GPIOB_1		1
-+#define GPIOB_2		2
-+#define GPIOB_3		3
-+#define GPIOB_4		4
-+#define GPIOB_5		5
-+#define GPIOB_6		6
-+#define GPIOB_7		7
-+#define GPIOB_8		8
-+#define GPIOB_9		9
-+#define GPIOB_10	10
-+#define GPIOB_11	11
-+#define GPIOB_12	12
-+
-+#define GPIOC_0		13
-+#define GPIOC_1		14
-+#define GPIOC_2		15
-+#define GPIOC_3		16
-+#define GPIOC_4		17
-+#define GPIOC_5		18
-+#define GPIOC_6		19
-+
-+#define GPIOX_0		20
-+#define GPIOX_1		21
-+#define GPIOX_2		22
-+#define GPIOX_3		23
-+#define GPIOX_4		24
-+#define GPIOX_5		25
-+#define GPIOX_6		26
-+#define GPIOX_7		27
-+#define GPIOX_8		28
-+#define GPIOX_9		29
-+#define GPIOX_10	30
-+#define GPIOX_11	31
-+#define GPIOX_12	32
-+#define GPIOX_13	33
-+#define GPIOX_14	34
-+#define GPIOX_15	35
-+#define GPIOX_16	36
-+#define GPIOX_17	37
-+#define GPIOX_18	38
-+#define GPIOX_19	39
-+
-+#define GPIOW_0		40
-+#define GPIOW_1		41
-+#define GPIOW_2		42
-+#define GPIOW_3		43
-+#define GPIOW_4		44
-+#define GPIOW_5		45
-+#define GPIOW_6		46
-+#define GPIOW_7		47
-+#define GPIOW_8		48
-+#define GPIOW_9		49
-+#define GPIOW_10	50
-+#define GPIOW_11	51
-+#define GPIOW_12	52
-+#define GPIOW_13	53
-+#define GPIOW_14	54
-+#define GPIOW_15	55
-+#define GPIOW_16	56
-+
-+#define GPIOD_0		57
-+#define GPIOD_1		58
-+#define GPIOD_2		59
-+#define GPIOD_3		60
-+#define GPIOD_4		61
-+#define GPIOD_5		62
-+#define GPIOD_6		63
-+#define GPIOD_7		64
-+#define GPIOD_8		65
-+#define GPIOD_9		66
-+#define GPIOD_10	67
-+#define GPIOD_11	68
-+#define GPIOD_12	69
-+
-+#define GPIOE_0		70
-+#define GPIOE_1		71
-+#define GPIOE_2		72
-+#define GPIOE_3		73
-+#define GPIOE_4		74
-+#define GPIOE_5		75
-+#define GPIOE_6		76
-+
-+#define GPIOZ_0		77
-+#define GPIOZ_1		78
-+#define GPIOZ_2		79
-+#define GPIOZ_3		80
-+#define GPIOZ_4		81
-+#define GPIOZ_5		82
-+#define GPIOZ_6		83
-+#define GPIOZ_7		84
-+#define GPIOZ_8		85
-+#define GPIOZ_9		86
-+#define GPIOZ_10	87
-+#define GPIOZ_11	88
-+#define GPIOZ_12	89
-+#define GPIOZ_13	90
-+
-+#define GPIOT_0		91
-+#define GPIOT_1		92
-+#define GPIOT_2		93
-+#define GPIOT_3		94
-+#define GPIOT_4		95
-+#define GPIOT_5		96
-+#define GPIOT_6		97
-+#define GPIOT_7		98
-+#define GPIOT_8		99
-+#define GPIOT_9		100
-+#define GPIOT_10	101
-+#define GPIOT_11	102
-+#define GPIOT_12	103
-+#define GPIOT_13	104
-+#define GPIOT_14	105
-+#define GPIOT_15	106
-+#define GPIOT_16	107
-+#define GPIOT_17	108
-+#define GPIOT_18	109
-+#define GPIOT_19	110
-+#define GPIOT_20	111
-+#define GPIOT_21	112
-+#define GPIOT_22	113
-+#define GPIOT_23	114
-+
-+#define GPIOM_0		115
-+#define GPIOM_1		116
-+#define GPIOM_2		117
-+#define GPIOM_3		118
-+#define GPIOM_4		119
-+#define GPIOM_5		120
-+#define GPIOM_6		121
-+#define GPIOM_7		122
-+#define GPIOM_8		123
-+#define GPIOM_9		124
-+#define GPIOM_10	125
-+#define GPIOM_11	126
-+#define GPIOM_12	127
-+#define GPIOM_13	128
-+
-+#define GPIOY_0		129
-+#define GPIOY_1		130
-+#define GPIOY_2		131
-+#define GPIOY_3		132
-+#define GPIOY_4		133
-+#define GPIOY_5		134
-+#define GPIOY_6		135
-+#define GPIOY_7		136
-+#define GPIOY_8		137
-+#define GPIOY_9		138
-+#define GPIOY_10	139
-+#define GPIOY_11	140
-+#define GPIOY_12	141
-+#define GPIOY_13	142
-+#define GPIOY_14	143
-+#define GPIOY_15	144
-+#define GPIOY_16	145
-+#define GPIOY_17	146
-+#define GPIOY_18	147
-+
-+#define GPIOH_0		148
-+#define GPIOH_1		149
-+#define GPIOH_2		150
-+#define GPIOH_3		151
-+#define GPIOH_4		152
-+#define GPIOH_5		153
-+#define GPIOH_6		154
-+#define GPIOH_7		155
-+
-+#define GPIO_TEST_N	156
-+
-+#endif /* _DT_BINDINGS_AMLOGIC_T7_GPIO_H */
-diff --git a/include/dt-bindings/gpio/amlogic-c3-gpio.h b/include/dt-bindings/gpio/amlogic-c3-gpio.h
-new file mode 100644
-index 000000000000..75c8da6f505f
--- /dev/null
-+++ b/include/dt-bindings/gpio/amlogic-c3-gpio.h
-@@ -0,0 +1,72 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (c) 2021 Amlogic, Inc. All rights reserved.
-+ * Author: Huqiang Qin <huqiang.qin@amlogic.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_AMLOGIC_C3_GPIO_H
-+#define _DT_BINDINGS_AMLOGIC_C3_GPIO_H
-+
-+#define	GPIOE_0		0
-+#define	GPIOE_1		1
-+#define	GPIOE_2		2
-+#define	GPIOE_3		3
-+#define	GPIOE_4		4
-+
-+#define	GPIOB_0		5
-+#define	GPIOB_1		6
-+#define	GPIOB_2		7
-+#define	GPIOB_3		8
-+#define	GPIOB_4		9
-+#define	GPIOB_5		10
-+#define	GPIOB_6		11
-+#define	GPIOB_7		12
-+#define	GPIOB_8		13
-+#define	GPIOB_9		14
-+#define	GPIOB_10	15
-+#define	GPIOB_11	16
-+#define	GPIOB_12	17
-+#define	GPIOB_13	18
-+#define	GPIOB_14	19
-+
-+#define	GPIOC_0		20
-+#define	GPIOC_1		21
-+#define	GPIOC_2		22
-+#define	GPIOC_3		23
-+#define	GPIOC_4		24
-+#define	GPIOC_5		25
-+#define	GPIOC_6		26
-+
-+#define	GPIOX_0		27
-+#define	GPIOX_1		28
-+#define	GPIOX_2		29
-+#define	GPIOX_3		30
-+#define	GPIOX_4		31
-+#define	GPIOX_5		32
-+#define	GPIOX_6		33
-+#define	GPIOX_7		34
-+#define	GPIOX_8		35
-+#define	GPIOX_9		36
-+#define	GPIOX_10	37
-+#define	GPIOX_11	38
-+#define	GPIOX_12	39
-+#define	GPIOX_13	40
-+
-+#define	GPIOD_0		41
-+#define	GPIOD_1		42
-+#define	GPIOD_2		43
-+#define	GPIOD_3		44
-+#define	GPIOD_4		45
-+#define	GPIOD_5		46
-+#define	GPIOD_6		47
-+
-+#define	GPIOA_0		48
-+#define	GPIOA_1		49
-+#define	GPIOA_2		50
-+#define	GPIOA_3		51
-+#define	GPIOA_4		52
-+#define	GPIOA_5		53
-+
-+#define	GPIO_TEST_N	54
-+
-+#endif /* _DT_BINDINGS_AMLOGIC_C3_GPIO_H */
-diff --git a/include/dt-bindings/gpio/gpio.h b/include/dt-bindings/gpio/gpio.h
-index 5566e58196a2..b5d531237448 100644
--- a/include/dt-bindings/gpio/gpio.h
-+++ b/include/dt-bindings/gpio/gpio.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
- /*
-  * This header provides constants for most GPIO bindings.
-  *
-diff --git a/include/dt-bindings/gpio/meson-g12a-gpio.h b/include/dt-bindings/gpio/meson-g12a-gpio.h
-index f7bd69350d18..fa7bb0bbf010 100644
--- a/include/dt-bindings/gpio/meson-g12a-gpio.h
-+++ b/include/dt-bindings/gpio/meson-g12a-gpio.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (c) 2018 Amlogic, Inc. All rights reserved.
-  * Author: Xingyu Chen <xingyu.chen@amlogic.com>
-diff --git a/include/dt-bindings/gpio/tegra234-gpio.h b/include/dt-bindings/gpio/tegra234-gpio.h
-index d7a1f2e298e8..784673c2c752 100644
--- a/include/dt-bindings/gpio/tegra234-gpio.h
-+++ b/include/dt-bindings/gpio/tegra234-gpio.h
-@@ -33,18 +33,14 @@
- #define TEGRA234_MAIN_GPIO_PORT_P  14
- #define TEGRA234_MAIN_GPIO_PORT_Q  15
- #define TEGRA234_MAIN_GPIO_PORT_R  16
-#define TEGRA234_MAIN_GPIO_PORT_S  17
-#define TEGRA234_MAIN_GPIO_PORT_T  18
-#define TEGRA234_MAIN_GPIO_PORT_U  19
-#define TEGRA234_MAIN_GPIO_PORT_V  20
-#define TEGRA234_MAIN_GPIO_PORT_X  21
-#define TEGRA234_MAIN_GPIO_PORT_Y  22
-#define TEGRA234_MAIN_GPIO_PORT_Z  23
-#define TEGRA234_MAIN_GPIO_PORT_AC 24
-#define TEGRA234_MAIN_GPIO_PORT_AD 25
-#define TEGRA234_MAIN_GPIO_PORT_AE 26
-#define TEGRA234_MAIN_GPIO_PORT_AF 27
-#define TEGRA234_MAIN_GPIO_PORT_AG 28
-+#define TEGRA234_MAIN_GPIO_PORT_X  17
-+#define TEGRA234_MAIN_GPIO_PORT_Y  18
-+#define TEGRA234_MAIN_GPIO_PORT_Z  19
-+#define TEGRA234_MAIN_GPIO_PORT_AC 20
-+#define TEGRA234_MAIN_GPIO_PORT_AD 21
-+#define TEGRA234_MAIN_GPIO_PORT_AE 22
-+#define TEGRA234_MAIN_GPIO_PORT_AF 23
-+#define TEGRA234_MAIN_GPIO_PORT_AG 24
- 
- #define TEGRA234_MAIN_GPIO(port, offset) \
- 	((TEGRA234_MAIN_GPIO_PORT_##port * 8) + offset)
-diff --git a/include/dt-bindings/gpio/tegra256-gpio.h b/include/dt-bindings/gpio/tegra256-gpio.h
-new file mode 100644
-index 000000000000..a0353a302aeb
--- /dev/null
-+++ b/include/dt-bindings/gpio/tegra256-gpio.h
-@@ -0,0 +1,28 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/* Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. */
-+
-+/*
-+ * This header provides constants for the nvidia,tegra256-gpio DT binding.
-+ *
-+ * The first cell in Tegra's GPIO specifier is the GPIO ID.
-+ * The macros below provide names for this.
-+ *
-+ * The second cell contains standard flag values specified in gpio.h.
-+ */
-+
-+#ifndef _DT_BINDINGS_GPIO_TEGRA256_GPIO_H
-+#define _DT_BINDINGS_GPIO_TEGRA256_GPIO_H
-+
-+#include <dt-bindings/gpio/gpio.h>
-+
-+/* GPIOs implemented by main GPIO controller */
-+#define TEGRA256_MAIN_GPIO_PORT_A 0
-+#define TEGRA256_MAIN_GPIO_PORT_B 1
-+#define TEGRA256_MAIN_GPIO_PORT_C 2
-+#define TEGRA256_MAIN_GPIO_PORT_D 3
-+
-+#define TEGRA256_MAIN_GPIO(port, offset) \
-+		((TEGRA256_MAIN_GPIO_PORT_##port * 8) + (offset))
-+
-+#endif
-+
-diff --git a/include/dt-bindings/i3c/i3c.h b/include/dt-bindings/i3c/i3c.h
-new file mode 100644
-index 000000000000..373439218bba
--- /dev/null
-+++ b/include/dt-bindings/i3c/i3c.h
-@@ -0,0 +1,16 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright 2024 NXP
-+ */
-+
-+#ifndef _DT_BINDINGS_I3C_I3C_H
-+#define _DT_BINDINGS_I3C_I3C_H
-+
-+#define I2C_FM      (1 << 4)
-+#define I2C_FM_PLUS (0 << 4)
-+
-+#define I2C_FILTER  (0 << 5)
-+#define I2C_NO_FILTER_HIGH_FREQUENCY    (1 << 5)
-+#define I2C_NO_FILTER_LOW_FREQUENCY     (2 << 5)
-+
-+#endif
-diff --git a/include/dt-bindings/iio/adc/adi,ad4695.h b/include/dt-bindings/iio/adc/adi,ad4695.h
-new file mode 100644
-index 000000000000..fea4525d2710
--- /dev/null
-+++ b/include/dt-bindings/iio/adc/adi,ad4695.h
-@@ -0,0 +1,16 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_ADI_AD4695_H
-+#define _DT_BINDINGS_ADI_AD4695_H
-+
-+#define AD4695_COMMON_MODE_REFGND	0xFF
-+#define AD4695_COMMON_MODE_COM		0xFE
-+
-+#define AD4695_TRIGGER_EVENT_BUSY	0
-+#define AD4695_TRIGGER_EVENT_ALERT	1
-+
-+#define AD4695_TRIGGER_PIN_GP0		0
-+#define AD4695_TRIGGER_PIN_GP2		2
-+#define AD4695_TRIGGER_PIN_GP3		3
-+
-+#endif /* _DT_BINDINGS_ADI_AD4695_H */
-diff --git a/include/dt-bindings/iio/adc/adi,ad7606.h b/include/dt-bindings/iio/adc/adi,ad7606.h
-new file mode 100644
-index 000000000000..f38a6d72b6dc
--- /dev/null
-+++ b/include/dt-bindings/iio/adc/adi,ad7606.h
-@@ -0,0 +1,9 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_ADI_AD7606_H
-+#define _DT_BINDINGS_ADI_AD7606_H
-+
-+#define AD7606_TRIGGER_EVENT_BUSY	0
-+#define AD7606_TRIGGER_EVENT_FRSTDATA	1
-+
-+#endif /* _DT_BINDINGS_ADI_AD7606_H */
-diff --git a/include/dt-bindings/iio/adc/adi,ad7768-1.h b/include/dt-bindings/iio/adc/adi,ad7768-1.h
-new file mode 100644
-index 000000000000..34d92856a50b
--- /dev/null
-+++ b/include/dt-bindings/iio/adc/adi,ad7768-1.h
-@@ -0,0 +1,10 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_ADI_AD7768_1_H
-+#define _DT_BINDINGS_ADI_AD7768_1_H
-+
-+#define AD7768_TRIGGER_SOURCE_SYNC_OUT  0
-+#define AD7768_TRIGGER_SOURCE_GPIO3     1
-+#define AD7768_TRIGGER_SOURCE_DRDY      2
-+
-+#endif /* _DT_BINDINGS_ADI_AD7768_1_H */
-diff --git a/include/dt-bindings/iio/adc/at91-sama5d2_adc.h b/include/dt-bindings/iio/adc/at91-sama5d2_adc.h
-index 70f99dbdbb42..866d36530583 100644
--- a/include/dt-bindings/iio/adc/at91-sama5d2_adc.h
-+++ b/include/dt-bindings/iio/adc/at91-sama5d2_adc.h
-@@ -13,4 +13,7 @@
- /* pressure channel index */
- #define AT91_SAMA5D2_ADC_P_CHANNEL		26
- 
-+/* SAMA7G5 Temperature sensor channel index. */
-+#define AT91_SAMA7G5_ADC_TEMP_CHANNEL		31
-+
- #endif
-diff --git a/include/dt-bindings/iio/adc/gehc,pmc-adc.h b/include/dt-bindings/iio/adc/gehc,pmc-adc.h
-new file mode 100644
-index 000000000000..2f291e3c76ae
--- /dev/null
-+++ b/include/dt-bindings/iio/adc/gehc,pmc-adc.h
-@@ -0,0 +1,10 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+
-+#ifndef _DT_BINDINGS_IIO_ADC_GEHC_PMC_ADC_H
-+#define _DT_BINDINGS_IIO_ADC_GEHC_PMC_ADC_H
-+
-+/* ADC channel type */
-+#define GEHC_PMC_ADC_VOLTAGE	0
-+#define GEHC_PMC_ADC_CURRENT	1
-+
-+#endif
-diff --git a/include/dt-bindings/iio/adc/mediatek,mt6357-auxadc.h b/include/dt-bindings/iio/adc/mediatek,mt6357-auxadc.h
-new file mode 100644
-index 000000000000..03ebb1d23953
--- /dev/null
-+++ b/include/dt-bindings/iio/adc/mediatek,mt6357-auxadc.h
-@@ -0,0 +1,21 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+
-+#ifndef _DT_BINDINGS_MEDIATEK_MT6357_AUXADC_H
-+#define _DT_BINDINGS_MEDIATEK_MT6357_AUXADC_H
-+
-+/* ADC Channel Index */
-+#define MT6357_AUXADC_BATADC		0
-+#define MT6357_AUXADC_ISENSE		1
-+#define MT6357_AUXADC_VCDT		2
-+#define MT6357_AUXADC_BAT_TEMP		3
-+#define MT6357_AUXADC_CHIP_TEMP		4
-+#define MT6357_AUXADC_ACCDET		5
-+#define MT6357_AUXADC_VDCXO		6
-+#define MT6357_AUXADC_TSX_TEMP		7
-+#define MT6357_AUXADC_HPOFS_CAL		8
-+#define MT6357_AUXADC_DCXO_TEMP		9
-+#define MT6357_AUXADC_VCORE_TEMP	10
-+#define MT6357_AUXADC_VPROC_TEMP	11
-+#define MT6357_AUXADC_VBAT		12
-+
-+#endif
-diff --git a/include/dt-bindings/iio/adc/mediatek,mt6358-auxadc.h b/include/dt-bindings/iio/adc/mediatek,mt6358-auxadc.h
-new file mode 100644
-index 000000000000..efa08398fafd
--- /dev/null
-+++ b/include/dt-bindings/iio/adc/mediatek,mt6358-auxadc.h
-@@ -0,0 +1,22 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+
-+#ifndef _DT_BINDINGS_MEDIATEK_MT6358_AUXADC_H
-+#define _DT_BINDINGS_MEDIATEK_MT6358_AUXADC_H
-+
-+/* ADC Channel Index */
-+#define MT6358_AUXADC_BATADC		0
-+#define MT6358_AUXADC_VCDT		1
-+#define MT6358_AUXADC_BAT_TEMP		2
-+#define MT6358_AUXADC_CHIP_TEMP		3
-+#define MT6358_AUXADC_ACCDET		4
-+#define MT6358_AUXADC_VDCXO		5
-+#define MT6358_AUXADC_TSX_TEMP		6
-+#define MT6358_AUXADC_HPOFS_CAL		7
-+#define MT6358_AUXADC_DCXO_TEMP		8
-+#define MT6358_AUXADC_VBIF		9
-+#define MT6358_AUXADC_VCORE_TEMP	10
-+#define MT6358_AUXADC_VPROC_TEMP	11
-+#define MT6358_AUXADC_VGPU_TEMP		12
-+#define MT6358_AUXADC_VBAT		13
-+
-+#endif
-diff --git a/include/dt-bindings/iio/adc/mediatek,mt6359-auxadc.h b/include/dt-bindings/iio/adc/mediatek,mt6359-auxadc.h
-new file mode 100644
-index 000000000000..59826393ee7e
--- /dev/null
-+++ b/include/dt-bindings/iio/adc/mediatek,mt6359-auxadc.h
-@@ -0,0 +1,22 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+
-+#ifndef _DT_BINDINGS_MEDIATEK_MT6359_AUXADC_H
-+#define _DT_BINDINGS_MEDIATEK_MT6359_AUXADC_H
-+
-+/* ADC Channel Index */
-+#define MT6359_AUXADC_BATADC		0
-+#define MT6359_AUXADC_BAT_TEMP		1
-+#define MT6359_AUXADC_CHIP_TEMP		2
-+#define MT6359_AUXADC_ACCDET		3
-+#define MT6359_AUXADC_VDCXO		4
-+#define MT6359_AUXADC_TSX_TEMP		5
-+#define MT6359_AUXADC_HPOFS_CAL		6
-+#define MT6359_AUXADC_DCXO_TEMP		7
-+#define MT6359_AUXADC_VBIF		8
-+#define MT6359_AUXADC_VCORE_TEMP	9
-+#define MT6359_AUXADC_VPROC_TEMP	10
-+#define MT6359_AUXADC_VGPU_TEMP		11
-+#define MT6359_AUXADC_VBAT		12
-+#define MT6359_AUXADC_IBAT		13
-+
-+#endif
-diff --git a/include/dt-bindings/iio/adc/mediatek,mt6363-auxadc.h b/include/dt-bindings/iio/adc/mediatek,mt6363-auxadc.h
-new file mode 100644
-index 000000000000..92d135477d0e
--- /dev/null
-+++ b/include/dt-bindings/iio/adc/mediatek,mt6363-auxadc.h
-@@ -0,0 +1,24 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+
-+#ifndef _DT_BINDINGS_MEDIATEK_MT6363_AUXADC_H
-+#define _DT_BINDINGS_MEDIATEK_MT6363_AUXADC_H
-+
-+/* ADC Channel Index */
-+#define MT6363_AUXADC_BATADC		0
-+#define MT6363_AUXADC_VCDT		1
-+#define MT6363_AUXADC_BAT_TEMP		2
-+#define MT6363_AUXADC_CHIP_TEMP		3
-+#define MT6363_AUXADC_VSYSSNS		4
-+#define MT6363_AUXADC_VTREF		5
-+#define MT6363_AUXADC_VCORE_TEMP	6
-+#define MT6363_AUXADC_VPROC_TEMP	7
-+#define MT6363_AUXADC_VGPU_TEMP		8
-+#define MT6363_AUXADC_VIN1		9
-+#define MT6363_AUXADC_VIN2		10
-+#define MT6363_AUXADC_VIN3		11
-+#define MT6363_AUXADC_VIN4		12
-+#define MT6363_AUXADC_VIN5		13
-+#define MT6363_AUXADC_VIN6		14
-+#define MT6363_AUXADC_VIN7		15
-+
-+#endif
-diff --git a/include/dt-bindings/iio/adc/mediatek,mt6370_adc.h b/include/dt-bindings/iio/adc/mediatek,mt6370_adc.h
-new file mode 100644
-index 000000000000..6ee725547763
--- /dev/null
-+++ b/include/dt-bindings/iio/adc/mediatek,mt6370_adc.h
-@@ -0,0 +1,18 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+
-+#ifndef __DT_BINDINGS_MEDIATEK_MT6370_ADC_H__
-+#define __DT_BINDINGS_MEDIATEK_MT6370_ADC_H__
-+
-+/* ADC Channel Index */
-+#define MT6370_CHAN_VBUSDIV5	0
-+#define MT6370_CHAN_VBUSDIV2	1
-+#define MT6370_CHAN_VSYS	2
-+#define MT6370_CHAN_VBAT	3
-+#define MT6370_CHAN_TS_BAT	4
-+#define MT6370_CHAN_IBUS	5
-+#define MT6370_CHAN_IBAT	6
-+#define MT6370_CHAN_CHG_VDDP	7
-+#define MT6370_CHAN_TEMP_JC	8
-+#define MT6370_CHAN_MAX		9
-+
-+#endif
-diff --git a/include/dt-bindings/iio/adc/mediatek,mt6373-auxadc.h b/include/dt-bindings/iio/adc/mediatek,mt6373-auxadc.h
-new file mode 100644
-index 000000000000..17cab86d355e
--- /dev/null
-+++ b/include/dt-bindings/iio/adc/mediatek,mt6373-auxadc.h
-@@ -0,0 +1,19 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+
-+#ifndef _DT_BINDINGS_MEDIATEK_MT6373_AUXADC_H
-+#define _DT_BINDINGS_MEDIATEK_MT6373_AUXADC_H
-+
-+/* ADC Channel Index */
-+#define MT6373_AUXADC_CHIP_TEMP		0
-+#define MT6373_AUXADC_VCORE_TEMP	1
-+#define MT6373_AUXADC_VPROC_TEMP	2
-+#define MT6373_AUXADC_VGPU_TEMP		3
-+#define MT6373_AUXADC_VIN1		4
-+#define MT6373_AUXADC_VIN2		5
-+#define MT6373_AUXADC_VIN3		6
-+#define MT6373_AUXADC_VIN4		7
-+#define MT6373_AUXADC_VIN5		8
-+#define MT6373_AUXADC_VIN6		9
-+#define MT6373_AUXADC_VIN7		10
-+
-+#endif
-diff --git a/include/dt-bindings/iio/qcom,spmi-adc7-pm7325.h b/include/dt-bindings/iio/qcom,spmi-adc7-pm7325.h
-new file mode 100644
-index 000000000000..96908014e09e
--- /dev/null
-+++ b/include/dt-bindings/iio/qcom,spmi-adc7-pm7325.h
-@@ -0,0 +1,69 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Copyright (c) 2020 The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_QCOM_SPMI_VADC_PM7325_H
-+#define _DT_BINDINGS_QCOM_SPMI_VADC_PM7325_H
-+
-+#ifndef PM7325_SID
-+#define PM7325_SID					1
-+#endif
-+
-+#include <dt-bindings/iio/qcom,spmi-vadc.h>
-+
-+/* ADC channels for PM7325_ADC for PMIC7 */
-+#define PM7325_ADC7_REF_GND			(PM7325_SID << 8 | ADC7_REF_GND)
-+#define PM7325_ADC7_1P25VREF			(PM7325_SID << 8 | ADC7_1P25VREF)
-+#define PM7325_ADC7_VREF_VADC			(PM7325_SID << 8 | ADC7_VREF_VADC)
-+#define PM7325_ADC7_DIE_TEMP			(PM7325_SID << 8 | ADC7_DIE_TEMP)
-+
-+#define PM7325_ADC7_AMUX_THM1			(PM7325_SID << 8 | ADC7_AMUX_THM1)
-+#define PM7325_ADC7_AMUX_THM2			(PM7325_SID << 8 | ADC7_AMUX_THM2)
-+#define PM7325_ADC7_AMUX_THM3			(PM7325_SID << 8 | ADC7_AMUX_THM3)
-+#define PM7325_ADC7_AMUX_THM4			(PM7325_SID << 8 | ADC7_AMUX_THM4)
-+#define PM7325_ADC7_AMUX_THM5			(PM7325_SID << 8 | ADC7_AMUX_THM5)
-+#define PM7325_ADC7_GPIO1			(PM7325_SID << 8 | ADC7_GPIO1)
-+#define PM7325_ADC7_GPIO2			(PM7325_SID << 8 | ADC7_GPIO2)
-+#define PM7325_ADC7_GPIO3			(PM7325_SID << 8 | ADC7_GPIO3)
-+#define PM7325_ADC7_GPIO4			(PM7325_SID << 8 | ADC7_GPIO4)
-+
-+/* 30k pull-up1 */
-+#define PM7325_ADC7_AMUX_THM1_30K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM1_30K_PU)
-+#define PM7325_ADC7_AMUX_THM2_30K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM2_30K_PU)
-+#define PM7325_ADC7_AMUX_THM3_30K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM3_30K_PU)
-+#define PM7325_ADC7_AMUX_THM4_30K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM4_30K_PU)
-+#define PM7325_ADC7_AMUX_THM5_30K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM5_30K_PU)
-+#define PM7325_ADC7_GPIO1_30K_PU		(PM7325_SID << 8 | ADC7_GPIO1_30K_PU)
-+#define PM7325_ADC7_GPIO2_30K_PU		(PM7325_SID << 8 | ADC7_GPIO2_30K_PU)
-+#define PM7325_ADC7_GPIO3_30K_PU		(PM7325_SID << 8 | ADC7_GPIO3_30K_PU)
-+#define PM7325_ADC7_GPIO4_30K_PU		(PM7325_SID << 8 | ADC7_GPIO4_30K_PU)
-+
-+/* 100k pull-up2 */
-+#define PM7325_ADC7_AMUX_THM1_100K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM1_100K_PU)
-+#define PM7325_ADC7_AMUX_THM2_100K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM2_100K_PU)
-+#define PM7325_ADC7_AMUX_THM3_100K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM3_100K_PU)
-+#define PM7325_ADC7_AMUX_THM4_100K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM4_100K_PU)
-+#define PM7325_ADC7_AMUX_THM5_100K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM5_100K_PU)
-+#define PM7325_ADC7_GPIO1_100K_PU		(PM7325_SID << 8 | ADC7_GPIO1_100K_PU)
-+#define PM7325_ADC7_GPIO2_100K_PU		(PM7325_SID << 8 | ADC7_GPIO2_100K_PU)
-+#define PM7325_ADC7_GPIO3_100K_PU		(PM7325_SID << 8 | ADC7_GPIO3_100K_PU)
-+#define PM7325_ADC7_GPIO4_100K_PU		(PM7325_SID << 8 | ADC7_GPIO4_100K_PU)
-+
-+/* 400k pull-up3 */
-+#define PM7325_ADC7_AMUX_THM1_400K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM1_400K_PU)
-+#define PM7325_ADC7_AMUX_THM2_400K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM2_400K_PU)
-+#define PM7325_ADC7_AMUX_THM3_400K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM3_400K_PU)
-+#define PM7325_ADC7_AMUX_THM4_400K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM4_400K_PU)
-+#define PM7325_ADC7_AMUX_THM5_400K_PU		(PM7325_SID << 8 | ADC7_AMUX_THM5_400K_PU)
-+#define PM7325_ADC7_GPIO1_400K_PU		(PM7325_SID << 8 | ADC7_GPIO1_400K_PU)
-+#define PM7325_ADC7_GPIO2_400K_PU		(PM7325_SID << 8 | ADC7_GPIO2_400K_PU)
-+#define PM7325_ADC7_GPIO3_400K_PU		(PM7325_SID << 8 | ADC7_GPIO3_400K_PU)
-+#define PM7325_ADC7_GPIO4_400K_PU		(PM7325_SID << 8 | ADC7_GPIO4_400K_PU)
-+
-+/* 1/3 Divider */
-+#define PM7325_ADC7_GPIO4_DIV3			(PM7325_SID << 8 | ADC7_GPIO4_DIV3)
-+
-+#define PM7325_ADC7_VPH_PWR			(PM7325_SID << 8 | ADC7_VPH_PWR)
-+
-+#endif /* _DT_BINDINGS_QCOM_SPMI_VADC_PM7325_H */
-diff --git a/include/dt-bindings/iio/qcom,spmi-adc7-pm8350.h b/include/dt-bindings/iio/qcom,spmi-adc7-pm8350.h
-index 9426f27a1946..5d98f7d48a1e 100644
--- a/include/dt-bindings/iio/qcom,spmi-adc7-pm8350.h
-+++ b/include/dt-bindings/iio/qcom,spmi-adc7-pm8350.h
-@@ -6,62 +6,60 @@
- #ifndef _DT_BINDINGS_QCOM_SPMI_VADC_PM8350_H
- #define _DT_BINDINGS_QCOM_SPMI_VADC_PM8350_H
- 
-#ifndef PM8350_SID
-#define PM8350_SID					1
-#endif
-+#include <dt-bindings/iio/qcom,spmi-vadc.h>
- 
- /* ADC channels for PM8350_ADC for PMIC7 */
-#define PM8350_ADC7_REF_GND			(PM8350_SID << 8 | 0x0)
-#define PM8350_ADC7_1P25VREF			(PM8350_SID << 8 | 0x01)
-#define PM8350_ADC7_VREF_VADC			(PM8350_SID << 8 | 0x02)
-#define PM8350_ADC7_DIE_TEMP			(PM8350_SID << 8 | 0x03)
-+#define PM8350_ADC7_REF_GND(sid)			((sid) << 8 | ADC7_REF_GND)
-+#define PM8350_ADC7_1P25VREF(sid)			((sid) << 8 | ADC7_1P25VREF)
-+#define PM8350_ADC7_VREF_VADC(sid)			((sid) << 8 | ADC7_VREF_VADC)
-+#define PM8350_ADC7_DIE_TEMP(sid)			((sid) << 8 | ADC7_DIE_TEMP)
- 
-#define PM8350_ADC7_AMUX_THM1			(PM8350_SID << 8 | 0x04)
-#define PM8350_ADC7_AMUX_THM2			(PM8350_SID << 8 | 0x05)
-#define PM8350_ADC7_AMUX_THM3			(PM8350_SID << 8 | 0x06)
-#define PM8350_ADC7_AMUX_THM4			(PM8350_SID << 8 | 0x07)
-#define PM8350_ADC7_AMUX_THM5			(PM8350_SID << 8 | 0x08)
-#define PM8350_ADC7_GPIO1			(PM8350_SID << 8 | 0x0a)
-#define PM8350_ADC7_GPIO2			(PM8350_SID << 8 | 0x0b)
-#define PM8350_ADC7_GPIO3			(PM8350_SID << 8 | 0x0c)
-#define PM8350_ADC7_GPIO4			(PM8350_SID << 8 | 0x0d)
-+#define PM8350_ADC7_AMUX_THM1(sid)			((sid) << 8 | ADC7_AMUX_THM1)
-+#define PM8350_ADC7_AMUX_THM2(sid)			((sid) << 8 | ADC7_AMUX_THM2)
-+#define PM8350_ADC7_AMUX_THM3(sid)			((sid) << 8 | ADC7_AMUX_THM3)
-+#define PM8350_ADC7_AMUX_THM4(sid)			((sid) << 8 | ADC7_AMUX_THM4)
-+#define PM8350_ADC7_AMUX_THM5(sid)			((sid) << 8 | ADC7_AMUX_THM5)
-+#define PM8350_ADC7_GPIO1(sid)				((sid) << 8 | ADC7_GPIO1)
-+#define PM8350_ADC7_GPIO2(sid)				((sid) << 8 | ADC7_GPIO2)
-+#define PM8350_ADC7_GPIO3(sid)				((sid) << 8 | ADC7_GPIO3)
-+#define PM8350_ADC7_GPIO4(sid)				((sid) << 8 | ADC7_GPIO4)
- 
- /* 30k pull-up1 */
-#define PM8350_ADC7_AMUX_THM1_30K_PU		(PM8350_SID << 8 | 0x24)
-#define PM8350_ADC7_AMUX_THM2_30K_PU		(PM8350_SID << 8 | 0x25)
-#define PM8350_ADC7_AMUX_THM3_30K_PU		(PM8350_SID << 8 | 0x26)
-#define PM8350_ADC7_AMUX_THM4_30K_PU		(PM8350_SID << 8 | 0x27)
-#define PM8350_ADC7_AMUX_THM5_30K_PU		(PM8350_SID << 8 | 0x28)
-#define PM8350_ADC7_GPIO1_30K_PU		(PM8350_SID << 8 | 0x2a)
-#define PM8350_ADC7_GPIO2_30K_PU		(PM8350_SID << 8 | 0x2b)
-#define PM8350_ADC7_GPIO3_30K_PU		(PM8350_SID << 8 | 0x2c)
-#define PM8350_ADC7_GPIO4_30K_PU		(PM8350_SID << 8 | 0x2d)
-+#define PM8350_ADC7_AMUX_THM1_30K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM1_30K_PU)
-+#define PM8350_ADC7_AMUX_THM2_30K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM2_30K_PU)
-+#define PM8350_ADC7_AMUX_THM3_30K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM3_30K_PU)
-+#define PM8350_ADC7_AMUX_THM4_30K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM4_30K_PU)
-+#define PM8350_ADC7_AMUX_THM5_30K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM5_30K_PU)
-+#define PM8350_ADC7_GPIO1_30K_PU(sid)			((sid) << 8 | ADC7_GPIO1_30K_PU)
-+#define PM8350_ADC7_GPIO2_30K_PU(sid)			((sid) << 8 | ADC7_GPIO2_30K_PU)
-+#define PM8350_ADC7_GPIO3_30K_PU(sid)			((sid) << 8 | ADC7_GPIO3_30K_PU)
-+#define PM8350_ADC7_GPIO4_30K_PU(sid)			((sid) << 8 | ADC7_GPIO4_30K_PU)
- 
- /* 100k pull-up2 */
-#define PM8350_ADC7_AMUX_THM1_100K_PU		(PM8350_SID << 8 | 0x44)
-#define PM8350_ADC7_AMUX_THM2_100K_PU		(PM8350_SID << 8 | 0x45)
-#define PM8350_ADC7_AMUX_THM3_100K_PU		(PM8350_SID << 8 | 0x46)
-#define PM8350_ADC7_AMUX_THM4_100K_PU		(PM8350_SID << 8 | 0x47)
-#define PM8350_ADC7_AMUX_THM5_100K_PU		(PM8350_SID << 8 | 0x48)
-#define PM8350_ADC7_GPIO1_100K_PU		(PM8350_SID << 8 | 0x4a)
-#define PM8350_ADC7_GPIO2_100K_PU		(PM8350_SID << 8 | 0x4b)
-#define PM8350_ADC7_GPIO3_100K_PU		(PM8350_SID << 8 | 0x4c)
-#define PM8350_ADC7_GPIO4_100K_PU		(PM8350_SID << 8 | 0x4d)
-+#define PM8350_ADC7_AMUX_THM1_100K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM1_100K_PU)
-+#define PM8350_ADC7_AMUX_THM2_100K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM2_100K_PU)
-+#define PM8350_ADC7_AMUX_THM3_100K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM3_100K_PU)
-+#define PM8350_ADC7_AMUX_THM4_100K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM4_100K_PU)
-+#define PM8350_ADC7_AMUX_THM5_100K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM5_100K_PU)
-+#define PM8350_ADC7_GPIO1_100K_PU(sid)			((sid) << 8 | ADC7_GPIO1_100K_PU)
-+#define PM8350_ADC7_GPIO2_100K_PU(sid)			((sid) << 8 | ADC7_GPIO2_100K_PU)
-+#define PM8350_ADC7_GPIO3_100K_PU(sid)			((sid) << 8 | ADC7_GPIO3_100K_PU)
-+#define PM8350_ADC7_GPIO4_100K_PU(sid)			((sid) << 8 | ADC7_GPIO4_100K_PU)
- 
- /* 400k pull-up3 */
-#define PM8350_ADC7_AMUX_THM1_400K_PU		(PM8350_SID << 8 | 0x64)
-#define PM8350_ADC7_AMUX_THM2_400K_PU		(PM8350_SID << 8 | 0x65)
-#define PM8350_ADC7_AMUX_THM3_400K_PU		(PM8350_SID << 8 | 0x66)
-#define PM8350_ADC7_AMUX_THM4_400K_PU		(PM8350_SID << 8 | 0x67)
-#define PM8350_ADC7_AMUX_THM5_400K_PU		(PM8350_SID << 8 | 0x68)
-#define PM8350_ADC7_GPIO1_400K_PU		(PM8350_SID << 8 | 0x6a)
-#define PM8350_ADC7_GPIO2_400K_PU		(PM8350_SID << 8 | 0x6b)
-#define PM8350_ADC7_GPIO3_400K_PU		(PM8350_SID << 8 | 0x6c)
-#define PM8350_ADC7_GPIO4_400K_PU		(PM8350_SID << 8 | 0x6d)
-+#define PM8350_ADC7_AMUX_THM1_400K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM1_400K_PU)
-+#define PM8350_ADC7_AMUX_THM2_400K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM2_400K_PU)
-+#define PM8350_ADC7_AMUX_THM3_400K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM3_400K_PU)
-+#define PM8350_ADC7_AMUX_THM4_400K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM4_400K_PU)
-+#define PM8350_ADC7_AMUX_THM5_400K_PU(sid)		((sid) << 8 | ADC7_AMUX_THM5_400K_PU)
-+#define PM8350_ADC7_GPIO1_400K_PU(sid)			((sid) << 8 | ADC7_GPIO1_400K_PU)
-+#define PM8350_ADC7_GPIO2_400K_PU(sid)			((sid) << 8 | ADC7_GPIO2_400K_PU)
-+#define PM8350_ADC7_GPIO3_400K_PU(sid)			((sid) << 8 | ADC7_GPIO3_400K_PU)
-+#define PM8350_ADC7_GPIO4_400K_PU(sid)			((sid) << 8 | ADC7_GPIO4_400K_PU)
- 
- /* 1/3 Divider */
-#define PM8350_ADC7_GPIO4_DIV3			(PM8350_SID << 8 | 0x8d)
-+#define PM8350_ADC7_GPIO4_DIV3(sid)			((sid) << 8 | ADC7_GPIO4_DIV3)
- 
-#define PM8350_ADC7_VPH_PWR			(PM8350_SID << 8 | 0x8e)
-+#define PM8350_ADC7_VPH_PWR(sid)			((sid) << 8 | ADC7_VPH_PWR)
- 
- #endif /* _DT_BINDINGS_QCOM_SPMI_VADC_PM8350_H */
-diff --git a/include/dt-bindings/iio/qcom,spmi-adc7-pm8350b.h b/include/dt-bindings/iio/qcom,spmi-adc7-pm8350b.h
-index dc2497c27e16..57c7977666d3 100644
--- a/include/dt-bindings/iio/qcom,spmi-adc7-pm8350b.h
-+++ b/include/dt-bindings/iio/qcom,spmi-adc7-pm8350b.h
-@@ -10,79 +10,81 @@
- #define PM8350B_SID					3
- #endif
- 
-+#include <dt-bindings/iio/qcom,spmi-vadc.h>
-+
- /* ADC channels for PM8350B_ADC for PMIC7 */
-#define PM8350B_ADC7_REF_GND			(PM8350B_SID << 8 | 0x0)
-#define PM8350B_ADC7_1P25VREF			(PM8350B_SID << 8 | 0x01)
-#define PM8350B_ADC7_VREF_VADC			(PM8350B_SID << 8 | 0x02)
-#define PM8350B_ADC7_DIE_TEMP			(PM8350B_SID << 8 | 0x03)
-+#define PM8350B_ADC7_REF_GND			(PM8350B_SID << 8 | ADC7_REF_GND)
-+#define PM8350B_ADC7_1P25VREF			(PM8350B_SID << 8 | ADC7_1P25VREF)
-+#define PM8350B_ADC7_VREF_VADC			(PM8350B_SID << 8 | ADC7_VREF_VADC)
-+#define PM8350B_ADC7_DIE_TEMP			(PM8350B_SID << 8 | ADC7_DIE_TEMP)
- 
-#define PM8350B_ADC7_AMUX_THM1			(PM8350B_SID << 8 | 0x04)
-#define PM8350B_ADC7_AMUX_THM2			(PM8350B_SID << 8 | 0x05)
-#define PM8350B_ADC7_AMUX_THM3			(PM8350B_SID << 8 | 0x06)
-#define PM8350B_ADC7_AMUX_THM4			(PM8350B_SID << 8 | 0x07)
-#define PM8350B_ADC7_AMUX_THM5			(PM8350B_SID << 8 | 0x08)
-#define PM8350B_ADC7_AMUX_THM6			(PM8350B_SID << 8 | 0x09)
-#define PM8350B_ADC7_GPIO1			(PM8350B_SID << 8 | 0x0a)
-#define PM8350B_ADC7_GPIO2			(PM8350B_SID << 8 | 0x0b)
-#define PM8350B_ADC7_GPIO3			(PM8350B_SID << 8 | 0x0c)
-#define PM8350B_ADC7_GPIO4			(PM8350B_SID << 8 | 0x0d)
-+#define PM8350B_ADC7_AMUX_THM1			(PM8350B_SID << 8 | ADC7_AMUX_THM1)
-+#define PM8350B_ADC7_AMUX_THM2			(PM8350B_SID << 8 | ADC7_AMUX_THM2)
-+#define PM8350B_ADC7_AMUX_THM3			(PM8350B_SID << 8 | ADC7_AMUX_THM3)
-+#define PM8350B_ADC7_AMUX_THM4			(PM8350B_SID << 8 | ADC7_AMUX_THM4)
-+#define PM8350B_ADC7_AMUX_THM5			(PM8350B_SID << 8 | ADC7_AMUX_THM5)
-+#define PM8350B_ADC7_AMUX_THM6			(PM8350B_SID << 8 | ADC7_AMUX_THM6)
-+#define PM8350B_ADC7_GPIO1			(PM8350B_SID << 8 | ADC7_GPIO1)
-+#define PM8350B_ADC7_GPIO2			(PM8350B_SID << 8 | ADC7_GPIO2)
-+#define PM8350B_ADC7_GPIO3			(PM8350B_SID << 8 | ADC7_GPIO3)
-+#define PM8350B_ADC7_GPIO4			(PM8350B_SID << 8 | ADC7_GPIO4)
- 
-#define PM8350B_ADC7_CHG_TEMP			(PM8350B_SID << 8 | 0x10)
-#define PM8350B_ADC7_USB_IN_V_16		(PM8350B_SID << 8 | 0x11)
-#define PM8350B_ADC7_VDC_16			(PM8350B_SID << 8 | 0x12)
-#define PM8350B_ADC7_CC1_ID			(PM8350B_SID << 8 | 0x13)
-#define PM8350B_ADC7_VREF_BAT_THERM		(PM8350B_SID << 8 | 0x15)
-#define PM8350B_ADC7_IIN_FB			(PM8350B_SID << 8 | 0x17)
-+#define PM8350B_ADC7_CHG_TEMP			(PM8350B_SID << 8 | ADC7_CHG_TEMP)
-+#define PM8350B_ADC7_USB_IN_V_16		(PM8350B_SID << 8 | ADC7_USB_IN_V_16)
-+#define PM8350B_ADC7_VDC_16			(PM8350B_SID << 8 | ADC7_VDC_16)
-+#define PM8350B_ADC7_CC1_ID			(PM8350B_SID << 8 | ADC7_CC1_ID)
-+#define PM8350B_ADC7_VREF_BAT_THERM		(PM8350B_SID << 8 | ADC7_VREF_BAT_THERM)
-+#define PM8350B_ADC7_IIN_FB			(PM8350B_SID << 8 | ADC7_IIN_FB)
- 
- /* 30k pull-up1 */
-#define PM8350B_ADC7_AMUX_THM1_30K_PU		(PM8350B_SID << 8 | 0x24)
-#define PM8350B_ADC7_AMUX_THM2_30K_PU		(PM8350B_SID << 8 | 0x25)
-#define PM8350B_ADC7_AMUX_THM3_30K_PU		(PM8350B_SID << 8 | 0x26)
-#define PM8350B_ADC7_AMUX_THM4_30K_PU		(PM8350B_SID << 8 | 0x27)
-#define PM8350B_ADC7_AMUX_THM5_30K_PU		(PM8350B_SID << 8 | 0x28)
-#define PM8350B_ADC7_AMUX_THM6_30K_PU		(PM8350B_SID << 8 | 0x29)
-#define PM8350B_ADC7_GPIO1_30K_PU		(PM8350B_SID << 8 | 0x2a)
-#define PM8350B_ADC7_GPIO2_30K_PU		(PM8350B_SID << 8 | 0x2b)
-#define PM8350B_ADC7_GPIO3_30K_PU		(PM8350B_SID << 8 | 0x2c)
-#define PM8350B_ADC7_GPIO4_30K_PU		(PM8350B_SID << 8 | 0x2d)
-#define PM8350B_ADC7_CC1_ID_30K_PU		(PM8350B_SID << 8 | 0x33)
-+#define PM8350B_ADC7_AMUX_THM1_30K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM1_30K_PU)
-+#define PM8350B_ADC7_AMUX_THM2_30K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM2_30K_PU)
-+#define PM8350B_ADC7_AMUX_THM3_30K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM3_30K_PU)
-+#define PM8350B_ADC7_AMUX_THM4_30K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM4_30K_PU)
-+#define PM8350B_ADC7_AMUX_THM5_30K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM5_30K_PU)
-+#define PM8350B_ADC7_AMUX_THM6_30K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM6_30K_PU)
-+#define PM8350B_ADC7_GPIO1_30K_PU		(PM8350B_SID << 8 | ADC7_GPIO1_30K_PU)
-+#define PM8350B_ADC7_GPIO2_30K_PU		(PM8350B_SID << 8 | ADC7_GPIO2_30K_PU)
-+#define PM8350B_ADC7_GPIO3_30K_PU		(PM8350B_SID << 8 | ADC7_GPIO3_30K_PU)
-+#define PM8350B_ADC7_GPIO4_30K_PU		(PM8350B_SID << 8 | ADC7_GPIO4_30K_PU)
-+#define PM8350B_ADC7_CC1_ID_30K_PU		(PM8350B_SID << 8 | ADC7_CC1_ID_30K_PU)
- 
- /* 100k pull-up2 */
-#define PM8350B_ADC7_AMUX_THM1_100K_PU		(PM8350B_SID << 8 | 0x44)
-#define PM8350B_ADC7_AMUX_THM2_100K_PU		(PM8350B_SID << 8 | 0x45)
-#define PM8350B_ADC7_AMUX_THM3_100K_PU		(PM8350B_SID << 8 | 0x46)
-#define PM8350B_ADC7_AMUX_THM4_100K_PU		(PM8350B_SID << 8 | 0x47)
-#define PM8350B_ADC7_AMUX_THM5_100K_PU		(PM8350B_SID << 8 | 0x48)
-#define PM8350B_ADC7_AMUX_THM6_100K_PU		(PM8350B_SID << 8 | 0x49)
-#define PM8350B_ADC7_GPIO1_100K_PU		(PM8350B_SID << 8 | 0x4a)
-#define PM8350B_ADC7_GPIO2_100K_PU		(PM8350B_SID << 8 | 0x4b)
-#define PM8350B_ADC7_GPIO3_100K_PU		(PM8350B_SID << 8 | 0x4c)
-#define PM8350B_ADC7_GPIO4_100K_PU		(PM8350B_SID << 8 | 0x4d)
-#define PM8350B_ADC7_CC1_ID_100K_PU		(PM8350B_SID << 8 | 0x53)
-+#define PM8350B_ADC7_AMUX_THM1_100K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM1_100K_PU)
-+#define PM8350B_ADC7_AMUX_THM2_100K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM2_100K_PU)
-+#define PM8350B_ADC7_AMUX_THM3_100K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM3_100K_PU)
-+#define PM8350B_ADC7_AMUX_THM4_100K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM4_100K_PU)
-+#define PM8350B_ADC7_AMUX_THM5_100K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM5_100K_PU)
-+#define PM8350B_ADC7_AMUX_THM6_100K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM6_100K_PU)
-+#define PM8350B_ADC7_GPIO1_100K_PU		(PM8350B_SID << 8 | ADC7_GPIO1_100K_PU)
-+#define PM8350B_ADC7_GPIO2_100K_PU		(PM8350B_SID << 8 | ADC7_GPIO2_100K_PU)
-+#define PM8350B_ADC7_GPIO3_100K_PU		(PM8350B_SID << 8 | ADC7_GPIO3_100K_PU)
-+#define PM8350B_ADC7_GPIO4_100K_PU		(PM8350B_SID << 8 | ADC7_GPIO4_100K_PU)
-+#define PM8350B_ADC7_CC1_ID_100K_PU		(PM8350B_SID << 8 | ADC7_CC1_ID_100K_PU)
- 
- /* 400k pull-up3 */
-#define PM8350B_ADC7_AMUX_THM1_400K_PU		(PM8350B_SID << 8 | 0x64)
-#define PM8350B_ADC7_AMUX_THM2_400K_PU		(PM8350B_SID << 8 | 0x65)
-#define PM8350B_ADC7_AMUX_THM3_400K_PU		(PM8350B_SID << 8 | 0x66)
-#define PM8350B_ADC7_AMUX_THM4_400K_PU		(PM8350B_SID << 8 | 0x67)
-#define PM8350B_ADC7_AMUX_THM5_400K_PU		(PM8350B_SID << 8 | 0x68)
-#define PM8350B_ADC7_AMUX_THM6_400K_PU		(PM8350B_SID << 8 | 0x69)
-#define PM8350B_ADC7_GPIO1_400K_PU		(PM8350B_SID << 8 | 0x6a)
-#define PM8350B_ADC7_GPIO2_400K_PU		(PM8350B_SID << 8 | 0x6b)
-#define PM8350B_ADC7_GPIO3_400K_PU		(PM8350B_SID << 8 | 0x6c)
-#define PM8350B_ADC7_GPIO4_400K_PU		(PM8350B_SID << 8 | 0x6d)
-#define PM8350B_ADC7_CC1_ID_400K_PU		(PM8350B_SID << 8 | 0x73)
-+#define PM8350B_ADC7_AMUX_THM1_400K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM1_400K_PU)
-+#define PM8350B_ADC7_AMUX_THM2_400K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM2_400K_PU)
-+#define PM8350B_ADC7_AMUX_THM3_400K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM3_400K_PU)
-+#define PM8350B_ADC7_AMUX_THM4_400K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM4_400K_PU)
-+#define PM8350B_ADC7_AMUX_THM5_400K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM5_400K_PU)
-+#define PM8350B_ADC7_AMUX_THM6_400K_PU		(PM8350B_SID << 8 | ADC7_AMUX_THM6_400K_PU)
-+#define PM8350B_ADC7_GPIO1_400K_PU		(PM8350B_SID << 8 | ADC7_GPIO1_400K_PU)
-+#define PM8350B_ADC7_GPIO2_400K_PU		(PM8350B_SID << 8 | ADC7_GPIO2_400K_PU)
-+#define PM8350B_ADC7_GPIO3_400K_PU		(PM8350B_SID << 8 | ADC7_GPIO3_400K_PU)
-+#define PM8350B_ADC7_GPIO4_400K_PU		(PM8350B_SID << 8 | ADC7_GPIO4_400K_PU)
-+#define PM8350B_ADC7_CC1_ID_400K_PU		(PM8350B_SID << 8 | ADC7_CC1_ID_400K_PU)
- 
- /* 1/3 Divider */
-#define PM8350B_ADC7_GPIO1_DIV3			(PM8350B_SID << 8 | 0x8a)
-#define PM8350B_ADC7_GPIO2_DIV3			(PM8350B_SID << 8 | 0x8b)
-#define PM8350B_ADC7_GPIO3_DIV3			(PM8350B_SID << 8 | 0x8c)
-#define PM8350B_ADC7_GPIO4_DIV3			(PM8350B_SID << 8 | 0x8d)
-+#define PM8350B_ADC7_GPIO1_DIV3			(PM8350B_SID << 8 | ADC7_GPIO1_DIV3)
-+#define PM8350B_ADC7_GPIO2_DIV3			(PM8350B_SID << 8 | ADC7_GPIO2_DIV3)
-+#define PM8350B_ADC7_GPIO3_DIV3			(PM8350B_SID << 8 | ADC7_GPIO3_DIV3)
-+#define PM8350B_ADC7_GPIO4_DIV3			(PM8350B_SID << 8 | ADC7_GPIO4_DIV3)
- 
-#define PM8350B_ADC7_VPH_PWR			(PM8350B_SID << 8 | 0x8e)
-#define PM8350B_ADC7_VBAT_SNS			(PM8350B_SID << 8 | 0x8f)
-+#define PM8350B_ADC7_VPH_PWR			(PM8350B_SID << 8 | ADC7_VPH_PWR)
-+#define PM8350B_ADC7_VBAT_SNS			(PM8350B_SID << 8 | ADC7_VBAT_SNS)
- 
-#define PM8350B_ADC7_SBUx			(PM8350B_SID << 8 | 0x94)
-#define PM8350B_ADC7_VBAT_2S_MID		(PM8350B_SID << 8 | 0x96)
-+#define PM8350B_ADC7_SBUx			(PM8350B_SID << 8 | ADC7_SBU)
-+#define PM8350B_ADC7_VBAT_2S_MID		(PM8350B_SID << 8 | ADC7_VBAT_2S_MID)
- 
- #endif /* _DT_BINDINGS_QCOM_SPMI_VADC_PM8350B_H */
-diff --git a/include/dt-bindings/iio/qcom,spmi-adc7-pmk8350.h b/include/dt-bindings/iio/qcom,spmi-adc7-pmk8350.h
-index 6c296870e95b..3d1a41a22cef 100644
--- a/include/dt-bindings/iio/qcom,spmi-adc7-pmk8350.h
-+++ b/include/dt-bindings/iio/qcom,spmi-adc7-pmk8350.h
-@@ -10,37 +10,39 @@
- #define PMK8350_SID					0
- #endif
- 
-+#include <dt-bindings/iio/qcom,spmi-vadc.h>
-+
- /* ADC channels for PMK8350_ADC for PMIC7 */
-#define PMK8350_ADC7_REF_GND			(PMK8350_SID << 8 | 0x0)
-#define PMK8350_ADC7_1P25VREF			(PMK8350_SID << 8 | 0x01)
-#define PMK8350_ADC7_VREF_VADC			(PMK8350_SID << 8 | 0x02)
-#define PMK8350_ADC7_DIE_TEMP			(PMK8350_SID << 8 | 0x03)
-+#define PMK8350_ADC7_REF_GND			(PMK8350_SID << 8 | ADC7_REF_GND)
-+#define PMK8350_ADC7_1P25VREF			(PMK8350_SID << 8 | ADC7_1P25VREF)
-+#define PMK8350_ADC7_VREF_VADC			(PMK8350_SID << 8 | ADC7_VREF_VADC)
-+#define PMK8350_ADC7_DIE_TEMP			(PMK8350_SID << 8 | ADC7_DIE_TEMP)
- 
-#define PMK8350_ADC7_AMUX_THM1			(PMK8350_SID << 8 | 0x04)
-#define PMK8350_ADC7_AMUX_THM2			(PMK8350_SID << 8 | 0x05)
-#define PMK8350_ADC7_AMUX_THM3			(PMK8350_SID << 8 | 0x06)
-#define PMK8350_ADC7_AMUX_THM4			(PMK8350_SID << 8 | 0x07)
-#define PMK8350_ADC7_AMUX_THM5			(PMK8350_SID << 8 | 0x08)
-+#define PMK8350_ADC7_AMUX_THM1			(PMK8350_SID << 8 | ADC7_AMUX_THM1)
-+#define PMK8350_ADC7_AMUX_THM2			(PMK8350_SID << 8 | ADC7_AMUX_THM2)
-+#define PMK8350_ADC7_AMUX_THM3			(PMK8350_SID << 8 | ADC7_AMUX_THM3)
-+#define PMK8350_ADC7_AMUX_THM4			(PMK8350_SID << 8 | ADC7_AMUX_THM4)
-+#define PMK8350_ADC7_AMUX_THM5			(PMK8350_SID << 8 | ADC7_AMUX_THM5)
- 
- /* 30k pull-up1 */
-#define PMK8350_ADC7_AMUX_THM1_30K_PU		(PMK8350_SID << 8 | 0x24)
-#define PMK8350_ADC7_AMUX_THM2_30K_PU		(PMK8350_SID << 8 | 0x25)
-#define PMK8350_ADC7_AMUX_THM3_30K_PU		(PMK8350_SID << 8 | 0x26)
-#define PMK8350_ADC7_AMUX_THM4_30K_PU		(PMK8350_SID << 8 | 0x27)
-#define PMK8350_ADC7_AMUX_THM5_30K_PU		(PMK8350_SID << 8 | 0x28)
-+#define PMK8350_ADC7_AMUX_THM1_30K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM1_30K_PU)
-+#define PMK8350_ADC7_AMUX_THM2_30K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM2_30K_PU)
-+#define PMK8350_ADC7_AMUX_THM3_30K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM3_30K_PU)
-+#define PMK8350_ADC7_AMUX_THM4_30K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM4_30K_PU)
-+#define PMK8350_ADC7_AMUX_THM5_30K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM5_30K_PU)
- 
- /* 100k pull-up2 */
-#define PMK8350_ADC7_AMUX_THM1_100K_PU		(PMK8350_SID << 8 | 0x44)
-#define PMK8350_ADC7_AMUX_THM2_100K_PU		(PMK8350_SID << 8 | 0x45)
-#define PMK8350_ADC7_AMUX_THM3_100K_PU		(PMK8350_SID << 8 | 0x46)
-#define PMK8350_ADC7_AMUX_THM4_100K_PU		(PMK8350_SID << 8 | 0x47)
-#define PMK8350_ADC7_AMUX_THM5_100K_PU		(PMK8350_SID << 8 | 0x48)
-+#define PMK8350_ADC7_AMUX_THM1_100K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM1_100K_PU)
-+#define PMK8350_ADC7_AMUX_THM2_100K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM2_100K_PU)
-+#define PMK8350_ADC7_AMUX_THM3_100K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM3_100K_PU)
-+#define PMK8350_ADC7_AMUX_THM4_100K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM4_100K_PU)
-+#define PMK8350_ADC7_AMUX_THM5_100K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM5_100K_PU)
- 
- /* 400k pull-up3 */
-#define PMK8350_ADC7_AMUX_THM1_400K_PU		(PMK8350_SID << 8 | 0x64)
-#define PMK8350_ADC7_AMUX_THM2_400K_PU		(PMK8350_SID << 8 | 0x65)
-#define PMK8350_ADC7_AMUX_THM3_400K_PU		(PMK8350_SID << 8 | 0x66)
-#define PMK8350_ADC7_AMUX_THM4_400K_PU		(PMK8350_SID << 8 | 0x67)
-#define PMK8350_ADC7_AMUX_THM5_400K_PU		(PMK8350_SID << 8 | 0x68)
-+#define PMK8350_ADC7_AMUX_THM1_400K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM1_400K_PU)
-+#define PMK8350_ADC7_AMUX_THM2_400K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM2_400K_PU)
-+#define PMK8350_ADC7_AMUX_THM3_400K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM3_400K_PU)
-+#define PMK8350_ADC7_AMUX_THM4_400K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM4_400K_PU)
-+#define PMK8350_ADC7_AMUX_THM5_400K_PU		(PMK8350_SID << 8 | ADC7_AMUX_THM5_400K_PU)
- 
- #endif /* _DT_BINDINGS_QCOM_SPMI_VADC_PMK8350_H */
-diff --git a/include/dt-bindings/iio/qcom,spmi-adc7-pmr735a.h b/include/dt-bindings/iio/qcom,spmi-adc7-pmr735a.h
-index d6df1b19e5ff..c5adfa82b20d 100644
--- a/include/dt-bindings/iio/qcom,spmi-adc7-pmr735a.h
-+++ b/include/dt-bindings/iio/qcom,spmi-adc7-pmr735a.h
-@@ -10,19 +10,21 @@
- #define PMR735A_SID					4
- #endif
- 
-+#include <dt-bindings/iio/qcom,spmi-vadc.h>
-+
- /* ADC channels for PMR735A_ADC for PMIC7 */
-#define PMR735A_ADC7_REF_GND			(PMR735A_SID << 8 | 0x0)
-#define PMR735A_ADC7_1P25VREF			(PMR735A_SID << 8 | 0x01)
-#define PMR735A_ADC7_VREF_VADC			(PMR735A_SID << 8 | 0x02)
-#define PMR735A_ADC7_DIE_TEMP			(PMR735A_SID << 8 | 0x03)
-+#define PMR735A_ADC7_REF_GND			(PMR735A_SID << 8 | ADC7_REF_GND)
-+#define PMR735A_ADC7_1P25VREF			(PMR735A_SID << 8 | ADC7_1P25VREF)
-+#define PMR735A_ADC7_VREF_VADC			(PMR735A_SID << 8 | ADC7_VREF_VADC)
-+#define PMR735A_ADC7_DIE_TEMP			(PMR735A_SID << 8 | ADC7_DIE_TEMP)
- 
-#define PMR735A_ADC7_GPIO1			(PMR735A_SID << 8 | 0x0a)
-#define PMR735A_ADC7_GPIO2			(PMR735A_SID << 8 | 0x0b)
-#define PMR735A_ADC7_GPIO3			(PMR735A_SID << 8 | 0x0c)
-+#define PMR735A_ADC7_GPIO1			(PMR735A_SID << 8 | ADC7_GPIO1)
-+#define PMR735A_ADC7_GPIO2			(PMR735A_SID << 8 | ADC7_GPIO2)
-+#define PMR735A_ADC7_GPIO3			(PMR735A_SID << 8 | ADC7_GPIO3)
- 
- /* 100k pull-up2 */
-#define PMR735A_ADC7_GPIO1_100K_PU		(PMR735A_SID << 8 | 0x4a)
-#define PMR735A_ADC7_GPIO2_100K_PU		(PMR735A_SID << 8 | 0x4b)
-#define PMR735A_ADC7_GPIO3_100K_PU		(PMR735A_SID << 8 | 0x4c)
-+#define PMR735A_ADC7_GPIO1_100K_PU		(PMR735A_SID << 8 | ADC7_GPIO1_100K_PU)
-+#define PMR735A_ADC7_GPIO2_100K_PU		(PMR735A_SID << 8 | ADC7_GPIO2_100K_PU)
-+#define PMR735A_ADC7_GPIO3_100K_PU		(PMR735A_SID << 8 | ADC7_GPIO3_100K_PU)
- 
- #endif /* _DT_BINDINGS_QCOM_SPMI_VADC_PMR735A_H */
-diff --git a/include/dt-bindings/iio/qcom,spmi-adc7-pmr735b.h b/include/dt-bindings/iio/qcom,spmi-adc7-pmr735b.h
-index 8da0e7dab315..fdb8dd9ae541 100644
--- a/include/dt-bindings/iio/qcom,spmi-adc7-pmr735b.h
-+++ b/include/dt-bindings/iio/qcom,spmi-adc7-pmr735b.h
-@@ -10,19 +10,21 @@
- #define PMR735B_SID					5
- #endif
- 
-+#include <dt-bindings/iio/qcom,spmi-vadc.h>
-+
- /* ADC channels for PMR735B_ADC for PMIC7 */
-#define PMR735B_ADC7_REF_GND			(PMR735B_SID << 8 | 0x0)
-#define PMR735B_ADC7_1P25VREF			(PMR735B_SID << 8 | 0x01)
-#define PMR735B_ADC7_VREF_VADC			(PMR735B_SID << 8 | 0x02)
-#define PMR735B_ADC7_DIE_TEMP			(PMR735B_SID << 8 | 0x03)
-+#define PMR735B_ADC7_REF_GND			(PMR735B_SID << 8 | ADC7_REF_GND)
-+#define PMR735B_ADC7_1P25VREF			(PMR735B_SID << 8 | ADC7_1P25VREF)
-+#define PMR735B_ADC7_VREF_VADC			(PMR735B_SID << 8 | ADC7_VREF_VADC)
-+#define PMR735B_ADC7_DIE_TEMP			(PMR735B_SID << 8 | ADC7_DIE_TEMP)
- 
-#define PMR735B_ADC7_GPIO1			(PMR735B_SID << 8 | 0x0a)
-#define PMR735B_ADC7_GPIO2			(PMR735B_SID << 8 | 0x0b)
-#define PMR735B_ADC7_GPIO3			(PMR735B_SID << 8 | 0x0c)
-+#define PMR735B_ADC7_GPIO1			(PMR735B_SID << 8 | ADC7_GPIO1)
-+#define PMR735B_ADC7_GPIO2			(PMR735B_SID << 8 | ADC7_GPIO2)
-+#define PMR735B_ADC7_GPIO3			(PMR735B_SID << 8 | ADC7_GPIO3)
- 
- /* 100k pull-up2 */
-#define PMR735B_ADC7_GPIO1_100K_PU		(PMR735B_SID << 8 | 0x4a)
-#define PMR735B_ADC7_GPIO2_100K_PU		(PMR735B_SID << 8 | 0x4b)
-#define PMR735B_ADC7_GPIO3_100K_PU		(PMR735B_SID << 8 | 0x4c)
-+#define PMR735B_ADC7_GPIO1_100K_PU		(PMR735B_SID << 8 | ADC7_GPIO1_100K_PU)
-+#define PMR735B_ADC7_GPIO2_100K_PU		(PMR735B_SID << 8 | ADC7_GPIO2_100K_PU)
-+#define PMR735B_ADC7_GPIO3_100K_PU		(PMR735B_SID << 8 | ADC7_GPIO3_100K_PU)
- 
- #endif /* _DT_BINDINGS_QCOM_SPMI_VADC_PMR735B_H */
-diff --git a/include/dt-bindings/iio/qcom,spmi-adc7-smb139x.h b/include/dt-bindings/iio/qcom,spmi-adc7-smb139x.h
-new file mode 100644
-index 000000000000..c0680d1285cf
--- /dev/null
-+++ b/include/dt-bindings/iio/qcom,spmi-adc7-smb139x.h
-@@ -0,0 +1,19 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */
-+/*
-+ * Copyright (c) 2020 The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_QCOM_SPMI_VADC_SMB139X_H
-+#define _DT_BINDINGS_QCOM_SPMI_VADC_SMB139X_H
-+
-+#include <dt-bindings/iio/qcom,spmi-vadc.h>
-+
-+#define SMB139x_1_ADC7_SMB_TEMP			(SMB139x_1_SID << 8 | ADC7_SMB_TEMP)
-+#define SMB139x_1_ADC7_ICHG_SMB			(SMB139x_1_SID << 8 | ADC7_ICHG_SMB)
-+#define SMB139x_1_ADC7_IIN_SMB			(SMB139x_1_SID << 8 | ADC7_IIN_SMB)
-+
-+#define SMB139x_2_ADC7_SMB_TEMP			(SMB139x_2_SID << 8 | ADC7_SMB_TEMP)
-+#define SMB139x_2_ADC7_ICHG_SMB			(SMB139x_2_SID << 8 | ADC7_ICHG_SMB)
-+#define SMB139x_2_ADC7_IIN_SMB			(SMB139x_2_SID << 8 | ADC7_IIN_SMB)
-+
-+#endif
-diff --git a/include/dt-bindings/iio/qcom,spmi-vadc.h b/include/dt-bindings/iio/qcom,spmi-vadc.h
-index 08adfe25964c..ef07ecd4d585 100644
--- a/include/dt-bindings/iio/qcom,spmi-vadc.h
-+++ b/include/dt-bindings/iio/qcom,spmi-vadc.h
-@@ -239,12 +239,15 @@
- #define ADC7_GPIO3				0x0c
- #define ADC7_GPIO4				0x0d
- 
-+#define ADC7_SMB_TEMP				0x06
- #define ADC7_CHG_TEMP				0x10
- #define ADC7_USB_IN_V_16			0x11
- #define ADC7_VDC_16				0x12
- #define ADC7_CC1_ID				0x13
- #define ADC7_VREF_BAT_THERM			0x15
- #define ADC7_IIN_FB				0x17
-+#define ADC7_ICHG_SMB				0x18
-+#define ADC7_IIN_SMB				0x19
- 
- /* 30k pull-up1 */
- #define ADC7_AMUX_THM1_30K_PU			0x24
-diff --git a/include/dt-bindings/input/cros-ec-keyboard.h b/include/dt-bindings/input/cros-ec-keyboard.h
-index f0ae03634a96..afc12f6aa642 100644
--- a/include/dt-bindings/input/cros-ec-keyboard.h
-+++ b/include/dt-bindings/input/cros-ec-keyboard.h
-@@ -100,4 +100,108 @@
- 	MATRIX_KEY(0x07, 0x0b, KEY_UP)		\
- 	MATRIX_KEY(0x07, 0x0c, KEY_LEFT)
- 
-+/* No numpad */
-+#define CROS_TOP_ROW_KEYMAP_V30 \
-+	MATRIX_KEY(0x00, 0x01, KEY_F11)		/* T11 */	\
-+	MATRIX_KEY(0x00, 0x02, KEY_F1)		/* T1 */	\
-+	MATRIX_KEY(0x00, 0x04, KEY_F10)		/* T10 */	\
-+	MATRIX_KEY(0x00, 0x0b, KEY_F14)		/* T14 */	\
-+	MATRIX_KEY(0x00, 0x0c, KEY_F15)		/* T15 */	\
-+	MATRIX_KEY(0x01, 0x02, KEY_F4)		/* T4 */	\
-+	MATRIX_KEY(0x01, 0x04, KEY_F7)		/* T7 */	\
-+	MATRIX_KEY(0x01, 0x05, KEY_F12)		/* T12 */	\
-+	MATRIX_KEY(0x01, 0x09, KEY_F9)		/* T9 */	\
-+	MATRIX_KEY(0x02, 0x02, KEY_F3)		/* T3 */	\
-+	MATRIX_KEY(0x02, 0x04, KEY_F6)		/* T6 */	\
-+	MATRIX_KEY(0x02, 0x0b, KEY_F8)		/* T8 */	\
-+	MATRIX_KEY(0x03, 0x02, KEY_F2)		/* T2 */	\
-+	MATRIX_KEY(0x03, 0x05, KEY_F13)		/* T13 */	\
-+	MATRIX_KEY(0x04, 0x04, KEY_F5)		/* T5 */
-+
-+#define CROS_MAIN_KEYMAP_V30			/* Keycode */	\
-+	MATRIX_KEY(0x00, 0x03, KEY_B)		/* 50 */	\
-+	MATRIX_KEY(0x00, 0x05, KEY_N)		/* 51 */	\
-+	MATRIX_KEY(0x00, 0x06, KEY_RO)		/* 56 (JIS) */	\
-+	MATRIX_KEY(0x00, 0x08, KEY_EQUAL)	/* 13 */	\
-+	MATRIX_KEY(0x00, 0x09, KEY_HOME)	/* 80 (Numpad) */	\
-+	MATRIX_KEY(0x00, 0x0a, KEY_RIGHTALT)	/* 62 */	\
-+	MATRIX_KEY(0x00, 0x10, KEY_FN)		/* 127 */	\
-+								\
-+	MATRIX_KEY(0x01, 0x01, KEY_ESC)		/* 110 */	\
-+	MATRIX_KEY(0x01, 0x03, KEY_G)		/* 35 */	\
-+	MATRIX_KEY(0x01, 0x06, KEY_H)		/* 36 */	\
-+	MATRIX_KEY(0x01, 0x08, KEY_APOSTROPHE)	/* 41 */	\
-+	MATRIX_KEY(0x01, 0x0b, KEY_BACKSPACE)	/* 15 */	\
-+	MATRIX_KEY(0x01, 0x0c, KEY_HENKAN)	/* 65 (JIS) */	\
-+	MATRIX_KEY(0x01, 0x0e, KEY_LEFTCTRL)	/* 58 */	\
-+								\
-+	MATRIX_KEY(0x02, 0x01, KEY_TAB)		/* 16 */	\
-+	MATRIX_KEY(0x02, 0x03, KEY_T)		/* 21 */	\
-+	MATRIX_KEY(0x02, 0x05, KEY_RIGHTBRACE)	/* 28 */	\
-+	MATRIX_KEY(0x02, 0x06, KEY_Y)		/* 22 */	\
-+	MATRIX_KEY(0x02, 0x08, KEY_LEFTBRACE)	/* 27 */	\
-+	MATRIX_KEY(0x02, 0x09, KEY_DELETE)	/* 76 (Numpad) */	\
-+	MATRIX_KEY(0x02, 0x0c, KEY_PAGEUP)	/* 85 (Numpad) */	\
-+	MATRIX_KEY(0x02, 0x011, KEY_YEN)	/* 14 (JIS) */	\
-+								\
-+	MATRIX_KEY(0x03, 0x00, KEY_LEFTMETA)	/* Launcher */	\
-+	MATRIX_KEY(0x03, 0x01, KEY_GRAVE)	/* 1 */	\
-+	MATRIX_KEY(0x03, 0x03, KEY_5)		/* 6 */	\
-+	MATRIX_KEY(0x03, 0x04, KEY_S)		/* 32 */	\
-+	MATRIX_KEY(0x03, 0x06, KEY_MINUS)	/* 12 */	\
-+	MATRIX_KEY(0x03, 0x08, KEY_6)		/* 7 */		\
-+	MATRIX_KEY(0x03, 0x09, KEY_SLEEP)	/* Lock */	\
-+	MATRIX_KEY(0x03, 0x0b, KEY_BACKSLASH)	/* 29 */	\
-+	MATRIX_KEY(0x03, 0x0c, KEY_MUHENKAN)	/* 63 (JIS) */	\
-+	MATRIX_KEY(0x03, 0x0e, KEY_RIGHTCTRL)	/* 64 */	\
-+								\
-+	MATRIX_KEY(0x04, 0x01, KEY_A)		/* 31 */	\
-+	MATRIX_KEY(0x04, 0x02, KEY_D)		/* 33 */	\
-+	MATRIX_KEY(0x04, 0x03, KEY_F)		/* 34 */	\
-+	MATRIX_KEY(0x04, 0x05, KEY_K)		/* 38 */	\
-+	MATRIX_KEY(0x04, 0x06, KEY_J)		/* 37 */	\
-+	MATRIX_KEY(0x04, 0x08, KEY_SEMICOLON)	/* 40 */	\
-+	MATRIX_KEY(0x04, 0x09, KEY_L)		/* 39 */	\
-+	MATRIX_KEY(0x04, 0x0b, KEY_ENTER)	/* 43 */	\
-+	MATRIX_KEY(0x04, 0x0c, KEY_END)		/* 81 (Numpad) */	\
-+								\
-+	MATRIX_KEY(0x05, 0x01, KEY_1)		/* 2 */	\
-+	MATRIX_KEY(0x05, 0x02, KEY_COMMA)	/* 53 */	\
-+	MATRIX_KEY(0x05, 0x03, KEY_DOT)		/* 54 */	\
-+	MATRIX_KEY(0x05, 0x04, KEY_SLASH)	/* 55 */	\
-+	MATRIX_KEY(0x05, 0x05, KEY_C)		/* 48 */	\
-+	MATRIX_KEY(0x05, 0x06, KEY_SPACE)	/* 61 */	\
-+	MATRIX_KEY(0x05, 0x07, KEY_LEFTSHIFT)	/* 44 */	\
-+	MATRIX_KEY(0x05, 0x08, KEY_X)		/* 47 */	\
-+	MATRIX_KEY(0x05, 0x09, KEY_V)		/* 49 */	\
-+	MATRIX_KEY(0x05, 0x0b, KEY_M)		/* 52 */	\
-+	MATRIX_KEY(0x05, 0x0c, KEY_PAGEDOWN)	/* 86 (Numpad) */	\
-+								\
-+	MATRIX_KEY(0x06, 0x01, KEY_Z)		/* 46 */	\
-+	MATRIX_KEY(0x06, 0x02, KEY_3)		/* 4 */		\
-+	MATRIX_KEY(0x06, 0x03, KEY_4)		/* 5 */		\
-+	MATRIX_KEY(0x06, 0x04, KEY_2)		/* 3 */		\
-+	MATRIX_KEY(0x06, 0x05, KEY_8)		/* 9 */		\
-+	MATRIX_KEY(0x06, 0x06, KEY_0)		/* 11 */	\
-+	MATRIX_KEY(0x06, 0x08, KEY_7)		/* 8 */		\
-+	MATRIX_KEY(0x06, 0x09, KEY_9)		/* 10 */	\
-+	MATRIX_KEY(0x06, 0x0b, KEY_DOWN)	/* 84 */	\
-+	MATRIX_KEY(0x06, 0x0c, KEY_RIGHT)	/* 89 */	\
-+	MATRIX_KEY(0x06, 0x0d, KEY_LEFTALT)	/* 60 */	\
-+	MATRIX_KEY(0x06, 0x0f, KEY_ASSISTANT)	/* 128 */	\
-+	MATRIX_KEY(0x06, 0x11, KEY_BACKSLASH)	/* 42 (JIS, ISO) */	\
-+								\
-+	MATRIX_KEY(0x07, 0x01, KEY_U)		/* 23 */	\
-+	MATRIX_KEY(0x07, 0x02, KEY_I)		/* 24 */	\
-+	MATRIX_KEY(0x07, 0x03, KEY_O)		/* 25 */	\
-+	MATRIX_KEY(0x07, 0x04, KEY_P)		/* 26 */	\
-+	MATRIX_KEY(0x07, 0x05, KEY_Q)		/* 17 */	\
-+	MATRIX_KEY(0x07, 0x06, KEY_W)		/* 18 */	\
-+	MATRIX_KEY(0x07, 0x07, KEY_RIGHTSHIFT)	/* 57 */	\
-+	MATRIX_KEY(0x07, 0x08, KEY_E)		/* 19 */	\
-+	MATRIX_KEY(0x07, 0x09, KEY_R)		/* 20 */	\
-+	MATRIX_KEY(0x07, 0x0b, KEY_UP)		/* 83 */	\
-+	MATRIX_KEY(0x07, 0x0c, KEY_LEFT)	/* 79 */	\
-+	MATRIX_KEY(0x07, 0x11, KEY_102ND)	/* 45 (ISO) */
-+
- #endif /* _CROS_EC_KEYBOARD_H */
-diff --git a/include/dt-bindings/interconnect/mediatek,mt8183.h b/include/dt-bindings/interconnect/mediatek,mt8183.h
-new file mode 100644
-index 000000000000..1088c350258d
--- /dev/null
-+++ b/include/dt-bindings/interconnect/mediatek,mt8183.h
-@@ -0,0 +1,23 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2021 MediaTek Inc.
-+ * Copyright (c) 2024 Collabora Ltd.
-+ *                    AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_MEDIATEK_MT8183_H
-+#define __DT_BINDINGS_INTERCONNECT_MEDIATEK_MT8183_H
-+
-+#define SLAVE_DDR_EMI		0
-+#define MASTER_MCUSYS		1
-+#define MASTER_MFG		2
-+#define MASTER_MMSYS		3
-+#define MASTER_MM_VPU		4
-+#define MASTER_MM_DISP		5
-+#define MASTER_MM_VDEC		6
-+#define MASTER_MM_VENC		7
-+#define MASTER_MM_CAM		8
-+#define MASTER_MM_IMG		9
-+#define MASTER_MM_MDP		10
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/mediatek,mt8195.h b/include/dt-bindings/interconnect/mediatek,mt8195.h
-new file mode 100644
-index 000000000000..33e0e6cde732
--- /dev/null
-+++ b/include/dt-bindings/interconnect/mediatek,mt8195.h
-@@ -0,0 +1,44 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2020 MediaTek Inc.
-+ * Copyright (c) 2024 Collabora Ltd.
-+ *                    AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_MEDIATEK_MT8195_H
-+#define __DT_BINDINGS_INTERCONNECT_MEDIATEK_MT8195_H
-+
-+#define SLAVE_DDR_EMI		0
-+#define MASTER_MCUSYS		1
-+#define MASTER_GPUSYS		2
-+#define MASTER_MMSYS		3
-+#define MASTER_MM_VPU		4
-+#define MASTER_MM_DISP		5
-+#define MASTER_MM_VDEC		6
-+#define MASTER_MM_VENC		7
-+#define MASTER_MM_CAM		8
-+#define MASTER_MM_IMG		9
-+#define MASTER_MM_MDP		10
-+#define MASTER_VPUSYS		11
-+#define MASTER_VPU_0		12
-+#define MASTER_VPU_1		13
-+#define MASTER_MDLASYS		14
-+#define MASTER_MDLA_0		15
-+#define MASTER_UFS		16
-+#define MASTER_PCIE_0		17
-+#define MASTER_PCIE_1		18
-+#define MASTER_USB		19
-+#define MASTER_DBGIF		20
-+#define SLAVE_HRT_DDR_EMI	21
-+#define MASTER_HRT_MMSYS	22
-+#define MASTER_HRT_MM_DISP	23
-+#define MASTER_HRT_MM_VDEC	24
-+#define MASTER_HRT_MM_VENC	25
-+#define MASTER_HRT_MM_CAM	26
-+#define MASTER_HRT_MM_IMG	27
-+#define MASTER_HRT_MM_MDP	28
-+#define MASTER_HRT_DBGIF	29
-+#define MASTER_WIFI		30
-+#define MASTER_BT		31
-+#define MASTER_NETSYS		32
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,glymur-rpmh.h b/include/dt-bindings/interconnect/qcom,glymur-rpmh.h
-new file mode 100644
-index 000000000000..6a0e754345e4
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,glymur-rpmh.h
-@@ -0,0 +1,205 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2025, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_GLYMUR_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_GLYMUR_H
-+
-+#define MASTER_CRYPTO				0
-+#define MASTER_SOCCP_PROC			1
-+#define MASTER_QDSS_ETR				2
-+#define MASTER_QDSS_ETR_1			3
-+#define SLAVE_A1NOC_SNOC			4
-+
-+#define MASTER_UFS_MEM				0
-+#define MASTER_USB3_2				1
-+#define MASTER_USB4_2				2
-+#define SLAVE_A2NOC_SNOC			3
-+
-+#define MASTER_QSPI_0				0
-+#define MASTER_QUP_0				1
-+#define MASTER_QUP_1				2
-+#define MASTER_QUP_2				3
-+#define MASTER_SP				4
-+#define MASTER_SDCC_2				5
-+#define MASTER_SDCC_4				6
-+#define MASTER_USB2				7
-+#define MASTER_USB3_MP				8
-+#define SLAVE_A3NOC_SNOC			9
-+
-+#define MASTER_USB3_0				0
-+#define MASTER_USB3_1				1
-+#define MASTER_USB4_0				2
-+#define MASTER_USB4_1				3
-+#define SLAVE_A4NOC_HSCNOC			4
-+
-+#define MASTER_QUP_CORE_0			0
-+#define MASTER_QUP_CORE_1			1
-+#define MASTER_QUP_CORE_2			2
-+#define SLAVE_QUP_CORE_0			3
-+#define SLAVE_QUP_CORE_1			4
-+#define SLAVE_QUP_CORE_2			5
-+
-+#define MASTER_CNOC_CFG				0
-+#define SLAVE_AHB2PHY_SOUTH			1
-+#define SLAVE_AHB2PHY_NORTH			2
-+#define SLAVE_AHB2PHY_2				3
-+#define SLAVE_AHB2PHY_3				4
-+#define SLAVE_AV1_ENC_CFG			5
-+#define SLAVE_CAMERA_CFG			6
-+#define SLAVE_CLK_CTL				7
-+#define SLAVE_CRYPTO_0_CFG			8
-+#define SLAVE_DISPLAY_CFG			9
-+#define SLAVE_GFX3D_CFG				10
-+#define SLAVE_IMEM_CFG				11
-+#define SLAVE_PCIE_0_CFG			12
-+#define SLAVE_PCIE_1_CFG			13
-+#define SLAVE_PCIE_2_CFG			14
-+#define SLAVE_PCIE_3A_CFG			15
-+#define SLAVE_PCIE_3B_CFG			16
-+#define SLAVE_PCIE_4_CFG			17
-+#define SLAVE_PCIE_5_CFG			18
-+#define SLAVE_PCIE_6_CFG			19
-+#define SLAVE_PCIE_RSCC				20
-+#define SLAVE_PDM				21
-+#define SLAVE_PRNG				22
-+#define SLAVE_QDSS_CFG				23
-+#define SLAVE_QSPI_0				24
-+#define SLAVE_QUP_0				25
-+#define SLAVE_QUP_1				26
-+#define SLAVE_QUP_2				27
-+#define SLAVE_SDCC_2				28
-+#define SLAVE_SDCC_4				29
-+#define SLAVE_SMMUV3_CFG			30
-+#define SLAVE_TCSR				31
-+#define SLAVE_TLMM				32
-+#define SLAVE_UFS_MEM_CFG			33
-+#define SLAVE_USB2				34
-+#define SLAVE_USB3_0				35
-+#define SLAVE_USB3_1				36
-+#define SLAVE_USB3_2				37
-+#define SLAVE_USB3_MP				38
-+#define SLAVE_USB4_0				39
-+#define SLAVE_USB4_1				40
-+#define SLAVE_USB4_2				41
-+#define SLAVE_VENUS_CFG				42
-+#define SLAVE_CNOC_PCIE_SLAVE_EAST_CFG		43
-+#define SLAVE_CNOC_PCIE_SLAVE_WEST_CFG		44
-+#define SLAVE_LPASS_QTB_CFG			45
-+#define SLAVE_CNOC_MNOC_CFG			46
-+#define SLAVE_NSP_QTB_CFG			47
-+#define SLAVE_PCIE_EAST_ANOC_CFG		48
-+#define SLAVE_PCIE_WEST_ANOC_CFG		49
-+#define SLAVE_QDSS_STM				50
-+#define SLAVE_TCU				51
-+
-+#define MASTER_HSCNOC_CNOC			0
-+#define SLAVE_AOSS				1
-+#define SLAVE_IPC_ROUTER_CFG			2
-+#define SLAVE_SOCCP				3
-+#define SLAVE_TME_CFG				4
-+#define SLAVE_APPSS				5
-+#define SLAVE_CNOC_CFG				6
-+#define SLAVE_BOOT_IMEM				7
-+#define SLAVE_IMEM				8
-+
-+#define MASTER_GPU_TCU				0
-+#define MASTER_PCIE_TCU				1
-+#define MASTER_SYS_TCU				2
-+#define MASTER_APPSS_PROC			3
-+#define MASTER_AGGRE_NOC_EAST			4
-+#define MASTER_GFX3D				5
-+#define MASTER_LPASS_GEM_NOC			6
-+#define MASTER_MNOC_HF_MEM_NOC			7
-+#define MASTER_MNOC_SF_MEM_NOC			8
-+#define MASTER_COMPUTE_NOC			9
-+#define MASTER_PCIE_EAST			10
-+#define MASTER_PCIE_WEST			11
-+#define MASTER_SNOC_SF_MEM_NOC			12
-+#define MASTER_WLAN_Q6				13
-+#define MASTER_GIC				14
-+#define SLAVE_HSCNOC_CNOC			15
-+#define SLAVE_LLCC				16
-+#define SLAVE_PCIE_EAST				17
-+#define SLAVE_PCIE_WEST				18
-+
-+#define MASTER_LPIAON_NOC			0
-+#define SLAVE_LPASS_GEM_NOC			1
-+
-+#define MASTER_LPASS_LPINOC			0
-+#define SLAVE_LPIAON_NOC_LPASS_AG_NOC		1
-+
-+#define MASTER_LPASS_PROC			0
-+#define SLAVE_LPICX_NOC_LPIAON_NOC		1
-+
-+#define MASTER_LLCC				0
-+#define SLAVE_EBI1				1
-+
-+#define MASTER_AV1_ENC				0
-+#define MASTER_CAMNOC_HF			1
-+#define MASTER_CAMNOC_ICP			2
-+#define MASTER_CAMNOC_SF			3
-+#define MASTER_EVA				4
-+#define MASTER_MDP				5
-+#define MASTER_CDSP_HCP				6
-+#define MASTER_VIDEO				7
-+#define MASTER_VIDEO_CV_PROC			8
-+#define MASTER_VIDEO_V_PROC			9
-+#define MASTER_CNOC_MNOC_CFG			10
-+#define SLAVE_MNOC_HF_MEM_NOC			11
-+#define SLAVE_MNOC_SF_MEM_NOC			12
-+#define SLAVE_SERVICE_MNOC			13
-+
-+#define MASTER_CPUCP				0
-+#define SLAVE_NSINOC_SYSTEM_NOC			1
-+#define SLAVE_SERVICE_NSINOC			2
-+
-+#define MASTER_CDSP_PROC			0
-+#define SLAVE_NSP0_HSC_NOC			1
-+
-+#define MASTER_OOBMSS_SP_PROC			0
-+#define SLAVE_OOBMSS_SNOC			1
-+
-+#define MASTER_PCIE_EAST_ANOC_CFG		0
-+#define MASTER_PCIE_0				1
-+#define MASTER_PCIE_1				2
-+#define MASTER_PCIE_5				3
-+#define SLAVE_PCIE_EAST_MEM_NOC			4
-+#define SLAVE_SERVICE_PCIE_EAST_AGGRE_NOC	5
-+
-+#define MASTER_HSCNOC_PCIE_EAST			0
-+#define MASTER_CNOC_PCIE_EAST_SLAVE_CFG		1
-+#define SLAVE_HSCNOC_PCIE_EAST_MS_MPU_CFG	2
-+#define SLAVE_SERVICE_PCIE_EAST			3
-+#define SLAVE_PCIE_0				4
-+#define SLAVE_PCIE_1				5
-+#define SLAVE_PCIE_5				6
-+
-+#define MASTER_PCIE_WEST_ANOC_CFG		0
-+#define MASTER_PCIE_2				1
-+#define MASTER_PCIE_3A				2
-+#define MASTER_PCIE_3B				3
-+#define MASTER_PCIE_4				4
-+#define MASTER_PCIE_6				5
-+#define SLAVE_PCIE_WEST_MEM_NOC			6
-+#define SLAVE_SERVICE_PCIE_WEST_AGGRE_NOC	7
-+
-+#define MASTER_HSCNOC_PCIE_WEST			0
-+#define MASTER_CNOC_PCIE_WEST_SLAVE_CFG		1
-+#define SLAVE_HSCNOC_PCIE_WEST_MS_MPU_CFG	2
-+#define SLAVE_SERVICE_PCIE_WEST			3
-+#define SLAVE_PCIE_2				4
-+#define SLAVE_PCIE_3A				5
-+#define SLAVE_PCIE_3B				6
-+#define SLAVE_PCIE_4				7
-+#define SLAVE_PCIE_6				8
-+
-+#define MASTER_A1NOC_SNOC			0
-+#define MASTER_A2NOC_SNOC			1
-+#define MASTER_A3NOC_SNOC			2
-+#define MASTER_NSINOC_SNOC			3
-+#define MASTER_OOBMSS				4
-+#define SLAVE_SNOC_GEM_NOC_SF			5
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,ipq5332.h b/include/dt-bindings/interconnect/qcom,ipq5332.h
-new file mode 100644
-index 000000000000..16475bb07a48
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,ipq5332.h
-@@ -0,0 +1,46 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+#ifndef INTERCONNECT_QCOM_IPQ5332_H
-+#define INTERCONNECT_QCOM_IPQ5332_H
-+
-+#define MASTER_SNOC_PCIE3_1_M		0
-+#define SLAVE_SNOC_PCIE3_1_M		1
-+#define MASTER_ANOC_PCIE3_1_S		2
-+#define SLAVE_ANOC_PCIE3_1_S		3
-+#define MASTER_SNOC_PCIE3_2_M		4
-+#define SLAVE_SNOC_PCIE3_2_M		5
-+#define MASTER_ANOC_PCIE3_2_S		6
-+#define SLAVE_ANOC_PCIE3_2_S		7
-+#define MASTER_SNOC_USB			8
-+#define SLAVE_SNOC_USB			9
-+#define MASTER_NSSNOC_NSSCC		10
-+#define SLAVE_NSSNOC_NSSCC		11
-+#define MASTER_NSSNOC_SNOC_0		12
-+#define SLAVE_NSSNOC_SNOC_0		13
-+#define MASTER_NSSNOC_SNOC_1		14
-+#define SLAVE_NSSNOC_SNOC_1		15
-+#define MASTER_NSSNOC_ATB		16
-+#define SLAVE_NSSNOC_ATB		17
-+#define MASTER_NSSNOC_PCNOC_1		18
-+#define SLAVE_NSSNOC_PCNOC_1		19
-+#define MASTER_NSSNOC_QOSGEN_REF	20
-+#define SLAVE_NSSNOC_QOSGEN_REF		21
-+#define MASTER_NSSNOC_TIMEOUT_REF	22
-+#define SLAVE_NSSNOC_TIMEOUT_REF	23
-+#define MASTER_NSSNOC_XO_DCD		24
-+#define SLAVE_NSSNOC_XO_DCD		25
-+
-+#define MASTER_NSSNOC_PPE		0
-+#define SLAVE_NSSNOC_PPE		1
-+#define MASTER_NSSNOC_PPE_CFG		2
-+#define SLAVE_NSSNOC_PPE_CFG		3
-+#define MASTER_NSSNOC_NSS_CSR		4
-+#define SLAVE_NSSNOC_NSS_CSR		5
-+#define MASTER_NSSNOC_CE_APB		6
-+#define SLAVE_NSSNOC_CE_APB		7
-+#define MASTER_NSSNOC_CE_AXI		8
-+#define SLAVE_NSSNOC_CE_AXI		9
-+
-+#define MASTER_CNOC_AHB			0
-+#define SLAVE_CNOC_AHB			1
-+
-+#endif /* INTERCONNECT_QCOM_IPQ5332_H */
-diff --git a/include/dt-bindings/interconnect/qcom,ipq5424.h b/include/dt-bindings/interconnect/qcom,ipq5424.h
-new file mode 100644
-index 000000000000..07b786bee7d6
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,ipq5424.h
-@@ -0,0 +1,60 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+#ifndef INTERCONNECT_QCOM_IPQ5424_H
-+#define INTERCONNECT_QCOM_IPQ5424_H
-+
-+#define MASTER_ANOC_PCIE0		0
-+#define SLAVE_ANOC_PCIE0		1
-+#define MASTER_CNOC_PCIE0		2
-+#define SLAVE_CNOC_PCIE0		3
-+#define MASTER_ANOC_PCIE1		4
-+#define SLAVE_ANOC_PCIE1		5
-+#define MASTER_CNOC_PCIE1		6
-+#define SLAVE_CNOC_PCIE1		7
-+#define MASTER_ANOC_PCIE2		8
-+#define SLAVE_ANOC_PCIE2		9
-+#define MASTER_CNOC_PCIE2		10
-+#define SLAVE_CNOC_PCIE2		11
-+#define MASTER_ANOC_PCIE3		12
-+#define SLAVE_ANOC_PCIE3		13
-+#define MASTER_CNOC_PCIE3		14
-+#define SLAVE_CNOC_PCIE3		15
-+#define MASTER_CNOC_USB			16
-+#define SLAVE_CNOC_USB			17
-+#define MASTER_NSSNOC_NSSCC		18
-+#define SLAVE_NSSNOC_NSSCC		19
-+#define MASTER_NSSNOC_SNOC_0		20
-+#define SLAVE_NSSNOC_SNOC_0		21
-+#define MASTER_NSSNOC_SNOC_1		22
-+#define SLAVE_NSSNOC_SNOC_1		23
-+#define MASTER_NSSNOC_PCNOC_1		24
-+#define SLAVE_NSSNOC_PCNOC_1		25
-+#define MASTER_NSSNOC_QOSGEN_REF	26
-+#define SLAVE_NSSNOC_QOSGEN_REF		27
-+#define MASTER_NSSNOC_TIMEOUT_REF	28
-+#define SLAVE_NSSNOC_TIMEOUT_REF	29
-+#define MASTER_NSSNOC_XO_DCD		30
-+#define SLAVE_NSSNOC_XO_DCD		31
-+#define MASTER_NSSNOC_ATB		32
-+#define SLAVE_NSSNOC_ATB		33
-+#define MASTER_CNOC_LPASS_CFG		34
-+#define SLAVE_CNOC_LPASS_CFG		35
-+#define MASTER_SNOC_LPASS		36
-+#define SLAVE_SNOC_LPASS		37
-+
-+#define MASTER_CPU			0
-+#define SLAVE_L3			1
-+
-+#define MASTER_NSSNOC_PPE		0
-+#define SLAVE_NSSNOC_PPE		1
-+#define MASTER_NSSNOC_PPE_CFG		2
-+#define SLAVE_NSSNOC_PPE_CFG		3
-+#define MASTER_NSSNOC_NSS_CSR		4
-+#define SLAVE_NSSNOC_NSS_CSR		5
-+#define MASTER_NSSNOC_CE_AXI		6
-+#define SLAVE_NSSNOC_CE_AXI		7
-+#define MASTER_NSSNOC_CE_APB		8
-+#define SLAVE_NSSNOC_CE_APB		9
-+#define MASTER_NSSNOC_EIP		10
-+#define SLAVE_NSSNOC_EIP		11
-+
-+#endif /* INTERCONNECT_QCOM_IPQ5424_H */
-diff --git a/include/dt-bindings/interconnect/qcom,ipq9574.h b/include/dt-bindings/interconnect/qcom,ipq9574.h
-new file mode 100644
-index 000000000000..42019335c7dd
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,ipq9574.h
-@@ -0,0 +1,59 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+#ifndef INTERCONNECT_QCOM_IPQ9574_H
-+#define INTERCONNECT_QCOM_IPQ9574_H
-+
-+#define MASTER_ANOC_PCIE0		0
-+#define SLAVE_ANOC_PCIE0		1
-+#define MASTER_SNOC_PCIE0		2
-+#define SLAVE_SNOC_PCIE0		3
-+#define MASTER_ANOC_PCIE1		4
-+#define SLAVE_ANOC_PCIE1		5
-+#define MASTER_SNOC_PCIE1		6
-+#define SLAVE_SNOC_PCIE1		7
-+#define MASTER_ANOC_PCIE2		8
-+#define SLAVE_ANOC_PCIE2		9
-+#define MASTER_SNOC_PCIE2		10
-+#define SLAVE_SNOC_PCIE2		11
-+#define MASTER_ANOC_PCIE3		12
-+#define SLAVE_ANOC_PCIE3		13
-+#define MASTER_SNOC_PCIE3		14
-+#define SLAVE_SNOC_PCIE3		15
-+#define MASTER_USB			16
-+#define SLAVE_USB			17
-+#define MASTER_USB_AXI			18
-+#define SLAVE_USB_AXI			19
-+#define MASTER_NSSNOC_NSSCC		20
-+#define SLAVE_NSSNOC_NSSCC		21
-+#define MASTER_NSSNOC_SNOC_0		22
-+#define SLAVE_NSSNOC_SNOC_0		23
-+#define MASTER_NSSNOC_SNOC_1		24
-+#define SLAVE_NSSNOC_SNOC_1		25
-+#define MASTER_NSSNOC_PCNOC_1		26
-+#define SLAVE_NSSNOC_PCNOC_1		27
-+#define MASTER_NSSNOC_QOSGEN_REF	28
-+#define SLAVE_NSSNOC_QOSGEN_REF		29
-+#define MASTER_NSSNOC_TIMEOUT_REF	30
-+#define SLAVE_NSSNOC_TIMEOUT_REF	31
-+#define MASTER_NSSNOC_XO_DCD		32
-+#define SLAVE_NSSNOC_XO_DCD		33
-+#define MASTER_NSSNOC_ATB		34
-+#define SLAVE_NSSNOC_ATB		35
-+#define MASTER_MEM_NOC_NSSNOC		36
-+#define SLAVE_MEM_NOC_NSSNOC		37
-+#define MASTER_NSSNOC_MEMNOC		38
-+#define SLAVE_NSSNOC_MEMNOC		39
-+#define MASTER_NSSNOC_MEM_NOC_1		40
-+#define SLAVE_NSSNOC_MEM_NOC_1		41
-+
-+#define MASTER_NSSNOC_PPE		0
-+#define SLAVE_NSSNOC_PPE		1
-+#define MASTER_NSSNOC_PPE_CFG		2
-+#define SLAVE_NSSNOC_PPE_CFG		3
-+#define MASTER_NSSNOC_NSS_CSR		4
-+#define SLAVE_NSSNOC_NSS_CSR		5
-+#define MASTER_NSSNOC_IMEM_QSB		6
-+#define SLAVE_NSSNOC_IMEM_QSB		7
-+#define MASTER_NSSNOC_IMEM_AHB		8
-+#define SLAVE_NSSNOC_IMEM_AHB		9
-+
-+#endif /* INTERCONNECT_QCOM_IPQ9574_H */
-diff --git a/include/dt-bindings/interconnect/qcom,kaanapali-rpmh.h b/include/dt-bindings/interconnect/qcom,kaanapali-rpmh.h
-new file mode 100644
-index 000000000000..dde3f9abd677
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,kaanapali-rpmh.h
-@@ -0,0 +1,149 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_KAANAPALI_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_KAANAPALI_H
-+
-+#define MASTER_QSPI_0				0
-+#define MASTER_CRYPTO				1
-+#define MASTER_QUP_1				2
-+#define MASTER_SDCC_4				3
-+#define MASTER_UFS_MEM				4
-+#define MASTER_USB3				5
-+#define MASTER_QUP_2				6
-+#define MASTER_QUP_3				7
-+#define MASTER_QUP_4				8
-+#define MASTER_IPA				9
-+#define MASTER_SOCCP_PROC			10
-+#define MASTER_SP				11
-+#define MASTER_QDSS_ETR				12
-+#define MASTER_QDSS_ETR_1			13
-+#define MASTER_SDCC_2				14
-+#define SLAVE_A1NOC_SNOC			15
-+#define SLAVE_A2NOC_SNOC			16
-+
-+#define MASTER_QUP_CORE_0			0
-+#define MASTER_QUP_CORE_1			1
-+#define MASTER_QUP_CORE_2			2
-+#define MASTER_QUP_CORE_3			3
-+#define MASTER_QUP_CORE_4			4
-+#define SLAVE_QUP_CORE_0			5
-+#define SLAVE_QUP_CORE_1			6
-+#define SLAVE_QUP_CORE_2			7
-+#define SLAVE_QUP_CORE_3			8
-+#define SLAVE_QUP_CORE_4			9
-+
-+#define MASTER_CNOC_CFG				0
-+#define SLAVE_AHB2PHY_SOUTH			1
-+#define SLAVE_AHB2PHY_NORTH			2
-+#define SLAVE_CAMERA_CFG			3
-+#define SLAVE_CLK_CTL				4
-+#define SLAVE_CRYPTO_0_CFG			5
-+#define SLAVE_DISPLAY_CFG			6
-+#define SLAVE_EVA_CFG				7
-+#define SLAVE_GFX3D_CFG				8
-+#define SLAVE_I2C				9
-+#define SLAVE_I3C_IBI0_CFG			10
-+#define SLAVE_I3C_IBI1_CFG			11
-+#define SLAVE_IMEM_CFG				12
-+#define SLAVE_IPC_ROUTER_CFG			13
-+#define SLAVE_CNOC_MSS				14
-+#define SLAVE_PCIE_CFG				15
-+#define SLAVE_PRNG				16
-+#define SLAVE_QDSS_CFG				17
-+#define SLAVE_QSPI_0				18
-+#define SLAVE_QUP_1				19
-+#define SLAVE_QUP_2				20
-+#define SLAVE_QUP_3				21
-+#define SLAVE_QUP_4				22
-+#define SLAVE_SDCC_2				23
-+#define SLAVE_SDCC_4				24
-+#define SLAVE_SPSS_CFG				25
-+#define SLAVE_TCSR				26
-+#define SLAVE_TLMM				27
-+#define SLAVE_UFS_MEM_CFG			28
-+#define SLAVE_USB3				29
-+#define SLAVE_VENUS_CFG				30
-+#define SLAVE_VSENSE_CTRL_CFG			31
-+#define SLAVE_CNOC_MNOC_CFG			32
-+#define SLAVE_PCIE_ANOC_CFG			33
-+#define SLAVE_QDSS_STM				34
-+#define SLAVE_TCU				35
-+
-+#define MASTER_GEM_NOC_CNOC			0
-+#define MASTER_GEM_NOC_PCIE_SNOC		1
-+#define SLAVE_AOSS				2
-+#define SLAVE_IPA_CFG				3
-+#define SLAVE_IPC_ROUTER_FENCE			4
-+#define SLAVE_SOCCP				5
-+#define SLAVE_TME_CFG				6
-+#define SLAVE_APPSS				7
-+#define SLAVE_CNOC_CFG				8
-+#define SLAVE_DDRSS_CFG				9
-+#define SLAVE_BOOT_IMEM				10
-+#define SLAVE_IMEM				11
-+#define SLAVE_PCIE_0				12
-+
-+#define MASTER_GPU_TCU				0
-+#define MASTER_SYS_TCU				1
-+#define MASTER_APPSS_PROC			2
-+#define MASTER_GFX3D				3
-+#define MASTER_LPASS_GEM_NOC			4
-+#define MASTER_MSS_PROC				5
-+#define MASTER_MNOC_HF_MEM_NOC			6
-+#define MASTER_MNOC_SF_MEM_NOC			7
-+#define MASTER_COMPUTE_NOC			8
-+#define MASTER_ANOC_PCIE_GEM_NOC		9
-+#define MASTER_QPACE				10
-+#define MASTER_SNOC_SF_MEM_NOC			11
-+#define MASTER_WLAN_Q6				12
-+#define MASTER_GIC				13
-+#define SLAVE_GEM_NOC_CNOC			14
-+#define SLAVE_LLCC				15
-+#define SLAVE_MEM_NOC_PCIE_SNOC			16
-+
-+#define MASTER_LPIAON_NOC			0
-+#define SLAVE_LPASS_GEM_NOC			1
-+
-+#define MASTER_LPASS_LPINOC			0
-+#define SLAVE_LPIAON_NOC_LPASS_AG_NOC		1
-+
-+#define MASTER_LPASS_PROC			0
-+#define SLAVE_LPICX_NOC_LPIAON_NOC		1
-+
-+#define MASTER_LLCC				0
-+#define SLAVE_EBI1				1
-+
-+#define MASTER_CAMNOC_HF			0
-+#define MASTER_CAMNOC_NRT_ICP_SF		1
-+#define MASTER_CAMNOC_RT_CDM_SF			2
-+#define MASTER_CAMNOC_SF			3
-+#define MASTER_MDP				4
-+#define MASTER_MDSS_DCP				5
-+#define MASTER_CDSP_HCP				6
-+#define MASTER_VIDEO_CV_PROC			7
-+#define MASTER_VIDEO_EVA			8
-+#define MASTER_VIDEO_MVP			9
-+#define MASTER_VIDEO_V_PROC			10
-+#define MASTER_CNOC_MNOC_CFG			11
-+#define SLAVE_MNOC_HF_MEM_NOC			12
-+#define SLAVE_MNOC_SF_MEM_NOC			13
-+#define SLAVE_SERVICE_MNOC			14
-+
-+#define MASTER_CDSP_PROC			0
-+#define SLAVE_CDSP_MEM_NOC			1
-+
-+#define MASTER_PCIE_ANOC_CFG			0
-+#define MASTER_PCIE_0				1
-+#define SLAVE_ANOC_PCIE_GEM_NOC			2
-+#define SLAVE_SERVICE_PCIE_ANOC			3
-+
-+#define MASTER_A1NOC_SNOC			0
-+#define MASTER_A2NOC_SNOC			1
-+#define MASTER_APSS_NOC				2
-+#define MASTER_CNOC_SNOC			3
-+#define SLAVE_SNOC_GEM_NOC_SF			4
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,milos-rpmh.h b/include/dt-bindings/interconnect/qcom,milos-rpmh.h
-new file mode 100644
-index 000000000000..9326d7d9c2a3
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,milos-rpmh.h
-@@ -0,0 +1,141 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2025, Luca Weiss <luca.weiss@fairphone.com>
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_MILOS_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_MILOS_H
-+
-+#define MASTER_QUP_1				0
-+#define MASTER_UFS_MEM				1
-+#define MASTER_USB3_0				2
-+#define SLAVE_A1NOC_SNOC			3
-+
-+#define MASTER_QDSS_BAM				0
-+#define MASTER_QSPI_0				1
-+#define MASTER_QUP_0				2
-+#define MASTER_CRYPTO				3
-+#define MASTER_IPA				4
-+#define MASTER_QDSS_ETR				5
-+#define MASTER_QDSS_ETR_1			6
-+#define MASTER_SDCC_1				7
-+#define MASTER_SDCC_2				8
-+#define SLAVE_A2NOC_SNOC			9
-+
-+#define MASTER_QUP_CORE_0			0
-+#define MASTER_QUP_CORE_1			1
-+#define SLAVE_QUP_CORE_0			2
-+#define SLAVE_QUP_CORE_1			3
-+
-+#define MASTER_CNOC_CFG				0
-+#define SLAVE_AHB2PHY_SOUTH			1
-+#define SLAVE_AHB2PHY_NORTH			2
-+#define SLAVE_CAMERA_CFG			3
-+#define SLAVE_CLK_CTL				4
-+#define SLAVE_RBCPR_CX_CFG			5
-+#define SLAVE_RBCPR_MXA_CFG			6
-+#define SLAVE_CRYPTO_0_CFG			7
-+#define SLAVE_CX_RDPM				8
-+#define SLAVE_GFX3D_CFG				9
-+#define SLAVE_IMEM_CFG				10
-+#define SLAVE_CNOC_MSS				11
-+#define SLAVE_MX_2_RDPM				12
-+#define SLAVE_MX_RDPM				13
-+#define SLAVE_PDM				14
-+#define SLAVE_QDSS_CFG				15
-+#define SLAVE_QSPI_0				16
-+#define SLAVE_QUP_0				17
-+#define SLAVE_QUP_1				18
-+#define SLAVE_SDC1				19
-+#define SLAVE_SDCC_2				20
-+#define SLAVE_TCSR				21
-+#define SLAVE_TLMM				22
-+#define SLAVE_UFS_MEM_CFG			23
-+#define SLAVE_USB3_0				24
-+#define SLAVE_VENUS_CFG				25
-+#define SLAVE_VSENSE_CTRL_CFG			26
-+#define SLAVE_WLAN				27
-+#define SLAVE_CNOC_MNOC_HF_CFG			28
-+#define SLAVE_CNOC_MNOC_SF_CFG			29
-+#define SLAVE_NSP_QTB_CFG			30
-+#define SLAVE_PCIE_ANOC_CFG			31
-+#define SLAVE_WLAN_Q6_THROTTLE_CFG		32
-+#define SLAVE_SERVICE_CNOC_CFG			33
-+#define SLAVE_QDSS_STM				34
-+#define SLAVE_TCU				35
-+
-+#define MASTER_GEM_NOC_CNOC			0
-+#define MASTER_GEM_NOC_PCIE_SNOC		1
-+#define SLAVE_AOSS				2
-+#define SLAVE_DISPLAY_CFG			3
-+#define SLAVE_IPA_CFG				4
-+#define SLAVE_IPC_ROUTER_CFG			5
-+#define SLAVE_PCIE_0_CFG			6
-+#define SLAVE_PCIE_1_CFG			7
-+#define SLAVE_PRNG				8
-+#define SLAVE_TME_CFG				9
-+#define SLAVE_APPSS				10
-+#define SLAVE_CNOC_CFG				11
-+#define SLAVE_DDRSS_CFG				12
-+#define SLAVE_IMEM				13
-+#define SLAVE_PIMEM				14
-+#define SLAVE_SERVICE_CNOC			15
-+#define SLAVE_PCIE_0				16
-+#define SLAVE_PCIE_1				17
-+
-+#define MASTER_GPU_TCU				0
-+#define MASTER_SYS_TCU				1
-+#define MASTER_APPSS_PROC			2
-+#define MASTER_GFX3D				3
-+#define MASTER_LPASS_GEM_NOC			4
-+#define MASTER_MSS_PROC				5
-+#define MASTER_MNOC_HF_MEM_NOC			6
-+#define MASTER_MNOC_SF_MEM_NOC			7
-+#define MASTER_COMPUTE_NOC			8
-+#define MASTER_ANOC_PCIE_GEM_NOC		9
-+#define MASTER_SNOC_GC_MEM_NOC			10
-+#define MASTER_SNOC_SF_MEM_NOC			11
-+#define MASTER_WLAN_Q6				12
-+#define SLAVE_GEM_NOC_CNOC			13
-+#define SLAVE_LLCC				14
-+#define SLAVE_MEM_NOC_PCIE_SNOC			15
-+
-+#define MASTER_LPASS_PROC			0
-+#define SLAVE_LPASS_GEM_NOC			1
-+
-+#define MASTER_LLCC				0
-+#define SLAVE_EBI1				1
-+
-+#define MASTER_CAMNOC_HF			0
-+#define MASTER_CAMNOC_ICP			1
-+#define MASTER_CAMNOC_SF			2
-+#define MASTER_MDP				3
-+#define MASTER_VIDEO				4
-+#define MASTER_CNOC_MNOC_HF_CFG			5
-+#define MASTER_CNOC_MNOC_SF_CFG			6
-+#define SLAVE_MNOC_HF_MEM_NOC			7
-+#define SLAVE_MNOC_SF_MEM_NOC			8
-+#define SLAVE_SERVICE_MNOC_HF			9
-+#define SLAVE_SERVICE_MNOC_SF			10
-+
-+#define MASTER_CDSP_PROC			0
-+#define SLAVE_CDSP_MEM_NOC			1
-+
-+#define MASTER_PCIE_ANOC_CFG			0
-+#define MASTER_PCIE_0				1
-+#define MASTER_PCIE_1				2
-+#define SLAVE_ANOC_PCIE_GEM_NOC			3
-+#define SLAVE_SERVICE_PCIE_ANOC			4
-+
-+#define MASTER_A1NOC_SNOC			0
-+#define MASTER_A2NOC_SNOC			1
-+#define MASTER_APSS_NOC				2
-+#define MASTER_CNOC_SNOC			3
-+#define MASTER_PIMEM				4
-+#define MASTER_GIC				5
-+#define SLAVE_SNOC_GEM_NOC_GC			6
-+#define SLAVE_SNOC_GEM_NOC_SF			7
-+
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,msm8909.h b/include/dt-bindings/interconnect/qcom,msm8909.h
-new file mode 100644
-index 000000000000..76365d8aec21
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,msm8909.h
-@@ -0,0 +1,93 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Qualcomm MSM8909 interconnect IDs
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_MSM8909_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_MSM8909_H
-+
-+/* BIMC fabric */
-+#define MAS_APPS_PROC			0
-+#define MAS_OXILI			1
-+#define MAS_SNOC_BIMC_0			2
-+#define MAS_SNOC_BIMC_1			3
-+#define MAS_TCU_0			4
-+#define MAS_TCU_1			5
-+#define SLV_EBI				6
-+#define SLV_BIMC_SNOC			7
-+
-+/* PCNOC fabric */
-+#define MAS_AUDIO			0
-+#define MAS_SPDM			1
-+#define MAS_DEHR			2
-+#define MAS_QPIC			3
-+#define MAS_BLSP_1			4
-+#define MAS_USB_HS			5
-+#define MAS_CRYPTO			6
-+#define MAS_SDCC_1			7
-+#define MAS_SDCC_2			8
-+#define MAS_SNOC_PCNOC			9
-+#define PCNOC_M_0			10
-+#define PCNOC_M_1			11
-+#define PCNOC_INT_0			12
-+#define PCNOC_INT_1			13
-+#define PCNOC_S_0			14
-+#define PCNOC_S_1			15
-+#define PCNOC_S_2			16
-+#define PCNOC_S_3			17
-+#define PCNOC_S_4			18
-+#define PCNOC_S_5			19
-+#define PCNOC_S_7			20
-+#define SLV_TCSR			21
-+#define SLV_SDCC_1			22
-+#define SLV_BLSP_1			23
-+#define SLV_CRYPTO_0_CFG		24
-+#define SLV_MESSAGE_RAM			25
-+#define SLV_PDM				26
-+#define SLV_PRNG			27
-+#define SLV_USB_HS			28
-+#define SLV_QPIC			29
-+#define SLV_SPDM			30
-+#define SLV_SDCC_2			31
-+#define SLV_AUDIO			32
-+#define SLV_DEHR_CFG			33
-+#define SLV_SNOC_CFG			34
-+#define SLV_QDSS_CFG			35
-+#define SLV_USB_PHY			36
-+#define SLV_CAMERA_SS_CFG		37
-+#define SLV_DISP_SS_CFG			38
-+#define SLV_VENUS_CFG			39
-+#define SLV_TLMM			40
-+#define SLV_GPU_CFG			41
-+#define SLV_IMEM_CFG			42
-+#define SLV_BIMC_CFG			43
-+#define SLV_PMIC_ARB			44
-+#define SLV_TCU				45
-+#define SLV_PCNOC_SNOC			46
-+
-+/* SNOC fabric */
-+#define MAS_QDSS_BAM			0
-+#define MAS_BIMC_SNOC			1
-+#define MAS_MDP				2
-+#define MAS_PCNOC_SNOC			3
-+#define MAS_VENUS			4
-+#define MAS_VFE				5
-+#define MAS_QDSS_ETR			6
-+#define MM_INT_0			7
-+#define MM_INT_1			8
-+#define MM_INT_2			9
-+#define MM_INT_BIMC			10
-+#define QDSS_INT			11
-+#define SNOC_INT_0			12
-+#define SNOC_INT_1			13
-+#define SNOC_INT_BIMC			14
-+#define SLV_KPSS_AHB			15
-+#define SLV_SNOC_BIMC_0			16
-+#define SLV_SNOC_BIMC_1			17
-+#define SLV_IMEM			18
-+#define SLV_SNOC_PCNOC			19
-+#define SLV_QDSS_STM			20
-+#define SLV_CATS_0			21
-+#define SLV_CATS_1			22
-+
-+#endif /* __DT_BINDINGS_INTERCONNECT_QCOM_MSM8909_H */
-diff --git a/include/dt-bindings/interconnect/qcom,msm8937.h b/include/dt-bindings/interconnect/qcom,msm8937.h
-new file mode 100644
-index 000000000000..98b8a4637aab
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,msm8937.h
-@@ -0,0 +1,93 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Qualcomm MSM8937 interconnect IDs
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_MSM8937_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_MSM8937_H
-+
-+/* BIMC fabric */
-+#define MAS_APPS_PROC		0
-+#define MAS_OXILI		1
-+#define MAS_SNOC_BIMC_0		2
-+#define MAS_SNOC_BIMC_2		3
-+#define MAS_SNOC_BIMC_1		4
-+#define MAS_TCU_0		5
-+#define SLV_EBI			6
-+#define SLV_BIMC_SNOC		7
-+
-+/* PCNOC fabric */
-+#define MAS_SPDM		0
-+#define MAS_BLSP_1		1
-+#define MAS_BLSP_2		2
-+#define MAS_USB_HS1		3
-+#define MAS_XI_USB_HS1		4
-+#define MAS_CRYPTO		5
-+#define MAS_SDCC_1		6
-+#define MAS_SDCC_2		7
-+#define MAS_SNOC_PCNOC		8
-+#define PCNOC_M_0		9
-+#define PCNOC_M_1		10
-+#define PCNOC_INT_0		11
-+#define PCNOC_INT_1		12
-+#define PCNOC_INT_2		13
-+#define PCNOC_INT_3		14
-+#define PCNOC_S_0		15
-+#define PCNOC_S_1		16
-+#define PCNOC_S_2		17
-+#define PCNOC_S_3		18
-+#define PCNOC_S_4		19
-+#define PCNOC_S_6		20
-+#define PCNOC_S_7		21
-+#define PCNOC_S_8		22
-+#define SLV_SDCC_2		23
-+#define SLV_SPDM		24
-+#define SLV_PDM			25
-+#define SLV_PRNG		26
-+#define SLV_TCSR		27
-+#define SLV_SNOC_CFG		28
-+#define SLV_MESSAGE_RAM		29
-+#define SLV_CAMERA_SS_CFG	30
-+#define SLV_DISP_SS_CFG		31
-+#define SLV_VENUS_CFG		32
-+#define SLV_GPU_CFG		33
-+#define SLV_TLMM		34
-+#define SLV_BLSP_1		35
-+#define SLV_BLSP_2		36
-+#define SLV_PMIC_ARB		37
-+#define SLV_SDCC_1		38
-+#define SLV_CRYPTO_0_CFG	39
-+#define SLV_USB_HS		40
-+#define SLV_TCU			41
-+#define SLV_PCNOC_SNOC		42
-+
-+/* SNOC fabric */
-+#define MAS_QDSS_BAM		0
-+#define MAS_BIMC_SNOC		1
-+#define MAS_PCNOC_SNOC		2
-+#define MAS_QDSS_ETR		3
-+#define QDSS_INT		4
-+#define SNOC_INT_0		5
-+#define SNOC_INT_1		6
-+#define SNOC_INT_2		7
-+#define SLV_KPSS_AHB		8
-+#define SLV_WCSS		9
-+#define SLV_SNOC_BIMC_1		10
-+#define SLV_IMEM		11
-+#define SLV_SNOC_PCNOC		12
-+#define SLV_QDSS_STM		13
-+#define SLV_CATS_1		14
-+#define SLV_LPASS		15
-+
-+/* SNOC-MM fabric */
-+#define MAS_JPEG		0
-+#define MAS_MDP			1
-+#define MAS_VENUS		2
-+#define MAS_VFE0		3
-+#define MAS_VFE1		4
-+#define MAS_CPP			5
-+#define SLV_SNOC_BIMC_0		6
-+#define SLV_SNOC_BIMC_2		7
-+#define SLV_CATS_0		8
-+
-+#endif /* __DT_BINDINGS_INTERCONNECT_QCOM_MSM8937_H */
-diff --git a/include/dt-bindings/interconnect/qcom,msm8953.h b/include/dt-bindings/interconnect/qcom,msm8953.h
-new file mode 100644
-index 000000000000..12564c434af7
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,msm8953.h
-@@ -0,0 +1,93 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Qualcomm MSM8953 interconnect IDs
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_MSM8953_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_MSM8953_H
-+
-+/* BIMC fabric */
-+#define MAS_APPS_PROC		0
-+#define MAS_OXILI		1
-+#define MAS_SNOC_BIMC_0		2
-+#define MAS_SNOC_BIMC_2		3
-+#define MAS_SNOC_BIMC_1		4
-+#define MAS_TCU_0		5
-+#define SLV_EBI			6
-+#define SLV_BIMC_SNOC		7
-+
-+/* PCNOC fabric */
-+#define MAS_SPDM		0
-+#define MAS_BLSP_1		1
-+#define MAS_BLSP_2		2
-+#define MAS_USB3		3
-+#define MAS_CRYPTO		4
-+#define MAS_SDCC_1		5
-+#define MAS_SDCC_2		6
-+#define MAS_SNOC_PCNOC		7
-+#define PCNOC_M_0		8
-+#define PCNOC_M_1		9
-+#define PCNOC_INT_1		10
-+#define PCNOC_INT_2		11
-+#define PCNOC_S_0		12
-+#define PCNOC_S_1		13
-+#define PCNOC_S_2		14
-+#define PCNOC_S_3		15
-+#define PCNOC_S_4		16
-+#define PCNOC_S_6		17
-+#define PCNOC_S_7		18
-+#define PCNOC_S_8		19
-+#define PCNOC_S_9		20
-+#define SLV_SPDM		21
-+#define SLV_PDM			22
-+#define SLV_TCSR		23
-+#define SLV_SNOC_CFG		24
-+#define SLV_TLMM		25
-+#define SLV_MESSAGE_RAM		26
-+#define SLV_BLSP_1		27
-+#define SLV_BLSP_2		28
-+#define SLV_PRNG		29
-+#define SLV_CAMERA_SS_CFG	30
-+#define SLV_DISP_SS_CFG		31
-+#define SLV_VENUS_CFG		32
-+#define SLV_GPU_CFG		33
-+#define SLV_SDCC_1		34
-+#define SLV_SDCC_2		35
-+#define SLV_CRYPTO_0_CFG	36
-+#define SLV_PMIC_ARB		37
-+#define SLV_USB3		38
-+#define SLV_IPA_CFG		39
-+#define SLV_TCU			40
-+#define SLV_PCNOC_SNOC		41
-+
-+/* SNOC fabric */
-+#define MAS_QDSS_BAM		0
-+#define MAS_BIMC_SNOC		1
-+#define MAS_PCNOC_SNOC		2
-+#define MAS_IPA			3
-+#define MAS_QDSS_ETR		4
-+#define QDSS_INT		5
-+#define SNOC_INT_0		6
-+#define SNOC_INT_1		7
-+#define SNOC_INT_2		8
-+#define SLV_KPSS_AHB		9
-+#define SLV_WCSS		10
-+#define SLV_SNOC_BIMC_1		11
-+#define SLV_IMEM		12
-+#define SLV_SNOC_PCNOC		13
-+#define SLV_QDSS_STM		14
-+#define SLV_CATS_1		15
-+#define SLV_LPASS		16
-+
-+/* SNOC-MM fabric */
-+#define MAS_JPEG		0
-+#define MAS_MDP			1
-+#define MAS_VENUS		2
-+#define MAS_VFE0		3
-+#define MAS_VFE1		4
-+#define MAS_CPP			5
-+#define SLV_SNOC_BIMC_0		6
-+#define SLV_SNOC_BIMC_2		7
-+#define SLV_CATS_0		8
-+
-+#endif /* __DT_BINDINGS_INTERCONNECT_QCOM_MSM8953_H */
-diff --git a/include/dt-bindings/interconnect/qcom,msm8976.h b/include/dt-bindings/interconnect/qcom,msm8976.h
-new file mode 100644
-index 000000000000..4ea90f22320e
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,msm8976.h
-@@ -0,0 +1,97 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Qualcomm MSM8976 interconnect IDs
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_MSM8976_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_MSM8976_H
-+
-+/* BIMC fabric */
-+#define MAS_APPS_PROC		0
-+#define MAS_SMMNOC_BIMC	        1
-+#define MAS_SNOC_BIMC		2
-+#define MAS_TCU_0		3
-+#define SLV_EBI		        4
-+#define SLV_BIMC_SNOC		5
-+
-+/* PCNOC fabric */
-+#define MAS_USB_HS2		0
-+#define MAS_BLSP_1		1
-+#define MAS_USB_HS1		2
-+#define MAS_BLSP_2		3
-+#define MAS_CRYPTO		4
-+#define MAS_SDCC_1		5
-+#define MAS_SDCC_2		6
-+#define MAS_SDCC_3		7
-+#define MAS_SNOC_PCNOC		8
-+#define MAS_LPASS_AHB		9
-+#define MAS_SPDM		10
-+#define MAS_DEHR		11
-+#define MAS_XM_USB_HS1		12
-+#define PCNOC_M_0		13
-+#define PCNOC_M_1		14
-+#define PCNOC_INT_0		15
-+#define PCNOC_INT_1		16
-+#define PCNOC_INT_2		17
-+#define PCNOC_S_1		18
-+#define PCNOC_S_2		19
-+#define PCNOC_S_3		20
-+#define PCNOC_S_4		21
-+#define PCNOC_S_8		22
-+#define PCNOC_S_9		23
-+#define SLV_TCSR		24
-+#define SLV_TLMM		25
-+#define SLV_CRYPTO_0_CFG	26
-+#define SLV_MESSAGE_RAM	        27
-+#define SLV_PDM		        28
-+#define SLV_PRNG		29
-+#define SLV_PMIC_ARB		30
-+#define SLV_SNOC_CFG		31
-+#define SLV_DCC_CFG		32
-+#define SLV_CAMERA_SS_CFG	33
-+#define SLV_DISP_SS_CFG	        34
-+#define SLV_VENUS_CFG		35
-+#define SLV_SDCC_1		36
-+#define SLV_BLSP_1		37
-+#define SLV_USB_HS		38
-+#define SLV_SDCC_3		39
-+#define SLV_SDCC_2		40
-+#define SLV_GPU_CFG		41
-+#define SLV_USB_HS2		42
-+#define SLV_BLSP_2		43
-+#define SLV_PCNOC_SNOC		44
-+
-+/* SNOC fabric */
-+#define MAS_QDSS_BAM		0
-+#define MAS_BIMC_SNOC		1
-+#define MAS_PCNOC_SNOC		2
-+#define MAS_QDSS_ETR		3
-+#define MAS_LPASS_PROC		4
-+#define MAS_IPA		        5
-+#define QDSS_INT		6
-+#define SNOC_INT_0		7
-+#define SNOC_INT_1		8
-+#define SNOC_INT_2		9
-+#define SLV_KPSS_AHB		10
-+#define SLV_SNOC_BIMC		11
-+#define SLV_IMEM		12
-+#define SLV_SNOC_PCNOC		13
-+#define SLV_QDSS_STM		14
-+#define SLV_CATS_0		15
-+#define SLV_CATS_1		16
-+#define SLV_LPASS		17
-+
-+/* SNOC-MM fabric */
-+#define MAS_JPEG		0
-+#define MAS_OXILI		1
-+#define MAS_MDP0		2
-+#define MAS_MDP1		3
-+#define MAS_VENUS_0		4
-+#define MAS_VENUS_1		5
-+#define MAS_VFE_0		6
-+#define MAS_VFE_1		7
-+#define MAS_CPP		        8
-+#define MM_INT_0		9
-+#define SLV_SMMNOC_BIMC		10
-+
-+#endif /* __DT_BINDINGS_INTERCONNECT_QCOM_MSM8976_H */
-diff --git a/include/dt-bindings/interconnect/qcom,msm8996-cbf.h b/include/dt-bindings/interconnect/qcom,msm8996-cbf.h
-new file mode 100644
-index 000000000000..aac5e69f6bd5
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,msm8996-cbf.h
-@@ -0,0 +1,12 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2023 Linaro Ltd. All rights reserved.
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_MSM8996_CBF_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_MSM8996_CBF_H
-+
-+#define MASTER_CBF_M4M		0
-+#define SLAVE_CBF_M4M		1
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,qcs615-rpmh.h b/include/dt-bindings/interconnect/qcom,qcs615-rpmh.h
-new file mode 100644
-index 000000000000..84ae0d39e73c
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,qcs615-rpmh.h
-@@ -0,0 +1,136 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_QCS615_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_QCS615_H
-+
-+#define MASTER_A1NOC_CFG		1
-+#define MASTER_QDSS_BAM			2
-+#define MASTER_QSPI			3
-+#define MASTER_QUP_0			4
-+#define MASTER_BLSP_1			5
-+#define MASTER_CNOC_A2NOC		6
-+#define MASTER_CRYPTO			7
-+#define MASTER_IPA			8
-+#define MASTER_EMAC_EVB			9
-+#define MASTER_PCIE			10
-+#define MASTER_QDSS_ETR			11
-+#define MASTER_SDCC_1			12
-+#define MASTER_SDCC_2			13
-+#define MASTER_UFS_MEM			14
-+#define MASTER_USB2			15
-+#define MASTER_USB3_0			16
-+#define SLAVE_A1NOC_SNOC		17
-+#define SLAVE_LPASS_SNOC		18
-+#define SLAVE_ANOC_PCIE_SNOC		19
-+#define SLAVE_SERVICE_A2NOC		20
-+
-+#define MASTER_CAMNOC_HF0_UNCOMP	1
-+#define MASTER_CAMNOC_HF1_UNCOMP	2
-+#define MASTER_CAMNOC_SF_UNCOMP		3
-+#define SLAVE_CAMNOC_UNCOMP		4
-+
-+#define MASTER_SPDM			1
-+#define MASTER_SNOC_CNOC		2
-+#define MASTER_QDSS_DAP			3
-+#define SLAVE_A1NOC_CFG			4
-+#define SLAVE_AHB2PHY_EAST		5
-+#define SLAVE_AHB2PHY_WEST		6
-+#define SLAVE_AOP			7
-+#define SLAVE_AOSS			8
-+#define SLAVE_CAMERA_CFG		9
-+#define SLAVE_CLK_CTL			10
-+#define SLAVE_RBCPR_CX_CFG		11
-+#define SLAVE_RBCPR_MX_CFG		12
-+#define SLAVE_CRYPTO_0_CFG		13
-+#define SLAVE_CNOC_DDRSS		14
-+#define SLAVE_DISPLAY_CFG		15
-+#define SLAVE_EMAC_AVB_CFG		16
-+#define SLAVE_GLM			17
-+#define SLAVE_GFX3D_CFG			18
-+#define SLAVE_IMEM_CFG			19
-+#define SLAVE_IPA_CFG			20
-+#define SLAVE_CNOC_MNOC_CFG		21
-+#define SLAVE_PCIE_CFG			22
-+#define SLAVE_PIMEM_CFG			23
-+#define SLAVE_PRNG			24
-+#define SLAVE_QDSS_CFG			25
-+#define SLAVE_QSPI			26
-+#define SLAVE_QUP_0			27
-+#define SLAVE_QUP_1			28
-+#define SLAVE_SDCC_1			29
-+#define SLAVE_SDCC_2			30
-+#define SLAVE_SNOC_CFG			31
-+#define SLAVE_SPDM_WRAPPER		32
-+#define SLAVE_TCSR			33
-+#define SLAVE_TLMM_EAST			34
-+#define SLAVE_TLMM_SOUTH		35
-+#define SLAVE_TLMM_WEST			36
-+#define SLAVE_UFS_MEM_CFG		37
-+#define SLAVE_USB2			38
-+#define SLAVE_USB3			39
-+#define SLAVE_VENUS_CFG			40
-+#define SLAVE_VSENSE_CTRL_CFG		41
-+#define SLAVE_CNOC_A2NOC		42
-+#define SLAVE_SERVICE_CNOC		43
-+
-+#define MASTER_CNOC_DC_NOC		1
-+#define SLAVE_DC_NOC_GEMNOC		2
-+#define SLAVE_LLCC_CFG			3
-+
-+#define MASTER_APPSS_PROC		1
-+#define MASTER_GPU_TCU			2
-+#define MASTER_SYS_TCU			3
-+#define MASTER_GEM_NOC_CFG		4
-+#define MASTER_GFX3D			5
-+#define MASTER_MNOC_HF_MEM_NOC		6
-+#define MASTER_MNOC_SF_MEM_NOC		7
-+#define MASTER_SNOC_GC_MEM_NOC		8
-+#define MASTER_SNOC_SF_MEM_NOC		9
-+#define SLAVE_MSS_PROC_MS_MPU_CFG	10
-+#define SLAVE_GEM_NOC_SNOC		11
-+#define SLAVE_LLCC			12
-+#define SLAVE_MEM_NOC_PCIE_SNOC		13
-+#define SLAVE_SERVICE_GEM_NOC		14
-+
-+#define MASTER_IPA_CORE			1
-+#define SLAVE_IPA_CORE			2
-+
-+#define MASTER_LLCC			1
-+#define SLAVE_EBI1			2
-+
-+#define MASTER_CNOC_MNOC_CFG		1
-+#define MASTER_CAMNOC_HF0		2
-+#define MASTER_CAMNOC_HF1		3
-+#define MASTER_CAMNOC_SF		4
-+#define MASTER_MDP0			5
-+#define MASTER_ROTATOR			6
-+#define MASTER_VIDEO_P0			7
-+#define MASTER_VIDEO_PROC		8
-+#define SLAVE_MNOC_SF_MEM_NOC		9
-+#define SLAVE_MNOC_HF_MEM_NOC		10
-+#define SLAVE_SERVICE_MNOC		11
-+
-+#define MASTER_SNOC_CFG			1
-+#define MASTER_A1NOC_SNOC		2
-+#define MASTER_GEM_NOC_SNOC		3
-+#define MASTER_GEM_NOC_PCIE_SNOC	4
-+#define MASTER_LPASS_ANOC		5
-+#define MASTER_ANOC_PCIE_SNOC		6
-+#define MASTER_PIMEM			7
-+#define MASTER_GIC			8
-+#define SLAVE_APPSS			9
-+#define SLAVE_SNOC_CNOC			10
-+#define SLAVE_SNOC_GEM_NOC_SF		11
-+#define SLAVE_SNOC_MEM_NOC_GC		12
-+#define SLAVE_IMEM			13
-+#define SLAVE_PIMEM			14
-+#define SLAVE_SERVICE_SNOC		15
-+#define SLAVE_PCIE_0			16
-+#define SLAVE_QDSS_STM			17
-+#define SLAVE_TCU			18
-+
-+#endif
-+
-diff --git a/include/dt-bindings/interconnect/qcom,qcs8300-rpmh.h b/include/dt-bindings/interconnect/qcom,qcs8300-rpmh.h
-new file mode 100644
-index 000000000000..c5eeafa1b1dd
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,qcs8300-rpmh.h
-@@ -0,0 +1,189 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_QCS8300_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_QCS8300_H
-+
-+#define MASTER_QUP_3				0
-+#define MASTER_EMAC				1
-+#define MASTER_SDC				2
-+#define MASTER_UFS_MEM				3
-+#define MASTER_USB2				4
-+#define MASTER_USB3_0				5
-+#define SLAVE_A1NOC_SNOC			6
-+
-+#define MASTER_QDSS_BAM				0
-+#define MASTER_QUP_0				1
-+#define MASTER_QUP_1				2
-+#define MASTER_CNOC_A2NOC			3
-+#define MASTER_CRYPTO_CORE0			4
-+#define MASTER_CRYPTO_CORE1			5
-+#define MASTER_IPA				6
-+#define MASTER_QDSS_ETR_0			7
-+#define MASTER_QDSS_ETR_1			8
-+#define SLAVE_A2NOC_SNOC			9
-+
-+#define MASTER_QUP_CORE_0			0
-+#define MASTER_QUP_CORE_1			1
-+#define MASTER_QUP_CORE_3			2
-+#define SLAVE_QUP_CORE_0			3
-+#define SLAVE_QUP_CORE_1			4
-+#define SLAVE_QUP_CORE_3			5
-+
-+#define MASTER_GEM_NOC_CNOC			0
-+#define MASTER_GEM_NOC_PCIE_SNOC		1
-+#define SLAVE_AHB2PHY_2				2
-+#define SLAVE_AHB2PHY_3				3
-+#define SLAVE_ANOC_THROTTLE_CFG			4
-+#define SLAVE_AOSS				5
-+#define SLAVE_APPSS				6
-+#define SLAVE_BOOT_ROM				7
-+#define SLAVE_CAMERA_CFG			8
-+#define SLAVE_CAMERA_NRT_THROTTLE_CFG		9
-+#define SLAVE_CAMERA_RT_THROTTLE_CFG		10
-+#define SLAVE_CLK_CTL				11
-+#define SLAVE_CDSP_CFG				12
-+#define SLAVE_RBCPR_CX_CFG			13
-+#define SLAVE_RBCPR_MMCX_CFG			14
-+#define SLAVE_RBCPR_MX_CFG			15
-+#define SLAVE_CPR_NSPCX				16
-+#define SLAVE_CPR_NSPHMX			17
-+#define SLAVE_CRYPTO_0_CFG			18
-+#define SLAVE_CX_RDPM				19
-+#define SLAVE_DISPLAY_CFG			20
-+#define SLAVE_DISPLAY_RT_THROTTLE_CFG		21
-+#define SLAVE_EMAC_CFG				22
-+#define SLAVE_GP_DSP0_CFG			23
-+#define SLAVE_GPDSP0_THROTTLE_CFG		24
-+#define SLAVE_GPU_TCU_THROTTLE_CFG		25
-+#define SLAVE_GFX3D_CFG				26
-+#define SLAVE_HWKM				27
-+#define SLAVE_IMEM_CFG				28
-+#define SLAVE_IPA_CFG				29
-+#define SLAVE_IPC_ROUTER_CFG			30
-+#define SLAVE_LPASS				31
-+#define SLAVE_LPASS_THROTTLE_CFG		32
-+#define SLAVE_MX_RDPM				33
-+#define SLAVE_MXC_RDPM				34
-+#define SLAVE_PCIE_0_CFG			35
-+#define SLAVE_PCIE_1_CFG			36
-+#define SLAVE_PCIE_TCU_THROTTLE_CFG		37
-+#define SLAVE_PCIE_THROTTLE_CFG			38
-+#define SLAVE_PDM				39
-+#define SLAVE_PIMEM_CFG				40
-+#define SLAVE_PKA_WRAPPER_CFG			41
-+#define SLAVE_QDSS_CFG				42
-+#define SLAVE_QM_CFG				43
-+#define SLAVE_QM_MPU_CFG			44
-+#define SLAVE_QUP_0				45
-+#define SLAVE_QUP_1				46
-+#define SLAVE_QUP_3				47
-+#define SLAVE_SAIL_THROTTLE_CFG			48
-+#define SLAVE_SDC1				49
-+#define SLAVE_SECURITY				50
-+#define SLAVE_SNOC_THROTTLE_CFG			51
-+#define SLAVE_TCSR				52
-+#define SLAVE_TLMM				53
-+#define SLAVE_TSC_CFG				54
-+#define SLAVE_UFS_MEM_CFG			55
-+#define SLAVE_USB2				56
-+#define SLAVE_USB3_0				57
-+#define SLAVE_VENUS_CFG				58
-+#define SLAVE_VENUS_CVP_THROTTLE_CFG		59
-+#define SLAVE_VENUS_V_CPU_THROTTLE_CFG		60
-+#define SLAVE_VENUS_VCODEC_THROTTLE_CFG		61
-+#define SLAVE_DDRSS_CFG				62
-+#define SLAVE_GPDSP_NOC_CFG			63
-+#define SLAVE_CNOC_MNOC_HF_CFG			64
-+#define SLAVE_CNOC_MNOC_SF_CFG			65
-+#define SLAVE_PCIE_ANOC_CFG			66
-+#define SLAVE_SNOC_CFG				67
-+#define SLAVE_BOOT_IMEM				68
-+#define SLAVE_IMEM				69
-+#define SLAVE_PIMEM				70
-+#define SLAVE_PCIE_0				71
-+#define SLAVE_PCIE_1				72
-+#define SLAVE_QDSS_STM				73
-+#define SLAVE_TCU				74
-+
-+#define MASTER_CNOC_DC_NOC			0
-+#define SLAVE_LLCC_CFG				1
-+#define SLAVE_GEM_NOC_CFG			2
-+
-+#define MASTER_GPU_TCU				0
-+#define MASTER_PCIE_TCU				1
-+#define MASTER_SYS_TCU				2
-+#define MASTER_APPSS_PROC			3
-+#define MASTER_COMPUTE_NOC			4
-+#define MASTER_GEM_NOC_CFG			5
-+#define MASTER_GPDSP_SAIL			6
-+#define MASTER_GFX3D				7
-+#define MASTER_MNOC_HF_MEM_NOC			8
-+#define MASTER_MNOC_SF_MEM_NOC			9
-+#define MASTER_ANOC_PCIE_GEM_NOC		10
-+#define MASTER_SNOC_GC_MEM_NOC			11
-+#define MASTER_SNOC_SF_MEM_NOC			12
-+#define SLAVE_GEM_NOC_CNOC			13
-+#define SLAVE_LLCC				14
-+#define SLAVE_GEM_NOC_PCIE_CNOC			15
-+#define SLAVE_SERVICE_GEM_NOC_1			16
-+#define SLAVE_SERVICE_GEM_NOC_2			17
-+#define SLAVE_SERVICE_GEM_NOC			18
-+#define SLAVE_SERVICE_GEM_NOC2			19
-+
-+#define MASTER_SAILSS_MD0			0
-+#define MASTER_DSP0				1
-+#define SLAVE_GP_DSP_SAIL_NOC			2
-+
-+#define MASTER_CNOC_LPASS_AG_NOC		0
-+#define MASTER_LPASS_PROC			1
-+#define SLAVE_LPASS_CORE_CFG			2
-+#define SLAVE_LPASS_LPI_CFG			3
-+#define SLAVE_LPASS_MPU_CFG			4
-+#define SLAVE_LPASS_TOP_CFG			5
-+#define SLAVE_LPASS_SNOC			6
-+#define SLAVE_SERVICES_LPASS_AML_NOC		7
-+#define SLAVE_SERVICE_LPASS_AG_NOC		8
-+
-+#define MASTER_LLCC				0
-+#define SLAVE_EBI1				1
-+
-+#define MASTER_CAMNOC_HF			0
-+#define MASTER_CAMNOC_ICP			1
-+#define MASTER_CAMNOC_SF			2
-+#define MASTER_MDP0				3
-+#define MASTER_MDP1				4
-+#define MASTER_CNOC_MNOC_HF_CFG			5
-+#define MASTER_CNOC_MNOC_SF_CFG			6
-+#define MASTER_VIDEO_P0				7
-+#define MASTER_VIDEO_PROC			8
-+#define MASTER_VIDEO_V_PROC			9
-+#define SLAVE_MNOC_HF_MEM_NOC			10
-+#define SLAVE_MNOC_SF_MEM_NOC			11
-+#define SLAVE_SERVICE_MNOC_HF			12
-+#define SLAVE_SERVICE_MNOC_SF			13
-+
-+#define MASTER_CDSP_NOC_CFG			0
-+#define MASTER_CDSP_PROC			1
-+#define SLAVE_HCP_A				2
-+#define SLAVE_CDSP_MEM_NOC			3
-+#define SLAVE_SERVICE_NSP_NOC			4
-+
-+#define MASTER_PCIE_0				0
-+#define MASTER_PCIE_1				1
-+#define SLAVE_ANOC_PCIE_GEM_NOC			2
-+
-+#define MASTER_GIC_AHB				0
-+#define MASTER_A1NOC_SNOC			1
-+#define MASTER_A2NOC_SNOC			2
-+#define MASTER_LPASS_ANOC			3
-+#define MASTER_SNOC_CFG				4
-+#define MASTER_PIMEM				5
-+#define MASTER_GIC				6
-+#define SLAVE_SNOC_GEM_NOC_GC			7
-+#define SLAVE_SNOC_GEM_NOC_SF			8
-+#define SLAVE_SERVICE_SNOC			9
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,qdu1000-rpmh.h b/include/dt-bindings/interconnect/qcom,qdu1000-rpmh.h
-new file mode 100644
-index 000000000000..7f0ad1571128
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,qdu1000-rpmh.h
-@@ -0,0 +1,98 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_QDU1000_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_QDU1000_H
-+
-+#define MASTER_QUP_CORE_0			0
-+#define MASTER_QUP_CORE_1			1
-+#define SLAVE_QUP_CORE_0			2
-+#define SLAVE_QUP_CORE_1			3
-+
-+#define MASTER_SYS_TCU				0
-+#define MASTER_APPSS_PROC			1
-+#define MASTER_GEMNOC_ECPRI_DMA			2
-+#define MASTER_FEC_2_GEMNOC			3
-+#define MASTER_ANOC_PCIE_GEM_NOC		4
-+#define MASTER_SNOC_GC_MEM_NOC			5
-+#define MASTER_SNOC_SF_MEM_NOC			6
-+#define MASTER_MSS_PROC				7
-+#define SLAVE_GEM_NOC_CNOC			8
-+#define SLAVE_LLCC				9
-+#define SLAVE_GEMNOC_MODEM_CNOC			10
-+#define SLAVE_MEM_NOC_PCIE_SNOC			11
-+
-+#define MASTER_LLCC				0
-+#define SLAVE_EBI1				1
-+
-+#define MASTER_GIC_AHB				0
-+#define MASTER_QDSS_BAM				1
-+#define MASTER_QPIC				2
-+#define MASTER_QSPI_0				3
-+#define MASTER_QUP_0				4
-+#define MASTER_QUP_1				5
-+#define MASTER_SNOC_CFG				6
-+#define MASTER_ANOC_SNOC			7
-+#define MASTER_ANOC_GSI				8
-+#define MASTER_GEM_NOC_CNOC			9
-+#define MASTER_GEMNOC_MODEM_CNOC		10
-+#define MASTER_GEM_NOC_PCIE_SNOC		11
-+#define MASTER_CRYPTO				12
-+#define MASTER_ECPRI_GSI			13
-+#define MASTER_PIMEM				14
-+#define MASTER_SNOC_ECPRI_DMA			15
-+#define MASTER_GIC				16
-+#define MASTER_PCIE				17
-+#define MASTER_QDSS_ETR				18
-+#define MASTER_QDSS_ETR_1			19
-+#define MASTER_SDCC_1				20
-+#define MASTER_USB3				21
-+#define SLAVE_AHB2PHY_SOUTH			22
-+#define SLAVE_AHB2PHY_NORTH			23
-+#define SLAVE_AHB2PHY_EAST			24
-+#define SLAVE_AOSS				25
-+#define SLAVE_CLK_CTL				26
-+#define SLAVE_RBCPR_CX_CFG			27
-+#define SLAVE_RBCPR_MX_CFG			28
-+#define SLAVE_CRYPTO_0_CFG			29
-+#define SLAVE_ECPRI_CFG				30
-+#define SLAVE_IMEM_CFG				31
-+#define SLAVE_IPC_ROUTER_CFG			32
-+#define SLAVE_CNOC_MSS				33
-+#define SLAVE_PCIE_CFG				34
-+#define SLAVE_PDM				35
-+#define SLAVE_PIMEM_CFG				36
-+#define SLAVE_PRNG				37
-+#define SLAVE_QDSS_CFG				38
-+#define SLAVE_QPIC				40
-+#define SLAVE_QSPI_0				41
-+#define SLAVE_QUP_0				42
-+#define SLAVE_QUP_1				43
-+#define SLAVE_SDCC_2				44
-+#define SLAVE_SMBUS_CFG				45
-+#define SLAVE_SNOC_CFG				46
-+#define SLAVE_TCSR				47
-+#define SLAVE_TLMM				48
-+#define SLAVE_TME_CFG				49
-+#define SLAVE_TSC_CFG				50
-+#define SLAVE_USB3_0				51
-+#define SLAVE_VSENSE_CTRL_CFG			52
-+#define SLAVE_A1NOC_SNOC			53
-+#define SLAVE_ANOC_SNOC_GSI			54
-+#define SLAVE_DDRSS_CFG				55
-+#define SLAVE_ECPRI_GEMNOC			56
-+#define SLAVE_SNOC_GEM_NOC_GC			57
-+#define SLAVE_SNOC_GEM_NOC_SF			58
-+#define SLAVE_MODEM_OFFLINE			59
-+#define SLAVE_ANOC_PCIE_GEM_NOC			60
-+#define SLAVE_IMEM				61
-+#define SLAVE_PIMEM				62
-+#define SLAVE_SERVICE_SNOC			63
-+#define SLAVE_ETHERNET_SS			64
-+#define SLAVE_PCIE_0				65
-+#define SLAVE_QDSS_STM				66
-+#define SLAVE_TCU				67
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,rpm-icc.h b/include/dt-bindings/interconnect/qcom,rpm-icc.h
-new file mode 100644
-index 000000000000..2cd56f91e5c5
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,rpm-icc.h
-@@ -0,0 +1,13 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_RPM_ICC_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_RPM_ICC_H
-+
-+#define RPM_ACTIVE_TAG		(1 << 0)
-+#define RPM_SLEEP_TAG		(1 << 1)
-+#define RPM_ALWAYS_TAG		(RPM_ACTIVE_TAG | RPM_SLEEP_TAG)
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,sa8775p-rpmh.h b/include/dt-bindings/interconnect/qcom,sa8775p-rpmh.h
-new file mode 100644
-index 000000000000..f21c39d0928e
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,sa8775p-rpmh.h
-@@ -0,0 +1,231 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_SA8775P_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_SA8775P_H
-+
-+/* aggre1_noc */
-+#define MASTER_QUP_3			0
-+#define MASTER_EMAC			1
-+#define MASTER_EMAC_1			2
-+#define MASTER_SDC			3
-+#define MASTER_UFS_MEM			4
-+#define MASTER_USB2			5
-+#define MASTER_USB3_0			6
-+#define MASTER_USB3_1			7
-+#define SLAVE_A1NOC_SNOC		8
-+
-+/* aggre2_noc */
-+#define MASTER_QDSS_BAM			0
-+#define MASTER_QUP_0			1
-+#define MASTER_QUP_1			2
-+#define MASTER_QUP_2			3
-+#define MASTER_CNOC_A2NOC		4
-+#define MASTER_CRYPTO_CORE0		5
-+#define MASTER_CRYPTO_CORE1		6
-+#define MASTER_IPA			7
-+#define MASTER_QDSS_ETR_0		8
-+#define MASTER_QDSS_ETR_1		9
-+#define MASTER_UFS_CARD			10
-+#define SLAVE_A2NOC_SNOC		11
-+
-+/* clk_virt */
-+#define MASTER_QUP_CORE_0		0
-+#define MASTER_QUP_CORE_1		1
-+#define MASTER_QUP_CORE_2		2
-+#define MASTER_QUP_CORE_3		3
-+#define SLAVE_QUP_CORE_0		4
-+#define SLAVE_QUP_CORE_1		5
-+#define SLAVE_QUP_CORE_2		6
-+#define SLAVE_QUP_CORE_3		7
-+
-+/* config_noc */
-+#define MASTER_GEM_NOC_CNOC		0
-+#define MASTER_GEM_NOC_PCIE_SNOC	1
-+#define SLAVE_AHB2PHY_0			2
-+#define SLAVE_AHB2PHY_1			3
-+#define SLAVE_AHB2PHY_2			4
-+#define SLAVE_AHB2PHY_3			5
-+#define SLAVE_ANOC_THROTTLE_CFG		6
-+#define SLAVE_AOSS			7
-+#define SLAVE_APPSS			8
-+#define SLAVE_BOOT_ROM			9
-+#define SLAVE_CAMERA_CFG		10
-+#define SLAVE_CAMERA_NRT_THROTTLE_CFG	11
-+#define SLAVE_CAMERA_RT_THROTTLE_CFG	12
-+#define SLAVE_CLK_CTL			13
-+#define SLAVE_CDSP_CFG			14
-+#define SLAVE_CDSP1_CFG			15
-+#define SLAVE_RBCPR_CX_CFG		16
-+#define SLAVE_RBCPR_MMCX_CFG		17
-+#define SLAVE_RBCPR_MX_CFG		18
-+#define SLAVE_CPR_NSPCX			19
-+#define SLAVE_CRYPTO_0_CFG		20
-+#define SLAVE_CX_RDPM			21
-+#define SLAVE_DISPLAY_CFG		22
-+#define SLAVE_DISPLAY_RT_THROTTLE_CFG	23
-+#define SLAVE_DISPLAY1_CFG		24
-+#define SLAVE_DISPLAY1_RT_THROTTLE_CFG  25
-+#define SLAVE_EMAC_CFG			26
-+#define SLAVE_EMAC1_CFG			27
-+#define SLAVE_GP_DSP0_CFG		28
-+#define SLAVE_GP_DSP1_CFG		29
-+#define SLAVE_GPDSP0_THROTTLE_CFG	30
-+#define SLAVE_GPDSP1_THROTTLE_CFG	31
-+#define SLAVE_GPU_TCU_THROTTLE_CFG	32
-+#define SLAVE_GFX3D_CFG			33
-+#define SLAVE_HWKM			34
-+#define SLAVE_IMEM_CFG			35
-+#define SLAVE_IPA_CFG			36
-+#define SLAVE_IPC_ROUTER_CFG		37
-+#define SLAVE_LPASS			38
-+#define SLAVE_LPASS_THROTTLE_CFG	39
-+#define SLAVE_MX_RDPM			40
-+#define SLAVE_MXC_RDPM			41
-+#define SLAVE_PCIE_0_CFG		42
-+#define SLAVE_PCIE_1_CFG		43
-+#define SLAVE_PCIE_RSC_CFG		44
-+#define SLAVE_PCIE_TCU_THROTTLE_CFG	45
-+#define SLAVE_PCIE_THROTTLE_CFG		46
-+#define SLAVE_PDM			47
-+#define SLAVE_PIMEM_CFG			48
-+#define SLAVE_PKA_WRAPPER_CFG		49
-+#define SLAVE_QDSS_CFG			50
-+#define SLAVE_QM_CFG			51
-+#define SLAVE_QM_MPU_CFG		52
-+#define SLAVE_QUP_0			53
-+#define SLAVE_QUP_1			54
-+#define SLAVE_QUP_2			55
-+#define SLAVE_QUP_3			56
-+#define SLAVE_SAIL_THROTTLE_CFG		57
-+#define SLAVE_SDC1			58
-+#define SLAVE_SECURITY			59
-+#define SLAVE_SNOC_THROTTLE_CFG		60
-+#define SLAVE_TCSR			61
-+#define SLAVE_TLMM			62
-+#define SLAVE_TSC_CFG			63
-+#define SLAVE_UFS_CARD_CFG		64
-+#define SLAVE_UFS_MEM_CFG		65
-+#define SLAVE_USB2			66
-+#define SLAVE_USB3_0			67
-+#define SLAVE_USB3_1			68
-+#define SLAVE_VENUS_CFG			69
-+#define SLAVE_VENUS_CVP_THROTTLE_CFG	70
-+#define SLAVE_VENUS_V_CPU_THROTTLE_CFG	71
-+#define SLAVE_VENUS_VCODEC_THROTTLE_CFG	72
-+#define SLAVE_DDRSS_CFG			73
-+#define SLAVE_GPDSP_NOC_CFG		74
-+#define SLAVE_CNOC_MNOC_HF_CFG		75
-+#define SLAVE_CNOC_MNOC_SF_CFG		76
-+#define SLAVE_PCIE_ANOC_CFG		77
-+#define SLAVE_SNOC_CFG			78
-+#define SLAVE_BOOT_IMEM			79
-+#define SLAVE_IMEM			80
-+#define SLAVE_PIMEM			81
-+#define SLAVE_PCIE_0			82
-+#define SLAVE_PCIE_1			83
-+#define SLAVE_QDSS_STM			84
-+#define SLAVE_TCU			85
-+
-+/* dc_noc */
-+#define MASTER_CNOC_DC_NOC		0
-+#define SLAVE_LLCC_CFG			1
-+#define SLAVE_GEM_NOC_CFG		2
-+
-+/* gem_noc */
-+#define MASTER_GPU_TCU			0
-+#define MASTER_PCIE_TCU			1
-+#define MASTER_SYS_TCU			2
-+#define MASTER_APPSS_PROC		3
-+#define MASTER_COMPUTE_NOC		4
-+#define MASTER_COMPUTE_NOC_1		5
-+#define MASTER_GEM_NOC_CFG		6
-+#define MASTER_GPDSP_SAIL		7
-+#define MASTER_GFX3D			8
-+#define MASTER_MNOC_HF_MEM_NOC		9
-+#define MASTER_MNOC_SF_MEM_NOC		10
-+#define MASTER_ANOC_PCIE_GEM_NOC	11
-+#define MASTER_SNOC_GC_MEM_NOC		12
-+#define MASTER_SNOC_SF_MEM_NOC		13
-+#define SLAVE_GEM_NOC_CNOC		14
-+#define SLAVE_LLCC			15
-+#define SLAVE_GEM_NOC_PCIE_CNOC		16
-+#define SLAVE_SERVICE_GEM_NOC_1		17
-+#define SLAVE_SERVICE_GEM_NOC_2		18
-+#define SLAVE_SERVICE_GEM_NOC		19
-+#define SLAVE_SERVICE_GEM_NOC2		20
-+
-+/* gpdsp_anoc */
-+#define MASTER_DSP0			0
-+#define MASTER_DSP1			1
-+#define SLAVE_GP_DSP_SAIL_NOC		2
-+
-+/* lpass_ag_noc */
-+#define MASTER_CNOC_LPASS_AG_NOC	0
-+#define MASTER_LPASS_PROC		1
-+#define SLAVE_LPASS_CORE_CFG		2
-+#define SLAVE_LPASS_LPI_CFG		3
-+#define SLAVE_LPASS_MPU_CFG		4
-+#define SLAVE_LPASS_TOP_CFG		5
-+#define SLAVE_LPASS_SNOC		6
-+#define SLAVE_SERVICES_LPASS_AML_NOC	7
-+#define SLAVE_SERVICE_LPASS_AG_NOC	8
-+
-+/* mc_virt */
-+#define MASTER_LLCC			0
-+#define SLAVE_EBI1			1
-+
-+/*mmss_noc */
-+#define MASTER_CAMNOC_HF		0
-+#define MASTER_CAMNOC_ICP		1
-+#define MASTER_CAMNOC_SF		2
-+#define MASTER_MDP0			3
-+#define MASTER_MDP1			4
-+#define MASTER_MDP_CORE1_0		5
-+#define MASTER_MDP_CORE1_1		6
-+#define MASTER_CNOC_MNOC_HF_CFG		7
-+#define MASTER_CNOC_MNOC_SF_CFG		8
-+#define MASTER_VIDEO_P0			9
-+#define MASTER_VIDEO_P1			10
-+#define MASTER_VIDEO_PROC		11
-+#define MASTER_VIDEO_V_PROC		12
-+#define SLAVE_MNOC_HF_MEM_NOC		13
-+#define SLAVE_MNOC_SF_MEM_NOC		14
-+#define SLAVE_SERVICE_MNOC_HF		15
-+#define SLAVE_SERVICE_MNOC_SF		16
-+
-+/* nspa_noc */
-+#define MASTER_CDSP_NOC_CFG		0
-+#define MASTER_CDSP_PROC		1
-+#define SLAVE_HCP_A			2
-+#define SLAVE_CDSP_MEM_NOC		3
-+#define SLAVE_SERVICE_NSP_NOC		4
-+
-+/* nspb_noc */
-+#define MASTER_CDSPB_NOC_CFG		0
-+#define MASTER_CDSP_PROC_B		1
-+#define SLAVE_CDSPB_MEM_NOC		2
-+#define SLAVE_HCP_B			3
-+#define SLAVE_SERVICE_NSPB_NOC		4
-+
-+/* pcie_anoc */
-+#define MASTER_PCIE_0			0
-+#define MASTER_PCIE_1			1
-+#define SLAVE_ANOC_PCIE_GEM_NOC		2
-+
-+/* system_noc */
-+#define MASTER_GIC_AHB			0
-+#define MASTER_A1NOC_SNOC		1
-+#define MASTER_A2NOC_SNOC		2
-+#define MASTER_LPASS_ANOC		3
-+#define MASTER_SNOC_CFG			4
-+#define MASTER_PIMEM			5
-+#define MASTER_GIC			6
-+#define SLAVE_SNOC_GEM_NOC_GC		7
-+#define SLAVE_SNOC_GEM_NOC_SF		8
-+#define SLAVE_SERVICE_SNOC		9
-+
-+#endif /* __DT_BINDINGS_INTERCONNECT_QCOM_SA8775P_H */
-diff --git a/include/dt-bindings/interconnect/qcom,sar2130p-rpmh.h b/include/dt-bindings/interconnect/qcom,sar2130p-rpmh.h
-new file mode 100644
-index 000000000000..aec7cbb7cd70
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,sar2130p-rpmh.h
-@@ -0,0 +1,137 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2024, Linaro Ltd.
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_SAR2130P_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_SAR2130P_H
-+
-+#define MASTER_QUP_CORE_0			0
-+#define MASTER_QUP_CORE_1			1
-+#define SLAVE_QUP_CORE_0			2
-+#define SLAVE_QUP_CORE_1			3
-+
-+#define MASTER_GEM_NOC_CNOC			0
-+#define MASTER_GEM_NOC_PCIE_SNOC		1
-+#define MASTER_QDSS_DAP				2
-+#define SLAVE_AHB2PHY_SOUTH			3
-+#define SLAVE_AOSS				4
-+#define SLAVE_CAMERA_CFG			5
-+#define SLAVE_CLK_CTL				6
-+#define SLAVE_CDSP_CFG				7
-+#define SLAVE_RBCPR_CX_CFG			8
-+#define SLAVE_RBCPR_MMCX_CFG			9
-+#define SLAVE_RBCPR_MXA_CFG			10
-+#define SLAVE_RBCPR_MXC_CFG			11
-+#define SLAVE_CPR_NSPCX				12
-+#define SLAVE_CRYPTO_0_CFG			13
-+#define SLAVE_CX_RDPM				14
-+#define SLAVE_DISPLAY_CFG			15
-+#define SLAVE_GFX3D_CFG				16
-+#define SLAVE_IMEM_CFG				17
-+#define SLAVE_IPC_ROUTER_CFG			18
-+#define SLAVE_LPASS				19
-+#define SLAVE_MX_RDPM				20
-+#define SLAVE_PCIE_0_CFG			21
-+#define SLAVE_PCIE_1_CFG			22
-+#define SLAVE_PDM				23
-+#define SLAVE_PIMEM_CFG				24
-+#define SLAVE_PRNG				25
-+#define SLAVE_QDSS_CFG				26
-+#define SLAVE_QSPI_0				27
-+#define SLAVE_QUP_0				28
-+#define SLAVE_QUP_1				29
-+#define SLAVE_SDCC_1				30
-+#define SLAVE_TCSR				31
-+#define SLAVE_TLMM				32
-+#define SLAVE_TME_CFG				33
-+#define SLAVE_USB3_0				34
-+#define SLAVE_VENUS_CFG				35
-+#define SLAVE_VSENSE_CTRL_CFG			36
-+#define SLAVE_WLAN_Q6_CFG			37
-+#define SLAVE_DDRSS_CFG				38
-+#define SLAVE_CNOC_MNOC_CFG			39
-+#define SLAVE_SNOC_CFG				40
-+#define SLAVE_IMEM				41
-+#define SLAVE_PIMEM				42
-+#define SLAVE_SERVICE_CNOC			43
-+#define SLAVE_PCIE_0				44
-+#define SLAVE_PCIE_1				45
-+#define SLAVE_QDSS_STM				46
-+#define SLAVE_TCU				47
-+
-+#define MASTER_GPU_TCU				0
-+#define MASTER_SYS_TCU				1
-+#define MASTER_APPSS_PROC			2
-+#define MASTER_GFX3D				3
-+#define MASTER_MNOC_HF_MEM_NOC			4
-+#define MASTER_MNOC_SF_MEM_NOC			5
-+#define MASTER_COMPUTE_NOC			6
-+#define MASTER_ANOC_PCIE_GEM_NOC		7
-+#define MASTER_SNOC_GC_MEM_NOC			8
-+#define MASTER_SNOC_SF_MEM_NOC			9
-+#define MASTER_WLAN_Q6				10
-+#define SLAVE_GEM_NOC_CNOC			11
-+#define SLAVE_LLCC				12
-+#define SLAVE_MEM_NOC_PCIE_SNOC			13
-+
-+#define MASTER_CNOC_LPASS_AG_NOC		0
-+#define MASTER_LPASS_PROC			1
-+#define SLAVE_LPASS_CORE_CFG			2
-+#define SLAVE_LPASS_LPI_CFG			3
-+#define SLAVE_LPASS_MPU_CFG			4
-+#define SLAVE_LPASS_TOP_CFG			5
-+#define SLAVE_LPASS_SNOC			6
-+#define SLAVE_SERVICES_LPASS_AML_NOC		7
-+#define SLAVE_SERVICE_LPASS_AG_NOC		8
-+
-+#define MASTER_LLCC				0
-+#define SLAVE_EBI1				1
-+
-+#define MASTER_CAMNOC_HF			0
-+#define MASTER_CAMNOC_ICP			1
-+#define MASTER_CAMNOC_SF			2
-+#define MASTER_LSR				3
-+#define MASTER_MDP				4
-+#define MASTER_CNOC_MNOC_CFG			5
-+#define MASTER_VIDEO				6
-+#define MASTER_VIDEO_CV_PROC			7
-+#define MASTER_VIDEO_PROC			8
-+#define MASTER_VIDEO_V_PROC			9
-+#define SLAVE_MNOC_HF_MEM_NOC			10
-+#define SLAVE_MNOC_SF_MEM_NOC			11
-+#define SLAVE_SERVICE_MNOC			12
-+
-+#define MASTER_CDSP_NOC_CFG			0
-+#define MASTER_CDSP_PROC			1
-+#define SLAVE_CDSP_MEM_NOC			2
-+#define SLAVE_SERVICE_NSP_NOC			3
-+
-+#define MASTER_PCIE_0				0
-+#define MASTER_PCIE_1				1
-+#define SLAVE_ANOC_PCIE_GEM_NOC			2
-+
-+#define MASTER_GIC_AHB				0
-+#define MASTER_QDSS_BAM				1
-+#define MASTER_QSPI_0				2
-+#define MASTER_QUP_0				3
-+#define MASTER_QUP_1				4
-+#define MASTER_A2NOC_SNOC			5
-+#define MASTER_CNOC_DATAPATH			6
-+#define MASTER_LPASS_ANOC			7
-+#define MASTER_SNOC_CFG				8
-+#define MASTER_CRYPTO				9
-+#define MASTER_PIMEM				10
-+#define MASTER_GIC				11
-+#define MASTER_QDSS_ETR				12
-+#define MASTER_QDSS_ETR_1			13
-+#define MASTER_SDCC_1				14
-+#define MASTER_USB3_0				15
-+#define SLAVE_A2NOC_SNOC			16
-+#define SLAVE_SNOC_GEM_NOC_GC			17
-+#define SLAVE_SNOC_GEM_NOC_SF			18
-+#define SLAVE_SERVICE_SNOC			19
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,sc7180.h b/include/dt-bindings/interconnect/qcom,sc7180.h
-index f9970f6032eb..de5d5867bd67 100644
--- a/include/dt-bindings/interconnect/qcom,sc7180.h
-+++ b/include/dt-bindings/interconnect/qcom,sc7180.h
-@@ -108,9 +108,6 @@
- #define SLAVE_LLCC			11
- #define SLAVE_SERVICE_GEM_NOC			12
- 
-#define MASTER_IPA_CORE			0
-#define SLAVE_IPA_CORE			1
-
- #define MASTER_LLCC			0
- #define SLAVE_EBI1			1
- 
-diff --git a/include/dt-bindings/interconnect/qcom,sc8180x.h b/include/dt-bindings/interconnect/qcom,sc8180x.h
-index e84cfec5afdd..0bdc8d6cb401 100644
--- a/include/dt-bindings/interconnect/qcom,sc8180x.h
-+++ b/include/dt-bindings/interconnect/qcom,sc8180x.h
-@@ -129,9 +129,6 @@
- #define SLAVE_SERVICE_GEM_NOC			16
- #define SLAVE_SERVICE_GEM_NOC_1			17
- 
-#define MASTER_IPA_CORE				0
-#define SLAVE_IPA_CORE				1
-
- #define MASTER_LLCC				0
- #define SLAVE_EBI_CH0				1
- 
-diff --git a/include/dt-bindings/interconnect/qcom,sc8280xp.h b/include/dt-bindings/interconnect/qcom,sc8280xp.h
-index a3e5fda7c127..f89f47e99c6d 100644
--- a/include/dt-bindings/interconnect/qcom,sc8280xp.h
-+++ b/include/dt-bindings/interconnect/qcom,sc8280xp.h
-@@ -48,11 +48,11 @@
- #define SLAVE_SERVICE_A2NOC		19
- 
- /* clk_virt */
-#define MASTER_IPA_CORE			0
-+/* 0 was used by MASTER_IPA_CORE, now represented as RPMh clock */
- #define MASTER_QUP_CORE_0		1
- #define MASTER_QUP_CORE_1		2
- #define MASTER_QUP_CORE_2		3
-#define SLAVE_IPA_CORE			4
-+/* 4 was used by SLAVE_IPA_CORE, now represented as RPMh clock */
- #define SLAVE_QUP_CORE_0		5
- #define SLAVE_QUP_CORE_1		6
- #define SLAVE_QUP_CORE_2		7
-diff --git a/include/dt-bindings/interconnect/qcom,sdm670-rpmh.h b/include/dt-bindings/interconnect/qcom,sdm670-rpmh.h
-new file mode 100644
-index 000000000000..9b516cc360bb
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,sdm670-rpmh.h
-@@ -0,0 +1,136 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Qualcomm SDM670 interconnect IDs
-+ *
-+ * Copyright (c) 2022, The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_SDM670_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_SDM670_H
-+
-+#define MASTER_A1NOC_CFG		0
-+#define MASTER_BLSP_1			1
-+#define MASTER_TSIF			2
-+#define MASTER_EMMC			3
-+#define MASTER_SDCC_2			4
-+#define MASTER_SDCC_4			5
-+#define MASTER_UFS_MEM			6
-+#define SLAVE_A1NOC_SNOC		7
-+#define SLAVE_SERVICE_A1NOC		8
-+
-+#define MASTER_A2NOC_CFG		0
-+#define MASTER_QDSS_BAM			1
-+#define MASTER_BLSP_2			2
-+#define MASTER_CNOC_A2NOC		3
-+#define MASTER_CRYPTO_CORE_0		4
-+#define MASTER_IPA			5
-+#define MASTER_QDSS_ETR			6
-+#define MASTER_USB3			7
-+#define SLAVE_A2NOC_SNOC		8
-+#define SLAVE_SERVICE_A2NOC		9
-+
-+
-+#define MASTER_SPDM			0
-+#define MASTER_SNOC_CNOC		1
-+#define SLAVE_A1NOC_CFG			2
-+#define SLAVE_A2NOC_CFG			3
-+#define SLAVE_AOP			4
-+#define SLAVE_AOSS			5
-+#define SLAVE_CAMERA_CFG		6
-+#define SLAVE_CLK_CTL			7
-+#define SLAVE_CDSP_CFG			8
-+#define SLAVE_RBCPR_CX_CFG		9
-+#define SLAVE_CRYPTO_0_CFG		10
-+#define SLAVE_DCC_CFG			11
-+#define SLAVE_CNOC_DDRSS		12
-+#define SLAVE_DISPLAY_CFG		13
-+#define SLAVE_EMMC_CFG			14
-+#define SLAVE_GLM			15
-+#define SLAVE_GRAPHICS_3D_CFG		16
-+#define SLAVE_IMEM_CFG			17
-+#define SLAVE_IPA_CFG			18
-+#define SLAVE_CNOC_MNOC_CFG		19
-+#define SLAVE_PDM			20
-+#define SLAVE_SOUTH_PHY_CFG		21
-+#define SLAVE_PIMEM_CFG			22
-+#define SLAVE_PRNG			23
-+#define SLAVE_QDSS_CFG			24
-+#define SLAVE_BLSP_2			25
-+#define SLAVE_BLSP_1			26
-+#define SLAVE_SDCC_2			27
-+#define SLAVE_SDCC_4			28
-+#define SLAVE_SNOC_CFG			29
-+#define SLAVE_SPDM_WRAPPER		30
-+#define SLAVE_TCSR			31
-+#define SLAVE_TLMM_NORTH		32
-+#define SLAVE_TLMM_SOUTH		33
-+#define SLAVE_TSIF			34
-+#define SLAVE_UFS_MEM_CFG		35
-+#define SLAVE_USB3			36
-+#define SLAVE_VENUS_CFG			37
-+#define SLAVE_VSENSE_CTRL_CFG		38
-+#define SLAVE_CNOC_A2NOC		39
-+#define SLAVE_SERVICE_CNOC		40
-+
-+#define MASTER_CNOC_DC_NOC		0
-+#define SLAVE_LLCC_CFG			1
-+#define SLAVE_MEM_NOC_CFG		2
-+
-+#define MASTER_AMPSS_M0			0
-+#define MASTER_GNOC_CFG			1
-+#define SLAVE_GNOC_SNOC			2
-+#define SLAVE_GNOC_MEM_NOC		3
-+#define SLAVE_SERVICE_GNOC		4
-+
-+#define MASTER_TCU_0			0
-+#define MASTER_MEM_NOC_CFG		1
-+#define MASTER_GNOC_MEM_NOC		2
-+#define MASTER_MNOC_HF_MEM_NOC		3
-+#define MASTER_MNOC_SF_MEM_NOC		4
-+#define MASTER_SNOC_GC_MEM_NOC		5
-+#define MASTER_SNOC_SF_MEM_NOC		6
-+#define MASTER_GRAPHICS_3D		7
-+#define SLAVE_MSS_PROC_MS_MPU_CFG	8
-+#define SLAVE_MEM_NOC_GNOC		9
-+#define SLAVE_LLCC			10
-+#define SLAVE_MEM_NOC_SNOC		11
-+#define SLAVE_SERVICE_MEM_NOC		12
-+#define MASTER_LLCC			13
-+#define SLAVE_EBI_CH0			14
-+
-+#define MASTER_CNOC_MNOC_CFG		0
-+#define MASTER_CAMNOC_HF0		1
-+#define MASTER_CAMNOC_HF1		2
-+#define MASTER_CAMNOC_SF		3
-+#define MASTER_MDP_PORT0		4
-+#define MASTER_MDP_PORT1		5
-+#define MASTER_ROTATOR			6
-+#define MASTER_VIDEO_P0			7
-+#define MASTER_VIDEO_P1			8
-+#define MASTER_VIDEO_PROC		9
-+#define SLAVE_MNOC_SF_MEM_NOC		10
-+#define SLAVE_MNOC_HF_MEM_NOC		11
-+#define SLAVE_SERVICE_MNOC		12
-+
-+#define MASTER_SNOC_CFG			0
-+#define MASTER_A1NOC_SNOC		1
-+#define MASTER_A2NOC_SNOC		2
-+#define MASTER_GNOC_SNOC		3
-+#define MASTER_MEM_NOC_SNOC		4
-+#define MASTER_PIMEM			5
-+#define MASTER_GIC			6
-+#define SLAVE_APPSS			7
-+#define SLAVE_SNOC_CNOC			8
-+#define SLAVE_SNOC_MEM_NOC_GC		9
-+#define SLAVE_SNOC_MEM_NOC_SF		10
-+#define SLAVE_OCIMEM			11
-+#define SLAVE_PIMEM			12
-+#define SLAVE_SERVICE_SNOC		13
-+#define SLAVE_QDSS_STM			14
-+#define SLAVE_TCU			15
-+#define MASTER_CAMNOC_HF0_UNCOMP	16
-+#define MASTER_CAMNOC_HF1_UNCOMP	17
-+#define MASTER_CAMNOC_SF_UNCOMP		18
-+#define SLAVE_CAMNOC_UNCOMP		19
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,sdx55.h b/include/dt-bindings/interconnect/qcom,sdx55.h
-index bfb6524a2d90..1925f0784ab2 100644
--- a/include/dt-bindings/interconnect/qcom,sdx55.h
-+++ b/include/dt-bindings/interconnect/qcom,sdx55.h
-@@ -70,7 +70,5 @@
- #define SLAVE_QDSS_STM			48
- #define SLAVE_TCU			49
- 
-#define MASTER_IPA_CORE			0
-#define SLAVE_IPA_CORE			1
- 
- #endif
-diff --git a/include/dt-bindings/interconnect/qcom,sdx75.h b/include/dt-bindings/interconnect/qcom,sdx75.h
-new file mode 100644
-index 000000000000..0e19ee8f1687
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,sdx75.h
-@@ -0,0 +1,100 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_SDX75_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_SDX75_H
-+
-+#define MASTER_QUP_CORE_0		1
-+#define SLAVE_QUP_CORE_0		3
-+
-+#define MASTER_LLCC			0
-+#define SLAVE_EBI1			1
-+
-+#define MASTER_CNOC_DC_NOC		0
-+#define SLAVE_LAGG_CFG			1
-+#define SLAVE_MCCC_MASTER		2
-+#define SLAVE_GEM_NOC_CFG		3
-+#define SLAVE_SNOOP_BWMON		4
-+
-+#define MASTER_SYS_TCU			0
-+#define MASTER_APPSS_PROC		1
-+#define MASTER_GEM_NOC_CFG		2
-+#define MASTER_MSS_PROC			3
-+#define MASTER_ANOC_PCIE_GEM_NOC	4
-+#define MASTER_SNOC_SF_MEM_NOC		5
-+#define MASTER_GIC			6
-+#define MASTER_IPA_PCIE			7
-+#define SLAVE_GEM_NOC_CNOC		8
-+#define SLAVE_LLCC			9
-+#define SLAVE_MEM_NOC_PCIE_SNOC		10
-+#define SLAVE_SERVICE_GEM_NOC		11
-+
-+#define MASTER_PCIE_0			0
-+#define MASTER_PCIE_1			1
-+#define MASTER_PCIE_2			2
-+#define SLAVE_ANOC_PCIE_GEM_NOC		3
-+
-+#define MASTER_AUDIO			0
-+#define MASTER_GIC_AHB			1
-+#define MASTER_PCIE_RSCC		2
-+#define MASTER_QDSS_BAM			3
-+#define MASTER_QPIC			4
-+#define MASTER_QUP_0			5
-+#define MASTER_ANOC_SNOC		6
-+#define MASTER_GEM_NOC_CNOC		7
-+#define MASTER_GEM_NOC_PCIE_SNOC	8
-+#define MASTER_SNOC_CFG			9
-+#define MASTER_PCIE_ANOC_CFG		10
-+#define MASTER_CRYPTO			11
-+#define MASTER_IPA			12
-+#define MASTER_MVMSS			13
-+#define MASTER_EMAC_0			14
-+#define MASTER_EMAC_1			15
-+#define MASTER_QDSS_ETR			16
-+#define MASTER_QDSS_ETR_1		17
-+#define MASTER_SDCC_1			18
-+#define MASTER_SDCC_4			19
-+#define MASTER_USB3_0			20
-+#define SLAVE_ETH0_CFG			21
-+#define SLAVE_ETH1_CFG			22
-+#define SLAVE_AUDIO			23
-+#define SLAVE_CLK_CTL			24
-+#define SLAVE_CRYPTO_0_CFG		25
-+#define SLAVE_IMEM_CFG			26
-+#define SLAVE_IPA_CFG			27
-+#define SLAVE_IPC_ROUTER_CFG		28
-+#define SLAVE_CNOC_MSS			29
-+#define SLAVE_ICBDI_MVMSS_CFG		30
-+#define SLAVE_PCIE_0_CFG		31
-+#define SLAVE_PCIE_1_CFG		32
-+#define SLAVE_PCIE_2_CFG		33
-+#define SLAVE_PCIE_RSC_CFG		34
-+#define SLAVE_PDM			35
-+#define SLAVE_PRNG			36
-+#define SLAVE_QDSS_CFG			37
-+#define SLAVE_QPIC			38
-+#define SLAVE_QUP_0			39
-+#define SLAVE_SDCC_1			40
-+#define SLAVE_SDCC_4			41
-+#define SLAVE_SPMI_VGI_COEX		42
-+#define SLAVE_TCSR			43
-+#define SLAVE_TLMM			44
-+#define SLAVE_USB3			45
-+#define SLAVE_USB3_PHY_CFG		46
-+#define SLAVE_A1NOC_CFG			47
-+#define SLAVE_DDRSS_CFG			48
-+#define SLAVE_SNOC_GEM_NOC_SF		49
-+#define SLAVE_SNOC_CFG			50
-+#define SLAVE_PCIE_ANOC_CFG		51
-+#define SLAVE_IMEM			52
-+#define SLAVE_SERVICE_PCIE_ANOC		53
-+#define SLAVE_SERVICE_SNOC		54
-+#define SLAVE_PCIE_0			55
-+#define SLAVE_PCIE_1			56
-+#define SLAVE_PCIE_2			57
-+#define SLAVE_QDSS_STM			58
-+#define SLAVE_TCU			59
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,sm6115.h b/include/dt-bindings/interconnect/qcom,sm6115.h
-new file mode 100644
-index 000000000000..21090e585f05
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,sm6115.h
-@@ -0,0 +1,111 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (c) 2020, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_SM6115_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_SM6115_H
-+
-+/* BIMC */
-+#define MASTER_AMPSS_M0				0
-+#define MASTER_SNOC_BIMC_RT			1
-+#define MASTER_SNOC_BIMC_NRT			2
-+#define SNOC_BIMC_MAS				3
-+#define MASTER_GRAPHICS_3D			4
-+#define MASTER_TCU_0				5
-+#define SLAVE_EBI_CH0				6
-+#define BIMC_SNOC_SLV				7
-+
-+/* CNOC */
-+#define SNOC_CNOC_MAS				0
-+#define MASTER_QDSS_DAP				1
-+#define SLAVE_AHB2PHY_USB			2
-+#define SLAVE_APSS_THROTTLE_CFG			3
-+#define SLAVE_BIMC_CFG				4
-+#define SLAVE_BOOT_ROM				5
-+#define SLAVE_CAMERA_NRT_THROTTLE_CFG		6
-+#define SLAVE_CAMERA_RT_THROTTLE_CFG		7
-+#define SLAVE_CAMERA_CFG			8
-+#define SLAVE_CLK_CTL				9
-+#define SLAVE_RBCPR_CX_CFG			10
-+#define SLAVE_RBCPR_MX_CFG			11
-+#define SLAVE_CRYPTO_0_CFG			12
-+#define SLAVE_DCC_CFG				13
-+#define SLAVE_DDR_PHY_CFG			14
-+#define SLAVE_DDR_SS_CFG			15
-+#define SLAVE_DISPLAY_CFG			16
-+#define SLAVE_DISPLAY_THROTTLE_CFG		17
-+#define SLAVE_GPU_CFG				18
-+#define SLAVE_GPU_THROTTLE_CFG			19
-+#define SLAVE_HWKM_CORE				20
-+#define SLAVE_IMEM_CFG				21
-+#define SLAVE_IPA_CFG				22
-+#define SLAVE_LPASS				23
-+#define SLAVE_MAPSS				24
-+#define SLAVE_MDSP_MPU_CFG			25
-+#define SLAVE_MESSAGE_RAM			26
-+#define SLAVE_CNOC_MSS				27
-+#define SLAVE_PDM				28
-+#define SLAVE_PIMEM_CFG				29
-+#define SLAVE_PKA_CORE				30
-+#define SLAVE_PMIC_ARB				31
-+#define SLAVE_QDSS_CFG				32
-+#define SLAVE_QM_CFG				33
-+#define SLAVE_QM_MPU_CFG			34
-+#define SLAVE_QPIC				35
-+#define SLAVE_QUP_0				36
-+#define SLAVE_RPM				37
-+#define SLAVE_SDCC_1				38
-+#define SLAVE_SDCC_2				39
-+#define SLAVE_SECURITY				40
-+#define SLAVE_SNOC_CFG				41
-+#define SLAVE_TCSR				42
-+#define SLAVE_TLMM				43
-+#define SLAVE_USB3				44
-+#define SLAVE_VENUS_CFG				45
-+#define SLAVE_VENUS_THROTTLE_CFG		46
-+#define SLAVE_VSENSE_CTRL_CFG			47
-+#define SLAVE_SERVICE_CNOC			48
-+
-+/* SNOC */
-+#define MASTER_CRYPTO_CORE0			0
-+#define MASTER_SNOC_CFG				1
-+#define MASTER_TIC				2
-+#define MASTER_ANOC_SNOC			3
-+#define BIMC_SNOC_MAS				4
-+#define MASTER_PIMEM				5
-+#define MASTER_QDSS_BAM				6
-+#define MASTER_QPIC				7
-+#define MASTER_QUP_0				8
-+#define MASTER_IPA				9
-+#define MASTER_QDSS_ETR				10
-+#define MASTER_SDCC_1				11
-+#define MASTER_SDCC_2				12
-+#define MASTER_USB3				13
-+#define SLAVE_APPSS				14
-+#define SNOC_CNOC_SLV				15
-+#define SLAVE_OCIMEM				16
-+#define SLAVE_PIMEM				17
-+#define SNOC_BIMC_SLV				18
-+#define SLAVE_SERVICE_SNOC			19
-+#define SLAVE_QDSS_STM				20
-+#define SLAVE_TCU				21
-+#define SLAVE_ANOC_SNOC				22
-+
-+/* CLK Virtual */
-+#define MASTER_QUP_CORE_0			0
-+#define SLAVE_QUP_CORE_0			1
-+
-+/* MMRT Virtual */
-+#define MASTER_CAMNOC_HF			0
-+#define MASTER_MDP_PORT0			1
-+#define SLAVE_SNOC_BIMC_RT			2
-+
-+/* MMNRT Virtual */
-+#define MASTER_CAMNOC_SF			0
-+#define MASTER_VIDEO_P0				1
-+#define MASTER_VIDEO_PROC			2
-+#define SLAVE_SNOC_BIMC_NRT			3
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,sm7150-rpmh.h b/include/dt-bindings/interconnect/qcom,sm7150-rpmh.h
-new file mode 100644
-index 000000000000..1f610eb832aa
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,sm7150-rpmh.h
-@@ -0,0 +1,150 @@
-+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
-+/*
-+ * Qualcomm SM7150 interconnect IDs
-+ *
-+ * Copyright (c) 2020, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2024, Danila Tikhonov <danila@jiaxyga.com>
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_SM7150_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_SM7150_H
-+
-+#define MASTER_A1NOC_CFG		0
-+#define MASTER_QUP_0			1
-+#define MASTER_TSIF			2
-+#define MASTER_EMMC			3
-+#define MASTER_SDCC_2			4
-+#define MASTER_SDCC_4			5
-+#define MASTER_UFS_MEM			6
-+#define A1NOC_SNOC_SLV			7
-+#define SLAVE_SERVICE_A1NOC		8
-+
-+#define MASTER_A2NOC_CFG		0
-+#define MASTER_QDSS_BAM			1
-+#define MASTER_QUP_1			2
-+#define MASTER_CNOC_A2NOC		3
-+#define MASTER_CRYPTO_CORE_0		4
-+#define MASTER_IPA			5
-+#define MASTER_PCIE			6
-+#define MASTER_QDSS_ETR			7
-+#define MASTER_USB3			8
-+#define A2NOC_SNOC_SLV			9
-+#define SLAVE_ANOC_PCIE_GEM_NOC		10
-+#define SLAVE_SERVICE_A2NOC		11
-+
-+#define MASTER_CAMNOC_HF0_UNCOMP	0
-+#define MASTER_CAMNOC_RT_UNCOMP		1
-+#define MASTER_CAMNOC_SF_UNCOMP		2
-+#define MASTER_CAMNOC_NRT_UNCOMP	3
-+#define SLAVE_CAMNOC_UNCOMP		4
-+
-+#define MASTER_NPU			0
-+#define SLAVE_CDSP_GEM_NOC		1
-+
-+#define MASTER_SPDM			0
-+#define SNOC_CNOC_MAS			1
-+#define MASTER_QDSS_DAP			2
-+#define SLAVE_A1NOC_CFG			3
-+#define SLAVE_A2NOC_CFG			4
-+#define SLAVE_AHB2PHY_NORTH		5
-+#define SLAVE_AHB2PHY_SOUTH		6
-+#define SLAVE_AHB2PHY_WEST		7
-+#define SLAVE_AOP			8
-+#define SLAVE_AOSS			9
-+#define SLAVE_CAMERA_CFG		10
-+#define SLAVE_CAMERA_NRT_THROTTLE_CFG	11
-+#define SLAVE_CAMERA_RT_THROTTLE_CFG	12
-+#define SLAVE_CLK_CTL			13
-+#define SLAVE_CDSP_CFG			14
-+#define SLAVE_RBCPR_CX_CFG		15
-+#define SLAVE_RBCPR_MX_CFG		16
-+#define SLAVE_CRYPTO_0_CFG		17
-+#define SLAVE_CNOC_DDRSS		18
-+#define SLAVE_DISPLAY_CFG		19
-+#define SLAVE_DISPLAY_THROTTLE_CFG	20
-+#define SLAVE_EMMC_CFG			21
-+#define SLAVE_GLM			22
-+#define SLAVE_GRAPHICS_3D_CFG		23
-+#define SLAVE_IMEM_CFG			24
-+#define SLAVE_IPA_CFG			25
-+#define SLAVE_CNOC_MNOC_CFG		26
-+#define SLAVE_PCIE_CFG			27
-+#define SLAVE_PDM			28
-+#define SLAVE_PIMEM_CFG			29
-+#define SLAVE_PRNG			30
-+#define SLAVE_QDSS_CFG			31
-+#define SLAVE_QUP_0			32
-+#define SLAVE_QUP_1			33
-+#define SLAVE_SDCC_2			34
-+#define SLAVE_SDCC_4			35
-+#define SLAVE_SNOC_CFG			36
-+#define SLAVE_SPDM_WRAPPER		37
-+#define SLAVE_TCSR			38
-+#define SLAVE_TLMM_NORTH		39
-+#define SLAVE_TLMM_SOUTH		40
-+#define SLAVE_TLMM_WEST			41
-+#define SLAVE_TSIF			42
-+#define SLAVE_UFS_MEM_CFG		43
-+#define SLAVE_USB3			44
-+#define SLAVE_VENUS_CFG			45
-+#define SLAVE_VENUS_CVP_THROTTLE_CFG	46
-+#define SLAVE_VENUS_THROTTLE_CFG	47
-+#define SLAVE_VSENSE_CTRL_CFG		48
-+#define SLAVE_CNOC_A2NOC		49
-+#define SLAVE_SERVICE_CNOC		50
-+
-+#define MASTER_CNOC_DC_NOC		0
-+#define SLAVE_GEM_NOC_CFG		1
-+#define SLAVE_LLCC_CFG			2
-+
-+#define MASTER_AMPSS_M0			0
-+#define MASTER_SYS_TCU			1
-+#define MASTER_GEM_NOC_CFG		2
-+#define MASTER_COMPUTE_NOC		3
-+#define MASTER_MNOC_HF_MEM_NOC		4
-+#define MASTER_MNOC_SF_MEM_NOC		5
-+#define MASTER_GEM_NOC_PCIE_SNOC	6
-+#define MASTER_SNOC_GC_MEM_NOC		7
-+#define MASTER_SNOC_SF_MEM_NOC		8
-+#define MASTER_GRAPHICS_3D		9
-+#define SLAVE_MSS_PROC_MS_MPU_CFG	10
-+#define SLAVE_GEM_NOC_SNOC		11
-+#define SLAVE_LLCC			12
-+#define SLAVE_SERVICE_GEM_NOC		13
-+
-+
-+#define MASTER_LLCC			0
-+#define SLAVE_EBI_CH0			1
-+
-+#define MASTER_CNOC_MNOC_CFG		0
-+#define MASTER_CAMNOC_HF0		1
-+#define MASTER_CAMNOC_NRT		2
-+#define MASTER_CAMNOC_RT		3
-+#define MASTER_CAMNOC_SF		4
-+#define MASTER_MDP_PORT0		5
-+#define MASTER_MDP_PORT1		6
-+#define MASTER_ROTATOR			7
-+#define MASTER_VIDEO_P0			8
-+#define MASTER_VIDEO_P1			9
-+#define MASTER_VIDEO_PROC		10
-+#define SLAVE_MNOC_SF_MEM_NOC		11
-+#define SLAVE_MNOC_HF_MEM_NOC		12
-+#define SLAVE_SERVICE_MNOC		13
-+
-+#define MASTER_SNOC_CFG			0
-+#define A1NOC_SNOC_MAS			1
-+#define A2NOC_SNOC_MAS			2
-+#define MASTER_GEM_NOC_SNOC		3
-+#define MASTER_PIMEM			4
-+#define MASTER_GIC			5
-+#define SLAVE_APPSS			6
-+#define SNOC_CNOC_SLV			7
-+#define SLAVE_SNOC_GEM_NOC_GC		8
-+#define SLAVE_SNOC_GEM_NOC_SF		9
-+#define SLAVE_OCIMEM			10
-+#define SLAVE_PIMEM			11
-+#define SLAVE_SERVICE_SNOC		12
-+#define SLAVE_QDSS_STM			13
-+#define SLAVE_TCU			14
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,sm8150.h b/include/dt-bindings/interconnect/qcom,sm8150.h
-index a25684680c42..ef292791f52e 100644
--- a/include/dt-bindings/interconnect/qcom,sm8150.h
-+++ b/include/dt-bindings/interconnect/qcom,sm8150.h
-@@ -121,9 +121,6 @@
- #define SLAVE_LLCC			15
- #define SLAVE_SERVICE_GEM_NOC		16
- 
-#define MASTER_IPA_CORE			0
-#define SLAVE_IPA_CORE			1
-
- #define MASTER_LLCC			0
- #define SLAVE_EBI_CH0			1
- 
-diff --git a/include/dt-bindings/interconnect/qcom,sm8250.h b/include/dt-bindings/interconnect/qcom,sm8250.h
-index 1b4d9fbe888d..2a656c02df4b 100644
--- a/include/dt-bindings/interconnect/qcom,sm8250.h
-+++ b/include/dt-bindings/interconnect/qcom,sm8250.h
-@@ -115,9 +115,6 @@
- #define SLAVE_SERVICE_GEM_NOC_2		15
- #define SLAVE_SERVICE_GEM_NOC		16
- 
-#define MASTER_IPA_CORE			0
-#define SLAVE_IPA_CORE			1
-
- #define MASTER_LLCC			0
- #define SLAVE_EBI_CH0			1
- 
-@@ -169,4 +166,11 @@
- #define SLAVE_QDSS_STM			17
- #define SLAVE_TCU			18
- 
-+#define MASTER_QUP_CORE_0		0
-+#define MASTER_QUP_CORE_1		1
-+#define MASTER_QUP_CORE_2		2
-+#define SLAVE_QUP_CORE_0		3
-+#define SLAVE_QUP_CORE_1		4
-+#define SLAVE_QUP_CORE_2		5
-+
- #endif
-diff --git a/include/dt-bindings/interconnect/qcom,sm8350.h b/include/dt-bindings/interconnect/qcom,sm8350.h
-index c7f7ed315aeb..2282f93607bc 100644
--- a/include/dt-bindings/interconnect/qcom,sm8350.h
-+++ b/include/dt-bindings/interconnect/qcom,sm8350.h
-@@ -119,9 +119,6 @@
- #define SLAVE_SERVICE_GEM_NOC_1		16
- #define SLAVE_SERVICE_GEM_NOC_2		17
- #define SLAVE_SERVICE_GEM_NOC		18
-#define MASTER_MNOC_HF_MEM_NOC_DISP	19
-#define MASTER_MNOC_SF_MEM_NOC_DISP	20
-#define SLAVE_LLCC_DISP			21
- 
- #define MASTER_CNOC_LPASS_AG_NOC	0
- #define SLAVE_LPASS_CORE_CFG		1
-@@ -133,8 +130,6 @@
- 
- #define MASTER_LLCC			0
- #define SLAVE_EBI1			1
-#define MASTER_LLCC_DISP		2
-#define SLAVE_EBI1_DISP			3
- 
- #define MASTER_CAMNOC_HF		0
- #define MASTER_CAMNOC_ICP		1
-@@ -149,11 +144,6 @@
- #define SLAVE_MNOC_HF_MEM_NOC		10
- #define SLAVE_MNOC_SF_MEM_NOC		11
- #define SLAVE_SERVICE_MNOC		12
-#define MASTER_MDP0_DISP		13
-#define MASTER_MDP1_DISP		14
-#define MASTER_ROTATOR_DISP		15
-#define SLAVE_MNOC_HF_MEM_NOC_DISP	16
-#define SLAVE_MNOC_SF_MEM_NOC_DISP	17
- 
- #define MASTER_CDSP_NOC_CFG		0
- #define MASTER_CDSP_PROC		1
-diff --git a/include/dt-bindings/interconnect/qcom,sm8550-rpmh.h b/include/dt-bindings/interconnect/qcom,sm8550-rpmh.h
-new file mode 100644
-index 000000000000..b38d0da7886f
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,sm8550-rpmh.h
-@@ -0,0 +1,189 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2022, Linaro Limited
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_SM8550_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_SM8550_H
-+
-+#define MASTER_QSPI_0				0
-+#define MASTER_QUP_1				1
-+#define MASTER_SDCC_4				2
-+#define MASTER_UFS_MEM				3
-+#define MASTER_USB3_0				4
-+#define SLAVE_A1NOC_SNOC			5
-+
-+#define MASTER_QDSS_BAM				0
-+#define MASTER_QUP_2				1
-+#define MASTER_CRYPTO				2
-+#define MASTER_IPA				3
-+#define MASTER_SP				4
-+#define MASTER_QDSS_ETR				5
-+#define MASTER_QDSS_ETR_1			6
-+#define MASTER_SDCC_2				7
-+#define SLAVE_A2NOC_SNOC			8
-+
-+#define MASTER_QUP_CORE_0			0
-+#define MASTER_QUP_CORE_1			1
-+#define MASTER_QUP_CORE_2			2
-+#define SLAVE_QUP_CORE_0			3
-+#define SLAVE_QUP_CORE_1			4
-+#define SLAVE_QUP_CORE_2			5
-+
-+#define MASTER_CNOC_CFG				0
-+#define SLAVE_AHB2PHY_SOUTH			1
-+#define SLAVE_AHB2PHY_NORTH			2
-+#define SLAVE_APPSS				3
-+#define SLAVE_CAMERA_CFG			4
-+#define SLAVE_CLK_CTL				5
-+#define SLAVE_RBCPR_CX_CFG			6
-+#define SLAVE_RBCPR_MMCX_CFG			7
-+#define SLAVE_RBCPR_MXA_CFG			8
-+#define SLAVE_RBCPR_MXC_CFG			9
-+#define SLAVE_CPR_NSPCX				10
-+#define SLAVE_CRYPTO_0_CFG			11
-+#define SLAVE_CX_RDPM				12
-+#define SLAVE_DISPLAY_CFG			13
-+#define SLAVE_GFX3D_CFG				14
-+#define SLAVE_I2C				15
-+#define SLAVE_IMEM_CFG				16
-+#define SLAVE_IPA_CFG				17
-+#define SLAVE_IPC_ROUTER_CFG			18
-+#define SLAVE_CNOC_MSS				19
-+#define SLAVE_MX_RDPM				20
-+#define SLAVE_PCIE_0_CFG			21
-+#define SLAVE_PCIE_1_CFG			22
-+#define SLAVE_PDM				23
-+#define SLAVE_PIMEM_CFG				24
-+#define SLAVE_PRNG				25
-+#define SLAVE_QDSS_CFG				26
-+#define SLAVE_QSPI_0				27
-+#define SLAVE_QUP_1				28
-+#define SLAVE_QUP_2				29
-+#define SLAVE_SDCC_2				30
-+#define SLAVE_SDCC_4				31
-+#define SLAVE_SPSS_CFG				32
-+#define SLAVE_TCSR				33
-+#define SLAVE_TLMM				34
-+#define SLAVE_UFS_MEM_CFG			35
-+#define SLAVE_USB3_0				36
-+#define SLAVE_VENUS_CFG				37
-+#define SLAVE_VSENSE_CTRL_CFG			38
-+#define SLAVE_LPASS_QTB_CFG			39
-+#define SLAVE_CNOC_MNOC_CFG			40
-+#define SLAVE_NSP_QTB_CFG			41
-+#define SLAVE_PCIE_ANOC_CFG			42
-+#define SLAVE_QDSS_STM				43
-+#define SLAVE_TCU				44
-+
-+#define MASTER_GEM_NOC_CNOC			0
-+#define MASTER_GEM_NOC_PCIE_SNOC		1
-+#define SLAVE_AOSS				2
-+#define SLAVE_TME_CFG				3
-+#define SLAVE_CNOC_CFG				4
-+#define SLAVE_DDRSS_CFG				5
-+#define SLAVE_BOOT_IMEM				6
-+#define SLAVE_IMEM				7
-+#define SLAVE_PCIE_0				8
-+#define SLAVE_PCIE_1				9
-+
-+#define MASTER_GPU_TCU				0
-+#define MASTER_SYS_TCU				1
-+#define MASTER_APPSS_PROC			2
-+#define MASTER_GFX3D				3
-+#define MASTER_LPASS_GEM_NOC			4
-+#define MASTER_MSS_PROC				5
-+#define MASTER_MNOC_HF_MEM_NOC			6
-+#define MASTER_MNOC_SF_MEM_NOC			7
-+#define MASTER_COMPUTE_NOC			8
-+#define MASTER_ANOC_PCIE_GEM_NOC		9
-+#define MASTER_SNOC_GC_MEM_NOC			10
-+#define MASTER_SNOC_SF_MEM_NOC			11
-+#define SLAVE_GEM_NOC_CNOC			12
-+#define SLAVE_LLCC				13
-+#define SLAVE_MEM_NOC_PCIE_SNOC			14
-+#define MASTER_MNOC_HF_MEM_NOC_DISP		15
-+#define MASTER_ANOC_PCIE_GEM_NOC_DISP		16
-+#define SLAVE_LLCC_DISP				17
-+#define MASTER_MNOC_HF_MEM_NOC_CAM_IFE_0	18
-+#define MASTER_MNOC_SF_MEM_NOC_CAM_IFE_0	19
-+#define MASTER_ANOC_PCIE_GEM_NOC_CAM_IFE_0	20
-+#define SLAVE_LLCC_CAM_IFE_0			21
-+#define MASTER_MNOC_HF_MEM_NOC_CAM_IFE_1	22
-+#define MASTER_MNOC_SF_MEM_NOC_CAM_IFE_1	23
-+#define MASTER_ANOC_PCIE_GEM_NOC_CAM_IFE_1	24
-+#define SLAVE_LLCC_CAM_IFE_1			25
-+#define MASTER_MNOC_HF_MEM_NOC_CAM_IFE_2	26
-+#define MASTER_MNOC_SF_MEM_NOC_CAM_IFE_2	27
-+#define MASTER_ANOC_PCIE_GEM_NOC_CAM_IFE_2	28
-+#define SLAVE_LLCC_CAM_IFE_2			29
-+
-+#define MASTER_LPIAON_NOC			0
-+#define SLAVE_LPASS_GEM_NOC			1
-+
-+#define MASTER_LPASS_LPINOC			0
-+#define SLAVE_LPIAON_NOC_LPASS_AG_NOC		1
-+
-+#define MASTER_LPASS_PROC			0
-+#define SLAVE_LPICX_NOC_LPIAON_NOC		1
-+
-+#define MASTER_LLCC				0
-+#define SLAVE_EBI1				1
-+#define MASTER_LLCC_DISP			2
-+#define SLAVE_EBI1_DISP				3
-+#define MASTER_LLCC_CAM_IFE_0			4
-+#define SLAVE_EBI1_CAM_IFE_0			5
-+#define MASTER_LLCC_CAM_IFE_1			6
-+#define SLAVE_EBI1_CAM_IFE_1			7
-+#define MASTER_LLCC_CAM_IFE_2			8
-+#define SLAVE_EBI1_CAM_IFE_2			9
-+
-+#define MASTER_CAMNOC_HF			0
-+#define MASTER_CAMNOC_ICP			1
-+#define MASTER_CAMNOC_SF			2
-+#define MASTER_MDP				3
-+#define MASTER_CDSP_HCP				4
-+#define MASTER_VIDEO				5
-+#define MASTER_VIDEO_CV_PROC			6
-+#define MASTER_VIDEO_PROC			7
-+#define MASTER_VIDEO_V_PROC			8
-+#define MASTER_CNOC_MNOC_CFG			9
-+#define SLAVE_MNOC_HF_MEM_NOC			10
-+#define SLAVE_MNOC_SF_MEM_NOC			11
-+#define SLAVE_SERVICE_MNOC			12
-+#define MASTER_MDP_DISP				13
-+#define SLAVE_MNOC_HF_MEM_NOC_DISP		14
-+#define MASTER_CAMNOC_HF_CAM_IFE_0		15
-+#define MASTER_CAMNOC_ICP_CAM_IFE_0		16
-+#define MASTER_CAMNOC_SF_CAM_IFE_0		17
-+#define SLAVE_MNOC_HF_MEM_NOC_CAM_IFE_0		18
-+#define SLAVE_MNOC_SF_MEM_NOC_CAM_IFE_0		19
-+#define MASTER_CAMNOC_HF_CAM_IFE_1		20
-+#define MASTER_CAMNOC_ICP_CAM_IFE_1		21
-+#define MASTER_CAMNOC_SF_CAM_IFE_1		22
-+#define SLAVE_MNOC_HF_MEM_NOC_CAM_IFE_1		23
-+#define SLAVE_MNOC_SF_MEM_NOC_CAM_IFE_1		24
-+#define MASTER_CAMNOC_HF_CAM_IFE_2		25
-+#define MASTER_CAMNOC_ICP_CAM_IFE_2		26
-+#define MASTER_CAMNOC_SF_CAM_IFE_2		27
-+#define SLAVE_MNOC_HF_MEM_NOC_CAM_IFE_2		28
-+#define SLAVE_MNOC_SF_MEM_NOC_CAM_IFE_2		29
-+
-+#define MASTER_CDSP_PROC			0
-+#define SLAVE_CDSP_MEM_NOC			1
-+
-+#define MASTER_PCIE_ANOC_CFG			0
-+#define MASTER_PCIE_0				1
-+#define MASTER_PCIE_1				2
-+#define SLAVE_ANOC_PCIE_GEM_NOC			3
-+#define SLAVE_SERVICE_PCIE_ANOC			4
-+
-+#define MASTER_GIC_AHB				0
-+#define MASTER_A1NOC_SNOC			1
-+#define MASTER_A2NOC_SNOC			2
-+#define MASTER_GIC				3
-+#define SLAVE_SNOC_GEM_NOC_GC			4
-+#define SLAVE_SNOC_GEM_NOC_SF			5
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,sm8650-rpmh.h b/include/dt-bindings/interconnect/qcom,sm8650-rpmh.h
-new file mode 100644
-index 000000000000..1216aa352d55
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,sm8650-rpmh.h
-@@ -0,0 +1,155 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_SM8650_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_SM8650_H
-+
-+#define MASTER_QSPI_0				0
-+#define MASTER_QUP_1				1
-+#define MASTER_QUP_3				2
-+#define MASTER_SDCC_4				3
-+#define MASTER_UFS_MEM				4
-+#define MASTER_USB3_0				5
-+#define SLAVE_A1NOC_SNOC			6
-+
-+#define MASTER_QDSS_BAM				0
-+#define MASTER_QUP_2				1
-+#define MASTER_CRYPTO				2
-+#define MASTER_IPA				3
-+#define MASTER_SP				4
-+#define MASTER_QDSS_ETR				5
-+#define MASTER_QDSS_ETR_1			6
-+#define MASTER_SDCC_2				7
-+#define SLAVE_A2NOC_SNOC			8
-+
-+#define MASTER_QUP_CORE_0			0
-+#define MASTER_QUP_CORE_1			1
-+#define MASTER_QUP_CORE_2			2
-+#define SLAVE_QUP_CORE_0			3
-+#define SLAVE_QUP_CORE_1			4
-+#define SLAVE_QUP_CORE_2			5
-+
-+#define MASTER_CNOC_CFG				0
-+#define SLAVE_AHB2PHY_SOUTH			1
-+#define SLAVE_AHB2PHY_NORTH			2
-+#define SLAVE_CAMERA_CFG			3
-+#define SLAVE_CLK_CTL				4
-+#define SLAVE_RBCPR_CX_CFG			5
-+#define SLAVE_CPR_HMX				6
-+#define SLAVE_RBCPR_MMCX_CFG			7
-+#define SLAVE_RBCPR_MXA_CFG			8
-+#define SLAVE_RBCPR_MXC_CFG			9
-+#define SLAVE_CPR_NSPCX				10
-+#define SLAVE_CRYPTO_0_CFG			11
-+#define SLAVE_CX_RDPM				12
-+#define SLAVE_DISPLAY_CFG			13
-+#define SLAVE_GFX3D_CFG				14
-+#define SLAVE_I2C				15
-+#define SLAVE_I3C_IBI0_CFG			16
-+#define SLAVE_I3C_IBI1_CFG			17
-+#define SLAVE_IMEM_CFG				18
-+#define SLAVE_CNOC_MSS				19
-+#define SLAVE_MX_2_RDPM				20
-+#define SLAVE_MX_RDPM				21
-+#define SLAVE_PCIE_0_CFG			22
-+#define SLAVE_PCIE_1_CFG			23
-+#define SLAVE_PCIE_RSCC				24
-+#define SLAVE_PDM				25
-+#define SLAVE_PRNG				26
-+#define SLAVE_QDSS_CFG				27
-+#define SLAVE_QSPI_0				28
-+#define SLAVE_QUP_3				29
-+#define SLAVE_QUP_1				30
-+#define SLAVE_QUP_2				31
-+#define SLAVE_SDCC_2				32
-+#define SLAVE_SDCC_4				33
-+#define SLAVE_SPSS_CFG				34
-+#define SLAVE_TCSR				35
-+#define SLAVE_TLMM				36
-+#define SLAVE_UFS_MEM_CFG			37
-+#define SLAVE_USB3_0				38
-+#define SLAVE_VENUS_CFG				39
-+#define SLAVE_VSENSE_CTRL_CFG			40
-+#define SLAVE_CNOC_MNOC_CFG			41
-+#define SLAVE_NSP_QTB_CFG			42
-+#define SLAVE_PCIE_ANOC_CFG			43
-+#define SLAVE_SERVICE_CNOC_CFG			44
-+#define SLAVE_QDSS_STM				45
-+#define SLAVE_TCU				46
-+
-+#define MASTER_GEM_NOC_CNOC			0
-+#define MASTER_GEM_NOC_PCIE_SNOC		1
-+#define SLAVE_AOSS				2
-+#define SLAVE_IPA_CFG				3
-+#define SLAVE_IPC_ROUTER_CFG			4
-+#define SLAVE_TME_CFG				5
-+#define SLAVE_APPSS				6
-+#define SLAVE_CNOC_CFG				7
-+#define SLAVE_DDRSS_CFG				8
-+#define SLAVE_IMEM				9
-+#define SLAVE_SERVICE_CNOC			10
-+#define SLAVE_PCIE_0				11
-+#define SLAVE_PCIE_1				12
-+
-+#define MASTER_GPU_TCU				0
-+#define MASTER_SYS_TCU				1
-+#define MASTER_UBWC_P_TCU			2
-+#define MASTER_APPSS_PROC			3
-+#define MASTER_GFX3D				4
-+#define MASTER_LPASS_GEM_NOC			5
-+#define MASTER_MSS_PROC				6
-+#define MASTER_MNOC_HF_MEM_NOC			7
-+#define MASTER_MNOC_SF_MEM_NOC			8
-+#define MASTER_COMPUTE_NOC			9
-+#define MASTER_ANOC_PCIE_GEM_NOC		10
-+#define MASTER_SNOC_SF_MEM_NOC			11
-+#define MASTER_UBWC_P				12
-+#define MASTER_GIC				13
-+#define SLAVE_GEM_NOC_CNOC			14
-+#define SLAVE_LLCC				15
-+#define SLAVE_MEM_NOC_PCIE_SNOC			16
-+
-+#define MASTER_LPIAON_NOC			0
-+#define SLAVE_LPASS_GEM_NOC			1
-+
-+#define MASTER_LPASS_LPINOC			0
-+#define SLAVE_LPIAON_NOC_LPASS_AG_NOC		1
-+
-+#define MASTER_LPASS_PROC			0
-+#define SLAVE_LPICX_NOC_LPIAON_NOC		1
-+
-+#define MASTER_LLCC				0
-+#define SLAVE_EBI1				1
-+
-+#define MASTER_CAMNOC_HF			0
-+#define MASTER_CAMNOC_ICP			1
-+#define MASTER_CAMNOC_SF			2
-+#define MASTER_MDP				3
-+#define MASTER_CDSP_HCP				4
-+#define MASTER_VIDEO				5
-+#define MASTER_VIDEO_CV_PROC			6
-+#define MASTER_VIDEO_PROC			7
-+#define MASTER_VIDEO_V_PROC			8
-+#define MASTER_CNOC_MNOC_CFG			9
-+#define SLAVE_MNOC_HF_MEM_NOC			10
-+#define SLAVE_MNOC_SF_MEM_NOC			11
-+#define SLAVE_SERVICE_MNOC			12
-+
-+#define MASTER_CDSP_PROC			0
-+#define SLAVE_CDSP_MEM_NOC			1
-+
-+#define MASTER_PCIE_ANOC_CFG			0
-+#define MASTER_PCIE_0				1
-+#define MASTER_PCIE_1				2
-+#define SLAVE_ANOC_PCIE_GEM_NOC			3
-+#define SLAVE_SERVICE_PCIE_ANOC			4
-+
-+#define MASTER_A1NOC_SNOC			0
-+#define MASTER_A2NOC_SNOC			1
-+#define SLAVE_SNOC_GEM_NOC_SF			2
-+#define MASTER_APSS_NOC				3
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,sm8750-rpmh.h b/include/dt-bindings/interconnect/qcom,sm8750-rpmh.h
-new file mode 100644
-index 000000000000..30563952a646
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,sm8750-rpmh.h
-@@ -0,0 +1,143 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_SM8750_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_SM8750_H
-+
-+#define MASTER_QSPI_0				0
-+#define MASTER_QUP_1				1
-+#define MASTER_QUP_3				2
-+#define MASTER_SDCC_4				3
-+#define MASTER_UFS_MEM				4
-+#define MASTER_USB3_0				5
-+#define SLAVE_A1NOC_SNOC			6
-+
-+#define MASTER_QDSS_BAM				0
-+#define MASTER_QUP_2				1
-+#define MASTER_CRYPTO				2
-+#define MASTER_IPA				3
-+#define MASTER_SOCCP_AGGR_NOC			4
-+#define MASTER_SP				5
-+#define MASTER_QDSS_ETR				6
-+#define MASTER_QDSS_ETR_1			7
-+#define MASTER_SDCC_2				8
-+#define SLAVE_A2NOC_SNOC			9
-+
-+#define MASTER_QUP_CORE_0			0
-+#define MASTER_QUP_CORE_1			1
-+#define MASTER_QUP_CORE_2			2
-+#define SLAVE_QUP_CORE_0			3
-+#define SLAVE_QUP_CORE_1			4
-+#define SLAVE_QUP_CORE_2			5
-+
-+#define MASTER_CNOC_CFG				0
-+#define SLAVE_AHB2PHY_SOUTH			1
-+#define SLAVE_AHB2PHY_NORTH			2
-+#define SLAVE_CAMERA_CFG			3
-+#define SLAVE_CLK_CTL				4
-+#define SLAVE_CRYPTO_0_CFG			5
-+#define SLAVE_DISPLAY_CFG			6
-+#define SLAVE_EVA_CFG				7
-+#define SLAVE_GFX3D_CFG				8
-+#define SLAVE_I2C				9
-+#define SLAVE_I3C_IBI0_CFG			10
-+#define SLAVE_I3C_IBI1_CFG			11
-+#define SLAVE_IMEM_CFG				12
-+#define SLAVE_CNOC_MSS				13
-+#define SLAVE_PCIE_CFG				14
-+#define SLAVE_PRNG				15
-+#define SLAVE_QDSS_CFG				16
-+#define SLAVE_QSPI_0				17
-+#define SLAVE_QUP_3				18
-+#define SLAVE_QUP_1				19
-+#define SLAVE_QUP_2				20
-+#define SLAVE_SDCC_2				21
-+#define SLAVE_SDCC_4				22
-+#define SLAVE_SPSS_CFG				23
-+#define SLAVE_TCSR				24
-+#define SLAVE_TLMM				25
-+#define SLAVE_UFS_MEM_CFG			26
-+#define SLAVE_USB3_0				27
-+#define SLAVE_VENUS_CFG				28
-+#define SLAVE_VSENSE_CTRL_CFG			29
-+#define SLAVE_CNOC_MNOC_CFG			30
-+#define SLAVE_PCIE_ANOC_CFG			31
-+#define SLAVE_QDSS_STM				32
-+#define SLAVE_TCU				33
-+
-+#define MASTER_GEM_NOC_CNOC			0
-+#define MASTER_GEM_NOC_PCIE_SNOC		1
-+#define SLAVE_AOSS				2
-+#define SLAVE_IPA_CFG				3
-+#define SLAVE_IPC_ROUTER_CFG			4
-+#define SLAVE_SOCCP				5
-+#define SLAVE_TME_CFG				6
-+#define SLAVE_APPSS				7
-+#define SLAVE_CNOC_CFG				8
-+#define SLAVE_DDRSS_CFG				9
-+#define SLAVE_BOOT_IMEM				10
-+#define SLAVE_IMEM				11
-+#define SLAVE_BOOT_IMEM_2			12
-+#define SLAVE_SERVICE_CNOC			13
-+#define SLAVE_PCIE_0				14
-+
-+#define MASTER_GPU_TCU				0
-+#define MASTER_SYS_TCU				1
-+#define MASTER_APPSS_PROC			2
-+#define MASTER_GFX3D				3
-+#define MASTER_LPASS_GEM_NOC			4
-+#define MASTER_MSS_PROC				5
-+#define MASTER_MNOC_HF_MEM_NOC			6
-+#define MASTER_MNOC_SF_MEM_NOC			7
-+#define MASTER_COMPUTE_NOC			8
-+#define MASTER_ANOC_PCIE_GEM_NOC		9
-+#define MASTER_SNOC_SF_MEM_NOC			10
-+#define MASTER_UBWC_P				11
-+#define MASTER_GIC				12
-+#define SLAVE_UBWC_P				13
-+#define SLAVE_GEM_NOC_CNOC			14
-+#define SLAVE_LLCC				15
-+#define SLAVE_MEM_NOC_PCIE_SNOC			16
-+
-+#define MASTER_LPIAON_NOC			0
-+#define SLAVE_LPASS_GEM_NOC			1
-+
-+#define MASTER_LPASS_LPINOC			0
-+#define SLAVE_LPIAON_NOC_LPASS_AG_NOC		1
-+
-+#define MASTER_LPASS_PROC			0
-+#define SLAVE_LPICX_NOC_LPIAON_NOC		1
-+
-+#define MASTER_LLCC				0
-+#define SLAVE_EBI1				1
-+
-+#define MASTER_CAMNOC_HF			0
-+#define MASTER_CAMNOC_NRT_ICP_SF		1
-+#define MASTER_CAMNOC_RT_CDM_SF			2
-+#define MASTER_CAMNOC_SF			3
-+#define MASTER_MDP				4
-+#define MASTER_CDSP_HCP				5
-+#define MASTER_VIDEO_CV_PROC			6
-+#define MASTER_VIDEO_EVA			7
-+#define MASTER_VIDEO_MVP			8
-+#define MASTER_VIDEO_V_PROC			9
-+#define MASTER_CNOC_MNOC_CFG			10
-+#define SLAVE_MNOC_HF_MEM_NOC			11
-+#define SLAVE_MNOC_SF_MEM_NOC			12
-+#define SLAVE_SERVICE_MNOC			13
-+
-+#define MASTER_CDSP_PROC			0
-+#define SLAVE_CDSP_MEM_NOC			1
-+
-+#define MASTER_PCIE_ANOC_CFG			0
-+#define MASTER_PCIE_0				1
-+#define SLAVE_ANOC_PCIE_GEM_NOC			2
-+#define SLAVE_SERVICE_PCIE_ANOC			3
-+
-+#define MASTER_A1NOC_SNOC			0
-+#define MASTER_A2NOC_SNOC			1
-+#define SLAVE_SNOC_GEM_NOC_SF			2
-+
-+#endif
-diff --git a/include/dt-bindings/interconnect/qcom,x1e80100-rpmh.h b/include/dt-bindings/interconnect/qcom,x1e80100-rpmh.h
-new file mode 100644
-index 000000000000..7d9710881149
--- /dev/null
-+++ b/include/dt-bindings/interconnect/qcom,x1e80100-rpmh.h
-@@ -0,0 +1,183 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_X1E80100_H
-+#define __DT_BINDINGS_INTERCONNECT_QCOM_X1E80100_H
-+
-+#define MASTER_QSPI_0				0
-+#define MASTER_QUP_1				1
-+#define MASTER_SDCC_4				2
-+#define MASTER_UFS_MEM				3
-+#define SLAVE_A1NOC_SNOC			4
-+
-+#define MASTER_QUP_0				0
-+#define MASTER_QUP_2				1
-+#define MASTER_CRYPTO				2
-+#define MASTER_SP				3
-+#define MASTER_QDSS_ETR				4
-+#define MASTER_QDSS_ETR_1			5
-+#define MASTER_SDCC_2				6
-+#define SLAVE_A2NOC_SNOC			7
-+
-+#define MASTER_DDR_PERF_MODE			0
-+#define MASTER_QUP_CORE_0			1
-+#define MASTER_QUP_CORE_1			2
-+#define MASTER_QUP_CORE_2			3
-+#define SLAVE_DDR_PERF_MODE			4
-+#define SLAVE_QUP_CORE_0			5
-+#define SLAVE_QUP_CORE_1			6
-+#define SLAVE_QUP_CORE_2			7
-+
-+#define MASTER_CNOC_CFG				0
-+#define SLAVE_AHB2PHY_SOUTH			1
-+#define SLAVE_AHB2PHY_NORTH			2
-+#define SLAVE_AHB2PHY_2				3
-+#define SLAVE_AV1_ENC_CFG			4
-+#define SLAVE_CAMERA_CFG			5
-+#define SLAVE_CLK_CTL				6
-+#define SLAVE_CRYPTO_0_CFG			7
-+#define SLAVE_DISPLAY_CFG			8
-+#define SLAVE_GFX3D_CFG				9
-+#define SLAVE_IMEM_CFG				10
-+#define SLAVE_IPC_ROUTER_CFG			11
-+#define SLAVE_PCIE_0_CFG			12
-+#define SLAVE_PCIE_1_CFG			13
-+#define SLAVE_PCIE_2_CFG			14
-+#define SLAVE_PCIE_3_CFG			15
-+#define SLAVE_PCIE_4_CFG			16
-+#define SLAVE_PCIE_5_CFG			17
-+#define SLAVE_PCIE_6A_CFG			18
-+#define SLAVE_PCIE_6B_CFG			19
-+#define SLAVE_PCIE_RSC_CFG			20
-+#define SLAVE_PDM				21
-+#define SLAVE_PRNG				22
-+#define SLAVE_QDSS_CFG				23
-+#define SLAVE_QSPI_0				24
-+#define SLAVE_QUP_0				25
-+#define SLAVE_QUP_1				26
-+#define SLAVE_QUP_2				27
-+#define SLAVE_SDCC_2				28
-+#define SLAVE_SDCC_4				29
-+#define SLAVE_SMMUV3_CFG			30
-+#define SLAVE_TCSR				31
-+#define SLAVE_TLMM				32
-+#define SLAVE_UFS_MEM_CFG			33
-+#define SLAVE_USB2				34
-+#define SLAVE_USB3_0				35
-+#define SLAVE_USB3_1				36
-+#define SLAVE_USB3_2				37
-+#define SLAVE_USB3_MP				38
-+#define SLAVE_USB4_0				39
-+#define SLAVE_USB4_1				40
-+#define SLAVE_USB4_2				41
-+#define SLAVE_VENUS_CFG				42
-+#define SLAVE_LPASS_QTB_CFG			43
-+#define SLAVE_CNOC_MNOC_CFG			44
-+#define SLAVE_NSP_QTB_CFG			45
-+#define SLAVE_QDSS_STM				46
-+#define SLAVE_TCU				47
-+
-+#define MASTER_GEM_NOC_CNOC			0
-+#define MASTER_GEM_NOC_PCIE_SNOC		1
-+#define SLAVE_AOSS				2
-+#define SLAVE_TME_CFG				3
-+#define SLAVE_APPSS				4
-+#define SLAVE_CNOC_CFG				5
-+#define SLAVE_BOOT_IMEM				6
-+#define SLAVE_IMEM				7
-+#define SLAVE_PCIE_0				8
-+#define SLAVE_PCIE_1				9
-+#define SLAVE_PCIE_2				10
-+#define SLAVE_PCIE_3				11
-+#define SLAVE_PCIE_4				12
-+#define SLAVE_PCIE_5				13
-+#define SLAVE_PCIE_6A				14
-+#define SLAVE_PCIE_6B				15
-+
-+#define MASTER_GPU_TCU				0
-+#define MASTER_PCIE_TCU				1
-+#define MASTER_SYS_TCU				2
-+#define MASTER_APPSS_PROC			3
-+#define MASTER_GFX3D				4
-+#define MASTER_LPASS_GEM_NOC			5
-+#define MASTER_MNOC_HF_MEM_NOC			6
-+#define MASTER_MNOC_SF_MEM_NOC			7
-+#define MASTER_COMPUTE_NOC			8
-+#define MASTER_ANOC_PCIE_GEM_NOC		9
-+#define MASTER_SNOC_SF_MEM_NOC			10
-+#define MASTER_GIC2				11
-+#define SLAVE_GEM_NOC_CNOC			12
-+#define SLAVE_LLCC				13
-+#define SLAVE_MEM_NOC_PCIE_SNOC			14
-+
-+#define MASTER_LPIAON_NOC			0
-+#define SLAVE_LPASS_GEM_NOC			1
-+
-+#define MASTER_LPASS_LPINOC			0
-+#define SLAVE_LPIAON_NOC_LPASS_AG_NOC		1
-+
-+#define MASTER_LPASS_PROC			0
-+#define SLAVE_LPICX_NOC_LPIAON_NOC		1
-+
-+#define MASTER_LLCC				0
-+#define SLAVE_EBI1				1
-+
-+#define MASTER_AV1_ENC				0
-+#define MASTER_CAMNOC_HF			1
-+#define MASTER_CAMNOC_ICP			2
-+#define MASTER_CAMNOC_SF			3
-+#define MASTER_EVA				4
-+#define MASTER_MDP				5
-+#define MASTER_VIDEO				6
-+#define MASTER_VIDEO_CV_PROC			7
-+#define MASTER_VIDEO_V_PROC			8
-+#define MASTER_CNOC_MNOC_CFG			9
-+#define SLAVE_MNOC_HF_MEM_NOC			10
-+#define SLAVE_MNOC_SF_MEM_NOC			11
-+#define SLAVE_SERVICE_MNOC			12
-+
-+#define MASTER_CDSP_PROC			0
-+#define SLAVE_CDSP_MEM_NOC			1
-+
-+#define MASTER_PCIE_NORTH			0
-+#define MASTER_PCIE_SOUTH			1
-+#define SLAVE_ANOC_PCIE_GEM_NOC			2
-+
-+#define MASTER_PCIE_3				0
-+#define MASTER_PCIE_4				1
-+#define MASTER_PCIE_5				2
-+#define SLAVE_PCIE_NORTH			3
-+
-+#define MASTER_PCIE_0				0
-+#define MASTER_PCIE_1				1
-+#define MASTER_PCIE_2				2
-+#define MASTER_PCIE_6A				3
-+#define MASTER_PCIE_6B				4
-+#define SLAVE_PCIE_SOUTH			5
-+
-+#define MASTER_A1NOC_SNOC			0
-+#define MASTER_A2NOC_SNOC			1
-+#define MASTER_GIC1				2
-+#define MASTER_USB_NOC_SNOC			3
-+#define SLAVE_SNOC_GEM_NOC_SF			4
-+
-+#define MASTER_AGGRE_USB_NORTH			0
-+#define MASTER_AGGRE_USB_SOUTH			1
-+#define SLAVE_USB_NOC_SNOC			2
-+
-+#define MASTER_USB2				0
-+#define MASTER_USB3_MP				1
-+#define SLAVE_AGGRE_USB_NORTH			2
-+
-+#define MASTER_USB3_0				0
-+#define MASTER_USB3_1				1
-+#define MASTER_USB3_2				2
-+#define MASTER_USB4_0				3
-+#define MASTER_USB4_1				4
-+#define MASTER_USB4_2				5
-+#define SLAVE_AGGRE_USB_SOUTH			6
-+
-+#endif
-diff --git a/include/dt-bindings/interrupt-controller/amlogic,meson-g12a-gpio-intc.h b/include/dt-bindings/interrupt-controller/amlogic,meson-g12a-gpio-intc.h
-new file mode 100644
-index 000000000000..bd415cb7b669
--- /dev/null
-+++ b/include/dt-bindings/interrupt-controller/amlogic,meson-g12a-gpio-intc.h
-@@ -0,0 +1,126 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (c) 2023 Amlogic, Inc. All rights reserved.
-+ * Author: Huqiang Qin <huqiang.qin@amlogic.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_IRQ_MESON_G12A_GPIO_H
-+#define _DT_BINDINGS_IRQ_MESON_G12A_GPIO_H
-+
-+/* IRQID[11:0] - GPIOAO[11:0] */
-+#define IRQID_GPIOAO_0		0
-+#define IRQID_GPIOAO_1		1
-+#define IRQID_GPIOAO_2		2
-+#define IRQID_GPIOAO_3		3
-+#define IRQID_GPIOAO_4		4
-+#define IRQID_GPIOAO_5		5
-+#define IRQID_GPIOAO_6		6
-+#define IRQID_GPIOAO_7		7
-+#define IRQID_GPIOAO_8		8
-+#define IRQID_GPIOAO_9		9
-+#define IRQID_GPIOAO_10		10
-+#define IRQID_GPIOAO_11		11
-+
-+/* IRQID[27:12] - GPIOZ[15:0] */
-+#define IRQID_GPIOZ_0		12
-+#define IRQID_GPIOZ_1		13
-+#define IRQID_GPIOZ_2		14
-+#define IRQID_GPIOZ_3		15
-+#define IRQID_GPIOZ_4		16
-+#define IRQID_GPIOZ_5		17
-+#define IRQID_GPIOZ_6		18
-+#define IRQID_GPIOZ_7		19
-+#define IRQID_GPIOZ_8		20
-+#define IRQID_GPIOZ_9		21
-+#define IRQID_GPIOZ_10		22
-+#define IRQID_GPIOZ_11		23
-+#define IRQID_GPIOZ_12		24
-+#define IRQID_GPIOZ_13		25
-+#define IRQID_GPIOZ_14		26
-+#define IRQID_GPIOZ_15		27
-+
-+/* IRQID[36:28] - GPIOH[8:0] */
-+#define IRQID_GPIOH_0		28
-+#define IRQID_GPIOH_1		29
-+#define IRQID_GPIOH_2		30
-+#define IRQID_GPIOH_3		31
-+#define IRQID_GPIOH_4		32
-+#define IRQID_GPIOH_5		33
-+#define IRQID_GPIOH_6		34
-+#define IRQID_GPIOH_7		35
-+#define IRQID_GPIOH_8		36
-+
-+/* IRQID[52:37] - BOOT[15:0] */
-+#define IRQID_BOOT_0		37
-+#define IRQID_BOOT_1		38
-+#define IRQID_BOOT_2		39
-+#define IRQID_BOOT_3		40
-+#define IRQID_BOOT_4		41
-+#define IRQID_BOOT_5		42
-+#define IRQID_BOOT_6		43
-+#define IRQID_BOOT_7		44
-+#define IRQID_BOOT_8		45
-+#define IRQID_BOOT_9		46
-+#define IRQID_BOOT_10		47
-+#define IRQID_BOOT_11		48
-+#define IRQID_BOOT_12		49
-+#define IRQID_BOOT_13		50
-+#define IRQID_BOOT_14		51
-+#define IRQID_BOOT_15		52
-+
-+/* IRQID[60:53] - GPIOC[7:0] */
-+#define IRQID_GPIOC_0		53
-+#define IRQID_GPIOC_1		54
-+#define IRQID_GPIOC_2		55
-+#define IRQID_GPIOC_3		56
-+#define IRQID_GPIOC_4		57
-+#define IRQID_GPIOC_5		58
-+#define IRQID_GPIOC_6		59
-+#define IRQID_GPIOC_7		60
-+
-+/* IRQID[76:61] - GPIOA[15:0] */
-+#define IRQID_GPIOA_0		61
-+#define IRQID_GPIOA_1		62
-+#define IRQID_GPIOA_2		63
-+#define IRQID_GPIOA_3		64
-+#define IRQID_GPIOA_4		65
-+#define IRQID_GPIOA_5		66
-+#define IRQID_GPIOA_6		67
-+#define IRQID_GPIOA_7		68
-+#define IRQID_GPIOA_8		69
-+#define IRQID_GPIOA_9		70
-+#define IRQID_GPIOA_10		71
-+#define IRQID_GPIOA_11		72
-+#define IRQID_GPIOA_12		73
-+#define IRQID_GPIOA_13		74
-+#define IRQID_GPIOA_14		75
-+#define IRQID_GPIOA_15		76
-+
-+/* IRQID[96:77] - GPIOX[19:0] */
-+#define IRQID_GPIOX_0		77
-+#define IRQID_GPIOX_1		78
-+#define IRQID_GPIOX_2		79
-+#define IRQID_GPIOX_3		80
-+#define IRQID_GPIOX_4		81
-+#define IRQID_GPIOX_5		82
-+#define IRQID_GPIOX_6		83
-+#define IRQID_GPIOX_7		84
-+#define IRQID_GPIOX_8		85
-+#define IRQID_GPIOX_9		86
-+#define IRQID_GPIOX_10		87
-+#define IRQID_GPIOX_11		88
-+#define IRQID_GPIOX_12		89
-+#define IRQID_GPIOX_13		90
-+#define IRQID_GPIOX_14		91
-+#define IRQID_GPIOX_15		92
-+#define IRQID_GPIOX_16		93
-+#define IRQID_GPIOX_17		94
-+#define IRQID_GPIOX_18		95
-+#define IRQID_GPIOX_19		96
-+
-+/* IRQID[99:97] - GPIOE[2:0] */
-+#define IRQID_GPIOE_0		97
-+#define IRQID_GPIOE_1		98
-+#define IRQID_GPIOE_2		99
-+
-+#endif /* _DT_BINDINGS_IRQ_MESON_G12A_GPIO_H */
-diff --git a/include/dt-bindings/interrupt-controller/arm-gic.h b/include/dt-bindings/interrupt-controller/arm-gic.h
-index 35b6f69b7db6..887f53363e8a 100644
--- a/include/dt-bindings/interrupt-controller/arm-gic.h
-+++ b/include/dt-bindings/interrupt-controller/arm-gic.h
-@@ -12,6 +12,8 @@
- 
- #define GIC_SPI 0
- #define GIC_PPI 1
-+#define GIC_ESPI 2
-+#define GIC_EPPI 3
- 
- /*
-  * Interrupt specifier cell 2.
-diff --git a/include/dt-bindings/interrupt-controller/aspeed-scu-ic.h b/include/dt-bindings/interrupt-controller/aspeed-scu-ic.h
-index f315d5a7f5ee..7dd04424afcc 100644
--- a/include/dt-bindings/interrupt-controller/aspeed-scu-ic.h
-+++ b/include/dt-bindings/interrupt-controller/aspeed-scu-ic.h
-@@ -20,4 +20,18 @@
- #define ASPEED_AST2600_SCU_IC1_LPC_RESET_LO_TO_HI	0
- #define ASPEED_AST2600_SCU_IC1_LPC_RESET_HI_TO_LO	1
- 
-+#define ASPEED_AST2700_SCU_IC0_PCIE_PERST_LO_TO_HI	3
-+#define ASPEED_AST2700_SCU_IC0_PCIE_PERST_HI_TO_LO	2
-+
-+#define ASPEED_AST2700_SCU_IC1_PCIE_RCRST_LO_TO_HI	3
-+#define ASPEED_AST2700_SCU_IC1_PCIE_RCRST_HI_TO_LO	2
-+
-+#define ASPEED_AST2700_SCU_IC2_PCIE_PERST_LO_TO_HI	3
-+#define ASPEED_AST2700_SCU_IC2_PCIE_PERST_HI_TO_LO	2
-+#define ASPEED_AST2700_SCU_IC2_LPC_RESET_LO_TO_HI	1
-+#define ASPEED_AST2700_SCU_IC2_LPC_RESET_HI_TO_LO	0
-+
-+#define ASPEED_AST2700_SCU_IC3_LPC_RESET_LO_TO_HI	1
-+#define ASPEED_AST2700_SCU_IC3_LPC_RESET_HI_TO_LO	0
-+
- #endif /* _DT_BINDINGS_INTERRUPT_CONTROLLER_ASPEED_SCU_IC_H_ */
-diff --git a/include/dt-bindings/interrupt-controller/irqc-rzg2l.h b/include/dt-bindings/interrupt-controller/irqc-rzg2l.h
-new file mode 100644
-index 000000000000..34ce778885a1
--- /dev/null
-+++ b/include/dt-bindings/interrupt-controller/irqc-rzg2l.h
-@@ -0,0 +1,25 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * This header provides constants for Renesas RZ/G2L family IRQC bindings.
-+ *
-+ * Copyright (C) 2022 Renesas Electronics Corp.
-+ *
-+ */
-+
-+#ifndef __DT_BINDINGS_IRQC_RZG2L_H
-+#define __DT_BINDINGS_IRQC_RZG2L_H
-+
-+/* NMI maps to SPI0 */
-+#define RZG2L_NMI	0
-+
-+/* IRQ0-7 map to SPI1-8 */
-+#define RZG2L_IRQ0	1
-+#define RZG2L_IRQ1	2
-+#define RZG2L_IRQ2	3
-+#define RZG2L_IRQ3	4
-+#define RZG2L_IRQ4	5
-+#define RZG2L_IRQ5	6
-+#define RZG2L_IRQ6	7
-+#define RZG2L_IRQ7	8
-+
-+#endif /* __DT_BINDINGS_IRQC_RZG2L_H */
-diff --git a/include/dt-bindings/leds/common.h b/include/dt-bindings/leds/common.h
-index 3be89a7c20a9..4f017bea0123 100644
--- a/include/dt-bindings/leds/common.h
-+++ b/include/dt-bindings/leds/common.h
-@@ -33,7 +33,12 @@
- #define LED_COLOR_ID_MULTI	8	/* For multicolor LEDs */
- #define LED_COLOR_ID_RGB	9	/* For multicolor LEDs that can do arbitrary color,
- 					   so this would include RGBW and similar */
-#define LED_COLOR_ID_MAX	10
-+#define LED_COLOR_ID_PURPLE	10
-+#define LED_COLOR_ID_ORANGE	11
-+#define LED_COLOR_ID_PINK	12
-+#define LED_COLOR_ID_CYAN	13
-+#define LED_COLOR_ID_LIME	14
-+#define LED_COLOR_ID_MAX	15
- 
- /* Standard LED functions */
- /* Keyboard LEDs, usually it would be input4::capslock etc. */
-@@ -41,6 +46,7 @@
- #define LED_FUNCTION_CAPSLOCK "capslock"
- #define LED_FUNCTION_SCROLLLOCK "scrolllock"
- #define LED_FUNCTION_NUMLOCK "numlock"
-+#define LED_FUNCTION_FNLOCK "fnlock"
- /*   Obsolete equivalents: "tpacpi::thinklight" (IBM/Lenovo Thinkpads),
-      "lp5523:kb{1,2,3,4,5,6}" (Nokia N900) */
- #define LED_FUNCTION_KBD_BACKLIGHT "kbd_backlight"
-@@ -85,17 +91,24 @@
- #define LED_FUNCTION_INDICATOR "indicator"
- #define LED_FUNCTION_LAN "lan"
- #define LED_FUNCTION_MAIL "mail"
-+#define LED_FUNCTION_MOBILE "mobile"
- #define LED_FUNCTION_MTD "mtd"
- #define LED_FUNCTION_PANIC "panic"
- #define LED_FUNCTION_PROGRAMMING "programming"
- #define LED_FUNCTION_RX "rx"
- #define LED_FUNCTION_SD "sd"
-+#define LED_FUNCTION_SPEED_LAN "speed-lan"
-+#define LED_FUNCTION_SPEED_WAN "speed-wan"
- #define LED_FUNCTION_STANDBY "standby"
- #define LED_FUNCTION_TORCH "torch"
- #define LED_FUNCTION_TX "tx"
- #define LED_FUNCTION_USB "usb"
- #define LED_FUNCTION_WAN "wan"
-+#define LED_FUNCTION_WAN_ONLINE "wan-online"
- #define LED_FUNCTION_WLAN "wlan"
-+#define LED_FUNCTION_WLAN_2GHZ "wlan-2ghz"
-+#define LED_FUNCTION_WLAN_5GHZ "wlan-5ghz"
-+#define LED_FUNCTION_WLAN_6GHZ "wlan-6ghz"
- #define LED_FUNCTION_WPS "wps"
- 
- #endif /* __DT_BINDINGS_LEDS_H */
-diff --git a/include/dt-bindings/leds/leds-lp55xx.h b/include/dt-bindings/leds/leds-lp55xx.h
-new file mode 100644
-index 000000000000..a4fb4567715d
--- /dev/null
-+++ b/include/dt-bindings/leds/leds-lp55xx.h
-@@ -0,0 +1,10 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+#ifndef _DT_BINDINGS_LEDS_LP55XX_H
-+#define _DT_BINDINGS_LEDS_LP55XX_H
-+
-+#define LP55XX_CP_OFF		0
-+#define LP55XX_CP_BYPASS	1
-+#define LP55XX_CP_BOOST		2
-+#define LP55XX_CP_AUTO		3
-+
-+#endif /* _DT_BINDINGS_LEDS_LP55XX_H */
-diff --git a/include/dt-bindings/mailbox/mediatek,mt8188-gce.h b/include/dt-bindings/mailbox/mediatek,mt8188-gce.h
-new file mode 100644
-index 000000000000..119865787b47
--- /dev/null
-+++ b/include/dt-bindings/mailbox/mediatek,mt8188-gce.h
-@@ -0,0 +1,967 @@
-+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
-+/*
-+ * Copyright (c) 2022 MediaTek Inc.
-+ *
-+ */
-+#ifndef _DT_BINDINGS_GCE_MT8188_H
-+#define _DT_BINDINGS_GCE_MT8188_H
-+
-+#define CMDQ_THR_PRIO_LOWEST		0
-+#define CMDQ_THR_PRIO_1			1
-+#define CMDQ_THR_PRIO_2			2
-+#define CMDQ_THR_PRIO_3			3
-+#define CMDQ_THR_PRIO_4			4
-+#define CMDQ_THR_PRIO_5			5
-+#define CMDQ_THR_PRIO_6			6
-+#define CMDQ_THR_PRIO_HIGHEST		7
-+
-+#define SUBSYS_1400XXXX                 0
-+#define SUBSYS_1401XXXX			1
-+#define SUBSYS_1402XXXX			2
-+#define SUBSYS_1c00XXXX			3
-+#define SUBSYS_1c01XXXX			4
-+#define SUBSYS_1c02XXXX			5
-+#define SUBSYS_1c10XXXX			6
-+#define SUBSYS_1c11XXXX			7
-+#define SUBSYS_1c12XXXX			8
-+#define SUBSYS_14f0XXXX			9
-+#define SUBSYS_14f1XXXX			10
-+#define SUBSYS_14f2XXXX			11
-+#define SUBSYS_1800XXXX			12
-+#define SUBSYS_1801XXXX			13
-+#define SUBSYS_1802XXXX			14
-+#define SUBSYS_1803XXXX			15
-+#define SUBSYS_1032XXXX			16
-+#define SUBSYS_1033XXXX			17
-+#define SUBSYS_1600XXXX			18
-+#define SUBSYS_1601XXXX			19
-+#define SUBSYS_14e0XXXX			20
-+#define SUBSYS_1c20XXXX			21
-+#define SUBSYS_1c30XXXX			22
-+#define SUBSYS_1c40XXXX			23
-+#define SUBSYS_1c50XXXX			24
-+#define SUBSYS_1c60XXXX			25
-+#define SUBSYS_NO_SUPPORT		99
-+
-+#define CMDQ_EVENT_IMG_SOF				0
-+#define CMDQ_EVENT_IMG_TRAW0_CQ_THR_DONE_0		1
-+#define CMDQ_EVENT_IMG_TRAW0_CQ_THR_DONE_1		2
-+#define CMDQ_EVENT_IMG_TRAW0_CQ_THR_DONE_2		3
-+#define CMDQ_EVENT_IMG_TRAW0_CQ_THR_DONE_3		4
-+#define CMDQ_EVENT_IMG_TRAW0_CQ_THR_DONE_4		5
-+#define CMDQ_EVENT_IMG_TRAW0_CQ_THR_DONE_5		6
-+#define CMDQ_EVENT_IMG_TRAW0_CQ_THR_DONE_6		7
-+#define CMDQ_EVENT_IMG_TRAW0_CQ_THR_DONE_7		8
-+#define CMDQ_EVENT_IMG_TRAW0_CQ_THR_DONE_8		9
-+#define CMDQ_EVENT_IMG_TRAW0_CQ_THR_DONE_9		10
-+#define CMDQ_EVENT_IMG_TRAW0_DMA_ERROR_INT		11
-+#define CMDQ_EVENT_IMG_TRAW1_CQ_THR_DONE_0		12
-+#define CMDQ_EVENT_IMG_TRAW1_CQ_THR_DONE_1		13
-+#define CMDQ_EVENT_IMG_TRAW1_CQ_THR_DONE_2		14
-+#define CMDQ_EVENT_IMG_TRAW1_CQ_THR_DONE_3		15
-+#define CMDQ_EVENT_IMG_TRAW1_CQ_THR_DONE_4		16
-+#define CMDQ_EVENT_IMG_TRAW1_CQ_THR_DONE_5		17
-+#define CMDQ_EVENT_IMG_TRAW1_CQ_THR_DONE_6		18
-+#define CMDQ_EVENT_IMG_TRAW1_CQ_THR_DONE_7		19
-+#define CMDQ_EVENT_IMG_TRAW1_CQ_THR_DONE_8		20
-+#define CMDQ_EVENT_IMG_TRAW1_CQ_THR_DONE_9		21
-+#define CMDQ_EVENT_IMG_TRAW1_DMA_ERROR_INT		22
-+#define CMDQ_EVENT_IMG_ADL_RESERVED			23
-+#define CMDQ_EVENT_IMG_DIP_CQ_THR_DONE_0		24
-+#define CMDQ_EVENT_IMG_DIP_CQ_THR_DONE_1		25
-+#define CMDQ_EVENT_IMG_DIP_CQ_THR_DONE_2		26
-+#define CMDQ_EVENT_IMG_DIP_CQ_THR_DONE_3		27
-+#define CMDQ_EVENT_IMG_DIP_CQ_THR_DONE_4		28
-+#define CMDQ_EVENT_IMG_DIP_CQ_THR_DONE_5		29
-+#define CMDQ_EVENT_IMG_DIP_CQ_THR_DONE_6		30
-+#define CMDQ_EVENT_IMG_DIP_CQ_THR_DONE_7		31
-+#define CMDQ_EVENT_IMG_DIP_CQ_THR_DONE_8		32
-+#define CMDQ_EVENT_IMG_DIP_CQ_THR_DONE_9		33
-+#define CMDQ_EVENT_IMG_DIP_DMA_ERR			34
-+#define CMDQ_EVENT_IMG_DIP_NR_DMA_ERR			35
-+#define CMDQ_EVENT_DIP_DUMMY_0				36
-+#define CMDQ_EVENT_DIP_DUMMY_1				37
-+#define CMDQ_EVENT_DIP_DUMMY_2				38
-+#define CMDQ_EVENT_IMG_WPE_EIS_GCE_FRAME_DONE	39
-+#define CMDQ_EVENT_IMG_WPE_EIS_DONE_SYNC_OUT	40
-+#define CMDQ_EVENT_IMG_WPE_EIS_CQ_THR_DONE_0	41
-+#define CMDQ_EVENT_IMG_WPE_EIS_CQ_THR_DONE_1	42
-+#define CMDQ_EVENT_IMG_WPE_EIS_CQ_THR_DONE_2	43
-+#define CMDQ_EVENT_IMG_WPE_EIS_CQ_THR_DONE_3	44
-+#define CMDQ_EVENT_IMG_WPE_EIS_CQ_THR_DONE_4	45
-+#define CMDQ_EVENT_IMG_WPE_EIS_CQ_THR_DONE_5	46
-+#define CMDQ_EVENT_IMG_WPE_EIS_CQ_THR_DONE_6	47
-+#define CMDQ_EVENT_IMG_WPE_EIS_CQ_THR_DONE_7	48
-+#define CMDQ_EVENT_IMG_WPE_EIS_CQ_THR_DONE_8	49
-+#define CMDQ_EVENT_IMG_WPE_EIS_CQ_THR_DONE_9	50
-+#define CMDQ_EVENT_IMG_PQDIP_A_CQ_THR_DONE_0	51
-+#define CMDQ_EVENT_IMG_PQDIP_A_CQ_THR_DONE_1	52
-+#define CMDQ_EVENT_IMG_PQDIP_A_CQ_THR_DONE_2	53
-+#define CMDQ_EVENT_IMG_PQDIP_A_CQ_THR_DONE_3	54
-+#define CMDQ_EVENT_IMG_PQDIP_A_CQ_THR_DONE_4	55
-+#define CMDQ_EVENT_IMG_PQDIP_A_CQ_THR_DONE_5	56
-+#define CMDQ_EVENT_IMG_PQDIP_A_CQ_THR_DONE_6	57
-+#define CMDQ_EVENT_IMG_PQDIP_A_CQ_THR_DONE_7	58
-+#define CMDQ_EVENT_IMG_PQDIP_A_CQ_THR_DONE_8	59
-+#define CMDQ_EVENT_IMG_PQDIP_A_CQ_THR_DONE_9	60
-+#define CMDQ_EVENT_IMG_PQDIP_A_DMA_ERR		61
-+#define CMDQ_EVENT_WPE0_DUMMY_0			62
-+#define CMDQ_EVENT_WPE0_DUMMY_1			63
-+#define CMDQ_EVENT_WPE0_DUMMY_2			64
-+#define CMDQ_EVENT_IMG_WPE_TNR_GCE_FRAME_DONE	65
-+#define CMDQ_EVENT_IMG_WPE_TNR_DONE_SYNC_OUT	66
-+#define CMDQ_EVENT_IMG_WPE_TNR_CQ_THR_DONE_0	67
-+#define CMDQ_EVENT_IMG_WPE_TNR_CQ_THR_DONE_1	68
-+#define CMDQ_EVENT_IMG_WPE_TNR_CQ_THR_DONE_2	69
-+#define CMDQ_EVENT_IMG_WPE_TNR_CQ_THR_DONE_3	70
-+#define CMDQ_EVENT_IMG_WPE_TNR_CQ_THR_DONE_4	71
-+#define CMDQ_EVENT_IMG_WPE_TNR_CQ_THR_DONE_5	72
-+#define CMDQ_EVENT_IMG_WPE_TNR_CQ_THR_DONE_6	73
-+#define CMDQ_EVENT_IMG_WPE_TNR_CQ_THR_DONE_7	74
-+#define CMDQ_EVENT_IMG_WPE_TNR_CQ_THR_DONE_8	75
-+#define CMDQ_EVENT_IMG_WPE_TNR_CQ_THR_DONE_9	76
-+#define CMDQ_EVENT_IMG_PQDIP_B_CQ_THR_DONE_0	77
-+#define CMDQ_EVENT_IMG_PQDIP_B_CQ_THR_DONE_1	78
-+#define CMDQ_EVENT_IMG_PQDIP_B_CQ_THR_DONE_2	79
-+#define CMDQ_EVENT_IMG_PQDIP_B_CQ_THR_DONE_3	80
-+#define CMDQ_EVENT_IMG_PQDIP_B_CQ_THR_DONE_4	81
-+#define CMDQ_EVENT_IMG_PQDIP_B_CQ_THR_DONE_5	82
-+#define CMDQ_EVENT_IMG_PQDIP_B_CQ_THR_DONE_6	83
-+#define CMDQ_EVENT_IMG_PQDIP_B_CQ_THR_DONE_7	84
-+#define CMDQ_EVENT_IMG_PQDIP_B_CQ_THR_DONE_8	85
-+#define CMDQ_EVENT_IMG_PQDIP_B_CQ_THR_DONE_9	86
-+#define CMDQ_EVENT_IMG_PQDIP_B_DMA_ERR		87
-+#define CMDQ_EVENT_WPE1_DUMMY_0			88
-+#define CMDQ_EVENT_WPE1_DUMMY_1			89
-+#define CMDQ_EVENT_WPE1_DUMMY_2			90
-+#define CMDQ_EVENT_IMG_WPE_LITE_GCE_FRAME_DONE	91
-+#define CMDQ_EVENT_IMG_WPE_LITE_DONE_SYNC_OUT	92
-+#define CMDQ_EVENT_IMG_WPE_LITE_CQ_THR_DONE_0	93
-+#define CMDQ_EVENT_IMG_WPE_LITE_CQ_THR_DONE_1	94
-+#define CMDQ_EVENT_IMG_WPE_LITE_CQ_THR_DONE_2	95
-+#define CMDQ_EVENT_IMG_WPE_LITE_CQ_THR_DONE_3	96
-+#define CMDQ_EVENT_IMG_WPE_LITE_CQ_THR_DONE_4	97
-+#define CMDQ_EVENT_IMG_WPE_LITE_CQ_THR_DONE_5	98
-+#define CMDQ_EVENT_IMG_WPE_LITE_CQ_THR_DONE_6	99
-+#define CMDQ_EVENT_IMG_WPE_LITE_CQ_THR_DONE_7	100
-+#define CMDQ_EVENT_IMG_WPE_LITE_CQ_THR_DONE_8	101
-+#define CMDQ_EVENT_IMG_WPE_LITE_CQ_THR_DONE_9	102
-+#define CMDQ_EVENT_IMG_XTRAW_CQ_THR_DONE_0	103
-+#define CMDQ_EVENT_IMG_XTRAW_CQ_THR_DONE_1	104
-+#define CMDQ_EVENT_IMG_XTRAW_CQ_THR_DONE_2	105
-+#define CMDQ_EVENT_IMG_XTRAW_CQ_THR_DONE_3	106
-+#define CMDQ_EVENT_IMG_XTRAW_CQ_THR_DONE_4	107
-+#define CMDQ_EVENT_IMG_XTRAW_CQ_THR_DONE_5	108
-+#define CMDQ_EVENT_IMG_XTRAW_CQ_THR_DONE_6	109
-+#define CMDQ_EVENT_IMG_XTRAW_CQ_THR_DONE_7	110
-+#define CMDQ_EVENT_IMG_XTRAW_CQ_THR_DONE_8	111
-+#define CMDQ_EVENT_IMG_XTRAW_CQ_THR_DONE_9	112
-+#define CMDQ_EVENT_IMG_XTRAW_DMA_ERR_EVENT	113
-+#define CMDQ_EVENT_WPE2_DUMMY_0			114
-+#define CMDQ_EVENT_WPE2_DUMMY_1			115
-+#define CMDQ_EVENT_WPE2_DUMMY_2			116
-+#define CMDQ_EVENT_IMG_IMGSYS_IPE_DUMMY		117
-+#define CMDQ_EVENT_IMG_IMGSYS_IPE_FDVT_DONE	118
-+#define CMDQ_EVENT_IMG_IMGSYS_IPE_ME_DONE	119
-+#define CMDQ_EVENT_IMG_IMGSYS_IPE_DVS_DONE	120
-+#define CMDQ_EVENT_IMG_IMGSYS_IPE_DVP_DONE	121
-+#define CMDQ_EVENT_FDVT1_RESERVED		122
-+#define CMDQ_EVENT_IMG_ENG_EVENT		123
-+#define CMDQ_EVENT_CAMSUBA_SW_PASS1_DONE		129
-+#define CMDQ_EVENT_CAMSUBB_SW_PASS1_DONE		130
-+#define CMDQ_EVENT_CAMSUBC_SW_PASS1_DONE		131
-+#define CMDQ_EVENT_GCAMSV_A_1_SW_PASS1_DONE		132
-+#define CMDQ_EVENT_GCAMSV_A_2_SW_PASS1_DONE		133
-+#define CMDQ_EVENT_GCAMSV_B_1_SW_PASS1_DONE		134
-+#define CMDQ_EVENT_GCAMSV_B_2_SW_PASS1_DONE		135
-+#define CMDQ_EVENT_GCAMSV_C_1_SW_PASS1_DONE		136
-+#define CMDQ_EVENT_GCAMSV_C_2_SW_PASS1_DONE		137
-+#define CMDQ_EVENT_GCAMSV_D_1_SW_PASS1_DONE		138
-+#define CMDQ_EVENT_GCAMSV_D_2_SW_PASS1_DONE		139
-+#define CMDQ_EVENT_GCAMSV_E_1_SW_PASS1_DONE		140
-+#define CMDQ_EVENT_GCAMSV_E_2_SW_PASS1_DONE		141
-+#define CMDQ_EVENT_GCAMSV_F_1_SW_PASS1_DONE		142
-+#define CMDQ_EVENT_GCAMSV_F_2_SW_PASS1_DONE		143
-+#define CMDQ_EVENT_GCAMSV_G_1_SW_PASS1_DONE		144
-+#define CMDQ_EVENT_GCAMSV_G_2_SW_PASS1_DONE		145
-+#define CMDQ_EVENT_GCAMSV_H_1_SW_PASS1_DONE		146
-+#define CMDQ_EVENT_GCAMSV_H_2_SW_PASS1_DONE		147
-+#define CMDQ_EVENT_GCAMSV_I_1_SW_PASS1_DONE		148
-+#define CMDQ_EVENT_GCAMSV_I_2_SW_PASS1_DONE		149
-+#define CMDQ_EVENT_GCAMSV_J_1_SW_PASS1_DONE		150
-+#define CMDQ_EVENT_GCAMSV_J_2_SW_PASS1_DONE		151
-+#define CMDQ_EVENT_MRAW_0_SW_PASS1_DONE			152
-+#define CMDQ_EVENT_MRAW_1_SW_PASS1_DONE			153
-+#define CMDQ_EVENT_MRAW_2_SW_PASS1_DONE			154
-+#define CMDQ_EVENT_MRAW_3_SW_PASS1_DONE			155
-+#define CMDQ_EVENT_SENINF_CAM0_FIFO_FULL		156
-+#define CMDQ_EVENT_SENINF_CAM1_FIFO_FULL		157
-+#define CMDQ_EVENT_SENINF_CAM2_FIFO_FULL		158
-+#define CMDQ_EVENT_SENINF_CAM3_FIFO_FULL		159
-+#define CMDQ_EVENT_SENINF_CAM4_FIFO_FULL		160
-+#define CMDQ_EVENT_SENINF_CAM5_FIFO_FULL		161
-+#define CMDQ_EVENT_SENINF_CAM6_FIFO_FULL		162
-+#define CMDQ_EVENT_SENINF_CAM7_FIFO_FULL		163
-+#define CMDQ_EVENT_SENINF_CAM8_FIFO_FULL		164
-+#define CMDQ_EVENT_SENINF_CAM9_FIFO_FULL		165
-+#define CMDQ_EVENT_SENINF_CAM10_FIFO_FULL		166
-+#define CMDQ_EVENT_SENINF_CAM11_FIFO_FULL		167
-+#define CMDQ_EVENT_SENINF_CAM12_FIFO_FULL		168
-+#define CMDQ_EVENT_SENINF_CAM13_FIFO_FULL		169
-+#define CMDQ_EVENT_SENINF_CAM14_FIFO_FULL		170
-+#define CMDQ_EVENT_SENINF_CAM15_FIFO_FULL		171
-+#define CMDQ_EVENT_SENINF_CAM16_FIFO_FULL		172
-+#define CMDQ_EVENT_SENINF_CAM17_FIFO_FULL		173
-+#define CMDQ_EVENT_SENINF_CAM18_FIFO_FULL		174
-+#define CMDQ_EVENT_SENINF_CAM19_FIFO_FULL		175
-+#define CMDQ_EVENT_SENINF_CAM20_FIFO_FULL		176
-+#define CMDQ_EVENT_SENINF_CAM21_FIFO_FULL		177
-+#define CMDQ_EVENT_SENINF_CAM22_FIFO_FULL		178
-+#define CMDQ_EVENT_SENINF_CAM23_FIFO_FULL		179
-+#define CMDQ_EVENT_SENINF_CAM24_FIFO_FULL		180
-+#define CMDQ_EVENT_SENINF_CAM25_FIFO_FULL		181
-+#define CMDQ_EVENT_SENINF_CAM26_FIFO_FULL		182
-+#define CMDQ_EVENT_TG_OVRUN_MRAW0_INT			183
-+#define CMDQ_EVENT_TG_OVRUN_MRAW1_INT			184
-+#define CMDQ_EVENT_TG_OVRUN_MRAW2_INT			185
-+#define CMDQ_EVENT_TG_OVRUN_MRAW3_INT			186
-+#define CMDQ_EVENT_DMA_R1_ERROR_MRAW0_INT		187
-+#define CMDQ_EVENT_DMA_R1_ERROR_MRAW1_INT		188
-+#define CMDQ_EVENT_DMA_R1_ERROR_MRAW2_INT		189
-+#define CMDQ_EVENT_DMA_R1_ERROR_MRAW3_INT		190
-+#define CMDQ_EVENT_PDA0_IRQO_EVENT_DONE_D1		191
-+#define CMDQ_EVENT_PDA1_IRQO_EVENT_DONE_D1		192
-+#define CMDQ_EVENT_CAM_SUBA_TG_INT1				193
-+#define CMDQ_EVENT_CAM_SUBA_TG_INT2				194
-+#define CMDQ_EVENT_CAM_SUBA_TG_INT3				195
-+#define CMDQ_EVENT_CAM_SUBA_TG_INT4				196
-+#define CMDQ_EVENT_CAM_SUBB_TG_INT1				197
-+#define CMDQ_EVENT_CAM_SUBB_TG_INT2				198
-+#define CMDQ_EVENT_CAM_SUBB_TG_INT3				199
-+#define CMDQ_EVENT_CAM_SUBB_TG_INT4				200
-+#define CMDQ_EVENT_CAM_SUBC_TG_INT1				201
-+#define CMDQ_EVENT_CAM_SUBC_TG_INT2				202
-+#define CMDQ_EVENT_CAM_SUBC_TG_INT3				203
-+#define CMDQ_EVENT_CAM_SUBC_TG_INT4				204
-+#define CMDQ_EVENT_CAM_SUBA_IMGO_R1_LOW_LATENCY_LINE_CNT_INT	205
-+#define CMDQ_EVENT_CAM_SUBA_YUVO_R1_LOW_LATENCY_LINE_CNT_INT	206
-+#define CMDQ_EVENT_CAM_SUBA_YUVO_R3_LOW_LATENCY_LINE_CNT_INT	207
-+#define CMDQ_EVENT_CAM_SUBA_DRZS4NO_R1_LOW_LATENCY_LINE_CNT_INT	208
-+#define CMDQ_EVENT_CAM_SUBB_IMGO_R1_LOW_LATENCY_LINE_CNT_INT	209
-+#define CMDQ_EVENT_CAM_SUBB_YUVO_R1_LOW_LATENCY_LINE_CNT_INT	210
-+#define CMDQ_EVENT_CAM_SUBB_YUVO_R3_LOW_LATENCY_LINE_CNT_INT	211
-+#define CMDQ_EVENT_CAM_SUBB_DRZS4NO_R1_LOW_LATENCY_LINE_CNT_INT	212
-+#define CMDQ_EVENT_CAM_SUBC_IMGO_R1_LOW_LATENCY_LINE_CNT_INT	213
-+#define CMDQ_EVENT_CAM_SUBC_YUVO_R1_LOW_LATENCY_LINE_CNT_INT	214
-+#define CMDQ_EVENT_CAM_SUBC_YUVO_R3_LOW_LATENCY_LINE_CNT_INT	215
-+#define CMDQ_EVENT_CAM_SUBC_DRZS4NO_R1_LOW_LATENCY_LINE_CNT_INT	216
-+#define CMDQ_EVENT_RAW_SEL_SOF_SUBA				217
-+#define CMDQ_EVENT_RAW_SEL_SOF_SUBB				218
-+#define CMDQ_EVENT_RAW_SEL_SOF_SUBC				219
-+#define CMDQ_EVENT_CAM_SUBA_RING_BUFFER_OVERFLOW_INT_IN		220
-+#define CMDQ_EVENT_CAM_SUBB_RING_BUFFER_OVERFLOW_INT_IN		221
-+#define CMDQ_EVENT_CAM_SUBC_RING_BUFFER_OVERFLOW_INT_IN		222
-+#define CMDQ_EVENT_VPP0_MDP_RDMA_SOF				256
-+#define CMDQ_EVENT_VPP0_MDP_FG_SOF				257
-+#define CMDQ_EVENT_VPP0_STITCH_SOF				258
-+#define CMDQ_EVENT_VPP0_MDP_HDR_SOF				259
-+#define CMDQ_EVENT_VPP0_MDP_AAL_SOF				260
-+#define CMDQ_EVENT_VPP0_MDP_RSZ_IN_RSZ_SOF			261
-+#define CMDQ_EVENT_VPP0_MDP_TDSHP_SOF				262
-+#define CMDQ_EVENT_VPP0_DISP_COLOR_SOF				263
-+#define CMDQ_EVENT_VPP0_DISP_OVL_NOAFBC_SOF			264
-+#define CMDQ_EVENT_VPP0_VPP_PADDING_IN_PADDING_SOF		265
-+#define CMDQ_EVENT_VPP0_MDP_TCC_IN_SOF				266
-+#define CMDQ_EVENT_VPP0_MDP_WROT_SOF				267
-+#define CMDQ_EVENT_VPP0_WARP0_MMSYS_TOP_RELAY_SOF_PRE		269
-+#define CMDQ_EVENT_VPP0_WARP1_MMSYS_TOP_RELAY_SOF_PRE		270
-+#define CMDQ_EVENT_VPP0_VPP1_MMSYS_TOP_RELAY_SOF		271
-+#define CMDQ_EVENT_VPP0_VPP1_IN_MMSYS_TOP_RELAY_SOF_PRE		272
-+#define CMDQ_EVENT_VPP0_DISP_RDMA_SOF				273
-+#define CMDQ_EVENT_VPP0_DISP_WDMA_SOF				274
-+#define CMDQ_EVENT_VPP0_MDP_HMS_SOF				275
-+#define CMDQ_EVENT_VPP0_MDP_RDMA_FRAME_DONE			288
-+#define CMDQ_EVENT_VPP0_MDP_FG_TILE_DONE			289
-+#define CMDQ_EVENT_VPP0_STITCH_FRAME_DONE			290
-+#define CMDQ_EVENT_VPP0_MDP_HDR_FRAME_DONE			291
-+#define CMDQ_EVENT_VPP0_MDP_AAL_FRAME_DONE			292
-+#define CMDQ_EVENT_VPP0_MDP_RSZ_FRAME_DONE			293
-+#define CMDQ_EVENT_VPP0_MDP_TDSHP_FRAME_DONE			294
-+#define CMDQ_EVENT_VPP0_DISP_COLOR_FRAME_DONE			295
-+#define CMDQ_EVENT_VPP0_DISP_OVL_NOAFBC_FRAME_DONE		296
-+#define CMDQ_EVENT_VPP0_VPP_PADDING_IN_PADDING_FRAME_DONE	297
-+#define CMDQ_EVENT_VPP0_MDP_TCC_TCC_FRAME_DONE			298
-+#define CMDQ_EVENT_VPP0_MDP_WROT_VIDO_WDONE			299
-+#define CMDQ_EVENT_VPP0_DISP_RDMA_FRAME_DONE			305
-+#define CMDQ_EVENT_VPP0_DISP_WDMA_FRAME_DONE			306
-+#define CMDQ_EVENT_VPP0_MDP_HMS_FRAME_DONE			307
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_0		320
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_1		321
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_2		322
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_3		323
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_4		324
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_5		325
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_6		326
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_7		327
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_8		328
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_9		329
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_10		330
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_11		331
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_12		332
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_13		333
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_14		334
-+#define CMDQ_EVENT_VPP0_DISP_MUTEX_STREAM_DONE_15		335
-+#define CMDQ_EVENT_VPP0_DISP_RDMA_0_UNDERRUN			336
-+#define CMDQ_EVENT_VPP0_DISP_RDMA_1_UNDERRUN			337
-+#define CMDQ_EVENT_VPP0_U_MERGE4_UNDERRUN			338
-+#define CMDQ_EVENT_VPP0_U_VPP_SPLIT_VIDEO_0_OVERFLOW		339
-+#define CMDQ_EVENT_VPP0_U_VPP_SPLIT_VIDEO_1_OVERFLOW		340
-+#define CMDQ_EVENT_VPP0_DSI_0_UNDERRUN				341
-+#define CMDQ_EVENT_VPP0_DSI_1_UNDERRUN				342
-+#define CMDQ_EVENT_VPP0_DP_INTF_0				343
-+#define CMDQ_EVENT_VPP0_DP_INTF_1				344
-+#define CMDQ_EVENT_VPP0_DPI_0					345
-+#define CMDQ_EVENT_VPP0_DPI_1					346
-+#define CMDQ_EVENT_VPP0_MDP_RDMA_SW_RST_DONE			352
-+#define CMDQ_EVENT_VPP0_MDP_RDMA_PM_VALID_EVENT			353
-+#define CMDQ_EVENT_VPP0_DISP_OVL_NOAFBC_FRAME_RESET_DONE_PULSE	354
-+#define CMDQ_EVENT_VPP0_MDP_WROT_SW_RST_DONE			355
-+#define CMDQ_EVENT_VPP0_DISP_OVL_NOAFBC_TARGET_MATCH_0		356
-+#define CMDQ_EVENT_VPP0_DISP_OVL_NOAFBC_TARGET_MATCH_1		357
-+#define CMDQ_EVENT_VPP0_DISP_OVL_NOAFBC_TARGET_MATCH_2		358
-+#define CMDQ_EVENT_VPP0_DISP_OVL_NOAFBC_TARGET_MATCH_3		359
-+#define CMDQ_EVENT_VPP0_DISP_OVL_NOAFBC_TARGET_MATCH_4		360
-+#define CMDQ_EVENT_VPP0_DISP_OVL_NOAFBC_TARGET_MATCH_5		361
-+#define CMDQ_EVENT_VPP0_DISP_OVL_NOAFBC_TARGET_MATCH_6		362
-+#define CMDQ_EVENT_VPP0_DISP_RDMA_DISP_RDMA_VALID_EVENT		363
-+#define CMDQ_EVENT_VPP0_DISP_RDMA_DISP_RDMA_TARGET_LINE_EVENT	364
-+#define CMDQ_EVENT_VPP0_DISP_WDMA_SW_RST_DONE			365
-+#define CMDQ_EVENT_VPP0_DISP_WDMA_WDMA_VALID_EVENT		366
-+#define CMDQ_EVENT_VPP0_DISP_WDMA_WDMA_TARGET_LINE_EVENT	367
-+#define CMDQ_EVENT_VPP1_HDMI_META_SOF				384
-+#define CMDQ_EVENT_VPP1_DGI_SOF					385
-+#define CMDQ_EVENT_VPP1_VPP_SPLIT_SOF				386
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_TCC_SOF			387
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_RDMA_SOF			388
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_RDMA_SOF			389
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_RDMA_SOF			390
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_FG_SOF			391
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_FG_SOF			392
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_FG_SOF			393
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_HDR_SOF			394
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_HDR_SOF			395
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_HDR_SOF			396
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_AAL_SOF			397
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_AAL_SOF			398
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_AAL_SOF			399
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_RSZ_SOF			400
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_RSZ_SOF			401
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_RSZ_SOF			402
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_TDSHP_SOF			403
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_TDSHP_SOF			404
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_TDSHP_SOF			405
-+#define CMDQ_EVENT_VPP1_SVPP2_VPP_MERGE_SOF			406
-+#define CMDQ_EVENT_VPP1_SVPP3_VPP_MERGE_SOF			407
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_COLOR_SOF			408
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_COLOR_SOF			409
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_COLOR_SOF			410
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_OVL_SOF			411
-+#define CMDQ_EVENT_VPP1_SVPP1_VPP_PAD_SOF			412
-+#define CMDQ_EVENT_VPP1_SVPP2_VPP_PAD_SOF			413
-+#define CMDQ_EVENT_VPP1_SVPP3_VPP_PAD_SOF			414
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_WROT_SOF			415
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_WROT_SOF			416
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_WROT_SOF			417
-+#define CMDQ_EVENT_VPP1_VPP0_DL_IRLY_SOF			418
-+#define CMDQ_EVENT_VPP1_VPP0_DL_ORLY_SOF			419
-+#define CMDQ_EVENT_VPP1_VDO0_DL_ORLY_0_SOF			420
-+#define CMDQ_EVENT_VPP1_VDO0_DL_ORLY_1_SOF			421
-+#define CMDQ_EVENT_VPP1_VDO1_DL_ORLY_0_SOF			422
-+#define CMDQ_EVENT_VPP1_VDO1_DL_ORLY_1_SOF			423
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_RDMA_FRAME_DONE		424
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_RDMA_FRAME_DONE		425
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_RDMA_FRAME_DONE		426
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_WROT_FRAME_DONE		427
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_WROT_FRAME_DONE		428
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_WROT_FRAME_DONE		429
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_OVL_FRAME_DONE		430
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_RSZ_FRAME_DONE		431
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_RSZ_FRAME_DONE		432
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_RSZ_FRAME_DONE		433
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_FG_TILE_DONE			434
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_FG_TILE_DONE			435
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_FG_TILE_DONE			436
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_HDR_FRAME_DONE		437
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_HDR_FRAME_DONE		438
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_HDR_FRAME_DONE		439
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_AAL_FRAME_DONE		440
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_AAL_FRAME_DONE		441
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_AAL_FRAME_DONE		442
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_TDSHP_FRAME_DONE		443
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_TDSHP_FRAME_DONE		444
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_TDSHP_FRAME_DONE		445
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_COLOR_FRAME_DONE		446
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_COLOR_FRAME_DONE		447
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_COLOR_FRAME_DONE		448
-+#define CMDQ_EVENT_VPP1_SVPP1_VPP_PAD_FRAME_DONE		449
-+#define CMDQ_EVENT_VPP1_SVPP2_VPP_PAD_FRAME_DONE		450
-+#define CMDQ_EVENT_VPP1_SVPP3_VPP_PAD_FRAME_DONE		451
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_TCC_FRAME_DONE		452
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_0		456
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_1		457
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_2		458
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_3		459
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_4		460
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_5		461
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_6		462
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_7		463
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_8		464
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_9		465
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_10		466
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_11		467
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_12		468
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_13		469
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_14		470
-+#define CMDQ_EVENT_VPP1_MUTEX_STREAM_DONE_GCE_EVENT_15		471
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_0		472
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_1		473
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_2		474
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_3		475
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_4		476
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_5		477
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_6		478
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_7		479
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_8		480
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_9		481
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_10		482
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_11		483
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_12		484
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_13		485
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_14		486
-+#define CMDQ_EVENT_VPP1_MUTEX_BUF_UNDERRUN_GCE_EVENT_15		487
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_0			488
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_1			489
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_2			490
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_3			491
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_4			492
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_5			493
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_6			494
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_7			495
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_8			496
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_9			497
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_10			498
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_11			499
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_12			500
-+#define CMDQ_EVENT_VPP1_DGI_SYNC_EVENT_13			501
-+#define CMDQ_EVENT_VPP1_SVPP3_VPP_MERGE_GCE_EVENT		502
-+#define CMDQ_EVENT_VPP1_SVPP2_VPP_MERGE_GCE_EVENT		503
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_OVL_GCE_EVENT			504
-+#define CMDQ_EVENT_VPP1_VPP_SPLIT_DGI_GCE_EVENT			505
-+#define CMDQ_EVENT_VPP1_VPP_SPLIT_HDMI_GCE_EVENT		506
-+#define CMDQ_EVENT_VPP1_SVPP3_MDP_WROT_SW_RST_DONE_GCE_EVENT	507
-+#define CMDQ_EVENT_VPP1_SVPP2_MDP_WROT_SW_RST_DONE_GCE_EVENT	508
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_WROT_SW_RST_DONE_GCE_EVENT	509
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_OVL_NEW_EVENT_0		510
-+#define CMDQ_EVENT_VPP1_SVPP1_MDP_OVL_NEW_EVENT_1		511
-+#define CMDQ_EVENT_VDO0_DISP_OVL0_SOF				512
-+#define CMDQ_EVENT_VDO0_DISP_WDMA0_SOF				513
-+#define CMDQ_EVENT_VDO0_DISP_RDMA0_SOF				514
-+#define CMDQ_EVENT_VDO0_DISP_COLOR0_SOF				515
-+#define CMDQ_EVENT_VDO0_DISP_CCORR0_SOF				516
-+#define CMDQ_EVENT_VDO0_DISP_AAL0_SOF				517
-+#define CMDQ_EVENT_VDO0_DISP_GAMMA0_SOF				518
-+#define CMDQ_EVENT_VDO0_DISP_DITHER0_SOF			519
-+#define CMDQ_EVENT_VDO0_DSI0_SOF				520
-+#define CMDQ_EVENT_VDO0_DSC_WRAP0C0_SOF			521
-+#define CMDQ_EVENT_VDO0_DISP_OVL1_SOF			522
-+#define CMDQ_EVENT_VDO0_DISP_WDMA1_SOF			523
-+#define CMDQ_EVENT_VDO0_DISP_RDMA1_SOF			524
-+#define CMDQ_EVENT_VDO0_DISP_COLOR1_SOF			525
-+#define CMDQ_EVENT_VDO0_DISP_CCORR1_SOF			526
-+#define CMDQ_EVENT_VDO0_DISP_AAL1_SOF			527
-+#define CMDQ_EVENT_VDO0_DISP_GAMMA1_SOF			528
-+#define CMDQ_EVENT_VDO0_DISP_DITHER1_SOF		529
-+#define CMDQ_EVENT_VDO0_DSI1_SOF			530
-+#define CMDQ_EVENT_VDO0_DSC_WRAP0C1_SOF			531
-+#define CMDQ_EVENT_VDO0_VPP_MERGE0_SOF			532
-+#define CMDQ_EVENT_VDO0_DP_INTF0_SOF			533
-+#define CMDQ_EVENT_VDO0_DISP_DPI0_SOF			534
-+#define CMDQ_EVENT_VDO0_DISP_DPI1_SOF			535
-+#define CMDQ_EVENT_VDO0_DISP_POSTMASK0_SOF		536
-+#define CMDQ_EVENT_VDO0_MDP_WROT0_SOF			537
-+#define CMDQ_EVENT_VDO0_DISP_RSZ0_SOF			538
-+#define CMDQ_EVENT_VDO0_VPP1_DL_RELAY0_SOF		539
-+#define CMDQ_EVENT_VDO0_VPP1_DL_RELAY1_SOF		540
-+#define CMDQ_EVENT_VDO0_VDO1_DL_RELAY2_SOF		541
-+#define CMDQ_EVENT_VDO0_VDO0_DL_RELAY3_SOF		542
-+#define CMDQ_EVENT_VDO0_VDO0_DL_RELAY4_SOF		543
-+#define CMDQ_EVENT_VDO0_DISP_PWM0_SOF			544
-+#define CMDQ_EVENT_VDO0_DISP_PWM1_SOF			545
-+#define CMDQ_EVENT_VDO0_DISP_OVL0_O_FRAME_DONE		546
-+#define CMDQ_EVENT_VDO0_DISP_WDMA0_FRAME_DONE		547
-+#define CMDQ_EVENT_VDO0_DISP_RDMA0_FRAME_DONE		548
-+#define CMDQ_EVENT_VDO0_DISP_COLOR0_O_FRAME_DONE	549
-+#define CMDQ_EVENT_VDO0_DISP_CCORR0_O_FRAME_DONE	550
-+#define CMDQ_EVENT_VDO0_DISP_AAL0_O_FRAME_DONE		551
-+#define CMDQ_EVENT_VDO0_DISP_GAMMA0_O_FRAME_DONE	552
-+#define CMDQ_EVENT_VDO0_DISP_DITHER0_O_FRAME_DONE	553
-+#define CMDQ_EVENT_VDO0_DSI0_FRAME_DONE			554
-+#define CMDQ_EVENT_VDO0_DSC_WRAP0_O_FRAME_DONE_0	555
-+#define CMDQ_EVENT_VDO0_DISP_OVL1_O_FRAME_DONE		556
-+#define CMDQ_EVENT_VDO0_DISP_WDMA1_O_FRAME_DONE		557
-+#define CMDQ_EVENT_VDO0_DISP_RDMA1_O_FRAME_DONE		558
-+#define CMDQ_EVENT_VDO0_DISP_COLOR1_O_FRAME_DONE	559
-+#define CMDQ_EVENT_VDO0_DISP_CCORR1_O_FRAME_DONE	560
-+#define CMDQ_EVENT_VDO0_DISP_AAL1_O_FRAME_DONE		561
-+#define CMDQ_EVENT_VDO0_DISP_GAMMA1_O_FRAME_DONE	562
-+#define CMDQ_EVENT_VDO0_DISP_DITHER1_O_FRAME_DONE	563
-+#define CMDQ_EVENT_VDO0_DSI1_FRAME_DONE			564
-+#define CMDQ_EVENT_VDO0_DSC_WRAP0_O_FRAME_DONE_1	565
-+#define CMDQ_EVENT_VDO0_DP_INTF0_FRAME_DONE		567
-+#define CMDQ_EVENT_VDO0_DISP_DPI0_O_FRAME_DONE		568
-+#define CMDQ_EVENT_VDO0_DISP_DPI1_O_FRAME_DONE		569
-+#define CMDQ_EVENT_VDO0_DISP_POSTMASK0_O_FRAME_DONE	570
-+#define CMDQ_EVENT_VDO0_MDP_WROT0_O_FRAME_DONE		571
-+#define CMDQ_EVENT_VDO0_DISP_RSZ0_O_FRAME_DONE		572
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_0		574
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_1		575
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_2		576
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_3		577
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_4		578
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_5		579
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_6		580
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_7		581
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_8		582
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_9		583
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_10		584
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_11		585
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_12		586
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_13		587
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_14		588
-+#define CMDQ_EVENT_VDO0_DISP_STREAM_DONE_15		589
-+#define CMDQ_EVENT_VDO0_DISP_RDMA_0_UNDERRUN		590
-+#define CMDQ_EVENT_VDO0_DISP_RDMA_1_UNDERRUN		591
-+#define CMDQ_EVENT_VDO0_U_MERGE4_UNDERRUN		592
-+#define CMDQ_EVENT_VDO0_DSI_0_UNDERRUN			595
-+#define CMDQ_EVENT_VDO0_DSI_1_UNDERRUN			596
-+#define CMDQ_EVENT_VDO0_DP_INTF_0			597
-+#define CMDQ_EVENT_VDO0_DP_INTF_1			598
-+#define CMDQ_EVENT_VDO0_DPI_0				599
-+#define CMDQ_EVENT_VDO0_DPI_1				600
-+#define CMDQ_EVENT_VDO0_DISP_SMIASSERT_ENG_EVENT	606
-+#define CMDQ_EVENT_VDO0_DSI0_O_DSI_IRQ_EVENT_MM		607
-+#define CMDQ_EVENT_VDO0_DSI0_TE_ENG_EVENT_MM		608
-+#define CMDQ_EVENT_VDO0_DSI0_O_DSI_DONE_EVENT_MM	609
-+#define CMDQ_EVENT_VDO0_DSI0_O_DSI_VACTL_EVENT_MM	610
-+#define CMDQ_EVENT_VDO0_DSI1_O_DSI_IRQ_EVENT_MM		611
-+#define CMDQ_EVENT_VDO0_DSI1_TE_ENG_EVENT_MM		612
-+#define CMDQ_EVENT_VDO0_DSI1_O_DSI_DONE_EVENT_MM	613
-+#define CMDQ_EVENT_VDO0_DSI1_O_DSI_VACTL_EVENT_MM	614
-+#define CMDQ_EVENT_VDO0_DP_INTF0_O_DP_VSYNC_START_EVENT_MM		615
-+#define CMDQ_EVENT_VDO0_DP_INTF0_O_DP_VSYNC_END_EVENT_MM		616
-+#define CMDQ_EVENT_VDO0_DP_INTF0_O_DP_VDE_START_EVENT_MM		617
-+#define CMDQ_EVENT_VDO0_DP_INTF0_O_DP_VDE_END_EVENT_MM			618
-+#define CMDQ_EVENT_VDO0_DP_INTF0_O_VACT_TARGET_LINE_EVENT_MM		619
-+#define CMDQ_EVENT_VDO0_DP_INTF0_O_LAST_SAFE_BLANK_EVENT_MM		620
-+#define CMDQ_EVENT_VDO0_DP_INTF0_O_LAST_LINE_EVENT_MM			621
-+#define CMDQ_EVENT_VDO0_DP_INTF0_O_TRIGGER_LOOP_CLEAR_EVENT_MM		622
-+#define CMDQ_EVENT_VDO0_DP_INTF0_O_TARGET_LINE_0_EVENT_MM		623
-+#define CMDQ_EVENT_VDO0_DP_INTF0_O_TARGET_LINE_1_EVENT_MM		624
-+#define CMDQ_EVENT_VDO0_DISP_POSTMASK0_O_FRAME_RESET_DONE_PULSE		625
-+#define CMDQ_EVENT_VDO0_VPP_MERGE0_O_VPP_MERGE_EVENT			626
-+#define CMDQ_EVENT_VDO0_DISP_OVL0_O_FRAME_RESET_DONE_PULSE		627
-+#define CMDQ_EVENT_VDO0_DISP_RDMA0_O_DISP_RDMA_TARGET_LINE_EVENT	628
-+#define CMDQ_EVENT_VDO0_DISP_WDMA0_O_WDMA_TARGET_LINE_EVENT		629
-+#define CMDQ_EVENT_VDO0_DISP_WDMA0_O_SW_RST_DONE			630
-+#define CMDQ_EVENT_VDO0_DISP_OVL0_O_TARGET_MATCH_EVENT_0		631
-+#define CMDQ_EVENT_VDO0_DISP_OVL0_O_TARGET_MATCH_EVENT_1		632
-+#define CMDQ_EVENT_VDO0_DISP_OVL0_O_TARGET_MATCH_EVENT_2		633
-+#define CMDQ_EVENT_VDO0_DISP_OVL0_O_TARGET_MATCH_EVENT_3		634
-+#define CMDQ_EVENT_VDO0_DISP_OVL0_O_TARGET_MATCH_EVENT_4		635
-+#define CMDQ_EVENT_VDO0_DISP_OVL0_O_TARGET_MATCH_EVENT_5		636
-+#define CMDQ_EVENT_VDO0_DISP_OVL0_O_TARGET_MATCH_EVENT_6		637
-+#define CMDQ_EVENT_VDO0_MDP_WROT0_O_SW_RST_DONE				638
-+#define CMDQ_EVENT_VDO0_RESERVED					639
-+#define CMDQ_EVENT_VDO1_MDP_RDMA0_SOF				640
-+#define CMDQ_EVENT_VDO1_MDP_RDMA1_SOF				641
-+#define CMDQ_EVENT_VDO1_MDP_RDMA2_SOF				642
-+#define CMDQ_EVENT_VDO1_MDP_RDMA3_SOF				643
-+#define CMDQ_EVENT_VDO1_MDP_RDMA4_SOF				644
-+#define CMDQ_EVENT_VDO1_MDP_RDMA5_SOF				645
-+#define CMDQ_EVENT_VDO1_MDP_RDMA6_SOF				646
-+#define CMDQ_EVENT_VDO1_MDP_RDMA7_SOF				647
-+#define CMDQ_EVENT_VDO1_DISP_PADDING0_SOF			648
-+#define CMDQ_EVENT_VDO1_DISP_PADDING1_SOF			649
-+#define CMDQ_EVENT_VDO1_DISP_PADDING2_SOF			650
-+#define CMDQ_EVENT_VDO1_DISP_PADDING3_SOF			651
-+#define CMDQ_EVENT_VDO1_DISP_PADDING4_SOF			652
-+#define CMDQ_EVENT_VDO1_DISP_PADDING5_SOF			653
-+#define CMDQ_EVENT_VDO1_DISP_PADDING6_SOF			654
-+#define CMDQ_EVENT_VDO1_DISP_PADDING7_SOF			655
-+#define CMDQ_EVENT_VDO1_DISP_RSZ0_SOF				656
-+#define CMDQ_EVENT_VDO1_DISP_RSZ1_SOF				657
-+#define CMDQ_EVENT_VDO1_DISP_RSZ2_SOF				658
-+#define CMDQ_EVENT_VDO1_DISP_RSZ3_SOF				659
-+#define CMDQ_EVENT_VDO1_VPP_MERGE0_SOF				660
-+#define CMDQ_EVENT_VDO1_VPP_MERGE1_SOF				661
-+#define CMDQ_EVENT_VDO1_VPP_MERGE2_SOF				662
-+#define CMDQ_EVENT_VDO1_VPP_MERGE3_SOF				663
-+#define CMDQ_EVENT_VDO1_VPP_MERGE4_SOF				664
-+#define CMDQ_EVENT_VDO1_VPP2_DL_RELAY_SOF			665
-+#define CMDQ_EVENT_VDO1_VPP3_DL_RELAY_SOF			666
-+#define CMDQ_EVENT_VDO0_DSC_DL_ASYNC_SOF			667
-+#define CMDQ_EVENT_VDO0_MERGE_DL_ASYNC_SOF			668
-+#define CMDQ_EVENT_VDO1_OUT_DL_RELAY_SOF			669
-+#define CMDQ_EVENT_VDO1_DISP_MIXER_SOF				670
-+#define CMDQ_EVENT_VDO1_HDR_VDO_FE0_SOF				671
-+#define CMDQ_EVENT_VDO1_HDR_VDO_FE1_SOF				672
-+#define CMDQ_EVENT_VDO1_HDR_GFX_FE0_SOF				673
-+#define CMDQ_EVENT_VDO1_HDR_GFX_FE1_SOF				674
-+#define CMDQ_EVENT_VDO1_HDR_VDO_BE0_SOF				675
-+#define CMDQ_EVENT_VDO1_HDR_MLOAD_SOF				676
-+#define CMDQ_EVENT_VDO1_DPI0_EXT_SOF				677
-+#define CMDQ_EVENT_VDO1_DPI1_EXT_SOF				678
-+#define CMDQ_EVENT_VDO1_DP_INTF_EXT_EXT_SOF			679
-+#define CMDQ_EVENT_VDO1_MDP_RDMA0_FRAME_DONE		680
-+#define CMDQ_EVENT_VDO1_MDP_RDMA1_FRAME_DONE		681
-+#define CMDQ_EVENT_VDO1_MDP_RDMA2_FRAME_DONE		682
-+#define CMDQ_EVENT_VDO1_MDP_RDMA3_FRAME_DONE		683
-+#define CMDQ_EVENT_VDO1_MDP_RDMA4_FRAME_DONE		684
-+#define CMDQ_EVENT_VDO1_MDP_RDMA5_FRAME_DONE		685
-+#define CMDQ_EVENT_VDO1_MDP_RDMA6_FRAME_DONE		686
-+#define CMDQ_EVENT_VDO1_MDP_RDMA7_FRAME_DONE		687
-+#define CMDQ_EVENT_VDO1_DISP_PADDING0_FRAME_DONE	688
-+#define CMDQ_EVENT_VDO1_DISP_PADDING1_FRAME_DONE	689
-+#define CMDQ_EVENT_VDO1_DISP_PADDING2_FRAME_DONE	690
-+#define CMDQ_EVENT_VDO1_DISP_PADDING3_FRAME_DONE	691
-+#define CMDQ_EVENT_VDO1_DISP_PADDING4_FRAME_DONE	692
-+#define CMDQ_EVENT_VDO1_DISP_PADDING5_FRAME_DONE	693
-+#define CMDQ_EVENT_VDO1_DISP_PADDING6_FRAME_DONE	694
-+#define CMDQ_EVENT_VDO1_DISP_PADDING7_FRAME_DONE	695
-+#define CMDQ_EVENT_VDO1_DISP_RSZ0_FRAME_DONE		696
-+#define CMDQ_EVENT_VDO1_DISP_RSZ1_FRAME_DONE		697
-+#define CMDQ_EVENT_VDO1_DISP_RSZ2_FRAME_DONE		698
-+#define CMDQ_EVENT_VDO1_DISP_RSZ3_FRAME_DONE		699
-+#define CMDQ_EVENT_VDO1_VPP_MERGE0_FRAME_DONE		700
-+#define CMDQ_EVENT_VDO1_VPP_MERGE1_FRAME_DONE		701
-+#define CMDQ_EVENT_VDO1_VPP_MERGE2_FRAME_DONE		702
-+#define CMDQ_EVENT_VDO1_VPP_MERGE3_FRAME_DONE		703
-+#define CMDQ_EVENT_VDO1_VPP_MERGE4_FRAME_DONE		704
-+#define CMDQ_EVENT_VDO1_DPI0_FRAME_DONE			705
-+#define CMDQ_EVENT_VDO1_DPI1_FRAME_DONE			706
-+#define CMDQ_EVENT_VDO1_DP_INTF0_FRAME_DONE		707
-+#define CMDQ_EVENT_VDO1_DISP_MIXER_FRAME_DONE_MM	708
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_0		709
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_1		710
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_2		711
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_3		712
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_4		713
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_5		714
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_6		715
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_7		716
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_8		717
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_9		718
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_10		719
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_11		720
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_12		721
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_13		722
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_14		723
-+#define CMDQ_EVENT_VDO1_STREAM_DONE_ENG_15		724
-+#define CMDQ_EVENT_VDO1_DISP_RDMA_0_UNDERRUN		725
-+#define CMDQ_EVENT_VDO1_DISP_RDMA_1_UNDERRUN		726
-+#define CMDQ_EVENT_VDO1_U_MERGE4_UNDERRUN		727
-+#define CMDQ_EVENT_VDO1_U_VPP_SPLIT_VIDEO_0_OVERFLOW	728
-+#define CMDQ_EVENT_VDO1_U_VPP_SPLIT_VIDEO_1_OVERFLOW	729
-+#define CMDQ_EVENT_VDO1_DSI_0_UNDERRUN			730
-+#define CMDQ_EVENT_VDO1_DSI_1_UNDERRUN			731
-+#define CMDQ_EVENT_VDO1_DP_INTF_0			732
-+#define CMDQ_EVENT_VDO1_DP_INTF_1			733
-+#define CMDQ_EVENT_VDO1_DPI_0				734
-+#define CMDQ_EVENT_VDO1_DPI_1				735
-+#define CMDQ_EVENT_VDO1_MDP_RDMA0_SW_RST_DONE		741
-+#define CMDQ_EVENT_VDO1_MDP_RDMA1_SW_RST_DONE		742
-+#define CMDQ_EVENT_VDO1_MDP_RDMA2_SW_RST_DONE		743
-+#define CMDQ_EVENT_VDO1_MDP_RDMA3_SW_RST_DONE		744
-+#define CMDQ_EVENT_VDO1_MDP_RDMA4_SW_RST_DONE		745
-+#define CMDQ_EVENT_VDO1_MDP_RDMA5_SW_RST_DONE		746
-+#define CMDQ_EVENT_VDO1_MDP_RDMA6_SW_RST_DONE		747
-+#define CMDQ_EVENT_VDO1_MDP_RDMA7_SW_RST_DONE		748
-+#define CMDQ_EVENT_VDO1_DP0_VDE_END_ENG_EVENT_MM	749
-+#define CMDQ_EVENT_VDO1_DP0_VDE_START_ENG_EVENT_MM	750
-+#define CMDQ_EVENT_VDO1_DP0_VSYNC_END_ENG_EVENT_MM	751
-+#define CMDQ_EVENT_VDO1_DP0_VSYNC_START_ENG_EVENT_MM	752
-+#define CMDQ_EVENT_VDO1_DP0_TARGET_LINE_ENG_EVENT_MM	753
-+#define CMDQ_EVENT_VDO1_VPP_MERGE0_EVENT		754
-+#define CMDQ_EVENT_VDO1_VPP_MERGE1_EVENT		755
-+#define CMDQ_EVENT_VDO1_VPP_MERGE2_EVENT		756
-+#define CMDQ_EVENT_VDO1_VPP_MERGE3_EVENT		757
-+#define CMDQ_EVENT_VDO1_VPP_MERGE4_EVENT		758
-+#define CMDQ_EVENT_VDO1_HDMITX_EVENT			759
-+#define CMDQ_EVENT_VDO1_HDR_VDO_BE0_ADL_TRIG_EVENT_MM		760
-+#define CMDQ_EVENT_VDO1_HDR_GFX_FE1_THDR_ADL_TRIG_EVENT_MM	761
-+#define CMDQ_EVENT_VDO1_HDR_GFX_FE1_DM_ADL_TRIG_EVENT_MM	762
-+#define CMDQ_EVENT_VDO1_HDR_GFX_FE0_THDR_ADL_TRIG_EVENT_MM	763
-+#define CMDQ_EVENT_VDO1_HDR_GFX_FE0_DM_ADL_TRIG_EVENT_MM	764
-+#define CMDQ_EVENT_VDO1_HDR_VDO_FE1_ADL_TRIG_EVENT_MM		765
-+#define CMDQ_EVENT_VDO1_HDR_VDO_FE1_AD0_TRIG_EVENT_MM		766
-+#define CMDQ_EVENT_VDO1_DPI0_TARGET_LINE_1_EVENT_MM		767
-+#define CMDQ_EVENT_HANDSHAKE_0		768
-+#define CMDQ_EVENT_HANDSHAKE_1		769
-+#define CMDQ_EVENT_HANDSHAKE_2		770
-+#define CMDQ_EVENT_HANDSHAKE_3		771
-+#define CMDQ_EVENT_HANDSHAKE_4		772
-+#define CMDQ_EVENT_HANDSHAKE_5		773
-+#define CMDQ_EVENT_HANDSHAKE_6		774
-+#define CMDQ_EVENT_HANDSHAKE_7		775
-+#define CMDQ_EVENT_HANDSHAKE_8		776
-+#define CMDQ_EVENT_HANDSHAKE_9		777
-+#define CMDQ_EVENT_HANDSHAKE_10		778
-+#define CMDQ_EVENT_HANDSHAKE_11		779
-+#define CMDQ_EVENT_HANDSHAKE_12		780
-+#define CMDQ_EVENT_HANDSHAKE_13		781
-+#define CMDQ_EVENT_HANDSHAKE_14		782
-+#define CMDQ_EVENT_HANDSHAKE_15		783
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_0	800
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_1	801
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_2	802
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_3	803
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_4	804
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_5	805
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_6	806
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_7	807
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_8	808
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_9	809
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_10	810
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_11	811
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_12	812
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_13	813
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_14	814
-+#define CMDQ_EVENT_VDEC_SOC_EVENT_15	815
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_0	832
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_1	833
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_2	834
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_3	835
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_4	836
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_5	837
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_6	838
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_7	839
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_8	840
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_9	841
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_10	842
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_11	843
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_12	844
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_13	845
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_14	846
-+#define CMDQ_EVENT_VDEC_CORE0_EVENT_15	847
-+#define CMDQ_EVENT_VENC_TOP_VENC_FRAME_DONE		865
-+#define CMDQ_EVENT_VENC_TOP_VENC_PAUSE_DONE		866
-+#define CMDQ_EVENT_VENC_TOP_JPGENC_DONE			867
-+#define CMDQ_EVENT_VENC_TOP_VENC_MB_DONE		868
-+#define CMDQ_EVENT_VENC_TOP_VENC_128BYTE_DONE		869
-+#define CMDQ_EVENT_VENC_TOP_JPGDEC_DONE			870
-+#define CMDQ_EVENT_VENC_TOP_VENC_SLICE_DONE		871
-+#define CMDQ_EVENT_VENC_TOP_JPGDEC_INSUFF_DONE		872
-+#define CMDQ_EVENT_VENC_TOP_WP_2ND_STAGE_DONE		874
-+#define CMDQ_EVENT_VENC_TOP_WP_3RD_STAGE_DONE		875
-+#define CMDQ_EVENT_VENC_TOP_PPS_HEADER_DONE		876
-+#define CMDQ_EVENT_VENC_TOP_SPS_HEADER_DONE		877
-+#define CMDQ_EVENT_VENC_TOP_VPS_HEADER_DONE		878
-+#define CMDQ_EVENT_WPE_VPP0_WPE_GCE_FRAME_DONE		882
-+#define CMDQ_EVENT_WPE_VPP0_WPE_DONE_SYNC_OUT		883
-+#define CMDQ_EVENT_SVPP1_MDP_OVL_NEW_EVENT_2		896
-+#define CMDQ_EVENT_SVPP1_MDP_OVL_NEW_EVENT_3		897
-+#define CMDQ_EVENT_SVPP1_MDP_OVL_NEW_EVENT_4		898
-+#define CMDQ_EVENT_SVPP1_MDP_OVL_NEW_EVENT_5		899
-+#define CMDQ_EVENT_SVPP1_MDP_OVL_NEW_EVENT_6		900
-+#define CMDQ_EVENT_VDO1_DPI0_TARGET_LINE_0_EVENT_MM		928
-+#define CMDQ_EVENT_VDO1_DPI0_TRIGGER_LOOP_CLEAR_EVENT_MM	929
-+#define CMDQ_EVENT_VDO1_DPI0_LAST_LINE_EVENT_MM			930
-+#define CMDQ_EVENT_VDO1_DPI0_LAST_SAFE_BLANK_EVENT_MM		931
-+#define CMDQ_EVENT_VDO1_DPI0_VSYNC_START_EVENT_MM		932
-+#define CMDQ_EVENT_VDO1_DPI1_TARGET_LINE_1_EVENT_MM		933
-+#define CMDQ_EVENT_VDO1_DPI1_TARGET_LINE_0_EVENT_MM		934
-+#define CMDQ_EVENT_VDO1_DPI1_TRIGGER_LOOP_CLEAR_EVENT_MM	935
-+#define CMDQ_EVENT_VDO1_DPI1_LAST_LINE_EVENT_MM			936
-+#define CMDQ_EVENT_VDO1_DPI1_LAST_SAFE_BLANK_EVENT_MM		937
-+#define CMDQ_EVENT_VDO1_DPI1_VSYNC_START_EVENT_MM		938
-+#define CMDQ_EVENT_VDO1_DP_INTF_TARGET_LINE_1_EVENT_MM		939
-+#define CMDQ_EVENT_VDO1_DP_INTF_TARGET_LINE_0_EVENT_MM		940
-+#define CMDQ_EVENT_VDO1_DP_INTF_TRIGGER_LOOP_CLEAR_EVENT_MM	941
-+#define CMDQ_EVENT_VDO1_DP_INTF_LAST_LINE_EVENT_MM		942
-+#define CMDQ_EVENT_VDO1_DP_INTF_LAST_SAFE_BLANK_EVENT_MM	943
-+#define CMDQ_EVENT_VBLANK_FALLING		946
-+#define CMDQ_EVENT_VSC_FINISH			947
-+#define CMDQ_EVENT_TPR_0			962
-+#define CMDQ_EVENT_TPR_1			963
-+#define CMDQ_EVENT_TPR_2			964
-+#define CMDQ_EVENT_TPR_3			965
-+#define CMDQ_EVENT_TPR_4			966
-+#define CMDQ_EVENT_TPR_5			967
-+#define CMDQ_EVENT_TPR_6			968
-+#define CMDQ_EVENT_TPR_7			969
-+#define CMDQ_EVENT_TPR_8			970
-+#define CMDQ_EVENT_TPR_9			971
-+#define CMDQ_EVENT_TPR_10			972
-+#define CMDQ_EVENT_TPR_11			973
-+#define CMDQ_EVENT_TPR_12			974
-+#define CMDQ_EVENT_TPR_13			975
-+#define CMDQ_EVENT_TPR_14			976
-+#define CMDQ_EVENT_TPR_15			977
-+#define CMDQ_EVENT_TPR_16			978
-+#define CMDQ_EVENT_TPR_17			979
-+#define CMDQ_EVENT_TPR_18			980
-+#define CMDQ_EVENT_TPR_19			981
-+#define CMDQ_EVENT_TPR_20			982
-+#define CMDQ_EVENT_TPR_21			983
-+#define CMDQ_EVENT_TPR_22			984
-+#define CMDQ_EVENT_TPR_23			985
-+#define CMDQ_EVENT_TPR_24			986
-+#define CMDQ_EVENT_TPR_25			987
-+#define CMDQ_EVENT_TPR_26			988
-+#define CMDQ_EVENT_TPR_27			989
-+#define CMDQ_EVENT_TPR_28			990
-+#define CMDQ_EVENT_TPR_29			991
-+#define CMDQ_EVENT_TPR_30			992
-+#define CMDQ_EVENT_TPR_31			993
-+#define CMDQ_EVENT_TPR_TIMEOUT_0	994
-+#define CMDQ_EVENT_TPR_TIMEOUT_1	995
-+#define CMDQ_EVENT_TPR_TIMEOUT_2	996
-+#define CMDQ_EVENT_TPR_TIMEOUT_3	997
-+#define CMDQ_EVENT_TPR_TIMEOUT_4	998
-+#define CMDQ_EVENT_TPR_TIMEOUT_5	999
-+#define CMDQ_EVENT_TPR_TIMEOUT_6	1000
-+#define CMDQ_EVENT_TPR_TIMEOUT_7	1001
-+#define CMDQ_EVENT_TPR_TIMEOUT_8	1002
-+#define CMDQ_EVENT_TPR_TIMEOUT_9	1003
-+#define CMDQ_EVENT_TPR_TIMEOUT_10	1004
-+#define CMDQ_EVENT_TPR_TIMEOUT_11	1005
-+#define CMDQ_EVENT_TPR_TIMEOUT_12	1006
-+#define CMDQ_EVENT_TPR_TIMEOUT_13	1007
-+#define CMDQ_EVENT_TPR_TIMEOUT_14	1008
-+#define CMDQ_EVENT_TPR_TIMEOUT_15	1009
-+#define CMDQ_EVENT_OUTPIN_0		1018
-+#define CMDQ_EVENT_OUTPIN_1		1019
-+
-+#define CMDQ_SYNC_TOKEN_IMGSYS_WPE_EIS			124
-+#define CMDQ_SYNC_TOKEN_IMGSYS_WPE_TNR			125
-+#define CMDQ_SYNC_TOKEN_IMGSYS_WPE_LITE			126
-+#define CMDQ_SYNC_TOKEN_IMGSYS_TRAW			127
-+#define CMDQ_SYNC_TOKEN_IMGSYS_LTRAW			128
-+#define CMDQ_SYNC_TOKEN_CAMSYS_POOL_1			223
-+#define CMDQ_SYNC_TOKEN_CAMSYS_POOL_2			224
-+#define CMDQ_SYNC_TOKEN_CAMSYS_POOL_3			225
-+#define CMDQ_SYNC_TOKEN_CAMSYS_POOL_4			226
-+#define CMDQ_SYNC_TOKEN_CAMSYS_POOL_5			227
-+#define CMDQ_SYNC_TOKEN_CAMSYS_POOL_6			228
-+#define CMDQ_SYNC_TOKEN_CAMSYS_POOL_7			229
-+#define CMDQ_SYNC_TOKEN_CAMSYS_POOL_8			230
-+#define CMDQ_SYNC_TOKEN_CAMSYS_POOL_9			231
-+#define CMDQ_SYNC_TOKEN_CAMSYS_POOL_10			232
-+#define CMDQ_SYNC_TOKEN_IMGSYS_XTRAW			233
-+#define CMDQ_SYNC_TOKEN_IMGSYS_DIP			234
-+#define CMDQ_SYNC_TOKEN_IMGSYS_PQDIP_A			235
-+#define CMDQ_SYNC_TOKEN_IMGSYS_PQDIP_B			236
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_1			237
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_2			238
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_3			239
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_4			240
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_5			241
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_6			242
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_7			243
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_8			244
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_9			245
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_10			246
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_11			247
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_12			248
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_13			249
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_14			250
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_15			251
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_16			252
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_17			253
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_18			254
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_19			255
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_20			276
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_21			277
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_22			278
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_23			279
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_24			280
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_25			281
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_26			282
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_27			283
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_28			284
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_29			285
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_30			286
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_31			287
-+#define CMDQ_SYNC_TOKEN_IPESYS_ME			300
-+#define CMDQ_SYNC_TOKEN_IMGSYS_VSS_TRAW			301
-+#define CMDQ_SYNC_TOKEN_IMGSYS_VSS_LTRAW		302
-+#define CMDQ_SYNC_TOKEN_IMGSYS_VSS_XTRAW		303
-+#define CMDQ_SYNC_TOKEN_IMGSYS_VSS_DIP			304
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_32			308
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_33			309
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_34			310
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_35			311
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_36			312
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_37			313
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_38			314
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_39			315
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_40			316
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_41			370
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_42			371
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_43			372
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_44			373
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_45			374
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_46			375
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_47			376
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_48			377
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_49			378
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_50			379
-+#define CMDQ_SYNC_TOKEN_TZMP_ISP_WAIT			380
-+#define CMDQ_SYNC_TOKEN_TZMP_ISP_SET			381
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_51			790
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_52			791
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_53			792
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_54			793
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_55			794
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_56			795
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_57			796
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_58			797
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_59			798
-+#define CMDQ_SYNC_TOKEN_IMGSYS_POOL_60			799
-+#define CMDQ_SYNC_TOKEN_PREBUILT_MDP_WAIT		816
-+#define CMDQ_SYNC_TOKEN_PREBUILT_MDP_SET		817
-+#define CMDQ_SYNC_TOKEN_PREBUILT_MDP_LOCK		818
-+#define CMDQ_SYNC_TOKEN_PREBUILT_MML_WAIT		819
-+#define CMDQ_SYNC_TOKEN_PREBUILT_MML_SET		820
-+#define CMDQ_SYNC_TOKEN_PREBUILT_MML_LOCK		821
-+#define CMDQ_SYNC_TOKEN_PREBUILT_VFMT_WAIT		822
-+#define CMDQ_SYNC_TOKEN_PREBUILT_VFMT_SET		823
-+#define CMDQ_SYNC_TOKEN_PREBUILT_VFMT_LOCK		824
-+#define CMDQ_SYNC_TOKEN_PREBUILT_DISP_WAIT		825
-+#define CMDQ_SYNC_TOKEN_PREBUILT_DISP_SET		826
-+#define CMDQ_SYNC_TOKEN_PREBUILT_DISP_LOCK		827
-+#define CMDQ_SYNC_TOKEN_CONFIG_DIRTY			848
-+#define CMDQ_SYNC_TOKEN_STREAM_EOF			849
-+#define CMDQ_SYNC_TOKEN_ESD_EOF				850
-+#define CMDQ_SYNC_TOKEN_STREAM_BLOCK			851
-+#define CMDQ_SYNC_TOKEN_CABC_EOF			852
-+#define CMDQ_SYNC_TOKEN_VENC_INPUT_READY		853
-+#define CMDQ_SYNC_TOKEN_VENC_EOF			854
-+#define CMDQ_SYNC_TOKEN_SECURE_THR_EOF			855
-+#define CMDQ_SYNC_TOKEN_USER_0				856
-+#define CMDQ_SYNC_TOKEN_USER_1				857
-+#define CMDQ_SYNC_TOKEN_POLL_MONITOR			858
-+#define CMDQ_TOKEN_TPR_LOCK				859
-+#define CMDQ_SYNC_TOKEN_MSS				860
-+#define CMDQ_SYNC_TOKEN_MSF				861
-+#define CMDQ_SYNC_TOKEN_GPR_SET_0			884
-+#define CMDQ_SYNC_TOKEN_GPR_SET_1			885
-+#define CMDQ_SYNC_TOKEN_GPR_SET_2			886
-+#define CMDQ_SYNC_TOKEN_GPR_SET_3			887
-+#define CMDQ_SYNC_TOKEN_GPR_SET_4			888
-+#define CMDQ_SYNC_RESOURCE_WROT0			889
-+#define CMDQ_SYNC_RESOURCE_WROT1			890
-+#define CMDQ_SYNC_TOKEN_DISP_VA_START			1012
-+#define CMDQ_SYNC_TOKEN_DISP_VA_END			1013
-+
-+#endif
-diff --git a/include/dt-bindings/mailbox/qcom-ipcc.h b/include/dt-bindings/mailbox/qcom-ipcc.h
-index fbfa3febc66d..fd85a79381b3 100644
--- a/include/dt-bindings/mailbox/qcom-ipcc.h
-+++ b/include/dt-bindings/mailbox/qcom-ipcc.h
-@@ -33,5 +33,7 @@
- #define IPCC_CLIENT_NSP1		18
- #define IPCC_CLIENT_TME			23
- #define IPCC_CLIENT_WPSS		24
-+#define IPCC_CLIENT_GPDSP0		31
-+#define IPCC_CLIENT_GPDSP1		32
- 
- #endif
-diff --git a/include/dt-bindings/media/c8sectpfe.h b/include/dt-bindings/media/c8sectpfe.h
-deleted file mode 100644
-index 6b1fb6f5413b..000000000000
--- a/include/dt-bindings/media/c8sectpfe.h
-+++ /dev/null
-@@ -1,13 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __DT_C8SECTPFE_H
-#define __DT_C8SECTPFE_H
-
-#define STV0367_TDA18212_NIMA_1	0
-#define STV0367_TDA18212_NIMA_2	1
-#define STV0367_TDA18212_NIMB_1	2
-#define STV0367_TDA18212_NIMB_2	3
-
-#define STV0903_6110_LNB24_NIMA	4
-#define STV0903_6110_LNB24_NIMB	5
-
-#endif /* __DT_C8SECTPFE_H */
-diff --git a/include/dt-bindings/media/tvp5150.h b/include/dt-bindings/media/tvp5150.h
-index dda00c038530..ba34c420c303 100644
--- a/include/dt-bindings/media/tvp5150.h
-+++ b/include/dt-bindings/media/tvp5150.h
-@@ -2,7 +2,7 @@
- /*
-     tvp5150.h - definition for tvp5150 inputs
- 
-    Copyright (C) 2006 Hans Verkuil (hverkuil@xs4all.nl)
-+    Copyright (C) 2006 Hans Verkuil (hverkuil@kernel.org)
- 
- */
- 
-diff --git a/include/dt-bindings/media/video-interfaces.h b/include/dt-bindings/media/video-interfaces.h
-new file mode 100644
-index 000000000000..0b19c9b2e627
--- /dev/null
-+++ b/include/dt-bindings/media/video-interfaces.h
-@@ -0,0 +1,27 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (C) 2022 Laurent Pinchart <laurent.pinchart@ideasonboard.com>
-+ */
-+
-+#ifndef __DT_BINDINGS_MEDIA_VIDEO_INTERFACES_H__
-+#define __DT_BINDINGS_MEDIA_VIDEO_INTERFACES_H__
-+
-+#define MEDIA_BUS_TYPE_CSI2_CPHY		1
-+#define MEDIA_BUS_TYPE_CSI1			2
-+#define MEDIA_BUS_TYPE_CCP2			3
-+#define MEDIA_BUS_TYPE_CSI2_DPHY		4
-+#define MEDIA_BUS_TYPE_PARALLEL			5
-+#define MEDIA_BUS_TYPE_BT656			6
-+
-+#define MEDIA_BUS_CSI2_CPHY_LINE_ORDER_ABC	0
-+#define MEDIA_BUS_CSI2_CPHY_LINE_ORDER_ACB	1
-+#define MEDIA_BUS_CSI2_CPHY_LINE_ORDER_BAC	2
-+#define MEDIA_BUS_CSI2_CPHY_LINE_ORDER_BCA	3
-+#define MEDIA_BUS_CSI2_CPHY_LINE_ORDER_CAB	4
-+#define MEDIA_BUS_CSI2_CPHY_LINE_ORDER_CBA	5
-+
-+#define MEDIA_PCLK_SAMPLE_FALLING_EDGE		0
-+#define MEDIA_PCLK_SAMPLE_RISING_EDGE		1
-+#define MEDIA_PCLK_SAMPLE_DUAL_EDGE		2
-+
-+#endif /* __DT_BINDINGS_MEDIA_VIDEO_INTERFACES_H__ */
-diff --git a/include/dt-bindings/memory/mediatek,mt6893-memory-port.h b/include/dt-bindings/memory/mediatek,mt6893-memory-port.h
-new file mode 100644
-index 000000000000..26e8b400db0d
--- /dev/null
-+++ b/include/dt-bindings/memory/mediatek,mt6893-memory-port.h
-@@ -0,0 +1,288 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2020 MediaTek Inc.
-+ * Copyright (c) 2025 Collabora Ltd
-+ *                    AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
-+ */
-+#ifndef _DT_BINDINGS_MEMORY_MT6893_LARB_PORT_H_
-+#define _DT_BINDINGS_MEMORY_MT6893_LARB_PORT_H_
-+
-+#include <dt-bindings/memory/mtk-memory-port.h>
-+
-+/*
-+ * MM IOMMU supports 16GB dma address.
-+ *
-+ * The address will preassign like this:
-+ *
-+ * modules    dma-address-region	larbs-ports
-+ * disp         0 ~ 4G                  larb0/2
-+ * vcodec      4G ~ 8G                  larb4/5/7
-+ * cam/mdp     8G ~ 12G                 larb9/11/13/14/16/17/18/19/20
-+ * CCU0    0x4000_0000 ~ 0x43ff_ffff    larb13: port 9/10
-+ * CCU1    0x4400_0000 ~ 0x47ff_ffff    larb14: port 4/5
-+ *
-+ * larb3/6/8/10/12/15 are null.
-+ */
-+
-+/* larb0 */
-+#define M4U_PORT_L0_DISP_POSTMASK0		MTK_M4U_DOM_ID(0, 0)
-+#define M4U_PORT_L0_MDP_RDMA4			MTK_M4U_DOM_ID(0, 1)
-+#define M4U_PORT_L0_OVL_RDMA0_HDR		MTK_M4U_DOM_ID(0, 2)
-+#define M4U_PORT_L0_OVL_2L_RDMA1_HDR		MTK_M4U_DOM_ID(0, 3)
-+#define M4U_PORT_L0_OVL_2L_RDMA3_HDR		MTK_M4U_DOM_ID(0, 4)
-+#define M4U_PORT_L0_OVL_RDMA0			MTK_M4U_DOM_ID(0, 5)
-+#define M4U_PORT_L0_OVL_2L_RDMA1		MTK_M4U_DOM_ID(0, 6)
-+#define M4U_PORT_L0_OVL_2L_RDMA3		MTK_M4U_DOM_ID(0, 7)
-+#define M4U_PORT_L0_OVL_RDMA1_SYSRAM		MTK_M4U_DOM_ID(0, 8)
-+#define M4U_PORT_L0_OVL_2L_RDMA0_SYSRAM		MTK_M4U_DOM_ID(0, 9)
-+#define M4U_PORT_L0_OVL_2L_RDMA2_SYSRAM		MTK_M4U_DOM_ID(0, 10)
-+#define M4U_PORT_L0_DISP_WDMA0			MTK_M4U_DOM_ID(0, 11)
-+#define M4U_PORT_L0_DISP_RDMA0			MTK_M4U_DOM_ID(0, 12)
-+#define M4U_PORT_L0_DISP_UFBC_WDMA0		MTK_M4U_DOM_ID(0, 13)
-+#define M4U_PORT_L0_DISP_FAKE0			MTK_M4U_DOM_ID(0, 14)
-+
-+/* larb1 */
-+#define M4U_PORT_L1_DISP_POSTMASK1		MTK_M4U_DOM_ID(1, 0)
-+#define M4U_PORT_L1_MDP_RDMA5			MTK_M4U_DOM_ID(1, 1)
-+#define M4U_PORT_L1_OVL_RDMA1_HDR		MTK_M4U_DOM_ID(1, 2)
-+#define M4U_PORT_L1_OVL_2L_RDMA0_HDR		MTK_M4U_DOM_ID(1, 3)
-+#define M4U_PORT_L1_OVL_2L_RDMA2_HDR		MTK_M4U_DOM_ID(1, 4)
-+#define M4U_PORT_L1_OVL_RDMA1			MTK_M4U_DOM_ID(1, 5)
-+#define M4U_PORT_L1_OVL_2L_RDMA0		MTK_M4U_DOM_ID(1, 6)
-+#define M4U_PORT_L1_OVL_2L_RDMA2		MTK_M4U_DOM_ID(1, 7)
-+#define M4U_PORT_L1_OVL_RDMA0_SYSRAM		MTK_M4U_DOM_ID(1, 8)
-+#define M4U_PORT_L1_OVL_2L_RDMA1_SYSRAM		MTK_M4U_DOM_ID(1, 9)
-+#define M4U_PORT_L1_OVL_2L_RDMA3_SYSRAM		MTK_M4U_DOM_ID(1, 10)
-+#define M4U_PORT_L1_DISP_WDMA1			MTK_M4U_DOM_ID(1, 11)
-+#define M4U_PORT_L1_DISP_RDMA1			MTK_M4U_DOM_ID(1, 12)
-+#define M4U_PORT_L1_DISP_UFBC_WDMA1		MTK_M4U_DOM_ID(1, 13)
-+#define M4U_PORT_L1_DISP_FAKE1			MTK_M4U_DOM_ID(1, 14)
-+
-+/* larb2 */
-+#define M4U_PORT_L2_MDP_RDMA0			MTK_M4U_DOM_ID(2, 0)
-+#define M4U_PORT_L2_MDP_RDMA2			MTK_M4U_DOM_ID(2, 1)
-+#define M4U_PORT_L2_MDP_WROT0			MTK_M4U_DOM_ID(2, 2)
-+#define M4U_PORT_L2_MDP_WROT2			MTK_M4U_DOM_ID(2, 3)
-+#define M4U_PORT_L2_MDP_FILMGRAIN0		MTK_M4U_DOM_ID(2, 4)
-+#define M4U_PORT_L2_MDP_FAKE0			MTK_M4U_DOM_ID(2, 5)
-+
-+/* larb3: null */
-+
-+/* larb4 */
-+#define M4U_PORT_L4_VDEC_MC_EXT_MDP		MTK_M4U_DOM_ID(4, 0)
-+#define M4U_PORT_L4_VDEC_UFO_EXT_MDP		MTK_M4U_DOM_ID(4, 1)
-+#define M4U_PORT_L4_VDEC_PP_EXT_MDP		MTK_M4U_DOM_ID(4, 2)
-+#define M4U_PORT_L4_VDEC_PRED_RD_EXT_MDP	MTK_M4U_DOM_ID(4, 3)
-+#define M4U_PORT_L4_VDEC_PRED_WR_EXT_MDP	MTK_M4U_DOM_ID(4, 4)
-+#define M4U_PORT_L4_VDEC_PPWRAP_EXT_MDP		MTK_M4U_DOM_ID(4, 5)
-+#define M4U_PORT_L4_VDEC_TILE_EXT_MDP		MTK_M4U_DOM_ID(4, 6)
-+#define M4U_PORT_L4_VDEC_VLD_EXT_MDP		MTK_M4U_DOM_ID(4, 7)
-+#define M4U_PORT_L4_VDEC_VLD2_EXT_MDP		MTK_M4U_DOM_ID(4, 8)
-+#define M4U_PORT_L4_VDEC_AVC_MV_EXT_MDP		MTK_M4U_DOM_ID(4, 9)
-+#define M4U_PORT_L4_VDEC_RG_CTRL_DMA_EXT_MDP	MTK_M4U_DOM_ID(4, 10)
-+
-+/* larb5 */
-+#define M4U_PORT_L5_VDEC_LAT0_VLD_EXT_DISP	MTK_M4U_DOM_ID(5, 0)
-+#define M4U_PORT_L5_VDEC_LAT0_VLD2_EXT_DISP	MTK_M4U_DOM_ID(5, 1)
-+#define M4U_PORT_L5_VDEC_LAT0_AVC_MV_EXT_DISP	MTK_M4U_DOM_ID(5, 2)
-+#define M4U_PORT_L5_VDEC_LAT0_PRED_RD_EXT_DISP	MTK_M4U_DOM_ID(5, 3)
-+#define M4U_PORT_L5_VDEC_LAT0_TILE_EXT_DISP	MTK_M4U_DOM_ID(5, 4)
-+#define M4U_PORT_L5_VDEC_LAT0_WDMA_EXT_DISP	MTK_M4U_DOM_ID(5, 5)
-+#define M4U_PORT_L5_VDEC_LAT0_RG_CTRL_DMA_EXT_DISP MTK_M4U_DOM_ID(5, 6)
-+#define M4U_PORT_L5_VDEC_UFO_ENC_EXT_DISP	MTK_M4U_DOM_ID(5, 7)
-+
-+/* larb6: null */
-+
-+/* larb7 */
-+#define M4U_PORT_L7_VENC_RCPU_DISP		MTK_M4U_DOM_ID(7, 0)
-+#define M4U_PORT_L7_VENC_REC_DISP		MTK_M4U_DOM_ID(7, 1)
-+#define M4U_PORT_L7_VENC_BSDMA_DISP		MTK_M4U_DOM_ID(7, 2)
-+#define M4U_PORT_L7_VENC_SV_COMV_DISP		MTK_M4U_DOM_ID(7, 3)
-+#define M4U_PORT_L7_VENC_RD_COMV_DISP		MTK_M4U_DOM_ID(7, 4)
-+#define M4U_PORT_L7_VENC_NBM_RDMA_DISP		MTK_M4U_DOM_ID(7, 5)
-+#define M4U_PORT_L7_VENC_NBM_RDMA_LITE_DISP	MTK_M4U_DOM_ID(7, 6)
-+#define M4U_PORT_L7_JPGENC_Y_RDMA_DISP		MTK_M4U_DOM_ID(7, 7)
-+#define M4U_PORT_L7_JPGENC_C_RDMA_DISP		MTK_M4U_DOM_ID(7, 8)
-+#define M4U_PORT_L7_JPGENC_Q_TABLE_DISP		MTK_M4U_DOM_ID(7, 9)
-+#define M4U_PORT_L7_JPGENC_BSDMA_DISP		MTK_M4U_DOM_ID(7, 10)
-+#define M4U_PORT_L7_JPGENC_WDMA0_DISP		MTK_M4U_DOM_ID(7, 11)
-+#define M4U_PORT_L7_JPGENC_BSDMA0_DISP		MTK_M4U_DOM_ID(7, 12)
-+#define M4U_PORT_L7_VENC_NBM_WDMA_DISP		MTK_M4U_DOM_ID(7, 13)
-+#define M4U_PORT_L7_VENC_NBM_WDMA_LITE_DISP	MTK_M4U_DOM_ID(7, 14)
-+#define M4U_PORT_L7_VENC_CUR_LUMA_DISP		MTK_M4U_DOM_ID(7, 15)
-+#define M4U_PORT_L7_VENC_CUR_CHROMA_DISP	MTK_M4U_DOM_ID(7, 16)
-+#define M4U_PORT_L7_VENC_REF_LUMA_DISP		MTK_M4U_DOM_ID(7, 17)
-+#define M4U_PORT_L7_VENC_REF_CHROMA_DISP	MTK_M4U_DOM_ID(7, 18)
-+#define M4U_PORT_L7_VENC_SUB_R_LUMA_DISP	MTK_M4U_DOM_ID(7, 19)
-+#define M4U_PORT_L7_VENC_SUB_W_LUMA_DISP	MTK_M4U_DOM_ID(7, 20)
-+#define M4U_PORT_L7_VENC_FCS_NBM_RDMA_DISP	MTK_M4U_DOM_ID(7, 21)
-+#define M4U_PORT_L7_VENC_FCS_NBM_WDMA_DISP	MTK_M4U_DOM_ID(7, 22)
-+#define M4U_PORT_L7_JPGENC_WDMA1_DISP		MTK_M4U_DOM_ID(7, 23)
-+#define M4U_PORT_L7_JPGENC_BSDMA1_DISP		MTK_M4U_DOM_ID(7, 24)
-+#define M4U_PORT_L7_JPGENC_HUFF_OFFSET1_DISP	MTK_M4U_DOM_ID(7, 25)
-+#define M4U_PORT_L7_JPGENC_HUFF_OFFSET0_DISP	MTK_M4U_DOM_ID(7, 26)
-+
-+/* larb8: null */
-+
-+/* larb9 */
-+#define M4U_PORT_L9_IMG_IMGI_D1_MDP		MTK_M4U_DOM_ID(9, 0)
-+#define M4U_PORT_L9_IMG_IMGBI_D1_MDP		MTK_M4U_DOM_ID(9, 1)
-+#define M4U_PORT_L9_IMG_DMGI_D1_MDP		MTK_M4U_DOM_ID(9, 2)
-+#define M4U_PORT_L9_IMG_DEPI_D1_MDP		MTK_M4U_DOM_ID(9, 3)
-+#define M4U_PORT_L9_IMG_ICE_D1_MDP		MTK_M4U_DOM_ID(9, 4)
-+#define M4U_PORT_L9_IMG_SMTI_D1_MDP		MTK_M4U_DOM_ID(9, 5)
-+#define M4U_PORT_L9_IMG_SMTO_D2_MDP		MTK_M4U_DOM_ID(9, 6)
-+#define M4U_PORT_L9_IMG_SMTO_D1_MDP		MTK_M4U_DOM_ID(9, 7)
-+#define M4U_PORT_L9_IMG_CRZO_D1_MDP		MTK_M4U_DOM_ID(9, 8)
-+#define M4U_PORT_L9_IMG_IMG3O_D1_MDP		MTK_M4U_DOM_ID(9, 9)
-+#define M4U_PORT_L9_IMG_VIPI_D1_MDP		MTK_M4U_DOM_ID(9, 10)
-+#define M4U_PORT_L9_IMG_SMTI_D5_MDP		MTK_M4U_DOM_ID(9, 11)
-+#define M4U_PORT_L9_IMG_TIMGO_D1_MDP		MTK_M4U_DOM_ID(9, 12)
-+#define M4U_PORT_L9_IMG_UFBC_W0_MDP		MTK_M4U_DOM_ID(9, 13)
-+#define M4U_PORT_L9_IMG_UFBC_R0_MDP		MTK_M4U_DOM_ID(9, 14)
-+#define M4U_PORT_L9_IMG_WPE_RDMA1_MDP		MTK_M4U_DOM_ID(9, 15)
-+#define M4U_PORT_L9_IMG_WPE_RDMA0_MDP		MTK_M4U_DOM_ID(9, 16)
-+#define M4U_PORT_L9_IMG_WPE_WDMA_MDP		MTK_M4U_DOM_ID(9, 17)
-+#define M4U_PORT_L9_IMG_MFB_RDMA0_MDP		MTK_M4U_DOM_ID(9, 18)
-+#define M4U_PORT_L9_IMG_MFB_RDMA1_MDP		MTK_M4U_DOM_ID(9, 19)
-+#define M4U_PORT_L9_IMG_MFB_RDMA2_MDP		MTK_M4U_DOM_ID(9, 20)
-+#define M4U_PORT_L9_IMG_MFB_RDMA3_MDP		MTK_M4U_DOM_ID(9, 21)
-+#define M4U_PORT_L9_IMG_MFB_RDMA4_MDP		MTK_M4U_DOM_ID(9, 22)
-+#define M4U_PORT_L9_IMG_MFB_RDMA5_MDP		MTK_M4U_DOM_ID(9, 23)
-+#define M4U_PORT_L9_IMG_MFB_WDMA0_MDP		MTK_M4U_DOM_ID(9, 24)
-+#define M4U_PORT_L9_IMG_MFB_WDMA1_MDP		MTK_M4U_DOM_ID(9, 25)
-+#define M4U_PORT_L9_IMG_RESERVE6_MDP		MTK_M4U_DOM_ID(9, 26)
-+#define M4U_PORT_L9_IMG_RESERVE7_MDP		MTK_M4U_DOM_ID(9, 27)
-+#define M4U_PORT_L9_IMG_RESERVE8_MDP		MTK_M4U_DOM_ID(9, 28)
-+
-+/* larb10: null */
-+
-+/* larb11 */
-+#define M4U_PORT_L11_IMG_IMGI_D1_DISP		MTK_M4U_DOM_ID(11, 0)
-+#define M4U_PORT_L11_IMG_IMGBI_D1_DISP		MTK_M4U_DOM_ID(11, 1)
-+#define M4U_PORT_L11_IMG_DMGI_D1_DISP		MTK_M4U_DOM_ID(11, 2)
-+#define M4U_PORT_L11_IMG_DEPI_D1_DISP		MTK_M4U_DOM_ID(11, 3)
-+#define M4U_PORT_L11_IMG_ICE_D1_DISP		MTK_M4U_DOM_ID(11, 4)
-+#define M4U_PORT_L11_IMG_SMTI_D1_DISP		MTK_M4U_DOM_ID(11, 5)
-+#define M4U_PORT_L11_IMG_SMTO_D2_DISP		MTK_M4U_DOM_ID(11, 6)
-+#define M4U_PORT_L11_IMG_SMTO_D1_DISP		MTK_M4U_DOM_ID(11, 7)
-+#define M4U_PORT_L11_IMG_CRZO_D1_DISP		MTK_M4U_DOM_ID(11, 8)
-+#define M4U_PORT_L11_IMG_IMG3O_D1_DISP		MTK_M4U_DOM_ID(11, 9)
-+#define M4U_PORT_L11_IMG_VIPI_D1_DISP		MTK_M4U_DOM_ID(11, 10)
-+#define M4U_PORT_L11_IMG_SMTI_D5_DISP		MTK_M4U_DOM_ID(11, 11)
-+#define M4U_PORT_L11_IMG_TIMGO_D1_DISP		MTK_M4U_DOM_ID(11, 12)
-+#define M4U_PORT_L11_IMG_UFBC_W0_DISP		MTK_M4U_DOM_ID(11, 13)
-+#define M4U_PORT_L11_IMG_UFBC_R0_DISP		MTK_M4U_DOM_ID(11, 14)
-+#define M4U_PORT_L11_IMG_WPE_RDMA1_DISP		MTK_M4U_DOM_ID(11, 15)
-+#define M4U_PORT_L11_IMG_WPE_RDMA0_DISP		MTK_M4U_DOM_ID(11, 16)
-+#define M4U_PORT_L11_IMG_WPE_WDMA_DISP		MTK_M4U_DOM_ID(11, 17)
-+#define M4U_PORT_L11_IMG_MFB_RDMA0_DISP		MTK_M4U_DOM_ID(11, 18)
-+#define M4U_PORT_L11_IMG_MFB_RDMA1_DISP		MTK_M4U_DOM_ID(11, 19)
-+#define M4U_PORT_L11_IMG_MFB_RDMA2_DISP		MTK_M4U_DOM_ID(11, 20)
-+#define M4U_PORT_L11_IMG_MFB_RDMA3_DISP		MTK_M4U_DOM_ID(11, 21)
-+#define M4U_PORT_L11_IMG_MFB_RDMA4_DISP		MTK_M4U_DOM_ID(11, 22)
-+#define M4U_PORT_L11_IMG_MFB_RDMA5_DISP		MTK_M4U_DOM_ID(11, 23)
-+#define M4U_PORT_L11_IMG_MFB_WDMA0_DISP		MTK_M4U_DOM_ID(11, 24)
-+#define M4U_PORT_L11_IMG_MFB_WDMA1_DISP		MTK_M4U_DOM_ID(11, 25)
-+#define M4U_PORT_L11_IMG_RESERVE6_DISP		MTK_M4U_DOM_ID(11, 26)
-+#define M4U_PORT_L11_IMG_RESERVE7_DISP		MTK_M4U_DOM_ID(11, 27)
-+#define M4U_PORT_L11_IMG_RESERVE8_DISP		MTK_M4U_DOM_ID(11, 28)
-+
-+/* larb12: null */
-+
-+/* larb13 */
-+#define M4U_PORT_L13_CAM_MRAWI_MDP		MTK_M4U_DOM_ID(13, 0)
-+#define M4U_PORT_L13_CAM_MRAWO0_MDP		MTK_M4U_DOM_ID(13, 1)
-+#define M4U_PORT_L13_CAM_MRAWO1_MDP		MTK_M4U_DOM_ID(13, 2)
-+#define M4U_PORT_L13_CAM_CAMSV1_MDP		MTK_M4U_DOM_ID(13, 3)
-+#define M4U_PORT_L13_CAM_CAMSV2_MDP		MTK_M4U_DOM_ID(13, 4)
-+#define M4U_PORT_L13_CAM_CAMSV3_MDP		MTK_M4U_DOM_ID(13, 5)
-+#define M4U_PORT_L13_CAM_CAMSV4_MDP		MTK_M4U_DOM_ID(13, 6)
-+#define M4U_PORT_L13_CAM_CAMSV5_MDP		MTK_M4U_DOM_ID(13, 7)
-+#define M4U_PORT_L13_CAM_CAMSV6_MDP		MTK_M4U_DOM_ID(13, 8)
-+#define M4U_PORT_L13_CAM_CCUI_MDP		MTK_M4U_DOM_ID(13, 9)
-+#define M4U_PORT_L13_CAM_CCUO_MDP		MTK_M4U_DOM_ID(13, 10)
-+#define M4U_PORT_L13_CAM_FAKE_MDP		MTK_M4U_DOM_ID(13, 11)
-+
-+/* larb14 */
-+#define M4U_PORT_L14_CAM_MRAWI_DISP		MTK_M4U_DOM_ID(14, 0)
-+#define M4U_PORT_L14_CAM_MRAWO0_DISP		MTK_M4U_DOM_ID(14, 1)
-+#define M4U_PORT_L14_CAM_MRAWO1_DISP		MTK_M4U_DOM_ID(14, 2)
-+#define M4U_PORT_L14_CAM_CAMSV0_DISP		MTK_M4U_DOM_ID(14, 3)
-+#define M4U_PORT_L14_CAM_CCUI_DISP		MTK_M4U_DOM_ID(14, 4)
-+#define M4U_PORT_L14_CAM_CCUO_DISP		MTK_M4U_DOM_ID(14, 5)
-+
-+/* larb15: null */
-+
-+/* larb16 */
-+#define M4U_PORT_L16_CAM_IMGO_R1_A_MDP		MTK_M4U_DOM_ID(16, 0)
-+#define M4U_PORT_L16_CAM_RRZO_R1_A_MDP		MTK_M4U_DOM_ID(16, 1)
-+#define M4U_PORT_L16_CAM_CQI_R1_A_MDP		MTK_M4U_DOM_ID(16, 2)
-+#define M4U_PORT_L16_CAM_BPCI_R1_A_MDP		MTK_M4U_DOM_ID(16, 3)
-+#define M4U_PORT_L16_CAM_YUVO_R1_A_MDP		MTK_M4U_DOM_ID(16, 4)
-+#define M4U_PORT_L16_CAM_UFDI_R2_A_MDP		MTK_M4U_DOM_ID(16, 5)
-+#define M4U_PORT_L16_CAM_RAWI_R2_A_MDP		MTK_M4U_DOM_ID(16, 6)
-+#define M4U_PORT_L16_CAM_RAWI_R3_A_MDP		MTK_M4U_DOM_ID(16, 7)
-+#define M4U_PORT_L16_CAM_AAO_R1_A_MDP		MTK_M4U_DOM_ID(16, 8)
-+#define M4U_PORT_L16_CAM_AFO_R1_A_MDP		MTK_M4U_DOM_ID(16, 9)
-+#define M4U_PORT_L16_CAM_FLKO_R1_A_MDP		MTK_M4U_DOM_ID(16, 10)
-+#define M4U_PORT_L16_CAM_LCESO_R1_A_MDP		MTK_M4U_DOM_ID(16, 11)
-+#define M4U_PORT_L16_CAM_CRZO_R1_A_MDP		MTK_M4U_DOM_ID(16, 12)
-+#define M4U_PORT_L16_CAM_LTMSO_R1_A_MDP		MTK_M4U_DOM_ID(16, 13)
-+#define M4U_PORT_L16_CAM_RSSO_R1_A_MDP		MTK_M4U_DOM_ID(16, 14)
-+#define M4U_PORT_L16_CAM_AAHO_R1_A_MDP		MTK_M4U_DOM_ID(16, 15)
-+#define M4U_PORT_L16_CAM_LSCI_R1_A_MDP		MTK_M4U_DOM_ID(16, 16)
-+
-+/* larb17 */
-+#define M4U_PORT_L17_CAM_IMGO_R1_B_DISP		MTK_M4U_DOM_ID(17, 0)
-+#define M4U_PORT_L17_CAM_RRZO_R1_B_DISP		MTK_M4U_DOM_ID(17, 1)
-+#define M4U_PORT_L17_CAM_CQI_R1_B_DISP		MTK_M4U_DOM_ID(17, 2)
-+#define M4U_PORT_L17_CAM_BPCI_R1_B_DISP		MTK_M4U_DOM_ID(17, 3)
-+#define M4U_PORT_L17_CAM_YUVO_R1_B_DISP		MTK_M4U_DOM_ID(17, 4)
-+#define M4U_PORT_L17_CAM_UFDI_R2_B_DISP		MTK_M4U_DOM_ID(17, 5)
-+#define M4U_PORT_L17_CAM_RAWI_R2_B_DISP		MTK_M4U_DOM_ID(17, 6)
-+#define M4U_PORT_L17_CAM_RAWI_R3_B_DISP		MTK_M4U_DOM_ID(17, 7)
-+#define M4U_PORT_L17_CAM_AAO_R1_B_DISP		MTK_M4U_DOM_ID(17, 8)
-+#define M4U_PORT_L17_CAM_AFO_R1_B_DISP		MTK_M4U_DOM_ID(17, 9)
-+#define M4U_PORT_L17_CAM_FLKO_R1_B_DISP		MTK_M4U_DOM_ID(17, 10)
-+#define M4U_PORT_L17_CAM_LCESO_R1_B_DISP	MTK_M4U_DOM_ID(17, 11)
-+#define M4U_PORT_L17_CAM_CRZO_R1_B_DISP		MTK_M4U_DOM_ID(17, 12)
-+#define M4U_PORT_L17_CAM_LTMSO_R1_B_DISP	MTK_M4U_DOM_ID(17, 13)
-+#define M4U_PORT_L17_CAM_RSSO_R1_B_DISP		MTK_M4U_DOM_ID(17, 14)
-+#define M4U_PORT_L17_CAM_AAHO_R1_B_DISP		MTK_M4U_DOM_ID(17, 15)
-+#define M4U_PORT_L17_CAM_LSCI_R1_B_DISP		MTK_M4U_DOM_ID(17, 16)
-+
-+/* larb18 */
-+#define M4U_PORT_L18_CAM_IMGO_R1_C_MDP		MTK_M4U_DOM_ID(18, 0)
-+#define M4U_PORT_L18_CAM_RRZO_R1_C_MDP		MTK_M4U_DOM_ID(18, 1)
-+#define M4U_PORT_L18_CAM_CQI_R1_C_MDP		MTK_M4U_DOM_ID(18, 2)
-+#define M4U_PORT_L18_CAM_BPCI_R1_C_MDP		MTK_M4U_DOM_ID(18, 3)
-+#define M4U_PORT_L18_CAM_YUVO_R1_C_MDP		MTK_M4U_DOM_ID(18, 4)
-+#define M4U_PORT_L18_CAM_UFDI_R2_C_MDP		MTK_M4U_DOM_ID(18, 5)
-+#define M4U_PORT_L18_CAM_RAWI_R2_C_MDP		MTK_M4U_DOM_ID(18, 6)
-+#define M4U_PORT_L18_CAM_RAWI_R3_C_MDP		MTK_M4U_DOM_ID(18, 7)
-+#define M4U_PORT_L18_CAM_AAO_R1_C_MDP		MTK_M4U_DOM_ID(18, 8)
-+#define M4U_PORT_L18_CAM_AFO_R1_C_MDP		MTK_M4U_DOM_ID(18, 9)
-+#define M4U_PORT_L18_CAM_FLKO_R1_C_MDP		MTK_M4U_DOM_ID(18, 10)
-+#define M4U_PORT_L18_CAM_LCESO_R1_C_MDP		MTK_M4U_DOM_ID(18, 11)
-+#define M4U_PORT_L18_CAM_CRZO_R1_C_MDP		MTK_M4U_DOM_ID(18, 12)
-+#define M4U_PORT_L18_CAM_LTMSO_R1_C_MDP		MTK_M4U_DOM_ID(18, 13)
-+#define M4U_PORT_L18_CAM_RSSO_R1_C_MDP		MTK_M4U_DOM_ID(18, 14)
-+#define M4U_PORT_L18_CAM_AAHO_R1_C_MDP		MTK_M4U_DOM_ID(18, 15)
-+#define M4U_PORT_L18_CAM_LSCI_R1_C_MDP		MTK_M4U_DOM_ID(18, 16)
-+
-+/* larb19 */
-+#define M4U_PORT_L19_IPE_DVS_RDMA_DISP		MTK_M4U_DOM_ID(19, 0)
-+#define M4U_PORT_L19_IPE_DVS_WDMA_DISP		MTK_M4U_DOM_ID(19, 1)
-+#define M4U_PORT_L19_IPE_DVP_RDMA_DISP		MTK_M4U_DOM_ID(19, 2)
-+#define M4U_PORT_L19_IPE_DVP_WDMA_DISP		MTK_M4U_DOM_ID(19, 3)
-+
-+/* larb20 */
-+#define M4U_PORT_L20_IPE_FDVT_RDA_DISP		MTK_M4U_DOM_ID(20, 0)
-+#define M4U_PORT_L20_IPE_FDVT_RDB_DISP		MTK_M4U_DOM_ID(20, 1)
-+#define M4U_PORT_L20_IPE_FDVT_WRA_DISP		MTK_M4U_DOM_ID(20, 2)
-+#define M4U_PORT_L20_IPE_FDVT_WRB_DISP		MTK_M4U_DOM_ID(20, 3)
-+#define M4U_PORT_L20_IPE_RSC_RDMA0_DISP		MTK_M4U_DOM_ID(20, 4)
-+#define M4U_PORT_L20_IPE_RSC_WDMA_DISP		MTK_M4U_DOM_ID(20, 5)
-+
-+#endif
-diff --git a/include/dt-bindings/memory/mediatek,mt8188-memory-port.h b/include/dt-bindings/memory/mediatek,mt8188-memory-port.h
-new file mode 100644
-index 000000000000..337ab11262af
--- /dev/null
-+++ b/include/dt-bindings/memory/mediatek,mt8188-memory-port.h
-@@ -0,0 +1,489 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022 MediaTek Inc.
-+ * Author: Chengci Xu <chengci.xu@mediatek.com>
-+ */
-+#ifndef _DT_BINDINGS_MEMORY_MEDIATEK_MT8188_LARB_PORT_H_
-+#define _DT_BINDINGS_MEMORY_MEDIATEK_MT8188_LARB_PORT_H_
-+
-+#include <dt-bindings/memory/mtk-memory-port.h>
-+
-+/*
-+ * MM IOMMU larbs:
-+ * From below, for example larb11 has larb11a/larb11b/larb11c,
-+ * the index of larb is not in order. So we reindexed these larbs from a
-+ * software view.
-+ */
-+#define SMI_L0_ID		0
-+#define SMI_L1_ID		1
-+#define SMI_L2_ID		2
-+#define SMI_L3_ID		3
-+#define SMI_L4_ID		4
-+#define SMI_L5_ID		5
-+#define SMI_L6_ID		6
-+#define SMI_L7_ID		7
-+#define SMI_L9_ID		8
-+#define SMI_L10_ID		9
-+#define SMI_L11A_ID		10
-+#define SMI_L11B_ID		11
-+#define SMI_L11C_ID		12
-+#define SMI_L12_ID		13
-+#define SMI_L13_ID		14
-+#define SMI_L14_ID		15
-+#define SMI_L15_ID		16
-+#define SMI_L16A_ID		17
-+#define SMI_L16B_ID		18
-+#define SMI_L17A_ID		19
-+#define SMI_L17B_ID		20
-+#define SMI_L19_ID		21
-+#define SMI_L21_ID		22
-+#define SMI_L23_ID		23
-+#define SMI_L27_ID		24
-+#define SMI_L28_ID		25
-+
-+/*
-+ * MM IOMMU supports 16GB dma address. We separate it to four ranges:
-+ * 0 ~ 4G; 4G ~ 8G; 8G ~ 12G; 12G ~ 16G, we could adjust these masters
-+ * locate in anyone region. BUT:
-+ * a) Make sure all the ports inside a larb are in one range.
-+ * b) The iova of any master can NOT cross the 4G/8G/12G boundary.
-+ *
-+ * This is the suggested mapping in this SoC:
-+ *
-+ * modules    dma-address-region	larbs-ports
-+ * disp         0 ~ 4G                  larb0/1/2/3
-+ * vcodec      4G ~ 8G                  larb19(21)[1]/21(22)/23
-+ * cam/mdp     8G ~ 12G                 the other larbs.
-+ * N/A         12G ~ 16G
-+ * CCU0   0x24000_0000 ~ 0x243ff_ffff   larb27(24): port 0/1
-+ * CCU1   0x24400_0000 ~ 0x247ff_ffff   larb27(24): port 2/3
-+ *
-+ * This SoC have two MM IOMMU HWs, this is the connected information:
-+ * iommu-vdo: larb0/2/5/9/10/11A/11C/13/16B/17B/19/21
-+ * iommu-vpp: larb1/3/4/6/7/11B/12/14/15/16A/17A/23/27
-+ *
-+ * [1]: This is larb19, but the index is 21 from the SW view.
-+ */
-+
-+/* MM IOMMU ports */
-+/* LARB 0 -- VDO-0 */
-+#define M4U_PORT_L0_DISP_RDMA1			MTK_M4U_ID(SMI_L0_ID, 0)
-+#define M4U_PORT_L0_DISP_WDMA0			MTK_M4U_ID(SMI_L0_ID, 1)
-+#define M4U_PORT_L0_DISP_OVL0_RDMA0		MTK_M4U_ID(SMI_L0_ID, 2)
-+#define M4U_PORT_L0_DISP_OVL0_RDMA1		MTK_M4U_ID(SMI_L0_ID, 3)
-+#define M4U_PORT_L0_DISP_OVL0_HDR		MTK_M4U_ID(SMI_L0_ID, 4)
-+#define M4U_PORT_L0_DISP_POSTMASK0		MTK_M4U_ID(SMI_L0_ID, 5)
-+#define M4U_PORT_L0_DISP_FAKE_ENG0		MTK_M4U_ID(SMI_L0_ID, 6)
-+
-+/* LARB 1 -- VD0-0 */
-+#define M4U_PORT_L1_DISP_RDMA0			MTK_M4U_ID(SMI_L1_ID, 0)
-+#define M4U_PORT_L1_DISP_WDMA1			MTK_M4U_ID(SMI_L1_ID, 1)
-+#define M4U_PORT_L1_DISP_OVL1_RDMA0		MTK_M4U_ID(SMI_L1_ID, 2)
-+#define M4U_PORT_L1_DISP_OVL1_RDMA1		MTK_M4U_ID(SMI_L1_ID, 3)
-+#define M4U_PORT_L1_DISP_OVL1_HDR		MTK_M4U_ID(SMI_L1_ID, 4)
-+#define M4U_PORT_L1_DISP_WROT0			MTK_M4U_ID(SMI_L1_ID, 5)
-+#define M4U_PORT_L1_DISP_FAKE_ENG1		MTK_M4U_ID(SMI_L1_ID, 6)
-+
-+/* LARB 2 -- VDO-1 */
-+#define M4U_PORT_L2_MDP_RDMA0			MTK_M4U_ID(SMI_L2_ID, 0)
-+#define M4U_PORT_L2_MDP_RDMA2			MTK_M4U_ID(SMI_L2_ID, 1)
-+#define M4U_PORT_L2_MDP_RDMA4			MTK_M4U_ID(SMI_L2_ID, 2)
-+#define M4U_PORT_L2_MDP_RDMA6			MTK_M4U_ID(SMI_L2_ID, 3)
-+#define M4U_PORT_L2_DISP_FAKE1			MTK_M4U_ID(SMI_L2_ID, 4)
-+
-+/* LARB 3 -- VDO-1 */
-+#define M4U_PORT_L3_MDP_RDMA1			MTK_M4U_ID(SMI_L3_ID, 0)
-+#define M4U_PORT_L3_MDP_RDMA3			MTK_M4U_ID(SMI_L3_ID, 1)
-+#define M4U_PORT_L3_MDP_RDMA5			MTK_M4U_ID(SMI_L3_ID, 2)
-+#define M4U_PORT_L3_MDP_RDMA7			MTK_M4U_ID(SMI_L3_ID, 3)
-+#define M4U_PORT_L3_HDR_DS_SMI			MTK_M4U_ID(SMI_L3_ID, 4)
-+#define M4U_PORT_L3_HDR_ADL_SMI			MTK_M4U_ID(SMI_L3_ID, 5)
-+#define M4U_PORT_L3_DISP_FAKE1			MTK_M4U_ID(SMI_L3_ID, 6)
-+
-+/* LARB 4 -- VPP-0 */
-+#define M4U_PORT_L4_MDP_RDMA			MTK_M4U_ID(SMI_L4_ID, 0)
-+#define M4U_PORT_L4_MDP_FG			MTK_M4U_ID(SMI_L4_ID, 1)
-+#define M4U_PORT_L4_MDP_OVL			MTK_M4U_ID(SMI_L4_ID, 2)
-+#define M4U_PORT_L4_MDP_WROT			MTK_M4U_ID(SMI_L4_ID, 3)
-+#define M4U_PORT_L4_FAKE_ENG			MTK_M4U_ID(SMI_L4_ID, 4)
-+#define M4U_PORT_L4_DISP_RDMA			MTK_M4U_ID(SMI_L4_ID, 5)
-+#define M4U_PORT_L4_DISP_WDMA			MTK_M4U_ID(SMI_L4_ID, 6)
-+
-+/* LARB 5 -- VPP-1 */
-+#define M4U_PORT_L5_SVPP1_MDP_RDMA		MTK_M4U_ID(SMI_L5_ID, 0)
-+#define M4U_PORT_L5_SVPP1_MDP_FG		MTK_M4U_ID(SMI_L5_ID, 1)
-+#define M4U_PORT_L5_SVPP1_MDP_OVL		MTK_M4U_ID(SMI_L5_ID, 2)
-+#define M4U_PORT_L5_SVPP1_MDP_WROT		MTK_M4U_ID(SMI_L5_ID, 3)
-+#define M4U_PORT_L5_SVPP2_MDP_RDMA		MTK_M4U_ID(SMI_L5_ID, 4)
-+#define M4U_PORT_L5_SVPP2_MDP_FG		MTK_M4U_ID(SMI_L5_ID, 5)
-+#define M4U_PORT_L5_SVPP2_MDP_WROT		MTK_M4U_ID(SMI_L5_ID, 6)
-+#define M4U_PORT_L5_LARB5_FAKE_ENG		MTK_M4U_ID(SMI_L5_ID, 7)
-+
-+/* LARB 6 -- VPP-1 */
-+#define M4U_PORT_L6_SVPP3_MDP_RDMA		MTK_M4U_ID(SMI_L6_ID, 0)
-+#define M4U_PORT_L6_SVPP3_MDP_FG		MTK_M4U_ID(SMI_L6_ID, 1)
-+#define M4U_PORT_L6_SVPP3_MDP_WROT		MTK_M4U_ID(SMI_L6_ID, 2)
-+#define M4U_PORT_L6_LARB6_FAKE_ENG		MTK_M4U_ID(SMI_L6_ID, 3)
-+
-+/* LARB 7 -- WPE */
-+#define M4U_PORT_L7_WPE_RDMA_0			MTK_M4U_ID(SMI_L7_ID, 0)
-+#define M4U_PORT_L7_WPE_RDMA_1			MTK_M4U_ID(SMI_L7_ID, 1)
-+#define M4U_PORT_L7_WPE_WDMA_0			MTK_M4U_ID(SMI_L7_ID, 2)
-+
-+/* LARB 9 -- IMG-M */
-+#define M4U_PORT_L9_IMGI_T1_A			MTK_M4U_ID(SMI_L9_ID, 0)
-+#define M4U_PORT_L9_UFDI_T1_A			MTK_M4U_ID(SMI_L9_ID, 1)
-+#define M4U_PORT_L9_IMGBI_T1_A			MTK_M4U_ID(SMI_L9_ID, 2)
-+#define M4U_PORT_L9_IMGCI_T1_A			MTK_M4U_ID(SMI_L9_ID, 3)
-+#define M4U_PORT_L9_SMTI_T1_A			MTK_M4U_ID(SMI_L9_ID, 4)
-+#define M4U_PORT_L9_SMTI_T4_A			MTK_M4U_ID(SMI_L9_ID, 5)
-+#define M4U_PORT_L9_TNCSTI_T1_A			MTK_M4U_ID(SMI_L9_ID, 6)
-+#define M4U_PORT_L9_TNCSTI_T4_A			MTK_M4U_ID(SMI_L9_ID, 7)
-+#define M4U_PORT_L9_YUVO_T1_A			MTK_M4U_ID(SMI_L9_ID, 8)
-+#define M4U_PORT_L9_YUVBO_T1_A			MTK_M4U_ID(SMI_L9_ID, 9)
-+#define M4U_PORT_L9_YUVCO_T1_A			MTK_M4U_ID(SMI_L9_ID, 10)
-+#define M4U_PORT_L9_TIMGO_T1_A			MTK_M4U_ID(SMI_L9_ID, 11)
-+#define M4U_PORT_L9_YUVO_T2_A			MTK_M4U_ID(SMI_L9_ID, 12)
-+#define M4U_PORT_L9_YUVO_T5_A			MTK_M4U_ID(SMI_L9_ID, 13)
-+#define M4U_PORT_L9_IMGI_T1_B			MTK_M4U_ID(SMI_L9_ID, 14)
-+#define M4U_PORT_L9_IMGBI_T1_B			MTK_M4U_ID(SMI_L9_ID, 15)
-+#define M4U_PORT_L9_IMGCI_T1_B			MTK_M4U_ID(SMI_L9_ID, 16)
-+#define M4U_PORT_L9_SMTI_T4_B			MTK_M4U_ID(SMI_L9_ID, 17)
-+#define M4U_PORT_L9_TNCSO_T1_A			MTK_M4U_ID(SMI_L9_ID, 18)
-+#define M4U_PORT_L9_SMTO_T1_A			MTK_M4U_ID(SMI_L9_ID, 19)
-+#define M4U_PORT_L9_SMTO_T4_A			MTK_M4U_ID(SMI_L9_ID, 20)
-+#define M4U_PORT_L9_TNCSTO_T1_A			MTK_M4U_ID(SMI_L9_ID, 21)
-+#define M4U_PORT_L9_YUVO_T2_B			MTK_M4U_ID(SMI_L9_ID, 22)
-+#define M4U_PORT_L9_YUVO_T5_B			MTK_M4U_ID(SMI_L9_ID, 23)
-+#define M4U_PORT_L9_SMTO_T4_B			MTK_M4U_ID(SMI_L9_ID, 24)
-+
-+/* LARB 10 -- IMG-D */
-+#define M4U_PORT_L10_IMGI_D1			MTK_M4U_ID(SMI_L10_ID, 0)
-+#define M4U_PORT_L10_IMGBI_D1			MTK_M4U_ID(SMI_L10_ID, 1)
-+#define M4U_PORT_L10_IMGCI_D1			MTK_M4U_ID(SMI_L10_ID, 2)
-+#define M4U_PORT_L10_IMGDI_D1			MTK_M4U_ID(SMI_L10_ID, 3)
-+#define M4U_PORT_L10_DEPI_D1			MTK_M4U_ID(SMI_L10_ID, 4)
-+#define M4U_PORT_L10_DMGI_D1			MTK_M4U_ID(SMI_L10_ID, 5)
-+#define M4U_PORT_L10_SMTI_D1			MTK_M4U_ID(SMI_L10_ID, 6)
-+#define M4U_PORT_L10_RECI_D1			MTK_M4U_ID(SMI_L10_ID, 7)
-+#define M4U_PORT_L10_RECI_D1_N			MTK_M4U_ID(SMI_L10_ID, 8)
-+#define M4U_PORT_L10_TNRWI_D1			MTK_M4U_ID(SMI_L10_ID, 9)
-+#define M4U_PORT_L10_TNRCI_D1			MTK_M4U_ID(SMI_L10_ID, 10)
-+#define M4U_PORT_L10_TNRCI_D1_N			MTK_M4U_ID(SMI_L10_ID, 11)
-+#define M4U_PORT_L10_IMG4O_D1			MTK_M4U_ID(SMI_L10_ID, 12)
-+#define M4U_PORT_L10_IMG4BO_D1			MTK_M4U_ID(SMI_L10_ID, 13)
-+#define M4U_PORT_L10_SMTI_D8			MTK_M4U_ID(SMI_L10_ID, 14)
-+#define M4U_PORT_L10_SMTO_D1			MTK_M4U_ID(SMI_L10_ID, 15)
-+#define M4U_PORT_L10_TNRMO_D1			MTK_M4U_ID(SMI_L10_ID, 16)
-+#define M4U_PORT_L10_TNRMO_D1_N			MTK_M4U_ID(SMI_L10_ID, 17)
-+#define M4U_PORT_L10_SMTO_D8			MTK_M4U_ID(SMI_L10_ID, 18)
-+#define M4U_PORT_L10_DBGO_D1			MTK_M4U_ID(SMI_L10_ID, 19)
-+
-+/* LARB 11A -- IMG-D */
-+#define M4U_PORT_L11A_WPE_RDMA_0		MTK_M4U_ID(SMI_L11A_ID, 0)
-+#define M4U_PORT_L11A_WPE_RDMA_1		MTK_M4U_ID(SMI_L11A_ID, 1)
-+#define M4U_PORT_L11A_WPE_RDMA_4P_0		MTK_M4U_ID(SMI_L11A_ID, 2)
-+#define M4U_PORT_L11A_WPE_RDMA_4P_1		MTK_M4U_ID(SMI_L11A_ID, 3)
-+#define M4U_PORT_L11A_WPE_CQ0			MTK_M4U_ID(SMI_L11A_ID, 4)
-+#define M4U_PORT_L11A_WPE_CQ1			MTK_M4U_ID(SMI_L11A_ID, 5)
-+#define M4U_PORT_L11A_PIMGI_P1			MTK_M4U_ID(SMI_L11A_ID, 6)
-+#define M4U_PORT_L11A_PIMGBI_P1			MTK_M4U_ID(SMI_L11A_ID, 7)
-+#define M4U_PORT_L11A_PIMGCI_P1			MTK_M4U_ID(SMI_L11A_ID, 8)
-+#define M4U_PORT_L11A_IMGI_T1_C			MTK_M4U_ID(SMI_L11A_ID, 9)
-+#define M4U_PORT_L11A_IMGBI_T1_C		MTK_M4U_ID(SMI_L11A_ID, 10)
-+#define M4U_PORT_L11A_IMGCI_T1_C		MTK_M4U_ID(SMI_L11A_ID, 11)
-+#define M4U_PORT_L11A_SMTI_T1_C			MTK_M4U_ID(SMI_L11A_ID, 12)
-+#define M4U_PORT_L11A_SMTI_T4_C			MTK_M4U_ID(SMI_L11A_ID, 13)
-+#define M4U_PORT_L11A_SMTI_T6_C			MTK_M4U_ID(SMI_L11A_ID, 14)
-+#define M4U_PORT_L11A_YUVO_T1_C			MTK_M4U_ID(SMI_L11A_ID, 15)
-+#define M4U_PORT_L11A_YUVBO_T1_C		MTK_M4U_ID(SMI_L11A_ID, 16)
-+#define M4U_PORT_L11A_YUVCO_T1_C		MTK_M4U_ID(SMI_L11A_ID, 17)
-+#define M4U_PORT_L11A_WPE_WDMA_0		MTK_M4U_ID(SMI_L11A_ID, 18)
-+#define M4U_PORT_L11A_WPE_WDMA_4P_0		MTK_M4U_ID(SMI_L11A_ID, 19)
-+#define M4U_PORT_L11A_WROT_P1			MTK_M4U_ID(SMI_L11A_ID, 20)
-+#define M4U_PORT_L11A_TCCSO_P1			MTK_M4U_ID(SMI_L11A_ID, 21)
-+#define M4U_PORT_L11A_TCCSI_P1			MTK_M4U_ID(SMI_L11A_ID, 22)
-+#define M4U_PORT_L11A_TIMGO_T1_C		MTK_M4U_ID(SMI_L11A_ID, 23)
-+#define M4U_PORT_L11A_YUVO_T2_C			MTK_M4U_ID(SMI_L11A_ID, 24)
-+#define M4U_PORT_L11A_YUVO_T5_C			MTK_M4U_ID(SMI_L11A_ID, 25)
-+#define M4U_PORT_L11A_SMTO_T1_C			MTK_M4U_ID(SMI_L11A_ID, 26)
-+#define M4U_PORT_L11A_SMTO_T4_C			MTK_M4U_ID(SMI_L11A_ID, 27)
-+#define M4U_PORT_L11A_SMTO_T6_C			MTK_M4U_ID(SMI_L11A_ID, 28)
-+#define M4U_PORT_L11A_DBGO_T1_C			MTK_M4U_ID(SMI_L11A_ID, 29)
-+
-+/* LARB 11B -- IMG-D */
-+#define M4U_PORT_L11B_WPE_RDMA_0		MTK_M4U_ID(SMI_L11B_ID, 0)
-+#define M4U_PORT_L11B_WPE_RDMA_1		MTK_M4U_ID(SMI_L11B_ID, 1)
-+#define M4U_PORT_L11B_WPE_RDMA_4P_0		MTK_M4U_ID(SMI_L11B_ID, 2)
-+#define M4U_PORT_L11B_WPE_RDMA_4P_1		MTK_M4U_ID(SMI_L11B_ID, 3)
-+#define M4U_PORT_L11B_WPE_CQ0			MTK_M4U_ID(SMI_L11B_ID, 4)
-+#define M4U_PORT_L11B_WPE_CQ1			MTK_M4U_ID(SMI_L11B_ID, 5)
-+#define M4U_PORT_L11B_PIMGI_P1			MTK_M4U_ID(SMI_L11B_ID, 6)
-+#define M4U_PORT_L11B_PIMGBI_P1			MTK_M4U_ID(SMI_L11B_ID, 7)
-+#define M4U_PORT_L11B_PIMGCI_P1			MTK_M4U_ID(SMI_L11B_ID, 8)
-+#define M4U_PORT_L11B_IMGI_T1_C			MTK_M4U_ID(SMI_L11B_ID, 9)
-+#define M4U_PORT_L11B_IMGBI_T1_C		MTK_M4U_ID(SMI_L11B_ID, 10)
-+#define M4U_PORT_L11B_IMGCI_T1_C		MTK_M4U_ID(SMI_L11B_ID, 11)
-+#define M4U_PORT_L11B_SMTI_T1_C			MTK_M4U_ID(SMI_L11B_ID, 12)
-+#define M4U_PORT_L11B_SMTI_T4_C			MTK_M4U_ID(SMI_L11B_ID, 13)
-+#define M4U_PORT_L11B_SMTI_T6_C			MTK_M4U_ID(SMI_L11B_ID, 14)
-+#define M4U_PORT_L11B_YUVO_T1_C			MTK_M4U_ID(SMI_L11B_ID, 15)
-+#define M4U_PORT_L11B_YUVBO_T1_C		MTK_M4U_ID(SMI_L11B_ID, 16)
-+#define M4U_PORT_L11B_YUVCO_T1_C		MTK_M4U_ID(SMI_L11B_ID, 17)
-+#define M4U_PORT_L11B_WPE_WDMA_0		MTK_M4U_ID(SMI_L11B_ID, 18)
-+#define M4U_PORT_L11B_WPE_WDMA_4P_0		MTK_M4U_ID(SMI_L11B_ID, 19)
-+#define M4U_PORT_L11B_WROT_P1			MTK_M4U_ID(SMI_L11B_ID, 20)
-+#define M4U_PORT_L11B_TCCSO_P1			MTK_M4U_ID(SMI_L11B_ID, 21)
-+#define M4U_PORT_L11B_TCCSI_P1			MTK_M4U_ID(SMI_L11B_ID, 22)
-+#define M4U_PORT_L11B_TIMGO_T1_C		MTK_M4U_ID(SMI_L11B_ID, 23)
-+#define M4U_PORT_L11B_YUVO_T2_C			MTK_M4U_ID(SMI_L11B_ID, 24)
-+#define M4U_PORT_L11B_YUVO_T5_C			MTK_M4U_ID(SMI_L11B_ID, 25)
-+#define M4U_PORT_L11B_SMTO_T1_C			MTK_M4U_ID(SMI_L11B_ID, 26)
-+#define M4U_PORT_L11B_SMTO_T4_C			MTK_M4U_ID(SMI_L11B_ID, 27)
-+#define M4U_PORT_L11B_SMTO_T6_C			MTK_M4U_ID(SMI_L11B_ID, 28)
-+#define M4U_PORT_L11B_DBGO_T1_C			MTK_M4U_ID(SMI_L11B_ID, 29)
-+
-+/* LARB 11C -- IMG-D */
-+#define M4U_PORT_L11C_WPE_RDMA_0		MTK_M4U_ID(SMI_L11C_ID, 0)
-+#define M4U_PORT_L11C_WPE_RDMA_1		MTK_M4U_ID(SMI_L11C_ID, 1)
-+#define M4U_PORT_L11C_WPE_RDMA_4P_0		MTK_M4U_ID(SMI_L11C_ID, 2)
-+#define M4U_PORT_L11C_WPE_RDMA_4P_1		MTK_M4U_ID(SMI_L11C_ID, 3)
-+#define M4U_PORT_L11C_WPE_CQ0			MTK_M4U_ID(SMI_L11C_ID, 4)
-+#define M4U_PORT_L11C_WPE_CQ1			MTK_M4U_ID(SMI_L11C_ID, 5)
-+#define M4U_PORT_L11C_PIMGI_P1			MTK_M4U_ID(SMI_L11C_ID, 6)
-+#define M4U_PORT_L11C_PIMGBI_P1			MTK_M4U_ID(SMI_L11C_ID, 7)
-+#define M4U_PORT_L11C_PIMGCI_P1			MTK_M4U_ID(SMI_L11C_ID, 8)
-+#define M4U_PORT_L11C_IMGI_T1_C			MTK_M4U_ID(SMI_L11C_ID, 9)
-+#define M4U_PORT_L11C_IMGBI_T1_C		MTK_M4U_ID(SMI_L11C_ID, 10)
-+#define M4U_PORT_L11C_IMGCI_T1_C		MTK_M4U_ID(SMI_L11C_ID, 11)
-+#define M4U_PORT_L11C_SMTI_T1_C			MTK_M4U_ID(SMI_L11C_ID, 12)
-+#define M4U_PORT_L11C_SMTI_T4_C			MTK_M4U_ID(SMI_L11C_ID, 13)
-+#define M4U_PORT_L11C_SMTI_T6_C			MTK_M4U_ID(SMI_L11C_ID, 14)
-+#define M4U_PORT_L11C_YUVO_T1_C			MTK_M4U_ID(SMI_L11C_ID, 15)
-+#define M4U_PORT_L11C_YUVBO_T1_C		MTK_M4U_ID(SMI_L11C_ID, 16)
-+#define M4U_PORT_L11C_YUVCO_T1_C		MTK_M4U_ID(SMI_L11C_ID, 17)
-+#define M4U_PORT_L11C_WPE_WDMA_0		MTK_M4U_ID(SMI_L11C_ID, 18)
-+#define M4U_PORT_L11C_WPE_WDMA_4P_0		MTK_M4U_ID(SMI_L11C_ID, 19)
-+#define M4U_PORT_L11C_WROT_P1			MTK_M4U_ID(SMI_L11C_ID, 20)
-+#define M4U_PORT_L11C_TCCSO_P1			MTK_M4U_ID(SMI_L11C_ID, 21)
-+#define M4U_PORT_L11C_TCCSI_P1			MTK_M4U_ID(SMI_L11C_ID, 22)
-+#define M4U_PORT_L11C_TIMGO_T1_C		MTK_M4U_ID(SMI_L11C_ID, 23)
-+#define M4U_PORT_L11C_YUVO_T2_C			MTK_M4U_ID(SMI_L11C_ID, 24)
-+#define M4U_PORT_L11C_YUVO_T5_C			MTK_M4U_ID(SMI_L11C_ID, 25)
-+#define M4U_PORT_L11C_SMTO_T1_C			MTK_M4U_ID(SMI_L11C_ID, 26)
-+#define M4U_PORT_L11C_SMTO_T4_C			MTK_M4U_ID(SMI_L11C_ID, 27)
-+#define M4U_PORT_L11C_SMTO_T6_C			MTK_M4U_ID(SMI_L11C_ID, 28)
-+#define M4U_PORT_L11C_DBGO_T1_C			MTK_M4U_ID(SMI_L11C_ID, 29)
-+
-+/* LARB 12 -- IPE */
-+#define M4U_PORT_L12_FDVT_RDA_0			MTK_M4U_ID(SMI_L12_ID, 0)
-+#define M4U_PORT_L12_FDVT_RDB_0			MTK_M4U_ID(SMI_L12_ID, 1)
-+#define M4U_PORT_L12_FDVT_WRA_0			MTK_M4U_ID(SMI_L12_ID, 2)
-+#define M4U_PORT_L12_FDVT_WRB_0			MTK_M4U_ID(SMI_L12_ID, 3)
-+#define M4U_PORT_L12_ME_RDMA			MTK_M4U_ID(SMI_L12_ID, 4)
-+#define M4U_PORT_L12_ME_WDMA			MTK_M4U_ID(SMI_L12_ID, 5)
-+#define M4U_PORT_L12_DVS_RDMA			MTK_M4U_ID(SMI_L12_ID, 6)
-+#define M4U_PORT_L12_DVS_WDMA			MTK_M4U_ID(SMI_L12_ID, 7)
-+#define M4U_PORT_L12_DVP_RDMA			MTK_M4U_ID(SMI_L12_ID, 8)
-+#define M4U_PORT_L12_DVP_WDMA			MTK_M4U_ID(SMI_L12_ID, 9)
-+#define M4U_PORT_L12_FDVT_2ND_RDA_0		MTK_M4U_ID(SMI_L12_ID, 10)
-+#define M4U_PORT_L12_FDVT_2ND_RDB_0		MTK_M4U_ID(SMI_L12_ID, 11)
-+#define M4U_PORT_L12_FDVT_2ND_WRA_0		MTK_M4U_ID(SMI_L12_ID, 12)
-+#define M4U_PORT_L12_FDVT_2ND_WRB_0		MTK_M4U_ID(SMI_L12_ID, 13)
-+#define M4U_PORT_L12_DHZEI_E1			MTK_M4U_ID(SMI_L12_ID, 14)
-+#define M4U_PORT_L12_DHZEO_E1			MTK_M4U_ID(SMI_L12_ID, 15)
-+
-+/* LARB 13 -- CAM-1 */
-+#define M4U_PORT_L13_CAMSV_CQI_E1		MTK_M4U_ID(SMI_L13_ID, 0)
-+#define M4U_PORT_L13_CAMSV_CQI_E2		MTK_M4U_ID(SMI_L13_ID, 1)
-+#define M4U_PORT_L13_GCAMSV_A_IMGO_1		MTK_M4U_ID(SMI_L13_ID, 2)
-+#define M4U_PORT_L13_GCAMSV_C_IMGO_1		MTK_M4U_ID(SMI_L13_ID, 3)
-+#define M4U_PORT_L13_GCAMSV_A_IMGO_2		MTK_M4U_ID(SMI_L13_ID, 4)
-+#define M4U_PORT_L13_GCAMSV_C_IMGO_2		MTK_M4U_ID(SMI_L13_ID, 5)
-+#define M4U_PORT_L13_PDAI_A_0			MTK_M4U_ID(SMI_L13_ID, 6)
-+#define M4U_PORT_L13_PDAI_A_1			MTK_M4U_ID(SMI_L13_ID, 7)
-+#define M4U_PORT_L13_CAMSV_CQI_B_E1		MTK_M4U_ID(SMI_L13_ID, 8)
-+#define M4U_PORT_L13_CAMSV_CQI_B_E2		MTK_M4U_ID(SMI_L13_ID, 9)
-+#define M4U_PORT_L13_CAMSV_CQI_C_E1		MTK_M4U_ID(SMI_L13_ID, 10)
-+#define M4U_PORT_L13_CAMSV_CQI_C_E2		MTK_M4U_ID(SMI_L13_ID, 11)
-+#define M4U_PORT_L13_GCAMSV_E_IMGO_1		MTK_M4U_ID(SMI_L13_ID, 12)
-+#define M4U_PORT_L13_GCAMSV_E_IMGO_2		MTK_M4U_ID(SMI_L13_ID, 13)
-+#define M4U_PORT_L13_GCAMSV_A_UFEO_1		MTK_M4U_ID(SMI_L13_ID, 14)
-+#define M4U_PORT_L13_GCAMSV_C_UFEO_1		MTK_M4U_ID(SMI_L13_ID, 15)
-+#define M4U_PORT_L13_GCAMSV_A_UFEO_2		MTK_M4U_ID(SMI_L13_ID, 16)
-+#define M4U_PORT_L13_GCAMSV_C_UFEO_2		MTK_M4U_ID(SMI_L13_ID, 17)
-+#define M4U_PORT_L13_GCAMSV_E_UFEO_1		MTK_M4U_ID(SMI_L13_ID, 18)
-+#define M4U_PORT_L13_GCAMSV_E_UFEO_2		MTK_M4U_ID(SMI_L13_ID, 19)
-+#define M4U_PORT_L13_GCAMSV_G_IMGO_1		MTK_M4U_ID(SMI_L13_ID, 20)
-+#define M4U_PORT_L13_GCAMSV_G_IMGO_2		MTK_M4U_ID(SMI_L13_ID, 21)
-+#define M4U_PORT_L13_PDAO_A			MTK_M4U_ID(SMI_L13_ID, 22)
-+#define M4U_PORT_L13_PDAO_C			MTK_M4U_ID(SMI_L13_ID, 23)
-+
-+/* LARB 14 -- CAM-1 */
-+#define M4U_PORT_L14_GCAMSV_B_IMGO_1		MTK_M4U_ID(SMI_L14_ID, 0)
-+#define M4U_PORT_L14_GCAMSV_B_IMGO_2		MTK_M4U_ID(SMI_L14_ID, 1)
-+#define M4U_PORT_L14_SCAMSV_A_IMGO_1		MTK_M4U_ID(SMI_L14_ID, 2)
-+#define M4U_PORT_L14_SCAMSV_A_IMGO_2		MTK_M4U_ID(SMI_L14_ID, 3)
-+#define M4U_PORT_L14_SCAMSV_B_IMGO_1		MTK_M4U_ID(SMI_L14_ID, 4)
-+#define M4U_PORT_L14_SCAMSV_B_IMGO_2		MTK_M4U_ID(SMI_L14_ID, 5)
-+#define M4U_PORT_L14_PDAI_B_0			MTK_M4U_ID(SMI_L14_ID, 6)
-+#define M4U_PORT_L14_PDAI_B_1			MTK_M4U_ID(SMI_L14_ID, 7)
-+#define M4U_PORT_L14_GCAMSV_D_IMGO_1		MTK_M4U_ID(SMI_L14_ID, 8)
-+#define M4U_PORT_L14_GCAMSV_D_IMGO_2		MTK_M4U_ID(SMI_L14_ID, 9)
-+#define M4U_PORT_L14_GCAMSV_F_IMGO_1		MTK_M4U_ID(SMI_L14_ID, 10)
-+#define M4U_PORT_L14_GCAMSV_F_IMGO_2		MTK_M4U_ID(SMI_L14_ID, 11)
-+#define M4U_PORT_L14_GCAMSV_H_IMGO_1		MTK_M4U_ID(SMI_L14_ID, 12)
-+#define M4U_PORT_L14_GCAMSV_H_IMGO_2		MTK_M4U_ID(SMI_L14_ID, 13)
-+#define M4U_PORT_L14_GCAMSV_B_UFEO_1		MTK_M4U_ID(SMI_L14_ID, 14)
-+#define M4U_PORT_L14_GCAMSV_B_UFEO_2		MTK_M4U_ID(SMI_L14_ID, 15)
-+#define M4U_PORT_L14_GCAMSV_D_UFEO_1		MTK_M4U_ID(SMI_L14_ID, 16)
-+#define M4U_PORT_L14_GCAMSV_D_UFEO_2		MTK_M4U_ID(SMI_L14_ID, 17)
-+#define M4U_PORT_L14_PDAO_B			MTK_M4U_ID(SMI_L14_ID, 18)
-+#define M4U_PORT_L14_IPUI			MTK_M4U_ID(SMI_L14_ID, 19)
-+#define M4U_PORT_L14_IPUO			MTK_M4U_ID(SMI_L14_ID, 20)
-+#define M4U_PORT_L14_IPU3O			MTK_M4U_ID(SMI_L14_ID, 21)
-+#define M4U_PORT_L14_FAKE			MTK_M4U_ID(SMI_L14_ID, 22)
-+
-+/* LARB 15 -- IMG-D */
-+#define M4U_PORT_L15_VIPI_D1			MTK_M4U_ID(SMI_L15_ID, 0)
-+#define M4U_PORT_L15_VIPBI_D1			MTK_M4U_ID(SMI_L15_ID, 1)
-+#define M4U_PORT_L15_SMTI_D6			MTK_M4U_ID(SMI_L15_ID, 2)
-+#define M4U_PORT_L15_TNCSTI_D1			MTK_M4U_ID(SMI_L15_ID, 3)
-+#define M4U_PORT_L15_TNCSTI_D4			MTK_M4U_ID(SMI_L15_ID, 4)
-+#define M4U_PORT_L15_SMTI_D4			MTK_M4U_ID(SMI_L15_ID, 5)
-+#define M4U_PORT_L15_IMG3O_D1			MTK_M4U_ID(SMI_L15_ID, 6)
-+#define M4U_PORT_L15_IMG3BO_D1			MTK_M4U_ID(SMI_L15_ID, 7)
-+#define M4U_PORT_L15_IMG3CO_D1			MTK_M4U_ID(SMI_L15_ID, 8)
-+#define M4U_PORT_L15_IMG2O_D1			MTK_M4U_ID(SMI_L15_ID, 9)
-+#define M4U_PORT_L15_SMTI_D9			MTK_M4U_ID(SMI_L15_ID, 10)
-+#define M4U_PORT_L15_SMTO_D4			MTK_M4U_ID(SMI_L15_ID, 11)
-+#define M4U_PORT_L15_FEO_D1			MTK_M4U_ID(SMI_L15_ID, 12)
-+#define M4U_PORT_L15_TNCSO_D1			MTK_M4U_ID(SMI_L15_ID, 13)
-+#define M4U_PORT_L15_TNCSTO_D1			MTK_M4U_ID(SMI_L15_ID, 14)
-+#define M4U_PORT_L15_SMTO_D6			MTK_M4U_ID(SMI_L15_ID, 15)
-+#define M4U_PORT_L15_SMTO_D9			MTK_M4U_ID(SMI_L15_ID, 16)
-+#define M4U_PORT_L15_TNCO_D1			MTK_M4U_ID(SMI_L15_ID, 17)
-+#define M4U_PORT_L15_TNCO_D1_N			MTK_M4U_ID(SMI_L15_ID, 18)
-+
-+/* LARB 16A -- CAM */
-+#define M4U_PORT_L16A_IMGO_R1			MTK_M4U_ID(SMI_L16A_ID, 0)
-+#define M4U_PORT_L16A_CQI_R1			MTK_M4U_ID(SMI_L16A_ID, 1)
-+#define M4U_PORT_L16A_CQI_R2			MTK_M4U_ID(SMI_L16A_ID, 2)
-+#define M4U_PORT_L16A_BPCI_R1			MTK_M4U_ID(SMI_L16A_ID, 3)
-+#define M4U_PORT_L16A_LSCI_R1			MTK_M4U_ID(SMI_L16A_ID, 4)
-+#define M4U_PORT_L16A_RAWI_R2			MTK_M4U_ID(SMI_L16A_ID, 5)
-+#define M4U_PORT_L16A_RAWI_R3			MTK_M4U_ID(SMI_L16A_ID, 6)
-+#define M4U_PORT_L16A_UFDI_R2			MTK_M4U_ID(SMI_L16A_ID, 7)
-+#define M4U_PORT_L16A_UFDI_R3			MTK_M4U_ID(SMI_L16A_ID, 8)
-+#define M4U_PORT_L16A_RAWI_R4			MTK_M4U_ID(SMI_L16A_ID, 9)
-+#define M4U_PORT_L16A_RAWI_R5			MTK_M4U_ID(SMI_L16A_ID, 10)
-+#define M4U_PORT_L16A_AAI_R1			MTK_M4U_ID(SMI_L16A_ID, 11)
-+#define M4U_PORT_L16A_UFDI_R5			MTK_M4U_ID(SMI_L16A_ID, 12)
-+#define M4U_PORT_L16A_FHO_R1			MTK_M4U_ID(SMI_L16A_ID, 13)
-+#define M4U_PORT_L16A_AAO_R1			MTK_M4U_ID(SMI_L16A_ID, 14)
-+#define M4U_PORT_L16A_TSFSO_R1			MTK_M4U_ID(SMI_L16A_ID, 15)
-+#define M4U_PORT_L16A_FLKO_R1			MTK_M4U_ID(SMI_L16A_ID, 16)
-+
-+/* LARB 16B -- CAM */
-+#define M4U_PORT_L16B_IMGO_R1			MTK_M4U_ID(SMI_L16B_ID, 0)
-+#define M4U_PORT_L16B_CQI_R1			MTK_M4U_ID(SMI_L16B_ID, 1)
-+#define M4U_PORT_L16B_CQI_R2			MTK_M4U_ID(SMI_L16B_ID, 2)
-+#define M4U_PORT_L16B_BPCI_R1			MTK_M4U_ID(SMI_L16B_ID, 3)
-+#define M4U_PORT_L16B_LSCI_R1			MTK_M4U_ID(SMI_L16B_ID, 4)
-+#define M4U_PORT_L16B_RAWI_R2			MTK_M4U_ID(SMI_L16B_ID, 5)
-+#define M4U_PORT_L16B_RAWI_R3			MTK_M4U_ID(SMI_L16B_ID, 6)
-+#define M4U_PORT_L16B_UFDI_R2			MTK_M4U_ID(SMI_L16B_ID, 7)
-+#define M4U_PORT_L16B_UFDI_R3			MTK_M4U_ID(SMI_L16B_ID, 8)
-+#define M4U_PORT_L16B_RAWI_R4			MTK_M4U_ID(SMI_L16B_ID, 9)
-+#define M4U_PORT_L16B_RAWI_R5			MTK_M4U_ID(SMI_L16B_ID, 10)
-+#define M4U_PORT_L16B_AAI_R1			MTK_M4U_ID(SMI_L16B_ID, 11)
-+#define M4U_PORT_L16B_UFDI_R5			MTK_M4U_ID(SMI_L16B_ID, 12)
-+#define M4U_PORT_L16B_FHO_R1			MTK_M4U_ID(SMI_L16B_ID, 13)
-+#define M4U_PORT_L16B_AAO_R1			MTK_M4U_ID(SMI_L16B_ID, 14)
-+#define M4U_PORT_L16B_TSFSO_R1			MTK_M4U_ID(SMI_L16B_ID, 15)
-+#define M4U_PORT_L16B_FLKO_R1			MTK_M4U_ID(SMI_L16B_ID, 16)
-+
-+/* LARB 17A -- CAM */
-+#define M4U_PORT_L17A_YUVO_R1			MTK_M4U_ID(SMI_L17A_ID, 0)
-+#define M4U_PORT_L17A_YUVO_R3			MTK_M4U_ID(SMI_L17A_ID, 1)
-+#define M4U_PORT_L17A_YUVCO_R1			MTK_M4U_ID(SMI_L17A_ID, 2)
-+#define M4U_PORT_L17A_YUVO_R2			MTK_M4U_ID(SMI_L17A_ID, 3)
-+#define M4U_PORT_L17A_RZH1N2TO_R1		MTK_M4U_ID(SMI_L17A_ID, 4)
-+#define M4U_PORT_L17A_DRZS4NO_R1		MTK_M4U_ID(SMI_L17A_ID, 5)
-+#define M4U_PORT_L17A_TNCSO_R1			MTK_M4U_ID(SMI_L17A_ID, 6)
-+
-+/* LARB 17B -- CAM */
-+#define M4U_PORT_L17B_YUVO_R1			MTK_M4U_ID(SMI_L17B_ID, 0)
-+#define M4U_PORT_L17B_YUVO_R3			MTK_M4U_ID(SMI_L17B_ID, 1)
-+#define M4U_PORT_L17B_YUVCO_R1			MTK_M4U_ID(SMI_L17B_ID, 2)
-+#define M4U_PORT_L17B_YUVO_R2			MTK_M4U_ID(SMI_L17B_ID, 3)
-+#define M4U_PORT_L17B_RZH1N2TO_R1		MTK_M4U_ID(SMI_L17B_ID, 4)
-+#define M4U_PORT_L17B_DRZS4NO_R1		MTK_M4U_ID(SMI_L17B_ID, 5)
-+#define M4U_PORT_L17B_TNCSO_R1			MTK_M4U_ID(SMI_L17B_ID, 6)
-+
-+/* LARB 19 -- VENC */
-+#define M4U_PORT_L19_VENC_RCPU			MTK_M4U_ID(SMI_L19_ID, 0)
-+#define M4U_PORT_L19_VENC_REC			MTK_M4U_ID(SMI_L19_ID, 1)
-+#define M4U_PORT_L19_VENC_BSDMA			MTK_M4U_ID(SMI_L19_ID, 2)
-+#define M4U_PORT_L19_VENC_SV_COMV		MTK_M4U_ID(SMI_L19_ID, 3)
-+#define M4U_PORT_L19_VENC_RD_COMV		MTK_M4U_ID(SMI_L19_ID, 4)
-+#define M4U_PORT_L19_VENC_NBM_RDMA		MTK_M4U_ID(SMI_L19_ID, 5)
-+#define M4U_PORT_L19_VENC_NBM_RDMA_LITE		MTK_M4U_ID(SMI_L19_ID, 6)
-+#define M4U_PORT_L19_JPGENC_Y_RDMA		MTK_M4U_ID(SMI_L19_ID, 7)
-+#define M4U_PORT_L19_JPGENC_C_RDMA		MTK_M4U_ID(SMI_L19_ID, 8)
-+#define M4U_PORT_L19_JPGENC_Q_TABLE		MTK_M4U_ID(SMI_L19_ID, 9)
-+#define M4U_PORT_L19_VENC_SUB_W_LUMA		MTK_M4U_ID(SMI_L19_ID, 10)
-+#define M4U_PORT_L19_VENC_FCS_NBM_RDMA		MTK_M4U_ID(SMI_L19_ID, 11)
-+#define M4U_PORT_L19_JPGENC_BSDMA		MTK_M4U_ID(SMI_L19_ID, 12)
-+#define M4U_PORT_L19_JPGDEC_WDMA_0		MTK_M4U_ID(SMI_L19_ID, 13)
-+#define M4U_PORT_L19_JPGDEC_BSDMA_0		MTK_M4U_ID(SMI_L19_ID, 14)
-+#define M4U_PORT_L19_VENC_NBM_WDMA		MTK_M4U_ID(SMI_L19_ID, 15)
-+#define M4U_PORT_L19_VENC_NBM_WDMA_LITE		MTK_M4U_ID(SMI_L19_ID, 16)
-+#define M4U_PORT_L19_VENC_FCS_NBM_WDMA		MTK_M4U_ID(SMI_L19_ID, 17)
-+#define M4U_PORT_L19_JPGDEC_WDMA_1		MTK_M4U_ID(SMI_L19_ID, 18)
-+#define M4U_PORT_L19_JPGDEC_BSDMA_1		MTK_M4U_ID(SMI_L19_ID, 19)
-+#define M4U_PORT_L19_JPGDEC_HUFF_OFFSET_1	MTK_M4U_ID(SMI_L19_ID, 20)
-+#define M4U_PORT_L19_JPGDEC_HUFF_OFFSET_0	MTK_M4U_ID(SMI_L19_ID, 21)
-+#define M4U_PORT_L19_VENC_CUR_LUMA		MTK_M4U_ID(SMI_L19_ID, 22)
-+#define M4U_PORT_L19_VENC_CUR_CHROMA		MTK_M4U_ID(SMI_L19_ID, 23)
-+#define M4U_PORT_L19_VENC_REF_LUMA		MTK_M4U_ID(SMI_L19_ID, 24)
-+#define M4U_PORT_L19_VENC_REF_CHROMA		MTK_M4U_ID(SMI_L19_ID, 25)
-+#define M4U_PORT_L19_VENC_SUB_R_LUMA		MTK_M4U_ID(SMI_L19_ID, 26)
-+
-+/* LARB 21 -- VDEC-CORE0 */
-+#define M4U_PORT_L21_HW_VDEC_MC_EXT		MTK_M4U_ID(SMI_L21_ID, 0)
-+#define M4U_PORT_L21_HW_VDEC_UFO_EXT		MTK_M4U_ID(SMI_L21_ID, 1)
-+#define M4U_PORT_L21_HW_VDEC_PP_EXT		MTK_M4U_ID(SMI_L21_ID, 2)
-+#define M4U_PORT_L21_HW_VDEC_PRED_RD_EXT	MTK_M4U_ID(SMI_L21_ID, 3)
-+#define M4U_PORT_L21_HW_VDEC_PRED_WR_EXT	MTK_M4U_ID(SMI_L21_ID, 4)
-+#define M4U_PORT_L21_HW_VDEC_PPWRAP_EXT		MTK_M4U_ID(SMI_L21_ID, 5)
-+#define M4U_PORT_L21_HW_VDEC_TILE_EXT		MTK_M4U_ID(SMI_L21_ID, 6)
-+#define M4U_PORT_L21_HW_VDEC_VLD_EXT		MTK_M4U_ID(SMI_L21_ID, 7)
-+#define M4U_PORT_L21_HW_VDEC_VLD2_EXT		MTK_M4U_ID(SMI_L21_ID, 8)
-+#define M4U_PORT_L21_HW_VDEC_AVC_MV_EXT		MTK_M4U_ID(SMI_L21_ID, 9)
-+#define M4U_PORT_L21_HW_VDEC_UFO_EXT_C		MTK_M4U_ID(SMI_L21_ID, 10)
-+
-+/* LARB 23 -- VDEC-SOC */
-+#define M4U_PORT_L23_HW_VDEC_LAT0_VLD_EXT	MTK_M4U_ID(SMI_L23_ID, 0)
-+#define M4U_PORT_L23_HW_VDEC_LAT0_VLD2_EXT	MTK_M4U_ID(SMI_L23_ID, 1)
-+#define M4U_PORT_L23_HW_VDEC_LAT0_AVC_MV_EXT	MTK_M4U_ID(SMI_L23_ID, 2)
-+#define M4U_PORT_L23_HW_VDEC_LAT0_PRED_RD_EXT	MTK_M4U_ID(SMI_L23_ID, 3)
-+#define M4U_PORT_L23_HW_VDEC_LAT0_TILE_EXT	MTK_M4U_ID(SMI_L23_ID, 4)
-+#define M4U_PORT_L23_HW_VDEC_LAT0_WDMA_EXT	MTK_M4U_ID(SMI_L23_ID, 5)
-+#define M4U_PORT_L23_HW_VDEC_UFO_ENC_EXT	MTK_M4U_ID(SMI_L23_ID, 6)
-+#define M4U_PORT_L23_HW_VDEC_UFO_ENC_EXT_C	MTK_M4U_ID(SMI_L23_ID, 7)
-+#define M4U_PORT_L23_HW_VDEC_MC_EXT_C		MTK_M4U_ID(SMI_L23_ID, 8)
-+
-+/* LARB 27 -- CCU */
-+#define M4U_PORT_L27_CCUI			MTK_M4U_ID(SMI_L27_ID, 0)
-+#define M4U_PORT_L27_CCUO			MTK_M4U_ID(SMI_L27_ID, 1)
-+#define M4U_PORT_L27_CCUI2			MTK_M4U_ID(SMI_L27_ID, 2)
-+#define M4U_PORT_L27_CCUO2			MTK_M4U_ID(SMI_L27_ID, 3)
-+
-+/* LARB 28 -- AXI-CCU */
-+#define M4U_PORT_L28_CCU_AXI_0			MTK_M4U_ID(SMI_L28_ID, 0)
-+
-+/* infra/peri */
-+#define IFR_IOMMU_PORT_PCIE_0			MTK_IFAIOMMU_PERI_ID(0)
-+
-+#endif
-diff --git a/include/dt-bindings/memory/mediatek,mt8189-memory-port.h b/include/dt-bindings/memory/mediatek,mt8189-memory-port.h
-new file mode 100644
-index 000000000000..849fead3d0f7
--- /dev/null
-+++ b/include/dt-bindings/memory/mediatek,mt8189-memory-port.h
-@@ -0,0 +1,283 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2025 MediaTek Inc.
-+ * Author: Zhengnan chen <zhengnan.chen@mediatek.com>
-+ */
-+#ifndef _DT_BINDINGS_MEMORY_MEDIATEK_MT8189_MEMORY_PORT_H_
-+#define _DT_BINDINGS_MEMORY_MEDIATEK_MT8189_MEMORY_PORT_H_
-+
-+#include <dt-bindings/memory/mtk-memory-port.h>
-+
-+#define SMI_L0_ID		(0)
-+#define SMI_L1_ID		(1)
-+#define SMI_L2_ID		(2)
-+#define SMI_L4_ID		(3)
-+#define SMI_L7_ID		(4)
-+#define SMI_L9_ID		(5)
-+#define SMI_L11_ID		(6)
-+#define SMI_L13_ID		(7)
-+#define SMI_L14_ID		(8)
-+#define SMI_L16_ID		(9)
-+#define SMI_L17_ID		(10)
-+#define SMI_L19_ID		(11)
-+#define SMI_L20_ID		(12)
-+
-+/*
-+ * MM IOMMU supports 16GB dma address. We separate it to four ranges:
-+ * 0 ~ 4G; 4G ~ 8G; 8G ~ 12G; 12G ~ 16G, we could adjust these masters
-+ * locate in anyone region. BUT:
-+ * a) Make sure all the ports inside a larb are in one range.
-+ * b) The iova of any master can NOT cross the 4G/8G/12G boundary.
-+ *
-+ * This is the suggested mapping in this SoC:
-+ *
-+ * modules		dma-address-region	larbs-ports
-+ * disp/mdp		0 ~ 4G			larb0/1/2
-+ * vcodec		4G ~ 8G                 larb4/7
-+ * imgsys/cam/ipesys	8G ~ 12G                the other larbs.
-+ * N/A			12G ~ 16G
-+ */
-+
-+/* Larb0 -- disp */
-+#define M4U_L0_P0_DISP_OVL0_4L_HDR		MTK_M4U_ID(SMI_L0_ID, 0)
-+#define M4U_L0_P1_DISP_OVL0_4L_RDMA0		MTK_M4U_ID(SMI_L0_ID, 1)
-+#define M4U_L0_P2_DISP_OVL1_4L_RDMA1		MTK_M4U_ID(SMI_L0_ID, 2)
-+#define M4U_L0_P3_DISP_OVL0_4L_RDMA2		MTK_M4U_ID(SMI_L0_ID, 3)
-+#define M4U_L0_P4_DISP_OVL1_4L_RDMA3		MTK_M4U_ID(SMI_L0_ID, 4)
-+#define M4U_L0_P5_DISP_RDMA0			MTK_M4U_ID(SMI_L0_ID, 5)
-+#define M4U_L0_P6_DISP_WDMA0			MTK_M4U_ID(SMI_L0_ID, 6)
-+#define M4U_L0_P7_DISP_FAKE_ENG0		MTK_M4U_ID(SMI_L0_ID, 7)
-+
-+/* Larb1 -- disp */
-+#define M4U_L1_P0_DISP_OVL1_4L_HDR		MTK_M4U_ID(SMI_L1_ID, 0)
-+#define M4U_L1_P1_DISP_OVL1_4L_RDMA0		MTK_M4U_ID(SMI_L1_ID, 1)
-+#define M4U_L1_P2_DISP_OVL0_4L_RDMA1		MTK_M4U_ID(SMI_L1_ID, 2)
-+#define M4U_L1_P3_DISP_OVL1_4L_RDMA2		MTK_M4U_ID(SMI_L1_ID, 3)
-+#define M4U_L1_P4_DISP_OVL0_4L_RDMA3		MTK_M4U_ID(SMI_L1_ID, 4)
-+#define M4U_L1_P5_DISP_RDMA1			MTK_M4U_ID(SMI_L1_ID, 5)
-+#define M4U_L1_P6_DISP_WDMA1			MTK_M4U_ID(SMI_L1_ID, 6)
-+#define M4U_L1_P7_DISP_FAKE_ENG1		MTK_M4U_ID(SMI_L1_ID, 7)
-+
-+/* Larb2 -- mmlsys(mdp) */
-+#define M4U_L2_P0_MDP_RDMA0			MTK_M4U_ID(SMI_L2_ID, 0)
-+#define M4U_L2_P1_MDP_RDMA1			MTK_M4U_ID(SMI_L2_ID, 1)
-+#define M4U_L2_P2_MDP_WROT0			MTK_M4U_ID(SMI_L2_ID, 2)
-+#define M4U_L2_P3_MDP_WROT1			MTK_M4U_ID(SMI_L2_ID, 3)
-+#define M4U_L2_P4_MDP_DUMMY0			MTK_M4U_ID(SMI_L2_ID, 4)
-+#define M4U_L2_P5_MDP_DUMMY1			MTK_M4U_ID(SMI_L2_ID, 5)
-+#define M4U_L2_P6_MDP_RDMA2			MTK_M4U_ID(SMI_L2_ID, 6)
-+#define M4U_L2_P7_MDP_RDMA3			MTK_M4U_ID(SMI_L2_ID, 7)
-+#define M4U_L2_P8_MDP_WROT2			MTK_M4U_ID(SMI_L2_ID, 8)
-+#define M4U_L2_P9_MDP_WROT3			MTK_M4U_ID(SMI_L2_ID, 9)
-+#define M4U_L2_P10_DISP_FAKE0			MTK_M4U_ID(SMI_L2_ID, 10)
-+
-+/* Larb3: null */
-+
-+/* Larb4 -- vdec */
-+#define M4U_L4_P0_HW_VDEC_MC_EXT		MTK_M4U_ID(SMI_L4_ID, 0)
-+#define M4U_L4_P1_HW_VDEC_UFO_EXT		MTK_M4U_ID(SMI_L4_ID, 1)
-+#define M4U_L4_P2_HW_VDEC_PP_EXT		MTK_M4U_ID(SMI_L4_ID, 2)
-+#define M4U_L4_P3_HW_VDEC_PRED_RD_EXT		MTK_M4U_ID(SMI_L4_ID, 3)
-+#define M4U_L4_P4_HW_VDEC_PRED_WR_EXT		MTK_M4U_ID(SMI_L4_ID, 4)
-+#define M4U_L4_P5_HW_VDEC_PPWRAP_EXT		MTK_M4U_ID(SMI_L4_ID, 5)
-+#define M4U_L4_P6_HW_VDEC_TILE_EXT		MTK_M4U_ID(SMI_L4_ID, 6)
-+#define M4U_L4_P7_HW_VDEC_VLD_EXT		MTK_M4U_ID(SMI_L4_ID, 7)
-+#define M4U_L4_P8_HW_VDEC_VLD2_EXT		MTK_M4U_ID(SMI_L4_ID, 8)
-+#define M4U_L4_P9_HW_VDEC_AVC_MV_EXT		MTK_M4U_ID(SMI_L4_ID, 9)
-+#define M4U_L4_P10_HW_VDEC_RG_CTRL_DMA_EXT	MTK_M4U_ID(SMI_L4_ID, 10)
-+#define M4U_L4_P11_HW_VDEC_UFO_ENC_EXT		MTK_M4U_ID(SMI_L4_ID, 11)
-+
-+/* Larb5: null */
-+
-+/* Larb6: null */
-+
-+/* Larb7 -- venc */
-+#define M4U_L7_P0_VENC_RCPU			MTK_M4U_ID(SMI_L7_ID, 0)
-+#define M4U_L7_P1_VENC_REC			MTK_M4U_ID(SMI_L7_ID, 1)
-+#define M4U_L7_P2_VENC_BSDMA			MTK_M4U_ID(SMI_L7_ID, 2)
-+#define M4U_L7_P3_VENC_SV_COMV			MTK_M4U_ID(SMI_L7_ID, 3)
-+#define M4U_L7_P4_VENC_RD_COMV			MTK_M4U_ID(SMI_L7_ID, 4)
-+#define M4U_L7_P5_JPGENC_Y_RDMA			MTK_M4U_ID(SMI_L7_ID, 5)
-+#define M4U_L7_P6_JPGENC_C_RDMA			MTK_M4U_ID(SMI_L7_ID, 6)
-+#define M4U_L7_P7_JPGENC_Q_RDMA			MTK_M4U_ID(SMI_L7_ID, 7)
-+#define M4U_L7_P8_VENC_SUB_W_LUMA		MTK_M4U_ID(SMI_L7_ID, 8)
-+#define M4U_L7_P9_JPGENC_BSDMA			MTK_M4U_ID(SMI_L7_ID, 9)
-+#define M4U_L7_P10_VENC_CUR_LUMA		MTK_M4U_ID(SMI_L7_ID, 10)
-+#define M4U_L7_P11_VENC_CUR_CHROMA		MTK_M4U_ID(SMI_L7_ID, 11)
-+#define M4U_L7_P12_VENC_REF_LUMA		MTK_M4U_ID(SMI_L7_ID, 12)
-+#define M4U_L7_P13_VENC_REF_CHROMA		MTK_M4U_ID(SMI_L7_ID, 13)
-+#define M4U_L7_P14_VENC_SUB_R_LUMA		MTK_M4U_ID(SMI_L7_ID, 14)
-+#define M4U_L7_P15_JPGDEC_WDMA			MTK_M4U_ID(SMI_L7_ID, 15)
-+#define M4U_L7_P16_JPGDEC_BSDMA			MTK_M4U_ID(SMI_L7_ID, 16)
-+#define M4U_L7_P17_JPGDEC_HUFF_OFFSET		MTK_M4U_ID(SMI_L7_ID, 17)
-+
-+/* Larb8: null */
-+
-+/* Larb9 --imgsys */
-+#define M4U_L9_P0_IMGI_D1			MTK_M4U_ID(SMI_L9_ID, 0)
-+#define M4U_L9_P1_IMGBI_D1			MTK_M4U_ID(SMI_L9_ID, 1)
-+#define M4U_L9_P2_DMGI_D1			MTK_M4U_ID(SMI_L9_ID, 2)
-+#define M4U_L9_P3_DEPI_D1			MTK_M4U_ID(SMI_L9_ID, 3)
-+#define M4U_L9_P4_LCE_D1			MTK_M4U_ID(SMI_L9_ID, 4)
-+#define M4U_L9_P5_SMTI_D1			MTK_M4U_ID(SMI_L9_ID, 5)
-+#define M4U_L9_P6_SMTO_D2			MTK_M4U_ID(SMI_L9_ID, 6)
-+#define M4U_L9_P7_SMTO_D1			MTK_M4U_ID(SMI_L9_ID, 7)
-+#define M4U_L9_P8_CRZO_D1			MTK_M4U_ID(SMI_L9_ID, 8)
-+#define M4U_L9_P9_IMG3O_D1			MTK_M4U_ID(SMI_L9_ID, 9)
-+#define M4U_L9_P10_VIPI_D1			MTK_M4U_ID(SMI_L9_ID, 10)
-+#define M4U_L9_P11_SMTI_D5			MTK_M4U_ID(SMI_L9_ID, 11)
-+#define M4U_L9_P12_TIMGO_D1			MTK_M4U_ID(SMI_L9_ID, 12)
-+#define M4U_L9_P13_UFBC_W0			MTK_M4U_ID(SMI_L9_ID, 13)
-+#define M4U_L9_P14_UFBC_R0			MTK_M4U_ID(SMI_L9_ID, 14)
-+#define M4U_L9_P15_WPE_RDMA1			MTK_M4U_ID(SMI_L9_ID, 15)
-+#define M4U_L9_P16_WPE_RDMA0			MTK_M4U_ID(SMI_L9_ID, 16)
-+#define M4U_L9_P17_WPE_WDMA			MTK_M4U_ID(SMI_L9_ID, 17)
-+#define M4U_L9_P18_MFB_RDMA0			MTK_M4U_ID(SMI_L9_ID, 18)
-+#define M4U_L9_P19_MFB_RDMA1			MTK_M4U_ID(SMI_L9_ID, 19)
-+#define M4U_L9_P20_MFB_RDMA2			MTK_M4U_ID(SMI_L9_ID, 20)
-+#define M4U_L9_P21_MFB_RDMA3			MTK_M4U_ID(SMI_L9_ID, 21)
-+#define M4U_L9_P22_MFB_RDMA4			MTK_M4U_ID(SMI_L9_ID, 22)
-+#define M4U_L9_P23_MFB_RDMA5			MTK_M4U_ID(SMI_L9_ID, 23)
-+#define M4U_L9_P24_MFB_WDMA0			MTK_M4U_ID(SMI_L9_ID, 24)
-+#define M4U_L9_P25_MFB_WDMA1			MTK_M4U_ID(SMI_L9_ID, 25)
-+#define M4U_L9_P26_RESERVE6			MTK_M4U_ID(SMI_L9_ID, 26)
-+#define M4U_L9_P27_RESERVE7			MTK_M4U_ID(SMI_L9_ID, 27)
-+#define M4U_L9_P28_RESERVE8			MTK_M4U_ID(SMI_L9_ID, 28)
-+
-+/* Larb10: null */
-+
-+/* Larb11 -- imgsys */
-+#define M4U_L11_P0_IMGI_D1			MTK_M4U_ID(SMI_L11_ID, 0)
-+#define M4U_L11_P1_IMGBI_D1			MTK_M4U_ID(SMI_L11_ID, 1)
-+#define M4U_L11_P2_DMGI_D1			MTK_M4U_ID(SMI_L11_ID, 2)
-+#define M4U_L11_P3_DEPI_D1			MTK_M4U_ID(SMI_L11_ID, 3)
-+#define M4U_L11_P4_LCE_D1			MTK_M4U_ID(SMI_L11_ID, 4)
-+#define M4U_L11_P5_SMTI_D1			MTK_M4U_ID(SMI_L11_ID, 5)
-+#define M4U_L11_P6_SMTO_D2			MTK_M4U_ID(SMI_L11_ID, 6)
-+#define M4U_L11_P7_SMTO_D1			MTK_M4U_ID(SMI_L11_ID, 7)
-+#define M4U_L11_P8_CRZO_D1			MTK_M4U_ID(SMI_L11_ID, 8)
-+#define M4U_L11_P9_IMG3O_D1			MTK_M4U_ID(SMI_L11_ID, 9)
-+#define M4U_L11_P10_VIPI_D1			MTK_M4U_ID(SMI_L11_ID, 10)
-+#define M4U_L11_P11_SMTI_D5			MTK_M4U_ID(SMI_L11_ID, 11)
-+#define M4U_L11_P12_TIMGO_D1			MTK_M4U_ID(SMI_L11_ID, 12)
-+#define M4U_L11_P13_UFBC_W0			MTK_M4U_ID(SMI_L11_ID, 13)
-+#define M4U_L11_P14_UFBC_R0			MTK_M4U_ID(SMI_L11_ID, 14)
-+#define M4U_L11_P15_WPE_RDMA1			MTK_M4U_ID(SMI_L11_ID, 15)
-+#define M4U_L11_P16_WPE_RDMA0			MTK_M4U_ID(SMI_L11_ID, 16)
-+#define M4U_L11_P17_WPE_WDMA			MTK_M4U_ID(SMI_L11_ID, 17)
-+#define M4U_L11_P18_MFB_RDMA0			MTK_M4U_ID(SMI_L11_ID, 18)
-+#define M4U_L11_P19_MFB_RDMA1			MTK_M4U_ID(SMI_L11_ID, 19)
-+#define M4U_L11_P20_MFB_RDMA2			MTK_M4U_ID(SMI_L11_ID, 20)
-+#define M4U_L11_P21_MFB_RDMA3			MTK_M4U_ID(SMI_L11_ID, 21)
-+#define M4U_L11_P22_MFB_RDMA4			MTK_M4U_ID(SMI_L11_ID, 22)
-+#define M4U_L11_P23_MFB_RDMA5			MTK_M4U_ID(SMI_L11_ID, 23)
-+#define M4U_L11_P24_MFB_WDMA0			MTK_M4U_ID(SMI_L11_ID, 24)
-+#define M4U_L11_P25_MFB_WDMA1			MTK_M4U_ID(SMI_L11_ID, 25)
-+#define M4U_L11_P26_RESERVE6			MTK_M4U_ID(SMI_L11_ID, 26)
-+#define M4U_L11_P27_RESERVE7			MTK_M4U_ID(SMI_L11_ID, 27)
-+#define M4U_L11_P28_RESERVE8			MTK_M4U_ID(SMI_L11_ID, 28)
-+
-+/* Larb12: null */
-+
-+/* Larb13 -- cam */
-+#define M4U_L13_P0_MRAWI			MTK_M4U_ID(SMI_L13_ID, 0)
-+#define M4U_L13_P1_MRAWO_0			MTK_M4U_ID(SMI_L13_ID, 1)
-+#define M4U_L13_P2_MRAWO_1			MTK_M4U_ID(SMI_L13_ID, 2)
-+#define M4U_L13_P3_CAMSV_1			MTK_M4U_ID(SMI_L13_ID, 3)
-+#define M4U_L13_P4_CAMSV_2			MTK_M4U_ID(SMI_L13_ID, 4)
-+#define M4U_L13_P5_CAMSV_3			MTK_M4U_ID(SMI_L13_ID, 5)
-+#define M4U_L13_P6_CAMSV_4			MTK_M4U_ID(SMI_L13_ID, 6)
-+#define M4U_L13_P7_CAMSV_5			MTK_M4U_ID(SMI_L13_ID, 7)
-+#define M4U_L13_P8_CAMSV_6			MTK_M4U_ID(SMI_L13_ID, 8)
-+#define M4U_L13_P9_CCUI				MTK_M4U_ID(SMI_L13_ID, 9)
-+#define M4U_L13_P10_CCUO			MTK_M4U_ID(SMI_L13_ID, 10)
-+#define M4U_L13_P11_FAKE			MTK_M4U_ID(SMI_L13_ID, 11)
-+#define M4U_L13_P12_PDAI_0			MTK_M4U_ID(SMI_L13_ID, 12)
-+#define M4U_L13_P13_PDAI_1			MTK_M4U_ID(SMI_L13_ID, 13)
-+#define M4U_L13_P14_PDAO			MTK_M4U_ID(SMI_L13_ID, 14)
-+
-+/* Larb14 -- cam */
-+#define M4U_L14_P0_RESERVE			MTK_M4U_ID(SMI_L14_ID, 0)
-+#define M4U_L14_P1_RESERVE			MTK_M4U_ID(SMI_L14_ID, 1)
-+#define M4U_L14_P2_RESERVE			MTK_M4U_ID(SMI_L14_ID, 2)
-+#define M4U_L14_P3_CAMSV_0			MTK_M4U_ID(SMI_L14_ID, 3)
-+#define M4U_L14_P4_CCUI				MTK_M4U_ID(SMI_L14_ID, 4)
-+#define M4U_L14_P5_CCUO				MTK_M4U_ID(SMI_L14_ID, 5)
-+#define M4U_L14_P6_CAMSV_7			MTK_M4U_ID(SMI_L14_ID, 6)
-+#define M4U_L14_P7_CAMSV_8			MTK_M4U_ID(SMI_L14_ID, 7)
-+#define M4U_L14_P8_CAMSV_9			MTK_M4U_ID(SMI_L14_ID, 8)
-+#define M4U_L14_P9_CAMSV_10			MTK_M4U_ID(SMI_L14_ID, 9)
-+
-+/* Larb15: null */
-+
-+/* Larb16 -- cam */
-+#define M4U_L16_P0_IMGO_R1_A			MTK_M4U_ID(SMI_L16_ID, 0)
-+#define M4U_L16_P1_RRZO_R1_A			MTK_M4U_ID(SMI_L16_ID, 1)
-+#define M4U_L16_P2_CQI_R1_A			MTK_M4U_ID(SMI_L16_ID, 2)
-+#define M4U_L16_P3_BPCI_R1_A			MTK_M4U_ID(SMI_L16_ID, 3)
-+#define M4U_L16_P4_YUVO_R1_A			MTK_M4U_ID(SMI_L16_ID, 4)
-+#define M4U_L16_P5_UFDI_R2_A			MTK_M4U_ID(SMI_L16_ID, 5)
-+#define M4U_L16_P6_RAWI_R2_A			MTK_M4U_ID(SMI_L16_ID, 6)
-+#define M4U_L16_P7_RAWI_R3_A			MTK_M4U_ID(SMI_L16_ID, 7)
-+#define M4U_L16_P8_AAO_R1_A			MTK_M4U_ID(SMI_L16_ID, 8)
-+#define M4U_L16_P9_AFO_R1_A			MTK_M4U_ID(SMI_L16_ID, 9)
-+#define M4U_L16_P10_FLKO_R1_A			MTK_M4U_ID(SMI_L16_ID, 10)
-+#define M4U_L16_P11_LCESO_R1_A			MTK_M4U_ID(SMI_L16_ID, 11)
-+#define M4U_L16_P12_CRZO_R1_A			MTK_M4U_ID(SMI_L16_ID, 12)
-+#define M4U_L16_P13_LTMSO_R1_A			MTK_M4U_ID(SMI_L16_ID, 13)
-+#define M4U_L16_P14_RSSO_R1_A			MTK_M4U_ID(SMI_L16_ID, 14)
-+#define M4U_L16_P15_AAHO_R1_A			MTK_M4U_ID(SMI_L16_ID, 15)
-+#define M4U_L16_P16_LSCI_R1_A			MTK_M4U_ID(SMI_L16_ID, 16)
-+
-+/* Larb17 -- cam */
-+#define M4U_L17_P0_IMGO_R1_B			MTK_M4U_ID(SMI_L17_ID, 0)
-+#define M4U_L17_P1_RRZO_R1_B			MTK_M4U_ID(SMI_L17_ID, 1)
-+#define M4U_L17_P2_CQI_R1_B			MTK_M4U_ID(SMI_L17_ID, 2)
-+#define M4U_L17_P3_BPCI_R1_B			MTK_M4U_ID(SMI_L17_ID, 3)
-+#define M4U_L17_P4_YUVO_R1_B			MTK_M4U_ID(SMI_L17_ID, 4)
-+#define M4U_L17_P5_UFDI_R2_B			MTK_M4U_ID(SMI_L17_ID, 5)
-+#define M4U_L17_P6_RAWI_R2_B			MTK_M4U_ID(SMI_L17_ID, 6)
-+#define M4U_L17_P7_RAWI_R3_B			MTK_M4U_ID(SMI_L17_ID, 7)
-+#define M4U_L17_P8_AAO_R1_B			MTK_M4U_ID(SMI_L17_ID, 8)
-+#define M4U_L17_P9_AFO_R1_B			MTK_M4U_ID(SMI_L17_ID, 9)
-+#define M4U_L17_P10_FLKO_R1_B			MTK_M4U_ID(SMI_L17_ID, 10)
-+#define M4U_L17_P11_LCESO_R1_B			MTK_M4U_ID(SMI_L17_ID, 11)
-+#define M4U_L17_P12_CRZO_R1_B			MTK_M4U_ID(SMI_L17_ID, 12)
-+#define M4U_L17_P13_LTMSO_R1_B			MTK_M4U_ID(SMI_L17_ID, 13)
-+#define M4U_L17_P14_RSSO_R1_B			MTK_M4U_ID(SMI_L17_ID, 14)
-+#define M4U_L17_P15_AAHO_R1_B			MTK_M4U_ID(SMI_L17_ID, 15)
-+#define M4U_L17_P16_LSCI_R1_B			MTK_M4U_ID(SMI_L17_ID, 16)
-+
-+/* Larb19 -- ipesys */
-+#define M4U_L19_P0_DVS_RDMA			MTK_M4U_ID(SMI_L19_ID, 0)
-+#define M4U_L19_P1_DVS_WDMA			MTK_M4U_ID(SMI_L19_ID, 1)
-+#define M4U_L19_P2_DVP_RDMA			MTK_M4U_ID(SMI_L19_ID, 2)
-+#define M4U_L19_P3_DVP_WDMA			MTK_M4U_ID(SMI_L19_ID, 3)
-+
-+/* Larb20 -- ipesys */
-+#define M4U_L20_P0_FDVT_RDA_0			MTK_M4U_ID(SMI_L20_ID, 0)
-+#define M4U_L20_P1_FDVT_RDB_0			MTK_M4U_ID(SMI_L20_ID, 1)
-+#define M4U_L20_P2_FDVT_WRA_0			MTK_M4U_ID(SMI_L20_ID, 2)
-+#define M4U_L20_P3_FDVT_WRB_0			MTK_M4U_ID(SMI_L20_ID, 3)
-+#define M4U_L20_P4_RSC_RDMA			MTK_M4U_ID(SMI_L20_ID, 4)
-+#define M4U_L20_P5_RSC_WDMA			MTK_M4U_ID(SMI_L20_ID, 5)
-+
-+/* fake larb21 for gce */
-+#define M4U_L21_GCE_DM				MTK_M4U_ID(21, 0)
-+#define M4U_L21_GCE_MM				MTK_M4U_ID(21, 1)
-+
-+/* fake larb & port for svp and dual svp and wfd */
-+#define M4U_PORT_SVP_HEAP			MTK_M4U_ID(22, 0)
-+#define M4U_PORT_DUAL_SVP_HEAP			MTK_M4U_ID(22, 1)
-+#define M4U_PORT_WFD_HEAP			MTK_M4U_ID(22, 2)
-+
-+/* fake larb0 for apu */
-+#define M4U_L0_APU_DATA				MTK_M4U_ID(0, 0)
-+#define M4U_L0_APU_CODE				MTK_M4U_ID(0, 1)
-+#define M4U_L0_APU_SECURE			MTK_M4U_ID(0, 2)
-+#define M4U_L0_APU_VLM				MTK_M4U_ID(0, 3)
-+
-+/* infra/peri */
-+#define IFR_IOMMU_PORT_PCIE_0			MTK_IFAIOMMU_PERI_ID(0, 26)
-+
-+#endif
-diff --git a/include/dt-bindings/memory/mediatek,mt8365-larb-port.h b/include/dt-bindings/memory/mediatek,mt8365-larb-port.h
-new file mode 100644
-index 000000000000..56d5a5dd519e
--- /dev/null
-+++ b/include/dt-bindings/memory/mediatek,mt8365-larb-port.h
-@@ -0,0 +1,90 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022 MediaTek Inc.
-+ * Author: Yong Wu <yong.wu@mediatek.com>
-+ */
-+#ifndef _DT_BINDINGS_MEMORY_MT8365_LARB_PORT_H_
-+#define _DT_BINDINGS_MEMORY_MT8365_LARB_PORT_H_
-+
-+#include <dt-bindings/memory/mtk-memory-port.h>
-+
-+#define M4U_LARB0_ID			0
-+#define M4U_LARB1_ID			1
-+#define M4U_LARB2_ID			2
-+#define M4U_LARB3_ID			3
-+
-+/* larb0 */
-+#define M4U_PORT_DISP_OVL0		MTK_M4U_ID(M4U_LARB0_ID, 0)
-+#define M4U_PORT_DISP_OVL0_2L		MTK_M4U_ID(M4U_LARB0_ID, 1)
-+#define M4U_PORT_DISP_RDMA0		MTK_M4U_ID(M4U_LARB0_ID, 2)
-+#define M4U_PORT_DISP_WDMA0		MTK_M4U_ID(M4U_LARB0_ID, 3)
-+#define M4U_PORT_DISP_RDMA1		MTK_M4U_ID(M4U_LARB0_ID, 4)
-+#define M4U_PORT_MDP_RDMA0		MTK_M4U_ID(M4U_LARB0_ID, 5)
-+#define M4U_PORT_MDP_WROT1		MTK_M4U_ID(M4U_LARB0_ID, 6)
-+#define M4U_PORT_MDP_WROT0		MTK_M4U_ID(M4U_LARB0_ID, 7)
-+#define M4U_PORT_MDP_RDMA1		MTK_M4U_ID(M4U_LARB0_ID, 8)
-+#define M4U_PORT_DISP_FAKE0		MTK_M4U_ID(M4U_LARB0_ID, 9)
-+#define M4U_PORT_APU_READ		MTK_M4U_ID(M4U_LARB0_ID, 10)
-+#define M4U_PORT_APU_WRITE		MTK_M4U_ID(M4U_LARB0_ID, 11)
-+
-+/* larb1 */
-+#define M4U_PORT_VENC_RCPU		MTK_M4U_ID(M4U_LARB1_ID, 0)
-+#define M4U_PORT_VENC_REC		MTK_M4U_ID(M4U_LARB1_ID, 1)
-+#define M4U_PORT_VENC_BSDMA		MTK_M4U_ID(M4U_LARB1_ID, 2)
-+#define M4U_PORT_VENC_SV_COMV		MTK_M4U_ID(M4U_LARB1_ID, 3)
-+#define M4U_PORT_VENC_RD_COMV		MTK_M4U_ID(M4U_LARB1_ID, 4)
-+#define M4U_PORT_VENC_NBM_RDMA		MTK_M4U_ID(M4U_LARB1_ID, 5)
-+#define M4U_PORT_VENC_NBM_RDMA_LITE	MTK_M4U_ID(M4U_LARB1_ID, 6)
-+#define M4U_PORT_JPGENC_Y_RDMA		MTK_M4U_ID(M4U_LARB1_ID, 7)
-+#define M4U_PORT_JPGENC_C_RDMA		MTK_M4U_ID(M4U_LARB1_ID, 8)
-+#define M4U_PORT_JPGENC_Q_TABLE		MTK_M4U_ID(M4U_LARB1_ID, 9)
-+#define M4U_PORT_JPGENC_BSDMA		MTK_M4U_ID(M4U_LARB1_ID, 10)
-+#define M4U_PORT_JPGDEC_WDMA		MTK_M4U_ID(M4U_LARB1_ID, 11)
-+#define M4U_PORT_JPGDEC_BSDMA		MTK_M4U_ID(M4U_LARB1_ID, 12)
-+#define M4U_PORT_VENC_NBM_WDMA		MTK_M4U_ID(M4U_LARB1_ID, 13)
-+#define M4U_PORT_VENC_NBM_WDMA_LITE	MTK_M4U_ID(M4U_LARB1_ID, 14)
-+#define M4U_PORT_VENC_CUR_LUMA		MTK_M4U_ID(M4U_LARB1_ID, 15)
-+#define M4U_PORT_VENC_CUR_CHROMA	MTK_M4U_ID(M4U_LARB1_ID, 16)
-+#define M4U_PORT_VENC_REF_LUMA		MTK_M4U_ID(M4U_LARB1_ID, 17)
-+#define M4U_PORT_VENC_REF_CHROMA	MTK_M4U_ID(M4U_LARB1_ID, 18)
-+
-+/* larb2 */
-+#define M4U_PORT_CAM_IMGO		MTK_M4U_ID(M4U_LARB2_ID, 0)
-+#define M4U_PORT_CAM_RRZO		MTK_M4U_ID(M4U_LARB2_ID, 1)
-+#define M4U_PORT_CAM_AAO		MTK_M4U_ID(M4U_LARB2_ID, 2)
-+#define M4U_PORT_CAM_LCS		MTK_M4U_ID(M4U_LARB2_ID, 3)
-+#define M4U_PORT_CAM_ESFKO		MTK_M4U_ID(M4U_LARB2_ID, 4)
-+#define M4U_PORT_CAM_CAM_SV0		MTK_M4U_ID(M4U_LARB2_ID, 5)
-+#define M4U_PORT_CAM_CAM_SV1		MTK_M4U_ID(M4U_LARB2_ID, 6)
-+#define M4U_PORT_CAM_LSCI		MTK_M4U_ID(M4U_LARB2_ID, 7)
-+#define M4U_PORT_CAM_LSCI_D		MTK_M4U_ID(M4U_LARB2_ID, 8)
-+#define M4U_PORT_CAM_AFO		MTK_M4U_ID(M4U_LARB2_ID, 9)
-+#define M4U_PORT_CAM_SPARE		MTK_M4U_ID(M4U_LARB2_ID, 10)
-+#define M4U_PORT_CAM_BPCI		MTK_M4U_ID(M4U_LARB2_ID, 11)
-+#define M4U_PORT_CAM_BPCI_D		MTK_M4U_ID(M4U_LARB2_ID, 12)
-+#define M4U_PORT_CAM_UFDI		MTK_M4U_ID(M4U_LARB2_ID, 13)
-+#define M4U_PORT_CAM_IMGI		MTK_M4U_ID(M4U_LARB2_ID, 14)
-+#define M4U_PORT_CAM_IMG2O		MTK_M4U_ID(M4U_LARB2_ID, 15)
-+#define M4U_PORT_CAM_IMG3O		MTK_M4U_ID(M4U_LARB2_ID, 16)
-+#define M4U_PORT_CAM_WPE0_I		MTK_M4U_ID(M4U_LARB2_ID, 17)
-+#define M4U_PORT_CAM_WPE1_I		MTK_M4U_ID(M4U_LARB2_ID, 18)
-+#define M4U_PORT_CAM_WPE_O		MTK_M4U_ID(M4U_LARB2_ID, 19)
-+#define M4U_PORT_CAM_FD0_I		MTK_M4U_ID(M4U_LARB2_ID, 20)
-+#define M4U_PORT_CAM_FD1_I		MTK_M4U_ID(M4U_LARB2_ID, 21)
-+#define M4U_PORT_CAM_FD0_O		MTK_M4U_ID(M4U_LARB2_ID, 22)
-+#define M4U_PORT_CAM_FD1_O		MTK_M4U_ID(M4U_LARB2_ID, 23)
-+
-+/* larb3 */
-+#define M4U_PORT_HW_VDEC_MC_EXT		MTK_M4U_ID(M4U_LARB3_ID, 0)
-+#define M4U_PORT_HW_VDEC_UFO_EXT	MTK_M4U_ID(M4U_LARB3_ID, 1)
-+#define M4U_PORT_HW_VDEC_PP_EXT		MTK_M4U_ID(M4U_LARB3_ID, 2)
-+#define M4U_PORT_HW_VDEC_PRED_RD_EXT	MTK_M4U_ID(M4U_LARB3_ID, 3)
-+#define M4U_PORT_HW_VDEC_PRED_WR_EXT	MTK_M4U_ID(M4U_LARB3_ID, 4)
-+#define M4U_PORT_HW_VDEC_PPWRAP_EXT	MTK_M4U_ID(M4U_LARB3_ID, 5)
-+#define M4U_PORT_HW_VDEC_TILE_EXT	MTK_M4U_ID(M4U_LARB3_ID, 6)
-+#define M4U_PORT_HW_VDEC_VLD_EXT	MTK_M4U_ID(M4U_LARB3_ID, 7)
-+#define M4U_PORT_HW_VDEC_VLD2_EXT	MTK_M4U_ID(M4U_LARB3_ID, 8)
-+#define M4U_PORT_HW_VDEC_AVC_MV_EXT	MTK_M4U_ID(M4U_LARB3_ID, 9)
-+#define M4U_PORT_HW_VDEC_RG_CTRL_DMA_EXT MTK_M4U_ID(M4U_LARB3_ID, 10)
-+
-+#endif
-diff --git a/include/dt-bindings/memory/mt6795-larb-port.h b/include/dt-bindings/memory/mt6795-larb-port.h
-new file mode 100644
-index 000000000000..58cf6a6b6372
--- /dev/null
-+++ b/include/dt-bindings/memory/mt6795-larb-port.h
-@@ -0,0 +1,95 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022 Collabora Ltd.
-+ * Author: AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_MEMORY_MT6795_LARB_PORT_H_
-+#define _DT_BINDINGS_MEMORY_MT6795_LARB_PORT_H_
-+
-+#include <dt-bindings/memory/mtk-memory-port.h>
-+
-+#define M4U_LARB0_ID			0
-+#define M4U_LARB1_ID			1
-+#define M4U_LARB2_ID			2
-+#define M4U_LARB3_ID			3
-+#define M4U_LARB4_ID			4
-+
-+/* larb0 */
-+#define M4U_PORT_DISP_OVL0		MTK_M4U_ID(M4U_LARB0_ID, 0)
-+#define M4U_PORT_DISP_RDMA0		MTK_M4U_ID(M4U_LARB0_ID, 1)
-+#define M4U_PORT_DISP_RDMA1		MTK_M4U_ID(M4U_LARB0_ID, 2)
-+#define M4U_PORT_DISP_WDMA0		MTK_M4U_ID(M4U_LARB0_ID, 3)
-+#define M4U_PORT_DISP_OVL1		MTK_M4U_ID(M4U_LARB0_ID, 4)
-+#define M4U_PORT_DISP_RDMA2		MTK_M4U_ID(M4U_LARB0_ID, 5)
-+#define M4U_PORT_DISP_WDMA1		MTK_M4U_ID(M4U_LARB0_ID, 6)
-+#define M4U_PORT_DISP_OD_R		MTK_M4U_ID(M4U_LARB0_ID, 7)
-+#define M4U_PORT_DISP_OD_W		MTK_M4U_ID(M4U_LARB0_ID, 8)
-+#define M4U_PORT_MDP_RDMA0		MTK_M4U_ID(M4U_LARB0_ID, 9)
-+#define M4U_PORT_MDP_RDMA1		MTK_M4U_ID(M4U_LARB0_ID, 10)
-+#define M4U_PORT_MDP_WDMA		MTK_M4U_ID(M4U_LARB0_ID, 11)
-+#define M4U_PORT_MDP_WROT0		MTK_M4U_ID(M4U_LARB0_ID, 12)
-+#define M4U_PORT_MDP_WROT1		MTK_M4U_ID(M4U_LARB0_ID, 13)
-+
-+/* larb1 */
-+#define M4U_PORT_VDEC_MC		MTK_M4U_ID(M4U_LARB1_ID, 0)
-+#define M4U_PORT_VDEC_PP		MTK_M4U_ID(M4U_LARB1_ID, 1)
-+#define M4U_PORT_VDEC_UFO		MTK_M4U_ID(M4U_LARB1_ID, 2)
-+#define M4U_PORT_VDEC_VLD		MTK_M4U_ID(M4U_LARB1_ID, 3)
-+#define M4U_PORT_VDEC_VLD2		MTK_M4U_ID(M4U_LARB1_ID, 4)
-+#define M4U_PORT_VDEC_AVC_MV		MTK_M4U_ID(M4U_LARB1_ID, 5)
-+#define M4U_PORT_VDEC_PRED_RD		MTK_M4U_ID(M4U_LARB1_ID, 6)
-+#define M4U_PORT_VDEC_PRED_WR		MTK_M4U_ID(M4U_LARB1_ID, 7)
-+#define M4U_PORT_VDEC_PPWRAP		MTK_M4U_ID(M4U_LARB1_ID, 8)
-+
-+/* larb2 */
-+#define M4U_PORT_CAM_IMGO		MTK_M4U_ID(M4U_LARB2_ID, 0)
-+#define M4U_PORT_CAM_RRZO		MTK_M4U_ID(M4U_LARB2_ID, 1)
-+#define M4U_PORT_CAM_AAO		MTK_M4U_ID(M4U_LARB2_ID, 2)
-+#define M4U_PORT_CAM_LCSO		MTK_M4U_ID(M4U_LARB2_ID, 3)
-+#define M4U_PORT_CAM_ESFKO		MTK_M4U_ID(M4U_LARB2_ID, 4)
-+#define M4U_PORT_CAM_IMGO_S		MTK_M4U_ID(M4U_LARB2_ID, 5)
-+#define M4U_PORT_CAM_LSCI		MTK_M4U_ID(M4U_LARB2_ID, 6)
-+#define M4U_PORT_CAM_LSCI_D		MTK_M4U_ID(M4U_LARB2_ID, 7)
-+#define M4U_PORT_CAM_BPCI		MTK_M4U_ID(M4U_LARB2_ID, 8)
-+#define M4U_PORT_CAM_BPCI_D		MTK_M4U_ID(M4U_LARB2_ID, 9)
-+#define M4U_PORT_CAM_UFDI		MTK_M4U_ID(M4U_LARB2_ID, 10)
-+#define M4U_PORT_CAM_IMGI		MTK_M4U_ID(M4U_LARB2_ID, 11)
-+#define M4U_PORT_CAM_IMG2O		MTK_M4U_ID(M4U_LARB2_ID, 12)
-+#define M4U_PORT_CAM_IMG3O		MTK_M4U_ID(M4U_LARB2_ID, 13)
-+#define M4U_PORT_CAM_VIPI		MTK_M4U_ID(M4U_LARB2_ID, 14)
-+#define M4U_PORT_CAM_VIP2I		MTK_M4U_ID(M4U_LARB2_ID, 15)
-+#define M4U_PORT_CAM_VIP3I		MTK_M4U_ID(M4U_LARB2_ID, 16)
-+#define M4U_PORT_CAM_LCEI		MTK_M4U_ID(M4U_LARB2_ID, 17)
-+#define M4U_PORT_CAM_RB			MTK_M4U_ID(M4U_LARB2_ID, 18)
-+#define M4U_PORT_CAM_RP			MTK_M4U_ID(M4U_LARB2_ID, 19)
-+#define M4U_PORT_CAM_WR			MTK_M4U_ID(M4U_LARB2_ID, 20)
-+
-+/* larb3 */
-+#define M4U_PORT_VENC_RCPU		MTK_M4U_ID(M4U_LARB3_ID, 0)
-+#define M4U_PORT_VENC_REC		MTK_M4U_ID(M4U_LARB3_ID, 1)
-+#define M4U_PORT_VENC_BSDMA		MTK_M4U_ID(M4U_LARB3_ID, 2)
-+#define M4U_PORT_VENC_SV_COMV		MTK_M4U_ID(M4U_LARB3_ID, 3)
-+#define M4U_PORT_VENC_RD_COMV		MTK_M4U_ID(M4U_LARB3_ID, 4)
-+#define M4U_PORT_JPGENC_BSDMA		MTK_M4U_ID(M4U_LARB3_ID, 5)
-+#define M4U_PORT_REMDC_SDMA		MTK_M4U_ID(M4U_LARB3_ID, 6)
-+#define M4U_PORT_REMDC_BSDMA		MTK_M4U_ID(M4U_LARB3_ID, 7)
-+#define M4U_PORT_JPGENC_RDMA		MTK_M4U_ID(M4U_LARB3_ID, 8)
-+#define M4U_PORT_JPGENC_SDMA		MTK_M4U_ID(M4U_LARB3_ID, 9)
-+#define M4U_PORT_JPGDEC_WDMA		MTK_M4U_ID(M4U_LARB3_ID, 10)
-+#define M4U_PORT_JPGDEC_BSDMA		MTK_M4U_ID(M4U_LARB3_ID, 11)
-+#define M4U_PORT_VENC_CUR_LUMA		MTK_M4U_ID(M4U_LARB3_ID, 12)
-+#define M4U_PORT_VENC_CUR_CHROMA	MTK_M4U_ID(M4U_LARB3_ID, 13)
-+#define M4U_PORT_VENC_REF_LUMA		MTK_M4U_ID(M4U_LARB3_ID, 14)
-+#define M4U_PORT_VENC_REF_CHROMA	MTK_M4U_ID(M4U_LARB3_ID, 15)
-+#define M4U_PORT_REMDC_WDMA		MTK_M4U_ID(M4U_LARB3_ID, 16)
-+#define M4U_PORT_VENC_NBM_RDMA		MTK_M4U_ID(M4U_LARB3_ID, 17)
-+#define M4U_PORT_VENC_NBM_WDMA		MTK_M4U_ID(M4U_LARB3_ID, 18)
-+
-+/* larb4 */
-+#define M4U_PORT_MJC_MV_RD		MTK_M4U_ID(M4U_LARB4_ID, 0)
-+#define M4U_PORT_MJC_MV_WR		MTK_M4U_ID(M4U_LARB4_ID, 1)
-+#define M4U_PORT_MJC_DMA_RD		MTK_M4U_ID(M4U_LARB4_ID, 2)
-+#define M4U_PORT_MJC_DMA_WR		MTK_M4U_ID(M4U_LARB4_ID, 3)
-+
-+#endif
-diff --git a/include/dt-bindings/memory/nvidia,tegra264.h b/include/dt-bindings/memory/nvidia,tegra264.h
-new file mode 100644
-index 000000000000..521405c01f84
--- /dev/null
-+++ b/include/dt-bindings/memory/nvidia,tegra264.h
-@@ -0,0 +1,136 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/* Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. */
-+
-+#ifndef DT_BINDINGS_MEMORY_NVIDIA_TEGRA264_H
-+#define DT_BINDINGS_MEMORY_NVIDIA_TEGRA264_H
-+
-+#define TEGRA264_SID(x) ((x) << 8)
-+
-+/*
-+ * SMMU stream IDs
-+ */
-+
-+#define TEGRA264_SID_AON       TEGRA264_SID(0x01)
-+#define TEGRA264_SID_APE       TEGRA264_SID(0x02)
-+#define TEGRA264_SID_ETR       TEGRA264_SID(0x03)
-+#define TEGRA264_SID_BPMP      TEGRA264_SID(0x04)
-+#define TEGRA264_SID_DCE       TEGRA264_SID(0x05)
-+#define TEGRA264_SID_EQOS      TEGRA264_SID(0x06)
-+#define TEGRA264_SID_GPCDMA    TEGRA264_SID(0x08)
-+#define TEGRA264_SID_DISP      TEGRA264_SID(0x09)
-+#define TEGRA264_SID_HDA       TEGRA264_SID(0x0a)
-+#define TEGRA264_SID_HOST1X    TEGRA264_SID(0x0b)
-+#define TEGRA264_SID_ISP0      TEGRA264_SID(0x0c)
-+#define TEGRA264_SID_ISP1      TEGRA264_SID(0x0d)
-+#define TEGRA264_SID_PMA0      TEGRA264_SID(0x0e)
-+#define TEGRA264_SID_FSI0      TEGRA264_SID(0x0f)
-+#define TEGRA264_SID_FSI1      TEGRA264_SID(0x10)
-+#define TEGRA264_SID_PVA       TEGRA264_SID(0x11)
-+#define TEGRA264_SID_SDMMC0    TEGRA264_SID(0x12)
-+#define TEGRA264_SID_MGBE0     TEGRA264_SID(0x13)
-+#define TEGRA264_SID_MGBE1     TEGRA264_SID(0x14)
-+#define TEGRA264_SID_MGBE2     TEGRA264_SID(0x15)
-+#define TEGRA264_SID_MGBE3     TEGRA264_SID(0x16)
-+#define TEGRA264_SID_MSSSEQ    TEGRA264_SID(0x17)
-+#define TEGRA264_SID_SE        TEGRA264_SID(0x18)
-+#define TEGRA264_SID_SEU1      TEGRA264_SID(0x19)
-+#define TEGRA264_SID_SEU2      TEGRA264_SID(0x1a)
-+#define TEGRA264_SID_SEU3      TEGRA264_SID(0x1b)
-+#define TEGRA264_SID_PSC       TEGRA264_SID(0x1c)
-+#define TEGRA264_SID_OESP      TEGRA264_SID(0x23)
-+#define TEGRA264_SID_SB        TEGRA264_SID(0x24)
-+#define TEGRA264_SID_XSPI0     TEGRA264_SID(0x25)
-+#define TEGRA264_SID_TSEC      TEGRA264_SID(0x29)
-+#define TEGRA264_SID_UFS       TEGRA264_SID(0x2a)
-+#define TEGRA264_SID_RCE       TEGRA264_SID(0x2b)
-+#define TEGRA264_SID_RCE1      TEGRA264_SID(0x2c)
-+#define TEGRA264_SID_VI        TEGRA264_SID(0x2e)
-+#define TEGRA264_SID_VI1       TEGRA264_SID(0x2f)
-+#define TEGRA264_SID_VIC       TEGRA264_SID(0x30)
-+#define TEGRA264_SID_XUSB_DEV  TEGRA264_SID(0x32)
-+#define TEGRA264_SID_XUSB_DEV1 TEGRA264_SID(0x33)
-+#define TEGRA264_SID_XUSB_DEV2 TEGRA264_SID(0x34)
-+#define TEGRA264_SID_XUSB_DEV3 TEGRA264_SID(0x35)
-+#define TEGRA264_SID_XUSB_DEV4 TEGRA264_SID(0x36)
-+#define TEGRA264_SID_XUSB_DEV5 TEGRA264_SID(0x37)
-+
-+/*
-+ * memory client IDs
-+ */
-+
-+/* HOST1X read client */
-+#define TEGRA264_MEMORY_CLIENT_HOST1XR		0x16
-+/* VIC read client */
-+#define TEGRA264_MEMORY_CLIENT_VICR		0x6c
-+/* VIC Write client */
-+#define TEGRA264_MEMORY_CLIENT_VICW		0x6d
-+/* VI R5 Write client */
-+#define TEGRA264_MEMORY_CLIENT_VIW		0x72
-+#define TEGRA264_MEMORY_CLIENT_NVDECSRD2MC	0x78
-+#define TEGRA264_MEMORY_CLIENT_NVDECSWR2MC	0x79
-+/* Audio processor(APE) Read client */
-+#define TEGRA264_MEMORY_CLIENT_APER		0x7a
-+/* Audio processor(APE) Write client */
-+#define TEGRA264_MEMORY_CLIENT_APEW		0x7b
-+/* Audio DMA Read client */
-+#define TEGRA264_MEMORY_CLIENT_APEDMAR		0x9f
-+/* Audio DMA Write client */
-+#define TEGRA264_MEMORY_CLIENT_APEDMAW		0xa0
-+#define TEGRA264_MEMORY_CLIENT_GPUR02MC		0xb6
-+#define TEGRA264_MEMORY_CLIENT_GPUW02MC		0xb7
-+/* VI Falcon Read client */
-+#define TEGRA264_MEMORY_CLIENT_VIFALCONR	0xbc
-+/* VI Falcon Write client */
-+#define TEGRA264_MEMORY_CLIENT_VIFALCONW	0xbd
-+/* Read Client of RCE */
-+#define TEGRA264_MEMORY_CLIENT_RCER		0xd2
-+/* Write client of RCE */
-+#define TEGRA264_MEMORY_CLIENT_RCEW		0xd3
-+/* PCIE0/MSI Write clients */
-+#define TEGRA264_MEMORY_CLIENT_PCIE0W		0xd9
-+/* PCIE1/RPX4 Read clients */
-+#define TEGRA264_MEMORY_CLIENT_PCIE1R		0xda
-+/* PCIE1/RPX4 Write clients */
-+#define TEGRA264_MEMORY_CLIENT_PCIE1W		0xdb
-+/* PCIE2/DMX4 Read clients */
-+#define TEGRA264_MEMORY_CLIENT_PCIE2AR		0xdc
-+/* PCIE2/DMX4 Write clients */
-+#define TEGRA264_MEMORY_CLIENT_PCIE2AW		0xdd
-+/* PCIE3/RPX4 Read clients */
-+#define TEGRA264_MEMORY_CLIENT_PCIE3R		0xde
-+/* PCIE3/RPX4 Write clients */
-+#define TEGRA264_MEMORY_CLIENT_PCIE3W		0xdf
-+/* PCIE4/DMX8 Read clients */
-+#define TEGRA264_MEMORY_CLIENT_PCIE4R		0xe0
-+/* PCIE4/DMX8 Write clients */
-+#define TEGRA264_MEMORY_CLIENT_PCIE4W		0xe1
-+/* PCIE5/DMX4 Read clients */
-+#define TEGRA264_MEMORY_CLIENT_PCIE5R		0xe2
-+/* PCIE5/DMX4 Write clients */
-+#define TEGRA264_MEMORY_CLIENT_PCIE5W		0xe3
-+/* UFS Read client */
-+#define TEGRA264_MEMORY_CLIENT_UFSR		0x15c
-+/* UFS write client */
-+#define TEGRA264_MEMORY_CLIENT_UFSW		0x15d
-+/* HDA Read client */
-+#define TEGRA264_MEMORY_CLIENT_HDAR		0x17c
-+/* HDA Write client */
-+#define TEGRA264_MEMORY_CLIENT_HDAW		0x17d
-+/* Disp ISO Read Client */
-+#define TEGRA264_MEMORY_CLIENT_DISPR		0x182
-+/* MGBE0 Read mccif */
-+#define TEGRA264_MEMORY_CLIENT_MGBE0R		0x1a2
-+/* MGBE0 Write mccif */
-+#define TEGRA264_MEMORY_CLIENT_MGBE0W		0x1a3
-+/* MGBE1 Read mccif */
-+#define TEGRA264_MEMORY_CLIENT_MGBE1R		0x1a4
-+/* MGBE1 Write mccif */
-+#define TEGRA264_MEMORY_CLIENT_MGBE1W		0x1a5
-+/* VI1 R5 Write client */
-+#define TEGRA264_MEMORY_CLIENT_VI1W		0x1a6
-+/* SDMMC0 Read mccif */
-+#define TEGRA264_MEMORY_CLIENT_SDMMC0R		0x1c2
-+/* SDMMC0 Write mccif */
-+#define TEGRA264_MEMORY_CLIENT_SDMMC0W		0x1c3
-+
-+#endif /* DT_BINDINGS_MEMORY_NVIDIA_TEGRA264_H */
-diff --git a/include/dt-bindings/memory/tegra210-mc.h b/include/dt-bindings/memory/tegra210-mc.h
-index 5e082547f179..881bf78aa8b2 100644
--- a/include/dt-bindings/memory/tegra210-mc.h
-+++ b/include/dt-bindings/memory/tegra210-mc.h
-@@ -75,4 +75,78 @@
- #define TEGRA210_MC_RESET_ETR		28
- #define TEGRA210_MC_RESET_TSECB		29
- 
-+#define TEGRA210_MC_PTCR		0
-+#define TEGRA210_MC_DISPLAY0A		1
-+#define TEGRA210_MC_DISPLAY0AB		2
-+#define TEGRA210_MC_DISPLAY0B		3
-+#define TEGRA210_MC_DISPLAY0BB		4
-+#define TEGRA210_MC_DISPLAY0C		5
-+#define TEGRA210_MC_DISPLAY0CB		6
-+#define TEGRA210_MC_AFIR		14
-+#define TEGRA210_MC_AVPCARM7R		15
-+#define TEGRA210_MC_DISPLAYHC		16
-+#define TEGRA210_MC_DISPLAYHCB		17
-+#define TEGRA210_MC_HDAR		21
-+#define TEGRA210_MC_HOST1XDMAR		22
-+#define TEGRA210_MC_HOST1XR		23
-+#define TEGRA210_MC_NVENCSRD		28
-+#define TEGRA210_MC_PPCSAHBDMAR		29
-+#define TEGRA210_MC_PPCSAHBSLVR		30
-+#define TEGRA210_MC_SATAR		31
-+#define TEGRA210_MC_MPCORER		39
-+#define TEGRA210_MC_NVENCSWR		43
-+#define TEGRA210_MC_AFIW		49
-+#define TEGRA210_MC_AVPCARM7W		50
-+#define TEGRA210_MC_HDAW		53
-+#define TEGRA210_MC_HOST1XW		54
-+#define TEGRA210_MC_MPCOREW		57
-+#define TEGRA210_MC_PPCSAHBDMAW		59
-+#define TEGRA210_MC_PPCSAHBSLVW		60
-+#define TEGRA210_MC_SATAW		61
-+#define TEGRA210_MC_ISPRA		68
-+#define TEGRA210_MC_ISPWA		70
-+#define TEGRA210_MC_ISPWB		71
-+#define TEGRA210_MC_XUSB_HOSTR		74
-+#define TEGRA210_MC_XUSB_HOSTW		75
-+#define TEGRA210_MC_XUSB_DEVR		76
-+#define TEGRA210_MC_XUSB_DEVW		77
-+#define TEGRA210_MC_ISPRAB		78
-+#define TEGRA210_MC_ISPWAB		80
-+#define TEGRA210_MC_ISPWBB		81
-+#define TEGRA210_MC_TSECSRD		84
-+#define TEGRA210_MC_TSECSWR		85
-+#define TEGRA210_MC_A9AVPSCR		86
-+#define TEGRA210_MC_A9AVPSCW		87
-+#define TEGRA210_MC_GPUSRD		88
-+#define TEGRA210_MC_GPUSWR		89
-+#define TEGRA210_MC_DISPLAYT		90
-+#define TEGRA210_MC_SDMMCRA		96
-+#define TEGRA210_MC_SDMMCRAA		97
-+#define TEGRA210_MC_SDMMCR		98
-+#define TEGRA210_MC_SDMMCRAB		99
-+#define TEGRA210_MC_SDMMCWA		100
-+#define TEGRA210_MC_SDMMCWAA		101
-+#define TEGRA210_MC_SDMMCW		102
-+#define TEGRA210_MC_SDMMCWAB		103
-+#define TEGRA210_MC_VICSRD		108
-+#define TEGRA210_MC_VICSWR		109
-+#define TEGRA210_MC_VIW			114
-+#define TEGRA210_MC_DISPLAYD		115
-+#define TEGRA210_MC_NVDECSRD		120
-+#define TEGRA210_MC_NVDECSWR		121
-+#define TEGRA210_MC_APER		122
-+#define TEGRA210_MC_APEW		123
-+#define TEGRA210_MC_NVJPGRD		126
-+#define TEGRA210_MC_NVJPGWR		127
-+#define TEGRA210_MC_SESRD		128
-+#define TEGRA210_MC_SESWR		129
-+#define TEGRA210_MC_AXIAPR		130
-+#define TEGRA210_MC_AXIAPW		131
-+#define TEGRA210_MC_ETRR		132
-+#define TEGRA210_MC_ETRW		133
-+#define TEGRA210_MC_TSECSRDB		134
-+#define TEGRA210_MC_TSECSWRB		135
-+#define TEGRA210_MC_GPUSRD2		136
-+#define TEGRA210_MC_GPUSWR2		137
-+
- #endif
-diff --git a/include/dt-bindings/memory/tegra234-mc.h b/include/dt-bindings/memory/tegra234-mc.h
-index 62987b47ce81..6e60d55491b3 100644
--- a/include/dt-bindings/memory/tegra234-mc.h
-+++ b/include/dt-bindings/memory/tegra234-mc.h
-@@ -8,51 +8,248 @@
- #define TEGRA234_SID_INVALID		0x00
- #define TEGRA234_SID_PASSTHROUGH	0x7f
- 
-+/* ISO stream IDs */
-+#define TEGRA234_SID_ISO_NVDISPLAY	0x01
-+#define TEGRA234_SID_ISO_VI		0x02
-+#define TEGRA234_SID_ISO_VIFALC		0x03
-+#define TEGRA234_SID_ISO_VI2		0x04
-+#define TEGRA234_SID_ISO_VI2FALC	0x05
-+#define TEGRA234_SID_ISO_VI_VM2		0x06
-+#define TEGRA234_SID_ISO_VI2_VM2	0x07
-+
- /* NISO0 stream IDs */
-#define TEGRA234_SID_APE	0x02
-#define TEGRA234_SID_HDA	0x03
-#define TEGRA234_SID_GPCDMA	0x04
-#define TEGRA234_SID_MGBE	0x06
-#define TEGRA234_SID_PCIE0	0x12
-#define TEGRA234_SID_PCIE4	0x13
-#define TEGRA234_SID_PCIE5	0x14
-#define TEGRA234_SID_PCIE6	0x15
-#define TEGRA234_SID_PCIE9	0x1f
-#define TEGRA234_SID_MGBE_VF1	0x49
-#define TEGRA234_SID_MGBE_VF2	0x4a
-#define TEGRA234_SID_MGBE_VF3	0x4b
-+#define TEGRA234_SID_AON		0x01
-+#define TEGRA234_SID_APE		0x02
-+#define TEGRA234_SID_HDA		0x03
-+#define TEGRA234_SID_GPCDMA		0x04
-+#define TEGRA234_SID_ETR		0x05
-+#define TEGRA234_SID_MGBE		0x06
-+#define TEGRA234_SID_NVDISPLAY		0x07
-+#define TEGRA234_SID_DCE		0x08
-+#define TEGRA234_SID_PSC		0x09
-+#define TEGRA234_SID_RCE		0x0a
-+#define TEGRA234_SID_SCE		0x0b
-+#define TEGRA234_SID_UFSHC		0x0c
-+#define TEGRA234_SID_APE_1		0x0d
-+#define TEGRA234_SID_GPCDMA_1		0x0e
-+#define TEGRA234_SID_GPCDMA_2		0x0f
-+#define TEGRA234_SID_GPCDMA_3		0x10
-+#define TEGRA234_SID_GPCDMA_4		0x11
-+#define TEGRA234_SID_PCIE0		0x12
-+#define TEGRA234_SID_PCIE4		0x13
-+#define TEGRA234_SID_PCIE5		0x14
-+#define TEGRA234_SID_PCIE6		0x15
-+#define TEGRA234_SID_RCE_VM2		0x16
-+#define TEGRA234_SID_RCE_SERVER		0x17
-+#define TEGRA234_SID_SMMU_TEST		0x18
-+#define TEGRA234_SID_UFS_1		0x19
-+#define TEGRA234_SID_UFS_2		0x1a
-+#define TEGRA234_SID_UFS_3		0x1b
-+#define TEGRA234_SID_UFS_4		0x1c
-+#define TEGRA234_SID_UFS_5		0x1d
-+#define TEGRA234_SID_UFS_6		0x1e
-+#define TEGRA234_SID_PCIE9		0x1f
-+#define TEGRA234_SID_VSE_GPCDMA_VM0	0x20
-+#define TEGRA234_SID_VSE_GPCDMA_VM1	0x21
-+#define TEGRA234_SID_VSE_GPCDMA_VM2	0x22
-+#define TEGRA234_SID_NVDLA1		0x23
-+#define TEGRA234_SID_NVENC		0x24
-+#define TEGRA234_SID_NVJPG1		0x25
-+#define TEGRA234_SID_OFA		0x26
-+#define TEGRA234_SID_MGBE_VF1		0x49
-+#define TEGRA234_SID_MGBE_VF2		0x4a
-+#define TEGRA234_SID_MGBE_VF3		0x4b
-+#define TEGRA234_SID_MGBE_VF4		0x4c
-+#define TEGRA234_SID_MGBE_VF5		0x4d
-+#define TEGRA234_SID_MGBE_VF6		0x4e
-+#define TEGRA234_SID_MGBE_VF7		0x4f
-+#define TEGRA234_SID_MGBE_VF8		0x50
-+#define TEGRA234_SID_MGBE_VF9		0x51
-+#define TEGRA234_SID_MGBE_VF10		0x52
-+#define TEGRA234_SID_MGBE_VF11		0x53
-+#define TEGRA234_SID_MGBE_VF12		0x54
-+#define TEGRA234_SID_MGBE_VF13		0x55
-+#define TEGRA234_SID_MGBE_VF14		0x56
-+#define TEGRA234_SID_MGBE_VF15		0x57
-+#define TEGRA234_SID_MGBE_VF16		0x58
-+#define TEGRA234_SID_MGBE_VF17		0x59
-+#define TEGRA234_SID_MGBE_VF18		0x5a
-+#define TEGRA234_SID_MGBE_VF19		0x5b
-+#define TEGRA234_SID_MGBE_VF20		0x5c
-+#define TEGRA234_SID_APE_2		0x5e
-+#define TEGRA234_SID_APE_3		0x5f
-+#define TEGRA234_SID_UFS_7		0x60
-+#define TEGRA234_SID_UFS_8		0x61
-+#define TEGRA234_SID_UFS_9		0x62
-+#define TEGRA234_SID_UFS_10		0x63
-+#define TEGRA234_SID_UFS_11		0x64
-+#define TEGRA234_SID_UFS_12		0x65
-+#define TEGRA234_SID_UFS_13		0x66
-+#define TEGRA234_SID_UFS_14		0x67
-+#define TEGRA234_SID_UFS_15		0x68
-+#define TEGRA234_SID_UFS_16		0x69
-+#define TEGRA234_SID_UFS_17		0x6a
-+#define TEGRA234_SID_UFS_18		0x6b
-+#define TEGRA234_SID_UFS_19		0x6c
-+#define TEGRA234_SID_UFS_20		0x6d
-+#define TEGRA234_SID_GPCDMA_5		0x6e
-+#define TEGRA234_SID_GPCDMA_6		0x6f
-+#define TEGRA234_SID_GPCDMA_7		0x70
-+#define TEGRA234_SID_GPCDMA_8		0x71
-+#define TEGRA234_SID_GPCDMA_9		0x72
- 
- /* NISO1 stream IDs */
-#define TEGRA234_SID_SDMMC4	0x02
-#define TEGRA234_SID_PCIE1	0x05
-#define TEGRA234_SID_PCIE2	0x06
-#define TEGRA234_SID_PCIE3	0x07
-#define TEGRA234_SID_PCIE7	0x08
-#define TEGRA234_SID_PCIE8	0x09
-#define TEGRA234_SID_PCIE10	0x0b
-#define TEGRA234_SID_BPMP	0x10
-#define TEGRA234_SID_HOST1X	0x27
-#define TEGRA234_SID_VIC	0x34
-+#define TEGRA234_SID_SDMMC1A		0x01
-+#define TEGRA234_SID_SDMMC4		0x02
-+#define TEGRA234_SID_EQOS		0x03
-+#define TEGRA234_SID_HWMP_PMA		0x04
-+#define TEGRA234_SID_PCIE1		0x05
-+#define TEGRA234_SID_PCIE2		0x06
-+#define TEGRA234_SID_PCIE3		0x07
-+#define TEGRA234_SID_PCIE7		0x08
-+#define TEGRA234_SID_PCIE8		0x09
-+#define TEGRA234_SID_PCIE10		0x0b
-+#define TEGRA234_SID_QSPI0		0x0c
-+#define TEGRA234_SID_QSPI1		0x0d
-+#define TEGRA234_SID_XUSB_HOST		0x0e
-+#define TEGRA234_SID_XUSB_DEV		0x0f
-+#define TEGRA234_SID_BPMP		0x10
-+#define TEGRA234_SID_FSI		0x11
-+#define TEGRA234_SID_PVA0_VM0		0x12
-+#define TEGRA234_SID_PVA0_VM1		0x13
-+#define TEGRA234_SID_PVA0_VM2		0x14
-+#define TEGRA234_SID_PVA0_VM3		0x15
-+#define TEGRA234_SID_PVA0_VM4		0x16
-+#define TEGRA234_SID_PVA0_VM5		0x17
-+#define TEGRA234_SID_PVA0_VM6		0x18
-+#define TEGRA234_SID_PVA0_VM7		0x19
-+#define TEGRA234_SID_XUSB_VF0		0x1a
-+#define TEGRA234_SID_XUSB_VF1		0x1b
-+#define TEGRA234_SID_XUSB_VF2		0x1c
-+#define TEGRA234_SID_XUSB_VF3		0x1d
-+#define TEGRA234_SID_EQOS_VF1		0x1e
-+#define TEGRA234_SID_EQOS_VF2		0x1f
-+#define TEGRA234_SID_EQOS_VF3		0x20
-+#define TEGRA234_SID_EQOS_VF4		0x21
-+#define TEGRA234_SID_ISP_VM2		0x22
-+#define TEGRA234_SID_HOST1X		0x27
-+#define TEGRA234_SID_ISP		0x28
-+#define TEGRA234_SID_NVDEC		0x29
-+#define TEGRA234_SID_NVJPG		0x2a
-+#define TEGRA234_SID_NVDLA0		0x2b
-+#define TEGRA234_SID_PVA0		0x2c
-+#define TEGRA234_SID_SES_SE0		0x2d
-+#define TEGRA234_SID_SES_SE1		0x2e
-+#define TEGRA234_SID_SES_SE2		0x2f
-+#define TEGRA234_SID_SEU1_SE0		0x30
-+#define TEGRA234_SID_SEU1_SE1		0x31
-+#define TEGRA234_SID_SEU1_SE2		0x32
-+#define TEGRA234_SID_TSEC		0x33
-+#define TEGRA234_SID_VIC		0x34
-+#define TEGRA234_SID_HC_VM0		0x3d
-+#define TEGRA234_SID_HC_VM1		0x3e
-+#define TEGRA234_SID_HC_VM2		0x3f
-+#define TEGRA234_SID_HC_VM3		0x40
-+#define TEGRA234_SID_HC_VM4		0x41
-+#define TEGRA234_SID_HC_VM5		0x42
-+#define TEGRA234_SID_HC_VM6		0x43
-+#define TEGRA234_SID_HC_VM7		0x44
-+#define TEGRA234_SID_SE_VM0		0x45
-+#define TEGRA234_SID_SE_VM1		0x46
-+#define TEGRA234_SID_SE_VM2		0x47
-+#define TEGRA234_SID_ISPFALC		0x48
-+#define TEGRA234_SID_NISO1_SMMU_TEST	0x49
-+#define TEGRA234_SID_TSEC_VM0		0x4a
-+
-+/* Shared stream IDs */
-+#define TEGRA234_SID_HOST1X_CTX0	0x35
-+#define TEGRA234_SID_HOST1X_CTX1	0x36
-+#define TEGRA234_SID_HOST1X_CTX2	0x37
-+#define TEGRA234_SID_HOST1X_CTX3	0x38
-+#define TEGRA234_SID_HOST1X_CTX4	0x39
-+#define TEGRA234_SID_HOST1X_CTX5	0x3a
-+#define TEGRA234_SID_HOST1X_CTX6	0x3b
-+#define TEGRA234_SID_HOST1X_CTX7	0x3c
- 
- /*
-  * memory client IDs
-  */
- 
-+/* Misses from System Memory Management Unit (SMMU) Page Table Cache (PTC) */
-+#define TEGRA234_MEMORY_CLIENT_PTCR 0x00
-+/* MSS internal memqual MIU7 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU7R 0x01
-+/* MSS internal memqual MIU7 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU7W 0x02
-+/* MSS internal memqual MIU8 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU8R 0x03
-+/* MSS internal memqual MIU8 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU8W 0x04
-+/* MSS internal memqual MIU9 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU9R 0x05
-+/* MSS internal memqual MIU9 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU9W 0x06
-+/* MSS internal memqual MIU10 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU10R 0x07
-+/* MSS internal memqual MIU10 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU10W 0x08
-+/* MSS internal memqual MIU11 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU11R 0x09
-+/* MSS internal memqual MIU11 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU11W 0x0a
-+/* MSS internal memqual MIU12 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU12R 0x0b
-+/* MSS internal memqual MIU12 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU12W 0x0c
-+/* MSS internal memqual MIU13 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU13R 0x0d
-+/* MSS internal memqual MIU13 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU13W 0x0e
-+#define TEGRA234_MEMORY_CLIENT_NVL5RHP 0x13
-+#define TEGRA234_MEMORY_CLIENT_NVL5R 0x14
- /* High-definition audio (HDA) read clients */
- #define TEGRA234_MEMORY_CLIENT_HDAR 0x15
-+/* Host channel data read clients */
- #define TEGRA234_MEMORY_CLIENT_HOST1XDMAR 0x16
-+#define TEGRA234_MEMORY_CLIENT_NVL5W 0x17
-+#define TEGRA234_MEMORY_CLIENT_NVL6RHP 0x18
-+#define TEGRA234_MEMORY_CLIENT_NVL6R 0x19
-+#define TEGRA234_MEMORY_CLIENT_NVL6W 0x1a
-+#define TEGRA234_MEMORY_CLIENT_NVL7RHP 0x1b
-+#define TEGRA234_MEMORY_CLIENT_NVENCSRD 0x1c
-+#define TEGRA234_MEMORY_CLIENT_NVL7R 0x1d
-+#define TEGRA234_MEMORY_CLIENT_NVL7W 0x1e
-+#define TEGRA234_MEMORY_CLIENT_NVL8RHP 0x20
-+#define TEGRA234_MEMORY_CLIENT_NVL8R 0x21
-+#define TEGRA234_MEMORY_CLIENT_NVL8W 0x22
-+#define TEGRA234_MEMORY_CLIENT_NVL9RHP 0x23
-+#define TEGRA234_MEMORY_CLIENT_NVL9R 0x24
-+#define TEGRA234_MEMORY_CLIENT_NVL9W 0x25
- /* PCIE6 read clients */
- #define TEGRA234_MEMORY_CLIENT_PCIE6AR 0x28
- /* PCIE6 write clients */
- #define TEGRA234_MEMORY_CLIENT_PCIE6AW 0x29
- /* PCIE7 read clients */
- #define TEGRA234_MEMORY_CLIENT_PCIE7AR 0x2a
-+#define TEGRA234_MEMORY_CLIENT_NVENCSWR 0x2b
-+/* DLA0ARDB read clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA0RDB 0x2c
-+/* DLA0ARDB1 read clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA0RDB1 0x2d
-+/* DLA0 writes */
-+#define TEGRA234_MEMORY_CLIENT_DLA0WRB 0x2e
-+/* DLA1ARDB read clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA1RDB 0x2f
- /* PCIE7 write clients */
- #define TEGRA234_MEMORY_CLIENT_PCIE7AW 0x30
- /* PCIE8 read clients */
- #define TEGRA234_MEMORY_CLIENT_PCIE8AR 0x32
- /* High-definition audio (HDA) write clients */
- #define TEGRA234_MEMORY_CLIENT_HDAW 0x35
-+/* Writes from Cortex-A9 4 CPU cores via the L2 cache */
-+#define TEGRA234_MEMORY_CLIENT_MPCOREW 0x39
-+/* OFAA client */
-+#define TEGRA234_MEMORY_CLIENT_OFAR1 0x3a
- /* PCIE8 write clients */
- #define TEGRA234_MEMORY_CLIENT_PCIE8AW 0x3b
- /* PCIE9 read clients */
-@@ -65,10 +262,32 @@
- #define TEGRA234_MEMORY_CLIENT_PCIE10AR 0x3f
- /* PCIE10 write clients */
- #define TEGRA234_MEMORY_CLIENT_PCIE10AW 0x40
-+/* ISP read client for Crossbar A */
-+#define TEGRA234_MEMORY_CLIENT_ISPRA 0x44
-+/* ISP read client 1 for Crossbar A */
-+#define TEGRA234_MEMORY_CLIENT_ISPFALR 0x45
-+/* ISP Write client for Crossbar A */
-+#define TEGRA234_MEMORY_CLIENT_ISPWA 0x46
-+/* ISP Write client Crossbar B */
-+#define TEGRA234_MEMORY_CLIENT_ISPWB 0x47
- /* PCIE10r1 read clients */
- #define TEGRA234_MEMORY_CLIENT_PCIE10AR1 0x48
- /* PCIE7r1 read clients */
- #define TEGRA234_MEMORY_CLIENT_PCIE7AR1 0x49
-+/* XUSB_HOST read clients */
-+#define TEGRA234_MEMORY_CLIENT_XUSB_HOSTR 0x4a
-+/* XUSB_HOST write clients */
-+#define TEGRA234_MEMORY_CLIENT_XUSB_HOSTW 0x4b
-+/* XUSB read clients */
-+#define TEGRA234_MEMORY_CLIENT_XUSB_DEVR 0x4c
-+/* XUSB_DEV write clients */
-+#define TEGRA234_MEMORY_CLIENT_XUSB_DEVW 0x4d
-+/* TSEC Memory Return Data Client Description */
-+#define TEGRA234_MEMORY_CLIENT_TSECSRD 0x54
-+/* TSEC Memory Write Client Description */
-+#define TEGRA234_MEMORY_CLIENT_TSECSWR 0x55
-+/* XSPI writes */
-+#define TEGRA234_MEMORY_CLIENT_XSPI1W 0x56
- /* MGBE0 read client */
- #define TEGRA234_MEMORY_CLIENT_MGBEARD 0x58
- /* MGBEB read client */
-@@ -79,18 +298,86 @@
- #define TEGRA234_MEMORY_CLIENT_MGBEDRD 0x5b
- /* MGBE0 write client */
- #define TEGRA234_MEMORY_CLIENT_MGBEAWR 0x5c
-+/* OFAA client */
-+#define TEGRA234_MEMORY_CLIENT_OFAR 0x5d
-+/* OFAA writes */
-+#define TEGRA234_MEMORY_CLIENT_OFAW 0x5e
- /* MGBEB write client */
- #define TEGRA234_MEMORY_CLIENT_MGBEBWR 0x5f
-+/* sdmmca memory read client */
-+#define TEGRA234_MEMORY_CLIENT_SDMMCRA 0x60
- /* MGBEC write client */
- #define TEGRA234_MEMORY_CLIENT_MGBECWR 0x61
- /* sdmmcd memory read client */
- #define TEGRA234_MEMORY_CLIENT_SDMMCRAB 0x63
-+/* sdmmca memory write client */
-+#define TEGRA234_MEMORY_CLIENT_SDMMCWA 0x64
- /* MGBED write client */
- #define TEGRA234_MEMORY_CLIENT_MGBEDWR 0x65
- /* sdmmcd memory write client */
- #define TEGRA234_MEMORY_CLIENT_SDMMCWAB 0x67
-+/* SE Memory Return Data Client Description */
-+#define TEGRA234_MEMORY_CLIENT_SEU1RD 0x68
-+/* SE Memory Write Client Description */
-+#define TEGRA234_MEMORY_CLIENT_SUE1WR 0x69
- #define TEGRA234_MEMORY_CLIENT_VICSRD 0x6c
- #define TEGRA234_MEMORY_CLIENT_VICSWR 0x6d
-+/* DLA1ARDB1 read clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA1RDB1 0x6e
-+/* DLA1 writes */
-+#define TEGRA234_MEMORY_CLIENT_DLA1WRB 0x6f
-+/* VI FLACON read clients */
-+#define TEGRA234_MEMORY_CLIENT_VI2FALR 0x71
-+/* VI Write client */
-+#define TEGRA234_MEMORY_CLIENT_VI2W 0x70
-+/* VI Write client */
-+#define TEGRA234_MEMORY_CLIENT_VIW 0x72
-+/* NISO display read client */
-+#define TEGRA234_MEMORY_CLIENT_NVDISPNISOR 0x73
-+/* NVDISPNISO writes */
-+#define TEGRA234_MEMORY_CLIENT_NVDISPNISOW 0x74
-+/* XSPI client */
-+#define TEGRA234_MEMORY_CLIENT_XSPI0R 0x75
-+/* XSPI writes */
-+#define TEGRA234_MEMORY_CLIENT_XSPI0W 0x76
-+/* XSPI client */
-+#define TEGRA234_MEMORY_CLIENT_XSPI1R 0x77
-+#define TEGRA234_MEMORY_CLIENT_NVDECSRD 0x78
-+#define TEGRA234_MEMORY_CLIENT_NVDECSWR 0x79
-+/* Audio Processing (APE) engine read clients */
-+#define TEGRA234_MEMORY_CLIENT_APER 0x7a
-+/* Audio Processing (APE) engine write clients */
-+#define TEGRA234_MEMORY_CLIENT_APEW 0x7b
-+/* VI2FAL writes */
-+#define TEGRA234_MEMORY_CLIENT_VI2FALW 0x7c
-+#define TEGRA234_MEMORY_CLIENT_NVJPGSRD 0x7e
-+#define TEGRA234_MEMORY_CLIENT_NVJPGSWR 0x7f
-+/* SE Memory Return Data Client Description */
-+#define TEGRA234_MEMORY_CLIENT_SESRD 0x80
-+/* SE Memory Write Client Description */
-+#define TEGRA234_MEMORY_CLIENT_SESWR 0x81
-+/* AXI AP and DFD-AUX0/1 read clients Both share the same interface on the on MSS */
-+#define TEGRA234_MEMORY_CLIENT_AXIAPR 0x82
-+/* AXI AP and DFD-AUX0/1 write clients Both sahre the same interface on MSS */
-+#define TEGRA234_MEMORY_CLIENT_AXIAPW 0x83
-+/* ETR read clients */
-+#define TEGRA234_MEMORY_CLIENT_ETRR 0x84
-+/* ETR write clients */
-+#define TEGRA234_MEMORY_CLIENT_ETRW 0x85
-+/* AXI Switch read client */
-+#define TEGRA234_MEMORY_CLIENT_AXISR 0x8c
-+/* AXI Switch write client */
-+#define TEGRA234_MEMORY_CLIENT_AXISW 0x8d
-+/* EQOS read client */
-+#define TEGRA234_MEMORY_CLIENT_EQOSR 0x8e
-+/* EQOS write client */
-+#define TEGRA234_MEMORY_CLIENT_EQOSW 0x8f
-+/* UFSHC read client */
-+#define TEGRA234_MEMORY_CLIENT_UFSHCR 0x90
-+/* UFSHC write client */
-+#define TEGRA234_MEMORY_CLIENT_UFSHCW 0x91
-+/* NVDISPLAY read client */
-+#define TEGRA234_MEMORY_CLIENT_NVDISPLAYR 0x92
- /* BPMP read client */
- #define TEGRA234_MEMORY_CLIENT_BPMPR 0x93
- /* BPMP write client */
-@@ -99,10 +386,97 @@
- #define TEGRA234_MEMORY_CLIENT_BPMPDMAR 0x95
- /* BPMPDMA write client */
- #define TEGRA234_MEMORY_CLIENT_BPMPDMAW 0x96
-+/* AON read client */
-+#define TEGRA234_MEMORY_CLIENT_AONR 0x97
-+/* AON write client */
-+#define TEGRA234_MEMORY_CLIENT_AONW 0x98
-+/* AONDMA read client */
-+#define TEGRA234_MEMORY_CLIENT_AONDMAR 0x99
-+/* AONDMA write client */
-+#define TEGRA234_MEMORY_CLIENT_AONDMAW 0x9a
-+/* SCE read client */
-+#define TEGRA234_MEMORY_CLIENT_SCER 0x9b
-+/* SCE write client */
-+#define TEGRA234_MEMORY_CLIENT_SCEW 0x9c
-+/* SCEDMA read client */
-+#define TEGRA234_MEMORY_CLIENT_SCEDMAR 0x9d
-+/* SCEDMA write client */
-+#define TEGRA234_MEMORY_CLIENT_SCEDMAW 0x9e
- /* APEDMA read client */
- #define TEGRA234_MEMORY_CLIENT_APEDMAR 0x9f
- /* APEDMA write client */
- #define TEGRA234_MEMORY_CLIENT_APEDMAW 0xa0
-+/* NVDISPLAY read client instance 2 */
-+#define TEGRA234_MEMORY_CLIENT_NVDISPLAYR1 0xa1
-+#define TEGRA234_MEMORY_CLIENT_VICSRD1 0xa2
-+/* MSS internal memqual MIU0 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU0R 0xa6
-+/* MSS internal memqual MIU0 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU0W 0xa7
-+/* MSS internal memqual MIU1 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU1R 0xa8
-+/* MSS internal memqual MIU1 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU1W 0xa9
-+/* MSS internal memqual MIU2 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU2R 0xae
-+/* MSS internal memqual MIU2 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU2W 0xaf
-+/* MSS internal memqual MIU3 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU3R 0xb0
-+/* MSS internal memqual MIU3 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU3W 0xb1
-+/* MSS internal memqual MIU4 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU4R 0xb2
-+/* MSS internal memqual MIU4 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU4W 0xb3
-+#define TEGRA234_MEMORY_CLIENT_DPMUR 0xb4
-+#define TEGRA234_MEMORY_CLIENT_DPMUW 0xb5
-+#define TEGRA234_MEMORY_CLIENT_NVL0R 0xb6
-+#define TEGRA234_MEMORY_CLIENT_NVL0W 0xb7
-+#define TEGRA234_MEMORY_CLIENT_NVL1R 0xb8
-+#define TEGRA234_MEMORY_CLIENT_NVL1W 0xb9
-+#define TEGRA234_MEMORY_CLIENT_NVL2R 0xba
-+#define TEGRA234_MEMORY_CLIENT_NVL2W 0xbb
-+/* VI FLACON read clients */
-+#define TEGRA234_MEMORY_CLIENT_VIFALR 0xbc
-+/* VIFAL write clients */
-+#define TEGRA234_MEMORY_CLIENT_VIFALW 0xbd
-+/* DLA0ARDA read clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA0RDA 0xbe
-+/* DLA0 Falcon read clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA0FALRDB 0xbf
-+/* DLA0 write clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA0WRA 0xc0
-+/* DLA0 write clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA0FALWRB 0xc1
-+/* DLA1ARDA read clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA1RDA 0xc2
-+/* DLA1 Falcon read clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA1FALRDB 0xc3
-+/* DLA1 write clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA1WRA 0xc4
-+/* DLA1 write clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA1FALWRB 0xc5
-+/* PVA0RDA read clients */
-+#define TEGRA234_MEMORY_CLIENT_PVA0RDA 0xc6
-+/* PVA0RDB read clients */
-+#define TEGRA234_MEMORY_CLIENT_PVA0RDB 0xc7
-+/* PVA0RDC read clients */
-+#define TEGRA234_MEMORY_CLIENT_PVA0RDC 0xc8
-+/* PVA0WRA write clients */
-+#define TEGRA234_MEMORY_CLIENT_PVA0WRA 0xc9
-+/* PVA0WRB write clients */
-+#define TEGRA234_MEMORY_CLIENT_PVA0WRB 0xca
-+/* PVA0WRC write clients */
-+#define TEGRA234_MEMORY_CLIENT_PVA0WRC 0xcb
-+/* RCE read client */
-+#define TEGRA234_MEMORY_CLIENT_RCER 0xd2
-+/* RCE write client */
-+#define TEGRA234_MEMORY_CLIENT_RCEW 0xd3
-+/* RCEDMA read client */
-+#define TEGRA234_MEMORY_CLIENT_RCEDMAR 0xd4
-+/* RCEDMA write client */
-+#define TEGRA234_MEMORY_CLIENT_RCEDMAW 0xd5
- /* PCIE0 read clients */
- #define TEGRA234_MEMORY_CLIENT_PCIE0R 0xd8
- /* PCIE0 write clients */
-@@ -127,7 +501,44 @@
- #define TEGRA234_MEMORY_CLIENT_PCIE5R 0xe2
- /* PCIE5 write clients */
- #define TEGRA234_MEMORY_CLIENT_PCIE5W 0xe3
-+/* ISP read client 1 for Crossbar A */
-+#define TEGRA234_MEMORY_CLIENT_ISPFALW 0xe4
-+#define TEGRA234_MEMORY_CLIENT_NVL3R 0xe5
-+#define TEGRA234_MEMORY_CLIENT_NVL3W 0xe6
-+#define TEGRA234_MEMORY_CLIENT_NVL4R 0xe7
-+#define TEGRA234_MEMORY_CLIENT_NVL4W 0xe8
-+/* DLA0ARDA1 read clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA0RDA1 0xe9
-+/* DLA1ARDA1 read clients */
-+#define TEGRA234_MEMORY_CLIENT_DLA1RDA1 0xea
-+/* PVA0RDA1 read clients */
-+#define TEGRA234_MEMORY_CLIENT_PVA0RDA1 0xeb
-+/* PVA0RDB1 read clients */
-+#define TEGRA234_MEMORY_CLIENT_PVA0RDB1 0xec
- /* PCIE5r1 read clients */
- #define TEGRA234_MEMORY_CLIENT_PCIE5R1 0xef
-+#define TEGRA234_MEMORY_CLIENT_NVENCSRD1 0xf0
-+/* ISP read client for Crossbar A */
-+#define TEGRA234_MEMORY_CLIENT_ISPRA1 0xf2
-+#define TEGRA234_MEMORY_CLIENT_NVL0RHP 0xf4
-+#define TEGRA234_MEMORY_CLIENT_NVL1RHP 0xf5
-+#define TEGRA234_MEMORY_CLIENT_NVL2RHP 0xf6
-+#define TEGRA234_MEMORY_CLIENT_NVL3RHP 0xf7
-+#define TEGRA234_MEMORY_CLIENT_NVL4RHP 0xf8
-+/* MSS internal memqual MIU5 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU5R 0xfc
-+/* MSS internal memqual MIU5 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU5W 0xfd
-+/* MSS internal memqual MIU6 read clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU6R 0xfe
-+/* MSS internal memqual MIU6 write clients */
-+#define TEGRA234_MEMORY_CLIENT_MIU6W 0xff
-+#define TEGRA234_MEMORY_CLIENT_NVJPG1SRD 0x123
-+#define TEGRA234_MEMORY_CLIENT_NVJPG1SWR 0x124
-+
-+/* ICC ID's for dummy MC clients used to represent CPU Clusters */
-+#define TEGRA_ICC_MC_CPU_CLUSTER0       1003
-+#define TEGRA_ICC_MC_CPU_CLUSTER1       1004
-+#define TEGRA_ICC_MC_CPU_CLUSTER2       1005
- 
- #endif
-diff --git a/include/dt-bindings/mfd/qcom-pm8008.h b/include/dt-bindings/mfd/qcom-pm8008.h
-deleted file mode 100644
-index eca9448df228..000000000000
--- a/include/dt-bindings/mfd/qcom-pm8008.h
-+++ /dev/null
-@@ -1,19 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2021 The Linux Foundation. All rights reserved.
- */
-
-#ifndef __DT_BINDINGS_MFD_QCOM_PM8008_H
-#define __DT_BINDINGS_MFD_QCOM_PM8008_H
-
-/* PM8008 IRQ numbers */
-#define PM8008_IRQ_MISC_UVLO	0
-#define PM8008_IRQ_MISC_OVLO	1
-#define PM8008_IRQ_MISC_OTST2	2
-#define PM8008_IRQ_MISC_OTST3	3
-#define PM8008_IRQ_MISC_LDO_OCP	4
-#define PM8008_IRQ_TEMP_ALARM	5
-#define PM8008_IRQ_GPIO1	6
-#define PM8008_IRQ_GPIO2	7
-
-#endif
-diff --git a/include/dt-bindings/mfd/st,stpmic1.h b/include/dt-bindings/mfd/st,stpmic1.h
-index 321cd08797d9..9dd15b9c743e 100644
--- a/include/dt-bindings/mfd/st,stpmic1.h
-+++ b/include/dt-bindings/mfd/st,stpmic1.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
- /*
-  * Copyright (C) STMicroelectronics 2018 - All Rights Reserved
-  * Author: Philippe Peurichard <philippe.peurichard@st.com>,
-diff --git a/include/dt-bindings/mfd/stm32f4-rcc.h b/include/dt-bindings/mfd/stm32f4-rcc.h
-index 309e8c79f27b..36448a5619a1 100644
--- a/include/dt-bindings/mfd/stm32f4-rcc.h
-+++ b/include/dt-bindings/mfd/stm32f4-rcc.h
-@@ -34,7 +34,6 @@
- #define STM32F4_AHB1_RESET(bit) (STM32F4_RCC_AHB1_##bit + (0x10 * 8))
- #define STM32F4_AHB1_CLOCK(bit) (STM32F4_RCC_AHB1_##bit)
- 
-
- /* AHB2 */
- #define STM32F4_RCC_AHB2_DCMI	0
- #define STM32F4_RCC_AHB2_CRYP	4
-diff --git a/include/dt-bindings/mfd/stm32f7-rcc.h b/include/dt-bindings/mfd/stm32f7-rcc.h
-index a90f3613c584..a4e4f9271395 100644
--- a/include/dt-bindings/mfd/stm32f7-rcc.h
-+++ b/include/dt-bindings/mfd/stm32f7-rcc.h
-@@ -64,6 +64,7 @@
- #define STM32F7_RCC_APB1_TIM14		8
- #define STM32F7_RCC_APB1_LPTIM1		9
- #define STM32F7_RCC_APB1_WWDG		11
-+#define STM32F7_RCC_APB1_CAN3		13
- #define STM32F7_RCC_APB1_SPI2		14
- #define STM32F7_RCC_APB1_SPI3		15
- #define STM32F7_RCC_APB1_SPDIFRX	16
-@@ -107,6 +108,7 @@
- #define STM32F7_RCC_APB2_SAI1		22
- #define STM32F7_RCC_APB2_SAI2		23
- #define STM32F7_RCC_APB2_LTDC		26
-+#define STM32F7_RCC_APB2_DSI		27
- 
- #define STM32F7_APB2_RESET(bit)	(STM32F7_RCC_APB2_##bit + (0x24 * 8))
- #define STM32F7_APB2_CLOCK(bit)	(STM32F7_RCC_APB2_##bit + 0xA0)
-diff --git a/include/dt-bindings/mux/ti-serdes.h b/include/dt-bindings/mux/ti-serdes.h
-index d3116c52ab72..b0b1091aad6d 100644
--- a/include/dt-bindings/mux/ti-serdes.h
-+++ b/include/dt-bindings/mux/ti-serdes.h
-@@ -6,6 +6,14 @@
- #ifndef _DT_BINDINGS_MUX_TI_SERDES
- #define _DT_BINDINGS_MUX_TI_SERDES
- 
-+/*
-+ * These bindings are deprecated, because they do not match the actual
-+ * concept of bindings but rather contain pure constants values used only
-+ * in DTS board files.
-+ * Instead include the header in the DTS source directory.
-+ */
-+#warning "These bindings are deprecated. Instead, use the header in the DTS source directory."
-+
- /* J721E */
- 
- #define J721E_SERDES0_LANE0_QSGMII_LANE1	0x0
-@@ -117,4 +125,66 @@
- #define J721S2_SERDES0_LANE3_USB		0x2
- #define J721S2_SERDES0_LANE3_IP4_UNUSED		0x3
- 
-+/* J784S4 */
-+
-+#define J784S4_SERDES0_LANE0_IP1_UNUSED		0x0
-+#define J784S4_SERDES0_LANE0_PCIE1_LANE0	0x1
-+#define J784S4_SERDES0_LANE0_IP3_UNUSED		0x2
-+#define J784S4_SERDES0_LANE0_IP4_UNUSED		0x3
-+
-+#define J784S4_SERDES0_LANE1_IP1_UNUSED		0x0
-+#define J784S4_SERDES0_LANE1_PCIE1_LANE1	0x1
-+#define J784S4_SERDES0_LANE1_IP3_UNUSED		0x2
-+#define J784S4_SERDES0_LANE1_IP4_UNUSED		0x3
-+
-+#define J784S4_SERDES0_LANE2_PCIE3_LANE0	0x0
-+#define J784S4_SERDES0_LANE2_PCIE1_LANE2	0x1
-+#define J784S4_SERDES0_LANE2_IP3_UNUSED		0x2
-+#define J784S4_SERDES0_LANE2_IP4_UNUSED		0x3
-+
-+#define J784S4_SERDES0_LANE3_PCIE3_LANE1	0x0
-+#define J784S4_SERDES0_LANE3_PCIE1_LANE3	0x1
-+#define J784S4_SERDES0_LANE3_USB		0x2
-+#define J784S4_SERDES0_LANE3_IP4_UNUSED		0x3
-+
-+#define J784S4_SERDES1_LANE0_QSGMII_LANE3	0x0
-+#define J784S4_SERDES1_LANE0_PCIE0_LANE0	0x1
-+#define J784S4_SERDES1_LANE0_IP3_UNUSED		0x2
-+#define J784S4_SERDES1_LANE0_IP4_UNUSED		0x3
-+
-+#define J784S4_SERDES1_LANE1_QSGMII_LANE4	0x0
-+#define J784S4_SERDES1_LANE1_PCIE0_LANE1	0x1
-+#define J784S4_SERDES1_LANE1_IP3_UNUSED		0x2
-+#define J784S4_SERDES1_LANE1_IP4_UNUSED		0x3
-+
-+#define J784S4_SERDES1_LANE2_QSGMII_LANE1	0x0
-+#define J784S4_SERDES1_LANE2_PCIE0_LANE2	0x1
-+#define J784S4_SERDES1_LANE2_PCIE2_LANE0	0x2
-+#define J784S4_SERDES1_LANE2_IP4_UNUSED		0x3
-+
-+#define J784S4_SERDES1_LANE3_QSGMII_LANE2	0x0
-+#define J784S4_SERDES1_LANE3_PCIE0_LANE3	0x1
-+#define J784S4_SERDES1_LANE3_PCIE2_LANE1	0x2
-+#define J784S4_SERDES1_LANE3_IP4_UNUSED		0x3
-+
-+#define J784S4_SERDES2_LANE0_QSGMII_LANE5	0x0
-+#define J784S4_SERDES2_LANE0_IP2_UNUSED		0x1
-+#define J784S4_SERDES2_LANE0_IP3_UNUSED		0x2
-+#define J784S4_SERDES2_LANE0_IP4_UNUSED		0x3
-+
-+#define J784S4_SERDES2_LANE1_QSGMII_LANE6	0x0
-+#define J784S4_SERDES2_LANE1_IP2_UNUSED		0x1
-+#define J784S4_SERDES2_LANE1_IP3_UNUSED		0x2
-+#define J784S4_SERDES2_LANE1_IP4_UNUSED		0x3
-+
-+#define J784S4_SERDES2_LANE2_QSGMII_LANE7	0x0
-+#define J784S4_SERDES2_LANE2_QSGMII_LANE1	0x1
-+#define J784S4_SERDES2_LANE2_IP3_UNUSED		0x2
-+#define J784S4_SERDES2_LANE2_IP4_UNUSED		0x3
-+
-+#define J784S4_SERDES2_LANE3_QSGMII_LANE8	0x0
-+#define J784S4_SERDES2_LANE3_QSGMII_LANE2	0x1
-+#define J784S4_SERDES2_LANE3_IP3_UNUSED		0x2
-+#define J784S4_SERDES2_LANE3_IP4_UNUSED		0x3
-+
- #endif /* _DT_BINDINGS_MUX_TI_SERDES */
-diff --git a/include/dt-bindings/net/renesas,r9a09g077-pcs-miic.h b/include/dt-bindings/net/renesas,r9a09g077-pcs-miic.h
-new file mode 100644
-index 000000000000..43a2b5743a63
--- /dev/null
-+++ b/include/dt-bindings/net/renesas,r9a09g077-pcs-miic.h
-@@ -0,0 +1,36 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2025 Renesas Electronics Corporation.
-+ */
-+
-+#ifndef _DT_BINDINGS_RENASAS_R9A09G077_PCS_MIIC_H
-+#define _DT_BINDINGS_RENASAS_R9A09G077_PCS_MIIC_H
-+
-+/*
-+ * Media Interface Connection Matrix
-+ * ===========================================================
-+ *
-+ * Selects the function of the Media interface of the MAC to be used
-+ *
-+ * SW_MODE[2:0] | Port 0      | Port 1      | Port 2      | Port 3
-+ * -------------|-------------|-------------|-------------|-------------
-+ * 000b         | ETHSW Port0 | ETHSW Port1 | ETHSW Port2 | GMAC1
-+ * 001b         | ESC Port0   | ESC Port1   | GMAC2       | GMAC1
-+ * 010b         | ESC Port0   | ESC Port1   | ETHSW Port2 | GMAC1
-+ * 011b         | ESC Port0   | ESC Port1   | ESC Port2   | GMAC1
-+ * 100b         | ETHSW Port0 | ESC Port1   | ESC Port2   | GMAC1
-+ * 101b         | ETHSW Port0 | ESC Port1   | ETHSW Port2 | GMAC1
-+ * 110b         | ETHSW Port0 | ETHSW Port1 | GMAC2       | GMAC1
-+ * 111b         | GMAC0       | GMAC1       | GMAC2       | -
-+ */
-+#define ETHSS_GMAC0_PORT		0
-+#define ETHSS_GMAC1_PORT		1
-+#define ETHSS_GMAC2_PORT		2
-+#define ETHSS_ESC_PORT0			3
-+#define ETHSS_ESC_PORT1			4
-+#define ETHSS_ESC_PORT2			5
-+#define ETHSS_ETHSW_PORT0		6
-+#define ETHSS_ETHSW_PORT1		7
-+#define ETHSS_ETHSW_PORT2		8
-+
-+#endif
-diff --git a/include/dt-bindings/net/ti-dp83867.h b/include/dt-bindings/net/ti-dp83867.h
-index 6fc4b445d3a1..b8a4f3ff4a3b 100644
--- a/include/dt-bindings/net/ti-dp83867.h
-+++ b/include/dt-bindings/net/ti-dp83867.h
-@@ -1,10 +1,10 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-+/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
- /*
-  * Device Tree constants for the Texas Instruments DP83867 PHY
-  *
-  * Author: Dan Murphy <dmurphy@ti.com>
-  *
- * Copyright:   (C) 2015 Texas Instruments, Inc.
-+ * Copyright (C) 2015-2024 Texas Instruments Incorporated - https://www.ti.com/
-  */
- 
- #ifndef _DT_BINDINGS_TI_DP83867_H
-diff --git a/include/dt-bindings/net/ti-dp83869.h b/include/dt-bindings/net/ti-dp83869.h
-index 218b1a64e975..917114aad7d0 100644
--- a/include/dt-bindings/net/ti-dp83869.h
-+++ b/include/dt-bindings/net/ti-dp83869.h
-@@ -1,10 +1,10 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-+/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
- /*
-  * Device Tree constants for the Texas Instruments DP83869 PHY
-  *
-  * Author: Dan Murphy <dmurphy@ti.com>
-  *
- * Copyright:   (C) 2019 Texas Instruments, Inc.
-+ * Copyright (C) 2015-2024 Texas Instruments Incorporated - https://www.ti.com/
-  */
- 
- #ifndef _DT_BINDINGS_TI_DP83869_H
-diff --git a/include/dt-bindings/phy/phy-qcom-qmp.h b/include/dt-bindings/phy/phy-qcom-qmp.h
-new file mode 100644
-index 000000000000..6b43ea9e0051
--- /dev/null
-+++ b/include/dt-bindings/phy/phy-qcom-qmp.h
-@@ -0,0 +1,24 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
-+/*
-+ * Qualcomm QMP PHY constants
-+ *
-+ * Copyright (C) 2022 Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_PHY_QMP
-+#define _DT_BINDINGS_PHY_QMP
-+
-+/* QMP USB4-USB3-DP clocks */
-+#define QMP_USB43DP_USB3_PIPE_CLK	0
-+#define QMP_USB43DP_DP_LINK_CLK		1
-+#define QMP_USB43DP_DP_VCO_DIV_CLK	2
-+
-+/* QMP USB4-USB3-DP PHYs */
-+#define QMP_USB43DP_USB3_PHY		0
-+#define QMP_USB43DP_DP_PHY		1
-+
-+/* QMP PCIE PHYs */
-+#define QMP_PCIE_PIPE_CLK		0
-+#define QMP_PCIE_PHY_AUX_CLK		1
-+
-+#endif /* _DT_BINDINGS_PHY_QMP */
-diff --git a/include/dt-bindings/phy/phy.h b/include/dt-bindings/phy/phy.h
-index f48c9acf251e..6b901b342348 100644
--- a/include/dt-bindings/phy/phy.h
-+++ b/include/dt-bindings/phy/phy.h
-@@ -22,5 +22,6 @@
- #define PHY_TYPE_QSGMII		9
- #define PHY_TYPE_DPHY		10
- #define PHY_TYPE_CPHY		11
-+#define PHY_TYPE_USXGMII	12
- 
- #endif /* _DT_BINDINGS_PHY */
-diff --git a/include/dt-bindings/pinctrl/amlogic,pinctrl.h b/include/dt-bindings/pinctrl/amlogic,pinctrl.h
-new file mode 100644
-index 000000000000..7d40aecc7147
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/amlogic,pinctrl.h
-@@ -0,0 +1,46 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (c) 2024 Amlogic, Inc. All rights reserved.
-+ * Author: Xianwei Zhao <xianwei.zhao@amlogic.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_AMLOGIC_PINCTRL_H
-+#define _DT_BINDINGS_AMLOGIC_PINCTRL_H
-+/* Normal PIN bank */
-+#define AMLOGIC_GPIO_A		0
-+#define AMLOGIC_GPIO_B		1
-+#define AMLOGIC_GPIO_C		2
-+#define AMLOGIC_GPIO_D		3
-+#define AMLOGIC_GPIO_E		4
-+#define AMLOGIC_GPIO_F		5
-+#define AMLOGIC_GPIO_G		6
-+#define AMLOGIC_GPIO_H		7
-+#define AMLOGIC_GPIO_I		8
-+#define AMLOGIC_GPIO_J		9
-+#define AMLOGIC_GPIO_K		10
-+#define AMLOGIC_GPIO_L		11
-+#define AMLOGIC_GPIO_M		12
-+#define AMLOGIC_GPIO_N		13
-+#define AMLOGIC_GPIO_O		14
-+#define AMLOGIC_GPIO_P		15
-+#define AMLOGIC_GPIO_Q		16
-+#define AMLOGIC_GPIO_R		17
-+#define AMLOGIC_GPIO_S		18
-+#define AMLOGIC_GPIO_T		19
-+#define AMLOGIC_GPIO_U		20
-+#define AMLOGIC_GPIO_V		21
-+#define AMLOGIC_GPIO_W		22
-+#define AMLOGIC_GPIO_X		23
-+#define AMLOGIC_GPIO_Y		24
-+#define AMLOGIC_GPIO_Z		25
-+
-+/* Special PIN bank */
-+#define AMLOGIC_GPIO_DV		26
-+#define AMLOGIC_GPIO_AO		27
-+#define AMLOGIC_GPIO_CC		28
-+#define AMLOGIC_GPIO_TEST_N	29
-+#define AMLOGIC_GPIO_ANALOG	30
-+
-+#define AML_PINMUX(bank, offset, mode)	(((((bank) << 8) + (offset)) << 8) | (mode))
-+
-+#endif /* _DT_BINDINGS_AMLOGIC_PINCTRL_H */
-diff --git a/include/dt-bindings/pinctrl/k3.h b/include/dt-bindings/pinctrl/k3.h
-deleted file mode 100644
-index a5204ab91d3e..000000000000
--- a/include/dt-bindings/pinctrl/k3.h
-+++ /dev/null
-@@ -1,47 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * This header provides constants for pinctrl bindings for TI's K3 SoC
- * family.
- *
- * Copyright (C) 2018-2021 Texas Instruments Incorporated - https://www.ti.com/
- */
-#ifndef _DT_BINDINGS_PINCTRL_TI_K3_H
-#define _DT_BINDINGS_PINCTRL_TI_K3_H
-
-#define PULLUDEN_SHIFT		(16)
-#define PULLTYPESEL_SHIFT	(17)
-#define RXACTIVE_SHIFT		(18)
-
-#define PULL_DISABLE		(1 << PULLUDEN_SHIFT)
-#define PULL_ENABLE		(0 << PULLUDEN_SHIFT)
-
-#define PULL_UP			(1 << PULLTYPESEL_SHIFT | PULL_ENABLE)
-#define PULL_DOWN		(0 << PULLTYPESEL_SHIFT | PULL_ENABLE)
-
-#define INPUT_EN		(1 << RXACTIVE_SHIFT)
-#define INPUT_DISABLE		(0 << RXACTIVE_SHIFT)
-
-/* Only these macros are expected be used directly in device tree files */
-#define PIN_OUTPUT		(INPUT_DISABLE | PULL_DISABLE)
-#define PIN_OUTPUT_PULLUP	(INPUT_DISABLE | PULL_UP)
-#define PIN_OUTPUT_PULLDOWN	(INPUT_DISABLE | PULL_DOWN)
-#define PIN_INPUT		(INPUT_EN | PULL_DISABLE)
-#define PIN_INPUT_PULLUP	(INPUT_EN | PULL_UP)
-#define PIN_INPUT_PULLDOWN	(INPUT_EN | PULL_DOWN)
-
-#define AM65X_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
-#define AM65X_WKUP_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
-
-#define J721E_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
-#define J721E_WKUP_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
-
-#define AM64X_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
-#define AM64X_MCU_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
-
-#define J721S2_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
-#define J721S2_WKUP_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
-
-#define AM62X_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
-#define AM62X_MCU_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
-
-#endif
-diff --git a/include/dt-bindings/pinctrl/mediatek,mt8188-pinfunc.h b/include/dt-bindings/pinctrl/mediatek,mt8188-pinfunc.h
-new file mode 100644
-index 000000000000..2688da2f621f
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/mediatek,mt8188-pinfunc.h
-@@ -0,0 +1,1280 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2022 MediaTek Inc.
-+ * Author: Hui Liu <hui.liu@mediatek.com>
-+ */
-+
-+#ifndef __MEDIATEK_MT8188_PINFUNC_H
-+#define __MEDIATEK_MT8188_PINFUNC_H
-+
-+#include "mt65xx.h"
-+
-+#define PINMUX_GPIO0__FUNC_B_GPIO0 (MTK_PIN_NO(0) | 0)
-+#define PINMUX_GPIO0__FUNC_B0_TP_GPIO0_AO (MTK_PIN_NO(0) | 1)
-+#define PINMUX_GPIO0__FUNC_O_SPIM5_CSB (MTK_PIN_NO(0) | 2)
-+#define PINMUX_GPIO0__FUNC_O_UTXD1 (MTK_PIN_NO(0) | 3)
-+#define PINMUX_GPIO0__FUNC_O_DMIC3_CLK (MTK_PIN_NO(0) | 4)
-+#define PINMUX_GPIO0__FUNC_B0_I2SIN_MCK (MTK_PIN_NO(0) | 5)
-+#define PINMUX_GPIO0__FUNC_O_I2SO2_MCK (MTK_PIN_NO(0) | 6)
-+#define PINMUX_GPIO0__FUNC_B0_DBG_MON_A0 (MTK_PIN_NO(0) | 7)
-+
-+#define PINMUX_GPIO1__FUNC_B_GPIO1 (MTK_PIN_NO(1) | 0)
-+#define PINMUX_GPIO1__FUNC_B0_TP_GPIO1_AO (MTK_PIN_NO(1) | 1)
-+#define PINMUX_GPIO1__FUNC_O_SPIM5_CLK (MTK_PIN_NO(1) | 2)
-+#define PINMUX_GPIO1__FUNC_I1_URXD1 (MTK_PIN_NO(1) | 3)
-+#define PINMUX_GPIO1__FUNC_I0_DMIC3_DAT (MTK_PIN_NO(1) | 4)
-+#define PINMUX_GPIO1__FUNC_B0_I2SIN_BCK (MTK_PIN_NO(1) | 5)
-+#define PINMUX_GPIO1__FUNC_B0_I2SO2_BCK (MTK_PIN_NO(1) | 6)
-+#define PINMUX_GPIO1__FUNC_B0_DBG_MON_A1 (MTK_PIN_NO(1) | 7)
-+
-+#define PINMUX_GPIO2__FUNC_B_GPIO2 (MTK_PIN_NO(2) | 0)
-+#define PINMUX_GPIO2__FUNC_B0_TP_GPIO2_AO (MTK_PIN_NO(2) | 1)
-+#define PINMUX_GPIO2__FUNC_B0_SPIM5_MOSI (MTK_PIN_NO(2) | 2)
-+#define PINMUX_GPIO2__FUNC_O_URTS1 (MTK_PIN_NO(2) | 3)
-+#define PINMUX_GPIO2__FUNC_I0_DMIC3_DAT_R (MTK_PIN_NO(2) | 4)
-+#define PINMUX_GPIO2__FUNC_B0_I2SIN_WS (MTK_PIN_NO(2) | 5)
-+#define PINMUX_GPIO2__FUNC_B0_I2SO2_WS (MTK_PIN_NO(2) | 6)
-+#define PINMUX_GPIO2__FUNC_B0_DBG_MON_A2 (MTK_PIN_NO(2) | 7)
-+
-+#define PINMUX_GPIO3__FUNC_B_GPIO3 (MTK_PIN_NO(3) | 0)
-+#define PINMUX_GPIO3__FUNC_B0_TP_GPIO3_AO (MTK_PIN_NO(3) | 1)
-+#define PINMUX_GPIO3__FUNC_B0_SPIM5_MISO (MTK_PIN_NO(3) | 2)
-+#define PINMUX_GPIO3__FUNC_I1_UCTS1 (MTK_PIN_NO(3) | 3)
-+#define PINMUX_GPIO3__FUNC_O_DMIC4_CLK (MTK_PIN_NO(3) | 4)
-+#define PINMUX_GPIO3__FUNC_I0_I2SIN_D0 (MTK_PIN_NO(3) | 5)
-+#define PINMUX_GPIO3__FUNC_O_I2SO2_D0 (MTK_PIN_NO(3) | 6)
-+#define PINMUX_GPIO3__FUNC_B0_DBG_MON_A3 (MTK_PIN_NO(3) | 7)
-+
-+#define PINMUX_GPIO4__FUNC_B_GPIO4 (MTK_PIN_NO(4) | 0)
-+#define PINMUX_GPIO4__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(4) | 1)
-+#define PINMUX_GPIO4__FUNC_I0_SPDIF_IN2 (MTK_PIN_NO(4) | 2)
-+#define PINMUX_GPIO4__FUNC_O_I2SO1_MCK (MTK_PIN_NO(4) | 3)
-+#define PINMUX_GPIO4__FUNC_I0_DMIC4_DAT (MTK_PIN_NO(4) | 4)
-+#define PINMUX_GPIO4__FUNC_I0_I2SIN_D1 (MTK_PIN_NO(4) | 5)
-+#define PINMUX_GPIO4__FUNC_O_I2SO2_D1 (MTK_PIN_NO(4) | 6)
-+#define PINMUX_GPIO4__FUNC_B0_DBG_MON_A4 (MTK_PIN_NO(4) | 7)
-+
-+#define PINMUX_GPIO5__FUNC_B_GPIO5 (MTK_PIN_NO(5) | 0)
-+#define PINMUX_GPIO5__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(5) | 1)
-+#define PINMUX_GPIO5__FUNC_I0_SPDIF_IN1 (MTK_PIN_NO(5) | 2)
-+#define PINMUX_GPIO5__FUNC_O_I2SO1_BCK (MTK_PIN_NO(5) | 3)
-+#define PINMUX_GPIO5__FUNC_I0_DMIC4_DAT_R (MTK_PIN_NO(5) | 4)
-+#define PINMUX_GPIO5__FUNC_I0_I2SIN_D2 (MTK_PIN_NO(5) | 5)
-+#define PINMUX_GPIO5__FUNC_O_I2SO2_D2 (MTK_PIN_NO(5) | 6)
-+#define PINMUX_GPIO5__FUNC_B0_DBG_MON_A5 (MTK_PIN_NO(5) | 7)
-+
-+#define PINMUX_GPIO6__FUNC_B_GPIO6 (MTK_PIN_NO(6) | 0)
-+#define PINMUX_GPIO6__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(6) | 1)
-+#define PINMUX_GPIO6__FUNC_I0_SPDIF_IN0 (MTK_PIN_NO(6) | 2)
-+#define PINMUX_GPIO6__FUNC_O_I2SO1_WS (MTK_PIN_NO(6) | 3)
-+#define PINMUX_GPIO6__FUNC_O_DMIC1_CLK (MTK_PIN_NO(6) | 4)
-+#define PINMUX_GPIO6__FUNC_I0_I2SIN_D3 (MTK_PIN_NO(6) | 5)
-+#define PINMUX_GPIO6__FUNC_O_I2SO2_D3 (MTK_PIN_NO(6) | 6)
-+#define PINMUX_GPIO6__FUNC_B0_MD32_0_GPIO0 (MTK_PIN_NO(6) | 7)
-+
-+#define PINMUX_GPIO7__FUNC_B_GPIO7 (MTK_PIN_NO(7) | 0)
-+#define PINMUX_GPIO7__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(7) | 1)
-+#define PINMUX_GPIO7__FUNC_O_SPIM3_CSB (MTK_PIN_NO(7) | 2)
-+#define PINMUX_GPIO7__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(7) | 3)
-+#define PINMUX_GPIO7__FUNC_I0_DMIC1_DAT (MTK_PIN_NO(7) | 4)
-+#define PINMUX_GPIO7__FUNC_O_CMVREF0 (MTK_PIN_NO(7) | 5)
-+#define PINMUX_GPIO7__FUNC_O_CLKM0 (MTK_PIN_NO(7) | 6)
-+#define PINMUX_GPIO7__FUNC_B0_DBG_MON_A6 (MTK_PIN_NO(7) | 7)
-+
-+#define PINMUX_GPIO8__FUNC_B_GPIO8 (MTK_PIN_NO(8) | 0)
-+#define PINMUX_GPIO8__FUNC_B0_TP_GPIO0_AO (MTK_PIN_NO(8) | 1)
-+#define PINMUX_GPIO8__FUNC_O_SPIM3_CLK (MTK_PIN_NO(8) | 2)
-+#define PINMUX_GPIO8__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(8) | 3)
-+#define PINMUX_GPIO8__FUNC_I0_DMIC1_DAT_R (MTK_PIN_NO(8) | 4)
-+#define PINMUX_GPIO8__FUNC_O_CMVREF1 (MTK_PIN_NO(8) | 5)
-+#define PINMUX_GPIO8__FUNC_O_CLKM1 (MTK_PIN_NO(8) | 6)
-+#define PINMUX_GPIO8__FUNC_B0_DBG_MON_A7 (MTK_PIN_NO(8) | 7)
-+
-+#define PINMUX_GPIO9__FUNC_B_GPIO9 (MTK_PIN_NO(9) | 0)
-+#define PINMUX_GPIO9__FUNC_B0_TP_GPIO1_AO (MTK_PIN_NO(9) | 1)
-+#define PINMUX_GPIO9__FUNC_B0_SPIM3_MOSI (MTK_PIN_NO(9) | 2)
-+#define PINMUX_GPIO9__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(9) | 3)
-+#define PINMUX_GPIO9__FUNC_O_DMIC2_CLK (MTK_PIN_NO(9) | 4)
-+#define PINMUX_GPIO9__FUNC_O_CMFLASH0 (MTK_PIN_NO(9) | 5)
-+#define PINMUX_GPIO9__FUNC_O_PWM_0 (MTK_PIN_NO(9) | 6)
-+#define PINMUX_GPIO9__FUNC_B0_DBG_MON_A8 (MTK_PIN_NO(9) | 7)
-+
-+#define PINMUX_GPIO10__FUNC_B_GPIO10 (MTK_PIN_NO(10) | 0)
-+#define PINMUX_GPIO10__FUNC_B0_TP_GPIO2_AO (MTK_PIN_NO(10) | 1)
-+#define PINMUX_GPIO10__FUNC_B0_SPIM3_MISO (MTK_PIN_NO(10) | 2)
-+#define PINMUX_GPIO10__FUNC_I0_TDMIN_DI (MTK_PIN_NO(10) | 3)
-+#define PINMUX_GPIO10__FUNC_I0_DMIC2_DAT (MTK_PIN_NO(10) | 4)
-+#define PINMUX_GPIO10__FUNC_O_CMFLASH1 (MTK_PIN_NO(10) | 5)
-+#define PINMUX_GPIO10__FUNC_O_PWM_1 (MTK_PIN_NO(10) | 6)
-+#define PINMUX_GPIO10__FUNC_B0_DBG_MON_A9 (MTK_PIN_NO(10) | 7)
-+
-+#define PINMUX_GPIO11__FUNC_B_GPIO11 (MTK_PIN_NO(11) | 0)
-+#define PINMUX_GPIO11__FUNC_B0_TP_GPIO3_AO (MTK_PIN_NO(11) | 1)
-+#define PINMUX_GPIO11__FUNC_O_SPDIF_OUT (MTK_PIN_NO(11) | 2)
-+#define PINMUX_GPIO11__FUNC_O_I2SO1_D0 (MTK_PIN_NO(11) | 3)
-+#define PINMUX_GPIO11__FUNC_I0_DMIC2_DAT_R (MTK_PIN_NO(11) | 4)
-+#define PINMUX_GPIO11__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(11) | 5)
-+#define PINMUX_GPIO11__FUNC_O_CMVREF6 (MTK_PIN_NO(11) | 6)
-+#define PINMUX_GPIO11__FUNC_B0_DBG_MON_A10 (MTK_PIN_NO(11) | 7)
-+
-+#define PINMUX_GPIO12__FUNC_B_GPIO12 (MTK_PIN_NO(12) | 0)
-+#define PINMUX_GPIO12__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(12) | 1)
-+#define PINMUX_GPIO12__FUNC_O_SPIM4_CSB (MTK_PIN_NO(12) | 2)
-+#define PINMUX_GPIO12__FUNC_B1_JTMS_SEL3 (MTK_PIN_NO(12) | 3)
-+#define PINMUX_GPIO12__FUNC_B1_APU_JTAG_TMS (MTK_PIN_NO(12) | 4)
-+#define PINMUX_GPIO12__FUNC_I0_VPU_UDI_TMS (MTK_PIN_NO(12) | 5)
-+#define PINMUX_GPIO12__FUNC_I0_IPU_JTAG_TMS (MTK_PIN_NO(12) | 6)
-+#define PINMUX_GPIO12__FUNC_I0_HDMITX20_HTPLG (MTK_PIN_NO(12) | 7)
-+
-+#define PINMUX_GPIO13__FUNC_B_GPIO13 (MTK_PIN_NO(13) | 0)
-+#define PINMUX_GPIO13__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(13) | 1)
-+#define PINMUX_GPIO13__FUNC_O_SPIM4_CLK (MTK_PIN_NO(13) | 2)
-+#define PINMUX_GPIO13__FUNC_I0_JTCK_SEL3 (MTK_PIN_NO(13) | 3)
-+#define PINMUX_GPIO13__FUNC_I0_APU_JTAG_TCK (MTK_PIN_NO(13) | 4)
-+#define PINMUX_GPIO13__FUNC_I0_VPU_UDI_TCK (MTK_PIN_NO(13) | 5)
-+#define PINMUX_GPIO13__FUNC_I0_IPU_JTAG_TCK (MTK_PIN_NO(13) | 6)
-+#define PINMUX_GPIO13__FUNC_B1_HDMITX20_CEC (MTK_PIN_NO(13) | 7)
-+
-+#define PINMUX_GPIO14__FUNC_B_GPIO14 (MTK_PIN_NO(14) | 0)
-+#define PINMUX_GPIO14__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(14) | 1)
-+#define PINMUX_GPIO14__FUNC_B0_SPIM4_MOSI (MTK_PIN_NO(14) | 2)
-+#define PINMUX_GPIO14__FUNC_I1_JTDI_SEL3 (MTK_PIN_NO(14) | 3)
-+#define PINMUX_GPIO14__FUNC_I1_APU_JTAG_TDI (MTK_PIN_NO(14) | 4)
-+#define PINMUX_GPIO14__FUNC_I0_VPU_UDI_TDI (MTK_PIN_NO(14) | 5)
-+#define PINMUX_GPIO14__FUNC_I0_IPU_JTAG_TDI (MTK_PIN_NO(14) | 6)
-+#define PINMUX_GPIO14__FUNC_B1_HDMITX20_SCL (MTK_PIN_NO(14) | 7)
-+
-+#define PINMUX_GPIO15__FUNC_B_GPIO15 (MTK_PIN_NO(15) | 0)
-+#define PINMUX_GPIO15__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(15) | 1)
-+#define PINMUX_GPIO15__FUNC_B0_SPIM4_MISO (MTK_PIN_NO(15) | 2)
-+#define PINMUX_GPIO15__FUNC_O_JTDO_SEL3 (MTK_PIN_NO(15) | 3)
-+#define PINMUX_GPIO15__FUNC_O_APU_JTAG_TDO (MTK_PIN_NO(15) | 4)
-+#define PINMUX_GPIO15__FUNC_O_VPU_UDI_TDO (MTK_PIN_NO(15) | 5)
-+#define PINMUX_GPIO15__FUNC_O_IPU_JTAG_TDO (MTK_PIN_NO(15) | 6)
-+#define PINMUX_GPIO15__FUNC_B1_HDMITX20_SDA (MTK_PIN_NO(15) | 7)
-+
-+#define PINMUX_GPIO16__FUNC_B_GPIO16 (MTK_PIN_NO(16) | 0)
-+#define PINMUX_GPIO16__FUNC_B0_TP_GPIO0_AO (MTK_PIN_NO(16) | 1)
-+#define PINMUX_GPIO16__FUNC_O_UTXD3 (MTK_PIN_NO(16) | 2)
-+#define PINMUX_GPIO16__FUNC_I1_JTRSTn_SEL3 (MTK_PIN_NO(16) | 3)
-+#define PINMUX_GPIO16__FUNC_I0_APU_JTAG_TRST (MTK_PIN_NO(16) | 4)
-+#define PINMUX_GPIO16__FUNC_I0_VPU_UDI_NTRST (MTK_PIN_NO(16) | 5)
-+#define PINMUX_GPIO16__FUNC_I0_IPU_JTAG_TRST (MTK_PIN_NO(16) | 6)
-+#define PINMUX_GPIO16__FUNC_O_HDMITX20_PWR5V (MTK_PIN_NO(16) | 7)
-+
-+#define PINMUX_GPIO17__FUNC_B_GPIO17 (MTK_PIN_NO(17) | 0)
-+#define PINMUX_GPIO17__FUNC_B0_TP_GPIO1_AO (MTK_PIN_NO(17) | 1)
-+#define PINMUX_GPIO17__FUNC_I1_URXD3 (MTK_PIN_NO(17) | 2)
-+#define PINMUX_GPIO17__FUNC_O_CMFLASH2 (MTK_PIN_NO(17) | 3)
-+#define PINMUX_GPIO17__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(17) | 4)
-+#define PINMUX_GPIO17__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(17) | 5)
-+#define PINMUX_GPIO17__FUNC_O_CMVREF7 (MTK_PIN_NO(17) | 6)
-+#define PINMUX_GPIO17__FUNC_B0_MD32_0_GPIO1 (MTK_PIN_NO(17) | 7)
-+
-+#define PINMUX_GPIO18__FUNC_B_GPIO18 (MTK_PIN_NO(18) | 0)
-+#define PINMUX_GPIO18__FUNC_B0_TP_GPIO2_AO (MTK_PIN_NO(18) | 1)
-+#define PINMUX_GPIO18__FUNC_O_CMFLASH0 (MTK_PIN_NO(18) | 2)
-+#define PINMUX_GPIO18__FUNC_O_CMVREF4 (MTK_PIN_NO(18) | 3)
-+#define PINMUX_GPIO18__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(18) | 4)
-+#define PINMUX_GPIO18__FUNC_O_UTXD1 (MTK_PIN_NO(18) | 5)
-+#define PINMUX_GPIO18__FUNC_O_TP_UTXD1_AO (MTK_PIN_NO(18) | 6)
-+#define PINMUX_GPIO18__FUNC_B0_DBG_MON_A11 (MTK_PIN_NO(18) | 7)
-+
-+#define PINMUX_GPIO19__FUNC_B_GPIO19 (MTK_PIN_NO(19) | 0)
-+#define PINMUX_GPIO19__FUNC_B0_TP_GPIO3_AO (MTK_PIN_NO(19) | 1)
-+#define PINMUX_GPIO19__FUNC_O_CMFLASH1 (MTK_PIN_NO(19) | 2)
-+#define PINMUX_GPIO19__FUNC_O_CMVREF5 (MTK_PIN_NO(19) | 3)
-+#define PINMUX_GPIO19__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(19) | 4)
-+#define PINMUX_GPIO19__FUNC_I1_URXD1 (MTK_PIN_NO(19) | 5)
-+#define PINMUX_GPIO19__FUNC_I1_TP_URXD1_AO (MTK_PIN_NO(19) | 6)
-+#define PINMUX_GPIO19__FUNC_B0_DBG_MON_A12 (MTK_PIN_NO(19) | 7)
-+
-+#define PINMUX_GPIO20__FUNC_B_GPIO20 (MTK_PIN_NO(20) | 0)
-+#define PINMUX_GPIO20__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(20) | 1)
-+#define PINMUX_GPIO20__FUNC_O_CMFLASH2 (MTK_PIN_NO(20) | 2)
-+#define PINMUX_GPIO20__FUNC_O_CLKM2 (MTK_PIN_NO(20) | 3)
-+#define PINMUX_GPIO20__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(20) | 4)
-+#define PINMUX_GPIO20__FUNC_O_URTS1 (MTK_PIN_NO(20) | 5)
-+#define PINMUX_GPIO20__FUNC_O_TP_URTS1_AO (MTK_PIN_NO(20) | 6)
-+#define PINMUX_GPIO20__FUNC_B0_DBG_MON_A13 (MTK_PIN_NO(20) | 7)
-+
-+#define PINMUX_GPIO21__FUNC_B_GPIO21 (MTK_PIN_NO(21) | 0)
-+#define PINMUX_GPIO21__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(21) | 1)
-+#define PINMUX_GPIO21__FUNC_O_CMFLASH3 (MTK_PIN_NO(21) | 2)
-+#define PINMUX_GPIO21__FUNC_O_CLKM3 (MTK_PIN_NO(21) | 3)
-+#define PINMUX_GPIO21__FUNC_I0_TDMIN_DI (MTK_PIN_NO(21) | 4)
-+#define PINMUX_GPIO21__FUNC_I1_UCTS1 (MTK_PIN_NO(21) | 5)
-+#define PINMUX_GPIO21__FUNC_I1_TP_UCTS1_AO (MTK_PIN_NO(21) | 6)
-+#define PINMUX_GPIO21__FUNC_B0_DBG_MON_A14 (MTK_PIN_NO(21) | 7)
-+
-+#define PINMUX_GPIO22__FUNC_B_GPIO22 (MTK_PIN_NO(22) | 0)
-+#define PINMUX_GPIO22__FUNC_O_CMMCLK0 (MTK_PIN_NO(22) | 1)
-+#define PINMUX_GPIO22__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(22) | 5)
-+#define PINMUX_GPIO22__FUNC_B0_DBG_MON_A15 (MTK_PIN_NO(22) | 7)
-+
-+#define PINMUX_GPIO23__FUNC_B_GPIO23 (MTK_PIN_NO(23) | 0)
-+#define PINMUX_GPIO23__FUNC_O_CMMCLK1 (MTK_PIN_NO(23) | 1)
-+#define PINMUX_GPIO23__FUNC_O_PWM_2 (MTK_PIN_NO(23) | 3)
-+#define PINMUX_GPIO23__FUNC_B1_PCIE_PHY_I2C_SCL (MTK_PIN_NO(23) | 4)
-+#define PINMUX_GPIO23__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(23) | 5)
-+#define PINMUX_GPIO23__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(23) | 6)
-+#define PINMUX_GPIO23__FUNC_B0_DBG_MON_A16 (MTK_PIN_NO(23) | 7)
-+
-+#define PINMUX_GPIO24__FUNC_B_GPIO24 (MTK_PIN_NO(24) | 0)
-+#define PINMUX_GPIO24__FUNC_O_CMMCLK2 (MTK_PIN_NO(24) | 1)
-+#define PINMUX_GPIO24__FUNC_O_PWM_3 (MTK_PIN_NO(24) | 3)
-+#define PINMUX_GPIO24__FUNC_B1_PCIE_PHY_I2C_SDA (MTK_PIN_NO(24) | 4)
-+#define PINMUX_GPIO24__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(24) | 5)
-+#define PINMUX_GPIO24__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(24) | 6)
-+#define PINMUX_GPIO24__FUNC_B0_MD32_0_GPIO2 (MTK_PIN_NO(24) | 7)
-+
-+#define PINMUX_GPIO25__FUNC_B_GPIO25 (MTK_PIN_NO(25) | 0)
-+#define PINMUX_GPIO25__FUNC_O_LCM_RST (MTK_PIN_NO(25) | 1)
-+#define PINMUX_GPIO25__FUNC_O_LCM1_RST (MTK_PIN_NO(25) | 2)
-+#define PINMUX_GPIO25__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(25) | 3)
-+
-+#define PINMUX_GPIO26__FUNC_B_GPIO26 (MTK_PIN_NO(26) | 0)
-+#define PINMUX_GPIO26__FUNC_I0_DSI_TE (MTK_PIN_NO(26) | 1)
-+#define PINMUX_GPIO26__FUNC_I0_DSI1_TE (MTK_PIN_NO(26) | 2)
-+#define PINMUX_GPIO26__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(26) | 3)
-+
-+#define PINMUX_GPIO27__FUNC_B_GPIO27 (MTK_PIN_NO(27) | 0)
-+#define PINMUX_GPIO27__FUNC_O_LCM1_RST (MTK_PIN_NO(27) | 1)
-+#define PINMUX_GPIO27__FUNC_O_LCM_RST (MTK_PIN_NO(27) | 2)
-+#define PINMUX_GPIO27__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(27) | 3)
-+#define PINMUX_GPIO27__FUNC_O_CMVREF2 (MTK_PIN_NO(27) | 4)
-+#define PINMUX_GPIO27__FUNC_O_mbistwriteen_trigger (MTK_PIN_NO(27) | 5)
-+#define PINMUX_GPIO27__FUNC_O_PWM_2 (MTK_PIN_NO(27) | 6)
-+#define PINMUX_GPIO27__FUNC_B0_DBG_MON_A17 (MTK_PIN_NO(27) | 7)
-+
-+#define PINMUX_GPIO28__FUNC_B_GPIO28 (MTK_PIN_NO(28) | 0)
-+#define PINMUX_GPIO28__FUNC_I0_DSI1_TE (MTK_PIN_NO(28) | 1)
-+#define PINMUX_GPIO28__FUNC_I0_DSI_TE (MTK_PIN_NO(28) | 2)
-+#define PINMUX_GPIO28__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(28) | 3)
-+#define PINMUX_GPIO28__FUNC_O_CMVREF3 (MTK_PIN_NO(28) | 4)
-+#define PINMUX_GPIO28__FUNC_O_mbistreaden_trigger (MTK_PIN_NO(28) | 5)
-+#define PINMUX_GPIO28__FUNC_O_PWM_3 (MTK_PIN_NO(28) | 6)
-+#define PINMUX_GPIO28__FUNC_B0_DBG_MON_A18 (MTK_PIN_NO(28) | 7)
-+
-+#define PINMUX_GPIO29__FUNC_B_GPIO29 (MTK_PIN_NO(29) | 0)
-+#define PINMUX_GPIO29__FUNC_O_DISP_PWM0 (MTK_PIN_NO(29) | 1)
-+#define PINMUX_GPIO29__FUNC_O_DISP_PWM1 (MTK_PIN_NO(29) | 2)
-+
-+#define PINMUX_GPIO30__FUNC_B_GPIO30 (MTK_PIN_NO(30) | 0)
-+#define PINMUX_GPIO30__FUNC_O_DISP_PWM1 (MTK_PIN_NO(30) | 1)
-+#define PINMUX_GPIO30__FUNC_O_DISP_PWM0 (MTK_PIN_NO(30) | 2)
-+#define PINMUX_GPIO30__FUNC_O_CMFLASH3 (MTK_PIN_NO(30) | 3)
-+#define PINMUX_GPIO30__FUNC_O_PWM_1 (MTK_PIN_NO(30) | 4)
-+#define PINMUX_GPIO30__FUNC_B0_DBG_MON_A19 (MTK_PIN_NO(30) | 7)
-+
-+#define PINMUX_GPIO31__FUNC_B_GPIO31 (MTK_PIN_NO(31) | 0)
-+#define PINMUX_GPIO31__FUNC_O_UTXD0 (MTK_PIN_NO(31) | 1)
-+#define PINMUX_GPIO31__FUNC_O_TP_UTXD1_AO (MTK_PIN_NO(31) | 2)
-+#define PINMUX_GPIO31__FUNC_O_ADSP_UTXD0 (MTK_PIN_NO(31) | 3)
-+#define PINMUX_GPIO31__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(31) | 4)
-+#define PINMUX_GPIO31__FUNC_O_MD32_0_TXD (MTK_PIN_NO(31) | 5)
-+#define PINMUX_GPIO31__FUNC_O_MD32_1_TXD (MTK_PIN_NO(31) | 6)
-+#define PINMUX_GPIO31__FUNC_O_SSPM_UTXD_AO (MTK_PIN_NO(31) | 7)
-+
-+#define PINMUX_GPIO32__FUNC_B_GPIO32 (MTK_PIN_NO(32) | 0)
-+#define PINMUX_GPIO32__FUNC_I1_URXD0 (MTK_PIN_NO(32) | 1)
-+#define PINMUX_GPIO32__FUNC_I1_TP_URXD1_AO (MTK_PIN_NO(32) | 2)
-+#define PINMUX_GPIO32__FUNC_I1_ADSP_URXD0 (MTK_PIN_NO(32) | 3)
-+#define PINMUX_GPIO32__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(32) | 4)
-+#define PINMUX_GPIO32__FUNC_I1_MD32_0_RXD (MTK_PIN_NO(32) | 5)
-+#define PINMUX_GPIO32__FUNC_I1_MD32_1_RXD (MTK_PIN_NO(32) | 6)
-+#define PINMUX_GPIO32__FUNC_I1_SSPM_URXD_AO (MTK_PIN_NO(32) | 7)
-+
-+#define PINMUX_GPIO33__FUNC_B_GPIO33 (MTK_PIN_NO(33) | 0)
-+#define PINMUX_GPIO33__FUNC_O_UTXD1 (MTK_PIN_NO(33) | 1)
-+#define PINMUX_GPIO33__FUNC_O_URTS2 (MTK_PIN_NO(33) | 2)
-+#define PINMUX_GPIO33__FUNC_O_ADSP_UTXD0 (MTK_PIN_NO(33) | 3)
-+#define PINMUX_GPIO33__FUNC_O_TP_UTXD1_AO (MTK_PIN_NO(33) | 4)
-+#define PINMUX_GPIO33__FUNC_O_mbistwriteen_trigger (MTK_PIN_NO(33) | 5)
-+#define PINMUX_GPIO33__FUNC_O_MD32_0_TXD (MTK_PIN_NO(33) | 6)
-+#define PINMUX_GPIO33__FUNC_O_SSPM_UTXD_AO (MTK_PIN_NO(33) | 7)
-+
-+#define PINMUX_GPIO34__FUNC_B_GPIO34 (MTK_PIN_NO(34) | 0)
-+#define PINMUX_GPIO34__FUNC_I1_URXD1 (MTK_PIN_NO(34) | 1)
-+#define PINMUX_GPIO34__FUNC_I1_UCTS2 (MTK_PIN_NO(34) | 2)
-+#define PINMUX_GPIO34__FUNC_I1_ADSP_URXD0 (MTK_PIN_NO(34) | 3)
-+#define PINMUX_GPIO34__FUNC_I1_TP_URXD1_AO (MTK_PIN_NO(34) | 4)
-+#define PINMUX_GPIO34__FUNC_O_mbistreaden_trigger (MTK_PIN_NO(34) | 5)
-+#define PINMUX_GPIO34__FUNC_I1_MD32_0_RXD (MTK_PIN_NO(34) | 6)
-+#define PINMUX_GPIO34__FUNC_I1_SSPM_URXD_AO (MTK_PIN_NO(34) | 7)
-+
-+#define PINMUX_GPIO35__FUNC_B_GPIO35 (MTK_PIN_NO(35) | 0)
-+#define PINMUX_GPIO35__FUNC_O_UTXD2 (MTK_PIN_NO(35) | 1)
-+#define PINMUX_GPIO35__FUNC_O_URTS1 (MTK_PIN_NO(35) | 2)
-+#define PINMUX_GPIO35__FUNC_O_ADSP_UTXD0 (MTK_PIN_NO(35) | 3)
-+#define PINMUX_GPIO35__FUNC_O_TP_URTS1_AO (MTK_PIN_NO(35) | 4)
-+#define PINMUX_GPIO35__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(35) | 5)
-+#define PINMUX_GPIO35__FUNC_O_MD32_1_TXD (MTK_PIN_NO(35) | 6)
-+#define PINMUX_GPIO35__FUNC_B0_DBG_MON_A20 (MTK_PIN_NO(35) | 7)
-+
-+#define PINMUX_GPIO36__FUNC_B_GPIO36 (MTK_PIN_NO(36) | 0)
-+#define PINMUX_GPIO36__FUNC_I1_URXD2 (MTK_PIN_NO(36) | 1)
-+#define PINMUX_GPIO36__FUNC_I1_UCTS1 (MTK_PIN_NO(36) | 2)
-+#define PINMUX_GPIO36__FUNC_I1_ADSP_URXD0 (MTK_PIN_NO(36) | 3)
-+#define PINMUX_GPIO36__FUNC_I1_TP_UCTS1_AO (MTK_PIN_NO(36) | 4)
-+#define PINMUX_GPIO36__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(36) | 5)
-+#define PINMUX_GPIO36__FUNC_I1_MD32_1_RXD (MTK_PIN_NO(36) | 6)
-+#define PINMUX_GPIO36__FUNC_B0_DBG_MON_A21 (MTK_PIN_NO(36) | 7)
-+
-+#define PINMUX_GPIO37__FUNC_B_GPIO37 (MTK_PIN_NO(37) | 0)
-+#define PINMUX_GPIO37__FUNC_B1_JTMS_SEL1 (MTK_PIN_NO(37) | 1)
-+#define PINMUX_GPIO37__FUNC_I0_UDI_TMS (MTK_PIN_NO(37) | 2)
-+#define PINMUX_GPIO37__FUNC_I1_SPM_JTAG_TMS (MTK_PIN_NO(37) | 3)
-+#define PINMUX_GPIO37__FUNC_I1_ADSP_JTAG0_TMS (MTK_PIN_NO(37) | 4)
-+#define PINMUX_GPIO37__FUNC_I1_SCP_JTAG0_TMS (MTK_PIN_NO(37) | 5)
-+#define PINMUX_GPIO37__FUNC_I1_CCU0_JTAG_TMS (MTK_PIN_NO(37) | 6)
-+#define PINMUX_GPIO37__FUNC_I1_MCUPM_JTAG_TMS (MTK_PIN_NO(37) | 7)
-+
-+#define PINMUX_GPIO38__FUNC_B_GPIO38 (MTK_PIN_NO(38) | 0)
-+#define PINMUX_GPIO38__FUNC_I0_JTCK_SEL1 (MTK_PIN_NO(38) | 1)
-+#define PINMUX_GPIO38__FUNC_I0_UDI_TCK (MTK_PIN_NO(38) | 2)
-+#define PINMUX_GPIO38__FUNC_I1_SPM_JTAG_TCK (MTK_PIN_NO(38) | 3)
-+#define PINMUX_GPIO38__FUNC_I0_ADSP_JTAG0_TCK (MTK_PIN_NO(38) | 4)
-+#define PINMUX_GPIO38__FUNC_I1_SCP_JTAG0_TCK (MTK_PIN_NO(38) | 5)
-+#define PINMUX_GPIO38__FUNC_I1_CCU0_JTAG_TCK (MTK_PIN_NO(38) | 6)
-+#define PINMUX_GPIO38__FUNC_I1_MCUPM_JTAG_TCK (MTK_PIN_NO(38) | 7)
-+
-+#define PINMUX_GPIO39__FUNC_B_GPIO39 (MTK_PIN_NO(39) | 0)
-+#define PINMUX_GPIO39__FUNC_I1_JTDI_SEL1 (MTK_PIN_NO(39) | 1)
-+#define PINMUX_GPIO39__FUNC_I0_UDI_TDI (MTK_PIN_NO(39) | 2)
-+#define PINMUX_GPIO39__FUNC_I1_SPM_JTAG_TDI (MTK_PIN_NO(39) | 3)
-+#define PINMUX_GPIO39__FUNC_I1_ADSP_JTAG0_TDI (MTK_PIN_NO(39) | 4)
-+#define PINMUX_GPIO39__FUNC_I1_SCP_JTAG0_TDI (MTK_PIN_NO(39) | 5)
-+#define PINMUX_GPIO39__FUNC_I1_CCU0_JTAG_TDI (MTK_PIN_NO(39) | 6)
-+#define PINMUX_GPIO39__FUNC_I1_MCUPM_JTAG_TDI (MTK_PIN_NO(39) | 7)
-+
-+#define PINMUX_GPIO40__FUNC_B_GPIO40 (MTK_PIN_NO(40) | 0)
-+#define PINMUX_GPIO40__FUNC_O_JTDO_SEL1 (MTK_PIN_NO(40) | 1)
-+#define PINMUX_GPIO40__FUNC_O_UDI_TDO (MTK_PIN_NO(40) | 2)
-+#define PINMUX_GPIO40__FUNC_O_SPM_JTAG_TDO (MTK_PIN_NO(40) | 3)
-+#define PINMUX_GPIO40__FUNC_O_ADSP_JTAG0_TDO (MTK_PIN_NO(40) | 4)
-+#define PINMUX_GPIO40__FUNC_O_SCP_JTAG0_TDO (MTK_PIN_NO(40) | 5)
-+#define PINMUX_GPIO40__FUNC_O_CCU0_JTAG_TDO (MTK_PIN_NO(40) | 6)
-+#define PINMUX_GPIO40__FUNC_O_MCUPM_JTAG_TDO (MTK_PIN_NO(40) | 7)
-+
-+#define PINMUX_GPIO41__FUNC_B_GPIO41 (MTK_PIN_NO(41) | 0)
-+#define PINMUX_GPIO41__FUNC_I1_JTRSTn_SEL1 (MTK_PIN_NO(41) | 1)
-+#define PINMUX_GPIO41__FUNC_I0_UDI_NTRST (MTK_PIN_NO(41) | 2)
-+#define PINMUX_GPIO41__FUNC_I0_SPM_JTAG_TRSTN (MTK_PIN_NO(41) | 3)
-+#define PINMUX_GPIO41__FUNC_I1_ADSP_JTAG0_TRSTN (MTK_PIN_NO(41) | 4)
-+#define PINMUX_GPIO41__FUNC_I0_SCP_JTAG0_TRSTN (MTK_PIN_NO(41) | 5)
-+#define PINMUX_GPIO41__FUNC_I1_CCU0_JTAG_TRST (MTK_PIN_NO(41) | 6)
-+#define PINMUX_GPIO41__FUNC_I0_MCUPM_JTAG_TRSTN (MTK_PIN_NO(41) | 7)
-+
-+#define PINMUX_GPIO42__FUNC_B_GPIO42 (MTK_PIN_NO(42) | 0)
-+#define PINMUX_GPIO42__FUNC_B1_KPCOL0 (MTK_PIN_NO(42) | 1)
-+
-+#define PINMUX_GPIO43__FUNC_B_GPIO43 (MTK_PIN_NO(43) | 0)
-+#define PINMUX_GPIO43__FUNC_B1_KPCOL1 (MTK_PIN_NO(43) | 1)
-+#define PINMUX_GPIO43__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(43) | 2)
-+#define PINMUX_GPIO43__FUNC_O_CMFLASH2 (MTK_PIN_NO(43) | 3)
-+#define PINMUX_GPIO43__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(43) | 4)
-+#define PINMUX_GPIO43__FUNC_O_mbistwriteen_trigger (MTK_PIN_NO(43) | 7)
-+
-+#define PINMUX_GPIO44__FUNC_B_GPIO44 (MTK_PIN_NO(44) | 0)
-+#define PINMUX_GPIO44__FUNC_B1_KPROW0 (MTK_PIN_NO(44) | 1)
-+
-+#define PINMUX_GPIO45__FUNC_B_GPIO45 (MTK_PIN_NO(45) | 0)
-+#define PINMUX_GPIO45__FUNC_B1_KPROW1 (MTK_PIN_NO(45) | 1)
-+#define PINMUX_GPIO45__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(45) | 2)
-+#define PINMUX_GPIO45__FUNC_O_CMFLASH3 (MTK_PIN_NO(45) | 3)
-+#define PINMUX_GPIO45__FUNC_B0_I2SIN_MCK (MTK_PIN_NO(45) | 4)
-+#define PINMUX_GPIO45__FUNC_O_mbistreaden_trigger (MTK_PIN_NO(45) | 7)
-+
-+#define PINMUX_GPIO46__FUNC_B_GPIO46 (MTK_PIN_NO(46) | 0)
-+#define PINMUX_GPIO46__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(46) | 1)
-+#define PINMUX_GPIO46__FUNC_O_PWM_0 (MTK_PIN_NO(46) | 2)
-+#define PINMUX_GPIO46__FUNC_I0_VBUSVALID_2P (MTK_PIN_NO(46) | 3)
-+#define PINMUX_GPIO46__FUNC_B0_DBG_MON_A22 (MTK_PIN_NO(46) | 7)
-+
-+#define PINMUX_GPIO47__FUNC_B_GPIO47 (MTK_PIN_NO(47) | 0)
-+#define PINMUX_GPIO47__FUNC_I1_WAKEN (MTK_PIN_NO(47) | 1)
-+#define PINMUX_GPIO47__FUNC_O_GDU_TROOPS_DET0 (MTK_PIN_NO(47) | 6)
-+
-+#define PINMUX_GPIO48__FUNC_B_GPIO48 (MTK_PIN_NO(48) | 0)
-+#define PINMUX_GPIO48__FUNC_O_PERSTN (MTK_PIN_NO(48) | 1)
-+#define PINMUX_GPIO48__FUNC_O_GDU_TROOPS_DET1 (MTK_PIN_NO(48) | 6)
-+
-+#define PINMUX_GPIO49__FUNC_B_GPIO49 (MTK_PIN_NO(49) | 0)
-+#define PINMUX_GPIO49__FUNC_B1_CLKREQN (MTK_PIN_NO(49) | 1)
-+#define PINMUX_GPIO49__FUNC_O_GDU_TROOPS_DET2 (MTK_PIN_NO(49) | 6)
-+
-+#define PINMUX_GPIO50__FUNC_B_GPIO50 (MTK_PIN_NO(50) | 0)
-+#define PINMUX_GPIO50__FUNC_O_HDMITX20_PWR5V (MTK_PIN_NO(50) | 1)
-+#define PINMUX_GPIO50__FUNC_I1_IDDIG_1P (MTK_PIN_NO(50) | 3)
-+#define PINMUX_GPIO50__FUNC_I1_SCP_JTAG1_TMS (MTK_PIN_NO(50) | 4)
-+#define PINMUX_GPIO50__FUNC_I1_SSPM_JTAG_TMS (MTK_PIN_NO(50) | 5)
-+#define PINMUX_GPIO50__FUNC_I1_MD32_0_JTAG_TMS (MTK_PIN_NO(50) | 6)
-+#define PINMUX_GPIO50__FUNC_I1_MD32_1_JTAG_TMS (MTK_PIN_NO(50) | 7)
-+
-+#define PINMUX_GPIO51__FUNC_B_GPIO51 (MTK_PIN_NO(51) | 0)
-+#define PINMUX_GPIO51__FUNC_I0_HDMITX20_HTPLG (MTK_PIN_NO(51) | 1)
-+#define PINMUX_GPIO51__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(51) | 2)
-+#define PINMUX_GPIO51__FUNC_O_USB_DRVVBUS_1P (MTK_PIN_NO(51) | 3)
-+#define PINMUX_GPIO51__FUNC_I1_SCP_JTAG1_TCK (MTK_PIN_NO(51) | 4)
-+#define PINMUX_GPIO51__FUNC_I1_SSPM_JTAG_TCK (MTK_PIN_NO(51) | 5)
-+#define PINMUX_GPIO51__FUNC_I1_MD32_0_JTAG_TCK (MTK_PIN_NO(51) | 6)
-+#define PINMUX_GPIO51__FUNC_I1_MD32_1_JTAG_TCK (MTK_PIN_NO(51) | 7)
-+
-+#define PINMUX_GPIO52__FUNC_B_GPIO52 (MTK_PIN_NO(52) | 0)
-+#define PINMUX_GPIO52__FUNC_B1_HDMITX20_CEC (MTK_PIN_NO(52) | 1)
-+#define PINMUX_GPIO52__FUNC_I0_VBUSVALID_1P (MTK_PIN_NO(52) | 3)
-+#define PINMUX_GPIO52__FUNC_I1_SCP_JTAG1_TDI (MTK_PIN_NO(52) | 4)
-+#define PINMUX_GPIO52__FUNC_I1_SSPM_JTAG_TDI (MTK_PIN_NO(52) | 5)
-+#define PINMUX_GPIO52__FUNC_I1_MD32_0_JTAG_TDI (MTK_PIN_NO(52) | 6)
-+#define PINMUX_GPIO52__FUNC_I1_MD32_1_JTAG_TDI (MTK_PIN_NO(52) | 7)
-+
-+#define PINMUX_GPIO53__FUNC_B_GPIO53 (MTK_PIN_NO(53) | 0)
-+#define PINMUX_GPIO53__FUNC_B1_HDMITX20_SCL (MTK_PIN_NO(53) | 1)
-+#define PINMUX_GPIO53__FUNC_I1_IDDIG_2P (MTK_PIN_NO(53) | 3)
-+#define PINMUX_GPIO53__FUNC_O_SCP_JTAG1_TDO (MTK_PIN_NO(53) | 4)
-+#define PINMUX_GPIO53__FUNC_O_SSPM_JTAG_TDO (MTK_PIN_NO(53) | 5)
-+#define PINMUX_GPIO53__FUNC_O_MD32_0_JTAG_TDO (MTK_PIN_NO(53) | 6)
-+#define PINMUX_GPIO53__FUNC_O_MD32_1_JTAG_TDO (MTK_PIN_NO(53) | 7)
-+
-+#define PINMUX_GPIO54__FUNC_B_GPIO54 (MTK_PIN_NO(54) | 0)
-+#define PINMUX_GPIO54__FUNC_B1_HDMITX20_SDA (MTK_PIN_NO(54) | 1)
-+#define PINMUX_GPIO54__FUNC_O_USB_DRVVBUS_2P (MTK_PIN_NO(54) | 3)
-+#define PINMUX_GPIO54__FUNC_I0_SCP_JTAG1_TRSTN (MTK_PIN_NO(54) | 4)
-+#define PINMUX_GPIO54__FUNC_I0_SSPM_JTAG_TRSTN (MTK_PIN_NO(54) | 5)
-+#define PINMUX_GPIO54__FUNC_I1_MD32_0_JTAG_TRST (MTK_PIN_NO(54) | 6)
-+#define PINMUX_GPIO54__FUNC_I1_MD32_1_JTAG_TRST (MTK_PIN_NO(54) | 7)
-+
-+#define PINMUX_GPIO55__FUNC_B_GPIO55 (MTK_PIN_NO(55) | 0)
-+#define PINMUX_GPIO55__FUNC_B1_SCL0 (MTK_PIN_NO(55) | 1)
-+#define PINMUX_GPIO55__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(55) | 2)
-+#define PINMUX_GPIO55__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(55) | 3)
-+#define PINMUX_GPIO55__FUNC_B1_PCIE_PHY_I2C_SCL (MTK_PIN_NO(55) | 4)
-+
-+#define PINMUX_GPIO56__FUNC_B_GPIO56 (MTK_PIN_NO(56) | 0)
-+#define PINMUX_GPIO56__FUNC_B1_SDA0 (MTK_PIN_NO(56) | 1)
-+#define PINMUX_GPIO56__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(56) | 2)
-+#define PINMUX_GPIO56__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(56) | 3)
-+#define PINMUX_GPIO56__FUNC_B1_PCIE_PHY_I2C_SDA (MTK_PIN_NO(56) | 4)
-+
-+#define PINMUX_GPIO57__FUNC_B_GPIO57 (MTK_PIN_NO(57) | 0)
-+#define PINMUX_GPIO57__FUNC_B1_SCL1 (MTK_PIN_NO(57) | 1)
-+
-+#define PINMUX_GPIO58__FUNC_B_GPIO58 (MTK_PIN_NO(58) | 0)
-+#define PINMUX_GPIO58__FUNC_B1_SDA1 (MTK_PIN_NO(58) | 1)
-+
-+#define PINMUX_GPIO59__FUNC_B_GPIO59 (MTK_PIN_NO(59) | 0)
-+#define PINMUX_GPIO59__FUNC_B1_SCL2 (MTK_PIN_NO(59) | 1)
-+#define PINMUX_GPIO59__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(59) | 2)
-+#define PINMUX_GPIO59__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(59) | 3)
-+
-+#define PINMUX_GPIO60__FUNC_B_GPIO60 (MTK_PIN_NO(60) | 0)
-+#define PINMUX_GPIO60__FUNC_B1_SDA2 (MTK_PIN_NO(60) | 1)
-+#define PINMUX_GPIO60__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(60) | 2)
-+#define PINMUX_GPIO60__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(60) | 3)
-+
-+#define PINMUX_GPIO61__FUNC_B_GPIO61 (MTK_PIN_NO(61) | 0)
-+#define PINMUX_GPIO61__FUNC_B1_SCL3 (MTK_PIN_NO(61) | 1)
-+#define PINMUX_GPIO61__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(61) | 2)
-+#define PINMUX_GPIO61__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(61) | 3)
-+#define PINMUX_GPIO61__FUNC_B1_PCIE_PHY_I2C_SCL (MTK_PIN_NO(61) | 4)
-+
-+#define PINMUX_GPIO62__FUNC_B_GPIO62 (MTK_PIN_NO(62) | 0)
-+#define PINMUX_GPIO62__FUNC_B1_SDA3 (MTK_PIN_NO(62) | 1)
-+#define PINMUX_GPIO62__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(62) | 2)
-+#define PINMUX_GPIO62__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(62) | 3)
-+#define PINMUX_GPIO62__FUNC_B1_PCIE_PHY_I2C_SDA (MTK_PIN_NO(62) | 4)
-+
-+#define PINMUX_GPIO63__FUNC_B_GPIO63 (MTK_PIN_NO(63) | 0)
-+#define PINMUX_GPIO63__FUNC_B1_SCL4 (MTK_PIN_NO(63) | 1)
-+
-+#define PINMUX_GPIO64__FUNC_B_GPIO64 (MTK_PIN_NO(64) | 0)
-+#define PINMUX_GPIO64__FUNC_B1_SDA4 (MTK_PIN_NO(64) | 1)
-+
-+#define PINMUX_GPIO65__FUNC_B_GPIO65 (MTK_PIN_NO(65) | 0)
-+#define PINMUX_GPIO65__FUNC_B1_SCL5 (MTK_PIN_NO(65) | 1)
-+#define PINMUX_GPIO65__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(65) | 2)
-+#define PINMUX_GPIO65__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(65) | 3)
-+
-+#define PINMUX_GPIO66__FUNC_B_GPIO66 (MTK_PIN_NO(66) | 0)
-+#define PINMUX_GPIO66__FUNC_B1_SDA5 (MTK_PIN_NO(66) | 1)
-+#define PINMUX_GPIO66__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(66) | 2)
-+#define PINMUX_GPIO66__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(66) | 3)
-+
-+#define PINMUX_GPIO67__FUNC_B_GPIO67 (MTK_PIN_NO(67) | 0)
-+#define PINMUX_GPIO67__FUNC_B1_SCL6 (MTK_PIN_NO(67) | 1)
-+#define PINMUX_GPIO67__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(67) | 2)
-+#define PINMUX_GPIO67__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(67) | 3)
-+#define PINMUX_GPIO67__FUNC_B1_PCIE_PHY_I2C_SCL (MTK_PIN_NO(67) | 4)
-+
-+#define PINMUX_GPIO68__FUNC_B_GPIO68 (MTK_PIN_NO(68) | 0)
-+#define PINMUX_GPIO68__FUNC_B1_SDA6 (MTK_PIN_NO(68) | 1)
-+#define PINMUX_GPIO68__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(68) | 2)
-+#define PINMUX_GPIO68__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(68) | 3)
-+#define PINMUX_GPIO68__FUNC_B1_PCIE_PHY_I2C_SDA (MTK_PIN_NO(68) | 4)
-+
-+#define PINMUX_GPIO69__FUNC_B_GPIO69 (MTK_PIN_NO(69) | 0)
-+#define PINMUX_GPIO69__FUNC_O_SPIM0_CSB (MTK_PIN_NO(69) | 1)
-+#define PINMUX_GPIO69__FUNC_O_SCP_SPI0_CS (MTK_PIN_NO(69) | 2)
-+#define PINMUX_GPIO69__FUNC_O_DMIC3_CLK (MTK_PIN_NO(69) | 3)
-+#define PINMUX_GPIO69__FUNC_B0_MD32_1_GPIO0 (MTK_PIN_NO(69) | 4)
-+#define PINMUX_GPIO69__FUNC_O_CMVREF0 (MTK_PIN_NO(69) | 5)
-+#define PINMUX_GPIO69__FUNC_O_GDU_SUM_TROOP0_0 (MTK_PIN_NO(69) | 6)
-+#define PINMUX_GPIO69__FUNC_B0_DBG_MON_A23 (MTK_PIN_NO(69) | 7)
-+
-+#define PINMUX_GPIO70__FUNC_B_GPIO70 (MTK_PIN_NO(70) | 0)
-+#define PINMUX_GPIO70__FUNC_O_SPIM0_CLK (MTK_PIN_NO(70) | 1)
-+#define PINMUX_GPIO70__FUNC_O_SCP_SPI0_CK (MTK_PIN_NO(70) | 2)
-+#define PINMUX_GPIO70__FUNC_I0_DMIC3_DAT (MTK_PIN_NO(70) | 3)
-+#define PINMUX_GPIO70__FUNC_B0_MD32_1_GPIO1 (MTK_PIN_NO(70) | 4)
-+#define PINMUX_GPIO70__FUNC_O_CMVREF1 (MTK_PIN_NO(70) | 5)
-+#define PINMUX_GPIO70__FUNC_O_GDU_SUM_TROOP0_1 (MTK_PIN_NO(70) | 6)
-+#define PINMUX_GPIO70__FUNC_B0_DBG_MON_A24 (MTK_PIN_NO(70) | 7)
-+
-+#define PINMUX_GPIO71__FUNC_B_GPIO71 (MTK_PIN_NO(71) | 0)
-+#define PINMUX_GPIO71__FUNC_B0_SPIM0_MOSI (MTK_PIN_NO(71) | 1)
-+#define PINMUX_GPIO71__FUNC_O_SCP_SPI0_MO (MTK_PIN_NO(71) | 2)
-+#define PINMUX_GPIO71__FUNC_I0_DMIC3_DAT_R (MTK_PIN_NO(71) | 3)
-+#define PINMUX_GPIO71__FUNC_B0_MD32_1_GPIO2 (MTK_PIN_NO(71) | 4)
-+#define PINMUX_GPIO71__FUNC_O_CMVREF2 (MTK_PIN_NO(71) | 5)
-+#define PINMUX_GPIO71__FUNC_O_GDU_SUM_TROOP0_2 (MTK_PIN_NO(71) | 6)
-+#define PINMUX_GPIO71__FUNC_B0_DBG_MON_A25 (MTK_PIN_NO(71) | 7)
-+
-+#define PINMUX_GPIO72__FUNC_B_GPIO72 (MTK_PIN_NO(72) | 0)
-+#define PINMUX_GPIO72__FUNC_B0_SPIM0_MISO (MTK_PIN_NO(72) | 1)
-+#define PINMUX_GPIO72__FUNC_I0_SCP_SPI0_MI (MTK_PIN_NO(72) | 2)
-+#define PINMUX_GPIO72__FUNC_O_DMIC4_CLK (MTK_PIN_NO(72) | 3)
-+#define PINMUX_GPIO72__FUNC_O_CMVREF3 (MTK_PIN_NO(72) | 5)
-+#define PINMUX_GPIO72__FUNC_O_GDU_SUM_TROOP1_0 (MTK_PIN_NO(72) | 6)
-+#define PINMUX_GPIO72__FUNC_B0_DBG_MON_A26 (MTK_PIN_NO(72) | 7)
-+
-+#define PINMUX_GPIO73__FUNC_B_GPIO73 (MTK_PIN_NO(73) | 0)
-+#define PINMUX_GPIO73__FUNC_B0_SPIM0_MIO2 (MTK_PIN_NO(73) | 1)
-+#define PINMUX_GPIO73__FUNC_O_UTXD3 (MTK_PIN_NO(73) | 2)
-+#define PINMUX_GPIO73__FUNC_I0_DMIC4_DAT (MTK_PIN_NO(73) | 3)
-+#define PINMUX_GPIO73__FUNC_O_CLKM0 (MTK_PIN_NO(73) | 4)
-+#define PINMUX_GPIO73__FUNC_O_CMVREF4 (MTK_PIN_NO(73) | 5)
-+#define PINMUX_GPIO73__FUNC_O_GDU_SUM_TROOP1_1 (MTK_PIN_NO(73) | 6)
-+#define PINMUX_GPIO73__FUNC_B0_DBG_MON_A27 (MTK_PIN_NO(73) | 7)
-+
-+#define PINMUX_GPIO74__FUNC_B_GPIO74 (MTK_PIN_NO(74) | 0)
-+#define PINMUX_GPIO74__FUNC_B0_SPIM0_MIO3 (MTK_PIN_NO(74) | 1)
-+#define PINMUX_GPIO74__FUNC_I1_URXD3 (MTK_PIN_NO(74) | 2)
-+#define PINMUX_GPIO74__FUNC_I0_DMIC4_DAT_R (MTK_PIN_NO(74) | 3)
-+#define PINMUX_GPIO74__FUNC_O_CLKM1 (MTK_PIN_NO(74) | 4)
-+#define PINMUX_GPIO74__FUNC_O_CMVREF5 (MTK_PIN_NO(74) | 5)
-+#define PINMUX_GPIO74__FUNC_O_GDU_SUM_TROOP1_2 (MTK_PIN_NO(74) | 6)
-+#define PINMUX_GPIO74__FUNC_B0_DBG_MON_A28 (MTK_PIN_NO(74) | 7)
-+
-+#define PINMUX_GPIO75__FUNC_B_GPIO75 (MTK_PIN_NO(75) | 0)
-+#define PINMUX_GPIO75__FUNC_O_SPIM1_CSB (MTK_PIN_NO(75) | 1)
-+#define PINMUX_GPIO75__FUNC_O_SCP_SPI1_A_CS (MTK_PIN_NO(75) | 2)
-+#define PINMUX_GPIO75__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(75) | 3)
-+#define PINMUX_GPIO75__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(75) | 4)
-+#define PINMUX_GPIO75__FUNC_O_CMVREF6 (MTK_PIN_NO(75) | 5)
-+#define PINMUX_GPIO75__FUNC_O_GDU_SUM_TROOP2_0 (MTK_PIN_NO(75) | 6)
-+#define PINMUX_GPIO75__FUNC_B0_DBG_MON_A29 (MTK_PIN_NO(75) | 7)
-+
-+#define PINMUX_GPIO76__FUNC_B_GPIO76 (MTK_PIN_NO(76) | 0)
-+#define PINMUX_GPIO76__FUNC_O_SPIM1_CLK (MTK_PIN_NO(76) | 1)
-+#define PINMUX_GPIO76__FUNC_O_SCP_SPI1_A_CK (MTK_PIN_NO(76) | 2)
-+#define PINMUX_GPIO76__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(76) | 3)
-+#define PINMUX_GPIO76__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(76) | 4)
-+#define PINMUX_GPIO76__FUNC_O_CMVREF7 (MTK_PIN_NO(76) | 5)
-+#define PINMUX_GPIO76__FUNC_O_GDU_SUM_TROOP2_1 (MTK_PIN_NO(76) | 6)
-+#define PINMUX_GPIO76__FUNC_B0_DBG_MON_A30 (MTK_PIN_NO(76) | 7)
-+
-+#define PINMUX_GPIO77__FUNC_B_GPIO77 (MTK_PIN_NO(77) | 0)
-+#define PINMUX_GPIO77__FUNC_B0_SPIM1_MOSI (MTK_PIN_NO(77) | 1)
-+#define PINMUX_GPIO77__FUNC_O_SCP_SPI1_A_MO (MTK_PIN_NO(77) | 2)
-+#define PINMUX_GPIO77__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(77) | 3)
-+#define PINMUX_GPIO77__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(77) | 4)
-+#define PINMUX_GPIO77__FUNC_O_GDU_SUM_TROOP2_2 (MTK_PIN_NO(77) | 6)
-+#define PINMUX_GPIO77__FUNC_B0_DBG_MON_A31 (MTK_PIN_NO(77) | 7)
-+
-+#define PINMUX_GPIO78__FUNC_B_GPIO78 (MTK_PIN_NO(78) | 0)
-+#define PINMUX_GPIO78__FUNC_B0_SPIM1_MISO (MTK_PIN_NO(78) | 1)
-+#define PINMUX_GPIO78__FUNC_I0_SCP_SPI1_A_MI (MTK_PIN_NO(78) | 2)
-+#define PINMUX_GPIO78__FUNC_I0_TDMIN_DI (MTK_PIN_NO(78) | 3)
-+#define PINMUX_GPIO78__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(78) | 4)
-+#define PINMUX_GPIO78__FUNC_B0_DBG_MON_A32 (MTK_PIN_NO(78) | 7)
-+
-+#define PINMUX_GPIO79__FUNC_B_GPIO79 (MTK_PIN_NO(79) | 0)
-+#define PINMUX_GPIO79__FUNC_O_SPIM2_CSB (MTK_PIN_NO(79) | 1)
-+#define PINMUX_GPIO79__FUNC_O_SCP_SPI2_CS (MTK_PIN_NO(79) | 2)
-+#define PINMUX_GPIO79__FUNC_O_I2SO1_MCK (MTK_PIN_NO(79) | 3)
-+#define PINMUX_GPIO79__FUNC_O_UTXD2 (MTK_PIN_NO(79) | 4)
-+#define PINMUX_GPIO79__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(79) | 5)
-+#define PINMUX_GPIO79__FUNC_B0_PCM_SYNC (MTK_PIN_NO(79) | 6)
-+#define PINMUX_GPIO79__FUNC_B0_DBG_MON_B0 (MTK_PIN_NO(79) | 7)
-+
-+#define PINMUX_GPIO80__FUNC_B_GPIO80 (MTK_PIN_NO(80) | 0)
-+#define PINMUX_GPIO80__FUNC_O_SPIM2_CLK (MTK_PIN_NO(80) | 1)
-+#define PINMUX_GPIO80__FUNC_O_SCP_SPI2_CK (MTK_PIN_NO(80) | 2)
-+#define PINMUX_GPIO80__FUNC_O_I2SO1_BCK (MTK_PIN_NO(80) | 3)
-+#define PINMUX_GPIO80__FUNC_I1_URXD2 (MTK_PIN_NO(80) | 4)
-+#define PINMUX_GPIO80__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(80) | 5)
-+#define PINMUX_GPIO80__FUNC_B0_PCM_CLK (MTK_PIN_NO(80) | 6)
-+#define PINMUX_GPIO80__FUNC_B0_DBG_MON_B1 (MTK_PIN_NO(80) | 7)
-+
-+#define PINMUX_GPIO81__FUNC_B_GPIO81 (MTK_PIN_NO(81) | 0)
-+#define PINMUX_GPIO81__FUNC_B0_SPIM2_MOSI (MTK_PIN_NO(81) | 1)
-+#define PINMUX_GPIO81__FUNC_O_SCP_SPI2_MO (MTK_PIN_NO(81) | 2)
-+#define PINMUX_GPIO81__FUNC_O_I2SO1_WS (MTK_PIN_NO(81) | 3)
-+#define PINMUX_GPIO81__FUNC_O_URTS2 (MTK_PIN_NO(81) | 4)
-+#define PINMUX_GPIO81__FUNC_O_TP_URTS2_AO (MTK_PIN_NO(81) | 5)
-+#define PINMUX_GPIO81__FUNC_O_PCM_DO (MTK_PIN_NO(81) | 6)
-+#define PINMUX_GPIO81__FUNC_B0_DBG_MON_B2 (MTK_PIN_NO(81) | 7)
-+
-+#define PINMUX_GPIO82__FUNC_B_GPIO82 (MTK_PIN_NO(82) | 0)
-+#define PINMUX_GPIO82__FUNC_B0_SPIM2_MISO (MTK_PIN_NO(82) | 1)
-+#define PINMUX_GPIO82__FUNC_I0_SCP_SPI2_MI (MTK_PIN_NO(82) | 2)
-+#define PINMUX_GPIO82__FUNC_O_I2SO1_D0 (MTK_PIN_NO(82) | 3)
-+#define PINMUX_GPIO82__FUNC_I1_UCTS2 (MTK_PIN_NO(82) | 4)
-+#define PINMUX_GPIO82__FUNC_I1_TP_UCTS2_AO (MTK_PIN_NO(82) | 5)
-+#define PINMUX_GPIO82__FUNC_I0_PCM_DI (MTK_PIN_NO(82) | 6)
-+#define PINMUX_GPIO82__FUNC_B0_DBG_MON_B3 (MTK_PIN_NO(82) | 7)
-+
-+#define PINMUX_GPIO83__FUNC_B_GPIO83 (MTK_PIN_NO(83) | 0)
-+#define PINMUX_GPIO83__FUNC_I1_IDDIG (MTK_PIN_NO(83) | 1)
-+
-+#define PINMUX_GPIO84__FUNC_B_GPIO84 (MTK_PIN_NO(84) | 0)
-+#define PINMUX_GPIO84__FUNC_O_USB_DRVVBUS (MTK_PIN_NO(84) | 1)
-+
-+#define PINMUX_GPIO85__FUNC_B_GPIO85 (MTK_PIN_NO(85) | 0)
-+#define PINMUX_GPIO85__FUNC_I0_VBUSVALID (MTK_PIN_NO(85) | 1)
-+
-+#define PINMUX_GPIO86__FUNC_B_GPIO86 (MTK_PIN_NO(86) | 0)
-+#define PINMUX_GPIO86__FUNC_I1_IDDIG_1P (MTK_PIN_NO(86) | 1)
-+#define PINMUX_GPIO86__FUNC_O_UTXD1 (MTK_PIN_NO(86) | 2)
-+#define PINMUX_GPIO86__FUNC_O_URTS2 (MTK_PIN_NO(86) | 3)
-+#define PINMUX_GPIO86__FUNC_O_PWM_2 (MTK_PIN_NO(86) | 4)
-+#define PINMUX_GPIO86__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(86) | 5)
-+#define PINMUX_GPIO86__FUNC_O_AUXIF_ST0 (MTK_PIN_NO(86) | 6)
-+#define PINMUX_GPIO86__FUNC_B0_DBG_MON_B4 (MTK_PIN_NO(86) | 7)
-+
-+#define PINMUX_GPIO87__FUNC_B_GPIO87 (MTK_PIN_NO(87) | 0)
-+#define PINMUX_GPIO87__FUNC_O_USB_DRVVBUS_1P (MTK_PIN_NO(87) | 1)
-+#define PINMUX_GPIO87__FUNC_I1_URXD1 (MTK_PIN_NO(87) | 2)
-+#define PINMUX_GPIO87__FUNC_I1_UCTS2 (MTK_PIN_NO(87) | 3)
-+#define PINMUX_GPIO87__FUNC_O_PWM_3 (MTK_PIN_NO(87) | 4)
-+#define PINMUX_GPIO87__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(87) | 5)
-+#define PINMUX_GPIO87__FUNC_O_AUXIF_CLK0 (MTK_PIN_NO(87) | 6)
-+#define PINMUX_GPIO87__FUNC_B0_DBG_MON_B5 (MTK_PIN_NO(87) | 7)
-+
-+#define PINMUX_GPIO88__FUNC_B_GPIO88 (MTK_PIN_NO(88) | 0)
-+#define PINMUX_GPIO88__FUNC_I0_VBUSVALID_1P (MTK_PIN_NO(88) | 1)
-+#define PINMUX_GPIO88__FUNC_O_UTXD2 (MTK_PIN_NO(88) | 2)
-+#define PINMUX_GPIO88__FUNC_O_URTS1 (MTK_PIN_NO(88) | 3)
-+#define PINMUX_GPIO88__FUNC_O_CLKM2 (MTK_PIN_NO(88) | 4)
-+#define PINMUX_GPIO88__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(88) | 5)
-+#define PINMUX_GPIO88__FUNC_O_AUXIF_ST1 (MTK_PIN_NO(88) | 6)
-+#define PINMUX_GPIO88__FUNC_B0_DBG_MON_B6 (MTK_PIN_NO(88) | 7)
-+
-+#define PINMUX_GPIO89__FUNC_B_GPIO89 (MTK_PIN_NO(89) | 0)
-+#define PINMUX_GPIO89__FUNC_I1_IDDIG_2P (MTK_PIN_NO(89) | 1)
-+#define PINMUX_GPIO89__FUNC_I1_URXD2 (MTK_PIN_NO(89) | 2)
-+#define PINMUX_GPIO89__FUNC_I1_UCTS1 (MTK_PIN_NO(89) | 3)
-+#define PINMUX_GPIO89__FUNC_O_CLKM3 (MTK_PIN_NO(89) | 4)
-+#define PINMUX_GPIO89__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(89) | 5)
-+#define PINMUX_GPIO89__FUNC_O_AUXIF_CLK1 (MTK_PIN_NO(89) | 6)
-+#define PINMUX_GPIO89__FUNC_B0_DBG_MON_B7 (MTK_PIN_NO(89) | 7)
-+
-+#define PINMUX_GPIO90__FUNC_B_GPIO90 (MTK_PIN_NO(90) | 0)
-+#define PINMUX_GPIO90__FUNC_O_USB_DRVVBUS_2P (MTK_PIN_NO(90) | 1)
-+#define PINMUX_GPIO90__FUNC_O_UTXD3 (MTK_PIN_NO(90) | 2)
-+#define PINMUX_GPIO90__FUNC_O_ADSP_UTXD0 (MTK_PIN_NO(90) | 3)
-+#define PINMUX_GPIO90__FUNC_O_SSPM_UTXD_AO (MTK_PIN_NO(90) | 4)
-+#define PINMUX_GPIO90__FUNC_O_MD32_0_TXD (MTK_PIN_NO(90) | 5)
-+#define PINMUX_GPIO90__FUNC_O_MD32_1_TXD (MTK_PIN_NO(90) | 6)
-+#define PINMUX_GPIO90__FUNC_B0_DBG_MON_B8 (MTK_PIN_NO(90) | 7)
-+
-+#define PINMUX_GPIO91__FUNC_B_GPIO91 (MTK_PIN_NO(91) | 0)
-+#define PINMUX_GPIO91__FUNC_I0_VBUSVALID_2P (MTK_PIN_NO(91) | 1)
-+#define PINMUX_GPIO91__FUNC_I1_URXD3 (MTK_PIN_NO(91) | 2)
-+#define PINMUX_GPIO91__FUNC_I1_ADSP_URXD0 (MTK_PIN_NO(91) | 3)
-+#define PINMUX_GPIO91__FUNC_I1_SSPM_URXD_AO (MTK_PIN_NO(91) | 4)
-+#define PINMUX_GPIO91__FUNC_I1_MD32_0_RXD (MTK_PIN_NO(91) | 5)
-+#define PINMUX_GPIO91__FUNC_I1_MD32_1_RXD (MTK_PIN_NO(91) | 6)
-+#define PINMUX_GPIO91__FUNC_B0_DBG_MON_B9 (MTK_PIN_NO(91) | 7)
-+
-+#define PINMUX_GPIO92__FUNC_B_GPIO92 (MTK_PIN_NO(92) | 0)
-+#define PINMUX_GPIO92__FUNC_O_PWRAP_SPI0_CSN (MTK_PIN_NO(92) | 1)
-+
-+#define PINMUX_GPIO93__FUNC_B_GPIO93 (MTK_PIN_NO(93) | 0)
-+#define PINMUX_GPIO93__FUNC_O_PWRAP_SPI0_CK (MTK_PIN_NO(93) | 1)
-+
-+#define PINMUX_GPIO94__FUNC_B_GPIO94 (MTK_PIN_NO(94) | 0)
-+#define PINMUX_GPIO94__FUNC_B0_PWRAP_SPI0_MO (MTK_PIN_NO(94) | 1)
-+#define PINMUX_GPIO94__FUNC_B0_PWRAP_SPI0_MI (MTK_PIN_NO(94) | 2)
-+
-+#define PINMUX_GPIO95__FUNC_B_GPIO95 (MTK_PIN_NO(95) | 0)
-+#define PINMUX_GPIO95__FUNC_B0_PWRAP_SPI0_MI (MTK_PIN_NO(95) | 1)
-+#define PINMUX_GPIO95__FUNC_B0_PWRAP_SPI0_MO (MTK_PIN_NO(95) | 2)
-+
-+#define PINMUX_GPIO96__FUNC_B_GPIO96 (MTK_PIN_NO(96) | 0)
-+#define PINMUX_GPIO96__FUNC_O_SRCLKENA0 (MTK_PIN_NO(96) | 1)
-+
-+#define PINMUX_GPIO97__FUNC_B_GPIO97 (MTK_PIN_NO(97) | 0)
-+#define PINMUX_GPIO97__FUNC_O_SRCLKENA1 (MTK_PIN_NO(97) | 1)
-+
-+#define PINMUX_GPIO98__FUNC_B_GPIO98 (MTK_PIN_NO(98) | 0)
-+#define PINMUX_GPIO98__FUNC_O_SCP_VREQ_VAO (MTK_PIN_NO(98) | 1)
-+#define PINMUX_GPIO98__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(98) | 2)
-+
-+#define PINMUX_GPIO99__FUNC_B_GPIO99 (MTK_PIN_NO(99) | 0)
-+#define PINMUX_GPIO99__FUNC_I0_RTC32K_CK (MTK_PIN_NO(99) | 1)
-+
-+#define PINMUX_GPIO100__FUNC_B_GPIO100 (MTK_PIN_NO(100) | 0)
-+#define PINMUX_GPIO100__FUNC_O_WATCHDOG (MTK_PIN_NO(100) | 1)
-+
-+#define PINMUX_GPIO101__FUNC_B_GPIO101 (MTK_PIN_NO(101) | 0)
-+#define PINMUX_GPIO101__FUNC_O_AUD_CLK_MOSI (MTK_PIN_NO(101) | 1)
-+#define PINMUX_GPIO101__FUNC_O_I2SO1_MCK (MTK_PIN_NO(101) | 2)
-+#define PINMUX_GPIO101__FUNC_B0_I2SIN_BCK (MTK_PIN_NO(101) | 3)
-+
-+#define PINMUX_GPIO102__FUNC_B_GPIO102 (MTK_PIN_NO(102) | 0)
-+#define PINMUX_GPIO102__FUNC_O_AUD_SYNC_MOSI (MTK_PIN_NO(102) | 1)
-+#define PINMUX_GPIO102__FUNC_O_I2SO1_BCK (MTK_PIN_NO(102) | 2)
-+#define PINMUX_GPIO102__FUNC_B0_I2SIN_WS (MTK_PIN_NO(102) | 3)
-+
-+#define PINMUX_GPIO103__FUNC_B_GPIO103 (MTK_PIN_NO(103) | 0)
-+#define PINMUX_GPIO103__FUNC_O_AUD_DAT_MOSI0 (MTK_PIN_NO(103) | 1)
-+#define PINMUX_GPIO103__FUNC_O_I2SO1_WS (MTK_PIN_NO(103) | 2)
-+#define PINMUX_GPIO103__FUNC_I0_I2SIN_D0 (MTK_PIN_NO(103) | 3)
-+
-+#define PINMUX_GPIO104__FUNC_B_GPIO104 (MTK_PIN_NO(104) | 0)
-+#define PINMUX_GPIO104__FUNC_O_AUD_DAT_MOSI1 (MTK_PIN_NO(104) | 1)
-+#define PINMUX_GPIO104__FUNC_O_I2SO1_D0 (MTK_PIN_NO(104) | 2)
-+#define PINMUX_GPIO104__FUNC_I0_I2SIN_D1 (MTK_PIN_NO(104) | 3)
-+
-+#define PINMUX_GPIO105__FUNC_B_GPIO105 (MTK_PIN_NO(105) | 0)
-+#define PINMUX_GPIO105__FUNC_I0_AUD_DAT_MISO0 (MTK_PIN_NO(105) | 1)
-+#define PINMUX_GPIO105__FUNC_I0_VOW_DAT_MISO (MTK_PIN_NO(105) | 2)
-+#define PINMUX_GPIO105__FUNC_I0_I2SIN_D2 (MTK_PIN_NO(105) | 3)
-+
-+#define PINMUX_GPIO106__FUNC_B_GPIO106 (MTK_PIN_NO(106) | 0)
-+#define PINMUX_GPIO106__FUNC_I0_AUD_DAT_MISO1 (MTK_PIN_NO(106) | 1)
-+#define PINMUX_GPIO106__FUNC_I0_VOW_CLK_MISO (MTK_PIN_NO(106) | 2)
-+#define PINMUX_GPIO106__FUNC_I0_I2SIN_D3 (MTK_PIN_NO(106) | 3)
-+
-+#define PINMUX_GPIO107__FUNC_B_GPIO107 (MTK_PIN_NO(107) | 0)
-+#define PINMUX_GPIO107__FUNC_B0_I2SIN_MCK (MTK_PIN_NO(107) | 1)
-+#define PINMUX_GPIO107__FUNC_I0_SPLIN_MCK (MTK_PIN_NO(107) | 2)
-+#define PINMUX_GPIO107__FUNC_I0_SPDIF_IN0 (MTK_PIN_NO(107) | 3)
-+#define PINMUX_GPIO107__FUNC_O_CMVREF4 (MTK_PIN_NO(107) | 4)
-+#define PINMUX_GPIO107__FUNC_O_AUXIF_ST0 (MTK_PIN_NO(107) | 5)
-+#define PINMUX_GPIO107__FUNC_O_PGD_LV_LSC_PWR0 (MTK_PIN_NO(107) | 6)
-+
-+#define PINMUX_GPIO108__FUNC_B_GPIO108 (MTK_PIN_NO(108) | 0)
-+#define PINMUX_GPIO108__FUNC_B0_I2SIN_BCK (MTK_PIN_NO(108) | 1)
-+#define PINMUX_GPIO108__FUNC_I0_SPLIN_LRCK (MTK_PIN_NO(108) | 2)
-+#define PINMUX_GPIO108__FUNC_O_DMIC4_CLK (MTK_PIN_NO(108) | 3)
-+#define PINMUX_GPIO108__FUNC_O_CMVREF5 (MTK_PIN_NO(108) | 4)
-+#define PINMUX_GPIO108__FUNC_O_AUXIF_CLK0 (MTK_PIN_NO(108) | 5)
-+#define PINMUX_GPIO108__FUNC_O_PGD_LV_LSC_PWR1 (MTK_PIN_NO(108) | 6)
-+#define PINMUX_GPIO108__FUNC_B0_DBG_MON_B10 (MTK_PIN_NO(108) | 7)
-+
-+#define PINMUX_GPIO109__FUNC_B_GPIO109 (MTK_PIN_NO(109) | 0)
-+#define PINMUX_GPIO109__FUNC_B0_I2SIN_WS (MTK_PIN_NO(109) | 1)
-+#define PINMUX_GPIO109__FUNC_I0_SPLIN_BCK (MTK_PIN_NO(109) | 2)
-+#define PINMUX_GPIO109__FUNC_I0_DMIC4_DAT (MTK_PIN_NO(109) | 3)
-+#define PINMUX_GPIO109__FUNC_O_CMVREF6 (MTK_PIN_NO(109) | 4)
-+#define PINMUX_GPIO109__FUNC_O_AUXIF_ST1 (MTK_PIN_NO(109) | 5)
-+#define PINMUX_GPIO109__FUNC_O_PGD_LV_LSC_PWR2 (MTK_PIN_NO(109) | 6)
-+#define PINMUX_GPIO109__FUNC_B0_DBG_MON_B11 (MTK_PIN_NO(109) | 7)
-+
-+#define PINMUX_GPIO110__FUNC_B_GPIO110 (MTK_PIN_NO(110) | 0)
-+#define PINMUX_GPIO110__FUNC_I0_I2SIN_D0 (MTK_PIN_NO(110) | 1)
-+#define PINMUX_GPIO110__FUNC_I0_SPLIN_D0 (MTK_PIN_NO(110) | 2)
-+#define PINMUX_GPIO110__FUNC_I0_DMIC4_DAT_R (MTK_PIN_NO(110) | 3)
-+#define PINMUX_GPIO110__FUNC_O_CMVREF7 (MTK_PIN_NO(110) | 4)
-+#define PINMUX_GPIO110__FUNC_O_AUXIF_CLK1 (MTK_PIN_NO(110) | 5)
-+#define PINMUX_GPIO110__FUNC_O_PGD_LV_LSC_PWR3 (MTK_PIN_NO(110) | 6)
-+#define PINMUX_GPIO110__FUNC_B0_DBG_MON_B12 (MTK_PIN_NO(110) | 7)
-+
-+#define PINMUX_GPIO111__FUNC_B_GPIO111 (MTK_PIN_NO(111) | 0)
-+#define PINMUX_GPIO111__FUNC_I0_I2SIN_D1 (MTK_PIN_NO(111) | 1)
-+#define PINMUX_GPIO111__FUNC_I0_SPLIN_D1 (MTK_PIN_NO(111) | 2)
-+#define PINMUX_GPIO111__FUNC_O_DMIC3_CLK (MTK_PIN_NO(111) | 3)
-+#define PINMUX_GPIO111__FUNC_O_SPDIF_OUT (MTK_PIN_NO(111) | 4)
-+#define PINMUX_GPIO111__FUNC_O_PGD_LV_LSC_PWR4 (MTK_PIN_NO(111) | 6)
-+#define PINMUX_GPIO111__FUNC_B0_DBG_MON_B13 (MTK_PIN_NO(111) | 7)
-+
-+#define PINMUX_GPIO112__FUNC_B_GPIO112 (MTK_PIN_NO(112) | 0)
-+#define PINMUX_GPIO112__FUNC_I0_I2SIN_D2 (MTK_PIN_NO(112) | 1)
-+#define PINMUX_GPIO112__FUNC_I0_SPLIN_D2 (MTK_PIN_NO(112) | 2)
-+#define PINMUX_GPIO112__FUNC_I0_DMIC3_DAT (MTK_PIN_NO(112) | 3)
-+#define PINMUX_GPIO112__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(112) | 4)
-+#define PINMUX_GPIO112__FUNC_O_I2SO1_WS (MTK_PIN_NO(112) | 5)
-+#define PINMUX_GPIO112__FUNC_O_PGD_LV_LSC_PWR5 (MTK_PIN_NO(112) | 6)
-+#define PINMUX_GPIO112__FUNC_B0_DBG_MON_B14 (MTK_PIN_NO(112) | 7)
-+
-+#define PINMUX_GPIO113__FUNC_B_GPIO113 (MTK_PIN_NO(113) | 0)
-+#define PINMUX_GPIO113__FUNC_I0_I2SIN_D3 (MTK_PIN_NO(113) | 1)
-+#define PINMUX_GPIO113__FUNC_I0_SPLIN_D3 (MTK_PIN_NO(113) | 2)
-+#define PINMUX_GPIO113__FUNC_I0_DMIC3_DAT_R (MTK_PIN_NO(113) | 3)
-+#define PINMUX_GPIO113__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(113) | 4)
-+#define PINMUX_GPIO113__FUNC_O_I2SO1_D0 (MTK_PIN_NO(113) | 5)
-+#define PINMUX_GPIO113__FUNC_B0_DBG_MON_B15 (MTK_PIN_NO(113) | 7)
-+
-+#define PINMUX_GPIO114__FUNC_B_GPIO114 (MTK_PIN_NO(114) | 0)
-+#define PINMUX_GPIO114__FUNC_O_I2SO2_MCK (MTK_PIN_NO(114) | 1)
-+#define PINMUX_GPIO114__FUNC_B0_I2SIN_MCK (MTK_PIN_NO(114) | 2)
-+#define PINMUX_GPIO114__FUNC_I1_MCUPM_JTAG_TMS (MTK_PIN_NO(114) | 3)
-+#define PINMUX_GPIO114__FUNC_B1_APU_JTAG_TMS (MTK_PIN_NO(114) | 4)
-+#define PINMUX_GPIO114__FUNC_I1_SCP_JTAG1_TMS (MTK_PIN_NO(114) | 5)
-+#define PINMUX_GPIO114__FUNC_I1_SPM_JTAG_TMS (MTK_PIN_NO(114) | 6)
-+#define PINMUX_GPIO114__FUNC_B0_DBG_MON_B16 (MTK_PIN_NO(114) | 7)
-+
-+#define PINMUX_GPIO115__FUNC_B_GPIO115 (MTK_PIN_NO(115) | 0)
-+#define PINMUX_GPIO115__FUNC_B0_I2SO2_BCK (MTK_PIN_NO(115) | 1)
-+#define PINMUX_GPIO115__FUNC_B0_I2SIN_BCK (MTK_PIN_NO(115) | 2)
-+#define PINMUX_GPIO115__FUNC_I1_MCUPM_JTAG_TCK (MTK_PIN_NO(115) | 3)
-+#define PINMUX_GPIO115__FUNC_I0_APU_JTAG_TCK (MTK_PIN_NO(115) | 4)
-+#define PINMUX_GPIO115__FUNC_I1_SCP_JTAG1_TCK (MTK_PIN_NO(115) | 5)
-+#define PINMUX_GPIO115__FUNC_I1_SPM_JTAG_TCK (MTK_PIN_NO(115) | 6)
-+#define PINMUX_GPIO115__FUNC_B0_DBG_MON_B17 (MTK_PIN_NO(115) | 7)
-+
-+#define PINMUX_GPIO116__FUNC_B_GPIO116 (MTK_PIN_NO(116) | 0)
-+#define PINMUX_GPIO116__FUNC_B0_I2SO2_WS (MTK_PIN_NO(116) | 1)
-+#define PINMUX_GPIO116__FUNC_B0_I2SIN_WS (MTK_PIN_NO(116) | 2)
-+#define PINMUX_GPIO116__FUNC_I1_MCUPM_JTAG_TDI (MTK_PIN_NO(116) | 3)
-+#define PINMUX_GPIO116__FUNC_I1_APU_JTAG_TDI (MTK_PIN_NO(116) | 4)
-+#define PINMUX_GPIO116__FUNC_I1_SCP_JTAG1_TDI (MTK_PIN_NO(116) | 5)
-+#define PINMUX_GPIO116__FUNC_I1_SPM_JTAG_TDI (MTK_PIN_NO(116) | 6)
-+#define PINMUX_GPIO116__FUNC_B0_DBG_MON_B18 (MTK_PIN_NO(116) | 7)
-+
-+#define PINMUX_GPIO117__FUNC_B_GPIO117 (MTK_PIN_NO(117) | 0)
-+#define PINMUX_GPIO117__FUNC_O_I2SO2_D0 (MTK_PIN_NO(117) | 1)
-+#define PINMUX_GPIO117__FUNC_I0_I2SIN_D0 (MTK_PIN_NO(117) | 2)
-+#define PINMUX_GPIO117__FUNC_O_MCUPM_JTAG_TDO (MTK_PIN_NO(117) | 3)
-+#define PINMUX_GPIO117__FUNC_O_APU_JTAG_TDO (MTK_PIN_NO(117) | 4)
-+#define PINMUX_GPIO117__FUNC_O_SCP_JTAG1_TDO (MTK_PIN_NO(117) | 5)
-+#define PINMUX_GPIO117__FUNC_O_SPM_JTAG_TDO (MTK_PIN_NO(117) | 6)
-+#define PINMUX_GPIO117__FUNC_B0_DBG_MON_B19 (MTK_PIN_NO(117) | 7)
-+
-+#define PINMUX_GPIO118__FUNC_B_GPIO118 (MTK_PIN_NO(118) | 0)
-+#define PINMUX_GPIO118__FUNC_O_I2SO2_D1 (MTK_PIN_NO(118) | 1)
-+#define PINMUX_GPIO118__FUNC_I0_I2SIN_D1 (MTK_PIN_NO(118) | 2)
-+#define PINMUX_GPIO118__FUNC_I0_MCUPM_JTAG_TRSTN (MTK_PIN_NO(118) | 3)
-+#define PINMUX_GPIO118__FUNC_I0_APU_JTAG_TRST (MTK_PIN_NO(118) | 4)
-+#define PINMUX_GPIO118__FUNC_I0_SCP_JTAG1_TRSTN (MTK_PIN_NO(118) | 5)
-+#define PINMUX_GPIO118__FUNC_I0_SPM_JTAG_TRSTN (MTK_PIN_NO(118) | 6)
-+#define PINMUX_GPIO118__FUNC_B0_DBG_MON_B20 (MTK_PIN_NO(118) | 7)
-+
-+#define PINMUX_GPIO119__FUNC_B_GPIO119 (MTK_PIN_NO(119) | 0)
-+#define PINMUX_GPIO119__FUNC_O_I2SO2_D2 (MTK_PIN_NO(119) | 1)
-+#define PINMUX_GPIO119__FUNC_I0_I2SIN_D2 (MTK_PIN_NO(119) | 2)
-+#define PINMUX_GPIO119__FUNC_O_UTXD3 (MTK_PIN_NO(119) | 3)
-+#define PINMUX_GPIO119__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(119) | 4)
-+#define PINMUX_GPIO119__FUNC_O_I2SO1_MCK (MTK_PIN_NO(119) | 5)
-+#define PINMUX_GPIO119__FUNC_O_SSPM_UTXD_AO (MTK_PIN_NO(119) | 6)
-+#define PINMUX_GPIO119__FUNC_B0_DBG_MON_B21 (MTK_PIN_NO(119) | 7)
-+
-+#define PINMUX_GPIO120__FUNC_B_GPIO120 (MTK_PIN_NO(120) | 0)
-+#define PINMUX_GPIO120__FUNC_O_I2SO2_D3 (MTK_PIN_NO(120) | 1)
-+#define PINMUX_GPIO120__FUNC_I0_I2SIN_D3 (MTK_PIN_NO(120) | 2)
-+#define PINMUX_GPIO120__FUNC_I1_URXD3 (MTK_PIN_NO(120) | 3)
-+#define PINMUX_GPIO120__FUNC_I0_TDMIN_DI (MTK_PIN_NO(120) | 4)
-+#define PINMUX_GPIO120__FUNC_O_I2SO1_BCK (MTK_PIN_NO(120) | 5)
-+#define PINMUX_GPIO120__FUNC_I1_SSPM_URXD_AO (MTK_PIN_NO(120) | 6)
-+#define PINMUX_GPIO120__FUNC_B0_DBG_MON_B22 (MTK_PIN_NO(120) | 7)
-+
-+#define PINMUX_GPIO121__FUNC_B_GPIO121 (MTK_PIN_NO(121) | 0)
-+#define PINMUX_GPIO121__FUNC_B0_PCM_CLK (MTK_PIN_NO(121) | 1)
-+#define PINMUX_GPIO121__FUNC_O_SPIM4_CSB (MTK_PIN_NO(121) | 2)
-+#define PINMUX_GPIO121__FUNC_O_SCP_SPI1_B_CS (MTK_PIN_NO(121) | 3)
-+#define PINMUX_GPIO121__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(121) | 4)
-+#define PINMUX_GPIO121__FUNC_O_AUXIF_ST0 (MTK_PIN_NO(121) | 5)
-+#define PINMUX_GPIO121__FUNC_O_PGD_DA_EFUSE_RDY (MTK_PIN_NO(121) | 6)
-+#define PINMUX_GPIO121__FUNC_B0_DBG_MON_B23 (MTK_PIN_NO(121) | 7)
-+
-+#define PINMUX_GPIO122__FUNC_B_GPIO122 (MTK_PIN_NO(122) | 0)
-+#define PINMUX_GPIO122__FUNC_B0_PCM_SYNC (MTK_PIN_NO(122) | 1)
-+#define PINMUX_GPIO122__FUNC_O_SPIM4_CLK (MTK_PIN_NO(122) | 2)
-+#define PINMUX_GPIO122__FUNC_O_SCP_SPI1_B_CK (MTK_PIN_NO(122) | 3)
-+#define PINMUX_GPIO122__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(122) | 4)
-+#define PINMUX_GPIO122__FUNC_O_AUXIF_CLK0 (MTK_PIN_NO(122) | 5)
-+#define PINMUX_GPIO122__FUNC_O_PGD_DA_EFUSE_RDY_PRE (MTK_PIN_NO(122) | 6)
-+#define PINMUX_GPIO122__FUNC_B0_DBG_MON_B24 (MTK_PIN_NO(122) | 7)
-+
-+#define PINMUX_GPIO123__FUNC_B_GPIO123 (MTK_PIN_NO(123) | 0)
-+#define PINMUX_GPIO123__FUNC_O_PCM_DO (MTK_PIN_NO(123) | 1)
-+#define PINMUX_GPIO123__FUNC_B0_SPIM4_MOSI (MTK_PIN_NO(123) | 2)
-+#define PINMUX_GPIO123__FUNC_O_SCP_SPI1_B_MO (MTK_PIN_NO(123) | 3)
-+#define PINMUX_GPIO123__FUNC_O_TP_URTS2_AO (MTK_PIN_NO(123) | 4)
-+#define PINMUX_GPIO123__FUNC_O_AUXIF_ST1 (MTK_PIN_NO(123) | 5)
-+#define PINMUX_GPIO123__FUNC_O_PGD_DA_PWRGD_RESET (MTK_PIN_NO(123) | 6)
-+#define PINMUX_GPIO123__FUNC_B0_DBG_MON_B25 (MTK_PIN_NO(123) | 7)
-+
-+#define PINMUX_GPIO124__FUNC_B_GPIO124 (MTK_PIN_NO(124) | 0)
-+#define PINMUX_GPIO124__FUNC_I0_PCM_DI (MTK_PIN_NO(124) | 1)
-+#define PINMUX_GPIO124__FUNC_B0_SPIM4_MISO (MTK_PIN_NO(124) | 2)
-+#define PINMUX_GPIO124__FUNC_I0_SCP_SPI1_B_MI (MTK_PIN_NO(124) | 3)
-+#define PINMUX_GPIO124__FUNC_I1_TP_UCTS2_AO (MTK_PIN_NO(124) | 4)
-+#define PINMUX_GPIO124__FUNC_O_AUXIF_CLK1 (MTK_PIN_NO(124) | 5)
-+#define PINMUX_GPIO124__FUNC_O_PGD_DA_PWRGD_ENB (MTK_PIN_NO(124) | 6)
-+#define PINMUX_GPIO124__FUNC_B0_DBG_MON_B26 (MTK_PIN_NO(124) | 7)
-+
-+#define PINMUX_GPIO125__FUNC_B_GPIO125 (MTK_PIN_NO(125) | 0)
-+#define PINMUX_GPIO125__FUNC_O_DMIC1_CLK (MTK_PIN_NO(125) | 1)
-+#define PINMUX_GPIO125__FUNC_O_SPINOR_CK (MTK_PIN_NO(125) | 2)
-+#define PINMUX_GPIO125__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(125) | 3)
-+#define PINMUX_GPIO125__FUNC_O_LVTS_FOUT (MTK_PIN_NO(125) | 6)
-+#define PINMUX_GPIO125__FUNC_B0_DBG_MON_B27 (MTK_PIN_NO(125) | 7)
-+
-+#define PINMUX_GPIO126__FUNC_B_GPIO126 (MTK_PIN_NO(126) | 0)
-+#define PINMUX_GPIO126__FUNC_I0_DMIC1_DAT (MTK_PIN_NO(126) | 1)
-+#define PINMUX_GPIO126__FUNC_O_SPINOR_CS (MTK_PIN_NO(126) | 2)
-+#define PINMUX_GPIO126__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(126) | 3)
-+#define PINMUX_GPIO126__FUNC_O_LVTS_SDO (MTK_PIN_NO(126) | 6)
-+#define PINMUX_GPIO126__FUNC_B0_DBG_MON_B28 (MTK_PIN_NO(126) | 7)
-+
-+#define PINMUX_GPIO127__FUNC_B_GPIO127 (MTK_PIN_NO(127) | 0)
-+#define PINMUX_GPIO127__FUNC_I0_DMIC1_DAT_R (MTK_PIN_NO(127) | 1)
-+#define PINMUX_GPIO127__FUNC_B0_SPINOR_IO0 (MTK_PIN_NO(127) | 2)
-+#define PINMUX_GPIO127__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(127) | 3)
-+#define PINMUX_GPIO127__FUNC_I0_LVTS_26M (MTK_PIN_NO(127) | 6)
-+#define PINMUX_GPIO127__FUNC_B0_DBG_MON_B29 (MTK_PIN_NO(127) | 7)
-+
-+#define PINMUX_GPIO128__FUNC_B_GPIO128 (MTK_PIN_NO(128) | 0)
-+#define PINMUX_GPIO128__FUNC_O_DMIC2_CLK (MTK_PIN_NO(128) | 1)
-+#define PINMUX_GPIO128__FUNC_B0_SPINOR_IO1 (MTK_PIN_NO(128) | 2)
-+#define PINMUX_GPIO128__FUNC_I0_TDMIN_DI (MTK_PIN_NO(128) | 3)
-+#define PINMUX_GPIO128__FUNC_I0_LVTS_SCF (MTK_PIN_NO(128) | 6)
-+#define PINMUX_GPIO128__FUNC_B0_DBG_MON_B30 (MTK_PIN_NO(128) | 7)
-+
-+#define PINMUX_GPIO129__FUNC_B_GPIO129 (MTK_PIN_NO(129) | 0)
-+#define PINMUX_GPIO129__FUNC_I0_DMIC2_DAT (MTK_PIN_NO(129) | 1)
-+#define PINMUX_GPIO129__FUNC_B0_SPINOR_IO2 (MTK_PIN_NO(129) | 2)
-+#define PINMUX_GPIO129__FUNC_I0_SPDIF_IN1 (MTK_PIN_NO(129) | 3)
-+#define PINMUX_GPIO129__FUNC_I0_LVTS_SCK (MTK_PIN_NO(129) | 6)
-+#define PINMUX_GPIO129__FUNC_B0_DBG_MON_B31 (MTK_PIN_NO(129) | 7)
-+
-+#define PINMUX_GPIO130__FUNC_B_GPIO130 (MTK_PIN_NO(130) | 0)
-+#define PINMUX_GPIO130__FUNC_I0_DMIC2_DAT_R (MTK_PIN_NO(130) | 1)
-+#define PINMUX_GPIO130__FUNC_B0_SPINOR_IO3 (MTK_PIN_NO(130) | 2)
-+#define PINMUX_GPIO130__FUNC_I0_SPDIF_IN2 (MTK_PIN_NO(130) | 3)
-+#define PINMUX_GPIO130__FUNC_I0_LVTS_SDI (MTK_PIN_NO(130) | 6)
-+#define PINMUX_GPIO130__FUNC_B0_DBG_MON_B32 (MTK_PIN_NO(130) | 7)
-+
-+#define PINMUX_GPIO131__FUNC_B_GPIO131 (MTK_PIN_NO(131) | 0)
-+#define PINMUX_GPIO131__FUNC_O_DPI_D0 (MTK_PIN_NO(131) | 1)
-+#define PINMUX_GPIO131__FUNC_O_GBE_TXD3 (MTK_PIN_NO(131) | 2)
-+#define PINMUX_GPIO131__FUNC_O_DMIC1_CLK (MTK_PIN_NO(131) | 3)
-+#define PINMUX_GPIO131__FUNC_O_I2SO2_MCK (MTK_PIN_NO(131) | 4)
-+#define PINMUX_GPIO131__FUNC_B0_TP_GPIO0_AO (MTK_PIN_NO(131) | 5)
-+#define PINMUX_GPIO131__FUNC_O_SPIM5_CSB (MTK_PIN_NO(131) | 6)
-+#define PINMUX_GPIO131__FUNC_O_PGD_LV_HSC_PWR0 (MTK_PIN_NO(131) | 7)
-+
-+#define PINMUX_GPIO132__FUNC_B_GPIO132 (MTK_PIN_NO(132) | 0)
-+#define PINMUX_GPIO132__FUNC_O_DPI_D1 (MTK_PIN_NO(132) | 1)
-+#define PINMUX_GPIO132__FUNC_O_GBE_TXD2 (MTK_PIN_NO(132) | 2)
-+#define PINMUX_GPIO132__FUNC_I0_DMIC1_DAT (MTK_PIN_NO(132) | 3)
-+#define PINMUX_GPIO132__FUNC_B0_I2SO2_BCK (MTK_PIN_NO(132) | 4)
-+#define PINMUX_GPIO132__FUNC_B0_TP_GPIO1_AO (MTK_PIN_NO(132) | 5)
-+#define PINMUX_GPIO132__FUNC_O_SPIM5_CLK (MTK_PIN_NO(132) | 6)
-+#define PINMUX_GPIO132__FUNC_O_PGD_LV_HSC_PWR1 (MTK_PIN_NO(132) | 7)
-+
-+#define PINMUX_GPIO133__FUNC_B_GPIO133 (MTK_PIN_NO(133) | 0)
-+#define PINMUX_GPIO133__FUNC_O_DPI_D2 (MTK_PIN_NO(133) | 1)
-+#define PINMUX_GPIO133__FUNC_O_GBE_TXD1 (MTK_PIN_NO(133) | 2)
-+#define PINMUX_GPIO133__FUNC_I0_DMIC1_DAT_R (MTK_PIN_NO(133) | 3)
-+#define PINMUX_GPIO133__FUNC_B0_I2SO2_WS (MTK_PIN_NO(133) | 4)
-+#define PINMUX_GPIO133__FUNC_B0_TP_GPIO2_AO (MTK_PIN_NO(133) | 5)
-+#define PINMUX_GPIO133__FUNC_B0_SPIM5_MOSI (MTK_PIN_NO(133) | 6)
-+#define PINMUX_GPIO133__FUNC_O_PGD_LV_HSC_PWR2 (MTK_PIN_NO(133) | 7)
-+
-+#define PINMUX_GPIO134__FUNC_B_GPIO134 (MTK_PIN_NO(134) | 0)
-+#define PINMUX_GPIO134__FUNC_O_DPI_D3 (MTK_PIN_NO(134) | 1)
-+#define PINMUX_GPIO134__FUNC_O_GBE_TXD0 (MTK_PIN_NO(134) | 2)
-+#define PINMUX_GPIO134__FUNC_O_DMIC2_CLK (MTK_PIN_NO(134) | 3)
-+#define PINMUX_GPIO134__FUNC_O_I2SO2_D0 (MTK_PIN_NO(134) | 4)
-+#define PINMUX_GPIO134__FUNC_B0_TP_GPIO3_AO (MTK_PIN_NO(134) | 5)
-+#define PINMUX_GPIO134__FUNC_B0_SPIM5_MISO (MTK_PIN_NO(134) | 6)
-+#define PINMUX_GPIO134__FUNC_O_PGD_LV_HSC_PWR3 (MTK_PIN_NO(134) | 7)
-+
-+#define PINMUX_GPIO135__FUNC_B_GPIO135 (MTK_PIN_NO(135) | 0)
-+#define PINMUX_GPIO135__FUNC_O_DPI_D4 (MTK_PIN_NO(135) | 1)
-+#define PINMUX_GPIO135__FUNC_I0_GBE_RXD3 (MTK_PIN_NO(135) | 2)
-+#define PINMUX_GPIO135__FUNC_I0_DMIC2_DAT (MTK_PIN_NO(135) | 3)
-+#define PINMUX_GPIO135__FUNC_O_I2SO2_D1 (MTK_PIN_NO(135) | 4)
-+#define PINMUX_GPIO135__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(135) | 5)
-+#define PINMUX_GPIO135__FUNC_I1_WAKEN (MTK_PIN_NO(135) | 6)
-+#define PINMUX_GPIO135__FUNC_O_PGD_LV_HSC_PWR4 (MTK_PIN_NO(135) | 7)
-+
-+#define PINMUX_GPIO136__FUNC_B_GPIO136 (MTK_PIN_NO(136) | 0)
-+#define PINMUX_GPIO136__FUNC_O_DPI_D5 (MTK_PIN_NO(136) | 1)
-+#define PINMUX_GPIO136__FUNC_I0_GBE_RXD2 (MTK_PIN_NO(136) | 2)
-+#define PINMUX_GPIO136__FUNC_I0_DMIC2_DAT_R (MTK_PIN_NO(136) | 3)
-+#define PINMUX_GPIO136__FUNC_O_I2SO2_D2 (MTK_PIN_NO(136) | 4)
-+#define PINMUX_GPIO136__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(136) | 5)
-+#define PINMUX_GPIO136__FUNC_O_PERSTN (MTK_PIN_NO(136) | 6)
-+#define PINMUX_GPIO136__FUNC_O_PGD_LV_HSC_PWR5 (MTK_PIN_NO(136) | 7)
-+
-+#define PINMUX_GPIO137__FUNC_B_GPIO137 (MTK_PIN_NO(137) | 0)
-+#define PINMUX_GPIO137__FUNC_O_DPI_D6 (MTK_PIN_NO(137) | 1)
-+#define PINMUX_GPIO137__FUNC_I0_GBE_RXD1 (MTK_PIN_NO(137) | 2)
-+#define PINMUX_GPIO137__FUNC_O_DMIC3_CLK (MTK_PIN_NO(137) | 3)
-+#define PINMUX_GPIO137__FUNC_O_I2SO2_D3 (MTK_PIN_NO(137) | 4)
-+#define PINMUX_GPIO137__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(137) | 5)
-+#define PINMUX_GPIO137__FUNC_B1_CLKREQN (MTK_PIN_NO(137) | 6)
-+#define PINMUX_GPIO137__FUNC_O_PWM_0 (MTK_PIN_NO(137) | 7)
-+
-+#define PINMUX_GPIO138__FUNC_B_GPIO138 (MTK_PIN_NO(138) | 0)
-+#define PINMUX_GPIO138__FUNC_O_DPI_D7 (MTK_PIN_NO(138) | 1)
-+#define PINMUX_GPIO138__FUNC_I0_GBE_RXD0 (MTK_PIN_NO(138) | 2)
-+#define PINMUX_GPIO138__FUNC_I0_DMIC3_DAT (MTK_PIN_NO(138) | 3)
-+#define PINMUX_GPIO138__FUNC_O_CLKM2 (MTK_PIN_NO(138) | 4)
-+#define PINMUX_GPIO138__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(138) | 5)
-+#define PINMUX_GPIO138__FUNC_B0_MD32_0_GPIO0 (MTK_PIN_NO(138) | 7)
-+
-+#define PINMUX_GPIO139__FUNC_B_GPIO139 (MTK_PIN_NO(139) | 0)
-+#define PINMUX_GPIO139__FUNC_O_DPI_D8 (MTK_PIN_NO(139) | 1)
-+#define PINMUX_GPIO139__FUNC_B0_GBE_TXC (MTK_PIN_NO(139) | 2)
-+#define PINMUX_GPIO139__FUNC_I0_DMIC3_DAT_R (MTK_PIN_NO(139) | 3)
-+#define PINMUX_GPIO139__FUNC_O_CLKM3 (MTK_PIN_NO(139) | 4)
-+#define PINMUX_GPIO139__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(139) | 5)
-+#define PINMUX_GPIO139__FUNC_O_UTXD2 (MTK_PIN_NO(139) | 6)
-+#define PINMUX_GPIO139__FUNC_B0_MD32_0_GPIO1 (MTK_PIN_NO(139) | 7)
-+
-+#define PINMUX_GPIO140__FUNC_B_GPIO140 (MTK_PIN_NO(140) | 0)
-+#define PINMUX_GPIO140__FUNC_O_DPI_D9 (MTK_PIN_NO(140) | 1)
-+#define PINMUX_GPIO140__FUNC_I0_GBE_RXC (MTK_PIN_NO(140) | 2)
-+#define PINMUX_GPIO140__FUNC_O_DMIC4_CLK (MTK_PIN_NO(140) | 3)
-+#define PINMUX_GPIO140__FUNC_O_PWM_2 (MTK_PIN_NO(140) | 4)
-+#define PINMUX_GPIO140__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(140) | 5)
-+#define PINMUX_GPIO140__FUNC_I1_URXD2 (MTK_PIN_NO(140) | 6)
-+#define PINMUX_GPIO140__FUNC_B0_MD32_0_GPIO2 (MTK_PIN_NO(140) | 7)
-+
-+#define PINMUX_GPIO141__FUNC_B_GPIO141 (MTK_PIN_NO(141) | 0)
-+#define PINMUX_GPIO141__FUNC_O_DPI_D10 (MTK_PIN_NO(141) | 1)
-+#define PINMUX_GPIO141__FUNC_I0_GBE_RXDV (MTK_PIN_NO(141) | 2)
-+#define PINMUX_GPIO141__FUNC_I0_DMIC4_DAT (MTK_PIN_NO(141) | 3)
-+#define PINMUX_GPIO141__FUNC_O_PWM_3 (MTK_PIN_NO(141) | 4)
-+#define PINMUX_GPIO141__FUNC_O_TP_URTS2_AO (MTK_PIN_NO(141) | 5)
-+#define PINMUX_GPIO141__FUNC_O_URTS2 (MTK_PIN_NO(141) | 6)
-+#define PINMUX_GPIO141__FUNC_B0_MD32_1_GPIO0 (MTK_PIN_NO(141) | 7)
-+
-+#define PINMUX_GPIO142__FUNC_B_GPIO142 (MTK_PIN_NO(142) | 0)
-+#define PINMUX_GPIO142__FUNC_O_DPI_D11 (MTK_PIN_NO(142) | 1)
-+#define PINMUX_GPIO142__FUNC_O_GBE_TXEN (MTK_PIN_NO(142) | 2)
-+#define PINMUX_GPIO142__FUNC_I0_DMIC4_DAT_R (MTK_PIN_NO(142) | 3)
-+#define PINMUX_GPIO142__FUNC_O_PWM_1 (MTK_PIN_NO(142) | 4)
-+#define PINMUX_GPIO142__FUNC_I1_TP_UCTS2_AO (MTK_PIN_NO(142) | 5)
-+#define PINMUX_GPIO142__FUNC_I1_UCTS2 (MTK_PIN_NO(142) | 6)
-+#define PINMUX_GPIO142__FUNC_B0_MD32_1_GPIO1 (MTK_PIN_NO(142) | 7)
-+
-+#define PINMUX_GPIO143__FUNC_B_GPIO143 (MTK_PIN_NO(143) | 0)
-+#define PINMUX_GPIO143__FUNC_O_DPI_D12 (MTK_PIN_NO(143) | 1)
-+#define PINMUX_GPIO143__FUNC_O_GBE_MDC (MTK_PIN_NO(143) | 2)
-+#define PINMUX_GPIO143__FUNC_B0_MD32_0_GPIO0 (MTK_PIN_NO(143) | 3)
-+#define PINMUX_GPIO143__FUNC_O_CLKM0 (MTK_PIN_NO(143) | 4)
-+#define PINMUX_GPIO143__FUNC_O_SPIM3_CSB (MTK_PIN_NO(143) | 5)
-+#define PINMUX_GPIO143__FUNC_O_UTXD1 (MTK_PIN_NO(143) | 6)
-+#define PINMUX_GPIO143__FUNC_B0_MD32_1_GPIO2 (MTK_PIN_NO(143) | 7)
-+
-+#define PINMUX_GPIO144__FUNC_B_GPIO144 (MTK_PIN_NO(144) | 0)
-+#define PINMUX_GPIO144__FUNC_O_DPI_D13 (MTK_PIN_NO(144) | 1)
-+#define PINMUX_GPIO144__FUNC_B1_GBE_MDIO (MTK_PIN_NO(144) | 2)
-+#define PINMUX_GPIO144__FUNC_B0_MD32_0_GPIO1 (MTK_PIN_NO(144) | 3)
-+#define PINMUX_GPIO144__FUNC_O_CLKM1 (MTK_PIN_NO(144) | 4)
-+#define PINMUX_GPIO144__FUNC_O_SPIM3_CLK (MTK_PIN_NO(144) | 5)
-+#define PINMUX_GPIO144__FUNC_I1_URXD1 (MTK_PIN_NO(144) | 6)
-+#define PINMUX_GPIO144__FUNC_O_PGD_HV_HSC_PWR0 (MTK_PIN_NO(144) | 7)
-+
-+#define PINMUX_GPIO145__FUNC_B_GPIO145 (MTK_PIN_NO(145) | 0)
-+#define PINMUX_GPIO145__FUNC_O_DPI_D14 (MTK_PIN_NO(145) | 1)
-+#define PINMUX_GPIO145__FUNC_O_GBE_TXER (MTK_PIN_NO(145) | 2)
-+#define PINMUX_GPIO145__FUNC_B0_MD32_1_GPIO0 (MTK_PIN_NO(145) | 3)
-+#define PINMUX_GPIO145__FUNC_O_CMFLASH0 (MTK_PIN_NO(145) | 4)
-+#define PINMUX_GPIO145__FUNC_B0_SPIM3_MOSI (MTK_PIN_NO(145) | 5)
-+#define PINMUX_GPIO145__FUNC_B0_GBE_AUX_PPS2 (MTK_PIN_NO(145) | 6)
-+#define PINMUX_GPIO145__FUNC_O_PGD_HV_HSC_PWR1 (MTK_PIN_NO(145) | 7)
-+
-+#define PINMUX_GPIO146__FUNC_B_GPIO146 (MTK_PIN_NO(146) | 0)
-+#define PINMUX_GPIO146__FUNC_O_DPI_D15 (MTK_PIN_NO(146) | 1)
-+#define PINMUX_GPIO146__FUNC_I0_GBE_RXER (MTK_PIN_NO(146) | 2)
-+#define PINMUX_GPIO146__FUNC_B0_MD32_1_GPIO1 (MTK_PIN_NO(146) | 3)
-+#define PINMUX_GPIO146__FUNC_O_CMFLASH1 (MTK_PIN_NO(146) | 4)
-+#define PINMUX_GPIO146__FUNC_B0_SPIM3_MISO (MTK_PIN_NO(146) | 5)
-+#define PINMUX_GPIO146__FUNC_B0_GBE_AUX_PPS3 (MTK_PIN_NO(146) | 6)
-+#define PINMUX_GPIO146__FUNC_O_PGD_HV_HSC_PWR2 (MTK_PIN_NO(146) | 7)
-+
-+#define PINMUX_GPIO147__FUNC_B_GPIO147 (MTK_PIN_NO(147) | 0)
-+#define PINMUX_GPIO147__FUNC_O_DPI_HSYNC (MTK_PIN_NO(147) | 1)
-+#define PINMUX_GPIO147__FUNC_I0_GBE_COL (MTK_PIN_NO(147) | 2)
-+#define PINMUX_GPIO147__FUNC_O_I2SO1_MCK (MTK_PIN_NO(147) | 3)
-+#define PINMUX_GPIO147__FUNC_O_CMVREF0 (MTK_PIN_NO(147) | 4)
-+#define PINMUX_GPIO147__FUNC_O_SPDIF_OUT (MTK_PIN_NO(147) | 5)
-+#define PINMUX_GPIO147__FUNC_O_URTS1 (MTK_PIN_NO(147) | 6)
-+#define PINMUX_GPIO147__FUNC_O_PGD_HV_HSC_PWR3 (MTK_PIN_NO(147) | 7)
-+
-+#define PINMUX_GPIO148__FUNC_B_GPIO148 (MTK_PIN_NO(148) | 0)
-+#define PINMUX_GPIO148__FUNC_O_DPI_VSYNC (MTK_PIN_NO(148) | 1)
-+#define PINMUX_GPIO148__FUNC_I0_GBE_INTR (MTK_PIN_NO(148) | 2)
-+#define PINMUX_GPIO148__FUNC_O_I2SO1_BCK (MTK_PIN_NO(148) | 3)
-+#define PINMUX_GPIO148__FUNC_O_CMVREF1 (MTK_PIN_NO(148) | 4)
-+#define PINMUX_GPIO148__FUNC_I0_SPDIF_IN0 (MTK_PIN_NO(148) | 5)
-+#define PINMUX_GPIO148__FUNC_I1_UCTS1 (MTK_PIN_NO(148) | 6)
-+#define PINMUX_GPIO148__FUNC_O_PGD_HV_HSC_PWR4 (MTK_PIN_NO(148) | 7)
-+
-+#define PINMUX_GPIO149__FUNC_B_GPIO149 (MTK_PIN_NO(149) | 0)
-+#define PINMUX_GPIO149__FUNC_O_DPI_DE (MTK_PIN_NO(149) | 1)
-+#define PINMUX_GPIO149__FUNC_B0_GBE_AUX_PPS0 (MTK_PIN_NO(149) | 2)
-+#define PINMUX_GPIO149__FUNC_O_I2SO1_WS (MTK_PIN_NO(149) | 3)
-+#define PINMUX_GPIO149__FUNC_O_CMVREF2 (MTK_PIN_NO(149) | 4)
-+#define PINMUX_GPIO149__FUNC_I0_SPDIF_IN1 (MTK_PIN_NO(149) | 5)
-+#define PINMUX_GPIO149__FUNC_O_UTXD3 (MTK_PIN_NO(149) | 6)
-+#define PINMUX_GPIO149__FUNC_O_PGD_HV_HSC_PWR5 (MTK_PIN_NO(149) | 7)
-+
-+#define PINMUX_GPIO150__FUNC_B_GPIO150 (MTK_PIN_NO(150) | 0)
-+#define PINMUX_GPIO150__FUNC_O_DPI_CK (MTK_PIN_NO(150) | 1)
-+#define PINMUX_GPIO150__FUNC_B0_GBE_AUX_PPS1 (MTK_PIN_NO(150) | 2)
-+#define PINMUX_GPIO150__FUNC_O_I2SO1_D0 (MTK_PIN_NO(150) | 3)
-+#define PINMUX_GPIO150__FUNC_O_CMVREF3 (MTK_PIN_NO(150) | 4)
-+#define PINMUX_GPIO150__FUNC_I0_SPDIF_IN2 (MTK_PIN_NO(150) | 5)
-+#define PINMUX_GPIO150__FUNC_I1_URXD3 (MTK_PIN_NO(150) | 6)
-+
-+#define PINMUX_GPIO151__FUNC_B_GPIO151 (MTK_PIN_NO(151) | 0)
-+#define PINMUX_GPIO151__FUNC_B1_MSDC0_DAT7 (MTK_PIN_NO(151) | 1)
-+
-+#define PINMUX_GPIO152__FUNC_B_GPIO152 (MTK_PIN_NO(152) | 0)
-+#define PINMUX_GPIO152__FUNC_B1_MSDC0_DAT6 (MTK_PIN_NO(152) | 1)
-+
-+#define PINMUX_GPIO153__FUNC_B_GPIO153 (MTK_PIN_NO(153) | 0)
-+#define PINMUX_GPIO153__FUNC_B1_MSDC0_DAT5 (MTK_PIN_NO(153) | 1)
-+
-+#define PINMUX_GPIO154__FUNC_B_GPIO154 (MTK_PIN_NO(154) | 0)
-+#define PINMUX_GPIO154__FUNC_B1_MSDC0_DAT4 (MTK_PIN_NO(154) | 1)
-+
-+#define PINMUX_GPIO155__FUNC_B_GPIO155 (MTK_PIN_NO(155) | 0)
-+#define PINMUX_GPIO155__FUNC_O_MSDC0_RSTB (MTK_PIN_NO(155) | 1)
-+
-+#define PINMUX_GPIO156__FUNC_B_GPIO156 (MTK_PIN_NO(156) | 0)
-+#define PINMUX_GPIO156__FUNC_B1_MSDC0_CMD (MTK_PIN_NO(156) | 1)
-+
-+#define PINMUX_GPIO157__FUNC_B_GPIO157 (MTK_PIN_NO(157) | 0)
-+#define PINMUX_GPIO157__FUNC_B1_MSDC0_CLK (MTK_PIN_NO(157) | 1)
-+
-+#define PINMUX_GPIO158__FUNC_B_GPIO158 (MTK_PIN_NO(158) | 0)
-+#define PINMUX_GPIO158__FUNC_B1_MSDC0_DAT3 (MTK_PIN_NO(158) | 1)
-+
-+#define PINMUX_GPIO159__FUNC_B_GPIO159 (MTK_PIN_NO(159) | 0)
-+#define PINMUX_GPIO159__FUNC_B1_MSDC0_DAT2 (MTK_PIN_NO(159) | 1)
-+
-+#define PINMUX_GPIO160__FUNC_B_GPIO160 (MTK_PIN_NO(160) | 0)
-+#define PINMUX_GPIO160__FUNC_B1_MSDC0_DAT1 (MTK_PIN_NO(160) | 1)
-+
-+#define PINMUX_GPIO161__FUNC_B_GPIO161 (MTK_PIN_NO(161) | 0)
-+#define PINMUX_GPIO161__FUNC_B1_MSDC0_DAT0 (MTK_PIN_NO(161) | 1)
-+
-+#define PINMUX_GPIO162__FUNC_B_GPIO162 (MTK_PIN_NO(162) | 0)
-+#define PINMUX_GPIO162__FUNC_B0_MSDC0_DSL (MTK_PIN_NO(162) | 1)
-+
-+#define PINMUX_GPIO163__FUNC_B_GPIO163 (MTK_PIN_NO(163) | 0)
-+#define PINMUX_GPIO163__FUNC_B1_MSDC1_CMD (MTK_PIN_NO(163) | 1)
-+#define PINMUX_GPIO163__FUNC_O_SPDIF_OUT (MTK_PIN_NO(163) | 2)
-+#define PINMUX_GPIO163__FUNC_I1_MD32_0_JTAG_TMS (MTK_PIN_NO(163) | 3)
-+#define PINMUX_GPIO163__FUNC_I1_ADSP_JTAG0_TMS (MTK_PIN_NO(163) | 4)
-+#define PINMUX_GPIO163__FUNC_I1_SCP_JTAG0_TMS (MTK_PIN_NO(163) | 5)
-+#define PINMUX_GPIO163__FUNC_I1_CCU0_JTAG_TMS (MTK_PIN_NO(163) | 6)
-+#define PINMUX_GPIO163__FUNC_I0_IPU_JTAG_TMS (MTK_PIN_NO(163) | 7)
-+
-+#define PINMUX_GPIO164__FUNC_B_GPIO164 (MTK_PIN_NO(164) | 0)
-+#define PINMUX_GPIO164__FUNC_B1_MSDC1_CLK (MTK_PIN_NO(164) | 1)
-+#define PINMUX_GPIO164__FUNC_I0_SPDIF_IN0 (MTK_PIN_NO(164) | 2)
-+#define PINMUX_GPIO164__FUNC_I1_MD32_0_JTAG_TCK (MTK_PIN_NO(164) | 3)
-+#define PINMUX_GPIO164__FUNC_I0_ADSP_JTAG0_TCK (MTK_PIN_NO(164) | 4)
-+#define PINMUX_GPIO164__FUNC_I1_SCP_JTAG0_TCK (MTK_PIN_NO(164) | 5)
-+#define PINMUX_GPIO164__FUNC_I1_CCU0_JTAG_TCK (MTK_PIN_NO(164) | 6)
-+#define PINMUX_GPIO164__FUNC_I0_IPU_JTAG_TCK (MTK_PIN_NO(164) | 7)
-+
-+#define PINMUX_GPIO165__FUNC_B_GPIO165 (MTK_PIN_NO(165) | 0)
-+#define PINMUX_GPIO165__FUNC_B1_MSDC1_DAT0 (MTK_PIN_NO(165) | 1)
-+#define PINMUX_GPIO165__FUNC_I0_SPDIF_IN1 (MTK_PIN_NO(165) | 2)
-+#define PINMUX_GPIO165__FUNC_I1_MD32_0_JTAG_TDI (MTK_PIN_NO(165) | 3)
-+#define PINMUX_GPIO165__FUNC_I1_ADSP_JTAG0_TDI (MTK_PIN_NO(165) | 4)
-+#define PINMUX_GPIO165__FUNC_I1_SCP_JTAG0_TDI (MTK_PIN_NO(165) | 5)
-+#define PINMUX_GPIO165__FUNC_I1_CCU0_JTAG_TDI (MTK_PIN_NO(165) | 6)
-+#define PINMUX_GPIO165__FUNC_I0_IPU_JTAG_TDI (MTK_PIN_NO(165) | 7)
-+
-+#define PINMUX_GPIO166__FUNC_B_GPIO166 (MTK_PIN_NO(166) | 0)
-+#define PINMUX_GPIO166__FUNC_B1_MSDC1_DAT1 (MTK_PIN_NO(166) | 1)
-+#define PINMUX_GPIO166__FUNC_I0_SPDIF_IN2 (MTK_PIN_NO(166) | 2)
-+#define PINMUX_GPIO166__FUNC_O_MD32_0_JTAG_TDO (MTK_PIN_NO(166) | 3)
-+#define PINMUX_GPIO166__FUNC_O_ADSP_JTAG0_TDO (MTK_PIN_NO(166) | 4)
-+#define PINMUX_GPIO166__FUNC_O_SCP_JTAG0_TDO (MTK_PIN_NO(166) | 5)
-+#define PINMUX_GPIO166__FUNC_O_CCU0_JTAG_TDO (MTK_PIN_NO(166) | 6)
-+#define PINMUX_GPIO166__FUNC_O_IPU_JTAG_TDO (MTK_PIN_NO(166) | 7)
-+
-+#define PINMUX_GPIO167__FUNC_B_GPIO167 (MTK_PIN_NO(167) | 0)
-+#define PINMUX_GPIO167__FUNC_B1_MSDC1_DAT2 (MTK_PIN_NO(167) | 1)
-+#define PINMUX_GPIO167__FUNC_O_PWM_0 (MTK_PIN_NO(167) | 2)
-+#define PINMUX_GPIO167__FUNC_I1_MD32_0_JTAG_TRST (MTK_PIN_NO(167) | 3)
-+#define PINMUX_GPIO167__FUNC_I1_ADSP_JTAG0_TRSTN (MTK_PIN_NO(167) | 4)
-+#define PINMUX_GPIO167__FUNC_I0_SCP_JTAG0_TRSTN (MTK_PIN_NO(167) | 5)
-+#define PINMUX_GPIO167__FUNC_I1_CCU0_JTAG_TRST (MTK_PIN_NO(167) | 6)
-+#define PINMUX_GPIO167__FUNC_I0_IPU_JTAG_TRST (MTK_PIN_NO(167) | 7)
-+
-+#define PINMUX_GPIO168__FUNC_B_GPIO168 (MTK_PIN_NO(168) | 0)
-+#define PINMUX_GPIO168__FUNC_B1_MSDC1_DAT3 (MTK_PIN_NO(168) | 1)
-+#define PINMUX_GPIO168__FUNC_O_PWM_1 (MTK_PIN_NO(168) | 2)
-+#define PINMUX_GPIO168__FUNC_O_CLKM0 (MTK_PIN_NO(168) | 3)
-+
-+#define PINMUX_GPIO169__FUNC_B_GPIO169 (MTK_PIN_NO(169) | 0)
-+#define PINMUX_GPIO169__FUNC_B1_MSDC2_CMD (MTK_PIN_NO(169) | 1)
-+#define PINMUX_GPIO169__FUNC_O_LVTS_FOUT (MTK_PIN_NO(169) | 2)
-+#define PINMUX_GPIO169__FUNC_I1_MD32_1_JTAG_TMS (MTK_PIN_NO(169) | 3)
-+#define PINMUX_GPIO169__FUNC_I0_UDI_TMS (MTK_PIN_NO(169) | 4)
-+#define PINMUX_GPIO169__FUNC_I0_VPU_UDI_TMS (MTK_PIN_NO(169) | 5)
-+#define PINMUX_GPIO169__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(169) | 6)
-+#define PINMUX_GPIO169__FUNC_I1_SSPM_JTAG_TMS (MTK_PIN_NO(169) | 7)
-+
-+#define PINMUX_GPIO170__FUNC_B_GPIO170 (MTK_PIN_NO(170) | 0)
-+#define PINMUX_GPIO170__FUNC_B1_MSDC2_CLK (MTK_PIN_NO(170) | 1)
-+#define PINMUX_GPIO170__FUNC_O_LVTS_SDO (MTK_PIN_NO(170) | 2)
-+#define PINMUX_GPIO170__FUNC_I1_MD32_1_JTAG_TCK (MTK_PIN_NO(170) | 3)
-+#define PINMUX_GPIO170__FUNC_I0_UDI_TCK (MTK_PIN_NO(170) | 4)
-+#define PINMUX_GPIO170__FUNC_I0_VPU_UDI_TCK (MTK_PIN_NO(170) | 5)
-+#define PINMUX_GPIO170__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(170) | 6)
-+#define PINMUX_GPIO170__FUNC_I1_SSPM_JTAG_TCK (MTK_PIN_NO(170) | 7)
-+
-+#define PINMUX_GPIO171__FUNC_B_GPIO171 (MTK_PIN_NO(171) | 0)
-+#define PINMUX_GPIO171__FUNC_B1_MSDC2_DAT0 (MTK_PIN_NO(171) | 1)
-+#define PINMUX_GPIO171__FUNC_I0_LVTS_26M (MTK_PIN_NO(171) | 2)
-+#define PINMUX_GPIO171__FUNC_I1_MD32_1_JTAG_TDI (MTK_PIN_NO(171) | 3)
-+#define PINMUX_GPIO171__FUNC_I0_UDI_TDI (MTK_PIN_NO(171) | 4)
-+#define PINMUX_GPIO171__FUNC_I0_VPU_UDI_TDI (MTK_PIN_NO(171) | 5)
-+#define PINMUX_GPIO171__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(171) | 6)
-+#define PINMUX_GPIO171__FUNC_I1_SSPM_JTAG_TDI (MTK_PIN_NO(171) | 7)
-+
-+#define PINMUX_GPIO172__FUNC_B_GPIO172 (MTK_PIN_NO(172) | 0)
-+#define PINMUX_GPIO172__FUNC_B1_MSDC2_DAT1 (MTK_PIN_NO(172) | 1)
-+#define PINMUX_GPIO172__FUNC_I0_LVTS_SCF (MTK_PIN_NO(172) | 2)
-+#define PINMUX_GPIO172__FUNC_O_MD32_1_JTAG_TDO (MTK_PIN_NO(172) | 3)
-+#define PINMUX_GPIO172__FUNC_O_UDI_TDO (MTK_PIN_NO(172) | 4)
-+#define PINMUX_GPIO172__FUNC_O_VPU_UDI_TDO (MTK_PIN_NO(172) | 5)
-+#define PINMUX_GPIO172__FUNC_I0_TDMIN_DI (MTK_PIN_NO(172) | 6)
-+#define PINMUX_GPIO172__FUNC_O_SSPM_JTAG_TDO (MTK_PIN_NO(172) | 7)
-+
-+#define PINMUX_GPIO173__FUNC_B_GPIO173 (MTK_PIN_NO(173) | 0)
-+#define PINMUX_GPIO173__FUNC_B1_MSDC2_DAT2 (MTK_PIN_NO(173) | 1)
-+#define PINMUX_GPIO173__FUNC_I0_LVTS_SCK (MTK_PIN_NO(173) | 2)
-+#define PINMUX_GPIO173__FUNC_I1_MD32_1_JTAG_TRST (MTK_PIN_NO(173) | 3)
-+#define PINMUX_GPIO173__FUNC_I0_UDI_NTRST (MTK_PIN_NO(173) | 4)
-+#define PINMUX_GPIO173__FUNC_I0_VPU_UDI_NTRST (MTK_PIN_NO(173) | 5)
-+#define PINMUX_GPIO173__FUNC_I0_SSPM_JTAG_TRSTN (MTK_PIN_NO(173) | 7)
-+
-+#define PINMUX_GPIO174__FUNC_B_GPIO174 (MTK_PIN_NO(174) | 0)
-+#define PINMUX_GPIO174__FUNC_B1_MSDC2_DAT3 (MTK_PIN_NO(174) | 1)
-+#define PINMUX_GPIO174__FUNC_I0_LVTS_SDI (MTK_PIN_NO(174) | 2)
-+
-+#define PINMUX_GPIO175__FUNC_B_GPIO175 (MTK_PIN_NO(175) | 0)
-+#define PINMUX_GPIO175__FUNC_B0_SPMI_M_SCL (MTK_PIN_NO(175) | 1)
-+
-+#define PINMUX_GPIO176__FUNC_B_GPIO176 (MTK_PIN_NO(176) | 0)
-+#define PINMUX_GPIO176__FUNC_B0_SPMI_M_SDA (MTK_PIN_NO(176) | 1)
-+
-+#endif /* __MEDIATEK_MT8188-PINFUNC_H */
-diff --git a/include/dt-bindings/pinctrl/mt6795-pinfunc.h b/include/dt-bindings/pinctrl/mt6795-pinfunc.h
-index bd1c5a9fad06..dfd3f6f13e0d 100644
--- a/include/dt-bindings/pinctrl/mt6795-pinfunc.h
-+++ b/include/dt-bindings/pinctrl/mt6795-pinfunc.h
-@@ -4,8 +4,8 @@
-  * Author: AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
-  */
- 
-#ifndef __DTS_MT8173_PINFUNC_H
-#define __DTS_MT8173_PINFUNC_H
-+#ifndef __DTS_MT6795_PINFUNC_H
-+#define __DTS_MT6795_PINFUNC_H
- 
- #include <dt-bindings/pinctrl/mt65xx.h>
- 
-diff --git a/include/dt-bindings/pinctrl/pinctrl-cv1800b.h b/include/dt-bindings/pinctrl/pinctrl-cv1800b.h
-new file mode 100644
-index 000000000000..0593fc33d470
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/pinctrl-cv1800b.h
-@@ -0,0 +1,63 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2024 Inochi Amaoto <inochiama@outlook.com>
-+ *
-+ * This file is generated from vendor pinout definition.
-+ */
-+
-+#ifndef _DT_BINDINGS_PINCTRL_CV1800B_H
-+#define _DT_BINDINGS_PINCTRL_CV1800B_H
-+
-+#include <dt-bindings/pinctrl/pinctrl-cv18xx.h>
-+
-+#define PIN_AUD_AOUTR			1
-+#define PIN_SD0_CLK			3
-+#define PIN_SD0_CMD			4
-+#define PIN_SD0_D0			5
-+#define PIN_SD0_D1			7
-+#define PIN_SD0_D2			8
-+#define PIN_SD0_D3			9
-+#define PIN_SD0_CD			11
-+#define PIN_SD0_PWR_EN			12
-+#define PIN_SPK_EN			14
-+#define PIN_UART0_TX			15
-+#define PIN_UART0_RX			16
-+#define PIN_SPINOR_HOLD_X		17
-+#define PIN_SPINOR_SCK			18
-+#define PIN_SPINOR_MOSI			19
-+#define PIN_SPINOR_WP_X			20
-+#define PIN_SPINOR_MISO			21
-+#define PIN_SPINOR_CS_X			22
-+#define PIN_IIC0_SCL			23
-+#define PIN_IIC0_SDA			24
-+#define PIN_AUX0			25
-+#define PIN_PWR_VBAT_DET		30
-+#define PIN_PWR_SEQ2			31
-+#define PIN_XTAL_XIN			33
-+#define PIN_SD1_GPIO0			35
-+#define PIN_SD1_GPIO1			36
-+#define PIN_SD1_D3			38
-+#define PIN_SD1_D2			39
-+#define PIN_SD1_D1			40
-+#define PIN_SD1_D0			41
-+#define PIN_SD1_CMD			42
-+#define PIN_SD1_CLK			43
-+#define PIN_ADC1			44
-+#define PIN_USB_VBUS_DET		45
-+#define PIN_ETH_TXP			47
-+#define PIN_ETH_TXM			48
-+#define PIN_ETH_RXP			49
-+#define PIN_ETH_RXM			50
-+#define PIN_MIPIRX4N			56
-+#define PIN_MIPIRX4P			57
-+#define PIN_MIPIRX3N			58
-+#define PIN_MIPIRX3P			59
-+#define PIN_MIPIRX2N			60
-+#define PIN_MIPIRX2P			61
-+#define PIN_MIPIRX1N			62
-+#define PIN_MIPIRX1P			63
-+#define PIN_MIPIRX0N			64
-+#define PIN_MIPIRX0P			65
-+#define PIN_AUD_AINL_MIC		67
-+
-+#endif /* _DT_BINDINGS_PINCTRL_CV1800B_H */
-diff --git a/include/dt-bindings/pinctrl/pinctrl-cv1812h.h b/include/dt-bindings/pinctrl/pinctrl-cv1812h.h
-new file mode 100644
-index 000000000000..2908de347919
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/pinctrl-cv1812h.h
-@@ -0,0 +1,127 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2024 Inochi Amaoto <inochiama@outlook.com>
-+ *
-+ * This file is generated from vendor pinout definition.
-+ */
-+
-+#ifndef _DT_BINDINGS_PINCTRL_CV1812H_H
-+#define _DT_BINDINGS_PINCTRL_CV1812H_H
-+
-+#include <dt-bindings/pinctrl/pinctrl-cv18xx.h>
-+
-+#define PINPOS(row, col)			\
-+	((((row) - 'A' + 1) << 8) + ((col) - 1))
-+
-+#define PIN_MIPI_TXM4			PINPOS('A', 2)
-+#define PIN_MIPIRX0N			PINPOS('A', 4)
-+#define PIN_MIPIRX3P			PINPOS('A', 6)
-+#define PIN_MIPIRX4P			PINPOS('A', 7)
-+#define PIN_VIVO_D2			PINPOS('A', 9)
-+#define PIN_VIVO_D3			PINPOS('A', 10)
-+#define PIN_VIVO_D10			PINPOS('A', 12)
-+#define PIN_USB_VBUS_DET		PINPOS('A', 13)
-+#define PIN_MIPI_TXP3			PINPOS('B', 1)
-+#define PIN_MIPI_TXM3			PINPOS('B', 2)
-+#define PIN_MIPI_TXP4			PINPOS('B', 3)
-+#define PIN_MIPIRX0P			PINPOS('B', 4)
-+#define PIN_MIPIRX1N			PINPOS('B', 5)
-+#define PIN_MIPIRX2N			PINPOS('B', 6)
-+#define PIN_MIPIRX4N			PINPOS('B', 7)
-+#define PIN_MIPIRX5N			PINPOS('B', 8)
-+#define PIN_VIVO_D1			PINPOS('B', 9)
-+#define PIN_VIVO_D5			PINPOS('B', 10)
-+#define PIN_VIVO_D7			PINPOS('B', 11)
-+#define PIN_VIVO_D9			PINPOS('B', 12)
-+#define PIN_USB_ID			PINPOS('B', 13)
-+#define PIN_ETH_RXM			PINPOS('B', 15)
-+#define PIN_MIPI_TXP2			PINPOS('C', 1)
-+#define PIN_MIPI_TXM2			PINPOS('C', 2)
-+#define PIN_CAM_PD0			PINPOS('C', 3)
-+#define PIN_CAM_MCLK0			PINPOS('C', 4)
-+#define PIN_MIPIRX1P			PINPOS('C', 5)
-+#define PIN_MIPIRX2P			PINPOS('C', 6)
-+#define PIN_MIPIRX3N			PINPOS('C', 7)
-+#define PIN_MIPIRX5P			PINPOS('C', 8)
-+#define PIN_VIVO_CLK			PINPOS('C', 9)
-+#define PIN_VIVO_D6			PINPOS('C', 10)
-+#define PIN_VIVO_D8			PINPOS('C', 11)
-+#define PIN_USB_VBUS_EN			PINPOS('C', 12)
-+#define PIN_ETH_RXP			PINPOS('C', 14)
-+#define PIN_GPIO_RTX			PINPOS('C', 15)
-+#define PIN_MIPI_TXP1			PINPOS('D', 1)
-+#define PIN_MIPI_TXM1			PINPOS('D', 2)
-+#define PIN_CAM_MCLK1			PINPOS('D', 3)
-+#define PIN_IIC3_SCL			PINPOS('D', 4)
-+#define PIN_VIVO_D4			PINPOS('D', 10)
-+#define PIN_ETH_TXM			PINPOS('D', 14)
-+#define PIN_ETH_TXP			PINPOS('D', 15)
-+#define PIN_MIPI_TXP0			PINPOS('E', 1)
-+#define PIN_MIPI_TXM0			PINPOS('E', 2)
-+#define PIN_CAM_PD1			PINPOS('E', 4)
-+#define PIN_CAM_RST0			PINPOS('E', 5)
-+#define PIN_VIVO_D0			PINPOS('E', 10)
-+#define PIN_ADC1			PINPOS('E', 13)
-+#define PIN_ADC2			PINPOS('E', 14)
-+#define PIN_ADC3			PINPOS('E', 15)
-+#define PIN_AUD_AOUTL			PINPOS('F', 2)
-+#define PIN_IIC3_SDA			PINPOS('F', 4)
-+#define PIN_SD1_D2			PINPOS('F', 14)
-+#define PIN_AUD_AOUTR			PINPOS('G', 2)
-+#define PIN_SD1_D3			PINPOS('G', 13)
-+#define PIN_SD1_CLK			PINPOS('G', 14)
-+#define PIN_SD1_CMD			PINPOS('G', 15)
-+#define PIN_AUD_AINL_MIC		PINPOS('H', 1)
-+#define PIN_RSTN			PINPOS('H', 12)
-+#define PIN_PWM0_BUCK			PINPOS('H', 13)
-+#define PIN_SD1_D1			PINPOS('H', 14)
-+#define PIN_SD1_D0			PINPOS('H', 15)
-+#define PIN_AUD_AINR_MIC		PINPOS('J', 1)
-+#define PIN_IIC2_SCL			PINPOS('J', 13)
-+#define PIN_IIC2_SDA			PINPOS('J', 14)
-+#define PIN_SD0_CD			PINPOS('K', 2)
-+#define PIN_SD0_D1			PINPOS('K', 3)
-+#define PIN_UART2_RX			PINPOS('K', 13)
-+#define PIN_UART2_CTS			PINPOS('K', 14)
-+#define PIN_UART2_TX			PINPOS('K', 15)
-+#define PIN_SD0_CLK			PINPOS('L', 1)
-+#define PIN_SD0_D0			PINPOS('L', 2)
-+#define PIN_SD0_CMD			PINPOS('L', 3)
-+#define PIN_CLK32K			PINPOS('L', 14)
-+#define PIN_UART2_RTS			PINPOS('L', 15)
-+#define PIN_SD0_D3			PINPOS('M', 1)
-+#define PIN_SD0_D2			PINPOS('M', 2)
-+#define PIN_UART0_RX			PINPOS('M', 4)
-+#define PIN_UART0_TX			PINPOS('M', 5)
-+#define PIN_JTAG_CPU_TRST		PINPOS('M', 6)
-+#define PIN_PWR_ON			PINPOS('M', 11)
-+#define PIN_PWR_GPIO2			PINPOS('M', 12)
-+#define PIN_PWR_GPIO0			PINPOS('M', 13)
-+#define PIN_CLK25M			PINPOS('M', 14)
-+#define PIN_SD0_PWR_EN			PINPOS('N', 1)
-+#define PIN_SPK_EN			PINPOS('N', 3)
-+#define PIN_JTAG_CPU_TCK		PINPOS('N', 4)
-+#define PIN_JTAG_CPU_TMS		PINPOS('N', 6)
-+#define PIN_PWR_WAKEUP1			PINPOS('N', 11)
-+#define PIN_PWR_WAKEUP0			PINPOS('N', 12)
-+#define PIN_PWR_GPIO1			PINPOS('N', 13)
-+#define PIN_EMMC_DAT3			PINPOS('P', 1)
-+#define PIN_EMMC_DAT0			PINPOS('P', 2)
-+#define PIN_EMMC_DAT2			PINPOS('P', 3)
-+#define PIN_EMMC_RSTN			PINPOS('P', 4)
-+#define PIN_AUX0			PINPOS('P', 5)
-+#define PIN_IIC0_SDA			PINPOS('P', 6)
-+#define PIN_PWR_SEQ3			PINPOS('P', 10)
-+#define PIN_PWR_VBAT_DET		PINPOS('P', 11)
-+#define PIN_PWR_SEQ1			PINPOS('P', 12)
-+#define PIN_PWR_BUTTON1			PINPOS('P', 13)
-+#define PIN_EMMC_DAT1			PINPOS('R', 2)
-+#define PIN_EMMC_CMD			PINPOS('R', 3)
-+#define PIN_EMMC_CLK			PINPOS('R', 4)
-+#define PIN_IIC0_SCL			PINPOS('R', 6)
-+#define PIN_GPIO_ZQ			PINPOS('R', 10)
-+#define PIN_PWR_RSTN			PINPOS('R', 11)
-+#define PIN_PWR_SEQ2			PINPOS('R', 12)
-+#define PIN_XTAL_XIN			PINPOS('R', 13)
-+
-+#endif /* _DT_BINDINGS_PINCTRL_CV1812H_H */
-diff --git a/include/dt-bindings/pinctrl/pinctrl-cv18xx.h b/include/dt-bindings/pinctrl/pinctrl-cv18xx.h
-new file mode 100644
-index 000000000000..bc92ad1067ec
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/pinctrl-cv18xx.h
-@@ -0,0 +1,19 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2023 Sophgo Ltd.
-+ *
-+ * Author: Inochi Amaoto <inochiama@outlook.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_PINCTRL_CV18XX_H
-+#define _DT_BINDINGS_PINCTRL_CV18XX_H
-+
-+#define PIN_MUX_INVALD				0xff
-+
-+#define PINMUX2(pin, mux, mux2)	\
-+	(((pin) & 0xffff) | (((mux) & 0xff) << 16) | (((mux2) & 0xff) << 24))
-+
-+#define PINMUX(pin, mux) \
-+	PINMUX2(pin, mux, PIN_MUX_INVALD)
-+
-+#endif /* _DT_BINDINGS_PINCTRL_CV18XX_H */
-diff --git a/include/dt-bindings/pinctrl/pinctrl-sg2000.h b/include/dt-bindings/pinctrl/pinctrl-sg2000.h
-new file mode 100644
-index 000000000000..4871f9a7c6c1
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/pinctrl-sg2000.h
-@@ -0,0 +1,127 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2024 Inochi Amaoto <inochiama@outlook.com>
-+ *
-+ * This file is generated from vendor pinout definition.
-+ */
-+
-+#ifndef _DT_BINDINGS_PINCTRL_SG2000_H
-+#define _DT_BINDINGS_PINCTRL_SG2000_H
-+
-+#include <dt-bindings/pinctrl/pinctrl-cv18xx.h>
-+
-+#define PINPOS(row, col)			\
-+	((((row) - 'A' + 1) << 8) + ((col) - 1))
-+
-+#define PIN_MIPI_TXM4			PINPOS('A', 2)
-+#define PIN_MIPIRX0N			PINPOS('A', 4)
-+#define PIN_MIPIRX3P			PINPOS('A', 6)
-+#define PIN_MIPIRX4P			PINPOS('A', 7)
-+#define PIN_VIVO_D2			PINPOS('A', 9)
-+#define PIN_VIVO_D3			PINPOS('A', 10)
-+#define PIN_VIVO_D10			PINPOS('A', 12)
-+#define PIN_USB_VBUS_DET		PINPOS('A', 13)
-+#define PIN_MIPI_TXP3			PINPOS('B', 1)
-+#define PIN_MIPI_TXM3			PINPOS('B', 2)
-+#define PIN_MIPI_TXP4			PINPOS('B', 3)
-+#define PIN_MIPIRX0P			PINPOS('B', 4)
-+#define PIN_MIPIRX1N			PINPOS('B', 5)
-+#define PIN_MIPIRX2N			PINPOS('B', 6)
-+#define PIN_MIPIRX4N			PINPOS('B', 7)
-+#define PIN_MIPIRX5N			PINPOS('B', 8)
-+#define PIN_VIVO_D1			PINPOS('B', 9)
-+#define PIN_VIVO_D5			PINPOS('B', 10)
-+#define PIN_VIVO_D7			PINPOS('B', 11)
-+#define PIN_VIVO_D9			PINPOS('B', 12)
-+#define PIN_USB_ID			PINPOS('B', 13)
-+#define PIN_ETH_RXM			PINPOS('B', 15)
-+#define PIN_MIPI_TXP2			PINPOS('C', 1)
-+#define PIN_MIPI_TXM2			PINPOS('C', 2)
-+#define PIN_CAM_PD0			PINPOS('C', 3)
-+#define PIN_CAM_MCLK0			PINPOS('C', 4)
-+#define PIN_MIPIRX1P			PINPOS('C', 5)
-+#define PIN_MIPIRX2P			PINPOS('C', 6)
-+#define PIN_MIPIRX3N			PINPOS('C', 7)
-+#define PIN_MIPIRX5P			PINPOS('C', 8)
-+#define PIN_VIVO_CLK			PINPOS('C', 9)
-+#define PIN_VIVO_D6			PINPOS('C', 10)
-+#define PIN_VIVO_D8			PINPOS('C', 11)
-+#define PIN_USB_VBUS_EN			PINPOS('C', 12)
-+#define PIN_ETH_RXP			PINPOS('C', 14)
-+#define PIN_GPIO_RTX			PINPOS('C', 15)
-+#define PIN_MIPI_TXP1			PINPOS('D', 1)
-+#define PIN_MIPI_TXM1			PINPOS('D', 2)
-+#define PIN_CAM_MCLK1			PINPOS('D', 3)
-+#define PIN_IIC3_SCL			PINPOS('D', 4)
-+#define PIN_VIVO_D4			PINPOS('D', 10)
-+#define PIN_ETH_TXM			PINPOS('D', 14)
-+#define PIN_ETH_TXP			PINPOS('D', 15)
-+#define PIN_MIPI_TXP0			PINPOS('E', 1)
-+#define PIN_MIPI_TXM0			PINPOS('E', 2)
-+#define PIN_CAM_PD1			PINPOS('E', 4)
-+#define PIN_CAM_RST0			PINPOS('E', 5)
-+#define PIN_VIVO_D0			PINPOS('E', 10)
-+#define PIN_ADC1			PINPOS('E', 13)
-+#define PIN_ADC2			PINPOS('E', 14)
-+#define PIN_ADC3			PINPOS('E', 15)
-+#define PIN_AUD_AOUTL			PINPOS('F', 2)
-+#define PIN_IIC3_SDA			PINPOS('F', 4)
-+#define PIN_SD1_D2			PINPOS('F', 14)
-+#define PIN_AUD_AOUTR			PINPOS('G', 2)
-+#define PIN_SD1_D3			PINPOS('G', 13)
-+#define PIN_SD1_CLK			PINPOS('G', 14)
-+#define PIN_SD1_CMD			PINPOS('G', 15)
-+#define PIN_AUD_AINL_MIC		PINPOS('H', 1)
-+#define PIN_RSTN			PINPOS('H', 12)
-+#define PIN_PWM0_BUCK			PINPOS('H', 13)
-+#define PIN_SD1_D1			PINPOS('H', 14)
-+#define PIN_SD1_D0			PINPOS('H', 15)
-+#define PIN_AUD_AINR_MIC		PINPOS('J', 1)
-+#define PIN_IIC2_SCL			PINPOS('J', 13)
-+#define PIN_IIC2_SDA			PINPOS('J', 14)
-+#define PIN_SD0_CD			PINPOS('K', 2)
-+#define PIN_SD0_D1			PINPOS('K', 3)
-+#define PIN_UART2_RX			PINPOS('K', 13)
-+#define PIN_UART2_CTS			PINPOS('K', 14)
-+#define PIN_UART2_TX			PINPOS('K', 15)
-+#define PIN_SD0_CLK			PINPOS('L', 1)
-+#define PIN_SD0_D0			PINPOS('L', 2)
-+#define PIN_SD0_CMD			PINPOS('L', 3)
-+#define PIN_CLK32K			PINPOS('L', 14)
-+#define PIN_UART2_RTS			PINPOS('L', 15)
-+#define PIN_SD0_D3			PINPOS('M', 1)
-+#define PIN_SD0_D2			PINPOS('M', 2)
-+#define PIN_UART0_RX			PINPOS('M', 4)
-+#define PIN_UART0_TX			PINPOS('M', 5)
-+#define PIN_JTAG_CPU_TRST		PINPOS('M', 6)
-+#define PIN_PWR_ON			PINPOS('M', 11)
-+#define PIN_PWR_GPIO2			PINPOS('M', 12)
-+#define PIN_PWR_GPIO0			PINPOS('M', 13)
-+#define PIN_CLK25M			PINPOS('M', 14)
-+#define PIN_SD0_PWR_EN			PINPOS('N', 1)
-+#define PIN_SPK_EN			PINPOS('N', 3)
-+#define PIN_JTAG_CPU_TCK		PINPOS('N', 4)
-+#define PIN_JTAG_CPU_TMS		PINPOS('N', 6)
-+#define PIN_PWR_WAKEUP1			PINPOS('N', 11)
-+#define PIN_PWR_WAKEUP0			PINPOS('N', 12)
-+#define PIN_PWR_GPIO1			PINPOS('N', 13)
-+#define PIN_EMMC_DAT3			PINPOS('P', 1)
-+#define PIN_EMMC_DAT0			PINPOS('P', 2)
-+#define PIN_EMMC_DAT2			PINPOS('P', 3)
-+#define PIN_EMMC_RSTN			PINPOS('P', 4)
-+#define PIN_AUX0			PINPOS('P', 5)
-+#define PIN_IIC0_SDA			PINPOS('P', 6)
-+#define PIN_PWR_SEQ3			PINPOS('P', 10)
-+#define PIN_PWR_VBAT_DET		PINPOS('P', 11)
-+#define PIN_PWR_SEQ1			PINPOS('P', 12)
-+#define PIN_PWR_BUTTON1			PINPOS('P', 13)
-+#define PIN_EMMC_DAT1			PINPOS('R', 2)
-+#define PIN_EMMC_CMD			PINPOS('R', 3)
-+#define PIN_EMMC_CLK			PINPOS('R', 4)
-+#define PIN_IIC0_SCL			PINPOS('R', 6)
-+#define PIN_GPIO_ZQ			PINPOS('R', 10)
-+#define PIN_PWR_RSTN			PINPOS('R', 11)
-+#define PIN_PWR_SEQ2			PINPOS('R', 12)
-+#define PIN_XTAL_XIN			PINPOS('R', 13)
-+
-+#endif /* _DT_BINDINGS_PINCTRL_SG2000_H */
-diff --git a/include/dt-bindings/pinctrl/pinctrl-sg2002.h b/include/dt-bindings/pinctrl/pinctrl-sg2002.h
-new file mode 100644
-index 000000000000..3c36cfa0a550
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/pinctrl-sg2002.h
-@@ -0,0 +1,79 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2024 Inochi Amaoto <inochiama@outlook.com>
-+ *
-+ * This file is generated from vendor pinout definition.
-+ */
-+
-+#ifndef _DT_BINDINGS_PINCTRL_SG2002_H
-+#define _DT_BINDINGS_PINCTRL_SG2002_H
-+
-+#include <dt-bindings/pinctrl/pinctrl-cv18xx.h>
-+
-+#define PIN_AUD_AINL_MIC		2
-+#define PIN_AUD_AOUTR			4
-+#define PIN_SD0_CLK			6
-+#define PIN_SD0_CMD			7
-+#define PIN_SD0_D0			8
-+#define PIN_SD0_D1			10
-+#define PIN_SD0_D2			11
-+#define PIN_SD0_D3			12
-+#define PIN_SD0_CD			14
-+#define PIN_SD0_PWR_EN			15
-+#define PIN_SPK_EN			17
-+#define PIN_UART0_TX			18
-+#define PIN_UART0_RX			19
-+#define PIN_EMMC_DAT2			20
-+#define PIN_EMMC_CLK			21
-+#define PIN_EMMC_DAT0			22
-+#define PIN_EMMC_DAT3			23
-+#define PIN_EMMC_CMD			24
-+#define PIN_EMMC_DAT1			25
-+#define PIN_JTAG_CPU_TMS		26
-+#define PIN_JTAG_CPU_TCK		27
-+#define PIN_IIC0_SCL			28
-+#define PIN_IIC0_SDA			29
-+#define PIN_AUX0			30
-+#define PIN_GPIO_ZQ			35
-+#define PIN_PWR_VBAT_DET		38
-+#define PIN_PWR_RSTN			39
-+#define PIN_PWR_SEQ1			40
-+#define PIN_PWR_SEQ2			41
-+#define PIN_PWR_WAKEUP0			43
-+#define PIN_PWR_BUTTON1			44
-+#define PIN_XTAL_XIN			45
-+#define PIN_PWR_GPIO0			47
-+#define PIN_PWR_GPIO1			48
-+#define PIN_PWR_GPIO2			49
-+#define PIN_SD1_D3			51
-+#define PIN_SD1_D2			52
-+#define PIN_SD1_D1			53
-+#define PIN_SD1_D0			54
-+#define PIN_SD1_CMD			55
-+#define PIN_SD1_CLK			56
-+#define PIN_PWM0_BUCK			58
-+#define PIN_ADC1			59
-+#define PIN_USB_VBUS_DET		60
-+#define PIN_ETH_TXP			62
-+#define PIN_ETH_TXM			63
-+#define PIN_ETH_RXP			64
-+#define PIN_ETH_RXM			65
-+#define PIN_GPIO_RTX			67
-+#define PIN_MIPIRX4N			72
-+#define PIN_MIPIRX4P			73
-+#define PIN_MIPIRX3N			74
-+#define PIN_MIPIRX3P			75
-+#define PIN_MIPIRX2N			76
-+#define PIN_MIPIRX2P			77
-+#define PIN_MIPIRX1N			78
-+#define PIN_MIPIRX1P			79
-+#define PIN_MIPIRX0N			80
-+#define PIN_MIPIRX0P			81
-+#define PIN_MIPI_TXM2			83
-+#define PIN_MIPI_TXP2			84
-+#define PIN_MIPI_TXM1			85
-+#define PIN_MIPI_TXP1			86
-+#define PIN_MIPI_TXM0			87
-+#define PIN_MIPI_TXP0			88
-+
-+#endif /* _DT_BINDINGS_PINCTRL_SG2002_H */
-diff --git a/include/dt-bindings/pinctrl/pinctrl-sg2042.h b/include/dt-bindings/pinctrl/pinctrl-sg2042.h
-new file mode 100644
-index 000000000000..79d5bb8e04f8
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/pinctrl-sg2042.h
-@@ -0,0 +1,196 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2024 Inochi Amaoto <inochiama@outlook.com>
-+ *
-+ */
-+
-+#ifndef _DT_BINDINGS_PINCTRL_SG2042_H
-+#define _DT_BINDINGS_PINCTRL_SG2042_H
-+
-+#define PINMUX(pin, mux) \
-+	(((pin) & 0xffff) | (((mux) & 0xff) << 16))
-+
-+#define PIN_LPC_LCLK			0
-+#define PIN_LPC_LFRAME			1
-+#define PIN_LPC_LAD0			2
-+#define PIN_LPC_LAD1			3
-+#define PIN_LPC_LAD2			4
-+#define PIN_LPC_LAD3			5
-+#define PIN_LPC_LDRQ0			6
-+#define PIN_LPC_LDRQ1			7
-+#define PIN_LPC_SERIRQ			8
-+#define PIN_LPC_CLKRUN			9
-+#define PIN_LPC_LPME			10
-+#define PIN_LPC_LPCPD			11
-+#define PIN_LPC_LSMI			12
-+#define PIN_PCIE0_L0_RESET		13
-+#define PIN_PCIE0_L1_RESET		14
-+#define PIN_PCIE0_L0_WAKEUP		15
-+#define PIN_PCIE0_L1_WAKEUP		16
-+#define PIN_PCIE0_L0_CLKREQ_IN		17
-+#define PIN_PCIE0_L1_CLKREQ_IN		18
-+#define PIN_PCIE1_L0_RESET		19
-+#define PIN_PCIE1_L1_RESET		20
-+#define PIN_PCIE1_L0_WAKEUP		21
-+#define PIN_PCIE1_L1_WAKEUP		22
-+#define PIN_PCIE1_L0_CLKREQ_IN		23
-+#define PIN_PCIE1_L1_CLKREQ_IN		24
-+#define PIN_SPIF0_CLK_SEL1		25
-+#define PIN_SPIF0_CLK_SEL0		26
-+#define PIN_SPIF0_WP			27
-+#define PIN_SPIF0_HOLD			28
-+#define PIN_SPIF0_SDI			29
-+#define PIN_SPIF0_CS			30
-+#define PIN_SPIF0_SCK			31
-+#define PIN_SPIF0_SDO			32
-+#define PIN_SPIF1_CLK_SEL1		33
-+#define PIN_SPIF1_CLK_SEL0		34
-+#define PIN_SPIF1_WP			35
-+#define PIN_SPIF1_HOLD			36
-+#define PIN_SPIF1_SDI			37
-+#define PIN_SPIF1_CS			38
-+#define PIN_SPIF1_SCK			39
-+#define PIN_SPIF1_SDO			40
-+#define PIN_EMMC_WP			41
-+#define PIN_EMMC_CD			42
-+#define PIN_EMMC_RST			43
-+#define PIN_EMMC_PWR_EN			44
-+#define PIN_SDIO_CD			45
-+#define PIN_SDIO_WP			46
-+#define PIN_SDIO_RST			47
-+#define PIN_SDIO_PWR_EN			48
-+#define PIN_RGMII0_TXD0			49
-+#define PIN_RGMII0_TXD1			50
-+#define PIN_RGMII0_TXD2			51
-+#define PIN_RGMII0_TXD3			52
-+#define PIN_RGMII0_TXCTRL		53
-+#define PIN_RGMII0_RXD0			54
-+#define PIN_RGMII0_RXD1			55
-+#define PIN_RGMII0_RXD2			56
-+#define PIN_RGMII0_RXD3			57
-+#define PIN_RGMII0_RXCTRL		58
-+#define PIN_RGMII0_TXC			59
-+#define PIN_RGMII0_RXC			60
-+#define PIN_RGMII0_REFCLKO		61
-+#define PIN_RGMII0_IRQ			62
-+#define PIN_RGMII0_MDC			63
-+#define PIN_RGMII0_MDIO			64
-+#define PIN_PWM0			65
-+#define PIN_PWM1			66
-+#define PIN_PWM2			67
-+#define PIN_PWM3			68
-+#define PIN_FAN0			69
-+#define PIN_FAN1			70
-+#define PIN_FAN2			71
-+#define PIN_FAN3			72
-+#define PIN_IIC0_SDA			73
-+#define PIN_IIC0_SCL			74
-+#define PIN_IIC1_SDA			75
-+#define PIN_IIC1_SCL			76
-+#define PIN_IIC2_SDA			77
-+#define PIN_IIC2_SCL			78
-+#define PIN_IIC3_SDA			79
-+#define PIN_IIC3_SCL			80
-+#define PIN_UART0_TX			81
-+#define PIN_UART0_RX			82
-+#define PIN_UART0_RTS			83
-+#define PIN_UART0_CTS			84
-+#define PIN_UART1_TX			85
-+#define PIN_UART1_RX			86
-+#define PIN_UART1_RTS			87
-+#define PIN_UART1_CTS			88
-+#define PIN_UART2_TX			89
-+#define PIN_UART2_RX			90
-+#define PIN_UART2_RTS			91
-+#define PIN_UART2_CTS			92
-+#define PIN_UART3_TX			93
-+#define PIN_UART3_RX			94
-+#define PIN_UART3_RTS			95
-+#define PIN_UART3_CTS			96
-+#define PIN_SPI0_CS0			97
-+#define PIN_SPI0_CS1			98
-+#define PIN_SPI0_SDI			99
-+#define PIN_SPI0_SDO			100
-+#define PIN_SPI0_SCK			101
-+#define PIN_SPI1_CS0			102
-+#define PIN_SPI1_CS1			103
-+#define PIN_SPI1_SDI			104
-+#define PIN_SPI1_SDO			105
-+#define PIN_SPI1_SCK			106
-+#define PIN_JTAG0_TDO			107
-+#define PIN_JTAG0_TCK			108
-+#define PIN_JTAG0_TDI			109
-+#define PIN_JTAG0_TMS			110
-+#define PIN_JTAG0_TRST			111
-+#define PIN_JTAG0_SRST			112
-+#define PIN_JTAG1_TDO			113
-+#define PIN_JTAG1_TCK			114
-+#define PIN_JTAG1_TDI			115
-+#define PIN_JTAG1_TMS			116
-+#define PIN_JTAG1_TRST			117
-+#define PIN_JTAG1_SRST			118
-+#define PIN_JTAG2_TDO			119
-+#define PIN_JTAG2_TCK			120
-+#define PIN_JTAG2_TDI			121
-+#define PIN_JTAG2_TMS			122
-+#define PIN_JTAG2_TRST			123
-+#define PIN_JTAG2_SRST			124
-+#define PIN_GPIO0			125
-+#define PIN_GPIO1			126
-+#define PIN_GPIO2			127
-+#define PIN_GPIO3			128
-+#define PIN_GPIO4			129
-+#define PIN_GPIO5			130
-+#define PIN_GPIO6			131
-+#define PIN_GPIO7			132
-+#define PIN_GPIO8			133
-+#define PIN_GPIO9			134
-+#define PIN_GPIO10			135
-+#define PIN_GPIO11			136
-+#define PIN_GPIO12			137
-+#define PIN_GPIO13			138
-+#define PIN_GPIO14			139
-+#define PIN_GPIO15			140
-+#define PIN_GPIO16			141
-+#define PIN_GPIO17			142
-+#define PIN_GPIO18			143
-+#define PIN_GPIO19			144
-+#define PIN_GPIO20			145
-+#define PIN_GPIO21			146
-+#define PIN_GPIO22			147
-+#define PIN_GPIO23			148
-+#define PIN_GPIO24			149
-+#define PIN_GPIO25			150
-+#define PIN_GPIO26			151
-+#define PIN_GPIO27			152
-+#define PIN_GPIO28			153
-+#define PIN_GPIO29			154
-+#define PIN_GPIO30			155
-+#define PIN_GPIO31			156
-+#define PIN_MODE_SEL0			157
-+#define PIN_MODE_SEL1			158
-+#define PIN_MODE_SEL2			159
-+#define PIN_BOOT_SEL0			160
-+#define PIN_BOOT_SEL1			161
-+#define PIN_BOOT_SEL2			162
-+#define PIN_BOOT_SEL3			163
-+#define PIN_BOOT_SEL4			164
-+#define PIN_BOOT_SEL5			165
-+#define PIN_BOOT_SEL6			166
-+#define PIN_BOOT_SEL7			167
-+#define PIN_MULTI_SCKT			168
-+#define PIN_SCKT_ID0			169
-+#define PIN_SCKT_ID1			170
-+#define PIN_PLL_CLK_IN_MAIN		171
-+#define PIN_PLL_CLK_IN_DDR_L		172
-+#define PIN_PLL_CLK_IN_DDR_R		173
-+#define PIN_XTAL_32K			174
-+#define PIN_SYS_RST			175
-+#define PIN_PWR_BUTTON			176
-+#define PIN_TEST_EN			177
-+#define PIN_TEST_MODE_MBIST		178
-+#define PIN_TEST_MODE_SCAN		179
-+#define PIN_TEST_MODE_BSD		180
-+#define PIN_BISR_BYP			181
-+
-+#endif /* _DT_BINDINGS_PINCTRL_SG2042_H */
-diff --git a/include/dt-bindings/pinctrl/pinctrl-sg2044.h b/include/dt-bindings/pinctrl/pinctrl-sg2044.h
-new file mode 100644
-index 000000000000..2a619f681c39
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/pinctrl-sg2044.h
-@@ -0,0 +1,221 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2024 Inochi Amaoto <inochiama@outlook.com>
-+ *
-+ */
-+
-+#ifndef _DT_BINDINGS_PINCTRL_SG2044_H
-+#define _DT_BINDINGS_PINCTRL_SG2044_H
-+
-+#define PINMUX(pin, mux) \
-+	(((pin) & 0xffff) | (((mux) & 0xff) << 16))
-+
-+#define PIN_IIC0_SMBSUS_IN		0
-+#define PIN_IIC0_SMBSUS_OUT		1
-+#define PIN_IIC0_SMBALERT		2
-+#define PIN_IIC1_SMBSUS_IN		3
-+#define PIN_IIC1_SMBSUS_OUT		4
-+#define PIN_IIC1_SMBALERT		5
-+#define PIN_IIC2_SMBSUS_IN		6
-+#define PIN_IIC2_SMBSUS_OUT		7
-+#define PIN_IIC2_SMBALERT		8
-+#define PIN_IIC3_SMBSUS_IN		9
-+#define PIN_IIC3_SMBSUS_OUT		10
-+#define PIN_IIC3_SMBALERT		11
-+#define PIN_PCIE0_L0_RESET		12
-+#define PIN_PCIE0_L1_RESET		13
-+#define PIN_PCIE0_L0_WAKEUP		14
-+#define PIN_PCIE0_L1_WAKEUP		15
-+#define PIN_PCIE0_L0_CLKREQ_IN		16
-+#define PIN_PCIE0_L1_CLKREQ_IN		17
-+#define PIN_PCIE1_L0_RESET		18
-+#define PIN_PCIE1_L1_RESET		19
-+#define PIN_PCIE1_L0_WAKEUP		20
-+#define PIN_PCIE1_L1_WAKEUP		21
-+#define PIN_PCIE1_L0_CLKREQ_IN		22
-+#define PIN_PCIE1_L1_CLKREQ_IN		23
-+#define PIN_PCIE2_L0_RESET		24
-+#define PIN_PCIE2_L1_RESET		25
-+#define PIN_PCIE2_L0_WAKEUP		26
-+#define PIN_PCIE2_L1_WAKEUP		27
-+#define PIN_PCIE2_L0_CLKREQ_IN		28
-+#define PIN_PCIE2_L1_CLKREQ_IN		29
-+#define PIN_PCIE3_L0_RESET		30
-+#define PIN_PCIE3_L1_RESET		31
-+#define PIN_PCIE3_L0_WAKEUP		32
-+#define PIN_PCIE3_L1_WAKEUP		33
-+#define PIN_PCIE3_L0_CLKREQ_IN		34
-+#define PIN_PCIE3_L1_CLKREQ_IN		35
-+#define PIN_PCIE4_L0_RESET		36
-+#define PIN_PCIE4_L1_RESET		37
-+#define PIN_PCIE4_L0_WAKEUP		38
-+#define PIN_PCIE4_L1_WAKEUP		39
-+#define PIN_PCIE4_L0_CLKREQ_IN		40
-+#define PIN_PCIE4_L1_CLKREQ_IN		41
-+#define PIN_SPIF0_CLK_SEL1		42
-+#define PIN_SPIF0_CLK_SEL0		43
-+#define PIN_SPIF0_WP			44
-+#define PIN_SPIF0_HOLD			45
-+#define PIN_SPIF0_SDI			46
-+#define PIN_SPIF0_CS			47
-+#define PIN_SPIF0_SCK			48
-+#define PIN_SPIF0_SDO			49
-+#define PIN_SPIF1_CLK_SEL1		50
-+#define PIN_SPIF1_CLK_SEL0		51
-+#define PIN_SPIF1_WP			52
-+#define PIN_SPIF1_HOLD			53
-+#define PIN_SPIF1_SDI			54
-+#define PIN_SPIF1_CS			55
-+#define PIN_SPIF1_SCK			56
-+#define PIN_SPIF1_SDO			57
-+#define PIN_EMMC_WP			58
-+#define PIN_EMMC_CD			59
-+#define PIN_EMMC_RST			60
-+#define PIN_EMMC_PWR_EN			61
-+#define PIN_SDIO_CD			62
-+#define PIN_SDIO_WP			63
-+#define PIN_SDIO_RST			64
-+#define PIN_SDIO_PWR_EN			65
-+#define PIN_RGMII0_TXD0			66
-+#define PIN_RGMII0_TXD1			67
-+#define PIN_RGMII0_TXD2			68
-+#define PIN_RGMII0_TXD3			69
-+#define PIN_RGMII0_TXCTRL		70
-+#define PIN_RGMII0_RXD0			71
-+#define PIN_RGMII0_RXD1			72
-+#define PIN_RGMII0_RXD2			73
-+#define PIN_RGMII0_RXD3			74
-+#define PIN_RGMII0_RXCTRL		75
-+#define PIN_RGMII0_TXC			76
-+#define PIN_RGMII0_RXC			77
-+#define PIN_RGMII0_REFCLKO		78
-+#define PIN_RGMII0_IRQ			79
-+#define PIN_RGMII0_MDC			80
-+#define PIN_RGMII0_MDIO			81
-+#define PIN_PWM0			82
-+#define PIN_PWM1			83
-+#define PIN_PWM2			84
-+#define PIN_PWM3			85
-+#define PIN_FAN0			86
-+#define PIN_FAN1			87
-+#define PIN_FAN2			88
-+#define PIN_FAN3			89
-+#define PIN_IIC0_SDA			90
-+#define PIN_IIC0_SCL			91
-+#define PIN_IIC1_SDA			92
-+#define PIN_IIC1_SCL			93
-+#define PIN_IIC2_SDA			94
-+#define PIN_IIC2_SCL			95
-+#define PIN_IIC3_SDA			96
-+#define PIN_IIC3_SCL			97
-+#define PIN_UART0_TX			98
-+#define PIN_UART0_RX			99
-+#define PIN_UART0_RTS			100
-+#define PIN_UART0_CTS			101
-+#define PIN_UART1_TX			102
-+#define PIN_UART1_RX			103
-+#define PIN_UART1_RTS			104
-+#define PIN_UART1_CTS			105
-+#define PIN_UART2_TX			106
-+#define PIN_UART2_RX			107
-+#define PIN_UART2_RTS			108
-+#define PIN_UART2_CTS			109
-+#define PIN_UART3_TX			110
-+#define PIN_UART3_RX			111
-+#define PIN_UART3_RTS			112
-+#define PIN_UART3_CTS			113
-+#define PIN_SPI0_CS0			114
-+#define PIN_SPI0_CS1			115
-+#define PIN_SPI0_SDI			116
-+#define PIN_SPI0_SDO			117
-+#define PIN_SPI0_SCK			118
-+#define PIN_SPI1_CS0			119
-+#define PIN_SPI1_CS1			120
-+#define PIN_SPI1_SDI			121
-+#define PIN_SPI1_SDO			122
-+#define PIN_SPI1_SCK			123
-+#define PIN_JTAG0_TDO			124
-+#define PIN_JTAG0_TCK			125
-+#define PIN_JTAG0_TDI			126
-+#define PIN_JTAG0_TMS			127
-+#define PIN_JTAG0_TRST			128
-+#define PIN_JTAG0_SRST			129
-+#define PIN_JTAG1_TDO			130
-+#define PIN_JTAG1_TCK			131
-+#define PIN_JTAG1_TDI			132
-+#define PIN_JTAG1_TMS			133
-+#define PIN_JTAG1_TRST			134
-+#define PIN_JTAG1_SRST			135
-+#define PIN_JTAG2_TDO			136
-+#define PIN_JTAG2_TCK			137
-+#define PIN_JTAG2_TDI			138
-+#define PIN_JTAG2_TMS			139
-+#define PIN_JTAG2_TRST			140
-+#define PIN_JTAG2_SRST			141
-+#define PIN_JTAG3_TDO			142
-+#define PIN_JTAG3_TCK			143
-+#define PIN_JTAG3_TDI			144
-+#define PIN_JTAG3_TMS			145
-+#define PIN_JTAG3_TRST			146
-+#define PIN_JTAG3_SRST			147
-+#define PIN_GPIO0			148
-+#define PIN_GPIO1			149
-+#define PIN_GPIO2			150
-+#define PIN_GPIO3			151
-+#define PIN_GPIO4			152
-+#define PIN_GPIO5			153
-+#define PIN_GPIO6			154
-+#define PIN_GPIO7			155
-+#define PIN_GPIO8			156
-+#define PIN_GPIO9			157
-+#define PIN_GPIO10			158
-+#define PIN_GPIO11			159
-+#define PIN_GPIO12			160
-+#define PIN_GPIO13			161
-+#define PIN_GPIO14			162
-+#define PIN_GPIO15			163
-+#define PIN_GPIO16			164
-+#define PIN_GPIO17			165
-+#define PIN_GPIO18			166
-+#define PIN_GPIO19			167
-+#define PIN_GPIO20			168
-+#define PIN_GPIO21			169
-+#define PIN_GPIO22			170
-+#define PIN_GPIO23			171
-+#define PIN_GPIO24			172
-+#define PIN_GPIO25			173
-+#define PIN_GPIO26			174
-+#define PIN_GPIO27			175
-+#define PIN_GPIO28			176
-+#define PIN_GPIO29			177
-+#define PIN_GPIO30			178
-+#define PIN_GPIO31			179
-+#define PIN_MODE_SEL0			180
-+#define PIN_MODE_SEL1			181
-+#define PIN_MODE_SEL2			182
-+#define PIN_BOOT_SEL0			183
-+#define PIN_BOOT_SEL1			184
-+#define PIN_BOOT_SEL2			185
-+#define PIN_BOOT_SEL3			186
-+#define PIN_BOOT_SEL4			187
-+#define PIN_BOOT_SEL5			188
-+#define PIN_BOOT_SEL6			189
-+#define PIN_BOOT_SEL7			190
-+#define PIN_MULTI_SCKT			191
-+#define PIN_SCKT_ID0			192
-+#define PIN_SCKT_ID1			193
-+#define PIN_PLL_CLK_IN_MAIN		194
-+#define PIN_PLL_CLK_IN_DDR_0		195
-+#define PIN_PLL_CLK_IN_DDR_1		196
-+#define PIN_PLL_CLK_IN_DDR_2		197
-+#define PIN_PLL_CLK_IN_DDR_3		198
-+#define PIN_XTAL_32K			199
-+#define PIN_SYS_RST			200
-+#define PIN_PWR_BUTTON			201
-+#define PIN_TEST_EN			202
-+#define PIN_TEST_MODE_MBIST		203
-+#define PIN_TEST_MODE_SCAN		204
-+#define PIN_TEST_MODE_BSD		205
-+#define PIN_BISR_BYP			206
-+
-+#endif /* _DT_BINDINGS_PINCTRL_SG2044_H */
-diff --git a/include/dt-bindings/pinctrl/pinctrl-starfive.h b/include/dt-bindings/pinctrl/pinctrl-starfive-jh7100.h
-index de4f75c2c9e8..a200f546d078 100644
--- a/include/dt-bindings/pinctrl/pinctrl-starfive.h
-+++ b/include/dt-bindings/pinctrl/pinctrl-starfive-jh7100.h
-@@ -3,8 +3,8 @@
-  * Copyright (C) 2021 Emil Renner Berthing <kernel@esmil.dk>
-  */
- 
-#ifndef __DT_BINDINGS_PINCTRL_STARFIVE_H__
-#define __DT_BINDINGS_PINCTRL_STARFIVE_H__
-+#ifndef __DT_BINDINGS_PINCTRL_STARFIVE_JH7100_H__
-+#define __DT_BINDINGS_PINCTRL_STARFIVE_JH7100_H__
- 
- #define PAD_GPIO_OFFSET		0
- #define PAD_FUNC_SHARE_OFFSET	64
-@@ -272,4 +272,4 @@
- 
- #define GPI_NONE				0xff
- 
-#endif /* __DT_BINDINGS_PINCTRL_STARFIVE_H__ */
-+#endif /* __DT_BINDINGS_PINCTRL_STARFIVE_JH7100_H__ */
-diff --git a/include/dt-bindings/pinctrl/renesas,r9a09g047-pinctrl.h b/include/dt-bindings/pinctrl/renesas,r9a09g047-pinctrl.h
-new file mode 100644
-index 000000000000..5917096720bd
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/renesas,r9a09g047-pinctrl.h
-@@ -0,0 +1,41 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * This header provides constants for Renesas RZ/G3E family pinctrl bindings.
-+ *
-+ * Copyright (C) 2024 Renesas Electronics Corp.
-+ *
-+ */
-+
-+#ifndef __DT_BINDINGS_PINCTRL_RENESAS_R9A09G047_PINCTRL_H__
-+#define __DT_BINDINGS_PINCTRL_RENESAS_R9A09G047_PINCTRL_H__
-+
-+#include <dt-bindings/pinctrl/rzg2l-pinctrl.h>
-+
-+/* RZG3E_Px = Offset address of PFC_P_mn  - 0x20 */
-+#define RZG3E_P0	0
-+#define RZG3E_P1	1
-+#define RZG3E_P2	2
-+#define RZG3E_P3	3
-+#define RZG3E_P4	4
-+#define RZG3E_P5	5
-+#define RZG3E_P6	6
-+#define RZG3E_P7	7
-+#define RZG3E_P8	8
-+#define RZG3E_PA	10
-+#define RZG3E_PB	11
-+#define RZG3E_PC	12
-+#define RZG3E_PD	13
-+#define RZG3E_PE	14
-+#define RZG3E_PF	15
-+#define RZG3E_PG	16
-+#define RZG3E_PH	17
-+#define RZG3E_PJ	19
-+#define RZG3E_PK	20
-+#define RZG3E_PL	21
-+#define RZG3E_PM	22
-+#define RZG3E_PS	28
-+
-+#define RZG3E_PORT_PINMUX(b, p, f)	RZG2L_PORT_PINMUX(RZG3E_P##b, p, f)
-+#define RZG3E_GPIO(port, pin)		RZG2L_GPIO(RZG3E_P##port, pin)
-+
-+#endif /* __DT_BINDINGS_PINCTRL_RENESAS_R9A09G047_PINCTRL_H__ */
-diff --git a/include/dt-bindings/pinctrl/renesas,r9a09g057-pinctrl.h b/include/dt-bindings/pinctrl/renesas,r9a09g057-pinctrl.h
-new file mode 100644
-index 000000000000..2e83bf43160b
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/renesas,r9a09g057-pinctrl.h
-@@ -0,0 +1,31 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * This header provides constants for Renesas RZ/V2H family pinctrl bindings.
-+ *
-+ * Copyright (C) 2024 Renesas Electronics Corp.
-+ *
-+ */
-+
-+#ifndef __DT_BINDINGS_PINCTRL_RENESAS_R9A09G057_PINCTRL_H__
-+#define __DT_BINDINGS_PINCTRL_RENESAS_R9A09G057_PINCTRL_H__
-+
-+#include <dt-bindings/pinctrl/rzg2l-pinctrl.h>
-+
-+/* RZV2H_Px = Offset address of PFC_P_mn  - 0x20 */
-+#define RZV2H_P0	0
-+#define RZV2H_P1	1
-+#define RZV2H_P2	2
-+#define RZV2H_P3	3
-+#define RZV2H_P4	4
-+#define RZV2H_P5	5
-+#define RZV2H_P6	6
-+#define RZV2H_P7	7
-+#define RZV2H_P8	8
-+#define RZV2H_P9	9
-+#define RZV2H_PA	10
-+#define RZV2H_PB	11
-+
-+#define RZV2H_PORT_PINMUX(b, p, f)	RZG2L_PORT_PINMUX(RZV2H_P##b, p, f)
-+#define RZV2H_GPIO(port, pin)		RZG2L_GPIO(RZV2H_P##port, pin)
-+
-+#endif /* __DT_BINDINGS_PINCTRL_RENESAS_R9A09G057_PINCTRL_H__ */
-diff --git a/include/dt-bindings/pinctrl/renesas,r9a09g077-pinctrl.h b/include/dt-bindings/pinctrl/renesas,r9a09g077-pinctrl.h
-new file mode 100644
-index 000000000000..f088793f23ee
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/renesas,r9a09g077-pinctrl.h
-@@ -0,0 +1,22 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * This header provides constants for Renesas RZ/T2H family pinctrl bindings.
-+ *
-+ * Copyright (C) 2025 Renesas Electronics Corp.
-+ */
-+
-+#ifndef __DT_BINDINGS_PINCTRL_RENESAS_R9A09G077_PINCTRL_H__
-+#define __DT_BINDINGS_PINCTRL_RENESAS_R9A09G077_PINCTRL_H__
-+
-+#define RZT2H_PINS_PER_PORT	8
-+
-+/*
-+ * Create the pin index from its bank and position numbers and store in
-+ * the upper 16 bits the alternate function identifier
-+ */
-+#define RZT2H_PORT_PINMUX(b, p, f)	((b) * RZT2H_PINS_PER_PORT + (p) | ((f) << 16))
-+
-+/* Convert a port and pin label to its global pin index */
-+#define RZT2H_GPIO(port, pin)	((port) * RZT2H_PINS_PER_PORT + (pin))
-+
-+#endif /* __DT_BINDINGS_PINCTRL_RENESAS_R9A09G077_PINCTRL_H__ */
-diff --git a/include/dt-bindings/pinctrl/samsung.h b/include/dt-bindings/pinctrl/samsung.h
-deleted file mode 100644
-index 950970634dfe..000000000000
--- a/include/dt-bindings/pinctrl/samsung.h
-+++ /dev/null
-@@ -1,88 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Samsung's Exynos pinctrl bindings
- *
- * Copyright (c) 2016 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- * Author: Krzysztof Kozlowski <krzk@kernel.org>
- */
-
-#ifndef __DT_BINDINGS_PINCTRL_SAMSUNG_H__
-#define __DT_BINDINGS_PINCTRL_SAMSUNG_H__
-
-#define EXYNOS_PIN_PULL_NONE		0
-#define EXYNOS_PIN_PULL_DOWN		1
-#define EXYNOS_PIN_PULL_UP		3
-
-#define S3C64XX_PIN_PULL_NONE		0
-#define S3C64XX_PIN_PULL_DOWN		1
-#define S3C64XX_PIN_PULL_UP		2
-
-/* Pin function in power down mode */
-#define EXYNOS_PIN_PDN_OUT0		0
-#define EXYNOS_PIN_PDN_OUT1		1
-#define EXYNOS_PIN_PDN_INPUT		2
-#define EXYNOS_PIN_PDN_PREV		3
-
-/* Drive strengths for Exynos3250, Exynos4 (all) and Exynos5250 */
-#define EXYNOS4_PIN_DRV_LV1		0
-#define EXYNOS4_PIN_DRV_LV2		2
-#define EXYNOS4_PIN_DRV_LV3		1
-#define EXYNOS4_PIN_DRV_LV4		3
-
-/* Drive strengths for Exynos5260 */
-#define EXYNOS5260_PIN_DRV_LV1		0
-#define EXYNOS5260_PIN_DRV_LV2		1
-#define EXYNOS5260_PIN_DRV_LV4		2
-#define EXYNOS5260_PIN_DRV_LV6		3
-
-/*
- * Drive strengths for Exynos5410, Exynos542x, Exynos5800 and Exynos850 (except
- * GPIO_HSI block)
- */
-#define EXYNOS5420_PIN_DRV_LV1		0
-#define EXYNOS5420_PIN_DRV_LV2		1
-#define EXYNOS5420_PIN_DRV_LV3		2
-#define EXYNOS5420_PIN_DRV_LV4		3
-
-/* Drive strengths for Exynos5433 */
-#define EXYNOS5433_PIN_DRV_FAST_SR1	0
-#define EXYNOS5433_PIN_DRV_FAST_SR2	1
-#define EXYNOS5433_PIN_DRV_FAST_SR3	2
-#define EXYNOS5433_PIN_DRV_FAST_SR4	3
-#define EXYNOS5433_PIN_DRV_FAST_SR5	4
-#define EXYNOS5433_PIN_DRV_FAST_SR6	5
-#define EXYNOS5433_PIN_DRV_SLOW_SR1	8
-#define EXYNOS5433_PIN_DRV_SLOW_SR2	9
-#define EXYNOS5433_PIN_DRV_SLOW_SR3	0xa
-#define EXYNOS5433_PIN_DRV_SLOW_SR4	0xb
-#define EXYNOS5433_PIN_DRV_SLOW_SR5	0xc
-#define EXYNOS5433_PIN_DRV_SLOW_SR6	0xf
-
-/* Drive strengths for Exynos850 GPIO_HSI block */
-#define EXYNOS850_HSI_PIN_DRV_LV1	0	/* 1x   */
-#define EXYNOS850_HSI_PIN_DRV_LV1_5	1	/* 1.5x */
-#define EXYNOS850_HSI_PIN_DRV_LV2	2	/* 2x   */
-#define EXYNOS850_HSI_PIN_DRV_LV2_5	3	/* 2.5x */
-#define EXYNOS850_HSI_PIN_DRV_LV3	4	/* 3x   */
-#define EXYNOS850_HSI_PIN_DRV_LV4	5	/* 4x   */
-
-#define EXYNOS_PIN_FUNC_INPUT		0
-#define EXYNOS_PIN_FUNC_OUTPUT		1
-#define EXYNOS_PIN_FUNC_2		2
-#define EXYNOS_PIN_FUNC_3		3
-#define EXYNOS_PIN_FUNC_4		4
-#define EXYNOS_PIN_FUNC_5		5
-#define EXYNOS_PIN_FUNC_6		6
-#define EXYNOS_PIN_FUNC_EINT		0xf
-#define EXYNOS_PIN_FUNC_F		EXYNOS_PIN_FUNC_EINT
-
-/* Drive strengths for Exynos7 FSYS1 block */
-#define EXYNOS7_FSYS1_PIN_DRV_LV1	0
-#define EXYNOS7_FSYS1_PIN_DRV_LV2	4
-#define EXYNOS7_FSYS1_PIN_DRV_LV3	2
-#define EXYNOS7_FSYS1_PIN_DRV_LV4	6
-#define EXYNOS7_FSYS1_PIN_DRV_LV5	1
-#define EXYNOS7_FSYS1_PIN_DRV_LV6	5
-
-#endif /* __DT_BINDINGS_PINCTRL_SAMSUNG_H__ */
-diff --git a/include/dt-bindings/pinctrl/starfive,jh7110-pinctrl.h b/include/dt-bindings/pinctrl/starfive,jh7110-pinctrl.h
-new file mode 100644
-index 000000000000..3865f0139639
--- /dev/null
-+++ b/include/dt-bindings/pinctrl/starfive,jh7110-pinctrl.h
-@@ -0,0 +1,137 @@
-+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
-+/*
-+ * Copyright (C) 2022 Emil Renner Berthing <kernel@esmil.dk>
-+ * Copyright (C) 2022 StarFive Technology Co., Ltd.
-+ */
-+
-+#ifndef __DT_BINDINGS_PINCTRL_STARFIVE_JH7110_H__
-+#define __DT_BINDINGS_PINCTRL_STARFIVE_JH7110_H__
-+
-+/* sys_iomux pins */
-+#define	PAD_GPIO0		0
-+#define	PAD_GPIO1		1
-+#define	PAD_GPIO2		2
-+#define	PAD_GPIO3		3
-+#define	PAD_GPIO4		4
-+#define	PAD_GPIO5		5
-+#define	PAD_GPIO6		6
-+#define	PAD_GPIO7		7
-+#define	PAD_GPIO8		8
-+#define	PAD_GPIO9		9
-+#define	PAD_GPIO10		10
-+#define	PAD_GPIO11		11
-+#define	PAD_GPIO12		12
-+#define	PAD_GPIO13		13
-+#define	PAD_GPIO14		14
-+#define	PAD_GPIO15		15
-+#define	PAD_GPIO16		16
-+#define	PAD_GPIO17		17
-+#define	PAD_GPIO18		18
-+#define	PAD_GPIO19		19
-+#define	PAD_GPIO20		20
-+#define	PAD_GPIO21		21
-+#define	PAD_GPIO22		22
-+#define	PAD_GPIO23		23
-+#define	PAD_GPIO24		24
-+#define	PAD_GPIO25		25
-+#define	PAD_GPIO26		26
-+#define	PAD_GPIO27		27
-+#define	PAD_GPIO28		28
-+#define	PAD_GPIO29		29
-+#define	PAD_GPIO30		30
-+#define	PAD_GPIO31		31
-+#define	PAD_GPIO32		32
-+#define	PAD_GPIO33		33
-+#define	PAD_GPIO34		34
-+#define	PAD_GPIO35		35
-+#define	PAD_GPIO36		36
-+#define	PAD_GPIO37		37
-+#define	PAD_GPIO38		38
-+#define	PAD_GPIO39		39
-+#define	PAD_GPIO40		40
-+#define	PAD_GPIO41		41
-+#define	PAD_GPIO42		42
-+#define	PAD_GPIO43		43
-+#define	PAD_GPIO44		44
-+#define	PAD_GPIO45		45
-+#define	PAD_GPIO46		46
-+#define	PAD_GPIO47		47
-+#define	PAD_GPIO48		48
-+#define	PAD_GPIO49		49
-+#define	PAD_GPIO50		50
-+#define	PAD_GPIO51		51
-+#define	PAD_GPIO52		52
-+#define	PAD_GPIO53		53
-+#define	PAD_GPIO54		54
-+#define	PAD_GPIO55		55
-+#define	PAD_GPIO56		56
-+#define	PAD_GPIO57		57
-+#define	PAD_GPIO58		58
-+#define	PAD_GPIO59		59
-+#define	PAD_GPIO60		60
-+#define	PAD_GPIO61		61
-+#define	PAD_GPIO62		62
-+#define	PAD_GPIO63		63
-+#define	PAD_SD0_CLK		64
-+#define	PAD_SD0_CMD		65
-+#define	PAD_SD0_DATA0		66
-+#define	PAD_SD0_DATA1		67
-+#define	PAD_SD0_DATA2		68
-+#define	PAD_SD0_DATA3		69
-+#define	PAD_SD0_DATA4		70
-+#define	PAD_SD0_DATA5		71
-+#define	PAD_SD0_DATA6		72
-+#define	PAD_SD0_DATA7		73
-+#define	PAD_SD0_STRB		74
-+#define	PAD_GMAC1_MDC		75
-+#define	PAD_GMAC1_MDIO		76
-+#define	PAD_GMAC1_RXD0		77
-+#define	PAD_GMAC1_RXD1		78
-+#define	PAD_GMAC1_RXD2		79
-+#define	PAD_GMAC1_RXD3		80
-+#define	PAD_GMAC1_RXDV		81
-+#define	PAD_GMAC1_RXC		82
-+#define	PAD_GMAC1_TXD0		83
-+#define	PAD_GMAC1_TXD1		84
-+#define	PAD_GMAC1_TXD2		85
-+#define	PAD_GMAC1_TXD3		86
-+#define	PAD_GMAC1_TXEN		87
-+#define	PAD_GMAC1_TXC		88
-+#define	PAD_QSPI_SCLK		89
-+#define	PAD_QSPI_CS0		90
-+#define	PAD_QSPI_DATA0		91
-+#define	PAD_QSPI_DATA1		92
-+#define	PAD_QSPI_DATA2		93
-+#define	PAD_QSPI_DATA3		94
-+
-+/* aon_iomux pins */
-+#define	PAD_TESTEN		0
-+#define	PAD_RGPIO0		1
-+#define	PAD_RGPIO1		2
-+#define	PAD_RGPIO2		3
-+#define	PAD_RGPIO3		4
-+#define	PAD_RSTN		5
-+#define	PAD_GMAC0_MDC		6
-+#define	PAD_GMAC0_MDIO		7
-+#define	PAD_GMAC0_RXD0		8
-+#define	PAD_GMAC0_RXD1		9
-+#define	PAD_GMAC0_RXD2		10
-+#define	PAD_GMAC0_RXD3		11
-+#define	PAD_GMAC0_RXDV		12
-+#define	PAD_GMAC0_RXC		13
-+#define	PAD_GMAC0_TXD0		14
-+#define	PAD_GMAC0_TXD1		15
-+#define	PAD_GMAC0_TXD2		16
-+#define	PAD_GMAC0_TXD3		17
-+#define	PAD_GMAC0_TXEN		18
-+#define	PAD_GMAC0_TXC		19
-+
-+#define GPOUT_LOW		0
-+#define GPOUT_HIGH		1
-+
-+#define GPOEN_ENABLE		0
-+#define GPOEN_DISABLE		1
-+
-+#define GPI_NONE		255
-+
-+#endif
-diff --git a/include/dt-bindings/pinctrl/stm32-pinfunc.h b/include/dt-bindings/pinctrl/stm32-pinfunc.h
-index e6fb8ada3f4d..af3fd388329a 100644
--- a/include/dt-bindings/pinctrl/stm32-pinfunc.h
-+++ b/include/dt-bindings/pinctrl/stm32-pinfunc.h
-@@ -26,6 +26,7 @@
- #define AF14	0xf
- #define AF15	0x10
- #define ANALOG	0x11
-+#define RSVD	0x12
- 
- /* define Pins number*/
- #define PIN_NO(port, line)	(((port) - 'A') * 0x10 + (line))
-@@ -37,6 +38,9 @@
- #define STM32MP_PKG_AB	0x2
- #define STM32MP_PKG_AC	0x4
- #define STM32MP_PKG_AD	0x8
-+#define STM32MP_PKG_AI	0x100
-+#define STM32MP_PKG_AK	0x400
-+#define STM32MP_PKG_AL	0x800
- 
- #endif /* _DT_BINDINGS_STM32_PINFUNC_H */
- 
-diff --git a/include/dt-bindings/power/allwinner,sun20i-d1-ppu.h b/include/dt-bindings/power/allwinner,sun20i-d1-ppu.h
-new file mode 100644
-index 000000000000..23cfb57256d6
--- /dev/null
-+++ b/include/dt-bindings/power/allwinner,sun20i-d1-ppu.h
-@@ -0,0 +1,10 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_POWER_SUN20I_D1_PPU_H_
-+#define _DT_BINDINGS_POWER_SUN20I_D1_PPU_H_
-+
-+#define PD_CPU			0
-+#define PD_VE			1
-+#define PD_DSP			2
-+
-+#endif /* _DT_BINDINGS_POWER_SUN20I_D1_PPU_H_ */
-diff --git a/include/dt-bindings/power/allwinner,sun55i-a523-pck-600.h b/include/dt-bindings/power/allwinner,sun55i-a523-pck-600.h
-new file mode 100644
-index 000000000000..6b3d8ea7bb69
--- /dev/null
-+++ b/include/dt-bindings/power/allwinner,sun55i-a523-pck-600.h
-@@ -0,0 +1,15 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_POWER_SUN55I_A523_PCK600_H_
-+#define _DT_BINDINGS_POWER_SUN55I_A523_PCK600_H_
-+
-+#define PD_VE			0
-+#define PD_GPU			1
-+#define PD_VI			2
-+#define PD_VO0			3
-+#define PD_VO1			4
-+#define PD_DE			5
-+#define PD_NAND			6
-+#define PD_PCIE			7
-+
-+#endif /* _DT_BINDINGS_POWER_SUN55I_A523_PCK600_H_ */
-diff --git a/include/dt-bindings/power/allwinner,sun55i-a523-ppu.h b/include/dt-bindings/power/allwinner,sun55i-a523-ppu.h
-new file mode 100644
-index 000000000000..bc9aba73c19a
--- /dev/null
-+++ b/include/dt-bindings/power/allwinner,sun55i-a523-ppu.h
-@@ -0,0 +1,12 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_POWER_SUN55I_A523_PPU_H_
-+#define _DT_BINDINGS_POWER_SUN55I_A523_PPU_H_
-+
-+#define PD_DSP			0
-+#define PD_NPU			1
-+#define PD_AUDIO		2
-+#define PD_SRAM			3
-+#define PD_RISCV		4
-+
-+#endif /* _DT_BINDINGS_POWER_SUN55I_A523_PPU_H_ */
-diff --git a/include/dt-bindings/power/allwinner,sun8i-v853-ppu.h b/include/dt-bindings/power/allwinner,sun8i-v853-ppu.h
-new file mode 100644
-index 000000000000..b1c18a490613
--- /dev/null
-+++ b/include/dt-bindings/power/allwinner,sun8i-v853-ppu.h
-@@ -0,0 +1,10 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_POWER_SUN8I_V853_PPU_H_
-+#define _DT_BINDINGS_POWER_SUN8I_V853_PPU_H_
-+
-+#define PD_RISCV		0
-+#define PD_NPU			1
-+#define PD_VE			2
-+
-+#endif
-diff --git a/include/dt-bindings/power/amlogic,a4-pwrc.h b/include/dt-bindings/power/amlogic,a4-pwrc.h
-new file mode 100644
-index 000000000000..bd2f9c558d22
--- /dev/null
-+++ b/include/dt-bindings/power/amlogic,a4-pwrc.h
-@@ -0,0 +1,21 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (C) 2024 Amlogic, Inc. All rights reserved
-+ */
-+#ifndef _DT_BINDINGS_AMLOGIC_A4_POWER_H
-+#define _DT_BINDINGS_AMLOGIC_A4_POWER_H
-+
-+#define PWRC_A4_AUDIO_ID				0
-+#define PWRC_A4_SDIOA_ID				1
-+#define PWRC_A4_EMMC_ID					2
-+#define PWRC_A4_USB_COMB_ID				3
-+#define PWRC_A4_ETH_ID					4
-+#define PWRC_A4_VOUT_ID					5
-+#define PWRC_A4_AUDIO_PDM_ID				6
-+#define PWRC_A4_DMC_ID					7
-+#define PWRC_A4_SYS_WRAP_ID				8
-+#define PWRC_A4_AO_I2C_S_ID				9
-+#define PWRC_A4_AO_UART_ID				10
-+#define PWRC_A4_AO_IR_ID				11
-+
-+#endif
-diff --git a/include/dt-bindings/power/amlogic,a5-pwrc.h b/include/dt-bindings/power/amlogic,a5-pwrc.h
-new file mode 100644
-index 000000000000..3a6f53eb959f
--- /dev/null
-+++ b/include/dt-bindings/power/amlogic,a5-pwrc.h
-@@ -0,0 +1,21 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (C) 2024 Amlogic, Inc. All rights reserved
-+ */
-+
-+#ifndef _DT_BINDINGS_AMLOGIC_A5_POWER_H
-+#define _DT_BINDINGS_AMLOGIC_A5_POWER_H
-+
-+#define PWRC_A5_NNA_ID			0
-+#define PWRC_A5_AUDIO_ID		1
-+#define PWRC_A5_SDIOA_ID		2
-+#define PWRC_A5_EMMC_ID			3
-+#define PWRC_A5_USB_COMB_ID		4
-+#define PWRC_A5_ETH_ID			5
-+#define PWRC_A5_RSA_ID			6
-+#define PWRC_A5_AUDIO_PDM_ID		7
-+#define PWRC_A5_DMC_ID			8
-+#define PWRC_A5_SYS_WRAP_ID		9
-+#define PWRC_A5_DSPA_ID			10
-+
-+#endif
-diff --git a/include/dt-bindings/power/amlogic,c3-pwrc.h b/include/dt-bindings/power/amlogic,c3-pwrc.h
-new file mode 100644
-index 000000000000..61759df4b2e7
--- /dev/null
-+++ b/include/dt-bindings/power/amlogic,c3-pwrc.h
-@@ -0,0 +1,25 @@
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
-+/*
-+ * Copyright (c) 2023 Amlogic, Inc.
-+ * Author: hongyu chen1 <hongyu.chen1@amlogic.com>
-+ */
-+#ifndef _DT_BINDINGS_AMLOGIC_C3_POWER_H
-+#define _DT_BINDINGS_AMLOGIC_C3_POWER_H
-+
-+#define PWRC_C3_NNA_ID				0
-+#define PWRC_C3_AUDIO_ID			1
-+#define PWRC_C3_RESV_SEC_ID			2
-+#define PWRC_C3_SDIOA_ID			3
-+#define PWRC_C3_EMMC_ID				4
-+#define PWRC_C3_USB_COMB_ID			5
-+#define PWRC_C3_SDCARD_ID			6
-+#define PWRC_C3_ETH_ID				7
-+#define PWRC_C3_RESV0_ID			8
-+#define PWRC_C3_GE2D_ID				9
-+#define PWRC_C3_CVE_ID				10
-+#define PWRC_C3_GDC_WRAP_ID			11
-+#define PWRC_C3_ISP_TOP_ID			12
-+#define PWRC_C3_MIPI_ISP_WRAP_ID		13
-+#define PWRC_C3_VCODEC_ID			14
-+
-+#endif
-diff --git a/include/dt-bindings/power/amlogic,s6-pwrc.h b/include/dt-bindings/power/amlogic,s6-pwrc.h
-new file mode 100644
-index 000000000000..2c005864ae73
--- /dev/null
-+++ b/include/dt-bindings/power/amlogic,s6-pwrc.h
-@@ -0,0 +1,29 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (C) 2025 Amlogic, Inc. All rights reserved
-+ */
-+#ifndef _DT_BINDINGS_AMLOGIC_S6_POWER_H
-+#define _DT_BINDINGS_AMLOGIC_S6_POWER_H
-+
-+#define PWRC_S6_DSPA_ID				0
-+#define PWRC_S6_DOS_HEVC_ID			1
-+#define PWRC_S6_DOS_VDEC_ID			2
-+#define PWRC_S6_VPU_HDMI_ID			3
-+#define PWRC_S6_U2DRD_ID			4
-+#define PWRC_S6_U3DRD_ID			5
-+#define PWRC_S6_SD_EMMC_C_ID			6
-+#define PWRC_S6_GE2D_ID				7
-+#define PWRC_S6_AMFC_ID				8
-+#define PWRC_S6_VC9000E_ID			9
-+#define PWRC_S6_DEWARP_ID			10
-+#define PWRC_S6_VICP_ID				11
-+#define PWRC_S6_SD_EMMC_A_ID			12
-+#define PWRC_S6_SD_EMMC_B_ID			13
-+#define PWRC_S6_ETH_ID				14
-+#define PWRC_S6_PCIE_ID				15
-+#define PWRC_S6_NNA_4T_ID			16
-+#define PWRC_S6_AUDIO_ID			17
-+#define PWRC_S6_AUCPU_ID			18
-+#define PWRC_S6_ADAPT_ID			19
-+
-+#endif
-diff --git a/include/dt-bindings/power/amlogic,s7-pwrc.h b/include/dt-bindings/power/amlogic,s7-pwrc.h
-new file mode 100644
-index 000000000000..3f21d095f784
--- /dev/null
-+++ b/include/dt-bindings/power/amlogic,s7-pwrc.h
-@@ -0,0 +1,20 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (C) 2025 Amlogic, Inc. All rights reserved
-+ */
-+#ifndef _DT_BINDINGS_AMLOGIC_S7_POWER_H
-+#define _DT_BINDINGS_AMLOGIC_S7_POWER_H
-+
-+#define PWRC_S7_DOS_HEVC_ID			0
-+#define PWRC_S7_DOS_VDEC_ID		        1
-+#define PWRC_S7_VPU_HDMI_ID		        2
-+#define PWRC_S7_USB_COMB_ID		        3
-+#define PWRC_S7_SD_EMMC_C_ID			4
-+#define PWRC_S7_GE2D_ID				5
-+#define PWRC_S7_SD_EMMC_A_ID			6
-+#define PWRC_S7_SD_EMMC_B_ID			7
-+#define PWRC_S7_ETH_ID				8
-+#define PWRC_S7_AUCPU_ID			9
-+#define PWRC_S7_AUDIO_ID			10
-+
-+#endif
-diff --git a/include/dt-bindings/power/amlogic,s7d-pwrc.h b/include/dt-bindings/power/amlogic,s7d-pwrc.h
-new file mode 100644
-index 000000000000..c6998553670a
--- /dev/null
-+++ b/include/dt-bindings/power/amlogic,s7d-pwrc.h
-@@ -0,0 +1,27 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (C) 2025 Amlogic, Inc. All rights reserved
-+ */
-+#ifndef _DT_BINDINGS_AMLOGIC_S7D_POWER_H
-+#define _DT_BINDINGS_AMLOGIC_S7D_POWER_H
-+
-+#define PWRC_S7D_DOS_HCODEC_ID			0
-+#define PWRC_S7D_DOS_HEVC_ID			1
-+#define PWRC_S7D_DOS_VDEC_ID			2
-+#define PWRC_S7D_VPU_HDMI_ID			3
-+#define PWRC_S7D_USB_U2DRD_ID			4
-+#define PWRC_S7D_USB_U2H_ID			5
-+#define PWRC_S7D_SSD_EMMC_C_ID			6
-+#define PWRC_S7D_GE2D_ID			7
-+#define PWRC_S7D_AMFC_ID			8
-+#define PWRC_S7D_EMMC_A_ID			9
-+#define PWRC_S7D_EMMC_B_ID			10
-+#define PWRC_S7D_ETH_ID				11
-+#define PWRC_S7D_AUCPU_ID			12
-+#define PWRC_S7D_AUDIO_ID			13
-+#define PWRC_S7D_SRAMA_ID			14
-+#define PWRC_S7D_DMC0_ID			15
-+#define PWRC_S7D_DMC1_ID			16
-+#define PWRC_S7D_DDR_ID				17
-+
-+#endif
-diff --git a/include/dt-bindings/power/amlogic,t7-pwrc.h b/include/dt-bindings/power/amlogic,t7-pwrc.h
-new file mode 100644
-index 000000000000..1f1f2739cc26
--- /dev/null
-+++ b/include/dt-bindings/power/amlogic,t7-pwrc.h
-@@ -0,0 +1,63 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (c) 2023 Amlogic, Inc.
-+ * Author: Hongyu Chen <hongyu.chen1@amlogic.com>
-+ */
-+#ifndef _DT_BINDINGS_AMLOGIC_T7_POWER_H
-+#define _DT_BINDINGS_AMLOGIC_T7_POWER_H
-+
-+#define PWRC_T7_DSPA_ID			0
-+#define PWRC_T7_DSPB_ID			1
-+#define PWRC_T7_DOS_HCODEC_ID		2
-+#define PWRC_T7_DOS_HEVC_ID		3
-+#define PWRC_T7_DOS_VDEC_ID		4
-+#define PWRC_T7_DOS_WAVE_ID		5
-+#define PWRC_T7_VPU_HDMI_ID		6
-+#define PWRC_T7_USB_COMB_ID		7
-+#define PWRC_T7_PCIE_ID			8
-+#define PWRC_T7_GE2D_ID			9
-+#define PWRC_T7_SRAMA_ID		10
-+#define PWRC_T7_SRAMB_ID		11
-+#define PWRC_T7_HDMIRX_ID		12
-+#define PWRC_T7_VI_CLK1_ID		13
-+#define PWRC_T7_VI_CLK2_ID		14
-+#define PWRC_T7_ETH_ID			15
-+#define PWRC_T7_ISP_ID			16
-+#define PWRC_T7_MIPI_ISP_ID		17
-+#define PWRC_T7_GDC_ID			18
-+#define PWRC_T7_CVE_ID			18
-+#define PWRC_T7_DEWARP_ID		19
-+#define PWRC_T7_SDIO_A_ID		20
-+#define PWRC_T7_SDIO_B_ID		21
-+#define PWRC_T7_EMMC_ID			22
-+#define PWRC_T7_MALI_SC0_ID		23
-+#define PWRC_T7_MALI_SC1_ID		24
-+#define PWRC_T7_MALI_SC2_ID		25
-+#define PWRC_T7_MALI_SC3_ID		26
-+#define PWRC_T7_MALI_TOP_ID		27
-+#define PWRC_T7_NNA_CORE0_ID		28
-+#define PWRC_T7_NNA_CORE1_ID		29
-+#define PWRC_T7_NNA_CORE2_ID		30
-+#define PWRC_T7_NNA_CORE3_ID		31
-+#define PWRC_T7_NNA_TOP_ID		32
-+#define PWRC_T7_DDR0_ID			33
-+#define PWRC_T7_DDR1_ID			34
-+#define PWRC_T7_DMC0_ID			35
-+#define PWRC_T7_DMC1_ID			36
-+#define PWRC_T7_NOC_ID			37
-+#define PWRC_T7_NIC2_ID			38
-+#define PWRC_T7_NIC3_ID			39
-+#define PWRC_T7_CCI_ID			40
-+#define PWRC_T7_MIPI_DSI0_ID		41
-+#define PWRC_T7_SPICC0_ID		42
-+#define PWRC_T7_SPICC1_ID		43
-+#define PWRC_T7_SPICC2_ID		44
-+#define PWRC_T7_SPICC3_ID		45
-+#define PWRC_T7_SPICC4_ID		46
-+#define PWRC_T7_SPICC5_ID		47
-+#define PWRC_T7_EDP0_ID			48
-+#define PWRC_T7_EDP1_ID			49
-+#define PWRC_T7_MIPI_DSI1_ID		50
-+#define PWRC_T7_AUDIO_ID		51
-+
-+#endif
-diff --git a/include/dt-bindings/power/fsl,imx93-power.h b/include/dt-bindings/power/fsl,imx93-power.h
-new file mode 100644
-index 000000000000..17f9f015bf7d
--- /dev/null
-+++ b/include/dt-bindings/power/fsl,imx93-power.h
-@@ -0,0 +1,15 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
-+/*
-+ *  Copyright 2022 NXP
-+ */
-+
-+#ifndef __DT_BINDINGS_IMX93_POWER_H__
-+#define __DT_BINDINGS_IMX93_POWER_H__
-+
-+#define IMX93_MEDIABLK_PD_MIPI_DSI		0
-+#define IMX93_MEDIABLK_PD_MIPI_CSI		1
-+#define IMX93_MEDIABLK_PD_PXP			2
-+#define IMX93_MEDIABLK_PD_LCDIF			3
-+#define IMX93_MEDIABLK_PD_ISI			4
-+
-+#endif
-diff --git a/include/dt-bindings/power/imx8mp-power.h b/include/dt-bindings/power/imx8mp-power.h
-index 7789bcca3223..2fe3c2abad13 100644
--- a/include/dt-bindings/power/imx8mp-power.h
-+++ b/include/dt-bindings/power/imx8mp-power.h
-@@ -49,5 +49,11 @@
- #define IMX8MP_HDMIBLK_PD_TRNG				4
- #define IMX8MP_HDMIBLK_PD_HDMI_TX			5
- #define IMX8MP_HDMIBLK_PD_HDMI_TX_PHY			6
-+#define IMX8MP_HDMIBLK_PD_HDCP				7
-+#define IMX8MP_HDMIBLK_PD_HRV				8
-+
-+#define IMX8MP_VPUBLK_PD_G1				0
-+#define IMX8MP_VPUBLK_PD_G2				1
-+#define IMX8MP_VPUBLK_PD_VC8000E			2
- 
- #endif
-diff --git a/include/dt-bindings/power/marvell,pxa1908-power.h b/include/dt-bindings/power/marvell,pxa1908-power.h
-new file mode 100644
-index 000000000000..19b088351af1
--- /dev/null
-+++ b/include/dt-bindings/power/marvell,pxa1908-power.h
-@@ -0,0 +1,17 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Marvell PXA1908 power domains
-+ *
-+ * Copyright 2025, Duje Mihanović <duje@dujemihanovic.xyz>
-+ */
-+
-+#ifndef __DTS_MARVELL_PXA1908_POWER_H
-+#define __DTS_MARVELL_PXA1908_POWER_H
-+
-+#define PXA1908_POWER_DOMAIN_VPU	0
-+#define PXA1908_POWER_DOMAIN_GPU	1
-+#define PXA1908_POWER_DOMAIN_GPU2D	2
-+#define PXA1908_POWER_DOMAIN_DSI	3
-+#define PXA1908_POWER_DOMAIN_ISP	4
-+
-+#endif
-diff --git a/include/dt-bindings/power/mediatek,mt6735-power-controller.h b/include/dt-bindings/power/mediatek,mt6735-power-controller.h
-new file mode 100644
-index 000000000000..6957075fcb9e
--- /dev/null
-+++ b/include/dt-bindings/power/mediatek,mt6735-power-controller.h
-@@ -0,0 +1,14 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_POWER_MT6735_POWER_CONTROLLER_H
-+#define _DT_BINDINGS_POWER_MT6735_POWER_CONTROLLER_H
-+
-+#define MT6735_POWER_DOMAIN_MD1		0
-+#define MT6735_POWER_DOMAIN_CONN	1
-+#define MT6735_POWER_DOMAIN_DIS		2
-+#define MT6735_POWER_DOMAIN_MFG		3
-+#define MT6735_POWER_DOMAIN_ISP		4
-+#define MT6735_POWER_DOMAIN_VDE		5
-+#define MT6735_POWER_DOMAIN_VEN		6
-+
-+#endif
-diff --git a/include/dt-bindings/power/mediatek,mt6893-power.h b/include/dt-bindings/power/mediatek,mt6893-power.h
-new file mode 100644
-index 000000000000..aeab51bb2ad8
--- /dev/null
-+++ b/include/dt-bindings/power/mediatek,mt6893-power.h
-@@ -0,0 +1,35 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
-+/*
-+ * Copyright (c) 2025 Collabora Ltd
-+ *                    AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_POWER_MT6893_POWER_H
-+#define _DT_BINDINGS_POWER_MT6893_POWER_H
-+
-+#define MT6893_POWER_DOMAIN_CONN		0
-+#define MT6893_POWER_DOMAIN_MFG0		1
-+#define MT6893_POWER_DOMAIN_MFG1		2
-+#define MT6893_POWER_DOMAIN_MFG2		3
-+#define MT6893_POWER_DOMAIN_MFG3		4
-+#define MT6893_POWER_DOMAIN_MFG4		5
-+#define MT6893_POWER_DOMAIN_MFG5		6
-+#define MT6893_POWER_DOMAIN_MFG6		7
-+#define MT6893_POWER_DOMAIN_ISP			8
-+#define MT6893_POWER_DOMAIN_ISP2		9
-+#define MT6893_POWER_DOMAIN_IPE			10
-+#define MT6893_POWER_DOMAIN_VDEC0		11
-+#define MT6893_POWER_DOMAIN_VDEC1		12
-+#define MT6893_POWER_DOMAIN_VENC0		13
-+#define MT6893_POWER_DOMAIN_VENC1		14
-+#define MT6893_POWER_DOMAIN_MDP			15
-+#define MT6893_POWER_DOMAIN_DISP		16
-+#define MT6893_POWER_DOMAIN_AUDIO		17
-+#define MT6893_POWER_DOMAIN_ADSP		18
-+#define MT6893_POWER_DOMAIN_CAM			19
-+#define MT6893_POWER_DOMAIN_CAM_RAWA		20
-+#define MT6893_POWER_DOMAIN_CAM_RAWB		21
-+#define MT6893_POWER_DOMAIN_CAM_RAWC		22
-+#define MT6893_POWER_DOMAIN_DP_TX		23
-+
-+#endif /* _DT_BINDINGS_POWER_MT6893_POWER_H */
-diff --git a/include/dt-bindings/power/mediatek,mt8188-power.h b/include/dt-bindings/power/mediatek,mt8188-power.h
-new file mode 100644
-index 000000000000..57e75cf3aa2c
--- /dev/null
-+++ b/include/dt-bindings/power/mediatek,mt8188-power.h
-@@ -0,0 +1,44 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
-+/*
-+ * Copyright (c) 2022 MediaTek Inc.
-+ * Author: Garmin Chang <garmin.chang@mediatek.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_POWER_MT8188_POWER_H
-+#define _DT_BINDINGS_POWER_MT8188_POWER_H
-+
-+#define MT8188_POWER_DOMAIN_MFG0			0
-+#define MT8188_POWER_DOMAIN_MFG1			1
-+#define MT8188_POWER_DOMAIN_MFG2			2
-+#define MT8188_POWER_DOMAIN_MFG3			3
-+#define MT8188_POWER_DOMAIN_MFG4			4
-+#define MT8188_POWER_DOMAIN_PEXTP_MAC_P0		5
-+#define MT8188_POWER_DOMAIN_PEXTP_PHY_TOP		6
-+#define MT8188_POWER_DOMAIN_CSIRX_TOP			7
-+#define MT8188_POWER_DOMAIN_ETHER			8
-+#define MT8188_POWER_DOMAIN_HDMI_TX			9
-+#define MT8188_POWER_DOMAIN_ADSP_AO			10
-+#define MT8188_POWER_DOMAIN_ADSP_INFRA			11
-+#define MT8188_POWER_DOMAIN_ADSP			12
-+#define MT8188_POWER_DOMAIN_AUDIO			13
-+#define MT8188_POWER_DOMAIN_AUDIO_ASRC			14
-+#define MT8188_POWER_DOMAIN_VPPSYS0			15
-+#define MT8188_POWER_DOMAIN_VDOSYS0			16
-+#define MT8188_POWER_DOMAIN_VDOSYS1			17
-+#define MT8188_POWER_DOMAIN_DP_TX			18
-+#define MT8188_POWER_DOMAIN_EDP_TX			19
-+#define MT8188_POWER_DOMAIN_VPPSYS1			20
-+#define MT8188_POWER_DOMAIN_WPE				21
-+#define MT8188_POWER_DOMAIN_VDEC0			22
-+#define MT8188_POWER_DOMAIN_VDEC1			23
-+#define MT8188_POWER_DOMAIN_VENC			24
-+#define MT8188_POWER_DOMAIN_IMG_VCORE			25
-+#define MT8188_POWER_DOMAIN_IMG_MAIN			26
-+#define MT8188_POWER_DOMAIN_DIP				27
-+#define MT8188_POWER_DOMAIN_IPE				28
-+#define MT8188_POWER_DOMAIN_CAM_VCORE			29
-+#define MT8188_POWER_DOMAIN_CAM_MAIN			30
-+#define MT8188_POWER_DOMAIN_CAM_SUBA			31
-+#define MT8188_POWER_DOMAIN_CAM_SUBB			32
-+
-+#endif /* _DT_BINDINGS_POWER_MT8188_POWER_H */
-diff --git a/include/dt-bindings/power/mediatek,mt8196-power.h b/include/dt-bindings/power/mediatek,mt8196-power.h
-new file mode 100644
-index 000000000000..0f622a93c807
--- /dev/null
-+++ b/include/dt-bindings/power/mediatek,mt8196-power.h
-@@ -0,0 +1,58 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
-+/*
-+ * Copyright (c) 2025 Collabora Ltd
-+ *                    AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_POWER_MT8196_POWER_H
-+#define _DT_BINDINGS_POWER_MT8196_POWER_H
-+
-+/* SCPSYS Secure Power Manager - Direct Control */
-+#define MT8196_POWER_DOMAIN_MD				0
-+#define MT8196_POWER_DOMAIN_CONN			1
-+#define MT8196_POWER_DOMAIN_SSUSB_P0			2
-+#define MT8196_POWER_DOMAIN_SSUSB_DP_PHY_P0		3
-+#define MT8196_POWER_DOMAIN_SSUSB_P1			4
-+#define MT8196_POWER_DOMAIN_SSUSB_P23			5
-+#define MT8196_POWER_DOMAIN_SSUSB_PHY_P2		6
-+#define MT8196_POWER_DOMAIN_PEXTP_MAC0			7
-+#define MT8196_POWER_DOMAIN_PEXTP_MAC1			8
-+#define MT8196_POWER_DOMAIN_PEXTP_MAC2			9
-+#define MT8196_POWER_DOMAIN_PEXTP_PHY0			10
-+#define MT8196_POWER_DOMAIN_PEXTP_PHY1			11
-+#define MT8196_POWER_DOMAIN_PEXTP_PHY2			12
-+#define MT8196_POWER_DOMAIN_AUDIO			13
-+#define MT8196_POWER_DOMAIN_ADSP_TOP_DORMANT		14
-+#define MT8196_POWER_DOMAIN_ADSP_INFRA			15
-+#define MT8196_POWER_DOMAIN_ADSP_AO			16
-+
-+/* SCPSYS Secure Power Manager - HW Voter */
-+#define MT8196_POWER_DOMAIN_MM_PROC_DORMANT		0
-+#define MT8196_POWER_DOMAIN_SSR				1
-+
-+/* HFRPSYS MultiMedia Power Control (MMPC) - HW Voter */
-+#define MT8196_POWER_DOMAIN_VDE0			0
-+#define MT8196_POWER_DOMAIN_VDE1			1
-+#define MT8196_POWER_DOMAIN_VDE_VCORE0			2
-+#define MT8196_POWER_DOMAIN_VEN0			3
-+#define MT8196_POWER_DOMAIN_VEN1			4
-+#define MT8196_POWER_DOMAIN_VEN2			5
-+#define MT8196_POWER_DOMAIN_DISP_VCORE			6
-+#define MT8196_POWER_DOMAIN_DIS0_DORMANT		7
-+#define MT8196_POWER_DOMAIN_DIS1_DORMANT		8
-+#define MT8196_POWER_DOMAIN_OVL0_DORMANT		9
-+#define MT8196_POWER_DOMAIN_OVL1_DORMANT		10
-+#define MT8196_POWER_DOMAIN_DISP_EDPTX_DORMANT		11
-+#define MT8196_POWER_DOMAIN_DISP_DPTX_DORMANT		12
-+#define MT8196_POWER_DOMAIN_MML0_SHUTDOWN		13
-+#define MT8196_POWER_DOMAIN_MML1_SHUTDOWN		14
-+#define MT8196_POWER_DOMAIN_MM_INFRA0			15
-+#define MT8196_POWER_DOMAIN_MM_INFRA1			16
-+#define MT8196_POWER_DOMAIN_MM_INFRA_AO			17
-+#define MT8196_POWER_DOMAIN_CSI_BS_RX			18
-+#define MT8196_POWER_DOMAIN_CSI_LS_RX			19
-+#define MT8196_POWER_DOMAIN_DSI_PHY0			20
-+#define MT8196_POWER_DOMAIN_DSI_PHY1			21
-+#define MT8196_POWER_DOMAIN_DSI_PHY2			22
-+
-+#endif /* _DT_BINDINGS_POWER_MT8196_POWER_H */
-diff --git a/include/dt-bindings/power/mediatek,mt8365-power.h b/include/dt-bindings/power/mediatek,mt8365-power.h
-new file mode 100644
-index 000000000000..e6cfd0ec7871
--- /dev/null
-+++ b/include/dt-bindings/power/mediatek,mt8365-power.h
-@@ -0,0 +1,19 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
-+/*
-+ * Copyright (c) 2022 MediaTek Inc.
-+ */
-+
-+#ifndef _DT_BINDINGS_POWER_MT8365_POWER_H
-+#define _DT_BINDINGS_POWER_MT8365_POWER_H
-+
-+#define MT8365_POWER_DOMAIN_MM		0
-+#define MT8365_POWER_DOMAIN_CONN	1
-+#define MT8365_POWER_DOMAIN_MFG		2
-+#define MT8365_POWER_DOMAIN_AUDIO	3
-+#define MT8365_POWER_DOMAIN_CAM		4
-+#define MT8365_POWER_DOMAIN_DSP		5
-+#define MT8365_POWER_DOMAIN_VDEC	6
-+#define MT8365_POWER_DOMAIN_VENC	7
-+#define MT8365_POWER_DOMAIN_APU		8
-+
-+#endif /* _DT_BINDINGS_POWER_MT8365_POWER_H */
-diff --git a/include/dt-bindings/power/meson-a1-power.h b/include/dt-bindings/power/meson-a1-power.h
-index 6cf50bfb8ccf..724c370d6853 100644
--- a/include/dt-bindings/power/meson-a1-power.h
-+++ b/include/dt-bindings/power/meson-a1-power.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (c) 2019 Amlogic, Inc.
-  * Author: Jianxin Pan <jianxin.pan@amlogic.com>
-diff --git a/include/dt-bindings/power/meson-axg-power.h b/include/dt-bindings/power/meson-axg-power.h
-index e5243884b249..ace0e468ce21 100644
--- a/include/dt-bindings/power/meson-axg-power.h
-+++ b/include/dt-bindings/power/meson-axg-power.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (c) 2020 BayLibre, SAS
-  * Author: Neil Armstrong <narmstrong@baylibre.com>
-diff --git a/include/dt-bindings/power/meson-g12a-power.h b/include/dt-bindings/power/meson-g12a-power.h
-index bb5e67a842de..01fd0ac4dd08 100644
--- a/include/dt-bindings/power/meson-g12a-power.h
-+++ b/include/dt-bindings/power/meson-g12a-power.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (c) 2019 BayLibre, SAS
-  * Author: Neil Armstrong <narmstrong@baylibre.com>
-@@ -9,5 +9,7 @@
- 
- #define PWRC_G12A_VPU_ID		0
- #define PWRC_G12A_ETH_ID		1
-+#define PWRC_G12A_NNA_ID		2
-+#define PWRC_G12A_ISP_ID		3
- 
- #endif
-diff --git a/include/dt-bindings/power/meson-gxbb-power.h b/include/dt-bindings/power/meson-gxbb-power.h
-index 1262dac696c0..8d0b32b6c02c 100644
--- a/include/dt-bindings/power/meson-gxbb-power.h
-+++ b/include/dt-bindings/power/meson-gxbb-power.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (c) 2019 BayLibre, SAS
-  * Author: Neil Armstrong <narmstrong@baylibre.com>
-diff --git a/include/dt-bindings/power/meson-s4-power.h b/include/dt-bindings/power/meson-s4-power.h
-index 462dd2cb938b..f210a524a592 100644
--- a/include/dt-bindings/power/meson-s4-power.h
-+++ b/include/dt-bindings/power/meson-s4-power.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (c) 2021 Amlogic, Inc.
-  * Author: Shunzhou Jiang <shunzhou.jiang@amlogic.com>
-diff --git a/include/dt-bindings/power/meson-sm1-power.h b/include/dt-bindings/power/meson-sm1-power.h
-index a020ab00c134..d78e710dbfff 100644
--- a/include/dt-bindings/power/meson-sm1-power.h
-+++ b/include/dt-bindings/power/meson-sm1-power.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (c) 2019 BayLibre, SAS
-  * Author: Neil Armstrong <narmstrong@baylibre.com>
-diff --git a/include/dt-bindings/power/meson8-power.h b/include/dt-bindings/power/meson8-power.h
-index dd8b2ddb82a7..7a55ba2cd22e 100644
--- a/include/dt-bindings/power/meson8-power.h
-+++ b/include/dt-bindings/power/meson8-power.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (c) 2019 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
-  */
-diff --git a/include/dt-bindings/power/nvidia,tegra264-bpmp.h b/include/dt-bindings/power/nvidia,tegra264-bpmp.h
-new file mode 100644
-index 000000000000..2eef4a2a02b0
--- /dev/null
-+++ b/include/dt-bindings/power/nvidia,tegra264-bpmp.h
-@@ -0,0 +1,24 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)  */
-+/* Copyright (c) 2022-2024, NVIDIA CORPORATION.  All rights reserved. */
-+
-+#ifndef DT_BINDINGS_POWER_NVIDIA_TEGRA264_BPMP_H
-+#define DT_BINDINGS_POWER_NVIDIA_TEGRA264_BPMP_H
-+
-+#define TEGRA264_POWER_DOMAIN_DISP	1
-+#define TEGRA264_POWER_DOMAIN_AUD	2
-+/* reserved 3:9 */
-+#define TEGRA264_POWER_DOMAIN_XUSB_SS	10
-+#define TEGRA264_POWER_DOMAIN_XUSB_DEV	11
-+#define TEGRA264_POWER_DOMAIN_XUSB_HOST	12
-+#define TEGRA264_POWER_DOMAIN_MGBE0	13
-+#define TEGRA264_POWER_DOMAIN_MGBE1	14
-+#define TEGRA264_POWER_DOMAIN_MGBE2	15
-+#define TEGRA264_POWER_DOMAIN_MGBE3	16
-+#define TEGRA264_POWER_DOMAIN_VI	17
-+#define TEGRA264_POWER_DOMAIN_VIC	18
-+#define TEGRA264_POWER_DOMAIN_ISP0	19
-+#define TEGRA264_POWER_DOMAIN_ISP1	20
-+#define TEGRA264_POWER_DOMAIN_PVA0	21
-+#define TEGRA264_POWER_DOMAIN_GPU	22
-+
-+#endif /* DT_BINDINGS_POWER_NVIDIA_TEGRA264_BPMP_H */
-diff --git a/include/dt-bindings/power/qcom,rpmhpd.h b/include/dt-bindings/power/qcom,rpmhpd.h
-new file mode 100644
-index 000000000000..50e7c886709d
--- /dev/null
-+++ b/include/dt-bindings/power/qcom,rpmhpd.h
-@@ -0,0 +1,268 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_POWER_QCOM_RPMHPD_H
-+#define _DT_BINDINGS_POWER_QCOM_RPMHPD_H
-+
-+/* Generic RPMH Power Domain Indexes */
-+#define RPMHPD_CX               0
-+#define RPMHPD_CX_AO		1
-+#define RPMHPD_EBI		2
-+#define RPMHPD_GFX		3
-+#define RPMHPD_LCX		4
-+#define RPMHPD_LMX		5
-+#define RPMHPD_MMCX		6
-+#define RPMHPD_MMCX_AO		7
-+#define RPMHPD_MX		8
-+#define RPMHPD_MX_AO		9
-+#define RPMHPD_MXC		10
-+#define RPMHPD_MXC_AO		11
-+#define RPMHPD_MSS              12
-+#define RPMHPD_NSP		13
-+#define RPMHPD_NSP0             14
-+#define RPMHPD_NSP1             15
-+#define RPMHPD_QPHY             16
-+#define RPMHPD_DDR              17
-+#define RPMHPD_XO               18
-+#define RPMHPD_NSP2             19
-+#define RPMHPD_GMXC		20
-+
-+/* RPMh Power Domain performance levels */
-+#define RPMH_REGULATOR_LEVEL_RETENTION		16
-+#define RPMH_REGULATOR_LEVEL_MIN_SVS		48
-+#define RPMH_REGULATOR_LEVEL_LOW_SVS_D3		50
-+#define RPMH_REGULATOR_LEVEL_LOW_SVS_D2_1	51
-+#define RPMH_REGULATOR_LEVEL_LOW_SVS_D2		52
-+#define RPMH_REGULATOR_LEVEL_LOW_SVS_D1_1	54
-+#define RPMH_REGULATOR_LEVEL_LOW_SVS_D1		56
-+#define RPMH_REGULATOR_LEVEL_LOW_SVS_D0		60
-+#define RPMH_REGULATOR_LEVEL_LOW_SVS		64
-+#define RPMH_REGULATOR_LEVEL_LOW_SVS_P1		72
-+#define RPMH_REGULATOR_LEVEL_LOW_SVS_L0		76
-+#define RPMH_REGULATOR_LEVEL_LOW_SVS_L1		80
-+#define RPMH_REGULATOR_LEVEL_LOW_SVS_L2		96
-+#define RPMH_REGULATOR_LEVEL_SVS		128
-+#define RPMH_REGULATOR_LEVEL_SVS_L0		144
-+#define RPMH_REGULATOR_LEVEL_SVS_L1		192
-+#define RPMH_REGULATOR_LEVEL_SVS_L2		224
-+#define RPMH_REGULATOR_LEVEL_NOM		256
-+#define RPMH_REGULATOR_LEVEL_NOM_L0		288
-+#define RPMH_REGULATOR_LEVEL_NOM_L1		320
-+#define RPMH_REGULATOR_LEVEL_NOM_L2		336
-+#define RPMH_REGULATOR_LEVEL_TURBO		384
-+#define RPMH_REGULATOR_LEVEL_TURBO_L0		400
-+#define RPMH_REGULATOR_LEVEL_TURBO_L1		416
-+#define RPMH_REGULATOR_LEVEL_TURBO_L2		432
-+#define RPMH_REGULATOR_LEVEL_TURBO_L3		448
-+#define RPMH_REGULATOR_LEVEL_TURBO_L4		452
-+#define RPMH_REGULATOR_LEVEL_TURBO_L5		456
-+#define RPMH_REGULATOR_LEVEL_SUPER_TURBO	464
-+#define RPMH_REGULATOR_LEVEL_SUPER_TURBO_NO_CPR	480
-+
-+/*
-+ * Platform-specific power domain bindings. Don't add new entries here, use
-+ * RPMHPD_* above.
-+ */
-+
-+/* SA8775P Power Domain Indexes */
-+#define SA8775P_CX	0
-+#define SA8775P_CX_AO	1
-+#define SA8775P_DDR	2
-+#define SA8775P_EBI	3
-+#define SA8775P_GFX	4
-+#define SA8775P_LCX	5
-+#define SA8775P_LMX	6
-+#define SA8775P_MMCX	7
-+#define SA8775P_MMCX_AO	8
-+#define SA8775P_MSS	9
-+#define SA8775P_MX	10
-+#define SA8775P_MX_AO	11
-+#define SA8775P_MXC	12
-+#define SA8775P_MXC_AO	13
-+#define SA8775P_NSP0	14
-+#define SA8775P_NSP1	15
-+#define SA8775P_XO	16
-+
-+/* SDM670 Power Domain Indexes */
-+#define SDM670_MX	0
-+#define SDM670_MX_AO	1
-+#define SDM670_CX	2
-+#define SDM670_CX_AO	3
-+#define SDM670_LMX	4
-+#define SDM670_LCX	5
-+#define SDM670_GFX	6
-+#define SDM670_MSS	7
-+
-+/* SDM845 Power Domain Indexes */
-+#define SDM845_EBI	0
-+#define SDM845_MX	1
-+#define SDM845_MX_AO	2
-+#define SDM845_CX	3
-+#define SDM845_CX_AO	4
-+#define SDM845_LMX	5
-+#define SDM845_LCX	6
-+#define SDM845_GFX	7
-+#define SDM845_MSS	8
-+
-+/* SDX55 Power Domain Indexes */
-+#define SDX55_MSS	0
-+#define SDX55_MX	1
-+#define SDX55_CX	2
-+
-+/* SDX65 Power Domain Indexes */
-+#define SDX65_MSS	0
-+#define SDX65_MX	1
-+#define SDX65_MX_AO	2
-+#define SDX65_CX	3
-+#define SDX65_CX_AO	4
-+#define SDX65_MXC	5
-+
-+/* SM6350 Power Domain Indexes */
-+#define SM6350_CX	0
-+#define SM6350_GFX	1
-+#define SM6350_LCX	2
-+#define SM6350_LMX	3
-+#define SM6350_MSS	4
-+#define SM6350_MX	5
-+
-+/* SM8150 Power Domain Indexes */
-+#define SM8150_MSS	0
-+#define SM8150_EBI	1
-+#define SM8150_LMX	2
-+#define SM8150_LCX	3
-+#define SM8150_GFX	4
-+#define SM8150_MX	5
-+#define SM8150_MX_AO	6
-+#define SM8150_CX	7
-+#define SM8150_CX_AO	8
-+#define SM8150_MMCX	9
-+#define SM8150_MMCX_AO	10
-+
-+/* SA8155P is a special case, kept for backwards compatibility */
-+#define SA8155P_CX	SM8150_CX
-+#define SA8155P_CX_AO	SM8150_CX_AO
-+#define SA8155P_EBI	SM8150_EBI
-+#define SA8155P_GFX	SM8150_GFX
-+#define SA8155P_MSS	SM8150_MSS
-+#define SA8155P_MX	SM8150_MX
-+#define SA8155P_MX_AO	SM8150_MX_AO
-+
-+/* SM8250 Power Domain Indexes */
-+#define SM8250_CX	0
-+#define SM8250_CX_AO	1
-+#define SM8250_EBI	2
-+#define SM8250_GFX	3
-+#define SM8250_LCX	4
-+#define SM8250_LMX	5
-+#define SM8250_MMCX	6
-+#define SM8250_MMCX_AO	7
-+#define SM8250_MX	8
-+#define SM8250_MX_AO	9
-+
-+/* SM8350 Power Domain Indexes */
-+#define SM8350_CX	0
-+#define SM8350_CX_AO	1
-+#define SM8350_EBI	2
-+#define SM8350_GFX	3
-+#define SM8350_LCX	4
-+#define SM8350_LMX	5
-+#define SM8350_MMCX	6
-+#define SM8350_MMCX_AO	7
-+#define SM8350_MX	8
-+#define SM8350_MX_AO	9
-+#define SM8350_MXC	10
-+#define SM8350_MXC_AO	11
-+#define SM8350_MSS	12
-+
-+/* SM8450 Power Domain Indexes */
-+#define SM8450_CX	0
-+#define SM8450_CX_AO	1
-+#define SM8450_EBI	2
-+#define SM8450_GFX	3
-+#define SM8450_LCX	4
-+#define SM8450_LMX	5
-+#define SM8450_MMCX	6
-+#define SM8450_MMCX_AO	7
-+#define SM8450_MX	8
-+#define SM8450_MX_AO	9
-+#define SM8450_MXC	10
-+#define SM8450_MXC_AO	11
-+#define SM8450_MSS	12
-+
-+/* SM8550 Power Domain Indexes */
-+#define SM8550_CX	0
-+#define SM8550_CX_AO	1
-+#define SM8550_EBI	2
-+#define SM8550_GFX	3
-+#define SM8550_LCX	4
-+#define SM8550_LMX	5
-+#define SM8550_MMCX	6
-+#define SM8550_MMCX_AO	7
-+#define SM8550_MX	8
-+#define SM8550_MX_AO	9
-+#define SM8550_MXC	10
-+#define SM8550_MXC_AO	11
-+#define SM8550_MSS	12
-+#define SM8550_NSP	13
-+
-+/* QDU1000/QRU1000 Power Domain Indexes */
-+#define QDU1000_EBI	0
-+#define QDU1000_MSS	1
-+#define QDU1000_CX	2
-+#define QDU1000_MX	3
-+
-+/* SC7180 Power Domain Indexes */
-+#define SC7180_CX	0
-+#define SC7180_CX_AO	1
-+#define SC7180_GFX	2
-+#define SC7180_MX	3
-+#define SC7180_MX_AO	4
-+#define SC7180_LMX	5
-+#define SC7180_LCX	6
-+#define SC7180_MSS	7
-+
-+/* SC7280 Power Domain Indexes */
-+#define SC7280_CX	0
-+#define SC7280_CX_AO	1
-+#define SC7280_EBI	2
-+#define SC7280_GFX	3
-+#define SC7280_MX	4
-+#define SC7280_MX_AO	5
-+#define SC7280_LMX	6
-+#define SC7280_LCX	7
-+#define SC7280_MSS	8
-+
-+/* SC8180X Power Domain Indexes */
-+#define SC8180X_CX	0
-+#define SC8180X_CX_AO	1
-+#define SC8180X_EBI	2
-+#define SC8180X_GFX	3
-+#define SC8180X_LCX	4
-+#define SC8180X_LMX	5
-+#define SC8180X_MMCX	6
-+#define SC8180X_MMCX_AO	7
-+#define SC8180X_MSS	8
-+#define SC8180X_MX	9
-+#define SC8180X_MX_AO	10
-+
-+/* SC8280XP Power Domain Indexes */
-+#define SC8280XP_CX		0
-+#define SC8280XP_CX_AO		1
-+#define SC8280XP_DDR		2
-+#define SC8280XP_EBI		3
-+#define SC8280XP_GFX		4
-+#define SC8280XP_LCX		5
-+#define SC8280XP_LMX		6
-+#define SC8280XP_MMCX		7
-+#define SC8280XP_MMCX_AO	8
-+#define SC8280XP_MSS		9
-+#define SC8280XP_MX		10
-+#define SC8280XP_MXC		12
-+#define SC8280XP_MX_AO		11
-+#define SC8280XP_NSP		13
-+#define SC8280XP_QPHY		14
-+#define SC8280XP_XO		15
-+
-+#endif
-diff --git a/include/dt-bindings/power/qcom-rpmpd.h b/include/dt-bindings/power/qcom-rpmpd.h
-index d81de63ae31c..4371ac941f29 100644
--- a/include/dt-bindings/power/qcom-rpmpd.h
-+++ b/include/dt-bindings/power/qcom-rpmpd.h
-@@ -4,171 +4,39 @@
- #ifndef _DT_BINDINGS_POWER_QCOM_RPMPD_H
- #define _DT_BINDINGS_POWER_QCOM_RPMPD_H
- 
-/* SDM845 Power Domain Indexes */
-#define SDM845_EBI	0
-#define SDM845_MX	1
-#define SDM845_MX_AO	2
-#define SDM845_CX	3
-#define SDM845_CX_AO	4
-#define SDM845_LMX	5
-#define SDM845_LCX	6
-#define SDM845_GFX	7
-#define SDM845_MSS	8
-
-/* SDX55 Power Domain Indexes */
-#define SDX55_MSS	0
-#define SDX55_MX	1
-#define SDX55_CX	2
-
-/* SDX65 Power Domain Indexes */
-#define SDX65_MSS	0
-#define SDX65_MX	1
-#define SDX65_MX_AO	2
-#define SDX65_CX	3
-#define SDX65_CX_AO	4
-#define SDX65_MXC	5
-
-/* SM6350 Power Domain Indexes */
-#define SM6350_CX	0
-#define SM6350_GFX	1
-#define SM6350_LCX	2
-#define SM6350_LMX	3
-#define SM6350_MSS	4
-#define SM6350_MX	5
-
-/* SM8150 Power Domain Indexes */
-#define SM8150_MSS	0
-#define SM8150_EBI	1
-#define SM8150_LMX	2
-#define SM8150_LCX	3
-#define SM8150_GFX	4
-#define SM8150_MX	5
-#define SM8150_MX_AO	6
-#define SM8150_CX	7
-#define SM8150_CX_AO	8
-#define SM8150_MMCX	9
-#define SM8150_MMCX_AO	10
-
-/* SM8250 Power Domain Indexes */
-#define SM8250_CX	0
-#define SM8250_CX_AO	1
-#define SM8250_EBI	2
-#define SM8250_GFX	3
-#define SM8250_LCX	4
-#define SM8250_LMX	5
-#define SM8250_MMCX	6
-#define SM8250_MMCX_AO	7
-#define SM8250_MX	8
-#define SM8250_MX_AO	9
-
-/* SM8350 Power Domain Indexes */
-#define SM8350_CX	0
-#define SM8350_CX_AO	1
-#define SM8350_EBI	2
-#define SM8350_GFX	3
-#define SM8350_LCX	4
-#define SM8350_LMX	5
-#define SM8350_MMCX	6
-#define SM8350_MMCX_AO	7
-#define SM8350_MX	8
-#define SM8350_MX_AO	9
-#define SM8350_MXC	10
-#define SM8350_MXC_AO	11
-#define SM8350_MSS	12
-
-/* SM8450 Power Domain Indexes */
-#define SM8450_CX	0
-#define SM8450_CX_AO	1
-#define SM8450_EBI	2
-#define SM8450_GFX	3
-#define SM8450_LCX	4
-#define SM8450_LMX	5
-#define SM8450_MMCX	6
-#define SM8450_MMCX_AO	7
-#define SM8450_MX	8
-#define SM8450_MX_AO	9
-#define SM8450_MXC	10
-#define SM8450_MXC_AO	11
-#define SM8450_MSS	12
-
-/* SC7180 Power Domain Indexes */
-#define SC7180_CX	0
-#define SC7180_CX_AO	1
-#define SC7180_GFX	2
-#define SC7180_MX	3
-#define SC7180_MX_AO	4
-#define SC7180_LMX	5
-#define SC7180_LCX	6
-#define SC7180_MSS	7
-
-/* SC7280 Power Domain Indexes */
-#define SC7280_CX	0
-#define SC7280_CX_AO	1
-#define SC7280_EBI	2
-#define SC7280_GFX	3
-#define SC7280_MX	4
-#define SC7280_MX_AO	5
-#define SC7280_LMX	6
-#define SC7280_LCX	7
-#define SC7280_MSS	8
-
-/* SC8180X Power Domain Indexes */
-#define SC8180X_CX	0
-#define SC8180X_CX_AO	1
-#define SC8180X_EBI	2
-#define SC8180X_GFX	3
-#define SC8180X_LCX	4
-#define SC8180X_LMX	5
-#define SC8180X_MMCX	6
-#define SC8180X_MMCX_AO	7
-#define SC8180X_MSS	8
-#define SC8180X_MX	9
-#define SC8180X_MX_AO	10
-
-/* SC8280XP Power Domain Indexes */
-#define SC8280XP_CX		0
-#define SC8280XP_CX_AO		1
-#define SC8280XP_DDR		2
-#define SC8280XP_EBI		3
-#define SC8280XP_GFX		4
-#define SC8280XP_LCX		5
-#define SC8280XP_LMX		6
-#define SC8280XP_MMCX		7
-#define SC8280XP_MMCX_AO	8
-#define SC8280XP_MSS		9
-#define SC8280XP_MX		10
-#define SC8280XP_MXC		12
-#define SC8280XP_MX_AO		11
-#define SC8280XP_NSP		13
-#define SC8280XP_QPHY		14
-#define SC8280XP_XO		15
-
-/* SDM845 Power Domain performance levels */
-#define RPMH_REGULATOR_LEVEL_RETENTION	16
-#define RPMH_REGULATOR_LEVEL_MIN_SVS	48
-#define RPMH_REGULATOR_LEVEL_LOW_SVS	64
-#define RPMH_REGULATOR_LEVEL_SVS	128
-#define RPMH_REGULATOR_LEVEL_SVS_L0	144
-#define RPMH_REGULATOR_LEVEL_SVS_L1	192
-#define RPMH_REGULATOR_LEVEL_SVS_L2	224
-#define RPMH_REGULATOR_LEVEL_NOM	256
-#define RPMH_REGULATOR_LEVEL_NOM_L1	320
-#define RPMH_REGULATOR_LEVEL_NOM_L2	336
-#define RPMH_REGULATOR_LEVEL_TURBO	384
-#define RPMH_REGULATOR_LEVEL_TURBO_L1	416
-+#include <dt-bindings/power/qcom,rpmhpd.h>
-+
-+/* Generic RPM Power Domain Indexes */
-+#define RPMPD_VDDCX		0
-+#define RPMPD_VDDCX_AO		1
-+/* VFC and VFL are mutually exclusive and can not be present on the same platform */
-+#define RPMPD_VDDCX_VFC		2
-+#define RPMPD_VDDCX_VFL		2
-+#define RPMPD_VDDMX		3
-+#define RPMPD_VDDMX_AO		4
-+#define RPMPD_VDDMX_VFL		5
-+#define RPMPD_SSCCX		6
-+#define RPMPD_SSCCX_VFL		7
-+#define RPMPD_SSCMX		8
-+#define RPMPD_SSCMX_VFL		9
-+
-+/*
-+ * Platform-specific power domain bindings. Don't add new entries here, use
-+ * RPMPD_* above.
-+ */
- 
- /* MDM9607 Power Domains */
-#define MDM9607_VDDCX		0
-#define MDM9607_VDDCX_AO	1
-#define MDM9607_VDDCX_VFL	2
-#define MDM9607_VDDMX		3
-#define MDM9607_VDDMX_AO	4
-#define MDM9607_VDDMX_VFL	5
-+#define MDM9607_VDDCX		RPMPD_VDDCX
-+#define MDM9607_VDDCX_AO	RPMPD_VDDCX_AO
-+#define MDM9607_VDDCX_VFL	RPMPD_VDDCX_VFL
-+#define MDM9607_VDDMX		RPMPD_VDDMX
-+#define MDM9607_VDDMX_AO	RPMPD_VDDMX_AO
-+#define MDM9607_VDDMX_VFL	RPMPD_VDDMX_VFL
- 
- /* MSM8226 Power Domain Indexes */
-#define MSM8226_VDDCX		0
-#define MSM8226_VDDCX_AO	1
-#define MSM8226_VDDCX_VFC	2
-+#define MSM8226_VDDCX		RPMPD_VDDCX
-+#define MSM8226_VDDCX_AO	RPMPD_VDDCX_AO
-+#define MSM8226_VDDCX_VFC	RPMPD_VDDCX_VFC
- 
- /* MSM8939 Power Domains */
- #define MSM8939_VDDMDCX		0
-@@ -181,11 +49,11 @@
- #define MSM8939_VDDMX_AO	7
- 
- /* MSM8916 Power Domain Indexes */
-#define MSM8916_VDDCX		0
-#define MSM8916_VDDCX_AO	1
-#define MSM8916_VDDCX_VFC	2
-#define MSM8916_VDDMX		3
-#define MSM8916_VDDMX_AO	4
-+#define MSM8916_VDDCX		RPMPD_VDDCX
-+#define MSM8916_VDDCX_AO	RPMPD_VDDCX_AO
-+#define MSM8916_VDDCX_VFC	RPMPD_VDDCX_VFC
-+#define MSM8916_VDDMX		RPMPD_VDDMX
-+#define MSM8916_VDDMX_AO	RPMPD_VDDMX_AO
- 
- /* MSM8909 Power Domain Indexes */
- #define MSM8909_VDDCX		MSM8916_VDDCX
-@@ -194,6 +62,27 @@
- #define MSM8909_VDDMX		MSM8916_VDDMX
- #define MSM8909_VDDMX_AO	MSM8916_VDDMX_AO
- 
-+/* MSM8917 Power Domain Indexes */
-+#define MSM8917_VDDCX		RPMPD_VDDCX
-+#define MSM8917_VDDCX_AO	RPMPD_VDDCX_AO
-+#define MSM8917_VDDCX_VFL	RPMPD_VDDCX_VFL
-+#define MSM8917_VDDMX		RPMPD_VDDMX
-+#define MSM8917_VDDMX_AO	RPMPD_VDDMX_AO
-+
-+/* MSM8937 Power Domain Indexes */
-+#define MSM8937_VDDCX		MSM8917_VDDCX
-+#define MSM8937_VDDCX_AO	MSM8917_VDDCX_AO
-+#define MSM8937_VDDCX_VFL	MSM8917_VDDCX_VFL
-+#define MSM8937_VDDMX		MSM8917_VDDMX
-+#define MSM8937_VDDMX_AO	MSM8917_VDDMX_AO
-+
-+/* QM215 Power Domain Indexes */
-+#define QM215_VDDCX		MSM8917_VDDCX
-+#define QM215_VDDCX_AO		MSM8917_VDDCX_AO
-+#define QM215_VDDCX_VFL		MSM8917_VDDCX_VFL
-+#define QM215_VDDMX		MSM8917_VDDMX
-+#define QM215_VDDMX_AO		MSM8917_VDDMX_AO
-+
- /* MSM8953 Power Domain Indexes */
- #define MSM8953_VDDMD		0
- #define MSM8953_VDDMD_AO	1
-@@ -203,13 +92,20 @@
- #define MSM8953_VDDMX		5
- #define MSM8953_VDDMX_AO	6
- 
-+/* MSM8974 Power Domain Indexes */
-+#define MSM8974_VDDCX		0
-+#define MSM8974_VDDCX_AO	1
-+#define MSM8974_VDDCX_VFC	2
-+#define MSM8974_VDDGFX		3
-+#define MSM8974_VDDGFX_VFC	4
-+
- /* MSM8976 Power Domain Indexes */
-#define MSM8976_VDDCX		0
-#define MSM8976_VDDCX_AO	1
-#define MSM8976_VDDCX_VFL	2
-#define MSM8976_VDDMX		3
-#define MSM8976_VDDMX_AO	4
-#define MSM8976_VDDMX_VFL	5
-+#define MSM8976_VDDCX		RPMPD_VDDCX
-+#define MSM8976_VDDCX_AO	RPMPD_VDDCX_AO
-+#define MSM8976_VDDCX_VFL	RPMPD_VDDCX_VFL
-+#define MSM8976_VDDMX		RPMPD_VDDMX
-+#define MSM8976_VDDMX_AO	RPMPD_VDDMX_AO
-+#define MSM8976_VDDMX_VFL	RPMPD_VDDMX_VFL
- 
- /* MSM8994 Power Domain Indexes */
- #define MSM8994_VDDCX		0
-@@ -230,16 +126,26 @@
- #define MSM8996_VDDSSCX_VFC	6
- 
- /* MSM8998 Power Domain Indexes */
-#define MSM8998_VDDCX		0
-#define MSM8998_VDDCX_AO	1
-#define MSM8998_VDDCX_VFL	2
-#define MSM8998_VDDMX		3
-#define MSM8998_VDDMX_AO	4
-#define MSM8998_VDDMX_VFL	5
-#define MSM8998_SSCCX		6
-#define MSM8998_SSCCX_VFL	7
-#define MSM8998_SSCMX		8
-#define MSM8998_SSCMX_VFL	9
-+#define MSM8998_VDDCX		RPMPD_VDDCX
-+#define MSM8998_VDDCX_AO	RPMPD_VDDCX_AO
-+#define MSM8998_VDDCX_VFL	RPMPD_VDDCX_VFL
-+#define MSM8998_VDDMX		RPMPD_VDDMX
-+#define MSM8998_VDDMX_AO	RPMPD_VDDMX_AO
-+#define MSM8998_VDDMX_VFL	RPMPD_VDDMX_VFL
-+#define MSM8998_SSCCX		RPMPD_SSCCX
-+#define MSM8998_SSCCX_VFL	RPMPD_SSCCX_VFL
-+#define MSM8998_SSCMX		RPMPD_SSCMX
-+#define MSM8998_SSCMX_VFL	RPMPD_SSCMX_VFL
-+
-+/* QCM2290 Power Domains */
-+#define QCM2290_VDDCX		0
-+#define QCM2290_VDDCX_AO	1
-+#define QCM2290_VDDCX_VFL	2
-+#define QCM2290_VDDMX		3
-+#define QCM2290_VDDMX_AO	4
-+#define QCM2290_VDDMX_VFL	5
-+#define QCM2290_VDD_LPI_CX	6
-+#define QCM2290_VDD_LPI_MX	7
- 
- /* QCS404 Power Domains */
- #define QCS404_VDDMX		0
-@@ -251,16 +157,16 @@
- #define QCS404_LPIMX_VFL	6
- 
- /* SDM660 Power Domains */
-#define SDM660_VDDCX		0
-#define SDM660_VDDCX_AO		1
-#define SDM660_VDDCX_VFL	2
-#define SDM660_VDDMX		3
-#define SDM660_VDDMX_AO		4
-#define SDM660_VDDMX_VFL	5
-#define SDM660_SSCCX		6
-#define SDM660_SSCCX_VFL	7
-#define SDM660_SSCMX		8
-#define SDM660_SSCMX_VFL	9
-+#define SDM660_VDDCX		RPMPD_VDDCX
-+#define SDM660_VDDCX_AO		RPMPD_VDDCX_AO
-+#define SDM660_VDDCX_VFL	RPMPD_VDDCX_VFL
-+#define SDM660_VDDMX		RPMPD_VDDMX
-+#define SDM660_VDDMX_AO		RPMPD_VDDMX_AO
-+#define SDM660_VDDMX_VFL	RPMPD_VDDMX_VFL
-+#define SDM660_SSCCX		RPMPD_SSCCX
-+#define SDM660_SSCCX_VFL	RPMPD_SSCCX_VFL
-+#define SDM660_SSCMX		RPMPD_SSCMX
-+#define SDM660_SSCMX_VFL	RPMPD_SSCMX_VFL
- 
- /* SM6115 Power Domains */
- #define SM6115_VDDCX		0
-@@ -273,22 +179,24 @@
- #define SM6115_VDD_LPI_MX	7
- 
- /* SM6125 Power Domains */
-#define SM6125_VDDCX		0
-#define SM6125_VDDCX_AO		1
-#define SM6125_VDDCX_VFL	2
-#define SM6125_VDDMX		3
-#define SM6125_VDDMX_AO		4
-#define SM6125_VDDMX_VFL	5
-
-/* QCM2290 Power Domains */
-#define QCM2290_VDDCX		0
-#define QCM2290_VDDCX_AO	1
-#define QCM2290_VDDCX_VFL	2
-#define QCM2290_VDDMX		3
-#define QCM2290_VDDMX_AO	4
-#define QCM2290_VDDMX_VFL	5
-#define QCM2290_VDD_LPI_CX	6
-#define QCM2290_VDD_LPI_MX	7
-+#define SM6125_VDDCX		RPMPD_VDDCX
-+#define SM6125_VDDCX_AO		RPMPD_VDDCX_AO
-+#define SM6125_VDDCX_VFL	RPMPD_VDDCX_VFL
-+#define SM6125_VDDMX		RPMPD_VDDMX
-+#define SM6125_VDDMX_AO		RPMPD_VDDMX_AO
-+#define SM6125_VDDMX_VFL	RPMPD_VDDMX_VFL
-+
-+/* SM6375 Power Domain Indexes */
-+#define SM6375_VDDCX		0
-+#define SM6375_VDDCX_AO	1
-+#define SM6375_VDDCX_VFL	2
-+#define SM6375_VDDMX		3
-+#define SM6375_VDDMX_AO	4
-+#define SM6375_VDDMX_VFL	5
-+#define SM6375_VDDGX		6
-+#define SM6375_VDDGX_AO	7
-+#define SM6375_VDD_LPI_CX	8
-+#define SM6375_VDD_LPI_MX	9
- 
- /* RPM SMD Power Domain performance levels */
- #define RPM_SMD_LEVEL_RETENTION       16
-diff --git a/include/dt-bindings/power/r8a7795-sysc.h b/include/dt-bindings/power/r8a7795-sysc.h
-index eea6ad69f0b0..ff5323858572 100644
--- a/include/dt-bindings/power/r8a7795-sysc.h
-+++ b/include/dt-bindings/power/r8a7795-sysc.h
-@@ -30,7 +30,6 @@
- #define R8A7795_PD_CA53_SCU		21
- #define R8A7795_PD_3DG_E		22
- #define R8A7795_PD_A3IR			24
-#define R8A7795_PD_A2VC0		25	/* ES1.x only */
- #define R8A7795_PD_A2VC1		26
- 
- /* Always-on power area */
-diff --git a/include/dt-bindings/power/r8a779f0-sysc.h b/include/dt-bindings/power/r8a779f0-sysc.h
-index 0ec8ad727ed9..cde1536e9ed0 100644
--- a/include/dt-bindings/power/r8a779f0-sysc.h
-+++ b/include/dt-bindings/power/r8a779f0-sysc.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0 or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
- /*
-  * Copyright (C) 2021 Renesas Electronics Corp.
-  */
-diff --git a/include/dt-bindings/power/r8a779g0-sysc.h b/include/dt-bindings/power/r8a779g0-sysc.h
-index 7daa70f1814e..c7b139fb075f 100644
--- a/include/dt-bindings/power/r8a779g0-sysc.h
-+++ b/include/dt-bindings/power/r8a779g0-sysc.h
-@@ -38,6 +38,7 @@
- #define R8A779G0_PD_A3VIP2		58
- #define R8A779G0_PD_A3ISP0		60
- #define R8A779G0_PD_A3ISP1		61
-+#define R8A779G0_PD_A3DUL		62
- 
- /* Always-on power area */
- #define R8A779G0_PD_ALWAYS_ON		64
-diff --git a/include/dt-bindings/power/renesas,r8a779h0-sysc.h b/include/dt-bindings/power/renesas,r8a779h0-sysc.h
-new file mode 100644
-index 000000000000..f27976f523e8
--- /dev/null
-+++ b/include/dt-bindings/power/renesas,r8a779h0-sysc.h
-@@ -0,0 +1,49 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2023 Renesas Electronics Corp.
-+ */
-+#ifndef __DT_BINDINGS_POWER_RENESAS_R8A779H0_SYSC_H__
-+#define __DT_BINDINGS_POWER_RENESAS_R8A779H0_SYSC_H__
-+
-+/*
-+ * These power domain indices match the Power Domain Register Numbers (PDR)
-+ */
-+
-+#define R8A779H0_PD_A1E0D0C0		0
-+#define R8A779H0_PD_A1E0D0C1		1
-+#define R8A779H0_PD_A1E0D0C2		2
-+#define R8A779H0_PD_A1E0D0C3		3
-+#define R8A779H0_PD_A2E0D0		16
-+#define R8A779H0_PD_A3CR0		21
-+#define R8A779H0_PD_A3CR1		22
-+#define R8A779H0_PD_A3CR2		23
-+#define R8A779H0_PD_A33DGA		24
-+#define R8A779H0_PD_A23DGB		25
-+#define R8A779H0_PD_C4			31
-+#define R8A779H0_PD_A1DSP0		33
-+#define R8A779H0_PD_A2IMP01		34
-+#define R8A779H0_PD_A2PSC		35
-+#define R8A779H0_PD_A2CV0		36
-+#define R8A779H0_PD_A2CV1		37
-+#define R8A779H0_PD_A3IMR0		38
-+#define R8A779H0_PD_A3IMR1		39
-+#define R8A779H0_PD_A3VC		40
-+#define R8A779H0_PD_A2CN0		42
-+#define R8A779H0_PD_A1CN0		44
-+#define R8A779H0_PD_A1DSP1		45
-+#define R8A779H0_PD_A2DMA		47
-+#define R8A779H0_PD_A2CV2		48
-+#define R8A779H0_PD_A2CV3		49
-+#define R8A779H0_PD_A3IMR2		50
-+#define R8A779H0_PD_A3IMR3		51
-+#define R8A779H0_PD_A3PCI		52
-+#define R8A779H0_PD_A2PCIPHY		53
-+#define R8A779H0_PD_A3VIP0		56
-+#define R8A779H0_PD_A3VIP2		58
-+#define R8A779H0_PD_A3ISP0		60
-+#define R8A779H0_PD_A3DUL		62
-+
-+/* Always-on power area */
-+#define R8A779H0_PD_ALWAYS_ON		64
-+
-+#endif /* __DT_BINDINGS_POWER_RENESAS_R8A779H0_SYSC_H__ */
-diff --git a/include/dt-bindings/power/rk3588-power.h b/include/dt-bindings/power/rk3588-power.h
-new file mode 100644
-index 000000000000..6b91a50cc6d6
--- /dev/null
-+++ b/include/dt-bindings/power/rk3588-power.h
-@@ -0,0 +1,69 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
-+#ifndef __DT_BINDINGS_POWER_RK3588_POWER_H__
-+#define __DT_BINDINGS_POWER_RK3588_POWER_H__
-+
-+/* VD_LITDSU */
-+#define RK3588_PD_CPU_0		0
-+#define RK3588_PD_CPU_1		1
-+#define RK3588_PD_CPU_2		2
-+#define RK3588_PD_CPU_3		3
-+
-+/* VD_BIGCORE0 */
-+#define RK3588_PD_CPU_4		4
-+#define RK3588_PD_CPU_5		5
-+
-+/* VD_BIGCORE1 */
-+#define RK3588_PD_CPU_6		6
-+#define RK3588_PD_CPU_7		7
-+
-+/* VD_NPU */
-+#define RK3588_PD_NPU		8
-+#define RK3588_PD_NPUTOP	9
-+#define RK3588_PD_NPU1		10
-+#define RK3588_PD_NPU2		11
-+
-+/* VD_GPU */
-+#define RK3588_PD_GPU		12
-+
-+/* VD_VCODEC */
-+#define RK3588_PD_VCODEC	13
-+#define RK3588_PD_RKVDEC0	14
-+#define RK3588_PD_RKVDEC1	15
-+#define RK3588_PD_VENC0		16
-+#define RK3588_PD_VENC1		17
-+
-+/* VD_DD01 */
-+#define RK3588_PD_DDR01		18
-+
-+/* VD_DD23 */
-+#define RK3588_PD_DDR23		19
-+
-+/* VD_LOGIC */
-+#define RK3588_PD_CENTER	20
-+#define RK3588_PD_VDPU		21
-+#define RK3588_PD_RGA30		22
-+#define RK3588_PD_AV1		23
-+#define RK3588_PD_VOP		24
-+#define RK3588_PD_VO0		25
-+#define RK3588_PD_VO1		26
-+#define RK3588_PD_VI		27
-+#define RK3588_PD_ISP1		28
-+#define RK3588_PD_FEC		29
-+#define RK3588_PD_RGA31		30
-+#define RK3588_PD_USB		31
-+#define RK3588_PD_PHP		32
-+#define RK3588_PD_GMAC		33
-+#define RK3588_PD_PCIE		34
-+#define RK3588_PD_NVM		35
-+#define RK3588_PD_NVM0		36
-+#define RK3588_PD_SDIO		37
-+#define RK3588_PD_AUDIO		38
-+#define RK3588_PD_SECURE	39
-+#define RK3588_PD_SDMMC		40
-+#define RK3588_PD_CRYPTO	41
-+#define RK3588_PD_BUS		42
-+
-+/* VD_PMU */
-+#define RK3588_PD_PMU1		43
-+
-+#endif
-diff --git a/include/dt-bindings/power/rockchip,rk3528-power.h b/include/dt-bindings/power/rockchip,rk3528-power.h
-new file mode 100644
-index 000000000000..318923cdaaf6
--- /dev/null
-+++ b/include/dt-bindings/power/rockchip,rk3528-power.h
-@@ -0,0 +1,19 @@
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
-+#ifndef __DT_BINDINGS_POWER_RK3528_POWER_H__
-+#define __DT_BINDINGS_POWER_RK3528_POWER_H__
-+
-+#define RK3528_PD_PMU		0
-+#define RK3528_PD_BUS		1
-+#define RK3528_PD_DDR		2
-+#define RK3528_PD_MSCH		3
-+
-+/* VD_GPU */
-+#define RK3528_PD_GPU		4
-+
-+/* VD_LOGIC */
-+#define RK3528_PD_RKVDEC	5
-+#define RK3528_PD_RKVENC	6
-+#define RK3528_PD_VO		7
-+#define RK3528_PD_VPU		8
-+
-+#endif
-diff --git a/include/dt-bindings/power/rockchip,rk3562-power.h b/include/dt-bindings/power/rockchip,rk3562-power.h
-new file mode 100644
-index 000000000000..5182c2427a55
--- /dev/null
-+++ b/include/dt-bindings/power/rockchip,rk3562-power.h
-@@ -0,0 +1,35 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022-2024 Rockchip Electronics Co., Ltd.
-+ */
-+#ifndef __DT_BINDINGS_POWER_RK3562_POWER_H__
-+#define __DT_BINDINGS_POWER_RK3562_POWER_H__
-+
-+/* VD_CORE */
-+#define RK3562_PD_CPU_0		0
-+#define RK3562_PD_CPU_1		1
-+#define RK3562_PD_CPU_2		2
-+#define RK3562_PD_CPU_3		3
-+#define RK3562_PD_CORE_ALIVE	4
-+
-+/* VD_PMU */
-+#define RK3562_PD_PMU		5
-+#define RK3562_PD_PMU_ALIVE	6
-+
-+/* VD_NPU */
-+#define RK3562_PD_NPU		7
-+
-+/* VD_GPU */
-+#define RK3562_PD_GPU		8
-+
-+/* VD_LOGIC */
-+#define RK3562_PD_DDR		9
-+#define RK3562_PD_VEPU		10
-+#define RK3562_PD_VDPU		11
-+#define RK3562_PD_VI		12
-+#define RK3562_PD_VO		13
-+#define RK3562_PD_RGA		14
-+#define RK3562_PD_PHP		15
-+#define RK3562_PD_LOGIC_ALIVE	16
-+
-+#endif
-diff --git a/include/dt-bindings/power/rockchip,rk3576-power.h b/include/dt-bindings/power/rockchip,rk3576-power.h
-new file mode 100644
-index 000000000000..324a056aa851
--- /dev/null
-+++ b/include/dt-bindings/power/rockchip,rk3576-power.h
-@@ -0,0 +1,30 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
-+#ifndef __DT_BINDINGS_POWER_RK3576_POWER_H__
-+#define __DT_BINDINGS_POWER_RK3576_POWER_H__
-+
-+/* VD_NPU */
-+#define RK3576_PD_NPU		0
-+#define RK3576_PD_NPUTOP	1
-+#define RK3576_PD_NPU0		2
-+#define RK3576_PD_NPU1		3
-+
-+/* VD_GPU */
-+#define RK3576_PD_GPU		4
-+
-+/* VD_LOGIC */
-+#define RK3576_PD_NVM		5
-+#define RK3576_PD_SDGMAC	6
-+#define RK3576_PD_USB		7
-+#define RK3576_PD_PHP		8
-+#define RK3576_PD_SUBPHP	9
-+#define RK3576_PD_AUDIO		10
-+#define RK3576_PD_VEPU0		11
-+#define RK3576_PD_VEPU1		12
-+#define RK3576_PD_VPU		13
-+#define RK3576_PD_VDEC		14
-+#define RK3576_PD_VI		15
-+#define RK3576_PD_VO0		16
-+#define RK3576_PD_VO1		17
-+#define RK3576_PD_VOP		18
-+
-+#endif
-diff --git a/include/dt-bindings/power/rockchip,rv1126-power.h b/include/dt-bindings/power/rockchip,rv1126-power.h
-new file mode 100644
-index 000000000000..38a68e000d38
--- /dev/null
-+++ b/include/dt-bindings/power/rockchip,rv1126-power.h
-@@ -0,0 +1,35 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+
-+#ifndef __DT_BINDINGS_POWER_RV1126_POWER_H__
-+#define __DT_BINDINGS_POWER_RV1126_POWER_H__
-+
-+/* VD_CORE */
-+#define RV1126_PD_CPU_0		0
-+#define RV1126_PD_CPU_1		1
-+#define RV1126_PD_CPU_2		2
-+#define RV1126_PD_CPU_3		3
-+#define RV1126_PD_CORE_ALIVE	4
-+
-+/* VD_PMU */
-+#define RV1126_PD_PMU		5
-+#define RV1126_PD_PMU_ALIVE	6
-+
-+/* VD_NPU */
-+#define RV1126_PD_NPU		7
-+
-+/* VD_VEPU */
-+#define RV1126_PD_VEPU		8
-+
-+/* VD_LOGIC */
-+#define RV1126_PD_VI		9
-+#define RV1126_PD_VO		10
-+#define RV1126_PD_ISPP		11
-+#define RV1126_PD_VDPU		12
-+#define RV1126_PD_CRYPTO	13
-+#define RV1126_PD_DDR		14
-+#define RV1126_PD_NVM		15
-+#define RV1126_PD_SDIO		16
-+#define RV1126_PD_USB		17
-+#define RV1126_PD_LOGIC_ALIVE	18
-+
-+#endif
-diff --git a/include/dt-bindings/power/rockchip,rv1126b-power-controller.h b/include/dt-bindings/power/rockchip,rv1126b-power-controller.h
-new file mode 100644
-index 000000000000..48ea87a4423c
--- /dev/null
-+++ b/include/dt-bindings/power/rockchip,rv1126b-power-controller.h
-@@ -0,0 +1,17 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (c) 2024 Rockchip Electronics Co., Ltd.
-+ * Author: Finley Xiao <finley.xiao@rock-chips.com>
-+ */
-+
-+#ifndef __DT_BINDINGS_POWER_RV1126B_POWER_CONTROLLER_H__
-+#define __DT_BINDINGS_POWER_RV1126B_POWER_CONTROLLER_H__
-+
-+/* VD_NPU */
-+#define RV1126B_PD_NPU		0
-+
-+/* VD_LOGIC */
-+#define RV1126B_PD_VDO		1
-+#define RV1126B_PD_AIISP	2
-+
-+#endif
-diff --git a/include/dt-bindings/power/starfive,jh7110-pmu.h b/include/dt-bindings/power/starfive,jh7110-pmu.h
-new file mode 100644
-index 000000000000..7b4f24927dee
--- /dev/null
-+++ b/include/dt-bindings/power/starfive,jh7110-pmu.h
-@@ -0,0 +1,21 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2022-2023 StarFive Technology Co., Ltd.
-+ * Author: Walker Chen <walker.chen@starfivetech.com>
-+ */
-+#ifndef __DT_BINDINGS_POWER_JH7110_POWER_H__
-+#define __DT_BINDINGS_POWER_JH7110_POWER_H__
-+
-+#define JH7110_PD_SYSTOP	0
-+#define JH7110_PD_CPU		1
-+#define JH7110_PD_GPUA		2
-+#define JH7110_PD_VDEC		3
-+#define JH7110_PD_VOUT		4
-+#define JH7110_PD_ISP		5
-+#define JH7110_PD_VENC		6
-+
-+/* AON Power Domain */
-+#define JH7110_AON_PD_DPHY_TX	0
-+#define JH7110_AON_PD_DPHY_RX	1
-+
-+#endif
-diff --git a/include/dt-bindings/power/summit,smb347-charger.h b/include/dt-bindings/power/summit,smb347-charger.h
-index 3205699b5e41..14f2f9cf2020 100644
--- a/include/dt-bindings/power/summit,smb347-charger.h
-+++ b/include/dt-bindings/power/summit,smb347-charger.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0-or-later or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0-or-later OR MIT) */
- /*
-  * Author: David Heidelberg <david@ixit.cz>
-  */
-diff --git a/include/dt-bindings/power/tegra234-powergate.h b/include/dt-bindings/power/tegra234-powergate.h
-index ae9286cef85c..b0fec2ddec84 100644
--- a/include/dt-bindings/power/tegra234-powergate.h
-+++ b/include/dt-bindings/power/tegra234-powergate.h
-@@ -4,6 +4,7 @@
- #ifndef __ABI_MACH_T234_POWERGATE_T234_H_
- #define __ABI_MACH_T234_POWERGATE_T234_H_
- 
-+#define TEGRA234_POWER_DOMAIN_OFA	1U
- #define TEGRA234_POWER_DOMAIN_AUD	2U
- #define TEGRA234_POWER_DOMAIN_DISP	3U
- #define TEGRA234_POWER_DOMAIN_PCIEX8A	5U
-@@ -11,6 +12,9 @@
- #define TEGRA234_POWER_DOMAIN_PCIEX4BA	7U
- #define TEGRA234_POWER_DOMAIN_PCIEX4BB	8U
- #define TEGRA234_POWER_DOMAIN_PCIEX1A	9U
-+#define TEGRA234_POWER_DOMAIN_XUSBA	10U
-+#define TEGRA234_POWER_DOMAIN_XUSBB	11U
-+#define TEGRA234_POWER_DOMAIN_XUSBC	12U
- #define TEGRA234_POWER_DOMAIN_PCIEX4CA	13U
- #define TEGRA234_POWER_DOMAIN_PCIEX4CB	14U
- #define TEGRA234_POWER_DOMAIN_PCIEX4CC	15U
-@@ -19,6 +23,17 @@
- #define TEGRA234_POWER_DOMAIN_MGBEB	18U
- #define TEGRA234_POWER_DOMAIN_MGBEC	19U
- #define TEGRA234_POWER_DOMAIN_MGBED	20U
-+#define TEGRA234_POWER_DOMAIN_ISPA	22U
-+#define TEGRA234_POWER_DOMAIN_NVDEC	23U
-+#define TEGRA234_POWER_DOMAIN_NVJPGA	24U
-+#define TEGRA234_POWER_DOMAIN_NVENC	25U
-+#define TEGRA234_POWER_DOMAIN_VI	28U
- #define TEGRA234_POWER_DOMAIN_VIC	29U
-+#define TEGRA234_POWER_DOMAIN_PVA	30U
-+#define TEGRA234_POWER_DOMAIN_DLAA	32U
-+#define TEGRA234_POWER_DOMAIN_DLAB	33U
-+#define TEGRA234_POWER_DOMAIN_CV	34U
-+#define TEGRA234_POWER_DOMAIN_GPU	35U
-+#define TEGRA234_POWER_DOMAIN_NVJPGB	36U
- 
- #endif
-diff --git a/include/dt-bindings/power/thead,th1520-power.h b/include/dt-bindings/power/thead,th1520-power.h
-new file mode 100644
-index 000000000000..8395bd1459f3
--- /dev/null
-+++ b/include/dt-bindings/power/thead,th1520-power.h
-@@ -0,0 +1,19 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2022 Alibaba Group Holding Limited.
-+ * Copyright (c) 2024 Samsung Electronics Co., Ltd.
-+ * Author: Michal Wilczynski <m.wilczynski@samsung.com>
-+ */
-+
-+#ifndef __DT_BINDINGS_POWER_TH1520_H
-+#define __DT_BINDINGS_POWER_TH1520_H
-+
-+#define TH1520_AUDIO_PD		0
-+#define TH1520_VDEC_PD		1
-+#define TH1520_NPU_PD		2
-+#define TH1520_VENC_PD		3
-+#define TH1520_GPU_PD		4
-+#define TH1520_DSP0_PD		5
-+#define TH1520_DSP1_PD		6
-+
-+#endif
-diff --git a/include/dt-bindings/power/xlnx-zynqmp-power.h b/include/dt-bindings/power/xlnx-zynqmp-power.h
-index 0d9a412fd5e0..618024cbb20d 100644
--- a/include/dt-bindings/power/xlnx-zynqmp-power.h
-+++ b/include/dt-bindings/power/xlnx-zynqmp-power.h
-@@ -6,6 +6,12 @@
- #ifndef _DT_BINDINGS_ZYNQMP_POWER_H
- #define _DT_BINDINGS_ZYNQMP_POWER_H
- 
-+#define		PD_RPU_0	7
-+#define		PD_RPU_1	8
-+#define		PD_R5_0_ATCM	15
-+#define		PD_R5_0_BTCM	16
-+#define		PD_R5_1_ATCM	17
-+#define		PD_R5_1_BTCM	18
- #define		PD_USB_0	22
- #define		PD_USB_1	23
- #define		PD_TTC_0	24
-diff --git a/include/dt-bindings/regulator/nxp,pca9450-regulator.h b/include/dt-bindings/regulator/nxp,pca9450-regulator.h
-new file mode 100644
-index 000000000000..08434caef429
--- /dev/null
-+++ b/include/dt-bindings/regulator/nxp,pca9450-regulator.h
-@@ -0,0 +1,18 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Device Tree binding constants for the NXP PCA9450A/B/C PMIC regulators
-+ */
-+
-+#ifndef _DT_BINDINGS_REGULATORS_NXP_PCA9450_H
-+#define _DT_BINDINGS_REGULATORS_NXP_PCA9450_H
-+
-+/*
-+ * Buck mode constants which may be used in devicetree properties (eg.
-+ * regulator-initial-mode, regulator-allowed-modes).
-+ * See the manufacturer's datasheet for more information on these modes.
-+ */
-+
-+#define PCA9450_BUCK_MODE_AUTO		0
-+#define PCA9450_BUCK_MODE_FORCE_PWM	1
-+
-+#endif
-diff --git a/include/dt-bindings/regulator/st,stm32mp13-regulator.h b/include/dt-bindings/regulator/st,stm32mp13-regulator.h
-new file mode 100644
-index 000000000000..b3a974dfc585
--- /dev/null
-+++ b/include/dt-bindings/regulator/st,stm32mp13-regulator.h
-@@ -0,0 +1,42 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2022, STMicroelectronics - All Rights Reserved
-+ */
-+
-+#ifndef __DT_BINDINGS_REGULATOR_ST_STM32MP13_REGULATOR_H
-+#define __DT_BINDINGS_REGULATOR_ST_STM32MP13_REGULATOR_H
-+
-+/* SCMI voltage domains identifiers */
-+
-+/* SOC Internal regulators */
-+#define VOLTD_SCMI_REG11		0
-+#define VOLTD_SCMI_REG18		1
-+#define VOLTD_SCMI_USB33		2
-+#define VOLTD_SCMI_SDMMC1_IO		3
-+#define VOLTD_SCMI_SDMMC2_IO		4
-+#define VOLTD_SCMI_VREFBUF		5
-+
-+/* STPMIC1 regulators */
-+#define VOLTD_SCMI_STPMIC1_BUCK1	6
-+#define VOLTD_SCMI_STPMIC1_BUCK2	7
-+#define VOLTD_SCMI_STPMIC1_BUCK3	8
-+#define VOLTD_SCMI_STPMIC1_BUCK4	9
-+#define VOLTD_SCMI_STPMIC1_LDO1	        10
-+#define VOLTD_SCMI_STPMIC1_LDO2	        11
-+#define VOLTD_SCMI_STPMIC1_LDO3	        12
-+#define VOLTD_SCMI_STPMIC1_LDO4	        13
-+#define VOLTD_SCMI_STPMIC1_LDO5	        14
-+#define VOLTD_SCMI_STPMIC1_LDO6	        15
-+#define VOLTD_SCMI_STPMIC1_VREFDDR	16
-+#define VOLTD_SCMI_STPMIC1_BOOST	17
-+#define VOLTD_SCMI_STPMIC1_PWR_SW1	18
-+#define VOLTD_SCMI_STPMIC1_PWR_SW2	19
-+
-+/* External regulators */
-+#define VOLTD_SCMI_REGU0		20
-+#define VOLTD_SCMI_REGU1		21
-+#define VOLTD_SCMI_REGU2		22
-+#define VOLTD_SCMI_REGU3		23
-+#define VOLTD_SCMI_REGU4		24
-+
-+#endif /*__DT_BINDINGS_REGULATOR_ST_STM32MP13_REGULATOR_H */
-diff --git a/include/dt-bindings/regulator/st,stm32mp15-regulator.h b/include/dt-bindings/regulator/st,stm32mp15-regulator.h
-new file mode 100644
-index 000000000000..7052507cb3e5
--- /dev/null
-+++ b/include/dt-bindings/regulator/st,stm32mp15-regulator.h
-@@ -0,0 +1,40 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2025, STMicroelectronics - All Rights Reserved
-+ */
-+
-+#ifndef __DT_BINDINGS_REGULATOR_ST_STM32MP15_REGULATOR_H
-+#define __DT_BINDINGS_REGULATOR_ST_STM32MP15_REGULATOR_H
-+
-+/* SCMI voltage domain identifiers */
-+
-+/* SOC Internal regulators */
-+#define VOLTD_SCMI_REG11		0
-+#define VOLTD_SCMI_REG18		1
-+#define VOLTD_SCMI_USB33		2
-+
-+/* STPMIC1 regulators */
-+#define VOLTD_SCMI_STPMIC1_BUCK1	3
-+#define VOLTD_SCMI_STPMIC1_BUCK2	4
-+#define VOLTD_SCMI_STPMIC1_BUCK3	5
-+#define VOLTD_SCMI_STPMIC1_BUCK4	6
-+#define VOLTD_SCMI_STPMIC1_LDO1		7
-+#define VOLTD_SCMI_STPMIC1_LDO2		8
-+#define VOLTD_SCMI_STPMIC1_LDO3		9
-+#define VOLTD_SCMI_STPMIC1_LDO4		10
-+#define VOLTD_SCMI_STPMIC1_LDO5		11
-+#define VOLTD_SCMI_STPMIC1_LDO6		12
-+#define VOLTD_SCMI_STPMIC1_VREFDDR	13
-+#define VOLTD_SCMI_STPMIC1_BOOST	14
-+#define VOLTD_SCMI_STPMIC1_PWR_SW1	15
-+#define VOLTD_SCMI_STPMIC1_PWR_SW2	16
-+#define VOLTD_SCMI_VREFBUF		17
-+
-+/* External regulators */
-+#define VOLTD_SCMI_REGU0		18
-+#define VOLTD_SCMI_REGU1		19
-+#define VOLTD_SCMI_REGU2		20
-+#define VOLTD_SCMI_REGU3		21
-+#define VOLTD_SCMI_REGU4		22
-+
-+#endif /*__DT_BINDINGS_REGULATOR_ST_STM32MP15_REGULATOR_H */
-diff --git a/include/dt-bindings/regulator/st,stm32mp25-regulator.h b/include/dt-bindings/regulator/st,stm32mp25-regulator.h
-new file mode 100644
-index 000000000000..3c3d30911dd0
--- /dev/null
-+++ b/include/dt-bindings/regulator/st,stm32mp25-regulator.h
-@@ -0,0 +1,48 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2024, STMicroelectronics - All Rights Reserved
-+ */
-+
-+#ifndef __DT_BINDINGS_REGULATOR_ST_STM32MP25_REGULATOR_H
-+#define __DT_BINDINGS_REGULATOR_ST_STM32MP25_REGULATOR_H
-+
-+/* SCMI voltage domains identifiers */
-+
-+/* SOC Internal regulators */
-+#define VOLTD_SCMI_VDDIO1		0
-+#define VOLTD_SCMI_VDDIO2		1
-+#define VOLTD_SCMI_VDDIO3		2
-+#define VOLTD_SCMI_VDDIO4		3
-+#define VOLTD_SCMI_VDDIO		4
-+#define VOLTD_SCMI_UCPD			5
-+#define VOLTD_SCMI_USB33		6
-+#define VOLTD_SCMI_ADC			7
-+#define VOLTD_SCMI_GPU			8
-+#define VOLTD_SCMI_VREFBUF		9
-+
-+/* STPMIC2 regulators */
-+#define VOLTD_SCMI_STPMIC2_BUCK1	10
-+#define VOLTD_SCMI_STPMIC2_BUCK2	11
-+#define VOLTD_SCMI_STPMIC2_BUCK3	12
-+#define VOLTD_SCMI_STPMIC2_BUCK4	13
-+#define VOLTD_SCMI_STPMIC2_BUCK5	14
-+#define VOLTD_SCMI_STPMIC2_BUCK6	15
-+#define VOLTD_SCMI_STPMIC2_BUCK7	16
-+#define VOLTD_SCMI_STPMIC2_LDO1		17
-+#define VOLTD_SCMI_STPMIC2_LDO2		18
-+#define VOLTD_SCMI_STPMIC2_LDO3		19
-+#define VOLTD_SCMI_STPMIC2_LDO4		20
-+#define VOLTD_SCMI_STPMIC2_LDO5		21
-+#define VOLTD_SCMI_STPMIC2_LDO6		22
-+#define VOLTD_SCMI_STPMIC2_LDO7		23
-+#define VOLTD_SCMI_STPMIC2_LDO8		24
-+#define VOLTD_SCMI_STPMIC2_REFDDR	25
-+
-+/* External regulators */
-+#define VOLTD_SCMI_REGU0		26
-+#define VOLTD_SCMI_REGU1		27
-+#define VOLTD_SCMI_REGU2		28
-+#define VOLTD_SCMI_REGU3		29
-+#define VOLTD_SCMI_REGU4		30
-+
-+#endif /*__DT_BINDINGS_REGULATOR_ST_STM32MP25_REGULATOR_H */
-diff --git a/include/dt-bindings/reset/airoha,en7523-reset.h b/include/dt-bindings/reset/airoha,en7523-reset.h
-new file mode 100644
-index 000000000000..211e8a23a21c
--- /dev/null
-+++ b/include/dt-bindings/reset/airoha,en7523-reset.h
-@@ -0,0 +1,61 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Copyright (C) 2024 iopsys Software Solutions AB.
-+ * Copyright (C) 2025 Genexis AB.
-+ *
-+ * Author: Mikhail Kshevetskiy <mikhail.kshevetskiy@iopsys.eu>
-+ *
-+ * based on
-+ *   include/dt-bindings/reset/airoha,en7581-reset.h
-+ * by Lorenzo Bianconi <lorenzo@kernel.org>
-+ */
-+
-+#ifndef __DT_BINDINGS_RESET_CONTROLLER_AIROHA_EN7523_H_
-+#define __DT_BINDINGS_RESET_CONTROLLER_AIROHA_EN7523_H_
-+
-+/* RST_CTRL2 */
-+#define EN7523_XPON_PHY_RST		 0
-+#define EN7523_XSI_MAC_RST		 1
-+#define EN7523_XSI_PHY_RST		 2
-+#define EN7523_NPU_RST			 3
-+#define EN7523_I2S_RST			 4
-+#define EN7523_TRNG_RST			 5
-+#define EN7523_TRNG_MSTART_RST		 6
-+#define EN7523_DUAL_HSI0_RST		 7
-+#define EN7523_DUAL_HSI1_RST		 8
-+#define EN7523_HSI_RST			 9
-+#define EN7523_DUAL_HSI0_MAC_RST	10
-+#define EN7523_DUAL_HSI1_MAC_RST	11
-+#define EN7523_HSI_MAC_RST		12
-+#define EN7523_WDMA_RST			13
-+#define EN7523_WOE0_RST			14
-+#define EN7523_WOE1_RST			15
-+#define EN7523_HSDMA_RST		16
-+#define EN7523_I2C2RBUS_RST		17
-+#define EN7523_TDMA_RST			18
-+/* RST_CTRL1 */
-+#define EN7523_PCM1_ZSI_ISI_RST		19
-+#define EN7523_FE_PDMA_RST		20
-+#define EN7523_FE_QDMA_RST		21
-+#define EN7523_PCM_SPIWP_RST		22
-+#define EN7523_CRYPTO_RST		23
-+#define EN7523_TIMER_RST		24
-+#define EN7523_PCM1_RST			25
-+#define EN7523_UART_RST			26
-+#define EN7523_GPIO_RST			27
-+#define EN7523_GDMA_RST			28
-+#define EN7523_I2C_MASTER_RST		29
-+#define EN7523_PCM2_ZSI_ISI_RST		30
-+#define EN7523_SFC_RST			31
-+#define EN7523_UART2_RST		32
-+#define EN7523_GDMP_RST			33
-+#define EN7523_FE_RST			34
-+#define EN7523_USB_HOST_P0_RST		35
-+#define EN7523_GSW_RST			36
-+#define EN7523_SFC2_PCM_RST		37
-+#define EN7523_PCIE0_RST		38
-+#define EN7523_PCIE1_RST		39
-+#define EN7523_PCIE_HB_RST		40
-+#define EN7523_XPON_MAC_RST		41
-+
-+#endif /* __DT_BINDINGS_RESET_CONTROLLER_AIROHA_EN7523_H_ */
-diff --git a/include/dt-bindings/reset/airoha,en7581-reset.h b/include/dt-bindings/reset/airoha,en7581-reset.h
-new file mode 100644
-index 000000000000..6544a1790b83
--- /dev/null
-+++ b/include/dt-bindings/reset/airoha,en7581-reset.h
-@@ -0,0 +1,66 @@
-+// SPDX-License-Identifier: GPL-2.0-only
-+/*
-+ * Copyright (c) 2024 AIROHA Inc
-+ * Author: Lorenzo Bianconi <lorenzo@kernel.org>
-+ */
-+
-+#ifndef __DT_BINDINGS_RESET_CONTROLLER_AIROHA_EN7581_H_
-+#define __DT_BINDINGS_RESET_CONTROLLER_AIROHA_EN7581_H_
-+
-+/* RST_CTRL2 */
-+#define EN7581_XPON_PHY_RST		 0
-+#define EN7581_CPU_TIMER2_RST		 1
-+#define EN7581_HSUART_RST		 2
-+#define EN7581_UART4_RST		 3
-+#define EN7581_UART5_RST		 4
-+#define EN7581_I2C2_RST			 5
-+#define EN7581_XSI_MAC_RST		 6
-+#define EN7581_XSI_PHY_RST		 7
-+#define EN7581_NPU_RST			 8
-+#define EN7581_I2S_RST			 9
-+#define EN7581_TRNG_RST			10
-+#define EN7581_TRNG_MSTART_RST		11
-+#define EN7581_DUAL_HSI0_RST		12
-+#define EN7581_DUAL_HSI1_RST		13
-+#define EN7581_HSI_RST			14
-+#define EN7581_DUAL_HSI0_MAC_RST	15
-+#define EN7581_DUAL_HSI1_MAC_RST	16
-+#define EN7581_HSI_MAC_RST		17
-+#define EN7581_WDMA_RST			18
-+#define EN7581_WOE0_RST			19
-+#define EN7581_WOE1_RST			20
-+#define EN7581_HSDMA_RST		21
-+#define EN7581_TDMA_RST			22
-+#define EN7581_EMMC_RST			23
-+#define EN7581_SOE_RST			24
-+#define EN7581_PCIE2_RST		25
-+#define EN7581_XFP_MAC_RST		26
-+#define EN7581_USB_HOST_P1_RST		27
-+#define EN7581_USB_HOST_P1_U3_PHY_RST	28
-+/* RST_CTRL1 */
-+#define EN7581_PCM1_ZSI_ISI_RST		29
-+#define EN7581_FE_PDMA_RST		30
-+#define EN7581_FE_QDMA_RST		31
-+#define EN7581_PCM_SPIWP_RST		32
-+#define EN7581_CRYPTO_RST		33
-+#define EN7581_TIMER_RST		34
-+#define EN7581_PCM1_RST			35
-+#define EN7581_UART_RST			36
-+#define EN7581_GPIO_RST			37
-+#define EN7581_GDMA_RST			38
-+#define EN7581_I2C_MASTER_RST		39
-+#define EN7581_PCM2_ZSI_ISI_RST		40
-+#define EN7581_SFC_RST			41
-+#define EN7581_UART2_RST		42
-+#define EN7581_GDMP_RST			43
-+#define EN7581_FE_RST			44
-+#define EN7581_USB_HOST_P0_RST		45
-+#define EN7581_GSW_RST			46
-+#define EN7581_SFC2_PCM_RST		47
-+#define EN7581_PCIE0_RST		48
-+#define EN7581_PCIE1_RST		49
-+#define EN7581_CPU_TIMER_RST		50
-+#define EN7581_PCIE_HB_RST		51
-+#define EN7581_XPON_MAC_RST		52
-+
-+#endif /* __DT_BINDINGS_RESET_CONTROLLER_AIROHA_EN7581_H_ */
-diff --git a/include/dt-bindings/reset/altr,rst-mgr-s10.h b/include/dt-bindings/reset/altr,rst-mgr-s10.h
-index 70ea3a09dbe1..04c4d0c6fd34 100644
--- a/include/dt-bindings/reset/altr,rst-mgr-s10.h
-+++ b/include/dt-bindings/reset/altr,rst-mgr-s10.h
-@@ -63,12 +63,15 @@
- #define I2C2_RESET		74
- #define I2C3_RESET		75
- #define I2C4_RESET		76
-/* 77-79 is empty */
-+#define I3C0_RESET		77
-+#define I3C1_RESET		78
-+/* 79 is empty */
- #define UART0_RESET		80
- #define UART1_RESET		81
- /* 82-87 is empty */
- #define GPIO0_RESET		88
- #define GPIO1_RESET		89
-+#define WATCHDOG4_RESET		90
- 
- /* BRGMODRST */
- #define SOC2FPGA_RESET		96
-diff --git a/include/dt-bindings/reset/amlogic,c3-reset.h b/include/dt-bindings/reset/amlogic,c3-reset.h
-new file mode 100644
-index 000000000000..d9127863f603
--- /dev/null
-+++ b/include/dt-bindings/reset/amlogic,c3-reset.h
-@@ -0,0 +1,119 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (c) 2023 Amlogic, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_AMLOGIC_C3_RESET_H
-+#define _DT_BINDINGS_AMLOGIC_C3_RESET_H
-+
-+/* RESET0 */
-+/*						0-3 */
-+#define RESET_USBCTRL				4
-+/*						5-7 */
-+#define RESET_USBPHY20				8
-+/*						9 */
-+#define RESET_USB2DRD				10
-+#define RESET_MIPI_DSI_HOST			11
-+#define RESET_MIPI_DSI_PHY			12
-+/*						13-20 */
-+#define RESET_GE2D				21
-+#define RESET_DWAP				22
-+/*						23-31 */
-+
-+/* RESET1 */
-+#define RESET_AUDIO				32
-+/*						33-34 */
-+#define RESET_DDRAPB				35
-+#define RESET_DDR				36
-+#define RESET_DOS_CAPB3				37
-+#define RESET_DOS				38
-+/*						39-46 */
-+#define RESET_NNA				47
-+#define RESET_ETHERNET				48
-+#define RESET_ISP				49
-+#define RESET_VC9000E_APB			50
-+#define RESET_VC9000E_A				51
-+/*						52 */
-+#define RESET_VC9000E_CORE			53
-+/*						54-63 */
-+
-+/* RESET2 */
-+#define RESET_ABUS_ARB				64
-+#define RESET_IRCTRL				65
-+/*						66 */
-+#define RESET_TEMP_PII				67
-+/*						68-72 */
-+#define RESET_SPICC_0				73
-+#define RESET_SPICC_1				74
-+#define RESET_RSA				75
-+
-+/*						76-79 */
-+#define RESET_MSR_CLK				80
-+#define RESET_SPIFC				81
-+#define RESET_SAR_ADC				82
-+/*						83-87 */
-+#define RESET_ACODEC				88
-+/*						89-90 */
-+#define RESET_WATCHDOG				91
-+/*						92-95 */
-+
-+/* RESET3 */
-+#define RESET_ISP_NIC_GPV			96
-+#define RESET_ISP_NIC_MAIN			97
-+#define RESET_ISP_NIC_VCLK			98
-+#define RESET_ISP_NIC_VOUT			99
-+#define RESET_ISP_NIC_ALL			100
-+#define RESET_VOUT				101
-+#define RESET_VOUT_VENC				102
-+/*						103 */
-+#define RESET_CVE_NIC_GPV			104
-+#define RESET_CVE_NIC_MAIN			105
-+#define RESET_CVE_NIC_GE2D			106
-+#define RESET_CVE_NIC_DW			106
-+#define RESET_CVE_NIC_CVE			108
-+#define RESET_CVE_NIC_ALL			109
-+#define RESET_CVE				110
-+/*						112-127 */
-+
-+/* RESET4 */
-+#define RESET_RTC				128
-+#define RESET_PWM_AB				129
-+#define RESET_PWM_CD				130
-+#define RESET_PWM_EF				131
-+#define RESET_PWM_GH				132
-+#define RESET_PWM_IJ				133
-+#define RESET_PWM_KL				134
-+#define RESET_PWM_MN				135
-+/*						136-137 */
-+#define RESET_UART_A				138
-+#define RESET_UART_B				139
-+#define RESET_UART_C				140
-+#define RESET_UART_D				141
-+#define RESET_UART_E				142
-+#define RESET_UART_F				143
-+#define RESET_I2C_S_A				144
-+#define RESET_I2C_M_A				145
-+#define RESET_I2C_M_B				146
-+#define RESET_I2C_M_C				147
-+#define RESET_I2C_M_D				148
-+/*						149-151 */
-+#define RESET_SD_EMMC_A				152
-+#define RESET_SD_EMMC_B				153
-+#define RESET_SD_EMMC_C				154
-+
-+/* RESET5 */
-+/*						160-172 */
-+#define RESET_BRG_NIC_NNA			173
-+#define RESET_BRG_MUX_NIC_MAIN			174
-+#define RESET_BRG_AO_NIC_ALL			175
-+/*						176-183 */
-+#define RESET_BRG_NIC_VAPB			184
-+#define RESET_BRG_NIC_SDIO_B			185
-+#define RESET_BRG_NIC_SDIO_A			186
-+#define RESET_BRG_NIC_EMMC			187
-+#define RESET_BRG_NIC_DSU			188
-+#define RESET_BRG_NIC_SYSCLK			189
-+#define RESET_BRG_NIC_MAIN			190
-+#define RESET_BRG_NIC_ALL			191
-+
-+#endif
-diff --git a/include/dt-bindings/reset/amlogic,meson-a1-audio-reset.h b/include/dt-bindings/reset/amlogic,meson-a1-audio-reset.h
-new file mode 100644
-index 000000000000..7693552f1507
--- /dev/null
-+++ b/include/dt-bindings/reset/amlogic,meson-a1-audio-reset.h
-@@ -0,0 +1,36 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
-+/*
-+ * Copyright (c) 2024, SaluteDevices. All Rights Reserved.
-+ *
-+ * Author: Jan Dakinevich <jan.dakinevich@salutedevices.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_AMLOGIC_MESON_A1_AUDIO_RESET_H
-+#define _DT_BINDINGS_AMLOGIC_MESON_A1_AUDIO_RESET_H
-+
-+#define AUD_RESET_DDRARB	0
-+#define AUD_RESET_TDMIN_A	1
-+#define AUD_RESET_TDMIN_B	2
-+#define AUD_RESET_TDMIN_LB	3
-+#define AUD_RESET_LOOPBACK	4
-+#define AUD_RESET_TDMOUT_A	5
-+#define AUD_RESET_TDMOUT_B	6
-+#define AUD_RESET_FRDDR_A	7
-+#define AUD_RESET_FRDDR_B	8
-+#define AUD_RESET_TODDR_A	9
-+#define AUD_RESET_TODDR_B	10
-+#define AUD_RESET_SPDIFIN	11
-+#define AUD_RESET_RESAMPLE	12
-+#define AUD_RESET_EQDRC		13
-+#define AUD_RESET_LOCKER	14
-+#define AUD_RESET_TOACODEC	30
-+#define AUD_RESET_CLKTREE	31
-+
-+#define AUD_VAD_RESET_DDRARB	0
-+#define AUD_VAD_RESET_PDM	1
-+#define AUD_VAD_RESET_TDMIN_VAD	2
-+#define AUD_VAD_RESET_TODDR_VAD	3
-+#define AUD_VAD_RESET_TOVAD	4
-+#define AUD_VAD_RESET_CLKTREE	5
-+
-+#endif /* _DT_BINDINGS_AMLOGIC_MESON_A1_AUDIO_RESET_H */
-diff --git a/include/dt-bindings/reset/amlogic,meson-g12a-reset.h b/include/dt-bindings/reset/amlogic,meson-g12a-reset.h
-index 6d487c5eba2c..45f6b8a951d0 100644
--- a/include/dt-bindings/reset/amlogic,meson-g12a-reset.h
-+++ b/include/dt-bindings/reset/amlogic,meson-g12a-reset.h
-@@ -69,7 +69,9 @@
- #define RESET_PARSER_FETCH		72
- #define RESET_CTL			73
- #define RESET_PARSER_TOP		74
-/*					75-77	*/
-+/*					75	*/
-+#define RESET_NNA			76
-+/*					77	*/
- #define RESET_DVALIN			78
- #define RESET_HDMITX			79
- /*					80-95	*/
-diff --git a/include/dt-bindings/reset/aspeed,ast2700-scu.h b/include/dt-bindings/reset/aspeed,ast2700-scu.h
-new file mode 100644
-index 000000000000..d53c719b7a66
--- /dev/null
-+++ b/include/dt-bindings/reset/aspeed,ast2700-scu.h
-@@ -0,0 +1,124 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Device Tree binding constants for AST2700 reset controller.
-+ *
-+ * Copyright (c) 2024 Aspeed Technology Inc.
-+ */
-+
-+#ifndef _MACH_ASPEED_AST2700_RESET_H_
-+#define _MACH_ASPEED_AST2700_RESET_H_
-+
-+/* SOC0 */
-+#define SCU0_RESET_SDRAM	0
-+#define SCU0_RESET_DDRPHY	1
-+#define SCU0_RESET_RSA		2
-+#define SCU0_RESET_SHA3		3
-+#define SCU0_RESET_HACE		4
-+#define SCU0_RESET_SOC		5
-+#define SCU0_RESET_VIDEO	6
-+#define SCU0_RESET_2D		7
-+#define SCU0_RESET_PCIS		8
-+#define SCU0_RESET_RVAS0	9
-+#define SCU0_RESET_RVAS1	10
-+#define SCU0_RESET_SM3		11
-+#define SCU0_RESET_SM4		12
-+#define SCU0_RESET_CRT0		13
-+#define SCU0_RESET_ECC		14
-+#define SCU0_RESET_DP_PCI	15
-+#define SCU0_RESET_UFS		16
-+#define SCU0_RESET_EMMC		17
-+#define SCU0_RESET_PCIE1RST	18
-+#define SCU0_RESET_PCIE1RSTOE	19
-+#define SCU0_RESET_PCIE0RST	20
-+#define SCU0_RESET_PCIE0RSTOE	21
-+#define SCU0_RESET_JTAG		22
-+#define SCU0_RESET_MCTP0	23
-+#define SCU0_RESET_MCTP1	24
-+#define SCU0_RESET_XDMA0	25
-+#define SCU0_RESET_XDMA1	26
-+#define SCU0_RESET_H2X1		27
-+#define SCU0_RESET_DP		28
-+#define SCU0_RESET_DP_MCU	29
-+#define SCU0_RESET_SSP		30
-+#define SCU0_RESET_H2X0		31
-+#define SCU0_RESET_PORTA_VHUB	32
-+#define SCU0_RESET_PORTA_PHY3	33
-+#define SCU0_RESET_PORTA_XHCI	34
-+#define SCU0_RESET_PORTB_VHUB	35
-+#define SCU0_RESET_PORTB_PHY3	36
-+#define SCU0_RESET_PORTB_XHCI	37
-+#define SCU0_RESET_PORTA_VHUB_EHCI	38
-+#define SCU0_RESET_PORTB_VHUB_EHCI	39
-+#define SCU0_RESET_UHCI		40
-+#define SCU0_RESET_TSP		41
-+#define SCU0_RESET_E2M0		42
-+#define SCU0_RESET_E2M1		43
-+#define SCU0_RESET_VLINK	44
-+
-+/* SOC1 */
-+#define SCU1_RESET_LPC0		0
-+#define SCU1_RESET_LPC1		1
-+#define SCU1_RESET_MII		2
-+#define SCU1_RESET_PECI		3
-+#define SCU1_RESET_PWM		4
-+#define SCU1_RESET_MAC0		5
-+#define SCU1_RESET_MAC1		6
-+#define SCU1_RESET_MAC2		7
-+#define SCU1_RESET_ADC		8
-+#define SCU1_RESET_SD		9
-+#define SCU1_RESET_ESPI0	10
-+#define SCU1_RESET_ESPI1	11
-+#define SCU1_RESET_JTAG1	12
-+#define SCU1_RESET_SPI0		13
-+#define SCU1_RESET_SPI1		14
-+#define SCU1_RESET_SPI2		15
-+#define SCU1_RESET_I3C0		16
-+#define SCU1_RESET_I3C1		17
-+#define SCU1_RESET_I3C2		18
-+#define SCU1_RESET_I3C3		19
-+#define SCU1_RESET_I3C4		20
-+#define SCU1_RESET_I3C5		21
-+#define SCU1_RESET_I3C6		22
-+#define SCU1_RESET_I3C7		23
-+#define SCU1_RESET_I3C8		24
-+#define SCU1_RESET_I3C9		25
-+#define SCU1_RESET_I3C10	26
-+#define SCU1_RESET_I3C11	27
-+#define SCU1_RESET_I3C12	28
-+#define SCU1_RESET_I3C13	29
-+#define SCU1_RESET_I3C14	30
-+#define SCU1_RESET_I3C15	31
-+#define SCU1_RESET_MCU0		32
-+#define SCU1_RESET_MCU1		33
-+#define SCU1_RESET_H2A_SPI1	34
-+#define SCU1_RESET_H2A_SPI2	35
-+#define SCU1_RESET_UART0	36
-+#define SCU1_RESET_UART1	37
-+#define SCU1_RESET_UART2	38
-+#define SCU1_RESET_UART3	39
-+#define SCU1_RESET_I2C_FILTER	40
-+#define SCU1_RESET_CALIPTRA	41
-+#define SCU1_RESET_XDMA		42
-+#define SCU1_RESET_FSI		43
-+#define SCU1_RESET_CAN		44
-+#define SCU1_RESET_MCTP		45
-+#define SCU1_RESET_I2C		46
-+#define SCU1_RESET_UART6	47
-+#define SCU1_RESET_UART7	48
-+#define SCU1_RESET_UART8	49
-+#define SCU1_RESET_UART9	50
-+#define SCU1_RESET_LTPI0	51
-+#define SCU1_RESET_VGAL		52
-+#define SCU1_RESET_LTPI1	53
-+#define SCU1_RESET_ACE		54
-+#define SCU1_RESET_E2M		55
-+#define SCU1_RESET_UHCI		56
-+#define SCU1_RESET_PORTC_USB2UART	57
-+#define SCU1_RESET_PORTC_VHUB_EHCI	58
-+#define SCU1_RESET_PORTD_USB2UART	59
-+#define SCU1_RESET_PORTD_VHUB_EHCI	60
-+#define SCU1_RESET_H2X		61
-+#define SCU1_RESET_I3CDMA	62
-+#define SCU1_RESET_PCIE2RST	63
-+
-+#endif  /* _MACH_ASPEED_AST2700_RESET_H_ */
-diff --git a/include/dt-bindings/reset/bcm63268-reset.h b/include/dt-bindings/reset/bcm63268-reset.h
-index 6a6403a4c2d5..d87a7882782a 100644
--- a/include/dt-bindings/reset/bcm63268-reset.h
-+++ b/include/dt-bindings/reset/bcm63268-reset.h
-@@ -23,4 +23,8 @@
- #define BCM63268_RST_PCIE_HARD	17
- #define BCM63268_RST_GPHY	18
- 
-+#define BCM63268_TRST_SW	29
-+#define BCM63268_TRST_HW	30
-+#define BCM63268_TRST_POR	31
-+
- #endif /* __DT_BINDINGS_RESET_BCM63268_H */
-diff --git a/include/dt-bindings/reset/bt1-ccu.h b/include/dt-bindings/reset/bt1-ccu.h
-index 3578e83026bc..c691efaa678f 100644
--- a/include/dt-bindings/reset/bt1-ccu.h
-+++ b/include/dt-bindings/reset/bt1-ccu.h
-@@ -21,5 +21,14 @@
- 
- #define CCU_SYS_SATA_REF_RST		0
- #define CCU_SYS_APB_RST			1
-+#define CCU_SYS_DDR_FULL_RST		2
-+#define CCU_SYS_DDR_INIT_RST		3
-+#define CCU_SYS_PCIE_PCS_PHY_RST	4
-+#define CCU_SYS_PCIE_PIPE0_RST		5
-+#define CCU_SYS_PCIE_CORE_RST		6
-+#define CCU_SYS_PCIE_PWR_RST		7
-+#define CCU_SYS_PCIE_STICKY_RST		8
-+#define CCU_SYS_PCIE_NSTICKY_RST	9
-+#define CCU_SYS_PCIE_HOT_RST		10
- 
- #endif /* __DT_BINDINGS_RESET_BT1_CCU_H */
-diff --git a/include/dt-bindings/reset/canaan,k230-rst.h b/include/dt-bindings/reset/canaan,k230-rst.h
-new file mode 100644
-index 000000000000..e4f6612607fe
--- /dev/null
-+++ b/include/dt-bindings/reset/canaan,k230-rst.h
-@@ -0,0 +1,90 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2023-2024 Canaan Bright Sight Co., Ltd
-+ * Copyright (C) 2024-2025 Junhui Liu <junhui.liu@pigmoral.tech>
-+ */
-+#ifndef _DT_BINDINGS_CANAAN_K230_RST_H_
-+#define _DT_BINDINGS_CANAAN_K230_RST_H_
-+
-+#define RST_CPU0		0
-+#define RST_CPU1		1
-+#define RST_CPU0_FLUSH		2
-+#define RST_CPU1_FLUSH		3
-+#define RST_AI			4
-+#define RST_VPU			5
-+#define RST_HISYS		6
-+#define RST_HISYS_AHB		7
-+#define RST_SDIO0		8
-+#define RST_SDIO1		9
-+#define RST_SDIO_AXI		10
-+#define RST_USB0		11
-+#define RST_USB1		12
-+#define RST_USB0_AHB		13
-+#define RST_USB1_AHB		14
-+#define RST_SPI0		15
-+#define RST_SPI1		16
-+#define RST_SPI2		17
-+#define RST_SEC			18
-+#define RST_PDMA		19
-+#define RST_SDMA		20
-+#define RST_DECOMPRESS		21
-+#define RST_SRAM		22
-+#define RST_SHRM_AXIM		23
-+#define RST_SHRM_AXIS		24
-+#define RST_NONAI2D		25
-+#define RST_MCTL		26
-+#define RST_ISP			27
-+#define RST_ISP_DW		28
-+#define RST_DPU			29
-+#define RST_DISP		30
-+#define RST_GPU			31
-+#define RST_AUDIO		32
-+#define RST_TIMER0		33
-+#define RST_TIMER1		34
-+#define RST_TIMER2		35
-+#define RST_TIMER3		36
-+#define RST_TIMER4		37
-+#define RST_TIMER5		38
-+#define RST_TIMER_APB		39
-+#define RST_HDI			40
-+#define RST_WDT0		41
-+#define RST_WDT1		42
-+#define RST_WDT0_APB		43
-+#define RST_WDT1_APB		44
-+#define RST_TS_APB		45
-+#define RST_MAILBOX		46
-+#define RST_STC			47
-+#define RST_PMU			48
-+#define RST_LOSYS_APB		49
-+#define RST_UART0		50
-+#define RST_UART1		51
-+#define RST_UART2		52
-+#define RST_UART3		53
-+#define RST_UART4		54
-+#define RST_I2C0		55
-+#define RST_I2C1		56
-+#define RST_I2C2		57
-+#define RST_I2C3		58
-+#define RST_I2C4		59
-+#define RST_JAMLINK0_APB	60
-+#define RST_JAMLINK1_APB	61
-+#define RST_JAMLINK2_APB	62
-+#define RST_JAMLINK3_APB	63
-+#define RST_CODEC_APB		64
-+#define RST_GPIO_DB		65
-+#define RST_GPIO_APB		66
-+#define RST_ADC			67
-+#define RST_ADC_APB		68
-+#define RST_PWM_APB		69
-+#define RST_SHRM_APB		70
-+#define RST_CSI0		71
-+#define RST_CSI1		72
-+#define RST_CSI2		73
-+#define RST_CSI_DPHY		74
-+#define RST_ISP_AHB		75
-+#define RST_M0			76
-+#define RST_M1			77
-+#define RST_M2			78
-+#define RST_SPI2AXI		79
-+
-+#endif
-diff --git a/include/dt-bindings/reset/eswin,eic7700-reset.h b/include/dt-bindings/reset/eswin,eic7700-reset.h
-new file mode 100644
-index 000000000000..a370c9f74307
--- /dev/null
-+++ b/include/dt-bindings/reset/eswin,eic7700-reset.h
-@@ -0,0 +1,298 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright 2025, Beijing ESWIN Computing Technology Co., Ltd..
-+ * All rights reserved.
-+ *
-+ * Device Tree binding constants for EIC7700 reset controller.
-+ *
-+ * Authors:
-+ *	Yifeng Huang <huangyifeng@eswincomputing.com>
-+ *	Xuyang Dong <dongxuyang@eswincomputing.com>
-+ */
-+
-+#ifndef __DT_ESWIN_EIC7700_RESET_H__
-+#define __DT_ESWIN_EIC7700_RESET_H__
-+
-+#define EIC7700_RESET_NOC_NSP		0
-+#define EIC7700_RESET_NOC_CFG		1
-+#define EIC7700_RESET_RNOC_NSP		2
-+#define EIC7700_RESET_SNOC_TCU		3
-+#define EIC7700_RESET_SNOC_U84		4
-+#define EIC7700_RESET_SNOC_PCIE_XSR	5
-+#define EIC7700_RESET_SNOC_PCIE_XMR	6
-+#define EIC7700_RESET_SNOC_PCIE_PR	7
-+#define EIC7700_RESET_SNOC_NPU		8
-+#define EIC7700_RESET_SNOC_JTAG		9
-+#define EIC7700_RESET_SNOC_DSP		10
-+#define EIC7700_RESET_SNOC_DDRC1_P2	11
-+#define EIC7700_RESET_SNOC_DDRC1_P1	12
-+#define EIC7700_RESET_SNOC_DDRC0_P2	13
-+#define EIC7700_RESET_SNOC_DDRC0_P1	14
-+#define EIC7700_RESET_SNOC_D2D		15
-+#define EIC7700_RESET_SNOC_AON		16
-+#define EIC7700_RESET_GPU_AXI		17
-+#define EIC7700_RESET_GPU_CFG		18
-+#define EIC7700_RESET_GPU_GRAY		19
-+#define EIC7700_RESET_GPU_JONES		20
-+#define EIC7700_RESET_GPU_SPU		21
-+#define EIC7700_RESET_DSP_AXI		22
-+#define EIC7700_RESET_DSP_CFG		23
-+#define EIC7700_RESET_DSP_DIV4		24
-+#define EIC7700_RESET_DSP_DIV0		25
-+#define EIC7700_RESET_DSP_DIV1		26
-+#define EIC7700_RESET_DSP_DIV2		27
-+#define EIC7700_RESET_DSP_DIV3		28
-+#define EIC7700_RESET_D2D_AXI		29
-+#define EIC7700_RESET_D2D_CFG		30
-+#define EIC7700_RESET_D2D_PRST		31
-+#define EIC7700_RESET_D2D_RAW_PCS	32
-+#define EIC7700_RESET_D2D_RX		33
-+#define EIC7700_RESET_D2D_TX		34
-+#define EIC7700_RESET_D2D_CORE		35
-+#define EIC7700_RESET_DDR1_ARST		36
-+#define EIC7700_RESET_DDR1_TRACE	37
-+#define EIC7700_RESET_DDR0_ARST		38
-+#define EIC7700_RESET_DDR_CFG		39
-+#define EIC7700_RESET_DDR0_TRACE	40
-+#define EIC7700_RESET_DDR_CORE		41
-+#define EIC7700_RESET_DDR_PRST		42
-+#define EIC7700_RESET_TCU_AXI		43
-+#define EIC7700_RESET_TCU_CFG		44
-+#define EIC7700_RESET_TCU_TBU0		45
-+#define EIC7700_RESET_TCU_TBU1		46
-+#define EIC7700_RESET_TCU_TBU2		47
-+#define EIC7700_RESET_TCU_TBU3		48
-+#define EIC7700_RESET_TCU_TBU4		49
-+#define EIC7700_RESET_TCU_TBU5		50
-+#define EIC7700_RESET_TCU_TBU6		51
-+#define EIC7700_RESET_TCU_TBU7		52
-+#define EIC7700_RESET_TCU_TBU8		53
-+#define EIC7700_RESET_TCU_TBU9		54
-+#define EIC7700_RESET_TCU_TBU10		55
-+#define EIC7700_RESET_TCU_TBU11		56
-+#define EIC7700_RESET_TCU_TBU12		57
-+#define EIC7700_RESET_TCU_TBU13		58
-+#define EIC7700_RESET_TCU_TBU14		59
-+#define EIC7700_RESET_TCU_TBU15		60
-+#define EIC7700_RESET_TCU_TBU16		61
-+#define EIC7700_RESET_NPU_AXI		62
-+#define EIC7700_RESET_NPU_CFG		63
-+#define EIC7700_RESET_NPU_CORE		64
-+#define EIC7700_RESET_NPU_E31CORE	65
-+#define EIC7700_RESET_NPU_E31BUS	66
-+#define EIC7700_RESET_NPU_E31DBG	67
-+#define EIC7700_RESET_NPU_LLC		68
-+#define EIC7700_RESET_HSP_AXI		69
-+#define EIC7700_RESET_HSP_CFG		70
-+#define EIC7700_RESET_HSP_POR		71
-+#define EIC7700_RESET_MSHC0_PHY		72
-+#define EIC7700_RESET_MSHC1_PHY		73
-+#define EIC7700_RESET_MSHC2_PHY		74
-+#define EIC7700_RESET_MSHC0_TXRX	75
-+#define EIC7700_RESET_MSHC1_TXRX	76
-+#define EIC7700_RESET_MSHC2_TXRX	77
-+#define EIC7700_RESET_SATA_ASIC0	78
-+#define EIC7700_RESET_SATA_OOB		79
-+#define EIC7700_RESET_SATA_PMALIVE	80
-+#define EIC7700_RESET_SATA_RBC		81
-+#define EIC7700_RESET_DMA0		82
-+#define EIC7700_RESET_HSP_DMA		83
-+#define EIC7700_RESET_USB0_VAUX		84
-+#define EIC7700_RESET_USB1_VAUX		85
-+#define EIC7700_RESET_HSP_SD1_PRST	86
-+#define EIC7700_RESET_HSP_SD0_PRST	87
-+#define EIC7700_RESET_HSP_EMMC_PRST	88
-+#define EIC7700_RESET_HSP_DMA_PRST	89
-+#define EIC7700_RESET_HSP_SD1_ARST	90
-+#define EIC7700_RESET_HSP_SD0_ARST	91
-+#define EIC7700_RESET_HSP_EMMC_ARST	92
-+#define EIC7700_RESET_HSP_DMA_ARST	93
-+#define EIC7700_RESET_HSP_ETH1_ARST	94
-+#define EIC7700_RESET_HSP_ETH0_ARST	95
-+#define EIC7700_RESET_SATA_ARST		96
-+#define EIC7700_RESET_PCIE_CFG		97
-+#define EIC7700_RESET_PCIE_POWEUP	98
-+#define EIC7700_RESET_PCIE_PERST	99
-+#define EIC7700_RESET_I2C0		100
-+#define EIC7700_RESET_I2C1		101
-+#define EIC7700_RESET_I2C2		102
-+#define EIC7700_RESET_I2C3		103
-+#define EIC7700_RESET_I2C4		104
-+#define EIC7700_RESET_I2C5		105
-+#define EIC7700_RESET_I2C6		106
-+#define EIC7700_RESET_I2C7		107
-+#define EIC7700_RESET_I2C8		108
-+#define EIC7700_RESET_I2C9		109
-+#define EIC7700_RESET_FAN		110
-+#define EIC7700_RESET_PVT0		111
-+#define EIC7700_RESET_PVT1		112
-+#define EIC7700_RESET_MBOX0		113
-+#define EIC7700_RESET_MBOX1		114
-+#define EIC7700_RESET_MBOX2		115
-+#define EIC7700_RESET_MBOX3		116
-+#define EIC7700_RESET_MBOX4		117
-+#define EIC7700_RESET_MBOX5		118
-+#define EIC7700_RESET_MBOX6		119
-+#define EIC7700_RESET_MBOX7		120
-+#define EIC7700_RESET_MBOX8		121
-+#define EIC7700_RESET_MBOX9		122
-+#define EIC7700_RESET_MBOX10		123
-+#define EIC7700_RESET_MBOX11		124
-+#define EIC7700_RESET_MBOX12		125
-+#define EIC7700_RESET_MBOX13		126
-+#define EIC7700_RESET_MBOX14		127
-+#define EIC7700_RESET_MBOX15		128
-+#define EIC7700_RESET_UART0		129
-+#define EIC7700_RESET_UART1		130
-+#define EIC7700_RESET_UART2		131
-+#define EIC7700_RESET_UART3		132
-+#define EIC7700_RESET_UART4		133
-+#define EIC7700_RESET_GPIO0		134
-+#define EIC7700_RESET_GPIO1		135
-+#define EIC7700_RESET_TIMER		136
-+#define EIC7700_RESET_SSI0		137
-+#define EIC7700_RESET_SSI1		138
-+#define EIC7700_RESET_WDT0		139
-+#define EIC7700_RESET_WDT1		140
-+#define EIC7700_RESET_WDT2		141
-+#define EIC7700_RESET_WDT3		142
-+#define EIC7700_RESET_LSP_CFG		143
-+#define EIC7700_RESET_U84_CORE0		144
-+#define EIC7700_RESET_U84_CORE1		145
-+#define EIC7700_RESET_U84_CORE2		146
-+#define EIC7700_RESET_U84_CORE3		147
-+#define EIC7700_RESET_U84_BUS		148
-+#define EIC7700_RESET_U84_DBG		149
-+#define EIC7700_RESET_U84_TRACECOM	150
-+#define EIC7700_RESET_U84_TRACE0	151
-+#define EIC7700_RESET_U84_TRACE1	152
-+#define EIC7700_RESET_U84_TRACE2	153
-+#define EIC7700_RESET_U84_TRACE3	154
-+#define EIC7700_RESET_SCPU_CORE		155
-+#define EIC7700_RESET_SCPU_BUS		156
-+#define EIC7700_RESET_SCPU_DBG		157
-+#define EIC7700_RESET_LPCPU_CORE	158
-+#define EIC7700_RESET_LPCPU_BUS		159
-+#define EIC7700_RESET_LPCPU_DBG		160
-+#define EIC7700_RESET_VC_CFG		161
-+#define EIC7700_RESET_VC_AXI		162
-+#define EIC7700_RESET_VC_MONCFG		163
-+#define EIC7700_RESET_JD_CFG		164
-+#define EIC7700_RESET_JD_AXI		165
-+#define EIC7700_RESET_JE_CFG		166
-+#define EIC7700_RESET_JE_AXI		167
-+#define EIC7700_RESET_VD_CFG		168
-+#define EIC7700_RESET_VD_AXI		169
-+#define EIC7700_RESET_VE_AXI		170
-+#define EIC7700_RESET_VE_CFG		171
-+#define EIC7700_RESET_G2D_CORE		172
-+#define EIC7700_RESET_G2D_CFG		173
-+#define EIC7700_RESET_G2D_AXI		174
-+#define EIC7700_RESET_VI_AXI		175
-+#define EIC7700_RESET_VI_CFG		176
-+#define EIC7700_RESET_VI_DWE		177
-+#define EIC7700_RESET_DVP		178
-+#define EIC7700_RESET_ISP0		179
-+#define EIC7700_RESET_ISP1		180
-+#define EIC7700_RESET_SHUTTR0		181
-+#define EIC7700_RESET_SHUTTR1		182
-+#define EIC7700_RESET_SHUTTR2		183
-+#define EIC7700_RESET_SHUTTR3		184
-+#define EIC7700_RESET_SHUTTR4		185
-+#define EIC7700_RESET_SHUTTR5		186
-+#define EIC7700_RESET_VO_MIPI		187
-+#define EIC7700_RESET_VO_PRST		188
-+#define EIC7700_RESET_VO_HDMI_PRST	189
-+#define EIC7700_RESET_VO_HDMI_PHY	190
-+#define EIC7700_RESET_VO_HDMI		191
-+#define EIC7700_RESET_VO_I2S		192
-+#define EIC7700_RESET_VO_I2S_PRST	193
-+#define EIC7700_RESET_VO_AXI		194
-+#define EIC7700_RESET_VO_CFG		195
-+#define EIC7700_RESET_VO_DC		196
-+#define EIC7700_RESET_VO_DC_PRST	197
-+#define EIC7700_RESET_BOOTSPI_HRST	198
-+#define EIC7700_RESET_BOOTSPI		199
-+#define EIC7700_RESET_ANO1		200
-+#define EIC7700_RESET_ANO0		201
-+#define EIC7700_RESET_DMA1_ARST		202
-+#define EIC7700_RESET_DMA1_HRST		203
-+#define EIC7700_RESET_FPRT		204
-+#define EIC7700_RESET_HBLOCK		205
-+#define EIC7700_RESET_SECSR		206
-+#define EIC7700_RESET_OTP		207
-+#define EIC7700_RESET_PKA		208
-+#define EIC7700_RESET_SPACC		209
-+#define EIC7700_RESET_TRNG		210
-+#define EIC7700_RESET_TIMER0_0		211
-+#define EIC7700_RESET_TIMER0_1		212
-+#define EIC7700_RESET_TIMER0_2		213
-+#define EIC7700_RESET_TIMER0_3		214
-+#define EIC7700_RESET_TIMER0_4		215
-+#define EIC7700_RESET_TIMER0_5		216
-+#define EIC7700_RESET_TIMER0_6		217
-+#define EIC7700_RESET_TIMER0_7		218
-+#define EIC7700_RESET_TIMER0_N		219
-+#define EIC7700_RESET_TIMER1_0		220
-+#define EIC7700_RESET_TIMER1_1		221
-+#define EIC7700_RESET_TIMER1_2		222
-+#define EIC7700_RESET_TIMER1_3		223
-+#define EIC7700_RESET_TIMER1_4		224
-+#define EIC7700_RESET_TIMER1_5		225
-+#define EIC7700_RESET_TIMER1_6		226
-+#define EIC7700_RESET_TIMER1_7		227
-+#define EIC7700_RESET_TIMER1_N		228
-+#define EIC7700_RESET_TIMER2_0		229
-+#define EIC7700_RESET_TIMER2_1		230
-+#define EIC7700_RESET_TIMER2_2		231
-+#define EIC7700_RESET_TIMER2_3		232
-+#define EIC7700_RESET_TIMER2_4		233
-+#define EIC7700_RESET_TIMER2_5		234
-+#define EIC7700_RESET_TIMER2_6		235
-+#define EIC7700_RESET_TIMER2_7		236
-+#define EIC7700_RESET_TIMER2_N		237
-+#define EIC7700_RESET_TIMER3_0		238
-+#define EIC7700_RESET_TIMER3_1		239
-+#define EIC7700_RESET_TIMER3_2		240
-+#define EIC7700_RESET_TIMER3_3		241
-+#define EIC7700_RESET_TIMER3_4		242
-+#define EIC7700_RESET_TIMER3_5		243
-+#define EIC7700_RESET_TIMER3_6		244
-+#define EIC7700_RESET_TIMER3_7		245
-+#define EIC7700_RESET_TIMER3_N		246
-+#define EIC7700_RESET_RTC		247
-+#define EIC7700_RESET_MNOC_SNOC_NSP	248
-+#define EIC7700_RESET_MNOC_VC		249
-+#define EIC7700_RESET_MNOC_CFG		250
-+#define EIC7700_RESET_MNOC_HSP		251
-+#define EIC7700_RESET_MNOC_GPU		252
-+#define EIC7700_RESET_MNOC_DDRC1_P3	253
-+#define EIC7700_RESET_MNOC_DDRC0_P3	254
-+#define EIC7700_RESET_RNOC_VO		255
-+#define EIC7700_RESET_RNOC_VI		256
-+#define EIC7700_RESET_RNOC_SNOC_NSP	257
-+#define EIC7700_RESET_RNOC_CFG		258
-+#define EIC7700_RESET_MNOC_DDRC1_P4	259
-+#define EIC7700_RESET_MNOC_DDRC0_P4	260
-+#define EIC7700_RESET_CNOC_VO_CFG	261
-+#define EIC7700_RESET_CNOC_VI_CFG	262
-+#define EIC7700_RESET_CNOC_VC_CFG	263
-+#define EIC7700_RESET_CNOC_TCU_CFG	264
-+#define EIC7700_RESET_CNOC_PCIE_CFG	265
-+#define EIC7700_RESET_CNOC_NPU_CFG	266
-+#define EIC7700_RESET_CNOC_LSP_CFG	267
-+#define EIC7700_RESET_CNOC_HSP_CFG	268
-+#define EIC7700_RESET_CNOC_GPU_CFG	269
-+#define EIC7700_RESET_CNOC_DSPT_CFG	270
-+#define EIC7700_RESET_CNOC_DDRT1_CFG	271
-+#define EIC7700_RESET_CNOC_DDRT0_CFG	272
-+#define EIC7700_RESET_CNOC_D2D_CFG	273
-+#define EIC7700_RESET_CNOC_CFG		274
-+#define EIC7700_RESET_CNOC_CLMM_CFG	275
-+#define EIC7700_RESET_CNOC_AON_CFG	276
-+#define EIC7700_RESET_LNOC_CFG		277
-+#define EIC7700_RESET_LNOC_NPU_LLC	278
-+#define EIC7700_RESET_LNOC_DDRC1_P0	279
-+#define EIC7700_RESET_LNOC_DDRC0_P0	280
-+
-+#endif /* __DT_ESWIN_EIC7700_RESET_H__ */
-diff --git a/include/dt-bindings/reset/fsl,imx8ulp-sim-lpav.h b/include/dt-bindings/reset/fsl,imx8ulp-sim-lpav.h
-new file mode 100644
-index 000000000000..adf95bb26d21
--- /dev/null
-+++ b/include/dt-bindings/reset/fsl,imx8ulp-sim-lpav.h
-@@ -0,0 +1,16 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright 2025 NXP
-+ */
-+
-+#ifndef DT_BINDING_RESET_IMX8ULP_SIM_LPAV_H
-+#define DT_BINDING_RESET_IMX8ULP_SIM_LPAV_H
-+
-+#define IMX8ULP_SIM_LPAV_HIFI4_DSP_DBG_RST	0
-+#define IMX8ULP_SIM_LPAV_HIFI4_DSP_RST		1
-+#define IMX8ULP_SIM_LPAV_HIFI4_DSP_STALL	2
-+#define IMX8ULP_SIM_LPAV_DSI_RST_BYTE_N		3
-+#define IMX8ULP_SIM_LPAV_DSI_RST_ESC_N		4
-+#define IMX8ULP_SIM_LPAV_DSI_RST_DPI_N		5
-+
-+#endif /* DT_BINDING_RESET_IMX8ULP_SIM_LPAV_H */
-diff --git a/include/dt-bindings/reset/imx8mp-reset-audiomix.h b/include/dt-bindings/reset/imx8mp-reset-audiomix.h
-new file mode 100644
-index 000000000000..746c1337ed99
--- /dev/null
-+++ b/include/dt-bindings/reset/imx8mp-reset-audiomix.h
-@@ -0,0 +1,13 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
-+/*
-+ * Copyright 2025 NXP
-+ */
-+
-+#ifndef DT_BINDING_RESET_IMX8MP_AUDIOMIX_H
-+#define DT_BINDING_RESET_IMX8MP_AUDIOMIX_H
-+
-+#define IMX8MP_AUDIOMIX_EARC_RESET	0
-+#define IMX8MP_AUDIOMIX_EARC_PHY_RESET	1
-+#define IMX8MP_AUDIOMIX_DSP_RUNSTALL	2
-+
-+#endif /* DT_BINDING_RESET_IMX8MP_AUDIOMIX_H */
-diff --git a/include/dt-bindings/reset/mediatek,mt6735-infracfg.h b/include/dt-bindings/reset/mediatek,mt6735-infracfg.h
-new file mode 100644
-index 000000000000..9df969090377
--- /dev/null
-+++ b/include/dt-bindings/reset/mediatek,mt6735-infracfg.h
-@@ -0,0 +1,27 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_RESET_MT6735_INFRACFG_H
-+#define _DT_BINDINGS_RESET_MT6735_INFRACFG_H
-+
-+#define MT6735_INFRA_RST0_EMI_REG		0
-+#define MT6735_INFRA_RST0_DRAMC0_AO		1
-+#define MT6735_INFRA_RST0_AP_CIRQ_EINT		2
-+#define MT6735_INFRA_RST0_APXGPT		3
-+#define MT6735_INFRA_RST0_SCPSYS		4
-+#define MT6735_INFRA_RST0_KP			5
-+#define MT6735_INFRA_RST0_PMIC_WRAP		6
-+#define MT6735_INFRA_RST0_CLDMA_AO_TOP		7
-+#define MT6735_INFRA_RST0_USBSIF_TOP		8
-+#define MT6735_INFRA_RST0_EMI			9
-+#define MT6735_INFRA_RST0_CCIF			10
-+#define MT6735_INFRA_RST0_DRAMC0		11
-+#define MT6735_INFRA_RST0_EMI_AO_REG		12
-+#define MT6735_INFRA_RST0_CCIF_AO		13
-+#define MT6735_INFRA_RST0_TRNG			14
-+#define MT6735_INFRA_RST0_SYS_CIRQ		15
-+#define MT6735_INFRA_RST0_GCE			16
-+#define MT6735_INFRA_RST0_M4U			17
-+#define MT6735_INFRA_RST0_CCIF1			18
-+#define MT6735_INFRA_RST0_CLDMA_TOP_PD		19
-+
-+#endif
-diff --git a/include/dt-bindings/reset/mediatek,mt6735-mfgcfg.h b/include/dt-bindings/reset/mediatek,mt6735-mfgcfg.h
-new file mode 100644
-index 000000000000..c489242b226e
--- /dev/null
-+++ b/include/dt-bindings/reset/mediatek,mt6735-mfgcfg.h
-@@ -0,0 +1,9 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_RESET_MT6735_MFGCFG_H
-+#define _DT_BINDINGS_RESET_MT6735_MFGCFG_H
-+
-+#define MT6735_MFG_RST0_AXI		0
-+#define MT6735_MFG_RST0_G3D		1
-+
-+#endif /* _DT_BINDINGS_RESET_MT6735_MFGCFG_H */
-diff --git a/include/dt-bindings/reset/mediatek,mt6735-pericfg.h b/include/dt-bindings/reset/mediatek,mt6735-pericfg.h
-new file mode 100644
-index 000000000000..a62bb192835a
--- /dev/null
-+++ b/include/dt-bindings/reset/mediatek,mt6735-pericfg.h
-@@ -0,0 +1,31 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_RESET_MT6735_PERICFG_H
-+#define _DT_BINDINGS_RESET_MT6735_PERICFG_H
-+
-+#define MT6735_PERI_RST0_UART0			0
-+#define MT6735_PERI_RST0_UART1			1
-+#define MT6735_PERI_RST0_UART2			2
-+#define MT6735_PERI_RST0_UART3			3
-+#define MT6735_PERI_RST0_UART4			4
-+#define MT6735_PERI_RST0_BTIF			5
-+#define MT6735_PERI_RST0_DISP_PWM_PERI		6
-+#define MT6735_PERI_RST0_PWM			7
-+#define MT6735_PERI_RST0_AUXADC			8
-+#define MT6735_PERI_RST0_DMA			9
-+#define MT6735_PERI_RST0_IRDA			10
-+#define MT6735_PERI_RST0_IRTX			11
-+#define MT6735_PERI_RST0_THERM			12
-+#define MT6735_PERI_RST0_MSDC2			13
-+#define MT6735_PERI_RST0_MSDC3			14
-+#define MT6735_PERI_RST0_MSDC0			15
-+#define MT6735_PERI_RST0_MSDC1			16
-+#define MT6735_PERI_RST0_I2C0			17
-+#define MT6735_PERI_RST0_I2C1			18
-+#define MT6735_PERI_RST0_I2C2			19
-+#define MT6735_PERI_RST0_I2C3			20
-+#define MT6735_PERI_RST0_USB			21
-+
-+#define MT6735_PERI_RST1_SPI0			22
-+
-+#endif
-diff --git a/include/dt-bindings/reset/mediatek,mt6735-vdecsys.h b/include/dt-bindings/reset/mediatek,mt6735-vdecsys.h
-new file mode 100644
-index 000000000000..b6ae5d249192
--- /dev/null
-+++ b/include/dt-bindings/reset/mediatek,mt6735-vdecsys.h
-@@ -0,0 +1,9 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_RESET_MT6735_VDECSYS_H
-+#define _DT_BINDINGS_RESET_MT6735_VDECSYS_H
-+
-+#define MT6735_VDEC_RST0_VDEC			0
-+#define MT6735_VDEC_RST1_SMI_LARB1		1
-+
-+#endif /* _DT_BINDINGS_RESET_MT6735_VDECSYS_H */
-diff --git a/include/dt-bindings/reset/mediatek,mt6735-wdt.h b/include/dt-bindings/reset/mediatek,mt6735-wdt.h
-new file mode 100644
-index 000000000000..c6056e676d46
--- /dev/null
-+++ b/include/dt-bindings/reset/mediatek,mt6735-wdt.h
-@@ -0,0 +1,17 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef _DT_BINDINGS_RESET_MEDIATEK_MT6735_WDT_H_
-+#define _DT_BINDINGS_RESET_MEDIATEK_MT6735_WDT_H_
-+
-+#define MT6735_TOPRGU_MM_RST		1
-+#define MT6735_TOPRGU_MFG_RST		2
-+#define MT6735_TOPRGU_VENC_RST		3
-+#define MT6735_TOPRGU_VDEC_RST		4
-+#define MT6735_TOPRGU_IMG_RST		5
-+#define MT6735_TOPRGU_MD_RST		7
-+#define MT6735_TOPRGU_CONN_RST		9
-+#define MT6735_TOPRGU_C2K_SW_RST	14
-+#define MT6735_TOPRGU_C2K_RST		15
-+#define MT6735_TOPRGU_RST_NUM		9
-+
-+#endif
-diff --git a/include/dt-bindings/reset/mediatek,mt6795-resets.h b/include/dt-bindings/reset/mediatek,mt6795-resets.h
-new file mode 100644
-index 000000000000..5464a4a79a70
--- /dev/null
-+++ b/include/dt-bindings/reset/mediatek,mt6795-resets.h
-@@ -0,0 +1,53 @@
-+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2022 Collabora Ltd.
-+ * Author: AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_CONTROLLER_MT6795
-+#define _DT_BINDINGS_RESET_CONTROLLER_MT6795
-+
-+/* INFRACFG resets */
-+#define MT6795_INFRA_RST0_SCPSYS_RST		0
-+#define MT6795_INFRA_RST0_PMIC_WRAP_RST		1
-+#define MT6795_INFRA_RST1_MIPI_DSI_RST		2
-+#define MT6795_INFRA_RST1_MIPI_CSI_RST		3
-+#define MT6795_INFRA_RST1_MM_IOMMU_RST		4
-+
-+/* MMSYS resets */
-+#define MT6795_MMSYS_SW0_RST_B_SMI_COMMON	0
-+#define MT6795_MMSYS_SW0_RST_B_SMI_LARB		1
-+#define MT6795_MMSYS_SW0_RST_B_CAM_MDP		2
-+#define MT6795_MMSYS_SW0_RST_B_MDP_RDMA0	3
-+#define MT6795_MMSYS_SW0_RST_B_MDP_RDMA1	4
-+#define MT6795_MMSYS_SW0_RST_B_MDP_RSZ0		5
-+#define MT6795_MMSYS_SW0_RST_B_MDP_RSZ1		6
-+#define MT6795_MMSYS_SW0_RST_B_MDP_RSZ2		7
-+#define MT6795_MMSYS_SW0_RST_B_MDP_TDSHP0	8
-+#define MT6795_MMSYS_SW0_RST_B_MDP_TDSHP1	9
-+#define MT6795_MMSYS_SW0_RST_B_MDP_WDMA		10
-+#define MT6795_MMSYS_SW0_RST_B_MDP_WROT0	11
-+#define MT6795_MMSYS_SW0_RST_B_MDP_WROT1	12
-+#define MT6795_MMSYS_SW0_RST_B_MDP_CROP		13
-+
-+/*  PERICFG resets */
-+#define MT6795_PERI_NFI_SW_RST			0
-+#define MT6795_PERI_THERM_SW_RST		1
-+#define MT6795_PERI_MSDC1_SW_RST		2
-+
-+/* TOPRGU resets */
-+#define MT6795_TOPRGU_INFRA_SW_RST		0
-+#define MT6795_TOPRGU_MM_SW_RST			1
-+#define MT6795_TOPRGU_MFG_SW_RST		2
-+#define MT6795_TOPRGU_VENC_SW_RST		3
-+#define MT6795_TOPRGU_VDEC_SW_RST		4
-+#define MT6795_TOPRGU_IMG_SW_RST		5
-+#define MT6795_TOPRGU_DDRPHY_SW_RST		6
-+#define MT6795_TOPRGU_MD_SW_RST			7
-+#define MT6795_TOPRGU_INFRA_AO_SW_RST		8
-+#define MT6795_TOPRGU_MD_LITE_SW_RST		9
-+#define MT6795_TOPRGU_APMIXED_SW_RST		10
-+#define MT6795_TOPRGU_PWRAP_SPI_CTL_RST		11
-+#define MT6795_TOPRGU_SW_RST_NUM		12
-+
-+#endif  /* _DT_BINDINGS_RESET_CONTROLLER_MT6795 */
-diff --git a/include/dt-bindings/reset/mediatek,mt7988-resets.h b/include/dt-bindings/reset/mediatek,mt7988-resets.h
-new file mode 100644
-index 000000000000..0eb152889a89
--- /dev/null
-+++ b/include/dt-bindings/reset/mediatek,mt7988-resets.h
-@@ -0,0 +1,19 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023 Daniel Golle <daniel@makrotopia.org>
-+ * Author: Daniel Golle <daniel@makrotopia.org>
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_CONTROLLER_MT7988
-+#define _DT_BINDINGS_RESET_CONTROLLER_MT7988
-+
-+/* ETHWARP resets */
-+#define MT7988_ETHWARP_RST_SWITCH		0
-+
-+/* INFRA resets */
-+#define MT7988_INFRA_RST0_PEXTP_MAC_SWRST	0
-+#define MT7988_INFRA_RST1_THERM_CTRL_SWRST	1
-+
-+
-+#endif  /* _DT_BINDINGS_RESET_CONTROLLER_MT7988 */
-+
-diff --git a/include/dt-bindings/reset/mediatek,mt8196-resets.h b/include/dt-bindings/reset/mediatek,mt8196-resets.h
-new file mode 100644
-index 000000000000..46ced0850d91
--- /dev/null
-+++ b/include/dt-bindings/reset/mediatek,mt8196-resets.h
-@@ -0,0 +1,26 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2025 Collabora Ltd.
-+ * Author: AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_CONTROLLER_MT8196
-+#define _DT_BINDINGS_RESET_CONTROLLER_MT8196
-+
-+/* PEXTP0 resets */
-+#define MT8196_PEXTP0_RST0_PCIE0_MAC		0
-+#define MT8196_PEXTP0_RST0_PCIE0_PHY		1
-+
-+/* PEXTP1 resets */
-+#define MT8196_PEXTP1_RST0_PCIE1_MAC		0
-+#define MT8196_PEXTP1_RST0_PCIE1_PHY		1
-+#define MT8196_PEXTP1_RST0_PCIE2_MAC		2
-+#define MT8196_PEXTP1_RST0_PCIE2_PHY		3
-+
-+/* UFS resets */
-+#define MT8196_UFSAO_RST0_UFS_MPHY		0
-+#define MT8196_UFSAO_RST1_UFS_UNIPRO		1
-+#define MT8196_UFSAO_RST1_UFS_CRYPTO		2
-+#define MT8196_UFSAO_RST1_UFSHCI		3
-+
-+#endif  /* _DT_BINDINGS_RESET_CONTROLLER_MT8196 */
-diff --git a/include/dt-bindings/reset/mt8188-resets.h b/include/dt-bindings/reset/mt8188-resets.h
-new file mode 100644
-index 000000000000..5a58c54e7d20
--- /dev/null
-+++ b/include/dt-bindings/reset/mt8188-resets.h
-@@ -0,0 +1,116 @@
-+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)*/
-+/*
-+ * Copyright (c) 2022 MediaTek Inc.
-+ * Author: Runyang Chen <runyang.chen@mediatek.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_CONTROLLER_MT8188
-+#define _DT_BINDINGS_RESET_CONTROLLER_MT8188
-+
-+#define MT8188_TOPRGU_CONN_MCU_SW_RST          0
-+#define MT8188_TOPRGU_INFRA_GRST_SW_RST        1
-+#define MT8188_TOPRGU_IPU0_SW_RST              2
-+#define MT8188_TOPRGU_IPU1_SW_RST              3
-+#define MT8188_TOPRGU_IPU2_SW_RST              4
-+#define MT8188_TOPRGU_AUD_ASRC_SW_RST          5
-+#define MT8188_TOPRGU_INFRA_SW_RST             6
-+#define MT8188_TOPRGU_MMSYS_SW_RST             7
-+#define MT8188_TOPRGU_MFG_SW_RST               8
-+#define MT8188_TOPRGU_VENC_SW_RST              9
-+#define MT8188_TOPRGU_VDEC_SW_RST              10
-+#define MT8188_TOPRGU_CAM_VCORE_SW_RST         11
-+#define MT8188_TOPRGU_SCP_SW_RST               12
-+#define MT8188_TOPRGU_APMIXEDSYS_SW_RST        13
-+#define MT8188_TOPRGU_AUDIO_SW_RST             14
-+#define MT8188_TOPRGU_CAMSYS_SW_RST            15
-+#define MT8188_TOPRGU_MJC_SW_RST               16
-+#define MT8188_TOPRGU_PERI_SW_RST              17
-+#define MT8188_TOPRGU_PERI_AO_SW_RST           18
-+#define MT8188_TOPRGU_PCIE_SW_RST              19
-+#define MT8188_TOPRGU_ADSPSYS_SW_RST           21
-+#define MT8188_TOPRGU_DPTX_SW_RST              22
-+#define MT8188_TOPRGU_SPMI_MST_SW_RST          23
-+
-+#define MT8188_TOPRGU_SW_RST_NUM               24
-+
-+/* INFRA resets */
-+#define MT8188_INFRA_RST1_THERMAL_MCU_RST          0
-+#define MT8188_INFRA_RST1_THERMAL_CTRL_RST         1
-+#define MT8188_INFRA_RST3_PTP_CTRL_RST             2
-+
-+#define MT8188_VDO0_RST_DISP_OVL0		0
-+#define MT8188_VDO0_RST_FAKE_ENG0		1
-+#define MT8188_VDO0_RST_DISP_CCORR0		2
-+#define MT8188_VDO0_RST_DISP_MUTEX0		3
-+#define MT8188_VDO0_RST_DISP_GAMMA0		4
-+#define MT8188_VDO0_RST_DISP_DITHER0		5
-+#define MT8188_VDO0_RST_DISP_WDMA0		6
-+#define MT8188_VDO0_RST_DISP_RDMA0		7
-+#define MT8188_VDO0_RST_DSI0			8
-+#define MT8188_VDO0_RST_DSI1			9
-+#define MT8188_VDO0_RST_DSC_WRAP0		10
-+#define MT8188_VDO0_RST_VPP_MERGE0		11
-+#define MT8188_VDO0_RST_DP_INTF0		12
-+#define MT8188_VDO0_RST_DISP_AAL0		13
-+#define MT8188_VDO0_RST_INLINEROT0		14
-+#define MT8188_VDO0_RST_APB_BUS			15
-+#define MT8188_VDO0_RST_DISP_COLOR0		16
-+#define MT8188_VDO0_RST_MDP_WROT0		17
-+#define MT8188_VDO0_RST_DISP_RSZ0		18
-+
-+#define MT8188_VDO1_RST_SMI_LARB2		0
-+#define MT8188_VDO1_RST_SMI_LARB3		1
-+#define MT8188_VDO1_RST_GALS			2
-+#define MT8188_VDO1_RST_FAKE_ENG0		3
-+#define MT8188_VDO1_RST_FAKE_ENG1		4
-+#define MT8188_VDO1_RST_MDP_RDMA0		5
-+#define MT8188_VDO1_RST_MDP_RDMA1		6
-+#define MT8188_VDO1_RST_MDP_RDMA2		7
-+#define MT8188_VDO1_RST_MDP_RDMA3		8
-+#define MT8188_VDO1_RST_VPP_MERGE0		9
-+#define MT8188_VDO1_RST_VPP_MERGE1		10
-+#define MT8188_VDO1_RST_VPP_MERGE2		11
-+#define MT8188_VDO1_RST_VPP_MERGE3		12
-+#define MT8188_VDO1_RST_VPP_MERGE4		13
-+#define MT8188_VDO1_RST_VPP2_TO_VDO1_DL_ASYNC	14
-+#define MT8188_VDO1_RST_VPP3_TO_VDO1_DL_ASYNC	15
-+#define MT8188_VDO1_RST_DISP_MUTEX		16
-+#define MT8188_VDO1_RST_MDP_RDMA4		17
-+#define MT8188_VDO1_RST_MDP_RDMA5		18
-+#define MT8188_VDO1_RST_MDP_RDMA6		19
-+#define MT8188_VDO1_RST_MDP_RDMA7		20
-+#define MT8188_VDO1_RST_DP_INTF1_MMCK		21
-+#define MT8188_VDO1_RST_DPI0_MM_CK		22
-+#define MT8188_VDO1_RST_DPI1_MM_CK		23
-+#define MT8188_VDO1_RST_MERGE0_DL_ASYNC		24
-+#define MT8188_VDO1_RST_MERGE1_DL_ASYNC		25
-+#define MT8188_VDO1_RST_MERGE2_DL_ASYNC		26
-+#define MT8188_VDO1_RST_MERGE3_DL_ASYNC		27
-+#define MT8188_VDO1_RST_MERGE4_DL_ASYNC		28
-+#define MT8188_VDO1_RST_VDO0_DSC_TO_VDO1_DL_ASYNC	29
-+#define MT8188_VDO1_RST_VDO0_MERGE_TO_VDO1_DL_ASYNC	30
-+#define MT8188_VDO1_RST_PADDING0		31
-+#define MT8188_VDO1_RST_PADDING1		32
-+#define MT8188_VDO1_RST_PADDING2		33
-+#define MT8188_VDO1_RST_PADDING3		34
-+#define MT8188_VDO1_RST_PADDING4		35
-+#define MT8188_VDO1_RST_PADDING5		36
-+#define MT8188_VDO1_RST_PADDING6		37
-+#define MT8188_VDO1_RST_PADDING7		38
-+#define MT8188_VDO1_RST_DISP_RSZ0		39
-+#define MT8188_VDO1_RST_DISP_RSZ1		40
-+#define MT8188_VDO1_RST_DISP_RSZ2		41
-+#define MT8188_VDO1_RST_DISP_RSZ3		42
-+#define MT8188_VDO1_RST_HDR_VDO_FE0		43
-+#define MT8188_VDO1_RST_HDR_GFX_FE0		44
-+#define MT8188_VDO1_RST_HDR_VDO_BE		45
-+#define MT8188_VDO1_RST_HDR_VDO_FE1		46
-+#define MT8188_VDO1_RST_HDR_GFX_FE1		47
-+#define MT8188_VDO1_RST_DISP_MIXER		48
-+#define MT8188_VDO1_RST_HDR_VDO_FE0_DL_ASYNC	49
-+#define MT8188_VDO1_RST_HDR_VDO_FE1_DL_ASYNC	50
-+#define MT8188_VDO1_RST_HDR_GFX_FE0_DL_ASYNC	51
-+#define MT8188_VDO1_RST_HDR_GFX_FE1_DL_ASYNC	52
-+#define MT8188_VDO1_RST_HDR_VDO_BE_DL_ASYNC	53
-+
-+#endif  /* _DT_BINDINGS_RESET_CONTROLLER_MT8188 */
-diff --git a/include/dt-bindings/reset/mt8195-resets.h b/include/dt-bindings/reset/mt8195-resets.h
-index 0b1937f14b36..e61660438d61 100644
--- a/include/dt-bindings/reset/mt8195-resets.h
-+++ b/include/dt-bindings/reset/mt8195-resets.h
-@@ -31,5 +31,53 @@
- #define MT8195_INFRA_RST0_THERM_CTRL_SWRST     0
- #define MT8195_INFRA_RST3_THERM_CTRL_PTP_SWRST 1
- #define MT8195_INFRA_RST4_THERM_CTRL_MCU_SWRST 2
-+#define MT8195_INFRA_RST2_PCIE_P0_SWRST        3
-+#define MT8195_INFRA_RST2_PCIE_P1_SWRST        4
-+#define MT8195_INFRA_RST2_USBSIF_P1_SWRST      5
-+
-+/* VDOSYS1 */
-+#define MT8195_VDOSYS1_SW0_RST_B_SMI_LARB2                     0
-+#define MT8195_VDOSYS1_SW0_RST_B_SMI_LARB3                     1
-+#define MT8195_VDOSYS1_SW0_RST_B_GALS                          2
-+#define MT8195_VDOSYS1_SW0_RST_B_FAKE_ENG0                     3
-+#define MT8195_VDOSYS1_SW0_RST_B_FAKE_ENG1                     4
-+#define MT8195_VDOSYS1_SW0_RST_B_MDP_RDMA0                     5
-+#define MT8195_VDOSYS1_SW0_RST_B_MDP_RDMA1                     6
-+#define MT8195_VDOSYS1_SW0_RST_B_MDP_RDMA2                     7
-+#define MT8195_VDOSYS1_SW0_RST_B_MDP_RDMA3                     8
-+#define MT8195_VDOSYS1_SW0_RST_B_VPP_MERGE0                    9
-+#define MT8195_VDOSYS1_SW0_RST_B_VPP_MERGE1                    10
-+#define MT8195_VDOSYS1_SW0_RST_B_VPP_MERGE2                    11
-+#define MT8195_VDOSYS1_SW0_RST_B_VPP_MERGE3                    12
-+#define MT8195_VDOSYS1_SW0_RST_B_VPP_MERGE4                    13
-+#define MT8195_VDOSYS1_SW0_RST_B_VPP2_TO_VDO1_DL_ASYNC         14
-+#define MT8195_VDOSYS1_SW0_RST_B_VPP3_TO_VDO1_DL_ASYNC         15
-+#define MT8195_VDOSYS1_SW0_RST_B_DISP_MUTEX                    16
-+#define MT8195_VDOSYS1_SW0_RST_B_MDP_RDMA4                     17
-+#define MT8195_VDOSYS1_SW0_RST_B_MDP_RDMA5                     18
-+#define MT8195_VDOSYS1_SW0_RST_B_MDP_RDMA6                     19
-+#define MT8195_VDOSYS1_SW0_RST_B_MDP_RDMA7                     20
-+#define MT8195_VDOSYS1_SW0_RST_B_DP_INTF0                      21
-+#define MT8195_VDOSYS1_SW0_RST_B_DPI0                          22
-+#define MT8195_VDOSYS1_SW0_RST_B_DPI1                          23
-+#define MT8195_VDOSYS1_SW0_RST_B_DISP_MONITOR                  24
-+#define MT8195_VDOSYS1_SW0_RST_B_MERGE0_DL_ASYNC               25
-+#define MT8195_VDOSYS1_SW0_RST_B_MERGE1_DL_ASYNC               26
-+#define MT8195_VDOSYS1_SW0_RST_B_MERGE2_DL_ASYNC               27
-+#define MT8195_VDOSYS1_SW0_RST_B_MERGE3_DL_ASYNC               28
-+#define MT8195_VDOSYS1_SW0_RST_B_MERGE4_DL_ASYNC               29
-+#define MT8195_VDOSYS1_SW0_RST_B_VDO0_DSC_TO_VDO1_DL_ASYNC     30
-+#define MT8195_VDOSYS1_SW0_RST_B_VDO0_MERGE_TO_VDO1_DL_ASYNC   31
-+#define MT8195_VDOSYS1_SW1_RST_B_HDR_VDO_FE0                   32
-+#define MT8195_VDOSYS1_SW1_RST_B_HDR_GFX_FE0                   33
-+#define MT8195_VDOSYS1_SW1_RST_B_HDR_VDO_BE                    34
-+#define MT8195_VDOSYS1_SW1_RST_B_HDR_VDO_FE1                   48
-+#define MT8195_VDOSYS1_SW1_RST_B_HDR_GFX_FE1                   49
-+#define MT8195_VDOSYS1_SW1_RST_B_DISP_MIXER                    50
-+#define MT8195_VDOSYS1_SW1_RST_B_HDR_VDO_FE0_DL_ASYNC          51
-+#define MT8195_VDOSYS1_SW1_RST_B_HDR_VDO_FE1_DL_ASYNC          52
-+#define MT8195_VDOSYS1_SW1_RST_B_HDR_GFX_FE0_DL_ASYNC          53
-+#define MT8195_VDOSYS1_SW1_RST_B_HDR_GFX_FE1_DL_ASYNC          54
-+#define MT8195_VDOSYS1_SW1_RST_B_HDR_VDO_BE_DL_ASYNC           55
- 
- #endif  /* _DT_BINDINGS_RESET_CONTROLLER_MT8195 */
-diff --git a/include/dt-bindings/reset/nuvoton,ma35d1-reset.h b/include/dt-bindings/reset/nuvoton,ma35d1-reset.h
-new file mode 100644
-index 000000000000..2e99ee0d68c5
--- /dev/null
-+++ b/include/dt-bindings/reset/nuvoton,ma35d1-reset.h
-@@ -0,0 +1,108 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (C) 2023 Nuvoton Technologies.
-+ * Author: Chi-Fen Li <cfli0@nuvoton.com>
-+ *
-+ * Device Tree binding constants for MA35D1 reset controller.
-+ */
-+
-+#ifndef __DT_BINDINGS_RESET_MA35D1_H
-+#define __DT_BINDINGS_RESET_MA35D1_H
-+
-+#define MA35D1_RESET_CHIP	0
-+#define MA35D1_RESET_CA35CR0	1
-+#define MA35D1_RESET_CA35CR1	2
-+#define MA35D1_RESET_CM4	3
-+#define MA35D1_RESET_PDMA0	4
-+#define MA35D1_RESET_PDMA1	5
-+#define MA35D1_RESET_PDMA2	6
-+#define MA35D1_RESET_PDMA3	7
-+#define MA35D1_RESET_DISP	8
-+#define MA35D1_RESET_VCAP0	9
-+#define MA35D1_RESET_VCAP1	10
-+#define MA35D1_RESET_GFX	11
-+#define MA35D1_RESET_VDEC	12
-+#define MA35D1_RESET_WHC0	13
-+#define MA35D1_RESET_WHC1	14
-+#define MA35D1_RESET_GMAC0	15
-+#define MA35D1_RESET_GMAC1	16
-+#define MA35D1_RESET_HWSEM	17
-+#define MA35D1_RESET_EBI	18
-+#define MA35D1_RESET_HSUSBH0	19
-+#define MA35D1_RESET_HSUSBH1	20
-+#define MA35D1_RESET_HSUSBD	21
-+#define MA35D1_RESET_USBHL	22
-+#define MA35D1_RESET_SDH0	23
-+#define MA35D1_RESET_SDH1	24
-+#define MA35D1_RESET_NAND	25
-+#define MA35D1_RESET_GPIO	26
-+#define MA35D1_RESET_MCTLP	27
-+#define MA35D1_RESET_MCTLC	28
-+#define MA35D1_RESET_DDRPUB	29
-+#define MA35D1_RESET_TMR0	30
-+#define MA35D1_RESET_TMR1	31
-+#define MA35D1_RESET_TMR2	32
-+#define MA35D1_RESET_TMR3	33
-+#define MA35D1_RESET_I2C0	34
-+#define MA35D1_RESET_I2C1	35
-+#define MA35D1_RESET_I2C2	36
-+#define MA35D1_RESET_I2C3	37
-+#define MA35D1_RESET_QSPI0	38
-+#define MA35D1_RESET_SPI0	39
-+#define MA35D1_RESET_SPI1	40
-+#define MA35D1_RESET_SPI2	41
-+#define MA35D1_RESET_UART0	42
-+#define MA35D1_RESET_UART1	43
-+#define MA35D1_RESET_UART2	44
-+#define MA35D1_RESET_UART3	45
-+#define MA35D1_RESET_UART4	46
-+#define MA35D1_RESET_UART5	47
-+#define MA35D1_RESET_UART6	48
-+#define MA35D1_RESET_UART7	49
-+#define MA35D1_RESET_CANFD0	50
-+#define MA35D1_RESET_CANFD1	51
-+#define MA35D1_RESET_EADC0	52
-+#define MA35D1_RESET_I2S0	53
-+#define MA35D1_RESET_SC0	54
-+#define MA35D1_RESET_SC1	55
-+#define MA35D1_RESET_QSPI1	56
-+#define MA35D1_RESET_SPI3	57
-+#define MA35D1_RESET_EPWM0	58
-+#define MA35D1_RESET_EPWM1	59
-+#define MA35D1_RESET_QEI0	60
-+#define MA35D1_RESET_QEI1	61
-+#define MA35D1_RESET_ECAP0	62
-+#define MA35D1_RESET_ECAP1	63
-+#define MA35D1_RESET_CANFD2	64
-+#define MA35D1_RESET_ADC0	65
-+#define MA35D1_RESET_TMR4	66
-+#define MA35D1_RESET_TMR5	67
-+#define MA35D1_RESET_TMR6	68
-+#define MA35D1_RESET_TMR7	69
-+#define MA35D1_RESET_TMR8	70
-+#define MA35D1_RESET_TMR9	71
-+#define MA35D1_RESET_TMR10	72
-+#define MA35D1_RESET_TMR11	73
-+#define MA35D1_RESET_UART8	74
-+#define MA35D1_RESET_UART9	75
-+#define MA35D1_RESET_UART10	76
-+#define MA35D1_RESET_UART11	77
-+#define MA35D1_RESET_UART12	78
-+#define MA35D1_RESET_UART13	79
-+#define MA35D1_RESET_UART14	80
-+#define MA35D1_RESET_UART15	81
-+#define MA35D1_RESET_UART16	82
-+#define MA35D1_RESET_I2S1	83
-+#define MA35D1_RESET_I2C4	84
-+#define MA35D1_RESET_I2C5	85
-+#define MA35D1_RESET_EPWM2	86
-+#define MA35D1_RESET_ECAP2	87
-+#define MA35D1_RESET_QEI2	88
-+#define MA35D1_RESET_CANFD3	89
-+#define MA35D1_RESET_KPI	90
-+#define MA35D1_RESET_GIC	91
-+#define MA35D1_RESET_SSMCC	92
-+#define MA35D1_RESET_SSPCC	93
-+#define MA35D1_RESET_COUNT	94
-+
-+#endif
-diff --git a/include/dt-bindings/reset/nvidia,tegra114-car.h b/include/dt-bindings/reset/nvidia,tegra114-car.h
-new file mode 100644
-index 000000000000..9b8c320402db
--- /dev/null
-+++ b/include/dt-bindings/reset/nvidia,tegra114-car.h
-@@ -0,0 +1,13 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * This header provides Tegra114-specific constants for binding
-+ * nvidia,tegra114-car.
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_NVIDIA_TEGRA114_CAR_H
-+#define _DT_BINDINGS_RESET_NVIDIA_TEGRA114_CAR_H
-+
-+#define TEGRA114_RESET(x)		(5 * 32 + (x))
-+#define TEGRA114_RST_DFLL_DVCO		TEGRA114_RESET(0)
-+
-+#endif	/* _DT_BINDINGS_RESET_NVIDIA_TEGRA114_CAR_H */
-diff --git a/include/dt-bindings/reset/nvidia,tegra264.h b/include/dt-bindings/reset/nvidia,tegra264.h
-new file mode 100644
-index 000000000000..a61a56bb232b
--- /dev/null
-+++ b/include/dt-bindings/reset/nvidia,tegra264.h
-@@ -0,0 +1,92 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/* Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. */
-+
-+#ifndef DT_BINDINGS_RESET_NVIDIA_TEGRA264_H
-+#define DT_BINDINGS_RESET_NVIDIA_TEGRA264_H
-+
-+#define TEGRA264_RESET_APE_TKE			1
-+#define TEGRA264_RESET_CEC			2
-+#define TEGRA264_RESET_ADSP_ALL			3
-+#define TEGRA264_RESET_RCE_ALL			4
-+#define TEGRA264_RESET_UFSHC			5
-+#define TEGRA264_RESET_UFSHC_AXI_M		6
-+#define TEGRA264_RESET_UFSHC_LP_SEQ		7
-+#define TEGRA264_RESET_DPAUX			8
-+#define TEGRA264_RESET_EQOS_PCS			9
-+#define TEGRA264_RESET_HWPM			10
-+#define TEGRA264_RESET_I2C1			11
-+#define TEGRA264_RESET_I2C2			12
-+#define TEGRA264_RESET_I2C3			13
-+#define TEGRA264_RESET_I2C4			14
-+#define TEGRA264_RESET_I2C6			15
-+#define TEGRA264_RESET_I2C7			16
-+#define TEGRA264_RESET_I2C8			17
-+#define TEGRA264_RESET_I2C9			18
-+#define TEGRA264_RESET_ISP			19
-+#define TEGRA264_RESET_LA			20
-+#define TEGRA264_RESET_NVCSI			21
-+#define TEGRA264_RESET_EQOS_MAC			22
-+#define TEGRA264_RESET_PWM10			23
-+#define TEGRA264_RESET_PWM2			24
-+#define TEGRA264_RESET_PWM3			25
-+#define TEGRA264_RESET_PWM4			26
-+#define TEGRA264_RESET_PWM5			27
-+#define TEGRA264_RESET_PWM9			28
-+#define TEGRA264_RESET_QSPI0			29
-+#define TEGRA264_RESET_HDA			30
-+#define TEGRA264_RESET_HDACODEC			31
-+#define TEGRA264_RESET_I2C0			32
-+#define TEGRA264_RESET_I2C10			33
-+#define TEGRA264_RESET_SDMMC1			34
-+#define TEGRA264_RESET_MIPI_CAL			35
-+#define TEGRA264_RESET_SPI1			36
-+#define TEGRA264_RESET_SPI2			37
-+#define TEGRA264_RESET_SPI3			38
-+#define TEGRA264_RESET_SPI4			39
-+#define TEGRA264_RESET_SPI5			40
-+#define TEGRA264_RESET_SPI7			41
-+#define TEGRA264_RESET_SPI8			42
-+#define TEGRA264_RESET_SPI9			43
-+#define TEGRA264_RESET_TACH0			44
-+#define TEGRA264_RESET_TSEC			45
-+#define TEGRA264_RESET_VI			46
-+#define TEGRA264_RESET_VI1			47
-+#define TEGRA264_RESET_PVA0_ALL			48
-+#define TEGRA264_RESET_VIC			49
-+#define TEGRA264_RESET_MPHY_CLK_CTL		50
-+#define TEGRA264_RESET_MPHY_L0_RX		51
-+#define TEGRA264_RESET_MPHY_L0_TX		52
-+#define TEGRA264_RESET_MPHY_L1_RX		53
-+#define TEGRA264_RESET_MPHY_L1_TX		54
-+#define TEGRA264_RESET_ISP1			55
-+#define TEGRA264_RESET_I2C11			56
-+#define TEGRA264_RESET_I2C12			57
-+#define TEGRA264_RESET_I2C14			58
-+#define TEGRA264_RESET_I2C15			59
-+#define TEGRA264_RESET_I2C16			60
-+#define TEGRA264_RESET_EQOS_MACSEC		61
-+#define TEGRA264_RESET_MGBE0_PCS		62
-+#define TEGRA264_RESET_MGBE0_MAC		63
-+#define TEGRA264_RESET_MGBE0_MACSEC		64
-+#define TEGRA264_RESET_MGBE1_PCS		65
-+#define TEGRA264_RESET_MGBE1_MAC		66
-+#define TEGRA264_RESET_MGBE1_MACSEC		67
-+#define TEGRA264_RESET_MGBE2_PCS		68
-+#define TEGRA264_RESET_MGBE2_MAC		69
-+#define TEGRA264_RESET_MGBE2_MACSEC		70
-+#define TEGRA264_RESET_MGBE3_PCS		71
-+#define TEGRA264_RESET_MGBE3_MAC		72
-+#define TEGRA264_RESET_MGBE3_MACSEC		73
-+#define TEGRA264_RESET_ADSP_CORE0		74
-+#define TEGRA264_RESET_ADSP_CORE1		75
-+#define TEGRA264_RESET_APE			76
-+#define TEGRA264_RESET_XUSB1_PADCTL		77
-+#define TEGRA264_RESET_AON_CPU_ALL		78
-+#define TEGRA264_RESET_AON_HSP			79
-+#define TEGRA264_RESET_UART4			80
-+#define TEGRA264_RESET_UART5			81
-+#define TEGRA264_RESET_UART9			82
-+#define TEGRA264_RESET_UART10			83
-+#define TEGRA264_RESET_UART8			84
-+
-+#endif /* DT_BINDINGS_RESET_NVIDIA_TEGRA264_H */
-diff --git a/include/dt-bindings/reset/qcom,gcc-ipq5018.h b/include/dt-bindings/reset/qcom,gcc-ipq5018.h
-new file mode 100644
-index 000000000000..8f03c92fc23b
--- /dev/null
-+++ b/include/dt-bindings/reset/qcom,gcc-ipq5018.h
-@@ -0,0 +1,122 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_IPQ_GCC_5018_H
-+#define _DT_BINDINGS_RESET_IPQ_GCC_5018_H
-+
-+#define GCC_APC0_VOLTAGE_DROOP_DETECTOR_BCR	0
-+#define GCC_BLSP1_BCR				1
-+#define GCC_BLSP1_QUP1_BCR			2
-+#define GCC_BLSP1_QUP2_BCR			3
-+#define GCC_BLSP1_QUP3_BCR			4
-+#define GCC_BLSP1_UART1_BCR			5
-+#define GCC_BLSP1_UART2_BCR			6
-+#define GCC_BOOT_ROM_BCR			7
-+#define GCC_BTSS_BCR				8
-+#define GCC_CMN_BLK_BCR				9
-+#define GCC_CMN_LDO_BCR				10
-+#define GCC_CE_BCR				11
-+#define GCC_CRYPTO_BCR				12
-+#define GCC_DCC_BCR				13
-+#define GCC_DCD_BCR				14
-+#define GCC_DDRSS_BCR				15
-+#define GCC_EDPD_BCR				16
-+#define GCC_GEPHY_BCR				17
-+#define GCC_GEPHY_MDC_SW_ARES			18
-+#define GCC_GEPHY_DSP_HW_ARES			19
-+#define GCC_GEPHY_RX_ARES			20
-+#define GCC_GEPHY_TX_ARES			21
-+#define GCC_GMAC0_BCR				22
-+#define GCC_GMAC0_CFG_ARES			23
-+#define GCC_GMAC0_SYS_ARES			24
-+#define GCC_GMAC1_BCR				25
-+#define GCC_GMAC1_CFG_ARES			26
-+#define GCC_GMAC1_SYS_ARES			27
-+#define GCC_IMEM_BCR				28
-+#define GCC_LPASS_BCR				29
-+#define GCC_MDIO0_BCR				30
-+#define GCC_MDIO1_BCR				31
-+#define GCC_MPM_BCR				32
-+#define GCC_PCIE0_BCR				33
-+#define GCC_PCIE0_LINK_DOWN_BCR			34
-+#define GCC_PCIE0_PHY_BCR			35
-+#define GCC_PCIE0PHY_PHY_BCR			36
-+#define GCC_PCIE0_PIPE_ARES			37
-+#define GCC_PCIE0_SLEEP_ARES			38
-+#define GCC_PCIE0_CORE_STICKY_ARES		39
-+#define GCC_PCIE0_AXI_MASTER_ARES		40
-+#define GCC_PCIE0_AXI_SLAVE_ARES		41
-+#define GCC_PCIE0_AHB_ARES			42
-+#define GCC_PCIE0_AXI_MASTER_STICKY_ARES	43
-+#define GCC_PCIE0_AXI_SLAVE_STICKY_ARES		44
-+#define GCC_PCIE1_BCR				45
-+#define GCC_PCIE1_LINK_DOWN_BCR			46
-+#define GCC_PCIE1_PHY_BCR			47
-+#define GCC_PCIE1PHY_PHY_BCR			48
-+#define GCC_PCIE1_PIPE_ARES			49
-+#define GCC_PCIE1_SLEEP_ARES			50
-+#define GCC_PCIE1_CORE_STICKY_ARES		51
-+#define GCC_PCIE1_AXI_MASTER_ARES		52
-+#define GCC_PCIE1_AXI_SLAVE_ARES		53
-+#define GCC_PCIE1_AHB_ARES			54
-+#define GCC_PCIE1_AXI_MASTER_STICKY_ARES	55
-+#define GCC_PCIE1_AXI_SLAVE_STICKY_ARES		56
-+#define GCC_PCNOC_BCR				57
-+#define GCC_PCNOC_BUS_TIMEOUT0_BCR		58
-+#define GCC_PCNOC_BUS_TIMEOUT1_BCR		59
-+#define GCC_PCNOC_BUS_TIMEOUT2_BCR		60
-+#define GCC_PCNOC_BUS_TIMEOUT3_BCR		61
-+#define GCC_PCNOC_BUS_TIMEOUT4_BCR		62
-+#define GCC_PCNOC_BUS_TIMEOUT5_BCR		63
-+#define GCC_PCNOC_BUS_TIMEOUT6_BCR		64
-+#define GCC_PCNOC_BUS_TIMEOUT7_BCR		65
-+#define GCC_PCNOC_BUS_TIMEOUT8_BCR		66
-+#define GCC_PCNOC_BUS_TIMEOUT9_BCR		67
-+#define GCC_PCNOC_BUS_TIMEOUT10_BCR		68
-+#define GCC_PCNOC_BUS_TIMEOUT11_BCR		69
-+#define GCC_PRNG_BCR				70
-+#define GCC_Q6SS_DBG_ARES			71
-+#define GCC_Q6_AHB_S_ARES			72
-+#define GCC_Q6_AHB_ARES				73
-+#define GCC_Q6_AXIM2_ARES			74
-+#define GCC_Q6_AXIM_ARES			75
-+#define GCC_Q6_AXIS_ARES			76
-+#define GCC_QDSS_BCR				77
-+#define GCC_QPIC_BCR				78
-+#define GCC_QUSB2_0_PHY_BCR			79
-+#define GCC_SDCC1_BCR				80
-+#define GCC_SEC_CTRL_BCR			81
-+#define GCC_SPDM_BCR				82
-+#define GCC_SYSTEM_NOC_BCR			83
-+#define GCC_TCSR_BCR				84
-+#define GCC_TLMM_BCR				85
-+#define GCC_UBI0_AXI_ARES			86
-+#define GCC_UBI0_AHB_ARES			87
-+#define GCC_UBI0_NC_AXI_ARES			88
-+#define GCC_UBI0_DBG_ARES			89
-+#define GCC_UBI0_UTCM_ARES			90
-+#define GCC_UBI0_CORE_ARES			91
-+#define GCC_UBI32_BCR				92
-+#define GCC_UNIPHY_BCR				93
-+#define GCC_UNIPHY_AHB_ARES			94
-+#define GCC_UNIPHY_SYS_ARES			95
-+#define GCC_UNIPHY_RX_ARES			96
-+#define GCC_UNIPHY_TX_ARES			97
-+#define GCC_USB0_BCR				98
-+#define GCC_USB0_PHY_BCR			99
-+#define GCC_WCSS_BCR				100
-+#define GCC_WCSS_DBG_ARES			101
-+#define GCC_WCSS_ECAHB_ARES			102
-+#define GCC_WCSS_ACMT_ARES			103
-+#define GCC_WCSS_DBG_BDG_ARES			104
-+#define GCC_WCSS_AHB_S_ARES			105
-+#define GCC_WCSS_AXI_M_ARES			106
-+#define GCC_WCSS_AXI_S_ARES			107
-+#define GCC_WCSS_Q6_BCR				108
-+#define GCC_WCSSAON_RESET			109
-+#define GCC_UNIPHY_SOFT_RESET			110
-+#define GCC_GEPHY_MISC_ARES			111
-+
-+#endif
-diff --git a/include/dt-bindings/reset/qcom,ipq5424-gcc.h b/include/dt-bindings/reset/qcom,ipq5424-gcc.h
-new file mode 100644
-index 000000000000..16a72771c79a
--- /dev/null
-+++ b/include/dt-bindings/reset/qcom,ipq5424-gcc.h
-@@ -0,0 +1,310 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2018,2020 The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_IPQ_GCC_IPQ5424_H
-+#define _DT_BINDINGS_RESET_IPQ_GCC_IPQ5424_H
-+
-+#define GCC_QUPV3_BCR				0
-+#define GCC_QUPV3_I2C0_BCR			1
-+#define GCC_QUPV3_UART0_BCR			2
-+#define GCC_QUPV3_I2C1_BCR			3
-+#define GCC_QUPV3_UART1_BCR			4
-+#define GCC_QUPV3_SPI0_BCR			5
-+#define GCC_QUPV3_SPI1_BCR			6
-+#define GCC_IMEM_BCR				7
-+#define GCC_TME_BCR				8
-+#define GCC_DDRSS_BCR				9
-+#define GCC_PRNG_BCR				10
-+#define GCC_BOOT_ROM_BCR			11
-+#define GCC_NSS_BCR				12
-+#define GCC_MDIO_BCR				13
-+#define GCC_UNIPHY0_BCR				14
-+#define GCC_UNIPHY1_BCR				15
-+#define GCC_UNIPHY2_BCR				16
-+#define GCC_WCSS_BCR				17
-+#define GCC_SEC_CTRL_BCR			19
-+#define GCC_TME_SEC_BUS_BCR			20
-+#define GCC_ADSS_BCR				21
-+#define GCC_LPASS_BCR				22
-+#define GCC_PCIE0_BCR				23
-+#define GCC_PCIE0_LINK_DOWN_BCR			24
-+#define GCC_PCIE0PHY_PHY_BCR			25
-+#define GCC_PCIE0_PHY_BCR			26
-+#define GCC_PCIE1_BCR				27
-+#define GCC_PCIE1_LINK_DOWN_BCR			28
-+#define GCC_PCIE1PHY_PHY_BCR			29
-+#define GCC_PCIE1_PHY_BCR			30
-+#define GCC_PCIE2_BCR				31
-+#define GCC_PCIE2_LINK_DOWN_BCR			32
-+#define GCC_PCIE2PHY_PHY_BCR			33
-+#define GCC_PCIE2_PHY_BCR			34
-+#define GCC_PCIE3_BCR				35
-+#define GCC_PCIE3_LINK_DOWN_BCR			36
-+#define GCC_PCIE3PHY_PHY_BCR			37
-+#define GCC_PCIE3_PHY_BCR			38
-+#define GCC_USB_BCR				39
-+#define GCC_QUSB2_0_PHY_BCR			40
-+#define GCC_USB0_PHY_BCR			41
-+#define GCC_USB3PHY_0_PHY_BCR			42
-+#define GCC_QDSS_BCR				43
-+#define GCC_SNOC_BCR				44
-+#define GCC_ANOC_BCR				45
-+#define GCC_PCNOC_BCR				46
-+#define GCC_PCNOC_BUS_TIMEOUT0_BCR		47
-+#define GCC_PCNOC_BUS_TIMEOUT1_BCR		48
-+#define GCC_PCNOC_BUS_TIMEOUT2_BCR		49
-+#define GCC_PCNOC_BUS_TIMEOUT3_BCR		50
-+#define GCC_PCNOC_BUS_TIMEOUT4_BCR		51
-+#define GCC_PCNOC_BUS_TIMEOUT5_BCR		52
-+#define GCC_PCNOC_BUS_TIMEOUT6_BCR		53
-+#define GCC_PCNOC_BUS_TIMEOUT7_BCR		54
-+#define GCC_PCNOC_BUS_TIMEOUT8_BCR		55
-+#define GCC_PCNOC_BUS_TIMEOUT9_BCR		56
-+#define GCC_QPIC_BCR				57
-+#define GCC_SDCC_BCR				58
-+#define GCC_DCC_BCR				59
-+#define GCC_SPDM_BCR				60
-+#define GCC_MPM_BCR				61
-+#define GCC_APC0_VOLTAGE_DROOP_DETECTOR_BCR	62
-+#define GCC_RBCPR_BCR				63
-+#define GCC_CMN_BLK_BCR				64
-+#define GCC_TCSR_BCR				65
-+#define GCC_TLMM_BCR				66
-+#define GCC_QUPV3_AHB_MST_ARES			67
-+#define GCC_QUPV3_CORE_ARES			68
-+#define GCC_QUPV3_2X_CORE_ARES			69
-+#define GCC_QUPV3_SLEEP_ARES			70
-+#define GCC_QUPV3_AHB_SLV_ARES			71
-+#define GCC_QUPV3_I2C0_ARES			72
-+#define GCC_QUPV3_UART0_ARES			73
-+#define GCC_QUPV3_I2C1_ARES			74
-+#define GCC_QUPV3_UART1_ARES			75
-+#define GCC_QUPV3_SPI0_ARES			76
-+#define GCC_QUPV3_SPI1_ARES			77
-+#define GCC_DEBUG_ARES				78
-+#define GCC_GP1_ARES				79
-+#define GCC_GP2_ARES				80
-+#define GCC_GP3_ARES				81
-+#define GCC_IMEM_AXI_ARES			82
-+#define GCC_IMEM_CFG_AHB_ARES			83
-+#define GCC_TME_ARES				84
-+#define GCC_TME_TS_ARES				85
-+#define GCC_TME_SLOW_ARES			86
-+#define GCC_TME_RTC_TOGGLE_ARES			87
-+#define GCC_TIC_ARES				88
-+#define GCC_PRNG_AHB_ARES			89
-+#define GCC_BOOT_ROM_AHB_ARES			90
-+#define GCC_NSSNOC_ATB_ARES			91
-+#define GCC_NSS_TS_ARES				92
-+#define GCC_NSSNOC_QOSGEN_REF_ARES		93
-+#define GCC_NSSNOC_TIMEOUT_REF_ARES		94
-+#define GCC_NSSNOC_MEMNOC_ARES			95
-+#define GCC_NSSNOC_SNOC_ARES			96
-+#define GCC_NSSCFG_ARES				97
-+#define GCC_NSSNOC_NSSCC_ARES			98
-+#define GCC_NSSCC_ARES				99
-+#define GCC_MDIO_AHB_ARES			100
-+#define GCC_UNIPHY0_SYS_ARES			101
-+#define GCC_UNIPHY0_AHB_ARES			102
-+#define GCC_UNIPHY1_SYS_ARES			103
-+#define GCC_UNIPHY1_AHB_ARES			104
-+#define GCC_UNIPHY2_SYS_ARES			105
-+#define GCC_UNIPHY2_AHB_ARES			106
-+#define GCC_NSSNOC_XO_DCD_ARES			107
-+#define GCC_NSSNOC_SNOC_1_ARES			108
-+#define GCC_NSSNOC_PCNOC_1_ARES			109
-+#define GCC_NSSNOC_MEMNOC_1_ARES		110
-+#define GCC_DDRSS_ATB_ARES			111
-+#define GCC_DDRSS_AHB_ARES			112
-+#define GCC_GEMNOC_AHB_ARES			113
-+#define GCC_GEMNOC_Q6_AXI_ARES			114
-+#define GCC_GEMNOC_NSSNOC_ARES			115
-+#define GCC_GEMNOC_SNOC_ARES			116
-+#define GCC_GEMNOC_APSS_ARES			117
-+#define GCC_GEMNOC_QOSGEN_EXTREF_ARES		118
-+#define GCC_GEMNOC_TS_ARES			119
-+#define GCC_DDRSS_SMS_SLOW_ARES			120
-+#define GCC_GEMNOC_CNOC_ARES			121
-+#define GCC_GEMNOC_XO_DBG_ARES			122
-+#define GCC_GEMNOC_ANOC_ARES			123
-+#define GCC_DDRSS_LLCC_ATB_ARES			124
-+#define GCC_LLCC_TPDM_CFG_ARES			125
-+#define GCC_TME_BUS_ARES			126
-+#define GCC_SEC_CTRL_ACC_ARES			127
-+#define GCC_SEC_CTRL_ARES			128
-+#define GCC_SEC_CTRL_SENSE_ARES			129
-+#define GCC_SEC_CTRL_AHB_ARES			130
-+#define GCC_SEC_CTRL_BOOT_ROM_PATCH_ARES	131
-+#define GCC_ADSS_PWM_ARES			132
-+#define GCC_TME_ATB_ARES			133
-+#define GCC_TME_DBGAPB_ARES			134
-+#define GCC_TME_DEBUG_ARES			135
-+#define GCC_TME_AT_ARES				136
-+#define GCC_TME_APB_ARES			137
-+#define GCC_TME_DMI_DBG_HS_ARES			138
-+#define GCC_APSS_AHB_ARES			139
-+#define GCC_APSS_AXI_ARES			140
-+#define GCC_CPUSS_TRIG_ARES			141
-+#define GCC_APSS_DBG_ARES			142
-+#define GCC_APSS_TS_ARES			143
-+#define GCC_APSS_ATB_ARES			144
-+#define GCC_Q6_AXIM_ARES			145
-+#define GCC_Q6_AXIS_ARES			146
-+#define GCC_Q6_AHB_ARES				147
-+#define GCC_Q6_AHB_S_ARES			148
-+#define GCC_Q6SS_ATBM_ARES			149
-+#define GCC_Q6_TSCTR_1TO2_ARES			150
-+#define GCC_Q6SS_PCLKDBG_ARES			151
-+#define GCC_Q6SS_TRIG_ARES			152
-+#define GCC_Q6SS_BOOT_CBCR_ARES			153
-+#define GCC_WCSS_DBG_IFC_APB_ARES		154
-+#define GCC_WCSS_DBG_IFC_ATB_ARES		155
-+#define GCC_WCSS_DBG_IFC_NTS_ARES		156
-+#define GCC_WCSS_DBG_IFC_DAPBUS_ARES		157
-+#define GCC_WCSS_DBG_IFC_APB_BDG_ARES		158
-+#define GCC_WCSS_DBG_IFC_NTS_BDG_ARES		159
-+#define GCC_WCSS_DBG_IFC_DAPBUS_BDG_ARES	160
-+#define GCC_WCSS_ECAHB_ARES			161
-+#define GCC_WCSS_ACMT_ARES			162
-+#define GCC_WCSS_AHB_S_ARES			163
-+#define GCC_WCSS_AXI_M_ARES			164
-+#define GCC_PCNOC_WAPSS_ARES			165
-+#define GCC_SNOC_WAPSS_ARES			166
-+#define GCC_LPASS_SWAY_ARES			167
-+#define GCC_LPASS_CORE_AXIM_ARES		168
-+#define GCC_PCIE0_AHB_ARES			169
-+#define GCC_PCIE0_AXI_M_ARES			170
-+#define GCC_PCIE0_AXI_S_ARES			171
-+#define GCC_PCIE0_AXI_S_BRIDGE_ARES		172
-+#define GCC_PCIE0_PIPE_ARES			173
-+#define GCC_PCIE0_AUX_ARES			174
-+#define GCC_PCIE1_AHB_ARES			175
-+#define GCC_PCIE1_AXI_M_ARES			176
-+#define GCC_PCIE1_AXI_S_ARES			177
-+#define GCC_PCIE1_AXI_S_BRIDGE_ARES		178
-+#define GCC_PCIE1_PIPE_ARES			179
-+#define GCC_PCIE1_AUX_ARES			180
-+#define GCC_PCIE2_AHB_ARES			181
-+#define GCC_PCIE2_AXI_M_ARES			182
-+#define GCC_PCIE2_AXI_S_ARES			183
-+#define GCC_PCIE2_AXI_S_BRIDGE_ARES		184
-+#define GCC_PCIE2_PIPE_ARES			185
-+#define GCC_PCIE2_AUX_ARES			186
-+#define GCC_PCIE3_AHB_ARES			187
-+#define GCC_PCIE3_AXI_M_ARES			188
-+#define GCC_PCIE3_AXI_S_ARES			189
-+#define GCC_PCIE3_AXI_S_BRIDGE_ARES		190
-+#define GCC_PCIE3_PIPE_ARES			191
-+#define GCC_PCIE3_AUX_ARES			192
-+#define GCC_USB0_MASTER_ARES			193
-+#define GCC_USB0_AUX_ARES			194
-+#define GCC_USB0_MOCK_UTMI_ARES			195
-+#define GCC_USB0_PIPE_ARES			196
-+#define GCC_USB0_SLEEP_ARES			197
-+#define GCC_USB0_PHY_CFG_AHB_ARES		198
-+#define GCC_QDSS_AT_ARES			199
-+#define GCC_QDSS_STM_ARES			200
-+#define GCC_QDSS_TRACECLKIN_ARES		201
-+#define GCC_QDSS_TSCTR_DIV2_ARES		202
-+#define GCC_QDSS_TSCTR_DIV3_ARES		203
-+#define GCC_QDSS_TSCTR_DIV4_ARES		204
-+#define GCC_QDSS_TSCTR_DIV8_ARES		205
-+#define GCC_QDSS_TSCTR_DIV16_ARES		206
-+#define GCC_QDSS_DAP_ARES			207
-+#define GCC_QDSS_APB2JTAG_ARES			208
-+#define GCC_QDSS_ETR_USB_ARES			209
-+#define GCC_QDSS_DAP_AHB_ARES			210
-+#define GCC_QDSS_CFG_AHB_ARES			211
-+#define GCC_QDSS_EUD_AT_ARES			212
-+#define GCC_QDSS_TS_ARES			213
-+#define GCC_QDSS_USB_ARES			214
-+#define GCC_SYS_NOC_AXI_ARES			215
-+#define GCC_SNOC_QOSGEN_EXTREF_ARES		216
-+#define GCC_CNOC_LPASS_CFG_ARES			217
-+#define GCC_SYS_NOC_AT_ARES			218
-+#define GCC_SNOC_PCNOC_AHB_ARES			219
-+#define GCC_SNOC_TME_ARES			220
-+#define GCC_SNOC_XO_DCD_ARES			221
-+#define GCC_SNOC_TS_ARES			222
-+#define GCC_ANOC0_AXI_ARES			223
-+#define GCC_ANOC_PCIE0_1LANE_M_ARES		224
-+#define GCC_ANOC_PCIE2_2LANE_M_ARES		225
-+#define GCC_ANOC_PCIE1_1LANE_M_ARES		226
-+#define GCC_ANOC_PCIE3_2LANE_M_ARES		227
-+#define GCC_ANOC_PCNOC_AHB_ARES			228
-+#define GCC_ANOC_QOSGEN_EXTREF_ARES		229
-+#define GCC_ANOC_XO_DCD_ARES			230
-+#define GCC_SNOC_XO_DBG_ARES			231
-+#define GCC_AGGRNOC_ATB_ARES			232
-+#define GCC_AGGRNOC_TS_ARES			233
-+#define GCC_USB0_EUD_AT_ARES			234
-+#define GCC_PCNOC_TIC_ARES			235
-+#define GCC_PCNOC_AHB_ARES			236
-+#define GCC_PCNOC_XO_DBG_ARES			237
-+#define GCC_SNOC_LPASS_ARES			238
-+#define GCC_PCNOC_AT_ARES			239
-+#define GCC_PCNOC_XO_DCD_ARES			240
-+#define GCC_PCNOC_TS_ARES			241
-+#define GCC_PCNOC_BUS_TIMEOUT0_AHB_ARES		242
-+#define GCC_PCNOC_BUS_TIMEOUT1_AHB_ARES		243
-+#define GCC_PCNOC_BUS_TIMEOUT2_AHB_ARES		244
-+#define GCC_PCNOC_BUS_TIMEOUT3_AHB_ARES		245
-+#define GCC_PCNOC_BUS_TIMEOUT4_AHB_ARES		246
-+#define GCC_PCNOC_BUS_TIMEOUT5_AHB_ARES		247
-+#define GCC_PCNOC_BUS_TIMEOUT6_AHB_ARES		248
-+#define GCC_PCNOC_BUS_TIMEOUT7_AHB_ARES		249
-+#define GCC_Q6_AXIM_RESET			250
-+#define GCC_Q6_AXIS_RESET			251
-+#define GCC_Q6_AHB_S_RESET			252
-+#define GCC_Q6_AHB_RESET			253
-+#define GCC_Q6SS_DBG_RESET			254
-+#define GCC_WCSS_ECAHB_RESET			255
-+#define GCC_WCSS_DBG_BDG_RESET			256
-+#define GCC_WCSS_DBG_RESET			257
-+#define GCC_WCSS_AXI_M_RESET			258
-+#define GCC_WCSS_AHB_S_RESET			259
-+#define GCC_WCSS_ACMT_RESET			260
-+#define GCC_WCSSAON_RESET			261
-+#define GCC_PCIE0_PIPE_RESET			262
-+#define GCC_PCIE0_CORE_STICKY_RESET		263
-+#define GCC_PCIE0_AXI_S_STICKY_RESET		264
-+#define GCC_PCIE0_AXI_S_RESET			265
-+#define GCC_PCIE0_AXI_M_STICKY_RESET		266
-+#define GCC_PCIE0_AXI_M_RESET			267
-+#define GCC_PCIE0_AUX_RESET			268
-+#define GCC_PCIE0_AHB_RESET			269
-+#define GCC_PCIE1_PIPE_RESET			270
-+#define GCC_PCIE1_CORE_STICKY_RESET		271
-+#define GCC_PCIE1_AXI_S_STICKY_RESET		272
-+#define GCC_PCIE1_AXI_S_RESET			273
-+#define GCC_PCIE1_AXI_M_STICKY_RESET		274
-+#define GCC_PCIE1_AXI_M_RESET			275
-+#define GCC_PCIE1_AUX_RESET			276
-+#define GCC_PCIE1_AHB_RESET			277
-+#define GCC_PCIE2_PIPE_RESET			278
-+#define GCC_PCIE2_CORE_STICKY_RESET		279
-+#define GCC_PCIE2_AXI_S_STICKY_RESET		280
-+#define GCC_PCIE2_AXI_S_RESET			281
-+#define GCC_PCIE2_AXI_M_STICKY_RESET		282
-+#define GCC_PCIE2_AXI_M_RESET			283
-+#define GCC_PCIE2_AUX_RESET			284
-+#define GCC_PCIE2_AHB_RESET			285
-+#define GCC_PCIE3_PIPE_RESET			286
-+#define GCC_PCIE3_CORE_STICKY_RESET		287
-+#define GCC_PCIE3_AXI_S_STICKY_RESET		288
-+#define GCC_PCIE3_AXI_S_RESET			289
-+#define GCC_PCIE3_AXI_M_STICKY_RESET		290
-+#define GCC_PCIE3_AXI_M_RESET			291
-+#define GCC_PCIE3_AUX_RESET			292
-+#define GCC_PCIE3_AHB_RESET			293
-+#define GCC_NSS_PARTIAL_RESET			294
-+#define GCC_UNIPHY0_XPCS_ARES			295
-+#define GCC_UNIPHY1_XPCS_ARES			296
-+#define GCC_UNIPHY2_XPCS_ARES			297
-+#define GCC_USB1_BCR				298
-+#define GCC_QUSB2_1_PHY_BCR			299
-+
-+#endif
-diff --git a/include/dt-bindings/reset/qcom,ipq5424-nsscc.h b/include/dt-bindings/reset/qcom,ipq5424-nsscc.h
-new file mode 100644
-index 000000000000..9627e3b0ad30
--- /dev/null
-+++ b/include/dt-bindings/reset/qcom,ipq5424-nsscc.h
-@@ -0,0 +1,46 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_QCOM_IPQ5424_NSSCC_H
-+#define _DT_BINDINGS_RESET_QCOM_IPQ5424_NSSCC_H
-+
-+#define NSS_CC_CE_APB_CLK_ARES					0
-+#define NSS_CC_CE_AXI_CLK_ARES					1
-+#define NSS_CC_DEBUG_CLK_ARES					2
-+#define NSS_CC_EIP_CLK_ARES					3
-+#define NSS_CC_NSS_CSR_CLK_ARES					4
-+#define NSS_CC_NSSNOC_CE_APB_CLK_ARES				5
-+#define NSS_CC_NSSNOC_CE_AXI_CLK_ARES				6
-+#define NSS_CC_NSSNOC_EIP_CLK_ARES				7
-+#define NSS_CC_NSSNOC_NSS_CSR_CLK_ARES				8
-+#define NSS_CC_NSSNOC_PPE_CLK_ARES				9
-+#define NSS_CC_NSSNOC_PPE_CFG_CLK_ARES				10
-+#define NSS_CC_PORT1_MAC_CLK_ARES				11
-+#define NSS_CC_PORT1_RX_CLK_ARES				12
-+#define NSS_CC_PORT1_TX_CLK_ARES				13
-+#define NSS_CC_PORT2_MAC_CLK_ARES				14
-+#define NSS_CC_PORT2_RX_CLK_ARES				15
-+#define NSS_CC_PORT2_TX_CLK_ARES				16
-+#define NSS_CC_PORT3_MAC_CLK_ARES				17
-+#define NSS_CC_PORT3_RX_CLK_ARES				18
-+#define NSS_CC_PORT3_TX_CLK_ARES				19
-+#define NSS_CC_PPE_BCR						20
-+#define NSS_CC_PPE_EDMA_CLK_ARES				21
-+#define NSS_CC_PPE_EDMA_CFG_CLK_ARES				22
-+#define NSS_CC_PPE_SWITCH_BTQ_CLK_ARES				23
-+#define NSS_CC_PPE_SWITCH_CLK_ARES				24
-+#define NSS_CC_PPE_SWITCH_CFG_CLK_ARES				25
-+#define NSS_CC_PPE_SWITCH_IPE_CLK_ARES				26
-+#define NSS_CC_UNIPHY_PORT1_RX_CLK_ARES				27
-+#define NSS_CC_UNIPHY_PORT1_TX_CLK_ARES				28
-+#define NSS_CC_UNIPHY_PORT2_RX_CLK_ARES				29
-+#define NSS_CC_UNIPHY_PORT2_TX_CLK_ARES				30
-+#define NSS_CC_UNIPHY_PORT3_RX_CLK_ARES				31
-+#define NSS_CC_UNIPHY_PORT3_TX_CLK_ARES				32
-+#define NSS_CC_XGMAC0_PTP_REF_CLK_ARES				33
-+#define NSS_CC_XGMAC1_PTP_REF_CLK_ARES				34
-+#define NSS_CC_XGMAC2_PTP_REF_CLK_ARES				35
-+
-+#endif
-diff --git a/include/dt-bindings/reset/qcom,ipq9574-gcc.h b/include/dt-bindings/reset/qcom,ipq9574-gcc.h
-new file mode 100644
-index 000000000000..c709d103673d
--- /dev/null
-+++ b/include/dt-bindings/reset/qcom,ipq9574-gcc.h
-@@ -0,0 +1,165 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2018-2023, The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_IPQ_GCC_9574_H
-+#define _DT_BINDINGS_RESET_IPQ_GCC_9574_H
-+
-+#define GCC_ADSS_BCR						0
-+#define GCC_APC0_VOLTAGE_DROOP_DETECTOR_BCR			1
-+#define GCC_BLSP1_BCR						2
-+#define GCC_BLSP1_QUP1_BCR					3
-+#define GCC_BLSP1_QUP2_BCR					4
-+#define GCC_BLSP1_QUP3_BCR					5
-+#define GCC_BLSP1_QUP4_BCR					6
-+#define GCC_BLSP1_QUP5_BCR					7
-+#define GCC_BLSP1_QUP6_BCR					8
-+#define GCC_BLSP1_UART1_BCR					9
-+#define GCC_BLSP1_UART2_BCR					10
-+#define GCC_BLSP1_UART3_BCR					11
-+#define GCC_BLSP1_UART4_BCR					12
-+#define GCC_BLSP1_UART5_BCR					13
-+#define GCC_BLSP1_UART6_BCR					14
-+#define GCC_BOOT_ROM_BCR					15
-+#define GCC_MDIO_BCR						16
-+#define GCC_NSS_BCR						17
-+#define GCC_NSS_TBU_BCR						18
-+#define GCC_PCIE0_BCR						19
-+#define GCC_PCIE0_LINK_DOWN_BCR					20
-+#define GCC_PCIE0_PHY_BCR					21
-+#define GCC_PCIE0PHY_PHY_BCR					22
-+#define GCC_PCIE1_BCR						23
-+#define GCC_PCIE1_LINK_DOWN_BCR					24
-+#define GCC_PCIE1_PHY_BCR					25
-+#define GCC_PCIE1PHY_PHY_BCR					26
-+#define GCC_PCIE2_BCR						27
-+#define GCC_PCIE2_LINK_DOWN_BCR					28
-+#define GCC_PCIE2_PHY_BCR					29
-+#define GCC_PCIE2PHY_PHY_BCR					30
-+#define GCC_PCIE3_BCR						31
-+#define GCC_PCIE3_LINK_DOWN_BCR					32
-+#define GCC_PCIE3_PHY_BCR					33
-+#define GCC_PCIE3PHY_PHY_BCR					34
-+#define GCC_PRNG_BCR						35
-+#define GCC_QUSB2_0_PHY_BCR					36
-+#define GCC_SDCC_BCR						37
-+#define GCC_TLMM_BCR						38
-+#define GCC_UNIPHY0_BCR						39
-+#define GCC_UNIPHY1_BCR						40
-+#define GCC_UNIPHY2_BCR						41
-+#define GCC_USB0_PHY_BCR					42
-+#define GCC_USB3PHY_0_PHY_BCR					43
-+#define GCC_USB_BCR						44
-+#define GCC_ANOC0_TBU_BCR					45
-+#define GCC_ANOC1_TBU_BCR					46
-+#define GCC_ANOC_BCR						47
-+#define GCC_APSS_TCU_BCR					48
-+#define GCC_CMN_BLK_BCR						49
-+#define GCC_CMN_BLK_AHB_ARES					50
-+#define GCC_CMN_BLK_SYS_ARES					51
-+#define GCC_CMN_BLK_APU_ARES					52
-+#define GCC_DCC_BCR						53
-+#define GCC_DDRSS_BCR						54
-+#define GCC_IMEM_BCR						55
-+#define GCC_LPASS_BCR						56
-+#define GCC_MPM_BCR						57
-+#define GCC_MSG_RAM_BCR						58
-+#define GCC_NSSNOC_MEMNOC_1_ARES				59
-+#define GCC_NSSNOC_PCNOC_1_ARES					60
-+#define GCC_NSSNOC_SNOC_1_ARES					61
-+#define GCC_NSSNOC_XO_DCD_ARES					62
-+#define GCC_NSSNOC_TS_ARES					63
-+#define GCC_NSSCC_ARES						64
-+#define GCC_NSSNOC_NSSCC_ARES					65
-+#define GCC_NSSNOC_ATB_ARES					66
-+#define GCC_NSSNOC_MEMNOC_ARES					67
-+#define GCC_NSSNOC_QOSGEN_REF_ARES				68
-+#define GCC_NSSNOC_SNOC_ARES					69
-+#define GCC_NSSNOC_TIMEOUT_REF_ARES				70
-+#define GCC_NSS_CFG_ARES					71
-+#define GCC_UBI0_DBG_ARES					72
-+#define GCC_PCIE0_AHB_ARES					73
-+#define GCC_PCIE0_AUX_ARES					74
-+#define GCC_PCIE0_AXI_M_ARES					75
-+#define GCC_PCIE0_AXI_M_STICKY_ARES				76
-+#define GCC_PCIE0_AXI_S_ARES					77
-+#define GCC_PCIE0_AXI_S_STICKY_ARES				78
-+#define GCC_PCIE0_CORE_STICKY_ARES				79
-+#define GCC_PCIE0_PIPE_ARES					80
-+#define GCC_PCIE1_AHB_ARES					81
-+#define GCC_PCIE1_AUX_ARES					82
-+#define GCC_PCIE1_AXI_M_ARES					83
-+#define GCC_PCIE1_AXI_M_STICKY_ARES				84
-+#define GCC_PCIE1_AXI_S_ARES					85
-+#define GCC_PCIE1_AXI_S_STICKY_ARES				86
-+#define GCC_PCIE1_CORE_STICKY_ARES				87
-+#define GCC_PCIE1_PIPE_ARES					88
-+#define GCC_PCIE2_AHB_ARES					89
-+#define GCC_PCIE2_AUX_ARES					90
-+#define GCC_PCIE2_AXI_M_ARES					91
-+#define GCC_PCIE2_AXI_M_STICKY_ARES				92
-+#define GCC_PCIE2_AXI_S_ARES					93
-+#define GCC_PCIE2_AXI_S_STICKY_ARES				94
-+#define GCC_PCIE2_CORE_STICKY_ARES				95
-+#define GCC_PCIE2_PIPE_ARES					96
-+#define GCC_PCIE3_AHB_ARES					97
-+#define GCC_PCIE3_AUX_ARES					98
-+#define GCC_PCIE3_AXI_M_ARES					99
-+#define GCC_PCIE3_AXI_M_STICKY_ARES				100
-+#define GCC_PCIE3_AXI_S_ARES					101
-+#define GCC_PCIE3_AXI_S_STICKY_ARES				102
-+#define GCC_PCIE3_CORE_STICKY_ARES				103
-+#define GCC_PCIE3_PIPE_ARES					104
-+#define GCC_PCNOC_BCR						105
-+#define GCC_PCNOC_BUS_TIMEOUT0_BCR				106
-+#define GCC_PCNOC_BUS_TIMEOUT1_BCR				107
-+#define GCC_PCNOC_BUS_TIMEOUT2_BCR				108
-+#define GCC_PCNOC_BUS_TIMEOUT3_BCR				109
-+#define GCC_PCNOC_BUS_TIMEOUT4_BCR				110
-+#define GCC_PCNOC_BUS_TIMEOUT5_BCR				111
-+#define GCC_PCNOC_BUS_TIMEOUT6_BCR				112
-+#define GCC_PCNOC_BUS_TIMEOUT7_BCR				113
-+#define GCC_PCNOC_BUS_TIMEOUT8_BCR				114
-+#define GCC_PCNOC_BUS_TIMEOUT9_BCR				115
-+#define GCC_PCNOC_TBU_BCR					116
-+#define GCC_Q6SS_DBG_ARES					117
-+#define GCC_Q6_AHB_ARES						118
-+#define GCC_Q6_AHB_S_ARES					119
-+#define GCC_Q6_AXIM2_ARES					120
-+#define GCC_Q6_AXIM_ARES					121
-+#define GCC_QDSS_BCR						122
-+#define GCC_QPIC_BCR						123
-+#define GCC_QPIC_AHB_ARES					124
-+#define GCC_QPIC_ARES						125
-+#define GCC_RBCPR_BCR						126
-+#define GCC_RBCPR_MX_BCR					127
-+#define GCC_SEC_CTRL_BCR					128
-+#define GCC_SMMU_CFG_BCR					129
-+#define GCC_SNOC_BCR						130
-+#define GCC_SPDM_BCR						131
-+#define GCC_TME_BCR						132
-+#define GCC_UNIPHY0_SYS_RESET					133
-+#define GCC_UNIPHY0_AHB_RESET					134
-+#define GCC_UNIPHY0_XPCS_RESET					135
-+#define GCC_UNIPHY1_SYS_RESET					136
-+#define GCC_UNIPHY1_AHB_RESET					137
-+#define GCC_UNIPHY1_XPCS_RESET					138
-+#define GCC_UNIPHY2_SYS_RESET					139
-+#define GCC_UNIPHY2_AHB_RESET					140
-+#define GCC_UNIPHY2_XPCS_RESET					141
-+#define GCC_USB_MISC_RESET					142
-+#define GCC_WCSSAON_RESET					143
-+#define GCC_WCSS_ACMT_ARES					144
-+#define GCC_WCSS_AHB_S_ARES					145
-+#define GCC_WCSS_AXI_M_ARES					146
-+#define GCC_WCSS_BCR						147
-+#define GCC_WCSS_DBG_ARES					148
-+#define GCC_WCSS_DBG_BDG_ARES					149
-+#define GCC_WCSS_ECAHB_ARES					150
-+#define GCC_WCSS_Q6_BCR						151
-+#define GCC_WCSS_Q6_TBU_BCR					152
-+#define GCC_TCSR_BCR						153
-+#define GCC_CRYPTO_BCR						154
-+
-+#endif
-diff --git a/include/dt-bindings/reset/qcom,ipq9574-nsscc.h b/include/dt-bindings/reset/qcom,ipq9574-nsscc.h
-new file mode 100644
-index 000000000000..7f152e98b99c
--- /dev/null
-+++ b/include/dt-bindings/reset/qcom,ipq9574-nsscc.h
-@@ -0,0 +1,134 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, 2025 The Linux Foundation. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_IPQ_NSSCC_9574_H
-+#define _DT_BINDINGS_RESET_IPQ_NSSCC_9574_H
-+
-+#define EDMA_HW_RESET                   0
-+#define NSS_CC_CE_BCR			1
-+#define NSS_CC_CLC_BCR			2
-+#define NSS_CC_EIP197_BCR		3
-+#define NSS_CC_HAQ_BCR			4
-+#define NSS_CC_IMEM_BCR			5
-+#define NSS_CC_MAC_BCR			6
-+#define NSS_CC_PPE_BCR			7
-+#define NSS_CC_UBI_BCR			8
-+#define NSS_CC_UNIPHY_BCR		9
-+#define UBI3_CLKRST_CLAMP_ENABLE	10
-+#define UBI3_CORE_CLAMP_ENABLE		11
-+#define UBI2_CLKRST_CLAMP_ENABLE	12
-+#define UBI2_CORE_CLAMP_ENABLE		13
-+#define UBI1_CLKRST_CLAMP_ENABLE	14
-+#define UBI1_CORE_CLAMP_ENABLE		15
-+#define UBI0_CLKRST_CLAMP_ENABLE	16
-+#define UBI0_CORE_CLAMP_ENABLE		17
-+#define NSSNOC_NSS_CSR_ARES		18
-+#define NSS_CSR_ARES			19
-+#define PPE_BTQ_ARES			20
-+#define PPE_IPE_ARES			21
-+#define PPE_ARES			22
-+#define PPE_CFG_ARES			23
-+#define PPE_EDMA_ARES			24
-+#define PPE_EDMA_CFG_ARES		25
-+#define CRY_PPE_ARES			26
-+#define NSSNOC_PPE_ARES			27
-+#define NSSNOC_PPE_CFG_ARES		28
-+#define PORT1_MAC_ARES			29
-+#define PORT2_MAC_ARES			30
-+#define PORT3_MAC_ARES			31
-+#define PORT4_MAC_ARES			32
-+#define PORT5_MAC_ARES			33
-+#define PORT6_MAC_ARES			34
-+#define XGMAC0_PTP_REF_ARES		35
-+#define XGMAC1_PTP_REF_ARES		36
-+#define XGMAC2_PTP_REF_ARES		37
-+#define XGMAC3_PTP_REF_ARES		38
-+#define XGMAC4_PTP_REF_ARES		39
-+#define XGMAC5_PTP_REF_ARES		40
-+#define HAQ_AHB_ARES			41
-+#define HAQ_AXI_ARES			42
-+#define NSSNOC_HAQ_AHB_ARES		43
-+#define NSSNOC_HAQ_AXI_ARES		44
-+#define CE_APB_ARES			45
-+#define CE_AXI_ARES			46
-+#define NSSNOC_CE_APB_ARES		47
-+#define NSSNOC_CE_AXI_ARES		48
-+#define CRYPTO_ARES			49
-+#define NSSNOC_CRYPTO_ARES		50
-+#define NSSNOC_NC_AXI0_1_ARES		51
-+#define UBI0_CORE_ARES			52
-+#define UBI1_CORE_ARES			53
-+#define UBI2_CORE_ARES			54
-+#define UBI3_CORE_ARES			55
-+#define NC_AXI0_ARES			56
-+#define UTCM0_ARES			57
-+#define NC_AXI1_ARES			58
-+#define UTCM1_ARES			59
-+#define NC_AXI2_ARES			60
-+#define UTCM2_ARES			61
-+#define NC_AXI3_ARES			62
-+#define UTCM3_ARES			63
-+#define NSSNOC_NC_AXI0_ARES		64
-+#define AHB0_ARES			65
-+#define INTR0_AHB_ARES			66
-+#define AHB1_ARES			67
-+#define INTR1_AHB_ARES			68
-+#define AHB2_ARES			69
-+#define INTR2_AHB_ARES			70
-+#define AHB3_ARES			71
-+#define INTR3_AHB_ARES			72
-+#define NSSNOC_AHB0_ARES		73
-+#define NSSNOC_INT0_AHB_ARES		74
-+#define AXI0_ARES			75
-+#define AXI1_ARES			76
-+#define AXI2_ARES			77
-+#define AXI3_ARES			78
-+#define NSSNOC_AXI0_ARES		79
-+#define IMEM_QSB_ARES			80
-+#define NSSNOC_IMEM_QSB_ARES		81
-+#define IMEM_AHB_ARES			82
-+#define NSSNOC_IMEM_AHB_ARES		83
-+#define UNIPHY_PORT1_RX_ARES		84
-+#define UNIPHY_PORT1_TX_ARES		85
-+#define UNIPHY_PORT2_RX_ARES		86
-+#define UNIPHY_PORT2_TX_ARES		87
-+#define UNIPHY_PORT3_RX_ARES		88
-+#define UNIPHY_PORT3_TX_ARES		89
-+#define UNIPHY_PORT4_RX_ARES		90
-+#define UNIPHY_PORT4_TX_ARES		91
-+#define UNIPHY_PORT5_RX_ARES		92
-+#define UNIPHY_PORT5_TX_ARES		93
-+#define UNIPHY_PORT6_RX_ARES		94
-+#define UNIPHY_PORT6_TX_ARES		95
-+#define PORT1_RX_ARES			96
-+#define PORT1_TX_ARES			97
-+#define PORT2_RX_ARES			98
-+#define PORT2_TX_ARES			99
-+#define PORT3_RX_ARES			100
-+#define PORT3_TX_ARES			101
-+#define PORT4_RX_ARES			102
-+#define PORT4_TX_ARES			103
-+#define PORT5_RX_ARES			104
-+#define PORT5_TX_ARES			105
-+#define PORT6_RX_ARES			106
-+#define PORT6_TX_ARES			107
-+#define PPE_FULL_RESET			108
-+#define UNIPHY0_SOFT_RESET		109
-+#define UNIPHY1_SOFT_RESET		110
-+#define UNIPHY2_SOFT_RESET		111
-+#define UNIPHY_PORT1_ARES		112
-+#define UNIPHY_PORT2_ARES		113
-+#define UNIPHY_PORT3_ARES		114
-+#define UNIPHY_PORT4_ARES		115
-+#define UNIPHY_PORT5_ARES		116
-+#define UNIPHY_PORT6_ARES		117
-+#define NSSPORT1_RESET			118
-+#define NSSPORT2_RESET			119
-+#define NSSPORT3_RESET			120
-+#define NSSPORT4_RESET			121
-+#define NSSPORT5_RESET			122
-+#define NSSPORT6_RESET			123
-+
-+#endif
-diff --git a/include/dt-bindings/reset/qcom,qca8k-nsscc.h b/include/dt-bindings/reset/qcom,qca8k-nsscc.h
-new file mode 100644
-index 000000000000..c71167a3bd41
--- /dev/null
-+++ b/include/dt-bindings/reset/qcom,qca8k-nsscc.h
-@@ -0,0 +1,76 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_QCOM_QCA8K_NSS_CC_H
-+#define _DT_BINDINGS_RESET_QCOM_QCA8K_NSS_CC_H
-+
-+#define NSS_CC_SWITCH_CORE_ARES				1
-+#define NSS_CC_APB_BRIDGE_ARES				2
-+#define NSS_CC_MAC0_TX_ARES				3
-+#define NSS_CC_MAC0_TX_SRDS1_ARES			4
-+#define NSS_CC_MAC0_RX_ARES				5
-+#define NSS_CC_MAC0_RX_SRDS1_ARES			6
-+#define NSS_CC_MAC1_SRDS1_CH0_RX_ARES			7
-+#define NSS_CC_MAC1_TX_ARES				8
-+#define NSS_CC_MAC1_GEPHY0_TX_ARES			9
-+#define NSS_CC_MAC1_SRDS1_CH0_XGMII_RX_ARES		10
-+#define NSS_CC_MAC1_SRDS1_CH0_TX_ARES			11
-+#define NSS_CC_MAC1_RX_ARES				12
-+#define NSS_CC_MAC1_GEPHY0_RX_ARES			13
-+#define NSS_CC_MAC1_SRDS1_CH0_XGMII_TX_ARES		14
-+#define NSS_CC_MAC2_SRDS1_CH1_RX_ARES			15
-+#define NSS_CC_MAC2_TX_ARES				16
-+#define NSS_CC_MAC2_GEPHY1_TX_ARES			17
-+#define NSS_CC_MAC2_SRDS1_CH1_XGMII_RX_ARES		18
-+#define NSS_CC_MAC2_SRDS1_CH1_TX_ARES			19
-+#define NSS_CC_MAC2_RX_ARES				20
-+#define NSS_CC_MAC2_GEPHY1_RX_ARES			21
-+#define NSS_CC_MAC2_SRDS1_CH1_XGMII_TX_ARES		22
-+#define NSS_CC_MAC3_SRDS1_CH2_RX_ARES			23
-+#define NSS_CC_MAC3_TX_ARES				24
-+#define NSS_CC_MAC3_GEPHY2_TX_ARES			25
-+#define NSS_CC_MAC3_SRDS1_CH2_XGMII_RX_ARES		26
-+#define NSS_CC_MAC3_SRDS1_CH2_TX_ARES			27
-+#define NSS_CC_MAC3_RX_ARES				28
-+#define NSS_CC_MAC3_GEPHY2_RX_ARES			29
-+#define NSS_CC_MAC3_SRDS1_CH2_XGMII_TX_ARES		30
-+#define NSS_CC_MAC4_SRDS1_CH3_RX_ARES			31
-+#define NSS_CC_MAC4_TX_ARES				32
-+#define NSS_CC_MAC4_GEPHY3_TX_ARES			33
-+#define NSS_CC_MAC4_SRDS1_CH3_XGMII_RX_ARES		34
-+#define NSS_CC_MAC4_SRDS1_CH3_TX_ARES			35
-+#define NSS_CC_MAC4_RX_ARES				36
-+#define NSS_CC_MAC4_GEPHY3_RX_ARES			37
-+#define NSS_CC_MAC4_SRDS1_CH3_XGMII_TX_ARES		38
-+#define NSS_CC_MAC5_TX_ARES				39
-+#define NSS_CC_MAC5_TX_SRDS0_ARES			40
-+#define NSS_CC_MAC5_RX_ARES				41
-+#define NSS_CC_MAC5_RX_SRDS0_ARES			42
-+#define NSS_CC_AHB_ARES					43
-+#define NSS_CC_SEC_CTRL_AHB_ARES			44
-+#define NSS_CC_TLMM_ARES				45
-+#define NSS_CC_TLMM_AHB_ARES				46
-+#define NSS_CC_CNOC_AHB_ARES				47
-+#define NSS_CC_MDIO_AHB_ARES				48
-+#define NSS_CC_MDIO_MASTER_AHB_ARES			49
-+#define NSS_CC_SRDS0_SYS_ARES				50
-+#define NSS_CC_SRDS1_SYS_ARES				51
-+#define NSS_CC_GEPHY0_SYS_ARES				52
-+#define NSS_CC_GEPHY1_SYS_ARES				53
-+#define NSS_CC_GEPHY2_SYS_ARES				54
-+#define NSS_CC_GEPHY3_SYS_ARES				55
-+#define NSS_CC_SEC_CTRL_ARES				56
-+#define NSS_CC_SEC_CTRL_SENSE_ARES			57
-+#define NSS_CC_SLEEP_ARES				58
-+#define NSS_CC_DEBUG_ARES				59
-+#define NSS_CC_GEPHY0_ARES				60
-+#define NSS_CC_GEPHY1_ARES				61
-+#define NSS_CC_GEPHY2_ARES				62
-+#define NSS_CC_GEPHY3_ARES				63
-+#define NSS_CC_DSP_ARES					64
-+#define NSS_CC_GEPHY_FULL_ARES				65
-+#define NSS_CC_GLOBAL_ARES				66
-+#define NSS_CC_XPCS_ARES				67
-+#endif
-diff --git a/include/dt-bindings/reset/qcom,sar2130p-gpucc.h b/include/dt-bindings/reset/qcom,sar2130p-gpucc.h
-new file mode 100644
-index 000000000000..99ba5f092e2a
--- /dev/null
-+++ b/include/dt-bindings/reset/qcom,sar2130p-gpucc.h
-@@ -0,0 +1,14 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2019, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2024, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_QCOM_GPU_CC_SAR2130P_H
-+#define _DT_BINDINGS_RESET_QCOM_GPU_CC_SAR2130P_H
-+
-+#define GPUCC_GPU_CC_GX_BCR			0
-+#define GPUCC_GPU_CC_ACD_BCR			1
-+#define GPUCC_GPU_CC_GX_ACD_IROOT_BCR		2
-+
-+#endif
-diff --git a/include/dt-bindings/reset/qcom,sm8350-videocc.h b/include/dt-bindings/reset/qcom,sm8350-videocc.h
-new file mode 100644
-index 000000000000..cd356b207a4a
--- /dev/null
-+++ b/include/dt-bindings/reset/qcom,sm8350-videocc.h
-@@ -0,0 +1,18 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2019, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_QCOM_VIDEO_CC_SM8350_H
-+#define _DT_BINDINGS_RESET_QCOM_VIDEO_CC_SM8350_H
-+
-+#define VIDEO_CC_CVP_INTERFACE_BCR				0
-+#define VIDEO_CC_CVP_MVS0_BCR					1
-+#define VIDEO_CC_MVS0C_CLK_ARES					2
-+#define VIDEO_CC_CVP_MVS0C_BCR					3
-+#define VIDEO_CC_CVP_MVS1_BCR					4
-+#define VIDEO_CC_MVS1C_CLK_ARES					5
-+#define VIDEO_CC_CVP_MVS1C_BCR					6
-+
-+#endif
-diff --git a/include/dt-bindings/reset/qcom,sm8450-gpucc.h b/include/dt-bindings/reset/qcom,sm8450-gpucc.h
-new file mode 100644
-index 000000000000..58ba8f987107
--- /dev/null
-+++ b/include/dt-bindings/reset/qcom,sm8450-gpucc.h
-@@ -0,0 +1,20 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2019, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_QCOM_GPU_CC_SM8450_H
-+#define _DT_BINDINGS_RESET_QCOM_GPU_CC_SM8450_H
-+
-+#define GPUCC_GPU_CC_ACD_BCR			0
-+#define GPUCC_GPU_CC_CX_BCR			1
-+#define GPUCC_GPU_CC_FAST_HUB_BCR		2
-+#define GPUCC_GPU_CC_FF_BCR			3
-+#define GPUCC_GPU_CC_GFX3D_AON_BCR		4
-+#define GPUCC_GPU_CC_GMU_BCR			5
-+#define GPUCC_GPU_CC_GX_BCR			6
-+#define GPUCC_GPU_CC_XO_BCR			7
-+#define GPUCC_GPU_CC_GX_ACD_IROOT_BCR		8
-+
-+#endif
-diff --git a/include/dt-bindings/reset/qcom,sm8650-gpucc.h b/include/dt-bindings/reset/qcom,sm8650-gpucc.h
-new file mode 100644
-index 000000000000..f021a6cccc66
--- /dev/null
-+++ b/include/dt-bindings/reset/qcom,sm8650-gpucc.h
-@@ -0,0 +1,20 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2019, The Linux Foundation. All rights reserved.
-+ * Copyright (c) 2023, Linaro Limited
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_QCOM_GPU_CC_SM8650_H
-+#define _DT_BINDINGS_RESET_QCOM_GPU_CC_SM8650_H
-+
-+#define GPUCC_GPU_CC_ACD_BCR			0
-+#define GPUCC_GPU_CC_CX_BCR			1
-+#define GPUCC_GPU_CC_FAST_HUB_BCR		2
-+#define GPUCC_GPU_CC_FF_BCR			3
-+#define GPUCC_GPU_CC_GFX3D_AON_BCR		4
-+#define GPUCC_GPU_CC_GMU_BCR			5
-+#define GPUCC_GPU_CC_GX_BCR			6
-+#define GPUCC_GPU_CC_XO_BCR			7
-+#define GPUCC_GPU_CC_GX_ACD_IROOT_BCR		8
-+
-+#endif
-diff --git a/include/dt-bindings/reset/qcom,x1e80100-gpucc.h b/include/dt-bindings/reset/qcom,x1e80100-gpucc.h
-new file mode 100644
-index 000000000000..32b43e71a16f
--- /dev/null
-+++ b/include/dt-bindings/reset/qcom,x1e80100-gpucc.h
-@@ -0,0 +1,19 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_QCOM_X1E80100_GPU_CC_H
-+#define _DT_BINDINGS_RESET_QCOM_X1E80100_GPU_CC_H
-+
-+#define GPUCC_GPU_CC_ACD_BCR					0
-+#define GPUCC_GPU_CC_CB_BCR					1
-+#define GPUCC_GPU_CC_CX_BCR					2
-+#define GPUCC_GPU_CC_FAST_HUB_BCR				3
-+#define GPUCC_GPU_CC_FF_BCR					4
-+#define GPUCC_GPU_CC_GFX3D_AON_BCR				5
-+#define GPUCC_GPU_CC_GMU_BCR					6
-+#define GPUCC_GPU_CC_GX_BCR					7
-+#define GPUCC_GPU_CC_XO_BCR					8
-+
-+#endif
-diff --git a/include/dt-bindings/reset/rockchip,rk3506-cru.h b/include/dt-bindings/reset/rockchip,rk3506-cru.h
-new file mode 100644
-index 000000000000..31c0d4aa410f
--- /dev/null
-+++ b/include/dt-bindings/reset/rockchip,rk3506-cru.h
-@@ -0,0 +1,211 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023-2025 Rockchip Electronics Co., Ltd.
-+ * Author: Finley Xiao <finley.xiao@rock-chips.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_REST_ROCKCHIP_RK3506_H
-+#define _DT_BINDINGS_REST_ROCKCHIP_RK3506_H
-+
-+/* CRU-->SOFTRST_CON00 */
-+#define SRST_NCOREPORESET0_AC	0
-+#define SRST_NCOREPORESET1_AC	1
-+#define SRST_NCOREPORESET2_AC	2
-+#define SRST_NCORESET0_AC	3
-+#define SRST_NCORESET1_AC	4
-+#define SRST_NCORESET2_AC	5
-+#define SRST_NL2RESET_AC	6
-+#define SRST_A_CORE_BIU_AC	7
-+#define SRST_H_M0_AC		8
-+
-+/* CRU-->SOFTRST_CON02 */
-+#define SRST_NDBGRESET		9
-+#define SRST_P_CORE_BIU		10
-+#define SRST_PMU		11
-+
-+/* CRU-->SOFTRST_CON03 */
-+#define SRST_P_DBG		12
-+#define SRST_POT_DBG		13
-+#define SRST_P_CORE_GRF		14
-+#define SRST_CORE_EMA_DETECT	15
-+#define SRST_REF_PVTPLL_CORE	16
-+#define SRST_P_GPIO1		17
-+#define SRST_DB_GPIO1		18
-+
-+/* CRU-->SOFTRST_CON04 */
-+#define SRST_A_CORE_PERI_BIU	19
-+#define SRST_A_DSMC		20
-+#define SRST_P_DSMC		21
-+#define SRST_FLEXBUS		22
-+#define SRST_A_FLEXBUS		23
-+#define SRST_H_FLEXBUS		24
-+#define SRST_A_DSMC_SLV		25
-+#define SRST_H_DSMC_SLV		26
-+#define SRST_DSMC_SLV		27
-+
-+/* CRU-->SOFTRST_CON05 */
-+#define SRST_A_BUS_BIU		28
-+#define SRST_H_BUS_BIU		29
-+#define SRST_P_BUS_BIU		30
-+#define SRST_A_SYSRAM		31
-+#define SRST_H_SYSRAM		32
-+#define SRST_A_DMAC0		33
-+#define SRST_A_DMAC1		34
-+#define SRST_H_M0		35
-+#define SRST_M0_JTAG		36
-+#define SRST_H_CRYPTO		37
-+
-+/* CRU-->SOFTRST_CON06 */
-+#define SRST_H_RNG		38
-+#define SRST_P_BUS_GRF		39
-+#define SRST_P_TIMER0		40
-+#define SRST_TIMER0_CH0		41
-+#define SRST_TIMER0_CH1		42
-+#define SRST_TIMER0_CH2		43
-+#define SRST_TIMER0_CH3		44
-+#define SRST_TIMER0_CH4		45
-+#define SRST_TIMER0_CH5		46
-+#define SRST_P_WDT0		47
-+#define SRST_T_WDT0		48
-+#define SRST_P_WDT1		49
-+#define SRST_T_WDT1		50
-+#define SRST_P_MAILBOX		51
-+#define SRST_P_INTMUX		52
-+#define SRST_P_SPINLOCK		53
-+
-+/* CRU-->SOFTRST_CON07 */
-+#define SRST_P_DDRC		54
-+#define SRST_H_DDRPHY		55
-+#define SRST_P_DDRMON		56
-+#define SRST_DDRMON_OSC		57
-+#define SRST_P_DDR_LPC		58
-+#define SRST_H_USBOTG0		59
-+#define SRST_USBOTG0_ADP	60
-+#define SRST_H_USBOTG1		61
-+#define SRST_USBOTG1_ADP	62
-+#define SRST_P_USBPHY		63
-+#define SRST_USBPHY_POR		64
-+#define SRST_USBPHY_OTG0	65
-+#define SRST_USBPHY_OTG1	66
-+
-+/* CRU-->SOFTRST_CON08 */
-+#define SRST_A_DMA2DDR		67
-+#define SRST_P_DMA2DDR		68
-+
-+/* CRU-->SOFTRST_CON09 */
-+#define SRST_USBOTG0_UTMI	69
-+#define SRST_USBOTG1_UTMI	70
-+
-+/* CRU-->SOFTRST_CON10 */
-+#define SRST_A_DDRC_0		71
-+#define SRST_A_DDRC_1		72
-+#define SRST_A_DDR_BIU		73
-+#define SRST_DDRC		74
-+#define SRST_DDRMON		75
-+
-+/* CRU-->SOFTRST_CON11 */
-+#define SRST_H_LSPERI_BIU	76
-+#define SRST_P_UART0		77
-+#define SRST_P_UART1		78
-+#define SRST_P_UART2		79
-+#define SRST_P_UART3		80
-+#define SRST_P_UART4		81
-+#define SRST_UART0		82
-+#define SRST_UART1		83
-+#define SRST_UART2		84
-+#define SRST_UART3		85
-+#define SRST_UART4		86
-+#define SRST_P_I2C0		87
-+#define SRST_I2C0		88
-+
-+/* CRU-->SOFTRST_CON12 */
-+#define SRST_P_I2C1		89
-+#define SRST_I2C1		90
-+#define SRST_P_I2C2		91
-+#define SRST_I2C2		92
-+#define SRST_P_PWM1		93
-+#define SRST_PWM1		94
-+#define SRST_P_SPI0		95
-+#define SRST_SPI0		96
-+#define SRST_P_SPI1		97
-+#define SRST_SPI1		98
-+#define SRST_P_GPIO2		99
-+#define SRST_DB_GPIO2		100
-+
-+/* CRU-->SOFTRST_CON13 */
-+#define SRST_P_GPIO3		101
-+#define SRST_DB_GPIO3		102
-+#define SRST_P_GPIO4		103
-+#define SRST_DB_GPIO4		104
-+#define SRST_H_CAN0		105
-+#define SRST_CAN0		106
-+#define SRST_H_CAN1		107
-+#define SRST_CAN1		108
-+#define SRST_H_PDM		109
-+#define SRST_M_PDM		110
-+#define SRST_PDM		111
-+#define SRST_SPDIFTX		112
-+#define SRST_H_SPDIFTX		113
-+#define SRST_H_SPDIFRX		114
-+#define SRST_SPDIFRX		115
-+#define SRST_M_SAI0		116
-+
-+/* CRU-->SOFTRST_CON14 */
-+#define SRST_H_SAI0		117
-+#define SRST_M_SAI1		118
-+#define SRST_H_SAI1		119
-+#define SRST_H_ASRC0		120
-+#define SRST_ASRC0		121
-+#define SRST_H_ASRC1		122
-+#define SRST_ASRC1		123
-+
-+/* CRU-->SOFTRST_CON17 */
-+#define SRST_H_HSPERI_BIU	124
-+#define SRST_H_SDMMC		125
-+#define SRST_H_FSPI		126
-+#define SRST_S_FSPI		127
-+#define SRST_P_SPI2		128
-+#define SRST_A_MAC0		129
-+#define SRST_A_MAC1		130
-+
-+/* CRU-->SOFTRST_CON18 */
-+#define SRST_M_SAI2		131
-+#define SRST_H_SAI2		132
-+#define SRST_H_SAI3		133
-+#define SRST_M_SAI3		134
-+#define SRST_H_SAI4		135
-+#define SRST_M_SAI4		136
-+#define SRST_H_DSM		137
-+#define SRST_M_DSM		138
-+#define SRST_P_AUDIO_ADC	139
-+#define SRST_M_AUDIO_ADC	140
-+
-+/* CRU-->SOFTRST_CON19 */
-+#define SRST_P_SARADC		141
-+#define SRST_SARADC		142
-+#define SRST_SARADC_PHY		143
-+#define SRST_P_OTPC_NS		144
-+#define SRST_SBPI_OTPC_NS	145
-+#define SRST_USER_OTPC_NS	146
-+#define SRST_P_UART5		147
-+#define SRST_UART5		148
-+#define SRST_P_GPIO234_IOC	149
-+
-+/* CRU-->SOFTRST_CON21 */
-+#define SRST_A_VIO_BIU		150
-+#define SRST_H_VIO_BIU		151
-+#define SRST_H_RGA		152
-+#define SRST_A_RGA		153
-+#define SRST_CORE_RGA		154
-+#define SRST_A_VOP		155
-+#define SRST_H_VOP		156
-+#define SRST_VOP		157
-+#define SRST_P_DPHY		158
-+#define SRST_P_DSI_HOST		159
-+#define SRST_P_TSADC		160
-+#define SRST_TSADC		161
-+
-+/* CRU-->SOFTRST_CON22 */
-+#define SRST_P_GPIO1_IOC	162
-+
-+#endif
-diff --git a/include/dt-bindings/reset/rockchip,rk3528-cru.h b/include/dt-bindings/reset/rockchip,rk3528-cru.h
-new file mode 100644
-index 000000000000..6b024c5f2e1c
--- /dev/null
-+++ b/include/dt-bindings/reset/rockchip,rk3528-cru.h
-@@ -0,0 +1,241 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (c) 2022 Rockchip Electronics Co. Ltd.
-+ * Copyright (c) 2024 Yao Zi <ziyao@disroot.org>
-+ * Author: Joseph Chen <chenjh@rock-chips.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_ROCKCHIP_RK3528_H
-+#define _DT_BINDINGS_RESET_ROCKCHIP_RK3528_H
-+
-+#define SRST_CORE0_PO		0
-+#define SRST_CORE1_PO		1
-+#define SRST_CORE2_PO		2
-+#define SRST_CORE3_PO		3
-+#define SRST_CORE0		4
-+#define SRST_CORE1		5
-+#define SRST_CORE2		6
-+#define SRST_CORE3		7
-+#define SRST_NL2		8
-+#define SRST_CORE_BIU		9
-+#define SRST_CORE_CRYPTO	10
-+#define SRST_P_DBG		11
-+#define SRST_POT_DBG		12
-+#define SRST_NT_DBG		13
-+#define SRST_P_CORE_GRF		14
-+#define SRST_P_DAPLITE_BIU	15
-+#define SRST_P_CPU_BIU		16
-+#define SRST_REF_PVTPLL_CORE	17
-+#define SRST_A_BUS_VOPGL_BIU	18
-+#define SRST_A_BUS_H_BIU	19
-+#define SRST_A_SYSMEM_BIU	20
-+#define SRST_A_BUS_BIU		21
-+#define SRST_H_BUS_BIU		22
-+#define SRST_P_BUS_BIU		23
-+#define SRST_P_DFT2APB		24
-+#define SRST_P_BUS_GRF		25
-+#define SRST_A_BUS_M_BIU	26
-+#define SRST_A_GIC		27
-+#define SRST_A_SPINLOCK		28
-+#define SRST_A_DMAC		29
-+#define SRST_P_TIMER		30
-+#define SRST_TIMER0		31
-+#define SRST_TIMER1		32
-+#define SRST_TIMER2		33
-+#define SRST_TIMER3		34
-+#define SRST_TIMER4		35
-+#define SRST_TIMER5		36
-+#define SRST_P_JDBCK_DAP	37
-+#define SRST_JDBCK_DAP		38
-+#define SRST_P_WDT_NS		39
-+#define SRST_T_WDT_NS		40
-+#define SRST_H_TRNG_NS		41
-+#define SRST_P_UART0		42
-+#define SRST_S_UART0		43
-+#define SRST_PKA_CRYPTO		44
-+#define SRST_A_CRYPTO		45
-+#define SRST_H_CRYPTO		46
-+#define SRST_P_DMA2DDR		47
-+#define SRST_A_DMA2DDR		48
-+#define SRST_P_PWM0		49
-+#define SRST_PWM0		50
-+#define SRST_P_PWM1		51
-+#define SRST_PWM1		52
-+#define SRST_P_SCR		53
-+#define SRST_A_DCF		54
-+#define SRST_P_INTMUX		55
-+#define SRST_A_VPU_BIU		56
-+#define SRST_H_VPU_BIU		57
-+#define SRST_P_VPU_BIU		58
-+#define SRST_A_VPU		59
-+#define SRST_H_VPU		60
-+#define SRST_P_CRU_PCIE		61
-+#define SRST_P_VPU_GRF		62
-+#define SRST_H_SFC		63
-+#define SRST_S_SFC		64
-+#define SRST_C_EMMC		65
-+#define SRST_H_EMMC		66
-+#define SRST_A_EMMC		67
-+#define SRST_B_EMMC		68
-+#define SRST_T_EMMC		69
-+#define SRST_P_GPIO1		70
-+#define SRST_DB_GPIO1		71
-+#define SRST_A_VPU_L_BIU	72
-+#define SRST_P_VPU_IOC		73
-+#define SRST_H_SAI_I2S0		74
-+#define SRST_M_SAI_I2S0		75
-+#define SRST_H_SAI_I2S2		76
-+#define SRST_M_SAI_I2S2		77
-+#define SRST_P_ACODEC		78
-+#define SRST_P_GPIO3		79
-+#define SRST_DB_GPIO3		80
-+#define SRST_P_SPI1		81
-+#define SRST_SPI1		82
-+#define SRST_P_UART2		83
-+#define SRST_S_UART2		84
-+#define SRST_P_UART5		85
-+#define SRST_S_UART5		86
-+#define SRST_P_UART6		87
-+#define SRST_S_UART6		88
-+#define SRST_P_UART7		89
-+#define SRST_S_UART7		90
-+#define SRST_P_I2C3		91
-+#define SRST_I2C3		92
-+#define SRST_P_I2C5		93
-+#define SRST_I2C5		94
-+#define SRST_P_I2C6		95
-+#define SRST_I2C6		96
-+#define SRST_A_MAC		97
-+#define SRST_P_PCIE		98
-+#define SRST_PCIE_PIPE_PHY	99
-+#define SRST_PCIE_POWER_UP	100
-+#define SRST_P_PCIE_PHY		101
-+#define SRST_P_PIPE_GRF		102
-+#define SRST_H_SDIO0		103
-+#define SRST_H_SDIO1		104
-+#define SRST_TS_0		105
-+#define SRST_TS_1		106
-+#define SRST_P_CAN2		107
-+#define SRST_CAN2		108
-+#define SRST_P_CAN3		109
-+#define SRST_CAN3		110
-+#define SRST_P_SARADC		111
-+#define SRST_SARADC		112
-+#define SRST_SARADC_PHY		113
-+#define SRST_P_TSADC		114
-+#define SRST_TSADC		115
-+#define SRST_A_USB3OTG		116
-+#define SRST_A_GPU_BIU		117
-+#define SRST_P_GPU_BIU		118
-+#define SRST_A_GPU		119
-+#define SRST_REF_PVTPLL_GPU	120
-+#define SRST_H_RKVENC_BIU	121
-+#define SRST_A_RKVENC_BIU	122
-+#define SRST_P_RKVENC_BIU	123
-+#define SRST_H_RKVENC		124
-+#define SRST_A_RKVENC		125
-+#define SRST_CORE_RKVENC	126
-+#define SRST_H_SAI_I2S1		127
-+#define SRST_M_SAI_I2S1		128
-+#define SRST_P_I2C1		129
-+#define SRST_I2C1		130
-+#define SRST_P_I2C0		131
-+#define SRST_I2C0		132
-+#define SRST_P_SPI0		133
-+#define SRST_SPI0		134
-+#define SRST_P_GPIO4		135
-+#define SRST_DB_GPIO4		136
-+#define SRST_P_RKVENC_IOC	137
-+#define SRST_H_SPDIF		138
-+#define SRST_M_SPDIF		139
-+#define SRST_H_PDM		140
-+#define SRST_M_PDM		141
-+#define SRST_P_UART1		142
-+#define SRST_S_UART1		143
-+#define SRST_P_UART3		144
-+#define SRST_S_UART3		145
-+#define SRST_P_RKVENC_GRF	146
-+#define SRST_P_CAN0		147
-+#define SRST_CAN0		148
-+#define SRST_P_CAN1		149
-+#define SRST_CAN1		150
-+#define SRST_A_VO_BIU		151
-+#define SRST_H_VO_BIU		152
-+#define SRST_P_VO_BIU		153
-+#define SRST_H_RGA2E		154
-+#define SRST_A_RGA2E		155
-+#define SRST_CORE_RGA2E		156
-+#define SRST_H_VDPP		157
-+#define SRST_A_VDPP		158
-+#define SRST_CORE_VDPP		159
-+#define SRST_P_VO_GRF		160
-+#define SRST_P_CRU		161
-+#define SRST_A_VOP_BIU		162
-+#define SRST_H_VOP		163
-+#define SRST_D_VOP0		164
-+#define SRST_D_VOP1		165
-+#define SRST_A_VOP		166
-+#define SRST_P_HDMI		167
-+#define SRST_HDMI		168
-+#define SRST_P_HDMIPHY		169
-+#define SRST_H_HDCP_KEY		170
-+#define SRST_A_HDCP		171
-+#define SRST_H_HDCP		172
-+#define SRST_P_HDCP		173
-+#define SRST_H_CVBS		174
-+#define SRST_D_CVBS_VOP		175
-+#define SRST_D_4X_CVBS_VOP	176
-+#define SRST_A_JPEG_DECODER	177
-+#define SRST_H_JPEG_DECODER	178
-+#define SRST_A_VO_L_BIU		179
-+#define SRST_A_MAC_VO		180
-+#define SRST_A_JPEG_BIU		181
-+#define SRST_H_SAI_I2S3		182
-+#define SRST_M_SAI_I2S3		183
-+#define SRST_MACPHY		184
-+#define SRST_P_VCDCPHY		185
-+#define SRST_P_GPIO2		186
-+#define SRST_DB_GPIO2		187
-+#define SRST_P_VO_IOC		188
-+#define SRST_H_SDMMC0		189
-+#define SRST_P_OTPC_NS		190
-+#define SRST_SBPI_OTPC_NS	191
-+#define SRST_USER_OTPC_NS	192
-+#define SRST_HDMIHDP0		193
-+#define SRST_H_USBHOST		194
-+#define SRST_H_USBHOST_ARB	195
-+#define SRST_HOST_UTMI		196
-+#define SRST_P_UART4		197
-+#define SRST_S_UART4		198
-+#define SRST_P_I2C4		199
-+#define SRST_I2C4		200
-+#define SRST_P_I2C7		201
-+#define SRST_I2C7		202
-+#define SRST_P_USBPHY		203
-+#define SRST_USBPHY_POR		204
-+#define SRST_USBPHY_OTG		205
-+#define SRST_USBPHY_HOST	206
-+#define SRST_P_DDRPHY_CRU	207
-+#define SRST_H_RKVDEC_BIU	208
-+#define SRST_A_RKVDEC_BIU	209
-+#define SRST_A_RKVDEC		210
-+#define SRST_H_RKVDEC		211
-+#define SRST_HEVC_CA_RKVDEC	212
-+#define SRST_REF_PVTPLL_RKVDEC	213
-+#define SRST_P_DDR_BIU		214
-+#define SRST_P_DDRC		215
-+#define SRST_P_DDRMON		216
-+#define SRST_TIMER_DDRMON	217
-+#define SRST_P_MSCH_BIU		218
-+#define SRST_P_DDR_GRF		219
-+#define SRST_P_DDR_HWLP		220
-+#define SRST_P_DDRPHY		221
-+#define SRST_MSCH_BIU		222
-+#define SRST_A_DDR_UPCTL	223
-+#define SRST_DDR_UPCTL		224
-+#define SRST_DDRMON		225
-+#define SRST_A_DDR_SCRAMBLE	226
-+#define SRST_A_SPLIT		227
-+#define SRST_DDR_PHY		228
-+
-+#endif // _DT_BINDINGS_RESET_ROCKCHIP_RK3528_H
-diff --git a/include/dt-bindings/reset/rockchip,rk3562-cru.h b/include/dt-bindings/reset/rockchip,rk3562-cru.h
-new file mode 100644
-index 000000000000..8df95113056e
--- /dev/null
-+++ b/include/dt-bindings/reset/rockchip,rk3562-cru.h
-@@ -0,0 +1,358 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024-2025 Rockchip Electronics Co. Ltd.
-+ *
-+ * Author: Elaine Zhang <zhangqing@rock-chips.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_ROCKCHIP_RK3562_H
-+#define _DT_BINDINGS_RESET_ROCKCHIP_RK3562_H
-+
-+/********Name=SOFTRST_CON01,Offset=0x404********/
-+#define SRST_A_TOP_BIU			0
-+#define SRST_A_TOP_VIO_BIU		1
-+#define SRST_REF_PVTPLL_LOGIC		2
-+/********Name=SOFTRST_CON03,Offset=0x40C********/
-+#define SRST_NCOREPORESET0		3
-+#define SRST_NCOREPORESET1		4
-+#define SRST_NCOREPORESET2		5
-+#define SRST_NCOREPORESET3		6
-+#define SRST_NCORESET0			7
-+#define SRST_NCORESET1			8
-+#define SRST_NCORESET2			9
-+#define SRST_NCORESET3			10
-+#define SRST_NL2RESET			11
-+/********Name=SOFTRST_CON04,Offset=0x410********/
-+#define SRST_DAP			12
-+#define SRST_P_DBG_DAPLITE		13
-+#define SRST_REF_PVTPLL_CORE		14
-+/********Name=SOFTRST_CON05,Offset=0x414********/
-+#define SRST_A_CORE_BIU			15
-+#define SRST_P_CORE_BIU			16
-+#define SRST_H_CORE_BIU			17
-+/********Name=SOFTRST_CON06,Offset=0x418********/
-+#define SRST_A_NPU_BIU			18
-+#define SRST_H_NPU_BIU			19
-+#define SRST_A_RKNN			20
-+#define SRST_H_RKNN			21
-+#define SRST_REF_PVTPLL_NPU		22
-+/********Name=SOFTRST_CON08,Offset=0x420********/
-+#define SRST_A_GPU_BIU			23
-+#define SRST_GPU			24
-+#define SRST_REF_PVTPLL_GPU		25
-+#define SRST_GPU_BRG_BIU		26
-+/********Name=SOFTRST_CON09,Offset=0x424********/
-+#define SRST_RKVENC_CORE		27
-+#define SRST_A_VEPU_BIU			28
-+#define SRST_H_VEPU_BIU			29
-+#define SRST_A_RKVENC			30
-+#define SRST_H_RKVENC			31
-+/********Name=SOFTRST_CON10,Offset=0x428********/
-+#define SRST_RKVDEC_HEVC_CA		32
-+#define SRST_A_VDPU_BIU			33
-+#define SRST_H_VDPU_BIU			34
-+#define SRST_A_RKVDEC			35
-+#define SRST_H_RKVDEC			36
-+/********Name=SOFTRST_CON11,Offset=0x42C********/
-+#define SRST_A_VI_BIU			37
-+#define SRST_H_VI_BIU			38
-+#define SRST_P_VI_BIU			39
-+#define SRST_ISP			40
-+#define SRST_A_VICAP			41
-+#define SRST_H_VICAP			42
-+#define SRST_D_VICAP			43
-+#define SRST_I0_VICAP			44
-+#define SRST_I1_VICAP			45
-+#define SRST_I2_VICAP			46
-+#define SRST_I3_VICAP			47
-+/********Name=SOFTRST_CON12,Offset=0x430********/
-+#define SRST_P_CSIHOST0			48
-+#define SRST_P_CSIHOST1			49
-+#define SRST_P_CSIHOST2			50
-+#define SRST_P_CSIHOST3			51
-+#define SRST_P_CSIPHY0			52
-+#define SRST_P_CSIPHY1			53
-+/********Name=SOFTRST_CON13,Offset=0x434********/
-+#define SRST_A_VO_BIU			54
-+#define SRST_H_VO_BIU			55
-+#define SRST_A_VOP			56
-+#define SRST_H_VOP			57
-+#define SRST_D_VOP			58
-+#define SRST_D_VOP1			59
-+/********Name=SOFTRST_CON14,Offset=0x438********/
-+#define SRST_A_RGA_BIU			60
-+#define SRST_H_RGA_BIU			61
-+#define SRST_A_RGA			62
-+#define SRST_H_RGA			63
-+#define SRST_RGA_CORE			64
-+#define SRST_A_JDEC			65
-+#define SRST_H_JDEC			66
-+/********Name=SOFTRST_CON15,Offset=0x43C********/
-+#define SRST_B_EBK_BIU			67
-+#define SRST_P_EBK_BIU			68
-+#define SRST_AHB2AXI_EBC		69
-+#define SRST_H_EBC			70
-+#define SRST_D_EBC			71
-+#define SRST_H_EINK			72
-+#define SRST_P_EINK			73
-+/********Name=SOFTRST_CON16,Offset=0x440********/
-+#define SRST_P_PHP_BIU			74
-+#define SRST_A_PHP_BIU			75
-+#define SRST_P_PCIE20			76
-+#define SRST_PCIE20_POWERUP		77
-+#define SRST_USB3OTG			78
-+/********Name=SOFTRST_CON17,Offset=0x444********/
-+#define SRST_PIPEPHY			79
-+/********Name=SOFTRST_CON18,Offset=0x448********/
-+#define SRST_A_BUS_BIU			80
-+#define SRST_H_BUS_BIU			81
-+#define SRST_P_BUS_BIU			82
-+/********Name=SOFTRST_CON19,Offset=0x44C********/
-+#define SRST_P_I2C1			83
-+#define SRST_P_I2C2			84
-+#define SRST_P_I2C3			85
-+#define SRST_P_I2C4			86
-+#define SRST_P_I2C5			87
-+#define SRST_I2C1			88
-+#define SRST_I2C2			89
-+#define SRST_I2C3			90
-+#define SRST_I2C4			91
-+#define SRST_I2C5			92
-+/********Name=SOFTRST_CON20,Offset=0x450********/
-+#define SRST_BUS_GPIO3			93
-+#define SRST_BUS_GPIO4			94
-+/********Name=SOFTRST_CON21,Offset=0x454********/
-+#define SRST_P_TIMER			95
-+#define SRST_TIMER0			96
-+#define SRST_TIMER1			97
-+#define SRST_TIMER2			98
-+#define SRST_TIMER3			99
-+#define SRST_TIMER4			100
-+#define SRST_TIMER5			101
-+#define SRST_P_STIMER			102
-+#define SRST_STIMER0			103
-+#define SRST_STIMER1			104
-+/********Name=SOFTRST_CON22,Offset=0x458********/
-+#define SRST_P_WDTNS			105
-+#define SRST_WDTNS			106
-+#define SRST_P_GRF			107
-+#define SRST_P_SGRF			108
-+#define SRST_P_MAILBOX			109
-+#define SRST_P_INTC			110
-+#define SRST_A_BUS_GIC400		111
-+#define SRST_A_BUS_GIC400_DEBUG		112
-+/********Name=SOFTRST_CON23,Offset=0x45C********/
-+#define SRST_A_BUS_SPINLOCK		113
-+#define SRST_A_DCF			114
-+#define SRST_P_DCF			115
-+#define SRST_F_BUS_CM0_CORE		116
-+#define SRST_T_BUS_CM0_JTAG		117
-+#define SRST_H_ICACHE			118
-+#define SRST_H_DCACHE			119
-+/********Name=SOFTRST_CON24,Offset=0x460********/
-+#define SRST_P_TSADC			120
-+#define SRST_TSADC			121
-+#define SRST_TSADCPHY			122
-+#define SRST_P_DFT2APB			123
-+/********Name=SOFTRST_CON25,Offset=0x464********/
-+#define SRST_A_GMAC			124
-+#define SRST_P_APB2ASB_VCCIO156		125
-+#define SRST_P_DSIPHY			126
-+#define SRST_P_DSITX			127
-+#define SRST_P_CPU_EMA_DET		128
-+#define SRST_P_HASH			129
-+#define SRST_P_TOPCRU			130
-+/********Name=SOFTRST_CON26,Offset=0x468********/
-+#define SRST_P_ASB2APB_VCCIO156		131
-+#define SRST_P_IOC_VCCIO156		132
-+#define SRST_P_GPIO3_VCCIO156		133
-+#define SRST_P_GPIO4_VCCIO156		134
-+#define SRST_P_SARADC_VCCIO156		135
-+#define SRST_SARADC_VCCIO156		136
-+#define SRST_SARADC_VCCIO156_PHY	137
-+/********Name=SOFTRST_CON27,Offset=0x46c********/
-+#define SRST_A_MAC100			138
-+
-+/********Name=PMU0SOFTRST_CON00,Offset=0x10200********/
-+#define SRST_P_PMU0_CRU			139
-+#define SRST_P_PMU0_PMU			140
-+#define SRST_PMU0_PMU			141
-+#define SRST_P_PMU0_HP_TIMER		142
-+#define SRST_PMU0_HP_TIMER		143
-+#define SRST_PMU0_32K_HP_TIMER		144
-+#define SRST_P_PMU0_PVTM		145
-+#define SRST_PMU0_PVTM			146
-+#define SRST_P_IOC_PMUIO		147
-+#define SRST_P_PMU0_GPIO0		148
-+#define SRST_PMU0_GPIO0			149
-+#define SRST_P_PMU0_GRF			150
-+#define SRST_P_PMU0_SGRF		151
-+/********Name=PMU0SOFTRST_CON01,Offset=0x10204********/
-+#define SRST_DDR_FAIL_SAFE		152
-+#define SRST_P_PMU0_SCRKEYGEN		153
-+/********Name=PMU0SOFTRST_CON02,Offset=0x10208********/
-+#define SRST_P_PMU0_I2C0		154
-+#define SRST_PMU0_I2C0			155
-+
-+/********Name=PMU1SOFTRST_CON00,Offset=0x18200********/
-+#define SRST_P_PMU1_CRU			156
-+#define SRST_H_PMU1_MEM			157
-+#define SRST_H_PMU1_BIU			158
-+#define SRST_P_PMU1_BIU			159
-+#define SRST_P_PMU1_UART0		160
-+#define SRST_S_PMU1_UART0		161
-+/********Name=PMU1SOFTRST_CON01,Offset=0x18204********/
-+#define SRST_P_PMU1_SPI0		162
-+#define SRST_PMU1_SPI0			163
-+#define SRST_P_PMU1_PWM0		164
-+#define SRST_PMU1_PWM0			165
-+/********Name=PMU1SOFTRST_CON02,Offset=0x18208********/
-+#define SRST_F_PMU1_CM0_CORE		166
-+#define SRST_T_PMU1_CM0_JTAG		167
-+#define SRST_P_PMU1_WDTNS		168
-+#define SRST_PMU1_WDTNS			169
-+#define SRST_PMU1_MAILBOX		170
-+
-+/********Name=DDRSOFTRST_CON00,Offset=0x20200********/
-+#define SRST_MSCH_BRG_BIU		171
-+#define SRST_P_MSCH_BIU			172
-+#define SRST_P_DDR_HWLP			173
-+#define SRST_P_DDR_PHY			290
-+#define SRST_P_DDR_DFICTL		174
-+#define SRST_P_DDR_DMA2DDR		175
-+/********Name=DDRSOFTRST_CON01,Offset=0x20204********/
-+#define SRST_P_DDR_MON			176
-+#define SRST_TM_DDR_MON			177
-+#define SRST_P_DDR_GRF			178
-+#define SRST_P_DDR_CRU			179
-+#define SRST_P_SUBDDR_CRU		180
-+
-+/********Name=SUBDDRSOFTRST_CON00,Offset=0x28200********/
-+#define SRST_MSCH_BIU			181
-+#define SRST_DDR_PHY			182
-+#define SRST_DDR_DFICTL			183
-+#define SRST_DDR_SCRAMBLE		184
-+#define SRST_DDR_MON			185
-+#define SRST_A_DDR_SPLIT		186
-+#define SRST_DDR_DMA2DDR		187
-+
-+/********Name=PERISOFTRST_CON01,Offset=0x30404********/
-+#define SRST_A_PERI_BIU			188
-+#define SRST_H_PERI_BIU			189
-+#define SRST_P_PERI_BIU			190
-+#define SRST_P_PERICRU			191
-+/********Name=PERISOFTRST_CON02,Offset=0x30408********/
-+#define SRST_H_SAI0_8CH			192
-+#define SRST_M_SAI0_8CH			193
-+#define SRST_H_SAI1_8CH			194
-+#define SRST_M_SAI1_8CH			195
-+#define SRST_H_SAI2_2CH			196
-+#define SRST_M_SAI2_2CH			197
-+/********Name=PERISOFTRST_CON03,Offset=0x3040C********/
-+#define SRST_H_DSM			198
-+#define SRST_DSM			199
-+#define SRST_H_PDM			200
-+#define SRST_M_PDM			201
-+#define SRST_H_SPDIF			202
-+#define SRST_M_SPDIF			203
-+/********Name=PERISOFTRST_CON04,Offset=0x30410********/
-+#define SRST_H_SDMMC0			204
-+#define SRST_H_SDMMC1			205
-+#define SRST_H_EMMC			206
-+#define SRST_A_EMMC			207
-+#define SRST_C_EMMC			208
-+#define SRST_B_EMMC			209
-+#define SRST_T_EMMC			210
-+#define SRST_S_SFC			211
-+#define SRST_H_SFC			212
-+/********Name=PERISOFTRST_CON05,Offset=0x30414********/
-+#define SRST_H_USB2HOST			213
-+#define SRST_H_USB2HOST_ARB		214
-+#define SRST_USB2HOST_UTMI		215
-+/********Name=PERISOFTRST_CON06,Offset=0x30418********/
-+#define SRST_P_SPI1			216
-+#define SRST_SPI1			217
-+#define SRST_P_SPI2			218
-+#define SRST_SPI2			219
-+/********Name=PERISOFTRST_CON07,Offset=0x3041C********/
-+#define SRST_P_UART1			220
-+#define SRST_P_UART2			221
-+#define SRST_P_UART3			222
-+#define SRST_P_UART4			223
-+#define SRST_P_UART5			224
-+#define SRST_P_UART6			225
-+#define SRST_P_UART7			226
-+#define SRST_P_UART8			227
-+#define SRST_P_UART9			228
-+#define SRST_S_UART1			229
-+#define SRST_S_UART2			230
-+/********Name=PERISOFTRST_CON08,Offset=0x30420********/
-+#define SRST_S_UART3			231
-+#define SRST_S_UART4			232
-+#define SRST_S_UART5			233
-+#define SRST_S_UART6			234
-+#define SRST_S_UART7			235
-+/********Name=PERISOFTRST_CON09,Offset=0x30424********/
-+#define SRST_S_UART8			236
-+#define SRST_S_UART9			237
-+/********Name=PERISOFTRST_CON10,Offset=0x30428********/
-+#define SRST_P_PWM1_PERI		238
-+#define SRST_PWM1_PERI			239
-+#define SRST_P_PWM2_PERI		240
-+#define SRST_PWM2_PERI			241
-+#define SRST_P_PWM3_PERI		242
-+#define SRST_PWM3_PERI			243
-+/********Name=PERISOFTRST_CON11,Offset=0x3042C********/
-+#define SRST_P_CAN0			244
-+#define SRST_CAN0			245
-+#define SRST_P_CAN1			246
-+#define SRST_CAN1			247
-+/********Name=PERISOFTRST_CON12,Offset=0x30430********/
-+#define SRST_A_CRYPTO			248
-+#define SRST_H_CRYPTO			249
-+#define SRST_P_CRYPTO			250
-+#define SRST_CORE_CRYPTO		251
-+#define SRST_PKA_CRYPTO			252
-+#define SRST_H_KLAD			253
-+#define SRST_P_KEY_READER		254
-+#define SRST_H_RK_RNG_NS		255
-+#define SRST_H_RK_RNG_S			256
-+#define SRST_H_TRNG_NS			257
-+#define SRST_H_TRNG_S			258
-+#define SRST_H_CRYPTO_S			259
-+/********Name=PERISOFTRST_CON13,Offset=0x30434********/
-+#define SRST_P_PERI_WDT			260
-+#define SRST_T_PERI_WDT			261
-+#define SRST_A_SYSMEM			262
-+#define SRST_H_BOOTROM			263
-+#define SRST_P_PERI_GRF			264
-+#define SRST_A_DMAC			265
-+#define SRST_A_RKDMAC			267
-+/********Name=PERISOFTRST_CON14,Offset=0x30438********/
-+#define SRST_P_OTPC_NS			268
-+#define SRST_SBPI_OTPC_NS		269
-+#define SRST_USER_OTPC_NS		270
-+#define SRST_P_OTPC_S			271
-+#define SRST_SBPI_OTPC_S		272
-+#define SRST_USER_OTPC_S		273
-+#define SRST_OTPC_ARB			274
-+#define SRST_P_OTPPHY			275
-+#define SRST_OTP_NPOR			276
-+/********Name=PERISOFTRST_CON15,Offset=0x3043C********/
-+#define SRST_P_USB2PHY			277
-+#define SRST_USB2PHY_POR		278
-+#define SRST_USB2PHY_OTG		279
-+#define SRST_USB2PHY_HOST		280
-+#define SRST_P_PIPEPHY			281
-+/********Name=PERISOFTRST_CON16,Offset=0x30440********/
-+#define SRST_P_SARADC			282
-+#define SRST_SARADC			283
-+#define SRST_SARADC_PHY			284
-+#define SRST_P_IOC_VCCIO234		285
-+/********Name=PERISOFTRST_CON17,Offset=0x30444********/
-+#define SRST_P_PERI_GPIO1		286
-+#define SRST_P_PERI_GPIO2		287
-+#define SRST_PERI_GPIO1			288
-+#define SRST_PERI_GPIO2			289
-+
-+#endif
-diff --git a/include/dt-bindings/reset/rockchip,rk3576-cru.h b/include/dt-bindings/reset/rockchip,rk3576-cru.h
-new file mode 100644
-index 000000000000..ae856906f3a3
--- /dev/null
-+++ b/include/dt-bindings/reset/rockchip,rk3576-cru.h
-@@ -0,0 +1,564 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
-+/*
-+ * Copyright (c) 2023 Rockchip Electronics Co. Ltd.
-+ * Copyright (c) 2024 Collabora Ltd.
-+ *
-+ * Author: Elaine Zhang <zhangqing@rock-chips.com>
-+ * Author: Detlev Casanova <detlev.casanova@collabora.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_ROCKCHIP_RK3576_H
-+#define _DT_BINDINGS_RESET_ROCKCHIP_RK3576_H
-+
-+#define SRST_A_TOP_BIU			0
-+#define SRST_P_TOP_BIU			1
-+#define SRST_A_TOP_MID_BIU		2
-+#define SRST_A_SECURE_HIGH_BIU		3
-+#define SRST_H_TOP_BIU			4
-+
-+#define SRST_H_VO0VOP_CHANNEL_BIU	5
-+#define SRST_A_VO0VOP_CHANNEL_BIU	6
-+
-+#define SRST_BISRINTF			7
-+
-+#define SRST_H_AUDIO_BIU		8
-+#define SRST_H_ASRC_2CH_0		9
-+#define SRST_H_ASRC_2CH_1		10
-+#define SRST_H_ASRC_4CH_0		11
-+#define SRST_H_ASRC_4CH_1		12
-+#define SRST_ASRC_2CH_0			13
-+#define SRST_ASRC_2CH_1			14
-+#define SRST_ASRC_4CH_0			15
-+#define SRST_ASRC_4CH_1			16
-+#define SRST_M_SAI0_8CH			17
-+#define SRST_H_SAI0_8CH			18
-+#define SRST_H_SPDIF_RX0		19
-+#define SRST_M_SPDIF_RX0		20
-+
-+#define SRST_H_SPDIF_RX1		21
-+#define SRST_M_SPDIF_RX1		22
-+#define SRST_M_SAI1_8CH			23
-+#define SRST_H_SAI1_8CH			24
-+#define SRST_M_SAI2_2CH			25
-+#define SRST_H_SAI2_2CH			26
-+#define SRST_M_SAI3_2CH			27
-+#define SRST_H_SAI3_2CH			28
-+
-+#define SRST_M_SAI4_2CH			29
-+#define SRST_H_SAI4_2CH			30
-+#define SRST_H_ACDCDIG_DSM		31
-+#define SRST_M_ACDCDIG_DSM		32
-+#define SRST_PDM1			33
-+#define SRST_H_PDM1			34
-+#define SRST_M_PDM1			35
-+#define SRST_H_SPDIF_TX0		36
-+#define SRST_M_SPDIF_TX0		37
-+#define SRST_H_SPDIF_TX1		38
-+#define SRST_M_SPDIF_TX1		39
-+
-+#define SRST_A_BUS_BIU			40
-+#define SRST_P_BUS_BIU			41
-+#define SRST_P_CRU			42
-+#define SRST_H_CAN0			43
-+#define SRST_CAN0			44
-+#define SRST_H_CAN1			45
-+#define SRST_CAN1			46
-+#define SRST_P_INTMUX2BUS		47
-+#define SRST_P_VCCIO_IOC		48
-+#define SRST_H_BUS_BIU			49
-+#define SRST_KEY_SHIFT			50
-+
-+#define SRST_P_I2C1			51
-+#define SRST_P_I2C2			52
-+#define SRST_P_I2C3			53
-+#define SRST_P_I2C4			54
-+#define SRST_P_I2C5			55
-+#define SRST_P_I2C6			56
-+#define SRST_P_I2C7			57
-+#define SRST_P_I2C8			58
-+#define SRST_P_I2C9			59
-+#define SRST_P_WDT_BUSMCU		60
-+#define SRST_T_WDT_BUSMCU		61
-+#define SRST_A_GIC			62
-+#define SRST_I2C1			63
-+#define SRST_I2C2			64
-+#define SRST_I2C3			65
-+#define SRST_I2C4			66
-+
-+#define SRST_I2C5			67
-+#define SRST_I2C6			68
-+#define SRST_I2C7			69
-+#define SRST_I2C8			70
-+#define SRST_I2C9			71
-+#define SRST_P_SARADC			72
-+#define SRST_SARADC			73
-+#define SRST_P_TSADC			74
-+#define SRST_TSADC			75
-+#define SRST_P_UART0			76
-+#define SRST_P_UART2			77
-+#define SRST_P_UART3			78
-+#define SRST_P_UART4			79
-+#define SRST_P_UART5			80
-+#define SRST_P_UART6			81
-+
-+#define SRST_P_UART7			82
-+#define SRST_P_UART8			83
-+#define SRST_P_UART9			84
-+#define SRST_P_UART10			85
-+#define SRST_P_UART11			86
-+#define SRST_S_UART0			87
-+#define SRST_S_UART2			88
-+#define SRST_S_UART3			89
-+#define SRST_S_UART4			90
-+#define SRST_S_UART5			91
-+
-+#define SRST_S_UART6			92
-+#define SRST_S_UART7			93
-+#define SRST_S_UART8			94
-+#define SRST_S_UART9			95
-+#define SRST_S_UART10			96
-+#define SRST_S_UART11			97
-+#define SRST_P_SPI0			98
-+#define SRST_P_SPI1			99
-+#define SRST_P_SPI2			100
-+
-+#define SRST_P_SPI3			101
-+#define SRST_P_SPI4			102
-+#define SRST_SPI0			103
-+#define SRST_SPI1			104
-+#define SRST_SPI2			105
-+#define SRST_SPI3			106
-+#define SRST_SPI4			107
-+#define SRST_P_WDT0			108
-+#define SRST_T_WDT0			109
-+#define SRST_P_SYS_GRF			110
-+#define SRST_P_PWM1			111
-+#define SRST_PWM1			112
-+
-+#define SRST_P_BUSTIMER0		113
-+#define SRST_P_BUSTIMER1		114
-+#define SRST_TIMER0			115
-+#define SRST_TIMER1			116
-+#define SRST_TIMER2			117
-+#define SRST_TIMER3			118
-+#define SRST_TIMER4			119
-+#define SRST_TIMER5			120
-+#define SRST_P_BUSIOC			121
-+#define SRST_P_MAILBOX0			122
-+#define SRST_P_GPIO1			123
-+
-+#define SRST_GPIO1			124
-+#define SRST_P_GPIO2			125
-+#define SRST_GPIO2			126
-+#define SRST_P_GPIO3			127
-+#define SRST_GPIO3			128
-+#define SRST_P_GPIO4			129
-+#define SRST_GPIO4			130
-+#define SRST_A_DECOM			131
-+#define SRST_P_DECOM			132
-+#define SRST_D_DECOM			133
-+#define SRST_TIMER6			134
-+#define SRST_TIMER7			135
-+#define SRST_TIMER8			136
-+#define SRST_TIMER9			137
-+#define SRST_TIMER10			138
-+
-+#define SRST_TIMER11			139
-+#define SRST_A_DMAC0			140
-+#define SRST_A_DMAC1			141
-+#define SRST_A_DMAC2			142
-+#define SRST_A_SPINLOCK			143
-+#define SRST_REF_PVTPLL_BUS		144
-+#define SRST_H_I3C0			145
-+#define SRST_H_I3C1			146
-+#define SRST_H_BUS_CM0_BIU		147
-+#define SRST_F_BUS_CM0_CORE		148
-+#define SRST_T_BUS_CM0_JTAG		149
-+
-+#define SRST_P_INTMUX2PMU		150
-+#define SRST_P_INTMUX2DDR		151
-+#define SRST_P_PVTPLL_BUS		152
-+#define SRST_P_PWM2			153
-+#define SRST_PWM2			154
-+#define SRST_FREQ_PWM1			155
-+#define SRST_COUNTER_PWM1		156
-+#define SRST_I3C0			157
-+#define SRST_I3C1			158
-+
-+#define SRST_P_DDR_MON_CH0		159
-+#define SRST_P_DDR_BIU			160
-+#define SRST_P_DDR_UPCTL_CH0		161
-+#define SRST_TM_DDR_MON_CH0		162
-+#define SRST_A_DDR_BIU			163
-+#define SRST_DFI_CH0			164
-+#define SRST_DDR_MON_CH0		165
-+#define SRST_P_DDR_HWLP_CH0		166
-+#define SRST_P_DDR_MON_CH1		167
-+#define SRST_P_DDR_HWLP_CH1		168
-+
-+#define SRST_P_DDR_UPCTL_CH1		169
-+#define SRST_TM_DDR_MON_CH1		170
-+#define SRST_DFI_CH1			171
-+#define SRST_A_DDR01_MSCH0		172
-+#define SRST_A_DDR01_MSCH1		173
-+#define SRST_DDR_MON_CH1		174
-+#define SRST_DDR_SCRAMBLE_CH0		175
-+#define SRST_DDR_SCRAMBLE_CH1		176
-+#define SRST_P_AHB2APB			177
-+#define SRST_H_AHB2APB			178
-+#define SRST_H_DDR_BIU			179
-+#define SRST_F_DDR_CM0_CORE		180
-+
-+#define SRST_P_DDR01_MSCH0		181
-+#define SRST_P_DDR01_MSCH1		182
-+#define SRST_DDR_TIMER0			183
-+#define SRST_DDR_TIMER1			184
-+#define SRST_T_WDT_DDR			185
-+#define SRST_P_WDT			186
-+#define SRST_P_TIMER			187
-+#define SRST_T_DDR_CM0_JTAG		188
-+#define SRST_P_DDR_GRF			189
-+
-+#define SRST_DDR_UPCTL_CH0		190
-+#define SRST_A_DDR_UPCTL_0_CH0		191
-+#define SRST_A_DDR_UPCTL_1_CH0		192
-+#define SRST_A_DDR_UPCTL_2_CH0		193
-+#define SRST_A_DDR_UPCTL_3_CH0		194
-+#define SRST_A_DDR_UPCTL_4_CH0		195
-+
-+#define SRST_DDR_UPCTL_CH1		196
-+#define SRST_A_DDR_UPCTL_0_CH1		197
-+#define SRST_A_DDR_UPCTL_1_CH1		198
-+#define SRST_A_DDR_UPCTL_2_CH1		199
-+#define SRST_A_DDR_UPCTL_3_CH1		200
-+#define SRST_A_DDR_UPCTL_4_CH1		201
-+
-+#define SRST_REF_PVTPLL_DDR		202
-+#define SRST_P_PVTPLL_DDR		203
-+
-+#define SRST_A_RKNN0			204
-+#define SRST_A_RKNN0_BIU		205
-+#define SRST_L_RKNN0_BIU		206
-+
-+#define SRST_A_RKNN1			207
-+#define SRST_A_RKNN1_BIU		208
-+#define SRST_L_RKNN1_BIU		209
-+
-+#define SRST_NPU_DAP			210
-+#define SRST_L_NPUSUBSYS_BIU		211
-+#define SRST_P_NPUTOP_BIU		212
-+#define SRST_P_NPU_TIMER		213
-+#define SRST_NPUTIMER0			214
-+#define SRST_NPUTIMER1			215
-+#define SRST_P_NPU_WDT			216
-+#define SRST_T_NPU_WDT			217
-+
-+#define SRST_A_RKNN_CBUF		218
-+#define SRST_A_RVCORE0			219
-+#define SRST_P_NPU_GRF			220
-+#define SRST_P_PVTPLL_NPU		221
-+#define SRST_NPU_PVTPLL			222
-+#define SRST_H_NPU_CM0_BIU		223
-+#define SRST_F_NPU_CM0_CORE		224
-+#define SRST_T_NPU_CM0_JTAG		225
-+#define SRST_A_RKNNTOP_BIU		226
-+#define SRST_H_RKNN_CBUF		227
-+#define SRST_H_RKNNTOP_BIU		228
-+
-+#define SRST_H_NVM_BIU			229
-+#define SRST_A_NVM_BIU			230
-+#define SRST_S_FSPI			231
-+#define SRST_H_FSPI			232
-+#define SRST_C_EMMC			233
-+#define SRST_H_EMMC			234
-+#define SRST_A_EMMC			235
-+#define SRST_B_EMMC			236
-+#define SRST_T_EMMC			237
-+
-+#define SRST_P_GRF			238
-+#define SRST_P_PHP_BIU			239
-+#define SRST_A_PHP_BIU			240
-+#define SRST_P_PCIE0			241
-+#define SRST_PCIE0_POWER_UP		242
-+
-+#define SRST_A_USB3OTG1			243
-+#define SRST_A_MMU0			244
-+#define SRST_A_SLV_MMU0			245
-+#define SRST_A_MMU1			246
-+
-+#define SRST_A_SLV_MMU1			247
-+#define SRST_P_PCIE1			248
-+#define SRST_PCIE1_POWER_UP		249
-+
-+#define SRST_RXOOB0			250
-+#define SRST_RXOOB1			251
-+#define SRST_PMALIVE0			252
-+#define SRST_PMALIVE1			253
-+#define SRST_A_SATA0			254
-+#define SRST_A_SATA1			255
-+#define SRST_ASIC1			256
-+#define SRST_ASIC0			257
-+
-+#define SRST_P_CSIDPHY1			258
-+#define SRST_SCAN_CSIDPHY1		259
-+
-+#define SRST_P_SDGMAC_GRF		260
-+#define SRST_P_SDGMAC_BIU		261
-+#define SRST_A_SDGMAC_BIU		262
-+#define SRST_H_SDGMAC_BIU		263
-+#define SRST_A_GMAC0			264
-+#define SRST_A_GMAC1			265
-+#define SRST_P_GMAC0			266
-+#define SRST_P_GMAC1			267
-+#define SRST_H_SDIO			268
-+
-+#define SRST_H_SDMMC0			269
-+#define SRST_S_FSPI1			270
-+#define SRST_H_FSPI1			271
-+#define SRST_A_DSMC_BIU			272
-+#define SRST_A_DSMC			273
-+#define SRST_P_DSMC			274
-+#define SRST_H_HSGPIO			275
-+#define SRST_HSGPIO			276
-+#define SRST_A_HSGPIO			277
-+
-+#define SRST_H_RKVDEC			278
-+#define SRST_H_RKVDEC_BIU		279
-+#define SRST_A_RKVDEC_BIU		280
-+#define SRST_RKVDEC_HEVC_CA		281
-+#define SRST_RKVDEC_CORE		282
-+
-+#define SRST_A_USB_BIU			283
-+#define SRST_P_USBUFS_BIU		284
-+#define SRST_A_USB3OTG0			285
-+#define SRST_A_UFS_BIU			286
-+#define SRST_A_MMU2			287
-+#define SRST_A_SLV_MMU2			288
-+#define SRST_A_UFS_SYS			289
-+
-+#define SRST_A_UFS			290
-+#define SRST_P_USBUFS_GRF		291
-+#define SRST_P_UFS_GRF			292
-+
-+#define SRST_H_VPU_BIU			293
-+#define SRST_A_JPEG_BIU			294
-+#define SRST_A_RGA_BIU			295
-+#define SRST_A_VDPP_BIU			296
-+#define SRST_A_EBC_BIU			297
-+#define SRST_H_RGA2E_0			298
-+#define SRST_A_RGA2E_0			299
-+#define SRST_CORE_RGA2E_0		300
-+
-+#define SRST_A_JPEG			301
-+#define SRST_H_JPEG			302
-+#define SRST_H_VDPP			303
-+#define SRST_A_VDPP			304
-+#define SRST_CORE_VDPP			305
-+#define SRST_H_RGA2E_1			306
-+#define SRST_A_RGA2E_1			307
-+#define SRST_CORE_RGA2E_1		308
-+#define SRST_H_EBC			309
-+#define SRST_A_EBC			310
-+#define SRST_D_EBC			311
-+
-+#define SRST_H_VEPU0_BIU		312
-+#define SRST_A_VEPU0_BIU		313
-+#define SRST_H_VEPU0			314
-+#define SRST_A_VEPU0			315
-+#define SRST_VEPU0_CORE			316
-+
-+#define SRST_A_VI_BIU			317
-+#define SRST_H_VI_BIU			318
-+#define SRST_P_VI_BIU			319
-+#define SRST_D_VICAP			320
-+#define SRST_A_VICAP			321
-+#define SRST_H_VICAP			322
-+#define SRST_ISP0			323
-+#define SRST_ISP0_VICAP			324
-+
-+#define SRST_CORE_VPSS			325
-+#define SRST_P_CSI_HOST_0		326
-+#define SRST_P_CSI_HOST_1		327
-+#define SRST_P_CSI_HOST_2		328
-+#define SRST_P_CSI_HOST_3		329
-+#define SRST_P_CSI_HOST_4		330
-+
-+#define SRST_CIFIN			331
-+#define SRST_VICAP_I0CLK		332
-+#define SRST_VICAP_I1CLK		333
-+#define SRST_VICAP_I2CLK		334
-+#define SRST_VICAP_I3CLK		335
-+#define SRST_VICAP_I4CLK		336
-+
-+#define SRST_A_VOP_BIU			337
-+#define SRST_A_VOP2_BIU			338
-+#define SRST_H_VOP_BIU			339
-+#define SRST_P_VOP_BIU			340
-+#define SRST_H_VOP			341
-+#define SRST_A_VOP			342
-+#define SRST_D_VP0			343
-+
-+#define SRST_D_VP1			344
-+#define SRST_D_VP2			345
-+#define SRST_P_VOP2_BIU			346
-+#define SRST_P_VOPGRF			347
-+
-+#define SRST_H_VO0_BIU			348
-+#define SRST_P_VO0_BIU			349
-+#define SRST_A_HDCP0_BIU		350
-+#define SRST_P_VO0_GRF			351
-+#define SRST_A_HDCP0			352
-+#define SRST_H_HDCP0			353
-+#define SRST_HDCP0			354
-+
-+#define SRST_P_DSIHOST0			355
-+#define SRST_DSIHOST0			356
-+#define SRST_P_HDMITX0			357
-+#define SRST_HDMITX0_REF		358
-+#define SRST_P_EDP0			359
-+#define SRST_EDP0_24M			360
-+
-+#define SRST_M_SAI5_8CH			361
-+#define SRST_H_SAI5_8CH			362
-+#define SRST_M_SAI6_8CH			363
-+#define SRST_H_SAI6_8CH			364
-+#define SRST_H_SPDIF_TX2		365
-+#define SRST_M_SPDIF_TX2		366
-+#define SRST_H_SPDIF_RX2		367
-+#define SRST_M_SPDIF_RX2		368
-+
-+#define SRST_H_SAI8_8CH			369
-+#define SRST_M_SAI8_8CH			370
-+
-+#define SRST_H_VO1_BIU			371
-+#define SRST_P_VO1_BIU			372
-+#define SRST_M_SAI7_8CH			373
-+#define SRST_H_SAI7_8CH			374
-+#define SRST_H_SPDIF_TX3		375
-+#define SRST_H_SPDIF_TX4		376
-+#define SRST_H_SPDIF_TX5		377
-+#define SRST_M_SPDIF_TX3		378
-+
-+#define SRST_DP0			379
-+#define SRST_P_VO1_GRF			380
-+#define SRST_A_HDCP1_BIU		381
-+#define SRST_A_HDCP1			382
-+#define SRST_H_HDCP1			383
-+#define SRST_HDCP1			384
-+#define SRST_H_SAI9_8CH			385
-+#define SRST_M_SAI9_8CH			386
-+#define SRST_M_SPDIF_TX4		387
-+#define SRST_M_SPDIF_TX5		388
-+
-+#define SRST_GPU			389
-+#define SRST_A_S_GPU_BIU		390
-+#define SRST_A_M0_GPU_BIU		391
-+#define SRST_P_GPU_BIU			392
-+#define SRST_P_GPU_GRF			393
-+#define SRST_GPU_PVTPLL			394
-+#define SRST_P_PVTPLL_GPU		395
-+
-+#define SRST_A_CENTER_BIU		396
-+#define SRST_A_DMA2DDR			397
-+#define SRST_A_DDR_SHAREMEM		398
-+#define SRST_A_DDR_SHAREMEM_BIU		399
-+#define SRST_H_CENTER_BIU		400
-+#define SRST_P_CENTER_GRF		401
-+#define SRST_P_DMA2DDR			402
-+#define SRST_P_SHAREMEM			403
-+#define SRST_P_CENTER_BIU		404
-+
-+#define SRST_LINKSYM_HDMITXPHY0		405
-+
-+#define SRST_DP0_PIXELCLK		406
-+#define SRST_PHY_DP0_TX			407
-+#define SRST_DP1_PIXELCLK		408
-+#define SRST_DP2_PIXELCLK		409
-+
-+#define SRST_H_VEPU1_BIU		410
-+#define SRST_A_VEPU1_BIU		411
-+#define SRST_H_VEPU1			412
-+#define SRST_A_VEPU1			413
-+#define SRST_VEPU1_CORE			414
-+
-+#define SRST_P_PHPPHY_CRU		415
-+#define SRST_P_APB2ASB_SLV_CHIP_TOP	416
-+#define SRST_P_PCIE2_COMBOPHY0		417
-+#define SRST_P_PCIE2_COMBOPHY0_GRF	418
-+#define SRST_P_PCIE2_COMBOPHY1		419
-+#define SRST_P_PCIE2_COMBOPHY1_GRF	420
-+
-+#define SRST_PCIE0_PIPE_PHY		421
-+#define SRST_PCIE1_PIPE_PHY		422
-+
-+#define SRST_H_CRYPTO_NS		423
-+#define SRST_H_TRNG_NS			424
-+#define SRST_P_OTPC_NS			425
-+#define SRST_OTPC_NS			426
-+
-+#define SRST_P_HDPTX_GRF		427
-+#define SRST_P_HDPTX_APB		428
-+#define SRST_P_MIPI_DCPHY		429
-+#define SRST_P_DCPHY_GRF		430
-+#define SRST_P_BOT0_APB2ASB		431
-+#define SRST_P_BOT1_APB2ASB		432
-+#define SRST_USB2DEBUG			433
-+#define SRST_P_CSIPHY_GRF		434
-+#define SRST_P_CSIPHY			435
-+#define SRST_P_USBPHY_GRF_0		436
-+#define SRST_P_USBPHY_GRF_1		437
-+#define SRST_P_USBDP_GRF		438
-+#define SRST_P_USBDPPHY			439
-+#define SRST_USBDP_COMBO_PHY_INIT	440
-+
-+#define SRST_USBDP_COMBO_PHY_CMN	441
-+#define SRST_USBDP_COMBO_PHY_LANE	442
-+#define SRST_USBDP_COMBO_PHY_PCS	443
-+#define SRST_M_MIPI_DCPHY		444
-+#define SRST_S_MIPI_DCPHY		445
-+#define SRST_SCAN_CSIPHY		446
-+#define SRST_P_VCCIO6_IOC		447
-+#define SRST_OTGPHY_0			448
-+#define SRST_OTGPHY_1			449
-+#define SRST_HDPTX_INIT			450
-+#define SRST_HDPTX_CMN			451
-+#define SRST_HDPTX_LANE			452
-+#define SRST_HDMITXHDP			453
-+
-+#define SRST_MPHY_INIT			454
-+#define SRST_P_MPHY_GRF			455
-+#define SRST_P_VCCIO7_IOC		456
-+
-+#define SRST_H_PMU1_BIU			457
-+#define SRST_P_PMU1_NIU			458
-+#define SRST_H_PMU_CM0_BIU		459
-+#define SRST_PMU_CM0_CORE		460
-+#define SRST_PMU_CM0_JTAG		461
-+
-+#define SRST_P_CRU_PMU1			462
-+#define SRST_P_PMU1_GRF			463
-+#define SRST_P_PMU1_IOC			464
-+#define SRST_P_PMU1WDT			465
-+#define SRST_T_PMU1WDT			466
-+#define SRST_P_PMUTIMER			467
-+#define SRST_PMUTIMER0			468
-+#define SRST_PMUTIMER1			469
-+#define SRST_P_PMU1PWM			470
-+#define SRST_PMU1PWM			471
-+
-+#define SRST_P_I2C0			472
-+#define SRST_I2C0			473
-+#define SRST_S_UART1			474
-+#define SRST_P_UART1			475
-+#define SRST_PDM0			476
-+#define SRST_H_PDM0			477
-+
-+#define SRST_M_PDM0			478
-+#define SRST_H_VAD			479
-+
-+#define SRST_P_PMU0GRF			480
-+#define SRST_P_PMU0IOC			481
-+#define SRST_P_GPIO0			482
-+#define SRST_DB_GPIO0			483
-+
-+#endif
-diff --git a/include/dt-bindings/reset/rockchip,rk3588-cru.h b/include/dt-bindings/reset/rockchip,rk3588-cru.h
-new file mode 100644
-index 000000000000..878beae6dc3b
--- /dev/null
-+++ b/include/dt-bindings/reset/rockchip,rk3588-cru.h
-@@ -0,0 +1,795 @@
-+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
-+/*
-+ * Copyright (c) 2021, 2024 Rockchip Electronics Co. Ltd.
-+ * Copyright (c) 2022 Collabora Ltd.
-+ *
-+ * Author: Elaine Zhang <zhangqing@rock-chips.com>
-+ * Author: Sebastian Reichel <sebastian.reichel@collabora.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_ROCKCHIP_RK3588_H
-+#define _DT_BINDINGS_RESET_ROCKCHIP_RK3588_H
-+
-+#define SRST_A_TOP_BIU			0
-+#define SRST_P_TOP_BIU			1
-+#define SRST_P_CSIPHY0			2
-+#define SRST_CSIPHY0			3
-+#define SRST_P_CSIPHY1			4
-+#define SRST_CSIPHY1			5
-+#define SRST_A_TOP_M500_BIU		6
-+
-+#define SRST_A_TOP_M400_BIU		7
-+#define SRST_A_TOP_S200_BIU		8
-+#define SRST_A_TOP_S400_BIU		9
-+#define SRST_A_TOP_M300_BIU		10
-+#define SRST_USBDP_COMBO_PHY0_INIT	11
-+#define SRST_USBDP_COMBO_PHY0_CMN	12
-+#define SRST_USBDP_COMBO_PHY0_LANE	13
-+#define SRST_USBDP_COMBO_PHY0_PCS	14
-+#define SRST_USBDP_COMBO_PHY1_INIT	15
-+
-+#define SRST_USBDP_COMBO_PHY1_CMN	16
-+#define SRST_USBDP_COMBO_PHY1_LANE	17
-+#define SRST_USBDP_COMBO_PHY1_PCS	18
-+#define SRST_DCPHY0			19
-+#define SRST_P_MIPI_DCPHY0		20
-+#define SRST_P_MIPI_DCPHY0_GRF		21
-+
-+#define SRST_DCPHY1			22
-+#define SRST_P_MIPI_DCPHY1		23
-+#define SRST_P_MIPI_DCPHY1_GRF		24
-+#define SRST_P_APB2ASB_SLV_CDPHY	25
-+#define SRST_P_APB2ASB_SLV_CSIPHY	26
-+#define SRST_P_APB2ASB_SLV_VCCIO3_5	27
-+#define SRST_P_APB2ASB_SLV_VCCIO6	28
-+#define SRST_P_APB2ASB_SLV_EMMCIO	29
-+#define SRST_P_APB2ASB_SLV_IOC_TOP	30
-+#define SRST_P_APB2ASB_SLV_IOC_RIGHT	31
-+
-+#define SRST_P_CRU			32
-+#define SRST_A_CHANNEL_SECURE2VO1USB	33
-+#define SRST_A_CHANNEL_SECURE2CENTER	34
-+#define SRST_H_CHANNEL_SECURE2VO1USB	35
-+#define SRST_H_CHANNEL_SECURE2CENTER	36
-+
-+#define SRST_P_CHANNEL_SECURE2VO1USB	37
-+#define SRST_P_CHANNEL_SECURE2CENTER	38
-+
-+#define SRST_H_AUDIO_BIU		39
-+#define SRST_P_AUDIO_BIU		40
-+#define SRST_H_I2S0_8CH			41
-+#define SRST_M_I2S0_8CH_TX		42
-+#define SRST_M_I2S0_8CH_RX		43
-+#define SRST_P_ACDCDIG			44
-+#define SRST_H_I2S2_2CH			45
-+#define SRST_H_I2S3_2CH			46
-+
-+#define SRST_M_I2S2_2CH			47
-+#define SRST_M_I2S3_2CH			48
-+#define SRST_DAC_ACDCDIG		49
-+#define SRST_H_SPDIF0			50
-+
-+#define SRST_M_SPDIF0			51
-+#define SRST_H_SPDIF1			52
-+#define SRST_M_SPDIF1			53
-+#define SRST_H_PDM1			54
-+#define SRST_PDM1			55
-+
-+#define SRST_A_BUS_BIU			56
-+#define SRST_P_BUS_BIU			57
-+#define SRST_A_GIC			58
-+#define SRST_A_GIC_DBG			59
-+#define SRST_A_DMAC0			60
-+#define SRST_A_DMAC1			61
-+#define SRST_A_DMAC2			62
-+#define SRST_P_I2C1			63
-+#define SRST_P_I2C2			64
-+#define SRST_P_I2C3			65
-+#define SRST_P_I2C4			66
-+#define SRST_P_I2C5			67
-+#define SRST_P_I2C6			68
-+#define SRST_P_I2C7			69
-+#define SRST_P_I2C8			70
-+
-+#define SRST_I2C1			71
-+#define SRST_I2C2			72
-+#define SRST_I2C3			73
-+#define SRST_I2C4			74
-+#define SRST_I2C5			75
-+#define SRST_I2C6			76
-+#define SRST_I2C7			77
-+#define SRST_I2C8			78
-+#define SRST_P_CAN0			79
-+#define SRST_CAN0			80
-+#define SRST_P_CAN1			81
-+#define SRST_CAN1			82
-+#define SRST_P_CAN2			83
-+#define SRST_CAN2			84
-+#define SRST_P_SARADC			85
-+
-+#define SRST_P_TSADC			86
-+#define SRST_TSADC			87
-+#define SRST_P_UART1			88
-+#define SRST_P_UART2			89
-+#define SRST_P_UART3			90
-+#define SRST_P_UART4			91
-+#define SRST_P_UART5			92
-+#define SRST_P_UART6			93
-+#define SRST_P_UART7			94
-+#define SRST_P_UART8			95
-+#define SRST_P_UART9			96
-+#define SRST_S_UART1			97
-+
-+#define SRST_S_UART2			98
-+#define SRST_S_UART3			99
-+#define SRST_S_UART4			100
-+#define SRST_S_UART5			101
-+#define SRST_S_UART6			102
-+#define SRST_S_UART7			103
-+
-+#define SRST_S_UART8			104
-+#define SRST_S_UART9			105
-+#define SRST_P_SPI0			106
-+#define SRST_P_SPI1			107
-+#define SRST_P_SPI2			108
-+#define SRST_P_SPI3			109
-+#define SRST_P_SPI4			110
-+#define SRST_SPI0			111
-+#define SRST_SPI1			112
-+#define SRST_SPI2			113
-+#define SRST_SPI3			114
-+#define SRST_SPI4			115
-+
-+#define SRST_P_WDT0			116
-+#define SRST_T_WDT0			117
-+#define SRST_P_SYS_GRF			118
-+#define SRST_P_PWM1			119
-+#define SRST_PWM1			120
-+#define SRST_P_PWM2			121
-+#define SRST_PWM2			122
-+#define SRST_P_PWM3			123
-+#define SRST_PWM3			124
-+#define SRST_P_BUSTIMER0		125
-+#define SRST_P_BUSTIMER1		126
-+#define SRST_BUSTIMER0			127
-+
-+#define SRST_BUSTIMER1			128
-+#define SRST_BUSTIMER2			129
-+#define SRST_BUSTIMER3			130
-+#define SRST_BUSTIMER4			131
-+#define SRST_BUSTIMER5			132
-+#define SRST_BUSTIMER6			133
-+#define SRST_BUSTIMER7			134
-+#define SRST_BUSTIMER8			135
-+#define SRST_BUSTIMER9			136
-+#define SRST_BUSTIMER10			137
-+#define SRST_BUSTIMER11			138
-+#define SRST_P_MAILBOX0			139
-+#define SRST_P_MAILBOX1			140
-+#define SRST_P_MAILBOX2			141
-+#define SRST_P_GPIO1			142
-+#define SRST_GPIO1			143
-+
-+#define SRST_P_GPIO2			144
-+#define SRST_GPIO2			145
-+#define SRST_P_GPIO3			146
-+#define SRST_GPIO3			147
-+#define SRST_P_GPIO4			148
-+#define SRST_GPIO4			149
-+#define SRST_A_DECOM			150
-+#define SRST_P_DECOM			151
-+#define SRST_D_DECOM			152
-+#define SRST_P_TOP			153
-+#define SRST_A_GICADB_GIC2CORE_BUS	154
-+#define SRST_P_DFT2APB			155
-+#define SRST_P_APB2ASB_MST_TOP		156
-+#define SRST_P_APB2ASB_MST_CDPHY	157
-+#define SRST_P_APB2ASB_MST_BOT_RIGHT	158
-+
-+#define SRST_P_APB2ASB_MST_IOC_TOP	159
-+#define SRST_P_APB2ASB_MST_IOC_RIGHT	160
-+#define SRST_P_APB2ASB_MST_CSIPHY	161
-+#define SRST_P_APB2ASB_MST_VCCIO3_5	162
-+#define SRST_P_APB2ASB_MST_VCCIO6	163
-+#define SRST_P_APB2ASB_MST_EMMCIO	164
-+#define SRST_A_SPINLOCK			165
-+#define SRST_P_OTPC_NS			166
-+#define SRST_OTPC_NS			167
-+#define SRST_OTPC_ARB			168
-+
-+#define SRST_P_BUSIOC			169
-+#define SRST_P_PMUCM0_INTMUX		170
-+#define SRST_P_DDRCM0_INTMUX		171
-+
-+#define SRST_P_DDR_DFICTL_CH0		172
-+#define SRST_P_DDR_MON_CH0		173
-+#define SRST_P_DDR_STANDBY_CH0		174
-+#define SRST_P_DDR_UPCTL_CH0		175
-+#define SRST_TM_DDR_MON_CH0		176
-+#define SRST_P_DDR_GRF_CH01		177
-+#define SRST_DFI_CH0			178
-+#define SRST_SBR_CH0			179
-+#define SRST_DDR_UPCTL_CH0		180
-+#define SRST_DDR_DFICTL_CH0		181
-+#define SRST_DDR_MON_CH0		182
-+#define SRST_DDR_STANDBY_CH0		183
-+#define SRST_A_DDR_UPCTL_CH0		184
-+#define SRST_P_DDR_DFICTL_CH1		185
-+#define SRST_P_DDR_MON_CH1		186
-+#define SRST_P_DDR_STANDBY_CH1		187
-+
-+#define SRST_P_DDR_UPCTL_CH1		188
-+#define SRST_TM_DDR_MON_CH1		189
-+#define SRST_DFI_CH1			190
-+#define SRST_SBR_CH1			191
-+#define SRST_DDR_UPCTL_CH1		192
-+#define SRST_DDR_DFICTL_CH1		193
-+#define SRST_DDR_MON_CH1		194
-+#define SRST_DDR_STANDBY_CH1		195
-+#define SRST_A_DDR_UPCTL_CH1		196
-+#define SRST_A_DDR01_MSCH0		197
-+#define SRST_A_DDR01_RS_MSCH0		198
-+#define SRST_A_DDR01_FRS_MSCH0		199
-+
-+#define SRST_A_DDR01_SCRAMBLE0		200
-+#define SRST_A_DDR01_FRS_SCRAMBLE0	201
-+#define SRST_A_DDR01_MSCH1		202
-+#define SRST_A_DDR01_RS_MSCH1		203
-+#define SRST_A_DDR01_FRS_MSCH1		204
-+#define SRST_A_DDR01_SCRAMBLE1		205
-+#define SRST_A_DDR01_FRS_SCRAMBLE1	206
-+#define SRST_P_DDR01_MSCH0		207
-+#define SRST_P_DDR01_MSCH1		208
-+
-+#define SRST_P_DDR_DFICTL_CH2		209
-+#define SRST_P_DDR_MON_CH2		210
-+#define SRST_P_DDR_STANDBY_CH2		211
-+#define SRST_P_DDR_UPCTL_CH2		212
-+#define SRST_TM_DDR_MON_CH2		213
-+#define SRST_P_DDR_GRF_CH23		214
-+#define SRST_DFI_CH2			215
-+#define SRST_SBR_CH2			216
-+#define SRST_DDR_UPCTL_CH2		217
-+#define SRST_DDR_DFICTL_CH2		218
-+#define SRST_DDR_MON_CH2		219
-+#define SRST_DDR_STANDBY_CH2		220
-+#define SRST_A_DDR_UPCTL_CH2		221
-+#define SRST_P_DDR_DFICTL_CH3		222
-+#define SRST_P_DDR_MON_CH3		223
-+#define SRST_P_DDR_STANDBY_CH3		224
-+
-+#define SRST_P_DDR_UPCTL_CH3		225
-+#define SRST_TM_DDR_MON_CH3		226
-+#define SRST_DFI_CH3			227
-+#define SRST_SBR_CH3			228
-+#define SRST_DDR_UPCTL_CH3		229
-+#define SRST_DDR_DFICTL_CH3		230
-+#define SRST_DDR_MON_CH3		231
-+#define SRST_DDR_STANDBY_CH3		232
-+#define SRST_A_DDR_UPCTL_CH3		233
-+#define SRST_A_DDR23_MSCH2		234
-+#define SRST_A_DDR23_RS_MSCH2		235
-+#define SRST_A_DDR23_FRS_MSCH2		236
-+
-+#define SRST_A_DDR23_SCRAMBLE2		237
-+#define SRST_A_DDR23_FRS_SCRAMBLE2	238
-+#define SRST_A_DDR23_MSCH3		239
-+#define SRST_A_DDR23_RS_MSCH3		240
-+#define SRST_A_DDR23_FRS_MSCH3		241
-+#define SRST_A_DDR23_SCRAMBLE3		242
-+#define SRST_A_DDR23_FRS_SCRAMBLE3	243
-+#define SRST_P_DDR23_MSCH2		244
-+#define SRST_P_DDR23_MSCH3		245
-+
-+#define SRST_ISP1			246
-+#define SRST_ISP1_VICAP			247
-+#define SRST_A_ISP1_BIU			248
-+#define SRST_H_ISP1_BIU			249
-+
-+#define SRST_A_RKNN1			250
-+#define SRST_A_RKNN1_BIU		251
-+#define SRST_H_RKNN1			252
-+#define SRST_H_RKNN1_BIU		253
-+
-+#define SRST_A_RKNN2			254
-+#define SRST_A_RKNN2_BIU		255
-+#define SRST_H_RKNN2			256
-+#define SRST_H_RKNN2_BIU		257
-+
-+#define SRST_A_RKNN_DSU0		258
-+#define SRST_P_NPUTOP_BIU		259
-+#define SRST_P_NPU_TIMER		260
-+#define SRST_NPUTIMER0			261
-+#define SRST_NPUTIMER1			262
-+#define SRST_P_NPU_WDT			263
-+#define SRST_T_NPU_WDT			264
-+#define SRST_P_NPU_PVTM			265
-+#define SRST_P_NPU_GRF			266
-+#define SRST_NPU_PVTM			267
-+
-+#define SRST_NPU_PVTPLL			268
-+#define SRST_H_NPU_CM0_BIU		269
-+#define SRST_F_NPU_CM0_CORE		270
-+#define SRST_T_NPU_CM0_JTAG		271
-+#define SRST_A_RKNN0			272
-+#define SRST_A_RKNN0_BIU		273
-+#define SRST_H_RKNN0			274
-+#define SRST_H_RKNN0_BIU		275
-+
-+#define SRST_H_NVM_BIU			276
-+#define SRST_A_NVM_BIU			277
-+#define SRST_H_EMMC			278
-+#define SRST_A_EMMC			279
-+#define SRST_C_EMMC			280
-+#define SRST_B_EMMC			281
-+#define SRST_T_EMMC			282
-+#define SRST_S_SFC			283
-+#define SRST_H_SFC			284
-+#define SRST_H_SFC_XIP			285
-+
-+#define SRST_P_GRF			286
-+#define SRST_P_DEC_BIU			287
-+#define SRST_P_PHP_BIU			288
-+#define SRST_A_PCIE_GRIDGE		289
-+#define SRST_A_PHP_BIU			290
-+#define SRST_A_GMAC0			291
-+#define SRST_A_GMAC1			292
-+#define SRST_A_PCIE_BIU			293
-+#define SRST_PCIE0_POWER_UP		294
-+#define SRST_PCIE1_POWER_UP		295
-+#define SRST_PCIE2_POWER_UP		296
-+
-+#define SRST_PCIE3_POWER_UP		297
-+#define SRST_PCIE4_POWER_UP		298
-+#define SRST_P_PCIE0			299
-+#define SRST_P_PCIE1			300
-+#define SRST_P_PCIE2			301
-+#define SRST_P_PCIE3			302
-+
-+#define SRST_P_PCIE4			303
-+#define SRST_A_PHP_GIC_ITS		304
-+#define SRST_A_MMU_PCIE			305
-+#define SRST_A_MMU_PHP			306
-+#define SRST_A_MMU_BIU			307
-+
-+#define SRST_A_USB3OTG2			308
-+
-+#define SRST_PMALIVE0			309
-+#define SRST_PMALIVE1			310
-+#define SRST_PMALIVE2			311
-+#define SRST_A_SATA0			312
-+#define SRST_A_SATA1			313
-+#define SRST_A_SATA2			314
-+#define SRST_RXOOB0			315
-+#define SRST_RXOOB1			316
-+#define SRST_RXOOB2			317
-+#define SRST_ASIC0			318
-+#define SRST_ASIC1			319
-+#define SRST_ASIC2			320
-+
-+#define SRST_A_RKVDEC_CCU		321
-+#define SRST_H_RKVDEC0			322
-+#define SRST_A_RKVDEC0			323
-+#define SRST_H_RKVDEC0_BIU		324
-+#define SRST_A_RKVDEC0_BIU		325
-+#define SRST_RKVDEC0_CA			326
-+#define SRST_RKVDEC0_HEVC_CA		327
-+#define SRST_RKVDEC0_CORE		328
-+
-+#define SRST_H_RKVDEC1			329
-+#define SRST_A_RKVDEC1			330
-+#define SRST_H_RKVDEC1_BIU		331
-+#define SRST_A_RKVDEC1_BIU		332
-+#define SRST_RKVDEC1_CA			333
-+#define SRST_RKVDEC1_HEVC_CA		334
-+#define SRST_RKVDEC1_CORE		335
-+
-+#define SRST_A_USB_BIU			336
-+#define SRST_H_USB_BIU			337
-+#define SRST_A_USB3OTG0			338
-+#define SRST_A_USB3OTG1			339
-+#define SRST_H_HOST0			340
-+#define SRST_H_HOST_ARB0		341
-+#define SRST_H_HOST1			342
-+#define SRST_H_HOST_ARB1		343
-+#define SRST_A_USB_GRF			344
-+#define SRST_C_USB2P0_HOST0		345
-+
-+#define SRST_C_USB2P0_HOST1		346
-+#define SRST_HOST_UTMI0			347
-+#define SRST_HOST_UTMI1			348
-+
-+#define SRST_A_VDPU_BIU			349
-+#define SRST_A_VDPU_LOW_BIU		350
-+#define SRST_H_VDPU_BIU			351
-+#define SRST_A_JPEG_DECODER_BIU		352
-+#define SRST_A_VPU			353
-+#define SRST_H_VPU			354
-+#define SRST_A_JPEG_ENCODER0		355
-+#define SRST_H_JPEG_ENCODER0		356
-+#define SRST_A_JPEG_ENCODER1		357
-+#define SRST_H_JPEG_ENCODER1		358
-+#define SRST_A_JPEG_ENCODER2		359
-+#define SRST_H_JPEG_ENCODER2		360
-+
-+#define SRST_A_JPEG_ENCODER3		361
-+#define SRST_H_JPEG_ENCODER3		362
-+#define SRST_A_JPEG_DECODER		363
-+#define SRST_H_JPEG_DECODER		364
-+#define SRST_H_IEP2P0			365
-+#define SRST_A_IEP2P0			366
-+#define SRST_IEP2P0_CORE		367
-+#define SRST_H_RGA2			368
-+#define SRST_A_RGA2			369
-+#define SRST_RGA2_CORE			370
-+#define SRST_H_RGA3_0			371
-+#define SRST_A_RGA3_0			372
-+#define SRST_RGA3_0_CORE		373
-+
-+#define SRST_H_RKVENC0_BIU		374
-+#define SRST_A_RKVENC0_BIU		375
-+#define SRST_H_RKVENC0			376
-+#define SRST_A_RKVENC0			377
-+#define SRST_RKVENC0_CORE		378
-+
-+#define SRST_H_RKVENC1_BIU		379
-+#define SRST_A_RKVENC1_BIU		380
-+#define SRST_H_RKVENC1			381
-+#define SRST_A_RKVENC1			382
-+#define SRST_RKVENC1_CORE		383
-+
-+#define SRST_A_VI_BIU			384
-+#define SRST_H_VI_BIU			385
-+#define SRST_P_VI_BIU			386
-+#define SRST_D_VICAP			387
-+#define SRST_A_VICAP			388
-+#define SRST_H_VICAP			389
-+#define SRST_ISP0			390
-+#define SRST_ISP0_VICAP			391
-+
-+#define SRST_FISHEYE0			392
-+#define SRST_FISHEYE1			393
-+#define SRST_P_CSI_HOST_0		394
-+#define SRST_P_CSI_HOST_1		395
-+#define SRST_P_CSI_HOST_2		396
-+#define SRST_P_CSI_HOST_3		397
-+#define SRST_P_CSI_HOST_4		398
-+#define SRST_P_CSI_HOST_5		399
-+
-+#define SRST_CSIHOST0_VICAP		400
-+#define SRST_CSIHOST1_VICAP		401
-+#define SRST_CSIHOST2_VICAP		402
-+#define SRST_CSIHOST3_VICAP		403
-+#define SRST_CSIHOST4_VICAP		404
-+#define SRST_CSIHOST5_VICAP		405
-+#define SRST_CIFIN			406
-+
-+#define SRST_A_VOP_BIU			407
-+#define SRST_A_VOP_LOW_BIU		408
-+#define SRST_H_VOP_BIU			409
-+#define SRST_P_VOP_BIU			410
-+#define SRST_H_VOP			411
-+#define SRST_A_VOP			412
-+#define SRST_D_VOP0			413
-+#define SRST_D_VOP2HDMI_BRIDGE0		414
-+#define SRST_D_VOP2HDMI_BRIDGE1		415
-+
-+#define SRST_D_VOP1			416
-+#define SRST_D_VOP2			417
-+#define SRST_D_VOP3			418
-+#define SRST_P_VOPGRF			419
-+#define SRST_P_DSIHOST0			420
-+#define SRST_P_DSIHOST1			421
-+#define SRST_DSIHOST0			422
-+#define SRST_DSIHOST1			423
-+#define SRST_VOP_PMU			424
-+#define SRST_P_VOP_CHANNEL_BIU		425
-+
-+#define SRST_H_VO0_BIU			426
-+#define SRST_H_VO0_S_BIU		427
-+#define SRST_P_VO0_BIU			428
-+#define SRST_P_VO0_S_BIU		429
-+#define SRST_A_HDCP0_BIU		430
-+#define SRST_P_VO0GRF			431
-+#define SRST_H_HDCP_KEY0		432
-+#define SRST_A_HDCP0			433
-+#define SRST_H_HDCP0			434
-+#define SRST_HDCP0			435
-+
-+#define SRST_P_TRNG0			436
-+#define SRST_DP0			437
-+#define SRST_DP1			438
-+#define SRST_H_I2S4_8CH			439
-+#define SRST_M_I2S4_8CH_TX		440
-+#define SRST_H_I2S8_8CH			441
-+
-+#define SRST_M_I2S8_8CH_TX		442
-+#define SRST_H_SPDIF2_DP0		443
-+#define SRST_M_SPDIF2_DP0		444
-+#define SRST_H_SPDIF5_DP1		445
-+#define SRST_M_SPDIF5_DP1		446
-+
-+#define SRST_A_HDCP1_BIU		447
-+#define SRST_A_VO1_BIU			448
-+#define SRST_H_VOP1_BIU			449
-+#define SRST_H_VOP1_S_BIU		450
-+#define SRST_P_VOP1_BIU			451
-+#define SRST_P_VO1GRF			452
-+#define SRST_P_VO1_S_BIU		453
-+
-+#define SRST_H_I2S7_8CH			454
-+#define SRST_M_I2S7_8CH_RX		455
-+#define SRST_H_HDCP_KEY1		456
-+#define SRST_A_HDCP1			457
-+#define SRST_H_HDCP1			458
-+#define SRST_HDCP1			459
-+#define SRST_P_TRNG1			460
-+#define SRST_P_HDMITX0			461
-+
-+#define SRST_HDMITX0_REF		462
-+#define SRST_P_HDMITX1			463
-+#define SRST_HDMITX1_REF		464
-+#define SRST_A_HDMIRX			465
-+#define SRST_P_HDMIRX			466
-+#define SRST_HDMIRX_REF			467
-+
-+#define SRST_P_EDP0			468
-+#define SRST_EDP0_24M			469
-+#define SRST_P_EDP1			470
-+#define SRST_EDP1_24M			471
-+#define SRST_M_I2S5_8CH_TX		472
-+#define SRST_H_I2S5_8CH			473
-+#define SRST_M_I2S6_8CH_TX		474
-+
-+#define SRST_M_I2S6_8CH_RX		475
-+#define SRST_H_I2S6_8CH			476
-+#define SRST_H_SPDIF3			477
-+#define SRST_M_SPDIF3			478
-+#define SRST_H_SPDIF4			479
-+#define SRST_M_SPDIF4			480
-+#define SRST_H_SPDIFRX0			481
-+#define SRST_M_SPDIFRX0			482
-+#define SRST_H_SPDIFRX1			483
-+#define SRST_M_SPDIFRX1			484
-+
-+#define SRST_H_SPDIFRX2			485
-+#define SRST_M_SPDIFRX2			486
-+#define SRST_LINKSYM_HDMITXPHY0		487
-+#define SRST_LINKSYM_HDMITXPHY1		488
-+#define SRST_VO1_BRIDGE0		489
-+#define SRST_VO1_BRIDGE1		490
-+
-+#define SRST_H_I2S9_8CH			491
-+#define SRST_M_I2S9_8CH_RX		492
-+#define SRST_H_I2S10_8CH		493
-+#define SRST_M_I2S10_8CH_RX		494
-+#define SRST_P_S_HDMIRX			495
-+
-+#define SRST_GPU			496
-+#define SRST_SYS_GPU			497
-+#define SRST_A_S_GPU_BIU		498
-+#define SRST_A_M0_GPU_BIU		499
-+#define SRST_A_M1_GPU_BIU		500
-+#define SRST_A_M2_GPU_BIU		501
-+#define SRST_A_M3_GPU_BIU		502
-+#define SRST_P_GPU_BIU			503
-+#define SRST_P_GPU_PVTM			504
-+
-+#define SRST_GPU_PVTM			505
-+#define SRST_P_GPU_GRF			506
-+#define SRST_GPU_PVTPLL			507
-+#define SRST_GPU_JTAG			508
-+
-+#define SRST_A_AV1_BIU			509
-+#define SRST_A_AV1			510
-+#define SRST_P_AV1_BIU			511
-+#define SRST_P_AV1			512
-+
-+#define SRST_A_DDR_BIU			513
-+#define SRST_A_DMA2DDR			514
-+#define SRST_A_DDR_SHAREMEM		515
-+#define SRST_A_DDR_SHAREMEM_BIU		516
-+#define SRST_A_CENTER_S200_BIU		517
-+#define SRST_A_CENTER_S400_BIU		518
-+#define SRST_H_AHB2APB			519
-+#define SRST_H_CENTER_BIU		520
-+#define SRST_F_DDR_CM0_CORE		521
-+
-+#define SRST_DDR_TIMER0			522
-+#define SRST_DDR_TIMER1			523
-+#define SRST_T_WDT_DDR			524
-+#define SRST_T_DDR_CM0_JTAG		525
-+#define SRST_P_CENTER_GRF		526
-+#define SRST_P_AHB2APB			527
-+#define SRST_P_WDT			528
-+#define SRST_P_TIMER			529
-+#define SRST_P_DMA2DDR			530
-+#define SRST_P_SHAREMEM			531
-+#define SRST_P_CENTER_BIU		532
-+#define SRST_P_CENTER_CHANNEL_BIU	533
-+
-+#define SRST_P_USBDPGRF0		534
-+#define SRST_P_USBDPPHY0		535
-+#define SRST_P_USBDPGRF1		536
-+#define SRST_P_USBDPPHY1		537
-+#define SRST_P_HDPTX0			538
-+#define SRST_P_HDPTX1			539
-+#define SRST_P_APB2ASB_SLV_BOT_RIGHT	540
-+#define SRST_P_USB2PHY_U3_0_GRF0	541
-+#define SRST_P_USB2PHY_U3_1_GRF0	542
-+#define SRST_P_USB2PHY_U2_0_GRF0	543
-+#define SRST_P_USB2PHY_U2_1_GRF0	544
-+#define SRST_HDPTX0_ROPLL		545
-+#define SRST_HDPTX0_LCPLL		546
-+#define SRST_HDPTX0			547
-+#define SRST_HDPTX1_ROPLL		548
-+
-+#define SRST_HDPTX1_LCPLL		549
-+#define SRST_HDPTX1			550
-+#define SRST_HDPTX0_HDMIRXPHY_SET	551
-+#define SRST_USBDP_COMBO_PHY0		552
-+#define SRST_USBDP_COMBO_PHY0_LCPLL	553
-+#define SRST_USBDP_COMBO_PHY0_ROPLL	554
-+#define SRST_USBDP_COMBO_PHY0_PCS_HS	555
-+#define SRST_USBDP_COMBO_PHY1		556
-+#define SRST_USBDP_COMBO_PHY1_LCPLL	557
-+#define SRST_USBDP_COMBO_PHY1_ROPLL	558
-+#define SRST_USBDP_COMBO_PHY1_PCS_HS	559
-+#define SRST_HDMIHDP0			560
-+#define SRST_HDMIHDP1			561
-+
-+#define SRST_A_VO1USB_TOP_BIU		562
-+#define SRST_H_VO1USB_TOP_BIU		563
-+
-+#define SRST_H_SDIO_BIU			564
-+#define SRST_H_SDIO			565
-+#define SRST_SDIO			566
-+
-+#define SRST_H_RGA3_BIU			567
-+#define SRST_A_RGA3_BIU			568
-+#define SRST_H_RGA3_1			569
-+#define SRST_A_RGA3_1			570
-+#define SRST_RGA3_1_CORE		571
-+
-+#define SRST_REF_PIPE_PHY0		572
-+#define SRST_REF_PIPE_PHY1		573
-+#define SRST_REF_PIPE_PHY2		574
-+
-+#define SRST_P_PHPTOP_CRU		575
-+#define SRST_P_PCIE2_GRF0		576
-+#define SRST_P_PCIE2_GRF1		577
-+#define SRST_P_PCIE2_GRF2		578
-+#define SRST_P_PCIE2_PHY0		579
-+#define SRST_P_PCIE2_PHY1		580
-+#define SRST_P_PCIE2_PHY2		581
-+#define SRST_P_PCIE3_PHY		582
-+#define SRST_P_APB2ASB_SLV_CHIP_TOP	583
-+#define SRST_PCIE30_PHY			584
-+
-+#define SRST_H_PMU1_BIU			585
-+#define SRST_P_PMU1_BIU			586
-+#define SRST_H_PMU_CM0_BIU		587
-+#define SRST_F_PMU_CM0_CORE		588
-+#define SRST_T_PMU1_CM0_JTAG		589
-+
-+#define SRST_DDR_FAIL_SAFE		590
-+#define SRST_P_CRU_PMU1			591
-+#define SRST_P_PMU1_GRF			592
-+#define SRST_P_PMU1_IOC			593
-+#define SRST_P_PMU1WDT			594
-+#define SRST_T_PMU1WDT			595
-+#define SRST_P_PMU1TIMER		596
-+#define SRST_PMU1TIMER0			597
-+#define SRST_PMU1TIMER1			598
-+#define SRST_P_PMU1PWM			599
-+#define SRST_PMU1PWM			600
-+
-+#define SRST_P_I2C0			601
-+#define SRST_I2C0			602
-+#define SRST_S_UART0			603
-+#define SRST_P_UART0			604
-+#define SRST_H_I2S1_8CH			605
-+#define SRST_M_I2S1_8CH_TX		606
-+#define SRST_M_I2S1_8CH_RX		607
-+#define SRST_H_PDM0			608
-+#define SRST_PDM0			609
-+
-+#define SRST_H_VAD			610
-+#define SRST_HDPTX0_INIT		611
-+#define SRST_HDPTX0_CMN			612
-+#define SRST_HDPTX0_LANE		613
-+#define SRST_HDPTX1_INIT		614
-+
-+#define SRST_HDPTX1_CMN			615
-+#define SRST_HDPTX1_LANE		616
-+#define SRST_M_MIPI_DCPHY0		617
-+#define SRST_S_MIPI_DCPHY0		618
-+#define SRST_M_MIPI_DCPHY1		619
-+#define SRST_S_MIPI_DCPHY1		620
-+#define SRST_OTGPHY_U3_0		621
-+#define SRST_OTGPHY_U3_1		622
-+#define SRST_OTGPHY_U2_0		623
-+#define SRST_OTGPHY_U2_1		624
-+
-+#define SRST_P_PMU0GRF			625
-+#define SRST_P_PMU0IOC			626
-+#define SRST_P_GPIO0			627
-+#define SRST_GPIO0			628
-+
-+#define SRST_A_SECURE_NS_BIU		629
-+#define SRST_H_SECURE_NS_BIU		630
-+#define SRST_A_SECURE_S_BIU		631
-+#define SRST_H_SECURE_S_BIU		632
-+#define SRST_P_SECURE_S_BIU		633
-+#define SRST_CRYPTO_CORE		634
-+
-+#define SRST_CRYPTO_PKA			635
-+#define SRST_CRYPTO_RNG			636
-+#define SRST_A_CRYPTO			637
-+#define SRST_H_CRYPTO			638
-+#define SRST_KEYLADDER_CORE		639
-+#define SRST_KEYLADDER_RNG		640
-+#define SRST_A_KEYLADDER		641
-+#define SRST_H_KEYLADDER		642
-+#define SRST_P_OTPC_S			643
-+#define SRST_OTPC_S			644
-+#define SRST_WDT_S			645
-+
-+#define SRST_T_WDT_S			646
-+#define SRST_H_BOOTROM			647
-+#define SRST_A_DCF			648
-+#define SRST_P_DCF			649
-+#define SRST_H_BOOTROM_NS		650
-+#define SRST_P_KEYLADDER		651
-+#define SRST_H_TRNG_S			652
-+
-+#define SRST_H_TRNG_NS			653
-+#define SRST_D_SDMMC_BUFFER		654
-+#define SRST_H_SDMMC			655
-+#define SRST_H_SDMMC_BUFFER		656
-+#define SRST_SDMMC			657
-+#define SRST_P_TRNG_CHK			658
-+#define SRST_TRNG_S			659
-+
-+#define SRST_A_HDMIRX_BIU		660
-+
-+/* SCMI Secure Resets */
-+
-+/* Name=SECURE_SOFTRST_CON00,Offset=0xA00 */
-+#define SCMI_SRST_A_SECURE_NS_BIU	10
-+#define SCMI_SRST_H_SECURE_NS_BIU	11
-+#define SCMI_SRST_A_SECURE_S_BIU	12
-+#define SCMI_SRST_H_SECURE_S_BIU	13
-+#define SCMI_SRST_P_SECURE_S_BIU	14
-+#define SCMI_SRST_CRYPTO_CORE		15
-+/* Name=SECURE_SOFTRST_CON01,Offset=0xA04 */
-+#define SCMI_SRST_CRYPTO_PKA		16
-+#define SCMI_SRST_CRYPTO_RNG		17
-+#define SCMI_SRST_A_CRYPTO		18
-+#define SCMI_SRST_H_CRYPTO		19
-+#define SCMI_SRST_KEYLADDER_CORE	25
-+#define SCMI_SRST_KEYLADDER_RNG		26
-+#define SCMI_SRST_A_KEYLADDER		27
-+#define SCMI_SRST_H_KEYLADDER		28
-+#define SCMI_SRST_P_OTPC_S		29
-+#define SCMI_SRST_OTPC_S		30
-+#define SCMI_SRST_WDT_S			31
-+/* Name=SECURE_SOFTRST_CON02,Offset=0xA08 */
-+#define SCMI_SRST_T_WDT_S		32
-+#define SCMI_SRST_H_BOOTROM		33
-+#define SCMI_SRST_A_DCF			34
-+#define SCMI_SRST_P_DCF			35
-+#define SCMI_SRST_H_BOOTROM_NS		37
-+#define SCMI_SRST_P_KEYLADDER		46
-+#define SCMI_SRST_H_TRNG_S		47
-+/* Name=SECURE_SOFTRST_CON03,Offset=0xA0C */
-+#define SCMI_SRST_H_TRNG_NS		48
-+#define SCMI_SRST_D_SDMMC_BUFFER	49
-+#define SCMI_SRST_H_SDMMC		50
-+#define SCMI_SRST_H_SDMMC_BUFFER	51
-+#define SCMI_SRST_SDMMC			52
-+#define SCMI_SRST_P_TRNG_CHK		53
-+#define SCMI_SRST_TRNG_S		54
-+
-+
-+#endif
-diff --git a/include/dt-bindings/reset/rockchip,rv1126b-cru.h b/include/dt-bindings/reset/rockchip,rv1126b-cru.h
-new file mode 100644
-index 000000000000..a7712db319d0
--- /dev/null
-+++ b/include/dt-bindings/reset/rockchip,rv1126b-cru.h
-@@ -0,0 +1,405 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (c) 2025 Rockchip Electronics Co., Ltd.
-+ * Author: Elaine Zhang <zhangqing@rock-chips.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_RESET_ROCKCHIP_RV1126B_H
-+#define _DT_BINDINGS_RESET_ROCKCHIP_RV1126B_H
-+
-+/* ==========================list all of reset fields id=========================== */
-+/* TOPCRU-->SOFTRST_CON00 */
-+
-+/* TOPCRU-->SOFTRST_CON15 */
-+#define SRST_P_CRU				0
-+#define SRST_P_CRU_BIU				1
-+
-+/* BUSCRU-->SOFTRST_CON00 */
-+#define SRST_A_TOP_BIU				2
-+#define SRST_A_RKCE_BIU				3
-+#define SRST_A_BUS_BIU				4
-+#define SRST_H_BUS_BIU				5
-+#define SRST_P_BUS_BIU				6
-+#define SRST_P_CRU_BUS				7
-+#define SRST_P_SYS_GRF				8
-+#define SRST_H_BOOTROM				9
-+#define SRST_A_GIC400				10
-+#define SRST_A_SPINLOCK				11
-+#define SRST_P_WDT_NS				12
-+#define SRST_T_WDT_NS				13
-+
-+/* BUSCRU-->SOFTRST_CON01 */
-+#define SRST_P_WDT_HPMCU			14
-+#define SRST_T_WDT_HPMCU			15
-+#define SRST_H_CACHE				16
-+#define SRST_P_HPMCU_MAILBOX			17
-+#define SRST_P_HPMCU_INTMUX			18
-+#define SRST_HPMCU_FULL_CLUSTER			19
-+#define SRST_HPMCU_PWUP				20
-+#define SRST_HPMCU_ONLY_CORE			21
-+#define SRST_T_HPMCU_JTAG			22
-+#define SRST_P_RKDMA				23
-+#define SRST_A_RKDMA				24
-+
-+/* BUSCRU-->SOFTRST_CON02 */
-+#define SRST_P_DCF				25
-+#define SRST_A_DCF				26
-+#define SRST_H_RGA				27
-+#define SRST_A_RGA				28
-+#define SRST_CORE_RGA				29
-+#define SRST_P_TIMER				30
-+#define SRST_TIMER0				31
-+#define SRST_TIMER1				32
-+#define SRST_TIMER2				33
-+#define SRST_TIMER3				34
-+#define SRST_TIMER4				35
-+#define SRST_TIMER5				36
-+#define SRST_A_RKCE				37
-+#define SRST_PKA_RKCE				38
-+#define SRST_H_RKRNG_S				39
-+#define SRST_H_RKRNG_NS				40
-+
-+/* BUSCRU-->SOFTRST_CON03 */
-+#define SRST_P_I2C0				41
-+#define SRST_I2C0				42
-+#define SRST_P_I2C1				43
-+#define SRST_I2C1				44
-+#define SRST_P_I2C3				45
-+#define SRST_I2C3				46
-+#define SRST_P_I2C4				47
-+#define SRST_I2C4				48
-+#define SRST_P_I2C5				49
-+#define SRST_I2C5				50
-+#define SRST_P_SPI0				51
-+#define SRST_SPI0				52
-+#define SRST_P_SPI1				53
-+#define SRST_SPI1				54
-+
-+/* BUSCRU-->SOFTRST_CON04 */
-+#define SRST_P_PWM0				55
-+#define SRST_PWM0				56
-+#define SRST_P_PWM2				57
-+#define SRST_PWM2				58
-+#define SRST_P_PWM3				59
-+#define SRST_PWM3				60
-+
-+/* BUSCRU-->SOFTRST_CON05 */
-+#define SRST_P_UART1				61
-+#define SRST_S_UART1				62
-+#define SRST_P_UART2				63
-+#define SRST_S_UART2				64
-+#define SRST_P_UART3				65
-+#define SRST_S_UART3				66
-+#define SRST_P_UART4				67
-+#define SRST_S_UART4				68
-+#define SRST_P_UART5				69
-+#define SRST_S_UART5				70
-+#define SRST_P_UART6				71
-+#define SRST_S_UART6				72
-+#define SRST_P_UART7				73
-+#define SRST_S_UART7				74
-+
-+/* BUSCRU-->SOFTRST_CON06 */
-+#define SRST_P_TSADC				75
-+#define SRST_TSADC				76
-+#define SRST_H_SAI0				77
-+#define SRST_M_SAI0				78
-+#define SRST_H_SAI1				79
-+#define SRST_M_SAI1				80
-+#define SRST_H_SAI2				81
-+#define SRST_M_SAI2				82
-+#define SRST_H_RKDSM				83
-+#define SRST_M_RKDSM				84
-+#define SRST_H_PDM				85
-+#define SRST_M_PDM				86
-+#define SRST_PDM				87
-+
-+/* BUSCRU-->SOFTRST_CON07 */
-+#define SRST_H_ASRC0				88
-+#define SRST_ASRC0				89
-+#define SRST_H_ASRC1				90
-+#define SRST_ASRC1				91
-+#define SRST_P_AUDIO_ADC_BUS			92
-+#define SRST_M_AUDIO_ADC_BUS			93
-+#define SRST_P_RKCE				94
-+#define SRST_H_NS_RKCE				95
-+#define SRST_P_OTPC_NS				96
-+#define SRST_SBPI_OTPC_NS			97
-+#define SRST_USER_OTPC_NS			98
-+#define SRST_OTPC_ARB				99
-+#define SRST_P_OTP_MASK				100
-+
-+/* PERICRU-->SOFTRST_CON00 */
-+#define SRST_A_PERI_BIU				101
-+#define SRST_P_PERI_BIU				102
-+#define SRST_P_RTC_BIU				103
-+#define SRST_P_CRU_PERI				104
-+#define SRST_P_PERI_GRF				105
-+#define SRST_P_GPIO1				106
-+#define SRST_DB_GPIO1				107
-+#define SRST_P_IOC_VCCIO1			108
-+#define SRST_A_USB3OTG				109
-+#define SRST_H_USB2HOST				110
-+#define SRST_H_ARB_USB2HOST			111
-+#define SRST_P_RTC_TEST				112
-+
-+/* PERICRU-->SOFTRST_CON01 */
-+#define SRST_H_EMMC				113
-+#define SRST_H_FSPI0				114
-+#define SRST_H_XIP_FSPI0			115
-+#define SRST_S_2X_FSPI0				116
-+#define SRST_UTMI_USB2HOST			117
-+#define SRST_REF_PIPEPHY			118
-+#define SRST_P_PIPEPHY				119
-+#define SRST_P_PIPEPHY_GRF			120
-+#define SRST_P_USB2PHY				121
-+#define SRST_POR_USB2PHY			122
-+#define SRST_OTG_USB2PHY			123
-+#define SRST_HOST_USB2PHY			124
-+
-+/* CORECRU-->SOFTRST_CON00 */
-+#define SRST_REF_PVTPLL_CORE			125
-+#define SRST_NCOREPORESET0			126
-+#define SRST_NCORESET0				127
-+#define SRST_NCOREPORESET1			128
-+#define SRST_NCORESET1				129
-+#define SRST_NCOREPORESET2			130
-+#define SRST_NCORESET2				131
-+#define SRST_NCOREPORESET3			132
-+#define SRST_NCORESET3				133
-+#define SRST_NDBGRESET				134
-+#define SRST_NL2RESET				135
-+
-+/* CORECRU-->SOFTRST_CON01 */
-+#define SRST_A_CORE_BIU				136
-+#define SRST_P_CORE_BIU				137
-+#define SRST_H_CORE_BIU				138
-+#define SRST_P_DBG				139
-+#define SRST_POT_DBG				140
-+#define SRST_NT_DBG				141
-+#define SRST_P_CORE_PVTPLL			142
-+#define SRST_P_CRU_CORE				143
-+#define SRST_P_CORE_GRF				144
-+#define SRST_P_DFT2APB				145
-+
-+/* PMUCRU-->SOFTRST_CON00 */
-+#define SRST_H_PMU_BIU				146
-+#define SRST_P_PMU_GPIO0			147
-+#define SRST_DB_PMU_GPIO0			148
-+#define SRST_P_PMU_HP_TIMER			149
-+#define SRST_PMU_HP_TIMER			150
-+#define SRST_PMU_32K_HP_TIMER			151
-+
-+/* PMUCRU-->SOFTRST_CON01 */
-+#define SRST_P_PWM1				152
-+#define SRST_PWM1				153
-+#define SRST_P_I2C2				154
-+#define SRST_I2C2				155
-+#define SRST_P_UART0				156
-+#define SRST_S_UART0				157
-+
-+/* PMUCRU-->SOFTRST_CON02 */
-+#define SRST_P_RCOSC_CTRL			158
-+#define SRST_REF_RCOSC_CTRL			159
-+#define SRST_P_IOC_PMUIO0			160
-+#define SRST_P_CRU_PMU				161
-+#define SRST_P_PMU_GRF				162
-+#define SRST_PREROLL				163
-+#define SRST_PREROLL_32K			164
-+#define SRST_H_PMU_SRAM				165
-+
-+/* PMUCRU-->SOFTRST_CON03 */
-+#define SRST_P_WDT_LPMCU			166
-+#define SRST_T_WDT_LPMCU			167
-+#define SRST_LPMCU_FULL_CLUSTER			168
-+#define SRST_LPMCU_PWUP				169
-+#define SRST_LPMCU_ONLY_CORE			170
-+#define SRST_T_LPMCU_JTAG			171
-+#define SRST_P_LPMCU_MAILBOX			172
-+
-+/* PMU1CRU-->SOFTRST_CON00 */
-+#define SRST_P_SPI2AHB				173
-+#define SRST_H_SPI2AHB				174
-+#define SRST_H_FSPI1				175
-+#define SRST_H_XIP_FSPI1			176
-+#define SRST_S_1X_FSPI1				177
-+#define SRST_P_IOC_PMUIO1			178
-+#define SRST_P_CRU_PMU1				179
-+#define SRST_P_AUDIO_ADC_PMU			180
-+#define SRST_M_AUDIO_ADC_PMU			181
-+#define SRST_H_PMU1_BIU				182
-+
-+/* PMU1CRU-->SOFTRST_CON01 */
-+#define SRST_P_LPDMA				183
-+#define SRST_A_LPDMA				184
-+#define SRST_H_LPSAI				185
-+#define SRST_M_LPSAI				186
-+#define SRST_P_AOA_TDD				187
-+#define SRST_P_AOA_FE				188
-+#define SRST_P_AOA_AAD				189
-+#define SRST_P_AOA_APB				190
-+#define SRST_P_AOA_SRAM				191
-+
-+/* DDRCRU-->SOFTRST_CON00 */
-+#define SRST_P_DDR_BIU				192
-+#define SRST_P_DDRC				193
-+#define SRST_P_DDRMON				194
-+#define SRST_TIMER_DDRMON			195
-+#define SRST_P_DFICTRL				196
-+#define SRST_P_DDR_GRF				197
-+#define SRST_P_CRU_DDR				198
-+#define SRST_P_DDRPHY				199
-+#define SRST_P_DMA2DDR				200
-+
-+/* SUBDDRCRU-->SOFTRST_CON00 */
-+#define SRST_A_SYSMEM_BIU			201
-+#define SRST_A_SYSMEM				202
-+#define SRST_A_DDR_BIU				203
-+#define SRST_A_DDRSCH0_CPU			204
-+#define SRST_A_DDRSCH1_NPU			205
-+#define SRST_A_DDRSCH2_POE			206
-+#define SRST_A_DDRSCH3_VI			207
-+#define SRST_CORE_DDRC				208
-+#define SRST_DDRMON				209
-+#define SRST_DFICTRL				210
-+#define SRST_RS					211
-+#define SRST_A_DMA2DDR				212
-+#define SRST_DDRPHY				213
-+
-+/* VICRU-->SOFTRST_CON00 */
-+#define SRST_REF_PVTPLL_ISP			214
-+#define SRST_A_GMAC_BIU				215
-+#define SRST_A_VI_BIU				216
-+#define SRST_H_VI_BIU				217
-+#define SRST_P_VI_BIU				218
-+#define SRST_P_CRU_VI				219
-+#define SRST_P_VI_GRF				220
-+#define SRST_P_VI_PVTPLL			221
-+#define SRST_P_DSMC				222
-+#define SRST_A_DSMC				223
-+#define SRST_H_CAN0				224
-+#define SRST_CAN0				225
-+#define SRST_H_CAN1				226
-+#define SRST_CAN1				227
-+
-+/* VICRU-->SOFTRST_CON01 */
-+#define SRST_P_GPIO2				228
-+#define SRST_DB_GPIO2				229
-+#define SRST_P_GPIO4				230
-+#define SRST_DB_GPIO4				231
-+#define SRST_P_GPIO5				232
-+#define SRST_DB_GPIO5				233
-+#define SRST_P_GPIO6				234
-+#define SRST_DB_GPIO6				235
-+#define SRST_P_GPIO7				236
-+#define SRST_DB_GPIO7				237
-+#define SRST_P_IOC_VCCIO2			238
-+#define SRST_P_IOC_VCCIO4			239
-+#define SRST_P_IOC_VCCIO5			240
-+#define SRST_P_IOC_VCCIO6			241
-+#define SRST_P_IOC_VCCIO7			242
-+
-+/* VICRU-->SOFTRST_CON02 */
-+#define SRST_CORE_ISP				243
-+#define SRST_H_VICAP				244
-+#define SRST_A_VICAP				245
-+#define SRST_D_VICAP				246
-+#define SRST_ISP0_VICAP				247
-+#define SRST_CORE_VPSS				248
-+#define SRST_CORE_VPSL				249
-+#define SRST_P_CSI2HOST0			250
-+#define SRST_P_CSI2HOST1			251
-+#define SRST_P_CSI2HOST2			252
-+#define SRST_P_CSI2HOST3			253
-+#define SRST_H_SDMMC0				254
-+#define SRST_A_GMAC				255
-+#define SRST_P_CSIPHY0				256
-+#define SRST_P_CSIPHY1				257
-+
-+/* VICRU-->SOFTRST_CON03 */
-+#define SRST_P_MACPHY				258
-+#define SRST_MACPHY				259
-+#define SRST_P_SARADC1				260
-+#define SRST_SARADC1				261
-+#define SRST_P_SARADC2				262
-+#define SRST_SARADC2				263
-+
-+/* VEPUCRU-->SOFTRST_CON00 */
-+#define SRST_REF_PVTPLL_VEPU			264
-+#define SRST_A_VEPU_BIU				265
-+#define SRST_H_VEPU_BIU				266
-+#define SRST_P_VEPU_BIU				267
-+#define SRST_P_CRU_VEPU				268
-+#define SRST_P_VEPU_GRF				269
-+#define SRST_P_GPIO3				270
-+#define SRST_DB_GPIO3				271
-+#define SRST_P_IOC_VCCIO3			272
-+#define SRST_P_SARADC0				273
-+#define SRST_SARADC0				274
-+#define SRST_H_SDMMC1				275
-+
-+/* VEPUCRU-->SOFTRST_CON01 */
-+#define SRST_P_VEPU_PVTPLL			276
-+#define SRST_H_VEPU				277
-+#define SRST_A_VEPU				278
-+#define SRST_CORE_VEPU				279
-+
-+/* NPUCRU-->SOFTRST_CON00 */
-+#define SRST_REF_PVTPLL_NPU			280
-+#define SRST_A_NPU_BIU				281
-+#define SRST_H_NPU_BIU				282
-+#define SRST_P_NPU_BIU				283
-+#define SRST_P_CRU_NPU				284
-+#define SRST_P_NPU_GRF				285
-+#define SRST_P_NPU_PVTPLL			286
-+#define SRST_H_RKNN				287
-+#define SRST_A_RKNN				288
-+
-+/* VDOCRU-->SOFTRST_CON00 */
-+#define SRST_A_RKVDEC_BIU			289
-+#define SRST_A_VDO_BIU				290
-+#define SRST_H_VDO_BIU				291
-+#define SRST_P_VDO_BIU				292
-+#define SRST_P_CRU_VDO				293
-+#define SRST_P_VDO_GRF				294
-+#define SRST_A_RKVDEC				295
-+#define SRST_H_RKVDEC				296
-+#define SRST_HEVC_CA_RKVDEC			297
-+#define SRST_A_VOP				298
-+#define SRST_H_VOP				299
-+#define SRST_D_VOP				300
-+#define SRST_A_OOC				301
-+#define SRST_H_OOC				302
-+#define SRST_D_OOC				303
-+
-+/* VDOCRU-->SOFTRST_CON01 */
-+#define SRST_H_RKJPEG				304
-+#define SRST_A_RKJPEG				305
-+#define SRST_A_RKMMU_DECOM			306
-+#define SRST_H_RKMMU_DECOM			307
-+#define SRST_D_DECOM				308
-+#define SRST_A_DECOM				309
-+#define SRST_P_DECOM				310
-+#define SRST_P_MIPI_DSI				311
-+#define SRST_P_DSIPHY				312
-+
-+/* VCPCRU-->SOFTRST_CON00 */
-+#define SRST_REF_PVTPLL_VCP			313
-+#define SRST_A_VCP_BIU				314
-+#define SRST_H_VCP_BIU				315
-+#define SRST_P_VCP_BIU				316
-+#define SRST_P_CRU_VCP				317
-+#define SRST_P_VCP_GRF				318
-+#define SRST_P_VCP_PVTPLL			319
-+#define SRST_A_AISP_BIU				320
-+#define SRST_H_AISP_BIU				321
-+#define SRST_CORE_AISP				322
-+
-+/* VCPCRU-->SOFTRST_CON01 */
-+#define SRST_H_FEC				323
-+#define SRST_A_FEC				324
-+#define SRST_CORE_FEC				325
-+#define SRST_H_AVSP				326
-+#define SRST_A_AVSP				327
-+
-+#endif
-diff --git a/include/dt-bindings/reset/sophgo,sg2042-reset.h b/include/dt-bindings/reset/sophgo,sg2042-reset.h
-new file mode 100644
-index 000000000000..9ab0980625c1
--- /dev/null
-+++ b/include/dt-bindings/reset/sophgo,sg2042-reset.h
-@@ -0,0 +1,87 @@
-+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
-+/*
-+ * Copyright (C) 2023 Sophgo Technology Inc. All rights reserved.
-+ */
-+
-+#ifndef __DT_BINDINGS_RESET_SOPHGO_SG2042_H_
-+#define __DT_BINDINGS_RESET_SOPHGO_SG2042_H_
-+
-+#define RST_MAIN_AP			0
-+#define RST_RISCV_CPU			1
-+#define RST_RISCV_LOW_SPEED_LOGIC	2
-+#define RST_RISCV_CMN			3
-+#define RST_HSDMA			4
-+#define RST_SYSDMA			5
-+#define RST_EFUSE0			6
-+#define RST_EFUSE1			7
-+#define RST_RTC				8
-+#define RST_TIMER			9
-+#define RST_WDT				10
-+#define RST_AHB_ROM0			11
-+#define RST_AHB_ROM1			12
-+#define RST_I2C0			13
-+#define RST_I2C1			14
-+#define RST_I2C2			15
-+#define RST_I2C3			16
-+#define RST_GPIO0			17
-+#define RST_GPIO1			18
-+#define RST_GPIO2			19
-+#define RST_PWM				20
-+#define RST_AXI_SRAM0			21
-+#define RST_AXI_SRAM1			22
-+#define RST_SF0				23
-+#define RST_SF1				24
-+#define RST_LPC				25
-+#define RST_ETH0			26
-+#define RST_EMMC			27
-+#define RST_SD				28
-+#define RST_UART0			29
-+#define RST_UART1			30
-+#define RST_UART2			31
-+#define RST_UART3			32
-+#define RST_SPI0			33
-+#define RST_SPI1			34
-+#define RST_DBG_I2C			35
-+#define RST_PCIE0			36
-+#define RST_PCIE1			37
-+#define RST_DDR0			38
-+#define RST_DDR1			39
-+#define RST_DDR2			40
-+#define RST_DDR3			41
-+#define RST_FAU0			42
-+#define RST_FAU1			43
-+#define RST_FAU2			44
-+#define RST_RXU0			45
-+#define RST_RXU1			46
-+#define RST_RXU2			47
-+#define RST_RXU3			48
-+#define RST_RXU4			49
-+#define RST_RXU5			50
-+#define RST_RXU6			51
-+#define RST_RXU7			52
-+#define RST_RXU8			53
-+#define RST_RXU9			54
-+#define RST_RXU10			55
-+#define RST_RXU11			56
-+#define RST_RXU12			57
-+#define RST_RXU13			58
-+#define RST_RXU14			59
-+#define RST_RXU15			60
-+#define RST_RXU16			61
-+#define RST_RXU17			62
-+#define RST_RXU18			63
-+#define RST_RXU19			64
-+#define RST_RXU20			65
-+#define RST_RXU21			66
-+#define RST_RXU22			67
-+#define RST_RXU23			68
-+#define RST_RXU24			69
-+#define RST_RXU25			70
-+#define RST_RXU26			71
-+#define RST_RXU27			72
-+#define RST_RXU28			73
-+#define RST_RXU29			74
-+#define RST_RXU30			75
-+#define RST_RXU31			76
-+
-+#endif /* __DT_BINDINGS_RESET_SOPHGO_SG2042_H_ */
-diff --git a/include/dt-bindings/reset/st,stm32mp21-rcc.h b/include/dt-bindings/reset/st,stm32mp21-rcc.h
-new file mode 100644
-index 000000000000..6463bd73d025
--- /dev/null
-+++ b/include/dt-bindings/reset/st,stm32mp21-rcc.h
-@@ -0,0 +1,138 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (C) STMicroelectronics 2025 - All Rights Reserved
-+ * Author: Gabriel Fernandez <gabriel.fernandez@foss.st.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_STM32MP21_RESET_H_
-+#define _DT_BINDINGS_STM32MP21_RESET_H_
-+
-+#define TIM1_R		0
-+#define TIM2_R		1
-+#define TIM3_R		2
-+#define TIM4_R		3
-+#define TIM5_R		4
-+#define TIM6_R		5
-+#define TIM7_R		6
-+#define TIM8_R		7
-+#define TIM10_R		8
-+#define TIM11_R		9
-+#define TIM12_R		10
-+#define TIM13_R		11
-+#define TIM14_R		12
-+#define TIM15_R		13
-+#define TIM16_R		14
-+#define TIM17_R		15
-+#define LPTIM1_R	16
-+#define LPTIM2_R	17
-+#define LPTIM3_R	18
-+#define LPTIM4_R	19
-+#define LPTIM5_R	20
-+#define SPI1_R		21
-+#define SPI2_R		22
-+#define SPI3_R		23
-+#define SPI4_R		24
-+#define SPI5_R		25
-+#define SPI6_R		26
-+#define SPDIFRX_R	27
-+#define USART1_R	28
-+#define USART2_R	29
-+#define USART3_R	30
-+#define UART4_R		31
-+#define UART5_R		32
-+#define USART6_R	33
-+#define UART7_R		34
-+#define LPUART1_R	35
-+#define I2C1_R		36
-+#define I2C2_R		37
-+#define I2C3_R		38
-+#define SAI1_R		39
-+#define SAI2_R		40
-+#define SAI3_R		41
-+#define SAI4_R		42
-+#define MDF1_R		43
-+#define FDCAN_R		44
-+#define HDP_R		45
-+#define ADC1_R		46
-+#define ADC2_R		47
-+#define ETH1_R		48
-+#define ETH2_R		49
-+#define USBH_R		50
-+#define USB2PHY1_R	51
-+#define USB2PHY2_R	52
-+#define SDMMC1_R	53
-+#define SDMMC1DLL_R	54
-+#define SDMMC2_R	55
-+#define SDMMC2DLL_R	56
-+#define SDMMC3_R	57
-+#define SDMMC3DLL_R	58
-+#define LTDC_R		59
-+#define CSI_R		60
-+#define DCMIPP_R	61
-+#define DCMIPSSI_R	62
-+#define WWDG1_R		63
-+#define VREF_R		64
-+#define DTS_R		65
-+#define CRC_R		66
-+#define SERC_R		67
-+#define I3C1_R		68
-+#define I3C2_R		69
-+#define I3C3_R		70
-+#define IWDG2_KER_R	71
-+#define IWDG4_KER_R	72
-+#define RNG1_R		73
-+#define RNG2_R		74
-+#define PKA_R		75
-+#define SAES_R		76
-+#define HASH1_R		77
-+#define HASH2_R		78
-+#define CRYP1_R		79
-+#define CRYP2_R		80
-+#define OSPI1_R		81
-+#define OSPI1DLL_R	82
-+#define OTG_R		83
-+#define FMC_R		84
-+#define DBG_R		85
-+#define GPIOA_R		86
-+#define GPIOB_R		87
-+#define GPIOC_R		88
-+#define GPIOD_R		89
-+#define GPIOE_R		90
-+#define GPIOF_R		91
-+#define GPIOG_R		92
-+#define GPIOH_R		93
-+#define GPIOI_R		94
-+#define GPIOZ_R		95
-+#define HPDMA1_R	96
-+#define HPDMA2_R	97
-+#define HPDMA3_R	98
-+#define IPCC1_R		99
-+#define C2_HOLDBOOT_R	100
-+#define C1_HOLDBOOT_R	101
-+#define C1_R		102
-+#define C1P1POR_R	103
-+#define C1P1_R		104
-+#define C2_R		105
-+#define SYS_R		106
-+#define VSW_R		107
-+#define C1MS_R		108
-+#define DDRCP_R		109
-+#define DDRCAPB_R	110
-+#define DDRPHYCAPB_R	111
-+#define DDRCFG_R	112
-+#define DDR_R		113
-+#define DDRPERFM_R	114
-+#define IWDG1_SYS_R	116
-+#define IWDG2_SYS_R	117
-+#define IWDG3_SYS_R	118
-+#define IWDG4_SYS_R	119
-+
-+#define RST_SCMI_C1_R		0
-+#define RST_SCMI_C2_R		1
-+#define RST_SCMI_C1_HOLDBOOT_R	2
-+#define RST_SCMI_C2_HOLDBOOT_R	3
-+#define RST_SCMI_FMC		4
-+#define RST_SCMI_OSPI1		5
-+#define RST_SCMI_OSPI1DLL	6
-+
-+#endif /* _DT_BINDINGS_STM32MP21_RESET_H_ */
-diff --git a/include/dt-bindings/reset/st,stm32mp25-rcc.h b/include/dt-bindings/reset/st,stm32mp25-rcc.h
-new file mode 100644
-index 000000000000..748e78ae20bd
--- /dev/null
-+++ b/include/dt-bindings/reset/st,stm32mp25-rcc.h
-@@ -0,0 +1,167 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Copyright (C) STMicroelectronics 2023 - All Rights Reserved
-+ * Author(s): Gabriel Fernandez <gabriel.fernandez@foss.st.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_STM32MP25_RESET_H_
-+#define _DT_BINDINGS_STM32MP25_RESET_H_
-+
-+#define TIM1_R		0
-+#define TIM2_R		1
-+#define TIM3_R		2
-+#define TIM4_R		3
-+#define TIM5_R		4
-+#define TIM6_R		5
-+#define TIM7_R		6
-+#define TIM8_R		7
-+#define TIM10_R		8
-+#define TIM11_R		9
-+#define TIM12_R		10
-+#define TIM13_R		11
-+#define TIM14_R		12
-+#define TIM15_R		13
-+#define TIM16_R		14
-+#define TIM17_R		15
-+#define TIM20_R		16
-+#define LPTIM1_R	17
-+#define LPTIM2_R	18
-+#define LPTIM3_R	19
-+#define LPTIM4_R	20
-+#define LPTIM5_R	21
-+#define SPI1_R		22
-+#define SPI2_R		23
-+#define SPI3_R		24
-+#define SPI4_R		25
-+#define SPI5_R		26
-+#define SPI6_R		27
-+#define SPI7_R		28
-+#define SPI8_R		29
-+#define SPDIFRX_R	30
-+#define USART1_R	31
-+#define USART2_R	32
-+#define USART3_R	33
-+#define UART4_R		34
-+#define UART5_R		35
-+#define USART6_R	36
-+#define UART7_R		37
-+#define UART8_R		38
-+#define UART9_R		39
-+#define LPUART1_R	40
-+#define IS2M_R		41
-+#define I2C1_R		42
-+#define I2C2_R		43
-+#define I2C3_R		44
-+#define I2C4_R		45
-+#define I2C5_R		46
-+#define I2C6_R		47
-+#define I2C7_R		48
-+#define I2C8_R		49
-+#define SAI1_R		50
-+#define SAI2_R		51
-+#define SAI3_R		52
-+#define SAI4_R		53
-+#define MDF1_R		54
-+#define MDF2_R		55
-+#define FDCAN_R		56
-+#define HDP_R		57
-+#define ADC12_R		58
-+#define ADC3_R		59
-+#define ETH1_R		60
-+#define ETH2_R		61
-+#define USBH_R		62
-+#define USB2PHY1_R	63
-+#define USB2PHY2_R	64
-+#define USB3DR_R	65
-+#define USB3PCIEPHY_R	66
-+#define USBTC_R		67
-+#define ETHSW_R		68
-+#define SDMMC1_R	69
-+#define SDMMC1DLL_R	70
-+#define SDMMC2_R	71
-+#define SDMMC2DLL_R	72
-+#define SDMMC3_R	73
-+#define SDMMC3DLL_R	74
-+#define GPU_R		75
-+#define LTDC_R		76
-+#define DSI_R		77
-+#define LVDS_R		78
-+#define CSI_R		79
-+#define DCMIPP_R	80
-+#define CCI_R		81
-+#define VDEC_R		82
-+#define VENC_R		83
-+#define WWDG1_R		84
-+#define WWDG2_R		85
-+#define VREF_R		86
-+#define DTS_R		87
-+#define CRC_R		88
-+#define SERC_R		89
-+#define OSPIIOM_R	90
-+#define I3C1_R		91
-+#define I3C2_R		92
-+#define I3C3_R		93
-+#define I3C4_R		94
-+#define IWDG2_KER_R	95
-+#define IWDG4_KER_R	96
-+#define RNG_R		97
-+#define PKA_R		98
-+#define SAES_R		99
-+#define HASH_R		100
-+#define CRYP1_R		101
-+#define CRYP2_R		102
-+#define PCIE_R		103
-+#define OSPI1_R		104
-+#define OSPI1DLL_R	105
-+#define OSPI2_R		106
-+#define OSPI2DLL_R	107
-+#define FMC_R		108
-+#define DBG_R		109
-+#define GPIOA_R		110
-+#define GPIOB_R		111
-+#define GPIOC_R		112
-+#define GPIOD_R		113
-+#define GPIOE_R		114
-+#define GPIOF_R		115
-+#define GPIOG_R		116
-+#define GPIOH_R		117
-+#define GPIOI_R		118
-+#define GPIOJ_R		119
-+#define GPIOK_R		120
-+#define GPIOZ_R		121
-+#define HPDMA1_R	122
-+#define HPDMA2_R	123
-+#define HPDMA3_R	124
-+#define LPDMA_R		125
-+#define HSEM_R		126
-+#define IPCC1_R		127
-+#define IPCC2_R		128
-+#define C2_HOLDBOOT_R	129
-+#define C1_HOLDBOOT_R	130
-+#define C1_R		131
-+#define C1P1POR_R	132
-+#define C1P1_R		133
-+#define C2_R		134
-+#define C3_R		135
-+#define SYS_R		136
-+#define VSW_R		137
-+#define C1MS_R		138
-+#define DDRCP_R		139
-+#define DDRCAPB_R	140
-+#define DDRPHYCAPB_R	141
-+#define DDRCFG_R	142
-+#define DDR_R		143
-+
-+#define STM32MP25_LAST_RESET	144
-+
-+#define RST_SCMI_C1_R		0
-+#define RST_SCMI_C2_R		1
-+#define RST_SCMI_C1_HOLDBOOT_R	2
-+#define RST_SCMI_C2_HOLDBOOT_R	3
-+#define RST_SCMI_FMC		4
-+#define RST_SCMI_OSPI1		5
-+#define RST_SCMI_OSPI1DLL	6
-+#define RST_SCMI_OSPI2		7
-+#define RST_SCMI_OSPI2DLL	8
-+
-+#endif /* _DT_BINDINGS_STM32MP25_RESET_H_ */
-diff --git a/include/dt-bindings/reset/starfive,jh7110-crg.h b/include/dt-bindings/reset/starfive,jh7110-crg.h
-new file mode 100644
-index 000000000000..eaf4a0d84f6a
--- /dev/null
-+++ b/include/dt-bindings/reset/starfive,jh7110-crg.h
-@@ -0,0 +1,214 @@
-+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
-+/*
-+ * Copyright (C) 2022 Emil Renner Berthing <kernel@esmil.dk>
-+ * Copyright (C) 2022 StarFive Technology Co., Ltd.
-+ */
-+
-+#ifndef __DT_BINDINGS_RESET_STARFIVE_JH7110_CRG_H__
-+#define __DT_BINDINGS_RESET_STARFIVE_JH7110_CRG_H__
-+
-+/* SYSCRG resets */
-+#define JH7110_SYSRST_JTAG_APB			0
-+#define JH7110_SYSRST_SYSCON_APB		1
-+#define JH7110_SYSRST_IOMUX_APB			2
-+#define JH7110_SYSRST_BUS			3
-+#define JH7110_SYSRST_DEBUG			4
-+#define JH7110_SYSRST_CORE0			5
-+#define JH7110_SYSRST_CORE1			6
-+#define JH7110_SYSRST_CORE2			7
-+#define JH7110_SYSRST_CORE3			8
-+#define JH7110_SYSRST_CORE4			9
-+#define JH7110_SYSRST_CORE0_ST			10
-+#define JH7110_SYSRST_CORE1_ST			11
-+#define JH7110_SYSRST_CORE2_ST			12
-+#define JH7110_SYSRST_CORE3_ST			13
-+#define JH7110_SYSRST_CORE4_ST			14
-+#define JH7110_SYSRST_TRACE0			15
-+#define JH7110_SYSRST_TRACE1			16
-+#define JH7110_SYSRST_TRACE2			17
-+#define JH7110_SYSRST_TRACE3			18
-+#define JH7110_SYSRST_TRACE4			19
-+#define JH7110_SYSRST_TRACE_COM			20
-+#define JH7110_SYSRST_GPU_APB			21
-+#define JH7110_SYSRST_GPU_DOMA			22
-+#define JH7110_SYSRST_NOC_BUS_APB		23
-+#define JH7110_SYSRST_NOC_BUS_AXICFG0_AXI	24
-+#define JH7110_SYSRST_NOC_BUS_CPU_AXI		25
-+#define JH7110_SYSRST_NOC_BUS_DISP_AXI		26
-+#define JH7110_SYSRST_NOC_BUS_GPU_AXI		27
-+#define JH7110_SYSRST_NOC_BUS_ISP_AXI		28
-+#define JH7110_SYSRST_NOC_BUS_DDRC		29
-+#define JH7110_SYSRST_NOC_BUS_STG_AXI		30
-+#define JH7110_SYSRST_NOC_BUS_VDEC_AXI		31
-+
-+#define JH7110_SYSRST_NOC_BUS_VENC_AXI		32
-+#define JH7110_SYSRST_AXI_CFG1_AHB		33
-+#define JH7110_SYSRST_AXI_CFG1_MAIN		34
-+#define JH7110_SYSRST_AXI_CFG0_MAIN		35
-+#define JH7110_SYSRST_AXI_CFG0_MAIN_DIV		36
-+#define JH7110_SYSRST_AXI_CFG0_HIFI4		37
-+#define JH7110_SYSRST_DDR_AXI			38
-+#define JH7110_SYSRST_DDR_OSC			39
-+#define JH7110_SYSRST_DDR_APB			40
-+#define JH7110_SYSRST_ISP_TOP			41
-+#define JH7110_SYSRST_ISP_TOP_AXI		42
-+#define JH7110_SYSRST_VOUT_TOP_SRC		43
-+#define JH7110_SYSRST_CODAJ12_AXI		44
-+#define JH7110_SYSRST_CODAJ12_CORE		45
-+#define JH7110_SYSRST_CODAJ12_APB		46
-+#define JH7110_SYSRST_WAVE511_AXI		47
-+#define JH7110_SYSRST_WAVE511_BPU		48
-+#define JH7110_SYSRST_WAVE511_VCE		49
-+#define JH7110_SYSRST_WAVE511_APB		50
-+#define JH7110_SYSRST_VDEC_JPG			51
-+#define JH7110_SYSRST_VDEC_MAIN			52
-+#define JH7110_SYSRST_AXIMEM0_AXI		53
-+#define JH7110_SYSRST_WAVE420L_AXI		54
-+#define JH7110_SYSRST_WAVE420L_BPU		55
-+#define JH7110_SYSRST_WAVE420L_VCE		56
-+#define JH7110_SYSRST_WAVE420L_APB		57
-+#define JH7110_SYSRST_AXIMEM1_AXI		58
-+#define JH7110_SYSRST_AXIMEM2_AXI		59
-+#define JH7110_SYSRST_INTMEM			60
-+#define JH7110_SYSRST_QSPI_AHB			61
-+#define JH7110_SYSRST_QSPI_APB			62
-+#define JH7110_SYSRST_QSPI_REF			63
-+
-+#define JH7110_SYSRST_SDIO0_AHB			64
-+#define JH7110_SYSRST_SDIO1_AHB			65
-+#define JH7110_SYSRST_GMAC1_AXI			66
-+#define JH7110_SYSRST_GMAC1_AHB			67
-+#define JH7110_SYSRST_MAILBOX_APB		68
-+#define JH7110_SYSRST_SPI0_APB			69
-+#define JH7110_SYSRST_SPI1_APB			70
-+#define JH7110_SYSRST_SPI2_APB			71
-+#define JH7110_SYSRST_SPI3_APB			72
-+#define JH7110_SYSRST_SPI4_APB			73
-+#define JH7110_SYSRST_SPI5_APB			74
-+#define JH7110_SYSRST_SPI6_APB			75
-+#define JH7110_SYSRST_I2C0_APB			76
-+#define JH7110_SYSRST_I2C1_APB			77
-+#define JH7110_SYSRST_I2C2_APB			78
-+#define JH7110_SYSRST_I2C3_APB			79
-+#define JH7110_SYSRST_I2C4_APB			80
-+#define JH7110_SYSRST_I2C5_APB			81
-+#define JH7110_SYSRST_I2C6_APB			82
-+#define JH7110_SYSRST_UART0_APB			83
-+#define JH7110_SYSRST_UART0_CORE		84
-+#define JH7110_SYSRST_UART1_APB			85
-+#define JH7110_SYSRST_UART1_CORE		86
-+#define JH7110_SYSRST_UART2_APB			87
-+#define JH7110_SYSRST_UART2_CORE		88
-+#define JH7110_SYSRST_UART3_APB			89
-+#define JH7110_SYSRST_UART3_CORE		90
-+#define JH7110_SYSRST_UART4_APB			91
-+#define JH7110_SYSRST_UART4_CORE		92
-+#define JH7110_SYSRST_UART5_APB			93
-+#define JH7110_SYSRST_UART5_CORE		94
-+#define JH7110_SYSRST_SPDIF_APB			95
-+
-+#define JH7110_SYSRST_PWMDAC_APB		96
-+#define JH7110_SYSRST_PDM_DMIC			97
-+#define JH7110_SYSRST_PDM_APB			98
-+#define JH7110_SYSRST_I2SRX_APB			99
-+#define JH7110_SYSRST_I2SRX_BCLK		100
-+#define JH7110_SYSRST_I2STX0_APB		101
-+#define JH7110_SYSRST_I2STX0_BCLK		102
-+#define JH7110_SYSRST_I2STX1_APB		103
-+#define JH7110_SYSRST_I2STX1_BCLK		104
-+#define JH7110_SYSRST_TDM_AHB			105
-+#define JH7110_SYSRST_TDM_CORE			106
-+#define JH7110_SYSRST_TDM_APB			107
-+#define JH7110_SYSRST_PWM_APB			108
-+#define JH7110_SYSRST_WDT_APB			109
-+#define JH7110_SYSRST_WDT_CORE			110
-+#define JH7110_SYSRST_CAN0_APB			111
-+#define JH7110_SYSRST_CAN0_CORE			112
-+#define JH7110_SYSRST_CAN0_TIMER		113
-+#define JH7110_SYSRST_CAN1_APB			114
-+#define JH7110_SYSRST_CAN1_CORE			115
-+#define JH7110_SYSRST_CAN1_TIMER		116
-+#define JH7110_SYSRST_TIMER_APB			117
-+#define JH7110_SYSRST_TIMER0			118
-+#define JH7110_SYSRST_TIMER1			119
-+#define JH7110_SYSRST_TIMER2			120
-+#define JH7110_SYSRST_TIMER3			121
-+#define JH7110_SYSRST_INT_CTRL_APB		122
-+#define JH7110_SYSRST_TEMP_APB			123
-+#define JH7110_SYSRST_TEMP_CORE			124
-+#define JH7110_SYSRST_JTAG_CERTIFICATION	125
-+
-+#define JH7110_SYSRST_END			126
-+
-+/* AONCRG resets */
-+#define JH7110_AONRST_GMAC0_AXI			0
-+#define JH7110_AONRST_GMAC0_AHB			1
-+#define JH7110_AONRST_IOMUX			2
-+#define JH7110_AONRST_PMU_APB			3
-+#define JH7110_AONRST_PMU_WKUP			4
-+#define JH7110_AONRST_RTC_APB			5
-+#define JH7110_AONRST_RTC_CAL			6
-+#define JH7110_AONRST_RTC_32K			7
-+
-+#define JH7110_AONRST_END			8
-+
-+/* STGCRG resets */
-+#define JH7110_STGRST_SYSCON			0
-+#define JH7110_STGRST_HIFI4_CORE		1
-+#define JH7110_STGRST_HIFI4_AXI			2
-+#define JH7110_STGRST_SEC_AHB			3
-+#define JH7110_STGRST_E24_CORE			4
-+#define JH7110_STGRST_DMA1P_AXI			5
-+#define JH7110_STGRST_DMA1P_AHB			6
-+#define JH7110_STGRST_USB0_AXI			7
-+#define JH7110_STGRST_USB0_APB			8
-+#define JH7110_STGRST_USB0_UTMI_APB		9
-+#define JH7110_STGRST_USB0_PWRUP		10
-+#define JH7110_STGRST_PCIE0_AXI_MST0		11
-+#define JH7110_STGRST_PCIE0_AXI_SLV0		12
-+#define JH7110_STGRST_PCIE0_AXI_SLV		13
-+#define JH7110_STGRST_PCIE0_BRG			14
-+#define JH7110_STGRST_PCIE0_CORE		15
-+#define JH7110_STGRST_PCIE0_APB			16
-+#define JH7110_STGRST_PCIE1_AXI_MST0		17
-+#define JH7110_STGRST_PCIE1_AXI_SLV0		18
-+#define JH7110_STGRST_PCIE1_AXI_SLV		19
-+#define JH7110_STGRST_PCIE1_BRG			20
-+#define JH7110_STGRST_PCIE1_CORE		21
-+#define JH7110_STGRST_PCIE1_APB			22
-+
-+#define JH7110_STGRST_END			23
-+
-+/* ISPCRG resets */
-+#define JH7110_ISPRST_ISPV2_TOP_WRAPPER_P	0
-+#define JH7110_ISPRST_ISPV2_TOP_WRAPPER_C	1
-+#define JH7110_ISPRST_M31DPHY_HW		2
-+#define JH7110_ISPRST_M31DPHY_B09_AON		3
-+#define JH7110_ISPRST_VIN_APB			4
-+#define JH7110_ISPRST_VIN_PIXEL_IF0		5
-+#define JH7110_ISPRST_VIN_PIXEL_IF1		6
-+#define JH7110_ISPRST_VIN_PIXEL_IF2		7
-+#define JH7110_ISPRST_VIN_PIXEL_IF3		8
-+#define JH7110_ISPRST_VIN_SYS			9
-+#define JH7110_ISPRST_VIN_P_AXI_RD		10
-+#define JH7110_ISPRST_VIN_P_AXI_WR		11
-+
-+#define JH7110_ISPRST_END			12
-+
-+/* VOUTCRG resets */
-+#define JH7110_VOUTRST_DC8200_AXI		0
-+#define JH7110_VOUTRST_DC8200_AHB		1
-+#define JH7110_VOUTRST_DC8200_CORE		2
-+#define JH7110_VOUTRST_DSITX_DPI		3
-+#define JH7110_VOUTRST_DSITX_APB		4
-+#define JH7110_VOUTRST_DSITX_RXESC		5
-+#define JH7110_VOUTRST_DSITX_SYS		6
-+#define JH7110_VOUTRST_DSITX_TXBYTEHS		7
-+#define JH7110_VOUTRST_DSITX_TXESC		8
-+#define JH7110_VOUTRST_HDMI_TX_HDMI		9
-+#define JH7110_VOUTRST_MIPITX_DPHY_SYS		10
-+#define JH7110_VOUTRST_MIPITX_DPHY_TXBYTEHS	11
-+
-+#define JH7110_VOUTRST_END			12
-+
-+#endif /* __DT_BINDINGS_RESET_STARFIVE_JH7110_CRG_H__ */
-diff --git a/include/dt-bindings/reset/stih415-resets.h b/include/dt-bindings/reset/stih415-resets.h
-deleted file mode 100644
-index 96f7831a1db0..000000000000
--- a/include/dt-bindings/reset/stih415-resets.h
-+++ /dev/null
-@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * This header provides constants for the reset controller
- * based peripheral powerdown requests on the STMicroelectronics
- * STiH415 SoC.
- */
-#ifndef _DT_BINDINGS_RESET_CONTROLLER_STIH415
-#define _DT_BINDINGS_RESET_CONTROLLER_STIH415
-
-#define STIH415_EMISS_POWERDOWN		0
-#define STIH415_NAND_POWERDOWN		1
-#define STIH415_KEYSCAN_POWERDOWN	2
-#define STIH415_USB0_POWERDOWN		3
-#define STIH415_USB1_POWERDOWN		4
-#define STIH415_USB2_POWERDOWN		5
-#define STIH415_SATA0_POWERDOWN		6
-#define STIH415_SATA1_POWERDOWN		7
-#define STIH415_PCIE_POWERDOWN		8
-
-#define STIH415_ETH0_SOFTRESET		0
-#define STIH415_ETH1_SOFTRESET		1
-#define STIH415_IRB_SOFTRESET		2
-#define STIH415_USB0_SOFTRESET		3
-#define STIH415_USB1_SOFTRESET		4
-#define STIH415_USB2_SOFTRESET		5
-#define STIH415_KEYSCAN_SOFTRESET	6
-
-#endif /* _DT_BINDINGS_RESET_CONTROLLER_STIH415 */
-diff --git a/include/dt-bindings/reset/stih416-resets.h b/include/dt-bindings/reset/stih416-resets.h
-deleted file mode 100644
-index f682c906ed5a..000000000000
--- a/include/dt-bindings/reset/stih416-resets.h
-+++ /dev/null
-@@ -1,52 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * This header provides constants for the reset controller
- * based peripheral powerdown requests on the STMicroelectronics
- * STiH416 SoC.
- */
-#ifndef _DT_BINDINGS_RESET_CONTROLLER_STIH416
-#define _DT_BINDINGS_RESET_CONTROLLER_STIH416
-
-#define STIH416_EMISS_POWERDOWN		0
-#define STIH416_NAND_POWERDOWN		1
-#define STIH416_KEYSCAN_POWERDOWN	2
-#define STIH416_USB0_POWERDOWN		3
-#define STIH416_USB1_POWERDOWN		4
-#define STIH416_USB2_POWERDOWN		5
-#define STIH416_USB3_POWERDOWN		6
-#define STIH416_SATA0_POWERDOWN		7
-#define STIH416_SATA1_POWERDOWN		8
-#define STIH416_PCIE0_POWERDOWN		9
-#define STIH416_PCIE1_POWERDOWN		10
-
-#define STIH416_ETH0_SOFTRESET		0
-#define STIH416_ETH1_SOFTRESET		1
-#define STIH416_IRB_SOFTRESET		2
-#define STIH416_USB0_SOFTRESET		3
-#define STIH416_USB1_SOFTRESET		4
-#define STIH416_USB2_SOFTRESET		5
-#define STIH416_USB3_SOFTRESET		6
-#define STIH416_SATA0_SOFTRESET		7
-#define STIH416_SATA1_SOFTRESET		8
-#define STIH416_PCIE0_SOFTRESET		9
-#define STIH416_PCIE1_SOFTRESET		10
-#define STIH416_AUD_DAC_SOFTRESET	11
-#define STIH416_HDTVOUT_SOFTRESET	12
-#define STIH416_VTAC_M_RX_SOFTRESET	13
-#define STIH416_VTAC_A_RX_SOFTRESET	14
-#define STIH416_SYNC_HD_SOFTRESET	15
-#define STIH416_SYNC_SD_SOFTRESET	16
-#define STIH416_BLITTER_SOFTRESET	17
-#define STIH416_GPU_SOFTRESET		18
-#define STIH416_VTAC_M_TX_SOFTRESET	19
-#define STIH416_VTAC_A_TX_SOFTRESET	20
-#define STIH416_VTG_AUX_SOFTRESET	21
-#define STIH416_JPEG_DEC_SOFTRESET	22
-#define STIH416_HVA_SOFTRESET		23
-#define STIH416_COMPO_M_SOFTRESET	24
-#define STIH416_COMPO_A_SOFTRESET	25
-#define STIH416_VP8_DEC_SOFTRESET	26
-#define STIH416_VTG_MAIN_SOFTRESET	27
-#define STIH416_KEYSCAN_SOFTRESET	28
-
-#endif /* _DT_BINDINGS_RESET_CONTROLLER_STIH416 */
-diff --git a/include/dt-bindings/reset/stm32mp1-resets.h b/include/dt-bindings/reset/stm32mp1-resets.h
-index 4ffa7c3612e6..9071f139649f 100644
--- a/include/dt-bindings/reset/stm32mp1-resets.h
-+++ b/include/dt-bindings/reset/stm32mp1-resets.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */
-+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
- /*
-  * Copyright (C) STMicroelectronics 2018 - All Rights Reserved
-  * Author: Gabriel Fernandez <gabriel.fernandez@st.com> for STMicroelectronics.
-diff --git a/include/dt-bindings/reset/stm32mp13-resets.h b/include/dt-bindings/reset/stm32mp13-resets.h
-index 934864e90da6..ecb37c7ddde1 100644
--- a/include/dt-bindings/reset/stm32mp13-resets.h
-+++ b/include/dt-bindings/reset/stm32mp13-resets.h
-@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */
- /*
-  * Copyright (C) STMicroelectronics 2018 - All Rights Reserved
- * Author: Gabriel Fernandez <gabriel.fernandez@st.com> for STMicroelectronics.
-+ * Author: Gabriel Fernandez <gabriel.fernandez@foss.st.com> for STMicroelectronics.
-  */
- 
- #ifndef _DT_BINDINGS_STM32MP13_RESET_H_
-diff --git a/include/dt-bindings/reset/sun20i-d1-ccu.h b/include/dt-bindings/reset/sun20i-d1-ccu.h
-index de9ff5203239..79e52aca5912 100644
--- a/include/dt-bindings/reset/sun20i-d1-ccu.h
-+++ b/include/dt-bindings/reset/sun20i-d1-ccu.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (c) 2020 huangzhenwei@allwinnertech.com
-  * Copyright (C) 2021 Samuel Holland <samuel@sholland.org>
-@@ -73,5 +73,7 @@
- #define RST_BUS_DSP_CFG		63
- #define RST_BUS_DSP_DBG		64
- #define RST_BUS_RISCV_CFG	65
-+#define RST_BUS_CAN0		66
-+#define RST_BUS_CAN1		67
- 
- #endif /* _DT_BINDINGS_RST_SUN20I_D1_CCU_H_ */
-diff --git a/include/dt-bindings/reset/sun20i-d1-r-ccu.h b/include/dt-bindings/reset/sun20i-d1-r-ccu.h
-index d93d6423d283..e20babc990af 100644
--- a/include/dt-bindings/reset/sun20i-d1-r-ccu.h
-+++ b/include/dt-bindings/reset/sun20i-d1-r-ccu.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (C) 2021 Samuel Holland <samuel@sholland.org>
-  */
-diff --git a/include/dt-bindings/reset/sun50i-a100-ccu.h b/include/dt-bindings/reset/sun50i-a100-ccu.h
-index 55c0ada99885..d13764bc1860 100644
--- a/include/dt-bindings/reset/sun50i-a100-ccu.h
-+++ b/include/dt-bindings/reset/sun50i-a100-ccu.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (c) 2020 Yangtao Li <frank@allwinnertech.com>
-  */
-diff --git a/include/dt-bindings/reset/sun50i-a100-r-ccu.h b/include/dt-bindings/reset/sun50i-a100-r-ccu.h
-index 737bf6f66626..1e7c4431f03c 100644
--- a/include/dt-bindings/reset/sun50i-a100-r-ccu.h
-+++ b/include/dt-bindings/reset/sun50i-a100-r-ccu.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (c) 2020 Yangtao Li <frank@allwinnertech.com>
-  */
-diff --git a/include/dt-bindings/reset/sun50i-h6-ccu.h b/include/dt-bindings/reset/sun50i-h6-ccu.h
-index 81106f455097..d038ddfa4818 100644
--- a/include/dt-bindings/reset/sun50i-h6-ccu.h
-+++ b/include/dt-bindings/reset/sun50i-h6-ccu.h
-@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: (GPL-2.0+ or MIT)
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (C) 2017 Icenowy Zheng <icenowy@aosc.io>
-  */
-diff --git a/include/dt-bindings/reset/sun50i-h6-r-ccu.h b/include/dt-bindings/reset/sun50i-h6-r-ccu.h
-index 7950e799c76d..d541ade884fc 100644
--- a/include/dt-bindings/reset/sun50i-h6-r-ccu.h
-+++ b/include/dt-bindings/reset/sun50i-h6-r-ccu.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (C) 2016 Icenowy Zheng <icenowy@aosc.xyz>
-  */
-diff --git a/include/dt-bindings/reset/sun50i-h616-ccu.h b/include/dt-bindings/reset/sun50i-h616-ccu.h
-index cb6285a8d128..ba626f7015b5 100644
--- a/include/dt-bindings/reset/sun50i-h616-ccu.h
-+++ b/include/dt-bindings/reset/sun50i-h616-ccu.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */
-+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
- /*
-  * Copyright (C) 2020 Arm Ltd.
-  */
-@@ -66,5 +66,9 @@
- #define RST_BUS_TVE0		57
- #define RST_BUS_HDCP		58
- #define RST_BUS_KEYADC		59
-+#define RST_BUS_GPADC		60
-+#define RST_BUS_TCON_LCD0	61
-+#define RST_BUS_TCON_LCD1	62
-+#define RST_BUS_LVDS		63
- 
- #endif /* _DT_BINDINGS_RESET_SUN50I_H616_H_ */
-diff --git a/include/dt-bindings/reset/sun55i-a523-ccu.h b/include/dt-bindings/reset/sun55i-a523-ccu.h
-new file mode 100644
-index 000000000000..70df503f34fe
--- /dev/null
-+++ b/include/dt-bindings/reset/sun55i-a523-ccu.h
-@@ -0,0 +1,88 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (c) 2024 Arm Ltd.
-+ */
-+
-+#ifndef _DT_BINDINGS_RST_SUN55I_A523_CCU_H_
-+#define _DT_BINDINGS_RST_SUN55I_A523_CCU_H_
-+
-+#define RST_MBUS		0
-+#define RST_BUS_NSI		1
-+#define RST_BUS_DE		2
-+#define RST_BUS_DI		3
-+#define RST_BUS_G2D		4
-+#define RST_BUS_SYS		5
-+#define RST_BUS_GPU		6
-+#define RST_BUS_CE		7
-+#define RST_BUS_SYS_CE		8
-+#define RST_BUS_VE		9
-+#define RST_BUS_DMA		10
-+#define RST_BUS_MSGBOX		11
-+#define RST_BUS_SPINLOCK	12
-+#define RST_BUS_CPUXTIMER	13
-+#define RST_BUS_DBG		14
-+#define RST_BUS_PWM0		15
-+#define RST_BUS_PWM1		16
-+#define RST_BUS_DRAM		17
-+#define RST_BUS_NAND		18
-+#define RST_BUS_MMC0		19
-+#define RST_BUS_MMC1		20
-+#define RST_BUS_MMC2		21
-+#define RST_BUS_SYSDAP		22
-+#define RST_BUS_UART0		23
-+#define RST_BUS_UART1		24
-+#define RST_BUS_UART2		25
-+#define RST_BUS_UART3		26
-+#define RST_BUS_UART4		27
-+#define RST_BUS_UART5		28
-+#define RST_BUS_UART6		29
-+#define RST_BUS_UART7		30
-+#define RST_BUS_I2C0		31
-+#define RST_BUS_I2C1		32
-+#define RST_BUS_I2C2		33
-+#define RST_BUS_I2C3		34
-+#define RST_BUS_I2C4		35
-+#define RST_BUS_I2C5		36
-+#define RST_BUS_CAN		37
-+#define RST_BUS_SPI0		38
-+#define RST_BUS_SPI1		39
-+#define RST_BUS_SPI2		40
-+#define RST_BUS_SPIFC		41
-+#define RST_BUS_EMAC0		42
-+#define RST_BUS_EMAC1		43
-+#define RST_BUS_IR_RX		44
-+#define RST_BUS_IR_TX		45
-+#define RST_BUS_GPADC0		46
-+#define RST_BUS_GPADC1		47
-+#define RST_BUS_THS		48
-+#define RST_USB_PHY0		49
-+#define RST_USB_PHY1		50
-+#define RST_BUS_OHCI0		51
-+#define RST_BUS_OHCI1		52
-+#define RST_BUS_EHCI0		53
-+#define RST_BUS_EHCI1		54
-+#define RST_BUS_OTG		55
-+#define RST_BUS_3		56
-+#define RST_BUS_LRADC		57
-+#define RST_BUS_PCIE_USB3	58
-+#define RST_BUS_DISPLAY0_TOP	59
-+#define RST_BUS_DISPLAY1_TOP	60
-+#define RST_BUS_HDMI_MAIN	61
-+#define RST_BUS_HDMI_SUB	62
-+#define RST_BUS_MIPI_DSI0	63
-+#define RST_BUS_MIPI_DSI1	64
-+#define RST_BUS_TCON_LCD0	65
-+#define RST_BUS_TCON_LCD1	66
-+#define RST_BUS_TCON_LCD2	67
-+#define RST_BUS_TCON_TV0	68
-+#define RST_BUS_TCON_TV1	69
-+#define RST_BUS_LVDS0		70
-+#define RST_BUS_LVDS1		71
-+#define RST_BUS_EDP		72
-+#define RST_BUS_VIDEO_OUT0	73
-+#define RST_BUS_VIDEO_OUT1	74
-+#define RST_BUS_LEDC		75
-+#define RST_BUS_CSI		76
-+#define RST_BUS_ISP		77
-+
-+#endif /* _DT_BINDINGS_RST_SUN55I_A523_CCU_H_ */
-diff --git a/include/dt-bindings/reset/sun55i-a523-mcu-ccu.h b/include/dt-bindings/reset/sun55i-a523-mcu-ccu.h
-new file mode 100644
-index 000000000000..a89a0b44f08b
--- /dev/null
-+++ b/include/dt-bindings/reset/sun55i-a523-mcu-ccu.h
-@@ -0,0 +1,30 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (C) 2025 Chen-Yu Tsai <wens@csie.org>
-+ */
-+
-+#ifndef _DT_BINDINGS_RST_SUN55I_A523_MCU_CCU_H_
-+#define _DT_BINDINGS_RST_SUN55I_A523_MCU_CCU_H_
-+
-+#define RST_BUS_MCU_I2S0		0
-+#define RST_BUS_MCU_I2S1		1
-+#define RST_BUS_MCU_I2S2		2
-+#define RST_BUS_MCU_I2S3		3
-+#define RST_BUS_MCU_SPDIF		4
-+#define RST_BUS_MCU_DMIC		5
-+#define RST_BUS_MCU_AUDIO_CODEC		6
-+#define RST_BUS_MCU_DSP_MSGBOX		7
-+#define RST_BUS_MCU_DSP_CFG		8
-+#define RST_BUS_MCU_NPU			9
-+#define RST_BUS_MCU_TIMER		10
-+#define RST_BUS_MCU_DSP_DEBUG		11
-+#define RST_BUS_MCU_DSP			12
-+#define RST_BUS_MCU_DMA			13
-+#define RST_BUS_MCU_PUBSRAM		14
-+#define RST_BUS_MCU_RISCV_CFG		15
-+#define RST_BUS_MCU_RISCV_DEBUG		16
-+#define RST_BUS_MCU_RISCV_CORE		17
-+#define RST_BUS_MCU_RISCV_MSGBOX	18
-+#define RST_BUS_MCU_PWM0		19
-+
-+#endif /* _DT_BINDINGS_RST_SUN55I_A523_MCU_CCU_H_ */
-diff --git a/include/dt-bindings/reset/sun55i-a523-r-ccu.h b/include/dt-bindings/reset/sun55i-a523-r-ccu.h
-new file mode 100644
-index 000000000000..eb31ae9958d6
--- /dev/null
-+++ b/include/dt-bindings/reset/sun55i-a523-r-ccu.h
-@@ -0,0 +1,26 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
-+/*
-+ * Copyright (C) 2024 Arm Ltd.
-+ */
-+
-+#ifndef _DT_BINDINGS_RST_SUN55I_A523_R_CCU_H_
-+#define _DT_BINDINGS_RST_SUN55I_A523_R_CCU_H_
-+
-+#define RST_BUS_R_TIMER		0
-+#define RST_BUS_R_TWD		1
-+#define RST_BUS_R_PWMCTRL	2
-+#define RST_BUS_R_SPI		3
-+#define RST_BUS_R_SPINLOCK	4
-+#define RST_BUS_R_MSGBOX	5
-+#define RST_BUS_R_UART0		6
-+#define RST_BUS_R_UART1		7
-+#define RST_BUS_R_I2C0		8
-+#define RST_BUS_R_I2C1		9
-+#define RST_BUS_R_I2C2		10
-+#define RST_BUS_R_PPU1		11
-+#define RST_BUS_R_IR_RX		12
-+#define RST_BUS_R_RTC		13
-+#define RST_BUS_R_CPUCFG	14
-+#define RST_BUS_R_PPU0		15
-+
-+#endif /* _DT_BINDINGS_RST_SUN55I_A523_R_CCU_H_ */
-diff --git a/include/dt-bindings/reset/tegra234-reset.h b/include/dt-bindings/reset/tegra234-reset.h
-index d48d22b2bc7f..85cc423a7bdf 100644
--- a/include/dt-bindings/reset/tegra234-reset.h
-+++ b/include/dt-bindings/reset/tegra234-reset.h
-@@ -10,14 +10,29 @@
-  * @brief Identifiers for Resets controllable by firmware
-  * @{
-  */
-+#define TEGRA234_RESET_ACTMON			1U
-+#define TEGRA234_RESET_ADSP_ALL			2U
-+#define TEGRA234_RESET_DSI_CORE			3U
-+#define TEGRA234_RESET_CAN1			4U
-+#define TEGRA234_RESET_CAN2			5U
-+#define TEGRA234_RESET_DLA0			6U
-+#define TEGRA234_RESET_DLA1			7U
-+#define TEGRA234_RESET_DPAUX			8U
-+#define TEGRA234_RESET_OFA			9U
-+#define TEGRA234_RESET_NVJPG1			10U
- #define TEGRA234_RESET_PEX1_CORE_6		11U
- #define TEGRA234_RESET_PEX1_CORE_6_APB		12U
- #define TEGRA234_RESET_PEX1_COMMON_APB		13U
- #define TEGRA234_RESET_PEX2_CORE_7		14U
- #define TEGRA234_RESET_PEX2_CORE_7_APB		15U
-+#define TEGRA234_RESET_NVDISPLAY		16U
-+#define TEGRA234_RESET_EQOS			17U
- #define TEGRA234_RESET_GPCDMA			18U
-+#define TEGRA234_RESET_GPU			19U
- #define TEGRA234_RESET_HDA			20U
- #define TEGRA234_RESET_HDACODEC			21U
-+#define TEGRA234_RESET_EQOS_MACSEC		22U
-+#define TEGRA234_RESET_EQOS_MACSEC_SECURE	23U
- #define TEGRA234_RESET_I2C1			24U
- #define TEGRA234_RESET_PEX2_CORE_8		25U
- #define TEGRA234_RESET_PEX2_CORE_8_APB		26U
-@@ -30,15 +45,36 @@
- #define TEGRA234_RESET_I2C7			33U
- #define TEGRA234_RESET_I2C8			34U
- #define TEGRA234_RESET_I2C9			35U
-+#define TEGRA234_RESET_ISP			36U
-+#define TEGRA234_RESET_MIPI_CAL			37U
-+#define TEGRA234_RESET_MPHY_CLK_CTL		38U
-+#define TEGRA234_RESET_MPHY_L0_RX		39U
-+#define TEGRA234_RESET_MPHY_L0_TX		40U
-+#define TEGRA234_RESET_MPHY_L1_RX		41U
-+#define TEGRA234_RESET_MPHY_L1_TX		42U
-+#define TEGRA234_RESET_NVCSI			43U
-+#define TEGRA234_RESET_NVDEC			44U
- #define TEGRA234_RESET_MGBE0_PCS		45U
- #define TEGRA234_RESET_MGBE0_MAC		46U
-+#define TEGRA234_RESET_MGBE0_MACSEC		47U
-+#define TEGRA234_RESET_MGBE0_MACSEC_SECURE	48U
- #define TEGRA234_RESET_MGBE1_PCS		49U
- #define TEGRA234_RESET_MGBE1_MAC		50U
-+#define TEGRA234_RESET_MGBE1_MACSEC		51U
-+#define TEGRA234_RESET_MGBE1_MACSEC_SECURE	52U
- #define TEGRA234_RESET_MGBE2_PCS		53U
- #define TEGRA234_RESET_MGBE2_MAC		54U
-+#define TEGRA234_RESET_MGBE2_MACSEC		55U
- #define TEGRA234_RESET_PEX2_CORE_10		56U
- #define TEGRA234_RESET_PEX2_CORE_10_APB		57U
- #define TEGRA234_RESET_PEX2_COMMON_APB		58U
-+#define TEGRA234_RESET_NVENC			59U
-+#define TEGRA234_RESET_MGBE2_MACSEC_SECURE	60U
-+#define TEGRA234_RESET_NVJPG			61U
-+#define TEGRA234_RESET_LA			64U
-+#define TEGRA234_RESET_HWPM			65U
-+#define TEGRA234_RESET_PVA0_ALL			66U
-+#define TEGRA234_RESET_CEC			67U
- #define TEGRA234_RESET_PWM1			68U
- #define TEGRA234_RESET_PWM2			69U
- #define TEGRA234_RESET_PWM3			70U
-@@ -49,11 +85,43 @@
- #define TEGRA234_RESET_PWM8			75U
- #define TEGRA234_RESET_QSPI0			76U
- #define TEGRA234_RESET_QSPI1			77U
-+#define TEGRA234_RESET_I2S7			78U
-+#define TEGRA234_RESET_I2S8			79U
-+#define TEGRA234_RESET_SCE_ALL			80U
-+#define TEGRA234_RESET_RCE_ALL			81U
-+#define TEGRA234_RESET_SDMMC1			82U
-+#define TEGRA234_RESET_RSVD_83			83U
-+#define TEGRA234_RESET_RSVD_84			84U
- #define TEGRA234_RESET_SDMMC4			85U
- #define TEGRA234_RESET_MGBE3_PCS		87U
- #define TEGRA234_RESET_MGBE3_MAC		88U
-+#define TEGRA234_RESET_MGBE3_MACSEC		89U
-+#define TEGRA234_RESET_MGBE3_MACSEC_SECURE	90U
-+#define TEGRA234_RESET_SPI1			91U
-+#define TEGRA234_RESET_SPI2			92U
-+#define TEGRA234_RESET_SPI3			93U
-+#define TEGRA234_RESET_SPI4			94U
-+#define TEGRA234_RESET_TACH0			95U
-+#define TEGRA234_RESET_TACH1			96U
-+#define TEGRA234_RESET_SPI5			97U
-+#define TEGRA234_RESET_TSEC			98U
-+#define TEGRA234_RESET_UARTI			99U
- #define TEGRA234_RESET_UARTA			100U
-#define TEGRA234_RESET_VIC                      113U
-+#define TEGRA234_RESET_UARTB			101U
-+#define TEGRA234_RESET_UARTC			102U
-+#define TEGRA234_RESET_UARTD			103U
-+#define TEGRA234_RESET_UARTE			104U
-+#define TEGRA234_RESET_UARTF			105U
-+#define TEGRA234_RESET_UARTJ			106U
-+#define TEGRA234_RESET_UARTH			107U
-+#define TEGRA234_RESET_UFSHC			108U
-+#define TEGRA234_RESET_UFSHC_AXI_M		109U
-+#define TEGRA234_RESET_UFSHC_LP_SEQ		110U
-+#define TEGRA234_RESET_RSVD_111			111U
-+#define TEGRA234_RESET_VI			112U
-+#define TEGRA234_RESET_VIC			113U
-+#define TEGRA234_RESET_XUSB_PADCTL		114U
-+#define TEGRA234_RESET_VI2			115U
- #define TEGRA234_RESET_PEX0_CORE_0		116U
- #define TEGRA234_RESET_PEX0_CORE_1		117U
- #define TEGRA234_RESET_PEX0_CORE_2		118U
-@@ -65,8 +133,49 @@
- #define TEGRA234_RESET_PEX0_CORE_3_APB		124U
- #define TEGRA234_RESET_PEX0_CORE_4_APB		125U
- #define TEGRA234_RESET_PEX0_COMMON_APB		126U
-+#define TEGRA234_RESET_RSVD_127			127U
-+#define TEGRA234_RESET_NVHS_UPHY_PLL1		128U
- #define TEGRA234_RESET_PEX1_CORE_5		129U
- #define TEGRA234_RESET_PEX1_CORE_5_APB		130U
-+#define TEGRA234_RESET_GBE_UPHY			131U
-+#define TEGRA234_RESET_GBE_UPHY_PM		132U
-+#define TEGRA234_RESET_NVHS_UPHY		133U
-+#define TEGRA234_RESET_NVHS_UPHY_PLL0		134U
-+#define TEGRA234_RESET_NVHS_UPHY_L0		135U
-+#define TEGRA234_RESET_NVHS_UPHY_L1		136U
-+#define TEGRA234_RESET_NVHS_UPHY_L2		137U
-+#define TEGRA234_RESET_NVHS_UPHY_L3		138U
-+#define TEGRA234_RESET_NVHS_UPHY_L4		139U
-+#define TEGRA234_RESET_NVHS_UPHY_L5		140U
-+#define TEGRA234_RESET_NVHS_UPHY_L6		141U
-+#define TEGRA234_RESET_NVHS_UPHY_L7		142U
-+#define TEGRA234_RESET_NVHS_UPHY_PM		143U
-+#define TEGRA234_RESET_DMIC5			144U
-+#define TEGRA234_RESET_APE			145U
-+#define TEGRA234_RESET_PEX_USB_UPHY		146U
-+#define TEGRA234_RESET_PEX_USB_UPHY_L0		147U
-+#define TEGRA234_RESET_PEX_USB_UPHY_L1		148U
-+#define TEGRA234_RESET_PEX_USB_UPHY_L2		149U
-+#define TEGRA234_RESET_PEX_USB_UPHY_L3		150U
-+#define TEGRA234_RESET_PEX_USB_UPHY_L4		151U
-+#define TEGRA234_RESET_PEX_USB_UPHY_L5		152U
-+#define TEGRA234_RESET_PEX_USB_UPHY_L6		153U
-+#define TEGRA234_RESET_PEX_USB_UPHY_L7		154U
-+#define TEGRA234_RESET_PEX_USB_UPHY_PLL0	159U
-+#define TEGRA234_RESET_PEX_USB_UPHY_PLL1	160U
-+#define TEGRA234_RESET_PEX_USB_UPHY_PLL2	161U
-+#define TEGRA234_RESET_PEX_USB_UPHY_PLL3	162U
-+#define TEGRA234_RESET_GBE_UPHY_L0		163U
-+#define TEGRA234_RESET_GBE_UPHY_L1		164U
-+#define TEGRA234_RESET_GBE_UPHY_L2		165U
-+#define TEGRA234_RESET_GBE_UPHY_L3		166U
-+#define TEGRA234_RESET_GBE_UPHY_L4		167U
-+#define TEGRA234_RESET_GBE_UPHY_L5		168U
-+#define TEGRA234_RESET_GBE_UPHY_L6		169U
-+#define TEGRA234_RESET_GBE_UPHY_L7		170U
-+#define TEGRA234_RESET_GBE_UPHY_PLL0		171U
-+#define TEGRA234_RESET_GBE_UPHY_PLL1		172U
-+#define TEGRA234_RESET_GBE_UPHY_PLL2		173U
- 
- /** @} */
- 
-diff --git a/include/dt-bindings/reset/thead,th1520-reset.h b/include/dt-bindings/reset/thead,th1520-reset.h
-new file mode 100644
-index 000000000000..ba6805b6b12a
--- /dev/null
-+++ b/include/dt-bindings/reset/thead,th1520-reset.h
-@@ -0,0 +1,236 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2024 Samsung Electronics Co., Ltd.
-+ * Author: Michal Wilczynski <m.wilczynski@samsung.com>
-+ */
-+
-+#ifndef _DT_BINDINGS_TH1520_RESET_H
-+#define _DT_BINDINGS_TH1520_RESET_H
-+
-+/* AO Subsystem */
-+#define TH1520_RESET_ID_SYSTEM		0
-+#define TH1520_RESET_ID_RTC_APB		1
-+#define TH1520_RESET_ID_RTC_REF		2
-+#define TH1520_RESET_ID_AOGPIO_DB	3
-+#define TH1520_RESET_ID_AOGPIO_APB	4
-+#define TH1520_RESET_ID_AOI2C_APB	5
-+#define TH1520_RESET_ID_PVT_APB		6
-+#define TH1520_RESET_ID_E902_CORE	7
-+#define TH1520_RESET_ID_E902_HAD	8
-+#define TH1520_RESET_ID_AOTIMER_APB	9
-+#define TH1520_RESET_ID_AOTIMER_CORE	10
-+#define TH1520_RESET_ID_AOWDT_APB	11
-+#define TH1520_RESET_ID_APSYS		12
-+#define TH1520_RESET_ID_NPUSYS		13
-+#define TH1520_RESET_ID_DDRSYS		14
-+#define TH1520_RESET_ID_AXI_AP2CP	15
-+#define TH1520_RESET_ID_AXI_CP2AP	16
-+#define TH1520_RESET_ID_AXI_CP2SRAM	17
-+#define TH1520_RESET_ID_AUDSYS_CORE	18
-+#define TH1520_RESET_ID_AUDSYS_IOPMP	19
-+#define TH1520_RESET_ID_AUDSYS		20
-+#define TH1520_RESET_ID_DSP0		21
-+#define TH1520_RESET_ID_DSP1		22
-+#define TH1520_RESET_ID_GPU_MODULE	23
-+#define TH1520_RESET_ID_VDEC		24
-+#define TH1520_RESET_ID_VENC		25
-+#define TH1520_RESET_ID_ADC_APB		26
-+#define TH1520_RESET_ID_AUDGPIO_DB	27
-+#define TH1520_RESET_ID_AUDGPIO_APB	28
-+#define TH1520_RESET_ID_AOUART_IF	29
-+#define TH1520_RESET_ID_AOUART_APB	30
-+#define TH1520_RESET_ID_SRAM_AXI_P0	31
-+#define TH1520_RESET_ID_SRAM_AXI_P1	32
-+#define TH1520_RESET_ID_SRAM_AXI_P2	33
-+#define TH1520_RESET_ID_SRAM_AXI_P3	34
-+#define TH1520_RESET_ID_SRAM_AXI_P4	35
-+#define TH1520_RESET_ID_SRAM_AXI_CORE	36
-+#define TH1520_RESET_ID_SE		37
-+
-+/* AP Subsystem */
-+#define TH1520_RESET_ID_BROM			0
-+#define TH1520_RESET_ID_C910_TOP		1
-+#define TH1520_RESET_ID_NPU			2
-+#define TH1520_RESET_ID_WDT0			3
-+#define TH1520_RESET_ID_WDT1			4
-+#define TH1520_RESET_ID_C910_C0			5
-+#define TH1520_RESET_ID_C910_C1			6
-+#define TH1520_RESET_ID_C910_C2			7
-+#define TH1520_RESET_ID_C910_C3			8
-+#define TH1520_RESET_ID_CHIP_DBG_CORE		9
-+#define TH1520_RESET_ID_CHIP_DBG_AXI		10
-+#define TH1520_RESET_ID_AXI4_CPUSYS2_AXI	11
-+#define TH1520_RESET_ID_AXI4_CPUSYS2_APB	12
-+#define TH1520_RESET_ID_X2H_CPUSYS		13
-+#define TH1520_RESET_ID_AHB2_CPUSYS		14
-+#define TH1520_RESET_ID_APB3_CPUSYS		15
-+#define TH1520_RESET_ID_MBOX0_APB		16
-+#define TH1520_RESET_ID_MBOX1_APB		17
-+#define TH1520_RESET_ID_MBOX2_APB		18
-+#define TH1520_RESET_ID_MBOX3_APB		19
-+#define TH1520_RESET_ID_TIMER0_APB		20
-+#define TH1520_RESET_ID_TIMER0_CORE		21
-+#define TH1520_RESET_ID_TIMER1_APB		22
-+#define TH1520_RESET_ID_TIMER1_CORE		23
-+#define TH1520_RESET_ID_PERISYS_AHB		24
-+#define TH1520_RESET_ID_PERISYS_APB1		25
-+#define TH1520_RESET_ID_PERISYS_APB2		26
-+#define TH1520_RESET_ID_GMAC0_APB		27
-+#define TH1520_RESET_ID_GMAC0_AHB		28
-+#define TH1520_RESET_ID_GMAC0_CLKGEN		29
-+#define TH1520_RESET_ID_GMAC0_AXI		30
-+#define TH1520_RESET_ID_UART0_APB		31
-+#define TH1520_RESET_ID_UART0_IF		32
-+#define TH1520_RESET_ID_UART1_APB		33
-+#define TH1520_RESET_ID_UART1_IF		34
-+#define TH1520_RESET_ID_UART2_APB		35
-+#define TH1520_RESET_ID_UART2_IF		36
-+#define TH1520_RESET_ID_UART3_APB		37
-+#define TH1520_RESET_ID_UART3_IF		38
-+#define TH1520_RESET_ID_UART4_APB		39
-+#define TH1520_RESET_ID_UART4_IF		40
-+#define TH1520_RESET_ID_UART5_APB		41
-+#define TH1520_RESET_ID_UART5_IF		42
-+#define TH1520_RESET_ID_QSPI0_IF		43
-+#define TH1520_RESET_ID_QSPI0_APB		44
-+#define TH1520_RESET_ID_QSPI1_IF		45
-+#define TH1520_RESET_ID_QSPI1_APB		46
-+#define TH1520_RESET_ID_SPI_IF			47
-+#define TH1520_RESET_ID_SPI_APB			48
-+#define TH1520_RESET_ID_I2C0_APB		49
-+#define TH1520_RESET_ID_I2C0_CORE		50
-+#define TH1520_RESET_ID_I2C1_APB		51
-+#define TH1520_RESET_ID_I2C1_CORE		52
-+#define TH1520_RESET_ID_I2C2_APB		53
-+#define TH1520_RESET_ID_I2C2_CORE		54
-+#define TH1520_RESET_ID_I2C3_APB		55
-+#define TH1520_RESET_ID_I2C3_CORE		56
-+#define TH1520_RESET_ID_I2C4_APB		57
-+#define TH1520_RESET_ID_I2C4_CORE		58
-+#define TH1520_RESET_ID_I2C5_APB		59
-+#define TH1520_RESET_ID_I2C5_CORE		60
-+#define TH1520_RESET_ID_GPIO0_DB		61
-+#define TH1520_RESET_ID_GPIO0_APB		62
-+#define TH1520_RESET_ID_GPIO1_DB		63
-+#define TH1520_RESET_ID_GPIO1_APB		64
-+#define TH1520_RESET_ID_GPIO2_DB		65
-+#define TH1520_RESET_ID_GPIO2_APB		66
-+#define TH1520_RESET_ID_PWM_COUNTER		67
-+#define TH1520_RESET_ID_PWM_APB			68
-+#define TH1520_RESET_ID_PADCTRL0_APB		69
-+#define TH1520_RESET_ID_CPU2PERI_X2H		70
-+#define TH1520_RESET_ID_CPU2AON_X2H		71
-+#define TH1520_RESET_ID_AON2CPU_A2X		72
-+#define TH1520_RESET_ID_NPUSYS_AXI		73
-+#define TH1520_RESET_ID_NPUSYS_AXI_APB		74
-+#define TH1520_RESET_ID_CPU2VP_X2P		75
-+#define TH1520_RESET_ID_CPU2VI_X2H		76
-+#define TH1520_RESET_ID_BMU_AXI			77
-+#define TH1520_RESET_ID_BMU_APB			78
-+#define TH1520_RESET_ID_DMAC_CPUSYS_AXI		79
-+#define TH1520_RESET_ID_DMAC_CPUSYS_AHB		80
-+#define TH1520_RESET_ID_SPINLOCK		81
-+#define TH1520_RESET_ID_CFG2TEE			82
-+#define TH1520_RESET_ID_DSMART			83
-+#define TH1520_RESET_ID_GPIO3_DB		84
-+#define TH1520_RESET_ID_GPIO3_APB		85
-+#define TH1520_RESET_ID_PERI_I2S		86
-+#define TH1520_RESET_ID_PERI_APB3		87
-+#define TH1520_RESET_ID_PERI2PERI1_APB		88
-+#define TH1520_RESET_ID_VPSYS_APB		89
-+#define TH1520_RESET_ID_PERISYS_APB4		90
-+#define TH1520_RESET_ID_GMAC1_APB		91
-+#define TH1520_RESET_ID_GMAC1_AHB		92
-+#define TH1520_RESET_ID_GMAC1_CLKGEN		93
-+#define TH1520_RESET_ID_GMAC1_AXI		94
-+#define TH1520_RESET_ID_GMAC_AXI		95
-+#define TH1520_RESET_ID_GMAC_AXI_APB		96
-+#define TH1520_RESET_ID_PADCTRL1_APB		97
-+#define TH1520_RESET_ID_VOSYS_AXI		98
-+#define TH1520_RESET_ID_VOSYS_AXI_APB		99
-+#define TH1520_RESET_ID_VOSYS_AXI_X2X		100
-+#define TH1520_RESET_ID_MISC2VP_X2X		101
-+#define TH1520_RESET_ID_DSPSYS			102
-+#define TH1520_RESET_ID_VISYS			103
-+#define TH1520_RESET_ID_VOSYS			104
-+#define TH1520_RESET_ID_VPSYS			105
-+
-+/* DSP Subsystem */
-+#define TH1520_RESET_ID_X2X_DSP1	0
-+#define TH1520_RESET_ID_X2X_DSP0	1
-+#define TH1520_RESET_ID_X2X_SLAVE_DSP1	2
-+#define TH1520_RESET_ID_X2X_SLAVE_DSP0	3
-+#define TH1520_RESET_ID_DSP0_CORE	4
-+#define TH1520_RESET_ID_DSP0_DEBUG	5
-+#define TH1520_RESET_ID_DSP0_APB	6
-+#define TH1520_RESET_ID_DSP1_CORE	7
-+#define TH1520_RESET_ID_DSP1_DEBUG	8
-+#define TH1520_RESET_ID_DSP1_APB	9
-+#define TH1520_RESET_ID_DSPSYS_APB	10
-+#define TH1520_RESET_ID_AXI4_DSPSYS_SLV	11
-+#define TH1520_RESET_ID_AXI4_DSPSYS	12
-+#define TH1520_RESET_ID_AXI4_DSP_RS	13
-+
-+/* MISC Subsystem */
-+#define TH1520_RESET_ID_EMMC_SDIO_CLKGEN	0
-+#define TH1520_RESET_ID_EMMC			1
-+#define TH1520_RESET_ID_MISCSYS_AXI		2
-+#define TH1520_RESET_ID_MISCSYS_AXI_APB		3
-+#define TH1520_RESET_ID_SDIO0			4
-+#define TH1520_RESET_ID_SDIO1			5
-+#define TH1520_RESET_ID_USB3_APB		6
-+#define TH1520_RESET_ID_USB3_PHY		7
-+#define TH1520_RESET_ID_USB3_VCC		8
-+
-+/* VI Subsystem */
-+#define TH1520_RESET_ID_ISP0		0
-+#define TH1520_RESET_ID_ISP1		1
-+#define TH1520_RESET_ID_CSI0_APB	2
-+#define TH1520_RESET_ID_CSI1_APB	3
-+#define TH1520_RESET_ID_CSI2_APB	4
-+#define TH1520_RESET_ID_MIPI_FIFO	5
-+#define TH1520_RESET_ID_ISP_VENC_APB	6
-+#define TH1520_RESET_ID_VIPRE_APB	7
-+#define TH1520_RESET_ID_VIPRE_AXI	8
-+#define TH1520_RESET_ID_DW200_APB	9
-+#define TH1520_RESET_ID_VISYS3_AXI	10
-+#define TH1520_RESET_ID_VISYS2_AXI	11
-+#define TH1520_RESET_ID_VISYS1_AXI	12
-+#define TH1520_RESET_ID_VISYS_AXI	13
-+#define TH1520_RESET_ID_VISYS_APB	14
-+#define TH1520_RESET_ID_ISP_VENC_AXI	15
-+
-+/* VO Subsystem */
-+#define TH1520_RESET_ID_GPU		0
-+#define TH1520_RESET_ID_GPU_CLKGEN	1
-+#define TH1520_RESET_ID_DPU_AHB		5
-+#define TH1520_RESET_ID_DPU_AXI		6
-+#define TH1520_RESET_ID_DPU_CORE	7
-+#define TH1520_RESET_ID_DSI0_APB	8
-+#define TH1520_RESET_ID_DSI1_APB	9
-+#define TH1520_RESET_ID_HDMI		10
-+#define TH1520_RESET_ID_HDMI_APB	11
-+#define TH1520_RESET_ID_VOAXI		12
-+#define TH1520_RESET_ID_VOAXI_APB	13
-+#define TH1520_RESET_ID_X2H_DPU_AXI	14
-+#define TH1520_RESET_ID_X2H_DPU_AHB	15
-+#define TH1520_RESET_ID_X2H_DPU1_AXI	16
-+#define TH1520_RESET_ID_X2H_DPU1_AHB	17
-+
-+/* VP Subsystem */
-+#define TH1520_RESET_ID_VPSYS_AXI_APB	0
-+#define TH1520_RESET_ID_VPSYS_AXI	1
-+#define TH1520_RESET_ID_FCE_APB		2
-+#define TH1520_RESET_ID_FCE_CORE	3
-+#define TH1520_RESET_ID_FCE_X2X_MASTER	4
-+#define TH1520_RESET_ID_FCE_X2X_SLAVE	5
-+#define TH1520_RESET_ID_G2D_APB		6
-+#define TH1520_RESET_ID_G2D_ACLK	7
-+#define TH1520_RESET_ID_G2D_CORE	8
-+#define TH1520_RESET_ID_VDEC_APB	9
-+#define TH1520_RESET_ID_VDEC_ACLK	10
-+#define TH1520_RESET_ID_VDEC_CORE	11
-+#define TH1520_RESET_ID_VENC_APB	12
-+#define TH1520_RESET_ID_VENC_CORE	13
-+
-+#endif /* _DT_BINDINGS_TH1520_RESET_H */
-diff --git a/include/dt-bindings/reset/toshiba,tmpv770x.h b/include/dt-bindings/reset/toshiba,tmpv770x.h
-index c1007acb1941..9452bef31425 100644
--- a/include/dt-bindings/reset/toshiba,tmpv770x.h
-+++ b/include/dt-bindings/reset/toshiba,tmpv770x.h
-@@ -36,6 +36,13 @@
- #define TMPV770X_RESET_PIPCMIF		29
- #define TMPV770X_RESET_PICKMON		30
- #define TMPV770X_RESET_SBUSCLK		31
-#define TMPV770X_NR_RESET		32
-+#define TMPV770X_RESET_VIIFBS0		32
-+#define TMPV770X_RESET_VIIFBS0_APB	33
-+#define TMPV770X_RESET_VIIFBS0_L2ISP	34
-+#define TMPV770X_RESET_VIIFBS0_L1ISP	35
-+#define TMPV770X_RESET_VIIFBS1		36
-+#define TMPV770X_RESET_VIIFBS1_APB	37
-+#define TMPV770X_RESET_VIIFBS1_L2ISP	38
-+#define TMPV770X_RESET_VIIFBS1_L1ISP	39
- 
- #endif /*_DT_BINDINGS_RESET_TOSHIBA_TMPV770X_H_ */
-diff --git a/include/dt-bindings/soc/cpm1-fsl,tsa.h b/include/dt-bindings/soc/cpm1-fsl,tsa.h
-new file mode 100644
-index 000000000000..2cc44e867dbe
--- /dev/null
-+++ b/include/dt-bindings/soc/cpm1-fsl,tsa.h
-@@ -0,0 +1,13 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+
-+#ifndef __DT_BINDINGS_SOC_FSL_TSA_H
-+#define __DT_BINDINGS_SOC_FSL_TSA_H
-+
-+#define FSL_CPM_TSA_NU		0	/* Pseuso Cell Id for not used item */
-+#define FSL_CPM_TSA_SCC2	1
-+#define FSL_CPM_TSA_SCC3	2
-+#define FSL_CPM_TSA_SCC4	3
-+#define FSL_CPM_TSA_SMC1	4
-+#define FSL_CPM_TSA_SMC2	5
-+
-+#endif
-diff --git a/include/dt-bindings/soc/qe-fsl,tsa.h b/include/dt-bindings/soc/qe-fsl,tsa.h
-new file mode 100644
-index 000000000000..3cf3df9c0968
--- /dev/null
-+++ b/include/dt-bindings/soc/qe-fsl,tsa.h
-@@ -0,0 +1,13 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+
-+#ifndef __DT_BINDINGS_SOC_FSL_QE_TSA_H
-+#define __DT_BINDINGS_SOC_FSL_QE_TSA_H
-+
-+#define FSL_QE_TSA_NU		0
-+#define FSL_QE_TSA_UCC1		1
-+#define FSL_QE_TSA_UCC2		2
-+#define FSL_QE_TSA_UCC3		3
-+#define FSL_QE_TSA_UCC4		4
-+#define FSL_QE_TSA_UCC5		5
-+
-+#endif
-diff --git a/include/dt-bindings/soc/rockchip,vop2.h b/include/dt-bindings/soc/rockchip,vop2.h
-index 6e66a802b96a..668f199df9f0 100644
--- a/include/dt-bindings/soc/rockchip,vop2.h
-+++ b/include/dt-bindings/soc/rockchip,vop2.h
-@@ -10,5 +10,9 @@
- #define ROCKCHIP_VOP2_EP_LVDS0	5
- #define ROCKCHIP_VOP2_EP_MIPI1	6
- #define ROCKCHIP_VOP2_EP_LVDS1	7
-+#define ROCKCHIP_VOP2_EP_HDMI1	8
-+#define ROCKCHIP_VOP2_EP_EDP1	9
-+#define ROCKCHIP_VOP2_EP_DP0	10
-+#define ROCKCHIP_VOP2_EP_DP1	11
- 
- #endif /* __DT_BINDINGS_ROCKCHIP_VOP2_H */
-diff --git a/include/dt-bindings/soc/samsung,exynos-usi.h b/include/dt-bindings/soc/samsung,exynos-usi.h
-index a01af169d249..b46de214dd09 100644
--- a/include/dt-bindings/soc/samsung,exynos-usi.h
-+++ b/include/dt-bindings/soc/samsung,exynos-usi.h
-@@ -9,9 +9,18 @@
- #ifndef __DT_BINDINGS_SAMSUNG_EXYNOS_USI_H
- #define __DT_BINDINGS_SAMSUNG_EXYNOS_USI_H
- 
-#define USI_V2_NONE		0
-#define USI_V2_UART		1
-#define USI_V2_SPI		2
-#define USI_V2_I2C		3
-+#define USI_MODE_NONE		0
-+#define USI_MODE_UART		1
-+#define USI_MODE_SPI		2
-+#define USI_MODE_I2C		3
-+#define USI_MODE_I2C1		4
-+#define USI_MODE_I2C0_1		5
-+#define USI_MODE_UART_I2C1	6
-+
-+/* Deprecated */
-+#define USI_V2_NONE		USI_MODE_NONE
-+#define USI_V2_UART		USI_MODE_UART
-+#define USI_V2_SPI		USI_MODE_SPI
-+#define USI_V2_I2C		USI_MODE_I2C
- 
- #endif /* __DT_BINDINGS_SAMSUNG_EXYNOS_USI_H */
-diff --git a/include/dt-bindings/sound/audio-graph.h b/include/dt-bindings/sound/audio-graph.h
-new file mode 100644
-index 000000000000..bdb70c6b7332
--- /dev/null
-+++ b/include/dt-bindings/sound/audio-graph.h
-@@ -0,0 +1,26 @@
-+/* SPDX-License-Identifier: GPL-2.0
-+ *
-+ * audio-graph.h
-+ *
-+ * Copyright (c) 2024 Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
-+ */
-+#ifndef __AUDIO_GRAPH_H
-+#define __AUDIO_GRAPH_H
-+
-+/*
-+ * used in
-+ *	link-trigger-order
-+ *	link-trigger-order-start
-+ *	link-trigger-order-stop
-+ *
-+ * default is
-+ *	link-trigger-order = <SND_SOC_TRIGGER_LINK
-+ *			      SND_SOC_TRIGGER_COMPONENT
-+ *			      SND_SOC_TRIGGER_DAI>;
-+ */
-+#define SND_SOC_TRIGGER_LINK		0
-+#define SND_SOC_TRIGGER_COMPONENT	1
-+#define SND_SOC_TRIGGER_DAI		2
-+#define SND_SOC_TRIGGER_SIZE		3	/* shoud be last */
-+
-+#endif /* __AUDIO_GRAPH_H */
-diff --git a/include/dt-bindings/sound/cs35l45.h b/include/dt-bindings/sound/cs35l45.h
-index 076da4b2c28d..25386af18445 100644
--- a/include/dt-bindings/sound/cs35l45.h
-+++ b/include/dt-bindings/sound/cs35l45.h
-@@ -17,4 +17,61 @@
- #define CS35L45_ASP_TX_HIZ_UNUSED	0x1
- #define CS35L45_ASP_TX_HIZ_DISABLED	0x2
- 
-+/*
-+ * Optional GPIOX Sub-nodes:
-+ *  The cs35l45 node can have up to three "cirrus,gpio-ctrlX" ('X' = [1,2,3])
-+ *  sub-nodes for configuring the GPIO pins.
-+ *
-+ * - gpio-dir : GPIO pin direction. Valid only when 'gpio-ctrl'
-+ *   is 1.
-+ *    0 = Output
-+ *    1 = Input (Default)
-+ *
-+ * - gpio-lvl : GPIO level. Valid only when 'gpio-ctrl' is 1 and 'gpio-dir' is 0.
-+ *
-+ *    0 = Low (Default)
-+ *    1 = High
-+ *
-+ * - gpio-op-cfg : GPIO output configuration. Valid only when 'gpio-ctrl' is 1
-+ *   and 'gpio-dir' is 0.
-+ *
-+ *    0 = CMOS (Default)
-+ *    1 = Open Drain
-+ *
-+ * - gpio-pol : GPIO output polarity select. Valid only when 'gpio-ctrl' is 1
-+ *   and 'gpio-dir' is 0.
-+ *
-+ *    0 = Non-inverted, Active High (Default)
-+ *    1 = Inverted, Active Low
-+ *
-+ * - gpio-invert : Defines the polarity of the GPIO pin if configured
-+ *   as input.
-+ *
-+ *    0 = Not inverted (Default)
-+ *    1 = Inverted
-+ *
-+ * - gpio-ctrl : Defines the function of the GPIO pin.
-+ *
-+ * GPIO1:
-+ *   0 = High impedance input (Default)
-+ *   1 = Pin acts as a GPIO, direction controlled by 'gpio-dir'
-+ *   2 = Pin acts as MDSYNC, direction controlled by MDSYNC
-+ *   3-7 = Reserved
-+ *
-+ * GPIO2:
-+ *   0 = High impedance input (Default)
-+ *   1 = Pin acts as a GPIO, direction controlled by 'gpio-dir'
-+ *   2 = Pin acts as open drain INT
-+ *   3 = Reserved
-+ *   4 = Pin acts as push-pull output INT. Active low.
-+ *   5 = Pin acts as push-pull output INT. Active high.
-+ *   6,7 = Reserved
-+ *
-+ * GPIO3:
-+ *   0 = High impedance input (Default)
-+ *   1 = Pin acts as a GPIO, direction controlled by 'gpio-dir'
-+ *   2-7 = Reserved
-+ */
-+#define CS35L45_NUM_GPIOS	0x3
-+
- #endif /* DT_CS35L45_H */
-diff --git a/include/dt-bindings/sound/cs48l32.h b/include/dt-bindings/sound/cs48l32.h
-new file mode 100644
-index 000000000000..4e82260fff67
--- /dev/null
-+++ b/include/dt-bindings/sound/cs48l32.h
-@@ -0,0 +1,20 @@
-+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-+/*
-+ * Device Tree defines for CS48L32 DSP.
-+ *
-+ * Copyright (C) 2016-2018, 2022, 2025 Cirrus Logic, Inc. and
-+ *               Cirrus Logic International Semiconductor Ltd.
-+ */
-+
-+#ifndef DT_BINDINGS_SOUND_CS48L32_H
-+#define DT_BINDINGS_SOUND_CS48L32_H
-+
-+/* Values for cirrus,in-type */
-+#define CS48L32_IN_TYPE_DIFF		0
-+#define CS48L32_IN_TYPE_SE		1
-+
-+/* Values for cirrus,pdm-sup */
-+#define CS48L32_PDM_SUP_VOUT_MIC	0
-+#define CS48L32_PDM_SUP_MICBIAS1	1
-+
-+#endif
-diff --git a/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h b/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h
-index 0d3276c8fc11..6d1ce7f5da51 100644
--- a/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h
-+++ b/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h
-@@ -131,6 +131,15 @@
- #define RX_CODEC_DMA_RX_7	126
- #define QUINARY_MI2S_RX		127
- #define QUINARY_MI2S_TX		128
-+#define DISPLAY_PORT_RX_0	DISPLAY_PORT_RX
-+#define DISPLAY_PORT_RX_1	129
-+#define DISPLAY_PORT_RX_2	130
-+#define DISPLAY_PORT_RX_3	131
-+#define DISPLAY_PORT_RX_4	132
-+#define DISPLAY_PORT_RX_5	133
-+#define DISPLAY_PORT_RX_6	134
-+#define DISPLAY_PORT_RX_7	135
-+#define USB_RX			136
- 
- #define LPASS_CLK_ID_PRI_MI2S_IBIT	1
- #define LPASS_CLK_ID_PRI_MI2S_EBIT	2
-@@ -193,6 +202,24 @@
- #define LPASS_CLK_ID_RX_CORE_MCLK	59
- #define LPASS_CLK_ID_RX_CORE_NPL_MCLK	60
- #define LPASS_CLK_ID_VA_CORE_2X_MCLK	61
-+/* Clock ID for MCLK for WSA2 core */
-+#define LPASS_CLK_ID_WSA2_CORE_MCLK	62
-+/* Clock ID for NPL MCLK for WSA2 core */
-+#define LPASS_CLK_ID_WSA2_CORE_2X_MCLK	63
-+/* Clock ID for RX Core TX MCLK */
-+#define LPASS_CLK_ID_RX_CORE_TX_MCLK	64
-+/* Clock ID for RX CORE TX 2X MCLK */
-+#define LPASS_CLK_ID_RX_CORE_TX_2X_MCLK	65
-+/* Clock ID for WSA core TX MCLK */
-+#define LPASS_CLK_ID_WSA_CORE_TX_MCLK	66
-+/* Clock ID for WSA core TX 2X MCLK */
-+#define LPASS_CLK_ID_WSA_CORE_TX_2X_MCLK	67
-+/* Clock ID for WSA2 core TX MCLK */
-+#define LPASS_CLK_ID_WSA2_CORE_TX_MCLK	68
-+/* Clock ID for WSA2 core TX 2X MCLK */
-+#define LPASS_CLK_ID_WSA2_CORE_TX_2X_MCLK	69
-+/* Clock ID for RX CORE MCLK2 2X  MCLK */
-+#define LPASS_CLK_ID_RX_CORE_MCLK2_2X_MCLK	70
- 
- #define LPASS_HW_AVTIMER_VOTE		101
- #define LPASS_HW_MACRO_VOTE		102
-diff --git a/include/dt-bindings/sound/qcom,wcd9335.h b/include/dt-bindings/sound/qcom,wcd9335.h
-index f5e9f1db091e..4fc68aeb9e04 100644
--- a/include/dt-bindings/sound/qcom,wcd9335.h
-+++ b/include/dt-bindings/sound/qcom,wcd9335.h
-@@ -10,6 +10,5 @@
- #define AIF3_PB                 4
- #define AIF3_CAP                5
- #define AIF4_PB                 6
-#define NUM_CODEC_DAIS          7
- 
- #endif
-diff --git a/include/dt-bindings/sound/qcom,wcd934x.h b/include/dt-bindings/sound/qcom,wcd934x.h
-new file mode 100644
-index 000000000000..8b30d34fcc87
--- /dev/null
-+++ b/include/dt-bindings/sound/qcom,wcd934x.h
-@@ -0,0 +1,16 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef __DT_SOUND_QCOM_WCD934x_H
-+#define __DT_SOUND_QCOM_WCD934x_H
-+
-+#define AIF1_PB                 0
-+#define AIF1_CAP                1
-+#define AIF2_PB                 2
-+#define AIF2_CAP                3
-+#define AIF3_PB                 4
-+#define AIF3_CAP                5
-+#define AIF4_PB                 6
-+#define AIF4_VIFEED             7
-+#define AIF4_MAD_TX             8
-+
-+#endif
-diff --git a/include/dt-bindings/thermal/mediatek,lvts-thermal.h b/include/dt-bindings/thermal/mediatek,lvts-thermal.h
-new file mode 100644
-index 000000000000..ddc7302a510a
--- /dev/null
-+++ b/include/dt-bindings/thermal/mediatek,lvts-thermal.h
-@@ -0,0 +1,83 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * Copyright (c) 2023 MediaTek Inc.
-+ * Author: Balsam CHIHI <bchihi@baylibre.com>
-+ */
-+
-+#ifndef __MEDIATEK_LVTS_DT_H
-+#define __MEDIATEK_LVTS_DT_H
-+
-+#define MT7988_CPU_0		0
-+#define MT7988_CPU_1		1
-+#define MT7988_ETH2P5G_0	2
-+#define MT7988_ETH2P5G_1	3
-+#define MT7988_TOPS_0		4
-+#define MT7988_TOPS_1		5
-+#define MT7988_ETHWARP_0	6
-+#define MT7988_ETHWARP_1	7
-+
-+#define MT8186_LITTLE_CPU0	0
-+#define MT8186_LITTLE_CPU1	1
-+#define MT8186_LITTLE_CPU2	2
-+#define MT8186_CAM		3
-+#define MT8186_BIG_CPU0	4
-+#define MT8186_BIG_CPU1	5
-+#define MT8186_NNA		6
-+#define MT8186_ADSP		7
-+#define MT8186_GPU		8
-+
-+#define MT8188_MCU_LITTLE_CPU0	0
-+#define MT8188_MCU_LITTLE_CPU1	1
-+#define MT8188_MCU_LITTLE_CPU2	2
-+#define MT8188_MCU_LITTLE_CPU3	3
-+#define MT8188_MCU_BIG_CPU0	4
-+#define MT8188_MCU_BIG_CPU1	5
-+
-+#define MT8188_AP_APU		0
-+#define MT8188_AP_GPU0		1
-+#define MT8188_AP_GPU1		2
-+#define MT8188_AP_ADSP		3
-+#define MT8188_AP_VDO		4
-+#define MT8188_AP_INFRA		5
-+#define MT8188_AP_CAM1		6
-+#define MT8188_AP_CAM2		7
-+
-+#define MT8195_MCU_BIG_CPU0     0
-+#define MT8195_MCU_BIG_CPU1     1
-+#define MT8195_MCU_BIG_CPU2     2
-+#define MT8195_MCU_BIG_CPU3     3
-+#define MT8195_MCU_LITTLE_CPU0  4
-+#define MT8195_MCU_LITTLE_CPU1  5
-+#define MT8195_MCU_LITTLE_CPU2  6
-+#define MT8195_MCU_LITTLE_CPU3  7
-+
-+#define MT8195_AP_VPU0  8
-+#define MT8195_AP_VPU1  9
-+#define MT8195_AP_GPU0  10
-+#define MT8195_AP_GPU1  11
-+#define MT8195_AP_VDEC  12
-+#define MT8195_AP_IMG   13
-+#define MT8195_AP_INFRA 14
-+#define MT8195_AP_CAM0  15
-+#define MT8195_AP_CAM1  16
-+
-+#define MT8192_MCU_BIG_CPU0     0
-+#define MT8192_MCU_BIG_CPU1     1
-+#define MT8192_MCU_BIG_CPU2     2
-+#define MT8192_MCU_BIG_CPU3     3
-+#define MT8192_MCU_LITTLE_CPU0  4
-+#define MT8192_MCU_LITTLE_CPU1  5
-+#define MT8192_MCU_LITTLE_CPU2  6
-+#define MT8192_MCU_LITTLE_CPU3  7
-+
-+#define MT8192_AP_VPU0  8
-+#define MT8192_AP_VPU1  9
-+#define MT8192_AP_GPU0  10
-+#define MT8192_AP_GPU1  11
-+#define MT8192_AP_INFRA 12
-+#define MT8192_AP_CAM   13
-+#define MT8192_AP_MD0   14
-+#define MT8192_AP_MD1   15
-+#define MT8192_AP_MD2   16
-+
-+#endif /* __MEDIATEK_LVTS_DT_H */
-diff --git a/include/dt-bindings/thermal/tegra114-soctherm.h b/include/dt-bindings/thermal/tegra114-soctherm.h
-new file mode 100644
-index 000000000000..b766a61cd1ce
--- /dev/null
-+++ b/include/dt-bindings/thermal/tegra114-soctherm.h
-@@ -0,0 +1,19 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+/*
-+ * This header provides constants for binding nvidia,tegra114-soctherm.
-+ */
-+
-+#ifndef _DT_BINDINGS_THERMAL_TEGRA114_SOCTHERM_H
-+#define _DT_BINDINGS_THERMAL_TEGRA114_SOCTHERM_H
-+
-+#define TEGRA114_SOCTHERM_SENSOR_CPU 0
-+#define TEGRA114_SOCTHERM_SENSOR_MEM 1
-+#define TEGRA114_SOCTHERM_SENSOR_GPU 2
-+#define TEGRA114_SOCTHERM_SENSOR_PLLX 3
-+
-+#define TEGRA114_SOCTHERM_THROT_LEVEL_NONE 0
-+#define TEGRA114_SOCTHERM_THROT_LEVEL_LOW  1
-+#define TEGRA114_SOCTHERM_THROT_LEVEL_MED  2
-+#define TEGRA114_SOCTHERM_THROT_LEVEL_HIGH 3
-+
-+#endif
-diff --git a/include/dt-bindings/thermal/tegra234-bpmp-thermal.h b/include/dt-bindings/thermal/tegra234-bpmp-thermal.h
-new file mode 100644
-index 000000000000..934787950932
--- /dev/null
-+++ b/include/dt-bindings/thermal/tegra234-bpmp-thermal.h
-@@ -0,0 +1,19 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * This header provides constants for binding nvidia,tegra234-bpmp-thermal.
-+ */
-+
-+#ifndef _DT_BINDINGS_THERMAL_TEGRA234_BPMP_THERMAL_H
-+#define _DT_BINDINGS_THERMAL_TEGRA234_BPMP_THERMAL_H
-+
-+#define TEGRA234_BPMP_THERMAL_ZONE_CPU		0
-+#define TEGRA234_BPMP_THERMAL_ZONE_GPU		1
-+#define TEGRA234_BPMP_THERMAL_ZONE_CV0		2
-+#define TEGRA234_BPMP_THERMAL_ZONE_CV1		3
-+#define TEGRA234_BPMP_THERMAL_ZONE_CV2		4
-+#define TEGRA234_BPMP_THERMAL_ZONE_SOC0		5
-+#define TEGRA234_BPMP_THERMAL_ZONE_SOC1		6
-+#define TEGRA234_BPMP_THERMAL_ZONE_SOC2		7
-+#define TEGRA234_BPMP_THERMAL_ZONE_TJ_MAX	8
-+
-+#endif
-diff --git a/include/dt-bindings/watchdog/aspeed-wdt.h b/include/dt-bindings/watchdog/aspeed-wdt.h
-new file mode 100644
-index 000000000000..89fa31ffce2d
--- /dev/null
-+++ b/include/dt-bindings/watchdog/aspeed-wdt.h
-@@ -0,0 +1,230 @@
-+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-+
-+#ifndef DT_BINDINGS_ASPEED_WDT_H
-+#define DT_BINDINGS_ASPEED_WDT_H
-+
-+#define AST2500_WDT_RESET_CPU		(1 << 0)
-+#define AST2500_WDT_RESET_COPROC	(1 << 1)
-+#define AST2500_WDT_RESET_SDRAM		(1 << 2)
-+#define AST2500_WDT_RESET_AHB		(1 << 3)
-+#define AST2500_WDT_RESET_I2C		(1 << 4)
-+#define AST2500_WDT_RESET_MAC0		(1 << 5)
-+#define AST2500_WDT_RESET_MAC1		(1 << 6)
-+#define AST2500_WDT_RESET_GRAPHICS	(1 << 7)
-+#define AST2500_WDT_RESET_USB2_HOST_HUB	(1 << 8)
-+#define AST2500_WDT_RESET_USB_HOST	(1 << 9)
-+#define AST2500_WDT_RESET_HID_EHCI	(1 << 10)
-+#define AST2500_WDT_RESET_VIDEO		(1 << 11)
-+#define AST2500_WDT_RESET_HAC		(1 << 12)
-+#define AST2500_WDT_RESET_LPC		(1 << 13)
-+#define AST2500_WDT_RESET_SDIO		(1 << 14)
-+#define AST2500_WDT_RESET_MIC		(1 << 15)
-+#define AST2500_WDT_RESET_CRT		(1 << 16)
-+#define AST2500_WDT_RESET_PWM		(1 << 17)
-+#define AST2500_WDT_RESET_PECI		(1 << 18)
-+#define AST2500_WDT_RESET_JTAG		(1 << 19)
-+#define AST2500_WDT_RESET_ADC		(1 << 20)
-+#define AST2500_WDT_RESET_GPIO		(1 << 21)
-+#define AST2500_WDT_RESET_MCTP		(1 << 22)
-+#define AST2500_WDT_RESET_XDMA		(1 << 23)
-+#define AST2500_WDT_RESET_SPI		(1 << 24)
-+#define AST2500_WDT_RESET_SOC_MISC	(1 << 25)
-+
-+#define AST2500_WDT_RESET_DEFAULT 0x023ffff3
-+
-+#define AST2600_WDT_RESET1_CPU		(1 << 0)
-+#define AST2600_WDT_RESET1_SDRAM	(1 << 1)
-+#define AST2600_WDT_RESET1_AHB		(1 << 2)
-+#define AST2600_WDT_RESET1_SLI		(1 << 3)
-+#define AST2600_WDT_RESET1_SOC_MISC0	(1 << 4)
-+#define AST2600_WDT_RESET1_COPROC	(1 << 5)
-+#define AST2600_WDT_RESET1_USB_A	(1 << 6)
-+#define AST2600_WDT_RESET1_USB_B	(1 << 7)
-+#define AST2600_WDT_RESET1_UHCI		(1 << 8)
-+#define AST2600_WDT_RESET1_GRAPHICS	(1 << 9)
-+#define AST2600_WDT_RESET1_CRT		(1 << 10)
-+#define AST2600_WDT_RESET1_VIDEO	(1 << 11)
-+#define AST2600_WDT_RESET1_HAC		(1 << 12)
-+#define AST2600_WDT_RESET1_DP		(1 << 13)
-+#define AST2600_WDT_RESET1_DP_MCU	(1 << 14)
-+#define AST2600_WDT_RESET1_GP_MCU	(1 << 15)
-+#define AST2600_WDT_RESET1_MAC0		(1 << 16)
-+#define AST2600_WDT_RESET1_MAC1		(1 << 17)
-+#define AST2600_WDT_RESET1_SDIO0	(1 << 18)
-+#define AST2600_WDT_RESET1_JTAG0	(1 << 19)
-+#define AST2600_WDT_RESET1_MCTP0	(1 << 20)
-+#define AST2600_WDT_RESET1_MCTP1	(1 << 21)
-+#define AST2600_WDT_RESET1_XDMA0	(1 << 22)
-+#define AST2600_WDT_RESET1_XDMA1	(1 << 23)
-+#define AST2600_WDT_RESET1_GPIO0	(1 << 24)
-+#define AST2600_WDT_RESET1_RVAS		(1 << 25)
-+
-+#define AST2600_WDT_RESET1_DEFAULT 0x030f1ff1
-+
-+#define AST2600_WDT_RESET2_CPU		(1 << 0)
-+#define AST2600_WDT_RESET2_SPI		(1 << 1)
-+#define AST2600_WDT_RESET2_AHB2		(1 << 2)
-+#define AST2600_WDT_RESET2_SLI2		(1 << 3)
-+#define AST2600_WDT_RESET2_SOC_MISC1	(1 << 4)
-+#define AST2600_WDT_RESET2_MAC2		(1 << 5)
-+#define AST2600_WDT_RESET2_MAC3		(1 << 6)
-+#define AST2600_WDT_RESET2_SDIO1	(1 << 7)
-+#define AST2600_WDT_RESET2_JTAG1	(1 << 8)
-+#define AST2600_WDT_RESET2_GPIO1	(1 << 9)
-+#define AST2600_WDT_RESET2_MDIO		(1 << 10)
-+#define AST2600_WDT_RESET2_LPC		(1 << 11)
-+#define AST2600_WDT_RESET2_PECI		(1 << 12)
-+#define AST2600_WDT_RESET2_PWM		(1 << 13)
-+#define AST2600_WDT_RESET2_ADC		(1 << 14)
-+#define AST2600_WDT_RESET2_FSI		(1 << 15)
-+#define AST2600_WDT_RESET2_I2C		(1 << 16)
-+#define AST2600_WDT_RESET2_I3C_GLOBAL	(1 << 17)
-+#define AST2600_WDT_RESET2_I3C0		(1 << 18)
-+#define AST2600_WDT_RESET2_I3C1		(1 << 19)
-+#define AST2600_WDT_RESET2_I3C2		(1 << 20)
-+#define AST2600_WDT_RESET2_I3C3		(1 << 21)
-+#define AST2600_WDT_RESET2_I3C4		(1 << 22)
-+#define AST2600_WDT_RESET2_I3C5		(1 << 23)
-+#define AST2600_WDT_RESET2_ESPI		(1 << 26)
-+
-+#define AST2600_WDT_RESET2_DEFAULT 0x03fffff1
-+
-+#define AST2700_WDT_RESET1_CPU		(1 << 0)
-+#define AST2700_WDT_RESET1_DRAM		(1 << 1)
-+#define AST2700_WDT_RESET1_SLI0		(1 << 2)
-+#define AST2700_WDT_RESET1_EHCI		(1 << 3)
-+#define AST2700_WDT_RESET1_HACE		(1 << 4)
-+#define AST2700_WDT_RESET1_SOC_MISC0	(1 << 5)
-+#define AST2700_WDT_RESET1_VIDEO	(1 << 6)
-+#define AST2700_WDT_RESET1_2D_GRAPHIC	(1 << 7)
-+#define AST2700_WDT_RESET1_RAVS0	(1 << 8)
-+#define AST2700_WDT_RESET1_RAVS1	(1 << 9)
-+#define AST2700_WDT_RESET1_GPIO0	(1 << 10)
-+#define AST2700_WDT_RESET1_SSP		(1 << 11)
-+#define AST2700_WDT_RESET1_TSP		(1 << 12)
-+#define AST2700_WDT_RESET1_CRT		(1 << 13)
-+#define AST2700_WDT_RESET1_USB20_HOST	(1 << 14)
-+#define AST2700_WDT_RESET1_USB11_HOST	(1 << 15)
-+#define AST2700_WDT_RESET1_UFS		(1 << 16)
-+#define AST2700_WDT_RESET1_EMMC		(1 << 17)
-+#define AST2700_WDT_RESET1_AHB_TO_PCIE1	(1 << 18)
-+#define AST2700_WDT_RESET1_XDMA0	(1 << 22)
-+#define AST2700_WDT_RESET1_MCTP1	(1 << 23)
-+#define AST2700_WDT_RESET1_MCTP0	(1 << 24)
-+#define AST2700_WDT_RESET1_JTAG0	(1 << 25)
-+#define AST2700_WDT_RESET1_ECC		(1 << 26)
-+#define AST2700_WDT_RESET1_XDMA1	(1 << 27)
-+#define AST2700_WDT_RESET1_DP		(1 << 28)
-+#define AST2700_WDT_RESET1_DP_MCU	(1 << 29)
-+#define AST2700_WDT_RESET1_AHB_TO_PCIE0	(1 << 31)
-+
-+#define AST2700_WDT_RESET1_DEFAULT 0x8207ff71
-+
-+#define AST2700_WDT_RESET2_USB3_A_HOST	(1 << 0)
-+#define AST2700_WDT_RESET2_USB3_A_VHUB3	(1 << 1)
-+#define AST2700_WDT_RESET2_USB3_A_VHUB2	(1 << 2)
-+#define AST2700_WDT_RESET2_USB3_B_HOST	(1 << 3)
-+#define AST2700_WDT_RESET2_USB3_B_VHUB3	(1 << 4)
-+#define AST2700_WDT_RESET2_USB3_B_VHUB2	(1 << 5)
-+#define AST2700_WDT_RESET2_SM3		(1 << 6)
-+#define AST2700_WDT_RESET2_SM4		(1 << 7)
-+#define AST2700_WDT_RESET2_SHA3		(1 << 8)
-+#define AST2700_WDT_RESET2_RSA		(1 << 9)
-+
-+#define AST2700_WDT_RESET2_DEFAULT 0x000003f6
-+
-+#define AST2700_WDT_RESET3_LPC0		(1 << 0)
-+#define AST2700_WDT_RESET3_LPC1		(1 << 1)
-+#define AST2700_WDT_RESET3_MDIO		(1 << 2)
-+#define AST2700_WDT_RESET3_PECI		(1 << 3)
-+#define AST2700_WDT_RESET3_PWM		(1 << 4)
-+#define AST2700_WDT_RESET3_MAC0		(1 << 5)
-+#define AST2700_WDT_RESET3_MAC1		(1 << 6)
-+#define AST2700_WDT_RESET3_MAC2		(1 << 7)
-+#define AST2700_WDT_RESET3_ADC		(1 << 8)
-+#define AST2700_WDT_RESET3_SDC		(1 << 9)
-+#define AST2700_WDT_RESET3_ESPI0	(1 << 10)
-+#define AST2700_WDT_RESET3_ESPI1	(1 << 11)
-+#define AST2700_WDT_RESET3_JTAG1	(1 << 12)
-+#define AST2700_WDT_RESET3_SPI0		(1 << 13)
-+#define AST2700_WDT_RESET3_SPI1		(1 << 14)
-+#define AST2700_WDT_RESET3_SPI2		(1 << 15)
-+#define AST2700_WDT_RESET3_I3C0		(1 << 16)
-+#define AST2700_WDT_RESET3_I3C1		(1 << 17)
-+#define AST2700_WDT_RESET3_I3C2		(1 << 18)
-+#define AST2700_WDT_RESET3_I3C3		(1 << 19)
-+#define AST2700_WDT_RESET3_I3C4		(1 << 20)
-+#define AST2700_WDT_RESET3_I3C5		(1 << 21)
-+#define AST2700_WDT_RESET3_I3C6		(1 << 22)
-+#define AST2700_WDT_RESET3_I3C7		(1 << 23)
-+#define AST2700_WDT_RESET3_I3C8		(1 << 24)
-+#define AST2700_WDT_RESET3_I3C9		(1 << 25)
-+#define AST2700_WDT_RESET3_I3C10	(1 << 26)
-+#define AST2700_WDT_RESET3_I3C11	(1 << 27)
-+#define AST2700_WDT_RESET3_I3C12	(1 << 28)
-+#define AST2700_WDT_RESET3_I3C13	(1 << 29)
-+#define AST2700_WDT_RESET3_I3C14	(1 << 30)
-+#define AST2700_WDT_RESET3_I3C15	(1 << 31)
-+
-+#define AST2700_WDT_RESET3_DEFAULT 0x000093ec
-+
-+#define AST2700_WDT_RESET4_FMC		(1 << 0)
-+#define AST2700_WDT_RESET4_SOC_MISC1	(1 << 1)
-+#define AST2700_WDT_RESET4_AHB		(1 << 2)
-+#define AST2700_WDT_RESET4_SLI1		(1 << 3)
-+#define AST2700_WDT_RESET4_UART0	(1 << 4)
-+#define AST2700_WDT_RESET4_UART1	(1 << 5)
-+#define AST2700_WDT_RESET4_UART2	(1 << 6)
-+#define AST2700_WDT_RESET4_UART3	(1 << 7)
-+#define AST2700_WDT_RESET4_I2C_MONITOR	(1 << 8)
-+#define AST2700_WDT_RESET4_HOST_TO_SPI1	(1 << 9)
-+#define AST2700_WDT_RESET4_HOST_TO_SPI2	(1 << 10)
-+#define AST2700_WDT_RESET4_GPIO1	(1 << 11)
-+#define AST2700_WDT_RESET4_FSI		(1 << 12)
-+#define AST2700_WDT_RESET4_CANBUS	(1 << 13)
-+#define AST2700_WDT_RESET4_MCTP		(1 << 14)
-+#define AST2700_WDT_RESET4_XDMA		(1 << 15)
-+#define AST2700_WDT_RESET4_UART5	(1 << 16)
-+#define AST2700_WDT_RESET4_UART6	(1 << 17)
-+#define AST2700_WDT_RESET4_UART7	(1 << 18)
-+#define AST2700_WDT_RESET4_UART8	(1 << 19)
-+#define AST2700_WDT_RESET4_BOOT_MCU	(1 << 20)
-+#define AST2700_WDT_RESET4_IO_MCU	(1 << 21)
-+#define AST2700_WDT_RESET4_LTPI0	(1 << 22)
-+#define AST2700_WDT_RESET4_VGA_LINK	(1 << 23)
-+#define AST2700_WDT_RESET4_LTPI1	(1 << 24)
-+#define AST2700_WDT_RESET4_LTPI_PHY	(1 << 25)
-+#define AST2700_WDT_RESET4_ACE		(1 << 26)
-+#define AST2700_WDT_RESET4_LTPI_GPIO0	(1 << 28)
-+#define AST2700_WDT_RESET4_LTPI_GPIO1	(1 << 29)
-+#define AST2700_WDT_RESET4_AHB_TO_PCIE1	(1 << 30)
-+#define AST2700_WDT_RESET4_I3C_DMA	(1 << 31)
-+
-+#define AST2700_WDT_RESET4_DEFAULT 0x40303803
-+
-+#define AST2700_WDT_RESET5_I2C_GLOBAL	(1 << 0)
-+#define AST2700_WDT_RESET5_I2C0		(1 << 1)
-+#define AST2700_WDT_RESET5_I2C1		(1 << 2)
-+#define AST2700_WDT_RESET5_I2C2		(1 << 3)
-+#define AST2700_WDT_RESET5_I2C3		(1 << 4)
-+#define AST2700_WDT_RESET5_I2C4		(1 << 5)
-+#define AST2700_WDT_RESET5_I2C5		(1 << 6)
-+#define AST2700_WDT_RESET5_I2C6		(1 << 7)
-+#define AST2700_WDT_RESET5_I2C7		(1 << 8)
-+#define AST2700_WDT_RESET5_I2C8		(1 << 9)
-+#define AST2700_WDT_RESET5_I2C9		(1 << 10)
-+#define AST2700_WDT_RESET5_I2C10	(1 << 11)
-+#define AST2700_WDT_RESET5_I2C11	(1 << 12)
-+#define AST2700_WDT_RESET5_I2C12	(1 << 13)
-+#define AST2700_WDT_RESET5_I2C13	(1 << 14)
-+#define AST2700_WDT_RESET5_I2C14	(1 << 15)
-+#define AST2700_WDT_RESET5_I2C15	(1 << 16)
-+#define AST2700_WDT_RESET5_UHCI		(1 << 17)
-+#define AST2700_WDT_RESET5_USB2_C_UART	(1 << 18)
-+#define AST2700_WDT_RESET5_USB2_C	(1 << 19)
-+#define AST2700_WDT_RESET5_USB2_D_UART	(1 << 20)
-+#define AST2700_WDT_RESET5_USB2_D	(1 << 21)
-+
-+#define AST2700_WDT_RESET5_DEFAULT 0x00320000
-+
-+#endif
-diff --git a/include/hyperv/hvgdk.h b/include/hyperv/hvgdk.h
-new file mode 100644
-index 000000000000..dd6d4939ea29
--- /dev/null
-+++ b/include/hyperv/hvgdk.h
-@@ -0,0 +1,308 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * Type definitions for the Microsoft Hypervisor.
-+ */
-+#ifndef _HV_HVGDK_H
-+#define _HV_HVGDK_H
-+
-+#include "hvgdk_mini.h"
-+#include "hvgdk_ext.h"
-+
-+/*
-+ * The guest OS needs to register the guest ID with the hypervisor.
-+ * The guest ID is a 64 bit entity and the structure of this ID is
-+ * specified in the Hyper-V TLFS specification.
-+ *
-+ * While the current guideline does not specify how Linux guest ID(s)
-+ * need to be generated, our plan is to publish the guidelines for
-+ * Linux and other guest operating systems that currently are hosted
-+ * on Hyper-V. The implementation here conforms to this yet
-+ * unpublished guidelines.
-+ *
-+ * Bit(s)
-+ * 63 - Indicates if the OS is Open Source or not; 1 is Open Source
-+ * 62:56 - Os Type; Linux is 0x100
-+ * 55:48 - Distro specific identification
-+ * 47:16 - Linux kernel version number
-+ * 15:0  - Distro specific identification
-+ */
-+
-+#define HV_LINUX_VENDOR_ID              0x8100
-+
-+/* HV_VMX_ENLIGHTENED_VMCS */
-+struct hv_enlightened_vmcs {
-+	u32 revision_id;
-+	u32 abort;
-+
-+	u16 host_es_selector;
-+	u16 host_cs_selector;
-+	u16 host_ss_selector;
-+	u16 host_ds_selector;
-+	u16 host_fs_selector;
-+	u16 host_gs_selector;
-+	u16 host_tr_selector;
-+
-+	u16 padding16_1;
-+
-+	u64 host_ia32_pat;
-+	u64 host_ia32_efer;
-+
-+	u64 host_cr0;
-+	u64 host_cr3;
-+	u64 host_cr4;
-+
-+	u64 host_ia32_sysenter_esp;
-+	u64 host_ia32_sysenter_eip;
-+	u64 host_rip;
-+	u32 host_ia32_sysenter_cs;
-+
-+	u32 pin_based_vm_exec_control;
-+	u32 vm_exit_controls;
-+	u32 secondary_vm_exec_control;
-+
-+	u64 io_bitmap_a;
-+	u64 io_bitmap_b;
-+	u64 msr_bitmap;
-+
-+	u16 guest_es_selector;
-+	u16 guest_cs_selector;
-+	u16 guest_ss_selector;
-+	u16 guest_ds_selector;
-+	u16 guest_fs_selector;
-+	u16 guest_gs_selector;
-+	u16 guest_ldtr_selector;
-+	u16 guest_tr_selector;
-+
-+	u32 guest_es_limit;
-+	u32 guest_cs_limit;
-+	u32 guest_ss_limit;
-+	u32 guest_ds_limit;
-+	u32 guest_fs_limit;
-+	u32 guest_gs_limit;
-+	u32 guest_ldtr_limit;
-+	u32 guest_tr_limit;
-+	u32 guest_gdtr_limit;
-+	u32 guest_idtr_limit;
-+
-+	u32 guest_es_ar_bytes;
-+	u32 guest_cs_ar_bytes;
-+	u32 guest_ss_ar_bytes;
-+	u32 guest_ds_ar_bytes;
-+	u32 guest_fs_ar_bytes;
-+	u32 guest_gs_ar_bytes;
-+	u32 guest_ldtr_ar_bytes;
-+	u32 guest_tr_ar_bytes;
-+
-+	u64 guest_es_base;
-+	u64 guest_cs_base;
-+	u64 guest_ss_base;
-+	u64 guest_ds_base;
-+	u64 guest_fs_base;
-+	u64 guest_gs_base;
-+	u64 guest_ldtr_base;
-+	u64 guest_tr_base;
-+	u64 guest_gdtr_base;
-+	u64 guest_idtr_base;
-+
-+	u64 padding64_1[3];
-+
-+	u64 vm_exit_msr_store_addr;
-+	u64 vm_exit_msr_load_addr;
-+	u64 vm_entry_msr_load_addr;
-+
-+	u64 cr3_target_value0;
-+	u64 cr3_target_value1;
-+	u64 cr3_target_value2;
-+	u64 cr3_target_value3;
-+
-+	u32 page_fault_error_code_mask;
-+	u32 page_fault_error_code_match;
-+
-+	u32 cr3_target_count;
-+	u32 vm_exit_msr_store_count;
-+	u32 vm_exit_msr_load_count;
-+	u32 vm_entry_msr_load_count;
-+
-+	u64 tsc_offset;
-+	u64 virtual_apic_page_addr;
-+	u64 vmcs_link_pointer;
-+
-+	u64 guest_ia32_debugctl;
-+	u64 guest_ia32_pat;
-+	u64 guest_ia32_efer;
-+
-+	u64 guest_pdptr0;
-+	u64 guest_pdptr1;
-+	u64 guest_pdptr2;
-+	u64 guest_pdptr3;
-+
-+	u64 guest_pending_dbg_exceptions;
-+	u64 guest_sysenter_esp;
-+	u64 guest_sysenter_eip;
-+
-+	u32 guest_activity_state;
-+	u32 guest_sysenter_cs;
-+
-+	u64 cr0_guest_host_mask;
-+	u64 cr4_guest_host_mask;
-+	u64 cr0_read_shadow;
-+	u64 cr4_read_shadow;
-+	u64 guest_cr0;
-+	u64 guest_cr3;
-+	u64 guest_cr4;
-+	u64 guest_dr7;
-+
-+	u64 host_fs_base;
-+	u64 host_gs_base;
-+	u64 host_tr_base;
-+	u64 host_gdtr_base;
-+	u64 host_idtr_base;
-+	u64 host_rsp;
-+
-+	u64 ept_pointer;
-+
-+	u16 virtual_processor_id;
-+	u16 padding16_2[3];
-+
-+	u64 padding64_2[5];
-+	u64 guest_physical_address;
-+
-+	u32 vm_instruction_error;
-+	u32 vm_exit_reason;
-+	u32 vm_exit_intr_info;
-+	u32 vm_exit_intr_error_code;
-+	u32 idt_vectoring_info_field;
-+	u32 idt_vectoring_error_code;
-+	u32 vm_exit_instruction_len;
-+	u32 vmx_instruction_info;
-+
-+	u64 exit_qualification;
-+	u64 exit_io_instruction_ecx;
-+	u64 exit_io_instruction_esi;
-+	u64 exit_io_instruction_edi;
-+	u64 exit_io_instruction_eip;
-+
-+	u64 guest_linear_address;
-+	u64 guest_rsp;
-+	u64 guest_rflags;
-+
-+	u32 guest_interruptibility_info;
-+	u32 cpu_based_vm_exec_control;
-+	u32 exception_bitmap;
-+	u32 vm_entry_controls;
-+	u32 vm_entry_intr_info_field;
-+	u32 vm_entry_exception_error_code;
-+	u32 vm_entry_instruction_len;
-+	u32 tpr_threshold;
-+
-+	u64 guest_rip;
-+
-+	u32 hv_clean_fields;
-+	u32 padding32_1;
-+	u32 hv_synthetic_controls;
-+	struct {
-+		u32 nested_flush_hypercall:1;
-+		u32 msr_bitmap:1;
-+		u32 reserved:30;
-+	}  __packed hv_enlightenments_control;
-+	u32 hv_vp_id;
-+	u32 padding32_2;
-+	u64 hv_vm_id;
-+	u64 partition_assist_page;
-+	u64 padding64_4[4];
-+	u64 guest_bndcfgs;
-+	u64 guest_ia32_perf_global_ctrl;
-+	u64 guest_ia32_s_cet;
-+	u64 guest_ssp;
-+	u64 guest_ia32_int_ssp_table_addr;
-+	u64 guest_ia32_lbr_ctl;
-+	u64 padding64_5[2];
-+	u64 xss_exit_bitmap;
-+	u64 encls_exiting_bitmap;
-+	u64 host_ia32_perf_global_ctrl;
-+	u64 tsc_multiplier;
-+	u64 host_ia32_s_cet;
-+	u64 host_ssp;
-+	u64 host_ia32_int_ssp_table_addr;
-+	u64 padding64_6;
-+} __packed;
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE			0
-+
-+
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP		BIT(0)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP		BIT(1)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2		BIT(2)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1		BIT(3)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC		BIT(4)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT		BIT(5)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY		BIT(6)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN		BIT(7)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR			BIT(8)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT		BIT(9)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC		BIT(10)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1		BIT(11)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2		BIT(12)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER		BIT(13)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1		BIT(14)
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_ENLIGHTENMENTSCONTROL	BIT(15)
-+
-+#define HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL			0xFFFF
-+
-+/*
-+ * Note, Hyper-V isn't actually stealing bit 28 from Intel, just abusing it by
-+ * pairing it with architecturally impossible exit reasons.  Bit 28 is set only
-+ * on SMI exits to a SMI transfer monitor (STM) and if and only if a MTF VM-Exit
-+ * is pending.  I.e. it will never be set by hardware for non-SMI exits (there
-+ * are only three), nor will it ever be set unless the VMM is an STM.
-+ */
-+#define HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH		0x10000031
-+
-+/*
-+ * Hyper-V uses the software reserved 32 bytes in VMCB control area to expose
-+ * SVM enlightenments to guests. This is documented in the TLFS doc.
-+ * Note on naming: SVM_NESTED_ENLIGHTENED_VMCB_FIELDS
-+ */
-+struct hv_vmcb_enlightenments {
-+	struct __packed hv_enlightenments_control {
-+		u32 nested_flush_hypercall : 1;
-+		u32 msr_bitmap : 1;
-+		u32 enlightened_npt_tlb: 1;
-+		u32 reserved : 29;
-+	} __packed hv_enlightenments_control;
-+	u32 hv_vp_id;
-+	u64 hv_vm_id;
-+	u64 partition_assist_page;
-+	u64 reserved;
-+} __packed;
-+
-+/*
-+ * Hyper-V uses the software reserved clean bit in VMCB.
-+ */
-+#define HV_VMCB_NESTED_ENLIGHTENMENTS		31
-+
-+/* Synthetic VM-Exit */
-+#define HV_SVM_EXITCODE_ENL			0xf0000000
-+#define HV_SVM_ENL_EXITCODE_TRAP_AFTER_FLUSH	(1)
-+
-+/* VM_PARTITION_ASSIST_PAGE */
-+struct hv_partition_assist_pg {
-+	u32 tlb_lock_count;
-+};
-+
-+/* Define connection identifier type. */
-+union hv_connection_id {
-+	u32 asu32;
-+	struct {
-+		u32 id : 24;
-+		u32 reserved : 8;
-+	} __packed u;
-+};
-+
-+struct hv_input_unmap_gpa_pages {
-+	u64 target_partition_id;
-+	u64 target_gpa_base;
-+	u32 unmap_flags;
-+	u32 padding;
-+} __packed;
-+
-+#endif /* #ifndef _HV_HVGDK_H */
-diff --git a/include/hyperv/hvgdk_ext.h b/include/hyperv/hvgdk_ext.h
-new file mode 100644
-index 000000000000..641b591ee61f
--- /dev/null
-+++ b/include/hyperv/hvgdk_ext.h
-@@ -0,0 +1,46 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * Type definitions for the Microsoft Hypervisor.
-+ */
-+#ifndef _HV_HVGDK_EXT_H
-+#define _HV_HVGDK_EXT_H
-+
-+#include "hvgdk_mini.h"
-+
-+/* Extended hypercalls */
-+#define HV_EXT_CALL_QUERY_CAPABILITIES		0x8001
-+#define HV_EXT_CALL_MEMORY_HEAT_HINT		0x8003
-+
-+/* Extended hypercalls */
-+enum {		/* HV_EXT_CALL */
-+	HV_EXTCALL_QUERY_CAPABILITIES = 0x8001,
-+	HV_EXTCALL_MEMORY_HEAT_HINT   = 0x8003,
-+};
-+
-+/* HV_EXT_OUTPUT_QUERY_CAPABILITIES */
-+#define HV_EXT_CAPABILITY_MEMORY_COLD_DISCARD_HINT BIT(8)
-+
-+enum {		/* HV_EXT_MEMORY_HEAT_HINT_TYPE */
-+	HV_EXTMEM_HEAT_HINT_COLD = 0,
-+	HV_EXTMEM_HEAT_HINT_HOT = 1,
-+	HV_EXTMEM_HEAT_HINT_COLD_DISCARD = 2,
-+	HV_EXTMEM_HEAT_HINT_MAX
-+};
-+
-+/*
-+ * The whole argument should fit in a page to be able to pass to the hypervisor
-+ * in one hypercall.
-+ */
-+#define HV_MEMORY_HINT_MAX_GPA_PAGE_RANGES  \
-+	((HV_HYP_PAGE_SIZE - sizeof(struct hv_memory_hint)) / \
-+		sizeof(union hv_gpa_page_range))
-+
-+/* HvExtCallMemoryHeatHint hypercall */
-+#define HV_EXT_MEMORY_HEAT_HINT_TYPE_COLD_DISCARD	2
-+struct hv_memory_hint {		/* HV_EXT_INPUT_MEMORY_HEAT_HINT */
-+	u64 heat_type : 2;	/* HV_EXTMEM_HEAT_HINT_* */
-+	u64 reserved : 62;
-+	union hv_gpa_page_range ranges[];
-+} __packed;
-+
-+#endif /* _HV_HVGDK_EXT_H */
-diff --git a/include/hyperv/hvgdk_mini.h b/include/hyperv/hvgdk_mini.h
-new file mode 100644
-index 000000000000..04b18d0e37af
--- /dev/null
-+++ b/include/hyperv/hvgdk_mini.h
-@@ -0,0 +1,1528 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * Type definitions for the Microsoft hypervisor.
-+ */
-+#ifndef _HV_HVGDK_MINI_H
-+#define _HV_HVGDK_MINI_H
-+
-+#include <linux/types.h>
-+#include <linux/bits.h>
-+
-+struct hv_u128 {
-+	u64 low_part;
-+	u64 high_part;
-+} __packed;
-+
-+/* NOTE: when adding below, update hv_result_to_string() */
-+#define HV_STATUS_SUCCESS			    0x0
-+#define HV_STATUS_INVALID_HYPERCALL_CODE	    0x2
-+#define HV_STATUS_INVALID_HYPERCALL_INPUT	    0x3
-+#define HV_STATUS_INVALID_ALIGNMENT		    0x4
-+#define HV_STATUS_INVALID_PARAMETER		    0x5
-+#define HV_STATUS_ACCESS_DENIED			    0x6
-+#define HV_STATUS_INVALID_PARTITION_STATE	    0x7
-+#define HV_STATUS_OPERATION_DENIED		    0x8
-+#define HV_STATUS_UNKNOWN_PROPERTY		    0x9
-+#define HV_STATUS_PROPERTY_VALUE_OUT_OF_RANGE	    0xA
-+#define HV_STATUS_INSUFFICIENT_MEMORY		    0xB
-+#define HV_STATUS_INVALID_PARTITION_ID		    0xD
-+#define HV_STATUS_INVALID_VP_INDEX		    0xE
-+#define HV_STATUS_NOT_FOUND			    0x10
-+#define HV_STATUS_INVALID_PORT_ID		    0x11
-+#define HV_STATUS_INVALID_CONNECTION_ID		    0x12
-+#define HV_STATUS_INSUFFICIENT_BUFFERS		    0x13
-+#define HV_STATUS_NOT_ACKNOWLEDGED		    0x14
-+#define HV_STATUS_INVALID_VP_STATE		    0x15
-+#define HV_STATUS_NO_RESOURCES			    0x1D
-+#define HV_STATUS_PROCESSOR_FEATURE_NOT_SUPPORTED   0x20
-+#define HV_STATUS_INVALID_LP_INDEX		    0x41
-+#define HV_STATUS_INVALID_REGISTER_VALUE	    0x50
-+#define HV_STATUS_OPERATION_FAILED		    0x71
-+#define HV_STATUS_TIME_OUT			    0x78
-+#define HV_STATUS_CALL_PENDING			    0x79
-+#define HV_STATUS_VTL_ALREADY_ENABLED		    0x86
-+
-+/*
-+ * The Hyper-V TimeRefCount register and the TSC
-+ * page provide a guest VM clock with 100ns tick rate
-+ */
-+#define HV_CLOCK_HZ (NSEC_PER_SEC / 100)
-+
-+#define HV_HYP_PAGE_SHIFT		12
-+#define HV_HYP_PAGE_SIZE		BIT(HV_HYP_PAGE_SHIFT)
-+#define HV_HYP_PAGE_MASK		(~(HV_HYP_PAGE_SIZE - 1))
-+#define HV_HYP_LARGE_PAGE_SHIFT		21
-+
-+#define HV_PARTITION_ID_INVALID		((u64)0)
-+#define HV_PARTITION_ID_SELF		((u64)-1)
-+
-+/* Hyper-V specific model specific registers (MSRs) */
-+
-+#if defined(CONFIG_X86)
-+/* HV_X64_SYNTHETIC_MSR */
-+#define HV_X64_MSR_GUEST_OS_ID			0x40000000
-+#define HV_X64_MSR_HYPERCALL			0x40000001
-+#define HV_X64_MSR_VP_INDEX			0x40000002
-+#define HV_X64_MSR_RESET			0x40000003
-+#define HV_X64_MSR_VP_RUNTIME			0x40000010
-+#define HV_X64_MSR_TIME_REF_COUNT		0x40000020
-+#define HV_X64_MSR_REFERENCE_TSC		0x40000021
-+#define HV_X64_MSR_TSC_FREQUENCY		0x40000022
-+#define HV_X64_MSR_APIC_FREQUENCY		0x40000023
-+
-+/* Define the virtual APIC registers */
-+#define HV_X64_MSR_EOI				0x40000070
-+#define HV_X64_MSR_ICR				0x40000071
-+#define HV_X64_MSR_TPR				0x40000072
-+#define HV_X64_MSR_VP_ASSIST_PAGE		0x40000073
-+
-+/* Define synthetic interrupt controller model specific registers. */
-+#define HV_X64_MSR_SCONTROL			0x40000080
-+#define HV_X64_MSR_SVERSION			0x40000081
-+#define HV_X64_MSR_SIEFP			0x40000082
-+#define HV_X64_MSR_SIMP				0x40000083
-+#define HV_X64_MSR_EOM				0x40000084
-+#define HV_X64_MSR_SIRBP			0x40000085
-+#define HV_X64_MSR_SINT0			0x40000090
-+#define HV_X64_MSR_SINT1			0x40000091
-+#define HV_X64_MSR_SINT2			0x40000092
-+#define HV_X64_MSR_SINT3			0x40000093
-+#define HV_X64_MSR_SINT4			0x40000094
-+#define HV_X64_MSR_SINT5			0x40000095
-+#define HV_X64_MSR_SINT6			0x40000096
-+#define HV_X64_MSR_SINT7			0x40000097
-+#define HV_X64_MSR_SINT8			0x40000098
-+#define HV_X64_MSR_SINT9			0x40000099
-+#define HV_X64_MSR_SINT10			0x4000009A
-+#define HV_X64_MSR_SINT11			0x4000009B
-+#define HV_X64_MSR_SINT12			0x4000009C
-+#define HV_X64_MSR_SINT13			0x4000009D
-+#define HV_X64_MSR_SINT14			0x4000009E
-+#define HV_X64_MSR_SINT15			0x4000009F
-+
-+/* Define synthetic interrupt controller model specific registers for nested hypervisor */
-+#define HV_X64_MSR_NESTED_SCONTROL		0x40001080
-+#define HV_X64_MSR_NESTED_SVERSION		0x40001081
-+#define HV_X64_MSR_NESTED_SIEFP			0x40001082
-+#define HV_X64_MSR_NESTED_SIMP			0x40001083
-+#define HV_X64_MSR_NESTED_EOM			0x40001084
-+#define HV_X64_MSR_NESTED_SINT0			0x40001090
-+
-+/*
-+ * Synthetic Timer MSRs. Four timers per vcpu.
-+ */
-+#define HV_X64_MSR_STIMER0_CONFIG		0x400000B0
-+#define HV_X64_MSR_STIMER0_COUNT		0x400000B1
-+#define HV_X64_MSR_STIMER1_CONFIG		0x400000B2
-+#define HV_X64_MSR_STIMER1_COUNT		0x400000B3
-+#define HV_X64_MSR_STIMER2_CONFIG		0x400000B4
-+#define HV_X64_MSR_STIMER2_COUNT		0x400000B5
-+#define HV_X64_MSR_STIMER3_CONFIG		0x400000B6
-+#define HV_X64_MSR_STIMER3_COUNT		0x400000B7
-+
-+/* Hyper-V guest idle MSR */
-+#define HV_X64_MSR_GUEST_IDLE			0x400000F0
-+
-+/* Hyper-V guest crash notification MSR's */
-+#define HV_X64_MSR_CRASH_P0			0x40000100
-+#define HV_X64_MSR_CRASH_P1			0x40000101
-+#define HV_X64_MSR_CRASH_P2			0x40000102
-+#define HV_X64_MSR_CRASH_P3			0x40000103
-+#define HV_X64_MSR_CRASH_P4			0x40000104
-+#define HV_X64_MSR_CRASH_CTL			0x40000105
-+
-+#define HV_X64_MSR_HYPERCALL_ENABLE		0x00000001
-+#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT	12
-+#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK	\
-+		(~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1))
-+
-+#define HV_X64_MSR_CRASH_PARAMS		\
-+		(1 + (HV_X64_MSR_CRASH_P4 - HV_X64_MSR_CRASH_P0))
-+
-+#define HV_IPI_LOW_VECTOR	 0x10
-+#define HV_IPI_HIGH_VECTOR	 0xff
-+
-+#define HV_X64_MSR_VP_ASSIST_PAGE_ENABLE	0x00000001
-+#define HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT	12
-+#define HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_MASK	\
-+		(~((1ull << HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT) - 1))
-+
-+/* Hyper-V Enlightened VMCS version mask in nested features CPUID */
-+#define HV_X64_ENLIGHTENED_VMCS_VERSION		0xff
-+
-+#define HV_X64_MSR_TSC_REFERENCE_ENABLE		0x00000001
-+#define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT	12
-+
-+/* Number of XMM registers used in hypercall input/output */
-+#define HV_HYPERCALL_MAX_XMM_REGISTERS		6
-+
-+struct hv_reenlightenment_control {
-+	u64 vector : 8;
-+	u64 reserved1 : 8;
-+	u64 enabled : 1;
-+	u64 reserved2 : 15;
-+	u64 target_vp : 32;
-+}  __packed;
-+
-+struct hv_tsc_emulation_status {	 /* HV_TSC_EMULATION_STATUS */
-+	u64 inprogress : 1;
-+	u64 reserved : 63;
-+} __packed;
-+
-+struct hv_tsc_emulation_control {	 /* HV_TSC_INVARIANT_CONTROL */
-+	u64 enabled : 1;
-+	u64 reserved : 63;
-+} __packed;
-+
-+/* TSC emulation after migration */
-+#define HV_X64_MSR_REENLIGHTENMENT_CONTROL	0x40000106
-+#define HV_X64_MSR_TSC_EMULATION_CONTROL	0x40000107
-+#define HV_X64_MSR_TSC_EMULATION_STATUS		0x40000108
-+#define HV_X64_MSR_TSC_INVARIANT_CONTROL	0x40000118
-+#define HV_EXPOSE_INVARIANT_TSC		BIT_ULL(0)
-+
-+#endif /* CONFIG_X86 */
-+
-+struct hv_output_get_partition_id {
-+	u64 partition_id;
-+} __packed;
-+
-+/* HV_CRASH_CTL_REG_CONTENTS */
-+#define HV_CRASH_CTL_CRASH_NOTIFY_MSG		 BIT_ULL(62)
-+#define HV_CRASH_CTL_CRASH_NOTIFY		 BIT_ULL(63)
-+
-+union hv_reference_tsc_msr {
-+	u64 as_uint64;
-+	struct {
-+		u64 enable : 1;
-+		u64 reserved : 11;
-+		u64 pfn : 52;
-+	} __packed;
-+};
-+
-+/* The maximum number of sparse vCPU banks which can be encoded by 'struct hv_vpset' */
-+#define HV_MAX_SPARSE_VCPU_BANKS (64)
-+/* The number of vCPUs in one sparse bank */
-+#define HV_VCPUS_PER_SPARSE_BANK (64)
-+
-+/*
-+ * Some of Hyper-V structs do not use hv_vpset where linux uses them.
-+ *
-+ * struct hv_vpset is usually used as part of hypercall input. The portion
-+ * that counts as "fixed size input header" vs. "variable size input header"
-+ * varies per hypercall. See comments at relevant hypercall call sites as to
-+ * how the "valid_bank_mask" field should be accounted.
-+ */
-+struct hv_vpset {	 /* HV_VP_SET */
-+	u64 format;
-+	u64 valid_bank_mask;
-+	u64 bank_contents[];
-+} __packed;
-+
-+/*
-+ * Version info reported by hypervisor
-+ * Changed to a union for convenience
-+ */
-+union hv_hypervisor_version_info {
-+	struct {
-+		u32 build_number;
-+
-+		u32 minor_version : 16;
-+		u32 major_version : 16;
-+
-+		u32 service_pack;
-+
-+		u32 service_number : 24;
-+		u32 service_branch : 8;
-+	};
-+	struct {
-+		u32 eax;
-+		u32 ebx;
-+		u32 ecx;
-+		u32 edx;
-+	};
-+};
-+
-+/* HV_CPUID_FUNCTION */
-+#define HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS	0x40000000
-+#define HYPERV_CPUID_INTERFACE			0x40000001
-+#define HYPERV_CPUID_VERSION			0x40000002
-+#define HYPERV_CPUID_FEATURES			0x40000003
-+#define HYPERV_CPUID_ENLIGHTMENT_INFO		0x40000004
-+#define HYPERV_CPUID_IMPLEMENT_LIMITS		0x40000005
-+#define HYPERV_CPUID_CPU_MANAGEMENT_FEATURES	0x40000007
-+#define HYPERV_CPUID_NESTED_FEATURES		0x4000000A
-+#define HYPERV_CPUID_ISOLATION_CONFIG		0x4000000C
-+
-+#define HYPERV_CPUID_VIRT_STACK_INTERFACE	 0x40000081
-+#define HYPERV_VS_INTERFACE_EAX_SIGNATURE	 0x31235356  /* "VS#1" */
-+
-+#define HYPERV_CPUID_VIRT_STACK_PROPERTIES	 0x40000082
-+/* Support for the extended IOAPIC RTE format */
-+#define HYPERV_VS_PROPERTIES_EAX_EXTENDED_IOAPIC_RTE	 BIT(2)
-+#define HYPERV_VS_PROPERTIES_EAX_CONFIDENTIAL_VMBUS_AVAILABLE	 BIT(3)
-+
-+#define HYPERV_HYPERVISOR_PRESENT_BIT		 0x80000000
-+#define HYPERV_CPUID_MIN			 0x40000005
-+#define HYPERV_CPUID_MAX			 0x4000ffff
-+
-+/*
-+ * HV_X64_HYPERVISOR_FEATURES (EAX), or
-+ * HV_PARTITION_PRIVILEGE_MASK [31-0]
-+ */
-+#define HV_MSR_VP_RUNTIME_AVAILABLE			BIT(0)
-+#define HV_MSR_TIME_REF_COUNT_AVAILABLE			BIT(1)
-+#define HV_MSR_SYNIC_AVAILABLE				BIT(2)
-+#define HV_MSR_SYNTIMER_AVAILABLE			BIT(3)
-+#define HV_MSR_APIC_ACCESS_AVAILABLE			BIT(4)
-+#define HV_MSR_HYPERCALL_AVAILABLE			BIT(5)
-+#define HV_MSR_VP_INDEX_AVAILABLE			BIT(6)
-+#define HV_MSR_RESET_AVAILABLE				BIT(7)
-+#define HV_MSR_STAT_PAGES_AVAILABLE			BIT(8)
-+#define HV_MSR_REFERENCE_TSC_AVAILABLE			BIT(9)
-+#define HV_MSR_GUEST_IDLE_AVAILABLE			BIT(10)
-+#define HV_ACCESS_FREQUENCY_MSRS			BIT(11)
-+#define HV_ACCESS_REENLIGHTENMENT			BIT(13)
-+#define HV_ACCESS_TSC_INVARIANT				BIT(15)
-+
-+/*
-+ * HV_X64_HYPERVISOR_FEATURES (EBX), or
-+ * HV_PARTITION_PRIVILEGE_MASK [63-32]
-+ */
-+#define HV_CREATE_PARTITIONS				BIT(0)
-+#define HV_ACCESS_PARTITION_ID				BIT(1)
-+#define HV_ACCESS_MEMORY_POOL				BIT(2)
-+#define HV_ADJUST_MESSAGE_BUFFERS			BIT(3)
-+#define HV_POST_MESSAGES				BIT(4)
-+#define HV_SIGNAL_EVENTS				BIT(5)
-+#define HV_CREATE_PORT					BIT(6)
-+#define HV_CONNECT_PORT					BIT(7)
-+#define HV_ACCESS_STATS					BIT(8)
-+#define HV_DEBUGGING					BIT(11)
-+#define HV_CPU_MANAGEMENT				BIT(12)
-+#define HV_ENABLE_EXTENDED_HYPERCALLS			BIT(20)
-+#define HV_ISOLATION					BIT(22)
-+
-+#if defined(CONFIG_X86)
-+/* HV_X64_HYPERVISOR_FEATURES (EDX) */
-+#define HV_X64_MWAIT_AVAILABLE				BIT(0)
-+#define HV_X64_GUEST_DEBUGGING_AVAILABLE		BIT(1)
-+#define HV_X64_PERF_MONITOR_AVAILABLE			BIT(2)
-+#define HV_X64_CPU_DYNAMIC_PARTITIONING_AVAILABLE	BIT(3)
-+#define HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE		BIT(4)
-+#define HV_X64_GUEST_IDLE_STATE_AVAILABLE		BIT(5)
-+#define HV_FEATURE_FREQUENCY_MSRS_AVAILABLE		BIT(8)
-+#define HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE		BIT(10)
-+#define HV_FEATURE_DEBUG_MSRS_AVAILABLE			BIT(11)
-+#define HV_FEATURE_EXT_GVA_RANGES_FLUSH			BIT(14)
-+/*
-+ * Support for returning hypercall output block via XMM
-+ * registers is available
-+ */
-+#define HV_X64_HYPERCALL_XMM_OUTPUT_AVAILABLE		BIT(15)
-+/* stimer Direct Mode is available */
-+#define HV_STIMER_DIRECT_MODE_AVAILABLE			BIT(19)
-+
-+/*
-+ * Implementation recommendations. Indicates which behaviors the hypervisor
-+ * recommends the OS implement for optimal performance.
-+ * These are HYPERV_CPUID_ENLIGHTMENT_INFO.EAX bits.
-+ */
-+/* HV_X64_ENLIGHTENMENT_INFORMATION */
-+#define HV_X64_AS_SWITCH_RECOMMENDED			BIT(0)
-+#define HV_X64_LOCAL_TLB_FLUSH_RECOMMENDED		BIT(1)
-+#define HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED		BIT(2)
-+#define HV_X64_APIC_ACCESS_RECOMMENDED			BIT(3)
-+#define HV_X64_SYSTEM_RESET_RECOMMENDED			BIT(4)
-+#define HV_X64_RELAXED_TIMING_RECOMMENDED		BIT(5)
-+#define HV_DEPRECATING_AEOI_RECOMMENDED			BIT(9)
-+#define HV_X64_CLUSTER_IPI_RECOMMENDED			BIT(10)
-+#define HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED		BIT(11)
-+#define HV_X64_HYPERV_NESTED				BIT(12)
-+#define HV_X64_ENLIGHTENED_VMCS_RECOMMENDED		BIT(14)
-+#define HV_X64_USE_MMIO_HYPERCALLS			BIT(21)
-+
-+/*
-+ * CPU management features identification.
-+ * These are HYPERV_CPUID_CPU_MANAGEMENT_FEATURES.EAX bits.
-+ */
-+#define HV_X64_START_LOGICAL_PROCESSOR			BIT(0)
-+#define HV_X64_CREATE_ROOT_VIRTUAL_PROCESSOR		BIT(1)
-+#define HV_X64_PERFORMANCE_COUNTER_SYNC			BIT(2)
-+#define HV_X64_RESERVED_IDENTITY_BIT			BIT(31)
-+
-+/*
-+ * Virtual processor will never share a physical core with another virtual
-+ * processor, except for virtual processors that are reported as sibling SMT
-+ * threads.
-+ */
-+#define HV_X64_NO_NONARCH_CORESHARING			BIT(18)
-+
-+/* Nested features. These are HYPERV_CPUID_NESTED_FEATURES.EAX bits. */
-+#define HV_X64_NESTED_DIRECT_FLUSH			BIT(17)
-+#define HV_X64_NESTED_GUEST_MAPPING_FLUSH		BIT(18)
-+#define HV_X64_NESTED_MSR_BITMAP			BIT(19)
-+
-+/* Nested features #2. These are HYPERV_CPUID_NESTED_FEATURES.EBX bits. */
-+#define HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL		BIT(0)
-+
-+/*
-+ * This is specific to AMD and specifies that enlightened TLB flush is
-+ * supported. If guest opts in to this feature, ASID invalidations only
-+ * flushes gva -> hpa mapping entries. To flush the TLB entries derived
-+ * from NPT, hypercalls should be used (HvFlushGuestPhysicalAddressSpace
-+ * or HvFlushGuestPhysicalAddressList).
-+ */
-+#define HV_X64_NESTED_ENLIGHTENED_TLB			BIT(22)
-+
-+/* HYPERV_CPUID_ISOLATION_CONFIG.EAX bits. */
-+#define HV_PARAVISOR_PRESENT				BIT(0)
-+
-+/* HYPERV_CPUID_ISOLATION_CONFIG.EBX bits. */
-+#define HV_ISOLATION_TYPE				GENMASK(3, 0)
-+#define HV_SHARED_GPA_BOUNDARY_ACTIVE			BIT(5)
-+#define HV_SHARED_GPA_BOUNDARY_BITS			GENMASK(11, 6)
-+
-+/* HYPERV_CPUID_FEATURES.ECX bits. */
-+#define HV_VP_DISPATCH_INTERRUPT_INJECTION_AVAILABLE	BIT(9)
-+#define HV_VP_GHCB_ROOT_MAPPING_AVAILABLE		BIT(10)
-+
-+enum hv_isolation_type {
-+	HV_ISOLATION_TYPE_NONE	= 0,	/* HV_PARTITION_ISOLATION_TYPE_NONE */
-+	HV_ISOLATION_TYPE_VBS	= 1,
-+	HV_ISOLATION_TYPE_SNP	= 2,
-+	HV_ISOLATION_TYPE_TDX	= 3
-+};
-+
-+union hv_x64_msr_hypercall_contents {
-+	u64 as_uint64;
-+	struct {
-+		u64 enable : 1;
-+		u64 reserved : 11;
-+		u64 guest_physical_address : 52;
-+	} __packed;
-+};
-+#endif /* CONFIG_X86 */
-+
-+#if defined(CONFIG_ARM64)
-+#define HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE	BIT(8)
-+#define HV_STIMER_DIRECT_MODE_AVAILABLE		BIT(13)
-+#endif /* CONFIG_ARM64 */
-+
-+#if defined(CONFIG_X86)
-+#define HV_MAXIMUM_PROCESSORS	    2048
-+#elif defined(CONFIG_ARM64) /* CONFIG_X86 */
-+#define HV_MAXIMUM_PROCESSORS	    320
-+#endif /* CONFIG_ARM64 */
-+
-+#define HV_MAX_VP_INDEX			(HV_MAXIMUM_PROCESSORS - 1)
-+#define HV_VP_INDEX_SELF		((u32)-2)
-+#define HV_ANY_VP			((u32)-1)
-+
-+union hv_vp_assist_msr_contents {	 /* HV_REGISTER_VP_ASSIST_PAGE */
-+	u64 as_uint64;
-+	struct {
-+		u64 enable : 1;
-+		u64 reserved : 11;
-+		u64 pfn : 52;
-+	} __packed;
-+};
-+
-+/* Declare the various hypercall operations. */
-+/* HV_CALL_CODE */
-+#define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE		0x0002
-+#define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST		0x0003
-+#define HVCALL_NOTIFY_LONG_SPIN_WAIT			0x0008
-+#define HVCALL_SEND_IPI					0x000b
-+#define HVCALL_ENABLE_VP_VTL				0x000f
-+#define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX		0x0013
-+#define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX		0x0014
-+#define HVCALL_SEND_IPI_EX				0x0015
-+#define HVCALL_CREATE_PARTITION				0x0040
-+#define HVCALL_INITIALIZE_PARTITION			0x0041
-+#define HVCALL_FINALIZE_PARTITION			0x0042
-+#define HVCALL_DELETE_PARTITION				0x0043
-+#define HVCALL_GET_PARTITION_PROPERTY			0x0044
-+#define HVCALL_SET_PARTITION_PROPERTY			0x0045
-+#define HVCALL_GET_PARTITION_ID				0x0046
-+#define HVCALL_DEPOSIT_MEMORY				0x0048
-+#define HVCALL_WITHDRAW_MEMORY				0x0049
-+#define HVCALL_MAP_GPA_PAGES				0x004b
-+#define HVCALL_UNMAP_GPA_PAGES				0x004c
-+#define HVCALL_INSTALL_INTERCEPT			0x004d
-+#define HVCALL_CREATE_VP				0x004e
-+#define HVCALL_DELETE_VP				0x004f
-+#define HVCALL_GET_VP_REGISTERS				0x0050
-+#define HVCALL_SET_VP_REGISTERS				0x0051
-+#define HVCALL_TRANSLATE_VIRTUAL_ADDRESS		0x0052
-+#define HVCALL_CLEAR_VIRTUAL_INTERRUPT			0x0056
-+#define HVCALL_DELETE_PORT				0x0058
-+#define HVCALL_DISCONNECT_PORT				0x005b
-+#define HVCALL_POST_MESSAGE				0x005c
-+#define HVCALL_SIGNAL_EVENT				0x005d
-+#define HVCALL_POST_DEBUG_DATA				0x0069
-+#define HVCALL_RETRIEVE_DEBUG_DATA			0x006a
-+#define HVCALL_RESET_DEBUG_SESSION			0x006b
-+#define HVCALL_MAP_STATS_PAGE				0x006c
-+#define HVCALL_UNMAP_STATS_PAGE				0x006d
-+#define HVCALL_SET_SYSTEM_PROPERTY			0x006f
-+#define HVCALL_ADD_LOGICAL_PROCESSOR			0x0076
-+#define HVCALL_GET_SYSTEM_PROPERTY			0x007b
-+#define HVCALL_MAP_DEVICE_INTERRUPT			0x007c
-+#define HVCALL_UNMAP_DEVICE_INTERRUPT			0x007d
-+#define HVCALL_RETARGET_INTERRUPT			0x007e
-+#define HVCALL_NOTIFY_PARTITION_EVENT                   0x0087
-+#define HVCALL_ENTER_SLEEP_STATE			0x0084
-+#define HVCALL_NOTIFY_PORT_RING_EMPTY			0x008b
-+#define HVCALL_REGISTER_INTERCEPT_RESULT		0x0091
-+#define HVCALL_ASSERT_VIRTUAL_INTERRUPT			0x0094
-+#define HVCALL_CREATE_PORT				0x0095
-+#define HVCALL_CONNECT_PORT				0x0096
-+#define HVCALL_START_VP					0x0099
-+#define HVCALL_GET_VP_INDEX_FROM_APIC_ID		0x009a
-+#define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE	0x00af
-+#define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST	0x00b0
-+#define HVCALL_SIGNAL_EVENT_DIRECT			0x00c0
-+#define HVCALL_POST_MESSAGE_DIRECT			0x00c1
-+#define HVCALL_DISPATCH_VP				0x00c2
-+#define HVCALL_GET_GPA_PAGES_ACCESS_STATES		0x00c9
-+#define HVCALL_ACQUIRE_SPARSE_SPA_PAGE_HOST_ACCESS	0x00d7
-+#define HVCALL_RELEASE_SPARSE_SPA_PAGE_HOST_ACCESS	0x00d8
-+#define HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY	0x00db
-+#define HVCALL_MAP_VP_STATE_PAGE			0x00e1
-+#define HVCALL_UNMAP_VP_STATE_PAGE			0x00e2
-+#define HVCALL_GET_VP_STATE				0x00e3
-+#define HVCALL_SET_VP_STATE				0x00e4
-+#define HVCALL_GET_VP_CPUID_VALUES			0x00f4
-+#define HVCALL_GET_PARTITION_PROPERTY_EX		0x0101
-+#define HVCALL_MMIO_READ				0x0106
-+#define HVCALL_MMIO_WRITE				0x0107
-+#define HVCALL_DISABLE_HYP_EX                           0x010f
-+#define HVCALL_MAP_STATS_PAGE2				0x0131
-+
-+/* HV_HYPERCALL_INPUT */
-+#define HV_HYPERCALL_RESULT_MASK	GENMASK_ULL(15, 0)
-+#define HV_HYPERCALL_FAST_BIT		BIT(16)
-+#define HV_HYPERCALL_VARHEAD_OFFSET	17
-+#define HV_HYPERCALL_VARHEAD_MASK	GENMASK_ULL(26, 17)
-+#define HV_HYPERCALL_RSVD0_MASK		GENMASK_ULL(31, 27)
-+#define HV_HYPERCALL_NESTED		BIT_ULL(31)
-+#define HV_HYPERCALL_REP_COMP_OFFSET	32
-+#define HV_HYPERCALL_REP_COMP_1		BIT_ULL(32)
-+#define HV_HYPERCALL_REP_COMP_MASK	GENMASK_ULL(43, 32)
-+#define HV_HYPERCALL_RSVD1_MASK		GENMASK_ULL(47, 44)
-+#define HV_HYPERCALL_REP_START_OFFSET	48
-+#define HV_HYPERCALL_REP_START_MASK	GENMASK_ULL(59, 48)
-+#define HV_HYPERCALL_RSVD2_MASK		GENMASK_ULL(63, 60)
-+#define HV_HYPERCALL_RSVD_MASK		(HV_HYPERCALL_RSVD0_MASK | \
-+					 HV_HYPERCALL_RSVD1_MASK | \
-+					 HV_HYPERCALL_RSVD2_MASK)
-+
-+/* HvFlushGuestPhysicalAddressSpace hypercalls */
-+struct hv_guest_mapping_flush {
-+	u64 address_space;
-+	u64 flags;
-+} __packed;
-+
-+/*
-+ *  HV_MAX_FLUSH_PAGES = "additional_pages" + 1. It's limited
-+ *  by the bitwidth of "additional_pages" in union hv_gpa_page_range.
-+ */
-+#define HV_MAX_FLUSH_PAGES (2048)
-+#define HV_GPA_PAGE_RANGE_PAGE_SIZE_2MB		0
-+#define HV_GPA_PAGE_RANGE_PAGE_SIZE_1GB		1
-+
-+#define HV_FLUSH_ALL_PROCESSORS			BIT(0)
-+#define HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES	BIT(1)
-+#define HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY	BIT(2)
-+#define HV_FLUSH_USE_EXTENDED_RANGE_FORMAT	BIT(3)
-+
-+/* HvFlushGuestPhysicalAddressList, HvExtCallMemoryHeatHint hypercall */
-+union hv_gpa_page_range {
-+	u64 address_space;
-+	struct {
-+		u64 additional_pages : 11;
-+		u64 largepage : 1;
-+		u64 basepfn : 52;
-+	} page;
-+	struct {
-+		u64 reserved : 12;
-+		u64 page_size : 1;
-+		u64 reserved1 : 8;
-+		u64 base_large_pfn : 43;
-+	};
-+};
-+
-+/*
-+ * All input flush parameters should be in single page. The max flush
-+ * count is equal with how many entries of union hv_gpa_page_range can
-+ * be populated into the input parameter page.
-+ */
-+#define HV_MAX_FLUSH_REP_COUNT ((HV_HYP_PAGE_SIZE - 2 * sizeof(u64)) / \
-+				sizeof(union hv_gpa_page_range))
-+
-+struct hv_guest_mapping_flush_list {
-+	u64 address_space;
-+	u64 flags;
-+	union hv_gpa_page_range gpa_list[HV_MAX_FLUSH_REP_COUNT];
-+};
-+
-+struct hv_tlb_flush {	 /* HV_INPUT_FLUSH_VIRTUAL_ADDRESS_LIST */
-+	u64 address_space;
-+	u64 flags;
-+	u64 processor_mask;
-+	u64 gva_list[];
-+} __packed;
-+
-+/* HvFlushVirtualAddressSpaceEx, HvFlushVirtualAddressListEx hypercalls */
-+struct hv_tlb_flush_ex {
-+	u64 address_space;
-+	u64 flags;
-+	struct hv_vpset hv_vp_set;
-+	u64 gva_list[];
-+} __packed;
-+
-+struct ms_hyperv_tsc_page {	 /* HV_REFERENCE_TSC_PAGE */
-+	volatile u32 tsc_sequence;
-+	u32 reserved1;
-+	volatile u64 tsc_scale;
-+	volatile s64 tsc_offset;
-+} __packed;
-+
-+/* Define the number of synthetic interrupt sources. */
-+#define HV_SYNIC_SINT_COUNT (16)
-+
-+/* Define the expected SynIC version. */
-+#define HV_SYNIC_VERSION_1		(0x1)
-+/* Valid SynIC vectors are 16-255. */
-+#define HV_SYNIC_FIRST_VALID_VECTOR	(16)
-+
-+#define HV_SYNIC_CONTROL_ENABLE		(1ULL << 0)
-+#define HV_SYNIC_SIMP_ENABLE		(1ULL << 0)
-+#define HV_SYNIC_SIEFP_ENABLE		(1ULL << 0)
-+#define HV_SYNIC_SINT_MASKED		(1ULL << 16)
-+#define HV_SYNIC_SINT_AUTO_EOI		(1ULL << 17)
-+#define HV_SYNIC_SINT_VECTOR_MASK	(0xFF)
-+
-+/* Hyper-V defined statically assigned SINTs */
-+#define HV_SYNIC_INTERCEPTION_SINT_INDEX 0x00000000
-+#define HV_SYNIC_IOMMU_FAULT_SINT_INDEX  0x00000001
-+#define HV_SYNIC_VMBUS_SINT_INDEX	 0x00000002
-+#define HV_SYNIC_FIRST_UNUSED_SINT_INDEX 0x00000005
-+
-+/* mshv assigned SINT for doorbell */
-+#define HV_SYNIC_DOORBELL_SINT_INDEX     HV_SYNIC_FIRST_UNUSED_SINT_INDEX
-+
-+enum hv_interrupt_type {
-+	HV_X64_INTERRUPT_TYPE_FIXED		= 0x0000,
-+	HV_X64_INTERRUPT_TYPE_LOWESTPRIORITY	= 0x0001,
-+	HV_X64_INTERRUPT_TYPE_SMI		= 0x0002,
-+	HV_X64_INTERRUPT_TYPE_REMOTEREAD	= 0x0003,
-+	HV_X64_INTERRUPT_TYPE_NMI		= 0x0004,
-+	HV_X64_INTERRUPT_TYPE_INIT		= 0x0005,
-+	HV_X64_INTERRUPT_TYPE_SIPI		= 0x0006,
-+	HV_X64_INTERRUPT_TYPE_EXTINT		= 0x0007,
-+	HV_X64_INTERRUPT_TYPE_LOCALINT0		= 0x0008,
-+	HV_X64_INTERRUPT_TYPE_LOCALINT1		= 0x0009,
-+	HV_X64_INTERRUPT_TYPE_MAXIMUM		= 0x000A,
-+};
-+
-+/* Define synthetic interrupt source. */
-+union hv_synic_sint {
-+	u64 as_uint64;
-+	struct {
-+		u64 vector : 8;
-+		u64 reserved1 : 8;
-+		u64 masked : 1;
-+		u64 auto_eoi : 1;
-+		u64 polling : 1;
-+		u64 as_intercept : 1;
-+		u64 proxy : 1;
-+		u64 reserved2 : 43;
-+	} __packed;
-+};
-+
-+union hv_x64_xsave_xfem_register {
-+	u64 as_uint64;
-+	struct {
-+		u32 low_uint32;
-+		u32 high_uint32;
-+	} __packed;
-+	struct {
-+		u64 legacy_x87 : 1;
-+		u64 legacy_sse : 1;
-+		u64 avx : 1;
-+		u64 mpx_bndreg : 1;
-+		u64 mpx_bndcsr : 1;
-+		u64 avx_512_op_mask : 1;
-+		u64 avx_512_zmmhi : 1;
-+		u64 avx_512_zmm16_31 : 1;
-+		u64 rsvd8_9 : 2;
-+		u64 pasid : 1;
-+		u64 cet_u : 1;
-+		u64 cet_s : 1;
-+		u64 rsvd13_16 : 4;
-+		u64 xtile_cfg : 1;
-+		u64 xtile_data : 1;
-+		u64 rsvd19_63 : 45;
-+	} __packed;
-+};
-+
-+/* Synthetic timer configuration */
-+union hv_stimer_config {	 /* HV_X64_MSR_STIMER_CONFIG_CONTENTS */
-+	u64 as_uint64;
-+	struct {
-+		u64 enable : 1;
-+		u64 periodic : 1;
-+		u64 lazy : 1;
-+		u64 auto_enable : 1;
-+		u64 apic_vector : 8;
-+		u64 direct_mode : 1;
-+		u64 reserved_z0 : 3;
-+		u64 sintx : 4;
-+		u64 reserved_z1 : 44;
-+	} __packed;
-+};
-+
-+/* Define the number of synthetic timers */
-+#define HV_SYNIC_STIMER_COUNT	(4)
-+
-+/* Define port identifier type. */
-+union hv_port_id {
-+	u32 asu32;
-+	struct {
-+		u32 id : 24;
-+		u32 reserved : 8;
-+	} __packed u;
-+};
-+
-+#define HV_MESSAGE_SIZE			(256)
-+#define HV_MESSAGE_PAYLOAD_BYTE_COUNT	(240)
-+#define HV_MESSAGE_PAYLOAD_QWORD_COUNT	(30)
-+
-+/* Define hypervisor message types. */
-+enum hv_message_type {
-+	HVMSG_NONE				= 0x00000000,
-+
-+	/* Memory access messages. */
-+	HVMSG_UNMAPPED_GPA			= 0x80000000,
-+	HVMSG_GPA_INTERCEPT			= 0x80000001,
-+
-+	/* Timer notification messages. */
-+	HVMSG_TIMER_EXPIRED			= 0x80000010,
-+
-+	/* Error messages. */
-+	HVMSG_INVALID_VP_REGISTER_VALUE		= 0x80000020,
-+	HVMSG_UNRECOVERABLE_EXCEPTION		= 0x80000021,
-+	HVMSG_UNSUPPORTED_FEATURE		= 0x80000022,
-+
-+	/*
-+	 * Opaque intercept message. The original intercept message is only
-+	 * accessible from the mapped intercept message page.
-+	 */
-+	HVMSG_OPAQUE_INTERCEPT			= 0x8000003F,
-+
-+	/* Trace buffer complete messages. */
-+	HVMSG_EVENTLOG_BUFFERCOMPLETE		= 0x80000040,
-+
-+	/* Hypercall intercept */
-+	HVMSG_HYPERCALL_INTERCEPT		= 0x80000050,
-+
-+	/* SynIC intercepts */
-+	HVMSG_SYNIC_EVENT_INTERCEPT		= 0x80000060,
-+	HVMSG_SYNIC_SINT_INTERCEPT		= 0x80000061,
-+	HVMSG_SYNIC_SINT_DELIVERABLE	= 0x80000062,
-+
-+	/* Async call completion intercept */
-+	HVMSG_ASYNC_CALL_COMPLETION		= 0x80000070,
-+
-+	/* Root scheduler messages */
-+	HVMSG_SCHEDULER_VP_SIGNAL_BITSET	= 0x80000100,
-+	HVMSG_SCHEDULER_VP_SIGNAL_PAIR		= 0x80000101,
-+
-+	/* Platform-specific processor intercept messages. */
-+	HVMSG_X64_IO_PORT_INTERCEPT		= 0x80010000,
-+	HVMSG_X64_MSR_INTERCEPT			= 0x80010001,
-+	HVMSG_X64_CPUID_INTERCEPT		= 0x80010002,
-+	HVMSG_X64_EXCEPTION_INTERCEPT		= 0x80010003,
-+	HVMSG_X64_APIC_EOI			= 0x80010004,
-+	HVMSG_X64_LEGACY_FP_ERROR		= 0x80010005,
-+	HVMSG_X64_IOMMU_PRQ			= 0x80010006,
-+	HVMSG_X64_HALT				= 0x80010007,
-+	HVMSG_X64_INTERRUPTION_DELIVERABLE	= 0x80010008,
-+	HVMSG_X64_SIPI_INTERCEPT		= 0x80010009,
-+};
-+
-+/* Define the format of the SIMP register */
-+union hv_synic_simp {
-+	u64 as_uint64;
-+	struct {
-+		u64 simp_enabled : 1;
-+		u64 preserved : 11;
-+		u64 base_simp_gpa : 52;
-+	} __packed;
-+};
-+
-+union hv_message_flags {
-+	u8 asu8;
-+	struct {
-+		u8 msg_pending : 1;
-+		u8 reserved : 7;
-+	} __packed;
-+};
-+
-+struct hv_message_header {
-+	u32 message_type;
-+	u8 payload_size;
-+	union hv_message_flags message_flags;
-+	u8 reserved[2];
-+	union {
-+		u64 sender;
-+		union hv_port_id port;
-+	};
-+} __packed;
-+
-+/*
-+ * Message format for notifications delivered via
-+ * intercept message(as_intercept=1)
-+ */
-+struct hv_notification_message_payload {
-+	u32 sint_index;
-+} __packed;
-+
-+struct hv_message {
-+	struct hv_message_header header;
-+	union {
-+		u64 payload[HV_MESSAGE_PAYLOAD_QWORD_COUNT];
-+	} u;
-+} __packed;
-+
-+/* Define the synthetic interrupt message page layout. */
-+struct hv_message_page {
-+	struct hv_message sint_message[HV_SYNIC_SINT_COUNT];
-+} __packed;
-+
-+/* Define timer message payload structure. */
-+struct hv_timer_message_payload {
-+	u32 timer_index;
-+	u32 reserved;
-+	u64 expiration_time;	/* When the timer expired */
-+	u64 delivery_time;	/* When the message was delivered */
-+} __packed;
-+
-+struct hv_x64_segment_register {
-+	u64 base;
-+	u32 limit;
-+	u16 selector;
-+	union {
-+		struct {
-+			u16 segment_type : 4;
-+			u16 non_system_segment : 1;
-+			u16 descriptor_privilege_level : 2;
-+			u16 present : 1;
-+			u16 reserved : 4;
-+			u16 available : 1;
-+			u16 _long : 1;
-+			u16 _default : 1;
-+			u16 granularity : 1;
-+		} __packed;
-+		u16 attributes;
-+	};
-+} __packed;
-+
-+struct hv_x64_table_register {
-+	u16 pad[3];
-+	u16 limit;
-+	u64 base;
-+} __packed;
-+
-+#define HV_NORMAL_VTL	0
-+
-+union hv_input_vtl {
-+	u8 as_uint8;
-+	struct {
-+		u8 target_vtl : 4;
-+		u8 use_target_vtl : 1;
-+		u8 reserved_z : 3;
-+	};
-+} __packed;
-+
-+struct hv_init_vp_context {
-+	u64 rip;
-+	u64 rsp;
-+	u64 rflags;
-+
-+	struct hv_x64_segment_register cs;
-+	struct hv_x64_segment_register ds;
-+	struct hv_x64_segment_register es;
-+	struct hv_x64_segment_register fs;
-+	struct hv_x64_segment_register gs;
-+	struct hv_x64_segment_register ss;
-+	struct hv_x64_segment_register tr;
-+	struct hv_x64_segment_register ldtr;
-+
-+	struct hv_x64_table_register idtr;
-+	struct hv_x64_table_register gdtr;
-+
-+	u64 efer;
-+	u64 cr0;
-+	u64 cr3;
-+	u64 cr4;
-+	u64 msr_cr_pat;
-+} __packed;
-+
-+struct hv_enable_vp_vtl {
-+	u64				partition_id;
-+	u32				vp_index;
-+	union hv_input_vtl		target_vtl;
-+	u8				mbz0;
-+	u16				mbz1;
-+	struct hv_init_vp_context	vp_context;
-+} __packed;
-+
-+struct hv_get_vp_from_apic_id_in {
-+	u64 partition_id;
-+	union hv_input_vtl target_vtl;
-+	u8 res[7];
-+	u32 apic_ids[];
-+} __packed;
-+
-+union hv_register_vsm_partition_config {
-+	u64 as_uint64;
-+	struct {
-+		u64 enable_vtl_protection : 1;
-+		u64 default_vtl_protection_mask : 4;
-+		u64 zero_memory_on_reset : 1;
-+		u64 deny_lower_vtl_startup : 1;
-+		u64 intercept_acceptance : 1;
-+		u64 intercept_enable_vtl_protection : 1;
-+		u64 intercept_vp_startup : 1;
-+		u64 intercept_cpuid_unimplemented : 1;
-+		u64 intercept_unrecoverable_exception : 1;
-+		u64 intercept_page : 1;
-+		u64 mbz : 51;
-+	} __packed;
-+};
-+
-+union hv_register_vsm_capabilities {
-+	u64 as_uint64;
-+	struct {
-+		u64 dr6_shared: 1;
-+		u64 mbec_vtl_mask: 16;
-+		u64 deny_lower_vtl_startup: 1;
-+		u64 supervisor_shadow_stack: 1;
-+		u64 hardware_hvpt_available: 1;
-+		u64 software_hvpt_available: 1;
-+		u64 hardware_hvpt_range_bits: 6;
-+		u64 intercept_page_available: 1;
-+		u64 return_action_available: 1;
-+		u64 reserved: 35;
-+	} __packed;
-+};
-+
-+union hv_register_vsm_page_offsets {
-+	struct {
-+		u64 vtl_call_offset : 12;
-+		u64 vtl_return_offset : 12;
-+		u64 reserved_mbz : 40;
-+	} __packed;
-+	u64 as_uint64;
-+};
-+
-+struct hv_nested_enlightenments_control {
-+	struct {
-+		u32 directhypercall : 1;
-+		u32 reserved : 31;
-+	} __packed features;
-+	struct {
-+		u32 inter_partition_comm : 1;
-+		u32 reserved : 31;
-+	} __packed hypercall_controls;
-+} __packed;
-+
-+/* Define virtual processor assist page structure. */
-+struct hv_vp_assist_page {
-+	u32 apic_assist;
-+	u32 reserved1;
-+	u32 vtl_entry_reason;
-+	u32 vtl_reserved;
-+	u64 vtl_ret_x64rax;
-+	u64 vtl_ret_x64rcx;
-+	struct hv_nested_enlightenments_control nested_control;
-+	u8 enlighten_vmentry;
-+	u8 reserved2[7];
-+	u64 current_nested_vmcs;
-+	u8 synthetic_time_unhalted_timer_expired;
-+	u8 reserved3[7];
-+	u8 virtualization_fault_information[40];
-+	u8 reserved4[8];
-+	u8 intercept_message[256];
-+	u8 vtl_ret_actions[256];
-+} __packed;
-+
-+enum hv_register_name {
-+	/* Suspend Registers */
-+	HV_REGISTER_EXPLICIT_SUSPEND				= 0x00000000,
-+	HV_REGISTER_INTERCEPT_SUSPEND				= 0x00000001,
-+	HV_REGISTER_DISPATCH_SUSPEND				= 0x00000003,
-+
-+	/* Version - 128-bit result same as CPUID 0x40000002 */
-+	HV_REGISTER_HYPERVISOR_VERSION				= 0x00000100,
-+
-+	/* Feature Access (registers are 128 bits) - same as CPUID 0x40000003 - 0x4000000B */
-+	HV_REGISTER_PRIVILEGES_AND_FEATURES_INFO		= 0x00000200,
-+	HV_REGISTER_FEATURES_INFO				= 0x00000201,
-+	HV_REGISTER_IMPLEMENTATION_LIMITS_INFO			= 0x00000202,
-+	HV_REGISTER_HARDWARE_FEATURES_INFO			= 0x00000203,
-+	HV_REGISTER_CPU_MANAGEMENT_FEATURES_INFO		= 0x00000204,
-+	HV_REGISTER_SVM_FEATURES_INFO				= 0x00000205,
-+	HV_REGISTER_SKIP_LEVEL_FEATURES_INFO			= 0x00000206,
-+	HV_REGISTER_NESTED_VIRT_FEATURES_INFO			= 0x00000207,
-+	HV_REGISTER_IPT_FEATURES_INFO				= 0x00000208,
-+
-+	/* Guest Crash Registers */
-+	HV_REGISTER_GUEST_CRASH_P0				= 0x00000210,
-+	HV_REGISTER_GUEST_CRASH_P1				= 0x00000211,
-+	HV_REGISTER_GUEST_CRASH_P2				= 0x00000212,
-+	HV_REGISTER_GUEST_CRASH_P3				= 0x00000213,
-+	HV_REGISTER_GUEST_CRASH_P4				= 0x00000214,
-+	HV_REGISTER_GUEST_CRASH_CTL				= 0x00000215,
-+
-+	/* Misc */
-+	HV_REGISTER_VP_RUNTIME					= 0x00090000,
-+	HV_REGISTER_GUEST_OS_ID					= 0x00090002,
-+	HV_REGISTER_VP_INDEX					= 0x00090003,
-+	HV_REGISTER_TIME_REF_COUNT				= 0x00090004,
-+	HV_REGISTER_CPU_MANAGEMENT_VERSION			= 0x00090007,
-+	HV_REGISTER_VP_ASSIST_PAGE				= 0x00090013,
-+	HV_REGISTER_VP_ROOT_SIGNAL_COUNT			= 0x00090014,
-+	HV_REGISTER_REFERENCE_TSC				= 0x00090017,
-+
-+	/* Hypervisor-defined Registers (Synic) */
-+	HV_REGISTER_SINT0					= 0x000A0000,
-+	HV_REGISTER_SINT1					= 0x000A0001,
-+	HV_REGISTER_SINT2					= 0x000A0002,
-+	HV_REGISTER_SINT3					= 0x000A0003,
-+	HV_REGISTER_SINT4					= 0x000A0004,
-+	HV_REGISTER_SINT5					= 0x000A0005,
-+	HV_REGISTER_SINT6					= 0x000A0006,
-+	HV_REGISTER_SINT7					= 0x000A0007,
-+	HV_REGISTER_SINT8					= 0x000A0008,
-+	HV_REGISTER_SINT9					= 0x000A0009,
-+	HV_REGISTER_SINT10					= 0x000A000A,
-+	HV_REGISTER_SINT11					= 0x000A000B,
-+	HV_REGISTER_SINT12					= 0x000A000C,
-+	HV_REGISTER_SINT13					= 0x000A000D,
-+	HV_REGISTER_SINT14					= 0x000A000E,
-+	HV_REGISTER_SINT15					= 0x000A000F,
-+	HV_REGISTER_SCONTROL					= 0x000A0010,
-+	HV_REGISTER_SVERSION					= 0x000A0011,
-+	HV_REGISTER_SIEFP					= 0x000A0012,
-+	HV_REGISTER_SIMP					= 0x000A0013,
-+	HV_REGISTER_EOM						= 0x000A0014,
-+	HV_REGISTER_SIRBP					= 0x000A0015,
-+
-+	HV_REGISTER_NESTED_SINT0				= 0x000A1000,
-+	HV_REGISTER_NESTED_SINT1				= 0x000A1001,
-+	HV_REGISTER_NESTED_SINT2				= 0x000A1002,
-+	HV_REGISTER_NESTED_SINT3				= 0x000A1003,
-+	HV_REGISTER_NESTED_SINT4				= 0x000A1004,
-+	HV_REGISTER_NESTED_SINT5				= 0x000A1005,
-+	HV_REGISTER_NESTED_SINT6				= 0x000A1006,
-+	HV_REGISTER_NESTED_SINT7				= 0x000A1007,
-+	HV_REGISTER_NESTED_SINT8				= 0x000A1008,
-+	HV_REGISTER_NESTED_SINT9				= 0x000A1009,
-+	HV_REGISTER_NESTED_SINT10				= 0x000A100A,
-+	HV_REGISTER_NESTED_SINT11				= 0x000A100B,
-+	HV_REGISTER_NESTED_SINT12				= 0x000A100C,
-+	HV_REGISTER_NESTED_SINT13				= 0x000A100D,
-+	HV_REGISTER_NESTED_SINT14				= 0x000A100E,
-+	HV_REGISTER_NESTED_SINT15				= 0x000A100F,
-+	HV_REGISTER_NESTED_SCONTROL				= 0x000A1010,
-+	HV_REGISTER_NESTED_SVERSION				= 0x000A1011,
-+	HV_REGISTER_NESTED_SIFP					= 0x000A1012,
-+	HV_REGISTER_NESTED_SIPP					= 0x000A1013,
-+	HV_REGISTER_NESTED_EOM					= 0x000A1014,
-+	HV_REGISTER_NESTED_SIRBP				= 0x000a1015,
-+
-+	/* Hypervisor-defined Registers (Synthetic Timers) */
-+	HV_REGISTER_STIMER0_CONFIG				= 0x000B0000,
-+	HV_REGISTER_STIMER0_COUNT				= 0x000B0001,
-+
-+	/* VSM */
-+	HV_REGISTER_VSM_VP_STATUS				= 0x000D0003,
-+
-+	/* Synthetic VSM registers */
-+	HV_REGISTER_VSM_CODE_PAGE_OFFSETS	= 0x000D0002,
-+	HV_REGISTER_VSM_CAPABILITIES		= 0x000D0006,
-+	HV_REGISTER_VSM_PARTITION_CONFIG	= 0x000D0007,
-+
-+#if defined(CONFIG_X86)
-+	/* X64 Debug Registers */
-+	HV_X64_REGISTER_DR0	= 0x00050000,
-+	HV_X64_REGISTER_DR1	= 0x00050001,
-+	HV_X64_REGISTER_DR2	= 0x00050002,
-+	HV_X64_REGISTER_DR3	= 0x00050003,
-+	HV_X64_REGISTER_DR6	= 0x00050004,
-+	HV_X64_REGISTER_DR7	= 0x00050005,
-+
-+	/* X64 Cache control MSRs */
-+	HV_X64_REGISTER_MSR_MTRR_CAP		= 0x0008000D,
-+	HV_X64_REGISTER_MSR_MTRR_DEF_TYPE	= 0x0008000E,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASE0	= 0x00080010,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASE1	= 0x00080011,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASE2	= 0x00080012,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASE3	= 0x00080013,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASE4	= 0x00080014,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASE5	= 0x00080015,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASE6	= 0x00080016,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASE7	= 0x00080017,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASE8	= 0x00080018,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASE9	= 0x00080019,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASEA	= 0x0008001A,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASEB	= 0x0008001B,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASEC	= 0x0008001C,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASED	= 0x0008001D,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASEE	= 0x0008001E,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_BASEF	= 0x0008001F,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASK0	= 0x00080040,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASK1	= 0x00080041,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASK2	= 0x00080042,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASK3	= 0x00080043,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASK4	= 0x00080044,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASK5	= 0x00080045,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASK6	= 0x00080046,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASK7	= 0x00080047,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASK8	= 0x00080048,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASK9	= 0x00080049,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASKA	= 0x0008004A,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASKB	= 0x0008004B,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASKC	= 0x0008004C,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASKD	= 0x0008004D,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASKE	= 0x0008004E,
-+	HV_X64_REGISTER_MSR_MTRR_PHYS_MASKF	= 0x0008004F,
-+	HV_X64_REGISTER_MSR_MTRR_FIX64K00000	= 0x00080070,
-+	HV_X64_REGISTER_MSR_MTRR_FIX16K80000	= 0x00080071,
-+	HV_X64_REGISTER_MSR_MTRR_FIX16KA0000	= 0x00080072,
-+	HV_X64_REGISTER_MSR_MTRR_FIX4KC0000	= 0x00080073,
-+	HV_X64_REGISTER_MSR_MTRR_FIX4KC8000	= 0x00080074,
-+	HV_X64_REGISTER_MSR_MTRR_FIX4KD0000	= 0x00080075,
-+	HV_X64_REGISTER_MSR_MTRR_FIX4KD8000	= 0x00080076,
-+	HV_X64_REGISTER_MSR_MTRR_FIX4KE0000	= 0x00080077,
-+	HV_X64_REGISTER_MSR_MTRR_FIX4KE8000	= 0x00080078,
-+	HV_X64_REGISTER_MSR_MTRR_FIX4KF0000	= 0x00080079,
-+	HV_X64_REGISTER_MSR_MTRR_FIX4KF8000	= 0x0008007A,
-+
-+	HV_X64_REGISTER_REG_PAGE	= 0x0009001C,
-+#endif
-+};
-+
-+/*
-+ * Arch compatibility regs for use with hv_set/get_register
-+ */
-+#if defined(CONFIG_X86)
-+
-+/*
-+ * To support arch-generic code calling hv_set/get_register:
-+ * - On x86, HV_MSR_ indicates an MSR accessed via rdmsrq/wrmsrq
-+ * - On ARM, HV_MSR_ indicates a VP register accessed via hypercall
-+ */
-+#define HV_MSR_CRASH_P0		(HV_X64_MSR_CRASH_P0)
-+#define HV_MSR_CRASH_P1		(HV_X64_MSR_CRASH_P1)
-+#define HV_MSR_CRASH_P2		(HV_X64_MSR_CRASH_P2)
-+#define HV_MSR_CRASH_P3		(HV_X64_MSR_CRASH_P3)
-+#define HV_MSR_CRASH_P4		(HV_X64_MSR_CRASH_P4)
-+#define HV_MSR_CRASH_CTL	(HV_X64_MSR_CRASH_CTL)
-+
-+#define HV_MSR_VP_INDEX		(HV_X64_MSR_VP_INDEX)
-+#define HV_MSR_TIME_REF_COUNT	(HV_X64_MSR_TIME_REF_COUNT)
-+#define HV_MSR_REFERENCE_TSC	(HV_X64_MSR_REFERENCE_TSC)
-+
-+#define HV_MSR_SINT0		(HV_X64_MSR_SINT0)
-+#define HV_MSR_SVERSION		(HV_X64_MSR_SVERSION)
-+#define HV_MSR_SCONTROL		(HV_X64_MSR_SCONTROL)
-+#define HV_MSR_SIEFP		(HV_X64_MSR_SIEFP)
-+#define HV_MSR_SIMP		(HV_X64_MSR_SIMP)
-+#define HV_MSR_EOM		(HV_X64_MSR_EOM)
-+#define HV_MSR_SIRBP		(HV_X64_MSR_SIRBP)
-+
-+#define HV_MSR_NESTED_SCONTROL	(HV_X64_MSR_NESTED_SCONTROL)
-+#define HV_MSR_NESTED_SVERSION	(HV_X64_MSR_NESTED_SVERSION)
-+#define HV_MSR_NESTED_SIEFP	(HV_X64_MSR_NESTED_SIEFP)
-+#define HV_MSR_NESTED_SIMP	(HV_X64_MSR_NESTED_SIMP)
-+#define HV_MSR_NESTED_EOM	(HV_X64_MSR_NESTED_EOM)
-+#define HV_MSR_NESTED_SINT0	(HV_X64_MSR_NESTED_SINT0)
-+
-+#define HV_MSR_STIMER0_CONFIG	(HV_X64_MSR_STIMER0_CONFIG)
-+#define HV_MSR_STIMER0_COUNT	(HV_X64_MSR_STIMER0_COUNT)
-+
-+#elif defined(CONFIG_ARM64) /* CONFIG_X86 */
-+
-+#define HV_MSR_CRASH_P0		(HV_REGISTER_GUEST_CRASH_P0)
-+#define HV_MSR_CRASH_P1		(HV_REGISTER_GUEST_CRASH_P1)
-+#define HV_MSR_CRASH_P2		(HV_REGISTER_GUEST_CRASH_P2)
-+#define HV_MSR_CRASH_P3		(HV_REGISTER_GUEST_CRASH_P3)
-+#define HV_MSR_CRASH_P4		(HV_REGISTER_GUEST_CRASH_P4)
-+#define HV_MSR_CRASH_CTL	(HV_REGISTER_GUEST_CRASH_CTL)
-+
-+#define HV_MSR_VP_INDEX		(HV_REGISTER_VP_INDEX)
-+#define HV_MSR_TIME_REF_COUNT	(HV_REGISTER_TIME_REF_COUNT)
-+#define HV_MSR_REFERENCE_TSC	(HV_REGISTER_REFERENCE_TSC)
-+
-+#define HV_MSR_SINT0		(HV_REGISTER_SINT0)
-+#define HV_MSR_SCONTROL		(HV_REGISTER_SCONTROL)
-+#define HV_MSR_SIEFP		(HV_REGISTER_SIEFP)
-+#define HV_MSR_SIMP		(HV_REGISTER_SIMP)
-+#define HV_MSR_EOM		(HV_REGISTER_EOM)
-+#define HV_MSR_SIRBP		(HV_REGISTER_SIRBP)
-+
-+#define HV_MSR_STIMER0_CONFIG	(HV_REGISTER_STIMER0_CONFIG)
-+#define HV_MSR_STIMER0_COUNT	(HV_REGISTER_STIMER0_COUNT)
-+
-+#endif /* CONFIG_ARM64 */
-+
-+union hv_explicit_suspend_register {
-+	u64 as_uint64;
-+	struct {
-+		u64 suspended : 1;
-+		u64 reserved : 63;
-+	} __packed;
-+};
-+
-+union hv_intercept_suspend_register {
-+	u64 as_uint64;
-+	struct {
-+		u64 suspended : 1;
-+		u64 reserved : 63;
-+	} __packed;
-+};
-+
-+union hv_dispatch_suspend_register {
-+	u64 as_uint64;
-+	struct {
-+		u64 suspended : 1;
-+		u64 reserved : 63;
-+	} __packed;
-+};
-+
-+union hv_arm64_pending_interruption_register {
-+	u64 as_uint64;
-+	struct {
-+		u64 interruption_pending : 1;
-+		u64 interruption_type: 1;
-+		u64 reserved : 30;
-+		u64 error_code : 32;
-+	} __packed;
-+};
-+
-+union hv_arm64_interrupt_state_register {
-+	u64 as_uint64;
-+	struct {
-+		u64 interrupt_shadow : 1;
-+		u64 reserved : 63;
-+	} __packed;
-+};
-+
-+union hv_arm64_pending_synthetic_exception_event {
-+	u64 as_uint64[2];
-+	struct {
-+		u8 event_pending : 1;
-+		u8 event_type : 3;
-+		u8 reserved : 4;
-+		u8 rsvd[3];
-+		u32 exception_type;
-+		u64 context;
-+	} __packed;
-+};
-+
-+union hv_x64_interrupt_state_register {
-+	u64 as_uint64;
-+	struct {
-+		u64 interrupt_shadow : 1;
-+		u64 nmi_masked : 1;
-+		u64 reserved : 62;
-+	} __packed;
-+};
-+
-+union hv_x64_pending_interruption_register {
-+	u64 as_uint64;
-+	struct {
-+		u32 interruption_pending : 1;
-+		u32 interruption_type : 3;
-+		u32 deliver_error_code : 1;
-+		u32 instruction_length : 4;
-+		u32 nested_event : 1;
-+		u32 reserved : 6;
-+		u32 interruption_vector : 16;
-+		u32 error_code;
-+	} __packed;
-+};
-+
-+union hv_register_value {
-+	struct hv_u128 reg128;
-+	u64 reg64;
-+	u32 reg32;
-+	u16 reg16;
-+	u8 reg8;
-+
-+	struct hv_x64_segment_register segment;
-+	struct hv_x64_table_register table;
-+	union hv_explicit_suspend_register explicit_suspend;
-+	union hv_intercept_suspend_register intercept_suspend;
-+	union hv_dispatch_suspend_register dispatch_suspend;
-+#ifdef CONFIG_ARM64
-+	union hv_arm64_interrupt_state_register interrupt_state;
-+	union hv_arm64_pending_interruption_register pending_interruption;
-+#endif
-+#ifdef CONFIG_X86
-+	union hv_x64_interrupt_state_register interrupt_state;
-+	union hv_x64_pending_interruption_register pending_interruption;
-+#endif
-+	union hv_arm64_pending_synthetic_exception_event pending_synthetic_exception_event;
-+};
-+
-+/* NOTE: Linux helper struct - NOT from Hyper-V code. */
-+struct hv_output_get_vp_registers {
-+	DECLARE_FLEX_ARRAY(union hv_register_value, values);
-+};
-+
-+#if defined(CONFIG_ARM64)
-+/* HvGetVpRegisters returns an array of these output elements */
-+struct hv_get_vp_registers_output {
-+	union {
-+		struct {
-+			u32 a;
-+			u32 b;
-+			u32 c;
-+			u32 d;
-+		} as32 __packed;
-+		struct {
-+			u64 low;
-+			u64 high;
-+		} as64 __packed;
-+	};
-+};
-+
-+#endif /* CONFIG_ARM64 */
-+
-+struct hv_register_assoc {
-+	u32 name;			/* enum hv_register_name */
-+	u32 reserved1;
-+	u64 reserved2;
-+	union hv_register_value value;
-+} __packed;
-+
-+struct hv_input_get_vp_registers {
-+	u64 partition_id;
-+	u32 vp_index;
-+	union hv_input_vtl input_vtl;
-+	u8  rsvd_z8;
-+	u16 rsvd_z16;
-+	u32 names[];
-+} __packed;
-+
-+struct hv_input_set_vp_registers {
-+	u64 partition_id;
-+	u32 vp_index;
-+	union hv_input_vtl input_vtl;
-+	u8  rsvd_z8;
-+	u16 rsvd_z16;
-+	struct hv_register_assoc elements[];
-+} __packed;
-+
-+#define HV_UNMAP_GPA_LARGE_PAGE		0x2
-+
-+/* HvCallSendSyntheticClusterIpi hypercall */
-+struct hv_send_ipi {	 /* HV_INPUT_SEND_SYNTHETIC_CLUSTER_IPI */
-+	u32 vector;
-+	u32 reserved;
-+	u64 cpu_mask;
-+} __packed;
-+
-+#define	HV_VTL_MASK			GENMASK(3, 0)
-+
-+/* Hyper-V memory host visibility */
-+enum hv_mem_host_visibility {
-+	VMBUS_PAGE_NOT_VISIBLE		= 0,
-+	VMBUS_PAGE_VISIBLE_READ_ONLY	= 1,
-+	VMBUS_PAGE_VISIBLE_READ_WRITE	= 3
-+};
-+
-+/* HvCallModifySparseGpaPageHostVisibility hypercall */
-+#define HV_MAX_MODIFY_GPA_REP_COUNT	((HV_HYP_PAGE_SIZE / sizeof(u64)) - 2)
-+struct hv_gpa_range_for_visibility {
-+	u64 partition_id;
-+	u32 host_visibility : 2;
-+	u32 reserved0 : 30;
-+	u32 reserved1;
-+	u64 gpa_page_list[HV_MAX_MODIFY_GPA_REP_COUNT];
-+} __packed;
-+
-+#if defined(CONFIG_X86)
-+union hv_msi_address_register { /* HV_MSI_ADDRESS */
-+	u32 as_uint32;
-+	struct {
-+		u32 reserved1 : 2;
-+		u32 destination_mode : 1;
-+		u32 redirection_hint : 1;
-+		u32 reserved2 : 8;
-+		u32 destination_id : 8;
-+		u32 msi_base : 12;
-+	};
-+} __packed;
-+
-+union hv_msi_data_register {	 /* HV_MSI_ENTRY.Data */
-+	u32 as_uint32;
-+	struct {
-+		u32 vector : 8;
-+		u32 delivery_mode : 3;
-+		u32 reserved1 : 3;
-+		u32 level_assert : 1;
-+		u32 trigger_mode : 1;
-+		u32 reserved2 : 16;
-+	};
-+} __packed;
-+
-+union hv_msi_entry {	 /* HV_MSI_ENTRY */
-+
-+	u64 as_uint64;
-+	struct {
-+		union hv_msi_address_register address;
-+		union hv_msi_data_register data;
-+	} __packed;
-+};
-+
-+#elif defined(CONFIG_ARM64) /* CONFIG_X86 */
-+
-+union hv_msi_entry {
-+	u64 as_uint64[2];
-+	struct {
-+		u64 address;
-+		u32 data;
-+		u32 reserved;
-+	} __packed;
-+};
-+#endif /* CONFIG_ARM64 */
-+
-+union hv_ioapic_rte {
-+	u64 as_uint64;
-+
-+	struct {
-+		u32 vector : 8;
-+		u32 delivery_mode : 3;
-+		u32 destination_mode : 1;
-+		u32 delivery_status : 1;
-+		u32 interrupt_polarity : 1;
-+		u32 remote_irr : 1;
-+		u32 trigger_mode : 1;
-+		u32 interrupt_mask : 1;
-+		u32 reserved1 : 15;
-+
-+		u32 reserved2 : 24;
-+		u32 destination_id : 8;
-+	};
-+
-+	struct {
-+		u32 low_uint32;
-+		u32 high_uint32;
-+	};
-+} __packed;
-+
-+enum hv_interrupt_source {	 /* HV_INTERRUPT_SOURCE */
-+	HV_INTERRUPT_SOURCE_MSI = 1, /* MSI and MSI-X */
-+	HV_INTERRUPT_SOURCE_IOAPIC,
-+};
-+
-+struct hv_interrupt_entry {	 /* HV_INTERRUPT_ENTRY */
-+	u32 source;
-+	u32 reserved1;
-+	union {
-+		union hv_msi_entry msi_entry;
-+		union hv_ioapic_rte ioapic_rte;
-+	};
-+} __packed;
-+
-+#define HV_DEVICE_INTERRUPT_TARGET_MULTICAST		1
-+#define HV_DEVICE_INTERRUPT_TARGET_PROCESSOR_SET	2
-+
-+struct hv_device_interrupt_target {	 /* HV_DEVICE_INTERRUPT_TARGET */
-+	u32 vector;
-+	u32 flags;		/* HV_DEVICE_INTERRUPT_TARGET_* above */
-+	union {
-+		u64 vp_mask;
-+		struct hv_vpset vp_set;
-+	};
-+} __packed;
-+
-+struct hv_retarget_device_interrupt {	 /* HV_INPUT_RETARGET_DEVICE_INTERRUPT */
-+	u64 partition_id;		/* use "self" */
-+	u64 device_id;
-+	struct hv_interrupt_entry int_entry;
-+	u64 reserved2;
-+	struct hv_device_interrupt_target int_target;
-+} __packed __aligned(8);
-+
-+enum hv_intercept_type {
-+#if defined(CONFIG_X86)
-+	HV_INTERCEPT_TYPE_X64_IO_PORT			= 0x00000000,
-+	HV_INTERCEPT_TYPE_X64_MSR			= 0x00000001,
-+	HV_INTERCEPT_TYPE_X64_CPUID			= 0x00000002,
-+#endif
-+	HV_INTERCEPT_TYPE_EXCEPTION			= 0x00000003,
-+	/* Used to be HV_INTERCEPT_TYPE_REGISTER */
-+	HV_INTERCEPT_TYPE_RESERVED0			= 0x00000004,
-+	HV_INTERCEPT_TYPE_MMIO				= 0x00000005,
-+#if defined(CONFIG_X86)
-+	HV_INTERCEPT_TYPE_X64_GLOBAL_CPUID		= 0x00000006,
-+	HV_INTERCEPT_TYPE_X64_APIC_SMI			= 0x00000007,
-+#endif
-+	HV_INTERCEPT_TYPE_HYPERCALL			= 0x00000008,
-+#if defined(CONFIG_X86)
-+	HV_INTERCEPT_TYPE_X64_APIC_INIT_SIPI		= 0x00000009,
-+	HV_INTERCEPT_MC_UPDATE_PATCH_LEVEL_MSR_READ	= 0x0000000A,
-+	HV_INTERCEPT_TYPE_X64_APIC_WRITE		= 0x0000000B,
-+	HV_INTERCEPT_TYPE_X64_MSR_INDEX			= 0x0000000C,
-+#endif
-+	HV_INTERCEPT_TYPE_MAX,
-+	HV_INTERCEPT_TYPE_INVALID			= 0xFFFFFFFF,
-+};
-+
-+union hv_intercept_parameters {
-+	/*  HV_INTERCEPT_PARAMETERS is defined to be an 8-byte field. */
-+	u64 as_uint64;
-+#if defined(CONFIG_X86)
-+	/* HV_INTERCEPT_TYPE_X64_IO_PORT */
-+	u16 io_port;
-+	/* HV_INTERCEPT_TYPE_X64_CPUID */
-+	u32 cpuid_index;
-+	/* HV_INTERCEPT_TYPE_X64_APIC_WRITE */
-+	u32 apic_write_mask;
-+	/* HV_INTERCEPT_TYPE_EXCEPTION */
-+	u16 exception_vector;
-+	/* HV_INTERCEPT_TYPE_X64_MSR_INDEX */
-+	u32 msr_index;
-+#endif
-+	/* N.B. Other intercept types do not have any parameters. */
-+};
-+
-+/* Data structures for HVCALL_MMIO_READ and HVCALL_MMIO_WRITE */
-+#define HV_HYPERCALL_MMIO_MAX_DATA_LENGTH 64
-+
-+struct hv_mmio_read_input { /* HV_INPUT_MEMORY_MAPPED_IO_READ */
-+	u64 gpa;
-+	u32 size;
-+	u32 reserved;
-+} __packed;
-+
-+struct hv_mmio_read_output {
-+	u8 data[HV_HYPERCALL_MMIO_MAX_DATA_LENGTH];
-+} __packed;
-+
-+struct hv_mmio_write_input {
-+	u64 gpa;
-+	u32 size;
-+	u32 reserved;
-+	u8 data[HV_HYPERCALL_MMIO_MAX_DATA_LENGTH];
-+} __packed;
-+
-+#endif /* _HV_HVGDK_MINI_H */
-diff --git a/include/hyperv/hvhdk.h b/include/hyperv/hvhdk.h
-new file mode 100644
-index 000000000000..469186df7826
--- /dev/null
-+++ b/include/hyperv/hvhdk.h
-@@ -0,0 +1,899 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * Type definitions for the Microsoft hypervisor.
-+ */
-+#ifndef _HV_HVHDK_H
-+#define _HV_HVHDK_H
-+
-+#include <linux/build_bug.h>
-+
-+#include "hvhdk_mini.h"
-+#include "hvgdk.h"
-+
-+/* Bits for dirty mask of hv_vp_register_page */
-+#define HV_X64_REGISTER_CLASS_GENERAL	0
-+#define HV_X64_REGISTER_CLASS_IP	1
-+#define HV_X64_REGISTER_CLASS_XMM	2
-+#define HV_X64_REGISTER_CLASS_SEGMENT	3
-+#define HV_X64_REGISTER_CLASS_FLAGS	4
-+
-+#define HV_VP_REGISTER_PAGE_VERSION_1	1u
-+
-+#define HV_VP_REGISTER_PAGE_MAX_VECTOR_COUNT		7
-+
-+union hv_vp_register_page_interrupt_vectors {
-+	u64 as_uint64;
-+	struct {
-+		u8 vector_count;
-+		u8 vector[HV_VP_REGISTER_PAGE_MAX_VECTOR_COUNT];
-+	} __packed;
-+};
-+
-+struct hv_vp_register_page {
-+	u16 version;
-+	u8 isvalid;
-+	u8 rsvdz;
-+	u32 dirty;
-+
-+#if IS_ENABLED(CONFIG_X86)
-+
-+	union {
-+		struct {
-+			/* General purpose registers
-+			 * (HV_X64_REGISTER_CLASS_GENERAL)
-+			 */
-+			union {
-+				struct {
-+					u64 rax;
-+					u64 rcx;
-+					u64 rdx;
-+					u64 rbx;
-+					u64 rsp;
-+					u64 rbp;
-+					u64 rsi;
-+					u64 rdi;
-+					u64 r8;
-+					u64 r9;
-+					u64 r10;
-+					u64 r11;
-+					u64 r12;
-+					u64 r13;
-+					u64 r14;
-+					u64 r15;
-+				} __packed;
-+
-+				u64 gp_registers[16];
-+			};
-+			/* Instruction pointer (HV_X64_REGISTER_CLASS_IP) */
-+			u64 rip;
-+			/* Flags (HV_X64_REGISTER_CLASS_FLAGS) */
-+			u64 rflags;
-+		} __packed;
-+
-+		u64 registers[18];
-+	};
-+	/* Volatile XMM registers (HV_X64_REGISTER_CLASS_XMM) */
-+	union {
-+		struct {
-+			struct hv_u128 xmm0;
-+			struct hv_u128 xmm1;
-+			struct hv_u128 xmm2;
-+			struct hv_u128 xmm3;
-+			struct hv_u128 xmm4;
-+			struct hv_u128 xmm5;
-+		} __packed;
-+
-+		struct hv_u128 xmm_registers[6];
-+	};
-+	/* Segment registers (HV_X64_REGISTER_CLASS_SEGMENT) */
-+	union {
-+		struct {
-+			struct hv_x64_segment_register es;
-+			struct hv_x64_segment_register cs;
-+			struct hv_x64_segment_register ss;
-+			struct hv_x64_segment_register ds;
-+			struct hv_x64_segment_register fs;
-+			struct hv_x64_segment_register gs;
-+		} __packed;
-+
-+		struct hv_x64_segment_register segment_registers[6];
-+	};
-+	/* Misc. control registers (cannot be set via this interface) */
-+	u64 cr0;
-+	u64 cr3;
-+	u64 cr4;
-+	u64 cr8;
-+	u64 efer;
-+	u64 dr7;
-+	union hv_x64_pending_interruption_register pending_interruption;
-+	union hv_x64_interrupt_state_register interrupt_state;
-+	u64 instruction_emulation_hints;
-+	u64 xfem;
-+
-+	/*
-+	 * Fields from this point are not included in the register page save chunk.
-+	 * The reserved field is intended to maintain alignment for unsaved fields.
-+	 */
-+	u8 reserved1[0x100];
-+
-+	/*
-+	 * Interrupts injected as part of HvCallDispatchVp.
-+	 */
-+	union hv_vp_register_page_interrupt_vectors interrupt_vectors;
-+
-+#elif IS_ENABLED(CONFIG_ARM64)
-+	/* Not yet supported in ARM */
-+#endif
-+} __packed;
-+
-+#define HV_PARTITION_PROCESSOR_FEATURES_BANKS 2
-+
-+union hv_partition_processor_features {
-+	u64 as_uint64[HV_PARTITION_PROCESSOR_FEATURES_BANKS];
-+	struct {
-+		u64 sse3_support : 1;
-+		u64 lahf_sahf_support : 1;
-+		u64 ssse3_support : 1;
-+		u64 sse4_1_support : 1;
-+		u64 sse4_2_support : 1;
-+		u64 sse4a_support : 1;
-+		u64 xop_support : 1;
-+		u64 pop_cnt_support : 1;
-+		u64 cmpxchg16b_support : 1;
-+		u64 altmovcr8_support : 1;
-+		u64 lzcnt_support : 1;
-+		u64 mis_align_sse_support : 1;
-+		u64 mmx_ext_support : 1;
-+		u64 amd3dnow_support : 1;
-+		u64 extended_amd3dnow_support : 1;
-+		u64 page_1gb_support : 1;
-+		u64 aes_support : 1;
-+		u64 pclmulqdq_support : 1;
-+		u64 pcid_support : 1;
-+		u64 fma4_support : 1;
-+		u64 f16c_support : 1;
-+		u64 rd_rand_support : 1;
-+		u64 rd_wr_fs_gs_support : 1;
-+		u64 smep_support : 1;
-+		u64 enhanced_fast_string_support : 1;
-+		u64 bmi1_support : 1;
-+		u64 bmi2_support : 1;
-+		u64 hle_support_deprecated : 1;
-+		u64 rtm_support_deprecated : 1;
-+		u64 movbe_support : 1;
-+		u64 npiep1_support : 1;
-+		u64 dep_x87_fpu_save_support : 1;
-+		u64 rd_seed_support : 1;
-+		u64 adx_support : 1;
-+		u64 intel_prefetch_support : 1;
-+		u64 smap_support : 1;
-+		u64 hle_support : 1;
-+		u64 rtm_support : 1;
-+		u64 rdtscp_support : 1;
-+		u64 clflushopt_support : 1;
-+		u64 clwb_support : 1;
-+		u64 sha_support : 1;
-+		u64 x87_pointers_saved_support : 1;
-+		u64 invpcid_support : 1;
-+		u64 ibrs_support : 1;
-+		u64 stibp_support : 1;
-+		u64 ibpb_support: 1;
-+		u64 unrestricted_guest_support : 1;
-+		u64 mdd_support : 1;
-+		u64 fast_short_rep_mov_support : 1;
-+		u64 l1dcache_flush_support : 1;
-+		u64 rdcl_no_support : 1;
-+		u64 ibrs_all_support : 1;
-+		u64 skip_l1df_support : 1;
-+		u64 ssb_no_support : 1;
-+		u64 rsb_a_no_support : 1;
-+		u64 virt_spec_ctrl_support : 1;
-+		u64 rd_pid_support : 1;
-+		u64 umip_support : 1;
-+		u64 mbs_no_support : 1;
-+		u64 mb_clear_support : 1;
-+		u64 taa_no_support : 1;
-+		u64 tsx_ctrl_support : 1;
-+		/*
-+		 * N.B. The final processor feature bit in bank 0 is reserved to
-+		 * simplify potential downlevel backports.
-+		 */
-+		u64 reserved_bank0 : 1;
-+
-+		/* N.B. Begin bank 1 processor features. */
-+		u64 acount_mcount_support : 1;
-+		u64 tsc_invariant_support : 1;
-+		u64 cl_zero_support : 1;
-+		u64 rdpru_support : 1;
-+		u64 la57_support : 1;
-+		u64 mbec_support : 1;
-+		u64 nested_virt_support : 1;
-+		u64 psfd_support : 1;
-+		u64 cet_ss_support : 1;
-+		u64 cet_ibt_support : 1;
-+		u64 vmx_exception_inject_support : 1;
-+		u64 enqcmd_support : 1;
-+		u64 umwait_tpause_support : 1;
-+		u64 movdiri_support : 1;
-+		u64 movdir64b_support : 1;
-+		u64 cldemote_support : 1;
-+		u64 serialize_support : 1;
-+		u64 tsc_deadline_tmr_support : 1;
-+		u64 tsc_adjust_support : 1;
-+		u64 fzlrep_movsb : 1;
-+		u64 fsrep_stosb : 1;
-+		u64 fsrep_cmpsb : 1;
-+		u64 reserved_bank1 : 42;
-+	} __packed;
-+};
-+
-+union hv_partition_processor_xsave_features {
-+	struct {
-+		u64 xsave_support : 1;
-+		u64 xsaveopt_support : 1;
-+		u64 avx_support : 1;
-+		u64 reserved1 : 61;
-+	} __packed;
-+	u64 as_uint64;
-+};
-+
-+struct hv_partition_creation_properties {
-+	union hv_partition_processor_features disabled_processor_features;
-+	union hv_partition_processor_xsave_features
-+		disabled_processor_xsave_features;
-+} __packed;
-+
-+#define HV_PARTITION_SYNTHETIC_PROCESSOR_FEATURES_BANKS 1
-+
-+union hv_partition_synthetic_processor_features {
-+	u64 as_uint64[HV_PARTITION_SYNTHETIC_PROCESSOR_FEATURES_BANKS];
-+
-+	struct {
-+		u64 hypervisor_present : 1;
-+		/* Support for HV#1: (CPUID leaves 0x40000000 - 0x40000006)*/
-+		u64 hv1 : 1;
-+		u64 access_vp_run_time_reg : 1; /* HV_X64_MSR_VP_RUNTIME */
-+		u64 access_partition_reference_counter : 1; /* HV_X64_MSR_TIME_REF_COUNT */
-+		u64 access_synic_regs : 1; /* SINT-related registers */
-+		/*
-+		 * Access to HV_X64_MSR_STIMER0_CONFIG through
-+		 * HV_X64_MSR_STIMER3_COUNT.
-+		 */
-+		u64 access_synthetic_timer_regs : 1;
-+		u64 access_intr_ctrl_regs : 1; /* APIC MSRs and VP assist page*/
-+		/* HV_X64_MSR_GUEST_OS_ID and HV_X64_MSR_HYPERCALL */
-+		u64 access_hypercall_regs : 1;
-+		u64 access_vp_index : 1;
-+		u64 access_partition_reference_tsc : 1;
-+		u64 access_guest_idle_reg : 1;
-+		u64 access_frequency_regs : 1;
-+		u64 reserved_z12 : 1;
-+		u64 reserved_z13 : 1;
-+		u64 reserved_z14 : 1;
-+		u64 enable_extended_gva_ranges_for_flush_virtual_address_list : 1;
-+		u64 reserved_z16 : 1;
-+		u64 reserved_z17 : 1;
-+		/* Use fast hypercall output. Corresponds to privilege. */
-+		u64 fast_hypercall_output : 1;
-+		u64 reserved_z19 : 1;
-+		u64 start_virtual_processor : 1; /* Can start VPs */
-+		u64 reserved_z21 : 1;
-+		/* Synthetic timers in direct mode. */
-+		u64 direct_synthetic_timers : 1;
-+		u64 reserved_z23 : 1;
-+		u64 extended_processor_masks : 1;
-+
-+		/* Enable various hypercalls */
-+		u64 tb_flush_hypercalls : 1;
-+		u64 synthetic_cluster_ipi : 1;
-+		u64 notify_long_spin_wait : 1;
-+		u64 query_numa_distance : 1;
-+		u64 signal_events : 1;
-+		u64 retarget_device_interrupt : 1;
-+		u64 restore_time : 1;
-+
-+		/* EnlightenedVmcs nested enlightenment is supported. */
-+		u64 enlightened_vmcs : 1;
-+		u64 reserved : 31;
-+	} __packed;
-+};
-+
-+#define HV_MAKE_COMPATIBILITY_VERSION(major_, minor_)	\
-+	((u32)((major_) << 8 | (minor_)))
-+
-+#define HV_COMPATIBILITY_21_H2		HV_MAKE_COMPATIBILITY_VERSION(0X6, 0X9)
-+
-+union hv_partition_isolation_properties {
-+	u64 as_uint64;
-+	struct {
-+		u64 isolation_type: 5;
-+		u64 isolation_host_type : 2;
-+		u64 rsvd_z: 5;
-+		u64 shared_gpa_boundary_page_number: 52;
-+	} __packed;
-+};
-+
-+/*
-+ * Various isolation types supported by MSHV.
-+ */
-+#define HV_PARTITION_ISOLATION_TYPE_NONE            0
-+#define HV_PARTITION_ISOLATION_TYPE_SNP             2
-+#define HV_PARTITION_ISOLATION_TYPE_TDX             3
-+
-+/*
-+ * Various host isolation types supported by MSHV.
-+ */
-+#define HV_PARTITION_ISOLATION_HOST_TYPE_NONE       0x0
-+#define HV_PARTITION_ISOLATION_HOST_TYPE_HARDWARE   0x1
-+#define HV_PARTITION_ISOLATION_HOST_TYPE_RESERVED   0x2
-+
-+/* Note: Exo partition is enabled by default */
-+#define HV_PARTITION_CREATION_FLAG_GPA_SUPER_PAGES_ENABLED		BIT(4)
-+#define HV_PARTITION_CREATION_FLAG_EXO_PARTITION			BIT(8)
-+#define HV_PARTITION_CREATION_FLAG_LAPIC_ENABLED			BIT(13)
-+#define HV_PARTITION_CREATION_FLAG_INTERCEPT_MESSAGE_PAGE_ENABLED	BIT(19)
-+#define HV_PARTITION_CREATION_FLAG_X2APIC_CAPABLE			BIT(22)
-+
-+struct hv_input_create_partition {
-+	u64 flags;
-+	struct hv_proximity_domain_info proximity_domain_info;
-+	u32 compatibility_version;
-+	u32 padding;
-+	struct hv_partition_creation_properties partition_creation_properties;
-+	union hv_partition_isolation_properties isolation_properties;
-+} __packed;
-+
-+struct hv_output_create_partition {
-+	u64 partition_id;
-+} __packed;
-+
-+struct hv_input_initialize_partition {
-+	u64 partition_id;
-+} __packed;
-+
-+struct hv_input_finalize_partition {
-+	u64 partition_id;
-+} __packed;
-+
-+struct hv_input_delete_partition {
-+	u64 partition_id;
-+} __packed;
-+
-+struct hv_input_get_partition_property {
-+	u64 partition_id;
-+	u32 property_code; /* enum hv_partition_property_code */
-+	u32 padding;
-+} __packed;
-+
-+struct hv_output_get_partition_property {
-+	u64 property_value;
-+} __packed;
-+
-+struct hv_input_set_partition_property {
-+	u64 partition_id;
-+	u32 property_code; /* enum hv_partition_property_code */
-+	u32 padding;
-+	u64 property_value;
-+} __packed;
-+
-+union hv_partition_property_arg {
-+	u64 as_uint64;
-+	struct {
-+		union {
-+			u32 arg;
-+			u32 vp_index;
-+		};
-+		u16 reserved0;
-+		u8 reserved1;
-+		u8 object_type;
-+	} __packed;
-+};
-+
-+struct hv_input_get_partition_property_ex {
-+	u64 partition_id;
-+	u32 property_code; /* enum hv_partition_property_code */
-+	u32 padding;
-+	union {
-+		union hv_partition_property_arg arg_data;
-+		u64 arg;
-+	};
-+} __packed;
-+
-+/*
-+ * NOTE: Should use hv_input_set_partition_property_ex_header to compute this
-+ * size, but hv_input_get_partition_property_ex is identical so it suffices
-+ */
-+#define HV_PARTITION_PROPERTY_EX_MAX_VAR_SIZE \
-+	(HV_HYP_PAGE_SIZE - sizeof(struct hv_input_get_partition_property_ex))
-+
-+union hv_partition_property_ex {
-+	u8 buffer[HV_PARTITION_PROPERTY_EX_MAX_VAR_SIZE];
-+	struct hv_partition_property_vmm_capabilities vmm_capabilities;
-+	/* More fields to be filled in when needed */
-+};
-+
-+struct hv_output_get_partition_property_ex {
-+	union hv_partition_property_ex property_value;
-+} __packed;
-+
-+enum hv_vp_state_page_type {
-+	HV_VP_STATE_PAGE_REGISTERS = 0,
-+	HV_VP_STATE_PAGE_INTERCEPT_MESSAGE = 1,
-+	HV_VP_STATE_PAGE_GHCB = 2,
-+	HV_VP_STATE_PAGE_COUNT
-+};
-+
-+struct hv_input_map_vp_state_page {
-+	u64 partition_id;
-+	u32 vp_index;
-+	u16 type; /* enum hv_vp_state_page_type */
-+	union hv_input_vtl input_vtl;
-+	union {
-+		u8 as_uint8;
-+		struct {
-+			u8 map_location_provided : 1;
-+			u8 reserved : 7;
-+		};
-+	} flags;
-+	u64 requested_map_location;
-+} __packed;
-+
-+struct hv_output_map_vp_state_page {
-+	u64 map_location; /* GPA page number */
-+} __packed;
-+
-+struct hv_input_unmap_vp_state_page {
-+	u64 partition_id;
-+	u32 vp_index;
-+	u16 type; /* enum hv_vp_state_page_type */
-+	union hv_input_vtl input_vtl;
-+	u8 reserved0;
-+} __packed;
-+
-+struct hv_x64_apic_eoi_message {
-+	u32 vp_index;
-+	u32 interrupt_vector;
-+} __packed;
-+
-+struct hv_opaque_intercept_message {
-+	u32 vp_index;
-+} __packed;
-+
-+enum hv_port_type {
-+	HV_PORT_TYPE_MESSAGE = 1,
-+	HV_PORT_TYPE_EVENT   = 2,
-+	HV_PORT_TYPE_MONITOR = 3,
-+	HV_PORT_TYPE_DOORBELL = 4	/* Root Partition only */
-+};
-+
-+struct hv_port_info {
-+	u32 port_type; /* enum hv_port_type */
-+	u32 padding;
-+	union {
-+		struct {
-+			u32 target_sint;
-+			u32 target_vp;
-+			u64 rsvdz;
-+		} message_port_info;
-+		struct {
-+			u32 target_sint;
-+			u32 target_vp;
-+			u16 base_flag_number;
-+			u16 flag_count;
-+			u32 rsvdz;
-+		} event_port_info;
-+		struct {
-+			u64 monitor_address;
-+			u64 rsvdz;
-+		} monitor_port_info;
-+		struct {
-+			u32 target_sint;
-+			u32 target_vp;
-+			u64 rsvdz;
-+		} doorbell_port_info;
-+	};
-+} __packed;
-+
-+struct hv_connection_info {
-+	u32 port_type;
-+	u32 padding;
-+	union {
-+		struct {
-+			u64 rsvdz;
-+		} message_connection_info;
-+		struct {
-+			u64 rsvdz;
-+		} event_connection_info;
-+		struct {
-+			u64 monitor_address;
-+		} monitor_connection_info;
-+		struct {
-+			u64 gpa;
-+			u64 trigger_value;
-+			u64 flags;
-+		} doorbell_connection_info;
-+	};
-+} __packed;
-+
-+/* Define synthetic interrupt controller flag constants. */
-+#define HV_EVENT_FLAGS_COUNT		(256 * 8)
-+#define HV_EVENT_FLAGS_BYTE_COUNT	(256)
-+#define HV_EVENT_FLAGS32_COUNT		(256 / sizeof(u32))
-+
-+/* linux side we create long version of flags to use long bit ops on flags */
-+#define HV_EVENT_FLAGS_UL_COUNT		(256 / sizeof(ulong))
-+
-+/* Define the synthetic interrupt controller event flags format. */
-+union hv_synic_event_flags {
-+	unsigned char flags8[HV_EVENT_FLAGS_BYTE_COUNT];
-+	u32 flags32[HV_EVENT_FLAGS32_COUNT];
-+	ulong flags[HV_EVENT_FLAGS_UL_COUNT];  /* linux only */
-+};
-+
-+struct hv_synic_event_flags_page {
-+	volatile union hv_synic_event_flags event_flags[HV_SYNIC_SINT_COUNT];
-+};
-+
-+#define HV_SYNIC_EVENT_RING_MESSAGE_COUNT 63
-+
-+struct hv_synic_event_ring {
-+	u8  signal_masked;
-+	u8  ring_full;
-+	u16 reserved_z;
-+	u32 data[HV_SYNIC_EVENT_RING_MESSAGE_COUNT];
-+} __packed;
-+
-+struct hv_synic_event_ring_page {
-+	struct hv_synic_event_ring sint_event_ring[HV_SYNIC_SINT_COUNT];
-+};
-+
-+/* Define SynIC control register. */
-+union hv_synic_scontrol {
-+	u64 as_uint64;
-+	struct {
-+		u64 enable : 1;
-+		u64 reserved : 63;
-+	} __packed;
-+};
-+
-+/* Define the format of the SIEFP register */
-+union hv_synic_siefp {
-+	u64 as_uint64;
-+	struct {
-+		u64 siefp_enabled : 1;
-+		u64 preserved : 11;
-+		u64 base_siefp_gpa : 52;
-+	} __packed;
-+};
-+
-+union hv_synic_sirbp {
-+	u64 as_uint64;
-+	struct {
-+		u64 sirbp_enabled : 1;
-+		u64 preserved : 11;
-+		u64 base_sirbp_gpa : 52;
-+	} __packed;
-+};
-+
-+union hv_interrupt_control {
-+	u64 as_uint64;
-+	struct {
-+		u32 interrupt_type; /* enum hv_interrupt_type */
-+#if IS_ENABLED(CONFIG_X86)
-+		u32 level_triggered : 1;
-+		u32 logical_dest_mode : 1;
-+		u32 rsvd : 30;
-+#elif IS_ENABLED(CONFIG_ARM64)
-+		u32 rsvd1 : 2;
-+		u32 asserted : 1;
-+		u32 rsvd2 : 29;
-+#endif
-+	} __packed;
-+};
-+
-+struct hv_stimer_state {
-+	struct {
-+		u32 undelivered_msg_pending : 1;
-+		u32 reserved : 31;
-+	} __packed flags;
-+	u32 resvd;
-+	u64 config;
-+	u64 count;
-+	u64 adjustment;
-+	u64 undelivered_exp_time;
-+} __packed;
-+
-+struct hv_synthetic_timers_state {
-+	struct hv_stimer_state timers[HV_SYNIC_STIMER_COUNT];
-+	u64 reserved[5];
-+} __packed;
-+
-+struct hv_async_completion_message_payload {
-+	u64 partition_id;
-+	u32 status;
-+	u32 completion_count;
-+	u64 sub_status;
-+} __packed;
-+
-+union hv_input_delete_vp {
-+	u64 as_uint64[2];
-+	struct {
-+		u64 partition_id;
-+		u32 vp_index;
-+		u8 reserved[4];
-+	} __packed;
-+} __packed;
-+
-+struct hv_input_assert_virtual_interrupt {
-+	u64 partition_id;
-+	union hv_interrupt_control control;
-+	u64 dest_addr; /* cpu's apic id */
-+	u32 vector;
-+	u8 target_vtl;
-+	u8 rsvd_z0;
-+	u16 rsvd_z1;
-+} __packed;
-+
-+struct hv_input_create_port {
-+	u64 port_partition_id;
-+	union hv_port_id port_id;
-+	u8 port_vtl;
-+	u8 min_connection_vtl;
-+	u16 padding;
-+	u64 connection_partition_id;
-+	struct hv_port_info port_info;
-+	struct hv_proximity_domain_info proximity_domain_info;
-+} __packed;
-+
-+union hv_input_delete_port {
-+	u64 as_uint64[2];
-+	struct {
-+		u64 port_partition_id;
-+		union hv_port_id port_id;
-+		u32 reserved;
-+	};
-+} __packed;
-+
-+struct hv_input_connect_port {
-+	u64 connection_partition_id;
-+	union hv_connection_id connection_id;
-+	u8 connection_vtl;
-+	u8 rsvdz0;
-+	u16 rsvdz1;
-+	u64 port_partition_id;
-+	union hv_port_id port_id;
-+	u32 reserved2;
-+	struct hv_connection_info connection_info;
-+	struct hv_proximity_domain_info proximity_domain_info;
-+} __packed;
-+
-+union hv_input_disconnect_port {
-+	u64 as_uint64[2];
-+	struct {
-+		u64 connection_partition_id;
-+		union hv_connection_id connection_id;
-+		u32 is_doorbell: 1;
-+		u32 reserved: 31;
-+	} __packed;
-+} __packed;
-+
-+union hv_input_notify_port_ring_empty {
-+	u64 as_uint64;
-+	struct {
-+		u32 sint_index;
-+		u32 reserved;
-+	};
-+} __packed;
-+
-+struct hv_vp_state_data_xsave {
-+	u64 flags;
-+	union hv_x64_xsave_xfem_register states;
-+} __packed;
-+
-+/*
-+ * For getting and setting VP state, there are two options based on the state type:
-+ *
-+ *     1.) Data that is accessed by PFNs in the input hypercall page. This is used
-+ *         for state which may not fit into the hypercall pages.
-+ *     2.) Data that is accessed directly in the input\output hypercall pages.
-+ *         This is used for state that will always fit into the hypercall pages.
-+ *
-+ * In the future this could be dynamic based on the size if needed.
-+ *
-+ * Note these hypercalls have an 8-byte aligned variable header size as per the tlfs
-+ */
-+
-+#define HV_GET_SET_VP_STATE_TYPE_PFN	BIT(31)
-+
-+enum hv_get_set_vp_state_type {
-+	/* HvGetSetVpStateLocalInterruptControllerState - APIC/GIC state */
-+	HV_GET_SET_VP_STATE_LAPIC_STATE	     = 0 | HV_GET_SET_VP_STATE_TYPE_PFN,
-+	HV_GET_SET_VP_STATE_XSAVE	     = 1 | HV_GET_SET_VP_STATE_TYPE_PFN,
-+	HV_GET_SET_VP_STATE_SIM_PAGE	     = 2 | HV_GET_SET_VP_STATE_TYPE_PFN,
-+	HV_GET_SET_VP_STATE_SIEF_PAGE	     = 3 | HV_GET_SET_VP_STATE_TYPE_PFN,
-+	HV_GET_SET_VP_STATE_SYNTHETIC_TIMERS = 4,
-+};
-+
-+struct hv_vp_state_data {
-+	u32 type;
-+	u32 rsvd;
-+	struct hv_vp_state_data_xsave xsave;
-+} __packed;
-+
-+struct hv_input_get_vp_state {
-+	u64 partition_id;
-+	u32 vp_index;
-+	u8 input_vtl;
-+	u8 rsvd0;
-+	u16 rsvd1;
-+	struct hv_vp_state_data state_data;
-+	u64 output_data_pfns[];
-+} __packed;
-+
-+union hv_output_get_vp_state {
-+	struct hv_synthetic_timers_state synthetic_timers_state;
-+} __packed;
-+
-+union hv_input_set_vp_state_data {
-+	u64 pfns;
-+	u8 bytes;
-+} __packed;
-+
-+struct hv_input_set_vp_state {
-+	u64 partition_id;
-+	u32 vp_index;
-+	u8 input_vtl;
-+	u8 rsvd0;
-+	u16 rsvd1;
-+	struct hv_vp_state_data state_data;
-+	union hv_input_set_vp_state_data data[];
-+} __packed;
-+
-+union hv_x64_vp_execution_state {
-+	u16 as_uint16;
-+	struct {
-+		u16 cpl:2;
-+		u16 cr0_pe:1;
-+		u16 cr0_am:1;
-+		u16 efer_lma:1;
-+		u16 debug_active:1;
-+		u16 interruption_pending:1;
-+		u16 vtl:4;
-+		u16 enclave_mode:1;
-+		u16 interrupt_shadow:1;
-+		u16 virtualization_fault_active:1;
-+		u16 reserved:2;
-+	} __packed;
-+};
-+
-+struct hv_x64_intercept_message_header {
-+	u32 vp_index;
-+	u8 instruction_length:4;
-+	u8 cr8:4; /* Only set for exo partitions */
-+	u8 intercept_access_type;
-+	union hv_x64_vp_execution_state execution_state;
-+	struct hv_x64_segment_register cs_segment;
-+	u64 rip;
-+	u64 rflags;
-+} __packed;
-+
-+union hv_x64_memory_access_info {
-+	u8 as_uint8;
-+	struct {
-+		u8 gva_valid:1;
-+		u8 gva_gpa_valid:1;
-+		u8 hypercall_output_pending:1;
-+		u8 tlb_locked_no_overlay:1;
-+		u8 reserved:4;
-+	} __packed;
-+};
-+
-+struct hv_x64_memory_intercept_message {
-+	struct hv_x64_intercept_message_header header;
-+	u32 cache_type; /* enum hv_cache_type */
-+	u8 instruction_byte_count;
-+	union hv_x64_memory_access_info memory_access_info;
-+	u8 tpr_priority;
-+	u8 reserved1;
-+	u64 guest_virtual_address;
-+	u64 guest_physical_address;
-+	u8 instruction_bytes[16];
-+} __packed;
-+
-+/*
-+ * Dispatch state for the VP communicated by the hypervisor to the
-+ * VP-dispatching thread in the root on return from HVCALL_DISPATCH_VP.
-+ */
-+enum hv_vp_dispatch_state {
-+	HV_VP_DISPATCH_STATE_INVALID	= 0,
-+	HV_VP_DISPATCH_STATE_BLOCKED	= 1,
-+	HV_VP_DISPATCH_STATE_READY	= 2,
-+};
-+
-+/*
-+ * Dispatch event that caused the current dispatch state on return from
-+ * HVCALL_DISPATCH_VP.
-+ */
-+enum hv_vp_dispatch_event {
-+	HV_VP_DISPATCH_EVENT_INVALID	= 0x00000000,
-+	HV_VP_DISPATCH_EVENT_SUSPEND	= 0x00000001,
-+	HV_VP_DISPATCH_EVENT_INTERCEPT	= 0x00000002,
-+};
-+
-+#define HV_ROOT_SCHEDULER_MAX_VPS_PER_CHILD_PARTITION   1024
-+/* The maximum array size of HV_GENERIC_SET (vp_set) buffer */
-+#define HV_GENERIC_SET_QWORD_COUNT(max) (((((max) - 1) >> 6) + 1) + 2)
-+
-+struct hv_vp_signal_bitset_scheduler_message {
-+	u64 partition_id;
-+	u32 overflow_count;
-+	u16 vp_count;
-+	u16 reserved;
-+
-+#define BITSET_BUFFER_SIZE \
-+	HV_GENERIC_SET_QWORD_COUNT(HV_ROOT_SCHEDULER_MAX_VPS_PER_CHILD_PARTITION)
-+	union {
-+		struct hv_vpset bitset;
-+		u64 bitset_buffer[BITSET_BUFFER_SIZE];
-+	} vp_bitset;
-+#undef BITSET_BUFFER_SIZE
-+} __packed;
-+
-+static_assert(sizeof(struct hv_vp_signal_bitset_scheduler_message) <=
-+	(sizeof(struct hv_message) - sizeof(struct hv_message_header)));
-+
-+#define HV_MESSAGE_MAX_PARTITION_VP_PAIR_COUNT \
-+	(((sizeof(struct hv_message) - sizeof(struct hv_message_header)) / \
-+	 (sizeof(u64 /* partition id */) + sizeof(u32 /* vp index */))) - 1)
-+
-+struct hv_vp_signal_pair_scheduler_message {
-+	u32 overflow_count;
-+	u8 vp_count;
-+	u8 reserved1[3];
-+
-+	u64 partition_ids[HV_MESSAGE_MAX_PARTITION_VP_PAIR_COUNT];
-+	u32 vp_indexes[HV_MESSAGE_MAX_PARTITION_VP_PAIR_COUNT];
-+
-+	u8 reserved2[4];
-+} __packed;
-+
-+static_assert(sizeof(struct hv_vp_signal_pair_scheduler_message) ==
-+	(sizeof(struct hv_message) - sizeof(struct hv_message_header)));
-+
-+/* Input and output structures for HVCALL_DISPATCH_VP */
-+#define HV_DISPATCH_VP_FLAG_CLEAR_INTERCEPT_SUSPEND	0x1
-+#define HV_DISPATCH_VP_FLAG_ENABLE_CALLER_INTERRUPTS	0x2
-+#define HV_DISPATCH_VP_FLAG_SET_CALLER_SPEC_CTRL	0x4
-+#define HV_DISPATCH_VP_FLAG_SKIP_VP_SPEC_FLUSH		0x8
-+#define HV_DISPATCH_VP_FLAG_SKIP_CALLER_SPEC_FLUSH	0x10
-+#define HV_DISPATCH_VP_FLAG_SKIP_CALLER_USER_SPEC_FLUSH	0x20
-+#define HV_DISPATCH_VP_FLAG_SCAN_INTERRUPT_INJECTION	0x40
-+
-+struct hv_input_dispatch_vp {
-+	u64 partition_id;
-+	u32 vp_index;
-+	u32 flags;
-+	u64 time_slice; /* in 100ns */
-+	u64 spec_ctrl;
-+} __packed;
-+
-+struct hv_output_dispatch_vp {
-+	u32 dispatch_state; /* enum hv_vp_dispatch_state */
-+	u32 dispatch_event; /* enum hv_vp_dispatch_event */
-+} __packed;
-+
-+struct hv_input_modify_sparse_spa_page_host_access {
-+	u32 host_access : 2;
-+	u32 reserved : 30;
-+	u32 flags;
-+	u64 partition_id;
-+	u64 spa_page_list[];
-+} __packed;
-+
-+/* hv_input_modify_sparse_spa_page_host_access flags */
-+#define HV_MODIFY_SPA_PAGE_HOST_ACCESS_MAKE_EXCLUSIVE  0x1
-+#define HV_MODIFY_SPA_PAGE_HOST_ACCESS_MAKE_SHARED     0x2
-+#define HV_MODIFY_SPA_PAGE_HOST_ACCESS_LARGE_PAGE      0x4
-+#define HV_MODIFY_SPA_PAGE_HOST_ACCESS_HUGE_PAGE       0x8
-+
-+#endif /* _HV_HVHDK_H */
-diff --git a/include/hyperv/hvhdk_mini.h b/include/hyperv/hvhdk_mini.h
-new file mode 100644
-index 000000000000..41a29bf8ec14
--- /dev/null
-+++ b/include/hyperv/hvhdk_mini.h
-@@ -0,0 +1,531 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * Type definitions for the Microsoft Hypervisor.
-+ */
-+#ifndef _HV_HVHDK_MINI_H
-+#define _HV_HVHDK_MINI_H
-+
-+#include "hvgdk_mini.h"
-+
-+/*
-+ * Doorbell connection_info flags.
-+ */
-+#define HV_DOORBELL_FLAG_TRIGGER_SIZE_MASK  0x00000007
-+#define HV_DOORBELL_FLAG_TRIGGER_SIZE_ANY   0x00000000
-+#define HV_DOORBELL_FLAG_TRIGGER_SIZE_BYTE  0x00000001
-+#define HV_DOORBELL_FLAG_TRIGGER_SIZE_WORD  0x00000002
-+#define HV_DOORBELL_FLAG_TRIGGER_SIZE_DWORD 0x00000003
-+#define HV_DOORBELL_FLAG_TRIGGER_SIZE_QWORD 0x00000004
-+#define HV_DOORBELL_FLAG_TRIGGER_ANY_VALUE  0x80000000
-+
-+/* Each generic set contains 64 elements */
-+#define HV_GENERIC_SET_SHIFT		(6)
-+#define HV_GENERIC_SET_MASK		(63)
-+
-+enum hv_generic_set_format {
-+	HV_GENERIC_SET_SPARSE_4K,
-+	HV_GENERIC_SET_ALL,
-+};
-+#define HV_GENERIC_SET_FORMAT hv_generic_set_format
-+
-+enum hv_scheduler_type {
-+	HV_SCHEDULER_TYPE_LP		= 1, /* Classic scheduler w/o SMT */
-+	HV_SCHEDULER_TYPE_LP_SMT	= 2, /* Classic scheduler w/ SMT */
-+	HV_SCHEDULER_TYPE_CORE_SMT	= 3, /* Core scheduler */
-+	HV_SCHEDULER_TYPE_ROOT		= 4, /* Root / integrated scheduler */
-+	HV_SCHEDULER_TYPE_MAX
-+};
-+
-+/* HV_STATS_AREA_TYPE */
-+enum hv_stats_area_type {
-+	HV_STATS_AREA_SELF = 0,
-+	HV_STATS_AREA_PARENT = 1,
-+	HV_STATS_AREA_INTERNAL = 2,
-+	HV_STATS_AREA_COUNT
-+};
-+
-+enum hv_stats_object_type {
-+	HV_STATS_OBJECT_HYPERVISOR		= 0x00000001,
-+	HV_STATS_OBJECT_LOGICAL_PROCESSOR	= 0x00000002,
-+	HV_STATS_OBJECT_PARTITION		= 0x00010001,
-+	HV_STATS_OBJECT_VP			= 0x00010002
-+};
-+
-+union hv_stats_object_identity {
-+	/* hv_stats_hypervisor */
-+	struct {
-+		u8 reserved[15];
-+		u8 stats_area_type;
-+	} __packed hv;
-+
-+	/* hv_stats_logical_processor */
-+	struct {
-+		u32 lp_index;
-+		u8 reserved[11];
-+		u8 stats_area_type;
-+	} __packed lp;
-+
-+	/* hv_stats_partition */
-+	struct {
-+		u64 partition_id;
-+		u8  reserved[7];
-+		u8  stats_area_type;
-+	} __packed partition;
-+
-+	/* hv_stats_vp */
-+	struct {
-+		u64 partition_id;
-+		u32 vp_index;
-+		u16 flags;
-+		u8  reserved;
-+		u8  stats_area_type;
-+	} __packed vp;
-+};
-+
-+enum hv_partition_property_code {
-+	/* Privilege properties */
-+	HV_PARTITION_PROPERTY_PRIVILEGE_FLAGS			= 0x00010000,
-+	HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES		= 0x00010001,
-+
-+	/* Resource properties */
-+	HV_PARTITION_PROPERTY_GPA_PAGE_ACCESS_TRACKING		= 0x00050005,
-+	HV_PARTITION_PROPERTY_UNIMPLEMENTED_MSR_ACTION		= 0x00050017,
-+
-+	/* Compatibility properties */
-+	HV_PARTITION_PROPERTY_PROCESSOR_XSAVE_FEATURES		= 0x00060002,
-+	HV_PARTITION_PROPERTY_XSAVE_STATES                      = 0x00060007,
-+	HV_PARTITION_PROPERTY_MAX_XSAVE_DATA_SIZE		= 0x00060008,
-+	HV_PARTITION_PROPERTY_PROCESSOR_CLOCK_FREQUENCY		= 0x00060009,
-+
-+	/* Extended properties with larger property values */
-+	HV_PARTITION_PROPERTY_VMM_CAPABILITIES			= 0x00090007,
-+};
-+
-+#define HV_PARTITION_VMM_CAPABILITIES_BANK_COUNT		1
-+#define HV_PARTITION_VMM_CAPABILITIES_RESERVED_BITFIELD_COUNT	59
-+
-+struct hv_partition_property_vmm_capabilities {
-+	u16 bank_count;
-+	u16 reserved[3];
-+	union {
-+		u64 as_uint64[HV_PARTITION_VMM_CAPABILITIES_BANK_COUNT];
-+		struct {
-+			u64 map_gpa_preserve_adjustable: 1;
-+			u64 vmm_can_provide_overlay_gpfn: 1;
-+			u64 vp_affinity_property: 1;
-+#if IS_ENABLED(CONFIG_ARM64)
-+			u64 vmm_can_provide_gic_overlay_locations: 1;
-+#else
-+			u64 reservedbit3: 1;
-+#endif
-+			u64 assignable_synthetic_proc_features: 1;
-+			u64 reserved0: HV_PARTITION_VMM_CAPABILITIES_RESERVED_BITFIELD_COUNT;
-+		} __packed;
-+	};
-+} __packed;
-+
-+enum hv_snp_status {
-+	HV_SNP_STATUS_NONE = 0,
-+	HV_SNP_STATUS_AVAILABLE = 1,
-+	HV_SNP_STATUS_INCOMPATIBLE = 2,
-+	HV_SNP_STATUS_PSP_UNAVAILABLE = 3,
-+	HV_SNP_STATUS_PSP_INIT_FAILED = 4,
-+	HV_SNP_STATUS_PSP_BAD_FW_VERSION = 5,
-+	HV_SNP_STATUS_BAD_CONFIGURATION = 6,
-+	HV_SNP_STATUS_PSP_FW_UPDATE_IN_PROGRESS = 7,
-+	HV_SNP_STATUS_PSP_RB_INIT_FAILED = 8,
-+	HV_SNP_STATUS_PSP_PLATFORM_STATUS_FAILED = 9,
-+	HV_SNP_STATUS_PSP_INIT_LATE_FAILED = 10,
-+};
-+
-+enum hv_system_property {
-+	/* Add more values when needed */
-+	HV_SYSTEM_PROPERTY_SLEEP_STATE = 3,
-+	HV_SYSTEM_PROPERTY_SCHEDULER_TYPE = 15,
-+	HV_DYNAMIC_PROCESSOR_FEATURE_PROPERTY = 21,
-+	HV_SYSTEM_PROPERTY_CRASHDUMPAREA = 47,
-+};
-+
-+#define HV_PFN_RANGE_PGBITS 24  /* HV_SPA_PAGE_RANGE_ADDITIONAL_PAGES_BITS */
-+union hv_pfn_range {            /* HV_SPA_PAGE_RANGE */
-+	u64 as_uint64;
-+	struct {
-+		/* 39:0: base pfn.  63:40: additional pages */
-+		u64 base_pfn : 64 - HV_PFN_RANGE_PGBITS;
-+		u64 add_pfns : HV_PFN_RANGE_PGBITS;
-+	} __packed;
-+};
-+
-+enum hv_sleep_state {
-+	HV_SLEEP_STATE_S1 = 1,
-+	HV_SLEEP_STATE_S2 = 2,
-+	HV_SLEEP_STATE_S3 = 3,
-+	HV_SLEEP_STATE_S4 = 4,
-+	HV_SLEEP_STATE_S5 = 5,
-+	/*
-+	 * After hypervisor has received this, any follow up sleep
-+	 * state registration requests will be rejected.
-+	 */
-+	HV_SLEEP_STATE_LOCK = 6
-+};
-+
-+enum hv_dynamic_processor_feature_property {
-+	/* Add more values when needed */
-+	HV_X64_DYNAMIC_PROCESSOR_FEATURE_MAX_ENCRYPTED_PARTITIONS = 13,
-+	HV_X64_DYNAMIC_PROCESSOR_FEATURE_SNP_STATUS = 16,
-+};
-+
-+struct hv_input_get_system_property {
-+	u32 property_id; /* enum hv_system_property */
-+	union {
-+		u32 as_uint32;
-+#if IS_ENABLED(CONFIG_X86)
-+		/* enum hv_dynamic_processor_feature_property */
-+		u32 hv_processor_feature;
-+#endif
-+		/* More fields to be filled in when needed */
-+	};
-+} __packed;
-+
-+struct hv_output_get_system_property {
-+	union {
-+		u32 scheduler_type; /* enum hv_scheduler_type */
-+#if IS_ENABLED(CONFIG_X86)
-+		u64 hv_processor_feature_value;
-+#endif
-+		union hv_pfn_range hv_cda_info; /* CrashdumpAreaAddress */
-+		u64 hv_tramp_pa;                /* CrashdumpTrampolineAddress */
-+	};
-+} __packed;
-+
-+struct hv_sleep_state_info {
-+	u32 sleep_state; /* enum hv_sleep_state */
-+	u8 pm1a_slp_typ;
-+	u8 pm1b_slp_typ;
-+} __packed;
-+
-+struct hv_input_set_system_property {
-+	u32 property_id; /* enum hv_system_property */
-+	u32 reserved;
-+	union {
-+		/* More fields to be filled in when needed */
-+		struct hv_sleep_state_info set_sleep_state_info;
-+
-+		/*
-+		 * Add a reserved field to ensure the union is 8-byte aligned as
-+		 * existing members may not be. This is a temporary measure
-+		 * until all remaining members are added.
-+		 */
-+		 u64 reserved0[8];
-+	};
-+} __packed;
-+
-+struct hv_input_enter_sleep_state {     /* HV_INPUT_ENTER_SLEEP_STATE */
-+	u32 sleep_state;        /* enum hv_sleep_state */
-+} __packed;
-+
-+struct hv_input_map_stats_page {
-+	u32 type; /* enum hv_stats_object_type */
-+	u32 padding;
-+	union hv_stats_object_identity identity;
-+} __packed;
-+
-+struct hv_input_map_stats_page2 {
-+	u32 type; /* enum hv_stats_object_type */
-+	u32 padding;
-+	union hv_stats_object_identity identity;
-+	u64 map_location;
-+} __packed;
-+
-+struct hv_output_map_stats_page {
-+	u64 map_location;
-+} __packed;
-+
-+struct hv_input_unmap_stats_page {
-+	u32 type; /* enum hv_stats_object_type */
-+	u32 padding;
-+	union hv_stats_object_identity identity;
-+} __packed;
-+
-+struct hv_proximity_domain_flags {
-+	u32 proximity_preferred : 1;
-+	u32 reserved : 30;
-+	u32 proximity_info_valid : 1;
-+} __packed;
-+
-+struct hv_proximity_domain_info {
-+	u32 domain_id;
-+	struct hv_proximity_domain_flags flags;
-+} __packed;
-+
-+/* HvDepositMemory hypercall */
-+struct hv_deposit_memory {	/* HV_INPUT_DEPOSIT_MEMORY */
-+	u64 partition_id;
-+	u64 gpa_page_list[];
-+} __packed;
-+
-+struct hv_input_withdraw_memory {
-+	u64 partition_id;
-+	struct hv_proximity_domain_info proximity_domain_info;
-+} __packed;
-+
-+struct hv_output_withdraw_memory {
-+	DECLARE_FLEX_ARRAY(u64, gpa_page_list);
-+} __packed;
-+
-+/* HV Map GPA (Guest Physical Address) Flags */
-+#define HV_MAP_GPA_PERMISSIONS_NONE	       0x0
-+#define HV_MAP_GPA_READABLE		       0x1
-+#define HV_MAP_GPA_WRITABLE		       0x2
-+#define HV_MAP_GPA_KERNEL_EXECUTABLE	       0x4
-+#define HV_MAP_GPA_USER_EXECUTABLE	       0x8
-+#define HV_MAP_GPA_EXECUTABLE		       0xC
-+#define HV_MAP_GPA_PERMISSIONS_MASK	       0xF
-+#define HV_MAP_GPA_ADJUSTABLE		    0x8000
-+#define HV_MAP_GPA_NO_ACCESS		   0x10000
-+#define HV_MAP_GPA_NOT_CACHED		  0x200000
-+#define HV_MAP_GPA_LARGE_PAGE		0x80000000
-+
-+struct hv_input_map_gpa_pages {
-+	u64 target_partition_id;
-+	u64 target_gpa_base;
-+	u32 map_flags;
-+	u32 padding;
-+	u64 source_gpa_page_list[];
-+} __packed;
-+
-+union hv_gpa_page_access_state_flags {
-+	struct {
-+		u64 clear_accessed : 1;
-+		u64 set_accessed : 1;
-+		u64 clear_dirty : 1;
-+		u64 set_dirty : 1;
-+		u64 reserved : 60;
-+	} __packed;
-+	u64 as_uint64;
-+};
-+
-+struct hv_input_get_gpa_pages_access_state {
-+	u64  partition_id;
-+	union hv_gpa_page_access_state_flags flags;
-+	u64 hv_gpa_page_number;
-+} __packed;
-+
-+union hv_gpa_page_access_state {
-+	struct {
-+		u8 accessed : 1;
-+		u8 dirty : 1;
-+		u8 reserved: 6;
-+	};
-+	u8 as_uint8;
-+} __packed;
-+
-+enum hv_crashdump_action {
-+	HV_CRASHDUMP_NONE = 0,
-+	HV_CRASHDUMP_SUSPEND_ALL_VPS,
-+	HV_CRASHDUMP_PREPARE_FOR_STATE_SAVE,
-+	HV_CRASHDUMP_STATE_SAVED,
-+	HV_CRASHDUMP_ENTRY,
-+};
-+
-+struct hv_partition_event_root_crashdump_input {
-+	u32 crashdump_action; /* enum hv_crashdump_action */
-+} __packed;
-+
-+struct hv_input_disable_hyp_ex {   /* HV_X64_INPUT_DISABLE_HYPERVISOR_EX */
-+	u64 rip;
-+	u64 arg;
-+} __packed;
-+
-+struct hv_crashdump_area {	   /* HV_CRASHDUMP_AREA */
-+	u32 version;
-+	union {
-+		u32 flags_as_uint32;
-+		struct {
-+			u32 cda_valid : 1;
-+			u32 cda_unused : 31;
-+		} __packed;
-+	};
-+	/* more unused fields */
-+} __packed;
-+
-+union hv_partition_event_input {
-+	struct hv_partition_event_root_crashdump_input crashdump_input;
-+};
-+
-+enum hv_partition_event {
-+	HV_PARTITION_EVENT_ROOT_CRASHDUMP = 2,
-+};
-+
-+struct hv_input_notify_partition_event {
-+	u32 event;      /* enum hv_partition_event */
-+	union hv_partition_event_input input;
-+} __packed;
-+
-+struct hv_lp_startup_status {
-+	u64 hv_status;
-+	u64 substatus1;
-+	u64 substatus2;
-+	u64 substatus3;
-+	u64 substatus4;
-+	u64 substatus5;
-+	u64 substatus6;
-+} __packed;
-+
-+struct hv_input_add_logical_processor {
-+	u32 lp_index;
-+	u32 apic_id;
-+	struct hv_proximity_domain_info proximity_domain_info;
-+} __packed;
-+
-+struct hv_output_add_logical_processor {
-+	struct hv_lp_startup_status startup_status;
-+} __packed;
-+
-+enum {	/* HV_SUBNODE_TYPE */
-+	HV_SUBNODE_ANY		= 0,
-+	HV_SUBNODE_SOCKET,
-+	HV_SUBNODE_CLUSTER,
-+	HV_SUBNODE_L3,
-+	HV_SUBNODE_COUNT,
-+	HV_SUBNODE_INVALID	= -1
-+};
-+
-+struct hv_create_vp {	/* HV_INPUT_CREATE_VP */
-+	u64 partition_id;
-+	u32 vp_index;
-+	u8 padding[3];
-+	u8 subnode_type;
-+	u64 subnode_id;
-+	struct hv_proximity_domain_info proximity_domain_info;
-+	u64 flags;
-+} __packed;
-+
-+/* HV_INTERRUPT_TRIGGER_MODE */
-+enum hv_interrupt_trigger_mode {
-+	HV_INTERRUPT_TRIGGER_MODE_EDGE	= 0,
-+	HV_INTERRUPT_TRIGGER_MODE_LEVEL	= 1,
-+};
-+
-+/* HV_DEVICE_INTERRUPT_DESCRIPTOR */
-+struct hv_device_interrupt_descriptor {
-+	u32 interrupt_type;
-+	u32 trigger_mode;
-+	u32 vector_count;
-+	u32 reserved;
-+	struct hv_device_interrupt_target target;
-+} __packed;
-+
-+/* HV_INPUT_MAP_DEVICE_INTERRUPT */
-+struct hv_input_map_device_interrupt {
-+	u64 partition_id;
-+	u64 device_id;
-+	u32 flags;
-+	u32 base_irt_idx;
-+	struct hv_interrupt_entry logical_interrupt_entry;
-+	struct hv_device_interrupt_descriptor interrupt_descriptor;
-+} __packed;
-+
-+/* HV_OUTPUT_MAP_DEVICE_INTERRUPT */
-+struct hv_output_map_device_interrupt {
-+	struct hv_interrupt_entry interrupt_entry;
-+	u64 ext_status_deprecated[5];
-+} __packed;
-+
-+/* HV_INPUT_UNMAP_DEVICE_INTERRUPT */
-+struct hv_input_unmap_device_interrupt {
-+	u64 partition_id;
-+	u64 device_id;
-+	struct hv_interrupt_entry interrupt_entry;
-+	u32 flags;
-+} __packed;
-+
-+#define HV_SOURCE_SHADOW_NONE		    0x0
-+#define HV_SOURCE_SHADOW_BRIDGE_BUS_RANGE   0x1
-+
-+struct hv_send_ipi_ex { /* HV_INPUT_SEND_SYNTHETIC_CLUSTER_IPI_EX */
-+	u32 vector;
-+	u32 reserved;
-+	struct hv_vpset vp_set;
-+} __packed;
-+
-+typedef u16 hv_pci_rid;		/* HV_PCI_RID */
-+typedef u16 hv_pci_segment;	/* HV_PCI_SEGMENT */
-+typedef u64 hv_logical_device_id;
-+union hv_pci_bdf {	/* HV_PCI_BDF */
-+	u16 as_uint16;
-+
-+	struct {
-+		u8 function : 3;
-+		u8 device : 5;
-+		u8 bus;
-+	};
-+} __packed;
-+
-+union hv_pci_bus_range {
-+	u16 as_uint16;
-+
-+	struct {
-+		u8 subordinate_bus;
-+		u8 secondary_bus;
-+	};
-+} __packed;
-+
-+enum hv_device_type {		/* HV_DEVICE_TYPE */
-+	HV_DEVICE_TYPE_LOGICAL	= 0,
-+	HV_DEVICE_TYPE_PCI	= 1,
-+	HV_DEVICE_TYPE_IOAPIC	= 2,
-+	HV_DEVICE_TYPE_ACPI	= 3,
-+};
-+
-+union hv_device_id {		/* HV_DEVICE_ID */
-+	u64 as_uint64;
-+
-+	struct {
-+		u64 reserved0 : 62;
-+		u64 device_type : 2;
-+	};
-+
-+	/* HV_DEVICE_TYPE_LOGICAL */
-+	struct {
-+		u64 id : 62;
-+		u64 device_type : 2;
-+	} logical;
-+
-+	/* HV_DEVICE_TYPE_PCI */
-+	struct {
-+		union {
-+			hv_pci_rid rid;
-+			union hv_pci_bdf bdf;
-+		};
-+
-+		hv_pci_segment segment;
-+		union hv_pci_bus_range shadow_bus_range;
-+
-+		u16 phantom_function_bits : 2;
-+		u16 source_shadow : 1;
-+
-+		u16 rsvdz0 : 11;
-+		u16 device_type : 2;
-+	} pci;
-+
-+	/* HV_DEVICE_TYPE_IOAPIC */
-+	struct {
-+		u8 ioapic_id;
-+		u8 rsvdz0;
-+		u16 rsvdz1;
-+		u16 rsvdz2;
-+
-+		u16 rsvdz3 : 14;
-+		u16 device_type : 2;
-+	} ioapic;
-+
-+	/* HV_DEVICE_TYPE_ACPI */
-+	struct {
-+		u32 input_mapping_base;
-+		u32 input_mapping_count : 30;
-+		u32 device_type : 2;
-+	} acpi;
-+} __packed;
-+
-+#endif /* _HV_HVHDK_MINI_H */
-diff --git a/include/keys/asymmetric-parser.h b/include/keys/asymmetric-parser.h
-index c47dc5405f79..516a3f51179e 100644
--- a/include/keys/asymmetric-parser.h
-+++ b/include/keys/asymmetric-parser.h
-@@ -10,6 +10,8 @@
- #ifndef _KEYS_ASYMMETRIC_PARSER_H
- #define _KEYS_ASYMMETRIC_PARSER_H
- 
-+struct key_preparsed_payload;
-+
- /*
-  * Key data parser.  Called during key instantiation.
-  */
-diff --git a/include/keys/asymmetric-type.h b/include/keys/asymmetric-type.h
-index 69a13e1e5b2e..1b91c8f98688 100644
--- a/include/keys/asymmetric-type.h
-+++ b/include/keys/asymmetric-type.h
-@@ -49,7 +49,7 @@ enum asymmetric_payload_bits {
-  */
- struct asymmetric_key_id {
- 	unsigned short	len;
-	unsigned char	data[];
-+	unsigned char	data[] __counted_by(len);
- };
- 
- struct asymmetric_key_ids {
-diff --git a/include/keys/dns_resolver-type.h b/include/keys/dns_resolver-type.h
-index 218ca22fb056..1b89088a2837 100644
--- a/include/keys/dns_resolver-type.h
-+++ b/include/keys/dns_resolver-type.h
-@@ -12,8 +12,4 @@
- 
- extern struct key_type key_type_dns_resolver;
- 
-extern int request_dns_resolver_key(const char *description,
-				    const char *callout_info,
-				    char **data);
-
- #endif /* _KEYS_DNS_RESOLVER_TYPE_H */
-diff --git a/include/keys/rxrpc-type.h b/include/keys/rxrpc-type.h
-index 333c0f49a9cd..0ddbe197a261 100644
--- a/include/keys/rxrpc-type.h
-+++ b/include/keys/rxrpc-type.h
-@@ -9,6 +9,7 @@
- #define _KEYS_RXRPC_TYPE_H
- 
- #include <linux/key.h>
-+#include <crypto/krb5.h>
- 
- /*
-  * key type for AF_RXRPC keys
-@@ -32,6 +33,21 @@ struct rxkad_key {
- };
- 
- /*
-+ * RxRPC key for YFS-RxGK (type-6 security)
-+ */
-+struct rxgk_key {
-+	s64		begintime;	/* Time at which the ticket starts */
-+	s64		endtime;	/* Time at which the ticket ends */
-+	u64		lifetime;	/* Maximum lifespan of a connection (seconds) */
-+	u64		bytelife;	/* Maximum number of bytes on a connection */
-+	unsigned int	enctype;	/* Encoding type */
-+	s8		level;		/* Negotiated security RXRPC_SECURITY_PLAIN/AUTH/ENCRYPT */
-+	struct krb5_buffer key;		/* Master key, K0 */
-+	struct krb5_buffer ticket;	/* Ticket to be passed to server */
-+	u8		_key[];		/* Key storage */
-+};
-+
-+/*
-  * list of tokens attached to an rxrpc key
-  */
- struct rxrpc_key_token {
-@@ -40,6 +56,7 @@ struct rxrpc_key_token {
- 	struct rxrpc_key_token *next;	/* the next token in the list */
- 	union {
- 		struct rxkad_key *kad;
-+		struct rxgk_key *rxgk;
- 	};
- };
- 
-diff --git a/include/keys/system_keyring.h b/include/keys/system_keyring.h
-index 91e080efb918..a6c2897bcc63 100644
--- a/include/keys/system_keyring.h
-+++ b/include/keys/system_keyring.h
-@@ -23,10 +23,15 @@ extern int restrict_link_by_builtin_trusted(struct key *keyring,
- 					    const struct key_type *type,
- 					    const union key_payload *payload,
- 					    struct key *restriction_key);
-+int restrict_link_by_digsig_builtin(struct key *dest_keyring,
-+				    const struct key_type *type,
-+				    const union key_payload *payload,
-+				    struct key *restriction_key);
- extern __init int load_module_cert(struct key *keyring);
- 
- #else
- #define restrict_link_by_builtin_trusted restrict_link_reject
-+#define restrict_link_by_digsig_builtin restrict_link_reject
- 
- static inline __init int load_module_cert(struct key *keyring)
- {
-@@ -41,8 +46,17 @@ extern int restrict_link_by_builtin_and_secondary_trusted(
- 	const struct key_type *type,
- 	const union key_payload *payload,
- 	struct key *restriction_key);
-+int restrict_link_by_digsig_builtin_and_secondary(struct key *keyring,
-+						  const struct key_type *type,
-+						  const union key_payload *payload,
-+						  struct key *restriction_key);
-+void __init add_to_secondary_keyring(const char *source, const void *data, size_t len);
- #else
- #define restrict_link_by_builtin_and_secondary_trusted restrict_link_by_builtin_trusted
-+#define restrict_link_by_digsig_builtin_and_secondary restrict_link_by_digsig_builtin
-+static inline void __init add_to_secondary_keyring(const char *source, const void *data, size_t len)
-+{
-+}
- #endif
- 
- #ifdef CONFIG_INTEGRITY_MACHINE_KEYRING
-@@ -59,7 +73,6 @@ static inline void __init set_machine_trusted_keys(struct key *keyring)
- }
- #endif
- 
-extern struct pkcs7_message *pkcs7;
- #ifdef CONFIG_SYSTEM_BLACKLIST_KEYRING
- extern int mark_hash_blacklisted(const u8 *hash, size_t hash_len,
- 			       enum blacklist_hash_type hash_type);
-@@ -79,6 +92,7 @@ static inline int is_binary_blacklisted(const u8 *hash, size_t hash_len)
- }
- #endif
- 
-+struct pkcs7_message;
- #ifdef CONFIG_SYSTEM_REVOCATION_LIST
- extern int add_key_to_revocation_list(const char *data, size_t size);
- extern int is_key_on_revocation_list(struct pkcs7_message *pkcs7);
-diff --git a/include/keys/trusted_dcp.h b/include/keys/trusted_dcp.h
-new file mode 100644
-index 000000000000..9aaa42075b40
--- /dev/null
-+++ b/include/keys/trusted_dcp.h
-@@ -0,0 +1,11 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Copyright (C) 2021 sigma star gmbh
-+ */
-+
-+#ifndef TRUSTED_DCP_H
-+#define TRUSTED_DCP_H
-+
-+extern struct trusted_key_ops dcp_trusted_key_ops;
-+
-+#endif
-diff --git a/include/keys/trusted_tpm.h b/include/keys/trusted_tpm.h
-index 7769b726863a..0fadc6a4f166 100644
--- a/include/keys/trusted_tpm.h
-+++ b/include/keys/trusted_tpm.h
-@@ -5,43 +5,8 @@
- #include <keys/trusted-type.h>
- #include <linux/tpm_command.h>
- 
-/* implementation specific TPM constants */
-#define MAX_BUF_SIZE			1024
-#define TPM_GETRANDOM_SIZE		14
-#define TPM_SIZE_OFFSET			2
-#define TPM_RETURN_OFFSET		6
-#define TPM_DATA_OFFSET			10
-
-#define LOAD32(buffer, offset)	(ntohl(*(uint32_t *)&buffer[offset]))
-#define LOAD32N(buffer, offset)	(*(uint32_t *)&buffer[offset])
-#define LOAD16(buffer, offset)	(ntohs(*(uint16_t *)&buffer[offset]))
-
- extern struct trusted_key_ops trusted_key_tpm_ops;
- 
-struct osapsess {
-	uint32_t handle;
-	unsigned char secret[SHA1_DIGEST_SIZE];
-	unsigned char enonce[TPM_NONCE_SIZE];
-};
-
-/* discrete values, but have to store in uint16_t for TPM use */
-enum {
-	SEAL_keytype = 1,
-	SRK_keytype = 4
-};
-
-int TSS_authhmac(unsigned char *digest, const unsigned char *key,
-			unsigned int keylen, unsigned char *h1,
-			unsigned char *h2, unsigned int h3, ...);
-int TSS_checkhmac1(unsigned char *buffer,
-			  const uint32_t command,
-			  const unsigned char *ononce,
-			  const unsigned char *key,
-			  unsigned int keylen, ...);
-
-int trusted_tpm_send(unsigned char *cmd, size_t buflen);
-int oiap(struct tpm_buf *tb, uint32_t *handle, unsigned char *nonce);
-
- int tpm2_seal_trusted(struct tpm_chip *chip,
- 		      struct trusted_key_payload *payload,
- 		      struct trusted_key_options *options);
-@@ -49,50 +14,4 @@ int tpm2_unseal_trusted(struct tpm_chip *chip,
- 			struct trusted_key_payload *payload,
- 			struct trusted_key_options *options);
- 
-#define TPM_DEBUG 0
-
-#if TPM_DEBUG
-static inline void dump_options(struct trusted_key_options *o)
-{
-	pr_info("sealing key type %d\n", o->keytype);
-	pr_info("sealing key handle %0X\n", o->keyhandle);
-	pr_info("pcrlock %d\n", o->pcrlock);
-	pr_info("pcrinfo %d\n", o->pcrinfo_len);
-	print_hex_dump(KERN_INFO, "pcrinfo ", DUMP_PREFIX_NONE,
-		       16, 1, o->pcrinfo, o->pcrinfo_len, 0);
-}
-
-static inline void dump_sess(struct osapsess *s)
-{
-	print_hex_dump(KERN_INFO, "trusted-key: handle ", DUMP_PREFIX_NONE,
-		       16, 1, &s->handle, 4, 0);
-	pr_info("secret:\n");
-	print_hex_dump(KERN_INFO, "", DUMP_PREFIX_NONE,
-		       16, 1, &s->secret, SHA1_DIGEST_SIZE, 0);
-	pr_info("trusted-key: enonce:\n");
-	print_hex_dump(KERN_INFO, "", DUMP_PREFIX_NONE,
-		       16, 1, &s->enonce, SHA1_DIGEST_SIZE, 0);
-}
-
-static inline void dump_tpm_buf(unsigned char *buf)
-{
-	int len;
-
-	pr_info("\ntpm buffer\n");
-	len = LOAD32(buf, TPM_SIZE_OFFSET);
-	print_hex_dump(KERN_INFO, "", DUMP_PREFIX_NONE, 16, 1, buf, len, 0);
-}
-#else
-static inline void dump_options(struct trusted_key_options *o)
-{
-}
-
-static inline void dump_sess(struct osapsess *s)
-{
-}
-
-static inline void dump_tpm_buf(unsigned char *buf)
-{
-}
-#endif
- #endif
-diff --git a/include/kunit/assert.h b/include/kunit/assert.h
-index 4b52e12c2ae8..bb879389f11d 100644
--- a/include/kunit/assert.h
-+++ b/include/kunit/assert.h
-@@ -42,16 +42,15 @@ struct kunit_loc {
- 
- /**
-  * struct kunit_assert - Data for printing a failed assertion or expectation.
- * @format: a function which formats the data in this kunit_assert to a string.
-  *
-  * Represents a failed expectation/assertion. Contains all the data necessary to
-  * format a string to a user reporting the failure.
-  */
-struct kunit_assert {
-	void (*format)(const struct kunit_assert *assert,
-		       const struct va_format *message,
-		       struct string_stream *stream);
-};
-+struct kunit_assert {};
-+
-+typedef void (*assert_format_t)(const struct kunit_assert *assert,
-+				const struct va_format *message,
-+				struct string_stream *stream);
- 
- void kunit_assert_prologue(const struct kunit_loc *loc,
- 			   enum kunit_assert_type type,
-@@ -61,7 +60,7 @@ void kunit_assert_prologue(const struct kunit_loc *loc,
-  * struct kunit_fail_assert - Represents a plain fail expectation/assertion.
-  * @assert: The parent of this type.
-  *
- * Represents a simple KUNIT_FAIL/KUNIT_ASSERT_FAILURE that always fails.
-+ * Represents a simple KUNIT_FAIL/KUNIT_FAIL_AND_ABORT that always fails.
-  */
- struct kunit_fail_assert {
- 	struct kunit_assert assert;
-@@ -72,16 +71,6 @@ void kunit_fail_assert_format(const struct kunit_assert *assert,
- 			      struct string_stream *stream);
- 
- /**
- * KUNIT_INIT_FAIL_ASSERT_STRUCT - Initializer for &struct kunit_fail_assert.
- *
- * Initializes a &struct kunit_fail_assert. Intended to be used in
- * KUNIT_EXPECT_* and KUNIT_ASSERT_* macros.
- */
-#define KUNIT_INIT_FAIL_ASSERT_STRUCT {					\
-	.assert = { .format = kunit_fail_assert_format },		\
-}
-
-/**
-  * struct kunit_unary_assert - Represents a KUNIT_{EXPECT|ASSERT}_{TRUE|FALSE}
-  * @assert: The parent of this type.
-  * @condition: A string representation of a conditional expression.
-@@ -102,20 +91,6 @@ void kunit_unary_assert_format(const struct kunit_assert *assert,
- 			       struct string_stream *stream);
- 
- /**
- * KUNIT_INIT_UNARY_ASSERT_STRUCT() - Initializes &struct kunit_unary_assert.
- * @cond: A string representation of the expression asserted true or false.
- * @expect_true: True if of type KUNIT_{EXPECT|ASSERT}_TRUE, false otherwise.
- *
- * Initializes a &struct kunit_unary_assert. Intended to be used in
- * KUNIT_EXPECT_* and KUNIT_ASSERT_* macros.
- */
-#define KUNIT_INIT_UNARY_ASSERT_STRUCT(cond, expect_true) {		       \
-	.assert = { .format = kunit_unary_assert_format },		       \
-	.condition = cond,						       \
-	.expected_true = expect_true					       \
-}
-
-/**
-  * struct kunit_ptr_not_err_assert - An expectation/assertion that a pointer is
-  *	not NULL and not a -errno.
-  * @assert: The parent of this type.
-@@ -136,21 +111,6 @@ void kunit_ptr_not_err_assert_format(const struct kunit_assert *assert,
- 				     struct string_stream *stream);
- 
- /**
- * KUNIT_INIT_PTR_NOT_ERR_ASSERT_STRUCT() - Initializes a
- *	&struct kunit_ptr_not_err_assert.
- * @txt: A string representation of the expression passed to the expectation.
- * @val: The actual evaluated pointer value of the expression.
- *
- * Initializes a &struct kunit_ptr_not_err_assert. Intended to be used in
- * KUNIT_EXPECT_* and KUNIT_ASSERT_* macros.
- */
-#define KUNIT_INIT_PTR_NOT_ERR_STRUCT(txt, val) {			       \
-	.assert = { .format = kunit_ptr_not_err_assert_format },	       \
-	.text = txt,							       \
-	.value = val							       \
-}
-
-/**
-  * struct kunit_binary_assert_text - holds strings for &struct
-  *	kunit_binary_assert and friends to try and make the structs smaller.
-  * @operation: A string representation of the comparison operator (e.g. "==").
-@@ -187,30 +147,6 @@ void kunit_binary_assert_format(const struct kunit_assert *assert,
- 				struct string_stream *stream);
- 
- /**
- * KUNIT_INIT_BINARY_ASSERT_STRUCT() - Initializes a binary assert like
- *	kunit_binary_assert, kunit_binary_ptr_assert, etc.
- *
- * @format_func: a function which formats the assert to a string.
- * @text_: Pointer to a kunit_binary_assert_text.
- * @left_val: The actual evaluated value of the expression in the left slot.
- * @right_val: The actual evaluated value of the expression in the right slot.
- *
- * Initializes a binary assert like kunit_binary_assert,
- * kunit_binary_ptr_assert, etc. This relies on these structs having the same
- * fields but with different types for left_val/right_val.
- * This is ultimately used by binary assertion macros like KUNIT_EXPECT_EQ, etc.
- */
-#define KUNIT_INIT_BINARY_ASSERT_STRUCT(format_func,			       \
-					text_,				       \
-					left_val,			       \
-					right_val) {			       \
-	.assert = { .format = format_func },				       \
-	.text = text_,							       \
-	.left_value = left_val,						       \
-	.right_value = right_val					       \
-}
-
-/**
-  * struct kunit_binary_ptr_assert - An expectation/assertion that compares two
-  *	pointer values (for example, KUNIT_EXPECT_PTR_EQ(test, foo, bar)).
-  * @assert: The parent of this type.
-@@ -256,4 +192,41 @@ void kunit_binary_str_assert_format(const struct kunit_assert *assert,
- 				    const struct va_format *message,
- 				    struct string_stream *stream);
- 
-+/**
-+ * struct kunit_mem_assert - An expectation/assertion that compares two
-+ *	memory blocks.
-+ * @assert: The parent of this type.
-+ * @text: Holds the textual representations of the operands and comparator.
-+ * @left_value: The actual evaluated value of the expression in the left slot.
-+ * @right_value: The actual evaluated value of the expression in the right slot.
-+ * @size: Size of the memory block analysed in bytes.
-+ *
-+ * Represents an expectation/assertion that compares two memory blocks. For
-+ * example, to expect that the first three bytes of foo is equal to the
-+ * first three bytes of bar, you can use the expectation
-+ * KUNIT_EXPECT_MEMEQ(test, foo, bar, 3);
-+ */
-+struct kunit_mem_assert {
-+	struct kunit_assert assert;
-+	const struct kunit_binary_assert_text *text;
-+	const void *left_value;
-+	const void *right_value;
-+	const size_t size;
-+};
-+
-+void kunit_mem_assert_format(const struct kunit_assert *assert,
-+			     const struct va_format *message,
-+			     struct string_stream *stream);
-+
-+#if IS_ENABLED(CONFIG_KUNIT)
-+void kunit_assert_print_msg(const struct va_format *message,
-+			    struct string_stream *stream);
-+bool is_literal(const char *text, long long value);
-+bool is_str_literal(const char *text, const char *value);
-+void kunit_assert_hexdump(struct string_stream *stream,
-+			  const void *buf,
-+			  const void *compared_buf,
-+			  const size_t len);
-+#endif
-+
- #endif /*  _KUNIT_ASSERT_H */
-diff --git a/include/kunit/attributes.h b/include/kunit/attributes.h
-new file mode 100644
-index 000000000000..bc76a0b786d2
--- /dev/null
-+++ b/include/kunit/attributes.h
-@@ -0,0 +1,50 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * KUnit API to save and access test attributes
-+ *
-+ * Copyright (C) 2023, Google LLC.
-+ * Author: Rae Moar <rmoar@google.com>
-+ */
-+
-+#ifndef _KUNIT_ATTRIBUTES_H
-+#define _KUNIT_ATTRIBUTES_H
-+
-+/*
-+ * struct kunit_attr_filter - representation of attributes filter with the
-+ * attribute object and string input
-+ */
-+struct kunit_attr_filter {
-+	struct kunit_attr *attr;
-+	char *input;
-+};
-+
-+/*
-+ * Returns the name of the filter's attribute.
-+ */
-+const char *kunit_attr_filter_name(struct kunit_attr_filter filter);
-+
-+/*
-+ * Print all test attributes for a test case or suite.
-+ * Output format for test cases: "# <test_name>.<attribute>: <value>"
-+ * Output format for test suites: "# <attribute>: <value>"
-+ */
-+void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level);
-+
-+/*
-+ * Returns the number of fitlers in input.
-+ */
-+int kunit_get_filter_count(char *input);
-+
-+/*
-+ * Parse attributes filter input and return an objects containing the
-+ * attribute object and the string input of the next filter.
-+ */
-+struct kunit_attr_filter kunit_next_attr_filter(char **filters, int *err);
-+
-+/*
-+ * Returns a copy of the suite containing only tests that pass the filter.
-+ */
-+struct kunit_suite *kunit_filter_attr_tests(const struct kunit_suite *const suite,
-+		struct kunit_attr_filter filter, char *action, int *err);
-+
-+#endif /* _KUNIT_ATTRIBUTES_H */
-diff --git a/include/kunit/clk.h b/include/kunit/clk.h
-new file mode 100644
-index 000000000000..f226044cc78d
--- /dev/null
-+++ b/include/kunit/clk.h
-@@ -0,0 +1,33 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _CLK_KUNIT_H
-+#define _CLK_KUNIT_H
-+
-+struct clk;
-+struct clk_hw;
-+struct device;
-+struct device_node;
-+struct of_phandle_args;
-+struct kunit;
-+
-+struct clk *
-+clk_get_kunit(struct kunit *test, struct device *dev, const char *con_id);
-+struct clk *
-+of_clk_get_kunit(struct kunit *test, struct device_node *np, int index);
-+
-+struct clk *
-+clk_hw_get_clk_kunit(struct kunit *test, struct clk_hw *hw, const char *con_id);
-+struct clk *
-+clk_hw_get_clk_prepared_enabled_kunit(struct kunit *test, struct clk_hw *hw,
-+				      const char *con_id);
-+
-+int clk_prepare_enable_kunit(struct kunit *test, struct clk *clk);
-+
-+int clk_hw_register_kunit(struct kunit *test, struct device *dev, struct clk_hw *hw);
-+int of_clk_hw_register_kunit(struct kunit *test, struct device_node *node,
-+			     struct clk_hw *hw);
-+
-+int of_clk_add_hw_provider_kunit(struct kunit *test, struct device_node *np,
-+				 struct clk_hw *(*get)(struct of_phandle_args *clkspec, void *data),
-+				 void *data);
-+
-+#endif
-diff --git a/include/kunit/device.h b/include/kunit/device.h
-new file mode 100644
-index 000000000000..2450110ad64e
--- /dev/null
-+++ b/include/kunit/device.h
-@@ -0,0 +1,80 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * KUnit basic device implementation
-+ *
-+ * Helpers for creating and managing fake devices for KUnit tests.
-+ *
-+ * Copyright (C) 2023, Google LLC.
-+ * Author: David Gow <davidgow@google.com>
-+ */
-+
-+#ifndef _KUNIT_DEVICE_H
-+#define _KUNIT_DEVICE_H
-+
-+#if IS_ENABLED(CONFIG_KUNIT)
-+
-+#include <kunit/test.h>
-+
-+struct device;
-+struct device_driver;
-+
-+/**
-+ * kunit_driver_create() - Create a struct device_driver attached to the kunit_bus
-+ * @test: The test context object.
-+ * @name: The name to give the created driver.
-+ *
-+ * Creates a struct device_driver attached to the kunit_bus, with the name @name.
-+ * This driver will automatically be cleaned up on test exit.
-+ *
-+ * Return: a stub struct device_driver, managed by KUnit, with the name @name.
-+ */
-+struct device_driver *kunit_driver_create(struct kunit *test, const char *name);
-+
-+/**
-+ * kunit_device_register() - Create a struct device for use in KUnit tests
-+ * @test: The test context object.
-+ * @name: The name to give the created device.
-+ *
-+ * Creates a struct kunit_device (which is a struct device) with the given name,
-+ * and a corresponding driver. The device and driver will be cleaned up on test
-+ * exit, or when kunit_device_unregister is called. See also
-+ * kunit_device_register_with_driver, if you wish to provide your own
-+ * struct device_driver.
-+ *
-+ * Return: a pointer to a struct device which will be cleaned up when the test
-+ * exits, or an error pointer if the device could not be allocated or registered.
-+ */
-+struct device *kunit_device_register(struct kunit *test, const char *name);
-+
-+/**
-+ * kunit_device_register_with_driver() - Create a struct device for use in KUnit tests
-+ * @test: The test context object.
-+ * @name: The name to give the created device.
-+ * @drv: The struct device_driver to associate with the device.
-+ *
-+ * Creates a struct kunit_device (which is a struct device) with the given
-+ * name, and driver. The device will be cleaned up on test exit, or when
-+ * kunit_device_unregister is called. See also kunit_device_register, if you
-+ * wish KUnit to create and manage a driver for you.
-+ *
-+ * Return: a pointer to a struct device which will be cleaned up when the test
-+ * exits, or an error pointer if the device could not be allocated or registered.
-+ */
-+struct device *kunit_device_register_with_driver(struct kunit *test,
-+						 const char *name,
-+						 const struct device_driver *drv);
-+
-+/**
-+ * kunit_device_unregister() - Unregister a KUnit-managed device
-+ * @test: The test context object which created the device
-+ * @dev: The device.
-+ *
-+ * Unregisters and destroys a struct device which was created with
-+ * kunit_device_register or kunit_device_register_with_driver. If KUnit created
-+ * a driver, cleans it up as well.
-+ */
-+void kunit_device_unregister(struct kunit *test, struct device *dev);
-+
-+#endif
-+
-+#endif
-diff --git a/include/kunit/of.h b/include/kunit/of.h
-new file mode 100644
-index 000000000000..75a760a4e2a5
--- /dev/null
-+++ b/include/kunit/of.h
-@@ -0,0 +1,121 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _KUNIT_OF_H
-+#define _KUNIT_OF_H
-+
-+#include <kunit/test.h>
-+
-+struct device_node;
-+
-+#ifdef CONFIG_OF
-+
-+void of_node_put_kunit(struct kunit *test, struct device_node *node);
-+
-+#else
-+
-+static inline
-+void of_node_put_kunit(struct kunit *test, struct device_node *node)
-+{
-+	kunit_skip(test, "requires CONFIG_OF");
-+}
-+
-+#endif /* !CONFIG_OF */
-+
-+#if defined(CONFIG_OF) && defined(CONFIG_OF_OVERLAY) && defined(CONFIG_OF_EARLY_FLATTREE)
-+
-+int of_overlay_fdt_apply_kunit(struct kunit *test, void *overlay_fdt,
-+			       u32 overlay_fdt_size, int *ovcs_id);
-+#else
-+
-+static inline int
-+of_overlay_fdt_apply_kunit(struct kunit *test, void *overlay_fdt,
-+			   u32 overlay_fdt_size, int *ovcs_id)
-+{
-+	kunit_skip(test, "requires CONFIG_OF and CONFIG_OF_OVERLAY and CONFIG_OF_EARLY_FLATTREE for root node");
-+	return -EINVAL;
-+}
-+
-+#endif
-+
-+/**
-+ * __of_overlay_apply_kunit() - Test managed of_overlay_fdt_apply() variant
-+ * @test: test context
-+ * @overlay_begin: start address of overlay to apply
-+ * @overlay_end: end address of overlay to apply
-+ *
-+ * This is mostly internal API. See of_overlay_apply_kunit() for the wrapper
-+ * that makes this easier to use.
-+ *
-+ * Similar to of_overlay_fdt_apply(), except the overlay is managed by the test
-+ * case and is automatically removed with of_overlay_remove() after the test
-+ * case concludes.
-+ *
-+ * Return: 0 on success, negative errno on failure
-+ */
-+static inline int __of_overlay_apply_kunit(struct kunit *test,
-+					   u8 *overlay_begin,
-+					   const u8 *overlay_end)
-+{
-+	int unused;
-+
-+	return of_overlay_fdt_apply_kunit(test, overlay_begin,
-+					  overlay_end - overlay_begin,
-+					  &unused);
-+}
-+
-+#define of_overlay_begin(overlay_name) __dtbo_##overlay_name##_begin
-+#define of_overlay_end(overlay_name) __dtbo_##overlay_name##_end
-+
-+#define OF_OVERLAY_DECLARE(overlay_name)			\
-+	extern uint8_t of_overlay_begin(overlay_name)[];	\
-+	extern uint8_t of_overlay_end(overlay_name)[]		\
-+
-+/**
-+ * of_overlay_apply_kunit() - Test managed of_overlay_fdt_apply() for built-in overlays
-+ * @test: test context
-+ * @overlay_name: name of overlay to apply
-+ *
-+ * This macro is used to apply a device tree overlay built with the
-+ * cmd_dt_S_dtbo rule in scripts/Makefile.lib that has been compiled into the
-+ * kernel image or KUnit test module. The overlay is automatically removed when
-+ * the test is finished.
-+ *
-+ * Unit tests that need device tree nodes should compile an overlay file with
-+ * @overlay_name\.dtbo.o in their Makefile along with their unit test and then
-+ * load the overlay during their test. The @overlay_name matches the filename
-+ * of the overlay without the dtbo filename extension. If CONFIG_OF_OVERLAY is
-+ * not enabled, the @test will be skipped.
-+ *
-+ * In the Makefile
-+ *
-+ * .. code-block:: none
-+ *
-+ *	obj-$(CONFIG_OF_OVERLAY_KUNIT_TEST) += overlay_test.o kunit_overlay_test.dtbo.o
-+ *
-+ * In the test
-+ *
-+ * .. code-block:: c
-+ *
-+ *	static void of_overlay_kunit_of_overlay_apply(struct kunit *test)
-+ *	{
-+ *		struct device_node *np;
-+ *
-+ *		KUNIT_ASSERT_EQ(test, 0,
-+ *				of_overlay_apply_kunit(test, kunit_overlay_test));
-+ *
-+ *		np = of_find_node_by_name(NULL, "test-kunit");
-+ *		KUNIT_EXPECT_NOT_ERR_OR_NULL(test, np);
-+ *		of_node_put(np);
-+ *	}
-+ *
-+ * Return: 0 on success, negative errno on failure.
-+ */
-+#define of_overlay_apply_kunit(test, overlay_name)		\
-+({								\
-+	OF_OVERLAY_DECLARE(overlay_name);			\
-+								\
-+	__of_overlay_apply_kunit((test),			\
-+			of_overlay_begin(overlay_name),		\
-+			of_overlay_end(overlay_name));		\
-+})
-+
-+#endif
-diff --git a/include/kunit/platform_device.h b/include/kunit/platform_device.h
-new file mode 100644
-index 000000000000..f8236a8536f7
--- /dev/null
-+++ b/include/kunit/platform_device.h
-@@ -0,0 +1,21 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _KUNIT_PLATFORM_DRIVER_H
-+#define _KUNIT_PLATFORM_DRIVER_H
-+
-+struct completion;
-+struct kunit;
-+struct platform_device;
-+struct platform_driver;
-+
-+struct platform_device *
-+kunit_platform_device_alloc(struct kunit *test, const char *name, int id);
-+int kunit_platform_device_add(struct kunit *test, struct platform_device *pdev);
-+
-+int kunit_platform_device_prepare_wait_for_probe(struct kunit *test,
-+						 struct platform_device *pdev,
-+						 struct completion *x);
-+
-+int kunit_platform_driver_register(struct kunit *test,
-+				   struct platform_driver *drv);
-+
-+#endif
-diff --git a/include/kunit/resource.h b/include/kunit/resource.h
-index 09c2b34d1c61..4ad69a2642a5 100644
--- a/include/kunit/resource.h
-+++ b/include/kunit/resource.h
-@@ -72,7 +72,7 @@ typedef void (*kunit_resource_free_t)(struct kunit_resource *);
-  *		params.gfp = gfp;
-  *
-  *		return kunit_alloc_resource(test, kunit_kmalloc_init,
- *			kunit_kmalloc_free, &params);
-+ *			kunit_kmalloc_free, gfp, &params);
-  *	}
-  *
-  * Resources can also be named, with lookup/removal done on a name
-@@ -301,22 +301,6 @@ typedef bool (*kunit_resource_match_t)(struct kunit *test,
- 				       void *match_data);
- 
- /**
- * kunit_resource_instance_match() - Match a resource with the same instance.
- * @test: Test case to which the resource belongs.
- * @res: The resource.
- * @match_data: The resource pointer to match against.
- *
- * An instance of kunit_resource_match_t that matches a resource whose
- * allocation matches @match_data.
- */
-static inline bool kunit_resource_instance_match(struct kunit *test,
-						 struct kunit_resource *res,
-						 void *match_data)
-{
-	return res->data == match_data;
-}
-
-/**
-  * kunit_resource_name_match() - Match a resource with the same name.
-  * @test: Test case to which the resource belongs.
-  * @res: The resource.
-@@ -403,4 +387,117 @@ static inline int kunit_destroy_named_resource(struct kunit *test,
-  */
- void kunit_remove_resource(struct kunit *test, struct kunit_resource *res);
- 
-+/* A 'deferred action' function to be used with kunit_add_action. */
-+typedef void (kunit_action_t)(void *);
-+
-+/**
-+ * KUNIT_DEFINE_ACTION_WRAPPER() - Wrap a function for use as a deferred action.
-+ *
-+ * @wrapper: The name of the new wrapper function define.
-+ * @orig: The original function to wrap.
-+ * @arg_type: The type of the argument accepted by @orig.
-+ *
-+ * Defines a wrapper for a function which accepts a single, pointer-sized
-+ * argument. This wrapper can then be passed to kunit_add_action() and
-+ * similar. This should be used in preference to casting a function
-+ * directly to kunit_action_t, as casting function pointers will break
-+ * control flow integrity (CFI), leading to crashes.
-+ */
-+#define KUNIT_DEFINE_ACTION_WRAPPER(wrapper, orig, arg_type)	\
-+	static void wrapper(void *in)				\
-+	{							\
-+		arg_type arg = (arg_type)in;			\
-+		orig(arg);					\
-+	}
-+
-+
-+/**
-+ * kunit_add_action() - Call a function when the test ends.
-+ * @test: Test case to associate the action with.
-+ * @action: The function to run on test exit
-+ * @ctx: Data passed into @func
-+ *
-+ * Defer the execution of a function until the test exits, either normally or
-+ * due to a failure.  @ctx is passed as additional context. All functions
-+ * registered with kunit_add_action() will execute in the opposite order to that
-+ * they were registered in.
-+ *
-+ * This is useful for cleaning up allocated memory and resources, as these
-+ * functions are called even if the test aborts early due to, e.g., a failed
-+ * assertion.
-+ *
-+ * See also: devm_add_action() for the devres equivalent.
-+ *
-+ * Returns:
-+ *   0 on success, an error if the action could not be deferred.
-+ */
-+int kunit_add_action(struct kunit *test, kunit_action_t *action, void *ctx);
-+
-+/**
-+ * kunit_add_action_or_reset() - Call a function when the test ends.
-+ * @test: Test case to associate the action with.
-+ * @action: The function to run on test exit
-+ * @ctx: Data passed into @func
-+ *
-+ * Defer the execution of a function until the test exits, either normally or
-+ * due to a failure.  @ctx is passed as additional context. All functions
-+ * registered with kunit_add_action() will execute in the opposite order to that
-+ * they were registered in.
-+ *
-+ * This is useful for cleaning up allocated memory and resources, as these
-+ * functions are called even if the test aborts early due to, e.g., a failed
-+ * assertion.
-+ *
-+ * If the action cannot be created (e.g., due to the system being out of memory),
-+ * then action(ctx) will be called immediately, and an error will be returned.
-+ *
-+ * See also: devm_add_action_or_reset() for the devres equivalent.
-+ *
-+ * Returns:
-+ *   0 on success, an error if the action could not be deferred.
-+ */
-+int kunit_add_action_or_reset(struct kunit *test, kunit_action_t *action,
-+			      void *ctx);
-+
-+/**
-+ * kunit_remove_action() - Cancel a matching deferred action.
-+ * @test: Test case the action is associated with.
-+ * @action: The deferred function to cancel.
-+ * @ctx: The context passed to the deferred function to trigger.
-+ *
-+ * Prevent an action deferred via kunit_add_action() from executing when the
-+ * test terminates.
-+ *
-+ * If the function/context pair was deferred multiple times, only the most
-+ * recent one will be cancelled.
-+ *
-+ * See also: devm_remove_action() for the devres equivalent.
-+ */
-+void kunit_remove_action(struct kunit *test,
-+			 kunit_action_t *action,
-+			 void *ctx);
-+
-+/**
-+ * kunit_release_action() - Run a matching action call immediately.
-+ * @test: Test case the action is associated with.
-+ * @action: The deferred function to trigger.
-+ * @ctx: The context passed to the deferred function to trigger.
-+ *
-+ * Execute a function deferred via kunit_add_action()) immediately, rather than
-+ * when the test ends.
-+ *
-+ * If the function/context pair was deferred multiple times, it will only be
-+ * executed once here. The most recent deferral will no longer execute when
-+ * the test ends.
-+ *
-+ * kunit_release_action(test, func, ctx);
-+ * is equivalent to
-+ * func(ctx);
-+ * kunit_remove_action(test, func, ctx);
-+ *
-+ * See also: devm_release_action() for the devres equivalent.
-+ */
-+void kunit_release_action(struct kunit *test,
-+			  kunit_action_t *action,
-+			  void *ctx);
- #endif /* _KUNIT_RESOURCE_H */
-diff --git a/include/kunit/run-in-irq-context.h b/include/kunit/run-in-irq-context.h
-new file mode 100644
-index 000000000000..108e96433ea4
--- /dev/null
-+++ b/include/kunit/run-in-irq-context.h
-@@ -0,0 +1,129 @@
-+/* SPDX-License-Identifier: GPL-2.0-or-later */
-+/*
-+ * Helper function for testing code in interrupt contexts
-+ *
-+ * Copyright 2025 Google LLC
-+ */
-+#ifndef _KUNIT_RUN_IN_IRQ_CONTEXT_H
-+#define _KUNIT_RUN_IN_IRQ_CONTEXT_H
-+
-+#include <kunit/test.h>
-+#include <linux/timekeeping.h>
-+#include <linux/hrtimer.h>
-+#include <linux/workqueue.h>
-+
-+#define KUNIT_IRQ_TEST_HRTIMER_INTERVAL us_to_ktime(5)
-+
-+struct kunit_irq_test_state {
-+	bool (*func)(void *test_specific_state);
-+	void *test_specific_state;
-+	bool task_func_reported_failure;
-+	bool hardirq_func_reported_failure;
-+	bool softirq_func_reported_failure;
-+	unsigned long hardirq_func_calls;
-+	unsigned long softirq_func_calls;
-+	struct hrtimer timer;
-+	struct work_struct bh_work;
-+};
-+
-+static enum hrtimer_restart kunit_irq_test_timer_func(struct hrtimer *timer)
-+{
-+	struct kunit_irq_test_state *state =
-+		container_of(timer, typeof(*state), timer);
-+
-+	WARN_ON_ONCE(!in_hardirq());
-+	state->hardirq_func_calls++;
-+
-+	if (!state->func(state->test_specific_state))
-+		state->hardirq_func_reported_failure = true;
-+
-+	hrtimer_forward_now(&state->timer, KUNIT_IRQ_TEST_HRTIMER_INTERVAL);
-+	queue_work(system_bh_wq, &state->bh_work);
-+	return HRTIMER_RESTART;
-+}
-+
-+static void kunit_irq_test_bh_work_func(struct work_struct *work)
-+{
-+	struct kunit_irq_test_state *state =
-+		container_of(work, typeof(*state), bh_work);
-+
-+	WARN_ON_ONCE(!in_serving_softirq());
-+	state->softirq_func_calls++;
-+
-+	if (!state->func(state->test_specific_state))
-+		state->softirq_func_reported_failure = true;
-+}
-+
-+/*
-+ * Helper function which repeatedly runs the given @func in task, softirq, and
-+ * hardirq context concurrently, and reports a failure to KUnit if any
-+ * invocation of @func in any context returns false.  @func is passed
-+ * @test_specific_state as its argument.  At most 3 invocations of @func will
-+ * run concurrently: one in each of task, softirq, and hardirq context.
-+ *
-+ * The main purpose of this interrupt context testing is to validate fallback
-+ * code paths that run in contexts where the normal code path cannot be used,
-+ * typically due to the FPU or vector registers already being in-use in kernel
-+ * mode.  These code paths aren't covered when the test code is executed only by
-+ * the KUnit test runner thread in task context.  The reason for the concurrency
-+ * is because merely using hardirq context is not sufficient to reach a fallback
-+ * code path on some architectures; the hardirq actually has to occur while the
-+ * FPU or vector unit was already in-use in kernel mode.
-+ *
-+ * Another purpose of this testing is to detect issues with the architecture's
-+ * irq_fpu_usable() and kernel_fpu_begin/end() or equivalent functions,
-+ * especially in softirq context when the softirq may have interrupted a task
-+ * already using kernel-mode FPU or vector (if the arch didn't prevent that).
-+ * Crypto functions are often executed in softirqs, so this is important.
-+ */
-+static inline void kunit_run_irq_test(struct kunit *test, bool (*func)(void *),
-+				      int max_iterations,
-+				      void *test_specific_state)
-+{
-+	struct kunit_irq_test_state state = {
-+		.func = func,
-+		.test_specific_state = test_specific_state,
-+	};
-+	unsigned long end_jiffies;
-+
-+	/*
-+	 * Set up a hrtimer (the way we access hardirq context) and a work
-+	 * struct for the BH workqueue (the way we access softirq context).
-+	 */
-+	hrtimer_setup_on_stack(&state.timer, kunit_irq_test_timer_func,
-+			       CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
-+	INIT_WORK_ONSTACK(&state.bh_work, kunit_irq_test_bh_work_func);
-+
-+	/* Run for up to max_iterations or 1 second, whichever comes first. */
-+	end_jiffies = jiffies + HZ;
-+	hrtimer_start(&state.timer, KUNIT_IRQ_TEST_HRTIMER_INTERVAL,
-+		      HRTIMER_MODE_REL_HARD);
-+	for (int i = 0; i < max_iterations && !time_after(jiffies, end_jiffies);
-+	     i++) {
-+		if (!func(test_specific_state))
-+			state.task_func_reported_failure = true;
-+	}
-+
-+	/* Cancel the timer and work. */
-+	hrtimer_cancel(&state.timer);
-+	flush_work(&state.bh_work);
-+
-+	/* Sanity check: the timer and BH functions should have been run. */
-+	KUNIT_EXPECT_GT_MSG(test, state.hardirq_func_calls, 0,
-+			    "Timer function was not called");
-+	KUNIT_EXPECT_GT_MSG(test, state.softirq_func_calls, 0,
-+			    "BH work function was not called");
-+
-+	/* Check for incorrect hash values reported from any context. */
-+	KUNIT_EXPECT_FALSE_MSG(
-+		test, state.task_func_reported_failure,
-+		"Incorrect hash values reported from task context");
-+	KUNIT_EXPECT_FALSE_MSG(
-+		test, state.hardirq_func_reported_failure,
-+		"Incorrect hash values reported from hardirq context");
-+	KUNIT_EXPECT_FALSE_MSG(
-+		test, state.softirq_func_reported_failure,
-+		"Incorrect hash values reported from softirq context");
-+}
-+
-+#endif /* _KUNIT_RUN_IN_IRQ_CONTEXT_H */
-diff --git a/include/kunit/skbuff.h b/include/kunit/skbuff.h
-new file mode 100644
-index 000000000000..07784694357c
--- /dev/null
-+++ b/include/kunit/skbuff.h
-@@ -0,0 +1,57 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * KUnit resource management helpers for SKBs (skbuff).
-+ *
-+ * Copyright (C) 2023 Intel Corporation
-+ */
-+
-+#ifndef _KUNIT_SKBUFF_H
-+#define _KUNIT_SKBUFF_H
-+
-+#include <kunit/resource.h>
-+#include <linux/skbuff.h>
-+
-+static void kunit_action_kfree_skb(void *p)
-+{
-+	kfree_skb((struct sk_buff *)p);
-+}
-+
-+/**
-+ * kunit_zalloc_skb() - Allocate and initialize a resource managed skb.
-+ * @test: The test case to which the skb belongs
-+ * @len: size to allocate
-+ * @gfp: allocation flags
-+ *
-+ * Allocate a new struct sk_buff with gfp flags, zero fill the given length
-+ * and add it as a resource to the kunit test for automatic cleanup.
-+ *
-+ * Returns: newly allocated SKB, or %NULL on error
-+ */
-+static inline struct sk_buff *kunit_zalloc_skb(struct kunit *test, int len,
-+					       gfp_t gfp)
-+{
-+	struct sk_buff *res = alloc_skb(len, gfp);
-+
-+	if (!res || skb_pad(res, len))
-+		return NULL;
-+
-+	if (kunit_add_action_or_reset(test, kunit_action_kfree_skb, res))
-+		return NULL;
-+
-+	return res;
-+}
-+
-+/**
-+ * kunit_kfree_skb() - Like kfree_skb except for allocations managed by KUnit.
-+ * @test: The test case to which the resource belongs.
-+ * @skb: The SKB to free.
-+ */
-+static inline void kunit_kfree_skb(struct kunit *test, struct sk_buff *skb)
-+{
-+	if (!skb)
-+		return;
-+
-+	kunit_release_action(test, kunit_action_kfree_skb, (void *)skb);
-+}
-+
-+#endif /* _KUNIT_SKBUFF_H */
-diff --git a/include/kunit/static_stub.h b/include/kunit/static_stub.h
-new file mode 100644
-index 000000000000..bf940322dfc0
--- /dev/null
-+++ b/include/kunit/static_stub.h
-@@ -0,0 +1,113 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * KUnit function redirection (static stubbing) API.
-+ *
-+ * Copyright (C) 2022, Google LLC.
-+ * Author: David Gow <davidgow@google.com>
-+ */
-+#ifndef _KUNIT_STATIC_STUB_H
-+#define _KUNIT_STATIC_STUB_H
-+
-+#if !IS_ENABLED(CONFIG_KUNIT)
-+
-+/* If CONFIG_KUNIT is not enabled, these stubs quietly disappear. */
-+#define KUNIT_STATIC_STUB_REDIRECT(real_fn_name, args...) do {} while (0)
-+
-+#else
-+
-+#include <kunit/test.h>
-+#include <kunit/test-bug.h>
-+
-+#include <linux/compiler.h> /* for {un,}likely() */
-+#include <linux/sched.h> /* for task_struct */
-+
-+
-+/**
-+ * KUNIT_STATIC_STUB_REDIRECT() - call a replacement 'static stub' if one exists
-+ * @real_fn_name: The name of this function (as an identifier, not a string)
-+ * @args: All of the arguments passed to this function
-+ *
-+ * This is a function prologue which is used to allow calls to the current
-+ * function to be redirected by a KUnit test. KUnit tests can call
-+ * kunit_activate_static_stub() to pass a replacement function in. The
-+ * replacement function will be called by KUNIT_STATIC_STUB_REDIRECT(), which
-+ * will then return from the function. If the caller is not in a KUnit context,
-+ * the function will continue execution as normal.
-+ *
-+ * Example:
-+ *
-+ * .. code-block:: c
-+ *
-+ *	int real_func(int n)
-+ *	{
-+ *		KUNIT_STATIC_STUB_REDIRECT(real_func, n);
-+ *		return 0;
-+ *	}
-+ *
-+ *	int replacement_func(int n)
-+ *	{
-+ *		return 42;
-+ *	}
-+ *
-+ *	void example_test(struct kunit *test)
-+ *	{
-+ *		kunit_activate_static_stub(test, real_func, replacement_func);
-+ *		KUNIT_EXPECT_EQ(test, real_func(1), 42);
-+ *	}
-+ *
-+ */
-+#define KUNIT_STATIC_STUB_REDIRECT(real_fn_name, args...)		\
-+do {									\
-+	typeof(&real_fn_name) replacement;				\
-+	struct kunit *current_test = kunit_get_current_test();		\
-+									\
-+	if (likely(!current_test))					\
-+		break;							\
-+									\
-+	replacement = kunit_hooks.get_static_stub_address(current_test,	\
-+							&real_fn_name);	\
-+									\
-+	if (unlikely(replacement))					\
-+		return replacement(args);				\
-+} while (0)
-+
-+/* Helper function for kunit_activate_static_stub(). The macro does
-+ * typechecking, so use it instead.
-+ */
-+void __kunit_activate_static_stub(struct kunit *test,
-+				  void *real_fn_addr,
-+				  void *replacement_addr);
-+
-+/**
-+ * kunit_activate_static_stub() - replace a function using static stubs.
-+ * @test: A pointer to the 'struct kunit' test context for the current test.
-+ * @real_fn_addr: The address of the function to replace.
-+ * @replacement_addr: The address of the function to replace it with.
-+ *
-+ * When activated, calls to real_fn_addr from within this test (even if called
-+ * indirectly) will instead call replacement_addr. The function pointed to by
-+ * real_fn_addr must begin with the static stub prologue in
-+ * KUNIT_STATIC_STUB_REDIRECT() for this to work. real_fn_addr and
-+ * replacement_addr must have the same type.
-+ *
-+ * The redirection can be disabled again with kunit_deactivate_static_stub().
-+ */
-+#define kunit_activate_static_stub(test, real_fn_addr, replacement_addr) do {	\
-+	typecheck_fn(typeof(&replacement_addr), real_fn_addr);			\
-+	__kunit_activate_static_stub(test, real_fn_addr, replacement_addr);	\
-+} while (0)
-+
-+
-+/**
-+ * kunit_deactivate_static_stub() - disable a function redirection
-+ * @test: A pointer to the 'struct kunit' test context for the current test.
-+ * @real_fn_addr: The address of the function to no-longer redirect
-+ *
-+ * Deactivates a redirection configured with kunit_activate_static_stub(). After
-+ * this function returns, calls to real_fn_addr() will execute the original
-+ * real_fn, not any previously-configured replacement.
-+ */
-+void kunit_deactivate_static_stub(struct kunit *test, void *real_fn_addr);
-+
-+#endif
-+#endif
-diff --git a/include/kunit/test-bug.h b/include/kunit/test-bug.h
-index 5fc58081d511..47aa8f21ccce 100644
--- a/include/kunit/test-bug.h
-+++ b/include/kunit/test-bug.h
-@@ -1,6 +1,6 @@
- /* SPDX-License-Identifier: GPL-2.0 */
- /*
- * KUnit API allowing dynamic analysis tools to interact with KUnit tests
-+ * KUnit API providing hooks for non-test code to interact with tests.
-  *
-  * Copyright (C) 2020, Google LLC.
-  * Author: Uriel Guajardo <urielguajardo@google.com>
-@@ -9,21 +9,63 @@
- #ifndef _KUNIT_TEST_BUG_H
- #define _KUNIT_TEST_BUG_H
- 
-#define kunit_fail_current_test(fmt, ...) \
-	__kunit_fail_current_test(__FILE__, __LINE__, fmt, ##__VA_ARGS__)
-+#include <linux/stddef.h> /* for NULL */
- 
-#if IS_BUILTIN(CONFIG_KUNIT)
-+#if IS_ENABLED(CONFIG_KUNIT)
- 
-extern __printf(3, 4) void __kunit_fail_current_test(const char *file, int line,
-						    const char *fmt, ...);
-+#include <linux/jump_label.h> /* For static branch */
-+#include <linux/sched.h>
- 
-#else
-+/* Static key if KUnit is running any tests. */
-+DECLARE_STATIC_KEY_FALSE(kunit_running);
-+
-+/* Hooks table: a table of function pointers filled in when kunit loads */
-+extern struct kunit_hooks_table {
-+	__printf(3, 4) void (*fail_current_test)(const char*, int, const char*, ...);
-+	void *(*get_static_stub_address)(struct kunit *test, void *real_fn_addr);
-+} kunit_hooks;
- 
-static inline __printf(3, 4) void __kunit_fail_current_test(const char *file, int line,
-							    const char *fmt, ...)
-+/**
-+ * kunit_get_current_test() - Return a pointer to the currently running
-+ *			      KUnit test.
-+ *
-+ * If a KUnit test is running in the current task, returns a pointer to its
-+ * associated struct kunit. This pointer can then be passed to any KUnit
-+ * function or assertion. If no test is running (or a test is running in a
-+ * different task), returns NULL.
-+ *
-+ * This function is safe to call even when KUnit is disabled. If CONFIG_KUNIT
-+ * is not enabled, it will compile down to nothing and will return quickly no
-+ * test is running.
-+ */
-+static inline struct kunit *kunit_get_current_test(void)
- {
-+	if (!static_branch_unlikely(&kunit_running))
-+		return NULL;
-+
-+	return current->kunit_test;
- }
- 
-+
-+/**
-+ * kunit_fail_current_test() - If a KUnit test is running, fail it.
-+ *
-+ * If a KUnit test is running in the current task, mark that test as failed.
-+ */
-+#define kunit_fail_current_test(fmt, ...) do {					\
-+		if (static_branch_unlikely(&kunit_running)) {			\
-+			/* Guaranteed to be non-NULL when kunit_running true*/	\
-+			kunit_hooks.fail_current_test(__FILE__, __LINE__,	\
-+						  fmt, ##__VA_ARGS__);		\
-+		}								\
-+	} while (0)
-+
-+#else
-+
-+static inline struct kunit *kunit_get_current_test(void) { return NULL; }
-+
-+#define kunit_fail_current_test(fmt, ...) do {} while (0)
-+
- #endif
- 
- #endif /* _KUNIT_TEST_BUG_H */
-diff --git a/include/kunit/test.h b/include/kunit/test.h
-index c958855681cc..5ec5182b5e57 100644
--- a/include/kunit/test.h
-+++ b/include/kunit/test.h
-@@ -12,10 +12,12 @@
- #include <kunit/assert.h>
- #include <kunit/try-catch.h>
- 
-+#include <linux/args.h>
- #include <linux/compiler.h>
- #include <linux/container_of.h>
- #include <linux/err.h>
- #include <linux/init.h>
-+#include <linux/jump_label.h>
- #include <linux/kconfig.h>
- #include <linux/kref.h>
- #include <linux/list.h>
-@@ -26,11 +28,13 @@
- #include <linux/types.h>
- 
- #include <asm/rwonce.h>
-+#include <asm/sections.h>
- 
-struct kunit;
-+/* Static key: true if any KUnit tests are currently running */
-+DECLARE_STATIC_KEY_FALSE(kunit_running);
- 
-/* Size of log associated with test. */
-#define KUNIT_LOG_SIZE	512
-+struct kunit;
-+struct string_stream;
- 
- /* Maximum size of parameter description string. */
- #define KUNIT_PARAM_DESC_SIZE 128
-@@ -43,6 +47,7 @@ struct kunit;
-  * sub-subtest.  See the "Subtests" section in
-  * https://node-tap.org/tap-protocol/
-  */
-+#define KUNIT_INDENT_LEN		4
- #define KUNIT_SUBTEST_INDENT		"    "
- #define KUNIT_SUBSUBTEST_INDENT		"        "
- 
-@@ -58,12 +63,37 @@ enum kunit_status {
- 	KUNIT_SKIPPED,
- };
- 
-+/* Attribute struct/enum definitions */
-+
-+/*
-+ * Speed Attribute is stored as an enum and separated into categories of
-+ * speed: very_slow, slow, and normal. These speeds are relative to
-+ * other KUnit tests.
-+ *
-+ * Note: unset speed attribute acts as default of KUNIT_SPEED_NORMAL.
-+ */
-+enum kunit_speed {
-+	KUNIT_SPEED_UNSET,
-+	KUNIT_SPEED_VERY_SLOW,
-+	KUNIT_SPEED_SLOW,
-+	KUNIT_SPEED_NORMAL,
-+	KUNIT_SPEED_MAX = KUNIT_SPEED_NORMAL,
-+};
-+
-+/* Holds attributes for each test case and suite */
-+struct kunit_attributes {
-+	enum kunit_speed speed;
-+};
-+
- /**
-  * struct kunit_case - represents an individual test case.
-  *
-  * @run_case: the function representing the actual test case.
-  * @name:     the name of the test case.
-  * @generate_params: the generator function for parameterized tests.
-+ * @attr:     the attributes associated with the test
-+ * @param_init: The init function to run before a parameterized test.
-+ * @param_exit: The exit function to run after a parameterized test.
-  *
-  * A test case is a function with the signature,
-  * ``void (*)(struct kunit *)``
-@@ -98,11 +128,16 @@ enum kunit_status {
- struct kunit_case {
- 	void (*run_case)(struct kunit *test);
- 	const char *name;
-	const void* (*generate_params)(const void *prev, char *desc);
-+	const void* (*generate_params)(struct kunit *test,
-+				       const void *prev, char *desc);
-+	struct kunit_attributes attr;
-+	int (*param_init)(struct kunit *test);
-+	void (*param_exit)(struct kunit *test);
- 
- 	/* private: internal use only. */
- 	enum kunit_status status;
-	char *log;
-+	char *module_name;
-+	struct string_stream *log;
- };
- 
- static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
-@@ -126,7 +161,32 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
-  * &struct kunit_case object from it. See the documentation for
-  * &struct kunit_case for an example on how to use it.
-  */
-#define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
-+#define KUNIT_CASE(test_name)			\
-+		{ .run_case = test_name, .name = #test_name,	\
-+		  .module_name = KBUILD_MODNAME}
-+
-+/**
-+ * KUNIT_CASE_ATTR - A helper for creating a &struct kunit_case
-+ * with attributes
-+ *
-+ * @test_name: a reference to a test case function.
-+ * @attributes: a reference to a struct kunit_attributes object containing
-+ * test attributes
-+ */
-+#define KUNIT_CASE_ATTR(test_name, attributes)			\
-+		{ .run_case = test_name, .name = #test_name,	\
-+		  .attr = attributes, .module_name = KBUILD_MODNAME}
-+
-+/**
-+ * KUNIT_CASE_SLOW - A helper for creating a &struct kunit_case
-+ * with the slow attribute
-+ *
-+ * @test_name: a reference to a test case function.
-+ */
-+
-+#define KUNIT_CASE_SLOW(test_name)			\
-+		{ .run_case = test_name, .name = #test_name,	\
-+		  .attr.speed = KUNIT_SPEED_SLOW, .module_name = KBUILD_MODNAME}
- 
- /**
-  * KUNIT_CASE_PARAM - A helper for creation a parameterized &struct kunit_case
-@@ -147,7 +207,46 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
-  */
- #define KUNIT_CASE_PARAM(test_name, gen_params)			\
- 		{ .run_case = test_name, .name = #test_name,	\
-		  .generate_params = gen_params }
-+		  .generate_params = gen_params, .module_name = KBUILD_MODNAME}
-+
-+/**
-+ * KUNIT_CASE_PARAM_ATTR - A helper for creating a parameterized &struct
-+ * kunit_case with attributes
-+ *
-+ * @test_name: a reference to a test case function.
-+ * @gen_params: a reference to a parameter generator function.
-+ * @attributes: a reference to a struct kunit_attributes object containing
-+ * test attributes
-+ */
-+#define KUNIT_CASE_PARAM_ATTR(test_name, gen_params, attributes)	\
-+		{ .run_case = test_name, .name = #test_name,	\
-+		  .generate_params = gen_params,				\
-+		  .attr = attributes, .module_name = KBUILD_MODNAME}
-+
-+/**
-+ * KUNIT_CASE_PARAM_WITH_INIT - Define a parameterized KUnit test case with custom
-+ * param_init() and param_exit() functions.
-+ * @test_name: The function implementing the test case.
-+ * @gen_params: The function to generate parameters for the test case.
-+ * @init: A reference to the param_init() function to run before a parameterized test.
-+ * @exit: A reference to the param_exit() function to run after a parameterized test.
-+ *
-+ * Provides the option to register param_init() and param_exit() functions.
-+ * param_init/exit will be passed the parameterized test context and run once
-+ * before and once after the parameterized test. The init function can be used
-+ * to add resources to share between parameter runs, pass parameter arrays,
-+ * and any other setup logic. The exit function can be used to clean up resources
-+ * that were not managed by the parameterized test, and any other teardown logic.
-+ *
-+ * Note: If you are registering a parameter array in param_init() with
-+ * kunit_register_param_array() then you need to pass kunit_array_gen_params()
-+ * to this as the generator function.
-+ */
-+#define KUNIT_CASE_PARAM_WITH_INIT(test_name, gen_params, init, exit)		\
-+		{ .run_case = test_name, .name = #test_name,			\
-+		  .generate_params = gen_params,				\
-+		  .param_init = init, .param_exit = exit,			\
-+		  .module_name = KBUILD_MODNAME}
- 
- /**
-  * struct kunit_suite - describes a related collection of &struct kunit_case
-@@ -158,12 +257,16 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
-  * @init:	called before every test case.
-  * @exit:	called after every test case.
-  * @test_cases:	a null terminated array of test cases.
-+ * @attr:	the attributes associated with the test suite
-  *
-  * A kunit_suite is a collection of related &struct kunit_case s, such that
-  * @init is called before every test case and @exit is called after every
-  * test case, similar to the notion of a *test fixture* or a *test class*
-  * in other unit testing frameworks like JUnit or Googletest.
-  *
-+ * Note that @exit and @suite_exit will run even if @init or @suite_init
-+ * fail: make sure they can handle any inconsistent state which may result.
-+ *
-  * Every &struct kunit_case must be associated with a kunit_suite for KUnit
-  * to run it.
-  */
-@@ -174,12 +277,34 @@ struct kunit_suite {
- 	int (*init)(struct kunit *test);
- 	void (*exit)(struct kunit *test);
- 	struct kunit_case *test_cases;
-+	struct kunit_attributes attr;
- 
- 	/* private: internal use only */
- 	char status_comment[KUNIT_STATUS_COMMENT_SIZE];
- 	struct dentry *debugfs;
-	char *log;
-+	struct string_stream *log;
- 	int suite_init_err;
-+	bool is_init;
-+};
-+
-+/* Stores an array of suites, end points one past the end */
-+struct kunit_suite_set {
-+	struct kunit_suite * const *start;
-+	struct kunit_suite * const *end;
-+};
-+
-+/* Stores the pointer to the parameter array and its metadata. */
-+struct kunit_params {
-+	/*
-+	 * Reference to the parameter array for a parameterized test. This
-+	 * is NULL if a parameter array wasn't directly passed to the
-+	 * parameterized test context struct kunit via kunit_register_params_array().
-+	 */
-+	const void *params;
-+	/* Reference to a function that gets the description of a parameter. */
-+	void (*get_description)(struct kunit *test, const void *param, char *desc);
-+	size_t num_params;
-+	size_t elem_size;
- };
- 
- /**
-@@ -187,18 +312,24 @@ struct kunit_suite {
-  *
-  * @priv: for user to store arbitrary data. Commonly used to pass data
-  *	  created in the init function (see &struct kunit_suite).
-+ * @parent: reference to the parent context of type struct kunit that can
-+ *	    be used for storing shared resources.
-+ * @params_array: for storing the parameter array.
-  *
-  * Used to store information about the current context under which the test
-  * is running. Most of this data is private and should only be accessed
- * indirectly via public functions; the one exception is @priv which can be
- * used by the test writer to store arbitrary data.
-+ * indirectly via public functions; the exceptions are @priv, @parent and
-+ * @params_array which can be used by the test writer to store arbitrary data,
-+ * access the parent context, and to store the parameter array, respectively.
-  */
- struct kunit {
- 	void *priv;
-+	struct kunit *parent;
-+	struct kunit_params params_array;
- 
- 	/* private: internal use only. */
- 	const char *name; /* Read only after initialization! */
-	char *log; /* Points at case log after initialization */
-+	struct string_stream *log; /* Points at case log after initialization */
- 	struct kunit_try_catch try_catch;
- 	/* param_value is the current parameter value for a test case. */
- 	const void *param_value;
-@@ -221,6 +352,8 @@ struct kunit {
- 	struct list_head resources; /* Protected by lock. */
- 
- 	char status_comment[KUNIT_STATUS_COMMENT_SIZE];
-+	/* Saves the last seen test. Useful to help with faults. */
-+	struct kunit_loc last_seen;
- };
- 
- static inline void kunit_set_failure(struct kunit *test)
-@@ -228,7 +361,14 @@ static inline void kunit_set_failure(struct kunit *test)
- 	WRITE_ONCE(test->status, KUNIT_FAILURE);
- }
- 
-void kunit_init_test(struct kunit *test, const char *name, char *log);
-+bool kunit_enabled(void);
-+bool kunit_autorun(void);
-+const char *kunit_action(void);
-+const char *kunit_filter_glob(void);
-+char *kunit_filter(void);
-+char *kunit_filter_action(void);
-+
-+void kunit_init_test(struct kunit *test, const char *name, struct string_stream *log);
- 
- int kunit_run_tests(struct kunit_suite *suite);
- 
-@@ -237,10 +377,27 @@ size_t kunit_suite_num_test_cases(struct kunit_suite *suite);
- unsigned int kunit_test_case_num(struct kunit_suite *suite,
- 				 struct kunit_case *test_case);
- 
-int __kunit_test_suites_init(struct kunit_suite * const * const suites, int num_suites);
-+struct kunit_suite_set
-+kunit_filter_suites(const struct kunit_suite_set *suite_set,
-+		    const char *filter_glob,
-+		    char *filters,
-+		    char *filter_action,
-+		    int *err);
-+void kunit_free_suite_set(struct kunit_suite_set suite_set);
-+
-+int __kunit_test_suites_init(struct kunit_suite * const * const suites, int num_suites,
-+			     bool run_tests);
- 
- void __kunit_test_suites_exit(struct kunit_suite **suites, int num_suites);
- 
-+void kunit_exec_run_tests(struct kunit_suite_set *suite_set, bool builtin);
-+void kunit_exec_list_tests(struct kunit_suite_set *suite_set, bool include_attr);
-+
-+struct kunit_suite_set kunit_merge_suite_sets(struct kunit_suite_set init_suite_set,
-+		struct kunit_suite_set suite_set);
-+
-+const void *kunit_array_gen_params(struct kunit *test, const void *prev, char *desc);
-+
- #if IS_BUILTIN(CONFIG_KUNIT)
- int kunit_run_all_tests(void);
- #else
-@@ -251,7 +408,6 @@ static inline int kunit_run_all_tests(void)
- #endif /* IS_BUILTIN(CONFIG_KUNIT) */
- 
- #define __kunit_test_suites(unique_array, ...)				       \
-	MODULE_INFO(test, "Y");						       \
- 	static struct kunit_suite *unique_array[]			       \
- 	__aligned(sizeof(struct kunit_suite *))				       \
- 	__used __section(".kunit_test_suites") = { __VA_ARGS__ }
-@@ -276,6 +432,11 @@ static inline int kunit_run_all_tests(void)
- 
- #define kunit_test_suite(suite)	kunit_test_suites(&suite)
- 
-+#define __kunit_init_test_suites(unique_array, ...)			       \
-+	static struct kunit_suite *unique_array[]			       \
-+	__aligned(sizeof(struct kunit_suite *))				       \
-+	__used __section(".kunit_init_test_suites") = { __VA_ARGS__ }
-+
- /**
-  * kunit_test_init_section_suites() - used to register one or more &struct
-  *				      kunit_suite containing init functions or
-@@ -283,22 +444,21 @@ static inline int kunit_run_all_tests(void)
-  *
-  * @__suites: a statically allocated list of &struct kunit_suite.
-  *
- * This functions identically as kunit_test_suites() except that it suppresses
- * modpost warnings for referencing functions marked __init or data marked
- * __initdata; this is OK because currently KUnit only runs tests upon boot
- * during the init phase or upon loading a module during the init phase.
-+ * This functions similar to kunit_test_suites() except that it compiles the
-+ * list of suites during init phase.
-  *
- * NOTE TO KUNIT DEVS: If we ever allow KUnit tests to be run after boot, these
- * tests must be excluded.
-+ * This macro also suffixes the array and suite declarations it makes with
-+ * _probe; so that modpost suppresses warnings about referencing init data
-+ * for symbols named in this manner.
-  *
- * The only thing this macro does that's different from kunit_test_suites is
- * that it suffixes the array and suite declarations it makes with _probe;
- * modpost suppresses warnings about referencing init data for symbols named in
- * this manner.
-+ * Note: these init tests are not able to be run after boot so there is no
-+ * "run" debugfs file generated for these tests.
-+ *
-+ * Also, do not mark the suite or test case structs with __initdata because
-+ * they will be used after the init phase with debugfs.
-  */
- #define kunit_test_init_section_suites(__suites...)			\
-	__kunit_test_suites(CONCATENATE(__UNIQUE_ID(array), _probe),	\
-			    CONCATENATE(__UNIQUE_ID(suites), _probe),	\
-+	__kunit_init_test_suites(CONCATENATE(__UNIQUE_ID(array), _probe), \
- 			    ##__suites)
- 
- #define kunit_test_init_section_suite(suite)	\
-@@ -317,8 +477,11 @@ enum kunit_status kunit_suite_has_succeeded(struct kunit_suite *suite);
-  * @gfp: flags passed to underlying kmalloc().
-  *
-  * Just like `kmalloc_array(...)`, except the allocation is managed by the test case
- * and is automatically cleaned up after the test case concludes. See &struct
- * kunit_resource for more information.
-+ * and is automatically cleaned up after the test case concludes. See kunit_add_action()
-+ * for more information.
-+ *
-+ * Note that some internal context data is also allocated with GFP_KERNEL,
-+ * regardless of the gfp passed in.
-  */
- void *kunit_kmalloc_array(struct kunit *test, size_t n, size_t size, gfp_t gfp);
- 
-@@ -329,6 +492,9 @@ void *kunit_kmalloc_array(struct kunit *test, size_t n, size_t size, gfp_t gfp);
-  * @gfp: flags passed to underlying kmalloc().
-  *
-  * See kmalloc() and kunit_kmalloc_array() for more information.
-+ *
-+ * Note that some internal context data is also allocated with GFP_KERNEL,
-+ * regardless of the gfp passed in.
-  */
- static inline void *kunit_kmalloc(struct kunit *test, size_t size, gfp_t gfp)
- {
-@@ -369,14 +535,91 @@ static inline void *kunit_kcalloc(struct kunit *test, size_t n, size_t size, gfp
- 	return kunit_kmalloc_array(test, n, size, gfp | __GFP_ZERO);
- }
- 
-+
-+/**
-+ * kunit_kfree_const() - conditionally free test managed memory
-+ * @test: The test context object.
-+ * @x: pointer to the memory
-+ *
-+ * Calls kunit_kfree() only if @x is not in .rodata section.
-+ * See kunit_kstrdup_const() for more information.
-+ */
-+void kunit_kfree_const(struct kunit *test, const void *x);
-+
-+/**
-+ * kunit_kstrdup() - Duplicates a string into a test managed allocation.
-+ *
-+ * @test: The test context object.
-+ * @str: The NULL-terminated string to duplicate.
-+ * @gfp: flags passed to underlying kmalloc().
-+ *
-+ * See kstrdup() and kunit_kmalloc_array() for more information.
-+ */
-+static inline char *kunit_kstrdup(struct kunit *test, const char *str, gfp_t gfp)
-+{
-+	size_t len;
-+	char *buf;
-+
-+	if (!str)
-+		return NULL;
-+
-+	len = strlen(str) + 1;
-+	buf = kunit_kmalloc(test, len, gfp);
-+	if (buf)
-+		memcpy(buf, str, len);
-+	return buf;
-+}
-+
-+/**
-+ * kunit_kstrdup_const() - Conditionally duplicates a string into a test managed allocation.
-+ *
-+ * @test: The test context object.
-+ * @str: The NULL-terminated string to duplicate.
-+ * @gfp: flags passed to underlying kmalloc().
-+ *
-+ * Calls kunit_kstrdup() only if @str is not in the rodata section. Must be freed with
-+ * kunit_kfree_const() -- not kunit_kfree().
-+ * See kstrdup_const() and kunit_kmalloc_array() for more information.
-+ */
-+const char *kunit_kstrdup_const(struct kunit *test, const char *str, gfp_t gfp);
-+
-+/**
-+ * kunit_attach_mm() - Create and attach a new mm if it doesn't already exist.
-+ *
-+ * Allocates a &struct mm_struct and attaches it to @current. In most cases, call
-+ * kunit_vm_mmap() without calling kunit_attach_mm() directly. Only necessary when
-+ * code under test accesses the mm before executing the mmap (e.g., to perform
-+ * additional initialization beforehand).
-+ *
-+ * Return: 0 on success, -errno on failure.
-+ */
-+int kunit_attach_mm(void);
-+
-+/**
-+ * kunit_vm_mmap() - Allocate KUnit-tracked vm_mmap() area
-+ * @test: The test context object.
-+ * @file: struct file pointer to map from, if any
-+ * @addr: desired address, if any
-+ * @len: how many bytes to allocate
-+ * @prot: mmap PROT_* bits
-+ * @flag: mmap flags
-+ * @offset: offset into @file to start mapping from.
-+ *
-+ * See vm_mmap() for more information.
-+ */
-+unsigned long kunit_vm_mmap(struct kunit *test, struct file *file,
-+			    unsigned long addr, unsigned long len,
-+			    unsigned long prot, unsigned long flag,
-+			    unsigned long offset);
-+
- void kunit_cleanup(struct kunit *test);
- 
-void __printf(2, 3) kunit_log_append(char *log, const char *fmt, ...);
-+void __printf(2, 3) kunit_log_append(struct string_stream *log, const char *fmt, ...);
- 
- /**
- * kunit_mark_skipped() - Marks @test_or_suite as skipped
-+ * kunit_mark_skipped() - Marks @test as skipped
-  *
- * @test_or_suite: The test context object.
-+ * @test: The test context object.
-  * @fmt:  A printk() style format string.
-  *
-  * Marks the test as skipped. @fmt is given output as the test status
-@@ -384,18 +627,18 @@ void __printf(2, 3) kunit_log_append(char *log, const char *fmt, ...);
-  *
-  * Test execution continues after kunit_mark_skipped() is called.
-  */
-#define kunit_mark_skipped(test_or_suite, fmt, ...)			\
-+#define kunit_mark_skipped(test, fmt, ...)				\
- 	do {								\
-		WRITE_ONCE((test_or_suite)->status, KUNIT_SKIPPED);	\
-		scnprintf((test_or_suite)->status_comment,		\
-+		WRITE_ONCE((test)->status, KUNIT_SKIPPED);		\
-+		scnprintf((test)->status_comment,			\
- 			  KUNIT_STATUS_COMMENT_SIZE,			\
- 			  fmt, ##__VA_ARGS__);				\
- 	} while (0)
- 
- /**
- * kunit_skip() - Marks @test_or_suite as skipped
-+ * kunit_skip() - Marks @test as skipped
-  *
- * @test_or_suite: The test context object.
-+ * @test: The test context object.
-  * @fmt:  A printk() style format string.
-  *
-  * Skips the test. @fmt is given output as the test status
-@@ -403,10 +646,10 @@ void __printf(2, 3) kunit_log_append(char *log, const char *fmt, ...);
-  *
-  * Test execution is halted after kunit_skip() is called.
-  */
-#define kunit_skip(test_or_suite, fmt, ...)				\
-+#define kunit_skip(test, fmt, ...)					\
- 	do {								\
-		kunit_mark_skipped((test_or_suite), fmt, ##__VA_ARGS__);\
-		kunit_try_catch_throw(&((test_or_suite)->try_catch));	\
-+		kunit_mark_skipped((test), fmt, ##__VA_ARGS__);		\
-+		kunit_try_catch_throw(&((test)->try_catch));		\
- 	} while (0)
- 
- /*
-@@ -416,7 +659,7 @@ void __printf(2, 3) kunit_log_append(char *log, const char *fmt, ...);
- #define kunit_log(lvl, test_or_suite, fmt, ...)				\
- 	do {								\
- 		printk(lvl fmt, ##__VA_ARGS__);				\
-		kunit_log_append((test_or_suite)->log,	fmt "\n",	\
-+		kunit_log_append((test_or_suite)->log,	fmt,		\
- 				 ##__VA_ARGS__);			\
- 	} while (0)
- 
-@@ -458,6 +701,15 @@ void __printf(2, 3) kunit_log_append(char *log, const char *fmt, ...);
- #define kunit_err(test, fmt, ...) \
- 	kunit_printk(KERN_ERR, test, fmt, ##__VA_ARGS__)
- 
-+/*
-+ * Must be called at the beginning of each KUNIT_*_ASSERTION().
-+ * Cf. KUNIT_CURRENT_LOC.
-+ */
-+#define _KUNIT_SAVE_LOC(test) do {					       \
-+	WRITE_ONCE(test->last_seen.file, __FILE__);			       \
-+	WRITE_ONCE(test->last_seen.line, __LINE__);			       \
-+} while (0)
-+
- /**
-  * KUNIT_SUCCEED() - A no-op expectation. Only exists for code clarity.
-  * @test: The test context object.
-@@ -466,36 +718,42 @@ void __printf(2, 3) kunit_log_append(char *log, const char *fmt, ...);
-  * words, it does nothing and only exists for code clarity. See
-  * KUNIT_EXPECT_TRUE() for more information.
-  */
-#define KUNIT_SUCCEED(test) do {} while (0)
-
-void kunit_do_failed_assertion(struct kunit *test,
-			       const struct kunit_loc *loc,
-			       enum kunit_assert_type type,
-			       const struct kunit_assert *assert,
-			       const char *fmt, ...);
-
-#define KUNIT_ASSERTION(test, assert_type, pass, assert_class, INITIALIZER, fmt, ...) do { \
-	if (unlikely(!(pass))) {					       \
-		static const struct kunit_loc __loc = KUNIT_CURRENT_LOC;       \
-		struct assert_class __assertion = INITIALIZER;		       \
-		kunit_do_failed_assertion(test,				       \
-					  &__loc,			       \
-					  assert_type,			       \
-					  &__assertion.assert,		       \
-					  fmt,				       \
-					  ##__VA_ARGS__);		       \
-	}								       \
-+#define KUNIT_SUCCEED(test) _KUNIT_SAVE_LOC(test)
-+
-+void __noreturn __kunit_abort(struct kunit *test);
-+
-+void __printf(6, 7) __kunit_do_failed_assertion(struct kunit *test,
-+						const struct kunit_loc *loc,
-+						enum kunit_assert_type type,
-+						const struct kunit_assert *assert,
-+						assert_format_t assert_format,
-+						const char *fmt, ...);
-+
-+#define _KUNIT_FAILED(test, assert_type, assert_class, assert_format, INITIALIZER, fmt, ...) do { \
-+	static const struct kunit_loc __loc = KUNIT_CURRENT_LOC;	       \
-+	const struct assert_class __assertion = INITIALIZER;		       \
-+	__kunit_do_failed_assertion(test,				       \
-+				    &__loc,				       \
-+				    assert_type,			       \
-+				    &__assertion.assert,		       \
-+				    assert_format,			       \
-+				    fmt,				       \
-+				    ##__VA_ARGS__);			       \
-+	if (assert_type == KUNIT_ASSERTION)				       \
-+		__kunit_abort(test);					       \
- } while (0)
- 
- 
-#define KUNIT_FAIL_ASSERTION(test, assert_type, fmt, ...)		       \
-	KUNIT_ASSERTION(test,						       \
-			assert_type,					       \
-			false,						       \
-			kunit_fail_assert,				       \
-			KUNIT_INIT_FAIL_ASSERT_STRUCT,			       \
-			fmt,						       \
-			##__VA_ARGS__)
-+#define KUNIT_FAIL_ASSERTION(test, assert_type, fmt, ...) do {		       \
-+	_KUNIT_SAVE_LOC(test);						       \
-+	_KUNIT_FAILED(test,						       \
-+		      assert_type,					       \
-+		      kunit_fail_assert,				       \
-+		      kunit_fail_assert_format,				       \
-+		      {},						       \
-+		      fmt,						       \
-+		      ##__VA_ARGS__);					       \
-+} while (0)
- 
- /**
-  * KUNIT_FAIL() - Always causes a test to fail when evaluated.
-@@ -514,20 +772,29 @@ void kunit_do_failed_assertion(struct kunit *test,
- 			     fmt,					       \
- 			     ##__VA_ARGS__)
- 
-+/* Helper to safely pass around an initializer list to other macros. */
-+#define KUNIT_INIT_ASSERT(initializers...) { initializers }
-+
- #define KUNIT_UNARY_ASSERTION(test,					       \
- 			      assert_type,				       \
-			      condition,				       \
-			      expected_true,				       \
-+			      condition_,				       \
-+			      expected_true_,				       \
- 			      fmt,					       \
- 			      ...)					       \
-	KUNIT_ASSERTION(test,						       \
-			assert_type,					       \
-			!!(condition) == !!expected_true,		       \
-			kunit_unary_assert,				       \
-			KUNIT_INIT_UNARY_ASSERT_STRUCT(#condition,	       \
-						       expected_true),	       \
-			fmt,						       \
-			##__VA_ARGS__)
-+do {									       \
-+	_KUNIT_SAVE_LOC(test);						       \
-+	if (likely(!!(condition_) == !!expected_true_))			       \
-+		break;							       \
-+									       \
-+	_KUNIT_FAILED(test,						       \
-+		      assert_type,					       \
-+		      kunit_unary_assert,				       \
-+		      kunit_unary_assert_format,			       \
-+		      KUNIT_INIT_ASSERT(.condition = #condition_,	       \
-+					.expected_true = expected_true_),      \
-+		      fmt,						       \
-+		      ##__VA_ARGS__);					       \
-+} while (0)
- 
- #define KUNIT_TRUE_MSG_ASSERTION(test, assert_type, condition, fmt, ...)       \
- 	KUNIT_UNARY_ASSERTION(test,					       \
-@@ -577,16 +844,19 @@ do {									       \
- 		.right_text = #right,					       \
- 	};								       \
- 									       \
-	KUNIT_ASSERTION(test,						       \
-			assert_type,					       \
-			__left op __right,				       \
-			assert_class,					       \
-			KUNIT_INIT_BINARY_ASSERT_STRUCT(format_func,	       \
-							&__text,	       \
-							__left,		       \
-							__right),	       \
-			fmt,						       \
-			##__VA_ARGS__);					       \
-+	_KUNIT_SAVE_LOC(test);						       \
-+	if (likely(__left op __right))					       \
-+		break;							       \
-+									       \
-+	_KUNIT_FAILED(test,						       \
-+		      assert_type,					       \
-+		      assert_class,					       \
-+		      format_func,					       \
-+		      KUNIT_INIT_ASSERT(.text = &__text,		       \
-+					.left_value = __left,		       \
-+					.right_value = __right),	       \
-+		      fmt,						       \
-+		      ##__VA_ARGS__);					       \
- } while (0)
- 
- #define KUNIT_BINARY_INT_ASSERTION(test,				       \
-@@ -635,16 +905,55 @@ do {									       \
- 		.right_text = #right,					       \
- 	};								       \
- 									       \
-	KUNIT_ASSERTION(test,						       \
-			assert_type,					       \
-			strcmp(__left, __right) op 0,			       \
-			kunit_binary_str_assert,			       \
-			KUNIT_INIT_BINARY_ASSERT_STRUCT(kunit_binary_str_assert_format,\
-							&__text,	       \
-							__left,		       \
-							__right),	       \
-			fmt,						       \
-			##__VA_ARGS__);					       \
-+	_KUNIT_SAVE_LOC(test);						       \
-+	if (likely((__left) && (__right) && (strcmp(__left, __right) op 0)))   \
-+		break;							       \
-+									       \
-+									       \
-+	_KUNIT_FAILED(test,						       \
-+		      assert_type,					       \
-+		      kunit_binary_str_assert,				       \
-+		      kunit_binary_str_assert_format,			       \
-+		      KUNIT_INIT_ASSERT(.text = &__text,		       \
-+					.left_value = __left,		       \
-+					.right_value = __right),	       \
-+		      fmt,						       \
-+		      ##__VA_ARGS__);					       \
-+} while (0)
-+
-+#define KUNIT_MEM_ASSERTION(test,					       \
-+			    assert_type,				       \
-+			    left,					       \
-+			    op,						       \
-+			    right,					       \
-+			    size_,					       \
-+			    fmt,					       \
-+			    ...)					       \
-+do {									       \
-+	const void *__left = (left);					       \
-+	const void *__right = (right);					       \
-+	const size_t __size = (size_);					       \
-+	static const struct kunit_binary_assert_text __text = {		       \
-+		.operation = #op,					       \
-+		.left_text = #left,					       \
-+		.right_text = #right,					       \
-+	};								       \
-+									       \
-+	_KUNIT_SAVE_LOC(test);						       \
-+	if (likely(__left && __right))					       \
-+		if (likely(memcmp(__left, __right, __size) op 0))	       \
-+			break;						       \
-+									       \
-+	_KUNIT_FAILED(test,						       \
-+		      assert_type,					       \
-+		      kunit_mem_assert,					       \
-+		      kunit_mem_assert_format,				       \
-+		      KUNIT_INIT_ASSERT(.text = &__text,		       \
-+					.left_value = __left,		       \
-+					.right_value = __right,		       \
-+					.size = __size),		       \
-+		      fmt,						       \
-+		      ##__VA_ARGS__);					       \
- } while (0)
- 
- #define KUNIT_PTR_NOT_ERR_OR_NULL_MSG_ASSERTION(test,			       \
-@@ -655,14 +964,17 @@ do {									       \
- do {									       \
- 	const typeof(ptr) __ptr = (ptr);				       \
- 									       \
-	KUNIT_ASSERTION(test,						       \
-			assert_type,					       \
-			!IS_ERR_OR_NULL(__ptr),				       \
-			kunit_ptr_not_err_assert,			       \
-			KUNIT_INIT_PTR_NOT_ERR_STRUCT(#ptr,		       \
-						      __ptr),		       \
-			fmt,						       \
-			##__VA_ARGS__);					       \
-+	_KUNIT_SAVE_LOC(test);						       \
-+	if (!IS_ERR_OR_NULL(__ptr))					       \
-+		break;							       \
-+									       \
-+	_KUNIT_FAILED(test,						       \
-+		      assert_type,					       \
-+		      kunit_ptr_not_err_assert,				       \
-+		      kunit_ptr_not_err_assert_format,			       \
-+		      KUNIT_INIT_ASSERT(.text = #ptr, .value = __ptr),	       \
-+		      fmt,						       \
-+		      ##__VA_ARGS__);					       \
- } while (0)
- 
- /**
-@@ -826,7 +1138,7 @@ do {									       \
- 
- #define KUNIT_EXPECT_LE_MSG(test, left, right, fmt, ...)		       \
- 	KUNIT_BINARY_INT_ASSERTION(test,				       \
-				   KUNIT_ASSERTION,			       \
-+				   KUNIT_EXPECTATION,			       \
- 				   left, <=, right,			       \
- 				   fmt,					       \
- 				    ##__VA_ARGS__)
-@@ -916,6 +1228,60 @@ do {									       \
- 				   ##__VA_ARGS__)
- 
- /**
-+ * KUNIT_EXPECT_MEMEQ() - Expects that the first @size bytes of @left and @right are equal.
-+ * @test: The test context object.
-+ * @left: An arbitrary expression that evaluates to the specified size.
-+ * @right: An arbitrary expression that evaluates to the specified size.
-+ * @size: Number of bytes compared.
-+ *
-+ * Sets an expectation that the values that @left and @right evaluate to are
-+ * equal. This is semantically equivalent to
-+ * KUNIT_EXPECT_TRUE(@test, !memcmp((@left), (@right), (@size))). See
-+ * KUNIT_EXPECT_TRUE() for more information.
-+ *
-+ * Although this expectation works for any memory block, it is not recommended
-+ * for comparing more structured data, such as structs. This expectation is
-+ * recommended for comparing, for example, data arrays.
-+ */
-+#define KUNIT_EXPECT_MEMEQ(test, left, right, size) \
-+	KUNIT_EXPECT_MEMEQ_MSG(test, left, right, size, NULL)
-+
-+#define KUNIT_EXPECT_MEMEQ_MSG(test, left, right, size, fmt, ...)	       \
-+	KUNIT_MEM_ASSERTION(test,					       \
-+			    KUNIT_EXPECTATION,				       \
-+			    left, ==, right,				       \
-+			    size,					       \
-+			    fmt,					       \
-+			    ##__VA_ARGS__)
-+
-+/**
-+ * KUNIT_EXPECT_MEMNEQ() - Expects that the first @size bytes of @left and @right are not equal.
-+ * @test: The test context object.
-+ * @left: An arbitrary expression that evaluates to the specified size.
-+ * @right: An arbitrary expression that evaluates to the specified size.
-+ * @size: Number of bytes compared.
-+ *
-+ * Sets an expectation that the values that @left and @right evaluate to are
-+ * not equal. This is semantically equivalent to
-+ * KUNIT_EXPECT_TRUE(@test, memcmp((@left), (@right), (@size))). See
-+ * KUNIT_EXPECT_TRUE() for more information.
-+ *
-+ * Although this expectation works for any memory block, it is not recommended
-+ * for comparing more structured data, such as structs. This expectation is
-+ * recommended for comparing, for example, data arrays.
-+ */
-+#define KUNIT_EXPECT_MEMNEQ(test, left, right, size) \
-+	KUNIT_EXPECT_MEMNEQ_MSG(test, left, right, size, NULL)
-+
-+#define KUNIT_EXPECT_MEMNEQ_MSG(test, left, right, size, fmt, ...)	       \
-+	KUNIT_MEM_ASSERTION(test,					       \
-+			    KUNIT_EXPECTATION,				       \
-+			    left, !=, right,				       \
-+			    size,					       \
-+			    fmt,					       \
-+			    ##__VA_ARGS__)
-+
-+/**
-  * KUNIT_EXPECT_NULL() - Expects that @ptr is null.
-  * @test: The test context object.
-  * @ptr: an arbitrary pointer.
-@@ -977,7 +1343,18 @@ do {									       \
- 						fmt,			       \
- 						##__VA_ARGS__)
- 
-#define KUNIT_ASSERT_FAILURE(test, fmt, ...) \
-+/**
-+ * KUNIT_FAIL_AND_ABORT() - Always causes a test to fail and abort when evaluated.
-+ * @test: The test context object.
-+ * @fmt: an informational message to be printed when the assertion is made.
-+ * @...: string format arguments.
-+ *
-+ * The opposite of KUNIT_SUCCEED(), it is an assertion that always fails. In
-+ * other words, it always results in a failed assertion, and consequently
-+ * always causes the test case to fail and abort when evaluated.
-+ * See KUNIT_ASSERT_TRUE() for more information.
-+ */
-+#define KUNIT_FAIL_AND_ABORT(test, fmt, ...) \
- 	KUNIT_FAIL_ASSERTION(test, KUNIT_ASSERTION, fmt, ##__VA_ARGS__)
- 
- /**
-@@ -1116,7 +1493,7 @@ do {									       \
- 
- #define KUNIT_ASSERT_LT_MSG(test, left, right, fmt, ...)		       \
- 	KUNIT_BINARY_INT_ASSERTION(test,				       \
-				   KUNIT_EXPECTATION,			       \
-+				   KUNIT_ASSERTION,			       \
- 				   left, <, right,			       \
- 				   fmt,					       \
- 				    ##__VA_ARGS__)
-@@ -1157,7 +1534,7 @@ do {									       \
- 
- #define KUNIT_ASSERT_GT_MSG(test, left, right, fmt, ...)		       \
- 	KUNIT_BINARY_INT_ASSERTION(test,				       \
-				   KUNIT_EXPECTATION,			       \
-+				   KUNIT_ASSERTION,			       \
- 				   left, >, right,			       \
- 				   fmt,					       \
- 				    ##__VA_ARGS__)
-@@ -1204,12 +1581,12 @@ do {									       \
- 				   ##__VA_ARGS__)
- 
- /**
- * KUNIT_ASSERT_STRNEQ() - Expects that strings @left and @right are not equal.
-+ * KUNIT_ASSERT_STRNEQ() - An assertion that strings @left and @right are not equal.
-  * @test: The test context object.
-  * @left: an arbitrary expression that evaluates to a null terminated string.
-  * @right: an arbitrary expression that evaluates to a null terminated string.
-  *
- * Sets an expectation that the values that @left and @right evaluate to are
-+ * Sets an assertion that the values that @left and @right evaluate to are
-  * not equal. This is semantically equivalent to
-  * KUNIT_ASSERT_TRUE(@test, strcmp((@left), (@right))). See KUNIT_ASSERT_TRUE()
-  * for more information.
-@@ -1225,6 +1602,60 @@ do {									       \
- 				   ##__VA_ARGS__)
- 
- /**
-+ * KUNIT_ASSERT_MEMEQ() - Asserts that the first @size bytes of @left and @right are equal.
-+ * @test: The test context object.
-+ * @left: An arbitrary expression that evaluates to the specified size.
-+ * @right: An arbitrary expression that evaluates to the specified size.
-+ * @size: Number of bytes compared.
-+ *
-+ * Sets an assertion that the values that @left and @right evaluate to are
-+ * equal. This is semantically equivalent to
-+ * KUNIT_ASSERT_TRUE(@test, !memcmp((@left), (@right), (@size))). See
-+ * KUNIT_ASSERT_TRUE() for more information.
-+ *
-+ * Although this assertion works for any memory block, it is not recommended
-+ * for comparing more structured data, such as structs. This assertion is
-+ * recommended for comparing, for example, data arrays.
-+ */
-+#define KUNIT_ASSERT_MEMEQ(test, left, right, size) \
-+	KUNIT_ASSERT_MEMEQ_MSG(test, left, right, size, NULL)
-+
-+#define KUNIT_ASSERT_MEMEQ_MSG(test, left, right, size, fmt, ...)	       \
-+	KUNIT_MEM_ASSERTION(test,					       \
-+			    KUNIT_ASSERTION,				       \
-+			    left, ==, right,				       \
-+			    size,					       \
-+			    fmt,					       \
-+			    ##__VA_ARGS__)
-+
-+/**
-+ * KUNIT_ASSERT_MEMNEQ() - Asserts that the first @size bytes of @left and @right are not equal.
-+ * @test: The test context object.
-+ * @left: An arbitrary expression that evaluates to the specified size.
-+ * @right: An arbitrary expression that evaluates to the specified size.
-+ * @size: Number of bytes compared.
-+ *
-+ * Sets an assertion that the values that @left and @right evaluate to are
-+ * not equal. This is semantically equivalent to
-+ * KUNIT_ASSERT_TRUE(@test, memcmp((@left), (@right), (@size))). See
-+ * KUNIT_ASSERT_TRUE() for more information.
-+ *
-+ * Although this assertion works for any memory block, it is not recommended
-+ * for comparing more structured data, such as structs. This assertion is
-+ * recommended for comparing, for example, data arrays.
-+ */
-+#define KUNIT_ASSERT_MEMNEQ(test, left, right, size) \
-+	KUNIT_ASSERT_MEMNEQ_MSG(test, left, right, size, NULL)
-+
-+#define KUNIT_ASSERT_MEMNEQ_MSG(test, left, right, size, fmt, ...)	       \
-+	KUNIT_MEM_ASSERTION(test,					       \
-+			    KUNIT_ASSERTION,				       \
-+			    left, !=, right,				       \
-+			    size,					       \
-+			    fmt,					       \
-+			    ##__VA_ARGS__)
-+
-+/**
-  * KUNIT_ASSERT_NULL() - Asserts that pointers @ptr is null.
-  * @test: The test context object.
-  * @ptr: an arbitrary pointer.
-@@ -1295,9 +1726,12 @@ do {									       \
-  * Define function @name_gen_params which uses @array to generate parameters.
-  */
- #define KUNIT_ARRAY_PARAM(name, array, get_desc)						\
-	static const void *name##_gen_params(const void *prev, char *desc)			\
-+	static const void *name##_gen_params(struct kunit *test,				\
-+					     const void *prev, char *desc)			\
- 	{											\
- 		typeof((array)[0]) *__next = prev ? ((typeof(__next)) prev) + 1 : (array);	\
-+		if (!prev)									\
-+			kunit_register_params_array(test, array, ARRAY_SIZE(array), NULL);	\
- 		if (__next - (array) < ARRAY_SIZE((array))) {					\
- 			void (*__get_desc)(typeof(__next), char *) = get_desc;			\
- 			if (__get_desc)								\
-@@ -1307,6 +1741,55 @@ do {									       \
- 		return NULL;									\
- 	}
- 
-+/**
-+ * KUNIT_ARRAY_PARAM_DESC() - Define test parameter generator from an array.
-+ * @name:  prefix for the test parameter generator function.
-+ * @array: array of test parameters.
-+ * @desc_member: structure member from array element to use as description
-+ *
-+ * Define function @name_gen_params which uses @array to generate parameters.
-+ */
-+#define KUNIT_ARRAY_PARAM_DESC(name, array, desc_member)					\
-+	static const void *name##_gen_params(struct kunit *test,				\
-+					     const void *prev, char *desc)			\
-+	{											\
-+		typeof((array)[0]) *__next = prev ? ((typeof(__next)) prev) + 1 : (array);	\
-+		if (!prev)									\
-+			kunit_register_params_array(test, array, ARRAY_SIZE(array), NULL);	\
-+		if (__next - (array) < ARRAY_SIZE((array))) {					\
-+			strscpy(desc, __next->desc_member, KUNIT_PARAM_DESC_SIZE);		\
-+			return __next;								\
-+		}										\
-+		return NULL;									\
-+	}
-+
-+/**
-+ * kunit_register_params_array() - Register parameter array for a KUnit test.
-+ * @test: The KUnit test structure to which parameters will be added.
-+ * @array: An array of test parameters.
-+ * @param_count: Number of parameters.
-+ * @get_desc: Function that generates a string description for a given parameter
-+ * element.
-+ *
-+ * This macro initializes the @test's parameter array data, storing information
-+ * including the parameter array, its count, the element size, and the parameter
-+ * description function within `test->params_array`.
-+ *
-+ * Note: If using this macro in param_init(), kunit_array_gen_params()
-+ * will then need to be manually provided as the parameter generator function to
-+ * KUNIT_CASE_PARAM_WITH_INIT(). kunit_array_gen_params() is a KUnit
-+ * function that uses the registered array to generate parameters
-+ */
-+#define kunit_register_params_array(test, array, param_count, get_desc)				\
-+	do {											\
-+		struct kunit *_test = (test);							\
-+		const typeof((array)[0]) * _params_ptr = &(array)[0];				\
-+		_test->params_array.params = _params_ptr;					\
-+		_test->params_array.num_params = (param_count);					\
-+		_test->params_array.elem_size = sizeof(*_params_ptr);				\
-+		_test->params_array.get_description = (get_desc);				\
-+	} while (0)
-+
- // TODO(dlatypov@google.com): consider eventually migrating users to explicitly
- // include resource.h themselves if they need it.
- #include <kunit/resource.h>
-diff --git a/include/kunit/try-catch.h b/include/kunit/try-catch.h
-index c507dd43119d..d4e1a5b98ed6 100644
--- a/include/kunit/try-catch.h
-+++ b/include/kunit/try-catch.h
-@@ -14,13 +14,11 @@
- 
- typedef void (*kunit_try_catch_func_t)(void *);
- 
-struct completion;
- struct kunit;
- 
- /**
-  * struct kunit_try_catch - provides a generic way to run code which might fail.
-  * @test: The test case that is currently being executed.
- * @try_completion: Completion that the control thread waits on while test runs.
-  * @try_result: Contains any errno obtained while running test case.
-  * @try: The function, the test case, to attempt to run.
-  * @catch: The function called if @try bails out.
-@@ -46,10 +44,10 @@ struct kunit;
- struct kunit_try_catch {
- 	/* private: internal use only. */
- 	struct kunit *test;
-	struct completion *try_completion;
- 	int try_result;
- 	kunit_try_catch_func_t try;
- 	kunit_try_catch_func_t catch;
-+	unsigned long timeout;
- 	void *context;
- };
- 
-diff --git a/include/kunit/visibility.h b/include/kunit/visibility.h
-new file mode 100644
-index 000000000000..7c34c8ffcf3b
--- /dev/null
-+++ b/include/kunit/visibility.h
-@@ -0,0 +1,33 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * KUnit API to allow symbols to be conditionally visible during KUnit
-+ * testing
-+ *
-+ * Copyright (C) 2022, Google LLC.
-+ * Author: Rae Moar <rmoar@google.com>
-+ */
-+
-+#ifndef _KUNIT_VISIBILITY_H
-+#define _KUNIT_VISIBILITY_H
-+
-+#if IS_ENABLED(CONFIG_KUNIT)
-+    /**
-+     * VISIBLE_IF_KUNIT - A macro that sets symbols to be static if
-+     * CONFIG_KUNIT is not enabled. Otherwise if CONFIG_KUNIT is enabled
-+     * there is no change to the symbol definition.
-+     */
-+    #define VISIBLE_IF_KUNIT
-+    /**
-+     * EXPORT_SYMBOL_IF_KUNIT(symbol) - Exports symbol into
-+     * EXPORTED_FOR_KUNIT_TESTING namespace only if CONFIG_KUNIT is
-+     * enabled. Must use MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING")
-+     * in test file in order to use symbols.
-+     * @symbol: the symbol identifier to export
-+     */
-+    #define EXPORT_SYMBOL_IF_KUNIT(symbol) EXPORT_SYMBOL_NS(symbol, "EXPORTED_FOR_KUNIT_TESTING")
-+#else
-+    #define VISIBLE_IF_KUNIT static
-+    #define EXPORT_SYMBOL_IF_KUNIT(symbol)
-+#endif
-+
-+#endif /* _KUNIT_VISIBILITY_H */
-diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h
-index cd6d8f260eab..7310841f4512 100644
--- a/include/kvm/arm_arch_timer.h
-+++ b/include/kvm/arm_arch_timer.h
-@@ -13,6 +13,9 @@
- enum kvm_arch_timers {
- 	TIMER_PTIMER,
- 	TIMER_VTIMER,
-+	NR_KVM_EL0_TIMERS,
-+	TIMER_HVTIMER = NR_KVM_EL0_TIMERS,
-+	TIMER_HPTIMER,
- 	NR_KVM_TIMERS
- };
- 
-@@ -21,17 +24,39 @@ enum kvm_arch_timer_regs {
- 	TIMER_REG_CVAL,
- 	TIMER_REG_TVAL,
- 	TIMER_REG_CTL,
-+	TIMER_REG_VOFF,
- };
- 
-struct arch_timer_context {
-	struct kvm_vcpu			*vcpu;
-+struct arch_timer_offset {
-+	/*
-+	 * If set, pointer to one of the offsets in the kvm's offset
-+	 * structure. If NULL, assume a zero offset.
-+	 */
-+	u64	*vm_offset;
-+	/*
-+	 * If set, pointer to one of the offsets in the vcpu's sysreg
-+	 * array. If NULL, assume a zero offset.
-+	 */
-+	u64	*vcpu_offset;
-+};
- 
-	/* Timer IRQ */
-	struct kvm_irq_level		irq;
-+struct arch_timer_vm_data {
-+	/* Offset applied to the virtual timer/counter */
-+	u64	voffset;
-+	/* Offset applied to the physical timer/counter */
-+	u64	poffset;
- 
-+	/* The PPI for each timer, global to the VM */
-+	u8	ppi[NR_KVM_TIMERS];
-+};
-+
-+struct arch_timer_context {
- 	/* Emulated Timer (may be unused) */
- 	struct hrtimer			hrtimer;
-+	u64				ns_frac;
- 
-+	/* Offset for this counter/timer */
-+	struct arch_timer_offset	offset;
- 	/*
- 	 * We have multiple paths which can save/restore the timer state onto
- 	 * the hardware, so we need some way of keeping track of where the
-@@ -39,17 +64,27 @@ struct arch_timer_context {
- 	 */
- 	bool				loaded;
- 
-+	/* Output level of the timer IRQ */
-+	struct {
-+		bool			level;
-+	} irq;
-+
-+	/* Who am I? */
-+	enum kvm_arch_timers		timer_id;
-+
- 	/* Duplicated state from arch_timer.c for convenience */
- 	u32				host_timer_irq;
-	u32				host_timer_irq_flags;
- };
- 
- struct timer_map {
- 	struct arch_timer_context *direct_vtimer;
- 	struct arch_timer_context *direct_ptimer;
-+	struct arch_timer_context *emul_vtimer;
- 	struct arch_timer_context *emul_ptimer;
- };
- 
-+void get_timer_map(struct kvm_vcpu *vcpu, struct timer_map *map);
-+
- struct arch_timer_cpu {
- 	struct arch_timer_context timers[NR_KVM_TIMERS];
- 
-@@ -60,17 +95,17 @@ struct arch_timer_cpu {
- 	bool			enabled;
- };
- 
-int kvm_timer_hyp_init(bool);
-+int __init kvm_timer_hyp_init(bool has_gic);
- int kvm_timer_enable(struct kvm_vcpu *vcpu);
-int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu);
-+void kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu);
- void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu);
-+void kvm_timer_sync_nested(struct kvm_vcpu *vcpu);
- void kvm_timer_sync_user(struct kvm_vcpu *vcpu);
- bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu);
- void kvm_timer_update_run(struct kvm_vcpu *vcpu);
- void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu);
- 
-u64 kvm_arm_timer_get_reg(struct kvm_vcpu *, u64 regid);
-int kvm_arm_timer_set_reg(struct kvm_vcpu *, u64 regid, u64 value);
-+void kvm_timer_init_vm(struct kvm *kvm);
- 
- int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
- int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
-@@ -83,14 +118,17 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu);
- 
- void kvm_timer_init_vhe(void);
- 
-bool kvm_arch_timer_get_input_level(int vintid);
-
- #define vcpu_timer(v)	(&(v)->arch.timer_cpu)
- #define vcpu_get_timer(v,t)	(&vcpu_timer(v)->timers[(t)])
- #define vcpu_vtimer(v)	(&(v)->arch.timer_cpu.timers[TIMER_VTIMER])
- #define vcpu_ptimer(v)	(&(v)->arch.timer_cpu.timers[TIMER_PTIMER])
-+#define vcpu_hvtimer(v)	(&(v)->arch.timer_cpu.timers[TIMER_HVTIMER])
-+#define vcpu_hptimer(v)	(&(v)->arch.timer_cpu.timers[TIMER_HPTIMER])
- 
-#define arch_timer_ctx_index(ctx)	((ctx) - vcpu_timer((ctx)->vcpu)->timers)
-+#define arch_timer_ctx_index(ctx)	((ctx)->timer_id)
-+#define timer_context_to_vcpu(ctx)	container_of((ctx), struct kvm_vcpu, arch.timer_cpu.timers[(ctx)->timer_id])
-+#define timer_vm_data(ctx)		(&(timer_context_to_vcpu(ctx)->kvm->arch.timer_data))
-+#define timer_irq(ctx)			(timer_vm_data(ctx)->ppi[arch_timer_ctx_index(ctx)])
- 
- u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu,
- 			      enum kvm_arch_timers tmr,
-@@ -104,4 +142,48 @@ void kvm_arm_timer_write_sysreg(struct kvm_vcpu *vcpu,
- u32 timer_get_ctl(struct arch_timer_context *ctxt);
- u64 timer_get_cval(struct arch_timer_context *ctxt);
- 
-+/* CPU HP callbacks */
-+void kvm_timer_cpu_up(void);
-+void kvm_timer_cpu_down(void);
-+
-+/* CNTKCTL_EL1 valid bits as of DDI0487J.a */
-+#define CNTKCTL_VALID_BITS	(BIT(17) | GENMASK_ULL(9, 0))
-+
-+DECLARE_STATIC_KEY_FALSE(broken_cntvoff_key);
-+
-+static inline bool has_broken_cntvoff(void)
-+{
-+	return static_branch_unlikely(&broken_cntvoff_key);
-+}
-+
-+static inline bool has_cntpoff(void)
-+{
-+	return (has_vhe() && cpus_have_final_cap(ARM64_HAS_ECV_CNTPOFF));
-+}
-+
-+static inline u64 timer_get_offset(struct arch_timer_context *ctxt)
-+{
-+	u64 offset = 0;
-+
-+	if (!ctxt)
-+		return 0;
-+
-+	if (ctxt->offset.vm_offset)
-+		offset += *ctxt->offset.vm_offset;
-+	if (ctxt->offset.vcpu_offset)
-+		offset += *ctxt->offset.vcpu_offset;
-+
-+	return offset;
-+}
-+
-+static inline void timer_set_offset(struct arch_timer_context *ctxt, u64 offset)
-+{
-+	if (!ctxt->offset.vm_offset) {
-+		WARN(offset, "timer %d\n", arch_timer_ctx_index(ctxt));
-+		return;
-+	}
-+
-+	WRITE_ONCE(*ctxt->offset.vm_offset, offset);
-+}
-+
- #endif
-diff --git a/include/kvm/arm_hypercalls.h b/include/kvm/arm_hypercalls.h
-index 1188f116cf4e..2df152207ccd 100644
--- a/include/kvm/arm_hypercalls.h
-+++ b/include/kvm/arm_hypercalls.h
-@@ -6,7 +6,7 @@
- 
- #include <asm/kvm_emulate.h>
- 
-int kvm_hvc_call_handler(struct kvm_vcpu *vcpu);
-+int kvm_smccc_call_handler(struct kvm_vcpu *vcpu);
- 
- static inline u32 smccc_get_function(struct kvm_vcpu *vcpu)
- {
-@@ -43,9 +43,13 @@ static inline void smccc_set_retval(struct kvm_vcpu *vcpu,
- struct kvm_one_reg;
- 
- void kvm_arm_init_hypercalls(struct kvm *kvm);
-+void kvm_arm_teardown_hypercalls(struct kvm *kvm);
- int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu);
- int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices);
- int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
- int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
- 
-+int kvm_vm_smccc_has_attr(struct kvm *kvm, struct kvm_device_attr *attr);
-+int kvm_vm_smccc_set_attr(struct kvm *kvm, struct kvm_device_attr *attr);
-+
- #endif
-diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
-index c0b868ce6a8f..96754b51b411 100644
--- a/include/kvm/arm_pmu.h
-+++ b/include/kvm/arm_pmu.h
-@@ -8,28 +8,25 @@
- #define __ASM_ARM_KVM_PMU_H
- 
- #include <linux/perf_event.h>
-#include <asm/perf_event.h>
-+#include <linux/perf/arm_pmuv3.h>
- 
-#define ARMV8_PMU_CYCLE_IDX		(ARMV8_PMU_MAX_COUNTERS - 1)
-#define ARMV8_PMU_MAX_COUNTER_PAIRS	((ARMV8_PMU_MAX_COUNTERS + 1) >> 1)
-
-#ifdef CONFIG_HW_PERF_EVENTS
-+#define KVM_ARMV8_PMU_MAX_COUNTERS	32
- 
-+#if IS_ENABLED(CONFIG_HW_PERF_EVENTS) && IS_ENABLED(CONFIG_KVM)
- struct kvm_pmc {
- 	u8 idx;	/* index into the pmu->pmc array */
- 	struct perf_event *perf_event;
- };
- 
- struct kvm_pmu_events {
-	u32 events_host;
-	u32 events_guest;
-+	u64 events_host;
-+	u64 events_guest;
- };
- 
- struct kvm_pmu {
- 	struct irq_work overflow_work;
- 	struct kvm_pmu_events events;
-	struct kvm_pmc pmc[ARMV8_PMU_MAX_COUNTERS];
-	DECLARE_BITMAP(chained, ARMV8_PMU_MAX_COUNTER_PAIRS);
-+	struct kvm_pmc pmc[KVM_ARMV8_PMU_MAX_COUNTERS];
- 	int irq_num;
- 	bool created;
- 	bool irq_level;
-@@ -40,23 +37,17 @@ struct arm_pmu_entry {
- 	struct arm_pmu *arm_pmu;
- };
- 
-DECLARE_STATIC_KEY_FALSE(kvm_arm_pmu_available);
-
-static __always_inline bool kvm_arm_support_pmu_v3(void)
-{
-	return static_branch_likely(&kvm_arm_pmu_available);
-}
-
-+bool kvm_supports_guest_pmuv3(void);
- #define kvm_arm_pmu_irq_initialized(v)	((v)->arch.pmu.irq_num >= VGIC_NR_SGIS)
- u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx);
- void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val);
-u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu);
-+void kvm_pmu_set_counter_value_user(struct kvm_vcpu *vcpu, u64 select_idx, u64 val);
-+u64 kvm_pmu_implemented_counter_mask(struct kvm_vcpu *vcpu);
-+u64 kvm_pmu_accessible_counter_mask(struct kvm_vcpu *vcpu);
- u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1);
- void kvm_pmu_vcpu_init(struct kvm_vcpu *vcpu);
-void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu);
- void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu);
-void kvm_pmu_disable_counter_mask(struct kvm_vcpu *vcpu, u64 val);
-void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val);
-+void kvm_pmu_reprogram_counter_mask(struct kvm_vcpu *vcpu, u64 val);
- void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu);
- void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu);
- bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu);
-@@ -65,6 +56,7 @@ void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val);
- void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val);
- void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
- 				    u64 select_idx);
-+void kvm_vcpu_reload_pmu(struct kvm_vcpu *vcpu);
- int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu,
- 			    struct kvm_device_attr *attr);
- int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu,
-@@ -76,9 +68,10 @@ int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu);
- struct kvm_pmu_events *kvm_get_pmu_events(void);
- void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu);
- void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu);
-+void kvm_vcpu_pmu_resync_el0(void);
- 
- #define kvm_vcpu_has_pmu(vcpu)					\
-	(test_bit(KVM_ARM_VCPU_PMU_V3, (vcpu)->arch.features))
-+	(vcpu_has_feature(vcpu, KVM_ARM_VCPU_PMU_V3))
- 
- /*
-  * Updates the vcpu's view of the pmu events for this cpu.
-@@ -87,15 +80,23 @@ void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu);
-  */
- #define kvm_pmu_update_vcpu_events(vcpu)				\
- 	do {								\
-		if (!has_vhe() && kvm_vcpu_has_pmu(vcpu))		\
-+		if (!has_vhe() && system_supports_pmuv3())		\
- 			vcpu->arch.pmu.events = *kvm_get_pmu_events();	\
- 	} while (0)
- 
-+u8 kvm_arm_pmu_get_pmuver_limit(void);
-+u64 kvm_pmu_evtyper_mask(struct kvm *kvm);
-+int kvm_arm_set_default_pmu(struct kvm *kvm);
-+u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm);
-+
-+u64 kvm_vcpu_read_pmcr(struct kvm_vcpu *vcpu);
-+bool kvm_pmu_counter_is_hyp(struct kvm_vcpu *vcpu, unsigned int idx);
-+void kvm_pmu_nested_transition(struct kvm_vcpu *vcpu);
- #else
- struct kvm_pmu {
- };
- 
-static inline bool kvm_arm_support_pmu_v3(void)
-+static inline bool kvm_supports_guest_pmuv3(void)
- {
- 	return false;
- }
-@@ -108,15 +109,19 @@ static inline u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
- }
- static inline void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu,
- 					     u64 select_idx, u64 val) {}
-static inline u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu)
-+static inline void kvm_pmu_set_counter_value_user(struct kvm_vcpu *vcpu,
-+						  u64 select_idx, u64 val) {}
-+static inline u64 kvm_pmu_implemented_counter_mask(struct kvm_vcpu *vcpu)
-+{
-+	return 0;
-+}
-+static inline u64 kvm_pmu_accessible_counter_mask(struct kvm_vcpu *vcpu)
- {
- 	return 0;
- }
- static inline void kvm_pmu_vcpu_init(struct kvm_vcpu *vcpu) {}
-static inline void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) {}
- static inline void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu) {}
-static inline void kvm_pmu_disable_counter_mask(struct kvm_vcpu *vcpu, u64 val) {}
-static inline void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val) {}
-+static inline void kvm_pmu_reprogram_counter_mask(struct kvm_vcpu *vcpu, u64 val) {}
- static inline void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu) {}
- static inline void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) {}
- static inline bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
-@@ -156,6 +161,38 @@ static inline u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1)
- static inline void kvm_pmu_update_vcpu_events(struct kvm_vcpu *vcpu) {}
- static inline void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) {}
- static inline void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) {}
-+static inline void kvm_vcpu_reload_pmu(struct kvm_vcpu *vcpu) {}
-+static inline u8 kvm_arm_pmu_get_pmuver_limit(void)
-+{
-+	return 0;
-+}
-+static inline u64 kvm_pmu_evtyper_mask(struct kvm *kvm)
-+{
-+	return 0;
-+}
-+static inline void kvm_vcpu_pmu_resync_el0(void) {}
-+
-+static inline int kvm_arm_set_default_pmu(struct kvm *kvm)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm)
-+{
-+	return 0;
-+}
-+
-+static inline u64 kvm_vcpu_read_pmcr(struct kvm_vcpu *vcpu)
-+{
-+	return 0;
-+}
-+
-+static inline bool kvm_pmu_counter_is_hyp(struct kvm_vcpu *vcpu, unsigned int idx)
-+{
-+	return false;
-+}
-+
-+static inline void kvm_pmu_nested_transition(struct kvm_vcpu *vcpu) {}
- 
- #endif
- 
-diff --git a/include/kvm/arm_psci.h b/include/kvm/arm_psci.h
-index 6e55b9283789..cbaec804eb83 100644
--- a/include/kvm/arm_psci.h
-+++ b/include/kvm/arm_psci.h
-@@ -14,8 +14,10 @@
- #define KVM_ARM_PSCI_0_2	PSCI_VERSION(0, 2)
- #define KVM_ARM_PSCI_1_0	PSCI_VERSION(1, 0)
- #define KVM_ARM_PSCI_1_1	PSCI_VERSION(1, 1)
-+#define KVM_ARM_PSCI_1_2	PSCI_VERSION(1, 2)
-+#define KVM_ARM_PSCI_1_3	PSCI_VERSION(1, 3)
- 
-#define KVM_ARM_PSCI_LATEST	KVM_ARM_PSCI_1_1
-+#define KVM_ARM_PSCI_LATEST	KVM_ARM_PSCI_1_3
- 
- static inline int kvm_psci_version(struct kvm_vcpu *vcpu)
- {
-@@ -26,7 +28,7 @@ static inline int kvm_psci_version(struct kvm_vcpu *vcpu)
- 	 * revisions. It is thus safe to return the latest, unless
- 	 * userspace has instructed us otherwise.
- 	 */
-	if (test_bit(KVM_ARM_VCPU_PSCI_0_2, vcpu->arch.features)) {
-+	if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_PSCI_0_2)) {
- 		if (vcpu->kvm->arch.psci_version)
- 			return vcpu->kvm->arch.psci_version;
- 
-diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
-index 4df9e73a8bb5..b261fb3968d0 100644
--- a/include/kvm/arm_vgic.h
-+++ b/include/kvm/arm_vgic.h
-@@ -8,11 +8,12 @@
- #include <linux/bits.h>
- #include <linux/kvm.h>
- #include <linux/irqreturn.h>
-#include <linux/kref.h>
- #include <linux/mutex.h>
-+#include <linux/refcount.h>
- #include <linux/spinlock.h>
- #include <linux/static_key.h>
- #include <linux/types.h>
-+#include <linux/xarray.h>
- #include <kvm/iodev.h>
- #include <linux/list.h>
- #include <linux/jump_label.h>
-@@ -25,7 +26,6 @@
- #define VGIC_NR_SGIS		16
- #define VGIC_NR_PPIS		16
- #define VGIC_NR_PRIVATE_IRQS	(VGIC_NR_SGIS + VGIC_NR_PPIS)
-#define VGIC_MAX_PRIVATE	(VGIC_NR_PRIVATE_IRQS - 1)
- #define VGIC_MAX_SPI		1019
- #define VGIC_MAX_RESERVED	1023
- #define VGIC_MIN_LPI		8192
-@@ -38,6 +38,7 @@
- enum vgic_type {
- 	VGIC_V2,		/* Good ol' GICv2 */
- 	VGIC_V3,		/* New fancy GICv3 */
-+	VGIC_V5,		/* Newer, fancier GICv5 */
- };
- 
- /* same for all guests, as depending only on the _host's_ GIC model */
-@@ -58,6 +59,9 @@ struct vgic_global {
- 	/* virtual control interface mapping, HYP VA */
- 	void __iomem		*vctrl_hyp;
- 
-+	/* Physical CPU interface, kernel VA */
-+	void __iomem		*gicc_base;
-+
- 	/* Number of implemented list registers */
- 	int			nr_lr;
- 
-@@ -77,9 +81,12 @@ struct vgic_global {
- 	/* Pseudo GICv3 from outer space */
- 	bool			no_hw_deactivation;
- 
-	/* GIC system register CPU interface */
-+	/* GICv3 system register CPU interface */
- 	struct static_key_false gicv3_cpuif;
- 
-+	/* GICv3 compat mode on a GICv5 host */
-+	bool			has_gcie_v3_compat;
-+
- 	u32			ich_vtr_el2;
- };
- 
-@@ -116,7 +123,8 @@ struct irq_ops {
- 
- struct vgic_irq {
- 	raw_spinlock_t irq_lock;	/* Protects the content of the struct */
-	struct list_head lpi_list;	/* Used to link all LPIs together */
-+	u32 intid;			/* Guest visible INTID */
-+	struct rcu_head rcu;
- 	struct list_head ap_list;
- 
- 	struct kvm_vcpu *vcpu;		/* SGIs and PPIs: The VCPU
-@@ -130,15 +138,19 @@ struct vgic_irq {
- 					 * affinity reg (v3).
- 					 */
- 
-	u32 intid;			/* Guest visible INTID */
-	bool line_level;		/* Level only */
-	bool pending_latch;		/* The pending latch state used to calculate
-+	bool pending_release:1;		/* Used for LPIs only, unreferenced IRQ
-+					 * pending a release */
-+
-+	bool pending_latch:1;		/* The pending latch state used to calculate
- 					 * the pending state for both level
- 					 * and edge triggered IRQs. */
-	bool active;			/* not used for LPIs */
-	bool enabled;
-	bool hw;			/* Tied to HW IRQ */
-	struct kref refcount;		/* Used for LPIs */
-+	enum vgic_irq_config config:1;	/* Level or edge */
-+	bool line_level:1;		/* Level only */
-+	bool enabled:1;
-+	bool active:1;
-+	bool hw:1;			/* Tied to HW IRQ */
-+	bool on_lr:1;			/* Present in a CPU LR */
-+	refcount_t refcount;		/* Used for LPIs */
- 	u32 hwintid;			/* HW INTID number */
- 	unsigned int host_irq;		/* linux irq corresponding to hwintid */
- 	union {
-@@ -149,7 +161,6 @@ struct vgic_irq {
- 	u8 active_source;		/* GICv2 SGIs only */
- 	u8 priority;
- 	u8 group;			/* 0 == group 0, 1 == group 1 */
-	enum vgic_irq_config config;	/* Level or edge */
- 
- 	struct irq_ops *ops;
- 
-@@ -209,6 +220,12 @@ struct vgic_its {
- 	struct mutex		its_lock;
- 	struct list_head	device_list;
- 	struct list_head	collection_list;
-+
-+	/*
-+	 * Caches the (device_id, event_id) -> vgic_irq translation for
-+	 * LPIs that are mapped and enabled.
-+	 */
-+	struct xarray		translation_cache;
- };
- 
- struct vgic_state_iter;
-@@ -243,6 +260,12 @@ struct vgic_dist {
- 
- 	int			nr_spis;
- 
-+	/* The GIC maintenance IRQ for nested hypervisors. */
-+	u32			mi_intid;
-+
-+	/* Track the number of in-flight active SPIs */
-+	atomic_t		active_spis;
-+
- 	/* base addresses in guest physical address space: */
- 	gpa_t			vgic_dist_base;		/* distributor */
- 	union {
-@@ -255,14 +278,19 @@ struct vgic_dist {
- 	/* distributor enabled */
- 	bool			enabled;
- 
-+	/* Supports SGIs without active state */
-+	bool			nassgicap;
-+
- 	/* Wants SGIs without active state */
- 	bool			nassgireq;
- 
- 	struct vgic_irq		*spis;
- 
- 	struct vgic_io_device	dist_iodev;
-+	struct vgic_io_device	cpuif_iodev;
- 
- 	bool			has_its;
-+	bool			table_write_in_progress;
- 
- 	/*
- 	 * Contains the attributes and gpa of the LPI configuration table.
-@@ -272,13 +300,8 @@ struct vgic_dist {
- 	 */
- 	u64			propbaser;
- 
-	/* Protects the lpi_list and the count value below. */
-	raw_spinlock_t		lpi_list_lock;
-	struct list_head	lpi_list_head;
-	int			lpi_list_count;
-
-	/* LPI translation cache */
-	struct list_head	lpi_translation_cache;
-+#define LPI_XA_MARK_DEBUG_ITER	XA_MARK_0
-+	struct xarray		lpi_xa;
- 
- 	/* used by vgic-debug */
- 	struct vgic_state_iter *iter;
-@@ -328,7 +351,7 @@ struct vgic_cpu {
- 		struct vgic_v3_cpu_if	vgic_v3;
- 	};
- 
-	struct vgic_irq private_irqs[VGIC_NR_PRIVATE_IRQS];
-+	struct vgic_irq *private_irqs;
- 
- 	raw_spinlock_t ap_list_lock;	/* Protects the ap_list */
- 
-@@ -363,10 +386,12 @@ struct vgic_cpu {
- 
- extern struct static_key_false vgic_v2_cpuif_trap;
- extern struct static_key_false vgic_v3_cpuif_trap;
-+extern struct static_key_false vgic_v3_has_v2_compat;
- 
- int kvm_set_legacy_vgic_v2_addr(struct kvm *kvm, struct kvm_arm_device_addr *dev_addr);
- void kvm_vgic_early_init(struct kvm *kvm);
- int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu);
-+int kvm_vgic_vcpu_nv_init(struct kvm_vcpu *vcpu);
- int kvm_vgic_create(struct kvm *kvm, u32 type);
- void kvm_vgic_destroy(struct kvm *kvm);
- void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu);
-@@ -374,22 +399,25 @@ int kvm_vgic_map_resources(struct kvm *kvm);
- int kvm_vgic_hyp_init(void);
- void kvm_vgic_init_cpu_hardware(void);
- 
-int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid,
-			bool level, void *owner);
-+int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
-+			unsigned int intid, bool level, void *owner);
- int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq,
- 			  u32 vintid, struct irq_ops *ops);
- int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int vintid);
-+int kvm_vgic_get_map(struct kvm_vcpu *vcpu, unsigned int vintid);
- bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int vintid);
- 
- int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
- 
- void kvm_vgic_load(struct kvm_vcpu *vcpu);
- void kvm_vgic_put(struct kvm_vcpu *vcpu);
-void kvm_vgic_vmcr_sync(struct kvm_vcpu *vcpu);
-+
-+u16 vgic_v3_get_eisr(struct kvm_vcpu *vcpu);
-+u16 vgic_v3_get_elrsr(struct kvm_vcpu *vcpu);
-+u64 vgic_v3_get_misr(struct kvm_vcpu *vcpu);
- 
- #define irqchip_in_kernel(k)	(!!((k)->arch.vgic.in_kernel))
- #define vgic_initialized(k)	((k)->arch.vgic.initialized)
-#define vgic_ready(k)		((k)->arch.vgic.ready)
- #define vgic_valid_spi(k, i)	(((i) >= VGIC_NR_PRIVATE_IRQS) && \
- 			((i) < (k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS))
- 
-@@ -397,6 +425,7 @@ bool kvm_vcpu_has_pending_irqs(struct kvm_vcpu *vcpu);
- void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu);
- void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu);
- void kvm_vgic_reset_mapped_irq(struct kvm_vcpu *vcpu, u32 vintid);
-+void kvm_vgic_process_async_update(struct kvm_vcpu *vcpu);
- 
- void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1);
- 
-@@ -424,11 +453,16 @@ struct kvm_kernel_irq_routing_entry;
- int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int irq,
- 			       struct kvm_kernel_irq_routing_entry *irq_entry);
- 
-int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int irq,
-				 struct kvm_kernel_irq_routing_entry *irq_entry);
-+void kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int host_irq);
- 
- int vgic_v4_load(struct kvm_vcpu *vcpu);
- void vgic_v4_commit(struct kvm_vcpu *vcpu);
-int vgic_v4_put(struct kvm_vcpu *vcpu, bool need_db);
-+int vgic_v4_put(struct kvm_vcpu *vcpu);
-+
-+bool vgic_state_is_nested(struct kvm_vcpu *vcpu);
-+
-+/* CPU HP callbacks */
-+void kvm_vgic_cpu_up(void);
-+void kvm_vgic_cpu_down(void);
- 
- #endif /* __KVM_ARM_VGIC_H */
-diff --git a/include/kvm/iodev.h b/include/kvm/iodev.h
-index d75fc4365746..56619e33251e 100644
--- a/include/kvm/iodev.h
-+++ b/include/kvm/iodev.h
-@@ -55,10 +55,4 @@ static inline int kvm_iodevice_write(struct kvm_vcpu *vcpu,
- 				 : -EOPNOTSUPP;
- }
- 
-static inline void kvm_iodevice_destructor(struct kvm_io_device *dev)
-{
-	if (dev->ops->destructor)
-		dev->ops->destructor(dev);
-}
-
- #endif /* __KVM_IODEV_H__ */
-diff --git a/include/linux/a.out.h b/include/linux/a.out.h
-deleted file mode 100644
-index 600cf45645c6..000000000000
--- a/include/linux/a.out.h
-+++ /dev/null
-@@ -1,18 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __A_OUT_GNU_H__
-#define __A_OUT_GNU_H__
-
-#include <uapi/linux/a.out.h>
-
-#ifndef __ASSEMBLY__
-#ifdef linux
-#include <asm/page.h>
-#if defined(__i386__) || defined(__mc68000__)
-#else
-#ifndef SEGMENT_SIZE
-#define SEGMENT_SIZE	PAGE_SIZE
-#endif
-#endif
-#endif
-#endif /*__ASSEMBLY__ */
-#endif /* __A_OUT_GNU_H__ */
-diff --git a/include/linux/acpi.h b/include/linux/acpi.h
-index 6f64b2f3dc54..fbf0c3a65f59 100644
--- a/include/linux/acpi.h
-+++ b/include/linux/acpi.h
-@@ -8,33 +8,48 @@
- #ifndef _LINUX_ACPI_H
- #define _LINUX_ACPI_H
- 
-+#include <linux/cleanup.h>
- #include <linux/errno.h>
- #include <linux/ioport.h>	/* for struct resource */
-#include <linux/irqdomain.h>
- #include <linux/resource_ext.h>
- #include <linux/device.h>
-+#include <linux/mod_devicetable.h>
- #include <linux/property.h>
- #include <linux/uuid.h>
-+#include <linux/node.h>
-+
-+struct irq_domain;
-+struct irq_domain_ops;
- 
- #ifndef _LINUX
- #define _LINUX
- #endif
- #include <acpi/acpi.h>
-+#include <acpi/acpi_numa.h>
- 
- #ifdef	CONFIG_ACPI
- 
- #include <linux/list.h>
-#include <linux/mod_devicetable.h>
- #include <linux/dynamic_debug.h>
- #include <linux/module.h>
- #include <linux/mutex.h>
-+#include <linux/fw_table.h>
- 
- #include <acpi/acpi_bus.h>
- #include <acpi/acpi_drivers.h>
-#include <acpi/acpi_numa.h>
- #include <acpi/acpi_io.h>
- #include <asm/acpi.h>
- 
-+#ifdef CONFIG_ACPI_TABLE_LIB
-+#define EXPORT_SYMBOL_ACPI_LIB(x) EXPORT_SYMBOL_NS_GPL(x, "ACPI")
-+#define __init_or_acpilib
-+#define __initdata_or_acpilib
-+#else
-+#define EXPORT_SYMBOL_ACPI_LIB(x)
-+#define __init_or_acpilib __init
-+#define __initdata_or_acpilib __initdata
-+#endif
-+
- static inline acpi_handle acpi_device_handle(struct acpi_device *adev)
- {
- 	return adev ? adev->handle : NULL;
-@@ -68,19 +83,6 @@ static inline void acpi_free_fwnode_static(struct fwnode_handle *fwnode)
- 	kfree(fwnode);
- }
- 
-/**
- * ACPI_DEVICE_CLASS - macro used to describe an ACPI device with
- * the PCI-defined class-code information
- *
- * @_cls : the class, subclass, prog-if triple for this device
- * @_msk : the class mask for this device
- *
- * This macro is used to create a struct acpi_device_id that matches a
- * specific PCI class. The .id and .driver_data fields will be left
- * initialized with the default value.
- */
-#define ACPI_DEVICE_CLASS(_cls, _msk)	.cls = (_cls), .cls_msk = (_msk),
-
- static inline bool has_acpi_companion(struct device *dev)
- {
- 	return is_acpi_device_node(dev->fwnode);
-@@ -106,6 +108,7 @@ enum acpi_irq_model_id {
- 	ACPI_IRQ_MODEL_PLATFORM,
- 	ACPI_IRQ_MODEL_GIC,
- 	ACPI_IRQ_MODEL_LPIC,
-+	ACPI_IRQ_MODEL_RINTC,
- 	ACPI_IRQ_MODEL_COUNT
- };
- 
-@@ -130,21 +133,8 @@ enum acpi_address_range_id {
- 
- 
- /* Table Handlers */
-union acpi_subtable_headers {
-	struct acpi_subtable_header common;
-	struct acpi_hmat_structure hmat;
-	struct acpi_prmt_module_header prmt;
-	struct acpi_cedt_header cedt;
-};
-
- typedef int (*acpi_tbl_table_handler)(struct acpi_table_header *table);
- 
-typedef int (*acpi_tbl_entry_handler)(union acpi_subtable_headers *header,
-				      const unsigned long end);
-
-typedef int (*acpi_tbl_entry_handler_arg)(union acpi_subtable_headers *header,
-					  void *arg, const unsigned long end);
-
- /* Debugger support */
- 
- struct acpi_debugger_ops {
-@@ -218,14 +208,6 @@ static inline int acpi_debugger_notify_command_complete(void)
- 		(!entry) || (unsigned long)entry + sizeof(*entry) > end ||  \
- 		((struct acpi_subtable_header *)entry)->length < sizeof(*entry))
- 
-struct acpi_subtable_proc {
-	int id;
-	acpi_tbl_entry_handler handler;
-	acpi_tbl_entry_handler_arg handler_arg;
-	void *arg;
-	int count;
-};
-
- void __iomem *__acpi_map_table(unsigned long phys, unsigned long size);
- void __acpi_unmap_table(void __iomem *map, unsigned long size);
- int early_acpi_boot_init(void);
-@@ -240,15 +222,16 @@ void acpi_reserve_initial_tables (void);
- void acpi_table_init_complete (void);
- int acpi_table_init (void);
- 
-#ifdef CONFIG_ACPI_TABLE_LIB
-#define EXPORT_SYMBOL_ACPI_LIB(x) EXPORT_SYMBOL_NS_GPL(x, ACPI)
-#define __init_or_acpilib
-#define __initdata_or_acpilib
-#else
-#define EXPORT_SYMBOL_ACPI_LIB(x)
-#define __init_or_acpilib __init
-#define __initdata_or_acpilib __initdata
-#endif
-+static inline struct acpi_table_header *acpi_get_table_pointer(char *signature, u32 instance)
-+{
-+	struct acpi_table_header *table;
-+	int status = acpi_get_table(signature, instance, &table);
-+
-+	if (ACPI_FAILURE(status))
-+		return ERR_PTR(-ENOENT);
-+	return table;
-+}
-+DEFINE_FREE(acpi_put_table, struct acpi_table_header *, if (!IS_ERR_OR_NULL(_T)) acpi_put_table(_T))
- 
- int acpi_table_parse(char *id, acpi_tbl_table_handler handler);
- int __init_or_acpilib acpi_table_parse_entries(char *id,
-@@ -267,10 +250,7 @@ acpi_table_parse_cedt(enum acpi_cedt_type id,
- int acpi_parse_mcfg (struct acpi_table_header *header);
- void acpi_table_print_madt_entry (struct acpi_subtable_header *madt);
- 
-/* the following numa functions are architecture-dependent */
-void acpi_numa_slit_init (struct acpi_table_slit *slit);
-
-#if defined(CONFIG_X86) || defined(CONFIG_IA64) || defined(CONFIG_LOONGARCH)
-+#if defined(CONFIG_X86) || defined(CONFIG_LOONGARCH)
- void acpi_numa_processor_affinity_init (struct acpi_srat_cpu_affinity *pa);
- #else
- static inline void
-@@ -279,17 +259,24 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) { }
- 
- void acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa);
- 
-+#if defined(CONFIG_ARM64) || defined(CONFIG_LOONGARCH)
-+void acpi_arch_dma_setup(struct device *dev);
-+#else
-+static inline void acpi_arch_dma_setup(struct device *dev) { }
-+#endif
-+
- #ifdef CONFIG_ARM64
- void acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa);
-void acpi_arch_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size);
- #else
- static inline void
- acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa) { }
-static inline void
-acpi_arch_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size) { }
- #endif
- 
-int acpi_numa_memory_affinity_init (struct acpi_srat_mem_affinity *ma);
-+#ifdef CONFIG_RISCV
-+void acpi_numa_rintc_affinity_init(struct acpi_srat_rintc_affinity *pa);
-+#else
-+static inline void acpi_numa_rintc_affinity_init(struct acpi_srat_rintc_affinity *pa) { }
-+#endif
- 
- #ifndef PHYS_CPUID_INVALID
- typedef u32 phys_cpuid_t;
-@@ -306,6 +293,9 @@ static inline bool invalid_phys_cpuid(phys_cpuid_t phys_id)
- 	return phys_id == PHYS_CPUID_INVALID;
- }
- 
-+
-+int __init acpi_get_madt_revision(void);
-+
- /* Validate the processor object's proc_id */
- bool acpi_duplicate_processor_id(int proc_id);
- /* Processor _CTS control */
-@@ -331,6 +321,8 @@ int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 acpi_id,
- int acpi_unmap_cpu(int cpu);
- #endif /* CONFIG_ACPI_HOTPLUG_CPU */
- 
-+acpi_handle acpi_get_processor_handle(int cpu);
-+
- #ifdef CONFIG_ACPI_HOTPLUG_IOAPIC
- int acpi_get_ioapic_id(acpi_handle handle, u32 gsi_base, u64 *phys_addr);
- #endif
-@@ -350,14 +342,16 @@ static inline bool acpi_sci_irq_valid(void)
- }
- 
- extern int sbf_port;
-extern unsigned long acpi_realmode_flags;
- 
- int acpi_register_gsi (struct device *dev, u32 gsi, int triggering, int polarity);
- int acpi_gsi_to_irq (u32 gsi, unsigned int *irq);
- int acpi_isa_irq_to_gsi (unsigned isa_irq, u32 *gsi);
- 
-+typedef struct fwnode_handle *(*acpi_gsi_domain_disp_fn)(u32);
-+
- void acpi_set_irq_model(enum acpi_irq_model_id model,
-			struct fwnode_handle *(*)(u32));
-+			acpi_gsi_domain_disp_fn fn);
-+acpi_gsi_domain_disp_fn acpi_get_gsi_dispatcher(void);
- void acpi_set_gsi_to_irq_fallback(u32 (*)(u32));
- 
- struct irq_domain *acpi_irq_create_hierarchy(unsigned int flags,
-@@ -383,6 +377,7 @@ void acpi_unregister_gsi (u32 gsi);
- 
- struct pci_dev;
- 
-+struct acpi_prt_entry *acpi_pci_irq_lookup(struct pci_dev *dev, int pin);
- int acpi_pci_irq_enable (struct pci_dev *dev);
- void acpi_penalize_isa_irq(int irq, int active);
- bool acpi_isa_irq_available(int irq);
-@@ -407,7 +402,9 @@ extern bool acpi_is_pnp_device(struct acpi_device *);
- 
- #if defined(CONFIG_ACPI_WMI) || defined(CONFIG_ACPI_WMI_MODULE)
- 
-typedef void (*wmi_notify_handler) (u32 value, void *context);
-+typedef void (*wmi_notify_handler) (union acpi_object *data, void *context);
-+
-+int wmi_instance_count(const char *guid);
- 
- extern acpi_status wmi_evaluate_method(const char *guid, u8 instance,
- 					u32 method_id,
-@@ -420,7 +417,6 @@ extern acpi_status wmi_set_block(const char *guid, u8 instance,
- extern acpi_status wmi_install_notify_handler(const char *guid,
- 					wmi_notify_handler handler, void *data);
- extern acpi_status wmi_remove_notify_handler(const char *guid);
-extern acpi_status wmi_get_event_data(u32 event, struct acpi_buffer *out);
- extern bool wmi_has_guid(const char *guid);
- extern char *wmi_get_acpi_device_uid(const char *guid);
- 
-@@ -441,10 +437,27 @@ extern char *wmi_get_acpi_device_uid(const char *guid);
- 
- extern char acpi_video_backlight_string[];
- extern long acpi_is_video_device(acpi_handle handle);
-extern int acpi_blacklisted(void);
-+
- extern void acpi_osi_setup(char *str);
- extern bool acpi_osi_is_win8(void);
- 
-+#ifdef CONFIG_ACPI_THERMAL_LIB
-+int thermal_acpi_active_trip_temp(struct acpi_device *adev, int id, int *ret_temp);
-+int thermal_acpi_passive_trip_temp(struct acpi_device *adev, int *ret_temp);
-+int thermal_acpi_hot_trip_temp(struct acpi_device *adev, int *ret_temp);
-+int thermal_acpi_critical_trip_temp(struct acpi_device *adev, int *ret_temp);
-+#endif
-+
-+#ifdef CONFIG_ACPI_HMAT
-+int acpi_get_genport_coordinates(u32 uid, struct access_coordinate *coord);
-+#else
-+static inline int acpi_get_genport_coordinates(u32 uid,
-+					       struct access_coordinate *coord)
-+{
-+	return -EOPNOTSUPP;
-+}
-+#endif
-+
- #ifdef CONFIG_ACPI_NUMA
- int acpi_map_pxm_to_node(int pxm);
- int acpi_get_node(acpi_handle handle);
-@@ -483,8 +496,6 @@ static inline int acpi_get_node(acpi_handle handle)
- 	return 0;
- }
- #endif
-extern int acpi_paddr_to_node(u64 start_addr, u64 size);
-
- extern int pnpacpi_disabled;
- 
- #define PXM_INVAL	(-1)
-@@ -495,7 +506,7 @@ bool acpi_dev_resource_address_space(struct acpi_resource *ares,
- 				     struct resource_win *win);
- bool acpi_dev_resource_ext_address_space(struct acpi_resource *ares,
- 					 struct resource_win *win);
-unsigned long acpi_dev_irq_flags(u8 triggering, u8 polarity, u8 shareable);
-+unsigned long acpi_dev_irq_flags(u8 triggering, u8 polarity, u8 shareable, u8 wake_capable);
- unsigned int acpi_dev_get_irq_type(int triggering, int polarity);
- bool acpi_dev_resource_interrupt(struct acpi_resource *ares, int index,
- 				 struct resource *res);
-@@ -506,6 +517,7 @@ int acpi_dev_get_resources(struct acpi_device *adev, struct list_head *list,
- 			   void *preproc_data);
- int acpi_dev_get_dma_resources(struct acpi_device *adev,
- 			       struct list_head *list);
-+int acpi_dev_get_memory_resources(struct acpi_device *adev, struct list_head *list);
- int acpi_dev_filter_resource_type(struct acpi_resource *ares,
- 				  unsigned long types);
- 
-@@ -577,11 +589,17 @@ acpi_status acpi_run_osc(acpi_handle handle, struct acpi_osc_context *context);
- #define OSC_SB_CPCV2_SUPPORT			0x00000040
- #define OSC_SB_PCLPI_SUPPORT			0x00000080
- #define OSC_SB_OSLPI_SUPPORT			0x00000100
-+#define OSC_SB_FAST_THERMAL_SAMPLING_SUPPORT	0x00000200
-+#define OSC_SB_OVER_16_PSTATES_SUPPORT		0x00000400
-+#define OSC_SB_GED_SUPPORT			0x00000800
- #define OSC_SB_CPC_DIVERSE_HIGH_SUPPORT		0x00001000
-#define OSC_SB_GENERIC_INITIATOR_SUPPORT	0x00002000
-+#define OSC_SB_IRQ_RESOURCE_SOURCE_SUPPORT	0x00002000
- #define OSC_SB_CPC_FLEXIBLE_ADR_SPACE		0x00004000
-+#define OSC_SB_GENERIC_INITIATOR_SUPPORT	0x00020000
- #define OSC_SB_NATIVE_USB4_SUPPORT		0x00040000
-+#define OSC_SB_BATTERY_CHARGE_LIMITING_SUPPORT	0x00080000
- #define OSC_SB_PRM_SUPPORT			0x00200000
-+#define OSC_SB_FFH_OPR_SUPPORT			0x00400000
- 
- extern bool osc_sb_apei_support_acked;
- extern bool osc_pc_lpi_support_confirmed;
-@@ -705,20 +723,22 @@ int acpi_match_platform_list(const struct acpi_platform_list *plat);
- 
- extern void acpi_early_init(void);
- extern void acpi_subsystem_init(void);
-extern void arch_post_acpi_subsys_init(void);
- 
- extern int acpi_nvs_register(__u64 start, __u64 size);
- 
- extern int acpi_nvs_for_each_region(int (*func)(__u64, __u64, void *),
- 				    void *data);
- 
-+const struct acpi_device_id *acpi_match_acpi_device(const struct acpi_device_id *ids,
-+						    const struct acpi_device *adev);
-+
- const struct acpi_device_id *acpi_match_device(const struct acpi_device_id *ids,
- 					       const struct device *dev);
- 
- const void *acpi_device_get_match_data(const struct device *dev);
- extern bool acpi_driver_match_device(struct device *dev,
- 				     const struct device_driver *drv);
-int acpi_device_uevent_modalias(struct device *, struct kobj_uevent_env *);
-+int acpi_device_uevent_modalias(const struct device *, struct kobj_uevent_env *);
- int acpi_device_modalias(struct device *, char *, int);
- 
- struct platform_device *acpi_create_platform_device(struct acpi_device *,
-@@ -747,25 +767,29 @@ int acpi_reconfig_notifier_unregister(struct notifier_block *nb);
- int acpi_gtdt_init(struct acpi_table_header *table, int *platform_timer_count);
- int acpi_gtdt_map_ppi(int type);
- bool acpi_gtdt_c3stop(int type);
-int acpi_arch_timer_mem_init(struct arch_timer_mem *timer_mem, int *timer_count);
- #endif
- 
- #ifndef ACPI_HAVE_ARCH_SET_ROOT_POINTER
-static inline void acpi_arch_set_root_pointer(u64 addr)
-+static __always_inline void acpi_arch_set_root_pointer(u64 addr)
- {
- }
- #endif
- 
- #ifndef ACPI_HAVE_ARCH_GET_ROOT_POINTER
-static inline u64 acpi_arch_get_root_pointer(void)
-+static __always_inline u64 acpi_arch_get_root_pointer(void)
- {
- 	return 0;
- }
- #endif
- 
-+int acpi_get_local_u64_address(acpi_handle handle, u64 *addr);
- int acpi_get_local_address(acpi_handle handle, u32 *addr);
- const char *acpi_get_subsystem_id(acpi_handle handle);
- 
-+#ifdef CONFIG_ACPI_MRRM
-+int acpi_mrrm_max_mem_region(void);
-+#endif
-+
- #else	/* !CONFIG_ACPI */
- 
- #define acpi_disabled 1
-@@ -774,9 +798,10 @@ const char *acpi_get_subsystem_id(acpi_handle handle);
- #define ACPI_COMPANION_SET(dev, adev)	do { } while (0)
- #define ACPI_HANDLE(dev)		(NULL)
- #define ACPI_HANDLE_FWNODE(fwnode)	(NULL)
-#define ACPI_DEVICE_CLASS(_cls, _msk)	.cls = (0), .cls_msk = (0),
- 
-#include <acpi/acpi_numa.h>
-+/* Get rid of the -Wunused-variable for adev */
-+#define acpi_dev_uid_match(adev, uid2)			(adev && false)
-+#define acpi_dev_hid_uid_match(adev, hid2, uid2)	(adev && false)
- 
- struct fwnode_handle;
- 
-@@ -792,10 +817,9 @@ static inline bool acpi_dev_present(const char *hid, const char *uid, s64 hrv)
- 
- struct acpi_device;
- 
-static inline bool
-acpi_dev_hid_uid_match(struct acpi_device *adev, const char *hid2, const char *uid2)
-+static inline int acpi_dev_uid_to_integer(struct acpi_device *adev, u64 *integer)
- {
-	return false;
-+	return -ENODEV;
- }
- 
- static inline struct acpi_device *
-@@ -847,6 +871,11 @@ static inline struct fwnode_handle *acpi_fwnode_handle(struct acpi_device *adev)
- 	return NULL;
- }
- 
-+static inline acpi_handle acpi_device_handle(struct acpi_device *adev)
-+{
-+	return NULL;
-+}
-+
- static inline bool has_acpi_companion(struct device *dev)
- {
- 	return false;
-@@ -923,6 +952,12 @@ static inline int acpi_nvs_for_each_region(int (*func)(__u64, __u64, void *),
- 
- struct acpi_device_id;
- 
-+static inline const struct acpi_device_id *acpi_match_acpi_device(
-+	const struct acpi_device_id *ids, const struct acpi_device *adev)
-+{
-+	return NULL;
-+}
-+
- static inline const struct acpi_device_id *acpi_match_device(
- 	const struct acpi_device_id *ids, const struct device *dev)
- {
-@@ -940,6 +975,12 @@ static inline bool acpi_driver_match_device(struct device *dev,
- 	return false;
- }
- 
-+static inline bool acpi_check_dsm(acpi_handle handle, const guid_t *guid,
-+				  u64 rev, u64 funcs)
-+{
-+	return false;
-+}
-+
- static inline union acpi_object *acpi_evaluate_dsm(acpi_handle handle,
- 						   const guid_t *guid,
- 						   u64 rev, u64 func,
-@@ -948,7 +989,16 @@ static inline union acpi_object *acpi_evaluate_dsm(acpi_handle handle,
- 	return NULL;
- }
- 
-static inline int acpi_device_uevent_modalias(struct device *dev,
-+static inline union acpi_object *acpi_evaluate_dsm_typed(acpi_handle handle,
-+							 const guid_t *guid,
-+							 u64 rev, u64 func,
-+							 union acpi_object *argv4,
-+							 acpi_object_type type)
-+{
-+	return NULL;
-+}
-+
-+static inline int acpi_device_uevent_modalias(const struct device *dev,
- 				struct kobj_uevent_env *env)
- {
- 	return -ENODEV;
-@@ -977,8 +1027,7 @@ static inline enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev)
- 	return DEV_DMA_NOT_SUPPORTED;
- }
- 
-static inline int acpi_dma_get_range(struct device *dev, u64 *dma_addr,
-				     u64 *offset, u64 *size)
-+static inline int acpi_dma_get_range(struct device *dev, const struct bus_dma_region **map)
- {
- 	return -ENODEV;
- }
-@@ -1051,8 +1100,36 @@ static inline u32 acpi_osc_ctx_get_cxl_control(struct acpi_osc_context *context)
- 	return 0;
- }
- 
-+static inline bool acpi_sleep_state_supported(u8 sleep_state)
-+{
-+	return false;
-+}
-+
-+static inline acpi_handle acpi_get_processor_handle(int cpu)
-+{
-+	return NULL;
-+}
-+
-+static inline int acpi_mrrm_max_mem_region(void)
-+{
-+	return 1;
-+}
-+
- #endif	/* !CONFIG_ACPI */
- 
-+#ifdef CONFIG_ACPI_HMAT
-+int hmat_get_extended_linear_cache_size(struct resource *backing_res, int nid,
-+					resource_size_t *size);
-+#else
-+static inline int hmat_get_extended_linear_cache_size(struct resource *backing_res,
-+						      int nid, resource_size_t *size)
-+{
-+	return -EOPNOTSUPP;
-+}
-+#endif
-+
-+extern void arch_post_acpi_subsys_init(void);
-+
- #ifdef CONFIG_ACPI_HOTPLUG_IOAPIC
- int acpi_ioapic_add(acpi_handle root);
- #else
-@@ -1071,23 +1148,25 @@ void acpi_os_set_prepare_extended_sleep(int (*func)(u8 sleep_state,
- 
- acpi_status acpi_os_prepare_extended_sleep(u8 sleep_state,
- 					   u32 val_a, u32 val_b);
-#ifdef CONFIG_X86
- struct acpi_s2idle_dev_ops {
- 	struct list_head list_node;
- 	void (*prepare)(void);
-+	void (*check)(void);
- 	void (*restore)(void);
- };
-+#if defined(CONFIG_SUSPEND) && defined(CONFIG_X86)
- int acpi_register_lps0_dev(struct acpi_s2idle_dev_ops *arg);
- void acpi_unregister_lps0_dev(struct acpi_s2idle_dev_ops *arg);
-#endif /* CONFIG_X86 */
-#ifndef CONFIG_IA64
-void arch_reserve_mem_area(acpi_physical_address addr, size_t size);
-#else
-static inline void arch_reserve_mem_area(acpi_physical_address addr,
-					  size_t size)
-+#else /* CONFIG_SUSPEND && CONFIG_X86 */
-+static inline int acpi_register_lps0_dev(struct acpi_s2idle_dev_ops *arg)
-+{
-+	return -ENODEV;
-+}
-+static inline void acpi_unregister_lps0_dev(struct acpi_s2idle_dev_ops *arg)
- {
- }
-#endif /* CONFIG_X86 */
-+#endif /* CONFIG_SUSPEND && CONFIG_X86 */
-+void arch_reserve_mem_area(acpi_physical_address addr, size_t size);
- #else
- #define acpi_os_set_prepare_sleep(func, pm1a_ctrl, pm1b_ctrl) do { } while (0)
- #endif
-@@ -1125,8 +1204,7 @@ int acpi_subsys_suspend_noirq(struct device *dev);
- int acpi_subsys_suspend(struct device *dev);
- int acpi_subsys_freeze(struct device *dev);
- int acpi_subsys_poweroff(struct device *dev);
-void acpi_ec_mark_gpe_for_wake(void);
-void acpi_ec_set_gpe_wake_mask(u8 action);
-+int acpi_subsys_restore_early(struct device *dev);
- #else
- static inline int acpi_subsys_prepare(struct device *dev) { return 0; }
- static inline void acpi_subsys_complete(struct device *dev) {}
-@@ -1135,11 +1213,19 @@ static inline int acpi_subsys_suspend_noirq(struct device *dev) { return 0; }
- static inline int acpi_subsys_suspend(struct device *dev) { return 0; }
- static inline int acpi_subsys_freeze(struct device *dev) { return 0; }
- static inline int acpi_subsys_poweroff(struct device *dev) { return 0; }
-+static inline int acpi_subsys_restore_early(struct device *dev) { return 0; }
-+#endif
-+
-+#if defined(CONFIG_ACPI_EC) && defined(CONFIG_PM_SLEEP)
-+void acpi_ec_mark_gpe_for_wake(void);
-+void acpi_ec_set_gpe_wake_mask(u8 action);
-+#else
- static inline void acpi_ec_mark_gpe_for_wake(void) {}
- static inline void acpi_ec_set_gpe_wake_mask(u8 action) {}
- #endif
- 
- #ifdef CONFIG_ACPI
-+char *acpi_handle_path(acpi_handle handle);
- __printf(3, 4)
- void acpi_handle_printk(const char *level, acpi_handle handle,
- 			const char *fmt, ...);
-@@ -1202,7 +1288,8 @@ bool acpi_gpio_get_irq_resource(struct acpi_resource *ares,
- 				struct acpi_resource_gpio **agpio);
- bool acpi_gpio_get_io_resource(struct acpi_resource *ares,
- 			       struct acpi_resource_gpio **agpio);
-int acpi_dev_gpio_irq_get_by(struct acpi_device *adev, const char *name, int index);
-+int acpi_dev_gpio_irq_wake_get_by(struct acpi_device *adev, const char *con_id, int index,
-+				  bool *wake_capable);
- #else
- static inline bool acpi_gpio_get_irq_resource(struct acpi_resource *ares,
- 					      struct acpi_resource_gpio **agpio)
-@@ -1214,16 +1301,28 @@ static inline bool acpi_gpio_get_io_resource(struct acpi_resource *ares,
- {
- 	return false;
- }
-static inline int acpi_dev_gpio_irq_get_by(struct acpi_device *adev,
-					   const char *name, int index)
-+static inline int acpi_dev_gpio_irq_wake_get_by(struct acpi_device *adev, const char *con_id,
-+						int index, bool *wake_capable)
- {
- 	return -ENXIO;
- }
- #endif
- 
-+static inline int acpi_dev_gpio_irq_wake_get(struct acpi_device *adev, int index,
-+					     bool *wake_capable)
-+{
-+	return acpi_dev_gpio_irq_wake_get_by(adev, NULL, index, wake_capable);
-+}
-+
-+static inline int acpi_dev_gpio_irq_get_by(struct acpi_device *adev, const char *con_id,
-+					   int index)
-+{
-+	return acpi_dev_gpio_irq_wake_get_by(adev, con_id, index, NULL);
-+}
-+
- static inline int acpi_dev_gpio_irq_get(struct acpi_device *adev, int index)
- {
-	return acpi_dev_gpio_irq_get_by(adev, NULL, index);
-+	return acpi_dev_gpio_irq_wake_get_by(adev, NULL, index, NULL);
- }
- 
- /* Device properties */
-@@ -1256,9 +1355,6 @@ acpi_data_add_props(struct acpi_device_data *data, const guid_t *guid,
- int acpi_node_prop_get(const struct fwnode_handle *fwnode, const char *propname,
- 		       void **valptr);
- 
-struct fwnode_handle *acpi_get_next_subnode(const struct fwnode_handle *fwnode,
-					    struct fwnode_handle *child);
-
- struct acpi_probe_entry;
- typedef bool (*acpi_probe_entry_validate_subtbl)(struct acpi_subtable_header *,
- 						 struct acpi_probe_entry *);
-@@ -1289,6 +1385,8 @@ struct acpi_probe_entry {
- 	kernel_ulong_t driver_data;
- };
- 
-+void arch_sort_irqchip_probe(struct acpi_probe_entry *ap_head, int nr);
-+
- #define ACPI_DECLARE_PROBE_ENTRY(table, name, table_id, subtable,	\
- 				 valid, data, fn)			\
- 	static const struct acpi_probe_entry __acpi_probe_##name	\
-@@ -1356,13 +1454,6 @@ static inline int acpi_node_prop_get(const struct fwnode_handle *fwnode,
- }
- 
- static inline struct fwnode_handle *
-acpi_get_next_subnode(const struct fwnode_handle *fwnode,
-		      struct fwnode_handle *child)
-{
-	return NULL;
-}
-
-static inline struct fwnode_handle *
- acpi_graph_get_next_endpoint(const struct fwnode_handle *fwnode,
- 			     struct fwnode_handle *prev)
- {
-@@ -1408,18 +1499,25 @@ int acpi_parse_spcr(bool enable_earlycon, bool enable_console);
- #else
- static inline int acpi_parse_spcr(bool enable_earlycon, bool enable_console)
- {
-	return 0;
-+	return -ENODEV;
- }
- #endif
- 
- #if IS_ENABLED(CONFIG_ACPI_GENERIC_GSI)
- int acpi_irq_get(acpi_handle handle, unsigned int index, struct resource *res);
-+const struct cpumask *acpi_irq_get_affinity(acpi_handle handle,
-+					    unsigned int index);
- #else
- static inline
- int acpi_irq_get(acpi_handle handle, unsigned int index, struct resource *res)
- {
- 	return -EINVAL;
- }
-+static inline const struct cpumask *acpi_irq_get_affinity(acpi_handle handle,
-+							  unsigned int index)
-+{
-+	return NULL;
-+}
- #endif
- 
- #ifdef CONFIG_ACPI_LPIT
-@@ -1431,12 +1529,24 @@ static inline int lpit_read_residency_count_address(u64 *address)
- }
- #endif
- 
-+#ifdef CONFIG_ACPI_PROCESSOR_IDLE
-+#ifndef arch_get_idle_state_flags
-+static inline unsigned int arch_get_idle_state_flags(u32 arch_flags)
-+{
-+	return 0;
-+}
-+#endif
-+#endif /* CONFIG_ACPI_PROCESSOR_IDLE */
-+
- #ifdef CONFIG_ACPI_PPTT
- int acpi_pptt_cpu_is_thread(unsigned int cpu);
- int find_acpi_cpu_topology(unsigned int cpu, int level);
- int find_acpi_cpu_topology_cluster(unsigned int cpu);
- int find_acpi_cpu_topology_package(unsigned int cpu);
- int find_acpi_cpu_topology_hetero_id(unsigned int cpu);
-+void acpi_pptt_get_cpus_from_container(u32 acpi_cpu_id, cpumask_t *cpus);
-+int find_acpi_cache_level_from_id(u32 cache_id);
-+int acpi_pptt_get_cpumask_from_cache_id(u32 cache_id, cpumask_t *cpus);
- #else
- static inline int acpi_pptt_cpu_is_thread(unsigned int cpu)
- {
-@@ -1458,14 +1568,37 @@ static inline int find_acpi_cpu_topology_hetero_id(unsigned int cpu)
- {
- 	return -EINVAL;
- }
-+static inline void acpi_pptt_get_cpus_from_container(u32 acpi_cpu_id,
-+						     cpumask_t *cpus) { }
-+static inline int find_acpi_cache_level_from_id(u32 cache_id)
-+{
-+	return -ENOENT;
-+}
-+static inline int acpi_pptt_get_cpumask_from_cache_id(u32 cache_id,
-+						      cpumask_t *cpus)
-+{
-+	return -ENOENT;
-+}
- #endif
- 
-+void acpi_arch_init(void);
-+
- #ifdef CONFIG_ACPI_PCC
- void acpi_init_pcc(void);
- #else
- static inline void acpi_init_pcc(void) { }
- #endif
- 
-+#ifdef CONFIG_ACPI_FFH
-+void acpi_init_ffh(void);
-+extern int acpi_ffh_address_space_arch_setup(void *handler_ctxt,
-+					     void **region_ctxt);
-+extern int acpi_ffh_address_space_arch_handler(acpi_integer *value,
-+					       void *region_context);
-+#else
-+static inline void acpi_init_ffh(void) { }
-+#endif
-+
- #ifdef CONFIG_ACPI
- extern void acpi_device_notify(struct device *dev);
- extern void acpi_device_notify_remove(struct device *dev);
-@@ -1474,4 +1607,18 @@ static inline void acpi_device_notify(struct device *dev) { }
- static inline void acpi_device_notify_remove(struct device *dev) { }
- #endif
- 
-+static inline void acpi_use_parent_companion(struct device *dev)
-+{
-+	ACPI_COMPANION_SET(dev, ACPI_COMPANION(dev->parent));
-+}
-+
-+#ifdef CONFIG_ACPI_NUMA
-+bool acpi_node_backed_by_real_pxm(int nid);
-+#else
-+static inline bool acpi_node_backed_by_real_pxm(int nid)
-+{
-+	return false;
-+}
-+#endif
-+
- #endif	/*_LINUX_ACPI_H*/
-diff --git a/include/linux/acpi_agdi.h b/include/linux/acpi_agdi.h
-deleted file mode 100644
-index f477f0b452fa..000000000000
--- a/include/linux/acpi_agdi.h
-+++ /dev/null
-@@ -1,13 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-
-#ifndef __ACPI_AGDI_H__
-#define __ACPI_AGDI_H__
-
-#include <linux/acpi.h>
-
-#ifdef CONFIG_ACPI_AGDI
-void __init acpi_agdi_init(void);
-#else
-static inline void acpi_agdi_init(void) {}
-#endif
-#endif /* __ACPI_AGDI_H__ */
-diff --git a/include/linux/acpi_amd_wbrf.h b/include/linux/acpi_amd_wbrf.h
-new file mode 100644
-index 000000000000..898f31d536d4
--- /dev/null
-+++ b/include/linux/acpi_amd_wbrf.h
-@@ -0,0 +1,91 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * Wifi Band Exclusion Interface (AMD ACPI Implementation)
-+ * Copyright (C) 2023 Advanced Micro Devices
-+ */
-+
-+#ifndef _ACPI_AMD_WBRF_H
-+#define _ACPI_AMD_WBRF_H
-+
-+#include <linux/device.h>
-+#include <linux/notifier.h>
-+
-+/* The maximum number of frequency band ranges */
-+#define MAX_NUM_OF_WBRF_RANGES		11
-+
-+/* Record actions */
-+#define WBRF_RECORD_ADD		0x0
-+#define WBRF_RECORD_REMOVE	0x1
-+
-+/**
-+ * struct freq_band_range - Wifi frequency band range definition
-+ * @start: start frequency point (in Hz)
-+ * @end: end frequency point (in Hz)
-+ */
-+struct freq_band_range {
-+	u64		start;
-+	u64		end;
-+};
-+
-+/**
-+ * struct wbrf_ranges_in_out - wbrf ranges info
-+ * @num_of_ranges: total number of band ranges in this struct
-+ * @band_list: array of Wifi band ranges
-+ */
-+struct wbrf_ranges_in_out {
-+	u64			num_of_ranges;
-+	struct freq_band_range	band_list[MAX_NUM_OF_WBRF_RANGES];
-+};
-+
-+/**
-+ * enum wbrf_notifier_actions - wbrf notifier actions index
-+ * @WBRF_CHANGED: there was some frequency band updates. The consumers
-+ *               should retrieve the latest active frequency bands.
-+ */
-+enum wbrf_notifier_actions {
-+	WBRF_CHANGED,
-+};
-+
-+#if IS_ENABLED(CONFIG_AMD_WBRF)
-+bool acpi_amd_wbrf_supported_producer(struct device *dev);
-+int acpi_amd_wbrf_add_remove(struct device *dev, uint8_t action, struct wbrf_ranges_in_out *in);
-+bool acpi_amd_wbrf_supported_consumer(struct device *dev);
-+int amd_wbrf_retrieve_freq_band(struct device *dev, struct wbrf_ranges_in_out *out);
-+int amd_wbrf_register_notifier(struct notifier_block *nb);
-+int amd_wbrf_unregister_notifier(struct notifier_block *nb);
-+#else
-+static inline
-+bool acpi_amd_wbrf_supported_consumer(struct device *dev)
-+{
-+	return false;
-+}
-+
-+static inline
-+int acpi_amd_wbrf_add_remove(struct device *dev, uint8_t action, struct wbrf_ranges_in_out *in)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline
-+bool acpi_amd_wbrf_supported_producer(struct device *dev)
-+{
-+	return false;
-+}
-+static inline
-+int amd_wbrf_retrieve_freq_band(struct device *dev, struct wbrf_ranges_in_out *out)
-+{
-+	return -ENODEV;
-+}
-+static inline
-+int amd_wbrf_register_notifier(struct notifier_block *nb)
-+{
-+	return -ENODEV;
-+}
-+static inline
-+int amd_wbrf_unregister_notifier(struct notifier_block *nb)
-+{
-+	return -ENODEV;
-+}
-+#endif /* CONFIG_AMD_WBRF */
-+
-+#endif /* _ACPI_AMD_WBRF_H */
-diff --git a/include/linux/acpi_dma.h b/include/linux/acpi_dma.h
-index 72cedb916a9c..e748b2877602 100644
--- a/include/linux/acpi_dma.h
-+++ b/include/linux/acpi_dma.h
-@@ -11,10 +11,11 @@
- #ifndef __LINUX_ACPI_DMA_H
- #define __LINUX_ACPI_DMA_H
- 
-#include <linux/list.h>
-#include <linux/device.h>
- #include <linux/err.h>
- #include <linux/dmaengine.h>
-+#include <linux/types.h>
-+
-+struct device;
- 
- /**
-  * struct acpi_dma_spec - slave device DMA resources
-@@ -65,7 +66,6 @@ int devm_acpi_dma_controller_register(struct device *dev,
- 		struct dma_chan *(*acpi_dma_xlate)
- 		(struct acpi_dma_spec *, struct acpi_dma *),
- 		void *data);
-void devm_acpi_dma_controller_free(struct device *dev);
- 
- struct dma_chan *acpi_dma_request_slave_chan_by_index(struct device *dev,
- 						      size_t index);
-@@ -94,9 +94,6 @@ static inline int devm_acpi_dma_controller_register(struct device *dev,
- {
- 	return -ENODEV;
- }
-static inline void devm_acpi_dma_controller_free(struct device *dev)
-{
-}
- 
- static inline struct dma_chan *acpi_dma_request_slave_chan_by_index(
- 		struct device *dev, size_t index)
-diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h
-index b43be0987b19..d4ed5622cf2b 100644
--- a/include/linux/acpi_iort.h
-+++ b/include/linux/acpi_iort.h
-@@ -21,29 +21,29 @@
-  */
- #define IORT_SMMU_V3_PMCG_GENERIC        0x00000000 /* Generic SMMUv3 PMCG */
- #define IORT_SMMU_V3_PMCG_HISI_HIP08     0x00000001 /* HiSilicon HIP08 PMCG */
-+#define IORT_SMMU_V3_PMCG_HISI_HIP09     0x00000002 /* HiSilicon HIP09 PMCG */
- 
- int iort_register_domain_token(int trans_id, phys_addr_t base,
- 			       struct fwnode_handle *fw_node);
- void iort_deregister_domain_token(int trans_id);
- struct fwnode_handle *iort_find_domain_token(int trans_id);
-+int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id);
-+
- #ifdef CONFIG_ACPI_IORT
-void acpi_iort_init(void);
- u32 iort_msi_map_id(struct device *dev, u32 id);
- struct irq_domain *iort_get_device_domain(struct device *dev, u32 id,
- 					  enum irq_domain_bus_token bus_token);
- void acpi_configure_pmsi_domain(struct device *dev);
-int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id);
- void iort_get_rmr_sids(struct fwnode_handle *iommu_fwnode,
- 		       struct list_head *head);
- void iort_put_rmr_sids(struct fwnode_handle *iommu_fwnode,
- 		       struct list_head *head);
- /* IOMMU interface */
-int iort_dma_get_ranges(struct device *dev, u64 *size);
-+int iort_dma_get_ranges(struct device *dev, u64 *limit);
- int iort_iommu_configure_id(struct device *dev, const u32 *id_in);
- void iort_iommu_get_resv_regions(struct device *dev, struct list_head *head);
- phys_addr_t acpi_iort_dma_get_max_cpu_address(void);
- #else
-static inline void acpi_iort_init(void) { }
- static inline u32 iort_msi_map_id(struct device *dev, u32 id)
- { return id; }
- static inline struct irq_domain *iort_get_device_domain(
-@@ -55,7 +55,7 @@ void iort_get_rmr_sids(struct fwnode_handle *iommu_fwnode, struct list_head *hea
- static inline
- void iort_put_rmr_sids(struct fwnode_handle *iommu_fwnode, struct list_head *head) { }
- /* IOMMU interface */
-static inline int iort_dma_get_ranges(struct device *dev, u64 *size)
-+static inline int iort_dma_get_ranges(struct device *dev, u64 *limit)
- { return -ENODEV; }
- static inline int iort_iommu_configure_id(struct device *dev, const u32 *id_in)
- { return -ENODEV; }
-diff --git a/include/linux/acpi_mdio.h b/include/linux/acpi_mdio.h
-index 0a24ab7cb66f..8e2eefa9fbc0 100644
--- a/include/linux/acpi_mdio.h
-+++ b/include/linux/acpi_mdio.h
-@@ -9,7 +9,14 @@
- #include <linux/phy.h>
- 
- #if IS_ENABLED(CONFIG_ACPI_MDIO)
-int acpi_mdiobus_register(struct mii_bus *mdio, struct fwnode_handle *fwnode);
-+int __acpi_mdiobus_register(struct mii_bus *mdio, struct fwnode_handle *fwnode,
-+			    struct module *owner);
-+
-+static inline int
-+acpi_mdiobus_register(struct mii_bus *mdio, struct fwnode_handle *handle)
-+{
-+	return __acpi_mdiobus_register(mdio, handle, THIS_MODULE);
-+}
- #else /* CONFIG_ACPI_MDIO */
- static inline int
- acpi_mdiobus_register(struct mii_bus *mdio, struct fwnode_handle *fwnode)
-diff --git a/include/linux/acpi_pmtmr.h b/include/linux/acpi_pmtmr.h
-index 50d88bf1498d..0ded9220d379 100644
--- a/include/linux/acpi_pmtmr.h
-+++ b/include/linux/acpi_pmtmr.h
-@@ -26,6 +26,19 @@ static inline u32 acpi_pm_read_early(void)
- 	return acpi_pm_read_verified() & ACPI_PM_MASK;
- }
- 
-+/**
-+ * Register callback for suspend and resume event
-+ *
-+ * @cb Callback triggered on suspend and resume
-+ * @data Data passed with the callback
-+ */
-+void acpi_pmtmr_register_suspend_resume_callback(void (*cb)(void *data, bool suspend), void *data);
-+
-+/**
-+ * Remove registered callback for suspend and resume event
-+ */
-+void acpi_pmtmr_unregister_suspend_resume_callback(void);
-+
- #else
- 
- static inline u32 acpi_pm_read_early(void)
-diff --git a/include/linux/acpi_rimt.h b/include/linux/acpi_rimt.h
-new file mode 100644
-index 000000000000..fad3adc4d899
--- /dev/null
-+++ b/include/linux/acpi_rimt.h
-@@ -0,0 +1,28 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ *  Copyright (C) 2024-2025, Ventana Micro Systems Inc.
-+ *	Author: Sunil V L <sunilvl@ventanamicro.com>
-+ */
-+
-+#ifndef _ACPI_RIMT_H
-+#define _ACPI_RIMT_H
-+
-+#ifdef CONFIG_ACPI_RIMT
-+int rimt_iommu_register(struct device *dev);
-+#else
-+static inline int rimt_iommu_register(struct device *dev)
-+{
-+	return -ENODEV;
-+}
-+#endif
-+
-+#if defined(CONFIG_IOMMU_API) && defined(CONFIG_ACPI_RIMT)
-+int rimt_iommu_configure_id(struct device *dev, const u32 *id_in);
-+#else
-+static inline int rimt_iommu_configure_id(struct device *dev, const u32 *id_in)
-+{
-+	return -ENODEV;
-+}
-+#endif
-+
-+#endif /* _ACPI_RIMT_H */
-diff --git a/include/linux/adi-axi-common.h b/include/linux/adi-axi-common.h
-new file mode 100644
-index 000000000000..37962ba530df
--- /dev/null
-+++ b/include/linux/adi-axi-common.h
-@@ -0,0 +1,77 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * Analog Devices AXI common registers & definitions
-+ *
-+ * Copyright 2019 Analog Devices Inc.
-+ *
-+ * https://wiki.analog.com/resources/fpga/docs/axi_ip
-+ * https://wiki.analog.com/resources/fpga/docs/hdl/regmap
-+ */
-+
-+#include <linux/types.h>
-+
-+#ifndef ADI_AXI_COMMON_H_
-+#define ADI_AXI_COMMON_H_
-+
-+#define ADI_AXI_REG_VERSION			0x0000
-+#define ADI_AXI_REG_FPGA_INFO			0x001C
-+
-+#define ADI_AXI_PCORE_VER(major, minor, patch)	\
-+	(((major) << 16) | ((minor) << 8) | (patch))
-+
-+#define ADI_AXI_PCORE_VER_MAJOR(version)	(((version) >> 16) & 0xff)
-+#define ADI_AXI_PCORE_VER_MINOR(version)	(((version) >> 8) & 0xff)
-+#define ADI_AXI_PCORE_VER_PATCH(version)	((version) & 0xff)
-+
-+/**
-+ * adi_axi_pcore_ver_gteq() - check if a version is satisfied
-+ * @version: the full version read from the hardware
-+ * @major: the major version to compare against
-+ * @minor: the minor version to compare against
-+ *
-+ * ADI AXI IP Cores use semantic versioning, so this can be used to check for
-+ * feature availability.
-+ *
-+ * Return: true if the version is greater than or equal to the specified
-+ *         major and minor version, false otherwise.
-+ */
-+static inline bool adi_axi_pcore_ver_gteq(u32 version, u32 major, u32 minor)
-+{
-+	return ADI_AXI_PCORE_VER_MAJOR(version) > (major) ||
-+	       (ADI_AXI_PCORE_VER_MAJOR(version) == (major) &&
-+		ADI_AXI_PCORE_VER_MINOR(version) >= (minor));
-+}
-+
-+#define ADI_AXI_INFO_FPGA_TECH(info)            (((info) >> 24) & 0xff)
-+#define ADI_AXI_INFO_FPGA_FAMILY(info)          (((info) >> 16) & 0xff)
-+#define ADI_AXI_INFO_FPGA_SPEED_GRADE(info)     (((info) >> 8) & 0xff)
-+
-+enum adi_axi_fpga_technology {
-+	ADI_AXI_FPGA_TECH_UNKNOWN = 0,
-+	ADI_AXI_FPGA_TECH_SERIES7,
-+	ADI_AXI_FPGA_TECH_ULTRASCALE,
-+	ADI_AXI_FPGA_TECH_ULTRASCALE_PLUS,
-+};
-+
-+enum adi_axi_fpga_family {
-+	ADI_AXI_FPGA_FAMILY_UNKNOWN = 0,
-+	ADI_AXI_FPGA_FAMILY_ARTIX,
-+	ADI_AXI_FPGA_FAMILY_KINTEX,
-+	ADI_AXI_FPGA_FAMILY_VIRTEX,
-+	ADI_AXI_FPGA_FAMILY_ZYNQ,
-+};
-+
-+enum adi_axi_fpga_speed_grade {
-+	ADI_AXI_FPGA_SPEED_UNKNOWN      = 0,
-+	ADI_AXI_FPGA_SPEED_1    = 10,
-+	ADI_AXI_FPGA_SPEED_1L   = 11,
-+	ADI_AXI_FPGA_SPEED_1H   = 12,
-+	ADI_AXI_FPGA_SPEED_1HV  = 13,
-+	ADI_AXI_FPGA_SPEED_1LV  = 14,
-+	ADI_AXI_FPGA_SPEED_2    = 20,
-+	ADI_AXI_FPGA_SPEED_2L   = 21,
-+	ADI_AXI_FPGA_SPEED_2LV  = 22,
-+	ADI_AXI_FPGA_SPEED_3    = 30,
-+};
-+
-+#endif /* ADI_AXI_COMMON_H_ */
-diff --git a/include/linux/adreno-smmu-priv.h b/include/linux/adreno-smmu-priv.h
-index c637e0997f6d..d83c9175828f 100644
--- a/include/linux/adreno-smmu-priv.h
-+++ b/include/linux/adreno-smmu-priv.h
-@@ -45,11 +45,16 @@ struct adreno_smmu_fault_info {
-  *                 TTBR0 translation is enabled with the specified cfg
-  * @get_fault_info: Called by the GPU fault handler to get information about
-  *                  the fault
- * @set_stall:     Configure whether stall on fault (CFCFG) is enabled.  Call
- *                 before set_ttbr0_cfg().  If stalling on fault is enabled,
- *                 the GPU driver must call resume_translation()
-+ * @set_stall:     Configure whether stall on fault (CFCFG) is enabled. If
-+ *                 stalling on fault is enabled, the GPU driver must call
-+ *                 resume_translation()
-  * @resume_translation: Resume translation after a fault
-  *
-+ * @set_prr_bit:   [optional] Configure the GPU's Partially Resident
-+ *                 Region (PRR) bit in the ACTLR register.
-+ * @set_prr_addr:  [optional] Configure the PRR_CFG_*ADDR register with
-+ *                 the physical address of PRR page passed from GPU
-+ *                 driver.
-  *
-  * The GPU driver (drm/msm) and adreno-smmu work together for controlling
-  * the GPU's SMMU instance.  This is by necessity, as the GPU is directly
-@@ -67,6 +72,8 @@ struct adreno_smmu_priv {
-     void (*get_fault_info)(const void *cookie, struct adreno_smmu_fault_info *info);
-     void (*set_stall)(const void *cookie, bool enabled);
-     void (*resume_translation)(const void *cookie, bool terminate);
-+    void (*set_prr_bit)(const void *cookie, bool set);
-+    void (*set_prr_addr)(const void *cookie, phys_addr_t page_addr);
- };
- 
- #endif /* __ADRENO_SMMU_PRIV_H */
-diff --git a/include/linux/aer.h b/include/linux/aer.h
-index 97f64ba1b34a..02940be66324 100644
--- a/include/linux/aer.h
-+++ b/include/linux/aer.h
-@@ -16,13 +16,26 @@
- #define AER_CORRECTABLE			2
- #define DPC_FATAL			3
- 
-+/*
-+ * AER and DPC capabilities TLP Logging register sizes (PCIe r6.2, sec 7.8.4
-+ * & 7.9.14).
-+ */
-+#define PCIE_STD_NUM_TLP_HEADERLOG     4
-+#define PCIE_STD_MAX_TLP_PREFIXLOG     4
-+#define PCIE_STD_MAX_TLP_HEADERLOG	(PCIE_STD_NUM_TLP_HEADERLOG + 10)
-+
- struct pci_dev;
- 
-struct aer_header_log_regs {
-	unsigned int dw0;
-	unsigned int dw1;
-	unsigned int dw2;
-	unsigned int dw3;
-+struct pcie_tlp_log {
-+	union {
-+		u32 dw[PCIE_STD_MAX_TLP_HEADERLOG];
-+		struct {
-+			u32 _do_not_use[PCIE_STD_NUM_TLP_HEADERLOG];
-+			u32 prefix[PCIE_STD_MAX_TLP_PREFIXLOG];
-+		};
-+	};
-+	u8 header_len;		/* Length of the Logged TLP Header in DWORDs */
-+	bool flit;		/* TLP was logged when in Flit mode */
- };
- 
- struct aer_capability_regs {
-@@ -33,7 +46,7 @@ struct aer_capability_regs {
- 	u32 cor_status;
- 	u32 cor_mask;
- 	u32 cap_control;
-	struct aer_header_log_regs header_log;
-+	struct pcie_tlp_log header_log;
- 	u32 root_command;
- 	u32 root_status;
- 	u16 cor_err_source;
-@@ -41,30 +54,17 @@ struct aer_capability_regs {
- };
- 
- #if defined(CONFIG_PCIEAER)
-/* PCIe port driver needs this function to enable AER */
-int pci_enable_pcie_error_reporting(struct pci_dev *dev);
-int pci_disable_pcie_error_reporting(struct pci_dev *dev);
- int pci_aer_clear_nonfatal_status(struct pci_dev *dev);
-void pci_save_aer_state(struct pci_dev *dev);
-void pci_restore_aer_state(struct pci_dev *dev);
-+int pcie_aer_is_native(struct pci_dev *dev);
- #else
-static inline int pci_enable_pcie_error_reporting(struct pci_dev *dev)
-{
-	return -EINVAL;
-}
-static inline int pci_disable_pcie_error_reporting(struct pci_dev *dev)
-{
-	return -EINVAL;
-}
- static inline int pci_aer_clear_nonfatal_status(struct pci_dev *dev)
- {
- 	return -EINVAL;
- }
-static inline void pci_save_aer_state(struct pci_dev *dev) {}
-static inline void pci_restore_aer_state(struct pci_dev *dev) {}
-+static inline int pcie_aer_is_native(struct pci_dev *dev) { return 0; }
- #endif
- 
-void cper_print_aer(struct pci_dev *dev, int aer_severity,
-+void pci_print_aer(struct pci_dev *dev, int aer_severity,
- 		    struct aer_capability_regs *aer);
- int cper_severity_to_aer(int cper_severity);
- void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn,
-diff --git a/include/linux/ahci_platform.h b/include/linux/ahci_platform.h
-index 49e5383d4222..fe0760ce34c8 100644
--- a/include/linux/ahci_platform.h
-+++ b/include/linux/ahci_platform.h
-@@ -13,6 +13,7 @@
- 
- #include <linux/compiler.h>
- 
-+struct clk;
- struct device;
- struct ata_port_info;
- struct ahci_host_priv;
-@@ -21,8 +22,12 @@ struct scsi_host_template;
- 
- int ahci_platform_enable_phys(struct ahci_host_priv *hpriv);
- void ahci_platform_disable_phys(struct ahci_host_priv *hpriv);
-+struct clk *ahci_platform_find_clk(struct ahci_host_priv *hpriv,
-+				   const char *con_id);
- int ahci_platform_enable_clks(struct ahci_host_priv *hpriv);
- void ahci_platform_disable_clks(struct ahci_host_priv *hpriv);
-+int ahci_platform_deassert_rsts(struct ahci_host_priv *hpriv);
-+int ahci_platform_assert_rsts(struct ahci_host_priv *hpriv);
- int ahci_platform_enable_regulators(struct ahci_host_priv *hpriv);
- void ahci_platform_disable_regulators(struct ahci_host_priv *hpriv);
- int ahci_platform_enable_resources(struct ahci_host_priv *hpriv);
-@@ -32,7 +37,7 @@ struct ahci_host_priv *ahci_platform_get_resources(
- int ahci_platform_init_host(struct platform_device *pdev,
- 			    struct ahci_host_priv *hpriv,
- 			    const struct ata_port_info *pi_template,
-			    struct scsi_host_template *sht);
-+			    const struct scsi_host_template *sht);
- 
- void ahci_platform_shutdown(struct platform_device *pdev);
- 
-@@ -41,6 +46,7 @@ int ahci_platform_resume_host(struct device *dev);
- int ahci_platform_suspend(struct device *dev);
- int ahci_platform_resume(struct device *dev);
- 
-#define AHCI_PLATFORM_GET_RESETS	0x01
-+#define AHCI_PLATFORM_GET_RESETS	BIT(0)
-+#define AHCI_PLATFORM_RST_TRIGGER	BIT(1)
- 
- #endif /* _AHCI_PLATFORM_H */
-diff --git a/include/linux/alarmtimer.h b/include/linux/alarmtimer.h
-index 05e758b8b894..3ffa5341dce2 100644
--- a/include/linux/alarmtimer.h
-+++ b/include/linux/alarmtimer.h
-@@ -20,12 +20,6 @@ enum alarmtimer_type {
- 	ALARM_BOOTTIME_FREEZER,
- };
- 
-enum alarmtimer_restart {
-	ALARMTIMER_NORESTART,
-	ALARMTIMER_RESTART,
-};
-
-
- #define ALARMTIMER_STATE_INACTIVE	0x00
- #define ALARMTIMER_STATE_ENQUEUED	0x01
- 
-@@ -42,14 +36,14 @@ enum alarmtimer_restart {
- struct alarm {
- 	struct timerqueue_node	node;
- 	struct hrtimer		timer;
-	enum alarmtimer_restart	(*function)(struct alarm *, ktime_t now);
-+	void			(*function)(struct alarm *, ktime_t now);
- 	enum alarmtimer_type	type;
- 	int			state;
- 	void			*data;
- };
- 
- void alarm_init(struct alarm *alarm, enum alarmtimer_type type,
-		enum alarmtimer_restart (*function)(struct alarm *, ktime_t));
-+		void (*function)(struct alarm *, ktime_t));
- void alarm_start(struct alarm *alarm, ktime_t start);
- void alarm_start_relative(struct alarm *alarm, ktime_t start);
- void alarm_restart(struct alarm *alarm);
-diff --git a/include/linux/alcor_pci.h b/include/linux/alcor_pci.h
-index 8274ed525e9f..dcb1d37dabc2 100644
--- a/include/linux/alcor_pci.h
-+++ b/include/linux/alcor_pci.h
-@@ -11,6 +11,7 @@
- #define ALCOR_SD_CARD 0
- #define ALCOR_MS_CARD 1
- 
-+#define DRV_NAME_ALCOR_PCI			"alcor_pci"
- #define DRV_NAME_ALCOR_PCI_SDMMC		"alcor_sdmmc"
- #define DRV_NAME_ALCOR_PCI_MS			"alcor_ms"
- 
-@@ -268,13 +269,6 @@ struct alcor_pci_priv {
- 	unsigned long id; /* idr id */
- 
- 	struct alcor_dev_cfg	*cfg;
-
-	/* PCI ASPM related vars */
-	int pdev_cap_off;
-	u8  pdev_aspm_cap;
-	int parent_cap_off;
-	u8  parent_aspm_cap;
-	u8 ext_config_dev_aspm;
- };
- 
- void alcor_write8(struct alcor_pci_priv *priv, u8 val, unsigned int addr);
-diff --git a/include/linux/align.h b/include/linux/align.h
-index 2b4acec7b95a..55debf105a5d 100644
--- a/include/linux/align.h
-+++ b/include/linux/align.h
-@@ -2,14 +2,6 @@
- #ifndef _LINUX_ALIGN_H
- #define _LINUX_ALIGN_H
- 
-#include <linux/const.h>
-
-/* @a is a power of 2 value */
-#define ALIGN(x, a)		__ALIGN_KERNEL((x), (a))
-#define ALIGN_DOWN(x, a)	__ALIGN_KERNEL((x) - ((a) - 1), (a))
-#define __ALIGN_MASK(x, mask)	__ALIGN_KERNEL_MASK((x), (mask))
-#define PTR_ALIGN(p, a)		((typeof(p))ALIGN((unsigned long)(p), (a)))
-#define PTR_ALIGN_DOWN(p, a)	((typeof(p))ALIGN_DOWN((unsigned long)(p), (a)))
-#define IS_ALIGNED(x, a)		(((x) & ((typeof(x))(a) - 1)) == 0)
-+#include <vdso/align.h>
- 
- #endif	/* _LINUX_ALIGN_H */
-diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
-new file mode 100644
-index 000000000000..d40ac39bfbe8
--- /dev/null
-+++ b/include/linux/alloc_tag.h
-@@ -0,0 +1,268 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * allocation tagging
-+ */
-+#ifndef _LINUX_ALLOC_TAG_H
-+#define _LINUX_ALLOC_TAG_H
-+
-+#include <linux/bug.h>
-+#include <linux/codetag.h>
-+#include <linux/container_of.h>
-+#include <linux/preempt.h>
-+#include <asm/percpu.h>
-+#include <linux/cpumask.h>
-+#include <linux/smp.h>
-+#include <linux/static_key.h>
-+#include <linux/irqflags.h>
-+
-+struct alloc_tag_counters {
-+	u64 bytes;
-+	u64 calls;
-+};
-+
-+/*
-+ * An instance of this structure is created in a special ELF section at every
-+ * allocation callsite. At runtime, the special section is treated as
-+ * an array of these. Embedded codetag utilizes codetag framework.
-+ */
-+struct alloc_tag {
-+	struct codetag			ct;
-+	struct alloc_tag_counters __percpu	*counters;
-+} __aligned(8);
-+
-+struct alloc_tag_kernel_section {
-+	struct alloc_tag *first_tag;
-+	unsigned long count;
-+};
-+
-+struct alloc_tag_module_section {
-+	union {
-+		unsigned long start_addr;
-+		struct alloc_tag *first_tag;
-+	};
-+	unsigned long end_addr;
-+	/* used size */
-+	unsigned long size;
-+};
-+
-+#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
-+
-+#define CODETAG_EMPTY	((void *)1)
-+
-+static inline bool is_codetag_empty(union codetag_ref *ref)
-+{
-+	return ref->ct == CODETAG_EMPTY;
-+}
-+
-+static inline void set_codetag_empty(union codetag_ref *ref)
-+{
-+	if (ref)
-+		ref->ct = CODETAG_EMPTY;
-+}
-+
-+#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
-+
-+static inline bool is_codetag_empty(union codetag_ref *ref) { return false; }
-+
-+static inline void set_codetag_empty(union codetag_ref *ref)
-+{
-+	if (ref)
-+		ref->ct = NULL;
-+}
-+
-+#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
-+
-+#ifdef CONFIG_MEM_ALLOC_PROFILING
-+
-+#define ALLOC_TAG_SECTION_NAME	"alloc_tags"
-+
-+struct codetag_bytes {
-+	struct codetag *ct;
-+	s64 bytes;
-+};
-+
-+size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep);
-+
-+static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct)
-+{
-+	return container_of(ct, struct alloc_tag, ct);
-+}
-+
-+#if defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE)
-+/*
-+ * When percpu variables are required to be defined as weak, static percpu
-+ * variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION).
-+ * Instead we will account all module allocations to a single counter.
-+ */
-+DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
-+
-+#define DEFINE_ALLOC_TAG(_alloc_tag)						\
-+	static struct alloc_tag _alloc_tag __used __aligned(8)			\
-+	__section(ALLOC_TAG_SECTION_NAME) = {					\
-+		.ct = CODE_TAG_INIT,						\
-+		.counters = &_shared_alloc_tag };
-+
-+#else /* CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU && MODULE */
-+
-+#ifdef MODULE
-+
-+#define DEFINE_ALLOC_TAG(_alloc_tag)						\
-+	static struct alloc_tag _alloc_tag __used __aligned(8)			\
-+	__section(ALLOC_TAG_SECTION_NAME) = {					\
-+		.ct = CODE_TAG_INIT,						\
-+		.counters = NULL };
-+
-+#else  /* MODULE */
-+
-+#define DEFINE_ALLOC_TAG(_alloc_tag)						\
-+	static DEFINE_PER_CPU(struct alloc_tag_counters, _alloc_tag_cntr);	\
-+	static struct alloc_tag _alloc_tag __used __aligned(8)			\
-+	__section(ALLOC_TAG_SECTION_NAME) = {					\
-+		.ct = CODE_TAG_INIT,						\
-+		.counters = &_alloc_tag_cntr };
-+
-+#endif /* MODULE */
-+
-+#endif /* CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU && MODULE */
-+
-+DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
-+			mem_alloc_profiling_key);
-+
-+static inline bool mem_alloc_profiling_enabled(void)
-+{
-+	return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
-+				   &mem_alloc_profiling_key);
-+}
-+
-+static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag)
-+{
-+	struct alloc_tag_counters v = { 0, 0 };
-+	struct alloc_tag_counters *counter;
-+	int cpu;
-+
-+	for_each_possible_cpu(cpu) {
-+		counter = per_cpu_ptr(tag->counters, cpu);
-+		v.bytes += counter->bytes;
-+		v.calls += counter->calls;
-+	}
-+
-+	return v;
-+}
-+
-+#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
-+static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag)
-+{
-+	WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),
-+		  "alloc_tag was not cleared (got tag for %s:%u)\n",
-+		  ref->ct->filename, ref->ct->lineno);
-+
-+	WARN_ONCE(!tag, "current->alloc_tag not set\n");
-+}
-+
-+static inline void alloc_tag_sub_check(union codetag_ref *ref)
-+{
-+	WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n");
-+}
-+#else
-+static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {}
-+static inline void alloc_tag_sub_check(union codetag_ref *ref) {}
-+#endif
-+
-+/* Caller should verify both ref and tag to be valid */
-+static inline bool __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
-+{
-+	alloc_tag_add_check(ref, tag);
-+	if (!ref || !tag)
-+		return false;
-+
-+	ref->ct = &tag->ct;
-+	return true;
-+}
-+
-+static inline bool alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
-+{
-+	if (unlikely(!__alloc_tag_ref_set(ref, tag)))
-+		return false;
-+
-+	/*
-+	 * We need in increment the call counter every time we have a new
-+	 * allocation or when we split a large allocation into smaller ones.
-+	 * Each new reference for every sub-allocation needs to increment call
-+	 * counter because when we free each part the counter will be decremented.
-+	 */
-+	this_cpu_inc(tag->counters->calls);
-+	return true;
-+}
-+
-+static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes)
-+{
-+	if (likely(alloc_tag_ref_set(ref, tag)))
-+		this_cpu_add(tag->counters->bytes, bytes);
-+}
-+
-+static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes)
-+{
-+	struct alloc_tag *tag;
-+
-+	alloc_tag_sub_check(ref);
-+	if (!ref || !ref->ct)
-+		return;
-+
-+	if (is_codetag_empty(ref)) {
-+		ref->ct = NULL;
-+		return;
-+	}
-+
-+	tag = ct_to_alloc_tag(ref->ct);
-+
-+	this_cpu_sub(tag->counters->bytes, bytes);
-+	this_cpu_dec(tag->counters->calls);
-+
-+	ref->ct = NULL;
-+}
-+
-+static inline void alloc_tag_set_inaccurate(struct alloc_tag *tag)
-+{
-+	tag->ct.flags |= CODETAG_FLAG_INACCURATE;
-+}
-+
-+static inline bool alloc_tag_is_inaccurate(struct alloc_tag *tag)
-+{
-+	return !!(tag->ct.flags & CODETAG_FLAG_INACCURATE);
-+}
-+
-+#define alloc_tag_record(p)	((p) = current->alloc_tag)
-+
-+#else /* CONFIG_MEM_ALLOC_PROFILING */
-+
-+#define DEFINE_ALLOC_TAG(_alloc_tag)
-+static inline bool mem_alloc_profiling_enabled(void) { return false; }
-+static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag,
-+				 size_t bytes) {}
-+static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {}
-+static inline void alloc_tag_set_inaccurate(struct alloc_tag *tag) {}
-+static inline bool alloc_tag_is_inaccurate(struct alloc_tag *tag) { return false; }
-+#define alloc_tag_record(p)	do {} while (0)
-+
-+#endif /* CONFIG_MEM_ALLOC_PROFILING */
-+
-+#define alloc_hooks_tag(_tag, _do_alloc)				\
-+({									\
-+	typeof(_do_alloc) _res;						\
-+	if (mem_alloc_profiling_enabled()) {				\
-+		struct alloc_tag * __maybe_unused _old;			\
-+		_old = alloc_tag_save(_tag);				\
-+		_res = _do_alloc;					\
-+		alloc_tag_restore(_tag, _old);				\
-+	} else								\
-+		_res = _do_alloc;					\
-+	_res;								\
-+})
-+
-+#define alloc_hooks(_do_alloc)						\
-+({									\
-+	DEFINE_ALLOC_TAG(_alloc_tag);					\
-+	alloc_hooks_tag(&_alloc_tag, _do_alloc);			\
-+})
-+
-+#endif /* _LINUX_ALLOC_TAG_H */
-diff --git a/include/linux/amba/bus.h b/include/linux/amba/bus.h
-index e94cdf235f1d..9946276aff73 100644
--- a/include/linux/amba/bus.h
-+++ b/include/linux/amba/bus.h
-@@ -67,6 +67,7 @@ struct amba_device {
- 	struct clk		*pclk;
- 	struct device_dma_parameters dma_parms;
- 	unsigned int		periphid;
-+	struct mutex		periphid_lock;
- 	unsigned int		cid;
- 	struct amba_cs_uci_id	uci;
- 	unsigned int		irq[AMBA_NR_IRQS];
-@@ -104,24 +105,36 @@ enum amba_vendor {
- 	AMBA_VENDOR_LSI = 0xb6,
- };
- 
-extern struct bus_type amba_bustype;
-+extern const struct bus_type amba_bustype;
- 
-#define to_amba_device(d)	container_of(d, struct amba_device, dev)
-+#define to_amba_device(d)	container_of_const(d, struct amba_device, dev)
- 
- #define amba_get_drvdata(d)	dev_get_drvdata(&d->dev)
- #define amba_set_drvdata(d,p)	dev_set_drvdata(&d->dev, p)
- 
-+/*
-+ * use a macro to avoid include chaining to get THIS_MODULE
-+ */
-+#define amba_driver_register(drv) \
-+	__amba_driver_register(drv, THIS_MODULE)
-+
- #ifdef CONFIG_ARM_AMBA
-int amba_driver_register(struct amba_driver *);
-+int __amba_driver_register(struct amba_driver *, struct module *);
- void amba_driver_unregister(struct amba_driver *);
-+bool dev_is_amba(const struct device *dev);
- #else
-static inline int amba_driver_register(struct amba_driver *drv)
-+static inline int __amba_driver_register(struct amba_driver *drv,
-+					 struct module *owner)
- {
- 	return -EINVAL;
- }
- static inline void amba_driver_unregister(struct amba_driver *drv)
- {
- }
-+static inline bool dev_is_amba(const struct device *dev)
-+{
-+	return false;
-+}
- #endif
- 
- struct amba_device *amba_device_alloc(const char *, resource_size_t, size_t);
-diff --git a/include/linux/amba/clcd-regs.h b/include/linux/amba/clcd-regs.h
-deleted file mode 100644
-index 421b0fa90d6a..000000000000
--- a/include/linux/amba/clcd-regs.h
-+++ /dev/null
-@@ -1,87 +0,0 @@
-/*
- * David A Rusling
- *
- * Copyright (C) 2001 ARM Limited
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file COPYING in the main directory of this archive
- * for more details.
- */
-
-#ifndef AMBA_CLCD_REGS_H
-#define AMBA_CLCD_REGS_H
-
-/*
- * CLCD Controller Internal Register addresses
- */
-#define CLCD_TIM0		0x00000000
-#define CLCD_TIM1 		0x00000004
-#define CLCD_TIM2 		0x00000008
-#define CLCD_TIM3 		0x0000000c
-#define CLCD_UBAS 		0x00000010
-#define CLCD_LBAS 		0x00000014
-
-#define CLCD_PL110_IENB		0x00000018
-#define CLCD_PL110_CNTL		0x0000001c
-#define CLCD_PL110_STAT		0x00000020
-#define CLCD_PL110_INTR 	0x00000024
-#define CLCD_PL110_UCUR		0x00000028
-#define CLCD_PL110_LCUR		0x0000002C
-
-#define CLCD_PL111_CNTL		0x00000018
-#define CLCD_PL111_IENB		0x0000001c
-#define CLCD_PL111_RIS		0x00000020
-#define CLCD_PL111_MIS		0x00000024
-#define CLCD_PL111_ICR		0x00000028
-#define CLCD_PL111_UCUR		0x0000002c
-#define CLCD_PL111_LCUR		0x00000030
-
-#define CLCD_PALL 		0x00000200
-#define CLCD_PALETTE		0x00000200
-
-#define TIM2_PCD_LO_MASK	GENMASK(4, 0)
-#define TIM2_PCD_LO_BITS	5
-#define TIM2_CLKSEL		(1 << 5)
-#define TIM2_ACB_MASK		GENMASK(10, 6)
-#define TIM2_IVS		(1 << 11)
-#define TIM2_IHS		(1 << 12)
-#define TIM2_IPC		(1 << 13)
-#define TIM2_IOE		(1 << 14)
-#define TIM2_BCD		(1 << 26)
-#define TIM2_PCD_HI_MASK	GENMASK(31, 27)
-#define TIM2_PCD_HI_BITS	5
-#define TIM2_PCD_HI_SHIFT	27
-
-#define CNTL_LCDEN		(1 << 0)
-#define CNTL_LCDBPP1		(0 << 1)
-#define CNTL_LCDBPP2		(1 << 1)
-#define CNTL_LCDBPP4		(2 << 1)
-#define CNTL_LCDBPP8		(3 << 1)
-#define CNTL_LCDBPP16		(4 << 1)
-#define CNTL_LCDBPP16_565	(6 << 1)
-#define CNTL_LCDBPP16_444	(7 << 1)
-#define CNTL_LCDBPP24		(5 << 1)
-#define CNTL_LCDBW		(1 << 4)
-#define CNTL_LCDTFT		(1 << 5)
-#define CNTL_LCDMONO8		(1 << 6)
-#define CNTL_LCDDUAL		(1 << 7)
-#define CNTL_BGR		(1 << 8)
-#define CNTL_BEBO		(1 << 9)
-#define CNTL_BEPO		(1 << 10)
-#define CNTL_LCDPWR		(1 << 11)
-#define CNTL_LCDVCOMP(x)	((x) << 12)
-#define CNTL_LDMAFIFOTIME	(1 << 15)
-#define CNTL_WATERMARK		(1 << 16)
-
-/* ST Microelectronics variant bits */
-#define CNTL_ST_1XBPP_444	0x0
-#define CNTL_ST_1XBPP_5551	(1 << 17)
-#define CNTL_ST_1XBPP_565	(1 << 18)
-#define CNTL_ST_CDWID_12	0x0
-#define CNTL_ST_CDWID_16	(1 << 19)
-#define CNTL_ST_CDWID_18	(1 << 20)
-#define CNTL_ST_CDWID_24	((1 << 19)|(1 << 20))
-#define CNTL_ST_CEAEN		(1 << 21)
-#define CNTL_ST_LCDBPP24_PACKED	(6 << 1)
-
-#endif /* AMBA_CLCD_REGS_H */
-diff --git a/include/linux/amba/clcd.h b/include/linux/amba/clcd.h
-deleted file mode 100644
-index b6e0cbeaf533..000000000000
--- a/include/linux/amba/clcd.h
-+++ /dev/null
-@@ -1,290 +0,0 @@
-/*
- * linux/include/asm-arm/hardware/amba_clcd.h -- Integrator LCD panel.
- *
- * David A Rusling
- *
- * Copyright (C) 2001 ARM Limited
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file COPYING in the main directory of this archive
- * for more details.
- */
-#include <linux/fb.h>
-#include <linux/amba/clcd-regs.h>
-
-enum {
-	/* individual formats */
-	CLCD_CAP_RGB444		= (1 << 0),
-	CLCD_CAP_RGB5551	= (1 << 1),
-	CLCD_CAP_RGB565		= (1 << 2),
-	CLCD_CAP_RGB888		= (1 << 3),
-	CLCD_CAP_BGR444		= (1 << 4),
-	CLCD_CAP_BGR5551	= (1 << 5),
-	CLCD_CAP_BGR565		= (1 << 6),
-	CLCD_CAP_BGR888		= (1 << 7),
-
-	/* connection layouts */
-	CLCD_CAP_444		= CLCD_CAP_RGB444 | CLCD_CAP_BGR444,
-	CLCD_CAP_5551		= CLCD_CAP_RGB5551 | CLCD_CAP_BGR5551,
-	CLCD_CAP_565		= CLCD_CAP_RGB565 | CLCD_CAP_BGR565,
-	CLCD_CAP_888		= CLCD_CAP_RGB888 | CLCD_CAP_BGR888,
-
-	/* red/blue ordering */
-	CLCD_CAP_RGB		= CLCD_CAP_RGB444 | CLCD_CAP_RGB5551 |
-				  CLCD_CAP_RGB565 | CLCD_CAP_RGB888,
-	CLCD_CAP_BGR		= CLCD_CAP_BGR444 | CLCD_CAP_BGR5551 |
-				  CLCD_CAP_BGR565 | CLCD_CAP_BGR888,
-
-	CLCD_CAP_ALL		= CLCD_CAP_BGR | CLCD_CAP_RGB,
-};
-
-struct backlight_device;
-
-struct clcd_panel {
-	struct fb_videomode	mode;
-	signed short		width;	/* width in mm */
-	signed short		height;	/* height in mm */
-	u32			tim2;
-	u32			tim3;
-	u32			cntl;
-	u32			caps;
-	unsigned int		bpp:8,
-				fixedtimings:1,
-				grayscale:1;
-	unsigned int		connector;
-	struct backlight_device	*backlight;
-	/*
-	 * If the B/R lines are switched between the CLCD
-	 * and the panel we need to know this and not try to
-	 * compensate with the BGR bit in the control register.
-	 */
-	bool			bgr_connection;
-};
-
-struct clcd_regs {
-	u32			tim0;
-	u32			tim1;
-	u32			tim2;
-	u32			tim3;
-	u32			cntl;
-	unsigned long		pixclock;
-};
-
-struct clcd_fb;
-
-/*
- * the board-type specific routines
- */
-struct clcd_board {
-	const char *name;
-
-	/*
-	 * Optional.  Hardware capability flags.
-	 */
-	u32	caps;
-
-	/*
-	 * Optional.  Check whether the var structure is acceptable
-	 * for this display.
-	 */
-	int	(*check)(struct clcd_fb *fb, struct fb_var_screeninfo *var);
-
-	/*
-	 * Compulsory.  Decode fb->fb.var into regs->*.  In the case of
-	 * fixed timing, set regs->* to the register values required.
-	 */
-	void	(*decode)(struct clcd_fb *fb, struct clcd_regs *regs);
-
-	/*
-	 * Optional.  Disable any extra display hardware.
-	 */
-	void	(*disable)(struct clcd_fb *);
-
-	/*
-	 * Optional.  Enable any extra display hardware.
-	 */
-	void	(*enable)(struct clcd_fb *);
-
-	/*
-	 * Setup platform specific parts of CLCD driver
-	 */
-	int	(*setup)(struct clcd_fb *);
-
-	/*
-	 * mmap the framebuffer memory
-	 */
-	int	(*mmap)(struct clcd_fb *, struct vm_area_struct *);
-
-	/*
-	 * Remove platform specific parts of CLCD driver
-	 */
-	void	(*remove)(struct clcd_fb *);
-};
-
-struct amba_device;
-struct clk;
-
-/* this data structure describes each frame buffer device we find */
-struct clcd_fb {
-	struct fb_info		fb;
-	struct amba_device	*dev;
-	struct clk		*clk;
-	struct clcd_panel	*panel;
-	struct clcd_board	*board;
-	void			*board_data;
-	void __iomem		*regs;
-	u16			off_ienb;
-	u16			off_cntl;
-	u32			clcd_cntl;
-	u32			cmap[16];
-	bool			clk_enabled;
-};
-
-static inline void clcdfb_decode(struct clcd_fb *fb, struct clcd_regs *regs)
-{
-	struct fb_var_screeninfo *var = &fb->fb.var;
-	u32 val, cpl;
-
-	/*
-	 * Program the CLCD controller registers and start the CLCD
-	 */
-	val = ((var->xres / 16) - 1) << 2;
-	val |= (var->hsync_len - 1) << 8;
-	val |= (var->right_margin - 1) << 16;
-	val |= (var->left_margin - 1) << 24;
-	regs->tim0 = val;
-
-	val = var->yres;
-	if (fb->panel->cntl & CNTL_LCDDUAL)
-		val /= 2;
-	val -= 1;
-	val |= (var->vsync_len - 1) << 10;
-	val |= var->lower_margin << 16;
-	val |= var->upper_margin << 24;
-	regs->tim1 = val;
-
-	val = fb->panel->tim2;
-	val |= var->sync & FB_SYNC_HOR_HIGH_ACT  ? 0 : TIM2_IHS;
-	val |= var->sync & FB_SYNC_VERT_HIGH_ACT ? 0 : TIM2_IVS;
-
-	cpl = var->xres_virtual;
-	if (fb->panel->cntl & CNTL_LCDTFT)	  /* TFT */
-		/* / 1 */;
-	else if (!var->grayscale)		  /* STN color */
-		cpl = cpl * 8 / 3;
-	else if (fb->panel->cntl & CNTL_LCDMONO8) /* STN monochrome, 8bit */
-		cpl /= 8;
-	else					  /* STN monochrome, 4bit */
-		cpl /= 4;
-
-	regs->tim2 = val | ((cpl - 1) << 16);
-
-	regs->tim3 = fb->panel->tim3;
-
-	val = fb->panel->cntl;
-	if (var->grayscale)
-		val |= CNTL_LCDBW;
-
-	if (fb->panel->caps && fb->board->caps && var->bits_per_pixel >= 16) {
-		/*
-		 * if board and panel supply capabilities, we can support
-		 * changing BGR/RGB depending on supplied parameters. Here
-		 * we switch to what the framebuffer is providing if need
-		 * be, so if the framebuffer is BGR but the display connection
-		 * is RGB (first case) we switch it around. Vice versa mutatis
-		 * mutandis if the framebuffer is RGB but the display connection
-		 * is BGR, we flip it around.
-		 */
-		if (var->red.offset == 0)
-			val &= ~CNTL_BGR;
-		else
-			val |= CNTL_BGR;
-		if (fb->panel->bgr_connection)
-			val ^= CNTL_BGR;
-	}
-
-	switch (var->bits_per_pixel) {
-	case 1:
-		val |= CNTL_LCDBPP1;
-		break;
-	case 2:
-		val |= CNTL_LCDBPP2;
-		break;
-	case 4:
-		val |= CNTL_LCDBPP4;
-		break;
-	case 8:
-		val |= CNTL_LCDBPP8;
-		break;
-	case 16:
-		/*
-		 * PL110 cannot choose between 5551 and 565 modes in its
-		 * control register.  It is possible to use 565 with
-		 * custom external wiring.
-		 */
-		if (amba_part(fb->dev) == 0x110 ||
-		    var->green.length == 5)
-			val |= CNTL_LCDBPP16;
-		else if (var->green.length == 6)
-			val |= CNTL_LCDBPP16_565;
-		else
-			val |= CNTL_LCDBPP16_444;
-		break;
-	case 32:
-		val |= CNTL_LCDBPP24;
-		break;
-	}
-
-	regs->cntl = val;
-	regs->pixclock = var->pixclock;
-}
-
-static inline int clcdfb_check(struct clcd_fb *fb, struct fb_var_screeninfo *var)
-{
-	var->xres_virtual = var->xres = (var->xres + 15) & ~15;
-	var->yres_virtual = var->yres = (var->yres + 1) & ~1;
-
-#define CHECK(e,l,h) (var->e < l || var->e > h)
-	if (CHECK(right_margin, (5+1), 256) ||	/* back porch */
-	    CHECK(left_margin, (5+1), 256) ||	/* front porch */
-	    CHECK(hsync_len, (5+1), 256) ||
-	    var->xres > 4096 ||
-	    var->lower_margin > 255 ||		/* back porch */
-	    var->upper_margin > 255 ||		/* front porch */
-	    var->vsync_len > 32 ||
-	    var->yres > 1024)
-		return -EINVAL;
-#undef CHECK
-
-	/* single panel mode: PCD = max(PCD, 1) */
-	/* dual panel mode: PCD = max(PCD, 5) */
-
-	/*
-	 * You can't change the grayscale setting, and
-	 * we can only do non-interlaced video.
-	 */
-	if (var->grayscale != fb->fb.var.grayscale ||
-	    (var->vmode & FB_VMODE_MASK) != FB_VMODE_NONINTERLACED)
-		return -EINVAL;
-
-#define CHECK(e) (var->e != fb->fb.var.e)
-	if (fb->panel->fixedtimings &&
-	    (CHECK(xres)		||
-	     CHECK(yres)		||
-	     CHECK(bits_per_pixel)	||
-	     CHECK(pixclock)		||
-	     CHECK(left_margin)		||
-	     CHECK(right_margin)	||
-	     CHECK(upper_margin)	||
-	     CHECK(lower_margin)	||
-	     CHECK(hsync_len)		||
-	     CHECK(vsync_len)		||
-	     CHECK(sync)))
-		return -EINVAL;
-#undef CHECK
-
-	var->nonstd = 0;
-	var->accel_flags = 0;
-
-	return 0;
-}
-diff --git a/include/linux/amba/pl022.h b/include/linux/amba/pl022.h
-index 9bf58aac0df2..d7b07d0311e1 100644
--- a/include/linux/amba/pl022.h
-+++ b/include/linux/amba/pl022.h
-@@ -16,6 +16,7 @@
- #ifndef _SSP_PL022_H
- #define _SSP_PL022_H
- 
-+#include <linux/dmaengine.h>
- #include <linux/types.h>
- 
- /**
-@@ -224,6 +225,7 @@ struct dma_chan;
-  * struct pl022_ssp_master - device.platform_data for SPI controller devices.
-  * @bus_id: identifier for this bus
-  * @enable_dma: if true enables DMA driven transfers.
-+ * @dma_filter: callback filter for dma_request_channel.
-  * @dma_rx_param: parameter to locate an RX DMA channel.
-  * @dma_tx_param: parameter to locate a TX DMA channel.
-  * @autosuspend_delay: delay in ms following transfer completion before the
-@@ -235,7 +237,7 @@ struct dma_chan;
- struct pl022_ssp_controller {
- 	u16 bus_id;
- 	u8 enable_dma:1;
-	bool (*dma_filter)(struct dma_chan *chan, void *filter_param);
-+	dma_filter_fn dma_filter;
- 	void *dma_rx_param;
- 	void *dma_tx_param;
- 	int autosuspend_delay;
-diff --git a/include/linux/amba/pl093.h b/include/linux/amba/pl093.h
-deleted file mode 100644
-index b17166e3b49a..000000000000
--- a/include/linux/amba/pl093.h
-+++ /dev/null
-@@ -1,77 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/* linux/amba/pl093.h
- *
- * Copyright (c) 2008 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * AMBA PL093 SSMC (synchronous static memory controller)
- *  See DDI0236.pdf (r0p4) for more details
-*/
-
-#define SMB_BANK(x)	((x) * 0x20) /* each bank control set is 0x20 apart */
-
-/* Offsets for SMBxxxxRy registers */
-
-#define SMBIDCYR	(0x00)
-#define SMBWSTRDR	(0x04)
-#define SMBWSTWRR	(0x08)
-#define SMBWSTOENR	(0x0C)
-#define SMBWSTWENR	(0x10)
-#define SMBCR		(0x14)
-#define SMBSR		(0x18)
-#define SMBWSTBRDR	(0x1C)
-
-/* Masks for SMB registers */
-#define IDCY_MASK	(0xf)
-#define WSTRD_MASK	(0xf)
-#define WSTWR_MASK	(0xf)
-#define WSTOEN_MASK	(0xf)
-#define WSTWEN_MASK	(0xf)
-
-/* Notes from datasheet:
- *	WSTOEN <= WSTRD
- *	WSTWEN <= WSTWR
- *
- * WSTOEN is not used with nWAIT
- */
-
-/* SMBCR bit definitions */
-#define SMBCR_BIWRITEEN		(1 << 21)
-#define SMBCR_ADDRVALIDWRITEEN	(1 << 20)
-#define SMBCR_SYNCWRITE		(1 << 17)
-#define SMBCR_BMWRITE		(1 << 16)
-#define SMBCR_WRAPREAD		(1 << 14)
-#define SMBCR_BIREADEN		(1 << 13)
-#define SMBCR_ADDRVALIDREADEN	(1 << 12)
-#define SMBCR_SYNCREAD		(1 << 9)
-#define SMBCR_BMREAD		(1 << 8)
-#define SMBCR_SMBLSPOL		(1 << 6)
-#define SMBCR_WP		(1 << 3)
-#define SMBCR_WAITEN		(1 << 2)
-#define SMBCR_WAITPOL		(1 << 1)
-#define SMBCR_RBLE		(1 << 0)
-
-#define SMBCR_BURSTLENWRITE_MASK	(3 << 18)
-#define SMBCR_BURSTLENWRITE_4		(0 << 18)
-#define SMBCR_BURSTLENWRITE_8		(1 << 18)
-#define SMBCR_BURSTLENWRITE_RESERVED	(2 << 18)
-#define SMBCR_BURSTLENWRITE_CONTINUOUS	(3 << 18)
-
-#define SMBCR_BURSTLENREAD_MASK		(3 << 10)
-#define SMBCR_BURSTLENREAD_4		(0 << 10)
-#define SMBCR_BURSTLENREAD_8		(1 << 10)
-#define SMBCR_BURSTLENREAD_16		(2 << 10)
-#define SMBCR_BURSTLENREAD_CONTINUOUS	(3 << 10)
-
-#define SMBCR_MW_MASK			(3 << 4)
-#define SMBCR_MW_8BIT			(0 << 4)
-#define SMBCR_MW_16BIT			(1 << 4)
-#define SMBCR_MW_M32BIT			(2 << 4)
-
-/* SSMC status registers */
-#define SSMCCSR		(0x200)
-#define SSMCCR		(0x204)
-#define SSMCITCR	(0x208)
-#define SSMCITIP	(0x20C)
-#define SSMCITIOP	(0x210)
-diff --git a/include/linux/amba/serial.h b/include/linux/amba/serial.h
-index a1307b58cc2c..9120de05ead0 100644
--- a/include/linux/amba/serial.h
-+++ b/include/linux/amba/serial.h
-@@ -10,6 +10,11 @@
- #ifndef ASM_ARM_HARDWARE_SERIAL_AMBA_H
- #define ASM_ARM_HARDWARE_SERIAL_AMBA_H
- 
-+#ifndef __ASSEMBLY__
-+#include <linux/bitfield.h>
-+#include <linux/bits.h>
-+#endif
-+
- #include <linux/types.h>
- 
- /* -------------------------------------------------------------------------------
-@@ -70,141 +75,145 @@
- #define ZX_UART011_ICR		0x4c
- #define ZX_UART011_DMACR	0x50
- 
-#define UART011_DR_OE		(1 << 11)
-#define UART011_DR_BE		(1 << 10)
-#define UART011_DR_PE		(1 << 9)
-#define UART011_DR_FE		(1 << 8)
-
-#define UART01x_RSR_OE 		0x08
-#define UART01x_RSR_BE 		0x04
-#define UART01x_RSR_PE 		0x02
-#define UART01x_RSR_FE 		0x01
-
-#define UART011_FR_RI		0x100
-#define UART011_FR_TXFE		0x080
-#define UART011_FR_RXFF		0x040
-#define UART01x_FR_TXFF		0x020
-#define UART01x_FR_RXFE		0x010
-#define UART01x_FR_BUSY		0x008
-#define UART01x_FR_DCD 		0x004
-#define UART01x_FR_DSR 		0x002
-#define UART01x_FR_CTS 		0x001
-+#define UART011_DR_OE		BIT(11)
-+#define UART011_DR_BE		BIT(10)
-+#define UART011_DR_PE		BIT(9)
-+#define UART011_DR_FE		BIT(8)
-+
-+#define UART01x_RSR_OE		BIT(3)
-+#define UART01x_RSR_BE		BIT(2)
-+#define UART01x_RSR_PE		BIT(1)
-+#define UART01x_RSR_FE		BIT(0)
-+
-+#define UART011_FR_RI		BIT(8)
-+#define UART011_FR_TXFE		BIT(7)
-+#define UART011_FR_RXFF		BIT(6)
-+#define UART01x_FR_TXFF		(1 << 5)	/* used in ASM */
-+#define UART01x_FR_RXFE		BIT(4)
-+#define UART01x_FR_BUSY		(1 << 3)	/* used in ASM */
-+#define UART01x_FR_DCD		BIT(2)
-+#define UART01x_FR_DSR		BIT(1)
-+#define UART01x_FR_CTS		BIT(0)
- #define UART01x_FR_TMSK		(UART01x_FR_TXFF + UART01x_FR_BUSY)
- 
- /*
-  * Some bits of Flag Register on ZTE device have different position from
-  * standard ones.
-  */
-#define ZX_UART01x_FR_BUSY	0x100
-#define ZX_UART01x_FR_DSR	0x008
-#define ZX_UART01x_FR_CTS	0x002
-#define ZX_UART011_FR_RI	0x001
-
-#define UART011_CR_CTSEN	0x8000	/* CTS hardware flow control */
-#define UART011_CR_RTSEN	0x4000	/* RTS hardware flow control */
-#define UART011_CR_OUT2		0x2000	/* OUT2 */
-#define UART011_CR_OUT1		0x1000	/* OUT1 */
-#define UART011_CR_RTS		0x0800	/* RTS */
-#define UART011_CR_DTR		0x0400	/* DTR */
-#define UART011_CR_RXE		0x0200	/* receive enable */
-#define UART011_CR_TXE		0x0100	/* transmit enable */
-#define UART011_CR_LBE		0x0080	/* loopback enable */
-#define UART010_CR_RTIE		0x0040
-#define UART010_CR_TIE 		0x0020
-#define UART010_CR_RIE 		0x0010
-#define UART010_CR_MSIE		0x0008
-#define ST_UART011_CR_OVSFACT	0x0008	/* Oversampling factor */
-#define UART01x_CR_IIRLP	0x0004	/* SIR low power mode */
-#define UART01x_CR_SIREN	0x0002	/* SIR enable */
-#define UART01x_CR_UARTEN	0x0001	/* UART enable */
- 
-#define UART011_LCRH_SPS	0x80
-+#define ZX_UART01x_FR_BUSY	BIT(8)
-+#define ZX_UART01x_FR_DSR	BIT(3)
-+#define ZX_UART01x_FR_CTS	BIT(1)
-+#define ZX_UART011_FR_RI	BIT(0)
-+
-+#define UART011_CR_CTSEN	BIT(15)	/* CTS hardware flow control */
-+#define UART011_CR_RTSEN	BIT(14)	/* RTS hardware flow control */
-+#define UART011_CR_OUT2		BIT(13)	/* OUT2 */
-+#define UART011_CR_OUT1		BIT(12)	/* OUT1 */
-+#define UART011_CR_RTS		BIT(11)	/* RTS */
-+#define UART011_CR_DTR		BIT(10)	/* DTR */
-+#define UART011_CR_RXE		BIT(9)	/* receive enable */
-+#define UART011_CR_TXE		BIT(8)	/* transmit enable */
-+#define UART011_CR_LBE		BIT(7)	/* loopback enable */
-+#define UART010_CR_RTIE		BIT(6)
-+#define UART010_CR_TIE		BIT(5)
-+#define UART010_CR_RIE		BIT(4)
-+#define UART010_CR_MSIE		BIT(3)
-+#define ST_UART011_CR_OVSFACT	BIT(3)	/* Oversampling factor */
-+#define UART01x_CR_IIRLP	BIT(2)	/* SIR low power mode */
-+#define UART01x_CR_SIREN	BIT(1)	/* SIR enable */
-+#define UART01x_CR_UARTEN	BIT(0)	/* UART enable */
-+
-+#define UART011_LCRH_SPS	BIT(7)
- #define UART01x_LCRH_WLEN_8	0x60
- #define UART01x_LCRH_WLEN_7	0x40
- #define UART01x_LCRH_WLEN_6	0x20
- #define UART01x_LCRH_WLEN_5	0x00
-#define UART01x_LCRH_FEN	0x10
-#define UART01x_LCRH_STP2	0x08
-#define UART01x_LCRH_EPS	0x04
-#define UART01x_LCRH_PEN	0x02
-#define UART01x_LCRH_BRK	0x01
-
-#define ST_UART011_DMAWM_RX_1	(0 << 3)
-#define ST_UART011_DMAWM_RX_2	(1 << 3)
-#define ST_UART011_DMAWM_RX_4	(2 << 3)
-#define ST_UART011_DMAWM_RX_8	(3 << 3)
-#define ST_UART011_DMAWM_RX_16	(4 << 3)
-#define ST_UART011_DMAWM_RX_32	(5 << 3)
-#define ST_UART011_DMAWM_RX_48	(6 << 3)
-#define ST_UART011_DMAWM_TX_1	0
-#define ST_UART011_DMAWM_TX_2	1
-#define ST_UART011_DMAWM_TX_4	2
-#define ST_UART011_DMAWM_TX_8	3
-#define ST_UART011_DMAWM_TX_16	4
-#define ST_UART011_DMAWM_TX_32	5
-#define ST_UART011_DMAWM_TX_48	6
-
-#define UART010_IIR_RTIS	0x08
-#define UART010_IIR_TIS		0x04
-#define UART010_IIR_RIS		0x02
-#define UART010_IIR_MIS		0x01
-
-#define UART011_IFLS_RX1_8	(0 << 3)
-#define UART011_IFLS_RX2_8	(1 << 3)
-#define UART011_IFLS_RX4_8	(2 << 3)
-#define UART011_IFLS_RX6_8	(3 << 3)
-#define UART011_IFLS_RX7_8	(4 << 3)
-#define UART011_IFLS_TX1_8	(0 << 0)
-#define UART011_IFLS_TX2_8	(1 << 0)
-#define UART011_IFLS_TX4_8	(2 << 0)
-#define UART011_IFLS_TX6_8	(3 << 0)
-#define UART011_IFLS_TX7_8	(4 << 0)
-+#define UART01x_LCRH_FEN	BIT(4)
-+#define UART01x_LCRH_STP2	BIT(3)
-+#define UART01x_LCRH_EPS	BIT(2)
-+#define UART01x_LCRH_PEN	BIT(1)
-+#define UART01x_LCRH_BRK	BIT(0)
-+
-+#define ST_UART011_DMAWM_RX	GENMASK(5, 3)
-+#define ST_UART011_DMAWM_RX_1	FIELD_PREP_CONST(ST_UART011_DMAWM_RX, 0)
-+#define ST_UART011_DMAWM_RX_2	FIELD_PREP_CONST(ST_UART011_DMAWM_RX, 1)
-+#define ST_UART011_DMAWM_RX_4	FIELD_PREP_CONST(ST_UART011_DMAWM_RX, 2)
-+#define ST_UART011_DMAWM_RX_8	FIELD_PREP_CONST(ST_UART011_DMAWM_RX, 3)
-+#define ST_UART011_DMAWM_RX_16	FIELD_PREP_CONST(ST_UART011_DMAWM_RX, 4)
-+#define ST_UART011_DMAWM_RX_32	FIELD_PREP_CONST(ST_UART011_DMAWM_RX, 5)
-+#define ST_UART011_DMAWM_RX_48	FIELD_PREP_CONST(ST_UART011_DMAWM_RX, 6)
-+#define ST_UART011_DMAWM_TX	GENMASK(2, 0)
-+#define ST_UART011_DMAWM_TX_1	FIELD_PREP_CONST(ST_UART011_DMAWM_TX, 0)
-+#define ST_UART011_DMAWM_TX_2	FIELD_PREP_CONST(ST_UART011_DMAWM_TX, 1)
-+#define ST_UART011_DMAWM_TX_4	FIELD_PREP_CONST(ST_UART011_DMAWM_TX, 2)
-+#define ST_UART011_DMAWM_TX_8	FIELD_PREP_CONST(ST_UART011_DMAWM_TX, 3)
-+#define ST_UART011_DMAWM_TX_16	FIELD_PREP_CONST(ST_UART011_DMAWM_TX, 4)
-+#define ST_UART011_DMAWM_TX_32	FIELD_PREP_CONST(ST_UART011_DMAWM_TX, 5)
-+#define ST_UART011_DMAWM_TX_48	FIELD_PREP_CONST(ST_UART011_DMAWM_TX, 6)
-+
-+#define UART010_IIR_RTIS	BIT(3)
-+#define UART010_IIR_TIS		BIT(2)
-+#define UART010_IIR_RIS		BIT(1)
-+#define UART010_IIR_MIS		BIT(0)
-+
-+#define UART011_IFLS_RXIFLSEL	GENMASK(5, 3)
-+#define UART011_IFLS_RX1_8	FIELD_PREP_CONST(UART011_IFLS_RXIFLSEL, 0)
-+#define UART011_IFLS_RX2_8	FIELD_PREP_CONST(UART011_IFLS_RXIFLSEL, 1)
-+#define UART011_IFLS_RX4_8	FIELD_PREP_CONST(UART011_IFLS_RXIFLSEL, 2)
-+#define UART011_IFLS_RX6_8	FIELD_PREP_CONST(UART011_IFLS_RXIFLSEL, 3)
-+#define UART011_IFLS_RX7_8	FIELD_PREP_CONST(UART011_IFLS_RXIFLSEL, 4)
-+#define UART011_IFLS_TXIFLSEL	GENMASK(2, 0)
-+#define UART011_IFLS_TX1_8	FIELD_PREP_CONST(UART011_IFLS_TXIFLSEL, 0)
-+#define UART011_IFLS_TX2_8	FIELD_PREP_CONST(UART011_IFLS_TXIFLSEL, 1)
-+#define UART011_IFLS_TX4_8	FIELD_PREP_CONST(UART011_IFLS_TXIFLSEL, 2)
-+#define UART011_IFLS_TX6_8	FIELD_PREP_CONST(UART011_IFLS_TXIFLSEL, 3)
-+#define UART011_IFLS_TX7_8	FIELD_PREP_CONST(UART011_IFLS_TXIFLSEL, 4)
- /* special values for ST vendor with deeper fifo */
-#define UART011_IFLS_RX_HALF	(5 << 3)
-#define UART011_IFLS_TX_HALF	(5 << 0)
-
-#define UART011_OEIM		(1 << 10)	/* overrun error interrupt mask */
-#define UART011_BEIM		(1 << 9)	/* break error interrupt mask */
-#define UART011_PEIM		(1 << 8)	/* parity error interrupt mask */
-#define UART011_FEIM		(1 << 7)	/* framing error interrupt mask */
-#define UART011_RTIM		(1 << 6)	/* receive timeout interrupt mask */
-#define UART011_TXIM		(1 << 5)	/* transmit interrupt mask */
-#define UART011_RXIM		(1 << 4)	/* receive interrupt mask */
-#define UART011_DSRMIM		(1 << 3)	/* DSR interrupt mask */
-#define UART011_DCDMIM		(1 << 2)	/* DCD interrupt mask */
-#define UART011_CTSMIM		(1 << 1)	/* CTS interrupt mask */
-#define UART011_RIMIM		(1 << 0)	/* RI interrupt mask */
-
-#define UART011_OEIS		(1 << 10)	/* overrun error interrupt status */
-#define UART011_BEIS		(1 << 9)	/* break error interrupt status */
-#define UART011_PEIS		(1 << 8)	/* parity error interrupt status */
-#define UART011_FEIS		(1 << 7)	/* framing error interrupt status */
-#define UART011_RTIS		(1 << 6)	/* receive timeout interrupt status */
-#define UART011_TXIS		(1 << 5)	/* transmit interrupt status */
-#define UART011_RXIS		(1 << 4)	/* receive interrupt status */
-#define UART011_DSRMIS		(1 << 3)	/* DSR interrupt status */
-#define UART011_DCDMIS		(1 << 2)	/* DCD interrupt status */
-#define UART011_CTSMIS		(1 << 1)	/* CTS interrupt status */
-#define UART011_RIMIS		(1 << 0)	/* RI interrupt status */
-
-#define UART011_OEIC		(1 << 10)	/* overrun error interrupt clear */
-#define UART011_BEIC		(1 << 9)	/* break error interrupt clear */
-#define UART011_PEIC		(1 << 8)	/* parity error interrupt clear */
-#define UART011_FEIC		(1 << 7)	/* framing error interrupt clear */
-#define UART011_RTIC		(1 << 6)	/* receive timeout interrupt clear */
-#define UART011_TXIC		(1 << 5)	/* transmit interrupt clear */
-#define UART011_RXIC		(1 << 4)	/* receive interrupt clear */
-#define UART011_DSRMIC		(1 << 3)	/* DSR interrupt clear */
-#define UART011_DCDMIC		(1 << 2)	/* DCD interrupt clear */
-#define UART011_CTSMIC		(1 << 1)	/* CTS interrupt clear */
-#define UART011_RIMIC		(1 << 0)	/* RI interrupt clear */
-
-#define UART011_DMAONERR	(1 << 2)	/* disable dma on error */
-#define UART011_TXDMAE		(1 << 1)	/* enable transmit dma */
-#define UART011_RXDMAE		(1 << 0)	/* enable receive dma */
-
-#define UART01x_RSR_ANY		(UART01x_RSR_OE|UART01x_RSR_BE|UART01x_RSR_PE|UART01x_RSR_FE)
-#define UART01x_FR_MODEM_ANY	(UART01x_FR_DCD|UART01x_FR_DSR|UART01x_FR_CTS)
-+#define UART011_IFLS_RX_HALF	FIELD_PREP_CONST(UART011_IFLS_RXIFLSEL, 5)
-+#define UART011_IFLS_TX_HALF	FIELD_PREP_CONST(UART011_IFLS_TXIFLSEL, 5)
-+
-+#define UART011_OEIM		BIT(10)	/* overrun error interrupt mask */
-+#define UART011_BEIM		BIT(9)	/* break error interrupt mask */
-+#define UART011_PEIM		BIT(8)	/* parity error interrupt mask */
-+#define UART011_FEIM		BIT(7)	/* framing error interrupt mask */
-+#define UART011_RTIM		BIT(6)	/* receive timeout interrupt mask */
-+#define UART011_TXIM		BIT(5)	/* transmit interrupt mask */
-+#define UART011_RXIM		BIT(4)	/* receive interrupt mask */
-+#define UART011_DSRMIM		BIT(3)	/* DSR interrupt mask */
-+#define UART011_DCDMIM		BIT(2)	/* DCD interrupt mask */
-+#define UART011_CTSMIM		BIT(1)	/* CTS interrupt mask */
-+#define UART011_RIMIM		BIT(0)	/* RI interrupt mask */
-+
-+#define UART011_OEIS		BIT(10)	/* overrun error interrupt status */
-+#define UART011_BEIS		BIT(9)	/* break error interrupt status */
-+#define UART011_PEIS		BIT(8)	/* parity error interrupt status */
-+#define UART011_FEIS		BIT(7)	/* framing error interrupt status */
-+#define UART011_RTIS		BIT(6)	/* receive timeout interrupt status */
-+#define UART011_TXIS		BIT(5)	/* transmit interrupt status */
-+#define UART011_RXIS		BIT(4)	/* receive interrupt status */
-+#define UART011_DSRMIS		BIT(3)	/* DSR interrupt status */
-+#define UART011_DCDMIS		BIT(2)	/* DCD interrupt status */
-+#define UART011_CTSMIS		BIT(1)	/* CTS interrupt status */
-+#define UART011_RIMIS		BIT(0)	/* RI interrupt status */
-+
-+#define UART011_OEIC		BIT(10)	/* overrun error interrupt clear */
-+#define UART011_BEIC		BIT(9)	/* break error interrupt clear */
-+#define UART011_PEIC		BIT(8)	/* parity error interrupt clear */
-+#define UART011_FEIC		BIT(7)	/* framing error interrupt clear */
-+#define UART011_RTIC		BIT(6)	/* receive timeout interrupt clear */
-+#define UART011_TXIC		BIT(5)	/* transmit interrupt clear */
-+#define UART011_RXIC		BIT(4)	/* receive interrupt clear */
-+#define UART011_DSRMIC		BIT(3)	/* DSR interrupt clear */
-+#define UART011_DCDMIC		BIT(2)	/* DCD interrupt clear */
-+#define UART011_CTSMIC		BIT(1)	/* CTS interrupt clear */
-+#define UART011_RIMIC		BIT(0)	/* RI interrupt clear */
-+
-+#define UART011_DMAONERR	BIT(2)	/* disable dma on error */
-+#define UART011_TXDMAE		BIT(1)	/* enable transmit dma */
-+#define UART011_RXDMAE		BIT(0)	/* enable receive dma */
-+
-+#define UART01x_RSR_ANY		(UART01x_RSR_OE | UART01x_RSR_BE | UART01x_RSR_PE | UART01x_RSR_FE)
-+#define UART01x_FR_MODEM_ANY	(UART01x_FR_DCD | UART01x_FR_DSR | UART01x_FR_CTS)
- 
- #ifndef __ASSEMBLY__
- struct amba_device; /* in uncompress this is included but amba/bus.h is not */
-@@ -220,8 +229,8 @@ struct amba_pl011_data {
- 	bool dma_rx_poll_enable;
- 	unsigned int dma_rx_poll_rate;
- 	unsigned int dma_rx_poll_timeout;
-        void (*init) (void);
-	void (*exit) (void);
-+	void (*init)(void);
-+	void (*exit)(void);
- };
- #endif
- 
-diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h
-index 953e6f12fa1c..edcee9f5335a 100644
--- a/include/linux/amd-iommu.h
-+++ b/include/linux/amd-iommu.h
-@@ -12,151 +12,16 @@
- 
- struct amd_iommu;
- 
-/*
- * This is mainly used to communicate information back-and-forth
- * between SVM and IOMMU for setting up and tearing down posted
- * interrupt
- */
-struct amd_iommu_pi_data {
-	u32 ga_tag;
-	u32 prev_ga_tag;
-	u64 base;
-	bool is_guest_mode;
-	struct vcpu_data *vcpu_data;
-	void *ir_data;
-};
-
- #ifdef CONFIG_AMD_IOMMU
- 
- struct task_struct;
- struct pci_dev;
- 
-extern int amd_iommu_detect(void);
-extern int amd_iommu_init_hardware(void);
-
-/**
- * amd_iommu_init_device() - Init device for use with IOMMUv2 driver
- * @pdev: The PCI device to initialize
- * @pasids: Number of PASIDs to support for this device
- *
- * This function does all setup for the device pdev so that it can be
- * used with IOMMUv2.
- * Returns 0 on success or negative value on error.
- */
-extern int amd_iommu_init_device(struct pci_dev *pdev, int pasids);
-+extern void amd_iommu_detect(void);
- 
-/**
- * amd_iommu_free_device() - Free all IOMMUv2 related device resources
- *			     and disable IOMMUv2 usage for this device
- * @pdev: The PCI device to disable IOMMUv2 usage for'
- */
-extern void amd_iommu_free_device(struct pci_dev *pdev);
-
-/**
- * amd_iommu_bind_pasid() - Bind a given task to a PASID on a device
- * @pdev: The PCI device to bind the task to
- * @pasid: The PASID on the device the task should be bound to
- * @task: the task to bind
- *
- * The function returns 0 on success or a negative value on error.
- */
-extern int amd_iommu_bind_pasid(struct pci_dev *pdev, u32 pasid,
-				struct task_struct *task);
-
-/**
- * amd_iommu_unbind_pasid() - Unbind a PASID from its task on
- *			      a device
- * @pdev: The device of the PASID
- * @pasid: The PASID to unbind
- *
- * When this function returns the device is no longer using the PASID
- * and the PASID is no longer bound to its task.
- */
-extern void amd_iommu_unbind_pasid(struct pci_dev *pdev, u32 pasid);
-
-/**
- * amd_iommu_set_invalid_ppr_cb() - Register a call-back for failed
- *				    PRI requests
- * @pdev: The PCI device the call-back should be registered for
- * @cb: The call-back function
- *
- * The IOMMUv2 driver invokes this call-back when it is unable to
- * successfully handle a PRI request. The device driver can then decide
- * which PRI response the device should see. Possible return values for
- * the call-back are:
- *
- * - AMD_IOMMU_INV_PRI_RSP_SUCCESS - Send SUCCESS back to the device
- * - AMD_IOMMU_INV_PRI_RSP_INVALID - Send INVALID back to the device
- * - AMD_IOMMU_INV_PRI_RSP_FAIL    - Send Failure back to the device,
- *				     the device is required to disable
- *				     PRI when it receives this response
- *
- * The function returns 0 on success or negative value on error.
- */
-#define AMD_IOMMU_INV_PRI_RSP_SUCCESS	0
-#define AMD_IOMMU_INV_PRI_RSP_INVALID	1
-#define AMD_IOMMU_INV_PRI_RSP_FAIL	2
-
-typedef int (*amd_iommu_invalid_ppr_cb)(struct pci_dev *pdev,
-					u32 pasid,
-					unsigned long address,
-					u16);
-
-extern int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev,
-					amd_iommu_invalid_ppr_cb cb);
-
-#define PPR_FAULT_EXEC	(1 << 1)
-#define PPR_FAULT_READ  (1 << 2)
-#define PPR_FAULT_WRITE (1 << 5)
-#define PPR_FAULT_USER  (1 << 6)
-#define PPR_FAULT_RSVD  (1 << 7)
-#define PPR_FAULT_GN    (1 << 8)
-
-/**
- * amd_iommu_device_info() - Get information about IOMMUv2 support of a
- *			     PCI device
- * @pdev: PCI device to query information from
- * @info: A pointer to an amd_iommu_device_info structure which will contain
- *	  the information about the PCI device
- *
- * Returns 0 on success, negative value on error
- */
-
-#define AMD_IOMMU_DEVICE_FLAG_ATS_SUP     0x1    /* ATS feature supported */
-#define AMD_IOMMU_DEVICE_FLAG_PRI_SUP     0x2    /* PRI feature supported */
-#define AMD_IOMMU_DEVICE_FLAG_PASID_SUP   0x4    /* PASID context supported */
-#define AMD_IOMMU_DEVICE_FLAG_EXEC_SUP    0x8    /* Device may request execution
-						    on memory pages */
-#define AMD_IOMMU_DEVICE_FLAG_PRIV_SUP   0x10    /* Device may request
-						    super-user privileges */
-
-struct amd_iommu_device_info {
-	int max_pasids;
-	u32 flags;
-};
-
-extern int amd_iommu_device_info(struct pci_dev *pdev,
-				 struct amd_iommu_device_info *info);
-
-/**
- * amd_iommu_set_invalidate_ctx_cb() - Register a call-back for invalidating
- *				       a pasid context. This call-back is
- *				       invoked when the IOMMUv2 driver needs to
- *				       invalidate a PASID context, for example
- *				       because the task that is bound to that
- *				       context is about to exit.
- *
- * @pdev: The PCI device the call-back should be registered for
- * @cb: The call-back function
- */
-
-typedef void (*amd_iommu_invalidate_ctx)(struct pci_dev *pdev, u32 pasid);
-
-extern int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev,
-					   amd_iommu_invalidate_ctx cb);
- #else /* CONFIG_AMD_IOMMU */
- 
-static inline int amd_iommu_detect(void) { return -ENODEV; }
-+static inline void amd_iommu_detect(void) { }
- 
- #endif /* CONFIG_AMD_IOMMU */
- 
-@@ -165,10 +30,8 @@ static inline int amd_iommu_detect(void) { return -ENODEV; }
- /* IOMMU AVIC Function */
- extern int amd_iommu_register_ga_log_notifier(int (*notifier)(u32));
- 
-extern int
-amd_iommu_update_ga(int cpu, bool is_run, void *data);
-
-extern int amd_iommu_activate_guest_mode(void *data);
-+extern int amd_iommu_update_ga(void *data, int cpu, bool ga_log_intr);
-+extern int amd_iommu_activate_guest_mode(void *data, int cpu, bool ga_log_intr);
- extern int amd_iommu_deactivate_guest_mode(void *data);
- 
- #else /* defined(CONFIG_AMD_IOMMU) && defined(CONFIG_IRQ_REMAP) */
-@@ -179,13 +42,12 @@ amd_iommu_register_ga_log_notifier(int (*notifier)(u32))
- 	return 0;
- }
- 
-static inline int
-amd_iommu_update_ga(int cpu, bool is_run, void *data)
-+static inline int amd_iommu_update_ga(void *data, int cpu, bool ga_log_intr)
- {
- 	return 0;
- }
- 
-static inline int amd_iommu_activate_guest_mode(void *data)
-+static inline int amd_iommu_activate_guest_mode(void *data, int cpu, bool ga_log_intr)
- {
- 	return 0;
- }
-@@ -206,8 +68,12 @@ int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn,
- 		u64 *value);
- struct amd_iommu *get_amd_iommu(unsigned int idx);
- 
-#ifdef CONFIG_AMD_MEM_ENCRYPT
-int amd_iommu_snp_enable(void);
-+#ifdef CONFIG_KVM_AMD_SEV
-+int amd_iommu_snp_disable(void);
-+extern bool amd_iommu_sev_tio_supported(void);
-+#else
-+static inline int amd_iommu_snp_disable(void) { return 0; }
-+static inline bool amd_iommu_sev_tio_supported(void) { return false; }
- #endif
- 
- #endif /* _ASM_X86_AMD_IOMMU_H */
-diff --git a/include/linux/amd-pmf-io.h b/include/linux/amd-pmf-io.h
-new file mode 100644
-index 000000000000..6fa510f419c0
--- /dev/null
-+++ b/include/linux/amd-pmf-io.h
-@@ -0,0 +1,65 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * AMD Platform Management Framework Interface
-+ *
-+ * Copyright (c) 2023, Advanced Micro Devices, Inc.
-+ * All Rights Reserved.
-+ *
-+ * Authors: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
-+ *          Basavaraj Natikar <Basavaraj.Natikar@amd.com>
-+ */
-+
-+#ifndef AMD_PMF_IO_H
-+#define AMD_PMF_IO_H
-+
-+#include <linux/types.h>
-+
-+/**
-+ * enum sfh_message_type - Query the SFH message type
-+ * @MT_HPD: Message ID to know the Human presence info from MP2 FW
-+ * @MT_ALS: Message ID to know the Ambient light info from MP2 FW
-+ * @MT_SRA: Message ID to know the SRA data from MP2 FW
-+ */
-+enum sfh_message_type {
-+	MT_HPD,
-+	MT_ALS,
-+	MT_SRA,
-+};
-+
-+/**
-+ * enum sfh_hpd_info - Query the Human presence information
-+ * @SFH_NOT_DETECTED: Check the HPD connection information from MP2 FW
-+ * @SFH_USER_PRESENT: Check if the user is present from HPD sensor
-+ * @SFH_USER_AWAY: Check if the user is away from HPD sensor
-+ */
-+enum sfh_hpd_info {
-+	SFH_NOT_DETECTED,
-+	SFH_USER_PRESENT,
-+	SFH_USER_AWAY,
-+};
-+
-+/**
-+ * struct amd_sfh_info - get HPD sensor info from MP2 FW
-+ * @ambient_light: Populates the ambient light information
-+ * @user_present: Populates the user presence information
-+ * @platform_type: Operating modes (clamshell, flat, tent, etc.)
-+ * @laptop_placement: Device states (ontable, onlap, outbag)
-+ */
-+struct amd_sfh_info {
-+	u32 ambient_light;
-+	u8 user_present;
-+	u32 platform_type;
-+	u32 laptop_placement;
-+};
-+
-+enum laptop_placement {
-+	LP_UNKNOWN = 0,
-+	ON_TABLE,
-+	ON_LAP_MOTION,
-+	IN_BAG,
-+	OUT_OF_BAG,
-+	LP_UNDEFINED,
-+};
-+
-+int amd_get_sfh_info(struct amd_sfh_info *sfh_info, enum sfh_message_type op);
-+#endif
-diff --git a/include/linux/annotate.h b/include/linux/annotate.h
-new file mode 100644
-index 000000000000..2f1599c9e573
--- /dev/null
-+++ b/include/linux/annotate.h
-@@ -0,0 +1,127 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_ANNOTATE_H
-+#define _LINUX_ANNOTATE_H
-+
-+#include <linux/objtool_types.h>
-+
-+#ifdef CONFIG_OBJTOOL
-+
-+#define __ASM_ANNOTATE(section, label, type)				\
-+	.pushsection section, "M", @progbits, 8;			\
-+	.long label - ., type;						\
-+	.popsection
-+
-+#ifndef __ASSEMBLY__
-+
-+#define ASM_ANNOTATE_LABEL(label, type)					\
-+	__stringify(__ASM_ANNOTATE(.discard.annotate_insn, label, type))
-+
-+#define ASM_ANNOTATE(type)						\
-+	"911: "								\
-+	__stringify(__ASM_ANNOTATE(.discard.annotate_insn, 911b, type))
-+
-+#define ASM_ANNOTATE_DATA(type)						\
-+	"912: "								\
-+	__stringify(__ASM_ANNOTATE(.discard.annotate_data, 912b, type))
-+
-+#else /* __ASSEMBLY__ */
-+
-+.macro ANNOTATE type
-+.Lhere_\@:
-+	__ASM_ANNOTATE(.discard.annotate_insn, .Lhere_\@, \type)
-+.endm
-+
-+.macro ANNOTATE_DATA type
-+.Lhere_\@:
-+	__ASM_ANNOTATE(.discard.annotate_data, .Lhere_\@, \type)
-+.endm
-+
-+#endif /* __ASSEMBLY__ */
-+
-+#else /* !CONFIG_OBJTOOL */
-+#ifndef __ASSEMBLY__
-+#define ASM_ANNOTATE_LABEL(label, type) ""
-+#define ASM_ANNOTATE(type)
-+#define ASM_ANNOTATE_DATA(type)
-+#else /* __ASSEMBLY__ */
-+.macro ANNOTATE type
-+.endm
-+.macro ANNOTATE_DATA type
-+.endm
-+#endif /* __ASSEMBLY__ */
-+#endif /* !CONFIG_OBJTOOL */
-+
-+#ifndef __ASSEMBLY__
-+
-+/*
-+ * Annotate away the various 'relocation to !ENDBR` complaints; knowing that
-+ * these relocations will never be used for indirect calls.
-+ */
-+#define ANNOTATE_NOENDBR		ASM_ANNOTATE(ANNOTYPE_NOENDBR)
-+#define ANNOTATE_NOENDBR_SYM(sym)	asm(ASM_ANNOTATE_LABEL(sym, ANNOTYPE_NOENDBR))
-+
-+/*
-+ * This should be used immediately before an indirect jump/call. It tells
-+ * objtool the subsequent indirect jump/call is vouched safe for retpoline
-+ * builds.
-+ */
-+#define ANNOTATE_RETPOLINE_SAFE		ASM_ANNOTATE(ANNOTYPE_RETPOLINE_SAFE)
-+/*
-+ * See linux/instrumentation.h
-+ */
-+#define ANNOTATE_INSTR_BEGIN(label)	ASM_ANNOTATE_LABEL(label, ANNOTYPE_INSTR_BEGIN)
-+#define ANNOTATE_INSTR_END(label)	ASM_ANNOTATE_LABEL(label, ANNOTYPE_INSTR_END)
-+/*
-+ * objtool annotation to ignore the alternatives and only consider the original
-+ * instruction(s).
-+ */
-+#define ANNOTATE_IGNORE_ALTERNATIVE	ASM_ANNOTATE(ANNOTYPE_IGNORE_ALTS)
-+/*
-+ * This macro indicates that the following intra-function call is valid.
-+ * Any non-annotated intra-function call will cause objtool to issue a warning.
-+ */
-+#define ANNOTATE_INTRA_FUNCTION_CALL	ASM_ANNOTATE(ANNOTYPE_INTRA_FUNCTION_CALL)
-+/*
-+ * Use objtool to validate the entry requirement that all code paths do
-+ * VALIDATE_UNRET_END before RET.
-+ *
-+ * NOTE: The macro must be used at the beginning of a global symbol, otherwise
-+ * it will be ignored.
-+ */
-+#define ANNOTATE_UNRET_BEGIN		ASM_ANNOTATE(ANNOTYPE_UNRET_BEGIN)
-+/*
-+ * This should be used to refer to an instruction that is considered
-+ * terminating, like a noreturn CALL or UD2 when we know they are not -- eg
-+ * WARN using UD2.
-+ */
-+#define ANNOTATE_REACHABLE(label)	ASM_ANNOTATE_LABEL(label, ANNOTYPE_REACHABLE)
-+/*
-+ * This should not be used; it annotates away CFI violations. There are a few
-+ * valid use cases like kexec handover to the next kernel image, and there is
-+ * no security concern there.
-+ *
-+ * There are also a few real issues annotated away, like EFI because we can't
-+ * control the EFI code.
-+ */
-+#define ANNOTATE_NOCFI_SYM(sym)		asm(ASM_ANNOTATE_LABEL(sym, ANNOTYPE_NOCFI))
-+
-+/*
-+ * Annotate a special section entry.  This emables livepatch module generation
-+ * to find and extract individual special section entries as needed.
-+ */
-+#define ANNOTATE_DATA_SPECIAL		ASM_ANNOTATE_DATA(ANNOTYPE_DATA_SPECIAL)
-+
-+#else /* __ASSEMBLY__ */
-+#define ANNOTATE_NOENDBR		ANNOTATE type=ANNOTYPE_NOENDBR
-+#define ANNOTATE_RETPOLINE_SAFE		ANNOTATE type=ANNOTYPE_RETPOLINE_SAFE
-+/*	ANNOTATE_INSTR_BEGIN		ANNOTATE type=ANNOTYPE_INSTR_BEGIN */
-+/*	ANNOTATE_INSTR_END		ANNOTATE type=ANNOTYPE_INSTR_END */
-+#define ANNOTATE_IGNORE_ALTERNATIVE	ANNOTATE type=ANNOTYPE_IGNORE_ALTS
-+#define ANNOTATE_INTRA_FUNCTION_CALL	ANNOTATE type=ANNOTYPE_INTRA_FUNCTION_CALL
-+#define ANNOTATE_UNRET_BEGIN		ANNOTATE type=ANNOTYPE_UNRET_BEGIN
-+#define ANNOTATE_REACHABLE		ANNOTATE type=ANNOTYPE_REACHABLE
-+#define ANNOTATE_NOCFI_SYM		ANNOTATE type=ANNOTYPE_NOCFI
-+#define ANNOTATE_DATA_SPECIAL		ANNOTATE_DATA type=ANNOTYPE_DATA_SPECIAL
-+#endif /* __ASSEMBLY__ */
-+
-+#endif /* _LINUX_ANNOTATE_H */
-diff --git a/include/linux/anon_inodes.h b/include/linux/anon_inodes.h
-index 5deaddbd7927..edef565c2a1a 100644
--- a/include/linux/anon_inodes.h
-+++ b/include/linux/anon_inodes.h
-@@ -9,19 +9,24 @@
- #ifndef _LINUX_ANON_INODES_H
- #define _LINUX_ANON_INODES_H
- 
-+#include <linux/types.h>
-+
- struct file_operations;
- struct inode;
- 
- struct file *anon_inode_getfile(const char *name,
- 				const struct file_operations *fops,
- 				void *priv, int flags);
-struct file *anon_inode_getfile_secure(const char *name,
-+struct file *anon_inode_getfile_fmode(const char *name,
-+				const struct file_operations *fops,
-+				void *priv, int flags, fmode_t f_mode);
-+struct file *anon_inode_create_getfile(const char *name,
- 				       const struct file_operations *fops,
- 				       void *priv, int flags,
- 				       const struct inode *context_inode);
- int anon_inode_getfd(const char *name, const struct file_operations *fops,
- 		     void *priv, int flags);
-int anon_inode_getfd_secure(const char *name,
-+int anon_inode_create_getfd(const char *name,
- 			    const struct file_operations *fops,
- 			    void *priv, int flags,
- 			    const struct inode *context_inode);
-diff --git a/include/linux/aperture.h b/include/linux/aperture.h
-index 442f15a57cad..1a9a88b11584 100644
--- a/include/linux/aperture.h
-+++ b/include/linux/aperture.h
-@@ -14,7 +14,9 @@ int devm_aperture_acquire_for_platform_device(struct platform_device *pdev,
- 					      resource_size_t size);
- 
- int aperture_remove_conflicting_devices(resource_size_t base, resource_size_t size,
-					bool primary, const char *name);
-+					const char *name);
-+
-+int __aperture_remove_legacy_vga_devices(struct pci_dev *pdev);
- 
- int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, const char *name);
- #else
-@@ -26,7 +28,12 @@ static inline int devm_aperture_acquire_for_platform_device(struct platform_devi
- }
- 
- static inline int aperture_remove_conflicting_devices(resource_size_t base, resource_size_t size,
-						      bool primary, const char *name)
-+						      const char *name)
-+{
-+	return 0;
-+}
-+
-+static inline int __aperture_remove_legacy_vga_devices(struct pci_dev *pdev)
- {
- 	return 0;
- }
-@@ -39,7 +46,6 @@ static inline int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev,
- 
- /**
-  * aperture_remove_all_conflicting_devices - remove all existing framebuffers
- * @primary: also kick vga16fb if present; only relevant for VGA devices
-  * @name: a descriptive name of the requesting driver
-  *
-  * This function removes all graphics device drivers. Use this function on systems
-@@ -48,9 +54,9 @@ static inline int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev,
-  * Returns:
-  * 0 on success, or a negative errno code otherwise
-  */
-static inline int aperture_remove_all_conflicting_devices(bool primary, const char *name)
-+static inline int aperture_remove_all_conflicting_devices(const char *name)
- {
-	return aperture_remove_conflicting_devices(0, (resource_size_t)-1, primary, name);
-+	return aperture_remove_conflicting_devices(0, (resource_size_t)-1, name);
- }
- 
- #endif
-diff --git a/include/linux/apple-gmux.h b/include/linux/apple-gmux.h
-index ddb10aa67b14..206d97ffda79 100644
--- a/include/linux/apple-gmux.h
-+++ b/include/linux/apple-gmux.h
-@@ -8,18 +8,154 @@
- #define LINUX_APPLE_GMUX_H
- 
- #include <linux/acpi.h>
-+#include <linux/io.h>
-+#include <linux/pnp.h>
- 
- #define GMUX_ACPI_HID "APP000B"
- 
-+/*
-+ * gmux port offsets. Many of these are not yet used, but may be in the
-+ * future, and it's useful to have them documented here anyhow.
-+ */
-+#define GMUX_PORT_VERSION_MAJOR		0x04
-+#define GMUX_PORT_VERSION_MINOR		0x05
-+#define GMUX_PORT_VERSION_RELEASE	0x06
-+#define GMUX_PORT_SWITCH_DISPLAY	0x10
-+#define GMUX_PORT_SWITCH_GET_DISPLAY	0x11
-+#define GMUX_PORT_INTERRUPT_ENABLE	0x14
-+#define GMUX_PORT_INTERRUPT_STATUS	0x16
-+#define GMUX_PORT_SWITCH_DDC		0x28
-+#define GMUX_PORT_SWITCH_EXTERNAL	0x40
-+#define GMUX_PORT_SWITCH_GET_EXTERNAL	0x41
-+#define GMUX_PORT_DISCRETE_POWER	0x50
-+#define GMUX_PORT_MAX_BRIGHTNESS	0x70
-+#define GMUX_PORT_BRIGHTNESS		0x74
-+#define GMUX_PORT_VALUE			0xc2
-+#define GMUX_PORT_READ			0xd0
-+#define GMUX_PORT_WRITE			0xd4
-+
-+#define GMUX_MMIO_PORT_SELECT		0x0e
-+#define GMUX_MMIO_COMMAND_SEND		0x0f
-+
-+#define GMUX_MMIO_READ			0x00
-+#define GMUX_MMIO_WRITE			0x40
-+
-+#define GMUX_MIN_IO_LEN			(GMUX_PORT_BRIGHTNESS + 4)
-+
-+enum apple_gmux_type {
-+	APPLE_GMUX_TYPE_PIO,
-+	APPLE_GMUX_TYPE_INDEXED,
-+	APPLE_GMUX_TYPE_MMIO,
-+};
-+
- #if IS_ENABLED(CONFIG_APPLE_GMUX)
-+static inline bool apple_gmux_is_indexed(unsigned long iostart)
-+{
-+	u16 val;
-+
-+	outb(0xaa, iostart + 0xcc);
-+	outb(0x55, iostart + 0xcd);
-+	outb(0x00, iostart + 0xce);
-+
-+	val = inb(iostart + 0xcc) | (inb(iostart + 0xcd) << 8);
-+	if (val == 0x55aa)
-+		return true;
-+
-+	return false;
-+}
-+
-+static inline bool apple_gmux_is_mmio(unsigned long iostart)
-+{
-+	u8 __iomem *iomem_base = ioremap(iostart, 16);
-+	u8 val;
-+
-+	if (!iomem_base)
-+		return false;
-+
-+	/*
-+	 * If this is 0xff, then gmux must not be present, as the gmux would
-+	 * reset it to 0x00, or it would be one of 0x1, 0x4, 0x41, 0x44 if a
-+	 * command is currently being processed.
-+	 */
-+	val = ioread8(iomem_base + GMUX_MMIO_COMMAND_SEND);
-+	iounmap(iomem_base);
-+	return (val != 0xff);
-+}
- 
- /**
- * apple_gmux_present() - detect if gmux is built into the machine
-+ * apple_gmux_detect() - detect if gmux is built into the machine
-+ *
-+ * @pnp_dev:     Device to probe or NULL to use the first matching device
-+ * @type_ret: Returns (by reference) the apple_gmux_type of the device
-+ *
-+ * Detect if a supported gmux device is present by actually probing it.
-+ * This avoids the false positives returned on some models by
-+ * apple_gmux_present().
-+ *
-+ * Return: %true if a supported gmux ACPI device is detected and the kernel
-+ * was configured with CONFIG_APPLE_GMUX, %false otherwise.
-+ */
-+static inline bool apple_gmux_detect(struct pnp_dev *pnp_dev, enum apple_gmux_type *type_ret)
-+{
-+	u8 ver_major, ver_minor, ver_release;
-+	struct device *dev = NULL;
-+	struct acpi_device *adev;
-+	struct resource *res;
-+	enum apple_gmux_type type = APPLE_GMUX_TYPE_PIO;
-+	bool ret = false;
-+
-+	if (!pnp_dev) {
-+		adev = acpi_dev_get_first_match_dev(GMUX_ACPI_HID, NULL, -1);
-+		if (!adev)
-+			return false;
-+
-+		dev = get_device(acpi_get_first_physical_node(adev));
-+		acpi_dev_put(adev);
-+		if (!dev)
-+			return false;
-+
-+		pnp_dev = to_pnp_dev(dev);
-+	}
-+
-+	res = pnp_get_resource(pnp_dev, IORESOURCE_IO, 0);
-+	if (res && resource_size(res) >= GMUX_MIN_IO_LEN) {
-+		/*
-+		 * Invalid version information may indicate either that the gmux
-+		 * device isn't present or that it's a new one that uses indexed io.
-+		 */
-+		ver_major = inb(res->start + GMUX_PORT_VERSION_MAJOR);
-+		ver_minor = inb(res->start + GMUX_PORT_VERSION_MINOR);
-+		ver_release = inb(res->start + GMUX_PORT_VERSION_RELEASE);
-+		if (ver_major == 0xff && ver_minor == 0xff && ver_release == 0xff) {
-+			if (apple_gmux_is_indexed(res->start))
-+				type = APPLE_GMUX_TYPE_INDEXED;
-+			else
-+				goto out;
-+		}
-+	} else {
-+		res = pnp_get_resource(pnp_dev, IORESOURCE_MEM, 0);
-+		if (res && apple_gmux_is_mmio(res->start))
-+			type = APPLE_GMUX_TYPE_MMIO;
-+		else
-+			goto out;
-+	}
-+
-+	if (type_ret)
-+		*type_ret = type;
-+
-+	ret = true;
-+out:
-+	put_device(dev);
-+	return ret;
-+}
-+
-+/**
-+ * apple_gmux_present() - check if gmux ACPI device is present
-  *
-  * Drivers may use this to activate quirks specific to dual GPU MacBook Pros
-  * and Mac Pros, e.g. for deferred probing, runtime pm and backlight.
-  *
- * Return: %true if gmux is present and the kernel was configured
-+ * Return: %true if gmux ACPI device is present and the kernel was configured
-  * with CONFIG_APPLE_GMUX, %false otherwise.
-  */
- static inline bool apple_gmux_present(void)
-@@ -34,6 +170,11 @@ static inline bool apple_gmux_present(void)
- 	return false;
- }
- 
-+static inline bool apple_gmux_detect(struct pnp_dev *pnp_dev, bool *indexed_ret)
-+{
-+	return false;
-+}
-+
- #endif /* !CONFIG_APPLE_GMUX */
- 
- #endif /* LINUX_APPLE_GMUX_H */
-diff --git a/include/linux/apple-mailbox.h b/include/linux/apple-mailbox.h
-deleted file mode 100644
-index 720fbb70294a..000000000000
--- a/include/linux/apple-mailbox.h
-+++ /dev/null
-@@ -1,19 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
-/*
- * Apple mailbox message format
- *
- * Copyright (C) 2021 The Asahi Linux Contributors
- */
-
-#ifndef _LINUX_APPLE_MAILBOX_H_
-#define _LINUX_APPLE_MAILBOX_H_
-
-#include <linux/types.h>
-
-/* encodes a single 96bit message sent over the single channel */
-struct apple_mbox_msg {
-	u64 msg0;
-	u32 msg1;
-};
-
-#endif
-diff --git a/include/linux/apple_bl.h b/include/linux/apple_bl.h
-deleted file mode 100644
-index 445af2e3cc21..000000000000
--- a/include/linux/apple_bl.h
-+++ /dev/null
-@@ -1,27 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * apple_bl exported symbols
- */
-
-#ifndef _LINUX_APPLE_BL_H
-#define _LINUX_APPLE_BL_H
-
-#if defined(CONFIG_BACKLIGHT_APPLE) || defined(CONFIG_BACKLIGHT_APPLE_MODULE)
-
-extern int apple_bl_register(void);
-extern void apple_bl_unregister(void);
-
-#else /* !CONFIG_BACKLIGHT_APPLE */
-
-static inline int apple_bl_register(void)
-{
-	return 0;
-}
-
-static inline void apple_bl_unregister(void)
-{
-}
-
-#endif /* !CONFIG_BACKLIGHT_APPLE */
-
-#endif /* _LINUX_APPLE_BL_H */
-diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h
-index a07b510e7dc5..ebd7f8935f96 100644
--- a/include/linux/arch_topology.h
-+++ b/include/linux/arch_topology.h
-@@ -11,22 +11,17 @@
- void topology_normalize_cpu_scale(void);
- int topology_update_cpu_topology(void);
- 
-#ifdef CONFIG_ACPI_CPPC_LIB
-void topology_init_cpu_capacity_cppc(void);
-#endif
-
- struct device_node;
- bool topology_parse_cpu_capacity(struct device_node *cpu_node, int cpu);
- 
-DECLARE_PER_CPU(unsigned long, cpu_scale);
- 
-static inline unsigned long topology_get_cpu_scale(int cpu)
-+DECLARE_PER_CPU(unsigned long, capacity_freq_ref);
-+
-+static inline unsigned long topology_get_freq_ref(int cpu)
- {
-	return per_cpu(cpu_scale, cpu);
-+	return per_cpu(capacity_freq_ref, cpu);
- }
- 
-void topology_set_cpu_scale(unsigned int cpu, unsigned long capacity);
-
- DECLARE_PER_CPU(unsigned long, arch_freq_scale);
- 
- static inline unsigned long topology_get_freq_scale(int cpu)
-@@ -42,6 +37,7 @@ enum scale_freq_source {
- 	SCALE_FREQ_SOURCE_CPUFREQ = 0,
- 	SCALE_FREQ_SOURCE_ARCH,
- 	SCALE_FREQ_SOURCE_CPPC,
-+	SCALE_FREQ_SOURCE_VIRT,
- };
- 
- struct scale_freq_data {
-@@ -53,14 +49,14 @@ void topology_scale_freq_tick(void);
- void topology_set_scale_freq_source(struct scale_freq_data *data, const struct cpumask *cpus);
- void topology_clear_scale_freq_source(enum scale_freq_source source, const struct cpumask *cpus);
- 
-DECLARE_PER_CPU(unsigned long, thermal_pressure);
-+DECLARE_PER_CPU(unsigned long, hw_pressure);
- 
-static inline unsigned long topology_get_thermal_pressure(int cpu)
-+static inline unsigned long topology_get_hw_pressure(int cpu)
- {
-	return per_cpu(thermal_pressure, cpu);
-+	return per_cpu(hw_pressure, cpu);
- }
- 
-void topology_update_thermal_pressure(const struct cpumask *cpus,
-+void topology_update_hw_pressure(const struct cpumask *cpus,
- 				      unsigned long capped_freq);
- 
- struct cpu_topology {
-@@ -84,6 +80,11 @@ extern struct cpu_topology cpu_topology[NR_CPUS];
- #define topology_sibling_cpumask(cpu)	(&cpu_topology[cpu].thread_sibling)
- #define topology_cluster_cpumask(cpu)	(&cpu_topology[cpu].cluster_sibling)
- #define topology_llc_cpumask(cpu)	(&cpu_topology[cpu].llc_sibling)
-+
-+#ifndef arch_cpu_is_threaded
-+#define arch_cpu_is_threaded()	(0)
-+#endif
-+
- void init_cpu_topology(void);
- void store_cpu_topology(unsigned int cpuid);
- const struct cpumask *cpu_coregroup_mask(int cpu);
-@@ -92,6 +93,22 @@ void update_siblings_masks(unsigned int cpu);
- void remove_cpu_topology(unsigned int cpuid);
- void reset_cpu_topology(void);
- int parse_acpi_topology(void);
-#endif
-+void freq_inv_set_max_ratio(int cpu, u64 max_rate);
-+
-+/*
-+ * Architectures like ARM64 don't have reliable architectural way to get SMT
-+ * information and depend on the firmware (ACPI/OF) report. Non-SMT core won't
-+ * initialize thread_id so we can use this to detect the SMT implementation.
-+ */
-+static inline bool topology_core_has_smt(int cpu)
-+{
-+	return cpu_topology[cpu].thread_id != -1;
-+}
-+
-+#else
-+
-+static inline bool topology_core_has_smt(int cpu) { return false; }
-+
-+#endif /* CONFIG_GENERIC_ARCH_TOPOLOGY */
- 
- #endif /* _LINUX_ARCH_TOPOLOGY_H_ */
-diff --git a/include/linux/args.h b/include/linux/args.h
-new file mode 100644
-index 000000000000..2e8e65d975c7
--- /dev/null
-+++ b/include/linux/args.h
-@@ -0,0 +1,28 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+
-+#ifndef _LINUX_ARGS_H
-+#define _LINUX_ARGS_H
-+
-+/*
-+ * How do these macros work?
-+ *
-+ * In __COUNT_ARGS() _0 to _12 are just placeholders from the start
-+ * in order to make sure _n is positioned over the correct number
-+ * from 12 to 0 (depending on X, which is a variadic argument list).
-+ * They serve no purpose other than occupying a position. Since each
-+ * macro parameter must have a distinct identifier, those identifiers
-+ * are as good as any.
-+ *
-+ * In COUNT_ARGS() we use actual integers, so __COUNT_ARGS() returns
-+ * that as _n.
-+ */
-+
-+/* This counts to 15. Any more, it will return 16th argument. */
-+#define __COUNT_ARGS(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _n, X...) _n
-+#define COUNT_ARGS(X...) __COUNT_ARGS(, ##X, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
-+
-+/* Concatenate two parameters, but allow them to be expanded beforehand. */
-+#define __CONCAT(a, b) a ## b
-+#define CONCATENATE(a, b) __CONCAT(a, b)
-+
-+#endif	/* _LINUX_ARGS_H */
-diff --git a/include/linux/arm-cci.h b/include/linux/arm-cci.h
-index d0e44201d855..7f7a576267bc 100644
--- a/include/linux/arm-cci.h
-+++ b/include/linux/arm-cci.h
-@@ -43,6 +43,8 @@ static inline int __cci_control_port_by_index(u32 port, bool enable)
- }
- #endif
- 
-+void cci_enable_port_for_self(void);
-+
- #define cci_disable_port_by_device(dev) \
- 	__cci_control_port_by_device(dev, false)
- #define cci_enable_port_by_device(dev) \
-diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h
-index 220c8c60e021..50b47eba7d01 100644
--- a/include/linux/arm-smccc.h
-+++ b/include/linux/arm-smccc.h
-@@ -5,7 +5,13 @@
- #ifndef __LINUX_ARM_SMCCC_H
- #define __LINUX_ARM_SMCCC_H
- 
-+#include <linux/args.h>
- #include <linux/init.h>
-+
-+#ifndef __ASSEMBLY__
-+#include <linux/uuid.h>
-+#endif
-+
- #include <uapi/linux/const.h>
- 
- /*
-@@ -66,6 +72,8 @@
- #define ARM_SMCCC_VERSION_1_3		0x10003
- 
- #define ARM_SMCCC_1_3_SVE_HINT		0x10000
-+#define ARM_SMCCC_CALL_HINTS		ARM_SMCCC_1_3_SVE_HINT
-+
- 
- #define ARM_SMCCC_VERSION_FUNC_ID					\
- 	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,				\
-@@ -104,14 +112,81 @@
- 			   ARM_SMCCC_FUNC_QUERY_CALL_UID)
- 
- /* KVM UID value: 28b46fb6-2ec5-11e9-a9ca-4b564d003a74 */
-#define ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_0	0xb66fb428U
-#define ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_1	0xe911c52eU
-#define ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_2	0x564bcaa9U
-#define ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_3	0x743a004dU
-+#define ARM_SMCCC_VENDOR_HYP_UID_KVM UUID_INIT(\
-+	0x28b46fb6, 0x2ec5, 0x11e9, \
-+	0xa9, 0xca, 0x4b, 0x56, \
-+	0x4d, 0x00, 0x3a, 0x74)
- 
- /* KVM "vendor specific" services */
- #define ARM_SMCCC_KVM_FUNC_FEATURES		0
- #define ARM_SMCCC_KVM_FUNC_PTP			1
-+/* Start of pKVM hypercall range */
-+#define ARM_SMCCC_KVM_FUNC_HYP_MEMINFO		2
-+#define ARM_SMCCC_KVM_FUNC_MEM_SHARE		3
-+#define ARM_SMCCC_KVM_FUNC_MEM_UNSHARE		4
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_5		5
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_6		6
-+#define ARM_SMCCC_KVM_FUNC_MMIO_GUARD		7
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_8		8
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_9		9
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_10		10
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_11		11
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_12		12
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_13		13
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_14		14
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_15		15
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_16		16
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_17		17
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_18		18
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_19		19
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_20		20
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_21		21
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_22		22
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_23		23
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_24		24
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_25		25
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_26		26
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_27		27
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_28		28
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_29		29
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_30		30
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_31		31
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_32		32
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_33		33
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_34		34
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_35		35
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_36		36
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_37		37
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_38		38
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_39		39
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_40		40
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_41		41
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_42		42
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_43		43
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_44		44
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_45		45
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_46		46
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_47		47
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_48		48
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_49		49
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_50		50
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_51		51
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_52		52
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_53		53
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_54		54
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_55		55
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_56		56
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_57		57
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_58		58
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_59		59
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_60		60
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_61		61
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_62		62
-+#define ARM_SMCCC_KVM_FUNC_PKVM_RESV_63		63
-+/* End of pKVM hypercall range */
-+#define ARM_SMCCC_KVM_FUNC_DISCOVER_IMPL_VER	64
-+#define ARM_SMCCC_KVM_FUNC_DISCOVER_IMPL_CPUS	65
-+
- #define ARM_SMCCC_KVM_FUNC_FEATURES_2		127
- #define ARM_SMCCC_KVM_NUM_FUNCS			128
- 
-@@ -134,6 +209,42 @@
- 			   ARM_SMCCC_OWNER_VENDOR_HYP,			\
- 			   ARM_SMCCC_KVM_FUNC_PTP)
- 
-+#define ARM_SMCCC_VENDOR_HYP_KVM_HYP_MEMINFO_FUNC_ID			\
-+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,				\
-+			   ARM_SMCCC_SMC_64,				\
-+			   ARM_SMCCC_OWNER_VENDOR_HYP,			\
-+			   ARM_SMCCC_KVM_FUNC_HYP_MEMINFO)
-+
-+#define ARM_SMCCC_VENDOR_HYP_KVM_MEM_SHARE_FUNC_ID			\
-+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,				\
-+			   ARM_SMCCC_SMC_64,				\
-+			   ARM_SMCCC_OWNER_VENDOR_HYP,			\
-+			   ARM_SMCCC_KVM_FUNC_MEM_SHARE)
-+
-+#define ARM_SMCCC_VENDOR_HYP_KVM_MEM_UNSHARE_FUNC_ID			\
-+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,				\
-+			   ARM_SMCCC_SMC_64,				\
-+			   ARM_SMCCC_OWNER_VENDOR_HYP,			\
-+			   ARM_SMCCC_KVM_FUNC_MEM_UNSHARE)
-+
-+#define ARM_SMCCC_VENDOR_HYP_KVM_MMIO_GUARD_FUNC_ID			\
-+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,				\
-+			   ARM_SMCCC_SMC_64,				\
-+			   ARM_SMCCC_OWNER_VENDOR_HYP,			\
-+			   ARM_SMCCC_KVM_FUNC_MMIO_GUARD)
-+
-+#define ARM_SMCCC_VENDOR_HYP_KVM_DISCOVER_IMPL_VER_FUNC_ID		\
-+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,				\
-+			   ARM_SMCCC_SMC_64,				\
-+			   ARM_SMCCC_OWNER_VENDOR_HYP,			\
-+			   ARM_SMCCC_KVM_FUNC_DISCOVER_IMPL_VER)
-+
-+#define ARM_SMCCC_VENDOR_HYP_KVM_DISCOVER_IMPL_CPUS_FUNC_ID		\
-+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,				\
-+			   ARM_SMCCC_SMC_64,				\
-+			   ARM_SMCCC_OWNER_VENDOR_HYP,			\
-+			   ARM_SMCCC_KVM_FUNC_DISCOVER_IMPL_CPUS)
-+
- /* ptp_kvm counter type ID */
- #define KVM_PTP_VIRT_COUNTER			0
- #define KVM_PTP_PHYS_COUNTER			1
-@@ -224,7 +335,74 @@ u32 arm_smccc_get_version(void);
- 
- void __init arm_smccc_version_init(u32 version, enum arm_smccc_conduit conduit);
- 
-extern u64 smccc_has_sve_hint;
-+/**
-+ * arm_smccc_get_soc_id_version()
-+ *
-+ * Returns the SOC ID version.
-+ *
-+ * When ARM_SMCCC_ARCH_SOC_ID is not present, returns SMCCC_RET_NOT_SUPPORTED.
-+ */
-+s32 arm_smccc_get_soc_id_version(void);
-+
-+/**
-+ * arm_smccc_get_soc_id_revision()
-+ *
-+ * Returns the SOC ID revision.
-+ *
-+ * When ARM_SMCCC_ARCH_SOC_ID is not present, returns SMCCC_RET_NOT_SUPPORTED.
-+ */
-+s32 arm_smccc_get_soc_id_revision(void);
-+
-+#ifndef __ASSEMBLY__
-+
-+/*
-+ * Returns whether a specific hypervisor UUID is advertised for the
-+ * Vendor Specific Hypervisor Service range.
-+ */
-+bool arm_smccc_hypervisor_has_uuid(const uuid_t *uuid);
-+
-+static inline uuid_t smccc_res_to_uuid(u32 r0, u32 r1, u32 r2, u32 r3)
-+{
-+	uuid_t uuid = {
-+		.b = {
-+			[0]  = (r0 >> 0)  & 0xff,
-+			[1]  = (r0 >> 8)  & 0xff,
-+			[2]  = (r0 >> 16) & 0xff,
-+			[3]  = (r0 >> 24) & 0xff,
-+
-+			[4]  = (r1 >> 0)  & 0xff,
-+			[5]  = (r1 >> 8)  & 0xff,
-+			[6]  = (r1 >> 16) & 0xff,
-+			[7]  = (r1 >> 24) & 0xff,
-+
-+			[8]  = (r2 >> 0)  & 0xff,
-+			[9]  = (r2 >> 8)  & 0xff,
-+			[10] = (r2 >> 16) & 0xff,
-+			[11] = (r2 >> 24) & 0xff,
-+
-+			[12] = (r3 >> 0)  & 0xff,
-+			[13] = (r3 >> 8)  & 0xff,
-+			[14] = (r3 >> 16) & 0xff,
-+			[15] = (r3 >> 24) & 0xff,
-+		},
-+	};
-+
-+	return uuid;
-+}
-+
-+static inline u32 smccc_uuid_to_reg(const uuid_t *uuid, int reg)
-+{
-+	u32 val = 0;
-+
-+	val |= (u32)(uuid->b[4 * reg + 0] << 0);
-+	val |= (u32)(uuid->b[4 * reg + 1] << 8);
-+	val |= (u32)(uuid->b[4 * reg + 2] << 16);
-+	val |= (u32)(uuid->b[4 * reg + 3] << 24);
-+
-+	return val;
-+}
-+
-+#endif /* !__ASSEMBLY__ */
- 
- /**
-  * struct arm_smccc_res - Result from SMC/HVC call
-@@ -306,15 +484,6 @@ struct arm_smccc_quirk {
- };
- 
- /**
- * __arm_smccc_sve_check() - Set the SVE hint bit when doing SMC calls
- *
- * Sets the SMCCC hint bit to indicate if there is live state in the SVE
- * registers, this modifies x0 in place and should never be called from C
- * code.
- */
-asmlinkage unsigned long __arm_smccc_sve_check(unsigned long x0);
-
-/**
-  * __arm_smccc_smc() - make SMC calls
-  * @a0-a7: arguments passed in registers 0 to 7
-  * @res: result values from registers 0 to 3
-@@ -381,45 +550,26 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1,
- 
- #endif
- 
-/* nVHE hypervisor doesn't have a current thread so needs separate checks */
-#if defined(CONFIG_ARM64_SVE) && !defined(__KVM_NVHE_HYPERVISOR__)
-
-#define SMCCC_SVE_CHECK ALTERNATIVE("nop \n",  "bl __arm_smccc_sve_check \n", \
-				    ARM64_SVE)
-#define smccc_sve_clobbers "x16", "x30", "cc",
-
-#else
-
-#define SMCCC_SVE_CHECK
-#define smccc_sve_clobbers
-
-#endif
-
-#define ___count_args(_0, _1, _2, _3, _4, _5, _6, _7, _8, x, ...) x
-+#define __constraint_read_2	"r" (arg0)
-+#define __constraint_read_3	__constraint_read_2, "r" (arg1)
-+#define __constraint_read_4	__constraint_read_3, "r" (arg2)
-+#define __constraint_read_5	__constraint_read_4, "r" (arg3)
-+#define __constraint_read_6	__constraint_read_5, "r" (arg4)
-+#define __constraint_read_7	__constraint_read_6, "r" (arg5)
-+#define __constraint_read_8	__constraint_read_7, "r" (arg6)
-+#define __constraint_read_9	__constraint_read_8, "r" (arg7)
- 
-#define __count_args(...)						\
-	___count_args(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0)
-
-#define __constraint_read_0	"r" (arg0)
-#define __constraint_read_1	__constraint_read_0, "r" (arg1)
-#define __constraint_read_2	__constraint_read_1, "r" (arg2)
-#define __constraint_read_3	__constraint_read_2, "r" (arg3)
-#define __constraint_read_4	__constraint_read_3, "r" (arg4)
-#define __constraint_read_5	__constraint_read_4, "r" (arg5)
-#define __constraint_read_6	__constraint_read_5, "r" (arg6)
-#define __constraint_read_7	__constraint_read_6, "r" (arg7)
-
-#define __declare_arg_0(a0, res)					\
-+#define __declare_arg_2(a0, res)					\
- 	struct arm_smccc_res   *___res = res;				\
- 	register unsigned long arg0 asm("r0") = (u32)a0
- 
-#define __declare_arg_1(a0, a1, res)					\
-+#define __declare_arg_3(a0, a1, res)					\
- 	typeof(a1) __a1 = a1;						\
- 	struct arm_smccc_res   *___res = res;				\
- 	register unsigned long arg0 asm("r0") = (u32)a0;			\
- 	register typeof(a1) arg1 asm("r1") = __a1
- 
-#define __declare_arg_2(a0, a1, a2, res)				\
-+#define __declare_arg_4(a0, a1, a2, res)				\
- 	typeof(a1) __a1 = a1;						\
- 	typeof(a2) __a2 = a2;						\
- 	struct arm_smccc_res   *___res = res;				\
-@@ -427,7 +577,7 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1,
- 	register typeof(a1) arg1 asm("r1") = __a1;			\
- 	register typeof(a2) arg2 asm("r2") = __a2
- 
-#define __declare_arg_3(a0, a1, a2, a3, res)				\
-+#define __declare_arg_5(a0, a1, a2, a3, res)				\
- 	typeof(a1) __a1 = a1;						\
- 	typeof(a2) __a2 = a2;						\
- 	typeof(a3) __a3 = a3;						\
-@@ -437,34 +587,26 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1,
- 	register typeof(a2) arg2 asm("r2") = __a2;			\
- 	register typeof(a3) arg3 asm("r3") = __a3
- 
-#define __declare_arg_4(a0, a1, a2, a3, a4, res)			\
-+#define __declare_arg_6(a0, a1, a2, a3, a4, res)			\
- 	typeof(a4) __a4 = a4;						\
-	__declare_arg_3(a0, a1, a2, a3, res);				\
-+	__declare_arg_5(a0, a1, a2, a3, res);				\
- 	register typeof(a4) arg4 asm("r4") = __a4
- 
-#define __declare_arg_5(a0, a1, a2, a3, a4, a5, res)			\
-+#define __declare_arg_7(a0, a1, a2, a3, a4, a5, res)			\
- 	typeof(a5) __a5 = a5;						\
-	__declare_arg_4(a0, a1, a2, a3, a4, res);			\
-+	__declare_arg_6(a0, a1, a2, a3, a4, res);			\
- 	register typeof(a5) arg5 asm("r5") = __a5
- 
-#define __declare_arg_6(a0, a1, a2, a3, a4, a5, a6, res)		\
-+#define __declare_arg_8(a0, a1, a2, a3, a4, a5, a6, res)		\
- 	typeof(a6) __a6 = a6;						\
-	__declare_arg_5(a0, a1, a2, a3, a4, a5, res);			\
-+	__declare_arg_7(a0, a1, a2, a3, a4, a5, res);			\
- 	register typeof(a6) arg6 asm("r6") = __a6
- 
-#define __declare_arg_7(a0, a1, a2, a3, a4, a5, a6, a7, res)		\
-+#define __declare_arg_9(a0, a1, a2, a3, a4, a5, a6, a7, res)		\
- 	typeof(a7) __a7 = a7;						\
-	__declare_arg_6(a0, a1, a2, a3, a4, a5, a6, res);		\
-+	__declare_arg_8(a0, a1, a2, a3, a4, a5, a6, res);		\
- 	register typeof(a7) arg7 asm("r7") = __a7
- 
-#define ___declare_args(count, ...) __declare_arg_ ## count(__VA_ARGS__)
-#define __declare_args(count, ...)  ___declare_args(count, __VA_ARGS__)
-
-#define ___constraints(count)						\
-	: __constraint_read_ ## count					\
-	: smccc_sve_clobbers "memory"
-#define __constraints(count)	___constraints(count)
-
- /*
-  * We have an output list that is not necessarily used, and GCC feels
-  * entitled to optimise the whole sequence away. "volatile" is what
-@@ -476,11 +618,13 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1,
- 		register unsigned long r1 asm("r1");			\
- 		register unsigned long r2 asm("r2");			\
- 		register unsigned long r3 asm("r3"); 			\
-		__declare_args(__count_args(__VA_ARGS__), __VA_ARGS__);	\
-		asm volatile(SMCCC_SVE_CHECK				\
-			     inst "\n" :				\
-+		CONCATENATE(__declare_arg_,				\
-+			    COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__);	\
-+		asm volatile(inst "\n" :				\
- 			     "=r" (r0), "=r" (r1), "=r" (r2), "=r" (r3)	\
-			     __constraints(__count_args(__VA_ARGS__)));	\
-+			     : CONCATENATE(__constraint_read_,		\
-+					   COUNT_ARGS(__VA_ARGS__))	\
-+			     : "memory");				\
- 		if (___res)						\
- 			*___res = (typeof(*___res)){r0, r1, r2, r3};	\
- 	} while (0)
-@@ -524,8 +668,12 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1,
-  */
- #define __fail_smccc_1_1(...)						\
- 	do {								\
-		__declare_args(__count_args(__VA_ARGS__), __VA_ARGS__);	\
-		asm ("" : __constraints(__count_args(__VA_ARGS__)));	\
-+		CONCATENATE(__declare_arg_,				\
-+			    COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__);	\
-+		asm ("" :						\
-+		     : CONCATENATE(__constraint_read_,			\
-+				   COUNT_ARGS(__VA_ARGS__))		\
-+		     : "memory");					\
- 		if (___res)						\
- 			___res->a0 = SMCCC_RET_NOT_SUPPORTED;		\
- 	} while (0)
-@@ -562,5 +710,45 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1,
- 		method;							\
- 	})
- 
-+#ifdef CONFIG_ARM64
-+
-+#define __fail_smccc_1_2(___res)					\
-+	do {								\
-+		if (___res)						\
-+			___res->a0 = SMCCC_RET_NOT_SUPPORTED;		\
-+	} while (0)
-+
-+/*
-+ * arm_smccc_1_2_invoke() - make an SMCCC v1.2 compliant call
-+ *
-+ * @args: SMC args are in the a0..a17 fields of the arm_smcc_1_2_regs structure
-+ * @res: result values from registers 0 to 17
-+ *
-+ * This macro will make either an HVC call or an SMC call depending on the
-+ * current SMCCC conduit. If no valid conduit is available then -1
-+ * (SMCCC_RET_NOT_SUPPORTED) is returned in @res.a0 (if supplied).
-+ *
-+ * The return value also provides the conduit that was used.
-+ */
-+#define arm_smccc_1_2_invoke(args, res) ({				\
-+		struct arm_smccc_1_2_regs *__args = args;		\
-+		struct arm_smccc_1_2_regs *__res = res;			\
-+		int method = arm_smccc_1_1_get_conduit();		\
-+		switch (method) {					\
-+		case SMCCC_CONDUIT_HVC:					\
-+			arm_smccc_1_2_hvc(__args, __res);		\
-+			break;						\
-+		case SMCCC_CONDUIT_SMC:					\
-+			arm_smccc_1_2_smc(__args, __res);		\
-+			break;						\
-+		default:						\
-+			__fail_smccc_1_2(__res);			\
-+			method = SMCCC_CONDUIT_NONE;			\
-+			break;						\
-+		}							\
-+		method;							\
-+	})
-+#endif /*CONFIG_ARM64*/
-+
- #endif /*__ASSEMBLY__*/
- #endif /*__LINUX_ARM_SMCCC_H*/
-diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h
-index e5c76c1ef9ed..81e603839c4a 100644
--- a/include/linux/arm_ffa.h
-+++ b/include/linux/arm_ffa.h
-@@ -6,17 +6,139 @@
- #ifndef _LINUX_ARM_FFA_H
- #define _LINUX_ARM_FFA_H
- 
-+#include <linux/bitfield.h>
- #include <linux/device.h>
- #include <linux/module.h>
- #include <linux/types.h>
- #include <linux/uuid.h>
- 
-+#define FFA_SMC(calling_convention, func_num)				\
-+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, (calling_convention),	\
-+			   ARM_SMCCC_OWNER_STANDARD, (func_num))
-+
-+#define FFA_SMC_32(func_num)	FFA_SMC(ARM_SMCCC_SMC_32, (func_num))
-+#define FFA_SMC_64(func_num)	FFA_SMC(ARM_SMCCC_SMC_64, (func_num))
-+
-+#define FFA_ERROR			FFA_SMC_32(0x60)
-+#define FFA_SUCCESS			FFA_SMC_32(0x61)
-+#define FFA_FN64_SUCCESS		FFA_SMC_64(0x61)
-+#define FFA_INTERRUPT			FFA_SMC_32(0x62)
-+#define FFA_VERSION			FFA_SMC_32(0x63)
-+#define FFA_FEATURES			FFA_SMC_32(0x64)
-+#define FFA_RX_RELEASE			FFA_SMC_32(0x65)
-+#define FFA_RXTX_MAP			FFA_SMC_32(0x66)
-+#define FFA_FN64_RXTX_MAP		FFA_SMC_64(0x66)
-+#define FFA_RXTX_UNMAP			FFA_SMC_32(0x67)
-+#define FFA_PARTITION_INFO_GET		FFA_SMC_32(0x68)
-+#define FFA_ID_GET			FFA_SMC_32(0x69)
-+#define FFA_MSG_POLL			FFA_SMC_32(0x6A)
-+#define FFA_MSG_WAIT			FFA_SMC_32(0x6B)
-+#define FFA_YIELD			FFA_SMC_32(0x6C)
-+#define FFA_RUN				FFA_SMC_32(0x6D)
-+#define FFA_MSG_SEND			FFA_SMC_32(0x6E)
-+#define FFA_MSG_SEND_DIRECT_REQ		FFA_SMC_32(0x6F)
-+#define FFA_FN64_MSG_SEND_DIRECT_REQ	FFA_SMC_64(0x6F)
-+#define FFA_MSG_SEND_DIRECT_RESP	FFA_SMC_32(0x70)
-+#define FFA_FN64_MSG_SEND_DIRECT_RESP	FFA_SMC_64(0x70)
-+#define FFA_MEM_DONATE			FFA_SMC_32(0x71)
-+#define FFA_FN64_MEM_DONATE		FFA_SMC_64(0x71)
-+#define FFA_MEM_LEND			FFA_SMC_32(0x72)
-+#define FFA_FN64_MEM_LEND		FFA_SMC_64(0x72)
-+#define FFA_MEM_SHARE			FFA_SMC_32(0x73)
-+#define FFA_FN64_MEM_SHARE		FFA_SMC_64(0x73)
-+#define FFA_MEM_RETRIEVE_REQ		FFA_SMC_32(0x74)
-+#define FFA_FN64_MEM_RETRIEVE_REQ	FFA_SMC_64(0x74)
-+#define FFA_MEM_RETRIEVE_RESP		FFA_SMC_32(0x75)
-+#define FFA_MEM_RELINQUISH		FFA_SMC_32(0x76)
-+#define FFA_MEM_RECLAIM			FFA_SMC_32(0x77)
-+#define FFA_MEM_OP_PAUSE		FFA_SMC_32(0x78)
-+#define FFA_MEM_OP_RESUME		FFA_SMC_32(0x79)
-+#define FFA_MEM_FRAG_RX			FFA_SMC_32(0x7A)
-+#define FFA_MEM_FRAG_TX			FFA_SMC_32(0x7B)
-+#define FFA_NORMAL_WORLD_RESUME		FFA_SMC_32(0x7C)
-+#define FFA_NOTIFICATION_BITMAP_CREATE	FFA_SMC_32(0x7D)
-+#define FFA_NOTIFICATION_BITMAP_DESTROY FFA_SMC_32(0x7E)
-+#define FFA_NOTIFICATION_BIND		FFA_SMC_32(0x7F)
-+#define FFA_NOTIFICATION_UNBIND		FFA_SMC_32(0x80)
-+#define FFA_NOTIFICATION_SET		FFA_SMC_32(0x81)
-+#define FFA_NOTIFICATION_GET		FFA_SMC_32(0x82)
-+#define FFA_NOTIFICATION_INFO_GET	FFA_SMC_32(0x83)
-+#define FFA_FN64_NOTIFICATION_INFO_GET	FFA_SMC_64(0x83)
-+#define FFA_RX_ACQUIRE			FFA_SMC_32(0x84)
-+#define FFA_SPM_ID_GET			FFA_SMC_32(0x85)
-+#define FFA_MSG_SEND2			FFA_SMC_32(0x86)
-+#define FFA_SECONDARY_EP_REGISTER	FFA_SMC_32(0x87)
-+#define FFA_FN64_SECONDARY_EP_REGISTER	FFA_SMC_64(0x87)
-+#define FFA_MEM_PERM_GET		FFA_SMC_32(0x88)
-+#define FFA_FN64_MEM_PERM_GET		FFA_SMC_64(0x88)
-+#define FFA_MEM_PERM_SET		FFA_SMC_32(0x89)
-+#define FFA_FN64_MEM_PERM_SET		FFA_SMC_64(0x89)
-+#define FFA_CONSOLE_LOG			FFA_SMC_32(0x8A)
-+#define FFA_PARTITION_INFO_GET_REGS	FFA_SMC_64(0x8B)
-+#define FFA_EL3_INTR_HANDLE		FFA_SMC_32(0x8C)
-+#define FFA_MSG_SEND_DIRECT_REQ2	FFA_SMC_64(0x8D)
-+#define FFA_MSG_SEND_DIRECT_RESP2	FFA_SMC_64(0x8E)
-+
-+/*
-+ * For some calls it is necessary to use SMC64 to pass or return 64-bit values.
-+ * For such calls FFA_FN_NATIVE(name) will choose the appropriate
-+ * (native-width) function ID.
-+ */
-+#ifdef CONFIG_64BIT
-+#define FFA_FN_NATIVE(name)	FFA_FN64_##name
-+#else
-+#define FFA_FN_NATIVE(name)	FFA_##name
-+#endif
-+
-+/* FFA error codes. */
-+#define FFA_RET_SUCCESS            (0)
-+#define FFA_RET_NOT_SUPPORTED      (-1)
-+#define FFA_RET_INVALID_PARAMETERS (-2)
-+#define FFA_RET_NO_MEMORY          (-3)
-+#define FFA_RET_BUSY               (-4)
-+#define FFA_RET_INTERRUPTED        (-5)
-+#define FFA_RET_DENIED             (-6)
-+#define FFA_RET_RETRY              (-7)
-+#define FFA_RET_ABORTED            (-8)
-+#define FFA_RET_NO_DATA            (-9)
-+
-+/* FFA version encoding */
-+#define FFA_MAJOR_VERSION_MASK	GENMASK(30, 16)
-+#define FFA_MINOR_VERSION_MASK	GENMASK(15, 0)
-+#define FFA_MAJOR_VERSION(x)	((u16)(FIELD_GET(FFA_MAJOR_VERSION_MASK, (x))))
-+#define FFA_MINOR_VERSION(x)	((u16)(FIELD_GET(FFA_MINOR_VERSION_MASK, (x))))
-+#define FFA_PACK_VERSION_INFO(major, minor)			\
-+	(FIELD_PREP(FFA_MAJOR_VERSION_MASK, (major)) |		\
-+	 FIELD_PREP(FFA_MINOR_VERSION_MASK, (minor)))
-+#define FFA_VERSION_1_0		FFA_PACK_VERSION_INFO(1, 0)
-+#define FFA_VERSION_1_1		FFA_PACK_VERSION_INFO(1, 1)
-+#define FFA_VERSION_1_2		FFA_PACK_VERSION_INFO(1, 2)
-+
-+/**
-+ * FF-A specification mentions explicitly about '4K pages'. This should
-+ * not be confused with the kernel PAGE_SIZE, which is the translation
-+ * granule kernel is configured and may be one among 4K, 16K and 64K.
-+ */
-+#define FFA_PAGE_SIZE		SZ_4K
-+
-+/*
-+ * Minimum buffer size/alignment encodings returned by an FFA_FEATURES
-+ * query for FFA_RXTX_MAP.
-+ */
-+#define FFA_FEAT_RXTX_MIN_SZ_4K		0
-+#define FFA_FEAT_RXTX_MIN_SZ_64K	1
-+#define FFA_FEAT_RXTX_MIN_SZ_16K	2
-+#define FFA_FEAT_RXTX_MIN_SZ_MASK	GENMASK(1, 0)
-+
- /* FFA Bus/Device/Driver related */
- struct ffa_device {
-+	u32 id;
-+	u32 properties;
- 	int vm_id;
- 	bool mode_32bit;
- 	uuid_t uuid;
- 	struct device dev;
-+	const struct ffa_ops *ops;
- };
- 
- #define to_ffa_dev(d) container_of(d, struct ffa_device, dev)
-@@ -34,7 +156,7 @@ struct ffa_driver {
- 	struct device_driver driver;
- };
- 
-#define to_ffa_driver(d) container_of(d, struct ffa_driver, driver)
-+#define to_ffa_driver(d) container_of_const(d, struct ffa_driver, driver)
- 
- static inline void ffa_dev_set_drvdata(struct ffa_device *fdev, void *data)
- {
-@@ -46,24 +168,31 @@ static inline void *ffa_dev_get_drvdata(struct ffa_device *fdev)
- 	return dev_get_drvdata(&fdev->dev);
- }
- 
-+struct ffa_partition_info;
-+
- #if IS_REACHABLE(CONFIG_ARM_FFA_TRANSPORT)
-struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id);
-+struct ffa_device *
-+ffa_device_register(const struct ffa_partition_info *part_info,
-+		    const struct ffa_ops *ops);
- void ffa_device_unregister(struct ffa_device *ffa_dev);
- int ffa_driver_register(struct ffa_driver *driver, struct module *owner,
- 			const char *mod_name);
- void ffa_driver_unregister(struct ffa_driver *driver);
-+void ffa_devices_unregister(void);
- bool ffa_device_is_valid(struct ffa_device *ffa_dev);
-const struct ffa_dev_ops *ffa_dev_ops_get(struct ffa_device *dev);
- 
- #else
-static inline
-struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id)
-+static inline struct ffa_device *
-+ffa_device_register(const struct ffa_partition_info *part_info,
-+		    const struct ffa_ops *ops)
- {
- 	return NULL;
- }
- 
- static inline void ffa_device_unregister(struct ffa_device *dev) {}
- 
-+static inline void ffa_devices_unregister(void) {}
-+
- static inline int
- ffa_driver_register(struct ffa_driver *driver, struct module *owner,
- 		    const char *mod_name)
-@@ -76,11 +205,6 @@ static inline void ffa_driver_unregister(struct ffa_driver *driver) {}
- static inline
- bool ffa_device_is_valid(struct ffa_device *ffa_dev) { return false; }
- 
-static inline
-const struct ffa_dev_ops *ffa_dev_ops_get(struct ffa_device *dev)
-{
-	return NULL;
-}
- #endif /* CONFIG_ARM_FFA_TRANSPORT */
- 
- #define ffa_register(driver) \
-@@ -99,6 +223,11 @@ const struct ffa_dev_ops *ffa_dev_ops_get(struct ffa_device *dev)
- #define module_ffa_driver(__ffa_driver)	\
- 	module_driver(__ffa_driver, ffa_register, ffa_unregister)
- 
-+extern const struct bus_type ffa_bus_type;
-+
-+/* The FF-A 1.0 partition structure lacks the uuid[4] */
-+#define FFA_1_0_PARTITON_INFO_SZ	(8)
-+
- /* FFA transport related */
- struct ffa_partition_info {
- 	u16 id;
-@@ -109,9 +238,37 @@ struct ffa_partition_info {
- #define FFA_PARTITION_DIRECT_SEND	BIT(1)
- /* partition can send and receive indirect messages. */
- #define FFA_PARTITION_INDIRECT_MSG	BIT(2)
-+/* partition can receive notifications */
-+#define FFA_PARTITION_NOTIFICATION_RECV	BIT(3)
-+/* partition runs in the AArch64 execution state. */
-+#define FFA_PARTITION_AARCH64_EXEC	BIT(8)
-+/* partition supports receipt of direct request2 */
-+#define FFA_PARTITION_DIRECT_REQ2_RECV	BIT(9)
-+/* partition can send direct request2. */
-+#define FFA_PARTITION_DIRECT_REQ2_SEND	BIT(10)
- 	u32 properties;
-+	uuid_t uuid;
- };
- 
-+static inline
-+bool ffa_partition_check_property(struct ffa_device *dev, u32 property)
-+{
-+	return dev->properties & property;
-+}
-+
-+#define ffa_partition_supports_notify_recv(dev)	\
-+	ffa_partition_check_property(dev, FFA_PARTITION_NOTIFICATION_RECV)
-+
-+#define ffa_partition_supports_indirect_msg(dev)	\
-+	ffa_partition_check_property(dev, FFA_PARTITION_INDIRECT_MSG)
-+
-+#define ffa_partition_supports_direct_recv(dev)	\
-+	ffa_partition_check_property(dev, FFA_PARTITION_DIRECT_RECV)
-+
-+#define ffa_partition_supports_direct_req2_recv(dev)	\
-+	(ffa_partition_check_property(dev, FFA_PARTITION_DIRECT_REQ2_RECV) && \
-+	 !dev->mode_32bit)
-+
- /* For use with FFA_MSG_SEND_DIRECT_{REQ,RESP} which pass data via registers */
- struct ffa_send_direct_data {
- 	unsigned long data0; /* w3/x3 */
-@@ -121,6 +278,21 @@ struct ffa_send_direct_data {
- 	unsigned long data4; /* w7/x7 */
- };
- 
-+struct ffa_indirect_msg_hdr {
-+	u32 flags;
-+	u32 res0;
-+	u32 offset;
-+	u32 send_recv_id;
-+	u32 size;
-+	u32 res1;
-+	uuid_t uuid;
-+};
-+
-+/* For use with FFA_MSG_SEND_DIRECT_{REQ,RESP}2 which pass data via registers */
-+struct ffa_send_direct_data2 {
-+	unsigned long data[14]; /* x4-x17 */
-+};
-+
- struct ffa_mem_region_addr_range {
- 	/* The base IPA of the constituent memory region, aligned to 4 kiB */
- 	u64 address;
-@@ -161,11 +333,12 @@ struct ffa_mem_region_attributes {
- 	 */
- #define FFA_MEM_RETRIEVE_SELF_BORROWER	BIT(0)
- 	u8 flag;
-	u32 composite_off;
- 	/*
- 	 * Offset in bytes from the start of the outer `ffa_memory_region` to
- 	 * an `struct ffa_mem_region_addr_range`.
- 	 */
-+	u32 composite_off;
-+	u8 impdef_val[16];
- 	u64 reserved;
- };
- 
-@@ -186,8 +359,8 @@ struct ffa_mem_region {
- #define FFA_MEM_NON_SHAREABLE	(0)
- #define FFA_MEM_OUTER_SHAREABLE	(2)
- #define FFA_MEM_INNER_SHAREABLE	(3)
-	u8 attributes;
-	u8 reserved_0;
-+	/* Memory region attributes, upper byte MBZ pre v1.1 */
-+	u16 attributes;
- /*
-  * Clear memory region contents after unmapping it from the sender and
-  * before mapping it for any receiver.
-@@ -225,27 +398,57 @@ struct ffa_mem_region {
- 	 * memory region.
- 	 */
- 	u64 tag;
-	u32 reserved_1;
-+	/* Size of each endpoint memory access descriptor, MBZ pre v1.1 */
-+	u32 ep_mem_size;
- 	/*
- 	 * The number of `ffa_mem_region_attributes` entries included in this
- 	 * transaction.
- 	 */
- 	u32 ep_count;
- 	/*
-	 * An array of endpoint memory access descriptors.
-	 * Each one specifies a memory region offset, an endpoint and the
-	 * attributes with which this memory region should be mapped in that
-	 * endpoint's page table.
-+	 * 16-byte aligned offset from the base address of this descriptor
-+	 * to the first element of the endpoint memory access descriptor array
-+	 * Valid only from v1.1
- 	 */
-	struct ffa_mem_region_attributes ep_mem_access[];
-+	u32 ep_mem_offset;
-+	/* MBZ, valid only from v1.1 */
-+	u32 reserved[3];
- };
- 
-#define	COMPOSITE_OFFSET(x)	\
-	(offsetof(struct ffa_mem_region, ep_mem_access[x]))
- #define CONSTITUENTS_OFFSET(x)	\
- 	(offsetof(struct ffa_composite_mem_region, constituents[x]))
-#define COMPOSITE_CONSTITUENTS_OFFSET(x, y)	\
-	(COMPOSITE_OFFSET(x) + CONSTITUENTS_OFFSET(y))
-+
-+#define FFA_EMAD_HAS_IMPDEF_FIELD(version)	((version) >= FFA_VERSION_1_2)
-+#define FFA_MEM_REGION_HAS_EP_MEM_OFFSET(version) ((version) > FFA_VERSION_1_0)
-+
-+static inline u32 ffa_emad_size_get(u32 ffa_version)
-+{
-+	u32 sz;
-+	struct ffa_mem_region_attributes *ep_mem_access;
-+
-+	if (FFA_EMAD_HAS_IMPDEF_FIELD(ffa_version))
-+		sz = sizeof(*ep_mem_access);
-+	else
-+		sz = sizeof(*ep_mem_access) - sizeof(ep_mem_access->impdef_val);
-+
-+	return sz;
-+}
-+
-+static inline u32
-+ffa_mem_desc_offset(struct ffa_mem_region *buf, int count, u32 ffa_version)
-+{
-+	u32 offset = count * ffa_emad_size_get(ffa_version);
-+	/*
-+	 * Earlier to v1.1, the endpoint memory descriptor array started at
-+	 * offset 32(i.e. offset of ep_mem_offset in the current structure)
-+	 */
-+	if (!FFA_MEM_REGION_HAS_EP_MEM_OFFSET(ffa_version))
-+		offset += offsetof(struct ffa_mem_region, ep_mem_offset);
-+	else
-+		offset += sizeof(struct ffa_mem_region);
-+
-+	return offset;
-+}
- 
- struct ffa_mem_ops_args {
- 	bool use_txbuf;
-@@ -257,18 +460,56 @@ struct ffa_mem_ops_args {
- 	struct ffa_mem_region_attributes *attrs;
- };
- 
-struct ffa_dev_ops {
-+struct ffa_info_ops {
- 	u32 (*api_version_get)(void);
- 	int (*partition_info_get)(const char *uuid_str,
- 				  struct ffa_partition_info *buffer);
-+};
-+
-+struct ffa_msg_ops {
- 	void (*mode_32bit_set)(struct ffa_device *dev);
- 	int (*sync_send_receive)(struct ffa_device *dev,
- 				 struct ffa_send_direct_data *data);
-+	int (*indirect_send)(struct ffa_device *dev, void *buf, size_t sz);
-+	int (*sync_send_receive2)(struct ffa_device *dev,
-+				  struct ffa_send_direct_data2 *data);
-+};
-+
-+struct ffa_mem_ops {
- 	int (*memory_reclaim)(u64 g_handle, u32 flags);
-	int (*memory_share)(struct ffa_device *dev,
-			    struct ffa_mem_ops_args *args);
-	int (*memory_lend)(struct ffa_device *dev,
-			   struct ffa_mem_ops_args *args);
-+	int (*memory_share)(struct ffa_mem_ops_args *args);
-+	int (*memory_lend)(struct ffa_mem_ops_args *args);
-+};
-+
-+struct ffa_cpu_ops {
-+	int (*run)(struct ffa_device *dev, u16 vcpu);
-+};
-+
-+typedef void (*ffa_sched_recv_cb)(u16 vcpu, bool is_per_vcpu, void *cb_data);
-+typedef void (*ffa_notifier_cb)(int notify_id, void *cb_data);
-+typedef void (*ffa_fwk_notifier_cb)(int notify_id, void *cb_data, void *buf);
-+
-+struct ffa_notifier_ops {
-+	int (*sched_recv_cb_register)(struct ffa_device *dev,
-+				      ffa_sched_recv_cb cb, void *cb_data);
-+	int (*sched_recv_cb_unregister)(struct ffa_device *dev);
-+	int (*notify_request)(struct ffa_device *dev, bool per_vcpu,
-+			      ffa_notifier_cb cb, void *cb_data, int notify_id);
-+	int (*notify_relinquish)(struct ffa_device *dev, int notify_id);
-+	int (*fwk_notify_request)(struct ffa_device *dev,
-+				  ffa_fwk_notifier_cb cb, void *cb_data,
-+				  int notify_id);
-+	int (*fwk_notify_relinquish)(struct ffa_device *dev, int notify_id);
-+	int (*notify_send)(struct ffa_device *dev, int notify_id, bool per_vcpu,
-+			   u16 vcpu);
-+};
-+
-+struct ffa_ops {
-+	const struct ffa_info_ops *info_ops;
-+	const struct ffa_msg_ops *msg_ops;
-+	const struct ffa_mem_ops *mem_ops;
-+	const struct ffa_cpu_ops *cpu_ops;
-+	const struct ffa_notifier_ops *notifier_ops;
- };
- 
- #endif /* _LINUX_ARM_FFA_H */
-diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h
-new file mode 100644
-index 000000000000..7f00c5285a32
--- /dev/null
-+++ b/include/linux/arm_mpam.h
-@@ -0,0 +1,66 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/* Copyright (C) 2025 Arm Ltd. */
-+
-+#ifndef __LINUX_ARM_MPAM_H
-+#define __LINUX_ARM_MPAM_H
-+
-+#include <linux/acpi.h>
-+#include <linux/types.h>
-+
-+struct mpam_msc;
-+
-+enum mpam_msc_iface {
-+	MPAM_IFACE_MMIO,	/* a real MPAM MSC */
-+	MPAM_IFACE_PCC,		/* a fake MPAM MSC */
-+};
-+
-+enum mpam_class_types {
-+	MPAM_CLASS_CACHE,	/* Caches, e.g. L2, L3 */
-+	MPAM_CLASS_MEMORY,	/* Main memory */
-+	MPAM_CLASS_UNKNOWN,	/* Everything else, e.g. SMMU */
-+};
-+
-+#define MPAM_CLASS_ID_DEFAULT	255
-+
-+#ifdef CONFIG_ACPI_MPAM
-+int acpi_mpam_parse_resources(struct mpam_msc *msc,
-+			      struct acpi_mpam_msc_node *tbl_msc);
-+
-+int acpi_mpam_count_msc(void);
-+#else
-+static inline int acpi_mpam_parse_resources(struct mpam_msc *msc,
-+					    struct acpi_mpam_msc_node *tbl_msc)
-+{
-+	return -EINVAL;
-+}
-+
-+static inline int acpi_mpam_count_msc(void) { return -EINVAL; }
-+#endif
-+
-+#ifdef CONFIG_ARM64_MPAM_DRIVER
-+int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx,
-+		    enum mpam_class_types type, u8 class_id, int component_id);
-+#else
-+static inline int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx,
-+				  enum mpam_class_types type, u8 class_id,
-+				  int component_id)
-+{
-+	return -EINVAL;
-+}
-+#endif
-+
-+/**
-+ * mpam_register_requestor() - Register a requestor with the MPAM driver
-+ * @partid_max:		The maximum PARTID value the requestor can generate.
-+ * @pmg_max:		The maximum PMG value the requestor can generate.
-+ *
-+ * Registers a requestor with the MPAM driver to ensure the chosen system-wide
-+ * minimum PARTID and PMG values will allow the requestors features to be used.
-+ *
-+ * Returns an error if the registration is too late, and a larger PARTID/PMG
-+ * value has been advertised to user-space. In this case the requestor should
-+ * not use its MPAM features. Returns 0 on success.
-+ */
-+int mpam_register_requestor(u16 partid_max, u8 pmg_max);
-+
-+#endif /* __LINUX_ARM_MPAM_H */
-diff --git a/include/linux/arm_sdei.h b/include/linux/arm_sdei.h
-index 14dc461b0e82..f652a5028b59 100644
--- a/include/linux/arm_sdei.h
-+++ b/include/linux/arm_sdei.h
-@@ -46,11 +46,13 @@ int sdei_unregister_ghes(struct ghes *ghes);
- /* For use by arch code when CPU hotplug notifiers are not appropriate. */
- int sdei_mask_local_cpu(void);
- int sdei_unmask_local_cpu(void);
-void __init sdei_init(void);
-+void __init acpi_sdei_init(void);
-+void sdei_handler_abort(void);
- #else
- static inline int sdei_mask_local_cpu(void) { return 0; }
- static inline int sdei_unmask_local_cpu(void) { return 0; }
-static inline void sdei_init(void) { }
-+static inline void acpi_sdei_init(void) { }
-+static inline void sdei_handler_abort(void) { }
- #endif /* CONFIG_ARM_SDE_INTERFACE */
- 
- 
-diff --git a/include/linux/array_size.h b/include/linux/array_size.h
-new file mode 100644
-index 000000000000..06d7d83196ca
--- /dev/null
-+++ b/include/linux/array_size.h
-@@ -0,0 +1,13 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_ARRAY_SIZE_H
-+#define _LINUX_ARRAY_SIZE_H
-+
-+#include <linux/compiler.h>
-+
-+/**
-+ * ARRAY_SIZE - get the number of elements in array @arr
-+ * @arr: array to be sized
-+ */
-+#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]) + __must_be_array(arr))
-+
-+#endif  /* _LINUX_ARRAY_SIZE_H */
-diff --git a/include/linux/asn1_decoder.h b/include/linux/asn1_decoder.h
-index 83f9c6e1e5e9..b41bce82a191 100644
--- a/include/linux/asn1_decoder.h
-+++ b/include/linux/asn1_decoder.h
-@@ -9,6 +9,7 @@
- #define _LINUX_ASN1_DECODER_H
- 
- #include <linux/asn1.h>
-+#include <linux/types.h>
- 
- struct asn1_decoder;
- 
-diff --git a/include/linux/asn1_encoder.h b/include/linux/asn1_encoder.h
-index 08cd0c2ad34f..d17484dffb74 100644
--- a/include/linux/asn1_encoder.h
-+++ b/include/linux/asn1_encoder.h
-@@ -6,7 +6,6 @@
- #include <linux/types.h>
- #include <linux/asn1.h>
- #include <linux/asn1_ber_bytecode.h>
-#include <linux/bug.h>
- 
- #define asn1_oid_len(oid) (sizeof(oid)/sizeof(u32))
- unsigned char *
-diff --git a/include/linux/async.h b/include/linux/async.h
-index cce4ad31e8fc..19b778d08600 100644
--- a/include/linux/async.h
-+++ b/include/linux/async.h
-@@ -90,6 +90,8 @@ async_schedule_dev(async_func_t func, struct device *dev)
- 	return async_schedule_node(func, dev, dev_to_node(dev));
- }
- 
-+bool async_schedule_dev_nocall(async_func_t func, struct device *dev);
-+
- /**
-  * async_schedule_dev_domain - A device specific version of async_schedule_domain
-  * @func: function to execute asynchronously
-@@ -118,4 +120,5 @@ extern void async_synchronize_cookie(async_cookie_t cookie);
- extern void async_synchronize_cookie_domain(async_cookie_t cookie,
- 					    struct async_domain *domain);
- extern bool current_is_async(void);
-+extern void async_init(void);
- #endif
-diff --git a/include/linux/async_tx.h b/include/linux/async_tx.h
-index 5cc73d7e5b52..1ca9f9e05f4f 100644
--- a/include/linux/async_tx.h
-+++ b/include/linux/async_tx.h
-@@ -168,11 +168,6 @@ async_xor_offs(struct page *dest, unsigned int offset,
- 		int src_cnt, size_t len, struct async_submit_ctl *submit);
- 
- struct dma_async_tx_descriptor *
-async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
-	      int src_cnt, size_t len, enum sum_check_flags *result,
-	      struct async_submit_ctl *submit);
-
-struct dma_async_tx_descriptor *
- async_xor_val_offs(struct page *dest, unsigned int offset,
- 		struct page **src_list, unsigned int *src_offset,
- 		int src_cnt, size_t len, enum sum_check_flags *result,
-diff --git a/include/linux/ata.h b/include/linux/ata.h
-index 21292b5bbb55..54b416e26995 100644
--- a/include/linux/ata.h
-+++ b/include/linux/ata.h
-@@ -16,7 +16,6 @@
- #include <linux/bits.h>
- #include <linux/string.h>
- #include <linux/types.h>
-#include <asm/byteorder.h>
- 
- /* defines only for the constants which don't work well as enums */
- #define ATA_DMA_BOUNDARY	0xffffUL
-@@ -30,6 +29,7 @@ enum {
- 	ATA_MAX_SECTORS_128	= 128,
- 	ATA_MAX_SECTORS		= 256,
- 	ATA_MAX_SECTORS_1024    = 1024,
-+	ATA_MAX_SECTORS_8191    = 8191,
- 	ATA_MAX_SECTORS_LBA48	= 65535,/* avoid count to be 0000h */
- 	ATA_MAX_SECTORS_TAPE	= 65535,
- 	ATA_MAX_TRIM_RNUM	= 64,	/* 512-byte payload / (6-byte LBA + 2-byte range per entry) */
-@@ -323,15 +323,21 @@ enum {
- 	ATA_LOG_SATA_NCQ	= 0x10,
- 	ATA_LOG_NCQ_NON_DATA	= 0x12,
- 	ATA_LOG_NCQ_SEND_RECV	= 0x13,
-+	ATA_LOG_CDL		= 0x18,
-+	ATA_LOG_CDL_SIZE	= ATA_SECT_SIZE,
- 	ATA_LOG_IDENTIFY_DEVICE	= 0x30,
-+	ATA_LOG_SENSE_NCQ	= 0x0F,
-+	ATA_LOG_SENSE_NCQ_SIZE	= ATA_SECT_SIZE * 2,
- 	ATA_LOG_CONCURRENT_POSITIONING_RANGES = 0x47,
- 
- 	/* Identify device log pages: */
-+	ATA_LOG_SUPPORTED_CAPABILITIES	= 0x03,
-+	ATA_LOG_CURRENT_SETTINGS  = 0x04,
- 	ATA_LOG_SECURITY	  = 0x06,
- 	ATA_LOG_SATA_SETTINGS	  = 0x08,
- 	ATA_LOG_ZONED_INFORMATION = 0x09,
- 
-	/* Identify device SATA settings log:*/
-+	/* Identify device SATA settings log: */
- 	ATA_LOG_DEVSLP_OFFSET	  = 0x30,
- 	ATA_LOG_DEVSLP_SIZE	  = 0x08,
- 	ATA_LOG_DEVSLP_MDAT	  = 0x00,
-@@ -416,6 +422,8 @@ enum {
- 	SETFEATURES_SATA_ENABLE = 0x10, /* Enable use of SATA feature */
- 	SETFEATURES_SATA_DISABLE = 0x90, /* Disable use of SATA feature */
- 
-+	SETFEATURES_CDL		= 0x0d, /* Enable/disable cmd duration limits */
-+
- 	/* SETFEATURE Sector counts for SATA features */
- 	SATA_FPDMA_OFFSET	= 0x01,	/* FPDMA non-zero buffer offsets */
- 	SATA_FPDMA_AA		= 0x02, /* FPDMA Setup FIS Auto-Activate */
-@@ -426,6 +434,7 @@ enum {
- 	SATA_DEVSLP		= 0x09,	/* Device Sleep */
- 
- 	SETFEATURE_SENSE_DATA	= 0xC3, /* Sense Data Reporting feature */
-+	SETFEATURE_SENSE_DATA_SUCC_NCQ = 0xC4, /* Sense Data for successful NCQ commands */
- 
- 	/* feature values for SET_MAX */
- 	ATA_SET_MAX_ADDR	= 0x00,
-@@ -558,6 +567,7 @@ struct ata_bmdma_prd {
- #define ata_id_has_ncq(id)	((id)[ATA_ID_SATA_CAPABILITY] & (1 << 8))
- #define ata_id_queue_depth(id)	(((id)[ATA_ID_QUEUE_DEPTH] & 0x1f) + 1)
- #define ata_id_removable(id)	((id)[ATA_ID_CONFIG] & (1 << 7))
-+#define ata_id_is_locked(id)	(((id)[ATA_ID_DLF] & 0x7) == 0x7)
- #define ata_id_has_atapi_AN(id)	\
- 	((((id)[ATA_ID_SATA_CAPABILITY] != 0x0000) && \
- 	  ((id)[ATA_ID_SATA_CAPABILITY] != 0xffff)) && \
-@@ -566,6 +576,18 @@ struct ata_bmdma_prd {
- 	((((id)[ATA_ID_SATA_CAPABILITY] != 0x0000) && \
- 	  ((id)[ATA_ID_SATA_CAPABILITY] != 0xffff)) && \
- 	 ((id)[ATA_ID_FEATURE_SUPP] & (1 << 2)))
-+#define ata_id_has_devslp(id)	\
-+	((((id)[ATA_ID_SATA_CAPABILITY] != 0x0000) && \
-+	  ((id)[ATA_ID_SATA_CAPABILITY] != 0xffff)) && \
-+	 ((id)[ATA_ID_FEATURE_SUPP] & (1 << 8)))
-+#define ata_id_has_ncq_autosense(id) \
-+	((((id)[ATA_ID_SATA_CAPABILITY] != 0x0000) && \
-+	  ((id)[ATA_ID_SATA_CAPABILITY] != 0xffff)) && \
-+	 ((id)[ATA_ID_FEATURE_SUPP] & (1 << 7)))
-+#define ata_id_has_dipm(id)	\
-+	((((id)[ATA_ID_SATA_CAPABILITY] != 0x0000) && \
-+	  ((id)[ATA_ID_SATA_CAPABILITY] != 0xffff)) && \
-+	 ((id)[ATA_ID_FEATURE_SUPP] & (1 << 3)))
- #define ata_id_iordy_disable(id) ((id)[ATA_ID_CAPABILITY] & (1 << 10))
- #define ata_id_has_iordy(id) ((id)[ATA_ID_CAPABILITY] & (1 << 11))
- #define ata_id_u32(id,n)	\
-@@ -578,9 +600,6 @@ struct ata_bmdma_prd {
- 
- #define ata_id_cdb_intr(id)	(((id)[ATA_ID_CONFIG] & 0x60) == 0x20)
- #define ata_id_has_da(id)	((id)[ATA_ID_SATA_CAPABILITY_2] & (1 << 4))
-#define ata_id_has_devslp(id)	((id)[ATA_ID_FEATURE_SUPP] & (1 << 8))
-#define ata_id_has_ncq_autosense(id) \
-				((id)[ATA_ID_FEATURE_SUPP] & (1 << 7))
- 
- static inline bool ata_id_has_hipm(const u16 *id)
- {
-@@ -592,17 +611,6 @@ static inline bool ata_id_has_hipm(const u16 *id)
- 	return val & (1 << 9);
- }
- 
-static inline bool ata_id_has_dipm(const u16 *id)
-{
-	u16 val = id[ATA_ID_FEATURE_SUPP];
-
-	if (val == 0 || val == 0xffff)
-		return false;
-
-	return val & (1 << 3);
-}
-
-
- static inline bool ata_id_has_fua(const u16 *id)
- {
- 	if ((id[ATA_ID_CFSSE] & 0xC000) != 0x4000)
-@@ -617,15 +625,6 @@ static inline bool ata_id_has_flush(const u16 *id)
- 	return id[ATA_ID_COMMAND_SET_2] & (1 << 12);
- }
- 
-static inline bool ata_id_flush_enabled(const u16 *id)
-{
-	if (ata_id_has_flush(id) == 0)
-		return false;
-	if ((id[ATA_ID_CSF_DEFAULT] & 0xC000) != 0x4000)
-		return false;
-	return id[ATA_ID_CFS_ENABLE_2] & (1 << 12);
-}
-
- static inline bool ata_id_has_flush_ext(const u16 *id)
- {
- 	if ((id[ATA_ID_COMMAND_SET_2] & 0xC000) != 0x4000)
-@@ -633,19 +632,6 @@ static inline bool ata_id_has_flush_ext(const u16 *id)
- 	return id[ATA_ID_COMMAND_SET_2] & (1 << 13);
- }
- 
-static inline bool ata_id_flush_ext_enabled(const u16 *id)
-{
-	if (ata_id_has_flush_ext(id) == 0)
-		return false;
-	if ((id[ATA_ID_CSF_DEFAULT] & 0xC000) != 0x4000)
-		return false;
-	/*
-	 * some Maxtor disks have bit 13 defined incorrectly
-	 * so check bit 10 too
-	 */
-	return (id[ATA_ID_CFS_ENABLE_2] & 0x2400) == 0x2400;
-}
-
- static inline u32 ata_id_logical_sector_size(const u16 *id)
- {
- 	/* T13/1699-D Revision 6a, Sep 6, 2008. Page 128.
-@@ -700,15 +686,6 @@ static inline bool ata_id_has_lba48(const u16 *id)
- 	return id[ATA_ID_COMMAND_SET_2] & (1 << 10);
- }
- 
-static inline bool ata_id_lba48_enabled(const u16 *id)
-{
-	if (ata_id_has_lba48(id) == 0)
-		return false;
-	if ((id[ATA_ID_CSF_DEFAULT] & 0xC000) != 0x4000)
-		return false;
-	return id[ATA_ID_CFS_ENABLE_2] & (1 << 10);
-}
-
- static inline bool ata_id_hpa_enabled(const u16 *id)
- {
- 	/* Yes children, word 83 valid bits cover word 82 data */
-@@ -771,16 +748,21 @@ static inline bool ata_id_has_read_log_dma_ext(const u16 *id)
- 
- static inline bool ata_id_has_sense_reporting(const u16 *id)
- {
-	if (!(id[ATA_ID_CFS_ENABLE_2] & (1 << 15)))
-+	if (!(id[ATA_ID_CFS_ENABLE_2] & BIT(15)))
-+		return false;
-+	if ((id[ATA_ID_COMMAND_SET_3] & (BIT(15) | BIT(14))) != BIT(14))
- 		return false;
-	return id[ATA_ID_COMMAND_SET_3] & (1 << 6);
-+	return id[ATA_ID_COMMAND_SET_3] & BIT(6);
- }
- 
- static inline bool ata_id_sense_reporting_enabled(const u16 *id)
- {
-	if (!(id[ATA_ID_CFS_ENABLE_2] & (1 << 15)))
-+	if (!ata_id_has_sense_reporting(id))
-+		return false;
-+	/* ata_id_has_sense_reporting() == true, word 86 must have bit 15 set */
-+	if ((id[ATA_ID_COMMAND_SET_4] & (BIT(15) | BIT(14))) != BIT(14))
- 		return false;
-	return id[ATA_ID_COMMAND_SET_4] & (1 << 6);
-+	return id[ATA_ID_COMMAND_SET_4] & BIT(6);
- }
- 
- /**
-@@ -1045,76 +1027,6 @@ static inline bool atapi_id_dmadir(const u16 *dev_id)
- 	return ata_id_major_version(dev_id) >= 7 && (dev_id[62] & 0x8000);
- }
- 
-/*
- * ata_id_is_lba_capacity_ok() performs a sanity check on
- * the claimed LBA capacity value for the device.
- *
- * Returns 1 if LBA capacity looks sensible, 0 otherwise.
- *
- * It is called only once for each device.
- */
-static inline bool ata_id_is_lba_capacity_ok(u16 *id)
-{
-	unsigned long lba_sects, chs_sects, head, tail;
-
-	/* No non-LBA info .. so valid! */
-	if (id[ATA_ID_CYLS] == 0)
-		return true;
-
-	lba_sects = ata_id_u32(id, ATA_ID_LBA_CAPACITY);
-
-	/*
-	 * The ATA spec tells large drives to return
-	 * C/H/S = 16383/16/63 independent of their size.
-	 * Some drives can be jumpered to use 15 heads instead of 16.
-	 * Some drives can be jumpered to use 4092 cyls instead of 16383.
-	 */
-	if ((id[ATA_ID_CYLS] == 16383 ||
-	     (id[ATA_ID_CYLS] == 4092 && id[ATA_ID_CUR_CYLS] == 16383)) &&
-	    id[ATA_ID_SECTORS] == 63 &&
-	    (id[ATA_ID_HEADS] == 15 || id[ATA_ID_HEADS] == 16) &&
-	    (lba_sects >= 16383 * 63 * id[ATA_ID_HEADS]))
-		return true;
-
-	chs_sects = id[ATA_ID_CYLS] * id[ATA_ID_HEADS] * id[ATA_ID_SECTORS];
-
-	/* perform a rough sanity check on lba_sects: within 10% is OK */
-	if (lba_sects - chs_sects < chs_sects/10)
-		return true;
-
-	/* some drives have the word order reversed */
-	head = (lba_sects >> 16) & 0xffff;
-	tail = lba_sects & 0xffff;
-	lba_sects = head | (tail << 16);
-
-	if (lba_sects - chs_sects < chs_sects/10) {
-		*(__le32 *)&id[ATA_ID_LBA_CAPACITY] = __cpu_to_le32(lba_sects);
-		return true;	/* LBA capacity is (now) good */
-	}
-
-	return false;	/* LBA capacity value may be bad */
-}
-
-static inline void ata_id_to_hd_driveid(u16 *id)
-{
-#ifdef __BIG_ENDIAN
-	/* accessed in struct hd_driveid as 8-bit values */
-	id[ATA_ID_MAX_MULTSECT]	 = __cpu_to_le16(id[ATA_ID_MAX_MULTSECT]);
-	id[ATA_ID_CAPABILITY]	 = __cpu_to_le16(id[ATA_ID_CAPABILITY]);
-	id[ATA_ID_OLD_PIO_MODES] = __cpu_to_le16(id[ATA_ID_OLD_PIO_MODES]);
-	id[ATA_ID_OLD_DMA_MODES] = __cpu_to_le16(id[ATA_ID_OLD_DMA_MODES]);
-	id[ATA_ID_MULTSECT]	 = __cpu_to_le16(id[ATA_ID_MULTSECT]);
-
-	/* as 32-bit values */
-	*(u32 *)&id[ATA_ID_LBA_CAPACITY] = ata_id_u32(id, ATA_ID_LBA_CAPACITY);
-	*(u32 *)&id[ATA_ID_SPG]		 = ata_id_u32(id, ATA_ID_SPG);
-
-	/* as 64-bit value */
-	*(u64 *)&id[ATA_ID_LBA_CAPACITY_2] =
-		ata_id_u64(id, ATA_ID_LBA_CAPACITY_2);
-#endif
-}
-
- static inline bool ata_ok(u8 status)
- {
- 	return ((status & (ATA_BUSY | ATA_DRDY | ATA_DF | ATA_DRQ | ATA_ERR))
-diff --git a/include/linux/ata_platform.h b/include/linux/ata_platform.h
-index 9cafec92282d..b9745cc08e38 100644
--- a/include/linux/ata_platform.h
-+++ b/include/linux/ata_platform.h
-@@ -19,7 +19,7 @@ extern int __pata_platform_probe(struct device *dev,
- 				 struct resource *irq_res,
- 				 unsigned int ioport_shift,
- 				 int __pio_mask,
-				 struct scsi_host_template *sht,
-+				 const struct scsi_host_template *sht,
- 				 bool use16bit);
- 
- /*
-diff --git a/include/linux/ath9k_platform.h b/include/linux/ath9k_platform.h
-deleted file mode 100644
-index 76860a461ed2..000000000000
--- a/include/linux/ath9k_platform.h
-+++ /dev/null
-@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2008 Atheros Communications Inc.
- * Copyright (c) 2009 Gabor Juhos <juhosg@openwrt.org>
- * Copyright (c) 2009 Imre Kaloz <kaloz@openwrt.org>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
- * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
- * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
- * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
-
-#ifndef _LINUX_ATH9K_PLATFORM_H
-#define _LINUX_ATH9K_PLATFORM_H
-
-#define ATH9K_PLAT_EEP_MAX_WORDS	2048
-
-struct ath9k_platform_data {
-	const char *eeprom_name;
-
-	u16 eeprom_data[ATH9K_PLAT_EEP_MAX_WORDS];
-	u8 *macaddr;
-
-	int led_pin;
-	u32 gpio_mask;
-	u32 gpio_val;
-
-	u32 bt_active_pin;
-	u32 bt_priority_pin;
-	u32 wlan_active_pin;
-
-	bool endian_check;
-	bool is_clk_25mhz;
-	bool tx_gain_buffalo;
-	bool disable_2ghz;
-	bool disable_5ghz;
-	bool led_active_high;
-
-	int (*get_mac_revision)(void);
-	int (*external_reset)(void);
-
-	bool use_eeprom;
-};
-
-#endif /* _LINUX_ATH9K_PLATFORM_H */
-diff --git a/include/linux/atmdev.h b/include/linux/atmdev.h
-index 9b02961d65ee..70807c679f1a 100644
--- a/include/linux/atmdev.h
-+++ b/include/linux/atmdev.h
-@@ -185,6 +185,7 @@ struct atmdev_ops { /* only send is required */
- 	int (*compat_ioctl)(struct atm_dev *dev,unsigned int cmd,
- 			    void __user *arg);
- #endif
-+	int (*pre_send)(struct atm_vcc *vcc, struct sk_buff *skb);
- 	int (*send)(struct atm_vcc *vcc,struct sk_buff *skb);
- 	int (*send_bh)(struct atm_vcc *vcc, struct sk_buff *skb);
- 	int (*send_oam)(struct atm_vcc *vcc,void *cell,int flags);
-@@ -249,6 +250,12 @@ static inline void atm_account_tx(struct atm_vcc *vcc, struct sk_buff *skb)
- 	ATM_SKB(skb)->atm_options = vcc->atm_options;
- }
- 
-+static inline void atm_return_tx(struct atm_vcc *vcc, struct sk_buff *skb)
-+{
-+	WARN_ON_ONCE(refcount_sub_and_test(ATM_SKB(skb)->acct_truesize,
-+					   &sk_atm(vcc)->sk_wmem_alloc));
-+}
-+
- static inline void atm_force_charge(struct atm_vcc *vcc,int truesize)
- {
- 	atomic_add(truesize, &sk_atm(vcc)->sk_rmem_alloc);
-diff --git a/include/linux/atmel-mci.h b/include/linux/atmel-mci.h
-deleted file mode 100644
-index 1491af38cc6e..000000000000
--- a/include/linux/atmel-mci.h
-+++ /dev/null
-@@ -1,46 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_ATMEL_MCI_H
-#define __LINUX_ATMEL_MCI_H
-
-#include <linux/types.h>
-#include <linux/dmaengine.h>
-
-#define ATMCI_MAX_NR_SLOTS	2
-
-/**
- * struct mci_slot_pdata - board-specific per-slot configuration
- * @bus_width: Number of data lines wired up the slot
- * @detect_pin: GPIO pin wired to the card detect switch
- * @wp_pin: GPIO pin wired to the write protect sensor
- * @detect_is_active_high: The state of the detect pin when it is active
- * @non_removable: The slot is not removable, only detect once
- *
- * If a given slot is not present on the board, @bus_width should be
- * set to 0. The other fields are ignored in this case.
- *
- * Any pins that aren't available should be set to a negative value.
- *
- * Note that support for multiple slots is experimental -- some cards
- * might get upset if we don't get the clock management exactly right.
- * But in most cases, it should work just fine.
- */
-struct mci_slot_pdata {
-	unsigned int		bus_width;
-	int			detect_pin;
-	int			wp_pin;
-	bool			detect_is_active_high;
-	bool			non_removable;
-};
-
-/**
- * struct mci_platform_data - board-specific MMC/SDcard configuration
- * @dma_slave: DMA slave interface to use in data transfers.
- * @slot: Per-slot configuration data.
- */
-struct mci_platform_data {
-	void			*dma_slave;
-	dma_filter_fn		dma_filter;
-	struct mci_slot_pdata	slot[ATMCI_MAX_NR_SLOTS];
-};
-
-#endif /* __LINUX_ATMEL_MCI_H */
-diff --git a/include/linux/atomic/atomic-arch-fallback.h b/include/linux/atomic/atomic-arch-fallback.h
-index 77bc5522e61c..2f9d36b72bd8 100644
--- a/include/linux/atomic/atomic-arch-fallback.h
-+++ b/include/linux/atomic/atomic-arch-fallback.h
-@@ -8,2452 +8,4686 @@
- 
- #include <linux/compiler.h>
- 
-#ifndef arch_xchg_relaxed
-#define arch_xchg_acquire arch_xchg
-#define arch_xchg_release arch_xchg
-#define arch_xchg_relaxed arch_xchg
-#else /* arch_xchg_relaxed */
-
-#ifndef arch_xchg_acquire
-#define arch_xchg_acquire(...) \
-	__atomic_op_acquire(arch_xchg, __VA_ARGS__)
-+#if defined(arch_xchg)
-+#define raw_xchg arch_xchg
-+#elif defined(arch_xchg_relaxed)
-+#define raw_xchg(...) \
-+	__atomic_op_fence(arch_xchg, __VA_ARGS__)
-+#else
-+extern void raw_xchg_not_implemented(void);
-+#define raw_xchg(...) raw_xchg_not_implemented()
- #endif
- 
-#ifndef arch_xchg_release
-#define arch_xchg_release(...) \
-	__atomic_op_release(arch_xchg, __VA_ARGS__)
-+#if defined(arch_xchg_acquire)
-+#define raw_xchg_acquire arch_xchg_acquire
-+#elif defined(arch_xchg_relaxed)
-+#define raw_xchg_acquire(...) \
-+	__atomic_op_acquire(arch_xchg, __VA_ARGS__)
-+#elif defined(arch_xchg)
-+#define raw_xchg_acquire arch_xchg
-+#else
-+extern void raw_xchg_acquire_not_implemented(void);
-+#define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented()
- #endif
- 
-#ifndef arch_xchg
-#define arch_xchg(...) \
-	__atomic_op_fence(arch_xchg, __VA_ARGS__)
-+#if defined(arch_xchg_release)
-+#define raw_xchg_release arch_xchg_release
-+#elif defined(arch_xchg_relaxed)
-+#define raw_xchg_release(...) \
-+	__atomic_op_release(arch_xchg, __VA_ARGS__)
-+#elif defined(arch_xchg)
-+#define raw_xchg_release arch_xchg
-+#else
-+extern void raw_xchg_release_not_implemented(void);
-+#define raw_xchg_release(...) raw_xchg_release_not_implemented()
-+#endif
-+
-+#if defined(arch_xchg_relaxed)
-+#define raw_xchg_relaxed arch_xchg_relaxed
-+#elif defined(arch_xchg)
-+#define raw_xchg_relaxed arch_xchg
-+#else
-+extern void raw_xchg_relaxed_not_implemented(void);
-+#define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented()
-+#endif
-+
-+#if defined(arch_cmpxchg)
-+#define raw_cmpxchg arch_cmpxchg
-+#elif defined(arch_cmpxchg_relaxed)
-+#define raw_cmpxchg(...) \
-+	__atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
-+#else
-+extern void raw_cmpxchg_not_implemented(void);
-+#define raw_cmpxchg(...) raw_cmpxchg_not_implemented()
- #endif
- 
-#endif /* arch_xchg_relaxed */
-
-#ifndef arch_cmpxchg_relaxed
-#define arch_cmpxchg_acquire arch_cmpxchg
-#define arch_cmpxchg_release arch_cmpxchg
-#define arch_cmpxchg_relaxed arch_cmpxchg
-#else /* arch_cmpxchg_relaxed */
-
-#ifndef arch_cmpxchg_acquire
-#define arch_cmpxchg_acquire(...) \
-+#if defined(arch_cmpxchg_acquire)
-+#define raw_cmpxchg_acquire arch_cmpxchg_acquire
-+#elif defined(arch_cmpxchg_relaxed)
-+#define raw_cmpxchg_acquire(...) \
- 	__atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
-+#elif defined(arch_cmpxchg)
-+#define raw_cmpxchg_acquire arch_cmpxchg
-+#else
-+extern void raw_cmpxchg_acquire_not_implemented(void);
-+#define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented()
- #endif
- 
-#ifndef arch_cmpxchg_release
-#define arch_cmpxchg_release(...) \
-+#if defined(arch_cmpxchg_release)
-+#define raw_cmpxchg_release arch_cmpxchg_release
-+#elif defined(arch_cmpxchg_relaxed)
-+#define raw_cmpxchg_release(...) \
- 	__atomic_op_release(arch_cmpxchg, __VA_ARGS__)
-+#elif defined(arch_cmpxchg)
-+#define raw_cmpxchg_release arch_cmpxchg
-+#else
-+extern void raw_cmpxchg_release_not_implemented(void);
-+#define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented()
-+#endif
-+
-+#if defined(arch_cmpxchg_relaxed)
-+#define raw_cmpxchg_relaxed arch_cmpxchg_relaxed
-+#elif defined(arch_cmpxchg)
-+#define raw_cmpxchg_relaxed arch_cmpxchg
-+#else
-+extern void raw_cmpxchg_relaxed_not_implemented(void);
-+#define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented()
-+#endif
-+
-+#if defined(arch_cmpxchg64)
-+#define raw_cmpxchg64 arch_cmpxchg64
-+#elif defined(arch_cmpxchg64_relaxed)
-+#define raw_cmpxchg64(...) \
-+	__atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
-+#else
-+extern void raw_cmpxchg64_not_implemented(void);
-+#define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented()
- #endif
- 
-#ifndef arch_cmpxchg
-#define arch_cmpxchg(...) \
-	__atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
-#endif
-
-#endif /* arch_cmpxchg_relaxed */
-
-#ifndef arch_cmpxchg64_relaxed
-#define arch_cmpxchg64_acquire arch_cmpxchg64
-#define arch_cmpxchg64_release arch_cmpxchg64
-#define arch_cmpxchg64_relaxed arch_cmpxchg64
-#else /* arch_cmpxchg64_relaxed */
-
-#ifndef arch_cmpxchg64_acquire
-#define arch_cmpxchg64_acquire(...) \
-+#if defined(arch_cmpxchg64_acquire)
-+#define raw_cmpxchg64_acquire arch_cmpxchg64_acquire
-+#elif defined(arch_cmpxchg64_relaxed)
-+#define raw_cmpxchg64_acquire(...) \
- 	__atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
-+#elif defined(arch_cmpxchg64)
-+#define raw_cmpxchg64_acquire arch_cmpxchg64
-+#else
-+extern void raw_cmpxchg64_acquire_not_implemented(void);
-+#define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented()
- #endif
- 
-#ifndef arch_cmpxchg64_release
-#define arch_cmpxchg64_release(...) \
-+#if defined(arch_cmpxchg64_release)
-+#define raw_cmpxchg64_release arch_cmpxchg64_release
-+#elif defined(arch_cmpxchg64_relaxed)
-+#define raw_cmpxchg64_release(...) \
- 	__atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
-+#elif defined(arch_cmpxchg64)
-+#define raw_cmpxchg64_release arch_cmpxchg64
-+#else
-+extern void raw_cmpxchg64_release_not_implemented(void);
-+#define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented()
-+#endif
-+
-+#if defined(arch_cmpxchg64_relaxed)
-+#define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed
-+#elif defined(arch_cmpxchg64)
-+#define raw_cmpxchg64_relaxed arch_cmpxchg64
-+#else
-+extern void raw_cmpxchg64_relaxed_not_implemented(void);
-+#define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented()
-+#endif
-+
-+#if defined(arch_cmpxchg128)
-+#define raw_cmpxchg128 arch_cmpxchg128
-+#elif defined(arch_cmpxchg128_relaxed)
-+#define raw_cmpxchg128(...) \
-+	__atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
-+#else
-+extern void raw_cmpxchg128_not_implemented(void);
-+#define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented()
-+#endif
-+
-+#if defined(arch_cmpxchg128_acquire)
-+#define raw_cmpxchg128_acquire arch_cmpxchg128_acquire
-+#elif defined(arch_cmpxchg128_relaxed)
-+#define raw_cmpxchg128_acquire(...) \
-+	__atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
-+#elif defined(arch_cmpxchg128)
-+#define raw_cmpxchg128_acquire arch_cmpxchg128
-+#else
-+extern void raw_cmpxchg128_acquire_not_implemented(void);
-+#define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented()
-+#endif
-+
-+#if defined(arch_cmpxchg128_release)
-+#define raw_cmpxchg128_release arch_cmpxchg128_release
-+#elif defined(arch_cmpxchg128_relaxed)
-+#define raw_cmpxchg128_release(...) \
-+	__atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
-+#elif defined(arch_cmpxchg128)
-+#define raw_cmpxchg128_release arch_cmpxchg128
-+#else
-+extern void raw_cmpxchg128_release_not_implemented(void);
-+#define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented()
-+#endif
-+
-+#if defined(arch_cmpxchg128_relaxed)
-+#define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed
-+#elif defined(arch_cmpxchg128)
-+#define raw_cmpxchg128_relaxed arch_cmpxchg128
-+#else
-+extern void raw_cmpxchg128_relaxed_not_implemented(void);
-+#define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented()
-+#endif
-+
-+#if defined(arch_try_cmpxchg)
-+#define raw_try_cmpxchg arch_try_cmpxchg
-+#elif defined(arch_try_cmpxchg_relaxed)
-+#define raw_try_cmpxchg(...) \
-+	__atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
-+#else
-+#define raw_try_cmpxchg(_ptr, _oldp, _new) \
-+({ \
-+	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-+	___r = raw_cmpxchg((_ptr), ___o, (_new)); \
-+	if (unlikely(___r != ___o)) \
-+		*___op = ___r; \
-+	likely(___r == ___o); \
-+})
- #endif
- 
-#ifndef arch_cmpxchg64
-#define arch_cmpxchg64(...) \
-	__atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
-+#if defined(arch_try_cmpxchg_acquire)
-+#define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire
-+#elif defined(arch_try_cmpxchg_relaxed)
-+#define raw_try_cmpxchg_acquire(...) \
-+	__atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
-+#elif defined(arch_try_cmpxchg)
-+#define raw_try_cmpxchg_acquire arch_try_cmpxchg
-+#else
-+#define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \
-+({ \
-+	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-+	___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \
-+	if (unlikely(___r != ___o)) \
-+		*___op = ___r; \
-+	likely(___r == ___o); \
-+})
- #endif
- 
-#endif /* arch_cmpxchg64_relaxed */
-
-#ifndef arch_try_cmpxchg_relaxed
-#ifdef arch_try_cmpxchg
-#define arch_try_cmpxchg_acquire arch_try_cmpxchg
-#define arch_try_cmpxchg_release arch_try_cmpxchg
-#define arch_try_cmpxchg_relaxed arch_try_cmpxchg
-#endif /* arch_try_cmpxchg */
-
-#ifndef arch_try_cmpxchg
-#define arch_try_cmpxchg(_ptr, _oldp, _new) \
-+#if defined(arch_try_cmpxchg_release)
-+#define raw_try_cmpxchg_release arch_try_cmpxchg_release
-+#elif defined(arch_try_cmpxchg_relaxed)
-+#define raw_try_cmpxchg_release(...) \
-+	__atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
-+#elif defined(arch_try_cmpxchg)
-+#define raw_try_cmpxchg_release arch_try_cmpxchg
-+#else
-+#define raw_try_cmpxchg_release(_ptr, _oldp, _new) \
- ({ \
- 	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-	___r = arch_cmpxchg((_ptr), ___o, (_new)); \
-+	___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \
- 	if (unlikely(___r != ___o)) \
- 		*___op = ___r; \
- 	likely(___r == ___o); \
- })
-#endif /* arch_try_cmpxchg */
-+#endif
- 
-#ifndef arch_try_cmpxchg_acquire
-#define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \
-+#if defined(arch_try_cmpxchg_relaxed)
-+#define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed
-+#elif defined(arch_try_cmpxchg)
-+#define raw_try_cmpxchg_relaxed arch_try_cmpxchg
-+#else
-+#define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
- ({ \
- 	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-	___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \
-+	___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \
- 	if (unlikely(___r != ___o)) \
- 		*___op = ___r; \
- 	likely(___r == ___o); \
- })
-#endif /* arch_try_cmpxchg_acquire */
-+#endif
- 
-#ifndef arch_try_cmpxchg_release
-#define arch_try_cmpxchg_release(_ptr, _oldp, _new) \
-+#if defined(arch_try_cmpxchg64)
-+#define raw_try_cmpxchg64 arch_try_cmpxchg64
-+#elif defined(arch_try_cmpxchg64_relaxed)
-+#define raw_try_cmpxchg64(...) \
-+	__atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
-+#else
-+#define raw_try_cmpxchg64(_ptr, _oldp, _new) \
- ({ \
- 	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-	___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \
-+	___r = raw_cmpxchg64((_ptr), ___o, (_new)); \
- 	if (unlikely(___r != ___o)) \
- 		*___op = ___r; \
- 	likely(___r == ___o); \
- })
-#endif /* arch_try_cmpxchg_release */
-+#endif
- 
-#ifndef arch_try_cmpxchg_relaxed
-#define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
-+#if defined(arch_try_cmpxchg64_acquire)
-+#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire
-+#elif defined(arch_try_cmpxchg64_relaxed)
-+#define raw_try_cmpxchg64_acquire(...) \
-+	__atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
-+#elif defined(arch_try_cmpxchg64)
-+#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64
-+#else
-+#define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
- ({ \
- 	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-	___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \
-+	___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \
- 	if (unlikely(___r != ___o)) \
- 		*___op = ___r; \
- 	likely(___r == ___o); \
- })
-#endif /* arch_try_cmpxchg_relaxed */
-
-#else /* arch_try_cmpxchg_relaxed */
-
-#ifndef arch_try_cmpxchg_acquire
-#define arch_try_cmpxchg_acquire(...) \
-	__atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
- #endif
- 
-#ifndef arch_try_cmpxchg_release
-#define arch_try_cmpxchg_release(...) \
-	__atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
-+#if defined(arch_try_cmpxchg64_release)
-+#define raw_try_cmpxchg64_release arch_try_cmpxchg64_release
-+#elif defined(arch_try_cmpxchg64_relaxed)
-+#define raw_try_cmpxchg64_release(...) \
-+	__atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
-+#elif defined(arch_try_cmpxchg64)
-+#define raw_try_cmpxchg64_release arch_try_cmpxchg64
-+#else
-+#define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \
-+({ \
-+	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-+	___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \
-+	if (unlikely(___r != ___o)) \
-+		*___op = ___r; \
-+	likely(___r == ___o); \
-+})
- #endif
- 
-#ifndef arch_try_cmpxchg
-#define arch_try_cmpxchg(...) \
-	__atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
-+#if defined(arch_try_cmpxchg64_relaxed)
-+#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed
-+#elif defined(arch_try_cmpxchg64)
-+#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64
-+#else
-+#define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
-+({ \
-+	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-+	___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
-+	if (unlikely(___r != ___o)) \
-+		*___op = ___r; \
-+	likely(___r == ___o); \
-+})
- #endif
- 
-#endif /* arch_try_cmpxchg_relaxed */
-
-#ifndef arch_try_cmpxchg64_relaxed
-#ifdef arch_try_cmpxchg64
-#define arch_try_cmpxchg64_acquire arch_try_cmpxchg64
-#define arch_try_cmpxchg64_release arch_try_cmpxchg64
-#define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64
-#endif /* arch_try_cmpxchg64 */
-+#if defined(arch_try_cmpxchg128)
-+#define raw_try_cmpxchg128 arch_try_cmpxchg128
-+#elif defined(arch_try_cmpxchg128_relaxed)
-+#define raw_try_cmpxchg128(...) \
-+	__atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
-+#else
-+#define raw_try_cmpxchg128(_ptr, _oldp, _new) \
-+({ \
-+	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-+	___r = raw_cmpxchg128((_ptr), ___o, (_new)); \
-+	if (unlikely(___r != ___o)) \
-+		*___op = ___r; \
-+	likely(___r == ___o); \
-+})
-+#endif
- 
-#ifndef arch_try_cmpxchg64
-#define arch_try_cmpxchg64(_ptr, _oldp, _new) \
-+#if defined(arch_try_cmpxchg128_acquire)
-+#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire
-+#elif defined(arch_try_cmpxchg128_relaxed)
-+#define raw_try_cmpxchg128_acquire(...) \
-+	__atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
-+#elif defined(arch_try_cmpxchg128)
-+#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128
-+#else
-+#define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
- ({ \
- 	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-	___r = arch_cmpxchg64((_ptr), ___o, (_new)); \
-+	___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \
- 	if (unlikely(___r != ___o)) \
- 		*___op = ___r; \
- 	likely(___r == ___o); \
- })
-#endif /* arch_try_cmpxchg64 */
-+#endif
- 
-#ifndef arch_try_cmpxchg64_acquire
-#define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
-+#if defined(arch_try_cmpxchg128_release)
-+#define raw_try_cmpxchg128_release arch_try_cmpxchg128_release
-+#elif defined(arch_try_cmpxchg128_relaxed)
-+#define raw_try_cmpxchg128_release(...) \
-+	__atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
-+#elif defined(arch_try_cmpxchg128)
-+#define raw_try_cmpxchg128_release arch_try_cmpxchg128
-+#else
-+#define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \
- ({ \
- 	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-	___r = arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \
-+	___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \
- 	if (unlikely(___r != ___o)) \
- 		*___op = ___r; \
- 	likely(___r == ___o); \
- })
-#endif /* arch_try_cmpxchg64_acquire */
-+#endif
- 
-#ifndef arch_try_cmpxchg64_release
-#define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \
-+#if defined(arch_try_cmpxchg128_relaxed)
-+#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed
-+#elif defined(arch_try_cmpxchg128)
-+#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128
-+#else
-+#define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
- ({ \
- 	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-	___r = arch_cmpxchg64_release((_ptr), ___o, (_new)); \
-+	___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
- 	if (unlikely(___r != ___o)) \
- 		*___op = ___r; \
- 	likely(___r == ___o); \
- })
-#endif /* arch_try_cmpxchg64_release */
-+#endif
- 
-#ifndef arch_try_cmpxchg64_relaxed
-#define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
-+#define raw_cmpxchg_local arch_cmpxchg_local
-+
-+#ifdef arch_try_cmpxchg_local
-+#define raw_try_cmpxchg_local arch_try_cmpxchg_local
-+#else
-+#define raw_try_cmpxchg_local(_ptr, _oldp, _new) \
- ({ \
- 	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-	___r = arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
-+	___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \
- 	if (unlikely(___r != ___o)) \
- 		*___op = ___r; \
- 	likely(___r == ___o); \
- })
-#endif /* arch_try_cmpxchg64_relaxed */
-+#endif
- 
-#else /* arch_try_cmpxchg64_relaxed */
-+#define raw_cmpxchg64_local arch_cmpxchg64_local
- 
-#ifndef arch_try_cmpxchg64_acquire
-#define arch_try_cmpxchg64_acquire(...) \
-	__atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
-+#ifdef arch_try_cmpxchg64_local
-+#define raw_try_cmpxchg64_local arch_try_cmpxchg64_local
-+#else
-+#define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \
-+({ \
-+	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-+	___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \
-+	if (unlikely(___r != ___o)) \
-+		*___op = ___r; \
-+	likely(___r == ___o); \
-+})
- #endif
- 
-#ifndef arch_try_cmpxchg64_release
-#define arch_try_cmpxchg64_release(...) \
-	__atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
-+#define raw_cmpxchg128_local arch_cmpxchg128_local
-+
-+#ifdef arch_try_cmpxchg128_local
-+#define raw_try_cmpxchg128_local arch_try_cmpxchg128_local
-+#else
-+#define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \
-+({ \
-+	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-+	___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \
-+	if (unlikely(___r != ___o)) \
-+		*___op = ___r; \
-+	likely(___r == ___o); \
-+})
- #endif
- 
-#ifndef arch_try_cmpxchg64
-#define arch_try_cmpxchg64(...) \
-	__atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
-+#define raw_sync_cmpxchg arch_sync_cmpxchg
-+
-+#ifdef arch_sync_try_cmpxchg
-+#define raw_sync_try_cmpxchg arch_sync_try_cmpxchg
-+#else
-+#define raw_sync_try_cmpxchg(_ptr, _oldp, _new) \
-+({ \
-+	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
-+	___r = raw_sync_cmpxchg((_ptr), ___o, (_new)); \
-+	if (unlikely(___r != ___o)) \
-+		*___op = ___r; \
-+	likely(___r == ___o); \
-+})
- #endif
- 
-#endif /* arch_try_cmpxchg64_relaxed */
-+/**
-+ * raw_atomic_read() - atomic load with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically loads the value of @v with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_read() elsewhere.
-+ *
-+ * Return: The value loaded from @v.
-+ */
-+static __always_inline int
-+raw_atomic_read(const atomic_t *v)
-+{
-+	return arch_atomic_read(v);
-+}
- 
-#ifndef arch_atomic_read_acquire
-+/**
-+ * raw_atomic_read_acquire() - atomic load with acquire ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically loads the value of @v with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere.
-+ *
-+ * Return: The value loaded from @v.
-+ */
- static __always_inline int
-arch_atomic_read_acquire(const atomic_t *v)
-+raw_atomic_read_acquire(const atomic_t *v)
- {
-+#if defined(arch_atomic_read_acquire)
-+	return arch_atomic_read_acquire(v);
-+#else
- 	int ret;
- 
- 	if (__native_word(atomic_t)) {
- 		ret = smp_load_acquire(&(v)->counter);
- 	} else {
-		ret = arch_atomic_read(v);
-+		ret = raw_atomic_read(v);
- 		__atomic_acquire_fence();
- 	}
- 
- 	return ret;
-}
-#define arch_atomic_read_acquire arch_atomic_read_acquire
- #endif
-+}
- 
-#ifndef arch_atomic_set_release
-+/**
-+ * raw_atomic_set() - atomic set with relaxed ordering
-+ * @v: pointer to atomic_t
-+ * @i: int value to assign
-+ *
-+ * Atomically sets @v to @i with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_set() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic_set_release(atomic_t *v, int i)
-+raw_atomic_set(atomic_t *v, int i)
- {
-+	arch_atomic_set(v, i);
-+}
-+
-+/**
-+ * raw_atomic_set_release() - atomic set with release ordering
-+ * @v: pointer to atomic_t
-+ * @i: int value to assign
-+ *
-+ * Atomically sets @v to @i with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_set_release() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic_set_release(atomic_t *v, int i)
-+{
-+#if defined(arch_atomic_set_release)
-+	arch_atomic_set_release(v, i);
-+#else
- 	if (__native_word(atomic_t)) {
- 		smp_store_release(&(v)->counter, i);
- 	} else {
- 		__atomic_release_fence();
-		arch_atomic_set(v, i);
-+		raw_atomic_set(v, i);
- 	}
-}
-#define arch_atomic_set_release arch_atomic_set_release
- #endif
-
-#ifndef arch_atomic_add_return_relaxed
-#define arch_atomic_add_return_acquire arch_atomic_add_return
-#define arch_atomic_add_return_release arch_atomic_add_return
-#define arch_atomic_add_return_relaxed arch_atomic_add_return
-#else /* arch_atomic_add_return_relaxed */
-
-#ifndef arch_atomic_add_return_acquire
-static __always_inline int
-arch_atomic_add_return_acquire(int i, atomic_t *v)
-{
-	int ret = arch_atomic_add_return_relaxed(i, v);
-	__atomic_acquire_fence();
-	return ret;
- }
-#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
-#endif
- 
-#ifndef arch_atomic_add_return_release
-static __always_inline int
-arch_atomic_add_return_release(int i, atomic_t *v)
-+/**
-+ * raw_atomic_add() - atomic add with relaxed ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_add() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic_add(int i, atomic_t *v)
- {
-	__atomic_release_fence();
-	return arch_atomic_add_return_relaxed(i, v);
-+	arch_atomic_add(i, v);
- }
-#define arch_atomic_add_return_release arch_atomic_add_return_release
-#endif
- 
-#ifndef arch_atomic_add_return
-+/**
-+ * raw_atomic_add_return() - atomic add with full ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_add_return() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_add_return(int i, atomic_t *v)
-+raw_atomic_add_return(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_add_return)
-+	return arch_atomic_add_return(i, v);
-+#elif defined(arch_atomic_add_return_relaxed)
- 	int ret;
- 	__atomic_pre_full_fence();
- 	ret = arch_atomic_add_return_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic_add_return arch_atomic_add_return
-+#else
-+#error "Unable to define raw_atomic_add_return"
- #endif
-+}
- 
-#endif /* arch_atomic_add_return_relaxed */
-
-#ifndef arch_atomic_fetch_add_relaxed
-#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
-#define arch_atomic_fetch_add_release arch_atomic_fetch_add
-#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
-#else /* arch_atomic_fetch_add_relaxed */
-
-#ifndef arch_atomic_fetch_add_acquire
-+/**
-+ * raw_atomic_add_return_acquire() - atomic add with acquire ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_add_acquire(int i, atomic_t *v)
-+raw_atomic_add_return_acquire(int i, atomic_t *v)
- {
-	int ret = arch_atomic_fetch_add_relaxed(i, v);
-+#if defined(arch_atomic_add_return_acquire)
-+	return arch_atomic_add_return_acquire(i, v);
-+#elif defined(arch_atomic_add_return_relaxed)
-+	int ret = arch_atomic_add_return_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
-+#elif defined(arch_atomic_add_return)
-+	return arch_atomic_add_return(i, v);
-+#else
-+#error "Unable to define raw_atomic_add_return_acquire"
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_add_release
-+/**
-+ * raw_atomic_add_return_release() - atomic add with release ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_add_release(int i, atomic_t *v)
-+raw_atomic_add_return_release(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_add_return_release)
-+	return arch_atomic_add_return_release(i, v);
-+#elif defined(arch_atomic_add_return_relaxed)
- 	__atomic_release_fence();
-	return arch_atomic_fetch_add_relaxed(i, v);
-+	return arch_atomic_add_return_relaxed(i, v);
-+#elif defined(arch_atomic_add_return)
-+	return arch_atomic_add_return(i, v);
-+#else
-+#error "Unable to define raw_atomic_add_return_release"
-+#endif
- }
-#define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
-+
-+/**
-+ * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
-+static __always_inline int
-+raw_atomic_add_return_relaxed(int i, atomic_t *v)
-+{
-+#if defined(arch_atomic_add_return_relaxed)
-+	return arch_atomic_add_return_relaxed(i, v);
-+#elif defined(arch_atomic_add_return)
-+	return arch_atomic_add_return(i, v);
-+#else
-+#error "Unable to define raw_atomic_add_return_relaxed"
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_add
-+/**
-+ * raw_atomic_fetch_add() - atomic add with full ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_add(int i, atomic_t *v)
-+raw_atomic_fetch_add(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_add)
-+	return arch_atomic_fetch_add(i, v);
-+#elif defined(arch_atomic_fetch_add_relaxed)
- 	int ret;
- 	__atomic_pre_full_fence();
- 	ret = arch_atomic_fetch_add_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_add arch_atomic_fetch_add
-+#else
-+#error "Unable to define raw_atomic_fetch_add"
- #endif
-+}
- 
-#endif /* arch_atomic_fetch_add_relaxed */
-
-#ifndef arch_atomic_sub_return_relaxed
-#define arch_atomic_sub_return_acquire arch_atomic_sub_return
-#define arch_atomic_sub_return_release arch_atomic_sub_return
-#define arch_atomic_sub_return_relaxed arch_atomic_sub_return
-#else /* arch_atomic_sub_return_relaxed */
-
-#ifndef arch_atomic_sub_return_acquire
-+/**
-+ * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_sub_return_acquire(int i, atomic_t *v)
-+raw_atomic_fetch_add_acquire(int i, atomic_t *v)
- {
-	int ret = arch_atomic_sub_return_relaxed(i, v);
-+#if defined(arch_atomic_fetch_add_acquire)
-+	return arch_atomic_fetch_add_acquire(i, v);
-+#elif defined(arch_atomic_fetch_add_relaxed)
-+	int ret = arch_atomic_fetch_add_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
-+#elif defined(arch_atomic_fetch_add)
-+	return arch_atomic_fetch_add(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_add_acquire"
- #endif
-+}
- 
-#ifndef arch_atomic_sub_return_release
-+/**
-+ * raw_atomic_fetch_add_release() - atomic add with release ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_sub_return_release(int i, atomic_t *v)
-+raw_atomic_fetch_add_release(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_add_release)
-+	return arch_atomic_fetch_add_release(i, v);
-+#elif defined(arch_atomic_fetch_add_relaxed)
- 	__atomic_release_fence();
-	return arch_atomic_sub_return_relaxed(i, v);
-+	return arch_atomic_fetch_add_relaxed(i, v);
-+#elif defined(arch_atomic_fetch_add)
-+	return arch_atomic_fetch_add(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_add_release"
-+#endif
- }
-#define arch_atomic_sub_return_release arch_atomic_sub_return_release
-+
-+/**
-+ * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
-+static __always_inline int
-+raw_atomic_fetch_add_relaxed(int i, atomic_t *v)
-+{
-+#if defined(arch_atomic_fetch_add_relaxed)
-+	return arch_atomic_fetch_add_relaxed(i, v);
-+#elif defined(arch_atomic_fetch_add)
-+	return arch_atomic_fetch_add(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_add_relaxed"
- #endif
-+}
-+
-+/**
-+ * raw_atomic_sub() - atomic subtract with relaxed ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_sub() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic_sub(int i, atomic_t *v)
-+{
-+	arch_atomic_sub(i, v);
-+}
- 
-#ifndef arch_atomic_sub_return
-+/**
-+ * raw_atomic_sub_return() - atomic subtract with full ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_sub_return(int i, atomic_t *v)
-+raw_atomic_sub_return(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_sub_return)
-+	return arch_atomic_sub_return(i, v);
-+#elif defined(arch_atomic_sub_return_relaxed)
- 	int ret;
- 	__atomic_pre_full_fence();
- 	ret = arch_atomic_sub_return_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic_sub_return arch_atomic_sub_return
-+#else
-+#error "Unable to define raw_atomic_sub_return"
- #endif
-+}
- 
-#endif /* arch_atomic_sub_return_relaxed */
-
-#ifndef arch_atomic_fetch_sub_relaxed
-#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
-#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
-#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
-#else /* arch_atomic_fetch_sub_relaxed */
-
-#ifndef arch_atomic_fetch_sub_acquire
-+/**
-+ * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
-+raw_atomic_sub_return_acquire(int i, atomic_t *v)
- {
-	int ret = arch_atomic_fetch_sub_relaxed(i, v);
-+#if defined(arch_atomic_sub_return_acquire)
-+	return arch_atomic_sub_return_acquire(i, v);
-+#elif defined(arch_atomic_sub_return_relaxed)
-+	int ret = arch_atomic_sub_return_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
-+#elif defined(arch_atomic_sub_return)
-+	return arch_atomic_sub_return(i, v);
-+#else
-+#error "Unable to define raw_atomic_sub_return_acquire"
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_sub_release
-+/**
-+ * raw_atomic_sub_return_release() - atomic subtract with release ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_sub_release(int i, atomic_t *v)
-+raw_atomic_sub_return_release(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_sub_return_release)
-+	return arch_atomic_sub_return_release(i, v);
-+#elif defined(arch_atomic_sub_return_relaxed)
- 	__atomic_release_fence();
-	return arch_atomic_fetch_sub_relaxed(i, v);
-+	return arch_atomic_sub_return_relaxed(i, v);
-+#elif defined(arch_atomic_sub_return)
-+	return arch_atomic_sub_return(i, v);
-+#else
-+#error "Unable to define raw_atomic_sub_return_release"
-+#endif
- }
-#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
-+
-+/**
-+ * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
-+static __always_inline int
-+raw_atomic_sub_return_relaxed(int i, atomic_t *v)
-+{
-+#if defined(arch_atomic_sub_return_relaxed)
-+	return arch_atomic_sub_return_relaxed(i, v);
-+#elif defined(arch_atomic_sub_return)
-+	return arch_atomic_sub_return(i, v);
-+#else
-+#error "Unable to define raw_atomic_sub_return_relaxed"
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_sub
-+/**
-+ * raw_atomic_fetch_sub() - atomic subtract with full ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_sub(int i, atomic_t *v)
-+raw_atomic_fetch_sub(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_sub)
-+	return arch_atomic_fetch_sub(i, v);
-+#elif defined(arch_atomic_fetch_sub_relaxed)
- 	int ret;
- 	__atomic_pre_full_fence();
- 	ret = arch_atomic_fetch_sub_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_sub arch_atomic_fetch_sub
-+#else
-+#error "Unable to define raw_atomic_fetch_sub"
- #endif
-
-#endif /* arch_atomic_fetch_sub_relaxed */
-
-#ifndef arch_atomic_inc
-static __always_inline void
-arch_atomic_inc(atomic_t *v)
-{
-	arch_atomic_add(1, v);
- }
-#define arch_atomic_inc arch_atomic_inc
-#endif
- 
-#ifndef arch_atomic_inc_return_relaxed
-#ifdef arch_atomic_inc_return
-#define arch_atomic_inc_return_acquire arch_atomic_inc_return
-#define arch_atomic_inc_return_release arch_atomic_inc_return
-#define arch_atomic_inc_return_relaxed arch_atomic_inc_return
-#endif /* arch_atomic_inc_return */
-
-#ifndef arch_atomic_inc_return
-+/**
-+ * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_inc_return(atomic_t *v)
-+raw_atomic_fetch_sub_acquire(int i, atomic_t *v)
- {
-	return arch_atomic_add_return(1, v);
-}
-#define arch_atomic_inc_return arch_atomic_inc_return
-+#if defined(arch_atomic_fetch_sub_acquire)
-+	return arch_atomic_fetch_sub_acquire(i, v);
-+#elif defined(arch_atomic_fetch_sub_relaxed)
-+	int ret = arch_atomic_fetch_sub_relaxed(i, v);
-+	__atomic_acquire_fence();
-+	return ret;
-+#elif defined(arch_atomic_fetch_sub)
-+	return arch_atomic_fetch_sub(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_sub_acquire"
- #endif
-
-#ifndef arch_atomic_inc_return_acquire
-static __always_inline int
-arch_atomic_inc_return_acquire(atomic_t *v)
-{
-	return arch_atomic_add_return_acquire(1, v);
- }
-#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
-#endif
- 
-#ifndef arch_atomic_inc_return_release
-+/**
-+ * raw_atomic_fetch_sub_release() - atomic subtract with release ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_inc_return_release(atomic_t *v)
-+raw_atomic_fetch_sub_release(int i, atomic_t *v)
- {
-	return arch_atomic_add_return_release(1, v);
-}
-#define arch_atomic_inc_return_release arch_atomic_inc_return_release
-+#if defined(arch_atomic_fetch_sub_release)
-+	return arch_atomic_fetch_sub_release(i, v);
-+#elif defined(arch_atomic_fetch_sub_relaxed)
-+	__atomic_release_fence();
-+	return arch_atomic_fetch_sub_relaxed(i, v);
-+#elif defined(arch_atomic_fetch_sub)
-+	return arch_atomic_fetch_sub(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_sub_release"
- #endif
-
-#ifndef arch_atomic_inc_return_relaxed
-static __always_inline int
-arch_atomic_inc_return_relaxed(atomic_t *v)
-{
-	return arch_atomic_add_return_relaxed(1, v);
- }
-#define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
-#endif
- 
-#else /* arch_atomic_inc_return_relaxed */
-
-#ifndef arch_atomic_inc_return_acquire
-+/**
-+ * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_inc_return_acquire(atomic_t *v)
-+raw_atomic_fetch_sub_relaxed(int i, atomic_t *v)
- {
-	int ret = arch_atomic_inc_return_relaxed(v);
-	__atomic_acquire_fence();
-	return ret;
-}
-#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
-+#if defined(arch_atomic_fetch_sub_relaxed)
-+	return arch_atomic_fetch_sub_relaxed(i, v);
-+#elif defined(arch_atomic_fetch_sub)
-+	return arch_atomic_fetch_sub(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_sub_relaxed"
- #endif
-+}
- 
-#ifndef arch_atomic_inc_return_release
-static __always_inline int
-arch_atomic_inc_return_release(atomic_t *v)
-+/**
-+ * raw_atomic_inc() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_inc() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic_inc(atomic_t *v)
- {
-	__atomic_release_fence();
-	return arch_atomic_inc_return_relaxed(v);
-}
-#define arch_atomic_inc_return_release arch_atomic_inc_return_release
-+#if defined(arch_atomic_inc)
-+	arch_atomic_inc(v);
-+#else
-+	raw_atomic_add(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic_inc_return
-+/**
-+ * raw_atomic_inc_return() - atomic increment with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_inc_return(atomic_t *v)
-+raw_atomic_inc_return(atomic_t *v)
- {
-+#if defined(arch_atomic_inc_return)
-+	return arch_atomic_inc_return(v);
-+#elif defined(arch_atomic_inc_return_relaxed)
- 	int ret;
- 	__atomic_pre_full_fence();
- 	ret = arch_atomic_inc_return_relaxed(v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic_inc_return arch_atomic_inc_return
-+#else
-+	return raw_atomic_add_return(1, v);
- #endif
-+}
- 
-#endif /* arch_atomic_inc_return_relaxed */
-
-#ifndef arch_atomic_fetch_inc_relaxed
-#ifdef arch_atomic_fetch_inc
-#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
-#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
-#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
-#endif /* arch_atomic_fetch_inc */
-
-#ifndef arch_atomic_fetch_inc
-+/**
-+ * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_inc(atomic_t *v)
-+raw_atomic_inc_return_acquire(atomic_t *v)
- {
-	return arch_atomic_fetch_add(1, v);
-}
-#define arch_atomic_fetch_inc arch_atomic_fetch_inc
-+#if defined(arch_atomic_inc_return_acquire)
-+	return arch_atomic_inc_return_acquire(v);
-+#elif defined(arch_atomic_inc_return_relaxed)
-+	int ret = arch_atomic_inc_return_relaxed(v);
-+	__atomic_acquire_fence();
-+	return ret;
-+#elif defined(arch_atomic_inc_return)
-+	return arch_atomic_inc_return(v);
-+#else
-+	return raw_atomic_add_return_acquire(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_inc_acquire
-+/**
-+ * raw_atomic_inc_return_release() - atomic increment with release ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_inc_acquire(atomic_t *v)
-+raw_atomic_inc_return_release(atomic_t *v)
- {
-	return arch_atomic_fetch_add_acquire(1, v);
-}
-#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
-+#if defined(arch_atomic_inc_return_release)
-+	return arch_atomic_inc_return_release(v);
-+#elif defined(arch_atomic_inc_return_relaxed)
-+	__atomic_release_fence();
-+	return arch_atomic_inc_return_relaxed(v);
-+#elif defined(arch_atomic_inc_return)
-+	return arch_atomic_inc_return(v);
-+#else
-+	return raw_atomic_add_return_release(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_inc_release
-+/**
-+ * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_inc_release(atomic_t *v)
-+raw_atomic_inc_return_relaxed(atomic_t *v)
- {
-	return arch_atomic_fetch_add_release(1, v);
-}
-#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
-+#if defined(arch_atomic_inc_return_relaxed)
-+	return arch_atomic_inc_return_relaxed(v);
-+#elif defined(arch_atomic_inc_return)
-+	return arch_atomic_inc_return(v);
-+#else
-+	return raw_atomic_add_return_relaxed(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_inc_relaxed
-+/**
-+ * raw_atomic_fetch_inc() - atomic increment with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_inc_relaxed(atomic_t *v)
-+raw_atomic_fetch_inc(atomic_t *v)
- {
-	return arch_atomic_fetch_add_relaxed(1, v);
-}
-#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
-+#if defined(arch_atomic_fetch_inc)
-+	return arch_atomic_fetch_inc(v);
-+#elif defined(arch_atomic_fetch_inc_relaxed)
-+	int ret;
-+	__atomic_pre_full_fence();
-+	ret = arch_atomic_fetch_inc_relaxed(v);
-+	__atomic_post_full_fence();
-+	return ret;
-+#else
-+	return raw_atomic_fetch_add(1, v);
- #endif
-+}
- 
-#else /* arch_atomic_fetch_inc_relaxed */
-
-#ifndef arch_atomic_fetch_inc_acquire
-+/**
-+ * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_inc_acquire(atomic_t *v)
-+raw_atomic_fetch_inc_acquire(atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_inc_acquire)
-+	return arch_atomic_fetch_inc_acquire(v);
-+#elif defined(arch_atomic_fetch_inc_relaxed)
- 	int ret = arch_atomic_fetch_inc_relaxed(v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
-+#elif defined(arch_atomic_fetch_inc)
-+	return arch_atomic_fetch_inc(v);
-+#else
-+	return raw_atomic_fetch_add_acquire(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_inc_release
-+/**
-+ * raw_atomic_fetch_inc_release() - atomic increment with release ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_inc_release(atomic_t *v)
-+raw_atomic_fetch_inc_release(atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_inc_release)
-+	return arch_atomic_fetch_inc_release(v);
-+#elif defined(arch_atomic_fetch_inc_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic_fetch_inc_relaxed(v);
-}
-#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
-+#elif defined(arch_atomic_fetch_inc)
-+	return arch_atomic_fetch_inc(v);
-+#else
-+	return raw_atomic_fetch_add_release(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_inc
-+/**
-+ * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_inc(atomic_t *v)
-+raw_atomic_fetch_inc_relaxed(atomic_t *v)
- {
-	int ret;
-	__atomic_pre_full_fence();
-	ret = arch_atomic_fetch_inc_relaxed(v);
-	__atomic_post_full_fence();
-	return ret;
-}
-#define arch_atomic_fetch_inc arch_atomic_fetch_inc
-+#if defined(arch_atomic_fetch_inc_relaxed)
-+	return arch_atomic_fetch_inc_relaxed(v);
-+#elif defined(arch_atomic_fetch_inc)
-+	return arch_atomic_fetch_inc(v);
-+#else
-+	return raw_atomic_fetch_add_relaxed(1, v);
- #endif
-+}
- 
-#endif /* arch_atomic_fetch_inc_relaxed */
-
-#ifndef arch_atomic_dec
-+/**
-+ * raw_atomic_dec() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_dec() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic_dec(atomic_t *v)
-+raw_atomic_dec(atomic_t *v)
- {
-	arch_atomic_sub(1, v);
-}
-#define arch_atomic_dec arch_atomic_dec
-+#if defined(arch_atomic_dec)
-+	arch_atomic_dec(v);
-+#else
-+	raw_atomic_sub(1, v);
- #endif
-
-#ifndef arch_atomic_dec_return_relaxed
-#ifdef arch_atomic_dec_return
-#define arch_atomic_dec_return_acquire arch_atomic_dec_return
-#define arch_atomic_dec_return_release arch_atomic_dec_return
-#define arch_atomic_dec_return_relaxed arch_atomic_dec_return
-#endif /* arch_atomic_dec_return */
-
-#ifndef arch_atomic_dec_return
-static __always_inline int
-arch_atomic_dec_return(atomic_t *v)
-{
-	return arch_atomic_sub_return(1, v);
- }
-#define arch_atomic_dec_return arch_atomic_dec_return
-#endif
- 
-#ifndef arch_atomic_dec_return_acquire
-+/**
-+ * raw_atomic_dec_return() - atomic decrement with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_dec_return_acquire(atomic_t *v)
-+raw_atomic_dec_return(atomic_t *v)
- {
-	return arch_atomic_sub_return_acquire(1, v);
-}
-#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
-+#if defined(arch_atomic_dec_return)
-+	return arch_atomic_dec_return(v);
-+#elif defined(arch_atomic_dec_return_relaxed)
-+	int ret;
-+	__atomic_pre_full_fence();
-+	ret = arch_atomic_dec_return_relaxed(v);
-+	__atomic_post_full_fence();
-+	return ret;
-+#else
-+	return raw_atomic_sub_return(1, v);
- #endif
-
-#ifndef arch_atomic_dec_return_release
-static __always_inline int
-arch_atomic_dec_return_release(atomic_t *v)
-{
-	return arch_atomic_sub_return_release(1, v);
- }
-#define arch_atomic_dec_return_release arch_atomic_dec_return_release
-#endif
- 
-#ifndef arch_atomic_dec_return_relaxed
-static __always_inline int
-arch_atomic_dec_return_relaxed(atomic_t *v)
-{
-	return arch_atomic_sub_return_relaxed(1, v);
-}
-#define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
-#endif
-
-#else /* arch_atomic_dec_return_relaxed */
-
-#ifndef arch_atomic_dec_return_acquire
-+/**
-+ * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_dec_return_acquire(atomic_t *v)
-+raw_atomic_dec_return_acquire(atomic_t *v)
- {
-+#if defined(arch_atomic_dec_return_acquire)
-+	return arch_atomic_dec_return_acquire(v);
-+#elif defined(arch_atomic_dec_return_relaxed)
- 	int ret = arch_atomic_dec_return_relaxed(v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
-+#elif defined(arch_atomic_dec_return)
-+	return arch_atomic_dec_return(v);
-+#else
-+	return raw_atomic_sub_return_acquire(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic_dec_return_release
-+/**
-+ * raw_atomic_dec_return_release() - atomic decrement with release ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
-arch_atomic_dec_return_release(atomic_t *v)
-+raw_atomic_dec_return_release(atomic_t *v)
- {
-+#if defined(arch_atomic_dec_return_release)
-+	return arch_atomic_dec_return_release(v);
-+#elif defined(arch_atomic_dec_return_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic_dec_return_relaxed(v);
-+#elif defined(arch_atomic_dec_return)
-+	return arch_atomic_dec_return(v);
-+#else
-+	return raw_atomic_sub_return_release(1, v);
-+#endif
- }
-#define arch_atomic_dec_return_release arch_atomic_dec_return_release
-+
-+/**
-+ * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
-+static __always_inline int
-+raw_atomic_dec_return_relaxed(atomic_t *v)
-+{
-+#if defined(arch_atomic_dec_return_relaxed)
-+	return arch_atomic_dec_return_relaxed(v);
-+#elif defined(arch_atomic_dec_return)
-+	return arch_atomic_dec_return(v);
-+#else
-+	return raw_atomic_sub_return_relaxed(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic_dec_return
-+/**
-+ * raw_atomic_fetch_dec() - atomic decrement with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_dec_return(atomic_t *v)
-+raw_atomic_fetch_dec(atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_dec)
-+	return arch_atomic_fetch_dec(v);
-+#elif defined(arch_atomic_fetch_dec_relaxed)
- 	int ret;
- 	__atomic_pre_full_fence();
-	ret = arch_atomic_dec_return_relaxed(v);
-+	ret = arch_atomic_fetch_dec_relaxed(v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic_dec_return arch_atomic_dec_return
-+#else
-+	return raw_atomic_fetch_sub(1, v);
- #endif
-
-#endif /* arch_atomic_dec_return_relaxed */
-
-#ifndef arch_atomic_fetch_dec_relaxed
-#ifdef arch_atomic_fetch_dec
-#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
-#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
-#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
-#endif /* arch_atomic_fetch_dec */
-
-#ifndef arch_atomic_fetch_dec
-static __always_inline int
-arch_atomic_fetch_dec(atomic_t *v)
-{
-	return arch_atomic_fetch_sub(1, v);
- }
-#define arch_atomic_fetch_dec arch_atomic_fetch_dec
-#endif
- 
-#ifndef arch_atomic_fetch_dec_acquire
-+/**
-+ * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_dec_acquire(atomic_t *v)
-+raw_atomic_fetch_dec_acquire(atomic_t *v)
- {
-	return arch_atomic_fetch_sub_acquire(1, v);
-}
-#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
-+#if defined(arch_atomic_fetch_dec_acquire)
-+	return arch_atomic_fetch_dec_acquire(v);
-+#elif defined(arch_atomic_fetch_dec_relaxed)
-+	int ret = arch_atomic_fetch_dec_relaxed(v);
-+	__atomic_acquire_fence();
-+	return ret;
-+#elif defined(arch_atomic_fetch_dec)
-+	return arch_atomic_fetch_dec(v);
-+#else
-+	return raw_atomic_fetch_sub_acquire(1, v);
- #endif
-
-#ifndef arch_atomic_fetch_dec_release
-static __always_inline int
-arch_atomic_fetch_dec_release(atomic_t *v)
-{
-	return arch_atomic_fetch_sub_release(1, v);
- }
-#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
-#endif
- 
-#ifndef arch_atomic_fetch_dec_relaxed
-+/**
-+ * raw_atomic_fetch_dec_release() - atomic decrement with release ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_dec_relaxed(atomic_t *v)
-+raw_atomic_fetch_dec_release(atomic_t *v)
- {
-	return arch_atomic_fetch_sub_relaxed(1, v);
-}
-#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
-+#if defined(arch_atomic_fetch_dec_release)
-+	return arch_atomic_fetch_dec_release(v);
-+#elif defined(arch_atomic_fetch_dec_relaxed)
-+	__atomic_release_fence();
-+	return arch_atomic_fetch_dec_relaxed(v);
-+#elif defined(arch_atomic_fetch_dec)
-+	return arch_atomic_fetch_dec(v);
-+#else
-+	return raw_atomic_fetch_sub_release(1, v);
- #endif
-+}
- 
-#else /* arch_atomic_fetch_dec_relaxed */
-
-#ifndef arch_atomic_fetch_dec_acquire
-+/**
-+ * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_dec_acquire(atomic_t *v)
-+raw_atomic_fetch_dec_relaxed(atomic_t *v)
- {
-	int ret = arch_atomic_fetch_dec_relaxed(v);
-	__atomic_acquire_fence();
-	return ret;
-}
-#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
-+#if defined(arch_atomic_fetch_dec_relaxed)
-+	return arch_atomic_fetch_dec_relaxed(v);
-+#elif defined(arch_atomic_fetch_dec)
-+	return arch_atomic_fetch_dec(v);
-+#else
-+	return raw_atomic_fetch_sub_relaxed(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_dec_release
-static __always_inline int
-arch_atomic_fetch_dec_release(atomic_t *v)
-+/**
-+ * raw_atomic_and() - atomic bitwise AND with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_and() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic_and(int i, atomic_t *v)
- {
-	__atomic_release_fence();
-	return arch_atomic_fetch_dec_relaxed(v);
-+	arch_atomic_and(i, v);
- }
-#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
-#endif
- 
-#ifndef arch_atomic_fetch_dec
-+/**
-+ * raw_atomic_fetch_and() - atomic bitwise AND with full ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_dec(atomic_t *v)
-+raw_atomic_fetch_and(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_and)
-+	return arch_atomic_fetch_and(i, v);
-+#elif defined(arch_atomic_fetch_and_relaxed)
- 	int ret;
- 	__atomic_pre_full_fence();
-	ret = arch_atomic_fetch_dec_relaxed(v);
-+	ret = arch_atomic_fetch_and_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_dec arch_atomic_fetch_dec
-+#else
-+#error "Unable to define raw_atomic_fetch_and"
- #endif
-+}
- 
-#endif /* arch_atomic_fetch_dec_relaxed */
-
-#ifndef arch_atomic_fetch_and_relaxed
-#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
-#define arch_atomic_fetch_and_release arch_atomic_fetch_and
-#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
-#else /* arch_atomic_fetch_and_relaxed */
-
-#ifndef arch_atomic_fetch_and_acquire
-+/**
-+ * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_and_acquire(int i, atomic_t *v)
-+raw_atomic_fetch_and_acquire(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_and_acquire)
-+	return arch_atomic_fetch_and_acquire(i, v);
-+#elif defined(arch_atomic_fetch_and_relaxed)
- 	int ret = arch_atomic_fetch_and_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
-+#elif defined(arch_atomic_fetch_and)
-+	return arch_atomic_fetch_and(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_and_acquire"
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_and_release
-+/**
-+ * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_and_release(int i, atomic_t *v)
-+raw_atomic_fetch_and_release(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_and_release)
-+	return arch_atomic_fetch_and_release(i, v);
-+#elif defined(arch_atomic_fetch_and_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic_fetch_and_relaxed(i, v);
-}
-#define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
-+#elif defined(arch_atomic_fetch_and)
-+	return arch_atomic_fetch_and(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_and_release"
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_and
-+/**
-+ * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_and(int i, atomic_t *v)
-+raw_atomic_fetch_and_relaxed(int i, atomic_t *v)
- {
-	int ret;
-	__atomic_pre_full_fence();
-	ret = arch_atomic_fetch_and_relaxed(i, v);
-	__atomic_post_full_fence();
-	return ret;
-}
-#define arch_atomic_fetch_and arch_atomic_fetch_and
-+#if defined(arch_atomic_fetch_and_relaxed)
-+	return arch_atomic_fetch_and_relaxed(i, v);
-+#elif defined(arch_atomic_fetch_and)
-+	return arch_atomic_fetch_and(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_and_relaxed"
- #endif
-
-#endif /* arch_atomic_fetch_and_relaxed */
-
-#ifndef arch_atomic_andnot
-static __always_inline void
-arch_atomic_andnot(int i, atomic_t *v)
-{
-	arch_atomic_and(~i, v);
- }
-#define arch_atomic_andnot arch_atomic_andnot
-#endif
-
-#ifndef arch_atomic_fetch_andnot_relaxed
-#ifdef arch_atomic_fetch_andnot
-#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
-#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
-#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
-#endif /* arch_atomic_fetch_andnot */
- 
-#ifndef arch_atomic_fetch_andnot
-static __always_inline int
-arch_atomic_fetch_andnot(int i, atomic_t *v)
-+/**
-+ * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_andnot() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic_andnot(int i, atomic_t *v)
- {
-	return arch_atomic_fetch_and(~i, v);
-}
-#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
-+#if defined(arch_atomic_andnot)
-+	arch_atomic_andnot(i, v);
-+#else
-+	raw_atomic_and(~i, v);
- #endif
-
-#ifndef arch_atomic_fetch_andnot_acquire
-static __always_inline int
-arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
-{
-	return arch_atomic_fetch_and_acquire(~i, v);
- }
-#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
-#endif
- 
-#ifndef arch_atomic_fetch_andnot_release
-+/**
-+ * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_andnot_release(int i, atomic_t *v)
-+raw_atomic_fetch_andnot(int i, atomic_t *v)
- {
-	return arch_atomic_fetch_and_release(~i, v);
-}
-#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
-+#if defined(arch_atomic_fetch_andnot)
-+	return arch_atomic_fetch_andnot(i, v);
-+#elif defined(arch_atomic_fetch_andnot_relaxed)
-+	int ret;
-+	__atomic_pre_full_fence();
-+	ret = arch_atomic_fetch_andnot_relaxed(i, v);
-+	__atomic_post_full_fence();
-+	return ret;
-+#else
-+	return raw_atomic_fetch_and(~i, v);
- #endif
-
-#ifndef arch_atomic_fetch_andnot_relaxed
-static __always_inline int
-arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
-{
-	return arch_atomic_fetch_and_relaxed(~i, v);
- }
-#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
-#endif
-
-#else /* arch_atomic_fetch_andnot_relaxed */
- 
-#ifndef arch_atomic_fetch_andnot_acquire
-+/**
-+ * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
-+raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_andnot_acquire)
-+	return arch_atomic_fetch_andnot_acquire(i, v);
-+#elif defined(arch_atomic_fetch_andnot_relaxed)
- 	int ret = arch_atomic_fetch_andnot_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
-+#elif defined(arch_atomic_fetch_andnot)
-+	return arch_atomic_fetch_andnot(i, v);
-+#else
-+	return raw_atomic_fetch_and_acquire(~i, v);
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_andnot_release
-+/**
-+ * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_andnot_release(int i, atomic_t *v)
-+raw_atomic_fetch_andnot_release(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_andnot_release)
-+	return arch_atomic_fetch_andnot_release(i, v);
-+#elif defined(arch_atomic_fetch_andnot_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic_fetch_andnot_relaxed(i, v);
-+#elif defined(arch_atomic_fetch_andnot)
-+	return arch_atomic_fetch_andnot(i, v);
-+#else
-+	return raw_atomic_fetch_and_release(~i, v);
-+#endif
- }
-#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
-+
-+/**
-+ * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
-+static __always_inline int
-+raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
-+{
-+#if defined(arch_atomic_fetch_andnot_relaxed)
-+	return arch_atomic_fetch_andnot_relaxed(i, v);
-+#elif defined(arch_atomic_fetch_andnot)
-+	return arch_atomic_fetch_andnot(i, v);
-+#else
-+	return raw_atomic_fetch_and_relaxed(~i, v);
- #endif
-+}
-+
-+/**
-+ * raw_atomic_or() - atomic bitwise OR with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v | @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_or() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic_or(int i, atomic_t *v)
-+{
-+	arch_atomic_or(i, v);
-+}
- 
-#ifndef arch_atomic_fetch_andnot
-+/**
-+ * raw_atomic_fetch_or() - atomic bitwise OR with full ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v | @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_andnot(int i, atomic_t *v)
-+raw_atomic_fetch_or(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_or)
-+	return arch_atomic_fetch_or(i, v);
-+#elif defined(arch_atomic_fetch_or_relaxed)
- 	int ret;
- 	__atomic_pre_full_fence();
-	ret = arch_atomic_fetch_andnot_relaxed(i, v);
-+	ret = arch_atomic_fetch_or_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
-+#else
-+#error "Unable to define raw_atomic_fetch_or"
- #endif
-+}
- 
-#endif /* arch_atomic_fetch_andnot_relaxed */
-
-#ifndef arch_atomic_fetch_or_relaxed
-#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
-#define arch_atomic_fetch_or_release arch_atomic_fetch_or
-#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
-#else /* arch_atomic_fetch_or_relaxed */
-
-#ifndef arch_atomic_fetch_or_acquire
-+/**
-+ * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v | @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_or_acquire(int i, atomic_t *v)
-+raw_atomic_fetch_or_acquire(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_or_acquire)
-+	return arch_atomic_fetch_or_acquire(i, v);
-+#elif defined(arch_atomic_fetch_or_relaxed)
- 	int ret = arch_atomic_fetch_or_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
-+#elif defined(arch_atomic_fetch_or)
-+	return arch_atomic_fetch_or(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_or_acquire"
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_or_release
-+/**
-+ * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v | @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_or_release(int i, atomic_t *v)
-+raw_atomic_fetch_or_release(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_or_release)
-+	return arch_atomic_fetch_or_release(i, v);
-+#elif defined(arch_atomic_fetch_or_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic_fetch_or_relaxed(i, v);
-+#elif defined(arch_atomic_fetch_or)
-+	return arch_atomic_fetch_or(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_or_release"
-+#endif
- }
-#define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
-+
-+/**
-+ * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v | @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
-+static __always_inline int
-+raw_atomic_fetch_or_relaxed(int i, atomic_t *v)
-+{
-+#if defined(arch_atomic_fetch_or_relaxed)
-+	return arch_atomic_fetch_or_relaxed(i, v);
-+#elif defined(arch_atomic_fetch_or)
-+	return arch_atomic_fetch_or(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_or_relaxed"
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_or
-+/**
-+ * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_xor() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic_xor(int i, atomic_t *v)
-+{
-+	arch_atomic_xor(i, v);
-+}
-+
-+/**
-+ * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_or(int i, atomic_t *v)
-+raw_atomic_fetch_xor(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_xor)
-+	return arch_atomic_fetch_xor(i, v);
-+#elif defined(arch_atomic_fetch_xor_relaxed)
- 	int ret;
- 	__atomic_pre_full_fence();
-	ret = arch_atomic_fetch_or_relaxed(i, v);
-+	ret = arch_atomic_fetch_xor_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_or arch_atomic_fetch_or
-+#else
-+#error "Unable to define raw_atomic_fetch_xor"
- #endif
-+}
- 
-#endif /* arch_atomic_fetch_or_relaxed */
-
-#ifndef arch_atomic_fetch_xor_relaxed
-#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
-#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
-#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
-#else /* arch_atomic_fetch_xor_relaxed */
-
-#ifndef arch_atomic_fetch_xor_acquire
-+/**
-+ * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
-+raw_atomic_fetch_xor_acquire(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_xor_acquire)
-+	return arch_atomic_fetch_xor_acquire(i, v);
-+#elif defined(arch_atomic_fetch_xor_relaxed)
- 	int ret = arch_atomic_fetch_xor_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
-+#elif defined(arch_atomic_fetch_xor)
-+	return arch_atomic_fetch_xor(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_xor_acquire"
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_xor_release
-+/**
-+ * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_xor_release(int i, atomic_t *v)
-+raw_atomic_fetch_xor_release(int i, atomic_t *v)
- {
-+#if defined(arch_atomic_fetch_xor_release)
-+	return arch_atomic_fetch_xor_release(i, v);
-+#elif defined(arch_atomic_fetch_xor_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic_fetch_xor_relaxed(i, v);
-+#elif defined(arch_atomic_fetch_xor)
-+	return arch_atomic_fetch_xor(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_xor_release"
-+#endif
- }
-#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
-+
-+/**
-+ * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
-+static __always_inline int
-+raw_atomic_fetch_xor_relaxed(int i, atomic_t *v)
-+{
-+#if defined(arch_atomic_fetch_xor_relaxed)
-+	return arch_atomic_fetch_xor_relaxed(i, v);
-+#elif defined(arch_atomic_fetch_xor)
-+	return arch_atomic_fetch_xor(i, v);
-+#else
-+#error "Unable to define raw_atomic_fetch_xor_relaxed"
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_xor
-+/**
-+ * raw_atomic_xchg() - atomic exchange with full ordering
-+ * @v: pointer to atomic_t
-+ * @new: int value to assign
-+ *
-+ * Atomically updates @v to @new with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_xchg() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_fetch_xor(int i, atomic_t *v)
-+raw_atomic_xchg(atomic_t *v, int new)
- {
-+#if defined(arch_atomic_xchg)
-+	return arch_atomic_xchg(v, new);
-+#elif defined(arch_atomic_xchg_relaxed)
- 	int ret;
- 	__atomic_pre_full_fence();
-	ret = arch_atomic_fetch_xor_relaxed(i, v);
-+	ret = arch_atomic_xchg_relaxed(v, new);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic_fetch_xor arch_atomic_fetch_xor
-+#else
-+	return raw_xchg(&v->counter, new);
- #endif
-+}
- 
-#endif /* arch_atomic_fetch_xor_relaxed */
-
-#ifndef arch_atomic_xchg_relaxed
-#define arch_atomic_xchg_acquire arch_atomic_xchg
-#define arch_atomic_xchg_release arch_atomic_xchg
-#define arch_atomic_xchg_relaxed arch_atomic_xchg
-#else /* arch_atomic_xchg_relaxed */
-
-#ifndef arch_atomic_xchg_acquire
-+/**
-+ * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering
-+ * @v: pointer to atomic_t
-+ * @new: int value to assign
-+ *
-+ * Atomically updates @v to @new with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_xchg_acquire(atomic_t *v, int i)
-+raw_atomic_xchg_acquire(atomic_t *v, int new)
- {
-	int ret = arch_atomic_xchg_relaxed(v, i);
-+#if defined(arch_atomic_xchg_acquire)
-+	return arch_atomic_xchg_acquire(v, new);
-+#elif defined(arch_atomic_xchg_relaxed)
-+	int ret = arch_atomic_xchg_relaxed(v, new);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
-+#elif defined(arch_atomic_xchg)
-+	return arch_atomic_xchg(v, new);
-+#else
-+	return raw_xchg_acquire(&v->counter, new);
- #endif
-+}
- 
-#ifndef arch_atomic_xchg_release
-+/**
-+ * raw_atomic_xchg_release() - atomic exchange with release ordering
-+ * @v: pointer to atomic_t
-+ * @new: int value to assign
-+ *
-+ * Atomically updates @v to @new with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_xchg_release(atomic_t *v, int i)
-+raw_atomic_xchg_release(atomic_t *v, int new)
- {
-+#if defined(arch_atomic_xchg_release)
-+	return arch_atomic_xchg_release(v, new);
-+#elif defined(arch_atomic_xchg_relaxed)
- 	__atomic_release_fence();
-	return arch_atomic_xchg_relaxed(v, i);
-+	return arch_atomic_xchg_relaxed(v, new);
-+#elif defined(arch_atomic_xchg)
-+	return arch_atomic_xchg(v, new);
-+#else
-+	return raw_xchg_release(&v->counter, new);
-+#endif
- }
-#define arch_atomic_xchg_release arch_atomic_xchg_release
-+
-+/**
-+ * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering
-+ * @v: pointer to atomic_t
-+ * @new: int value to assign
-+ *
-+ * Atomically updates @v to @new with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
-+static __always_inline int
-+raw_atomic_xchg_relaxed(atomic_t *v, int new)
-+{
-+#if defined(arch_atomic_xchg_relaxed)
-+	return arch_atomic_xchg_relaxed(v, new);
-+#elif defined(arch_atomic_xchg)
-+	return arch_atomic_xchg(v, new);
-+#else
-+	return raw_xchg_relaxed(&v->counter, new);
- #endif
-+}
- 
-#ifndef arch_atomic_xchg
-+/**
-+ * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering
-+ * @v: pointer to atomic_t
-+ * @old: int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_xchg(atomic_t *v, int i)
-+raw_atomic_cmpxchg(atomic_t *v, int old, int new)
- {
-+#if defined(arch_atomic_cmpxchg)
-+	return arch_atomic_cmpxchg(v, old, new);
-+#elif defined(arch_atomic_cmpxchg_relaxed)
- 	int ret;
- 	__atomic_pre_full_fence();
-	ret = arch_atomic_xchg_relaxed(v, i);
-+	ret = arch_atomic_cmpxchg_relaxed(v, old, new);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic_xchg arch_atomic_xchg
-+#else
-+	return raw_cmpxchg(&v->counter, old, new);
- #endif
-+}
- 
-#endif /* arch_atomic_xchg_relaxed */
-
-#ifndef arch_atomic_cmpxchg_relaxed
-#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
-#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
-#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
-#else /* arch_atomic_cmpxchg_relaxed */
-
-#ifndef arch_atomic_cmpxchg_acquire
-+/**
-+ * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
-+ * @v: pointer to atomic_t
-+ * @old: int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
-+raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
- {
-+#if defined(arch_atomic_cmpxchg_acquire)
-+	return arch_atomic_cmpxchg_acquire(v, old, new);
-+#elif defined(arch_atomic_cmpxchg_relaxed)
- 	int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
-+#elif defined(arch_atomic_cmpxchg)
-+	return arch_atomic_cmpxchg(v, old, new);
-+#else
-+	return raw_cmpxchg_acquire(&v->counter, old, new);
- #endif
-+}
- 
-#ifndef arch_atomic_cmpxchg_release
-+/**
-+ * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering
-+ * @v: pointer to atomic_t
-+ * @old: int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with release ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
-+raw_atomic_cmpxchg_release(atomic_t *v, int old, int new)
- {
-+#if defined(arch_atomic_cmpxchg_release)
-+	return arch_atomic_cmpxchg_release(v, old, new);
-+#elif defined(arch_atomic_cmpxchg_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic_cmpxchg_relaxed(v, old, new);
-}
-#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
-+#elif defined(arch_atomic_cmpxchg)
-+	return arch_atomic_cmpxchg(v, old, new);
-+#else
-+	return raw_cmpxchg_release(&v->counter, old, new);
- #endif
-+}
- 
-#ifndef arch_atomic_cmpxchg
-+/**
-+ * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
-+ * @v: pointer to atomic_t
-+ * @old: int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-arch_atomic_cmpxchg(atomic_t *v, int old, int new)
-+raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
- {
-	int ret;
-	__atomic_pre_full_fence();
-	ret = arch_atomic_cmpxchg_relaxed(v, old, new);
-	__atomic_post_full_fence();
-	return ret;
-}
-#define arch_atomic_cmpxchg arch_atomic_cmpxchg
-+#if defined(arch_atomic_cmpxchg_relaxed)
-+	return arch_atomic_cmpxchg_relaxed(v, old, new);
-+#elif defined(arch_atomic_cmpxchg)
-+	return arch_atomic_cmpxchg(v, old, new);
-+#else
-+	return raw_cmpxchg_relaxed(&v->counter, old, new);
- #endif
-+}
- 
-#endif /* arch_atomic_cmpxchg_relaxed */
-
-#ifndef arch_atomic_try_cmpxchg_relaxed
-#ifdef arch_atomic_try_cmpxchg
-#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
-#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
-#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
-#endif /* arch_atomic_try_cmpxchg */
-
-#ifndef arch_atomic_try_cmpxchg
-+/**
-+ * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering
-+ * @v: pointer to atomic_t
-+ * @old: pointer to int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with full ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
-+raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
- {
-+#if defined(arch_atomic_try_cmpxchg)
-+	return arch_atomic_try_cmpxchg(v, old, new);
-+#elif defined(arch_atomic_try_cmpxchg_relaxed)
-+	bool ret;
-+	__atomic_pre_full_fence();
-+	ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
-+	__atomic_post_full_fence();
-+	return ret;
-+#else
- 	int r, o = *old;
-	r = arch_atomic_cmpxchg(v, o, new);
-+	r = raw_atomic_cmpxchg(v, o, new);
- 	if (unlikely(r != o))
- 		*old = r;
- 	return likely(r == o);
-}
-#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
- #endif
-+}
- 
-#ifndef arch_atomic_try_cmpxchg_acquire
-+/**
-+ * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
-+ * @v: pointer to atomic_t
-+ * @old: pointer to int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
-+raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
- {
-+#if defined(arch_atomic_try_cmpxchg_acquire)
-+	return arch_atomic_try_cmpxchg_acquire(v, old, new);
-+#elif defined(arch_atomic_try_cmpxchg_relaxed)
-+	bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
-+	__atomic_acquire_fence();
-+	return ret;
-+#elif defined(arch_atomic_try_cmpxchg)
-+	return arch_atomic_try_cmpxchg(v, old, new);
-+#else
- 	int r, o = *old;
-	r = arch_atomic_cmpxchg_acquire(v, o, new);
-+	r = raw_atomic_cmpxchg_acquire(v, o, new);
- 	if (unlikely(r != o))
- 		*old = r;
- 	return likely(r == o);
-}
-#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
- #endif
-+}
- 
-#ifndef arch_atomic_try_cmpxchg_release
-+/**
-+ * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
-+ * @v: pointer to atomic_t
-+ * @old: pointer to int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with release ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
-+raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
- {
-+#if defined(arch_atomic_try_cmpxchg_release)
-+	return arch_atomic_try_cmpxchg_release(v, old, new);
-+#elif defined(arch_atomic_try_cmpxchg_relaxed)
-+	__atomic_release_fence();
-+	return arch_atomic_try_cmpxchg_relaxed(v, old, new);
-+#elif defined(arch_atomic_try_cmpxchg)
-+	return arch_atomic_try_cmpxchg(v, old, new);
-+#else
- 	int r, o = *old;
-	r = arch_atomic_cmpxchg_release(v, o, new);
-+	r = raw_atomic_cmpxchg_release(v, o, new);
- 	if (unlikely(r != o))
- 		*old = r;
- 	return likely(r == o);
-}
-#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
- #endif
-+}
- 
-#ifndef arch_atomic_try_cmpxchg_relaxed
-+/**
-+ * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
-+ * @v: pointer to atomic_t
-+ * @old: pointer to int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
-+raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
- {
-+#if defined(arch_atomic_try_cmpxchg_relaxed)
-+	return arch_atomic_try_cmpxchg_relaxed(v, old, new);
-+#elif defined(arch_atomic_try_cmpxchg)
-+	return arch_atomic_try_cmpxchg(v, old, new);
-+#else
- 	int r, o = *old;
-	r = arch_atomic_cmpxchg_relaxed(v, o, new);
-+	r = raw_atomic_cmpxchg_relaxed(v, o, new);
- 	if (unlikely(r != o))
- 		*old = r;
- 	return likely(r == o);
-}
-#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
- #endif
-+}
- 
-#else /* arch_atomic_try_cmpxchg_relaxed */
-
-#ifndef arch_atomic_try_cmpxchg_acquire
-+/**
-+ * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
-+raw_atomic_sub_and_test(int i, atomic_t *v)
- {
-	bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
-	__atomic_acquire_fence();
-	return ret;
-}
-#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
-+#if defined(arch_atomic_sub_and_test)
-+	return arch_atomic_sub_and_test(i, v);
-+#else
-+	return raw_atomic_sub_return(i, v) == 0;
- #endif
-+}
- 
-#ifndef arch_atomic_try_cmpxchg_release
-+/**
-+ * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
-+raw_atomic_dec_and_test(atomic_t *v)
- {
-	__atomic_release_fence();
-	return arch_atomic_try_cmpxchg_relaxed(v, old, new);
-}
-#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
-+#if defined(arch_atomic_dec_and_test)
-+	return arch_atomic_dec_and_test(v);
-+#else
-+	return raw_atomic_dec_return(v) == 0;
- #endif
-+}
- 
-#ifndef arch_atomic_try_cmpxchg
-+/**
-+ * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
-+raw_atomic_inc_and_test(atomic_t *v)
- {
-	bool ret;
-	__atomic_pre_full_fence();
-	ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
-	__atomic_post_full_fence();
-	return ret;
-}
-#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
-+#if defined(arch_atomic_inc_and_test)
-+	return arch_atomic_inc_and_test(v);
-+#else
-+	return raw_atomic_inc_return(v) == 0;
- #endif
-+}
- 
-#endif /* arch_atomic_try_cmpxchg_relaxed */
-
-#ifndef arch_atomic_sub_and_test
- /**
- * arch_atomic_sub_and_test - subtract value from variable and test result
- * @i: integer value to subtract
- * @v: pointer of type atomic_t
-+ * raw_atomic_add_negative() - atomic add and test if negative with full ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere.
-  *
- * Atomically subtracts @i from @v and returns
- * true if the result is zero, or false for all
- * other cases.
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-  */
- static __always_inline bool
-arch_atomic_sub_and_test(int i, atomic_t *v)
-+raw_atomic_add_negative(int i, atomic_t *v)
- {
-	return arch_atomic_sub_return(i, v) == 0;
-}
-#define arch_atomic_sub_and_test arch_atomic_sub_and_test
-+#if defined(arch_atomic_add_negative)
-+	return arch_atomic_add_negative(i, v);
-+#elif defined(arch_atomic_add_negative_relaxed)
-+	bool ret;
-+	__atomic_pre_full_fence();
-+	ret = arch_atomic_add_negative_relaxed(i, v);
-+	__atomic_post_full_fence();
-+	return ret;
-+#else
-+	return raw_atomic_add_return(i, v) < 0;
- #endif
-+}
- 
-#ifndef arch_atomic_dec_and_test
- /**
- * arch_atomic_dec_and_test - decrement and test
- * @v: pointer of type atomic_t
-+ * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-  *
- * Atomically decrements @v by 1 and
- * returns true if the result is 0, or false for all other
- * cases.
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-  */
- static __always_inline bool
-arch_atomic_dec_and_test(atomic_t *v)
-+raw_atomic_add_negative_acquire(int i, atomic_t *v)
- {
-	return arch_atomic_dec_return(v) == 0;
-}
-#define arch_atomic_dec_and_test arch_atomic_dec_and_test
-+#if defined(arch_atomic_add_negative_acquire)
-+	return arch_atomic_add_negative_acquire(i, v);
-+#elif defined(arch_atomic_add_negative_relaxed)
-+	bool ret = arch_atomic_add_negative_relaxed(i, v);
-+	__atomic_acquire_fence();
-+	return ret;
-+#elif defined(arch_atomic_add_negative)
-+	return arch_atomic_add_negative(i, v);
-+#else
-+	return raw_atomic_add_return_acquire(i, v) < 0;
- #endif
-+}
- 
-#ifndef arch_atomic_inc_and_test
- /**
- * arch_atomic_inc_and_test - increment and test
- * @v: pointer of type atomic_t
-+ * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-  *
- * Atomically increments @v by 1
- * and returns true if the result is zero, or false for all
- * other cases.
-+ * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-  */
- static __always_inline bool
-arch_atomic_inc_and_test(atomic_t *v)
-+raw_atomic_add_negative_release(int i, atomic_t *v)
- {
-	return arch_atomic_inc_return(v) == 0;
-}
-#define arch_atomic_inc_and_test arch_atomic_inc_and_test
-+#if defined(arch_atomic_add_negative_release)
-+	return arch_atomic_add_negative_release(i, v);
-+#elif defined(arch_atomic_add_negative_relaxed)
-+	__atomic_release_fence();
-+	return arch_atomic_add_negative_relaxed(i, v);
-+#elif defined(arch_atomic_add_negative)
-+	return arch_atomic_add_negative(i, v);
-+#else
-+	return raw_atomic_add_return_release(i, v) < 0;
- #endif
-+}
- 
-#ifndef arch_atomic_add_negative
- /**
- * arch_atomic_add_negative - add and test if negative
- * @i: integer value to add
- * @v: pointer of type atomic_t
-+ * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-  *
- * Atomically adds @i to @v and returns true
- * if the result is negative, or false when
- * result is greater than or equal to zero.
-+ * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-  */
- static __always_inline bool
-arch_atomic_add_negative(int i, atomic_t *v)
-+raw_atomic_add_negative_relaxed(int i, atomic_t *v)
- {
-	return arch_atomic_add_return(i, v) < 0;
-}
-#define arch_atomic_add_negative arch_atomic_add_negative
-+#if defined(arch_atomic_add_negative_relaxed)
-+	return arch_atomic_add_negative_relaxed(i, v);
-+#elif defined(arch_atomic_add_negative)
-+	return arch_atomic_add_negative(i, v);
-+#else
-+	return raw_atomic_add_return_relaxed(i, v) < 0;
- #endif
-+}
- 
-#ifndef arch_atomic_fetch_add_unless
- /**
- * arch_atomic_fetch_add_unless - add unless the number is already a given value
- * @v: pointer of type atomic_t
- * @a: the amount to add to v...
- * @u: ...unless v is equal to u.
-+ * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering
-+ * @v: pointer to atomic_t
-+ * @a: int value to add
-+ * @u: int value to compare with
-+ *
-+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere.
-  *
- * Atomically adds @a to @v, so long as @v was not already @u.
- * Returns original value of @v
-+ * Return: The original value of @v.
-  */
- static __always_inline int
-arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
-+raw_atomic_fetch_add_unless(atomic_t *v, int a, int u)
- {
-	int c = arch_atomic_read(v);
-+#if defined(arch_atomic_fetch_add_unless)
-+	return arch_atomic_fetch_add_unless(v, a, u);
-+#else
-+	int c = raw_atomic_read(v);
- 
- 	do {
- 		if (unlikely(c == u))
- 			break;
-	} while (!arch_atomic_try_cmpxchg(v, &c, c + a));
-+	} while (!raw_atomic_try_cmpxchg(v, &c, c + a));
- 
- 	return c;
-}
-#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
- #endif
-+}
- 
-#ifndef arch_atomic_add_unless
- /**
- * arch_atomic_add_unless - add unless the number is already a given value
- * @v: pointer of type atomic_t
- * @a: the amount to add to v...
- * @u: ...unless v is equal to u.
-+ * raw_atomic_add_unless() - atomic add unless value with full ordering
-+ * @v: pointer to atomic_t
-+ * @a: int value to add
-+ * @u: int value to compare with
-+ *
-+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere.
-  *
- * Atomically adds @a to @v, if @v was not already @u.
- * Returns true if the addition was done.
-+ * Return: @true if @v was updated, @false otherwise.
-  */
- static __always_inline bool
-arch_atomic_add_unless(atomic_t *v, int a, int u)
-+raw_atomic_add_unless(atomic_t *v, int a, int u)
- {
-	return arch_atomic_fetch_add_unless(v, a, u) != u;
-}
-#define arch_atomic_add_unless arch_atomic_add_unless
-+#if defined(arch_atomic_add_unless)
-+	return arch_atomic_add_unless(v, a, u);
-+#else
-+	return raw_atomic_fetch_add_unless(v, a, u) != u;
- #endif
-+}
- 
-#ifndef arch_atomic_inc_not_zero
- /**
- * arch_atomic_inc_not_zero - increment unless the number is zero
- * @v: pointer of type atomic_t
-+ * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-  *
- * Atomically increments @v by 1, if @v is non-zero.
- * Returns true if the increment was done.
-+ * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-  */
- static __always_inline bool
-arch_atomic_inc_not_zero(atomic_t *v)
-+raw_atomic_inc_not_zero(atomic_t *v)
- {
-	return arch_atomic_add_unless(v, 1, 0);
-}
-#define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
-+#if defined(arch_atomic_inc_not_zero)
-+	return arch_atomic_inc_not_zero(v);
-+#else
-+	return raw_atomic_add_unless(v, 1, 0);
- #endif
-+}
- 
-#ifndef arch_atomic_inc_unless_negative
-+/**
-+ * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_inc_unless_negative(atomic_t *v)
-+raw_atomic_inc_unless_negative(atomic_t *v)
- {
-	int c = arch_atomic_read(v);
-+#if defined(arch_atomic_inc_unless_negative)
-+	return arch_atomic_inc_unless_negative(v);
-+#else
-+	int c = raw_atomic_read(v);
- 
- 	do {
- 		if (unlikely(c < 0))
- 			return false;
-	} while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
-+	} while (!raw_atomic_try_cmpxchg(v, &c, c + 1));
- 
- 	return true;
-}
-#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
- #endif
-+}
- 
-#ifndef arch_atomic_dec_unless_positive
-+/**
-+ * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_dec_unless_positive(atomic_t *v)
-+raw_atomic_dec_unless_positive(atomic_t *v)
- {
-	int c = arch_atomic_read(v);
-+#if defined(arch_atomic_dec_unless_positive)
-+	return arch_atomic_dec_unless_positive(v);
-+#else
-+	int c = raw_atomic_read(v);
- 
- 	do {
- 		if (unlikely(c > 0))
- 			return false;
-	} while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
-+	} while (!raw_atomic_try_cmpxchg(v, &c, c - 1));
- 
- 	return true;
-}
-#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
- #endif
-+}
- 
-#ifndef arch_atomic_dec_if_positive
-+/**
-+ * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere.
-+ *
-+ * Return: The old value of (@v - 1), regardless of whether @v was updated.
-+ */
- static __always_inline int
-arch_atomic_dec_if_positive(atomic_t *v)
-+raw_atomic_dec_if_positive(atomic_t *v)
- {
-	int dec, c = arch_atomic_read(v);
-+#if defined(arch_atomic_dec_if_positive)
-+	return arch_atomic_dec_if_positive(v);
-+#else
-+	int dec, c = raw_atomic_read(v);
- 
- 	do {
- 		dec = c - 1;
- 		if (unlikely(dec < 0))
- 			break;
-	} while (!arch_atomic_try_cmpxchg(v, &c, dec));
-+	} while (!raw_atomic_try_cmpxchg(v, &c, dec));
- 
- 	return dec;
-}
-#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
- #endif
-+}
- 
- #ifdef CONFIG_GENERIC_ATOMIC64
- #include <asm-generic/atomic64.h>
- #endif
- 
-#ifndef arch_atomic64_read_acquire
-+/**
-+ * raw_atomic64_read() - atomic load with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically loads the value of @v with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_read() elsewhere.
-+ *
-+ * Return: The value loaded from @v.
-+ */
- static __always_inline s64
-arch_atomic64_read_acquire(const atomic64_t *v)
-+raw_atomic64_read(const atomic64_t *v)
- {
-+	return arch_atomic64_read(v);
-+}
-+
-+/**
-+ * raw_atomic64_read_acquire() - atomic load with acquire ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically loads the value of @v with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere.
-+ *
-+ * Return: The value loaded from @v.
-+ */
-+static __always_inline s64
-+raw_atomic64_read_acquire(const atomic64_t *v)
-+{
-+#if defined(arch_atomic64_read_acquire)
-+	return arch_atomic64_read_acquire(v);
-+#else
- 	s64 ret;
- 
- 	if (__native_word(atomic64_t)) {
- 		ret = smp_load_acquire(&(v)->counter);
- 	} else {
-		ret = arch_atomic64_read(v);
-+		ret = raw_atomic64_read(v);
- 		__atomic_acquire_fence();
- 	}
- 
- 	return ret;
-}
-#define arch_atomic64_read_acquire arch_atomic64_read_acquire
- #endif
-+}
-+
-+/**
-+ * raw_atomic64_set() - atomic set with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ * @i: s64 value to assign
-+ *
-+ * Atomically sets @v to @i with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_set() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic64_set(atomic64_t *v, s64 i)
-+{
-+	arch_atomic64_set(v, i);
-+}
- 
-#ifndef arch_atomic64_set_release
-+/**
-+ * raw_atomic64_set_release() - atomic set with release ordering
-+ * @v: pointer to atomic64_t
-+ * @i: s64 value to assign
-+ *
-+ * Atomically sets @v to @i with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic64_set_release(atomic64_t *v, s64 i)
-+raw_atomic64_set_release(atomic64_t *v, s64 i)
- {
-+#if defined(arch_atomic64_set_release)
-+	arch_atomic64_set_release(v, i);
-+#else
- 	if (__native_word(atomic64_t)) {
- 		smp_store_release(&(v)->counter, i);
- 	} else {
- 		__atomic_release_fence();
-		arch_atomic64_set(v, i);
-+		raw_atomic64_set(v, i);
- 	}
-}
-#define arch_atomic64_set_release arch_atomic64_set_release
- #endif
-
-#ifndef arch_atomic64_add_return_relaxed
-#define arch_atomic64_add_return_acquire arch_atomic64_add_return
-#define arch_atomic64_add_return_release arch_atomic64_add_return
-#define arch_atomic64_add_return_relaxed arch_atomic64_add_return
-#else /* arch_atomic64_add_return_relaxed */
-
-#ifndef arch_atomic64_add_return_acquire
-static __always_inline s64
-arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
-{
-	s64 ret = arch_atomic64_add_return_relaxed(i, v);
-	__atomic_acquire_fence();
-	return ret;
- }
-#define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
-#endif
- 
-#ifndef arch_atomic64_add_return_release
-static __always_inline s64
-arch_atomic64_add_return_release(s64 i, atomic64_t *v)
-+/**
-+ * raw_atomic64_add() - atomic add with relaxed ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_add() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic64_add(s64 i, atomic64_t *v)
- {
-	__atomic_release_fence();
-	return arch_atomic64_add_return_relaxed(i, v);
-+	arch_atomic64_add(i, v);
- }
-#define arch_atomic64_add_return_release arch_atomic64_add_return_release
-#endif
- 
-#ifndef arch_atomic64_add_return
-+/**
-+ * raw_atomic64_add_return() - atomic add with full ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_add_return(s64 i, atomic64_t *v)
-+raw_atomic64_add_return(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_add_return)
-+	return arch_atomic64_add_return(i, v);
-+#elif defined(arch_atomic64_add_return_relaxed)
- 	s64 ret;
- 	__atomic_pre_full_fence();
- 	ret = arch_atomic64_add_return_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic64_add_return arch_atomic64_add_return
-+#else
-+#error "Unable to define raw_atomic64_add_return"
- #endif
-+}
- 
-#endif /* arch_atomic64_add_return_relaxed */
-
-#ifndef arch_atomic64_fetch_add_relaxed
-#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
-#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
-#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
-#else /* arch_atomic64_fetch_add_relaxed */
-
-#ifndef arch_atomic64_fetch_add_acquire
-+/**
-+ * raw_atomic64_add_return_acquire() - atomic add with acquire ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
-+raw_atomic64_add_return_acquire(s64 i, atomic64_t *v)
- {
-	s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
-+#if defined(arch_atomic64_add_return_acquire)
-+	return arch_atomic64_add_return_acquire(i, v);
-+#elif defined(arch_atomic64_add_return_relaxed)
-+	s64 ret = arch_atomic64_add_return_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
-+#elif defined(arch_atomic64_add_return)
-+	return arch_atomic64_add_return(i, v);
-+#else
-+#error "Unable to define raw_atomic64_add_return_acquire"
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_add_release
-+/**
-+ * raw_atomic64_add_return_release() - atomic add with release ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
-+raw_atomic64_add_return_release(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_add_return_release)
-+	return arch_atomic64_add_return_release(i, v);
-+#elif defined(arch_atomic64_add_return_relaxed)
- 	__atomic_release_fence();
-	return arch_atomic64_fetch_add_relaxed(i, v);
-+	return arch_atomic64_add_return_relaxed(i, v);
-+#elif defined(arch_atomic64_add_return)
-+	return arch_atomic64_add_return(i, v);
-+#else
-+#error "Unable to define raw_atomic64_add_return_release"
-+#endif
- }
-#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
-+
-+/**
-+ * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
-+static __always_inline s64
-+raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v)
-+{
-+#if defined(arch_atomic64_add_return_relaxed)
-+	return arch_atomic64_add_return_relaxed(i, v);
-+#elif defined(arch_atomic64_add_return)
-+	return arch_atomic64_add_return(i, v);
-+#else
-+#error "Unable to define raw_atomic64_add_return_relaxed"
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_add
-+/**
-+ * raw_atomic64_fetch_add() - atomic add with full ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_add(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_add(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_add)
-+	return arch_atomic64_fetch_add(i, v);
-+#elif defined(arch_atomic64_fetch_add_relaxed)
- 	s64 ret;
- 	__atomic_pre_full_fence();
- 	ret = arch_atomic64_fetch_add_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_add arch_atomic64_fetch_add
-+#else
-+#error "Unable to define raw_atomic64_fetch_add"
- #endif
-+}
- 
-#endif /* arch_atomic64_fetch_add_relaxed */
-
-#ifndef arch_atomic64_sub_return_relaxed
-#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
-#define arch_atomic64_sub_return_release arch_atomic64_sub_return
-#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
-#else /* arch_atomic64_sub_return_relaxed */
-
-#ifndef arch_atomic64_sub_return_acquire
-+/**
-+ * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
- {
-	s64 ret = arch_atomic64_sub_return_relaxed(i, v);
-+#if defined(arch_atomic64_fetch_add_acquire)
-+	return arch_atomic64_fetch_add_acquire(i, v);
-+#elif defined(arch_atomic64_fetch_add_relaxed)
-+	s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
-+#elif defined(arch_atomic64_fetch_add)
-+	return arch_atomic64_fetch_add(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_add_acquire"
- #endif
-+}
- 
-#ifndef arch_atomic64_sub_return_release
-+/**
-+ * raw_atomic64_fetch_add_release() - atomic add with release ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_add_release(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_add_release)
-+	return arch_atomic64_fetch_add_release(i, v);
-+#elif defined(arch_atomic64_fetch_add_relaxed)
- 	__atomic_release_fence();
-	return arch_atomic64_sub_return_relaxed(i, v);
-+	return arch_atomic64_fetch_add_relaxed(i, v);
-+#elif defined(arch_atomic64_fetch_add)
-+	return arch_atomic64_fetch_add(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_add_release"
-+#endif
- }
-#define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
-+
-+/**
-+ * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
-+static __always_inline s64
-+raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
-+{
-+#if defined(arch_atomic64_fetch_add_relaxed)
-+	return arch_atomic64_fetch_add_relaxed(i, v);
-+#elif defined(arch_atomic64_fetch_add)
-+	return arch_atomic64_fetch_add(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_add_relaxed"
- #endif
-+}
-+
-+/**
-+ * raw_atomic64_sub() - atomic subtract with relaxed ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_sub() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic64_sub(s64 i, atomic64_t *v)
-+{
-+	arch_atomic64_sub(i, v);
-+}
- 
-#ifndef arch_atomic64_sub_return
-+/**
-+ * raw_atomic64_sub_return() - atomic subtract with full ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_sub_return(s64 i, atomic64_t *v)
-+raw_atomic64_sub_return(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_sub_return)
-+	return arch_atomic64_sub_return(i, v);
-+#elif defined(arch_atomic64_sub_return_relaxed)
- 	s64 ret;
- 	__atomic_pre_full_fence();
- 	ret = arch_atomic64_sub_return_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic64_sub_return arch_atomic64_sub_return
-+#else
-+#error "Unable to define raw_atomic64_sub_return"
- #endif
-+}
- 
-#endif /* arch_atomic64_sub_return_relaxed */
-
-#ifndef arch_atomic64_fetch_sub_relaxed
-#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
-#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
-#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
-#else /* arch_atomic64_fetch_sub_relaxed */
-
-#ifndef arch_atomic64_fetch_sub_acquire
-+/**
-+ * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
-+raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
- {
-	s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
-+#if defined(arch_atomic64_sub_return_acquire)
-+	return arch_atomic64_sub_return_acquire(i, v);
-+#elif defined(arch_atomic64_sub_return_relaxed)
-+	s64 ret = arch_atomic64_sub_return_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
-+#elif defined(arch_atomic64_sub_return)
-+	return arch_atomic64_sub_return(i, v);
-+#else
-+#error "Unable to define raw_atomic64_sub_return_acquire"
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_sub_release
-+/**
-+ * raw_atomic64_sub_return_release() - atomic subtract with release ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
-+raw_atomic64_sub_return_release(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_sub_return_release)
-+	return arch_atomic64_sub_return_release(i, v);
-+#elif defined(arch_atomic64_sub_return_relaxed)
- 	__atomic_release_fence();
-	return arch_atomic64_fetch_sub_relaxed(i, v);
-+	return arch_atomic64_sub_return_relaxed(i, v);
-+#elif defined(arch_atomic64_sub_return)
-+	return arch_atomic64_sub_return(i, v);
-+#else
-+#error "Unable to define raw_atomic64_sub_return_release"
-+#endif
- }
-#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
-+
-+/**
-+ * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
-+static __always_inline s64
-+raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
-+{
-+#if defined(arch_atomic64_sub_return_relaxed)
-+	return arch_atomic64_sub_return_relaxed(i, v);
-+#elif defined(arch_atomic64_sub_return)
-+	return arch_atomic64_sub_return(i, v);
-+#else
-+#error "Unable to define raw_atomic64_sub_return_relaxed"
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_sub
-+/**
-+ * raw_atomic64_fetch_sub() - atomic subtract with full ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_sub(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_sub)
-+	return arch_atomic64_fetch_sub(i, v);
-+#elif defined(arch_atomic64_fetch_sub_relaxed)
- 	s64 ret;
- 	__atomic_pre_full_fence();
- 	ret = arch_atomic64_fetch_sub_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
-+#else
-+#error "Unable to define raw_atomic64_fetch_sub"
- #endif
-
-#endif /* arch_atomic64_fetch_sub_relaxed */
-
-#ifndef arch_atomic64_inc
-static __always_inline void
-arch_atomic64_inc(atomic64_t *v)
-{
-	arch_atomic64_add(1, v);
- }
-#define arch_atomic64_inc arch_atomic64_inc
-#endif
-
-#ifndef arch_atomic64_inc_return_relaxed
-#ifdef arch_atomic64_inc_return
-#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
-#define arch_atomic64_inc_return_release arch_atomic64_inc_return
-#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
-#endif /* arch_atomic64_inc_return */
- 
-#ifndef arch_atomic64_inc_return
-+/**
-+ * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_inc_return(atomic64_t *v)
-+raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
- {
-	return arch_atomic64_add_return(1, v);
-}
-#define arch_atomic64_inc_return arch_atomic64_inc_return
-+#if defined(arch_atomic64_fetch_sub_acquire)
-+	return arch_atomic64_fetch_sub_acquire(i, v);
-+#elif defined(arch_atomic64_fetch_sub_relaxed)
-+	s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
-+	__atomic_acquire_fence();
-+	return ret;
-+#elif defined(arch_atomic64_fetch_sub)
-+	return arch_atomic64_fetch_sub(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_sub_acquire"
- #endif
-
-#ifndef arch_atomic64_inc_return_acquire
-static __always_inline s64
-arch_atomic64_inc_return_acquire(atomic64_t *v)
-{
-	return arch_atomic64_add_return_acquire(1, v);
- }
-#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
-#endif
- 
-#ifndef arch_atomic64_inc_return_release
-+/**
-+ * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_inc_return_release(atomic64_t *v)
-+raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
- {
-	return arch_atomic64_add_return_release(1, v);
-}
-#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
-+#if defined(arch_atomic64_fetch_sub_release)
-+	return arch_atomic64_fetch_sub_release(i, v);
-+#elif defined(arch_atomic64_fetch_sub_relaxed)
-+	__atomic_release_fence();
-+	return arch_atomic64_fetch_sub_relaxed(i, v);
-+#elif defined(arch_atomic64_fetch_sub)
-+	return arch_atomic64_fetch_sub(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_sub_release"
- #endif
-
-#ifndef arch_atomic64_inc_return_relaxed
-static __always_inline s64
-arch_atomic64_inc_return_relaxed(atomic64_t *v)
-{
-	return arch_atomic64_add_return_relaxed(1, v);
- }
-#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
-#endif
-
-#else /* arch_atomic64_inc_return_relaxed */
- 
-#ifndef arch_atomic64_inc_return_acquire
-+/**
-+ * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_inc_return_acquire(atomic64_t *v)
-+raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
- {
-	s64 ret = arch_atomic64_inc_return_relaxed(v);
-	__atomic_acquire_fence();
-	return ret;
-}
-#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
-+#if defined(arch_atomic64_fetch_sub_relaxed)
-+	return arch_atomic64_fetch_sub_relaxed(i, v);
-+#elif defined(arch_atomic64_fetch_sub)
-+	return arch_atomic64_fetch_sub(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_sub_relaxed"
- #endif
-+}
- 
-#ifndef arch_atomic64_inc_return_release
-static __always_inline s64
-arch_atomic64_inc_return_release(atomic64_t *v)
-+/**
-+ * raw_atomic64_inc() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_inc() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic64_inc(atomic64_t *v)
- {
-	__atomic_release_fence();
-	return arch_atomic64_inc_return_relaxed(v);
-}
-#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
-+#if defined(arch_atomic64_inc)
-+	arch_atomic64_inc(v);
-+#else
-+	raw_atomic64_add(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic64_inc_return
-+/**
-+ * raw_atomic64_inc_return() - atomic increment with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_inc_return(atomic64_t *v)
-+raw_atomic64_inc_return(atomic64_t *v)
- {
-+#if defined(arch_atomic64_inc_return)
-+	return arch_atomic64_inc_return(v);
-+#elif defined(arch_atomic64_inc_return_relaxed)
- 	s64 ret;
- 	__atomic_pre_full_fence();
- 	ret = arch_atomic64_inc_return_relaxed(v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic64_inc_return arch_atomic64_inc_return
-+#else
-+	return raw_atomic64_add_return(1, v);
- #endif
-+}
- 
-#endif /* arch_atomic64_inc_return_relaxed */
-
-#ifndef arch_atomic64_fetch_inc_relaxed
-#ifdef arch_atomic64_fetch_inc
-#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
-#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
-#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
-#endif /* arch_atomic64_fetch_inc */
-
-#ifndef arch_atomic64_fetch_inc
-+/**
-+ * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_inc(atomic64_t *v)
-+raw_atomic64_inc_return_acquire(atomic64_t *v)
- {
-	return arch_atomic64_fetch_add(1, v);
-}
-#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
-+#if defined(arch_atomic64_inc_return_acquire)
-+	return arch_atomic64_inc_return_acquire(v);
-+#elif defined(arch_atomic64_inc_return_relaxed)
-+	s64 ret = arch_atomic64_inc_return_relaxed(v);
-+	__atomic_acquire_fence();
-+	return ret;
-+#elif defined(arch_atomic64_inc_return)
-+	return arch_atomic64_inc_return(v);
-+#else
-+	return raw_atomic64_add_return_acquire(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_inc_acquire
-+/**
-+ * raw_atomic64_inc_return_release() - atomic increment with release ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_inc_acquire(atomic64_t *v)
-+raw_atomic64_inc_return_release(atomic64_t *v)
- {
-	return arch_atomic64_fetch_add_acquire(1, v);
-}
-#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
-+#if defined(arch_atomic64_inc_return_release)
-+	return arch_atomic64_inc_return_release(v);
-+#elif defined(arch_atomic64_inc_return_relaxed)
-+	__atomic_release_fence();
-+	return arch_atomic64_inc_return_relaxed(v);
-+#elif defined(arch_atomic64_inc_return)
-+	return arch_atomic64_inc_return(v);
-+#else
-+	return raw_atomic64_add_return_release(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_inc_release
-+/**
-+ * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_inc_release(atomic64_t *v)
-+raw_atomic64_inc_return_relaxed(atomic64_t *v)
- {
-	return arch_atomic64_fetch_add_release(1, v);
-}
-#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
-+#if defined(arch_atomic64_inc_return_relaxed)
-+	return arch_atomic64_inc_return_relaxed(v);
-+#elif defined(arch_atomic64_inc_return)
-+	return arch_atomic64_inc_return(v);
-+#else
-+	return raw_atomic64_add_return_relaxed(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_inc_relaxed
-+/**
-+ * raw_atomic64_fetch_inc() - atomic increment with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
-+raw_atomic64_fetch_inc(atomic64_t *v)
- {
-	return arch_atomic64_fetch_add_relaxed(1, v);
-}
-#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
-+#if defined(arch_atomic64_fetch_inc)
-+	return arch_atomic64_fetch_inc(v);
-+#elif defined(arch_atomic64_fetch_inc_relaxed)
-+	s64 ret;
-+	__atomic_pre_full_fence();
-+	ret = arch_atomic64_fetch_inc_relaxed(v);
-+	__atomic_post_full_fence();
-+	return ret;
-+#else
-+	return raw_atomic64_fetch_add(1, v);
- #endif
-+}
- 
-#else /* arch_atomic64_fetch_inc_relaxed */
-
-#ifndef arch_atomic64_fetch_inc_acquire
-+/**
-+ * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_inc_acquire(atomic64_t *v)
-+raw_atomic64_fetch_inc_acquire(atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_inc_acquire)
-+	return arch_atomic64_fetch_inc_acquire(v);
-+#elif defined(arch_atomic64_fetch_inc_relaxed)
- 	s64 ret = arch_atomic64_fetch_inc_relaxed(v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
-+#elif defined(arch_atomic64_fetch_inc)
-+	return arch_atomic64_fetch_inc(v);
-+#else
-+	return raw_atomic64_fetch_add_acquire(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_inc_release
-+/**
-+ * raw_atomic64_fetch_inc_release() - atomic increment with release ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_inc_release(atomic64_t *v)
-+raw_atomic64_fetch_inc_release(atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_inc_release)
-+	return arch_atomic64_fetch_inc_release(v);
-+#elif defined(arch_atomic64_fetch_inc_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic64_fetch_inc_relaxed(v);
-}
-#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
-+#elif defined(arch_atomic64_fetch_inc)
-+	return arch_atomic64_fetch_inc(v);
-+#else
-+	return raw_atomic64_fetch_add_release(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_inc
-+/**
-+ * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_inc(atomic64_t *v)
-+raw_atomic64_fetch_inc_relaxed(atomic64_t *v)
- {
-	s64 ret;
-	__atomic_pre_full_fence();
-	ret = arch_atomic64_fetch_inc_relaxed(v);
-	__atomic_post_full_fence();
-	return ret;
-}
-#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
-+#if defined(arch_atomic64_fetch_inc_relaxed)
-+	return arch_atomic64_fetch_inc_relaxed(v);
-+#elif defined(arch_atomic64_fetch_inc)
-+	return arch_atomic64_fetch_inc(v);
-+#else
-+	return raw_atomic64_fetch_add_relaxed(1, v);
- #endif
-
-#endif /* arch_atomic64_fetch_inc_relaxed */
-
-#ifndef arch_atomic64_dec
-static __always_inline void
-arch_atomic64_dec(atomic64_t *v)
-{
-	arch_atomic64_sub(1, v);
- }
-#define arch_atomic64_dec arch_atomic64_dec
-#endif
-
-#ifndef arch_atomic64_dec_return_relaxed
-#ifdef arch_atomic64_dec_return
-#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
-#define arch_atomic64_dec_return_release arch_atomic64_dec_return
-#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
-#endif /* arch_atomic64_dec_return */
- 
-#ifndef arch_atomic64_dec_return
-static __always_inline s64
-arch_atomic64_dec_return(atomic64_t *v)
-+/**
-+ * raw_atomic64_dec() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_dec() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic64_dec(atomic64_t *v)
- {
-	return arch_atomic64_sub_return(1, v);
-}
-#define arch_atomic64_dec_return arch_atomic64_dec_return
-+#if defined(arch_atomic64_dec)
-+	arch_atomic64_dec(v);
-+#else
-+	raw_atomic64_sub(1, v);
- #endif
-
-#ifndef arch_atomic64_dec_return_acquire
-static __always_inline s64
-arch_atomic64_dec_return_acquire(atomic64_t *v)
-{
-	return arch_atomic64_sub_return_acquire(1, v);
- }
-#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
-#endif
- 
-#ifndef arch_atomic64_dec_return_release
-+/**
-+ * raw_atomic64_dec_return() - atomic decrement with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_dec_return_release(atomic64_t *v)
-+raw_atomic64_dec_return(atomic64_t *v)
- {
-	return arch_atomic64_sub_return_release(1, v);
-}
-#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
-+#if defined(arch_atomic64_dec_return)
-+	return arch_atomic64_dec_return(v);
-+#elif defined(arch_atomic64_dec_return_relaxed)
-+	s64 ret;
-+	__atomic_pre_full_fence();
-+	ret = arch_atomic64_dec_return_relaxed(v);
-+	__atomic_post_full_fence();
-+	return ret;
-+#else
-+	return raw_atomic64_sub_return(1, v);
- #endif
-
-#ifndef arch_atomic64_dec_return_relaxed
-static __always_inline s64
-arch_atomic64_dec_return_relaxed(atomic64_t *v)
-{
-	return arch_atomic64_sub_return_relaxed(1, v);
- }
-#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
-#endif
-
-#else /* arch_atomic64_dec_return_relaxed */
- 
-#ifndef arch_atomic64_dec_return_acquire
-+/**
-+ * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_dec_return_acquire(atomic64_t *v)
-+raw_atomic64_dec_return_acquire(atomic64_t *v)
- {
-+#if defined(arch_atomic64_dec_return_acquire)
-+	return arch_atomic64_dec_return_acquire(v);
-+#elif defined(arch_atomic64_dec_return_relaxed)
- 	s64 ret = arch_atomic64_dec_return_relaxed(v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
-+#elif defined(arch_atomic64_dec_return)
-+	return arch_atomic64_dec_return(v);
-+#else
-+	return raw_atomic64_sub_return_acquire(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic64_dec_return_release
-+/**
-+ * raw_atomic64_dec_return_release() - atomic decrement with release ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_dec_return_release(atomic64_t *v)
-+raw_atomic64_dec_return_release(atomic64_t *v)
- {
-+#if defined(arch_atomic64_dec_return_release)
-+	return arch_atomic64_dec_return_release(v);
-+#elif defined(arch_atomic64_dec_return_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic64_dec_return_relaxed(v);
-+#elif defined(arch_atomic64_dec_return)
-+	return arch_atomic64_dec_return(v);
-+#else
-+	return raw_atomic64_sub_return_release(1, v);
-+#endif
- }
-#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
-+
-+/**
-+ * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
-+static __always_inline s64
-+raw_atomic64_dec_return_relaxed(atomic64_t *v)
-+{
-+#if defined(arch_atomic64_dec_return_relaxed)
-+	return arch_atomic64_dec_return_relaxed(v);
-+#elif defined(arch_atomic64_dec_return)
-+	return arch_atomic64_dec_return(v);
-+#else
-+	return raw_atomic64_sub_return_relaxed(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic64_dec_return
-+/**
-+ * raw_atomic64_fetch_dec() - atomic decrement with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_dec_return(atomic64_t *v)
-+raw_atomic64_fetch_dec(atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_dec)
-+	return arch_atomic64_fetch_dec(v);
-+#elif defined(arch_atomic64_fetch_dec_relaxed)
- 	s64 ret;
- 	__atomic_pre_full_fence();
-	ret = arch_atomic64_dec_return_relaxed(v);
-+	ret = arch_atomic64_fetch_dec_relaxed(v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic64_dec_return arch_atomic64_dec_return
-+#else
-+	return raw_atomic64_fetch_sub(1, v);
- #endif
-
-#endif /* arch_atomic64_dec_return_relaxed */
-
-#ifndef arch_atomic64_fetch_dec_relaxed
-#ifdef arch_atomic64_fetch_dec
-#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
-#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
-#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
-#endif /* arch_atomic64_fetch_dec */
-
-#ifndef arch_atomic64_fetch_dec
-static __always_inline s64
-arch_atomic64_fetch_dec(atomic64_t *v)
-{
-	return arch_atomic64_fetch_sub(1, v);
- }
-#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
-#endif
- 
-#ifndef arch_atomic64_fetch_dec_acquire
-+/**
-+ * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_dec_acquire(atomic64_t *v)
-+raw_atomic64_fetch_dec_acquire(atomic64_t *v)
- {
-	return arch_atomic64_fetch_sub_acquire(1, v);
-}
-#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
-+#if defined(arch_atomic64_fetch_dec_acquire)
-+	return arch_atomic64_fetch_dec_acquire(v);
-+#elif defined(arch_atomic64_fetch_dec_relaxed)
-+	s64 ret = arch_atomic64_fetch_dec_relaxed(v);
-+	__atomic_acquire_fence();
-+	return ret;
-+#elif defined(arch_atomic64_fetch_dec)
-+	return arch_atomic64_fetch_dec(v);
-+#else
-+	return raw_atomic64_fetch_sub_acquire(1, v);
- #endif
-
-#ifndef arch_atomic64_fetch_dec_release
-static __always_inline s64
-arch_atomic64_fetch_dec_release(atomic64_t *v)
-{
-	return arch_atomic64_fetch_sub_release(1, v);
- }
-#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
-#endif
- 
-#ifndef arch_atomic64_fetch_dec_relaxed
-+/**
-+ * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
-+raw_atomic64_fetch_dec_release(atomic64_t *v)
- {
-	return arch_atomic64_fetch_sub_relaxed(1, v);
-}
-#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
-+#if defined(arch_atomic64_fetch_dec_release)
-+	return arch_atomic64_fetch_dec_release(v);
-+#elif defined(arch_atomic64_fetch_dec_relaxed)
-+	__atomic_release_fence();
-+	return arch_atomic64_fetch_dec_relaxed(v);
-+#elif defined(arch_atomic64_fetch_dec)
-+	return arch_atomic64_fetch_dec(v);
-+#else
-+	return raw_atomic64_fetch_sub_release(1, v);
- #endif
-+}
- 
-#else /* arch_atomic64_fetch_dec_relaxed */
-
-#ifndef arch_atomic64_fetch_dec_acquire
-+/**
-+ * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_dec_acquire(atomic64_t *v)
-+raw_atomic64_fetch_dec_relaxed(atomic64_t *v)
- {
-	s64 ret = arch_atomic64_fetch_dec_relaxed(v);
-	__atomic_acquire_fence();
-	return ret;
-}
-#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
-+#if defined(arch_atomic64_fetch_dec_relaxed)
-+	return arch_atomic64_fetch_dec_relaxed(v);
-+#elif defined(arch_atomic64_fetch_dec)
-+	return arch_atomic64_fetch_dec(v);
-+#else
-+	return raw_atomic64_fetch_sub_relaxed(1, v);
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_dec_release
-static __always_inline s64
-arch_atomic64_fetch_dec_release(atomic64_t *v)
-+/**
-+ * raw_atomic64_and() - atomic bitwise AND with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_and() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic64_and(s64 i, atomic64_t *v)
- {
-	__atomic_release_fence();
-	return arch_atomic64_fetch_dec_relaxed(v);
-+	arch_atomic64_and(i, v);
- }
-#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
-#endif
- 
-#ifndef arch_atomic64_fetch_dec
-+/**
-+ * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_dec(atomic64_t *v)
-+raw_atomic64_fetch_and(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_and)
-+	return arch_atomic64_fetch_and(i, v);
-+#elif defined(arch_atomic64_fetch_and_relaxed)
- 	s64 ret;
- 	__atomic_pre_full_fence();
-	ret = arch_atomic64_fetch_dec_relaxed(v);
-+	ret = arch_atomic64_fetch_and_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
-+#else
-+#error "Unable to define raw_atomic64_fetch_and"
- #endif
-+}
- 
-#endif /* arch_atomic64_fetch_dec_relaxed */
-
-#ifndef arch_atomic64_fetch_and_relaxed
-#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
-#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
-#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
-#else /* arch_atomic64_fetch_and_relaxed */
-
-#ifndef arch_atomic64_fetch_and_acquire
-+/**
-+ * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_and_acquire)
-+	return arch_atomic64_fetch_and_acquire(i, v);
-+#elif defined(arch_atomic64_fetch_and_relaxed)
- 	s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
-+#elif defined(arch_atomic64_fetch_and)
-+	return arch_atomic64_fetch_and(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_and_acquire"
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_and_release
-+/**
-+ * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_and_release(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_and_release)
-+	return arch_atomic64_fetch_and_release(i, v);
-+#elif defined(arch_atomic64_fetch_and_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic64_fetch_and_relaxed(i, v);
-}
-#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
-+#elif defined(arch_atomic64_fetch_and)
-+	return arch_atomic64_fetch_and(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_and_release"
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_and
-+/**
-+ * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_and(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
- {
-	s64 ret;
-	__atomic_pre_full_fence();
-	ret = arch_atomic64_fetch_and_relaxed(i, v);
-	__atomic_post_full_fence();
-	return ret;
-}
-#define arch_atomic64_fetch_and arch_atomic64_fetch_and
-+#if defined(arch_atomic64_fetch_and_relaxed)
-+	return arch_atomic64_fetch_and_relaxed(i, v);
-+#elif defined(arch_atomic64_fetch_and)
-+	return arch_atomic64_fetch_and(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_and_relaxed"
- #endif
-
-#endif /* arch_atomic64_fetch_and_relaxed */
-
-#ifndef arch_atomic64_andnot
-static __always_inline void
-arch_atomic64_andnot(s64 i, atomic64_t *v)
-{
-	arch_atomic64_and(~i, v);
- }
-#define arch_atomic64_andnot arch_atomic64_andnot
-#endif
- 
-#ifndef arch_atomic64_fetch_andnot_relaxed
-#ifdef arch_atomic64_fetch_andnot
-#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
-#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
-#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
-#endif /* arch_atomic64_fetch_andnot */
-
-#ifndef arch_atomic64_fetch_andnot
-static __always_inline s64
-arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
-+/**
-+ * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic64_andnot(s64 i, atomic64_t *v)
- {
-	return arch_atomic64_fetch_and(~i, v);
-}
-#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
-+#if defined(arch_atomic64_andnot)
-+	arch_atomic64_andnot(i, v);
-+#else
-+	raw_atomic64_and(~i, v);
- #endif
-
-#ifndef arch_atomic64_fetch_andnot_acquire
-static __always_inline s64
-arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
-{
-	return arch_atomic64_fetch_and_acquire(~i, v);
- }
-#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
-#endif
- 
-#ifndef arch_atomic64_fetch_andnot_release
-+/**
-+ * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_andnot(s64 i, atomic64_t *v)
- {
-	return arch_atomic64_fetch_and_release(~i, v);
-}
-#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
-+#if defined(arch_atomic64_fetch_andnot)
-+	return arch_atomic64_fetch_andnot(i, v);
-+#elif defined(arch_atomic64_fetch_andnot_relaxed)
-+	s64 ret;
-+	__atomic_pre_full_fence();
-+	ret = arch_atomic64_fetch_andnot_relaxed(i, v);
-+	__atomic_post_full_fence();
-+	return ret;
-+#else
-+	return raw_atomic64_fetch_and(~i, v);
- #endif
-
-#ifndef arch_atomic64_fetch_andnot_relaxed
-static __always_inline s64
-arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
-{
-	return arch_atomic64_fetch_and_relaxed(~i, v);
- }
-#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
-#endif
-
-#else /* arch_atomic64_fetch_andnot_relaxed */
- 
-#ifndef arch_atomic64_fetch_andnot_acquire
-+/**
-+ * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_andnot_acquire)
-+	return arch_atomic64_fetch_andnot_acquire(i, v);
-+#elif defined(arch_atomic64_fetch_andnot_relaxed)
- 	s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
-+#elif defined(arch_atomic64_fetch_andnot)
-+	return arch_atomic64_fetch_andnot(i, v);
-+#else
-+	return raw_atomic64_fetch_and_acquire(~i, v);
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_andnot_release
-+/**
-+ * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_andnot_release)
-+	return arch_atomic64_fetch_andnot_release(i, v);
-+#elif defined(arch_atomic64_fetch_andnot_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic64_fetch_andnot_relaxed(i, v);
-+#elif defined(arch_atomic64_fetch_andnot)
-+	return arch_atomic64_fetch_andnot(i, v);
-+#else
-+	return raw_atomic64_fetch_and_release(~i, v);
-+#endif
- }
-#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
-+
-+/**
-+ * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
-+static __always_inline s64
-+raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
-+{
-+#if defined(arch_atomic64_fetch_andnot_relaxed)
-+	return arch_atomic64_fetch_andnot_relaxed(i, v);
-+#elif defined(arch_atomic64_fetch_andnot)
-+	return arch_atomic64_fetch_andnot(i, v);
-+#else
-+	return raw_atomic64_fetch_and_relaxed(~i, v);
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_andnot
-+/**
-+ * raw_atomic64_or() - atomic bitwise OR with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v | @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_or() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic64_or(s64 i, atomic64_t *v)
-+{
-+	arch_atomic64_or(i, v);
-+}
-+
-+/**
-+ * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v | @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_or(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_or)
-+	return arch_atomic64_fetch_or(i, v);
-+#elif defined(arch_atomic64_fetch_or_relaxed)
- 	s64 ret;
- 	__atomic_pre_full_fence();
-	ret = arch_atomic64_fetch_andnot_relaxed(i, v);
-+	ret = arch_atomic64_fetch_or_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
-+#else
-+#error "Unable to define raw_atomic64_fetch_or"
- #endif
-+}
- 
-#endif /* arch_atomic64_fetch_andnot_relaxed */
-
-#ifndef arch_atomic64_fetch_or_relaxed
-#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
-#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
-#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or
-#else /* arch_atomic64_fetch_or_relaxed */
-
-#ifndef arch_atomic64_fetch_or_acquire
-+/**
-+ * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v | @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_or_acquire)
-+	return arch_atomic64_fetch_or_acquire(i, v);
-+#elif defined(arch_atomic64_fetch_or_relaxed)
- 	s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
-+#elif defined(arch_atomic64_fetch_or)
-+	return arch_atomic64_fetch_or(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_or_acquire"
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_or_release
-+/**
-+ * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v | @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_or_release(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_or_release(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_or_release)
-+	return arch_atomic64_fetch_or_release(i, v);
-+#elif defined(arch_atomic64_fetch_or_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic64_fetch_or_relaxed(i, v);
-+#elif defined(arch_atomic64_fetch_or)
-+	return arch_atomic64_fetch_or(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_or_release"
-+#endif
- }
-#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
-+
-+/**
-+ * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v | @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
-+static __always_inline s64
-+raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
-+{
-+#if defined(arch_atomic64_fetch_or_relaxed)
-+	return arch_atomic64_fetch_or_relaxed(i, v);
-+#elif defined(arch_atomic64_fetch_or)
-+	return arch_atomic64_fetch_or(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_or_relaxed"
- #endif
-+}
-+
-+/**
-+ * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_xor() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
-+static __always_inline void
-+raw_atomic64_xor(s64 i, atomic64_t *v)
-+{
-+	arch_atomic64_xor(i, v);
-+}
- 
-#ifndef arch_atomic64_fetch_or
-+/**
-+ * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_or(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_xor(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_xor)
-+	return arch_atomic64_fetch_xor(i, v);
-+#elif defined(arch_atomic64_fetch_xor_relaxed)
- 	s64 ret;
- 	__atomic_pre_full_fence();
-	ret = arch_atomic64_fetch_or_relaxed(i, v);
-+	ret = arch_atomic64_fetch_xor_relaxed(i, v);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_or arch_atomic64_fetch_or
-+#else
-+#error "Unable to define raw_atomic64_fetch_xor"
- #endif
-+}
- 
-#endif /* arch_atomic64_fetch_or_relaxed */
-
-#ifndef arch_atomic64_fetch_xor_relaxed
-#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
-#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
-#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor
-#else /* arch_atomic64_fetch_xor_relaxed */
-
-#ifndef arch_atomic64_fetch_xor_acquire
-+/**
-+ * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_xor_acquire)
-+	return arch_atomic64_fetch_xor_acquire(i, v);
-+#elif defined(arch_atomic64_fetch_xor_relaxed)
- 	s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
-+#elif defined(arch_atomic64_fetch_xor)
-+	return arch_atomic64_fetch_xor(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_xor_acquire"
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_xor_release
-+/**
-+ * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
-+raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
- {
-+#if defined(arch_atomic64_fetch_xor_release)
-+	return arch_atomic64_fetch_xor_release(i, v);
-+#elif defined(arch_atomic64_fetch_xor_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic64_fetch_xor_relaxed(i, v);
-+#elif defined(arch_atomic64_fetch_xor)
-+	return arch_atomic64_fetch_xor(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_xor_release"
-+#endif
- }
-#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
-+
-+/**
-+ * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
-+static __always_inline s64
-+raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
-+{
-+#if defined(arch_atomic64_fetch_xor_relaxed)
-+	return arch_atomic64_fetch_xor_relaxed(i, v);
-+#elif defined(arch_atomic64_fetch_xor)
-+	return arch_atomic64_fetch_xor(i, v);
-+#else
-+#error "Unable to define raw_atomic64_fetch_xor_relaxed"
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_xor
-+/**
-+ * raw_atomic64_xchg() - atomic exchange with full ordering
-+ * @v: pointer to atomic64_t
-+ * @new: s64 value to assign
-+ *
-+ * Atomically updates @v to @new with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
-+raw_atomic64_xchg(atomic64_t *v, s64 new)
- {
-+#if defined(arch_atomic64_xchg)
-+	return arch_atomic64_xchg(v, new);
-+#elif defined(arch_atomic64_xchg_relaxed)
- 	s64 ret;
- 	__atomic_pre_full_fence();
-	ret = arch_atomic64_fetch_xor_relaxed(i, v);
-+	ret = arch_atomic64_xchg_relaxed(v, new);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
-+#else
-+	return raw_xchg(&v->counter, new);
- #endif
-+}
- 
-#endif /* arch_atomic64_fetch_xor_relaxed */
-
-#ifndef arch_atomic64_xchg_relaxed
-#define arch_atomic64_xchg_acquire arch_atomic64_xchg
-#define arch_atomic64_xchg_release arch_atomic64_xchg
-#define arch_atomic64_xchg_relaxed arch_atomic64_xchg
-#else /* arch_atomic64_xchg_relaxed */
-
-#ifndef arch_atomic64_xchg_acquire
-+/**
-+ * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering
-+ * @v: pointer to atomic64_t
-+ * @new: s64 value to assign
-+ *
-+ * Atomically updates @v to @new with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_xchg_acquire(atomic64_t *v, s64 i)
-+raw_atomic64_xchg_acquire(atomic64_t *v, s64 new)
- {
-	s64 ret = arch_atomic64_xchg_relaxed(v, i);
-+#if defined(arch_atomic64_xchg_acquire)
-+	return arch_atomic64_xchg_acquire(v, new);
-+#elif defined(arch_atomic64_xchg_relaxed)
-+	s64 ret = arch_atomic64_xchg_relaxed(v, new);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire
-+#elif defined(arch_atomic64_xchg)
-+	return arch_atomic64_xchg(v, new);
-+#else
-+	return raw_xchg_acquire(&v->counter, new);
- #endif
-+}
- 
-#ifndef arch_atomic64_xchg_release
-+/**
-+ * raw_atomic64_xchg_release() - atomic exchange with release ordering
-+ * @v: pointer to atomic64_t
-+ * @new: s64 value to assign
-+ *
-+ * Atomically updates @v to @new with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_xchg_release(atomic64_t *v, s64 i)
-+raw_atomic64_xchg_release(atomic64_t *v, s64 new)
- {
-+#if defined(arch_atomic64_xchg_release)
-+	return arch_atomic64_xchg_release(v, new);
-+#elif defined(arch_atomic64_xchg_relaxed)
- 	__atomic_release_fence();
-	return arch_atomic64_xchg_relaxed(v, i);
-+	return arch_atomic64_xchg_relaxed(v, new);
-+#elif defined(arch_atomic64_xchg)
-+	return arch_atomic64_xchg(v, new);
-+#else
-+	return raw_xchg_release(&v->counter, new);
-+#endif
- }
-#define arch_atomic64_xchg_release arch_atomic64_xchg_release
-+
-+/**
-+ * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ * @new: s64 value to assign
-+ *
-+ * Atomically updates @v to @new with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
-+static __always_inline s64
-+raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new)
-+{
-+#if defined(arch_atomic64_xchg_relaxed)
-+	return arch_atomic64_xchg_relaxed(v, new);
-+#elif defined(arch_atomic64_xchg)
-+	return arch_atomic64_xchg(v, new);
-+#else
-+	return raw_xchg_relaxed(&v->counter, new);
- #endif
-+}
- 
-#ifndef arch_atomic64_xchg
-+/**
-+ * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering
-+ * @v: pointer to atomic64_t
-+ * @old: s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_xchg(atomic64_t *v, s64 i)
-+raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
- {
-+#if defined(arch_atomic64_cmpxchg)
-+	return arch_atomic64_cmpxchg(v, old, new);
-+#elif defined(arch_atomic64_cmpxchg_relaxed)
- 	s64 ret;
- 	__atomic_pre_full_fence();
-	ret = arch_atomic64_xchg_relaxed(v, i);
-+	ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
- 	__atomic_post_full_fence();
- 	return ret;
-}
-#define arch_atomic64_xchg arch_atomic64_xchg
-+#else
-+	return raw_cmpxchg(&v->counter, old, new);
- #endif
-+}
- 
-#endif /* arch_atomic64_xchg_relaxed */
-
-#ifndef arch_atomic64_cmpxchg_relaxed
-#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg
-#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg
-#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg
-#else /* arch_atomic64_cmpxchg_relaxed */
-
-#ifndef arch_atomic64_cmpxchg_acquire
-+/**
-+ * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
-+ * @v: pointer to atomic64_t
-+ * @old: s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
-+raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
- {
-+#if defined(arch_atomic64_cmpxchg_acquire)
-+	return arch_atomic64_cmpxchg_acquire(v, old, new);
-+#elif defined(arch_atomic64_cmpxchg_relaxed)
- 	s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
- 	__atomic_acquire_fence();
- 	return ret;
-}
-#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire
-+#elif defined(arch_atomic64_cmpxchg)
-+	return arch_atomic64_cmpxchg(v, old, new);
-+#else
-+	return raw_cmpxchg_acquire(&v->counter, old, new);
- #endif
-+}
- 
-#ifndef arch_atomic64_cmpxchg_release
-+/**
-+ * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
-+ * @v: pointer to atomic64_t
-+ * @old: s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with release ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
-+raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
- {
-+#if defined(arch_atomic64_cmpxchg_release)
-+	return arch_atomic64_cmpxchg_release(v, old, new);
-+#elif defined(arch_atomic64_cmpxchg_relaxed)
- 	__atomic_release_fence();
- 	return arch_atomic64_cmpxchg_relaxed(v, old, new);
-}
-#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release
-+#elif defined(arch_atomic64_cmpxchg)
-+	return arch_atomic64_cmpxchg(v, old, new);
-+#else
-+	return raw_cmpxchg_release(&v->counter, old, new);
- #endif
-+}
- 
-#ifndef arch_atomic64_cmpxchg
-+/**
-+ * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ * @old: s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
-+raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
- {
-	s64 ret;
-	__atomic_pre_full_fence();
-	ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
-	__atomic_post_full_fence();
-	return ret;
-}
-#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
-+#if defined(arch_atomic64_cmpxchg_relaxed)
-+	return arch_atomic64_cmpxchg_relaxed(v, old, new);
-+#elif defined(arch_atomic64_cmpxchg)
-+	return arch_atomic64_cmpxchg(v, old, new);
-+#else
-+	return raw_cmpxchg_relaxed(&v->counter, old, new);
- #endif
-+}
- 
-#endif /* arch_atomic64_cmpxchg_relaxed */
-
-#ifndef arch_atomic64_try_cmpxchg_relaxed
-#ifdef arch_atomic64_try_cmpxchg
-#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg
-#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg
-#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg
-#endif /* arch_atomic64_try_cmpxchg */
-
-#ifndef arch_atomic64_try_cmpxchg
-+/**
-+ * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
-+ * @v: pointer to atomic64_t
-+ * @old: pointer to s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with full ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
-+raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
- {
-+#if defined(arch_atomic64_try_cmpxchg)
-+	return arch_atomic64_try_cmpxchg(v, old, new);
-+#elif defined(arch_atomic64_try_cmpxchg_relaxed)
-+	bool ret;
-+	__atomic_pre_full_fence();
-+	ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
-+	__atomic_post_full_fence();
-+	return ret;
-+#else
- 	s64 r, o = *old;
-	r = arch_atomic64_cmpxchg(v, o, new);
-+	r = raw_atomic64_cmpxchg(v, o, new);
- 	if (unlikely(r != o))
- 		*old = r;
- 	return likely(r == o);
-}
-#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
- #endif
-+}
- 
-#ifndef arch_atomic64_try_cmpxchg_acquire
-+/**
-+ * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
-+ * @v: pointer to atomic64_t
-+ * @old: pointer to s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
-+raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
- {
-+#if defined(arch_atomic64_try_cmpxchg_acquire)
-+	return arch_atomic64_try_cmpxchg_acquire(v, old, new);
-+#elif defined(arch_atomic64_try_cmpxchg_relaxed)
-+	bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
-+	__atomic_acquire_fence();
-+	return ret;
-+#elif defined(arch_atomic64_try_cmpxchg)
-+	return arch_atomic64_try_cmpxchg(v, old, new);
-+#else
- 	s64 r, o = *old;
-	r = arch_atomic64_cmpxchg_acquire(v, o, new);
-+	r = raw_atomic64_cmpxchg_acquire(v, o, new);
- 	if (unlikely(r != o))
- 		*old = r;
- 	return likely(r == o);
-}
-#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
- #endif
-+}
- 
-#ifndef arch_atomic64_try_cmpxchg_release
-+/**
-+ * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
-+ * @v: pointer to atomic64_t
-+ * @old: pointer to s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with release ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
-+raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
- {
-+#if defined(arch_atomic64_try_cmpxchg_release)
-+	return arch_atomic64_try_cmpxchg_release(v, old, new);
-+#elif defined(arch_atomic64_try_cmpxchg_relaxed)
-+	__atomic_release_fence();
-+	return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
-+#elif defined(arch_atomic64_try_cmpxchg)
-+	return arch_atomic64_try_cmpxchg(v, old, new);
-+#else
- 	s64 r, o = *old;
-	r = arch_atomic64_cmpxchg_release(v, o, new);
-+	r = raw_atomic64_cmpxchg_release(v, o, new);
- 	if (unlikely(r != o))
- 		*old = r;
- 	return likely(r == o);
-}
-#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
- #endif
-+}
- 
-#ifndef arch_atomic64_try_cmpxchg_relaxed
-+/**
-+ * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ * @old: pointer to s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
-+raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
- {
-+#if defined(arch_atomic64_try_cmpxchg_relaxed)
-+	return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
-+#elif defined(arch_atomic64_try_cmpxchg)
-+	return arch_atomic64_try_cmpxchg(v, old, new);
-+#else
- 	s64 r, o = *old;
-	r = arch_atomic64_cmpxchg_relaxed(v, o, new);
-+	r = raw_atomic64_cmpxchg_relaxed(v, o, new);
- 	if (unlikely(r != o))
- 		*old = r;
- 	return likely(r == o);
-}
-#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed
- #endif
-+}
- 
-#else /* arch_atomic64_try_cmpxchg_relaxed */
-
-#ifndef arch_atomic64_try_cmpxchg_acquire
-+/**
-+ * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
-+raw_atomic64_sub_and_test(s64 i, atomic64_t *v)
- {
-	bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
-	__atomic_acquire_fence();
-	return ret;
-}
-#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
-+#if defined(arch_atomic64_sub_and_test)
-+	return arch_atomic64_sub_and_test(i, v);
-+#else
-+	return raw_atomic64_sub_return(i, v) == 0;
- #endif
-+}
- 
-#ifndef arch_atomic64_try_cmpxchg_release
-+/**
-+ * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
-+raw_atomic64_dec_and_test(atomic64_t *v)
- {
-	__atomic_release_fence();
-	return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
-}
-#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
-+#if defined(arch_atomic64_dec_and_test)
-+	return arch_atomic64_dec_and_test(v);
-+#else
-+	return raw_atomic64_dec_return(v) == 0;
- #endif
-+}
- 
-#ifndef arch_atomic64_try_cmpxchg
-+/**
-+ * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
-+raw_atomic64_inc_and_test(atomic64_t *v)
- {
-	bool ret;
-	__atomic_pre_full_fence();
-	ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
-	__atomic_post_full_fence();
-	return ret;
-}
-#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
-+#if defined(arch_atomic64_inc_and_test)
-+	return arch_atomic64_inc_and_test(v);
-+#else
-+	return raw_atomic64_inc_return(v) == 0;
- #endif
-+}
- 
-#endif /* arch_atomic64_try_cmpxchg_relaxed */
-
-#ifndef arch_atomic64_sub_and_test
- /**
- * arch_atomic64_sub_and_test - subtract value from variable and test result
- * @i: integer value to subtract
- * @v: pointer of type atomic64_t
-+ * raw_atomic64_add_negative() - atomic add and test if negative with full ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-  *
- * Atomically subtracts @i from @v and returns
- * true if the result is zero, or false for all
- * other cases.
-+ * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-  */
- static __always_inline bool
-arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
-+raw_atomic64_add_negative(s64 i, atomic64_t *v)
- {
-	return arch_atomic64_sub_return(i, v) == 0;
-}
-#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
-+#if defined(arch_atomic64_add_negative)
-+	return arch_atomic64_add_negative(i, v);
-+#elif defined(arch_atomic64_add_negative_relaxed)
-+	bool ret;
-+	__atomic_pre_full_fence();
-+	ret = arch_atomic64_add_negative_relaxed(i, v);
-+	__atomic_post_full_fence();
-+	return ret;
-+#else
-+	return raw_atomic64_add_return(i, v) < 0;
- #endif
-+}
- 
-#ifndef arch_atomic64_dec_and_test
- /**
- * arch_atomic64_dec_and_test - decrement and test
- * @v: pointer of type atomic64_t
-+ * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-  *
- * Atomically decrements @v by 1 and
- * returns true if the result is 0, or false for all other
- * cases.
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-  */
- static __always_inline bool
-arch_atomic64_dec_and_test(atomic64_t *v)
-+raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
- {
-	return arch_atomic64_dec_return(v) == 0;
-}
-#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
-+#if defined(arch_atomic64_add_negative_acquire)
-+	return arch_atomic64_add_negative_acquire(i, v);
-+#elif defined(arch_atomic64_add_negative_relaxed)
-+	bool ret = arch_atomic64_add_negative_relaxed(i, v);
-+	__atomic_acquire_fence();
-+	return ret;
-+#elif defined(arch_atomic64_add_negative)
-+	return arch_atomic64_add_negative(i, v);
-+#else
-+	return raw_atomic64_add_return_acquire(i, v) < 0;
- #endif
-+}
- 
-#ifndef arch_atomic64_inc_and_test
- /**
- * arch_atomic64_inc_and_test - increment and test
- * @v: pointer of type atomic64_t
-+ * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-  *
- * Atomically increments @v by 1
- * and returns true if the result is zero, or false for all
- * other cases.
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-  */
- static __always_inline bool
-arch_atomic64_inc_and_test(atomic64_t *v)
-+raw_atomic64_add_negative_release(s64 i, atomic64_t *v)
- {
-	return arch_atomic64_inc_return(v) == 0;
-}
-#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
-+#if defined(arch_atomic64_add_negative_release)
-+	return arch_atomic64_add_negative_release(i, v);
-+#elif defined(arch_atomic64_add_negative_relaxed)
-+	__atomic_release_fence();
-+	return arch_atomic64_add_negative_relaxed(i, v);
-+#elif defined(arch_atomic64_add_negative)
-+	return arch_atomic64_add_negative(i, v);
-+#else
-+	return raw_atomic64_add_return_release(i, v) < 0;
- #endif
-+}
- 
-#ifndef arch_atomic64_add_negative
- /**
- * arch_atomic64_add_negative - add and test if negative
- * @i: integer value to add
- * @v: pointer of type atomic64_t
-+ * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere.
-  *
- * Atomically adds @i to @v and returns true
- * if the result is negative, or false when
- * result is greater than or equal to zero.
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-  */
- static __always_inline bool
-arch_atomic64_add_negative(s64 i, atomic64_t *v)
-+raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
- {
-	return arch_atomic64_add_return(i, v) < 0;
-}
-#define arch_atomic64_add_negative arch_atomic64_add_negative
-+#if defined(arch_atomic64_add_negative_relaxed)
-+	return arch_atomic64_add_negative_relaxed(i, v);
-+#elif defined(arch_atomic64_add_negative)
-+	return arch_atomic64_add_negative(i, v);
-+#else
-+	return raw_atomic64_add_return_relaxed(i, v) < 0;
- #endif
-+}
- 
-#ifndef arch_atomic64_fetch_add_unless
- /**
- * arch_atomic64_fetch_add_unless - add unless the number is already a given value
- * @v: pointer of type atomic64_t
- * @a: the amount to add to v...
- * @u: ...unless v is equal to u.
-+ * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering
-+ * @v: pointer to atomic64_t
-+ * @a: s64 value to add
-+ * @u: s64 value to compare with
-+ *
-+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-  *
- * Atomically adds @a to @v, so long as @v was not already @u.
- * Returns original value of @v
-+ * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere.
-+ *
-+ * Return: The original value of @v.
-  */
- static __always_inline s64
-arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
-+raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
- {
-	s64 c = arch_atomic64_read(v);
-+#if defined(arch_atomic64_fetch_add_unless)
-+	return arch_atomic64_fetch_add_unless(v, a, u);
-+#else
-+	s64 c = raw_atomic64_read(v);
- 
- 	do {
- 		if (unlikely(c == u))
- 			break;
-	} while (!arch_atomic64_try_cmpxchg(v, &c, c + a));
-+	} while (!raw_atomic64_try_cmpxchg(v, &c, c + a));
- 
- 	return c;
-}
-#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
- #endif
-+}
- 
-#ifndef arch_atomic64_add_unless
- /**
- * arch_atomic64_add_unless - add unless the number is already a given value
- * @v: pointer of type atomic64_t
- * @a: the amount to add to v...
- * @u: ...unless v is equal to u.
-+ * raw_atomic64_add_unless() - atomic add unless value with full ordering
-+ * @v: pointer to atomic64_t
-+ * @a: s64 value to add
-+ * @u: s64 value to compare with
-  *
- * Atomically adds @a to @v, if @v was not already @u.
- * Returns true if the addition was done.
-+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-  */
- static __always_inline bool
-arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
-+raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
- {
-	return arch_atomic64_fetch_add_unless(v, a, u) != u;
-}
-#define arch_atomic64_add_unless arch_atomic64_add_unless
-+#if defined(arch_atomic64_add_unless)
-+	return arch_atomic64_add_unless(v, a, u);
-+#else
-+	return raw_atomic64_fetch_add_unless(v, a, u) != u;
- #endif
-+}
- 
-#ifndef arch_atomic64_inc_not_zero
- /**
- * arch_atomic64_inc_not_zero - increment unless the number is zero
- * @v: pointer of type atomic64_t
-+ * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-  *
- * Atomically increments @v by 1, if @v is non-zero.
- * Returns true if the increment was done.
-+ * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-  */
- static __always_inline bool
-arch_atomic64_inc_not_zero(atomic64_t *v)
-+raw_atomic64_inc_not_zero(atomic64_t *v)
- {
-	return arch_atomic64_add_unless(v, 1, 0);
-}
-#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
-+#if defined(arch_atomic64_inc_not_zero)
-+	return arch_atomic64_inc_not_zero(v);
-+#else
-+	return raw_atomic64_add_unless(v, 1, 0);
- #endif
-+}
- 
-#ifndef arch_atomic64_inc_unless_negative
-+/**
-+ * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic64_inc_unless_negative(atomic64_t *v)
-+raw_atomic64_inc_unless_negative(atomic64_t *v)
- {
-	s64 c = arch_atomic64_read(v);
-+#if defined(arch_atomic64_inc_unless_negative)
-+	return arch_atomic64_inc_unless_negative(v);
-+#else
-+	s64 c = raw_atomic64_read(v);
- 
- 	do {
- 		if (unlikely(c < 0))
- 			return false;
-	} while (!arch_atomic64_try_cmpxchg(v, &c, c + 1));
-+	} while (!raw_atomic64_try_cmpxchg(v, &c, c + 1));
- 
- 	return true;
-}
-#define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
- #endif
-+}
- 
-#ifndef arch_atomic64_dec_unless_positive
-+/**
-+ * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic64_dec_unless_positive(atomic64_t *v)
-+raw_atomic64_dec_unless_positive(atomic64_t *v)
- {
-	s64 c = arch_atomic64_read(v);
-+#if defined(arch_atomic64_dec_unless_positive)
-+	return arch_atomic64_dec_unless_positive(v);
-+#else
-+	s64 c = raw_atomic64_read(v);
- 
- 	do {
- 		if (unlikely(c > 0))
- 			return false;
-	} while (!arch_atomic64_try_cmpxchg(v, &c, c - 1));
-+	} while (!raw_atomic64_try_cmpxchg(v, &c, c - 1));
- 
- 	return true;
-}
-#define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
- #endif
-+}
- 
-#ifndef arch_atomic64_dec_if_positive
-+/**
-+ * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere.
-+ *
-+ * Return: The old value of (@v - 1), regardless of whether @v was updated.
-+ */
- static __always_inline s64
-arch_atomic64_dec_if_positive(atomic64_t *v)
-+raw_atomic64_dec_if_positive(atomic64_t *v)
- {
-	s64 dec, c = arch_atomic64_read(v);
-+#if defined(arch_atomic64_dec_if_positive)
-+	return arch_atomic64_dec_if_positive(v);
-+#else
-+	s64 dec, c = raw_atomic64_read(v);
- 
- 	do {
- 		dec = c - 1;
- 		if (unlikely(dec < 0))
- 			break;
-	} while (!arch_atomic64_try_cmpxchg(v, &c, dec));
-+	} while (!raw_atomic64_try_cmpxchg(v, &c, dec));
- 
- 	return dec;
-}
-#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
- #endif
-+}
- 
- #endif /* _LINUX_ATOMIC_FALLBACK_H */
-// b5e87bdd5ede61470c29f7a7e4de781af3770f09
-+// b565db590afeeff0d7c9485ccbca5bb6e155749f
-diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h
-index 7a139ec030b0..37ab6314a9f7 100644
--- a/include/linux/atomic/atomic-instrumented.h
-+++ b/include/linux/atomic/atomic-instrumented.h
-@@ -4,15 +4,10 @@
- // DO NOT MODIFY THIS FILE DIRECTLY
- 
- /*
- * This file provides wrappers with KASAN instrumentation for atomic operations.
- * To use this functionality an arch's atomic.h file needs to define all
- * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
- * this file at the end. This file provides atomic_read() that forwards to
- * arch_atomic_read() for actual atomic operation.
- * Note: if an arch atomic operation is implemented by means of other atomic
- * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
- * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
- * double instrumentation.
-+ * This file provoides atomic operations with explicit instrumentation (e.g.
-+ * KASAN, KCSAN), which should be used unless it is necessary to avoid
-+ * instrumentation. Where it is necessary to aovid instrumenation, the
-+ * raw_atomic*() operations should be used.
-  */
- #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
- #define _LINUX_ATOMIC_INSTRUMENTED_H
-@@ -21,1951 +16,4858 @@
- #include <linux/compiler.h>
- #include <linux/instrumented.h>
- 
-+/**
-+ * atomic_read() - atomic load with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically loads the value of @v with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_read() there.
-+ *
-+ * Return: The value loaded from @v.
-+ */
- static __always_inline int
- atomic_read(const atomic_t *v)
- {
- 	instrument_atomic_read(v, sizeof(*v));
-	return arch_atomic_read(v);
-}
-
-+	return raw_atomic_read(v);
-+}
-+
-+/**
-+ * atomic_read_acquire() - atomic load with acquire ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically loads the value of @v with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_read_acquire() there.
-+ *
-+ * Return: The value loaded from @v.
-+ */
- static __always_inline int
- atomic_read_acquire(const atomic_t *v)
- {
- 	instrument_atomic_read(v, sizeof(*v));
-	return arch_atomic_read_acquire(v);
-}
-
-+	return raw_atomic_read_acquire(v);
-+}
-+
-+/**
-+ * atomic_set() - atomic set with relaxed ordering
-+ * @v: pointer to atomic_t
-+ * @i: int value to assign
-+ *
-+ * Atomically sets @v to @i with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_set() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_set(atomic_t *v, int i)
- {
- 	instrument_atomic_write(v, sizeof(*v));
-	arch_atomic_set(v, i);
-}
-
-+	raw_atomic_set(v, i);
-+}
-+
-+/**
-+ * atomic_set_release() - atomic set with release ordering
-+ * @v: pointer to atomic_t
-+ * @i: int value to assign
-+ *
-+ * Atomically sets @v to @i with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_set_release() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_set_release(atomic_t *v, int i)
- {
- 	kcsan_release();
- 	instrument_atomic_write(v, sizeof(*v));
-	arch_atomic_set_release(v, i);
-}
-
-+	raw_atomic_set_release(v, i);
-+}
-+
-+/**
-+ * atomic_add() - atomic add with relaxed ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_add() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_add(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_add(i, v);
-+	raw_atomic_add(i, v);
- }
- 
-+/**
-+ * atomic_add_return() - atomic add with full ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_add_return() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_add_return(int i, atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_add_return(i, v);
-+	return raw_atomic_add_return(i, v);
- }
- 
-+/**
-+ * atomic_add_return_acquire() - atomic add with acquire ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_add_return_acquire() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_add_return_acquire(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_add_return_acquire(i, v);
-+	return raw_atomic_add_return_acquire(i, v);
- }
- 
-+/**
-+ * atomic_add_return_release() - atomic add with release ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_add_return_release() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_add_return_release(int i, atomic_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_add_return_release(i, v);
-+	return raw_atomic_add_return_release(i, v);
- }
- 
-+/**
-+ * atomic_add_return_relaxed() - atomic add with relaxed ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_add_return_relaxed() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_add_return_relaxed(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_add_return_relaxed(i, v);
-+	return raw_atomic_add_return_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_fetch_add() - atomic add with full ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_add() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_add(int i, atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_add(i, v);
-+	return raw_atomic_fetch_add(i, v);
- }
- 
-+/**
-+ * atomic_fetch_add_acquire() - atomic add with acquire ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_add_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_add_acquire(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_add_acquire(i, v);
-+	return raw_atomic_fetch_add_acquire(i, v);
- }
- 
-+/**
-+ * atomic_fetch_add_release() - atomic add with release ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_add_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_add_release(int i, atomic_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_add_release(i, v);
-+	return raw_atomic_fetch_add_release(i, v);
- }
- 
-+/**
-+ * atomic_fetch_add_relaxed() - atomic add with relaxed ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_add_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_add_relaxed(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_add_relaxed(i, v);
-+	return raw_atomic_fetch_add_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_sub() - atomic subtract with relaxed ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_sub() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_sub(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_sub(i, v);
-+	raw_atomic_sub(i, v);
- }
- 
-+/**
-+ * atomic_sub_return() - atomic subtract with full ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_sub_return() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_sub_return(int i, atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_sub_return(i, v);
-+	return raw_atomic_sub_return(i, v);
- }
- 
-+/**
-+ * atomic_sub_return_acquire() - atomic subtract with acquire ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_sub_return_acquire() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_sub_return_acquire(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_sub_return_acquire(i, v);
-+	return raw_atomic_sub_return_acquire(i, v);
- }
- 
-+/**
-+ * atomic_sub_return_release() - atomic subtract with release ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_sub_return_release() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_sub_return_release(int i, atomic_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_sub_return_release(i, v);
-+	return raw_atomic_sub_return_release(i, v);
- }
- 
-+/**
-+ * atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_sub_return_relaxed() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_sub_return_relaxed(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_sub_return_relaxed(i, v);
-+	return raw_atomic_sub_return_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_fetch_sub() - atomic subtract with full ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_sub() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_sub(int i, atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_sub(i, v);
-+	return raw_atomic_fetch_sub(i, v);
- }
- 
-+/**
-+ * atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_sub_acquire(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_sub_acquire(i, v);
-+	return raw_atomic_fetch_sub_acquire(i, v);
- }
- 
-+/**
-+ * atomic_fetch_sub_release() - atomic subtract with release ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_sub_release(int i, atomic_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_sub_release(i, v);
-+	return raw_atomic_fetch_sub_release(i, v);
- }
- 
-+/**
-+ * atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_sub_relaxed(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_sub_relaxed(i, v);
-+	return raw_atomic_fetch_sub_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_inc() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_inc() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_inc(atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_inc(v);
-+	raw_atomic_inc(v);
- }
- 
-+/**
-+ * atomic_inc_return() - atomic increment with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_inc_return() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_inc_return(atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_inc_return(v);
-+	return raw_atomic_inc_return(v);
- }
- 
-+/**
-+ * atomic_inc_return_acquire() - atomic increment with acquire ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_inc_return_acquire() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_inc_return_acquire(atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_inc_return_acquire(v);
-+	return raw_atomic_inc_return_acquire(v);
- }
- 
-+/**
-+ * atomic_inc_return_release() - atomic increment with release ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_inc_return_release() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_inc_return_release(atomic_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_inc_return_release(v);
-+	return raw_atomic_inc_return_release(v);
- }
- 
-+/**
-+ * atomic_inc_return_relaxed() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_inc_return_relaxed() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_inc_return_relaxed(atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_inc_return_relaxed(v);
-+	return raw_atomic_inc_return_relaxed(v);
- }
- 
-+/**
-+ * atomic_fetch_inc() - atomic increment with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_inc() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_inc(atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_inc(v);
-+	return raw_atomic_fetch_inc(v);
- }
- 
-+/**
-+ * atomic_fetch_inc_acquire() - atomic increment with acquire ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_inc_acquire(atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_inc_acquire(v);
-+	return raw_atomic_fetch_inc_acquire(v);
- }
- 
-+/**
-+ * atomic_fetch_inc_release() - atomic increment with release ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_inc_release(atomic_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_inc_release(v);
-+	return raw_atomic_fetch_inc_release(v);
- }
- 
-+/**
-+ * atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_inc_relaxed(atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_inc_relaxed(v);
-+	return raw_atomic_fetch_inc_relaxed(v);
- }
- 
-+/**
-+ * atomic_dec() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_dec() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_dec(atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_dec(v);
-+	raw_atomic_dec(v);
- }
- 
-+/**
-+ * atomic_dec_return() - atomic decrement with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_dec_return() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_dec_return(atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_dec_return(v);
-+	return raw_atomic_dec_return(v);
- }
- 
-+/**
-+ * atomic_dec_return_acquire() - atomic decrement with acquire ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_dec_return_acquire() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_dec_return_acquire(atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_dec_return_acquire(v);
-+	return raw_atomic_dec_return_acquire(v);
- }
- 
-+/**
-+ * atomic_dec_return_release() - atomic decrement with release ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_dec_return_release() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_dec_return_release(atomic_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_dec_return_release(v);
-+	return raw_atomic_dec_return_release(v);
- }
- 
-+/**
-+ * atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_dec_return_relaxed() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline int
- atomic_dec_return_relaxed(atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_dec_return_relaxed(v);
-+	return raw_atomic_dec_return_relaxed(v);
- }
- 
-+/**
-+ * atomic_fetch_dec() - atomic decrement with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_dec() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_dec(atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_dec(v);
-+	return raw_atomic_fetch_dec(v);
- }
- 
-+/**
-+ * atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_dec_acquire(atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_dec_acquire(v);
-+	return raw_atomic_fetch_dec_acquire(v);
- }
- 
-+/**
-+ * atomic_fetch_dec_release() - atomic decrement with release ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_dec_release(atomic_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_dec_release(v);
-+	return raw_atomic_fetch_dec_release(v);
- }
- 
-+/**
-+ * atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_dec_relaxed(atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_dec_relaxed(v);
-+	return raw_atomic_fetch_dec_relaxed(v);
- }
- 
-+/**
-+ * atomic_and() - atomic bitwise AND with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_and() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_and(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_and(i, v);
-+	raw_atomic_and(i, v);
- }
- 
-+/**
-+ * atomic_fetch_and() - atomic bitwise AND with full ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_and() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_and(int i, atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_and(i, v);
-+	return raw_atomic_fetch_and(i, v);
- }
- 
-+/**
-+ * atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_and_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_and_acquire(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_and_acquire(i, v);
-+	return raw_atomic_fetch_and_acquire(i, v);
- }
- 
-+/**
-+ * atomic_fetch_and_release() - atomic bitwise AND with release ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_and_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_and_release(int i, atomic_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_and_release(i, v);
-+	return raw_atomic_fetch_and_release(i, v);
- }
- 
-+/**
-+ * atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_and_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_and_relaxed(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_and_relaxed(i, v);
-+	return raw_atomic_fetch_and_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_andnot() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_andnot(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_andnot(i, v);
-+	raw_atomic_andnot(i, v);
- }
- 
-+/**
-+ * atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_andnot(int i, atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_andnot(i, v);
-+	return raw_atomic_fetch_andnot(i, v);
- }
- 
-+/**
-+ * atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_andnot_acquire(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_andnot_acquire(i, v);
-+	return raw_atomic_fetch_andnot_acquire(i, v);
- }
- 
-+/**
-+ * atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_andnot_release(int i, atomic_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_andnot_release(i, v);
-+	return raw_atomic_fetch_andnot_release(i, v);
- }
- 
-+/**
-+ * atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_andnot_relaxed(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_andnot_relaxed(i, v);
-+	return raw_atomic_fetch_andnot_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_or() - atomic bitwise OR with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v | @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_or() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_or(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_or(i, v);
-+	raw_atomic_or(i, v);
- }
- 
-+/**
-+ * atomic_fetch_or() - atomic bitwise OR with full ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v | @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_or() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_or(int i, atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_or(i, v);
-+	return raw_atomic_fetch_or(i, v);
- }
- 
-+/**
-+ * atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v | @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_or_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_or_acquire(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_or_acquire(i, v);
-+	return raw_atomic_fetch_or_acquire(i, v);
- }
- 
-+/**
-+ * atomic_fetch_or_release() - atomic bitwise OR with release ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v | @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_or_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_or_release(int i, atomic_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_or_release(i, v);
-+	return raw_atomic_fetch_or_release(i, v);
- }
- 
-+/**
-+ * atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v | @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_or_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_or_relaxed(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_or_relaxed(i, v);
-+	return raw_atomic_fetch_or_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_xor() - atomic bitwise XOR with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_xor() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_xor(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_xor(i, v);
-+	raw_atomic_xor(i, v);
- }
- 
-+/**
-+ * atomic_fetch_xor() - atomic bitwise XOR with full ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_xor() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_xor(int i, atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_xor(i, v);
-+	return raw_atomic_fetch_xor(i, v);
- }
- 
-+/**
-+ * atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_xor_acquire(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_xor_acquire(i, v);
-+	return raw_atomic_fetch_xor_acquire(i, v);
- }
- 
-+/**
-+ * atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_xor_release(int i, atomic_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_xor_release(i, v);
-+	return raw_atomic_fetch_xor_release(i, v);
- }
- 
-+/**
-+ * atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
-+ * @i: int value
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_xor_relaxed(int i, atomic_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_xor_relaxed(i, v);
-+	return raw_atomic_fetch_xor_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_xchg() - atomic exchange with full ordering
-+ * @v: pointer to atomic_t
-+ * @new: int value to assign
-+ *
-+ * Atomically updates @v to @new with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_xchg() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-atomic_xchg(atomic_t *v, int i)
-+atomic_xchg(atomic_t *v, int new)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_xchg(v, i);
-+	return raw_atomic_xchg(v, new);
- }
- 
-+/**
-+ * atomic_xchg_acquire() - atomic exchange with acquire ordering
-+ * @v: pointer to atomic_t
-+ * @new: int value to assign
-+ *
-+ * Atomically updates @v to @new with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_xchg_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-atomic_xchg_acquire(atomic_t *v, int i)
-+atomic_xchg_acquire(atomic_t *v, int new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_xchg_acquire(v, i);
-+	return raw_atomic_xchg_acquire(v, new);
- }
- 
-+/**
-+ * atomic_xchg_release() - atomic exchange with release ordering
-+ * @v: pointer to atomic_t
-+ * @new: int value to assign
-+ *
-+ * Atomically updates @v to @new with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_xchg_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-atomic_xchg_release(atomic_t *v, int i)
-+atomic_xchg_release(atomic_t *v, int new)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_xchg_release(v, i);
-+	return raw_atomic_xchg_release(v, new);
- }
- 
-+/**
-+ * atomic_xchg_relaxed() - atomic exchange with relaxed ordering
-+ * @v: pointer to atomic_t
-+ * @new: int value to assign
-+ *
-+ * Atomically updates @v to @new with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_xchg_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
-atomic_xchg_relaxed(atomic_t *v, int i)
-+atomic_xchg_relaxed(atomic_t *v, int new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_xchg_relaxed(v, i);
-+	return raw_atomic_xchg_relaxed(v, new);
- }
- 
-+/**
-+ * atomic_cmpxchg() - atomic compare and exchange with full ordering
-+ * @v: pointer to atomic_t
-+ * @old: int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_cmpxchg() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_cmpxchg(atomic_t *v, int old, int new)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_cmpxchg(v, old, new);
-}
-
-+	return raw_atomic_cmpxchg(v, old, new);
-+}
-+
-+/**
-+ * atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
-+ * @v: pointer to atomic_t
-+ * @old: int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_cmpxchg_acquire(v, old, new);
-+	return raw_atomic_cmpxchg_acquire(v, old, new);
- }
- 
-+/**
-+ * atomic_cmpxchg_release() - atomic compare and exchange with release ordering
-+ * @v: pointer to atomic_t
-+ * @old: int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with release ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_cmpxchg_release(atomic_t *v, int old, int new)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_cmpxchg_release(v, old, new);
-}
-
-+	return raw_atomic_cmpxchg_release(v, old, new);
-+}
-+
-+/**
-+ * atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
-+ * @v: pointer to atomic_t
-+ * @old: int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_cmpxchg_relaxed(v, old, new);
-+	return raw_atomic_cmpxchg_relaxed(v, old, new);
- }
- 
-+/**
-+ * atomic_try_cmpxchg() - atomic compare and exchange with full ordering
-+ * @v: pointer to atomic_t
-+ * @old: pointer to int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with full ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
- atomic_try_cmpxchg(atomic_t *v, int *old, int new)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	instrument_atomic_read_write(old, sizeof(*old));
-	return arch_atomic_try_cmpxchg(v, old, new);
-}
-
-+	instrument_read_write(old, sizeof(*old));
-+	return raw_atomic_try_cmpxchg(v, old, new);
-+}
-+
-+/**
-+ * atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
-+ * @v: pointer to atomic_t
-+ * @old: pointer to int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
- atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	instrument_atomic_read_write(old, sizeof(*old));
-	return arch_atomic_try_cmpxchg_acquire(v, old, new);
-+	instrument_read_write(old, sizeof(*old));
-+	return raw_atomic_try_cmpxchg_acquire(v, old, new);
- }
- 
-+/**
-+ * atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
-+ * @v: pointer to atomic_t
-+ * @old: pointer to int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with release ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
- atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	instrument_atomic_read_write(old, sizeof(*old));
-	return arch_atomic_try_cmpxchg_release(v, old, new);
-}
-
-+	instrument_read_write(old, sizeof(*old));
-+	return raw_atomic_try_cmpxchg_release(v, old, new);
-+}
-+
-+/**
-+ * atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
-+ * @v: pointer to atomic_t
-+ * @old: pointer to int value to compare with
-+ * @new: int value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
- atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	instrument_atomic_read_write(old, sizeof(*old));
-	return arch_atomic_try_cmpxchg_relaxed(v, old, new);
-+	instrument_read_write(old, sizeof(*old));
-+	return raw_atomic_try_cmpxchg_relaxed(v, old, new);
- }
- 
-+/**
-+ * atomic_sub_and_test() - atomic subtract and test if zero with full ordering
-+ * @i: int value to subtract
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_sub_and_test() there.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
- atomic_sub_and_test(int i, atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_sub_and_test(i, v);
-+	return raw_atomic_sub_and_test(i, v);
- }
- 
-+/**
-+ * atomic_dec_and_test() - atomic decrement and test if zero with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_dec_and_test() there.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
- atomic_dec_and_test(atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_dec_and_test(v);
-+	return raw_atomic_dec_and_test(v);
- }
- 
-+/**
-+ * atomic_inc_and_test() - atomic increment and test if zero with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_inc_and_test() there.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
- atomic_inc_and_test(atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_inc_and_test(v);
-+	return raw_atomic_inc_and_test(v);
- }
- 
-+/**
-+ * atomic_add_negative() - atomic add and test if negative with full ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_add_negative() there.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
- static __always_inline bool
- atomic_add_negative(int i, atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_add_negative(i, v);
-+	return raw_atomic_add_negative(i, v);
-+}
-+
-+/**
-+ * atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_add_negative_acquire() there.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
-+static __always_inline bool
-+atomic_add_negative_acquire(int i, atomic_t *v)
-+{
-+	instrument_atomic_read_write(v, sizeof(*v));
-+	return raw_atomic_add_negative_acquire(i, v);
- }
- 
-+/**
-+ * atomic_add_negative_release() - atomic add and test if negative with release ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_add_negative_release() there.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
-+static __always_inline bool
-+atomic_add_negative_release(int i, atomic_t *v)
-+{
-+	kcsan_release();
-+	instrument_atomic_read_write(v, sizeof(*v));
-+	return raw_atomic_add_negative_release(i, v);
-+}
-+
-+/**
-+ * atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
-+ * @i: int value to add
-+ * @v: pointer to atomic_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_add_negative_relaxed() there.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
-+static __always_inline bool
-+atomic_add_negative_relaxed(int i, atomic_t *v)
-+{
-+	instrument_atomic_read_write(v, sizeof(*v));
-+	return raw_atomic_add_negative_relaxed(i, v);
-+}
-+
-+/**
-+ * atomic_fetch_add_unless() - atomic add unless value with full ordering
-+ * @v: pointer to atomic_t
-+ * @a: int value to add
-+ * @u: int value to compare with
-+ *
-+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_fetch_add_unless() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline int
- atomic_fetch_add_unless(atomic_t *v, int a, int u)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_fetch_add_unless(v, a, u);
-}
-
-+	return raw_atomic_fetch_add_unless(v, a, u);
-+}
-+
-+/**
-+ * atomic_add_unless() - atomic add unless value with full ordering
-+ * @v: pointer to atomic_t
-+ * @a: int value to add
-+ * @u: int value to compare with
-+ *
-+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_add_unless() there.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
- atomic_add_unless(atomic_t *v, int a, int u)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_add_unless(v, a, u);
-+	return raw_atomic_add_unless(v, a, u);
- }
- 
-+/**
-+ * atomic_inc_not_zero() - atomic increment unless zero with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_inc_not_zero() there.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
- atomic_inc_not_zero(atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_inc_not_zero(v);
-+	return raw_atomic_inc_not_zero(v);
- }
- 
-+/**
-+ * atomic_inc_unless_negative() - atomic increment unless negative with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_inc_unless_negative() there.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
- atomic_inc_unless_negative(atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_inc_unless_negative(v);
-+	return raw_atomic_inc_unless_negative(v);
- }
- 
-+/**
-+ * atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_dec_unless_positive() there.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
- atomic_dec_unless_positive(atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_dec_unless_positive(v);
-+	return raw_atomic_dec_unless_positive(v);
- }
- 
-+/**
-+ * atomic_dec_if_positive() - atomic decrement if positive with full ordering
-+ * @v: pointer to atomic_t
-+ *
-+ * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there.
-+ *
-+ * Return: The old value of (@v - 1), regardless of whether @v was updated.
-+ */
- static __always_inline int
- atomic_dec_if_positive(atomic_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_dec_if_positive(v);
-+	return raw_atomic_dec_if_positive(v);
- }
- 
-+/**
-+ * atomic64_read() - atomic load with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically loads the value of @v with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_read() there.
-+ *
-+ * Return: The value loaded from @v.
-+ */
- static __always_inline s64
- atomic64_read(const atomic64_t *v)
- {
- 	instrument_atomic_read(v, sizeof(*v));
-	return arch_atomic64_read(v);
-}
-
-+	return raw_atomic64_read(v);
-+}
-+
-+/**
-+ * atomic64_read_acquire() - atomic load with acquire ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically loads the value of @v with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_read_acquire() there.
-+ *
-+ * Return: The value loaded from @v.
-+ */
- static __always_inline s64
- atomic64_read_acquire(const atomic64_t *v)
- {
- 	instrument_atomic_read(v, sizeof(*v));
-	return arch_atomic64_read_acquire(v);
-}
-
-+	return raw_atomic64_read_acquire(v);
-+}
-+
-+/**
-+ * atomic64_set() - atomic set with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ * @i: s64 value to assign
-+ *
-+ * Atomically sets @v to @i with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_set() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic64_set(atomic64_t *v, s64 i)
- {
- 	instrument_atomic_write(v, sizeof(*v));
-	arch_atomic64_set(v, i);
-}
-
-+	raw_atomic64_set(v, i);
-+}
-+
-+/**
-+ * atomic64_set_release() - atomic set with release ordering
-+ * @v: pointer to atomic64_t
-+ * @i: s64 value to assign
-+ *
-+ * Atomically sets @v to @i with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_set_release() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic64_set_release(atomic64_t *v, s64 i)
- {
- 	kcsan_release();
- 	instrument_atomic_write(v, sizeof(*v));
-	arch_atomic64_set_release(v, i);
-}
-
-+	raw_atomic64_set_release(v, i);
-+}
-+
-+/**
-+ * atomic64_add() - atomic add with relaxed ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_add() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic64_add(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic64_add(i, v);
-+	raw_atomic64_add(i, v);
- }
- 
-+/**
-+ * atomic64_add_return() - atomic add with full ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_add_return() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_add_return(s64 i, atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_add_return(i, v);
-+	return raw_atomic64_add_return(i, v);
- }
- 
-+/**
-+ * atomic64_add_return_acquire() - atomic add with acquire ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_add_return_acquire() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_add_return_acquire(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_add_return_acquire(i, v);
-+	return raw_atomic64_add_return_acquire(i, v);
- }
- 
-+/**
-+ * atomic64_add_return_release() - atomic add with release ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_add_return_release() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_add_return_release(s64 i, atomic64_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_add_return_release(i, v);
-+	return raw_atomic64_add_return_release(i, v);
- }
- 
-+/**
-+ * atomic64_add_return_relaxed() - atomic add with relaxed ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_add_return_relaxed() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_add_return_relaxed(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_add_return_relaxed(i, v);
-+	return raw_atomic64_add_return_relaxed(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_add() - atomic add with full ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_add() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_add(s64 i, atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_add(i, v);
-+	return raw_atomic64_fetch_add(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_add_acquire() - atomic add with acquire ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_add_acquire(i, v);
-+	return raw_atomic64_fetch_add_acquire(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_add_release() - atomic add with release ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_add_release(s64 i, atomic64_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_add_release(i, v);
-+	return raw_atomic64_fetch_add_release(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_add_relaxed(i, v);
-+	return raw_atomic64_fetch_add_relaxed(i, v);
- }
- 
-+/**
-+ * atomic64_sub() - atomic subtract with relaxed ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_sub() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic64_sub(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic64_sub(i, v);
-+	raw_atomic64_sub(i, v);
- }
- 
-+/**
-+ * atomic64_sub_return() - atomic subtract with full ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_sub_return() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_sub_return(s64 i, atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_sub_return(i, v);
-+	return raw_atomic64_sub_return(i, v);
- }
- 
-+/**
-+ * atomic64_sub_return_acquire() - atomic subtract with acquire ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_sub_return_acquire() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_sub_return_acquire(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_sub_return_acquire(i, v);
-+	return raw_atomic64_sub_return_acquire(i, v);
- }
- 
-+/**
-+ * atomic64_sub_return_release() - atomic subtract with release ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_sub_return_release() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_sub_return_release(s64 i, atomic64_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_sub_return_release(i, v);
-+	return raw_atomic64_sub_return_release(i, v);
- }
- 
-+/**
-+ * atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_sub_return_relaxed() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_sub_return_relaxed(i, v);
-+	return raw_atomic64_sub_return_relaxed(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_sub() - atomic subtract with full ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_sub(s64 i, atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_sub(i, v);
-+	return raw_atomic64_fetch_sub(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_sub_acquire(i, v);
-+	return raw_atomic64_fetch_sub_acquire(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_sub_release() - atomic subtract with release ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_sub_release(s64 i, atomic64_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_sub_release(i, v);
-+	return raw_atomic64_fetch_sub_release(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_sub_relaxed(i, v);
-+	return raw_atomic64_fetch_sub_relaxed(i, v);
- }
- 
-+/**
-+ * atomic64_inc() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_inc() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic64_inc(atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic64_inc(v);
-+	raw_atomic64_inc(v);
- }
- 
-+/**
-+ * atomic64_inc_return() - atomic increment with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_inc_return() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_inc_return(atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_inc_return(v);
-+	return raw_atomic64_inc_return(v);
- }
- 
-+/**
-+ * atomic64_inc_return_acquire() - atomic increment with acquire ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_inc_return_acquire() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_inc_return_acquire(atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_inc_return_acquire(v);
-+	return raw_atomic64_inc_return_acquire(v);
- }
- 
-+/**
-+ * atomic64_inc_return_release() - atomic increment with release ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_inc_return_release() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_inc_return_release(atomic64_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_inc_return_release(v);
-+	return raw_atomic64_inc_return_release(v);
- }
- 
-+/**
-+ * atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_inc_return_relaxed() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_inc_return_relaxed(atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_inc_return_relaxed(v);
-+	return raw_atomic64_inc_return_relaxed(v);
- }
- 
-+/**
-+ * atomic64_fetch_inc() - atomic increment with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_inc(atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_inc(v);
-+	return raw_atomic64_fetch_inc(v);
- }
- 
-+/**
-+ * atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_inc_acquire(atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_inc_acquire(v);
-+	return raw_atomic64_fetch_inc_acquire(v);
- }
- 
-+/**
-+ * atomic64_fetch_inc_release() - atomic increment with release ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_inc_release(atomic64_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_inc_release(v);
-+	return raw_atomic64_fetch_inc_release(v);
- }
- 
-+/**
-+ * atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_inc_relaxed(atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_inc_relaxed(v);
-+	return raw_atomic64_fetch_inc_relaxed(v);
- }
- 
-+/**
-+ * atomic64_dec() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_dec() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic64_dec(atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic64_dec(v);
-+	raw_atomic64_dec(v);
- }
- 
-+/**
-+ * atomic64_dec_return() - atomic decrement with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_dec_return() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_dec_return(atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_dec_return(v);
-+	return raw_atomic64_dec_return(v);
- }
- 
-+/**
-+ * atomic64_dec_return_acquire() - atomic decrement with acquire ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_dec_return_acquire() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_dec_return_acquire(atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_dec_return_acquire(v);
-+	return raw_atomic64_dec_return_acquire(v);
- }
- 
-+/**
-+ * atomic64_dec_return_release() - atomic decrement with release ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_dec_return_release() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_dec_return_release(atomic64_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_dec_return_release(v);
-+	return raw_atomic64_dec_return_release(v);
- }
- 
-+/**
-+ * atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_dec_return_relaxed() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline s64
- atomic64_dec_return_relaxed(atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_dec_return_relaxed(v);
-+	return raw_atomic64_dec_return_relaxed(v);
- }
- 
-+/**
-+ * atomic64_fetch_dec() - atomic decrement with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_dec(atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_dec(v);
-+	return raw_atomic64_fetch_dec(v);
- }
- 
-+/**
-+ * atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_dec_acquire(atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_dec_acquire(v);
-+	return raw_atomic64_fetch_dec_acquire(v);
- }
- 
-+/**
-+ * atomic64_fetch_dec_release() - atomic decrement with release ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_dec_release(atomic64_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_dec_release(v);
-+	return raw_atomic64_fetch_dec_release(v);
- }
- 
-+/**
-+ * atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_dec_relaxed(atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_dec_relaxed(v);
-+	return raw_atomic64_fetch_dec_relaxed(v);
- }
- 
-+/**
-+ * atomic64_and() - atomic bitwise AND with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_and() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic64_and(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic64_and(i, v);
-+	raw_atomic64_and(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_and() - atomic bitwise AND with full ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_and() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_and(s64 i, atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_and(i, v);
-+	return raw_atomic64_fetch_and(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_and_acquire(i, v);
-+	return raw_atomic64_fetch_and_acquire(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_and_release() - atomic bitwise AND with release ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_and_release(s64 i, atomic64_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_and_release(i, v);
-+	return raw_atomic64_fetch_and_release(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_and_relaxed(i, v);
-+	return raw_atomic64_fetch_and_relaxed(i, v);
- }
- 
-+/**
-+ * atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_andnot() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic64_andnot(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic64_andnot(i, v);
-+	raw_atomic64_andnot(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_andnot(s64 i, atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_andnot(i, v);
-+	return raw_atomic64_fetch_andnot(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_andnot_acquire(i, v);
-+	return raw_atomic64_fetch_andnot_acquire(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_andnot_release(i, v);
-+	return raw_atomic64_fetch_andnot_release(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_andnot_relaxed(i, v);
-+	return raw_atomic64_fetch_andnot_relaxed(i, v);
- }
- 
-+/**
-+ * atomic64_or() - atomic bitwise OR with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v | @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_or() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic64_or(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic64_or(i, v);
-+	raw_atomic64_or(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_or() - atomic bitwise OR with full ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v | @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_or() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_or(s64 i, atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_or(i, v);
-+	return raw_atomic64_fetch_or(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v | @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_or_acquire(i, v);
-+	return raw_atomic64_fetch_or_acquire(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_or_release() - atomic bitwise OR with release ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v | @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_or_release(s64 i, atomic64_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_or_release(i, v);
-+	return raw_atomic64_fetch_or_release(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v | @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_or_relaxed(i, v);
-+	return raw_atomic64_fetch_or_relaxed(i, v);
- }
- 
-+/**
-+ * atomic64_xor() - atomic bitwise XOR with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_xor() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic64_xor(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic64_xor(i, v);
-+	raw_atomic64_xor(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_xor() - atomic bitwise XOR with full ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_xor(s64 i, atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_xor(i, v);
-+	return raw_atomic64_fetch_xor(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_xor_acquire(i, v);
-+	return raw_atomic64_fetch_xor_acquire(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_xor_release(s64 i, atomic64_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_xor_release(i, v);
-+	return raw_atomic64_fetch_xor_release(i, v);
- }
- 
-+/**
-+ * atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
-+ * @i: s64 value
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_xor_relaxed(i, v);
-+	return raw_atomic64_fetch_xor_relaxed(i, v);
- }
- 
-+/**
-+ * atomic64_xchg() - atomic exchange with full ordering
-+ * @v: pointer to atomic64_t
-+ * @new: s64 value to assign
-+ *
-+ * Atomically updates @v to @new with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_xchg() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-atomic64_xchg(atomic64_t *v, s64 i)
-+atomic64_xchg(atomic64_t *v, s64 new)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_xchg(v, i);
-+	return raw_atomic64_xchg(v, new);
- }
- 
-+/**
-+ * atomic64_xchg_acquire() - atomic exchange with acquire ordering
-+ * @v: pointer to atomic64_t
-+ * @new: s64 value to assign
-+ *
-+ * Atomically updates @v to @new with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_xchg_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-atomic64_xchg_acquire(atomic64_t *v, s64 i)
-+atomic64_xchg_acquire(atomic64_t *v, s64 new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_xchg_acquire(v, i);
-+	return raw_atomic64_xchg_acquire(v, new);
- }
- 
-+/**
-+ * atomic64_xchg_release() - atomic exchange with release ordering
-+ * @v: pointer to atomic64_t
-+ * @new: s64 value to assign
-+ *
-+ * Atomically updates @v to @new with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_xchg_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-atomic64_xchg_release(atomic64_t *v, s64 i)
-+atomic64_xchg_release(atomic64_t *v, s64 new)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_xchg_release(v, i);
-+	return raw_atomic64_xchg_release(v, new);
- }
- 
-+/**
-+ * atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ * @new: s64 value to assign
-+ *
-+ * Atomically updates @v to @new with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_xchg_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
-atomic64_xchg_relaxed(atomic64_t *v, s64 i)
-+atomic64_xchg_relaxed(atomic64_t *v, s64 new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_xchg_relaxed(v, i);
-+	return raw_atomic64_xchg_relaxed(v, new);
- }
- 
-+/**
-+ * atomic64_cmpxchg() - atomic compare and exchange with full ordering
-+ * @v: pointer to atomic64_t
-+ * @old: s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_cmpxchg(v, old, new);
-}
-
-+	return raw_atomic64_cmpxchg(v, old, new);
-+}
-+
-+/**
-+ * atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
-+ * @v: pointer to atomic64_t
-+ * @old: s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_cmpxchg_acquire(v, old, new);
-+	return raw_atomic64_cmpxchg_acquire(v, old, new);
- }
- 
-+/**
-+ * atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
-+ * @v: pointer to atomic64_t
-+ * @old: s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with release ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_cmpxchg_release(v, old, new);
-}
-
-+	return raw_atomic64_cmpxchg_release(v, old, new);
-+}
-+
-+/**
-+ * atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ * @old: s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_cmpxchg_relaxed(v, old, new);
-+	return raw_atomic64_cmpxchg_relaxed(v, old, new);
- }
- 
-+/**
-+ * atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
-+ * @v: pointer to atomic64_t
-+ * @old: pointer to s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with full ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
- atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	instrument_atomic_read_write(old, sizeof(*old));
-	return arch_atomic64_try_cmpxchg(v, old, new);
-}
-
-+	instrument_read_write(old, sizeof(*old));
-+	return raw_atomic64_try_cmpxchg(v, old, new);
-+}
-+
-+/**
-+ * atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
-+ * @v: pointer to atomic64_t
-+ * @old: pointer to s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
- atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	instrument_atomic_read_write(old, sizeof(*old));
-	return arch_atomic64_try_cmpxchg_acquire(v, old, new);
-+	instrument_read_write(old, sizeof(*old));
-+	return raw_atomic64_try_cmpxchg_acquire(v, old, new);
- }
- 
-+/**
-+ * atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
-+ * @v: pointer to atomic64_t
-+ * @old: pointer to s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with release ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
- atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	instrument_atomic_read_write(old, sizeof(*old));
-	return arch_atomic64_try_cmpxchg_release(v, old, new);
-}
-
-+	instrument_read_write(old, sizeof(*old));
-+	return raw_atomic64_try_cmpxchg_release(v, old, new);
-+}
-+
-+/**
-+ * atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
-+ * @v: pointer to atomic64_t
-+ * @old: pointer to s64 value to compare with
-+ * @new: s64 value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
- atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	instrument_atomic_read_write(old, sizeof(*old));
-	return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
-+	instrument_read_write(old, sizeof(*old));
-+	return raw_atomic64_try_cmpxchg_relaxed(v, old, new);
- }
- 
-+/**
-+ * atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
-+ * @i: s64 value to subtract
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_sub_and_test() there.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
- atomic64_sub_and_test(s64 i, atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_sub_and_test(i, v);
-+	return raw_atomic64_sub_and_test(i, v);
- }
- 
-+/**
-+ * atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_dec_and_test() there.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
- atomic64_dec_and_test(atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_dec_and_test(v);
-+	return raw_atomic64_dec_and_test(v);
- }
- 
-+/**
-+ * atomic64_inc_and_test() - atomic increment and test if zero with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_inc_and_test() there.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
- atomic64_inc_and_test(atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_inc_and_test(v);
-+	return raw_atomic64_inc_and_test(v);
- }
- 
-+/**
-+ * atomic64_add_negative() - atomic add and test if negative with full ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_add_negative() there.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
- static __always_inline bool
- atomic64_add_negative(s64 i, atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_add_negative(i, v);
-+	return raw_atomic64_add_negative(i, v);
-+}
-+
-+/**
-+ * atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_add_negative_acquire() there.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
-+static __always_inline bool
-+atomic64_add_negative_acquire(s64 i, atomic64_t *v)
-+{
-+	instrument_atomic_read_write(v, sizeof(*v));
-+	return raw_atomic64_add_negative_acquire(i, v);
-+}
-+
-+/**
-+ * atomic64_add_negative_release() - atomic add and test if negative with release ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_add_negative_release() there.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
-+static __always_inline bool
-+atomic64_add_negative_release(s64 i, atomic64_t *v)
-+{
-+	kcsan_release();
-+	instrument_atomic_read_write(v, sizeof(*v));
-+	return raw_atomic64_add_negative_release(i, v);
- }
- 
-+/**
-+ * atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
-+ * @i: s64 value to add
-+ * @v: pointer to atomic64_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_add_negative_relaxed() there.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
-+static __always_inline bool
-+atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
-+{
-+	instrument_atomic_read_write(v, sizeof(*v));
-+	return raw_atomic64_add_negative_relaxed(i, v);
-+}
-+
-+/**
-+ * atomic64_fetch_add_unless() - atomic add unless value with full ordering
-+ * @v: pointer to atomic64_t
-+ * @a: s64 value to add
-+ * @u: s64 value to compare with
-+ *
-+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_unless() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline s64
- atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_fetch_add_unless(v, a, u);
-}
-
-+	return raw_atomic64_fetch_add_unless(v, a, u);
-+}
-+
-+/**
-+ * atomic64_add_unless() - atomic add unless value with full ordering
-+ * @v: pointer to atomic64_t
-+ * @a: s64 value to add
-+ * @u: s64 value to compare with
-+ *
-+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_add_unless() there.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
- atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_add_unless(v, a, u);
-+	return raw_atomic64_add_unless(v, a, u);
- }
- 
-+/**
-+ * atomic64_inc_not_zero() - atomic increment unless zero with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_inc_not_zero() there.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
- atomic64_inc_not_zero(atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_inc_not_zero(v);
-+	return raw_atomic64_inc_not_zero(v);
- }
- 
-+/**
-+ * atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_inc_unless_negative() there.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
- atomic64_inc_unless_negative(atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_inc_unless_negative(v);
-+	return raw_atomic64_inc_unless_negative(v);
- }
- 
-+/**
-+ * atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_dec_unless_positive() there.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
- atomic64_dec_unless_positive(atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_dec_unless_positive(v);
-+	return raw_atomic64_dec_unless_positive(v);
- }
- 
-+/**
-+ * atomic64_dec_if_positive() - atomic decrement if positive with full ordering
-+ * @v: pointer to atomic64_t
-+ *
-+ * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there.
-+ *
-+ * Return: The old value of (@v - 1), regardless of whether @v was updated.
-+ */
- static __always_inline s64
- atomic64_dec_if_positive(atomic64_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic64_dec_if_positive(v);
-+	return raw_atomic64_dec_if_positive(v);
- }
- 
-+/**
-+ * atomic_long_read() - atomic load with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically loads the value of @v with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_read() there.
-+ *
-+ * Return: The value loaded from @v.
-+ */
- static __always_inline long
- atomic_long_read(const atomic_long_t *v)
- {
- 	instrument_atomic_read(v, sizeof(*v));
-	return arch_atomic_long_read(v);
-}
-
-+	return raw_atomic_long_read(v);
-+}
-+
-+/**
-+ * atomic_long_read_acquire() - atomic load with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically loads the value of @v with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_read_acquire() there.
-+ *
-+ * Return: The value loaded from @v.
-+ */
- static __always_inline long
- atomic_long_read_acquire(const atomic_long_t *v)
- {
- 	instrument_atomic_read(v, sizeof(*v));
-	return arch_atomic_long_read_acquire(v);
-}
-
-+	return raw_atomic_long_read_acquire(v);
-+}
-+
-+/**
-+ * atomic_long_set() - atomic set with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ * @i: long value to assign
-+ *
-+ * Atomically sets @v to @i with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_set() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_long_set(atomic_long_t *v, long i)
- {
- 	instrument_atomic_write(v, sizeof(*v));
-	arch_atomic_long_set(v, i);
-}
-
-+	raw_atomic_long_set(v, i);
-+}
-+
-+/**
-+ * atomic_long_set_release() - atomic set with release ordering
-+ * @v: pointer to atomic_long_t
-+ * @i: long value to assign
-+ *
-+ * Atomically sets @v to @i with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_set_release() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_long_set_release(atomic_long_t *v, long i)
- {
- 	kcsan_release();
- 	instrument_atomic_write(v, sizeof(*v));
-	arch_atomic_long_set_release(v, i);
-}
-
-+	raw_atomic_long_set_release(v, i);
-+}
-+
-+/**
-+ * atomic_long_add() - atomic add with relaxed ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_add() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_long_add(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_long_add(i, v);
-+	raw_atomic_long_add(i, v);
- }
- 
-+/**
-+ * atomic_long_add_return() - atomic add with full ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_add_return() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_add_return(long i, atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_add_return(i, v);
-+	return raw_atomic_long_add_return(i, v);
- }
- 
-+/**
-+ * atomic_long_add_return_acquire() - atomic add with acquire ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_add_return_acquire() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_add_return_acquire(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_add_return_acquire(i, v);
-+	return raw_atomic_long_add_return_acquire(i, v);
- }
- 
-+/**
-+ * atomic_long_add_return_release() - atomic add with release ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_add_return_release() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_add_return_release(long i, atomic_long_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_add_return_release(i, v);
-+	return raw_atomic_long_add_return_release(i, v);
- }
- 
-+/**
-+ * atomic_long_add_return_relaxed() - atomic add with relaxed ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_add_return_relaxed() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_add_return_relaxed(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_add_return_relaxed(i, v);
-+	return raw_atomic_long_add_return_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_add() - atomic add with full ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_add(long i, atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_add(i, v);
-+	return raw_atomic_long_fetch_add(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_add_acquire() - atomic add with acquire ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_add_acquire(i, v);
-+	return raw_atomic_long_fetch_add_acquire(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_add_release() - atomic add with release ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_add_release(long i, atomic_long_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_add_release(i, v);
-+	return raw_atomic_long_fetch_add_release(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_add_relaxed(i, v);
-+	return raw_atomic_long_fetch_add_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_long_sub() - atomic subtract with relaxed ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_sub() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_long_sub(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_long_sub(i, v);
-+	raw_atomic_long_sub(i, v);
- }
- 
-+/**
-+ * atomic_long_sub_return() - atomic subtract with full ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_sub_return() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_sub_return(long i, atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_sub_return(i, v);
-+	return raw_atomic_long_sub_return(i, v);
- }
- 
-+/**
-+ * atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_acquire() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_sub_return_acquire(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_sub_return_acquire(i, v);
-+	return raw_atomic_long_sub_return_acquire(i, v);
- }
- 
-+/**
-+ * atomic_long_sub_return_release() - atomic subtract with release ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_release() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_sub_return_release(long i, atomic_long_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_sub_return_release(i, v);
-+	return raw_atomic_long_sub_return_release(i, v);
- }
- 
-+/**
-+ * atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_relaxed() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_sub_return_relaxed(i, v);
-+	return raw_atomic_long_sub_return_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_sub() - atomic subtract with full ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_sub(long i, atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_sub(i, v);
-+	return raw_atomic_long_fetch_sub(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_sub_acquire(i, v);
-+	return raw_atomic_long_fetch_sub_acquire(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_sub_release() - atomic subtract with release ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_sub_release(long i, atomic_long_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_sub_release(i, v);
-+	return raw_atomic_long_fetch_sub_release(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_sub_relaxed(i, v);
-+	return raw_atomic_long_fetch_sub_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_long_inc() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_inc() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_long_inc(atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_long_inc(v);
-+	raw_atomic_long_inc(v);
- }
- 
-+/**
-+ * atomic_long_inc_return() - atomic increment with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_return() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_inc_return(atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_inc_return(v);
-+	return raw_atomic_long_inc_return(v);
- }
- 
-+/**
-+ * atomic_long_inc_return_acquire() - atomic increment with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_acquire() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_inc_return_acquire(atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_inc_return_acquire(v);
-+	return raw_atomic_long_inc_return_acquire(v);
- }
- 
-+/**
-+ * atomic_long_inc_return_release() - atomic increment with release ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_release() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_inc_return_release(atomic_long_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_inc_return_release(v);
-+	return raw_atomic_long_inc_return_release(v);
- }
- 
-+/**
-+ * atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_relaxed() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_inc_return_relaxed(atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_inc_return_relaxed(v);
-+	return raw_atomic_long_inc_return_relaxed(v);
- }
- 
-+/**
-+ * atomic_long_fetch_inc() - atomic increment with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_inc(atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_inc(v);
-+	return raw_atomic_long_fetch_inc(v);
- }
- 
-+/**
-+ * atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_inc_acquire(atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_inc_acquire(v);
-+	return raw_atomic_long_fetch_inc_acquire(v);
- }
- 
-+/**
-+ * atomic_long_fetch_inc_release() - atomic increment with release ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_inc_release(atomic_long_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_inc_release(v);
-+	return raw_atomic_long_fetch_inc_release(v);
- }
- 
-+/**
-+ * atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_inc_relaxed(atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_inc_relaxed(v);
-+	return raw_atomic_long_fetch_inc_relaxed(v);
- }
- 
-+/**
-+ * atomic_long_dec() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_dec() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_long_dec(atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_long_dec(v);
-+	raw_atomic_long_dec(v);
- }
- 
-+/**
-+ * atomic_long_dec_return() - atomic decrement with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_return() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_dec_return(atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_dec_return(v);
-+	return raw_atomic_long_dec_return(v);
- }
- 
-+/**
-+ * atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_acquire() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_dec_return_acquire(atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_dec_return_acquire(v);
-+	return raw_atomic_long_dec_return_acquire(v);
- }
- 
-+/**
-+ * atomic_long_dec_return_release() - atomic decrement with release ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_release() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_dec_return_release(atomic_long_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_dec_return_release(v);
-+	return raw_atomic_long_dec_return_release(v);
- }
- 
-+/**
-+ * atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_relaxed() there.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
- atomic_long_dec_return_relaxed(atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_dec_return_relaxed(v);
-+	return raw_atomic_long_dec_return_relaxed(v);
- }
- 
-+/**
-+ * atomic_long_fetch_dec() - atomic decrement with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_dec(atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_dec(v);
-+	return raw_atomic_long_fetch_dec(v);
- }
- 
-+/**
-+ * atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_dec_acquire(atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_dec_acquire(v);
-+	return raw_atomic_long_fetch_dec_acquire(v);
- }
- 
-+/**
-+ * atomic_long_fetch_dec_release() - atomic decrement with release ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_dec_release(atomic_long_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_dec_release(v);
-+	return raw_atomic_long_fetch_dec_release(v);
- }
- 
-+/**
-+ * atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_dec_relaxed(atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_dec_relaxed(v);
-+	return raw_atomic_long_fetch_dec_relaxed(v);
- }
- 
-+/**
-+ * atomic_long_and() - atomic bitwise AND with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_and() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_long_and(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_long_and(i, v);
-+	raw_atomic_long_and(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_and() - atomic bitwise AND with full ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_and(long i, atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_and(i, v);
-+	return raw_atomic_long_fetch_and(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_and_acquire(i, v);
-+	return raw_atomic_long_fetch_and_acquire(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_and_release(long i, atomic_long_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_and_release(i, v);
-+	return raw_atomic_long_fetch_and_release(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_and_relaxed(i, v);
-+	return raw_atomic_long_fetch_and_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_andnot() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_long_andnot(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_long_andnot(i, v);
-+	raw_atomic_long_andnot(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_andnot(long i, atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_andnot(i, v);
-+	return raw_atomic_long_fetch_andnot(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_andnot_acquire(i, v);
-+	return raw_atomic_long_fetch_andnot_acquire(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_andnot_release(i, v);
-+	return raw_atomic_long_fetch_andnot_release(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_andnot_relaxed(i, v);
-+	return raw_atomic_long_fetch_andnot_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_long_or() - atomic bitwise OR with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v | @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_or() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_long_or(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_long_or(i, v);
-+	raw_atomic_long_or(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_or() - atomic bitwise OR with full ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v | @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_or(long i, atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_or(i, v);
-+	return raw_atomic_long_fetch_or(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v | @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_or_acquire(i, v);
-+	return raw_atomic_long_fetch_or_acquire(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v | @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_or_release(long i, atomic_long_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_or_release(i, v);
-+	return raw_atomic_long_fetch_or_release(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v | @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_or_relaxed(i, v);
-+	return raw_atomic_long_fetch_or_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_long_xor() - atomic bitwise XOR with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_xor() there.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
- atomic_long_xor(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	arch_atomic_long_xor(i, v);
-+	raw_atomic_long_xor(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_xor(long i, atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_xor(i, v);
-+	return raw_atomic_long_fetch_xor(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_xor_acquire(i, v);
-+	return raw_atomic_long_fetch_xor_acquire(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_xor_release(long i, atomic_long_t *v)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_xor_release(i, v);
-+	return raw_atomic_long_fetch_xor_release(i, v);
- }
- 
-+/**
-+ * atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_xor_relaxed(i, v);
-+	return raw_atomic_long_fetch_xor_relaxed(i, v);
- }
- 
-+/**
-+ * atomic_long_xchg() - atomic exchange with full ordering
-+ * @v: pointer to atomic_long_t
-+ * @new: long value to assign
-+ *
-+ * Atomically updates @v to @new with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_xchg() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-atomic_long_xchg(atomic_long_t *v, long i)
-+atomic_long_xchg(atomic_long_t *v, long new)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_xchg(v, i);
-+	return raw_atomic_long_xchg(v, new);
- }
- 
-+/**
-+ * atomic_long_xchg_acquire() - atomic exchange with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ * @new: long value to assign
-+ *
-+ * Atomically updates @v to @new with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_xchg_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-atomic_long_xchg_acquire(atomic_long_t *v, long i)
-+atomic_long_xchg_acquire(atomic_long_t *v, long new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_xchg_acquire(v, i);
-+	return raw_atomic_long_xchg_acquire(v, new);
- }
- 
-+/**
-+ * atomic_long_xchg_release() - atomic exchange with release ordering
-+ * @v: pointer to atomic_long_t
-+ * @new: long value to assign
-+ *
-+ * Atomically updates @v to @new with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_xchg_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-atomic_long_xchg_release(atomic_long_t *v, long i)
-+atomic_long_xchg_release(atomic_long_t *v, long new)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_xchg_release(v, i);
-+	return raw_atomic_long_xchg_release(v, new);
- }
- 
-+/**
-+ * atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ * @new: long value to assign
-+ *
-+ * Atomically updates @v to @new with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_xchg_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-atomic_long_xchg_relaxed(atomic_long_t *v, long i)
-+atomic_long_xchg_relaxed(atomic_long_t *v, long new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_xchg_relaxed(v, i);
-+	return raw_atomic_long_xchg_relaxed(v, new);
- }
- 
-+/**
-+ * atomic_long_cmpxchg() - atomic compare and exchange with full ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_cmpxchg(v, old, new);
-}
-
-+	return raw_atomic_long_cmpxchg(v, old, new);
-+}
-+
-+/**
-+ * atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_acquire() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_cmpxchg_acquire(v, old, new);
-+	return raw_atomic_long_cmpxchg_acquire(v, old, new);
- }
- 
-+/**
-+ * atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with release ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_release() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_cmpxchg_release(v, old, new);
-}
-
-+	return raw_atomic_long_cmpxchg_release(v, old, new);
-+}
-+
-+/**
-+ * atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_relaxed() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_cmpxchg_relaxed(v, old, new);
-+	return raw_atomic_long_cmpxchg_relaxed(v, old, new);
- }
- 
-+/**
-+ * atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: pointer to long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with full ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
- atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	instrument_atomic_read_write(old, sizeof(*old));
-	return arch_atomic_long_try_cmpxchg(v, old, new);
-}
-
-+	instrument_read_write(old, sizeof(*old));
-+	return raw_atomic_long_try_cmpxchg(v, old, new);
-+}
-+
-+/**
-+ * atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: pointer to long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
- atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	instrument_atomic_read_write(old, sizeof(*old));
-	return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
-+	instrument_read_write(old, sizeof(*old));
-+	return raw_atomic_long_try_cmpxchg_acquire(v, old, new);
- }
- 
-+/**
-+ * atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: pointer to long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with release ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
- atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
- {
- 	kcsan_release();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	instrument_atomic_read_write(old, sizeof(*old));
-	return arch_atomic_long_try_cmpxchg_release(v, old, new);
-}
-
-+	instrument_read_write(old, sizeof(*old));
-+	return raw_atomic_long_try_cmpxchg_release(v, old, new);
-+}
-+
-+/**
-+ * atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: pointer to long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
- atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
- {
- 	instrument_atomic_read_write(v, sizeof(*v));
-	instrument_atomic_read_write(old, sizeof(*old));
-	return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
-+	instrument_read_write(old, sizeof(*old));
-+	return raw_atomic_long_try_cmpxchg_relaxed(v, old, new);
- }
- 
-+/**
-+ * atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_sub_and_test() there.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
- atomic_long_sub_and_test(long i, atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_sub_and_test(i, v);
-+	return raw_atomic_long_sub_and_test(i, v);
- }
- 
-+/**
-+ * atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_and_test() there.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
- atomic_long_dec_and_test(atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_dec_and_test(v);
-+	return raw_atomic_long_dec_and_test(v);
- }
- 
-+/**
-+ * atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_and_test() there.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
- atomic_long_inc_and_test(atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_inc_and_test(v);
-+	return raw_atomic_long_inc_and_test(v);
- }
- 
-+/**
-+ * atomic_long_add_negative() - atomic add and test if negative with full ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_add_negative() there.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
- static __always_inline bool
- atomic_long_add_negative(long i, atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_add_negative(i, v);
-+	return raw_atomic_long_add_negative(i, v);
-+}
-+
-+/**
-+ * atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_acquire() there.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
-+static __always_inline bool
-+atomic_long_add_negative_acquire(long i, atomic_long_t *v)
-+{
-+	instrument_atomic_read_write(v, sizeof(*v));
-+	return raw_atomic_long_add_negative_acquire(i, v);
-+}
-+
-+/**
-+ * atomic_long_add_negative_release() - atomic add and test if negative with release ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_release() there.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
-+static __always_inline bool
-+atomic_long_add_negative_release(long i, atomic_long_t *v)
-+{
-+	kcsan_release();
-+	instrument_atomic_read_write(v, sizeof(*v));
-+	return raw_atomic_long_add_negative_release(i, v);
- }
- 
-+/**
-+ * atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_relaxed() there.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
-+static __always_inline bool
-+atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
-+{
-+	instrument_atomic_read_write(v, sizeof(*v));
-+	return raw_atomic_long_add_negative_relaxed(i, v);
-+}
-+
-+/**
-+ * atomic_long_fetch_add_unless() - atomic add unless value with full ordering
-+ * @v: pointer to atomic_long_t
-+ * @a: long value to add
-+ * @u: long value to compare with
-+ *
-+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_unless() there.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
- atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_fetch_add_unless(v, a, u);
-}
-
-+	return raw_atomic_long_fetch_add_unless(v, a, u);
-+}
-+
-+/**
-+ * atomic_long_add_unless() - atomic add unless value with full ordering
-+ * @v: pointer to atomic_long_t
-+ * @a: long value to add
-+ * @u: long value to compare with
-+ *
-+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_add_unless() there.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
- atomic_long_add_unless(atomic_long_t *v, long a, long u)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_add_unless(v, a, u);
-+	return raw_atomic_long_add_unless(v, a, u);
- }
- 
-+/**
-+ * atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_not_zero() there.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
- atomic_long_inc_not_zero(atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_inc_not_zero(v);
-+	return raw_atomic_long_inc_not_zero(v);
- }
- 
-+/**
-+ * atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_unless_negative() there.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
- atomic_long_inc_unless_negative(atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_inc_unless_negative(v);
-+	return raw_atomic_long_inc_unless_negative(v);
- }
- 
-+/**
-+ * atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_unless_positive() there.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
- atomic_long_dec_unless_positive(atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_dec_unless_positive(v);
-+	return raw_atomic_long_dec_unless_positive(v);
- }
- 
-+/**
-+ * atomic_long_dec_if_positive() - atomic decrement if positive with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_if_positive() there.
-+ *
-+ * Return: The old value of (@v - 1), regardless of whether @v was updated.
-+ */
- static __always_inline long
- atomic_long_dec_if_positive(atomic_long_t *v)
- {
- 	kcsan_mb();
- 	instrument_atomic_read_write(v, sizeof(*v));
-	return arch_atomic_long_dec_if_positive(v);
-+	return raw_atomic_long_dec_if_positive(v);
- }
- 
- #define xchg(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	kcsan_mb(); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_xchg(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_xchg(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define xchg_acquire(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_xchg_acquire(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define xchg_release(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	kcsan_release(); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_xchg_release(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_xchg_release(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define xchg_relaxed(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define cmpxchg(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	kcsan_mb(); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define cmpxchg_acquire(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define cmpxchg_release(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	kcsan_release(); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define cmpxchg_relaxed(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define cmpxchg64(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	kcsan_mb(); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg64(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define cmpxchg64_acquire(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define cmpxchg64_release(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	kcsan_release(); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define cmpxchg64_relaxed(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
-+})
-+
-+#define cmpxchg128(ptr, ...) \
-+({ \
-+	typeof(ptr) __ai_ptr = (ptr); \
-+	kcsan_mb(); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg128(__ai_ptr, __VA_ARGS__); \
-+})
-+
-+#define cmpxchg128_acquire(ptr, ...) \
-+({ \
-+	typeof(ptr) __ai_ptr = (ptr); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \
-+})
-+
-+#define cmpxchg128_release(ptr, ...) \
-+({ \
-+	typeof(ptr) __ai_ptr = (ptr); \
-+	kcsan_release(); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \
-+})
-+
-+#define cmpxchg128_relaxed(ptr, ...) \
-+({ \
-+	typeof(ptr) __ai_ptr = (ptr); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define try_cmpxchg(ptr, oldp, ...) \
-@@ -1973,18 +4875,18 @@ atomic_long_dec_if_positive(atomic_long_t *v)
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	typeof(oldp) __ai_oldp = (oldp); \
- 	kcsan_mb(); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
-	arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
- })
- 
- #define try_cmpxchg_acquire(ptr, oldp, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	typeof(oldp) __ai_oldp = (oldp); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
-	arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
- })
- 
- #define try_cmpxchg_release(ptr, oldp, ...) \
-@@ -1992,18 +4894,18 @@ atomic_long_dec_if_positive(atomic_long_t *v)
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	typeof(oldp) __ai_oldp = (oldp); \
- 	kcsan_release(); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
-	arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
- })
- 
- #define try_cmpxchg_relaxed(ptr, oldp, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	typeof(oldp) __ai_oldp = (oldp); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
-	arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
- })
- 
- #define try_cmpxchg64(ptr, oldp, ...) \
-@@ -2011,18 +4913,18 @@ atomic_long_dec_if_positive(atomic_long_t *v)
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	typeof(oldp) __ai_oldp = (oldp); \
- 	kcsan_mb(); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
-	arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
- })
- 
- #define try_cmpxchg64_acquire(ptr, oldp, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	typeof(oldp) __ai_oldp = (oldp); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
-	arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
- })
- 
- #define try_cmpxchg64_release(ptr, oldp, ...) \
-@@ -2030,57 +4932,122 @@ atomic_long_dec_if_positive(atomic_long_t *v)
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	typeof(oldp) __ai_oldp = (oldp); \
- 	kcsan_release(); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
-	arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
- })
- 
- #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	typeof(oldp) __ai_oldp = (oldp); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
-	arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+})
-+
-+#define try_cmpxchg128(ptr, oldp, ...) \
-+({ \
-+	typeof(ptr) __ai_ptr = (ptr); \
-+	typeof(oldp) __ai_oldp = (oldp); \
-+	kcsan_mb(); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+})
-+
-+#define try_cmpxchg128_acquire(ptr, oldp, ...) \
-+({ \
-+	typeof(ptr) __ai_ptr = (ptr); \
-+	typeof(oldp) __ai_oldp = (oldp); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+})
-+
-+#define try_cmpxchg128_release(ptr, oldp, ...) \
-+({ \
-+	typeof(ptr) __ai_ptr = (ptr); \
-+	typeof(oldp) __ai_oldp = (oldp); \
-+	kcsan_release(); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+})
-+
-+#define try_cmpxchg128_relaxed(ptr, oldp, ...) \
-+({ \
-+	typeof(ptr) __ai_ptr = (ptr); \
-+	typeof(oldp) __ai_oldp = (oldp); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
- })
- 
- #define cmpxchg_local(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define cmpxchg64_local(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
-+})
-+
-+#define cmpxchg128_local(ptr, ...) \
-+({ \
-+	typeof(ptr) __ai_ptr = (ptr); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \
- })
- 
- #define sync_cmpxchg(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
- 	kcsan_mb(); \
-	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-	arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
- })
- 
-#define cmpxchg_double(ptr, ...) \
-+#define try_cmpxchg_local(ptr, oldp, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
-	kcsan_mb(); \
-	instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
-	arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
-+	typeof(oldp) __ai_oldp = (oldp); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+})
-+
-+#define try_cmpxchg64_local(ptr, oldp, ...) \
-+({ \
-+	typeof(ptr) __ai_ptr = (ptr); \
-+	typeof(oldp) __ai_oldp = (oldp); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
- })
- 
-+#define try_cmpxchg128_local(ptr, oldp, ...) \
-+({ \
-+	typeof(ptr) __ai_ptr = (ptr); \
-+	typeof(oldp) __ai_oldp = (oldp); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
-+	raw_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
-+})
- 
-#define cmpxchg_double_local(ptr, ...) \
-+#define sync_try_cmpxchg(ptr, ...) \
- ({ \
- 	typeof(ptr) __ai_ptr = (ptr); \
-	instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
-	arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
-+	kcsan_mb(); \
-+	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
-+	raw_sync_try_cmpxchg(__ai_ptr, __VA_ARGS__); \
- })
- 
-+
- #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
-// 764f741eb77a7ad565dc8d99ce2837d5542e8aee
-+// f618ac667f868941a84ce0ab2242f1786e049ed4
-diff --git a/include/linux/atomic/atomic-long.h b/include/linux/atomic/atomic-long.h
-index 800b8c35992d..f86b29d90877 100644
--- a/include/linux/atomic/atomic-long.h
-+++ b/include/linux/atomic/atomic-long.h
-@@ -21,994 +21,1792 @@ typedef atomic_t atomic_long_t;
- #define atomic_long_cond_read_relaxed	atomic_cond_read_relaxed
- #endif
- 
-#ifdef CONFIG_64BIT
-
-static __always_inline long
-arch_atomic_long_read(const atomic_long_t *v)
-+/**
-+ * raw_atomic_long_read() - atomic load with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically loads the value of @v with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_read() elsewhere.
-+ *
-+ * Return: The value loaded from @v.
-+ */
-+static __always_inline long
-+raw_atomic_long_read(const atomic_long_t *v)
- {
-	return arch_atomic64_read(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_read(v);
-+#else
-+	return raw_atomic_read(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_read_acquire() - atomic load with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically loads the value of @v with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_read_acquire() elsewhere.
-+ *
-+ * Return: The value loaded from @v.
-+ */
- static __always_inline long
-arch_atomic_long_read_acquire(const atomic_long_t *v)
-+raw_atomic_long_read_acquire(const atomic_long_t *v)
- {
-	return arch_atomic64_read_acquire(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_read_acquire(v);
-+#else
-+	return raw_atomic_read_acquire(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_set() - atomic set with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ * @i: long value to assign
-+ *
-+ * Atomically sets @v to @i with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_set() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic_long_set(atomic_long_t *v, long i)
-+raw_atomic_long_set(atomic_long_t *v, long i)
- {
-	arch_atomic64_set(v, i);
-+#ifdef CONFIG_64BIT
-+	raw_atomic64_set(v, i);
-+#else
-+	raw_atomic_set(v, i);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_set_release() - atomic set with release ordering
-+ * @v: pointer to atomic_long_t
-+ * @i: long value to assign
-+ *
-+ * Atomically sets @v to @i with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_set_release() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic_long_set_release(atomic_long_t *v, long i)
-+raw_atomic_long_set_release(atomic_long_t *v, long i)
- {
-	arch_atomic64_set_release(v, i);
-+#ifdef CONFIG_64BIT
-+	raw_atomic64_set_release(v, i);
-+#else
-+	raw_atomic_set_release(v, i);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_add() - atomic add with relaxed ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_add() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic_long_add(long i, atomic_long_t *v)
-+raw_atomic_long_add(long i, atomic_long_t *v)
- {
-	arch_atomic64_add(i, v);
-+#ifdef CONFIG_64BIT
-+	raw_atomic64_add(i, v);
-+#else
-+	raw_atomic_add(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_add_return() - atomic add with full ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_add_return() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_add_return(long i, atomic_long_t *v)
-+raw_atomic_long_add_return(long i, atomic_long_t *v)
- {
-	return arch_atomic64_add_return(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_add_return(i, v);
-+#else
-+	return raw_atomic_add_return(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_add_return_acquire() - atomic add with acquire ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_add_return_acquire() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_add_return_acquire(long i, atomic_long_t *v)
-+raw_atomic_long_add_return_acquire(long i, atomic_long_t *v)
- {
-	return arch_atomic64_add_return_acquire(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_add_return_acquire(i, v);
-+#else
-+	return raw_atomic_add_return_acquire(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_add_return_release() - atomic add with release ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_add_return_release() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_add_return_release(long i, atomic_long_t *v)
-+raw_atomic_long_add_return_release(long i, atomic_long_t *v)
- {
-	return arch_atomic64_add_return_release(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_add_return_release(i, v);
-+#else
-+	return raw_atomic_add_return_release(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_add_return_relaxed() - atomic add with relaxed ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_add_return_relaxed() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_add_return_relaxed(long i, atomic_long_t *v)
-+raw_atomic_long_add_return_relaxed(long i, atomic_long_t *v)
- {
-	return arch_atomic64_add_return_relaxed(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_add_return_relaxed(i, v);
-+#else
-+	return raw_atomic_add_return_relaxed(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_add() - atomic add with full ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_add() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_add(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_add(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_add(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_add(i, v);
-+#else
-+	return raw_atomic_fetch_add(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_add_acquire() - atomic add with acquire ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_add_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_add_acquire(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_add_acquire(i, v);
-+#else
-+	return raw_atomic_fetch_add_acquire(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_add_release() - atomic add with release ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_add_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_add_release(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_add_release(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_add_release(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_add_release(i, v);
-+#else
-+	return raw_atomic_fetch_add_release(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_add_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_add_relaxed(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_add_relaxed(i, v);
-+#else
-+	return raw_atomic_fetch_add_relaxed(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_sub() - atomic subtract with relaxed ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_sub() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic_long_sub(long i, atomic_long_t *v)
-+raw_atomic_long_sub(long i, atomic_long_t *v)
- {
-	arch_atomic64_sub(i, v);
-+#ifdef CONFIG_64BIT
-+	raw_atomic64_sub(i, v);
-+#else
-+	raw_atomic_sub(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_sub_return() - atomic subtract with full ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_sub_return() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_sub_return(long i, atomic_long_t *v)
-+raw_atomic_long_sub_return(long i, atomic_long_t *v)
- {
-	return arch_atomic64_sub_return(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_sub_return(i, v);
-+#else
-+	return raw_atomic_sub_return(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_sub_return_acquire() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_sub_return_acquire(long i, atomic_long_t *v)
-+raw_atomic_long_sub_return_acquire(long i, atomic_long_t *v)
- {
-	return arch_atomic64_sub_return_acquire(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_sub_return_acquire(i, v);
-+#else
-+	return raw_atomic_sub_return_acquire(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_sub_return_release() - atomic subtract with release ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_sub_return_release() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_sub_return_release(long i, atomic_long_t *v)
-+raw_atomic_long_sub_return_release(long i, atomic_long_t *v)
- {
-	return arch_atomic64_sub_return_release(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_sub_return_release(i, v);
-+#else
-+	return raw_atomic_sub_return_release(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_sub_return_relaxed() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
-+raw_atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
- {
-	return arch_atomic64_sub_return_relaxed(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_sub_return_relaxed(i, v);
-+#else
-+	return raw_atomic_sub_return_relaxed(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_sub() - atomic subtract with full ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_sub() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_sub(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_sub(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_sub(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_sub(i, v);
-+#else
-+	return raw_atomic_fetch_sub(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_sub_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_sub_acquire(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_sub_acquire(i, v);
-+#else
-+	return raw_atomic_fetch_sub_acquire(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_sub_release() - atomic subtract with release ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_sub_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_sub_release(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_sub_release(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_sub_release(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_sub_release(i, v);
-+#else
-+	return raw_atomic_fetch_sub_release(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_sub_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_sub_relaxed(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_sub_relaxed(i, v);
-+#else
-+	return raw_atomic_fetch_sub_relaxed(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_inc() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_inc() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic_long_inc(atomic_long_t *v)
-+raw_atomic_long_inc(atomic_long_t *v)
- {
-	arch_atomic64_inc(v);
-+#ifdef CONFIG_64BIT
-+	raw_atomic64_inc(v);
-+#else
-+	raw_atomic_inc(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_inc_return() - atomic increment with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_inc_return() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_inc_return(atomic_long_t *v)
-+raw_atomic_long_inc_return(atomic_long_t *v)
- {
-	return arch_atomic64_inc_return(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_inc_return(v);
-+#else
-+	return raw_atomic_inc_return(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_inc_return_acquire() - atomic increment with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_inc_return_acquire() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_inc_return_acquire(atomic_long_t *v)
-+raw_atomic_long_inc_return_acquire(atomic_long_t *v)
- {
-	return arch_atomic64_inc_return_acquire(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_inc_return_acquire(v);
-+#else
-+	return raw_atomic_inc_return_acquire(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_inc_return_release() - atomic increment with release ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_inc_return_release() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_inc_return_release(atomic_long_t *v)
-+raw_atomic_long_inc_return_release(atomic_long_t *v)
- {
-	return arch_atomic64_inc_return_release(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_inc_return_release(v);
-+#else
-+	return raw_atomic_inc_return_release(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_inc_return_relaxed() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_inc_return_relaxed(atomic_long_t *v)
-+raw_atomic_long_inc_return_relaxed(atomic_long_t *v)
- {
-	return arch_atomic64_inc_return_relaxed(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_inc_return_relaxed(v);
-+#else
-+	return raw_atomic_inc_return_relaxed(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_inc() - atomic increment with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_inc() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_inc(atomic_long_t *v)
-+raw_atomic_long_fetch_inc(atomic_long_t *v)
- {
-	return arch_atomic64_fetch_inc(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_inc(v);
-+#else
-+	return raw_atomic_fetch_inc(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_inc_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_inc_acquire(atomic_long_t *v)
-+raw_atomic_long_fetch_inc_acquire(atomic_long_t *v)
- {
-	return arch_atomic64_fetch_inc_acquire(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_inc_acquire(v);
-+#else
-+	return raw_atomic_fetch_inc_acquire(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_inc_release() - atomic increment with release ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_inc_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_inc_release(atomic_long_t *v)
-+raw_atomic_long_fetch_inc_release(atomic_long_t *v)
- {
-	return arch_atomic64_fetch_inc_release(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_inc_release(v);
-+#else
-+	return raw_atomic_fetch_inc_release(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_inc_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_inc_relaxed(atomic_long_t *v)
-+raw_atomic_long_fetch_inc_relaxed(atomic_long_t *v)
- {
-	return arch_atomic64_fetch_inc_relaxed(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_inc_relaxed(v);
-+#else
-+	return raw_atomic_fetch_inc_relaxed(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_dec() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_dec() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic_long_dec(atomic_long_t *v)
-+raw_atomic_long_dec(atomic_long_t *v)
- {
-	arch_atomic64_dec(v);
-+#ifdef CONFIG_64BIT
-+	raw_atomic64_dec(v);
-+#else
-+	raw_atomic_dec(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_dec_return() - atomic decrement with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_dec_return() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_dec_return(atomic_long_t *v)
-+raw_atomic_long_dec_return(atomic_long_t *v)
- {
-	return arch_atomic64_dec_return(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_dec_return(v);
-+#else
-+	return raw_atomic_dec_return(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_dec_return_acquire() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_dec_return_acquire(atomic_long_t *v)
-+raw_atomic_long_dec_return_acquire(atomic_long_t *v)
- {
-	return arch_atomic64_dec_return_acquire(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_dec_return_acquire(v);
-+#else
-+	return raw_atomic_dec_return_acquire(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_dec_return_release() - atomic decrement with release ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_dec_return_release() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_dec_return_release(atomic_long_t *v)
-+raw_atomic_long_dec_return_release(atomic_long_t *v)
- {
-	return arch_atomic64_dec_return_release(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_dec_return_release(v);
-+#else
-+	return raw_atomic_dec_return_release(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_dec_return_relaxed() elsewhere.
-+ *
-+ * Return: The updated value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_dec_return_relaxed(atomic_long_t *v)
-+raw_atomic_long_dec_return_relaxed(atomic_long_t *v)
- {
-	return arch_atomic64_dec_return_relaxed(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_dec_return_relaxed(v);
-+#else
-+	return raw_atomic_dec_return_relaxed(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_dec() - atomic decrement with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_dec() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_dec(atomic_long_t *v)
-+raw_atomic_long_fetch_dec(atomic_long_t *v)
- {
-	return arch_atomic64_fetch_dec(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_dec(v);
-+#else
-+	return raw_atomic_fetch_dec(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_dec_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_dec_acquire(atomic_long_t *v)
-+raw_atomic_long_fetch_dec_acquire(atomic_long_t *v)
- {
-	return arch_atomic64_fetch_dec_acquire(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_dec_acquire(v);
-+#else
-+	return raw_atomic_fetch_dec_acquire(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_dec_release() - atomic decrement with release ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_dec_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_dec_release(atomic_long_t *v)
-+raw_atomic_long_fetch_dec_release(atomic_long_t *v)
- {
-	return arch_atomic64_fetch_dec_release(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_dec_release(v);
-+#else
-+	return raw_atomic_fetch_dec_release(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_dec_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_dec_relaxed(atomic_long_t *v)
-+raw_atomic_long_fetch_dec_relaxed(atomic_long_t *v)
- {
-	return arch_atomic64_fetch_dec_relaxed(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_dec_relaxed(v);
-+#else
-+	return raw_atomic_fetch_dec_relaxed(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_and() - atomic bitwise AND with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_and() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic_long_and(long i, atomic_long_t *v)
-+raw_atomic_long_and(long i, atomic_long_t *v)
- {
-	arch_atomic64_and(i, v);
-+#ifdef CONFIG_64BIT
-+	raw_atomic64_and(i, v);
-+#else
-+	raw_atomic_and(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_and() - atomic bitwise AND with full ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_and() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_and(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_and(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_and(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_and(i, v);
-+#else
-+	return raw_atomic_fetch_and(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_and_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_and_acquire(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_and_acquire(i, v);
-+#else
-+	return raw_atomic_fetch_and_acquire(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_and_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_and_release(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_and_release(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_and_release(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_and_release(i, v);
-+#else
-+	return raw_atomic_fetch_and_release(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_and_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_and_relaxed(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_and_relaxed(i, v);
-+#else
-+	return raw_atomic_fetch_and_relaxed(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_andnot() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic_long_andnot(long i, atomic_long_t *v)
-+raw_atomic_long_andnot(long i, atomic_long_t *v)
- {
-	arch_atomic64_andnot(i, v);
-+#ifdef CONFIG_64BIT
-+	raw_atomic64_andnot(i, v);
-+#else
-+	raw_atomic_andnot(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_andnot() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_andnot(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_andnot(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_andnot(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_andnot(i, v);
-+#else
-+	return raw_atomic_fetch_andnot(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_andnot_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_andnot_acquire(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_andnot_acquire(i, v);
-+#else
-+	return raw_atomic_fetch_andnot_acquire(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_andnot_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_andnot_release(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_andnot_release(i, v);
-+#else
-+	return raw_atomic_fetch_andnot_release(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_andnot_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_andnot_relaxed(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_andnot_relaxed(i, v);
-+#else
-+	return raw_atomic_fetch_andnot_relaxed(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_or() - atomic bitwise OR with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v | @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_or() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic_long_or(long i, atomic_long_t *v)
-+raw_atomic_long_or(long i, atomic_long_t *v)
- {
-	arch_atomic64_or(i, v);
-+#ifdef CONFIG_64BIT
-+	raw_atomic64_or(i, v);
-+#else
-+	raw_atomic_or(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_or() - atomic bitwise OR with full ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v | @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_or() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_or(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_or(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_or(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_or(i, v);
-+#else
-+	return raw_atomic_fetch_or(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v | @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_or_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_or_acquire(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_or_acquire(i, v);
-+#else
-+	return raw_atomic_fetch_or_acquire(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v | @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_or_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_or_release(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_or_release(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_or_release(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_or_release(i, v);
-+#else
-+	return raw_atomic_fetch_or_release(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v | @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_or_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_or_relaxed(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_or_relaxed(i, v);
-+#else
-+	return raw_atomic_fetch_or_relaxed(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_xor() - atomic bitwise XOR with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_xor() elsewhere.
-+ *
-+ * Return: Nothing.
-+ */
- static __always_inline void
-arch_atomic_long_xor(long i, atomic_long_t *v)
-{
-	arch_atomic64_xor(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_xor(long i, atomic_long_t *v)
-+raw_atomic_long_xor(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_xor(i, v);
-+#ifdef CONFIG_64BIT
-+	raw_atomic64_xor(i, v);
-+#else
-+	raw_atomic_xor(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_xor() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_xor(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_xor_acquire(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_xor(i, v);
-+#else
-+	return raw_atomic_fetch_xor(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_xor_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_xor_release(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_xor_release(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_xor_acquire(i, v);
-+#else
-+	return raw_atomic_fetch_xor_acquire(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_xor_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
-+raw_atomic_long_fetch_xor_release(long i, atomic_long_t *v)
- {
-	return arch_atomic64_fetch_xor_relaxed(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_xor_release(i, v);
-+#else
-+	return raw_atomic_fetch_xor_release(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
-+ * @i: long value
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_xor_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_xchg(atomic_long_t *v, long i)
-+raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
- {
-	return arch_atomic64_xchg(v, i);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_xor_relaxed(i, v);
-+#else
-+	return raw_atomic_fetch_xor_relaxed(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_xchg() - atomic exchange with full ordering
-+ * @v: pointer to atomic_long_t
-+ * @new: long value to assign
-+ *
-+ * Atomically updates @v to @new with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_xchg() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_xchg_acquire(atomic_long_t *v, long i)
-+raw_atomic_long_xchg(atomic_long_t *v, long new)
- {
-	return arch_atomic64_xchg_acquire(v, i);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_xchg(v, new);
-+#else
-+	return raw_atomic_xchg(v, new);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_xchg_acquire() - atomic exchange with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ * @new: long value to assign
-+ *
-+ * Atomically updates @v to @new with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_xchg_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_xchg_release(atomic_long_t *v, long i)
-+raw_atomic_long_xchg_acquire(atomic_long_t *v, long new)
- {
-	return arch_atomic64_xchg_release(v, i);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_xchg_acquire(v, new);
-+#else
-+	return raw_atomic_xchg_acquire(v, new);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_xchg_release() - atomic exchange with release ordering
-+ * @v: pointer to atomic_long_t
-+ * @new: long value to assign
-+ *
-+ * Atomically updates @v to @new with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_xchg_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_xchg_relaxed(atomic_long_t *v, long i)
-+raw_atomic_long_xchg_release(atomic_long_t *v, long new)
- {
-	return arch_atomic64_xchg_relaxed(v, i);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_xchg_release(v, new);
-+#else
-+	return raw_atomic_xchg_release(v, new);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ * @new: long value to assign
-+ *
-+ * Atomically updates @v to @new with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_xchg_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
-+raw_atomic_long_xchg_relaxed(atomic_long_t *v, long new)
- {
-	return arch_atomic64_cmpxchg(v, old, new);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_xchg_relaxed(v, new);
-+#else
-+	return raw_atomic_xchg_relaxed(v, new);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_cmpxchg() - atomic compare and exchange with full ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_cmpxchg() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
-+raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
- {
-	return arch_atomic64_cmpxchg_acquire(v, old, new);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_cmpxchg(v, old, new);
-+#else
-+	return raw_atomic_cmpxchg(v, old, new);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_cmpxchg_acquire() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
-+raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
- {
-	return arch_atomic64_cmpxchg_release(v, old, new);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_cmpxchg_acquire(v, old, new);
-+#else
-+	return raw_atomic_cmpxchg_acquire(v, old, new);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with release ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_cmpxchg_release() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
-{
-	return arch_atomic64_cmpxchg_relaxed(v, old, new);
-}
-
-static __always_inline bool
-arch_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
-{
-	return arch_atomic64_try_cmpxchg(v, (s64 *)old, new);
-}
-
-static __always_inline bool
-arch_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
-{
-	return arch_atomic64_try_cmpxchg_acquire(v, (s64 *)old, new);
-}
-
-static __always_inline bool
-arch_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
-{
-	return arch_atomic64_try_cmpxchg_release(v, (s64 *)old, new);
-}
-
-static __always_inline bool
-arch_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
-{
-	return arch_atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new);
-}
-
-static __always_inline bool
-arch_atomic_long_sub_and_test(long i, atomic_long_t *v)
-{
-	return arch_atomic64_sub_and_test(i, v);
-}
-
-static __always_inline bool
-arch_atomic_long_dec_and_test(atomic_long_t *v)
-{
-	return arch_atomic64_dec_and_test(v);
-}
-
-static __always_inline bool
-arch_atomic_long_inc_and_test(atomic_long_t *v)
-{
-	return arch_atomic64_inc_and_test(v);
-}
-
-static __always_inline bool
-arch_atomic_long_add_negative(long i, atomic_long_t *v)
-+raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
- {
-	return arch_atomic64_add_negative(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_cmpxchg_release(v, old, new);
-+#else
-+	return raw_atomic_cmpxchg_release(v, old, new);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_cmpxchg_relaxed() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
-{
-	return arch_atomic64_fetch_add_unless(v, a, u);
-}
-
-static __always_inline bool
-arch_atomic_long_add_unless(atomic_long_t *v, long a, long u)
-+raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
- {
-	return arch_atomic64_add_unless(v, a, u);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_cmpxchg_relaxed(v, old, new);
-+#else
-+	return raw_atomic_cmpxchg_relaxed(v, old, new);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: pointer to long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with full ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg() elsewhere.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_inc_not_zero(atomic_long_t *v)
-+raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
- {
-	return arch_atomic64_inc_not_zero(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_try_cmpxchg(v, (s64 *)old, new);
-+#else
-+	return raw_atomic_try_cmpxchg(v, (int *)old, new);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: pointer to long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_acquire() elsewhere.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_inc_unless_negative(atomic_long_t *v)
-+raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
- {
-	return arch_atomic64_inc_unless_negative(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_try_cmpxchg_acquire(v, (s64 *)old, new);
-+#else
-+	return raw_atomic_try_cmpxchg_acquire(v, (int *)old, new);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: pointer to long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with release ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_release() elsewhere.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_dec_unless_positive(atomic_long_t *v)
-{
-	return arch_atomic64_dec_unless_positive(v);
-}
-
-static __always_inline long
-arch_atomic_long_dec_if_positive(atomic_long_t *v)
-{
-	return arch_atomic64_dec_if_positive(v);
-}
-
-#else /* CONFIG_64BIT */
-
-static __always_inline long
-arch_atomic_long_read(const atomic_long_t *v)
-{
-	return arch_atomic_read(v);
-}
-
-static __always_inline long
-arch_atomic_long_read_acquire(const atomic_long_t *v)
-{
-	return arch_atomic_read_acquire(v);
-}
-
-static __always_inline void
-arch_atomic_long_set(atomic_long_t *v, long i)
-+raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
- {
-	arch_atomic_set(v, i);
-}
-
-static __always_inline void
-arch_atomic_long_set_release(atomic_long_t *v, long i)
-{
-	arch_atomic_set_release(v, i);
-}
-
-static __always_inline void
-arch_atomic_long_add(long i, atomic_long_t *v)
-{
-	arch_atomic_add(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_add_return(long i, atomic_long_t *v)
-{
-	return arch_atomic_add_return(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_add_return_acquire(long i, atomic_long_t *v)
-{
-	return arch_atomic_add_return_acquire(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_add_return_release(long i, atomic_long_t *v)
-{
-	return arch_atomic_add_return_release(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_add_return_relaxed(long i, atomic_long_t *v)
-{
-	return arch_atomic_add_return_relaxed(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_add(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_add(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_add_acquire(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_add_release(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_add_release(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_add_relaxed(i, v);
-}
-
-static __always_inline void
-arch_atomic_long_sub(long i, atomic_long_t *v)
-{
-	arch_atomic_sub(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_sub_return(long i, atomic_long_t *v)
-{
-	return arch_atomic_sub_return(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_sub_return_acquire(long i, atomic_long_t *v)
-{
-	return arch_atomic_sub_return_acquire(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_sub_return_release(long i, atomic_long_t *v)
-{
-	return arch_atomic_sub_return_release(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
-{
-	return arch_atomic_sub_return_relaxed(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_sub(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_sub(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_sub_acquire(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_sub_release(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_sub_release(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_sub_relaxed(i, v);
-}
-
-static __always_inline void
-arch_atomic_long_inc(atomic_long_t *v)
-{
-	arch_atomic_inc(v);
-}
-
-static __always_inline long
-arch_atomic_long_inc_return(atomic_long_t *v)
-{
-	return arch_atomic_inc_return(v);
-}
-
-static __always_inline long
-arch_atomic_long_inc_return_acquire(atomic_long_t *v)
-{
-	return arch_atomic_inc_return_acquire(v);
-}
-
-static __always_inline long
-arch_atomic_long_inc_return_release(atomic_long_t *v)
-{
-	return arch_atomic_inc_return_release(v);
-}
-
-static __always_inline long
-arch_atomic_long_inc_return_relaxed(atomic_long_t *v)
-{
-	return arch_atomic_inc_return_relaxed(v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_inc(atomic_long_t *v)
-{
-	return arch_atomic_fetch_inc(v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_inc_acquire(atomic_long_t *v)
-{
-	return arch_atomic_fetch_inc_acquire(v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_inc_release(atomic_long_t *v)
-{
-	return arch_atomic_fetch_inc_release(v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_inc_relaxed(atomic_long_t *v)
-{
-	return arch_atomic_fetch_inc_relaxed(v);
-}
-
-static __always_inline void
-arch_atomic_long_dec(atomic_long_t *v)
-{
-	arch_atomic_dec(v);
-}
-
-static __always_inline long
-arch_atomic_long_dec_return(atomic_long_t *v)
-{
-	return arch_atomic_dec_return(v);
-}
-
-static __always_inline long
-arch_atomic_long_dec_return_acquire(atomic_long_t *v)
-{
-	return arch_atomic_dec_return_acquire(v);
-}
-
-static __always_inline long
-arch_atomic_long_dec_return_release(atomic_long_t *v)
-{
-	return arch_atomic_dec_return_release(v);
-}
-
-static __always_inline long
-arch_atomic_long_dec_return_relaxed(atomic_long_t *v)
-{
-	return arch_atomic_dec_return_relaxed(v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_dec(atomic_long_t *v)
-{
-	return arch_atomic_fetch_dec(v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_dec_acquire(atomic_long_t *v)
-{
-	return arch_atomic_fetch_dec_acquire(v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_dec_release(atomic_long_t *v)
-{
-	return arch_atomic_fetch_dec_release(v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_dec_relaxed(atomic_long_t *v)
-{
-	return arch_atomic_fetch_dec_relaxed(v);
-}
-
-static __always_inline void
-arch_atomic_long_and(long i, atomic_long_t *v)
-{
-	arch_atomic_and(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_and(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_and(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_and_acquire(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_and_release(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_and_release(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_and_relaxed(i, v);
-}
-
-static __always_inline void
-arch_atomic_long_andnot(long i, atomic_long_t *v)
-{
-	arch_atomic_andnot(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_andnot(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_andnot(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_andnot_acquire(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_andnot_release(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_andnot_relaxed(i, v);
-}
-
-static __always_inline void
-arch_atomic_long_or(long i, atomic_long_t *v)
-{
-	arch_atomic_or(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_or(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_or(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_or_acquire(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_or_release(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_or_release(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_or_relaxed(i, v);
-}
-
-static __always_inline void
-arch_atomic_long_xor(long i, atomic_long_t *v)
-{
-	arch_atomic_xor(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_xor(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_xor(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_xor_acquire(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_xor_release(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_xor_release(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
-{
-	return arch_atomic_fetch_xor_relaxed(i, v);
-}
-
-static __always_inline long
-arch_atomic_long_xchg(atomic_long_t *v, long i)
-{
-	return arch_atomic_xchg(v, i);
-}
-
-static __always_inline long
-arch_atomic_long_xchg_acquire(atomic_long_t *v, long i)
-{
-	return arch_atomic_xchg_acquire(v, i);
-}
-
-static __always_inline long
-arch_atomic_long_xchg_release(atomic_long_t *v, long i)
-{
-	return arch_atomic_xchg_release(v, i);
-}
-
-static __always_inline long
-arch_atomic_long_xchg_relaxed(atomic_long_t *v, long i)
-{
-	return arch_atomic_xchg_relaxed(v, i);
-}
-
-static __always_inline long
-arch_atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
-{
-	return arch_atomic_cmpxchg(v, old, new);
-}
-
-static __always_inline long
-arch_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
-{
-	return arch_atomic_cmpxchg_acquire(v, old, new);
-}
-
-static __always_inline long
-arch_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
-{
-	return arch_atomic_cmpxchg_release(v, old, new);
-}
-
-static __always_inline long
-arch_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
-{
-	return arch_atomic_cmpxchg_relaxed(v, old, new);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_try_cmpxchg_release(v, (s64 *)old, new);
-+#else
-+	return raw_atomic_try_cmpxchg_release(v, (int *)old, new);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
-+ * @v: pointer to atomic_long_t
-+ * @old: pointer to long value to compare with
-+ * @new: long value to assign
-+ *
-+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
-+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
-+ * and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_relaxed() elsewhere.
-+ *
-+ * Return: @true if the exchange occured, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
-+raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
- {
-	return arch_atomic_try_cmpxchg(v, (int *)old, new);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new);
-+#else
-+	return raw_atomic_try_cmpxchg_relaxed(v, (int *)old, new);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
-+ * @i: long value to subtract
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_sub_and_test() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
-+raw_atomic_long_sub_and_test(long i, atomic_long_t *v)
- {
-	return arch_atomic_try_cmpxchg_acquire(v, (int *)old, new);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_sub_and_test(i, v);
-+#else
-+	return raw_atomic_sub_and_test(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v - 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_dec_and_test() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
-+raw_atomic_long_dec_and_test(atomic_long_t *v)
- {
-	return arch_atomic_try_cmpxchg_release(v, (int *)old, new);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_dec_and_test(v);
-+#else
-+	return raw_atomic_dec_and_test(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + 1) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_inc_and_test() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is zero, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
-+raw_atomic_long_inc_and_test(atomic_long_t *v)
- {
-	return arch_atomic_try_cmpxchg_relaxed(v, (int *)old, new);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_inc_and_test(v);
-+#else
-+	return raw_atomic_inc_and_test(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_add_negative() - atomic add and test if negative with full ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with full ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_add_negative() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_sub_and_test(long i, atomic_long_t *v)
-+raw_atomic_long_add_negative(long i, atomic_long_t *v)
- {
-	return arch_atomic_sub_and_test(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_add_negative(i, v);
-+#else
-+	return raw_atomic_add_negative(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with acquire ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_add_negative_acquire() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_dec_and_test(atomic_long_t *v)
-+raw_atomic_long_add_negative_acquire(long i, atomic_long_t *v)
- {
-	return arch_atomic_dec_and_test(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_add_negative_acquire(i, v);
-+#else
-+	return raw_atomic_add_negative_acquire(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_add_negative_release() - atomic add and test if negative with release ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with release ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_add_negative_release() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_inc_and_test(atomic_long_t *v)
-+raw_atomic_long_add_negative_release(long i, atomic_long_t *v)
- {
-	return arch_atomic_inc_and_test(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_add_negative_release(i, v);
-+#else
-+	return raw_atomic_add_negative_release(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
-+ * @i: long value to add
-+ * @v: pointer to atomic_long_t
-+ *
-+ * Atomically updates @v to (@v + @i) with relaxed ordering.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_add_negative_relaxed() elsewhere.
-+ *
-+ * Return: @true if the resulting value of @v is negative, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_add_negative(long i, atomic_long_t *v)
-+raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
- {
-	return arch_atomic_add_negative(i, v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_add_negative_relaxed(i, v);
-+#else
-+	return raw_atomic_add_negative_relaxed(i, v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_fetch_add_unless() - atomic add unless value with full ordering
-+ * @v: pointer to atomic_long_t
-+ * @a: long value to add
-+ * @u: long value to compare with
-+ *
-+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_fetch_add_unless() elsewhere.
-+ *
-+ * Return: The original value of @v.
-+ */
- static __always_inline long
-arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
-+raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
- {
-	return arch_atomic_fetch_add_unless(v, a, u);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_fetch_add_unless(v, a, u);
-+#else
-+	return raw_atomic_fetch_add_unless(v, a, u);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_add_unless() - atomic add unless value with full ordering
-+ * @v: pointer to atomic_long_t
-+ * @a: long value to add
-+ * @u: long value to compare with
-+ *
-+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_add_unless() elsewhere.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_add_unless(atomic_long_t *v, long a, long u)
-+raw_atomic_long_add_unless(atomic_long_t *v, long a, long u)
- {
-	return arch_atomic_add_unless(v, a, u);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_add_unless(v, a, u);
-+#else
-+	return raw_atomic_add_unless(v, a, u);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_inc_not_zero() elsewhere.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_inc_not_zero(atomic_long_t *v)
-+raw_atomic_long_inc_not_zero(atomic_long_t *v)
- {
-	return arch_atomic_inc_not_zero(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_inc_not_zero(v);
-+#else
-+	return raw_atomic_inc_not_zero(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_inc_unless_negative() elsewhere.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_inc_unless_negative(atomic_long_t *v)
-+raw_atomic_long_inc_unless_negative(atomic_long_t *v)
- {
-	return arch_atomic_inc_unless_negative(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_inc_unless_negative(v);
-+#else
-+	return raw_atomic_inc_unless_negative(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_dec_unless_positive() elsewhere.
-+ *
-+ * Return: @true if @v was updated, @false otherwise.
-+ */
- static __always_inline bool
-arch_atomic_long_dec_unless_positive(atomic_long_t *v)
-+raw_atomic_long_dec_unless_positive(atomic_long_t *v)
- {
-	return arch_atomic_dec_unless_positive(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_dec_unless_positive(v);
-+#else
-+	return raw_atomic_dec_unless_positive(v);
-+#endif
- }
- 
-+/**
-+ * raw_atomic_long_dec_if_positive() - atomic decrement if positive with full ordering
-+ * @v: pointer to atomic_long_t
-+ *
-+ * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
-+ * Otherwise, @v is not modified and relaxed ordering is provided.
-+ *
-+ * Safe to use in noinstr code; prefer atomic_long_dec_if_positive() elsewhere.
-+ *
-+ * Return: The old value of (@v - 1), regardless of whether @v was updated.
-+ */
- static __always_inline long
-arch_atomic_long_dec_if_positive(atomic_long_t *v)
-+raw_atomic_long_dec_if_positive(atomic_long_t *v)
- {
-	return arch_atomic_dec_if_positive(v);
-+#ifdef CONFIG_64BIT
-+	return raw_atomic64_dec_if_positive(v);
-+#else
-+	return raw_atomic_dec_if_positive(v);
-+#endif
- }
- 
-#endif /* CONFIG_64BIT */
- #endif /* _LINUX_ATOMIC_LONG_H */
-// e8f0e08ff072b74d180eabe2ad001282b38c2c88
-+// eadf183c3600b8b92b91839dd3be6bcc560c752d
-diff --git a/include/linux/attribute_container.h b/include/linux/attribute_container.h
-index e4004d1e6725..b3643de9931d 100644
--- a/include/linux/attribute_container.h
-+++ b/include/linux/attribute_container.h
-@@ -61,14 +61,8 @@ int attribute_container_device_trigger_safe(struct device *dev,
- 					    int (*undo)(struct attribute_container *,
- 							struct device *,
- 							struct device *));
-void attribute_container_trigger(struct device *dev, 
-				 int (*fn)(struct attribute_container *,
-					   struct device *));
- int attribute_container_add_attrs(struct device *classdev);
- int attribute_container_add_class_device(struct device *classdev);
-int attribute_container_add_class_device_adapter(struct attribute_container *cont,
-						 struct device *dev,
-						 struct device *classdev);
- void attribute_container_remove_attrs(struct device *classdev);
- void attribute_container_class_device_del(struct device *classdev);
- struct attribute_container *attribute_container_classdev_to_container(struct device *);
-diff --git a/include/linux/audit.h b/include/linux/audit.h
-index 3608992848d3..536f8ee8da81 100644
--- a/include/linux/audit.h
-+++ b/include/linux/audit.h
-@@ -14,6 +14,7 @@
- #include <linux/audit_arch.h>
- #include <uapi/linux/audit.h>
- #include <uapi/linux/netfilter/nf_tables.h>
-+#include <uapi/linux/fanotify.h>
- 
- #define AUDIT_INO_UNSET ((unsigned long)-1)
- #define AUDIT_DEV_UNSET ((dev_t)-1)
-@@ -35,6 +36,9 @@ struct mqstat;
- struct audit_watch;
- struct audit_tree;
- struct sk_buff;
-+struct kern_ipc_perm;
-+struct lsm_id;
-+struct lsm_prop;
- 
- struct audit_krule {
- 	u32			pflags;
-@@ -116,6 +120,8 @@ enum audit_nfcfgop {
- 	AUDIT_NFT_OP_OBJ_RESET,
- 	AUDIT_NFT_OP_FLOWTABLE_REGISTER,
- 	AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
-+	AUDIT_NFT_OP_SETELEM_RESET,
-+	AUDIT_NFT_OP_RULE_RESET,
- 	AUDIT_NFT_OP_INVALID,
- };
- 
-@@ -129,8 +135,6 @@ extern unsigned compat_dir_class[];
- extern unsigned compat_chattr_class[];
- extern unsigned compat_signal_class[];
- 
-extern int audit_classify_compat_syscall(int abi, unsigned syscall);
-
- /* audit_names->type values */
- #define	AUDIT_TYPE_UNKNOWN	0	/* we don't know yet */
- #define	AUDIT_TYPE_NORMAL	1	/* a "normal" audit record */
-@@ -145,6 +149,10 @@ extern int audit_classify_compat_syscall(int abi, unsigned syscall);
- #define AUDIT_TTY_ENABLE	BIT(0)
- #define AUDIT_TTY_LOG_PASSWD	BIT(1)
- 
-+/* bit values for audit_cfg_lsm */
-+#define AUDIT_CFG_LSM_SECCTX_SUBJECT	BIT(0)
-+#define AUDIT_CFG_LSM_SECCTX_OBJECT	BIT(1)
-+
- struct filename;
- 
- #define AUDIT_OFF	0
-@@ -183,6 +191,8 @@ extern void		    audit_log_path_denied(int type,
- 						  const char *operation);
- extern void		    audit_log_lost(const char *message);
- 
-+extern int audit_log_subj_ctx(struct audit_buffer *ab, struct lsm_prop *prop);
-+extern int audit_log_obj_ctx(struct audit_buffer *ab, struct lsm_prop *prop);
- extern int audit_log_task_context(struct audit_buffer *ab);
- extern void audit_log_task_info(struct audit_buffer *ab);
- 
-@@ -208,6 +218,8 @@ extern u32 audit_enabled;
- 
- extern int audit_signal_info(int sig, struct task_struct *t);
- 
-+extern void audit_cfg_lsm(const struct lsm_id *lsmid, int flags);
-+
- #else /* CONFIG_AUDIT */
- static inline __printf(4, 5)
- void audit_log(struct audit_context *ctx, gfp_t gfp_mask, int type,
-@@ -243,6 +255,16 @@ static inline void audit_log_key(struct audit_buffer *ab, char *key)
- { }
- static inline void audit_log_path_denied(int type, const char *operation)
- { }
-+static inline int audit_log_subj_ctx(struct audit_buffer *ab,
-+				     struct lsm_prop *prop)
-+{
-+	return 0;
-+}
-+static inline int audit_log_obj_ctx(struct audit_buffer *ab,
-+				    struct lsm_prop *prop)
-+{
-+	return 0;
-+}
- static inline int audit_log_task_context(struct audit_buffer *ab)
- {
- 	return 0;
-@@ -267,6 +289,9 @@ static inline int audit_signal_info(int sig, struct task_struct *t)
- 	return 0;
- }
- 
-+static inline void audit_cfg_lsm(const struct lsm_id *lsmid, int flags)
-+{ }
-+
- #endif /* CONFIG_AUDIT */
- 
- #ifdef CONFIG_AUDIT_COMPAT_GENERIC
-@@ -415,8 +440,8 @@ extern int __audit_log_bprm_fcaps(struct linux_binprm *bprm,
- extern void __audit_log_capset(const struct cred *new, const struct cred *old);
- extern void __audit_mmap_fd(int fd, int flags);
- extern void __audit_openat2_how(struct open_how *how);
-extern void __audit_log_kern_module(char *name);
-extern void __audit_fanotify(unsigned int response);
-+extern void __audit_log_kern_module(const char *name);
-+extern void __audit_fanotify(u32 response, struct fanotify_response_info_audit_rule *friar);
- extern void __audit_tk_injoffset(struct timespec64 offset);
- extern void __audit_ntp_log(const struct audit_ntp_data *ad);
- extern void __audit_log_nfcfg(const char *name, u8 af, unsigned int nentries,
-@@ -517,16 +542,16 @@ static inline void audit_openat2_how(struct open_how *how)
- 		__audit_openat2_how(how);
- }
- 
-static inline void audit_log_kern_module(char *name)
-+static inline void audit_log_kern_module(const char *name)
- {
- 	if (!audit_dummy_context())
- 		__audit_log_kern_module(name);
- }
- 
-static inline void audit_fanotify(unsigned int response)
-+static inline void audit_fanotify(u32 response, struct fanotify_response_info_audit_rule *friar)
- {
-	if (!audit_dummy_context())
-		__audit_fanotify(response);
-+	if (audit_enabled)
-+		__audit_fanotify(response, friar);
- }
- 
- static inline void audit_tk_injoffset(struct timespec64 offset)
-@@ -675,11 +700,10 @@ static inline void audit_mmap_fd(int fd, int flags)
- static inline void audit_openat2_how(struct open_how *how)
- { }
- 
-static inline void audit_log_kern_module(char *name)
-{
-}
-+static inline void audit_log_kern_module(const char *name)
-+{ }
- 
-static inline void audit_fanotify(unsigned int response)
-+static inline void audit_fanotify(u32 response, struct fanotify_response_info_audit_rule *friar)
- { }
- 
- static inline void audit_tk_injoffset(struct timespec64 offset)
-diff --git a/include/linux/audit_arch.h b/include/linux/audit_arch.h
-index 8fdb1afe251a..0e34d673ef17 100644
--- a/include/linux/audit_arch.h
-+++ b/include/linux/audit_arch.h
-@@ -21,4 +21,6 @@ enum auditsc_class_t {
- 	AUDITSC_NVALS /* count */
- };
- 
-+extern int audit_classify_compat_syscall(int abi, unsigned syscall);
-+
- #endif
-diff --git a/include/linux/auxiliary_bus.h b/include/linux/auxiliary_bus.h
-index de21d9d24a95..4086afd0cc6b 100644
--- a/include/linux/auxiliary_bus.h
-+++ b/include/linux/auxiliary_bus.h
-@@ -58,6 +58,10 @@
-  *       in
-  * @name: Match name found by the auxiliary device driver,
-  * @id: unique identitier if multiple devices of the same name are exported,
-+ * @sysfs: embedded struct which hold all sysfs related fields,
-+ * @sysfs.irqs: irqs xarray contains irq indices which are used by the device,
-+ * @sysfs.lock: Synchronize irq sysfs creation,
-+ * @sysfs.irq_dir_exists: whether "irqs" directory exists,
-  *
-  * An auxiliary_device represents a part of its parent device's functionality.
-  * It is given a name that, combined with the registering drivers
-@@ -139,6 +143,11 @@ struct auxiliary_device {
- 	struct device dev;
- 	const char *name;
- 	u32 id;
-+	struct {
-+		struct xarray irqs;
-+		struct mutex lock; /* Synchronize irq sysfs creation */
-+		bool irq_dir_exists;
-+	} sysfs;
- };
- 
- /**
-@@ -203,7 +212,7 @@ static inline struct auxiliary_device *to_auxiliary_dev(struct device *dev)
- 	return container_of(dev, struct auxiliary_device, dev);
- }
- 
-static inline struct auxiliary_driver *to_auxiliary_drv(struct device_driver *drv)
-+static inline const struct auxiliary_driver *to_auxiliary_drv(const struct device_driver *drv)
- {
- 	return container_of(drv, struct auxiliary_driver, driver);
- }
-@@ -212,8 +221,24 @@ int auxiliary_device_init(struct auxiliary_device *auxdev);
- int __auxiliary_device_add(struct auxiliary_device *auxdev, const char *modname);
- #define auxiliary_device_add(auxdev) __auxiliary_device_add(auxdev, KBUILD_MODNAME)
- 
-+#ifdef CONFIG_SYSFS
-+int auxiliary_device_sysfs_irq_add(struct auxiliary_device *auxdev, int irq);
-+void auxiliary_device_sysfs_irq_remove(struct auxiliary_device *auxdev,
-+				       int irq);
-+#else /* CONFIG_SYSFS */
-+static inline int
-+auxiliary_device_sysfs_irq_add(struct auxiliary_device *auxdev, int irq)
-+{
-+	return 0;
-+}
-+
-+static inline void
-+auxiliary_device_sysfs_irq_remove(struct auxiliary_device *auxdev, int irq) {}
-+#endif
-+
- static inline void auxiliary_device_uninit(struct auxiliary_device *auxdev)
- {
-+	mutex_destroy(&auxdev->sysfs.lock);
- 	put_device(&auxdev->dev);
- }
- 
-@@ -229,6 +254,23 @@ int __auxiliary_driver_register(struct auxiliary_driver *auxdrv, struct module *
- 
- void auxiliary_driver_unregister(struct auxiliary_driver *auxdrv);
- 
-+struct auxiliary_device *auxiliary_device_create(struct device *dev,
-+						 const char *modname,
-+						 const char *devname,
-+						 void *platform_data,
-+						 int id);
-+void auxiliary_device_destroy(void *auxdev);
-+
-+struct auxiliary_device *__devm_auxiliary_device_create(struct device *dev,
-+							const char *modname,
-+							const char *devname,
-+							void *platform_data,
-+							int id);
-+
-+#define devm_auxiliary_device_create(dev, devname, platform_data)     \
-+	__devm_auxiliary_device_create(dev, KBUILD_MODNAME, devname,  \
-+				       platform_data, 0)
-+
- /**
-  * module_auxiliary_driver() - Helper macro for registering an auxiliary driver
-  * @__auxiliary_driver: auxiliary driver struct
-@@ -244,8 +286,4 @@ void auxiliary_driver_unregister(struct auxiliary_driver *auxdrv);
- #define module_auxiliary_driver(__auxiliary_driver) \
- 	module_driver(__auxiliary_driver, auxiliary_driver_register, auxiliary_driver_unregister)
- 
-struct auxiliary_device *auxiliary_find_device(struct device *start,
-					       const void *data,
-					       int (*match)(struct device *dev, const void *data));
-
- #endif /* _AUXILIARY_BUS_H_ */
-diff --git a/include/linux/auxvec.h b/include/linux/auxvec.h
-index f68d0ec2d740..407f7005e6d6 100644
--- a/include/linux/auxvec.h
-+++ b/include/linux/auxvec.h
-@@ -4,6 +4,6 @@
- 
- #include <uapi/linux/auxvec.h>
- 
-#define AT_VECTOR_SIZE_BASE 20 /* NEW_AUX_ENT entries in auxiliary table */
-+#define AT_VECTOR_SIZE_BASE 22 /* NEW_AUX_ENT entries in auxiliary table */
-   /* number of "#define AT_.*" above, minus {AT_NULL, AT_IGNORE, AT_NOTELF} */
- #endif /* _LINUX_AUXVEC_H */
-diff --git a/include/linux/avf/virtchnl.h b/include/linux/avf/virtchnl.h
-index 2ce27e8e4f19..11bdab5522fd 100644
--- a/include/linux/avf/virtchnl.h
-+++ b/include/linux/avf/virtchnl.h
-@@ -1,21 +1,18 @@
- /* SPDX-License-Identifier: GPL-2.0-only */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
-+/* Copyright (c) 2013-2022, Intel Corporation. */
- 
- #ifndef _VIRTCHNL_H_
- #define _VIRTCHNL_H_
- 
-+#include <linux/bitops.h>
-+#include <linux/bits.h>
-+#include <linux/overflow.h>
-+#include <uapi/linux/if_ether.h>
-+
- /* Description:
- * This header file describes the VF-PF communication protocol used
- * by the drivers for all devices starting from our 40G product line
-+ * This header file describes the Virtual Function (VF) - Physical Function
-+ * (PF) communication protocol used by the drivers for all devices starting
-+ * from our 40G product line
-  *
-  * Admin queue buffer usage:
-  * desc->opcode is always aqc_opc_send_msg_to_pf
-@@ -29,8 +26,8 @@
-  * have a maximum of sixteen queues for all of its VSIs.
-  *
-  * The PF is required to return a status code in v_retval for all messages
- * except RESET_VF, which does not require any response. The return value
- * is of status_code type, defined in the shared type.h.
-+ * except RESET_VF, which does not require any response. The returned value
-+ * is of virtchnl_status_code type, defined here.
-  *
-  * In general, VF driver initialization should roughly follow the order of
-  * these opcodes. The VF driver must first validate the API version of the
-@@ -92,6 +89,9 @@ enum virtchnl_rx_hsplit {
- 	VIRTCHNL_RX_HSPLIT_SPLIT_SCTP    = 8,
- };
- 
-+enum virtchnl_bw_limit_type {
-+	VIRTCHNL_BW_SHAPER = 0,
-+};
- /* END GENERIC DEFINES */
- 
- /* Opcodes for VF-PF communication. These are placed in the v_opcode field
-@@ -122,13 +122,18 @@ enum virtchnl_ops {
- 	VIRTCHNL_OP_GET_STATS = 15,
- 	VIRTCHNL_OP_RSVD = 16,
- 	VIRTCHNL_OP_EVENT = 17, /* must ALWAYS be 17 */
-+	VIRTCHNL_OP_CONFIG_RSS_HFUNC = 18,
-+	/* opcode 19 is reserved */
- 	VIRTCHNL_OP_IWARP = 20, /* advanced opcode */
-+	VIRTCHNL_OP_RDMA = VIRTCHNL_OP_IWARP,
- 	VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP = 21, /* advanced opcode */
-+	VIRTCHNL_OP_CONFIG_RDMA_IRQ_MAP = VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP,
- 	VIRTCHNL_OP_RELEASE_IWARP_IRQ_MAP = 22, /* advanced opcode */
-+	VIRTCHNL_OP_RELEASE_RDMA_IRQ_MAP = VIRTCHNL_OP_RELEASE_IWARP_IRQ_MAP,
- 	VIRTCHNL_OP_CONFIG_RSS_KEY = 23,
- 	VIRTCHNL_OP_CONFIG_RSS_LUT = 24,
-	VIRTCHNL_OP_GET_RSS_HENA_CAPS = 25,
-	VIRTCHNL_OP_SET_RSS_HENA = 26,
-+	VIRTCHNL_OP_GET_RSS_HASHCFG_CAPS = 25,
-+	VIRTCHNL_OP_SET_RSS_HASHCFG = 26,
- 	VIRTCHNL_OP_ENABLE_VLAN_STRIPPING = 27,
- 	VIRTCHNL_OP_DISABLE_VLAN_STRIPPING = 28,
- 	VIRTCHNL_OP_REQUEST_QUEUES = 29,
-@@ -136,7 +141,8 @@ enum virtchnl_ops {
- 	VIRTCHNL_OP_DISABLE_CHANNELS = 31,
- 	VIRTCHNL_OP_ADD_CLOUD_FILTER = 32,
- 	VIRTCHNL_OP_DEL_CLOUD_FILTER = 33,
-	/* opcode 34 - 44 are reserved */
-+	/* opcode 34 - 43 are reserved */
-+	VIRTCHNL_OP_GET_SUPPORTED_RXDIDS = 44,
- 	VIRTCHNL_OP_ADD_RSS_CFG = 45,
- 	VIRTCHNL_OP_DEL_RSS_CFG = 46,
- 	VIRTCHNL_OP_ADD_FDIR_FILTER = 47,
-@@ -148,6 +154,14 @@ enum virtchnl_ops {
- 	VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2 = 55,
- 	VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2 = 56,
- 	VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2 = 57,
-+	/* opcode 58 and 59 are reserved */
-+	VIRTCHNL_OP_1588_PTP_GET_CAPS = 60,
-+	VIRTCHNL_OP_1588_PTP_GET_TIME = 61,
-+	/* opcode 62 - 65 are reserved */
-+	VIRTCHNL_OP_GET_QOS_CAPS = 66,
-+	/* opcode 68 through 111 are reserved */
-+	VIRTCHNL_OP_CONFIG_QUEUE_BW = 112,
-+	VIRTCHNL_OP_CONFIG_QUANTA = 113,
- 	VIRTCHNL_OP_MAX,
- };
- 
-@@ -161,19 +175,6 @@ enum virtchnl_ops {
- #define VIRTCHNL_CHECK_UNION_LEN(n, X) enum virtchnl_static_asset_enum_##X \
- 	{ virtchnl_static_assert_##X = (n)/((sizeof(union X) == (n)) ? 1 : 0) }
- 
-/* Virtual channel message descriptor. This overlays the admin queue
- * descriptor. All other data is passed in external buffers.
- */
-
-struct virtchnl_msg {
-	u8 pad[8];			 /* AQ flags/opcode/len/retval fields */
-	enum virtchnl_ops v_opcode; /* avoid confusion with desc->opcode */
-	enum virtchnl_status_code v_retval;  /* ditto for desc->retval */
-	u32 vfid;			 /* used by PF when sending to VF */
-};
-
-VIRTCHNL_CHECK_STRUCT_LEN(20, virtchnl_msg);
-
- /* Message descriptions and data structures. */
- 
- /* VIRTCHNL_OP_VERSION
-@@ -234,7 +235,9 @@ enum virtchnl_vsi_type {
- struct virtchnl_vsi_resource {
- 	u16 vsi_id;
- 	u16 num_queue_pairs;
-	enum virtchnl_vsi_type vsi_type;
-+
-+	/* see enum virtchnl_vsi_type */
-+	s32 vsi_type;
- 	u16 qset_handle;
- 	u8 default_mac_addr[ETH_ALEN];
- };
-@@ -246,13 +249,16 @@ VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_vsi_resource);
-  * TX/RX Checksum offloading and TSO for non-tunnelled packets.
-  */
- #define VIRTCHNL_VF_OFFLOAD_L2			BIT(0)
-#define VIRTCHNL_VF_OFFLOAD_IWARP		BIT(1)
-+#define VIRTCHNL_VF_OFFLOAD_RDMA		BIT(1)
-+#define VIRTCHNL_VF_CAP_RDMA			VIRTCHNL_VF_OFFLOAD_RDMA
- #define VIRTCHNL_VF_OFFLOAD_RSS_AQ		BIT(3)
- #define VIRTCHNL_VF_OFFLOAD_RSS_REG		BIT(4)
- #define VIRTCHNL_VF_OFFLOAD_WB_ON_ITR		BIT(5)
- #define VIRTCHNL_VF_OFFLOAD_REQ_QUEUES		BIT(6)
- /* used to negotiate communicating link speeds in Mbps */
- #define VIRTCHNL_VF_CAP_ADV_LINK_SPEED		BIT(7)
-+#define  VIRTCHNL_VF_OFFLOAD_CRC		BIT(10)
-+#define VIRTCHNL_VF_OFFLOAD_TC_U32		BIT(11)
- #define VIRTCHNL_VF_OFFLOAD_VLAN_V2		BIT(15)
- #define VIRTCHNL_VF_OFFLOAD_VLAN		BIT(16)
- #define VIRTCHNL_VF_OFFLOAD_RX_POLLING		BIT(17)
-@@ -263,8 +269,11 @@ VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_vsi_resource);
- #define VIRTCHNL_VF_OFFLOAD_RX_ENCAP_CSUM	BIT(22)
- #define VIRTCHNL_VF_OFFLOAD_ADQ			BIT(23)
- #define VIRTCHNL_VF_OFFLOAD_USO			BIT(25)
-+#define VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC	BIT(26)
- #define VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF		BIT(27)
- #define VIRTCHNL_VF_OFFLOAD_FDIR_PF		BIT(28)
-+#define VIRTCHNL_VF_OFFLOAD_QOS			BIT(29)
-+#define VIRTCHNL_VF_CAP_PTP			BIT(31)
- 
- #define VF_BASE_MODE_OFFLOADS (VIRTCHNL_VF_OFFLOAD_L2 | \
- 			       VIRTCHNL_VF_OFFLOAD_VLAN | \
-@@ -280,10 +289,11 @@ struct virtchnl_vf_resource {
- 	u32 rss_key_size;
- 	u32 rss_lut_size;
- 
-	struct virtchnl_vsi_resource vsi_res[1];
-+	struct virtchnl_vsi_resource vsi_res[];
- };
- 
-VIRTCHNL_CHECK_STRUCT_LEN(36, virtchnl_vf_resource);
-+VIRTCHNL_CHECK_STRUCT_LEN(20, virtchnl_vf_resource);
-+#define virtchnl_vf_resource_LEGACY_SIZEOF	36
- 
- /* VIRTCHNL_OP_CONFIG_TX_QUEUE
-  * VF sends this message to set up parameters for one TX queue.
-@@ -303,10 +313,70 @@ struct virtchnl_txq_info {
- 
- VIRTCHNL_CHECK_STRUCT_LEN(24, virtchnl_txq_info);
- 
-+/* RX descriptor IDs (range from 0 to 63) */
-+enum virtchnl_rx_desc_ids {
-+	VIRTCHNL_RXDID_0_16B_BASE		= 0,
-+	VIRTCHNL_RXDID_1_32B_BASE		= 1,
-+	VIRTCHNL_RXDID_2_FLEX_SQ_NIC		= 2,
-+	VIRTCHNL_RXDID_3_FLEX_SQ_SW		= 3,
-+	VIRTCHNL_RXDID_4_FLEX_SQ_NIC_VEB	= 4,
-+	VIRTCHNL_RXDID_5_FLEX_SQ_NIC_ACL	= 5,
-+	VIRTCHNL_RXDID_6_FLEX_SQ_NIC_2		= 6,
-+	VIRTCHNL_RXDID_7_HW_RSVD		= 7,
-+	/* 8 through 15 are reserved */
-+	VIRTCHNL_RXDID_16_COMMS_GENERIC		= 16,
-+	VIRTCHNL_RXDID_17_COMMS_AUX_VLAN	= 17,
-+	VIRTCHNL_RXDID_18_COMMS_AUX_IPV4	= 18,
-+	VIRTCHNL_RXDID_19_COMMS_AUX_IPV6	= 19,
-+	VIRTCHNL_RXDID_20_COMMS_AUX_FLOW	= 20,
-+	VIRTCHNL_RXDID_21_COMMS_AUX_TCP		= 21,
-+	/* 22 through 63 are reserved */
-+};
-+
-+#define VIRTCHNL_RXDID_BIT(x)			BIT_ULL(VIRTCHNL_RXDID_##x)
-+
-+/* RX descriptor ID bitmasks */
-+enum virtchnl_rx_desc_id_bitmasks {
-+	VIRTCHNL_RXDID_0_16B_BASE_M		= VIRTCHNL_RXDID_BIT(0_16B_BASE),
-+	VIRTCHNL_RXDID_1_32B_BASE_M		= VIRTCHNL_RXDID_BIT(1_32B_BASE),
-+	VIRTCHNL_RXDID_2_FLEX_SQ_NIC_M		= VIRTCHNL_RXDID_BIT(2_FLEX_SQ_NIC),
-+	VIRTCHNL_RXDID_3_FLEX_SQ_SW_M		= VIRTCHNL_RXDID_BIT(3_FLEX_SQ_SW),
-+	VIRTCHNL_RXDID_4_FLEX_SQ_NIC_VEB_M	= VIRTCHNL_RXDID_BIT(4_FLEX_SQ_NIC_VEB),
-+	VIRTCHNL_RXDID_5_FLEX_SQ_NIC_ACL_M	= VIRTCHNL_RXDID_BIT(5_FLEX_SQ_NIC_ACL),
-+	VIRTCHNL_RXDID_6_FLEX_SQ_NIC_2_M	= VIRTCHNL_RXDID_BIT(6_FLEX_SQ_NIC_2),
-+	VIRTCHNL_RXDID_7_HW_RSVD_M		= VIRTCHNL_RXDID_BIT(7_HW_RSVD),
-+	/* 8 through 15 are reserved */
-+	VIRTCHNL_RXDID_16_COMMS_GENERIC_M	= VIRTCHNL_RXDID_BIT(16_COMMS_GENERIC),
-+	VIRTCHNL_RXDID_17_COMMS_AUX_VLAN_M	= VIRTCHNL_RXDID_BIT(17_COMMS_AUX_VLAN),
-+	VIRTCHNL_RXDID_18_COMMS_AUX_IPV4_M	= VIRTCHNL_RXDID_BIT(18_COMMS_AUX_IPV4),
-+	VIRTCHNL_RXDID_19_COMMS_AUX_IPV6_M	= VIRTCHNL_RXDID_BIT(19_COMMS_AUX_IPV6),
-+	VIRTCHNL_RXDID_20_COMMS_AUX_FLOW_M	= VIRTCHNL_RXDID_BIT(20_COMMS_AUX_FLOW),
-+	VIRTCHNL_RXDID_21_COMMS_AUX_TCP_M	= VIRTCHNL_RXDID_BIT(21_COMMS_AUX_TCP),
-+	/* 22 through 63 are reserved */
-+};
-+
-+/* virtchnl_rxq_info_flags - definition of bits in the flags field of the
-+ *			     virtchnl_rxq_info structure.
-+ *
-+ * @VIRTCHNL_PTP_RX_TSTAMP: request to enable Rx timestamping
-+ *
-+ * Other flag bits are currently reserved and they may be extended in the
-+ * future.
-+ */
-+enum virtchnl_rxq_info_flags {
-+	VIRTCHNL_PTP_RX_TSTAMP = BIT(0),
-+};
-+
- /* VIRTCHNL_OP_CONFIG_RX_QUEUE
-  * VF sends this message to set up parameters for one RX queue.
-  * External data buffer contains one instance of virtchnl_rxq_info.
- * PF configures requested queue and returns a status code.
-+ * PF configures requested queue and returns a status code. The
-+ * crc_disable flag disables CRC stripping on the VF. Setting
-+ * the crc_disable flag to 1 will disable CRC stripping for each
-+ * queue in the VF where the flag is set. The VIRTCHNL_VF_OFFLOAD_CRC
-+ * offload must have been set prior to sending this info or the PF
-+ * will ignore the request. This flag should be set the same for
-+ * all of the queues for a VF.
-  */
- 
- /* Rx queue config info */
-@@ -318,9 +388,19 @@ struct virtchnl_rxq_info {
- 	u16 splithdr_enabled; /* deprecated with AVF 1.0 */
- 	u32 databuffer_size;
- 	u32 max_pkt_size;
-	u32 pad1;
-+	u8 crc_disable;
-+	/* see enum virtchnl_rx_desc_ids;
-+	 * only used when VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC is supported. Note
-+	 * that when the offload is not supported, the descriptor format aligns
-+	 * with VIRTCHNL_RXDID_1_32B_BASE.
-+	 */
-+	enum virtchnl_rx_desc_ids rxdid:8;
-+	enum virtchnl_rxq_info_flags flags:8; /* see virtchnl_rxq_info_flags */
-+	u8 pad1;
- 	u64 dma_ring_addr;
-	enum virtchnl_rx_hsplit rx_split_pos; /* deprecated with AVF 1.0 */
-+
-+	/* see enum virtchnl_rx_hsplit; deprecated with AVF 1.0 */
-+	s32 rx_split_pos;
- 	u32 pad2;
- };
- 
-@@ -332,6 +412,9 @@ VIRTCHNL_CHECK_STRUCT_LEN(40, virtchnl_rxq_info);
-  * PF configures queues and returns status.
-  * If the number of queues specified is greater than the number of queues
-  * associated with the VSI, an error is returned and no queues are configured.
-+ * NOTE: The VF is not required to configure all queues in a single request.
-+ * It may send multiple messages. PF drivers must correctly handle all VF
-+ * requests.
-  */
- struct virtchnl_queue_pair_info {
- 	/* NOTE: vsi_id and queue_id should be identical for both queues. */
-@@ -345,10 +428,11 @@ struct virtchnl_vsi_queue_config_info {
- 	u16 vsi_id;
- 	u16 num_queue_pairs;
- 	u32 pad;
-	struct virtchnl_queue_pair_info qpair[1];
-+	struct virtchnl_queue_pair_info qpair[];
- };
- 
-VIRTCHNL_CHECK_STRUCT_LEN(72, virtchnl_vsi_queue_config_info);
-+VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_vsi_queue_config_info);
-+#define virtchnl_vsi_queue_config_info_LEGACY_SIZEOF	72
- 
- /* VIRTCHNL_OP_REQUEST_QUEUES
-  * VF sends this message to request the PF to allocate additional queues to
-@@ -369,8 +453,13 @@ struct virtchnl_vf_res_request {
-  * VF uses this message to map vectors to queues.
-  * The rxq_map and txq_map fields are bitmaps used to indicate which queues
-  * are to be associated with the specified vector.
- * The "other" causes are always mapped to vector 0.
-+ * The "other" causes are always mapped to vector 0. The VF may not request
-+ * that vector 0 be used for traffic.
-  * PF configures interrupt mapping and returns status.
-+ * NOTE: due to hardware requirements, all active queues (both TX and RX)
-+ * should be mapped to interrupts, even if the driver intends to operate
-+ * only in polling mode. In this case the interrupt may be disabled, but
-+ * the ITR timer will still run to trigger writebacks.
-  */
- struct virtchnl_vector_map {
- 	u16 vsi_id;
-@@ -385,10 +474,11 @@ VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_vector_map);
- 
- struct virtchnl_irq_map_info {
- 	u16 num_vectors;
-	struct virtchnl_vector_map vecmap[1];
-+	struct virtchnl_vector_map vecmap[];
- };
- 
-VIRTCHNL_CHECK_STRUCT_LEN(14, virtchnl_irq_map_info);
-+VIRTCHNL_CHECK_STRUCT_LEN(2, virtchnl_irq_map_info);
-+#define virtchnl_irq_map_info_LEGACY_SIZEOF	14
- 
- /* VIRTCHNL_OP_ENABLE_QUEUES
-  * VIRTCHNL_OP_DISABLE_QUEUES
-@@ -397,6 +487,9 @@ VIRTCHNL_CHECK_STRUCT_LEN(14, virtchnl_irq_map_info);
-  * (Currently, we only support 16 queues per VF, but we make the field
-  * u32 to allow for expansion.)
-  * PF performs requested action and returns status.
-+ * NOTE: The VF is not required to enable/disable all queues in a single
-+ * request. It may send multiple messages.
-+ * PF drivers must correctly handle all VF requests.
-  */
- struct virtchnl_queue_select {
- 	u16 vsi_id;
-@@ -456,10 +549,11 @@ VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_ether_addr);
- struct virtchnl_ether_addr_list {
- 	u16 vsi_id;
- 	u16 num_elements;
-	struct virtchnl_ether_addr list[1];
-+	struct virtchnl_ether_addr list[];
- };
- 
-VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_ether_addr_list);
-+VIRTCHNL_CHECK_STRUCT_LEN(4, virtchnl_ether_addr_list);
-+#define virtchnl_ether_addr_list_LEGACY_SIZEOF	12
- 
- /* VIRTCHNL_OP_ADD_VLAN
-  * VF sends this message to add one or more VLAN tag filters for receives.
-@@ -478,10 +572,11 @@ VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_ether_addr_list);
- struct virtchnl_vlan_filter_list {
- 	u16 vsi_id;
- 	u16 num_elements;
-	u16 vlan_id[1];
-+	u16 vlan_id[];
- };
- 
-VIRTCHNL_CHECK_STRUCT_LEN(6, virtchnl_vlan_filter_list);
-+VIRTCHNL_CHECK_STRUCT_LEN(4, virtchnl_vlan_filter_list);
-+#define virtchnl_vlan_filter_list_LEGACY_SIZEOF	6
- 
- /* This enum is used for all of the VIRTCHNL_VF_OFFLOAD_VLAN_V2_CAPS related
-  * structures and opcodes.
-@@ -708,10 +803,11 @@ struct virtchnl_vlan_filter_list_v2 {
- 	u16 vport_id;
- 	u16 num_elements;
- 	u8 pad[4];
-	struct virtchnl_vlan_filter filters[1];
-+	struct virtchnl_vlan_filter filters[];
- };
- 
-VIRTCHNL_CHECK_STRUCT_LEN(40, virtchnl_vlan_filter_list_v2);
-+VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_vlan_filter_list_v2);
-+#define virtchnl_vlan_filter_list_v2_LEGACY_SIZEOF	40
- 
- /* VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2
-  * VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2
-@@ -863,31 +959,57 @@ VIRTCHNL_CHECK_STRUCT_LEN(4, virtchnl_promisc_info);
- struct virtchnl_rss_key {
- 	u16 vsi_id;
- 	u16 key_len;
-	u8 key[1];         /* RSS hash key, packed bytes */
-+	u8 key[];          /* RSS hash key, packed bytes */
- };
- 
-VIRTCHNL_CHECK_STRUCT_LEN(6, virtchnl_rss_key);
-+VIRTCHNL_CHECK_STRUCT_LEN(4, virtchnl_rss_key);
-+#define virtchnl_rss_key_LEGACY_SIZEOF	6
- 
- struct virtchnl_rss_lut {
- 	u16 vsi_id;
- 	u16 lut_entries;
-	u8 lut[1];        /* RSS lookup table */
-+	u8 lut[];         /* RSS lookup table */
- };
- 
-VIRTCHNL_CHECK_STRUCT_LEN(6, virtchnl_rss_lut);
-+VIRTCHNL_CHECK_STRUCT_LEN(4, virtchnl_rss_lut);
-+#define virtchnl_rss_lut_LEGACY_SIZEOF	6
- 
-/* VIRTCHNL_OP_GET_RSS_HENA_CAPS
- * VIRTCHNL_OP_SET_RSS_HENA
- * VF sends these messages to get and set the hash filter enable bits for RSS.
-+/* VIRTCHNL_OP_GET_RSS_HASHCFG_CAPS
-+ * VIRTCHNL_OP_SET_RSS_HASHCFG
-+ * VF sends these messages to get and set the hash filter configuration for RSS.
-  * By default, the PF sets these to all possible traffic types that the
-  * hardware supports. The VF can query this value if it wants to change the
-  * traffic types that are hashed by the hardware.
-  */
-struct virtchnl_rss_hena {
-	u64 hena;
-+struct virtchnl_rss_hashcfg {
-+	/* Bits defined by enum libie_filter_pctype */
-+	u64 hashcfg;
-+};
-+
-+VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_rss_hashcfg);
-+
-+/* Type of RSS algorithm */
-+enum virtchnl_rss_algorithm {
-+	VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC	= 0,
-+	VIRTCHNL_RSS_ALG_R_ASYMMETRIC		= 1,
-+	VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC	= 2,
-+	VIRTCHNL_RSS_ALG_XOR_SYMMETRIC		= 3,
-+};
-+
-+/* VIRTCHNL_OP_CONFIG_RSS_HFUNC
-+ * VF sends this message to configure the RSS hash function. Only supported
-+ * if both PF and VF drivers set the VIRTCHNL_VF_OFFLOAD_RSS_PF bit during
-+ * configuration negotiation.
-+ * The hash function is initialized to VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC
-+ * by the PF.
-+ */
-+struct virtchnl_rss_hfunc {
-+	u16 vsi_id;
-+	u16 rss_algorithm; /* enum virtchnl_rss_algorithm */
-+	u32 reserved;
- };
- 
-VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_rss_hena);
-+VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_rss_hfunc);
- 
- /* VIRTCHNL_OP_ENABLE_CHANNELS
-  * VIRTCHNL_OP_DISABLE_CHANNELS
-@@ -908,10 +1030,11 @@ VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_channel_info);
- struct virtchnl_tc_info {
- 	u32	num_tc;
- 	u32	pad;
-	struct	virtchnl_channel_info list[1];
-+	struct virtchnl_channel_info list[];
- };
- 
-VIRTCHNL_CHECK_STRUCT_LEN(24, virtchnl_tc_info);
-+VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_tc_info);
-+#define virtchnl_tc_info_LEGACY_SIZEOF	24
- 
- /* VIRTCHNL_ADD_CLOUD_FILTER
-  * VIRTCHNL_DEL_CLOUD_FILTER
-@@ -961,8 +1084,12 @@ enum virtchnl_flow_type {
- struct virtchnl_filter {
- 	union	virtchnl_flow_spec data;
- 	union	virtchnl_flow_spec mask;
-	enum	virtchnl_flow_type flow_type;
-	enum	virtchnl_action action;
-+
-+	/* see enum virtchnl_flow_type */
-+	s32	flow_type;
-+
-+	/* see enum virtchnl_action */
-+	s32	action;
- 	u32	action_meta;
- 	u8	field_flags;
- 	u8	pad[3];
-@@ -986,7 +1113,8 @@ enum virtchnl_event_codes {
- #define PF_EVENT_SEVERITY_CERTAIN_DOOM	255
- 
- struct virtchnl_pf_event {
-	enum virtchnl_event_codes event;
-+	/* see enum virtchnl_event_codes */
-+	s32 event;
- 	union {
- 		/* If the PF driver does not support the new speed reporting
- 		 * capabilities then use link_event else use link_event_adv to
-@@ -999,6 +1127,7 @@ struct virtchnl_pf_event {
- 		struct {
- 			enum virtchnl_link_speed link_speed;
- 			bool link_status;
-+			u8 pad[3];
- 		} link_event;
- 		struct {
- 			/* link_speed provided in Mbps */
-@@ -1008,39 +1137,42 @@ struct virtchnl_pf_event {
- 		} link_event_adv;
- 	} event_data;
- 
-	int severity;
-+	s32 severity;
- };
- 
- VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_pf_event);
- 
-/* VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP
- * VF uses this message to request PF to map IWARP vectors to IWARP queues.
- * The request for this originates from the VF IWARP driver through
- * a client interface between VF LAN and VF IWARP driver.
-+/* used to specify if a ceq_idx or aeq_idx is invalid */
-+#define VIRTCHNL_RDMA_INVALID_QUEUE_IDX	0xFFFF
-+/* VIRTCHNL_OP_CONFIG_RDMA_IRQ_MAP
-+ * VF uses this message to request PF to map RDMA vectors to RDMA queues.
-+ * The request for this originates from the VF RDMA driver through
-+ * a client interface between VF LAN and VF RDMA driver.
-  * A vector could have an AEQ and CEQ attached to it although
- * there is a single AEQ per VF IWARP instance in which case
- * most vectors will have an INVALID_IDX for aeq and valid idx for ceq.
- * There will never be a case where there will be multiple CEQs attached
- * to a single vector.
-+ * there is a single AEQ per VF RDMA instance in which case
-+ * most vectors will have an VIRTCHNL_RDMA_INVALID_QUEUE_IDX for aeq and valid
-+ * idx for ceqs There will never be a case where there will be multiple CEQs
-+ * attached to a single vector.
-  * PF configures interrupt mapping and returns status.
-  */
- 
-struct virtchnl_iwarp_qv_info {
-+struct virtchnl_rdma_qv_info {
- 	u32 v_idx; /* msix_vector */
-	u16 ceq_idx;
-	u16 aeq_idx;
-+	u16 ceq_idx; /* set to VIRTCHNL_RDMA_INVALID_QUEUE_IDX if invalid */
-+	u16 aeq_idx; /* set to VIRTCHNL_RDMA_INVALID_QUEUE_IDX if invalid */
- 	u8 itr_idx;
- 	u8 pad[3];
- };
- 
-VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_iwarp_qv_info);
-+VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_rdma_qv_info);
- 
-struct virtchnl_iwarp_qvlist_info {
-+struct virtchnl_rdma_qvlist_info {
- 	u32 num_vectors;
-	struct virtchnl_iwarp_qv_info qv_info[1];
-+	struct virtchnl_rdma_qv_info qv_info[];
- };
- 
-VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_iwarp_qvlist_info);
-+VIRTCHNL_CHECK_STRUCT_LEN(4, virtchnl_rdma_qvlist_info);
-+#define virtchnl_rdma_qvlist_info_LEGACY_SIZEOF	16
- 
- /* VF reset states - these are written into the RSTAT register:
-  * VFGEN_RSTAT on the VF
-@@ -1059,15 +1191,8 @@ enum virtchnl_vfr_states {
- 	VIRTCHNL_VFR_VFACTIVE,
- };
- 
-/* Type of RSS algorithm */
-enum virtchnl_rss_algorithm {
-	VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC	= 0,
-	VIRTCHNL_RSS_ALG_R_ASYMMETRIC		= 1,
-	VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC	= 2,
-	VIRTCHNL_RSS_ALG_XOR_SYMMETRIC		= 3,
-};
-
- #define VIRTCHNL_MAX_NUM_PROTO_HDRS	32
-+#define VIRTCHNL_MAX_SIZE_RAW_PACKET	1024
- #define PROTO_HDR_SHIFT			5
- #define PROTO_HDR_FIELD_START(proto_hdr_type) ((proto_hdr_type) << PROTO_HDR_SHIFT)
- #define PROTO_HDR_FIELD_MASK ((1UL << PROTO_HDR_SHIFT) - 1)
-@@ -1099,7 +1224,7 @@ enum virtchnl_rss_algorithm {
- #define VIRTCHNL_GET_PROTO_HDR_TYPE(hdr) \
- 	(((hdr)->type) >> PROTO_HDR_SHIFT)
- #define VIRTCHNL_TEST_PROTO_HDR_TYPE(hdr, val) \
-	((hdr)->type == ((val) >> PROTO_HDR_SHIFT))
-+	((hdr)->type == ((s32)((val) >> PROTO_HDR_SHIFT)))
- #define VIRTCHNL_TEST_PROTO_HDR(hdr, val) \
- 	(VIRTCHNL_TEST_PROTO_HDR_TYPE((hdr), (val)) && \
- 	 VIRTCHNL_TEST_PROTO_HDR_FIELD((hdr), (val)))
-@@ -1128,6 +1253,17 @@ enum virtchnl_proto_hdr_type {
- 	VIRTCHNL_PROTO_HDR_ESP,
- 	VIRTCHNL_PROTO_HDR_AH,
- 	VIRTCHNL_PROTO_HDR_PFCP,
-+	VIRTCHNL_PROTO_HDR_GTPC,
-+	VIRTCHNL_PROTO_HDR_ECPRI,
-+	VIRTCHNL_PROTO_HDR_L2TPV2,
-+	VIRTCHNL_PROTO_HDR_PPP,
-+	/* IPv4 and IPv6 Fragment header types are only associated to
-+	 * VIRTCHNL_PROTO_HDR_IPV4 and VIRTCHNL_PROTO_HDR_IPV6 respectively,
-+	 * cannot be used independently.
-+	 */
-+	VIRTCHNL_PROTO_HDR_IPV4_FRAG,
-+	VIRTCHNL_PROTO_HDR_IPV6_EH_FRAG,
-+	VIRTCHNL_PROTO_HDR_GRE,
- };
- 
- /* Protocol header field within a protocol header. */
-@@ -1150,6 +1286,7 @@ enum virtchnl_proto_hdr_field {
- 	VIRTCHNL_PROTO_HDR_IPV4_DSCP,
- 	VIRTCHNL_PROTO_HDR_IPV4_TTL,
- 	VIRTCHNL_PROTO_HDR_IPV4_PROT,
-+	VIRTCHNL_PROTO_HDR_IPV4_CHKSUM,
- 	/* IPV6 */
- 	VIRTCHNL_PROTO_HDR_IPV6_SRC =
- 		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_IPV6),
-@@ -1157,18 +1294,34 @@ enum virtchnl_proto_hdr_field {
- 	VIRTCHNL_PROTO_HDR_IPV6_TC,
- 	VIRTCHNL_PROTO_HDR_IPV6_HOP_LIMIT,
- 	VIRTCHNL_PROTO_HDR_IPV6_PROT,
-+	/* IPV6 Prefix */
-+	VIRTCHNL_PROTO_HDR_IPV6_PREFIX32_SRC,
-+	VIRTCHNL_PROTO_HDR_IPV6_PREFIX32_DST,
-+	VIRTCHNL_PROTO_HDR_IPV6_PREFIX40_SRC,
-+	VIRTCHNL_PROTO_HDR_IPV6_PREFIX40_DST,
-+	VIRTCHNL_PROTO_HDR_IPV6_PREFIX48_SRC,
-+	VIRTCHNL_PROTO_HDR_IPV6_PREFIX48_DST,
-+	VIRTCHNL_PROTO_HDR_IPV6_PREFIX56_SRC,
-+	VIRTCHNL_PROTO_HDR_IPV6_PREFIX56_DST,
-+	VIRTCHNL_PROTO_HDR_IPV6_PREFIX64_SRC,
-+	VIRTCHNL_PROTO_HDR_IPV6_PREFIX64_DST,
-+	VIRTCHNL_PROTO_HDR_IPV6_PREFIX96_SRC,
-+	VIRTCHNL_PROTO_HDR_IPV6_PREFIX96_DST,
- 	/* TCP */
- 	VIRTCHNL_PROTO_HDR_TCP_SRC_PORT =
- 		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_TCP),
- 	VIRTCHNL_PROTO_HDR_TCP_DST_PORT,
-+	VIRTCHNL_PROTO_HDR_TCP_CHKSUM,
- 	/* UDP */
- 	VIRTCHNL_PROTO_HDR_UDP_SRC_PORT =
- 		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_UDP),
- 	VIRTCHNL_PROTO_HDR_UDP_DST_PORT,
-+	VIRTCHNL_PROTO_HDR_UDP_CHKSUM,
- 	/* SCTP */
- 	VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT =
- 		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_SCTP),
- 	VIRTCHNL_PROTO_HDR_SCTP_DST_PORT,
-+	VIRTCHNL_PROTO_HDR_SCTP_CHKSUM,
- 	/* GTPU_IP */
- 	VIRTCHNL_PROTO_HDR_GTPU_IP_TEID =
- 		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_GTPU_IP),
-@@ -1192,10 +1345,33 @@ enum virtchnl_proto_hdr_field {
- 	VIRTCHNL_PROTO_HDR_PFCP_S_FIELD =
- 		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_PFCP),
- 	VIRTCHNL_PROTO_HDR_PFCP_SEID,
-+	/* GTPC */
-+	VIRTCHNL_PROTO_HDR_GTPC_TEID =
-+		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_GTPC),
-+	/* ECPRI */
-+	VIRTCHNL_PROTO_HDR_ECPRI_MSG_TYPE =
-+		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_ECPRI),
-+	VIRTCHNL_PROTO_HDR_ECPRI_PC_RTC_ID,
-+	/* IPv4 Dummy Fragment */
-+	VIRTCHNL_PROTO_HDR_IPV4_FRAG_PKID =
-+		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_IPV4_FRAG),
-+	/* IPv6 Extension Fragment */
-+	VIRTCHNL_PROTO_HDR_IPV6_EH_FRAG_PKID =
-+		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_IPV6_EH_FRAG),
-+	/* GTPU_DWN/UP */
-+	VIRTCHNL_PROTO_HDR_GTPU_DWN_QFI =
-+		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_GTPU_EH_PDU_DWN),
-+	VIRTCHNL_PROTO_HDR_GTPU_UP_QFI =
-+		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_GTPU_EH_PDU_UP),
-+	/* L2TPv2 */
-+	VIRTCHNL_PROTO_HDR_L2TPV2_SESS_ID =
-+		PROTO_HDR_FIELD_START(VIRTCHNL_PROTO_HDR_L2TPV2),
-+	VIRTCHNL_PROTO_HDR_L2TPV2_LEN_SESS_ID,
- };
- 
- struct virtchnl_proto_hdr {
-	enum virtchnl_proto_hdr_type type;
-+	/* see enum virtchnl_proto_hdr_type */
-+	s32 type;
- 	u32 field_selector; /* a bit mask to select field for header type */
- 	u8 buffer[64];
- 	/**
-@@ -1212,28 +1388,40 @@ struct virtchnl_proto_hdrs {
- 	u8 pad[3];
- 	/**
- 	 * specify where protocol header start from.
-+	 * must be 0 when sending a raw packet request.
- 	 * 0 - from the outer layer
- 	 * 1 - from the first inner layer
- 	 * 2 - from the second inner layer
- 	 * ....
- 	 **/
-	int count; /* the proto layers must < VIRTCHNL_MAX_NUM_PROTO_HDRS */
-	struct virtchnl_proto_hdr proto_hdr[VIRTCHNL_MAX_NUM_PROTO_HDRS];
-+	u32 count; /* the proto layers must < VIRTCHNL_MAX_NUM_PROTO_HDRS */
-+	union {
-+		struct virtchnl_proto_hdr
-+			proto_hdr[VIRTCHNL_MAX_NUM_PROTO_HDRS];
-+		struct {
-+			u16 pkt_len;
-+			u8 spec[VIRTCHNL_MAX_SIZE_RAW_PACKET];
-+			u8 mask[VIRTCHNL_MAX_SIZE_RAW_PACKET];
-+		} raw;
-+	};
- };
- 
- VIRTCHNL_CHECK_STRUCT_LEN(2312, virtchnl_proto_hdrs);
- 
- struct virtchnl_rss_cfg {
- 	struct virtchnl_proto_hdrs proto_hdrs;	   /* protocol headers */
-	enum virtchnl_rss_algorithm rss_algorithm; /* RSS algorithm type */
-	u8 reserved[128];			   /* reserve for future */
-+
-+	/* see enum virtchnl_rss_algorithm; rss algorithm type */
-+	s32 rss_algorithm;
-+	u8 reserved[128];                          /* reserve for future */
- };
- 
- VIRTCHNL_CHECK_STRUCT_LEN(2444, virtchnl_rss_cfg);
- 
- /* action configuration for FDIR */
- struct virtchnl_filter_action {
-	enum virtchnl_action type;
-+	/* see enum virtchnl_action type */
-+	s32 type;
- 	union {
- 		/* used for queue and qgroup action */
- 		struct {
-@@ -1258,7 +1446,7 @@ VIRTCHNL_CHECK_STRUCT_LEN(36, virtchnl_filter_action);
- 
- struct virtchnl_filter_action_set {
- 	/* action number must be less then VIRTCHNL_MAX_NUM_ACTIONS */
-	int count;
-+	u32 count;
- 	struct virtchnl_filter_action actions[VIRTCHNL_MAX_NUM_ACTIONS];
- };
- 
-@@ -1275,7 +1463,7 @@ VIRTCHNL_CHECK_STRUCT_LEN(2604, virtchnl_fdir_rule);
- /* Status returned to VF after VF requests FDIR commands
-  * VIRTCHNL_FDIR_SUCCESS
-  * VF FDIR related request is successfully done by PF
- * The request can be OP_ADD/DEL.
-+ * The request can be OP_ADD/DEL/QUERY_FDIR_FILTER.
-  *
-  * VIRTCHNL_FDIR_FAILURE_RULE_NORESOURCE
-  * OP_ADD_FDIR_FILTER request is failed due to no Hardware resource.
-@@ -1296,6 +1484,10 @@ VIRTCHNL_CHECK_STRUCT_LEN(2604, virtchnl_fdir_rule);
-  * VIRTCHNL_FDIR_FAILURE_RULE_TIMEOUT
-  * OP_ADD/DEL_FDIR_FILTER request is failed due to timing out
-  * for programming.
-+ *
-+ * VIRTCHNL_FDIR_FAILURE_QUERY_INVALID
-+ * OP_QUERY_FDIR_FILTER request is failed due to parameters validation,
-+ * for example, VF query counter of a rule who has no counter action.
-  */
- enum virtchnl_fdir_prgm_status {
- 	VIRTCHNL_FDIR_SUCCESS = 0,
-@@ -1305,6 +1497,7 @@ enum virtchnl_fdir_prgm_status {
- 	VIRTCHNL_FDIR_FAILURE_RULE_NONEXIST,
- 	VIRTCHNL_FDIR_FAILURE_RULE_INVALID,
- 	VIRTCHNL_FDIR_FAILURE_RULE_TIMEOUT,
-+	VIRTCHNL_FDIR_FAILURE_QUERY_INVALID,
- };
- 
- /* VIRTCHNL_OP_ADD_FDIR_FILTER
-@@ -1321,7 +1514,9 @@ struct virtchnl_fdir_add {
- 	u16 validate_only; /* INPUT */
- 	u32 flow_id;       /* OUTPUT */
- 	struct virtchnl_fdir_rule rule_cfg; /* INPUT */
-	enum virtchnl_fdir_prgm_status status; /* OUTPUT */
-+
-+	/* see enum virtchnl_fdir_prgm_status; OUTPUT */
-+	s32 status;
- };
- 
- VIRTCHNL_CHECK_STRUCT_LEN(2616, virtchnl_fdir_add);
-@@ -1334,11 +1529,175 @@ struct virtchnl_fdir_del {
- 	u16 vsi_id;  /* INPUT */
- 	u16 pad;
- 	u32 flow_id; /* INPUT */
-	enum virtchnl_fdir_prgm_status status; /* OUTPUT */
-+
-+	/* see enum virtchnl_fdir_prgm_status; OUTPUT */
-+	s32 status;
- };
- 
- VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_fdir_del);
- 
-+#define VIRTCHNL_1588_PTP_CAP_RX_TSTAMP		BIT(1)
-+#define VIRTCHNL_1588_PTP_CAP_READ_PHC		BIT(2)
-+
-+/**
-+ * struct virtchnl_ptp_caps - Defines the PTP caps available to the VF.
-+ * @caps: On send, VF sets what capabilities it requests. On reply, PF
-+ *        indicates what has been enabled for this VF. The PF shall not set
-+ *        bits which were not requested by the VF.
-+ * @rsvd: Reserved bits for future extension.
-+ *
-+ * Structure that defines the PTP capabilities available to the VF. The VF
-+ * sends VIRTCHNL_OP_1588_PTP_GET_CAPS, and must fill in the ptp_caps field
-+ * indicating what capabilities it is requesting. The PF will respond with the
-+ * same message with the virtchnl_ptp_caps structure indicating what is
-+ * enabled for the VF.
-+ *
-+ * VIRTCHNL_1588_PTP_CAP_RX_TSTAMP indicates that the VF receive queues have
-+ * receive timestamps enabled in the flexible descriptors. Note that this
-+ * requires a VF to also negotiate to enable advanced flexible descriptors in
-+ * the receive path instead of the default legacy descriptor format.
-+ *
-+ * VIRTCHNL_1588_PTP_CAP_READ_PHC indicates that the VF may read the PHC time
-+ * via the VIRTCHNL_OP_1588_PTP_GET_TIME command.
-+ *
-+ * Note that in the future, additional capability flags may be added which
-+ * indicate additional extended support. All fields marked as reserved by this
-+ * header will be set to zero. VF implementations should verify this to ensure
-+ * that future extensions do not break compatibility.
-+ */
-+struct virtchnl_ptp_caps {
-+	u32 caps;
-+	u8 rsvd[44];
-+};
-+
-+VIRTCHNL_CHECK_STRUCT_LEN(48, virtchnl_ptp_caps);
-+
-+/**
-+ * struct virtchnl_phc_time - Contains the 64bits of PHC clock time in ns.
-+ * @time: PHC time in nanoseconds
-+ * @rsvd: Reserved for future extension
-+ *
-+ * Structure received with VIRTCHNL_OP_1588_PTP_GET_TIME. Contains the 64bits
-+ * of PHC clock time in nanoseconds.
-+ *
-+ * VIRTCHNL_OP_1588_PTP_GET_TIME may be sent to request the current time of
-+ * the PHC. This op is available in case direct access via the PHC registers
-+ * is not available.
-+ */
-+struct virtchnl_phc_time {
-+	u64 time;
-+	u8 rsvd[8];
-+};
-+
-+VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_phc_time);
-+
-+struct virtchnl_shaper_bw {
-+	/* Unit is Kbps */
-+	u32 committed;
-+	u32 peak;
-+};
-+
-+VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_shaper_bw);
-+
-+/* VIRTCHNL_OP_GET_QOS_CAPS
-+ * VF sends this message to get its QoS Caps, such as
-+ * TC number, Arbiter and Bandwidth.
-+ */
-+struct virtchnl_qos_cap_elem {
-+	u8 tc_num;
-+	u8 tc_prio;
-+#define VIRTCHNL_ABITER_STRICT      0
-+#define VIRTCHNL_ABITER_ETS         2
-+	u8 arbiter;
-+#define VIRTCHNL_STRICT_WEIGHT      1
-+	u8 weight;
-+	enum virtchnl_bw_limit_type type;
-+	union {
-+		struct virtchnl_shaper_bw shaper;
-+		u8 pad2[32];
-+	};
-+};
-+
-+VIRTCHNL_CHECK_STRUCT_LEN(40, virtchnl_qos_cap_elem);
-+
-+struct virtchnl_qos_cap_list {
-+	u16 vsi_id;
-+	u16 num_elem;
-+	struct virtchnl_qos_cap_elem cap[];
-+};
-+
-+VIRTCHNL_CHECK_STRUCT_LEN(4, virtchnl_qos_cap_list);
-+#define virtchnl_qos_cap_list_LEGACY_SIZEOF	44
-+
-+/* VIRTCHNL_OP_CONFIG_QUEUE_BW */
-+struct virtchnl_queue_bw {
-+	u16 queue_id;
-+	u8 tc;
-+	u8 pad;
-+	struct virtchnl_shaper_bw shaper;
-+};
-+
-+VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_queue_bw);
-+
-+struct virtchnl_queues_bw_cfg {
-+	u16 vsi_id;
-+	u16 num_queues;
-+	struct virtchnl_queue_bw cfg[];
-+};
-+
-+VIRTCHNL_CHECK_STRUCT_LEN(4, virtchnl_queues_bw_cfg);
-+#define virtchnl_queues_bw_cfg_LEGACY_SIZEOF	16
-+
-+enum virtchnl_queue_type {
-+	VIRTCHNL_QUEUE_TYPE_TX			= 0,
-+	VIRTCHNL_QUEUE_TYPE_RX			= 1,
-+};
-+
-+/* structure to specify a chunk of contiguous queues */
-+struct virtchnl_queue_chunk {
-+	/* see enum virtchnl_queue_type */
-+	s32 type;
-+	u16 start_queue_id;
-+	u16 num_queues;
-+};
-+
-+VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_queue_chunk);
-+
-+struct virtchnl_quanta_cfg {
-+	u16 quanta_size;
-+	u16 pad;
-+	struct virtchnl_queue_chunk queue_select;
-+};
-+
-+VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_quanta_cfg);
-+
-+#define __vss_byone(p, member, count, old)				      \
-+	(struct_size(p, member, count) + (old - 1 - struct_size(p, member, 0)))
-+
-+#define __vss_byelem(p, member, count, old)				      \
-+	(struct_size(p, member, count - 1) + (old - struct_size(p, member, 0)))
-+
-+#define __vss_full(p, member, count, old)				      \
-+	(struct_size(p, member, count) + (old - struct_size(p, member, 0)))
-+
-+#define __vss(type, func, p, member, count)		\
-+	struct type: func(p, member, count, type##_LEGACY_SIZEOF)
-+
-+#define virtchnl_struct_size(p, m, c)					      \
-+	_Generic(*p,							      \
-+		 __vss(virtchnl_vf_resource, __vss_full, p, m, c),	      \
-+		 __vss(virtchnl_vsi_queue_config_info, __vss_full, p, m, c),  \
-+		 __vss(virtchnl_irq_map_info, __vss_full, p, m, c),	      \
-+		 __vss(virtchnl_ether_addr_list, __vss_full, p, m, c),	      \
-+		 __vss(virtchnl_vlan_filter_list, __vss_full, p, m, c),	      \
-+		 __vss(virtchnl_vlan_filter_list_v2, __vss_byelem, p, m, c),  \
-+		 __vss(virtchnl_tc_info, __vss_byelem, p, m, c),	      \
-+		 __vss(virtchnl_rdma_qvlist_info, __vss_byelem, p, m, c),     \
-+		 __vss(virtchnl_qos_cap_list, __vss_byelem, p, m, c),	      \
-+		 __vss(virtchnl_queues_bw_cfg, __vss_byelem, p, m, c),	      \
-+		 __vss(virtchnl_rss_key, __vss_byone, p, m, c),		      \
-+		 __vss(virtchnl_rss_lut, __vss_byone, p, m, c))
-+
- /**
-  * virtchnl_vc_validate_vf_msg
-  * @ver: Virtchnl version info
-@@ -1353,7 +1712,7 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode,
- 			    u8 *msg, u16 msglen)
- {
- 	bool err_msg_format = false;
-	int valid_len = 0;
-+	u32 valid_len = 0;
- 
- 	/* Validate message length. */
- 	switch (v_opcode) {
-@@ -1373,24 +1732,23 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode,
- 		valid_len = sizeof(struct virtchnl_rxq_info);
- 		break;
- 	case VIRTCHNL_OP_CONFIG_VSI_QUEUES:
-		valid_len = sizeof(struct virtchnl_vsi_queue_config_info);
-+		valid_len = virtchnl_vsi_queue_config_info_LEGACY_SIZEOF;
- 		if (msglen >= valid_len) {
- 			struct virtchnl_vsi_queue_config_info *vqc =
- 			    (struct virtchnl_vsi_queue_config_info *)msg;
-			valid_len += (vqc->num_queue_pairs *
-				      sizeof(struct
-					     virtchnl_queue_pair_info));
-+			valid_len = virtchnl_struct_size(vqc, qpair,
-+							 vqc->num_queue_pairs);
- 			if (vqc->num_queue_pairs == 0)
- 				err_msg_format = true;
- 		}
- 		break;
- 	case VIRTCHNL_OP_CONFIG_IRQ_MAP:
-		valid_len = sizeof(struct virtchnl_irq_map_info);
-+		valid_len = virtchnl_irq_map_info_LEGACY_SIZEOF;
- 		if (msglen >= valid_len) {
- 			struct virtchnl_irq_map_info *vimi =
- 			    (struct virtchnl_irq_map_info *)msg;
-			valid_len += (vimi->num_vectors *
-				      sizeof(struct virtchnl_vector_map));
-+			valid_len = virtchnl_struct_size(vimi, vecmap,
-+							 vimi->num_vectors);
- 			if (vimi->num_vectors == 0)
- 				err_msg_format = true;
- 		}
-@@ -1401,23 +1759,24 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode,
- 		break;
- 	case VIRTCHNL_OP_ADD_ETH_ADDR:
- 	case VIRTCHNL_OP_DEL_ETH_ADDR:
-		valid_len = sizeof(struct virtchnl_ether_addr_list);
-+		valid_len = virtchnl_ether_addr_list_LEGACY_SIZEOF;
- 		if (msglen >= valid_len) {
- 			struct virtchnl_ether_addr_list *veal =
- 			    (struct virtchnl_ether_addr_list *)msg;
-			valid_len += veal->num_elements *
-			    sizeof(struct virtchnl_ether_addr);
-+			valid_len = virtchnl_struct_size(veal, list,
-+							 veal->num_elements);
- 			if (veal->num_elements == 0)
- 				err_msg_format = true;
- 		}
- 		break;
- 	case VIRTCHNL_OP_ADD_VLAN:
- 	case VIRTCHNL_OP_DEL_VLAN:
-		valid_len = sizeof(struct virtchnl_vlan_filter_list);
-+		valid_len = virtchnl_vlan_filter_list_LEGACY_SIZEOF;
- 		if (msglen >= valid_len) {
- 			struct virtchnl_vlan_filter_list *vfl =
- 			    (struct virtchnl_vlan_filter_list *)msg;
-			valid_len += vfl->num_elements * sizeof(u16);
-+			valid_len = virtchnl_struct_size(vfl, vlan_id,
-+							 vfl->num_elements);
- 			if (vfl->num_elements == 0)
- 				err_msg_format = true;
- 		}
-@@ -1428,7 +1787,7 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode,
- 	case VIRTCHNL_OP_GET_STATS:
- 		valid_len = sizeof(struct virtchnl_queue_select);
- 		break;
-	case VIRTCHNL_OP_IWARP:
-+	case VIRTCHNL_OP_RDMA:
- 		/* These messages are opaque to us and will be validated in
- 		 * the RDMA client code. We just need to check for nonzero
- 		 * length. The firmware will enforce max length restrictions.
-@@ -1438,41 +1797,43 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode,
- 		else
- 			err_msg_format = true;
- 		break;
-	case VIRTCHNL_OP_RELEASE_IWARP_IRQ_MAP:
-+	case VIRTCHNL_OP_RELEASE_RDMA_IRQ_MAP:
- 		break;
-	case VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP:
-		valid_len = sizeof(struct virtchnl_iwarp_qvlist_info);
-+	case VIRTCHNL_OP_CONFIG_RDMA_IRQ_MAP:
-+		valid_len = virtchnl_rdma_qvlist_info_LEGACY_SIZEOF;
- 		if (msglen >= valid_len) {
-			struct virtchnl_iwarp_qvlist_info *qv =
-				(struct virtchnl_iwarp_qvlist_info *)msg;
-			if (qv->num_vectors == 0) {
-				err_msg_format = true;
-				break;
-			}
-			valid_len += ((qv->num_vectors - 1) *
-				sizeof(struct virtchnl_iwarp_qv_info));
-+			struct virtchnl_rdma_qvlist_info *qv =
-+				(struct virtchnl_rdma_qvlist_info *)msg;
-+
-+			valid_len = virtchnl_struct_size(qv, qv_info,
-+							 qv->num_vectors);
- 		}
- 		break;
- 	case VIRTCHNL_OP_CONFIG_RSS_KEY:
-		valid_len = sizeof(struct virtchnl_rss_key);
-+		valid_len = virtchnl_rss_key_LEGACY_SIZEOF;
- 		if (msglen >= valid_len) {
- 			struct virtchnl_rss_key *vrk =
- 				(struct virtchnl_rss_key *)msg;
-			valid_len += vrk->key_len - 1;
-+			valid_len = virtchnl_struct_size(vrk, key,
-+							 vrk->key_len);
- 		}
- 		break;
- 	case VIRTCHNL_OP_CONFIG_RSS_LUT:
-		valid_len = sizeof(struct virtchnl_rss_lut);
-+		valid_len = virtchnl_rss_lut_LEGACY_SIZEOF;
- 		if (msglen >= valid_len) {
- 			struct virtchnl_rss_lut *vrl =
- 				(struct virtchnl_rss_lut *)msg;
-			valid_len += vrl->lut_entries - 1;
-+			valid_len = virtchnl_struct_size(vrl, lut,
-+							 vrl->lut_entries);
- 		}
- 		break;
-	case VIRTCHNL_OP_GET_RSS_HENA_CAPS:
-+	case VIRTCHNL_OP_CONFIG_RSS_HFUNC:
-+		valid_len = sizeof(struct virtchnl_rss_hfunc);
- 		break;
-	case VIRTCHNL_OP_SET_RSS_HENA:
-		valid_len = sizeof(struct virtchnl_rss_hena);
-+	case VIRTCHNL_OP_GET_RSS_HASHCFG_CAPS:
-+		break;
-+	case VIRTCHNL_OP_SET_RSS_HASHCFG:
-+		valid_len = sizeof(struct virtchnl_rss_hashcfg);
- 		break;
- 	case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING:
- 	case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING:
-@@ -1481,12 +1842,12 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode,
- 		valid_len = sizeof(struct virtchnl_vf_res_request);
- 		break;
- 	case VIRTCHNL_OP_ENABLE_CHANNELS:
-		valid_len = sizeof(struct virtchnl_tc_info);
-+		valid_len = virtchnl_tc_info_LEGACY_SIZEOF;
- 		if (msglen >= valid_len) {
- 			struct virtchnl_tc_info *vti =
- 				(struct virtchnl_tc_info *)msg;
-			valid_len += (vti->num_tc - 1) *
-				     sizeof(struct virtchnl_channel_info);
-+			valid_len = virtchnl_struct_size(vti, list,
-+							 vti->num_tc);
- 			if (vti->num_tc == 0)
- 				err_msg_format = true;
- 		}
-@@ -1494,11 +1855,11 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode,
- 	case VIRTCHNL_OP_DISABLE_CHANNELS:
- 		break;
- 	case VIRTCHNL_OP_ADD_CLOUD_FILTER:
-		valid_len = sizeof(struct virtchnl_filter);
-		break;
- 	case VIRTCHNL_OP_DEL_CLOUD_FILTER:
- 		valid_len = sizeof(struct virtchnl_filter);
- 		break;
-+	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
-+		break;
- 	case VIRTCHNL_OP_ADD_RSS_CFG:
- 	case VIRTCHNL_OP_DEL_RSS_CFG:
- 		valid_len = sizeof(struct virtchnl_rss_cfg);
-@@ -1513,13 +1874,13 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode,
- 		break;
- 	case VIRTCHNL_OP_ADD_VLAN_V2:
- 	case VIRTCHNL_OP_DEL_VLAN_V2:
-		valid_len = sizeof(struct virtchnl_vlan_filter_list_v2);
-+		valid_len = virtchnl_vlan_filter_list_v2_LEGACY_SIZEOF;
- 		if (msglen >= valid_len) {
- 			struct virtchnl_vlan_filter_list_v2 *vfl =
- 			    (struct virtchnl_vlan_filter_list_v2 *)msg;
- 
-			valid_len += (vfl->num_elements - 1) *
-				sizeof(struct virtchnl_vlan_filter);
-+			valid_len = virtchnl_struct_size(vfl, filters,
-+							 vfl->num_elements);
- 
- 			if (vfl->num_elements == 0) {
- 				err_msg_format = true;
-@@ -1533,6 +1894,41 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode,
- 	case VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2:
- 		valid_len = sizeof(struct virtchnl_vlan_setting);
- 		break;
-+	case VIRTCHNL_OP_GET_QOS_CAPS:
-+		break;
-+	case VIRTCHNL_OP_CONFIG_QUEUE_BW:
-+		valid_len = virtchnl_queues_bw_cfg_LEGACY_SIZEOF;
-+		if (msglen >= valid_len) {
-+			struct virtchnl_queues_bw_cfg *q_bw =
-+				(struct virtchnl_queues_bw_cfg *)msg;
-+
-+			valid_len = virtchnl_struct_size(q_bw, cfg,
-+							 q_bw->num_queues);
-+			if (q_bw->num_queues == 0) {
-+				err_msg_format = true;
-+				break;
-+			}
-+		}
-+		break;
-+	case VIRTCHNL_OP_CONFIG_QUANTA:
-+		valid_len = sizeof(struct virtchnl_quanta_cfg);
-+		if (msglen >= valid_len) {
-+			struct virtchnl_quanta_cfg *q_quanta =
-+				(struct virtchnl_quanta_cfg *)msg;
-+
-+			if (q_quanta->quanta_size == 0 ||
-+			    q_quanta->queue_select.num_queues == 0) {
-+				err_msg_format = true;
-+				break;
-+			}
-+		}
-+		break;
-+	case VIRTCHNL_OP_1588_PTP_GET_CAPS:
-+		valid_len = sizeof(struct virtchnl_ptp_caps);
-+		break;
-+	case VIRTCHNL_OP_1588_PTP_GET_TIME:
-+		valid_len = sizeof(struct virtchnl_phc_time);
-+		break;
- 	/* These are always errors coming from the VF. */
- 	case VIRTCHNL_OP_EVENT:
- 	case VIRTCHNL_OP_UNKNOWN:
-diff --git a/include/linux/backing-dev-defs.h b/include/linux/backing-dev-defs.h
-index ae12696ec492..0217c1073735 100644
--- a/include/linux/backing-dev-defs.h
-+++ b/include/linux/backing-dev-defs.h
-@@ -63,6 +63,8 @@ enum wb_reason {
- struct wb_completion {
- 	atomic_t		cnt;
- 	wait_queue_head_t	*waitq;
-+	unsigned long progress_stamp;	/* The jiffies when slow progress is detected */
-+	unsigned long wait_start;	/* The jiffies when waiting for the writeback work to finish */
- };
- 
- #define __WB_COMPLETION_INIT(_waitq)	\
-@@ -141,8 +143,6 @@ struct bdi_writeback {
- 	struct delayed_work dwork;	/* work item used for writeback */
- 	struct delayed_work bw_dwork;	/* work item used for bandwidth estimate */
- 
-	unsigned long dirty_sleep;	/* last wait */
-
- 	struct list_head bdi_node;	/* anchored at bdi->wb_list */
- 
- #ifdef CONFIG_CGROUP_WRITEBACK
-@@ -154,6 +154,10 @@ struct bdi_writeback {
- 	struct list_head blkcg_node;	/* anchored at blkcg->cgwb_list */
- 	struct list_head b_attached;	/* attached inodes, protected by list_lock */
- 	struct list_head offline_node;	/* anchored at offline_cgwbs */
-+	struct work_struct switch_work;	/* work used to perform inode switching
-+					 * to this wb */
-+	struct llist_head switch_wbs_ctxs;	/* queued contexts for
-+						 * writeback switching */
- 
- 	union {
- 		struct work_struct release_work;
-@@ -166,7 +170,9 @@ struct backing_dev_info {
- 	u64 id;
- 	struct rb_node rb_node; /* keyed by ->id */
- 	struct list_head bdi_list;
-	unsigned long ra_pages;	/* max readahead in PAGE_SIZE units */
-+	/* max readahead in PAGE_SIZE units */
-+	unsigned long __data_racy ra_pages;
-+
- 	unsigned long io_pages;	/* max allowed IO size */
- 
- 	struct kref refcnt;	/* Reference counter for the structure */
-@@ -179,6 +185,11 @@ struct backing_dev_info {
- 	 * any dirty wbs, which is depended upon by bdi_has_dirty().
- 	 */
- 	atomic_long_t tot_write_bandwidth;
-+	/*
-+	 * Jiffies when last process was dirty throttled on this bdi. Used by
-+	 * blk-wbt.
-+	 */
-+	unsigned long last_bdp_sleep;
- 
- 	struct bdi_writeback wb;  /* the root writeback info for this bdi */
- 	struct list_head wb_list; /* list of all wbs */
-diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h
-index 439815cc1ab9..0c8342747cab 100644
--- a/include/linux/backing-dev.h
-+++ b/include/linux/backing-dev.h
-@@ -38,7 +38,6 @@ struct backing_dev_info *bdi_alloc(int node_id);
- 
- void wb_start_background_writeback(struct bdi_writeback *wb);
- void wb_workfn(struct work_struct *work);
-void wb_wakeup_delayed(struct bdi_writeback *wb);
- 
- void wb_wait_for_completion(struct wb_completion *done);
- 
-@@ -46,7 +45,6 @@ extern spinlock_t bdi_lock;
- extern struct list_head bdi_list;
- 
- extern struct workqueue_struct *bdi_wq;
-extern struct workqueue_struct *bdi_async_bio_wq;
- 
- static inline bool wb_has_dirty_io(struct bdi_writeback *wb)
- {
-@@ -68,16 +66,6 @@ static inline void wb_stat_mod(struct bdi_writeback *wb,
- 	percpu_counter_add_batch(&wb->stat[item], amount, WB_STAT_BATCH);
- }
- 
-static inline void inc_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
-{
-	wb_stat_mod(wb, item, 1);
-}
-
-static inline void dec_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
-{
-	wb_stat_mod(wb, item, -1);
-}
-
- static inline s64 wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
- {
- 	return percpu_counter_read_positive(&wb->stat[item]);
-@@ -102,20 +90,28 @@ static inline unsigned long wb_stat_error(void)
- #endif
- }
- 
-+/* BDI ratio is expressed as part per 1000000 for finer granularity. */
-+#define BDI_RATIO_SCALE 10000
-+
-+u64 bdi_get_min_bytes(struct backing_dev_info *bdi);
-+u64 bdi_get_max_bytes(struct backing_dev_info *bdi);
- int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio);
- int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio);
-+int bdi_set_min_ratio_no_scale(struct backing_dev_info *bdi, unsigned int min_ratio);
-+int bdi_set_max_ratio_no_scale(struct backing_dev_info *bdi, unsigned int max_ratio);
-+int bdi_set_min_bytes(struct backing_dev_info *bdi, u64 min_bytes);
-+int bdi_set_max_bytes(struct backing_dev_info *bdi, u64 max_bytes);
-+int bdi_set_strict_limit(struct backing_dev_info *bdi, unsigned int strict_limit);
- 
- /*
-  * Flags in backing_dev_info::capability
-  *
-  * BDI_CAP_WRITEBACK:		Supports dirty page writeback, and dirty pages
-  *				should contribute to accounting
- * BDI_CAP_WRITEBACK_ACCT:	Automatically account writeback pages
-  * BDI_CAP_STRICTLIMIT:		Keep number of dirty pages below bdi threshold
-  */
- #define BDI_CAP_WRITEBACK		(1 << 0)
-#define BDI_CAP_WRITEBACK_ACCT		(1 << 1)
-#define BDI_CAP_STRICTLIMIT		(1 << 2)
-+#define BDI_CAP_STRICTLIMIT		(1 << 1)
- 
- extern struct backing_dev_info noop_backing_dev_info;
- 
-@@ -241,6 +237,7 @@ static inline struct bdi_writeback *inode_to_wb(const struct inode *inode)
- {
- #ifdef CONFIG_LOCKDEP
- 	WARN_ON_ONCE(debug_locks &&
-+		     (inode->i_sb->s_iflags & SB_I_CGROUPWB) &&
- 		     (!lockdep_is_held(&inode->i_lock) &&
- 		      !lockdep_is_held(&inode->i_mapping->i_pages.xa_lock) &&
- 		      !lockdep_is_held(&inode->i_wb->list_lock)));
-@@ -280,10 +277,11 @@ unlocked_inode_to_wb_begin(struct inode *inode, struct wb_lock_cookie *cookie)
- 	rcu_read_lock();
- 
- 	/*
-	 * Paired with store_release in inode_switch_wbs_work_fn() and
-+	 * Paired with a release fence in inode_do_switch_wbs() and
- 	 * ensures that we see the new wb if we see cleared I_WB_SWITCH.
- 	 */
-	cookie->locked = smp_load_acquire(&inode->i_state) & I_WB_SWITCH;
-+	cookie->locked = inode_state_read_once(inode) & I_WB_SWITCH;
-+	smp_rmb();
- 
- 	if (unlikely(cookie->locked))
- 		xa_lock_irqsave(&inode->i_mapping->i_pages, cookie->flags);
-diff --git a/include/linux/backing-file.h b/include/linux/backing-file.h
-new file mode 100644
-index 000000000000..1476a6ed1bfd
--- /dev/null
-+++ b/include/linux/backing-file.h
-@@ -0,0 +1,44 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Common helpers for stackable filesystems and backing files.
-+ *
-+ * Copyright (C) 2023 CTERA Networks.
-+ */
-+
-+#ifndef _LINUX_BACKING_FILE_H
-+#define _LINUX_BACKING_FILE_H
-+
-+#include <linux/file.h>
-+#include <linux/uio.h>
-+#include <linux/fs.h>
-+
-+struct backing_file_ctx {
-+	const struct cred *cred;
-+	void (*accessed)(struct file *file);
-+	void (*end_write)(struct kiocb *iocb, ssize_t);
-+};
-+
-+struct file *backing_file_open(const struct path *user_path, int flags,
-+			       const struct path *real_path,
-+			       const struct cred *cred);
-+struct file *backing_tmpfile_open(const struct path *user_path, int flags,
-+				  const struct path *real_parentpath,
-+				  umode_t mode, const struct cred *cred);
-+ssize_t backing_file_read_iter(struct file *file, struct iov_iter *iter,
-+			       struct kiocb *iocb, int flags,
-+			       struct backing_file_ctx *ctx);
-+ssize_t backing_file_write_iter(struct file *file, struct iov_iter *iter,
-+				struct kiocb *iocb, int flags,
-+				struct backing_file_ctx *ctx);
-+ssize_t backing_file_splice_read(struct file *in, struct kiocb *iocb,
-+				 struct pipe_inode_info *pipe, size_t len,
-+				 unsigned int flags,
-+				 struct backing_file_ctx *ctx);
-+ssize_t backing_file_splice_write(struct pipe_inode_info *pipe,
-+				  struct file *out, struct kiocb *iocb,
-+				  size_t len, unsigned int flags,
-+				  struct backing_file_ctx *ctx);
-+int backing_file_mmap(struct file *file, struct vm_area_struct *vma,
-+		      struct backing_file_ctx *ctx);
-+
-+#endif /* _LINUX_BACKING_FILE_H */
-diff --git a/include/linux/backlight.h b/include/linux/backlight.h
-index 614653e07e3a..f29a9ef1052e 100644
--- a/include/linux/backlight.h
-+++ b/include/linux/backlight.h
-@@ -10,9 +10,8 @@
- #define _LINUX_BACKLIGHT_H
- 
- #include <linux/device.h>
-#include <linux/fb.h>
- #include <linux/mutex.h>
-#include <linux/notifier.h>
-+#include <linux/types.h>
- 
- /**
-  * enum backlight_update_reason - what method was used to update backlight
-@@ -65,24 +64,6 @@ enum backlight_type {
- 	BACKLIGHT_TYPE_MAX,
- };
- 
-/**
- * enum backlight_notification - the type of notification
- *
- * The notifications that is used for notification sent to the receiver
- * that registered notifications using backlight_register_notifier().
- */
-enum backlight_notification {
-	/**
-	 * @BACKLIGHT_REGISTERED: The backlight device is registered.
-	 */
-	BACKLIGHT_REGISTERED,
-
-	/**
-	 * @BACKLIGHT_UNREGISTERED: The backlight revice is unregistered.
-	 */
-	BACKLIGHT_UNREGISTERED,
-};
-
- /** enum backlight_scale - the type of scale used for brightness values
-  *
-  * The type of scale used for brightness values.
-@@ -110,7 +91,6 @@ enum backlight_scale {
- };
- 
- struct backlight_device;
-struct fb_info;
- 
- /**
-  * struct backlight_ops - backlight operations
-@@ -160,18 +140,18 @@ struct backlight_ops {
- 	int (*get_brightness)(struct backlight_device *);
- 
- 	/**
-	 * @check_fb: Check the framebuffer device.
-+	 * @controls_device: Check against the display device
- 	 *
-	 * Check if given framebuffer device is the one bound to this backlight.
-	 * This operation is optional and if not implemented it is assumed that the
-	 * fbdev is always the one bound to the backlight.
-+	 * Check if the backlight controls the given display device. This
-+	 * operation is optional and if not implemented it is assumed that
-+	 * the display is always the one controlled by the backlight.
- 	 *
- 	 * RETURNS:
- 	 *
-	 * If info is NULL or the info matches the fbdev bound to the backlight return true.
-	 * If info does not match the fbdev bound to the backlight return false.
-+	 * If display_dev is NULL or display_dev matches the device controlled by
-+	 * the backlight, return true. Otherwise return false.
- 	 */
-	int (*check_fb)(struct backlight_device *bd, struct fb_info *info);
-+	bool (*controls_device)(struct backlight_device *bd, struct device *display_dev);
- };
- 
- /**
-@@ -209,33 +189,18 @@ struct backlight_properties {
- 	 * attribute: /sys/class/backlight/<backlight>/bl_power
- 	 * When the power property is updated update_status() is called.
- 	 *
-	 * The possible values are: (0: full on, 1 to 3: power saving
-	 * modes; 4: full off), see FB_BLANK_XXX.
-+	 * The possible values are: (0: full on, 4: full off), see
-+	 * BACKLIGHT_POWER constants.
- 	 *
-	 * When the backlight device is enabled @power is set
-	 * to FB_BLANK_UNBLANK. When the backlight device is disabled
-	 * @power is set to FB_BLANK_POWERDOWN.
-+	 * When the backlight device is enabled, @power is set to
-+	 * BACKLIGHT_POWER_ON. When the backlight device is disabled,
-+	 * @power is set to BACKLIGHT_POWER_OFF.
- 	 */
- 	int power;
- 
-	/**
-	 * @fb_blank: The power state from the FBIOBLANK ioctl.
-	 *
-	 * When the FBIOBLANK ioctl is called @fb_blank is set to the
-	 * blank parameter and the update_status() operation is called.
-	 *
-	 * When the backlight device is enabled @fb_blank is set
-	 * to FB_BLANK_UNBLANK. When the backlight device is disabled
-	 * @fb_blank is set to FB_BLANK_POWERDOWN.
-	 *
-	 * Backlight drivers should avoid using this property. It has been
-	 * replaced by state & BL_CORE_FBLANK (although most drivers should
-	 * use backlight_is_blank() as the preferred means to get the blank
-	 * state).
-	 *
-	 * fb_blank is deprecated and will be removed.
-	 */
-	int fb_blank;
-+#define BACKLIGHT_POWER_ON		(0)
-+#define BACKLIGHT_POWER_OFF		(4)
-+#define BACKLIGHT_POWER_REDUCED		(1) // deprecated; don't use in new code
- 
- 	/**
- 	 * @type: The type of backlight supported.
-@@ -312,11 +277,6 @@ struct backlight_device {
- 	const struct backlight_ops *ops;
- 
- 	/**
-	 * @fb_notif: The framebuffer notifier block
-	 */
-	struct notifier_block fb_notif;
-
-	/**
- 	 * @entry: List entry of all registered backlight devices
- 	 */
- 	struct list_head entry;
-@@ -327,15 +287,7 @@ struct backlight_device {
- 	struct device dev;
- 
- 	/**
-	 * @fb_bl_on: The state of individual fbdev's.
-	 *
-	 * Multiple fbdev's may share one backlight device. The fb_bl_on
-	 * records the state of the individual fbdev.
-	 */
-	bool fb_bl_on[FB_MAX];
-
-	/**
-	 * @use_count: The number of uses of fb_bl_on.
-+	 * @use_count: The number of unblanked displays.
- 	 */
- 	int use_count;
- };
-@@ -365,8 +317,7 @@ static inline int backlight_enable(struct backlight_device *bd)
- 	if (!bd)
- 		return 0;
- 
-	bd->props.power = FB_BLANK_UNBLANK;
-	bd->props.fb_blank = FB_BLANK_UNBLANK;
-+	bd->props.power = BACKLIGHT_POWER_ON;
- 	bd->props.state &= ~BL_CORE_FBBLANK;
- 
- 	return backlight_update_status(bd);
-@@ -381,8 +332,7 @@ static inline int backlight_disable(struct backlight_device *bd)
- 	if (!bd)
- 		return 0;
- 
-	bd->props.power = FB_BLANK_POWERDOWN;
-	bd->props.fb_blank = FB_BLANK_POWERDOWN;
-+	bd->props.power = BACKLIGHT_POWER_OFF;
- 	bd->props.state |= BL_CORE_FBBLANK;
- 
- 	return backlight_update_status(bd);
-@@ -395,15 +345,13 @@ static inline int backlight_disable(struct backlight_device *bd)
-  * Display is expected to be blank if any of these is true::
-  *
-  *   1) if power in not UNBLANK
- *   2) if fb_blank is not UNBLANK
- *   3) if state indicate BLANK or SUSPENDED
-+ *   2) if state indicate BLANK or SUSPENDED
-  *
-  * Returns true if display is expected to be blank, false otherwise.
-  */
- static inline bool backlight_is_blank(const struct backlight_device *bd)
- {
-	return bd->props.power != FB_BLANK_UNBLANK ||
-	       bd->props.fb_blank != FB_BLANK_UNBLANK ||
-+	return bd->props.power != BACKLIGHT_POWER_ON ||
- 	       bd->props.state & (BL_CORE_SUSPENDED | BL_CORE_FBBLANK);
- }
- 
-@@ -440,13 +388,27 @@ void devm_backlight_device_unregister(struct device *dev,
- 				      struct backlight_device *bd);
- void backlight_force_update(struct backlight_device *bd,
- 			    enum backlight_update_reason reason);
-int backlight_register_notifier(struct notifier_block *nb);
-int backlight_unregister_notifier(struct notifier_block *nb);
- struct backlight_device *backlight_device_get_by_name(const char *name);
- struct backlight_device *backlight_device_get_by_type(enum backlight_type type);
- int backlight_device_set_brightness(struct backlight_device *bd,
- 				    unsigned long brightness);
- 
-+#if IS_REACHABLE(CONFIG_BACKLIGHT_CLASS_DEVICE)
-+void backlight_notify_blank(struct backlight_device *bd,
-+			    struct device *display_dev,
-+			    bool fb_on, bool prev_fb_on);
-+void backlight_notify_blank_all(struct device *display_dev,
-+				bool fb_on, bool prev_fb_on);
-+#else
-+static inline void backlight_notify_blank(struct backlight_device *bd,
-+					  struct device *display_dev,
-+					  bool fb_on, bool prev_fb_on)
-+{ }
-+static inline void backlight_notify_blank_all(struct device *display_dev,
-+					      bool fb_on, bool prev_fb_on)
-+{ }
-+#endif
-+
- #define to_backlight_device(obj) container_of(obj, struct backlight_device, dev)
- 
- /**
-diff --git a/include/linux/badblocks.h b/include/linux/badblocks.h
-index 2426276b9bd3..996493917f36 100644
--- a/include/linux/badblocks.h
-+++ b/include/linux/badblocks.h
-@@ -15,6 +15,7 @@
- #define BB_OFFSET(x)	(((x) & BB_OFFSET_MASK) >> 9)
- #define BB_LEN(x)	(((x) & BB_LEN_MASK) + 1)
- #define BB_ACK(x)	(!!((x) & BB_ACK_MASK))
-+#define BB_END(x)	(BB_OFFSET(x) + BB_LEN(x))
- #define BB_MAKE(a, l, ack) (((a)<<9) | ((l)-1) | ((u64)(!!(ack)) << 63))
- 
- /* Bad block numbers are stored sorted in a single page.
-@@ -41,11 +42,17 @@ struct badblocks {
- 	sector_t size;		/* in sectors */
- };
- 
-int badblocks_check(struct badblocks *bb, sector_t s, int sectors,
-		   sector_t *first_bad, int *bad_sectors);
-int badblocks_set(struct badblocks *bb, sector_t s, int sectors,
-			int acknowledged);
-int badblocks_clear(struct badblocks *bb, sector_t s, int sectors);
-+struct badblocks_context {
-+	sector_t	start;
-+	sector_t	len;
-+	int		ack;
-+};
-+
-+int badblocks_check(struct badblocks *bb, sector_t s, sector_t sectors,
-+		    sector_t *first_bad, sector_t *bad_sectors);
-+bool badblocks_set(struct badblocks *bb, sector_t s, sector_t sectors,
-+		   int acknowledged);
-+bool badblocks_clear(struct badblocks *bb, sector_t s, sector_t sectors);
- void ack_all_badblocks(struct badblocks *bb);
- ssize_t badblocks_show(struct badblocks *bb, char *page, int unack);
- ssize_t badblocks_store(struct badblocks *bb, const char *page, size_t len,
-@@ -63,4 +70,27 @@ static inline void devm_exit_badblocks(struct device *dev, struct badblocks *bb)
- 	}
- 	badblocks_exit(bb);
- }
-+
-+static inline int badblocks_full(struct badblocks *bb)
-+{
-+	return (bb->count >= MAX_BADBLOCKS);
-+}
-+
-+static inline int badblocks_empty(struct badblocks *bb)
-+{
-+	return (bb->count == 0);
-+}
-+
-+static inline void set_changed(struct badblocks *bb)
-+{
-+	if (bb->changed != 1)
-+		bb->changed = 1;
-+}
-+
-+static inline void clear_changed(struct badblocks *bb)
-+{
-+	if (bb->changed != 0)
-+		bb->changed = 0;
-+}
-+
- #endif
-diff --git a/include/linux/balloon_compaction.h b/include/linux/balloon_compaction.h
-index 5ca2d5699620..7cfe48769239 100644
--- a/include/linux/balloon_compaction.h
-+++ b/include/linux/balloon_compaction.h
-@@ -4,12 +4,13 @@
-  *
-  * Common interface definitions for making balloon pages movable by compaction.
-  *
- * Balloon page migration makes use of the general non-lru movable page
-+ * Balloon page migration makes use of the general "movable_ops page migration"
-  * feature.
-  *
-  * page->private is used to reference the responsible balloon device.
- * page->mapping is used in context of non-lru page migration to reference
- * the address space operations for page isolation/migration/compaction.
-+ * That these pages have movable_ops, and which movable_ops apply,
-+ * is derived from the page type (PageOffline()) combined with the
-+ * PG_movable_ops flag (PageMovableOps()).
-  *
-  * As the page isolation scanning step a compaction thread does is a lockless
-  * procedure (from a page standpoint), it might bring some racy situations while
-@@ -17,12 +18,10 @@
-  * and safely perform balloon's page compaction and migration we must, always,
-  * ensure following these simple rules:
-  *
- *   i. when updating a balloon's page ->mapping element, strictly do it under
- *      the following lock order, independently of the far superior
- *      locking scheme (lru_lock, balloon_lock):
-+ *   i. Setting the PG_movable_ops flag and page->private with the following
-+ *	lock order
-  *	    +-page_lock(page);
-  *	      +--spin_lock_irq(&b_dev_info->pages_lock);
- *	            ... page->mapping updates here ...
-  *
-  *  ii. isolation or dequeueing procedure must remove the page from balloon
-  *      device page list under b_dev_info->pages_lock.
-@@ -78,6 +77,15 @@ static inline void balloon_devinfo_init(struct balloon_dev_info *balloon)
- 
- #ifdef CONFIG_BALLOON_COMPACTION
- extern const struct movable_operations balloon_mops;
-+/*
-+ * balloon_page_device - get the b_dev_info descriptor for the balloon device
-+ *			 that enqueues the given page.
-+ */
-+static inline struct balloon_dev_info *balloon_page_device(struct page *page)
-+{
-+	return (struct balloon_dev_info *)page_private(page);
-+}
-+#endif /* CONFIG_BALLOON_COMPACTION */
- 
- /*
-  * balloon_page_insert - insert a page into the balloon's page list and make
-@@ -92,68 +100,34 @@ static inline void balloon_page_insert(struct balloon_dev_info *balloon,
- 				       struct page *page)
- {
- 	__SetPageOffline(page);
-	__SetPageMovable(page, &balloon_mops);
-	set_page_private(page, (unsigned long)balloon);
-+	if (IS_ENABLED(CONFIG_BALLOON_COMPACTION)) {
-+		SetPageMovableOps(page);
-+		set_page_private(page, (unsigned long)balloon);
-+	}
- 	list_add(&page->lru, &balloon->pages);
- }
- 
-/*
- * balloon_page_delete - delete a page from balloon's page list and clear
- *			 the page->private assignement accordingly.
- * @page    : page to be released from balloon's page list
- *
- * Caller must ensure the page is locked and the spin_lock protecting balloon
- * pages list is held before deleting a page from the balloon device.
- */
-static inline void balloon_page_delete(struct page *page)
-+static inline gfp_t balloon_mapping_gfp_mask(void)
- {
-	__ClearPageOffline(page);
-	__ClearPageMovable(page);
-	set_page_private(page, 0);
-	/*
-	 * No touch page.lru field once @page has been isolated
-	 * because VM is using the field.
-	 */
-	if (!PageIsolated(page))
-		list_del(&page->lru);
-+	if (IS_ENABLED(CONFIG_BALLOON_COMPACTION))
-+		return GFP_HIGHUSER_MOVABLE;
-+	return GFP_HIGHUSER;
- }
- 
- /*
- * balloon_page_device - get the b_dev_info descriptor for the balloon device
- *			 that enqueues the given page.
-+ * balloon_page_finalize - prepare a balloon page that was removed from the
-+ *			   balloon list for release to the page allocator
-+ * @page: page to be released to the page allocator
-+ *
-+ * Caller must ensure that the page is locked.
-  */
-static inline struct balloon_dev_info *balloon_page_device(struct page *page)
-+static inline void balloon_page_finalize(struct page *page)
- {
-	return (struct balloon_dev_info *)page_private(page);
-+	if (IS_ENABLED(CONFIG_BALLOON_COMPACTION))
-+		set_page_private(page, 0);
-+	/* PageOffline is sticky until the page is freed to the buddy. */
- }
- 
-static inline gfp_t balloon_mapping_gfp_mask(void)
-{
-	return GFP_HIGHUSER_MOVABLE;
-}
-
-#else /* !CONFIG_BALLOON_COMPACTION */
-
-static inline void balloon_page_insert(struct balloon_dev_info *balloon,
-				       struct page *page)
-{
-	__SetPageOffline(page);
-	list_add(&page->lru, &balloon->pages);
-}
-
-static inline void balloon_page_delete(struct page *page)
-{
-	__ClearPageOffline(page);
-	list_del(&page->lru);
-}
-
-static inline gfp_t balloon_mapping_gfp_mask(void)
-{
-	return GFP_HIGHUSER;
-}
-
-#endif /* CONFIG_BALLOON_COMPACTION */
-
- /*
-  * balloon_page_push - insert a page into a page list.
-  * @head : pointer to list
-diff --git a/include/linux/base64.h b/include/linux/base64.h
-index 660d4cb1ef31..a2c6c9222da3 100644
--- a/include/linux/base64.h
-+++ b/include/linux/base64.h
-@@ -8,9 +8,15 @@
- 
- #include <linux/types.h>
- 
-+enum base64_variant {
-+	BASE64_STD,       /* RFC 4648 (standard) */
-+	BASE64_URLSAFE,   /* RFC 4648 (base64url) */
-+	BASE64_IMAP,      /* RFC 3501 */
-+};
-+
- #define BASE64_CHARS(nbytes)   DIV_ROUND_UP((nbytes) * 4, 3)
- 
-int base64_encode(const u8 *src, int len, char *dst);
-int base64_decode(const char *src, int len, u8 *dst);
-+int base64_encode(const u8 *src, int len, char *dst, bool padding, enum base64_variant variant);
-+int base64_decode(const char *src, int len, u8 *dst, bool padding, enum base64_variant variant);
- 
- #endif /* _LINUX_BASE64_H */
-diff --git a/include/linux/bcd.h b/include/linux/bcd.h
-index 118bea36d7d4..abbc8149178e 100644
--- a/include/linux/bcd.h
-+++ b/include/linux/bcd.h
-@@ -14,8 +14,12 @@
- 		const_bin2bcd(x) :			\
- 		_bin2bcd(x))
- 
-+#define bcd_is_valid(x)					\
-+		const_bcd_is_valid(x)
-+
- #define const_bcd2bin(x)	(((x) & 0x0f) + ((x) >> 4) * 10)
- #define const_bin2bcd(x)	((((x) / 10) << 4) + (x) % 10)
-+#define const_bcd_is_valid(x)	(((x) & 0x0f) < 10 && ((x) >> 4) < 10)
- 
- unsigned _bcd2bin(unsigned char val) __attribute_const__;
- unsigned char _bin2bcd(unsigned val) __attribute_const__;
-diff --git a/include/linux/bcm47xx_nvram.h b/include/linux/bcm47xx_nvram.h
-index 53b31f69b74a..e4b6ce953ddb 100644
--- a/include/linux/bcm47xx_nvram.h
-+++ b/include/linux/bcm47xx_nvram.h
-@@ -7,10 +7,10 @@
- 
- #include <linux/errno.h>
- #include <linux/types.h>
-#include <linux/kernel.h>
- #include <linux/vmalloc.h>
- 
- #ifdef CONFIG_BCM47XX_NVRAM
-+int bcm47xx_nvram_init_from_iomem(void __iomem *nvram_start, size_t res_size);
- int bcm47xx_nvram_init_from_mem(u32 base, u32 lim);
- int bcm47xx_nvram_getenv(const char *name, char *val, size_t val_len);
- int bcm47xx_nvram_gpio_pin(const char *name);
-@@ -20,6 +20,11 @@ static inline void bcm47xx_nvram_release_contents(char *nvram)
- 	vfree(nvram);
- };
- #else
-+static inline int bcm47xx_nvram_init_from_iomem(void __iomem *nvram_start,
-+						size_t res_size)
-+{
-+	return -ENOTSUPP;
-+}
- static inline int bcm47xx_nvram_init_from_mem(u32 base, u32 lim)
- {
- 	return -ENOTSUPP;
-diff --git a/include/linux/bcm47xx_sprom.h b/include/linux/bcm47xx_sprom.h
-index f8254fd53e15..40a7da3ef50e 100644
--- a/include/linux/bcm47xx_sprom.h
-+++ b/include/linux/bcm47xx_sprom.h
-@@ -5,8 +5,8 @@
- #ifndef __BCM47XX_SPROM_H
- #define __BCM47XX_SPROM_H
- 
-+#include <linux/errno.h>
- #include <linux/types.h>
-#include <linux/kernel.h>
- #include <linux/vmalloc.h>
- 
- struct ssb_sprom;
-diff --git a/include/linux/bcm963xx_nvram.h b/include/linux/bcm963xx_nvram.h
-index c8c7f01159fe..48830bf18042 100644
--- a/include/linux/bcm963xx_nvram.h
-+++ b/include/linux/bcm963xx_nvram.h
-@@ -81,25 +81,21 @@ static int __maybe_unused bcm963xx_nvram_checksum(
- 	const struct bcm963xx_nvram *nvram,
- 	u32 *expected_out, u32 *actual_out)
- {
-+	const u32 zero = 0;
- 	u32 expected, actual;
- 	size_t len;
- 
- 	if (nvram->version <= 4) {
- 		expected = nvram->checksum_v4;
-		len = BCM963XX_NVRAM_V4_SIZE - sizeof(u32);
-+		len = BCM963XX_NVRAM_V4_SIZE;
- 	} else {
- 		expected = nvram->checksum_v5;
-		len = BCM963XX_NVRAM_V5_SIZE - sizeof(u32);
-+		len = BCM963XX_NVRAM_V5_SIZE;
- 	}
- 
-	/*
-	 * Calculate the CRC32 value for the nvram with a checksum value
-	 * of 0 without modifying or copying the nvram by combining:
-	 * - The CRC32 of the nvram without the checksum value
-	 * - The CRC32 of a zero checksum value (which is also 0)
-	 */
-	actual = crc32_le_combine(
-		crc32_le(~0, (u8 *)nvram, len), 0, sizeof(u32));
-+	/* Calculate the CRC32 of the nvram with the checksum field set to 0. */
-+	actual = crc32_le(~0, nvram, len - sizeof(u32));
-+	actual = crc32_le(actual, &zero, sizeof(u32));
- 
- 	if (expected_out)
- 		*expected_out = expected;
-diff --git a/include/linux/bcma/bcma_driver_chipcommon.h b/include/linux/bcma/bcma_driver_chipcommon.h
-index e3314f746bfa..0cb6638b55e5 100644
--- a/include/linux/bcma/bcma_driver_chipcommon.h
-+++ b/include/linux/bcma/bcma_driver_chipcommon.h
-@@ -4,7 +4,7 @@
- 
- #include <linux/platform_device.h>
- #include <linux/platform_data/brcmnand.h>
-#include <linux/gpio.h>
-+#include <linux/gpio/driver.h>
- 
- /** ChipCommon core registers. **/
- #define BCMA_CC_ID			0x0000
-@@ -271,6 +271,7 @@
- #define  BCMA_CC_SROM_CONTROL_OP_WRDIS	0x40000000
- #define  BCMA_CC_SROM_CONTROL_OP_WREN	0x60000000
- #define  BCMA_CC_SROM_CONTROL_OTPSEL	0x00000010
-+#define  BCMA_CC_SROM_CONTROL_OTP_PRESENT	0x00000020
- #define  BCMA_CC_SROM_CONTROL_LOCK	0x00000008
- #define  BCMA_CC_SROM_CONTROL_SIZE_MASK	0x00000006
- #define  BCMA_CC_SROM_CONTROL_SIZE_1K	0x00000000
-diff --git a/include/linux/bcma/bcma_driver_pci.h b/include/linux/bcma/bcma_driver_pci.h
-index 68da8dba5162..dba41b65ae0d 100644
--- a/include/linux/bcma/bcma_driver_pci.h
-+++ b/include/linux/bcma/bcma_driver_pci.h
-@@ -203,7 +203,7 @@ struct pci_dev;
- #define BCMA_CORE_PCI_MDIO_RXCTRL0		0x840
- 
- /* PCIE Root Capability Register bits (Host mode only) */
-#define BCMA_CORE_PCI_RC_CRS_VISIBILITY		0x0001
-+#define BCMA_CORE_PCI_RC_RRS_VISIBILITY		0x0001
- 
- struct bcma_drv_pci;
- struct bcma_bus;
-diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h
-index 3dc20c4f394c..65abd5ab8836 100644
--- a/include/linux/binfmts.h
-+++ b/include/linux/binfmts.h
-@@ -19,13 +19,13 @@ struct linux_binprm {
- #ifdef CONFIG_MMU
- 	struct vm_area_struct *vma;
- 	unsigned long vma_pages;
-+	unsigned long argmin; /* rlimit marker for copy_strings() */
- #else
- # define MAX_ARG_PAGES	32
- 	struct page *page[MAX_ARG_PAGES];
- #endif
- 	struct mm_struct *mm;
- 	unsigned long p; /* current top of mem */
-	unsigned long argmin; /* rlimit marker for copy_strings() */
- 	unsigned int
- 		/* Should an execfd be passed to userspace? */
- 		have_execfd:1,
-@@ -42,10 +42,14 @@ struct linux_binprm {
- 		 * Set when errors can no longer be returned to the
- 		 * original userspace.
- 		 */
-		point_of_no_return:1;
-#ifdef __alpha__
-	unsigned int taso:1;
-#endif
-+		point_of_no_return:1,
-+		/* Set when "comm" must come from the dentry. */
-+		comm_from_dentry:1,
-+		/*
-+		 * Set by user space to check executability according to the
-+		 * caller's environment.
-+		 */
-+		is_check:1;
- 	struct file *executable; /* Executable to pass to the interpreter */
- 	struct file *interpreter;
- 	struct file *file;
-@@ -60,7 +64,7 @@ struct linux_binprm {
- 	const char *fdpath;	/* generated filename for execveat */
- 	unsigned interp_flags;
- 	int execfd;		/* File descriptor of the executable */
-	unsigned long loader, exec;
-+	unsigned long exec;
- 
- 	struct rlimit rlim_stack; /* Saved RLIMIT_STACK used during exec. */
- 
-@@ -86,13 +90,22 @@ struct linux_binfmt {
- 	struct list_head lh;
- 	struct module *module;
- 	int (*load_binary)(struct linux_binprm *);
-	int (*load_shlib)(struct file *);
- #ifdef CONFIG_COREDUMP
- 	int (*core_dump)(struct coredump_params *cprm);
- 	unsigned long min_coredump;	/* minimal dump size */
- #endif
- } __randomize_layout;
- 
-+#if IS_ENABLED(CONFIG_BINFMT_MISC)
-+struct binfmt_misc {
-+	struct list_head entries;
-+	rwlock_t entries_lock;
-+	bool enabled;
-+} __randomize_layout;
-+
-+extern struct binfmt_misc init_binfmt_misc;
-+#endif
-+
- extern void __register_binfmt(struct linux_binfmt *fmt, int insert);
- 
- /* Registration of default binfmt handlers */
-diff --git a/include/linux/bio-integrity.h b/include/linux/bio-integrity.h
-new file mode 100644
-index 000000000000..21e4652dcfd2
--- /dev/null
-+++ b/include/linux/bio-integrity.h
-@@ -0,0 +1,148 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_BIO_INTEGRITY_H
-+#define _LINUX_BIO_INTEGRITY_H
-+
-+#include <linux/bio.h>
-+
-+enum bip_flags {
-+	BIP_BLOCK_INTEGRITY	= 1 << 0, /* block layer owns integrity data */
-+	BIP_MAPPED_INTEGRITY	= 1 << 1, /* ref tag has been remapped */
-+	BIP_DISK_NOCHECK	= 1 << 2, /* disable disk integrity checking */
-+	BIP_IP_CHECKSUM		= 1 << 3, /* IP checksum */
-+	BIP_COPY_USER		= 1 << 4, /* Kernel bounce buffer in use */
-+	BIP_CHECK_GUARD		= 1 << 5, /* guard check */
-+	BIP_CHECK_REFTAG	= 1 << 6, /* reftag check */
-+	BIP_CHECK_APPTAG	= 1 << 7, /* apptag check */
-+
-+	BIP_MEMPOOL		= 1 << 15, /* buffer backed by mempool */
-+};
-+
-+struct bio_integrity_payload {
-+	struct bvec_iter	bip_iter;
-+
-+	unsigned short		bip_vcnt;	/* # of integrity bio_vecs */
-+	unsigned short		bip_max_vcnt;	/* integrity bio_vec slots */
-+	unsigned short		bip_flags;	/* control flags */
-+	u16			app_tag;	/* application tag value */
-+
-+	struct bio_vec		*bip_vec;
-+};
-+
-+#define BIP_CLONE_FLAGS (BIP_MAPPED_INTEGRITY | BIP_IP_CHECKSUM | \
-+			 BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG)
-+
-+#ifdef CONFIG_BLK_DEV_INTEGRITY
-+
-+#define bip_for_each_vec(bvl, bip, iter)				\
-+	for_each_bvec(bvl, (bip)->bip_vec, iter, (bip)->bip_iter)
-+
-+#define bio_for_each_integrity_vec(_bvl, _bio, _iter)			\
-+	for_each_bio(_bio)						\
-+		bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
-+
-+static inline struct bio_integrity_payload *bio_integrity(struct bio *bio)
-+{
-+	if (bio->bi_opf & REQ_INTEGRITY)
-+		return bio->bi_integrity;
-+
-+	return NULL;
-+}
-+
-+static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
-+{
-+	struct bio_integrity_payload *bip = bio_integrity(bio);
-+
-+	if (bip)
-+		return bip->bip_flags & flag;
-+
-+	return false;
-+}
-+
-+static inline sector_t bip_get_seed(struct bio_integrity_payload *bip)
-+{
-+	return bip->bip_iter.bi_sector;
-+}
-+
-+static inline void bip_set_seed(struct bio_integrity_payload *bip,
-+				sector_t seed)
-+{
-+	bip->bip_iter.bi_sector = seed;
-+}
-+
-+void bio_integrity_init(struct bio *bio, struct bio_integrity_payload *bip,
-+		struct bio_vec *bvecs, unsigned int nr_vecs);
-+struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, gfp_t gfp,
-+		unsigned int nr);
-+int bio_integrity_add_page(struct bio *bio, struct page *page, unsigned int len,
-+		unsigned int offset);
-+int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter);
-+int bio_integrity_map_iter(struct bio *bio, struct uio_meta *meta);
-+void bio_integrity_unmap_user(struct bio *bio);
-+bool bio_integrity_prep(struct bio *bio);
-+void bio_integrity_advance(struct bio *bio, unsigned int bytes_done);
-+void bio_integrity_trim(struct bio *bio);
-+int bio_integrity_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp_mask);
-+
-+#else /* CONFIG_BLK_DEV_INTEGRITY */
-+
-+static inline struct bio_integrity_payload *bio_integrity(struct bio *bio)
-+{
-+	return NULL;
-+}
-+
-+static inline int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
-+{
-+	return -EINVAL;
-+}
-+
-+static inline int bio_integrity_map_iter(struct bio *bio, struct uio_meta *meta)
-+{
-+	return -EINVAL;
-+}
-+
-+static inline void bio_integrity_unmap_user(struct bio *bio)
-+{
-+}
-+
-+static inline bool bio_integrity_prep(struct bio *bio)
-+{
-+	return true;
-+}
-+
-+static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
-+		gfp_t gfp_mask)
-+{
-+	return 0;
-+}
-+
-+static inline void bio_integrity_advance(struct bio *bio,
-+		unsigned int bytes_done)
-+{
-+}
-+
-+static inline void bio_integrity_trim(struct bio *bio)
-+{
-+}
-+
-+static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
-+{
-+	return false;
-+}
-+
-+static inline struct bio_integrity_payload *
-+bio_integrity_alloc(struct bio *bio, gfp_t gfp, unsigned int nr)
-+{
-+	return ERR_PTR(-EINVAL);
-+}
-+
-+static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
-+					unsigned int len, unsigned int offset)
-+{
-+	return 0;
-+}
-+#endif /* CONFIG_BLK_DEV_INTEGRITY */
-+
-+void bio_integrity_alloc_buf(struct bio *bio, bool zero_buffer);
-+void bio_integrity_free_buf(struct bio_integrity_payload *bip);
-+
-+#endif /* _LINUX_BIO_INTEGRITY_H */
-diff --git a/include/linux/bio.h b/include/linux/bio.h
-index ca22b06700a9..ad2d57908c1c 100644
--- a/include/linux/bio.h
-+++ b/include/linux/bio.h
-@@ -11,15 +11,15 @@
- #include <linux/uio.h>
- 
- #define BIO_MAX_VECS		256U
-+#define BIO_MAX_INLINE_VECS	UIO_MAXIOV
-+
-+struct queue_limits;
- 
- static inline unsigned int bio_max_segs(unsigned int nr_segs)
- {
- 	return min(nr_segs, BIO_MAX_VECS);
- }
- 
-#define bio_prio(bio)			(bio)->bi_ioprio
-#define bio_set_prio(bio, prio)		((bio)->bi_ioprio = prio)
-
- #define bio_iter_iovec(bio, iter)				\
- 	bvec_iter_bvec((bio)->bi_io_vec, (iter))
- 
-@@ -227,7 +227,7 @@ static inline void bio_cnt_set(struct bio *bio, unsigned int count)
- 
- static inline bool bio_flagged(struct bio *bio, unsigned int bit)
- {
-	return (bio->bi_flags & (1U << bit)) != 0;
-+	return bio->bi_flags & (1U << bit);
- }
- 
- static inline void bio_set_flag(struct bio *bio, unsigned int bit)
-@@ -251,6 +251,11 @@ static inline struct page *bio_first_page_all(struct bio *bio)
- 	return bio_first_bvec_all(bio)->bv_page;
- }
- 
-+static inline struct folio *bio_first_folio_all(struct bio *bio)
-+{
-+	return page_folio(bio_first_page_all(bio));
-+}
-+
- static inline struct bio_vec *bio_last_bvec_all(struct bio *bio)
- {
- 	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
-@@ -279,9 +284,14 @@ static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio,
- {
- 	struct bio_vec *bvec = bio_first_bvec_all(bio) + i;
- 
-+	if (unlikely(i >= bio->bi_vcnt)) {
-+		fi->folio = NULL;
-+		return;
-+	}
-+
- 	fi->folio = page_folio(bvec->bv_page);
- 	fi->offset = bvec->bv_offset +
-			PAGE_SIZE * (bvec->bv_page - &fi->folio->page);
-+			PAGE_SIZE * folio_page_idx(fi->folio, bvec->bv_page);
- 	fi->_seg_count = bvec->bv_len;
- 	fi->length = min(folio_size(fi->folio) - fi->offset, fi->_seg_count);
- 	fi->_next = folio_next(fi->folio);
-@@ -296,10 +306,8 @@ static inline void bio_next_folio(struct folio_iter *fi, struct bio *bio)
- 		fi->offset = 0;
- 		fi->length = min(folio_size(fi->folio), fi->_seg_count);
- 		fi->_next = folio_next(fi->folio);
-	} else if (fi->_i + 1 < bio->bi_vcnt) {
-		bio_first_folio(fi, bio, fi->_i + 1);
- 	} else {
-		fi->folio = NULL;
-+		bio_first_folio(fi, bio, fi->_i + 1);
- 	}
- }
- 
-@@ -311,70 +319,13 @@ static inline void bio_next_folio(struct folio_iter *fi, struct bio *bio)
- #define bio_for_each_folio_all(fi, bio)				\
- 	for (bio_first_folio(&fi, bio, 0); fi.folio; bio_next_folio(&fi, bio))
- 
-enum bip_flags {
-	BIP_BLOCK_INTEGRITY	= 1 << 0, /* block layer owns integrity data */
-	BIP_MAPPED_INTEGRITY	= 1 << 1, /* ref tag has been remapped */
-	BIP_CTRL_NOCHECK	= 1 << 2, /* disable HBA integrity checking */
-	BIP_DISK_NOCHECK	= 1 << 3, /* disable disk integrity checking */
-	BIP_IP_CHECKSUM		= 1 << 4, /* IP checksum */
-};
-
-/*
- * bio integrity payload
- */
-struct bio_integrity_payload {
-	struct bio		*bip_bio;	/* parent bio */
-
-	struct bvec_iter	bip_iter;
-
-	unsigned short		bip_vcnt;	/* # of integrity bio_vecs */
-	unsigned short		bip_max_vcnt;	/* integrity bio_vec slots */
-	unsigned short		bip_flags;	/* control flags */
-
-	struct bvec_iter	bio_iter;	/* for rewinding parent bio */
-
-	struct work_struct	bip_work;	/* I/O completion */
-
-	struct bio_vec		*bip_vec;
-	struct bio_vec		bip_inline_vecs[];/* embedded bvec array */
-};
-
-#if defined(CONFIG_BLK_DEV_INTEGRITY)
-
-static inline struct bio_integrity_payload *bio_integrity(struct bio *bio)
-{
-	if (bio->bi_opf & REQ_INTEGRITY)
-		return bio->bi_integrity;
-
-	return NULL;
-}
-
-static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
-{
-	struct bio_integrity_payload *bip = bio_integrity(bio);
-
-	if (bip)
-		return bip->bip_flags & flag;
-
-	return false;
-}
-
-static inline sector_t bip_get_seed(struct bio_integrity_payload *bip)
-{
-	return bip->bip_iter.bi_sector;
-}
-
-static inline void bip_set_seed(struct bio_integrity_payload *bip,
-				sector_t seed)
-{
-	bip->bip_iter.bi_sector = seed;
-}
-
-#endif /* CONFIG_BLK_DEV_INTEGRITY */
-
- void bio_trim(struct bio *bio, sector_t offset, sector_t size);
- extern struct bio *bio_split(struct bio *bio, int sectors,
- 			     gfp_t gfp, struct bio_set *bs);
-+int bio_split_io_at(struct bio *bio, const struct queue_limits *lim,
-+		unsigned *segs, unsigned max_bytes, unsigned len_align);
-+u8 bio_seg_gap(struct request_queue *q, struct bio *prev, struct bio *next,
-+		u8 gaps_bit);
- 
- /**
-  * bio_next_split - get next @sectors from a bio, splitting if necessary
-@@ -454,23 +405,53 @@ static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs)
- 
- struct request_queue;
- 
-extern int submit_bio_wait(struct bio *bio);
- void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
- 	      unsigned short max_vecs, blk_opf_t opf);
-+static inline void bio_init_inline(struct bio *bio, struct block_device *bdev,
-+	      unsigned short max_vecs, blk_opf_t opf)
-+{
-+	bio_init(bio, bdev, bio_inline_vecs(bio), max_vecs, opf);
-+}
- extern void bio_uninit(struct bio *);
- void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf);
- void bio_chain(struct bio *, struct bio *);
- 
-int bio_add_page(struct bio *, struct page *, unsigned len, unsigned off);
-bool bio_add_folio(struct bio *, struct folio *, size_t len, size_t off);
-extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
-			   unsigned int, unsigned int);
-int bio_add_zone_append_page(struct bio *bio, struct page *page,
-			     unsigned int len, unsigned int offset);
-+int __must_check bio_add_page(struct bio *bio, struct page *page, unsigned len,
-+			      unsigned off);
-+bool __must_check bio_add_folio(struct bio *bio, struct folio *folio,
-+				size_t len, size_t off);
- void __bio_add_page(struct bio *bio, struct page *page,
- 		unsigned int len, unsigned int off);
-int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);
-void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter);
-+void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len,
-+			  size_t off);
-+void bio_add_virt_nofail(struct bio *bio, void *vaddr, unsigned len);
-+
-+/**
-+ * bio_add_max_vecs - number of bio_vecs needed to add data to a bio
-+ * @kaddr: kernel virtual address to add
-+ * @len: length in bytes to add
-+ *
-+ * Calculate how many bio_vecs need to be allocated to add the kernel virtual
-+ * address range in [@kaddr:@len] in the worse case.
-+ */
-+static inline unsigned int bio_add_max_vecs(void *kaddr, unsigned int len)
-+{
-+	if (is_vmalloc_addr(kaddr))
-+		return DIV_ROUND_UP(offset_in_page(kaddr) + len, PAGE_SIZE);
-+	return 1;
-+}
-+
-+unsigned int bio_add_vmalloc_chunk(struct bio *bio, void *vaddr, unsigned len);
-+bool bio_add_vmalloc(struct bio *bio, void *vaddr, unsigned int len);
-+
-+int submit_bio_wait(struct bio *bio);
-+int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data,
-+		size_t len, enum req_op op);
-+
-+int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
-+		unsigned len_align_mask);
-+
-+void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter);
- void __bio_release_pages(struct bio *bio, bool mark_dirty);
- extern void bio_set_pages_dirty(struct bio *bio);
- extern void bio_check_pages_dirty(struct bio *bio);
-@@ -480,11 +461,16 @@ extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
- extern void bio_copy_data(struct bio *dst, struct bio *src);
- extern void bio_free_pages(struct bio *bio);
- void guard_bio_eod(struct bio *bio);
-void zero_fill_bio(struct bio *bio);
-+void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter);
-+
-+static inline void zero_fill_bio(struct bio *bio)
-+{
-+	zero_fill_bio_iter(bio, bio->bi_iter);
-+}
- 
- static inline void bio_release_pages(struct bio *bio, bool mark_dirty)
- {
-	if (!bio_flagged(bio, BIO_NO_PAGE_REF))
-+	if (bio_flagged(bio, BIO_PAGE_PINNED))
- 		__bio_release_pages(bio, mark_dirty);
- }
- 
-@@ -496,6 +482,7 @@ void bio_associate_blkg(struct bio *bio);
- void bio_associate_blkg_from_css(struct bio *bio,
- 				 struct cgroup_subsys_state *css);
- void bio_clone_blkg_association(struct bio *dst, struct bio *src);
-+void blkcg_punt_bio_submit(struct bio *bio);
- #else	/* CONFIG_BLK_CGROUP */
- static inline void bio_associate_blkg(struct bio *bio) { }
- static inline void bio_associate_blkg_from_css(struct bio *bio,
-@@ -503,13 +490,17 @@ static inline void bio_associate_blkg_from_css(struct bio *bio,
- { }
- static inline void bio_clone_blkg_association(struct bio *dst,
- 					      struct bio *src) { }
-+static inline void blkcg_punt_bio_submit(struct bio *bio)
-+{
-+	submit_bio(bio);
-+}
- #endif	/* CONFIG_BLK_CGROUP */
- 
- static inline void bio_set_dev(struct bio *bio, struct block_device *bdev)
- {
- 	bio_clear_flag(bio, BIO_REMAPPED);
- 	if (bio->bi_bdev != bdev)
-		bio_clear_flag(bio, BIO_THROTTLED);
-+		bio_clear_flag(bio, BIO_BPS_THROTTLED);
- 	bio->bi_bdev = bdev;
- 	bio_associate_blkg(bio);
- }
-@@ -587,6 +578,13 @@ static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
- 	bl->tail = bl2->tail;
- }
- 
-+static inline void bio_list_merge_init(struct bio_list *bl,
-+		struct bio_list *bl2)
-+{
-+	bio_list_merge(bl, bl2);
-+	bio_list_init(bl2);
-+}
-+
- static inline void bio_list_merge_head(struct bio_list *bl,
- 				       struct bio_list *bl2)
- {
-@@ -660,10 +658,6 @@ struct bio_set {
- 
- 	mempool_t bio_pool;
- 	mempool_t bvec_pool;
-#if defined(CONFIG_BLK_DEV_INTEGRITY)
-	mempool_t bio_integrity_pool;
-	mempool_t bvec_integrity_pool;
-#endif
- 
- 	unsigned int back_pad;
- 	/*
-@@ -686,88 +680,6 @@ static inline bool bioset_initialized(struct bio_set *bs)
- 	return bs->bio_slab != NULL;
- }
- 
-#if defined(CONFIG_BLK_DEV_INTEGRITY)
-
-#define bip_for_each_vec(bvl, bip, iter)				\
-	for_each_bvec(bvl, (bip)->bip_vec, iter, (bip)->bip_iter)
-
-#define bio_for_each_integrity_vec(_bvl, _bio, _iter)			\
-	for_each_bio(_bio)						\
-		bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
-
-extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
-extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
-extern bool bio_integrity_prep(struct bio *);
-extern void bio_integrity_advance(struct bio *, unsigned int);
-extern void bio_integrity_trim(struct bio *);
-extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
-extern int bioset_integrity_create(struct bio_set *, int);
-extern void bioset_integrity_free(struct bio_set *);
-extern void bio_integrity_init(void);
-
-#else /* CONFIG_BLK_DEV_INTEGRITY */
-
-static inline void *bio_integrity(struct bio *bio)
-{
-	return NULL;
-}
-
-static inline int bioset_integrity_create(struct bio_set *bs, int pool_size)
-{
-	return 0;
-}
-
-static inline void bioset_integrity_free (struct bio_set *bs)
-{
-	return;
-}
-
-static inline bool bio_integrity_prep(struct bio *bio)
-{
-	return true;
-}
-
-static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
-				      gfp_t gfp_mask)
-{
-	return 0;
-}
-
-static inline void bio_integrity_advance(struct bio *bio,
-					 unsigned int bytes_done)
-{
-	return;
-}
-
-static inline void bio_integrity_trim(struct bio *bio)
-{
-	return;
-}
-
-static inline void bio_integrity_init(void)
-{
-	return;
-}
-
-static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
-{
-	return false;
-}
-
-static inline void *bio_integrity_alloc(struct bio * bio, gfp_t gfp,
-								unsigned int nr)
-{
-	return ERR_PTR(-EINVAL);
-}
-
-static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
-					unsigned int len, unsigned int offset)
-{
-	return 0;
-}
-
-#endif /* CONFIG_BLK_DEV_INTEGRITY */
-
- /*
-  * Mark a bio as polled. Note that for async polled IO, the caller must
-  * expect -EWOULDBLOCK if we cannot allocate a request (or other resources).
-@@ -778,17 +690,37 @@ static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
- static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb)
- {
- 	bio->bi_opf |= REQ_POLLED;
-	if (!is_sync_kiocb(kiocb))
-+	if (kiocb->ki_flags & IOCB_NOWAIT)
- 		bio->bi_opf |= REQ_NOWAIT;
- }
- 
- static inline void bio_clear_polled(struct bio *bio)
- {
-	/* can't support alloc cache if we turn off polling */
-	bio->bi_opf &= ~(REQ_POLLED | REQ_ALLOC_CACHE);
-+	bio->bi_opf &= ~REQ_POLLED;
-+}
-+
-+/**
-+ * bio_is_zone_append - is this a zone append bio?
-+ * @bio:	bio to check
-+ *
-+ * Check if @bio is a zone append operation.  Core block layer code and end_io
-+ * handlers must use this instead of an open coded REQ_OP_ZONE_APPEND check
-+ * because the block layer can rewrite REQ_OP_ZONE_APPEND to REQ_OP_WRITE if
-+ * it is not natively supported.
-+ */
-+static inline bool bio_is_zone_append(struct bio *bio)
-+{
-+	if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED))
-+		return false;
-+	return bio_op(bio) == REQ_OP_ZONE_APPEND ||
-+		bio_flagged(bio, BIO_EMULATES_ZONE_APPEND);
- }
- 
- struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev,
- 		unsigned int nr_pages, blk_opf_t opf, gfp_t gfp);
-+struct bio *bio_chain_and_submit(struct bio *prev, struct bio *new);
-+
-+struct bio *blk_alloc_discard_bio(struct block_device *bdev,
-+		sector_t *sector, sector_t *nr_sects, gfp_t gfp_mask);
- 
- #endif /* __LINUX_BIO_H */
-diff --git a/include/linux/bit_spinlock.h b/include/linux/bit_spinlock.h
-index bbc4730a6505..c0989b5b0407 100644
--- a/include/linux/bit_spinlock.h
-+++ b/include/linux/bit_spinlock.h
-@@ -13,7 +13,7 @@
-  * Don't use this unless you really need to: spin_lock() and spin_unlock()
-  * are significantly faster.
-  */
-static inline void bit_spin_lock(int bitnum, unsigned long *addr)
-+static __always_inline void bit_spin_lock(int bitnum, unsigned long *addr)
- {
- 	/*
- 	 * Assuming the lock is uncontended, this never enters
-@@ -38,7 +38,7 @@ static inline void bit_spin_lock(int bitnum, unsigned long *addr)
- /*
-  * Return true if it was acquired
-  */
-static inline int bit_spin_trylock(int bitnum, unsigned long *addr)
-+static __always_inline int bit_spin_trylock(int bitnum, unsigned long *addr)
- {
- 	preempt_disable();
- #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
-@@ -54,7 +54,7 @@ static inline int bit_spin_trylock(int bitnum, unsigned long *addr)
- /*
-  *  bit-based spin_unlock()
-  */
-static inline void bit_spin_unlock(int bitnum, unsigned long *addr)
-+static __always_inline void bit_spin_unlock(int bitnum, unsigned long *addr)
- {
- #ifdef CONFIG_DEBUG_SPINLOCK
- 	BUG_ON(!test_bit(bitnum, addr));
-@@ -71,7 +71,7 @@ static inline void bit_spin_unlock(int bitnum, unsigned long *addr)
-  *  non-atomic version, which can be used eg. if the bit lock itself is
-  *  protecting the rest of the flags in the word.
-  */
-static inline void __bit_spin_unlock(int bitnum, unsigned long *addr)
-+static __always_inline void __bit_spin_unlock(int bitnum, unsigned long *addr)
- {
- #ifdef CONFIG_DEBUG_SPINLOCK
- 	BUG_ON(!test_bit(bitnum, addr));
-diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h
-index c9be1657f03d..126dc5b380af 100644
--- a/include/linux/bitfield.h
-+++ b/include/linux/bitfield.h
-@@ -8,6 +8,7 @@
- #define _LINUX_BITFIELD_H
- 
- #include <linux/build_bug.h>
-+#include <linux/typecheck.h>
- #include <asm/byteorder.h>
- 
- /*
-@@ -16,6 +17,7 @@
-  * FIELD_{GET,PREP} macros take as first parameter shifted mask
-  * from which they extract the base mask and shift amount.
-  * Mask must be a compilation time constant.
-+ * field_{get,prep} are variants that take a non-const mask.
-  *
-  * Example:
-  *
-@@ -38,8 +40,7 @@
-  *	  FIELD_PREP(REG_FIELD_D, 0x40);
-  *
-  * Modify:
- *  reg &= ~REG_FIELD_C;
- *  reg |= FIELD_PREP(REG_FIELD_C, c);
-+ *  FIELD_MODIFY(REG_FIELD_C, &reg, c);
-  */
- 
- #define __bf_shf(x) (__builtin_ffsll(x) - 1)
-@@ -60,21 +61,42 @@
- 
- #define __bf_cast_unsigned(type, x)	((__unsigned_scalar_typeof(type))(x))
- 
-#define __BF_FIELD_CHECK(_mask, _reg, _val, _pfx)			\
-+#define __BF_FIELD_CHECK_MASK(_mask, _val, _pfx)			\
- 	({								\
- 		BUILD_BUG_ON_MSG(!__builtin_constant_p(_mask),		\
- 				 _pfx "mask is not constant");		\
- 		BUILD_BUG_ON_MSG((_mask) == 0, _pfx "mask is zero");	\
- 		BUILD_BUG_ON_MSG(__builtin_constant_p(_val) ?		\
-				 ~((_mask) >> __bf_shf(_mask)) & (_val) : 0, \
-+				 ~((_mask) >> __bf_shf(_mask)) &	\
-+					(0 + (_val)) : 0,		\
- 				 _pfx "value too large for the field"); \
-		BUILD_BUG_ON_MSG(__bf_cast_unsigned(_mask, _mask) >	\
-				 __bf_cast_unsigned(_reg, ~0ull),	\
-				 _pfx "type of reg too small for mask"); \
- 		__BUILD_BUG_ON_NOT_POWER_OF_2((_mask) +			\
- 					      (1ULL << __bf_shf(_mask))); \
- 	})
- 
-+#define __BF_FIELD_CHECK_REG(mask, reg, pfx)				\
-+	BUILD_BUG_ON_MSG(__bf_cast_unsigned(mask, mask) >		\
-+			 __bf_cast_unsigned(reg, ~0ull),		\
-+			 pfx "type of reg too small for mask")
-+
-+#define __BF_FIELD_CHECK(mask, reg, val, pfx)				\
-+	({								\
-+		__BF_FIELD_CHECK_MASK(mask, val, pfx);			\
-+		__BF_FIELD_CHECK_REG(mask, reg, pfx);			\
-+	})
-+
-+#define __FIELD_PREP(mask, val, pfx)					\
-+	({								\
-+		__BF_FIELD_CHECK_MASK(mask, val, pfx);			\
-+		((typeof(mask))(val) << __bf_shf(mask)) & (mask);	\
-+	})
-+
-+#define __FIELD_GET(mask, reg, pfx)					\
-+	({								\
-+		__BF_FIELD_CHECK_MASK(mask, 0U, pfx);			\
-+		(typeof(mask))(((reg) & (mask)) >> __bf_shf(mask));	\
-+	})
-+
- /**
-  * FIELD_MAX() - produce the maximum value representable by a field
-  * @_mask: shifted mask defining the field's length and position
-@@ -111,10 +133,36 @@
-  */
- #define FIELD_PREP(_mask, _val)						\
- 	({								\
-		__BF_FIELD_CHECK(_mask, 0ULL, _val, "FIELD_PREP: ");	\
-		((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask);	\
-+		__BF_FIELD_CHECK_REG(_mask, 0ULL, "FIELD_PREP: ");	\
-+		__FIELD_PREP(_mask, _val, "FIELD_PREP: ");		\
- 	})
- 
-+#define __BF_CHECK_POW2(n)	BUILD_BUG_ON_ZERO(((n) & ((n) - 1)) != 0)
-+
-+/**
-+ * FIELD_PREP_CONST() - prepare a constant bitfield element
-+ * @_mask: shifted mask defining the field's length and position
-+ * @_val:  value to put in the field
-+ *
-+ * FIELD_PREP_CONST() masks and shifts up the value.  The result should
-+ * be combined with other fields of the bitfield using logical OR.
-+ *
-+ * Unlike FIELD_PREP() this is a constant expression and can therefore
-+ * be used in initializers. Error checking is less comfortable for this
-+ * version, and non-constant masks cannot be used.
-+ */
-+#define FIELD_PREP_CONST(_mask, _val)					\
-+	(								\
-+		/* mask must be non-zero */				\
-+		BUILD_BUG_ON_ZERO((_mask) == 0) +			\
-+		/* check if value fits */				\
-+		BUILD_BUG_ON_ZERO(~((_mask) >> __bf_shf(_mask)) & (_val)) + \
-+		/* check if mask is contiguous */			\
-+		__BF_CHECK_POW2((_mask) + (1ULL << __bf_shf(_mask))) +	\
-+		/* and create the value */				\
-+		(((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask))	\
-+	)
-+
- /**
-  * FIELD_GET() - extract a bitfield element
-  * @_mask: shifted mask defining the field's length and position
-@@ -125,8 +173,25 @@
-  */
- #define FIELD_GET(_mask, _reg)						\
- 	({								\
-		__BF_FIELD_CHECK(_mask, _reg, 0U, "FIELD_GET: ");	\
-		(typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask));	\
-+		__BF_FIELD_CHECK_REG(_mask, _reg, "FIELD_GET: ");	\
-+		__FIELD_GET(_mask, _reg, "FIELD_GET: ");		\
-+	})
-+
-+/**
-+ * FIELD_MODIFY() - modify a bitfield element
-+ * @_mask: shifted mask defining the field's length and position
-+ * @_reg_p: pointer to the memory that should be updated
-+ * @_val: value to store in the bitfield
-+ *
-+ * FIELD_MODIFY() modifies the set of bits in @_reg_p specified by @_mask,
-+ * by replacing them with the bitfield value passed in as @_val.
-+ */
-+#define FIELD_MODIFY(_mask, _reg_p, _val)						\
-+	({										\
-+		typecheck_pointer(_reg_p);						\
-+		__BF_FIELD_CHECK(_mask, *(_reg_p), _val, "FIELD_MODIFY: ");		\
-+		*(_reg_p) &= ~(_mask);							\
-+		*(_reg_p) |= (((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask));	\
- 	})
- 
- extern void __compiletime_error("value doesn't fit into mask")
-@@ -145,14 +210,14 @@ static __always_inline u64 field_mask(u64 field)
- }
- #define field_max(field)	((typeof(field))field_mask(field))
- #define ____MAKE_OP(type,base,to,from)					\
-static __always_inline __##type type##_encode_bits(base v, base field)	\
-+static __always_inline __##type __must_check type##_encode_bits(base v, base field)	\
- {									\
- 	if (__builtin_constant_p(v) && (v & ~field_mask(field)))	\
- 		__field_overflow();					\
- 	return to((v & field_mask(field)) * field_multiplier(field));	\
- }									\
-static __always_inline __##type type##_replace_bits(__##type old,	\
-					base val, base field)		\
-+static __always_inline __##type __must_check type##_replace_bits(__##type old,	\
-+							base val, base field)	\
- {									\
- 	return (old & ~to(field)) | type##_encode_bits(val, field);	\
- }									\
-@@ -161,7 +226,7 @@ static __always_inline void type##p_replace_bits(__##type *p,		\
- {									\
- 	*p = (*p & ~to(field)) | type##_encode_bits(val, field);	\
- }									\
-static __always_inline base type##_get_bits(__##type v, base field)	\
-+static __always_inline base __must_check type##_get_bits(__##type v, base field)	\
- {									\
- 	return (from(v) & field)/field_multiplier(field);		\
- }
-@@ -176,4 +241,62 @@ __MAKE_OP(64)
- #undef __MAKE_OP
- #undef ____MAKE_OP
- 
-+#define __field_prep(mask, val)						\
-+	({								\
-+		__auto_type __mask = (mask);				\
-+		typeof(__mask) __val = (val);				\
-+		unsigned int __shift = BITS_PER_TYPE(__mask) <= 32 ?	\
-+				       __ffs(__mask) : __ffs64(__mask);	\
-+		(__val << __shift) & __mask;				\
-+	})
-+
-+#define __field_get(mask, reg)						\
-+	({								\
-+		__auto_type __mask = (mask);				\
-+		typeof(__mask) __reg =  (reg);				\
-+		unsigned int __shift = BITS_PER_TYPE(__mask) <= 32 ?	\
-+				       __ffs(__mask) : __ffs64(__mask);	\
-+		(__reg & __mask) >> __shift;				\
-+	})
-+
-+/**
-+ * field_prep() - prepare a bitfield element
-+ * @mask: shifted mask defining the field's length and position, must be
-+ *        non-zero
-+ * @val:  value to put in the field
-+ *
-+ * Return: field value masked and shifted to its final destination
-+ *
-+ * field_prep() masks and shifts up the value.  The result should be
-+ * combined with other fields of the bitfield using logical OR.
-+ * Unlike FIELD_PREP(), @mask is not limited to a compile-time constant.
-+ * Typical usage patterns are a value stored in a table, or calculated by
-+ * shifting a constant by a variable number of bits.
-+ * If you want to ensure that @mask is a compile-time constant, please use
-+ * FIELD_PREP() directly instead.
-+ */
-+#define field_prep(mask, val)						\
-+	(__builtin_constant_p(mask) ? __FIELD_PREP(mask, val, "field_prep: ") \
-+				    : __field_prep(mask, val))
-+
-+/**
-+ * field_get() - extract a bitfield element
-+ * @mask: shifted mask defining the field's length and position, must be
-+ *        non-zero
-+ * @reg:  value of entire bitfield
-+ *
-+ * Return: extracted field value
-+ *
-+ * field_get() extracts the field specified by @mask from the
-+ * bitfield passed in as @reg by masking and shifting it down.
-+ * Unlike FIELD_GET(), @mask is not limited to a compile-time constant.
-+ * Typical usage patterns are a value stored in a table, or calculated by
-+ * shifting a constant by a variable number of bits.
-+ * If you want to ensure that @mask is a compile-time constant, please use
-+ * FIELD_GET() directly instead.
-+ */
-+#define field_get(mask, reg)						\
-+	(__builtin_constant_p(mask) ? __FIELD_GET(mask, reg, "field_get: ") \
-+				    : __field_get(mask, reg))
-+
- #endif
-diff --git a/include/linux/bitmap-str.h b/include/linux/bitmap-str.h
-new file mode 100644
-index 000000000000..53d3e1b32d3d
--- /dev/null
-+++ b/include/linux/bitmap-str.h
-@@ -0,0 +1,18 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef __LINUX_BITMAP_STR_H
-+#define __LINUX_BITMAP_STR_H
-+
-+#include <linux/types.h>
-+
-+int bitmap_parse_user(const char __user *ubuf, unsigned int ulen, unsigned long *dst, int nbits);
-+int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, int nmaskbits);
-+int bitmap_print_bitmask_to_buf(char *buf, const unsigned long *maskp, int nmaskbits,
-+				loff_t off, size_t count);
-+int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp, int nmaskbits,
-+			     loff_t off, size_t count);
-+int bitmap_parse(const char *buf, unsigned int buflen, unsigned long *dst, int nbits);
-+int bitmap_parselist(const char *buf, unsigned long *maskp, int nmaskbits);
-+int bitmap_parselist_user(const char __user *ubuf, unsigned int ulen,
-+					unsigned long *dst, int nbits);
-+
-+#endif /* __LINUX_BITMAP_STR_H */
-diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h
-index f65410a49fda..b0395e4ccf90 100644
--- a/include/linux/bitmap.h
-+++ b/include/linux/bitmap.h
-@@ -6,10 +6,13 @@
- 
- #include <linux/align.h>
- #include <linux/bitops.h>
-+#include <linux/cleanup.h>
-+#include <linux/errno.h>
- #include <linux/find.h>
- #include <linux/limits.h>
- #include <linux/string.h>
- #include <linux/types.h>
-+#include <linux/bitmap-str.h>
- 
- struct device;
- 
-@@ -20,7 +23,7 @@ struct device;
-  *
-  * Function implementations generic to all architectures are in
-  * lib/bitmap.c.  Functions implementations that are architecture
- * specific are in various include/asm-<arch>/bitops.h headers
-+ * specific are in various arch/<arch>/include/asm/bitops.h headers
-  * and other arch/<arch> specific files.
-  *
-  * See lib/bitmap.c for more details.
-@@ -42,6 +45,7 @@ struct device;
-  *  bitmap_copy(dst, src, nbits)                *dst = *src
-  *  bitmap_and(dst, src1, src2, nbits)          *dst = *src1 & *src2
-  *  bitmap_or(dst, src1, src2, nbits)           *dst = *src1 | *src2
-+ *  bitmap_weighted_or(dst, src1, src2, nbits)	*dst = *src1 | *src2. Returns Hamming Weight of dst
-  *  bitmap_xor(dst, src1, src2, nbits)          *dst = *src1 ^ *src2
-  *  bitmap_andnot(dst, src1, src2, nbits)       *dst = *src1 & ~(*src2)
-  *  bitmap_complement(dst, src, nbits)          *dst = ~(*src)
-@@ -51,6 +55,8 @@ struct device;
-  *  bitmap_empty(src, nbits)                    Are all bits zero in *src?
-  *  bitmap_full(src, nbits)                     Are all bits set in *src?
-  *  bitmap_weight(src, nbits)                   Hamming Weight: number set bits
-+ *  bitmap_weight_and(src1, src2, nbits)        Hamming Weight of and'ed bitmap
-+ *  bitmap_weight_andnot(src1, src2, nbits)     Hamming Weight of andnot'ed bitmap
-  *  bitmap_set(dst, pos, nbits)                 Set specified bit area
-  *  bitmap_clear(dst, pos, nbits)               Clear specified bit area
-  *  bitmap_find_next_zero_area(buf, len, pos, n, mask)  Find bit free area
-@@ -59,6 +65,8 @@ struct device;
-  *  bitmap_shift_left(dst, src, n, nbits)       *dst = *src << n
-  *  bitmap_cut(dst, src, first, n, nbits)       Cut n bits from first, copy rest
-  *  bitmap_replace(dst, old, new, mask, nbits)  *dst = (*old & ~(*mask)) | (*new & *mask)
-+ *  bitmap_scatter(dst, src, mask, nbits)	*dst = map(dense, sparse)(src)
-+ *  bitmap_gather(dst, src, mask, nbits)	*dst = map(sparse, dense)(src)
-  *  bitmap_remap(dst, src, old, new, nbits)     *dst = map(old, new)(src)
-  *  bitmap_bitremap(oldbit, old, new, nbits)    newbit = map(old, new)(oldbit)
-  *  bitmap_onto(dst, orig, relmap, nbits)       *dst = orig relative to relmap
-@@ -76,6 +84,10 @@ struct device;
-  *  bitmap_to_arr64(buf, src, nbits)            Copy nbits from buf to u64[] dst
-  *  bitmap_get_value8(map, start)               Get 8bit value from map at start
-  *  bitmap_set_value8(map, value, start)        Set 8bit value to map at start
-+ *  bitmap_read(map, start, nbits)              Read an nbits-sized value from
-+ *                                              map at start
-+ *  bitmap_write(map, value, start, nbits)      Write an nbits-sized value to
-+ *                                              map at start
-  *
-  * Note, bitmap_zero() and bitmap_fill() operate over the region of
-  * unsigned longs, that is, bits behind bitmap till the unsigned long
-@@ -124,6 +136,8 @@ unsigned long *bitmap_alloc_node(unsigned int nbits, gfp_t flags, int node);
- unsigned long *bitmap_zalloc_node(unsigned int nbits, gfp_t flags, int node);
- void bitmap_free(const unsigned long *bitmap);
- 
-+DEFINE_FREE(bitmap, unsigned long *, if (_T) bitmap_free(_T))
-+
- /* Managed variants of the above. */
- unsigned long *devm_bitmap_alloc(struct device *dev,
- 				 unsigned int nbits, gfp_t flags);
-@@ -152,6 +166,8 @@ bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
- 		 const unsigned long *bitmap2, unsigned int nbits);
- void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
- 		 const unsigned long *bitmap2, unsigned int nbits);
-+unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1,
-+				  const unsigned long *bitmap2, unsigned int nbits);
- void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
- 		  const unsigned long *bitmap2, unsigned int nbits);
- bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
-@@ -164,6 +180,10 @@ bool __bitmap_intersects(const unsigned long *bitmap1,
- bool __bitmap_subset(const unsigned long *bitmap1,
- 		     const unsigned long *bitmap2, unsigned int nbits);
- unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits);
-+unsigned int __bitmap_weight_and(const unsigned long *bitmap1,
-+				 const unsigned long *bitmap2, unsigned int nbits);
-+unsigned int __bitmap_weight_andnot(const unsigned long *bitmap1,
-+				    const unsigned long *bitmap2, unsigned int nbits);
- void __bitmap_set(unsigned long *map, unsigned int start, int len);
- void __bitmap_clear(unsigned long *map, unsigned int start, int len);
- 
-@@ -186,25 +206,17 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map,
-  * the bit offset of all zero areas this function finds is multiples of that
-  * power of 2. A @align_mask of 0 means no alignment is required.
-  */
-static inline unsigned long
-bitmap_find_next_zero_area(unsigned long *map,
-			   unsigned long size,
-			   unsigned long start,
-			   unsigned int nr,
-			   unsigned long align_mask)
-+static __always_inline
-+unsigned long bitmap_find_next_zero_area(unsigned long *map,
-+					 unsigned long size,
-+					 unsigned long start,
-+					 unsigned int nr,
-+					 unsigned long align_mask)
- {
- 	return bitmap_find_next_zero_area_off(map, size, start, nr,
- 					      align_mask, 0);
- }
- 
-int bitmap_parse(const char *buf, unsigned int buflen,
-			unsigned long *dst, int nbits);
-int bitmap_parse_user(const char __user *ubuf, unsigned int ulen,
-			unsigned long *dst, int nbits);
-int bitmap_parselist(const char *buf, unsigned long *maskp,
-			int nmaskbits);
-int bitmap_parselist_user(const char __user *ubuf, unsigned int ulen,
-			unsigned long *dst, int nbits);
- void bitmap_remap(unsigned long *dst, const unsigned long *src,
- 		const unsigned long *old, const unsigned long *new, unsigned int nbits);
- int bitmap_bitremap(int oldbit,
-@@ -213,31 +225,15 @@ void bitmap_onto(unsigned long *dst, const unsigned long *orig,
- 		const unsigned long *relmap, unsigned int bits);
- void bitmap_fold(unsigned long *dst, const unsigned long *orig,
- 		unsigned int sz, unsigned int nbits);
-int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order);
-void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order);
-int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order);
-
-#ifdef __BIG_ENDIAN
-void bitmap_copy_le(unsigned long *dst, const unsigned long *src, unsigned int nbits);
-#else
-#define bitmap_copy_le bitmap_copy
-#endif
-unsigned int bitmap_ord_to_pos(const unsigned long *bitmap, unsigned int ord, unsigned int nbits);
-int bitmap_print_to_pagebuf(bool list, char *buf,
-				   const unsigned long *maskp, int nmaskbits);
-
-extern int bitmap_print_bitmask_to_buf(char *buf, const unsigned long *maskp,
-				      int nmaskbits, loff_t off, size_t count);
-
-extern int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp,
-				      int nmaskbits, loff_t off, size_t count);
- 
- #define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) & (BITS_PER_LONG - 1)))
- #define BITMAP_LAST_WORD_MASK(nbits) (~0UL >> (-(nbits) & (BITS_PER_LONG - 1)))
- 
-static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
-+#define bitmap_size(nbits)	(ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE)
-+
-+static __always_inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
- {
-	unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
-+	unsigned int len = bitmap_size(nbits);
- 
- 	if (small_const_nbits(nbits))
- 		*dst = 0;
-@@ -245,9 +241,9 @@ static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
- 		memset(dst, 0, len);
- }
- 
-static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
-+static __always_inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
- {
-	unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
-+	unsigned int len = bitmap_size(nbits);
- 
- 	if (small_const_nbits(nbits))
- 		*dst = ~0UL;
-@@ -255,10 +251,10 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
- 		memset(dst, 0xff, len);
- }
- 
-static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,
-			unsigned int nbits)
-+static __always_inline
-+void bitmap_copy(unsigned long *dst, const unsigned long *src, unsigned int nbits)
- {
-	unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
-+	unsigned int len = bitmap_size(nbits);
- 
- 	if (small_const_nbits(nbits))
- 		*dst = *src;
-@@ -269,14 +265,26 @@ static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,
- /*
-  * Copy bitmap and clear tail bits in last word.
-  */
-static inline void bitmap_copy_clear_tail(unsigned long *dst,
-		const unsigned long *src, unsigned int nbits)
-+static __always_inline
-+void bitmap_copy_clear_tail(unsigned long *dst, const unsigned long *src, unsigned int nbits)
- {
- 	bitmap_copy(dst, src, nbits);
- 	if (nbits % BITS_PER_LONG)
- 		dst[nbits / BITS_PER_LONG] &= BITMAP_LAST_WORD_MASK(nbits);
- }
- 
-+static inline void bitmap_copy_and_extend(unsigned long *to,
-+					  const unsigned long *from,
-+					  unsigned int count, unsigned int size)
-+{
-+	unsigned int copy = BITS_TO_LONGS(count);
-+
-+	memcpy(to, from, copy * sizeof(long));
-+	if (count % BITS_PER_LONG)
-+		to[copy - 1] &= BITMAP_LAST_WORD_MASK(count);
-+	memset(to + copy, 0, bitmap_size(size) - copy * sizeof(long));
-+}
-+
- /*
-  * On 32-bit systems bitmaps are represented as u32 arrays internally. On LE64
-  * machines the order of hi and lo parts of numbers match the bitmap structure.
-@@ -300,12 +308,10 @@ void bitmap_to_arr32(u32 *buf, const unsigned long *bitmap,
- #endif
- 
- /*
- * On 64-bit systems bitmaps are represented as u64 arrays internally. On LE32
- * machines the order of hi and lo parts of numbers match the bitmap structure.
- * In both cases conversion is not needed when copying data from/to arrays of
- * u64.
-+ * On 64-bit systems bitmaps are represented as u64 arrays internally. So,
-+ * the conversion is not needed when copying data from/to arrays of u64.
-  */
-#if (BITS_PER_LONG == 32) && defined(__BIG_ENDIAN)
-+#if BITS_PER_LONG == 32
- void bitmap_from_arr64(unsigned long *bitmap, const u64 *buf, unsigned int nbits);
- void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits);
- #else
-@@ -315,16 +321,18 @@ void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits);
- 	bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits))
- #endif
- 
-static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1,
-			const unsigned long *src2, unsigned int nbits)
-+static __always_inline
-+bool bitmap_and(unsigned long *dst, const unsigned long *src1,
-+		const unsigned long *src2, unsigned int nbits)
- {
- 	if (small_const_nbits(nbits))
- 		return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0;
- 	return __bitmap_and(dst, src1, src2, nbits);
- }
- 
-static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,
-			const unsigned long *src2, unsigned int nbits)
-+static __always_inline
-+void bitmap_or(unsigned long *dst, const unsigned long *src1,
-+	       const unsigned long *src2, unsigned int nbits)
- {
- 	if (small_const_nbits(nbits))
- 		*dst = *src1 | *src2;
-@@ -332,8 +340,21 @@ static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,
- 		__bitmap_or(dst, src1, src2, nbits);
- }
- 
-static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,
-			const unsigned long *src2, unsigned int nbits)
-+static __always_inline
-+unsigned int bitmap_weighted_or(unsigned long *dst, const unsigned long *src1,
-+				const unsigned long *src2, unsigned int nbits)
-+{
-+	if (small_const_nbits(nbits)) {
-+		*dst = *src1 | *src2;
-+		return hweight_long(*dst & BITMAP_LAST_WORD_MASK(nbits));
-+	} else {
-+		return __bitmap_weighted_or(dst, src1, src2, nbits);
-+	}
-+}
-+
-+static __always_inline
-+void bitmap_xor(unsigned long *dst, const unsigned long *src1,
-+		const unsigned long *src2, unsigned int nbits)
- {
- 	if (small_const_nbits(nbits))
- 		*dst = *src1 ^ *src2;
-@@ -341,16 +362,17 @@ static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,
- 		__bitmap_xor(dst, src1, src2, nbits);
- }
- 
-static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,
-			const unsigned long *src2, unsigned int nbits)
-+static __always_inline
-+bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,
-+		   const unsigned long *src2, unsigned int nbits)
- {
- 	if (small_const_nbits(nbits))
- 		return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
- 	return __bitmap_andnot(dst, src1, src2, nbits);
- }
- 
-static inline void bitmap_complement(unsigned long *dst, const unsigned long *src,
-			unsigned int nbits)
-+static __always_inline
-+void bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int nbits)
- {
- 	if (small_const_nbits(nbits))
- 		*dst = ~(*src);
-@@ -365,8 +387,8 @@ static inline void bitmap_complement(unsigned long *dst, const unsigned long *sr
- #endif
- #define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1)
- 
-static inline bool bitmap_equal(const unsigned long *src1,
-				const unsigned long *src2, unsigned int nbits)
-+static __always_inline
-+bool bitmap_equal(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
- {
- 	if (small_const_nbits(nbits))
- 		return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));
-@@ -385,10 +407,9 @@ static inline bool bitmap_equal(const unsigned long *src1,
-  *
-  * Returns: True if (*@src1 | *@src2) == *@src3, false otherwise
-  */
-static inline bool bitmap_or_equal(const unsigned long *src1,
-				   const unsigned long *src2,
-				   const unsigned long *src3,
-				   unsigned int nbits)
-+static __always_inline
-+bool bitmap_or_equal(const unsigned long *src1, const unsigned long *src2,
-+		     const unsigned long *src3, unsigned int nbits)
- {
- 	if (!small_const_nbits(nbits))
- 		return __bitmap_or_equal(src1, src2, src3, nbits);
-@@ -396,9 +417,8 @@ static inline bool bitmap_or_equal(const unsigned long *src1,
- 	return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits));
- }
- 
-static inline bool bitmap_intersects(const unsigned long *src1,
-				     const unsigned long *src2,
-				     unsigned int nbits)
-+static __always_inline
-+bool bitmap_intersects(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
- {
- 	if (small_const_nbits(nbits))
- 		return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
-@@ -406,8 +426,8 @@ static inline bool bitmap_intersects(const unsigned long *src1,
- 		return __bitmap_intersects(src1, src2, nbits);
- }
- 
-static inline bool bitmap_subset(const unsigned long *src1,
-				 const unsigned long *src2, unsigned int nbits)
-+static __always_inline
-+bool bitmap_subset(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
- {
- 	if (small_const_nbits(nbits))
- 		return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits));
-@@ -415,7 +435,8 @@ static inline bool bitmap_subset(const unsigned long *src1,
- 		return __bitmap_subset(src1, src2, nbits);
- }
- 
-static inline bool bitmap_empty(const unsigned long *src, unsigned nbits)
-+static __always_inline
-+bool bitmap_empty(const unsigned long *src, unsigned nbits)
- {
- 	if (small_const_nbits(nbits))
- 		return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
-@@ -423,7 +444,8 @@ static inline bool bitmap_empty(const unsigned long *src, unsigned nbits)
- 	return find_first_bit(src, nbits) == nbits;
- }
- 
-static inline bool bitmap_full(const unsigned long *src, unsigned int nbits)
-+static __always_inline
-+bool bitmap_full(const unsigned long *src, unsigned int nbits)
- {
- 	if (small_const_nbits(nbits))
- 		return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
-@@ -439,8 +461,26 @@ unsigned int bitmap_weight(const unsigned long *src, unsigned int nbits)
- 	return __bitmap_weight(src, nbits);
- }
- 
-static __always_inline void bitmap_set(unsigned long *map, unsigned int start,
-		unsigned int nbits)
-+static __always_inline
-+unsigned long bitmap_weight_and(const unsigned long *src1,
-+				const unsigned long *src2, unsigned int nbits)
-+{
-+	if (small_const_nbits(nbits))
-+		return hweight_long(*src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits));
-+	return __bitmap_weight_and(src1, src2, nbits);
-+}
-+
-+static __always_inline
-+unsigned long bitmap_weight_andnot(const unsigned long *src1,
-+				   const unsigned long *src2, unsigned int nbits)
-+{
-+	if (small_const_nbits(nbits))
-+		return hweight_long(*src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits));
-+	return __bitmap_weight_andnot(src1, src2, nbits);
-+}
-+
-+static __always_inline
-+void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits)
- {
- 	if (__builtin_constant_p(nbits) && nbits == 1)
- 		__set_bit(start, map);
-@@ -455,8 +495,8 @@ static __always_inline void bitmap_set(unsigned long *map, unsigned int start,
- 		__bitmap_set(map, start, nbits);
- }
- 
-static __always_inline void bitmap_clear(unsigned long *map, unsigned int start,
-		unsigned int nbits)
-+static __always_inline
-+void bitmap_clear(unsigned long *map, unsigned int start, unsigned int nbits)
- {
- 	if (__builtin_constant_p(nbits) && nbits == 1)
- 		__clear_bit(start, map);
-@@ -471,8 +511,9 @@ static __always_inline void bitmap_clear(unsigned long *map, unsigned int start,
- 		__bitmap_clear(map, start, nbits);
- }
- 
-static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *src,
-				unsigned int shift, unsigned int nbits)
-+static __always_inline
-+void bitmap_shift_right(unsigned long *dst, const unsigned long *src,
-+			unsigned int shift, unsigned int nbits)
- {
- 	if (small_const_nbits(nbits))
- 		*dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> shift;
-@@ -480,8 +521,9 @@ static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *s
- 		__bitmap_shift_right(dst, src, shift, nbits);
- }
- 
-static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *src,
-				unsigned int shift, unsigned int nbits)
-+static __always_inline
-+void bitmap_shift_left(unsigned long *dst, const unsigned long *src,
-+		       unsigned int shift, unsigned int nbits)
- {
- 	if (small_const_nbits(nbits))
- 		*dst = (*src << shift) & BITMAP_LAST_WORD_MASK(nbits);
-@@ -489,11 +531,12 @@ static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *sr
- 		__bitmap_shift_left(dst, src, shift, nbits);
- }
- 
-static inline void bitmap_replace(unsigned long *dst,
-				  const unsigned long *old,
-				  const unsigned long *new,
-				  const unsigned long *mask,
-				  unsigned int nbits)
-+static __always_inline
-+void bitmap_replace(unsigned long *dst,
-+		    const unsigned long *old,
-+		    const unsigned long *new,
-+		    const unsigned long *mask,
-+		    unsigned int nbits)
- {
- 	if (small_const_nbits(nbits))
- 		*dst = (*old & ~(*mask)) | (*new & *mask);
-@@ -501,15 +544,183 @@ static inline void bitmap_replace(unsigned long *dst,
- 		__bitmap_replace(dst, old, new, mask, nbits);
- }
- 
-static inline void bitmap_next_set_region(unsigned long *bitmap,
-					  unsigned int *rs, unsigned int *re,
-					  unsigned int end)
-+/**
-+ * bitmap_scatter - Scatter a bitmap according to the given mask
-+ * @dst: scattered bitmap
-+ * @src: gathered bitmap
-+ * @mask: mask representing bits to assign to in the scattered bitmap
-+ * @nbits: number of bits in each of these bitmaps
-+ *
-+ * Scatters bitmap with sequential bits according to the given @mask.
-+ *
-+ * Example:
-+ * If @src bitmap = 0x005a, with @mask = 0x1313, @dst will be 0x0302.
-+ *
-+ * Or in binary form
-+ * @src			@mask			@dst
-+ * 0000000001011010	0001001100010011	0000001100000010
-+ *
-+ * (Bits 0, 1, 2, 3, 4, 5 are copied to the bits 0, 1, 4, 8, 9, 12)
-+ *
-+ * A more 'visual' description of the operation::
-+ *
-+ *	src:  0000000001011010
-+ *	                ||||||
-+ *	         +------+|||||
-+ *	         |  +----+||||
-+ *	         |  |+----+|||
-+ *	         |  ||   +-+||
-+ *	         |  ||   |  ||
-+ *	mask: ...v..vv...v..vv
-+ *	      ...0..11...0..10
-+ *	dst:  0000001100000010
-+ *
-+ * A relationship exists between bitmap_scatter() and bitmap_gather(). See
-+ * bitmap_gather() for the bitmap gather detailed operations. TL;DR:
-+ * bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation.
-+ */
-+static __always_inline
-+void bitmap_scatter(unsigned long *dst, const unsigned long *src,
-+		    const unsigned long *mask, unsigned int nbits)
-+{
-+	unsigned int n = 0;
-+	unsigned int bit;
-+
-+	bitmap_zero(dst, nbits);
-+
-+	for_each_set_bit(bit, mask, nbits)
-+		__assign_bit(bit, dst, test_bit(n++, src));
-+}
-+
-+/**
-+ * bitmap_gather - Gather a bitmap according to given mask
-+ * @dst: gathered bitmap
-+ * @src: scattered bitmap
-+ * @mask: mask representing bits to extract from in the scattered bitmap
-+ * @nbits: number of bits in each of these bitmaps
-+ *
-+ * Gathers bitmap with sparse bits according to the given @mask.
-+ *
-+ * Example:
-+ * If @src bitmap = 0x0302, with @mask = 0x1313, @dst will be 0x001a.
-+ *
-+ * Or in binary form
-+ * @src			@mask			@dst
-+ * 0000001100000010	0001001100010011	0000000000011010
-+ *
-+ * (Bits 0, 1, 4, 8, 9, 12 are copied to the bits 0, 1, 2, 3, 4, 5)
-+ *
-+ * A more 'visual' description of the operation::
-+ *
-+ *	mask: ...v..vv...v..vv
-+ *	src:  0000001100000010
-+ *	         ^  ^^   ^   0
-+ *	         |  ||   |  10
-+ *	         |  ||   > 010
-+ *	         |  |+--> 1010
-+ *	         |  +--> 11010
-+ *	         +----> 011010
-+ *	dst:  0000000000011010
-+ *
-+ * A relationship exists between bitmap_gather() and bitmap_scatter(). See
-+ * bitmap_scatter() for the bitmap scatter detailed operations. TL;DR:
-+ * bitmap_scatter() can be seen as the 'reverse' bitmap_gather() operation.
-+ *
-+ * Suppose scattered computed using bitmap_scatter(scattered, src, mask, n).
-+ * The operation bitmap_gather(result, scattered, mask, n) leads to a result
-+ * equal or equivalent to src.
-+ *
-+ * The result can be 'equivalent' because bitmap_scatter() and bitmap_gather()
-+ * are not bijective.
-+ * The result and src values are equivalent in that sense that a call to
-+ * bitmap_scatter(res, src, mask, n) and a call to
-+ * bitmap_scatter(res, result, mask, n) will lead to the same res value.
-+ */
-+static __always_inline
-+void bitmap_gather(unsigned long *dst, const unsigned long *src,
-+		   const unsigned long *mask, unsigned int nbits)
-+{
-+	unsigned int n = 0;
-+	unsigned int bit;
-+
-+	bitmap_zero(dst, nbits);
-+
-+	for_each_set_bit(bit, mask, nbits)
-+		__assign_bit(n++, dst, test_bit(bit, src));
-+}
-+
-+static __always_inline
-+void bitmap_next_set_region(unsigned long *bitmap, unsigned int *rs,
-+			    unsigned int *re, unsigned int end)
- {
- 	*rs = find_next_bit(bitmap, end, *rs);
- 	*re = find_next_zero_bit(bitmap, end, *rs + 1);
- }
- 
- /**
-+ * bitmap_release_region - release allocated bitmap region
-+ *	@bitmap: array of unsigned longs corresponding to the bitmap
-+ *	@pos: beginning of bit region to release
-+ *	@order: region size (log base 2 of number of bits) to release
-+ *
-+ * This is the complement to __bitmap_find_free_region() and releases
-+ * the found region (by clearing it in the bitmap).
-+ */
-+static __always_inline
-+void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)
-+{
-+	bitmap_clear(bitmap, pos, BIT(order));
-+}
-+
-+/**
-+ * bitmap_allocate_region - allocate bitmap region
-+ *	@bitmap: array of unsigned longs corresponding to the bitmap
-+ *	@pos: beginning of bit region to allocate
-+ *	@order: region size (log base 2 of number of bits) to allocate
-+ *
-+ * Allocate (set bits in) a specified region of a bitmap.
-+ *
-+ * Returns: 0 on success, or %-EBUSY if specified region wasn't
-+ * free (not all bits were zero).
-+ */
-+static __always_inline
-+int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)
-+{
-+	unsigned int len = BIT(order);
-+
-+	if (find_next_bit(bitmap, pos + len, pos) < pos + len)
-+		return -EBUSY;
-+	bitmap_set(bitmap, pos, len);
-+	return 0;
-+}
-+
-+/**
-+ * bitmap_find_free_region - find a contiguous aligned mem region
-+ *	@bitmap: array of unsigned longs corresponding to the bitmap
-+ *	@bits: number of bits in the bitmap
-+ *	@order: region size (log base 2 of number of bits) to find
-+ *
-+ * Find a region of free (zero) bits in a @bitmap of @bits bits and
-+ * allocate them (set them to one).  Only consider regions of length
-+ * a power (@order) of two, aligned to that power of two, which
-+ * makes the search algorithm much faster.
-+ *
-+ * Returns: the bit offset in bitmap of the allocated region,
-+ * or -errno on failure.
-+ */
-+static __always_inline
-+int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)
-+{
-+	unsigned int pos, end;		/* scans bitmap by regions of size order */
-+
-+	for (pos = 0; (end = pos + BIT(order)) <= bits; pos = end) {
-+		if (!bitmap_allocate_region(bitmap, pos, order))
-+			return pos;
-+	}
-+	return -ENOMEM;
-+}
-+
-+/**
-  * BITMAP_FROM_U64() - Represent u64 value in the format suitable for bitmap.
-  * @n: u64 value
-  *
-@@ -552,44 +763,89 @@ static inline void bitmap_next_set_region(unsigned long *bitmap,
-  * That is ``(u32 *)(&val)[0]`` gets the upper 32 bits,
-  * but we expect the lower 32-bits of u64.
-  */
-static inline void bitmap_from_u64(unsigned long *dst, u64 mask)
-+static __always_inline void bitmap_from_u64(unsigned long *dst, u64 mask)
- {
- 	bitmap_from_arr64(dst, &mask, 64);
- }
- 
- /**
- * bitmap_get_value8 - get an 8-bit value within a memory region
-+ * bitmap_read - read a value of n-bits from the memory region
-  * @map: address to the bitmap memory region
- * @start: bit offset of the 8-bit value; must be a multiple of 8
-+ * @start: bit offset of the n-bit value
-+ * @nbits: size of value in bits, nonzero, up to BITS_PER_LONG
-  *
- * Returns the 8-bit value located at the @start bit offset within the @src
- * memory region.
-+ * Returns: value of @nbits bits located at the @start bit offset within the
-+ * @map memory region. For @nbits = 0 and @nbits > BITS_PER_LONG the return
-+ * value is undefined.
-  */
-static inline unsigned long bitmap_get_value8(const unsigned long *map,
-					      unsigned long start)
-+static __always_inline
-+unsigned long bitmap_read(const unsigned long *map, unsigned long start, unsigned long nbits)
- {
-	const size_t index = BIT_WORD(start);
-	const unsigned long offset = start % BITS_PER_LONG;
-+	size_t index = BIT_WORD(start);
-+	unsigned long offset = start % BITS_PER_LONG;
-+	unsigned long space = BITS_PER_LONG - offset;
-+	unsigned long value_low, value_high;
-+
-+	if (unlikely(!nbits || nbits > BITS_PER_LONG))
-+		return 0;
-+
-+	if (space >= nbits)
-+		return (map[index] >> offset) & BITMAP_LAST_WORD_MASK(nbits);
- 
-	return (map[index] >> offset) & 0xFF;
-+	value_low = map[index] & BITMAP_FIRST_WORD_MASK(start);
-+	value_high = map[index + 1] & BITMAP_LAST_WORD_MASK(start + nbits);
-+	return (value_low >> offset) | (value_high << space);
- }
- 
- /**
- * bitmap_set_value8 - set an 8-bit value within a memory region
-+ * bitmap_write - write n-bit value within a memory region
-  * @map: address to the bitmap memory region
- * @value: the 8-bit value; values wider than 8 bits may clobber bitmap
- * @start: bit offset of the 8-bit value; must be a multiple of 8
-+ * @value: value to write, clamped to nbits
-+ * @start: bit offset of the n-bit value
-+ * @nbits: size of value in bits, nonzero, up to BITS_PER_LONG.
-+ *
-+ * bitmap_write() behaves as-if implemented as @nbits calls of __assign_bit(),
-+ * i.e. bits beyond @nbits are ignored:
-+ *
-+ *   for (bit = 0; bit < nbits; bit++)
-+ *           __assign_bit(start + bit, bitmap, val & BIT(bit));
-+ *
-+ * For @nbits == 0 and @nbits > BITS_PER_LONG no writes are performed.
-  */
-static inline void bitmap_set_value8(unsigned long *map, unsigned long value,
-				     unsigned long start)
-+static __always_inline
-+void bitmap_write(unsigned long *map, unsigned long value,
-+		  unsigned long start, unsigned long nbits)
- {
-	const size_t index = BIT_WORD(start);
-	const unsigned long offset = start % BITS_PER_LONG;
-
-	map[index] &= ~(0xFFUL << offset);
-+	size_t index;
-+	unsigned long offset;
-+	unsigned long space;
-+	unsigned long mask;
-+	bool fit;
-+
-+	if (unlikely(!nbits || nbits > BITS_PER_LONG))
-+		return;
-+
-+	mask = BITMAP_LAST_WORD_MASK(nbits);
-+	value &= mask;
-+	offset = start % BITS_PER_LONG;
-+	space = BITS_PER_LONG - offset;
-+	fit = space >= nbits;
-+	index = BIT_WORD(start);
-+
-+	map[index] &= (fit ? (~(mask << offset)) : ~BITMAP_FIRST_WORD_MASK(start));
- 	map[index] |= value << offset;
-+	if (fit)
-+		return;
-+
-+	map[index + 1] &= BITMAP_FIRST_WORD_MASK(start + nbits);
-+	map[index + 1] |= (value >> space);
- }
- 
-+#define bitmap_get_value8(map, start)			\
-+	bitmap_read(map, start, BITS_PER_BYTE)
-+#define bitmap_set_value8(map, value, start)		\
-+	bitmap_write(map, value, start, BITS_PER_BYTE)
-+
- #endif /* __ASSEMBLY__ */
- 
- #endif /* __LINUX_BITMAP_H */
-diff --git a/include/linux/bitops.h b/include/linux/bitops.h
-index cf9bf65039f2..ea7898cc5903 100644
--- a/include/linux/bitops.h
-+++ b/include/linux/bitops.h
-@@ -8,19 +8,13 @@
- 
- #include <uapi/linux/kernel.h>
- 
-/* Set bits in the first 'n' bytes when loaded from memory */
-#ifdef __LITTLE_ENDIAN
-#  define aligned_byte_mask(n) ((1UL << 8*(n))-1)
-#else
-#  define aligned_byte_mask(n) (~0xffUL << (BITS_PER_LONG - 8 - 8*(n)))
-#endif
-
-#define BITS_PER_TYPE(type)	(sizeof(type) * BITS_PER_BYTE)
- #define BITS_TO_LONGS(nr)	__KERNEL_DIV_ROUND_UP(nr, BITS_PER_TYPE(long))
- #define BITS_TO_U64(nr)		__KERNEL_DIV_ROUND_UP(nr, BITS_PER_TYPE(u64))
- #define BITS_TO_U32(nr)		__KERNEL_DIV_ROUND_UP(nr, BITS_PER_TYPE(u32))
- #define BITS_TO_BYTES(nr)	__KERNEL_DIV_ROUND_UP(nr, BITS_PER_TYPE(char))
- 
-+#define BYTES_TO_BITS(nb)	((nb) * BITS_PER_BYTE)
-+
- extern unsigned int __sw_hweight8(unsigned int w);
- extern unsigned int __sw_hweight16(unsigned int w);
- extern unsigned int __sw_hweight32(unsigned int w);
-@@ -52,13 +46,19 @@ extern unsigned long __sw_hweight64(__u64 w);
- 	  __builtin_constant_p(*(const unsigned long *)(addr))) ?	\
- 	 const##op(nr, addr) : op(nr, addr))
- 
-+/*
-+ * The following macros are non-atomic versions of their non-underscored
-+ * counterparts.
-+ */
- #define __set_bit(nr, addr)		bitop(___set_bit, nr, addr)
- #define __clear_bit(nr, addr)		bitop(___clear_bit, nr, addr)
- #define __change_bit(nr, addr)		bitop(___change_bit, nr, addr)
- #define __test_and_set_bit(nr, addr)	bitop(___test_and_set_bit, nr, addr)
- #define __test_and_clear_bit(nr, addr)	bitop(___test_and_clear_bit, nr, addr)
- #define __test_and_change_bit(nr, addr)	bitop(___test_and_change_bit, nr, addr)
-+
- #define test_bit(nr, addr)		bitop(_test_bit, nr, addr)
-+#define test_bit_acquire(nr, addr)	bitop(_test_bit_acquire, nr, addr)
- 
- /*
-  * Include this here because some architectures need generic_ffs/fls in
-@@ -79,6 +79,7 @@ __check_bitop_pr(__test_and_set_bit);
- __check_bitop_pr(__test_and_clear_bit);
- __check_bitop_pr(__test_and_change_bit);
- __check_bitop_pr(test_bit);
-+__check_bitop_pr(test_bit_acquire);
- 
- #undef __check_bitop_pr
- 
-@@ -199,7 +200,7 @@ static __always_inline __s64 sign_extend64(__u64 value, int index)
- 	return (__s64)(value << shift) >> shift;
- }
- 
-static inline unsigned fls_long(unsigned long l)
-+static inline unsigned int fls_long(unsigned long l)
- {
- 	if (sizeof(l) == 4)
- 		return fls(l);
-@@ -228,6 +229,37 @@ static inline int get_count_order_long(unsigned long l)
- }
- 
- /**
-+ * parity8 - get the parity of an u8 value
-+ * @value: the value to be examined
-+ *
-+ * Determine the parity of the u8 argument.
-+ *
-+ * Returns:
-+ * 0 for even parity, 1 for odd parity
-+ *
-+ * Note: This function informs you about the current parity. Example to bail
-+ * out when parity is odd:
-+ *
-+ *	if (parity8(val) == 1)
-+ *		return -EBADMSG;
-+ *
-+ * If you need to calculate a parity bit, you need to draw the conclusion from
-+ * this result yourself. Example to enforce odd parity, parity bit is bit 7:
-+ *
-+ *	if (parity8(val) == 0)
-+ *		val ^= BIT(7);
-+ */
-+static inline int parity8(u8 val)
-+{
-+	/*
-+	 * One explanation of this algorithm:
-+	 * https://funloop.org/codex/problem/parity/README.html
-+	 */
-+	val ^= val >> 4;
-+	return (0x6996 >> (val & 0xf)) & 1;
-+}
-+
-+/**
-  * __ffs64 - find first set bit in a 64 bit word
-  * @word: The 64 bit word
-  *
-@@ -235,7 +267,7 @@ static inline int get_count_order_long(unsigned long l)
-  * The result is not defined if no bits are set, so check that @word
-  * is non-zero before calling this.
-  */
-static inline unsigned long __ffs64(u64 word)
-+static inline __attribute_const__ unsigned int __ffs64(u64 word)
- {
- #if BITS_PER_LONG == 32
- 	if (((u32)word) == 0UL)
-@@ -247,28 +279,29 @@ static inline unsigned long __ffs64(u64 word)
- }
- 
- /**
-+ * fns - find N'th set bit in a word
-+ * @word: The word to search
-+ * @n: Bit to find
-+ */
-+static inline unsigned int fns(unsigned long word, unsigned int n)
-+{
-+	while (word && n--)
-+		word &= word - 1;
-+
-+	return word ? __ffs(word) : BITS_PER_LONG;
-+}
-+
-+/**
-  * assign_bit - Assign value to a bit in memory
-  * @nr: the bit to set
-  * @addr: the address to start counting from
-  * @value: the value to assign
-  */
-static __always_inline void assign_bit(long nr, volatile unsigned long *addr,
-				       bool value)
-{
-	if (value)
-		set_bit(nr, addr);
-	else
-		clear_bit(nr, addr);
-}
-+#define assign_bit(nr, addr, value)					\
-+	((value) ? set_bit((nr), (addr)) : clear_bit((nr), (addr)))
- 
-static __always_inline void __assign_bit(long nr, volatile unsigned long *addr,
-					 bool value)
-{
-	if (value)
-		__set_bit(nr, addr);
-	else
-		__clear_bit(nr, addr);
-}
-+#define __assign_bit(nr, addr, value)					\
-+	((value) ? __set_bit((nr), (addr)) : __clear_bit((nr), (addr)))
- 
- /**
-  * __ptr_set_bit - Set bit in a pointer's value
-@@ -327,10 +360,10 @@ static __always_inline void __assign_bit(long nr, volatile unsigned long *addr,
- 	const typeof(*(ptr)) mask__ = (mask), bits__ = (bits);	\
- 	typeof(*(ptr)) old__, new__;				\
- 								\
-+	old__ = READ_ONCE(*(ptr));				\
- 	do {							\
-		old__ = READ_ONCE(*(ptr));			\
- 		new__ = (old__ & ~mask__) | bits__;		\
-	} while (cmpxchg(ptr, old__, new__) != old__);		\
-+	} while (!try_cmpxchg(ptr, &old__, new__));		\
- 								\
- 	old__;							\
- })
-@@ -342,11 +375,12 @@ static __always_inline void __assign_bit(long nr, volatile unsigned long *addr,
- 	const typeof(*(ptr)) clear__ = (clear), test__ = (test);\
- 	typeof(*(ptr)) old__, new__;				\
- 								\
-+	old__ = READ_ONCE(*(ptr));				\
- 	do {							\
-		old__ = READ_ONCE(*(ptr));			\
-+		if (old__ & test__)				\
-+			break;					\
- 		new__ = old__ & ~clear__;			\
-	} while (!(old__ & test__) &&				\
-		 cmpxchg(ptr, old__, new__) != old__);		\
-+	} while (!try_cmpxchg(ptr, &old__, new__));		\
- 								\
- 	!(old__ & test__);					\
- })
-diff --git a/include/linux/bits.h b/include/linux/bits.h
-index 87d112650dfb..a40cc861b3a7 100644
--- a/include/linux/bits.h
-+++ b/include/linux/bits.h
-@@ -2,16 +2,15 @@
- #ifndef __LINUX_BITS_H
- #define __LINUX_BITS_H
- 
-#include <linux/const.h>
- #include <vdso/bits.h>
-#include <asm/bitsperlong.h>
-+#include <uapi/linux/bits.h>
- 
-#define BIT_ULL(nr)		(ULL(1) << (nr))
- #define BIT_MASK(nr)		(UL(1) << ((nr) % BITS_PER_LONG))
- #define BIT_WORD(nr)		((nr) / BITS_PER_LONG)
- #define BIT_ULL_MASK(nr)	(ULL(1) << ((nr) % BITS_PER_LONG_LONG))
- #define BIT_ULL_WORD(nr)	((nr) / BITS_PER_LONG_LONG)
- #define BITS_PER_BYTE		8
-+#define BITS_PER_TYPE(type)	(sizeof(type) * BITS_PER_BYTE)
- 
- /*
-  * Create a contiguous bitmask starting at bit position @l and ending at
-@@ -19,28 +18,72 @@
-  * GENMASK_ULL(39, 21) gives us the 64bit vector 0x000000ffffe00000.
-  */
- #if !defined(__ASSEMBLY__)
-+
-+/*
-+ * Missing asm support
-+ *
-+ * GENMASK_U*() and BIT_U*() depend on BITS_PER_TYPE() which relies on sizeof(),
-+ * something not available in asm. Nevertheless, fixed width integers is a C
-+ * concept. Assembly code can rely on the long and long long versions instead.
-+ */
-+
- #include <linux/build_bug.h>
-#define GENMASK_INPUT_CHECK(h, l) \
-	(BUILD_BUG_ON_ZERO(__builtin_choose_expr( \
-		__is_constexpr((l) > (h)), (l) > (h), 0)))
-#else
-+#include <linux/compiler.h>
-+#include <linux/overflow.h>
-+
-+#define GENMASK_INPUT_CHECK(h, l) BUILD_BUG_ON_ZERO(const_true((l) > (h)))
-+
-+/*
-+ * Generate a mask for the specified type @t. Additional checks are made to
-+ * guarantee the value returned fits in that type, relying on
-+ * -Wshift-count-overflow compiler check to detect incompatible arguments.
-+ * For example, all these create build errors or warnings:
-+ *
-+ * - GENMASK(15, 20): wrong argument order
-+ * - GENMASK(72, 15): doesn't fit unsigned long
-+ * - GENMASK_U32(33, 15): doesn't fit in a u32
-+ */
-+#define GENMASK_TYPE(t, h, l)					\
-+	((t)(GENMASK_INPUT_CHECK(h, l) +			\
-+	     (type_max(t) << (l) &				\
-+	      type_max(t) >> (BITS_PER_TYPE(t) - 1 - (h)))))
-+
-+#define GENMASK(h, l)		GENMASK_TYPE(unsigned long, h, l)
-+#define GENMASK_ULL(h, l)	GENMASK_TYPE(unsigned long long, h, l)
-+
-+#define GENMASK_U8(h, l)	GENMASK_TYPE(u8, h, l)
-+#define GENMASK_U16(h, l)	GENMASK_TYPE(u16, h, l)
-+#define GENMASK_U32(h, l)	GENMASK_TYPE(u32, h, l)
-+#define GENMASK_U64(h, l)	GENMASK_TYPE(u64, h, l)
-+#define GENMASK_U128(h, l)	GENMASK_TYPE(u128, h, l)
-+
-+/*
-+ * Fixed-type variants of BIT(), with additional checks like GENMASK_TYPE(). The
-+ * following examples generate compiler warnings due to -Wshift-count-overflow:
-+ *
-+ * - BIT_U8(8)
-+ * - BIT_U32(-1)
-+ * - BIT_U32(40)
-+ */
-+#define BIT_INPUT_CHECK(type, nr) \
-+	BUILD_BUG_ON_ZERO(const_true((nr) >= BITS_PER_TYPE(type)))
-+
-+#define BIT_TYPE(type, nr) ((type)(BIT_INPUT_CHECK(type, nr) + BIT_ULL(nr)))
-+
-+#define BIT_U8(nr)	BIT_TYPE(u8, nr)
-+#define BIT_U16(nr)	BIT_TYPE(u16, nr)
-+#define BIT_U32(nr)	BIT_TYPE(u32, nr)
-+#define BIT_U64(nr)	BIT_TYPE(u64, nr)
-+
-+#else /* defined(__ASSEMBLY__) */
-+
- /*
-  * BUILD_BUG_ON_ZERO is not available in h files included from asm files,
-  * disable the input check if that is the case.
-  */
-#define GENMASK_INPUT_CHECK(h, l) 0
-#endif
-
-#define __GENMASK(h, l) \
-	(((~UL(0)) - (UL(1) << (l)) + 1) & \
-	 (~UL(0) >> (BITS_PER_LONG - 1 - (h))))
-#define GENMASK(h, l) \
-	(GENMASK_INPUT_CHECK(h, l) + __GENMASK(h, l))
-
-#define __GENMASK_ULL(h, l) \
-	(((~ULL(0)) - (ULL(1) << (l)) + 1) & \
-	 (~ULL(0) >> (BITS_PER_LONG_LONG - 1 - (h))))
-#define GENMASK_ULL(h, l) \
-	(GENMASK_INPUT_CHECK(h, l) + __GENMASK_ULL(h, l))
-+#define GENMASK(h, l)		__GENMASK(h, l)
-+#define GENMASK_ULL(h, l)	__GENMASK_ULL(h, l)
-+
-+#endif /* !defined(__ASSEMBLY__) */
- 
- #endif	/* __LINUX_BITS_H */
-diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h
-index 9f40dbc65f82..dd5841a42c33 100644
--- a/include/linux/blk-cgroup.h
-+++ b/include/linux/blk-cgroup.h
-@@ -18,14 +18,14 @@
- 
- struct bio;
- struct cgroup_subsys_state;
-struct request_queue;
-+struct gendisk;
- 
- #define FC_APPID_LEN              129
- 
- #ifdef CONFIG_BLK_CGROUP
- extern struct cgroup_subsys_state * const blkcg_root_css;
- 
-void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay);
-+void blkcg_schedule_throttle(struct gendisk *disk, bool use_memdelay);
- void blkcg_maybe_throttle_current(void);
- bool blk_cgroup_congested(void);
- void blkcg_pin_online(struct cgroup_subsys_state *blkcg_css);
-@@ -39,7 +39,6 @@ struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio);
- 
- static inline void blkcg_maybe_throttle_current(void) { }
- static inline bool blk_cgroup_congested(void) { return false; }
-static inline void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay) { }
- static inline struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio)
- {
- 	return NULL;
-diff --git a/include/linux/blk-crypto-profile.h b/include/linux/blk-crypto-profile.h
-index bbab65bd5428..4f39e9cd7576 100644
--- a/include/linux/blk-crypto-profile.h
-+++ b/include/linux/blk-crypto-profile.h
-@@ -57,6 +57,62 @@ struct blk_crypto_ll_ops {
- 	int (*keyslot_evict)(struct blk_crypto_profile *profile,
- 			     const struct blk_crypto_key *key,
- 			     unsigned int slot);
-+
-+	/**
-+	 * @derive_sw_secret: Derive the software secret from a hardware-wrapped
-+	 *		      key in ephemerally-wrapped form.
-+	 *
-+	 * This only needs to be implemented if BLK_CRYPTO_KEY_TYPE_HW_WRAPPED
-+	 * is supported.
-+	 *
-+	 * Must return 0 on success, -EBADMSG if the key is invalid, or another
-+	 * -errno code on other errors.
-+	 */
-+	int (*derive_sw_secret)(struct blk_crypto_profile *profile,
-+				const u8 *eph_key, size_t eph_key_size,
-+				u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]);
-+
-+	/**
-+	 * @import_key: Create a hardware-wrapped key by importing a raw key.
-+	 *
-+	 * This only needs to be implemented if BLK_CRYPTO_KEY_TYPE_HW_WRAPPED
-+	 * is supported.
-+	 *
-+	 * On success, must write the new key in long-term wrapped form to
-+	 * @lt_key and return its size in bytes.  On failure, must return a
-+	 * -errno value.
-+	 */
-+	int (*import_key)(struct blk_crypto_profile *profile,
-+			  const u8 *raw_key, size_t raw_key_size,
-+			  u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]);
-+
-+	/**
-+	 * @generate_key: Generate a hardware-wrapped key.
-+	 *
-+	 * This only needs to be implemented if BLK_CRYPTO_KEY_TYPE_HW_WRAPPED
-+	 * is supported.
-+	 *
-+	 * On success, must write the new key in long-term wrapped form to
-+	 * @lt_key and return its size in bytes.  On failure, must return a
-+	 * -errno value.
-+	 */
-+	int (*generate_key)(struct blk_crypto_profile *profile,
-+			    u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]);
-+
-+	/**
-+	 * @prepare_key: Prepare a hardware-wrapped key to be used.
-+	 *
-+	 * Prepare a hardware-wrapped key to be used by converting it from
-+	 * long-term wrapped form to ephemerally-wrapped form.  This only needs
-+	 * to be implemented if BLK_CRYPTO_KEY_TYPE_HW_WRAPPED is supported.
-+	 *
-+	 * On success, must write the key in ephemerally-wrapped form to
-+	 * @eph_key and return its size in bytes.  On failure, must return
-+	 * -EBADMSG if the key is invalid, or another -errno on other error.
-+	 */
-+	int (*prepare_key)(struct blk_crypto_profile *profile,
-+			   const u8 *lt_key, size_t lt_key_size,
-+			   u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]);
- };
- 
- /**
-@@ -85,6 +141,12 @@ struct blk_crypto_profile {
- 	unsigned int max_dun_bytes_supported;
- 
- 	/**
-+	 * @key_types_supported: A bitmask of the supported key types:
-+	 * BLK_CRYPTO_KEY_TYPE_RAW and/or BLK_CRYPTO_KEY_TYPE_HW_WRAPPED.
-+	 */
-+	unsigned int key_types_supported;
-+
-+	/**
- 	 * @modes_supported: Array of bitmasks that specifies whether each
- 	 * combination of crypto mode and data unit size is supported.
- 	 * Specifically, the i'th bit of modes_supported[crypto_mode] is set if
-@@ -111,6 +173,7 @@ struct blk_crypto_profile {
- 	 * keyslots while ensuring that they can't be changed concurrently.
- 	 */
- 	struct rw_semaphore lock;
-+	struct lock_class_key lockdep_key;
- 
- 	/* List of idle slots, with least recently used slot at front */
- 	wait_queue_head_t idle_slots_wait_queue;
-@@ -138,21 +201,20 @@ int devm_blk_crypto_profile_init(struct device *dev,
- 
- unsigned int blk_crypto_keyslot_index(struct blk_crypto_keyslot *slot);
- 
-blk_status_t blk_crypto_get_keyslot(struct blk_crypto_profile *profile,
-				    const struct blk_crypto_key *key,
-				    struct blk_crypto_keyslot **slot_ptr);
-
-void blk_crypto_put_keyslot(struct blk_crypto_keyslot *slot);
-+void blk_crypto_reprogram_all_keys(struct blk_crypto_profile *profile);
- 
-bool __blk_crypto_cfg_supported(struct blk_crypto_profile *profile,
-				const struct blk_crypto_config *cfg);
-+void blk_crypto_profile_destroy(struct blk_crypto_profile *profile);
- 
-int __blk_crypto_evict_key(struct blk_crypto_profile *profile,
-			   const struct blk_crypto_key *key);
-+int blk_crypto_import_key(struct blk_crypto_profile *profile,
-+			  const u8 *raw_key, size_t raw_key_size,
-+			  u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]);
- 
-void blk_crypto_reprogram_all_keys(struct blk_crypto_profile *profile);
-+int blk_crypto_generate_key(struct blk_crypto_profile *profile,
-+			    u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]);
- 
-void blk_crypto_profile_destroy(struct blk_crypto_profile *profile);
-+int blk_crypto_prepare_key(struct blk_crypto_profile *profile,
-+			   const u8 *lt_key, size_t lt_key_size,
-+			   u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]);
- 
- void blk_crypto_intersect_capabilities(struct blk_crypto_profile *parent,
- 				       const struct blk_crypto_profile *child);
-diff --git a/include/linux/blk-crypto.h b/include/linux/blk-crypto.h
-index 69b24fe92cbf..58b0c5254a67 100644
--- a/include/linux/blk-crypto.h
-+++ b/include/linux/blk-crypto.h
-@@ -6,17 +6,68 @@
- #ifndef __LINUX_BLK_CRYPTO_H
- #define __LINUX_BLK_CRYPTO_H
- 
-+#include <linux/minmax.h>
- #include <linux/types.h>
-+#include <uapi/linux/blk-crypto.h>
- 
- enum blk_crypto_mode_num {
- 	BLK_ENCRYPTION_MODE_INVALID,
- 	BLK_ENCRYPTION_MODE_AES_256_XTS,
- 	BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV,
- 	BLK_ENCRYPTION_MODE_ADIANTUM,
-+	BLK_ENCRYPTION_MODE_SM4_XTS,
- 	BLK_ENCRYPTION_MODE_MAX,
- };
- 
-#define BLK_CRYPTO_MAX_KEY_SIZE		64
-+/*
-+ * Supported types of keys.  Must be bitflags due to their use in
-+ * blk_crypto_profile::key_types_supported.
-+ */
-+enum blk_crypto_key_type {
-+	/*
-+	 * Raw keys (i.e. "software keys").  These keys are simply kept in raw,
-+	 * plaintext form in kernel memory.
-+	 */
-+	BLK_CRYPTO_KEY_TYPE_RAW = 0x1,
-+
-+	/*
-+	 * Hardware-wrapped keys.  These keys are only present in kernel memory
-+	 * in ephemerally-wrapped form, and they can only be unwrapped by
-+	 * dedicated hardware.  For details, see the "Hardware-wrapped keys"
-+	 * section of Documentation/block/inline-encryption.rst.
-+	 */
-+	BLK_CRYPTO_KEY_TYPE_HW_WRAPPED = 0x2,
-+};
-+
-+/*
-+ * Currently the maximum raw key size is 64 bytes, as that is the key size of
-+ * BLK_ENCRYPTION_MODE_AES_256_XTS which takes the longest key.
-+ *
-+ * The maximum hardware-wrapped key size depends on the hardware's key wrapping
-+ * algorithm, which is a hardware implementation detail, so it isn't precisely
-+ * specified.  But currently 128 bytes is plenty in practice.  Implementations
-+ * are recommended to wrap a 32-byte key for the hardware KDF with AES-256-GCM,
-+ * which should result in a size closer to 64 bytes than 128.
-+ *
-+ * Both of these values can trivially be increased if ever needed.
-+ */
-+#define BLK_CRYPTO_MAX_RAW_KEY_SIZE		64
-+#define BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE	128
-+
-+#define BLK_CRYPTO_MAX_ANY_KEY_SIZE \
-+	MAX(BLK_CRYPTO_MAX_RAW_KEY_SIZE, BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE)
-+
-+/*
-+ * Size of the "software secret" which can be derived from a hardware-wrapped
-+ * key.  This is currently always 32 bytes.  Note, the choice of 32 bytes
-+ * assumes that the software secret is only used directly for algorithms that
-+ * don't require more than a 256-bit key to get the desired security strength.
-+ * If it were to be used e.g. directly as an AES-256-XTS key, then this would
-+ * need to be increased (which is possible if hardware supports it, but care
-+ * would need to be taken to avoid breaking users who need exactly 32 bytes).
-+ */
-+#define BLK_CRYPTO_SW_SECRET_SIZE	32
-+
- /**
-  * struct blk_crypto_config - an inline encryption key's crypto configuration
-  * @crypto_mode: encryption algorithm this key is for
-@@ -25,20 +76,23 @@ enum blk_crypto_mode_num {
-  *	ciphertext.  This is always a power of 2.  It might be e.g. the
-  *	filesystem block size or the disk sector size.
-  * @dun_bytes: the maximum number of bytes of DUN used when using this key
-+ * @key_type: the type of this key -- either raw or hardware-wrapped
-  */
- struct blk_crypto_config {
- 	enum blk_crypto_mode_num crypto_mode;
- 	unsigned int data_unit_size;
- 	unsigned int dun_bytes;
-+	enum blk_crypto_key_type key_type;
- };
- 
- /**
-  * struct blk_crypto_key - an inline encryption key
- * @crypto_cfg: the crypto configuration (like crypto_mode, key size) for this
- *		key
-+ * @crypto_cfg: the crypto mode, data unit size, key type, and other
-+ *		characteristics of this key and how it will be used
-  * @data_unit_size_bits: log2 of data_unit_size
- * @size: size of this key in bytes (determined by @crypto_cfg.crypto_mode)
- * @raw: the raw bytes of this key.  Only the first @size bytes are used.
-+ * @size: size of this key in bytes.  The size of a raw key is fixed for a given
-+ *	  crypto mode, but the size of a hardware-wrapped key can vary.
-+ * @bytes: the bytes of this key.  Only the first @size bytes are significant.
-  *
-  * A blk_crypto_key is immutable once created, and many bios can reference it at
-  * the same time.  It must not be freed until all bios using it have completed
-@@ -48,7 +102,7 @@ struct blk_crypto_key {
- 	struct blk_crypto_config crypto_cfg;
- 	unsigned int data_unit_size_bits;
- 	unsigned int size;
-	u8 raw[BLK_CRYPTO_MAX_KEY_SIZE];
-+	u8 bytes[BLK_CRYPTO_MAX_ANY_KEY_SIZE];
- };
- 
- #define BLK_CRYPTO_MAX_IV_SIZE		32
-@@ -71,9 +125,6 @@ struct bio_crypt_ctx {
- #include <linux/blk_types.h>
- #include <linux/blkdev.h>
- 
-struct request;
-struct request_queue;
-
- #ifdef CONFIG_BLK_INLINE_ENCRYPTION
- 
- static inline bool bio_has_crypt_ctx(struct bio *bio)
-@@ -89,20 +140,28 @@ bool bio_crypt_dun_is_contiguous(const struct bio_crypt_ctx *bc,
- 				 unsigned int bytes,
- 				 const u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]);
- 
-int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *raw_key,
-+int blk_crypto_init_key(struct blk_crypto_key *blk_key,
-+			const u8 *key_bytes, size_t key_size,
-+			enum blk_crypto_key_type key_type,
- 			enum blk_crypto_mode_num crypto_mode,
- 			unsigned int dun_bytes,
- 			unsigned int data_unit_size);
- 
-int blk_crypto_start_using_key(const struct blk_crypto_key *key,
-			       struct request_queue *q);
-+int blk_crypto_start_using_key(struct block_device *bdev,
-+			       const struct blk_crypto_key *key);
- 
-int blk_crypto_evict_key(struct request_queue *q,
-			 const struct blk_crypto_key *key);
-+void blk_crypto_evict_key(struct block_device *bdev,
-+			  const struct blk_crypto_key *key);
- 
-bool blk_crypto_config_supported(struct request_queue *q,
-+bool blk_crypto_config_supported_natively(struct block_device *bdev,
-+					  const struct blk_crypto_config *cfg);
-+bool blk_crypto_config_supported(struct block_device *bdev,
- 				 const struct blk_crypto_config *cfg);
- 
-+int blk_crypto_derive_sw_secret(struct block_device *bdev,
-+				const u8 *eph_key, size_t eph_key_size,
-+				u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]);
-+
- #else /* CONFIG_BLK_INLINE_ENCRYPTION */
- 
- static inline bool bio_has_crypt_ctx(struct bio *bio)
-diff --git a/include/linux/blk-integrity.h b/include/linux/blk-integrity.h
-index 378b2459efe2..a6b84206eb94 100644
--- a/include/linux/blk-integrity.h
-+++ b/include/linux/blk-integrity.h
-@@ -3,54 +3,58 @@
- #define _LINUX_BLK_INTEGRITY_H
- 
- #include <linux/blk-mq.h>
-+#include <linux/bio-integrity.h>
-+#include <linux/blk-mq-dma.h>
- 
- struct request;
- 
-+/*
-+ * Maximum contiguous integrity buffer allocation.
-+ */
-+#define BLK_INTEGRITY_MAX_SIZE		SZ_2M
-+
- enum blk_integrity_flags {
-	BLK_INTEGRITY_VERIFY		= 1 << 0,
-	BLK_INTEGRITY_GENERATE		= 1 << 1,
-+	BLK_INTEGRITY_NOVERIFY		= 1 << 0,
-+	BLK_INTEGRITY_NOGENERATE	= 1 << 1,
- 	BLK_INTEGRITY_DEVICE_CAPABLE	= 1 << 2,
-	BLK_INTEGRITY_IP_CHECKSUM	= 1 << 3,
-+	BLK_INTEGRITY_REF_TAG		= 1 << 3,
-+	BLK_INTEGRITY_STACKED		= 1 << 4,
- };
- 
-struct blk_integrity_iter {
-	void			*prot_buf;
-	void			*data_buf;
-	sector_t		seed;
-	unsigned int		data_size;
-	unsigned short		interval;
-	unsigned char		tuple_size;
-	const char		*disk_name;
-};
-
-typedef blk_status_t (integrity_processing_fn) (struct blk_integrity_iter *);
-typedef void (integrity_prepare_fn) (struct request *);
-typedef void (integrity_complete_fn) (struct request *, unsigned int);
-
-struct blk_integrity_profile {
-	integrity_processing_fn		*generate_fn;
-	integrity_processing_fn		*verify_fn;
-	integrity_prepare_fn		*prepare_fn;
-	integrity_complete_fn		*complete_fn;
-	const char			*name;
-};
-+const char *blk_integrity_profile_name(struct blk_integrity *bi);
-+bool queue_limits_stack_integrity(struct queue_limits *t,
-+		struct queue_limits *b);
-+static inline bool queue_limits_stack_integrity_bdev(struct queue_limits *t,
-+		struct block_device *bdev)
-+{
-+	return queue_limits_stack_integrity(t, &bdev->bd_disk->queue->limits);
-+}
- 
- #ifdef CONFIG_BLK_DEV_INTEGRITY
-void blk_integrity_register(struct gendisk *, struct blk_integrity *);
-void blk_integrity_unregister(struct gendisk *);
-int blk_integrity_compare(struct gendisk *, struct gendisk *);
-int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
-				   struct scatterlist *);
-+int blk_rq_map_integrity_sg(struct request *, struct scatterlist *);
-+
- int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
-+int blk_rq_integrity_map_user(struct request *rq, void __user *ubuf,
-+			      ssize_t bytes);
-+int blk_get_meta_cap(struct block_device *bdev, unsigned int cmd,
-+		     struct logical_block_metadata_cap __user *argp);
-+bool blk_rq_integrity_dma_map_iter_start(struct request *req,
-+		struct device *dma_dev,  struct dma_iova_state *state,
-+		struct blk_dma_iter *iter);
-+bool blk_rq_integrity_dma_map_iter_next(struct request *req,
-+		struct device *dma_dev, struct blk_dma_iter *iter);
- 
-static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
-+static inline bool
-+blk_integrity_queue_supports_integrity(struct request_queue *q)
- {
-	struct blk_integrity *bi = &disk->queue->integrity;
-+	return q->limits.integrity.metadata_size;
-+}
- 
-	if (!bi->profile)
-+static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
-+{
-+	if (!blk_integrity_queue_supports_integrity(disk->queue))
- 		return NULL;
-
-	return bi;
-+	return &disk->queue->limits.integrity;
- }
- 
- static inline struct blk_integrity *
-@@ -59,18 +63,6 @@ bdev_get_integrity(struct block_device *bdev)
- 	return blk_get_integrity(bdev->bd_disk);
- }
- 
-static inline bool
-blk_integrity_queue_supports_integrity(struct request_queue *q)
-{
-	return q->integrity.profile;
-}
-
-static inline void blk_queue_max_integrity_segments(struct request_queue *q,
-						    unsigned int segs)
-{
-	q->limits.max_integrity_segments = segs;
-}
-
- static inline unsigned short
- queue_max_integrity_segments(const struct request_queue *q)
- {
-@@ -96,7 +88,7 @@ static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
- static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi,
- 					       unsigned int sectors)
- {
-	return bio_integrity_intervals(bi, sectors) * bi->tuple_size;
-+	return bio_integrity_intervals(bi, sectors) * bi->metadata_size;
- }
- 
- static inline bool blk_integrity_rq(struct request *rq)
-@@ -105,54 +97,59 @@ static inline bool blk_integrity_rq(struct request *rq)
- }
- 
- /*
- * Return the first bvec that contains integrity data.  Only drivers that are
- * limited to a single integrity segment should use this helper.
-+ * Return the current bvec that contains the integrity data. bip_iter may be
-+ * advanced to iterate over the integrity data.
-  */
-static inline struct bio_vec *rq_integrity_vec(struct request *rq)
-+static inline struct bio_vec rq_integrity_vec(struct request *rq)
- {
-	if (WARN_ON_ONCE(queue_max_integrity_segments(rq->q) > 1))
-		return NULL;
-	return rq->bio->bi_integrity->bip_vec;
-+	return mp_bvec_iter_bvec(rq->bio->bi_integrity->bip_vec,
-+				 rq->bio->bi_integrity->bip_iter);
- }
- #else /* CONFIG_BLK_DEV_INTEGRITY */
-+static inline int blk_get_meta_cap(struct block_device *bdev, unsigned int cmd,
-+				   struct logical_block_metadata_cap __user *argp)
-+{
-+	return -ENOIOCTLCMD;
-+}
- static inline int blk_rq_count_integrity_sg(struct request_queue *q,
- 					    struct bio *b)
- {
- 	return 0;
- }
-static inline int blk_rq_map_integrity_sg(struct request_queue *q,
-					  struct bio *b,
-+static inline int blk_rq_map_integrity_sg(struct request *q,
- 					  struct scatterlist *s)
- {
- 	return 0;
- }
-static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
-{
-	return NULL;
-}
-static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
-+static inline int blk_rq_integrity_map_user(struct request *rq,
-+					    void __user *ubuf,
-+					    ssize_t bytes)
- {
-	return NULL;
-+	return -EINVAL;
- }
-static inline bool
-blk_integrity_queue_supports_integrity(struct request_queue *q)
-+static inline bool blk_rq_integrity_dma_map_iter_start(struct request *req,
-+		struct device *dma_dev,  struct dma_iova_state *state,
-+		struct blk_dma_iter *iter)
- {
- 	return false;
- }
-static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
-+static inline bool blk_rq_integrity_dma_map_iter_next(struct request *req,
-+		struct device *dma_dev, struct blk_dma_iter *iter)
- {
-	return 0;
-+	return false;
- }
-static inline void blk_integrity_register(struct gendisk *d,
-					 struct blk_integrity *b)
-+static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
- {
-+	return NULL;
- }
-static inline void blk_integrity_unregister(struct gendisk *d)
-+static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
- {
-+	return NULL;
- }
-static inline void blk_queue_max_integrity_segments(struct request_queue *q,
-						    unsigned int segs)
-+static inline bool
-+blk_integrity_queue_supports_integrity(struct request_queue *q)
- {
-+	return false;
- }
- static inline unsigned short
- queue_max_integrity_segments(const struct request_queue *q)
-@@ -176,9 +173,11 @@ static inline int blk_integrity_rq(struct request *rq)
- 	return 0;
- }
- 
-static inline struct bio_vec *rq_integrity_vec(struct request *rq)
-+static inline struct bio_vec rq_integrity_vec(struct request *rq)
- {
-	return NULL;
-+	/* the optimizer will remove all calls to this function */
-+	return (struct bio_vec){ };
- }
- #endif /* CONFIG_BLK_DEV_INTEGRITY */
-+
- #endif /* _LINUX_BLK_INTEGRITY_H */
-diff --git a/include/linux/blk-mq-dma.h b/include/linux/blk-mq-dma.h
-new file mode 100644
-index 000000000000..cb88fc791fbd
--- /dev/null
-+++ b/include/linux/blk-mq-dma.h
-@@ -0,0 +1,76 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+#ifndef BLK_MQ_DMA_H
-+#define BLK_MQ_DMA_H
-+
-+#include <linux/blk-mq.h>
-+#include <linux/pci-p2pdma.h>
-+
-+struct blk_map_iter {
-+	struct bvec_iter		iter;
-+	struct bio			*bio;
-+	struct bio_vec			*bvecs;
-+	bool				is_integrity;
-+};
-+
-+struct blk_dma_iter {
-+	/* Output address range for this iteration */
-+	dma_addr_t			addr;
-+	u32				len;
-+	struct pci_p2pdma_map_state	p2pdma;
-+
-+	/* Status code. Only valid when blk_rq_dma_map_iter_* returned false */
-+	blk_status_t			status;
-+
-+	/* Internal to blk_rq_dma_map_iter_* */
-+	struct blk_map_iter		iter;
-+};
-+
-+bool blk_rq_dma_map_iter_start(struct request *req, struct device *dma_dev,
-+		struct dma_iova_state *state, struct blk_dma_iter *iter);
-+bool blk_rq_dma_map_iter_next(struct request *req, struct device *dma_dev,
-+		struct dma_iova_state *state, struct blk_dma_iter *iter);
-+
-+/**
-+ * blk_rq_dma_map_coalesce - were all segments coalesced?
-+ * @state: DMA state to check
-+ *
-+ * Returns true if blk_rq_dma_map_iter_start coalesced all segments into a
-+ * single DMA range.
-+ */
-+static inline bool blk_rq_dma_map_coalesce(struct dma_iova_state *state)
-+{
-+	return dma_use_iova(state);
-+}
-+
-+/**
-+ * blk_rq_dma_unmap - try to DMA unmap a request
-+ * @req:	request to unmap
-+ * @dma_dev:	device to unmap from
-+ * @state:	DMA IOVA state
-+ * @mapped_len: number of bytes to unmap
-+ * @map:	peer-to-peer mapping type
-+ *
-+ * Returns %false if the callers need to manually unmap every DMA segment
-+ * mapped using @iter or %true if no work is left to be done.
-+ */
-+static inline bool blk_rq_dma_unmap(struct request *req, struct device *dma_dev,
-+		struct dma_iova_state *state, size_t mapped_len,
-+		enum pci_p2pdma_map_type map)
-+{
-+	if (map == PCI_P2PDMA_MAP_BUS_ADDR)
-+		return true;
-+
-+	if (dma_use_iova(state)) {
-+		unsigned int attrs = 0;
-+
-+		if (map == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE)
-+			attrs |= DMA_ATTR_MMIO;
-+
-+		dma_iova_destroy(dma_dev, state, mapped_len, rq_dma_dir(req),
-+				 attrs);
-+		return true;
-+	}
-+
-+	return !dma_need_unmap(dma_dev);
-+}
-+#endif /* BLK_MQ_DMA_H */
-diff --git a/include/linux/blk-mq-pci.h b/include/linux/blk-mq-pci.h
-deleted file mode 100644
-index 0b1f45c62623..000000000000
--- a/include/linux/blk-mq-pci.h
-+++ /dev/null
-@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _LINUX_BLK_MQ_PCI_H
-#define _LINUX_BLK_MQ_PCI_H
-
-struct blk_mq_queue_map;
-struct pci_dev;
-
-int blk_mq_pci_map_queues(struct blk_mq_queue_map *qmap, struct pci_dev *pdev,
-			  int offset);
-
-#endif /* _LINUX_BLK_MQ_PCI_H */
-diff --git a/include/linux/blk-mq-rdma.h b/include/linux/blk-mq-rdma.h
-deleted file mode 100644
-index 5cc5f0f36218..000000000000
--- a/include/linux/blk-mq-rdma.h
-+++ /dev/null
-@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _LINUX_BLK_MQ_RDMA_H
-#define _LINUX_BLK_MQ_RDMA_H
-
-struct blk_mq_tag_set;
-struct ib_device;
-
-int blk_mq_rdma_map_queues(struct blk_mq_queue_map *map,
-		struct ib_device *dev, int first_vec);
-
-#endif /* _LINUX_BLK_MQ_RDMA_H */
-diff --git a/include/linux/blk-mq-virtio.h b/include/linux/blk-mq-virtio.h
-deleted file mode 100644
-index 687ae287e1dc..000000000000
--- a/include/linux/blk-mq-virtio.h
-+++ /dev/null
-@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _LINUX_BLK_MQ_VIRTIO_H
-#define _LINUX_BLK_MQ_VIRTIO_H
-
-struct blk_mq_queue_map;
-struct virtio_device;
-
-int blk_mq_virtio_map_queues(struct blk_mq_queue_map *qmap,
-		struct virtio_device *vdev, int first_vec);
-
-#endif /* _LINUX_BLK_MQ_VIRTIO_H */
-diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
-index 92294a5fb083..cae9e857aea4 100644
--- a/include/linux/blk-mq.h
-+++ b/include/linux/blk-mq.h
-@@ -7,6 +7,9 @@
- #include <linux/lockdep.h>
- #include <linux/scatterlist.h>
- #include <linux/prefetch.h>
-+#include <linux/srcu.h>
-+#include <linux/rw_hint.h>
-+#include <linux/rwsem.h>
- 
- struct blk_mq_tags;
- struct blk_flush_queue;
-@@ -14,54 +17,76 @@ struct blk_flush_queue;
- #define BLKDEV_MIN_RQ	4
- #define BLKDEV_DEFAULT_RQ	128
- 
-typedef void (rq_end_io_fn)(struct request *, blk_status_t);
-+enum rq_end_io_ret {
-+	RQ_END_IO_NONE,
-+	RQ_END_IO_FREE,
-+};
-+
-+typedef enum rq_end_io_ret (rq_end_io_fn)(struct request *, blk_status_t);
- 
- /*
-  * request flags */
- typedef __u32 __bitwise req_flags_t;
- 
-/* drive already may have started this one */
-#define RQF_STARTED		((__force req_flags_t)(1 << 1))
-/* may not be passed by ioscheduler */
-#define RQF_SOFTBARRIER		((__force req_flags_t)(1 << 3))
-/* request for flush sequence */
-#define RQF_FLUSH_SEQ		((__force req_flags_t)(1 << 4))
-/* merge of different types, fail separately */
-#define RQF_MIXED_MERGE		((__force req_flags_t)(1 << 5))
-/* track inflight for MQ */
-#define RQF_MQ_INFLIGHT		((__force req_flags_t)(1 << 6))
-/* don't call prep for this one */
-#define RQF_DONTPREP		((__force req_flags_t)(1 << 7))
-/* vaguely specified driver internal error.  Ignored by the block layer */
-#define RQF_FAILED		((__force req_flags_t)(1 << 10))
-/* don't warn about errors */
-#define RQF_QUIET		((__force req_flags_t)(1 << 11))
-/* elevator private data attached */
-#define RQF_ELVPRIV		((__force req_flags_t)(1 << 12))
-/* account into disk and partition IO statistics */
-#define RQF_IO_STAT		((__force req_flags_t)(1 << 13))
-/* runtime pm request */
-#define RQF_PM			((__force req_flags_t)(1 << 15))
-/* on IO scheduler merge hash */
-#define RQF_HASHED		((__force req_flags_t)(1 << 16))
-/* track IO completion time */
-#define RQF_STATS		((__force req_flags_t)(1 << 17))
-/* Look at ->special_vec for the actual data payload instead of the
-   bio chain. */
-#define RQF_SPECIAL_PAYLOAD	((__force req_flags_t)(1 << 18))
-/* The per-zone write lock is held for this request */
-#define RQF_ZONE_WRITE_LOCKED	((__force req_flags_t)(1 << 19))
-/* already slept for hybrid poll */
-#define RQF_MQ_POLL_SLEPT	((__force req_flags_t)(1 << 20))
-/* ->timeout has been called, don't expire again */
-#define RQF_TIMED_OUT		((__force req_flags_t)(1 << 21))
-/* queue has elevator attached */
-#define RQF_ELV			((__force req_flags_t)(1 << 22))
-#define RQF_RESV			((__force req_flags_t)(1 << 23))
-+/* Keep rqf_name[] in sync with the definitions below */
-+enum rqf_flags {
-+	/* drive already may have started this one */
-+	__RQF_STARTED,
-+	/* request for flush sequence */
-+	__RQF_FLUSH_SEQ,
-+	/* merge of different types, fail separately */
-+	__RQF_MIXED_MERGE,
-+	/* don't call prep for this one */
-+	__RQF_DONTPREP,
-+	/* use hctx->sched_tags */
-+	__RQF_SCHED_TAGS,
-+	/* use an I/O scheduler for this request */
-+	__RQF_USE_SCHED,
-+	/* vaguely specified driver internal error.  Ignored by block layer */
-+	__RQF_FAILED,
-+	/* don't warn about errors */
-+	__RQF_QUIET,
-+	/* account into disk and partition IO statistics */
-+	__RQF_IO_STAT,
-+	/* runtime pm request */
-+	__RQF_PM,
-+	/* on IO scheduler merge hash */
-+	__RQF_HASHED,
-+	/* track IO completion time */
-+	__RQF_STATS,
-+	/* Look at ->special_vec for the actual data payload instead of the
-+	   bio chain. */
-+	__RQF_SPECIAL_PAYLOAD,
-+	/* request completion needs to be signaled to zone write plugging. */
-+	__RQF_ZONE_WRITE_PLUGGING,
-+	/* ->timeout has been called, don't expire again */
-+	__RQF_TIMED_OUT,
-+	__RQF_RESV,
-+	__RQF_BITS
-+};
-+
-+#define RQF_STARTED		((__force req_flags_t)(1 << __RQF_STARTED))
-+#define RQF_FLUSH_SEQ		((__force req_flags_t)(1 << __RQF_FLUSH_SEQ))
-+#define RQF_MIXED_MERGE		((__force req_flags_t)(1 << __RQF_MIXED_MERGE))
-+#define RQF_DONTPREP		((__force req_flags_t)(1 << __RQF_DONTPREP))
-+#define RQF_SCHED_TAGS		((__force req_flags_t)(1 << __RQF_SCHED_TAGS))
-+#define RQF_USE_SCHED		((__force req_flags_t)(1 << __RQF_USE_SCHED))
-+#define RQF_FAILED		((__force req_flags_t)(1 << __RQF_FAILED))
-+#define RQF_QUIET		((__force req_flags_t)(1 << __RQF_QUIET))
-+#define RQF_IO_STAT		((__force req_flags_t)(1 << __RQF_IO_STAT))
-+#define RQF_PM			((__force req_flags_t)(1 << __RQF_PM))
-+#define RQF_HASHED		((__force req_flags_t)(1 << __RQF_HASHED))
-+#define RQF_STATS		((__force req_flags_t)(1 << __RQF_STATS))
-+#define RQF_SPECIAL_PAYLOAD	\
-+			((__force req_flags_t)(1 << __RQF_SPECIAL_PAYLOAD))
-+#define RQF_ZONE_WRITE_PLUGGING	\
-+			((__force req_flags_t)(1 << __RQF_ZONE_WRITE_PLUGGING))
-+#define RQF_TIMED_OUT		((__force req_flags_t)(1 << __RQF_TIMED_OUT))
-+#define RQF_RESV		((__force req_flags_t)(1 << __RQF_RESV))
- 
- /* flags that prevent us from merging requests: */
- #define RQF_NOMERGE_FLAGS \
-	(RQF_STARTED | RQF_SOFTBARRIER | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)
-+	(RQF_STARTED | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)
- 
- enum mq_rq_state {
- 	MQ_RQ_IDLE		= 0,
-@@ -125,19 +150,21 @@ struct request {
- 	 * physical address coalescing is performed.
- 	 */
- 	unsigned short nr_phys_segments;
-
-#ifdef CONFIG_BLK_DEV_INTEGRITY
- 	unsigned short nr_integrity_segments;
-#endif
-+
-+	/*
-+	 * The lowest set bit for address gaps between physical segments. This
-+	 * provides information necessary for dma optimization opprotunities,
-+	 * like for testing if the segments can be coalesced against the
-+	 * device's iommu granule.
-+	 */
-+	unsigned char phys_gap_bit;
- 
- #ifdef CONFIG_BLK_INLINE_ENCRYPTION
- 	struct bio_crypt_ctx *crypt_ctx;
- 	struct blk_crypto_keyslot *crypt_keyslot;
- #endif
- 
-	unsigned short write_hint;
-	unsigned short ioprio;
-
- 	enum mq_rq_state state;
- 	atomic_t ref;
- 
-@@ -157,39 +184,30 @@ struct request {
- 
- 	/*
- 	 * The rb_node is only used inside the io scheduler, requests
-	 * are pruned when moved to the dispatch queue. So let the
-	 * completion_data share space with the rb_node.
-+	 * are pruned when moved to the dispatch queue. special_vec must
-+	 * only be used if RQF_SPECIAL_PAYLOAD is set, and those cannot be
-+	 * insert into an IO scheduler.
- 	 */
- 	union {
- 		struct rb_node rb_node;	/* sort/lookup */
- 		struct bio_vec special_vec;
-		void *completion_data;
- 	};
- 
-
- 	/*
- 	 * Three pointers are available for the IO schedulers, if they need
-	 * more they have to dynamically allocate it.  Flush requests are
-	 * never put on the IO scheduler. So let the flush fields share
-	 * space with the elevator data.
-+	 * more they have to dynamically allocate it.
- 	 */
-	union {
-		struct {
-			struct io_cq		*icq;
-			void			*priv[2];
-		} elv;
-
-		struct {
-			unsigned int		seq;
-			struct list_head	list;
-			rq_end_io_fn		*saved_end_io;
-		} flush;
-	};
-+	struct {
-+		struct io_cq		*icq;
-+		void			*priv[2];
-+	} elv;
- 
-	union {
-		struct __call_single_data csd;
-		u64 fifo_time;
-	};
-+	struct {
-+		unsigned int		seq;
-+		rq_end_io_fn		*saved_end_io;
-+	} flush;
-+
-+	u64 fifo_time;
- 
- 	/*
- 	 * completion callback.
-@@ -198,6 +216,14 @@ struct request {
- 	void *end_io_data;
- };
- 
-+/*
-+ * Returns a mask with all bits starting at req->phys_gap_bit set to 1.
-+ */
-+static inline unsigned long req_phys_gap_mask(const struct request *req)
-+{
-+	return ~(((1 << req->phys_gap_bit) >> 1) - 1);
-+}
-+
- static inline enum req_op req_op(const struct request *req)
- {
- 	return req->cmd_flags & REQ_OP_MASK;
-@@ -205,12 +231,14 @@ static inline enum req_op req_op(const struct request *req)
- 
- static inline bool blk_rq_is_passthrough(struct request *rq)
- {
-	return blk_op_is_passthrough(req_op(rq));
-+	return blk_op_is_passthrough(rq->cmd_flags);
- }
- 
- static inline unsigned short req_get_ioprio(struct request *req)
- {
-	return req->ioprio;
-+	if (req->bio)
-+		return req->bio->bi_ioprio;
-+	return 0;
- }
- 
- #define rq_data_dir(rq)		(op_is_write(req_op(rq)) ? WRITE : READ)
-@@ -218,64 +246,73 @@ static inline unsigned short req_get_ioprio(struct request *req)
- #define rq_dma_dir(rq) \
- 	(op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE)
- 
-#define rq_list_add(listptr, rq)	do {		\
-	(rq)->rq_next = *(listptr);			\
-	*(listptr) = rq;				\
-} while (0)
-
-#define rq_list_pop(listptr)				\
-({							\
-	struct request *__req = NULL;			\
-	if ((listptr) && *(listptr))	{		\
-		__req = *(listptr);			\
-		*(listptr) = __req->rq_next;		\
-	}						\
-	__req;						\
-})
-+static inline int rq_list_empty(const struct rq_list *rl)
-+{
-+	return rl->head == NULL;
-+}
- 
-#define rq_list_peek(listptr)				\
-({							\
-	struct request *__req = NULL;			\
-	if ((listptr) && *(listptr))			\
-		__req = *(listptr);			\
-	__req;						\
-})
-+static inline void rq_list_init(struct rq_list *rl)
-+{
-+	rl->head = NULL;
-+	rl->tail = NULL;
-+}
- 
-#define rq_list_for_each(listptr, pos)			\
-	for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos))
-+static inline void rq_list_add_tail(struct rq_list *rl, struct request *rq)
-+{
-+	rq->rq_next = NULL;
-+	if (rl->tail)
-+		rl->tail->rq_next = rq;
-+	else
-+		rl->head = rq;
-+	rl->tail = rq;
-+}
- 
-#define rq_list_for_each_safe(listptr, pos, nxt)			\
-	for (pos = rq_list_peek((listptr)), nxt = rq_list_next(pos);	\
-		pos; pos = nxt, nxt = pos ? rq_list_next(pos) : NULL)
-+static inline void rq_list_add_head(struct rq_list *rl, struct request *rq)
-+{
-+	rq->rq_next = rl->head;
-+	rl->head = rq;
-+	if (!rl->tail)
-+		rl->tail = rq;
-+}
- 
-#define rq_list_next(rq)	(rq)->rq_next
-#define rq_list_empty(list)	((list) == (struct request *) NULL)
-+static inline struct request *rq_list_pop(struct rq_list *rl)
-+{
-+	struct request *rq = rl->head;
- 
-/**
- * rq_list_move() - move a struct request from one list to another
- * @src: The source list @rq is currently in
- * @dst: The destination list that @rq will be appended to
- * @rq: The request to move
- * @prev: The request preceding @rq in @src (NULL if @rq is the head)
- */
-static inline void rq_list_move(struct request **src, struct request **dst,
-				struct request *rq, struct request *prev)
-+	if (rq) {
-+		rl->head = rl->head->rq_next;
-+		if (!rl->head)
-+			rl->tail = NULL;
-+		rq->rq_next = NULL;
-+	}
-+
-+	return rq;
-+}
-+
-+static inline struct request *rq_list_peek(struct rq_list *rl)
- {
-	if (prev)
-		prev->rq_next = rq->rq_next;
-	else
-		*src = rq->rq_next;
-	rq_list_add(dst, rq);
-+	return rl->head;
- }
- 
-+#define rq_list_for_each(rl, pos)					\
-+	for (pos = rq_list_peek((rl)); (pos); pos = pos->rq_next)
-+
-+#define rq_list_for_each_safe(rl, pos, nxt)				\
-+	for (pos = rq_list_peek((rl)), nxt = pos->rq_next;		\
-+		pos; pos = nxt, nxt = pos ? pos->rq_next : NULL)
-+
-+/**
-+ * enum blk_eh_timer_return - How the timeout handler should proceed
-+ * @BLK_EH_DONE: The block driver completed the command or will complete it at
-+ *	a later time.
-+ * @BLK_EH_RESET_TIMER: Reset the request timer and continue waiting for the
-+ *	request to complete.
-+ */
- enum blk_eh_timer_return {
-	BLK_EH_DONE,		/* drivers has completed the command */
-	BLK_EH_RESET_TIMER,	/* reset timer and try again */
-+	BLK_EH_DONE,
-+	BLK_EH_RESET_TIMER,
- };
- 
-#define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */
-#define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */
-
- /**
-  * struct blk_mq_hw_ctx - State for a hardware queue facing the hardware
-  * block device
-@@ -388,11 +425,6 @@ struct blk_mq_hw_ctx {
- 	 */
- 	struct blk_mq_tags	*sched_tags;
- 
-	/** @queued: Number of queued requests. */
-	unsigned long		queued;
-	/** @run: Number of dispatched requests. */
-	unsigned long		run;
-
- 	/** @numa_node: NUMA node the storage adapter has been connected to. */
- 	unsigned int		numa_node;
- 	/** @queue_num: Index of this hardware queue. */
-@@ -461,6 +493,7 @@ enum hctx_type {
- 
- /**
-  * struct blk_mq_tag_set - tag set that can be shared between request queues
-+ * @ops:	   Pointers to functions that implement block driver behavior.
-  * @map:	   One or more ctx -> hctx mappings. One map exists for each
-  *		   hardware queue type (enum hctx_type) that the driver wishes
-  *		   to support. There are no restrictions on maps being of the
-@@ -468,7 +501,6 @@ enum hctx_type {
-  *		   types.
-  * @nr_maps:	   Number of elements in the @map array. A number in the range
-  *		   [1, HCTX_MAX_TYPES].
- * @ops:	   Pointers to functions that implement block driver behavior.
-  * @nr_hw_queues:  Number of hardware queues supported by the block driver that
-  *		   owns this data structure.
-  * @queue_depth:   Number of tags per hardware queue, reserved tags included.
-@@ -489,11 +521,18 @@ enum hctx_type {
-  * @tag_list_lock: Serializes tag_list accesses.
-  * @tag_list:	   List of the request queues that use this tag set. See also
-  *		   request_queue.tag_set_list.
-+ * @srcu:	   Use as lock when type of the request queue is blocking
-+ *		   (BLK_MQ_F_BLOCKING).
-+ * @tags_srcu:	   SRCU used to defer freeing of tags page_list to prevent
-+ *		   use-after-free when iterating tags.
-+ * @update_nr_hwq_lock:
-+ * 		   Synchronize updating nr_hw_queues with add/del disk &
-+ * 		   switching elevator.
-  */
- struct blk_mq_tag_set {
-+	const struct blk_mq_ops	*ops;
- 	struct blk_mq_queue_map	map[HCTX_MAX_TYPES];
- 	unsigned int		nr_maps;
-	const struct blk_mq_ops	*ops;
- 	unsigned int		nr_hw_queues;
- 	unsigned int		queue_depth;
- 	unsigned int		reserved_tags;
-@@ -509,6 +548,10 @@ struct blk_mq_tag_set {
- 
- 	struct mutex		tag_list_lock;
- 	struct list_head	tag_list;
-+	struct srcu_struct	*srcu;
-+	struct srcu_struct	tags_srcu;
-+
-+	struct rw_semaphore	update_nr_hwq_lock;
- };
- 
- /**
-@@ -550,7 +593,7 @@ struct blk_mq_ops {
- 	 * empty the @rqlist completely, then the rest will be queued
- 	 * individually by the block layer upon return.
- 	 */
-	void (*queue_rqs)(struct request **rqlist);
-+	void (*queue_rqs)(struct rq_list *rqlist);
- 
- 	/**
- 	 * @get_budget: Reserve budget before queue request, once .queue_rq is
-@@ -630,7 +673,7 @@ struct blk_mq_ops {
- 	 * @map_queues: This allows drivers specify their own queue mapping by
- 	 * overriding the setup-time function that builds the mq_map.
- 	 */
-	int (*map_queues)(struct blk_mq_tag_set *set);
-+	void (*map_queues)(struct blk_mq_tag_set *set);
- 
- #ifdef CONFIG_BLK_DEBUG_FS
- 	/**
-@@ -641,8 +684,8 @@ struct blk_mq_ops {
- #endif
- };
- 
-+/* Keep hctx_flag_name[] in sync with the definitions below */
- enum {
-	BLK_MQ_F_SHOULD_MERGE	= 1 << 0,
- 	BLK_MQ_F_TAG_QUEUE_SHARED = 1 << 1,
- 	/*
- 	 * Set when this device requires underlying blk-mq device for
-@@ -650,48 +693,48 @@ enum {
- 	 */
- 	BLK_MQ_F_STACKING	= 1 << 2,
- 	BLK_MQ_F_TAG_HCTX_SHARED = 1 << 3,
-	BLK_MQ_F_BLOCKING	= 1 << 5,
-	/* Do not allow an I/O scheduler to be configured. */
-	BLK_MQ_F_NO_SCHED	= 1 << 6,
-+	BLK_MQ_F_BLOCKING	= 1 << 4,
-+
-+	/*
-+	 * Alloc tags on a round-robin base instead of the first available one.
-+	 */
-+	BLK_MQ_F_TAG_RR		= 1 << 5,
-+
- 	/*
- 	 * Select 'none' during queue registration in case of a single hwq
- 	 * or shared hwqs instead of 'mq-deadline'.
- 	 */
-	BLK_MQ_F_NO_SCHED_BY_DEFAULT	= 1 << 7,
-	BLK_MQ_F_ALLOC_POLICY_START_BIT = 8,
-	BLK_MQ_F_ALLOC_POLICY_BITS = 1,
-
-	BLK_MQ_S_STOPPED	= 0,
-	BLK_MQ_S_TAG_ACTIVE	= 1,
-	BLK_MQ_S_SCHED_RESTART	= 2,
-
-	/* hw queue is inactive after all its CPUs become offline */
-	BLK_MQ_S_INACTIVE	= 3,
-+	BLK_MQ_F_NO_SCHED_BY_DEFAULT	= 1 << 6,
- 
-	BLK_MQ_MAX_DEPTH	= 10240,
-
-	BLK_MQ_CPU_WORK_BATCH	= 8,
-+	BLK_MQ_F_MAX = 1 << 7,
- };
-#define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \
-	((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \
-		((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1))
-#define BLK_ALLOC_POLICY_TO_MQ_FLAG(policy) \
-	((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \
-		<< BLK_MQ_F_ALLOC_POLICY_START_BIT)
- 
-+#define BLK_MQ_MAX_DEPTH	(10240)
- #define BLK_MQ_NO_HCTX_IDX	(-1U)
- 
-struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata,
-+enum {
-+	/* Keep hctx_state_name[] in sync with the definitions below */
-+	BLK_MQ_S_STOPPED,
-+	BLK_MQ_S_TAG_ACTIVE,
-+	BLK_MQ_S_SCHED_RESTART,
-+	/* hw queue is inactive after all its CPUs become offline */
-+	BLK_MQ_S_INACTIVE,
-+	BLK_MQ_S_MAX
-+};
-+
-+struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set,
-+		struct queue_limits *lim, void *queuedata,
- 		struct lock_class_key *lkclass);
-#define blk_mq_alloc_disk(set, queuedata)				\
-+#define blk_mq_alloc_disk(set, lim, queuedata)				\
- ({									\
- 	static struct lock_class_key __key;				\
- 									\
-	__blk_mq_alloc_disk(set, queuedata, &__key);			\
-+	__blk_mq_alloc_disk(set, lim, queuedata, &__key);		\
- })
- struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q,
- 		struct lock_class_key *lkclass);
-struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *);
-+struct request_queue *blk_mq_alloc_queue(struct blk_mq_tag_set *set,
-+		struct queue_limits *lim, void *queuedata);
- int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
- 		struct request_queue *q);
- void blk_mq_destroy_queue(struct request_queue *);
-@@ -703,6 +746,8 @@ int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
- void blk_mq_free_tag_set(struct blk_mq_tag_set *set);
- 
- void blk_mq_free_request(struct request *rq);
-+int blk_rq_poll(struct request *rq, struct io_comp_batch *iob,
-+		unsigned int poll_flags);
- 
- bool blk_mq_queue_inflight(struct request_queue *q);
- 
-@@ -727,8 +772,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
- struct blk_mq_tags {
- 	unsigned int nr_tags;
- 	unsigned int nr_reserved_tags;
-
-	atomic_t active_queues;
-+	unsigned int active_queues;
- 
- 	struct sbitmap_queue bitmap_tags;
- 	struct sbitmap_queue breserved_tags;
-@@ -742,6 +786,7 @@ struct blk_mq_tags {
- 	 * request pool
- 	 */
- 	spinlock_t lock;
-+	struct rcu_head rcu_head;
- };
- 
- static inline struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags,
-@@ -825,7 +870,7 @@ void blk_mq_end_request_batch(struct io_comp_batch *ib);
-  */
- static inline bool blk_mq_need_time_stamp(struct request *rq)
- {
-	return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_ELV));
-+	return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_USE_SCHED));
- }
- 
- static inline bool blk_mq_is_reserved_rq(struct request *rq)
-@@ -833,22 +878,46 @@ static inline bool blk_mq_is_reserved_rq(struct request *rq)
- 	return rq->rq_flags & RQF_RESV;
- }
- 
-/*
-+/**
-+ * blk_mq_add_to_batch() - add a request to the completion batch
-+ * @req: The request to add to batch
-+ * @iob: The batch to add the request
-+ * @is_error: Specify true if the request failed with an error
-+ * @complete: The completaion handler for the request
-+ *
-  * Batched completions only work when there is no I/O error and no special
-  * ->end_io handler.
-+ *
-+ * Return: true when the request was added to the batch, otherwise false
-  */
- static inline bool blk_mq_add_to_batch(struct request *req,
-				       struct io_comp_batch *iob, int ioerror,
-+				       struct io_comp_batch *iob, bool is_error,
- 				       void (*complete)(struct io_comp_batch *))
- {
-	if (!iob || (req->rq_flags & RQF_ELV) || req->end_io || ioerror)
-+	/*
-+	 * Check various conditions that exclude batch processing:
-+	 * 1) No batch container
-+	 * 2) Has scheduler data attached
-+	 * 3) Not a passthrough request and end_io set
-+	 * 4) Not a passthrough request and failed with an error
-+	 */
-+	if (!iob)
-+		return false;
-+	if (req->rq_flags & RQF_SCHED_TAGS)
- 		return false;
-+	if (!blk_rq_is_passthrough(req)) {
-+		if (req->end_io)
-+			return false;
-+		if (is_error)
-+			return false;
-+	}
-+
- 	if (!iob->complete)
- 		iob->complete = complete;
- 	else if (iob->complete != complete)
- 		return false;
- 	iob->need_ts |= blk_mq_need_time_stamp(req);
-	rq_list_add(&iob->req_list, req);
-+	rq_list_add_tail(&iob->req_list, req);
- 	return true;
- }
- 
-@@ -864,7 +933,9 @@ void blk_mq_start_hw_queues(struct request_queue *q);
- void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
- void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async);
- void blk_mq_quiesce_queue(struct request_queue *q);
-void blk_mq_wait_quiesce_done(struct request_queue *q);
-+void blk_mq_wait_quiesce_done(struct blk_mq_tag_set *set);
-+void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set);
-+void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set);
- void blk_mq_unquiesce_queue(struct request_queue *q);
- void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
- void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
-@@ -873,14 +944,34 @@ void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs);
- void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
- 		busy_tag_iter_fn *fn, void *priv);
- void blk_mq_tagset_wait_completed_request(struct blk_mq_tag_set *tagset);
-void blk_mq_freeze_queue(struct request_queue *q);
-void blk_mq_unfreeze_queue(struct request_queue *q);
-+void blk_mq_freeze_queue_nomemsave(struct request_queue *q);
-+void blk_mq_unfreeze_queue_nomemrestore(struct request_queue *q);
-+static inline unsigned int __must_check
-+blk_mq_freeze_queue(struct request_queue *q)
-+{
-+	unsigned int memflags = memalloc_noio_save();
-+
-+	blk_mq_freeze_queue_nomemsave(q);
-+	return memflags;
-+}
-+static inline void
-+blk_mq_unfreeze_queue(struct request_queue *q, unsigned int memflags)
-+{
-+	blk_mq_unfreeze_queue_nomemrestore(q);
-+	memalloc_noio_restore(memflags);
-+}
- void blk_freeze_queue_start(struct request_queue *q);
- void blk_mq_freeze_queue_wait(struct request_queue *q);
- int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
- 				     unsigned long timeout);
-
-int blk_mq_map_queues(struct blk_mq_queue_map *qmap);
-+void blk_mq_unfreeze_queue_non_owner(struct request_queue *q);
-+void blk_freeze_queue_start_non_owner(struct request_queue *q);
-+
-+unsigned int blk_mq_num_possible_queues(unsigned int max_queues);
-+unsigned int blk_mq_num_online_queues(unsigned int max_queues);
-+void blk_mq_map_queues(struct blk_mq_queue_map *qmap);
-+void blk_mq_map_hw_queues(struct blk_mq_queue_map *qmap,
-+			  struct device *dev, unsigned int offset);
- void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues);
- 
- void blk_mq_quiesce_queue_nowait(struct request_queue *q);
-@@ -924,8 +1015,20 @@ static inline void *blk_mq_rq_to_pdu(struct request *rq)
- 	return rq + 1;
- }
- 
-+static inline struct blk_mq_hw_ctx *queue_hctx(struct request_queue *q, int id)
-+{
-+	struct blk_mq_hw_ctx *hctx;
-+
-+	rcu_read_lock();
-+	hctx = rcu_dereference(q->queue_hw_ctx)[id];
-+	rcu_read_unlock();
-+
-+	return hctx;
-+}
-+
- #define queue_for_each_hw_ctx(q, hctx, i)				\
-	xa_for_each(&(q)->hctx_table, (i), (hctx))
-+	for ((i) = 0; (i) < (q)->nr_hw_queues &&			\
-+	     ({ hctx = queue_hctx((q), i); 1; }); (i)++)
- 
- #define hctx_for_each_ctx(hctx, ctx, i)					\
- 	for ((i) = 0; (i) < (hctx)->nr_ctx &&				\
-@@ -937,15 +1040,6 @@ static inline void blk_mq_cleanup_rq(struct request *rq)
- 		rq->q->mq_ops->cleanup_rq(rq);
- }
- 
-static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio,
-		unsigned int nr_segs)
-{
-	rq->nr_phys_segments = nr_segs;
-	rq->__data_len = bio->bi_iter.bi_size;
-	rq->bio = rq->biotail = bio;
-	rq->ioprio = bio_prio(bio);
-}
-
- void blk_mq_hctx_set_fq_lock_class(struct blk_mq_hw_ctx *hctx,
- 		struct lock_class_key *key);
- 
-@@ -963,23 +1057,26 @@ blk_status_t blk_insert_cloned_request(struct request *rq);
- 
- struct rq_map_data {
- 	struct page **pages;
-	int page_order;
-	int nr_entries;
- 	unsigned long offset;
-	int null_mapped;
-	int from_user;
-+	unsigned short page_order;
-+	unsigned short nr_entries;
-+	bool null_mapped;
-+	bool from_user;
- };
- 
- int blk_rq_map_user(struct request_queue *, struct request *,
- 		struct rq_map_data *, void __user *, unsigned long, gfp_t);
-+int blk_rq_map_user_io(struct request *, struct rq_map_data *,
-+		void __user *, unsigned long, gfp_t, bool, int, bool, int);
- int blk_rq_map_user_iov(struct request_queue *, struct request *,
- 		struct rq_map_data *, const struct iov_iter *, gfp_t);
- int blk_rq_unmap_user(struct bio *);
-int blk_rq_map_kern(struct request_queue *, struct request *, void *,
-		unsigned int, gfp_t);
-+int blk_rq_map_kern(struct request *rq, void *kbuf, unsigned int len,
-+		gfp_t gfp);
- int blk_rq_append_bio(struct request *rq, struct bio *bio);
- void blk_execute_rq_nowait(struct request *rq, bool at_head);
- blk_status_t blk_execute_rq(struct request *rq, bool at_head);
-+bool blk_rq_is_poll(struct request *rq);
- 
- struct req_iterator {
- 	struct bvec_iter iter;
-@@ -1116,79 +1213,32 @@ static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)
- 	return max_t(unsigned short, rq->nr_phys_segments, 1);
- }
- 
-int __blk_rq_map_sg(struct request_queue *q, struct request *rq,
-		struct scatterlist *sglist, struct scatterlist **last_sg);
-static inline int blk_rq_map_sg(struct request_queue *q, struct request *rq,
-		struct scatterlist *sglist)
-{
-	struct scatterlist *last_sg = NULL;
-
-	return __blk_rq_map_sg(q, rq, sglist, &last_sg);
-}
-void blk_dump_rq_flags(struct request *, char *);
-
-#ifdef CONFIG_BLK_DEV_ZONED
-static inline unsigned int blk_rq_zone_no(struct request *rq)
-{
-	return disk_zone_no(rq->q->disk, blk_rq_pos(rq));
-}
-
-static inline unsigned int blk_rq_zone_is_seq(struct request *rq)
-{
-	return disk_zone_is_seq(rq->q->disk, blk_rq_pos(rq));
-}
-
-bool blk_req_needs_zone_write_lock(struct request *rq);
-bool blk_req_zone_write_trylock(struct request *rq);
-void __blk_req_zone_write_lock(struct request *rq);
-void __blk_req_zone_write_unlock(struct request *rq);
-
-static inline void blk_req_zone_write_lock(struct request *rq)
-+/**
-+ * blk_rq_nr_bvec - return number of bvecs in a request
-+ * @rq: request to calculate bvecs for
-+ *
-+ * Returns the number of bvecs.
-+ */
-+static inline unsigned int blk_rq_nr_bvec(struct request *rq)
- {
-	if (blk_req_needs_zone_write_lock(rq))
-		__blk_req_zone_write_lock(rq);
-}
-+	struct req_iterator rq_iter;
-+	struct bio_vec bv;
-+	unsigned int nr_bvec = 0;
- 
-static inline void blk_req_zone_write_unlock(struct request *rq)
-{
-	if (rq->rq_flags & RQF_ZONE_WRITE_LOCKED)
-		__blk_req_zone_write_unlock(rq);
-}
-+	rq_for_each_bvec(bv, rq, rq_iter)
-+		nr_bvec++;
- 
-static inline bool blk_req_zone_is_write_locked(struct request *rq)
-{
-	return rq->q->disk->seq_zones_wlock &&
-		test_bit(blk_rq_zone_no(rq), rq->q->disk->seq_zones_wlock);
-+	return nr_bvec;
- }
- 
-static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
-{
-	if (!blk_req_needs_zone_write_lock(rq))
-		return true;
-	return !blk_req_zone_is_write_locked(rq);
-}
-#else /* CONFIG_BLK_DEV_ZONED */
-static inline bool blk_req_needs_zone_write_lock(struct request *rq)
-+int __blk_rq_map_sg(struct request *rq, struct scatterlist *sglist,
-+		struct scatterlist **last_sg);
-+static inline int blk_rq_map_sg(struct request *rq, struct scatterlist *sglist)
- {
-	return false;
-}
-
-static inline void blk_req_zone_write_lock(struct request *rq)
-{
-}
-
-static inline void blk_req_zone_write_unlock(struct request *rq)
-{
-}
-static inline bool blk_req_zone_is_write_locked(struct request *rq)
-{
-	return false;
-}
-+	struct scatterlist *last_sg = NULL;
- 
-static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
-{
-	return true;
-+	return __blk_rq_map_sg(rq, sglist, &last_sg);
- }
-#endif /* CONFIG_BLK_DEV_ZONED */
-+void blk_dump_rq_flags(struct request *, char *);
- 
- #endif /* BLK_MQ_H */
-diff --git a/include/linux/blk-pm.h b/include/linux/blk-pm.h
-index 2580e05a8ab6..004b38a538ff 100644
--- a/include/linux/blk-pm.h
-+++ b/include/linux/blk-pm.h
-@@ -15,7 +15,6 @@ extern int blk_pre_runtime_suspend(struct request_queue *q);
- extern void blk_post_runtime_suspend(struct request_queue *q, int err);
- extern void blk_pre_runtime_resume(struct request_queue *q);
- extern void blk_post_runtime_resume(struct request_queue *q);
-extern void blk_set_runtime_active(struct request_queue *q);
- #else
- static inline void blk_pm_runtime_init(struct request_queue *q,
- 				       struct device *dev) {}
-diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
-index 1ef99790f6ed..5dc061d318a4 100644
--- a/include/linux/blk_types.h
-+++ b/include/linux/blk_types.h
-@@ -10,6 +10,7 @@
- #include <linux/bvec.h>
- #include <linux/device.h>
- #include <linux/ktime.h>
-+#include <linux/rw_hint.h>
- 
- struct bio_set;
- struct bio;
-@@ -40,34 +41,44 @@ struct bio_crypt_ctx;
- struct block_device {
- 	sector_t		bd_start_sect;
- 	sector_t		bd_nr_sectors;
-+	struct gendisk *	bd_disk;
-+	struct request_queue *	bd_queue;
- 	struct disk_stats __percpu *bd_stats;
- 	unsigned long		bd_stamp;
-	bool			bd_read_only;	/* read-only policy */
-+	atomic_t		__bd_flags;	// partition number + flags
-+#define BD_PARTNO		255	// lower 8 bits; assign-once
-+#define BD_READ_ONLY		(1u<<8) // read-only policy
-+#define BD_WRITE_HOLDER		(1u<<9)
-+#define BD_HAS_SUBMIT_BIO	(1u<<10)
-+#define BD_RO_WARNED		(1u<<11)
-+#ifdef CONFIG_FAIL_MAKE_REQUEST
-+#define BD_MAKE_IT_FAIL		(1u<<12)
-+#endif
- 	dev_t			bd_dev;
-+	struct address_space	*bd_mapping;	/* page cache */
-+
- 	atomic_t		bd_openers;
-	struct inode *		bd_inode;	/* will die */
-	struct super_block *	bd_super;
-+	spinlock_t		bd_size_lock; /* for bd_inode->i_size updates */
- 	void *			bd_claiming;
-	struct device		bd_device;
- 	void *			bd_holder;
-+	const struct blk_holder_ops *bd_holder_ops;
-+	struct mutex		bd_holder_lock;
- 	int			bd_holders;
-	bool			bd_write_holder;
- 	struct kobject		*bd_holder_dir;
-	u8			bd_partno;
-	spinlock_t		bd_size_lock; /* for bd_inode->i_size updates */
-	struct gendisk *	bd_disk;
-	struct request_queue *	bd_queue;
- 
-	/* The counter of freeze processes */
-	int			bd_fsfreeze_count;
-	/* Mutex for freeze */
-	struct mutex		bd_fsfreeze_mutex;
-	struct super_block	*bd_fsfreeze_sb;
-+	atomic_t		bd_fsfreeze_count; /* number of freeze requests */
-+	struct mutex		bd_fsfreeze_mutex; /* serialize freeze/thaw */
- 
- 	struct partition_meta_info *bd_meta_info;
-#ifdef CONFIG_FAIL_MAKE_REQUEST
-	bool			bd_make_it_fail;
-+	int			bd_writers;
-+#ifdef CONFIG_SECURITY
-+	void			*bd_security;
- #endif
-+	/*
-+	 * keep this out-of-line as it's both big and not needed in the fast
-+	 * path
-+	 */
-+	struct device		bd_device;
- } __randomize_layout;
- 
- #define bdev_whole(_bdev) \
-@@ -81,22 +92,16 @@ struct block_device {
- 
- /*
-  * Block error status values.  See block/blk-core:blk_errors for the details.
- * Alpha cannot write a byte atomically, so we need to use 32-bit value.
-  */
-#if defined(CONFIG_ALPHA) && !defined(__alpha_bwx__)
-typedef u32 __bitwise blk_status_t;
-typedef u32 blk_short_t;
-#else
- typedef u8 __bitwise blk_status_t;
- typedef u16 blk_short_t;
-#endif
- #define	BLK_STS_OK 0
- #define BLK_STS_NOTSUPP		((__force blk_status_t)1)
- #define BLK_STS_TIMEOUT		((__force blk_status_t)2)
- #define BLK_STS_NOSPC		((__force blk_status_t)3)
- #define BLK_STS_TRANSPORT	((__force blk_status_t)4)
- #define BLK_STS_TARGET		((__force blk_status_t)5)
-#define BLK_STS_NEXUS		((__force blk_status_t)6)
-+#define BLK_STS_RESV_CONFLICT	((__force blk_status_t)6)
- #define BLK_STS_MEDIUM		((__force blk_status_t)7)
- #define BLK_STS_PROTECTION	((__force blk_status_t)8)
- #define BLK_STS_RESOURCE	((__force blk_status_t)9)
-@@ -130,25 +135,13 @@ typedef u16 blk_short_t;
- #define BLK_STS_DEV_RESOURCE	((__force blk_status_t)13)
- 
- /*
- * BLK_STS_ZONE_RESOURCE is returned from the driver to the block layer if zone
- * related resources are unavailable, but the driver can guarantee the queue
- * will be rerun in the future once the resources become available again.
- *
- * This is different from BLK_STS_DEV_RESOURCE in that it explicitly references
- * a zone specific resource and IO to a different zone on the same device could
- * still be served. Examples of that are zones that are write-locked, but a read
- * to the same zone could be served.
- */
-#define BLK_STS_ZONE_RESOURCE	((__force blk_status_t)14)
-
-/*
-  * BLK_STS_ZONE_OPEN_RESOURCE is returned from the driver in the completion
-  * path if the device returns a status indicating that too many zone resources
-  * are currently open. The same command should be successful if resubmitted
-  * after the number of open zones decreases below the device's limits, which is
-  * reported in the request_queue's max_open_zones.
-  */
-#define BLK_STS_ZONE_OPEN_RESOURCE	((__force blk_status_t)15)
-+#define BLK_STS_ZONE_OPEN_RESOURCE	((__force blk_status_t)14)
- 
- /*
-  * BLK_STS_ZONE_ACTIVE_RESOURCE is returned from the driver in the completion
-@@ -157,14 +150,25 @@ typedef u16 blk_short_t;
-  * after the number of active zones decreases below the device's limits, which
-  * is reported in the request_queue's max_active_zones.
-  */
-#define BLK_STS_ZONE_ACTIVE_RESOURCE	((__force blk_status_t)16)
-+#define BLK_STS_ZONE_ACTIVE_RESOURCE	((__force blk_status_t)15)
- 
- /*
-  * BLK_STS_OFFLINE is returned from the driver when the target device is offline
-  * or is being taken offline. This could help differentiate the case where a
-  * device is intentionally being shut down from a real I/O error.
-  */
-#define BLK_STS_OFFLINE		((__force blk_status_t)17)
-+#define BLK_STS_OFFLINE		((__force blk_status_t)16)
-+
-+/*
-+ * BLK_STS_DURATION_LIMIT is returned from the driver when the target device
-+ * aborted the command because it exceeded one of its Command Duration Limits.
-+ */
-+#define BLK_STS_DURATION_LIMIT	((__force blk_status_t)17)
-+
-+/*
-+ * Invalid size or alignment.
-+ */
-+#define BLK_STS_INVAL	((__force blk_status_t)19)
- 
- /**
-  * blk_path_error - returns true if error may be path related
-@@ -184,7 +188,7 @@ static inline bool blk_path_error(blk_status_t error)
- 	case BLK_STS_NOTSUPP:
- 	case BLK_STS_NOSPC:
- 	case BLK_STS_TARGET:
-	case BLK_STS_NEXUS:
-+	case BLK_STS_RESV_CONFLICT:
- 	case BLK_STS_MEDIUM:
- 	case BLK_STS_PROTECTION:
- 		return false;
-@@ -194,52 +198,6 @@ static inline bool blk_path_error(blk_status_t error)
- 	return true;
- }
- 
-/*
- * From most significant bit:
- * 1 bit: reserved for other usage, see below
- * 12 bits: original size of bio
- * 51 bits: issue time of bio
- */
-#define BIO_ISSUE_RES_BITS      1
-#define BIO_ISSUE_SIZE_BITS     12
-#define BIO_ISSUE_RES_SHIFT     (64 - BIO_ISSUE_RES_BITS)
-#define BIO_ISSUE_SIZE_SHIFT    (BIO_ISSUE_RES_SHIFT - BIO_ISSUE_SIZE_BITS)
-#define BIO_ISSUE_TIME_MASK     ((1ULL << BIO_ISSUE_SIZE_SHIFT) - 1)
-#define BIO_ISSUE_SIZE_MASK     \
-	(((1ULL << BIO_ISSUE_SIZE_BITS) - 1) << BIO_ISSUE_SIZE_SHIFT)
-#define BIO_ISSUE_RES_MASK      (~((1ULL << BIO_ISSUE_RES_SHIFT) - 1))
-
-/* Reserved bit for blk-throtl */
-#define BIO_ISSUE_THROTL_SKIP_LATENCY (1ULL << 63)
-
-struct bio_issue {
-	u64 value;
-};
-
-static inline u64 __bio_issue_time(u64 time)
-{
-	return time & BIO_ISSUE_TIME_MASK;
-}
-
-static inline u64 bio_issue_time(struct bio_issue *issue)
-{
-	return __bio_issue_time(issue->value);
-}
-
-static inline sector_t bio_issue_size(struct bio_issue *issue)
-{
-	return ((issue->value & BIO_ISSUE_SIZE_MASK) >> BIO_ISSUE_SIZE_SHIFT);
-}
-
-static inline void bio_issue_init(struct bio_issue *issue,
-				       sector_t size)
-{
-	size &= (1ULL << BIO_ISSUE_SIZE_BITS) - 1;
-	issue->value = ((issue->value & BIO_ISSUE_RES_MASK) |
-			(ktime_get_ns() & BIO_ISSUE_TIME_MASK) |
-			((u64)size << BIO_ISSUE_SIZE_SHIFT));
-}
-
- typedef __u32 __bitwise blk_opf_t;
- 
- typedef unsigned int blk_qc_t;
-@@ -257,12 +215,31 @@ struct bio {
- 						 */
- 	unsigned short		bi_flags;	/* BIO_* below */
- 	unsigned short		bi_ioprio;
-+	enum rw_hint		bi_write_hint;
-+	u8			bi_write_stream;
- 	blk_status_t		bi_status;
-+
-+	/*
-+	 * The bvec gap bit indicates the lowest set bit in any address offset
-+	 * between all bi_io_vecs. This field is initialized only after the bio
-+	 * is split to the hardware limits (see bio_split_io_at()). The value
-+	 * may be used to consider DMA optimization when performing that
-+	 * mapping. The value is compared to a power of two mask where the
-+	 * result depends on any bit set within the mask, so saving the lowest
-+	 * bit is sufficient to know if any segment gap collides with the mask.
-+	 */
-+	u8			bi_bvec_gap_bit;
-+
- 	atomic_t		__bi_remaining;
- 
- 	struct bvec_iter	bi_iter;
- 
-	blk_qc_t		bi_cookie;
-+	union {
-+		/* for polled bios: */
-+		blk_qc_t		bi_cookie;
-+		/* for plugged zoned writes only: */
-+		unsigned int		__bi_nr_segments;
-+	};
- 	bio_end_io_t		*bi_end_io;
- 	void			*bi_private;
- #ifdef CONFIG_BLK_CGROUP
-@@ -273,7 +250,8 @@ struct bio {
- 	 * on release of the bio.
- 	 */
- 	struct blkcg_gq		*bi_blkg;
-	struct bio_issue	bi_issue;
-+	/* Time that this bio was issued. */
-+	u64			issue_time_ns;
- #ifdef CONFIG_BLK_CGROUP_IOCOST
- 	u64			bi_iocost_cost;
- #endif
-@@ -283,11 +261,9 @@ struct bio {
- 	struct bio_crypt_ctx	*bi_crypt_context;
- #endif
- 
-	union {
- #if defined(CONFIG_BLK_DEV_INTEGRITY)
-		struct bio_integrity_payload *bi_integrity; /* data integrity */
-+	struct bio_integrity_payload *bi_integrity; /* data integrity */
- #endif
-	};
- 
- 	unsigned short		bi_vcnt;	/* how many bio_vec's */
- 
-@@ -302,38 +278,43 @@ struct bio {
- 	struct bio_vec		*bi_io_vec;	/* the actual vec list */
- 
- 	struct bio_set		*bi_pool;
-
-	/*
-	 * We can inline a number of vecs at the end of the bio, to avoid
-	 * double allocations for a small number of bio_vecs. This member
-	 * MUST obviously be kept at the very end of the bio.
-	 */
-	struct bio_vec		bi_inline_vecs[];
- };
- 
- #define BIO_RESET_BYTES		offsetof(struct bio, bi_max_vecs)
- #define BIO_MAX_SECTORS		(UINT_MAX >> SECTOR_SHIFT)
- 
-+static inline struct bio_vec *bio_inline_vecs(struct bio *bio)
-+{
-+	return (struct bio_vec *)(bio + 1);
-+}
-+
- /*
-  * bio flags
-  */
- enum {
-	BIO_NO_PAGE_REF,	/* don't put release vec pages */
-+	BIO_PAGE_PINNED,	/* Unpin pages in bio_release_pages() */
- 	BIO_CLONED,		/* doesn't own data */
-	BIO_BOUNCED,		/* bio is a bounce bio */
-	BIO_WORKINGSET,		/* contains userspace workingset pages */
- 	BIO_QUIET,		/* Make BIO Quiet */
- 	BIO_CHAIN,		/* chained bio, ->bi_remaining in effect */
- 	BIO_REFFED,		/* bio has elevated ->bi_cnt */
-	BIO_THROTTLED,		/* This bio has already been subjected to
-+	BIO_BPS_THROTTLED,	/* This bio has already been subjected to
- 				 * throttling rules. Don't do it again. */
- 	BIO_TRACE_COMPLETION,	/* bio_endio() should trace the final completion
- 				 * of this bio. */
- 	BIO_CGROUP_ACCT,	/* has been accounted to a cgroup */
- 	BIO_QOS_THROTTLED,	/* bio went through rq_qos throttle path */
-+	/*
-+	 * This bio has completed bps throttling at the single tg granularity,
-+	 * which is different from BIO_BPS_THROTTLED. When the bio is enqueued
-+	 * into the sq->queued of the upper tg, or is about to be dispatched,
-+	 * this flag needs to be cleared. Since blk-throttle and rq_qos are not
-+	 * on the same hierarchical level, reuse the value.
-+	 */
-+	BIO_TG_BPS_THROTTLED = BIO_QOS_THROTTLED,
- 	BIO_QOS_MERGED,		/* but went through rq_qos merge path */
- 	BIO_REMAPPED,
-	BIO_ZONE_WRITE_LOCKED,	/* Owns a zoned device zone write lock */
-+	BIO_ZONE_WRITE_PLUGGING, /* bio handled through zone write plugging */
-+	BIO_EMULATES_ZONE_APPEND, /* bio emulates a zone append operation */
- 	BIO_FLAG_LAST
- };
- 
-@@ -367,20 +348,20 @@ enum req_op {
- 	REQ_OP_DISCARD		= (__force blk_opf_t)3,
- 	/* securely erase sectors */
- 	REQ_OP_SECURE_ERASE	= (__force blk_opf_t)5,
-+	/* write data at the current zone write pointer */
-+	REQ_OP_ZONE_APPEND	= (__force blk_opf_t)7,
- 	/* write the zero filled sector many times */
- 	REQ_OP_WRITE_ZEROES	= (__force blk_opf_t)9,
- 	/* Open a zone */
-	REQ_OP_ZONE_OPEN	= (__force blk_opf_t)10,
-+	REQ_OP_ZONE_OPEN	= (__force blk_opf_t)11,
- 	/* Close a zone */
-	REQ_OP_ZONE_CLOSE	= (__force blk_opf_t)11,
-+	REQ_OP_ZONE_CLOSE	= (__force blk_opf_t)13,
- 	/* Transition a zone to full */
-	REQ_OP_ZONE_FINISH	= (__force blk_opf_t)12,
-	/* write data at the current zone write pointer */
-	REQ_OP_ZONE_APPEND	= (__force blk_opf_t)13,
-+	REQ_OP_ZONE_FINISH	= (__force blk_opf_t)15,
- 	/* reset a zone write pointer */
-	REQ_OP_ZONE_RESET	= (__force blk_opf_t)15,
-+	REQ_OP_ZONE_RESET	= (__force blk_opf_t)17,
- 	/* reset all the zone present on the device */
-	REQ_OP_ZONE_RESET_ALL	= (__force blk_opf_t)17,
-+	REQ_OP_ZONE_RESET_ALL	= (__force blk_opf_t)19,
- 
- 	/* Driver private requests */
- 	REQ_OP_DRV_IN		= (__force blk_opf_t)34,
-@@ -389,6 +370,7 @@ enum req_op {
- 	REQ_OP_LAST		= (__force blk_opf_t)36,
- };
- 
-+/* Keep cmd_flag_name[] in sync with the definitions below */
- enum req_flag_bits {
- 	__REQ_FAILFAST_DEV =	/* no driver retries of device errors */
- 		REQ_OP_BITS,
-@@ -405,19 +387,12 @@ enum req_flag_bits {
- 	__REQ_RAHEAD,		/* read ahead, can fail anytime */
- 	__REQ_BACKGROUND,	/* background IO */
- 	__REQ_NOWAIT,           /* Don't wait if request will block */
-	/*
-	 * When a shared kthread needs to issue a bio for a cgroup, doing
-	 * so synchronously can lead to priority inversions as the kthread
-	 * can be trapped waiting for that cgroup.  CGROUP_PUNT flag makes
-	 * submit_bio() punt the actual issuing to a dedicated per-blkcg
-	 * work item to avoid such priority inversions.
-	 */
-	__REQ_CGROUP_PUNT,
- 	__REQ_POLLED,		/* caller polls for completion using bio_poll */
- 	__REQ_ALLOC_CACHE,	/* allocate IO from cache if available */
- 	__REQ_SWAP,		/* swap I/O */
- 	__REQ_DRV,		/* for driver use */
-
-+	__REQ_FS_PRIVATE,	/* for file system (submitter) use */
-+	__REQ_ATOMIC,		/* for atomic write operations */
- 	/*
- 	 * Command specific flags, keep last:
- 	 */
-@@ -444,14 +419,14 @@ enum req_flag_bits {
- #define REQ_RAHEAD	(__force blk_opf_t)(1ULL << __REQ_RAHEAD)
- #define REQ_BACKGROUND	(__force blk_opf_t)(1ULL << __REQ_BACKGROUND)
- #define REQ_NOWAIT	(__force blk_opf_t)(1ULL << __REQ_NOWAIT)
-#define REQ_CGROUP_PUNT	(__force blk_opf_t)(1ULL << __REQ_CGROUP_PUNT)
-
-#define REQ_NOUNMAP	(__force blk_opf_t)(1ULL << __REQ_NOUNMAP)
- #define REQ_POLLED	(__force blk_opf_t)(1ULL << __REQ_POLLED)
- #define REQ_ALLOC_CACHE	(__force blk_opf_t)(1ULL << __REQ_ALLOC_CACHE)
-
-#define REQ_DRV		(__force blk_opf_t)(1ULL << __REQ_DRV)
- #define REQ_SWAP	(__force blk_opf_t)(1ULL << __REQ_SWAP)
-+#define REQ_DRV		(__force blk_opf_t)(1ULL << __REQ_DRV)
-+#define REQ_FS_PRIVATE	(__force blk_opf_t)(1ULL << __REQ_FS_PRIVATE)
-+#define REQ_ATOMIC	(__force blk_opf_t)(1ULL << __REQ_ATOMIC)
-+
-+#define REQ_NOUNMAP	(__force blk_opf_t)(1ULL << __REQ_NOUNMAP)
- 
- #define REQ_FAILFAST_MASK \
- 	(REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER)
-@@ -473,13 +448,6 @@ static inline enum req_op bio_op(const struct bio *bio)
- 	return bio->bi_opf & REQ_OP_MASK;
- }
- 
-/* obsolete, don't use in new code */
-static inline void bio_set_op_attrs(struct bio *bio, enum req_op op,
-				    blk_opf_t op_flags)
-{
-	bio->bi_opf = op | op_flags;
-}
-
- static inline bool op_is_write(blk_opf_t op)
- {
- 	return !!(op & (__force blk_opf_t)1);
-@@ -511,15 +479,13 @@ static inline bool op_is_discard(blk_opf_t op)
- }
- 
- /*
- * Check if a bio or request operation is a zone management operation, with
- * the exception of REQ_OP_ZONE_RESET_ALL which is treated as a special case
- * due to its different handling in the block layer and device response in
- * case of command failure.
-+ * Check if a bio or request operation is a zone management operation.
-  */
- static inline bool op_is_zone_mgmt(enum req_op op)
- {
- 	switch (op & REQ_OP_MASK) {
- 	case REQ_OP_ZONE_RESET:
-+	case REQ_OP_ZONE_RESET_ALL:
- 	case REQ_OP_ZONE_OPEN:
- 	case REQ_OP_ZONE_CLOSE:
- 	case REQ_OP_ZONE_FINISH:
-diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
-index 84b13fdd34a7..72e34acd439c 100644
--- a/include/linux/blkdev.h
-+++ b/include/linux/blkdev.h
-@@ -22,9 +22,10 @@
- #include <linux/blkzoned.h>
- #include <linux/sched.h>
- #include <linux/sbitmap.h>
-#include <linux/srcu.h>
- #include <linux/uuid.h>
- #include <linux/xarray.h>
-+#include <linux/file.h>
-+#include <linux/lockdep.h>
- 
- struct module;
- struct request_queue;
-@@ -37,19 +38,14 @@ struct blk_flush_queue;
- struct kiocb;
- struct pr_ops;
- struct rq_qos;
-+struct blk_report_zones_args;
- struct blk_queue_stats;
- struct blk_stat_callback;
- struct blk_crypto_profile;
- 
- extern const struct device_type disk_type;
-extern struct device_type part_type;
-extern struct class block_class;
-
-/* Must be consistent with blk_mq_poll_stats_bkt() */
-#define BLK_MQ_POLL_STATS_BKTS 16
-
-/* Doing classic polling */
-#define BLK_MQ_POLL_CLASSIC -1
-+extern const struct device_type part_type;
-+extern const struct class block_class;
- 
- /*
-  * Maximum number of blkcg policies allowed to be registered concurrently.
-@@ -111,14 +107,40 @@ enum {
- struct disk_events;
- struct badblocks;
- 
-+enum blk_integrity_checksum {
-+	BLK_INTEGRITY_CSUM_NONE		= 0,
-+	BLK_INTEGRITY_CSUM_IP		= 1,
-+	BLK_INTEGRITY_CSUM_CRC		= 2,
-+	BLK_INTEGRITY_CSUM_CRC64	= 3,
-+} __packed ;
-+
- struct blk_integrity {
-	const struct blk_integrity_profile	*profile;
- 	unsigned char				flags;
-	unsigned char				tuple_size;
-+	enum blk_integrity_checksum		csum_type;
-+	unsigned char				metadata_size;
-+	unsigned char				pi_offset;
- 	unsigned char				interval_exp;
- 	unsigned char				tag_size;
-+	unsigned char				pi_tuple_size;
- };
- 
-+typedef unsigned int __bitwise blk_mode_t;
-+
-+/* open for reading */
-+#define BLK_OPEN_READ		((__force blk_mode_t)(1 << 0))
-+/* open for writing */
-+#define BLK_OPEN_WRITE		((__force blk_mode_t)(1 << 1))
-+/* open exclusively (vs other exclusive openers */
-+#define BLK_OPEN_EXCL		((__force blk_mode_t)(1 << 2))
-+/* opened with O_NDELAY */
-+#define BLK_OPEN_NDELAY		((__force blk_mode_t)(1 << 3))
-+/* open for "writes" only for ioctls (specialy hack for floppy.c) */
-+#define BLK_OPEN_WRITE_IOCTL	((__force blk_mode_t)(1 << 4))
-+/* open is exclusive wrt all other BLK_OPEN_WRITE opens to the device */
-+#define BLK_OPEN_RESTRICT_WRITES	((__force blk_mode_t)(1 << 5))
-+/* return partition scanning errors */
-+#define BLK_OPEN_STRICT_SCAN	((__force blk_mode_t)(1 << 6))
-+
- struct gendisk {
- 	/*
- 	 * major/first_minor/minors should not be set by any new driver, the
-@@ -151,42 +173,37 @@ struct gendisk {
- #define GD_ADDED			4
- #define GD_SUPPRESS_PART_SCAN		5
- #define GD_OWNS_QUEUE			6
-+#define GD_ZONE_APPEND_USED		7
- 
- 	struct mutex open_mutex;	/* open/close mutex */
- 	unsigned open_partitions;	/* number of open partitions */
- 
- 	struct backing_dev_info	*bdi;
-+	struct kobject queue_kobj;	/* the queue/ directory */
- 	struct kobject *slave_dir;
- #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
- 	struct list_head slave_bdevs;
- #endif
- 	struct timer_rand_state *random;
-	atomic_t sync_io;		/* RAID */
- 	struct disk_events *ev;
-#ifdef  CONFIG_BLK_DEV_INTEGRITY
-	struct kobject integrity_kobj;
-#endif	/* CONFIG_BLK_DEV_INTEGRITY */
- 
- #ifdef CONFIG_BLK_DEV_ZONED
- 	/*
-	 * Zoned block device information for request dispatch control.
-	 * nr_zones is the total number of zones of the device. This is always
-	 * 0 for regular block devices. conv_zones_bitmap is a bitmap of nr_zones
-	 * bits which indicates if a zone is conventional (bit set) or
-	 * sequential (bit clear). seq_zones_wlock is a bitmap of nr_zones
-	 * bits which indicates if a zone is write locked, that is, if a write
-	 * request targeting the zone was dispatched.
-	 *
-	 * Reads of this information must be protected with blk_queue_enter() /
-	 * blk_queue_exit(). Modifying this information is only allowed while
-	 * no requests are being processed. See also blk_mq_freeze_queue() and
-	 * blk_mq_unfreeze_queue().
-+	 * Zoned block device information. Reads of this information must be
-+	 * protected with blk_queue_enter() / blk_queue_exit(). Modifying this
-+	 * information is only allowed while no requests are being processed.
-+	 * See also blk_mq_freeze_queue() and blk_mq_unfreeze_queue().
- 	 */
- 	unsigned int		nr_zones;
-	unsigned int		max_open_zones;
-	unsigned int		max_active_zones;
-	unsigned long		*conv_zones_bitmap;
-	unsigned long		*seq_zones_wlock;
-+	unsigned int		zone_capacity;
-+	unsigned int		last_zone_capacity;
-+	u8 __rcu		*zones_cond;
-+	unsigned int		zone_wplugs_hash_bits;
-+	atomic_t		nr_zone_wplugs;
-+	spinlock_t		zone_wplugs_lock;
-+	struct mempool		*zone_wplugs_pool;
-+	struct hlist_head	*zone_wplugs_hash;
-+	struct workqueue_struct *zone_wplugs_wq;
- #endif /* CONFIG_BLK_DEV_ZONED */
- 
- #if IS_ENABLED(CONFIG_CDROM)
-@@ -196,18 +213,16 @@ struct gendisk {
- 	struct badblocks *bb;
- 	struct lockdep_map lockdep_map;
- 	u64 diskseq;
-+	blk_mode_t open_mode;
- 
- 	/*
- 	 * Independent sector access ranges. This is always NULL for
- 	 * devices that do not have multiple independent access ranges.
- 	 */
- 	struct blk_independent_access_ranges *ia_ranges;
-};
- 
-static inline bool disk_live(struct gendisk *disk)
-{
-	return !inode_unhashed(disk->part0->bd_inode);
-}
-+	struct mutex rqos_state_mutex;	/* rqos state change mutex */
-+};
- 
- /**
-  * disk_openers - returns how many openers are there for a disk
-@@ -224,6 +239,19 @@ static inline unsigned int disk_openers(struct gendisk *disk)
- 	return atomic_read(&disk->part0->bd_openers);
- }
- 
-+/**
-+ * disk_has_partscan - return %true if partition scanning is enabled on a disk
-+ * @disk: disk to check
-+ *
-+ * Returns %true if partitions scanning is enabled for @disk, or %false if
-+ * partition scanning is disabled either permanently or temporarily.
-+ */
-+static inline bool disk_has_partscan(struct gendisk *disk)
-+{
-+	return !(disk->flags & (GENHD_FL_NO_PART | GENHD_FL_HIDDEN)) &&
-+		!test_bit(GD_SUPPRESS_PART_SCAN, &disk->state);
-+}
-+
- /*
-  * The gendisk is refcounted by the part0 block_device, and the bd_device
-  * therein is also used for device model presentation in sysfs.
-@@ -244,9 +272,21 @@ static inline dev_t disk_devt(struct gendisk *disk)
- 	return MKDEV(disk->major, disk->first_minor);
- }
- 
-+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-+/*
-+ * We should strive for 1 << (PAGE_SHIFT + MAX_PAGECACHE_ORDER)
-+ * however we constrain this to what we can validate and test.
-+ */
-+#define BLK_MAX_BLOCK_SIZE      SZ_64K
-+#else
-+#define BLK_MAX_BLOCK_SIZE      PAGE_SIZE
-+#endif
-+
-+
-+/* blk_validate_limits() validates bsize, so drivers don't usually need to */
- static inline int blk_validate_block_size(unsigned long bsize)
- {
-	if (bsize < 512 || bsize > PAGE_SIZE || !is_power_of_2(bsize))
-+	if (bsize < 512 || bsize > BLK_MAX_BLOCK_SIZE || !is_power_of_2(bsize))
- 		return -EINVAL;
- 
- 	return 0;
-@@ -258,29 +298,79 @@ static inline bool blk_op_is_passthrough(blk_opf_t op)
- 	return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
- }
- 
-/*
- * Zoned block device models (zoned limit).
- *
- * Note: This needs to be ordered from the least to the most severe
- * restrictions for the inheritance in blk_stack_limits() to work.
- */
-enum blk_zoned_model {
-	BLK_ZONED_NONE = 0,	/* Regular block device */
-	BLK_ZONED_HA,		/* Host-aware zoned block device */
-	BLK_ZONED_HM,		/* Host-managed zoned block device */
-};
-+/* flags set by the driver in queue_limits.features */
-+typedef unsigned int __bitwise blk_features_t;
-+
-+/* supports a volatile write cache */
-+#define BLK_FEAT_WRITE_CACHE		((__force blk_features_t)(1u << 0))
-+
-+/* supports passing on the FUA bit */
-+#define BLK_FEAT_FUA			((__force blk_features_t)(1u << 1))
-+
-+/* rotational device (hard drive or floppy) */
-+#define BLK_FEAT_ROTATIONAL		((__force blk_features_t)(1u << 2))
-+
-+/* contributes to the random number pool */
-+#define BLK_FEAT_ADD_RANDOM		((__force blk_features_t)(1u << 3))
-+
-+/* do disk/partitions IO accounting */
-+#define BLK_FEAT_IO_STAT		((__force blk_features_t)(1u << 4))
-+
-+/* don't modify data until writeback is done */
-+#define BLK_FEAT_STABLE_WRITES		((__force blk_features_t)(1u << 5))
-+
-+/* always completes in submit context */
-+#define BLK_FEAT_SYNCHRONOUS		((__force blk_features_t)(1u << 6))
-+
-+/* supports REQ_NOWAIT */
-+#define BLK_FEAT_NOWAIT			((__force blk_features_t)(1u << 7))
-+
-+/* supports DAX */
-+#define BLK_FEAT_DAX			((__force blk_features_t)(1u << 8))
-+
-+/* supports I/O polling */
-+#define BLK_FEAT_POLL			((__force blk_features_t)(1u << 9))
-+
-+/* is a zoned device */
-+#define BLK_FEAT_ZONED			((__force blk_features_t)(1u << 10))
-+
-+/* supports PCI(e) p2p requests */
-+#define BLK_FEAT_PCI_P2PDMA		((__force blk_features_t)(1u << 12))
-+
-+/* skip this queue in blk_mq_(un)quiesce_tagset */
-+#define BLK_FEAT_SKIP_TAGSET_QUIESCE	((__force blk_features_t)(1u << 13))
-+
-+/* undocumented magic for bcache */
-+#define BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE \
-+	((__force blk_features_t)(1u << 15))
-+
-+/* atomic writes enabled */
-+#define BLK_FEAT_ATOMIC_WRITES \
-+	((__force blk_features_t)(1u << 16))
- 
- /*
- * BLK_BOUNCE_NONE:	never bounce (default)
- * BLK_BOUNCE_HIGH:	bounce all highmem pages
-+ * Flags automatically inherited when stacking limits.
-  */
-enum blk_bounce {
-	BLK_BOUNCE_NONE,
-	BLK_BOUNCE_HIGH,
-};
-+#define BLK_FEAT_INHERIT_MASK \
-+	(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL | \
-+	 BLK_FEAT_STABLE_WRITES | BLK_FEAT_ZONED | \
-+	 BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE)
-+
-+/* internal flags in queue_limits.flags */
-+typedef unsigned int __bitwise blk_flags_t;
-+
-+/* do not send FLUSH/FUA commands despite advertising a write cache */
-+#define BLK_FLAG_WRITE_CACHE_DISABLED	((__force blk_flags_t)(1u << 0))
-+
-+/* I/O topology is misaligned */
-+#define BLK_FLAG_MISALIGNED		((__force blk_flags_t)(1u << 1))
-+
-+/* passthrough command IO accounting */
-+#define BLK_FLAG_IOSTATS_PASSTHROUGH	((__force blk_flags_t)(1u << 2))
- 
- struct queue_limits {
-	enum blk_bounce		bounce;
-+	blk_features_t		features;
-+	blk_flags_t		flags;
- 	unsigned long		seg_boundary_mask;
- 	unsigned long		virt_boundary_mask;
- 
-@@ -288,7 +378,9 @@ struct queue_limits {
- 	unsigned int		max_dev_sectors;
- 	unsigned int		chunk_sectors;
- 	unsigned int		max_sectors;
-+	unsigned int		max_user_sectors;
- 	unsigned int		max_segment_size;
-+	unsigned int		max_fast_segment_size;
- 	unsigned int		physical_block_size;
- 	unsigned int		logical_block_size;
- 	unsigned int		alignment_offset;
-@@ -296,67 +388,66 @@ struct queue_limits {
- 	unsigned int		io_opt;
- 	unsigned int		max_discard_sectors;
- 	unsigned int		max_hw_discard_sectors;
-+	unsigned int		max_user_discard_sectors;
- 	unsigned int		max_secure_erase_sectors;
- 	unsigned int		max_write_zeroes_sectors;
-+	unsigned int		max_wzeroes_unmap_sectors;
-+	unsigned int		max_hw_wzeroes_unmap_sectors;
-+	unsigned int		max_user_wzeroes_unmap_sectors;
-+	unsigned int		max_hw_zone_append_sectors;
- 	unsigned int		max_zone_append_sectors;
- 	unsigned int		discard_granularity;
- 	unsigned int		discard_alignment;
- 	unsigned int		zone_write_granularity;
- 
-+	/* atomic write limits */
-+	unsigned int		atomic_write_hw_max;
-+	unsigned int		atomic_write_max_sectors;
-+	unsigned int		atomic_write_hw_boundary;
-+	unsigned int		atomic_write_boundary_sectors;
-+	unsigned int		atomic_write_hw_unit_min;
-+	unsigned int		atomic_write_unit_min;
-+	unsigned int		atomic_write_hw_unit_max;
-+	unsigned int		atomic_write_unit_max;
-+
- 	unsigned short		max_segments;
- 	unsigned short		max_integrity_segments;
- 	unsigned short		max_discard_segments;
- 
-	unsigned char		misaligned;
-	unsigned char		discard_misaligned;
-	unsigned char		raid_partial_stripes_expensive;
-	enum blk_zoned_model	zoned;
-+	unsigned short		max_write_streams;
-+	unsigned int		write_stream_granularity;
-+
-+	unsigned int		max_open_zones;
-+	unsigned int		max_active_zones;
-+
-+	/*
-+	 * Drivers that set dma_alignment to less than 511 must be prepared to
-+	 * handle individual bvec's that are not a multiple of a SECTOR_SIZE
-+	 * due to possible offsets.
-+	 */
-+	unsigned int		dma_alignment;
-+	unsigned int		dma_pad_mask;
-+
-+	struct blk_integrity	integrity;
- };
- 
- typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,
- 			       void *data);
- 
-void disk_set_zoned(struct gendisk *disk, enum blk_zoned_model model);
-+int disk_report_zone(struct gendisk *disk, struct blk_zone *zone,
-+		     unsigned int idx, struct blk_report_zones_args *args);
- 
-#ifdef CONFIG_BLK_DEV_ZONED
-+int blkdev_get_zone_info(struct block_device *bdev, sector_t sector,
-+			 struct blk_zone *zone);
- 
- #define BLK_ALL_ZONES  ((unsigned int)-1)
- int blkdev_report_zones(struct block_device *bdev, sector_t sector,
-			unsigned int nr_zones, report_zones_cb cb, void *data);
-unsigned int bdev_nr_zones(struct block_device *bdev);
-extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op,
-			    sector_t sectors, sector_t nr_sectors,
-			    gfp_t gfp_mask);
-int blk_revalidate_disk_zones(struct gendisk *disk,
-			      void (*update_driver_data)(struct gendisk *disk));
-
-extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
-				     unsigned int cmd, unsigned long arg);
-extern int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode,
-				  unsigned int cmd, unsigned long arg);
-
-#else /* CONFIG_BLK_DEV_ZONED */
-
-static inline unsigned int bdev_nr_zones(struct block_device *bdev)
-{
-	return 0;
-}
-
-static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
-					    fmode_t mode, unsigned int cmd,
-					    unsigned long arg)
-{
-	return -ENOTTY;
-}
-
-static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev,
-					 fmode_t mode, unsigned int cmd,
-					 unsigned long arg)
-{
-	return -ENOTTY;
-}
-
-#endif /* CONFIG_BLK_DEV_ZONED */
-+		unsigned int nr_zones, report_zones_cb cb, void *data);
-+int blkdev_report_zones_cached(struct block_device *bdev, sector_t sector,
-+		unsigned int nr_zones, report_zones_cb cb, void *data);
-+int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op,
-+		sector_t sectors, sector_t nr_sectors);
-+int blk_revalidate_disk_zones(struct gendisk *disk);
- 
- /*
-  * Independent access ranges: struct blk_independent_access_range describes
-@@ -386,64 +477,55 @@ struct blk_independent_access_ranges {
- };
- 
- struct request_queue {
-	struct request		*last_merge;
-	struct elevator_queue	*elevator;
-
-	struct percpu_ref	q_usage_counter;
-+	/*
-+	 * The queue owner gets to use this for whatever they like.
-+	 * ll_rw_blk doesn't touch it.
-+	 */
-+	void			*queuedata;
- 
-	struct blk_queue_stats	*stats;
-	struct rq_qos		*rq_qos;
-+	struct elevator_queue	*elevator;
- 
- 	const struct blk_mq_ops	*mq_ops;
- 
- 	/* sw queues */
- 	struct blk_mq_ctx __percpu	*queue_ctx;
- 
-+	/*
-+	 * various queue flags, see QUEUE_* below
-+	 */
-+	unsigned long		queue_flags;
-+
-+	unsigned int __data_racy rq_timeout;
-+
- 	unsigned int		queue_depth;
- 
-+	refcount_t		refs;
-+
- 	/* hw dispatch queues */
-	struct xarray		hctx_table;
- 	unsigned int		nr_hw_queues;
-+	struct blk_mq_hw_ctx * __rcu *queue_hw_ctx;
- 
-	/*
-	 * The queue owner gets to use this for whatever they like.
-	 * ll_rw_blk doesn't touch it.
-	 */
-	void			*queuedata;
-+	struct percpu_ref	q_usage_counter;
-+	struct lock_class_key	io_lock_cls_key;
-+	struct lockdep_map	io_lockdep_map;
- 
-	/*
-	 * various queue flags, see QUEUE_* below
-	 */
-	unsigned long		queue_flags;
-	/*
-	 * Number of contexts that have called blk_set_pm_only(). If this
-	 * counter is above zero then only RQF_PM requests are processed.
-	 */
-	atomic_t		pm_only;
-+	struct lock_class_key	q_lock_cls_key;
-+	struct lockdep_map	q_lockdep_map;
- 
-	/*
-	 * ida allocated id for this queue.  Used to index queues from
-	 * ioctx.
-	 */
-	int			id;
-+	struct request		*last_merge;
- 
- 	spinlock_t		queue_lock;
- 
-	struct gendisk		*disk;
-+	int			quiesce_depth;
- 
-	/*
-	 * queue kobject
-	 */
-	struct kobject kobj;
-+	struct gendisk		*disk;
- 
- 	/*
- 	 * mq queue kobject
- 	 */
- 	struct kobject *mq_kobj;
- 
-#ifdef  CONFIG_BLK_DEV_INTEGRITY
-	struct blk_integrity integrity;
-#endif	/* CONFIG_BLK_DEV_INTEGRITY */
-+	struct queue_limits	limits;
- 
- #ifdef CONFIG_PM
- 	struct device		*dev;
-@@ -451,29 +533,31 @@ struct request_queue {
- #endif
- 
- 	/*
-	 * queue settings
-+	 * Number of contexts that have called blk_set_pm_only(). If this
-+	 * counter is above zero then only RQF_PM requests are processed.
- 	 */
-	unsigned long		nr_requests;	/* Max # of requests */
-+	atomic_t		pm_only;
-+
-+	struct blk_queue_stats	*stats;
-+	struct rq_qos		*rq_qos;
-+	struct mutex		rq_qos_mutex;
- 
-	unsigned int		dma_pad_mask;
- 	/*
-	 * Drivers that set dma_alignment to less than 511 must be prepared to
-	 * handle individual bvec's that are not a multiple of a SECTOR_SIZE
-	 * due to possible offsets.
-+	 * ida allocated id for this queue.  Used to index queues from
-+	 * ioctx.
- 	 */
-	unsigned int		dma_alignment;
-+	int			id;
-+
-+	/*
-+	 * queue settings
-+	 */
-+	unsigned long		nr_requests;	/* Max # of requests */
- 
- #ifdef CONFIG_BLK_INLINE_ENCRYPTION
- 	struct blk_crypto_profile *crypto_profile;
- 	struct kobject *crypto_kobject;
- #endif
- 
-	unsigned int		rq_timeout;
-	int			poll_nsec;
-
-	struct blk_stat_callback	*poll_cb;
-	struct blk_rq_stat	*poll_stat;
-
- 	struct timer_list	timeout;
- 	struct work_struct	timeout_work;
- 
-@@ -486,13 +570,15 @@ struct request_queue {
- 	DECLARE_BITMAP		(blkcg_pols, BLKCG_MAX_POLS);
- 	struct blkcg_gq		*root_blkg;
- 	struct list_head	blkg_list;
-+	struct mutex		blkcg_mutex;
- #endif
- 
-	struct queue_limits	limits;
-+	int			node;
- 
-	unsigned int		required_elevator_features;
-+	spinlock_t		requeue_lock;
-+	struct list_head	requeue_list;
-+	struct delayed_work	requeue_work;
- 
-	int			node;
- #ifdef CONFIG_BLK_DEV_IO_TRACE
- 	struct blk_trace __rcu	*blk_trace;
- #endif
-@@ -500,13 +586,25 @@ struct request_queue {
- 	 * for flush operations
- 	 */
- 	struct blk_flush_queue	*fq;
-+	struct list_head	flush_list;
- 
-	struct list_head	requeue_list;
-	spinlock_t		requeue_lock;
-	struct delayed_work	requeue_work;
-+	/*
-+	 * Protects against I/O scheduler switching, particularly when updating
-+	 * q->elevator. Since the elevator update code path may also modify q->
-+	 * nr_requests and wbt latency, this lock also protects the sysfs attrs
-+	 * nr_requests and wbt_lat_usec. Additionally the nr_hw_queues update
-+	 * may modify hctx tags, reserved-tags and cpumask, so this lock also
-+	 * helps protect the hctx sysfs/debugfs attrs. To ensure proper locking
-+	 * order during an elevator or nr_hw_queue update, first freeze the
-+	 * queue, then acquire ->elevator_lock.
-+	 */
-+	struct mutex		elevator_lock;
- 
- 	struct mutex		sysfs_lock;
-	struct mutex		sysfs_dir_lock;
-+	/*
-+	 * Protects queue limits and also sysfs attribute read_ahead_kb.
-+	 */
-+	struct mutex		limits_lock;
- 
- 	/*
- 	 * for reusing dead hctx instance in case of updating
-@@ -522,6 +620,16 @@ struct request_queue {
- 	struct throtl_data *td;
- #endif
- 	struct rcu_head		rcu_head;
-+#ifdef CONFIG_LOCKDEP
-+	struct task_struct	*mq_freeze_owner;
-+	int			mq_freeze_owner_depth;
-+	/*
-+	 * Records disk & queue state in current context, used in unfreeze
-+	 * queue
-+	 */
-+	bool			mq_freeze_disk_dead;
-+	bool			mq_freeze_queue_dying;
-+#endif
- 	wait_queue_head_t	mq_freeze_wq;
- 	/*
- 	 * Protect concurrent access to q_usage_counter by
-@@ -529,8 +637,6 @@ struct request_queue {
- 	 */
- 	struct mutex		mq_freeze_lock;
- 
-	int			quiesce_depth;
-
- 	struct blk_mq_tag_set	*tag_set;
- 	struct list_head	tag_set_list;
- 
-@@ -541,70 +647,46 @@ struct request_queue {
- 	 * Serializes all debugfs metadata operations using the above dentries.
- 	 */
- 	struct mutex		debugfs_mutex;
-
-	bool			mq_sysfs_init_done;
-
-	/**
-	 * @srcu: Sleepable RCU. Use as lock when type of the request queue
-	 * is blocking (BLK_MQ_F_BLOCKING). Must be the last member
-	 */
-	struct srcu_struct	srcu[];
- };
- 
- /* Keep blk_queue_flag_name[] in sync with the definitions below */
-#define QUEUE_FLAG_STOPPED	0	/* queue is stopped */
-#define QUEUE_FLAG_DYING	1	/* queue being torn down */
-#define QUEUE_FLAG_HAS_SRCU	2	/* SRCU is allocated */
-#define QUEUE_FLAG_NOMERGES     3	/* disable merge attempts */
-#define QUEUE_FLAG_SAME_COMP	4	/* complete on same CPU-group */
-#define QUEUE_FLAG_FAIL_IO	5	/* fake timeout */
-#define QUEUE_FLAG_NONROT	6	/* non-rotational device (SSD) */
-#define QUEUE_FLAG_VIRT		QUEUE_FLAG_NONROT /* paravirt device */
-#define QUEUE_FLAG_IO_STAT	7	/* do disk/partitions IO accounting */
-#define QUEUE_FLAG_NOXMERGES	9	/* No extended merges */
-#define QUEUE_FLAG_ADD_RANDOM	10	/* Contributes to random pool */
-#define QUEUE_FLAG_SAME_FORCE	12	/* force complete on same CPU */
-#define QUEUE_FLAG_INIT_DONE	14	/* queue is initialized */
-#define QUEUE_FLAG_STABLE_WRITES 15	/* don't modify blks until WB is done */
-#define QUEUE_FLAG_POLL		16	/* IO polling enabled if set */
-#define QUEUE_FLAG_WC		17	/* Write back caching */
-#define QUEUE_FLAG_FUA		18	/* device supports FUA writes */
-#define QUEUE_FLAG_DAX		19	/* device supports DAX */
-#define QUEUE_FLAG_STATS	20	/* track IO start and completion times */
-#define QUEUE_FLAG_REGISTERED	22	/* queue has been registered to a disk */
-#define QUEUE_FLAG_QUIESCED	24	/* queue has been quiesced */
-#define QUEUE_FLAG_PCI_P2PDMA	25	/* device supports PCI p2p requests */
-#define QUEUE_FLAG_ZONE_RESETALL 26	/* supports Zone Reset All */
-#define QUEUE_FLAG_RQ_ALLOC_TIME 27	/* record rq->alloc_time_ns */
-#define QUEUE_FLAG_HCTX_ACTIVE	28	/* at least one blk-mq hctx is active */
-#define QUEUE_FLAG_NOWAIT       29	/* device supports NOWAIT */
-#define QUEUE_FLAG_SQ_SCHED     30	/* single queue style io dispatch */
-
-#define QUEUE_FLAG_MQ_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
-				 (1 << QUEUE_FLAG_SAME_COMP) |		\
-				 (1 << QUEUE_FLAG_NOWAIT))
-+enum {
-+	QUEUE_FLAG_DYING,		/* queue being torn down */
-+	QUEUE_FLAG_NOMERGES,		/* disable merge attempts */
-+	QUEUE_FLAG_SAME_COMP,		/* complete on same CPU-group */
-+	QUEUE_FLAG_FAIL_IO,		/* fake timeout */
-+	QUEUE_FLAG_NOXMERGES,		/* No extended merges */
-+	QUEUE_FLAG_SAME_FORCE,		/* force complete on same CPU */
-+	QUEUE_FLAG_INIT_DONE,		/* queue is initialized */
-+	QUEUE_FLAG_STATS,		/* track IO start and completion times */
-+	QUEUE_FLAG_REGISTERED,		/* queue has been registered to a disk */
-+	QUEUE_FLAG_QUIESCED,		/* queue has been quiesced */
-+	QUEUE_FLAG_RQ_ALLOC_TIME,	/* record rq->alloc_time_ns */
-+	QUEUE_FLAG_HCTX_ACTIVE,		/* at least one blk-mq hctx is active */
-+	QUEUE_FLAG_SQ_SCHED,		/* single queue style io dispatch */
-+	QUEUE_FLAG_DISABLE_WBT_DEF,	/* for sched to disable/enable wbt */
-+	QUEUE_FLAG_NO_ELV_SWITCH,	/* can't switch elevator any more */
-+	QUEUE_FLAG_QOS_ENABLED,		/* qos is enabled */
-+	QUEUE_FLAG_BIO_ISSUE_TIME,	/* record bio->issue_time_ns */
-+	QUEUE_FLAG_MAX
-+};
-+
-+#define QUEUE_FLAG_MQ_DEFAULT	(1UL << QUEUE_FLAG_SAME_COMP)
- 
- void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
- void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
-bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
- 
-#define blk_queue_stopped(q)	test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
- #define blk_queue_dying(q)	test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
-#define blk_queue_has_srcu(q)	test_bit(QUEUE_FLAG_HAS_SRCU, &(q)->queue_flags)
- #define blk_queue_init_done(q)	test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
- #define blk_queue_nomerges(q)	test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
- #define blk_queue_noxmerges(q)	\
- 	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
-#define blk_queue_nonrot(q)	test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
-#define blk_queue_stable_writes(q) \
-	test_bit(QUEUE_FLAG_STABLE_WRITES, &(q)->queue_flags)
-#define blk_queue_io_stat(q)	test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
-#define blk_queue_add_random(q)	test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
-#define blk_queue_zone_resetall(q)	\
-	test_bit(QUEUE_FLAG_ZONE_RESETALL, &(q)->queue_flags)
-#define blk_queue_dax(q)	test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags)
-#define blk_queue_pci_p2pdma(q)	\
-	test_bit(QUEUE_FLAG_PCI_P2PDMA, &(q)->queue_flags)
-+#define blk_queue_nonrot(q)	(!((q)->limits.features & BLK_FEAT_ROTATIONAL))
-+#define blk_queue_io_stat(q)	((q)->limits.features & BLK_FEAT_IO_STAT)
-+#define blk_queue_passthrough_stat(q)	\
-+	((q)->limits.flags & BLK_FLAG_IOSTATS_PASSTHROUGH)
-+#define blk_queue_dax(q)	((q)->limits.features & BLK_FEAT_DAX)
-+#define blk_queue_pci_p2pdma(q)	((q)->limits.features & BLK_FEAT_PCI_P2PDMA)
- #ifdef CONFIG_BLK_RQ_ALLOC_TIME
- #define blk_queue_rq_alloc_time(q)	\
- 	test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags)
-@@ -618,8 +700,13 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
- #define blk_queue_quiesced(q)	test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
- #define blk_queue_pm_only(q)	atomic_read(&(q)->pm_only)
- #define blk_queue_registered(q)	test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
-#define blk_queue_nowait(q)	test_bit(QUEUE_FLAG_NOWAIT, &(q)->queue_flags)
- #define blk_queue_sq_sched(q)	test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags)
-+#define blk_queue_skip_tagset_quiesce(q) \
-+	((q)->limits.features & BLK_FEAT_SKIP_TAGSET_QUIESCE)
-+#define blk_queue_disable_wbt(q)	\
-+	test_bit(QUEUE_FLAG_DISABLE_WBT_DEF, &(q)->queue_flags)
-+#define blk_queue_no_elv_switch(q)	\
-+	test_bit(QUEUE_FLAG_NO_ELV_SWITCH, &(q)->queue_flags)
- 
- extern void blk_set_pm_only(struct request_queue *q);
- extern void blk_clear_pm_only(struct request_queue *q);
-@@ -647,29 +734,10 @@ static inline enum rpm_status queue_rpm_status(struct request_queue *q)
- }
- #endif
- 
-static inline enum blk_zoned_model
-blk_queue_zoned_model(struct request_queue *q)
-{
-	if (IS_ENABLED(CONFIG_BLK_DEV_ZONED))
-		return q->limits.zoned;
-	return BLK_ZONED_NONE;
-}
-
- static inline bool blk_queue_is_zoned(struct request_queue *q)
- {
-	switch (blk_queue_zoned_model(q)) {
-	case BLK_ZONED_HA:
-	case BLK_ZONED_HM:
-		return true;
-	default:
-		return false;
-	}
-}
-
-#ifdef CONFIG_BLK_DEV_ZONED
-static inline unsigned int disk_nr_zones(struct gendisk *disk)
-{
-	return blk_queue_is_zoned(disk->queue) ? disk->nr_zones : 0;
-+	return IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
-+		(q->limits.features & BLK_FEAT_ZONED);
- }
- 
- static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector)
-@@ -679,61 +747,16 @@ static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector)
- 	return sector >> ilog2(disk->queue->limits.chunk_sectors);
- }
- 
-static inline bool disk_zone_is_seq(struct gendisk *disk, sector_t sector)
-{
-	if (!blk_queue_is_zoned(disk->queue))
-		return false;
-	if (!disk->conv_zones_bitmap)
-		return true;
-	return !test_bit(disk_zone_no(disk, sector), disk->conv_zones_bitmap);
-}
-
-static inline void disk_set_max_open_zones(struct gendisk *disk,
-		unsigned int max_open_zones)
-{
-	disk->max_open_zones = max_open_zones;
-}
-
-static inline void disk_set_max_active_zones(struct gendisk *disk,
-		unsigned int max_active_zones)
-{
-	disk->max_active_zones = max_active_zones;
-}
-
- static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
- {
-	return bdev->bd_disk->max_open_zones;
-+	return bdev->bd_disk->queue->limits.max_open_zones;
- }
- 
- static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
- {
-	return bdev->bd_disk->max_active_zones;
-+	return bdev->bd_disk->queue->limits.max_active_zones;
- }
- 
-#else /* CONFIG_BLK_DEV_ZONED */
-static inline unsigned int disk_nr_zones(struct gendisk *disk)
-{
-	return 0;
-}
-static inline bool disk_zone_is_seq(struct gendisk *disk, sector_t sector)
-{
-	return false;
-}
-static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector)
-{
-	return 0;
-}
-static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
-{
-	return 0;
-}
-
-static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
-{
-	return 0;
-}
-#endif /* CONFIG_BLK_DEV_ZONED */
-
- static inline unsigned int blk_queue_depth(struct request_queue *q)
- {
- 	if (q->queue_depth)
-@@ -752,6 +775,9 @@ static inline unsigned int blk_queue_depth(struct request_queue *q)
- #define for_each_bio(_bio)		\
- 	for (; _bio; _bio = _bio->bi_next)
- 
-+int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk,
-+				 const struct attribute_group **groups,
-+				 struct fwnode_handle *fwnode);
- int __must_check device_add_disk(struct device *parent, struct gendisk *disk,
- 				 const struct attribute_group **groups);
- static inline int __must_check add_disk(struct gendisk *disk)
-@@ -763,19 +789,40 @@ void invalidate_disk(struct gendisk *disk);
- void set_disk_ro(struct gendisk *disk, bool read_only);
- void disk_uevent(struct gendisk *disk, enum kobject_action action);
- 
-static inline int get_disk_ro(struct gendisk *disk)
-+static inline u8 bdev_partno(const struct block_device *bdev)
-+{
-+	return atomic_read(&bdev->__bd_flags) & BD_PARTNO;
-+}
-+
-+static inline bool bdev_test_flag(const struct block_device *bdev, unsigned flag)
-+{
-+	return atomic_read(&bdev->__bd_flags) & flag;
-+}
-+
-+static inline void bdev_set_flag(struct block_device *bdev, unsigned flag)
-+{
-+	atomic_or(flag, &bdev->__bd_flags);
-+}
-+
-+static inline void bdev_clear_flag(struct block_device *bdev, unsigned flag)
-+{
-+	atomic_andnot(flag, &bdev->__bd_flags);
-+}
-+
-+static inline bool get_disk_ro(struct gendisk *disk)
- {
-	return disk->part0->bd_read_only ||
-+	return bdev_test_flag(disk->part0, BD_READ_ONLY) ||
- 		test_bit(GD_READ_ONLY, &disk->state);
- }
- 
-static inline int bdev_read_only(struct block_device *bdev)
-+static inline bool bdev_read_only(struct block_device *bdev)
- {
-	return bdev->bd_read_only || get_disk_ro(bdev->bd_disk);
-+	return bdev_test_flag(bdev, BD_READ_ONLY) || get_disk_ro(bdev->bd_disk);
- }
- 
- bool set_capacity_and_notify(struct gendisk *disk, sector_t size);
-bool disk_force_media_change(struct gendisk *disk, unsigned int events);
-+void disk_force_media_change(struct gendisk *disk);
-+void bdev_mark_dead(struct block_device *bdev, bool surprise);
- 
- void add_disk_randomness(struct gendisk *disk) __latent_entropy;
- void rand_initialize_disk(struct gendisk *disk);
-@@ -806,25 +853,137 @@ static inline u64 sb_bdev_nr_blocks(struct super_block *sb)
- 		(sb->s_blocksize_bits - SECTOR_SHIFT);
- }
- 
-+#ifdef CONFIG_BLK_DEV_ZONED
-+static inline unsigned int disk_nr_zones(struct gendisk *disk)
-+{
-+	return disk->nr_zones;
-+}
-+
-+/**
-+ * bio_needs_zone_write_plugging - Check if a BIO needs to be handled with zone
-+ *				   write plugging
-+ * @bio: The BIO being submitted
-+ *
-+ * Return true whenever @bio execution needs to be handled through zone
-+ * write plugging (using blk_zone_plug_bio()). Return false otherwise.
-+ */
-+static inline bool bio_needs_zone_write_plugging(struct bio *bio)
-+{
-+	enum req_op op = bio_op(bio);
-+
-+	/*
-+	 * Only zoned block devices have a zone write plug hash table. But not
-+	 * all of them have one (e.g. DM devices may not need one).
-+	 */
-+	if (!bio->bi_bdev->bd_disk->zone_wplugs_hash)
-+		return false;
-+
-+	/* Only write operations need zone write plugging. */
-+	if (!op_is_write(op))
-+		return false;
-+
-+	/* Ignore empty flush */
-+	if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
-+		return false;
-+
-+	/* Ignore BIOs that already have been handled by zone write plugging. */
-+	if (bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING))
-+		return false;
-+
-+	/*
-+	 * All zone write operations must be handled through zone write plugging
-+	 * using blk_zone_plug_bio().
-+	 */
-+	switch (op) {
-+	case REQ_OP_ZONE_APPEND:
-+	case REQ_OP_WRITE:
-+	case REQ_OP_WRITE_ZEROES:
-+	case REQ_OP_ZONE_FINISH:
-+	case REQ_OP_ZONE_RESET:
-+	case REQ_OP_ZONE_RESET_ALL:
-+		return true;
-+	default:
-+		return false;
-+	}
-+}
-+
-+bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs);
-+
-+/**
-+ * disk_zone_capacity - returns the zone capacity of zone containing @sector
-+ * @disk:	disk to work with
-+ * @sector:	sector number within the querying zone
-+ *
-+ * Returns the zone capacity of a zone containing @sector. @sector can be any
-+ * sector in the zone.
-+ */
-+static inline unsigned int disk_zone_capacity(struct gendisk *disk,
-+					      sector_t sector)
-+{
-+	sector_t zone_sectors = disk->queue->limits.chunk_sectors;
-+
-+	if (sector + zone_sectors >= get_capacity(disk))
-+		return disk->last_zone_capacity;
-+	return disk->zone_capacity;
-+}
-+static inline unsigned int bdev_zone_capacity(struct block_device *bdev,
-+					      sector_t pos)
-+{
-+	return disk_zone_capacity(bdev->bd_disk, pos);
-+}
-+
-+bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector);
-+
-+#else /* CONFIG_BLK_DEV_ZONED */
-+static inline unsigned int disk_nr_zones(struct gendisk *disk)
-+{
-+	return 0;
-+}
-+
-+static inline bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector)
-+{
-+	return false;
-+}
-+
-+static inline bool bio_needs_zone_write_plugging(struct bio *bio)
-+{
-+	return false;
-+}
-+
-+static inline bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs)
-+{
-+	return false;
-+}
-+#endif /* CONFIG_BLK_DEV_ZONED */
-+
-+static inline unsigned int bdev_nr_zones(struct block_device *bdev)
-+{
-+	return disk_nr_zones(bdev->bd_disk);
-+}
-+
- int bdev_disk_changed(struct gendisk *disk, bool invalidate);
- 
- void put_disk(struct gendisk *disk);
-struct gendisk *__blk_alloc_disk(int node, struct lock_class_key *lkclass);
-+struct gendisk *__blk_alloc_disk(struct queue_limits *lim, int node,
-+		struct lock_class_key *lkclass);
- 
- /**
-  * blk_alloc_disk - allocate a gendisk structure
-+ * @lim: queue limits to be used for this disk.
-  * @node_id: numa node to allocate on
-  *
-  * Allocate and pre-initialize a gendisk structure for use with BIO based
-  * drivers.
-  *
-+ * Returns an ERR_PTR on error, else the allocated disk.
-+ *
-  * Context: can sleep
-  */
-#define blk_alloc_disk(node_id)						\
-+#define blk_alloc_disk(lim, node_id)					\
- ({									\
- 	static struct lock_class_key __key;				\
- 									\
-	__blk_alloc_disk(node_id, &__key);				\
-+	__blk_alloc_disk(lim, node_id, &__key);				\
- })
- 
- int __register_blkdev(unsigned int major, const char *name,
-@@ -833,14 +992,12 @@ int __register_blkdev(unsigned int major, const char *name,
- 	__register_blkdev(major, name, NULL)
- void unregister_blkdev(unsigned int major, const char *name);
- 
-bool bdev_check_media_change(struct block_device *bdev);
-int __invalidate_device(struct block_device *bdev, bool kill_dirty);
-+bool disk_check_media_change(struct gendisk *disk);
- void set_capacity(struct gendisk *disk, sector_t size);
- 
- #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
- int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk);
- void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk);
-int bd_register_pending_holders(struct gendisk *disk);
- #else
- static inline int bd_link_disk_holder(struct block_device *bdev,
- 				      struct gendisk *disk)
-@@ -851,21 +1008,18 @@ static inline void bd_unlink_disk_holder(struct block_device *bdev,
- 					 struct gendisk *disk)
- {
- }
-static inline int bd_register_pending_holders(struct gendisk *disk)
-{
-	return 0;
-}
- #endif /* CONFIG_BLOCK_HOLDER_DEPRECATED */
- 
- dev_t part_devt(struct gendisk *disk, u8 partno);
- void inc_diskseq(struct gendisk *disk);
-dev_t blk_lookup_devt(const char *name, int partno);
- void blk_request_module(dev_t devt);
- 
- extern int blk_register_queue(struct gendisk *disk);
- extern void blk_unregister_queue(struct gendisk *disk);
- void submit_bio_noacct(struct bio *bio);
- struct bio *bio_split_to_limits(struct bio *bio);
-+struct bio *bio_submit_split_bioset(struct bio *bio, unsigned int split_sectors,
-+				    struct bio_set *bs);
- 
- extern int blk_lld_busy(struct request_queue *q);
- extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
-@@ -877,11 +1031,10 @@ extern const char *blk_op_str(enum req_op op);
- 
- int blk_status_to_errno(blk_status_t status);
- blk_status_t errno_to_blk_status(int errno);
-+const char *blk_status_to_str(blk_status_t status);
- 
- /* only poll the hardware once, don't continue until a completion was found */
- #define BLK_POLL_ONESHOT		(1 << 0)
-/* do not sleep to wait for the expected completion time */
-#define BLK_POLL_NOSLEEP		(1 << 1)
- int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags);
- int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
- 			unsigned int flags);
-@@ -899,87 +1052,111 @@ static inline unsigned int bio_zone_no(struct bio *bio)
- 	return disk_zone_no(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector);
- }
- 
-static inline unsigned int bio_zone_is_seq(struct bio *bio)
-+static inline bool bio_straddles_zones(struct bio *bio)
- {
-	return disk_zone_is_seq(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector);
-+	return bio_sectors(bio) &&
-+		bio_zone_no(bio) !=
-+		disk_zone_no(bio->bi_bdev->bd_disk, bio_end_sector(bio) - 1);
- }
- 
- /*
- * Return how much of the chunk is left to be used for I/O at a given offset.
-+ * Return how much within the boundary is left to be used for I/O at a given
-+ * offset.
-  */
-static inline unsigned int blk_chunk_sectors_left(sector_t offset,
-		unsigned int chunk_sectors)
-+static inline unsigned int blk_boundary_sectors_left(sector_t offset,
-+		unsigned int boundary_sectors)
- {
-	if (unlikely(!is_power_of_2(chunk_sectors)))
-		return chunk_sectors - sector_div(offset, chunk_sectors);
-	return chunk_sectors - (offset & (chunk_sectors - 1));
-+	if (unlikely(!is_power_of_2(boundary_sectors)))
-+		return boundary_sectors - sector_div(offset, boundary_sectors);
-+	return boundary_sectors - (offset & (boundary_sectors - 1));
-+}
-+
-+/**
-+ * queue_limits_start_update - start an atomic update of queue limits
-+ * @q:		queue to update
-+ *
-+ * This functions starts an atomic update of the queue limits.  It takes a lock
-+ * to prevent other updates and returns a snapshot of the current limits that
-+ * the caller can modify.  The caller must call queue_limits_commit_update()
-+ * to finish the update.
-+ *
-+ * Context: process context.
-+ */
-+static inline struct queue_limits
-+queue_limits_start_update(struct request_queue *q)
-+{
-+	mutex_lock(&q->limits_lock);
-+	return q->limits;
-+}
-+int queue_limits_commit_update_frozen(struct request_queue *q,
-+		struct queue_limits *lim);
-+int queue_limits_commit_update(struct request_queue *q,
-+		struct queue_limits *lim);
-+int queue_limits_set(struct request_queue *q, struct queue_limits *lim);
-+int blk_validate_limits(struct queue_limits *lim);
-+
-+/**
-+ * queue_limits_cancel_update - cancel an atomic update of queue limits
-+ * @q:		queue to update
-+ *
-+ * This functions cancels an atomic update of the queue limits started by
-+ * queue_limits_start_update() and should be used when an error occurs after
-+ * starting update.
-+ */
-+static inline void queue_limits_cancel_update(struct request_queue *q)
-+{
-+	mutex_unlock(&q->limits_lock);
-+}
-+
-+/*
-+ * These helpers are for drivers that have sloppy feature negotiation and might
-+ * have to disable DISCARD, WRITE_ZEROES or SECURE_DISCARD from the I/O
-+ * completion handler when the device returned an indicator that the respective
-+ * feature is not actually supported.  They are racy and the driver needs to
-+ * cope with that.  Try to avoid this scheme if you can.
-+ */
-+static inline void blk_queue_disable_discard(struct request_queue *q)
-+{
-+	q->limits.max_discard_sectors = 0;
-+}
-+
-+static inline void blk_queue_disable_secure_erase(struct request_queue *q)
-+{
-+	q->limits.max_secure_erase_sectors = 0;
-+}
-+
-+static inline void blk_queue_disable_write_zeroes(struct request_queue *q)
-+{
-+	q->limits.max_write_zeroes_sectors = 0;
-+	q->limits.max_wzeroes_unmap_sectors = 0;
- }
- 
- /*
-  * Access functions for manipulating queue properties
-  */
-void blk_queue_bounce_limit(struct request_queue *q, enum blk_bounce limit);
-extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
-extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
-extern void blk_queue_max_segments(struct request_queue *, unsigned short);
-extern void blk_queue_max_discard_segments(struct request_queue *,
-		unsigned short);
-void blk_queue_max_secure_erase_sectors(struct request_queue *q,
-		unsigned int max_sectors);
-extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
-extern void blk_queue_max_discard_sectors(struct request_queue *q,
-		unsigned int max_discard_sectors);
-extern void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
-		unsigned int max_write_same_sectors);
-extern void blk_queue_logical_block_size(struct request_queue *, unsigned int);
-extern void blk_queue_max_zone_append_sectors(struct request_queue *q,
-		unsigned int max_zone_append_sectors);
-extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
-void blk_queue_zone_write_granularity(struct request_queue *q,
-				      unsigned int size);
-extern void blk_queue_alignment_offset(struct request_queue *q,
-				       unsigned int alignment);
-void disk_update_readahead(struct gendisk *disk);
-extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
-extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
-extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
-extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
- extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
-extern void blk_set_default_limits(struct queue_limits *lim);
- extern void blk_set_stacking_limits(struct queue_limits *lim);
- extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
- 			    sector_t offset);
-extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
-			      sector_t offset);
-extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
-extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
-extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
-extern void blk_queue_dma_alignment(struct request_queue *, int);
-extern void blk_queue_update_dma_alignment(struct request_queue *, int);
-+void queue_limits_stack_bdev(struct queue_limits *t, struct block_device *bdev,
-+		sector_t offset, const char *pfx);
- extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
-extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
- 
- struct blk_independent_access_ranges *
- disk_alloc_independent_access_ranges(struct gendisk *disk, int nr_ia_ranges);
- void disk_set_independent_access_ranges(struct gendisk *disk,
- 				struct blk_independent_access_ranges *iars);
- 
-/*
- * Elevator features for blk_queue_required_elevator_features:
- */
-/* Supports zoned block devices sequential write constraint */
-#define ELEVATOR_F_ZBD_SEQ_WRITE	(1U << 0)
-
-extern void blk_queue_required_elevator_features(struct request_queue *q,
-						 unsigned int features);
-extern bool blk_queue_can_use_dma_map_merging(struct request_queue *q,
-					      struct device *dev);
-
- bool __must_check blk_get_queue(struct request_queue *);
- extern void blk_put_queue(struct request_queue *);
- 
- void blk_mark_disk_dead(struct gendisk *disk);
- 
-+struct rq_list {
-+	struct request *head;
-+	struct request *tail;
-+};
-+
- #ifdef CONFIG_BLOCK
- /*
-  * blk_plug permits building a queue of related requests by holding the I/O
-@@ -993,17 +1170,17 @@ void blk_mark_disk_dead(struct gendisk *disk);
-  * blk_flush_plug() is called.
-  */
- struct blk_plug {
-	struct request *mq_list; /* blk-mq requests */
-+	struct rq_list mq_list; /* blk-mq requests */
- 
- 	/* if ios_left is > 1, we can batch tag/rq allocations */
-	struct request *cached_rq;
-+	struct rq_list cached_rqs;
-+	u64 cur_ktime;
- 	unsigned short nr_ios;
- 
- 	unsigned short rq_count;
- 
- 	bool multiple_queues;
- 	bool has_elevator;
-	bool nowait;
- 
- 	struct list_head cb_list; /* md requires an unplug callback */
- };
-@@ -1028,6 +1205,18 @@ static inline void blk_flush_plug(struct blk_plug *plug, bool async)
- 		__blk_flush_plug(plug, async);
- }
- 
-+/*
-+ * tsk == current here
-+ */
-+static inline void blk_plug_invalidate_ts(struct task_struct *tsk)
-+{
-+	struct blk_plug *plug = tsk->plug;
-+
-+	if (plug)
-+		plug->cur_ktime = 0;
-+	current->flags &= ~PF_BLOCK_TS;
-+}
-+
- int blkdev_issue_flush(struct block_device *bdev);
- long nr_blockdev_pages(void);
- #else /* CONFIG_BLOCK */
-@@ -1051,6 +1240,10 @@ static inline void blk_flush_plug(struct blk_plug *plug, bool async)
- {
- }
- 
-+static inline void blk_plug_invalidate_ts(struct task_struct *tsk)
-+{
-+}
-+
- static inline int blkdev_issue_flush(struct block_device *bdev)
- {
- 	return 0;
-@@ -1073,6 +1266,7 @@ int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,
- 
- #define BLKDEV_ZERO_NOUNMAP	(1 << 0)  /* do not free blocks */
- #define BLKDEV_ZERO_NOFALLBACK	(1 << 1)  /* don't write explicit zeroes */
-+#define BLKDEV_ZERO_KILLABLE	(1 << 2)  /* interruptible by fatal signals */
- 
- extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
- 		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
-@@ -1103,17 +1297,21 @@ static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
- 
- static inline bool bdev_is_partition(struct block_device *bdev)
- {
-	return bdev->bd_partno;
-+	return bdev_partno(bdev) != 0;
- }
- 
- enum blk_default_limits {
- 	BLK_MAX_SEGMENTS	= 128,
- 	BLK_SAFE_MAX_SECTORS	= 255,
-	BLK_DEF_MAX_SECTORS	= 2560,
- 	BLK_MAX_SEGMENT_SIZE	= 65536,
- 	BLK_SEG_BOUNDARY_MASK	= 0xFFFFFFFFUL,
- };
- 
-+static inline struct queue_limits *bdev_limits(struct block_device *bdev)
-+{
-+	return &bdev_get_queue(bdev)->limits;
-+}
-+
- static inline unsigned long queue_segment_boundary(const struct request_queue *q)
- {
- 	return q->limits.seg_boundary_mask;
-@@ -1154,18 +1352,20 @@ static inline unsigned int queue_max_segment_size(const struct request_queue *q)
- 	return q->limits.max_segment_size;
- }
- 
-static inline unsigned int queue_max_zone_append_sectors(const struct request_queue *q)
-+static inline bool queue_emulates_zone_append(struct request_queue *q)
- {
-+	return blk_queue_is_zoned(q) && !q->limits.max_hw_zone_append_sectors;
-+}
- 
-	const struct queue_limits *l = &q->limits;
-
-	return min(l->max_zone_append_sectors, l->max_sectors);
-+static inline bool bdev_emulates_zone_append(struct block_device *bdev)
-+{
-+	return queue_emulates_zone_append(bdev_get_queue(bdev));
- }
- 
- static inline unsigned int
- bdev_max_zone_append_sectors(struct block_device *bdev)
- {
-	return queue_max_zone_append_sectors(bdev_get_queue(bdev));
-+	return bdev_limits(bdev)->max_zone_append_sectors;
- }
- 
- static inline unsigned int bdev_max_segments(struct block_device *bdev)
-@@ -1173,14 +1373,16 @@ static inline unsigned int bdev_max_segments(struct block_device *bdev)
- 	return queue_max_segments(bdev_get_queue(bdev));
- }
- 
-static inline unsigned queue_logical_block_size(const struct request_queue *q)
-+static inline unsigned short bdev_max_write_streams(struct block_device *bdev)
- {
-	int retval = 512;
-
-	if (q && q->limits.logical_block_size)
-		retval = q->limits.logical_block_size;
-+	if (bdev_is_partition(bdev))
-+		return 0;
-+	return bdev_limits(bdev)->max_write_streams;
-+}
- 
-	return retval;
-+static inline unsigned queue_logical_block_size(const struct request_queue *q)
-+{
-+	return q->limits.logical_block_size;
- }
- 
- static inline unsigned int bdev_logical_block_size(struct block_device *bdev)
-@@ -1203,7 +1405,7 @@ static inline unsigned int queue_io_min(const struct request_queue *q)
- 	return q->limits.io_min;
- }
- 
-static inline int bdev_io_min(struct block_device *bdev)
-+static inline unsigned int bdev_io_min(struct block_device *bdev)
- {
- 	return queue_io_min(bdev_get_queue(bdev));
- }
-@@ -1213,7 +1415,7 @@ static inline unsigned int queue_io_opt(const struct request_queue *q)
- 	return q->limits.io_opt;
- }
- 
-static inline int bdev_io_opt(struct block_device *bdev)
-+static inline unsigned int bdev_io_opt(struct block_device *bdev)
- {
- 	return queue_io_opt(bdev_get_queue(bdev));
- }
-@@ -1235,28 +1437,29 @@ unsigned int bdev_discard_alignment(struct block_device *bdev);
- 
- static inline unsigned int bdev_max_discard_sectors(struct block_device *bdev)
- {
-	return bdev_get_queue(bdev)->limits.max_discard_sectors;
-+	return bdev_limits(bdev)->max_discard_sectors;
- }
- 
- static inline unsigned int bdev_discard_granularity(struct block_device *bdev)
- {
-	return bdev_get_queue(bdev)->limits.discard_granularity;
-+	return bdev_limits(bdev)->discard_granularity;
- }
- 
- static inline unsigned int
- bdev_max_secure_erase_sectors(struct block_device *bdev)
- {
-	return bdev_get_queue(bdev)->limits.max_secure_erase_sectors;
-+	return bdev_limits(bdev)->max_secure_erase_sectors;
- }
- 
- static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
- {
-	struct request_queue *q = bdev_get_queue(bdev);
-
-	if (q)
-		return q->limits.max_write_zeroes_sectors;
-+	return bdev_limits(bdev)->max_write_zeroes_sectors;
-+}
- 
-	return 0;
-+static inline unsigned int
-+bdev_write_zeroes_unmap_sectors(struct block_device *bdev)
-+{
-+	return bdev_limits(bdev)->max_wzeroes_unmap_sectors;
- }
- 
- static inline bool bdev_nonrot(struct block_device *bdev)
-@@ -1264,40 +1467,50 @@ static inline bool bdev_nonrot(struct block_device *bdev)
- 	return blk_queue_nonrot(bdev_get_queue(bdev));
- }
- 
-+static inline bool bdev_synchronous(struct block_device *bdev)
-+{
-+	return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS;
-+}
-+
- static inline bool bdev_stable_writes(struct block_device *bdev)
- {
-	return test_bit(QUEUE_FLAG_STABLE_WRITES,
-			&bdev_get_queue(bdev)->queue_flags);
-+	struct request_queue *q = bdev_get_queue(bdev);
-+
-+	if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
-+	    q->limits.integrity.csum_type != BLK_INTEGRITY_CSUM_NONE)
-+		return true;
-+	return q->limits.features & BLK_FEAT_STABLE_WRITES;
-+}
-+
-+static inline bool blk_queue_write_cache(struct request_queue *q)
-+{
-+	return (q->limits.features & BLK_FEAT_WRITE_CACHE) &&
-+		!(q->limits.flags & BLK_FLAG_WRITE_CACHE_DISABLED);
- }
- 
- static inline bool bdev_write_cache(struct block_device *bdev)
- {
-	return test_bit(QUEUE_FLAG_WC, &bdev_get_queue(bdev)->queue_flags);
-+	return blk_queue_write_cache(bdev_get_queue(bdev));
- }
- 
- static inline bool bdev_fua(struct block_device *bdev)
- {
-	return test_bit(QUEUE_FLAG_FUA, &bdev_get_queue(bdev)->queue_flags);
-+	return bdev_limits(bdev)->features & BLK_FEAT_FUA;
- }
- 
-static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev)
-+static inline bool bdev_nowait(struct block_device *bdev)
- {
-	struct request_queue *q = bdev_get_queue(bdev);
-
-	if (q)
-		return blk_queue_zoned_model(q);
-
-	return BLK_ZONED_NONE;
-+	return bdev->bd_disk->queue->limits.features & BLK_FEAT_NOWAIT;
- }
- 
- static inline bool bdev_is_zoned(struct block_device *bdev)
- {
-	struct request_queue *q = bdev_get_queue(bdev);
-
-	if (q)
-		return blk_queue_is_zoned(q);
-+	return blk_queue_is_zoned(bdev_get_queue(bdev));
-+}
- 
-	return false;
-+static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec)
-+{
-+	return disk_zone_no(bdev->bd_disk, sec);
- }
- 
- static inline sector_t bdev_zone_sectors(struct block_device *bdev)
-@@ -1309,9 +1522,67 @@ static inline sector_t bdev_zone_sectors(struct block_device *bdev)
- 	return q->limits.chunk_sectors;
- }
- 
-static inline int queue_dma_alignment(const struct request_queue *q)
-+static inline sector_t bdev_zone_start(struct block_device *bdev,
-+				       sector_t sector)
- {
-	return q ? q->dma_alignment : 511;
-+	return sector & ~(bdev_zone_sectors(bdev) - 1);
-+}
-+
-+static inline sector_t bdev_offset_from_zone_start(struct block_device *bdev,
-+						   sector_t sector)
-+{
-+	return sector & (bdev_zone_sectors(bdev) - 1);
-+}
-+
-+static inline sector_t bio_offset_from_zone_start(struct bio *bio)
-+{
-+	return bdev_offset_from_zone_start(bio->bi_bdev,
-+					   bio->bi_iter.bi_sector);
-+}
-+
-+static inline bool bdev_is_zone_start(struct block_device *bdev,
-+				      sector_t sector)
-+{
-+	return bdev_offset_from_zone_start(bdev, sector) == 0;
-+}
-+
-+/* Check whether @sector is a multiple of the zone size. */
-+static inline bool bdev_is_zone_aligned(struct block_device *bdev,
-+					sector_t sector)
-+{
-+	return bdev_is_zone_start(bdev, sector);
-+}
-+
-+int blk_zone_issue_zeroout(struct block_device *bdev, sector_t sector,
-+			   sector_t nr_sects, gfp_t gfp_mask);
-+
-+static inline unsigned int queue_dma_alignment(const struct request_queue *q)
-+{
-+	return q->limits.dma_alignment;
-+}
-+
-+static inline unsigned int
-+queue_atomic_write_unit_max_bytes(const struct request_queue *q)
-+{
-+	return q->limits.atomic_write_unit_max;
-+}
-+
-+static inline unsigned int
-+queue_atomic_write_unit_min_bytes(const struct request_queue *q)
-+{
-+	return q->limits.atomic_write_unit_min;
-+}
-+
-+static inline unsigned int
-+queue_atomic_write_boundary_bytes(const struct request_queue *q)
-+{
-+	return q->limits.atomic_write_boundary_sectors << SECTOR_SHIFT;
-+}
-+
-+static inline unsigned int
-+queue_atomic_write_max_bytes(const struct request_queue *q)
-+{
-+	return q->limits.atomic_write_max_sectors << SECTOR_SHIFT;
- }
- 
- static inline unsigned int bdev_dma_alignment(struct block_device *bdev)
-@@ -1319,34 +1590,24 @@ static inline unsigned int bdev_dma_alignment(struct block_device *bdev)
- 	return queue_dma_alignment(bdev_get_queue(bdev));
- }
- 
-static inline bool bdev_iter_is_aligned(struct block_device *bdev,
-					struct iov_iter *iter)
-+static inline unsigned int
-+blk_lim_dma_alignment_and_pad(struct queue_limits *lim)
- {
-	return iov_iter_is_aligned(iter, bdev_dma_alignment(bdev),
-				   bdev_logical_block_size(bdev) - 1);
-+	return lim->dma_alignment | lim->dma_pad_mask;
- }
- 
-static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
-+static inline bool blk_rq_aligned(struct request_queue *q, unsigned long addr,
- 				 unsigned int len)
- {
-	unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
-+	unsigned int alignment = blk_lim_dma_alignment_and_pad(&q->limits);
-+
- 	return !(addr & alignment) && !(len & alignment);
- }
- 
- /* assumes size > 256 */
- static inline unsigned int blksize_bits(unsigned int size)
- {
-	unsigned int bits = 8;
-	do {
-		bits++;
-		size >>= 1;
-	} while (size > 256);
-	return bits;
-}
-
-static inline unsigned int block_size(struct block_device *bdev)
-{
-	return 1 << bdev->bd_inode->i_blkbits;
-+	return order_base_2(size >> SECTOR_SHIFT) + SECTOR_SHIFT;
- }
- 
- int kblockd_schedule_work(struct work_struct *work);
-@@ -1379,27 +1640,27 @@ enum blk_unique_id {
- 	BLK_UID_NAA	= 3,
- };
- 
-#define NFL4_UFLG_MASK			0x0000003F
-
- struct block_device_operations {
- 	void (*submit_bio)(struct bio *bio);
- 	int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob,
- 			unsigned int flags);
-	int (*open) (struct block_device *, fmode_t);
-	void (*release) (struct gendisk *, fmode_t);
-	int (*rw_page)(struct block_device *, sector_t, struct page *, enum req_op);
-	int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
-	int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
-+	int (*open)(struct gendisk *disk, blk_mode_t mode);
-+	void (*release)(struct gendisk *disk);
-+	int (*ioctl)(struct block_device *bdev, blk_mode_t mode,
-+			unsigned cmd, unsigned long arg);
-+	int (*compat_ioctl)(struct block_device *bdev, blk_mode_t mode,
-+			unsigned cmd, unsigned long arg);
- 	unsigned int (*check_events) (struct gendisk *disk,
- 				      unsigned int clearing);
- 	void (*unlock_native_capacity) (struct gendisk *);
-	int (*getgeo)(struct block_device *, struct hd_geometry *);
-+	int (*getgeo)(struct gendisk *, struct hd_geometry *);
- 	int (*set_read_only)(struct block_device *bdev, bool ro);
- 	void (*free_disk)(struct gendisk *disk);
- 	/* this callback is with swap_lock and sometimes page table lock held */
- 	void (*swap_slot_free_notify) (struct block_device *, unsigned long);
- 	int (*report_zones)(struct gendisk *, sector_t sector,
-			unsigned int nr_zones, report_zones_cb cb, void *data);
-+			    unsigned int nr_zones,
-+			    struct blk_report_zones_args *args);
- 	char *(*devnode)(struct gendisk *disk, umode_t *mode);
- 	/* returns the length of the identifier or a negative errno: */
- 	int (*get_unique_id)(struct gendisk *disk, u8 id[16],
-@@ -1416,16 +1677,12 @@ struct block_device_operations {
- };
- 
- #ifdef CONFIG_COMPAT
-extern int blkdev_compat_ptr_ioctl(struct block_device *, fmode_t,
-+extern int blkdev_compat_ptr_ioctl(struct block_device *, blk_mode_t,
- 				      unsigned int, unsigned long);
- #else
- #define blkdev_compat_ptr_ioctl NULL
- #endif
- 
-extern int bdev_read_page(struct block_device *, sector_t, struct page *);
-extern int bdev_write_page(struct block_device *, sector_t, struct page *,
-						struct writeback_control *);
-
- static inline void blk_wake_io_task(struct task_struct *waiter)
- {
- 	/*
-@@ -1439,13 +1696,11 @@ static inline void blk_wake_io_task(struct task_struct *waiter)
- 		wake_up_process(waiter);
- }
- 
-unsigned long bdev_start_io_acct(struct block_device *bdev,
-				 unsigned int sectors, enum req_op op,
-+unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op,
- 				 unsigned long start_time);
- void bdev_end_io_acct(struct block_device *bdev, enum req_op op,
-		unsigned long start_time);
-+		      unsigned int sectors, unsigned long start_time);
- 
-void bio_start_io_acct_time(struct bio *bio, unsigned long start_time);
- unsigned long bio_start_io_acct(struct bio *bio);
- void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
- 		struct block_device *orig_bdev);
-@@ -1460,8 +1715,8 @@ static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time)
- 	return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev);
- }
- 
-int bdev_read_only(struct block_device *bdev);
-int set_blocksize(struct block_device *bdev, int size);
-+int bdev_validate_blocksize(struct block_device *bdev, int block_size);
-+int set_blocksize(struct file *file, int size);
- 
- int lookup_bdev(const char *pathname, dev_t *dev);
- 
-@@ -1475,22 +1730,52 @@ void blkdev_show(struct seq_file *seqf, off_t offset);
- #define BLKDEV_MAJOR_MAX	0
- #endif
- 
-struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
-		void *holder);
-struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder);
-int bd_prepare_to_claim(struct block_device *bdev, void *holder);
-void bd_abort_claiming(struct block_device *bdev, void *holder);
-void blkdev_put(struct block_device *bdev, fmode_t mode);
-+struct blk_holder_ops {
-+	void (*mark_dead)(struct block_device *bdev, bool surprise);
-+
-+	/*
-+	 * Sync the file system mounted on the block device.
-+	 */
-+	void (*sync)(struct block_device *bdev);
-+
-+	/*
-+	 * Freeze the file system mounted on the block device.
-+	 */
-+	int (*freeze)(struct block_device *bdev);
-+
-+	/*
-+	 * Thaw the file system mounted on the block device.
-+	 */
-+	int (*thaw)(struct block_device *bdev);
-+};
-+
-+/*
-+ * For filesystems using @fs_holder_ops, the @holder argument passed to
-+ * helpers used to open and claim block devices via
-+ * bd_prepare_to_claim() must point to a superblock.
-+ */
-+extern const struct blk_holder_ops fs_holder_ops;
- 
-/* just for blk-cgroup, don't use elsewhere */
-struct block_device *blkdev_get_no_open(dev_t dev);
-void blkdev_put_no_open(struct block_device *bdev);
-+/*
-+ * Return the correct open flags for blkdev_get_by_* for super block flags
-+ * as stored in sb->s_flags.
-+ */
-+#define sb_open_mode(flags) \
-+	(BLK_OPEN_READ | BLK_OPEN_RESTRICT_WRITES | \
-+	 (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE))
-+
-+struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,
-+		const struct blk_holder_ops *hops);
-+struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode,
-+		void *holder, const struct blk_holder_ops *hops);
-+int bd_prepare_to_claim(struct block_device *bdev, void *holder,
-+		const struct blk_holder_ops *hops);
-+void bd_abort_claiming(struct block_device *bdev, void *holder);
- 
-struct block_device *bdev_alloc(struct gendisk *disk, u8 partno);
-void bdev_add(struct block_device *bdev, dev_t dev);
- struct block_device *I_BDEV(struct inode *inode);
-int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart,
-		loff_t lend);
-+struct block_device *file_bdev(struct file *bdev_file);
-+bool disk_live(struct gendisk *disk);
-+unsigned int block_size(struct block_device *bdev);
- 
- #ifdef CONFIG_BLOCK
- void invalidate_bdev(struct block_device *bdev);
-@@ -1498,7 +1783,9 @@ int sync_blockdev(struct block_device *bdev);
- int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend);
- int sync_blockdev_nowait(struct block_device *bdev);
- void sync_bdevs(bool wait);
-+void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask);
- void printk_all_partitions(void);
-+int __init early_lookup_bdev(const char *pathname, dev_t *dev);
- #else
- static inline void invalidate_bdev(struct block_device *bdev)
- {
-@@ -1514,22 +1801,76 @@ static inline int sync_blockdev_nowait(struct block_device *bdev)
- static inline void sync_bdevs(bool wait)
- {
- }
-+static inline void bdev_statx(const struct path *path, struct kstat *stat,
-+		u32 request_mask)
-+{
-+}
- static inline void printk_all_partitions(void)
- {
- }
-+static inline int early_lookup_bdev(const char *pathname, dev_t *dev)
-+{
-+	return -EINVAL;
-+}
- #endif /* CONFIG_BLOCK */
- 
-int fsync_bdev(struct block_device *bdev);
-
-int freeze_bdev(struct block_device *bdev);
-int thaw_bdev(struct block_device *bdev);
-+int bdev_freeze(struct block_device *bdev);
-+int bdev_thaw(struct block_device *bdev);
-+void bdev_fput(struct file *bdev_file);
- 
- struct io_comp_batch {
-	struct request *req_list;
-+	struct rq_list req_list;
- 	bool need_ts;
- 	void (*complete)(struct io_comp_batch *);
- };
- 
-+static inline bool blk_atomic_write_start_sect_aligned(sector_t sector,
-+						struct queue_limits *limits)
-+{
-+	unsigned int alignment = max(limits->atomic_write_hw_unit_min,
-+				limits->atomic_write_hw_boundary);
-+
-+	return IS_ALIGNED(sector, alignment >> SECTOR_SHIFT);
-+}
-+
-+static inline bool bdev_can_atomic_write(struct block_device *bdev)
-+{
-+	struct request_queue *bd_queue = bdev->bd_queue;
-+	struct queue_limits *limits = &bd_queue->limits;
-+
-+	if (!limits->atomic_write_unit_min)
-+		return false;
-+
-+	if (bdev_is_partition(bdev))
-+		return blk_atomic_write_start_sect_aligned(bdev->bd_start_sect,
-+							limits);
-+
-+	return true;
-+}
-+
-+static inline unsigned int
-+bdev_atomic_write_unit_min_bytes(struct block_device *bdev)
-+{
-+	if (!bdev_can_atomic_write(bdev))
-+		return 0;
-+	return queue_atomic_write_unit_min_bytes(bdev_get_queue(bdev));
-+}
-+
-+static inline unsigned int
-+bdev_atomic_write_unit_max_bytes(struct block_device *bdev)
-+{
-+	if (!bdev_can_atomic_write(bdev))
-+		return 0;
-+	return queue_atomic_write_unit_max_bytes(bdev_get_queue(bdev));
-+}
-+
-+static inline int bio_split_rw_at(struct bio *bio,
-+		const struct queue_limits *lim,
-+		unsigned *segs, unsigned max_bytes)
-+{
-+	return bio_split_io_at(bio, lim, segs, max_bytes, lim->dma_alignment);
-+}
-+
- #define DEFINE_IO_COMP_BATCH(name)	struct io_comp_batch name = { }
- 
- #endif /* _LINUX_BLKDEV_H */
-diff --git a/include/linux/blktrace_api.h b/include/linux/blktrace_api.h
-index cfbda114348c..05c8754456aa 100644
--- a/include/linux/blktrace_api.h
-+++ b/include/linux/blktrace_api.h
-@@ -14,11 +14,12 @@
- #include <linux/sysfs.h>
- 
- struct blk_trace {
-+	int version;
- 	int trace_state;
- 	struct rchan *rchan;
- 	unsigned long __percpu *sequence;
- 	unsigned char __percpu *msg_data;
-	u16 act_mask;
-+	u64 act_mask;
- 	u64 start_lba;
- 	u64 end_lba;
- 	u32 pid;
-@@ -85,10 +86,14 @@ extern int blk_trace_remove(struct request_queue *q);
- # define blk_add_driver_data(rq, data, len)		do {} while (0)
- # define blk_trace_setup(q, name, dev, bdev, arg)	(-ENOTTY)
- # define blk_trace_startstop(q, start)			(-ENOTTY)
-# define blk_trace_remove(q)				(-ENOTTY)
- # define blk_add_trace_msg(q, fmt, ...)			do { } while (0)
- # define blk_add_cgroup_trace_msg(q, cg, fmt, ...)	do { } while (0)
- # define blk_trace_note_message_enabled(q)		(false)
-+
-+static inline int blk_trace_remove(struct request_queue *q)
-+{
-+	return -ENOTTY;
-+}
- #endif /* CONFIG_BLK_DEV_IO_TRACE */
- 
- #ifdef CONFIG_COMPAT
-diff --git a/include/linux/bma150.h b/include/linux/bma150.h
-index 31c9e323a391..4d4a62d49341 100644
--- a/include/linux/bma150.h
-+++ b/include/linux/bma150.h
-@@ -33,8 +33,8 @@ struct bma150_cfg {
- 	unsigned char lg_hyst;		/* Low-G hysterisis */
- 	unsigned char lg_dur;		/* Low-G duration */
- 	unsigned char lg_thres;		/* Low-G threshold */
-	unsigned char range;		/* one of BMA0150_RANGE_xxx */
-	unsigned char bandwidth;	/* one of BMA0150_BW_xxx */
-+	unsigned char range;		/* one of BMA150_RANGE_xxx */
-+	unsigned char bandwidth;	/* one of BMA150_BW_xxx */
- };
- 
- struct bma150_platform_data {
-diff --git a/include/linux/bnxt/hsi.h b/include/linux/bnxt/hsi.h
-new file mode 100644
-index 000000000000..47c34990cf23
--- /dev/null
-+++ b/include/linux/bnxt/hsi.h
-@@ -0,0 +1,11166 @@
-+/* Broadcom NetXtreme-C/E network driver.
-+ *
-+ * Copyright (c) 2014-2016 Broadcom Corporation
-+ * Copyright (c) 2014-2018 Broadcom Limited
-+ * Copyright (c) 2018-2025 Broadcom Inc.
-+ *
-+ * This program is free software; you can redistribute it and/or modify
-+ * it under the terms of the GNU General Public License as published by
-+ * the Free Software Foundation.
-+ *
-+ * DO NOT MODIFY!!! This file is automatically generated.
-+ */
-+
-+#ifndef _BNXT_HSI_H_
-+#define _BNXT_HSI_H_
-+
-+/* hwrm_cmd_hdr (size:128b/16B) */
-+struct hwrm_cmd_hdr {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+};
-+
-+/* hwrm_resp_hdr (size:64b/8B) */
-+struct hwrm_resp_hdr {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+};
-+
-+#define CMD_DISCR_TLV_ENCAP 0x8000UL
-+#define CMD_DISCR_LAST     CMD_DISCR_TLV_ENCAP
-+
-+
-+#define TLV_TYPE_HWRM_REQUEST                    0x1UL
-+#define TLV_TYPE_HWRM_RESPONSE                   0x2UL
-+#define TLV_TYPE_ROCE_SP_COMMAND                 0x3UL
-+#define TLV_TYPE_QUERY_ROCE_CC_GEN1              0x4UL
-+#define TLV_TYPE_MODIFY_ROCE_CC_GEN1             0x5UL
-+#define TLV_TYPE_QUERY_ROCE_CC_GEN2              0x6UL
-+#define TLV_TYPE_MODIFY_ROCE_CC_GEN2             0x7UL
-+#define TLV_TYPE_QUERY_ROCE_CC_GEN1_EXT          0x8UL
-+#define TLV_TYPE_MODIFY_ROCE_CC_GEN1_EXT         0x9UL
-+#define TLV_TYPE_QUERY_ROCE_CC_GEN2_EXT          0xaUL
-+#define TLV_TYPE_MODIFY_ROCE_CC_GEN2_EXT         0xbUL
-+#define TLV_TYPE_ENGINE_CKV_ALIAS_ECC_PUBLIC_KEY 0x8001UL
-+#define TLV_TYPE_ENGINE_CKV_IV                   0x8003UL
-+#define TLV_TYPE_ENGINE_CKV_AUTH_TAG             0x8004UL
-+#define TLV_TYPE_ENGINE_CKV_CIPHERTEXT           0x8005UL
-+#define TLV_TYPE_ENGINE_CKV_HOST_ALGORITHMS      0x8006UL
-+#define TLV_TYPE_ENGINE_CKV_HOST_ECC_PUBLIC_KEY  0x8007UL
-+#define TLV_TYPE_ENGINE_CKV_ECDSA_SIGNATURE      0x8008UL
-+#define TLV_TYPE_ENGINE_CKV_FW_ECC_PUBLIC_KEY    0x8009UL
-+#define TLV_TYPE_ENGINE_CKV_FW_ALGORITHMS        0x800aUL
-+#define TLV_TYPE_LAST                           TLV_TYPE_ENGINE_CKV_FW_ALGORITHMS
-+
-+
-+/* tlv (size:64b/8B) */
-+struct tlv {
-+	__le16	cmd_discr;
-+	u8	reserved_8b;
-+	u8	flags;
-+	#define TLV_FLAGS_MORE         0x1UL
-+	#define TLV_FLAGS_MORE_LAST      0x0UL
-+	#define TLV_FLAGS_MORE_NOT_LAST  0x1UL
-+	#define TLV_FLAGS_REQUIRED     0x2UL
-+	#define TLV_FLAGS_REQUIRED_NO    (0x0UL << 1)
-+	#define TLV_FLAGS_REQUIRED_YES   (0x1UL << 1)
-+	#define TLV_FLAGS_REQUIRED_LAST TLV_FLAGS_REQUIRED_YES
-+	__le16	tlv_type;
-+	__le16	length;
-+};
-+
-+/* input (size:128b/16B) */
-+struct input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+};
-+
-+/* output (size:64b/8B) */
-+struct output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+};
-+
-+/* hwrm_short_input (size:128b/16B) */
-+struct hwrm_short_input {
-+	__le16	req_type;
-+	__le16	signature;
-+	#define SHORT_REQ_SIGNATURE_SHORT_CMD 0x4321UL
-+	#define SHORT_REQ_SIGNATURE_LAST     SHORT_REQ_SIGNATURE_SHORT_CMD
-+	__le16	target_id;
-+	#define SHORT_REQ_TARGET_ID_DEFAULT 0x0UL
-+	#define SHORT_REQ_TARGET_ID_TOOLS   0xfffdUL
-+	#define SHORT_REQ_TARGET_ID_LAST   SHORT_REQ_TARGET_ID_TOOLS
-+	__le16	size;
-+	__le64	req_addr;
-+};
-+
-+/* cmd_nums (size:64b/8B) */
-+struct cmd_nums {
-+	__le16	req_type;
-+	#define HWRM_VER_GET                              0x0UL
-+	#define HWRM_FUNC_ECHO_RESPONSE                   0xbUL
-+	#define HWRM_ERROR_RECOVERY_QCFG                  0xcUL
-+	#define HWRM_FUNC_DRV_IF_CHANGE                   0xdUL
-+	#define HWRM_FUNC_BUF_UNRGTR                      0xeUL
-+	#define HWRM_FUNC_VF_CFG                          0xfUL
-+	#define HWRM_RESERVED1                            0x10UL
-+	#define HWRM_FUNC_RESET                           0x11UL
-+	#define HWRM_FUNC_GETFID                          0x12UL
-+	#define HWRM_FUNC_VF_ALLOC                        0x13UL
-+	#define HWRM_FUNC_VF_FREE                         0x14UL
-+	#define HWRM_FUNC_QCAPS                           0x15UL
-+	#define HWRM_FUNC_QCFG                            0x16UL
-+	#define HWRM_FUNC_CFG                             0x17UL
-+	#define HWRM_FUNC_QSTATS                          0x18UL
-+	#define HWRM_FUNC_CLR_STATS                       0x19UL
-+	#define HWRM_FUNC_DRV_UNRGTR                      0x1aUL
-+	#define HWRM_FUNC_VF_RESC_FREE                    0x1bUL
-+	#define HWRM_FUNC_VF_VNIC_IDS_QUERY               0x1cUL
-+	#define HWRM_FUNC_DRV_RGTR                        0x1dUL
-+	#define HWRM_FUNC_DRV_QVER                        0x1eUL
-+	#define HWRM_FUNC_BUF_RGTR                        0x1fUL
-+	#define HWRM_PORT_PHY_CFG                         0x20UL
-+	#define HWRM_PORT_MAC_CFG                         0x21UL
-+	#define HWRM_PORT_TS_QUERY                        0x22UL
-+	#define HWRM_PORT_QSTATS                          0x23UL
-+	#define HWRM_PORT_LPBK_QSTATS                     0x24UL
-+	#define HWRM_PORT_CLR_STATS                       0x25UL
-+	#define HWRM_PORT_LPBK_CLR_STATS                  0x26UL
-+	#define HWRM_PORT_PHY_QCFG                        0x27UL
-+	#define HWRM_PORT_MAC_QCFG                        0x28UL
-+	#define HWRM_PORT_MAC_PTP_QCFG                    0x29UL
-+	#define HWRM_PORT_PHY_QCAPS                       0x2aUL
-+	#define HWRM_PORT_PHY_I2C_WRITE                   0x2bUL
-+	#define HWRM_PORT_PHY_I2C_READ                    0x2cUL
-+	#define HWRM_PORT_LED_CFG                         0x2dUL
-+	#define HWRM_PORT_LED_QCFG                        0x2eUL
-+	#define HWRM_PORT_LED_QCAPS                       0x2fUL
-+	#define HWRM_QUEUE_QPORTCFG                       0x30UL
-+	#define HWRM_QUEUE_QCFG                           0x31UL
-+	#define HWRM_QUEUE_CFG                            0x32UL
-+	#define HWRM_FUNC_VLAN_CFG                        0x33UL
-+	#define HWRM_FUNC_VLAN_QCFG                       0x34UL
-+	#define HWRM_QUEUE_PFCENABLE_QCFG                 0x35UL
-+	#define HWRM_QUEUE_PFCENABLE_CFG                  0x36UL
-+	#define HWRM_QUEUE_PRI2COS_QCFG                   0x37UL
-+	#define HWRM_QUEUE_PRI2COS_CFG                    0x38UL
-+	#define HWRM_QUEUE_COS2BW_QCFG                    0x39UL
-+	#define HWRM_QUEUE_COS2BW_CFG                     0x3aUL
-+	#define HWRM_QUEUE_DSCP_QCAPS                     0x3bUL
-+	#define HWRM_QUEUE_DSCP2PRI_QCFG                  0x3cUL
-+	#define HWRM_QUEUE_DSCP2PRI_CFG                   0x3dUL
-+	#define HWRM_VNIC_ALLOC                           0x40UL
-+	#define HWRM_VNIC_FREE                            0x41UL
-+	#define HWRM_VNIC_CFG                             0x42UL
-+	#define HWRM_VNIC_QCFG                            0x43UL
-+	#define HWRM_VNIC_TPA_CFG                         0x44UL
-+	#define HWRM_VNIC_TPA_QCFG                        0x45UL
-+	#define HWRM_VNIC_RSS_CFG                         0x46UL
-+	#define HWRM_VNIC_RSS_QCFG                        0x47UL
-+	#define HWRM_VNIC_PLCMODES_CFG                    0x48UL
-+	#define HWRM_VNIC_PLCMODES_QCFG                   0x49UL
-+	#define HWRM_VNIC_QCAPS                           0x4aUL
-+	#define HWRM_VNIC_UPDATE                          0x4bUL
-+	#define HWRM_RING_ALLOC                           0x50UL
-+	#define HWRM_RING_FREE                            0x51UL
-+	#define HWRM_RING_CMPL_RING_QAGGINT_PARAMS        0x52UL
-+	#define HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS     0x53UL
-+	#define HWRM_RING_AGGINT_QCAPS                    0x54UL
-+	#define HWRM_RING_SCHQ_ALLOC                      0x55UL
-+	#define HWRM_RING_SCHQ_CFG                        0x56UL
-+	#define HWRM_RING_SCHQ_FREE                       0x57UL
-+	#define HWRM_RING_RESET                           0x5eUL
-+	#define HWRM_RING_GRP_ALLOC                       0x60UL
-+	#define HWRM_RING_GRP_FREE                        0x61UL
-+	#define HWRM_RING_CFG                             0x62UL
-+	#define HWRM_RING_QCFG                            0x63UL
-+	#define HWRM_RESERVED5                            0x64UL
-+	#define HWRM_RESERVED6                            0x65UL
-+	#define HWRM_VNIC_RSS_COS_LB_CTX_ALLOC            0x70UL
-+	#define HWRM_VNIC_RSS_COS_LB_CTX_FREE             0x71UL
-+	#define HWRM_QUEUE_MPLS_QCAPS                     0x80UL
-+	#define HWRM_QUEUE_MPLSTC2PRI_QCFG                0x81UL
-+	#define HWRM_QUEUE_MPLSTC2PRI_CFG                 0x82UL
-+	#define HWRM_QUEUE_VLANPRI_QCAPS                  0x83UL
-+	#define HWRM_QUEUE_VLANPRI2PRI_QCFG               0x84UL
-+	#define HWRM_QUEUE_VLANPRI2PRI_CFG                0x85UL
-+	#define HWRM_QUEUE_GLOBAL_CFG                     0x86UL
-+	#define HWRM_QUEUE_GLOBAL_QCFG                    0x87UL
-+	#define HWRM_QUEUE_ADPTV_QOS_RX_FEATURE_QCFG      0x88UL
-+	#define HWRM_QUEUE_ADPTV_QOS_RX_FEATURE_CFG       0x89UL
-+	#define HWRM_QUEUE_ADPTV_QOS_TX_FEATURE_QCFG      0x8aUL
-+	#define HWRM_QUEUE_ADPTV_QOS_TX_FEATURE_CFG       0x8bUL
-+	#define HWRM_QUEUE_QCAPS                          0x8cUL
-+	#define HWRM_QUEUE_ADPTV_QOS_RX_TUNING_QCFG       0x8dUL
-+	#define HWRM_QUEUE_ADPTV_QOS_RX_TUNING_CFG        0x8eUL
-+	#define HWRM_QUEUE_ADPTV_QOS_TX_TUNING_QCFG       0x8fUL
-+	#define HWRM_CFA_L2_FILTER_ALLOC                  0x90UL
-+	#define HWRM_CFA_L2_FILTER_FREE                   0x91UL
-+	#define HWRM_CFA_L2_FILTER_CFG                    0x92UL
-+	#define HWRM_CFA_L2_SET_RX_MASK                   0x93UL
-+	#define HWRM_CFA_VLAN_ANTISPOOF_CFG               0x94UL
-+	#define HWRM_CFA_TUNNEL_FILTER_ALLOC              0x95UL
-+	#define HWRM_CFA_TUNNEL_FILTER_FREE               0x96UL
-+	#define HWRM_CFA_ENCAP_RECORD_ALLOC               0x97UL
-+	#define HWRM_CFA_ENCAP_RECORD_FREE                0x98UL
-+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC              0x99UL
-+	#define HWRM_CFA_NTUPLE_FILTER_FREE               0x9aUL
-+	#define HWRM_CFA_NTUPLE_FILTER_CFG                0x9bUL
-+	#define HWRM_CFA_EM_FLOW_ALLOC                    0x9cUL
-+	#define HWRM_CFA_EM_FLOW_FREE                     0x9dUL
-+	#define HWRM_CFA_EM_FLOW_CFG                      0x9eUL
-+	#define HWRM_TUNNEL_DST_PORT_QUERY                0xa0UL
-+	#define HWRM_TUNNEL_DST_PORT_ALLOC                0xa1UL
-+	#define HWRM_TUNNEL_DST_PORT_FREE                 0xa2UL
-+	#define HWRM_QUEUE_ADPTV_QOS_TX_TUNING_CFG        0xa3UL
-+	#define HWRM_STAT_CTX_ENG_QUERY                   0xafUL
-+	#define HWRM_STAT_CTX_ALLOC                       0xb0UL
-+	#define HWRM_STAT_CTX_FREE                        0xb1UL
-+	#define HWRM_STAT_CTX_QUERY                       0xb2UL
-+	#define HWRM_STAT_CTX_CLR_STATS                   0xb3UL
-+	#define HWRM_PORT_QSTATS_EXT                      0xb4UL
-+	#define HWRM_PORT_PHY_MDIO_WRITE                  0xb5UL
-+	#define HWRM_PORT_PHY_MDIO_READ                   0xb6UL
-+	#define HWRM_PORT_PHY_MDIO_BUS_ACQUIRE            0xb7UL
-+	#define HWRM_PORT_PHY_MDIO_BUS_RELEASE            0xb8UL
-+	#define HWRM_PORT_QSTATS_EXT_PFC_WD               0xb9UL
-+	#define HWRM_RESERVED7                            0xbaUL
-+	#define HWRM_PORT_TX_FIR_CFG                      0xbbUL
-+	#define HWRM_PORT_TX_FIR_QCFG                     0xbcUL
-+	#define HWRM_PORT_ECN_QSTATS                      0xbdUL
-+	#define HWRM_FW_LIVEPATCH_QUERY                   0xbeUL
-+	#define HWRM_FW_LIVEPATCH                         0xbfUL
-+	#define HWRM_FW_RESET                             0xc0UL
-+	#define HWRM_FW_QSTATUS                           0xc1UL
-+	#define HWRM_FW_HEALTH_CHECK                      0xc2UL
-+	#define HWRM_FW_SYNC                              0xc3UL
-+	#define HWRM_FW_STATE_QCAPS                       0xc4UL
-+	#define HWRM_FW_STATE_QUIESCE                     0xc5UL
-+	#define HWRM_FW_STATE_BACKUP                      0xc6UL
-+	#define HWRM_FW_STATE_RESTORE                     0xc7UL
-+	#define HWRM_FW_SET_TIME                          0xc8UL
-+	#define HWRM_FW_GET_TIME                          0xc9UL
-+	#define HWRM_FW_SET_STRUCTURED_DATA               0xcaUL
-+	#define HWRM_FW_GET_STRUCTURED_DATA               0xcbUL
-+	#define HWRM_FW_IPC_MAILBOX                       0xccUL
-+	#define HWRM_FW_ECN_CFG                           0xcdUL
-+	#define HWRM_FW_ECN_QCFG                          0xceUL
-+	#define HWRM_FW_SECURE_CFG                        0xcfUL
-+	#define HWRM_EXEC_FWD_RESP                        0xd0UL
-+	#define HWRM_REJECT_FWD_RESP                      0xd1UL
-+	#define HWRM_FWD_RESP                             0xd2UL
-+	#define HWRM_FWD_ASYNC_EVENT_CMPL                 0xd3UL
-+	#define HWRM_OEM_CMD                              0xd4UL
-+	#define HWRM_PORT_PRBS_TEST                       0xd5UL
-+	#define HWRM_PORT_SFP_SIDEBAND_CFG                0xd6UL
-+	#define HWRM_PORT_SFP_SIDEBAND_QCFG               0xd7UL
-+	#define HWRM_FW_STATE_UNQUIESCE                   0xd8UL
-+	#define HWRM_PORT_DSC_DUMP                        0xd9UL
-+	#define HWRM_PORT_EP_TX_QCFG                      0xdaUL
-+	#define HWRM_PORT_EP_TX_CFG                       0xdbUL
-+	#define HWRM_PORT_CFG                             0xdcUL
-+	#define HWRM_PORT_QCFG                            0xddUL
-+	#define HWRM_PORT_MAC_QCAPS                       0xdfUL
-+	#define HWRM_TEMP_MONITOR_QUERY                   0xe0UL
-+	#define HWRM_REG_POWER_QUERY                      0xe1UL
-+	#define HWRM_CORE_FREQUENCY_QUERY                 0xe2UL
-+	#define HWRM_REG_POWER_HISTOGRAM                  0xe3UL
-+	#define HWRM_MONITOR_PAX_HISTOGRAM_START          0xe4UL
-+	#define HWRM_MONITOR_PAX_HISTOGRAM_COLLECT        0xe5UL
-+	#define HWRM_STAT_QUERY_ROCE_STATS                0xe6UL
-+	#define HWRM_STAT_QUERY_ROCE_STATS_EXT            0xe7UL
-+	#define HWRM_WOL_FILTER_ALLOC                     0xf0UL
-+	#define HWRM_WOL_FILTER_FREE                      0xf1UL
-+	#define HWRM_WOL_FILTER_QCFG                      0xf2UL
-+	#define HWRM_WOL_REASON_QCFG                      0xf3UL
-+	#define HWRM_CFA_METER_QCAPS                      0xf4UL
-+	#define HWRM_CFA_METER_PROFILE_ALLOC              0xf5UL
-+	#define HWRM_CFA_METER_PROFILE_FREE               0xf6UL
-+	#define HWRM_CFA_METER_PROFILE_CFG                0xf7UL
-+	#define HWRM_CFA_METER_INSTANCE_ALLOC             0xf8UL
-+	#define HWRM_CFA_METER_INSTANCE_FREE              0xf9UL
-+	#define HWRM_CFA_METER_INSTANCE_CFG               0xfaUL
-+	#define HWRM_CFA_VFR_ALLOC                        0xfdUL
-+	#define HWRM_CFA_VFR_FREE                         0xfeUL
-+	#define HWRM_CFA_VF_PAIR_ALLOC                    0x100UL
-+	#define HWRM_CFA_VF_PAIR_FREE                     0x101UL
-+	#define HWRM_CFA_VF_PAIR_INFO                     0x102UL
-+	#define HWRM_CFA_FLOW_ALLOC                       0x103UL
-+	#define HWRM_CFA_FLOW_FREE                        0x104UL
-+	#define HWRM_CFA_FLOW_FLUSH                       0x105UL
-+	#define HWRM_CFA_FLOW_STATS                       0x106UL
-+	#define HWRM_CFA_FLOW_INFO                        0x107UL
-+	#define HWRM_CFA_DECAP_FILTER_ALLOC               0x108UL
-+	#define HWRM_CFA_DECAP_FILTER_FREE                0x109UL
-+	#define HWRM_CFA_VLAN_ANTISPOOF_QCFG              0x10aUL
-+	#define HWRM_CFA_REDIRECT_TUNNEL_TYPE_ALLOC       0x10bUL
-+	#define HWRM_CFA_REDIRECT_TUNNEL_TYPE_FREE        0x10cUL
-+	#define HWRM_CFA_PAIR_ALLOC                       0x10dUL
-+	#define HWRM_CFA_PAIR_FREE                        0x10eUL
-+	#define HWRM_CFA_PAIR_INFO                        0x10fUL
-+	#define HWRM_FW_IPC_MSG                           0x110UL
-+	#define HWRM_CFA_REDIRECT_TUNNEL_TYPE_INFO        0x111UL
-+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE       0x112UL
-+	#define HWRM_CFA_FLOW_AGING_TIMER_RESET           0x113UL
-+	#define HWRM_CFA_FLOW_AGING_CFG                   0x114UL
-+	#define HWRM_CFA_FLOW_AGING_QCFG                  0x115UL
-+	#define HWRM_CFA_FLOW_AGING_QCAPS                 0x116UL
-+	#define HWRM_CFA_CTX_MEM_RGTR                     0x117UL
-+	#define HWRM_CFA_CTX_MEM_UNRGTR                   0x118UL
-+	#define HWRM_CFA_CTX_MEM_QCTX                     0x119UL
-+	#define HWRM_CFA_CTX_MEM_QCAPS                    0x11aUL
-+	#define HWRM_CFA_COUNTER_QCAPS                    0x11bUL
-+	#define HWRM_CFA_COUNTER_CFG                      0x11cUL
-+	#define HWRM_CFA_COUNTER_QCFG                     0x11dUL
-+	#define HWRM_CFA_COUNTER_QSTATS                   0x11eUL
-+	#define HWRM_CFA_TCP_FLAG_PROCESS_QCFG            0x11fUL
-+	#define HWRM_CFA_EEM_QCAPS                        0x120UL
-+	#define HWRM_CFA_EEM_CFG                          0x121UL
-+	#define HWRM_CFA_EEM_QCFG                         0x122UL
-+	#define HWRM_CFA_EEM_OP                           0x123UL
-+	#define HWRM_CFA_ADV_FLOW_MGNT_QCAPS              0x124UL
-+	#define HWRM_CFA_TFLIB                            0x125UL
-+	#define HWRM_CFA_LAG_GROUP_MEMBER_RGTR            0x126UL
-+	#define HWRM_CFA_LAG_GROUP_MEMBER_UNRGTR          0x127UL
-+	#define HWRM_CFA_TLS_FILTER_ALLOC                 0x128UL
-+	#define HWRM_CFA_TLS_FILTER_FREE                  0x129UL
-+	#define HWRM_CFA_RELEASE_AFM_FUNC                 0x12aUL
-+	#define HWRM_ENGINE_CKV_STATUS                    0x12eUL
-+	#define HWRM_ENGINE_CKV_CKEK_ADD                  0x12fUL
-+	#define HWRM_ENGINE_CKV_CKEK_DELETE               0x130UL
-+	#define HWRM_ENGINE_CKV_KEY_ADD                   0x131UL
-+	#define HWRM_ENGINE_CKV_KEY_DELETE                0x132UL
-+	#define HWRM_ENGINE_CKV_FLUSH                     0x133UL
-+	#define HWRM_ENGINE_CKV_RNG_GET                   0x134UL
-+	#define HWRM_ENGINE_CKV_KEY_GEN                   0x135UL
-+	#define HWRM_ENGINE_CKV_KEY_LABEL_CFG             0x136UL
-+	#define HWRM_ENGINE_CKV_KEY_LABEL_QCFG            0x137UL
-+	#define HWRM_ENGINE_QG_CONFIG_QUERY               0x13cUL
-+	#define HWRM_ENGINE_QG_QUERY                      0x13dUL
-+	#define HWRM_ENGINE_QG_METER_PROFILE_CONFIG_QUERY 0x13eUL
-+	#define HWRM_ENGINE_QG_METER_PROFILE_QUERY        0x13fUL
-+	#define HWRM_ENGINE_QG_METER_PROFILE_ALLOC        0x140UL
-+	#define HWRM_ENGINE_QG_METER_PROFILE_FREE         0x141UL
-+	#define HWRM_ENGINE_QG_METER_QUERY                0x142UL
-+	#define HWRM_ENGINE_QG_METER_BIND                 0x143UL
-+	#define HWRM_ENGINE_QG_METER_UNBIND               0x144UL
-+	#define HWRM_ENGINE_QG_FUNC_BIND                  0x145UL
-+	#define HWRM_ENGINE_SG_CONFIG_QUERY               0x146UL
-+	#define HWRM_ENGINE_SG_QUERY                      0x147UL
-+	#define HWRM_ENGINE_SG_METER_QUERY                0x148UL
-+	#define HWRM_ENGINE_SG_METER_CONFIG               0x149UL
-+	#define HWRM_ENGINE_SG_QG_BIND                    0x14aUL
-+	#define HWRM_ENGINE_QG_SG_UNBIND                  0x14bUL
-+	#define HWRM_ENGINE_CONFIG_QUERY                  0x154UL
-+	#define HWRM_ENGINE_STATS_CONFIG                  0x155UL
-+	#define HWRM_ENGINE_STATS_CLEAR                   0x156UL
-+	#define HWRM_ENGINE_STATS_QUERY                   0x157UL
-+	#define HWRM_ENGINE_STATS_QUERY_CONTINUOUS_ERROR  0x158UL
-+	#define HWRM_ENGINE_RQ_ALLOC                      0x15eUL
-+	#define HWRM_ENGINE_RQ_FREE                       0x15fUL
-+	#define HWRM_ENGINE_CQ_ALLOC                      0x160UL
-+	#define HWRM_ENGINE_CQ_FREE                       0x161UL
-+	#define HWRM_ENGINE_NQ_ALLOC                      0x162UL
-+	#define HWRM_ENGINE_NQ_FREE                       0x163UL
-+	#define HWRM_ENGINE_ON_DIE_RQE_CREDITS            0x164UL
-+	#define HWRM_ENGINE_FUNC_QCFG                     0x165UL
-+	#define HWRM_FUNC_RESOURCE_QCAPS                  0x190UL
-+	#define HWRM_FUNC_VF_RESOURCE_CFG                 0x191UL
-+	#define HWRM_FUNC_BACKING_STORE_QCAPS             0x192UL
-+	#define HWRM_FUNC_BACKING_STORE_CFG               0x193UL
-+	#define HWRM_FUNC_BACKING_STORE_QCFG              0x194UL
-+	#define HWRM_FUNC_VF_BW_CFG                       0x195UL
-+	#define HWRM_FUNC_VF_BW_QCFG                      0x196UL
-+	#define HWRM_FUNC_HOST_PF_IDS_QUERY               0x197UL
-+	#define HWRM_FUNC_QSTATS_EXT                      0x198UL
-+	#define HWRM_STAT_EXT_CTX_QUERY                   0x199UL
-+	#define HWRM_FUNC_SPD_CFG                         0x19aUL
-+	#define HWRM_FUNC_SPD_QCFG                        0x19bUL
-+	#define HWRM_FUNC_PTP_PIN_QCFG                    0x19cUL
-+	#define HWRM_FUNC_PTP_PIN_CFG                     0x19dUL
-+	#define HWRM_FUNC_PTP_CFG                         0x19eUL
-+	#define HWRM_FUNC_PTP_TS_QUERY                    0x19fUL
-+	#define HWRM_FUNC_PTP_EXT_CFG                     0x1a0UL
-+	#define HWRM_FUNC_PTP_EXT_QCFG                    0x1a1UL
-+	#define HWRM_FUNC_KEY_CTX_ALLOC                   0x1a2UL
-+	#define HWRM_FUNC_BACKING_STORE_CFG_V2            0x1a3UL
-+	#define HWRM_FUNC_BACKING_STORE_QCFG_V2           0x1a4UL
-+	#define HWRM_FUNC_DBR_PACING_CFG                  0x1a5UL
-+	#define HWRM_FUNC_DBR_PACING_QCFG                 0x1a6UL
-+	#define HWRM_FUNC_DBR_PACING_BROADCAST_EVENT      0x1a7UL
-+	#define HWRM_FUNC_BACKING_STORE_QCAPS_V2          0x1a8UL
-+	#define HWRM_FUNC_DBR_PACING_NQLIST_QUERY         0x1a9UL
-+	#define HWRM_FUNC_DBR_RECOVERY_COMPLETED          0x1aaUL
-+	#define HWRM_FUNC_SYNCE_CFG                       0x1abUL
-+	#define HWRM_FUNC_SYNCE_QCFG                      0x1acUL
-+	#define HWRM_FUNC_KEY_CTX_FREE                    0x1adUL
-+	#define HWRM_FUNC_LAG_MODE_CFG                    0x1aeUL
-+	#define HWRM_FUNC_LAG_MODE_QCFG                   0x1afUL
-+	#define HWRM_FUNC_LAG_CREATE                      0x1b0UL
-+	#define HWRM_FUNC_LAG_UPDATE                      0x1b1UL
-+	#define HWRM_FUNC_LAG_FREE                        0x1b2UL
-+	#define HWRM_FUNC_LAG_QCFG                        0x1b3UL
-+	#define HWRM_FUNC_TTX_PACING_RATE_PROF_QUERY      0x1c3UL
-+	#define HWRM_FUNC_TTX_PACING_RATE_QUERY           0x1c4UL
-+	#define HWRM_SELFTEST_QLIST                       0x200UL
-+	#define HWRM_SELFTEST_EXEC                        0x201UL
-+	#define HWRM_SELFTEST_IRQ                         0x202UL
-+	#define HWRM_SELFTEST_RETRIEVE_SERDES_DATA        0x203UL
-+	#define HWRM_PCIE_QSTATS                          0x204UL
-+	#define HWRM_MFG_FRU_WRITE_CONTROL                0x205UL
-+	#define HWRM_MFG_TIMERS_QUERY                     0x206UL
-+	#define HWRM_MFG_OTP_CFG                          0x207UL
-+	#define HWRM_MFG_OTP_QCFG                         0x208UL
-+	#define HWRM_MFG_HDMA_TEST                        0x209UL
-+	#define HWRM_MFG_FRU_EEPROM_WRITE                 0x20aUL
-+	#define HWRM_MFG_FRU_EEPROM_READ                  0x20bUL
-+	#define HWRM_MFG_SOC_IMAGE                        0x20cUL
-+	#define HWRM_MFG_SOC_QSTATUS                      0x20dUL
-+	#define HWRM_MFG_PARAM_CRITICAL_DATA_FINALIZE     0x20eUL
-+	#define HWRM_MFG_PARAM_CRITICAL_DATA_READ         0x20fUL
-+	#define HWRM_MFG_PARAM_CRITICAL_DATA_HEALTH       0x210UL
-+	#define HWRM_MFG_PRVSN_EXPORT_CSR                 0x211UL
-+	#define HWRM_MFG_PRVSN_IMPORT_CERT                0x212UL
-+	#define HWRM_MFG_PRVSN_GET_STATE                  0x213UL
-+	#define HWRM_MFG_GET_NVM_MEASUREMENT              0x214UL
-+	#define HWRM_MFG_PSOC_QSTATUS                     0x215UL
-+	#define HWRM_MFG_SELFTEST_QLIST                   0x216UL
-+	#define HWRM_MFG_SELFTEST_EXEC                    0x217UL
-+	#define HWRM_STAT_GENERIC_QSTATS                  0x218UL
-+	#define HWRM_MFG_PRVSN_EXPORT_CERT                0x219UL
-+	#define HWRM_STAT_DB_ERROR_QSTATS                 0x21aUL
-+	#define HWRM_MFG_TESTS                            0x21bUL
-+	#define HWRM_MFG_WRITE_CERT_NVM                   0x21cUL
-+	#define HWRM_PORT_POE_CFG                         0x230UL
-+	#define HWRM_PORT_POE_QCFG                        0x231UL
-+	#define HWRM_PORT_PHY_FDRSTAT                     0x232UL
-+	#define HWRM_UDCC_QCAPS                           0x258UL
-+	#define HWRM_UDCC_CFG                             0x259UL
-+	#define HWRM_UDCC_QCFG                            0x25aUL
-+	#define HWRM_UDCC_SESSION_CFG                     0x25bUL
-+	#define HWRM_UDCC_SESSION_QCFG                    0x25cUL
-+	#define HWRM_UDCC_SESSION_QUERY                   0x25dUL
-+	#define HWRM_UDCC_COMP_CFG                        0x25eUL
-+	#define HWRM_UDCC_COMP_QCFG                       0x25fUL
-+	#define HWRM_UDCC_COMP_QUERY                      0x260UL
-+	#define HWRM_QUEUE_PFCWD_TIMEOUT_QCAPS            0x261UL
-+	#define HWRM_QUEUE_PFCWD_TIMEOUT_CFG              0x262UL
-+	#define HWRM_QUEUE_PFCWD_TIMEOUT_QCFG             0x263UL
-+	#define HWRM_QUEUE_ADPTV_QOS_RX_QCFG              0x264UL
-+	#define HWRM_QUEUE_ADPTV_QOS_TX_QCFG              0x265UL
-+	#define HWRM_TF                                   0x2bcUL
-+	#define HWRM_TF_VERSION_GET                       0x2bdUL
-+	#define HWRM_TF_SESSION_OPEN                      0x2c6UL
-+	#define HWRM_TF_SESSION_REGISTER                  0x2c8UL
-+	#define HWRM_TF_SESSION_UNREGISTER                0x2c9UL
-+	#define HWRM_TF_SESSION_CLOSE                     0x2caUL
-+	#define HWRM_TF_SESSION_QCFG                      0x2cbUL
-+	#define HWRM_TF_SESSION_RESC_QCAPS                0x2ccUL
-+	#define HWRM_TF_SESSION_RESC_ALLOC                0x2cdUL
-+	#define HWRM_TF_SESSION_RESC_FREE                 0x2ceUL
-+	#define HWRM_TF_SESSION_RESC_FLUSH                0x2cfUL
-+	#define HWRM_TF_SESSION_RESC_INFO                 0x2d0UL
-+	#define HWRM_TF_SESSION_HOTUP_STATE_SET           0x2d1UL
-+	#define HWRM_TF_SESSION_HOTUP_STATE_GET           0x2d2UL
-+	#define HWRM_TF_TBL_TYPE_GET                      0x2daUL
-+	#define HWRM_TF_TBL_TYPE_SET                      0x2dbUL
-+	#define HWRM_TF_TBL_TYPE_BULK_GET                 0x2dcUL
-+	#define HWRM_TF_EM_INSERT                         0x2eaUL
-+	#define HWRM_TF_EM_DELETE                         0x2ebUL
-+	#define HWRM_TF_EM_HASH_INSERT                    0x2ecUL
-+	#define HWRM_TF_EM_MOVE                           0x2edUL
-+	#define HWRM_TF_TCAM_SET                          0x2f8UL
-+	#define HWRM_TF_TCAM_GET                          0x2f9UL
-+	#define HWRM_TF_TCAM_MOVE                         0x2faUL
-+	#define HWRM_TF_TCAM_FREE                         0x2fbUL
-+	#define HWRM_TF_GLOBAL_CFG_SET                    0x2fcUL
-+	#define HWRM_TF_GLOBAL_CFG_GET                    0x2fdUL
-+	#define HWRM_TF_IF_TBL_SET                        0x2feUL
-+	#define HWRM_TF_IF_TBL_GET                        0x2ffUL
-+	#define HWRM_TF_RESC_USAGE_SET                    0x300UL
-+	#define HWRM_TF_RESC_USAGE_QUERY                  0x301UL
-+	#define HWRM_TF_TBL_TYPE_ALLOC                    0x302UL
-+	#define HWRM_TF_TBL_TYPE_FREE                     0x303UL
-+	#define HWRM_TFC_TBL_SCOPE_QCAPS                  0x380UL
-+	#define HWRM_TFC_TBL_SCOPE_ID_ALLOC               0x381UL
-+	#define HWRM_TFC_TBL_SCOPE_CONFIG                 0x382UL
-+	#define HWRM_TFC_TBL_SCOPE_DECONFIG               0x383UL
-+	#define HWRM_TFC_TBL_SCOPE_FID_ADD                0x384UL
-+	#define HWRM_TFC_TBL_SCOPE_FID_REM                0x385UL
-+	#define HWRM_TFC_TBL_SCOPE_POOL_ALLOC             0x386UL
-+	#define HWRM_TFC_TBL_SCOPE_POOL_FREE              0x387UL
-+	#define HWRM_TFC_SESSION_ID_ALLOC                 0x388UL
-+	#define HWRM_TFC_SESSION_FID_ADD                  0x389UL
-+	#define HWRM_TFC_SESSION_FID_REM                  0x38aUL
-+	#define HWRM_TFC_IDENT_ALLOC                      0x38bUL
-+	#define HWRM_TFC_IDENT_FREE                       0x38cUL
-+	#define HWRM_TFC_IDX_TBL_ALLOC                    0x38dUL
-+	#define HWRM_TFC_IDX_TBL_ALLOC_SET                0x38eUL
-+	#define HWRM_TFC_IDX_TBL_SET                      0x38fUL
-+	#define HWRM_TFC_IDX_TBL_GET                      0x390UL
-+	#define HWRM_TFC_IDX_TBL_FREE                     0x391UL
-+	#define HWRM_TFC_GLOBAL_ID_ALLOC                  0x392UL
-+	#define HWRM_TFC_TCAM_SET                         0x393UL
-+	#define HWRM_TFC_TCAM_GET                         0x394UL
-+	#define HWRM_TFC_TCAM_ALLOC                       0x395UL
-+	#define HWRM_TFC_TCAM_ALLOC_SET                   0x396UL
-+	#define HWRM_TFC_TCAM_FREE                        0x397UL
-+	#define HWRM_TFC_IF_TBL_SET                       0x398UL
-+	#define HWRM_TFC_IF_TBL_GET                       0x399UL
-+	#define HWRM_TFC_TBL_SCOPE_CONFIG_GET             0x39aUL
-+	#define HWRM_TFC_RESC_USAGE_QUERY                 0x39bUL
-+	#define HWRM_TFC_GLOBAL_ID_FREE                   0x39cUL
-+	#define HWRM_TFC_TCAM_PRI_UPDATE                  0x39dUL
-+	#define HWRM_TFC_HOT_UPGRADE_PROCESS              0x3a0UL
-+	#define HWRM_SV                                   0x400UL
-+	#define HWRM_DBG_SERDES_TEST                      0xff0eUL
-+	#define HWRM_DBG_LOG_BUFFER_FLUSH                 0xff0fUL
-+	#define HWRM_DBG_READ_DIRECT                      0xff10UL
-+	#define HWRM_DBG_READ_INDIRECT                    0xff11UL
-+	#define HWRM_DBG_WRITE_DIRECT                     0xff12UL
-+	#define HWRM_DBG_WRITE_INDIRECT                   0xff13UL
-+	#define HWRM_DBG_DUMP                             0xff14UL
-+	#define HWRM_DBG_ERASE_NVM                        0xff15UL
-+	#define HWRM_DBG_CFG                              0xff16UL
-+	#define HWRM_DBG_COREDUMP_LIST                    0xff17UL
-+	#define HWRM_DBG_COREDUMP_INITIATE                0xff18UL
-+	#define HWRM_DBG_COREDUMP_RETRIEVE                0xff19UL
-+	#define HWRM_DBG_FW_CLI                           0xff1aUL
-+	#define HWRM_DBG_I2C_CMD                          0xff1bUL
-+	#define HWRM_DBG_RING_INFO_GET                    0xff1cUL
-+	#define HWRM_DBG_CRASHDUMP_HEADER                 0xff1dUL
-+	#define HWRM_DBG_CRASHDUMP_ERASE                  0xff1eUL
-+	#define HWRM_DBG_DRV_TRACE                        0xff1fUL
-+	#define HWRM_DBG_QCAPS                            0xff20UL
-+	#define HWRM_DBG_QCFG                             0xff21UL
-+	#define HWRM_DBG_CRASHDUMP_MEDIUM_CFG             0xff22UL
-+	#define HWRM_DBG_USEQ_ALLOC                       0xff23UL
-+	#define HWRM_DBG_USEQ_FREE                        0xff24UL
-+	#define HWRM_DBG_USEQ_FLUSH                       0xff25UL
-+	#define HWRM_DBG_USEQ_QCAPS                       0xff26UL
-+	#define HWRM_DBG_USEQ_CW_CFG                      0xff27UL
-+	#define HWRM_DBG_USEQ_SCHED_CFG                   0xff28UL
-+	#define HWRM_DBG_USEQ_RUN                         0xff29UL
-+	#define HWRM_DBG_USEQ_DELIVERY_REQ                0xff2aUL
-+	#define HWRM_DBG_USEQ_RESP_HDR                    0xff2bUL
-+	#define HWRM_DBG_COREDUMP_CAPTURE                 0xff2cUL
-+	#define HWRM_DBG_PTRACE                           0xff2dUL
-+	#define HWRM_DBG_SIM_CABLE_STATE                  0xff2eUL
-+	#define HWRM_DBG_TOKEN_QUERY_AUTH_IDS             0xff2fUL
-+	#define HWRM_DBG_TOKEN_CFG                        0xff30UL
-+	#define HWRM_NVM_GET_VPD_FIELD_INFO               0xffeaUL
-+	#define HWRM_NVM_SET_VPD_FIELD_INFO               0xffebUL
-+	#define HWRM_NVM_DEFRAG                           0xffecUL
-+	#define HWRM_NVM_REQ_ARBITRATION                  0xffedUL
-+	#define HWRM_NVM_FACTORY_DEFAULTS                 0xffeeUL
-+	#define HWRM_NVM_VALIDATE_OPTION                  0xffefUL
-+	#define HWRM_NVM_FLUSH                            0xfff0UL
-+	#define HWRM_NVM_GET_VARIABLE                     0xfff1UL
-+	#define HWRM_NVM_SET_VARIABLE                     0xfff2UL
-+	#define HWRM_NVM_INSTALL_UPDATE                   0xfff3UL
-+	#define HWRM_NVM_MODIFY                           0xfff4UL
-+	#define HWRM_NVM_VERIFY_UPDATE                    0xfff5UL
-+	#define HWRM_NVM_GET_DEV_INFO                     0xfff6UL
-+	#define HWRM_NVM_ERASE_DIR_ENTRY                  0xfff7UL
-+	#define HWRM_NVM_MOD_DIR_ENTRY                    0xfff8UL
-+	#define HWRM_NVM_FIND_DIR_ENTRY                   0xfff9UL
-+	#define HWRM_NVM_GET_DIR_ENTRIES                  0xfffaUL
-+	#define HWRM_NVM_GET_DIR_INFO                     0xfffbUL
-+	#define HWRM_NVM_RAW_DUMP                         0xfffcUL
-+	#define HWRM_NVM_READ                             0xfffdUL
-+	#define HWRM_NVM_WRITE                            0xfffeUL
-+	#define HWRM_NVM_RAW_WRITE_BLK                    0xffffUL
-+	#define HWRM_LAST                                HWRM_NVM_RAW_WRITE_BLK
-+	__le16	unused_0[3];
-+};
-+
-+/* ret_codes (size:64b/8B) */
-+struct ret_codes {
-+	__le16	error_code;
-+	#define HWRM_ERR_CODE_SUCCESS                      0x0UL
-+	#define HWRM_ERR_CODE_FAIL                         0x1UL
-+	#define HWRM_ERR_CODE_INVALID_PARAMS               0x2UL
-+	#define HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED       0x3UL
-+	#define HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR         0x4UL
-+	#define HWRM_ERR_CODE_INVALID_FLAGS                0x5UL
-+	#define HWRM_ERR_CODE_INVALID_ENABLES              0x6UL
-+	#define HWRM_ERR_CODE_UNSUPPORTED_TLV              0x7UL
-+	#define HWRM_ERR_CODE_NO_BUFFER                    0x8UL
-+	#define HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR       0x9UL
-+	#define HWRM_ERR_CODE_HOT_RESET_PROGRESS           0xaUL
-+	#define HWRM_ERR_CODE_HOT_RESET_FAIL               0xbUL
-+	#define HWRM_ERR_CODE_NO_FLOW_COUNTER_DURING_ALLOC 0xcUL
-+	#define HWRM_ERR_CODE_KEY_HASH_COLLISION           0xdUL
-+	#define HWRM_ERR_CODE_KEY_ALREADY_EXISTS           0xeUL
-+	#define HWRM_ERR_CODE_HWRM_ERROR                   0xfUL
-+	#define HWRM_ERR_CODE_BUSY                         0x10UL
-+	#define HWRM_ERR_CODE_RESOURCE_LOCKED              0x11UL
-+	#define HWRM_ERR_CODE_PF_UNAVAILABLE               0x12UL
-+	#define HWRM_ERR_CODE_ENTITY_NOT_PRESENT           0x13UL
-+	#define HWRM_ERR_CODE_SECURE_SOC_ERROR             0x14UL
-+	#define HWRM_ERR_CODE_TLV_ENCAPSULATED_RESPONSE    0x8000UL
-+	#define HWRM_ERR_CODE_UNKNOWN_ERR                  0xfffeUL
-+	#define HWRM_ERR_CODE_CMD_NOT_SUPPORTED            0xffffUL
-+	#define HWRM_ERR_CODE_LAST                        HWRM_ERR_CODE_CMD_NOT_SUPPORTED
-+	__le16	unused_0[3];
-+};
-+
-+/* hwrm_err_output (size:128b/16B) */
-+struct hwrm_err_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	opaque_0;
-+	__le16	opaque_1;
-+	u8	cmd_err;
-+	u8	valid;
-+};
-+#define HWRM_NA_SIGNATURE ((__le32)(-1))
-+#define HWRM_MAX_REQ_LEN 128
-+#define HWRM_MAX_RESP_LEN 704
-+#define HW_HASH_INDEX_SIZE 0x80
-+#define HW_HASH_KEY_SIZE 40
-+#define HWRM_RESP_VALID_KEY 1
-+#define HWRM_TARGET_ID_BONO 0xFFF8
-+#define HWRM_TARGET_ID_KONG 0xFFF9
-+#define HWRM_TARGET_ID_APE 0xFFFA
-+#define HWRM_TARGET_ID_TOOLS 0xFFFD
-+#define HWRM_VERSION_MAJOR 1
-+#define HWRM_VERSION_MINOR 10
-+#define HWRM_VERSION_UPDATE 3
-+#define HWRM_VERSION_RSVD 133
-+#define HWRM_VERSION_STR "1.10.3.133"
-+
-+/* hwrm_ver_get_input (size:192b/24B) */
-+struct hwrm_ver_get_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	hwrm_intf_maj;
-+	u8	hwrm_intf_min;
-+	u8	hwrm_intf_upd;
-+	u8	unused_0[5];
-+};
-+
-+/* hwrm_ver_get_output (size:1408b/176B) */
-+struct hwrm_ver_get_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	hwrm_intf_maj_8b;
-+	u8	hwrm_intf_min_8b;
-+	u8	hwrm_intf_upd_8b;
-+	u8	hwrm_intf_rsvd_8b;
-+	u8	hwrm_fw_maj_8b;
-+	u8	hwrm_fw_min_8b;
-+	u8	hwrm_fw_bld_8b;
-+	u8	hwrm_fw_rsvd_8b;
-+	u8	mgmt_fw_maj_8b;
-+	u8	mgmt_fw_min_8b;
-+	u8	mgmt_fw_bld_8b;
-+	u8	mgmt_fw_rsvd_8b;
-+	u8	netctrl_fw_maj_8b;
-+	u8	netctrl_fw_min_8b;
-+	u8	netctrl_fw_bld_8b;
-+	u8	netctrl_fw_rsvd_8b;
-+	__le32	dev_caps_cfg;
-+	#define VER_GET_RESP_DEV_CAPS_CFG_SECURE_FW_UPD_SUPPORTED                  0x1UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_FW_DCBX_AGENT_SUPPORTED                  0x2UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED                      0x4UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED                       0x8UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED                   0x10UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED              0x20UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_L2_FILTER_TYPES_ROCE_OR_L2_SUPPORTED     0x40UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_VIRTIO_VSWITCH_OFFLOAD_SUPPORTED         0x80UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED                     0x100UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_FLOW_AGING_SUPPORTED                     0x200UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_ADV_FLOW_COUNTERS_SUPPORTED              0x400UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_CFA_EEM_SUPPORTED                        0x800UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED              0x1000UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_CFA_TFLIB_SUPPORTED                      0x2000UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_CFA_TRUFLOW_SUPPORTED                    0x4000UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_SECURE_BOOT_CAPABLE                      0x8000UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_SECURE_SOC_CAPABLE                       0x10000UL
-+	#define VER_GET_RESP_DEV_CAPS_CFG_DEBUG_TOKEN_SUPPORTED                    0x20000UL
-+	u8	roce_fw_maj_8b;
-+	u8	roce_fw_min_8b;
-+	u8	roce_fw_bld_8b;
-+	u8	roce_fw_rsvd_8b;
-+	char	hwrm_fw_name[16];
-+	char	mgmt_fw_name[16];
-+	char	netctrl_fw_name[16];
-+	char	active_pkg_name[16];
-+	char	roce_fw_name[16];
-+	__le16	chip_num;
-+	u8	chip_rev;
-+	u8	chip_metal;
-+	u8	chip_bond_id;
-+	u8	chip_platform_type;
-+	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_ASIC      0x0UL
-+	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_FPGA      0x1UL
-+	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_PALLADIUM 0x2UL
-+	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_LAST     VER_GET_RESP_CHIP_PLATFORM_TYPE_PALLADIUM
-+	__le16	max_req_win_len;
-+	__le16	max_resp_len;
-+	__le16	def_req_timeout;
-+	u8	flags;
-+	#define VER_GET_RESP_FLAGS_DEV_NOT_RDY                   0x1UL
-+	#define VER_GET_RESP_FLAGS_EXT_VER_AVAIL                 0x2UL
-+	#define VER_GET_RESP_FLAGS_DEV_NOT_RDY_BACKING_STORE     0x4UL
-+	u8	unused_0[2];
-+	u8	always_1;
-+	__le16	hwrm_intf_major;
-+	__le16	hwrm_intf_minor;
-+	__le16	hwrm_intf_build;
-+	__le16	hwrm_intf_patch;
-+	__le16	hwrm_fw_major;
-+	__le16	hwrm_fw_minor;
-+	__le16	hwrm_fw_build;
-+	__le16	hwrm_fw_patch;
-+	__le16	mgmt_fw_major;
-+	__le16	mgmt_fw_minor;
-+	__le16	mgmt_fw_build;
-+	__le16	mgmt_fw_patch;
-+	__le16	netctrl_fw_major;
-+	__le16	netctrl_fw_minor;
-+	__le16	netctrl_fw_build;
-+	__le16	netctrl_fw_patch;
-+	__le16	roce_fw_major;
-+	__le16	roce_fw_minor;
-+	__le16	roce_fw_build;
-+	__le16	roce_fw_patch;
-+	__le16	max_ext_req_len;
-+	__le16	max_req_timeout;
-+	u8	unused_1[3];
-+	u8	valid;
-+};
-+
-+/* eject_cmpl (size:128b/16B) */
-+struct eject_cmpl {
-+	__le16	type;
-+	#define EJECT_CMPL_TYPE_MASK       0x3fUL
-+	#define EJECT_CMPL_TYPE_SFT        0
-+	#define EJECT_CMPL_TYPE_STAT_EJECT   0x1aUL
-+	#define EJECT_CMPL_TYPE_LAST        EJECT_CMPL_TYPE_STAT_EJECT
-+	#define EJECT_CMPL_FLAGS_MASK      0xffc0UL
-+	#define EJECT_CMPL_FLAGS_SFT       6
-+	#define EJECT_CMPL_FLAGS_ERROR      0x40UL
-+	__le16	len;
-+	__le32	opaque;
-+	__le16	v;
-+	#define EJECT_CMPL_V                              0x1UL
-+	#define EJECT_CMPL_ERRORS_MASK                    0xfffeUL
-+	#define EJECT_CMPL_ERRORS_SFT                     1
-+	#define EJECT_CMPL_ERRORS_BUFFER_ERROR_MASK        0xeUL
-+	#define EJECT_CMPL_ERRORS_BUFFER_ERROR_SFT         1
-+	#define EJECT_CMPL_ERRORS_BUFFER_ERROR_NO_BUFFER     (0x0UL << 1)
-+	#define EJECT_CMPL_ERRORS_BUFFER_ERROR_DID_NOT_FIT   (0x1UL << 1)
-+	#define EJECT_CMPL_ERRORS_BUFFER_ERROR_BAD_FORMAT    (0x3UL << 1)
-+	#define EJECT_CMPL_ERRORS_BUFFER_ERROR_FLUSH         (0x5UL << 1)
-+	#define EJECT_CMPL_ERRORS_BUFFER_ERROR_LAST         EJECT_CMPL_ERRORS_BUFFER_ERROR_FLUSH
-+	__le16	reserved16;
-+	__le32	unused_2;
-+};
-+
-+/* hwrm_cmpl (size:128b/16B) */
-+struct hwrm_cmpl {
-+	__le16	type;
-+	#define CMPL_TYPE_MASK     0x3fUL
-+	#define CMPL_TYPE_SFT      0
-+	#define CMPL_TYPE_HWRM_DONE  0x20UL
-+	#define CMPL_TYPE_LAST      CMPL_TYPE_HWRM_DONE
-+	__le16	sequence_id;
-+	__le32	unused_1;
-+	__le32	v;
-+	#define CMPL_V     0x1UL
-+	__le32	unused_3;
-+};
-+
-+/* hwrm_fwd_req_cmpl (size:128b/16B) */
-+struct hwrm_fwd_req_cmpl {
-+	__le16	req_len_type;
-+	#define FWD_REQ_CMPL_TYPE_MASK        0x3fUL
-+	#define FWD_REQ_CMPL_TYPE_SFT         0
-+	#define FWD_REQ_CMPL_TYPE_HWRM_FWD_REQ  0x22UL
-+	#define FWD_REQ_CMPL_TYPE_LAST         FWD_REQ_CMPL_TYPE_HWRM_FWD_REQ
-+	#define FWD_REQ_CMPL_REQ_LEN_MASK     0xffc0UL
-+	#define FWD_REQ_CMPL_REQ_LEN_SFT      6
-+	__le16	source_id;
-+	__le32	unused0;
-+	__le32	req_buf_addr_v[2];
-+	#define FWD_REQ_CMPL_V                0x1UL
-+	#define FWD_REQ_CMPL_REQ_BUF_ADDR_MASK 0xfffffffeUL
-+	#define FWD_REQ_CMPL_REQ_BUF_ADDR_SFT 1
-+};
-+
-+/* hwrm_fwd_resp_cmpl (size:128b/16B) */
-+struct hwrm_fwd_resp_cmpl {
-+	__le16	type;
-+	#define FWD_RESP_CMPL_TYPE_MASK         0x3fUL
-+	#define FWD_RESP_CMPL_TYPE_SFT          0
-+	#define FWD_RESP_CMPL_TYPE_HWRM_FWD_RESP  0x24UL
-+	#define FWD_RESP_CMPL_TYPE_LAST          FWD_RESP_CMPL_TYPE_HWRM_FWD_RESP
-+	__le16	source_id;
-+	__le16	resp_len;
-+	__le16	unused_1;
-+	__le32	resp_buf_addr_v[2];
-+	#define FWD_RESP_CMPL_V                 0x1UL
-+	#define FWD_RESP_CMPL_RESP_BUF_ADDR_MASK 0xfffffffeUL
-+	#define FWD_RESP_CMPL_RESP_BUF_ADDR_SFT 1
-+};
-+
-+/* hwrm_async_event_cmpl (size:128b/16B) */
-+struct hwrm_async_event_cmpl {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_TYPE_LAST             ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE              0x0UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_MTU_CHANGE                 0x1UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE               0x2UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE               0x3UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED           0x4UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED      0x5UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE           0x6UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE             0x7UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY                    0x8UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY                  0x9UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG                0xaUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_UNLOAD                0x10UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_LOAD                  0x11UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_FUNC_FLR_PROC_CMPLT             0x12UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD                  0x20UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_LOAD                    0x21UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_VF_FLR                          0x30UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_VF_MAC_ADDR_CHANGE              0x31UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_PF_VF_COMM_STATUS_CHANGE        0x32UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE                   0x33UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_LLFC_PFC_CHANGE                 0x34UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_DEFAULT_VNIC_CHANGE             0x35UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_HW_FLOW_AGED                    0x36UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION              0x37UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_EEM_CACHE_FLUSH_REQ             0x38UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_EEM_CACHE_FLUSH_DONE            0x39UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_TCP_FLAG_ACTION_CHANGE          0x3aUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_EEM_FLOW_ACTIVE                 0x3bUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_EEM_CFG_CHANGE                  0x3cUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_TFLIB_DEFAULT_VNIC_CHANGE       0x3dUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_TFLIB_LINK_STATUS_CHANGE        0x3eUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_QUIESCE_DONE                    0x3fUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE               0x40UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_PFC_WATCHDOG_CFG_CHANGE         0x41UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST                    0x42UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE                      0x43UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP                   0x44UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT                    0x45UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD       0x46UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_RSS_CHANGE                      0x47UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE       0x48UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_HW_DOORBELL_RECOVERY_READ_ERROR 0x49UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_CTX_ERROR                       0x4aUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_UDCC_SESSION_CHANGE             0x4bUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_DBG_BUF_PRODUCER                0x4cUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_PEER_MMAP_CHANGE                0x4dUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_REPRESENTOR_PAIR_CHANGE         0x4eUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_VF_STAT_CHANGE                  0x4fUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_HOST_COREDUMP                   0x50UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_ADPTV_QOS                       0x51UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_MAX_RGTR_EVENT_ID               0x52UL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_FW_TRACE_MSG                    0xfeUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR                      0xffUL
-+	#define ASYNC_EVENT_CMPL_EVENT_ID_LAST                           ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+};
-+
-+/* hwrm_async_event_cmpl_link_status_change (size:128b/16B) */
-+struct hwrm_async_event_cmpl_link_status_change {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_LAST             ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_ID_LINK_STATUS_CHANGE 0x0UL
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_ID_LAST              ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_ID_LINK_STATUS_CHANGE
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE     0x1UL
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE_DOWN  0x0UL
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE_UP    0x1UL
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE_LAST ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE_UP
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_MASK       0xeUL
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_SFT        1
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_ID_MASK    0xffff0UL
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_ID_SFT     4
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PF_ID_MASK      0xff00000UL
-+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PF_ID_SFT       20
-+};
-+
-+/* hwrm_async_event_cmpl_port_conn_not_allowed (size:128b/16B) */
-+struct hwrm_async_event_cmpl_port_conn_not_allowed {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_LAST             ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_ID_PORT_CONN_NOT_ALLOWED 0x4UL
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_ID_LAST                 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_ID_PORT_CONN_NOT_ALLOWED
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK                 0xffffUL
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_SFT                  0
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_MASK      0xff0000UL
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_SFT       16
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_NONE        (0x0UL << 16)
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_DISABLETX   (0x1UL << 16)
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_WARNINGMSG  (0x2UL << 16)
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_PWRDOWN     (0x3UL << 16)
-+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_LAST       ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_PWRDOWN
-+};
-+
-+/* hwrm_async_event_cmpl_link_speed_cfg_change (size:128b/16B) */
-+struct hwrm_async_event_cmpl_link_speed_cfg_change {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_LAST             ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_ID_LINK_SPEED_CFG_CHANGE 0x6UL
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_ID_LAST                 ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_ID_LINK_SPEED_CFG_CHANGE
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_DATA1_PORT_ID_MASK                     0xffffUL
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_DATA1_PORT_ID_SFT                      0
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_DATA1_SUPPORTED_LINK_SPEEDS_CHANGE     0x10000UL
-+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_DATA1_ILLEGAL_LINK_SPEED_CFG           0x20000UL
-+};
-+
-+/* hwrm_async_event_cmpl_reset_notify (size:128b/16B) */
-+struct hwrm_async_event_cmpl_reset_notify {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_TYPE_LAST             ASYNC_EVENT_CMPL_RESET_NOTIFY_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_ID_RESET_NOTIFY 0x8UL
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_ID_LAST        ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_ID_RESET_NOTIFY
-+	__le32	event_data2;
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA2_FW_STATUS_CODE_MASK 0xffffUL
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA2_FW_STATUS_CODE_SFT 0
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DRIVER_ACTION_MASK                  0xffUL
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DRIVER_ACTION_SFT                   0
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DRIVER_ACTION_DRIVER_STOP_TX_QUEUE    0x1UL
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DRIVER_ACTION_DRIVER_IFDOWN           0x2UL
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DRIVER_ACTION_LAST                   ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DRIVER_ACTION_DRIVER_IFDOWN
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_MASK                    0xff00UL
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_SFT                     8
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_MANAGEMENT_RESET_REQUEST  (0x1UL << 8)
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_EXCEPTION_FATAL        (0x2UL << 8)
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_EXCEPTION_NON_FATAL    (0x3UL << 8)
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FAST_RESET                (0x4UL << 8)
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_ACTIVATION             (0x5UL << 8)
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_LAST                     ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_ACTIVATION
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DELAY_IN_100MS_TICKS_MASK           0xffff0000UL
-+	#define ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DELAY_IN_100MS_TICKS_SFT            16
-+};
-+
-+/* hwrm_async_event_cmpl_error_recovery (size:128b/16B) */
-+struct hwrm_async_event_cmpl_error_recovery {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_TYPE_LAST             ASYNC_EVENT_CMPL_ERROR_RECOVERY_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_ID_ERROR_RECOVERY 0x9UL
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_ID_LAST          ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_ID_ERROR_RECOVERY
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_DATA1_FLAGS_MASK                 0xffUL
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_DATA1_FLAGS_SFT                  0
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_DATA1_FLAGS_MASTER_FUNC           0x1UL
-+	#define ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_DATA1_FLAGS_RECOVERY_ENABLED      0x2UL
-+};
-+
-+/* hwrm_async_event_cmpl_ring_monitor_msg (size:128b/16B) */
-+struct hwrm_async_event_cmpl_ring_monitor_msg {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_TYPE_LAST             ASYNC_EVENT_CMPL_RING_MONITOR_MSG_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_ID_RING_MONITOR_MSG 0xaUL
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_ID_LAST            ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_ID_RING_MONITOR_MSG
-+	__le32	event_data2;
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK 0xffUL
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_SFT 0
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_TX    0x0UL
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX    0x1UL
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_CMPL  0x2UL
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_LAST ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_CMPL
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_RING_MONITOR_MSG_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+};
-+
-+/* hwrm_async_event_cmpl_vf_cfg_change (size:128b/16B) */
-+struct hwrm_async_event_cmpl_vf_cfg_change {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_LAST             ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_ID_VF_CFG_CHANGE 0x33UL
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_ID_LAST         ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_ID_VF_CFG_CHANGE
-+	__le32	event_data2;
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA2_VF_ID_MASK 0xffffUL
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA2_VF_ID_SFT 0
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_MTU_CHANGE                0x1UL
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_MRU_CHANGE                0x2UL
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_DFLT_MAC_ADDR_CHANGE      0x4UL
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_DFLT_VLAN_CHANGE          0x8UL
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_TRUSTED_VF_CFG_CHANGE     0x10UL
-+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_TF_OWNERSHIP_RELEASE      0x20UL
-+};
-+
-+/* hwrm_async_event_cmpl_default_vnic_change (size:128b/16B) */
-+struct hwrm_async_event_cmpl_default_vnic_change {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_TYPE_LAST             ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_TYPE_HWRM_ASYNC_EVENT
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_UNUSED1_MASK         0xffc0UL
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_UNUSED1_SFT          6
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_ID_ALLOC_FREE_NOTIFICATION 0x35UL
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_ID_LAST                   ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_ID_ALLOC_FREE_NOTIFICATION
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_DEF_VNIC_STATE_MASK          0x3UL
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_DEF_VNIC_STATE_SFT           0
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_DEF_VNIC_STATE_DEF_VNIC_ALLOC  0x1UL
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_DEF_VNIC_STATE_DEF_VNIC_FREE   0x2UL
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_DEF_VNIC_STATE_LAST           ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_DEF_VNIC_STATE_DEF_VNIC_FREE
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_PF_ID_MASK                   0x3fcUL
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_PF_ID_SFT                    2
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_VF_ID_MASK                   0x3fffc00UL
-+	#define ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_VF_ID_SFT                    10
-+};
-+
-+/* hwrm_async_event_cmpl_hw_flow_aged (size:128b/16B) */
-+struct hwrm_async_event_cmpl_hw_flow_aged {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_TYPE_LAST             ASYNC_EVENT_CMPL_HW_FLOW_AGED_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_EVENT_ID_HW_FLOW_AGED 0x36UL
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_EVENT_ID_LAST        ASYNC_EVENT_CMPL_HW_FLOW_AGED_EVENT_ID_HW_FLOW_AGED
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_EVENT_DATA1_FLOW_ID_MASK       0x7fffffffUL
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_EVENT_DATA1_FLOW_ID_SFT        0
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_EVENT_DATA1_FLOW_DIRECTION     0x80000000UL
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_EVENT_DATA1_FLOW_DIRECTION_RX    (0x0UL << 31)
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_EVENT_DATA1_FLOW_DIRECTION_TX    (0x1UL << 31)
-+	#define ASYNC_EVENT_CMPL_HW_FLOW_AGED_EVENT_DATA1_FLOW_DIRECTION_LAST ASYNC_EVENT_CMPL_HW_FLOW_AGED_EVENT_DATA1_FLOW_DIRECTION_TX
-+};
-+
-+/* hwrm_async_event_cmpl_eem_cache_flush_req (size:128b/16B) */
-+struct hwrm_async_event_cmpl_eem_cache_flush_req {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_REQ_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_REQ_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_REQ_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_REQ_TYPE_LAST             ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_REQ_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_REQ_EVENT_ID_EEM_CACHE_FLUSH_REQ 0x38UL
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_REQ_EVENT_ID_LAST               ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_REQ_EVENT_ID_EEM_CACHE_FLUSH_REQ
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_REQ_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_REQ_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_REQ_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+};
-+
-+/* hwrm_async_event_cmpl_eem_cache_flush_done (size:128b/16B) */
-+struct hwrm_async_event_cmpl_eem_cache_flush_done {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_TYPE_LAST             ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_EVENT_ID_EEM_CACHE_FLUSH_DONE 0x39UL
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_EVENT_ID_LAST                ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_EVENT_ID_EEM_CACHE_FLUSH_DONE
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_EVENT_DATA1_FID_MASK 0xffffUL
-+	#define ASYNC_EVENT_CMPL_EEM_CACHE_FLUSH_DONE_EVENT_DATA1_FID_SFT 0
-+};
-+
-+/* hwrm_async_event_cmpl_deferred_response (size:128b/16B) */
-+struct hwrm_async_event_cmpl_deferred_response {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_TYPE_LAST             ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_EVENT_ID_DEFERRED_RESPONSE 0x40UL
-+	#define ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_EVENT_ID_LAST             ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_EVENT_ID_DEFERRED_RESPONSE
-+	__le32	event_data2;
-+	#define ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_EVENT_DATA2_SEQ_ID_MASK 0xffffUL
-+	#define ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_EVENT_DATA2_SEQ_ID_SFT 0
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_DEFERRED_RESPONSE_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+};
-+
-+/* hwrm_async_event_cmpl_echo_request (size:128b/16B) */
-+struct hwrm_async_event_cmpl_echo_request {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_ECHO_REQUEST_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_ECHO_REQUEST_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_ECHO_REQUEST_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_ECHO_REQUEST_TYPE_LAST             ASYNC_EVENT_CMPL_ECHO_REQUEST_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_ECHO_REQUEST_EVENT_ID_ECHO_REQUEST 0x42UL
-+	#define ASYNC_EVENT_CMPL_ECHO_REQUEST_EVENT_ID_LAST        ASYNC_EVENT_CMPL_ECHO_REQUEST_EVENT_ID_ECHO_REQUEST
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_ECHO_REQUEST_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_ECHO_REQUEST_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_ECHO_REQUEST_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+};
-+
-+/* hwrm_async_event_cmpl_phc_update (size:128b/16B) */
-+struct hwrm_async_event_cmpl_phc_update {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_TYPE_LAST             ASYNC_EVENT_CMPL_PHC_UPDATE_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_ID_PHC_UPDATE 0x43UL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_ID_LAST      ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_ID_PHC_UPDATE
-+	__le32	event_data2;
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA2_PHC_MASTER_FID_MASK 0xffffUL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA2_PHC_MASTER_FID_SFT 0
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA2_PHC_SEC_FID_MASK   0xffff0000UL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA2_PHC_SEC_FID_SFT    16
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK          0xfUL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT           0
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_MASTER      0x1UL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_SECONDARY   0x2UL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_FAILOVER    0x3UL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE  0x4UL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_LAST           ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK   0xffff0UL
-+	#define ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT    4
-+};
-+
-+/* hwrm_async_event_cmpl_pps_timestamp (size:128b/16B) */
-+struct hwrm_async_event_cmpl_pps_timestamp {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_TYPE_LAST             ASYNC_EVENT_CMPL_PPS_TIMESTAMP_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_ID_PPS_TIMESTAMP 0x44UL
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_ID_LAST         ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_ID_PPS_TIMESTAMP
-+	__le32	event_data2;
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_EVENT_TYPE              0x1UL
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_EVENT_TYPE_INTERNAL       0x0UL
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_EVENT_TYPE_EXTERNAL       0x1UL
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_EVENT_TYPE_LAST          ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_EVENT_TYPE_EXTERNAL
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_PIN_NUMBER_MASK         0xeUL
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_PIN_NUMBER_SFT          1
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_PPS_TIMESTAMP_UPPER_MASK 0xffff0UL
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_PPS_TIMESTAMP_UPPER_SFT 4
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA1_PPS_TIMESTAMP_LOWER_MASK 0xffffffffUL
-+	#define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA1_PPS_TIMESTAMP_LOWER_SFT 0
-+};
-+
-+/* hwrm_async_event_cmpl_error_report (size:128b/16B) */
-+struct hwrm_async_event_cmpl_error_report {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_TYPE_LAST             ASYNC_EVENT_CMPL_ERROR_REPORT_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_EVENT_ID_ERROR_REPORT 0x45UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_EVENT_ID_LAST        ASYNC_EVENT_CMPL_ERROR_REPORT_EVENT_ID_ERROR_REPORT
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_EVENT_DATA1_ERROR_TYPE_MASK 0xffUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_EVENT_DATA1_ERROR_TYPE_SFT 0
-+};
-+
-+/* hwrm_async_event_cmpl_dbg_buf_producer (size:128b/16B) */
-+struct hwrm_async_event_cmpl_dbg_buf_producer {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_TYPE_LAST             ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_ID_DBG_BUF_PRODUCER 0x4cUL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_ID_LAST            ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_ID_DBG_BUF_PRODUCER
-+	__le32	event_data2;
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA2_CURR_OFF_MASK 0xffffffffUL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA2_CURR_OFF_SFT 0
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_MASK               0xffffUL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_SFT                0
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_SRT_TRACE            0x0UL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_SRT2_TRACE           0x1UL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_CRT_TRACE            0x2UL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_CRT2_TRACE           0x3UL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_RIGP0_TRACE          0x4UL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_L2_HWRM_TRACE        0x5UL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_ROCE_HWRM_TRACE      0x6UL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_CA0_TRACE            0x7UL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_CA1_TRACE            0x8UL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_CA2_TRACE            0x9UL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_RIGP1_TRACE          0xaUL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_AFM_KONG_HWRM_TRACE  0xbUL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_ERR_QPC_TRACE        0xcUL
-+	#define ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_LAST                ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_ERR_QPC_TRACE
-+};
-+
-+/* hwrm_async_event_cmpl_hwrm_error (size:128b/16B) */
-+struct hwrm_async_event_cmpl_hwrm_error {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_LAST             ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_ID_HWRM_ERROR 0xffUL
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_ID_LAST      ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_ID_HWRM_ERROR
-+	__le32	event_data2;
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_MASK    0xffUL
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_SFT     0
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_WARNING   0x0UL
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_NONFATAL  0x1UL
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_FATAL     0x2UL
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_LAST     ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_FATAL
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA1_TIMESTAMP     0x1UL
-+};
-+
-+/* hwrm_async_event_cmpl_error_report_base (size:128b/16B) */
-+struct hwrm_async_event_cmpl_error_report_base {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_TYPE_LAST             ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_ID_ERROR_REPORT 0x45UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_ID_LAST        ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_ID_ERROR_REPORT
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_MASK                        0xffUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_SFT                         0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_RESERVED                      0x0UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM                   0x1UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL                0x2UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_NVM                           0x3UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD       0x4UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_THERMAL_THRESHOLD             0x5UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DUAL_DATA_RATE_NOT_SUPPORTED  0x6UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DUP_UDCC_SES                  0x7UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DB_DROP                       0x8UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_MD_TEMP                       0x9UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_VNIC_ERR                      0xaUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_LAST                         ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_VNIC_ERR
-+};
-+
-+/* hwrm_async_event_cmpl_error_report_pause_storm (size:128b/16B) */
-+struct hwrm_async_event_cmpl_error_report_pause_storm {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_TYPE_LAST             ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_ID_ERROR_REPORT 0x45UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_ID_LAST        ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_ID_ERROR_REPORT
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_DATA1_ERROR_TYPE_MASK       0xffUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_DATA1_ERROR_TYPE_SFT        0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM  0x1UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_DATA1_ERROR_TYPE_LAST        ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM
-+};
-+
-+/* hwrm_async_event_cmpl_error_report_invalid_signal (size:128b/16B) */
-+struct hwrm_async_event_cmpl_error_report_invalid_signal {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_TYPE_LAST             ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_ID_ERROR_REPORT 0x45UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_ID_LAST        ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_ID_ERROR_REPORT
-+	__le32	event_data2;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA2_PIN_ID_MASK 0xffUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA2_PIN_ID_SFT 0
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA1_ERROR_TYPE_MASK          0xffUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA1_ERROR_TYPE_SFT           0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL  0x2UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA1_ERROR_TYPE_LAST           ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL
-+};
-+
-+/* hwrm_async_event_cmpl_error_report_nvm (size:128b/16B) */
-+struct hwrm_async_event_cmpl_error_report_nvm {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_TYPE_LAST             ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_ID_ERROR_REPORT 0x45UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_ID_LAST        ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_ID_ERROR_REPORT
-+	__le32	event_data2;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA2_ERR_ADDR_MASK 0xffffffffUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA2_ERR_ADDR_SFT 0
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_ERROR_TYPE_MASK     0xffUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_ERROR_TYPE_SFT      0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_ERROR_TYPE_NVM_ERROR  0x3UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_ERROR_TYPE_LAST      ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_ERROR_TYPE_NVM_ERROR
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_MASK   0xff00UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_SFT    8
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_WRITE    (0x1UL << 8)
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_ERASE    (0x2UL << 8)
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_LAST    ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_ERASE
-+};
-+
-+/* hwrm_async_event_cmpl_error_report_doorbell_drop_threshold (size:128b/16B) */
-+struct hwrm_async_event_cmpl_error_report_doorbell_drop_threshold {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_TYPE_LAST             ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_ID_ERROR_REPORT 0x45UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_ID_LAST        ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_ID_ERROR_REPORT
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_DATA1_ERROR_TYPE_MASK                   0xffUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_DATA1_ERROR_TYPE_SFT                    0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD  0x4UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_DATA1_ERROR_TYPE_LAST                    ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_DATA1_EPOCH_MASK                        0xffffff00UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_DATA1_EPOCH_SFT                         8
-+};
-+
-+/* hwrm_async_event_cmpl_error_report_thermal (size:128b/16B) */
-+struct hwrm_async_event_cmpl_error_report_thermal {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_TYPE_LAST             ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_ID_ERROR_REPORT 0x45UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_ID_LAST        ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_ID_ERROR_REPORT
-+	__le32	event_data2;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_CURRENT_TEMP_MASK  0xffUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_CURRENT_TEMP_SFT   0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_MASK 0xff00UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_SFT 8
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_ERROR_TYPE_MASK          0xffUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_ERROR_TYPE_SFT           0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_ERROR_TYPE_THERMAL_EVENT   0x5UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_ERROR_TYPE_LAST           ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_ERROR_TYPE_THERMAL_EVENT
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_MASK      0x700UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_SFT       8
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_WARN        (0x0UL << 8)
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_CRITICAL    (0x1UL << 8)
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_FATAL       (0x2UL << 8)
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_SHUTDOWN    (0x3UL << 8)
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_LAST       ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_SHUTDOWN
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR           0x800UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_DECREASING  (0x0UL << 11)
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_INCREASING  (0x1UL << 11)
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_LAST       ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_INCREASING
-+};
-+
-+/* hwrm_async_event_cmpl_error_report_dual_data_rate_not_supported (size:128b/16B) */
-+struct hwrm_async_event_cmpl_error_report_dual_data_rate_not_supported {
-+	__le16	type;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_TYPE_MASK            0x3fUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_TYPE_SFT             0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_TYPE_HWRM_ASYNC_EVENT  0x2eUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_TYPE_LAST             ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_TYPE_HWRM_ASYNC_EVENT
-+	__le16	event_id;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_EVENT_ID_ERROR_REPORT 0x45UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_EVENT_ID_LAST        ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_EVENT_ID_ERROR_REPORT
-+	__le32	event_data2;
-+	u8	opaque_v;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_V          0x1UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_OPAQUE_MASK 0xfeUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_OPAQUE_SFT 1
-+	u8	timestamp_lo;
-+	__le16	timestamp_hi;
-+	__le32	event_data1;
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_EVENT_DATA1_ERROR_TYPE_MASK                        0xffUL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_EVENT_DATA1_ERROR_TYPE_SFT                         0
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_EVENT_DATA1_ERROR_TYPE_DUAL_DATA_RATE_NOT_SUPPORTED  0x6UL
-+	#define ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_EVENT_DATA1_ERROR_TYPE_LAST                         ASYNC_EVENT_CMPL_ERROR_REPORT_DUAL_DATA_RATE_NOT_SUPPORTED_EVENT_DATA1_ERROR_TYPE_DUAL_DATA_RATE_NOT_SUPPORTED
-+};
-+
-+/* hwrm_func_reset_input (size:192b/24B) */
-+struct hwrm_func_reset_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	#define FUNC_RESET_REQ_ENABLES_VF_ID_VALID     0x1UL
-+	__le16	vf_id;
-+	u8	func_reset_level;
-+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETALL      0x0UL
-+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETME       0x1UL
-+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETCHILDREN 0x2UL
-+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETVF       0x3UL
-+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_LAST         FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETVF
-+	u8	unused_0;
-+};
-+
-+/* hwrm_func_reset_output (size:128b/16B) */
-+struct hwrm_func_reset_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_getfid_input (size:192b/24B) */
-+struct hwrm_func_getfid_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	#define FUNC_GETFID_REQ_ENABLES_PCI_ID     0x1UL
-+	__le16	pci_id;
-+	u8	unused_0[2];
-+};
-+
-+/* hwrm_func_getfid_output (size:128b/16B) */
-+struct hwrm_func_getfid_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	fid;
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_vf_alloc_input (size:192b/24B) */
-+struct hwrm_func_vf_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	#define FUNC_VF_ALLOC_REQ_ENABLES_FIRST_VF_ID     0x1UL
-+	__le16	first_vf_id;
-+	__le16	num_vfs;
-+};
-+
-+/* hwrm_func_vf_alloc_output (size:128b/16B) */
-+struct hwrm_func_vf_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	first_vf_id;
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_vf_free_input (size:192b/24B) */
-+struct hwrm_func_vf_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	#define FUNC_VF_FREE_REQ_ENABLES_FIRST_VF_ID     0x1UL
-+	__le16	first_vf_id;
-+	__le16	num_vfs;
-+};
-+
-+/* hwrm_func_vf_free_output (size:128b/16B) */
-+struct hwrm_func_vf_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_vf_cfg_input (size:576b/72B) */
-+struct hwrm_func_vf_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	#define FUNC_VF_CFG_REQ_ENABLES_MTU                      0x1UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_GUEST_VLAN               0x2UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR           0x4UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_DFLT_MAC_ADDR            0x8UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS          0x10UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS           0x20UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS             0x40UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS             0x80UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS              0x100UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS                0x200UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS            0x400UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS         0x800UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_KTLS_TX_KEY_CTXS     0x1000UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_KTLS_RX_KEY_CTXS     0x2000UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_QUIC_TX_KEY_CTXS     0x4000UL
-+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_QUIC_RX_KEY_CTXS     0x8000UL
-+	__le16	mtu;
-+	__le16	guest_vlan;
-+	__le16	async_event_cr;
-+	u8	dflt_mac_addr[6];
-+	__le32	flags;
-+	#define FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST             0x1UL
-+	#define FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST             0x2UL
-+	#define FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST           0x4UL
-+	#define FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST     0x8UL
-+	#define FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST       0x10UL
-+	#define FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST       0x20UL
-+	#define FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST           0x40UL
-+	#define FUNC_VF_CFG_REQ_FLAGS_L2_CTX_ASSETS_TEST         0x80UL
-+	#define FUNC_VF_CFG_REQ_FLAGS_PPP_PUSH_MODE_ENABLE       0x100UL
-+	#define FUNC_VF_CFG_REQ_FLAGS_PPP_PUSH_MODE_DISABLE      0x200UL
-+	__le16	num_rsscos_ctxs;
-+	__le16	num_cmpl_rings;
-+	__le16	num_tx_rings;
-+	__le16	num_rx_rings;
-+	__le16	num_l2_ctxs;
-+	__le16	num_vnics;
-+	__le16	num_stat_ctxs;
-+	__le16	num_hw_ring_grps;
-+	__le32	num_ktls_tx_key_ctxs;
-+	__le32	num_ktls_rx_key_ctxs;
-+	__le16	num_msix;
-+	u8	unused[2];
-+	__le32	num_quic_tx_key_ctxs;
-+	__le32	num_quic_rx_key_ctxs;
-+};
-+
-+/* hwrm_func_vf_cfg_output (size:128b/16B) */
-+struct hwrm_func_vf_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_qcaps_input (size:192b/24B) */
-+struct hwrm_func_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	fid;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_func_qcaps_output (size:1152b/144B) */
-+struct hwrm_func_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	fid;
-+	__le16	port_id;
-+	__le32	flags;
-+	#define FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED                   0x1UL
-+	#define FUNC_QCAPS_RESP_FLAGS_GLOBAL_MSIX_AUTOMASKING               0x2UL
-+	#define FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED                         0x4UL
-+	#define FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED                     0x8UL
-+	#define FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED                     0x10UL
-+	#define FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED                0x20UL
-+	#define FUNC_QCAPS_RESP_FLAGS_WOL_BMP_SUPPORTED                     0x40UL
-+	#define FUNC_QCAPS_RESP_FLAGS_TX_RING_RL_SUPPORTED                  0x80UL
-+	#define FUNC_QCAPS_RESP_FLAGS_TX_BW_CFG_SUPPORTED                   0x100UL
-+	#define FUNC_QCAPS_RESP_FLAGS_VF_TX_RING_RL_SUPPORTED               0x200UL
-+	#define FUNC_QCAPS_RESP_FLAGS_VF_BW_CFG_SUPPORTED                   0x400UL
-+	#define FUNC_QCAPS_RESP_FLAGS_STD_TX_RING_MODE_SUPPORTED            0x800UL
-+	#define FUNC_QCAPS_RESP_FLAGS_GENEVE_TUN_FLAGS_SUPPORTED            0x1000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_NVGRE_TUN_FLAGS_SUPPORTED             0x2000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_GRE_TUN_FLAGS_SUPPORTED               0x4000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_MPLS_TUN_FLAGS_SUPPORTED              0x8000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED                  0x10000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_ADOPTED_PF_SUPPORTED                  0x20000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_ADMIN_PF_SUPPORTED                    0x40000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_LINK_ADMIN_STATUS_SUPPORTED           0x80000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_WCB_PUSH_MODE                         0x100000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_DYNAMIC_TX_RING_ALLOC                 0x200000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE                     0x400000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE                0x800000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED                   0x1000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD                    0x2000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_NOTIFY_VF_DEF_VNIC_CHNG_SUPPORTED     0x4000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED         0x8000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_COREDUMP_CMD_SUPPORTED                0x10000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_CRASHDUMP_CMD_SUPPORTED               0x20000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_PFC_WD_STATS_SUPPORTED                0x40000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED               0x80000000UL
-+	u8	mac_address[6];
-+	__le16	max_rsscos_ctx;
-+	__le16	max_cmpl_rings;
-+	__le16	max_tx_rings;
-+	__le16	max_rx_rings;
-+	__le16	max_l2_ctxs;
-+	__le16	max_vnics;
-+	__le16	first_vf_id;
-+	__le16	max_vfs;
-+	__le16	max_stat_ctx;
-+	__le32	max_encap_records;
-+	__le32	max_decap_records;
-+	__le32	max_tx_em_flows;
-+	__le32	max_tx_wm_flows;
-+	__le32	max_rx_em_flows;
-+	__le32	max_rx_wm_flows;
-+	__le32	max_mcast_filters;
-+	__le32	max_flow_id;
-+	__le32	max_hw_ring_grps;
-+	__le16	max_sp_tx_rings;
-+	__le16	max_msix_vfs;
-+	__le32	flags_ext;
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_ECN_MARK_SUPPORTED                          0x1UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_ECN_STATS_SUPPORTED                         0x2UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED                      0x4UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT                        0x8UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_PROXY_MODE_SUPPORT                          0x10UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_TX_PROXY_SRC_INTF_OVERRIDE_SUPPORT          0x20UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_SCHQ_SUPPORTED                              0x40UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_PPP_PUSH_MODE_SUPPORTED                     0x80UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_EVB_MODE_CFG_NOT_SUPPORTED                  0x100UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_SOC_SPD_SUPPORTED                           0x200UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED                      0x400UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_FAST_RESET_CAPABLE                          0x800UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_TX_METADATA_CFG_CAPABLE                     0x1000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_NVM_OPTION_ACTION_SUPPORTED                 0x2000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_BD_METADATA_SUPPORTED                       0x4000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_ECHO_REQUEST_SUPPORTED                      0x8000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_NPAR_1_2_SUPPORTED                          0x10000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PTM_SUPPORTED                           0x20000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED                           0x40000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_VF_CFG_ASYNC_FOR_PF_SUPPORTED               0x80000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_PARTITION_BW_SUPPORTED                      0x100000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_DFLT_VLAN_TPID_PCP_SUPPORTED                0x200000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_KTLS_SUPPORTED                              0x400000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_EP_RATE_CONTROL                             0x800000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_MIN_BW_SUPPORTED                            0x1000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_TX_COAL_CMPL_CAP                            0x2000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_BS_V2_SUPPORTED                             0x4000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_BS_V2_REQUIRED                              0x8000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED                     0x10000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_DBR_PACING_SUPPORTED                        0x20000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_HW_DBR_DROP_RECOV_SUPPORTED                 0x40000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT_DISABLE_CQ_OVERFLOW_DETECTION_SUPPORTED     0x80000000UL
-+	u8	max_schqs;
-+	u8	mpc_chnls_cap;
-+	#define FUNC_QCAPS_RESP_MPC_CHNLS_CAP_TCE         0x1UL
-+	#define FUNC_QCAPS_RESP_MPC_CHNLS_CAP_RCE         0x2UL
-+	#define FUNC_QCAPS_RESP_MPC_CHNLS_CAP_TE_CFA      0x4UL
-+	#define FUNC_QCAPS_RESP_MPC_CHNLS_CAP_RE_CFA      0x8UL
-+	#define FUNC_QCAPS_RESP_MPC_CHNLS_CAP_PRIMATE     0x10UL
-+	__le16	max_key_ctxs_alloc;
-+	__le32	flags_ext2;
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED      0x1UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_QUIC_SUPPORTED                        0x2UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_KDNET_SUPPORTED                       0x4UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED              0x8UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_SW_DBR_DROP_RECOVERY_SUPPORTED        0x10UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_GENERIC_STATS_SUPPORTED               0x20UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_UDP_GSO_SUPPORTED                     0x40UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_SYNCE_SUPPORTED                       0x80UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_DBR_PACING_V0_SUPPORTED               0x100UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_TX_PKT_TS_CMPL_SUPPORTED              0x200UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_HW_LAG_SUPPORTED                      0x400UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_ON_CHIP_CTX_SUPPORTED                 0x800UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_STEERING_TAG_SUPPORTED                0x1000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_ENHANCED_VF_SCALE_SUPPORTED           0x2000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_KEY_XID_PARTITION_SUPPORTED           0x4000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_CONCURRENT_KTLS_QUIC_SUPPORTED        0x8000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_SCHQ_CROSS_TC_CAP_SUPPORTED           0x10000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_SCHQ_PER_TC_CAP_SUPPORTED             0x20000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_SCHQ_PER_TC_RESERVATION_SUPPORTED     0x40000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_DB_ERROR_STATS_SUPPORTED              0x80000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_ROCE_VF_RESOURCE_MGMT_SUPPORTED       0x100000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_UDCC_SUPPORTED                        0x200000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_TIMED_TX_SO_TXTIME_SUPPORTED          0x400000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_SW_MAX_RESOURCE_LIMITS_SUPPORTED      0x800000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_TF_INGRESS_NIC_FLOW_SUPPORTED         0x1000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_LPBK_STATS_SUPPORTED                  0x2000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_TF_EGRESS_NIC_FLOW_SUPPORTED          0x4000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_MULTI_LOSSLESS_QUEUES_SUPPORTED       0x8000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_PEER_MMAP_SUPPORTED                   0x10000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_TIMED_TX_PACING_SUPPORTED             0x20000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_VF_STAT_EJECTION_SUPPORTED            0x40000000UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT2_HOST_COREDUMP_SUPPORTED               0x80000000UL
-+	__le16	tunnel_disable_flag;
-+	#define FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_VXLAN      0x1UL
-+	#define FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_NGE        0x2UL
-+	#define FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_NVGRE      0x4UL
-+	#define FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_L2GRE      0x8UL
-+	#define FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_GRE        0x10UL
-+	#define FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_IPINIP     0x20UL
-+	#define FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_MPLS       0x40UL
-+	#define FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_PPPOE      0x80UL
-+	__le16	xid_partition_cap;
-+	#define FUNC_QCAPS_RESP_XID_PARTITION_CAP_TX_CK     0x1UL
-+	#define FUNC_QCAPS_RESP_XID_PARTITION_CAP_RX_CK     0x2UL
-+	u8	device_serial_number[8];
-+	__le16	ctxs_per_partition;
-+	__le16	max_tso_segs;
-+	__le32	roce_vf_max_av;
-+	__le32	roce_vf_max_cq;
-+	__le32	roce_vf_max_mrw;
-+	__le32	roce_vf_max_qp;
-+	__le32	roce_vf_max_srq;
-+	__le32	roce_vf_max_gid;
-+	__le32	flags_ext3;
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT3_RM_RSV_WHILE_ALLOC_CAP            0x1UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT3_REQUIRE_L2_FILTER                 0x2UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT3_MAX_ROCE_VFS_SUPPORTED            0x4UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT3_RX_RATE_PROFILE_SEL_SUPPORTED     0x8UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT3_BIDI_OPT_SUPPORTED                0x10UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT3_MIRROR_ON_ROCE_SUPPORTED          0x20UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT3_ROCE_VF_DYN_ALLOC_SUPPORT         0x40UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT3_CHANGE_UDP_SRCPORT_SUPPORT        0x80UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT3_PCIE_COMPLIANCE_SUPPORTED         0x100UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT3_MULTI_L2_DB_SUPPORTED             0x200UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT3_PCIE_SECURE_ATS_SUPPORTED         0x400UL
-+	#define FUNC_QCAPS_RESP_FLAGS_EXT3_MBUF_STATS_SUPPORTED              0x800UL
-+	__le16	max_roce_vfs;
-+	__le16	max_crypto_rx_flow_filters;
-+	u8	unused_3[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_qcfg_input (size:192b/24B) */
-+struct hwrm_func_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	fid;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_func_qcfg_output (size:1408b/176B) */
-+struct hwrm_func_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	fid;
-+	__le16	port_id;
-+	__le16	vlan;
-+	__le16	flags;
-+	#define FUNC_QCFG_RESP_FLAGS_OOB_WOL_MAGICPKT_ENABLED     0x1UL
-+	#define FUNC_QCFG_RESP_FLAGS_OOB_WOL_BMP_ENABLED          0x2UL
-+	#define FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED        0x4UL
-+	#define FUNC_QCFG_RESP_FLAGS_STD_TX_RING_MODE_ENABLED     0x8UL
-+	#define FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED        0x10UL
-+	#define FUNC_QCFG_RESP_FLAGS_MULTI_HOST                   0x20UL
-+	#define FUNC_QCFG_RESP_FLAGS_TRUSTED_VF                   0x40UL
-+	#define FUNC_QCFG_RESP_FLAGS_SECURE_MODE_ENABLED          0x80UL
-+	#define FUNC_QCFG_RESP_FLAGS_PREBOOT_LEGACY_L2_RINGS      0x100UL
-+	#define FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED            0x200UL
-+	#define FUNC_QCFG_RESP_FLAGS_PPP_PUSH_MODE_ENABLED        0x400UL
-+	#define FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED         0x800UL
-+	#define FUNC_QCFG_RESP_FLAGS_FAST_RESET_ALLOWED           0x1000UL
-+	#define FUNC_QCFG_RESP_FLAGS_MULTI_ROOT                   0x2000UL
-+	#define FUNC_QCFG_RESP_FLAGS_ENABLE_RDMA_SRIOV            0x4000UL
-+	#define FUNC_QCFG_RESP_FLAGS_ROCE_VNIC_ID_VALID           0x8000UL
-+	u8	mac_address[6];
-+	__le16	pci_id;
-+	__le16	alloc_rsscos_ctx;
-+	__le16	alloc_cmpl_rings;
-+	__le16	alloc_tx_rings;
-+	__le16	alloc_rx_rings;
-+	__le16	alloc_l2_ctx;
-+	__le16	alloc_vnics;
-+	__le16	admin_mtu;
-+	__le16	mru;
-+	__le16	stat_ctx_id;
-+	u8	port_partition_type;
-+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_SPF     0x0UL
-+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_MPFS    0x1UL
-+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0 0x2UL
-+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5 0x3UL
-+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0 0x4UL
-+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_2 0x5UL
-+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_UNKNOWN 0xffUL
-+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_LAST   FUNC_QCFG_RESP_PORT_PARTITION_TYPE_UNKNOWN
-+	u8	port_pf_cnt;
-+	#define FUNC_QCFG_RESP_PORT_PF_CNT_UNAVAIL 0x0UL
-+	#define FUNC_QCFG_RESP_PORT_PF_CNT_LAST   FUNC_QCFG_RESP_PORT_PF_CNT_UNAVAIL
-+	__le16	dflt_vnic_id;
-+	__le16	max_mtu_configured;
-+	__le32	min_bw;
-+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_SFT              0
-+	#define FUNC_QCFG_RESP_MIN_BW_SCALE                     0x10000000UL
-+	#define FUNC_QCFG_RESP_MIN_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define FUNC_QCFG_RESP_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define FUNC_QCFG_RESP_MIN_BW_SCALE_LAST                 FUNC_QCFG_RESP_MIN_BW_SCALE_BYTES
-+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_SFT         29
-+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_LAST         FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_INVALID
-+	__le32	max_bw;
-+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_SFT              0
-+	#define FUNC_QCFG_RESP_MAX_BW_SCALE                     0x10000000UL
-+	#define FUNC_QCFG_RESP_MAX_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define FUNC_QCFG_RESP_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define FUNC_QCFG_RESP_MAX_BW_SCALE_LAST                 FUNC_QCFG_RESP_MAX_BW_SCALE_BYTES
-+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_SFT         29
-+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_LAST         FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_INVALID
-+	u8	evb_mode;
-+	#define FUNC_QCFG_RESP_EVB_MODE_NO_EVB 0x0UL
-+	#define FUNC_QCFG_RESP_EVB_MODE_VEB    0x1UL
-+	#define FUNC_QCFG_RESP_EVB_MODE_VEPA   0x2UL
-+	#define FUNC_QCFG_RESP_EVB_MODE_LAST  FUNC_QCFG_RESP_EVB_MODE_VEPA
-+	u8	options;
-+	#define FUNC_QCFG_RESP_OPTIONS_CACHE_LINESIZE_MASK         0x3UL
-+	#define FUNC_QCFG_RESP_OPTIONS_CACHE_LINESIZE_SFT          0
-+	#define FUNC_QCFG_RESP_OPTIONS_CACHE_LINESIZE_SIZE_64        0x0UL
-+	#define FUNC_QCFG_RESP_OPTIONS_CACHE_LINESIZE_SIZE_128       0x1UL
-+	#define FUNC_QCFG_RESP_OPTIONS_CACHE_LINESIZE_LAST          FUNC_QCFG_RESP_OPTIONS_CACHE_LINESIZE_SIZE_128
-+	#define FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_MASK       0xcUL
-+	#define FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_SFT        2
-+	#define FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_FORCED_DOWN  (0x0UL << 2)
-+	#define FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_FORCED_UP    (0x1UL << 2)
-+	#define FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_AUTO         (0x2UL << 2)
-+	#define FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_LAST        FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_AUTO
-+	#define FUNC_QCFG_RESP_OPTIONS_RSVD_MASK                   0xf0UL
-+	#define FUNC_QCFG_RESP_OPTIONS_RSVD_SFT                    4
-+	__le16	alloc_vfs;
-+	__le32	alloc_mcast_filters;
-+	__le32	alloc_hw_ring_grps;
-+	__le16	alloc_sp_tx_rings;
-+	__le16	alloc_stat_ctx;
-+	__le16	alloc_msix;
-+	__le16	registered_vfs;
-+	__le16	l2_doorbell_bar_size_kb;
-+	u8	active_endpoints;
-+	u8	always_1;
-+	__le32	reset_addr_poll;
-+	__le16	legacy_l2_db_size_kb;
-+	__le16	svif_info;
-+	#define FUNC_QCFG_RESP_SVIF_INFO_SVIF_MASK      0x7fffUL
-+	#define FUNC_QCFG_RESP_SVIF_INFO_SVIF_SFT       0
-+	#define FUNC_QCFG_RESP_SVIF_INFO_SVIF_VALID     0x8000UL
-+	u8	mpc_chnls;
-+	#define FUNC_QCFG_RESP_MPC_CHNLS_TCE_ENABLED         0x1UL
-+	#define FUNC_QCFG_RESP_MPC_CHNLS_RCE_ENABLED         0x2UL
-+	#define FUNC_QCFG_RESP_MPC_CHNLS_TE_CFA_ENABLED      0x4UL
-+	#define FUNC_QCFG_RESP_MPC_CHNLS_RE_CFA_ENABLED      0x8UL
-+	#define FUNC_QCFG_RESP_MPC_CHNLS_PRIMATE_ENABLED     0x10UL
-+	u8	db_page_size;
-+	#define FUNC_QCFG_RESP_DB_PAGE_SIZE_4KB   0x0UL
-+	#define FUNC_QCFG_RESP_DB_PAGE_SIZE_8KB   0x1UL
-+	#define FUNC_QCFG_RESP_DB_PAGE_SIZE_16KB  0x2UL
-+	#define FUNC_QCFG_RESP_DB_PAGE_SIZE_32KB  0x3UL
-+	#define FUNC_QCFG_RESP_DB_PAGE_SIZE_64KB  0x4UL
-+	#define FUNC_QCFG_RESP_DB_PAGE_SIZE_128KB 0x5UL
-+	#define FUNC_QCFG_RESP_DB_PAGE_SIZE_256KB 0x6UL
-+	#define FUNC_QCFG_RESP_DB_PAGE_SIZE_512KB 0x7UL
-+	#define FUNC_QCFG_RESP_DB_PAGE_SIZE_1MB   0x8UL
-+	#define FUNC_QCFG_RESP_DB_PAGE_SIZE_2MB   0x9UL
-+	#define FUNC_QCFG_RESP_DB_PAGE_SIZE_4MB   0xaUL
-+	#define FUNC_QCFG_RESP_DB_PAGE_SIZE_LAST FUNC_QCFG_RESP_DB_PAGE_SIZE_4MB
-+	__le16	roce_vnic_id;
-+	__le32	partition_min_bw;
-+	#define FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_SFT              0
-+	#define FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE                     0x10000000UL
-+	#define FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE_LAST                 FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE_BYTES
-+	#define FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_SFT         29
-+	#define FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_LAST         FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_PERCENT1_100
-+	__le32	partition_max_bw;
-+	#define FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_SFT              0
-+	#define FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE                     0x10000000UL
-+	#define FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE_LAST                 FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE_BYTES
-+	#define FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_SFT         29
-+	#define FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_LAST         FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_PERCENT1_100
-+	__le16	host_mtu;
-+	__le16	flags2;
-+	#define FUNC_QCFG_RESP_FLAGS2_SRIOV_DSCP_INSERT_ENABLED     0x1UL
-+	__le16	stag_vid;
-+	u8	port_kdnet_mode;
-+	#define FUNC_QCFG_RESP_PORT_KDNET_MODE_DISABLED 0x0UL
-+	#define FUNC_QCFG_RESP_PORT_KDNET_MODE_ENABLED  0x1UL
-+	#define FUNC_QCFG_RESP_PORT_KDNET_MODE_LAST    FUNC_QCFG_RESP_PORT_KDNET_MODE_ENABLED
-+	u8	kdnet_pcie_function;
-+	__le16	port_kdnet_fid;
-+	u8	unused_5;
-+	u8	roce_bidi_opt_mode;
-+	#define FUNC_QCFG_RESP_ROCE_BIDI_OPT_MODE_DISABLED      0x1UL
-+	#define FUNC_QCFG_RESP_ROCE_BIDI_OPT_MODE_DEDICATED     0x2UL
-+	#define FUNC_QCFG_RESP_ROCE_BIDI_OPT_MODE_SHARED        0x4UL
-+	__le32	num_ktls_tx_key_ctxs;
-+	__le32	num_ktls_rx_key_ctxs;
-+	u8	lag_id;
-+	u8	parif;
-+	u8	fw_lag_id;
-+	u8	unused_6;
-+	__le32	num_quic_tx_key_ctxs;
-+	__le32	num_quic_rx_key_ctxs;
-+	__le32	roce_max_av_per_vf;
-+	__le32	roce_max_cq_per_vf;
-+	__le32	roce_max_mrw_per_vf;
-+	__le32	roce_max_qp_per_vf;
-+	__le32	roce_max_srq_per_vf;
-+	__le32	roce_max_gid_per_vf;
-+	__le16	xid_partition_cfg;
-+	#define FUNC_QCFG_RESP_XID_PARTITION_CFG_TX_CK     0x1UL
-+	#define FUNC_QCFG_RESP_XID_PARTITION_CFG_RX_CK     0x2UL
-+	__le16	mirror_vnic_id;
-+	u8	max_link_width;
-+	#define FUNC_QCFG_RESP_MAX_LINK_WIDTH_UNKNOWN 0x0UL
-+	#define FUNC_QCFG_RESP_MAX_LINK_WIDTH_X1      0x1UL
-+	#define FUNC_QCFG_RESP_MAX_LINK_WIDTH_X2      0x2UL
-+	#define FUNC_QCFG_RESP_MAX_LINK_WIDTH_X4      0x4UL
-+	#define FUNC_QCFG_RESP_MAX_LINK_WIDTH_X8      0x8UL
-+	#define FUNC_QCFG_RESP_MAX_LINK_WIDTH_X16     0x10UL
-+	#define FUNC_QCFG_RESP_MAX_LINK_WIDTH_LAST   FUNC_QCFG_RESP_MAX_LINK_WIDTH_X16
-+	u8	max_link_speed;
-+	#define FUNC_QCFG_RESP_MAX_LINK_SPEED_UNKNOWN 0x0UL
-+	#define FUNC_QCFG_RESP_MAX_LINK_SPEED_G1      0x1UL
-+	#define FUNC_QCFG_RESP_MAX_LINK_SPEED_G2      0x2UL
-+	#define FUNC_QCFG_RESP_MAX_LINK_SPEED_G3      0x3UL
-+	#define FUNC_QCFG_RESP_MAX_LINK_SPEED_G4      0x4UL
-+	#define FUNC_QCFG_RESP_MAX_LINK_SPEED_G5      0x5UL
-+	#define FUNC_QCFG_RESP_MAX_LINK_SPEED_LAST   FUNC_QCFG_RESP_MAX_LINK_SPEED_G5
-+	u8	negotiated_link_width;
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_UNKNOWN 0x0UL
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_X1      0x1UL
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_X2      0x2UL
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_X4      0x4UL
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_X8      0x8UL
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_X16     0x10UL
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_LAST   FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_X16
-+	u8	negotiated_link_speed;
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_UNKNOWN 0x0UL
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_G1      0x1UL
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_G2      0x2UL
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_G3      0x3UL
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_G4      0x4UL
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_G5      0x5UL
-+	#define FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_LAST   FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_G5
-+	u8	unused_7[2];
-+	u8	pcie_compliance;
-+	u8	unused_8;
-+	__le16	l2_db_multi_page_size_kb;
-+	u8	unused_9[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_cfg_input (size:1280b/160B) */
-+struct hwrm_func_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	fid;
-+	__le16	num_msix;
-+	__le32	flags;
-+	#define FUNC_CFG_REQ_FLAGS_SRC_MAC_ADDR_CHECK_DISABLE     0x1UL
-+	#define FUNC_CFG_REQ_FLAGS_SRC_MAC_ADDR_CHECK_ENABLE      0x2UL
-+	#define FUNC_CFG_REQ_FLAGS_RSVD_MASK                      0x1fcUL
-+	#define FUNC_CFG_REQ_FLAGS_RSVD_SFT                       2
-+	#define FUNC_CFG_REQ_FLAGS_STD_TX_RING_MODE_ENABLE        0x200UL
-+	#define FUNC_CFG_REQ_FLAGS_STD_TX_RING_MODE_DISABLE       0x400UL
-+	#define FUNC_CFG_REQ_FLAGS_VIRT_MAC_PERSIST               0x800UL
-+	#define FUNC_CFG_REQ_FLAGS_NO_AUTOCLEAR_STATISTIC         0x1000UL
-+	#define FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST                 0x2000UL
-+	#define FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST                 0x4000UL
-+	#define FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST               0x8000UL
-+	#define FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST         0x10000UL
-+	#define FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST           0x20000UL
-+	#define FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST           0x40000UL
-+	#define FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST               0x80000UL
-+	#define FUNC_CFG_REQ_FLAGS_L2_CTX_ASSETS_TEST             0x100000UL
-+	#define FUNC_CFG_REQ_FLAGS_TRUSTED_VF_ENABLE              0x200000UL
-+	#define FUNC_CFG_REQ_FLAGS_DYNAMIC_TX_RING_ALLOC          0x400000UL
-+	#define FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST                 0x800000UL
-+	#define FUNC_CFG_REQ_FLAGS_TRUSTED_VF_DISABLE             0x1000000UL
-+	#define FUNC_CFG_REQ_FLAGS_PREBOOT_LEGACY_L2_RINGS        0x2000000UL
-+	#define FUNC_CFG_REQ_FLAGS_HOT_RESET_IF_EN_DIS            0x4000000UL
-+	#define FUNC_CFG_REQ_FLAGS_PPP_PUSH_MODE_ENABLE           0x8000000UL
-+	#define FUNC_CFG_REQ_FLAGS_PPP_PUSH_MODE_DISABLE          0x10000000UL
-+	#define FUNC_CFG_REQ_FLAGS_BD_METADATA_ENABLE             0x20000000UL
-+	#define FUNC_CFG_REQ_FLAGS_BD_METADATA_DISABLE            0x40000000UL
-+	__le32	enables;
-+	#define FUNC_CFG_REQ_ENABLES_ADMIN_MTU                0x1UL
-+	#define FUNC_CFG_REQ_ENABLES_MRU                      0x2UL
-+	#define FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS          0x4UL
-+	#define FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS           0x8UL
-+	#define FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS             0x10UL
-+	#define FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS             0x20UL
-+	#define FUNC_CFG_REQ_ENABLES_NUM_L2_CTXS              0x40UL
-+	#define FUNC_CFG_REQ_ENABLES_NUM_VNICS                0x80UL
-+	#define FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS            0x100UL
-+	#define FUNC_CFG_REQ_ENABLES_DFLT_MAC_ADDR            0x200UL
-+	#define FUNC_CFG_REQ_ENABLES_DFLT_VLAN                0x400UL
-+	#define FUNC_CFG_REQ_ENABLES_DFLT_IP_ADDR             0x800UL
-+	#define FUNC_CFG_REQ_ENABLES_MIN_BW                   0x1000UL
-+	#define FUNC_CFG_REQ_ENABLES_MAX_BW                   0x2000UL
-+	#define FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR           0x4000UL
-+	#define FUNC_CFG_REQ_ENABLES_VLAN_ANTISPOOF_MODE      0x8000UL
-+	#define FUNC_CFG_REQ_ENABLES_ALLOWED_VLAN_PRIS        0x10000UL
-+	#define FUNC_CFG_REQ_ENABLES_EVB_MODE                 0x20000UL
-+	#define FUNC_CFG_REQ_ENABLES_NUM_MCAST_FILTERS        0x40000UL
-+	#define FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS         0x80000UL
-+	#define FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE           0x100000UL
-+	#define FUNC_CFG_REQ_ENABLES_NUM_MSIX                 0x200000UL
-+	#define FUNC_CFG_REQ_ENABLES_ADMIN_LINK_STATE         0x400000UL
-+	#define FUNC_CFG_REQ_ENABLES_HOT_RESET_IF_SUPPORT     0x800000UL
-+	#define FUNC_CFG_REQ_ENABLES_SCHQ_ID                  0x1000000UL
-+	#define FUNC_CFG_REQ_ENABLES_MPC_CHNLS                0x2000000UL
-+	#define FUNC_CFG_REQ_ENABLES_PARTITION_MIN_BW         0x4000000UL
-+	#define FUNC_CFG_REQ_ENABLES_PARTITION_MAX_BW         0x8000000UL
-+	#define FUNC_CFG_REQ_ENABLES_TPID                     0x10000000UL
-+	#define FUNC_CFG_REQ_ENABLES_HOST_MTU                 0x20000000UL
-+	#define FUNC_CFG_REQ_ENABLES_KTLS_TX_KEY_CTXS         0x40000000UL
-+	#define FUNC_CFG_REQ_ENABLES_KTLS_RX_KEY_CTXS         0x80000000UL
-+	__le16	admin_mtu;
-+	__le16	mru;
-+	__le16	num_rsscos_ctxs;
-+	__le16	num_cmpl_rings;
-+	__le16	num_tx_rings;
-+	__le16	num_rx_rings;
-+	__le16	num_l2_ctxs;
-+	__le16	num_vnics;
-+	__le16	num_stat_ctxs;
-+	__le16	num_hw_ring_grps;
-+	u8	dflt_mac_addr[6];
-+	__le16	dflt_vlan;
-+	__be32	dflt_ip_addr[4];
-+	__le32	min_bw;
-+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_SFT              0
-+	#define FUNC_CFG_REQ_MIN_BW_SCALE                     0x10000000UL
-+	#define FUNC_CFG_REQ_MIN_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define FUNC_CFG_REQ_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define FUNC_CFG_REQ_MIN_BW_SCALE_LAST                 FUNC_CFG_REQ_MIN_BW_SCALE_BYTES
-+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_SFT         29
-+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_LAST         FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_INVALID
-+	__le32	max_bw;
-+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_SFT              0
-+	#define FUNC_CFG_REQ_MAX_BW_SCALE                     0x10000000UL
-+	#define FUNC_CFG_REQ_MAX_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define FUNC_CFG_REQ_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define FUNC_CFG_REQ_MAX_BW_SCALE_LAST                 FUNC_CFG_REQ_MAX_BW_SCALE_BYTES
-+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_SFT         29
-+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_LAST         FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_INVALID
-+	__le16	async_event_cr;
-+	u8	vlan_antispoof_mode;
-+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_NOCHECK                 0x0UL
-+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_VALIDATE_VLAN           0x1UL
-+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_INSERT_IF_VLANDNE       0x2UL
-+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_INSERT_OR_OVERRIDE_VLAN 0x3UL
-+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_LAST                   FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_INSERT_OR_OVERRIDE_VLAN
-+	u8	allowed_vlan_pris;
-+	u8	evb_mode;
-+	#define FUNC_CFG_REQ_EVB_MODE_NO_EVB 0x0UL
-+	#define FUNC_CFG_REQ_EVB_MODE_VEB    0x1UL
-+	#define FUNC_CFG_REQ_EVB_MODE_VEPA   0x2UL
-+	#define FUNC_CFG_REQ_EVB_MODE_LAST  FUNC_CFG_REQ_EVB_MODE_VEPA
-+	u8	options;
-+	#define FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_MASK         0x3UL
-+	#define FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SFT          0
-+	#define FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64        0x0UL
-+	#define FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128       0x1UL
-+	#define FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_LAST          FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128
-+	#define FUNC_CFG_REQ_OPTIONS_LINK_ADMIN_STATE_MASK       0xcUL
-+	#define FUNC_CFG_REQ_OPTIONS_LINK_ADMIN_STATE_SFT        2
-+	#define FUNC_CFG_REQ_OPTIONS_LINK_ADMIN_STATE_FORCED_DOWN  (0x0UL << 2)
-+	#define FUNC_CFG_REQ_OPTIONS_LINK_ADMIN_STATE_FORCED_UP    (0x1UL << 2)
-+	#define FUNC_CFG_REQ_OPTIONS_LINK_ADMIN_STATE_AUTO         (0x2UL << 2)
-+	#define FUNC_CFG_REQ_OPTIONS_LINK_ADMIN_STATE_LAST        FUNC_CFG_REQ_OPTIONS_LINK_ADMIN_STATE_AUTO
-+	#define FUNC_CFG_REQ_OPTIONS_RSVD_MASK                   0xf0UL
-+	#define FUNC_CFG_REQ_OPTIONS_RSVD_SFT                    4
-+	__le16	num_mcast_filters;
-+	__le16	schq_id;
-+	__le16	mpc_chnls;
-+	#define FUNC_CFG_REQ_MPC_CHNLS_TCE_ENABLE          0x1UL
-+	#define FUNC_CFG_REQ_MPC_CHNLS_TCE_DISABLE         0x2UL
-+	#define FUNC_CFG_REQ_MPC_CHNLS_RCE_ENABLE          0x4UL
-+	#define FUNC_CFG_REQ_MPC_CHNLS_RCE_DISABLE         0x8UL
-+	#define FUNC_CFG_REQ_MPC_CHNLS_TE_CFA_ENABLE       0x10UL
-+	#define FUNC_CFG_REQ_MPC_CHNLS_TE_CFA_DISABLE      0x20UL
-+	#define FUNC_CFG_REQ_MPC_CHNLS_RE_CFA_ENABLE       0x40UL
-+	#define FUNC_CFG_REQ_MPC_CHNLS_RE_CFA_DISABLE      0x80UL
-+	#define FUNC_CFG_REQ_MPC_CHNLS_PRIMATE_ENABLE      0x100UL
-+	#define FUNC_CFG_REQ_MPC_CHNLS_PRIMATE_DISABLE     0x200UL
-+	__le32	partition_min_bw;
-+	#define FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_SFT              0
-+	#define FUNC_CFG_REQ_PARTITION_MIN_BW_SCALE                     0x10000000UL
-+	#define FUNC_CFG_REQ_PARTITION_MIN_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define FUNC_CFG_REQ_PARTITION_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define FUNC_CFG_REQ_PARTITION_MIN_BW_SCALE_LAST                 FUNC_CFG_REQ_PARTITION_MIN_BW_SCALE_BYTES
-+	#define FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_UNIT_SFT         29
-+	#define FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_UNIT_LAST         FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_UNIT_PERCENT1_100
-+	__le32	partition_max_bw;
-+	#define FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_SFT              0
-+	#define FUNC_CFG_REQ_PARTITION_MAX_BW_SCALE                     0x10000000UL
-+	#define FUNC_CFG_REQ_PARTITION_MAX_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define FUNC_CFG_REQ_PARTITION_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define FUNC_CFG_REQ_PARTITION_MAX_BW_SCALE_LAST                 FUNC_CFG_REQ_PARTITION_MAX_BW_SCALE_BYTES
-+	#define FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_UNIT_SFT         29
-+	#define FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_UNIT_LAST         FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_UNIT_PERCENT1_100
-+	__be16	tpid;
-+	__le16	host_mtu;
-+	__le32	flags2;
-+	#define FUNC_CFG_REQ_FLAGS2_KTLS_KEY_CTX_ASSETS_TEST     0x1UL
-+	#define FUNC_CFG_REQ_FLAGS2_QUIC_KEY_CTX_ASSETS_TEST     0x2UL
-+	__le32	enables2;
-+	#define FUNC_CFG_REQ_ENABLES2_KDNET                    0x1UL
-+	#define FUNC_CFG_REQ_ENABLES2_DB_PAGE_SIZE             0x2UL
-+	#define FUNC_CFG_REQ_ENABLES2_QUIC_TX_KEY_CTXS         0x4UL
-+	#define FUNC_CFG_REQ_ENABLES2_QUIC_RX_KEY_CTXS         0x8UL
-+	#define FUNC_CFG_REQ_ENABLES2_ROCE_MAX_AV_PER_VF       0x10UL
-+	#define FUNC_CFG_REQ_ENABLES2_ROCE_MAX_CQ_PER_VF       0x20UL
-+	#define FUNC_CFG_REQ_ENABLES2_ROCE_MAX_MRW_PER_VF      0x40UL
-+	#define FUNC_CFG_REQ_ENABLES2_ROCE_MAX_QP_PER_VF       0x80UL
-+	#define FUNC_CFG_REQ_ENABLES2_ROCE_MAX_SRQ_PER_VF      0x100UL
-+	#define FUNC_CFG_REQ_ENABLES2_ROCE_MAX_GID_PER_VF      0x200UL
-+	#define FUNC_CFG_REQ_ENABLES2_XID_PARTITION_CFG        0x400UL
-+	#define FUNC_CFG_REQ_ENABLES2_PHYSICAL_SLOT_NUMBER     0x800UL
-+	#define FUNC_CFG_REQ_ENABLES2_PCIE_COMPLIANCE          0x1000UL
-+	u8	port_kdnet_mode;
-+	#define FUNC_CFG_REQ_PORT_KDNET_MODE_DISABLED 0x0UL
-+	#define FUNC_CFG_REQ_PORT_KDNET_MODE_ENABLED  0x1UL
-+	#define FUNC_CFG_REQ_PORT_KDNET_MODE_LAST    FUNC_CFG_REQ_PORT_KDNET_MODE_ENABLED
-+	u8	db_page_size;
-+	#define FUNC_CFG_REQ_DB_PAGE_SIZE_4KB   0x0UL
-+	#define FUNC_CFG_REQ_DB_PAGE_SIZE_8KB   0x1UL
-+	#define FUNC_CFG_REQ_DB_PAGE_SIZE_16KB  0x2UL
-+	#define FUNC_CFG_REQ_DB_PAGE_SIZE_32KB  0x3UL
-+	#define FUNC_CFG_REQ_DB_PAGE_SIZE_64KB  0x4UL
-+	#define FUNC_CFG_REQ_DB_PAGE_SIZE_128KB 0x5UL
-+	#define FUNC_CFG_REQ_DB_PAGE_SIZE_256KB 0x6UL
-+	#define FUNC_CFG_REQ_DB_PAGE_SIZE_512KB 0x7UL
-+	#define FUNC_CFG_REQ_DB_PAGE_SIZE_1MB   0x8UL
-+	#define FUNC_CFG_REQ_DB_PAGE_SIZE_2MB   0x9UL
-+	#define FUNC_CFG_REQ_DB_PAGE_SIZE_4MB   0xaUL
-+	#define FUNC_CFG_REQ_DB_PAGE_SIZE_LAST FUNC_CFG_REQ_DB_PAGE_SIZE_4MB
-+	__le16	physical_slot_number;
-+	__le32	num_ktls_tx_key_ctxs;
-+	__le32	num_ktls_rx_key_ctxs;
-+	__le32	num_quic_tx_key_ctxs;
-+	__le32	num_quic_rx_key_ctxs;
-+	__le32	roce_max_av_per_vf;
-+	__le32	roce_max_cq_per_vf;
-+	__le32	roce_max_mrw_per_vf;
-+	__le32	roce_max_qp_per_vf;
-+	__le32	roce_max_srq_per_vf;
-+	__le32	roce_max_gid_per_vf;
-+	__le16	xid_partition_cfg;
-+	#define FUNC_CFG_REQ_XID_PARTITION_CFG_TX_CK     0x1UL
-+	#define FUNC_CFG_REQ_XID_PARTITION_CFG_RX_CK     0x2UL
-+	u8	pcie_compliance;
-+	u8	unused_2;
-+};
-+
-+/* hwrm_func_cfg_output (size:128b/16B) */
-+struct hwrm_func_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_cfg_cmd_err (size:64b/8B) */
-+struct hwrm_func_cfg_cmd_err {
-+	u8	code;
-+	#define FUNC_CFG_CMD_ERR_CODE_UNKNOWN                      0x0UL
-+	#define FUNC_CFG_CMD_ERR_CODE_PARTITION_BW_OUT_OF_RANGE    0x1UL
-+	#define FUNC_CFG_CMD_ERR_CODE_NPAR_PARTITION_DOWN_FAILED   0x2UL
-+	#define FUNC_CFG_CMD_ERR_CODE_TPID_SET_DFLT_VLAN_NOT_SET   0x3UL
-+	#define FUNC_CFG_CMD_ERR_CODE_RES_ARRAY_ALLOC_FAILED       0x4UL
-+	#define FUNC_CFG_CMD_ERR_CODE_TX_RING_ASSET_TEST_FAILED    0x5UL
-+	#define FUNC_CFG_CMD_ERR_CODE_TX_RING_RES_UPDATE_FAILED    0x6UL
-+	#define FUNC_CFG_CMD_ERR_CODE_APPLY_MAX_BW_FAILED          0x7UL
-+	#define FUNC_CFG_CMD_ERR_CODE_ENABLE_EVB_FAILED            0x8UL
-+	#define FUNC_CFG_CMD_ERR_CODE_RSS_CTXT_ASSET_TEST_FAILED   0x9UL
-+	#define FUNC_CFG_CMD_ERR_CODE_RSS_CTXT_RES_UPDATE_FAILED   0xaUL
-+	#define FUNC_CFG_CMD_ERR_CODE_CMPL_RING_ASSET_TEST_FAILED  0xbUL
-+	#define FUNC_CFG_CMD_ERR_CODE_CMPL_RING_RES_UPDATE_FAILED  0xcUL
-+	#define FUNC_CFG_CMD_ERR_CODE_NQ_ASSET_TEST_FAILED         0xdUL
-+	#define FUNC_CFG_CMD_ERR_CODE_NQ_RES_UPDATE_FAILED         0xeUL
-+	#define FUNC_CFG_CMD_ERR_CODE_RX_RING_ASSET_TEST_FAILED    0xfUL
-+	#define FUNC_CFG_CMD_ERR_CODE_RX_RING_RES_UPDATE_FAILED    0x10UL
-+	#define FUNC_CFG_CMD_ERR_CODE_VNIC_ASSET_TEST_FAILED       0x11UL
-+	#define FUNC_CFG_CMD_ERR_CODE_VNIC_RES_UPDATE_FAILED       0x12UL
-+	#define FUNC_CFG_CMD_ERR_CODE_FAILED_TO_START_STATS_THREAD 0x13UL
-+	#define FUNC_CFG_CMD_ERR_CODE_RDMA_SRIOV_DISABLED          0x14UL
-+	#define FUNC_CFG_CMD_ERR_CODE_TX_KTLS_DISABLED             0x15UL
-+	#define FUNC_CFG_CMD_ERR_CODE_TX_KTLS_ASSET_TEST_FAILED    0x16UL
-+	#define FUNC_CFG_CMD_ERR_CODE_TX_KTLS_RES_UPDATE_FAILED    0x17UL
-+	#define FUNC_CFG_CMD_ERR_CODE_RX_KTLS_DISABLED             0x18UL
-+	#define FUNC_CFG_CMD_ERR_CODE_RX_KTLS_ASSET_TEST_FAILED    0x19UL
-+	#define FUNC_CFG_CMD_ERR_CODE_RX_KTLS_RES_UPDATE_FAILED    0x1aUL
-+	#define FUNC_CFG_CMD_ERR_CODE_TX_QUIC_DISABLED             0x1bUL
-+	#define FUNC_CFG_CMD_ERR_CODE_TX_QUIC_ASSET_TEST_FAILED    0x1cUL
-+	#define FUNC_CFG_CMD_ERR_CODE_TX_QUIC_RES_UPDATE_FAILED    0x1dUL
-+	#define FUNC_CFG_CMD_ERR_CODE_RX_QUIC_DISABLED             0x1eUL
-+	#define FUNC_CFG_CMD_ERR_CODE_RX_QUIC_ASSET_TEST_FAILED    0x1fUL
-+	#define FUNC_CFG_CMD_ERR_CODE_RX_QUIC_RES_UPDATE_FAILED    0x20UL
-+	#define FUNC_CFG_CMD_ERR_CODE_INVALID_KDNET_MODE           0x21UL
-+	#define FUNC_CFG_CMD_ERR_CODE_SCHQ_CFG_FAIL                0x22UL
-+	#define FUNC_CFG_CMD_ERR_CODE_LAST                        FUNC_CFG_CMD_ERR_CODE_SCHQ_CFG_FAIL
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_func_qstats_input (size:192b/24B) */
-+struct hwrm_func_qstats_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	fid;
-+	u8	flags;
-+	#define FUNC_QSTATS_REQ_FLAGS_ROCE_ONLY        0x1UL
-+	#define FUNC_QSTATS_REQ_FLAGS_COUNTER_MASK     0x2UL
-+	#define FUNC_QSTATS_REQ_FLAGS_L2_ONLY          0x4UL
-+	u8	unused_0[5];
-+};
-+
-+/* hwrm_func_qstats_output (size:1408b/176B) */
-+struct hwrm_func_qstats_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le64	tx_ucast_pkts;
-+	__le64	tx_mcast_pkts;
-+	__le64	tx_bcast_pkts;
-+	__le64	tx_discard_pkts;
-+	__le64	tx_drop_pkts;
-+	__le64	tx_ucast_bytes;
-+	__le64	tx_mcast_bytes;
-+	__le64	tx_bcast_bytes;
-+	__le64	rx_ucast_pkts;
-+	__le64	rx_mcast_pkts;
-+	__le64	rx_bcast_pkts;
-+	__le64	rx_discard_pkts;
-+	__le64	rx_drop_pkts;
-+	__le64	rx_ucast_bytes;
-+	__le64	rx_mcast_bytes;
-+	__le64	rx_bcast_bytes;
-+	__le64	rx_agg_pkts;
-+	__le64	rx_agg_bytes;
-+	__le64	rx_agg_events;
-+	__le64	rx_agg_aborts;
-+	u8	clear_seq;
-+	u8	unused_0[6];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_qstats_ext_input (size:256b/32B) */
-+struct hwrm_func_qstats_ext_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	fid;
-+	u8	flags;
-+	#define FUNC_QSTATS_EXT_REQ_FLAGS_ROCE_ONLY        0x1UL
-+	#define FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK     0x2UL
-+	u8	unused_0[1];
-+	__le32	enables;
-+	#define FUNC_QSTATS_EXT_REQ_ENABLES_SCHQ_ID     0x1UL
-+	__le16	schq_id;
-+	__le16	traffic_class;
-+	u8	unused_1[4];
-+};
-+
-+/* hwrm_func_qstats_ext_output (size:1536b/192B) */
-+struct hwrm_func_qstats_ext_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le64	rx_ucast_pkts;
-+	__le64	rx_mcast_pkts;
-+	__le64	rx_bcast_pkts;
-+	__le64	rx_discard_pkts;
-+	__le64	rx_error_pkts;
-+	__le64	rx_ucast_bytes;
-+	__le64	rx_mcast_bytes;
-+	__le64	rx_bcast_bytes;
-+	__le64	tx_ucast_pkts;
-+	__le64	tx_mcast_pkts;
-+	__le64	tx_bcast_pkts;
-+	__le64	tx_error_pkts;
-+	__le64	tx_discard_pkts;
-+	__le64	tx_ucast_bytes;
-+	__le64	tx_mcast_bytes;
-+	__le64	tx_bcast_bytes;
-+	__le64	rx_tpa_eligible_pkt;
-+	__le64	rx_tpa_eligible_bytes;
-+	__le64	rx_tpa_pkt;
-+	__le64	rx_tpa_bytes;
-+	__le64	rx_tpa_errors;
-+	__le64	rx_tpa_events;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_clr_stats_input (size:192b/24B) */
-+struct hwrm_func_clr_stats_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	fid;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_func_clr_stats_output (size:128b/16B) */
-+struct hwrm_func_clr_stats_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_vf_resc_free_input (size:192b/24B) */
-+struct hwrm_func_vf_resc_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	vf_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_func_vf_resc_free_output (size:128b/16B) */
-+struct hwrm_func_vf_resc_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_drv_rgtr_input (size:896b/112B) */
-+struct hwrm_func_drv_rgtr_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_FWD_ALL_MODE                     0x1UL
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_FWD_NONE_MODE                    0x2UL
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE                   0x4UL
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE           0x8UL
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT                0x10UL
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT           0x20UL
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT                   0x40UL
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_FAST_RESET_SUPPORT               0x80UL
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_RSS_STRICT_HASH_TYPE_SUPPORT     0x100UL
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_NPAR_1_2_SUPPORT                 0x200UL
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_ASYM_QUEUE_CFG_SUPPORT           0x400UL
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_TF_INGRESS_NIC_FLOW_MODE         0x800UL
-+	#define FUNC_DRV_RGTR_REQ_FLAGS_TF_EGRESS_NIC_FLOW_MODE          0x1000UL
-+	__le32	enables;
-+	#define FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE             0x1UL
-+	#define FUNC_DRV_RGTR_REQ_ENABLES_VER                 0x2UL
-+	#define FUNC_DRV_RGTR_REQ_ENABLES_TIMESTAMP           0x4UL
-+	#define FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD          0x8UL
-+	#define FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD     0x10UL
-+	__le16	os_type;
-+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_UNKNOWN   0x0UL
-+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_OTHER     0x1UL
-+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_MSDOS     0xeUL
-+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WINDOWS   0x12UL
-+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_SOLARIS   0x1dUL
-+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX     0x24UL
-+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_FREEBSD   0x2aUL
-+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_ESXI      0x68UL
-+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WIN864    0x73UL
-+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WIN2012R2 0x74UL
-+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_UEFI      0x8000UL
-+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_LAST     FUNC_DRV_RGTR_REQ_OS_TYPE_UEFI
-+	u8	ver_maj_8b;
-+	u8	ver_min_8b;
-+	u8	ver_upd_8b;
-+	u8	unused_0[3];
-+	__le32	timestamp;
-+	u8	unused_1[4];
-+	__le32	vf_req_fwd[8];
-+	__le32	async_event_fwd[8];
-+	__le16	ver_maj;
-+	__le16	ver_min;
-+	__le16	ver_upd;
-+	__le16	ver_patch;
-+};
-+
-+/* hwrm_func_drv_rgtr_output (size:128b/16B) */
-+struct hwrm_func_drv_rgtr_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	flags;
-+	#define FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED     0x1UL
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_drv_unrgtr_input (size:192b/24B) */
-+struct hwrm_func_drv_unrgtr_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define FUNC_DRV_UNRGTR_REQ_FLAGS_PREPARE_FOR_SHUTDOWN     0x1UL
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_func_drv_unrgtr_output (size:128b/16B) */
-+struct hwrm_func_drv_unrgtr_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_buf_rgtr_input (size:1024b/128B) */
-+struct hwrm_func_buf_rgtr_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	#define FUNC_BUF_RGTR_REQ_ENABLES_VF_ID            0x1UL
-+	#define FUNC_BUF_RGTR_REQ_ENABLES_ERR_BUF_ADDR     0x2UL
-+	__le16	vf_id;
-+	__le16	req_buf_num_pages;
-+	__le16	req_buf_page_size;
-+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_16B 0x4UL
-+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_4K  0xcUL
-+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_8K  0xdUL
-+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_64K 0x10UL
-+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_2M  0x15UL
-+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_4M  0x16UL
-+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_1G  0x1eUL
-+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_LAST FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_1G
-+	__le16	req_buf_len;
-+	__le16	resp_buf_len;
-+	u8	unused_0[2];
-+	__le64	req_buf_page_addr0;
-+	__le64	req_buf_page_addr1;
-+	__le64	req_buf_page_addr2;
-+	__le64	req_buf_page_addr3;
-+	__le64	req_buf_page_addr4;
-+	__le64	req_buf_page_addr5;
-+	__le64	req_buf_page_addr6;
-+	__le64	req_buf_page_addr7;
-+	__le64	req_buf_page_addr8;
-+	__le64	req_buf_page_addr9;
-+	__le64	error_buf_addr;
-+	__le64	resp_buf_addr;
-+};
-+
-+/* hwrm_func_buf_rgtr_output (size:128b/16B) */
-+struct hwrm_func_buf_rgtr_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_drv_qver_input (size:192b/24B) */
-+struct hwrm_func_drv_qver_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	reserved;
-+	__le16	fid;
-+	u8	driver_type;
-+	#define FUNC_DRV_QVER_REQ_DRIVER_TYPE_L2   0x0UL
-+	#define FUNC_DRV_QVER_REQ_DRIVER_TYPE_ROCE 0x1UL
-+	#define FUNC_DRV_QVER_REQ_DRIVER_TYPE_LAST FUNC_DRV_QVER_REQ_DRIVER_TYPE_ROCE
-+	u8	unused_0;
-+};
-+
-+/* hwrm_func_drv_qver_output (size:256b/32B) */
-+struct hwrm_func_drv_qver_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	os_type;
-+	#define FUNC_DRV_QVER_RESP_OS_TYPE_UNKNOWN   0x0UL
-+	#define FUNC_DRV_QVER_RESP_OS_TYPE_OTHER     0x1UL
-+	#define FUNC_DRV_QVER_RESP_OS_TYPE_MSDOS     0xeUL
-+	#define FUNC_DRV_QVER_RESP_OS_TYPE_WINDOWS   0x12UL
-+	#define FUNC_DRV_QVER_RESP_OS_TYPE_SOLARIS   0x1dUL
-+	#define FUNC_DRV_QVER_RESP_OS_TYPE_LINUX     0x24UL
-+	#define FUNC_DRV_QVER_RESP_OS_TYPE_FREEBSD   0x2aUL
-+	#define FUNC_DRV_QVER_RESP_OS_TYPE_ESXI      0x68UL
-+	#define FUNC_DRV_QVER_RESP_OS_TYPE_WIN864    0x73UL
-+	#define FUNC_DRV_QVER_RESP_OS_TYPE_WIN2012R2 0x74UL
-+	#define FUNC_DRV_QVER_RESP_OS_TYPE_UEFI      0x8000UL
-+	#define FUNC_DRV_QVER_RESP_OS_TYPE_LAST     FUNC_DRV_QVER_RESP_OS_TYPE_UEFI
-+	u8	ver_maj_8b;
-+	u8	ver_min_8b;
-+	u8	ver_upd_8b;
-+	u8	unused_0[3];
-+	__le16	ver_maj;
-+	__le16	ver_min;
-+	__le16	ver_upd;
-+	__le16	ver_patch;
-+	u8	unused_1[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_resource_qcaps_input (size:192b/24B) */
-+struct hwrm_func_resource_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	fid;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_func_resource_qcaps_output (size:704b/88B) */
-+struct hwrm_func_resource_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	max_vfs;
-+	__le16	max_msix;
-+	__le16	vf_reservation_strategy;
-+	#define FUNC_RESOURCE_QCAPS_RESP_VF_RESERVATION_STRATEGY_MAXIMAL        0x0UL
-+	#define FUNC_RESOURCE_QCAPS_RESP_VF_RESERVATION_STRATEGY_MINIMAL        0x1UL
-+	#define FUNC_RESOURCE_QCAPS_RESP_VF_RESERVATION_STRATEGY_MINIMAL_STATIC 0x2UL
-+	#define FUNC_RESOURCE_QCAPS_RESP_VF_RESERVATION_STRATEGY_LAST          FUNC_RESOURCE_QCAPS_RESP_VF_RESERVATION_STRATEGY_MINIMAL_STATIC
-+	__le16	min_rsscos_ctx;
-+	__le16	max_rsscos_ctx;
-+	__le16	min_cmpl_rings;
-+	__le16	max_cmpl_rings;
-+	__le16	min_tx_rings;
-+	__le16	max_tx_rings;
-+	__le16	min_rx_rings;
-+	__le16	max_rx_rings;
-+	__le16	min_l2_ctxs;
-+	__le16	max_l2_ctxs;
-+	__le16	min_vnics;
-+	__le16	max_vnics;
-+	__le16	min_stat_ctx;
-+	__le16	max_stat_ctx;
-+	__le16	min_hw_ring_grps;
-+	__le16	max_hw_ring_grps;
-+	__le16	max_tx_scheduler_inputs;
-+	__le16	flags;
-+	#define FUNC_RESOURCE_QCAPS_RESP_FLAGS_MIN_GUARANTEED     0x1UL
-+	__le16	min_msix;
-+	__le32	min_ktls_tx_key_ctxs;
-+	__le32	max_ktls_tx_key_ctxs;
-+	__le32	min_ktls_rx_key_ctxs;
-+	__le32	max_ktls_rx_key_ctxs;
-+	__le32	min_quic_tx_key_ctxs;
-+	__le32	max_quic_tx_key_ctxs;
-+	__le32	min_quic_rx_key_ctxs;
-+	__le32	max_quic_rx_key_ctxs;
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_vf_resource_cfg_input (size:704b/88B) */
-+struct hwrm_func_vf_resource_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	vf_id;
-+	__le16	max_msix;
-+	__le16	min_rsscos_ctx;
-+	__le16	max_rsscos_ctx;
-+	__le16	min_cmpl_rings;
-+	__le16	max_cmpl_rings;
-+	__le16	min_tx_rings;
-+	__le16	max_tx_rings;
-+	__le16	min_rx_rings;
-+	__le16	max_rx_rings;
-+	__le16	min_l2_ctxs;
-+	__le16	max_l2_ctxs;
-+	__le16	min_vnics;
-+	__le16	max_vnics;
-+	__le16	min_stat_ctx;
-+	__le16	max_stat_ctx;
-+	__le16	min_hw_ring_grps;
-+	__le16	max_hw_ring_grps;
-+	__le16	flags;
-+	#define FUNC_VF_RESOURCE_CFG_REQ_FLAGS_MIN_GUARANTEED     0x1UL
-+	__le16	min_msix;
-+	__le32	min_ktls_tx_key_ctxs;
-+	__le32	max_ktls_tx_key_ctxs;
-+	__le32	min_ktls_rx_key_ctxs;
-+	__le32	max_ktls_rx_key_ctxs;
-+	__le32	min_quic_tx_key_ctxs;
-+	__le32	max_quic_tx_key_ctxs;
-+	__le32	min_quic_rx_key_ctxs;
-+	__le32	max_quic_rx_key_ctxs;
-+};
-+
-+/* hwrm_func_vf_resource_cfg_output (size:384b/48B) */
-+struct hwrm_func_vf_resource_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	reserved_rsscos_ctx;
-+	__le16	reserved_cmpl_rings;
-+	__le16	reserved_tx_rings;
-+	__le16	reserved_rx_rings;
-+	__le16	reserved_l2_ctxs;
-+	__le16	reserved_vnics;
-+	__le16	reserved_stat_ctx;
-+	__le16	reserved_hw_ring_grps;
-+	__le32	reserved_ktls_tx_key_ctxs;
-+	__le32	reserved_ktls_rx_key_ctxs;
-+	__le32	reserved_quic_tx_key_ctxs;
-+	__le32	reserved_quic_rx_key_ctxs;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_backing_store_qcaps_input (size:128b/16B) */
-+struct hwrm_func_backing_store_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+};
-+
-+/* hwrm_func_backing_store_qcaps_output (size:832b/104B) */
-+struct hwrm_func_backing_store_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	qp_max_entries;
-+	__le16	qp_min_qp1_entries;
-+	__le16	qp_max_l2_entries;
-+	__le16	qp_entry_size;
-+	__le16	srq_max_l2_entries;
-+	__le32	srq_max_entries;
-+	__le16	srq_entry_size;
-+	__le16	cq_max_l2_entries;
-+	__le32	cq_max_entries;
-+	__le16	cq_entry_size;
-+	__le16	vnic_max_vnic_entries;
-+	__le16	vnic_max_ring_table_entries;
-+	__le16	vnic_entry_size;
-+	__le32	stat_max_entries;
-+	__le16	stat_entry_size;
-+	__le16	tqm_entry_size;
-+	__le32	tqm_min_entries_per_ring;
-+	__le32	tqm_max_entries_per_ring;
-+	__le32	mrav_max_entries;
-+	__le16	mrav_entry_size;
-+	__le16	tim_entry_size;
-+	__le32	tim_max_entries;
-+	__le16	mrav_num_entries_units;
-+	u8	tqm_entries_multiple;
-+	u8	ctx_kind_initializer;
-+	__le16	ctx_init_mask;
-+	#define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_QP       0x1UL
-+	#define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_SRQ      0x2UL
-+	#define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_CQ       0x4UL
-+	#define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_VNIC     0x8UL
-+	#define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_STAT     0x10UL
-+	#define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_MRAV     0x20UL
-+	#define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_TKC      0x40UL
-+	#define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_RKC      0x80UL
-+	u8	qp_init_offset;
-+	u8	srq_init_offset;
-+	u8	cq_init_offset;
-+	u8	vnic_init_offset;
-+	u8	tqm_fp_rings_count;
-+	u8	stat_init_offset;
-+	u8	mrav_init_offset;
-+	u8	tqm_fp_rings_count_ext;
-+	u8	tkc_init_offset;
-+	u8	rkc_init_offset;
-+	__le16	tkc_entry_size;
-+	__le16	rkc_entry_size;
-+	__le32	tkc_max_entries;
-+	__le32	rkc_max_entries;
-+	__le16	fast_qpmd_qp_num_entries;
-+	u8	rsvd1[5];
-+	u8	valid;
-+};
-+
-+/* tqm_fp_ring_cfg (size:128b/16B) */
-+struct tqm_fp_ring_cfg {
-+	u8	tqm_ring_pg_size_tqm_ring_lvl;
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_LVL_MASK      0xfUL
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_LVL_SFT       0
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_LVL_LVL_0       0x0UL
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_LVL_LVL_1       0x1UL
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_LVL_LVL_2       0x2UL
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_LVL_LAST       TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_LVL_LVL_2
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_PG_SIZE_MASK  0xf0UL
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_PG_SIZE_SFT   4
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_PG_SIZE_LAST   TQM_FP_RING_CFG_TQM_RING_CFG_TQM_RING_PG_SIZE_PG_1G
-+	u8	unused[3];
-+	__le32	tqm_ring_num_entries;
-+	__le64	tqm_ring_page_dir;
-+};
-+
-+/* hwrm_func_backing_store_cfg_input (size:2688b/336B) */
-+struct hwrm_func_backing_store_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define FUNC_BACKING_STORE_CFG_REQ_FLAGS_PREBOOT_MODE               0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT     0x2UL
-+	__le32	enables;
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP               0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ              0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ               0x4UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC             0x8UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT             0x10UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP           0x20UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING0        0x40UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING1        0x80UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING2        0x100UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING3        0x200UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING4        0x400UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING5        0x800UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING6        0x1000UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING7        0x2000UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV             0x4000UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM              0x8000UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING8        0x10000UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING9        0x20000UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING10       0x40000UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TKC              0x80000UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_RKC              0x100000UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD     0x200000UL
-+	u8	qpc_pg_size_qpc_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_1G
-+	u8	srq_pg_size_srq_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_1G
-+	u8	cq_pg_size_cq_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_1G
-+	u8	vnic_pg_size_vnic_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_1G
-+	u8	stat_pg_size_stat_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_1G
-+	u8	tqm_sp_pg_size_tqm_sp_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_1G
-+	u8	tqm_ring0_pg_size_tqm_ring0_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_1G
-+	u8	tqm_ring1_pg_size_tqm_ring1_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_1G
-+	u8	tqm_ring2_pg_size_tqm_ring2_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_1G
-+	u8	tqm_ring3_pg_size_tqm_ring3_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_1G
-+	u8	tqm_ring4_pg_size_tqm_ring4_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_1G
-+	u8	tqm_ring5_pg_size_tqm_ring5_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_1G
-+	u8	tqm_ring6_pg_size_tqm_ring6_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_1G
-+	u8	tqm_ring7_pg_size_tqm_ring7_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_1G
-+	u8	mrav_pg_size_mrav_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_1G
-+	u8	tim_pg_size_tim_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_1G
-+	__le64	qpc_page_dir;
-+	__le64	srq_page_dir;
-+	__le64	cq_page_dir;
-+	__le64	vnic_page_dir;
-+	__le64	stat_page_dir;
-+	__le64	tqm_sp_page_dir;
-+	__le64	tqm_ring0_page_dir;
-+	__le64	tqm_ring1_page_dir;
-+	__le64	tqm_ring2_page_dir;
-+	__le64	tqm_ring3_page_dir;
-+	__le64	tqm_ring4_page_dir;
-+	__le64	tqm_ring5_page_dir;
-+	__le64	tqm_ring6_page_dir;
-+	__le64	tqm_ring7_page_dir;
-+	__le64	mrav_page_dir;
-+	__le64	tim_page_dir;
-+	__le32	qp_num_entries;
-+	__le32	srq_num_entries;
-+	__le32	cq_num_entries;
-+	__le32	stat_num_entries;
-+	__le32	tqm_sp_num_entries;
-+	__le32	tqm_ring0_num_entries;
-+	__le32	tqm_ring1_num_entries;
-+	__le32	tqm_ring2_num_entries;
-+	__le32	tqm_ring3_num_entries;
-+	__le32	tqm_ring4_num_entries;
-+	__le32	tqm_ring5_num_entries;
-+	__le32	tqm_ring6_num_entries;
-+	__le32	tqm_ring7_num_entries;
-+	__le32	mrav_num_entries;
-+	__le32	tim_num_entries;
-+	__le16	qp_num_qp1_entries;
-+	__le16	qp_num_l2_entries;
-+	__le16	qp_entry_size;
-+	__le16	srq_num_l2_entries;
-+	__le16	srq_entry_size;
-+	__le16	cq_num_l2_entries;
-+	__le16	cq_entry_size;
-+	__le16	vnic_num_vnic_entries;
-+	__le16	vnic_num_ring_table_entries;
-+	__le16	vnic_entry_size;
-+	__le16	stat_entry_size;
-+	__le16	tqm_entry_size;
-+	__le16	mrav_entry_size;
-+	__le16	tim_entry_size;
-+	u8	tqm_ring8_pg_size_tqm_ring_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_1G
-+	u8	ring8_unused[3];
-+	__le32	tqm_ring8_num_entries;
-+	__le64	tqm_ring8_page_dir;
-+	u8	tqm_ring9_pg_size_tqm_ring_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_1G
-+	u8	ring9_unused[3];
-+	__le32	tqm_ring9_num_entries;
-+	__le64	tqm_ring9_page_dir;
-+	u8	tqm_ring10_pg_size_tqm_ring_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_1G
-+	u8	ring10_unused[3];
-+	__le32	tqm_ring10_num_entries;
-+	__le64	tqm_ring10_page_dir;
-+	__le32	tkc_num_entries;
-+	__le32	rkc_num_entries;
-+	__le64	tkc_page_dir;
-+	__le64	rkc_page_dir;
-+	__le16	tkc_entry_size;
-+	__le16	rkc_entry_size;
-+	u8	tkc_pg_size_tkc_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_1G
-+	u8	rkc_pg_size_rkc_lvl;
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_MASK      0xfUL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_SFT       0
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LVL_0       0x0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LVL_1       0x1UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LVL_2       0x2UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LAST       FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_LAST   FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_1G
-+	__le16	qp_num_fast_qpmd_entries;
-+};
-+
-+/* hwrm_func_backing_store_cfg_output (size:128b/16B) */
-+struct hwrm_func_backing_store_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_error_recovery_qcfg_input (size:192b/24B) */
-+struct hwrm_error_recovery_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	unused_0[8];
-+};
-+
-+/* hwrm_error_recovery_qcfg_output (size:1664b/208B) */
-+struct hwrm_error_recovery_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	flags;
-+	#define ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST       0x1UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU     0x2UL
-+	__le32	driver_polling_freq;
-+	__le32	master_func_wait_period;
-+	__le32	normal_func_wait_period;
-+	__le32	master_func_wait_period_after_reset;
-+	__le32	max_bailout_time_after_reset;
-+	__le32	fw_health_status_reg;
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEALTH_STATUS_REG_ADDR_SPACE_MASK    0x3UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEALTH_STATUS_REG_ADDR_SPACE_SFT     0
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEALTH_STATUS_REG_ADDR_SPACE_PCIE_CFG  0x0UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEALTH_STATUS_REG_ADDR_SPACE_GRC       0x1UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEALTH_STATUS_REG_ADDR_SPACE_BAR0      0x2UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEALTH_STATUS_REG_ADDR_SPACE_BAR1      0x3UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEALTH_STATUS_REG_ADDR_SPACE_LAST     ERROR_RECOVERY_QCFG_RESP_FW_HEALTH_STATUS_REG_ADDR_SPACE_BAR1
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEALTH_STATUS_REG_ADDR_MASK          0xfffffffcUL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEALTH_STATUS_REG_ADDR_SFT           2
-+	__le32	fw_heartbeat_reg;
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEARTBEAT_REG_ADDR_SPACE_MASK    0x3UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEARTBEAT_REG_ADDR_SPACE_SFT     0
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEARTBEAT_REG_ADDR_SPACE_PCIE_CFG  0x0UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEARTBEAT_REG_ADDR_SPACE_GRC       0x1UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEARTBEAT_REG_ADDR_SPACE_BAR0      0x2UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEARTBEAT_REG_ADDR_SPACE_BAR1      0x3UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEARTBEAT_REG_ADDR_SPACE_LAST     ERROR_RECOVERY_QCFG_RESP_FW_HEARTBEAT_REG_ADDR_SPACE_BAR1
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEARTBEAT_REG_ADDR_MASK          0xfffffffcUL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_HEARTBEAT_REG_ADDR_SFT           2
-+	__le32	fw_reset_cnt_reg;
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_RESET_CNT_REG_ADDR_SPACE_MASK    0x3UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_RESET_CNT_REG_ADDR_SPACE_SFT     0
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_RESET_CNT_REG_ADDR_SPACE_PCIE_CFG  0x0UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_RESET_CNT_REG_ADDR_SPACE_GRC       0x1UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_RESET_CNT_REG_ADDR_SPACE_BAR0      0x2UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_RESET_CNT_REG_ADDR_SPACE_BAR1      0x3UL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_RESET_CNT_REG_ADDR_SPACE_LAST     ERROR_RECOVERY_QCFG_RESP_FW_RESET_CNT_REG_ADDR_SPACE_BAR1
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_RESET_CNT_REG_ADDR_MASK          0xfffffffcUL
-+	#define ERROR_RECOVERY_QCFG_RESP_FW_RESET_CNT_REG_ADDR_SFT           2
-+	__le32	reset_inprogress_reg;
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_INPROGRESS_REG_ADDR_SPACE_MASK    0x3UL
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_INPROGRESS_REG_ADDR_SPACE_SFT     0
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_INPROGRESS_REG_ADDR_SPACE_PCIE_CFG  0x0UL
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_INPROGRESS_REG_ADDR_SPACE_GRC       0x1UL
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_INPROGRESS_REG_ADDR_SPACE_BAR0      0x2UL
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_INPROGRESS_REG_ADDR_SPACE_BAR1      0x3UL
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_INPROGRESS_REG_ADDR_SPACE_LAST     ERROR_RECOVERY_QCFG_RESP_RESET_INPROGRESS_REG_ADDR_SPACE_BAR1
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_INPROGRESS_REG_ADDR_MASK          0xfffffffcUL
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_INPROGRESS_REG_ADDR_SFT           2
-+	__le32	reset_inprogress_reg_mask;
-+	u8	unused_0[3];
-+	u8	reg_array_cnt;
-+	__le32	reset_reg[16];
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_REG_ADDR_SPACE_MASK    0x3UL
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_REG_ADDR_SPACE_SFT     0
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_REG_ADDR_SPACE_PCIE_CFG  0x0UL
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_REG_ADDR_SPACE_GRC       0x1UL
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_REG_ADDR_SPACE_BAR0      0x2UL
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_REG_ADDR_SPACE_BAR1      0x3UL
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_REG_ADDR_SPACE_LAST     ERROR_RECOVERY_QCFG_RESP_RESET_REG_ADDR_SPACE_BAR1
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_REG_ADDR_MASK          0xfffffffcUL
-+	#define ERROR_RECOVERY_QCFG_RESP_RESET_REG_ADDR_SFT           2
-+	__le32	reset_reg_val[16];
-+	u8	delay_after_reset[16];
-+	__le32	err_recovery_cnt_reg;
-+	#define ERROR_RECOVERY_QCFG_RESP_ERR_RECOVERY_CNT_REG_ADDR_SPACE_MASK    0x3UL
-+	#define ERROR_RECOVERY_QCFG_RESP_ERR_RECOVERY_CNT_REG_ADDR_SPACE_SFT     0
-+	#define ERROR_RECOVERY_QCFG_RESP_ERR_RECOVERY_CNT_REG_ADDR_SPACE_PCIE_CFG  0x0UL
-+	#define ERROR_RECOVERY_QCFG_RESP_ERR_RECOVERY_CNT_REG_ADDR_SPACE_GRC       0x1UL
-+	#define ERROR_RECOVERY_QCFG_RESP_ERR_RECOVERY_CNT_REG_ADDR_SPACE_BAR0      0x2UL
-+	#define ERROR_RECOVERY_QCFG_RESP_ERR_RECOVERY_CNT_REG_ADDR_SPACE_BAR1      0x3UL
-+	#define ERROR_RECOVERY_QCFG_RESP_ERR_RECOVERY_CNT_REG_ADDR_SPACE_LAST     ERROR_RECOVERY_QCFG_RESP_ERR_RECOVERY_CNT_REG_ADDR_SPACE_BAR1
-+	#define ERROR_RECOVERY_QCFG_RESP_ERR_RECOVERY_CNT_REG_ADDR_MASK          0xfffffffcUL
-+	#define ERROR_RECOVERY_QCFG_RESP_ERR_RECOVERY_CNT_REG_ADDR_SFT           2
-+	u8	unused_1[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_echo_response_input (size:192b/24B) */
-+struct hwrm_func_echo_response_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	event_data1;
-+	__le32	event_data2;
-+};
-+
-+/* hwrm_func_echo_response_output (size:128b/16B) */
-+struct hwrm_func_echo_response_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_ptp_pin_qcfg_input (size:192b/24B) */
-+struct hwrm_func_ptp_pin_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	unused_0[8];
-+};
-+
-+/* hwrm_func_ptp_pin_qcfg_output (size:128b/16B) */
-+struct hwrm_func_ptp_pin_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	num_pins;
-+	u8	state;
-+	#define FUNC_PTP_PIN_QCFG_RESP_STATE_PIN0_ENABLED     0x1UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_STATE_PIN1_ENABLED     0x2UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_STATE_PIN2_ENABLED     0x4UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_STATE_PIN3_ENABLED     0x8UL
-+	u8	pin0_usage;
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_NONE     0x0UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_PPS_IN   0x1UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_PPS_OUT  0x2UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_SYNC_IN  0x3UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_SYNC_OUT 0x4UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_LAST    FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_SYNC_OUT
-+	u8	pin1_usage;
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_NONE     0x0UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_PPS_IN   0x1UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_PPS_OUT  0x2UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_SYNC_IN  0x3UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_SYNC_OUT 0x4UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_LAST    FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_SYNC_OUT
-+	u8	pin2_usage;
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_NONE                      0x0UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_PPS_IN                    0x1UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_PPS_OUT                   0x2UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_SYNC_IN                   0x3UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_SYNC_OUT                  0x4UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_SYNCE_PRIMARY_CLOCK_OUT   0x5UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_SYNCE_SECONDARY_CLOCK_OUT 0x6UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_LAST                     FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_SYNCE_SECONDARY_CLOCK_OUT
-+	u8	pin3_usage;
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_NONE                      0x0UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_PPS_IN                    0x1UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_PPS_OUT                   0x2UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_SYNC_IN                   0x3UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_SYNC_OUT                  0x4UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_SYNCE_PRIMARY_CLOCK_OUT   0x5UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_SYNCE_SECONDARY_CLOCK_OUT 0x6UL
-+	#define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_LAST                     FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_SYNCE_SECONDARY_CLOCK_OUT
-+	u8	unused_0;
-+	u8	valid;
-+};
-+
-+/* hwrm_func_ptp_pin_cfg_input (size:256b/32B) */
-+struct hwrm_func_ptp_pin_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	#define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN0_STATE     0x1UL
-+	#define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN0_USAGE     0x2UL
-+	#define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN1_STATE     0x4UL
-+	#define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN1_USAGE     0x8UL
-+	#define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN2_STATE     0x10UL
-+	#define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN2_USAGE     0x20UL
-+	#define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN3_STATE     0x40UL
-+	#define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN3_USAGE     0x80UL
-+	u8	pin0_state;
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN0_STATE_DISABLED 0x0UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN0_STATE_ENABLED  0x1UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN0_STATE_LAST    FUNC_PTP_PIN_CFG_REQ_PIN0_STATE_ENABLED
-+	u8	pin0_usage;
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_NONE     0x0UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_PPS_IN   0x1UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_PPS_OUT  0x2UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_SYNC_IN  0x3UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_SYNC_OUT 0x4UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_LAST    FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_SYNC_OUT
-+	u8	pin1_state;
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN1_STATE_DISABLED 0x0UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN1_STATE_ENABLED  0x1UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN1_STATE_LAST    FUNC_PTP_PIN_CFG_REQ_PIN1_STATE_ENABLED
-+	u8	pin1_usage;
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_NONE     0x0UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_PPS_IN   0x1UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_PPS_OUT  0x2UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_SYNC_IN  0x3UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_SYNC_OUT 0x4UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_LAST    FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_SYNC_OUT
-+	u8	pin2_state;
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN2_STATE_DISABLED 0x0UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN2_STATE_ENABLED  0x1UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN2_STATE_LAST    FUNC_PTP_PIN_CFG_REQ_PIN2_STATE_ENABLED
-+	u8	pin2_usage;
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_NONE                      0x0UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_PPS_IN                    0x1UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_PPS_OUT                   0x2UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_SYNC_IN                   0x3UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_SYNC_OUT                  0x4UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_SYNCE_PRIMARY_CLOCK_OUT   0x5UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_SYNCE_SECONDARY_CLOCK_OUT 0x6UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_LAST                     FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_SYNCE_SECONDARY_CLOCK_OUT
-+	u8	pin3_state;
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN3_STATE_DISABLED 0x0UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN3_STATE_ENABLED  0x1UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN3_STATE_LAST    FUNC_PTP_PIN_CFG_REQ_PIN3_STATE_ENABLED
-+	u8	pin3_usage;
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_NONE                      0x0UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_PPS_IN                    0x1UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_PPS_OUT                   0x2UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_SYNC_IN                   0x3UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_SYNC_OUT                  0x4UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_SYNCE_PRIMARY_CLOCK_OUT   0x5UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_SYNCE_SECONDARY_CLOCK_OUT 0x6UL
-+	#define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_LAST                     FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_SYNCE_SECONDARY_CLOCK_OUT
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_func_ptp_pin_cfg_output (size:128b/16B) */
-+struct hwrm_func_ptp_pin_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_ptp_cfg_input (size:384b/48B) */
-+struct hwrm_func_ptp_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	enables;
-+	#define FUNC_PTP_CFG_REQ_ENABLES_PTP_PPS_EVENT               0x1UL
-+	#define FUNC_PTP_CFG_REQ_ENABLES_PTP_FREQ_ADJ_DLL_SOURCE     0x2UL
-+	#define FUNC_PTP_CFG_REQ_ENABLES_PTP_FREQ_ADJ_DLL_PHASE      0x4UL
-+	#define FUNC_PTP_CFG_REQ_ENABLES_PTP_FREQ_ADJ_EXT_PERIOD     0x8UL
-+	#define FUNC_PTP_CFG_REQ_ENABLES_PTP_FREQ_ADJ_EXT_UP         0x10UL
-+	#define FUNC_PTP_CFG_REQ_ENABLES_PTP_FREQ_ADJ_EXT_PHASE      0x20UL
-+	#define FUNC_PTP_CFG_REQ_ENABLES_PTP_SET_TIME                0x40UL
-+	u8	ptp_pps_event;
-+	#define FUNC_PTP_CFG_REQ_PTP_PPS_EVENT_INTERNAL     0x1UL
-+	#define FUNC_PTP_CFG_REQ_PTP_PPS_EVENT_EXTERNAL     0x2UL
-+	u8	ptp_freq_adj_dll_source;
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_NONE    0x0UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_TSIO_0  0x1UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_TSIO_1  0x2UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_TSIO_2  0x3UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_TSIO_3  0x4UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_PORT_0  0x5UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_PORT_1  0x6UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_PORT_2  0x7UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_PORT_3  0x8UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_INVALID 0xffUL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_LAST   FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_INVALID
-+	u8	ptp_freq_adj_dll_phase;
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_NONE 0x0UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_4K   0x1UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_8K   0x2UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_10M  0x3UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_25M  0x4UL
-+	#define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_LAST FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_25M
-+	u8	unused_0[3];
-+	__le32	ptp_freq_adj_ext_period;
-+	__le32	ptp_freq_adj_ext_up;
-+	__le32	ptp_freq_adj_ext_phase_lower;
-+	__le32	ptp_freq_adj_ext_phase_upper;
-+	__le64	ptp_set_time;
-+};
-+
-+/* hwrm_func_ptp_cfg_output (size:128b/16B) */
-+struct hwrm_func_ptp_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_ptp_ts_query_input (size:192b/24B) */
-+struct hwrm_func_ptp_ts_query_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define FUNC_PTP_TS_QUERY_REQ_FLAGS_PPS_TIME     0x1UL
-+	#define FUNC_PTP_TS_QUERY_REQ_FLAGS_PTM_TIME     0x2UL
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_func_ptp_ts_query_output (size:320b/40B) */
-+struct hwrm_func_ptp_ts_query_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le64	pps_event_ts;
-+	__le64	ptm_local_ts;
-+	__le64	ptm_system_ts;
-+	__le32	ptm_link_delay;
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_ptp_ext_cfg_input (size:256b/32B) */
-+struct hwrm_func_ptp_ext_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	enables;
-+	#define FUNC_PTP_EXT_CFG_REQ_ENABLES_PHC_MASTER_FID     0x1UL
-+	#define FUNC_PTP_EXT_CFG_REQ_ENABLES_PHC_SEC_FID        0x2UL
-+	#define FUNC_PTP_EXT_CFG_REQ_ENABLES_PHC_SEC_MODE       0x4UL
-+	#define FUNC_PTP_EXT_CFG_REQ_ENABLES_FAILOVER_TIMER     0x8UL
-+	__le16	phc_master_fid;
-+	__le16	phc_sec_fid;
-+	u8	phc_sec_mode;
-+	#define FUNC_PTP_EXT_CFG_REQ_PHC_SEC_MODE_SWITCH  0x0UL
-+	#define FUNC_PTP_EXT_CFG_REQ_PHC_SEC_MODE_ALL     0x1UL
-+	#define FUNC_PTP_EXT_CFG_REQ_PHC_SEC_MODE_PF_ONLY 0x2UL
-+	#define FUNC_PTP_EXT_CFG_REQ_PHC_SEC_MODE_LAST   FUNC_PTP_EXT_CFG_REQ_PHC_SEC_MODE_PF_ONLY
-+	u8	unused_0;
-+	__le32	failover_timer;
-+	u8	unused_1[4];
-+};
-+
-+/* hwrm_func_ptp_ext_cfg_output (size:128b/16B) */
-+struct hwrm_func_ptp_ext_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_ptp_ext_qcfg_input (size:192b/24B) */
-+struct hwrm_func_ptp_ext_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	unused_0[8];
-+};
-+
-+/* hwrm_func_ptp_ext_qcfg_output (size:256b/32B) */
-+struct hwrm_func_ptp_ext_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	phc_master_fid;
-+	__le16	phc_sec_fid;
-+	__le16	phc_active_fid0;
-+	__le16	phc_active_fid1;
-+	__le32	last_failover_event;
-+	__le16	from_fid;
-+	__le16	to_fid;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_backing_store_cfg_v2_input (size:512b/64B) */
-+struct hwrm_func_backing_store_cfg_v2_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	type;
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_QP                  0x0UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_SRQ                 0x1UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CQ                  0x2UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_VNIC                0x3UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_STAT                0x4UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_SP_TQM_RING         0x5UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_FP_TQM_RING         0x6UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_MRAV                0xeUL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_TIM                 0xfUL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_TX_CK               0x13UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_RX_CK               0x14UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_MP_TQM_RING         0x15UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_SQ_DB_SHADOW        0x16UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_RQ_DB_SHADOW        0x17UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_SRQ_DB_SHADOW       0x18UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CQ_DB_SHADOW        0x19UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_TBL_SCOPE           0x1cUL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_XID_PARTITION       0x1dUL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_SRT_TRACE           0x1eUL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_SRT2_TRACE          0x1fUL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CRT_TRACE           0x20UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CRT2_TRACE          0x21UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_RIGP0_TRACE         0x22UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_L2_HWRM_TRACE       0x23UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_ROCE_HWRM_TRACE     0x24UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_TTX_PACING_TQM_RING 0x25UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CA0_TRACE           0x26UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CA1_TRACE           0x27UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CA2_TRACE           0x28UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_RIGP1_TRACE         0x29UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_AFM_KONG_HWRM_TRACE 0x2aUL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_ERR_QPC_TRACE       0x2bUL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_INVALID             0xffffUL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_LAST               FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_INVALID
-+	__le16	instance;
-+	__le32	flags;
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_PREBOOT_MODE        0x1UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE     0x2UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_EXTEND           0x4UL
-+	__le64	page_dir;
-+	__le32	num_entries;
-+	__le16	entry_size;
-+	u8	page_size_pbl_level;
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_MASK  0xfUL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_SFT   0
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_LVL_0   0x0UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_LVL_1   0x1UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_LVL_2   0x2UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_LAST   FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_LVL_2
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_LAST   FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_1G
-+	u8	subtype_valid_cnt;
-+	__le32	split_entry_0;
-+	__le32	split_entry_1;
-+	__le32	split_entry_2;
-+	__le32	split_entry_3;
-+	__le32	enables;
-+	#define FUNC_BACKING_STORE_CFG_V2_REQ_ENABLES_NEXT_BS_OFFSET     0x1UL
-+	__le32	next_bs_offset;
-+};
-+
-+/* hwrm_func_backing_store_cfg_v2_output (size:128b/16B) */
-+struct hwrm_func_backing_store_cfg_v2_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	rsvd0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_backing_store_qcfg_v2_input (size:192b/24B) */
-+struct hwrm_func_backing_store_qcfg_v2_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	type;
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_QP                  0x0UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_SRQ                 0x1UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CQ                  0x2UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_VNIC                0x3UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_STAT                0x4UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_SP_TQM_RING         0x5UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_FP_TQM_RING         0x6UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_MRAV                0xeUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_TIM                 0xfUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_TX_CK               0x13UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_RX_CK               0x14UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_MP_TQM_RING         0x15UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_SQ_DB_SHADOW        0x16UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_RQ_DB_SHADOW        0x17UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_SRQ_DB_SHADOW       0x18UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CQ_DB_SHADOW        0x19UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_TBL_SCOPE           0x1cUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_XID_PARTITION_TABLE 0x1dUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_SRT_TRACE           0x1eUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_SRT2_TRACE          0x1fUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CRT_TRACE           0x20UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CRT2_TRACE          0x21UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_RIGP0_TRACE         0x22UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_L2_HWRM_TRACE       0x23UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_ROCE_HWRM_TRACE     0x24UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_TTX_PACING_TQM_RING 0x25UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CA0_TRACE           0x26UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CA1_TRACE           0x27UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CA2_TRACE           0x28UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_RIGP1_TRACE         0x29UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_AFM_KONG_HWRM_TRACE 0x2aUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_ERR_QPC_TRACE       0x2bUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_INVALID             0xffffUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_LAST               FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_INVALID
-+	__le16	instance;
-+	u8	rsvd[4];
-+};
-+
-+/* hwrm_func_backing_store_qcfg_v2_output (size:448b/56B) */
-+struct hwrm_func_backing_store_qcfg_v2_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	type;
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_QP                  0x0UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_SRQ                 0x1UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_CQ                  0x2UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_VNIC                0x3UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_STAT                0x4UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_SP_TQM_RING         0x5UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_FP_TQM_RING         0x6UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_MRAV                0xeUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_TIM                 0xfUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_TX_CK               0x13UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_RX_CK               0x14UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_MP_TQM_RING         0x15UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_TBL_SCOPE           0x1cUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_XID_PARTITION       0x1dUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_SRT_TRACE           0x1eUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_SRT2_TRACE          0x1fUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_CRT_TRACE           0x20UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_CRT2_TRACE          0x21UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_RIGP0_TRACE         0x22UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_L2_HWRM_TRACE       0x23UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_ROCE_HWRM_TRACE     0x24UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_TTX_PACING_TQM_RING 0x25UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_CA0_TRACE           0x26UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_CA1_TRACE           0x27UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_CA2_TRACE           0x28UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_RIGP1_TRACE         0x29UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_ERR_QPC_TRACE       0x2aUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_INVALID             0xffffUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_LAST               FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_INVALID
-+	__le16	instance;
-+	__le32	flags;
-+	__le64	page_dir;
-+	__le32	num_entries;
-+	u8	page_size_pbl_level;
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_MASK  0xfUL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_SFT   0
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_LVL_0   0x0UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_LVL_1   0x1UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_LVL_2   0x2UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_LAST   FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_LVL_2
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_MASK  0xf0UL
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_SFT   4
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_4K   (0x0UL << 4)
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_8K   (0x1UL << 4)
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_64K  (0x2UL << 4)
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_2M   (0x3UL << 4)
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_8M   (0x4UL << 4)
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_1G   (0x5UL << 4)
-+	#define FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_LAST   FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_1G
-+	u8	subtype_valid_cnt;
-+	u8	rsvd[2];
-+	__le32	split_entry_0;
-+	__le32	split_entry_1;
-+	__le32	split_entry_2;
-+	__le32	split_entry_3;
-+	u8	rsvd2[7];
-+	u8	valid;
-+};
-+
-+/* qpc_split_entries (size:128b/16B) */
-+struct qpc_split_entries {
-+	__le32	qp_num_l2_entries;
-+	__le32	qp_num_qp1_entries;
-+	__le32	qp_num_fast_qpmd_entries;
-+	__le32	rsvd;
-+};
-+
-+/* srq_split_entries (size:128b/16B) */
-+struct srq_split_entries {
-+	__le32	srq_num_l2_entries;
-+	__le32	rsvd;
-+	__le32	rsvd2[2];
-+};
-+
-+/* cq_split_entries (size:128b/16B) */
-+struct cq_split_entries {
-+	__le32	cq_num_l2_entries;
-+	__le32	rsvd;
-+	__le32	rsvd2[2];
-+};
-+
-+/* vnic_split_entries (size:128b/16B) */
-+struct vnic_split_entries {
-+	__le32	vnic_num_vnic_entries;
-+	__le32	rsvd;
-+	__le32	rsvd2[2];
-+};
-+
-+/* mrav_split_entries (size:128b/16B) */
-+struct mrav_split_entries {
-+	__le32	mrav_num_av_entries;
-+	__le32	rsvd;
-+	__le32	rsvd2[2];
-+};
-+
-+/* ts_split_entries (size:128b/16B) */
-+struct ts_split_entries {
-+	__le32	region_num_entries;
-+	u8	tsid;
-+	u8	lkup_static_bkt_cnt_exp[2];
-+	u8	locked;
-+	__le32	rsvd2[2];
-+};
-+
-+/* ck_split_entries (size:128b/16B) */
-+struct ck_split_entries {
-+	__le32	num_quic_entries;
-+	__le32	rsvd;
-+	__le32	rsvd2[2];
-+};
-+
-+/* hwrm_func_backing_store_qcaps_v2_input (size:192b/24B) */
-+struct hwrm_func_backing_store_qcaps_v2_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	type;
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_QP                  0x0UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_SRQ                 0x1UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CQ                  0x2UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_VNIC                0x3UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_STAT                0x4UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_SP_TQM_RING         0x5UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_FP_TQM_RING         0x6UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_MRAV                0xeUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_TIM                 0xfUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_TX_CK               0x13UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_RX_CK               0x14UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_MP_TQM_RING         0x15UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_SQ_DB_SHADOW        0x16UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_RQ_DB_SHADOW        0x17UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_SRQ_DB_SHADOW       0x18UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CQ_DB_SHADOW        0x19UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_TBL_SCOPE           0x1cUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_XID_PARTITION       0x1dUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_SRT_TRACE           0x1eUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_SRT2_TRACE          0x1fUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CRT_TRACE           0x20UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CRT2_TRACE          0x21UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_RIGP0_TRACE         0x22UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_L2_HWRM_TRACE       0x23UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_ROCE_HWRM_TRACE     0x24UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_TTX_PACING_TQM_RING 0x25UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CA0_TRACE           0x26UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CA1_TRACE           0x27UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CA2_TRACE           0x28UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_RIGP1_TRACE         0x29UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_AFM_KONG_HWRM_TRACE 0x2aUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_ERR_QPC_TRACE       0x2bUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_INVALID             0xffffUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_LAST               FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_INVALID
-+	u8	rsvd[6];
-+};
-+
-+/* hwrm_func_backing_store_qcaps_v2_output (size:448b/56B) */
-+struct hwrm_func_backing_store_qcaps_v2_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	type;
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_QP                  0x0UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_SRQ                 0x1UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CQ                  0x2UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_VNIC                0x3UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_STAT                0x4UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_SP_TQM_RING         0x5UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_FP_TQM_RING         0x6UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_MRAV                0xeUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_TIM                 0xfUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_TX_CK               0x13UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_RX_CK               0x14UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_MP_TQM_RING         0x15UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_SQ_DB_SHADOW        0x16UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_RQ_DB_SHADOW        0x17UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_SRQ_DB_SHADOW       0x18UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CQ_DB_SHADOW        0x19UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_TBL_SCOPE           0x1cUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_XID_PARTITION       0x1dUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_SRT_TRACE           0x1eUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_SRT2_TRACE          0x1fUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CRT_TRACE           0x20UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CRT2_TRACE          0x21UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_RIGP0_TRACE         0x22UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_L2_HWRM_TRACE       0x23UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_ROCE_HWRM_TRACE     0x24UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_TTX_PACING_TQM_RING 0x25UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CA0_TRACE           0x26UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CA1_TRACE           0x27UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CA2_TRACE           0x28UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_RIGP1_TRACE         0x29UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_AFM_KONG_HWRM_TRACE 0x2aUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_ERR_QPC_TRACE       0x2bUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_INVALID             0xffffUL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_LAST               FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_INVALID
-+	__le16	entry_size;
-+	__le32	flags;
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_ENABLE_CTX_KIND_INIT            0x1UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_TYPE_VALID                      0x2UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_DRIVER_MANAGED_MEMORY           0x4UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_ROCE_QP_PSEUDO_STATIC_ALLOC     0x8UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_FW_DBG_TRACE                    0x10UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_FW_BIN_DBG_TRACE                0x20UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_NEXT_BS_OFFSET                  0x40UL
-+	__le32	instance_bit_map;
-+	u8	ctx_init_value;
-+	u8	ctx_init_offset;
-+	u8	entry_multiple;
-+	u8	rsvd;
-+	__le32	max_num_entries;
-+	__le32	min_num_entries;
-+	__le16	next_valid_type;
-+	u8	subtype_valid_cnt;
-+	u8	exact_cnt_bit_map;
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_EXACT_CNT_BIT_MAP_SPLIT_ENTRY_0_EXACT     0x1UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_EXACT_CNT_BIT_MAP_SPLIT_ENTRY_1_EXACT     0x2UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_EXACT_CNT_BIT_MAP_SPLIT_ENTRY_2_EXACT     0x4UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_EXACT_CNT_BIT_MAP_SPLIT_ENTRY_3_EXACT     0x8UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_EXACT_CNT_BIT_MAP_UNUSED_MASK             0xf0UL
-+	#define FUNC_BACKING_STORE_QCAPS_V2_RESP_EXACT_CNT_BIT_MAP_UNUSED_SFT              4
-+	__le32	split_entry_0;
-+	__le32	split_entry_1;
-+	__le32	split_entry_2;
-+	__le32	split_entry_3;
-+	__le16	max_instance_count;
-+	u8	rsvd3;
-+	u8	valid;
-+};
-+
-+/* hwrm_func_dbr_pacing_qcfg_input (size:128b/16B) */
-+struct hwrm_func_dbr_pacing_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+};
-+
-+/* hwrm_func_dbr_pacing_qcfg_output (size:512b/64B) */
-+struct hwrm_func_dbr_pacing_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	flags;
-+	#define FUNC_DBR_PACING_QCFG_RESP_FLAGS_DBR_NQ_EVENT_ENABLED     0x1UL
-+	u8	unused_0[7];
-+	__le32	dbr_stat_db_fifo_reg;
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK    0x3UL
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_SFT     0
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_PCIE_CFG  0x0UL
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC       0x1UL
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_BAR0      0x2UL
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_BAR1      0x3UL
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_LAST     FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_BAR1
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_MASK          0xfffffffcUL
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SFT           2
-+	__le32	dbr_stat_db_fifo_reg_watermark_mask;
-+	u8	dbr_stat_db_fifo_reg_watermark_shift;
-+	u8	unused_1[3];
-+	__le32	dbr_stat_db_fifo_reg_fifo_room_mask;
-+	u8	dbr_stat_db_fifo_reg_fifo_room_shift;
-+	u8	unused_2[3];
-+	__le32	dbr_throttling_aeq_arm_reg;
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_MASK    0x3UL
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_SFT     0
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_PCIE_CFG  0x0UL
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_GRC       0x1UL
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_BAR0      0x2UL
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_BAR1      0x3UL
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_LAST     FUNC_DBR_PACING_QCFG_RESP_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_BAR1
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_THROTTLING_AEQ_ARM_REG_ADDR_MASK          0xfffffffcUL
-+	#define FUNC_DBR_PACING_QCFG_RESP_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SFT           2
-+	u8	dbr_throttling_aeq_arm_reg_val;
-+	u8	unused_3[3];
-+	__le32	dbr_stat_db_max_fifo_depth;
-+	__le32	primary_nq_id;
-+	__le32	pacing_threshold;
-+	u8	unused_4[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_func_drv_if_change_input (size:192b/24B) */
-+struct hwrm_func_drv_if_change_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP     0x1UL
-+	__le32	unused;
-+};
-+
-+/* hwrm_func_drv_if_change_output (size:128b/16B) */
-+struct hwrm_func_drv_if_change_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	flags;
-+	#define FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE           0x1UL
-+	#define FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE     0x2UL
-+	#define FUNC_DRV_IF_CHANGE_RESP_FLAGS_CAPS_CHANGE           0x4UL
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_phy_cfg_input (size:512b/64B) */
-+struct hwrm_port_phy_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define PORT_PHY_CFG_REQ_FLAGS_RESET_PHY                  0x1UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_DEPRECATED                 0x2UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FORCE                      0x4UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG            0x8UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE                 0x10UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE                0x20UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE          0x40UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE         0x80UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_AUTONEG_ENABLE         0x100UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_AUTONEG_DISABLE        0x200UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE74_ENABLE        0x400UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE74_DISABLE       0x800UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE91_ENABLE        0x1000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE91_DISABLE       0x2000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN             0x4000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_1XN_ENABLE       0x8000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_1XN_DISABLE      0x10000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_IEEE_ENABLE      0x20000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_IEEE_DISABLE     0x40000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_1XN_ENABLE       0x80000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_1XN_DISABLE      0x100000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_IEEE_ENABLE      0x200000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_IEEE_DISABLE     0x400000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_LINK_TRAINING_ENABLE       0x800000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_LINK_TRAINING_DISABLE      0x1000000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_PRECODING_ENABLE           0x2000000UL
-+	#define PORT_PHY_CFG_REQ_FLAGS_PRECODING_DISABLE          0x4000000UL
-+	__le32	enables;
-+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE                     0x1UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_DUPLEX                   0x2UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE                    0x4UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED               0x8UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK          0x10UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_WIRESPEED                     0x20UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_LPBK                          0x40UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_PREEMPHASIS                   0x80UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE                   0x100UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_EEE_LINK_SPEED_MASK           0x200UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_TX_LPI_TIMER                  0x400UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED         0x800UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK     0x1000UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2            0x2000UL
-+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK        0x4000UL
-+	__le16	port_id;
-+	__le16	force_link_speed;
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100MB 0x1UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_1GB   0xaUL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_2GB   0x14UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_2_5GB 0x19UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10GB  0x64UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_20GB  0xc8UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_25GB  0xfaUL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_40GB  0x190UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_50GB  0x1f4UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100GB 0x3e8UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10MB  0xffffUL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_LAST PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10MB
-+	u8	auto_mode;
-+	#define PORT_PHY_CFG_REQ_AUTO_MODE_NONE         0x0UL
-+	#define PORT_PHY_CFG_REQ_AUTO_MODE_ALL_SPEEDS   0x1UL
-+	#define PORT_PHY_CFG_REQ_AUTO_MODE_ONE_SPEED    0x2UL
-+	#define PORT_PHY_CFG_REQ_AUTO_MODE_ONE_OR_BELOW 0x3UL
-+	#define PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK   0x4UL
-+	#define PORT_PHY_CFG_REQ_AUTO_MODE_LAST        PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK
-+	u8	auto_duplex;
-+	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_HALF 0x0UL
-+	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_FULL 0x1UL
-+	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_BOTH 0x2UL
-+	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_LAST PORT_PHY_CFG_REQ_AUTO_DUPLEX_BOTH
-+	u8	auto_pause;
-+	#define PORT_PHY_CFG_REQ_AUTO_PAUSE_TX                0x1UL
-+	#define PORT_PHY_CFG_REQ_AUTO_PAUSE_RX                0x2UL
-+	#define PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE     0x4UL
-+	u8	mgmt_flag;
-+	#define PORT_PHY_CFG_REQ_MGMT_FLAG_LINK_RELEASE     0x1UL
-+	#define PORT_PHY_CFG_REQ_MGMT_FLAG_MGMT_VALID       0x80UL
-+	__le16	auto_link_speed;
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_100MB 0x1UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_1GB   0xaUL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_2GB   0x14UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_2_5GB 0x19UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10GB  0x64UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_20GB  0xc8UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_25GB  0xfaUL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_40GB  0x190UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_50GB  0x1f4UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_100GB 0x3e8UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10MB  0xffffUL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_LAST PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10MB
-+	__le16	auto_link_speed_mask;
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100MBHD     0x1UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100MB       0x2UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_1GBHD       0x4UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_1GB         0x8UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_2GB         0x10UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_2_5GB       0x20UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10GB        0x40UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_20GB        0x80UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_25GB        0x100UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_40GB        0x200UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_50GB        0x400UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100GB       0x800UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10MBHD      0x1000UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10MB        0x2000UL
-+	u8	wirespeed;
-+	#define PORT_PHY_CFG_REQ_WIRESPEED_OFF 0x0UL
-+	#define PORT_PHY_CFG_REQ_WIRESPEED_ON  0x1UL
-+	#define PORT_PHY_CFG_REQ_WIRESPEED_LAST PORT_PHY_CFG_REQ_WIRESPEED_ON
-+	u8	lpbk;
-+	#define PORT_PHY_CFG_REQ_LPBK_NONE     0x0UL
-+	#define PORT_PHY_CFG_REQ_LPBK_LOCAL    0x1UL
-+	#define PORT_PHY_CFG_REQ_LPBK_REMOTE   0x2UL
-+	#define PORT_PHY_CFG_REQ_LPBK_EXTERNAL 0x3UL
-+	#define PORT_PHY_CFG_REQ_LPBK_LAST    PORT_PHY_CFG_REQ_LPBK_EXTERNAL
-+	u8	force_pause;
-+	#define PORT_PHY_CFG_REQ_FORCE_PAUSE_TX     0x1UL
-+	#define PORT_PHY_CFG_REQ_FORCE_PAUSE_RX     0x2UL
-+	u8	unused_1;
-+	__le32	preemphasis;
-+	__le16	eee_link_speed_mask;
-+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD1     0x1UL
-+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_100MB     0x2UL
-+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD2     0x4UL
-+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_1GB       0x8UL
-+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD3     0x10UL
-+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD4     0x20UL
-+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_10GB      0x40UL
-+	__le16	force_pam4_link_speed;
-+	#define PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_50GB  0x1f4UL
-+	#define PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_100GB 0x3e8UL
-+	#define PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_200GB 0x7d0UL
-+	#define PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_LAST PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_200GB
-+	__le32	tx_lpi_timer;
-+	#define PORT_PHY_CFG_REQ_TX_LPI_TIMER_MASK 0xffffffUL
-+	#define PORT_PHY_CFG_REQ_TX_LPI_TIMER_SFT 0
-+	__le16	auto_link_pam4_speed_mask;
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_PAM4_SPEED_MASK_50G      0x1UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_PAM4_SPEED_MASK_100G     0x2UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_PAM4_SPEED_MASK_200G     0x4UL
-+	__le16	force_link_speeds2;
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_1GB            0xaUL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_10GB           0x64UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_25GB           0xfaUL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_40GB           0x190UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB           0x1f4UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB          0x3e8UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB_PAM4_56   0x1f5UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_56  0x3e9UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_56  0x7d1UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_56  0xfa1UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_112 0x3eaUL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_112 0x7d2UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_112 0xfa2UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_800GB_PAM4_112 0x1f42UL
-+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_LAST          PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_800GB_PAM4_112
-+	__le16	auto_link_speeds2_mask;
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_1GB                0x1UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_10GB               0x2UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_25GB               0x4UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_40GB               0x8UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_50GB               0x10UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_100GB              0x20UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_50GB_PAM4_56       0x40UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_100GB_PAM4_56      0x80UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_200GB_PAM4_56      0x100UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_400GB_PAM4_56      0x200UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_100GB_PAM4_112     0x400UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_200GB_PAM4_112     0x800UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_400GB_PAM4_112     0x1000UL
-+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_800GB_PAM4_112     0x2000UL
-+	u8	unused_2[6];
-+};
-+
-+/* hwrm_port_phy_cfg_output (size:128b/16B) */
-+struct hwrm_port_phy_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_phy_cfg_cmd_err (size:64b/8B) */
-+struct hwrm_port_phy_cfg_cmd_err {
-+	u8	code;
-+	#define PORT_PHY_CFG_CMD_ERR_CODE_UNKNOWN       0x0UL
-+	#define PORT_PHY_CFG_CMD_ERR_CODE_ILLEGAL_SPEED 0x1UL
-+	#define PORT_PHY_CFG_CMD_ERR_CODE_RETRY         0x2UL
-+	#define PORT_PHY_CFG_CMD_ERR_CODE_LAST         PORT_PHY_CFG_CMD_ERR_CODE_RETRY
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_port_phy_qcfg_input (size:192b/24B) */
-+struct hwrm_port_phy_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_port_phy_qcfg_output (size:832b/104B) */
-+struct hwrm_port_phy_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	link;
-+	#define PORT_PHY_QCFG_RESP_LINK_NO_LINK 0x0UL
-+	#define PORT_PHY_QCFG_RESP_LINK_SIGNAL  0x1UL
-+	#define PORT_PHY_QCFG_RESP_LINK_LINK    0x2UL
-+	#define PORT_PHY_QCFG_RESP_LINK_LAST   PORT_PHY_QCFG_RESP_LINK_LINK
-+	u8	active_fec_signal_mode;
-+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK                0xfUL
-+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_SFT                 0
-+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ                   0x0UL
-+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4                  0x1UL
-+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112              0x2UL
-+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_LAST                 PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112
-+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK                 0xf0UL
-+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_SFT                  4
-+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE        (0x0UL << 4)
-+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE    (0x1UL << 4)
-+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE    (0x2UL << 4)
-+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE   (0x3UL << 4)
-+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE  (0x4UL << 4)
-+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE   (0x5UL << 4)
-+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE  (0x6UL << 4)
-+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_LAST                  PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE
-+	__le16	link_speed;
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100MB 0x1UL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_1GB   0xaUL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_2GB   0x14UL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_2_5GB 0x19UL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10GB  0x64UL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_20GB  0xc8UL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_25GB  0xfaUL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_40GB  0x190UL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_50GB  0x1f4UL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100GB 0x3e8UL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_200GB 0x7d0UL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_400GB 0xfa0UL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_800GB 0x1f40UL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10MB  0xffffUL
-+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_LAST PORT_PHY_QCFG_RESP_LINK_SPEED_10MB
-+	u8	duplex_cfg;
-+	#define PORT_PHY_QCFG_RESP_DUPLEX_CFG_HALF 0x0UL
-+	#define PORT_PHY_QCFG_RESP_DUPLEX_CFG_FULL 0x1UL
-+	#define PORT_PHY_QCFG_RESP_DUPLEX_CFG_LAST PORT_PHY_QCFG_RESP_DUPLEX_CFG_FULL
-+	u8	pause;
-+	#define PORT_PHY_QCFG_RESP_PAUSE_TX     0x1UL
-+	#define PORT_PHY_QCFG_RESP_PAUSE_RX     0x2UL
-+	__le16	support_speeds;
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_100MBHD     0x1UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_100MB       0x2UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_1GBHD       0x4UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_1GB         0x8UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_2GB         0x10UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_2_5GB       0x20UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_10GB        0x40UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_20GB        0x80UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_25GB        0x100UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_40GB        0x200UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_50GB        0x400UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_100GB       0x800UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_10MBHD      0x1000UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_10MB        0x2000UL
-+	__le16	force_link_speed;
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_100MB 0x1UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_1GB   0xaUL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_2GB   0x14UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_2_5GB 0x19UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_10GB  0x64UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_20GB  0xc8UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_25GB  0xfaUL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_40GB  0x190UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_50GB  0x1f4UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_100GB 0x3e8UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_10MB  0xffffUL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_LAST PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_10MB
-+	u8	auto_mode;
-+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_NONE         0x0UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ALL_SPEEDS   0x1UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ONE_SPEED    0x2UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ONE_OR_BELOW 0x3UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_SPEED_MASK   0x4UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_LAST        PORT_PHY_QCFG_RESP_AUTO_MODE_SPEED_MASK
-+	u8	auto_pause;
-+	#define PORT_PHY_QCFG_RESP_AUTO_PAUSE_TX                0x1UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_PAUSE_RX                0x2UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_PAUSE_AUTONEG_PAUSE     0x4UL
-+	__le16	auto_link_speed;
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_100MB 0x1UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_1GB   0xaUL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_2GB   0x14UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_2_5GB 0x19UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_10GB  0x64UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_20GB  0xc8UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_25GB  0xfaUL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_40GB  0x190UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_50GB  0x1f4UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_100GB 0x3e8UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_10MB  0xffffUL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_LAST PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_10MB
-+	__le16	auto_link_speed_mask;
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_100MBHD     0x1UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_100MB       0x2UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_1GBHD       0x4UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_1GB         0x8UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_2GB         0x10UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_2_5GB       0x20UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_10GB        0x40UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_20GB        0x80UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_25GB        0x100UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_40GB        0x200UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_50GB        0x400UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_100GB       0x800UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_10MBHD      0x1000UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_10MB        0x2000UL
-+	u8	wirespeed;
-+	#define PORT_PHY_QCFG_RESP_WIRESPEED_OFF 0x0UL
-+	#define PORT_PHY_QCFG_RESP_WIRESPEED_ON  0x1UL
-+	#define PORT_PHY_QCFG_RESP_WIRESPEED_LAST PORT_PHY_QCFG_RESP_WIRESPEED_ON
-+	u8	lpbk;
-+	#define PORT_PHY_QCFG_RESP_LPBK_NONE     0x0UL
-+	#define PORT_PHY_QCFG_RESP_LPBK_LOCAL    0x1UL
-+	#define PORT_PHY_QCFG_RESP_LPBK_REMOTE   0x2UL
-+	#define PORT_PHY_QCFG_RESP_LPBK_EXTERNAL 0x3UL
-+	#define PORT_PHY_QCFG_RESP_LPBK_LAST    PORT_PHY_QCFG_RESP_LPBK_EXTERNAL
-+	u8	force_pause;
-+	#define PORT_PHY_QCFG_RESP_FORCE_PAUSE_TX     0x1UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_PAUSE_RX     0x2UL
-+	u8	module_status;
-+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NONE          0x0UL
-+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX     0x1UL
-+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG    0x2UL
-+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN       0x3UL
-+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NOTINSERTED   0x4UL
-+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_CURRENTFAULT  0x5UL
-+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_OVERHEATED    0x6UL
-+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NOTAPPLICABLE 0xffUL
-+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_LAST         PORT_PHY_QCFG_RESP_MODULE_STATUS_NOTAPPLICABLE
-+	__le32	preemphasis;
-+	u8	phy_maj;
-+	u8	phy_min;
-+	u8	phy_bld;
-+	u8	phy_type;
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_UNKNOWN          0x0UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASECR           0x1UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR4          0x2UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASELR           0x3UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASESR           0x4UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR2          0x5UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKX           0x6UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR           0x7UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASET            0x8UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASETE           0x9UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_SGMIIEXTPHY      0xaUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_25G_BASECR_CA_L  0xbUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_25G_BASECR_CA_S  0xcUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_25G_BASECR_CA_N  0xdUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_25G_BASESR       0xeUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASECR4     0xfUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR4     0x10UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASELR4     0x11UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASEER4     0x12UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR10    0x13UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASECR4      0x14UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASESR4      0x15UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASELR4      0x16UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASEER4      0x17UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_ACTIVE_CABLE 0x18UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASET         0x19UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASESX        0x1aUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASECX        0x1bUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASECR4     0x1cUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASESR4     0x1dUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASELR4     0x1eUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASEER4     0x1fUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASECR       0x20UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASESR       0x21UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASELR       0x22UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASEER       0x23UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASECR2     0x24UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR2     0x25UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASELR2     0x26UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASEER2     0x27UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASECR      0x28UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR      0x29UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASELR      0x2aUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASEER      0x2bUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASECR2     0x2cUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASESR2     0x2dUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASELR2     0x2eUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASEER2     0x2fUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASECR8     0x30UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASESR8     0x31UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASELR8     0x32UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASEER8     0x33UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASECR4     0x34UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASESR4     0x35UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASELR4     0x36UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASEER4     0x37UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_800G_BASECR8     0x38UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_800G_BASESR8     0x39UL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_800G_BASELR8     0x3aUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_800G_BASEER8     0x3bUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_800G_BASEFR8     0x3cUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_800G_BASEDR8     0x3dUL
-+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_LAST            PORT_PHY_QCFG_RESP_PHY_TYPE_800G_BASEDR8
-+	u8	media_type;
-+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_UNKNOWN   0x0UL
-+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_TP        0x1UL
-+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_DAC       0x2UL
-+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_FIBRE     0x3UL
-+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_BACKPLANE 0x4UL
-+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_LAST     PORT_PHY_QCFG_RESP_MEDIA_TYPE_BACKPLANE
-+	u8	xcvr_pkg_type;
-+	#define PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_INTERNAL 0x1UL
-+	#define PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_EXTERNAL 0x2UL
-+	#define PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_LAST         PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_EXTERNAL
-+	u8	eee_config_phy_addr;
-+	#define PORT_PHY_QCFG_RESP_PHY_ADDR_MASK              0x1fUL
-+	#define PORT_PHY_QCFG_RESP_PHY_ADDR_SFT               0
-+	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_MASK            0xe0UL
-+	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_SFT             5
-+	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED      0x20UL
-+	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE       0x40UL
-+	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI       0x80UL
-+	u8	parallel_detect;
-+	#define PORT_PHY_QCFG_RESP_PARALLEL_DETECT     0x1UL
-+	__le16	link_partner_adv_speeds;
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_100MBHD     0x1UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_100MB       0x2UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_1GBHD       0x4UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_1GB         0x8UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_2GB         0x10UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_2_5GB       0x20UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_10GB        0x40UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_20GB        0x80UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_25GB        0x100UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_40GB        0x200UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_50GB        0x400UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_100GB       0x800UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_10MBHD      0x1000UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_10MB        0x2000UL
-+	u8	link_partner_adv_auto_mode;
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_NONE         0x0UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ALL_SPEEDS   0x1UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ONE_SPEED    0x2UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ONE_OR_BELOW 0x3UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_SPEED_MASK   0x4UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_LAST        PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_SPEED_MASK
-+	u8	link_partner_adv_pause;
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_PAUSE_TX     0x1UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_PAUSE_RX     0x2UL
-+	__le16	adv_eee_link_speed_mask;
-+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_RSVD1     0x1UL
-+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_100MB     0x2UL
-+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_RSVD2     0x4UL
-+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_1GB       0x8UL
-+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_RSVD3     0x10UL
-+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_RSVD4     0x20UL
-+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_10GB      0x40UL
-+	__le16	link_partner_adv_eee_link_speed_mask;
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD1     0x1UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_100MB     0x2UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD2     0x4UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_1GB       0x8UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD3     0x10UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD4     0x20UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_10GB      0x40UL
-+	__le32	xcvr_identifier_type_tx_lpi_timer;
-+	#define PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK            0xffffffUL
-+	#define PORT_PHY_QCFG_RESP_TX_LPI_TIMER_SFT             0
-+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_MASK    0xff000000UL
-+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_SFT     24
-+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_UNKNOWN   (0x0UL << 24)
-+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_SFP       (0x3UL << 24)
-+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_QSFP      (0xcUL << 24)
-+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_QSFPPLUS  (0xdUL << 24)
-+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_QSFP28    (0x11UL << 24)
-+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_QSFPDD    (0x18UL << 24)
-+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_QSFP112   (0x1eUL << 24)
-+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_SFPDD     (0x1fUL << 24)
-+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_CSFP      (0x20UL << 24)
-+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_LAST     PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_CSFP
-+	__le16	fec_cfg;
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED           0x1UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_AUTONEG_SUPPORTED        0x2UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_AUTONEG_ENABLED          0x4UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE74_SUPPORTED       0x8UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE74_ENABLED         0x10UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_SUPPORTED       0x20UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_ENABLED         0x40UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_1XN_SUPPORTED      0x80UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_1XN_ENABLED        0x100UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_IEEE_SUPPORTED     0x200UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_IEEE_ENABLED       0x400UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_1XN_SUPPORTED      0x800UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_1XN_ENABLED        0x1000UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_IEEE_SUPPORTED     0x2000UL
-+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_IEEE_ENABLED       0x4000UL
-+	u8	duplex_state;
-+	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_HALF 0x0UL
-+	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL 0x1UL
-+	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_LAST PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL
-+	u8	option_flags;
-+	#define PORT_PHY_QCFG_RESP_OPTION_FLAGS_MEDIA_AUTO_DETECT     0x1UL
-+	#define PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN     0x2UL
-+	#define PORT_PHY_QCFG_RESP_OPTION_FLAGS_SPEEDS2_SUPPORTED     0x4UL
-+	#define PORT_PHY_QCFG_RESP_OPTION_FLAGS_LINK_TRAINING         0x8UL
-+	#define PORT_PHY_QCFG_RESP_OPTION_FLAGS_PRECODING             0x10UL
-+	char	phy_vendor_name[16];
-+	char	phy_vendor_partnumber[16];
-+	__le16	support_pam4_speeds;
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_PAM4_SPEEDS_50G      0x1UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_PAM4_SPEEDS_100G     0x2UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_PAM4_SPEEDS_200G     0x4UL
-+	__le16	force_pam4_link_speed;
-+	#define PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_50GB  0x1f4UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_100GB 0x3e8UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_200GB 0x7d0UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_LAST PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_200GB
-+	__le16	auto_pam4_link_speed_mask;
-+	#define PORT_PHY_QCFG_RESP_AUTO_PAM4_LINK_SPEED_MASK_50G      0x1UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_PAM4_LINK_SPEED_MASK_100G     0x2UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_PAM4_LINK_SPEED_MASK_200G     0x4UL
-+	u8	link_partner_pam4_adv_speeds;
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_PAM4_ADV_SPEEDS_50GB      0x1UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_PAM4_ADV_SPEEDS_100GB     0x2UL
-+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_PAM4_ADV_SPEEDS_200GB     0x4UL
-+	u8	link_down_reason;
-+	#define PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_RF                      0x1UL
-+	#define PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_OTP_SPEED_VIOLATION     0x2UL
-+	#define PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_CABLE_REMOVED           0x4UL
-+	#define PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_MODULE_FAULT            0x8UL
-+	#define PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_BMC_REQUEST             0x10UL
-+	#define PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_TX_LASER_DISABLED       0x20UL
-+	__le16	support_speeds2;
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_1GB                0x1UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_10GB               0x2UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_25GB               0x4UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_40GB               0x8UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_50GB               0x10UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_100GB              0x20UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_50GB_PAM4_56       0x40UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_100GB_PAM4_56      0x80UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_200GB_PAM4_56      0x100UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_400GB_PAM4_56      0x200UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_100GB_PAM4_112     0x400UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_200GB_PAM4_112     0x800UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_400GB_PAM4_112     0x1000UL
-+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_800GB_PAM4_112     0x2000UL
-+	__le16	force_link_speeds2;
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_1GB            0xaUL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_10GB           0x64UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_25GB           0xfaUL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_40GB           0x190UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_50GB           0x1f4UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_100GB          0x3e8UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_50GB_PAM4_56   0x1f5UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_100GB_PAM4_56  0x3e9UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_200GB_PAM4_56  0x7d1UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_400GB_PAM4_56  0xfa1UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_100GB_PAM4_112 0x3eaUL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_200GB_PAM4_112 0x7d2UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_400GB_PAM4_112 0xfa2UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_800GB_PAM4_112 0x1f42UL
-+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_LAST          PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_800GB_PAM4_112
-+	__le16	auto_link_speeds2;
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_1GB                0x1UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_10GB               0x2UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_25GB               0x4UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_40GB               0x8UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_50GB               0x10UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_100GB              0x20UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_50GB_PAM4_56       0x40UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_100GB_PAM4_56      0x80UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_200GB_PAM4_56      0x100UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_400GB_PAM4_56      0x200UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_100GB_PAM4_112     0x400UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_200GB_PAM4_112     0x800UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_400GB_PAM4_112     0x1000UL
-+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_800GB_PAM4_112     0x2000UL
-+	u8	active_lanes;
-+	u8	valid;
-+};
-+
-+/* hwrm_port_mac_cfg_input (size:448b/56B) */
-+struct hwrm_port_mac_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define PORT_MAC_CFG_REQ_FLAGS_MATCH_LINK                    0x1UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_VLAN_PRI2COS_ENABLE           0x2UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_TUNNEL_PRI2COS_ENABLE         0x4UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_IP_DSCP2COS_ENABLE            0x8UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_PTP_RX_TS_CAPTURE_ENABLE      0x10UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_PTP_RX_TS_CAPTURE_DISABLE     0x20UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_PTP_TX_TS_CAPTURE_ENABLE      0x40UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_PTP_TX_TS_CAPTURE_DISABLE     0x80UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_OOB_WOL_ENABLE                0x100UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_OOB_WOL_DISABLE               0x200UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_VLAN_PRI2COS_DISABLE          0x400UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_TUNNEL_PRI2COS_DISABLE        0x800UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_IP_DSCP2COS_DISABLE           0x1000UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_PTP_ONE_STEP_TX_TS            0x2000UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_ALL_RX_TS_CAPTURE_ENABLE      0x4000UL
-+	#define PORT_MAC_CFG_REQ_FLAGS_ALL_RX_TS_CAPTURE_DISABLE     0x8000UL
-+	__le32	enables;
-+	#define PORT_MAC_CFG_REQ_ENABLES_IPG                            0x1UL
-+	#define PORT_MAC_CFG_REQ_ENABLES_LPBK                           0x2UL
-+	#define PORT_MAC_CFG_REQ_ENABLES_VLAN_PRI2COS_MAP_PRI           0x4UL
-+	#define PORT_MAC_CFG_REQ_ENABLES_TUNNEL_PRI2COS_MAP_PRI         0x10UL
-+	#define PORT_MAC_CFG_REQ_ENABLES_DSCP2COS_MAP_PRI               0x20UL
-+	#define PORT_MAC_CFG_REQ_ENABLES_RX_TS_CAPTURE_PTP_MSG_TYPE     0x40UL
-+	#define PORT_MAC_CFG_REQ_ENABLES_TX_TS_CAPTURE_PTP_MSG_TYPE     0x80UL
-+	#define PORT_MAC_CFG_REQ_ENABLES_COS_FIELD_CFG                  0x100UL
-+	#define PORT_MAC_CFG_REQ_ENABLES_PTP_FREQ_ADJ_PPB               0x200UL
-+	#define PORT_MAC_CFG_REQ_ENABLES_PTP_ADJ_PHASE                  0x400UL
-+	#define PORT_MAC_CFG_REQ_ENABLES_PTP_LOAD_CONTROL               0x800UL
-+	__le16	port_id;
-+	u8	ipg;
-+	u8	lpbk;
-+	#define PORT_MAC_CFG_REQ_LPBK_NONE   0x0UL
-+	#define PORT_MAC_CFG_REQ_LPBK_LOCAL  0x1UL
-+	#define PORT_MAC_CFG_REQ_LPBK_REMOTE 0x2UL
-+	#define PORT_MAC_CFG_REQ_LPBK_LAST  PORT_MAC_CFG_REQ_LPBK_REMOTE
-+	u8	vlan_pri2cos_map_pri;
-+	u8	reserved1;
-+	u8	tunnel_pri2cos_map_pri;
-+	u8	dscp2pri_map_pri;
-+	__le16	rx_ts_capture_ptp_msg_type;
-+	__le16	tx_ts_capture_ptp_msg_type;
-+	u8	cos_field_cfg;
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_RSVD1                     0x1UL
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_MASK         0x6UL
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_SFT          1
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_INNERMOST      (0x0UL << 1)
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_OUTER          (0x1UL << 1)
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_OUTERMOST      (0x2UL << 1)
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_UNSPECIFIED    (0x3UL << 1)
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_LAST          PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_UNSPECIFIED
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_MASK       0x18UL
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_SFT        3
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_INNERMOST    (0x0UL << 3)
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_OUTER        (0x1UL << 3)
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_OUTERMOST    (0x2UL << 3)
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_UNSPECIFIED  (0x3UL << 3)
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_LAST        PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_UNSPECIFIED
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_DEFAULT_COS_MASK          0xe0UL
-+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_DEFAULT_COS_SFT           5
-+	u8	unused_0[3];
-+	__le32	ptp_freq_adj_ppb;
-+	u8	unused_1[3];
-+	u8	ptp_load_control;
-+	#define PORT_MAC_CFG_REQ_PTP_LOAD_CONTROL_NONE      0x0UL
-+	#define PORT_MAC_CFG_REQ_PTP_LOAD_CONTROL_IMMEDIATE 0x1UL
-+	#define PORT_MAC_CFG_REQ_PTP_LOAD_CONTROL_PPS_EVENT 0x2UL
-+	#define PORT_MAC_CFG_REQ_PTP_LOAD_CONTROL_LAST     PORT_MAC_CFG_REQ_PTP_LOAD_CONTROL_PPS_EVENT
-+	__le64	ptp_adj_phase;
-+};
-+
-+/* hwrm_port_mac_cfg_output (size:128b/16B) */
-+struct hwrm_port_mac_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	mru;
-+	__le16	mtu;
-+	u8	ipg;
-+	u8	lpbk;
-+	#define PORT_MAC_CFG_RESP_LPBK_NONE   0x0UL
-+	#define PORT_MAC_CFG_RESP_LPBK_LOCAL  0x1UL
-+	#define PORT_MAC_CFG_RESP_LPBK_REMOTE 0x2UL
-+	#define PORT_MAC_CFG_RESP_LPBK_LAST  PORT_MAC_CFG_RESP_LPBK_REMOTE
-+	u8	unused_0;
-+	u8	valid;
-+};
-+
-+/* hwrm_port_mac_ptp_qcfg_input (size:192b/24B) */
-+struct hwrm_port_mac_ptp_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_port_mac_ptp_qcfg_output (size:704b/88B) */
-+struct hwrm_port_mac_ptp_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	flags;
-+	#define PORT_MAC_PTP_QCFG_RESP_FLAGS_DIRECT_ACCESS                       0x1UL
-+	#define PORT_MAC_PTP_QCFG_RESP_FLAGS_ONE_STEP_TX_TS                      0x4UL
-+	#define PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS                         0x8UL
-+	#define PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK     0x10UL
-+	#define PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED                      0x20UL
-+	#define PORT_MAC_PTP_QCFG_RESP_FLAGS_64B_PHC_TIME                        0x40UL
-+	u8	unused_0[3];
-+	__le32	rx_ts_reg_off_lower;
-+	__le32	rx_ts_reg_off_upper;
-+	__le32	rx_ts_reg_off_seq_id;
-+	__le32	rx_ts_reg_off_src_id_0;
-+	__le32	rx_ts_reg_off_src_id_1;
-+	__le32	rx_ts_reg_off_src_id_2;
-+	__le32	rx_ts_reg_off_domain_id;
-+	__le32	rx_ts_reg_off_fifo;
-+	__le32	rx_ts_reg_off_fifo_adv;
-+	__le32	rx_ts_reg_off_granularity;
-+	__le32	tx_ts_reg_off_lower;
-+	__le32	tx_ts_reg_off_upper;
-+	__le32	tx_ts_reg_off_seq_id;
-+	__le32	tx_ts_reg_off_fifo;
-+	__le32	tx_ts_reg_off_granularity;
-+	__le32	ts_ref_clock_reg_lower;
-+	__le32	ts_ref_clock_reg_upper;
-+	u8	unused_1[7];
-+	u8	valid;
-+};
-+
-+/* tx_port_stats (size:3264b/408B) */
-+struct tx_port_stats {
-+	__le64	tx_64b_frames;
-+	__le64	tx_65b_127b_frames;
-+	__le64	tx_128b_255b_frames;
-+	__le64	tx_256b_511b_frames;
-+	__le64	tx_512b_1023b_frames;
-+	__le64	tx_1024b_1518b_frames;
-+	__le64	tx_good_vlan_frames;
-+	__le64	tx_1519b_2047b_frames;
-+	__le64	tx_2048b_4095b_frames;
-+	__le64	tx_4096b_9216b_frames;
-+	__le64	tx_9217b_16383b_frames;
-+	__le64	tx_good_frames;
-+	__le64	tx_total_frames;
-+	__le64	tx_ucast_frames;
-+	__le64	tx_mcast_frames;
-+	__le64	tx_bcast_frames;
-+	__le64	tx_pause_frames;
-+	__le64	tx_pfc_frames;
-+	__le64	tx_jabber_frames;
-+	__le64	tx_fcs_err_frames;
-+	__le64	tx_control_frames;
-+	__le64	tx_oversz_frames;
-+	__le64	tx_single_dfrl_frames;
-+	__le64	tx_multi_dfrl_frames;
-+	__le64	tx_single_coll_frames;
-+	__le64	tx_multi_coll_frames;
-+	__le64	tx_late_coll_frames;
-+	__le64	tx_excessive_coll_frames;
-+	__le64	tx_frag_frames;
-+	__le64	tx_err;
-+	__le64	tx_tagged_frames;
-+	__le64	tx_dbl_tagged_frames;
-+	__le64	tx_runt_frames;
-+	__le64	tx_fifo_underruns;
-+	__le64	tx_pfc_ena_frames_pri0;
-+	__le64	tx_pfc_ena_frames_pri1;
-+	__le64	tx_pfc_ena_frames_pri2;
-+	__le64	tx_pfc_ena_frames_pri3;
-+	__le64	tx_pfc_ena_frames_pri4;
-+	__le64	tx_pfc_ena_frames_pri5;
-+	__le64	tx_pfc_ena_frames_pri6;
-+	__le64	tx_pfc_ena_frames_pri7;
-+	__le64	tx_eee_lpi_events;
-+	__le64	tx_eee_lpi_duration;
-+	__le64	tx_llfc_logical_msgs;
-+	__le64	tx_hcfc_msgs;
-+	__le64	tx_total_collisions;
-+	__le64	tx_bytes;
-+	__le64	tx_xthol_frames;
-+	__le64	tx_stat_discard;
-+	__le64	tx_stat_error;
-+};
-+
-+/* rx_port_stats (size:4224b/528B) */
-+struct rx_port_stats {
-+	__le64	rx_64b_frames;
-+	__le64	rx_65b_127b_frames;
-+	__le64	rx_128b_255b_frames;
-+	__le64	rx_256b_511b_frames;
-+	__le64	rx_512b_1023b_frames;
-+	__le64	rx_1024b_1518b_frames;
-+	__le64	rx_good_vlan_frames;
-+	__le64	rx_1519b_2047b_frames;
-+	__le64	rx_2048b_4095b_frames;
-+	__le64	rx_4096b_9216b_frames;
-+	__le64	rx_9217b_16383b_frames;
-+	__le64	rx_total_frames;
-+	__le64	rx_ucast_frames;
-+	__le64	rx_mcast_frames;
-+	__le64	rx_bcast_frames;
-+	__le64	rx_fcs_err_frames;
-+	__le64	rx_ctrl_frames;
-+	__le64	rx_pause_frames;
-+	__le64	rx_pfc_frames;
-+	__le64	rx_unsupported_opcode_frames;
-+	__le64	rx_unsupported_da_pausepfc_frames;
-+	__le64	rx_wrong_sa_frames;
-+	__le64	rx_align_err_frames;
-+	__le64	rx_oor_len_frames;
-+	__le64	rx_code_err_frames;
-+	__le64	rx_false_carrier_frames;
-+	__le64	rx_ovrsz_frames;
-+	__le64	rx_jbr_frames;
-+	__le64	rx_mtu_err_frames;
-+	__le64	rx_match_crc_frames;
-+	__le64	rx_promiscuous_frames;
-+	__le64	rx_tagged_frames;
-+	__le64	rx_double_tagged_frames;
-+	__le64	rx_trunc_frames;
-+	__le64	rx_good_frames;
-+	__le64	rx_pfc_xon2xoff_frames_pri0;
-+	__le64	rx_pfc_xon2xoff_frames_pri1;
-+	__le64	rx_pfc_xon2xoff_frames_pri2;
-+	__le64	rx_pfc_xon2xoff_frames_pri3;
-+	__le64	rx_pfc_xon2xoff_frames_pri4;
-+	__le64	rx_pfc_xon2xoff_frames_pri5;
-+	__le64	rx_pfc_xon2xoff_frames_pri6;
-+	__le64	rx_pfc_xon2xoff_frames_pri7;
-+	__le64	rx_pfc_ena_frames_pri0;
-+	__le64	rx_pfc_ena_frames_pri1;
-+	__le64	rx_pfc_ena_frames_pri2;
-+	__le64	rx_pfc_ena_frames_pri3;
-+	__le64	rx_pfc_ena_frames_pri4;
-+	__le64	rx_pfc_ena_frames_pri5;
-+	__le64	rx_pfc_ena_frames_pri6;
-+	__le64	rx_pfc_ena_frames_pri7;
-+	__le64	rx_sch_crc_err_frames;
-+	__le64	rx_undrsz_frames;
-+	__le64	rx_frag_frames;
-+	__le64	rx_eee_lpi_events;
-+	__le64	rx_eee_lpi_duration;
-+	__le64	rx_llfc_physical_msgs;
-+	__le64	rx_llfc_logical_msgs;
-+	__le64	rx_llfc_msgs_with_crc_err;
-+	__le64	rx_hcfc_msgs;
-+	__le64	rx_hcfc_msgs_with_crc_err;
-+	__le64	rx_bytes;
-+	__le64	rx_runt_bytes;
-+	__le64	rx_runt_frames;
-+	__le64	rx_stat_discard;
-+	__le64	rx_stat_err;
-+};
-+
-+/* hwrm_port_qstats_input (size:320b/40B) */
-+struct hwrm_port_qstats_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	u8	flags;
-+	#define PORT_QSTATS_REQ_FLAGS_COUNTER_MASK     0x1UL
-+	u8	unused_0[5];
-+	__le64	tx_stat_host_addr;
-+	__le64	rx_stat_host_addr;
-+};
-+
-+/* hwrm_port_qstats_output (size:128b/16B) */
-+struct hwrm_port_qstats_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	tx_stat_size;
-+	__le16	rx_stat_size;
-+	u8	flags;
-+	#define PORT_QSTATS_RESP_FLAGS_CLEARED     0x1UL
-+	u8	unused_0[2];
-+	u8	valid;
-+};
-+
-+/* tx_port_stats_ext (size:2048b/256B) */
-+struct tx_port_stats_ext {
-+	__le64	tx_bytes_cos0;
-+	__le64	tx_bytes_cos1;
-+	__le64	tx_bytes_cos2;
-+	__le64	tx_bytes_cos3;
-+	__le64	tx_bytes_cos4;
-+	__le64	tx_bytes_cos5;
-+	__le64	tx_bytes_cos6;
-+	__le64	tx_bytes_cos7;
-+	__le64	tx_packets_cos0;
-+	__le64	tx_packets_cos1;
-+	__le64	tx_packets_cos2;
-+	__le64	tx_packets_cos3;
-+	__le64	tx_packets_cos4;
-+	__le64	tx_packets_cos5;
-+	__le64	tx_packets_cos6;
-+	__le64	tx_packets_cos7;
-+	__le64	pfc_pri0_tx_duration_us;
-+	__le64	pfc_pri0_tx_transitions;
-+	__le64	pfc_pri1_tx_duration_us;
-+	__le64	pfc_pri1_tx_transitions;
-+	__le64	pfc_pri2_tx_duration_us;
-+	__le64	pfc_pri2_tx_transitions;
-+	__le64	pfc_pri3_tx_duration_us;
-+	__le64	pfc_pri3_tx_transitions;
-+	__le64	pfc_pri4_tx_duration_us;
-+	__le64	pfc_pri4_tx_transitions;
-+	__le64	pfc_pri5_tx_duration_us;
-+	__le64	pfc_pri5_tx_transitions;
-+	__le64	pfc_pri6_tx_duration_us;
-+	__le64	pfc_pri6_tx_transitions;
-+	__le64	pfc_pri7_tx_duration_us;
-+	__le64	pfc_pri7_tx_transitions;
-+};
-+
-+/* rx_port_stats_ext (size:3904b/488B) */
-+struct rx_port_stats_ext {
-+	__le64	link_down_events;
-+	__le64	continuous_pause_events;
-+	__le64	resume_pause_events;
-+	__le64	continuous_roce_pause_events;
-+	__le64	resume_roce_pause_events;
-+	__le64	rx_bytes_cos0;
-+	__le64	rx_bytes_cos1;
-+	__le64	rx_bytes_cos2;
-+	__le64	rx_bytes_cos3;
-+	__le64	rx_bytes_cos4;
-+	__le64	rx_bytes_cos5;
-+	__le64	rx_bytes_cos6;
-+	__le64	rx_bytes_cos7;
-+	__le64	rx_packets_cos0;
-+	__le64	rx_packets_cos1;
-+	__le64	rx_packets_cos2;
-+	__le64	rx_packets_cos3;
-+	__le64	rx_packets_cos4;
-+	__le64	rx_packets_cos5;
-+	__le64	rx_packets_cos6;
-+	__le64	rx_packets_cos7;
-+	__le64	pfc_pri0_rx_duration_us;
-+	__le64	pfc_pri0_rx_transitions;
-+	__le64	pfc_pri1_rx_duration_us;
-+	__le64	pfc_pri1_rx_transitions;
-+	__le64	pfc_pri2_rx_duration_us;
-+	__le64	pfc_pri2_rx_transitions;
-+	__le64	pfc_pri3_rx_duration_us;
-+	__le64	pfc_pri3_rx_transitions;
-+	__le64	pfc_pri4_rx_duration_us;
-+	__le64	pfc_pri4_rx_transitions;
-+	__le64	pfc_pri5_rx_duration_us;
-+	__le64	pfc_pri5_rx_transitions;
-+	__le64	pfc_pri6_rx_duration_us;
-+	__le64	pfc_pri6_rx_transitions;
-+	__le64	pfc_pri7_rx_duration_us;
-+	__le64	pfc_pri7_rx_transitions;
-+	__le64	rx_bits;
-+	__le64	rx_buffer_passed_threshold;
-+	__le64	rx_pcs_symbol_err;
-+	__le64	rx_corrected_bits;
-+	__le64	rx_discard_bytes_cos0;
-+	__le64	rx_discard_bytes_cos1;
-+	__le64	rx_discard_bytes_cos2;
-+	__le64	rx_discard_bytes_cos3;
-+	__le64	rx_discard_bytes_cos4;
-+	__le64	rx_discard_bytes_cos5;
-+	__le64	rx_discard_bytes_cos6;
-+	__le64	rx_discard_bytes_cos7;
-+	__le64	rx_discard_packets_cos0;
-+	__le64	rx_discard_packets_cos1;
-+	__le64	rx_discard_packets_cos2;
-+	__le64	rx_discard_packets_cos3;
-+	__le64	rx_discard_packets_cos4;
-+	__le64	rx_discard_packets_cos5;
-+	__le64	rx_discard_packets_cos6;
-+	__le64	rx_discard_packets_cos7;
-+	__le64	rx_fec_corrected_blocks;
-+	__le64	rx_fec_uncorrectable_blocks;
-+	__le64	rx_filter_miss;
-+	__le64	rx_fec_symbol_err;
-+};
-+
-+/* hwrm_port_qstats_ext_input (size:320b/40B) */
-+struct hwrm_port_qstats_ext_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	__le16	tx_stat_size;
-+	__le16	rx_stat_size;
-+	u8	flags;
-+	#define PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK     0x1UL
-+	u8	unused_0;
-+	__le64	tx_stat_host_addr;
-+	__le64	rx_stat_host_addr;
-+};
-+
-+/* hwrm_port_qstats_ext_output (size:128b/16B) */
-+struct hwrm_port_qstats_ext_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	tx_stat_size;
-+	__le16	rx_stat_size;
-+	__le16	total_active_cos_queues;
-+	u8	flags;
-+	#define PORT_QSTATS_EXT_RESP_FLAGS_CLEAR_ROCE_COUNTERS_SUPPORTED     0x1UL
-+	#define PORT_QSTATS_EXT_RESP_FLAGS_CLEARED                           0x2UL
-+	u8	valid;
-+};
-+
-+/* hwrm_port_lpbk_qstats_input (size:256b/32B) */
-+struct hwrm_port_lpbk_qstats_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	lpbk_stat_size;
-+	u8	flags;
-+	#define PORT_LPBK_QSTATS_REQ_FLAGS_COUNTER_MASK     0x1UL
-+	u8	unused_0[5];
-+	__le64	lpbk_stat_host_addr;
-+};
-+
-+/* hwrm_port_lpbk_qstats_output (size:128b/16B) */
-+struct hwrm_port_lpbk_qstats_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	lpbk_stat_size;
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* port_lpbk_stats (size:640b/80B) */
-+struct port_lpbk_stats {
-+	__le64	lpbk_ucast_frames;
-+	__le64	lpbk_mcast_frames;
-+	__le64	lpbk_bcast_frames;
-+	__le64	lpbk_ucast_bytes;
-+	__le64	lpbk_mcast_bytes;
-+	__le64	lpbk_bcast_bytes;
-+	__le64	lpbk_tx_discards;
-+	__le64	lpbk_tx_errors;
-+	__le64	lpbk_rx_discards;
-+	__le64	lpbk_rx_errors;
-+};
-+
-+/* hwrm_port_ecn_qstats_input (size:256b/32B) */
-+struct hwrm_port_ecn_qstats_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	__le16	ecn_stat_buf_size;
-+	u8	flags;
-+	#define PORT_ECN_QSTATS_REQ_FLAGS_COUNTER_MASK     0x1UL
-+	u8	unused_0[3];
-+	__le64	ecn_stat_host_addr;
-+};
-+
-+/* hwrm_port_ecn_qstats_output (size:128b/16B) */
-+struct hwrm_port_ecn_qstats_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	ecn_stat_buf_size;
-+	u8	mark_en;
-+	u8	unused_0[4];
-+	u8	valid;
-+};
-+
-+/* port_stats_ecn (size:512b/64B) */
-+struct port_stats_ecn {
-+	__le64	mark_cnt_cos0;
-+	__le64	mark_cnt_cos1;
-+	__le64	mark_cnt_cos2;
-+	__le64	mark_cnt_cos3;
-+	__le64	mark_cnt_cos4;
-+	__le64	mark_cnt_cos5;
-+	__le64	mark_cnt_cos6;
-+	__le64	mark_cnt_cos7;
-+};
-+
-+/* hwrm_port_clr_stats_input (size:192b/24B) */
-+struct hwrm_port_clr_stats_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	u8	flags;
-+	#define PORT_CLR_STATS_REQ_FLAGS_ROCE_COUNTERS     0x1UL
-+	u8	unused_0[5];
-+};
-+
-+/* hwrm_port_clr_stats_output (size:128b/16B) */
-+struct hwrm_port_clr_stats_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_lpbk_clr_stats_input (size:192b/24B) */
-+struct hwrm_port_lpbk_clr_stats_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_port_lpbk_clr_stats_output (size:128b/16B) */
-+struct hwrm_port_lpbk_clr_stats_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_ts_query_input (size:320b/40B) */
-+struct hwrm_port_ts_query_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define PORT_TS_QUERY_REQ_FLAGS_PATH             0x1UL
-+	#define PORT_TS_QUERY_REQ_FLAGS_PATH_TX            0x0UL
-+	#define PORT_TS_QUERY_REQ_FLAGS_PATH_RX            0x1UL
-+	#define PORT_TS_QUERY_REQ_FLAGS_PATH_LAST         PORT_TS_QUERY_REQ_FLAGS_PATH_RX
-+	#define PORT_TS_QUERY_REQ_FLAGS_CURRENT_TIME     0x2UL
-+	__le16	port_id;
-+	u8	unused_0[2];
-+	__le16	enables;
-+	#define PORT_TS_QUERY_REQ_ENABLES_TS_REQ_TIMEOUT     0x1UL
-+	#define PORT_TS_QUERY_REQ_ENABLES_PTP_SEQ_ID         0x2UL
-+	#define PORT_TS_QUERY_REQ_ENABLES_PTP_HDR_OFFSET     0x4UL
-+	__le16	ts_req_timeout;
-+	__le32	ptp_seq_id;
-+	__le16	ptp_hdr_offset;
-+	u8	unused_1[6];
-+};
-+
-+/* hwrm_port_ts_query_output (size:192b/24B) */
-+struct hwrm_port_ts_query_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le64	ptp_msg_ts;
-+	__le16	ptp_msg_seqid;
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_phy_qcaps_input (size:192b/24B) */
-+struct hwrm_port_phy_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_port_phy_qcaps_output (size:320b/40B) */
-+struct hwrm_port_phy_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	flags;
-+	#define PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED                    0x1UL
-+	#define PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED          0x2UL
-+	#define PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED           0x4UL
-+	#define PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED         0x8UL
-+	#define PORT_PHY_QCAPS_RESP_FLAGS_CUMULATIVE_COUNTERS_ON_RESET     0x10UL
-+	#define PORT_PHY_QCAPS_RESP_FLAGS_LOCAL_LPBK_NOT_SUPPORTED         0x20UL
-+	#define PORT_PHY_QCAPS_RESP_FLAGS_FW_MANAGED_LINK_DOWN             0x40UL
-+	#define PORT_PHY_QCAPS_RESP_FLAGS_NO_FCS                           0x80UL
-+	u8	port_cnt;
-+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_UNKNOWN 0x0UL
-+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_1       0x1UL
-+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_2       0x2UL
-+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_3       0x3UL
-+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_4       0x4UL
-+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_12      0xcUL
-+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_LAST   PORT_PHY_QCAPS_RESP_PORT_CNT_12
-+	__le16	supported_speeds_force_mode;
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_100MBHD     0x1UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_100MB       0x2UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_1GBHD       0x4UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_1GB         0x8UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_2GB         0x10UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_2_5GB       0x20UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_10GB        0x40UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_20GB        0x80UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_25GB        0x100UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_40GB        0x200UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_50GB        0x400UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_100GB       0x800UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_10MBHD      0x1000UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_10MB        0x2000UL
-+	__le16	supported_speeds_auto_mode;
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_100MBHD     0x1UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_100MB       0x2UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_1GBHD       0x4UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_1GB         0x8UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_2GB         0x10UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_2_5GB       0x20UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_10GB        0x40UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_20GB        0x80UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_25GB        0x100UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_40GB        0x200UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_50GB        0x400UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_100GB       0x800UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_10MBHD      0x1000UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_10MB        0x2000UL
-+	__le16	supported_speeds_eee_mode;
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_RSVD1     0x1UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_100MB     0x2UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_RSVD2     0x4UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_1GB       0x8UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_RSVD3     0x10UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_RSVD4     0x20UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_10GB      0x40UL
-+	__le32	tx_lpi_timer_low;
-+	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK 0xffffffUL
-+	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_SFT 0
-+	#define PORT_PHY_QCAPS_RESP_RSVD2_MASK           0xff000000UL
-+	#define PORT_PHY_QCAPS_RESP_RSVD2_SFT            24
-+	__le32	valid_tx_lpi_timer_high;
-+	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK 0xffffffUL
-+	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_SFT 0
-+	#define PORT_PHY_QCAPS_RESP_RSVD_MASK             0xff000000UL
-+	#define PORT_PHY_QCAPS_RESP_RSVD_SFT              24
-+	__le16	supported_pam4_speeds_auto_mode;
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_AUTO_MODE_50G      0x1UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_AUTO_MODE_100G     0x2UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_AUTO_MODE_200G     0x4UL
-+	__le16	supported_pam4_speeds_force_mode;
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_FORCE_MODE_50G      0x1UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_FORCE_MODE_100G     0x2UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_FORCE_MODE_200G     0x4UL
-+	__le16	flags2;
-+	#define PORT_PHY_QCAPS_RESP_FLAGS2_PAUSE_UNSUPPORTED           0x1UL
-+	#define PORT_PHY_QCAPS_RESP_FLAGS2_PFC_UNSUPPORTED             0x2UL
-+	#define PORT_PHY_QCAPS_RESP_FLAGS2_BANK_ADDR_SUPPORTED         0x4UL
-+	#define PORT_PHY_QCAPS_RESP_FLAGS2_SPEEDS2_SUPPORTED           0x8UL
-+	#define PORT_PHY_QCAPS_RESP_FLAGS2_REMOTE_LPBK_UNSUPPORTED     0x10UL
-+	u8	internal_port_cnt;
-+	u8	unused_0;
-+	__le16	supported_speeds2_force_mode;
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_1GB                0x1UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_10GB               0x2UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_25GB               0x4UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_40GB               0x8UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_50GB               0x10UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_100GB              0x20UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_50GB_PAM4_56       0x40UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_100GB_PAM4_56      0x80UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_200GB_PAM4_56      0x100UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_400GB_PAM4_56      0x200UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_100GB_PAM4_112     0x400UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_200GB_PAM4_112     0x800UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_400GB_PAM4_112     0x1000UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_800GB_PAM4_112     0x2000UL
-+	__le16	supported_speeds2_auto_mode;
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_1GB                0x1UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_10GB               0x2UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_25GB               0x4UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_40GB               0x8UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_50GB               0x10UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_100GB              0x20UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_50GB_PAM4_56       0x40UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_100GB_PAM4_56      0x80UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_200GB_PAM4_56      0x100UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_400GB_PAM4_56      0x200UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_100GB_PAM4_112     0x400UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_200GB_PAM4_112     0x800UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_400GB_PAM4_112     0x1000UL
-+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_800GB_PAM4_112     0x2000UL
-+	u8	unused_1[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_phy_i2c_write_input (size:832b/104B) */
-+struct hwrm_port_phy_i2c_write_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	__le32	enables;
-+	#define PORT_PHY_I2C_WRITE_REQ_ENABLES_PAGE_OFFSET     0x1UL
-+	#define PORT_PHY_I2C_WRITE_REQ_ENABLES_BANK_NUMBER     0x2UL
-+	__le16	port_id;
-+	u8	i2c_slave_addr;
-+	u8	bank_number;
-+	__le16	page_number;
-+	__le16	page_offset;
-+	u8	data_length;
-+	u8	unused_1[7];
-+	__le32	data[16];
-+};
-+
-+/* hwrm_port_phy_i2c_write_output (size:128b/16B) */
-+struct hwrm_port_phy_i2c_write_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_phy_i2c_read_input (size:320b/40B) */
-+struct hwrm_port_phy_i2c_read_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	__le32	enables;
-+	#define PORT_PHY_I2C_READ_REQ_ENABLES_PAGE_OFFSET     0x1UL
-+	#define PORT_PHY_I2C_READ_REQ_ENABLES_BANK_NUMBER     0x2UL
-+	__le16	port_id;
-+	u8	i2c_slave_addr;
-+	u8	bank_number;
-+	__le16	page_number;
-+	__le16	page_offset;
-+	u8	data_length;
-+	u8	unused_1[7];
-+};
-+
-+/* hwrm_port_phy_i2c_read_output (size:640b/80B) */
-+struct hwrm_port_phy_i2c_read_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	data[16];
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_phy_mdio_write_input (size:320b/40B) */
-+struct hwrm_port_phy_mdio_write_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	unused_0[2];
-+	__le16	port_id;
-+	u8	phy_addr;
-+	u8	dev_addr;
-+	__le16	reg_addr;
-+	__le16	reg_data;
-+	u8	cl45_mdio;
-+	u8	unused_1[7];
-+};
-+
-+/* hwrm_port_phy_mdio_write_output (size:128b/16B) */
-+struct hwrm_port_phy_mdio_write_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_phy_mdio_read_input (size:256b/32B) */
-+struct hwrm_port_phy_mdio_read_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	unused_0[2];
-+	__le16	port_id;
-+	u8	phy_addr;
-+	u8	dev_addr;
-+	__le16	reg_addr;
-+	u8	cl45_mdio;
-+	u8	unused_1;
-+};
-+
-+/* hwrm_port_phy_mdio_read_output (size:128b/16B) */
-+struct hwrm_port_phy_mdio_read_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	reg_data;
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_led_cfg_input (size:512b/64B) */
-+struct hwrm_port_led_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	#define PORT_LED_CFG_REQ_ENABLES_LED0_ID            0x1UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED0_STATE         0x2UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED0_COLOR         0x4UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED0_BLINK_ON      0x8UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED0_BLINK_OFF     0x10UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED0_GROUP_ID      0x20UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED1_ID            0x40UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED1_STATE         0x80UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED1_COLOR         0x100UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED1_BLINK_ON      0x200UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED1_BLINK_OFF     0x400UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED1_GROUP_ID      0x800UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED2_ID            0x1000UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED2_STATE         0x2000UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED2_COLOR         0x4000UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED2_BLINK_ON      0x8000UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED2_BLINK_OFF     0x10000UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED2_GROUP_ID      0x20000UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED3_ID            0x40000UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED3_STATE         0x80000UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED3_COLOR         0x100000UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED3_BLINK_ON      0x200000UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED3_BLINK_OFF     0x400000UL
-+	#define PORT_LED_CFG_REQ_ENABLES_LED3_GROUP_ID      0x800000UL
-+	__le16	port_id;
-+	u8	num_leds;
-+	u8	rsvd;
-+	u8	led0_id;
-+	u8	led0_state;
-+	#define PORT_LED_CFG_REQ_LED0_STATE_DEFAULT  0x0UL
-+	#define PORT_LED_CFG_REQ_LED0_STATE_OFF      0x1UL
-+	#define PORT_LED_CFG_REQ_LED0_STATE_ON       0x2UL
-+	#define PORT_LED_CFG_REQ_LED0_STATE_BLINK    0x3UL
-+	#define PORT_LED_CFG_REQ_LED0_STATE_BLINKALT 0x4UL
-+	#define PORT_LED_CFG_REQ_LED0_STATE_LAST    PORT_LED_CFG_REQ_LED0_STATE_BLINKALT
-+	u8	led0_color;
-+	#define PORT_LED_CFG_REQ_LED0_COLOR_DEFAULT    0x0UL
-+	#define PORT_LED_CFG_REQ_LED0_COLOR_AMBER      0x1UL
-+	#define PORT_LED_CFG_REQ_LED0_COLOR_GREEN      0x2UL
-+	#define PORT_LED_CFG_REQ_LED0_COLOR_GREENAMBER 0x3UL
-+	#define PORT_LED_CFG_REQ_LED0_COLOR_LAST      PORT_LED_CFG_REQ_LED0_COLOR_GREENAMBER
-+	u8	unused_0;
-+	__le16	led0_blink_on;
-+	__le16	led0_blink_off;
-+	u8	led0_group_id;
-+	u8	rsvd0;
-+	u8	led1_id;
-+	u8	led1_state;
-+	#define PORT_LED_CFG_REQ_LED1_STATE_DEFAULT  0x0UL
-+	#define PORT_LED_CFG_REQ_LED1_STATE_OFF      0x1UL
-+	#define PORT_LED_CFG_REQ_LED1_STATE_ON       0x2UL
-+	#define PORT_LED_CFG_REQ_LED1_STATE_BLINK    0x3UL
-+	#define PORT_LED_CFG_REQ_LED1_STATE_BLINKALT 0x4UL
-+	#define PORT_LED_CFG_REQ_LED1_STATE_LAST    PORT_LED_CFG_REQ_LED1_STATE_BLINKALT
-+	u8	led1_color;
-+	#define PORT_LED_CFG_REQ_LED1_COLOR_DEFAULT    0x0UL
-+	#define PORT_LED_CFG_REQ_LED1_COLOR_AMBER      0x1UL
-+	#define PORT_LED_CFG_REQ_LED1_COLOR_GREEN      0x2UL
-+	#define PORT_LED_CFG_REQ_LED1_COLOR_GREENAMBER 0x3UL
-+	#define PORT_LED_CFG_REQ_LED1_COLOR_LAST      PORT_LED_CFG_REQ_LED1_COLOR_GREENAMBER
-+	u8	unused_1;
-+	__le16	led1_blink_on;
-+	__le16	led1_blink_off;
-+	u8	led1_group_id;
-+	u8	rsvd1;
-+	u8	led2_id;
-+	u8	led2_state;
-+	#define PORT_LED_CFG_REQ_LED2_STATE_DEFAULT  0x0UL
-+	#define PORT_LED_CFG_REQ_LED2_STATE_OFF      0x1UL
-+	#define PORT_LED_CFG_REQ_LED2_STATE_ON       0x2UL
-+	#define PORT_LED_CFG_REQ_LED2_STATE_BLINK    0x3UL
-+	#define PORT_LED_CFG_REQ_LED2_STATE_BLINKALT 0x4UL
-+	#define PORT_LED_CFG_REQ_LED2_STATE_LAST    PORT_LED_CFG_REQ_LED2_STATE_BLINKALT
-+	u8	led2_color;
-+	#define PORT_LED_CFG_REQ_LED2_COLOR_DEFAULT    0x0UL
-+	#define PORT_LED_CFG_REQ_LED2_COLOR_AMBER      0x1UL
-+	#define PORT_LED_CFG_REQ_LED2_COLOR_GREEN      0x2UL
-+	#define PORT_LED_CFG_REQ_LED2_COLOR_GREENAMBER 0x3UL
-+	#define PORT_LED_CFG_REQ_LED2_COLOR_LAST      PORT_LED_CFG_REQ_LED2_COLOR_GREENAMBER
-+	u8	unused_2;
-+	__le16	led2_blink_on;
-+	__le16	led2_blink_off;
-+	u8	led2_group_id;
-+	u8	rsvd2;
-+	u8	led3_id;
-+	u8	led3_state;
-+	#define PORT_LED_CFG_REQ_LED3_STATE_DEFAULT  0x0UL
-+	#define PORT_LED_CFG_REQ_LED3_STATE_OFF      0x1UL
-+	#define PORT_LED_CFG_REQ_LED3_STATE_ON       0x2UL
-+	#define PORT_LED_CFG_REQ_LED3_STATE_BLINK    0x3UL
-+	#define PORT_LED_CFG_REQ_LED3_STATE_BLINKALT 0x4UL
-+	#define PORT_LED_CFG_REQ_LED3_STATE_LAST    PORT_LED_CFG_REQ_LED3_STATE_BLINKALT
-+	u8	led3_color;
-+	#define PORT_LED_CFG_REQ_LED3_COLOR_DEFAULT    0x0UL
-+	#define PORT_LED_CFG_REQ_LED3_COLOR_AMBER      0x1UL
-+	#define PORT_LED_CFG_REQ_LED3_COLOR_GREEN      0x2UL
-+	#define PORT_LED_CFG_REQ_LED3_COLOR_GREENAMBER 0x3UL
-+	#define PORT_LED_CFG_REQ_LED3_COLOR_LAST      PORT_LED_CFG_REQ_LED3_COLOR_GREENAMBER
-+	u8	unused_3;
-+	__le16	led3_blink_on;
-+	__le16	led3_blink_off;
-+	u8	led3_group_id;
-+	u8	rsvd3;
-+};
-+
-+/* hwrm_port_led_cfg_output (size:128b/16B) */
-+struct hwrm_port_led_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_led_qcfg_input (size:192b/24B) */
-+struct hwrm_port_led_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_port_led_qcfg_output (size:448b/56B) */
-+struct hwrm_port_led_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	num_leds;
-+	u8	led0_id;
-+	u8	led0_type;
-+	#define PORT_LED_QCFG_RESP_LED0_TYPE_SPEED    0x0UL
-+	#define PORT_LED_QCFG_RESP_LED0_TYPE_ACTIVITY 0x1UL
-+	#define PORT_LED_QCFG_RESP_LED0_TYPE_INVALID  0xffUL
-+	#define PORT_LED_QCFG_RESP_LED0_TYPE_LAST    PORT_LED_QCFG_RESP_LED0_TYPE_INVALID
-+	u8	led0_state;
-+	#define PORT_LED_QCFG_RESP_LED0_STATE_DEFAULT  0x0UL
-+	#define PORT_LED_QCFG_RESP_LED0_STATE_OFF      0x1UL
-+	#define PORT_LED_QCFG_RESP_LED0_STATE_ON       0x2UL
-+	#define PORT_LED_QCFG_RESP_LED0_STATE_BLINK    0x3UL
-+	#define PORT_LED_QCFG_RESP_LED0_STATE_BLINKALT 0x4UL
-+	#define PORT_LED_QCFG_RESP_LED0_STATE_LAST    PORT_LED_QCFG_RESP_LED0_STATE_BLINKALT
-+	u8	led0_color;
-+	#define PORT_LED_QCFG_RESP_LED0_COLOR_DEFAULT    0x0UL
-+	#define PORT_LED_QCFG_RESP_LED0_COLOR_AMBER      0x1UL
-+	#define PORT_LED_QCFG_RESP_LED0_COLOR_GREEN      0x2UL
-+	#define PORT_LED_QCFG_RESP_LED0_COLOR_GREENAMBER 0x3UL
-+	#define PORT_LED_QCFG_RESP_LED0_COLOR_LAST      PORT_LED_QCFG_RESP_LED0_COLOR_GREENAMBER
-+	u8	unused_0;
-+	__le16	led0_blink_on;
-+	__le16	led0_blink_off;
-+	u8	led0_group_id;
-+	u8	led1_id;
-+	u8	led1_type;
-+	#define PORT_LED_QCFG_RESP_LED1_TYPE_SPEED    0x0UL
-+	#define PORT_LED_QCFG_RESP_LED1_TYPE_ACTIVITY 0x1UL
-+	#define PORT_LED_QCFG_RESP_LED1_TYPE_INVALID  0xffUL
-+	#define PORT_LED_QCFG_RESP_LED1_TYPE_LAST    PORT_LED_QCFG_RESP_LED1_TYPE_INVALID
-+	u8	led1_state;
-+	#define PORT_LED_QCFG_RESP_LED1_STATE_DEFAULT  0x0UL
-+	#define PORT_LED_QCFG_RESP_LED1_STATE_OFF      0x1UL
-+	#define PORT_LED_QCFG_RESP_LED1_STATE_ON       0x2UL
-+	#define PORT_LED_QCFG_RESP_LED1_STATE_BLINK    0x3UL
-+	#define PORT_LED_QCFG_RESP_LED1_STATE_BLINKALT 0x4UL
-+	#define PORT_LED_QCFG_RESP_LED1_STATE_LAST    PORT_LED_QCFG_RESP_LED1_STATE_BLINKALT
-+	u8	led1_color;
-+	#define PORT_LED_QCFG_RESP_LED1_COLOR_DEFAULT    0x0UL
-+	#define PORT_LED_QCFG_RESP_LED1_COLOR_AMBER      0x1UL
-+	#define PORT_LED_QCFG_RESP_LED1_COLOR_GREEN      0x2UL
-+	#define PORT_LED_QCFG_RESP_LED1_COLOR_GREENAMBER 0x3UL
-+	#define PORT_LED_QCFG_RESP_LED1_COLOR_LAST      PORT_LED_QCFG_RESP_LED1_COLOR_GREENAMBER
-+	u8	unused_1;
-+	__le16	led1_blink_on;
-+	__le16	led1_blink_off;
-+	u8	led1_group_id;
-+	u8	led2_id;
-+	u8	led2_type;
-+	#define PORT_LED_QCFG_RESP_LED2_TYPE_SPEED    0x0UL
-+	#define PORT_LED_QCFG_RESP_LED2_TYPE_ACTIVITY 0x1UL
-+	#define PORT_LED_QCFG_RESP_LED2_TYPE_INVALID  0xffUL
-+	#define PORT_LED_QCFG_RESP_LED2_TYPE_LAST    PORT_LED_QCFG_RESP_LED2_TYPE_INVALID
-+	u8	led2_state;
-+	#define PORT_LED_QCFG_RESP_LED2_STATE_DEFAULT  0x0UL
-+	#define PORT_LED_QCFG_RESP_LED2_STATE_OFF      0x1UL
-+	#define PORT_LED_QCFG_RESP_LED2_STATE_ON       0x2UL
-+	#define PORT_LED_QCFG_RESP_LED2_STATE_BLINK    0x3UL
-+	#define PORT_LED_QCFG_RESP_LED2_STATE_BLINKALT 0x4UL
-+	#define PORT_LED_QCFG_RESP_LED2_STATE_LAST    PORT_LED_QCFG_RESP_LED2_STATE_BLINKALT
-+	u8	led2_color;
-+	#define PORT_LED_QCFG_RESP_LED2_COLOR_DEFAULT    0x0UL
-+	#define PORT_LED_QCFG_RESP_LED2_COLOR_AMBER      0x1UL
-+	#define PORT_LED_QCFG_RESP_LED2_COLOR_GREEN      0x2UL
-+	#define PORT_LED_QCFG_RESP_LED2_COLOR_GREENAMBER 0x3UL
-+	#define PORT_LED_QCFG_RESP_LED2_COLOR_LAST      PORT_LED_QCFG_RESP_LED2_COLOR_GREENAMBER
-+	u8	unused_2;
-+	__le16	led2_blink_on;
-+	__le16	led2_blink_off;
-+	u8	led2_group_id;
-+	u8	led3_id;
-+	u8	led3_type;
-+	#define PORT_LED_QCFG_RESP_LED3_TYPE_SPEED    0x0UL
-+	#define PORT_LED_QCFG_RESP_LED3_TYPE_ACTIVITY 0x1UL
-+	#define PORT_LED_QCFG_RESP_LED3_TYPE_INVALID  0xffUL
-+	#define PORT_LED_QCFG_RESP_LED3_TYPE_LAST    PORT_LED_QCFG_RESP_LED3_TYPE_INVALID
-+	u8	led3_state;
-+	#define PORT_LED_QCFG_RESP_LED3_STATE_DEFAULT  0x0UL
-+	#define PORT_LED_QCFG_RESP_LED3_STATE_OFF      0x1UL
-+	#define PORT_LED_QCFG_RESP_LED3_STATE_ON       0x2UL
-+	#define PORT_LED_QCFG_RESP_LED3_STATE_BLINK    0x3UL
-+	#define PORT_LED_QCFG_RESP_LED3_STATE_BLINKALT 0x4UL
-+	#define PORT_LED_QCFG_RESP_LED3_STATE_LAST    PORT_LED_QCFG_RESP_LED3_STATE_BLINKALT
-+	u8	led3_color;
-+	#define PORT_LED_QCFG_RESP_LED3_COLOR_DEFAULT    0x0UL
-+	#define PORT_LED_QCFG_RESP_LED3_COLOR_AMBER      0x1UL
-+	#define PORT_LED_QCFG_RESP_LED3_COLOR_GREEN      0x2UL
-+	#define PORT_LED_QCFG_RESP_LED3_COLOR_GREENAMBER 0x3UL
-+	#define PORT_LED_QCFG_RESP_LED3_COLOR_LAST      PORT_LED_QCFG_RESP_LED3_COLOR_GREENAMBER
-+	u8	unused_3;
-+	__le16	led3_blink_on;
-+	__le16	led3_blink_off;
-+	u8	led3_group_id;
-+	u8	unused_4[6];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_led_qcaps_input (size:192b/24B) */
-+struct hwrm_port_led_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_port_led_qcaps_output (size:384b/48B) */
-+struct hwrm_port_led_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	num_leds;
-+	u8	unused[3];
-+	u8	led0_id;
-+	u8	led0_type;
-+	#define PORT_LED_QCAPS_RESP_LED0_TYPE_SPEED    0x0UL
-+	#define PORT_LED_QCAPS_RESP_LED0_TYPE_ACTIVITY 0x1UL
-+	#define PORT_LED_QCAPS_RESP_LED0_TYPE_INVALID  0xffUL
-+	#define PORT_LED_QCAPS_RESP_LED0_TYPE_LAST    PORT_LED_QCAPS_RESP_LED0_TYPE_INVALID
-+	u8	led0_group_id;
-+	u8	unused_0;
-+	__le16	led0_state_caps;
-+	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_ENABLED                 0x1UL
-+	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_OFF_SUPPORTED           0x2UL
-+	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_ON_SUPPORTED            0x4UL
-+	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_BLINK_SUPPORTED         0x8UL
-+	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_BLINK_ALT_SUPPORTED     0x10UL
-+	__le16	led0_color_caps;
-+	#define PORT_LED_QCAPS_RESP_LED0_COLOR_CAPS_RSVD                 0x1UL
-+	#define PORT_LED_QCAPS_RESP_LED0_COLOR_CAPS_AMBER_SUPPORTED      0x2UL
-+	#define PORT_LED_QCAPS_RESP_LED0_COLOR_CAPS_GREEN_SUPPORTED      0x4UL
-+	#define PORT_LED_QCAPS_RESP_LED0_COLOR_CAPS_GRNAMB_SUPPORTED     0x8UL
-+	u8	led1_id;
-+	u8	led1_type;
-+	#define PORT_LED_QCAPS_RESP_LED1_TYPE_SPEED    0x0UL
-+	#define PORT_LED_QCAPS_RESP_LED1_TYPE_ACTIVITY 0x1UL
-+	#define PORT_LED_QCAPS_RESP_LED1_TYPE_INVALID  0xffUL
-+	#define PORT_LED_QCAPS_RESP_LED1_TYPE_LAST    PORT_LED_QCAPS_RESP_LED1_TYPE_INVALID
-+	u8	led1_group_id;
-+	u8	unused_1;
-+	__le16	led1_state_caps;
-+	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_ENABLED                 0x1UL
-+	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_OFF_SUPPORTED           0x2UL
-+	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_ON_SUPPORTED            0x4UL
-+	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_BLINK_SUPPORTED         0x8UL
-+	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_BLINK_ALT_SUPPORTED     0x10UL
-+	__le16	led1_color_caps;
-+	#define PORT_LED_QCAPS_RESP_LED1_COLOR_CAPS_RSVD                 0x1UL
-+	#define PORT_LED_QCAPS_RESP_LED1_COLOR_CAPS_AMBER_SUPPORTED      0x2UL
-+	#define PORT_LED_QCAPS_RESP_LED1_COLOR_CAPS_GREEN_SUPPORTED      0x4UL
-+	#define PORT_LED_QCAPS_RESP_LED1_COLOR_CAPS_GRNAMB_SUPPORTED     0x8UL
-+	u8	led2_id;
-+	u8	led2_type;
-+	#define PORT_LED_QCAPS_RESP_LED2_TYPE_SPEED    0x0UL
-+	#define PORT_LED_QCAPS_RESP_LED2_TYPE_ACTIVITY 0x1UL
-+	#define PORT_LED_QCAPS_RESP_LED2_TYPE_INVALID  0xffUL
-+	#define PORT_LED_QCAPS_RESP_LED2_TYPE_LAST    PORT_LED_QCAPS_RESP_LED2_TYPE_INVALID
-+	u8	led2_group_id;
-+	u8	unused_2;
-+	__le16	led2_state_caps;
-+	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_ENABLED                 0x1UL
-+	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_OFF_SUPPORTED           0x2UL
-+	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_ON_SUPPORTED            0x4UL
-+	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_BLINK_SUPPORTED         0x8UL
-+	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_BLINK_ALT_SUPPORTED     0x10UL
-+	__le16	led2_color_caps;
-+	#define PORT_LED_QCAPS_RESP_LED2_COLOR_CAPS_RSVD                 0x1UL
-+	#define PORT_LED_QCAPS_RESP_LED2_COLOR_CAPS_AMBER_SUPPORTED      0x2UL
-+	#define PORT_LED_QCAPS_RESP_LED2_COLOR_CAPS_GREEN_SUPPORTED      0x4UL
-+	#define PORT_LED_QCAPS_RESP_LED2_COLOR_CAPS_GRNAMB_SUPPORTED     0x8UL
-+	u8	led3_id;
-+	u8	led3_type;
-+	#define PORT_LED_QCAPS_RESP_LED3_TYPE_SPEED    0x0UL
-+	#define PORT_LED_QCAPS_RESP_LED3_TYPE_ACTIVITY 0x1UL
-+	#define PORT_LED_QCAPS_RESP_LED3_TYPE_INVALID  0xffUL
-+	#define PORT_LED_QCAPS_RESP_LED3_TYPE_LAST    PORT_LED_QCAPS_RESP_LED3_TYPE_INVALID
-+	u8	led3_group_id;
-+	u8	unused_3;
-+	__le16	led3_state_caps;
-+	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_ENABLED                 0x1UL
-+	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_OFF_SUPPORTED           0x2UL
-+	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_ON_SUPPORTED            0x4UL
-+	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_BLINK_SUPPORTED         0x8UL
-+	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_BLINK_ALT_SUPPORTED     0x10UL
-+	__le16	led3_color_caps;
-+	#define PORT_LED_QCAPS_RESP_LED3_COLOR_CAPS_RSVD                 0x1UL
-+	#define PORT_LED_QCAPS_RESP_LED3_COLOR_CAPS_AMBER_SUPPORTED      0x2UL
-+	#define PORT_LED_QCAPS_RESP_LED3_COLOR_CAPS_GREEN_SUPPORTED      0x4UL
-+	#define PORT_LED_QCAPS_RESP_LED3_COLOR_CAPS_GRNAMB_SUPPORTED     0x8UL
-+	u8	unused_4[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_port_mac_qcaps_input (size:192b/24B) */
-+struct hwrm_port_mac_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_port_mac_qcaps_output (size:128b/16B) */
-+struct hwrm_port_mac_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	flags;
-+	#define PORT_MAC_QCAPS_RESP_FLAGS_LOCAL_LPBK_NOT_SUPPORTED     0x1UL
-+	#define PORT_MAC_QCAPS_RESP_FLAGS_REMOTE_LPBK_SUPPORTED        0x2UL
-+	u8	unused_0[6];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_qportcfg_input (size:192b/24B) */
-+struct hwrm_queue_qportcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH     0x1UL
-+	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_TX    0x0UL
-+	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_RX    0x1UL
-+	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_LAST QUEUE_QPORTCFG_REQ_FLAGS_PATH_RX
-+	__le16	port_id;
-+	u8	drv_qmap_cap;
-+	#define QUEUE_QPORTCFG_REQ_DRV_QMAP_CAP_DISABLED 0x0UL
-+	#define QUEUE_QPORTCFG_REQ_DRV_QMAP_CAP_ENABLED  0x1UL
-+	#define QUEUE_QPORTCFG_REQ_DRV_QMAP_CAP_LAST    QUEUE_QPORTCFG_REQ_DRV_QMAP_CAP_ENABLED
-+	u8	unused_0;
-+};
-+
-+/* hwrm_queue_qportcfg_output (size:1344b/168B) */
-+struct hwrm_queue_qportcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	max_configurable_queues;
-+	u8	max_configurable_lossless_queues;
-+	u8	queue_cfg_allowed;
-+	u8	queue_cfg_info;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG             0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_USE_PROFILE_TYPE     0x2UL
-+	u8	queue_pfcenable_cfg_allowed;
-+	u8	queue_pri2cos_cfg_allowed;
-+	u8	queue_cos2bw_cfg_allowed;
-+	u8	queue_id0;
-+	u8	queue_id0_service_profile;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LOSSY          0x0UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS       0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS_ROCE  0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LOSSY_ROCE_CNP 0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS_NIC   0x3UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_UNKNOWN        0xffUL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LAST          QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_UNKNOWN
-+	u8	queue_id1;
-+	u8	queue_id1_service_profile;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LOSSY          0x0UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LOSSLESS       0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LOSSLESS_ROCE  0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LOSSY_ROCE_CNP 0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LOSSLESS_NIC   0x3UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_UNKNOWN        0xffUL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LAST          QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_UNKNOWN
-+	u8	queue_id2;
-+	u8	queue_id2_service_profile;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LOSSY          0x0UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LOSSLESS       0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LOSSLESS_ROCE  0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LOSSY_ROCE_CNP 0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LOSSLESS_NIC   0x3UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_UNKNOWN        0xffUL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LAST          QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_UNKNOWN
-+	u8	queue_id3;
-+	u8	queue_id3_service_profile;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LOSSY          0x0UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LOSSLESS       0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LOSSLESS_ROCE  0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LOSSY_ROCE_CNP 0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LOSSLESS_NIC   0x3UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_UNKNOWN        0xffUL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LAST          QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_UNKNOWN
-+	u8	queue_id4;
-+	u8	queue_id4_service_profile;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LOSSY          0x0UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LOSSLESS       0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LOSSLESS_ROCE  0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LOSSY_ROCE_CNP 0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LOSSLESS_NIC   0x3UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_UNKNOWN        0xffUL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LAST          QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_UNKNOWN
-+	u8	queue_id5;
-+	u8	queue_id5_service_profile;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LOSSY          0x0UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LOSSLESS       0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LOSSLESS_ROCE  0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LOSSY_ROCE_CNP 0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LOSSLESS_NIC   0x3UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_UNKNOWN        0xffUL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LAST          QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_UNKNOWN
-+	u8	queue_id6;
-+	u8	queue_id6_service_profile;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LOSSY          0x0UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LOSSLESS       0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LOSSLESS_ROCE  0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LOSSY_ROCE_CNP 0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LOSSLESS_NIC   0x3UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_UNKNOWN        0xffUL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LAST          QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_UNKNOWN
-+	u8	queue_id7;
-+	u8	queue_id7_service_profile;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LOSSY          0x0UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LOSSLESS       0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LOSSLESS_ROCE  0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LOSSY_ROCE_CNP 0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LOSSLESS_NIC   0x3UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_UNKNOWN        0xffUL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LAST          QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_UNKNOWN
-+	u8	queue_id0_service_profile_type;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_TYPE_ROCE     0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_TYPE_NIC      0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_TYPE_CNP      0x4UL
-+	char	qid0_name[16];
-+	char	qid1_name[16];
-+	char	qid2_name[16];
-+	char	qid3_name[16];
-+	char	qid4_name[16];
-+	char	qid5_name[16];
-+	char	qid6_name[16];
-+	char	qid7_name[16];
-+	u8	queue_id1_service_profile_type;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_TYPE_ROCE     0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_TYPE_NIC      0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_TYPE_CNP      0x4UL
-+	u8	queue_id2_service_profile_type;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_TYPE_ROCE     0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_TYPE_NIC      0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_TYPE_CNP      0x4UL
-+	u8	queue_id3_service_profile_type;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_TYPE_ROCE     0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_TYPE_NIC      0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_TYPE_CNP      0x4UL
-+	u8	queue_id4_service_profile_type;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_TYPE_ROCE     0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_TYPE_NIC      0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_TYPE_CNP      0x4UL
-+	u8	queue_id5_service_profile_type;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_TYPE_ROCE     0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_TYPE_NIC      0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_TYPE_CNP      0x4UL
-+	u8	queue_id6_service_profile_type;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_TYPE_ROCE     0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_TYPE_NIC      0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_TYPE_CNP      0x4UL
-+	u8	queue_id7_service_profile_type;
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_TYPE_ROCE     0x1UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_TYPE_NIC      0x2UL
-+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_TYPE_CNP      0x4UL
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_qcfg_input (size:192b/24B) */
-+struct hwrm_queue_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define QUEUE_QCFG_REQ_FLAGS_PATH     0x1UL
-+	#define QUEUE_QCFG_REQ_FLAGS_PATH_TX    0x0UL
-+	#define QUEUE_QCFG_REQ_FLAGS_PATH_RX    0x1UL
-+	#define QUEUE_QCFG_REQ_FLAGS_PATH_LAST QUEUE_QCFG_REQ_FLAGS_PATH_RX
-+	__le32	queue_id;
-+};
-+
-+/* hwrm_queue_qcfg_output (size:128b/16B) */
-+struct hwrm_queue_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	queue_len;
-+	u8	service_profile;
-+	#define QUEUE_QCFG_RESP_SERVICE_PROFILE_LOSSY    0x0UL
-+	#define QUEUE_QCFG_RESP_SERVICE_PROFILE_LOSSLESS 0x1UL
-+	#define QUEUE_QCFG_RESP_SERVICE_PROFILE_UNKNOWN  0xffUL
-+	#define QUEUE_QCFG_RESP_SERVICE_PROFILE_LAST    QUEUE_QCFG_RESP_SERVICE_PROFILE_UNKNOWN
-+	u8	queue_cfg_info;
-+	#define QUEUE_QCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG     0x1UL
-+	u8	unused_0;
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_cfg_input (size:320b/40B) */
-+struct hwrm_queue_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define QUEUE_CFG_REQ_FLAGS_PATH_MASK 0x3UL
-+	#define QUEUE_CFG_REQ_FLAGS_PATH_SFT  0
-+	#define QUEUE_CFG_REQ_FLAGS_PATH_TX     0x0UL
-+	#define QUEUE_CFG_REQ_FLAGS_PATH_RX     0x1UL
-+	#define QUEUE_CFG_REQ_FLAGS_PATH_BIDIR  0x2UL
-+	#define QUEUE_CFG_REQ_FLAGS_PATH_LAST  QUEUE_CFG_REQ_FLAGS_PATH_BIDIR
-+	__le32	enables;
-+	#define QUEUE_CFG_REQ_ENABLES_DFLT_LEN            0x1UL
-+	#define QUEUE_CFG_REQ_ENABLES_SERVICE_PROFILE     0x2UL
-+	__le32	queue_id;
-+	__le32	dflt_len;
-+	u8	service_profile;
-+	#define QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSY    0x0UL
-+	#define QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSLESS 0x1UL
-+	#define QUEUE_CFG_REQ_SERVICE_PROFILE_UNKNOWN  0xffUL
-+	#define QUEUE_CFG_REQ_SERVICE_PROFILE_LAST    QUEUE_CFG_REQ_SERVICE_PROFILE_UNKNOWN
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_queue_cfg_output (size:128b/16B) */
-+struct hwrm_queue_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_pfcenable_qcfg_input (size:192b/24B) */
-+struct hwrm_queue_pfcenable_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_queue_pfcenable_qcfg_output (size:128b/16B) */
-+struct hwrm_queue_pfcenable_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	flags;
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI0_PFC_ENABLED              0x1UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI1_PFC_ENABLED              0x2UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI2_PFC_ENABLED              0x4UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI3_PFC_ENABLED              0x8UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI4_PFC_ENABLED              0x10UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI5_PFC_ENABLED              0x20UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI6_PFC_ENABLED              0x40UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI7_PFC_ENABLED              0x80UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI0_PFC_WATCHDOG_ENABLED     0x100UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI1_PFC_WATCHDOG_ENABLED     0x200UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI2_PFC_WATCHDOG_ENABLED     0x400UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI3_PFC_WATCHDOG_ENABLED     0x800UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI4_PFC_WATCHDOG_ENABLED     0x1000UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI5_PFC_WATCHDOG_ENABLED     0x2000UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI6_PFC_WATCHDOG_ENABLED     0x4000UL
-+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI7_PFC_WATCHDOG_ENABLED     0x8000UL
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_pfcenable_cfg_input (size:192b/24B) */
-+struct hwrm_queue_pfcenable_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI0_PFC_ENABLED              0x1UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI1_PFC_ENABLED              0x2UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI2_PFC_ENABLED              0x4UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI3_PFC_ENABLED              0x8UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI4_PFC_ENABLED              0x10UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI5_PFC_ENABLED              0x20UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI6_PFC_ENABLED              0x40UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI7_PFC_ENABLED              0x80UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI0_PFC_WATCHDOG_ENABLED     0x100UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI1_PFC_WATCHDOG_ENABLED     0x200UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI2_PFC_WATCHDOG_ENABLED     0x400UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI3_PFC_WATCHDOG_ENABLED     0x800UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI4_PFC_WATCHDOG_ENABLED     0x1000UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI5_PFC_WATCHDOG_ENABLED     0x2000UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI6_PFC_WATCHDOG_ENABLED     0x4000UL
-+	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI7_PFC_WATCHDOG_ENABLED     0x8000UL
-+	__le16	port_id;
-+	u8	unused_0[2];
-+};
-+
-+/* hwrm_queue_pfcenable_cfg_output (size:128b/16B) */
-+struct hwrm_queue_pfcenable_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_pri2cos_qcfg_input (size:192b/24B) */
-+struct hwrm_queue_pri2cos_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH      0x1UL
-+	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH_TX     0x0UL
-+	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH_RX     0x1UL
-+	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH_LAST  QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH_RX
-+	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN     0x2UL
-+	u8	port_id;
-+	u8	unused_0[3];
-+};
-+
-+/* hwrm_queue_pri2cos_qcfg_output (size:192b/24B) */
-+struct hwrm_queue_pri2cos_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	pri0_cos_queue_id;
-+	u8	pri1_cos_queue_id;
-+	u8	pri2_cos_queue_id;
-+	u8	pri3_cos_queue_id;
-+	u8	pri4_cos_queue_id;
-+	u8	pri5_cos_queue_id;
-+	u8	pri6_cos_queue_id;
-+	u8	pri7_cos_queue_id;
-+	u8	queue_cfg_info;
-+	#define QUEUE_PRI2COS_QCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG     0x1UL
-+	u8	unused_0[6];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_pri2cos_cfg_input (size:320b/40B) */
-+struct hwrm_queue_pri2cos_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_MASK 0x3UL
-+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_SFT  0
-+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_TX     0x0UL
-+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_RX     0x1UL
-+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_BIDIR  0x2UL
-+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_LAST  QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_BIDIR
-+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_IVLAN     0x4UL
-+	__le32	enables;
-+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI0_COS_QUEUE_ID     0x1UL
-+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI1_COS_QUEUE_ID     0x2UL
-+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI2_COS_QUEUE_ID     0x4UL
-+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI3_COS_QUEUE_ID     0x8UL
-+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI4_COS_QUEUE_ID     0x10UL
-+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI5_COS_QUEUE_ID     0x20UL
-+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI6_COS_QUEUE_ID     0x40UL
-+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI7_COS_QUEUE_ID     0x80UL
-+	u8	port_id;
-+	u8	pri0_cos_queue_id;
-+	u8	pri1_cos_queue_id;
-+	u8	pri2_cos_queue_id;
-+	u8	pri3_cos_queue_id;
-+	u8	pri4_cos_queue_id;
-+	u8	pri5_cos_queue_id;
-+	u8	pri6_cos_queue_id;
-+	u8	pri7_cos_queue_id;
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_queue_pri2cos_cfg_output (size:128b/16B) */
-+struct hwrm_queue_pri2cos_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_cos2bw_qcfg_input (size:192b/24B) */
-+struct hwrm_queue_cos2bw_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_queue_cos2bw_qcfg_output (size:896b/112B) */
-+struct hwrm_queue_cos2bw_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	queue_id0;
-+	u8	unused_0;
-+	__le16	unused_1;
-+	__le32	queue_id0_min_bw;
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_SFT              0
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE                     0x10000000UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE_BYTES
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_SFT         29
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID
-+	__le32	queue_id0_max_bw;
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_SFT              0
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE                     0x10000000UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE_BYTES
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_SFT         29
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID
-+	u8	queue_id0_tsa_assign;
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_SP             0x0UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_ETS            0x1UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_RESERVED_LAST  0xffUL
-+	u8	queue_id0_pri_lvl;
-+	u8	queue_id0_bw_weight;
-+	struct {
-+		u8	queue_id;
-+		__le32	queue_id_min_bw;
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_BW_VALUE_SFT              0
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_SCALE                     0x10000000UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_SCALE_BYTES
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_SFT         29
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_INVALID
-+		__le32	queue_id_max_bw;
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_BW_VALUE_SFT              0
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_SCALE                     0x10000000UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_SCALE_BYTES
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_SFT         29
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_INVALID
-+		u8	queue_id_tsa_assign;
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_TSA_ASSIGN_SP             0x0UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_TSA_ASSIGN_ETS            0x1UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID_TSA_ASSIGN_RESERVED_LAST  0xffUL
-+		u8	queue_id_pri_lvl;
-+		u8	queue_id_bw_weight;
-+	} __packed cfg[7];
-+	u8	unused_2[4];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_cos2bw_cfg_input (size:1024b/128B) */
-+struct hwrm_queue_cos2bw_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	__le32	enables;
-+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID0_VALID     0x1UL
-+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID1_VALID     0x2UL
-+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID2_VALID     0x4UL
-+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID3_VALID     0x8UL
-+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID4_VALID     0x10UL
-+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID5_VALID     0x20UL
-+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID6_VALID     0x40UL
-+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID7_VALID     0x80UL
-+	__le16	port_id;
-+	u8	queue_id0;
-+	u8	unused_0;
-+	__le32	queue_id0_min_bw;
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_SFT              0
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE                     0x10000000UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE_BYTES
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_SFT         29
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID
-+	__le32	queue_id0_max_bw;
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_SFT              0
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE                     0x10000000UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE_BYTES
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_SFT         29
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID
-+	u8	queue_id0_tsa_assign;
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_SP             0x0UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_ETS            0x1UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_RESERVED_LAST  0xffUL
-+	u8	queue_id0_pri_lvl;
-+	u8	queue_id0_bw_weight;
-+	struct {
-+		u8	queue_id;
-+		__le32	queue_id_min_bw;
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_BW_VALUE_SFT              0
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_SCALE                     0x10000000UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_SCALE_BYTES
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_SFT         29
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MIN_BW_BW_VALUE_UNIT_INVALID
-+		__le32	queue_id_max_bw;
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_BW_VALUE_SFT              0
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_SCALE                     0x10000000UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_SCALE_BYTES
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_SFT         29
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID_MAX_BW_BW_VALUE_UNIT_INVALID
-+		u8	queue_id_tsa_assign;
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_TSA_ASSIGN_SP             0x0UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_TSA_ASSIGN_ETS            0x1UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID_TSA_ASSIGN_RESERVED_LAST  0xffUL
-+		u8	queue_id_pri_lvl;
-+		u8	queue_id_bw_weight;
-+	} __packed cfg[7];
-+	u8	unused_1[5];
-+};
-+
-+/* hwrm_queue_cos2bw_cfg_output (size:128b/16B) */
-+struct hwrm_queue_cos2bw_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_dscp_qcaps_input (size:192b/24B) */
-+struct hwrm_queue_dscp_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	port_id;
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_queue_dscp_qcaps_output (size:128b/16B) */
-+struct hwrm_queue_dscp_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	num_dscp_bits;
-+	u8	unused_0;
-+	__le16	max_entries;
-+	u8	unused_1[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_dscp2pri_qcfg_input (size:256b/32B) */
-+struct hwrm_queue_dscp2pri_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	dest_data_addr;
-+	u8	port_id;
-+	u8	unused_0;
-+	__le16	dest_data_buffer_size;
-+	u8	unused_1[4];
-+};
-+
-+/* hwrm_queue_dscp2pri_qcfg_output (size:128b/16B) */
-+struct hwrm_queue_dscp2pri_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	entry_cnt;
-+	u8	default_pri;
-+	u8	unused_0[4];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_dscp2pri_cfg_input (size:320b/40B) */
-+struct hwrm_queue_dscp2pri_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	src_data_addr;
-+	__le32	flags;
-+	#define QUEUE_DSCP2PRI_CFG_REQ_FLAGS_USE_HW_DEFAULT_PRI     0x1UL
-+	__le32	enables;
-+	#define QUEUE_DSCP2PRI_CFG_REQ_ENABLES_DEFAULT_PRI     0x1UL
-+	u8	port_id;
-+	u8	default_pri;
-+	__le16	entry_cnt;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_queue_dscp2pri_cfg_output (size:128b/16B) */
-+struct hwrm_queue_dscp2pri_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_pfcwd_timeout_qcaps_input (size:128b/16B) */
-+struct hwrm_queue_pfcwd_timeout_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+};
-+
-+/* hwrm_queue_pfcwd_timeout_qcaps_output (size:128b/16B) */
-+struct hwrm_queue_pfcwd_timeout_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	max_pfcwd_timeout;
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_pfcwd_timeout_cfg_input (size:192b/24B) */
-+struct hwrm_queue_pfcwd_timeout_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	pfcwd_timeout_value;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_queue_pfcwd_timeout_cfg_output (size:128b/16B) */
-+struct hwrm_queue_pfcwd_timeout_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_queue_pfcwd_timeout_qcfg_input (size:128b/16B) */
-+struct hwrm_queue_pfcwd_timeout_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+};
-+
-+/* hwrm_queue_pfcwd_timeout_qcfg_output (size:128b/16B) */
-+struct hwrm_queue_pfcwd_timeout_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	pfcwd_timeout_value;
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_vnic_alloc_input (size:192b/24B) */
-+struct hwrm_vnic_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define VNIC_ALLOC_REQ_FLAGS_DEFAULT                  0x1UL
-+	#define VNIC_ALLOC_REQ_FLAGS_VIRTIO_NET_FID_VALID     0x2UL
-+	#define VNIC_ALLOC_REQ_FLAGS_VNIC_ID_VALID            0x4UL
-+	__le16	virtio_net_fid;
-+	__le16	vnic_id;
-+};
-+
-+/* hwrm_vnic_alloc_output (size:128b/16B) */
-+struct hwrm_vnic_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	vnic_id;
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_vnic_update_input (size:256b/32B) */
-+struct hwrm_vnic_update_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	vnic_id;
-+	__le32	enables;
-+	#define VNIC_UPDATE_REQ_ENABLES_VNIC_STATE_VALID               0x1UL
-+	#define VNIC_UPDATE_REQ_ENABLES_MRU_VALID                      0x2UL
-+	#define VNIC_UPDATE_REQ_ENABLES_METADATA_FORMAT_TYPE_VALID     0x4UL
-+	u8	vnic_state;
-+	#define VNIC_UPDATE_REQ_VNIC_STATE_NORMAL 0x0UL
-+	#define VNIC_UPDATE_REQ_VNIC_STATE_DROP   0x1UL
-+	#define VNIC_UPDATE_REQ_VNIC_STATE_LAST  VNIC_UPDATE_REQ_VNIC_STATE_DROP
-+	u8	metadata_format_type;
-+	#define VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_0 0x0UL
-+	#define VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_1 0x1UL
-+	#define VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_2 0x2UL
-+	#define VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_3 0x3UL
-+	#define VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_4 0x4UL
-+	#define VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_LAST VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_4
-+	__le16	mru;
-+	u8	unused_1[4];
-+};
-+
-+/* hwrm_vnic_update_output (size:128b/16B) */
-+struct hwrm_vnic_update_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_vnic_free_input (size:192b/24B) */
-+struct hwrm_vnic_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	vnic_id;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_vnic_free_output (size:128b/16B) */
-+struct hwrm_vnic_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_vnic_cfg_input (size:384b/48B) */
-+struct hwrm_vnic_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define VNIC_CFG_REQ_FLAGS_DEFAULT                              0x1UL
-+	#define VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE                      0x2UL
-+	#define VNIC_CFG_REQ_FLAGS_BD_STALL_MODE                        0x4UL
-+	#define VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE                  0x8UL
-+	#define VNIC_CFG_REQ_FLAGS_ROCE_ONLY_VNIC_MODE                  0x10UL
-+	#define VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE                     0x20UL
-+	#define VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE     0x40UL
-+	#define VNIC_CFG_REQ_FLAGS_PORTCOS_MAPPING_MODE                 0x80UL
-+	__le32	enables;
-+	#define VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP            0x1UL
-+	#define VNIC_CFG_REQ_ENABLES_RSS_RULE                 0x2UL
-+	#define VNIC_CFG_REQ_ENABLES_COS_RULE                 0x4UL
-+	#define VNIC_CFG_REQ_ENABLES_LB_RULE                  0x8UL
-+	#define VNIC_CFG_REQ_ENABLES_MRU                      0x10UL
-+	#define VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID       0x20UL
-+	#define VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID     0x40UL
-+	#define VNIC_CFG_REQ_ENABLES_QUEUE_ID                 0x80UL
-+	#define VNIC_CFG_REQ_ENABLES_RX_CSUM_V2_MODE          0x100UL
-+	#define VNIC_CFG_REQ_ENABLES_L2_CQE_MODE              0x200UL
-+	#define VNIC_CFG_REQ_ENABLES_RAW_QP_ID                0x400UL
-+	__le16	vnic_id;
-+	__le16	dflt_ring_grp;
-+	__le16	rss_rule;
-+	__le16	cos_rule;
-+	__le16	lb_rule;
-+	__le16	mru;
-+	__le16	default_rx_ring_id;
-+	__le16	default_cmpl_ring_id;
-+	__le16	queue_id;
-+	u8	rx_csum_v2_mode;
-+	#define VNIC_CFG_REQ_RX_CSUM_V2_MODE_DEFAULT 0x0UL
-+	#define VNIC_CFG_REQ_RX_CSUM_V2_MODE_ALL_OK  0x1UL
-+	#define VNIC_CFG_REQ_RX_CSUM_V2_MODE_MAX     0x2UL
-+	#define VNIC_CFG_REQ_RX_CSUM_V2_MODE_LAST   VNIC_CFG_REQ_RX_CSUM_V2_MODE_MAX
-+	u8	l2_cqe_mode;
-+	#define VNIC_CFG_REQ_L2_CQE_MODE_DEFAULT    0x0UL
-+	#define VNIC_CFG_REQ_L2_CQE_MODE_COMPRESSED 0x1UL
-+	#define VNIC_CFG_REQ_L2_CQE_MODE_MIXED      0x2UL
-+	#define VNIC_CFG_REQ_L2_CQE_MODE_LAST      VNIC_CFG_REQ_L2_CQE_MODE_MIXED
-+	__le32	raw_qp_id;
-+};
-+
-+/* hwrm_vnic_cfg_output (size:128b/16B) */
-+struct hwrm_vnic_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_vnic_qcaps_input (size:192b/24B) */
-+struct hwrm_vnic_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_vnic_qcaps_output (size:192b/24B) */
-+struct hwrm_vnic_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	mru;
-+	u8	unused_0[2];
-+	__le32	flags;
-+	#define VNIC_QCAPS_RESP_FLAGS_UNUSED                                  0x1UL
-+	#define VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP                          0x2UL
-+	#define VNIC_QCAPS_RESP_FLAGS_BD_STALL_CAP                            0x4UL
-+	#define VNIC_QCAPS_RESP_FLAGS_ROCE_DUAL_VNIC_CAP                      0x8UL
-+	#define VNIC_QCAPS_RESP_FLAGS_ROCE_ONLY_VNIC_CAP                      0x10UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP                         0x20UL
-+	#define VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP         0x40UL
-+	#define VNIC_QCAPS_RESP_FLAGS_OUTERMOST_RSS_CAP                       0x80UL
-+	#define VNIC_QCAPS_RESP_FLAGS_COS_ASSIGNMENT_CAP                      0x100UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RX_CMPL_V2_CAP                          0x200UL
-+	#define VNIC_QCAPS_RESP_FLAGS_VNIC_STATE_CAP                          0x400UL
-+	#define VNIC_QCAPS_RESP_FLAGS_VIRTIO_NET_VNIC_ALLOC_CAP               0x800UL
-+	#define VNIC_QCAPS_RESP_FLAGS_METADATA_FORMAT_CAP                     0x1000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RSS_STRICT_HASH_TYPE_CAP                0x2000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP                 0x4000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RING_SELECT_MODE_TOEPLITZ_CAP           0x8000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RING_SELECT_MODE_XOR_CAP                0x10000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RING_SELECT_MODE_TOEPLITZ_CHKSM_CAP     0x20000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RSS_IPV6_FLOW_LABEL_CAP                 0x40000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RX_CMPL_V3_CAP                          0x80000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_L2_CQE_MODE_CAP                         0x100000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV4_CAP               0x200000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV4_CAP              0x400000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV6_CAP               0x800000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV6_CAP              0x1000000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_OUTERMOST_RSS_TRUSTED_VF_CAP            0x2000000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_PORTCOS_MAPPING_MODE                    0x4000000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RSS_PROF_TCAM_MODE_ENABLED              0x8000000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_VNIC_RSS_HASH_MODE_CAP                  0x10000000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_HW_TUNNEL_TPA_CAP                       0x20000000UL
-+	#define VNIC_QCAPS_RESP_FLAGS_RE_FLUSH_CAP                            0x40000000UL
-+	__le16	max_aggs_supported;
-+	u8	unused_1[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_vnic_tpa_cfg_input (size:384b/48B) */
-+struct hwrm_vnic_tpa_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define VNIC_TPA_CFG_REQ_FLAGS_TPA                       0x1UL
-+	#define VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA                 0x2UL
-+	#define VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE            0x4UL
-+	#define VNIC_TPA_CFG_REQ_FLAGS_GRO                       0x8UL
-+	#define VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN              0x10UL
-+	#define VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ     0x20UL
-+	#define VNIC_TPA_CFG_REQ_FLAGS_GRO_IPID_CHECK            0x40UL
-+	#define VNIC_TPA_CFG_REQ_FLAGS_GRO_TTL_CHECK             0x80UL
-+	#define VNIC_TPA_CFG_REQ_FLAGS_AGG_PACK_AS_GRO           0x100UL
-+	__le32	enables;
-+	#define VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS      0x1UL
-+	#define VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS          0x2UL
-+	#define VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_TIMER     0x4UL
-+	#define VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN       0x8UL
-+	#define VNIC_TPA_CFG_REQ_ENABLES_TNL_TPA_EN        0x10UL
-+	__le16	vnic_id;
-+	__le16	max_agg_segs;
-+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_1   0x0UL
-+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_2   0x1UL
-+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_4   0x2UL
-+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_8   0x3UL
-+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_MAX 0x1fUL
-+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_LAST VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_MAX
-+	__le16	max_aggs;
-+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_1   0x0UL
-+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_2   0x1UL
-+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_4   0x2UL
-+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_8   0x3UL
-+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_16  0x4UL
-+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_MAX 0x7UL
-+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_LAST VNIC_TPA_CFG_REQ_MAX_AGGS_MAX
-+	u8	unused_0[2];
-+	__le32	max_agg_timer;
-+	__le32	min_agg_len;
-+	__le32	tnl_tpa_en_bitmap;
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN           0x1UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GENEVE          0x2UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_NVGRE           0x4UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GRE             0x8UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV4            0x10UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV6            0x20UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN_GPE       0x40UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN_CUST1     0x80UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GRE_CUST1       0x100UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR1           0x200UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR2           0x400UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR3           0x800UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR4           0x1000UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR5           0x2000UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR6           0x4000UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR7           0x8000UL
-+	#define VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR8           0x10000UL
-+	u8	unused_1[4];
-+};
-+
-+/* hwrm_vnic_tpa_cfg_output (size:128b/16B) */
-+struct hwrm_vnic_tpa_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_vnic_tpa_qcfg_input (size:192b/24B) */
-+struct hwrm_vnic_tpa_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	vnic_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_vnic_tpa_qcfg_output (size:256b/32B) */
-+struct hwrm_vnic_tpa_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	flags;
-+	#define VNIC_TPA_QCFG_RESP_FLAGS_TPA                       0x1UL
-+	#define VNIC_TPA_QCFG_RESP_FLAGS_ENCAP_TPA                 0x2UL
-+	#define VNIC_TPA_QCFG_RESP_FLAGS_RSC_WND_UPDATE            0x4UL
-+	#define VNIC_TPA_QCFG_RESP_FLAGS_GRO                       0x8UL
-+	#define VNIC_TPA_QCFG_RESP_FLAGS_AGG_WITH_ECN              0x10UL
-+	#define VNIC_TPA_QCFG_RESP_FLAGS_AGG_WITH_SAME_GRE_SEQ     0x20UL
-+	#define VNIC_TPA_QCFG_RESP_FLAGS_GRO_IPID_CHECK            0x40UL
-+	#define VNIC_TPA_QCFG_RESP_FLAGS_GRO_TTL_CHECK             0x80UL
-+	__le16	max_agg_segs;
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_1   0x0UL
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_2   0x1UL
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_4   0x2UL
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_8   0x3UL
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_MAX 0x1fUL
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_LAST VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_MAX
-+	__le16	max_aggs;
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_1   0x0UL
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_2   0x1UL
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_4   0x2UL
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_8   0x3UL
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_16  0x4UL
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_MAX 0x7UL
-+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_LAST VNIC_TPA_QCFG_RESP_MAX_AGGS_MAX
-+	__le32	max_agg_timer;
-+	__le32	min_agg_len;
-+	__le32	tnl_tpa_en_bitmap;
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_VXLAN           0x1UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_GENEVE          0x2UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_NVGRE           0x4UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_GRE             0x8UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_IPV4            0x10UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_IPV6            0x20UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_VXLAN_GPE       0x40UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_VXLAN_CUST1     0x80UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_GRE_CUST1       0x100UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR1           0x200UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR2           0x400UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR3           0x800UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR4           0x1000UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR5           0x2000UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR6           0x4000UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR7           0x8000UL
-+	#define VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR8           0x10000UL
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_vnic_rss_cfg_input (size:384b/48B) */
-+struct hwrm_vnic_rss_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	hash_type;
-+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4                0x1UL
-+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4            0x2UL
-+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4            0x4UL
-+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6                0x8UL
-+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6            0x10UL
-+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6            0x20UL
-+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6_FLOW_LABEL     0x40UL
-+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_AH_SPI_IPV4         0x80UL
-+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_ESP_SPI_IPV4        0x100UL
-+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_AH_SPI_IPV6         0x200UL
-+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_ESP_SPI_IPV6        0x400UL
-+	__le16	vnic_id;
-+	u8	ring_table_pair_index;
-+	u8	hash_mode_flags;
-+	#define VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT         0x1UL
-+	#define VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_INNERMOST_4     0x2UL
-+	#define VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_INNERMOST_2     0x4UL
-+	#define VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_OUTERMOST_4     0x8UL
-+	#define VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_OUTERMOST_2     0x10UL
-+	__le64	ring_grp_tbl_addr;
-+	__le64	hash_key_tbl_addr;
-+	__le16	rss_ctx_idx;
-+	u8	flags;
-+	#define VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE               0x1UL
-+	#define VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE               0x2UL
-+	#define VNIC_RSS_CFG_REQ_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT     0x4UL
-+	u8	ring_select_mode;
-+	#define VNIC_RSS_CFG_REQ_RING_SELECT_MODE_TOEPLITZ          0x0UL
-+	#define VNIC_RSS_CFG_REQ_RING_SELECT_MODE_XOR               0x1UL
-+	#define VNIC_RSS_CFG_REQ_RING_SELECT_MODE_TOEPLITZ_CHECKSUM 0x2UL
-+	#define VNIC_RSS_CFG_REQ_RING_SELECT_MODE_LAST             VNIC_RSS_CFG_REQ_RING_SELECT_MODE_TOEPLITZ_CHECKSUM
-+	u8	unused_1[4];
-+};
-+
-+/* hwrm_vnic_rss_cfg_output (size:128b/16B) */
-+struct hwrm_vnic_rss_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_vnic_rss_cfg_cmd_err (size:64b/8B) */
-+struct hwrm_vnic_rss_cfg_cmd_err {
-+	u8	code;
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_UNKNOWN                      0x0UL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_INTERFACE_NOT_READY          0x1UL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_UNABLE_TO_GET_RSS_CFG        0x2UL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_HASH_TYPE_UNSUPPORTED        0x3UL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_HASH_TYPE_ERR                0x4UL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_HASH_MODE_FAIL               0x5UL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_RING_GRP_TABLE_ALLOC_ERR     0x6UL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_HASH_KEY_ALLOC_ERR           0x7UL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_DMA_FAILED                   0x8UL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_RX_RING_ALLOC_ERR            0x9UL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_CMPL_RING_ALLOC_ERR          0xaUL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_HW_SET_RSS_FAILED            0xbUL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_CTX_INVALID                  0xcUL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_VNIC_INVALID                 0xdUL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_VNIC_RING_TABLE_PAIR_INVALID 0xeUL
-+	#define VNIC_RSS_CFG_CMD_ERR_CODE_LAST                        VNIC_RSS_CFG_CMD_ERR_CODE_VNIC_RING_TABLE_PAIR_INVALID
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_vnic_rss_qcfg_input (size:192b/24B) */
-+struct hwrm_vnic_rss_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	rss_ctx_idx;
-+	__le16	vnic_id;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_vnic_rss_qcfg_output (size:512b/64B) */
-+struct hwrm_vnic_rss_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	hash_type;
-+	#define VNIC_RSS_QCFG_RESP_HASH_TYPE_IPV4                0x1UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_TYPE_TCP_IPV4            0x2UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_TYPE_UDP_IPV4            0x4UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_TYPE_IPV6                0x8UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_TYPE_TCP_IPV6            0x10UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_TYPE_UDP_IPV6            0x20UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_TYPE_IPV6_FLOW_LABEL     0x40UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_TYPE_AH_SPI_IPV4         0x80UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_TYPE_ESP_SPI_IPV4        0x100UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_TYPE_AH_SPI_IPV6         0x200UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_TYPE_ESP_SPI_IPV6        0x400UL
-+	u8	unused_0[4];
-+	__le32	hash_key[10];
-+	u8	hash_mode_flags;
-+	#define VNIC_RSS_QCFG_RESP_HASH_MODE_FLAGS_DEFAULT         0x1UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_MODE_FLAGS_INNERMOST_4     0x2UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_MODE_FLAGS_INNERMOST_2     0x4UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_MODE_FLAGS_OUTERMOST_4     0x8UL
-+	#define VNIC_RSS_QCFG_RESP_HASH_MODE_FLAGS_OUTERMOST_2     0x10UL
-+	u8	ring_select_mode;
-+	#define VNIC_RSS_QCFG_RESP_RING_SELECT_MODE_TOEPLITZ          0x0UL
-+	#define VNIC_RSS_QCFG_RESP_RING_SELECT_MODE_XOR               0x1UL
-+	#define VNIC_RSS_QCFG_RESP_RING_SELECT_MODE_TOEPLITZ_CHECKSUM 0x2UL
-+	#define VNIC_RSS_QCFG_RESP_RING_SELECT_MODE_LAST             VNIC_RSS_QCFG_RESP_RING_SELECT_MODE_TOEPLITZ_CHECKSUM
-+	u8	unused_1[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_vnic_plcmodes_cfg_input (size:320b/40B) */
-+struct hwrm_vnic_plcmodes_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_REGULAR_PLACEMENT     0x1UL
-+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT       0x2UL
-+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4              0x4UL
-+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6              0x8UL
-+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_FCOE              0x10UL
-+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_ROCE              0x20UL
-+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_VIRTIO_PLACEMENT      0x40UL
-+	__le32	enables;
-+	#define VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID      0x1UL
-+	#define VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_OFFSET_VALID        0x2UL
-+	#define VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID     0x4UL
-+	#define VNIC_PLCMODES_CFG_REQ_ENABLES_MAX_BDS_VALID           0x8UL
-+	__le32	vnic_id;
-+	__le16	jumbo_thresh;
-+	__le16	hds_offset;
-+	__le16	hds_threshold;
-+	__le16	max_bds;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_vnic_plcmodes_cfg_output (size:128b/16B) */
-+struct hwrm_vnic_plcmodes_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_vnic_plcmodes_cfg_cmd_err (size:64b/8B) */
-+struct hwrm_vnic_plcmodes_cfg_cmd_err {
-+	u8	code;
-+	#define VNIC_PLCMODES_CFG_CMD_ERR_CODE_UNKNOWN               0x0UL
-+	#define VNIC_PLCMODES_CFG_CMD_ERR_CODE_INVALID_HDS_THRESHOLD 0x1UL
-+	#define VNIC_PLCMODES_CFG_CMD_ERR_CODE_LAST                 VNIC_PLCMODES_CFG_CMD_ERR_CODE_INVALID_HDS_THRESHOLD
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_vnic_rss_cos_lb_ctx_alloc_input (size:128b/16B) */
-+struct hwrm_vnic_rss_cos_lb_ctx_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+};
-+
-+/* hwrm_vnic_rss_cos_lb_ctx_alloc_output (size:128b/16B) */
-+struct hwrm_vnic_rss_cos_lb_ctx_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	rss_cos_lb_ctx_id;
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_vnic_rss_cos_lb_ctx_free_input (size:192b/24B) */
-+struct hwrm_vnic_rss_cos_lb_ctx_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	rss_cos_lb_ctx_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_vnic_rss_cos_lb_ctx_free_output (size:128b/16B) */
-+struct hwrm_vnic_rss_cos_lb_ctx_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_ring_alloc_input (size:768b/96B) */
-+struct hwrm_ring_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	#define RING_ALLOC_REQ_ENABLES_RING_ARB_CFG              0x2UL
-+	#define RING_ALLOC_REQ_ENABLES_STAT_CTX_ID_VALID         0x8UL
-+	#define RING_ALLOC_REQ_ENABLES_MAX_BW_VALID              0x20UL
-+	#define RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID          0x40UL
-+	#define RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID          0x80UL
-+	#define RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID         0x100UL
-+	#define RING_ALLOC_REQ_ENABLES_SCHQ_ID                   0x200UL
-+	#define RING_ALLOC_REQ_ENABLES_MPC_CHNLS_TYPE            0x400UL
-+	#define RING_ALLOC_REQ_ENABLES_STEERING_TAG_VALID        0x800UL
-+	#define RING_ALLOC_REQ_ENABLES_RX_RATE_PROFILE_VALID     0x1000UL
-+	#define RING_ALLOC_REQ_ENABLES_DPI_VALID                 0x2000UL
-+	u8	ring_type;
-+	#define RING_ALLOC_REQ_RING_TYPE_L2_CMPL   0x0UL
-+	#define RING_ALLOC_REQ_RING_TYPE_TX        0x1UL
-+	#define RING_ALLOC_REQ_RING_TYPE_RX        0x2UL
-+	#define RING_ALLOC_REQ_RING_TYPE_ROCE_CMPL 0x3UL
-+	#define RING_ALLOC_REQ_RING_TYPE_RX_AGG    0x4UL
-+	#define RING_ALLOC_REQ_RING_TYPE_NQ        0x5UL
-+	#define RING_ALLOC_REQ_RING_TYPE_LAST     RING_ALLOC_REQ_RING_TYPE_NQ
-+	u8	cmpl_coal_cnt;
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_OFF 0x0UL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_4   0x1UL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_8   0x2UL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_12  0x3UL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_16  0x4UL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_24  0x5UL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_32  0x6UL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_48  0x7UL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_64  0x8UL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_96  0x9UL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_128 0xaUL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_192 0xbUL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_256 0xcUL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_320 0xdUL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_384 0xeUL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_MAX 0xfUL
-+	#define RING_ALLOC_REQ_CMPL_COAL_CNT_LAST    RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_MAX
-+	__le16	flags;
-+	#define RING_ALLOC_REQ_FLAGS_RX_SOP_PAD                        0x1UL
-+	#define RING_ALLOC_REQ_FLAGS_DISABLE_CQ_OVERFLOW_DETECTION     0x2UL
-+	#define RING_ALLOC_REQ_FLAGS_NQ_DBR_PACING                     0x4UL
-+	#define RING_ALLOC_REQ_FLAGS_TX_PKT_TS_CMPL_ENABLE             0x8UL
-+	__le64	page_tbl_addr;
-+	__le32	fbo;
-+	u8	page_size;
-+	u8	page_tbl_depth;
-+	__le16	schq_id;
-+	__le32	length;
-+	__le16	logical_id;
-+	__le16	cmpl_ring_id;
-+	__le16	queue_id;
-+	__le16	rx_buf_size;
-+	__le16	rx_ring_id;
-+	__le16	nq_ring_id;
-+	__le16	ring_arb_cfg;
-+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_MASK      0xfUL
-+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_SFT       0
-+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_SP          0x1UL
-+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_WFQ         0x2UL
-+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_LAST       RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_WFQ
-+	#define RING_ALLOC_REQ_RING_ARB_CFG_RSVD_MASK            0xf0UL
-+	#define RING_ALLOC_REQ_RING_ARB_CFG_RSVD_SFT             4
-+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_PARAM_MASK 0xff00UL
-+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_PARAM_SFT 8
-+	__le16	steering_tag;
-+	__le32	reserved3;
-+	__le32	stat_ctx_id;
-+	__le32	reserved4;
-+	__le32	max_bw;
-+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_MASK             0xfffffffUL
-+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_SFT              0
-+	#define RING_ALLOC_REQ_MAX_BW_SCALE                     0x10000000UL
-+	#define RING_ALLOC_REQ_MAX_BW_SCALE_BITS                  (0x0UL << 28)
-+	#define RING_ALLOC_REQ_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
-+	#define RING_ALLOC_REQ_MAX_BW_SCALE_LAST                 RING_ALLOC_REQ_MAX_BW_SCALE_BYTES
-+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
-+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_SFT         29
-+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
-+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
-+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
-+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
-+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
-+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_LAST         RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_INVALID
-+	u8	int_mode;
-+	#define RING_ALLOC_REQ_INT_MODE_LEGACY 0x0UL
-+	#define RING_ALLOC_REQ_INT_MODE_RSVD   0x1UL
-+	#define RING_ALLOC_REQ_INT_MODE_MSIX   0x2UL
-+	#define RING_ALLOC_REQ_INT_MODE_POLL   0x3UL
-+	#define RING_ALLOC_REQ_INT_MODE_LAST  RING_ALLOC_REQ_INT_MODE_POLL
-+	u8	mpc_chnls_type;
-+	#define RING_ALLOC_REQ_MPC_CHNLS_TYPE_TCE     0x0UL
-+	#define RING_ALLOC_REQ_MPC_CHNLS_TYPE_RCE     0x1UL
-+	#define RING_ALLOC_REQ_MPC_CHNLS_TYPE_TE_CFA  0x2UL
-+	#define RING_ALLOC_REQ_MPC_CHNLS_TYPE_RE_CFA  0x3UL
-+	#define RING_ALLOC_REQ_MPC_CHNLS_TYPE_PRIMATE 0x4UL
-+	#define RING_ALLOC_REQ_MPC_CHNLS_TYPE_LAST   RING_ALLOC_REQ_MPC_CHNLS_TYPE_PRIMATE
-+	u8	rx_rate_profile_sel;
-+	#define RING_ALLOC_REQ_RX_RATE_PROFILE_SEL_DEFAULT   0x0UL
-+	#define RING_ALLOC_REQ_RX_RATE_PROFILE_SEL_POLL_MODE 0x1UL
-+	#define RING_ALLOC_REQ_RX_RATE_PROFILE_SEL_LAST     RING_ALLOC_REQ_RX_RATE_PROFILE_SEL_POLL_MODE
-+	u8	unused_4;
-+	__le64	cq_handle;
-+	__le16	dpi;
-+	__le16	unused_5[3];
-+};
-+
-+/* hwrm_ring_alloc_output (size:128b/16B) */
-+struct hwrm_ring_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	ring_id;
-+	__le16	logical_ring_id;
-+	u8	push_buffer_index;
-+	#define RING_ALLOC_RESP_PUSH_BUFFER_INDEX_PING_BUFFER 0x0UL
-+	#define RING_ALLOC_RESP_PUSH_BUFFER_INDEX_PONG_BUFFER 0x1UL
-+	#define RING_ALLOC_RESP_PUSH_BUFFER_INDEX_LAST       RING_ALLOC_RESP_PUSH_BUFFER_INDEX_PONG_BUFFER
-+	u8	unused_0[2];
-+	u8	valid;
-+};
-+
-+/* hwrm_ring_free_input (size:256b/32B) */
-+struct hwrm_ring_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	ring_type;
-+	#define RING_FREE_REQ_RING_TYPE_L2_CMPL   0x0UL
-+	#define RING_FREE_REQ_RING_TYPE_TX        0x1UL
-+	#define RING_FREE_REQ_RING_TYPE_RX        0x2UL
-+	#define RING_FREE_REQ_RING_TYPE_ROCE_CMPL 0x3UL
-+	#define RING_FREE_REQ_RING_TYPE_RX_AGG    0x4UL
-+	#define RING_FREE_REQ_RING_TYPE_NQ        0x5UL
-+	#define RING_FREE_REQ_RING_TYPE_LAST     RING_FREE_REQ_RING_TYPE_NQ
-+	u8	flags;
-+	#define RING_FREE_REQ_FLAGS_VIRTIO_RING_VALID 0x1UL
-+	#define RING_FREE_REQ_FLAGS_LAST             RING_FREE_REQ_FLAGS_VIRTIO_RING_VALID
-+	__le16	ring_id;
-+	__le32	prod_idx;
-+	__le32	opaque;
-+	__le32	unused_1;
-+};
-+
-+/* hwrm_ring_free_output (size:128b/16B) */
-+struct hwrm_ring_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_ring_reset_input (size:192b/24B) */
-+struct hwrm_ring_reset_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	ring_type;
-+	#define RING_RESET_REQ_RING_TYPE_L2_CMPL     0x0UL
-+	#define RING_RESET_REQ_RING_TYPE_TX          0x1UL
-+	#define RING_RESET_REQ_RING_TYPE_RX          0x2UL
-+	#define RING_RESET_REQ_RING_TYPE_ROCE_CMPL   0x3UL
-+	#define RING_RESET_REQ_RING_TYPE_RX_RING_GRP 0x6UL
-+	#define RING_RESET_REQ_RING_TYPE_LAST       RING_RESET_REQ_RING_TYPE_RX_RING_GRP
-+	u8	unused_0;
-+	__le16	ring_id;
-+	u8	unused_1[4];
-+};
-+
-+/* hwrm_ring_reset_output (size:128b/16B) */
-+struct hwrm_ring_reset_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	push_buffer_index;
-+	#define RING_RESET_RESP_PUSH_BUFFER_INDEX_PING_BUFFER 0x0UL
-+	#define RING_RESET_RESP_PUSH_BUFFER_INDEX_PONG_BUFFER 0x1UL
-+	#define RING_RESET_RESP_PUSH_BUFFER_INDEX_LAST       RING_RESET_RESP_PUSH_BUFFER_INDEX_PONG_BUFFER
-+	u8	unused_0[3];
-+	u8	consumer_idx[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_ring_aggint_qcaps_input (size:128b/16B) */
-+struct hwrm_ring_aggint_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+};
-+
-+/* hwrm_ring_aggint_qcaps_output (size:384b/48B) */
-+struct hwrm_ring_aggint_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	cmpl_params;
-+	#define RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN                  0x1UL
-+	#define RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MAX                  0x2UL
-+	#define RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET                      0x4UL
-+	#define RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE                        0x8UL
-+	#define RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR                0x10UL
-+	#define RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT     0x20UL
-+	#define RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_CMPL_AGGR_DMA_TMR                0x40UL
-+	#define RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_CMPL_AGGR_DMA_TMR_DURING_INT     0x80UL
-+	#define RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_AGGR_INT                0x100UL
-+	__le32	nq_params;
-+	#define RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN     0x1UL
-+	__le16	num_cmpl_dma_aggr_min;
-+	__le16	num_cmpl_dma_aggr_max;
-+	__le16	num_cmpl_dma_aggr_during_int_min;
-+	__le16	num_cmpl_dma_aggr_during_int_max;
-+	__le16	cmpl_aggr_dma_tmr_min;
-+	__le16	cmpl_aggr_dma_tmr_max;
-+	__le16	cmpl_aggr_dma_tmr_during_int_min;
-+	__le16	cmpl_aggr_dma_tmr_during_int_max;
-+	__le16	int_lat_tmr_min_min;
-+	__le16	int_lat_tmr_min_max;
-+	__le16	int_lat_tmr_max_min;
-+	__le16	int_lat_tmr_max_max;
-+	__le16	num_cmpl_aggr_int_min;
-+	__le16	num_cmpl_aggr_int_max;
-+	__le16	timer_units;
-+	u8	unused_0[1];
-+	u8	valid;
-+};
-+
-+/* hwrm_ring_cmpl_ring_qaggint_params_input (size:192b/24B) */
-+struct hwrm_ring_cmpl_ring_qaggint_params_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	ring_id;
-+	__le16	flags;
-+	#define RING_CMPL_RING_QAGGINT_PARAMS_REQ_FLAGS_UNUSED_0_MASK 0x3UL
-+	#define RING_CMPL_RING_QAGGINT_PARAMS_REQ_FLAGS_UNUSED_0_SFT 0
-+	#define RING_CMPL_RING_QAGGINT_PARAMS_REQ_FLAGS_IS_NQ        0x4UL
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_ring_cmpl_ring_qaggint_params_output (size:256b/32B) */
-+struct hwrm_ring_cmpl_ring_qaggint_params_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	flags;
-+	#define RING_CMPL_RING_QAGGINT_PARAMS_RESP_FLAGS_TIMER_RESET     0x1UL
-+	#define RING_CMPL_RING_QAGGINT_PARAMS_RESP_FLAGS_RING_IDLE       0x2UL
-+	__le16	num_cmpl_dma_aggr;
-+	__le16	num_cmpl_dma_aggr_during_int;
-+	__le16	cmpl_aggr_dma_tmr;
-+	__le16	cmpl_aggr_dma_tmr_during_int;
-+	__le16	int_lat_tmr_min;
-+	__le16	int_lat_tmr_max;
-+	__le16	num_cmpl_aggr_int;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_ring_cmpl_ring_cfg_aggint_params_input (size:320b/40B) */
-+struct hwrm_ring_cmpl_ring_cfg_aggint_params_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	ring_id;
-+	__le16	flags;
-+	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET     0x1UL
-+	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE       0x2UL
-+	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ           0x4UL
-+	__le16	num_cmpl_dma_aggr;
-+	__le16	num_cmpl_dma_aggr_during_int;
-+	__le16	cmpl_aggr_dma_tmr;
-+	__le16	cmpl_aggr_dma_tmr_during_int;
-+	__le16	int_lat_tmr_min;
-+	__le16	int_lat_tmr_max;
-+	__le16	num_cmpl_aggr_int;
-+	__le16	enables;
-+	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_ENABLES_NUM_CMPL_DMA_AGGR                0x1UL
-+	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_ENABLES_NUM_CMPL_DMA_AGGR_DURING_INT     0x2UL
-+	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_ENABLES_CMPL_AGGR_DMA_TMR                0x4UL
-+	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_ENABLES_INT_LAT_TMR_MIN                  0x8UL
-+	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_ENABLES_INT_LAT_TMR_MAX                  0x10UL
-+	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_ENABLES_NUM_CMPL_AGGR_INT                0x20UL
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_ring_cmpl_ring_cfg_aggint_params_output (size:128b/16B) */
-+struct hwrm_ring_cmpl_ring_cfg_aggint_params_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_ring_grp_alloc_input (size:192b/24B) */
-+struct hwrm_ring_grp_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	cr;
-+	__le16	rr;
-+	__le16	ar;
-+	__le16	sc;
-+};
-+
-+/* hwrm_ring_grp_alloc_output (size:128b/16B) */
-+struct hwrm_ring_grp_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	ring_group_id;
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_ring_grp_free_input (size:192b/24B) */
-+struct hwrm_ring_grp_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	ring_group_id;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_ring_grp_free_output (size:128b/16B) */
-+struct hwrm_ring_grp_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+#define DEFAULT_FLOW_ID 0xFFFFFFFFUL
-+#define ROCEV1_FLOW_ID 0xFFFFFFFEUL
-+#define ROCEV2_FLOW_ID 0xFFFFFFFDUL
-+#define ROCEV2_CNP_FLOW_ID 0xFFFFFFFCUL
-+
-+/* hwrm_cfa_l2_filter_alloc_input (size:768b/96B) */
-+struct hwrm_cfa_l2_filter_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH              0x1UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_TX             0x0UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX             0x1UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_LAST          CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_LOOPBACK          0x2UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_DROP              0x4UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST         0x8UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_MASK      0x30UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_SFT       4
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_NO_ROCE_L2  (0x0UL << 4)
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_L2          (0x1UL << 4)
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_ROCE        (0x2UL << 4)
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_LAST       CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_ROCE
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_XDP_DISABLE       0x40UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_SOURCE_VALID      0x80UL
-+	__le32	enables;
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR             0x1UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK        0x2UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_OVLAN            0x4UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_OVLAN_MASK       0x8UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN            0x10UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN_MASK       0x20UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_ADDR           0x40UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_ADDR_MASK      0x80UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_OVLAN          0x100UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_OVLAN_MASK     0x200UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_IVLAN          0x400UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_IVLAN_MASK     0x800UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_SRC_TYPE            0x1000UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_SRC_ID              0x2000UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE         0x4000UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID              0x8000UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID      0x10000UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_NUM_VLANS           0x20000UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_NUM_VLANS         0x40000UL
-+	u8	l2_addr[6];
-+	u8	num_vlans;
-+	u8	t_num_vlans;
-+	u8	l2_addr_mask[6];
-+	__le16	l2_ovlan;
-+	__le16	l2_ovlan_mask;
-+	__le16	l2_ivlan;
-+	__le16	l2_ivlan_mask;
-+	u8	unused_1[2];
-+	u8	t_l2_addr[6];
-+	u8	unused_2[2];
-+	u8	t_l2_addr_mask[6];
-+	__le16	t_l2_ovlan;
-+	__le16	t_l2_ovlan_mask;
-+	__le16	t_l2_ivlan;
-+	__le16	t_l2_ivlan_mask;
-+	u8	src_type;
-+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_NPORT 0x0UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_PF    0x1UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_VF    0x2UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_VNIC  0x3UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_KONG  0x4UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_APE   0x5UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_BONO  0x6UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_TANG  0x7UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_LAST CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_TANG
-+	u8	unused_3;
-+	__le32	src_id;
-+	u8	tunnel_type;
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL    0x0UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN        0x1UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE        0x2UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE        0x3UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP         0x4UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE       0x5UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS         0x6UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT          0x7UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE        0x8UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4     0x9UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE_V1     0xaUL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2_ETYPE     0xbUL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE_V6 0xcUL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE    0x10UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL    0xffUL
-+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_LAST        CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL
-+	u8	unused_4;
-+	__le16	dst_id;
-+	__le16	mirror_vnic_id;
-+	u8	pri_hint;
-+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER    0x0UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_ABOVE_FILTER 0x1UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_BELOW_FILTER 0x2UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MAX          0x3UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN          0x4UL
-+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_LAST        CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN
-+	u8	unused_5;
-+	__le32	unused_6;
-+	__le64	l2_filter_id_hint;
-+};
-+
-+/* hwrm_cfa_l2_filter_alloc_output (size:192b/24B) */
-+struct hwrm_cfa_l2_filter_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le64	l2_filter_id;
-+	__le32	flow_id;
-+	#define CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_VALUE_MASK 0x3fffffffUL
-+	#define CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_VALUE_SFT 0
-+	#define CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_TYPE      0x40000000UL
-+	#define CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_TYPE_INT    (0x0UL << 30)
-+	#define CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_TYPE_EXT    (0x1UL << 30)
-+	#define CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_TYPE_LAST  CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_TYPE_EXT
-+	#define CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_DIR       0x80000000UL
-+	#define CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_DIR_RX      (0x0UL << 31)
-+	#define CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_DIR_TX      (0x1UL << 31)
-+	#define CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_DIR_LAST   CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_DIR_TX
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_l2_filter_free_input (size:192b/24B) */
-+struct hwrm_cfa_l2_filter_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	l2_filter_id;
-+};
-+
-+/* hwrm_cfa_l2_filter_free_output (size:128b/16B) */
-+struct hwrm_cfa_l2_filter_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_l2_filter_cfg_input (size:384b/48B) */
-+struct hwrm_cfa_l2_filter_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_PATH                  0x1UL
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_TX                 0x0UL
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_RX                 0x1UL
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_LAST              CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_RX
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_DROP                  0x2UL
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_MASK          0xcUL
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_SFT           2
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_NO_ROCE_L2      (0x0UL << 2)
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_L2              (0x1UL << 2)
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_ROCE            (0x2UL << 2)
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_LAST           CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_ROCE
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_MASK         0x30UL
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_SFT          4
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_NO_UPDATE      (0x0UL << 4)
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_BYPASS_LKUP    (0x1UL << 4)
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_ENABLE_LKUP    (0x2UL << 4)
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_RESTORE_FW_OP  (0x3UL << 4)
-+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_LAST          CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_RESTORE_FW_OP
-+	__le32	enables;
-+	#define CFA_L2_FILTER_CFG_REQ_ENABLES_DST_ID                 0x1UL
-+	#define CFA_L2_FILTER_CFG_REQ_ENABLES_NEW_MIRROR_VNIC_ID     0x2UL
-+	#define CFA_L2_FILTER_CFG_REQ_ENABLES_PROF_FUNC              0x4UL
-+	#define CFA_L2_FILTER_CFG_REQ_ENABLES_L2_CONTEXT_ID          0x8UL
-+	__le64	l2_filter_id;
-+	__le32	dst_id;
-+	__le32	new_mirror_vnic_id;
-+	__le32	prof_func;
-+	__le32	l2_context_id;
-+};
-+
-+/* hwrm_cfa_l2_filter_cfg_output (size:128b/16B) */
-+struct hwrm_cfa_l2_filter_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_l2_set_rx_mask_input (size:448b/56B) */
-+struct hwrm_cfa_l2_set_rx_mask_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	vnic_id;
-+	__le32	mask;
-+	#define CFA_L2_SET_RX_MASK_REQ_MASK_MCAST               0x2UL
-+	#define CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST           0x4UL
-+	#define CFA_L2_SET_RX_MASK_REQ_MASK_BCAST               0x8UL
-+	#define CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS         0x10UL
-+	#define CFA_L2_SET_RX_MASK_REQ_MASK_OUTERMOST           0x20UL
-+	#define CFA_L2_SET_RX_MASK_REQ_MASK_VLANONLY            0x40UL
-+	#define CFA_L2_SET_RX_MASK_REQ_MASK_VLAN_NONVLAN        0x80UL
-+	#define CFA_L2_SET_RX_MASK_REQ_MASK_ANYVLAN_NONVLAN     0x100UL
-+	__le64	mc_tbl_addr;
-+	__le32	num_mc_entries;
-+	u8	unused_0[4];
-+	__le64	vlan_tag_tbl_addr;
-+	__le32	num_vlan_tags;
-+	u8	unused_1[4];
-+};
-+
-+/* hwrm_cfa_l2_set_rx_mask_output (size:128b/16B) */
-+struct hwrm_cfa_l2_set_rx_mask_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_l2_set_rx_mask_cmd_err (size:64b/8B) */
-+struct hwrm_cfa_l2_set_rx_mask_cmd_err {
-+	u8	code;
-+	#define CFA_L2_SET_RX_MASK_CMD_ERR_CODE_UNKNOWN                    0x0UL
-+	#define CFA_L2_SET_RX_MASK_CMD_ERR_CODE_NTUPLE_FILTER_CONFLICT_ERR 0x1UL
-+	#define CFA_L2_SET_RX_MASK_CMD_ERR_CODE_MAX_VLAN_TAGS              0x2UL
-+	#define CFA_L2_SET_RX_MASK_CMD_ERR_CODE_INVALID_VNIC_ID            0x3UL
-+	#define CFA_L2_SET_RX_MASK_CMD_ERR_CODE_INVALID_ACTION             0x4UL
-+	#define CFA_L2_SET_RX_MASK_CMD_ERR_CODE_LAST                      CFA_L2_SET_RX_MASK_CMD_ERR_CODE_INVALID_ACTION
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_cfa_tunnel_filter_alloc_input (size:704b/88B) */
-+struct hwrm_cfa_tunnel_filter_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_FLAGS_LOOPBACK     0x1UL
-+	__le32	enables;
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID       0x1UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L2_ADDR            0x2UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN           0x4UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L3_ADDR            0x8UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L3_ADDR_TYPE       0x10UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_T_L3_ADDR_TYPE     0x20UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_T_L3_ADDR          0x40UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE        0x80UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_VNI                0x100UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_DST_VNIC_ID        0x200UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID     0x400UL
-+	__le64	l2_filter_id;
-+	u8	l2_addr[6];
-+	__le16	l2_ivlan;
-+	__le32	l3_addr[4];
-+	__le32	t_l3_addr[4];
-+	u8	l3_addr_type;
-+	u8	t_l3_addr_type;
-+	u8	tunnel_type;
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL    0x0UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN        0x1UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE        0x2UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE        0x3UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP         0x4UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE       0x5UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS         0x6UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT          0x7UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE        0x8UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4     0x9UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE_V1     0xaUL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2_ETYPE     0xbUL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE_V6 0xcUL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE    0x10UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL    0xffUL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_LAST        CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL
-+	u8	tunnel_flags;
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_FLAGS_TUN_FLAGS_OAM_CHECKSUM_EXPLHDR     0x1UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_FLAGS_TUN_FLAGS_CRITICAL_OPT_S1          0x2UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_FLAGS_TUN_FLAGS_EXTHDR_SEQNUM_S0         0x4UL
-+	__le32	vni;
-+	__le32	dst_vnic_id;
-+	__le32	mirror_vnic_id;
-+};
-+
-+/* hwrm_cfa_tunnel_filter_alloc_output (size:192b/24B) */
-+struct hwrm_cfa_tunnel_filter_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le64	tunnel_filter_id;
-+	__le32	flow_id;
-+	#define CFA_TUNNEL_FILTER_ALLOC_RESP_FLOW_ID_VALUE_MASK 0x3fffffffUL
-+	#define CFA_TUNNEL_FILTER_ALLOC_RESP_FLOW_ID_VALUE_SFT 0
-+	#define CFA_TUNNEL_FILTER_ALLOC_RESP_FLOW_ID_TYPE      0x40000000UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_RESP_FLOW_ID_TYPE_INT    (0x0UL << 30)
-+	#define CFA_TUNNEL_FILTER_ALLOC_RESP_FLOW_ID_TYPE_EXT    (0x1UL << 30)
-+	#define CFA_TUNNEL_FILTER_ALLOC_RESP_FLOW_ID_TYPE_LAST  CFA_TUNNEL_FILTER_ALLOC_RESP_FLOW_ID_TYPE_EXT
-+	#define CFA_TUNNEL_FILTER_ALLOC_RESP_FLOW_ID_DIR       0x80000000UL
-+	#define CFA_TUNNEL_FILTER_ALLOC_RESP_FLOW_ID_DIR_RX      (0x0UL << 31)
-+	#define CFA_TUNNEL_FILTER_ALLOC_RESP_FLOW_ID_DIR_TX      (0x1UL << 31)
-+	#define CFA_TUNNEL_FILTER_ALLOC_RESP_FLOW_ID_DIR_LAST   CFA_TUNNEL_FILTER_ALLOC_RESP_FLOW_ID_DIR_TX
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_tunnel_filter_free_input (size:192b/24B) */
-+struct hwrm_cfa_tunnel_filter_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	tunnel_filter_id;
-+};
-+
-+/* hwrm_cfa_tunnel_filter_free_output (size:128b/16B) */
-+struct hwrm_cfa_tunnel_filter_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_vxlan_ipv4_hdr (size:128b/16B) */
-+struct hwrm_vxlan_ipv4_hdr {
-+	u8	ver_hlen;
-+	#define VXLAN_IPV4_HDR_VER_HLEN_HEADER_LENGTH_MASK 0xfUL
-+	#define VXLAN_IPV4_HDR_VER_HLEN_HEADER_LENGTH_SFT 0
-+	#define VXLAN_IPV4_HDR_VER_HLEN_VERSION_MASK      0xf0UL
-+	#define VXLAN_IPV4_HDR_VER_HLEN_VERSION_SFT       4
-+	u8	tos;
-+	__be16	ip_id;
-+	__be16	flags_frag_offset;
-+	u8	ttl;
-+	u8	protocol;
-+	__be32	src_ip_addr;
-+	__be32	dest_ip_addr;
-+};
-+
-+/* hwrm_vxlan_ipv6_hdr (size:320b/40B) */
-+struct hwrm_vxlan_ipv6_hdr {
-+	__be32	ver_tc_flow_label;
-+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_VER_SFT         0x1cUL
-+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_VER_MASK        0xf0000000UL
-+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_TC_SFT          0x14UL
-+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_TC_MASK         0xff00000UL
-+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_FLOW_LABEL_SFT  0x0UL
-+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_FLOW_LABEL_MASK 0xfffffUL
-+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_LAST           VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_FLOW_LABEL_MASK
-+	__be16	payload_len;
-+	u8	next_hdr;
-+	u8	ttl;
-+	__be32	src_ip_addr[4];
-+	__be32	dest_ip_addr[4];
-+};
-+
-+/* hwrm_cfa_encap_data_vxlan (size:640b/80B) */
-+struct hwrm_cfa_encap_data_vxlan {
-+	u8	src_mac_addr[6];
-+	__le16	unused_0;
-+	u8	dst_mac_addr[6];
-+	u8	num_vlan_tags;
-+	u8	unused_1;
-+	__be16	ovlan_tpid;
-+	__be16	ovlan_tci;
-+	__be16	ivlan_tpid;
-+	__be16	ivlan_tci;
-+	__le32	l3[10];
-+	#define CFA_ENCAP_DATA_VXLAN_L3_VER_MASK 0xfUL
-+	#define CFA_ENCAP_DATA_VXLAN_L3_VER_IPV4 0x4UL
-+	#define CFA_ENCAP_DATA_VXLAN_L3_VER_IPV6 0x6UL
-+	#define CFA_ENCAP_DATA_VXLAN_L3_LAST    CFA_ENCAP_DATA_VXLAN_L3_VER_IPV6
-+	__be16	src_port;
-+	__be16	dst_port;
-+	__be32	vni;
-+	u8	hdr_rsvd0[3];
-+	u8	hdr_rsvd1;
-+	u8	hdr_flags;
-+	u8	unused[3];
-+};
-+
-+/* hwrm_cfa_encap_record_alloc_input (size:832b/104B) */
-+struct hwrm_cfa_encap_record_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_FLAGS_LOOPBACK     0x1UL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_FLAGS_EXTERNAL     0x2UL
-+	u8	encap_type;
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN        0x1UL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_NVGRE        0x2UL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_L2GRE        0x3UL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPIP         0x4UL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_GENEVE       0x5UL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_MPLS         0x6UL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VLAN         0x7UL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPGRE        0x8UL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN_V4     0x9UL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPGRE_V1     0xaUL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_L2_ETYPE     0xbUL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN_GPE_V6 0xcUL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN_GPE    0x10UL
-+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_LAST        CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN_GPE
-+	u8	unused_0[3];
-+	__le32	encap_data[20];
-+};
-+
-+/* hwrm_cfa_encap_record_alloc_output (size:128b/16B) */
-+struct hwrm_cfa_encap_record_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	encap_record_id;
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_encap_record_free_input (size:192b/24B) */
-+struct hwrm_cfa_encap_record_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	encap_record_id;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_cfa_encap_record_free_output (size:128b/16B) */
-+struct hwrm_cfa_encap_record_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_ntuple_filter_alloc_input (size:1024b/128B) */
-+struct hwrm_cfa_ntuple_filter_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_LOOPBACK              0x1UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DROP                  0x2UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_METER                 0x4UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_FID              0x8UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_ARP_REPLY             0x10UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX     0x20UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_NO_L2_CONTEXT         0x40UL
-+	__le32	enables;
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID         0x1UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE            0x2UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE          0x4UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR          0x8UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE          0x10UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR           0x20UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK      0x40UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR           0x80UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK      0x100UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL          0x200UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT             0x400UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK        0x800UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT             0x1000UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK        0x2000UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_PRI_HINT             0x4000UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_NTUPLE_FILTER_ID     0x8000UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID               0x10000UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID       0x20000UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR          0x40000UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX     0x80000UL
-+	__le64	l2_filter_id;
-+	u8	src_macaddr[6];
-+	__be16	ethertype;
-+	u8	ip_addr_type;
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_UNKNOWN 0x0UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4    0x4UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6    0x6UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_LAST   CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6
-+	u8	ip_protocol;
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UNKNOWN 0x0UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_TCP     0x6UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP     0x11UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_ICMP    0x1UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_ICMPV6  0x3aUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_RSVD    0xffUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_LAST   CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_RSVD
-+	__le16	dst_id;
-+	__le16	rfs_ring_tbl_idx;
-+	u8	tunnel_type;
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL    0x0UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN        0x1UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE        0x2UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE        0x3UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP         0x4UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE       0x5UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS         0x6UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT          0x7UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE        0x8UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4     0x9UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE_V1     0xaUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2_ETYPE     0xbUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE_V6 0xcUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE    0x10UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL    0xffUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_LAST        CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL
-+	u8	pri_hint;
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER 0x0UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_ABOVE     0x1UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_BELOW     0x2UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_HIGHEST   0x3UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LOWEST    0x4UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LAST     CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LOWEST
-+	__be32	src_ipaddr[4];
-+	__be32	src_ipaddr_mask[4];
-+	__be32	dst_ipaddr[4];
-+	__be32	dst_ipaddr_mask[4];
-+	__be16	src_port;
-+	__be16	src_port_mask;
-+	__be16	dst_port;
-+	__be16	dst_port_mask;
-+	__le64	ntuple_filter_id_hint;
-+};
-+
-+/* hwrm_cfa_ntuple_filter_alloc_output (size:192b/24B) */
-+struct hwrm_cfa_ntuple_filter_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le64	ntuple_filter_id;
-+	__le32	flow_id;
-+	#define CFA_NTUPLE_FILTER_ALLOC_RESP_FLOW_ID_VALUE_MASK 0x3fffffffUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_RESP_FLOW_ID_VALUE_SFT 0
-+	#define CFA_NTUPLE_FILTER_ALLOC_RESP_FLOW_ID_TYPE      0x40000000UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_RESP_FLOW_ID_TYPE_INT    (0x0UL << 30)
-+	#define CFA_NTUPLE_FILTER_ALLOC_RESP_FLOW_ID_TYPE_EXT    (0x1UL << 30)
-+	#define CFA_NTUPLE_FILTER_ALLOC_RESP_FLOW_ID_TYPE_LAST  CFA_NTUPLE_FILTER_ALLOC_RESP_FLOW_ID_TYPE_EXT
-+	#define CFA_NTUPLE_FILTER_ALLOC_RESP_FLOW_ID_DIR       0x80000000UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_RESP_FLOW_ID_DIR_RX      (0x0UL << 31)
-+	#define CFA_NTUPLE_FILTER_ALLOC_RESP_FLOW_ID_DIR_TX      (0x1UL << 31)
-+	#define CFA_NTUPLE_FILTER_ALLOC_RESP_FLOW_ID_DIR_LAST   CFA_NTUPLE_FILTER_ALLOC_RESP_FLOW_ID_DIR_TX
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_ntuple_filter_alloc_cmd_err (size:64b/8B) */
-+struct hwrm_cfa_ntuple_filter_alloc_cmd_err {
-+	u8	code;
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_UNKNOWN            0x0UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_ZERO_MAC           0x65UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_BC_MC_MAC          0x66UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_INVALID_VNIC       0x67UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_INVALID_PF_FID     0x68UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_INVALID_L2_CTXT_ID 0x69UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_NULL_L2_CTXT_CFG   0x6aUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_NULL_L2_DATA_FLD   0x6bUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_INVALID_CFA_LAYOUT 0x6cUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_L2_CTXT_ALLOC_FAIL 0x6dUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_ROCE_FLOW_ERR      0x6eUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_INVALID_OWNER_FID  0x6fUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_ZERO_REF_CNT       0x70UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_INVALID_FLOW_TYPE  0x71UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_INVALID_IVLAN      0x72UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_MAX_VLAN_ID        0x73UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_INVALID_TNL_REQ    0x74UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_L2_ADDR            0x75UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_L2_IVLAN           0x76UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_L3_ADDR            0x77UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_L3_ADDR_TYPE       0x78UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_T_L3_ADDR_TYPE     0x79UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_DST_VNIC_ID        0x7aUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_VNI                0x7bUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_INVALID_DST_ID     0x7cUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_FAIL_ROCE_L2_FLOW  0x7dUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_INVALID_NPAR_VLAN  0x7eUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_ATSP_ADD           0x7fUL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_DFLT_VLAN_FAIL     0x80UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_INVALID_L3_TYPE    0x81UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_VAL_FAIL_TNL_FLOW  0x82UL
-+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_LAST              CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_VAL_FAIL_TNL_FLOW
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_cfa_ntuple_filter_free_input (size:192b/24B) */
-+struct hwrm_cfa_ntuple_filter_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	ntuple_filter_id;
-+};
-+
-+/* hwrm_cfa_ntuple_filter_free_output (size:128b/16B) */
-+struct hwrm_cfa_ntuple_filter_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_ntuple_filter_cfg_input (size:384b/48B) */
-+struct hwrm_cfa_ntuple_filter_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	#define CFA_NTUPLE_FILTER_CFG_REQ_ENABLES_NEW_DST_ID                0x1UL
-+	#define CFA_NTUPLE_FILTER_CFG_REQ_ENABLES_NEW_MIRROR_VNIC_ID        0x2UL
-+	#define CFA_NTUPLE_FILTER_CFG_REQ_ENABLES_NEW_METER_INSTANCE_ID     0x4UL
-+	__le32	flags;
-+	#define CFA_NTUPLE_FILTER_CFG_REQ_FLAGS_DEST_FID              0x1UL
-+	#define CFA_NTUPLE_FILTER_CFG_REQ_FLAGS_DEST_RFS_RING_IDX     0x2UL
-+	#define CFA_NTUPLE_FILTER_CFG_REQ_FLAGS_NO_L2_CONTEXT         0x4UL
-+	__le64	ntuple_filter_id;
-+	__le32	new_dst_id;
-+	__le32	new_mirror_vnic_id;
-+	__le16	new_meter_instance_id;
-+	#define CFA_NTUPLE_FILTER_CFG_REQ_NEW_METER_INSTANCE_ID_INVALID 0xffffUL
-+	#define CFA_NTUPLE_FILTER_CFG_REQ_NEW_METER_INSTANCE_ID_LAST   CFA_NTUPLE_FILTER_CFG_REQ_NEW_METER_INSTANCE_ID_INVALID
-+	u8	unused_1[6];
-+};
-+
-+/* hwrm_cfa_ntuple_filter_cfg_output (size:128b/16B) */
-+struct hwrm_cfa_ntuple_filter_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_decap_filter_alloc_input (size:832b/104B) */
-+struct hwrm_cfa_decap_filter_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_FLAGS_OVS_TUNNEL     0x1UL
-+	__le32	enables;
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE        0x1UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_ID          0x2UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR        0x4UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR        0x8UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_OVLAN_VID          0x10UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IVLAN_VID          0x20UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_T_OVLAN_VID        0x40UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_T_IVLAN_VID        0x80UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE          0x100UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR         0x200UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR         0x400UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE        0x800UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL        0x1000UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_PORT           0x2000UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_PORT           0x4000UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_ID             0x8000UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID     0x10000UL
-+	__be32	tunnel_id;
-+	u8	tunnel_type;
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL    0x0UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN        0x1UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE        0x2UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE        0x3UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP         0x4UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE       0x5UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS         0x6UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT          0x7UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE        0x8UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4     0x9UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE_V1     0xaUL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2_ETYPE     0xbUL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE_V6 0xcUL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE    0x10UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL    0xffUL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_LAST        CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL
-+	u8	unused_0;
-+	__le16	unused_1;
-+	u8	src_macaddr[6];
-+	u8	unused_2[2];
-+	u8	dst_macaddr[6];
-+	__be16	ovlan_vid;
-+	__be16	ivlan_vid;
-+	__be16	t_ovlan_vid;
-+	__be16	t_ivlan_vid;
-+	__be16	ethertype;
-+	u8	ip_addr_type;
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_UNKNOWN 0x0UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4    0x4UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6    0x6UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_LAST   CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6
-+	u8	ip_protocol;
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UNKNOWN 0x0UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_TCP     0x6UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP     0x11UL
-+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_LAST   CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP
-+	__le16	unused_3;
-+	__le32	unused_4;
-+	__be32	src_ipaddr[4];
-+	__be32	dst_ipaddr[4];
-+	__be16	src_port;
-+	__be16	dst_port;
-+	__le16	dst_id;
-+	__le16	l2_ctxt_ref_id;
-+};
-+
-+/* hwrm_cfa_decap_filter_alloc_output (size:128b/16B) */
-+struct hwrm_cfa_decap_filter_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	decap_filter_id;
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_decap_filter_free_input (size:192b/24B) */
-+struct hwrm_cfa_decap_filter_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	decap_filter_id;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_cfa_decap_filter_free_output (size:128b/16B) */
-+struct hwrm_cfa_decap_filter_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_flow_alloc_input (size:1024b/128B) */
-+struct hwrm_cfa_flow_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	flags;
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_TUNNEL                 0x1UL
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_MASK          0x6UL
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_SFT           1
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_NONE            (0x0UL << 1)
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE             (0x1UL << 1)
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_TWO             (0x2UL << 1)
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_LAST           CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_TWO
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_MASK          0x38UL
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_SFT           3
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2              (0x0UL << 3)
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4            (0x1UL << 3)
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6            (0x2UL << 3)
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_LAST           CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_PATH_TX                0x40UL
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_PATH_RX                0x80UL
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_MATCH_VXLAN_IP_VNI     0x100UL
-+	#define CFA_FLOW_ALLOC_REQ_FLAGS_VHOST_ID_USE_VLAN      0x200UL
-+	__le16	src_fid;
-+	__le32	tunnel_handle;
-+	__le16	action_flags;
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD                       0x1UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_RECYCLE                   0x2UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP                      0x4UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_METER                     0x8UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TUNNEL                    0x10UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NAT_SRC                   0x20UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NAT_DEST                  0x40UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NAT_IPV4_ADDRESS          0x80UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE         0x100UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TTL_DECREMENT             0x200UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TUNNEL_IP                 0x400UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FLOW_AGING_ENABLED        0x800UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_PRI_HINT                  0x1000UL
-+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NO_FLOW_COUNTER_ALLOC     0x2000UL
-+	__le16	dst_fid;
-+	__be16	l2_rewrite_vlan_tpid;
-+	__be16	l2_rewrite_vlan_tci;
-+	__le16	act_meter_id;
-+	__le16	ref_flow_handle;
-+	__be16	ethertype;
-+	__be16	outer_vlan_tci;
-+	__be16	dmac[3];
-+	__be16	inner_vlan_tci;
-+	__be16	smac[3];
-+	u8	ip_dst_mask_len;
-+	u8	ip_src_mask_len;
-+	__be32	ip_dst[4];
-+	__be32	ip_src[4];
-+	__be16	l4_src_port;
-+	__be16	l4_src_port_mask;
-+	__be16	l4_dst_port;
-+	__be16	l4_dst_port_mask;
-+	__be32	nat_ip_address[4];
-+	__be16	l2_rewrite_dmac[3];
-+	__be16	nat_port;
-+	__be16	l2_rewrite_smac[3];
-+	u8	ip_proto;
-+	u8	tunnel_type;
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL    0x0UL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_VXLAN        0x1UL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_NVGRE        0x2UL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_L2GRE        0x3UL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_IPIP         0x4UL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_GENEVE       0x5UL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_MPLS         0x6UL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_STT          0x7UL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_IPGRE        0x8UL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4     0x9UL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_IPGRE_V1     0xaUL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_L2_ETYPE     0xbUL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE_V6 0xcUL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE    0x10UL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL    0xffUL
-+	#define CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_LAST        CFA_FLOW_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL
-+};
-+
-+/* hwrm_cfa_flow_alloc_output (size:256b/32B) */
-+struct hwrm_cfa_flow_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	flow_handle;
-+	u8	unused_0[2];
-+	__le32	flow_id;
-+	#define CFA_FLOW_ALLOC_RESP_FLOW_ID_VALUE_MASK 0x3fffffffUL
-+	#define CFA_FLOW_ALLOC_RESP_FLOW_ID_VALUE_SFT 0
-+	#define CFA_FLOW_ALLOC_RESP_FLOW_ID_TYPE      0x40000000UL
-+	#define CFA_FLOW_ALLOC_RESP_FLOW_ID_TYPE_INT    (0x0UL << 30)
-+	#define CFA_FLOW_ALLOC_RESP_FLOW_ID_TYPE_EXT    (0x1UL << 30)
-+	#define CFA_FLOW_ALLOC_RESP_FLOW_ID_TYPE_LAST  CFA_FLOW_ALLOC_RESP_FLOW_ID_TYPE_EXT
-+	#define CFA_FLOW_ALLOC_RESP_FLOW_ID_DIR       0x80000000UL
-+	#define CFA_FLOW_ALLOC_RESP_FLOW_ID_DIR_RX      (0x0UL << 31)
-+	#define CFA_FLOW_ALLOC_RESP_FLOW_ID_DIR_TX      (0x1UL << 31)
-+	#define CFA_FLOW_ALLOC_RESP_FLOW_ID_DIR_LAST   CFA_FLOW_ALLOC_RESP_FLOW_ID_DIR_TX
-+	__le64	ext_flow_handle;
-+	__le32	flow_counter_id;
-+	u8	unused_1[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_flow_alloc_cmd_err (size:64b/8B) */
-+struct hwrm_cfa_flow_alloc_cmd_err {
-+	u8	code;
-+	#define CFA_FLOW_ALLOC_CMD_ERR_CODE_UNKNOWN         0x0UL
-+	#define CFA_FLOW_ALLOC_CMD_ERR_CODE_L2_CONTEXT_TCAM 0x1UL
-+	#define CFA_FLOW_ALLOC_CMD_ERR_CODE_ACTION_RECORD   0x2UL
-+	#define CFA_FLOW_ALLOC_CMD_ERR_CODE_FLOW_COUNTER    0x3UL
-+	#define CFA_FLOW_ALLOC_CMD_ERR_CODE_WILD_CARD_TCAM  0x4UL
-+	#define CFA_FLOW_ALLOC_CMD_ERR_CODE_HASH_COLLISION  0x5UL
-+	#define CFA_FLOW_ALLOC_CMD_ERR_CODE_KEY_EXISTS      0x6UL
-+	#define CFA_FLOW_ALLOC_CMD_ERR_CODE_FLOW_CTXT_DB    0x7UL
-+	#define CFA_FLOW_ALLOC_CMD_ERR_CODE_LAST           CFA_FLOW_ALLOC_CMD_ERR_CODE_FLOW_CTXT_DB
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_cfa_flow_free_input (size:256b/32B) */
-+struct hwrm_cfa_flow_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	flow_handle;
-+	__le16	unused_0;
-+	__le32	flow_counter_id;
-+	__le64	ext_flow_handle;
-+};
-+
-+/* hwrm_cfa_flow_free_output (size:256b/32B) */
-+struct hwrm_cfa_flow_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le64	packet;
-+	__le64	byte;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_flow_info_input (size:256b/32B) */
-+struct hwrm_cfa_flow_info_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	flow_handle;
-+	#define CFA_FLOW_INFO_REQ_FLOW_HANDLE_MAX_MASK      0xfffUL
-+	#define CFA_FLOW_INFO_REQ_FLOW_HANDLE_CNP_CNT       0x1000UL
-+	#define CFA_FLOW_INFO_REQ_FLOW_HANDLE_ROCEV1_CNT    0x2000UL
-+	#define CFA_FLOW_INFO_REQ_FLOW_HANDLE_NIC_TX        0x3000UL
-+	#define CFA_FLOW_INFO_REQ_FLOW_HANDLE_ROCEV2_CNT    0x4000UL
-+	#define CFA_FLOW_INFO_REQ_FLOW_HANDLE_DIR_RX        0x8000UL
-+	#define CFA_FLOW_INFO_REQ_FLOW_HANDLE_CNP_CNT_RX    0x9000UL
-+	#define CFA_FLOW_INFO_REQ_FLOW_HANDLE_ROCEV1_CNT_RX 0xa000UL
-+	#define CFA_FLOW_INFO_REQ_FLOW_HANDLE_NIC_RX        0xb000UL
-+	#define CFA_FLOW_INFO_REQ_FLOW_HANDLE_ROCEV2_CNT_RX 0xc000UL
-+	#define CFA_FLOW_INFO_REQ_FLOW_HANDLE_LAST         CFA_FLOW_INFO_REQ_FLOW_HANDLE_ROCEV2_CNT_RX
-+	u8	unused_0[6];
-+	__le64	ext_flow_handle;
-+};
-+
-+/* hwrm_cfa_flow_info_output (size:5632b/704B) */
-+struct hwrm_cfa_flow_info_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	flags;
-+	#define CFA_FLOW_INFO_RESP_FLAGS_PATH_TX     0x1UL
-+	#define CFA_FLOW_INFO_RESP_FLAGS_PATH_RX     0x2UL
-+	u8	profile;
-+	__le16	src_fid;
-+	__le16	dst_fid;
-+	__le16	l2_ctxt_id;
-+	__le64	em_info;
-+	__le64	tcam_info;
-+	__le64	vfp_tcam_info;
-+	__le16	ar_id;
-+	__le16	flow_handle;
-+	__le32	tunnel_handle;
-+	__le16	flow_timer;
-+	u8	unused_0[6];
-+	__le32	flow_key_data[130];
-+	__le32	flow_action_info[30];
-+	u8	unused_1[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_flow_stats_input (size:640b/80B) */
-+struct hwrm_cfa_flow_stats_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	num_flows;
-+	__le16	flow_handle_0;
-+	__le16	flow_handle_1;
-+	__le16	flow_handle_2;
-+	__le16	flow_handle_3;
-+	__le16	flow_handle_4;
-+	__le16	flow_handle_5;
-+	__le16	flow_handle_6;
-+	__le16	flow_handle_7;
-+	__le16	flow_handle_8;
-+	__le16	flow_handle_9;
-+	u8	unused_0[2];
-+	__le32	flow_id_0;
-+	__le32	flow_id_1;
-+	__le32	flow_id_2;
-+	__le32	flow_id_3;
-+	__le32	flow_id_4;
-+	__le32	flow_id_5;
-+	__le32	flow_id_6;
-+	__le32	flow_id_7;
-+	__le32	flow_id_8;
-+	__le32	flow_id_9;
-+};
-+
-+/* hwrm_cfa_flow_stats_output (size:1408b/176B) */
-+struct hwrm_cfa_flow_stats_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le64	packet_0;
-+	__le64	packet_1;
-+	__le64	packet_2;
-+	__le64	packet_3;
-+	__le64	packet_4;
-+	__le64	packet_5;
-+	__le64	packet_6;
-+	__le64	packet_7;
-+	__le64	packet_8;
-+	__le64	packet_9;
-+	__le64	byte_0;
-+	__le64	byte_1;
-+	__le64	byte_2;
-+	__le64	byte_3;
-+	__le64	byte_4;
-+	__le64	byte_5;
-+	__le64	byte_6;
-+	__le64	byte_7;
-+	__le64	byte_8;
-+	__le64	byte_9;
-+	__le16	flow_hits;
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_vfr_alloc_input (size:448b/56B) */
-+struct hwrm_cfa_vfr_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	vf_id;
-+	__le16	reserved;
-+	u8	unused_0[4];
-+	char	vfr_name[32];
-+};
-+
-+/* hwrm_cfa_vfr_alloc_output (size:128b/16B) */
-+struct hwrm_cfa_vfr_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	rx_cfa_code;
-+	__le16	tx_cfa_action;
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_vfr_free_input (size:448b/56B) */
-+struct hwrm_cfa_vfr_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	char	vfr_name[32];
-+	__le16	vf_id;
-+	__le16	reserved;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_cfa_vfr_free_output (size:128b/16B) */
-+struct hwrm_cfa_vfr_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_eem_qcaps_input (size:192b/24B) */
-+struct hwrm_cfa_eem_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define CFA_EEM_QCAPS_REQ_FLAGS_PATH_TX               0x1UL
-+	#define CFA_EEM_QCAPS_REQ_FLAGS_PATH_RX               0x2UL
-+	#define CFA_EEM_QCAPS_REQ_FLAGS_PREFERRED_OFFLOAD     0x4UL
-+	__le32	unused_0;
-+};
-+
-+/* hwrm_cfa_eem_qcaps_output (size:320b/40B) */
-+struct hwrm_cfa_eem_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	flags;
-+	#define CFA_EEM_QCAPS_RESP_FLAGS_PATH_TX                                         0x1UL
-+	#define CFA_EEM_QCAPS_RESP_FLAGS_PATH_RX                                         0x2UL
-+	#define CFA_EEM_QCAPS_RESP_FLAGS_CENTRALIZED_MEMORY_MODEL_SUPPORTED              0x4UL
-+	#define CFA_EEM_QCAPS_RESP_FLAGS_DETACHED_CENTRALIZED_MEMORY_MODEL_SUPPORTED     0x8UL
-+	__le32	unused_0;
-+	__le32	supported;
-+	#define CFA_EEM_QCAPS_RESP_SUPPORTED_KEY0_TABLE                       0x1UL
-+	#define CFA_EEM_QCAPS_RESP_SUPPORTED_KEY1_TABLE                       0x2UL
-+	#define CFA_EEM_QCAPS_RESP_SUPPORTED_EXTERNAL_RECORD_TABLE            0x4UL
-+	#define CFA_EEM_QCAPS_RESP_SUPPORTED_EXTERNAL_FLOW_COUNTERS_TABLE     0x8UL
-+	#define CFA_EEM_QCAPS_RESP_SUPPORTED_FID_TABLE                        0x10UL
-+	__le32	max_entries_supported;
-+	__le16	key_entry_size;
-+	__le16	record_entry_size;
-+	__le16	efc_entry_size;
-+	__le16	fid_entry_size;
-+	u8	unused_1[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_eem_cfg_input (size:384b/48B) */
-+struct hwrm_cfa_eem_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define CFA_EEM_CFG_REQ_FLAGS_PATH_TX               0x1UL
-+	#define CFA_EEM_CFG_REQ_FLAGS_PATH_RX               0x2UL
-+	#define CFA_EEM_CFG_REQ_FLAGS_PREFERRED_OFFLOAD     0x4UL
-+	#define CFA_EEM_CFG_REQ_FLAGS_SECONDARY_PF          0x8UL
-+	__le16	group_id;
-+	__le16	unused_0;
-+	__le32	num_entries;
-+	__le32	unused_1;
-+	__le16	key0_ctx_id;
-+	__le16	key1_ctx_id;
-+	__le16	record_ctx_id;
-+	__le16	efc_ctx_id;
-+	__le16	fid_ctx_id;
-+	__le16	unused_2;
-+	__le32	unused_3;
-+};
-+
-+/* hwrm_cfa_eem_cfg_output (size:128b/16B) */
-+struct hwrm_cfa_eem_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_eem_qcfg_input (size:192b/24B) */
-+struct hwrm_cfa_eem_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define CFA_EEM_QCFG_REQ_FLAGS_PATH_TX     0x1UL
-+	#define CFA_EEM_QCFG_REQ_FLAGS_PATH_RX     0x2UL
-+	__le32	unused_0;
-+};
-+
-+/* hwrm_cfa_eem_qcfg_output (size:256b/32B) */
-+struct hwrm_cfa_eem_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	flags;
-+	#define CFA_EEM_QCFG_RESP_FLAGS_PATH_TX               0x1UL
-+	#define CFA_EEM_QCFG_RESP_FLAGS_PATH_RX               0x2UL
-+	#define CFA_EEM_QCFG_RESP_FLAGS_PREFERRED_OFFLOAD     0x4UL
-+	__le32	num_entries;
-+	__le16	key0_ctx_id;
-+	__le16	key1_ctx_id;
-+	__le16	record_ctx_id;
-+	__le16	efc_ctx_id;
-+	__le16	fid_ctx_id;
-+	u8	unused_2[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_eem_op_input (size:192b/24B) */
-+struct hwrm_cfa_eem_op_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define CFA_EEM_OP_REQ_FLAGS_PATH_TX     0x1UL
-+	#define CFA_EEM_OP_REQ_FLAGS_PATH_RX     0x2UL
-+	__le16	unused_0;
-+	__le16	op;
-+	#define CFA_EEM_OP_REQ_OP_RESERVED    0x0UL
-+	#define CFA_EEM_OP_REQ_OP_EEM_DISABLE 0x1UL
-+	#define CFA_EEM_OP_REQ_OP_EEM_ENABLE  0x2UL
-+	#define CFA_EEM_OP_REQ_OP_EEM_CLEANUP 0x3UL
-+	#define CFA_EEM_OP_REQ_OP_LAST       CFA_EEM_OP_REQ_OP_EEM_CLEANUP
-+};
-+
-+/* hwrm_cfa_eem_op_output (size:128b/16B) */
-+struct hwrm_cfa_eem_op_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_cfa_adv_flow_mgnt_qcaps_input (size:256b/32B) */
-+struct hwrm_cfa_adv_flow_mgnt_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	unused_0[4];
-+};
-+
-+/* hwrm_cfa_adv_flow_mgnt_qcaps_output (size:128b/16B) */
-+struct hwrm_cfa_adv_flow_mgnt_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	flags;
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_HND_16BIT_SUPPORTED                     0x1UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_HND_64BIT_SUPPORTED                     0x2UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_BATCH_DELETE_SUPPORTED                  0x4UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_RESET_ALL_SUPPORTED                     0x8UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_DEST_FUNC_SUPPORTED              0x10UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_TX_EEM_FLOW_SUPPORTED                        0x20UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RX_EEM_FLOW_SUPPORTED                        0x40UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_COUNTER_ALLOC_SUPPORTED                 0x80UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_SUPPORTED                   0x100UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_UNTAGGED_VLAN_SUPPORTED                      0x200UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_XDP_SUPPORTED                                0x400UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_L2_HEADER_SOURCE_FIELDS_SUPPORTED            0x800UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_ARP_SUPPORTED                 0x1000UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED                0x2000UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_ETHERTYPE_IP_SUPPORTED        0x4000UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_TRUFLOW_CAPABLE                              0x8000UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_L2_FILTER_TRAFFIC_TYPE_L2_ROCE_SUPPORTED     0x10000UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_LAG_SUPPORTED                                0x20000UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_NO_L2CTX_SUPPORTED               0x40000UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NIC_FLOW_STATS_SUPPORTED                     0x80000UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_EXT_IP_PROTO_SUPPORTED        0x100000UL
-+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V3_SUPPORTED                0x200000UL
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_tunnel_dst_port_query_input (size:192b/24B) */
-+struct hwrm_tunnel_dst_port_query_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	tunnel_type;
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_VXLAN              0x1UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_GENEVE             0x5UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_VXLAN_V4           0x9UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_IPGRE_V1           0xaUL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_L2_ETYPE           0xbUL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_VXLAN_GPE_V6       0xcUL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_CUSTOM_GRE         0xdUL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_ECPRI              0xeUL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_SRV6               0xfUL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_VXLAN_GPE          0x10UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_GRE                0x11UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_ULP_DYN_UPAR       0x12UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES01 0x13UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES02 0x14UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES03 0x15UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES04 0x16UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES05 0x17UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES06 0x18UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES07 0x19UL
-+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_LAST              TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES07
-+	u8	tunnel_next_proto;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_tunnel_dst_port_query_output (size:128b/16B) */
-+struct hwrm_tunnel_dst_port_query_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	tunnel_dst_port_id;
-+	__be16	tunnel_dst_port_val;
-+	u8	upar_in_use;
-+	#define TUNNEL_DST_PORT_QUERY_RESP_UPAR_IN_USE_UPAR0     0x1UL
-+	#define TUNNEL_DST_PORT_QUERY_RESP_UPAR_IN_USE_UPAR1     0x2UL
-+	#define TUNNEL_DST_PORT_QUERY_RESP_UPAR_IN_USE_UPAR2     0x4UL
-+	#define TUNNEL_DST_PORT_QUERY_RESP_UPAR_IN_USE_UPAR3     0x8UL
-+	#define TUNNEL_DST_PORT_QUERY_RESP_UPAR_IN_USE_UPAR4     0x10UL
-+	#define TUNNEL_DST_PORT_QUERY_RESP_UPAR_IN_USE_UPAR5     0x20UL
-+	#define TUNNEL_DST_PORT_QUERY_RESP_UPAR_IN_USE_UPAR6     0x40UL
-+	#define TUNNEL_DST_PORT_QUERY_RESP_UPAR_IN_USE_UPAR7     0x80UL
-+	u8	status;
-+	#define TUNNEL_DST_PORT_QUERY_RESP_STATUS_CHIP_LEVEL     0x1UL
-+	#define TUNNEL_DST_PORT_QUERY_RESP_STATUS_FUNC_LEVEL     0x2UL
-+	u8	unused_0;
-+	u8	valid;
-+};
-+
-+/* hwrm_tunnel_dst_port_alloc_input (size:192b/24B) */
-+struct hwrm_tunnel_dst_port_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	tunnel_type;
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN              0x1UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE             0x5UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4           0x9UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_IPGRE_V1           0xaUL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_L2_ETYPE           0xbUL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE_V6       0xcUL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_CUSTOM_GRE         0xdUL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_ECPRI              0xeUL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_SRV6               0xfUL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE          0x10UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GRE                0x11UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_ULP_DYN_UPAR       0x12UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES01 0x13UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES02 0x14UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES03 0x15UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES04 0x16UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES05 0x17UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES06 0x18UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES07 0x19UL
-+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_LAST              TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES07
-+	u8	tunnel_next_proto;
-+	__be16	tunnel_dst_port_val;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_tunnel_dst_port_alloc_output (size:128b/16B) */
-+struct hwrm_tunnel_dst_port_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	tunnel_dst_port_id;
-+	u8	error_info;
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_ERROR_INFO_SUCCESS         0x0UL
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_ERROR_INFO_ERR_ALLOCATED   0x1UL
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_ERROR_INFO_ERR_NO_RESOURCE 0x2UL
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_ERROR_INFO_ERR_ENABLED     0x3UL
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_ERROR_INFO_LAST           TUNNEL_DST_PORT_ALLOC_RESP_ERROR_INFO_ERR_ENABLED
-+	u8	upar_in_use;
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_UPAR_IN_USE_UPAR0     0x1UL
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_UPAR_IN_USE_UPAR1     0x2UL
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_UPAR_IN_USE_UPAR2     0x4UL
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_UPAR_IN_USE_UPAR3     0x8UL
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_UPAR_IN_USE_UPAR4     0x10UL
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_UPAR_IN_USE_UPAR5     0x20UL
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_UPAR_IN_USE_UPAR6     0x40UL
-+	#define TUNNEL_DST_PORT_ALLOC_RESP_UPAR_IN_USE_UPAR7     0x80UL
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_tunnel_dst_port_free_input (size:192b/24B) */
-+struct hwrm_tunnel_dst_port_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	tunnel_type;
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN              0x1UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE             0x5UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_V4           0x9UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_IPGRE_V1           0xaUL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_L2_ETYPE           0xbUL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE_V6       0xcUL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_CUSTOM_GRE         0xdUL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_ECPRI              0xeUL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_SRV6               0xfUL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE          0x10UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GRE                0x11UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_ULP_DYN_UPAR       0x12UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES01 0x13UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES02 0x14UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES03 0x15UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES04 0x16UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES05 0x17UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES06 0x18UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES07 0x19UL
-+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_LAST              TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_ULP_DYN_UPAR_RES07
-+	u8	tunnel_next_proto;
-+	__le16	tunnel_dst_port_id;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_tunnel_dst_port_free_output (size:128b/16B) */
-+struct hwrm_tunnel_dst_port_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	error_info;
-+	#define TUNNEL_DST_PORT_FREE_RESP_ERROR_INFO_SUCCESS           0x0UL
-+	#define TUNNEL_DST_PORT_FREE_RESP_ERROR_INFO_ERR_NOT_OWNER     0x1UL
-+	#define TUNNEL_DST_PORT_FREE_RESP_ERROR_INFO_ERR_NOT_ALLOCATED 0x2UL
-+	#define TUNNEL_DST_PORT_FREE_RESP_ERROR_INFO_LAST             TUNNEL_DST_PORT_FREE_RESP_ERROR_INFO_ERR_NOT_ALLOCATED
-+	u8	unused_1[6];
-+	u8	valid;
-+};
-+
-+/* ctx_hw_stats (size:1280b/160B) */
-+struct ctx_hw_stats {
-+	__le64	rx_ucast_pkts;
-+	__le64	rx_mcast_pkts;
-+	__le64	rx_bcast_pkts;
-+	__le64	rx_discard_pkts;
-+	__le64	rx_error_pkts;
-+	__le64	rx_ucast_bytes;
-+	__le64	rx_mcast_bytes;
-+	__le64	rx_bcast_bytes;
-+	__le64	tx_ucast_pkts;
-+	__le64	tx_mcast_pkts;
-+	__le64	tx_bcast_pkts;
-+	__le64	tx_error_pkts;
-+	__le64	tx_discard_pkts;
-+	__le64	tx_ucast_bytes;
-+	__le64	tx_mcast_bytes;
-+	__le64	tx_bcast_bytes;
-+	__le64	tpa_pkts;
-+	__le64	tpa_bytes;
-+	__le64	tpa_events;
-+	__le64	tpa_aborts;
-+};
-+
-+/* ctx_hw_stats_ext (size:1408b/176B) */
-+struct ctx_hw_stats_ext {
-+	__le64	rx_ucast_pkts;
-+	__le64	rx_mcast_pkts;
-+	__le64	rx_bcast_pkts;
-+	__le64	rx_discard_pkts;
-+	__le64	rx_error_pkts;
-+	__le64	rx_ucast_bytes;
-+	__le64	rx_mcast_bytes;
-+	__le64	rx_bcast_bytes;
-+	__le64	tx_ucast_pkts;
-+	__le64	tx_mcast_pkts;
-+	__le64	tx_bcast_pkts;
-+	__le64	tx_error_pkts;
-+	__le64	tx_discard_pkts;
-+	__le64	tx_ucast_bytes;
-+	__le64	tx_mcast_bytes;
-+	__le64	tx_bcast_bytes;
-+	__le64	rx_tpa_eligible_pkt;
-+	__le64	rx_tpa_eligible_bytes;
-+	__le64	rx_tpa_pkt;
-+	__le64	rx_tpa_bytes;
-+	__le64	rx_tpa_errors;
-+	__le64	rx_tpa_events;
-+};
-+
-+/* hwrm_stat_ctx_alloc_input (size:384b/48B) */
-+struct hwrm_stat_ctx_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	stats_dma_addr;
-+	__le32	update_period_ms;
-+	u8	stat_ctx_flags;
-+	#define STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE             0x1UL
-+	#define STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_DUP_HOST_BUF     0x2UL
-+	u8	unused_0;
-+	__le16	stats_dma_length;
-+	__le16	flags;
-+	#define STAT_CTX_ALLOC_REQ_FLAGS_STEERING_TAG_VALID     0x1UL
-+	__le16	steering_tag;
-+	__le32	stat_ctx_id;
-+	__le16	alloc_seq_id;
-+	u8	unused_1[6];
-+};
-+
-+/* hwrm_stat_ctx_alloc_output (size:128b/16B) */
-+struct hwrm_stat_ctx_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	stat_ctx_id;
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_stat_ctx_free_input (size:192b/24B) */
-+struct hwrm_stat_ctx_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	stat_ctx_id;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_stat_ctx_free_output (size:128b/16B) */
-+struct hwrm_stat_ctx_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	stat_ctx_id;
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_stat_ctx_query_input (size:192b/24B) */
-+struct hwrm_stat_ctx_query_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	stat_ctx_id;
-+	u8	flags;
-+	#define STAT_CTX_QUERY_REQ_FLAGS_COUNTER_MASK     0x1UL
-+	u8	unused_0[3];
-+};
-+
-+/* hwrm_stat_ctx_query_output (size:1408b/176B) */
-+struct hwrm_stat_ctx_query_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le64	tx_ucast_pkts;
-+	__le64	tx_mcast_pkts;
-+	__le64	tx_bcast_pkts;
-+	__le64	tx_discard_pkts;
-+	__le64	tx_error_pkts;
-+	__le64	tx_ucast_bytes;
-+	__le64	tx_mcast_bytes;
-+	__le64	tx_bcast_bytes;
-+	__le64	rx_ucast_pkts;
-+	__le64	rx_mcast_pkts;
-+	__le64	rx_bcast_pkts;
-+	__le64	rx_discard_pkts;
-+	__le64	rx_error_pkts;
-+	__le64	rx_ucast_bytes;
-+	__le64	rx_mcast_bytes;
-+	__le64	rx_bcast_bytes;
-+	__le64	rx_agg_pkts;
-+	__le64	rx_agg_bytes;
-+	__le64	rx_agg_events;
-+	__le64	rx_agg_aborts;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_stat_ext_ctx_query_input (size:192b/24B) */
-+struct hwrm_stat_ext_ctx_query_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	stat_ctx_id;
-+	u8	flags;
-+	#define STAT_EXT_CTX_QUERY_REQ_FLAGS_COUNTER_MASK     0x1UL
-+	u8	unused_0[3];
-+};
-+
-+/* hwrm_stat_ext_ctx_query_output (size:1536b/192B) */
-+struct hwrm_stat_ext_ctx_query_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le64	rx_ucast_pkts;
-+	__le64	rx_mcast_pkts;
-+	__le64	rx_bcast_pkts;
-+	__le64	rx_discard_pkts;
-+	__le64	rx_error_pkts;
-+	__le64	rx_ucast_bytes;
-+	__le64	rx_mcast_bytes;
-+	__le64	rx_bcast_bytes;
-+	__le64	tx_ucast_pkts;
-+	__le64	tx_mcast_pkts;
-+	__le64	tx_bcast_pkts;
-+	__le64	tx_error_pkts;
-+	__le64	tx_discard_pkts;
-+	__le64	tx_ucast_bytes;
-+	__le64	tx_mcast_bytes;
-+	__le64	tx_bcast_bytes;
-+	__le64	rx_tpa_eligible_pkt;
-+	__le64	rx_tpa_eligible_bytes;
-+	__le64	rx_tpa_pkt;
-+	__le64	rx_tpa_bytes;
-+	__le64	rx_tpa_errors;
-+	__le64	rx_tpa_events;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_stat_ctx_clr_stats_input (size:192b/24B) */
-+struct hwrm_stat_ctx_clr_stats_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	stat_ctx_id;
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_stat_ctx_clr_stats_output (size:128b/16B) */
-+struct hwrm_stat_ctx_clr_stats_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_pcie_qstats_input (size:256b/32B) */
-+struct hwrm_pcie_qstats_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	pcie_stat_size;
-+	u8	unused_0[6];
-+	__le64	pcie_stat_host_addr;
-+};
-+
-+/* hwrm_pcie_qstats_output (size:128b/16B) */
-+struct hwrm_pcie_qstats_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	pcie_stat_size;
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* pcie_ctx_hw_stats (size:768b/96B) */
-+struct pcie_ctx_hw_stats {
-+	__le64	pcie_pl_signal_integrity;
-+	__le64	pcie_dl_signal_integrity;
-+	__le64	pcie_tl_signal_integrity;
-+	__le64	pcie_link_integrity;
-+	__le64	pcie_tx_traffic_rate;
-+	__le64	pcie_rx_traffic_rate;
-+	__le64	pcie_tx_dllp_statistics;
-+	__le64	pcie_rx_dllp_statistics;
-+	__le64	pcie_equalization_time;
-+	__le32	pcie_ltssm_histogram[4];
-+	__le64	pcie_recovery_histogram;
-+};
-+
-+/* pcie_ctx_hw_stats_v2 (size:4544b/568B) */
-+struct pcie_ctx_hw_stats_v2 {
-+	__le64	pcie_pl_signal_integrity;
-+	__le64	pcie_dl_signal_integrity;
-+	__le64	pcie_tl_signal_integrity;
-+	__le64	pcie_link_integrity;
-+	__le64	pcie_tx_traffic_rate;
-+	__le64	pcie_rx_traffic_rate;
-+	__le64	pcie_tx_dllp_statistics;
-+	__le64	pcie_rx_dllp_statistics;
-+	__le64	pcie_equalization_time;
-+	__le32	pcie_ltssm_histogram[4];
-+	__le64	pcie_recovery_histogram;
-+	__le32	pcie_tl_credit_nph_histogram[8];
-+	__le32	pcie_tl_credit_ph_histogram[8];
-+	__le32	pcie_tl_credit_pd_histogram[8];
-+	__le32	pcie_cmpl_latest_times[4];
-+	__le32	pcie_cmpl_longest_time;
-+	__le32	pcie_cmpl_shortest_time;
-+	__le32	unused_0[2];
-+	__le32	pcie_cmpl_latest_headers[4][4];
-+	__le32	pcie_cmpl_longest_headers[4][4];
-+	__le32	pcie_cmpl_shortest_headers[4][4];
-+	__le32	pcie_wr_latency_histogram[12];
-+	__le32	pcie_wr_latency_all_normal_count;
-+	__le32	unused_1;
-+	__le64	pcie_posted_packet_count;
-+	__le64	pcie_non_posted_packet_count;
-+	__le64	pcie_other_packet_count;
-+	__le64	pcie_blocked_packet_count;
-+	__le64	pcie_cmpl_packet_count;
-+	__le32	pcie_rd_latency_histogram[12];
-+	__le32	pcie_rd_latency_all_normal_count;
-+	__le32	unused_2;
-+};
-+
-+/* hwrm_stat_generic_qstats_input (size:256b/32B) */
-+struct hwrm_stat_generic_qstats_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	generic_stat_size;
-+	u8	flags;
-+	#define STAT_GENERIC_QSTATS_REQ_FLAGS_COUNTER_MASK     0x1UL
-+	u8	unused_0[5];
-+	__le64	generic_stat_host_addr;
-+};
-+
-+/* hwrm_stat_generic_qstats_output (size:128b/16B) */
-+struct hwrm_stat_generic_qstats_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	generic_stat_size;
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* generic_sw_hw_stats (size:1472b/184B) */
-+struct generic_sw_hw_stats {
-+	__le64	pcie_statistics_tx_tlp;
-+	__le64	pcie_statistics_rx_tlp;
-+	__le64	pcie_credit_fc_hdr_posted;
-+	__le64	pcie_credit_fc_hdr_nonposted;
-+	__le64	pcie_credit_fc_hdr_cmpl;
-+	__le64	pcie_credit_fc_data_posted;
-+	__le64	pcie_credit_fc_data_nonposted;
-+	__le64	pcie_credit_fc_data_cmpl;
-+	__le64	pcie_credit_fc_tgt_nonposted;
-+	__le64	pcie_credit_fc_tgt_data_posted;
-+	__le64	pcie_credit_fc_tgt_hdr_posted;
-+	__le64	pcie_credit_fc_cmpl_hdr_posted;
-+	__le64	pcie_credit_fc_cmpl_data_posted;
-+	__le64	pcie_cmpl_longest;
-+	__le64	pcie_cmpl_shortest;
-+	__le64	cache_miss_count_cfcq;
-+	__le64	cache_miss_count_cfcs;
-+	__le64	cache_miss_count_cfcc;
-+	__le64	cache_miss_count_cfcm;
-+	__le64	hw_db_recov_dbs_dropped;
-+	__le64	hw_db_recov_drops_serviced;
-+	__le64	hw_db_recov_dbs_recovered;
-+	__le64	hw_db_recov_oo_drop_count;
-+};
-+
-+/* hwrm_fw_reset_input (size:192b/24B) */
-+struct hwrm_fw_reset_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	embedded_proc_type;
-+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_BOOT                  0x0UL
-+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_MGMT                  0x1UL
-+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_NETCTRL               0x2UL
-+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_ROCE                  0x3UL
-+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_HOST                  0x4UL
-+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_AP                    0x5UL
-+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP                  0x6UL
-+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_HOST_RESOURCE_REINIT  0x7UL
-+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_IMPACTLESS_ACTIVATION 0x8UL
-+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_LAST                 FW_RESET_REQ_EMBEDDED_PROC_TYPE_IMPACTLESS_ACTIVATION
-+	u8	selfrst_status;
-+	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTNONE      0x0UL
-+	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP      0x1UL
-+	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTPCIERST   0x2UL
-+	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTIMMEDIATE 0x3UL
-+	#define FW_RESET_REQ_SELFRST_STATUS_LAST            FW_RESET_REQ_SELFRST_STATUS_SELFRSTIMMEDIATE
-+	u8	host_idx;
-+	u8	flags;
-+	#define FW_RESET_REQ_FLAGS_RESET_GRACEFUL     0x1UL
-+	#define FW_RESET_REQ_FLAGS_FW_ACTIVATION      0x2UL
-+	u8	unused_0[4];
-+};
-+
-+/* hwrm_fw_reset_output (size:128b/16B) */
-+struct hwrm_fw_reset_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	selfrst_status;
-+	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTNONE      0x0UL
-+	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTASAP      0x1UL
-+	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTPCIERST   0x2UL
-+	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTIMMEDIATE 0x3UL
-+	#define FW_RESET_RESP_SELFRST_STATUS_LAST            FW_RESET_RESP_SELFRST_STATUS_SELFRSTIMMEDIATE
-+	u8	unused_0[6];
-+	u8	valid;
-+};
-+
-+/* hwrm_fw_qstatus_input (size:192b/24B) */
-+struct hwrm_fw_qstatus_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	embedded_proc_type;
-+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_BOOT    0x0UL
-+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_MGMT    0x1UL
-+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_NETCTRL 0x2UL
-+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_ROCE    0x3UL
-+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_HOST    0x4UL
-+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_AP      0x5UL
-+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_CHIP    0x6UL
-+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_LAST   FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_CHIP
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_fw_qstatus_output (size:128b/16B) */
-+struct hwrm_fw_qstatus_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	selfrst_status;
-+	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTNONE    0x0UL
-+	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTASAP    0x1UL
-+	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTPCIERST 0x2UL
-+	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTPOWER   0x3UL
-+	#define FW_QSTATUS_RESP_SELFRST_STATUS_LAST          FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTPOWER
-+	u8	nvm_option_action_status;
-+	#define FW_QSTATUS_RESP_NVM_OPTION_ACTION_STATUS_NVMOPT_ACTION_NONE     0x0UL
-+	#define FW_QSTATUS_RESP_NVM_OPTION_ACTION_STATUS_NVMOPT_ACTION_HOTRESET 0x1UL
-+	#define FW_QSTATUS_RESP_NVM_OPTION_ACTION_STATUS_NVMOPT_ACTION_WARMBOOT 0x2UL
-+	#define FW_QSTATUS_RESP_NVM_OPTION_ACTION_STATUS_NVMOPT_ACTION_COLDBOOT 0x3UL
-+	#define FW_QSTATUS_RESP_NVM_OPTION_ACTION_STATUS_LAST                  FW_QSTATUS_RESP_NVM_OPTION_ACTION_STATUS_NVMOPT_ACTION_COLDBOOT
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_fw_set_time_input (size:256b/32B) */
-+struct hwrm_fw_set_time_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	year;
-+	#define FW_SET_TIME_REQ_YEAR_UNKNOWN 0x0UL
-+	#define FW_SET_TIME_REQ_YEAR_LAST   FW_SET_TIME_REQ_YEAR_UNKNOWN
-+	u8	month;
-+	u8	day;
-+	u8	hour;
-+	u8	minute;
-+	u8	second;
-+	u8	unused_0;
-+	__le16	millisecond;
-+	__le16	zone;
-+	#define FW_SET_TIME_REQ_ZONE_UTC     0
-+	#define FW_SET_TIME_REQ_ZONE_UNKNOWN 65535
-+	#define FW_SET_TIME_REQ_ZONE_LAST   FW_SET_TIME_REQ_ZONE_UNKNOWN
-+	u8	unused_1[4];
-+};
-+
-+/* hwrm_fw_set_time_output (size:128b/16B) */
-+struct hwrm_fw_set_time_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_struct_hdr (size:128b/16B) */
-+struct hwrm_struct_hdr {
-+	__le16	struct_id;
-+	#define STRUCT_HDR_STRUCT_ID_LLDP_CFG              0x41bUL
-+	#define STRUCT_HDR_STRUCT_ID_DCBX_ETS              0x41dUL
-+	#define STRUCT_HDR_STRUCT_ID_DCBX_PFC              0x41fUL
-+	#define STRUCT_HDR_STRUCT_ID_DCBX_APP              0x421UL
-+	#define STRUCT_HDR_STRUCT_ID_DCBX_FEATURE_STATE    0x422UL
-+	#define STRUCT_HDR_STRUCT_ID_LLDP_GENERIC          0x424UL
-+	#define STRUCT_HDR_STRUCT_ID_LLDP_DEVICE           0x426UL
-+	#define STRUCT_HDR_STRUCT_ID_POWER_BKUP            0x427UL
-+	#define STRUCT_HDR_STRUCT_ID_PEER_MMAP             0x429UL
-+	#define STRUCT_HDR_STRUCT_ID_AFM_OPAQUE            0x1UL
-+	#define STRUCT_HDR_STRUCT_ID_PORT_DESCRIPTION      0xaUL
-+	#define STRUCT_HDR_STRUCT_ID_RSS_V2                0x64UL
-+	#define STRUCT_HDR_STRUCT_ID_MSIX_PER_VF           0xc8UL
-+	#define STRUCT_HDR_STRUCT_ID_UDCC_RTT_BUCKET_COUNT 0x12cUL
-+	#define STRUCT_HDR_STRUCT_ID_UDCC_RTT_BUCKET_BOUND 0x12dUL
-+	#define STRUCT_HDR_STRUCT_ID_DBG_TOKEN_CLAIMS      0x190UL
-+	#define STRUCT_HDR_STRUCT_ID_LAST                 STRUCT_HDR_STRUCT_ID_DBG_TOKEN_CLAIMS
-+	__le16	len;
-+	u8	version;
-+	#define STRUCT_HDR_VERSION_0 0x0UL
-+	#define STRUCT_HDR_VERSION_1 0x1UL
-+	#define STRUCT_HDR_VERSION_LAST STRUCT_HDR_VERSION_1
-+	u8	count;
-+	__le16	subtype;
-+	__le16	next_offset;
-+	#define STRUCT_HDR_NEXT_OFFSET_LAST 0x0UL
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_struct_data_dcbx_app (size:64b/8B) */
-+struct hwrm_struct_data_dcbx_app {
-+	__be16	protocol_id;
-+	u8	protocol_selector;
-+	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_ETHER_TYPE   0x1UL
-+	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_TCP_PORT     0x2UL
-+	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_UDP_PORT     0x3UL
-+	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_TCP_UDP_PORT 0x4UL
-+	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_LAST        STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_TCP_UDP_PORT
-+	u8	priority;
-+	u8	valid;
-+	u8	unused_0[3];
-+};
-+
-+/* hwrm_fw_set_structured_data_input (size:256b/32B) */
-+struct hwrm_fw_set_structured_data_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	src_data_addr;
-+	__le16	data_len;
-+	u8	hdr_cnt;
-+	u8	unused_0[5];
-+};
-+
-+/* hwrm_fw_set_structured_data_output (size:128b/16B) */
-+struct hwrm_fw_set_structured_data_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_fw_set_structured_data_cmd_err (size:64b/8B) */
-+struct hwrm_fw_set_structured_data_cmd_err {
-+	u8	code;
-+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_UNKNOWN       0x0UL
-+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_HDR_CNT   0x1UL
-+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_FMT       0x2UL
-+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID        0x3UL
-+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_ALREADY_ADDED 0x4UL
-+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_INST_IN_PROG  0x5UL
-+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_LAST         FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_INST_IN_PROG
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_fw_get_structured_data_input (size:256b/32B) */
-+struct hwrm_fw_get_structured_data_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	dest_data_addr;
-+	__le16	data_len;
-+	__le16	structure_id;
-+	__le16	subtype;
-+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_UNUSED                  0x0UL
-+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_ALL                     0xffffUL
-+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NEAR_BRIDGE_ADMIN       0x100UL
-+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NEAR_BRIDGE_PEER        0x101UL
-+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NEAR_BRIDGE_OPERATIONAL 0x102UL
-+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NON_TPMR_ADMIN          0x200UL
-+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NON_TPMR_PEER           0x201UL
-+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NON_TPMR_OPERATIONAL    0x202UL
-+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_HOST_OPERATIONAL        0x300UL
-+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_CLAIMS_SUPPORTED        0x320UL
-+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_CLAIMS_ACTIVE           0x321UL
-+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_LAST                   FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_CLAIMS_ACTIVE
-+	u8	count;
-+	u8	unused_0;
-+};
-+
-+/* hwrm_fw_get_structured_data_output (size:128b/16B) */
-+struct hwrm_fw_get_structured_data_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	hdr_cnt;
-+	u8	unused_0[6];
-+	u8	valid;
-+};
-+
-+/* hwrm_fw_get_structured_data_cmd_err (size:64b/8B) */
-+struct hwrm_fw_get_structured_data_cmd_err {
-+	u8	code;
-+	#define FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_UNKNOWN 0x0UL
-+	#define FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID  0x3UL
-+	#define FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_LAST   FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_fw_livepatch_query_input (size:192b/24B) */
-+struct hwrm_fw_livepatch_query_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	fw_target;
-+	#define FW_LIVEPATCH_QUERY_REQ_FW_TARGET_COMMON_FW 0x1UL
-+	#define FW_LIVEPATCH_QUERY_REQ_FW_TARGET_SECURE_FW 0x2UL
-+	#define FW_LIVEPATCH_QUERY_REQ_FW_TARGET_LAST     FW_LIVEPATCH_QUERY_REQ_FW_TARGET_SECURE_FW
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_fw_livepatch_query_output (size:640b/80B) */
-+struct hwrm_fw_livepatch_query_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	char	install_ver[32];
-+	char	active_ver[32];
-+	__le16	status_flags;
-+	#define FW_LIVEPATCH_QUERY_RESP_STATUS_FLAGS_INSTALL     0x1UL
-+	#define FW_LIVEPATCH_QUERY_RESP_STATUS_FLAGS_ACTIVE      0x2UL
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_fw_livepatch_input (size:256b/32B) */
-+struct hwrm_fw_livepatch_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	opcode;
-+	#define FW_LIVEPATCH_REQ_OPCODE_ACTIVATE   0x1UL
-+	#define FW_LIVEPATCH_REQ_OPCODE_DEACTIVATE 0x2UL
-+	#define FW_LIVEPATCH_REQ_OPCODE_LAST      FW_LIVEPATCH_REQ_OPCODE_DEACTIVATE
-+	u8	fw_target;
-+	#define FW_LIVEPATCH_REQ_FW_TARGET_COMMON_FW 0x1UL
-+	#define FW_LIVEPATCH_REQ_FW_TARGET_SECURE_FW 0x2UL
-+	#define FW_LIVEPATCH_REQ_FW_TARGET_LAST     FW_LIVEPATCH_REQ_FW_TARGET_SECURE_FW
-+	u8	loadtype;
-+	#define FW_LIVEPATCH_REQ_LOADTYPE_NVM_INSTALL   0x1UL
-+	#define FW_LIVEPATCH_REQ_LOADTYPE_MEMORY_DIRECT 0x2UL
-+	#define FW_LIVEPATCH_REQ_LOADTYPE_LAST         FW_LIVEPATCH_REQ_LOADTYPE_MEMORY_DIRECT
-+	u8	flags;
-+	__le32	patch_len;
-+	__le64	host_addr;
-+};
-+
-+/* hwrm_fw_livepatch_output (size:128b/16B) */
-+struct hwrm_fw_livepatch_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_fw_livepatch_cmd_err (size:64b/8B) */
-+struct hwrm_fw_livepatch_cmd_err {
-+	u8	code;
-+	#define FW_LIVEPATCH_CMD_ERR_CODE_UNKNOWN         0x0UL
-+	#define FW_LIVEPATCH_CMD_ERR_CODE_INVALID_OPCODE  0x1UL
-+	#define FW_LIVEPATCH_CMD_ERR_CODE_INVALID_TARGET  0x2UL
-+	#define FW_LIVEPATCH_CMD_ERR_CODE_NOT_SUPPORTED   0x3UL
-+	#define FW_LIVEPATCH_CMD_ERR_CODE_NOT_INSTALLED   0x4UL
-+	#define FW_LIVEPATCH_CMD_ERR_CODE_NOT_PATCHED     0x5UL
-+	#define FW_LIVEPATCH_CMD_ERR_CODE_AUTH_FAIL       0x6UL
-+	#define FW_LIVEPATCH_CMD_ERR_CODE_INVALID_HEADER  0x7UL
-+	#define FW_LIVEPATCH_CMD_ERR_CODE_INVALID_SIZE    0x8UL
-+	#define FW_LIVEPATCH_CMD_ERR_CODE_ALREADY_PATCHED 0x9UL
-+	#define FW_LIVEPATCH_CMD_ERR_CODE_LAST           FW_LIVEPATCH_CMD_ERR_CODE_ALREADY_PATCHED
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_exec_fwd_resp_input (size:1024b/128B) */
-+struct hwrm_exec_fwd_resp_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	encap_request[26];
-+	__le16	encap_resp_target_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_exec_fwd_resp_output (size:128b/16B) */
-+struct hwrm_exec_fwd_resp_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_reject_fwd_resp_input (size:1024b/128B) */
-+struct hwrm_reject_fwd_resp_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	encap_request[26];
-+	__le16	encap_resp_target_id;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_reject_fwd_resp_output (size:128b/16B) */
-+struct hwrm_reject_fwd_resp_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_fwd_resp_input (size:1024b/128B) */
-+struct hwrm_fwd_resp_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	encap_resp_target_id;
-+	__le16	encap_resp_cmpl_ring;
-+	__le16	encap_resp_len;
-+	u8	unused_0;
-+	u8	unused_1;
-+	__le64	encap_resp_addr;
-+	__le32	encap_resp[24];
-+};
-+
-+/* hwrm_fwd_resp_output (size:128b/16B) */
-+struct hwrm_fwd_resp_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_fwd_async_event_cmpl_input (size:320b/40B) */
-+struct hwrm_fwd_async_event_cmpl_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	encap_async_event_target_id;
-+	u8	unused_0[6];
-+	__le32	encap_async_event_cmpl[4];
-+};
-+
-+/* hwrm_fwd_async_event_cmpl_output (size:128b/16B) */
-+struct hwrm_fwd_async_event_cmpl_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_temp_monitor_query_input (size:128b/16B) */
-+struct hwrm_temp_monitor_query_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+};
-+
-+/* hwrm_temp_monitor_query_output (size:192b/24B) */
-+struct hwrm_temp_monitor_query_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	temp;
-+	u8	phy_temp;
-+	u8	om_temp;
-+	u8	flags;
-+	#define TEMP_MONITOR_QUERY_RESP_FLAGS_TEMP_NOT_AVAILABLE             0x1UL
-+	#define TEMP_MONITOR_QUERY_RESP_FLAGS_PHY_TEMP_NOT_AVAILABLE         0x2UL
-+	#define TEMP_MONITOR_QUERY_RESP_FLAGS_OM_NOT_PRESENT                 0x4UL
-+	#define TEMP_MONITOR_QUERY_RESP_FLAGS_OM_TEMP_NOT_AVAILABLE          0x8UL
-+	#define TEMP_MONITOR_QUERY_RESP_FLAGS_EXT_TEMP_FIELDS_AVAILABLE      0x10UL
-+	#define TEMP_MONITOR_QUERY_RESP_FLAGS_THRESHOLD_VALUES_AVAILABLE     0x20UL
-+	u8	temp2;
-+	u8	phy_temp2;
-+	u8	om_temp2;
-+	u8	warn_threshold;
-+	u8	critical_threshold;
-+	u8	fatal_threshold;
-+	u8	shutdown_threshold;
-+	u8	unused_0[4];
-+	u8	valid;
-+};
-+
-+/* hwrm_wol_filter_alloc_input (size:512b/64B) */
-+struct hwrm_wol_filter_alloc_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	__le32	enables;
-+	#define WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS           0x1UL
-+	#define WOL_FILTER_ALLOC_REQ_ENABLES_PATTERN_OFFSET        0x2UL
-+	#define WOL_FILTER_ALLOC_REQ_ENABLES_PATTERN_BUF_SIZE      0x4UL
-+	#define WOL_FILTER_ALLOC_REQ_ENABLES_PATTERN_BUF_ADDR      0x8UL
-+	#define WOL_FILTER_ALLOC_REQ_ENABLES_PATTERN_MASK_ADDR     0x10UL
-+	#define WOL_FILTER_ALLOC_REQ_ENABLES_PATTERN_MASK_SIZE     0x20UL
-+	__le16	port_id;
-+	u8	wol_type;
-+	#define WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT 0x0UL
-+	#define WOL_FILTER_ALLOC_REQ_WOL_TYPE_BMP      0x1UL
-+	#define WOL_FILTER_ALLOC_REQ_WOL_TYPE_INVALID  0xffUL
-+	#define WOL_FILTER_ALLOC_REQ_WOL_TYPE_LAST    WOL_FILTER_ALLOC_REQ_WOL_TYPE_INVALID
-+	u8	unused_0[5];
-+	u8	mac_address[6];
-+	__le16	pattern_offset;
-+	__le16	pattern_buf_size;
-+	__le16	pattern_mask_size;
-+	u8	unused_1[4];
-+	__le64	pattern_buf_addr;
-+	__le64	pattern_mask_addr;
-+};
-+
-+/* hwrm_wol_filter_alloc_output (size:128b/16B) */
-+struct hwrm_wol_filter_alloc_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	wol_filter_id;
-+	u8	unused_0[6];
-+	u8	valid;
-+};
-+
-+/* hwrm_wol_filter_free_input (size:256b/32B) */
-+struct hwrm_wol_filter_free_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	flags;
-+	#define WOL_FILTER_FREE_REQ_FLAGS_FREE_ALL_WOL_FILTERS     0x1UL
-+	__le32	enables;
-+	#define WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID     0x1UL
-+	__le16	port_id;
-+	u8	wol_filter_id;
-+	u8	unused_0[5];
-+};
-+
-+/* hwrm_wol_filter_free_output (size:128b/16B) */
-+struct hwrm_wol_filter_free_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_wol_filter_qcfg_input (size:448b/56B) */
-+struct hwrm_wol_filter_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	__le16	handle;
-+	u8	unused_0[4];
-+	__le64	pattern_buf_addr;
-+	__le16	pattern_buf_size;
-+	u8	unused_1[6];
-+	__le64	pattern_mask_addr;
-+	__le16	pattern_mask_size;
-+	u8	unused_2[6];
-+};
-+
-+/* hwrm_wol_filter_qcfg_output (size:256b/32B) */
-+struct hwrm_wol_filter_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	next_handle;
-+	u8	wol_filter_id;
-+	u8	wol_type;
-+	#define WOL_FILTER_QCFG_RESP_WOL_TYPE_MAGICPKT 0x0UL
-+	#define WOL_FILTER_QCFG_RESP_WOL_TYPE_BMP      0x1UL
-+	#define WOL_FILTER_QCFG_RESP_WOL_TYPE_INVALID  0xffUL
-+	#define WOL_FILTER_QCFG_RESP_WOL_TYPE_LAST    WOL_FILTER_QCFG_RESP_WOL_TYPE_INVALID
-+	__le32	unused_0;
-+	u8	mac_address[6];
-+	__le16	pattern_offset;
-+	__le16	pattern_size;
-+	__le16	pattern_mask_size;
-+	u8	unused_1[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_wol_reason_qcfg_input (size:320b/40B) */
-+struct hwrm_wol_reason_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	port_id;
-+	u8	unused_0[6];
-+	__le64	wol_pkt_buf_addr;
-+	__le16	wol_pkt_buf_size;
-+	u8	unused_1[6];
-+};
-+
-+/* hwrm_wol_reason_qcfg_output (size:128b/16B) */
-+struct hwrm_wol_reason_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	wol_filter_id;
-+	u8	wol_reason;
-+	#define WOL_REASON_QCFG_RESP_WOL_REASON_MAGICPKT 0x0UL
-+	#define WOL_REASON_QCFG_RESP_WOL_REASON_BMP      0x1UL
-+	#define WOL_REASON_QCFG_RESP_WOL_REASON_INVALID  0xffUL
-+	#define WOL_REASON_QCFG_RESP_WOL_REASON_LAST    WOL_REASON_QCFG_RESP_WOL_REASON_INVALID
-+	u8	wol_pkt_len;
-+	u8	unused_0[4];
-+	u8	valid;
-+};
-+
-+/* hwrm_dbg_read_direct_input (size:256b/32B) */
-+struct hwrm_dbg_read_direct_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	host_dest_addr;
-+	__le32	read_addr;
-+	__le32	read_len32;
-+};
-+
-+/* hwrm_dbg_read_direct_output (size:128b/16B) */
-+struct hwrm_dbg_read_direct_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	crc32;
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_dbg_qcaps_input (size:192b/24B) */
-+struct hwrm_dbg_qcaps_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	fid;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_dbg_qcaps_output (size:192b/24B) */
-+struct hwrm_dbg_qcaps_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	fid;
-+	u8	unused_0[2];
-+	__le32	coredump_component_disable_caps;
-+	#define DBG_QCAPS_RESP_COREDUMP_COMPONENT_DISABLE_CAPS_NVRAM     0x1UL
-+	__le32	flags;
-+	#define DBG_QCAPS_RESP_FLAGS_CRASHDUMP_NVM             0x1UL
-+	#define DBG_QCAPS_RESP_FLAGS_CRASHDUMP_HOST_DDR        0x2UL
-+	#define DBG_QCAPS_RESP_FLAGS_CRASHDUMP_SOC_DDR         0x4UL
-+	#define DBG_QCAPS_RESP_FLAGS_USEQ                      0x8UL
-+	#define DBG_QCAPS_RESP_FLAGS_COREDUMP_HOST_DDR         0x10UL
-+	#define DBG_QCAPS_RESP_FLAGS_COREDUMP_HOST_CAPTURE     0x20UL
-+	#define DBG_QCAPS_RESP_FLAGS_PTRACE                    0x40UL
-+	#define DBG_QCAPS_RESP_FLAGS_REG_ACCESS_RESTRICTED     0x80UL
-+	u8	unused_1[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_dbg_qcfg_input (size:192b/24B) */
-+struct hwrm_dbg_qcfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	fid;
-+	__le16	flags;
-+	#define DBG_QCFG_REQ_FLAGS_CRASHDUMP_SIZE_FOR_DEST_MASK         0x3UL
-+	#define DBG_QCFG_REQ_FLAGS_CRASHDUMP_SIZE_FOR_DEST_SFT          0
-+	#define DBG_QCFG_REQ_FLAGS_CRASHDUMP_SIZE_FOR_DEST_DEST_NVM       0x0UL
-+	#define DBG_QCFG_REQ_FLAGS_CRASHDUMP_SIZE_FOR_DEST_DEST_HOST_DDR  0x1UL
-+	#define DBG_QCFG_REQ_FLAGS_CRASHDUMP_SIZE_FOR_DEST_DEST_SOC_DDR   0x2UL
-+	#define DBG_QCFG_REQ_FLAGS_CRASHDUMP_SIZE_FOR_DEST_LAST          DBG_QCFG_REQ_FLAGS_CRASHDUMP_SIZE_FOR_DEST_DEST_SOC_DDR
-+	__le32	coredump_component_disable_flags;
-+	#define DBG_QCFG_REQ_COREDUMP_COMPONENT_DISABLE_FLAGS_NVRAM     0x1UL
-+};
-+
-+/* hwrm_dbg_qcfg_output (size:256b/32B) */
-+struct hwrm_dbg_qcfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	fid;
-+	u8	unused_0[2];
-+	__le32	coredump_size;
-+	__le32	flags;
-+	#define DBG_QCFG_RESP_FLAGS_UART_LOG               0x1UL
-+	#define DBG_QCFG_RESP_FLAGS_UART_LOG_SECONDARY     0x2UL
-+	#define DBG_QCFG_RESP_FLAGS_FW_TRACE               0x4UL
-+	#define DBG_QCFG_RESP_FLAGS_FW_TRACE_SECONDARY     0x8UL
-+	#define DBG_QCFG_RESP_FLAGS_DEBUG_NOTIFY           0x10UL
-+	#define DBG_QCFG_RESP_FLAGS_JTAG_DEBUG             0x20UL
-+	__le16	async_cmpl_ring;
-+	u8	unused_2[2];
-+	__le32	crashdump_size;
-+	u8	unused_3[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_dbg_crashdump_medium_cfg_input (size:320b/40B) */
-+struct hwrm_dbg_crashdump_medium_cfg_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	output_dest_flags;
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_TYPE_DDR     0x1UL
-+	__le16	pg_size_lvl;
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_LVL_MASK      0x3UL
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_LVL_SFT       0
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_LVL_LVL_0       0x0UL
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_LVL_LVL_1       0x1UL
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_LVL_LVL_2       0x2UL
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_LVL_LAST       DBG_CRASHDUMP_MEDIUM_CFG_REQ_LVL_LVL_2
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_MASK  0x1cUL
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_SFT   2
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_4K   (0x0UL << 2)
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_8K   (0x1UL << 2)
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_64K  (0x2UL << 2)
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_2M   (0x3UL << 2)
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_8M   (0x4UL << 2)
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_1G   (0x5UL << 2)
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_LAST   DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_1G
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_UNUSED11_MASK 0xffe0UL
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_UNUSED11_SFT  5
-+	__le32	size;
-+	__le32	coredump_component_disable_flags;
-+	#define DBG_CRASHDUMP_MEDIUM_CFG_REQ_NVRAM     0x1UL
-+	__le32	unused_0;
-+	__le64	pbl;
-+};
-+
-+/* hwrm_dbg_crashdump_medium_cfg_output (size:128b/16B) */
-+struct hwrm_dbg_crashdump_medium_cfg_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_1[7];
-+	u8	valid;
-+};
-+
-+/* coredump_segment_record (size:128b/16B) */
-+struct coredump_segment_record {
-+	__le16	component_id;
-+	__le16	segment_id;
-+	__le16	max_instances;
-+	u8	version_hi;
-+	u8	version_low;
-+	u8	seg_flags;
-+	u8	compress_flags;
-+	#define SFLAG_COMPRESSED_ZLIB     0x1UL
-+	u8	unused_0[2];
-+	__le32	segment_len;
-+};
-+
-+/* hwrm_dbg_coredump_list_input (size:256b/32B) */
-+struct hwrm_dbg_coredump_list_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	host_dest_addr;
-+	__le32	host_buf_len;
-+	__le16	seq_no;
-+	u8	flags;
-+	#define DBG_COREDUMP_LIST_REQ_FLAGS_CRASHDUMP     0x1UL
-+	u8	unused_0[1];
-+};
-+
-+/* hwrm_dbg_coredump_list_output (size:128b/16B) */
-+struct hwrm_dbg_coredump_list_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	flags;
-+	#define DBG_COREDUMP_LIST_RESP_FLAGS_MORE     0x1UL
-+	u8	unused_0;
-+	__le16	total_segments;
-+	__le16	data_len;
-+	u8	unused_1;
-+	u8	valid;
-+};
-+
-+/* hwrm_dbg_coredump_initiate_input (size:256b/32B) */
-+struct hwrm_dbg_coredump_initiate_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	component_id;
-+	__le16	segment_id;
-+	__le16	instance;
-+	__le16	unused_0;
-+	u8	seg_flags;
-+	#define DBG_COREDUMP_INITIATE_REQ_SEG_FLAGS_LIVE_DATA                0x1UL
-+	#define DBG_COREDUMP_INITIATE_REQ_SEG_FLAGS_CRASH_DATA               0x2UL
-+	#define DBG_COREDUMP_INITIATE_REQ_SEG_FLAGS_COLLECT_CTX_L1_CACHE     0x4UL
-+	u8	unused_1[7];
-+};
-+
-+/* hwrm_dbg_coredump_initiate_output (size:128b/16B) */
-+struct hwrm_dbg_coredump_initiate_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* coredump_data_hdr (size:128b/16B) */
-+struct coredump_data_hdr {
-+	__le32	address;
-+	__le32	flags_length;
-+	#define COREDUMP_DATA_HDR_FLAGS_LENGTH_ACTUAL_LEN_MASK     0xffffffUL
-+	#define COREDUMP_DATA_HDR_FLAGS_LENGTH_ACTUAL_LEN_SFT      0
-+	#define COREDUMP_DATA_HDR_FLAGS_LENGTH_INDIRECT_ACCESS     0x1000000UL
-+	__le32	instance;
-+	__le32	next_offset;
-+};
-+
-+/* hwrm_dbg_coredump_retrieve_input (size:448b/56B) */
-+struct hwrm_dbg_coredump_retrieve_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	host_dest_addr;
-+	__le32	host_buf_len;
-+	__le32	unused_0;
-+	__le16	component_id;
-+	__le16	segment_id;
-+	__le16	instance;
-+	__le16	unused_1;
-+	u8	seg_flags;
-+	u8	unused_2;
-+	__le16	unused_3;
-+	__le32	unused_4;
-+	__le32	seq_no;
-+	__le32	unused_5;
-+};
-+
-+/* hwrm_dbg_coredump_retrieve_output (size:128b/16B) */
-+struct hwrm_dbg_coredump_retrieve_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	flags;
-+	#define DBG_COREDUMP_RETRIEVE_RESP_FLAGS_MORE     0x1UL
-+	u8	unused_0;
-+	__le16	data_len;
-+	u8	unused_1[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_dbg_ring_info_get_input (size:192b/24B) */
-+struct hwrm_dbg_ring_info_get_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	ring_type;
-+	#define DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL 0x0UL
-+	#define DBG_RING_INFO_GET_REQ_RING_TYPE_TX      0x1UL
-+	#define DBG_RING_INFO_GET_REQ_RING_TYPE_RX      0x2UL
-+	#define DBG_RING_INFO_GET_REQ_RING_TYPE_NQ      0x3UL
-+	#define DBG_RING_INFO_GET_REQ_RING_TYPE_LAST   DBG_RING_INFO_GET_REQ_RING_TYPE_NQ
-+	u8	unused_0[3];
-+	__le32	fw_ring_id;
-+};
-+
-+/* hwrm_dbg_ring_info_get_output (size:192b/24B) */
-+struct hwrm_dbg_ring_info_get_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	producer_index;
-+	__le32	consumer_index;
-+	__le32	cag_vector_ctrl;
-+	__le16	st_tag;
-+	u8	unused_0;
-+	u8	valid;
-+};
-+
-+/* hwrm_dbg_log_buffer_flush_input (size:192b/24B) */
-+struct hwrm_dbg_log_buffer_flush_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	type;
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_SRT_TRACE           0x0UL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_SRT2_TRACE          0x1UL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CRT_TRACE           0x2UL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CRT2_TRACE          0x3UL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_RIGP0_TRACE         0x4UL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_L2_HWRM_TRACE       0x5UL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_ROCE_HWRM_TRACE     0x6UL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CA0_TRACE           0x7UL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CA1_TRACE           0x8UL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CA2_TRACE           0x9UL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_RIGP1_TRACE         0xaUL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_AFM_KONG_HWRM_TRACE 0xbUL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_ERR_QPC_TRACE       0xcUL
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_TYPE_LAST               DBG_LOG_BUFFER_FLUSH_REQ_TYPE_ERR_QPC_TRACE
-+	u8	unused_1[2];
-+	__le32	flags;
-+	#define DBG_LOG_BUFFER_FLUSH_REQ_FLAGS_FLUSH_ALL_BUFFERS     0x1UL
-+};
-+
-+/* hwrm_dbg_log_buffer_flush_output (size:128b/16B) */
-+struct hwrm_dbg_log_buffer_flush_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	current_buffer_offset;
-+	u8	unused_1[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_read_input (size:320b/40B) */
-+struct hwrm_nvm_read_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	host_dest_addr;
-+	__le16	dir_idx;
-+	u8	unused_0[2];
-+	__le32	offset;
-+	__le32	len;
-+	u8	unused_1[4];
-+};
-+
-+/* hwrm_nvm_read_output (size:128b/16B) */
-+struct hwrm_nvm_read_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_get_dir_entries_input (size:192b/24B) */
-+struct hwrm_nvm_get_dir_entries_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	host_dest_addr;
-+};
-+
-+/* hwrm_nvm_get_dir_entries_output (size:128b/16B) */
-+struct hwrm_nvm_get_dir_entries_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_get_dir_info_input (size:128b/16B) */
-+struct hwrm_nvm_get_dir_info_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+};
-+
-+/* hwrm_nvm_get_dir_info_output (size:192b/24B) */
-+struct hwrm_nvm_get_dir_info_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	entries;
-+	__le32	entry_length;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_write_input (size:448b/56B) */
-+struct hwrm_nvm_write_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	host_src_addr;
-+	__le16	dir_type;
-+	__le16	dir_ordinal;
-+	__le16	dir_ext;
-+	__le16	dir_attr;
-+	__le32	dir_data_length;
-+	__le16	option;
-+	__le16	flags;
-+	#define NVM_WRITE_REQ_FLAGS_KEEP_ORIG_ACTIVE_IMG     0x1UL
-+	#define NVM_WRITE_REQ_FLAGS_BATCH_MODE               0x2UL
-+	#define NVM_WRITE_REQ_FLAGS_BATCH_LAST               0x4UL
-+	#define NVM_WRITE_REQ_FLAGS_SKIP_CRID_CHECK          0x8UL
-+	__le32	dir_item_length;
-+	__le32	offset;
-+	__le32	len;
-+	__le32	unused_0;
-+};
-+
-+/* hwrm_nvm_write_output (size:128b/16B) */
-+struct hwrm_nvm_write_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	dir_item_length;
-+	__le16	dir_idx;
-+	u8	unused_0;
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_write_cmd_err (size:64b/8B) */
-+struct hwrm_nvm_write_cmd_err {
-+	u8	code;
-+	#define NVM_WRITE_CMD_ERR_CODE_UNKNOWN              0x0UL
-+	#define NVM_WRITE_CMD_ERR_CODE_FRAG_ERR             0x1UL
-+	#define NVM_WRITE_CMD_ERR_CODE_NO_SPACE             0x2UL
-+	#define NVM_WRITE_CMD_ERR_CODE_WRITE_FAILED         0x3UL
-+	#define NVM_WRITE_CMD_ERR_CODE_REQD_ERASE_FAILED    0x4UL
-+	#define NVM_WRITE_CMD_ERR_CODE_VERIFY_FAILED        0x5UL
-+	#define NVM_WRITE_CMD_ERR_CODE_INVALID_HEADER       0x6UL
-+	#define NVM_WRITE_CMD_ERR_CODE_UPDATE_DIGEST_FAILED 0x7UL
-+	#define NVM_WRITE_CMD_ERR_CODE_LAST                NVM_WRITE_CMD_ERR_CODE_UPDATE_DIGEST_FAILED
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_nvm_modify_input (size:320b/40B) */
-+struct hwrm_nvm_modify_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	host_src_addr;
-+	__le16	dir_idx;
-+	__le16	flags;
-+	#define NVM_MODIFY_REQ_FLAGS_BATCH_MODE     0x1UL
-+	#define NVM_MODIFY_REQ_FLAGS_BATCH_LAST     0x2UL
-+	__le32	offset;
-+	__le32	len;
-+	u8	unused_1[4];
-+};
-+
-+/* hwrm_nvm_modify_output (size:128b/16B) */
-+struct hwrm_nvm_modify_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_find_dir_entry_input (size:256b/32B) */
-+struct hwrm_nvm_find_dir_entry_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	#define NVM_FIND_DIR_ENTRY_REQ_ENABLES_DIR_IDX_VALID     0x1UL
-+	__le16	dir_idx;
-+	__le16	dir_type;
-+	__le16	dir_ordinal;
-+	__le16	dir_ext;
-+	u8	opt_ordinal;
-+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_MASK 0x3UL
-+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_SFT 0
-+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_EQ    0x0UL
-+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_GE    0x1UL
-+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_GT    0x2UL
-+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_LAST NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_GT
-+	u8	unused_0[3];
-+};
-+
-+/* hwrm_nvm_find_dir_entry_output (size:256b/32B) */
-+struct hwrm_nvm_find_dir_entry_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le32	dir_item_length;
-+	__le32	dir_data_length;
-+	__le32	fw_ver;
-+	__le16	dir_ordinal;
-+	__le16	dir_idx;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_erase_dir_entry_input (size:192b/24B) */
-+struct hwrm_nvm_erase_dir_entry_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	dir_idx;
-+	u8	unused_0[6];
-+};
-+
-+/* hwrm_nvm_erase_dir_entry_output (size:128b/16B) */
-+struct hwrm_nvm_erase_dir_entry_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_get_dev_info_input (size:192b/24B) */
-+struct hwrm_nvm_get_dev_info_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	flags;
-+	#define NVM_GET_DEV_INFO_REQ_FLAGS_SECURITY_SOC_NVM     0x1UL
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_nvm_get_dev_info_output (size:768b/96B) */
-+struct hwrm_nvm_get_dev_info_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	manufacturer_id;
-+	__le16	device_id;
-+	__le32	sector_size;
-+	__le32	nvram_size;
-+	__le32	reserved_size;
-+	__le32	available_size;
-+	u8	nvm_cfg_ver_maj;
-+	u8	nvm_cfg_ver_min;
-+	u8	nvm_cfg_ver_upd;
-+	u8	flags;
-+	#define NVM_GET_DEV_INFO_RESP_FLAGS_FW_VER_VALID     0x1UL
-+	char	pkg_name[16];
-+	__le16	hwrm_fw_major;
-+	__le16	hwrm_fw_minor;
-+	__le16	hwrm_fw_build;
-+	__le16	hwrm_fw_patch;
-+	__le16	mgmt_fw_major;
-+	__le16	mgmt_fw_minor;
-+	__le16	mgmt_fw_build;
-+	__le16	mgmt_fw_patch;
-+	__le16	roce_fw_major;
-+	__le16	roce_fw_minor;
-+	__le16	roce_fw_build;
-+	__le16	roce_fw_patch;
-+	__le16	netctrl_fw_major;
-+	__le16	netctrl_fw_minor;
-+	__le16	netctrl_fw_build;
-+	__le16	netctrl_fw_patch;
-+	__le16	srt2_fw_major;
-+	__le16	srt2_fw_minor;
-+	__le16	srt2_fw_build;
-+	__le16	srt2_fw_patch;
-+	u8	security_soc_fw_major;
-+	u8	security_soc_fw_minor;
-+	u8	security_soc_fw_build;
-+	u8	security_soc_fw_patch;
-+	u8	unused_0[3];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_mod_dir_entry_input (size:256b/32B) */
-+struct hwrm_nvm_mod_dir_entry_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	enables;
-+	#define NVM_MOD_DIR_ENTRY_REQ_ENABLES_CHECKSUM     0x1UL
-+	__le16	dir_idx;
-+	__le16	dir_ordinal;
-+	__le16	dir_ext;
-+	__le16	dir_attr;
-+	__le32	checksum;
-+};
-+
-+/* hwrm_nvm_mod_dir_entry_output (size:128b/16B) */
-+struct hwrm_nvm_mod_dir_entry_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_verify_update_input (size:192b/24B) */
-+struct hwrm_nvm_verify_update_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le16	dir_type;
-+	__le16	dir_ordinal;
-+	__le16	dir_ext;
-+	u8	unused_0[2];
-+};
-+
-+/* hwrm_nvm_verify_update_output (size:128b/16B) */
-+struct hwrm_nvm_verify_update_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_install_update_input (size:192b/24B) */
-+struct hwrm_nvm_install_update_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le32	install_type;
-+	#define NVM_INSTALL_UPDATE_REQ_INSTALL_TYPE_NORMAL 0x0UL
-+	#define NVM_INSTALL_UPDATE_REQ_INSTALL_TYPE_ALL    0xffffffffUL
-+	#define NVM_INSTALL_UPDATE_REQ_INSTALL_TYPE_LAST  NVM_INSTALL_UPDATE_REQ_INSTALL_TYPE_ALL
-+	__le16	flags;
-+	#define NVM_INSTALL_UPDATE_REQ_FLAGS_ERASE_UNUSED_SPACE     0x1UL
-+	#define NVM_INSTALL_UPDATE_REQ_FLAGS_REMOVE_UNUSED_PKG      0x2UL
-+	#define NVM_INSTALL_UPDATE_REQ_FLAGS_ALLOWED_TO_DEFRAG      0x4UL
-+	#define NVM_INSTALL_UPDATE_REQ_FLAGS_VERIFY_ONLY            0x8UL
-+	u8	unused_0[2];
-+};
-+
-+/* hwrm_nvm_install_update_output (size:192b/24B) */
-+struct hwrm_nvm_install_update_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le64	installed_items;
-+	u8	result;
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_SUCCESS                      0x0UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_FAILURE                      0xffUL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_MALLOC_FAILURE               0xfdUL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_INDEX_PARAMETER      0xfbUL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_TYPE_PARAMETER       0xf3UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_PREREQUISITE         0xf2UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_FILE_HEADER          0xecUL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_SIGNATURE            0xebUL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_PROP_STREAM          0xeaUL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_PROP_LENGTH          0xe9UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_MANIFEST             0xe8UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_TRAILER              0xe7UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_CHECKSUM             0xe6UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_ITEM_CHECKSUM        0xe5UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_DATA_LENGTH          0xe4UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INVALID_DIRECTIVE            0xe1UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_UNSUPPORTED_CHIP_REV         0xceUL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_UNSUPPORTED_DEVICE_ID        0xcdUL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_UNSUPPORTED_SUBSYS_VENDOR    0xccUL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_UNSUPPORTED_SUBSYS_ID        0xcbUL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_UNSUPPORTED_PLATFORM         0xc5UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_DUPLICATE_ITEM               0xc4UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_ZERO_LENGTH_ITEM             0xc3UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INSTALL_CHECKSUM_ERROR       0xb9UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INSTALL_DATA_ERROR           0xb8UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_INSTALL_AUTHENTICATION_ERROR 0xb7UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_ITEM_NOT_FOUND               0xb0UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_ITEM_LOCKED                  0xa7UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESULT_LAST                        NVM_INSTALL_UPDATE_RESP_RESULT_ITEM_LOCKED
-+	u8	problem_item;
-+	#define NVM_INSTALL_UPDATE_RESP_PROBLEM_ITEM_NONE    0x0UL
-+	#define NVM_INSTALL_UPDATE_RESP_PROBLEM_ITEM_PACKAGE 0xffUL
-+	#define NVM_INSTALL_UPDATE_RESP_PROBLEM_ITEM_LAST   NVM_INSTALL_UPDATE_RESP_PROBLEM_ITEM_PACKAGE
-+	u8	reset_required;
-+	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_NONE  0x0UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_PCI   0x1UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_POWER 0x2UL
-+	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_LAST NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_POWER
-+	u8	unused_0[4];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_install_update_cmd_err (size:64b/8B) */
-+struct hwrm_nvm_install_update_cmd_err {
-+	u8	code;
-+	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_UNKNOWN            0x0UL
-+	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_FRAG_ERR           0x1UL
-+	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_NO_SPACE           0x2UL
-+	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_ANTI_ROLLBACK      0x3UL
-+	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_NO_VOLTREG_SUPPORT 0x4UL
-+	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_DEFRAG_FAILED      0x5UL
-+	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_UNKNOWN_DIR_ERR    0x6UL
-+	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_LAST              NVM_INSTALL_UPDATE_CMD_ERR_CODE_UNKNOWN_DIR_ERR
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_nvm_get_variable_input (size:320b/40B) */
-+struct hwrm_nvm_get_variable_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	dest_data_addr;
-+	__le16	data_len;
-+	__le16	option_num;
-+	#define NVM_GET_VARIABLE_REQ_OPTION_NUM_RSVD_0    0x0UL
-+	#define NVM_GET_VARIABLE_REQ_OPTION_NUM_RSVD_FFFF 0xffffUL
-+	#define NVM_GET_VARIABLE_REQ_OPTION_NUM_LAST     NVM_GET_VARIABLE_REQ_OPTION_NUM_RSVD_FFFF
-+	__le16	dimensions;
-+	__le16	index_0;
-+	__le16	index_1;
-+	__le16	index_2;
-+	__le16	index_3;
-+	u8	flags;
-+	#define NVM_GET_VARIABLE_REQ_FLAGS_FACTORY_DFLT           0x1UL
-+	#define NVM_GET_VARIABLE_REQ_FLAGS_VALIDATE_OPT_VALUE     0x2UL
-+	u8	unused_0;
-+};
-+
-+/* hwrm_nvm_get_variable_output (size:128b/16B) */
-+struct hwrm_nvm_get_variable_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	__le16	data_len;
-+	__le16	option_num;
-+	#define NVM_GET_VARIABLE_RESP_OPTION_NUM_RSVD_0    0x0UL
-+	#define NVM_GET_VARIABLE_RESP_OPTION_NUM_RSVD_FFFF 0xffffUL
-+	#define NVM_GET_VARIABLE_RESP_OPTION_NUM_LAST     NVM_GET_VARIABLE_RESP_OPTION_NUM_RSVD_FFFF
-+	u8	flags;
-+	#define NVM_GET_VARIABLE_RESP_FLAGS_VALIDATE_OPT_VALUE     0x1UL
-+	u8	unused_0[2];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_get_variable_cmd_err (size:64b/8B) */
-+struct hwrm_nvm_get_variable_cmd_err {
-+	u8	code;
-+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_UNKNOWN          0x0UL
-+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_VAR_NOT_EXIST    0x1UL
-+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_CORRUPT_VAR      0x2UL
-+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_LEN_TOO_SHORT    0x3UL
-+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_INDEX_INVALID    0x4UL
-+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_ACCESS_DENIED    0x5UL
-+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_CB_FAILED        0x6UL
-+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_INVALID_DATA_LEN 0x7UL
-+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_NO_MEM           0x8UL
-+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_LAST            NVM_GET_VARIABLE_CMD_ERR_CODE_NO_MEM
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_nvm_set_variable_input (size:320b/40B) */
-+struct hwrm_nvm_set_variable_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	__le64	src_data_addr;
-+	__le16	data_len;
-+	__le16	option_num;
-+	#define NVM_SET_VARIABLE_REQ_OPTION_NUM_RSVD_0    0x0UL
-+	#define NVM_SET_VARIABLE_REQ_OPTION_NUM_RSVD_FFFF 0xffffUL
-+	#define NVM_SET_VARIABLE_REQ_OPTION_NUM_LAST     NVM_SET_VARIABLE_REQ_OPTION_NUM_RSVD_FFFF
-+	__le16	dimensions;
-+	__le16	index_0;
-+	__le16	index_1;
-+	__le16	index_2;
-+	__le16	index_3;
-+	u8	flags;
-+	#define NVM_SET_VARIABLE_REQ_FLAGS_FORCE_FLUSH                0x1UL
-+	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_MASK          0xeUL
-+	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_SFT           1
-+	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_NONE            (0x0UL << 1)
-+	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_HMAC_SHA1       (0x1UL << 1)
-+	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_AES256          (0x2UL << 1)
-+	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_HMAC_SHA1_AUTH  (0x3UL << 1)
-+	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_LAST           NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_HMAC_SHA1_AUTH
-+	#define NVM_SET_VARIABLE_REQ_FLAGS_FLAGS_UNUSED_0_MASK        0x70UL
-+	#define NVM_SET_VARIABLE_REQ_FLAGS_FLAGS_UNUSED_0_SFT         4
-+	#define NVM_SET_VARIABLE_REQ_FLAGS_FACTORY_DEFAULT            0x80UL
-+	u8	unused_0;
-+};
-+
-+/* hwrm_nvm_set_variable_output (size:128b/16B) */
-+struct hwrm_nvm_set_variable_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* hwrm_nvm_set_variable_cmd_err (size:64b/8B) */
-+struct hwrm_nvm_set_variable_cmd_err {
-+	u8	code;
-+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_UNKNOWN              0x0UL
-+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_VAR_NOT_EXIST        0x1UL
-+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_CORRUPT_VAR          0x2UL
-+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_LEN_TOO_SHORT        0x3UL
-+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_ACTION_NOT_SUPPORTED 0x4UL
-+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_INDEX_INVALID        0x5UL
-+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_ACCESS_DENIED        0x6UL
-+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_CB_FAILED            0x7UL
-+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_INVALID_DATA_LEN     0x8UL
-+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_NO_MEM               0x9UL
-+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_LAST                NVM_SET_VARIABLE_CMD_ERR_CODE_NO_MEM
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_selftest_qlist_input (size:128b/16B) */
-+struct hwrm_selftest_qlist_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+};
-+
-+/* hwrm_selftest_qlist_output (size:2240b/280B) */
-+struct hwrm_selftest_qlist_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	num_tests;
-+	u8	available_tests;
-+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_NVM_TEST                 0x1UL
-+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_LINK_TEST                0x2UL
-+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_REGISTER_TEST            0x4UL
-+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_MEMORY_TEST              0x8UL
-+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_PCIE_SERDES_TEST         0x10UL
-+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_ETHERNET_SERDES_TEST     0x20UL
-+	u8	offline_tests;
-+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_NVM_TEST                 0x1UL
-+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_LINK_TEST                0x2UL
-+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_REGISTER_TEST            0x4UL
-+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_MEMORY_TEST              0x8UL
-+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_PCIE_SERDES_TEST         0x10UL
-+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_ETHERNET_SERDES_TEST     0x20UL
-+	u8	unused_0;
-+	__le16	test_timeout;
-+	u8	unused_1[2];
-+	char	test_name[8][32];
-+	u8	eyescope_target_BER_support;
-+	#define SELFTEST_QLIST_RESP_EYESCOPE_TARGET_BER_SUPPORT_BER_1E8_SUPPORTED  0x0UL
-+	#define SELFTEST_QLIST_RESP_EYESCOPE_TARGET_BER_SUPPORT_BER_1E9_SUPPORTED  0x1UL
-+	#define SELFTEST_QLIST_RESP_EYESCOPE_TARGET_BER_SUPPORT_BER_1E10_SUPPORTED 0x2UL
-+	#define SELFTEST_QLIST_RESP_EYESCOPE_TARGET_BER_SUPPORT_BER_1E11_SUPPORTED 0x3UL
-+	#define SELFTEST_QLIST_RESP_EYESCOPE_TARGET_BER_SUPPORT_BER_1E12_SUPPORTED 0x4UL
-+	#define SELFTEST_QLIST_RESP_EYESCOPE_TARGET_BER_SUPPORT_LAST              SELFTEST_QLIST_RESP_EYESCOPE_TARGET_BER_SUPPORT_BER_1E12_SUPPORTED
-+	u8	unused_2[6];
-+	u8	valid;
-+};
-+
-+/* hwrm_selftest_exec_input (size:192b/24B) */
-+struct hwrm_selftest_exec_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+	u8	flags;
-+	#define SELFTEST_EXEC_REQ_FLAGS_NVM_TEST                 0x1UL
-+	#define SELFTEST_EXEC_REQ_FLAGS_LINK_TEST                0x2UL
-+	#define SELFTEST_EXEC_REQ_FLAGS_REGISTER_TEST            0x4UL
-+	#define SELFTEST_EXEC_REQ_FLAGS_MEMORY_TEST              0x8UL
-+	#define SELFTEST_EXEC_REQ_FLAGS_PCIE_SERDES_TEST         0x10UL
-+	#define SELFTEST_EXEC_REQ_FLAGS_ETHERNET_SERDES_TEST     0x20UL
-+	u8	unused_0[7];
-+};
-+
-+/* hwrm_selftest_exec_output (size:128b/16B) */
-+struct hwrm_selftest_exec_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	requested_tests;
-+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_NVM_TEST                 0x1UL
-+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_LINK_TEST                0x2UL
-+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_REGISTER_TEST            0x4UL
-+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_MEMORY_TEST              0x8UL
-+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_PCIE_SERDES_TEST         0x10UL
-+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_ETHERNET_SERDES_TEST     0x20UL
-+	u8	test_success;
-+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_NVM_TEST                 0x1UL
-+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_LINK_TEST                0x2UL
-+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_REGISTER_TEST            0x4UL
-+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_MEMORY_TEST              0x8UL
-+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_PCIE_SERDES_TEST         0x10UL
-+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_ETHERNET_SERDES_TEST     0x20UL
-+	u8	unused_0[5];
-+	u8	valid;
-+};
-+
-+/* hwrm_selftest_irq_input (size:128b/16B) */
-+struct hwrm_selftest_irq_input {
-+	__le16	req_type;
-+	__le16	cmpl_ring;
-+	__le16	seq_id;
-+	__le16	target_id;
-+	__le64	resp_addr;
-+};
-+
-+/* hwrm_selftest_irq_output (size:128b/16B) */
-+struct hwrm_selftest_irq_output {
-+	__le16	error_code;
-+	__le16	req_type;
-+	__le16	seq_id;
-+	__le16	resp_len;
-+	u8	unused_0[7];
-+	u8	valid;
-+};
-+
-+/* dbc_dbc (size:64b/8B) */
-+struct dbc_dbc {
-+	__le32	index;
-+	#define DBC_DBC_INDEX_MASK 0xffffffUL
-+	#define DBC_DBC_INDEX_SFT  0
-+	#define DBC_DBC_EPOCH      0x1000000UL
-+	#define DBC_DBC_TOGGLE_MASK 0x6000000UL
-+	#define DBC_DBC_TOGGLE_SFT 25
-+	__le32	type_path_xid;
-+	#define DBC_DBC_XID_MASK          0xfffffUL
-+	#define DBC_DBC_XID_SFT           0
-+	#define DBC_DBC_PATH_MASK         0x3000000UL
-+	#define DBC_DBC_PATH_SFT          24
-+	#define DBC_DBC_PATH_ROCE           (0x0UL << 24)
-+	#define DBC_DBC_PATH_L2             (0x1UL << 24)
-+	#define DBC_DBC_PATH_ENGINE         (0x2UL << 24)
-+	#define DBC_DBC_PATH_LAST          DBC_DBC_PATH_ENGINE
-+	#define DBC_DBC_VALID             0x4000000UL
-+	#define DBC_DBC_DEBUG_TRACE       0x8000000UL
-+	#define DBC_DBC_TYPE_MASK         0xf0000000UL
-+	#define DBC_DBC_TYPE_SFT          28
-+	#define DBC_DBC_TYPE_SQ             (0x0UL << 28)
-+	#define DBC_DBC_TYPE_RQ             (0x1UL << 28)
-+	#define DBC_DBC_TYPE_SRQ            (0x2UL << 28)
-+	#define DBC_DBC_TYPE_SRQ_ARM        (0x3UL << 28)
-+	#define DBC_DBC_TYPE_CQ             (0x4UL << 28)
-+	#define DBC_DBC_TYPE_CQ_ARMSE       (0x5UL << 28)
-+	#define DBC_DBC_TYPE_CQ_ARMALL      (0x6UL << 28)
-+	#define DBC_DBC_TYPE_CQ_ARMENA      (0x7UL << 28)
-+	#define DBC_DBC_TYPE_SRQ_ARMENA     (0x8UL << 28)
-+	#define DBC_DBC_TYPE_CQ_CUTOFF_ACK  (0x9UL << 28)
-+	#define DBC_DBC_TYPE_NQ             (0xaUL << 28)
-+	#define DBC_DBC_TYPE_NQ_ARM         (0xbUL << 28)
-+	#define DBC_DBC_TYPE_NQ_MASK        (0xeUL << 28)
-+	#define DBC_DBC_TYPE_NULL           (0xfUL << 28)
-+	#define DBC_DBC_TYPE_LAST          DBC_DBC_TYPE_NULL
-+};
-+
-+/* db_push_start (size:64b/8B) */
-+struct db_push_start {
-+	u64	db;
-+	#define DB_PUSH_START_DB_INDEX_MASK     0xffffffUL
-+	#define DB_PUSH_START_DB_INDEX_SFT      0
-+	#define DB_PUSH_START_DB_PI_LO_MASK     0xff000000UL
-+	#define DB_PUSH_START_DB_PI_LO_SFT      24
-+	#define DB_PUSH_START_DB_XID_MASK       0xfffff00000000ULL
-+	#define DB_PUSH_START_DB_XID_SFT        32
-+	#define DB_PUSH_START_DB_PI_HI_MASK     0xf0000000000000ULL
-+	#define DB_PUSH_START_DB_PI_HI_SFT      52
-+	#define DB_PUSH_START_DB_TYPE_MASK      0xf000000000000000ULL
-+	#define DB_PUSH_START_DB_TYPE_SFT       60
-+	#define DB_PUSH_START_DB_TYPE_PUSH_START  (0xcULL << 60)
-+	#define DB_PUSH_START_DB_TYPE_PUSH_END    (0xdULL << 60)
-+	#define DB_PUSH_START_DB_TYPE_LAST       DB_PUSH_START_DB_TYPE_PUSH_END
-+};
-+
-+/* db_push_end (size:64b/8B) */
-+struct db_push_end {
-+	u64	db;
-+	#define DB_PUSH_END_DB_INDEX_MASK      0xffffffUL
-+	#define DB_PUSH_END_DB_INDEX_SFT       0
-+	#define DB_PUSH_END_DB_PI_LO_MASK      0xff000000UL
-+	#define DB_PUSH_END_DB_PI_LO_SFT       24
-+	#define DB_PUSH_END_DB_XID_MASK        0xfffff00000000ULL
-+	#define DB_PUSH_END_DB_XID_SFT         32
-+	#define DB_PUSH_END_DB_PI_HI_MASK      0xf0000000000000ULL
-+	#define DB_PUSH_END_DB_PI_HI_SFT       52
-+	#define DB_PUSH_END_DB_PATH_MASK       0x300000000000000ULL
-+	#define DB_PUSH_END_DB_PATH_SFT        56
-+	#define DB_PUSH_END_DB_PATH_ROCE         (0x0ULL << 56)
-+	#define DB_PUSH_END_DB_PATH_L2           (0x1ULL << 56)
-+	#define DB_PUSH_END_DB_PATH_ENGINE       (0x2ULL << 56)
-+	#define DB_PUSH_END_DB_PATH_LAST        DB_PUSH_END_DB_PATH_ENGINE
-+	#define DB_PUSH_END_DB_DEBUG_TRACE     0x800000000000000ULL
-+	#define DB_PUSH_END_DB_TYPE_MASK       0xf000000000000000ULL
-+	#define DB_PUSH_END_DB_TYPE_SFT        60
-+	#define DB_PUSH_END_DB_TYPE_PUSH_START   (0xcULL << 60)
-+	#define DB_PUSH_END_DB_TYPE_PUSH_END     (0xdULL << 60)
-+	#define DB_PUSH_END_DB_TYPE_LAST        DB_PUSH_END_DB_TYPE_PUSH_END
-+};
-+
-+/* db_push_info (size:64b/8B) */
-+struct db_push_info {
-+	u32	push_size_push_index;
-+	#define DB_PUSH_INFO_PUSH_INDEX_MASK 0xffffffUL
-+	#define DB_PUSH_INFO_PUSH_INDEX_SFT 0
-+	#define DB_PUSH_INFO_PUSH_SIZE_MASK 0x1f000000UL
-+	#define DB_PUSH_INFO_PUSH_SIZE_SFT  24
-+	u32	reserved32;
-+};
-+
-+/* fw_status_reg (size:32b/4B) */
-+struct fw_status_reg {
-+	u32	fw_status;
-+	#define FW_STATUS_REG_CODE_MASK              0xffffUL
-+	#define FW_STATUS_REG_CODE_SFT               0
-+	#define FW_STATUS_REG_CODE_READY               0x8000UL
-+	#define FW_STATUS_REG_CODE_LAST               FW_STATUS_REG_CODE_READY
-+	#define FW_STATUS_REG_IMAGE_DEGRADED         0x10000UL
-+	#define FW_STATUS_REG_RECOVERABLE            0x20000UL
-+	#define FW_STATUS_REG_CRASHDUMP_ONGOING      0x40000UL
-+	#define FW_STATUS_REG_CRASHDUMP_COMPLETE     0x80000UL
-+	#define FW_STATUS_REG_SHUTDOWN               0x100000UL
-+	#define FW_STATUS_REG_CRASHED_NO_MASTER      0x200000UL
-+	#define FW_STATUS_REG_RECOVERING             0x400000UL
-+	#define FW_STATUS_REG_MANU_DEBUG_STATUS      0x800000UL
-+};
-+
-+/* hcomm_status (size:64b/8B) */
-+struct hcomm_status {
-+	u32	sig_ver;
-+	#define HCOMM_STATUS_VER_MASK      0xffUL
-+	#define HCOMM_STATUS_VER_SFT       0
-+	#define HCOMM_STATUS_VER_LATEST      0x1UL
-+	#define HCOMM_STATUS_VER_LAST       HCOMM_STATUS_VER_LATEST
-+	#define HCOMM_STATUS_SIGNATURE_MASK 0xffffff00UL
-+	#define HCOMM_STATUS_SIGNATURE_SFT 8
-+	#define HCOMM_STATUS_SIGNATURE_VAL   (0x484353UL << 8)
-+	#define HCOMM_STATUS_SIGNATURE_LAST HCOMM_STATUS_SIGNATURE_VAL
-+	u32	fw_status_loc;
-+	#define HCOMM_STATUS_TRUE_ADDR_SPACE_MASK    0x3UL
-+	#define HCOMM_STATUS_TRUE_ADDR_SPACE_SFT     0
-+	#define HCOMM_STATUS_FW_STATUS_LOC_ADDR_SPACE_PCIE_CFG  0x0UL
-+	#define HCOMM_STATUS_FW_STATUS_LOC_ADDR_SPACE_GRC       0x1UL
-+	#define HCOMM_STATUS_FW_STATUS_LOC_ADDR_SPACE_BAR0      0x2UL
-+	#define HCOMM_STATUS_FW_STATUS_LOC_ADDR_SPACE_BAR1      0x3UL
-+	#define HCOMM_STATUS_FW_STATUS_LOC_ADDR_SPACE_LAST     HCOMM_STATUS_FW_STATUS_LOC_ADDR_SPACE_BAR1
-+	#define HCOMM_STATUS_TRUE_OFFSET_MASK        0xfffffffcUL
-+	#define HCOMM_STATUS_TRUE_OFFSET_SFT         2
-+};
-+#define HCOMM_STATUS_STRUCT_LOC 0x31001F0UL
-+
-+#endif /* _BNXT_HSI_H_ */
-diff --git a/include/linux/bootconfig.h b/include/linux/bootconfig.h
-index 1611f9db878e..25df9260d206 100644
--- a/include/linux/bootconfig.h
-+++ b/include/linux/bootconfig.h
-@@ -10,6 +10,7 @@
- #ifdef __KERNEL__
- #include <linux/kernel.h>
- #include <linux/types.h>
-+bool __init cmdline_has_extra_options(void);
- #else /* !__KERNEL__ */
- /*
-  * NOTE: This is only for tools/bootconfig, because tools/bootconfig will
-@@ -59,7 +60,7 @@ struct xbc_node {
- /* Maximum size of boot config is 32KB - 1 */
- #define XBC_DATA_MAX	(XBC_VALUE - 1)
- 
-#define XBC_NODE_MAX	1024
-+#define XBC_NODE_MAX	8192
- #define XBC_KEYLEN_MAX	256
- #define XBC_DEPTH_MAX	16
- 
-@@ -287,7 +288,12 @@ int __init xbc_init(const char *buf, size_t size, const char **emsg, int *epos);
- int __init xbc_get_info(int *node_size, size_t *data_size);
- 
- /* XBC cleanup data structures */
-void __init xbc_exit(void);
-+void __init _xbc_exit(bool early);
-+
-+static __always_inline void xbc_exit(void)
-+{
-+	_xbc_exit(false);
-+}
- 
- /* XBC embedded bootconfig data in kernel */
- #ifdef CONFIG_BOOT_CONFIG_EMBED
-diff --git a/include/linux/bootmem_info.h b/include/linux/bootmem_info.h
-index cc35d010fa94..4c506e76a808 100644
--- a/include/linux/bootmem_info.h
-+++ b/include/linux/bootmem_info.h
-@@ -3,13 +3,13 @@
- #define __LINUX_BOOTMEM_INFO_H
- 
- #include <linux/mm.h>
-+#include <linux/kmemleak.h>
- 
- /*
- * Types for free bootmem stored in page->lru.next. These have to be in
- * some random range in unsigned long space for debugging purposes.
-+ * Types for free bootmem stored in the low bits of page->private.
-  */
-enum {
-	MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE = 12,
-+enum bootmem_type {
-+	MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE = 1,
- 	SECTION_INFO = MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE,
- 	MIX_SECTION_INFO,
- 	NODE_INFO,
-@@ -18,11 +18,23 @@ enum {
- 
- #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
- void __init register_page_bootmem_info_node(struct pglist_data *pgdat);
-+void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
-+				  unsigned long nr_pages);
- 
- void get_page_bootmem(unsigned long info, struct page *page,
-		      unsigned long type);
-+		enum bootmem_type type);
- void put_page_bootmem(struct page *page);
- 
-+static inline enum bootmem_type bootmem_type(const struct page *page)
-+{
-+	return (unsigned long)page->private & 0xf;
-+}
-+
-+static inline unsigned long bootmem_info(const struct page *page)
-+{
-+	return (unsigned long)page->private >> 4;
-+}
-+
- /*
-  * Any memory allocated via the memblock allocator and not via the
-  * buddy will be marked reserved already in the memmap. For those
-@@ -30,7 +42,7 @@ void put_page_bootmem(struct page *page);
-  */
- static inline void free_bootmem_page(struct page *page)
- {
-	unsigned long magic = page->index;
-+	enum bootmem_type type = bootmem_type(page);
- 
- 	/*
- 	 * The reserve_bootmem_region sets the reserved flag on bootmem
-@@ -38,7 +50,7 @@ static inline void free_bootmem_page(struct page *page)
- 	 */
- 	VM_BUG_ON_PAGE(page_ref_count(page) != 2, page);
- 
-	if (magic == SECTION_INFO || magic == MIX_SECTION_INFO)
-+	if (type == SECTION_INFO || type == MIX_SECTION_INFO)
- 		put_page_bootmem(page);
- 	else
- 		VM_BUG_ON_PAGE(1, page);
-@@ -48,17 +60,33 @@ static inline void register_page_bootmem_info_node(struct pglist_data *pgdat)
- {
- }
- 
-+static inline void register_page_bootmem_memmap(unsigned long section_nr,
-+		struct page *map, unsigned long nr_pages)
-+{
-+}
-+
- static inline void put_page_bootmem(struct page *page)
- {
- }
- 
-+static inline enum bootmem_type bootmem_type(const struct page *page)
-+{
-+	return SECTION_INFO;
-+}
-+
-+static inline unsigned long bootmem_info(const struct page *page)
-+{
-+	return 0;
-+}
-+
- static inline void get_page_bootmem(unsigned long info, struct page *page,
-				    unsigned long type)
-+				    enum bootmem_type type)
- {
- }
- 
- static inline void free_bootmem_page(struct page *page)
- {
-+	kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE);
- 	free_reserved_page(page);
- }
- #endif
-diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h
-index 7b121bd780eb..c9e6b26abab6 100644
--- a/include/linux/bpf-cgroup-defs.h
-+++ b/include/linux/bpf-cgroup-defs.h
-@@ -28,19 +28,24 @@ enum cgroup_bpf_attach_type {
- 	CGROUP_INET6_BIND,
- 	CGROUP_INET4_CONNECT,
- 	CGROUP_INET6_CONNECT,
-+	CGROUP_UNIX_CONNECT,
- 	CGROUP_INET4_POST_BIND,
- 	CGROUP_INET6_POST_BIND,
- 	CGROUP_UDP4_SENDMSG,
- 	CGROUP_UDP6_SENDMSG,
-+	CGROUP_UNIX_SENDMSG,
- 	CGROUP_SYSCTL,
- 	CGROUP_UDP4_RECVMSG,
- 	CGROUP_UDP6_RECVMSG,
-+	CGROUP_UNIX_RECVMSG,
- 	CGROUP_GETSOCKOPT,
- 	CGROUP_SETSOCKOPT,
- 	CGROUP_INET4_GETPEERNAME,
- 	CGROUP_INET6_GETPEERNAME,
-+	CGROUP_UNIX_GETPEERNAME,
- 	CGROUP_INET4_GETSOCKNAME,
- 	CGROUP_INET6_GETSOCKNAME,
-+	CGROUP_UNIX_GETSOCKNAME,
- 	CGROUP_INET_SOCK_RELEASE,
- 	CGROUP_LSM_START,
- 	CGROUP_LSM_END = CGROUP_LSM_START + CGROUP_LSM_NUM - 1,
-@@ -58,6 +63,7 @@ struct cgroup_bpf {
- 	 */
- 	struct hlist_head progs[MAX_CGROUP_BPF_ATTACH_TYPE];
- 	u8 flags[MAX_CGROUP_BPF_ATTACH_TYPE];
-+	u64 revisions[MAX_CGROUP_BPF_ATTACH_TYPE];
- 
- 	/* list of cgroup shared storages */
- 	struct list_head storages;
-diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h
-index 2bd1b5f8de9b..d1eb5c7729cb 100644
--- a/include/linux/bpf-cgroup.h
-+++ b/include/linux/bpf-cgroup.h
-@@ -48,19 +48,24 @@ to_cgroup_bpf_attach_type(enum bpf_attach_type attach_type)
- 	CGROUP_ATYPE(CGROUP_INET6_BIND);
- 	CGROUP_ATYPE(CGROUP_INET4_CONNECT);
- 	CGROUP_ATYPE(CGROUP_INET6_CONNECT);
-+	CGROUP_ATYPE(CGROUP_UNIX_CONNECT);
- 	CGROUP_ATYPE(CGROUP_INET4_POST_BIND);
- 	CGROUP_ATYPE(CGROUP_INET6_POST_BIND);
- 	CGROUP_ATYPE(CGROUP_UDP4_SENDMSG);
- 	CGROUP_ATYPE(CGROUP_UDP6_SENDMSG);
-+	CGROUP_ATYPE(CGROUP_UNIX_SENDMSG);
- 	CGROUP_ATYPE(CGROUP_SYSCTL);
- 	CGROUP_ATYPE(CGROUP_UDP4_RECVMSG);
- 	CGROUP_ATYPE(CGROUP_UDP6_RECVMSG);
-+	CGROUP_ATYPE(CGROUP_UNIX_RECVMSG);
- 	CGROUP_ATYPE(CGROUP_GETSOCKOPT);
- 	CGROUP_ATYPE(CGROUP_SETSOCKOPT);
- 	CGROUP_ATYPE(CGROUP_INET4_GETPEERNAME);
- 	CGROUP_ATYPE(CGROUP_INET6_GETPEERNAME);
-+	CGROUP_ATYPE(CGROUP_UNIX_GETPEERNAME);
- 	CGROUP_ATYPE(CGROUP_INET4_GETSOCKNAME);
- 	CGROUP_ATYPE(CGROUP_INET6_GETSOCKNAME);
-+	CGROUP_ATYPE(CGROUP_UNIX_GETSOCKNAME);
- 	CGROUP_ATYPE(CGROUP_INET_SOCK_RELEASE);
- 	default:
- 		return CGROUP_BPF_ATTACH_TYPE_INVALID;
-@@ -72,9 +77,6 @@ to_cgroup_bpf_attach_type(enum bpf_attach_type attach_type)
- extern struct static_key_false cgroup_bpf_enabled_key[MAX_CGROUP_BPF_ATTACH_TYPE];
- #define cgroup_bpf_enabled(atype) static_branch_unlikely(&cgroup_bpf_enabled_key[atype])
- 
-#define for_each_cgroup_storage_type(stype) \
-	for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
-
- struct bpf_cgroup_storage_map;
- 
- struct bpf_storage_buffer {
-@@ -98,7 +100,6 @@ struct bpf_cgroup_storage {
- struct bpf_cgroup_link {
- 	struct bpf_link link;
- 	struct cgroup *cgroup;
-	enum bpf_attach_type type;
- };
- 
- struct bpf_prog_list {
-@@ -106,10 +107,10 @@ struct bpf_prog_list {
- 	struct bpf_prog *prog;
- 	struct bpf_cgroup_link *link;
- 	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
-+	u32 flags;
- };
- 
-int cgroup_bpf_inherit(struct cgroup *cgrp);
-void cgroup_bpf_offline(struct cgroup *cgrp);
-+void __init cgroup_bpf_lifetime_notifier_init(void);
- 
- int __cgroup_bpf_run_filter_skb(struct sock *sk,
- 				struct sk_buff *skb,
-@@ -119,7 +120,8 @@ int __cgroup_bpf_run_filter_sk(struct sock *sk,
- 			       enum cgroup_bpf_attach_type atype);
- 
- int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
-				      struct sockaddr *uaddr,
-+				      struct sockaddr_unsized *uaddr,
-+				      int *uaddrlen,
- 				      enum cgroup_bpf_attach_type atype,
- 				      void *t_ctx,
- 				      u32 *flags);
-@@ -132,16 +134,17 @@ int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,
- 				      short access, enum cgroup_bpf_attach_type atype);
- 
- int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head,
-				   struct ctl_table *table, int write,
-+				   const struct ctl_table *table, int write,
- 				   char **buf, size_t *pcount, loff_t *ppos,
- 				   enum cgroup_bpf_attach_type atype);
- 
- int __cgroup_bpf_run_filter_setsockopt(struct sock *sock, int *level,
-				       int *optname, char __user *optval,
-+				       int *optname, sockptr_t optval,
- 				       int *optlen, char **kernel_optval);
-+
- int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,
-				       int optname, char __user *optval,
-				       int __user *optlen, int max_optlen,
-+				       int optname, sockptr_t optval,
-+				       sockptr_t optlen, int max_optlen,
- 				       int retval);
- 
- int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level,
-@@ -189,7 +192,8 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk,
- ({									      \
- 	int __ret = 0;							      \
- 	if (cgroup_bpf_enabled(CGROUP_INET_INGRESS) &&			      \
-	    cgroup_bpf_sock_enabled(sk, CGROUP_INET_INGRESS))		      \
-+	    cgroup_bpf_sock_enabled(sk, CGROUP_INET_INGRESS) && sk &&	      \
-+	    sk_fullsock(sk))						      \
- 		__ret = __cgroup_bpf_run_filter_skb(sk, skb,		      \
- 						    CGROUP_INET_INGRESS); \
- 									      \
-@@ -199,9 +203,9 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk,
- #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb)			       \
- ({									       \
- 	int __ret = 0;							       \
-	if (cgroup_bpf_enabled(CGROUP_INET_EGRESS) && sk && sk == skb->sk) { \
-+	if (cgroup_bpf_enabled(CGROUP_INET_EGRESS) && sk) {		       \
- 		typeof(sk) __sk = sk_to_full_sk(sk);			       \
-		if (sk_fullsock(__sk) &&				       \
-+		if (__sk && __sk == skb_to_full_sk(skb) &&	       \
- 		    cgroup_bpf_sock_enabled(__sk, CGROUP_INET_EGRESS))	       \
- 			__ret = __cgroup_bpf_run_filter_skb(__sk, skb,	       \
- 						      CGROUP_INET_EGRESS); \
-@@ -230,22 +234,24 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk,
- #define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk)				       \
- 	BPF_CGROUP_RUN_SK_PROG(sk, CGROUP_INET6_POST_BIND)
- 
-#define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, atype)				       \
-+#define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, uaddrlen, atype)		       \
- ({									       \
- 	int __ret = 0;							       \
- 	if (cgroup_bpf_enabled(atype))					       \
-		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, atype,     \
-							  NULL, NULL);	       \
-+		__ret = __cgroup_bpf_run_filter_sock_addr(sk,		       \
-+				(struct sockaddr_unsized *)uaddr, uaddrlen,     \
-+				atype, NULL, NULL);			       \
- 	__ret;								       \
- })
- 
-#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, atype, t_ctx)		       \
-+#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, atype, t_ctx)	       \
- ({									       \
- 	int __ret = 0;							       \
- 	if (cgroup_bpf_enabled(atype))	{				       \
- 		lock_sock(sk);						       \
-		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, atype,     \
-							  t_ctx, NULL);	       \
-+		__ret = __cgroup_bpf_run_filter_sock_addr(sk,		       \
-+				(struct sockaddr_unsized *)uaddr, uaddrlen,     \
-+				atype, t_ctx, NULL);			       \
- 		release_sock(sk);					       \
- 	}								       \
- 	__ret;								       \
-@@ -256,14 +262,15 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk,
-  * (at bit position 0) is to indicate CAP_NET_BIND_SERVICE capability check
-  * should be bypassed (BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE).
-  */
-#define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, atype, bind_flags)	       \
-+#define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, uaddrlen, atype, bind_flags) \
- ({									       \
- 	u32 __flags = 0;						       \
- 	int __ret = 0;							       \
- 	if (cgroup_bpf_enabled(atype))	{				       \
- 		lock_sock(sk);						       \
-		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, atype,     \
-							  NULL, &__flags);     \
-+		__ret = __cgroup_bpf_run_filter_sock_addr(sk,		       \
-+				(struct sockaddr_unsized *)uaddr, uaddrlen,     \
-+				atype, NULL, &__flags);			       \
- 		release_sock(sk);					       \
- 		if (__flags & BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE)	       \
- 			*bind_flags |= BIND_NO_CAP_NET_BIND_SERVICE;	       \
-@@ -276,29 +283,38 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk,
- 	  cgroup_bpf_enabled(CGROUP_INET6_CONNECT)) &&		       \
- 	 (sk)->sk_prot->pre_connect)
- 
-#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr)			       \
-	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, CGROUP_INET4_CONNECT)
-+#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr, uaddrlen)			\
-+	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, uaddrlen, CGROUP_INET4_CONNECT)
-+
-+#define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, uaddrlen)			\
-+	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, uaddrlen, CGROUP_INET6_CONNECT)
-+
-+#define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr, uaddrlen)		\
-+	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, CGROUP_INET4_CONNECT, NULL)
-+
-+#define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr, uaddrlen)		\
-+	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, CGROUP_INET6_CONNECT, NULL)
- 
-#define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr)			       \
-	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, CGROUP_INET6_CONNECT)
-+#define BPF_CGROUP_RUN_PROG_UNIX_CONNECT_LOCK(sk, uaddr, uaddrlen)		\
-+	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, CGROUP_UNIX_CONNECT, NULL)
- 
-#define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr)		       \
-	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, CGROUP_INET4_CONNECT, NULL)
-+#define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, uaddrlen, t_ctx)	\
-+	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, CGROUP_UDP4_SENDMSG, t_ctx)
- 
-#define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr)		       \
-	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, CGROUP_INET6_CONNECT, NULL)
-+#define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, uaddrlen, t_ctx)	\
-+	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, CGROUP_UDP6_SENDMSG, t_ctx)
- 
-#define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx)		       \
-	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, CGROUP_UDP4_SENDMSG, t_ctx)
-+#define BPF_CGROUP_RUN_PROG_UNIX_SENDMSG_LOCK(sk, uaddr, uaddrlen, t_ctx)	\
-+	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, CGROUP_UNIX_SENDMSG, t_ctx)
- 
-#define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx)		       \
-	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, CGROUP_UDP6_SENDMSG, t_ctx)
-+#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr, uaddrlen)		\
-+	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, CGROUP_UDP4_RECVMSG, NULL)
- 
-#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr)			\
-	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, CGROUP_UDP4_RECVMSG, NULL)
-+#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr, uaddrlen)		\
-+	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, CGROUP_UDP6_RECVMSG, NULL)
- 
-#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr)			\
-	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, CGROUP_UDP6_RECVMSG, NULL)
-+#define BPF_CGROUP_RUN_PROG_UNIX_RECVMSG_LOCK(sk, uaddr, uaddrlen)		\
-+	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, CGROUP_UNIX_RECVMSG, NULL)
- 
- /* The SOCK_OPS"_SK" macro should be used when sock_ops->sk is not a
-  * fullsock and its parent fullsock cannot be traced by
-@@ -373,14 +389,6 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk,
- 	__ret;								       \
- })
- 
-#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen)			       \
-({									       \
-	int __ret = 0;							       \
-	if (cgroup_bpf_enabled(CGROUP_GETSOCKOPT))			       \
-		get_user(__ret, optlen);				       \
-	__ret;								       \
-})
-
- #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, optlen,   \
- 				       max_optlen, retval)		       \
- ({									       \
-@@ -414,10 +422,15 @@ int cgroup_bpf_prog_detach(const union bpf_attr *attr,
- int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
- int cgroup_bpf_prog_query(const union bpf_attr *attr,
- 			  union bpf_attr __user *uattr);
-+
-+const struct bpf_func_proto *
-+cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
- #else
- 
-static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
-static inline void cgroup_bpf_offline(struct cgroup *cgrp) {}
-+static inline void cgroup_bpf_lifetime_notifier_init(void)
-+{
-+	return;
-+}
- 
- static inline int cgroup_bpf_prog_attach(const union bpf_attr *attr,
- 					 enum bpf_prog_type ptype,
-@@ -444,6 +457,12 @@ static inline int cgroup_bpf_prog_query(const union bpf_attr *attr,
- 	return -EINVAL;
- }
- 
-+static inline const struct bpf_func_proto *
-+cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
-+{
-+	return NULL;
-+}
-+
- static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
- 					    struct bpf_map *map) { return 0; }
- static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
-@@ -460,28 +479,30 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,
- }
- 
- #define cgroup_bpf_enabled(atype) (0)
-#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, atype, t_ctx) ({ 0; })
-#define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, atype) ({ 0; })
-+#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, atype, t_ctx) ({ 0; })
-+#define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, uaddrlen, atype) ({ 0; })
- #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0)
- #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; })
- #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; })
- #define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; })
- #define BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk) ({ 0; })
-#define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, atype, flags) ({ 0; })
-+#define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, uaddrlen, atype, flags) ({ 0; })
- #define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; })
- #define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; })
-#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; })
-#define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) ({ 0; })
-#define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) ({ 0; })
-#define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) ({ 0; })
-#define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; })
-#define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; })
-#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr) ({ 0; })
-#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr) ({ 0; })
-+#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr, uaddrlen) ({ 0; })
-+#define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr, uaddrlen) ({ 0; })
-+#define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, uaddrlen) ({ 0; })
-+#define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr, uaddrlen) ({ 0; })
-+#define BPF_CGROUP_RUN_PROG_UNIX_CONNECT_LOCK(sk, uaddr, uaddrlen) ({ 0; })
-+#define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, uaddrlen, t_ctx) ({ 0; })
-+#define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, uaddrlen, t_ctx) ({ 0; })
-+#define BPF_CGROUP_RUN_PROG_UNIX_SENDMSG_LOCK(sk, uaddr, uaddrlen, t_ctx) ({ 0; })
-+#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; })
-+#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; })
-+#define BPF_CGROUP_RUN_PROG_UNIX_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; })
- #define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; })
- #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) ({ 0; })
- #define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos) ({ 0; })
-#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; })
- #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \
- 				       optlen, max_optlen, retval) ({ retval; })
- #define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval, \
-@@ -489,8 +510,6 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,
- #define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \
- 				       kernel_optval) ({ 0; })
- 
-#define for_each_cgroup_storage_type(stype) for (; false; )
-
- #endif /* CONFIG_CGROUP_BPF */
- 
- #endif /* _BPF_CGROUP_H */
-diff --git a/include/linux/bpf.h b/include/linux/bpf.h
-index 20c26aed7896..6498be4c44f8 100644
--- a/include/linux/bpf.h
-+++ b/include/linux/bpf.h
-@@ -7,6 +7,7 @@
- #include <uapi/linux/bpf.h>
- #include <uapi/linux/filter.h>
- 
-+#include <crypto/sha2.h>
- #include <linux/workqueue.h>
- #include <linux/file.h>
- #include <linux/percpu.h>
-@@ -27,6 +28,10 @@
- #include <linux/bpfptr.h>
- #include <linux/btf.h>
- #include <linux/rcupdate_trace.h>
-+#include <linux/static_call.h>
-+#include <linux/memcontrol.h>
-+#include <linux/cfi.h>
-+#include <asm/rqspinlock.h>
- 
- struct bpf_verifier_env;
- struct bpf_verifier_log;
-@@ -34,6 +39,7 @@ struct perf_event;
- struct bpf_prog;
- struct bpf_prog_aux;
- struct bpf_map;
-+struct bpf_arena;
- struct sock;
- struct seq_file;
- struct btf;
-@@ -48,10 +54,17 @@ struct mem_cgroup;
- struct module;
- struct bpf_func_state;
- struct ftrace_ops;
-+struct cgroup;
-+struct bpf_token;
-+struct user_namespace;
-+struct super_block;
-+struct inode;
- 
- extern struct idr btf_idr;
- extern spinlock_t btf_idr_lock;
- extern struct kobject *btf_kobj;
-+extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma;
-+extern bool bpf_global_ma_set;
- 
- typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64);
- typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,
-@@ -83,24 +96,30 @@ struct bpf_map_ops {
- 	int (*map_lookup_and_delete_batch)(struct bpf_map *map,
- 					   const union bpf_attr *attr,
- 					   union bpf_attr __user *uattr);
-	int (*map_update_batch)(struct bpf_map *map, const union bpf_attr *attr,
-+	int (*map_update_batch)(struct bpf_map *map, struct file *map_file,
-+				const union bpf_attr *attr,
- 				union bpf_attr __user *uattr);
- 	int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr,
- 				union bpf_attr __user *uattr);
- 
- 	/* funcs callable from userspace and from eBPF programs */
- 	void *(*map_lookup_elem)(struct bpf_map *map, void *key);
-	int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
-	int (*map_delete_elem)(struct bpf_map *map, void *key);
-	int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);
-	int (*map_pop_elem)(struct bpf_map *map, void *value);
-	int (*map_peek_elem)(struct bpf_map *map, void *value);
-+	long (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
-+	long (*map_delete_elem)(struct bpf_map *map, void *key);
-+	long (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);
-+	long (*map_pop_elem)(struct bpf_map *map, void *value);
-+	long (*map_peek_elem)(struct bpf_map *map, void *value);
- 	void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu);
-+	int (*map_get_hash)(struct bpf_map *map, u32 hash_buf_size, void *hash_buf);
- 
- 	/* funcs called by prog_array and perf_event_array map */
- 	void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
- 				int fd);
-	void (*map_fd_put_ptr)(void *ptr);
-+	/* If need_defer is true, the implementation should guarantee that
-+	 * the to-be-put element is still alive before the bpf program, which
-+	 * may manipulate it, exists.
-+	 */
-+	void (*map_fd_put_ptr)(struct bpf_map *map, void *ptr, bool need_defer);
- 	int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
- 	u32 (*map_fd_sys_lookup_elem)(void *ptr);
- 	void (*map_seq_show_elem)(struct bpf_map *map, void *key,
-@@ -124,6 +143,9 @@ struct bpf_map_ops {
- 	int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
- 	__poll_t (*map_poll)(struct bpf_map *map, struct file *filp,
- 			     struct poll_table_struct *pts);
-+	unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr,
-+					       unsigned long len, unsigned long pgoff,
-+					       unsigned long flags);
- 
- 	/* Functions called by bpf_local_storage maps */
- 	int (*map_local_storage_charge)(struct bpf_local_storage_map *smap,
-@@ -133,7 +155,7 @@ struct bpf_map_ops {
- 	struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner);
- 
- 	/* Misc helpers.*/
-	int (*map_redirect)(struct bpf_map *map, u32 ifindex, u64 flags);
-+	long (*map_redirect)(struct bpf_map *map, u64 key, u64 flags);
- 
- 	/* map_meta_equal must be implemented for maps that can be
- 	 * used as an inner map.  It is a runtime check to ensure
-@@ -151,10 +173,12 @@ struct bpf_map_ops {
- 	int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env,
- 					      struct bpf_func_state *caller,
- 					      struct bpf_func_state *callee);
-	int (*map_for_each_callback)(struct bpf_map *map,
-+	long (*map_for_each_callback)(struct bpf_map *map,
- 				     bpf_callback_t callback_fn,
- 				     void *callback_ctx, u64 flags);
- 
-+	u64 (*map_mem_usage)(const struct bpf_map *map);
-+
- 	/* BTF id of struct allocated by map_alloc */
- 	int *map_btf_id;
- 
-@@ -163,45 +187,114 @@ struct bpf_map_ops {
- };
- 
- enum {
-	/* Support at most 8 pointers in a BPF map value */
-	BPF_MAP_VALUE_OFF_MAX = 8,
-	BPF_MAP_OFF_ARR_MAX   = BPF_MAP_VALUE_OFF_MAX +
-				1 + /* for bpf_spin_lock */
-				1,  /* for bpf_timer */
-+	/* Support at most 11 fields in a BTF type */
-+	BTF_FIELDS_MAX	   = 11,
-+};
-+
-+enum btf_field_type {
-+	BPF_SPIN_LOCK  = (1 << 0),
-+	BPF_TIMER      = (1 << 1),
-+	BPF_KPTR_UNREF = (1 << 2),
-+	BPF_KPTR_REF   = (1 << 3),
-+	BPF_KPTR_PERCPU = (1 << 4),
-+	BPF_KPTR       = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU,
-+	BPF_LIST_HEAD  = (1 << 5),
-+	BPF_LIST_NODE  = (1 << 6),
-+	BPF_RB_ROOT    = (1 << 7),
-+	BPF_RB_NODE    = (1 << 8),
-+	BPF_GRAPH_NODE = BPF_RB_NODE | BPF_LIST_NODE,
-+	BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD,
-+	BPF_REFCOUNT   = (1 << 9),
-+	BPF_WORKQUEUE  = (1 << 10),
-+	BPF_UPTR       = (1 << 11),
-+	BPF_RES_SPIN_LOCK = (1 << 12),
-+	BPF_TASK_WORK  = (1 << 13),
- };
- 
-enum bpf_kptr_type {
-	BPF_KPTR_UNREF,
-	BPF_KPTR_REF,
-+enum bpf_cgroup_storage_type {
-+	BPF_CGROUP_STORAGE_SHARED,
-+	BPF_CGROUP_STORAGE_PERCPU,
-+	__BPF_CGROUP_STORAGE_MAX
-+#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
- };
- 
-struct bpf_map_value_off_desc {
-	u32 offset;
-	enum bpf_kptr_type type;
-	struct {
-		struct btf *btf;
-		struct module *module;
-		btf_dtor_kfunc_t dtor;
-		u32 btf_id;
-	} kptr;
-+#ifdef CONFIG_CGROUP_BPF
-+# define for_each_cgroup_storage_type(stype) \
-+	for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
-+#else
-+# define for_each_cgroup_storage_type(stype) for (; false; )
-+#endif /* CONFIG_CGROUP_BPF */
-+
-+typedef void (*btf_dtor_kfunc_t)(void *);
-+
-+struct btf_field_kptr {
-+	struct btf *btf;
-+	struct module *module;
-+	/* dtor used if btf_is_kernel(btf), otherwise the type is
-+	 * program-allocated, dtor is NULL,  and __bpf_obj_drop_impl is used
-+	 */
-+	btf_dtor_kfunc_t dtor;
-+	u32 btf_id;
- };
- 
-struct bpf_map_value_off {
-	u32 nr_off;
-	struct bpf_map_value_off_desc off[];
-+struct btf_field_graph_root {
-+	struct btf *btf;
-+	u32 value_btf_id;
-+	u32 node_offset;
-+	struct btf_record *value_rec;
- };
- 
-struct bpf_map_off_arr {
-+struct btf_field {
-+	u32 offset;
-+	u32 size;
-+	enum btf_field_type type;
-+	union {
-+		struct btf_field_kptr kptr;
-+		struct btf_field_graph_root graph_root;
-+	};
-+};
-+
-+struct btf_record {
- 	u32 cnt;
-	u32 field_off[BPF_MAP_OFF_ARR_MAX];
-	u8 field_sz[BPF_MAP_OFF_ARR_MAX];
-+	u32 field_mask;
-+	int spin_lock_off;
-+	int res_spin_lock_off;
-+	int timer_off;
-+	int wq_off;
-+	int refcount_off;
-+	int task_work_off;
-+	struct btf_field fields[];
-+};
-+
-+/* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */
-+struct bpf_rb_node_kern {
-+	struct rb_node rb_node;
-+	void *owner;
-+} __attribute__((aligned(8)));
-+
-+/* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */
-+struct bpf_list_node_kern {
-+	struct list_head list_head;
-+	void *owner;
-+} __attribute__((aligned(8)));
-+
-+/* 'Ownership' of program-containing map is claimed by the first program
-+ * that is going to use this map or by the first program which FD is
-+ * stored in the map to make sure that all callers and callees have the
-+ * same prog type, JITed flag and xdp_has_frags flag.
-+ */
-+struct bpf_map_owner {
-+	enum bpf_prog_type type;
-+	bool jited;
-+	bool xdp_has_frags;
-+	u64 storage_cookie[MAX_BPF_CGROUP_STORAGE_TYPE];
-+	const struct btf_type *attach_func_proto;
-+	enum bpf_attach_type expected_attach_type;
- };
- 
- struct bpf_map {
-	/* The first two cachelines with read-mostly members of which some
-	 * are also accessed in fast-path (e.g. ops, max_entries).
-	 */
-	const struct bpf_map_ops *ops ____cacheline_aligned;
-+	u8 sha[SHA256_DIGEST_SIZE];
-+	const struct bpf_map_ops *ops;
- 	struct bpf_map *inner_map_meta;
- #ifdef CONFIG_SECURITY
- 	void *security;
-@@ -212,95 +305,317 @@ struct bpf_map {
- 	u32 max_entries;
- 	u64 map_extra; /* any per-map-type extra fields */
- 	u32 map_flags;
-	int spin_lock_off; /* >=0 valid offset, <0 error */
-	struct bpf_map_value_off *kptr_off_tab;
-	int timer_off; /* >=0 valid offset, <0 error */
- 	u32 id;
-+	struct btf_record *record;
- 	int numa_node;
- 	u32 btf_key_type_id;
- 	u32 btf_value_type_id;
- 	u32 btf_vmlinux_value_type_id;
- 	struct btf *btf;
-#ifdef CONFIG_MEMCG_KMEM
-+#ifdef CONFIG_MEMCG
- 	struct obj_cgroup *objcg;
- #endif
- 	char name[BPF_OBJ_NAME_LEN];
-	struct bpf_map_off_arr *off_arr;
-	/* The 3rd and 4th cacheline with misc members to avoid false sharing
-	 * particularly with refcounting.
-	 */
-	atomic64_t refcnt ____cacheline_aligned;
-	atomic64_t usercnt;
-	struct work_struct work;
- 	struct mutex freeze_mutex;
-+	atomic64_t refcnt;
-+	atomic64_t usercnt;
-+	/* rcu is used before freeing and work is only used during freeing */
-+	union {
-+		struct work_struct work;
-+		struct rcu_head rcu;
-+	};
- 	atomic64_t writecnt;
-	/* 'Ownership' of program-containing map is claimed by the first program
-	 * that is going to use this map or by the first program which FD is
-	 * stored in the map to make sure that all callers and callees have the
-	 * same prog type, JITed flag and xdp_has_frags flag.
-	 */
-	struct {
-		spinlock_t lock;
-		enum bpf_prog_type type;
-		bool jited;
-		bool xdp_has_frags;
-	} owner;
-+	spinlock_t owner_lock;
-+	struct bpf_map_owner *owner;
- 	bool bypass_spec_v1;
- 	bool frozen; /* write-once; write-protected by freeze_mutex */
-};
-+	bool free_after_mult_rcu_gp;
-+	bool free_after_rcu_gp;
-+	atomic64_t sleepable_refcnt;
-+	s64 __percpu *elem_count;
-+	u64 cookie; /* write-once */
-+	char *excl_prog_sha;
-+};
-+
-+static inline const char *btf_field_type_name(enum btf_field_type type)
-+{
-+	switch (type) {
-+	case BPF_SPIN_LOCK:
-+		return "bpf_spin_lock";
-+	case BPF_RES_SPIN_LOCK:
-+		return "bpf_res_spin_lock";
-+	case BPF_TIMER:
-+		return "bpf_timer";
-+	case BPF_WORKQUEUE:
-+		return "bpf_wq";
-+	case BPF_KPTR_UNREF:
-+	case BPF_KPTR_REF:
-+		return "kptr";
-+	case BPF_KPTR_PERCPU:
-+		return "percpu_kptr";
-+	case BPF_UPTR:
-+		return "uptr";
-+	case BPF_LIST_HEAD:
-+		return "bpf_list_head";
-+	case BPF_LIST_NODE:
-+		return "bpf_list_node";
-+	case BPF_RB_ROOT:
-+		return "bpf_rb_root";
-+	case BPF_RB_NODE:
-+		return "bpf_rb_node";
-+	case BPF_REFCOUNT:
-+		return "bpf_refcount";
-+	case BPF_TASK_WORK:
-+		return "bpf_task_work";
-+	default:
-+		WARN_ON_ONCE(1);
-+		return "unknown";
-+	}
-+}
- 
-static inline bool map_value_has_spin_lock(const struct bpf_map *map)
-{
-	return map->spin_lock_off >= 0;
-+#if IS_ENABLED(CONFIG_DEBUG_KERNEL)
-+#define BPF_WARN_ONCE(cond, format...) WARN_ONCE(cond, format)
-+#else
-+#define BPF_WARN_ONCE(cond, format...) BUILD_BUG_ON_INVALID(cond)
-+#endif
-+
-+static inline u32 btf_field_type_size(enum btf_field_type type)
-+{
-+	switch (type) {
-+	case BPF_SPIN_LOCK:
-+		return sizeof(struct bpf_spin_lock);
-+	case BPF_RES_SPIN_LOCK:
-+		return sizeof(struct bpf_res_spin_lock);
-+	case BPF_TIMER:
-+		return sizeof(struct bpf_timer);
-+	case BPF_WORKQUEUE:
-+		return sizeof(struct bpf_wq);
-+	case BPF_KPTR_UNREF:
-+	case BPF_KPTR_REF:
-+	case BPF_KPTR_PERCPU:
-+	case BPF_UPTR:
-+		return sizeof(u64);
-+	case BPF_LIST_HEAD:
-+		return sizeof(struct bpf_list_head);
-+	case BPF_LIST_NODE:
-+		return sizeof(struct bpf_list_node);
-+	case BPF_RB_ROOT:
-+		return sizeof(struct bpf_rb_root);
-+	case BPF_RB_NODE:
-+		return sizeof(struct bpf_rb_node);
-+	case BPF_REFCOUNT:
-+		return sizeof(struct bpf_refcount);
-+	case BPF_TASK_WORK:
-+		return sizeof(struct bpf_task_work);
-+	default:
-+		WARN_ON_ONCE(1);
-+		return 0;
-+	}
- }
- 
-static inline bool map_value_has_timer(const struct bpf_map *map)
-+static inline u32 btf_field_type_align(enum btf_field_type type)
-+{
-+	switch (type) {
-+	case BPF_SPIN_LOCK:
-+		return __alignof__(struct bpf_spin_lock);
-+	case BPF_RES_SPIN_LOCK:
-+		return __alignof__(struct bpf_res_spin_lock);
-+	case BPF_TIMER:
-+		return __alignof__(struct bpf_timer);
-+	case BPF_WORKQUEUE:
-+		return __alignof__(struct bpf_wq);
-+	case BPF_KPTR_UNREF:
-+	case BPF_KPTR_REF:
-+	case BPF_KPTR_PERCPU:
-+	case BPF_UPTR:
-+		return __alignof__(u64);
-+	case BPF_LIST_HEAD:
-+		return __alignof__(struct bpf_list_head);
-+	case BPF_LIST_NODE:
-+		return __alignof__(struct bpf_list_node);
-+	case BPF_RB_ROOT:
-+		return __alignof__(struct bpf_rb_root);
-+	case BPF_RB_NODE:
-+		return __alignof__(struct bpf_rb_node);
-+	case BPF_REFCOUNT:
-+		return __alignof__(struct bpf_refcount);
-+	case BPF_TASK_WORK:
-+		return __alignof__(struct bpf_task_work);
-+	default:
-+		WARN_ON_ONCE(1);
-+		return 0;
-+	}
-+}
-+
-+static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)
-+{
-+	memset(addr, 0, field->size);
-+
-+	switch (field->type) {
-+	case BPF_REFCOUNT:
-+		refcount_set((refcount_t *)addr, 1);
-+		break;
-+	case BPF_RB_NODE:
-+		RB_CLEAR_NODE((struct rb_node *)addr);
-+		break;
-+	case BPF_LIST_HEAD:
-+	case BPF_LIST_NODE:
-+		INIT_LIST_HEAD((struct list_head *)addr);
-+		break;
-+	case BPF_RB_ROOT:
-+		/* RB_ROOT_CACHED 0-inits, no need to do anything after memset */
-+	case BPF_SPIN_LOCK:
-+	case BPF_RES_SPIN_LOCK:
-+	case BPF_TIMER:
-+	case BPF_WORKQUEUE:
-+	case BPF_KPTR_UNREF:
-+	case BPF_KPTR_REF:
-+	case BPF_KPTR_PERCPU:
-+	case BPF_UPTR:
-+	case BPF_TASK_WORK:
-+		break;
-+	default:
-+		WARN_ON_ONCE(1);
-+		return;
-+	}
-+}
-+
-+static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type)
- {
-	return map->timer_off >= 0;
-+	if (IS_ERR_OR_NULL(rec))
-+		return false;
-+	return rec->field_mask & type;
- }
- 
-static inline bool map_value_has_kptrs(const struct bpf_map *map)
-+static inline void bpf_obj_init(const struct btf_record *rec, void *obj)
- {
-	return !IS_ERR_OR_NULL(map->kptr_off_tab);
-+	int i;
-+
-+	if (IS_ERR_OR_NULL(rec))
-+		return;
-+	for (i = 0; i < rec->cnt; i++)
-+		bpf_obj_init_field(&rec->fields[i], obj + rec->fields[i].offset);
- }
- 
-+/* 'dst' must be a temporary buffer and should not point to memory that is being
-+ * used in parallel by a bpf program or bpf syscall, otherwise the access from
-+ * the bpf program or bpf syscall may be corrupted by the reinitialization,
-+ * leading to weird problems. Even 'dst' is newly-allocated from bpf memory
-+ * allocator, it is still possible for 'dst' to be used in parallel by a bpf
-+ * program or bpf syscall.
-+ */
- static inline void check_and_init_map_value(struct bpf_map *map, void *dst)
- {
-	if (unlikely(map_value_has_spin_lock(map)))
-		memset(dst + map->spin_lock_off, 0, sizeof(struct bpf_spin_lock));
-	if (unlikely(map_value_has_timer(map)))
-		memset(dst + map->timer_off, 0, sizeof(struct bpf_timer));
-	if (unlikely(map_value_has_kptrs(map))) {
-		struct bpf_map_value_off *tab = map->kptr_off_tab;
-		int i;
-+	bpf_obj_init(map->record, dst);
-+}
-+
-+/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
-+ * forced to use 'long' read/writes to try to atomically copy long counters.
-+ * Best-effort only.  No barriers here, since it _will_ race with concurrent
-+ * updates from BPF programs. Called from bpf syscall and mostly used with
-+ * size 8 or 16 bytes, so ask compiler to inline it.
-+ */
-+static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
-+{
-+	const long *lsrc = src;
-+	long *ldst = dst;
- 
-		for (i = 0; i < tab->nr_off; i++)
-			*(u64 *)(dst + tab->off[i].offset) = 0;
-+	size /= sizeof(long);
-+	while (size--)
-+		data_race(*ldst++ = *lsrc++);
-+}
-+
-+/* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */
-+static inline void bpf_obj_memcpy(struct btf_record *rec,
-+				  void *dst, void *src, u32 size,
-+				  bool long_memcpy)
-+{
-+	u32 curr_off = 0;
-+	int i;
-+
-+	if (IS_ERR_OR_NULL(rec)) {
-+		if (long_memcpy)
-+			bpf_long_memcpy(dst, src, round_up(size, 8));
-+		else
-+			memcpy(dst, src, size);
-+		return;
-+	}
-+
-+	for (i = 0; i < rec->cnt; i++) {
-+		u32 next_off = rec->fields[i].offset;
-+		u32 sz = next_off - curr_off;
-+
-+		memcpy(dst + curr_off, src + curr_off, sz);
-+		curr_off += rec->fields[i].size + sz;
- 	}
-+	memcpy(dst + curr_off, src + curr_off, size - curr_off);
- }
- 
-/* copy everything but bpf_spin_lock and bpf_timer. There could be one of each. */
- static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
- {
-+	bpf_obj_memcpy(map->record, dst, src, map->value_size, false);
-+}
-+
-+static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)
-+{
-+	bpf_obj_memcpy(map->record, dst, src, map->value_size, true);
-+}
-+
-+static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src)
-+{
-+	unsigned long *src_uptr, *dst_uptr;
-+	const struct btf_field *field;
-+	int i;
-+
-+	if (!btf_record_has_field(rec, BPF_UPTR))
-+		return;
-+
-+	for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {
-+		if (field->type != BPF_UPTR)
-+			continue;
-+
-+		src_uptr = src + field->offset;
-+		dst_uptr = dst + field->offset;
-+		swap(*src_uptr, *dst_uptr);
-+	}
-+}
-+
-+static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size)
-+{
- 	u32 curr_off = 0;
- 	int i;
- 
-	if (likely(!map->off_arr)) {
-		memcpy(dst, src, map->value_size);
-+	if (IS_ERR_OR_NULL(rec)) {
-+		memset(dst, 0, size);
- 		return;
- 	}
- 
-	for (i = 0; i < map->off_arr->cnt; i++) {
-		u32 next_off = map->off_arr->field_off[i];
-+	for (i = 0; i < rec->cnt; i++) {
-+		u32 next_off = rec->fields[i].offset;
-+		u32 sz = next_off - curr_off;
- 
-		memcpy(dst + curr_off, src + curr_off, next_off - curr_off);
-		curr_off += map->off_arr->field_sz[i];
-+		memset(dst + curr_off, 0, sz);
-+		curr_off += rec->fields[i].size + sz;
- 	}
-	memcpy(dst + curr_off, src + curr_off, map->value_size - curr_off);
-+	memset(dst + curr_off, 0, size - curr_off);
-+}
-+
-+static inline void zero_map_value(struct bpf_map *map, void *dst)
-+{
-+	bpf_obj_memzero(map->record, dst, map->value_size);
- }
-+
- void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
- 			   bool lock_src);
- void bpf_timer_cancel_and_free(void *timer);
-+void bpf_wq_cancel_and_free(void *timer);
-+void bpf_task_work_cancel_and_free(void *timer);
-+void bpf_list_head_free(const struct btf_field *field, void *list_head,
-+			struct bpf_spin_lock *spin_lock);
-+void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
-+		      struct bpf_spin_lock *spin_lock);
-+u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena);
-+u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena);
- int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size);
- 
- struct bpf_offload_dev;
-@@ -348,6 +663,16 @@ int map_check_no_btf(const struct bpf_map *map,
- bool bpf_map_meta_equal(const struct bpf_map *meta0,
- 			const struct bpf_map *meta1);
- 
-+static inline bool bpf_map_has_internal_structs(struct bpf_map *map)
-+{
-+	return btf_record_has_field(map->record, BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK);
-+}
-+
-+void bpf_map_free_internal_structs(struct bpf_map *map, void *obj);
-+
-+int bpf_dynptr_from_file_sleepable(struct file *file, u32 flags,
-+				   struct bpf_dynptr *ptr__uninit);
-+
- extern const struct bpf_map_ops bpf_map_offload_ops;
- 
- /* bpf_type_flag contains a set of flags that are applicable to the values of
-@@ -369,10 +694,8 @@ enum bpf_type_flag {
- 	 */
- 	MEM_RDONLY		= BIT(1 + BPF_BASE_TYPE_BITS),
- 
-	/* MEM was "allocated" from a different helper, and cannot be mixed
-	 * with regular non-MEM_ALLOC'ed MEM types.
-	 */
-	MEM_ALLOC		= BIT(2 + BPF_BASE_TYPE_BITS),
-+	/* MEM points to BPF ring buffer reservation. */
-+	MEM_RINGBUF		= BIT(2 + BPF_BASE_TYPE_BITS),
- 
- 	/* MEM is in user address space. */
- 	MEM_USER		= BIT(3 + BPF_BASE_TYPE_BITS),
-@@ -396,22 +719,91 @@ enum bpf_type_flag {
- 	 */
- 	PTR_UNTRUSTED		= BIT(6 + BPF_BASE_TYPE_BITS),
- 
-+	/* MEM can be uninitialized. */
- 	MEM_UNINIT		= BIT(7 + BPF_BASE_TYPE_BITS),
- 
- 	/* DYNPTR points to memory local to the bpf program. */
- 	DYNPTR_TYPE_LOCAL	= BIT(8 + BPF_BASE_TYPE_BITS),
- 
-	/* DYNPTR points to a ringbuf record. */
-+	/* DYNPTR points to a kernel-produced ringbuf record. */
- 	DYNPTR_TYPE_RINGBUF	= BIT(9 + BPF_BASE_TYPE_BITS),
- 
- 	/* Size is known at compile time. */
- 	MEM_FIXED_SIZE		= BIT(10 + BPF_BASE_TYPE_BITS),
- 
-+	/* MEM is of an allocated object of type in program BTF. This is used to
-+	 * tag PTR_TO_BTF_ID allocated using bpf_obj_new.
-+	 */
-+	MEM_ALLOC		= BIT(11 + BPF_BASE_TYPE_BITS),
-+
-+	/* PTR was passed from the kernel in a trusted context, and may be
-+	 * passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions.
-+	 * Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above.
-+	 * PTR_UNTRUSTED refers to a kptr that was read directly from a map
-+	 * without invoking bpf_kptr_xchg(). What we really need to know is
-+	 * whether a pointer is safe to pass to a kfunc or BPF helper function.
-+	 * While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF
-+	 * helpers, they do not cover all possible instances of unsafe
-+	 * pointers. For example, a pointer that was obtained from walking a
-+	 * struct will _not_ get the PTR_UNTRUSTED type modifier, despite the
-+	 * fact that it may be NULL, invalid, etc. This is due to backwards
-+	 * compatibility requirements, as this was the behavior that was first
-+	 * introduced when kptrs were added. The behavior is now considered
-+	 * deprecated, and PTR_UNTRUSTED will eventually be removed.
-+	 *
-+	 * PTR_TRUSTED, on the other hand, is a pointer that the kernel
-+	 * guarantees to be valid and safe to pass to kfuncs and BPF helpers.
-+	 * For example, pointers passed to tracepoint arguments are considered
-+	 * PTR_TRUSTED, as are pointers that are passed to struct_ops
-+	 * callbacks. As alluded to above, pointers that are obtained from
-+	 * walking PTR_TRUSTED pointers are _not_ trusted. For example, if a
-+	 * struct task_struct *task is PTR_TRUSTED, then accessing
-+	 * task->last_wakee will lose the PTR_TRUSTED modifier when it's stored
-+	 * in a BPF register. Similarly, pointers passed to certain programs
-+	 * types such as kretprobes are not guaranteed to be valid, as they may
-+	 * for example contain an object that was recently freed.
-+	 */
-+	PTR_TRUSTED		= BIT(12 + BPF_BASE_TYPE_BITS),
-+
-+	/* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */
-+	MEM_RCU			= BIT(13 + BPF_BASE_TYPE_BITS),
-+
-+	/* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning.
-+	 * Currently only valid for linked-list and rbtree nodes. If the nodes
-+	 * have a bpf_refcount_field, they must be tagged MEM_RCU as well.
-+	 */
-+	NON_OWN_REF		= BIT(14 + BPF_BASE_TYPE_BITS),
-+
-+	/* DYNPTR points to sk_buff */
-+	DYNPTR_TYPE_SKB		= BIT(15 + BPF_BASE_TYPE_BITS),
-+
-+	/* DYNPTR points to xdp_buff */
-+	DYNPTR_TYPE_XDP		= BIT(16 + BPF_BASE_TYPE_BITS),
-+
-+	/* Memory must be aligned on some architectures, used in combination with
-+	 * MEM_FIXED_SIZE.
-+	 */
-+	MEM_ALIGNED		= BIT(17 + BPF_BASE_TYPE_BITS),
-+
-+	/* MEM is being written to, often combined with MEM_UNINIT. Non-presence
-+	 * of MEM_WRITE means that MEM is only being read. MEM_WRITE without the
-+	 * MEM_UNINIT means that memory needs to be initialized since it is also
-+	 * read.
-+	 */
-+	MEM_WRITE		= BIT(18 + BPF_BASE_TYPE_BITS),
-+
-+	/* DYNPTR points to skb_metadata_end()-skb_metadata_len() */
-+	DYNPTR_TYPE_SKB_META	= BIT(19 + BPF_BASE_TYPE_BITS),
-+
-+	/* DYNPTR points to file */
-+	DYNPTR_TYPE_FILE	= BIT(20 + BPF_BASE_TYPE_BITS),
-+
- 	__BPF_TYPE_FLAG_MAX,
- 	__BPF_TYPE_LAST_FLAG	= __BPF_TYPE_FLAG_MAX - 1,
- };
- 
-#define DYNPTR_TYPE_FLAG_MASK	(DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF)
-+#define DYNPTR_TYPE_FLAG_MASK	(DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \
-+				 | DYNPTR_TYPE_XDP | DYNPTR_TYPE_SKB_META | DYNPTR_TYPE_FILE)
- 
- /* Max number of base types. */
- #define BPF_BASE_TYPE_LIMIT	(1UL << BPF_BASE_TYPE_BITS)
-@@ -434,6 +826,7 @@ enum bpf_arg_type {
- 	 * on eBPF program stack
- 	 */
- 	ARG_PTR_TO_MEM,		/* pointer to valid memory (stack, packet, map value) */
-+	ARG_PTR_TO_ARENA,
- 
- 	ARG_CONST_SIZE,		/* number of bytes accessed from memory */
- 	ARG_CONST_SIZE_OR_ZERO,	/* number of bytes accessed from memory or 0 */
-@@ -442,11 +835,9 @@ enum bpf_arg_type {
- 	ARG_ANYTHING,		/* any (initialized) argument is ok */
- 	ARG_PTR_TO_SPIN_LOCK,	/* pointer to bpf_spin_lock */
- 	ARG_PTR_TO_SOCK_COMMON,	/* pointer to sock_common */
-	ARG_PTR_TO_INT,		/* pointer to int */
-	ARG_PTR_TO_LONG,	/* pointer to long */
- 	ARG_PTR_TO_SOCKET,	/* pointer to bpf_sock (fullsock) */
- 	ARG_PTR_TO_BTF_ID,	/* pointer to in-kernel struct */
-	ARG_PTR_TO_ALLOC_MEM,	/* pointer to dynamically allocated memory */
-+	ARG_PTR_TO_RINGBUF_MEM,	/* pointer to dynamically reserved ringbuf memory */
- 	ARG_CONST_ALLOC_SIZE_OR_ZERO,	/* number of allocated bytes requested */
- 	ARG_PTR_TO_BTF_ID_SOCK_COMMON,	/* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
- 	ARG_PTR_TO_PERCPU_BTF_ID,	/* pointer to in-kernel percpu type */
-@@ -454,7 +845,7 @@ enum bpf_arg_type {
- 	ARG_PTR_TO_STACK,	/* pointer to stack */
- 	ARG_PTR_TO_CONST_STR,	/* pointer to a null terminated read-only string */
- 	ARG_PTR_TO_TIMER,	/* pointer to bpf_timer */
-	ARG_PTR_TO_KPTR,	/* pointer to referenced kptr */
-+	ARG_KPTR_XCHG_DEST,	/* pointer to destination that kptrs are bpf_kptr_xchg'd into */
- 	ARG_PTR_TO_DYNPTR,      /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */
- 	__BPF_ARG_TYPE_MAX,
- 
-@@ -463,13 +854,12 @@ enum bpf_arg_type {
- 	ARG_PTR_TO_MEM_OR_NULL		= PTR_MAYBE_NULL | ARG_PTR_TO_MEM,
- 	ARG_PTR_TO_CTX_OR_NULL		= PTR_MAYBE_NULL | ARG_PTR_TO_CTX,
- 	ARG_PTR_TO_SOCKET_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET,
-	ARG_PTR_TO_ALLOC_MEM_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_ALLOC_MEM,
- 	ARG_PTR_TO_STACK_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_STACK,
- 	ARG_PTR_TO_BTF_ID_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_BTF_ID,
-	/* pointer to memory does not need to be initialized, helper function must fill
-	 * all bytes or clear them in error case.
-+	/* Pointer to memory does not need to be initialized, since helper function
-+	 * fills all bytes or clears them in error case.
- 	 */
-	ARG_PTR_TO_UNINIT_MEM		= MEM_UNINIT | ARG_PTR_TO_MEM,
-+	ARG_PTR_TO_UNINIT_MEM		= MEM_UNINIT | MEM_WRITE | ARG_PTR_TO_MEM,
- 	/* Pointer to valid memory of size known at compile time. */
- 	ARG_PTR_TO_FIXED_SIZE_MEM	= MEM_FIXED_SIZE | ARG_PTR_TO_MEM,
- 
-@@ -488,7 +878,7 @@ enum bpf_return_type {
- 	RET_PTR_TO_SOCKET,		/* returns a pointer to a socket */
- 	RET_PTR_TO_TCP_SOCK,		/* returns a pointer to a tcp_sock */
- 	RET_PTR_TO_SOCK_COMMON,		/* returns a pointer to a sock_common */
-	RET_PTR_TO_ALLOC_MEM,		/* returns a pointer to dynamically allocated memory */
-+	RET_PTR_TO_MEM,			/* returns a pointer to memory */
- 	RET_PTR_TO_MEM_OR_BTF_ID,	/* returns a pointer to a valid memory or a btf_id */
- 	RET_PTR_TO_BTF_ID,		/* returns a pointer to a btf_id */
- 	__BPF_RET_TYPE_MAX,
-@@ -498,9 +888,10 @@ enum bpf_return_type {
- 	RET_PTR_TO_SOCKET_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_SOCKET,
- 	RET_PTR_TO_TCP_SOCK_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK,
- 	RET_PTR_TO_SOCK_COMMON_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON,
-	RET_PTR_TO_ALLOC_MEM_OR_NULL	= PTR_MAYBE_NULL | MEM_ALLOC | RET_PTR_TO_ALLOC_MEM,
-	RET_PTR_TO_DYNPTR_MEM_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_ALLOC_MEM,
-+	RET_PTR_TO_RINGBUF_MEM_OR_NULL	= PTR_MAYBE_NULL | MEM_RINGBUF | RET_PTR_TO_MEM,
-+	RET_PTR_TO_DYNPTR_MEM_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_MEM,
- 	RET_PTR_TO_BTF_ID_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID,
-+	RET_PTR_TO_BTF_ID_TRUSTED	= PTR_TRUSTED	 | RET_PTR_TO_BTF_ID,
- 
- 	/* This must be the last entry. Its purpose is to ensure the enum is
- 	 * wide enough to hold the higher bits reserved for bpf_type_flag.
-@@ -517,6 +908,13 @@ struct bpf_func_proto {
- 	u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
- 	bool gpl_only;
- 	bool pkt_access;
-+	bool might_sleep;
-+	/* set to true if helper follows contract for llvm
-+	 * attribute bpf_fastcall:
-+	 * - void functions do not scratch r0
-+	 * - functions taking N arguments scratch only registers r1-rN
-+	 */
-+	bool allow_fastcall;
- 	enum bpf_return_type ret_type;
- 	union {
- 		struct {
-@@ -599,13 +997,12 @@ enum bpf_reg_type {
- 	 * additional context, assume the value is non-null.
- 	 */
- 	PTR_TO_BTF_ID,
-	/* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
-	 * been checked for null. Used primarily to inform the verifier
-	 * an explicit null check is required for this struct.
-	 */
- 	PTR_TO_MEM,		 /* reg points to valid memory region */
-+	PTR_TO_ARENA,
- 	PTR_TO_BUF,		 /* reg points to a read/write buffer */
- 	PTR_TO_FUNC,		 /* reg points to a bpf program function */
-+	PTR_TO_INSN,		 /* reg points to a bpf program instruction */
-+	CONST_PTR_TO_DYNPTR,	 /* reg points to a const struct bpf_dynptr */
- 	__BPF_REG_TYPE_MAX,
- 
- 	/* Extended reg_types. */
-@@ -613,6 +1010,10 @@ enum bpf_reg_type {
- 	PTR_TO_SOCKET_OR_NULL		= PTR_MAYBE_NULL | PTR_TO_SOCKET,
- 	PTR_TO_SOCK_COMMON_OR_NULL	= PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,
- 	PTR_TO_TCP_SOCK_OR_NULL		= PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,
-+	/* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
-+	 * been checked for null. Used primarily to inform the verifier
-+	 * an explicit null check is required for this struct.
-+	 */
- 	PTR_TO_BTF_ID_OR_NULL		= PTR_MAYBE_NULL | PTR_TO_BTF_ID,
- 
- 	/* This must be the last entry. Its purpose is to ensure the enum is
-@@ -627,14 +1028,17 @@ static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
-  */
- struct bpf_insn_access_aux {
- 	enum bpf_reg_type reg_type;
-+	bool is_ldsx;
- 	union {
- 		int ctx_field_size;
- 		struct {
- 			struct btf *btf;
- 			u32 btf_id;
-+			u32 ref_obj_id;
- 		};
- 	};
- 	struct bpf_verifier_log *log; /* for verbose logs */
-+	bool is_retval; /* is accessing function return value ? */
- };
- 
- static inline void
-@@ -643,10 +1047,29 @@ bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
- 	aux->ctx_field_size = size;
- }
- 
-+static bool bpf_is_ldimm64(const struct bpf_insn *insn)
-+{
-+	return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
-+}
-+
- static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
- {
-	return insn->code == (BPF_LD | BPF_IMM | BPF_DW) &&
-	       insn->src_reg == BPF_PSEUDO_FUNC;
-+	return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
-+}
-+
-+/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an
-+ * atomic load or store, and false if it is a read-modify-write instruction.
-+ */
-+static inline bool
-+bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn)
-+{
-+	switch (atomic_insn->imm) {
-+	case BPF_LOAD_ACQ:
-+	case BPF_STORE_REL:
-+		return true;
-+	default:
-+		return false;
-+	}
- }
- 
- struct bpf_prog_ops {
-@@ -654,6 +1077,7 @@ struct bpf_prog_ops {
- 			union bpf_attr __user *uattr);
- };
- 
-+struct bpf_reg_state;
- struct bpf_verifier_ops {
- 	/* return eBPF function prototype for verification */
- 	const struct bpf_func_proto *
-@@ -668,6 +1092,8 @@ struct bpf_verifier_ops {
- 				struct bpf_insn_access_aux *info);
- 	int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
- 			    const struct bpf_prog *prog);
-+	int (*gen_epilogue)(struct bpf_insn *insn, const struct bpf_prog *prog,
-+			    s16 ctx_stack_off);
- 	int (*gen_ld_abs)(const struct bpf_insn *orig,
- 			  struct bpf_insn *insn_buf);
- 	u32 (*convert_ctx_access)(enum bpf_access_type type,
-@@ -675,10 +1101,8 @@ struct bpf_verifier_ops {
- 				  struct bpf_insn *dst,
- 				  struct bpf_prog *prog, u32 *target_size);
- 	int (*btf_struct_access)(struct bpf_verifier_log *log,
-				 const struct btf *btf,
-				 const struct btf_type *t, int off, int size,
-				 enum bpf_access_type atype,
-				 u32 *next_btf_id, enum bpf_type_flag *flag);
-+				 const struct bpf_reg_state *reg,
-+				 int off, int size);
- };
- 
- struct bpf_prog_offload_ops {
-@@ -708,14 +1132,6 @@ struct bpf_prog_offload {
- 	u32			jited_len;
- };
- 
-enum bpf_cgroup_storage_type {
-	BPF_CGROUP_STORAGE_SHARED,
-	BPF_CGROUP_STORAGE_PERCPU,
-	__BPF_CGROUP_STORAGE_MAX
-};
-
-#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
-
- /* The longest tracepoint has 12 args.
-  * See include/trace/bpf_probe.h
-  */
-@@ -726,10 +1142,18 @@ enum bpf_cgroup_storage_type {
-  */
- #define MAX_BPF_FUNC_REG_ARGS 5
- 
-+/* The argument is a structure or a union. */
-+#define BTF_FMODEL_STRUCT_ARG		BIT(0)
-+
-+/* The argument is signed. */
-+#define BTF_FMODEL_SIGNED_ARG		BIT(1)
-+
- struct btf_func_model {
- 	u8 ret_size;
-+	u8 ret_flags;
- 	u8 nr_args;
- 	u8 arg_size[MAX_BPF_FUNC_ARGS];
-+	u8 arg_flags[MAX_BPF_FUNC_ARGS];
- };
- 
- /* Restore arguments before returning from trampoline to let original function
-@@ -762,10 +1186,32 @@ struct btf_func_model {
-  */
- #define BPF_TRAMP_F_SHARE_IPMODIFY	BIT(6)
- 
-+/* Indicate that current trampoline is in a tail call context. Then, it has to
-+ * cache and restore tail_call_cnt to avoid infinite tail call loop.
-+ */
-+#define BPF_TRAMP_F_TAIL_CALL_CTX	BIT(7)
-+
-+/*
-+ * Indicate the trampoline should be suitable to receive indirect calls;
-+ * without this indirectly calling the generated code can result in #UD/#CP,
-+ * depending on the CFI options.
-+ *
-+ * Used by bpf_struct_ops.
-+ *
-+ * Incompatible with FENTRY usage, overloads @func_addr argument.
-+ */
-+#define BPF_TRAMP_F_INDIRECT		BIT(8)
-+
- /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
-  * bytes on x86.
-  */
-#define BPF_MAX_TRAMP_LINKS 38
-+enum {
-+#if defined(__s390x__)
-+	BPF_MAX_TRAMP_LINKS = 27,
-+#else
-+	BPF_MAX_TRAMP_LINKS = 38,
-+#endif
-+};
- 
- struct bpf_tramp_links {
- 	struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];
-@@ -795,22 +1241,40 @@ struct bpf_tramp_run_ctx;
-  *      fexit = a set of program to run after original function
-  */
- struct bpf_tramp_image;
-int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,
-+int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
- 				const struct btf_func_model *m, u32 flags,
- 				struct bpf_tramp_links *tlinks,
-				void *orig_call);
-/* these two functions are called from generated trampoline */
-u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);
-void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx);
-u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);
-void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
-				       struct bpf_tramp_run_ctx *run_ctx);
-u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
-					struct bpf_tramp_run_ctx *run_ctx);
-void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
-					struct bpf_tramp_run_ctx *run_ctx);
-+				void *func_addr);
-+void *arch_alloc_bpf_trampoline(unsigned int size);
-+void arch_free_bpf_trampoline(void *image, unsigned int size);
-+int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size);
-+int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
-+			     struct bpf_tramp_links *tlinks, void *func_addr);
-+
-+u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
-+					     struct bpf_tramp_run_ctx *run_ctx);
-+void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
-+					     struct bpf_tramp_run_ctx *run_ctx);
- void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
- void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
-+typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog,
-+				      struct bpf_tramp_run_ctx *run_ctx);
-+typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start,
-+				      struct bpf_tramp_run_ctx *run_ctx);
-+bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog);
-+bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog);
-+
-+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP
-+static inline bool bpf_trampoline_use_jmp(u64 flags)
-+{
-+	return flags & BPF_TRAMP_F_CALL_ORIG && !(flags & BPF_TRAMP_F_SKIP_FRAME);
-+}
-+#else
-+static inline bool bpf_trampoline_use_jmp(u64 flags)
-+{
-+	return false;
-+}
-+#endif
- 
- struct bpf_ksym {
- 	unsigned long		 start;
-@@ -831,6 +1295,7 @@ enum bpf_tramp_prog_type {
- 
- struct bpf_tramp_image {
- 	void *image;
-+	int size;
- 	struct bpf_ksym ksym;
- 	struct percpu_ref pcref;
- 	void *ip_after_call;
-@@ -866,13 +1331,12 @@ struct bpf_trampoline {
- 	int progs_cnt[BPF_TRAMP_MAX];
- 	/* Executable image of trampoline */
- 	struct bpf_tramp_image *cur_image;
-	u64 selector;
-	struct module *mod;
- };
- 
- struct bpf_attach_target_info {
- 	struct btf_func_model fmodel;
- 	long tgt_addr;
-+	struct module *tgt_mod;
- 	const char *tgt_name;
- 	const struct btf_type *tgt_type;
- };
-@@ -891,11 +1355,20 @@ struct bpf_dispatcher {
- 	struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX];
- 	int num_progs;
- 	void *image;
-+	void *rw_image;
- 	u32 image_off;
- 	struct bpf_ksym ksym;
-+#ifdef CONFIG_HAVE_STATIC_CALL
-+	struct static_call_key *sc_key;
-+	void *sc_tramp;
-+#endif
- };
- 
-static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
-+#ifndef __bpfcall
-+#define __bpfcall __nocfi
-+#endif
-+
-+static __always_inline __bpfcall unsigned int bpf_dispatcher_nop_func(
- 	const void *ctx,
- 	const struct bpf_insn *insnsi,
- 	bpf_func_t bpf_func)
-@@ -903,13 +1376,97 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
- 	return bpf_func(ctx, insnsi);
- }
- 
-+/* the implementation of the opaque uapi struct bpf_dynptr */
-+struct bpf_dynptr_kern {
-+	void *data;
-+	/* Size represents the number of usable bytes of dynptr data.
-+	 * If for example the offset is at 4 for a local dynptr whose data is
-+	 * of type u64, the number of usable bytes is 4.
-+	 *
-+	 * The upper 8 bits are reserved. It is as follows:
-+	 * Bits 0 - 23 = size
-+	 * Bits 24 - 30 = dynptr type
-+	 * Bit 31 = whether dynptr is read-only
-+	 */
-+	u32 size;
-+	u32 offset;
-+} __aligned(8);
-+
-+enum bpf_dynptr_type {
-+	BPF_DYNPTR_TYPE_INVALID,
-+	/* Points to memory that is local to the bpf program */
-+	BPF_DYNPTR_TYPE_LOCAL,
-+	/* Underlying data is a ringbuf record */
-+	BPF_DYNPTR_TYPE_RINGBUF,
-+	/* Underlying data is a sk_buff */
-+	BPF_DYNPTR_TYPE_SKB,
-+	/* Underlying data is a xdp_buff */
-+	BPF_DYNPTR_TYPE_XDP,
-+	/* Points to skb_metadata_end()-skb_metadata_len() */
-+	BPF_DYNPTR_TYPE_SKB_META,
-+	/* Underlying data is a file */
-+	BPF_DYNPTR_TYPE_FILE,
-+};
-+
-+int bpf_dynptr_check_size(u64 size);
-+u64 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr);
-+const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u64 len);
-+void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u64 len);
-+bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);
-+int __bpf_dynptr_write(const struct bpf_dynptr_kern *dst, u64 offset,
-+		       void *src, u64 len, u64 flags);
-+void *bpf_dynptr_slice_rdwr(const struct bpf_dynptr *p, u64 offset,
-+			    void *buffer__opt, u64 buffer__szk);
-+
-+static inline int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u64 offset, u64 len)
-+{
-+	u64 size = __bpf_dynptr_size(ptr);
-+
-+	if (len > size || offset > size - len)
-+		return -E2BIG;
-+
-+	return 0;
-+}
-+
- #ifdef CONFIG_BPF_JIT
-int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
-int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
-+int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
-+			     struct bpf_trampoline *tr,
-+			     struct bpf_prog *tgt_prog);
-+int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
-+			       struct bpf_trampoline *tr,
-+			       struct bpf_prog *tgt_prog);
- struct bpf_trampoline *bpf_trampoline_get(u64 key,
- 					  struct bpf_attach_target_info *tgt_info);
- void bpf_trampoline_put(struct bpf_trampoline *tr);
-int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs);
-+int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);
-+
-+/*
-+ * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
-+ * indirection with a direct call to the bpf program. If the architecture does
-+ * not have STATIC_CALL, avoid a double-indirection.
-+ */
-+#ifdef CONFIG_HAVE_STATIC_CALL
-+
-+#define __BPF_DISPATCHER_SC_INIT(_name)				\
-+	.sc_key = &STATIC_CALL_KEY(_name),			\
-+	.sc_tramp = STATIC_CALL_TRAMP_ADDR(_name),
-+
-+#define __BPF_DISPATCHER_SC(name)				\
-+	DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func)
-+
-+#define __BPF_DISPATCHER_CALL(name)				\
-+	static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func)
-+
-+#define __BPF_DISPATCHER_UPDATE(_d, _new)			\
-+	__static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new))
-+
-+#else
-+#define __BPF_DISPATCHER_SC_INIT(name)
-+#define __BPF_DISPATCHER_SC(name)
-+#define __BPF_DISPATCHER_CALL(name)		bpf_func(ctx, insnsi)
-+#define __BPF_DISPATCHER_UPDATE(_d, _new)
-+#endif
-+
- #define BPF_DISPATCHER_INIT(_name) {				\
- 	.mutex = __MUTEX_INITIALIZER(_name.mutex),		\
- 	.func = &_name##_func,					\
-@@ -921,32 +1478,36 @@ int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs);
- 		.name  = #_name,				\
- 		.lnode = LIST_HEAD_INIT(_name.ksym.lnode),	\
- 	},							\
-+	__BPF_DISPATCHER_SC_INIT(_name##_call)			\
- }
- 
- #define DEFINE_BPF_DISPATCHER(name)					\
-	noinline __nocfi unsigned int bpf_dispatcher_##name##_func(	\
-+	__BPF_DISPATCHER_SC(name);					\
-+	noinline __bpfcall unsigned int bpf_dispatcher_##name##_func(	\
- 		const void *ctx,					\
- 		const struct bpf_insn *insnsi,				\
- 		bpf_func_t bpf_func)					\
- 	{								\
-		return bpf_func(ctx, insnsi);				\
-+		return __BPF_DISPATCHER_CALL(name);			\
- 	}								\
- 	EXPORT_SYMBOL(bpf_dispatcher_##name##_func);			\
- 	struct bpf_dispatcher bpf_dispatcher_##name =			\
- 		BPF_DISPATCHER_INIT(bpf_dispatcher_##name);
-+
- #define DECLARE_BPF_DISPATCHER(name)					\
- 	unsigned int bpf_dispatcher_##name##_func(			\
- 		const void *ctx,					\
- 		const struct bpf_insn *insnsi,				\
- 		bpf_func_t bpf_func);					\
- 	extern struct bpf_dispatcher bpf_dispatcher_##name;
-+
- #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func
- #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name)
- void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
- 				struct bpf_prog *to);
- /* Called only from JIT-enabled code, so there's no need for stubs. */
-void *bpf_jit_alloc_exec_page(void);
-void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym);
-+void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym);
-+void bpf_image_ksym_add(struct bpf_ksym *ksym);
- void bpf_image_ksym_del(struct bpf_ksym *ksym);
- void bpf_ksym_add(struct bpf_ksym *ksym);
- void bpf_ksym_del(struct bpf_ksym *ksym);
-@@ -955,19 +1516,21 @@ void bpf_jit_uncharge_modmem(u32 size);
- bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
- #else
- static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
-					   struct bpf_trampoline *tr)
-+					   struct bpf_trampoline *tr,
-+					   struct bpf_prog *tgt_prog)
- {
- 	return -ENOTSUPP;
- }
- static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
-					     struct bpf_trampoline *tr)
-+					     struct bpf_trampoline *tr,
-+					     struct bpf_prog *tgt_prog)
- {
- 	return -ENOTSUPP;
- }
- static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
- 							struct bpf_attach_target_info *tgt_info)
- {
-	return ERR_PTR(-EOPNOTSUPP);
-+	return NULL;
- }
- static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}
- #define DEFINE_BPF_DISPATCHER(name)
-@@ -990,6 +1553,8 @@ static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
- struct bpf_func_info_aux {
- 	u16 linkage;
- 	bool unreliable;
-+	bool called : 1;
-+	bool verified : 1;
- };
- 
- enum bpf_jit_poke_reason {
-@@ -1018,7 +1583,10 @@ struct bpf_jit_poke_descriptor {
- struct bpf_ctx_arg_aux {
- 	u32 offset;
- 	enum bpf_reg_type reg_type;
-+	struct btf *btf;
- 	u32 btf_id;
-+	u32 ref_obj_id;
-+	bool refcounted;
- };
- 
- struct btf_mod_pair {
-@@ -1028,6 +1596,37 @@ struct btf_mod_pair {
- 
- struct bpf_kfunc_desc_tab;
- 
-+enum bpf_stream_id {
-+	BPF_STDOUT = 1,
-+	BPF_STDERR = 2,
-+};
-+
-+struct bpf_stream_elem {
-+	struct llist_node node;
-+	int total_len;
-+	int consumed_len;
-+	char str[];
-+};
-+
-+enum {
-+	/* 100k bytes */
-+	BPF_STREAM_MAX_CAPACITY = 100000ULL,
-+};
-+
-+struct bpf_stream {
-+	atomic_t capacity;
-+	struct llist_head log;	/* list of in-flight stream elements in LIFO order */
-+
-+	struct mutex lock;  /* lock protecting backlog_{head,tail} */
-+	struct llist_node *backlog_head; /* list of in-flight stream elements in FIFO order */
-+	struct llist_node *backlog_tail; /* tail of the list above */
-+};
-+
-+struct bpf_stream_stage {
-+	struct llist_head log;
-+	int len;
-+};
-+
- struct bpf_prog_aux {
- 	atomic64_t refcnt;
- 	u32 used_map_cnt;
-@@ -1038,37 +1637,59 @@ struct bpf_prog_aux {
- 	u32 stack_depth;
- 	u32 id;
- 	u32 func_cnt; /* used by non-func prog as the number of func progs */
-+	u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */
- 	u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
- 	u32 attach_btf_id; /* in-kernel BTF type id to attach to */
-+	u32 attach_st_ops_member_off;
- 	u32 ctx_arg_info_size;
- 	u32 max_rdonly_access;
- 	u32 max_rdwr_access;
-+	u32 subprog_start;
- 	struct btf *attach_btf;
-	const struct bpf_ctx_arg_aux *ctx_arg_info;
-+	struct bpf_ctx_arg_aux *ctx_arg_info;
-+	void __percpu *priv_stack_ptr;
- 	struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
- 	struct bpf_prog *dst_prog;
- 	struct bpf_trampoline *dst_trampoline;
- 	enum bpf_prog_type saved_dst_prog_type;
- 	enum bpf_attach_type saved_dst_attach_type;
- 	bool verifier_zext; /* Zero extensions has been inserted by verifier. */
-	bool offload_requested;
-+	bool dev_bound; /* Program is bound to the netdev. */
-+	bool offload_requested; /* Program is bound and offloaded to the netdev. */
- 	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
-+	bool attach_tracing_prog; /* true if tracing another tracing program */
- 	bool func_proto_unreliable;
-	bool sleepable;
- 	bool tail_call_reachable;
- 	bool xdp_has_frags;
-+	bool exception_cb;
-+	bool exception_boundary;
-+	bool is_extended; /* true if extended by freplace program */
-+	bool jits_use_priv_stack;
-+	bool priv_stack_requested;
-+	bool changes_pkt_data;
-+	bool might_sleep;
-+	bool kprobe_write_ctx;
-+	u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
-+	struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
-+	struct bpf_arena *arena;
-+	void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */
- 	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
- 	const struct btf_type *attach_func_proto;
- 	/* function name for valid attach_btf_id */
- 	const char *attach_func_name;
- 	struct bpf_prog **func;
-+	struct bpf_prog_aux *main_prog_aux;
- 	void *jit_data; /* JIT specific data. arch dependent */
- 	struct bpf_jit_poke_descriptor *poke_tab;
- 	struct bpf_kfunc_desc_tab *kfunc_tab;
- 	struct bpf_kfunc_btf_tab *kfunc_btf_tab;
- 	u32 size_poke_tab;
-+#ifdef CONFIG_FINEIBT
-+	struct bpf_ksym ksym_prefix;
-+#endif
- 	struct bpf_ksym ksym;
- 	const struct bpf_prog_ops *ops;
-+	const struct bpf_struct_ops *st_ops;
- 	struct bpf_map **used_maps;
- 	struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
- 	struct btf_mod_pair *used_btfs;
-@@ -1079,9 +1700,11 @@ struct bpf_prog_aux {
- 	int cgroup_atype; /* enum cgroup_bpf_attach_type */
- 	struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
- 	char name[BPF_OBJ_NAME_LEN];
-+	u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);
- #ifdef CONFIG_SECURITY
- 	void *security;
- #endif
-+	struct bpf_token *token;
- 	struct bpf_prog_offload *offload;
- 	struct btf *btf;
- 	struct bpf_func_info *func_info;
-@@ -1108,12 +1731,14 @@ struct bpf_prog_aux {
- 	 * main prog always has linfo_idx == 0
- 	 */
- 	u32 linfo_idx;
-+	struct module *mod;
- 	u32 num_exentries;
- 	struct exception_table_entry *extable;
- 	union {
- 		struct work_struct work;
- 		struct rcu_head	rcu;
- 	};
-+	struct bpf_stream stream[2];
- };
- 
- struct bpf_prog {
-@@ -1131,12 +1756,16 @@ struct bpf_prog {
- 				enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
- 				call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
- 				call_get_func_ip:1, /* Do we call get_func_ip() */
-				tstamp_type_access:1; /* Accessed __sk_buff->tstamp_type */
-+				tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */
-+				sleepable:1;	/* BPF program is sleepable */
- 	enum bpf_prog_type	type;		/* Type of BPF program */
- 	enum bpf_attach_type	expected_attach_type; /* For some prog types */
- 	u32			len;		/* Number of filter blocks */
- 	u32			jited_len;	/* Size of jited insns in bytes */
-	u8			tag[BPF_TAG_SIZE];
-+	union {
-+		u8 digest[SHA256_DIGEST_SIZE];
-+		u8 tag[BPF_TAG_SIZE];
-+	};
- 	struct bpf_prog_stats __percpu *stats;
- 	int __percpu		*active;
- 	unsigned int		(*bpf_func)(const void *ctx,
-@@ -1164,18 +1793,46 @@ struct bpf_link {
- 	enum bpf_link_type type;
- 	const struct bpf_link_ops *ops;
- 	struct bpf_prog *prog;
-	struct work_struct work;
-+
-+	u32 flags;
-+	enum bpf_attach_type attach_type;
-+
-+	/* rcu is used before freeing, work can be used to schedule that
-+	 * RCU-based freeing before that, so they never overlap
-+	 */
-+	union {
-+		struct rcu_head rcu;
-+		struct work_struct work;
-+	};
-+	/* whether BPF link itself has "sleepable" semantics, which can differ
-+	 * from underlying BPF program having a "sleepable" semantics, as BPF
-+	 * link's semantics is determined by target attach hook
-+	 */
-+	bool sleepable;
- };
- 
- struct bpf_link_ops {
- 	void (*release)(struct bpf_link *link);
-+	/* deallocate link resources callback, called without RCU grace period
-+	 * waiting
-+	 */
- 	void (*dealloc)(struct bpf_link *link);
-+	/* deallocate link resources callback, called after RCU grace period;
-+	 * if either the underlying BPF program is sleepable or BPF link's
-+	 * target hook is sleepable, we'll go through tasks trace RCU GP and
-+	 * then "classic" RCU GP; this need for chaining tasks trace and
-+	 * classic RCU GPs is designated by setting bpf_link->sleepable flag
-+	 */
-+	void (*dealloc_deferred)(struct bpf_link *link);
- 	int (*detach)(struct bpf_link *link);
- 	int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog,
- 			   struct bpf_prog *old_prog);
- 	void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq);
- 	int (*fill_link_info)(const struct bpf_link *link,
- 			      struct bpf_link_info *info);
-+	int (*update_map)(struct bpf_link *link, struct bpf_map *new_map,
-+			  struct bpf_map *old_map);
-+	__poll_t (*poll)(struct file *file, struct poll_table_struct *pts);
- };
- 
- struct bpf_tramp_link {
-@@ -1191,11 +1848,16 @@ struct bpf_shim_tramp_link {
- 
- struct bpf_tracing_link {
- 	struct bpf_tramp_link link;
-	enum bpf_attach_type attach_type;
- 	struct bpf_trampoline *trampoline;
- 	struct bpf_prog *tgt_prog;
- };
- 
-+struct bpf_raw_tp_link {
-+	struct bpf_link link;
-+	struct bpf_raw_event_map *btp;
-+	u64 cookie;
-+};
-+
- struct bpf_link_primer {
- 	struct bpf_link *link;
- 	struct file *file;
-@@ -1203,40 +1865,167 @@ struct bpf_link_primer {
- 	u32 id;
- };
- 
-+struct bpf_mount_opts {
-+	kuid_t uid;
-+	kgid_t gid;
-+	umode_t mode;
-+
-+	/* BPF token-related delegation options */
-+	u64 delegate_cmds;
-+	u64 delegate_maps;
-+	u64 delegate_progs;
-+	u64 delegate_attachs;
-+};
-+
-+struct bpf_token {
-+	struct work_struct work;
-+	atomic64_t refcnt;
-+	struct user_namespace *userns;
-+	u64 allowed_cmds;
-+	u64 allowed_maps;
-+	u64 allowed_progs;
-+	u64 allowed_attachs;
-+#ifdef CONFIG_SECURITY
-+	void *security;
-+#endif
-+};
-+
- struct bpf_struct_ops_value;
- struct btf_member;
- 
- #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
-+/**
-+ * struct bpf_struct_ops - A structure of callbacks allowing a subsystem to
-+ *			   define a BPF_MAP_TYPE_STRUCT_OPS map type composed
-+ *			   of BPF_PROG_TYPE_STRUCT_OPS progs.
-+ * @verifier_ops: A structure of callbacks that are invoked by the verifier
-+ *		  when determining whether the struct_ops progs in the
-+ *		  struct_ops map are valid.
-+ * @init: A callback that is invoked a single time, and before any other
-+ *	  callback, to initialize the structure. A nonzero return value means
-+ *	  the subsystem could not be initialized.
-+ * @check_member: When defined, a callback invoked by the verifier to allow
-+ *		  the subsystem to determine if an entry in the struct_ops map
-+ *		  is valid. A nonzero return value means that the map is
-+ *		  invalid and should be rejected by the verifier.
-+ * @init_member: A callback that is invoked for each member of the struct_ops
-+ *		 map to allow the subsystem to initialize the member. A nonzero
-+ *		 value means the member could not be initialized. This callback
-+ *		 is exclusive with the @type, @type_id, @value_type, and
-+ *		 @value_id fields.
-+ * @reg: A callback that is invoked when the struct_ops map has been
-+ *	 initialized and is being attached to. Zero means the struct_ops map
-+ *	 has been successfully registered and is live. A nonzero return value
-+ *	 means the struct_ops map could not be registered.
-+ * @unreg: A callback that is invoked when the struct_ops map should be
-+ *	   unregistered.
-+ * @update: A callback that is invoked when the live struct_ops map is being
-+ *	    updated to contain new values. This callback is only invoked when
-+ *	    the struct_ops map is loaded with BPF_F_LINK. If not defined, the
-+ *	    it is assumed that the struct_ops map cannot be updated.
-+ * @validate: A callback that is invoked after all of the members have been
-+ *	      initialized. This callback should perform static checks on the
-+ *	      map, meaning that it should either fail or succeed
-+ *	      deterministically. A struct_ops map that has been validated may
-+ *	      not necessarily succeed in being registered if the call to @reg
-+ *	      fails. For example, a valid struct_ops map may be loaded, but
-+ *	      then fail to be registered due to there being another active
-+ *	      struct_ops map on the system in the subsystem already. For this
-+ *	      reason, if this callback is not defined, the check is skipped as
-+ *	      the struct_ops map will have final verification performed in
-+ *	      @reg.
-+ * @cfi_stubs: Pointer to a structure of stub functions for CFI. These stubs
-+ *	       provide the correct Control Flow Integrity hashes for the
-+ *	       trampolines generated by BPF struct_ops.
-+ * @owner: The module that owns this struct_ops. Used for module reference
-+ *	   counting to ensure the module providing the struct_ops cannot be
-+ *	   unloaded while in use.
-+ * @name: The name of the struct bpf_struct_ops object.
-+ * @func_models: Func models
-+ */
- struct bpf_struct_ops {
- 	const struct bpf_verifier_ops *verifier_ops;
- 	int (*init)(struct btf *btf);
- 	int (*check_member)(const struct btf_type *t,
-			    const struct btf_member *member);
-+			    const struct btf_member *member,
-+			    const struct bpf_prog *prog);
- 	int (*init_member)(const struct btf_type *t,
- 			   const struct btf_member *member,
- 			   void *kdata, const void *udata);
-	int (*reg)(void *kdata);
-	void (*unreg)(void *kdata);
-	const struct btf_type *type;
-	const struct btf_type *value_type;
-+	int (*reg)(void *kdata, struct bpf_link *link);
-+	void (*unreg)(void *kdata, struct bpf_link *link);
-+	int (*update)(void *kdata, void *old_kdata, struct bpf_link *link);
-+	int (*validate)(void *kdata);
-+	void *cfi_stubs;
-+	struct module *owner;
- 	const char *name;
- 	struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
-+};
-+
-+/* Every member of a struct_ops type has an instance even a member is not
-+ * an operator (function pointer). The "info" field will be assigned to
-+ * prog->aux->ctx_arg_info of BPF struct_ops programs to provide the
-+ * argument information required by the verifier to verify the program.
-+ *
-+ * btf_ctx_access() will lookup prog->aux->ctx_arg_info to find the
-+ * corresponding entry for an given argument.
-+ */
-+struct bpf_struct_ops_arg_info {
-+	struct bpf_ctx_arg_aux *info;
-+	u32 cnt;
-+};
-+
-+struct bpf_struct_ops_desc {
-+	struct bpf_struct_ops *st_ops;
-+
-+	const struct btf_type *type;
-+	const struct btf_type *value_type;
- 	u32 type_id;
- 	u32 value_id;
-+
-+	/* Collection of argument information for each member */
-+	struct bpf_struct_ops_arg_info *arg_info;
-+};
-+
-+enum bpf_struct_ops_state {
-+	BPF_STRUCT_OPS_STATE_INIT,
-+	BPF_STRUCT_OPS_STATE_INUSE,
-+	BPF_STRUCT_OPS_STATE_TOBEFREE,
-+	BPF_STRUCT_OPS_STATE_READY,
-+};
-+
-+struct bpf_struct_ops_common_value {
-+	refcount_t refcnt;
-+	enum bpf_struct_ops_state state;
- };
- 
- #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
-+/* This macro helps developer to register a struct_ops type and generate
-+ * type information correctly. Developers should use this macro to register
-+ * a struct_ops type instead of calling __register_bpf_struct_ops() directly.
-+ */
-+#define register_bpf_struct_ops(st_ops, type)				\
-+	({								\
-+		struct bpf_struct_ops_##type {				\
-+			struct bpf_struct_ops_common_value common;	\
-+			struct type data ____cacheline_aligned_in_smp;	\
-+		};							\
-+		BTF_TYPE_EMIT(struct bpf_struct_ops_##type);		\
-+		__register_bpf_struct_ops(st_ops);			\
-+	})
- #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
-const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id);
-void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log);
- bool bpf_struct_ops_get(const void *kdata);
- void bpf_struct_ops_put(const void *kdata);
-+int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff);
- int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
- 				       void *value);
- int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
- 				      struct bpf_tramp_link *link,
- 				      const struct btf_func_model *model,
-				      void *image, void *image_end);
-+				      void *stub_func,
-+				      void **image, u32 *image_off,
-+				      bool allow_alloc);
-+void bpf_struct_ops_image_free(void *image);
- static inline bool bpf_try_module_get(const void *data, struct module *owner)
- {
- 	if (owner == BPF_MODULE_OWNER)
-@@ -1251,6 +2040,8 @@ static inline void bpf_module_put(const void *data, struct module *owner)
- 	else
- 		module_put(owner);
- }
-+int bpf_struct_ops_link_create(union bpf_attr *attr);
-+u32 bpf_struct_ops_id(const void *kdata);
- 
- #ifdef CONFIG_NET
- /* Define it here to avoid the use of forward declaration */
-@@ -1262,20 +2053,19 @@ struct bpf_dummy_ops {
- 	int (*test_1)(struct bpf_dummy_ops_state *cb);
- 	int (*test_2)(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2,
- 		      char a3, unsigned long a4);
-+	int (*test_sleepable)(struct bpf_dummy_ops_state *cb);
- };
- 
- int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
- 			    union bpf_attr __user *uattr);
- #endif
-+int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
-+			     struct btf *btf,
-+			     struct bpf_verifier_log *log);
-+void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map);
-+void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc);
- #else
-static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
-{
-	return NULL;
-}
-static inline void bpf_struct_ops_init(struct btf *btf,
-				       struct bpf_verifier_log *log)
-{
-}
-+#define register_bpf_struct_ops(st_ops, type) ({ (void *)(st_ops); 0; })
- static inline bool bpf_try_module_get(const void *data, struct module *owner)
- {
- 	return try_module_get(owner);
-@@ -1284,21 +2074,42 @@ static inline void bpf_module_put(const void *data, struct module *owner)
- {
- 	module_put(owner);
- }
-+static inline int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
-+{
-+	return -ENOTSUPP;
-+}
- static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,
- 						     void *key,
- 						     void *value)
- {
- 	return -EINVAL;
- }
-+static inline int bpf_struct_ops_link_create(union bpf_attr *attr)
-+{
-+	return -EOPNOTSUPP;
-+}
-+static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
-+{
-+}
-+
-+static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)
-+{
-+}
-+
- #endif
- 
-+int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
-+			       const struct bpf_ctx_arg_aux *info, u32 cnt);
-+
- #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
- int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
-				    int cgroup_atype);
-+				    int cgroup_atype,
-+				    enum bpf_attach_type attach_type);
- void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
- #else
- static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
-						  int cgroup_atype)
-+						  int cgroup_atype,
-+						  enum bpf_attach_type attach_type)
- {
- 	return -EOPNOTSUPP;
- }
-@@ -1313,17 +2124,28 @@ struct bpf_array {
- 	u32 index_mask;
- 	struct bpf_array_aux *aux;
- 	union {
-		char value[0] __aligned(8);
-		void *ptrs[0] __aligned(8);
-		void __percpu *pptrs[0] __aligned(8);
-+		DECLARE_FLEX_ARRAY(char, value) __aligned(8);
-+		DECLARE_FLEX_ARRAY(void *, ptrs) __aligned(8);
-+		DECLARE_FLEX_ARRAY(void __percpu *, pptrs) __aligned(8);
- 	};
- };
- 
-+/*
-+ * The bpf_array_get_next_key() function may be used for all array-like
-+ * maps, i.e., maps with u32 keys with range [0 ,..., max_entries)
-+ */
-+int bpf_array_get_next_key(struct bpf_map *map, void *key, void *next_key);
-+
- #define BPF_COMPLEXITY_LIMIT_INSNS      1000000 /* yes. 1M insns */
- #define MAX_TAIL_CALL_CNT 33
- 
-/* Maximum number of loops for bpf_loop */
-#define BPF_MAX_LOOPS	BIT(23)
-+/* Maximum number of loops for bpf_loop and bpf_iter_num.
-+ * It's enum to expose it (and thus make it discoverable) through BTF.
-+ */
-+enum {
-+	BPF_MAX_LOOPS = 8 * 1024 * 1024,
-+	BPF_MAX_TIMED_LOOPS = 0xffff,
-+};
- 
- #define BPF_F_ACCESS_MASK	(BPF_F_RDONLY |		\
- 				 BPF_F_RDONLY_PROG |	\
-@@ -1333,6 +2155,11 @@ struct bpf_array {
- #define BPF_MAP_CAN_READ	BIT(0)
- #define BPF_MAP_CAN_WRITE	BIT(1)
- 
-+/* Maximum number of user-producer ring buffer samples that can be drained in
-+ * a call to bpf_user_ringbuf_drain().
-+ */
-+#define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024)
-+
- static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)
- {
- 	u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
-@@ -1354,6 +2181,16 @@ static inline bool bpf_map_flags_access_ok(u32 access_flags)
- 	       (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
- }
- 
-+static inline struct bpf_map_owner *bpf_map_owner_alloc(struct bpf_map *map)
-+{
-+	return kzalloc(sizeof(*map->owner), GFP_ATOMIC);
-+}
-+
-+static inline void bpf_map_owner_free(struct bpf_map *map)
-+{
-+	kfree(map->owner);
-+}
-+
- struct bpf_event_entry {
- 	struct perf_event *event;
- 	struct file *perf_file;
-@@ -1374,6 +2211,8 @@ int bpf_prog_calc_tag(struct bpf_prog *fp);
- const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
- const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);
- 
-+const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void);
-+
- typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
- 					unsigned long off, unsigned long len);
- typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
-@@ -1457,6 +2296,7 @@ struct bpf_cg_run_ctx {
- struct bpf_trace_run_ctx {
- 	struct bpf_run_ctx run_ctx;
- 	u64 bpf_cookie;
-+	bool is_uprobe;
- };
- 
- struct bpf_tramp_run_ctx {
-@@ -1505,6 +2345,8 @@ bpf_prog_run_array(const struct bpf_prog_array *array,
- 	if (unlikely(!array))
- 		return ret;
- 
-+	run_ctx.is_uprobe = false;
-+
- 	migrate_disable();
- 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
- 	item = &array->items[0];
-@@ -1529,44 +2371,49 @@ bpf_prog_run_array(const struct bpf_prog_array *array,
-  * rcu-protected dynamically sized maps.
-  */
- static __always_inline u32
-bpf_prog_run_array_sleepable(const struct bpf_prog_array __rcu *array_rcu,
-			     const void *ctx, bpf_prog_run_fn run_prog)
-+bpf_prog_run_array_uprobe(const struct bpf_prog_array *array,
-+			  const void *ctx, bpf_prog_run_fn run_prog)
- {
- 	const struct bpf_prog_array_item *item;
- 	const struct bpf_prog *prog;
-	const struct bpf_prog_array *array;
- 	struct bpf_run_ctx *old_run_ctx;
- 	struct bpf_trace_run_ctx run_ctx;
- 	u32 ret = 1;
- 
- 	might_fault();
-+	RCU_LOCKDEP_WARN(!rcu_read_lock_trace_held(), "no rcu lock held");
-+
-+	if (unlikely(!array))
-+		return ret;
- 
-	rcu_read_lock_trace();
- 	migrate_disable();
- 
-	array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held());
-	if (unlikely(!array))
-		goto out;
-+	run_ctx.is_uprobe = true;
-+
- 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
- 	item = &array->items[0];
- 	while ((prog = READ_ONCE(item->prog))) {
-		if (!prog->aux->sleepable)
-+		if (!prog->sleepable)
- 			rcu_read_lock();
- 
- 		run_ctx.bpf_cookie = item->bpf_cookie;
- 		ret &= run_prog(prog, ctx);
- 		item++;
- 
-		if (!prog->aux->sleepable)
-+		if (!prog->sleepable)
- 			rcu_read_unlock();
- 	}
- 	bpf_reset_run_ctx(old_run_ctx);
-out:
- 	migrate_enable();
-	rcu_read_unlock_trace();
- 	return ret;
- }
- 
-+bool bpf_jit_bypass_spec_v1(void);
-+bool bpf_jit_bypass_spec_v4(void);
-+
-+#define bpf_rcu_lock_held() \
-+	(rcu_read_lock_held() || rcu_read_lock_trace_held() || rcu_read_lock_bh_held())
-+
- #ifdef CONFIG_BPF_SYSCALL
- DECLARE_PER_CPU(int, bpf_prog_active);
- extern struct mutex bpf_stats_enabled_mutex;
-@@ -1589,9 +2436,11 @@ static inline void bpf_enable_instrumentation(void)
- 	migrate_enable();
- }
- 
-+extern const struct super_operations bpf_super_ops;
- extern const struct file_operations bpf_map_fops;
- extern const struct file_operations bpf_prog_fops;
- extern const struct file_operations bpf_iter_fops;
-+extern const struct file_operations bpf_token_fops;
- 
- #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
- 	extern const struct bpf_prog_ops _name ## _prog_ops; \
-@@ -1617,20 +2466,53 @@ void bpf_prog_inc(struct bpf_prog *prog);
- struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
- void bpf_prog_put(struct bpf_prog *prog);
- 
-void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock);
-void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock);
-
-struct bpf_map_value_off_desc *bpf_map_kptr_off_contains(struct bpf_map *map, u32 offset);
-void bpf_map_free_kptr_off_tab(struct bpf_map *map);
-struct bpf_map_value_off *bpf_map_copy_kptr_off_tab(const struct bpf_map *map);
-bool bpf_map_equal_kptr_off_tab(const struct bpf_map *map_a, const struct bpf_map *map_b);
-void bpf_map_free_kptrs(struct bpf_map *map, void *map_value);
-+void bpf_prog_free_id(struct bpf_prog *prog);
-+void bpf_map_free_id(struct bpf_map *map);
-+
-+struct btf_field *btf_record_find(const struct btf_record *rec,
-+				  u32 offset, u32 field_mask);
-+void btf_record_free(struct btf_record *rec);
-+void bpf_map_free_record(struct bpf_map *map);
-+struct btf_record *btf_record_dup(const struct btf_record *rec);
-+bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b);
-+void bpf_obj_free_timer(const struct btf_record *rec, void *obj);
-+void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj);
-+void bpf_obj_free_task_work(const struct btf_record *rec, void *obj);
-+void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
-+void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);
- 
- struct bpf_map *bpf_map_get(u32 ufd);
- struct bpf_map *bpf_map_get_with_uref(u32 ufd);
-struct bpf_map *__bpf_map_get(struct fd f);
-+
-+/*
-+ * The __bpf_map_get() and __btf_get_by_fd() functions parse a file
-+ * descriptor and return a corresponding map or btf object.
-+ * Their names are double underscored to emphasize the fact that they
-+ * do not increase refcnt. To also increase refcnt use corresponding
-+ * bpf_map_get() and btf_get_by_fd() functions.
-+ */
-+
-+static inline struct bpf_map *__bpf_map_get(struct fd f)
-+{
-+	if (fd_empty(f))
-+		return ERR_PTR(-EBADF);
-+	if (unlikely(fd_file(f)->f_op != &bpf_map_fops))
-+		return ERR_PTR(-EINVAL);
-+	return fd_file(f)->private_data;
-+}
-+
-+static inline struct btf *__btf_get_by_fd(struct fd f)
-+{
-+	if (fd_empty(f))
-+		return ERR_PTR(-EBADF);
-+	if (unlikely(fd_file(f)->f_op != &btf_fops))
-+		return ERR_PTR(-EINVAL);
-+	return fd_file(f)->private_data;
-+}
-+
- void bpf_map_inc(struct bpf_map *map);
- void bpf_map_inc_with_uref(struct bpf_map *map);
-+struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref);
- struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map);
- void bpf_map_put_with_uref(struct bpf_map *map);
- void bpf_map_put(struct bpf_map *map);
-@@ -1642,7 +2524,7 @@ void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
- int  generic_map_lookup_batch(struct bpf_map *map,
- 			      const union bpf_attr *attr,
- 			      union bpf_attr __user *uattr);
-int  generic_map_update_batch(struct bpf_map *map,
-+int  generic_map_update_batch(struct bpf_map *map, struct file *map_file,
- 			      const union bpf_attr *attr,
- 			      union bpf_attr __user *uattr);
- int  generic_map_delete_batch(struct bpf_map *map,
-@@ -1651,86 +2533,170 @@ int  generic_map_delete_batch(struct bpf_map *map,
- struct bpf_map *bpf_map_get_curr_or_next(u32 *id);
- struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);
- 
-#ifdef CONFIG_MEMCG_KMEM
-+
-+int bpf_map_alloc_pages(const struct bpf_map *map, int nid,
-+			unsigned long nr_pages, struct page **page_array);
-+#ifdef CONFIG_MEMCG
- void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
- 			   int node);
-+void *bpf_map_kmalloc_nolock(const struct bpf_map *map, size_t size, gfp_t flags,
-+			     int node);
- void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags);
-+void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
-+		       gfp_t flags);
- void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
- 				    size_t align, gfp_t flags);
- #else
-static inline void *
-bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
-		     int node)
-+/*
-+ * These specialized allocators have to be macros for their allocations to be
-+ * accounted separately (to have separate alloc_tag).
-+ */
-+#define bpf_map_kmalloc_node(_map, _size, _flags, _node)	\
-+		kmalloc_node(_size, _flags, _node)
-+#define bpf_map_kmalloc_nolock(_map, _size, _flags, _node)	\
-+		kmalloc_nolock(_size, _flags, _node)
-+#define bpf_map_kzalloc(_map, _size, _flags)			\
-+		kzalloc(_size, _flags)
-+#define bpf_map_kvcalloc(_map, _n, _size, _flags)		\
-+		kvcalloc(_n, _size, _flags)
-+#define bpf_map_alloc_percpu(_map, _size, _align, _flags)	\
-+		__alloc_percpu_gfp(_size, _align, _flags)
-+#endif
-+
-+static inline int
-+bpf_map_init_elem_count(struct bpf_map *map)
-+{
-+	size_t size = sizeof(*map->elem_count), align = size;
-+	gfp_t flags = GFP_USER | __GFP_NOWARN;
-+
-+	map->elem_count = bpf_map_alloc_percpu(map, size, align, flags);
-+	if (!map->elem_count)
-+		return -ENOMEM;
-+
-+	return 0;
-+}
-+
-+static inline void
-+bpf_map_free_elem_count(struct bpf_map *map)
- {
-	return kmalloc_node(size, flags, node);
-+	free_percpu(map->elem_count);
- }
- 
-static inline void *
-bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
-+static inline void bpf_map_inc_elem_count(struct bpf_map *map)
- {
-	return kzalloc(size, flags);
-+	this_cpu_inc(*map->elem_count);
- }
- 
-static inline void __percpu *
-bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align,
-		     gfp_t flags)
-+static inline void bpf_map_dec_elem_count(struct bpf_map *map)
- {
-	return __alloc_percpu_gfp(size, align, flags);
-+	this_cpu_dec(*map->elem_count);
- }
-#endif
- 
- extern int sysctl_unprivileged_bpf_disabled;
- 
-static inline bool bpf_allow_ptr_leaks(void)
-{
-	return perfmon_capable();
-}
-+bool bpf_token_capable(const struct bpf_token *token, int cap);
- 
-static inline bool bpf_allow_uninit_stack(void)
-+static inline bool bpf_allow_ptr_leaks(const struct bpf_token *token)
- {
-	return perfmon_capable();
-+	return bpf_token_capable(token, CAP_PERFMON);
- }
- 
-static inline bool bpf_allow_ptr_to_map_access(void)
-+static inline bool bpf_allow_uninit_stack(const struct bpf_token *token)
- {
-	return perfmon_capable();
-+	return bpf_token_capable(token, CAP_PERFMON);
- }
- 
-static inline bool bpf_bypass_spec_v1(void)
-+static inline bool bpf_bypass_spec_v1(const struct bpf_token *token)
- {
-	return perfmon_capable();
-+	return bpf_jit_bypass_spec_v1() ||
-+		cpu_mitigations_off() ||
-+		bpf_token_capable(token, CAP_PERFMON);
- }
- 
-static inline bool bpf_bypass_spec_v4(void)
-+static inline bool bpf_bypass_spec_v4(const struct bpf_token *token)
- {
-	return perfmon_capable();
-+	return bpf_jit_bypass_spec_v4() ||
-+		cpu_mitigations_off() ||
-+		bpf_token_capable(token, CAP_PERFMON);
- }
- 
- int bpf_map_new_fd(struct bpf_map *map, int flags);
- int bpf_prog_new_fd(struct bpf_prog *prog);
- 
- void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
-		   const struct bpf_link_ops *ops, struct bpf_prog *prog);
-+		   const struct bpf_link_ops *ops, struct bpf_prog *prog,
-+		   enum bpf_attach_type attach_type);
-+void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
-+			     const struct bpf_link_ops *ops, struct bpf_prog *prog,
-+			     enum bpf_attach_type attach_type, bool sleepable);
- int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer);
- int bpf_link_settle(struct bpf_link_primer *primer);
- void bpf_link_cleanup(struct bpf_link_primer *primer);
- void bpf_link_inc(struct bpf_link *link);
-+struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link);
- void bpf_link_put(struct bpf_link *link);
- int bpf_link_new_fd(struct bpf_link *link);
-struct file *bpf_link_new_file(struct bpf_link *link, int *reserved_fd);
- struct bpf_link *bpf_link_get_from_fd(u32 ufd);
- struct bpf_link *bpf_link_get_curr_or_next(u32 *id);
- 
-int bpf_obj_pin_user(u32 ufd, const char __user *pathname);
-int bpf_obj_get_user(const char __user *pathname, int flags);
-+void bpf_token_inc(struct bpf_token *token);
-+void bpf_token_put(struct bpf_token *token);
-+int bpf_token_create(union bpf_attr *attr);
-+struct bpf_token *bpf_token_get_from_fd(u32 ufd);
-+int bpf_token_get_info_by_fd(struct bpf_token *token,
-+			     const union bpf_attr *attr,
-+			     union bpf_attr __user *uattr);
-+
-+bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd);
-+bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type);
-+bool bpf_token_allow_prog_type(const struct bpf_token *token,
-+			       enum bpf_prog_type prog_type,
-+			       enum bpf_attach_type attach_type);
-+
-+int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname);
-+int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags);
-+struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir,
-+			    umode_t mode);
- 
- #define BPF_ITER_FUNC_PREFIX "bpf_iter_"
- #define DEFINE_BPF_ITER_FUNC(target, args...)			\
- 	extern int bpf_iter_ ## target(args);			\
- 	int __init bpf_iter_ ## target(args) { return 0; }
- 
-+/*
-+ * The task type of iterators.
-+ *
-+ * For BPF task iterators, they can be parameterized with various
-+ * parameters to visit only some of tasks.
-+ *
-+ * BPF_TASK_ITER_ALL (default)
-+ *	Iterate over resources of every task.
-+ *
-+ * BPF_TASK_ITER_TID
-+ *	Iterate over resources of a task/tid.
-+ *
-+ * BPF_TASK_ITER_TGID
-+ *	Iterate over resources of every task of a process / task group.
-+ */
-+enum bpf_iter_task_type {
-+	BPF_TASK_ITER_ALL = 0,
-+	BPF_TASK_ITER_TID,
-+	BPF_TASK_ITER_TGID,
-+};
-+
- struct bpf_iter_aux_info {
-+	/* for map_elem iter */
- 	struct bpf_map *map;
-+
-+	/* for cgroup iter */
-+	struct {
-+		struct cgroup *start; /* starting cgroup */
-+		enum bpf_cgroup_iter_order order;
-+	} cgroup;
-+	struct {
-+		enum bpf_iter_task_type	type;
-+		u32 pid;
-+	} task;
- };
- 
- typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog,
-@@ -1778,7 +2744,7 @@ struct bpf_iter__bpf_map_elem {
- 
- int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
- void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
-bool bpf_iter_prog_supported(struct bpf_prog *prog);
-+int bpf_iter_prog_supported(struct bpf_prog *prog);
- const struct bpf_func_proto *
- bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
- int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog);
-@@ -1802,7 +2768,7 @@ int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
- int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
- 			    u64 flags);
- 
-int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);
-+int bpf_stackmap_extract(struct bpf_map *map, void *key, void *value, bool delete);
- 
- int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
- 				 void *key, void *value, u64 map_flags);
-@@ -1815,24 +2781,8 @@ int bpf_get_file_flag(int flags);
- int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
- 			     size_t actual_size);
- 
-/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
- * forced to use 'long' read/writes to try to atomically copy long counters.
- * Best-effort only.  No barriers here, since it _will_ race with concurrent
- * updates from BPF programs. Called from bpf syscall and mostly used with
- * size 8 or 16 bytes, so ask compiler to inline it.
- */
-static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
-{
-	const long *lsrc = src;
-	long *ldst = dst;
-
-	size /= sizeof(long);
-	while (size--)
-		*ldst++ = *lsrc++;
-}
-
- /* verify correctness of eBPF program */
-int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr);
-+int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);
- 
- #ifndef CONFIG_BPF_JIT_ALWAYS_ON
- void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
-@@ -1846,7 +2796,7 @@ struct sk_buff;
- struct bpf_dtab_netdev;
- struct bpf_cpu_map_entry;
- 
-void __dev_flush(void);
-+void __dev_flush(struct list_head *flush_list);
- int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
- 		    struct net_device *dev_rx);
- int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
-@@ -1854,12 +2804,12 @@ int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
- int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
- 			  struct bpf_map *map, bool exclude_ingress);
- int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
-			     struct bpf_prog *xdp_prog);
-+			     const struct bpf_prog *xdp_prog);
- int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
-			   struct bpf_prog *xdp_prog, struct bpf_map *map,
-			   bool exclude_ingress);
-+			   const struct bpf_prog *xdp_prog,
-+			   struct bpf_map *map, bool exclude_ingress);
- 
-void __cpu_map_flush(void);
-+void __cpu_map_flush(struct list_head *flush_list);
- int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
- 		    struct net_device *dev_rx);
- int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
-@@ -1891,6 +2841,9 @@ int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
- int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
- 				const union bpf_attr *kattr,
- 				union bpf_attr __user *uattr);
-+int bpf_prog_test_run_nf(struct bpf_prog *prog,
-+			 const union bpf_attr *kattr,
-+			 union bpf_attr __user *uattr);
- bool btf_ctx_access(int off, int size, enum bpf_access_type type,
- 		    const struct bpf_prog *prog,
- 		    struct bpf_insn_access_aux *info);
-@@ -1917,10 +2870,10 @@ static inline bool bpf_tracing_btf_ctx_access(int off, int size,
- 	return btf_ctx_access(off, size, type, prog, info);
- }
- 
-int btf_struct_access(struct bpf_verifier_log *log, const struct btf *btf,
-		      const struct btf_type *t, int off, int size,
-		      enum bpf_access_type atype,
-		      u32 *next_btf_id, enum bpf_type_flag *flag);
-+int btf_struct_access(struct bpf_verifier_log *log,
-+		      const struct bpf_reg_state *reg,
-+		      int off, int size, enum bpf_access_type atype,
-+		      u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name);
- bool btf_struct_ids_match(struct bpf_verifier_log *log,
- 			  const struct btf *btf, u32 id, int off,
- 			  const struct btf *need_btf, u32 need_type_id,
-@@ -1933,31 +2886,41 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
- 			   struct btf_func_model *m);
- 
- struct bpf_reg_state;
-int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
-				struct bpf_reg_state *regs);
-int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
-			      const struct btf *btf, u32 func_id,
-			      struct bpf_reg_state *regs,
-			      u32 kfunc_flags);
-int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
-			  struct bpf_reg_state *reg);
-+int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog);
- int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
- 			 struct btf *btf, const struct btf_type *t);
-+const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt,
-+				    int comp_idx, const char *tag_key);
-+int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt,
-+			   int comp_idx, const char *tag_key, int last_id);
- 
- struct bpf_prog *bpf_prog_by_id(u32 id);
- struct bpf_link *bpf_link_by_id(u32 id);
- 
-const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id);
-+const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id,
-+						 const struct bpf_prog *prog);
- void bpf_task_storage_free(struct task_struct *task);
-+void bpf_cgrp_storage_free(struct cgroup *cgroup);
- bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
- const struct btf_func_model *
- bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
- 			 const struct bpf_insn *insn);
-+int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
-+		       u16 btf_fd_idx, u8 **func_addr);
-+
- struct bpf_core_ctx {
- 	struct bpf_verifier_log *log;
- 	const struct btf *btf;
- };
- 
-+bool btf_nested_type_is_trusted(struct bpf_verifier_log *log,
-+				const struct bpf_reg_state *reg,
-+				const char *field_name, u32 btf_id, const char *suffix);
-+
-+bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log,
-+			       const struct btf *reg_btf, u32 reg_id,
-+			       const struct btf *arg_btf, u32 arg_id);
-+
- int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
- 		   int relo_idx, void *insn);
- 
-@@ -1966,6 +2929,24 @@ static inline bool unprivileged_ebpf_enabled(void)
- 	return !sysctl_unprivileged_bpf_disabled;
- }
- 
-+/* Not all bpf prog type has the bpf_ctx.
-+ * For the bpf prog type that has initialized the bpf_ctx,
-+ * this function can be used to decide if a kernel function
-+ * is called by a bpf program.
-+ */
-+static inline bool has_current_bpf_ctx(void)
-+{
-+	return !!current->bpf_ctx;
-+}
-+
-+void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
-+
-+void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
-+		     enum bpf_dynptr_type type, u32 offset, u32 size);
-+void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
-+void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
-+void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip);
-+
- #else /* !CONFIG_BPF_SYSCALL */
- static inline struct bpf_prog *bpf_prog_get(u32 ufd)
- {
-@@ -2003,7 +2984,13 @@ bpf_prog_inc_not_zero(struct bpf_prog *prog)
- 
- static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
- 				 const struct bpf_link_ops *ops,
-				 struct bpf_prog *prog)
-+				 struct bpf_prog *prog, enum bpf_attach_type attach_type)
-+{
-+}
-+
-+static inline void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
-+					   const struct bpf_link_ops *ops, struct bpf_prog *prog,
-+					   enum bpf_attach_type attach_type, bool sleepable)
- {
- }
- 
-@@ -2026,6 +3013,11 @@ static inline void bpf_link_inc(struct bpf_link *link)
- {
- }
- 
-+static inline struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
-+{
-+	return NULL;
-+}
-+
- static inline void bpf_link_put(struct bpf_link *link)
- {
- }
-@@ -2035,7 +3027,32 @@ static inline int bpf_obj_get_user(const char __user *pathname, int flags)
- 	return -EOPNOTSUPP;
- }
- 
-static inline void __dev_flush(void)
-+static inline bool bpf_token_capable(const struct bpf_token *token, int cap)
-+{
-+	return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN));
-+}
-+
-+static inline void bpf_token_inc(struct bpf_token *token)
-+{
-+}
-+
-+static inline void bpf_token_put(struct bpf_token *token)
-+{
-+}
-+
-+static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd)
-+{
-+	return ERR_PTR(-EOPNOTSUPP);
-+}
-+
-+static inline int bpf_token_get_info_by_fd(struct bpf_token *token,
-+					   const union bpf_attr *attr,
-+					   union bpf_attr __user *uattr)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline void __dev_flush(struct list_head *flush_list)
- {
- }
- 
-@@ -2068,20 +3085,20 @@ struct sk_buff;
- 
- static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,
- 					   struct sk_buff *skb,
-					   struct bpf_prog *xdp_prog)
-+					   const struct bpf_prog *xdp_prog)
- {
- 	return 0;
- }
- 
- static inline
- int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
-			   struct bpf_prog *xdp_prog, struct bpf_map *map,
-			   bool exclude_ingress)
-+			   const struct bpf_prog *xdp_prog,
-+			   struct bpf_map *map, bool exclude_ingress)
- {
- 	return 0;
- }
- 
-static inline void __cpu_map_flush(void)
-+static inline void __cpu_map_flush(struct list_head *flush_list)
- {
- }
- 
-@@ -2148,8 +3165,17 @@ static inline struct bpf_prog *bpf_prog_by_id(u32 id)
- 	return ERR_PTR(-ENOTSUPP);
- }
- 
-+static inline int btf_struct_access(struct bpf_verifier_log *log,
-+				    const struct bpf_reg_state *reg,
-+				    int off, int size, enum bpf_access_type atype,
-+				    u32 *next_btf_id, enum bpf_type_flag *flag,
-+				    const char **field_name)
-+{
-+	return -EACCES;
-+}
-+
- static inline const struct bpf_func_proto *
-bpf_base_func_proto(enum bpf_func_id func_id)
-+bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
- {
- 	return NULL;
- }
-@@ -2170,15 +3196,63 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
- 	return NULL;
- }
- 
-+static inline int
-+bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
-+		   u16 btf_fd_idx, u8 **func_addr)
-+{
-+	return -ENOTSUPP;
-+}
-+
- static inline bool unprivileged_ebpf_enabled(void)
- {
- 	return false;
- }
- 
-+static inline bool has_current_bpf_ctx(void)
-+{
-+	return false;
-+}
-+
-+static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
-+{
-+}
-+
-+static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)
-+{
-+}
-+
-+static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
-+				   enum bpf_dynptr_type type, u32 offset, u32 size)
-+{
-+}
-+
-+static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
-+{
-+}
-+
-+static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
-+{
-+}
-+
-+static inline void bpf_prog_report_arena_violation(bool write, unsigned long addr,
-+						   unsigned long fault_ip)
-+{
-+}
- #endif /* CONFIG_BPF_SYSCALL */
- 
-void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
-			  struct btf_mod_pair *used_btfs, u32 len);
-+static __always_inline int
-+bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr)
-+{
-+	int ret = -EFAULT;
-+
-+	if (IS_ENABLED(CONFIG_BPF_EVENTS))
-+		ret = copy_from_kernel_nofault(dst, unsafe_ptr, size);
-+	if (unlikely(ret < 0))
-+		memset(dst, 0, size);
-+	return ret;
-+}
-+
-+void __bpf_free_used_btfs(struct btf_mod_pair *used_btfs, u32 len);
- 
- static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
- 						 enum bpf_prog_type type)
-@@ -2192,7 +3266,7 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux,
- bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool);
- 
- int bpf_prog_offload_compile(struct bpf_prog *prog);
-void bpf_prog_offload_destroy(struct bpf_prog *prog);
-+void bpf_prog_dev_bound_destroy(struct bpf_prog *prog);
- int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
- 			       struct bpf_prog *prog);
- 
-@@ -2220,20 +3294,33 @@ bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);
- void unpriv_ebpf_notify(int new_state);
- 
- #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
-int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr);
-+int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
-+			      struct bpf_prog_aux *prog_aux);
-+void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id);
-+int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr);
-+int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog);
-+void bpf_dev_bound_netdev_unregister(struct net_device *dev);
- 
- static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
- {
-+	return aux->dev_bound;
-+}
-+
-+static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux *aux)
-+{
- 	return aux->offload_requested;
- }
- 
-static inline bool bpf_map_is_dev_bound(struct bpf_map *map)
-+bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs);
-+
-+static inline bool bpf_map_is_offloaded(struct bpf_map *map)
- {
- 	return unlikely(map->ops == &bpf_map_offload_ops);
- }
- 
- struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr);
- void bpf_map_offload_map_free(struct bpf_map *map);
-+u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map);
- int bpf_prog_test_run_syscall(struct bpf_prog *prog,
- 			      const union bpf_attr *kattr,
- 			      union bpf_attr __user *uattr);
-@@ -2243,23 +3330,56 @@ int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype);
- int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags);
- int sock_map_bpf_prog_query(const union bpf_attr *attr,
- 			    union bpf_attr __user *uattr);
-+int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog);
- 
- void sock_map_unhash(struct sock *sk);
- void sock_map_destroy(struct sock *sk);
- void sock_map_close(struct sock *sk, long timeout);
- #else
-static inline int bpf_prog_offload_init(struct bpf_prog *prog,
-					union bpf_attr *attr)
-+static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
-+					    struct bpf_prog_aux *prog_aux)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog,
-+						u32 func_id)
-+{
-+	return NULL;
-+}
-+
-+static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog,
-+					  union bpf_attr *attr)
- {
- 	return -EOPNOTSUPP;
- }
- 
-static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux)
-+static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog,
-+					     struct bpf_prog *old_prog)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev)
-+{
-+}
-+
-+static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
-+{
-+	return false;
-+}
-+
-+static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux *aux)
-+{
-+	return false;
-+}
-+
-+static inline bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs)
- {
- 	return false;
- }
- 
-static inline bool bpf_map_is_dev_bound(struct bpf_map *map)
-+static inline bool bpf_map_is_offloaded(struct bpf_map *map)
- {
- 	return false;
- }
-@@ -2273,6 +3393,11 @@ static inline void bpf_map_offload_map_free(struct bpf_map *map)
- {
- }
- 
-+static inline u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map)
-+{
-+	return 0;
-+}
-+
- static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog,
- 					    const union bpf_attr *kattr,
- 					    union bpf_attr __user *uattr)
-@@ -2304,9 +3429,30 @@ static inline int sock_map_bpf_prog_query(const union bpf_attr *attr,
- {
- 	return -EINVAL;
- }
-+
-+static inline int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog)
-+{
-+	return -EOPNOTSUPP;
-+}
- #endif /* CONFIG_BPF_SYSCALL */
- #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */
- 
-+static __always_inline void
-+bpf_prog_inc_misses_counters(const struct bpf_prog_array *array)
-+{
-+	const struct bpf_prog_array_item *item;
-+	struct bpf_prog *prog;
-+
-+	if (unlikely(!array))
-+		return;
-+
-+	item = &array->items[0];
-+	while ((prog = READ_ONCE(item->prog))) {
-+		bpf_prog_inc_misses_counter(prog);
-+		item++;
-+	}
-+}
-+
- #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
- void bpf_sk_reuseport_detach(struct sock *sk);
- int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,
-@@ -2334,6 +3480,38 @@ static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
- #endif /* CONFIG_BPF_SYSCALL */
- #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */
- 
-+#if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL)
-+
-+struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags);
-+struct bpf_key *bpf_lookup_system_key(u64 id);
-+void bpf_key_put(struct bpf_key *bkey);
-+int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p,
-+			       struct bpf_dynptr *sig_p,
-+			       struct bpf_key *trusted_keyring);
-+
-+#else
-+static inline struct bpf_key *bpf_lookup_user_key(u32 serial, u64 flags)
-+{
-+	return NULL;
-+}
-+
-+static inline struct bpf_key *bpf_lookup_system_key(u64 id)
-+{
-+	return NULL;
-+}
-+
-+static inline void bpf_key_put(struct bpf_key *bkey)
-+{
-+}
-+
-+static inline int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p,
-+					     struct bpf_dynptr *sig_p,
-+					     struct bpf_key *trusted_keyring)
-+{
-+	return -EOPNOTSUPP;
-+}
-+#endif /* defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) */
-+
- /* verifier prototypes for helper functions called from eBPF programs */
- extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
- extern const struct bpf_func_proto bpf_map_update_elem_proto;
-@@ -2349,18 +3527,23 @@ extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
- extern const struct bpf_func_proto bpf_tail_call_proto;
- extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
- extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto;
-+extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto;
- extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
- extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
- extern const struct bpf_func_proto bpf_get_current_comm_proto;
- extern const struct bpf_func_proto bpf_get_stackid_proto;
- extern const struct bpf_func_proto bpf_get_stack_proto;
-+extern const struct bpf_func_proto bpf_get_stack_sleepable_proto;
- extern const struct bpf_func_proto bpf_get_task_stack_proto;
-+extern const struct bpf_func_proto bpf_get_task_stack_sleepable_proto;
- extern const struct bpf_func_proto bpf_get_stackid_proto_pe;
- extern const struct bpf_func_proto bpf_get_stack_proto_pe;
- extern const struct bpf_func_proto bpf_sock_map_update_proto;
- extern const struct bpf_func_proto bpf_sock_hash_update_proto;
- extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
- extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto;
-+extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto;
-+extern const struct bpf_func_proto bpf_current_task_under_cgroup_proto;
- extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
- extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
- extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;
-@@ -2397,7 +3580,9 @@ extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
- extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;
- extern const struct bpf_func_proto bpf_sock_from_file_proto;
- extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;
-+extern const struct bpf_func_proto bpf_task_storage_get_recur_proto;
- extern const struct bpf_func_proto bpf_task_storage_get_proto;
-+extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto;
- extern const struct bpf_func_proto bpf_task_storage_delete_proto;
- extern const struct bpf_func_proto bpf_for_each_map_elem_proto;
- extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;
-@@ -2410,6 +3595,9 @@ extern const struct bpf_func_proto bpf_loop_proto;
- extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
- extern const struct bpf_func_proto bpf_set_retval_proto;
- extern const struct bpf_func_proto bpf_get_retval_proto;
-+extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto;
-+extern const struct bpf_func_proto bpf_cgrp_storage_get_proto;
-+extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto;
- 
- const struct bpf_func_proto *tracing_prog_func_proto(
-   enum bpf_func_id func_id, const struct bpf_prog *prog);
-@@ -2430,6 +3618,8 @@ u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
- 				struct bpf_insn *insn_buf,
- 				struct bpf_prog *prog,
- 				u32 *target_size);
-+int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
-+			       struct bpf_dynptr *ptr);
- #else
- static inline bool bpf_sock_common_is_valid_access(int off, int size,
- 						   enum bpf_access_type type,
-@@ -2451,6 +3641,11 @@ static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
- {
- 	return 0;
- }
-+static inline int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
-+					     struct bpf_dynptr *ptr)
-+{
-+	return -EOPNOTSUPP;
-+}
- #endif
- 
- #ifdef CONFIG_INET
-@@ -2515,12 +3710,17 @@ static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
- #endif /* CONFIG_INET */
- 
- enum bpf_text_poke_type {
-+	BPF_MOD_NOP,
- 	BPF_MOD_CALL,
- 	BPF_MOD_JUMP,
- };
- 
-int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
-		       void *addr1, void *addr2);
-+int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
-+		       enum bpf_text_poke_type new_t, void *old_addr,
-+		       void *new_addr);
-+
-+void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
-+			       struct bpf_prog *new, struct bpf_prog *old);
- 
- void *bpf_arch_text_copy(void *dst, void *src, size_t len);
- int bpf_arch_text_invalidate(void *dst, size_t len);
-@@ -2529,39 +3729,52 @@ struct btf_id_set;
- bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
- 
- #define MAX_BPRINTF_VARARGS		12
-+#define MAX_BPRINTF_BUF			1024
- 
-int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
-			u32 **bin_buf, u32 num_args);
-void bpf_bprintf_cleanup(void);
-
-/* the implementation of the opaque uapi struct bpf_dynptr */
-struct bpf_dynptr_kern {
-	void *data;
-	/* Size represents the number of usable bytes of dynptr data.
-	 * If for example the offset is at 4 for a local dynptr whose data is
-	 * of type u64, the number of usable bytes is 4.
-	 *
-	 * The upper 8 bits are reserved. It is as follows:
-	 * Bits 0 - 23 = size
-	 * Bits 24 - 30 = dynptr type
-	 * Bit 31 = whether dynptr is read-only
-	 */
-	u32 size;
-	u32 offset;
-} __aligned(8);
-
-enum bpf_dynptr_type {
-	BPF_DYNPTR_TYPE_INVALID,
-	/* Points to memory that is local to the bpf program */
-	BPF_DYNPTR_TYPE_LOCAL,
-	/* Underlying data is a ringbuf record */
-	BPF_DYNPTR_TYPE_RINGBUF,
-};
-
-void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
-		     enum bpf_dynptr_type type, u32 offset, u32 size);
-void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
-int bpf_dynptr_check_size(u32 size);
-+/* Per-cpu temp buffers used by printf-like helpers to store the bprintf binary
-+ * arguments representation.
-+ */
-+#define MAX_BPRINTF_BIN_ARGS	512
-+
-+struct bpf_bprintf_buffers {
-+	char bin_args[MAX_BPRINTF_BIN_ARGS];
-+	char buf[MAX_BPRINTF_BUF];
-+};
-+
-+struct bpf_bprintf_data {
-+	u32 *bin_args;
-+	char *buf;
-+	bool get_bin_args;
-+	bool get_buf;
-+};
-+
-+int bpf_bprintf_prepare(const char *fmt, u32 fmt_size, const u64 *raw_args,
-+			u32 num_args, struct bpf_bprintf_data *data);
-+void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);
-+int bpf_try_get_buffers(struct bpf_bprintf_buffers **bufs);
-+void bpf_put_buffers(void);
-+
-+void bpf_prog_stream_init(struct bpf_prog *prog);
-+void bpf_prog_stream_free(struct bpf_prog *prog);
-+int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len);
-+void bpf_stream_stage_init(struct bpf_stream_stage *ss);
-+void bpf_stream_stage_free(struct bpf_stream_stage *ss);
-+__printf(2, 3)
-+int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...);
-+int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog,
-+			    enum bpf_stream_id stream_id);
-+int bpf_stream_stage_dump_stack(struct bpf_stream_stage *ss);
-+
-+#define bpf_stream_printk(ss, ...) bpf_stream_stage_printk(&ss, __VA_ARGS__)
-+#define bpf_stream_dump_stack(ss) bpf_stream_stage_dump_stack(&ss)
-+
-+#define bpf_stream_stage(ss, prog, stream_id, expr)            \
-+	({                                                     \
-+		bpf_stream_stage_init(&ss);                    \
-+		(expr);                                        \
-+		bpf_stream_stage_commit(&ss, prog, stream_id); \
-+		bpf_stream_stage_free(&ss);                    \
-+	})
- 
- #ifdef CONFIG_BPF_LSM
- void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);
-@@ -2571,4 +3784,60 @@ static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {}
- static inline void bpf_cgroup_atype_put(int cgroup_atype) {}
- #endif /* CONFIG_BPF_LSM */
- 
-+struct key;
-+
-+#ifdef CONFIG_KEYS
-+struct bpf_key {
-+	struct key *key;
-+	bool has_ref;
-+};
-+#endif /* CONFIG_KEYS */
-+
-+static inline bool type_is_alloc(u32 type)
-+{
-+	return type & MEM_ALLOC;
-+}
-+
-+static inline gfp_t bpf_memcg_flags(gfp_t flags)
-+{
-+	if (memcg_bpf_enabled())
-+		return flags | __GFP_ACCOUNT;
-+	return flags;
-+}
-+
-+static inline bool bpf_is_subprog(const struct bpf_prog *prog)
-+{
-+	return prog->aux->func_idx != 0;
-+}
-+
-+int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep,
-+			   const char **linep, int *nump);
-+struct bpf_prog *bpf_prog_find_from_stack(void);
-+
-+int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog);
-+int bpf_insn_array_ready(struct bpf_map *map);
-+void bpf_insn_array_release(struct bpf_map *map);
-+void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len);
-+void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len);
-+
-+#ifdef CONFIG_BPF_SYSCALL
-+void bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image);
-+#else
-+static inline void
-+bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image)
-+{
-+}
-+#endif
-+
-+static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags)
-+{
-+	if (flags & ~allowed_flags)
-+		return -EINVAL;
-+
-+	if ((flags & BPF_F_LOCK) && !btf_record_has_field(map->record, BPF_SPIN_LOCK))
-+		return -EINVAL;
-+
-+	return 0;
-+}
-+
- #endif /* _LINUX_BPF_H */
-diff --git a/include/linux/bpf_crypto.h b/include/linux/bpf_crypto.h
-new file mode 100644
-index 000000000000..a41e71d4e2d9
--- /dev/null
-+++ b/include/linux/bpf_crypto.h
-@@ -0,0 +1,24 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
-+#ifndef _BPF_CRYPTO_H
-+#define _BPF_CRYPTO_H
-+
-+struct bpf_crypto_type {
-+	void *(*alloc_tfm)(const char *algo);
-+	void (*free_tfm)(void *tfm);
-+	int (*has_algo)(const char *algo);
-+	int (*setkey)(void *tfm, const u8 *key, unsigned int keylen);
-+	int (*setauthsize)(void *tfm, unsigned int authsize);
-+	int (*encrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv);
-+	int (*decrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv);
-+	unsigned int (*ivsize)(void *tfm);
-+	unsigned int (*statesize)(void *tfm);
-+	u32 (*get_flags)(void *tfm);
-+	struct module *owner;
-+	char name[14];
-+};
-+
-+int bpf_crypto_register_type(const struct bpf_crypto_type *type);
-+int bpf_crypto_unregister_type(const struct bpf_crypto_type *type);
-+
-+#endif /* _BPF_CRYPTO_H */
-diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h
-index 7ea18d4da84b..66432248cd81 100644
--- a/include/linux/bpf_local_storage.h
-+++ b/include/linux/bpf_local_storage.h
-@@ -13,13 +13,11 @@
- #include <linux/list.h>
- #include <linux/hash.h>
- #include <linux/types.h>
-+#include <linux/bpf_mem_alloc.h>
- #include <uapi/linux/btf.h>
- 
- #define BPF_LOCAL_STORAGE_CACHE_SIZE	16
- 
-#define bpf_rcu_lock_held()                                                    \
-	(rcu_read_lock_held() || rcu_read_lock_trace_held() ||                 \
-	 rcu_read_lock_bh_held())
- struct bpf_local_storage_map_bucket {
- 	struct hlist_head list;
- 	raw_spinlock_t lock;
-@@ -55,6 +53,7 @@ struct bpf_local_storage_map {
- 	u32 bucket_log;
- 	u16 elem_size;
- 	u16 cache_idx;
-+	bool use_kmalloc_nolock;
- };
- 
- struct bpf_local_storage_data {
-@@ -73,7 +72,13 @@ struct bpf_local_storage_elem {
- 	struct hlist_node map_node;	/* Linked to bpf_local_storage_map */
- 	struct hlist_node snode;	/* Linked to bpf_local_storage */
- 	struct bpf_local_storage __rcu *local_storage;
-	struct rcu_head rcu;
-+	union {
-+		struct rcu_head rcu;
-+		struct hlist_node free_node;	/* used to postpone
-+						 * bpf_selem_free
-+						 * after raw_spin_unlock
-+						 */
-+	};
- 	/* 8 bytes hole */
- 	/* The data is stored in another cacheline to minimize
- 	 * the number of cachelines access during a cache hit.
-@@ -83,12 +88,14 @@ struct bpf_local_storage_elem {
- 
- struct bpf_local_storage {
- 	struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE];
-+	struct bpf_local_storage_map __rcu *smap;
- 	struct hlist_head list; /* List of bpf_local_storage_elem */
- 	void *owner;		/* The object that owns the above "list" of
- 				 * bpf_local_storage_elem.
- 				 */
- 	struct rcu_head rcu;
- 	raw_spinlock_t lock;	/* Protect adding/removing from the "list" */
-+	bool use_kmalloc_nolock;
- };
- 
- /* U16_MAX is much more than enough for sk local storage
-@@ -116,21 +123,49 @@ static struct bpf_local_storage_cache name = {			\
- 	.idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock),	\
- }
- 
-u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache);
-void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache,
-				      u16 idx);
-
- /* Helper functions for bpf_local_storage */
- int bpf_local_storage_map_alloc_check(union bpf_attr *attr);
- 
-struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr);
-+struct bpf_map *
-+bpf_local_storage_map_alloc(union bpf_attr *attr,
-+			    struct bpf_local_storage_cache *cache,
-+			    bool use_kmalloc_nolock);
- 
-struct bpf_local_storage_data *
-+void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage,
-+				      struct bpf_local_storage_map *smap,
-+				      struct bpf_local_storage_elem *selem);
-+/* If cacheit_lockit is false, this lookup function is lockless */
-+static inline struct bpf_local_storage_data *
- bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
- 			 struct bpf_local_storage_map *smap,
-			 bool cacheit_lockit);
-+			 bool cacheit_lockit)
-+{
-+	struct bpf_local_storage_data *sdata;
-+	struct bpf_local_storage_elem *selem;
-+
-+	/* Fast path (cache hit) */
-+	sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx],
-+				      bpf_rcu_lock_held());
-+	if (sdata && rcu_access_pointer(sdata->smap) == smap)
-+		return sdata;
-+
-+	/* Slow path (cache miss) */
-+	hlist_for_each_entry_rcu(selem, &local_storage->list, snode,
-+				  rcu_read_lock_trace_held())
-+		if (rcu_access_pointer(SDATA(selem)->smap) == smap)
-+			break;
-+
-+	if (!selem)
-+		return NULL;
-+	if (cacheit_lockit)
-+		__bpf_local_storage_insert_cache(local_storage, smap, selem);
-+	return SDATA(selem);
-+}
- 
-void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,
-+void bpf_local_storage_destroy(struct bpf_local_storage *local_storage);
-+
-+void bpf_local_storage_map_free(struct bpf_map *map,
-+				struct bpf_local_storage_cache *cache,
- 				int __percpu *busy_counter);
- 
- int bpf_local_storage_map_check_btf(const struct bpf_map *map,
-@@ -141,20 +176,17 @@ int bpf_local_storage_map_check_btf(const struct bpf_map *map,
- void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
- 				   struct bpf_local_storage_elem *selem);
- 
-bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
-				     struct bpf_local_storage_elem *selem,
-				     bool uncharge_omem, bool use_trace_rcu);
-
-void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool use_trace_rcu);
-+void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now);
- 
- void bpf_selem_link_map(struct bpf_local_storage_map *smap,
- 			struct bpf_local_storage_elem *selem);
- 
-void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem);
-
- struct bpf_local_storage_elem *
- bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
-		bool charge_mem, gfp_t gfp_flags);
-+		bool swap_uptrs, gfp_t gfp_flags);
-+
-+void bpf_selem_free(struct bpf_local_storage_elem *selem,
-+		    bool reuse_now);
- 
- int
- bpf_local_storage_alloc(void *owner,
-@@ -164,8 +196,8 @@ bpf_local_storage_alloc(void *owner,
- 
- struct bpf_local_storage_data *
- bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
-			 void *value, u64 map_flags, gfp_t gfp_flags);
-+			 void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags);
- 
-void bpf_local_storage_free_rcu(struct rcu_head *rcu);
-+u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map);
- 
- #endif /* _BPF_LOCAL_STORAGE_H */
-diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h
-index 4bcf76a9bb06..643809cc78c3 100644
--- a/include/linux/bpf_lsm.h
-+++ b/include/linux/bpf_lsm.h
-@@ -9,6 +9,7 @@
- 
- #include <linux/sched.h>
- #include <linux/bpf.h>
-+#include <linux/bpf_verifier.h>
- #include <linux/lsm_hooks.h>
- 
- #ifdef CONFIG_BPF_LSM
-@@ -28,6 +29,7 @@ int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
- 			const struct bpf_prog *prog);
- 
- bool bpf_lsm_is_sleepable_hook(u32 btf_id);
-+bool bpf_lsm_is_trusted(const struct bpf_prog *prog);
- 
- static inline struct bpf_storage_blob *bpf_inode(
- 	const struct inode *inode)
-@@ -44,6 +46,13 @@ void bpf_inode_storage_free(struct inode *inode);
- 
- void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, bpf_func_t *bpf_func);
- 
-+int bpf_lsm_get_retval_range(const struct bpf_prog *prog,
-+			     struct bpf_retval_range *range);
-+int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str,
-+				const struct bpf_dynptr *value_p, int flags);
-+int bpf_remove_dentry_xattr_locked(struct dentry *dentry, const char *name__str);
-+bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog);
-+
- #else /* !CONFIG_BPF_LSM */
- 
- static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)
-@@ -51,6 +60,11 @@ static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)
- 	return false;
- }
- 
-+static inline bool bpf_lsm_is_trusted(const struct bpf_prog *prog)
-+{
-+	return false;
-+}
-+
- static inline int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
- 				      const struct bpf_prog *prog)
- {
-@@ -72,6 +86,24 @@ static inline void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog,
- {
- }
- 
-+static inline int bpf_lsm_get_retval_range(const struct bpf_prog *prog,
-+					   struct bpf_retval_range *range)
-+{
-+	return -EOPNOTSUPP;
-+}
-+static inline int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str,
-+					      const struct bpf_dynptr *value_p, int flags)
-+{
-+	return -EOPNOTSUPP;
-+}
-+static inline int bpf_remove_dentry_xattr_locked(struct dentry *dentry, const char *name__str)
-+{
-+	return -EOPNOTSUPP;
-+}
-+static inline bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog)
-+{
-+	return false;
-+}
- #endif /* CONFIG_BPF_LSM */
- 
- #endif /* _LINUX_BPF_LSM_H */
-diff --git a/include/linux/bpf_mem_alloc.h b/include/linux/bpf_mem_alloc.h
-new file mode 100644
-index 000000000000..e45162ef59bb
--- /dev/null
-+++ b/include/linux/bpf_mem_alloc.h
-@@ -0,0 +1,51 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
-+#ifndef _BPF_MEM_ALLOC_H
-+#define _BPF_MEM_ALLOC_H
-+#include <linux/compiler_types.h>
-+#include <linux/workqueue.h>
-+
-+struct bpf_mem_cache;
-+struct bpf_mem_caches;
-+
-+struct bpf_mem_alloc {
-+	struct bpf_mem_caches __percpu *caches;
-+	struct bpf_mem_cache __percpu *cache;
-+	struct obj_cgroup *objcg;
-+	bool percpu;
-+	struct work_struct work;
-+};
-+
-+/* 'size != 0' is for bpf_mem_alloc which manages fixed-size objects.
-+ * Alloc and free are done with bpf_mem_cache_{alloc,free}().
-+ *
-+ * 'size = 0' is for bpf_mem_alloc which manages many fixed-size objects.
-+ * Alloc and free are done with bpf_mem_{alloc,free}() and the size of
-+ * the returned object is given by the size argument of bpf_mem_alloc().
-+ * If percpu equals true, error will be returned in order to avoid
-+ * large memory consumption and the below bpf_mem_alloc_percpu_unit_init()
-+ * should be used to do on-demand per-cpu allocation for each size.
-+ */
-+int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu);
-+/* Initialize a non-fix-size percpu memory allocator */
-+int bpf_mem_alloc_percpu_init(struct bpf_mem_alloc *ma, struct obj_cgroup *objcg);
-+/* The percpu allocation with a specific unit size. */
-+int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size);
-+void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma);
-+
-+/* Check the allocation size for kmalloc equivalent allocator */
-+int bpf_mem_alloc_check_size(bool percpu, size_t size);
-+
-+/* kmalloc/kfree equivalent: */
-+void *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size);
-+void bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr);
-+void bpf_mem_free_rcu(struct bpf_mem_alloc *ma, void *ptr);
-+
-+/* kmem_cache_alloc/free equivalent: */
-+void *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma);
-+void bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr);
-+void bpf_mem_cache_free_rcu(struct bpf_mem_alloc *ma, void *ptr);
-+void bpf_mem_cache_raw_free(void *ptr);
-+void *bpf_mem_cache_alloc_flags(struct bpf_mem_alloc *ma, gfp_t flags);
-+
-+#endif /* _BPF_MEM_ALLOC_H */
-diff --git a/include/linux/bpf_mprog.h b/include/linux/bpf_mprog.h
-new file mode 100644
-index 000000000000..929225f7b095
--- /dev/null
-+++ b/include/linux/bpf_mprog.h
-@@ -0,0 +1,343 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/* Copyright (c) 2023 Isovalent */
-+#ifndef __BPF_MPROG_H
-+#define __BPF_MPROG_H
-+
-+#include <linux/bpf.h>
-+
-+/* bpf_mprog framework:
-+ *
-+ * bpf_mprog is a generic layer for multi-program attachment. In-kernel users
-+ * of the bpf_mprog don't need to care about the dependency resolution
-+ * internals, they can just consume it with few API calls. Currently available
-+ * dependency directives are BPF_F_{BEFORE,AFTER} which enable insertion of
-+ * a BPF program or BPF link relative to an existing BPF program or BPF link
-+ * inside the multi-program array as well as prepend and append behavior if
-+ * no relative object was specified, see corresponding selftests for concrete
-+ * examples (e.g. tc_links and tc_opts test cases of test_progs).
-+ *
-+ * Usage of bpf_mprog_{attach,detach,query}() core APIs with pseudo code:
-+ *
-+ *  Attach case:
-+ *
-+ *   struct bpf_mprog_entry *entry, *entry_new;
-+ *   int ret;
-+ *
-+ *   // bpf_mprog user-side lock
-+ *   // fetch active @entry from attach location
-+ *   [...]
-+ *   ret = bpf_mprog_attach(entry, &entry_new, [...]);
-+ *   if (!ret) {
-+ *       if (entry != entry_new) {
-+ *           // swap @entry to @entry_new at attach location
-+ *           // ensure there are no inflight users of @entry:
-+ *           synchronize_rcu();
-+ *       }
-+ *       bpf_mprog_commit(entry);
-+ *   } else {
-+ *       // error path, bail out, propagate @ret
-+ *   }
-+ *   // bpf_mprog user-side unlock
-+ *
-+ *  Detach case:
-+ *
-+ *   struct bpf_mprog_entry *entry, *entry_new;
-+ *   int ret;
-+ *
-+ *   // bpf_mprog user-side lock
-+ *   // fetch active @entry from attach location
-+ *   [...]
-+ *   ret = bpf_mprog_detach(entry, &entry_new, [...]);
-+ *   if (!ret) {
-+ *       // all (*) marked is optional and depends on the use-case
-+ *       // whether bpf_mprog_bundle should be freed or not
-+ *       if (!bpf_mprog_total(entry_new))     (*)
-+ *           entry_new = NULL                 (*)
-+ *       // swap @entry to @entry_new at attach location
-+ *       // ensure there are no inflight users of @entry:
-+ *       synchronize_rcu();
-+ *       bpf_mprog_commit(entry);
-+ *       if (!entry_new)                      (*)
-+ *           // free bpf_mprog_bundle         (*)
-+ *   } else {
-+ *       // error path, bail out, propagate @ret
-+ *   }
-+ *   // bpf_mprog user-side unlock
-+ *
-+ *  Query case:
-+ *
-+ *   struct bpf_mprog_entry *entry;
-+ *   int ret;
-+ *
-+ *   // bpf_mprog user-side lock
-+ *   // fetch active @entry from attach location
-+ *   [...]
-+ *   ret = bpf_mprog_query(attr, uattr, entry);
-+ *   // bpf_mprog user-side unlock
-+ *
-+ *  Data/fast path:
-+ *
-+ *   struct bpf_mprog_entry *entry;
-+ *   struct bpf_mprog_fp *fp;
-+ *   struct bpf_prog *prog;
-+ *   int ret = [...];
-+ *
-+ *   rcu_read_lock();
-+ *   // fetch active @entry from attach location
-+ *   [...]
-+ *   bpf_mprog_foreach_prog(entry, fp, prog) {
-+ *       ret = bpf_prog_run(prog, [...]);
-+ *       // process @ret from program
-+ *   }
-+ *   [...]
-+ *   rcu_read_unlock();
-+ *
-+ * bpf_mprog locking considerations:
-+ *
-+ * bpf_mprog_{attach,detach,query}() must be protected by an external lock
-+ * (like RTNL in case of tcx).
-+ *
-+ * bpf_mprog_entry pointer can be an __rcu annotated pointer (in case of tcx
-+ * the netdevice has tcx_ingress and tcx_egress __rcu pointer) which gets
-+ * updated via rcu_assign_pointer() pointing to the active bpf_mprog_entry of
-+ * the bpf_mprog_bundle.
-+ *
-+ * Fast path accesses the active bpf_mprog_entry within RCU critical section
-+ * (in case of tcx it runs in NAPI which provides RCU protection there,
-+ * other users might need explicit rcu_read_lock()). The bpf_mprog_commit()
-+ * assumes that for the old bpf_mprog_entry there are no inflight users
-+ * anymore.
-+ *
-+ * The READ_ONCE()/WRITE_ONCE() pairing for bpf_mprog_fp's prog access is for
-+ * the replacement case where we don't swap the bpf_mprog_entry.
-+ */
-+
-+#define bpf_mprog_foreach_tuple(entry, fp, cp, t)			\
-+	for (fp = &entry->fp_items[0], cp = &entry->parent->cp_items[0];\
-+	     ({								\
-+		t.prog = READ_ONCE(fp->prog);				\
-+		t.link = cp->link;					\
-+		t.prog;							\
-+	      });							\
-+	     fp++, cp++)
-+
-+#define bpf_mprog_foreach_prog(entry, fp, p)				\
-+	for (fp = &entry->fp_items[0];					\
-+	     (p = READ_ONCE(fp->prog));					\
-+	     fp++)
-+
-+#define BPF_MPROG_MAX 64
-+
-+struct bpf_mprog_fp {
-+	struct bpf_prog *prog;
-+};
-+
-+struct bpf_mprog_cp {
-+	struct bpf_link *link;
-+};
-+
-+struct bpf_mprog_entry {
-+	struct bpf_mprog_fp fp_items[BPF_MPROG_MAX];
-+	struct bpf_mprog_bundle *parent;
-+};
-+
-+struct bpf_mprog_bundle {
-+	struct bpf_mprog_entry a;
-+	struct bpf_mprog_entry b;
-+	struct bpf_mprog_cp cp_items[BPF_MPROG_MAX];
-+	struct bpf_prog *ref;
-+	atomic64_t revision;
-+	u32 count;
-+};
-+
-+struct bpf_tuple {
-+	struct bpf_prog *prog;
-+	struct bpf_link *link;
-+};
-+
-+static inline struct bpf_mprog_entry *
-+bpf_mprog_peer(const struct bpf_mprog_entry *entry)
-+{
-+	if (entry == &entry->parent->a)
-+		return &entry->parent->b;
-+	else
-+		return &entry->parent->a;
-+}
-+
-+static inline void bpf_mprog_bundle_init(struct bpf_mprog_bundle *bundle)
-+{
-+	BUILD_BUG_ON(sizeof(bundle->a.fp_items[0]) > sizeof(u64));
-+	BUILD_BUG_ON(ARRAY_SIZE(bundle->a.fp_items) !=
-+		     ARRAY_SIZE(bundle->cp_items));
-+
-+	memset(bundle, 0, sizeof(*bundle));
-+	atomic64_set(&bundle->revision, 1);
-+	bundle->a.parent = bundle;
-+	bundle->b.parent = bundle;
-+}
-+
-+static inline void bpf_mprog_inc(struct bpf_mprog_entry *entry)
-+{
-+	entry->parent->count++;
-+}
-+
-+static inline void bpf_mprog_dec(struct bpf_mprog_entry *entry)
-+{
-+	entry->parent->count--;
-+}
-+
-+static inline int bpf_mprog_max(void)
-+{
-+	return ARRAY_SIZE(((struct bpf_mprog_entry *)NULL)->fp_items) - 1;
-+}
-+
-+static inline int bpf_mprog_total(struct bpf_mprog_entry *entry)
-+{
-+	int total = entry->parent->count;
-+
-+	WARN_ON_ONCE(total > bpf_mprog_max());
-+	return total;
-+}
-+
-+static inline bool bpf_mprog_exists(struct bpf_mprog_entry *entry,
-+				    struct bpf_prog *prog)
-+{
-+	const struct bpf_mprog_fp *fp;
-+	const struct bpf_prog *tmp;
-+
-+	bpf_mprog_foreach_prog(entry, fp, tmp) {
-+		if (tmp == prog)
-+			return true;
-+	}
-+	return false;
-+}
-+
-+static inline void bpf_mprog_mark_for_release(struct bpf_mprog_entry *entry,
-+					      struct bpf_tuple *tuple)
-+{
-+	WARN_ON_ONCE(entry->parent->ref);
-+	if (!tuple->link)
-+		entry->parent->ref = tuple->prog;
-+}
-+
-+static inline void bpf_mprog_complete_release(struct bpf_mprog_entry *entry)
-+{
-+	/* In the non-link case prog deletions can only drop the reference
-+	 * to the prog after the bpf_mprog_entry got swapped and the
-+	 * bpf_mprog ensured that there are no inflight users anymore.
-+	 *
-+	 * Paired with bpf_mprog_mark_for_release().
-+	 */
-+	if (entry->parent->ref) {
-+		bpf_prog_put(entry->parent->ref);
-+		entry->parent->ref = NULL;
-+	}
-+}
-+
-+static inline void bpf_mprog_revision_new(struct bpf_mprog_entry *entry)
-+{
-+	atomic64_inc(&entry->parent->revision);
-+}
-+
-+static inline void bpf_mprog_commit(struct bpf_mprog_entry *entry)
-+{
-+	bpf_mprog_complete_release(entry);
-+	bpf_mprog_revision_new(entry);
-+}
-+
-+static inline u64 bpf_mprog_revision(struct bpf_mprog_entry *entry)
-+{
-+	return atomic64_read(&entry->parent->revision);
-+}
-+
-+static inline void bpf_mprog_entry_copy(struct bpf_mprog_entry *dst,
-+					struct bpf_mprog_entry *src)
-+{
-+	memcpy(dst->fp_items, src->fp_items, sizeof(src->fp_items));
-+}
-+
-+static inline void bpf_mprog_entry_clear(struct bpf_mprog_entry *dst)
-+{
-+	memset(dst->fp_items, 0, sizeof(dst->fp_items));
-+}
-+
-+static inline void bpf_mprog_clear_all(struct bpf_mprog_entry *entry,
-+				       struct bpf_mprog_entry **entry_new)
-+{
-+	struct bpf_mprog_entry *peer;
-+
-+	peer = bpf_mprog_peer(entry);
-+	bpf_mprog_entry_clear(peer);
-+	peer->parent->count = 0;
-+	*entry_new = peer;
-+}
-+
-+static inline void bpf_mprog_entry_grow(struct bpf_mprog_entry *entry, int idx)
-+{
-+	int total = bpf_mprog_total(entry);
-+
-+	memmove(entry->fp_items + idx + 1,
-+		entry->fp_items + idx,
-+		(total - idx) * sizeof(struct bpf_mprog_fp));
-+
-+	memmove(entry->parent->cp_items + idx + 1,
-+		entry->parent->cp_items + idx,
-+		(total - idx) * sizeof(struct bpf_mprog_cp));
-+}
-+
-+static inline void bpf_mprog_entry_shrink(struct bpf_mprog_entry *entry, int idx)
-+{
-+	/* Total array size is needed in this case to enure the NULL
-+	 * entry is copied at the end.
-+	 */
-+	int total = ARRAY_SIZE(entry->fp_items);
-+
-+	memmove(entry->fp_items + idx,
-+		entry->fp_items + idx + 1,
-+		(total - idx - 1) * sizeof(struct bpf_mprog_fp));
-+
-+	memmove(entry->parent->cp_items + idx,
-+		entry->parent->cp_items + idx + 1,
-+		(total - idx - 1) * sizeof(struct bpf_mprog_cp));
-+}
-+
-+static inline void bpf_mprog_read(struct bpf_mprog_entry *entry, u32 idx,
-+				  struct bpf_mprog_fp **fp,
-+				  struct bpf_mprog_cp **cp)
-+{
-+	*fp = &entry->fp_items[idx];
-+	*cp = &entry->parent->cp_items[idx];
-+}
-+
-+static inline void bpf_mprog_write(struct bpf_mprog_fp *fp,
-+				   struct bpf_mprog_cp *cp,
-+				   struct bpf_tuple *tuple)
-+{
-+	WRITE_ONCE(fp->prog, tuple->prog);
-+	cp->link = tuple->link;
-+}
-+
-+int bpf_mprog_attach(struct bpf_mprog_entry *entry,
-+		     struct bpf_mprog_entry **entry_new,
-+		     struct bpf_prog *prog_new, struct bpf_link *link,
-+		     struct bpf_prog *prog_old,
-+		     u32 flags, u32 id_or_fd, u64 revision);
-+
-+int bpf_mprog_detach(struct bpf_mprog_entry *entry,
-+		     struct bpf_mprog_entry **entry_new,
-+		     struct bpf_prog *prog, struct bpf_link *link,
-+		     u32 flags, u32 id_or_fd, u64 revision);
-+
-+int bpf_mprog_query(const union bpf_attr *attr, union bpf_attr __user *uattr,
-+		    struct bpf_mprog_entry *entry);
-+
-+static inline bool bpf_mprog_supported(enum bpf_prog_type type)
-+{
-+	switch (type) {
-+	case BPF_PROG_TYPE_SCHED_CLS:
-+		return true;
-+	default:
-+		return false;
-+	}
-+}
-+#endif /* __BPF_MPROG_H */
-diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h
-index 2b9112b80171..b13de31e163f 100644
--- a/include/linux/bpf_types.h
-+++ b/include/linux/bpf_types.h
-@@ -79,6 +79,10 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_LSM, lsm,
- #endif
- BPF_PROG_TYPE(BPF_PROG_TYPE_SYSCALL, bpf_syscall,
- 	      void *, void *)
-+#ifdef CONFIG_NETFILTER_BPF_LINK
-+BPF_PROG_TYPE(BPF_PROG_TYPE_NETFILTER, netfilter,
-+	      struct bpf_nf_ctx, struct bpf_nf_ctx)
-+#endif
- 
- BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops)
- BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops)
-@@ -86,6 +90,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_PROG_ARRAY, prog_array_map_ops)
- BPF_MAP_TYPE(BPF_MAP_TYPE_PERF_EVENT_ARRAY, perf_event_array_map_ops)
- #ifdef CONFIG_CGROUPS
- BPF_MAP_TYPE(BPF_MAP_TYPE_CGROUP_ARRAY, cgroup_array_map_ops)
-+BPF_MAP_TYPE(BPF_MAP_TYPE_CGRP_STORAGE, cgrp_storage_map_ops)
- #endif
- #ifdef CONFIG_CGROUP_BPF
- BPF_MAP_TYPE(BPF_MAP_TYPE_CGROUP_STORAGE, cgroup_storage_map_ops)
-@@ -126,6 +131,9 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_STRUCT_OPS, bpf_struct_ops_map_ops)
- #endif
- BPF_MAP_TYPE(BPF_MAP_TYPE_RINGBUF, ringbuf_map_ops)
- BPF_MAP_TYPE(BPF_MAP_TYPE_BLOOM_FILTER, bloom_filter_map_ops)
-+BPF_MAP_TYPE(BPF_MAP_TYPE_USER_RINGBUF, user_ringbuf_map_ops)
-+BPF_MAP_TYPE(BPF_MAP_TYPE_ARENA, arena_map_ops)
-+BPF_MAP_TYPE(BPF_MAP_TYPE_INSN_ARRAY, insn_array_map_ops)
- 
- BPF_LINK_TYPE(BPF_LINK_TYPE_RAW_TRACEPOINT, raw_tracepoint)
- BPF_LINK_TYPE(BPF_LINK_TYPE_TRACING, tracing)
-@@ -136,9 +144,14 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_ITER, iter)
- #ifdef CONFIG_NET
- BPF_LINK_TYPE(BPF_LINK_TYPE_NETNS, netns)
- BPF_LINK_TYPE(BPF_LINK_TYPE_XDP, xdp)
-+BPF_LINK_TYPE(BPF_LINK_TYPE_NETFILTER, netfilter)
-+BPF_LINK_TYPE(BPF_LINK_TYPE_TCX, tcx)
-+BPF_LINK_TYPE(BPF_LINK_TYPE_NETKIT, netkit)
-+BPF_LINK_TYPE(BPF_LINK_TYPE_SOCKMAP, sockmap)
- #endif
- #ifdef CONFIG_PERF_EVENTS
- BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf)
- #endif
- BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi)
- BPF_LINK_TYPE(BPF_LINK_TYPE_STRUCT_OPS, struct_ops)
-+BPF_LINK_TYPE(BPF_LINK_TYPE_UPROBE_MULTI, uprobe_multi)
-diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
-index 2e3bad8640dc..130bcbd66f60 100644
--- a/include/linux/bpf_verifier.h
-+++ b/include/linux/bpf_verifier.h
-@@ -18,35 +18,29 @@
-  * that converting umax_value to int cannot overflow.
-  */
- #define BPF_MAX_VAR_SIZ	(1 << 29)
-/* size of type_str_buf in bpf_verifier. */
-#define TYPE_STR_BUF_LEN 64
-
-/* Liveness marks, used for registers and spilled-regs (in stack slots).
- * Read marks propagate upwards until they find a write mark; they record that
- * "one of this state's descendants read this reg" (and therefore the reg is
- * relevant for states_equal() checks).
- * Write marks collect downwards and do not propagate; they record that "the
- * straight-line code that reached this state (from its parent) wrote this reg"
- * (and therefore that reads propagated from this state or its descendants
- * should not propagate to its parent).
- * A state with a write mark can receive read marks; it just won't propagate
- * them to its parent, since the write mark is a property, not of the state,
- * but of the link between it and its parent.  See mark_reg_read() and
- * mark_stack_slot_read() in kernel/bpf/verifier.c.
-+/* size of tmp_str_buf in bpf_verifier.
-+ * we need at least 306 bytes to fit full stack mask representation
-+ * (in the "-8,-16,...,-512" form)
-  */
-enum bpf_reg_liveness {
-	REG_LIVE_NONE = 0, /* reg hasn't been read or written this branch */
-	REG_LIVE_READ32 = 0x1, /* reg was read, so we're sensitive to initial value */
-	REG_LIVE_READ64 = 0x2, /* likewise, but full 64-bit content matters */
-	REG_LIVE_READ = REG_LIVE_READ32 | REG_LIVE_READ64,
-	REG_LIVE_WRITTEN = 0x4, /* reg was written first, screening off later reads */
-	REG_LIVE_DONE = 0x8, /* liveness won't be updating this register anymore */
-+#define TMP_STR_BUF_LEN 320
-+/* Patch buffer size */
-+#define INSN_BUF_SIZE 32
-+
-+#define ITER_PREFIX "bpf_iter_"
-+
-+enum bpf_iter_state {
-+	BPF_ITER_STATE_INVALID, /* for non-first slot */
-+	BPF_ITER_STATE_ACTIVE,
-+	BPF_ITER_STATE_DRAINED,
- };
- 
- struct bpf_reg_state {
- 	/* Ordering of fields matters.  See states_equal() */
- 	enum bpf_reg_type type;
-	/* Fixed part of pointer offset, pointer types only */
-+	/*
-+	 * Fixed part of pointer offset, pointer types only.
-+	 * Or constant delta between "linked" scalars with the same ID.
-+	 */
- 	s32 off;
- 	union {
- 		/* valid when type == PTR_TO_PACKET */
-@@ -70,7 +64,10 @@ struct bpf_reg_state {
- 			u32 btf_id;
- 		};
- 
-		u32 mem_size; /* for PTR_TO_MEM | PTR_TO_MEM_OR_NULL */
-+		struct { /* for PTR_TO_MEM | PTR_TO_MEM_OR_NULL */
-+			u32 mem_size;
-+			u32 dynptr_id; /* for dynptr slices */
-+		};
- 
- 		/* For dynptr stack slots */
- 		struct {
-@@ -84,6 +81,26 @@ struct bpf_reg_state {
- 			bool first_slot;
- 		} dynptr;
- 
-+		/* For bpf_iter stack slots */
-+		struct {
-+			/* BTF container and BTF type ID describing
-+			 * struct bpf_iter_<type> of an iterator state
-+			 */
-+			struct btf *btf;
-+			u32 btf_id;
-+			/* packing following two fields to fit iter state into 16 bytes */
-+			enum bpf_iter_state state:2;
-+			int depth:30;
-+		} iter;
-+
-+		/* For irq stack slots */
-+		struct {
-+			enum {
-+				IRQ_NATIVE_KFUNC,
-+				IRQ_LOCK_KFUNC,
-+			} kfunc_class;
-+		} irq;
-+
- 		/* Max size from any of the above. */
- 		struct {
- 			unsigned long raw1;
-@@ -92,6 +109,26 @@ struct bpf_reg_state {
- 
- 		u32 subprogno; /* for PTR_TO_FUNC */
- 	};
-+	/* For scalar types (SCALAR_VALUE), this represents our knowledge of
-+	 * the actual value.
-+	 * For pointer types, this represents the variable part of the offset
-+	 * from the pointed-to object, and is shared with all bpf_reg_states
-+	 * with the same id as us.
-+	 */
-+	struct tnum var_off;
-+	/* Used to determine if any memory access using this register will
-+	 * result in a bad access.
-+	 * These refer to the same value as var_off, not necessarily the actual
-+	 * contents of the register.
-+	 */
-+	s64 smin_value; /* minimum possible (s64)value */
-+	s64 smax_value; /* maximum possible (s64)value */
-+	u64 umin_value; /* minimum possible (u64)value */
-+	u64 umax_value; /* maximum possible (u64)value */
-+	s32 s32_min_value; /* minimum possible (s32)value */
-+	s32 s32_max_value; /* maximum possible (s32)value */
-+	u32 u32_min_value; /* minimum possible (u32)value */
-+	u32 u32_max_value; /* maximum possible (u32)value */
- 	/* For PTR_TO_PACKET, used to find other pointers with the same variable
- 	 * offset, so they can share range knowledge.
- 	 * For PTR_TO_MAP_VALUE_OR_NULL this is used to share which map value we
-@@ -102,7 +139,16 @@ struct bpf_reg_state {
- 	 * same reference to the socket, to determine proper reference freeing.
- 	 * For stack slots that are dynptrs, this is used to track references to
- 	 * the dynptr to determine proper reference freeing.
-+	 * Similarly to dynptrs, we use ID to track "belonging" of a reference
-+	 * to a specific instance of bpf_iter.
-+	 */
-+	/*
-+	 * Upper bit of ID is used to remember relationship between "linked"
-+	 * registers. Example:
-+	 * r1 = r2;    both will have r1->id == r2->id == N
-+	 * r1 += 10;   r1->id == N | BPF_ADD_CONST and r1->off == 10
- 	 */
-+#define BPF_ADD_CONST (1U << 31)
- 	u32 id;
- 	/* PTR_TO_SOCKET and PTR_TO_TCP_SOCK could be a ptr returned
- 	 * from a pointer-cast helper, bpf_sk_fullsock() and
-@@ -144,28 +190,6 @@ struct bpf_reg_state {
- 	 * allowed and has the same effect as bpf_sk_release(sk).
- 	 */
- 	u32 ref_obj_id;
-	/* For scalar types (SCALAR_VALUE), this represents our knowledge of
-	 * the actual value.
-	 * For pointer types, this represents the variable part of the offset
-	 * from the pointed-to object, and is shared with all bpf_reg_states
-	 * with the same id as us.
-	 */
-	struct tnum var_off;
-	/* Used to determine if any memory access using this register will
-	 * result in a bad access.
-	 * These refer to the same value as var_off, not necessarily the actual
-	 * contents of the register.
-	 */
-	s64 smin_value; /* minimum possible (s64)value */
-	s64 smax_value; /* maximum possible (s64)value */
-	u64 umin_value; /* minimum possible (u64)value */
-	u64 umax_value; /* maximum possible (u64)value */
-	s32 s32_min_value; /* minimum possible (s32)value */
-	s32 s32_max_value; /* maximum possible (s32)value */
-	u32 u32_min_value; /* minimum possible (u32)value */
-	u32 u32_max_value; /* maximum possible (u32)value */
-	/* parentage chain for liveness checking */
-	struct bpf_reg_state *parent;
- 	/* Inside the callee two registers can be both PTR_TO_STACK like
- 	 * R1=fp-8 and R2=fp-8, but one of them points to this function stack
- 	 * while another to the caller's stack. To differentiate them 'frameno'
-@@ -178,7 +202,6 @@ struct bpf_reg_state {
- 	 * patching which only happens after main verification finished.
- 	 */
- 	s32 subreg_def;
-	enum bpf_reg_liveness live;
- 	/* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */
- 	bool precise;
- };
-@@ -192,9 +215,16 @@ enum bpf_stack_slot_type {
- 	 * is stored in bpf_stack_state->spilled_ptr.dynptr.type
- 	 */
- 	STACK_DYNPTR,
-+	STACK_ITER,
-+	STACK_IRQ_FLAG,
- };
- 
- #define BPF_REG_SIZE 8	/* size of eBPF register in bytes */
-+
-+#define BPF_REGMASK_ARGS ((1 << BPF_REG_1) | (1 << BPF_REG_2) | \
-+			  (1 << BPF_REG_3) | (1 << BPF_REG_4) | \
-+			  (1 << BPF_REG_5))
-+
- #define BPF_DYNPTR_SIZE		sizeof(struct bpf_dynptr_kern)
- #define BPF_DYNPTR_NR_SLOTS		(BPF_DYNPTR_SIZE / BPF_REG_SIZE)
- 
-@@ -204,6 +234,17 @@ struct bpf_stack_state {
- };
- 
- struct bpf_reference_state {
-+	/* Each reference object has a type. Ensure REF_TYPE_PTR is zero to
-+	 * default to pointer reference on zero initialization of a state.
-+	 */
-+	enum ref_state_type {
-+		REF_TYPE_PTR		= (1 << 1),
-+		REF_TYPE_IRQ		= (1 << 2),
-+		REF_TYPE_LOCK		= (1 << 3),
-+		REF_TYPE_RES_LOCK 	= (1 << 4),
-+		REF_TYPE_RES_LOCK_IRQ	= (1 << 5),
-+		REF_TYPE_LOCK_MASK	= REF_TYPE_LOCK | REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ,
-+	} type;
- 	/* Track each reference created with a unique id, even if the same
- 	 * instruction creates the reference multiple times (eg, via CALL).
- 	 */
-@@ -212,6 +253,15 @@ struct bpf_reference_state {
- 	 * is used purely to inform the user of a reference leak.
- 	 */
- 	int insn_idx;
-+	/* Use to keep track of the source object of a lock, to ensure
-+	 * it matches on unlock.
-+	 */
-+	void *ptr;
-+};
-+
-+struct bpf_retval_range {
-+	s32 minval;
-+	s32 maxval;
- };
- 
- /* state of the program:
-@@ -236,33 +286,81 @@ struct bpf_func_state {
- 	 * void foo(void) { bpf_timer_set_callback(,foo); }
- 	 */
- 	u32 async_entry_cnt;
-+	struct bpf_retval_range callback_ret_range;
- 	bool in_callback_fn;
- 	bool in_async_callback_fn;
-+	bool in_exception_callback_fn;
-+	/* For callback calling functions that limit number of possible
-+	 * callback executions (e.g. bpf_loop) keeps track of current
-+	 * simulated iteration number.
-+	 * Value in frame N refers to number of times callback with frame
-+	 * N+1 was simulated, e.g. for the following call:
-+	 *
-+	 *   bpf_loop(..., fn, ...); | suppose current frame is N
-+	 *                           | fn would be simulated in frame N+1
-+	 *                           | number of simulations is tracked in frame N
-+	 */
-+	u32 callback_depth;
- 
- 	/* The following fields should be last. See copy_func_state() */
-	int acquired_refs;
-	struct bpf_reference_state *refs;
-	int allocated_stack;
-+	/* The state of the stack. Each element of the array describes BPF_REG_SIZE
-+	 * (i.e. 8) bytes worth of stack memory.
-+	 * stack[0] represents bytes [*(r10-8)..*(r10-1)]
-+	 * stack[1] represents bytes [*(r10-16)..*(r10-9)]
-+	 * ...
-+	 * stack[allocated_stack/8 - 1] represents [*(r10-allocated_stack)..*(r10-allocated_stack+7)]
-+	 */
- 	struct bpf_stack_state *stack;
-+	/* Size of the current stack, in bytes. The stack state is tracked below, in
-+	 * `stack`. allocated_stack is always a multiple of BPF_REG_SIZE.
-+	 */
-+	int allocated_stack;
- };
- 
-struct bpf_idx_pair {
-	u32 prev_idx;
-	u32 idx;
-+#define MAX_CALL_FRAMES 8
-+
-+/* instruction history flags, used in bpf_jmp_history_entry.flags field */
-+enum {
-+	/* instruction references stack slot through PTR_TO_STACK register;
-+	 * we also store stack's frame number in lower 3 bits (MAX_CALL_FRAMES is 8)
-+	 * and accessed stack slot's index in next 6 bits (MAX_BPF_STACK is 512,
-+	 * 8 bytes per slot, so slot index (spi) is [0, 63])
-+	 */
-+	INSN_F_FRAMENO_MASK = 0x7, /* 3 bits */
-+
-+	INSN_F_SPI_MASK = 0x3f, /* 6 bits */
-+	INSN_F_SPI_SHIFT = 3, /* shifted 3 bits to the left */
-+
-+	INSN_F_STACK_ACCESS = BIT(9),
-+
-+	INSN_F_DST_REG_STACK = BIT(10), /* dst_reg is PTR_TO_STACK */
-+	INSN_F_SRC_REG_STACK = BIT(11), /* src_reg is PTR_TO_STACK */
-+	/* total 12 bits are used now. */
- };
- 
-struct bpf_id_pair {
-	u32 old;
-	u32 cur;
-+static_assert(INSN_F_FRAMENO_MASK + 1 >= MAX_CALL_FRAMES);
-+static_assert(INSN_F_SPI_MASK + 1 >= MAX_BPF_STACK / 8);
-+
-+struct bpf_jmp_history_entry {
-+	u32 idx;
-+	/* insn idx can't be bigger than 1 million */
-+	u32 prev_idx : 20;
-+	/* special INSN_F_xxx flags */
-+	u32 flags : 12;
-+	/* additional registers that need precision tracking when this
-+	 * jump is backtracked, vector of six 10-bit records
-+	 */
-+	u64 linked_regs;
- };
- 
- /* Maximum number of register states that can exist at once */
-#define BPF_ID_MAP_SIZE (MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE)
-#define MAX_CALL_FRAMES 8
-+#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE) * MAX_CALL_FRAMES)
- struct bpf_verifier_state {
- 	/* call stack tracking */
- 	struct bpf_func_state *frame[MAX_CALL_FRAMES];
- 	struct bpf_verifier_state *parent;
-+	/* Acquired reference states */
-+	struct bpf_reference_state *refs;
- 	/*
- 	 * 'branches' field is the number of branches left to explore:
- 	 * 0 - all possible paths from this state reached bpf_exit or
-@@ -311,37 +409,80 @@ struct bpf_verifier_state {
- 	u32 branches;
- 	u32 insn_idx;
- 	u32 curframe;
-	u32 active_spin_lock;
-+
-+	u32 acquired_refs;
-+	u32 active_locks;
-+	u32 active_preempt_locks;
-+	u32 active_irq_id;
-+	u32 active_lock_id;
-+	void *active_lock_ptr;
-+	u32 active_rcu_locks;
-+
- 	bool speculative;
-+	bool in_sleepable;
-+	bool cleaned;
- 
- 	/* first and last insn idx of this verifier state */
- 	u32 first_insn_idx;
- 	u32 last_insn_idx;
-+	/* if this state is a backedge state then equal_state
-+	 * records cached state to which this state is equal.
-+	 */
-+	struct bpf_verifier_state *equal_state;
- 	/* jmp history recorded from first to last.
- 	 * backtracking is using it to go from last to first.
- 	 * For most states jmp_history_cnt is [0-3].
- 	 * For loops can go up to ~40.
- 	 */
-	struct bpf_idx_pair *jmp_history;
-+	struct bpf_jmp_history_entry *jmp_history;
- 	u32 jmp_history_cnt;
-+	u32 dfs_depth;
-+	u32 callback_unroll_depth;
-+	u32 may_goto_depth;
- };
- 
-#define bpf_get_spilled_reg(slot, frame)				\
-+#define bpf_get_spilled_reg(slot, frame, mask)				\
- 	(((slot < frame->allocated_stack / BPF_REG_SIZE) &&		\
-	  (frame->stack[slot].slot_type[0] == STACK_SPILL))		\
-+	  ((1 << frame->stack[slot].slot_type[BPF_REG_SIZE - 1]) & (mask))) \
- 	 ? &frame->stack[slot].spilled_ptr : NULL)
- 
- /* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */
-#define bpf_for_each_spilled_reg(iter, frame, reg)			\
-	for (iter = 0, reg = bpf_get_spilled_reg(iter, frame);		\
-+#define bpf_for_each_spilled_reg(iter, frame, reg, mask)			\
-+	for (iter = 0, reg = bpf_get_spilled_reg(iter, frame, mask);		\
- 	     iter < frame->allocated_stack / BPF_REG_SIZE;		\
-	     iter++, reg = bpf_get_spilled_reg(iter, frame))
-+	     iter++, reg = bpf_get_spilled_reg(iter, frame, mask))
-+
-+#define bpf_for_each_reg_in_vstate_mask(__vst, __state, __reg, __mask, __expr)   \
-+	({                                                               \
-+		struct bpf_verifier_state *___vstate = __vst;            \
-+		int ___i, ___j;                                          \
-+		for (___i = 0; ___i <= ___vstate->curframe; ___i++) {    \
-+			struct bpf_reg_state *___regs;                   \
-+			__state = ___vstate->frame[___i];                \
-+			___regs = __state->regs;                         \
-+			for (___j = 0; ___j < MAX_BPF_REG; ___j++) {     \
-+				__reg = &___regs[___j];                  \
-+				(void)(__expr);                          \
-+			}                                                \
-+			bpf_for_each_spilled_reg(___j, __state, __reg, __mask) { \
-+				if (!__reg)                              \
-+					continue;                        \
-+				(void)(__expr);                          \
-+			}                                                \
-+		}                                                        \
-+	})
-+
-+/* Invoke __expr over regsiters in __vst, setting __state and __reg */
-+#define bpf_for_each_reg_in_vstate(__vst, __state, __reg, __expr) \
-+	bpf_for_each_reg_in_vstate_mask(__vst, __state, __reg, 1 << STACK_SPILL, __expr)
- 
- /* linked list of verifier states used to prune search */
- struct bpf_verifier_state_list {
- 	struct bpf_verifier_state state;
-	struct bpf_verifier_state_list *next;
-	int miss_cnt, hit_cnt;
-+	struct list_head node;
-+	u32 miss_cnt;
-+	u32 hit_cnt:31;
-+	u32 in_free_list:1;
- };
- 
- struct bpf_loop_inline_state {
-@@ -352,6 +493,13 @@ struct bpf_loop_inline_state {
- 	u32 callback_subprogno; /* valid when fit_for_inline is true */
- };
- 
-+/* pointer and state for maps */
-+struct bpf_map_ptr_state {
-+	struct bpf_map *map_ptr;
-+	bool poison;
-+	bool unpriv;
-+};
-+
- /* Possible states for alu_state member. */
- #define BPF_ALU_SANITIZE_SRC		(1U << 0)
- #define BPF_ALU_SANITIZE_DST		(1U << 1)
-@@ -361,10 +509,19 @@ struct bpf_loop_inline_state {
- #define BPF_ALU_SANITIZE		(BPF_ALU_SANITIZE_SRC | \
- 					 BPF_ALU_SANITIZE_DST)
- 
-+/*
-+ * An array of BPF instructions.
-+ * Primary usage: return value of bpf_insn_successors.
-+ */
-+struct bpf_iarray {
-+	int cnt;
-+	u32 items[];
-+};
-+
- struct bpf_insn_aux_data {
- 	union {
- 		enum bpf_reg_type ptr_type;	/* pointer type for load/store insns */
-		unsigned long map_ptr_state;	/* pointer/poison value for maps */
-+		struct bpf_map_ptr_state map_ptr_state;
- 		s32 call_imm;			/* saved imm field of call insn */
- 		u32 alu_limit;			/* limit for add/sub register with pointer */
- 		struct {
-@@ -386,16 +543,54 @@ struct bpf_insn_aux_data {
- 		 */
- 		struct bpf_loop_inline_state loop_inline_state;
- 	};
-+	union {
-+		/* remember the size of type passed to bpf_obj_new to rewrite R1 */
-+		u64 obj_new_size;
-+		/* remember the offset of node field within type to rewrite */
-+		u64 insert_off;
-+	};
-+	struct bpf_iarray *jt;	/* jump table for gotox or bpf_tailcall call instruction */
-+	struct btf_struct_meta *kptr_struct_meta;
- 	u64 map_key_state; /* constant (32 bit) key tracking for maps */
- 	int ctx_field_size; /* the ctx field size for load insn, maybe 0 */
- 	u32 seen; /* this insn was processed by the verifier at env->pass_cnt */
-	bool sanitize_stack_spill; /* subject to Spectre v4 sanitation */
-+	bool nospec; /* do not execute this instruction speculatively */
-+	bool nospec_result; /* result is unsafe under speculation, nospec must follow */
- 	bool zext_dst; /* this insn zero extends dst reg */
-+	bool needs_zext; /* alu op needs to clear upper bits */
-+	bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */
-+	bool is_iter_next; /* bpf_iter_<type>_next() kfunc call */
-+	bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */
- 	u8 alu_state; /* used in combination with alu_limit */
-+	/* true if STX or LDX instruction is a part of a spill/fill
-+	 * pattern for a bpf_fastcall call.
-+	 */
-+	u8 fastcall_pattern:1;
-+	/* for CALL instructions, a number of spill/fill pairs in the
-+	 * bpf_fastcall pattern.
-+	 */
-+	u8 fastcall_spills_num:3;
-+	u8 arg_prog:4;
- 
- 	/* below fields are initialized once */
- 	unsigned int orig_idx; /* original instruction index */
-+	bool jmp_point;
- 	bool prune_point;
-+	/* ensure we check state equivalence and save state checkpoint and
-+	 * this instruction, regardless of any heuristics
-+	 */
-+	bool force_checkpoint;
-+	/* true if instruction is a call to a helper function that
-+	 * accepts callback function as a parameter.
-+	 */
-+	bool calls_callback;
-+	/*
-+	 * CFG strongly connected component this instruction belongs to,
-+	 * zero if it is a singleton SCC.
-+	 */
-+	u32 scc;
-+	/* registers alive before this instruction. */
-+	u16 live_regs_before;
- };
- 
- #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */
-@@ -404,54 +599,141 @@ struct bpf_insn_aux_data {
- #define BPF_VERIFIER_TMP_LOG_SIZE	1024
- 
- struct bpf_verifier_log {
-	u32 level;
-	char kbuf[BPF_VERIFIER_TMP_LOG_SIZE];
-+	/* Logical start and end positions of a "log window" of the verifier log.
-+	 * start_pos == 0 means we haven't truncated anything.
-+	 * Once truncation starts to happen, start_pos + len_total == end_pos,
-+	 * except during log reset situations, in which (end_pos - start_pos)
-+	 * might get smaller than len_total (see bpf_vlog_reset()).
-+	 * Generally, (end_pos - start_pos) gives number of useful data in
-+	 * user log buffer.
-+	 */
-+	u64 start_pos;
-+	u64 end_pos;
- 	char __user *ubuf;
-	u32 len_used;
-+	u32 level;
- 	u32 len_total;
-+	u32 len_max;
-+	char kbuf[BPF_VERIFIER_TMP_LOG_SIZE];
- };
- 
-static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log)
-{
-	return log->len_used >= log->len_total - 1;
-}
-
- #define BPF_LOG_LEVEL1	1
- #define BPF_LOG_LEVEL2	2
- #define BPF_LOG_STATS	4
-+#define BPF_LOG_FIXED	8
- #define BPF_LOG_LEVEL	(BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2)
-#define BPF_LOG_MASK	(BPF_LOG_LEVEL | BPF_LOG_STATS)
-+#define BPF_LOG_MASK	(BPF_LOG_LEVEL | BPF_LOG_STATS | BPF_LOG_FIXED)
- #define BPF_LOG_KERNEL	(BPF_LOG_MASK + 1) /* kernel internal flag */
- #define BPF_LOG_MIN_ALIGNMENT 8U
- #define BPF_LOG_ALIGNMENT 40U
- 
- static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
- {
-	return log &&
-		((log->level && log->ubuf && !bpf_verifier_log_full(log)) ||
-		 log->level == BPF_LOG_KERNEL);
-}
-
-static inline bool
-bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log)
-{
-	return log->len_total >= 128 && log->len_total <= UINT_MAX >> 2 &&
-	       log->level && log->ubuf && !(log->level & ~BPF_LOG_MASK);
-+	return log && log->level;
- }
- 
- #define BPF_MAX_SUBPROGS 256
- 
-+struct bpf_subprog_arg_info {
-+	enum bpf_arg_type arg_type;
-+	union {
-+		u32 mem_size;
-+		u32 btf_id;
-+	};
-+};
-+
-+enum priv_stack_mode {
-+	PRIV_STACK_UNKNOWN,
-+	NO_PRIV_STACK,
-+	PRIV_STACK_ADAPTIVE,
-+};
-+
- struct bpf_subprog_info {
- 	/* 'start' has to be the first field otherwise find_subprog() won't work */
- 	u32 start; /* insn idx of function entry point */
- 	u32 linfo_idx; /* The idx to the main_prog->aux->linfo */
-+	u32 postorder_start; /* The idx to the env->cfg.insn_postorder */
-+	u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */
- 	u16 stack_depth; /* max. stack depth used by this function */
-	bool has_tail_call;
-	bool tail_call_reachable;
-	bool has_ld_abs;
-	bool is_async_cb;
-+	u16 stack_extra;
-+	/* offsets in range [stack_depth .. fastcall_stack_off)
-+	 * are used for bpf_fastcall spills and fills.
-+	 */
-+	s16 fastcall_stack_off;
-+	bool has_tail_call: 1;
-+	bool tail_call_reachable: 1;
-+	bool has_ld_abs: 1;
-+	bool is_cb: 1;
-+	bool is_async_cb: 1;
-+	bool is_exception_cb: 1;
-+	bool args_cached: 1;
-+	/* true if bpf_fastcall stack region is used by functions that can't be inlined */
-+	bool keep_fastcall_stack: 1;
-+	bool changes_pkt_data: 1;
-+	bool might_sleep: 1;
-+	u8 arg_cnt:3;
-+
-+	enum priv_stack_mode priv_stack_mode;
-+	struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS];
- };
- 
-+struct bpf_verifier_env;
-+
-+struct backtrack_state {
-+	struct bpf_verifier_env *env;
-+	u32 frame;
-+	u32 reg_masks[MAX_CALL_FRAMES];
-+	u64 stack_masks[MAX_CALL_FRAMES];
-+};
-+
-+struct bpf_id_pair {
-+	u32 old;
-+	u32 cur;
-+};
-+
-+struct bpf_idmap {
-+	u32 tmp_id_gen;
-+	struct bpf_id_pair map[BPF_ID_MAP_SIZE];
-+};
-+
-+struct bpf_idset {
-+	u32 count;
-+	u32 ids[BPF_ID_MAP_SIZE];
-+};
-+
-+/* see verifier.c:compute_scc_callchain() */
-+struct bpf_scc_callchain {
-+	/* call sites from bpf_verifier_state->frame[*]->callsite leading to this SCC */
-+	u32 callsites[MAX_CALL_FRAMES - 1];
-+	/* last frame in a chain is identified by SCC id */
-+	u32 scc;
-+};
-+
-+/* verifier state waiting for propagate_backedges() */
-+struct bpf_scc_backedge {
-+	struct bpf_scc_backedge *next;
-+	struct bpf_verifier_state state;
-+};
-+
-+struct bpf_scc_visit {
-+	struct bpf_scc_callchain callchain;
-+	/* first state in current verification path that entered SCC
-+	 * identified by the callchain
-+	 */
-+	struct bpf_verifier_state *entry_state;
-+	struct bpf_scc_backedge *backedges; /* list of backedges */
-+	u32 num_backedges;
-+};
-+
-+/* An array of bpf_scc_visit structs sharing tht same bpf_scc_callchain->scc
-+ * but having different bpf_scc_callchain->callsites.
-+ */
-+struct bpf_scc_info {
-+	u32 num_visits;
-+	struct bpf_scc_visit visits[];
-+};
-+
-+struct bpf_liveness;
-+
- /* single container for all structs
-  * one verifier_env per bpf_check() call
-  */
-@@ -460,36 +742,61 @@ struct bpf_verifier_env {
- 	u32 prev_insn_idx;
- 	struct bpf_prog *prog;		/* eBPF program being verified */
- 	const struct bpf_verifier_ops *ops;
-+	struct module *attach_btf_mod;	/* The owner module of prog->aux->attach_btf */
- 	struct bpf_verifier_stack_elem *head; /* stack of verifier states to be processed */
- 	int stack_size;			/* number of states to be processed */
- 	bool strict_alignment;		/* perform strict pointer alignment checks */
- 	bool test_state_freq;		/* test verifier with different pruning frequency */
-+	bool test_reg_invariants;	/* fail verification on register invariants violations */
- 	struct bpf_verifier_state *cur_state; /* current verifier state */
-	struct bpf_verifier_state_list **explored_states; /* search pruning optimization */
-	struct bpf_verifier_state_list *free_list;
-+	/* Search pruning optimization, array of list_heads for
-+	 * lists of struct bpf_verifier_state_list.
-+	 */
-+	struct list_head *explored_states;
-+	struct list_head free_list;	/* list of struct bpf_verifier_state_list */
- 	struct bpf_map *used_maps[MAX_USED_MAPS]; /* array of map's used by eBPF program */
- 	struct btf_mod_pair used_btfs[MAX_USED_BTFS]; /* array of BTF's used by BPF program */
-+	struct bpf_map *insn_array_maps[MAX_USED_MAPS]; /* array of INSN_ARRAY map's to be relocated */
- 	u32 used_map_cnt;		/* number of used maps */
- 	u32 used_btf_cnt;		/* number of used BTF objects */
-+	u32 insn_array_map_cnt;		/* number of used maps of type BPF_MAP_TYPE_INSN_ARRAY */
- 	u32 id_gen;			/* used to generate unique reg IDs */
-+	u32 hidden_subprog_cnt;		/* number of hidden subprogs */
-+	int exception_callback_subprog;
- 	bool explore_alu_limits;
- 	bool allow_ptr_leaks;
-+	/* Allow access to uninitialized stack memory. Writes with fixed offset are
-+	 * always allowed, so this refers to reads (with fixed or variable offset),
-+	 * to writes with variable offset and to indirect (helper) accesses.
-+	 */
- 	bool allow_uninit_stack;
-	bool allow_ptr_to_map_access;
- 	bool bpf_capable;
- 	bool bypass_spec_v1;
- 	bool bypass_spec_v4;
- 	bool seen_direct_write;
-+	bool seen_exception;
- 	struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */
- 	const struct bpf_line_info *prev_linfo;
- 	struct bpf_verifier_log log;
-	struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 1];
-	struct bpf_id_pair idmap_scratch[BPF_ID_MAP_SIZE];
-+	struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */
-+	union {
-+		struct bpf_idmap idmap_scratch;
-+		struct bpf_idset idset_scratch;
-+	};
- 	struct {
- 		int *insn_state;
- 		int *insn_stack;
-+		/*
-+		 * vector of instruction indexes sorted in post-order, grouped by subprogram,
-+		 * see bpf_subprog_info->postorder_start.
-+		 */
-+		int *insn_postorder;
- 		int cur_stack;
-+		/* current position in the insn_postorder vector */
-+		int cur_postorder;
- 	} cfg;
-+	struct backtrack_state bt;
-+	struct bpf_jmp_history_entry *cur_hist_ent;
- 	u32 pass_cnt; /* number of times do_check() was called */
- 	u32 subprog_cnt;
- 	/* number of instructions analyzed by the verifier */
-@@ -509,6 +816,9 @@ struct bpf_verifier_env {
- 	u32 peak_states;
- 	/* longest register parentage chain walked for liveness marking */
- 	u32 longest_mark_read_walk;
-+	u32 free_list_size;
-+	u32 explored_states_size;
-+	u32 num_backedges;
- 	bpfptr_t fd_array;
- 
- 	/* bit mask to keep track of whether a register has been accessed
-@@ -517,17 +827,61 @@ struct bpf_verifier_env {
- 	u32 scratched_regs;
- 	/* Same as scratched_regs but for stack slots */
- 	u64 scratched_stack_slots;
-	u32 prev_log_len, prev_insn_print_len;
-	/* buffer used in reg_type_str() to generate reg_type string */
-	char type_str_buf[TYPE_STR_BUF_LEN];
-+	u64 prev_log_pos, prev_insn_print_pos;
-+	/* buffer used to temporary hold constants as scalar registers */
-+	struct bpf_reg_state fake_reg[2];
-+	/* buffer used to generate temporary string representations,
-+	 * e.g., in reg_type_str() to generate reg_type string
-+	 */
-+	char tmp_str_buf[TMP_STR_BUF_LEN];
-+	struct bpf_insn insn_buf[INSN_BUF_SIZE];
-+	struct bpf_insn epilogue_buf[INSN_BUF_SIZE];
-+	struct bpf_scc_callchain callchain_buf;
-+	struct bpf_liveness *liveness;
-+	/* array of pointers to bpf_scc_info indexed by SCC id */
-+	struct bpf_scc_info **scc_info;
-+	u32 scc_cnt;
-+	struct bpf_iarray *succ;
-+	struct bpf_iarray *gotox_tmp_buf;
- };
- 
-+static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)
-+{
-+	return &env->prog->aux->func_info_aux[subprog];
-+}
-+
-+static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env, int subprog)
-+{
-+	return &env->subprog_info[subprog];
-+}
-+
- __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
- 				      const char *fmt, va_list args);
- __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
- 					   const char *fmt, ...);
- __printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
- 			    const char *fmt, ...);
-+int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,
-+		  char __user *log_buf, u32 log_size);
-+void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos);
-+int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual);
-+
-+__printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
-+				  u32 insn_off,
-+				  const char *prefix_fmt, ...);
-+
-+#define verifier_bug_if(cond, env, fmt, args...)						\
-+	({											\
-+		bool __cond = (cond);								\
-+		if (unlikely(__cond))								\
-+			verifier_bug(env, fmt " (" #cond ")", ##args);				\
-+		(__cond);									\
-+	})
-+#define verifier_bug(env, fmt, args...)								\
-+	({											\
-+		BPF_WARN_ONCE(1, "verifier bug: " fmt "\n", ##args);				\
-+		bpf_log(&env->log, "verifier bug: " fmt "\n", ##args);				\
-+	})
- 
- static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)
- {
-@@ -551,16 +905,6 @@ bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,
- void
- bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt);
- 
-int check_ptr_off_reg(struct bpf_verifier_env *env,
-		      const struct bpf_reg_state *reg, int regno);
-int check_func_arg_reg_off(struct bpf_verifier_env *env,
-			   const struct bpf_reg_state *reg, int regno,
-			   enum bpf_arg_type arg_type);
-int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
-			     u32 regno);
-int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
-		   u32 regno, u32 mem_size);
-
- /* this lives here instead of in bpf.h because it needs to dereference tgt_prog */
- static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,
- 					     struct btf *btf, u32 btf_id)
-@@ -587,6 +931,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
- 			    struct bpf_attach_target_info *tgt_info);
- void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab);
- 
-+int mark_chain_precision(struct bpf_verifier_env *env, int regno);
-+
- #define BPF_BASE_TYPE_MASK	GENMASK(BPF_BASE_TYPE_BITS - 1, 0)
- 
- /* extract base type from bpf_{arg, return, reg}_type. */
-@@ -602,10 +948,143 @@ static inline u32 type_flag(u32 type)
- }
- 
- /* only use after check_attach_btf_id() */
-static inline enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog)
-+static inline enum bpf_prog_type resolve_prog_type(const struct bpf_prog *prog)
-+{
-+	return (prog->type == BPF_PROG_TYPE_EXT && prog->aux->saved_dst_prog_type) ?
-+		prog->aux->saved_dst_prog_type : prog->type;
-+}
-+
-+static inline bool bpf_prog_check_recur(const struct bpf_prog *prog)
-+{
-+	switch (resolve_prog_type(prog)) {
-+	case BPF_PROG_TYPE_TRACING:
-+		return prog->expected_attach_type != BPF_TRACE_ITER;
-+	case BPF_PROG_TYPE_STRUCT_OPS:
-+		return prog->aux->jits_use_priv_stack;
-+	case BPF_PROG_TYPE_LSM:
-+	case BPF_PROG_TYPE_SYSCALL:
-+		return false;
-+	default:
-+		return true;
-+	}
-+}
-+
-+#define BPF_REG_TRUSTED_MODIFIERS (MEM_ALLOC | PTR_TRUSTED | NON_OWN_REF)
-+
-+static inline bool bpf_type_has_unsafe_modifiers(u32 type)
-+{
-+	return type_flag(type) & ~BPF_REG_TRUSTED_MODIFIERS;
-+}
-+
-+static inline bool type_is_ptr_alloc_obj(u32 type)
-+{
-+	return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC;
-+}
-+
-+static inline bool type_is_non_owning_ref(u32 type)
-+{
-+	return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF;
-+}
-+
-+static inline bool type_is_pkt_pointer(enum bpf_reg_type type)
-+{
-+	type = base_type(type);
-+	return type == PTR_TO_PACKET ||
-+	       type == PTR_TO_PACKET_META;
-+}
-+
-+static inline bool type_is_sk_pointer(enum bpf_reg_type type)
-+{
-+	return type == PTR_TO_SOCKET ||
-+		type == PTR_TO_SOCK_COMMON ||
-+		type == PTR_TO_TCP_SOCK ||
-+		type == PTR_TO_XDP_SOCK;
-+}
-+
-+static inline bool type_may_be_null(u32 type)
-+{
-+	return type & PTR_MAYBE_NULL;
-+}
-+
-+static inline void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)
-+{
-+	env->scratched_regs |= 1U << regno;
-+}
-+
-+static inline void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi)
- {
-	return prog->type == BPF_PROG_TYPE_EXT ?
-		prog->aux->dst_prog->type : prog->type;
-+	env->scratched_stack_slots |= 1ULL << spi;
- }
- 
-+static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)
-+{
-+	return (env->scratched_regs >> regno) & 1;
-+}
-+
-+static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno)
-+{
-+	return (env->scratched_stack_slots >> regno) & 1;
-+}
-+
-+static inline bool verifier_state_scratched(const struct bpf_verifier_env *env)
-+{
-+	return env->scratched_regs || env->scratched_stack_slots;
-+}
-+
-+static inline void mark_verifier_state_clean(struct bpf_verifier_env *env)
-+{
-+	env->scratched_regs = 0U;
-+	env->scratched_stack_slots = 0ULL;
-+}
-+
-+/* Used for printing the entire verifier state. */
-+static inline void mark_verifier_state_scratched(struct bpf_verifier_env *env)
-+{
-+	env->scratched_regs = ~0U;
-+	env->scratched_stack_slots = ~0ULL;
-+}
-+
-+static inline bool bpf_stack_narrow_access_ok(int off, int fill_size, int spill_size)
-+{
-+#ifdef __BIG_ENDIAN
-+	off -= spill_size - fill_size;
-+#endif
-+
-+	return !(off % BPF_REG_SIZE);
-+}
-+
-+static inline bool insn_is_gotox(struct bpf_insn *insn)
-+{
-+	return BPF_CLASS(insn->code) == BPF_JMP &&
-+	       BPF_OP(insn->code) == BPF_JA &&
-+	       BPF_SRC(insn->code) == BPF_X;
-+}
-+
-+const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type);
-+const char *dynptr_type_str(enum bpf_dynptr_type type);
-+const char *iter_type_str(const struct btf *btf, u32 btf_id);
-+const char *iter_state_str(enum bpf_iter_state state);
-+
-+void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,
-+			  u32 frameno, bool print_all);
-+void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,
-+		      u32 frameno);
-+
-+struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off);
-+int bpf_jmp_offset(struct bpf_insn *insn);
-+struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);
-+void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);
-+bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx);
-+
-+int bpf_stack_liveness_init(struct bpf_verifier_env *env);
-+void bpf_stack_liveness_free(struct bpf_verifier_env *env);
-+int bpf_update_live_stack(struct bpf_verifier_env *env);
-+int bpf_mark_stack_read(struct bpf_verifier_env *env, u32 frameno, u32 insn_idx, u64 mask);
-+void bpf_mark_stack_write(struct bpf_verifier_env *env, u32 frameno, u64 mask);
-+int bpf_reset_stack_write_marks(struct bpf_verifier_env *env, u32 insn_idx);
-+int bpf_commit_stack_write_marks(struct bpf_verifier_env *env);
-+int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
-+bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi);
-+void bpf_reset_live_stack_callchain(struct bpf_verifier_env *env);
-+
- #endif /* _LINUX_BPF_VERIFIER_H */
-diff --git a/include/linux/bpfilter.h b/include/linux/bpfilter.h
-deleted file mode 100644
-index 2ae3c8e1d83c..000000000000
--- a/include/linux/bpfilter.h
-+++ /dev/null
-@@ -1,25 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _LINUX_BPFILTER_H
-#define _LINUX_BPFILTER_H
-
-#include <uapi/linux/bpfilter.h>
-#include <linux/usermode_driver.h>
-#include <linux/sockptr.h>
-
-struct sock;
-int bpfilter_ip_set_sockopt(struct sock *sk, int optname, sockptr_t optval,
-			    unsigned int optlen);
-int bpfilter_ip_get_sockopt(struct sock *sk, int optname, char __user *optval,
-			    int __user *optlen);
-void bpfilter_umh_cleanup(struct umd_info *info);
-
-struct bpfilter_umh_ops {
-	struct umd_info info;
-	/* since ip_getsockopt() can run in parallel, serialize access to umh */
-	struct mutex lock;
-	int (*sockopt)(struct sock *sk, int optname, sockptr_t optval,
-		       unsigned int optlen, bool is_set);
-	int (*start)(void);
-};
-extern struct bpfilter_umh_ops bpfilter_ops;
-#endif
-diff --git a/include/linux/bpfptr.h b/include/linux/bpfptr.h
-index 79b2f78eec1a..f6e0795db484 100644
--- a/include/linux/bpfptr.h
-+++ b/include/linux/bpfptr.h
-@@ -65,9 +65,9 @@ static inline int copy_to_bpfptr_offset(bpfptr_t dst, size_t offset,
- 	return copy_to_sockptr_offset((sockptr_t) dst, offset, src, size);
- }
- 
-static inline void *kvmemdup_bpfptr(bpfptr_t src, size_t len)
-+static inline void *kvmemdup_bpfptr_noprof(bpfptr_t src, size_t len)
- {
-	void *p = kvmalloc(len, GFP_USER | __GFP_NOWARN);
-+	void *p = kvmalloc_node_align_noprof(len, 1, GFP_USER | __GFP_NOWARN, NUMA_NO_NODE);
- 
- 	if (!p)
- 		return ERR_PTR(-ENOMEM);
-@@ -77,6 +77,7 @@ static inline void *kvmemdup_bpfptr(bpfptr_t src, size_t len)
- 	}
- 	return p;
- }
-+#define kvmemdup_bpfptr(...)	alloc_hooks(kvmemdup_bpfptr_noprof(__VA_ARGS__))
- 
- static inline long strncpy_from_bpfptr(char *dst, bpfptr_t src, size_t count)
- {
-diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h
-index 6ff567ece34a..115a964f3006 100644
--- a/include/linux/brcmphy.h
-+++ b/include/linux/brcmphy.h
-@@ -11,6 +11,7 @@
- 
- #define PHY_ID_BCM50610			0x0143bd60
- #define PHY_ID_BCM50610M		0x0143bd70
-+#define PHY_ID_BCM5221			0x004061e0
- #define PHY_ID_BCM5241			0x0143bc30
- #define PHY_ID_BCMAC131			0x0143bc70
- #define PHY_ID_BCM5481			0x0143bca0
-@@ -44,6 +45,7 @@
- #define PHY_ID_BCM7366			0x600d8490
- #define PHY_ID_BCM7346			0x600d8650
- #define PHY_ID_BCM7362			0x600d84b0
-+#define PHY_ID_BCM74165			0x359052c0
- #define PHY_ID_BCM7425			0x600d86b0
- #define PHY_ID_BCM7429			0x600d8730
- #define PHY_ID_BCM7435			0x600d8750
-@@ -89,6 +91,7 @@
- #define MII_BCM54XX_EXP_SEL	0x17	/* Expansion register select */
- #define MII_BCM54XX_EXP_SEL_TOP	0x0d00	/* TOP_MISC expansion register select */
- #define MII_BCM54XX_EXP_SEL_SSD	0x0e00	/* Secondary SerDes select */
-+#define MII_BCM54XX_EXP_SEL_WOL	0x0e00	/* Wake-on-LAN expansion select register */
- #define MII_BCM54XX_EXP_SEL_ER	0x0f00	/* Expansion register select */
- #define MII_BCM54XX_EXP_SEL_ETC	0x0d00	/* Expansion register spare + 2k mem */
- 
-@@ -134,6 +137,7 @@
- 
- #define MII_BCM54XX_AUXCTL_SHDWSEL_MISC			0x07
- #define MII_BCM54XX_AUXCTL_SHDWSEL_MISC_WIRESPEED_EN	0x0010
-+#define MII_BCM54XX_AUXCTL_SHDWSEL_MISC_RSVD		0x0060
- #define MII_BCM54XX_AUXCTL_SHDWSEL_MISC_RGMII_EN	0x0080
- #define MII_BCM54XX_AUXCTL_SHDWSEL_MISC_RGMII_SKEW_EN	0x0100
- #define MII_BCM54XX_AUXCTL_MISC_FORCE_AMDIX		0x0200
-@@ -160,6 +164,7 @@
- #define BCM_LED_SRC_OPENSHORT	0xb
- #define BCM_LED_SRC_OFF		0xe	/* Tied high */
- #define BCM_LED_SRC_ON		0xf	/* Tied low */
-+#define BCM_LED_SRC_MASK	GENMASK(3, 0)
- 
- /*
-  * Broadcom Multicolor LED configurations (expansion register 4)
-@@ -179,6 +184,12 @@
- #define BCM_LED_MULTICOLOR_PROGRAM	0xa
- 
- /*
-+ * Broadcom Synchronous Ethernet Controls (expansion register 0x0E)
-+ */
-+#define BCM_EXP_SYNC_ETHERNET		(MII_BCM54XX_EXP_SEL_ER + 0x0E)
-+#define BCM_EXP_SYNC_ETHERNET_MII_LITE	BIT(11)
-+
-+/*
-  * BCM5482: Shadow registers
-  * Shadow values go into bits [14:10] of register 0x1c to select a shadow
-  * register to access.
-@@ -205,11 +216,13 @@
- #define  BCM_NO_ANEG_APD_EN		0x0060 /* bits 5 & 6 */
- #define  BCM_APD_SINGLELP_EN	0x0100 /* Bit 8 */
- 
-#define BCM5482_SHD_LEDS1	0x0d	/* 01101: LED Selector 1 */
-+#define BCM54XX_SHD_LEDS1	0x0d	/* 01101: LED Selector 1 */
- 					/* LED3 / ~LINKSPD[2] selector */
-#define BCM5482_SHD_LEDS1_LED3(src)	((src & 0xf) << 4)
-+#define BCM54XX_SHD_LEDS_SHIFT(led)	(4 * (led))
-+#define BCM54XX_SHD_LEDS1_LED3(src)	((src & 0xf) << 4)
- 					/* LED1 / ~LINKSPD[1] selector */
-#define BCM5482_SHD_LEDS1_LED1(src)	((src & 0xf) << 0)
-+#define BCM54XX_SHD_LEDS1_LED1(src)	((src & 0xf) << 0)
-+#define BCM54XX_SHD_LEDS2	0x0e	/* 01110: LED Selector 2 */
- #define BCM54XX_SHD_RGMII_MODE	0x0b	/* 01011: RGMII Mode Selector */
- #define BCM5482_SHD_SSD		0x14	/* 10100: Secondary SerDes control */
- #define BCM5482_SHD_SSD_LEDM	0x0008	/* SSD LED Mode enable */
-@@ -253,6 +266,9 @@
- #define BCM54XX_TOP_MISC_IDDQ_SD		(1 << 2)
- #define BCM54XX_TOP_MISC_IDDQ_SR		(1 << 3)
- 
-+#define BCM54XX_TOP_MISC_LED_CTL		(MII_BCM54XX_EXP_SEL_TOP + 0x0C)
-+#define  BCM54XX_LED4_SEL_INTR			BIT(1)
-+
- /*
-  * BCM5482: Secondary SerDes registers
-  */
-@@ -262,16 +278,164 @@
- #define BCM5482_SSD_SGMII_SLAVE_EN	0x0002	/* Slave mode enable */
- #define BCM5482_SSD_SGMII_SLAVE_AD	0x0001	/* Slave auto-detection */
- 
-+/* BroadR-Reach LRE Registers. */
-+#define MII_BCM54XX_LRECR		0x00	/* LRE Control Register                    */
-+#define MII_BCM54XX_LRESR		0x01	/* LRE Status Register                     */
-+#define MII_BCM54XX_LREPHYSID1		0x02	/* LRE PHYS ID 1                           */
-+#define MII_BCM54XX_LREPHYSID2		0x03	/* LRE PHYS ID 2                           */
-+#define MII_BCM54XX_LREANAA		0x04	/* LDS Auto-Negotiation Advertised Ability */
-+#define MII_BCM54XX_LREANAC		0x05	/* LDS Auto-Negotiation Advertised Control */
-+#define MII_BCM54XX_LREANPT		0x06	/* LDS Ability Next Page Transmit          */
-+#define MII_BCM54XX_LRELPA		0x07	/* LDS Link Partner Ability                */
-+#define MII_BCM54XX_LRELPNPM		0x08	/* LDS Link Partner Next Page Message      */
-+#define MII_BCM54XX_LRELPNPC		0x09	/* LDS Link Partner Next Page Control      */
-+#define MII_BCM54XX_LRELDSE		0x0a	/* LDS Expansion Register                  */
-+#define MII_BCM54XX_LREES		0x0f	/* LRE Extended Status                     */
-+
-+/* LRE control register. */
-+#define LRECR_RESET			0x8000	/* Reset to default state      */
-+#define LRECR_LOOPBACK			0x4000	/* Internal Loopback           */
-+#define LRECR_LDSRES			0x2000	/* Restart LDS Process         */
-+#define LRECR_LDSEN			0x1000	/* LDS Enable                  */
-+#define LRECR_PDOWN			0x0800	/* Enable low power state      */
-+#define LRECR_ISOLATE			0x0400	/* Isolate data paths from MII */
-+#define LRECR_SPEED100			0x0200	/* Select 100 Mbps             */
-+#define LRECR_SPEED10			0x0000	/* Select 10 Mbps              */
-+#define LRECR_4PAIRS			0x0020	/* Select 4 Pairs              */
-+#define LRECR_2PAIRS			0x0010	/* Select 2 Pairs              */
-+#define LRECR_1PAIR			0x0000	/* Select 1 Pair               */
-+#define LRECR_MASTER			0x0008	/* Force Master when LDS disabled */
-+#define LRECR_SLAVE			0x0000	/* Force Slave when LDS disabled  */
-+
-+/* LRE status register. */
-+#define LRESR_100_1PAIR			0x2000	/* Can do 100Mbps 1 Pair       */
-+#define LRESR_100_4PAIR			0x1000	/* Can do 100Mbps 4 Pairs      */
-+#define LRESR_100_2PAIR			0x0800	/* Can do 100Mbps 2 Pairs      */
-+#define LRESR_10_2PAIR			0x0400	/* Can do 10Mbps 2 Pairs       */
-+#define LRESR_10_1PAIR			0x0200	/* Can do 10Mbps 1 Pair        */
-+#define LRESR_ESTATEN			0x0100	/* Extended Status in R15      */
-+#define LRESR_RESV			0x0080	/* Unused...                   */
-+#define LRESR_MFPS			0x0040	/* Can suppress Management Frames Preamble */
-+#define LRESR_LDSCOMPLETE		0x0020	/* LDS Auto-negotiation complete */
-+#define LRESR_8023			0x0010	/* Has IEEE 802.3 Support      */
-+#define LRESR_LDSABILITY		0x0008	/* LDS auto-negotiation capable */
-+#define LRESR_LSTATUS			0x0004	/* Link status                 */
-+#define LRESR_JCD			0x0002	/* Jabber detected             */
-+#define LRESR_ERCAP			0x0001	/* Ext-reg capability          */
-+
-+/* LDS Auto-Negotiation Advertised Ability. */
-+#define LREANAA_PAUSE_ASYM		0x8000	/* Can pause asymmetrically    */
-+#define LREANAA_PAUSE			0x4000	/* Can pause                   */
-+#define LREANAA_100_1PAIR		0x0020	/* Can do 100Mbps 1 Pair       */
-+#define LREANAA_100_4PAIR		0x0010	/* Can do 100Mbps 4 Pair       */
-+#define LREANAA_100_2PAIR		0x0008	/* Can do 100Mbps 2 Pair       */
-+#define LREANAA_10_2PAIR		0x0004	/* Can do 10Mbps 2 Pair        */
-+#define LREANAA_10_1PAIR		0x0002	/* Can do 10Mbps 1 Pair        */
-+
-+#define LRE_ADVERTISE_FULL		(LREANAA_100_1PAIR | LREANAA_100_4PAIR | \
-+					 LREANAA_100_2PAIR | LREANAA_10_2PAIR | \
-+					 LREANAA_10_1PAIR)
-+
-+#define LRE_ADVERTISE_ALL		LRE_ADVERTISE_FULL
-+
-+/* LDS Link Partner Ability. */
-+#define LRELPA_PAUSE_ASYM		0x8000	/* Supports asymmetric pause   */
-+#define LRELPA_PAUSE			0x4000	/* Supports pause capability   */
-+#define LRELPA_100_1PAIR		0x0020	/* 100Mbps 1 Pair capable      */
-+#define LRELPA_100_4PAIR		0x0010	/* 100Mbps 4 Pair capable      */
-+#define LRELPA_100_2PAIR		0x0008	/* 100Mbps 2 Pair capable      */
-+#define LRELPA_10_2PAIR			0x0004	/* 10Mbps 2 Pair capable       */
-+#define LRELPA_10_1PAIR			0x0002	/* 10Mbps 1 Pair capable       */
-+
-+/* LDS Expansion register. */
-+#define LDSE_DOWNGRADE			0x8000	/* Can do LDS Speed Downgrade  */
-+#define LDSE_MASTER			0x4000	/* Master / Slave              */
-+#define LDSE_PAIRS_MASK			0x3000	/* Pair Count Mask             */
-+#define LDSE_PAIRS_SHIFT		12
-+#define LDSE_4PAIRS			(2 << LDSE_PAIRS_SHIFT)	/* 4 Pairs Connection */
-+#define LDSE_2PAIRS			(1 << LDSE_PAIRS_SHIFT)	/* 2 Pairs Connection */
-+#define LDSE_1PAIR			(0 << LDSE_PAIRS_SHIFT)	/* 1 Pair  Connection */
-+#define LDSE_CABLEN_MASK		0x0FFF	/* Cable Length Mask           */
-+
- /* BCM54810 Registers */
- #define BCM54810_EXP_BROADREACH_LRE_MISC_CTL	(MII_BCM54XX_EXP_SEL_ER + 0x90)
- #define BCM54810_EXP_BROADREACH_LRE_MISC_CTL_EN	(1 << 0)
- #define BCM54810_SHD_CLK_CTL			0x3
- #define BCM54810_SHD_CLK_CTL_GTXCLK_EN		(1 << 9)
- 
-+/* BCM54811 Registers */
-+#define BCM54811_EXP_BROADREACH_LRE_OVERLAY_CTL	(MII_BCM54XX_EXP_SEL_ER + 0x9A)
-+/* Access Control Override Enable */
-+#define BCM54811_EXP_BROADREACH_LRE_OVERLAY_CTL_EN		BIT(15)
-+/* Access Control Override Value */
-+#define BCM54811_EXP_BROADREACH_LRE_OVERLAY_CTL_OVERRIDE_VAL	BIT(14)
-+/* Access Control Value */
-+#define BCM54811_EXP_BROADREACH_LRE_OVERLAY_CTL_VAL		BIT(13)
-+
- /* BCM54612E Registers */
- #define BCM54612E_EXP_SPARE0		(MII_BCM54XX_EXP_SEL_ETC + 0x34)
- #define BCM54612E_LED4_CLK125OUT_EN	(1 << 1)
- 
-+
-+/* Wake-on-LAN registers */
-+#define BCM54XX_WOL_MAIN_CTL		(MII_BCM54XX_EXP_SEL_WOL + 0x80)
-+#define  BCM54XX_WOL_EN			BIT(0)
-+#define  BCM54XX_WOL_MODE_SINGLE_MPD	0
-+#define  BCM54XX_WOL_MODE_SINGLE_MPDSEC	1
-+#define  BCM54XX_WOL_MODE_DUAL		2
-+#define  BCM54XX_WOL_MODE_SHIFT		1
-+#define  BCM54XX_WOL_MODE_MASK		0x3
-+#define  BCM54XX_WOL_MP_MSB_FF_EN	BIT(3)
-+#define  BCM54XX_WOL_SECKEY_OPT_4B	0
-+#define  BCM54XX_WOL_SECKEY_OPT_6B	1
-+#define  BCM54XX_WOL_SECKEY_OPT_8B	2
-+#define  BCM54XX_WOL_SECKEY_OPT_SHIFT	4
-+#define  BCM54XX_WOL_SECKEY_OPT_MASK	0x3
-+#define  BCM54XX_WOL_L2_TYPE_CHK	BIT(6)
-+#define  BCM54XX_WOL_L4IPV4UDP_CHK	BIT(7)
-+#define  BCM54XX_WOL_L4IPV6UDP_CHK	BIT(8)
-+#define  BCM54XX_WOL_UDPPORT_CHK	BIT(9)
-+#define  BCM54XX_WOL_CRC_CHK		BIT(10)
-+#define  BCM54XX_WOL_SECKEY_MODE	BIT(11)
-+#define  BCM54XX_WOL_RST		BIT(12)
-+#define  BCM54XX_WOL_DIR_PKT_EN		BIT(13)
-+#define  BCM54XX_WOL_MASK_MODE_DA_FF	0
-+#define  BCM54XX_WOL_MASK_MODE_DA_MPD	1
-+#define  BCM54XX_WOL_MASK_MODE_DA_ONLY	2
-+#define  BCM54XX_WOL_MASK_MODE_MPD	3
-+#define  BCM54XX_WOL_MASK_MODE_SHIFT	14
-+#define  BCM54XX_WOL_MASK_MODE_MASK	0x3
-+
-+#define BCM54XX_WOL_INNER_PROTO		(MII_BCM54XX_EXP_SEL_WOL + 0x81)
-+#define BCM54XX_WOL_OUTER_PROTO		(MII_BCM54XX_EXP_SEL_WOL + 0x82)
-+#define BCM54XX_WOL_OUTER_PROTO2	(MII_BCM54XX_EXP_SEL_WOL + 0x83)
-+
-+#define BCM54XX_WOL_MPD_DATA1(x)	(MII_BCM54XX_EXP_SEL_WOL + 0x84 + (x))
-+#define BCM54XX_WOL_MPD_DATA2(x)	(MII_BCM54XX_EXP_SEL_WOL + 0x87 + (x))
-+#define BCM54XX_WOL_SEC_KEY_8B		(MII_BCM54XX_EXP_SEL_WOL + 0x8A)
-+#define BCM54XX_WOL_MASK(x)		(MII_BCM54XX_EXP_SEL_WOL + 0x8B + (x))
-+#define BCM54XX_SEC_KEY_STORE(x)	(MII_BCM54XX_EXP_SEL_WOL + 0x8E)
-+#define BCM54XX_WOL_SHARED_CNT		(MII_BCM54XX_EXP_SEL_WOL + 0x92)
-+
-+#define BCM54XX_WOL_INT_MASK		(MII_BCM54XX_EXP_SEL_WOL + 0x93)
-+#define  BCM54XX_WOL_PKT1		BIT(0)
-+#define  BCM54XX_WOL_PKT2		BIT(1)
-+#define  BCM54XX_WOL_DIR		BIT(2)
-+#define  BCM54XX_WOL_ALL_INTRS		(BCM54XX_WOL_PKT1 | \
-+					 BCM54XX_WOL_PKT2 | \
-+					 BCM54XX_WOL_DIR)
-+
-+#define BCM54XX_WOL_INT_STATUS		(MII_BCM54XX_EXP_SEL_WOL + 0x94)
-+
-+/* BCM5221 Registers */
-+#define BCM5221_AEGSR			0x1C
-+#define BCM5221_AEGSR_MDIX_STATUS	BIT(13)
-+#define BCM5221_AEGSR_MDIX_MAN_SWAP	BIT(12)
-+#define BCM5221_AEGSR_MDIX_DIS		BIT(11)
-+
-+#define BCM5221_SHDW_AM4_EN_CLK_LPM	BIT(2)
-+#define BCM5221_SHDW_AM4_FORCE_LPM	BIT(1)
-+
- /*****************************************************************************/
- /* Fast Ethernet Transceiver definitions. */
- /*****************************************************************************/
-@@ -293,6 +457,7 @@
- #define MII_BRCM_FET_SHDW_MC_FAME	0x4000	/* Force Auto MDIX enable */
- 
- #define MII_BRCM_FET_SHDW_AUXMODE4	0x1a	/* Auxiliary mode 4 */
-+#define MII_BRCM_FET_SHDW_AM4_STANDBY	0x0008	/* Standby enable */
- #define MII_BRCM_FET_SHDW_AM4_LED_MASK	0x0003
- #define MII_BRCM_FET_SHDW_AM4_LED_MODE1 0x0001
- 
-@@ -303,6 +468,8 @@
- #define LPI_FEATURE_EN			0x8000
- #define LPI_FEATURE_EN_DIG1000X		0x4000
- 
-+#define BRCM_CL45VEN_EEE_LPI_CNT	0x803f
-+
- /* Core register definitions*/
- #define MII_BRCM_CORE_BASE12	0x12
- #define MII_BRCM_CORE_BASE13	0x13
-diff --git a/include/linux/bsg-lib.h b/include/linux/bsg-lib.h
-index 9e97ced2896d..14fa93268630 100644
--- a/include/linux/bsg-lib.h
-+++ b/include/linux/bsg-lib.h
-@@ -65,7 +65,8 @@ struct bsg_job {
- void bsg_job_done(struct bsg_job *job, int result,
- 		  unsigned int reply_payload_rcv_len);
- struct request_queue *bsg_setup_queue(struct device *dev, const char *name,
-		bsg_job_fn *job_fn, bsg_timeout_fn *timeout, int dd_job_size);
-+		struct queue_limits *lim, bsg_job_fn *job_fn,
-+		bsg_timeout_fn *timeout, int dd_job_size);
- void bsg_remove_queue(struct request_queue *q);
- void bsg_job_put(struct bsg_job *job);
- int __must_check bsg_job_get(struct bsg_job *job);
-diff --git a/include/linux/bsg.h b/include/linux/bsg.h
-index 1ac81c809da9..ee2df73edf83 100644
--- a/include/linux/bsg.h
-+++ b/include/linux/bsg.h
-@@ -9,7 +9,7 @@ struct device;
- struct request_queue;
- 
- typedef int (bsg_sg_io_fn)(struct request_queue *, struct sg_io_v4 *hdr,
-		fmode_t mode, unsigned int timeout);
-+		bool open_for_write, unsigned int timeout);
- 
- struct bsg_device *bsg_register_queue(struct request_queue *q,
- 		struct device *parent, const char *name,
-diff --git a/include/linux/btf.h b/include/linux/btf.h
-index cdb376d53238..f06976ffb63f 100644
--- a/include/linux/btf.h
-+++ b/include/linux/btf.h
-@@ -6,6 +6,8 @@
- 
- #include <linux/types.h>
- #include <linux/bpfptr.h>
-+#include <linux/bsearch.h>
-+#include <linux/btf_ids.h>
- #include <uapi/linux/btf.h>
- #include <uapi/linux/bpf.h>
- 
-@@ -16,39 +18,95 @@
- #define KF_ACQUIRE	(1 << 0) /* kfunc is an acquire function */
- #define KF_RELEASE	(1 << 1) /* kfunc is a release function */
- #define KF_RET_NULL	(1 << 2) /* kfunc returns a pointer that may be NULL */
-#define KF_KPTR_GET	(1 << 3) /* kfunc returns reference to a kptr */
-/* Trusted arguments are those which are meant to be referenced arguments with
- * unchanged offset. It is used to enforce that pointers obtained from acquire
- * kfuncs remain unmodified when being passed to helpers taking trusted args.
-+/* Trusted arguments are those which are guaranteed to be valid when passed to
-+ * the kfunc. It is used to enforce that pointers obtained from either acquire
-+ * kfuncs, or from the main kernel on a tracepoint or struct_ops callback
-+ * invocation, remain unmodified when being passed to helpers taking trusted
-+ * args.
-  *
- * Consider
- *	struct foo {
- *		int data;
- *		struct foo *next;
- *	};
-+ * Consider, for example, the following new task tracepoint:
-  *
- *	struct bar {
- *		int data;
- *		struct foo f;
- *	};
-+ *	SEC("tp_btf/task_newtask")
-+ *	int BPF_PROG(new_task_tp, struct task_struct *task, u64 clone_flags)
-+ *	{
-+ *		...
-+ *	}
-  *
- *	struct foo *f = alloc_foo(); // Acquire kfunc
- *	struct bar *b = alloc_bar(); // Acquire kfunc
-+ * And the following kfunc:
-  *
- * If a kfunc set_foo_data() wants to operate only on the allocated object, it
- * will set the KF_TRUSTED_ARGS flag, which will prevent unsafe usage like:
-+ *	BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_TRUSTED_ARGS)
-  *
- *	set_foo_data(f, 42);	   // Allowed
- *	set_foo_data(f->next, 42); // Rejected, non-referenced pointer
- *	set_foo_data(&f->next, 42);// Rejected, referenced, but wrong type
- *	set_foo_data(&b->f, 42);   // Rejected, referenced, but bad offset
-+ * All invocations to the kfunc must pass the unmodified, unwalked task:
-  *
- * In the final case, usually for the purposes of type matching, it is deduced
- * by looking at the type of the member at the offset, but due to the
- * requirement of trusted argument, this deduction will be strict and not done
- * for this case.
-+ *	bpf_task_acquire(task);		    // Allowed
-+ *	bpf_task_acquire(task->last_wakee); // Rejected, walked task
-+ *
-+ * Programs may also pass referenced tasks directly to the kfunc:
-+ *
-+ *	struct task_struct *acquired;
-+ *
-+ *	acquired = bpf_task_acquire(task);	// Allowed, same as above
-+ *	bpf_task_acquire(acquired);		// Allowed
-+ *	bpf_task_acquire(task);			// Allowed
-+ *	bpf_task_acquire(acquired->last_wakee); // Rejected, walked task
-+ *
-+ * Programs may _not_, however, pass a task from an arbitrary fentry/fexit, or
-+ * kprobe/kretprobe to the kfunc, as BPF cannot guarantee that all of these
-+ * pointers are guaranteed to be safe. For example, the following BPF program
-+ * would be rejected:
-+ *
-+ * SEC("kretprobe/free_task")
-+ * int BPF_PROG(free_task_probe, struct task_struct *tsk)
-+ * {
-+ *	struct task_struct *acquired;
-+ *
-+ *	acquired = bpf_task_acquire(acquired); // Rejected, not a trusted pointer
-+ *	bpf_task_release(acquired);
-+ *
-+ *	return 0;
-+ * }
-  */
- #define KF_TRUSTED_ARGS (1 << 4) /* kfunc only takes trusted pointer arguments */
-+#define KF_SLEEPABLE    (1 << 5) /* kfunc may sleep */
-+#define KF_DESTRUCTIVE  (1 << 6) /* kfunc performs destructive actions */
-+#define KF_RCU          (1 << 7) /* kfunc takes either rcu or trusted pointer arguments */
-+/* only one of KF_ITER_{NEW,NEXT,DESTROY} could be specified per kfunc */
-+#define KF_ITER_NEW     (1 << 8) /* kfunc implements BPF iter constructor */
-+#define KF_ITER_NEXT    (1 << 9) /* kfunc implements BPF iter next method */
-+#define KF_ITER_DESTROY (1 << 10) /* kfunc implements BPF iter destructor */
-+#define KF_RCU_PROTECTED (1 << 11) /* kfunc should be protected by rcu cs when they are invoked */
-+#define KF_FASTCALL     (1 << 12) /* kfunc supports bpf_fastcall protocol */
-+#define KF_ARENA_RET    (1 << 13) /* kfunc returns an arena pointer */
-+#define KF_ARENA_ARG1   (1 << 14) /* kfunc takes an arena pointer as its first argument */
-+#define KF_ARENA_ARG2   (1 << 15) /* kfunc takes an arena pointer as its second argument */
-+
-+/*
-+ * Tag marking a kernel function as a kfunc. This is meant to minimize the
-+ * amount of copy-paste that kfunc authors have to include for correctness so
-+ * as to avoid issues such as the compiler inlining or eliding either a static
-+ * kfunc, or a global kfunc in an LTO build.
-+ */
-+#define __bpf_kfunc __used __retain __noclone noinline
-+
-+#define __bpf_kfunc_start_defs()					       \
-+	__diag_push();							       \
-+	__diag_ignore_all("-Wmissing-declarations",			       \
-+			  "Global kfuncs as their definitions will be in BTF");\
-+	__diag_ignore_all("-Wmissing-prototypes",			       \
-+			  "Global kfuncs as their definitions will be in BTF")
-+
-+#define __bpf_kfunc_end_defs() __diag_pop()
-+#define __bpf_hook_start() __bpf_kfunc_start_defs()
-+#define __bpf_hook_end() __bpf_kfunc_end_defs()
-+
-+/*
-+ * Return the name of the passed struct, if exists, or halt the build if for
-+ * example the structure gets renamed. In this way, developers have to revisit
-+ * the code using that structure name, and update it accordingly.
-+ */
-+#define stringify_struct(x)			\
-+	({ BUILD_BUG_ON(sizeof(struct x) < 0);	\
-+	   __stringify(x); })
- 
- struct btf;
- struct btf_member;
-@@ -56,10 +114,14 @@ struct btf_type;
- union bpf_attr;
- struct btf_show;
- struct btf_id_set;
-+struct bpf_prog;
-+
-+typedef int (*btf_kfunc_filter_t)(const struct bpf_prog *prog, u32 kfunc_id);
- 
- struct btf_kfunc_id_set {
- 	struct module *owner;
- 	struct btf_id_set8 *set;
-+	btf_kfunc_filter_t filter;
- };
- 
- struct btf_id_dtor_kfunc {
-@@ -67,13 +129,23 @@ struct btf_id_dtor_kfunc {
- 	u32 kfunc_btf_id;
- };
- 
-typedef void (*btf_dtor_kfunc_t)(void *);
-+struct btf_struct_meta {
-+	u32 btf_id;
-+	struct btf_record *record;
-+};
-+
-+struct btf_struct_metas {
-+	u32 cnt;
-+	struct btf_struct_meta types[];
-+};
- 
- extern const struct file_operations btf_fops;
- 
-+const char *btf_get_name(const struct btf *btf);
- void btf_get(struct btf *btf);
- void btf_put(struct btf *btf);
-int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr);
-+const struct btf_header *btf_header(const struct btf *btf);
-+int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_sz);
- struct btf *btf_get_by_fd(int fd);
- int btf_get_info_by_fd(const struct btf *btf,
- 		       const union bpf_attr *attr,
-@@ -145,17 +217,22 @@ int btf_get_fd_by_id(u32 id);
- u32 btf_obj_id(const struct btf *btf);
- bool btf_is_kernel(const struct btf *btf);
- bool btf_is_module(const struct btf *btf);
-+bool btf_is_vmlinux(const struct btf *btf);
- struct module *btf_try_get_module(const struct btf *btf);
- u32 btf_nr_types(const struct btf *btf);
-+struct btf *btf_base_btf(const struct btf *btf);
-+bool btf_type_is_i32(const struct btf_type *t);
-+bool btf_type_is_i64(const struct btf_type *t);
-+bool btf_type_is_primitive(const struct btf_type *t);
- bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s,
- 			   const struct btf_member *m,
- 			   u32 expected_offset, u32 expected_size);
-int btf_find_spin_lock(const struct btf *btf, const struct btf_type *t);
-int btf_find_timer(const struct btf *btf, const struct btf_type *t);
-struct bpf_map_value_off *btf_parse_kptrs(const struct btf *btf,
-					  const struct btf_type *t);
-+struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type *t,
-+				    u32 field_mask, u32 value_size);
-+int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec);
- bool btf_type_is_void(const struct btf_type *t);
- s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind);
-+s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p);
- const struct btf_type *btf_type_skip_modifiers(const struct btf *btf,
- 					       u32 id, u32 *res_id);
- const struct btf_type *btf_type_resolve_ptr(const struct btf *btf,
-@@ -192,6 +269,16 @@ static inline bool btf_type_is_small_int(const struct btf_type *t)
- 	return btf_type_is_int(t) && t->size <= sizeof(u64);
- }
- 
-+static inline u8 btf_int_encoding(const struct btf_type *t)
-+{
-+	return BTF_INT_ENCODING(*(u32 *)(t + 1));
-+}
-+
-+static inline bool btf_type_is_signed_int(const struct btf_type *t)
-+{
-+	return btf_type_is_int(t) && (btf_int_encoding(t) & BTF_INT_SIGNED);
-+}
-+
- static inline bool btf_type_is_enum(const struct btf_type *t)
- {
- 	return BTF_INFO_KIND(t->info) == BTF_KIND_ENUM;
-@@ -262,9 +349,9 @@ static inline u8 btf_int_offset(const struct btf_type *t)
- 	return BTF_INT_OFFSET(*(u32 *)(t + 1));
- }
- 
-static inline u8 btf_int_encoding(const struct btf_type *t)
-+static inline __u8 btf_int_bits(const struct btf_type *t)
- {
-	return BTF_INT_ENCODING(*(u32 *)(t + 1));
-+	return BTF_INT_BITS(*(__u32 *)(t + 1));
- }
- 
- static inline bool btf_type_is_scalar(const struct btf_type *t)
-@@ -272,11 +359,21 @@ static inline bool btf_type_is_scalar(const struct btf_type *t)
- 	return btf_type_is_int(t) || btf_type_is_enum(t);
- }
- 
-+static inline bool btf_type_is_fwd(const struct btf_type *t)
-+{
-+	return BTF_INFO_KIND(t->info) == BTF_KIND_FWD;
-+}
-+
- static inline bool btf_type_is_typedef(const struct btf_type *t)
- {
- 	return BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF;
- }
- 
-+static inline bool btf_type_is_volatile(const struct btf_type *t)
-+{
-+	return BTF_INFO_KIND(t->info) == BTF_KIND_VOLATILE;
-+}
-+
- static inline bool btf_type_is_func(const struct btf_type *t)
- {
- 	return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC;
-@@ -307,6 +404,16 @@ static inline bool btf_type_is_struct(const struct btf_type *t)
- 	return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION;
- }
- 
-+static inline bool __btf_type_is_struct(const struct btf_type *t)
-+{
-+	return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT;
-+}
-+
-+static inline bool btf_type_is_array(const struct btf_type *t)
-+{
-+	return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
-+}
-+
- static inline u16 btf_type_vlen(const struct btf_type *t)
- {
- 	return BTF_INFO_VLEN(t->info);
-@@ -391,35 +498,149 @@ static inline struct btf_param *btf_params(const struct btf_type *t)
- 	return (struct btf_param *)(t + 1);
- }
- 
-#ifdef CONFIG_BPF_SYSCALL
-struct bpf_prog;
-+static inline struct btf_decl_tag *btf_decl_tag(const struct btf_type *t)
-+{
-+	return (struct btf_decl_tag *)(t + 1);
-+}
-+
-+static inline int btf_id_cmp_func(const void *a, const void *b)
-+{
-+	const int *pa = a, *pb = b;
-+
-+	return *pa - *pb;
-+}
-+
-+static inline bool btf_id_set_contains(const struct btf_id_set *set, u32 id)
-+{
-+	return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL;
-+}
-+
-+static inline void *btf_id_set8_contains(const struct btf_id_set8 *set, u32 id)
-+{
-+	return bsearch(&id, set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func);
-+}
-+
-+bool btf_param_match_suffix(const struct btf *btf,
-+			    const struct btf_param *arg,
-+			    const char *suffix);
-+int btf_ctx_arg_offset(const struct btf *btf, const struct btf_type *func_proto,
-+		       u32 arg_no);
-+u32 btf_ctx_arg_idx(struct btf *btf, const struct btf_type *func_proto, int off);
-+
-+struct bpf_verifier_log;
-+
-+#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
-+struct bpf_struct_ops;
-+int __register_bpf_struct_ops(struct bpf_struct_ops *st_ops);
-+const struct bpf_struct_ops_desc *bpf_struct_ops_find_value(struct btf *btf, u32 value_id);
-+const struct bpf_struct_ops_desc *bpf_struct_ops_find(struct btf *btf, u32 type_id);
-+#else
-+static inline const struct bpf_struct_ops_desc *bpf_struct_ops_find(struct btf *btf, u32 type_id)
-+{
-+	return NULL;
-+}
-+#endif
- 
-+enum btf_field_iter_kind {
-+	BTF_FIELD_ITER_IDS,
-+	BTF_FIELD_ITER_STRS,
-+};
-+
-+struct btf_field_desc {
-+	/* once-per-type offsets */
-+	int t_off_cnt, t_offs[2];
-+	/* member struct size, or zero, if no members */
-+	int m_sz;
-+	/* repeated per-member offsets */
-+	int m_off_cnt, m_offs[1];
-+};
-+
-+struct btf_field_iter {
-+	struct btf_field_desc desc;
-+	void *p;
-+	int m_idx;
-+	int off_idx;
-+	int vlen;
-+};
-+
-+#ifdef CONFIG_BPF_SYSCALL
- const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id);
-+void btf_set_base_btf(struct btf *btf, const struct btf *base_btf);
-+int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids);
-+int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t,
-+			enum btf_field_iter_kind iter_kind);
-+__u32 *btf_field_iter_next(struct btf_field_iter *it);
-+
- const char *btf_name_by_offset(const struct btf *btf, u32 offset);
-+const char *btf_str_by_offset(const struct btf *btf, u32 offset);
- struct btf *btf_parse_vmlinux(void);
- struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog);
-u32 *btf_kfunc_id_set_contains(const struct btf *btf,
-			       enum bpf_prog_type prog_type,
-			       u32 kfunc_btf_id);
-+u32 *btf_kfunc_id_set_contains(const struct btf *btf, u32 kfunc_btf_id,
-+			       const struct bpf_prog *prog);
-+u32 *btf_kfunc_is_modify_return(const struct btf *btf, u32 kfunc_btf_id,
-+				const struct bpf_prog *prog);
- int register_btf_kfunc_id_set(enum bpf_prog_type prog_type,
- 			      const struct btf_kfunc_id_set *s);
-+int register_btf_fmodret_id_set(const struct btf_kfunc_id_set *kset);
- s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id);
- int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_cnt,
- 				struct module *owner);
-+struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id);
-+bool btf_is_projection_of(const char *pname, const char *tname);
-+bool btf_is_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf,
-+			   const struct btf_type *t, enum bpf_prog_type prog_type,
-+			   int arg);
-+int get_kern_ctx_btf_id(struct bpf_verifier_log *log, enum bpf_prog_type prog_type);
-+bool btf_types_are_same(const struct btf *btf1, u32 id1,
-+			const struct btf *btf2, u32 id2);
-+int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx);
-+
-+static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t)
-+{
-+	if (!btf_type_is_ptr(t))
-+		return false;
-+
-+	t = btf_type_skip_modifiers(btf, t->type, NULL);
-+
-+	return btf_type_is_struct(t);
-+}
- #else
- static inline const struct btf_type *btf_type_by_id(const struct btf *btf,
- 						    u32 type_id)
- {
- 	return NULL;
- }
-+
-+static inline void btf_set_base_btf(struct btf *btf, const struct btf *base_btf)
-+{
-+}
-+
-+static inline int btf_relocate(void *log, struct btf *btf, const struct btf *base_btf,
-+			       __u32 **map_ids)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t,
-+				      enum btf_field_iter_kind iter_kind)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline __u32 *btf_field_iter_next(struct btf_field_iter *it)
-+{
-+	return NULL;
-+}
-+
- static inline const char *btf_name_by_offset(const struct btf *btf,
- 					     u32 offset)
- {
- 	return NULL;
- }
- static inline u32 *btf_kfunc_id_set_contains(const struct btf *btf,
-					     enum bpf_prog_type prog_type,
-					     u32 kfunc_btf_id)
-+					     u32 kfunc_btf_id,
-+					     struct bpf_prog *prog)
-+
- {
- 	return NULL;
- }
-@@ -437,6 +658,29 @@ static inline int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dt
- {
- 	return 0;
- }
-+static inline struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id)
-+{
-+	return NULL;
-+}
-+static inline bool
-+btf_is_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf,
-+		     const struct btf_type *t, enum bpf_prog_type prog_type,
-+		     int arg)
-+{
-+	return false;
-+}
-+static inline int get_kern_ctx_btf_id(struct bpf_verifier_log *log,
-+				      enum bpf_prog_type prog_type) {
-+	return -EINVAL;
-+}
-+static inline bool btf_types_are_same(const struct btf *btf1, u32 id1,
-+				      const struct btf *btf2, u32 id2)
-+{
-+	return false;
-+}
-+static inline int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx)
-+{
-+	return -EOPNOTSUPP;
-+}
- #endif
-
- #endif
-diff --git a/include/linux/btf_ids.h b/include/linux/btf_ids.h
-index 2aea877d644f..139bdececdcf 100644
--- a/include/linux/btf_ids.h
-+++ b/include/linux/btf_ids.h
-@@ -3,11 +3,16 @@
- #ifndef _LINUX_BTF_IDS_H
- #define _LINUX_BTF_IDS_H
- 
-+#include <linux/types.h> /* for u32 */
-+
- struct btf_id_set {
- 	u32 cnt;
- 	u32 ids[];
- };
- 
-+/* This flag implies BTF_SET8 holds kfunc(s) */
-+#define BTF_SET8_KFUNCS		(1 << 0)
-+
- struct btf_id_set8 {
- 	u32 cnt;
- 	u32 flags;
-@@ -21,6 +26,7 @@ struct btf_id_set8 {
- 
- #include <linux/compiler.h> /* for __PASTE */
- #include <linux/compiler_attributes.h> /* for __maybe_unused */
-+#include <linux/stringify.h>
- 
- /*
-  * Following macros help to define lists of BTF IDs placed
-@@ -49,7 +55,7 @@ word							\
- 	____BTF_ID(symbol, word)
- 
- #define __ID(prefix) \
-	__PASTE(prefix, __COUNTER__)
-+	__PASTE(__PASTE(prefix, __COUNTER__), __LINE__)
- 
- /*
-  * The BTF_ID defines unique symbol for each ID pointing
-@@ -183,17 +189,18 @@ extern struct btf_id_set name;
-  * .word (1 << 3) | (1 << 1) | (1 << 2)
-  *
-  */
-#define __BTF_SET8_START(name, scope)			\
-+#define __BTF_SET8_START(name, scope, flags)		\
-+__BTF_ID_LIST(name, local)				\
- asm(							\
- ".pushsection " BTF_IDS_SECTION ",\"a\";       \n"	\
- "." #scope " __BTF_ID__set8__" #name ";        \n"	\
- "__BTF_ID__set8__" #name ":;                   \n"	\
-".zero 8                                       \n"	\
-+".zero 4                                       \n"	\
-+".long " __stringify(flags)                   "\n"	\
- ".popsection;                                  \n");
- 
- #define BTF_SET8_START(name)				\
-__BTF_ID_LIST(name, local)				\
-__BTF_SET8_START(name, local)
-+__BTF_SET8_START(name, local, 0)
- 
- #define BTF_SET8_END(name)				\
- asm(							\
-@@ -202,9 +209,15 @@ asm(							\
- ".popsection;                                 \n");	\
- extern struct btf_id_set8 name;
- 
-+#define BTF_KFUNCS_START(name)				\
-+__BTF_SET8_START(name, local, BTF_SET8_KFUNCS)
-+
-+#define BTF_KFUNCS_END(name)				\
-+BTF_SET8_END(name)
-+
- #else
- 
-#define BTF_ID_LIST(name) static u32 __maybe_unused name[5];
-+#define BTF_ID_LIST(name) static u32 __maybe_unused name[64];
- #define BTF_ID(prefix, name)
- #define BTF_ID_FLAGS(prefix, name, ...)
- #define BTF_ID_UNUSED
-@@ -216,6 +229,8 @@ extern struct btf_id_set8 name;
- #define BTF_SET_END(name)
- #define BTF_SET8_START(name) static struct btf_id_set8 __maybe_unused name = { 0 };
- #define BTF_SET8_END(name)
-+#define BTF_KFUNCS_START(name) static struct btf_id_set8 __maybe_unused name = { .flags = BTF_SET8_KFUNCS };
-+#define BTF_KFUNCS_END(name)
- 
- #endif /* CONFIG_DEBUG_INFO_BTF */
- 
-@@ -265,5 +280,9 @@ MAX_BTF_TRACING_TYPE,
- };
- 
- extern u32 btf_tracing_ids[];
-+extern u32 bpf_cgroup_btf_id[];
-+extern u32 bpf_local_storage_map_btf_id[];
-+extern u32 btf_bpf_map_id[];
-+extern u32 bpf_kmem_cache_btf_id[];
- 
- #endif
-diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h
-index def8b8d30ccc..b16b88bfbc3e 100644
--- a/include/linux/buffer_head.h
-+++ b/include/linux/buffer_head.h
-@@ -16,8 +16,6 @@
- #include <linux/wait.h>
- #include <linux/atomic.h>
- 
-#ifdef CONFIG_BLOCK
-
- enum bh_state_bits {
- 	BH_Uptodate,	/* Contains valid data */
- 	BH_Dirty,	/* Is dirty */
-@@ -36,6 +34,7 @@ enum bh_state_bits {
- 	BH_Meta,	/* Buffer contains metadata */
- 	BH_Prio,	/* Buffer should be submitted with REQ_PRIO */
- 	BH_Defer_Completion, /* Defer AIO completion to workqueue */
-+	BH_Migrate,     /* Buffer is being migrated (norefs) */
- 
- 	BH_PrivateStart,/* not a state bit, but the first bit available
- 			 * for private allocation by other entities
-@@ -55,13 +54,16 @@ typedef void (bh_end_io_t)(struct buffer_head *bh, int uptodate);
-  * filesystem and block layers.  Nowadays the basic I/O unit
-  * is the bio, and buffer_heads are used for extracting block
-  * mappings (via a get_block_t call), for tracking state within
- * a page (via a page_mapping) and for wrapping bio submission
-+ * a folio (via a folio_mapping) and for wrapping bio submission
-  * for backward compatibility reasons (e.g. submit_bh).
-  */
- struct buffer_head {
- 	unsigned long b_state;		/* buffer state bitmap (see above) */
- 	struct buffer_head *b_this_page;/* circular list of page's buffers */
-	struct page *b_page;		/* the page this bh is mapped to */
-+	union {
-+		struct page *b_page;	/* the page this bh is mapped to */
-+		struct folio *b_folio;	/* the folio this bh is mapped to */
-+	};
- 
- 	sector_t b_blocknr;		/* start block number */
- 	size_t b_size;			/* size of mapping */
-@@ -138,6 +140,17 @@ BUFFER_FNS(Defer_Completion, defer_completion)
- static __always_inline void set_buffer_uptodate(struct buffer_head *bh)
- {
- 	/*
-+	 * If somebody else already set this uptodate, they will
-+	 * have done the memory barrier, and a reader will thus
-+	 * see *some* valid buffer state.
-+	 *
-+	 * Any other serialization (with IO errors or whatever that
-+	 * might clear the bit) has to come from other state (eg BH_Lock).
-+	 */
-+	if (test_bit(BH_Uptodate, &bh->b_state))
-+		return;
-+
-+	/*
- 	 * make it consistent with folio_mark_uptodate
- 	 * pairs with smp_load_acquire in buffer_uptodate
- 	 */
-@@ -156,10 +169,13 @@ static __always_inline int buffer_uptodate(const struct buffer_head *bh)
- 	 * make it consistent with folio_test_uptodate
- 	 * pairs with smp_mb__before_atomic in set_buffer_uptodate
- 	 */
-	return (smp_load_acquire(&bh->b_state) & (1UL << BH_Uptodate)) != 0;
-+	return test_bit_acquire(BH_Uptodate, &bh->b_state);
- }
- 
-#define bh_offset(bh)		((unsigned long)(bh)->b_data & ~PAGE_MASK)
-+static inline unsigned long bh_offset(const struct buffer_head *bh)
-+{
-+	return (unsigned long)(bh)->b_data & (page_size(bh->b_page) - 1);
-+}
- 
- /* If we *know* page->private refers to buffer_heads */
- #define page_buffers(page)					\
-@@ -167,7 +183,6 @@ static __always_inline int buffer_uptodate(const struct buffer_head *bh)
- 		BUG_ON(!PagePrivate(page));			\
- 		((struct buffer_head *)page_private(page));	\
- 	})
-#define page_has_buffers(page)	PagePrivate(page)
- #define folio_buffers(folio)		folio_get_private(folio)
- 
- void buffer_check_dirty_writeback(struct folio *folio,
-@@ -180,23 +195,22 @@ void buffer_check_dirty_writeback(struct folio *folio,
- void mark_buffer_dirty(struct buffer_head *bh);
- void mark_buffer_write_io_error(struct buffer_head *bh);
- void touch_buffer(struct buffer_head *bh);
-void set_bh_page(struct buffer_head *bh,
-		struct page *page, unsigned long offset);
-bool try_to_free_buffers(struct folio *);
-struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
-		bool retry);
-void create_empty_buffers(struct page *, unsigned long,
-			unsigned long b_state);
-+void folio_set_bh(struct buffer_head *bh, struct folio *folio,
-+		  unsigned long offset);
-+struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size,
-+					gfp_t gfp);
-+struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size);
-+struct buffer_head *create_empty_buffers(struct folio *folio,
-+		unsigned long blocksize, unsigned long b_state);
- void end_buffer_read_sync(struct buffer_head *bh, int uptodate);
- void end_buffer_write_sync(struct buffer_head *bh, int uptodate);
-void end_buffer_async_write(struct buffer_head *bh, int uptodate);
- 
- /* Things to do with buffers at mapping->private_list */
- void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode);
-int inode_has_buffers(struct inode *);
-void invalidate_inode_buffers(struct inode *);
-int remove_inode_buffers(struct inode *inode);
-int sync_mapping_buffers(struct address_space *mapping);
-+int generic_buffers_fsync_noflush(struct file *file, loff_t start, loff_t end,
-+				  bool datasync);
-+int generic_buffers_fsync(struct file *file, loff_t start, loff_t end,
-+			  bool datasync);
- void clean_bdev_aliases(struct block_device *bdev, sector_t block,
- 			sector_t len);
- static inline void clean_bdev_bh_alias(struct buffer_head *bh)
-@@ -209,77 +223,57 @@ void __wait_on_buffer(struct buffer_head *);
- wait_queue_head_t *bh_waitq_head(struct buffer_head *bh);
- struct buffer_head *__find_get_block(struct block_device *bdev, sector_t block,
- 			unsigned size);
-struct buffer_head *__getblk_gfp(struct block_device *bdev, sector_t block,
-				  unsigned size, gfp_t gfp);
-+struct buffer_head *__find_get_block_nonatomic(struct block_device *bdev,
-+			sector_t block, unsigned size);
-+struct buffer_head *bdev_getblk(struct block_device *bdev, sector_t block,
-+		unsigned size, gfp_t gfp);
- void __brelse(struct buffer_head *);
- void __bforget(struct buffer_head *);
- void __breadahead(struct block_device *, sector_t block, unsigned int size);
-void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size,
-		  gfp_t gfp);
- struct buffer_head *__bread_gfp(struct block_device *,
- 				sector_t block, unsigned size, gfp_t gfp);
-void invalidate_bh_lrus(void);
-void invalidate_bh_lrus_cpu(void);
-bool has_bh_in_lru(int cpu, void *dummy);
- struct buffer_head *alloc_buffer_head(gfp_t gfp_flags);
- void free_buffer_head(struct buffer_head * bh);
- void unlock_buffer(struct buffer_head *bh);
- void __lock_buffer(struct buffer_head *bh);
-void ll_rw_block(blk_opf_t, int, struct buffer_head * bh[]);
- int sync_dirty_buffer(struct buffer_head *bh);
- int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags);
- void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags);
-int submit_bh(blk_opf_t, struct buffer_head *);
-+void submit_bh(blk_opf_t, struct buffer_head *);
- void write_boundary_block(struct block_device *bdev,
- 			sector_t bblock, unsigned blocksize);
- int bh_uptodate_or_lock(struct buffer_head *bh);
-int bh_submit_read(struct buffer_head *bh);
-
-extern int buffer_heads_over_limit;
-+int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait);
-+void __bh_read_batch(int nr, struct buffer_head *bhs[],
-+		     blk_opf_t op_flags, bool force_lock);
- 
- /*
-  * Generic address_space_operations implementations for buffer_head-backed
-  * address_spaces.
-  */
- void block_invalidate_folio(struct folio *folio, size_t offset, size_t length);
-int block_write_full_page(struct page *page, get_block_t *get_block,
-				struct writeback_control *wbc);
-int __block_write_full_page(struct inode *inode, struct page *page,
-			get_block_t *get_block, struct writeback_control *wbc,
-			bh_end_io_t *handler);
-+int block_write_full_folio(struct folio *folio, struct writeback_control *wbc,
-+		void *get_block);
-+int __block_write_full_folio(struct inode *inode, struct folio *folio,
-+		get_block_t *get_block, struct writeback_control *wbc);
- int block_read_full_folio(struct folio *, get_block_t *);
- bool block_is_partially_uptodate(struct folio *, size_t from, size_t count);
- int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
-		struct page **pagep, get_block_t *get_block);
-int __block_write_begin(struct page *page, loff_t pos, unsigned len,
-+		struct folio **foliop, get_block_t *get_block);
-+int __block_write_begin(struct folio *folio, loff_t pos, unsigned len,
- 		get_block_t *get_block);
-int block_write_end(struct file *, struct address_space *,
-				loff_t, unsigned, unsigned,
-				struct page *, void *);
-int generic_write_end(struct file *, struct address_space *,
-				loff_t, unsigned, unsigned,
-				struct page *, void *);
-void page_zero_new_buffers(struct page *page, unsigned from, unsigned to);
-void clean_page_buffers(struct page *page);
-int cont_write_begin(struct file *, struct address_space *, loff_t,
-			unsigned, struct page **, void **,
-+int block_write_end(loff_t pos, unsigned len, unsigned copied, struct folio *);
-+int generic_write_end(const struct kiocb *, struct address_space *,
-+				loff_t, unsigned len, unsigned copied,
-+				struct folio *, void *);
-+void folio_zero_new_buffers(struct folio *folio, size_t from, size_t to);
-+int cont_write_begin(const struct kiocb *, struct address_space *, loff_t,
-+			unsigned, struct folio **, void **,
- 			get_block_t *, loff_t *);
- int generic_cont_expand_simple(struct inode *inode, loff_t size);
-int block_commit_write(struct page *page, unsigned from, unsigned to);
-+void block_commit_write(struct folio *folio, size_t from, size_t to);
- int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
- 				get_block_t get_block);
-/* Convert errno to return value from ->page_mkwrite() call */
-static inline vm_fault_t block_page_mkwrite_return(int err)
-{
-	if (err == 0)
-		return VM_FAULT_LOCKED;
-	if (err == -EFAULT || err == -EAGAIN)
-		return VM_FAULT_NOPAGE;
-	if (err == -ENOMEM)
-		return VM_FAULT_OOM;
-	/* -ENOSPC, -EDQUOT, -EIO ... */
-	return VM_FAULT_SIGBUS;
-}
- sector_t generic_block_bmap(struct address_space *, sector_t, get_block_t *);
- int block_truncate_page(struct address_space *, loff_t, get_block_t *);
- 
-@@ -293,8 +287,6 @@ extern int buffer_migrate_folio_norefs(struct address_space *,
- #define buffer_migrate_folio_norefs NULL
- #endif
- 
-void buffer_init(void);
-
- /*
-  * inline definitions
-  */
-@@ -310,12 +302,38 @@ static inline void put_bh(struct buffer_head *bh)
-         atomic_dec(&bh->b_count);
- }
- 
-+/**
-+ * brelse - Release a buffer.
-+ * @bh: The buffer to release.
-+ *
-+ * Decrement a buffer_head's reference count.  If @bh is NULL, this
-+ * function is a no-op.
-+ *
-+ * If all buffers on a folio have zero reference count, are clean
-+ * and unlocked, and if the folio is unlocked and not under writeback
-+ * then try_to_free_buffers() may strip the buffers from the folio in
-+ * preparation for freeing it (sometimes, rarely, buffers are removed
-+ * from a folio but it ends up not being freed, and buffers may later
-+ * be reattached).
-+ *
-+ * Context: Any context.
-+ */
- static inline void brelse(struct buffer_head *bh)
- {
- 	if (bh)
- 		__brelse(bh);
- }
- 
-+/**
-+ * bforget - Discard any dirty data in a buffer.
-+ * @bh: The buffer to forget.
-+ *
-+ * Call this function instead of brelse() if the data written to a buffer
-+ * no longer needs to be written back.  It will clear the buffer's dirty
-+ * flag so writeback of this buffer will be skipped.
-+ *
-+ * Context: Any context.
-+ */
- static inline void bforget(struct buffer_head *bh)
- {
- 	if (bh)
-@@ -340,23 +358,38 @@ sb_breadahead(struct super_block *sb, sector_t block)
- 	__breadahead(sb->s_bdev, block, sb->s_blocksize);
- }
- 
-static inline void
-sb_breadahead_unmovable(struct super_block *sb, sector_t block)
-+static inline struct buffer_head *getblk_unmovable(struct block_device *bdev,
-+		sector_t block, unsigned size)
- {
-	__breadahead_gfp(sb->s_bdev, block, sb->s_blocksize, 0);
-+	gfp_t gfp;
-+
-+	gfp = mapping_gfp_constraint(bdev->bd_mapping, ~__GFP_FS);
-+	gfp |= __GFP_NOFAIL;
-+
-+	return bdev_getblk(bdev, block, size, gfp);
- }
- 
-static inline struct buffer_head *
-sb_getblk(struct super_block *sb, sector_t block)
-+static inline struct buffer_head *__getblk(struct block_device *bdev,
-+		sector_t block, unsigned size)
- {
-	return __getblk_gfp(sb->s_bdev, block, sb->s_blocksize, __GFP_MOVABLE);
-+	gfp_t gfp;
-+
-+	gfp = mapping_gfp_constraint(bdev->bd_mapping, ~__GFP_FS);
-+	gfp |= __GFP_MOVABLE | __GFP_NOFAIL;
-+
-+	return bdev_getblk(bdev, block, size, gfp);
- }
- 
-+static inline struct buffer_head *sb_getblk(struct super_block *sb,
-+		sector_t block)
-+{
-+	return __getblk(sb->s_bdev, block, sb->s_blocksize);
-+}
- 
-static inline struct buffer_head *
-sb_getblk_gfp(struct super_block *sb, sector_t block, gfp_t gfp)
-+static inline struct buffer_head *sb_getblk_gfp(struct super_block *sb,
-+		sector_t block, gfp_t gfp)
- {
-	return __getblk_gfp(sb->s_bdev, block, sb->s_blocksize, gfp);
-+	return bdev_getblk(sb->s_bdev, block, sb->s_blocksize, gfp);
- }
- 
- static inline struct buffer_head *
-@@ -365,6 +398,12 @@ sb_find_get_block(struct super_block *sb, sector_t block)
- 	return __find_get_block(sb->s_bdev, block, sb->s_blocksize);
- }
- 
-+static inline struct buffer_head *
-+sb_find_get_block_nonatomic(struct super_block *sb, sector_t block)
-+{
-+	return __find_get_block_nonatomic(sb->s_bdev, block, sb->s_blocksize);
-+}
-+
- static inline void
- map_bh(struct buffer_head *bh, struct super_block *sb, sector_t block)
- {
-@@ -393,39 +432,99 @@ static inline void lock_buffer(struct buffer_head *bh)
- 		__lock_buffer(bh);
- }
- 
-static inline struct buffer_head *getblk_unmovable(struct block_device *bdev,
-						   sector_t block,
-						   unsigned size)
-+static inline void bh_readahead(struct buffer_head *bh, blk_opf_t op_flags)
- {
-	return __getblk_gfp(bdev, block, size, 0);
-+	if (!buffer_uptodate(bh) && trylock_buffer(bh)) {
-+		if (!buffer_uptodate(bh))
-+			__bh_read(bh, op_flags, false);
-+		else
-+			unlock_buffer(bh);
-+	}
- }
- 
-static inline struct buffer_head *__getblk(struct block_device *bdev,
-					   sector_t block,
-					   unsigned size)
-+static inline void bh_read_nowait(struct buffer_head *bh, blk_opf_t op_flags)
-+{
-+	if (!bh_uptodate_or_lock(bh))
-+		__bh_read(bh, op_flags, false);
-+}
-+
-+/* Returns 1 if buffer uptodated, 0 on success, and -EIO on error. */
-+static inline int bh_read(struct buffer_head *bh, blk_opf_t op_flags)
-+{
-+	if (bh_uptodate_or_lock(bh))
-+		return 1;
-+	return __bh_read(bh, op_flags, true);
-+}
-+
-+static inline void bh_read_batch(int nr, struct buffer_head *bhs[])
- {
-	return __getblk_gfp(bdev, block, size, __GFP_MOVABLE);
-+	__bh_read_batch(nr, bhs, 0, true);
-+}
-+
-+static inline void bh_readahead_batch(int nr, struct buffer_head *bhs[],
-+				      blk_opf_t op_flags)
-+{
-+	__bh_read_batch(nr, bhs, op_flags, false);
- }
- 
- /**
- *  __bread() - reads a specified block and returns the bh
- *  @bdev: the block_device to read from
- *  @block: number of block
- *  @size: size (in bytes) to read
-+ * __bread() - Read a block.
-+ * @bdev: The block device to read from.
-+ * @block: Block number in units of block size.
-+ * @size: The block size of this device in bytes.
-+ *
-+ * Read a specified block, and return the buffer head that refers
-+ * to it.  The memory is allocated from the movable area so that it can
-+ * be migrated.  The returned buffer head has its refcount increased.
-+ * The caller should call brelse() when it has finished with the buffer.
-  *
- *  Reads a specified block, and returns buffer head that contains it.
- *  The page cache is allocated from movable area so that it can be migrated.
- *  It returns NULL if the block was unreadable.
-+ * Context: May sleep waiting for I/O.
-+ * Return: NULL if the block was unreadable.
-  */
-static inline struct buffer_head *
-__bread(struct block_device *bdev, sector_t block, unsigned size)
-+static inline struct buffer_head *__bread(struct block_device *bdev,
-+		sector_t block, unsigned size)
- {
- 	return __bread_gfp(bdev, block, size, __GFP_MOVABLE);
- }
- 
-+/**
-+ * get_nth_bh - Get a reference on the n'th buffer after this one.
-+ * @bh: The buffer to start counting from.
-+ * @count: How many buffers to skip.
-+ *
-+ * This is primarily useful for finding the nth buffer in a folio; in
-+ * that case you pass the head buffer and the byte offset in the folio
-+ * divided by the block size.  It can be used for other purposes, but
-+ * it will wrap at the end of the folio rather than returning NULL or
-+ * proceeding to the next folio for you.
-+ *
-+ * Return: The requested buffer with an elevated refcount.
-+ */
-+static inline __must_check
-+struct buffer_head *get_nth_bh(struct buffer_head *bh, unsigned int count)
-+{
-+	while (count--)
-+		bh = bh->b_this_page;
-+	get_bh(bh);
-+	return bh;
-+}
-+
- bool block_dirty_folio(struct address_space *mapping, struct folio *folio);
- 
-#else /* CONFIG_BLOCK */
-+#ifdef CONFIG_BUFFER_HEAD
-+
-+void buffer_init(void);
-+bool try_to_free_buffers(struct folio *folio);
-+int inode_has_buffers(struct inode *inode);
-+void invalidate_inode_buffers(struct inode *inode);
-+int remove_inode_buffers(struct inode *inode);
-+int sync_mapping_buffers(struct address_space *mapping);
-+void invalidate_bh_lrus(void);
-+void invalidate_bh_lrus_cpu(void);
-+bool has_bh_in_lru(int cpu, void *dummy);
-+extern int buffer_heads_over_limit;
-+
-+#else /* CONFIG_BUFFER_HEAD */
- 
- static inline void buffer_init(void) {}
- static inline bool try_to_free_buffers(struct folio *folio) { return true; }
-@@ -433,9 +532,10 @@ static inline int inode_has_buffers(struct inode *inode) { return 0; }
- static inline void invalidate_inode_buffers(struct inode *inode) {}
- static inline int remove_inode_buffers(struct inode *inode) { return 1; }
- static inline int sync_mapping_buffers(struct address_space *mapping) { return 0; }
-+static inline void invalidate_bh_lrus(void) {}
- static inline void invalidate_bh_lrus_cpu(void) {}
- static inline bool has_bh_in_lru(int cpu, void *dummy) { return false; }
- #define buffer_heads_over_limit 0
- 
-#endif /* CONFIG_BLOCK */
-+#endif /* CONFIG_BUFFER_HEAD */
- #endif /* _LINUX_BUFFER_HEAD_H */
-diff --git a/include/linux/bug.h b/include/linux/bug.h
-index 348acf2558f3..17a4933c611b 100644
--- a/include/linux/bug.h
-+++ b/include/linux/bug.h
-@@ -42,6 +42,7 @@ void bug_get_file_line(struct bug_entry *bug, const char **file,
- struct bug_entry *find_bug(unsigned long bugaddr);
- 
- enum bug_trap_type report_bug(unsigned long bug_addr, struct pt_regs *regs);
-+enum bug_trap_type report_bug_entry(struct bug_entry *bug, struct pt_regs *regs);
- 
- /* These are defined by the architecture */
- int is_valid_bugaddr(unsigned long addr);
-@@ -62,6 +63,13 @@ static inline enum bug_trap_type report_bug(unsigned long bug_addr,
- }
- 
- struct bug_entry;
-+
-+static inline enum bug_trap_type
-+report_bug_entry(struct bug_entry *bug, struct pt_regs *regs)
-+{
-+	return BUG_TRAP_TYPE_BUG;
-+}
-+
- static inline void bug_get_file_line(struct bug_entry *bug, const char **file,
- 				     unsigned int *line)
- {
-@@ -73,15 +81,23 @@ static inline void generic_bug_clear_once(void) {}
- 
- #endif	/* CONFIG_GENERIC_BUG */
- 
-+#ifdef CONFIG_PRINTK
-+void mem_dump_obj(void *object);
-+#else
-+static inline void mem_dump_obj(void *object) {}
-+#endif
-+
- /*
-  * Since detected data corruption should stop operation on the affected
-  * structures. Return value must be checked and sanely acted on by caller.
-  */
- static inline __must_check bool check_data_corruption(bool v) { return v; }
-#define CHECK_DATA_CORRUPTION(condition, fmt, ...)			 \
-+#define CHECK_DATA_CORRUPTION(condition, addr, fmt, ...)		 \
- 	check_data_corruption(({					 \
- 		bool corruption = unlikely(condition);			 \
- 		if (corruption) {					 \
-+			if (addr)					 \
-+				mem_dump_obj(addr);			 \
- 			if (IS_ENABLED(CONFIG_BUG_ON_DATA_CORRUPTION)) { \
- 				pr_err(fmt, ##__VA_ARGS__);		 \
- 				BUG();					 \
-diff --git a/include/linux/build_bug.h b/include/linux/build_bug.h
-index e3a0be2c90ad..2cfbb4c65c78 100644
--- a/include/linux/build_bug.h
-+++ b/include/linux/build_bug.h
-@@ -4,17 +4,17 @@
- 
- #include <linux/compiler.h>
- 
-#ifdef __CHECKER__
-#define BUILD_BUG_ON_ZERO(e) (0)
-#else /* __CHECKER__ */
- /*
-  * Force a compilation error if condition is true, but also produce a
-  * result (of value 0 and type int), so the expression can be used
-  * e.g. in a structure initializer (or where-ever else comma expressions
-  * aren't permitted).
-+ *
-+ * Take an error message as an optional second argument. If omitted,
-+ * default to the stringification of the tested expression.
-  */
-#define BUILD_BUG_ON_ZERO(e) ((int)(sizeof(struct { int:(-!!(e)); })))
-#endif /* __CHECKER__ */
-+#define BUILD_BUG_ON_ZERO(e, ...) \
-+	__BUILD_BUG_ON_ZERO_MSG(e, ##__VA_ARGS__, #e " is true")
- 
- /* Force a compilation error if a constant expression is not a power of 2 */
- #define __BUILD_BUG_ON_NOT_POWER_OF_2(n)	\
-@@ -77,4 +77,13 @@
- #define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr)
- #define __static_assert(expr, msg, ...) _Static_assert(expr, msg)
- 
-+
-+/*
-+ * Compile time check that field has an expected offset
-+ */
-+#define ASSERT_STRUCT_OFFSET(type, field, expected_offset)	\
-+	BUILD_BUG_ON_MSG(offsetof(type, field) != (expected_offset),	\
-+		"Offset of " #field " in " #type " has changed.")
-+
-+
- #endif	/* _LINUX_BUILD_BUG_H */
-diff --git a/include/linux/buildid.h b/include/linux/buildid.h
-index 3b7a0ff4642f..831c1b4b626c 100644
--- a/include/linux/buildid.h
-+++ b/include/linux/buildid.h
-@@ -2,19 +2,45 @@
- #ifndef _LINUX_BUILDID_H
- #define _LINUX_BUILDID_H
- 
-#include <linux/mm_types.h>
-+#include <linux/types.h>
- 
- #define BUILD_ID_SIZE_MAX 20
- 
-int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id,
-		   __u32 *size);
-+struct vm_area_struct;
-+int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, __u32 *size);
-+int build_id_parse_nofault(struct vm_area_struct *vma, unsigned char *build_id, __u32 *size);
- int build_id_parse_buf(const void *buf, unsigned char *build_id, u32 buf_size);
- 
-#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) || IS_ENABLED(CONFIG_CRASH_CORE)
-+#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) || IS_ENABLED(CONFIG_VMCORE_INFO)
- extern unsigned char vmlinux_build_id[BUILD_ID_SIZE_MAX];
- void init_vmlinux_build_id(void);
- #else
- static inline void init_vmlinux_build_id(void) { }
- #endif
- 
-+struct freader {
-+	void *buf;
-+	u32 buf_sz;
-+	int err;
-+	union {
-+		struct {
-+			struct file *file;
-+			struct folio *folio;
-+			void *addr;
-+			loff_t folio_off;
-+			bool may_fault;
-+		};
-+		struct {
-+			const char *data;
-+			u64 data_sz;
-+		};
-+	};
-+};
-+
-+void freader_init_from_file(struct freader *r, void *buf, u32 buf_sz,
-+			    struct file *file, bool may_fault);
-+void freader_init_from_mem(struct freader *r, const char *data, u64 data_sz);
-+const void *freader_fetch(struct freader *r, loff_t file_off, size_t sz);
-+void freader_cleanup(struct freader *r);
-+
- #endif
-diff --git a/include/linux/bus/stm32_firewall_device.h b/include/linux/bus/stm32_firewall_device.h
-new file mode 100644
-index 000000000000..eaa7a3f54450
--- /dev/null
-+++ b/include/linux/bus/stm32_firewall_device.h
-@@ -0,0 +1,145 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Copyright (C) 2023, STMicroelectronics - All Rights Reserved
-+ */
-+
-+#ifndef STM32_FIREWALL_DEVICE_H
-+#define STM32_FIREWALL_DEVICE_H
-+
-+#include <linux/of.h>
-+#include <linux/platform_device.h>
-+#include <linux/types.h>
-+
-+#define STM32_FIREWALL_MAX_EXTRA_ARGS		5
-+
-+/* Opaque reference to stm32_firewall_controller */
-+struct stm32_firewall_controller;
-+
-+/**
-+ * struct stm32_firewall - Information on a device's firewall. Each device can have more than one
-+ *			   firewall.
-+ *
-+ * @firewall_ctrl:		Pointer referencing a firewall controller of the device. It is
-+ *				opaque so a device cannot manipulate the controller's ops or access
-+ *				the controller's data
-+ * @extra_args:			Extra arguments that are implementation dependent
-+ * @entry:			Name of the firewall entry
-+ * @extra_args_size:		Number of extra arguments
-+ * @firewall_id:		Firewall ID associated the device for this firewall controller
-+ */
-+struct stm32_firewall {
-+	struct stm32_firewall_controller *firewall_ctrl;
-+	u32 extra_args[STM32_FIREWALL_MAX_EXTRA_ARGS];
-+	const char *entry;
-+	size_t extra_args_size;
-+	u32 firewall_id;
-+};
-+
-+#if IS_ENABLED(CONFIG_STM32_FIREWALL)
-+/**
-+ * stm32_firewall_get_firewall - Get the firewall(s) associated to given device.
-+ *				 The firewall controller reference is always the first argument
-+ *				 of each of the access-controller property entries.
-+ *				 The firewall ID is always the second argument of each of the
-+ *				 access-controller  property entries.
-+ *				 If there's no argument linked to the phandle, then the firewall ID
-+ *				 field is set to U32_MAX, which is an invalid ID.
-+ *
-+ * @np:				Device node to parse
-+ * @firewall:			Array of firewall references
-+ * @nb_firewall:		Number of firewall references to get. Must be at least 1.
-+ *
-+ * Returns 0 on success, -ENODEV if there's no match with a firewall controller or appropriate errno
-+ * code if error occurred.
-+ */
-+int stm32_firewall_get_firewall(struct device_node *np, struct stm32_firewall *firewall,
-+				unsigned int nb_firewall);
-+
-+/**
-+ * stm32_firewall_grant_access - Request firewall access rights and grant access.
-+ *
-+ * @firewall:			Firewall reference containing the ID to check against its firewall
-+ *				controller
-+ *
-+ * Returns 0 if access is granted, -EACCES if access is denied, -ENODEV if firewall is null or
-+ * appropriate errno code if error occurred
-+ */
-+int stm32_firewall_grant_access(struct stm32_firewall *firewall);
-+
-+/**
-+ * stm32_firewall_release_access - Release access granted from a call to
-+ *				   stm32_firewall_grant_access().
-+ *
-+ * @firewall:			Firewall reference containing the ID to check against its firewall
-+ *				controller
-+ */
-+void stm32_firewall_release_access(struct stm32_firewall *firewall);
-+
-+/**
-+ * stm32_firewall_grant_access_by_id - Request firewall access rights of a given device
-+ *				       based on a specific firewall ID
-+ *
-+ * Warnings:
-+ * There is no way to ensure that the given ID will correspond to the firewall referenced in the
-+ * device node if the ID did not come from stm32_firewall_get_firewall(). In that case, this
-+ * function must be used with caution.
-+ * This function should be used for subsystem resources that do not have the same firewall ID
-+ * as their parent.
-+ * U32_MAX is an invalid ID.
-+ *
-+ * @firewall:			Firewall reference containing the firewall controller
-+ * @subsystem_id:		Firewall ID of the subsystem resource
-+ *
-+ * Returns 0 if access is granted, -EACCES if access is denied, -ENODEV if firewall is null or
-+ * appropriate errno code if error occurred
-+ */
-+int stm32_firewall_grant_access_by_id(struct stm32_firewall *firewall, u32 subsystem_id);
-+
-+/**
-+ * stm32_firewall_release_access_by_id - Release access granted from a call to
-+ *					 stm32_firewall_grant_access_by_id().
-+ *
-+ * Warnings:
-+ * There is no way to ensure that the given ID will correspond to the firewall referenced in the
-+ * device node if the ID did not come from stm32_firewall_get_firewall(). In that case, this
-+ * function must be used with caution.
-+ * This function should be used for subsystem resources that do not have the same firewall ID
-+ * as their parent.
-+ * U32_MAX is an invalid ID.
-+ *
-+ * @firewall:			Firewall reference containing the firewall controller
-+ * @subsystem_id:		Firewall ID of the subsystem resource
-+ */
-+void stm32_firewall_release_access_by_id(struct stm32_firewall *firewall, u32 subsystem_id);
-+
-+#else /* CONFIG_STM32_FIREWALL */
-+
-+static inline int stm32_firewall_get_firewall(struct device_node *np,
-+					      struct stm32_firewall *firewall,
-+					      unsigned int nb_firewall)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int stm32_firewall_grant_access(struct stm32_firewall *firewall)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline void stm32_firewall_release_access(struct stm32_firewall *firewall)
-+{
-+}
-+
-+static inline int stm32_firewall_grant_access_by_id(struct stm32_firewall *firewall,
-+						    u32 subsystem_id)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline void stm32_firewall_release_access_by_id(struct stm32_firewall *firewall,
-+						       u32 subsystem_id)
-+{
-+}
-+
-+#endif /* CONFIG_STM32_FIREWALL */
-+#endif /* STM32_FIREWALL_DEVICE_H */
-diff --git a/include/linux/bvec.h b/include/linux/bvec.h
-index 35c25dff651a..3fc0efa0825b 100644
--- a/include/linux/bvec.h
-+++ b/include/linux/bvec.h
-@@ -12,7 +12,6 @@
- #include <linux/errno.h>
- #include <linux/limits.h>
- #include <linux/minmax.h>
-#include <linux/mm.h>
- #include <linux/types.h>
- 
- struct page;
-@@ -23,11 +22,8 @@ struct page;
-  * @bv_len:    Number of bytes in the address range.
-  * @bv_offset: Start of the address range relative to the start of @bv_page.
-  *
- * The following holds for a bvec if n * PAGE_SIZE < bv_offset + bv_len:
- *
- *   nth_page(@bv_page, n) == @bv_page + n
- *
- * This holds because page_is_mergeable() checks the above property.
-+ * All pages within a bio_vec starting from @bv_page are contiguous and
-+ * can simply be iterated (see bvec_advance()).
-  */
- struct bio_vec {
- 	struct page	*bv_page;
-@@ -35,6 +31,49 @@ struct bio_vec {
- 	unsigned int	bv_offset;
- };
- 
-+/**
-+ * bvec_set_page - initialize a bvec based off a struct page
-+ * @bv:		bvec to initialize
-+ * @page:	page the bvec should point to
-+ * @len:	length of the bvec
-+ * @offset:	offset into the page
-+ */
-+static inline void bvec_set_page(struct bio_vec *bv, struct page *page,
-+		unsigned int len, unsigned int offset)
-+{
-+	bv->bv_page = page;
-+	bv->bv_len = len;
-+	bv->bv_offset = offset;
-+}
-+
-+/**
-+ * bvec_set_folio - initialize a bvec based off a struct folio
-+ * @bv:		bvec to initialize
-+ * @folio:	folio the bvec should point to
-+ * @len:	length of the bvec
-+ * @offset:	offset into the folio
-+ */
-+static inline void bvec_set_folio(struct bio_vec *bv, struct folio *folio,
-+		size_t len, size_t offset)
-+{
-+	unsigned long nr = offset / PAGE_SIZE;
-+
-+	WARN_ON_ONCE(len > UINT_MAX);
-+	bvec_set_page(bv, folio_page(folio, nr), len, offset % PAGE_SIZE);
-+}
-+
-+/**
-+ * bvec_set_virt - initialize a bvec based on a virtual address
-+ * @bv:		bvec to initialize
-+ * @vaddr:	virtual address to set the bvec to
-+ * @len:	length of the bvec
-+ */
-+static inline void bvec_set_virt(struct bio_vec *bv, void *vaddr,
-+		unsigned int len)
-+{
-+	bvec_set_page(bv, virt_to_page(vaddr), len, offset_in_page(vaddr));
-+}
-+
- struct bvec_iter {
- 	sector_t		bi_sector;	/* device address in 512 byte
- 						   sectors */
-@@ -44,7 +83,7 @@ struct bvec_iter {
- 
- 	unsigned int            bi_bvec_done;	/* number of bytes completed in
- 						   current bvec */
-} __packed;
-+} __packed __aligned(4);
- 
- struct bvec_iter_all {
- 	struct bio_vec	bv;
-@@ -145,6 +184,12 @@ static inline void bvec_iter_advance_single(const struct bio_vec *bv,
- 		((bvl = bvec_iter_bvec((bio_vec), (iter))), 1);	\
- 	     bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len))
- 
-+#define for_each_mp_bvec(bvl, bio_vec, iter, start)			\
-+	for (iter = (start);						\
-+	     (iter).bi_size &&						\
-+		((bvl = mp_bvec_iter_bvec((bio_vec), (iter))), 1);	\
-+	     bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len))
-+
- /* for iterating one bio from start to end */
- #define BVEC_ITER_ALL_INIT (struct bvec_iter)				\
- {									\
-@@ -241,4 +286,13 @@ static inline void *bvec_virt(struct bio_vec *bvec)
- 	return page_address(bvec->bv_page) + bvec->bv_offset;
- }
- 
-+/**
-+ * bvec_phys - return the physical address for a bvec
-+ * @bvec: bvec to return the physical address for
-+ */
-+static inline phys_addr_t bvec_phys(const struct bio_vec *bvec)
-+{
-+	return page_to_phys(bvec->bv_page) + bvec->bv_offset;
-+}
-+
- #endif /* __LINUX_BVEC_H */
-diff --git a/include/linux/byteorder/generic.h b/include/linux/byteorder/generic.h
-index c9a4c96c9943..55a44199de87 100644
--- a/include/linux/byteorder/generic.h
-+++ b/include/linux/byteorder/generic.h
-@@ -173,6 +173,38 @@ static inline void cpu_to_le32_array(u32 *buf, unsigned int words)
- 	}
- }
- 
-+static inline void le64_to_cpu_array(u64 *buf, unsigned int words)
-+{
-+	while (words--) {
-+		__le64_to_cpus(buf);
-+		buf++;
-+	}
-+}
-+
-+static inline void cpu_to_le64_array(u64 *buf, unsigned int words)
-+{
-+	while (words--) {
-+		__cpu_to_le64s(buf);
-+		buf++;
-+	}
-+}
-+
-+static inline void memcpy_from_le32(u32 *dst, const __le32 *src, size_t words)
-+{
-+	size_t i;
-+
-+	for (i = 0; i < words; i++)
-+		dst[i] = le32_to_cpu(src[i]);
-+}
-+
-+static inline void memcpy_to_le32(__le32 *dst, const u32 *src, size_t words)
-+{
-+	size_t i;
-+
-+	for (i = 0; i < words; i++)
-+		dst[i] = cpu_to_le32(src[i]);
-+}
-+
- static inline void be16_add_cpu(__be16 *var, u16 val)
- {
- 	*var = cpu_to_be16(be16_to_cpu(*var) + val);
-diff --git a/include/linux/cache.h b/include/linux/cache.h
-index d742c57eaee5..e69768f50d53 100644
--- a/include/linux/cache.h
-+++ b/include/linux/cache.h
-@@ -3,14 +3,37 @@
- #define __LINUX_CACHE_H
- 
- #include <uapi/linux/kernel.h>
-+#include <vdso/cache.h>
- #include <asm/cache.h>
- 
- #ifndef L1_CACHE_ALIGN
- #define L1_CACHE_ALIGN(x) __ALIGN_KERNEL(x, L1_CACHE_BYTES)
- #endif
- 
-#ifndef SMP_CACHE_BYTES
-#define SMP_CACHE_BYTES L1_CACHE_BYTES
-+/**
-+ * SMP_CACHE_ALIGN - align a value to the L2 cacheline size
-+ * @x: value to align
-+ *
-+ * On some architectures, L2 ("SMP") CL size is bigger than L1, and sometimes,
-+ * this needs to be accounted.
-+ *
-+ * Return: aligned value.
-+ */
-+#ifndef SMP_CACHE_ALIGN
-+#define SMP_CACHE_ALIGN(x)	ALIGN(x, SMP_CACHE_BYTES)
-+#endif
-+
-+/*
-+ * ``__aligned_largest`` aligns a field to the value most optimal for the
-+ * target architecture to perform memory operations. Get the actual value
-+ * to be able to use it anywhere else.
-+ */
-+#ifndef __LARGEST_ALIGN
-+#define __LARGEST_ALIGN		sizeof(struct { long x; } __aligned_largest)
-+#endif
-+
-+#ifndef LARGEST_ALIGN
-+#define LARGEST_ALIGN(x)	ALIGN(x, __LARGEST_ALIGN)
- #endif
- 
- /*
-@@ -37,10 +60,6 @@
- #define __ro_after_init __section(".data..ro_after_init")
- #endif
- 
-#ifndef ____cacheline_aligned
-#define ____cacheline_aligned __attribute__((__aligned__(SMP_CACHE_BYTES)))
-#endif
-
- #ifndef ____cacheline_aligned_in_smp
- #ifdef CONFIG_SMP
- #define ____cacheline_aligned_in_smp ____cacheline_aligned
-@@ -85,4 +104,81 @@
- #define cache_line_size()	L1_CACHE_BYTES
- #endif
- 
-+#ifndef __cacheline_group_begin
-+#define __cacheline_group_begin(GROUP) \
-+	__u8 __cacheline_group_begin__##GROUP[0]
-+#endif
-+
-+#ifndef __cacheline_group_end
-+#define __cacheline_group_end(GROUP) \
-+	__u8 __cacheline_group_end__##GROUP[0]
-+#endif
-+
-+/**
-+ * __cacheline_group_begin_aligned - declare an aligned group start
-+ * @GROUP: name of the group
-+ * @...: optional group alignment
-+ *
-+ * The following block inside a struct:
-+ *
-+ *	__cacheline_group_begin_aligned(grp);
-+ *	field a;
-+ *	field b;
-+ *	__cacheline_group_end_aligned(grp);
-+ *
-+ * will always be aligned to either the specified alignment or
-+ * ``SMP_CACHE_BYTES``.
-+ */
-+#define __cacheline_group_begin_aligned(GROUP, ...)		\
-+	__cacheline_group_begin(GROUP)				\
-+	__aligned((__VA_ARGS__ + 0) ? : SMP_CACHE_BYTES)
-+
-+/**
-+ * __cacheline_group_end_aligned - declare an aligned group end
-+ * @GROUP: name of the group
-+ * @...: optional alignment (same as was in __cacheline_group_begin_aligned())
-+ *
-+ * Note that the end marker is aligned to sizeof(long) to allow more precise
-+ * size assertion. It also declares a padding at the end to avoid next field
-+ * falling into this cacheline.
-+ */
-+#define __cacheline_group_end_aligned(GROUP, ...)		\
-+	__cacheline_group_end(GROUP) __aligned(sizeof(long));	\
-+	struct { } __cacheline_group_pad__##GROUP		\
-+	__aligned((__VA_ARGS__ + 0) ? : SMP_CACHE_BYTES)
-+
-+#ifndef CACHELINE_ASSERT_GROUP_MEMBER
-+#define CACHELINE_ASSERT_GROUP_MEMBER(TYPE, GROUP, MEMBER) \
-+	BUILD_BUG_ON(!(offsetof(TYPE, MEMBER) >= \
-+		       offsetofend(TYPE, __cacheline_group_begin__##GROUP) && \
-+		       offsetofend(TYPE, MEMBER) <= \
-+		       offsetof(TYPE, __cacheline_group_end__##GROUP)))
-+#endif
-+
-+#ifndef CACHELINE_ASSERT_GROUP_SIZE
-+#define CACHELINE_ASSERT_GROUP_SIZE(TYPE, GROUP, SIZE) \
-+	BUILD_BUG_ON(offsetof(TYPE, __cacheline_group_end__##GROUP) - \
-+		     offsetofend(TYPE, __cacheline_group_begin__##GROUP) > \
-+		     SIZE)
-+#endif
-+
-+/*
-+ * Helper to add padding within a struct to ensure data fall into separate
-+ * cachelines.
-+ */
-+#if defined(CONFIG_SMP)
-+struct cacheline_padding {
-+	char x[0];
-+} ____cacheline_internodealigned_in_smp;
-+#define CACHELINE_PADDING(name)		struct cacheline_padding name
-+#else
-+#define CACHELINE_PADDING(name)
-+#endif
-+
-+#ifdef ARCH_DMA_MINALIGN
-+#define ARCH_HAS_DMA_MINALIGN
-+#else
-+#define ARCH_DMA_MINALIGN __alignof__(unsigned long long)
-+#endif
-+
- #endif /* __LINUX_CACHE_H */
-diff --git a/include/linux/cache_coherency.h b/include/linux/cache_coherency.h
-new file mode 100644
-index 000000000000..cc81c5733e31
--- /dev/null
-+++ b/include/linux/cache_coherency.h
-@@ -0,0 +1,61 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * Cache coherency maintenance operation device drivers
-+ *
-+ * Copyright Huawei 2025
-+ */
-+#ifndef _LINUX_CACHE_COHERENCY_H_
-+#define _LINUX_CACHE_COHERENCY_H_
-+
-+#include <linux/list.h>
-+#include <linux/kref.h>
-+#include <linux/types.h>
-+
-+struct cc_inval_params {
-+	phys_addr_t addr;
-+	size_t size;
-+};
-+
-+struct cache_coherency_ops_inst;
-+
-+struct cache_coherency_ops {
-+	int (*wbinv)(struct cache_coherency_ops_inst *cci,
-+		     struct cc_inval_params *invp);
-+	int (*done)(struct cache_coherency_ops_inst *cci);
-+};
-+
-+struct cache_coherency_ops_inst {
-+	struct kref kref;
-+	struct list_head node;
-+	const struct cache_coherency_ops *ops;
-+};
-+
-+int cache_coherency_ops_instance_register(struct cache_coherency_ops_inst *cci);
-+void cache_coherency_ops_instance_unregister(struct cache_coherency_ops_inst *cci);
-+
-+struct cache_coherency_ops_inst *
-+_cache_coherency_ops_instance_alloc(const struct cache_coherency_ops *ops,
-+				    size_t size);
-+/**
-+ * cache_coherency_ops_instance_alloc - Allocate cache coherency ops instance
-+ * @ops: Cache maintenance operations
-+ * @drv_struct: structure that contains the struct cache_coherency_ops_inst
-+ * @member: Name of the struct cache_coherency_ops_inst member in @drv_struct.
-+ *
-+ * This allocates a driver specific structure and initializes the
-+ * cache_coherency_ops_inst embedded in the drv_struct. Upon success the
-+ * pointer must be freed via cache_coherency_ops_instance_put().
-+ *
-+ * Returns a &drv_struct * on success, %NULL on error.
-+ */
-+#define cache_coherency_ops_instance_alloc(ops, drv_struct, member)	    \
-+	({								    \
-+		static_assert(__same_type(struct cache_coherency_ops_inst,  \
-+					  ((drv_struct *)NULL)->member));   \
-+		static_assert(offsetof(drv_struct, member) == 0);	    \
-+		(drv_struct *)_cache_coherency_ops_instance_alloc(ops,	    \
-+			sizeof(drv_struct));				    \
-+	})
-+void cache_coherency_ops_instance_put(struct cache_coherency_ops_inst *cci);
-+
-+#endif
-diff --git a/include/linux/cacheflush.h b/include/linux/cacheflush.h
-index a6189d21f2ba..55f297b2c23f 100644
--- a/include/linux/cacheflush.h
-+++ b/include/linux/cacheflush.h
-@@ -7,14 +7,23 @@
- struct folio;
- 
- #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
-#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_FOLIO
-+#ifndef flush_dcache_folio
- void flush_dcache_folio(struct folio *folio);
- #endif
- #else
- static inline void flush_dcache_folio(struct folio *folio)
- {
- }
-#define ARCH_IMPLEMENTS_FLUSH_DCACHE_FOLIO 0
-+#define flush_dcache_folio flush_dcache_folio
- #endif /* ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE */
- 
-+#ifndef flush_icache_pages
-+static inline void flush_icache_pages(struct vm_area_struct *vma,
-+				     struct page *page, unsigned int nr)
-+{
-+}
-+#endif
-+
-+#define flush_icache_page(vma, page)	flush_icache_pages(vma, page, 1)
-+
- #endif /* _LINUX_CACHEFLUSH_H */
-diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h
-index 00b7a6ae8617..c8f4f0a0b874 100644
--- a/include/linux/cacheinfo.h
-+++ b/include/linux/cacheinfo.h
-@@ -3,7 +3,8 @@
- #define _LINUX_CACHEINFO_H
- 
- #include <linux/bitops.h>
-#include <linux/cpumask.h>
-+#include <linux/cpuhplock.h>
-+#include <linux/cpumask_types.h>
- #include <linux/smp.h>
- 
- struct device_node;
-@@ -73,55 +74,95 @@ struct cacheinfo {
- 
- struct cpu_cacheinfo {
- 	struct cacheinfo *info_list;
-+	unsigned int per_cpu_data_slice_size;
- 	unsigned int num_levels;
- 	unsigned int num_leaves;
- 	bool cpu_map_populated;
-+	bool early_ci_levels;
- };
- 
- struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu);
-+int early_cache_level(unsigned int cpu);
- int init_cache_level(unsigned int cpu);
-+int init_of_cache_level(unsigned int cpu);
- int populate_cache_leaves(unsigned int cpu);
- int cache_setup_acpi(unsigned int cpu);
- bool last_level_cache_is_valid(unsigned int cpu);
- bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y);
-+int fetch_cache_info(unsigned int cpu);
- int detect_cache_attributes(unsigned int cpu);
- #ifndef CONFIG_ACPI_PPTT
- /*
- * acpi_find_last_cache_level is only called on ACPI enabled
-+ * acpi_get_cache_info() is only called on ACPI enabled
-  * platforms using the PPTT for topology. This means that if
-  * the platform supports other firmware configuration methods
-  * we need to stub out the call when ACPI is disabled.
-  * ACPI enabled platforms not using PPTT won't be making calls
-  * to this function so we need not worry about them.
-  */
-static inline int acpi_find_last_cache_level(unsigned int cpu)
-+static inline
-+int acpi_get_cache_info(unsigned int cpu,
-+			unsigned int *levels, unsigned int *split_levels)
- {
-	return 0;
-+	return -ENOENT;
- }
- #else
-int acpi_find_last_cache_level(unsigned int cpu);
-+int acpi_get_cache_info(unsigned int cpu,
-+			unsigned int *levels, unsigned int *split_levels);
- #endif
- 
- const struct attribute_group *cache_get_priv_group(struct cacheinfo *this_leaf);
- 
- /*
- * Get the id of the cache associated with @cpu at level @level.
-+ * Get the cacheinfo structure for the cache associated with @cpu at
-+ * level @level.
-  * cpuhp lock must be held.
-  */
-static inline int get_cpu_cacheinfo_id(int cpu, int level)
-+static inline struct cacheinfo *get_cpu_cacheinfo_level(int cpu, int level)
- {
- 	struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu);
- 	int i;
- 
-+	lockdep_assert_cpus_held();
-+
- 	for (i = 0; i < ci->num_leaves; i++) {
- 		if (ci->info_list[i].level == level) {
- 			if (ci->info_list[i].attributes & CACHE_ID)
-				return ci->info_list[i].id;
-			return -1;
-+				return &ci->info_list[i];
-+			return NULL;
- 		}
- 	}
- 
-	return -1;
-+	return NULL;
- }
- 
-+/*
-+ * Get the id of the cache associated with @cpu at level @level.
-+ * cpuhp lock must be held.
-+ */
-+static inline int get_cpu_cacheinfo_id(int cpu, int level)
-+{
-+	struct cacheinfo *ci = get_cpu_cacheinfo_level(cpu, level);
-+
-+	return ci ? ci->id : -1;
-+}
-+
-+#if defined(CONFIG_ARM64) || defined(CONFIG_ARM)
-+#define use_arch_cache_info()	(true)
-+#else
-+#define use_arch_cache_info()	(false)
-+#endif
-+
-+#ifndef CONFIG_ARCH_HAS_CPU_CACHE_ALIASING
-+#define cpu_dcache_is_aliasing()	false
-+#define cpu_icache_is_aliasing()	cpu_dcache_is_aliasing()
-+#else
-+#include <asm/cachetype.h>
-+
-+#ifndef cpu_icache_is_aliasing
-+#define cpu_icache_is_aliasing()	cpu_dcache_is_aliasing()
-+#endif
-+
-+#endif
-+
- #endif /* _LINUX_CACHEINFO_H */
-diff --git a/include/linux/call_once.h b/include/linux/call_once.h
-new file mode 100644
-index 000000000000..13cd6469e7e5
--- /dev/null
-+++ b/include/linux/call_once.h
-@@ -0,0 +1,66 @@
-+#ifndef _LINUX_CALL_ONCE_H
-+#define _LINUX_CALL_ONCE_H
-+
-+#include <linux/types.h>
-+#include <linux/mutex.h>
-+
-+#define ONCE_NOT_STARTED 0
-+#define ONCE_RUNNING     1
-+#define ONCE_COMPLETED   2
-+
-+struct once {
-+        atomic_t state;
-+        struct mutex lock;
-+};
-+
-+static inline void __once_init(struct once *once, const char *name,
-+			       struct lock_class_key *key)
-+{
-+        atomic_set(&once->state, ONCE_NOT_STARTED);
-+        __mutex_init(&once->lock, name, key);
-+}
-+
-+#define once_init(once)							\
-+do {									\
-+	static struct lock_class_key __key;				\
-+	__once_init((once), #once, &__key);				\
-+} while (0)
-+
-+/*
-+ * call_once - Ensure a function has been called exactly once
-+ *
-+ * @once: Tracking struct
-+ * @cb: Function to be called
-+ *
-+ * If @once has never completed successfully before, call @cb and, if
-+ * it returns a zero or positive value, mark @once as completed.  Return
-+ * the value returned by @cb
-+ *
-+ * If @once has completed succesfully before, return 0.
-+ *
-+ * The call to @cb is implicitly surrounded by a mutex, though for
-+ * efficiency the * function avoids taking it after the first call.
-+ */
-+static inline int call_once(struct once *once, int (*cb)(struct once *))
-+{
-+	int r, state;
-+
-+	/* Pairs with atomic_set_release() below.  */
-+	if (atomic_read_acquire(&once->state) == ONCE_COMPLETED)
-+		return 0;
-+
-+	guard(mutex)(&once->lock);
-+	state = atomic_read(&once->state);
-+	if (unlikely(state != ONCE_NOT_STARTED))
-+		return WARN_ON_ONCE(state != ONCE_COMPLETED) ? -EINVAL : 0;
-+
-+	atomic_set(&once->state, ONCE_RUNNING);
-+	r = cb(once);
-+	if (r < 0)
-+		atomic_set(&once->state, ONCE_NOT_STARTED);
-+	else
-+		atomic_set_release(&once->state, ONCE_COMPLETED);
-+	return r;
-+}
-+
-+#endif /* _LINUX_CALL_ONCE_H */
-diff --git a/include/linux/can/bittiming.h b/include/linux/can/bittiming.h
-index ef0a77173e3c..726d909e87ce 100644
--- a/include/linux/can/bittiming.h
-+++ b/include/linux/can/bittiming.h
-@@ -1,6 +1,6 @@
- /* SPDX-License-Identifier: GPL-2.0-only */
- /* Copyright (c) 2020 Pengutronix, Marc Kleine-Budde <kernel@pengutronix.de>
- * Copyright (c) 2021 Vincent Mailhol <mailhol.vincent@wanadoo.fr>
-+ * Copyright (c) 2021-2025 Vincent Mailhol <mailhol@kernel.org>
-  */
- 
- #ifndef _CAN_BITTIMING_H
-@@ -14,8 +14,14 @@
- #define CAN_BITRATE_UNSET 0
- #define CAN_BITRATE_UNKNOWN (-1U)
- 
-#define CAN_CTRLMODE_TDC_MASK					\
-+#define CAN_CTRLMODE_FD_TDC_MASK				\
- 	(CAN_CTRLMODE_TDC_AUTO | CAN_CTRLMODE_TDC_MANUAL)
-+#define CAN_CTRLMODE_XL_TDC_MASK				\
-+	(CAN_CTRLMODE_XL_TDC_AUTO | CAN_CTRLMODE_XL_TDC_MANUAL)
-+#define CAN_CTRLMODE_TDC_AUTO_MASK				\
-+	(CAN_CTRLMODE_TDC_AUTO | CAN_CTRLMODE_XL_TDC_AUTO)
-+#define CAN_CTRLMODE_TDC_MANUAL_MASK				\
-+	(CAN_CTRLMODE_TDC_MANUAL | CAN_CTRLMODE_XL_TDC_MANUAL)
- 
- /*
-  * struct can_tdc - CAN FD Transmission Delay Compensation parameters
-@@ -81,6 +87,11 @@ struct can_tdc {
- 	u32 tdcf;
- };
- 
-+/* The transceiver decoding margin corresponds to t_Decode in ISO 11898-2 */
-+#define CAN_PWM_DECODE_NS 5
-+/* Maximum PWM symbol duration. Corresponds to t_SymbolNom_MAX - t_Decode */
-+#define CAN_PWM_NS_MAX (205 - CAN_PWM_DECODE_NS)
-+
- /*
-  * struct can_tdc_const - CAN hardware-dependent constant for
-  *	Transmission Delay Compensation
-@@ -114,34 +125,131 @@ struct can_tdc_const {
- 	u32 tdcf_max;
- };
- 
-+/*
-+ * struct can_pwm - CAN Pulse-Width Modulation (PWM) parameters
-+ *
-+ * @pwms: pulse width modulation short phase
-+ * @pwml: pulse width modulation long phase
-+ * @pwmo: pulse width modulation offset
-+ */
-+struct can_pwm {
-+	u32 pwms;
-+	u32 pwml;
-+	u32 pwmo;
-+};
-+
-+/*
-+ * struct can_pwm - CAN hardware-dependent constants for Pulse-Width
-+ *	Modulation (PWM)
-+ *
-+ * @pwms_min: PWM short phase minimum value. Must be at least 1.
-+ * @pwms_max: PWM short phase maximum value
-+ * @pwml_min: PWM long phase minimum value. Must be at least 1.
-+ * @pwml_max: PWM long phase maximum value
-+ * @pwmo_min: PWM offset phase minimum value
-+ * @pwmo_max: PWM offset phase maximum value
-+ */
-+struct can_pwm_const {
-+	u32 pwms_min;
-+	u32 pwms_max;
-+	u32 pwml_min;
-+	u32 pwml_max;
-+	u32 pwmo_min;
-+	u32 pwmo_max;
-+};
-+
-+struct data_bittiming_params {
-+	const struct can_bittiming_const *data_bittiming_const;
-+	struct can_bittiming data_bittiming;
-+	const struct can_tdc_const *tdc_const;
-+	const struct can_pwm_const *pwm_const;
-+	union {
-+		struct can_tdc tdc;
-+		struct can_pwm pwm;
-+	};
-+	const u32 *data_bitrate_const;
-+	unsigned int data_bitrate_const_cnt;
-+	int (*do_set_data_bittiming)(struct net_device *dev);
-+	int (*do_get_auto_tdcv)(const struct net_device *dev, u32 *tdcv);
-+};
-+
- #ifdef CONFIG_CAN_CALC_BITTIMING
- int can_calc_bittiming(const struct net_device *dev, struct can_bittiming *bt,
-		       const struct can_bittiming_const *btc);
-+		       const struct can_bittiming_const *btc, struct netlink_ext_ack *extack);
- 
- void can_calc_tdco(struct can_tdc *tdc, const struct can_tdc_const *tdc_const,
- 		   const struct can_bittiming *dbt,
-		   u32 *ctrlmode, u32 ctrlmode_supported);
-+		   u32 tdc_mask, u32 *ctrlmode, u32 ctrlmode_supported);
-+
-+int can_calc_pwm(struct net_device *dev, struct netlink_ext_ack *extack);
- #else /* !CONFIG_CAN_CALC_BITTIMING */
- static inline int
- can_calc_bittiming(const struct net_device *dev, struct can_bittiming *bt,
-		   const struct can_bittiming_const *btc)
-+		   const struct can_bittiming_const *btc, struct netlink_ext_ack *extack)
- {
-	netdev_err(dev, "bit-timing calculation not available\n");
-+	NL_SET_ERR_MSG(extack, "bit-timing calculation not available\n");
- 	return -EINVAL;
- }
- 
- static inline void
- can_calc_tdco(struct can_tdc *tdc, const struct can_tdc_const *tdc_const,
- 	      const struct can_bittiming *dbt,
-	      u32 *ctrlmode, u32 ctrlmode_supported)
-+	      u32 tdc_mask, u32 *ctrlmode, u32 ctrlmode_supported)
-+{
-+}
-+
-+static inline int
-+can_calc_pwm(struct net_device *dev, struct netlink_ext_ack *extack)
- {
-+	NL_SET_ERR_MSG(extack,
-+		       "bit-timing calculation not available: manually provide PWML and PWMS\n");
-+	return -EINVAL;
- }
- #endif /* CONFIG_CAN_CALC_BITTIMING */
- 
-+void can_sjw_set_default(struct can_bittiming *bt);
-+
-+int can_sjw_check(const struct net_device *dev, const struct can_bittiming *bt,
-+		  const struct can_bittiming_const *btc, struct netlink_ext_ack *extack);
-+
- int can_get_bittiming(const struct net_device *dev, struct can_bittiming *bt,
- 		      const struct can_bittiming_const *btc,
- 		      const u32 *bitrate_const,
-		      const unsigned int bitrate_const_cnt);
-+		      const unsigned int bitrate_const_cnt,
-+		      struct netlink_ext_ack *extack);
-+
-+int can_validate_pwm_bittiming(const struct net_device *dev,
-+			       const struct can_pwm *pwm,
-+			       struct netlink_ext_ack *extack);
-+
-+/*
-+ * can_get_relative_tdco() - TDCO relative to the sample point
-+ *
-+ * struct can_tdc::tdco represents the absolute offset from TDCV. Some
-+ * controllers use instead an offset relative to the Sample Point (SP)
-+ * such that:
-+ *
-+ * SSP = TDCV + absolute TDCO
-+ *     = TDCV + SP + relative TDCO
-+ *
-+ * -+----------- one bit ----------+-- TX pin
-+ *  |<--- Sample Point --->|
-+ *
-+ *                         --+----------- one bit ----------+-- RX pin
-+ *  |<-------- TDCV -------->|
-+ *                           |<------------------------>| absolute TDCO
-+ *                           |<--- Sample Point --->|
-+ *                           |                      |<->| relative TDCO
-+ *  |<------------- Secondary Sample Point ------------>|
-+ */
-+static inline s32 can_get_relative_tdco(const struct data_bittiming_params *dbt_params)
-+{
-+	const struct can_bittiming *dbt = &dbt_params->data_bittiming;
-+	s32 sample_point_in_tc = (CAN_SYNC_SEG + dbt->prop_seg +
-+				  dbt->phase_seg1) * dbt->brp;
-+
-+	return (s32)dbt_params->tdc.tdco - sample_point_in_tc;
-+}
- 
- /*
-  * can_bit_time() - Duration of one bit
-@@ -156,4 +264,17 @@ static inline unsigned int can_bit_time(const struct can_bittiming *bt)
- 	return CAN_SYNC_SEG + bt->prop_seg + bt->phase_seg1 + bt->phase_seg2;
- }
- 
-+/* Duration of one bit in minimum time quantum */
-+static inline unsigned int can_bit_time_tqmin(const struct can_bittiming *bt)
-+{
-+	return can_bit_time(bt) * bt->brp;
-+}
-+
-+/* Convert a duration from minimum a minimum time quantum to nano seconds */
-+static inline u32 can_tqmin_to_ns(u32 tqmin, u32 clock_freq)
-+{
-+	return DIV_U64_ROUND_CLOSEST(mul_u32_u32(tqmin, NSEC_PER_SEC),
-+				     clock_freq);
-+}
-+
- #endif /* !_CAN_BITTIMING_H */
-diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h
-index c3e50e537e39..f6416a56e95d 100644
--- a/include/linux/can/dev.h
-+++ b/include/linux/can/dev.h
-@@ -45,16 +45,11 @@ struct can_priv {
- 	struct net_device *dev;
- 	struct can_device_stats can_stats;
- 
-	const struct can_bittiming_const *bittiming_const,
-		*data_bittiming_const;
-	struct can_bittiming bittiming, data_bittiming;
-	const struct can_tdc_const *tdc_const;
-	struct can_tdc tdc;
-
-+	const struct can_bittiming_const *bittiming_const;
-+	struct can_bittiming bittiming;
-+	struct data_bittiming_params fd, xl;
- 	unsigned int bitrate_const_cnt;
- 	const u32 *bitrate_const;
-	const u32 *data_bitrate_const;
-	unsigned int data_bitrate_const_cnt;
- 	u32 bitrate_max;
- 	struct can_clock clock;
- 
-@@ -77,74 +72,32 @@ struct can_priv {
- 	struct delayed_work restart_work;
- 
- 	int (*do_set_bittiming)(struct net_device *dev);
-	int (*do_set_data_bittiming)(struct net_device *dev);
- 	int (*do_set_mode)(struct net_device *dev, enum can_mode mode);
- 	int (*do_set_termination)(struct net_device *dev, u16 term);
- 	int (*do_get_state)(const struct net_device *dev,
- 			    enum can_state *state);
- 	int (*do_get_berr_counter)(const struct net_device *dev,
- 				   struct can_berr_counter *bec);
-	int (*do_get_auto_tdcv)(const struct net_device *dev, u32 *tdcv);
- };
- 
-static inline bool can_tdc_is_enabled(const struct can_priv *priv)
-+static inline bool can_fd_tdc_is_enabled(const struct can_priv *priv)
- {
-	return !!(priv->ctrlmode & CAN_CTRLMODE_TDC_MASK);
-+	return !!(priv->ctrlmode & CAN_CTRLMODE_FD_TDC_MASK);
- }
- 
-/*
- * can_get_relative_tdco() - TDCO relative to the sample point
- *
- * struct can_tdc::tdco represents the absolute offset from TDCV. Some
- * controllers use instead an offset relative to the Sample Point (SP)
- * such that:
- *
- * SSP = TDCV + absolute TDCO
- *     = TDCV + SP + relative TDCO
- *
- * -+----------- one bit ----------+-- TX pin
- *  |<--- Sample Point --->|
- *
- *                         --+----------- one bit ----------+-- RX pin
- *  |<-------- TDCV -------->|
- *                           |<------------------------>| absolute TDCO
- *                           |<--- Sample Point --->|
- *                           |                      |<->| relative TDCO
- *  |<------------- Secondary Sample Point ------------>|
- */
-static inline s32 can_get_relative_tdco(const struct can_priv *priv)
-+static inline bool can_xl_tdc_is_enabled(const struct can_priv *priv)
- {
-	const struct can_bittiming *dbt = &priv->data_bittiming;
-	s32 sample_point_in_tc = (CAN_SYNC_SEG + dbt->prop_seg +
-				  dbt->phase_seg1) * dbt->brp;
-
-	return (s32)priv->tdc.tdco - sample_point_in_tc;
-+	return !!(priv->ctrlmode & CAN_CTRLMODE_XL_TDC_MASK);
- }
- 
-/* helper to define static CAN controller features at device creation time */
-static inline int __must_check can_set_static_ctrlmode(struct net_device *dev,
-						       u32 static_mode)
-+static inline u32 can_get_static_ctrlmode(struct can_priv *priv)
- {
-	struct can_priv *priv = netdev_priv(dev);
-
-	/* alloc_candev() succeeded => netdev_priv() is valid at this point */
-	if (priv->ctrlmode_supported & static_mode) {
-		netdev_warn(dev,
-			    "Controller features can not be supported and static at the same time\n");
-		return -EINVAL;
-	}
-	priv->ctrlmode = static_mode;
-
-	/* override MTU which was set by default in can_setup()? */
-	if (static_mode & CAN_CTRLMODE_FD)
-		dev->mtu = CANFD_MTU;
-
-	return 0;
-+	return priv->ctrlmode & ~priv->ctrlmode_supported;
- }
- 
-static inline u32 can_get_static_ctrlmode(struct can_priv *priv)
-+static inline bool can_is_canxl_dev_mtu(unsigned int mtu)
- {
-	return priv->ctrlmode & ~priv->ctrlmode_supported;
-+	return (mtu >= CANXL_MIN_MTU && mtu <= CANXL_MAX_MTU);
- }
- 
- void can_setup(struct net_device *dev);
-@@ -158,14 +111,27 @@ struct net_device *alloc_candev_mqs(int sizeof_priv, unsigned int echo_skb_max,
- void free_candev(struct net_device *dev);
- 
- /* a candev safe wrapper around netdev_priv */
-+#if IS_ENABLED(CONFIG_CAN_NETLINK)
- struct can_priv *safe_candev_priv(struct net_device *dev);
-+#else
-+static inline struct can_priv *safe_candev_priv(struct net_device *dev)
-+{
-+	return NULL;
-+}
-+#endif
- 
- int open_candev(struct net_device *dev);
- void close_candev(struct net_device *dev);
-int can_change_mtu(struct net_device *dev, int new_mtu);
-int can_eth_ioctl_hwts(struct net_device *netdev, struct ifreq *ifr, int cmd);
-+void can_set_default_mtu(struct net_device *dev);
-+int __must_check can_set_static_ctrlmode(struct net_device *dev,
-+					 u32 static_mode);
-+int can_hwtstamp_get(struct net_device *netdev,
-+		     struct kernel_hwtstamp_config *cfg);
-+int can_hwtstamp_set(struct net_device *netdev,
-+		     struct kernel_hwtstamp_config *cfg,
-+		     struct netlink_ext_ack *extack);
- int can_ethtool_op_get_ts_info_hwts(struct net_device *dev,
-				    struct ethtool_ts_info *info);
-+				    struct kernel_ethtool_ts_info *info);
- 
- int register_candev(struct net_device *dev);
- void unregister_candev(struct net_device *dev);
-@@ -174,6 +140,57 @@ int can_restart_now(struct net_device *dev);
- void can_bus_off(struct net_device *dev);
- 
- const char *can_get_state_str(const enum can_state state);
-+const char *can_get_ctrlmode_str(u32 ctrlmode);
-+
-+static inline bool can_dev_in_xl_only_mode(struct can_priv *priv)
-+{
-+	const u32 mixed_mode = CAN_CTRLMODE_FD | CAN_CTRLMODE_XL;
-+
-+	/* When CAN XL is enabled but FD is disabled we are running in
-+	 * the so-called 'CANXL-only mode' where the error signalling is
-+	 * disabled. This helper function determines the required value
-+	 * to disable error signalling in the CAN XL controller.
-+	 * The so-called CC/FD/XL 'mixed mode' requires error signalling.
-+	 */
-+	return ((priv->ctrlmode & mixed_mode) == CAN_CTRLMODE_XL);
-+}
-+
-+/* drop skb if it does not contain a valid CAN frame for sending */
-+static inline bool can_dev_dropped_skb(struct net_device *dev, struct sk_buff *skb)
-+{
-+	struct can_priv *priv = netdev_priv(dev);
-+	u32 silent_mode = priv->ctrlmode & (CAN_CTRLMODE_LISTENONLY |
-+					    CAN_CTRLMODE_RESTRICTED);
-+
-+	if (silent_mode) {
-+		netdev_info_once(dev, "interface in %s mode, dropping skb\n",
-+				 can_get_ctrlmode_str(silent_mode));
-+		goto invalid_skb;
-+	}
-+
-+	if (!(priv->ctrlmode & CAN_CTRLMODE_FD) && can_is_canfd_skb(skb)) {
-+		netdev_info_once(dev, "CAN FD is disabled, dropping skb\n");
-+		goto invalid_skb;
-+	}
-+
-+	if (can_dev_in_xl_only_mode(priv) && !can_is_canxl_skb(skb)) {
-+		netdev_info_once(dev,
-+				 "Error signaling is disabled, dropping skb\n");
-+		goto invalid_skb;
-+	}
-+
-+	return can_dropped_invalid_skb(dev, skb);
-+
-+invalid_skb:
-+	kfree_skb(skb);
-+	dev->stats.tx_dropped++;
-+	return true;
-+}
-+
-+void can_state_get_by_berr_counter(const struct net_device *dev,
-+				   const struct can_berr_counter *bec,
-+				   enum can_state *tx_state,
-+				   enum can_state *rx_state);
- void can_change_state(struct net_device *dev, struct can_frame *cf,
- 		      enum can_state tx_state, enum can_state rx_state);
- 
-diff --git a/include/linux/can/dev/peak_canfd.h b/include/linux/can/dev/peak_canfd.h
-index f38772fd0c07..d3788a3d0942 100644
--- a/include/linux/can/dev/peak_canfd.h
-+++ b/include/linux/can/dev/peak_canfd.h
-@@ -2,8 +2,8 @@
- /*
-  * CAN driver for PEAK System micro-CAN based adapters
-  *
- * Copyright (C) 2003-2011 PEAK System-Technik GmbH
- * Copyright (C) 2011-2013 Stephane Grosjean <s.grosjean@peak-system.com>
-+ * Copyright (C) 2003-2025 PEAK System-Technik GmbH
-+ * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
-  */
- #ifndef PUCAN_H
- #define PUCAN_H
-diff --git a/include/linux/can/length.h b/include/linux/can/length.h
-index 6995092b774e..abc978b38f79 100644
--- a/include/linux/can/length.h
-+++ b/include/linux/can/length.h
-@@ -1,126 +1,258 @@
- /* SPDX-License-Identifier: GPL-2.0 */
- /* Copyright (C) 2020 Oliver Hartkopp <socketcan@hartkopp.net>
-  * Copyright (C) 2020 Marc Kleine-Budde <kernel@pengutronix.de>
-+ * Copyright (C) 2020, 2023 Vincent Mailhol <mailhol.vincent@wanadoo.fr>
-  */
- 
- #ifndef _CAN_LENGTH_H
- #define _CAN_LENGTH_H
- 
-+#include <linux/bits.h>
-+#include <linux/can.h>
-+#include <linux/can/netlink.h>
-+#include <linux/math.h>
-+
- /*
- * Size of a Classical CAN Standard Frame
-+ * Size of a Classical CAN Standard Frame header in bits
-  *
- * Name of Field			Bits
-+ * Name of Field				Bits
-  * ---------------------------------------------------------
- * Start-of-frame			1
- * Identifier				11
- * Remote transmission request (RTR)	1
- * Identifier extension bit (IDE)	1
- * Reserved bit (r0)			1
- * Data length code (DLC)		4
- * Data field				0...64
- * CRC					15
- * CRC delimiter			1
- * ACK slot				1
- * ACK delimiter			1
- * End-of-frame (EOF)			7
- * Inter frame spacing			3
-+ * Start Of Frame (SOF)				1
-+ * Arbitration field:
-+ *	base ID					11
-+ *	Remote Transmission Request (RTR)	1
-+ * Control field:
-+ *	IDentifier Extension bit (IDE)		1
-+ *	FD Format indicator (FDF)		1
-+ *	Data Length Code (DLC)			4
-+ *
-+ * including all fields preceding the data field, ignoring bitstuffing
-+ */
-+#define CAN_FRAME_HEADER_SFF_BITS 19
-+
-+/*
-+ * Size of a Classical CAN Extended Frame header in bits
-+ *
-+ * Name of Field				Bits
-+ * ---------------------------------------------------------
-+ * Start Of Frame (SOF)				1
-+ * Arbitration field:
-+ *	base ID					11
-+ *	Substitute Remote Request (SRR)		1
-+ *	IDentifier Extension bit (IDE)		1
-+ *	ID extension				18
-+ *	Remote Transmission Request (RTR)	1
-+ * Control field:
-+ *	FD Format indicator (FDF)		1
-+ *	Reserved bit (r0)			1
-+ *	Data length code (DLC)			4
-+ *
-+ * including all fields preceding the data field, ignoring bitstuffing
-+ */
-+#define CAN_FRAME_HEADER_EFF_BITS 39
-+
-+/*
-+ * Size of a CAN-FD Standard Frame in bits
-  *
- * rounded up and ignoring bitstuffing
-+ * Name of Field				Bits
-+ * ---------------------------------------------------------
-+ * Start Of Frame (SOF)				1
-+ * Arbitration field:
-+ *	base ID					11
-+ *	Remote Request Substitution (RRS)	1
-+ * Control field:
-+ *	IDentifier Extension bit (IDE)		1
-+ *	FD Format indicator (FDF)		1
-+ *	Reserved bit (res)			1
-+ *	Bit Rate Switch (BRS)			1
-+ *	Error Status Indicator (ESI)		1
-+ *	Data length code (DLC)			4
-+ *
-+ * including all fields preceding the data field, ignoring bitstuffing
-+ */
-+#define CANFD_FRAME_HEADER_SFF_BITS 22
-+
-+/*
-+ * Size of a CAN-FD Extended Frame in bits
-+ *
-+ * Name of Field				Bits
-+ * ---------------------------------------------------------
-+ * Start Of Frame (SOF)				1
-+ * Arbitration field:
-+ *	base ID					11
-+ *	Substitute Remote Request (SRR)		1
-+ *	IDentifier Extension bit (IDE)		1
-+ *	ID extension				18
-+ *	Remote Request Substitution (RRS)	1
-+ * Control field:
-+ *	FD Format indicator (FDF)		1
-+ *	Reserved bit (res)			1
-+ *	Bit Rate Switch (BRS)			1
-+ *	Error Status Indicator (ESI)		1
-+ *	Data length code (DLC)			4
-+ *
-+ * including all fields preceding the data field, ignoring bitstuffing
-  */
-#define CAN_FRAME_OVERHEAD_SFF DIV_ROUND_UP(47, 8)
-+#define CANFD_FRAME_HEADER_EFF_BITS 41
- 
- /*
- * Size of a Classical CAN Extended Frame
-+ * Size of a CAN CRC Field in bits
-  *
-  * Name of Field			Bits
-  * ---------------------------------------------------------
- * Start-of-frame			1
- * Identifier A				11
- * Substitute remote request (SRR)	1
- * Identifier extension bit (IDE)	1
- * Identifier B				18
- * Remote transmission request (RTR)	1
- * Reserved bits (r1, r0)		2
- * Data length code (DLC)		4
- * Data field				0...64
- * CRC					15
- * CRC delimiter			1
- * ACK slot				1
- * ACK delimiter			1
- * End-of-frame (EOF)			7
- * Inter frame spacing			3
-+ * CRC sequence (CRC15)			15
-+ * CRC Delimiter			1
-  *
- * rounded up and ignoring bitstuffing
-+ * ignoring bitstuffing
-  */
-#define CAN_FRAME_OVERHEAD_EFF DIV_ROUND_UP(67, 8)
-+#define CAN_FRAME_CRC_FIELD_BITS 16
- 
- /*
- * Size of a CAN-FD Standard Frame
-+ * Size of a CAN-FD CRC17 Field in bits (length: 0..16)
-  *
-  * Name of Field			Bits
-  * ---------------------------------------------------------
- * Start-of-frame			1
- * Identifier				11
- * Reserved bit (r1)			1
- * Identifier extension bit (IDE)	1
- * Flexible data rate format (FDF)	1
- * Reserved bit (r0)			1
- * Bit Rate Switch (BRS)		1
- * Error Status Indicator (ESI)		1
- * Data length code (DLC)		4
- * Data field				0...512
- * Stuff Bit Count (SBC)		0...16: 4 20...64:5
- * CRC					0...16: 17 20...64:21
- * CRC delimiter (CD)			1
- * ACK slot (AS)			1
- * ACK delimiter (AD)			1
- * End-of-frame (EOF)			7
- * Inter frame spacing			3
- *
- * assuming CRC21, rounded up and ignoring bitstuffing
- */
-#define CANFD_FRAME_OVERHEAD_SFF DIV_ROUND_UP(61, 8)
-+ * Stuff Count				4
-+ * CRC Sequence (CRC17)			17
-+ * CRC Delimiter			1
-+ * Fixed stuff bits			6
-+ */
-+#define CANFD_FRAME_CRC17_FIELD_BITS 28
- 
- /*
- * Size of a CAN-FD Extended Frame
-+ * Size of a CAN-FD CRC21 Field in bits (length: 20..64)
-  *
-  * Name of Field			Bits
-  * ---------------------------------------------------------
- * Start-of-frame			1
- * Identifier A				11
- * Substitute remote request (SRR)	1
- * Identifier extension bit (IDE)	1
- * Identifier B				18
- * Reserved bit (r1)			1
- * Flexible data rate format (FDF)	1
- * Reserved bit (r0)			1
- * Bit Rate Switch (BRS)		1
- * Error Status Indicator (ESI)		1
- * Data length code (DLC)		4
- * Data field				0...512
- * Stuff Bit Count (SBC)		0...16: 4 20...64:5
- * CRC					0...16: 17 20...64:21
- * CRC delimiter (CD)			1
- * ACK slot (AS)			1
- * ACK delimiter (AD)			1
- * End-of-frame (EOF)			7
- * Inter frame spacing			3
- *
- * assuming CRC21, rounded up and ignoring bitstuffing
- */
-#define CANFD_FRAME_OVERHEAD_EFF DIV_ROUND_UP(80, 8)
-+ * Stuff Count				4
-+ * CRC sequence (CRC21)			21
-+ * CRC Delimiter			1
-+ * Fixed stuff bits			7
-+ */
-+#define CANFD_FRAME_CRC21_FIELD_BITS 33
-+
-+/*
-+ * Size of a CAN(-FD) Frame footer in bits
-+ *
-+ * Name of Field			Bits
-+ * ---------------------------------------------------------
-+ * ACK slot				1
-+ * ACK delimiter			1
-+ * End Of Frame (EOF)			7
-+ *
-+ * including all fields following the CRC field
-+ */
-+#define CAN_FRAME_FOOTER_BITS 9
-+
-+/*
-+ * First part of the Inter Frame Space
-+ * (a.k.a. IMF - intermission field)
-+ */
-+#define CAN_INTERMISSION_BITS 3
-+
-+/**
-+ * can_bitstuffing_len() - Calculate the maximum length with bitstuffing
-+ * @destuffed_len: length of a destuffed bit stream
-+ *
-+ * The worst bit stuffing case is a sequence in which dominant and
-+ * recessive bits alternate every four bits:
-+ *
-+ *   Destuffed: 1 1111  0000  1111  0000  1111
-+ *   Stuffed:   1 1111o 0000i 1111o 0000i 1111o
-+ *
-+ * Nomenclature
-+ *
-+ *  - "0": dominant bit
-+ *  - "o": dominant stuff bit
-+ *  - "1": recessive bit
-+ *  - "i": recessive stuff bit
-+ *
-+ * Aside from the first bit, one stuff bit is added every four bits.
-+ *
-+ * Return: length of the stuffed bit stream in the worst case scenario.
-+ */
-+#define can_bitstuffing_len(destuffed_len)			\
-+	(destuffed_len + (destuffed_len - 1) / 4)
-+
-+#define __can_bitstuffing_len(bitstuffing, destuffed_len)	\
-+	(bitstuffing ? can_bitstuffing_len(destuffed_len) :	\
-+		       destuffed_len)
-+
-+#define __can_cc_frame_bits(is_eff, bitstuffing,		\
-+			    intermission, data_len)		\
-+(								\
-+	__can_bitstuffing_len(bitstuffing,			\
-+		(is_eff ? CAN_FRAME_HEADER_EFF_BITS :		\
-+			  CAN_FRAME_HEADER_SFF_BITS) +		\
-+		(data_len) * BITS_PER_BYTE +			\
-+		CAN_FRAME_CRC_FIELD_BITS) +			\
-+	CAN_FRAME_FOOTER_BITS +					\
-+	(intermission ? CAN_INTERMISSION_BITS : 0)		\
-+)
-+
-+#define __can_fd_frame_bits(is_eff, bitstuffing,		\
-+			    intermission, data_len)		\
-+(								\
-+	__can_bitstuffing_len(bitstuffing,			\
-+		(is_eff ? CANFD_FRAME_HEADER_EFF_BITS :		\
-+			  CANFD_FRAME_HEADER_SFF_BITS) +	\
-+		(data_len) * BITS_PER_BYTE) +			\
-+	((data_len) <= 16 ?					\
-+		CANFD_FRAME_CRC17_FIELD_BITS :			\
-+		CANFD_FRAME_CRC21_FIELD_BITS) +			\
-+	CAN_FRAME_FOOTER_BITS +					\
-+	(intermission ? CAN_INTERMISSION_BITS : 0)		\
-+)
-+
-+/**
-+ * can_frame_bits() - Calculate the number of bits on the wire in a
-+ *	CAN frame
-+ * @is_fd: true: CAN-FD frame; false: Classical CAN frame.
-+ * @is_eff: true: Extended frame; false: Standard frame.
-+ * @bitstuffing: true: calculate the bitstuffing worst case; false:
-+ *	calculate the bitstuffing best case (no dynamic
-+ *	bitstuffing). CAN-FD's fixed stuff bits are always included.
-+ * @intermission: if and only if true, include the inter frame space
-+ *	assuming no bus idle (i.e. only the intermission). Strictly
-+ *	speaking, the inter frame space is not part of the
-+ *	frame. However, it is needed when calculating the delay
-+ *	between the Start Of Frame of two consecutive frames.
-+ * @data_len: length of the data field in bytes. Correspond to
-+ *	can(fd)_frame->len. Should be zero for remote frames. No
-+ *	sanitization is done on @data_len and it shall have no side
-+ *	effects.
-+ *
-+ * Return: the numbers of bits on the wire of a CAN frame.
-+ */
-+#define can_frame_bits(is_fd, is_eff, bitstuffing,		\
-+		       intermission, data_len)			\
-+(								\
-+	is_fd ? __can_fd_frame_bits(is_eff, bitstuffing,	\
-+				    intermission, data_len) :	\
-+		__can_cc_frame_bits(is_eff, bitstuffing,	\
-+				    intermission, data_len)	\
-+)
-+
-+/*
-+ * Number of bytes in a CAN frame
-+ * (rounded up, including intermission)
-+ */
-+#define can_frame_bytes(is_fd, is_eff, bitstuffing, data_len)	\
-+	DIV_ROUND_UP(can_frame_bits(is_fd, is_eff, bitstuffing,	\
-+				    true, data_len),		\
-+		     BITS_PER_BYTE)
- 
- /*
-  * Maximum size of a Classical CAN frame
- * (rounded up and ignoring bitstuffing)
-+ * (rounded up, ignoring bitstuffing but including intermission)
-  */
-#define CAN_FRAME_LEN_MAX (CAN_FRAME_OVERHEAD_EFF + CAN_MAX_DLEN)
-+#define CAN_FRAME_LEN_MAX can_frame_bytes(false, true, false, CAN_MAX_DLEN)
- 
- /*
-  * Maximum size of a CAN-FD frame
- * (rounded up and ignoring bitstuffing)
-+ * (rounded up, ignoring dynamic bitstuffing but including intermission)
-  */
-#define CANFD_FRAME_LEN_MAX (CANFD_FRAME_OVERHEAD_EFF + CANFD_MAX_DLEN)
-+#define CANFD_FRAME_LEN_MAX can_frame_bytes(true, true, false, CANFD_MAX_DLEN)
- 
- /*
-  * can_cc_dlc2len(value) - convert a given data length code (dlc) of a
-diff --git a/include/linux/can/platform/sja1000.h b/include/linux/can/platform/sja1000.h
-index 5755ae5a4712..6a869682c120 100644
--- a/include/linux/can/platform/sja1000.h
-+++ b/include/linux/can/platform/sja1000.h
-@@ -14,7 +14,7 @@
- #define OCR_MODE_TEST     0x01
- #define OCR_MODE_NORMAL   0x02
- #define OCR_MODE_CLOCK    0x03
-#define OCR_MODE_MASK     0x07
-+#define OCR_MODE_MASK     0x03
- #define OCR_TX0_INVERT    0x04
- #define OCR_TX0_PULLDOWN  0x08
- #define OCR_TX0_PULLUP    0x10
-diff --git a/include/linux/can/rx-offload.h b/include/linux/can/rx-offload.h
-index c205c51d79c9..d29bb4521947 100644
--- a/include/linux/can/rx-offload.h
-+++ b/include/linux/can/rx-offload.h
-@@ -3,7 +3,7 @@
-  * linux/can/rx-offload.h
-  *
-  * Copyright (c) 2014 David Jander, Protonic Holland
- * Copyright (c) 2014-2017 Pengutronix, Marc Kleine-Budde <kernel@pengutronix.de>
-+ * Copyright (c) 2014-2017, 2023 Pengutronix, Marc Kleine-Budde <kernel@pengutronix.de>
-  */
- 
- #ifndef _CAN_RX_OFFLOAD_H
-@@ -44,11 +44,14 @@ int can_rx_offload_irq_offload_timestamp(struct can_rx_offload *offload,
- int can_rx_offload_irq_offload_fifo(struct can_rx_offload *offload);
- int can_rx_offload_queue_timestamp(struct can_rx_offload *offload,
- 				   struct sk_buff *skb, u32 timestamp);
-unsigned int can_rx_offload_get_echo_skb(struct can_rx_offload *offload,
-					 unsigned int idx, u32 timestamp,
-					 unsigned int *frame_len_ptr);
-+unsigned int can_rx_offload_get_echo_skb_queue_timestamp(struct can_rx_offload *offload,
-+							 unsigned int idx, u32 timestamp,
-+							 unsigned int *frame_len_ptr);
- int can_rx_offload_queue_tail(struct can_rx_offload *offload,
- 			      struct sk_buff *skb);
-+unsigned int can_rx_offload_get_echo_skb_queue_tail(struct can_rx_offload *offload,
-+						    unsigned int idx,
-+						    unsigned int *frame_len_ptr);
- void can_rx_offload_irq_finish(struct can_rx_offload *offload);
- void can_rx_offload_threaded_irq_finish(struct can_rx_offload *offload);
- void can_rx_offload_del(struct can_rx_offload *offload);
-diff --git a/include/linux/can/skb.h b/include/linux/can/skb.h
-index 182749e858b3..1abc25a8d144 100644
--- a/include/linux/can/skb.h
-+++ b/include/linux/can/skb.h
-@@ -20,7 +20,8 @@ void can_flush_echo_skb(struct net_device *dev);
- int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev,
- 		     unsigned int idx, unsigned int frame_len);
- struct sk_buff *__can_get_echo_skb(struct net_device *dev, unsigned int idx,
-				   u8 *len_ptr, unsigned int *frame_len_ptr);
-+				   unsigned int *len_ptr,
-+				   unsigned int *frame_len_ptr);
- unsigned int __must_check can_get_echo_skb(struct net_device *dev,
- 					   unsigned int idx,
- 					   unsigned int *frame_len_ptr);
-@@ -29,6 +30,9 @@ void can_free_echo_skb(struct net_device *dev, unsigned int idx,
- struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf);
- struct sk_buff *alloc_canfd_skb(struct net_device *dev,
- 				struct canfd_frame **cfd);
-+struct sk_buff *alloc_canxl_skb(struct net_device *dev,
-+				struct canxl_frame **cxl,
-+				unsigned int data_len);
- struct sk_buff *alloc_can_err_skb(struct net_device *dev,
- 				  struct can_frame **cf);
- bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb);
-@@ -97,10 +101,59 @@ static inline struct sk_buff *can_create_echo_skb(struct sk_buff *skb)
- 	return nskb;
- }
- 
-+static inline bool can_is_can_skb(const struct sk_buff *skb)
-+{
-+	struct can_frame *cf = (struct can_frame *)skb->data;
-+
-+	/* the CAN specific type of skb is identified by its data length */
-+	return (skb->len == CAN_MTU && cf->len <= CAN_MAX_DLEN);
-+}
-+
- static inline bool can_is_canfd_skb(const struct sk_buff *skb)
- {
-+	struct canfd_frame *cfd = (struct canfd_frame *)skb->data;
-+
- 	/* the CAN specific type of skb is identified by its data length */
-	return skb->len == CANFD_MTU;
-+	return (skb->len == CANFD_MTU && cfd->len <= CANFD_MAX_DLEN);
-+}
-+
-+static inline bool can_is_canxl_skb(const struct sk_buff *skb)
-+{
-+	const struct canxl_frame *cxl = (struct canxl_frame *)skb->data;
-+
-+	if (skb->len < CANXL_HDR_SIZE + CANXL_MIN_DLEN || skb->len > CANXL_MTU)
-+		return false;
-+
-+	/* this also checks valid CAN XL data length boundaries */
-+	if (skb->len != CANXL_HDR_SIZE + cxl->len)
-+		return false;
-+
-+	return cxl->flags & CANXL_XLF;
-+}
-+
-+/* get length element value from can[|fd|xl]_frame structure */
-+static inline unsigned int can_skb_get_len_val(struct sk_buff *skb)
-+{
-+	const struct canxl_frame *cxl = (struct canxl_frame *)skb->data;
-+	const struct canfd_frame *cfd = (struct canfd_frame *)skb->data;
-+
-+	if (can_is_canxl_skb(skb))
-+		return cxl->len;
-+
-+	return cfd->len;
-+}
-+
-+/* get needed data length inside CAN frame for all frame types (RTR aware) */
-+static inline unsigned int can_skb_get_data_len(struct sk_buff *skb)
-+{
-+	unsigned int len = can_skb_get_len_val(skb);
-+	const struct can_frame *cf = (struct can_frame *)skb->data;
-+
-+	/* RTR frames have an actual length of zero */
-+	if (can_is_can_skb(skb) && cf->can_id & CAN_RTR_FLAG)
-+		return 0;
-+
-+	return len;
- }
- 
- #endif /* !_CAN_SKB_H */
-diff --git a/include/linux/capability.h b/include/linux/capability.h
-index 65efb74c3585..1fb08922552c 100644
--- a/include/linux/capability.h
-+++ b/include/linux/capability.h
-@@ -15,43 +15,31 @@
- 
- #include <uapi/linux/capability.h>
- #include <linux/uidgid.h>
-+#include <linux/bits.h>
- 
- #define _KERNEL_CAPABILITY_VERSION _LINUX_CAPABILITY_VERSION_3
-#define _KERNEL_CAPABILITY_U32S    _LINUX_CAPABILITY_U32S_3
- 
- extern int file_caps_enabled;
- 
-typedef struct kernel_cap_struct {
-	__u32 cap[_KERNEL_CAPABILITY_U32S];
-} kernel_cap_t;
-+typedef struct { u64 val; } kernel_cap_t;
- 
- /* same as vfs_ns_cap_data but in cpu endian and always filled completely */
- struct cpu_vfs_cap_data {
- 	__u32 magic_etc;
-+	kuid_t rootid;
- 	kernel_cap_t permitted;
- 	kernel_cap_t inheritable;
-	kuid_t rootid;
- };
- 
- #define _USER_CAP_HEADER_SIZE  (sizeof(struct __user_cap_header_struct))
- #define _KERNEL_CAP_T_SIZE     (sizeof(kernel_cap_t))
- 
-
- struct file;
- struct inode;
- struct dentry;
- struct task_struct;
- struct user_namespace;
-
-extern const kernel_cap_t __cap_empty_set;
-extern const kernel_cap_t __cap_init_eff_set;
-
-/*
- * Internal kernel functions only
- */
-
-#define CAP_FOR_EACH_U32(__capi)  \
-	for (__capi = 0; __capi < _KERNEL_CAPABILITY_U32S; ++__capi)
-+struct mnt_idmap;
- 
- /*
-  * CAP_FS_MASK and CAP_NFSD_MASKS:
-@@ -66,94 +54,52 @@ extern const kernel_cap_t __cap_init_eff_set;
-  *   2. The security.* and trusted.* xattrs are fs-related MAC permissions
-  */
- 
-# define CAP_FS_MASK_B0     (CAP_TO_MASK(CAP_CHOWN)		\
-			    | CAP_TO_MASK(CAP_MKNOD)		\
-			    | CAP_TO_MASK(CAP_DAC_OVERRIDE)	\
-			    | CAP_TO_MASK(CAP_DAC_READ_SEARCH)	\
-			    | CAP_TO_MASK(CAP_FOWNER)		\
-			    | CAP_TO_MASK(CAP_FSETID))
-
-# define CAP_FS_MASK_B1     (CAP_TO_MASK(CAP_MAC_OVERRIDE))
-
-#if _KERNEL_CAPABILITY_U32S != 2
-# error Fix up hand-coded capability macro initializers
-#else /* HAND-CODED capability initializers */
-+# define CAP_FS_MASK     (BIT_ULL(CAP_CHOWN)		\
-+			| BIT_ULL(CAP_MKNOD)		\
-+			| BIT_ULL(CAP_DAC_OVERRIDE)	\
-+			| BIT_ULL(CAP_DAC_READ_SEARCH)	\
-+			| BIT_ULL(CAP_FOWNER)		\
-+			| BIT_ULL(CAP_FSETID)		\
-+			| BIT_ULL(CAP_MAC_OVERRIDE))
-+#define CAP_VALID_MASK	 (BIT_ULL(CAP_LAST_CAP+1)-1)
- 
-#define CAP_LAST_U32			((_KERNEL_CAPABILITY_U32S) - 1)
-#define CAP_LAST_U32_VALID_MASK		(CAP_TO_MASK(CAP_LAST_CAP + 1) -1)
-+# define CAP_EMPTY_SET    ((kernel_cap_t) { 0 })
-+# define CAP_FULL_SET     ((kernel_cap_t) { CAP_VALID_MASK })
-+# define CAP_FS_SET       ((kernel_cap_t) { CAP_FS_MASK | BIT_ULL(CAP_LINUX_IMMUTABLE) })
-+# define CAP_NFSD_SET     ((kernel_cap_t) { CAP_FS_MASK | BIT_ULL(CAP_SYS_RESOURCE) })
- 
-# define CAP_EMPTY_SET    ((kernel_cap_t){{ 0, 0 }})
-# define CAP_FULL_SET     ((kernel_cap_t){{ ~0, CAP_LAST_U32_VALID_MASK }})
-# define CAP_FS_SET       ((kernel_cap_t){{ CAP_FS_MASK_B0 \
-				    | CAP_TO_MASK(CAP_LINUX_IMMUTABLE), \
-				    CAP_FS_MASK_B1 } })
-# define CAP_NFSD_SET     ((kernel_cap_t){{ CAP_FS_MASK_B0 \
-				    | CAP_TO_MASK(CAP_SYS_RESOURCE), \
-				    CAP_FS_MASK_B1 } })
-+# define cap_clear(c)         do { (c).val = 0; } while (0)
- 
-#endif /* _KERNEL_CAPABILITY_U32S != 2 */
-
-# define cap_clear(c)         do { (c) = __cap_empty_set; } while (0)
-
-#define cap_raise(c, flag)  ((c).cap[CAP_TO_INDEX(flag)] |= CAP_TO_MASK(flag))
-#define cap_lower(c, flag)  ((c).cap[CAP_TO_INDEX(flag)] &= ~CAP_TO_MASK(flag))
-#define cap_raised(c, flag) ((c).cap[CAP_TO_INDEX(flag)] & CAP_TO_MASK(flag))
-
-#define CAP_BOP_ALL(c, a, b, OP)                                    \
-do {                                                                \
-	unsigned __capi;                                            \
-	CAP_FOR_EACH_U32(__capi) {                                  \
-		c.cap[__capi] = a.cap[__capi] OP b.cap[__capi];     \
-	}                                                           \
-} while (0)
-
-#define CAP_UOP_ALL(c, a, OP)                                       \
-do {                                                                \
-	unsigned __capi;                                            \
-	CAP_FOR_EACH_U32(__capi) {                                  \
-		c.cap[__capi] = OP a.cap[__capi];                   \
-	}                                                           \
-} while (0)
-+#define cap_raise(c, flag)  ((c).val |= BIT_ULL(flag))
-+#define cap_lower(c, flag)  ((c).val &= ~BIT_ULL(flag))
-+#define cap_raised(c, flag) (((c).val & BIT_ULL(flag)) != 0)
- 
- static inline kernel_cap_t cap_combine(const kernel_cap_t a,
- 				       const kernel_cap_t b)
- {
-	kernel_cap_t dest;
-	CAP_BOP_ALL(dest, a, b, |);
-	return dest;
-+	return (kernel_cap_t) { a.val | b.val };
- }
- 
- static inline kernel_cap_t cap_intersect(const kernel_cap_t a,
- 					 const kernel_cap_t b)
- {
-	kernel_cap_t dest;
-	CAP_BOP_ALL(dest, a, b, &);
-	return dest;
-+	return (kernel_cap_t) { a.val & b.val };
- }
- 
- static inline kernel_cap_t cap_drop(const kernel_cap_t a,
- 				    const kernel_cap_t drop)
- {
-	kernel_cap_t dest;
-	CAP_BOP_ALL(dest, a, drop, &~);
-	return dest;
-+	return (kernel_cap_t) { a.val &~ drop.val };
- }
- 
-static inline kernel_cap_t cap_invert(const kernel_cap_t c)
-+static inline bool cap_isclear(const kernel_cap_t a)
- {
-	kernel_cap_t dest;
-	CAP_UOP_ALL(dest, c, ~);
-	return dest;
-+	return !a.val;
- }
- 
-static inline bool cap_isclear(const kernel_cap_t a)
-+static inline bool cap_isidentical(const kernel_cap_t a, const kernel_cap_t b)
- {
-	unsigned __capi;
-	CAP_FOR_EACH_U32(__capi) {
-		if (a.cap[__capi] != 0)
-			return false;
-	}
-	return true;
-+	return a.val == b.val;
- }
- 
- /*
-@@ -165,43 +111,34 @@ static inline bool cap_isclear(const kernel_cap_t a)
-  */
- static inline bool cap_issubset(const kernel_cap_t a, const kernel_cap_t set)
- {
-	kernel_cap_t dest;
-	dest = cap_drop(a, set);
-	return cap_isclear(dest);
-+	return !(a.val & ~set.val);
- }
- 
- /* Used to decide between falling back on the old suser() or fsuser(). */
- 
- static inline kernel_cap_t cap_drop_fs_set(const kernel_cap_t a)
- {
-	const kernel_cap_t __cap_fs_set = CAP_FS_SET;
-	return cap_drop(a, __cap_fs_set);
-+	return cap_drop(a, CAP_FS_SET);
- }
- 
- static inline kernel_cap_t cap_raise_fs_set(const kernel_cap_t a,
- 					    const kernel_cap_t permitted)
- {
-	const kernel_cap_t __cap_fs_set = CAP_FS_SET;
-	return cap_combine(a,
-			   cap_intersect(permitted, __cap_fs_set));
-+	return cap_combine(a, cap_intersect(permitted, CAP_FS_SET));
- }
- 
- static inline kernel_cap_t cap_drop_nfsd_set(const kernel_cap_t a)
- {
-	const kernel_cap_t __cap_fs_set = CAP_NFSD_SET;
-	return cap_drop(a, __cap_fs_set);
-+	return cap_drop(a, CAP_NFSD_SET);
- }
- 
- static inline kernel_cap_t cap_raise_nfsd_set(const kernel_cap_t a,
- 					      const kernel_cap_t permitted)
- {
-	const kernel_cap_t __cap_nfsd_set = CAP_NFSD_SET;
-	return cap_combine(a,
-			   cap_intersect(permitted, __cap_nfsd_set));
-+	return cap_combine(a, cap_intersect(permitted, CAP_NFSD_SET));
- }
- 
- #ifdef CONFIG_MULTIUSER
-extern bool has_capability(struct task_struct *t, int cap);
- extern bool has_ns_capability(struct task_struct *t,
- 			      struct user_namespace *ns, int cap);
- extern bool has_capability_noaudit(struct task_struct *t, int cap);
-@@ -212,10 +149,6 @@ extern bool ns_capable(struct user_namespace *ns, int cap);
- extern bool ns_capable_noaudit(struct user_namespace *ns, int cap);
- extern bool ns_capable_setid(struct user_namespace *ns, int cap);
- #else
-static inline bool has_capability(struct task_struct *t, int cap)
-{
-	return true;
-}
- static inline bool has_ns_capability(struct task_struct *t,
- 			      struct user_namespace *ns, int cap)
- {
-@@ -248,9 +181,9 @@ static inline bool ns_capable_setid(struct user_namespace *ns, int cap)
- }
- #endif /* CONFIG_MULTIUSER */
- bool privileged_wrt_inode_uidgid(struct user_namespace *ns,
-				 struct user_namespace *mnt_userns,
-+				 struct mnt_idmap *idmap,
- 				 const struct inode *inode);
-bool capable_wrt_inode_uidgid(struct user_namespace *mnt_userns,
-+bool capable_wrt_inode_uidgid(struct mnt_idmap *idmap,
- 			      const struct inode *inode, int cap);
- extern bool file_ns_capable(const struct file *file, struct user_namespace *ns, int cap);
- extern bool ptracer_capable(struct task_struct *tsk, struct user_namespace *ns);
-@@ -271,11 +204,11 @@ static inline bool checkpoint_restore_ns_capable(struct user_namespace *ns)
- }
- 
- /* audit system wants to get cap info from files as well */
-int get_vfs_caps_from_disk(struct user_namespace *mnt_userns,
-+int get_vfs_caps_from_disk(struct mnt_idmap *idmap,
- 			   const struct dentry *dentry,
- 			   struct cpu_vfs_cap_data *cpu_caps);
- 
-int cap_convert_nscap(struct user_namespace *mnt_userns, struct dentry *dentry,
-+int cap_convert_nscap(struct mnt_idmap *idmap, struct dentry *dentry,
- 		      const void **ivalue, size_t size);
- 
- #endif /* !_LINUX_CAPABILITY_H */
-diff --git a/include/linux/cc_platform.h b/include/linux/cc_platform.h
-index cb0d6cd1c12f..559353ad64ac 100644
--- a/include/linux/cc_platform.h
-+++ b/include/linux/cc_platform.h
-@@ -74,7 +74,7 @@ enum cc_attr {
- 	CC_ATTR_GUEST_UNROLL_STRING_IO,
- 
- 	/**
-	 * @CC_ATTR_SEV_SNP: Guest SNP is active.
-+	 * @CC_ATTR_GUEST_SEV_SNP: Guest SNP is active.
- 	 *
- 	 * The platform/OS is running as a guest/virtual machine and actively
- 	 * using AMD SEV-SNP features.
-@@ -82,14 +82,28 @@ enum cc_attr {
- 	CC_ATTR_GUEST_SEV_SNP,
- 
- 	/**
-	 * @CC_ATTR_HOTPLUG_DISABLED: Hotplug is not supported or disabled.
-+	 * @CC_ATTR_GUEST_SNP_SECURE_TSC: SNP Secure TSC is active.
- 	 *
-	 * The platform/OS is running as a guest/virtual machine does not
-	 * support CPU hotplug feature.
-+	 * The platform/OS is running as a guest/virtual machine and actively
-+	 * using AMD SEV-SNP Secure TSC feature.
-+	 */
-+	CC_ATTR_GUEST_SNP_SECURE_TSC,
-+
-+	/**
-+	 * @CC_ATTR_HOST_SEV_SNP: AMD SNP enabled on the host.
-+	 *
-+	 * The host kernel is running with the necessary features
-+	 * enabled to run SEV-SNP guests.
-+	 */
-+	CC_ATTR_HOST_SEV_SNP,
-+
-+	/**
-+	 * @CC_ATTR_SNP_SECURE_AVIC: Secure AVIC mode is active.
- 	 *
-	 * Examples include TDX Guest.
-+	 * The host kernel is running with the necessary features enabled
-+	 * to run SEV-SNP guests with full Secure AVIC capabilities.
- 	 */
-	CC_ATTR_HOTPLUG_DISABLED,
-+	CC_ATTR_SNP_SECURE_AVIC,
- };
- 
- #ifdef CONFIG_ARCH_HAS_CC_PLATFORM
-@@ -107,10 +121,14 @@ enum cc_attr {
-  * * FALSE - Specified Confidential Computing attribute is not active
-  */
- bool cc_platform_has(enum cc_attr attr);
-+void cc_platform_set(enum cc_attr attr);
-+void cc_platform_clear(enum cc_attr attr);
- 
- #else	/* !CONFIG_ARCH_HAS_CC_PLATFORM */
- 
- static inline bool cc_platform_has(enum cc_attr attr) { return false; }
-+static inline void cc_platform_set(enum cc_attr attr) { }
-+static inline void cc_platform_clear(enum cc_attr attr) { }
- 
- #endif	/* CONFIG_ARCH_HAS_CC_PLATFORM */
- 
-diff --git a/include/linux/cdrom.h b/include/linux/cdrom.h
-index 67caa909e3e6..b907e6c2307d 100644
--- a/include/linux/cdrom.h
-+++ b/include/linux/cdrom.h
-@@ -13,6 +13,7 @@
- 
- #include <linux/fs.h>		/* not really needed, later.. */
- #include <linux/list.h>
-+#include <linux/blkdev.h>
- #include <scsi/scsi_common.h>
- #include <uapi/linux/cdrom.h>
- 
-@@ -61,9 +62,8 @@ struct cdrom_device_info {
- 	__u8 last_sense;
- 	__u8 media_written;		/* dirty flag, DVD+RW bookkeeping */
- 	unsigned short mmc3_profile;	/* current MMC3 profile */
-	int for_data;
-	int (*exit)(struct cdrom_device_info *);
- 	int mrw_mode_page;
-+	bool opened_for_data;
- 	__s64 last_media_change_ms;
- };
- 
-@@ -76,7 +76,7 @@ struct cdrom_device_ops {
- 				      unsigned int clearing, int slot);
- 	int (*tray_move) (struct cdrom_device_info *, int);
- 	int (*lock_door) (struct cdrom_device_info *, int);
-	int (*select_speed) (struct cdrom_device_info *, int);
-+	int (*select_speed) (struct cdrom_device_info *, unsigned long);
- 	int (*get_last_session) (struct cdrom_device_info *,
- 				 struct cdrom_multisession *);
- 	int (*get_mcn) (struct cdrom_device_info *,
-@@ -101,11 +101,10 @@ int cdrom_read_tocentry(struct cdrom_device_info *cdi,
- 		struct cdrom_tocentry *entry);
- 
- /* the general block_device operations structure: */
-extern int cdrom_open(struct cdrom_device_info *cdi, struct block_device *bdev,
-			fmode_t mode);
-extern void cdrom_release(struct cdrom_device_info *cdi, fmode_t mode);
-extern int cdrom_ioctl(struct cdrom_device_info *cdi, struct block_device *bdev,
-		       fmode_t mode, unsigned int cmd, unsigned long arg);
-+int cdrom_open(struct cdrom_device_info *cdi, blk_mode_t mode);
-+void cdrom_release(struct cdrom_device_info *cdi);
-+int cdrom_ioctl(struct cdrom_device_info *cdi, struct block_device *bdev,
-+		unsigned int cmd, unsigned long arg);
- extern unsigned int cdrom_check_events(struct cdrom_device_info *cdi,
- 				       unsigned int clearing);
- 
-diff --git a/include/linux/cdx/bitfield.h b/include/linux/cdx/bitfield.h
-new file mode 100644
-index 000000000000..567f8ec47582
--- /dev/null
-+++ b/include/linux/cdx/bitfield.h
-@@ -0,0 +1,90 @@
-+/* SPDX-License-Identifier: GPL-2.0
-+ *
-+ * Copyright 2005-2006 Fen Systems Ltd.
-+ * Copyright 2006-2013 Solarflare Communications Inc.
-+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
-+ */
-+
-+#ifndef CDX_BITFIELD_H
-+#define CDX_BITFIELD_H
-+
-+#include <linux/bitfield.h>
-+
-+/* Lowest bit numbers and widths */
-+#define CDX_DWORD_LBN 0
-+#define CDX_DWORD_WIDTH 32
-+
-+/* Specified attribute (e.g. LBN) of the specified field */
-+#define CDX_VAL(field, attribute) field ## _ ## attribute
-+/* Low bit number of the specified field */
-+#define CDX_LOW_BIT(field) CDX_VAL(field, LBN)
-+/* Bit width of the specified field */
-+#define CDX_WIDTH(field) CDX_VAL(field, WIDTH)
-+/* High bit number of the specified field */
-+#define CDX_HIGH_BIT(field) (CDX_LOW_BIT(field) + CDX_WIDTH(field) - 1)
-+
-+/* A doubleword (i.e. 4 byte) datatype - little-endian in HW */
-+struct cdx_dword {
-+	__le32 cdx_u32;
-+};
-+
-+/* Value expanders for printk */
-+#define CDX_DWORD_VAL(dword)				\
-+	((unsigned int)le32_to_cpu((dword).cdx_u32))
-+
-+/*
-+ * Extract bit field portion [low,high) from the 32-bit little-endian
-+ * element which contains bits [min,max)
-+ */
-+#define CDX_DWORD_FIELD(dword, field)					\
-+	(FIELD_GET(GENMASK(CDX_HIGH_BIT(field), CDX_LOW_BIT(field)),	\
-+		   le32_to_cpu((dword).cdx_u32)))
-+
-+/*
-+ * Creates the portion of the named bit field that lies within the
-+ * range [min,max).
-+ */
-+#define CDX_INSERT_FIELD(field, value)				\
-+	(FIELD_PREP(GENMASK(CDX_HIGH_BIT(field),		\
-+			    CDX_LOW_BIT(field)), value))
-+
-+/*
-+ * Creates the portion of the named bit fields that lie within the
-+ * range [min,max).
-+ */
-+#define CDX_INSERT_FIELDS(field1, value1,		\
-+			  field2, value2,		\
-+			  field3, value3,		\
-+			  field4, value4,		\
-+			  field5, value5,		\
-+			  field6, value6,		\
-+			  field7, value7)		\
-+	(CDX_INSERT_FIELD(field1, (value1)) |		\
-+	 CDX_INSERT_FIELD(field2, (value2)) |		\
-+	 CDX_INSERT_FIELD(field3, (value3)) |		\
-+	 CDX_INSERT_FIELD(field4, (value4)) |		\
-+	 CDX_INSERT_FIELD(field5, (value5)) |		\
-+	 CDX_INSERT_FIELD(field6, (value6)) |		\
-+	 CDX_INSERT_FIELD(field7, (value7)))
-+
-+#define CDX_POPULATE_DWORD(dword, ...)					\
-+	(dword).cdx_u32 = cpu_to_le32(CDX_INSERT_FIELDS(__VA_ARGS__))
-+
-+/* Populate a dword field with various numbers of arguments */
-+#define CDX_POPULATE_DWORD_7 CDX_POPULATE_DWORD
-+#define CDX_POPULATE_DWORD_6(dword, ...) \
-+	CDX_POPULATE_DWORD_7(dword, CDX_DWORD, 0, __VA_ARGS__)
-+#define CDX_POPULATE_DWORD_5(dword, ...) \
-+	CDX_POPULATE_DWORD_6(dword, CDX_DWORD, 0, __VA_ARGS__)
-+#define CDX_POPULATE_DWORD_4(dword, ...) \
-+	CDX_POPULATE_DWORD_5(dword, CDX_DWORD, 0, __VA_ARGS__)
-+#define CDX_POPULATE_DWORD_3(dword, ...) \
-+	CDX_POPULATE_DWORD_4(dword, CDX_DWORD, 0, __VA_ARGS__)
-+#define CDX_POPULATE_DWORD_2(dword, ...) \
-+	CDX_POPULATE_DWORD_3(dword, CDX_DWORD, 0, __VA_ARGS__)
-+#define CDX_POPULATE_DWORD_1(dword, ...) \
-+	CDX_POPULATE_DWORD_2(dword, CDX_DWORD, 0, __VA_ARGS__)
-+#define CDX_SET_DWORD(dword) \
-+	CDX_POPULATE_DWORD_1(dword, CDX_DWORD, 0xffffffff)
-+
-+#endif /* CDX_BITFIELD_H */
-diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h
-new file mode 100644
-index 000000000000..b1ba97f6c9ad
--- /dev/null
-+++ b/include/linux/cdx/cdx_bus.h
-@@ -0,0 +1,291 @@
-+/* SPDX-License-Identifier: GPL-2.0
-+ *
-+ * CDX bus public interface
-+ *
-+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
-+ *
-+ */
-+
-+#ifndef _CDX_BUS_H_
-+#define _CDX_BUS_H_
-+
-+#include <linux/device.h>
-+#include <linux/list.h>
-+#include <linux/mod_devicetable.h>
-+#include <linux/msi.h>
-+
-+#define MAX_CDX_DEV_RESOURCES	4
-+#define CDX_CONTROLLER_ID_SHIFT 4
-+#define CDX_BUS_NUM_MASK 0xF
-+
-+/* Forward declaration for CDX controller */
-+struct cdx_controller;
-+
-+enum {
-+	CDX_DEV_MSI_CONF,
-+	CDX_DEV_BUS_MASTER_CONF,
-+	CDX_DEV_RESET_CONF,
-+	CDX_DEV_MSI_ENABLE,
-+};
-+
-+struct cdx_msi_config {
-+	u64 addr;
-+	u32 data;
-+	u16 msi_index;
-+};
-+
-+struct cdx_device_config {
-+	u8 type;
-+	union {
-+		struct cdx_msi_config msi;
-+		bool bus_master_enable;
-+		bool msi_enable;
-+	};
-+};
-+
-+typedef int (*cdx_bus_enable_cb)(struct cdx_controller *cdx, u8 bus_num);
-+
-+typedef int (*cdx_bus_disable_cb)(struct cdx_controller *cdx, u8 bus_num);
-+
-+typedef int (*cdx_scan_cb)(struct cdx_controller *cdx);
-+
-+typedef int (*cdx_dev_configure_cb)(struct cdx_controller *cdx,
-+				    u8 bus_num, u8 dev_num,
-+				    struct cdx_device_config *dev_config);
-+
-+/**
-+ * CDX_DEVICE - macro used to describe a specific CDX device
-+ * @vend: the 16 bit CDX Vendor ID
-+ * @dev: the 16 bit CDX Device ID
-+ *
-+ * This macro is used to create a struct cdx_device_id that matches a
-+ * specific device. The subvendor and subdevice fields will be set to
-+ * CDX_ANY_ID.
-+ */
-+#define CDX_DEVICE(vend, dev) \
-+	.vendor = (vend), .device = (dev), \
-+	.subvendor = CDX_ANY_ID, .subdevice = CDX_ANY_ID
-+
-+/**
-+ * CDX_DEVICE_DRIVER_OVERRIDE - macro used to describe a CDX device with
-+ *                              override_only flags.
-+ * @vend: the 16 bit CDX Vendor ID
-+ * @dev: the 16 bit CDX Device ID
-+ * @driver_override: the 32 bit CDX Device override_only
-+ *
-+ * This macro is used to create a struct cdx_device_id that matches only a
-+ * driver_override device. The subvendor and subdevice fields will be set to
-+ * CDX_ANY_ID.
-+ */
-+#define CDX_DEVICE_DRIVER_OVERRIDE(vend, dev, driver_override) \
-+	.vendor = (vend), .device = (dev), .subvendor = CDX_ANY_ID,\
-+	.subdevice = CDX_ANY_ID, .override_only = (driver_override)
-+
-+/**
-+ * struct cdx_ops - Callbacks supported by CDX controller.
-+ * @bus_enable: enable bus on the controller
-+ * @bus_disable: disable bus on the controller
-+ * @scan: scan the devices on the controller
-+ * @dev_configure: configuration like reset, master_enable,
-+ *		   msi_config etc for a CDX device
-+ */
-+struct cdx_ops {
-+	cdx_bus_enable_cb bus_enable;
-+	cdx_bus_disable_cb bus_disable;
-+	cdx_scan_cb scan;
-+	cdx_dev_configure_cb dev_configure;
-+};
-+
-+/**
-+ * struct cdx_controller: CDX controller object
-+ * @dev: Linux device associated with the CDX controller.
-+ * @priv: private data
-+ * @msi_domain: MSI domain
-+ * @id: Controller ID
-+ * @controller_registered: controller registered with bus
-+ * @ops: CDX controller ops
-+ */
-+struct cdx_controller {
-+	struct device *dev;
-+	void *priv;
-+	struct irq_domain *msi_domain;
-+	u32 id;
-+	bool controller_registered;
-+	struct cdx_ops *ops;
-+};
-+
-+/**
-+ * struct cdx_device - CDX device object
-+ * @dev: Linux driver model device object
-+ * @cdx: CDX controller associated with the device
-+ * @vendor: Vendor ID for CDX device
-+ * @device: Device ID for CDX device
-+ * @subsystem_vendor: Subsystem Vendor ID for CDX device
-+ * @subsystem_device: Subsystem Device ID for CDX device
-+ * @class: Class for the CDX device
-+ * @revision: Revision of the CDX device
-+ * @bus_num: Bus number for this CDX device
-+ * @dev_num: Device number for this device
-+ * @res: array of MMIO region entries
-+ * @res_attr: resource binary attribute
-+ * @debugfs_dir: debugfs directory for this device
-+ * @res_count: number of valid MMIO regions
-+ * @dma_mask: Default DMA mask
-+ * @flags: CDX device flags
-+ * @req_id: Requestor ID associated with CDX device
-+ * @is_bus: Is this bus device
-+ * @enabled: is this bus enabled
-+ * @msi_dev_id: MSI Device ID associated with CDX device
-+ * @num_msi: Number of MSI's supported by the device
-+ * @driver_override: driver name to force a match; do not set directly,
-+ *                   because core frees it; use driver_set_override() to
-+ *                   set or clear it.
-+ * @irqchip_lock: lock to synchronize irq/msi configuration
-+ * @msi_write_pending: MSI write pending for this device
-+ */
-+struct cdx_device {
-+	struct device dev;
-+	struct cdx_controller *cdx;
-+	u16 vendor;
-+	u16 device;
-+	u16 subsystem_vendor;
-+	u16 subsystem_device;
-+	u32 class;
-+	u8 revision;
-+	u8 bus_num;
-+	u8 dev_num;
-+	struct resource res[MAX_CDX_DEV_RESOURCES];
-+	struct bin_attribute *res_attr[MAX_CDX_DEV_RESOURCES];
-+	struct dentry *debugfs_dir;
-+	u8 res_count;
-+	u64 dma_mask;
-+	u16 flags;
-+	u32 req_id;
-+	bool is_bus;
-+	bool enabled;
-+	u32 msi_dev_id;
-+	u32 num_msi;
-+	const char *driver_override;
-+	struct mutex irqchip_lock;
-+	bool msi_write_pending;
-+};
-+
-+#define to_cdx_device(_dev) \
-+	container_of(_dev, struct cdx_device, dev)
-+
-+#define cdx_resource_start(dev, num)	((dev)->res[(num)].start)
-+#define cdx_resource_end(dev, num)	((dev)->res[(num)].end)
-+#define cdx_resource_flags(dev, num)	((dev)->res[(num)].flags)
-+#define cdx_resource_len(dev, num) \
-+	((cdx_resource_start((dev), (num)) == 0 &&	\
-+	  cdx_resource_end((dev), (num)) ==		\
-+	  cdx_resource_start((dev), (num))) ? 0 :	\
-+	 (cdx_resource_end((dev), (num)) -		\
-+	  cdx_resource_start((dev), (num)) + 1))
-+/**
-+ * struct cdx_driver - CDX device driver
-+ * @driver: Generic device driver
-+ * @match_id_table: table of supported device matching Ids
-+ * @probe: Function called when a device is added
-+ * @remove: Function called when a device is removed
-+ * @shutdown: Function called at shutdown time to quiesce the device
-+ * @reset_prepare: Function called before is reset to notify driver
-+ * @reset_done: Function called after reset is complete to notify driver
-+ * @driver_managed_dma: Device driver doesn't use kernel DMA API for DMA.
-+ *		For most device drivers, no need to care about this flag
-+ *		as long as all DMAs are handled through the kernel DMA API.
-+ *		For some special ones, for example VFIO drivers, they know
-+ *		how to manage the DMA themselves and set this flag so that
-+ *		the IOMMU layer will allow them to setup and manage their
-+ *		own I/O address space.
-+ */
-+struct cdx_driver {
-+	struct device_driver driver;
-+	const struct cdx_device_id *match_id_table;
-+	int (*probe)(struct cdx_device *dev);
-+	int (*remove)(struct cdx_device *dev);
-+	void (*shutdown)(struct cdx_device *dev);
-+	void (*reset_prepare)(struct cdx_device *dev);
-+	void (*reset_done)(struct cdx_device *dev);
-+	bool driver_managed_dma;
-+};
-+
-+#define to_cdx_driver(_drv) \
-+	container_of_const(_drv, struct cdx_driver, driver)
-+
-+/* Macro to avoid include chaining to get THIS_MODULE */
-+#define cdx_driver_register(drv) \
-+	__cdx_driver_register(drv, THIS_MODULE)
-+
-+/**
-+ * __cdx_driver_register - registers a CDX device driver
-+ * @cdx_driver: CDX driver to register
-+ * @owner: module owner
-+ *
-+ * Return: -errno on failure, 0 on success.
-+ */
-+int __must_check __cdx_driver_register(struct cdx_driver *cdx_driver,
-+				       struct module *owner);
-+
-+/**
-+ * cdx_driver_unregister - unregisters a device driver from the
-+ * CDX bus.
-+ * @cdx_driver: CDX driver to register
-+ */
-+void cdx_driver_unregister(struct cdx_driver *cdx_driver);
-+
-+extern const struct bus_type cdx_bus_type;
-+
-+/**
-+ * cdx_dev_reset - Reset CDX device
-+ * @dev: device pointer
-+ *
-+ * Return: 0 for success, -errno on failure
-+ */
-+int cdx_dev_reset(struct device *dev);
-+
-+/**
-+ * cdx_set_master - enables bus-mastering for CDX device
-+ * @cdx_dev: the CDX device to enable
-+ *
-+ * Return: 0 for success, -errno on failure
-+ */
-+int cdx_set_master(struct cdx_device *cdx_dev);
-+
-+/**
-+ * cdx_clear_master - disables bus-mastering for CDX device
-+ * @cdx_dev: the CDX device to disable
-+ *
-+ * Return: 0 for success, -errno on failure
-+ */
-+int cdx_clear_master(struct cdx_device *cdx_dev);
-+
-+#ifdef CONFIG_GENERIC_MSI_IRQ
-+/**
-+ * cdx_enable_msi - Enable MSI for the CDX device.
-+ * @cdx_dev: device pointer
-+ *
-+ * Return: 0 for success, -errno on failure
-+ */
-+int cdx_enable_msi(struct cdx_device *cdx_dev);
-+
-+/**
-+ * cdx_disable_msi - Disable MSI for the CDX device.
-+ * @cdx_dev: device pointer
-+ */
-+void cdx_disable_msi(struct cdx_device *cdx_dev);
-+
-+#else /* CONFIG_GENERIC_MSI_IRQ */
-+
-+static inline int cdx_enable_msi(struct cdx_device *cdx_dev)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline void cdx_disable_msi(struct cdx_device *cdx_dev)
-+{
-+}
-+
-+#endif /* CONFIG_GENERIC_MSI_IRQ */
-+
-+#endif /* _CDX_BUS_H_ */
-diff --git a/include/linux/cdx/edac_cdx_pcol.h b/include/linux/cdx/edac_cdx_pcol.h
-new file mode 100644
-index 000000000000..749db33bb482
--- /dev/null
-+++ b/include/linux/cdx/edac_cdx_pcol.h
-@@ -0,0 +1,28 @@
-+/* SPDX-License-Identifier: GPL-2.0
-+ *
-+ * Driver for AMD network controllers and boards
-+ *
-+ * Copyright (C) 2021, Xilinx, Inc.
-+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
-+ */
-+
-+#ifndef MC_CDX_PCOL_H
-+#define MC_CDX_PCOL_H
-+#include  <linux/cdx/mcdi.h>
-+
-+#define MC_CMD_EDAC_GET_DDR_CONFIG_OUT_WORD_LENGTH_LEN		4
-+/* Number of registers for the DDR controller */
-+#define MC_CMD_GET_DDR_CONFIG_OFST	4
-+#define MC_CMD_GET_DDR_CONFIG_LEN	4
-+
-+/***********************************/
-+/* MC_CMD_EDAC_GET_DDR_CONFIG
-+ * Provides detailed configuration for the DDR controller of the given index.
-+ */
-+#define MC_CMD_EDAC_GET_DDR_CONFIG 0x3
-+
-+/* MC_CMD_EDAC_GET_DDR_CONFIG_IN msgrequest */
-+#define MC_CMD_EDAC_GET_DDR_CONFIG_IN_CONTROLLER_INDEX_OFST		0
-+#define MC_CMD_EDAC_GET_DDR_CONFIG_IN_CONTROLLER_INDEX_LEN		4
-+
-+#endif /* MC_CDX_PCOL_H */
-diff --git a/include/linux/cdx/mcdi.h b/include/linux/cdx/mcdi.h
-new file mode 100644
-index 000000000000..74075305cba4
--- /dev/null
-+++ b/include/linux/cdx/mcdi.h
-@@ -0,0 +1,199 @@
-+/* SPDX-License-Identifier: GPL-2.0
-+ *
-+ * Copyright 2008-2013 Solarflare Communications Inc.
-+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
-+ */
-+
-+#ifndef CDX_MCDI_H
-+#define CDX_MCDI_H
-+
-+#include <linux/mutex.h>
-+#include <linux/kref.h>
-+#include <linux/rpmsg.h>
-+
-+#include "linux/cdx/bitfield.h"
-+
-+/**
-+ * enum cdx_mcdi_mode - MCDI transaction mode
-+ * @MCDI_MODE_EVENTS: wait for an mcdi response callback.
-+ * @MCDI_MODE_FAIL: we think MCDI is dead, so fail-fast all calls
-+ */
-+enum cdx_mcdi_mode {
-+	MCDI_MODE_EVENTS,
-+	MCDI_MODE_FAIL,
-+};
-+
-+#define MCDI_RPC_TIMEOUT	(10 * HZ)
-+#define MCDI_RPC_LONG_TIMEOU	(60 * HZ)
-+#define MCDI_RPC_POST_RST_TIME	(10 * HZ)
-+
-+/**
-+ * enum cdx_mcdi_cmd_state - State for an individual MCDI command
-+ * @MCDI_STATE_QUEUED: Command not started and is waiting to run.
-+ * @MCDI_STATE_RETRY: Command was submitted and MC rejected with no resources,
-+ *	as MC have too many outstanding commands. Command will be retried once
-+ *	another command returns.
-+ * @MCDI_STATE_RUNNING: Command was accepted and is running.
-+ * @MCDI_STATE_RUNNING_CANCELLED: Command is running but the issuer cancelled
-+ *	the command.
-+ * @MCDI_STATE_FINISHED: Processing of this command has completed.
-+ */
-+
-+enum cdx_mcdi_cmd_state {
-+	MCDI_STATE_QUEUED,
-+	MCDI_STATE_RETRY,
-+	MCDI_STATE_RUNNING,
-+	MCDI_STATE_RUNNING_CANCELLED,
-+	MCDI_STATE_FINISHED,
-+};
-+
-+/**
-+ * struct cdx_mcdi - CDX MCDI Firmware interface, to interact
-+ *	with CDX controller.
-+ * @mcdi: MCDI interface
-+ * @mcdi_ops: MCDI operations
-+ * @r5_rproc : R5 Remoteproc device handle
-+ * @rpdev: RPMsg device
-+ * @ept: RPMsg endpoint
-+ * @work: Post probe work
-+ */
-+struct cdx_mcdi {
-+	/* MCDI interface */
-+	struct cdx_mcdi_data *mcdi;
-+	const struct cdx_mcdi_ops *mcdi_ops;
-+
-+	struct rproc *r5_rproc;
-+	struct rpmsg_device *rpdev;
-+	struct rpmsg_endpoint *ept;
-+	struct work_struct work;
-+};
-+
-+struct cdx_mcdi_ops {
-+	void (*mcdi_request)(struct cdx_mcdi *cdx,
-+			     const struct cdx_dword *hdr, size_t hdr_len,
-+			     const struct cdx_dword *sdu, size_t sdu_len);
-+	unsigned int (*mcdi_rpc_timeout)(struct cdx_mcdi *cdx, unsigned int cmd);
-+};
-+
-+typedef void cdx_mcdi_async_completer(struct cdx_mcdi *cdx,
-+				      unsigned long cookie, int rc,
-+				      struct cdx_dword *outbuf,
-+				      size_t outlen_actual);
-+
-+/**
-+ * struct cdx_mcdi_cmd - An outstanding MCDI command
-+ * @ref: Reference count. There will be one reference if the command is
-+ *	in the mcdi_iface cmd_list, another if it's on a cleanup list,
-+ *	and a third if it's queued in the work queue.
-+ * @list: The data for this entry in mcdi->cmd_list
-+ * @cleanup_list: The data for this entry in a cleanup list
-+ * @work: The work item for this command, queued in mcdi->workqueue
-+ * @mcdi: The mcdi_iface for this command
-+ * @state: The state of this command
-+ * @inlen: inbuf length
-+ * @inbuf: Input buffer
-+ * @quiet: Whether to silence errors
-+ * @reboot_seen: Whether a reboot has been seen during this command,
-+ *	to prevent duplicates
-+ * @seq: Sequence number
-+ * @started: Jiffies this command was started at
-+ * @cookie: Context for completion function
-+ * @completer: Completion function
-+ * @handle: Command handle
-+ * @cmd: Command number
-+ * @rc: Return code
-+ * @outlen: Length of output buffer
-+ * @outbuf: Output buffer
-+ */
-+struct cdx_mcdi_cmd {
-+	struct kref ref;
-+	struct list_head list;
-+	struct list_head cleanup_list;
-+	struct work_struct work;
-+	struct cdx_mcdi_iface *mcdi;
-+	enum cdx_mcdi_cmd_state state;
-+	size_t inlen;
-+	const struct cdx_dword *inbuf;
-+	bool quiet;
-+	bool reboot_seen;
-+	u8 seq;
-+	unsigned long started;
-+	unsigned long cookie;
-+	cdx_mcdi_async_completer *completer;
-+	unsigned int handle;
-+	unsigned int cmd;
-+	int rc;
-+	size_t outlen;
-+	struct cdx_dword *outbuf;
-+	/* followed by inbuf data if necessary */
-+};
-+
-+/**
-+ * struct cdx_mcdi_iface - MCDI protocol context
-+ * @cdx: The associated NIC
-+ * @iface_lock: Serialise access to this structure
-+ * @outstanding_cleanups: Count of cleanups
-+ * @cmd_list: List of outstanding and running commands
-+ * @workqueue: Workqueue used for delayed processing
-+ * @cmd_complete_wq: Waitqueue for command completion
-+ * @db_held_by: Command the MC doorbell is in use by
-+ * @seq_held_by: Command each sequence number is in use by
-+ * @prev_handle: The last used command handle
-+ * @mode: Poll for mcdi completion, or wait for an mcdi_event
-+ * @prev_seq: The last used sequence number
-+ * @new_epoch: Indicates start of day or start of MC reboot recovery
-+ */
-+struct cdx_mcdi_iface {
-+	struct cdx_mcdi *cdx;
-+	/* Serialise access */
-+	struct mutex iface_lock;
-+	unsigned int outstanding_cleanups;
-+	struct list_head cmd_list;
-+	struct workqueue_struct *workqueue;
-+	wait_queue_head_t cmd_complete_wq;
-+	struct cdx_mcdi_cmd *db_held_by;
-+	struct cdx_mcdi_cmd *seq_held_by[16];
-+	unsigned int prev_handle;
-+	enum cdx_mcdi_mode mode;
-+	u8 prev_seq;
-+	bool new_epoch;
-+};
-+
-+/**
-+ * struct cdx_mcdi_data - extra state for NICs that implement MCDI
-+ * @iface: Interface/protocol state
-+ * @fn_flags: Flags for this function, as returned by %MC_CMD_DRV_ATTACH.
-+ */
-+struct cdx_mcdi_data {
-+	struct cdx_mcdi_iface iface;
-+	u32 fn_flags;
-+};
-+
-+void cdx_mcdi_finish(struct cdx_mcdi *cdx);
-+int cdx_mcdi_init(struct cdx_mcdi *cdx);
-+void cdx_mcdi_process_cmd(struct cdx_mcdi *cdx, struct cdx_dword *outbuf, int len);
-+int cdx_mcdi_rpc(struct cdx_mcdi *cdx, unsigned int cmd,
-+		 const struct cdx_dword *inbuf, size_t inlen,
-+		 struct cdx_dword *outbuf, size_t outlen, size_t *outlen_actual);
-+
-+/*
-+ * We expect that 16- and 32-bit fields in MCDI requests and responses
-+ * are appropriately aligned, but 64-bit fields are only
-+ * 32-bit-aligned.
-+ */
-+#define MCDI_DECLARE_BUF(_name, _len) struct cdx_dword _name[DIV_ROUND_UP(_len, 4)] = {{0}}
-+#define _MCDI_PTR(_buf, _offset)					\
-+	((u8 *)(_buf) + (_offset))
-+#define MCDI_PTR(_buf, _field)						\
-+	_MCDI_PTR(_buf, MC_CMD_ ## _field ## _OFST)
-+#define _MCDI_CHECK_ALIGN(_ofst, _align)				\
-+	((void)BUILD_BUG_ON_ZERO((_ofst) & ((_align) - 1)),		\
-+	 (_ofst))
-+#define _MCDI_DWORD(_buf, _field)					\
-+	((_buf) + (_MCDI_CHECK_ALIGN(MC_CMD_ ## _field ## _OFST, 4) >> 2))
-+
-+#define MCDI_SET_DWORD(_buf, _field, _value)				\
-+	CDX_POPULATE_DWORD_1(*_MCDI_DWORD(_buf, _field), CDX_DWORD, _value)
-+#define MCDI_DWORD(_buf, _field)					\
-+	CDX_DWORD_FIELD(*_MCDI_DWORD(_buf, _field), CDX_DWORD)
-+#endif /* CDX_MCDI_H */
-diff --git a/include/linux/ceph/ceph_debug.h b/include/linux/ceph/ceph_debug.h
-index d5a5da838caf..5f904591fa5f 100644
--- a/include/linux/ceph/ceph_debug.h
-+++ b/include/linux/ceph/ceph_debug.h
-@@ -19,12 +19,21 @@
- 	pr_debug("%.*s %12.12s:%-4d : " fmt,				\
- 		 8 - (int)sizeof(KBUILD_MODNAME), "    ",		\
- 		 kbasename(__FILE__), __LINE__, ##__VA_ARGS__)
-+#  define doutc(client, fmt, ...)					\
-+	pr_debug("%.*s %12.12s:%-4d : [%pU %llu] " fmt,			\
-+		 8 - (int)sizeof(KBUILD_MODNAME), "    ",		\
-+		 kbasename(__FILE__), __LINE__,				\
-+		 &client->fsid, client->monc.auth->global_id,		\
-+		 ##__VA_ARGS__)
- # else
- /* faux printk call just to see any compiler warnings. */
-#  define dout(fmt, ...)	do {				\
-		if (0)						\
-			printk(KERN_DEBUG fmt, ##__VA_ARGS__);	\
-	} while (0)
-+#  define dout(fmt, ...)					\
-+		no_printk(KERN_DEBUG fmt, ##__VA_ARGS__)
-+#  define doutc(client, fmt, ...)				\
-+		no_printk(KERN_DEBUG "[%pU %llu] " fmt,		\
-+			  &client->fsid,			\
-+			  client->monc.auth->global_id,		\
-+			  ##__VA_ARGS__)
- # endif
- 
- #else
-@@ -33,7 +42,32 @@
-  * or, just wrap pr_debug
-  */
- # define dout(fmt, ...)	pr_debug(" " fmt, ##__VA_ARGS__)
-+# define doutc(client, fmt, ...)					\
-+	pr_debug(" [%pU %llu] %s: " fmt, &client->fsid,			\
-+		 client->monc.auth->global_id, __func__, ##__VA_ARGS__)
- 
- #endif
- 
-+#define pr_notice_client(client, fmt, ...)				\
-+	pr_notice("[%pU %llu]: " fmt, &client->fsid,			\
-+		  client->monc.auth->global_id, ##__VA_ARGS__)
-+#define pr_info_client(client, fmt, ...)				\
-+	pr_info("[%pU %llu]: " fmt, &client->fsid,			\
-+		client->monc.auth->global_id, ##__VA_ARGS__)
-+#define pr_warn_client(client, fmt, ...)				\
-+	pr_warn("[%pU %llu]: " fmt, &client->fsid,			\
-+		client->monc.auth->global_id, ##__VA_ARGS__)
-+#define pr_warn_once_client(client, fmt, ...)				\
-+	pr_warn_once("[%pU %llu]: " fmt, &client->fsid,			\
-+		     client->monc.auth->global_id, ##__VA_ARGS__)
-+#define pr_err_client(client, fmt, ...)					\
-+	pr_err("[%pU %llu]: " fmt, &client->fsid,			\
-+	       client->monc.auth->global_id, ##__VA_ARGS__)
-+#define pr_warn_ratelimited_client(client, fmt, ...)			\
-+	pr_warn_ratelimited("[%pU %llu]: " fmt, &client->fsid,		\
-+			    client->monc.auth->global_id, ##__VA_ARGS__)
-+#define pr_err_ratelimited_client(client, fmt, ...)			\
-+	pr_err_ratelimited("[%pU %llu]: " fmt, &client->fsid,		\
-+			   client->monc.auth->global_id, ##__VA_ARGS__)
-+
- #endif
-diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h
-index 49586ff26152..c7f2c63b3bc3 100644
--- a/include/linux/ceph/ceph_fs.h
-+++ b/include/linux/ceph/ceph_fs.h
-@@ -357,16 +357,26 @@ enum {
- 	CEPH_MDS_OP_RENAMESNAP = 0x01403,
- };
- 
-extern const char *ceph_mds_op_name(int op);
-+#define IS_CEPH_MDS_OP_NEWINODE(op) (op == CEPH_MDS_OP_CREATE     || \
-+				     op == CEPH_MDS_OP_MKNOD      || \
-+				     op == CEPH_MDS_OP_MKDIR      || \
-+				     op == CEPH_MDS_OP_SYMLINK)
- 
-+extern const char *ceph_mds_op_name(int op);
- 
-#define CEPH_SETATTR_MODE   1
-#define CEPH_SETATTR_UID    2
-#define CEPH_SETATTR_GID    4
-#define CEPH_SETATTR_MTIME  8
-#define CEPH_SETATTR_ATIME 16
-#define CEPH_SETATTR_SIZE  32
-#define CEPH_SETATTR_CTIME 64
-+#define CEPH_SETATTR_MODE              (1 << 0)
-+#define CEPH_SETATTR_UID               (1 << 1)
-+#define CEPH_SETATTR_GID               (1 << 2)
-+#define CEPH_SETATTR_MTIME             (1 << 3)
-+#define CEPH_SETATTR_ATIME             (1 << 4)
-+#define CEPH_SETATTR_SIZE              (1 << 5)
-+#define CEPH_SETATTR_CTIME             (1 << 6)
-+#define CEPH_SETATTR_MTIME_NOW         (1 << 7)
-+#define CEPH_SETATTR_ATIME_NOW         (1 << 8)
-+#define CEPH_SETATTR_BTIME             (1 << 9)
-+#define CEPH_SETATTR_KILL_SGUID        (1 << 10)
-+#define CEPH_SETATTR_FSCRYPT_AUTH      (1 << 11)
-+#define CEPH_SETATTR_FSCRYPT_FILE      (1 << 12)
- 
- /*
-  * Ceph setxattr request flags.
-@@ -479,7 +489,7 @@ union ceph_mds_request_args_ext {
- #define CEPH_MDS_FLAG_WANT_DENTRY	2 /* want dentry in reply */
- #define CEPH_MDS_FLAG_ASYNC		4 /* request is asynchronous */
- 
-struct ceph_mds_request_head_old {
-+struct ceph_mds_request_head_legacy {
- 	__le64 oldest_client_tid;
- 	__le32 mdsmap_epoch;           /* on client */
- 	__le32 flags;                  /* CEPH_MDS_FLAG_* */
-@@ -492,20 +502,26 @@ struct ceph_mds_request_head_old {
- 	union ceph_mds_request_args args;
- } __attribute__ ((packed));
- 
-#define CEPH_MDS_REQUEST_HEAD_VERSION  1
-+#define CEPH_MDS_REQUEST_HEAD_VERSION  3
- 
- struct ceph_mds_request_head {
- 	__le16 version;                /* struct version */
- 	__le64 oldest_client_tid;
- 	__le32 mdsmap_epoch;           /* on client */
- 	__le32 flags;                  /* CEPH_MDS_FLAG_* */
-	__u8 num_retry, num_fwd;       /* count retry, fwd attempts */
-+	__u8 num_retry, num_fwd;       /* legacy count retry and fwd attempts */
- 	__le16 num_releases;           /* # include cap/lease release records */
- 	__le32 op;                     /* mds op code */
- 	__le32 caller_uid, caller_gid;
- 	__le64 ino;                    /* use this ino for openc, mkdir, mknod,
- 					  etc. (if replaying) */
- 	union ceph_mds_request_args_ext args;
-+
-+	__le32 ext_num_retry;          /* new count retry attempts */
-+	__le32 ext_num_fwd;            /* new count fwd attempts */
-+
-+	__le32 struct_len;             /* to store size of struct ceph_mds_request_head */
-+	__le32 owner_uid, owner_gid;   /* used for OPs which create inodes */
- } __attribute__ ((packed));
- 
- /* cap/lease release record */
-@@ -778,7 +794,7 @@ struct ceph_mds_caps {
- 
- struct ceph_mds_cap_peer {
- 	__le64 cap_id;
-	__le32 seq;
-+	__le32 issue_seq;
- 	__le32 mseq;
- 	__le32 mds;
- 	__u8   flags;
-@@ -792,7 +808,7 @@ struct ceph_mds_cap_release {
- struct ceph_mds_cap_item {
- 	__le64 ino;
- 	__le64 cap_id;
-	__le32 migrate_seq, seq;
-+	__le32 migrate_seq, issue_seq;
- } __attribute__ ((packed));
- 
- #define CEPH_MDS_LEASE_REVOKE           1  /*    mds  -> client */
-diff --git a/include/linux/ceph/decode.h b/include/linux/ceph/decode.h
-index 04f3ace5787b..8fc1aed64113 100644
--- a/include/linux/ceph/decode.h
-+++ b/include/linux/ceph/decode.h
-@@ -6,7 +6,7 @@
- #include <linux/bug.h>
- #include <linux/slab.h>
- #include <linux/time.h>
-#include <asm/unaligned.h>
-+#include <linux/unaligned.h>
- 
- #include <linux/ceph/types.h>
- 
-diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h
-index 00af2c98da75..63e0e2aa1ce9 100644
--- a/include/linux/ceph/libceph.h
-+++ b/include/linux/ceph/libceph.h
-@@ -4,7 +4,7 @@
- 
- #include <linux/ceph/ceph_debug.h>
- 
-#include <asm/unaligned.h>
-+#include <linux/unaligned.h>
- #include <linux/backing-dev.h>
- #include <linux/completion.h>
- #include <linux/exportfs.h>
-@@ -99,16 +99,6 @@ struct ceph_options {
- 
- #define CEPH_AUTH_NAME_DEFAULT   "guest"
- 
-/* mount state */
-enum {
-	CEPH_MOUNT_MOUNTING,
-	CEPH_MOUNT_MOUNTED,
-	CEPH_MOUNT_UNMOUNTING,
-	CEPH_MOUNT_UNMOUNTED,
-	CEPH_MOUNT_SHUTDOWN,
-	CEPH_MOUNT_RECOVER,
-};
-
- static inline unsigned long ceph_timeout_jiffies(unsigned long timeout)
- {
- 	return timeout ?: MAX_SCHEDULE_TIMEOUT;
-@@ -316,8 +306,7 @@ struct ceph_entity_addr *ceph_client_addr(struct ceph_client *client);
- u64 ceph_client_gid(struct ceph_client *client);
- extern void ceph_destroy_client(struct ceph_client *client);
- extern void ceph_reset_client_addr(struct ceph_client *client);
-extern int __ceph_open_session(struct ceph_client *client,
-			       unsigned long started);
-+extern int __ceph_open_session(struct ceph_client *client);
- extern int ceph_open_session(struct ceph_client *client);
- int ceph_wait_for_latest_osdmap(struct ceph_client *client,
- 				unsigned long timeout);
-@@ -327,12 +316,6 @@ extern void ceph_release_page_vector(struct page **pages, int num_pages);
- extern void ceph_put_page_vector(struct page **pages, int num_pages,
- 				 bool dirty);
- extern struct page **ceph_alloc_page_vector(int num_pages, gfp_t flags);
-extern int ceph_copy_user_to_page_vector(struct page **pages,
-					 const void __user *data,
-					 loff_t off, size_t len);
-extern void ceph_copy_to_page_vector(struct page **pages,
-				    const void *data,
-				    loff_t off, size_t len);
- extern void ceph_copy_from_page_vector(struct page **pages,
- 				    void *data,
- 				    loff_t off, size_t len);
-diff --git a/include/linux/ceph/mdsmap.h b/include/linux/ceph/mdsmap.h
-deleted file mode 100644
-index 4c3e0648dc27..000000000000
--- a/include/linux/ceph/mdsmap.h
-+++ /dev/null
-@@ -1,72 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _FS_CEPH_MDSMAP_H
-#define _FS_CEPH_MDSMAP_H
-
-#include <linux/bug.h>
-#include <linux/ceph/types.h>
-
-/*
- * mds map - describe servers in the mds cluster.
- *
- * we limit fields to those the client actually xcares about
- */
-struct ceph_mds_info {
-	u64 global_id;
-	struct ceph_entity_addr addr;
-	s32 state;
-	int num_export_targets;
-	bool laggy;
-	u32 *export_targets;
-};
-
-struct ceph_mdsmap {
-	u32 m_epoch, m_client_epoch, m_last_failure;
-	u32 m_root;
-	u32 m_session_timeout;          /* seconds */
-	u32 m_session_autoclose;        /* seconds */
-	u64 m_max_file_size;
-	u64 m_max_xattr_size;		/* maximum size for xattrs blob */
-	u32 m_max_mds;			/* expected up:active mds number */
-	u32 m_num_active_mds;		/* actual up:active mds number */
-	u32 possible_max_rank;		/* possible max rank index */
-	struct ceph_mds_info *m_info;
-
-	/* which object pools file data can be stored in */
-	int m_num_data_pg_pools;
-	u64 *m_data_pg_pools;
-	u64 m_cas_pg_pool;
-
-	bool m_enabled;
-	bool m_damaged;
-	int m_num_laggy;
-};
-
-static inline struct ceph_entity_addr *
-ceph_mdsmap_get_addr(struct ceph_mdsmap *m, int w)
-{
-	if (w >= m->possible_max_rank)
-		return NULL;
-	return &m->m_info[w].addr;
-}
-
-static inline int ceph_mdsmap_get_state(struct ceph_mdsmap *m, int w)
-{
-	BUG_ON(w < 0);
-	if (w >= m->possible_max_rank)
-		return CEPH_MDS_STATE_DNE;
-	return m->m_info[w].state;
-}
-
-static inline bool ceph_mdsmap_is_laggy(struct ceph_mdsmap *m, int w)
-{
-	if (w >= 0 && w < m->possible_max_rank)
-		return m->m_info[w].laggy;
-	return false;
-}
-
-extern int ceph_mdsmap_get_random_mds(struct ceph_mdsmap *m);
-struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end, bool msgr2);
-extern void ceph_mdsmap_destroy(struct ceph_mdsmap *m);
-extern bool ceph_mdsmap_is_cluster_available(struct ceph_mdsmap *m);
-
-#endif
-diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h
-index e7f2fb2fc207..6aa4c6478c9f 100644
--- a/include/linux/ceph/messenger.h
-+++ b/include/linux/ceph/messenger.h
-@@ -2,6 +2,7 @@
- #ifndef __FS_CEPH_MESSENGER_H
- #define __FS_CEPH_MESSENGER_H
- 
-+#include <crypto/sha2.h>
- #include <linux/bvec.h>
- #include <linux/crypto.h>
- #include <linux/kref.h>
-@@ -17,6 +18,7 @@
- 
- struct ceph_msg;
- struct ceph_connection;
-+struct ceph_msg_data_cursor;
- 
- /*
-  * Ceph defines these callbacks for handling connection events.
-@@ -70,6 +72,30 @@ struct ceph_connection_operations {
- 				      int used_proto, int result,
- 				      const int *allowed_protos, int proto_cnt,
- 				      const int *allowed_modes, int mode_cnt);
-+
-+	/**
-+	 * sparse_read: read sparse data
-+	 * @con: connection we're reading from
-+	 * @cursor: data cursor for reading extents
-+	 * @buf: optional buffer to read into
-+	 *
-+	 * This should be called more than once, each time setting up to
-+	 * receive an extent into the current cursor position, and zeroing
-+	 * the holes between them.
-+	 *
-+	 * Returns amount of data to be read (in bytes), 0 if reading is
-+	 * complete, or -errno if there was an error.
-+	 *
-+	 * If @buf is set on a >0 return, then the data should be read into
-+	 * the provided buffer. Otherwise, it should be read into the cursor.
-+	 *
-+	 * The sparse read operation is expected to initialize the cursor
-+	 * with a length covering up to the end of the last extent.
-+	 */
-+	int (*sparse_read)(struct ceph_connection *con,
-+			   struct ceph_msg_data_cursor *cursor,
-+			   char **buf);
-+
- };
- 
- /* use format string %s%lld */
-@@ -98,6 +124,7 @@ enum ceph_msg_data_type {
- 	CEPH_MSG_DATA_BIO,	/* data source/destination is a bio list */
- #endif /* CONFIG_BLOCK */
- 	CEPH_MSG_DATA_BVECS,	/* data source/destination is a bio_vec array */
-+	CEPH_MSG_DATA_ITER,	/* data source/destination is an iov_iter */
- };
- 
- #ifdef CONFIG_BLOCK
-@@ -199,6 +226,7 @@ struct ceph_msg_data {
- 			bool		own_pages;
- 		};
- 		struct ceph_pagelist	*pagelist;
-+		struct iov_iter		iter;
- 	};
- };
- 
-@@ -207,7 +235,7 @@ struct ceph_msg_data_cursor {
- 
- 	struct ceph_msg_data	*data;		/* current data item */
- 	size_t			resid;		/* bytes not yet consumed */
-	bool			last_piece;	/* current is last piece */
-+	int			sr_resid;	/* residual sparse_read len */
- 	bool			need_crc;	/* crc update needed */
- 	union {
- #ifdef CONFIG_BLOCK
-@@ -223,6 +251,10 @@ struct ceph_msg_data_cursor {
- 			struct page	*page;		/* page from list */
- 			size_t		offset;		/* bytes from list */
- 		};
-+		struct {
-+			struct iov_iter		iov_iter;
-+			unsigned int		lastlen;
-+		};
- 	};
- };
- 
-@@ -252,6 +284,7 @@ struct ceph_msg {
- 	struct kref kref;
- 	bool more_to_follow;
- 	bool needs_out_seq;
-+	u64 sparse_read_total;
- 	int front_alloc_len;
- 
- 	struct ceph_msgpool *pool;
-@@ -310,6 +343,10 @@ struct ceph_connection_v1_info {
- 
- 	int in_base_pos;     /* bytes read */
- 
-+	/* sparse reads */
-+	struct kvec in_sr_kvec; /* current location to receive into */
-+	u64 in_sr_len;		/* amount of data in this extent */
-+
- 	/* message in temps */
- 	u8 in_tag;           /* protocol control byte */
- 	struct ceph_msg_header in_hdr;
-@@ -376,7 +413,8 @@ struct ceph_connection_v2_info {
- 	struct ceph_msg_data_cursor in_cursor;
- 	struct ceph_msg_data_cursor out_cursor;
- 
-	struct crypto_shash *hmac_tfm;  /* post-auth signature */
-+	struct hmac_sha256_key hmac_key;  /* post-auth signature */
-+	bool hmac_key_set;
- 	struct crypto_aead *gcm_tfm;  /* on-wire encryption */
- 	struct aead_request *gcm_req;
- 	struct crypto_wait gcm_wait;
-@@ -396,6 +434,7 @@ struct ceph_connection_v2_info {
- 
- 	void *conn_bufs[16];
- 	int conn_buf_cnt;
-+	int data_len_remain;
- 
- 	struct kvec in_sign_kvecs[8];
- 	struct kvec out_sign_kvecs[8];
-@@ -498,8 +537,7 @@ void ceph_con_discard_requeued(struct ceph_connection *con, u64 reconnect_seq);
- void ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor,
- 			       struct ceph_msg *msg, size_t length);
- struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor,
-				size_t *page_offset, size_t *length,
-				bool *last_piece);
-+				size_t *page_offset, size_t *length);
- void ceph_msg_data_advance(struct ceph_msg_data_cursor *cursor, size_t bytes);
- 
- u32 ceph_crc32c_page(u32 crc, struct page *page, unsigned int page_offset,
-@@ -512,12 +550,12 @@ void ceph_addr_set_port(struct ceph_entity_addr *addr, int p);
- void ceph_con_process_message(struct ceph_connection *con);
- int ceph_con_in_msg_alloc(struct ceph_connection *con,
- 			  struct ceph_msg_header *hdr, int *skip);
-void ceph_con_get_out_msg(struct ceph_connection *con);
-+struct ceph_msg *ceph_con_get_out_msg(struct ceph_connection *con);
- 
- /* messenger_v1.c */
- int ceph_con_v1_try_read(struct ceph_connection *con);
- int ceph_con_v1_try_write(struct ceph_connection *con);
-void ceph_con_v1_revoke(struct ceph_connection *con);
-+void ceph_con_v1_revoke(struct ceph_connection *con, struct ceph_msg *msg);
- void ceph_con_v1_revoke_incoming(struct ceph_connection *con);
- bool ceph_con_v1_opened(struct ceph_connection *con);
- void ceph_con_v1_reset_session(struct ceph_connection *con);
-@@ -526,7 +564,7 @@ void ceph_con_v1_reset_protocol(struct ceph_connection *con);
- /* messenger_v2.c */
- int ceph_con_v2_try_read(struct ceph_connection *con);
- int ceph_con_v2_try_write(struct ceph_connection *con);
-void ceph_con_v2_revoke(struct ceph_connection *con);
-+void ceph_con_v2_revoke(struct ceph_connection *con, struct ceph_msg *msg);
- void ceph_con_v2_revoke_incoming(struct ceph_connection *con);
- bool ceph_con_v2_opened(struct ceph_connection *con);
- void ceph_con_v2_reset_session(struct ceph_connection *con);
-@@ -575,6 +613,8 @@ void ceph_msg_data_add_bio(struct ceph_msg *msg, struct ceph_bio_iter *bio_pos,
- #endif /* CONFIG_BLOCK */
- void ceph_msg_data_add_bvecs(struct ceph_msg *msg,
- 			     struct ceph_bvec_iter *bvec_pos);
-+void ceph_msg_data_add_iter(struct ceph_msg *msg,
-+			    struct iov_iter *iter);
- 
- struct ceph_msg *ceph_msg_new2(int type, int front_len, int max_data_items,
- 			       gfp_t flags, bool can_fail);
-diff --git a/include/linux/ceph/mon_client.h b/include/linux/ceph/mon_client.h
-index b658961156a0..7a9a40163c0f 100644
--- a/include/linux/ceph/mon_client.h
-+++ b/include/linux/ceph/mon_client.h
-@@ -19,7 +19,7 @@ struct ceph_monmap {
- 	struct ceph_fsid fsid;
- 	u32 epoch;
- 	u32 num_mon;
-	struct ceph_entity_inst mon_inst[];
-+	struct ceph_entity_inst mon_inst[] __counted_by(num_mon);
- };
- 
- struct ceph_mon_client;
-diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h
-index fb6be72104df..50b14a5661c7 100644
--- a/include/linux/ceph/osd_client.h
-+++ b/include/linux/ceph/osd_client.h
-@@ -29,14 +29,63 @@ typedef void (*ceph_osdc_callback_t)(struct ceph_osd_request *);
- 
- #define CEPH_HOMELESS_OSD	-1
- 
-/* a given osd we're communicating with */
-+/*
-+ * A single extent in a SPARSE_READ reply.
-+ *
-+ * Note that these come from the OSD as little-endian values. On BE arches,
-+ * we convert them in-place after receipt.
-+ */
-+struct ceph_sparse_extent {
-+	u64	off;
-+	u64	len;
-+} __packed;
-+
-+/* Sparse read state machine state values */
-+enum ceph_sparse_read_state {
-+	CEPH_SPARSE_READ_HDR	= 0,
-+	CEPH_SPARSE_READ_EXTENTS,
-+	CEPH_SPARSE_READ_DATA_LEN,
-+	CEPH_SPARSE_READ_DATA_PRE,
-+	CEPH_SPARSE_READ_DATA,
-+};
-+
-+/*
-+ * A SPARSE_READ reply is a 32-bit count of extents, followed by an array of
-+ * 64-bit offset/length pairs, and then all of the actual file data
-+ * concatenated after it (sans holes).
-+ *
-+ * Unfortunately, we don't know how long the extent array is until we've
-+ * started reading the data section of the reply. The caller should send down
-+ * a destination buffer for the array, but we'll alloc one if it's too small
-+ * or if the caller doesn't.
-+ */
-+struct ceph_sparse_read {
-+	enum ceph_sparse_read_state	sr_state;    /* state machine state */
-+	u64				sr_req_off;  /* orig request offset */
-+	u64				sr_req_len;  /* orig request length */
-+	u64				sr_pos;      /* current pos in buffer */
-+	int				sr_index;    /* current extent index */
-+	u32				sr_datalen;  /* length of actual data */
-+	u32				sr_count;    /* extent count in reply */
-+	int				sr_ext_len;  /* length of extent array */
-+	struct ceph_sparse_extent	*sr_extent;  /* extent array */
-+};
-+
-+/*
-+ * A given osd we're communicating with.
-+ *
-+ * Note that the o_requests tree can be searched while holding the "lock" mutex
-+ * or the "o_requests_lock" spinlock. Insertion or removal requires both!
-+ */
- struct ceph_osd {
- 	refcount_t o_ref;
-+	int o_sparse_op_idx;
- 	struct ceph_osd_client *o_osdc;
- 	int o_osd;
- 	int o_incarnation;
- 	struct rb_node o_node;
- 	struct ceph_connection o_con;
-+	spinlock_t o_requests_lock;
- 	struct rb_root o_requests;
- 	struct rb_root o_linger_requests;
- 	struct rb_root o_backoff_mappings;
-@@ -46,6 +95,7 @@ struct ceph_osd {
- 	unsigned long lru_ttl;
- 	struct list_head o_keepalive_item;
- 	struct mutex lock;
-+	struct ceph_sparse_read	o_sparse_read;
- };
- 
- #define CEPH_OSD_SLAB_OPS	2
-@@ -59,6 +109,7 @@ enum ceph_osd_data_type {
- 	CEPH_OSD_DATA_TYPE_BIO,
- #endif /* CONFIG_BLOCK */
- 	CEPH_OSD_DATA_TYPE_BVECS,
-+	CEPH_OSD_DATA_TYPE_ITER,
- };
- 
- struct ceph_osd_data {
-@@ -82,6 +133,7 @@ struct ceph_osd_data {
- 			struct ceph_bvec_iter	bvec_pos;
- 			u32			num_bvecs;
- 		};
-+		struct iov_iter		iter;
- 	};
- };
- 
-@@ -98,6 +150,8 @@ struct ceph_osd_req_op {
- 			u64 offset, length;
- 			u64 truncate_size;
- 			u32 truncate_seq;
-+			int sparse_ext_cnt;
-+			struct ceph_sparse_extent *sparse_ext;
- 			struct ceph_osd_data osd_data;
- 		} extent;
- 		struct {
-@@ -145,6 +199,9 @@ struct ceph_osd_req_op {
- 			u32 src_fadvise_flags;
- 			struct ceph_osd_data osd_data;
- 		} copy_from;
-+		struct {
-+			u64 ver;
-+		} assert_ver;
- 	};
- };
- 
-@@ -199,6 +256,7 @@ struct ceph_osd_request {
- 	struct ceph_osd_client *r_osdc;
- 	struct kref       r_kref;
- 	bool              r_mempool;
-+	bool		  r_linger;           /* don't resend on failure */
- 	struct completion r_completion;       /* private to osd_client.c */
- 	ceph_osdc_callback_t r_callback;
- 
-@@ -211,9 +269,9 @@ struct ceph_osd_request {
- 	struct ceph_snap_context *r_snapc;    /* for writes */
- 	struct timespec64 r_mtime;            /* ditto */
- 	u64 r_data_offset;                    /* ditto */
-	bool r_linger;                        /* don't resend on failure */
- 
- 	/* internal */
-+	u64 r_version;			      /* data version sent in reply */
- 	unsigned long r_stamp;                /* jiffies, send or check time */
- 	unsigned long r_start_stamp;          /* jiffies */
- 	ktime_t r_start_latency;              /* ktime_t */
-@@ -221,7 +279,7 @@ struct ceph_osd_request {
- 	int r_attempts;
- 	u32 r_map_dne_bound;
- 
-	struct ceph_osd_req_op r_ops[];
-+	struct ceph_osd_req_op r_ops[] __counted_by(r_num_ops);
- };
- 
- struct ceph_request_redirect {
-@@ -391,8 +449,6 @@ extern int ceph_osdc_init(struct ceph_osd_client *osdc,
- extern void ceph_osdc_stop(struct ceph_osd_client *osdc);
- extern void ceph_osdc_reopen_osds(struct ceph_osd_client *osdc);
- 
-extern void ceph_osdc_handle_reply(struct ceph_osd_client *osdc,
-				   struct ceph_msg *msg);
- extern void ceph_osdc_handle_map(struct ceph_osd_client *osdc,
- 				 struct ceph_msg *msg);
- void ceph_osdc_update_epoch_barrier(struct ceph_osd_client *osdc, u32 eb);
-@@ -434,9 +490,6 @@ extern void osd_req_op_extent_osd_data_pages(struct ceph_osd_request *,
- 					struct page **pages, u64 length,
- 					u32 alignment, bool pages_from_pool,
- 					bool own_pages);
-extern void osd_req_op_extent_osd_data_pagelist(struct ceph_osd_request *,
-					unsigned int which,
-					struct ceph_pagelist *pagelist);
- #ifdef CONFIG_BLOCK
- void osd_req_op_extent_osd_data_bio(struct ceph_osd_request *osd_req,
- 				    unsigned int which,
-@@ -450,10 +503,9 @@ void osd_req_op_extent_osd_data_bvecs(struct ceph_osd_request *osd_req,
- void osd_req_op_extent_osd_data_bvec_pos(struct ceph_osd_request *osd_req,
- 					 unsigned int which,
- 					 struct ceph_bvec_iter *bvec_pos);
-+void osd_req_op_extent_osd_iter(struct ceph_osd_request *osd_req,
-+				unsigned int which, struct iov_iter *iter);
- 
-extern void osd_req_op_cls_request_data_pagelist(struct ceph_osd_request *,
-					unsigned int which,
-					struct ceph_pagelist *pagelist);
- extern void osd_req_op_cls_request_data_pages(struct ceph_osd_request *,
- 					unsigned int which,
- 					struct page **pages, u64 length,
-@@ -504,6 +556,23 @@ extern struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *,
- 				      u32 truncate_seq, u64 truncate_size,
- 				      bool use_mempool);
- 
-+int __ceph_alloc_sparse_ext_map(struct ceph_osd_req_op *op, int cnt);
-+
-+/*
-+ * How big an extent array should we preallocate for a sparse read? This is
-+ * just a starting value.  If we get more than this back from the OSD, the
-+ * receiver will reallocate.
-+ */
-+#define CEPH_SPARSE_EXT_ARRAY_INITIAL  16
-+
-+static inline int ceph_alloc_sparse_ext_map(struct ceph_osd_req_op *op, int cnt)
-+{
-+	if (!cnt)
-+		cnt = CEPH_SPARSE_EXT_ARRAY_INITIAL;
-+
-+	return __ceph_alloc_sparse_ext_map(op, cnt);
-+}
-+
- extern void ceph_osdc_get_request(struct ceph_osd_request *req);
- extern void ceph_osdc_put_request(struct ceph_osd_request *req);
- 
-@@ -551,12 +620,24 @@ int ceph_osdc_notify(struct ceph_osd_client *osdc,
- 		     u32 timeout,
- 		     struct page ***preply_pages,
- 		     size_t *preply_len);
-int ceph_osdc_watch_check(struct ceph_osd_client *osdc,
-			  struct ceph_osd_linger_request *lreq);
- int ceph_osdc_list_watchers(struct ceph_osd_client *osdc,
- 			    struct ceph_object_id *oid,
- 			    struct ceph_object_locator *oloc,
- 			    struct ceph_watch_item **watchers,
- 			    u32 *num_watchers);
-#endif
- 
-+/* Find offset into the buffer of the end of the extent map */
-+static inline u64 ceph_sparse_ext_map_end(struct ceph_osd_req_op *op)
-+{
-+	struct ceph_sparse_extent *ext;
-+
-+	/* No extents? No data */
-+	if (op->extent.sparse_ext_cnt == 0)
-+		return 0;
-+
-+	ext = &op->extent.sparse_ext[op->extent.sparse_ext_cnt - 1];
-+
-+	return ext->off + ext->len - op->extent.offset;
-+}
-+
-+#endif
-diff --git a/include/linux/ceph/pagelist.h b/include/linux/ceph/pagelist.h
-index 5dead8486fd8..879bec0863aa 100644
--- a/include/linux/ceph/pagelist.h
-+++ b/include/linux/ceph/pagelist.h
-@@ -17,12 +17,6 @@ struct ceph_pagelist {
- 	refcount_t refcnt;
- };
- 
-struct ceph_pagelist_cursor {
-	struct ceph_pagelist *pl;   /* pagelist, for error checking */
-	struct list_head *page_lru; /* page in list */
-	size_t room;		    /* room remaining to reset to */
-};
-
- struct ceph_pagelist *ceph_pagelist_alloc(gfp_t gfp_flags);
- 
- extern void ceph_pagelist_release(struct ceph_pagelist *pl);
-@@ -33,12 +27,6 @@ extern int ceph_pagelist_reserve(struct ceph_pagelist *pl, size_t space);
- 
- extern int ceph_pagelist_free_reserve(struct ceph_pagelist *pl);
- 
-extern void ceph_pagelist_set_cursor(struct ceph_pagelist *pl,
-				     struct ceph_pagelist_cursor *c);
-
-extern int ceph_pagelist_truncate(struct ceph_pagelist *pl,
-				  struct ceph_pagelist_cursor *c);
-
- static inline int ceph_pagelist_encode_64(struct ceph_pagelist *pl, u64 v)
- {
- 	__le64 ev = cpu_to_le64(v);
-diff --git a/include/linux/ceph/rados.h b/include/linux/ceph/rados.h
-index 43a7a1573b51..73c3efbec36c 100644
--- a/include/linux/ceph/rados.h
-+++ b/include/linux/ceph/rados.h
-@@ -524,6 +524,10 @@ struct ceph_osd_op {
- 			__le64 cookie;
- 		} __attribute__ ((packed)) notify;
- 		struct {
-+			__le64 unused;
-+			__le64 ver;
-+		} __attribute__ ((packed)) assert_ver;
-+		struct {
- 			__le64 offset, length;
- 			__le64 src_offset;
- 		} __attribute__ ((packed)) clonerange;
-diff --git a/include/linux/cfag12864b.h b/include/linux/cfag12864b.h
-index 6617d9c68d86..83e6613d12ae 100644
--- a/include/linux/cfag12864b.h
-+++ b/include/linux/cfag12864b.h
-@@ -28,13 +28,6 @@
- extern unsigned char * cfag12864b_buffer;
- 
- /*
- * Get the refresh rate of the LCD
- *
- * Returns the refresh rate (hertz).
- */
-extern unsigned int cfag12864b_getrate(void);
-
-/*
-  * Enable refreshing
-  *
-  * Returns 0 if successful (anyone was using it),
-@@ -50,16 +43,6 @@ extern unsigned char cfag12864b_enable(void);
- extern void cfag12864b_disable(void);
- 
- /*
- * Is enabled refreshing? (is anyone using the module?)
- *
- * Returns 0 if refreshing is not enabled (anyone is using it),
- * or != 0 if refreshing is enabled (someone is using it).
- *
- * Useful for buffer read-only modules.
- */
-extern unsigned char cfag12864b_isenabled(void);
-
-/*
-  * Is the module inited?
-  */
- extern unsigned char cfag12864b_isinited(void);
-diff --git a/include/linux/cfi.h b/include/linux/cfi.h
-index c6dfc1ed0626..1fd22ea6eba4 100644
--- a/include/linux/cfi.h
-+++ b/include/linux/cfi.h
-@@ -2,49 +2,85 @@
- /*
-  * Clang Control Flow Integrity (CFI) support.
-  *
- * Copyright (C) 2021 Google LLC
-+ * Copyright (C) 2022 Google LLC
-  */
- #ifndef _LINUX_CFI_H
- #define _LINUX_CFI_H
- 
-#ifdef CONFIG_CFI_CLANG
-typedef void (*cfi_check_fn)(uint64_t id, void *ptr, void *diag);
-+#include <linux/bug.h>
-+#include <linux/module.h>
-+#include <asm/cfi.h>
- 
-/* Compiler-generated function in each module, and the kernel */
-extern void __cfi_check(uint64_t id, void *ptr, void *diag);
-+#ifdef CONFIG_CFI
-+extern bool cfi_warn;
- 
-+enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr,
-+				      unsigned long *target, u32 type);
-+
-+static inline enum bug_trap_type report_cfi_failure_noaddr(struct pt_regs *regs,
-+							   unsigned long addr)
-+{
-+	return report_cfi_failure(regs, addr, NULL, 0);
-+}
-+
-+#ifndef cfi_get_offset
- /*
- * Force the compiler to generate a CFI jump table entry for a function
- * and store the jump table address to __cfi_jt_<function>.
-+ * Returns the CFI prefix offset. By default, the compiler emits only
-+ * a 4-byte CFI type hash before the function. If an architecture
-+ * uses -fpatchable-function-entry=N,M where M>0 to change the prefix
-+ * offset, they must override this function.
-  */
-#define __CFI_ADDRESSABLE(fn, __attr) \
-	const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn
-+static inline int cfi_get_offset(void)
-+{
-+	return 4;
-+}
-+#endif
- 
-#ifdef CONFIG_CFI_CLANG_SHADOW
-+#ifndef cfi_get_func_hash
-+static inline u32 cfi_get_func_hash(void *func)
-+{
-+	u32 hash;
- 
-extern void cfi_module_add(struct module *mod, unsigned long base_addr);
-extern void cfi_module_remove(struct module *mod, unsigned long base_addr);
-+	if (get_kernel_nofault(hash, func - cfi_get_offset()))
-+		return 0;
- 
-#else
-+	return hash;
-+}
-+#endif
- 
-static inline void cfi_module_add(struct module *mod, unsigned long base_addr) {}
-static inline void cfi_module_remove(struct module *mod, unsigned long base_addr) {}
-+/* CFI type hashes for BPF function types */
-+extern u32 cfi_bpf_hash;
-+extern u32 cfi_bpf_subprog_hash;
- 
-#endif /* CONFIG_CFI_CLANG_SHADOW */
-+#else /* CONFIG_CFI */
- 
-#else /* !CONFIG_CFI_CLANG */
-+static inline int cfi_get_offset(void) { return 0; }
-+static inline u32 cfi_get_func_hash(void *func) { return 0; }
- 
-#ifdef CONFIG_X86_KERNEL_IBT
-+#define cfi_bpf_hash 0U
-+#define cfi_bpf_subprog_hash 0U
- 
-#define __CFI_ADDRESSABLE(fn, __attr) \
-	const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn
-+#endif /* CONFIG_CFI */
- 
-#endif /* CONFIG_X86_KERNEL_IBT */
-+#ifdef CONFIG_ARCH_USES_CFI_TRAPS
-+bool is_cfi_trap(unsigned long addr);
-+#else
-+static inline bool is_cfi_trap(unsigned long addr) { return false; }
-+#endif
- 
-#endif /* CONFIG_CFI_CLANG */
-+#ifdef CONFIG_MODULES
-+#ifdef CONFIG_ARCH_USES_CFI_TRAPS
-+void module_cfi_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
-+			 struct module *mod);
-+#else
-+static inline void module_cfi_finalize(const Elf_Ehdr *hdr,
-+				       const Elf_Shdr *sechdrs,
-+				       struct module *mod) {}
-+#endif /* CONFIG_ARCH_USES_CFI_TRAPS */
-+#endif /* CONFIG_MODULES */
- 
-#ifndef __CFI_ADDRESSABLE
-#define __CFI_ADDRESSABLE(fn, __attr)
-+#ifndef CFI_NOSEAL
-+#define CFI_NOSEAL(x)
- #endif
- 
- #endif /* _LINUX_CFI_H */
-diff --git a/include/linux/cfi_types.h b/include/linux/cfi_types.h
-new file mode 100644
-index 000000000000..a86af9bc8bdc
--- /dev/null
-+++ b/include/linux/cfi_types.h
-@@ -0,0 +1,68 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * Clang Control Flow Integrity (CFI) type definitions.
-+ */
-+#ifndef _LINUX_CFI_TYPES_H
-+#define _LINUX_CFI_TYPES_H
-+
-+#ifdef __ASSEMBLY__
-+#include <linux/linkage.h>
-+
-+#ifdef CONFIG_CFI
-+/*
-+ * Use the __kcfi_typeid_<function> type identifier symbol to
-+ * annotate indirectly called assembly functions. The compiler emits
-+ * these symbols for all address-taken function declarations in C
-+ * code.
-+ */
-+#ifndef __CFI_TYPE
-+#define __CFI_TYPE(name)				\
-+	.4byte __kcfi_typeid_##name
-+#endif
-+
-+#define SYM_TYPED_ENTRY(name, linkage, align...)	\
-+	linkage(name) ASM_NL				\
-+	align ASM_NL					\
-+	__CFI_TYPE(name) ASM_NL				\
-+	name:
-+
-+#define SYM_TYPED_START(name, linkage, align...)	\
-+	SYM_TYPED_ENTRY(name, linkage, align)
-+
-+#else /* CONFIG_CFI */
-+
-+#define SYM_TYPED_START(name, linkage, align...)	\
-+	SYM_START(name, linkage, align)
-+
-+#endif /* CONFIG_CFI */
-+
-+#ifndef SYM_TYPED_FUNC_START
-+#define SYM_TYPED_FUNC_START(name) 			\
-+	SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN)
-+#endif
-+
-+#else /* __ASSEMBLY__ */
-+
-+#ifdef CONFIG_CFI
-+#define DEFINE_CFI_TYPE(name, func)						\
-+	/*									\
-+	 * Force a reference to the function so the compiler generates		\
-+	 * __kcfi_typeid_<func>.						\
-+	 */									\
-+	__ADDRESSABLE(func);							\
-+	/* u32 name __ro_after_init = __kcfi_typeid_<func> */			\
-+	extern u32 name;							\
-+	asm (									\
-+	"	.pushsection	.data..ro_after_init,\"aw\",\%progbits	\n"	\
-+	"	.type	" #name ",\%object				\n"	\
-+	"	.globl	" #name "					\n"	\
-+	"	.p2align	2, 0x0					\n"	\
-+	#name ":							\n"	\
-+	"	.4byte	__kcfi_typeid_" #func "				\n"	\
-+	"	.size	" #name ", 4					\n"	\
-+	"	.popsection						\n"	\
-+	);
-+#endif
-+
-+#endif /* __ASSEMBLY__ */
-+#endif /* _LINUX_CFI_TYPES_H */
-diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h
-index 4bcf56b3491c..b760a3c470a5 100644
--- a/include/linux/cgroup-defs.h
-+++ b/include/linux/cgroup-defs.h
-@@ -71,9 +71,6 @@ enum {
- 
- 	/* Cgroup is frozen. */
- 	CGRP_FROZEN,
-
-	/* Control group has to be killed. */
-	CGRP_KILL,
- };
- 
- /* cgroup_root->flags */
-@@ -94,6 +91,12 @@ enum {
- 	 * cgroup_threadgroup_rwsem. This makes hot path operations such as
- 	 * forks and exits into the slow path and more expensive.
- 	 *
-+	 * Alleviate the contention between fork, exec, exit operations and
-+	 * writing to cgroup.procs by taking a per threadgroup rwsem instead of
-+	 * the global cgroup_threadgroup_rwsem. Fork and other operations
-+	 * from threads in different thread groups no longer contend with
-+	 * writing to cgroup.procs.
-+	 *
- 	 * The static usage pattern of creating a cgroup, enabling controllers,
- 	 * and then seeding it with CLONE_INTO_CGROUP doesn't require write
- 	 * locking cgroup_threadgroup_rwsem and thus doesn't benefit from
-@@ -115,6 +118,16 @@ enum {
- 	 * Enable recursive subtree protection
- 	 */
- 	CGRP_ROOT_MEMORY_RECURSIVE_PROT = (1 << 18),
-+
-+	/*
-+	 * Enable hugetlb accounting for the memory controller.
-+	 */
-+	CGRP_ROOT_MEMORY_HUGETLB_ACCOUNTING = (1 << 19),
-+
-+	/*
-+	 * Enable legacy local pids.events.
-+	 */
-+	CGRP_ROOT_PIDS_LOCAL_EVENTS = (1 << 20),
- };
- 
- /* cftype->flags */
-@@ -126,11 +139,22 @@ enum {
- 	CFTYPE_NO_PREFIX	= (1 << 3),	/* (DON'T USE FOR NEW FILES) no subsys prefix */
- 	CFTYPE_WORLD_WRITABLE	= (1 << 4),	/* (DON'T USE FOR NEW FILES) S_IWUGO */
- 	CFTYPE_DEBUG		= (1 << 5),	/* create when cgroup_debug */
-	CFTYPE_PRESSURE		= (1 << 6),	/* only if pressure feature is enabled */
- 
- 	/* internal flags, do not use outside cgroup core proper */
- 	__CFTYPE_ONLY_ON_DFL	= (1 << 16),	/* only on default hierarchy */
- 	__CFTYPE_NOT_ON_DFL	= (1 << 17),	/* not on default hierarchy */
-+	__CFTYPE_ADDED		= (1 << 18),
-+};
-+
-+enum cgroup_attach_lock_mode {
-+	/* Default */
-+	CGRP_ATTACH_LOCK_GLOBAL,
-+
-+	/* When pid=0 && threadgroup=false, see comments in cgroup_procs_write_start */
-+	CGRP_ATTACH_LOCK_NONE,
-+
-+	/* When favordynmods is on, see comments above CGRP_ROOT_FAVOR_DYNMODS */
-+	CGRP_ATTACH_LOCK_PER_THREADGROUP,
- };
- 
- /*
-@@ -162,13 +186,31 @@ struct cgroup_subsys_state {
- 	/* reference count - access via css_[try]get() and css_put() */
- 	struct percpu_ref refcnt;
- 
-	/* siblings list anchored at the parent's ->children */
-+	/*
-+	 * Depending on the context, this field is initialized
-+	 * via css_rstat_init() at different places:
-+	 *
-+	 * when css is associated with cgroup::self
-+	 *   when css->cgroup is the root cgroup
-+	 *     performed in cgroup_init()
-+	 *   when css->cgroup is not the root cgroup
-+	 *     performed in cgroup_create()
-+	 * when css is associated with a subsystem
-+	 *   when css->cgroup is the root cgroup
-+	 *     performed in cgroup_init_subsys() in the non-early path
-+	 *   when css->cgroup is not the root cgroup
-+	 *     performed in css_create()
-+	 */
-+	struct css_rstat_cpu __percpu *rstat_cpu;
-+
-+	/*
-+	 * siblings list anchored at the parent's ->children
-+	 *
-+	 * linkage is protected by cgroup_mutex or RCU
-+	 */
- 	struct list_head sibling;
- 	struct list_head children;
- 
-	/* flush target list anchored at cgrp->rstat_css_list */
-	struct list_head rstat_css_node;
-
- 	/*
- 	 * PI: Subsys-unique ID.  0 is unused and root is always 1.  The
- 	 * matching css can be looked up using css_from_id().
-@@ -200,6 +242,24 @@ struct cgroup_subsys_state {
- 	 * fields of the containing structure.
- 	 */
- 	struct cgroup_subsys_state *parent;
-+
-+	/*
-+	 * Keep track of total numbers of visible descendant CSSes.
-+	 * The total number of dying CSSes is tracked in
-+	 * css->cgroup->nr_dying_subsys[ssid].
-+	 * Protected by cgroup_mutex.
-+	 */
-+	int nr_descendants;
-+
-+	/*
-+	 * A singly-linked list of css structures to be rstat flushed.
-+	 * This is a scratch field to be used exclusively by
-+	 * css_rstat_flush().
-+	 *
-+	 * Protected by rstat_base_lock when css is cgroup::self.
-+	 * Protected by css->ss->rstat_ss_lock otherwise.
-+	 */
-+	struct cgroup_subsys_state *rstat_flush_next;
- };
- 
- /*
-@@ -238,7 +298,7 @@ struct css_set {
- 	 * Lists running through all tasks using this cgroup group.
- 	 * mg_tasks lists tasks which belong to this cset but are in the
- 	 * process of being migrated out or in.  Protected by
-	 * css_set_rwsem, but, during migration, once tasks are moved to
-+	 * css_set_lock, but, during migration, once tasks are moved to
- 	 * mg_tasks, it can be read safely while holding cgroup_mutex.
- 	 */
- 	struct list_head tasks;
-@@ -305,14 +365,15 @@ struct cgroup_base_stat {
- #ifdef CONFIG_SCHED_CORE
- 	u64 forceidle_sum;
- #endif
-+	u64 ntime;
- };
- 
- /*
-  * rstat - cgroup scalable recursive statistics.  Accounting is done
- * per-cpu in cgroup_rstat_cpu which is then lazily propagated up the
-+ * per-cpu in css_rstat_cpu which is then lazily propagated up the
-  * hierarchy on reads.
-  *
- * When a stat gets updated, the cgroup_rstat_cpu and its ancestors are
-+ * When a stat gets updated, the css_rstat_cpu and its ancestors are
-  * linked into the updated tree.  On the following read, propagation only
-  * considers and consumes the updated tree.  This makes reading O(the
-  * number of descendants which have been active since last read) instead of
-@@ -324,10 +385,26 @@ struct cgroup_base_stat {
-  * frequency decreases the cost of each read.
-  *
-  * This struct hosts both the fields which implement the above -
- * updated_children and updated_next - and the fields which track basic
- * resource statistics on top of it - bsync, bstat and last_bstat.
-+ * updated_children and updated_next.
-+ */
-+struct css_rstat_cpu {
-+	/*
-+	 * Child cgroups with stat updates on this cpu since the last read
-+	 * are linked on the parent's ->updated_children through
-+	 * ->updated_next. updated_children is terminated by its container css.
-+	 */
-+	struct cgroup_subsys_state *updated_children;
-+	struct cgroup_subsys_state *updated_next;	/* NULL if not on the list */
-+
-+	struct llist_node lnode;		/* lockless list for update */
-+	struct cgroup_subsys_state *owner;	/* back pointer */
-+};
-+
-+/*
-+ * This struct hosts the fields which track basic resource statistics on
-+ * top of it - bsync, bstat and last_bstat.
-  */
-struct cgroup_rstat_cpu {
-+struct cgroup_rstat_base_cpu {
- 	/*
- 	 * ->bsync protects ->bstat.  These are the only fields which get
- 	 * updated in the hot path.
-@@ -342,18 +419,18 @@ struct cgroup_rstat_cpu {
- 	struct cgroup_base_stat last_bstat;
- 
- 	/*
-	 * Child cgroups with stat updates on this cpu since the last read
-	 * are linked on the parent's ->updated_children through
-	 * ->updated_next.
-	 *
-	 * In addition to being more compact, singly-linked list pointing
-	 * to the cgroup makes it unnecessary for each per-cpu struct to
-	 * point back to the associated cgroup.
-	 *
-	 * Protected by per-cpu cgroup_rstat_cpu_lock.
-+	 * This field is used to record the cumulative per-cpu time of
-+	 * the cgroup and its descendants. Currently it can be read via
-+	 * eBPF/drgn etc, and we are still trying to determine how to
-+	 * expose it in the cgroupfs interface.
-+	 */
-+	struct cgroup_base_stat subtree_bstat;
-+
-+	/*
-+	 * Snapshots at the last reading. These are used to calculate the
-+	 * deltas to propagate to the per-cpu subtree_bstat.
- 	 */
-	struct cgroup *updated_children;	/* terminated by self cgroup */
-	struct cgroup *updated_next;		/* NULL iff not on the list */
-+	struct cgroup_base_stat last_subtree_bstat;
- };
- 
- struct cgroup_freezer_state {
-@@ -361,7 +438,7 @@ struct cgroup_freezer_state {
- 	bool freeze;
- 
- 	/* Should the cgroup actually be frozen? */
-	int e_freeze;
-+	bool e_freeze;
- 
- 	/* Fields below are protected by css_set_lock */
- 
-@@ -373,6 +450,23 @@ struct cgroup_freezer_state {
- 	 * frozen, SIGSTOPped, and PTRACEd.
- 	 */
- 	int nr_frozen_tasks;
-+
-+	/* Freeze time data consistency protection */
-+	seqcount_spinlock_t freeze_seq;
-+
-+	/*
-+	 * Most recent time the cgroup was requested to freeze.
-+	 * Accesses guarded by freeze_seq counter. Writes serialized
-+	 * by css_set_lock.
-+	 */
-+	u64 freeze_start_nsec;
-+
-+	/*
-+	 * Total duration the cgroup has spent freezing.
-+	 * Accesses guarded by freeze_seq counter. Writes serialized
-+	 * by css_set_lock.
-+	 */
-+	u64 frozen_nsec;
- };
- 
- struct cgroup {
-@@ -384,7 +478,7 @@ struct cgroup {
- 	/*
- 	 * The depth this cgroup is at.  The root is at depth zero and each
- 	 * step down the hierarchy increments the level.  This along with
-	 * ancestor_ids[] can determine whether a given cgroup is a
-+	 * ancestors[] can determine whether a given cgroup is a
- 	 * descendant of another without traversing the hierarchy.
- 	 */
- 	int level;
-@@ -424,10 +518,16 @@ struct cgroup {
- 
- 	int nr_threaded_children;	/* # of live threaded child cgroups */
- 
-+	/* sequence number for cgroup.kill, serialized by css_set_lock. */
-+	unsigned int kill_seq;
-+
- 	struct kernfs_node *kn;		/* cgroup kernfs entry */
- 	struct cgroup_file procs_file;	/* handle for "cgroup.procs" */
- 	struct cgroup_file events_file;	/* handle for "cgroup.events" */
- 
-+	/* handles for "{cpu,memory,io,irq}.pressure" */
-+	struct cgroup_file psi_files[NR_PSI_RESOURCES];
-+
- 	/*
- 	 * The bitmask of subsystems enabled on the child cgroups.
- 	 * ->subtree_control is the one configured through
-@@ -443,6 +543,12 @@ struct cgroup {
- 	/* Private pointers for each registered subsystem */
- 	struct cgroup_subsys_state __rcu *subsys[CGROUP_SUBSYS_COUNT];
- 
-+	/*
-+	 * Keep track of total number of dying CSSes at and below this cgroup.
-+	 * Protected by cgroup_mutex.
-+	 */
-+	int nr_dying_subsys[CGROUP_SUBSYS_COUNT];
-+
- 	struct cgroup_root *root;
- 
- 	/*
-@@ -470,9 +576,23 @@ struct cgroup {
- 	struct cgroup *dom_cgrp;
- 	struct cgroup *old_dom_cgrp;		/* used while enabling threaded */
- 
-	/* per-cpu recursive resource statistics */
-	struct cgroup_rstat_cpu __percpu *rstat_cpu;
-	struct list_head rstat_css_list;
-+	/*
-+	 * Depending on the context, this field is initialized via
-+	 * css_rstat_init() at different places:
-+	 *
-+	 * when cgroup is the root cgroup
-+	 *   performed in cgroup_setup_root()
-+	 * otherwise
-+	 *   performed in cgroup_create()
-+	 */
-+	struct cgroup_rstat_base_cpu __percpu *rstat_base_cpu;
-+
-+	/*
-+	 * Add padding to keep the read mostly rstat per-cpu pointer on a
-+	 * different cacheline than the following *bstat fields which can have
-+	 * frequent updates.
-+	 */
-+	CACHELINE_PADDING(_pad_);
- 
- 	/* cgroup basic resource statistics */
- 	struct cgroup_base_stat last_bstat;
-@@ -498,14 +618,15 @@ struct cgroup {
- 	/* used to store eBPF programs */
- 	struct cgroup_bpf bpf;
- 
-	/* If there is block congestion on this cgroup. */
-	atomic_t congestion_count;
-
- 	/* Used to store internal freezer state */
- 	struct cgroup_freezer_state freezer;
- 
-	/* ids of the ancestors at each level including self */
-	u64 ancestor_ids[];
-+#ifdef CONFIG_BPF_SYSCALL
-+	struct bpf_local_storage __rcu  *bpf_cgrp_storage;
-+#endif
-+
-+	/* All ancestors including self */
-+	struct cgroup *ancestors[];
- };
- 
- /*
-@@ -522,18 +643,23 @@ struct cgroup_root {
- 	/* Unique id for this hierarchy. */
- 	int hierarchy_id;
- 
-	/* The root cgroup.  Root is destroyed on its release. */
-+	/* A list running through the active hierarchies */
-+	struct list_head root_list;
-+	struct rcu_head rcu;	/* Must be near the top */
-+
-+	/*
-+	 * The root cgroup. The containing cgroup_root will be destroyed on its
-+	 * release. cgrp->ancestors[0] will be used overflowing into the
-+	 * following field. cgrp_ancestor_storage must immediately follow.
-+	 */
- 	struct cgroup cgrp;
- 
-	/* for cgrp->ancestor_ids[0] */
-	u64 cgrp_ancestor_id_storage;
-+	/* must follow cgrp for cgrp->ancestors[0], see above */
-+	struct cgroup *cgrp_ancestor_storage;
- 
- 	/* Number of cgroups in the hierarchy, used only for /proc/cgroups */
- 	atomic_t nr_cgrps;
- 
-	/* A list running through the active hierarchies */
-	struct list_head root_list;
-
- 	/* Hierarchy-specific flags */
- 	unsigned int flags;
- 
-@@ -553,9 +679,8 @@ struct cgroup_root {
-  */
- struct cftype {
- 	/*
-	 * By convention, the name should begin with the name of the
-	 * subsystem, followed by a period.  Zero length string indicates
-	 * end of cftype array.
-+	 * Name of the subsystem is prepended in cgroup_file_name().
-+	 * Zero length string indicates end of cftype array.
- 	 */
- 	char name[MAX_CFTYPE_NAME];
- 	unsigned long private;
-@@ -631,9 +756,7 @@ struct cftype {
- 	__poll_t (*poll)(struct kernfs_open_file *of,
- 			 struct poll_table_struct *pt);
- 
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
- 	struct lock_class_key	lockdep_key;
-#endif
- };
- 
- /*
-@@ -647,14 +770,16 @@ struct cgroup_subsys {
- 	void (*css_released)(struct cgroup_subsys_state *css);
- 	void (*css_free)(struct cgroup_subsys_state *css);
- 	void (*css_reset)(struct cgroup_subsys_state *css);
-+	void (*css_killed)(struct cgroup_subsys_state *css);
- 	void (*css_rstat_flush)(struct cgroup_subsys_state *css, int cpu);
- 	int (*css_extra_stat_show)(struct seq_file *seq,
- 				   struct cgroup_subsys_state *css);
-+	int (*css_local_stat_show)(struct seq_file *seq,
-+				   struct cgroup_subsys_state *css);
- 
- 	int (*can_attach)(struct cgroup_taskset *tset);
- 	void (*cancel_attach)(struct cgroup_taskset *tset);
- 	void (*attach)(struct cgroup_taskset *tset);
-	void (*post_attach)(void);
- 	int (*can_fork)(struct task_struct *task,
- 			struct css_set *cset);
- 	void (*cancel_fork)(struct task_struct *task, struct css_set *cset);
-@@ -724,20 +849,32 @@ struct cgroup_subsys {
- 	 * specifies the mask of subsystems that this one depends on.
- 	 */
- 	unsigned int depends_on;
-+
-+	spinlock_t rstat_ss_lock;
-+	struct llist_head __percpu *lhead; /* lockless update list head */
- };
- 
- extern struct percpu_rw_semaphore cgroup_threadgroup_rwsem;
-+extern bool cgroup_enable_per_threadgroup_rwsem;
-+
-+struct cgroup_of_peak {
-+	unsigned long		value;
-+	struct list_head	list;
-+};
- 
- /**
-  * cgroup_threadgroup_change_begin - threadgroup exclusion for cgroups
-  * @tsk: target task
-  *
-  * Allows cgroup operations to synchronize against threadgroup changes
- * using a percpu_rw_semaphore.
-+ * using a global percpu_rw_semaphore and a per threadgroup rw_semaphore when
-+ * favordynmods is on. See the comment above CGRP_ROOT_FAVOR_DYNMODS definition.
-  */
- static inline void cgroup_threadgroup_change_begin(struct task_struct *tsk)
- {
- 	percpu_down_read(&cgroup_threadgroup_rwsem);
-+	if (cgroup_enable_per_threadgroup_rwsem)
-+		down_read(&tsk->signal->cgroup_threadgroup_rwsem);
- }
- 
- /**
-@@ -748,6 +885,8 @@ static inline void cgroup_threadgroup_change_begin(struct task_struct *tsk)
-  */
- static inline void cgroup_threadgroup_change_end(struct task_struct *tsk)
- {
-+	if (cgroup_enable_per_threadgroup_rwsem)
-+		up_read(&tsk->signal->cgroup_threadgroup_rwsem);
- 	percpu_up_read(&cgroup_threadgroup_rwsem);
- }
- 
-@@ -795,14 +934,12 @@ static inline u16 sock_cgroup_prioidx(const struct sock_cgroup_data *skcd)
- #endif
- }
- 
-+#ifdef CONFIG_CGROUP_NET_CLASSID
- static inline u32 sock_cgroup_classid(const struct sock_cgroup_data *skcd)
- {
-#ifdef CONFIG_CGROUP_NET_CLASSID
- 	return READ_ONCE(skcd->classid);
-#else
-	return 0;
-#endif
- }
-+#endif
- 
- static inline void sock_cgroup_set_prioidx(struct sock_cgroup_data *skcd,
- 					   u16 prioidx)
-@@ -812,13 +949,13 @@ static inline void sock_cgroup_set_prioidx(struct sock_cgroup_data *skcd,
- #endif
- }
- 
-+#ifdef CONFIG_CGROUP_NET_CLASSID
- static inline void sock_cgroup_set_classid(struct sock_cgroup_data *skcd,
- 					   u32 classid)
- {
-#ifdef CONFIG_CGROUP_NET_CLASSID
- 	WRITE_ONCE(skcd->classid, classid);
-#endif
- }
-+#endif
- 
- #else	/* CONFIG_SOCK_CGROUP_DATA */
- 
-diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
-index ed53bfe7c46c..bc892e3b37ee 100644
--- a/include/linux/cgroup.h
-+++ b/include/linux/cgroup.h
-@@ -10,8 +10,8 @@
-  */
- 
- #include <linux/sched.h>
-#include <linux/cpumask.h>
- #include <linux/nodemask.h>
-+#include <linux/list.h>
- #include <linux/rculist.h>
- #include <linux/cgroupstats.h>
- #include <linux/fs.h>
-@@ -19,6 +19,7 @@
- #include <linux/kernfs.h>
- #include <linux/jump_label.h>
- #include <linux/types.h>
-+#include <linux/notifier.h>
- #include <linux/ns_common.h>
- #include <linux/nsproxy.h>
- #include <linux/user_namespace.h>
-@@ -26,11 +27,10 @@
- #include <linux/kernel_stat.h>
- 
- #include <linux/cgroup-defs.h>
-+#include <linux/cgroup_namespace.h>
- 
- struct kernel_clone_args;
- 
-#ifdef CONFIG_CGROUPS
-
- /*
-  * All weight knobs on the default hierarchy should use the following min,
-  * default and max values.  The default value is the logarithmic center of
-@@ -40,13 +40,13 @@ struct kernel_clone_args;
- #define CGROUP_WEIGHT_DFL		100
- #define CGROUP_WEIGHT_MAX		10000
- 
-/* walk only threadgroup leaders */
-#define CSS_TASK_ITER_PROCS		(1U << 0)
-/* walk all threaded css_sets in the domain */
-#define CSS_TASK_ITER_THREADED		(1U << 1)
-+#ifdef CONFIG_CGROUPS
- 
-/* internal flags */
-#define CSS_TASK_ITER_SKIPPED		(1U << 16)
-+enum css_task_iter_flags {
-+	CSS_TASK_ITER_PROCS    = (1U << 0),  /* walk only threadgroup leaders */
-+	CSS_TASK_ITER_THREADED = (1U << 1),  /* walk all threaded css_sets in the domain */
-+	CSS_TASK_ITER_SKIPPED  = (1U << 16), /* internal flags */
-+};
- 
- /* a css_task_iter should be treated as an opaque object */
- struct css_task_iter {
-@@ -68,8 +68,16 @@ struct css_task_iter {
- 	struct list_head		iters_node;	/* css_set->task_iters */
- };
- 
-+enum cgroup_lifetime_events {
-+	CGROUP_LIFETIME_ONLINE,
-+	CGROUP_LIFETIME_OFFLINE,
-+};
-+
-+extern struct file_system_type cgroup_fs_type;
- extern struct cgroup_root cgrp_dfl_root;
- extern struct css_set init_css_set;
-+extern spinlock_t css_set_lock;
-+extern struct blocking_notifier_head cgroup_lifetime_notifier;
- 
- #define SUBSYS(_x) extern struct cgroup_subsys _x ## _cgrp_subsys;
- #include <linux/cgroup_subsys.h>
-@@ -106,16 +114,18 @@ struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
- 
- struct cgroup *cgroup_get_from_path(const char *path);
- struct cgroup *cgroup_get_from_fd(int fd);
-+struct cgroup *cgroup_v1v2_get_from_fd(int fd);
- 
- int cgroup_attach_task_all(struct task_struct *from, struct task_struct *);
- int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from);
- 
- int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts);
- int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts);
-+int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts);
- int cgroup_rm_cftypes(struct cftype *cfts);
- void cgroup_file_notify(struct cgroup_file *cfile);
-+void cgroup_file_show(struct cgroup_file *cfile, bool show);
- 
-int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen);
- int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry);
- int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
- 		     struct pid *pid, struct task_struct *tsk);
-@@ -127,9 +137,10 @@ extern void cgroup_cancel_fork(struct task_struct *p,
- 			       struct kernel_clone_args *kargs);
- extern void cgroup_post_fork(struct task_struct *p,
- 			     struct kernel_clone_args *kargs);
-void cgroup_exit(struct task_struct *p);
-void cgroup_release(struct task_struct *p);
-void cgroup_free(struct task_struct *p);
-+void cgroup_task_exit(struct task_struct *p);
-+void cgroup_task_dead(struct task_struct *p);
-+void cgroup_task_release(struct task_struct *p);
-+void cgroup_task_free(struct task_struct *p);
- 
- int cgroup_init_early(void);
- int cgroup_init(void);
-@@ -307,72 +318,25 @@ void css_task_iter_end(struct css_task_iter *it);
-  * Inline functions.
-  */
- 
-+#ifdef CONFIG_DEBUG_CGROUP_REF
-+void css_get(struct cgroup_subsys_state *css);
-+void css_get_many(struct cgroup_subsys_state *css, unsigned int n);
-+bool css_tryget(struct cgroup_subsys_state *css);
-+bool css_tryget_online(struct cgroup_subsys_state *css);
-+void css_put(struct cgroup_subsys_state *css);
-+void css_put_many(struct cgroup_subsys_state *css, unsigned int n);
-+#else
-+#define CGROUP_REF_FN_ATTRS	static inline
-+#define CGROUP_REF_EXPORT(fn)
-+#include <linux/cgroup_refcnt.h>
-+#endif
-+
- static inline u64 cgroup_id(const struct cgroup *cgrp)
- {
- 	return cgrp->kn->id;
- }
- 
- /**
- * css_get - obtain a reference on the specified css
- * @css: target css
- *
- * The caller must already have a reference.
- */
-static inline void css_get(struct cgroup_subsys_state *css)
-{
-	if (!(css->flags & CSS_NO_REF))
-		percpu_ref_get(&css->refcnt);
-}
-
-/**
- * css_get_many - obtain references on the specified css
- * @css: target css
- * @n: number of references to get
- *
- * The caller must already have a reference.
- */
-static inline void css_get_many(struct cgroup_subsys_state *css, unsigned int n)
-{
-	if (!(css->flags & CSS_NO_REF))
-		percpu_ref_get_many(&css->refcnt, n);
-}
-
-/**
- * css_tryget - try to obtain a reference on the specified css
- * @css: target css
- *
- * Obtain a reference on @css unless it already has reached zero and is
- * being released.  This function doesn't care whether @css is on or
- * offline.  The caller naturally needs to ensure that @css is accessible
- * but doesn't have to be holding a reference on it - IOW, RCU protected
- * access is good enough for this function.  Returns %true if a reference
- * count was successfully obtained; %false otherwise.
- */
-static inline bool css_tryget(struct cgroup_subsys_state *css)
-{
-	if (!(css->flags & CSS_NO_REF))
-		return percpu_ref_tryget(&css->refcnt);
-	return true;
-}
-
-/**
- * css_tryget_online - try to obtain a reference on the specified css if online
- * @css: target css
- *
- * Obtain a reference on @css if it's online.  The caller naturally needs
- * to ensure that @css is accessible but doesn't have to be holding a
- * reference on it - IOW, RCU protected access is good enough for this
- * function.  Returns %true if a reference count was successfully obtained;
- * %false otherwise.
- */
-static inline bool css_tryget_online(struct cgroup_subsys_state *css)
-{
-	if (!(css->flags & CSS_NO_REF))
-		return percpu_ref_tryget_live(&css->refcnt);
-	return true;
-}
-
-/**
-  * css_is_dying - test whether the specified css is dying
-  * @css: target css
-  *
-@@ -389,32 +353,23 @@ static inline bool css_tryget_online(struct cgroup_subsys_state *css)
-  */
- static inline bool css_is_dying(struct cgroup_subsys_state *css)
- {
-	return !(css->flags & CSS_NO_REF) && percpu_ref_is_dying(&css->refcnt);
-+	return css->flags & CSS_DYING;
- }
- 
-/**
- * css_put - put a css reference
- * @css: target css
- *
- * Put a reference obtained via css_get() and css_tryget_online().
- */
-static inline void css_put(struct cgroup_subsys_state *css)
-+static inline bool css_is_online(struct cgroup_subsys_state *css)
- {
-	if (!(css->flags & CSS_NO_REF))
-		percpu_ref_put(&css->refcnt);
-+	return css->flags & CSS_ONLINE;
- }
- 
-/**
- * css_put_many - put css references
- * @css: target css
- * @n: number of references to put
- *
- * Put references obtained via css_get() and css_tryget_online().
- */
-static inline void css_put_many(struct cgroup_subsys_state *css, unsigned int n)
-+static inline bool css_is_self(struct cgroup_subsys_state *css)
- {
-	if (!(css->flags & CSS_NO_REF))
-		percpu_ref_put_many(&css->refcnt, n);
-+	if (css == &css->cgroup->self) {
-+		/* cgroup::self should not have subsystem association */
-+		WARN_ON(css->ss != NULL);
-+		return true;
-+	}
-+
-+	return false;
- }
- 
- static inline void cgroup_get(struct cgroup *cgrp)
-@@ -432,6 +387,18 @@ static inline void cgroup_put(struct cgroup *cgrp)
- 	css_put(&cgrp->self);
- }
- 
-+extern struct mutex cgroup_mutex;
-+
-+static inline void cgroup_lock(void)
-+{
-+	mutex_lock(&cgroup_mutex);
-+}
-+
-+static inline void cgroup_unlock(void)
-+{
-+	mutex_unlock(&cgroup_mutex);
-+}
-+
- /**
-  * task_css_set_check - obtain a task's css_set with extra access conditions
-  * @task: the task to obtain css_set for
-@@ -446,8 +413,6 @@ static inline void cgroup_put(struct cgroup *cgrp)
-  * as locks used during the cgroup_subsys::attach() methods.
-  */
- #ifdef CONFIG_PROVE_RCU
-extern struct mutex cgroup_mutex;
-extern spinlock_t css_set_lock;
- #define task_css_set_check(task, __c)					\
- 	rcu_dereference_check((task)->cgroups,				\
- 		rcu_read_lock_sched_held() ||				\
-@@ -574,7 +539,7 @@ static inline bool cgroup_is_descendant(struct cgroup *cgrp,
- {
- 	if (cgrp->root != ancestor->root || cgrp->level < ancestor->level)
- 		return false;
-	return cgrp->ancestor_ids[ancestor->level] == cgroup_id(ancestor);
-+	return cgrp->ancestors[ancestor->level] == ancestor;
- }
- 
- /**
-@@ -591,11 +556,9 @@ static inline bool cgroup_is_descendant(struct cgroup *cgrp,
- static inline struct cgroup *cgroup_ancestor(struct cgroup *cgrp,
- 					     int ancestor_level)
- {
-	if (cgrp->level < ancestor_level)
-+	if (ancestor_level < 0 || ancestor_level > cgrp->level)
- 		return NULL;
-	while (cgrp && cgrp->level > ancestor_level)
-		cgrp = cgroup_parent(cgrp);
-	return cgrp;
-+	return cgrp->ancestors[ancestor_level];
- }
- 
- /**
-@@ -672,11 +635,6 @@ static inline void pr_cont_cgroup_path(struct cgroup *cgrp)
- 	pr_cont_kernfs_path(cgrp->kn);
- }
- 
-static inline struct psi_group *cgroup_psi(struct cgroup *cgrp)
-{
-	return cgrp->psi;
-}
-
- bool cgroup_psi_enabled(void);
- 
- static inline void cgroup_init_kthreadd(void)
-@@ -699,6 +657,7 @@ static inline void cgroup_kthread_ready(void)
- }
- 
- void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen);
-+struct cgroup *__cgroup_get_from_id(u64 id);
- struct cgroup *cgroup_get_from_id(u64 id);
- #else /* !CONFIG_CGROUPS */
- 
-@@ -708,6 +667,8 @@ struct cgroup;
- static inline u64 cgroup_id(const struct cgroup *cgrp) { return 1; }
- static inline void css_get(struct cgroup_subsys_state *css) {}
- static inline void css_put(struct cgroup_subsys_state *css) {}
-+static inline void cgroup_lock(void) {}
-+static inline void cgroup_unlock(void) {}
- static inline int cgroup_attach_task_all(struct task_struct *from,
- 					 struct task_struct *t) { return 0; }
- static inline int cgroupstats_build(struct cgroupstats *stats,
-@@ -720,9 +681,10 @@ static inline void cgroup_cancel_fork(struct task_struct *p,
- 				      struct kernel_clone_args *kargs) {}
- static inline void cgroup_post_fork(struct task_struct *p,
- 				    struct kernel_clone_args *kargs) {}
-static inline void cgroup_exit(struct task_struct *p) {}
-static inline void cgroup_release(struct task_struct *p) {}
-static inline void cgroup_free(struct task_struct *p) {}
-+static inline void cgroup_task_exit(struct task_struct *p) {}
-+static inline void cgroup_task_dead(struct task_struct *p) {}
-+static inline void cgroup_task_release(struct task_struct *p) {}
-+static inline void cgroup_task_free(struct task_struct *p) {}
- 
- static inline int cgroup_init_early(void) { return 0; }
- static inline int cgroup_init(void) { return 0; }
-@@ -734,11 +696,6 @@ static inline struct cgroup *cgroup_parent(struct cgroup *cgrp)
- 	return NULL;
- }
- 
-static inline struct psi_group *cgroup_psi(struct cgroup *cgrp)
-{
-	return NULL;
-}
-
- static inline bool cgroup_psi_enabled(void)
- {
- 	return false;
-@@ -752,22 +709,14 @@ static inline bool task_under_cgroup_hierarchy(struct task_struct *task,
- 
- static inline void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen)
- {}
-
-static inline struct cgroup *cgroup_get_from_id(u64 id)
-{
-	return NULL;
-}
- #endif /* !CONFIG_CGROUPS */
- 
- #ifdef CONFIG_CGROUPS
- /*
-  * cgroup scalable recursive statistics.
-  */
-void cgroup_rstat_updated(struct cgroup *cgrp, int cpu);
-void cgroup_rstat_flush(struct cgroup *cgrp);
-void cgroup_rstat_flush_irqsafe(struct cgroup *cgrp);
-void cgroup_rstat_flush_hold(struct cgroup *cgrp);
-void cgroup_rstat_flush_release(void);
-+void css_rstat_updated(struct cgroup_subsys_state *css, int cpu);
-+void css_rstat_flush(struct cgroup_subsys_state *css);
- 
- /*
-  * Basic resource stats.
-@@ -843,50 +792,6 @@ static inline void cgroup_sk_free(struct sock_cgroup_data *skcd) {}
- 
- #endif	/* CONFIG_CGROUP_DATA */
- 
-struct cgroup_namespace {
-	struct ns_common	ns;
-	struct user_namespace	*user_ns;
-	struct ucounts		*ucounts;
-	struct css_set          *root_cset;
-};
-
-extern struct cgroup_namespace init_cgroup_ns;
-
-#ifdef CONFIG_CGROUPS
-
-void free_cgroup_ns(struct cgroup_namespace *ns);
-
-struct cgroup_namespace *copy_cgroup_ns(unsigned long flags,
-					struct user_namespace *user_ns,
-					struct cgroup_namespace *old_ns);
-
-int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen,
-		   struct cgroup_namespace *ns);
-
-#else /* !CONFIG_CGROUPS */
-
-static inline void free_cgroup_ns(struct cgroup_namespace *ns) { }
-static inline struct cgroup_namespace *
-copy_cgroup_ns(unsigned long flags, struct user_namespace *user_ns,
-	       struct cgroup_namespace *old_ns)
-{
-	return old_ns;
-}
-
-#endif /* !CONFIG_CGROUPS */
-
-static inline void get_cgroup_ns(struct cgroup_namespace *ns)
-{
-	if (ns)
-		refcount_inc(&ns->ns.count);
-}
-
-static inline void put_cgroup_ns(struct cgroup_namespace *ns)
-{
-	if (ns && refcount_dec_and_test(&ns->ns.count))
-		free_cgroup_ns(ns);
-}
-
- #ifdef CONFIG_CGROUPS
- 
- void cgroup_enter_frozen(void);
-@@ -930,4 +835,8 @@ static inline void cgroup_bpf_put(struct cgroup *cgrp) {}
- 
- #endif /* CONFIG_CGROUP_BPF */
- 
-+struct cgroup *task_get_cgroup1(struct task_struct *tsk, int hierarchy_id);
-+
-+struct cgroup_of_peak *of_peak(struct kernfs_open_file *of);
-+
- #endif /* _LINUX_CGROUP_H */
-diff --git a/include/linux/cgroup_dmem.h b/include/linux/cgroup_dmem.h
-new file mode 100644
-index 000000000000..dd4869f1d736
--- /dev/null
-+++ b/include/linux/cgroup_dmem.h
-@@ -0,0 +1,66 @@
-+/* SPDX-License-Identifier: MIT */
-+/*
-+ * Copyright © 2023-2024 Intel Corporation
-+ */
-+
-+#ifndef _CGROUP_DMEM_H
-+#define _CGROUP_DMEM_H
-+
-+#include <linux/types.h>
-+#include <linux/llist.h>
-+
-+struct dmem_cgroup_pool_state;
-+
-+/* Opaque definition of a cgroup region, used internally */
-+struct dmem_cgroup_region;
-+
-+#if IS_ENABLED(CONFIG_CGROUP_DMEM)
-+struct dmem_cgroup_region *dmem_cgroup_register_region(u64 size, const char *name_fmt, ...) __printf(2,3);
-+void dmem_cgroup_unregister_region(struct dmem_cgroup_region *region);
-+int dmem_cgroup_try_charge(struct dmem_cgroup_region *region, u64 size,
-+			   struct dmem_cgroup_pool_state **ret_pool,
-+			   struct dmem_cgroup_pool_state **ret_limit_pool);
-+void dmem_cgroup_uncharge(struct dmem_cgroup_pool_state *pool, u64 size);
-+bool dmem_cgroup_state_evict_valuable(struct dmem_cgroup_pool_state *limit_pool,
-+				      struct dmem_cgroup_pool_state *test_pool,
-+				      bool ignore_low, bool *ret_hit_low);
-+
-+void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool);
-+#else
-+static inline __printf(2,3) struct dmem_cgroup_region *
-+dmem_cgroup_register_region(u64 size, const char *name_fmt, ...)
-+{
-+	return NULL;
-+}
-+
-+static inline void dmem_cgroup_unregister_region(struct dmem_cgroup_region *region)
-+{ }
-+
-+static inline int dmem_cgroup_try_charge(struct dmem_cgroup_region *region, u64 size,
-+					 struct dmem_cgroup_pool_state **ret_pool,
-+					 struct dmem_cgroup_pool_state **ret_limit_pool)
-+{
-+	*ret_pool = NULL;
-+
-+	if (ret_limit_pool)
-+		*ret_limit_pool = NULL;
-+
-+	return 0;
-+}
-+
-+static inline void dmem_cgroup_uncharge(struct dmem_cgroup_pool_state *pool, u64 size)
-+{ }
-+
-+static inline
-+bool dmem_cgroup_state_evict_valuable(struct dmem_cgroup_pool_state *limit_pool,
-+				      struct dmem_cgroup_pool_state *test_pool,
-+				      bool ignore_low, bool *ret_hit_low)
-+{
-+	return true;
-+}
-+
-+static inline void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool)
-+{ }
-+
-+#endif
-+#endif	/* _CGROUP_DMEM_H */
-diff --git a/include/linux/cgroup_namespace.h b/include/linux/cgroup_namespace.h
-new file mode 100644
-index 000000000000..78a8418558a4
--- /dev/null
-+++ b/include/linux/cgroup_namespace.h
-@@ -0,0 +1,58 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_CGROUP_NAMESPACE_H
-+#define _LINUX_CGROUP_NAMESPACE_H
-+
-+#include <linux/ns_common.h>
-+
-+struct cgroup_namespace {
-+	struct ns_common	ns;
-+	struct user_namespace	*user_ns;
-+	struct ucounts		*ucounts;
-+	struct css_set          *root_cset;
-+};
-+
-+extern struct cgroup_namespace init_cgroup_ns;
-+
-+#ifdef CONFIG_CGROUPS
-+
-+static inline struct cgroup_namespace *to_cg_ns(struct ns_common *ns)
-+{
-+	return container_of(ns, struct cgroup_namespace, ns);
-+}
-+
-+void free_cgroup_ns(struct cgroup_namespace *ns);
-+
-+struct cgroup_namespace *copy_cgroup_ns(u64 flags,
-+					struct user_namespace *user_ns,
-+					struct cgroup_namespace *old_ns);
-+
-+int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen,
-+		   struct cgroup_namespace *ns);
-+
-+static inline void get_cgroup_ns(struct cgroup_namespace *ns)
-+{
-+	ns_ref_inc(ns);
-+}
-+
-+static inline void put_cgroup_ns(struct cgroup_namespace *ns)
-+{
-+	if (ns_ref_put(ns))
-+		free_cgroup_ns(ns);
-+}
-+
-+#else /* !CONFIG_CGROUPS */
-+
-+static inline void free_cgroup_ns(struct cgroup_namespace *ns) { }
-+static inline struct cgroup_namespace *
-+copy_cgroup_ns(u64 flags, struct user_namespace *user_ns,
-+	       struct cgroup_namespace *old_ns)
-+{
-+	return old_ns;
-+}
-+
-+static inline void get_cgroup_ns(struct cgroup_namespace *ns) { }
-+static inline void put_cgroup_ns(struct cgroup_namespace *ns) { }
-+
-+#endif /* !CONFIG_CGROUPS */
-+
-+#endif /* _LINUX_CGROUP_NAMESPACE_H */
-diff --git a/include/linux/cgroup_refcnt.h b/include/linux/cgroup_refcnt.h
-new file mode 100644
-index 000000000000..2eea0a69ecfc
--- /dev/null
-+++ b/include/linux/cgroup_refcnt.h
-@@ -0,0 +1,96 @@
-+/**
-+ * css_get - obtain a reference on the specified css
-+ * @css: target css
-+ *
-+ * The caller must already have a reference.
-+ */
-+CGROUP_REF_FN_ATTRS
-+void css_get(struct cgroup_subsys_state *css)
-+{
-+	if (!(css->flags & CSS_NO_REF))
-+		percpu_ref_get(&css->refcnt);
-+}
-+CGROUP_REF_EXPORT(css_get)
-+
-+/**
-+ * css_get_many - obtain references on the specified css
-+ * @css: target css
-+ * @n: number of references to get
-+ *
-+ * The caller must already have a reference.
-+ */
-+CGROUP_REF_FN_ATTRS
-+void css_get_many(struct cgroup_subsys_state *css, unsigned int n)
-+{
-+	if (!(css->flags & CSS_NO_REF))
-+		percpu_ref_get_many(&css->refcnt, n);
-+}
-+CGROUP_REF_EXPORT(css_get_many)
-+
-+/**
-+ * css_tryget - try to obtain a reference on the specified css
-+ * @css: target css
-+ *
-+ * Obtain a reference on @css unless it already has reached zero and is
-+ * being released.  This function doesn't care whether @css is on or
-+ * offline.  The caller naturally needs to ensure that @css is accessible
-+ * but doesn't have to be holding a reference on it - IOW, RCU protected
-+ * access is good enough for this function.  Returns %true if a reference
-+ * count was successfully obtained; %false otherwise.
-+ */
-+CGROUP_REF_FN_ATTRS
-+bool css_tryget(struct cgroup_subsys_state *css)
-+{
-+	if (!(css->flags & CSS_NO_REF))
-+		return percpu_ref_tryget(&css->refcnt);
-+	return true;
-+}
-+CGROUP_REF_EXPORT(css_tryget)
-+
-+/**
-+ * css_tryget_online - try to obtain a reference on the specified css if online
-+ * @css: target css
-+ *
-+ * Obtain a reference on @css if it's online.  The caller naturally needs
-+ * to ensure that @css is accessible but doesn't have to be holding a
-+ * reference on it - IOW, RCU protected access is good enough for this
-+ * function.  Returns %true if a reference count was successfully obtained;
-+ * %false otherwise.
-+ */
-+CGROUP_REF_FN_ATTRS
-+bool css_tryget_online(struct cgroup_subsys_state *css)
-+{
-+	if (!(css->flags & CSS_NO_REF))
-+		return percpu_ref_tryget_live(&css->refcnt);
-+	return true;
-+}
-+CGROUP_REF_EXPORT(css_tryget_online)
-+
-+/**
-+ * css_put - put a css reference
-+ * @css: target css
-+ *
-+ * Put a reference obtained via css_get() and css_tryget_online().
-+ */
-+CGROUP_REF_FN_ATTRS
-+void css_put(struct cgroup_subsys_state *css)
-+{
-+	if (!(css->flags & CSS_NO_REF))
-+		percpu_ref_put(&css->refcnt);
-+}
-+CGROUP_REF_EXPORT(css_put)
-+
-+/**
-+ * css_put_many - put css references
-+ * @css: target css
-+ * @n: number of references to put
-+ *
-+ * Put references obtained via css_get() and css_tryget_online().
-+ */
-+CGROUP_REF_FN_ATTRS
-+void css_put_many(struct cgroup_subsys_state *css, unsigned int n)
-+{
-+	if (!(css->flags & CSS_NO_REF))
-+		percpu_ref_put_many(&css->refcnt, n);
-+}
-+CGROUP_REF_EXPORT(css_put_many)
-diff --git a/include/linux/cgroup_subsys.h b/include/linux/cgroup_subsys.h
-index 445235487230..3fd0bcbf3080 100644
--- a/include/linux/cgroup_subsys.h
-+++ b/include/linux/cgroup_subsys.h
-@@ -65,6 +65,10 @@ SUBSYS(rdma)
- SUBSYS(misc)
- #endif
- 
-+#if IS_ENABLED(CONFIG_CGROUP_DMEM)
-+SUBSYS(dmem)
-+#endif
-+
- /*
-  * The following subsystems are not supported on the default hierarchy.
-  */
-diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h
-new file mode 100644
-index 000000000000..8d41b917c77d
--- /dev/null
-+++ b/include/linux/cleanup.h
-@@ -0,0 +1,534 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_CLEANUP_H
-+#define _LINUX_CLEANUP_H
-+
-+#include <linux/compiler.h>
-+#include <linux/err.h>
-+#include <linux/args.h>
-+
-+/**
-+ * DOC: scope-based cleanup helpers
-+ *
-+ * The "goto error" pattern is notorious for introducing subtle resource
-+ * leaks. It is tedious and error prone to add new resource acquisition
-+ * constraints into code paths that already have several unwind
-+ * conditions. The "cleanup" helpers enable the compiler to help with
-+ * this tedium and can aid in maintaining LIFO (last in first out)
-+ * unwind ordering to avoid unintentional leaks.
-+ *
-+ * As drivers make up the majority of the kernel code base, here is an
-+ * example of using these helpers to clean up PCI drivers. The target of
-+ * the cleanups are occasions where a goto is used to unwind a device
-+ * reference (pci_dev_put()), or unlock the device (pci_dev_unlock())
-+ * before returning.
-+ *
-+ * The DEFINE_FREE() macro can arrange for PCI device references to be
-+ * dropped when the associated variable goes out of scope::
-+ *
-+ *	DEFINE_FREE(pci_dev_put, struct pci_dev *, if (_T) pci_dev_put(_T))
-+ *	...
-+ *	struct pci_dev *dev __free(pci_dev_put) =
-+ *		pci_get_slot(parent, PCI_DEVFN(0, 0));
-+ *
-+ * The above will automatically call pci_dev_put() if @dev is non-NULL
-+ * when @dev goes out of scope (automatic variable scope). If a function
-+ * wants to invoke pci_dev_put() on error, but return @dev (i.e. without
-+ * freeing it) on success, it can do::
-+ *
-+ *	return no_free_ptr(dev);
-+ *
-+ * ...or::
-+ *
-+ *	return_ptr(dev);
-+ *
-+ * The DEFINE_GUARD() macro can arrange for the PCI device lock to be
-+ * dropped when the scope where guard() is invoked ends::
-+ *
-+ *	DEFINE_GUARD(pci_dev, struct pci_dev *, pci_dev_lock(_T), pci_dev_unlock(_T))
-+ *	...
-+ *	guard(pci_dev)(dev);
-+ *
-+ * The lifetime of the lock obtained by the guard() helper follows the
-+ * scope of automatic variable declaration. Take the following example::
-+ *
-+ *	func(...)
-+ *	{
-+ *		if (...) {
-+ *			...
-+ *			guard(pci_dev)(dev); // pci_dev_lock() invoked here
-+ *			...
-+ *		} // <- implied pci_dev_unlock() triggered here
-+ *	}
-+ *
-+ * Observe the lock is held for the remainder of the "if ()" block not
-+ * the remainder of "func()".
-+ *
-+ * The ACQUIRE() macro can be used in all places that guard() can be
-+ * used and additionally support conditional locks::
-+ *
-+ *	DEFINE_GUARD_COND(pci_dev, _try, pci_dev_trylock(_T))
-+ *	...
-+ *	ACQUIRE(pci_dev_try, lock)(dev);
-+ *	rc = ACQUIRE_ERR(pci_dev_try, &lock);
-+ *	if (rc)
-+ *		return rc;
-+ *	// @lock is held
-+ *
-+ * Now, when a function uses both __free() and guard()/ACQUIRE(), or
-+ * multiple instances of __free(), the LIFO order of variable definition
-+ * order matters. GCC documentation says:
-+ *
-+ * "When multiple variables in the same scope have cleanup attributes,
-+ * at exit from the scope their associated cleanup functions are run in
-+ * reverse order of definition (last defined, first cleanup)."
-+ *
-+ * When the unwind order matters it requires that variables be defined
-+ * mid-function scope rather than at the top of the file.  Take the
-+ * following example and notice the bug highlighted by "!!"::
-+ *
-+ *	LIST_HEAD(list);
-+ *	DEFINE_MUTEX(lock);
-+ *
-+ *	struct object {
-+ *	        struct list_head node;
-+ *	};
-+ *
-+ *	static struct object *alloc_add(void)
-+ *	{
-+ *	        struct object *obj;
-+ *
-+ *	        lockdep_assert_held(&lock);
-+ *	        obj = kzalloc(sizeof(*obj), GFP_KERNEL);
-+ *	        if (obj) {
-+ *	                LIST_HEAD_INIT(&obj->node);
-+ *	                list_add(obj->node, &list):
-+ *	        }
-+ *	        return obj;
-+ *	}
-+ *
-+ *	static void remove_free(struct object *obj)
-+ *	{
-+ *	        lockdep_assert_held(&lock);
-+ *	        list_del(&obj->node);
-+ *	        kfree(obj);
-+ *	}
-+ *
-+ *	DEFINE_FREE(remove_free, struct object *, if (_T) remove_free(_T))
-+ *	static int init(void)
-+ *	{
-+ *	        struct object *obj __free(remove_free) = NULL;
-+ *	        int err;
-+ *
-+ *	        guard(mutex)(&lock);
-+ *	        obj = alloc_add();
-+ *
-+ *	        if (!obj)
-+ *	                return -ENOMEM;
-+ *
-+ *	        err = other_init(obj);
-+ *	        if (err)
-+ *	                return err; // remove_free() called without the lock!!
-+ *
-+ *	        no_free_ptr(obj);
-+ *	        return 0;
-+ *	}
-+ *
-+ * That bug is fixed by changing init() to call guard() and define +
-+ * initialize @obj in this order::
-+ *
-+ *	guard(mutex)(&lock);
-+ *	struct object *obj __free(remove_free) = alloc_add();
-+ *
-+ * Given that the "__free(...) = NULL" pattern for variables defined at
-+ * the top of the function poses this potential interdependency problem
-+ * the recommendation is to always define and assign variables in one
-+ * statement and not group variable definitions at the top of the
-+ * function when __free() is used.
-+ *
-+ * Lastly, given that the benefit of cleanup helpers is removal of
-+ * "goto", and that the "goto" statement can jump between scopes, the
-+ * expectation is that usage of "goto" and cleanup helpers is never
-+ * mixed in the same function. I.e. for a given routine, convert all
-+ * resources that need a "goto" cleanup to scope-based cleanup, or
-+ * convert none of them.
-+ */
-+
-+/*
-+ * DEFINE_FREE(name, type, free):
-+ *	simple helper macro that defines the required wrapper for a __free()
-+ *	based cleanup function. @free is an expression using '_T' to access the
-+ *	variable. @free should typically include a NULL test before calling a
-+ *	function, see the example below.
-+ *
-+ * __free(name):
-+ *	variable attribute to add a scoped based cleanup to the variable.
-+ *
-+ * no_free_ptr(var):
-+ *	like a non-atomic xchg(var, NULL), such that the cleanup function will
-+ *	be inhibited -- provided it sanely deals with a NULL value.
-+ *
-+ *	NOTE: this has __must_check semantics so that it is harder to accidentally
-+ *	leak the resource.
-+ *
-+ * return_ptr(p):
-+ *	returns p while inhibiting the __free().
-+ *
-+ * Ex.
-+ *
-+ * DEFINE_FREE(kfree, void *, if (_T) kfree(_T))
-+ *
-+ * void *alloc_obj(...)
-+ * {
-+ *	struct obj *p __free(kfree) = kmalloc(...);
-+ *	if (!p)
-+ *		return NULL;
-+ *
-+ *	if (!init_obj(p))
-+ *		return NULL;
-+ *
-+ *	return_ptr(p);
-+ * }
-+ *
-+ * NOTE: the DEFINE_FREE()'s @free expression includes a NULL test even though
-+ * kfree() is fine to be called with a NULL value. This is on purpose. This way
-+ * the compiler sees the end of our alloc_obj() function as:
-+ *
-+ *	tmp = p;
-+ *	p = NULL;
-+ *	if (p)
-+ *		kfree(p);
-+ *	return tmp;
-+ *
-+ * And through the magic of value-propagation and dead-code-elimination, it
-+ * eliminates the actual cleanup call and compiles into:
-+ *
-+ *	return p;
-+ *
-+ * Without the NULL test it turns into a mess and the compiler can't help us.
-+ */
-+
-+#define DEFINE_FREE(_name, _type, _free) \
-+	static __always_inline void __free_##_name(void *p) { _type _T = *(_type *)p; _free; }
-+
-+#define __free(_name)	__cleanup(__free_##_name)
-+
-+#define __get_and_null(p, nullvalue)	    \
-+	({                                  \
-+		auto __ptr = &(p);	    \
-+		auto __val = *__ptr;	    \
-+		*__ptr = nullvalue;         \
-+		__val;                      \
-+	})
-+
-+static __always_inline __must_check
-+const volatile void * __must_check_fn(const volatile void *val)
-+{ return val; }
-+
-+#define no_free_ptr(p) \
-+	((typeof(p)) __must_check_fn((__force const volatile void *)__get_and_null(p, NULL)))
-+
-+#define return_ptr(p)	return no_free_ptr(p)
-+
-+/*
-+ * Only for situations where an allocation is handed in to another function
-+ * and consumed by that function on success.
-+ *
-+ *	struct foo *f __free(kfree) = kzalloc(sizeof(*f), GFP_KERNEL);
-+ *
-+ *	setup(f);
-+ *	if (some_condition)
-+ *		return -EINVAL;
-+ *	....
-+ *	ret = bar(f);
-+ *	if (!ret)
-+ *		retain_and_null_ptr(f);
-+ *	return ret;
-+ *
-+ * After retain_and_null_ptr(f) the variable f is NULL and cannot be
-+ * dereferenced anymore.
-+ */
-+#define retain_and_null_ptr(p)		((void)__get_and_null(p, NULL))
-+
-+/*
-+ * DEFINE_CLASS(name, type, exit, init, init_args...):
-+ *	helper to define the destructor and constructor for a type.
-+ *	@exit is an expression using '_T' -- similar to FREE above.
-+ *	@init is an expression in @init_args resulting in @type
-+ *
-+ * EXTEND_CLASS(name, ext, init, init_args...):
-+ *	extends class @name to @name@ext with the new constructor
-+ *
-+ * CLASS(name, var)(args...):
-+ *	declare the variable @var as an instance of the named class
-+ *
-+ * CLASS_INIT(name, var, init_expr):
-+ *	declare the variable @var as an instance of the named class with
-+ *	custom initialization expression.
-+ *
-+ * Ex.
-+ *
-+ * DEFINE_CLASS(fdget, struct fd, fdput(_T), fdget(fd), int fd)
-+ *
-+ *	CLASS(fdget, f)(fd);
-+ *	if (fd_empty(f))
-+ *		return -EBADF;
-+ *
-+ *	// use 'f' without concern
-+ */
-+
-+#define DEFINE_CLASS(_name, _type, _exit, _init, _init_args...)		\
-+typedef _type class_##_name##_t;					\
-+static __always_inline void class_##_name##_destructor(_type *p)	\
-+{ _type _T = *p; _exit; }						\
-+static __always_inline _type class_##_name##_constructor(_init_args)	\
-+{ _type t = _init; return t; }
-+
-+#define EXTEND_CLASS(_name, ext, _init, _init_args...)			\
-+typedef class_##_name##_t class_##_name##ext##_t;			\
-+static __always_inline void class_##_name##ext##_destructor(class_##_name##_t *p) \
-+{ class_##_name##_destructor(p); }					\
-+static __always_inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \
-+{ class_##_name##_t t = _init; return t; }
-+
-+#define CLASS(_name, var)						\
-+	class_##_name##_t var __cleanup(class_##_name##_destructor) =	\
-+		class_##_name##_constructor
-+
-+#define CLASS_INIT(_name, _var, _init_expr)                             \
-+        class_##_name##_t _var __cleanup(class_##_name##_destructor) = (_init_expr)
-+
-+#define __scoped_class(_name, var, _label, args...)        \
-+	for (CLASS(_name, var)(args); ; ({ goto _label; })) \
-+		if (0) {                                   \
-+_label:                                                    \
-+			break;                             \
-+		} else
-+
-+#define scoped_class(_name, var, args...) \
-+	__scoped_class(_name, var, __UNIQUE_ID(label), args)
-+
-+/*
-+ * DEFINE_GUARD(name, type, lock, unlock):
-+ *	trivial wrapper around DEFINE_CLASS() above specifically
-+ *	for locks.
-+ *
-+ * DEFINE_GUARD_COND(name, ext, condlock)
-+ *	wrapper around EXTEND_CLASS above to add conditional lock
-+ *	variants to a base class, eg. mutex_trylock() or
-+ *	mutex_lock_interruptible().
-+ *
-+ * guard(name):
-+ *	an anonymous instance of the (guard) class, not recommended for
-+ *	conditional locks.
-+ *
-+ * scoped_guard (name, args...) { }:
-+ *	similar to CLASS(name, scope)(args), except the variable (with the
-+ *	explicit name 'scope') is declard in a for-loop such that its scope is
-+ *	bound to the next (compound) statement.
-+ *
-+ *	for conditional locks the loop body is skipped when the lock is not
-+ *	acquired.
-+ *
-+ * scoped_cond_guard (name, fail, args...) { }:
-+ *      similar to scoped_guard(), except it does fail when the lock
-+ *      acquire fails.
-+ *
-+ *      Only for conditional locks.
-+ *
-+ * ACQUIRE(name, var):
-+ *	a named instance of the (guard) class, suitable for conditional
-+ *	locks when paired with ACQUIRE_ERR().
-+ *
-+ * ACQUIRE_ERR(name, &var):
-+ *	a helper that is effectively a PTR_ERR() conversion of the guard
-+ *	pointer. Returns 0 when the lock was acquired and a negative
-+ *	error code otherwise.
-+ */
-+
-+#define __DEFINE_CLASS_IS_CONDITIONAL(_name, _is_cond)	\
-+static __maybe_unused const bool class_##_name##_is_conditional = _is_cond
-+
-+#define DEFINE_CLASS_IS_UNCONDITIONAL(_name)		\
-+	__DEFINE_CLASS_IS_CONDITIONAL(_name, false);	\
-+	static inline void * class_##_name##_lock_ptr(class_##_name##_t *_T) \
-+	{ return (void *)1; }
-+
-+#define __GUARD_IS_ERR(_ptr)                                       \
-+	({                                                         \
-+		unsigned long _rc = (__force unsigned long)(_ptr); \
-+		unlikely((_rc - 1) >= -MAX_ERRNO - 1);             \
-+	})
-+
-+#define __DEFINE_GUARD_LOCK_PTR(_name, _exp)                                \
-+	static __always_inline void *class_##_name##_lock_ptr(class_##_name##_t *_T) \
-+	{                                                                   \
-+		void *_ptr = (void *)(__force unsigned long)*(_exp);        \
-+		if (IS_ERR(_ptr)) {                                         \
-+			_ptr = NULL;                                        \
-+		}                                                           \
-+		return _ptr;                                                \
-+	}                                                                   \
-+	static __always_inline int class_##_name##_lock_err(class_##_name##_t *_T) \
-+	{                                                                   \
-+		long _rc = (__force unsigned long)*(_exp);                  \
-+		if (!_rc) {                                                 \
-+			_rc = -EBUSY;                                       \
-+		}                                                           \
-+		if (!IS_ERR_VALUE(_rc)) {                                   \
-+			_rc = 0;                                            \
-+		}                                                           \
-+		return _rc;                                                 \
-+	}
-+
-+#define DEFINE_CLASS_IS_GUARD(_name) \
-+	__DEFINE_CLASS_IS_CONDITIONAL(_name, false); \
-+	__DEFINE_GUARD_LOCK_PTR(_name, _T)
-+
-+#define DEFINE_CLASS_IS_COND_GUARD(_name) \
-+	__DEFINE_CLASS_IS_CONDITIONAL(_name, true); \
-+	__DEFINE_GUARD_LOCK_PTR(_name, _T)
-+
-+#define DEFINE_GUARD(_name, _type, _lock, _unlock) \
-+	DEFINE_CLASS(_name, _type, if (!__GUARD_IS_ERR(_T)) { _unlock; }, ({ _lock; _T; }), _type _T); \
-+	DEFINE_CLASS_IS_GUARD(_name)
-+
-+#define DEFINE_GUARD_COND_4(_name, _ext, _lock, _cond) \
-+	__DEFINE_CLASS_IS_CONDITIONAL(_name##_ext, true); \
-+	EXTEND_CLASS(_name, _ext, \
-+		     ({ void *_t = _T; int _RET = (_lock); if (_T && !(_cond)) _t = ERR_PTR(_RET); _t; }), \
-+		     class_##_name##_t _T) \
-+	static __always_inline void * class_##_name##_ext##_lock_ptr(class_##_name##_t *_T) \
-+	{ return class_##_name##_lock_ptr(_T); } \
-+	static __always_inline int class_##_name##_ext##_lock_err(class_##_name##_t *_T) \
-+	{ return class_##_name##_lock_err(_T); }
-+
-+/*
-+ * Default binary condition; success on 'true'.
-+ */
-+#define DEFINE_GUARD_COND_3(_name, _ext, _lock) \
-+	DEFINE_GUARD_COND_4(_name, _ext, _lock, _RET)
-+
-+#define DEFINE_GUARD_COND(X...) CONCATENATE(DEFINE_GUARD_COND_, COUNT_ARGS(X))(X)
-+
-+#define guard(_name) \
-+	CLASS(_name, __UNIQUE_ID(guard))
-+
-+#define __guard_ptr(_name) class_##_name##_lock_ptr
-+#define __guard_err(_name) class_##_name##_lock_err
-+#define __is_cond_ptr(_name) class_##_name##_is_conditional
-+
-+#define ACQUIRE(_name, _var)     CLASS(_name, _var)
-+#define ACQUIRE_ERR(_name, _var) __guard_err(_name)(_var)
-+
-+/*
-+ * Helper macro for scoped_guard().
-+ *
-+ * Note that the "!__is_cond_ptr(_name)" part of the condition ensures that
-+ * compiler would be sure that for the unconditional locks the body of the
-+ * loop (caller-provided code glued to the else clause) could not be skipped.
-+ * It is needed because the other part - "__guard_ptr(_name)(&scope)" - is too
-+ * hard to deduce (even if could be proven true for unconditional locks).
-+ */
-+#define __scoped_guard(_name, _label, args...)				\
-+	for (CLASS(_name, scope)(args);					\
-+	     __guard_ptr(_name)(&scope) || !__is_cond_ptr(_name);	\
-+	     ({ goto _label; }))					\
-+		if (0) {						\
-+_label:									\
-+			break;						\
-+		} else
-+
-+#define scoped_guard(_name, args...)	\
-+	__scoped_guard(_name, __UNIQUE_ID(label), args)
-+
-+#define __scoped_cond_guard(_name, _fail, _label, args...)		\
-+	for (CLASS(_name, scope)(args); true; ({ goto _label; }))	\
-+		if (!__guard_ptr(_name)(&scope)) {			\
-+			BUILD_BUG_ON(!__is_cond_ptr(_name));		\
-+			_fail;						\
-+_label:									\
-+			break;						\
-+		} else
-+
-+#define scoped_cond_guard(_name, _fail, args...)	\
-+	__scoped_cond_guard(_name, _fail, __UNIQUE_ID(label), args)
-+
-+/*
-+ * Additional helper macros for generating lock guards with types, either for
-+ * locks that don't have a native type (eg. RCU, preempt) or those that need a
-+ * 'fat' pointer (eg. spin_lock_irqsave).
-+ *
-+ * DEFINE_LOCK_GUARD_0(name, lock, unlock, ...)
-+ * DEFINE_LOCK_GUARD_1(name, type, lock, unlock, ...)
-+ * DEFINE_LOCK_GUARD_1_COND(name, ext, condlock)
-+ *
-+ * will result in the following type:
-+ *
-+ *   typedef struct {
-+ *	type *lock;		// 'type := void' for the _0 variant
-+ *	__VA_ARGS__;
-+ *   } class_##name##_t;
-+ *
-+ * As above, both _lock and _unlock are statements, except this time '_T' will
-+ * be a pointer to the above struct.
-+ */
-+
-+#define __DEFINE_UNLOCK_GUARD(_name, _type, _unlock, ...)		\
-+typedef struct {							\
-+	_type *lock;							\
-+	__VA_ARGS__;							\
-+} class_##_name##_t;							\
-+									\
-+static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \
-+{									\
-+	if (!__GUARD_IS_ERR(_T->lock)) { _unlock; }			\
-+}									\
-+									\
-+__DEFINE_GUARD_LOCK_PTR(_name, &_T->lock)
-+
-+#define __DEFINE_LOCK_GUARD_1(_name, _type, _lock)			\
-+static __always_inline class_##_name##_t class_##_name##_constructor(_type *l) \
-+{									\
-+	class_##_name##_t _t = { .lock = l }, *_T = &_t;		\
-+	_lock;								\
-+	return _t;							\
-+}
-+
-+#define __DEFINE_LOCK_GUARD_0(_name, _lock)				\
-+static __always_inline class_##_name##_t class_##_name##_constructor(void) \
-+{									\
-+	class_##_name##_t _t = { .lock = (void*)1 },			\
-+			 *_T __maybe_unused = &_t;			\
-+	_lock;								\
-+	return _t;							\
-+}
-+
-+#define DEFINE_LOCK_GUARD_1(_name, _type, _lock, _unlock, ...)		\
-+__DEFINE_CLASS_IS_CONDITIONAL(_name, false);				\
-+__DEFINE_UNLOCK_GUARD(_name, _type, _unlock, __VA_ARGS__)		\
-+__DEFINE_LOCK_GUARD_1(_name, _type, _lock)
-+
-+#define DEFINE_LOCK_GUARD_0(_name, _lock, _unlock, ...)			\
-+__DEFINE_CLASS_IS_CONDITIONAL(_name, false);				\
-+__DEFINE_UNLOCK_GUARD(_name, void, _unlock, __VA_ARGS__)		\
-+__DEFINE_LOCK_GUARD_0(_name, _lock)
-+
-+#define DEFINE_LOCK_GUARD_1_COND_4(_name, _ext, _lock, _cond)		\
-+	__DEFINE_CLASS_IS_CONDITIONAL(_name##_ext, true);		\
-+	EXTEND_CLASS(_name, _ext,					\
-+		     ({ class_##_name##_t _t = { .lock = l }, *_T = &_t;\
-+		        int _RET = (_lock);                             \
-+		        if (_T->lock && !(_cond)) _T->lock = ERR_PTR(_RET);\
-+			_t; }),						\
-+		     typeof_member(class_##_name##_t, lock) l)		\
-+	static __always_inline void * class_##_name##_ext##_lock_ptr(class_##_name##_t *_T) \
-+	{ return class_##_name##_lock_ptr(_T); } \
-+	static __always_inline int class_##_name##_ext##_lock_err(class_##_name##_t *_T) \
-+	{ return class_##_name##_lock_err(_T); }
-+
-+#define DEFINE_LOCK_GUARD_1_COND_3(_name, _ext, _lock) \
-+	DEFINE_LOCK_GUARD_1_COND_4(_name, _ext, _lock, _RET)
-+
-+#define DEFINE_LOCK_GUARD_1_COND(X...) CONCATENATE(DEFINE_LOCK_GUARD_1_COND_, COUNT_ARGS(X))(X)
-+
-+#endif /* _LINUX_CLEANUP_H */
-diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h
-index 1615010aa0ec..630705a47129 100644
--- a/include/linux/clk-provider.h
-+++ b/include/linux/clk-provider.h
-@@ -42,6 +42,9 @@ struct dentry;
-  * struct clk_rate_request - Structure encoding the clk constraints that
-  * a clock user might require.
-  *
-+ * Should be initialized by calling clk_hw_init_rate_request().
-+ *
-+ * @core: 		Pointer to the struct clk_core affected by this request
-  * @rate:		Requested clock rate. This field will be adjusted by
-  *			clock drivers according to hardware capabilities.
-  * @min_rate:		Minimum rate imposed by clk users.
-@@ -53,6 +56,7 @@ struct dentry;
-  *
-  */
- struct clk_rate_request {
-+	struct clk_core *core;
- 	unsigned long rate;
- 	unsigned long min_rate;
- 	unsigned long max_rate;
-@@ -60,8 +64,17 @@ struct clk_rate_request {
- 	struct clk_hw *best_parent_hw;
- };
- 
-+void clk_hw_init_rate_request(const struct clk_hw *hw,
-+			      struct clk_rate_request *req,
-+			      unsigned long rate);
-+void clk_hw_forward_rate_request(const struct clk_hw *core,
-+				 const struct clk_rate_request *old_req,
-+				 const struct clk_hw *parent,
-+				 struct clk_rate_request *req,
-+				 unsigned long parent_rate);
-+
- /**
- * struct clk_duty - Struture encoding the duty cycle ratio of a clock
-+ * struct clk_duty - Structure encoding the duty cycle ratio of a clock
-  *
-  * @num:	Numerator of the duty cycle ratio
-  * @den:	Denominator of the duty cycle ratio
-@@ -116,10 +129,11 @@ struct clk_duty {
-  * @restore_context: Restore the context of the clock after a restoration
-  *		of power.
-  *
- * @recalc_rate	Recalculate the rate of this clock, by querying hardware. The
-+ * @recalc_rate: Recalculate the rate of this clock, by querying hardware. The
-  *		parent rate is an input parameter.  It is up to the caller to
- *		ensure that the prepare_mutex is held across this call.
- *		Returns the calculated rate.  Optional, but recommended - if
-+ *		ensure that the prepare_mutex is held across this call. If the
-+ *		driver cannot figure out a rate for this clock, it must return
-+ *		0. Returns the calculated rate. Optional, but recommended - if
-  *		this op is not set then clock rate will be initialized to 0.
-  *
-  * @round_rate:	Given a target rate as input, returns the closest rate actually
-@@ -350,7 +364,7 @@ struct clk_hw *__clk_hw_register_fixed_rate(struct device *dev,
- 		const char *parent_name, const struct clk_hw *parent_hw,
- 		const struct clk_parent_data *parent_data, unsigned long flags,
- 		unsigned long fixed_rate, unsigned long fixed_accuracy,
-		unsigned long clk_fixed_flags);
-+		unsigned long clk_fixed_flags, bool devm);
- struct clk *clk_register_fixed_rate(struct device *dev, const char *name,
- 		const char *parent_name, unsigned long flags,
- 		unsigned long fixed_rate);
-@@ -365,7 +379,34 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name,
-  */
- #define clk_hw_register_fixed_rate(dev, name, parent_name, flags, fixed_rate)  \
- 	__clk_hw_register_fixed_rate((dev), NULL, (name), (parent_name), NULL, \
-				     NULL, (flags), (fixed_rate), 0, 0)
-+				     NULL, (flags), (fixed_rate), 0, 0, false)
-+
-+/**
-+ * devm_clk_hw_register_fixed_rate - register fixed-rate clock with the clock
-+ * framework
-+ * @dev: device that is registering this clock
-+ * @name: name of this clock
-+ * @parent_name: name of clock's parent
-+ * @flags: framework-specific flags
-+ * @fixed_rate: non-adjustable clock rate
-+ */
-+#define devm_clk_hw_register_fixed_rate(dev, name, parent_name, flags, fixed_rate)  \
-+	__clk_hw_register_fixed_rate((dev), NULL, (name), (parent_name), NULL, \
-+				     NULL, (flags), (fixed_rate), 0, 0, true)
-+/**
-+ * devm_clk_hw_register_fixed_rate_parent_data - register fixed-rate clock with
-+ * the clock framework
-+ * @dev: device that is registering this clock
-+ * @name: name of this clock
-+ * @parent_data: parent clk data
-+ * @flags: framework-specific flags
-+ * @fixed_rate: non-adjustable clock rate
-+ */
-+#define devm_clk_hw_register_fixed_rate_parent_data(dev, name, parent_data, flags, \
-+						    fixed_rate)			   \
-+	__clk_hw_register_fixed_rate((dev), NULL, (name), NULL, NULL,		   \
-+				     (parent_data), (flags), (fixed_rate), 0,	   \
-+				     0, true)
- /**
-  * clk_hw_register_fixed_rate_parent_hw - register fixed-rate clock with
-  * the clock framework
-@@ -378,7 +419,7 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name,
- #define clk_hw_register_fixed_rate_parent_hw(dev, name, parent_hw, flags,     \
- 					     fixed_rate)		      \
- 	__clk_hw_register_fixed_rate((dev), NULL, (name), NULL, (parent_hw),  \
-				     NULL, (flags), (fixed_rate), 0, 0)
-+				     NULL, (flags), (fixed_rate), 0, 0, false)
- /**
-  * clk_hw_register_fixed_rate_parent_data - register fixed-rate clock with
-  * the clock framework
-@@ -388,11 +429,11 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name,
-  * @flags: framework-specific flags
-  * @fixed_rate: non-adjustable clock rate
-  */
-#define clk_hw_register_fixed_rate_parent_data(dev, name, parent_hw, flags,   \
-+#define clk_hw_register_fixed_rate_parent_data(dev, name, parent_data, flags, \
- 					     fixed_rate)		      \
- 	__clk_hw_register_fixed_rate((dev), NULL, (name), NULL, NULL,	      \
- 				     (parent_data), (flags), (fixed_rate), 0, \
-				     0)
-+				     0, false)
- /**
-  * clk_hw_register_fixed_rate_with_accuracy - register fixed-rate clock with
-  * the clock framework
-@@ -408,7 +449,7 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name,
- 						 fixed_accuracy)	      \
- 	__clk_hw_register_fixed_rate((dev), NULL, (name), (parent_name),      \
- 				     NULL, NULL, (flags), (fixed_rate),       \
-				     (fixed_accuracy), 0)
-+				     (fixed_accuracy), 0, false)
- /**
-  * clk_hw_register_fixed_rate_with_accuracy_parent_hw - register fixed-rate
-  * clock with the clock framework
-@@ -421,15 +462,15 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name,
-  */
- #define clk_hw_register_fixed_rate_with_accuracy_parent_hw(dev, name,	      \
- 		parent_hw, flags, fixed_rate, fixed_accuracy)		      \
-	__clk_hw_register_fixed_rate((dev), NULL, (name), NULL, (parent_hw)   \
-				     NULL, NULL, (flags), (fixed_rate),	      \
-				     (fixed_accuracy), 0)
-+	__clk_hw_register_fixed_rate((dev), NULL, (name), NULL, (parent_hw),  \
-+				     NULL, (flags), (fixed_rate),	      \
-+				     (fixed_accuracy), 0, false)
- /**
-  * clk_hw_register_fixed_rate_with_accuracy_parent_data - register fixed-rate
-  * clock with the clock framework
-  * @dev: device that is registering this clock
-  * @name: name of this clock
- * @parent_name: name of clock's parent
-+ * @parent_data: name of clock's parent
-  * @flags: framework-specific flags
-  * @fixed_rate: non-adjustable clock rate
-  * @fixed_accuracy: non-adjustable clock accuracy
-@@ -438,7 +479,21 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name,
- 		parent_data, flags, fixed_rate, fixed_accuracy)		      \
- 	__clk_hw_register_fixed_rate((dev), NULL, (name), NULL, NULL,	      \
- 				     (parent_data), NULL, (flags),	      \
-				     (fixed_rate), (fixed_accuracy), 0)
-+				     (fixed_rate), (fixed_accuracy), 0, false)
-+/**
-+ * clk_hw_register_fixed_rate_parent_accuracy - register fixed-rate clock with
-+ * the clock framework
-+ * @dev: device that is registering this clock
-+ * @name: name of this clock
-+ * @parent_data: name of clock's parent
-+ * @flags: framework-specific flags
-+ * @fixed_rate: non-adjustable clock rate
-+ */
-+#define clk_hw_register_fixed_rate_parent_accuracy(dev, name, parent_data,    \
-+						   flags, fixed_rate)	      \
-+	__clk_hw_register_fixed_rate((dev), NULL, (name), NULL, NULL,      \
-+				     (parent_data), (flags), (fixed_rate), 0,    \
-+				     CLK_FIXED_RATE_PARENT_ACCURACY, false)
- 
- void clk_unregister_fixed_rate(struct clk *clk);
- void clk_hw_unregister_fixed_rate(struct clk_hw *hw);
-@@ -567,6 +622,43 @@ struct clk *clk_register_gate(struct device *dev, const char *name,
- 	__devm_clk_hw_register_gate((dev), NULL, (name), (parent_name), NULL, \
- 			       NULL, (flags), (reg), (bit_idx),		      \
- 			       (clk_gate_flags), (lock))
-+/**
-+ * devm_clk_hw_register_gate_parent_hw - register a gate clock with the clock
-+ * framework
-+ * @dev: device that is registering this clock
-+ * @name: name of this clock
-+ * @parent_hw: pointer to parent clk
-+ * @flags: framework-specific flags for this clock
-+ * @reg: register address to control gating of this clock
-+ * @bit_idx: which bit in the register controls gating of this clock
-+ * @clk_gate_flags: gate-specific flags for this clock
-+ * @lock: shared register lock for this clock
-+ */
-+#define devm_clk_hw_register_gate_parent_hw(dev, name, parent_hw, flags,      \
-+					    reg, bit_idx, clk_gate_flags,     \
-+					    lock)			      \
-+	__devm_clk_hw_register_gate((dev), NULL, (name), NULL, (parent_hw),   \
-+				    NULL, (flags), (reg), (bit_idx),	      \
-+				    (clk_gate_flags), (lock))
-+/**
-+ * devm_clk_hw_register_gate_parent_data - register a gate clock with the
-+ * clock framework
-+ * @dev: device that is registering this clock
-+ * @name: name of this clock
-+ * @parent_data: parent clk data
-+ * @flags: framework-specific flags for this clock
-+ * @reg: register address to control gating of this clock
-+ * @bit_idx: which bit in the register controls gating of this clock
-+ * @clk_gate_flags: gate-specific flags for this clock
-+ * @lock: shared register lock for this clock
-+ */
-+#define devm_clk_hw_register_gate_parent_data(dev, name, parent_data, flags,  \
-+					      reg, bit_idx, clk_gate_flags,   \
-+					      lock)			      \
-+	__devm_clk_hw_register_gate((dev), NULL, (name), NULL, NULL,	      \
-+				    (parent_data), (flags), (reg), (bit_idx), \
-+				    (clk_gate_flags), (lock))
-+
- void clk_unregister_gate(struct clk *clk);
- void clk_hw_unregister_gate(struct clk_hw *hw);
- int clk_gate_is_enabled(struct clk_hw *hw);
-@@ -589,7 +681,7 @@ struct clk_div_table {
-  * Clock with an adjustable divider affecting its output frequency.  Implements
-  * .recalc_rate, .set_rate and .round_rate
-  *
- * Flags:
-+ * @flags:
-  * CLK_DIVIDER_ONE_BASED - by default the divisor is the value read from the
-  *	register plus one.  If CLK_DIVIDER_ONE_BASED is set then the divider is
-  *	the raw value read from the register, with the value of zero considered
-@@ -615,13 +707,15 @@ struct clk_div_table {
-  * CLK_DIVIDER_BIG_ENDIAN - By default little endian register accesses are used
-  *	for the divider register.  Setting this flag makes the register accesses
-  *	big endian.
-+ * CLK_DIVIDER_EVEN_INTEGERS - clock divisor is 2, 4, 6, 8, 10, etc.
-+ *	Formula is 2 * (value read from hardware + 1).
-  */
- struct clk_divider {
- 	struct clk_hw	hw;
- 	void __iomem	*reg;
- 	u8		shift;
- 	u8		width;
-	u8		flags;
-+	u16		flags;
- 	const struct clk_div_table	*table;
- 	spinlock_t	*lock;
- };
-@@ -637,6 +731,7 @@ struct clk_divider {
- #define CLK_DIVIDER_READ_ONLY		BIT(5)
- #define CLK_DIVIDER_MAX_AT_ZERO		BIT(6)
- #define CLK_DIVIDER_BIG_ENDIAN		BIT(7)
-+#define CLK_DIVIDER_EVEN_INTEGERS	BIT(8)
- 
- extern const struct clk_ops clk_divider_ops;
- extern const struct clk_ops clk_divider_ro_ops;
-@@ -666,19 +761,21 @@ struct clk_hw *__clk_hw_register_divider(struct device *dev,
- 		struct device_node *np, const char *name,
- 		const char *parent_name, const struct clk_hw *parent_hw,
- 		const struct clk_parent_data *parent_data, unsigned long flags,
-		void __iomem *reg, u8 shift, u8 width, u8 clk_divider_flags,
-+		void __iomem *reg, u8 shift, u8 width,
-+		unsigned long clk_divider_flags,
- 		const struct clk_div_table *table, spinlock_t *lock);
- struct clk_hw *__devm_clk_hw_register_divider(struct device *dev,
- 		struct device_node *np, const char *name,
- 		const char *parent_name, const struct clk_hw *parent_hw,
- 		const struct clk_parent_data *parent_data, unsigned long flags,
-		void __iomem *reg, u8 shift, u8 width, u8 clk_divider_flags,
-+		void __iomem *reg, u8 shift, u8 width,
-+		unsigned long clk_divider_flags,
- 		const struct clk_div_table *table, spinlock_t *lock);
- struct clk *clk_register_divider_table(struct device *dev, const char *name,
- 		const char *parent_name, unsigned long flags,
- 		void __iomem *reg, u8 shift, u8 width,
-		u8 clk_divider_flags, const struct clk_div_table *table,
-		spinlock_t *lock);
-+		unsigned long clk_divider_flags,
-+		const struct clk_div_table *table, spinlock_t *lock);
- /**
-  * clk_register_divider - register a divider clock with the clock framework
-  * @dev: device registering this clock
-@@ -957,6 +1054,13 @@ struct clk *clk_register_mux_table(struct device *dev, const char *name,
- 			      (parent_names), NULL, NULL, (flags), (reg),     \
- 			      (shift), (mask), (clk_mux_flags), (table),      \
- 			      (lock))
-+#define clk_hw_register_mux_table_parent_data(dev, name, parent_data,	      \
-+				  num_parents, flags, reg, shift, mask,	      \
-+				  clk_mux_flags, table, lock)		      \
-+	__clk_hw_register_mux((dev), NULL, (name), (num_parents),	      \
-+			      NULL, NULL, (parent_data), (flags), (reg),      \
-+			      (shift), (mask), (clk_mux_flags), (table),      \
-+			      (lock))
- #define clk_hw_register_mux(dev, name, parent_names, num_parents, flags, reg, \
- 			    shift, width, clk_mux_flags, lock)		      \
- 	__clk_hw_register_mux((dev), NULL, (name), (num_parents),	      \
-@@ -974,6 +1078,13 @@ struct clk *clk_register_mux_table(struct device *dev, const char *name,
- 	__clk_hw_register_mux((dev), NULL, (name), (num_parents), NULL, NULL, \
- 			      (parent_data), (flags), (reg), (shift),	      \
- 			      BIT((width)) - 1, (clk_mux_flags), NULL, (lock))
-+#define clk_hw_register_mux_parent_data_table(dev, name, parent_data,	      \
-+					      num_parents, flags, reg, shift, \
-+					      width, clk_mux_flags, table,    \
-+					      lock)			      \
-+	__clk_hw_register_mux((dev), NULL, (name), (num_parents), NULL, NULL, \
-+			      (parent_data), (flags), (reg), (shift),	      \
-+			      BIT((width)) - 1, (clk_mux_flags), table, (lock))
- #define devm_clk_hw_register_mux(dev, name, parent_names, num_parents, flags, reg, \
- 			    shift, width, clk_mux_flags, lock)		      \
- 	__devm_clk_hw_register_mux((dev), NULL, (name), (num_parents),	      \
-@@ -987,6 +1098,13 @@ struct clk *clk_register_mux_table(struct device *dev, const char *name,
- 				   (parent_hws), NULL, (flags), (reg),        \
- 				   (shift), BIT((width)) - 1,		      \
- 				   (clk_mux_flags), NULL, (lock))
-+#define devm_clk_hw_register_mux_parent_data_table(dev, name, parent_data,    \
-+					      num_parents, flags, reg, shift, \
-+					      width, clk_mux_flags, table,    \
-+					      lock)			      \
-+	__devm_clk_hw_register_mux((dev), NULL, (name), (num_parents), NULL,  \
-+			      NULL, (parent_data), (flags), (reg), (shift),   \
-+			      BIT((width)) - 1, (clk_mux_flags), table, (lock))
- 
- int clk_mux_val_to_index(struct clk_hw *hw, const u32 *table, unsigned int flags,
- 			 unsigned int val);
-@@ -1003,18 +1121,28 @@ void of_fixed_factor_clk_setup(struct device_node *node);
-  * @hw:		handle between common and hardware-specific interfaces
-  * @mult:	multiplier
-  * @div:	divider
-+ * @acc:	fixed accuracy in ppb
-+ * @flags:	behavior modifying flags
-  *
-  * Clock with a fixed multiplier and divider. The output frequency is the
-  * parent clock rate divided by div and multiplied by mult.
- * Implements .recalc_rate, .set_rate and .round_rate
-+ * Implements .recalc_rate, .set_rate, .round_rate and .recalc_accuracy
-+ *
-+ * Flags:
-+ * * CLK_FIXED_FACTOR_FIXED_ACCURACY - Use the value in @acc instead of the
-+ *                                     parent clk accuracy.
-  */
- 
- struct clk_fixed_factor {
- 	struct clk_hw	hw;
- 	unsigned int	mult;
- 	unsigned int	div;
-+	unsigned long	acc;
-+	unsigned int	flags;
- };
- 
-+#define CLK_FIXED_FACTOR_FIXED_ACCURACY	BIT(0)
-+
- #define to_clk_fixed_factor(_hw) container_of(_hw, struct clk_fixed_factor, hw)
- 
- extern const struct clk_ops clk_fixed_factor_ops;
-@@ -1025,10 +1153,27 @@ void clk_unregister_fixed_factor(struct clk *clk);
- struct clk_hw *clk_hw_register_fixed_factor(struct device *dev,
- 		const char *name, const char *parent_name, unsigned long flags,
- 		unsigned int mult, unsigned int div);
-+struct clk_hw *clk_hw_register_fixed_factor_fwname(struct device *dev,
-+		struct device_node *np, const char *name, const char *fw_name,
-+		unsigned long flags, unsigned int mult, unsigned int div);
-+struct clk_hw *clk_hw_register_fixed_factor_with_accuracy_fwname(struct device *dev,
-+		struct device_node *np, const char *name, const char *fw_name,
-+		unsigned long flags, unsigned int mult, unsigned int div,
-+		unsigned long acc);
-+struct clk_hw *clk_hw_register_fixed_factor_index(struct device *dev,
-+		const char *name, unsigned int index, unsigned long flags,
-+		unsigned int mult, unsigned int div);
- void clk_hw_unregister_fixed_factor(struct clk_hw *hw);
- struct clk_hw *devm_clk_hw_register_fixed_factor(struct device *dev,
- 		const char *name, const char *parent_name, unsigned long flags,
- 		unsigned int mult, unsigned int div);
-+struct clk_hw *devm_clk_hw_register_fixed_factor_fwname(struct device *dev,
-+		struct device_node *np, const char *name, const char *fw_name,
-+		unsigned long flags, unsigned int mult, unsigned int div);
-+struct clk_hw *devm_clk_hw_register_fixed_factor_with_accuracy_fwname(struct device *dev,
-+		struct device_node *np, const char *name, const char *fw_name,
-+		unsigned long flags, unsigned int mult, unsigned int div,
-+		unsigned long acc);
- struct clk_hw *devm_clk_hw_register_fixed_factor_index(struct device *dev,
- 		const char *name, unsigned int index, unsigned long flags,
- 		unsigned int mult, unsigned int div);
-@@ -1049,11 +1194,12 @@ struct clk_hw *clk_hw_register_fixed_factor_parent_hw(struct device *dev,
-  * @mwidth:	width of the numerator bit field
-  * @nshift:	shift to the denominator bit field
-  * @nwidth:	width of the denominator bit field
-+ * @approximation: clk driver's callback for calculating the divider clock
-  * @lock:	register lock
-  *
-  * Clock with adjustable fractional divider affecting its output frequency.
-  *
- * Flags:
-+ * @flags:
-  * CLK_FRAC_DIVIDER_ZERO_BASED - by default the numerator and denominator
-  *	is the value read from the register. If CLK_FRAC_DIVIDER_ZERO_BASED
-  *	is set then the numerator and denominator are both the value read
-@@ -1073,10 +1219,8 @@ struct clk_fractional_divider {
- 	void __iomem	*reg;
- 	u8		mshift;
- 	u8		mwidth;
-	u32		mmask;
- 	u8		nshift;
- 	u8		nwidth;
-	u32		nmask;
- 	u8		flags;
- 	void		(*approximation)(struct clk_hw *hw,
- 				unsigned long rate, unsigned long *parent_rate,
-@@ -1112,7 +1256,7 @@ void clk_hw_unregister_fractional_divider(struct clk_hw *hw);
-  * Clock with an adjustable multiplier affecting its output frequency.
-  * Implements .recalc_rate, .set_rate and .round_rate
-  *
- * Flags:
-+ * @flags:
-  * CLK_MULTIPLIER_ZERO_BYPASS - By default, the multiplier is the value read
-  *	from the register, with 0 being a valid value effectively
-  *	zeroing the output clock rate. If CLK_MULTIPLIER_ZERO_BYPASS is
-@@ -1216,6 +1360,32 @@ void clk_hw_unregister(struct clk_hw *hw);
- /* helper functions */
- const char *__clk_get_name(const struct clk *clk);
- const char *clk_hw_get_name(const struct clk_hw *hw);
-+
-+/**
-+ * clk_hw_get_dev() - get device from an hardware clock.
-+ * @hw: the clk_hw pointer to get the struct device from
-+ *
-+ * This is a helper to get the struct device associated with a hardware
-+ * clock. Some clock controllers, such as the one registered with
-+ * CLK_OF_DECLARE(), may have not provided a device pointer while
-+ * registering the clock.
-+ *
-+ * Return: the struct device associated with the clock, or NULL if there
-+ * is none.
-+ */
-+struct device *clk_hw_get_dev(const struct clk_hw *hw);
-+
-+/**
-+ * clk_hw_get_of_node() - get device_node from a hardware clock.
-+ * @hw: the clk_hw pointer to get the struct device_node from
-+ *
-+ * This is a helper to get the struct device_node associated with a
-+ * hardware clock.
-+ *
-+ * Return: the struct device_node associated with the clock, or NULL
-+ * if there is none.
-+ */
-+struct device_node *clk_hw_get_of_node(const struct clk_hw *hw);
- #ifdef CONFIG_COMMON_CLK
- struct clk_hw *__clk_get_hw(struct clk *clk);
- #else
-@@ -1242,7 +1412,6 @@ unsigned long clk_hw_get_flags(const struct clk_hw *hw);
- 	(clk_hw_get_flags((hw)) & CLK_SET_RATE_PARENT)
- 
- bool clk_hw_is_prepared(const struct clk_hw *hw);
-bool clk_hw_rate_is_protected(const struct clk_hw *hw);
- bool clk_hw_is_enabled(const struct clk_hw *hw);
- bool __clk_is_enabled(struct clk *clk);
- struct clk *__clk_lookup(const char *name);
-@@ -1254,7 +1423,11 @@ int __clk_mux_determine_rate_closest(struct clk_hw *hw,
- int clk_mux_determine_rate_flags(struct clk_hw *hw,
- 				 struct clk_rate_request *req,
- 				 unsigned long flags);
-+int clk_hw_determine_rate_no_reparent(struct clk_hw *hw,
-+				      struct clk_rate_request *req);
- void clk_hw_reparent(struct clk_hw *hw, struct clk_hw *new_parent);
-+void clk_hw_get_rate_range(struct clk_hw *hw, unsigned long *min_rate,
-+			   unsigned long *max_rate);
- void clk_hw_set_rate_range(struct clk_hw *hw, unsigned long min_rate,
- 			   unsigned long max_rate);
- 
-@@ -1296,10 +1469,16 @@ struct clk_onecell_data {
- 
- struct clk_hw_onecell_data {
- 	unsigned int num;
-	struct clk_hw *hws[];
-+	struct clk_hw *hws[] __counted_by(num);
- };
- 
-#define CLK_OF_DECLARE(name, compat, fn) OF_DECLARE_1(clk, name, compat, fn)
-+#define CLK_OF_DECLARE(name, compat, fn) \
-+	static void __init __##name##_of_clk_init_declare(struct device_node *np) \
-+	{								\
-+		fn(np);							\
-+		fwnode_dev_initialized(of_fwnode_handle(np), true);	\
-+	}								\
-+	OF_DECLARE_1(clk, name, compat, __##name##_of_clk_init_declare)
- 
- /*
-  * Use this macro when you have a driver that requires two initialization
-@@ -1454,7 +1633,7 @@ int devm_of_clk_add_hw_provider(struct device *dev,
- 						 void *data),
- 			   void *data);
- void of_clk_del_provider(struct device_node *np);
-void devm_of_clk_del_provider(struct device *dev);
-+
- struct clk *of_clk_src_simple_get(struct of_phandle_args *clkspec,
- 				  void *data);
- struct clk_hw *of_clk_hw_simple_get(struct of_phandle_args *clkspec,
-@@ -1491,7 +1670,7 @@ static inline int devm_of_clk_add_hw_provider(struct device *dev,
- 	return 0;
- }
- static inline void of_clk_del_provider(struct device_node *np) {}
-static inline void devm_of_clk_del_provider(struct device *dev) {}
-+
- static inline struct clk *of_clk_src_simple_get(
- 	struct of_phandle_args *clkspec, void *data)
- {
-diff --git a/include/linux/clk.h b/include/linux/clk.h
-index c13061cabdfc..b607482ca77e 100644
--- a/include/linux/clk.h
-+++ b/include/linux/clk.h
-@@ -183,6 +183,51 @@ int clk_get_scaled_duty_cycle(struct clk *clk, unsigned int scale);
-  */
- bool clk_is_match(const struct clk *p, const struct clk *q);
- 
-+/**
-+ * clk_rate_exclusive_get - get exclusivity over the rate control of a
-+ *                          producer
-+ * @clk: clock source
-+ *
-+ * This function allows drivers to get exclusive control over the rate of a
-+ * provider. It prevents any other consumer to execute, even indirectly,
-+ * opereation which could alter the rate of the provider or cause glitches
-+ *
-+ * If exlusivity is claimed more than once on clock, even by the same driver,
-+ * the rate effectively gets locked as exclusivity can't be preempted.
-+ *
-+ * Must not be called from within atomic context.
-+ *
-+ * Returns success (0) or negative errno.
-+ */
-+int clk_rate_exclusive_get(struct clk *clk);
-+
-+/**
-+ * devm_clk_rate_exclusive_get - devm variant of clk_rate_exclusive_get
-+ * @dev: device the exclusivity is bound to
-+ * @clk: clock source
-+ *
-+ * Calls clk_rate_exclusive_get() on @clk and registers a devm cleanup handler
-+ * on @dev to call clk_rate_exclusive_put().
-+ *
-+ * Must not be called from within atomic context.
-+ */
-+int devm_clk_rate_exclusive_get(struct device *dev, struct clk *clk);
-+
-+/**
-+ * clk_rate_exclusive_put - release exclusivity over the rate control of a
-+ *                          producer
-+ * @clk: clock source
-+ *
-+ * This function allows drivers to release the exclusivity it previously got
-+ * from clk_rate_exclusive_get()
-+ *
-+ * The caller must balance the number of clk_rate_exclusive_get() and
-+ * clk_rate_exclusive_put() calls.
-+ *
-+ * Must not be called from within atomic context.
-+ */
-+void clk_rate_exclusive_put(struct clk *clk);
-+
- #else
- 
- static inline int clk_notifier_register(struct clk *clk,
-@@ -236,6 +281,18 @@ static inline bool clk_is_match(const struct clk *p, const struct clk *q)
- 	return p == q;
- }
- 
-+static inline int clk_rate_exclusive_get(struct clk *clk)
-+{
-+	return 0;
-+}
-+
-+static inline int devm_clk_rate_exclusive_get(struct device *dev, struct clk *clk)
-+{
-+	return 0;
-+}
-+
-+static inline void clk_rate_exclusive_put(struct clk *clk) {}
-+
- #endif
- 
- #ifdef CONFIG_HAVE_CLK_PREPARE
-@@ -439,6 +496,24 @@ int __must_check devm_clk_bulk_get_all(struct device *dev,
- 				       struct clk_bulk_data **clks);
- 
- /**
-+ * devm_clk_bulk_get_all_enabled - Get and enable all clocks of the consumer (managed)
-+ * @dev: device for clock "consumer"
-+ * @clks: pointer to the clk_bulk_data table of consumer
-+ *
-+ * Returns a positive value for the number of clocks obtained while the
-+ * clock references are stored in the clk_bulk_data table in @clks field.
-+ * Returns 0 if there're none and a negative value if something failed.
-+ *
-+ * This helper function allows drivers to get all clocks of the
-+ * consumer and enables them in one operation with management.
-+ * The clks will automatically be disabled and freed when the device
-+ * is unbound.
-+ */
-+
-+int __must_check devm_clk_bulk_get_all_enabled(struct device *dev,
-+					       struct clk_bulk_data **clks);
-+
-+/**
-  * devm_clk_get - lookup and obtain a managed reference to a clock producer.
-  * @dev: device for clock "consumer"
-  * @id: clock consumer ID
-@@ -568,6 +643,32 @@ struct clk *devm_clk_get_optional_prepared(struct device *dev, const char *id);
- struct clk *devm_clk_get_optional_enabled(struct device *dev, const char *id);
- 
- /**
-+ * devm_clk_get_optional_enabled_with_rate - devm_clk_get_optional() +
-+ *                                           clk_set_rate() +
-+ *                                           clk_prepare_enable()
-+ * @dev: device for clock "consumer"
-+ * @id: clock consumer ID
-+ * @rate: new clock rate
-+ *
-+ * Context: May sleep.
-+ *
-+ * Return: a struct clk corresponding to the clock producer, or
-+ * valid IS_ERR() condition containing errno.  The implementation
-+ * uses @dev and @id to determine the clock consumer, and thereby
-+ * the clock producer.  If no such clk is found, it returns NULL
-+ * which serves as a dummy clk.  That's the only difference compared
-+ * to devm_clk_get_enabled().
-+ *
-+ * The returned clk (if valid) is prepared and enabled and rate was set.
-+ *
-+ * The clock will automatically be disabled, unprepared and freed
-+ * when the device is unbound from the bus.
-+ */
-+struct clk *devm_clk_get_optional_enabled_with_rate(struct device *dev,
-+						    const char *id,
-+						    unsigned long rate);
-+
-+/**
-  * devm_get_clk_from_child - lookup and obtain a managed reference to a
-  *			     clock producer from child node.
-  * @dev: device for clock "consumer"
-@@ -583,38 +684,6 @@ struct clk *devm_clk_get_optional_enabled(struct device *dev, const char *id);
-  */
- struct clk *devm_get_clk_from_child(struct device *dev,
- 				    struct device_node *np, const char *con_id);
-/**
- * clk_rate_exclusive_get - get exclusivity over the rate control of a
- *                          producer
- * @clk: clock source
- *
- * This function allows drivers to get exclusive control over the rate of a
- * provider. It prevents any other consumer to execute, even indirectly,
- * opereation which could alter the rate of the provider or cause glitches
- *
- * If exlusivity is claimed more than once on clock, even by the same driver,
- * the rate effectively gets locked as exclusivity can't be preempted.
- *
- * Must not be called from within atomic context.
- *
- * Returns success (0) or negative errno.
- */
-int clk_rate_exclusive_get(struct clk *clk);
-
-/**
- * clk_rate_exclusive_put - release exclusivity over the rate control of a
- *                          producer
- * @clk: clock source
- *
- * This function allows drivers to release the exclusivity it previously got
- * from clk_rate_exclusive_get()
- *
- * The caller must balance the number of clk_rate_exclusive_get() and
- * clk_rate_exclusive_put() calls.
- *
- * Must not be called from within atomic context.
- */
-void clk_rate_exclusive_put(struct clk *clk);
- 
- /**
-  * clk_enable - inform the system when the clock source should be running.
-@@ -799,7 +868,7 @@ int clk_set_rate_exclusive(struct clk *clk, unsigned long rate);
-  *
-  * Returns true if @parent is a possible parent for @clk, false otherwise.
-  */
-bool clk_has_parent(struct clk *clk, struct clk *parent);
-+bool clk_has_parent(const struct clk *clk, const struct clk *parent);
- 
- /**
-  * clk_set_rate_range - set a rate range for a clock source
-@@ -941,6 +1010,13 @@ static inline struct clk *devm_clk_get_optional_enabled(struct device *dev,
- 	return NULL;
- }
- 
-+static inline struct clk *
-+devm_clk_get_optional_enabled_with_rate(struct device *dev, const char *id,
-+					unsigned long rate)
-+{
-+	return NULL;
-+}
-+
- static inline int __must_check devm_clk_bulk_get(struct device *dev, int num_clks,
- 						 struct clk_bulk_data *clks)
- {
-@@ -960,6 +1036,12 @@ static inline int __must_check devm_clk_bulk_get_all(struct device *dev,
- 	return 0;
- }
- 
-+static inline int __must_check devm_clk_bulk_get_all_enabled(struct device *dev,
-+						struct clk_bulk_data **clks)
-+{
-+	return 0;
-+}
-+
- static inline struct clk *devm_get_clk_from_child(struct device *dev,
- 				struct device_node *np, const char *con_id)
- {
-@@ -974,14 +1056,6 @@ static inline void clk_bulk_put_all(int num_clks, struct clk_bulk_data *clks) {}
- 
- static inline void devm_clk_put(struct device *dev, struct clk *clk) {}
- 
-
-static inline int clk_rate_exclusive_get(struct clk *clk)
-{
-	return 0;
-}
-
-static inline void clk_rate_exclusive_put(struct clk *clk) {}
-
- static inline int clk_enable(struct clk *clk)
- {
- 	return 0;
-diff --git a/include/linux/clk/at91_pmc.h b/include/linux/clk/at91_pmc.h
-index 3484309b59bf..d60ce9708ea2 100644
--- a/include/linux/clk/at91_pmc.h
-+++ b/include/linux/clk/at91_pmc.h
-@@ -12,6 +12,8 @@
- #ifndef AT91_PMC_H
- #define AT91_PMC_H
- 
-+#include <linux/bits.h>
-+
- #define AT91_PMC_V1		(1)			/* PMC version 1 */
- #define AT91_PMC_V2		(2)			/* PMC version 2 [SAM9X60] */
- 
-@@ -45,8 +47,6 @@
- #define	AT91_PMC_PCSR		0x18			/* Peripheral Clock Status Register */
- 
- #define AT91_PMC_PLL_ACR	0x18			/* PLL Analog Control Register [for SAM9X60] */
-#define		AT91_PMC_PLL_ACR_DEFAULT_UPLL	0x12020010UL	/* Default PLL ACR value for UPLL */
-#define		AT91_PMC_PLL_ACR_DEFAULT_PLLA	0x00020010UL	/* Default PLL ACR value for PLLA */
- #define		AT91_PMC_PLL_ACR_UTMIVR		(1 << 12)	/* UPLL Voltage regulator Control */
- #define		AT91_PMC_PLL_ACR_UTMIBG		(1 << 13)	/* UPLL Bandgap Control */
- 
-diff --git a/include/linux/clk/davinci.h b/include/linux/clk/davinci.h
-index 8a7b5cd7eac0..787a81116b00 100644
--- a/include/linux/clk/davinci.h
-+++ b/include/linux/clk/davinci.h
-@@ -12,29 +12,6 @@
- #include <linux/regmap.h>
- 
- /* function for registering clocks in early boot */
-
-#ifdef CONFIG_ARCH_DAVINCI_DA830
-int da830_pll_init(struct device *dev, void __iomem *base, struct regmap *cfgchip);
-#endif
-#ifdef CONFIG_ARCH_DAVINCI_DA850
- int da850_pll0_init(struct device *dev, void __iomem *base, struct regmap *cfgchip);
-#endif
-#ifdef CONFIG_ARCH_DAVINCI_DM355
-int dm355_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip);
-int dm355_psc_init(struct device *dev, void __iomem *base);
-#endif
-#ifdef CONFIG_ARCH_DAVINCI_DM365
-int dm365_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip);
-int dm365_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip);
-int dm365_psc_init(struct device *dev, void __iomem *base);
-#endif
-#ifdef CONFIG_ARCH_DAVINCI_DM644x
-int dm644x_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip);
-int dm644x_psc_init(struct device *dev, void __iomem *base);
-#endif
-#ifdef CONFIG_ARCH_DAVINCI_DM646x
-int dm646x_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip);
-int dm646x_psc_init(struct device *dev, void __iomem *base);
-#endif
- 
- #endif /* __LINUX_CLK_DAVINCI_PLL_H___ */
-diff --git a/include/linux/clk/mmp.h b/include/linux/clk/mmp.h
-deleted file mode 100644
-index 445130460380..000000000000
--- a/include/linux/clk/mmp.h
-+++ /dev/null
-@@ -1,18 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __CLK_MMP_H
-#define __CLK_MMP_H
-
-#include <linux/types.h>
-
-extern void pxa168_clk_init(phys_addr_t mpmu_phys,
-			    phys_addr_t apmu_phys,
-			    phys_addr_t apbc_phys);
-extern void pxa910_clk_init(phys_addr_t mpmu_phys,
-			    phys_addr_t apmu_phys,
-			    phys_addr_t apbc_phys,
-			    phys_addr_t apbcp_phys);
-extern void mmp2_clk_init(phys_addr_t mpmu_phys,
-			  phys_addr_t apmu_phys,
-			  phys_addr_t apbc_phys);
-
-#endif
-diff --git a/include/linux/clk/renesas.h b/include/linux/clk/renesas.h
-index 0ebbe2f0b45e..69d8159deee3 100644
--- a/include/linux/clk/renesas.h
-+++ b/include/linux/clk/renesas.h
-@@ -10,7 +10,9 @@
- #ifndef __LINUX_CLK_RENESAS_H_
- #define __LINUX_CLK_RENESAS_H_
- 
-+#include <linux/clk-provider.h>
- #include <linux/types.h>
-+#include <linux/units.h>
- 
- struct device;
- struct device_node;
-@@ -32,4 +34,147 @@ void cpg_mssr_detach_dev(struct generic_pm_domain *unused, struct device *dev);
- #define cpg_mssr_attach_dev	NULL
- #define cpg_mssr_detach_dev	NULL
- #endif
-+
-+/**
-+ * struct rzv2h_pll_limits - PLL parameter constraints
-+ *
-+ * This structure defines the minimum and maximum allowed values for
-+ * various parameters used to configure a PLL. These limits ensure
-+ * the PLL operates within valid and stable ranges.
-+ *
-+ * @fout: Output frequency range (in MHz)
-+ * @fout.min: Minimum allowed output frequency
-+ * @fout.max: Maximum allowed output frequency
-+ *
-+ * @fvco: PLL oscillation frequency range (in MHz)
-+ * @fvco.min: Minimum allowed VCO frequency
-+ * @fvco.max: Maximum allowed VCO frequency
-+ *
-+ * @m: Main-divider range
-+ * @m.min: Minimum main-divider value
-+ * @m.max: Maximum main-divider value
-+ *
-+ * @p: Pre-divider range
-+ * @p.min: Minimum pre-divider value
-+ * @p.max: Maximum pre-divider value
-+ *
-+ * @s: Divider range
-+ * @s.min: Minimum divider value
-+ * @s.max: Maximum divider value
-+ *
-+ * @k: Delta-sigma modulator range (signed)
-+ * @k.min: Minimum delta-sigma value
-+ * @k.max: Maximum delta-sigma value
-+ */
-+struct rzv2h_pll_limits {
-+	struct {
-+		u32 min;
-+		u32 max;
-+	} fout;
-+
-+	struct {
-+		u32 min;
-+		u32 max;
-+	} fvco;
-+
-+	struct {
-+		u16 min;
-+		u16 max;
-+	} m;
-+
-+	struct {
-+		u8 min;
-+		u8 max;
-+	} p;
-+
-+	struct {
-+		u8 min;
-+		u8 max;
-+	} s;
-+
-+	struct {
-+		s16 min;
-+		s16 max;
-+	} k;
-+};
-+
-+/**
-+ * struct rzv2h_pll_pars - PLL configuration parameters
-+ *
-+ * This structure contains the configuration parameters for the
-+ * Phase-Locked Loop (PLL), used to achieve a specific output frequency.
-+ *
-+ * @m: Main divider value
-+ * @p: Pre-divider value
-+ * @s: Output divider value
-+ * @k: Delta-sigma modulation value
-+ * @freq_millihz: Calculated PLL output frequency in millihertz
-+ * @error_millihz: Frequency error from target in millihertz (signed)
-+ */
-+struct rzv2h_pll_pars {
-+	u16 m;
-+	u8 p;
-+	u8 s;
-+	s16 k;
-+	u64 freq_millihz;
-+	s64 error_millihz;
-+};
-+
-+/**
-+ * struct rzv2h_pll_div_pars - PLL parameters with post-divider
-+ *
-+ * This structure is used for PLLs that include an additional post-divider
-+ * stage after the main PLL block. It contains both the PLL configuration
-+ * parameters and the resulting frequency/error values after the divider.
-+ *
-+ * @pll: Main PLL configuration parameters (see struct rzv2h_pll_pars)
-+ *
-+ * @div: Post-divider configuration and result
-+ * @div.divider_value: Divider applied to the PLL output
-+ * @div.freq_millihz: Output frequency after divider in millihertz
-+ * @div.error_millihz: Frequency error from target in millihertz (signed)
-+ */
-+struct rzv2h_pll_div_pars {
-+	struct rzv2h_pll_pars pll;
-+	struct {
-+		u8 divider_value;
-+		u64 freq_millihz;
-+		s64 error_millihz;
-+	} div;
-+};
-+
-+#define RZV2H_CPG_PLL_DSI_LIMITS(name)					\
-+	static const struct rzv2h_pll_limits (name) = {			\
-+		.fout = { .min = 25 * MEGA, .max = 375 * MEGA },	\
-+		.fvco = { .min = 1600 * MEGA, .max = 3200 * MEGA },	\
-+		.m = { .min = 64, .max = 533 },				\
-+		.p = { .min = 1, .max = 4 },				\
-+		.s = { .min = 0, .max = 6 },				\
-+		.k = { .min = -32768, .max = 32767 },			\
-+	}								\
-+
-+#ifdef CONFIG_CLK_RZV2H
-+bool rzv2h_get_pll_pars(const struct rzv2h_pll_limits *limits,
-+			struct rzv2h_pll_pars *pars, u64 freq_millihz);
-+
-+bool rzv2h_get_pll_divs_pars(const struct rzv2h_pll_limits *limits,
-+			     struct rzv2h_pll_div_pars *pars,
-+			     const u8 *table, u8 table_size, u64 freq_millihz);
-+#else
-+static inline bool rzv2h_get_pll_pars(const struct rzv2h_pll_limits *limits,
-+				      struct rzv2h_pll_pars *pars,
-+				      u64 freq_millihz)
-+{
-+	return false;
-+}
-+
-+static inline bool rzv2h_get_pll_divs_pars(const struct rzv2h_pll_limits *limits,
-+					   struct rzv2h_pll_div_pars *pars,
-+					   const u8 *table, u8 table_size,
-+					   u64 freq_millihz)
-+{
-+	return false;
-+}
-+#endif
-+
- #endif
-diff --git a/include/linux/clk/samsung.h b/include/linux/clk/samsung.h
-index 38b774001712..0cf7aac83439 100644
--- a/include/linux/clk/samsung.h
-+++ b/include/linux/clk/samsung.h
-@@ -21,36 +21,4 @@ static inline void s3c64xx_clk_init(struct device_node *np,
- 				    bool s3c6400, void __iomem *base) { }
- #endif /* CONFIG_S3C64XX_COMMON_CLK */
- 
-#ifdef CONFIG_S3C2410_COMMON_CLK
-void s3c2410_common_clk_init(struct device_node *np, unsigned long xti_f,
-			     int current_soc,
-			     void __iomem *reg_base);
-#else
-static inline void s3c2410_common_clk_init(struct device_node *np,
-					   unsigned long xti_f,
-					   int current_soc,
-					   void __iomem *reg_base) { }
-#endif /* CONFIG_S3C2410_COMMON_CLK */
-
-#ifdef CONFIG_S3C2412_COMMON_CLK
-void s3c2412_common_clk_init(struct device_node *np, unsigned long xti_f,
-			     unsigned long ext_f, void __iomem *reg_base);
-#else
-static inline void s3c2412_common_clk_init(struct device_node *np,
-					   unsigned long xti_f,
-					   unsigned long ext_f,
-					   void __iomem *reg_base) { }
-#endif /* CONFIG_S3C2412_COMMON_CLK */
-
-#ifdef CONFIG_S3C2443_COMMON_CLK
-void s3c2443_common_clk_init(struct device_node *np, unsigned long xti_f,
-			     int current_soc,
-			     void __iomem *reg_base);
-#else
-static inline void s3c2443_common_clk_init(struct device_node *np,
-					   unsigned long xti_f,
-					   int current_soc,
-					   void __iomem *reg_base) { }
-#endif /* CONFIG_S3C2443_COMMON_CLK */
-
- #endif /* __LINUX_CLK_SAMSUNG_H_ */
-diff --git a/include/linux/clk/spear.h b/include/linux/clk/spear.h
-index a64d034ceddd..eaf95ca656f8 100644
--- a/include/linux/clk/spear.h
-+++ b/include/linux/clk/spear.h
-@@ -8,6 +8,20 @@
- #ifndef __LINUX_CLK_SPEAR_H
- #define __LINUX_CLK_SPEAR_H
- 
-+#ifdef CONFIG_ARCH_SPEAR3XX
-+void __init spear3xx_clk_init(void __iomem *misc_base,
-+			      void __iomem *soc_config_base);
-+#else
-+static inline void __init spear3xx_clk_init(void __iomem *misc_base,
-+					    void __iomem *soc_config_base) {}
-+#endif
-+
-+#ifdef CONFIG_ARCH_SPEAR6XX
-+void __init spear6xx_clk_init(void __iomem *misc_base);
-+#else
-+static inline void __init spear6xx_clk_init(void __iomem *misc_base) {}
-+#endif
-+
- #ifdef CONFIG_MACH_SPEAR1310
- void __init spear1310_clk_init(void __iomem *misc_base, void __iomem *ras_base);
- #else
-diff --git a/include/linux/clk/ti.h b/include/linux/clk/ti.h
-index cbfcbf186ce3..54a3fa370004 100644
--- a/include/linux/clk/ti.h
-+++ b/include/linux/clk/ti.h
-@@ -13,11 +13,14 @@
- /**
-  * struct clk_omap_reg - OMAP register declaration
-  * @offset: offset from the master IP module base address
-+ * @bit: register bit offset
-  * @index: index of the master IP module
-+ * @flags: flags
-  */
- struct clk_omap_reg {
- 	void __iomem *ptr;
- 	u16 offset;
-+	u8 bit;
- 	u8 index;
- 	u8 flags;
- };
-@@ -31,14 +34,14 @@ struct clk_omap_reg {
-  * @clk_ref: struct clk_hw pointer to the clock's reference clock input
-  * @control_reg: register containing the DPLL mode bitfield
-  * @enable_mask: mask of the DPLL mode bitfield in @control_reg
- * @last_rounded_rate: cache of the last rate result of omap2_dpll_round_rate()
- * @last_rounded_m: cache of the last M result of omap2_dpll_round_rate()
-+ * @last_rounded_rate: cache of the last rate result of omap2_dpll_determine_rate()
-+ * @last_rounded_m: cache of the last M result of omap2_dpll_determine_rate()
-  * @last_rounded_m4xen: cache of the last M4X result of
- *			omap4_dpll_regm4xen_round_rate()
-+ *			omap4_dpll_regm4xen_determine_rate()
-  * @last_rounded_lpmode: cache of the last lpmode result of
-  *			 omap4_dpll_lpmode_recalc()
-  * @max_multiplier: maximum valid non-bypass multiplier value (actual)
- * @last_rounded_n: cache of the last N result of omap2_dpll_round_rate()
-+ * @last_rounded_n: cache of the last N result of omap2_dpll_determine_rate()
-  * @min_divider: minimum valid non-bypass divider value (actual)
-  * @max_divider: maximum valid non-bypass divider value (actual)
-  * @max_rate: maximum clock rate for the DPLL
-diff --git a/include/linux/clkdev.h b/include/linux/clkdev.h
-index 8a8423eb8e9a..45570bc21a43 100644
--- a/include/linux/clkdev.h
-+++ b/include/linux/clkdev.h
-@@ -46,6 +46,4 @@ int clk_hw_register_clkdev(struct clk_hw *, const char *, const char *);
- 
- int devm_clk_hw_register_clkdev(struct device *dev, struct clk_hw *hw,
- 				const char *con_id, const char *dev_id);
-void devm_clk_release_clkdev(struct device *dev, const char *con_id,
-			     const char *dev_id);
- #endif
-diff --git a/include/linux/clockchips.h b/include/linux/clockchips.h
-index 8ae9a95ebf5b..b0df28ddd394 100644
--- a/include/linux/clockchips.h
-+++ b/include/linux/clockchips.h
-@@ -12,7 +12,7 @@
- #ifdef CONFIG_GENERIC_CLOCKEVENTS
- 
- # include <linux/clocksource.h>
-# include <linux/cpumask.h>
-+# include <linux/cpumask_types.h>
- # include <linux/ktime.h>
- # include <linux/notifier.h>
- 
-@@ -211,7 +211,7 @@ extern int tick_receive_broadcast(void);
- extern void tick_setup_hrtimer_broadcast(void);
- extern int tick_check_broadcast_expired(void);
- # else
-static inline int tick_check_broadcast_expired(void) { return 0; }
-+static __always_inline int tick_check_broadcast_expired(void) { return 0; }
- static inline void tick_setup_hrtimer_broadcast(void) { }
- # endif
- 
-@@ -219,7 +219,7 @@ static inline void tick_setup_hrtimer_broadcast(void) { }
- 
- static inline void clockevents_suspend(void) { }
- static inline void clockevents_resume(void) { }
-static inline int tick_check_broadcast_expired(void) { return 0; }
-+static __always_inline int tick_check_broadcast_expired(void) { return 0; }
- static inline void tick_setup_hrtimer_broadcast(void) { }
- 
- #endif /* !CONFIG_GENERIC_CLOCKEVENTS */
-diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h
-index 1d42d4b17327..65b7c41471c3 100644
--- a/include/linux/clocksource.h
-+++ b/include/linux/clocksource.h
-@@ -21,6 +21,7 @@
- #include <asm/div64.h>
- #include <asm/io.h>
- 
-+struct clocksource_base;
- struct clocksource;
- struct module;
- 
-@@ -48,8 +49,10 @@ struct module;
-  * @archdata:		Optional arch-specific data
-  * @max_cycles:		Maximum safe cycle value which won't overflow on
-  *			multiplication
-+ * @max_raw_delta:	Maximum safe delta value for negative motion detection
-  * @name:		Pointer to clocksource name
-  * @list:		List head for registration (internal)
-+ * @freq_khz:		Clocksource frequency in khz.
-  * @rating:		Rating value for selection (higher is better)
-  *			To avoid rating inflation the following
-  *			list should give you a guide as to how
-@@ -70,6 +73,8 @@ struct module;
-  *			validate the clocksource from which the snapshot was
-  *			taken.
-  * @flags:		Flags describing special properties
-+ * @base:		Hardware abstraction for clock on which a clocksource
-+ *			is based
-  * @enable:		Optional function to enable the clocksource
-  * @disable:		Optional function to disable the clocksource
-  * @suspend:		Optional suspend function for the clocksource
-@@ -105,12 +110,15 @@ struct clocksource {
- 	struct arch_clocksource_data archdata;
- #endif
- 	u64			max_cycles;
-+	u64			max_raw_delta;
- 	const char		*name;
- 	struct list_head	list;
-+	u32			freq_khz;
- 	int			rating;
- 	enum clocksource_ids	id;
- 	enum vdso_clock_mode	vdso_clock_mode;
- 	unsigned long		flags;
-+	struct clocksource_base *base;
- 
- 	int			(*enable)(struct clocksource *cs);
- 	void			(*disable)(struct clocksource *cs);
-@@ -209,7 +217,6 @@ static inline s64 clocksource_cyc2ns(u64 cycles, u32 mult, u32 shift)
- 
- extern int clocksource_unregister(struct clocksource*);
- extern void clocksource_touch_watchdog(void);
-extern void clocksource_change_rating(struct clocksource *cs, int rating);
- extern void clocksource_suspend(void);
- extern void clocksource_resume(void);
- extern struct clocksource * __init clocksource_default_clock(void);
-@@ -291,7 +298,40 @@ static inline void timer_probe(void) {}
- #define TIMER_ACPI_DECLARE(name, table_id, fn)		\
- 	ACPI_DECLARE_PROBE_ENTRY(timer, name, table_id, 0, NULL, 0, fn)
- 
-extern ulong max_cswd_read_retries;
-+static inline unsigned int clocksource_get_max_watchdog_retry(void)
-+{
-+	/*
-+	 * When system is in the boot phase or under heavy workload, there
-+	 * can be random big latencies during the clocksource/watchdog
-+	 * read, so allow retries to filter the noise latency. As the
-+	 * latency's frequency and maximum value goes up with the number of
-+	 * CPUs, scale the number of retries with the number of online
-+	 * CPUs.
-+	 */
-+	return (ilog2(num_online_cpus()) / 2) + 1;
-+}
-+
- void clocksource_verify_percpu(struct clocksource *cs);
- 
-+/**
-+ * struct clocksource_base - hardware abstraction for clock on which a clocksource
-+ *			is based
-+ * @id:			Defaults to CSID_GENERIC. The id value is used for conversion
-+ *			functions which require that the current clocksource is based
-+ *			on a clocksource_base with a particular ID in certain snapshot
-+ *			functions to allow callers to validate the clocksource from
-+ *			which the snapshot was taken.
-+ * @freq_khz:		Nominal frequency of the base clock in kHz
-+ * @offset:		Offset between the base clock and the clocksource
-+ * @numerator:		Numerator of the clock ratio between base clock and the clocksource
-+ * @denominator:	Denominator of the clock ratio between base clock and the clocksource
-+ */
-+struct clocksource_base {
-+	enum clocksource_ids	id;
-+	u32			freq_khz;
-+	u64			offset;
-+	u32			numerator;
-+	u32			denominator;
-+};
-+
- #endif /* _LINUX_CLOCKSOURCE_H */
-diff --git a/include/linux/clocksource_ids.h b/include/linux/clocksource_ids.h
-index 16775d7d8f8d..c4ef4ae2eded 100644
--- a/include/linux/clocksource_ids.h
-+++ b/include/linux/clocksource_ids.h
-@@ -6,6 +6,11 @@
- enum clocksource_ids {
- 	CSID_GENERIC		= 0,
- 	CSID_ARM_ARCH_COUNTER,
-+	CSID_S390_TOD,
-+	CSID_X86_TSC_EARLY,
-+	CSID_X86_TSC,
-+	CSID_X86_KVM_CLK,
-+	CSID_X86_ART,
- 	CSID_MAX,
- };
- 
-diff --git a/include/linux/closure.h b/include/linux/closure.h
-new file mode 100644
-index 000000000000..880fe85e35e9
--- /dev/null
-+++ b/include/linux/closure.h
-@@ -0,0 +1,492 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_CLOSURE_H
-+#define _LINUX_CLOSURE_H
-+
-+#include <linux/llist.h>
-+#include <linux/sched.h>
-+#include <linux/sched/task_stack.h>
-+#include <linux/workqueue.h>
-+
-+/*
-+ * Closure is perhaps the most overused and abused term in computer science, but
-+ * since I've been unable to come up with anything better you're stuck with it
-+ * again.
-+ *
-+ * What are closures?
-+ *
-+ * They embed a refcount. The basic idea is they count "things that are in
-+ * progress" - in flight bios, some other thread that's doing something else -
-+ * anything you might want to wait on.
-+ *
-+ * The refcount may be manipulated with closure_get() and closure_put().
-+ * closure_put() is where many of the interesting things happen, when it causes
-+ * the refcount to go to 0.
-+ *
-+ * Closures can be used to wait on things both synchronously and asynchronously,
-+ * and synchronous and asynchronous use can be mixed without restriction. To
-+ * wait synchronously, use closure_sync() - you will sleep until your closure's
-+ * refcount hits 1.
-+ *
-+ * To wait asynchronously, use
-+ *   continue_at(cl, next_function, workqueue);
-+ *
-+ * passing it, as you might expect, the function to run when nothing is pending
-+ * and the workqueue to run that function out of.
-+ *
-+ * continue_at() also, critically, requires a 'return' immediately following the
-+ * location where this macro is referenced, to return to the calling function.
-+ * There's good reason for this.
-+ *
-+ * To use safely closures asynchronously, they must always have a refcount while
-+ * they are running owned by the thread that is running them. Otherwise, suppose
-+ * you submit some bios and wish to have a function run when they all complete:
-+ *
-+ * foo_endio(struct bio *bio)
-+ * {
-+ *	closure_put(cl);
-+ * }
-+ *
-+ * closure_init(cl);
-+ *
-+ * do_stuff();
-+ * closure_get(cl);
-+ * bio1->bi_endio = foo_endio;
-+ * bio_submit(bio1);
-+ *
-+ * do_more_stuff();
-+ * closure_get(cl);
-+ * bio2->bi_endio = foo_endio;
-+ * bio_submit(bio2);
-+ *
-+ * continue_at(cl, complete_some_read, system_wq);
-+ *
-+ * If closure's refcount started at 0, complete_some_read() could run before the
-+ * second bio was submitted - which is almost always not what you want! More
-+ * importantly, it wouldn't be possible to say whether the original thread or
-+ * complete_some_read()'s thread owned the closure - and whatever state it was
-+ * associated with!
-+ *
-+ * So, closure_init() initializes a closure's refcount to 1 - and when a
-+ * closure_fn is run, the refcount will be reset to 1 first.
-+ *
-+ * Then, the rule is - if you got the refcount with closure_get(), release it
-+ * with closure_put() (i.e, in a bio->bi_endio function). If you have a refcount
-+ * on a closure because you called closure_init() or you were run out of a
-+ * closure - _always_ use continue_at(). Doing so consistently will help
-+ * eliminate an entire class of particularly pernicious races.
-+ *
-+ * Lastly, you might have a wait list dedicated to a specific event, and have no
-+ * need for specifying the condition - you just want to wait until someone runs
-+ * closure_wake_up() on the appropriate wait list. In that case, just use
-+ * closure_wait(). It will return either true or false, depending on whether the
-+ * closure was already on a wait list or not - a closure can only be on one wait
-+ * list at a time.
-+ *
-+ * Parents:
-+ *
-+ * closure_init() takes two arguments - it takes the closure to initialize, and
-+ * a (possibly null) parent.
-+ *
-+ * If parent is non null, the new closure will have a refcount for its lifetime;
-+ * a closure is considered to be "finished" when its refcount hits 0 and the
-+ * function to run is null. Hence
-+ *
-+ * continue_at(cl, NULL, NULL);
-+ *
-+ * returns up the (spaghetti) stack of closures, precisely like normal return
-+ * returns up the C stack. continue_at() with non null fn is better thought of
-+ * as doing a tail call.
-+ *
-+ * All this implies that a closure should typically be embedded in a particular
-+ * struct (which its refcount will normally control the lifetime of), and that
-+ * struct can very much be thought of as a stack frame.
-+ */
-+
-+struct closure;
-+struct closure_syncer;
-+typedef void (closure_fn) (struct work_struct *);
-+extern struct dentry *bcache_debug;
-+
-+struct closure_waitlist {
-+	struct llist_head	list;
-+};
-+
-+enum closure_state {
-+	/*
-+	 * CLOSURE_WAITING: Set iff the closure is on a waitlist. Must be set by
-+	 * the thread that owns the closure, and cleared by the thread that's
-+	 * waking up the closure.
-+	 *
-+	 * The rest are for debugging and don't affect behaviour:
-+	 *
-+	 * CLOSURE_RUNNING: Set when a closure is running (i.e. by
-+	 * closure_init() and when closure_put() runs then next function), and
-+	 * must be cleared before remaining hits 0. Primarily to help guard
-+	 * against incorrect usage and accidentally transferring references.
-+	 * continue_at() and closure_return() clear it for you, if you're doing
-+	 * something unusual you can use closure_set_dead() which also helps
-+	 * annotate where references are being transferred.
-+	 */
-+
-+	CLOSURE_BITS_START	= (1U << 26),
-+	CLOSURE_DESTRUCTOR	= (1U << 26),
-+	CLOSURE_WAITING		= (1U << 28),
-+	CLOSURE_RUNNING		= (1U << 30),
-+};
-+
-+#define CLOSURE_GUARD_MASK					\
-+	((CLOSURE_DESTRUCTOR|CLOSURE_WAITING|CLOSURE_RUNNING) << 1)
-+
-+#define CLOSURE_REMAINING_MASK		(CLOSURE_BITS_START - 1)
-+#define CLOSURE_REMAINING_INITIALIZER	(1|CLOSURE_RUNNING)
-+
-+struct closure {
-+	union {
-+		struct {
-+			struct workqueue_struct *wq;
-+			struct closure_syncer	*s;
-+			struct llist_node	list;
-+			closure_fn		*fn;
-+		};
-+		struct work_struct	work;
-+	};
-+
-+	struct closure		*parent;
-+
-+	atomic_t		remaining;
-+	bool			closure_get_happened;
-+
-+#ifdef CONFIG_DEBUG_CLOSURES
-+#define CLOSURE_MAGIC_DEAD	0xc054dead
-+#define CLOSURE_MAGIC_ALIVE	0xc054a11e
-+#define CLOSURE_MAGIC_STACK	0xc05451cc
-+
-+	unsigned int		magic;
-+	struct list_head	all;
-+	unsigned long		ip;
-+	unsigned long		waiting_on;
-+#endif
-+};
-+
-+void closure_sub(struct closure *cl, int v);
-+void closure_put(struct closure *cl);
-+void __closure_wake_up(struct closure_waitlist *list);
-+bool closure_wait(struct closure_waitlist *list, struct closure *cl);
-+void __closure_sync(struct closure *cl);
-+
-+static inline unsigned closure_nr_remaining(struct closure *cl)
-+{
-+	return atomic_read(&cl->remaining) & CLOSURE_REMAINING_MASK;
-+}
-+
-+/**
-+ * closure_sync - sleep until a closure a closure has nothing left to wait on
-+ *
-+ * Sleeps until the refcount hits 1 - the thread that's running the closure owns
-+ * the last refcount.
-+ */
-+static inline void closure_sync(struct closure *cl)
-+{
-+#ifdef CONFIG_DEBUG_CLOSURES
-+	BUG_ON(closure_nr_remaining(cl) != 1 && !cl->closure_get_happened);
-+#endif
-+
-+	if (cl->closure_get_happened)
-+		__closure_sync(cl);
-+}
-+
-+int __closure_sync_timeout(struct closure *cl, unsigned long timeout);
-+
-+static inline int closure_sync_timeout(struct closure *cl, unsigned long timeout)
-+{
-+#ifdef CONFIG_DEBUG_CLOSURES
-+	BUG_ON(closure_nr_remaining(cl) != 1 && !cl->closure_get_happened);
-+#endif
-+	return cl->closure_get_happened
-+		? __closure_sync_timeout(cl, timeout)
-+		: 0;
-+}
-+
-+#ifdef CONFIG_DEBUG_CLOSURES
-+
-+void closure_debug_create(struct closure *cl);
-+void closure_debug_destroy(struct closure *cl);
-+
-+#else
-+
-+static inline void closure_debug_create(struct closure *cl) {}
-+static inline void closure_debug_destroy(struct closure *cl) {}
-+
-+#endif
-+
-+static inline void closure_set_ip(struct closure *cl)
-+{
-+#ifdef CONFIG_DEBUG_CLOSURES
-+	cl->ip = _THIS_IP_;
-+#endif
-+}
-+
-+static inline void closure_set_ret_ip(struct closure *cl)
-+{
-+#ifdef CONFIG_DEBUG_CLOSURES
-+	cl->ip = _RET_IP_;
-+#endif
-+}
-+
-+static inline void closure_set_waiting(struct closure *cl, unsigned long f)
-+{
-+#ifdef CONFIG_DEBUG_CLOSURES
-+	cl->waiting_on = f;
-+#endif
-+}
-+
-+static inline void closure_set_stopped(struct closure *cl)
-+{
-+	atomic_sub(CLOSURE_RUNNING, &cl->remaining);
-+}
-+
-+static inline void set_closure_fn(struct closure *cl, closure_fn *fn,
-+				  struct workqueue_struct *wq)
-+{
-+	closure_set_ip(cl);
-+	cl->fn = fn;
-+	cl->wq = wq;
-+}
-+
-+static inline void closure_queue(struct closure *cl)
-+{
-+	struct workqueue_struct *wq = cl->wq;
-+	/**
-+	 * Changes made to closure, work_struct, or a couple of other structs
-+	 * may cause work.func not pointing to the right location.
-+	 */
-+	BUILD_BUG_ON(offsetof(struct closure, fn)
-+		     != offsetof(struct work_struct, func));
-+
-+	if (wq) {
-+		INIT_WORK(&cl->work, cl->work.func);
-+		BUG_ON(!queue_work(wq, &cl->work));
-+	} else
-+		cl->fn(&cl->work);
-+}
-+
-+/**
-+ * closure_get - increment a closure's refcount
-+ */
-+static inline void closure_get(struct closure *cl)
-+{
-+	cl->closure_get_happened = true;
-+
-+#ifdef CONFIG_DEBUG_CLOSURES
-+	BUG_ON((atomic_inc_return(&cl->remaining) &
-+		CLOSURE_REMAINING_MASK) <= 1);
-+#else
-+	atomic_inc(&cl->remaining);
-+#endif
-+}
-+
-+/**
-+ * closure_get_not_zero
-+ */
-+static inline bool closure_get_not_zero(struct closure *cl)
-+{
-+	unsigned old = atomic_read(&cl->remaining);
-+	do {
-+		if (!(old & CLOSURE_REMAINING_MASK))
-+			return false;
-+
-+	} while (!atomic_try_cmpxchg_acquire(&cl->remaining, &old, old + 1));
-+
-+	return true;
-+}
-+
-+/**
-+ * closure_init - Initialize a closure, setting the refcount to 1
-+ * @cl:		closure to initialize
-+ * @parent:	parent of the new closure. cl will take a refcount on it for its
-+ *		lifetime; may be NULL.
-+ */
-+static inline void closure_init(struct closure *cl, struct closure *parent)
-+{
-+	cl->fn = NULL;
-+	cl->parent = parent;
-+	if (parent)
-+		closure_get(parent);
-+
-+	atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER);
-+	cl->closure_get_happened = false;
-+
-+	closure_debug_create(cl);
-+	closure_set_ip(cl);
-+}
-+
-+static inline void closure_init_stack(struct closure *cl)
-+{
-+	memset(cl, 0, sizeof(struct closure));
-+	atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER);
-+#ifdef CONFIG_DEBUG_CLOSURES
-+	cl->magic = CLOSURE_MAGIC_STACK;
-+#endif
-+}
-+
-+static inline void closure_init_stack_release(struct closure *cl)
-+{
-+	memset(cl, 0, sizeof(struct closure));
-+	atomic_set_release(&cl->remaining, CLOSURE_REMAINING_INITIALIZER);
-+#ifdef CONFIG_DEBUG_CLOSURES
-+	cl->magic = CLOSURE_MAGIC_STACK;
-+#endif
-+}
-+
-+/**
-+ * closure_wake_up - wake up all closures on a wait list,
-+ *		     with memory barrier
-+ */
-+static inline void closure_wake_up(struct closure_waitlist *list)
-+{
-+	/* Memory barrier for the wait list */
-+	smp_mb();
-+	__closure_wake_up(list);
-+}
-+
-+#define CLOSURE_CALLBACK(name)	void name(struct work_struct *ws)
-+#define closure_type(name, type, member)				\
-+	struct closure *cl = container_of(ws, struct closure, work);	\
-+	type *name = container_of(cl, type, member)
-+
-+/**
-+ * continue_at - jump to another function with barrier
-+ *
-+ * After @cl is no longer waiting on anything (i.e. all outstanding refs have
-+ * been dropped with closure_put()), it will resume execution at @fn running out
-+ * of @wq (or, if @wq is NULL, @fn will be called by closure_put() directly).
-+ *
-+ * This is because after calling continue_at() you no longer have a ref on @cl,
-+ * and whatever @cl owns may be freed out from under you - a running closure fn
-+ * has a ref on its own closure which continue_at() drops.
-+ *
-+ * Note you are expected to immediately return after using this macro.
-+ */
-+#define continue_at(_cl, _fn, _wq)					\
-+do {									\
-+	set_closure_fn(_cl, _fn, _wq);					\
-+	closure_sub(_cl, CLOSURE_RUNNING + 1);				\
-+} while (0)
-+
-+/**
-+ * closure_return - finish execution of a closure
-+ *
-+ * This is used to indicate that @cl is finished: when all outstanding refs on
-+ * @cl have been dropped @cl's ref on its parent closure (as passed to
-+ * closure_init()) will be dropped, if one was specified - thus this can be
-+ * thought of as returning to the parent closure.
-+ */
-+#define closure_return(_cl)	continue_at((_cl), NULL, NULL)
-+
-+void closure_return_sync(struct closure *cl);
-+
-+/**
-+ * continue_at_nobarrier - jump to another function without barrier
-+ *
-+ * Causes @fn to be executed out of @cl, in @wq context (or called directly if
-+ * @wq is NULL).
-+ *
-+ * The ref the caller of continue_at_nobarrier() had on @cl is now owned by @fn,
-+ * thus it's not safe to touch anything protected by @cl after a
-+ * continue_at_nobarrier().
-+ */
-+#define continue_at_nobarrier(_cl, _fn, _wq)				\
-+do {									\
-+	set_closure_fn(_cl, _fn, _wq);					\
-+	closure_queue(_cl);						\
-+} while (0)
-+
-+/**
-+ * closure_return_with_destructor - finish execution of a closure,
-+ *				    with destructor
-+ *
-+ * Works like closure_return(), except @destructor will be called when all
-+ * outstanding refs on @cl have been dropped; @destructor may be used to safely
-+ * free the memory occupied by @cl, and it is called with the ref on the parent
-+ * closure still held - so @destructor could safely return an item to a
-+ * freelist protected by @cl's parent.
-+ */
-+#define closure_return_with_destructor(_cl, _destructor)		\
-+do {									\
-+	set_closure_fn(_cl, _destructor, NULL);				\
-+	closure_sub(_cl, CLOSURE_RUNNING - CLOSURE_DESTRUCTOR + 1);	\
-+} while (0)
-+
-+/**
-+ * closure_call - execute @fn out of a new, uninitialized closure
-+ *
-+ * Typically used when running out of one closure, and we want to run @fn
-+ * asynchronously out of a new closure - @parent will then wait for @cl to
-+ * finish.
-+ */
-+static inline void closure_call(struct closure *cl, closure_fn fn,
-+				struct workqueue_struct *wq,
-+				struct closure *parent)
-+{
-+	closure_init(cl, parent);
-+	continue_at_nobarrier(cl, fn, wq);
-+}
-+
-+#define __closure_wait_event(waitlist, _cond)				\
-+do {									\
-+	struct closure cl;						\
-+									\
-+	closure_init_stack(&cl);					\
-+									\
-+	while (1) {							\
-+		closure_wait(waitlist, &cl);				\
-+		if (_cond)						\
-+			break;						\
-+		closure_sync(&cl);					\
-+	}								\
-+	closure_wake_up(waitlist);					\
-+	closure_sync(&cl);						\
-+} while (0)
-+
-+#define closure_wait_event(waitlist, _cond)				\
-+do {									\
-+	if (!(_cond))							\
-+		__closure_wait_event(waitlist, _cond);			\
-+} while (0)
-+
-+#define __closure_wait_event_timeout(waitlist, _cond, _until)		\
-+({									\
-+	struct closure cl;						\
-+	long _t;							\
-+									\
-+	closure_init_stack(&cl);					\
-+									\
-+	while (1) {							\
-+		closure_wait(waitlist, &cl);				\
-+		if (_cond) {						\
-+			_t = max_t(long, 1L, _until - jiffies);		\
-+			break;						\
-+		}							\
-+		_t = max_t(long, 0L, _until - jiffies);			\
-+		if (!_t)						\
-+			break;						\
-+		closure_sync_timeout(&cl, _t);				\
-+	}								\
-+	closure_wake_up(waitlist);					\
-+	closure_sync(&cl);						\
-+	_t;								\
-+})
-+
-+/*
-+ * Returns 0 if timeout expired, remaining time in jiffies (at least 1) if
-+ * condition became true
-+ */
-+#define closure_wait_event_timeout(waitlist, _cond, _timeout)		\
-+({									\
-+	unsigned long _until = jiffies + _timeout;			\
-+	(_cond)								\
-+		? max_t(long, 1L, _until - jiffies)			\
-+		: __closure_wait_event_timeout(waitlist, _cond, _until);\
-+})
-+
-+#endif /* _LINUX_CLOSURE_H */
-diff --git a/include/linux/cm4000_cs.h b/include/linux/cm4000_cs.h
-deleted file mode 100644
-index ea4958e07a14..000000000000
--- a/include/linux/cm4000_cs.h
-+++ /dev/null
-@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef	_CM4000_H_
-#define	_CM4000_H_
-
-#include <uapi/linux/cm4000_cs.h>
-
-
-#define	DEVICE_NAME		"cmm"
-#define	MODULE_NAME		"cm4000_cs"
-
-#endif	/* _CM4000_H_ */
-diff --git a/include/linux/cma.h b/include/linux/cma.h
-index 63873b93deaa..62d9c1cf6326 100644
--- a/include/linux/cma.h
-+++ b/include/linux/cma.h
-@@ -6,12 +6,8 @@
- #include <linux/types.h>
- #include <linux/numa.h>
- 
-/*
- * There is always at least global CMA area and a few optional
- * areas configured in kernel .config.
- */
- #ifdef CONFIG_CMA_AREAS
-#define MAX_CMA_AREAS	(1 + CONFIG_CMA_AREAS)
-+#define MAX_CMA_AREAS	CONFIG_CMA_AREAS
- #endif
- 
- #define CMA_MAX_NAME 64
-@@ -44,6 +40,9 @@ static inline int __init cma_declare_contiguous(phys_addr_t base,
- 	return cma_declare_contiguous_nid(base, size, limit, alignment,
- 			order_per_bit, fixed, name, res_cma, NUMA_NO_NODE);
- }
-+extern int __init cma_declare_contiguous_multi(phys_addr_t size,
-+			phys_addr_t align, unsigned int order_per_bit,
-+			const char *name, struct cma **res_cma, int nid);
- extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size,
- 					unsigned int order_per_bit,
- 					const char *name,
-@@ -54,6 +53,28 @@ extern bool cma_pages_valid(struct cma *cma, const struct page *pages, unsigned
- extern bool cma_release(struct cma *cma, const struct page *pages, unsigned long count);
- 
- extern int cma_for_each_area(int (*it)(struct cma *cma, void *data), void *data);
-+extern bool cma_intersects(struct cma *cma, unsigned long start, unsigned long end);
- 
- extern void cma_reserve_pages_on_error(struct cma *cma);
-+
-+#ifdef CONFIG_CMA
-+struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp);
-+bool cma_free_folio(struct cma *cma, const struct folio *folio);
-+bool cma_validate_zones(struct cma *cma);
-+#else
-+static inline struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp)
-+{
-+	return NULL;
-+}
-+
-+static inline bool cma_free_folio(struct cma *cma, const struct folio *folio)
-+{
-+	return false;
-+}
-+static inline bool cma_validate_zones(struct cma *cma)
-+{
-+	return false;
-+}
-+#endif
-+
- #endif
-diff --git a/include/linux/cmpxchg-emu.h b/include/linux/cmpxchg-emu.h
-new file mode 100644
-index 000000000000..998deec67740
--- /dev/null
-+++ b/include/linux/cmpxchg-emu.h
-@@ -0,0 +1,15 @@
-+/* SPDX-License-Identifier: GPL-2.0+ */
-+/*
-+ * Emulated 1-byte and 2-byte cmpxchg operations for architectures
-+ * lacking direct support for these sizes.  These are implemented in terms
-+ * of 4-byte cmpxchg operations.
-+ *
-+ * Copyright (C) 2024 Paul E. McKenney.
-+ */
-+
-+#ifndef __LINUX_CMPXCHG_EMU_H
-+#define __LINUX_CMPXCHG_EMU_H
-+
-+uintptr_t cmpxchg_emu_u8(volatile u8 *p, uintptr_t old, uintptr_t new);
-+
-+#endif /* __LINUX_CMPXCHG_EMU_H */
-diff --git a/include/linux/codetag.h b/include/linux/codetag.h
-new file mode 100644
-index 000000000000..8ea2a5f7c98a
--- /dev/null
-+++ b/include/linux/codetag.h
-@@ -0,0 +1,115 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * code tagging framework
-+ */
-+#ifndef _LINUX_CODETAG_H
-+#define _LINUX_CODETAG_H
-+
-+#include <linux/types.h>
-+
-+struct codetag_iterator;
-+struct codetag_type;
-+struct codetag_module;
-+struct seq_buf;
-+struct module;
-+
-+#define CODETAG_SECTION_START_PREFIX	"__start_"
-+#define CODETAG_SECTION_STOP_PREFIX	"__stop_"
-+
-+/* codetag flags */
-+#define CODETAG_FLAG_INACCURATE	(1 << 0)
-+
-+/*
-+ * An instance of this structure is created in a special ELF section at every
-+ * code location being tagged.  At runtime, the special section is treated as
-+ * an array of these.
-+ */
-+struct codetag {
-+	unsigned int flags;
-+	unsigned int lineno;
-+	const char *modname;
-+	const char *function;
-+	const char *filename;
-+} __aligned(8);
-+
-+union codetag_ref {
-+	struct codetag *ct;
-+};
-+
-+struct codetag_type_desc {
-+	const char *section;
-+	size_t tag_size;
-+	int (*module_load)(struct module *mod,
-+			   struct codetag *start, struct codetag *end);
-+	void (*module_unload)(struct module *mod,
-+			      struct codetag *start, struct codetag *end);
-+#ifdef CONFIG_MODULES
-+	void (*module_replaced)(struct module *mod, struct module *new_mod);
-+	bool (*needs_section_mem)(struct module *mod, unsigned long size);
-+	void *(*alloc_section_mem)(struct module *mod, unsigned long size,
-+				   unsigned int prepend, unsigned long align);
-+	void (*free_section_mem)(struct module *mod, bool used);
-+#endif
-+};
-+
-+struct codetag_iterator {
-+	struct codetag_type *cttype;
-+	struct codetag_module *cmod;
-+	unsigned long mod_id;
-+	struct codetag *ct;
-+	unsigned long mod_seq;
-+};
-+
-+#ifdef MODULE
-+#define CT_MODULE_NAME KBUILD_MODNAME
-+#else
-+#define CT_MODULE_NAME NULL
-+#endif
-+
-+#define CODE_TAG_INIT {					\
-+	.modname	= CT_MODULE_NAME,		\
-+	.function	= __func__,			\
-+	.filename	= __FILE__,			\
-+	.lineno		= __LINE__,			\
-+	.flags		= 0,				\
-+}
-+
-+void codetag_lock_module_list(struct codetag_type *cttype, bool lock);
-+bool codetag_trylock_module_list(struct codetag_type *cttype);
-+struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype);
-+struct codetag *codetag_next_ct(struct codetag_iterator *iter);
-+
-+void codetag_to_text(struct seq_buf *out, struct codetag *ct);
-+
-+struct codetag_type *
-+codetag_register_type(const struct codetag_type_desc *desc);
-+
-+#if defined(CONFIG_CODE_TAGGING) && defined(CONFIG_MODULES)
-+
-+bool codetag_needs_module_section(struct module *mod, const char *name,
-+				  unsigned long size);
-+void *codetag_alloc_module_section(struct module *mod, const char *name,
-+				   unsigned long size, unsigned int prepend,
-+				   unsigned long align);
-+void codetag_free_module_sections(struct module *mod);
-+void codetag_module_replaced(struct module *mod, struct module *new_mod);
-+int codetag_load_module(struct module *mod);
-+void codetag_unload_module(struct module *mod);
-+
-+#else /* defined(CONFIG_CODE_TAGGING) && defined(CONFIG_MODULES) */
-+
-+static inline bool
-+codetag_needs_module_section(struct module *mod, const char *name,
-+			     unsigned long size) { return false; }
-+static inline void *
-+codetag_alloc_module_section(struct module *mod, const char *name,
-+			     unsigned long size, unsigned int prepend,
-+			     unsigned long align) { return NULL; }
-+static inline void codetag_free_module_sections(struct module *mod) {}
-+static inline void codetag_module_replaced(struct module *mod, struct module *new_mod) {}
-+static inline int codetag_load_module(struct module *mod) { return 0; }
-+static inline void codetag_unload_module(struct module *mod) {}
-+
-+#endif /* defined(CONFIG_CODE_TAGGING) && defined(CONFIG_MODULES) */
-+
-+#endif /* _LINUX_CODETAG_H */
-diff --git a/include/linux/comedi/comedi_8254.h b/include/linux/comedi/comedi_8254.h
-index d8264417e53c..d527f04400df 100644
--- a/include/linux/comedi/comedi_8254.h
-+++ b/include/linux/comedi/comedi_8254.h
-@@ -12,6 +12,8 @@
- #define _COMEDI_8254_H
- 
- #include <linux/types.h>
-+#include <linux/errno.h>
-+#include <linux/err.h>
- 
- struct comedi_device;
- struct comedi_insn;
-@@ -57,10 +59,24 @@ struct comedi_subdevice;
- /* counter maps zero to 0x10000 */
- #define I8254_MAX_COUNT			0x10000
- 
-+struct comedi_8254;
-+
-+/**
-+ * typedef comedi_8254_iocb_fn - call-back function type for 8254 register access
-+ * @i8254:		pointer to struct comedi_8254
-+ * @dir:		direction (0 = read, 1 = write)
-+ * @reg:		register number
-+ * @val:		value to write
-+ *
-+ * Return: Register value when reading, 0 when writing.
-+ */
-+typedef unsigned int comedi_8254_iocb_fn(struct comedi_8254 *i8254, int dir,
-+					 unsigned int reg, unsigned int val);
-+
- /**
-  * struct comedi_8254 - private data used by this module
- * @iobase:		PIO base address of the registers (in/out)
- * @mmio:		MMIO base address of the registers (read/write)
-+ * @iocb:		I/O call-back function for register access
-+ * @context:		context for register access (e.g. a base address)
-  * @iosize:		I/O size used to access the registers (b/w/l)
-  * @regshift:		register gap shift
-  * @osc_base:		cascaded oscillator speed in ns
-@@ -76,8 +92,8 @@ struct comedi_subdevice;
-  * @insn_config:	driver specific (*insn_config) callback
-  */
- struct comedi_8254 {
-	unsigned long iobase;
-	void __iomem *mmio;
-+	comedi_8254_iocb_fn *iocb;
-+	unsigned long context;
- 	unsigned int iosize;
- 	unsigned int regshift;
- 	unsigned int osc_base;
-@@ -122,13 +138,24 @@ void comedi_8254_set_busy(struct comedi_8254 *i8254,
- void comedi_8254_subdevice_init(struct comedi_subdevice *s,
- 				struct comedi_8254 *i8254);
- 
-struct comedi_8254 *comedi_8254_init(unsigned long iobase,
-				     unsigned int osc_base,
-				     unsigned int iosize,
-				     unsigned int regshift);
-struct comedi_8254 *comedi_8254_mm_init(void __iomem *mmio,
-					unsigned int osc_base,
-					unsigned int iosize,
-					unsigned int regshift);
-+#ifdef CONFIG_HAS_IOPORT
-+struct comedi_8254 *comedi_8254_io_alloc(unsigned long iobase,
-+					 unsigned int osc_base,
-+					 unsigned int iosize,
-+					 unsigned int regshift);
-+#else
-+static inline struct comedi_8254 *comedi_8254_io_alloc(unsigned long iobase,
-+						       unsigned int osc_base,
-+						       unsigned int iosize,
-+						       unsigned int regshift)
-+{
-+	return ERR_PTR(-ENXIO);
-+}
-+#endif
-+
-+struct comedi_8254 *comedi_8254_mm_alloc(void __iomem *mmio,
-+					 unsigned int osc_base,
-+					 unsigned int iosize,
-+					 unsigned int regshift);
- 
- #endif	/* _COMEDI_8254_H */
-diff --git a/include/linux/comedi/comedi_8255.h b/include/linux/comedi/comedi_8255.h
-index b2a5bc6b3a49..d24a69da389b 100644
--- a/include/linux/comedi/comedi_8255.h
-+++ b/include/linux/comedi/comedi_8255.h
-@@ -10,6 +10,8 @@
- #ifndef _COMEDI_8255_H
- #define _COMEDI_8255_H
- 
-+#include <linux/errno.h>
-+
- #define I8255_SIZE		0x04
- 
- #define I8255_DATA_A_REG	0x00
-@@ -27,16 +29,26 @@
- struct comedi_device;
- struct comedi_subdevice;
- 
-int subdev_8255_init(struct comedi_device *dev, struct comedi_subdevice *s,
-		     int (*io)(struct comedi_device *dev, int dir, int port,
-			       int data, unsigned long regbase),
-		     unsigned long regbase);
-+#ifdef CONFIG_HAS_IOPORT
-+int subdev_8255_io_init(struct comedi_device *dev, struct comedi_subdevice *s,
-+			unsigned long regbase);
-+#else
-+static inline int subdev_8255_io_init(struct comedi_device *dev,
-+				      struct comedi_subdevice *s,
-+				      unsigned long regbase)
-+{
-+	return -ENXIO;
-+}
-+#endif
- 
- int subdev_8255_mm_init(struct comedi_device *dev, struct comedi_subdevice *s,
-			int (*io)(struct comedi_device *dev, int dir, int port,
-				  int data, unsigned long regbase),
- 			unsigned long regbase);
- 
-+int subdev_8255_cb_init(struct comedi_device *dev, struct comedi_subdevice *s,
-+			int (*io)(struct comedi_device *dev, int dir, int port,
-+				  int data, unsigned long context),
-+			unsigned long context);
-+
- unsigned long subdev_8255_regbase(struct comedi_subdevice *s);
- 
- #endif
-diff --git a/include/linux/comedi/comedidev.h b/include/linux/comedi/comedidev.h
-index 0a1150900ef3..35fdc41845ce 100644
--- a/include/linux/comedi/comedidev.h
-+++ b/include/linux/comedi/comedidev.h
-@@ -15,6 +15,7 @@
- #include <linux/spinlock_types.h>
- #include <linux/rwsem.h>
- #include <linux/kref.h>
-+#include <linux/completion.h>
- #include <linux/comedi.h>
- 
- #define COMEDI_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))
-@@ -234,16 +235,12 @@ struct comedi_buf_page {
-  *
-  * A COMEDI data buffer is allocated as individual pages, either in
-  * conventional memory or DMA coherent memory, depending on the attached,
- * low-level hardware device.  (The buffer pages also get mapped into the
- * kernel's contiguous virtual address space pointed to by the 'prealloc_buf'
- * member of &struct comedi_async.)
-+ * low-level hardware device.
-  *
-  * The buffer is normally freed when the COMEDI device is detached from the
-  * low-level driver (which may happen due to device removal), but if it happens
-  * to be mmapped at the time, the pages cannot be freed until the buffer has
- * been munmapped.  That is what the reference counter is for.  (The virtual
- * address space pointed by 'prealloc_buf' is freed when the COMEDI device is
- * detached.)
-+ * been munmapped.  That is what the reference counter is for.
-  */
- struct comedi_buf_map {
- 	struct device *dma_hw_dev;
-@@ -255,7 +252,6 @@ struct comedi_buf_map {
- 
- /**
-  * struct comedi_async - Control data for asynchronous COMEDI commands
- * @prealloc_buf: Kernel virtual address of allocated acquisition buffer.
-  * @prealloc_bufsz: Buffer size (in bytes).
-  * @buf_map: Map of buffer pages.
-  * @max_bufsize: Maximum allowed buffer size (in bytes).
-@@ -277,6 +273,8 @@ struct comedi_buf_map {
-  * @events: Bit-vector of events that have occurred.
-  * @cmd: Details of comedi command in progress.
-  * @wait_head: Task wait queue for file reader or writer.
-+ * @run_complete: "run complete" completion event.
-+ * @run_active: "run active" reference counter.
-  * @cb_mask: Bit-vector of events that should wake waiting tasks.
-  * @inttrig: Software trigger function for command, or NULL.
-  *
-@@ -344,7 +342,6 @@ struct comedi_buf_map {
-  * less than or equal to UINT_MAX).
-  */
- struct comedi_async {
-	void *prealloc_buf;
- 	unsigned int prealloc_bufsz;
- 	struct comedi_buf_map *buf_map;
- 	unsigned int max_bufsize;
-@@ -363,6 +360,8 @@ struct comedi_async {
- 	unsigned int events;
- 	struct comedi_cmd cmd;
- 	wait_queue_head_t wait_head;
-+	struct completion run_complete;
-+	refcount_t run_active;
- 	unsigned int cb_mask;
- 	int (*inttrig)(struct comedi_device *dev, struct comedi_subdevice *s,
- 		       unsigned int x);
-@@ -590,6 +589,8 @@ struct comedi_device *comedi_dev_get_from_minor(unsigned int minor);
- int comedi_dev_put(struct comedi_device *dev);
- 
- bool comedi_is_subdevice_running(struct comedi_subdevice *s);
-+bool comedi_get_is_subdevice_running(struct comedi_subdevice *s);
-+void comedi_put_is_subdevice_running(struct comedi_subdevice *s);
- 
- void *comedi_alloc_spriv(struct comedi_subdevice *s, size_t size);
- void comedi_set_spriv_auto_free(struct comedi_subdevice *s);
-@@ -633,7 +634,7 @@ extern const struct comedi_lrange range_unknown;
-  */
- struct comedi_lrange {
- 	int length;
-	struct comedi_krange range[];
-+	struct comedi_krange range[] __counted_by(length);
- };
- 
- /**
-diff --git a/include/linux/comedi/comedilib.h b/include/linux/comedi/comedilib.h
-index 0223c9cd9215..1f2b22b383cc 100644
--- a/include/linux/comedi/comedilib.h
-+++ b/include/linux/comedi/comedilib.h
-@@ -10,8 +10,38 @@
- #ifndef _LINUX_COMEDILIB_H
- #define _LINUX_COMEDILIB_H
- 
-struct comedi_device *comedi_open(const char *path);
-int comedi_close(struct comedi_device *dev);
-+struct comedi_device *comedi_open_from(const char *path, int from);
-+
-+/**
-+ * comedi_open() - Open a COMEDI device from the kernel
-+ * @filename: Fake pathname of the form "/dev/comediN".
-+ *
-+ * Converts @filename to a COMEDI device number and "opens" it if it exists
-+ * and is attached to a low-level COMEDI driver.
-+ *
-+ * Return: A pointer to the COMEDI device on success.
-+ * Return %NULL on failure.
-+ */
-+static inline struct comedi_device *comedi_open(const char *path)
-+{
-+	return comedi_open_from(path, -1);
-+}
-+
-+int comedi_close_from(struct comedi_device *dev, int from);
-+
-+/**
-+ * comedi_close() - Close a COMEDI device from the kernel
-+ * @dev: COMEDI device.
-+ *
-+ * Closes a COMEDI device previously opened by comedi_open().
-+ *
-+ * Returns: 0
-+ */
-+static inline int comedi_close(struct comedi_device *dev)
-+{
-+	return comedi_close_from(dev, -1);
-+}
-+
- int comedi_dio_get_config(struct comedi_device *dev, unsigned int subdev,
- 			  unsigned int chan, unsigned int *io);
- int comedi_dio_config(struct comedi_device *dev, unsigned int subdev,
-diff --git a/include/linux/compaction.h b/include/linux/compaction.h
-index 52a9ff65faee..173d9c07a895 100644
--- a/include/linux/compaction.h
-+++ b/include/linux/compaction.h
-@@ -80,14 +80,12 @@ static inline unsigned long compact_gap(unsigned int order)
- 	return 2UL << order;
- }
- 
-+static inline int current_is_kcompactd(void)
-+{
-+	return current->flags & PF_KCOMPACTD;
-+}
-+
- #ifdef CONFIG_COMPACTION
-extern unsigned int sysctl_compaction_proactiveness;
-extern int sysctl_compaction_handler(struct ctl_table *table, int write,
-			void *buffer, size_t *length, loff_t *ppos);
-extern int compaction_proactiveness_sysctl_handler(struct ctl_table *table,
-		int write, void *buffer, size_t *length, loff_t *ppos);
-extern int sysctl_extfrag_threshold;
-extern int sysctl_compact_unevictable_allowed;
- 
- extern unsigned int extfrag_for_order(struct zone *zone, unsigned int order);
- extern int fragmentation_index(struct zone *zone, unsigned int order);
-@@ -96,89 +94,17 @@ extern enum compact_result try_to_compact_pages(gfp_t gfp_mask,
- 		const struct alloc_context *ac, enum compact_priority prio,
- 		struct page **page);
- extern void reset_isolation_suitable(pg_data_t *pgdat);
-extern enum compact_result compaction_suitable(struct zone *zone, int order,
-		unsigned int alloc_flags, int highest_zoneidx);
-+extern bool compaction_suitable(struct zone *zone, int order,
-+				unsigned long watermark, int highest_zoneidx);
- 
- extern void compaction_defer_reset(struct zone *zone, int order,
- 				bool alloc_success);
- 
-/* Compaction has made some progress and retrying makes sense */
-static inline bool compaction_made_progress(enum compact_result result)
-{
-	/*
-	 * Even though this might sound confusing this in fact tells us
-	 * that the compaction successfully isolated and migrated some
-	 * pageblocks.
-	 */
-	if (result == COMPACT_SUCCESS)
-		return true;
-
-	return false;
-}
-
-/* Compaction has failed and it doesn't make much sense to keep retrying. */
-static inline bool compaction_failed(enum compact_result result)
-{
-	/* All zones were scanned completely and still not result. */
-	if (result == COMPACT_COMPLETE)
-		return true;
-
-	return false;
-}
-
-/* Compaction needs reclaim to be performed first, so it can continue. */
-static inline bool compaction_needs_reclaim(enum compact_result result)
-{
-	/*
-	 * Compaction backed off due to watermark checks for order-0
-	 * so the regular reclaim has to try harder and reclaim something.
-	 */
-	if (result == COMPACT_SKIPPED)
-		return true;
-
-	return false;
-}
-
-/*
- * Compaction has backed off for some reason after doing some work or none
- * at all. It might be throttling or lock contention. Retrying might be still
- * worthwhile, but with a higher priority if allowed.
- */
-static inline bool compaction_withdrawn(enum compact_result result)
-{
-	/*
-	 * If compaction is deferred for high-order allocations, it is
-	 * because sync compaction recently failed. If this is the case
-	 * and the caller requested a THP allocation, we do not want
-	 * to heavily disrupt the system, so we fail the allocation
-	 * instead of entering direct reclaim.
-	 */
-	if (result == COMPACT_DEFERRED)
-		return true;
-
-	/*
-	 * If compaction in async mode encounters contention or blocks higher
-	 * priority task we back off early rather than cause stalls.
-	 */
-	if (result == COMPACT_CONTENDED)
-		return true;
-
-	/*
-	 * Page scanners have met but we haven't scanned full zones so this
-	 * is a back off in fact.
-	 */
-	if (result == COMPACT_PARTIAL_SKIPPED)
-		return true;
-
-	return false;
-}
-
-
- bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
- 					int alloc_flags);
- 
-extern void kcompactd_run(int nid);
-extern void kcompactd_stop(int nid);
-+extern void __meminit kcompactd_run(int nid);
-+extern void __meminit kcompactd_stop(int nid);
- extern void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx);
- 
- #else
-@@ -186,32 +112,13 @@ static inline void reset_isolation_suitable(pg_data_t *pgdat)
- {
- }
- 
-static inline enum compact_result compaction_suitable(struct zone *zone, int order,
-					int alloc_flags, int highest_zoneidx)
-{
-	return COMPACT_SKIPPED;
-}
-
-static inline bool compaction_made_progress(enum compact_result result)
-+static inline bool compaction_suitable(struct zone *zone, int order,
-+				       unsigned long watermark,
-+				       int highest_zoneidx)
- {
- 	return false;
- }
- 
-static inline bool compaction_failed(enum compact_result result)
-{
-	return false;
-}
-
-static inline bool compaction_needs_reclaim(enum compact_result result)
-{
-	return false;
-}
-
-static inline bool compaction_withdrawn(enum compact_result result)
-{
-	return true;
-}
-
- static inline void kcompactd_run(int nid)
- {
- }
-diff --git a/include/linux/compat.h b/include/linux/compat.h
-index 594357881b0b..56cebaff0c91 100644
--- a/include/linux/compat.h
-+++ b/include/linux/compat.h
-@@ -126,11 +126,9 @@ struct compat_tms {
- 
- #define _COMPAT_NSIG_WORDS	(_COMPAT_NSIG / _COMPAT_NSIG_BPW)
- 
-#ifndef compat_sigset_t
- typedef struct {
- 	compat_sigset_word	sig[_COMPAT_NSIG_WORDS];
- } compat_sigset_t;
-#endif
- 
- int set_compat_user_sigmask(const compat_sigset_t __user *umask,
- 			    size_t sigsetsize);
-@@ -583,11 +581,6 @@ asmlinkage long compat_sys_io_pgetevents_time64(compat_aio_context_t ctx_id,
- 					struct io_event __user *events,
- 					struct __kernel_timespec __user *timeout,
- 					const struct __compat_aio_sigset __user *usig);
-
-/* fs/cookies.c */
-asmlinkage long compat_sys_lookup_dcookie(u32, u32, char __user *, compat_size_t);
-
-/* fs/eventpoll.c */
- asmlinkage long compat_sys_epoll_pwait(int epfd,
- 			struct epoll_event __user *events,
- 			int maxevents, int timeout,
-@@ -599,18 +592,12 @@ asmlinkage long compat_sys_epoll_pwait2(int epfd,
- 			const struct __kernel_timespec __user *timeout,
- 			const compat_sigset_t __user *sigmask,
- 			compat_size_t sigsetsize);
-
-/* fs/fcntl.c */
- asmlinkage long compat_sys_fcntl(unsigned int fd, unsigned int cmd,
- 				 compat_ulong_t arg);
- asmlinkage long compat_sys_fcntl64(unsigned int fd, unsigned int cmd,
- 				   compat_ulong_t arg);
-
-/* fs/ioctl.c */
- asmlinkage long compat_sys_ioctl(unsigned int fd, unsigned int cmd,
- 				 compat_ulong_t arg);
-
-/* fs/open.c */
- asmlinkage long compat_sys_statfs(const char __user *pathname,
- 				  struct compat_statfs __user *buf);
- asmlinkage long compat_sys_statfs64(const char __user *pathname,
-@@ -621,17 +608,13 @@ asmlinkage long compat_sys_fstatfs(unsigned int fd,
- asmlinkage long compat_sys_fstatfs64(unsigned int fd, compat_size_t sz,
- 				     struct compat_statfs64 __user *buf);
- asmlinkage long compat_sys_truncate(const char __user *, compat_off_t);
-asmlinkage long compat_sys_ftruncate(unsigned int, compat_ulong_t);
-+asmlinkage long compat_sys_ftruncate(unsigned int, compat_off_t);
- /* No generic prototype for truncate64, ftruncate64, fallocate */
- asmlinkage long compat_sys_openat(int dfd, const char __user *filename,
- 				  int flags, umode_t mode);
-
-/* fs/readdir.c */
- asmlinkage long compat_sys_getdents(unsigned int fd,
- 				    struct compat_linux_dirent __user *dirent,
- 				    unsigned int count);
-
-/* fs/read_write.c */
- asmlinkage long compat_sys_lseek(unsigned int, compat_off_t, unsigned int);
- /* No generic prototype for pread64 and pwrite64 */
- asmlinkage ssize_t compat_sys_preadv(compat_ulong_t fd,
-@@ -651,14 +634,10 @@ asmlinkage long compat_sys_pwritev64(unsigned long fd,
- 		const struct iovec __user *vec,
- 		unsigned long vlen, loff_t pos);
- #endif
-
-/* fs/sendfile.c */
- asmlinkage long compat_sys_sendfile(int out_fd, int in_fd,
- 				    compat_off_t __user *offset, compat_size_t count);
- asmlinkage long compat_sys_sendfile64(int out_fd, int in_fd,
- 				    compat_loff_t __user *offset, compat_size_t count);
-
-/* fs/select.c */
- asmlinkage long compat_sys_pselect6_time32(int n, compat_ulong_t __user *inp,
- 				    compat_ulong_t __user *outp,
- 				    compat_ulong_t __user *exp,
-@@ -679,68 +658,45 @@ asmlinkage long compat_sys_ppoll_time64(struct pollfd __user *ufds,
- 				 struct __kernel_timespec __user *tsp,
- 				 const compat_sigset_t __user *sigmask,
- 				 compat_size_t sigsetsize);
-
-/* fs/signalfd.c */
- asmlinkage long compat_sys_signalfd4(int ufd,
- 				     const compat_sigset_t __user *sigmask,
- 				     compat_size_t sigsetsize, int flags);
-
-/* fs/stat.c */
- asmlinkage long compat_sys_newfstatat(unsigned int dfd,
- 				      const char __user *filename,
- 				      struct compat_stat __user *statbuf,
- 				      int flag);
- asmlinkage long compat_sys_newfstat(unsigned int fd,
- 				    struct compat_stat __user *statbuf);
-
-/* fs/sync.c: No generic prototype for sync_file_range and sync_file_range2 */
-
-/* kernel/exit.c */
-+/* No generic prototype for sync_file_range and sync_file_range2 */
- asmlinkage long compat_sys_waitid(int, compat_pid_t,
- 		struct compat_siginfo __user *, int,
- 		struct compat_rusage __user *);
-
-
-
-/* kernel/futex.c */
- asmlinkage long
- compat_sys_set_robust_list(struct compat_robust_list_head __user *head,
- 			   compat_size_t len);
- asmlinkage long
- compat_sys_get_robust_list(int pid, compat_uptr_t __user *head_ptr,
- 			   compat_size_t __user *len_ptr);
-
-/* kernel/itimer.c */
- asmlinkage long compat_sys_getitimer(int which,
- 				     struct old_itimerval32 __user *it);
- asmlinkage long compat_sys_setitimer(int which,
- 				     struct old_itimerval32 __user *in,
- 				     struct old_itimerval32 __user *out);
-
-/* kernel/kexec.c */
- asmlinkage long compat_sys_kexec_load(compat_ulong_t entry,
- 				      compat_ulong_t nr_segments,
- 				      struct compat_kexec_segment __user *,
- 				      compat_ulong_t flags);
-
-/* kernel/posix-timers.c */
- asmlinkage long compat_sys_timer_create(clockid_t which_clock,
- 			struct compat_sigevent __user *timer_event_spec,
- 			timer_t __user *created_timer_id);
-
-/* kernel/ptrace.c */
- asmlinkage long compat_sys_ptrace(compat_long_t request, compat_long_t pid,
- 				  compat_long_t addr, compat_long_t data);
-
-/* kernel/sched/core.c */
- asmlinkage long compat_sys_sched_setaffinity(compat_pid_t pid,
- 				     unsigned int len,
- 				     compat_ulong_t __user *user_mask_ptr);
- asmlinkage long compat_sys_sched_getaffinity(compat_pid_t pid,
- 				     unsigned int len,
- 				     compat_ulong_t __user *user_mask_ptr);
-
-/* kernel/signal.c */
- asmlinkage long compat_sys_sigaltstack(const compat_stack_t __user *uss_ptr,
- 				       compat_stack_t __user *uoss_ptr);
- asmlinkage long compat_sys_rt_sigsuspend(compat_sigset_t __user *unewset,
-@@ -765,25 +721,17 @@ asmlinkage long compat_sys_rt_sigtimedwait_time64(compat_sigset_t __user *uthese
- asmlinkage long compat_sys_rt_sigqueueinfo(compat_pid_t pid, int sig,
- 				struct compat_siginfo __user *uinfo);
- /* No generic prototype for rt_sigreturn */
-
-/* kernel/sys.c */
- asmlinkage long compat_sys_times(struct compat_tms __user *tbuf);
- asmlinkage long compat_sys_getrlimit(unsigned int resource,
- 				     struct compat_rlimit __user *rlim);
- asmlinkage long compat_sys_setrlimit(unsigned int resource,
- 				     struct compat_rlimit __user *rlim);
- asmlinkage long compat_sys_getrusage(int who, struct compat_rusage __user *ru);
-
-/* kernel/time.c */
- asmlinkage long compat_sys_gettimeofday(struct old_timeval32 __user *tv,
- 		struct timezone __user *tz);
- asmlinkage long compat_sys_settimeofday(struct old_timeval32 __user *tv,
- 		struct timezone __user *tz);
-
-/* kernel/timer.c */
- asmlinkage long compat_sys_sysinfo(struct compat_sysinfo __user *info);
-
-/* ipc/mqueue.c */
- asmlinkage long compat_sys_mq_open(const char __user *u_name,
- 			int oflag, compat_mode_t mode,
- 			struct compat_mq_attr __user *u_attr);
-@@ -792,22 +740,14 @@ asmlinkage long compat_sys_mq_notify(mqd_t mqdes,
- asmlinkage long compat_sys_mq_getsetattr(mqd_t mqdes,
- 			const struct compat_mq_attr __user *u_mqstat,
- 			struct compat_mq_attr __user *u_omqstat);
-
-/* ipc/msg.c */
- asmlinkage long compat_sys_msgctl(int first, int second, void __user *uptr);
- asmlinkage long compat_sys_msgrcv(int msqid, compat_uptr_t msgp,
- 		compat_ssize_t msgsz, compat_long_t msgtyp, int msgflg);
- asmlinkage long compat_sys_msgsnd(int msqid, compat_uptr_t msgp,
- 		compat_ssize_t msgsz, int msgflg);
-
-/* ipc/sem.c */
- asmlinkage long compat_sys_semctl(int semid, int semnum, int cmd, int arg);
-
-/* ipc/shm.c */
- asmlinkage long compat_sys_shmctl(int first, int second, void __user *uptr);
- asmlinkage long compat_sys_shmat(int shmid, compat_uptr_t shmaddr, int shmflg);
-
-/* net/socket.c */
- asmlinkage long compat_sys_recvfrom(int fd, void __user *buf, compat_size_t len,
- 			    unsigned flags, struct sockaddr __user *addr,
- 			    int __user *addrlen);
-@@ -815,20 +755,13 @@ asmlinkage long compat_sys_sendmsg(int fd, struct compat_msghdr __user *msg,
- 				   unsigned flags);
- asmlinkage long compat_sys_recvmsg(int fd, struct compat_msghdr __user *msg,
- 				   unsigned int flags);
-
-/* mm/filemap.c: No generic prototype for readahead */
-
-/* security/keys/keyctl.c */
-+/* No generic prototype for readahead */
- asmlinkage long compat_sys_keyctl(u32 option,
- 			      u32 arg2, u32 arg3, u32 arg4, u32 arg5);
-
-/* arch/example/kernel/sys_example.c */
- asmlinkage long compat_sys_execve(const char __user *filename, const compat_uptr_t __user *argv,
- 		     const compat_uptr_t __user *envp);
-
-/* mm/fadvise.c: No generic prototype for fadvise64_64 */
-
-/* mm/, CONFIG_MMU only */
-+/* No generic prototype for fadvise64_64 */
-+/* CONFIG_MMU only */
- asmlinkage long compat_sys_rt_tgsigqueueinfo(compat_pid_t tgid,
- 					compat_pid_t pid, int sig,
- 					struct compat_siginfo __user *uinfo);
-@@ -898,18 +831,18 @@ asmlinkage long compat_sys_ustat(unsigned dev, struct compat_ustat __user *u32);
- asmlinkage long compat_sys_recv(int fd, void __user *buf, compat_size_t len,
- 				unsigned flags);
- 
-/* obsolete: fs/readdir.c */
-+/* obsolete */
- asmlinkage long compat_sys_old_readdir(unsigned int fd,
- 				       struct compat_old_linux_dirent __user *,
- 				       unsigned int count);
- 
-/* obsolete: fs/select.c */
-+/* obsolete */
- asmlinkage long compat_sys_old_select(struct compat_sel_arg_struct __user *arg);
- 
-/* obsolete: ipc */
-+/* obsolete */
- asmlinkage long compat_sys_ipc(u32, int, int, u32, compat_uptr_t, u32);
- 
-/* obsolete: kernel/signal.c */
-+/* obsolete */
- #ifdef __ARCH_WANT_SYS_SIGPENDING
- asmlinkage long compat_sys_sigpending(compat_old_sigset_t __user *set);
- #endif
-@@ -924,7 +857,7 @@ asmlinkage long compat_sys_sigaction(int sig,
-                                    struct compat_old_sigaction __user *oact);
- #endif
- 
-/* obsolete: net/socket.c */
-+/* obsolete */
- asmlinkage long compat_sys_socketcall(int call, u32 __user *args);
- 
- #ifdef __ARCH_WANT_COMPAT_TRUNCATE64
-diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h
-index c84fec767445..107ce05bd16e 100644
--- a/include/linux/compiler-clang.h
-+++ b/include/linux/compiler-clang.h
-@@ -1,36 +1,59 @@
- /* SPDX-License-Identifier: GPL-2.0 */
- #ifndef __LINUX_COMPILER_TYPES_H
-#error "Please don't include <linux/compiler-clang.h> directly, include <linux/compiler.h> instead."
-+#error "Please do not include <linux/compiler-clang.h> directly, include <linux/compiler.h> instead."
- #endif
- 
- /* Compiler specific definitions for Clang compiler */
- 
-/* same as gcc, this was present in clang-2.6 so we can assume it works
- * with any version that can compile the kernel
-+/*
-+ * Clang prior to 17 is being silly and considers many __cleanup() variables
-+ * as unused (because they are, their sole purpose is to go out of scope).
-+ *
-+ * https://github.com/llvm/llvm-project/commit/877210faa447f4cc7db87812f8ed80e398fedd61
-  */
-#define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __COUNTER__)
-+#undef __cleanup
-+#define __cleanup(func) __maybe_unused __attribute__((__cleanup__(func)))
- 
- /* all clang versions usable with the kernel support KASAN ABI version 5 */
- #define KASAN_ABI_VERSION 5
- 
- /*
-+ * Clang 22 added preprocessor macros to match GCC, in hopes of eventually
-+ * dropping __has_feature support for sanitizers:
-+ * https://github.com/llvm/llvm-project/commit/568c23bbd3303518c5056d7f03444dae4fdc8a9c
-+ * Create these macros for older versions of clang so that it is easy to clean
-+ * up once the minimum supported version of LLVM for building the kernel always
-+ * creates these macros.
-+ *
-  * Note: Checking __has_feature(*_sanitizer) is only true if the feature is
-  * enabled. Therefore it is not required to additionally check defined(CONFIG_*)
-  * to avoid adding redundant attributes in other configurations.
-  */
-+#if __has_feature(address_sanitizer) && !defined(__SANITIZE_ADDRESS__)
-+#define __SANITIZE_ADDRESS__
-+#endif
-+#if __has_feature(hwaddress_sanitizer) && !defined(__SANITIZE_HWADDRESS__)
-+#define __SANITIZE_HWADDRESS__
-+#endif
-+#if __has_feature(thread_sanitizer) && !defined(__SANITIZE_THREAD__)
-+#define __SANITIZE_THREAD__
-+#endif
- 
-#if __has_feature(address_sanitizer) || __has_feature(hwaddress_sanitizer)
-/* Emulate GCC's __SANITIZE_ADDRESS__ flag */
-+/*
-+ * Treat __SANITIZE_HWADDRESS__ the same as __SANITIZE_ADDRESS__ in the kernel.
-+ */
-+#ifdef __SANITIZE_HWADDRESS__
- #define __SANITIZE_ADDRESS__
-+#endif
-+
-+#ifdef __SANITIZE_ADDRESS__
- #define __no_sanitize_address \
- 		__attribute__((no_sanitize("address", "hwaddress")))
- #else
- #define __no_sanitize_address
- #endif
- 
-#if __has_feature(thread_sanitizer)
-/* emulate gcc's __SANITIZE_THREAD__ flag */
-#define __SANITIZE_THREAD__
-+#ifdef __SANITIZE_THREAD__
- #define __no_sanitize_thread \
- 		__attribute__((no_sanitize("thread")))
- #else
-@@ -51,6 +74,29 @@
- #define __no_sanitize_undefined
- #endif
- 
-+#if __has_feature(memory_sanitizer)
-+#define __SANITIZE_MEMORY__
-+/*
-+ * Unlike other sanitizers, KMSAN still inserts code into functions marked with
-+ * no_sanitize("kernel-memory"). Using disable_sanitizer_instrumentation
-+ * provides the behavior consistent with other __no_sanitize_ attributes,
-+ * guaranteeing that __no_sanitize_memory functions remain uninstrumented.
-+ */
-+#define __no_sanitize_memory __disable_sanitizer_instrumentation
-+
-+/*
-+ * The __no_kmsan_checks attribute ensures that a function does not produce
-+ * false positive reports by:
-+ *  - initializing all local variables and memory stores in this function;
-+ *  - skipping all shadow checks;
-+ *  - passing initialized arguments to this function's callees.
-+ */
-+#define __no_kmsan_checks __attribute__((no_sanitize("kernel-memory")))
-+#else
-+#define __no_sanitize_memory
-+#define __no_kmsan_checks
-+#endif
-+
- /*
-  * Support for __has_feature(coverage_sanitizer) was added in Clang 13 together
-  * with no_sanitize("coverage"). Prior versions of Clang support coverage
-@@ -62,23 +108,13 @@
- #define __no_sanitize_coverage
- #endif
- 
-+/* Only Clang needs to disable the coverage sanitizer for kstack_erase. */
-+#define __no_kstack_erase	__no_sanitize_coverage
-+
- #if __has_feature(shadow_call_stack)
- # define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
- #endif
- 
-#define __nocfi		__attribute__((__no_sanitize__("cfi")))
-#define __cficanonical	__attribute__((__cfi_canonical_jump_table__))
-
-#if defined(CONFIG_CFI_CLANG)
-/*
- * With CONFIG_CFI_CLANG, the compiler replaces function address
- * references with the address of the function's CFI jump table
- * entry. The function_nocfi macro always returns the address of the
- * actual function instead.
- */
-#define function_nocfi(x)	__builtin_function_start(x)
-#endif
-
- /*
-  * Turn individual warnings and errors on and off locally, depending
-  * on version.
-@@ -95,11 +131,25 @@
- #define __diag_str(s)		__diag_str1(s)
- #define __diag(s)		_Pragma(__diag_str(clang diagnostic s))
- 
-#if CONFIG_CLANG_VERSION >= 110000
-#define __diag_clang_11(s)	__diag(s)
-#else
-#define __diag_clang_11(s)
-#endif
-+#define __diag_clang_13(s)	__diag(s)
- 
- #define __diag_ignore_all(option, comment) \
-	__diag_clang(11, ignore, option)
-+	__diag_clang(13, ignore, option)
-+
-+/*
-+ * clang has horrible behavior with "g" or "rm" constraints for asm
-+ * inputs, turning them into something worse than "m". Avoid using
-+ * constraints with multiple possible uses (but "ir" seems to be ok):
-+ *
-+ *	https://github.com/llvm/llvm-project/issues/20571
-+ */
-+#define ASM_INPUT_G "ir"
-+#define ASM_INPUT_RM "r"
-+
-+/*
-+ * Declare compiler support for __typeof_unqual__() operator.
-+ *
-+ * Bindgen uses LLVM even if our C compiler is GCC, so we cannot
-+ * rely on the auto-detected CONFIG_CC_HAS_TYPEOF_UNQUAL.
-+ */
-+#define CC_HAS_TYPEOF_UNQUAL (__clang_major__ >= 19)
-diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h
-index 9b157b71036f..5de824a0b3d7 100644
--- a/include/linux/compiler-gcc.h
-+++ b/include/linux/compiler-gcc.h
-@@ -1,6 +1,6 @@
- /* SPDX-License-Identifier: GPL-2.0 */
- #ifndef __LINUX_COMPILER_TYPES_H
-#error "Please don't include <linux/compiler-gcc.h> directly, include <linux/compiler.h> instead."
-+#error "Please do not include <linux/compiler-gcc.h> directly, include <linux/compiler.h> instead."
- #endif
- 
- /*
-@@ -35,12 +35,6 @@
- 	(typeof(ptr)) (__ptr + (off));					\
- })
- 
-#ifdef CONFIG_RETPOLINE
-#define __noretpoline __attribute__((__indirect_branch__("keep")))
-#endif
-
-#define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __COUNTER__)
-
- #if defined(LATENT_ENTROPY_PLUGIN) && !defined(__CHECKER__)
- #define __latent_entropy __attribute__((latent_entropy))
- #endif
-@@ -54,18 +48,6 @@
-  */
- #define barrier_before_unreachable() asm volatile("")
- 
-/*
- * Mark a position in code as unreachable.  This can be used to
- * suppress control flow warnings after asm blocks that transfer
- * control elsewhere.
- */
-#define unreachable() \
-	do {					\
-		annotate_unreachable();		\
-		barrier_before_unreachable();	\
-		__builtin_unreachable();	\
-	} while (0)
-
- #if defined(CONFIG_ARCH_USE_BUILTIN_BSWAP)
- #define __HAVE_BUILTIN_BSWAP32__
- #define __HAVE_BUILTIN_BSWAP64__
-@@ -82,26 +64,25 @@
- #define __noscs __attribute__((__no_sanitize__("shadow-call-stack")))
- #endif
- 
-#if __has_attribute(__no_sanitize_address__)
-#define __no_sanitize_address __attribute__((no_sanitize_address))
-+#ifdef __SANITIZE_HWADDRESS__
-+#define __no_sanitize_address __attribute__((__no_sanitize__("hwaddress")))
- #else
-#define __no_sanitize_address
-+#define __no_sanitize_address __attribute__((__no_sanitize_address__))
- #endif
- 
-#if defined(__SANITIZE_THREAD__) && __has_attribute(__no_sanitize_thread__)
-#define __no_sanitize_thread __attribute__((no_sanitize_thread))
-+#if defined(__SANITIZE_THREAD__)
-+#define __no_sanitize_thread __attribute__((__no_sanitize_thread__))
- #else
- #define __no_sanitize_thread
- #endif
- 
-#if __has_attribute(__no_sanitize_undefined__)
-#define __no_sanitize_undefined __attribute__((no_sanitize_undefined))
-#else
-#define __no_sanitize_undefined
-#endif
-+#define __no_sanitize_undefined __attribute__((__no_sanitize_undefined__))
- 
-+/*
-+ * Only supported since gcc >= 12
-+ */
- #if defined(CONFIG_KCOV) && __has_attribute(__no_sanitize_coverage__)
-#define __no_sanitize_coverage __attribute__((no_sanitize_coverage))
-+#define __no_sanitize_coverage __attribute__((__no_sanitize_coverage__))
- #else
- #define __no_sanitize_coverage
- #endif
-@@ -115,6 +96,12 @@
- #endif
- 
- /*
-+ * GCC does not support KMSAN.
-+ */
-+#define __no_sanitize_memory
-+#define __no_kmsan_checks
-+
-+/*
-  * Turn individual warnings and errors on and off locally, depending
-  * on version.
-  */
-@@ -136,8 +123,10 @@
- #define __diag_GCC_8(s)
- #endif
- 
-+#define __diag_GCC_all(s)	__diag(s)
-+
- #define __diag_ignore_all(option, comment) \
-	__diag_GCC(8, ignore, option)
-+	__diag(__diag_GCC_ignore option)
- 
- /*
-  * Prior to 9.1, -Wno-alloc-size-larger-than (and therefore the "alloc_size"
-@@ -146,3 +135,11 @@
- #if GCC_VERSION < 90100
- #undef __alloc_size__
- #endif
-+
-+/*
-+ * Declare compiler support for __typeof_unqual__() operator.
-+ *
-+ * Bindgen uses LLVM even if our C compiler is GCC, so we cannot
-+ * rely on the auto-detected CONFIG_CC_HAS_TYPEOF_UNQUAL.
-+ */
-+#define CC_HAS_TYPEOF_UNQUAL (__GNUC__ >= 14)
-diff --git a/include/linux/compiler-intel.h b/include/linux/compiler-intel.h
-deleted file mode 100644
-index b17f3cd18334..000000000000
--- a/include/linux/compiler-intel.h
-+++ /dev/null
-@@ -1,34 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_COMPILER_TYPES_H
-#error "Please don't include <linux/compiler-intel.h> directly, include <linux/compiler.h> instead."
-#endif
-
-#ifdef __ECC
-
-/* Compiler specific definitions for Intel ECC compiler */
-
-#include <asm/intrinsics.h>
-
-/* Intel ECC compiler doesn't support gcc specific asm stmts.
- * It uses intrinsics to do the equivalent things.
- */
-
-#define barrier() __memory_barrier()
-#define barrier_data(ptr) barrier()
-
-#define RELOC_HIDE(ptr, off)					\
-  ({ unsigned long __ptr;					\
-     __ptr = (unsigned long) (ptr);				\
-    (typeof(ptr)) (__ptr + (off)); })
-
-/* This should act as an optimization barrier on var.
- * Given that this compiler does not have inline assembly, a compiler barrier
- * is the best we can do.
- */
-#define OPTIMIZER_HIDE_VAR(var) barrier()
-
-#endif
-
-/* icc has this, but it's called _bswap16 */
-#define __HAVE_BUILTIN_BSWAP16__
-#define __builtin_bswap16 _bswap16
-diff --git a/include/linux/compiler-version.h b/include/linux/compiler-version.h
-index 573fa85b6c0c..ac1665a98a15 100644
--- a/include/linux/compiler-version.h
-+++ b/include/linux/compiler-version.h
-@@ -12,3 +12,33 @@
-  * and add dependency on include/config/CC_VERSION_TEXT, which is touched
-  * by Kconfig when the version string from the compiler changes.
-  */
-+
-+/* Additional tree-wide dependencies start here. */
-+
-+/*
-+ * If any of the GCC plugins change, we need to rebuild everything that
-+ * was built with them, as they may have changed their behavior and those
-+ * behaviors may need to be synchronized across all translation units.
-+ */
-+#ifdef GCC_PLUGINS
-+#include <generated/gcc-plugins.h>
-+#endif
-+
-+/*
-+ * If the randstruct seed itself changes (whether for GCC plugins or
-+ * Clang), the entire tree needs to be rebuilt since the randomization of
-+ * structures may change between compilation units if not.
-+ */
-+#ifdef RANDSTRUCT
-+#include <generated/randstruct_hash.h>
-+#endif
-+
-+/*
-+ * If any external changes affect Clang's integer wrapping sanitizer
-+ * behavior, a full rebuild is needed as the coverage for wrapping types
-+ * may have changed, which may impact the expected behaviors that should
-+ * not differ between compilation units.
-+ */
-+#ifdef INTEGER_WRAP
-+#include <generated/integer-wrap.h>
-+#endif
-diff --git a/include/linux/compiler.h b/include/linux/compiler.h
-index 01ce94b58b42..04487c9bd751 100644
--- a/include/linux/compiler.h
-+++ b/include/linux/compiler.h
-@@ -12,11 +12,10 @@
-  * Note: DISABLE_BRANCH_PROFILING can be used by special lowlevel code
-  * to disable branch tracing on a per file basis.
-  */
-#if defined(CONFIG_TRACE_BRANCH_PROFILING) \
-    && !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__)
- void ftrace_likely_update(struct ftrace_likely_data *f, int val,
- 			  int expect, int is_constant);
-
-+#if defined(CONFIG_TRACE_BRANCH_PROFILING) \
-+    && !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__)
- #define likely_notrace(x)	__builtin_expect(!!(x), 1)
- #define unlikely_notrace(x)	__builtin_expect(!!(x), 0)
- 
-@@ -110,35 +109,21 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
- 
- /* Unreachable code */
- #ifdef CONFIG_OBJTOOL
-/*
- * These macros help objtool understand GCC code flow for unreachable code.
- * The __COUNTER__ based labels are a hack to make each instance of the macros
- * unique, to convince GCC not to merge duplicate inline asm statements.
- */
-#define __stringify_label(n) #n
-
-#define __annotate_unreachable(c) ({					\
-	asm volatile(__stringify_label(c) ":\n\t"			\
-		     ".pushsection .discard.unreachable\n\t"		\
-		     ".long " __stringify_label(c) "b - .\n\t"		\
-		     ".popsection\n\t" : : "i" (c));			\
-})
-#define annotate_unreachable() __annotate_unreachable(__COUNTER__)
-
- /* Annotate a C jump table to allow objtool to follow the code flow */
-#define __annotate_jump_table __section(".rodata..c_jump_table")
-
-+#define __annotate_jump_table __section(".data.rel.ro.c_jump_table")
- #else /* !CONFIG_OBJTOOL */
-#define annotate_unreachable()
- #define __annotate_jump_table
- #endif /* CONFIG_OBJTOOL */
- 
-#ifndef unreachable
-# define unreachable() do {		\
-	annotate_unreachable();		\
-+/*
-+ * Mark a position in code as unreachable.  This can be used to
-+ * suppress control flow warnings after asm blocks that transfer
-+ * control elsewhere.
-+ */
-+#define unreachable() do {		\
-+	barrier_before_unreachable();	\
- 	__builtin_unreachable();	\
- } while (0)
-#endif
- 
- /*
-  * KENTRY - kernel entry point
-@@ -178,10 +163,11 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
- 	__asm__ ("" : "=r" (var) : "0" (var))
- #endif
- 
-/* Not-quite-unique ID. */
-#ifndef __UNIQUE_ID
-# define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __LINE__)
-#endif
-+/* Format: __UNIQUE_ID_<name>_<__COUNTER__> */
-+#define __UNIQUE_ID(name)					\
-+	__PASTE(__UNIQUE_ID_,					\
-+	__PASTE(name,						\
-+	__PASTE(_, __COUNTER__)))
- 
- /**
-  * data_race - mark an expression as containing intentional data races
-@@ -189,41 +175,92 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
-  * This data_race() macro is useful for situations in which data races
-  * should be forgiven.  One example is diagnostic code that accesses
-  * shared variables but is not a part of the core synchronization design.
-+ * For example, if accesses to a given variable are protected by a lock,
-+ * except for diagnostic code, then the accesses under the lock should
-+ * be plain C-language accesses and those in the diagnostic code should
-+ * use data_race().  This way, KCSAN will complain if buggy lockless
-+ * accesses to that variable are introduced, even if the buggy accesses
-+ * are protected by READ_ONCE() or WRITE_ONCE().
-  *
-  * This macro *does not* affect normal code generation, but is a hint
- * to tooling that data races here are to be ignored.
-+ * to tooling that data races here are to be ignored.  If the access must
-+ * be atomic *and* KCSAN should ignore the access, use both data_race()
-+ * and READ_ONCE(), for example, data_race(READ_ONCE(x)).
-  */
- #define data_race(expr)							\
- ({									\
-	__unqual_scalar_typeof(({ expr; })) __v = ({			\
-		__kcsan_disable_current();				\
-		expr;							\
-	});								\
-+	__kcsan_disable_current();					\
-+	auto __v = (expr);						\
- 	__kcsan_enable_current();					\
- 	__v;								\
- })
- 
-+#ifdef __CHECKER__
-+#define __BUILD_BUG_ON_ZERO_MSG(e, msg, ...) (0)
-+#else /* __CHECKER__ */
-+#define __BUILD_BUG_ON_ZERO_MSG(e, msg, ...) ((int)sizeof(struct {_Static_assert(!(e), msg);}))
-+#endif /* __CHECKER__ */
-+
-+/* &a[0] degrades to a pointer: a different type from an array */
-+#define __is_array(a)		(!__same_type((a), &(a)[0]))
-+#define __must_be_array(a)	__BUILD_BUG_ON_ZERO_MSG(!__is_array(a), \
-+							"must be array")
-+
-+#define __is_byte_array(a)	(__is_array(a) && sizeof((a)[0]) == 1)
-+#define __must_be_byte_array(a)	__BUILD_BUG_ON_ZERO_MSG(!__is_byte_array(a), \
-+							"must be byte array")
-+
- /*
- * With CONFIG_CFI_CLANG, the compiler replaces function addresses in
- * instrumented C code with jump table addresses. Architectures that
- * support CFI can define this macro to return the actual function address
- * when needed.
-+ * If the "nonstring" attribute isn't available, we have to return true
-+ * so the __must_*() checks pass when "nonstring" isn't supported.
-+ */
-+#if __has_attribute(__nonstring__) && defined(__annotated)
-+#define __is_cstr(a)		(!__annotated(a, nonstring))
-+#define __is_noncstr(a)		(__annotated(a, nonstring))
-+#else
-+#define __is_cstr(a)		(true)
-+#define __is_noncstr(a)		(true)
-+#endif
-+
-+/* Require C Strings (i.e. NUL-terminated) lack the "nonstring" attribute. */
-+#define __must_be_cstr(p) \
-+	__BUILD_BUG_ON_ZERO_MSG(!__is_cstr(p), \
-+				"must be C-string (NUL-terminated)")
-+#define __must_be_noncstr(p) \
-+	__BUILD_BUG_ON_ZERO_MSG(!__is_noncstr(p), \
-+				"must be non-C-string (not NUL-terminated)")
-+
-+/*
-+ * Use __typeof_unqual__() when available.
-+ *
-+ * XXX: Remove test for __CHECKER__ once
-+ * sparse learns about __typeof_unqual__().
-  */
-#ifndef function_nocfi
-#define function_nocfi(x) (x)
-+#if CC_HAS_TYPEOF_UNQUAL && !defined(__CHECKER__)
-+# define USE_TYPEOF_UNQUAL 1
-+#endif
-+
-+/*
-+ * Define TYPEOF_UNQUAL() to use __typeof_unqual__() as typeof
-+ * operator when available, to return an unqualified type of the exp.
-+ */
-+#if defined(USE_TYPEOF_UNQUAL)
-+# define TYPEOF_UNQUAL(exp) __typeof_unqual__(exp)
-+#else
-+# define TYPEOF_UNQUAL(exp) __typeof__(exp)
- #endif
- 
- #endif /* __KERNEL__ */
- 
-+#if defined(CONFIG_CFI) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO)
- /*
- * Force the compiler to emit 'sym' as a symbol, so that we can reference
- * it from inline assembler. Necessary in case 'sym' could be inlined
- * otherwise, or eliminated entirely due to lack of references that are
- * visible to the compiler.
-+ * Force a reference to the external symbol so the compiler generates
-+ * __kcfi_typid.
-  */
-#define __ADDRESSABLE(sym) \
-	static void * __section(".discard.addressable") __used \
-		__UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;
-+#define KCFI_REFERENCE(sym) __ADDRESSABLE(sym)
-+#else
-+#define KCFI_REFERENCE(sym)
-+#endif
- 
- /**
-  * offset_to_ptr - convert a relative memory offset to an absolute pointer
-@@ -236,8 +273,103 @@ static inline void *offset_to_ptr(const int *off)
- 
- #endif /* __ASSEMBLY__ */
- 
-/* &a[0] degrades to a pointer: a different type from an array */
-#define __must_be_array(a)	BUILD_BUG_ON_ZERO(__same_type((a), &(a)[0]))
-+/*
-+ * Force the compiler to emit 'sym' as a symbol, so that we can reference
-+ * it from inline assembler. Necessary in case 'sym' could be inlined
-+ * otherwise, or eliminated entirely due to lack of references that are
-+ * visible to the compiler.
-+ */
-+#define ___ADDRESSABLE(sym, __attrs)						\
-+	static void * __used __attrs						\
-+	__UNIQUE_ID(__PASTE(addressable_, sym)) = (void *)(uintptr_t)&sym;
-+
-+#define __ADDRESSABLE(sym) \
-+	___ADDRESSABLE(sym, __section(".discard.addressable"))
-+
-+/*
-+ * This returns a constant expression while determining if an argument is
-+ * a constant expression, most importantly without evaluating the argument.
-+ * Glory to Martin Uecker <Martin.Uecker@med.uni-goettingen.de>
-+ *
-+ * Details:
-+ * - sizeof() return an integer constant expression, and does not evaluate
-+ *   the value of its operand; it only examines the type of its operand.
-+ * - The results of comparing two integer constant expressions is also
-+ *   an integer constant expression.
-+ * - The first literal "8" isn't important. It could be any literal value.
-+ * - The second literal "8" is to avoid warnings about unaligned pointers;
-+ *   this could otherwise just be "1".
-+ * - (long)(x) is used to avoid warnings about 64-bit types on 32-bit
-+ *   architectures.
-+ * - The C Standard defines "null pointer constant", "(void *)0", as
-+ *   distinct from other void pointers.
-+ * - If (x) is an integer constant expression, then the "* 0l" resolves
-+ *   it into an integer constant expression of value 0. Since it is cast to
-+ *   "void *", this makes the second operand a null pointer constant.
-+ * - If (x) is not an integer constant expression, then the second operand
-+ *   resolves to a void pointer (but not a null pointer constant: the value
-+ *   is not an integer constant 0).
-+ * - The conditional operator's third operand, "(int *)8", is an object
-+ *   pointer (to type "int").
-+ * - The behavior (including the return type) of the conditional operator
-+ *   ("operand1 ? operand2 : operand3") depends on the kind of expressions
-+ *   given for the second and third operands. This is the central mechanism
-+ *   of the macro:
-+ *   - When one operand is a null pointer constant (i.e. when x is an integer
-+ *     constant expression) and the other is an object pointer (i.e. our
-+ *     third operand), the conditional operator returns the type of the
-+ *     object pointer operand (i.e. "int *"). Here, within the sizeof(), we
-+ *     would then get:
-+ *       sizeof(*((int *)(...))  == sizeof(int)  == 4
-+ *   - When one operand is a void pointer (i.e. when x is not an integer
-+ *     constant expression) and the other is an object pointer (i.e. our
-+ *     third operand), the conditional operator returns a "void *" type.
-+ *     Here, within the sizeof(), we would then get:
-+ *       sizeof(*((void *)(...)) == sizeof(void) == 1
-+ * - The equality comparison to "sizeof(int)" therefore depends on (x):
-+ *     sizeof(int) == sizeof(int)     (x) was a constant expression
-+ *     sizeof(int) != sizeof(void)    (x) was not a constant expression
-+ */
-+#define __is_constexpr(x) \
-+	(sizeof(int) == sizeof(*(8 ? ((void *)((long)(x) * 0l)) : (int *)8)))
-+
-+/*
-+ * Whether 'type' is a signed type or an unsigned type. Supports scalar types,
-+ * bool and also pointer types.
-+ */
-+#define is_signed_type(type) (((type)(-1)) < (__force type)1)
-+#define is_unsigned_type(type) (!is_signed_type(type))
-+
-+/*
-+ * Useful shorthand for "is this condition known at compile-time?"
-+ *
-+ * Note that the condition may involve non-constant values,
-+ * but the compiler may know enough about the details of the
-+ * values to determine that the condition is statically true.
-+ */
-+#define statically_true(x) (__builtin_constant_p(x) && (x))
-+
-+/*
-+ * Similar to statically_true() but produces a constant expression
-+ *
-+ * To be used in conjunction with macros, such as BUILD_BUG_ON_ZERO(),
-+ * which require their input to be a constant expression and for which
-+ * statically_true() would otherwise fail.
-+ *
-+ * This is a trade-off: const_true() requires all its operands to be
-+ * compile time constants. Else, it would always returns false even on
-+ * the most trivial cases like:
-+ *
-+ *   true || non_const_var
-+ *
-+ * On the opposite, statically_true() is able to fold more complex
-+ * tautologies and will return true on expressions such as:
-+ *
-+ *   !(non_const_var * 8 % 4)
-+ *
-+ * For the general case, statically_true() is better.
-+ */
-+#define const_true(x) __builtin_choose_expr(__is_constexpr(x), x, false)
- 
- /*
-  * This is needed in functions which generate the stack canary, see
-diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h
-index 445e80517cab..c16d4199bf92 100644
--- a/include/linux/compiler_attributes.h
-+++ b/include/linux/compiler_attributes.h
-@@ -35,7 +35,8 @@
- 
- /*
-  * Note: do not use this directly. Instead, use __alloc_size() since it is conditionally
- * available and includes other attributes.
-+ * available and includes other attributes. For GCC < 9.1, __alloc_size__ gets undefined
-+ * in compiler-gcc.h, due to misbehaviors.
-  *
-  *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-alloc_005fsize-function-attribute
-  * clang: https://clang.llvm.org/docs/AttributeReference.html#alloc-size
-@@ -63,22 +64,16 @@
-  * compiler should see some alignment anyway, when the return value is
-  * massaged by 'flags = ptr & 3; ptr &= ~3;').
-  *
- * Optional: not supported by icc
- *
-  *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-assume_005faligned-function-attribute
-  * clang: https://clang.llvm.org/docs/AttributeReference.html#assume-aligned
-  */
-#if __has_attribute(__assume_aligned__)
-# define __assume_aligned(a, ...)       __attribute__((__assume_aligned__(a, ## __VA_ARGS__)))
-#else
-# define __assume_aligned(a, ...)
-#endif
-+#define __assume_aligned(a, ...)        __attribute__((__assume_aligned__(a, ## __VA_ARGS__)))
- 
- /*
- *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-cold-function-attribute
- *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Label-Attributes.html#index-cold-label-attribute
-+ *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-cleanup-variable-attribute
-+ * clang: https://clang.llvm.org/docs/AttributeReference.html#cleanup
-  */
-#define __cold                          __attribute__((__cold__))
-+#define __cleanup(func)			__attribute__((__cleanup__(func)))
- 
- /*
-  * Note the long name.
-@@ -90,7 +85,6 @@
- /*
-  * Optional: only supported since gcc >= 9
-  * Optional: not supported by clang
- * Optional: not supported by icc
-  *
-  *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-copy-function-attribute
-  */
-@@ -103,7 +97,6 @@
- /*
-  * Optional: not supported by gcc
-  * Optional: only supported since clang >= 14.0
- * Optional: not supported by icc
-  *
-  * clang: https://clang.llvm.org/docs/AttributeReference.html#diagnose_as_builtin
-  */
-@@ -127,7 +120,6 @@
- 
- /*
-  * Optional: not supported by clang
- * Optional: not supported by icc
-  *
-  *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Type-Attributes.html#index-designated_005finit-type-attribute
-  */
-@@ -241,7 +233,6 @@
- /*
-  * Optional: only supported since gcc >= 8
-  * Optional: not supported by clang
- * Optional: not supported by icc
-  *
-  *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-nonstring-variable-attribute
-  */
-@@ -271,8 +262,19 @@
- #define __noreturn                      __attribute__((__noreturn__))
- 
- /*
-+ * Optional: only supported since GCC >= 11.1, clang >= 7.0.
-+ *
-+ *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-no_005fstack_005fprotector-function-attribute
-+ *   clang: https://clang.llvm.org/docs/AttributeReference.html#no-stack-protector-safebuffers
-+ */
-+#if __has_attribute(__no_stack_protector__)
-+# define __no_stack_protector		__attribute__((__no_stack_protector__))
-+#else
-+# define __no_stack_protector
-+#endif
-+
-+/*
-  * Optional: not supported by gcc.
- * Optional: not supported by icc.
-  *
-  * clang: https://clang.llvm.org/docs/AttributeReference.html#overloadable
-  */
-@@ -292,10 +294,14 @@
-  * Note: the "type" argument should match any __builtin_object_size(p, type) usage.
-  *
-  * Optional: not supported by gcc.
- * Optional: not supported by icc.
-  *
-  * clang: https://clang.llvm.org/docs/AttributeReference.html#pass-object-size-pass-dynamic-object-size
-  */
-+#if __has_attribute(__pass_dynamic_object_size__)
-+# define __pass_dynamic_object_size(type)	__attribute__((__pass_dynamic_object_size__(type)))
-+#else
-+# define __pass_dynamic_object_size(type)
-+#endif
- #if __has_attribute(__pass_object_size__)
- # define __pass_object_size(type)	__attribute__((__pass_object_size__(type)))
- #else
-@@ -315,6 +321,18 @@
- #define __section(section)              __attribute__((__section__(section)))
- 
- /*
-+ * Optional: only supported since gcc >= 12
-+ *
-+ *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-uninitialized-variable-attribute
-+ * clang: https://clang.llvm.org/docs/AttributeReference.html#uninitialized
-+ */
-+#if __has_attribute(__uninitialized__)
-+# define __uninitialized		__attribute__((__uninitialized__))
-+#else
-+# define __uninitialized
-+#endif
-+
-+/*
-  *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-unused-function-attribute
-  *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Type-Attributes.html#index-unused-type-attribute
-  *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-unused-variable-attribute
-@@ -331,6 +349,19 @@
- #define __used                          __attribute__((__used__))
- 
- /*
-+ * The __used attribute guarantees that the attributed variable will be
-+ * always emitted by a compiler. It doesn't prevent the compiler from
-+ * throwing 'unused' warnings when it can't detect how the variable is
-+ * actually used. It's a compiler implementation details either emit
-+ * the warning in that case or not.
-+ *
-+ * The combination of both 'used' and 'unused' attributes ensures that
-+ * the variable would be emitted, and will not trigger 'unused' warnings.
-+ * The attribute is applicable for functions, static and global variables.
-+ */
-+#define __always_used			__used __maybe_unused
-+
-+/*
-  *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-warn_005funused_005fresult-function-attribute
-  * clang: https://clang.llvm.org/docs/AttributeReference.html#nodiscard-warn-unused-result
-  */
-@@ -371,4 +402,11 @@
-  */
- #define __weak                          __attribute__((__weak__))
- 
-+/*
-+ * Used by functions that use '__builtin_return_address'. These function
-+ * don't want to be splited or made inline, which can make
-+ * the '__builtin_return_address' get unexpected address.
-+ */
-+#define __fix_address noinline __noclone
-+
- #endif /* __LINUX_COMPILER_ATTRIBUTES_H */
-diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h
-index 4f2a819fd60a..1280693766b9 100644
--- a/include/linux/compiler_types.h
-+++ b/include/linux/compiler_types.h
-@@ -2,10 +2,40 @@
- #ifndef __LINUX_COMPILER_TYPES_H
- #define __LINUX_COMPILER_TYPES_H
- 
-+/*
-+ * __has_builtin is supported on gcc >= 10, clang >= 3 and icc >= 21.
-+ * In the meantime, to support gcc < 10, we implement __has_builtin
-+ * by hand.
-+ */
-+#ifndef __has_builtin
-+#define __has_builtin(x) (0)
-+#endif
-+
-+/* Indirect macros required for expanded argument pasting, eg. __LINE__. */
-+#define ___PASTE(a, b) a##b
-+#define __PASTE(a, b) ___PASTE(a, b)
-+
- #ifndef __ASSEMBLY__
- 
-+/*
-+ * C23 introduces "auto" as a standard way to define type-inferred
-+ * variables, but "auto" has been a (useless) keyword even since K&R C,
-+ * so it has always been "namespace reserved."
-+ *
-+ * Until at some future time we require C23 support, we need the gcc
-+ * extension __auto_type, but there is no reason to put that elsewhere
-+ * in the source code.
-+ */
-+#if __STDC_VERSION__ < 202311L
-+# define auto __auto_type
-+#endif
-+
-+/*
-+ * Skipped when running bindgen due to a libclang issue;
-+ * see https://github.com/rust-lang/rust-bindgen/issues/2244.
-+ */
- #if defined(CONFIG_DEBUG_INFO_BTF) && defined(CONFIG_PAHOLE_HAS_BTF_TAG) && \
-	__has_attribute(btf_type_tag)
-+	__has_attribute(btf_type_tag) && !defined(__BINDGEN__)
- # define BTF_TYPE_TAG(value) __attribute__((btf_type_tag(#value)))
- #else
- # define BTF_TYPE_TAG(value) /* nothing */
-@@ -44,8 +74,9 @@ static inline void __chk_io_ptr(const volatile void __iomem *ptr) { }
- #  define __user	BTF_TYPE_TAG(user)
- # endif
- # define __iomem
-# define __percpu	BTF_TYPE_TAG(percpu)
-# define __rcu
-+# define __percpu	__percpu_qual BTF_TYPE_TAG(percpu)
-+# define __rcu		BTF_TYPE_TAG(rcu)
-+
- # define __chk_user_ptr(x)	(void)0
- # define __chk_io_ptr(x)	(void)0
- /* context/locking */
-@@ -65,31 +96,92 @@ static inline void __chk_io_ptr(const volatile void __iomem *ptr) { }
- # define __builtin_warning(x, y...) (1)
- #endif /* __CHECKER__ */
- 
-/* Indirect macros required for expanded argument pasting, eg. __LINE__. */
-#define ___PASTE(a,b) a##b
-#define __PASTE(a,b) ___PASTE(a,b)
-
- #ifdef __KERNEL__
- 
- /* Attributes */
- #include <linux/compiler_attributes.h>
- 
-/* Builtins */
-+#if CONFIG_FUNCTION_ALIGNMENT > 0
-+#define __function_aligned		__aligned(CONFIG_FUNCTION_ALIGNMENT)
-+#else
-+#define __function_aligned
-+#endif
- 
- /*
- * __has_builtin is supported on gcc >= 10, clang >= 3 and icc >= 21.
- * In the meantime, to support gcc < 10, we implement __has_builtin
- * by hand.
-+ *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-cold-function-attribute
-+ *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Label-Attributes.html#index-cold-label-attribute
-+ *
-+ * When -falign-functions=N is in use, we must avoid the cold attribute as
-+ * GCC drops the alignment for cold functions. Worse, GCC can implicitly mark
-+ * callees of cold functions as cold themselves, so it's not sufficient to add
-+ * __function_aligned here as that will not ensure that callees are correctly
-+ * aligned.
-+ *
-+ * See:
-+ *
-+ *   https://lore.kernel.org/lkml/Y77%2FqVgvaJidFpYt@FVFF77S0Q05N
-+ *   https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88345#c9
-  */
-#ifndef __has_builtin
-#define __has_builtin(x) (0)
-+#if defined(CONFIG_CC_HAS_SANE_FUNCTION_ALIGNMENT) || (CONFIG_FUNCTION_ALIGNMENT == 0)
-+#define __cold				__attribute__((__cold__))
-+#else
-+#define __cold
-+#endif
-+
-+/*
-+ * On x86-64 and arm64 targets, __preserve_most changes the calling convention
-+ * of a function to make the code in the caller as unintrusive as possible. This
-+ * convention behaves identically to the C calling convention on how arguments
-+ * and return values are passed, but uses a different set of caller- and callee-
-+ * saved registers.
-+ *
-+ * The purpose is to alleviates the burden of saving and recovering a large
-+ * register set before and after the call in the caller.  This is beneficial for
-+ * rarely taken slow paths, such as error-reporting functions that may be called
-+ * from hot paths.
-+ *
-+ * Note: This may conflict with instrumentation inserted on function entry which
-+ * does not use __preserve_most or equivalent convention (if in assembly). Since
-+ * function tracing assumes the normal C calling convention, where the attribute
-+ * is supported, __preserve_most implies notrace.  It is recommended to restrict
-+ * use of the attribute to functions that should or already disable tracing.
-+ *
-+ * Optional: not supported by gcc.
-+ *
-+ * clang: https://clang.llvm.org/docs/AttributeReference.html#preserve-most
-+ */
-+#if __has_attribute(__preserve_most__) && (defined(CONFIG_X86_64) || defined(CONFIG_ARM64))
-+# define __preserve_most notrace __attribute__((__preserve_most__))
-+#else
-+# define __preserve_most
-+#endif
-+
-+/*
-+ * Annotating a function/variable with __retain tells the compiler to place
-+ * the object in its own section and set the flag SHF_GNU_RETAIN. This flag
-+ * instructs the linker to retain the object during garbage-cleanup or LTO
-+ * phases.
-+ *
-+ * Note that the __used macro is also used to prevent functions or data
-+ * being optimized out, but operates at the compiler/IR-level and may still
-+ * allow unintended removal of objects during linking.
-+ *
-+ * Optional: only supported since gcc >= 11, clang >= 13
-+ *
-+ *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-retain-function-attribute
-+ * clang: https://clang.llvm.org/docs/AttributeReference.html#retain
-+ */
-+#if __has_attribute(__retain__) && \
-+	(defined(CONFIG_LD_DEAD_CODE_DATA_ELIMINATION) || \
-+	 defined(CONFIG_LTO_CLANG))
-+# define __retain			__attribute__((__retain__))
-+#else
-+# define __retain
- #endif
- 
- /* Compiler specific macros. */
- #ifdef __clang__
- #include <linux/compiler-clang.h>
-#elif defined(__INTEL_COMPILER)
-#include <linux/compiler-intel.h>
- #elif defined(__GNUC__)
- /* The above compilers also define __GNUC__, so order is important here. */
- #include <linux/compiler-gcc.h>
-@@ -171,10 +263,9 @@ struct ftrace_likely_data {
- /*
-  * GCC does not warn about unused static inline functions for -Wunused-function.
-  * Suppress the warning in clang as well by using __maybe_unused, but enable it
- * for W=1 build. This will allow clang to find unused functions. Remove the
- * __inline_maybe_unused entirely after fixing most of -Wunused-function warnings.
-+ * for W=2 build. This will allow clang to find unused functions.
-  */
-#ifdef KBUILD_EXTRA_WARN1
-+#ifdef KBUILD_EXTRA_WARN2
- #define __inline_maybe_unused
- #else
- #define __inline_maybe_unused __maybe_unused
-@@ -187,6 +278,12 @@ struct ftrace_likely_data {
- #define noinline_for_stack noinline
- 
- /*
-+ * Use noinline_for_tracing for functions that should not be inlined.
-+ * For tracing reasons.
-+ */
-+#define noinline_for_tracing noinline
-+
-+/*
-  * Sanitizer helper attributes: Because using __always_inline and
-  * __no_sanitize_* conflict, provide helper attributes that will either expand
-  * to __no_sanitize_* in compilation units where instrumentation is enabled
-@@ -217,19 +314,139 @@ struct ftrace_likely_data {
-  * disable all instrumentation. See Kconfig.kcsan where this is mandatory.
-  */
- # define __no_kcsan __no_sanitize_thread __disable_sanitizer_instrumentation
-+/*
-+ * Type qualifier to mark variables where all data-racy accesses should be
-+ * ignored by KCSAN. Note, the implementation simply marks these variables as
-+ * volatile, since KCSAN will treat such accesses as "marked".
-+ */
-+# define __data_racy volatile
- # define __no_sanitize_or_inline __no_kcsan notrace __maybe_unused
- #else
- # define __no_kcsan
-+# define __data_racy
-+#endif
-+
-+#ifdef __SANITIZE_MEMORY__
-+/*
-+ * Similarly to KASAN and KCSAN, KMSAN loses function attributes of inlined
-+ * functions, therefore disabling KMSAN checks also requires disabling inlining.
-+ *
-+ * __no_sanitize_or_inline effectively prevents KMSAN from reporting errors
-+ * within the function and marks all its outputs as initialized.
-+ */
-+# define __no_sanitize_or_inline __no_kmsan_checks notrace __maybe_unused
- #endif
- 
- #ifndef __no_sanitize_or_inline
- #define __no_sanitize_or_inline __always_inline
- #endif
- 
-+/*
-+ * The assume attribute is used to indicate that a certain condition is
-+ * assumed to be true. If this condition is violated at runtime, the behavior
-+ * is undefined. Compilers may or may not use this indication to generate
-+ * optimized code.
-+ *
-+ * Note that the clang documentation states that optimizers may react
-+ * differently to this attribute, and this may even have a negative
-+ * performance impact. Therefore this attribute should be used with care.
-+ *
-+ * Optional: only supported since gcc >= 13
-+ * Optional: only supported since clang >= 19
-+ *
-+ *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Statement-Attributes.html#index-assume-statement-attribute
-+ * clang: https://clang.llvm.org/docs/AttributeReference.html#id13
-+ *
-+ */
-+#ifdef CONFIG_CC_HAS_ASSUME
-+# define __assume(expr)			__attribute__((__assume__(expr)))
-+#else
-+# define __assume(expr)
-+#endif
-+
-+/*
-+ * Optional: only supported since gcc >= 15
-+ * Optional: only supported since clang >= 18
-+ *
-+ *   gcc: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108896
-+ * clang: https://github.com/llvm/llvm-project/pull/76348
-+ *
-+ * __bdos on clang < 19.1.2 can erroneously return 0:
-+ * https://github.com/llvm/llvm-project/pull/110497
-+ *
-+ * __bdos on clang < 19.1.3 can be off by 4:
-+ * https://github.com/llvm/llvm-project/pull/112636
-+ */
-+#ifdef CONFIG_CC_HAS_COUNTED_BY
-+# define __counted_by(member)		__attribute__((__counted_by__(member)))
-+#else
-+# define __counted_by(member)
-+#endif
-+
-+/*
-+ * Optional: only supported since gcc >= 15
-+ * Optional: not supported by Clang
-+ *
-+ * gcc: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=117178
-+ */
-+#ifdef CONFIG_CC_HAS_MULTIDIMENSIONAL_NONSTRING
-+# define __nonstring_array		__attribute__((__nonstring__))
-+#else
-+# define __nonstring_array
-+#endif
-+
-+/*
-+ * Apply __counted_by() when the Endianness matches to increase test coverage.
-+ */
-+#ifdef __LITTLE_ENDIAN
-+#define __counted_by_le(member)	__counted_by(member)
-+#define __counted_by_be(member)
-+#else
-+#define __counted_by_le(member)
-+#define __counted_by_be(member)	__counted_by(member)
-+#endif
-+
-+/*
-+ * This designates the minimum number of elements a passed array parameter must
-+ * have. For example:
-+ *
-+ *     void some_function(u8 param[at_least 7]);
-+ *
-+ * If a caller passes an array with fewer than 7 elements, the compiler will
-+ * emit a warning.
-+ */
-+#ifndef __CHECKER__
-+#define at_least static
-+#else
-+#define at_least
-+#endif
-+
-+/* Do not trap wrapping arithmetic within an annotated function. */
-+#ifdef CONFIG_UBSAN_INTEGER_WRAP
-+# define __signed_wrap __attribute__((no_sanitize("signed-integer-overflow")))
-+#else
-+# define __signed_wrap
-+#endif
-+
- /* Section for code which can't be instrumented at all */
-#define noinstr								\
-	noinline notrace __attribute((__section__(".noinstr.text")))	\
-	__no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage
-+#define __noinstr_section(section)					\
-+	noinline notrace __attribute((__section__(section)))		\
-+	__no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage \
-+	__no_sanitize_memory __signed_wrap
-+
-+#define noinstr __noinstr_section(".noinstr.text")
-+
-+/*
-+ * The __cpuidle section is used twofold:
-+ *
-+ *  1) the original use -- identifying if a CPU is 'stuck' in idle state based
-+ *     on it's instruction pointer. See cpu_in_idle().
-+ *
-+ *  2) supressing instrumentation around where cpuidle disables RCU; where the
-+ *     function isn't strictly required for #1, this is interchangeable with
-+ *     noinstr.
-+ */
-+#define __cpuidle __noinstr_section(".cpuidle.text")
- 
- #endif /* __KERNEL__ */
- 
-@@ -257,32 +474,85 @@ struct ftrace_likely_data {
- # define randomized_struct_fields_end
- #endif
- 
-+#ifndef __no_kstack_erase
-+# define __no_kstack_erase
-+#endif
-+
- #ifndef __noscs
- # define __noscs
- #endif
- 
-#ifndef __nocfi
-+#if defined(CONFIG_CFI)
-+# define __nocfi		__attribute__((__no_sanitize__("kcfi")))
-+#else
- # define __nocfi
- #endif
- 
-#ifndef __cficanonical
-# define __cficanonical
-+#if defined(CONFIG_ARCH_USES_CFI_GENERIC_LLVM_PASS)
-+# define __nocfi_generic	__nocfi
-+#else
-+# define __nocfi_generic
- #endif
- 
- /*
-  * Any place that could be marked with the "alloc_size" attribute is also
- * a place to be marked with the "malloc" attribute. Do this as part of the
- * __alloc_size macro to avoid redundant attributes and to avoid missing a
- * __malloc marking.
-+ * a place to be marked with the "malloc" attribute, except those that may
-+ * be performing a _reallocation_, as that may alias the existing pointer.
-+ * For these, use __realloc_size().
-  */
- #ifdef __alloc_size__
- # define __alloc_size(x, ...)	__alloc_size__(x, ## __VA_ARGS__) __malloc
-+# define __realloc_size(x, ...)	__alloc_size__(x, ## __VA_ARGS__)
- #else
- # define __alloc_size(x, ...)	__malloc
-+# define __realloc_size(x, ...)
-+#endif
-+
-+/*
-+ * When the size of an allocated object is needed, use the best available
-+ * mechanism to find it. (For cases where sizeof() cannot be used.)
-+ *
-+ * Optional: only supported since gcc >= 12
-+ *
-+ *   gcc: https://gcc.gnu.org/onlinedocs/gcc/Object-Size-Checking.html
-+ * clang: https://clang.llvm.org/docs/LanguageExtensions.html#evaluating-object-size
-+ */
-+#if __has_builtin(__builtin_dynamic_object_size)
-+#define __struct_size(p)	__builtin_dynamic_object_size(p, 0)
-+#define __member_size(p)	__builtin_dynamic_object_size(p, 1)
-+#else
-+#define __struct_size(p)	__builtin_object_size(p, 0)
-+#define __member_size(p)	__builtin_object_size(p, 1)
-+#endif
-+
-+/*
-+ * Determine if an attribute has been applied to a variable.
-+ * Using __annotated needs to check for __annotated being available,
-+ * or negative tests may fail when annotation cannot be checked. For
-+ * example, see the definition of __is_cstr().
-+ */
-+#if __has_builtin(__builtin_has_attribute)
-+#define __annotated(var, attr)	__builtin_has_attribute(var, attr)
-+#endif
-+
-+/*
-+ * Some versions of gcc do not mark 'asm goto' volatile:
-+ *
-+ *  https://gcc.gnu.org/bugzilla/show_bug.cgi?id=103979
-+ *
-+ * We do it here by hand, because it doesn't hurt.
-+ */
-+#ifndef asm_goto_output
-+#define asm_goto_output(x...) asm volatile goto(x)
- #endif
- 
-#ifndef asm_volatile_goto
-#define asm_volatile_goto(x...) asm goto(x)
-+/*
-+ * Clang has trouble with constraints with multiple
-+ * alternative behaviors (mainly "g" and "rm").
-+ */
-+#ifndef ASM_INPUT_G
-+  #define ASM_INPUT_G "g"
-+  #define ASM_INPUT_RM "rm"
- #endif
- 
- #ifdef CONFIG_CC_HAS_ASM_INLINE
-@@ -322,6 +592,12 @@ struct ftrace_likely_data {
- 	 sizeof(t) == sizeof(int) || sizeof(t) == sizeof(long))
- 
- #ifdef __OPTIMIZE__
-+/*
-+ * #ifdef __OPTIMIZE__ is only a good approximation; for instance "make
-+ * CFLAGS_foo.o=-Og" defines __OPTIMIZE__, does not elide the conditional code
-+ * and can break compilation with wrong error message(s). Combine with
-+ * -U__OPTIMIZE__ when needed.
-+ */
- # define __compiletime_assert(condition, msg, prefix, suffix)		\
- 	do {								\
- 		/*							\
-@@ -335,7 +611,7 @@ struct ftrace_likely_data {
- 			prefix ## suffix();				\
- 	} while (0)
- #else
-# define __compiletime_assert(condition, msg, prefix, suffix) do { } while (0)
-+# define __compiletime_assert(condition, msg, prefix, suffix) ((void)(condition))
- #endif
- 
- #define _compiletime_assert(condition, msg, prefix, suffix) \
-diff --git a/include/linux/completion.h b/include/linux/completion.h
-index 51d9ab079629..fb2915676574 100644
--- a/include/linux/completion.h
-+++ b/include/linux/completion.h
-@@ -103,6 +103,7 @@ extern void wait_for_completion(struct completion *);
- extern void wait_for_completion_io(struct completion *);
- extern int wait_for_completion_interruptible(struct completion *x);
- extern int wait_for_completion_killable(struct completion *x);
-+extern int wait_for_completion_state(struct completion *x, unsigned int state);
- extern unsigned long wait_for_completion_timeout(struct completion *x,
- 						   unsigned long timeout);
- extern unsigned long wait_for_completion_io_timeout(struct completion *x,
-@@ -115,6 +116,7 @@ extern bool try_wait_for_completion(struct completion *x);
- extern bool completion_done(struct completion *x);
- 
- extern void complete(struct completion *);
-+extern void complete_on_current_cpu(struct completion *x);
- extern void complete_all(struct completion *);
- 
- #endif
-diff --git a/include/linux/component.h b/include/linux/component.h
-index df4aa75c9e7c..9d6c66401280 100644
--- a/include/linux/component.h
-+++ b/include/linux/component.h
-@@ -3,7 +3,7 @@
- #define COMPONENT_H
- 
- #include <linux/stddef.h>
-
-+#include <linux/types.h>
- 
- struct device;
- 
-@@ -90,6 +90,8 @@ int component_compare_dev_name(struct device *dev, void *data);
- 
- void component_master_del(struct device *,
- 	const struct component_master_ops *);
-+bool component_master_is_bound(struct device *parent,
-+	const struct component_master_ops *ops);
- 
- struct component_match;
- 
-diff --git a/include/linux/configfs.h b/include/linux/configfs.h
-index 97cfd13bae51..ef65c75beeaa 100644
--- a/include/linux/configfs.h
-+++ b/include/linux/configfs.h
-@@ -64,8 +64,8 @@ extern void config_item_put(struct config_item *);
- 
- struct config_item_type {
- 	struct module				*ct_owner;
-	struct configfs_item_operations		*ct_item_ops;
-	struct configfs_group_operations	*ct_group_ops;
-+	const struct configfs_item_operations	*ct_item_ops;
-+	const struct configfs_group_operations	*ct_group_ops;
- 	struct configfs_attribute		**ct_attrs;
- 	struct configfs_bin_attribute		**ct_bin_attrs;
- };
-@@ -120,15 +120,19 @@ struct configfs_attribute {
- 	ssize_t (*store)(struct config_item *, const char *, size_t);
- };
- 
-#define CONFIGFS_ATTR(_pfx, _name)			\
-+#define CONFIGFS_ATTR_PERM(_pfx, _name, _perm)		\
- static struct configfs_attribute _pfx##attr_##_name = {	\
- 	.ca_name	= __stringify(_name),		\
-	.ca_mode	= S_IRUGO | S_IWUSR,		\
-+	.ca_mode	= _perm,			\
- 	.ca_owner	= THIS_MODULE,			\
- 	.show		= _pfx##_name##_show,		\
- 	.store		= _pfx##_name##_store,		\
- }
- 
-+#define CONFIGFS_ATTR(_pfx, _name) CONFIGFS_ATTR_PERM(	\
-+		_pfx, _name, S_IRUGO | S_IWUSR		\
-+)
-+
- #define CONFIGFS_ATTR_RO(_pfx, _name)			\
- static struct configfs_attribute _pfx##attr_##_name = {	\
- 	.ca_name	= __stringify(_name),		\
-@@ -204,8 +208,6 @@ static struct configfs_bin_attribute _pfx##attr_##_name = {	\
-  * group children.  default_groups may coexist alongsize make_group() or
-  * make_item(), but if the group wishes to have only default_groups
-  * children (disallowing mkdir(2)), it need not provide either function.
- * If the group has commit(), it supports pending and committed (active)
- * items.
-  */
- struct configfs_item_operations {
- 	void (*release)(struct config_item *);
-@@ -216,9 +218,11 @@ struct configfs_item_operations {
- struct configfs_group_operations {
- 	struct config_item *(*make_item)(struct config_group *group, const char *name);
- 	struct config_group *(*make_group)(struct config_group *group, const char *name);
-	int (*commit_item)(struct config_item *item);
- 	void (*disconnect_notify)(struct config_group *group, struct config_item *item);
- 	void (*drop_item)(struct config_group *group, struct config_item *item);
-+	bool (*is_visible)(struct config_item *item, struct configfs_attribute *attr, int n);
-+	bool (*is_bin_visible)(struct config_item *item, struct configfs_bin_attribute *attr,
-+			       int n);
- };
- 
- struct configfs_subsystem {
-diff --git a/include/linux/connector.h b/include/linux/connector.h
-index 487350bb19c3..70bc1160f3d8 100644
--- a/include/linux/connector.h
-+++ b/include/linux/connector.h
-@@ -90,13 +90,18 @@ void cn_del_callback(const struct cb_id *id);
-  *		If @group is not zero, then message will be delivered
-  *		to the specified group.
-  * @gfp_mask:	GFP mask.
-+ * @filter:     Filter function to be used at netlink layer.
-+ * @filter_data:Filter data to be supplied to the filter function
-  *
-  * It can be safely called from softirq context, but may silently
-  * fail under strong memory pressure.
-  *
-  * If there are no listeners for given group %-ESRCH can be returned.
-  */
-int cn_netlink_send_mult(struct cn_msg *msg, u16 len, u32 portid, u32 group, gfp_t gfp_mask);
-+int cn_netlink_send_mult(struct cn_msg *msg, u16 len, u32 portid,
-+			 u32 group, gfp_t gfp_mask,
-+			 netlink_filter_fn filter,
-+			 void *filter_data);
- 
- /**
-  * cn_netlink_send - Sends message to the specified groups.
-diff --git a/include/linux/console.h b/include/linux/console.h
-index 8c1686e2c233..fc9f5c5c1b04 100644
--- a/include/linux/console.h
-+++ b/include/linux/console.h
-@@ -15,7 +15,13 @@
- #define _LINUX_CONSOLE_H_ 1
- 
- #include <linux/atomic.h>
-+#include <linux/bits.h>
-+#include <linux/irq_work.h>
-+#include <linux/rculist.h>
-+#include <linux/rcuwait.h>
-+#include <linux/smp.h>
- #include <linux/types.h>
-+#include <linux/vesa.h>
- 
- struct vc_data;
- struct console_font_op;
-@@ -34,62 +40,91 @@ enum vc_intensity;
- /**
-  * struct consw - callbacks for consoles
-  *
-+ * @owner:      the module to get references of when this console is used
-+ * @con_startup: set up the console and return its name (like VGA, EGA, ...)
-+ * @con_init:   initialize the console on @vc. @init is true for the very first
-+ *		call on this @vc.
-+ * @con_deinit: deinitialize the console from @vc.
-+ * @con_clear:  erase @count characters at [@x, @y] on @vc. @count >= 1.
-+ * @con_putc:   emit one character with attributes @ca to [@x, @y] on @vc.
-+ *		(optional -- @con_putcs would be called instead)
-+ * @con_putcs:  emit @count characters with attributes @s to [@x, @y] on @vc.
-+ * @con_cursor: enable/disable cursor depending on @enable
-  * @con_scroll: move lines from @top to @bottom in direction @dir by @lines.
-  *		Return true if no generic handling should be done.
-  *		Invoked by csi_M and printing to the console.
- * @con_set_palette: sets the palette of the console to @table (optional)
-+ * @con_switch: notifier about the console switch; it is supposed to return
-+ *		true if a redraw is needed.
-+ * @con_blank:  blank/unblank the console. The target mode is passed in @blank.
-+ *		@mode_switch is set if changing from/to text/graphics. The hook
-+ *		is supposed to return true if a redraw is needed.
-+ * @con_font_set: set console @vc font to @font with height @vpitch. @flags can
-+ *		be %KD_FONT_FLAG_DONT_RECALC. (optional)
-+ * @con_font_get: fetch the current font on @vc of height @vpitch into @font.
-+ *		(optional)
-+ * @con_font_default: set default font on @vc. @name can be %NULL or font name
-+ *		to search for. @font can be filled back. (optional)
-+ * @con_resize:	resize the @vc console to @width x @height. @from_user is true
-+ *		when this change comes from the user space.
-+ * @con_set_palette: sets the palette of the console @vc to @table (optional)
-  * @con_scrolldelta: the contents of the console should be scrolled by @lines.
-  *		     Invoked by user. (optional)
-+ * @con_set_origin: set origin (see &vc_data::vc_origin) of the @vc. If not
-+ *		provided or returns false, the origin is set to
-+ *		@vc->vc_screenbuf. (optional)
-+ * @con_save_screen: save screen content into @vc->vc_screenbuf. Called e.g.
-+ *		upon entering graphics. (optional)
-+ * @con_build_attr: build attributes based on @color, @intensity and other
-+ *		parameters. The result is used for both normal and erase
-+ *		characters. (optional)
-+ * @con_invert_region: invert a region of length @count on @vc starting at @p.
-+ *		(optional)
-+ * @con_debug_enter: prepare the console for the debugger. This includes, but
-+ *		is not limited to, unblanking the console, loading an
-+ *		appropriate palette, and allowing debugger generated output.
-+ *		(optional)
-+ * @con_debug_leave: restore the console to its pre-debug state as closely as
-+ *		possible. (optional)
-  */
- struct consw {
- 	struct module *owner;
- 	const char *(*con_startup)(void);
-	void	(*con_init)(struct vc_data *vc, int init);
-+	void	(*con_init)(struct vc_data *vc, bool init);
- 	void	(*con_deinit)(struct vc_data *vc);
-	void	(*con_clear)(struct vc_data *vc, int sy, int sx, int height,
-			int width);
-	void	(*con_putc)(struct vc_data *vc, int c, int ypos, int xpos);
-	void	(*con_putcs)(struct vc_data *vc, const unsigned short *s,
-			int count, int ypos, int xpos);
-	void	(*con_cursor)(struct vc_data *vc, int mode);
-+	void	(*con_clear)(struct vc_data *vc, unsigned int y,
-+			     unsigned int x, unsigned int count);
-+	void	(*con_putc)(struct vc_data *vc, u16 ca, unsigned int y,
-+			    unsigned int x);
-+	void	(*con_putcs)(struct vc_data *vc, const u16 *s,
-+			     unsigned int count, unsigned int ypos,
-+			     unsigned int xpos);
-+	void	(*con_cursor)(struct vc_data *vc, bool enable);
- 	bool	(*con_scroll)(struct vc_data *vc, unsigned int top,
- 			unsigned int bottom, enum con_scroll dir,
- 			unsigned int lines);
-	int	(*con_switch)(struct vc_data *vc);
-	int	(*con_blank)(struct vc_data *vc, int blank, int mode_switch);
-	int	(*con_font_set)(struct vc_data *vc, struct console_font *font,
-			unsigned int flags);
-	int	(*con_font_get)(struct vc_data *vc, struct console_font *font);
-+	bool	(*con_switch)(struct vc_data *vc);
-+	bool	(*con_blank)(struct vc_data *vc, enum vesa_blank_mode blank,
-+			     bool mode_switch);
-+	int	(*con_font_set)(struct vc_data *vc,
-+				const struct console_font *font,
-+				unsigned int vpitch, unsigned int flags);
-+	int	(*con_font_get)(struct vc_data *vc, struct console_font *font,
-+			unsigned int vpitch);
- 	int	(*con_font_default)(struct vc_data *vc,
-			struct console_font *font, char *name);
-+			struct console_font *font, const char *name);
- 	int     (*con_resize)(struct vc_data *vc, unsigned int width,
-			unsigned int height, unsigned int user);
-+			      unsigned int height, bool from_user);
- 	void	(*con_set_palette)(struct vc_data *vc,
- 			const unsigned char *table);
- 	void	(*con_scrolldelta)(struct vc_data *vc, int lines);
-	int	(*con_set_origin)(struct vc_data *vc);
-+	bool	(*con_set_origin)(struct vc_data *vc);
- 	void	(*con_save_screen)(struct vc_data *vc);
- 	u8	(*con_build_attr)(struct vc_data *vc, u8 color,
- 			enum vc_intensity intensity,
- 			bool blink, bool underline, bool reverse, bool italic);
- 	void	(*con_invert_region)(struct vc_data *vc, u16 *p, int count);
-	u16    *(*con_screen_pos)(const struct vc_data *vc, int offset);
-	unsigned long (*con_getxy)(struct vc_data *vc, unsigned long position,
-			int *px, int *py);
-	/*
-	 * Flush the video console driver's scrollback buffer
-	 */
-	void	(*con_flush_scrollback)(struct vc_data *vc);
-	/*
-	 * Prepare the console for the debugger.  This includes, but is not
-	 * limited to, unblanking the console, loading an appropriate
-	 * palette, and allowing debugger generated output.
-	 */
-	int	(*con_debug_enter)(struct vc_data *vc);
-	/*
-	 * Restore the console to its pre-debug state as closely as possible.
-	 */
-	int	(*con_debug_leave)(struct vc_data *vc);
-+	void	(*con_debug_enter)(struct vc_data *vc);
-+	void	(*con_debug_leave)(struct vc_data *vc);
- };
- 
- extern const struct consw *conswitchp;
-@@ -98,70 +133,554 @@ extern const struct consw dummy_con;	/* dummy console buffer */
- extern const struct consw vga_con;	/* VGA text console */
- extern const struct consw newport_con;	/* SGI Newport console  */
- 
-+struct screen_info;
-+#ifdef CONFIG_VGA_CONSOLE
-+void vgacon_register_screen(struct screen_info *si);
-+#else
-+static inline void vgacon_register_screen(struct screen_info *si) { }
-+#endif
-+
- int con_is_bound(const struct consw *csw);
- int do_unregister_con_driver(const struct consw *csw);
- int do_take_over_console(const struct consw *sw, int first, int last, int deflt);
- void give_up_console(const struct consw *sw);
-#ifdef CONFIG_HW_CONSOLE
-int con_debug_enter(struct vc_data *vc);
-int con_debug_leave(void);
-+#ifdef CONFIG_VT
-+void con_debug_enter(struct vc_data *vc);
-+void con_debug_leave(void);
-+#else
-+static inline void con_debug_enter(struct vc_data *vc) { }
-+static inline void con_debug_leave(void) { }
-+#endif
-+
-+/*
-+ * The interface for a console, or any other device that wants to capture
-+ * console messages (printer driver?)
-+ */
-+
-+/**
-+ * enum cons_flags - General console flags
-+ * @CON_PRINTBUFFER:	Used by newly registered consoles to avoid duplicate
-+ *			output of messages that were already shown by boot
-+ *			consoles or read by userspace via syslog() syscall.
-+ * @CON_CONSDEV:	Indicates that the console driver is backing
-+ *			/dev/console.
-+ * @CON_ENABLED:	Indicates if a console is allowed to print records. If
-+ *			false, the console also will not advance to later
-+ *			records.
-+ * @CON_BOOT:		Marks the console driver as early console driver which
-+ *			is used during boot before the real driver becomes
-+ *			available. It will be automatically unregistered
-+ *			when the real console driver is registered unless
-+ *			"keep_bootcon" parameter is used.
-+ * @CON_ANYTIME:	A misnomed historical flag which tells the core code
-+ *			that the legacy @console::write callback can be invoked
-+ *			on a CPU which is marked OFFLINE. That is misleading as
-+ *			it suggests that there is no contextual limit for
-+ *			invoking the callback. The original motivation was
-+ *			readiness of the per-CPU areas.
-+ * @CON_BRL:		Indicates a braille device which is exempt from
-+ *			receiving the printk spam for obvious reasons.
-+ * @CON_EXTENDED:	The console supports the extended output format of
-+ *			/dev/kmesg which requires a larger output buffer.
-+ * @CON_SUSPENDED:	Indicates if a console is suspended. If true, the
-+ *			printing callbacks must not be called.
-+ * @CON_NBCON:		Console can operate outside of the legacy style console_lock
-+ *			constraints.
-+ * @CON_NBCON_ATOMIC_UNSAFE: The write_atomic() callback is not safe and is
-+ *			therefore only used by nbcon_atomic_flush_unsafe().
-+ */
-+enum cons_flags {
-+	CON_PRINTBUFFER		= BIT(0),
-+	CON_CONSDEV		= BIT(1),
-+	CON_ENABLED		= BIT(2),
-+	CON_BOOT		= BIT(3),
-+	CON_ANYTIME		= BIT(4),
-+	CON_BRL			= BIT(5),
-+	CON_EXTENDED		= BIT(6),
-+	CON_SUSPENDED		= BIT(7),
-+	CON_NBCON		= BIT(8),
-+	CON_NBCON_ATOMIC_UNSAFE	= BIT(9),
-+};
-+
-+/**
-+ * struct nbcon_state - console state for nbcon consoles
-+ * @atom:	Compound of the state fields for atomic operations
-+ *
-+ * @req_prio:		The priority of a handover request
-+ * @prio:		The priority of the current owner
-+ * @unsafe:		Console is busy in a non takeover region
-+ * @unsafe_takeover:	A hostile takeover in an unsafe state happened in the
-+ *			past. The console cannot be safe until re-initialized.
-+ * @cpu:		The CPU on which the owner runs
-+ *
-+ * To be used for reading and preparing of the value stored in the nbcon
-+ * state variable @console::nbcon_state.
-+ *
-+ * The @prio and @req_prio fields are particularly important to allow
-+ * spin-waiting to timeout and give up without the risk of a waiter being
-+ * assigned the lock after giving up.
-+ */
-+struct nbcon_state {
-+	union {
-+		unsigned int	atom;
-+		struct {
-+			unsigned int prio		:  2;
-+			unsigned int req_prio		:  2;
-+			unsigned int unsafe		:  1;
-+			unsigned int unsafe_takeover	:  1;
-+			unsigned int cpu		: 24;
-+		};
-+	};
-+};
-+
-+/*
-+ * The nbcon_state struct is used to easily create and interpret values that
-+ * are stored in the @console::nbcon_state variable. Ensure this struct stays
-+ * within the size boundaries of the atomic variable's underlying type in
-+ * order to avoid any accidental truncation.
-+ */
-+static_assert(sizeof(struct nbcon_state) <= sizeof(int));
-+
-+/**
-+ * enum nbcon_prio - console owner priority for nbcon consoles
-+ * @NBCON_PRIO_NONE:		Unused
-+ * @NBCON_PRIO_NORMAL:		Normal (non-emergency) usage
-+ * @NBCON_PRIO_EMERGENCY:	Emergency output (WARN/OOPS...)
-+ * @NBCON_PRIO_PANIC:		Panic output
-+ * @NBCON_PRIO_MAX:		The number of priority levels
-+ *
-+ * A higher priority context can takeover the console when it is
-+ * in the safe state. The final attempt to flush consoles in panic()
-+ * can be allowed to do so even in an unsafe state (Hope and pray).
-+ */
-+enum nbcon_prio {
-+	NBCON_PRIO_NONE = 0,
-+	NBCON_PRIO_NORMAL,
-+	NBCON_PRIO_EMERGENCY,
-+	NBCON_PRIO_PANIC,
-+	NBCON_PRIO_MAX,
-+};
-+
-+struct console;
-+struct printk_buffers;
-+
-+/**
-+ * struct nbcon_context - Context for console acquire/release
-+ * @console:			The associated console
-+ * @spinwait_max_us:		Limit for spin-wait acquire
-+ * @prio:			Priority of the context
-+ * @allow_unsafe_takeover:	Allow performing takeover even if unsafe. Can
-+ *				be used only with NBCON_PRIO_PANIC @prio. It
-+ *				might cause a system freeze when the console
-+ *				is used later.
-+ * @backlog:			Ringbuffer has pending records
-+ * @pbufs:			Pointer to the text buffer for this context
-+ * @seq:			The sequence number to print for this context
-+ */
-+struct nbcon_context {
-+	/* members set by caller */
-+	struct console		*console;
-+	unsigned int		spinwait_max_us;
-+	enum nbcon_prio		prio;
-+	unsigned int		allow_unsafe_takeover	: 1;
-+
-+	/* members set by emit */
-+	unsigned int		backlog			: 1;
-+
-+	/* members set by acquire */
-+	struct printk_buffers	*pbufs;
-+	u64			seq;
-+};
-+
-+/**
-+ * struct nbcon_write_context - Context handed to the nbcon write callbacks
-+ * @ctxt:		The core console context
-+ * @outbuf:		Pointer to the text buffer for output
-+ * @len:		Length to write
-+ * @unsafe_takeover:	If a hostile takeover in an unsafe state has occurred
-+ */
-+struct nbcon_write_context {
-+	struct nbcon_context	__private ctxt;
-+	char			*outbuf;
-+	unsigned int		len;
-+	bool			unsafe_takeover;
-+};
-+
-+/**
-+ * struct console - The console descriptor structure
-+ * @name:		The name of the console driver
-+ * @write:		Legacy write callback to output messages (Optional)
-+ * @read:		Read callback for console input (Optional)
-+ * @device:		The underlying TTY device driver (Optional)
-+ * @unblank:		Callback to unblank the console (Optional)
-+ * @setup:		Callback for initializing the console (Optional)
-+ * @exit:		Callback for teardown of the console (Optional)
-+ * @match:		Callback for matching a console (Optional)
-+ * @flags:		Console flags. See enum cons_flags
-+ * @index:		Console index, e.g. port number
-+ * @cflag:		TTY control mode flags
-+ * @ispeed:		TTY input speed
-+ * @ospeed:		TTY output speed
-+ * @seq:		Sequence number of the next ringbuffer record to print
-+ * @dropped:		Number of unreported dropped ringbuffer records
-+ * @data:		Driver private data
-+ * @node:		hlist node for the console list
-+ *
-+ * @nbcon_state:	State for nbcon consoles
-+ * @nbcon_seq:		Sequence number of the next record for nbcon to print
-+ * @nbcon_device_ctxt:	Context available for non-printing operations
-+ * @nbcon_prev_seq:	Seq num the previous nbcon owner was assigned to print
-+ * @pbufs:		Pointer to nbcon private buffer
-+ * @kthread:		Printer kthread for this console
-+ * @rcuwait:		RCU-safe wait object for @kthread waking
-+ * @irq_work:		Defer @kthread waking to IRQ work context
-+ */
-+struct console {
-+	char			name[16];
-+	void			(*write)(struct console *co, const char *s, unsigned int count);
-+	int			(*read)(struct console *co, char *s, unsigned int count);
-+	struct tty_driver	*(*device)(struct console *co, int *index);
-+	void			(*unblank)(void);
-+	int			(*setup)(struct console *co, char *options);
-+	int			(*exit)(struct console *co);
-+	int			(*match)(struct console *co, char *name, int idx, char *options);
-+	short			flags;
-+	short			index;
-+	int			cflag;
-+	uint			ispeed;
-+	uint			ospeed;
-+	u64			seq;
-+	unsigned long		dropped;
-+	void			*data;
-+	struct hlist_node	node;
-+
-+	/* nbcon console specific members */
-+
-+	/**
-+	 * @write_atomic:
-+	 *
-+	 * NBCON callback to write out text in any context. (Optional)
-+	 *
-+	 * This callback is called with the console already acquired. However,
-+	 * a higher priority context is allowed to take it over by default.
-+	 *
-+	 * The callback must call nbcon_enter_unsafe() and nbcon_exit_unsafe()
-+	 * around any code where the takeover is not safe, for example, when
-+	 * manipulating the serial port registers.
-+	 *
-+	 * nbcon_enter_unsafe() will fail if the context has lost the console
-+	 * ownership in the meantime. In this case, the callback is no longer
-+	 * allowed to go forward. It must back out immediately and carefully.
-+	 * The buffer content is also no longer trusted since it no longer
-+	 * belongs to the context.
-+	 *
-+	 * The callback should allow the takeover whenever it is safe. It
-+	 * increases the chance to see messages when the system is in trouble.
-+	 * If the driver must reacquire ownership in order to finalize or
-+	 * revert hardware changes, nbcon_reacquire_nobuf() can be used.
-+	 * However, on reacquire the buffer content is no longer available. A
-+	 * reacquire cannot be used to resume printing.
-+	 *
-+	 * The callback can be called from any context (including NMI).
-+	 * Therefore it must avoid usage of any locking and instead rely
-+	 * on the console ownership for synchronization.
-+	 */
-+	void (*write_atomic)(struct console *con, struct nbcon_write_context *wctxt);
-+
-+	/**
-+	 * @write_thread:
-+	 *
-+	 * NBCON callback to write out text in task context.
-+	 *
-+	 * This callback must be called only in task context with both
-+	 * device_lock() and the nbcon console acquired with
-+	 * NBCON_PRIO_NORMAL.
-+	 *
-+	 * The same rules for console ownership verification and unsafe
-+	 * sections handling applies as with write_atomic().
-+	 *
-+	 * The console ownership handling is necessary for synchronization
-+	 * against write_atomic() which is synchronized only via the context.
-+	 *
-+	 * The device_lock() provides the primary serialization for operations
-+	 * on the device. It might be as relaxed (mutex)[*] or as tight
-+	 * (disabled preemption and interrupts) as needed. It allows
-+	 * the kthread to operate in the least restrictive mode[**].
-+	 *
-+	 * [*] Standalone nbcon_context_try_acquire() is not safe with
-+	 *     the preemption enabled, see nbcon_owner_matches(). But it
-+	 *     can be safe when always called in the preemptive context
-+	 *     under the device_lock().
-+	 *
-+	 * [**] The device_lock() makes sure that nbcon_context_try_acquire()
-+	 *      would never need to spin which is important especially with
-+	 *      PREEMPT_RT.
-+	 */
-+	void (*write_thread)(struct console *con, struct nbcon_write_context *wctxt);
-+
-+	/**
-+	 * @device_lock:
-+	 *
-+	 * NBCON callback to begin synchronization with driver code.
-+	 *
-+	 * Console drivers typically must deal with access to the hardware
-+	 * via user input/output (such as an interactive login shell) and
-+	 * output of kernel messages via printk() calls. This callback is
-+	 * called by the printk-subsystem whenever it needs to synchronize
-+	 * with hardware access by the driver. It should be implemented to
-+	 * use whatever synchronization mechanism the driver is using for
-+	 * itself (for example, the port lock for uart serial consoles).
-+	 *
-+	 * The callback is always called from task context. It may use any
-+	 * synchronization method required by the driver.
-+	 *
-+	 * IMPORTANT: The callback MUST disable migration. The console driver
-+	 *	may be using a synchronization mechanism that already takes
-+	 *	care of this (such as spinlocks). Otherwise this function must
-+	 *	explicitly call migrate_disable().
-+	 *
-+	 * The flags argument is provided as a convenience to the driver. It
-+	 * will be passed again to device_unlock(). It can be ignored if the
-+	 * driver does not need it.
-+	 */
-+	void (*device_lock)(struct console *con, unsigned long *flags);
-+
-+	/**
-+	 * @device_unlock:
-+	 *
-+	 * NBCON callback to finish synchronization with driver code.
-+	 *
-+	 * It is the counterpart to device_lock().
-+	 *
-+	 * This callback is always called from task context. It must
-+	 * appropriately re-enable migration (depending on how device_lock()
-+	 * disabled migration).
-+	 *
-+	 * The flags argument is the value of the same variable that was
-+	 * passed to device_lock().
-+	 */
-+	void (*device_unlock)(struct console *con, unsigned long flags);
-+
-+	atomic_t		__private nbcon_state;
-+	atomic_long_t		__private nbcon_seq;
-+	struct nbcon_context	__private nbcon_device_ctxt;
-+	atomic_long_t           __private nbcon_prev_seq;
-+
-+	struct printk_buffers	*pbufs;
-+	struct task_struct	*kthread;
-+	struct rcuwait		rcuwait;
-+	struct irq_work		irq_work;
-+};
-+
-+#ifdef CONFIG_LOCKDEP
-+extern void lockdep_assert_console_list_lock_held(void);
- #else
-static inline int con_debug_enter(struct vc_data *vc)
-+static inline void lockdep_assert_console_list_lock_held(void)
- {
-	return 0;
- }
-static inline int con_debug_leave(void)
-+#endif
-+
-+#ifdef CONFIG_DEBUG_LOCK_ALLOC
-+extern bool console_srcu_read_lock_is_held(void);
-+#else
-+static inline bool console_srcu_read_lock_is_held(void)
- {
-	return 0;
-+	return 1;
- }
- #endif
- 
-/* cursor */
-#define CM_DRAW     (1)
-#define CM_ERASE    (2)
-#define CM_MOVE     (3)
-+extern int console_srcu_read_lock(void);
-+extern void console_srcu_read_unlock(int cookie);
-+
-+extern void console_list_lock(void) __acquires(console_mutex);
-+extern void console_list_unlock(void) __releases(console_mutex);
-+
-+extern struct hlist_head console_list;
-+
-+/**
-+ * console_srcu_read_flags - Locklessly read flags of a possibly registered
-+ *				console
-+ * @con:	struct console pointer of console to read flags from
-+ *
-+ * Locklessly reading @con->flags provides a consistent read value because
-+ * there is at most one CPU modifying @con->flags and that CPU is using only
-+ * read-modify-write operations to do so.
-+ *
-+ * Requires console_srcu_read_lock to be held, which implies that @con might
-+ * be a registered console. The purpose of holding console_srcu_read_lock is
-+ * to guarantee that the console state is valid (CON_SUSPENDED/CON_ENABLED)
-+ * and that no exit/cleanup routines will run if the console is currently
-+ * undergoing unregistration.
-+ *
-+ * If the caller is holding the console_list_lock or it is _certain_ that
-+ * @con is not and will not become registered, the caller may read
-+ * @con->flags directly instead.
-+ *
-+ * Context: Any context.
-+ * Return: The current value of the @con->flags field.
-+ */
-+static inline short console_srcu_read_flags(const struct console *con)
-+{
-+	WARN_ON_ONCE(!console_srcu_read_lock_is_held());
-+
-+	/*
-+	 * The READ_ONCE() matches the WRITE_ONCE() when @flags are modified
-+	 * for registered consoles with console_srcu_write_flags().
-+	 */
-+	return data_race(READ_ONCE(con->flags));
-+}
-+
-+/**
-+ * console_srcu_write_flags - Write flags for a registered console
-+ * @con:	struct console pointer of console to write flags to
-+ * @flags:	new flags value to write
-+ *
-+ * Only use this function to write flags for registered consoles. It
-+ * requires holding the console_list_lock.
-+ *
-+ * Context: Any context.
-+ */
-+static inline void console_srcu_write_flags(struct console *con, short flags)
-+{
-+	lockdep_assert_console_list_lock_held();
-+
-+	/* This matches the READ_ONCE() in console_srcu_read_flags(). */
-+	WRITE_ONCE(con->flags, flags);
-+}
-+
-+/* Variant of console_is_registered() when the console_list_lock is held. */
-+static inline bool console_is_registered_locked(const struct console *con)
-+{
-+	lockdep_assert_console_list_lock_held();
-+	return !hlist_unhashed(&con->node);
-+}
- 
- /*
- * The interface for a console, or any other device that wants to capture
- * console messages (printer driver?)
-+ * console_is_registered - Check if the console is registered
-+ * @con:	struct console pointer of console to check
-  *
- * If a console driver is marked CON_BOOT then it will be auto-unregistered
- * when the first real console is registered.  This is for early-printk drivers.
-+ * Context: Process context. May sleep while acquiring console list lock.
-+ * Return: true if the console is in the console list, otherwise false.
-+ *
-+ * If false is returned for a console that was previously registered, it
-+ * can be assumed that the console's unregistration is fully completed,
-+ * including the exit() callback after console list removal.
-  */
-+static inline bool console_is_registered(const struct console *con)
-+{
-+	bool ret;
- 
-#define CON_PRINTBUFFER	(1)
-#define CON_CONSDEV	(2) /* Preferred console, /dev/console */
-#define CON_ENABLED	(4)
-#define CON_BOOT	(8)
-#define CON_ANYTIME	(16) /* Safe to call when cpu is offline */
-#define CON_BRL		(32) /* Used for a braille device */
-#define CON_EXTENDED	(64) /* Use the extended output format a la /dev/kmsg */
-+	console_list_lock();
-+	ret = console_is_registered_locked(con);
-+	console_list_unlock();
-+	return ret;
-+}
- 
-struct console {
-	char	name[16];
-	void	(*write)(struct console *, const char *, unsigned);
-	int	(*read)(struct console *, char *, unsigned);
-	struct tty_driver *(*device)(struct console *, int *);
-	void	(*unblank)(void);
-	int	(*setup)(struct console *, char *);
-	int	(*exit)(struct console *);
-	int	(*match)(struct console *, char *name, int idx, char *options);
-	short	flags;
-	short	index;
-	int	cflag;
-	uint	ispeed;
-	uint	ospeed;
-	u64	seq;
-	unsigned long dropped;
-	void	*data;
-	struct	 console *next;
-};
-+/**
-+ * for_each_console_srcu() - Iterator over registered consoles
-+ * @con:	struct console pointer used as loop cursor
-+ *
-+ * Although SRCU guarantees the console list will be consistent, the
-+ * struct console fields may be updated by other CPUs while iterating.
-+ *
-+ * Requires console_srcu_read_lock to be held. Can be invoked from
-+ * any context.
-+ */
-+#define for_each_console_srcu(con)					\
-+	hlist_for_each_entry_srcu(con, &console_list, node,		\
-+				  console_srcu_read_lock_is_held())
-+
-+/**
-+ * for_each_console() - Iterator over registered consoles
-+ * @con:	struct console pointer used as loop cursor
-+ *
-+ * The console list and the &console.flags are immutable while iterating.
-+ *
-+ * Requires console_list_lock to be held.
-+ */
-+#define for_each_console(con)						\
-+	lockdep_assert_console_list_lock_held();			\
-+	hlist_for_each_entry(con, &console_list, node)
-+
-+#ifdef CONFIG_PRINTK
-+extern void nbcon_cpu_emergency_enter(void);
-+extern void nbcon_cpu_emergency_exit(void);
-+extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt);
-+extern void nbcon_write_context_set_buf(struct nbcon_write_context *wctxt,
-+					char *buf, unsigned int len);
-+extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt);
-+extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt);
-+extern void nbcon_reacquire_nobuf(struct nbcon_write_context *wctxt);
-+extern bool nbcon_allow_unsafe_takeover(void);
-+extern bool nbcon_kdb_try_acquire(struct console *con,
-+				  struct nbcon_write_context *wctxt);
-+extern void nbcon_kdb_release(struct nbcon_write_context *wctxt);
- 
- /*
- * for_each_console() allows you to iterate on each console
-+ * Check if the given console is currently capable and allowed to print
-+ * records. Note that this function does not consider the current context,
-+ * which can also play a role in deciding if @con can be used to print
-+ * records.
-  */
-#define for_each_console(con) \
-	for (con = console_drivers; con != NULL; con = con->next)
-+static inline bool console_is_usable(struct console *con, short flags, bool use_atomic)
-+{
-+	if (!(flags & CON_ENABLED))
-+		return false;
-+
-+	if ((flags & CON_SUSPENDED))
-+		return false;
-+
-+	if (flags & CON_NBCON) {
-+		if (use_atomic) {
-+			/* The write_atomic() callback is optional. */
-+			if (!con->write_atomic)
-+				return false;
-+
-+			/*
-+			 * An unsafe write_atomic() callback is only usable
-+			 * when unsafe takeovers are allowed.
-+			 */
-+			if ((flags & CON_NBCON_ATOMIC_UNSAFE) && !nbcon_allow_unsafe_takeover())
-+				return false;
-+		}
-+
-+		/*
-+		 * For the !use_atomic case, @printk_kthreads_running is not
-+		 * checked because the write_thread() callback is also used
-+		 * via the legacy loop when the printer threads are not
-+		 * available.
-+		 */
-+	} else {
-+		if (!con->write)
-+			return false;
-+	}
-+
-+	/*
-+	 * Console drivers may assume that per-cpu resources have been
-+	 * allocated. So unless they're explicitly marked as being able to
-+	 * cope (CON_ANYTIME) don't call them until this CPU is officially up.
-+	 */
-+	if (!cpu_online(raw_smp_processor_id()) && !(flags & CON_ANYTIME))
-+		return false;
-+
-+	return true;
-+}
-+
-+#else
-+static inline void nbcon_cpu_emergency_enter(void) { }
-+static inline void nbcon_cpu_emergency_exit(void) { }
-+static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; }
-+static inline void nbcon_write_context_set_buf(struct nbcon_write_context *wctxt,
-+					       char *buf, unsigned int len) { }
-+static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; }
-+static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; }
-+static inline void nbcon_reacquire_nobuf(struct nbcon_write_context *wctxt) { }
-+static inline bool nbcon_kdb_try_acquire(struct console *con,
-+					 struct nbcon_write_context *wctxt) { return false; }
-+static inline void nbcon_kdb_release(struct nbcon_write_context *wctxt) { }
-+static inline bool console_is_usable(struct console *con, short flags,
-+				     bool use_atomic) { return false; }
-+#endif
- 
- extern int console_set_on_cmdline;
- extern struct console *early_console;
-@@ -171,10 +690,10 @@ enum con_flush_mode {
- 	CONSOLE_REPLAY_ALL,
- };
- 
-extern int add_preferred_console(char *name, int idx, char *options);
-+extern int add_preferred_console(const char *name, const short idx, char *options);
-+extern void console_force_preferred_locked(struct console *con);
- extern void register_console(struct console *);
- extern int unregister_console(struct console *);
-extern struct console *console_drivers;
- extern void console_lock(void);
- extern int console_trylock(void);
- extern void console_unlock(void);
-@@ -182,8 +701,8 @@ extern void console_conditional_schedule(void);
- extern void console_unblank(void);
- extern void console_flush_on_panic(enum con_flush_mode mode);
- extern struct tty_driver *console_device(int *);
-extern void console_stop(struct console *);
-extern void console_start(struct console *);
-+extern void console_suspend(struct console *);
-+extern void console_resume(struct console *);
- extern int is_console_locked(void);
- extern int braille_register_console(struct console *, int index,
- 		char *console_options, char *braille_options);
-@@ -197,8 +716,8 @@ static inline void console_sysfs_notify(void)
- extern bool console_suspend_enabled;
- 
- /* Suspend and resume console messages over PM events */
-extern void suspend_console(void);
-extern void resume_console(void);
-+extern void console_suspend_all(void);
-+extern void console_resume_all(void);
- 
- int mda_console_init(void);
- 
-@@ -215,11 +734,7 @@ void vcs_remove_sysfs(int index);
-  */
- extern atomic_t ignore_console_lock_warning;
- 
-/* VESA Blanking Levels */
-#define VESA_NO_BLANKING        0
-#define VESA_VSYNC_SUSPEND      1
-#define VESA_HSYNC_SUSPEND      2
-#define VESA_POWERDOWN          3
-+DEFINE_LOCK_GUARD_0(console_lock, console_lock(), console_unlock());
- 
- extern void console_init(void);
- 
-diff --git a/include/linux/console_struct.h b/include/linux/console_struct.h
-index 1518568aaf0f..13b35637bd5a 100644
--- a/include/linux/console_struct.h
-+++ b/include/linux/console_struct.h
-@@ -18,7 +18,6 @@
- #include <linux/workqueue.h>
- 
- struct uni_pagedict;
-struct uni_screen;
- 
- #define NPAR 16
- #define VC_TABSTOPS_COUNT	256U
-@@ -146,20 +145,23 @@ struct vc_data {
- 	unsigned int	vc_need_wrap	: 1;
- 	unsigned int	vc_can_do_color	: 1;
- 	unsigned int	vc_report_mouse : 2;
-+	unsigned int	vc_bracketed_paste : 1;
- 	unsigned char	vc_utf		: 1;	/* Unicode UTF-8 encoding */
- 	unsigned char	vc_utf_count;
- 		 int	vc_utf_char;
- 	DECLARE_BITMAP(vc_tab_stop, VC_TABSTOPS_COUNT);	/* Tab stops. 256 columns. */
- 	unsigned char   vc_palette[16*3];       /* Colour palette for VGA+ */
- 	unsigned short * vc_translate;
-	unsigned int    vc_resize_user;         /* resize request from user */
- 	unsigned int	vc_bell_pitch;		/* Console bell pitch */
- 	unsigned int	vc_bell_duration;	/* Console bell duration */
- 	unsigned short	vc_cur_blink_ms;	/* Cursor blink duration */
- 	struct vc_data **vc_display_fg;		/* [!] Ptr to var holding fg console for this display */
- 	struct uni_pagedict *uni_pagedict;
- 	struct uni_pagedict **uni_pagedict_loc; /* [!] Location of uni_pagedict variable for this console */
-	struct uni_screen *vc_uni_screen;	/* unicode screen content */
-+	u32 **vc_uni_lines;			/* unicode screen content */
-+	u16		*vc_saved_screen;
-+	unsigned int	vc_saved_cols;
-+	unsigned int	vc_saved_rows;
- 	/* additional information is in vt_kern.h */
- };
- 
-diff --git a/include/linux/consolemap.h b/include/linux/consolemap.h
-index c35db4896c37..6180b803795c 100644
--- a/include/linux/consolemap.h
-+++ b/include/linux/consolemap.h
-@@ -28,6 +28,10 @@ int conv_uni_to_pc(struct vc_data *conp, long ucs);
- u32 conv_8bit_to_uni(unsigned char c);
- int conv_uni_to_8bit(u32 uni);
- void console_map_init(void);
-+bool ucs_is_double_width(uint32_t cp);
-+bool ucs_is_zero_width(uint32_t cp);
-+u32 ucs_recompose(u32 base, u32 mark);
-+u32 ucs_get_fallback(u32 cp);
- #else
- static inline u16 inverse_translate(const struct vc_data *conp, u16 glyph,
- 		bool use_unicode)
-@@ -57,6 +61,26 @@ static inline int conv_uni_to_8bit(u32 uni)
- }
- 
- static inline void console_map_init(void) { }
-+
-+static inline bool ucs_is_double_width(uint32_t cp)
-+{
-+	return false;
-+}
-+
-+static inline bool ucs_is_zero_width(uint32_t cp)
-+{
-+	return false;
-+}
-+
-+static inline u32 ucs_recompose(u32 base, u32 mark)
-+{
-+	return 0;
-+}
-+
-+static inline u32 ucs_get_fallback(u32 cp)
-+{
-+	return 0;
-+}
- #endif /* CONFIG_CONSOLE_TRANSLATIONS */
- 
- #endif /* __LINUX_CONSOLEMAP_H__ */
-diff --git a/include/linux/const.h b/include/linux/const.h
-index 435ddd72d2c4..81b8aae5a855 100644
--- a/include/linux/const.h
-+++ b/include/linux/const.h
-@@ -3,12 +3,4 @@
- 
- #include <vdso/const.h>
- 
-/*
- * This returns a constant expression while determining if an argument is
- * a constant expression, most importantly without evaluating the argument.
- * Glory to Martin Uecker <Martin.Uecker@med.uni-goettingen.de>
- */
-#define __is_constexpr(x) \
-	(sizeof(int) == sizeof(*(8 ? ((void *)((long)(x) * 0l)) : (int *)8)))
-
- #endif /* _LINUX_CONST_H */
-diff --git a/include/linux/container.h b/include/linux/container.h
-index 2566a1baa736..dd00cc918a92 100644
--- a/include/linux/container.h
-+++ b/include/linux/container.h
-@@ -12,7 +12,7 @@
- #include <linux/device.h>
- 
- /* drivers/base/power/container.c */
-extern struct bus_type container_subsys;
-+extern const struct bus_type container_subsys;
- 
- struct container_dev {
- 	struct device dev;
-diff --git a/include/linux/container_of.h b/include/linux/container_of.h
-index 2f4944b791b8..1f6ebf27d962 100644
--- a/include/linux/container_of.h
-+++ b/include/linux/container_of.h
-@@ -3,7 +3,7 @@
- #define _LINUX_CONTAINER_OF_H
- 
- #include <linux/build_bug.h>
-#include <linux/err.h>
-+#include <linux/stddef.h>
- 
- #define typeof_member(T, m)	typeof(((T*)0)->m)
- 
-@@ -13,6 +13,8 @@
-  * @type:	the type of the container struct this is embedded in.
-  * @member:	the name of the member within the struct.
-  *
-+ * WARNING: any const qualifier of @ptr is lost.
-+ * Do not use container_of() in new code.
-  */
- #define container_of(ptr, type, member) ({				\
- 	void *__mptr = (void *)(ptr);					\
-@@ -22,19 +24,18 @@
- 	((type *)(__mptr - offsetof(type, member))); })
- 
- /**
- * container_of_safe - cast a member of a structure out to the containing structure
- * @ptr:	the pointer to the member.
- * @type:	the type of the container struct this is embedded in.
- * @member:	the name of the member within the struct.
-+ * container_of_const - cast a member of a structure out to the containing
-+ *			structure and preserve the const-ness of the pointer
-+ * @ptr:		the pointer to the member
-+ * @type:		the type of the container struct this is embedded in.
-+ * @member:		the name of the member within the struct.
-  *
- * If IS_ERR_OR_NULL(ptr), ptr is returned unchanged.
-+ * Always prefer container_of_const() instead of container_of() in new code.
-  */
-#define container_of_safe(ptr, type, member) ({				\
-	void *__mptr = (void *)(ptr);					\
-	static_assert(__same_type(*(ptr), ((type *)0)->member) ||	\
-		      __same_type(*(ptr), void),			\
-		      "pointer type mismatch in container_of_safe()");	\
-	IS_ERR_OR_NULL(__mptr) ? ERR_CAST(__mptr) :			\
-		((type *)(__mptr - offsetof(type, member))); })
-+#define container_of_const(ptr, type, member)				\
-+	_Generic(ptr,							\
-+		const typeof(*(ptr)) *: ((const type *)container_of(ptr, type, member)),\
-+		default: ((type *)container_of(ptr, type, member))	\
-+	)
- 
- #endif	/* _LINUX_CONTAINER_OF_H */
-diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
-index dcef4a9e4d63..af9fe87a0922 100644
--- a/include/linux/context_tracking.h
-+++ b/include/linux/context_tracking.h
-@@ -26,26 +26,26 @@ extern void user_exit_callable(void);
- static inline void user_enter(void)
- {
- 	if (context_tracking_enabled())
-		ct_user_enter(CONTEXT_USER);
-+		ct_user_enter(CT_STATE_USER);
- 
- }
- static inline void user_exit(void)
- {
- 	if (context_tracking_enabled())
-		ct_user_exit(CONTEXT_USER);
-+		ct_user_exit(CT_STATE_USER);
- }
- 
- /* Called with interrupts disabled.  */
- static __always_inline void user_enter_irqoff(void)
- {
- 	if (context_tracking_enabled())
-		__ct_user_enter(CONTEXT_USER);
-+		__ct_user_enter(CT_STATE_USER);
- 
- }
- static __always_inline void user_exit_irqoff(void)
- {
- 	if (context_tracking_enabled())
-		__ct_user_exit(CONTEXT_USER);
-+		__ct_user_exit(CT_STATE_USER);
- }
- 
- static inline enum ctx_state exception_enter(void)
-@@ -57,7 +57,7 @@ static inline enum ctx_state exception_enter(void)
- 		return 0;
- 
- 	prev_ctx = __ct_state();
-	if (prev_ctx != CONTEXT_KERNEL)
-+	if (prev_ctx != CT_STATE_KERNEL)
- 		ct_user_exit(prev_ctx);
- 
- 	return prev_ctx;
-@@ -67,7 +67,7 @@ static inline void exception_exit(enum ctx_state prev_ctx)
- {
- 	if (!IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) &&
- 	    context_tracking_enabled()) {
-		if (prev_ctx != CONTEXT_KERNEL)
-+		if (prev_ctx != CT_STATE_KERNEL)
- 			ct_user_enter(prev_ctx);
- 	}
- }
-@@ -75,15 +75,17 @@ static inline void exception_exit(enum ctx_state prev_ctx)
- static __always_inline bool context_tracking_guest_enter(void)
- {
- 	if (context_tracking_enabled())
-		__ct_user_enter(CONTEXT_GUEST);
-+		__ct_user_enter(CT_STATE_GUEST);
- 
- 	return context_tracking_enabled_this_cpu();
- }
- 
-static __always_inline void context_tracking_guest_exit(void)
-+static __always_inline bool context_tracking_guest_exit(void)
- {
- 	if (context_tracking_enabled())
-		__ct_user_exit(CONTEXT_GUEST);
-+		__ct_user_exit(CT_STATE_GUEST);
-+
-+	return context_tracking_enabled_this_cpu();
- }
- 
- #define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond))
-@@ -96,8 +98,9 @@ static inline void user_exit_irqoff(void) { }
- static inline int exception_enter(void) { return 0; }
- static inline void exception_exit(enum ctx_state prev_ctx) { }
- static inline int ct_state(void) { return -1; }
-+static inline int __ct_state(void) { return -1; }
- static __always_inline bool context_tracking_guest_enter(void) { return false; }
-static inline void context_tracking_guest_exit(void) { }
-+static __always_inline bool context_tracking_guest_exit(void) { return false; }
- #define CT_WARN_ON(cond) do { } while (0)
- #endif /* !CONFIG_CONTEXT_TRACKING_USER */
- 
-@@ -112,13 +115,17 @@ extern void ct_idle_enter(void);
- extern void ct_idle_exit(void);
- 
- /*
- * Is the current CPU in an extended quiescent state?
-+ * Is RCU watching the current CPU (IOW, it is not in an extended quiescent state)?
-+ *
-+ * Note that this returns the actual boolean data (watching / not watching),
-+ * whereas ct_rcu_watching() returns the RCU_WATCHING subvariable of
-+ * context_tracking.state.
-  *
-  * No ordering, as we are sampling CPU-local information.
-  */
-static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
-+static __always_inline bool rcu_is_watching_curr_cpu(void)
- {
-	return !(arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & RCU_DYNTICKS_IDX);
-+	return raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_RCU_WATCHING;
- }
- 
- /*
-@@ -127,12 +134,39 @@ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
-  */
- static __always_inline unsigned long ct_state_inc(int incby)
- {
-	return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state));
-+	return raw_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state));
-+}
-+
-+static __always_inline bool warn_rcu_enter(void)
-+{
-+	bool ret = false;
-+
-+	/*
-+	 * Horrible hack to shut up recursive RCU isn't watching fail since
-+	 * lots of the actual reporting also relies on RCU.
-+	 */
-+	preempt_disable_notrace();
-+	if (!rcu_is_watching_curr_cpu()) {
-+		ret = true;
-+		ct_state_inc(CT_RCU_WATCHING);
-+	}
-+
-+	return ret;
-+}
-+
-+static __always_inline void warn_rcu_exit(bool rcu)
-+{
-+	if (rcu)
-+		ct_state_inc(CT_RCU_WATCHING);
-+	preempt_enable_notrace();
- }
- 
- #else
- static inline void ct_idle_enter(void) { }
- static inline void ct_idle_exit(void) { }
-+
-+static __always_inline bool warn_rcu_enter(void) { return false; }
-+static __always_inline void warn_rcu_exit(bool rcu) { }
- #endif /* !CONFIG_CONTEXT_TRACKING_IDLE */
- 
- #endif
-diff --git a/include/linux/context_tracking_irq.h b/include/linux/context_tracking_irq.h
-index c50b5670c4a5..197916ee91a4 100644
--- a/include/linux/context_tracking_irq.h
-+++ b/include/linux/context_tracking_irq.h
-@@ -10,12 +10,12 @@ void ct_irq_exit_irqson(void);
- void ct_nmi_enter(void);
- void ct_nmi_exit(void);
- #else
-static inline void ct_irq_enter(void) { }
-static inline void ct_irq_exit(void) { }
-+static __always_inline void ct_irq_enter(void) { }
-+static __always_inline void ct_irq_exit(void) { }
- static inline void ct_irq_enter_irqson(void) { }
- static inline void ct_irq_exit_irqson(void) { }
-static inline void ct_nmi_enter(void) { }
-static inline void ct_nmi_exit(void) { }
-+static __always_inline void ct_nmi_enter(void) { }
-+static __always_inline void ct_nmi_exit(void) { }
- #endif
- 
- #endif
-diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h
-index 4a4d56f77180..0b81248aa03e 100644
--- a/include/linux/context_tracking_state.h
-+++ b/include/linux/context_tracking_state.h
-@@ -7,23 +7,17 @@
- #include <linux/context_tracking_irq.h>
- 
- /* Offset to allow distinguishing irq vs. task-based idle entry/exit. */
-#define DYNTICK_IRQ_NONIDLE	((LONG_MAX / 2) + 1)
-+#define CT_NESTING_IRQ_NONIDLE	((LONG_MAX / 2) + 1)
- 
- enum ctx_state {
-	CONTEXT_DISABLED	= -1,	/* returned by ct_state() if unknown */
-	CONTEXT_KERNEL		= 0,
-	CONTEXT_IDLE		= 1,
-	CONTEXT_USER		= 2,
-	CONTEXT_GUEST		= 3,
-	CONTEXT_MAX		= 4,
-+	CT_STATE_DISABLED	= -1,	/* returned by ct_state() if unknown */
-+	CT_STATE_KERNEL		= 0,
-+	CT_STATE_IDLE		= 1,
-+	CT_STATE_USER		= 2,
-+	CT_STATE_GUEST		= 3,
-+	CT_STATE_MAX		= 4,
- };
- 
-/* Even value for idle, else odd. */
-#define RCU_DYNTICKS_IDX CONTEXT_MAX
-
-#define CT_STATE_MASK (CONTEXT_MAX - 1)
-#define CT_DYNTICKS_MASK (~CT_STATE_MASK)
-
- struct context_tracking {
- #ifdef CONFIG_CONTEXT_TRACKING_USER
- 	/*
-@@ -39,62 +33,100 @@ struct context_tracking {
- 	atomic_t state;
- #endif
- #ifdef CONFIG_CONTEXT_TRACKING_IDLE
-	long dynticks_nesting;		/* Track process nesting level. */
-	long dynticks_nmi_nesting;	/* Track irq/NMI nesting level. */
-+	long nesting;		/* Track process nesting level. */
-+	long nmi_nesting;	/* Track irq/NMI nesting level. */
- #endif
- };
- 
-+/*
-+ * We cram two different things within the same atomic variable:
-+ *
-+ *                     CT_RCU_WATCHING_START  CT_STATE_START
-+ *                                |                |
-+ *                                v                v
-+ *     MSB [ RCU watching counter ][ context_state ] LSB
-+ *         ^                       ^
-+ *         |                       |
-+ * CT_RCU_WATCHING_END        CT_STATE_END
-+ *
-+ * Bits are used from the LSB upwards, so unused bits (if any) will always be in
-+ * upper bits of the variable.
-+ */
- #ifdef CONFIG_CONTEXT_TRACKING
-+#define CT_SIZE (sizeof(((struct context_tracking *)0)->state) * BITS_PER_BYTE)
-+
-+#define CT_STATE_WIDTH bits_per(CT_STATE_MAX - 1)
-+#define CT_STATE_START 0
-+#define CT_STATE_END   (CT_STATE_START + CT_STATE_WIDTH - 1)
-+
-+#define CT_RCU_WATCHING_MAX_WIDTH (CT_SIZE - CT_STATE_WIDTH)
-+#define CT_RCU_WATCHING_WIDTH     (IS_ENABLED(CONFIG_RCU_DYNTICKS_TORTURE) ? 2 : CT_RCU_WATCHING_MAX_WIDTH)
-+#define CT_RCU_WATCHING_START     (CT_STATE_END + 1)
-+#define CT_RCU_WATCHING_END       (CT_RCU_WATCHING_START + CT_RCU_WATCHING_WIDTH - 1)
-+#define CT_RCU_WATCHING           BIT(CT_RCU_WATCHING_START)
-+
-+#define CT_STATE_MASK        GENMASK(CT_STATE_END,        CT_STATE_START)
-+#define CT_RCU_WATCHING_MASK GENMASK(CT_RCU_WATCHING_END, CT_RCU_WATCHING_START)
-+
-+#define CT_UNUSED_WIDTH (CT_RCU_WATCHING_MAX_WIDTH - CT_RCU_WATCHING_WIDTH)
-+
-+static_assert(CT_STATE_WIDTH        +
-+	      CT_RCU_WATCHING_WIDTH +
-+	      CT_UNUSED_WIDTH       ==
-+	      CT_SIZE);
-+
- DECLARE_PER_CPU(struct context_tracking, context_tracking);
-+#endif	/* CONFIG_CONTEXT_TRACKING */
- 
-+#ifdef CONFIG_CONTEXT_TRACKING_USER
- static __always_inline int __ct_state(void)
- {
-	return arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK;
-+	return raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK;
- }
- #endif
- 
- #ifdef CONFIG_CONTEXT_TRACKING_IDLE
-static __always_inline int ct_dynticks(void)
-+static __always_inline int ct_rcu_watching(void)
- {
-	return atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_DYNTICKS_MASK;
-+	return atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_RCU_WATCHING_MASK;
- }
- 
-static __always_inline int ct_dynticks_cpu(int cpu)
-+static __always_inline int ct_rcu_watching_cpu(int cpu)
- {
- 	struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
- 
-	return atomic_read(&ct->state) & CT_DYNTICKS_MASK;
-+	return atomic_read(&ct->state) & CT_RCU_WATCHING_MASK;
- }
- 
-static __always_inline int ct_dynticks_cpu_acquire(int cpu)
-+static __always_inline int ct_rcu_watching_cpu_acquire(int cpu)
- {
- 	struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
- 
-	return atomic_read_acquire(&ct->state) & CT_DYNTICKS_MASK;
-+	return atomic_read_acquire(&ct->state) & CT_RCU_WATCHING_MASK;
- }
- 
-static __always_inline long ct_dynticks_nesting(void)
-+static __always_inline long ct_nesting(void)
- {
-	return __this_cpu_read(context_tracking.dynticks_nesting);
-+	return __this_cpu_read(context_tracking.nesting);
- }
- 
-static __always_inline long ct_dynticks_nesting_cpu(int cpu)
-+static __always_inline long ct_nesting_cpu(int cpu)
- {
- 	struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
- 
-	return ct->dynticks_nesting;
-+	return ct->nesting;
- }
- 
-static __always_inline long ct_dynticks_nmi_nesting(void)
-+static __always_inline long ct_nmi_nesting(void)
- {
-	return __this_cpu_read(context_tracking.dynticks_nmi_nesting);
-+	return __this_cpu_read(context_tracking.nmi_nesting);
- }
- 
-static __always_inline long ct_dynticks_nmi_nesting_cpu(int cpu)
-+static __always_inline long ct_nmi_nesting_cpu(int cpu)
- {
- 	struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
- 
-	return ct->dynticks_nmi_nesting;
-+	return ct->nmi_nesting;
- }
- #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */
- 
-@@ -111,7 +143,7 @@ static __always_inline bool context_tracking_enabled_cpu(int cpu)
- 	return context_tracking_enabled() && per_cpu(context_tracking.active, cpu);
- }
- 
-static inline bool context_tracking_enabled_this_cpu(void)
-+static __always_inline bool context_tracking_enabled_this_cpu(void)
- {
- 	return context_tracking_enabled() && __this_cpu_read(context_tracking.active);
- }
-@@ -121,14 +153,14 @@ static inline bool context_tracking_enabled_this_cpu(void)
-  *
-  * Returns the current cpu's context tracking state if context tracking
-  * is enabled.  If context tracking is disabled, returns
- * CONTEXT_DISABLED.  This should be used primarily for debugging.
-+ * CT_STATE_DISABLED.  This should be used primarily for debugging.
-  */
- static __always_inline int ct_state(void)
- {
- 	int ret;
- 
- 	if (!context_tracking_enabled())
-		return CONTEXT_DISABLED;
-+		return CT_STATE_DISABLED;
- 
- 	preempt_disable();
- 	ret = __ct_state();
-diff --git a/include/linux/coredump.h b/include/linux/coredump.h
-index 08a1d3e7e46d..68861da4cf7c 100644
--- a/include/linux/coredump.h
-+++ b/include/linux/coredump.h
-@@ -10,7 +10,7 @@
- #ifdef CONFIG_COREDUMP
- struct core_vma_metadata {
- 	unsigned long start, end;
-	unsigned long flags;
-+	vm_flags_t flags;
- 	unsigned long dump_size;
- 	unsigned long pgoff;
- 	struct file   *file;
-@@ -18,18 +18,21 @@ struct core_vma_metadata {
- 
- struct coredump_params {
- 	const kernel_siginfo_t *siginfo;
-	struct pt_regs *regs;
- 	struct file *file;
- 	unsigned long limit;
- 	unsigned long mm_flags;
-+	int cpu;
- 	loff_t written;
- 	loff_t pos;
- 	loff_t to_skip;
- 	int vma_count;
- 	size_t vma_data_size;
- 	struct core_vma_metadata *vma_meta;
-+	struct pid *pid;
- };
- 
-+extern unsigned int core_file_note_size_limit;
-+
- /*
-  * These are the only things you should do on a core-file: use only these
-  * functions to write out all the necessary info.
-@@ -40,9 +43,31 @@ extern int dump_emit(struct coredump_params *cprm, const void *addr, int nr);
- extern int dump_align(struct coredump_params *cprm, int align);
- int dump_user_range(struct coredump_params *cprm, unsigned long start,
- 		    unsigned long len);
-extern void do_coredump(const kernel_siginfo_t *siginfo);
-+extern void vfs_coredump(const kernel_siginfo_t *siginfo);
-+
-+/*
-+ * Logging for the coredump code, ratelimited.
-+ * The TGID and comm fields are added to the message.
-+ */
-+
-+#define __COREDUMP_PRINTK(Level, Format, ...) \
-+	do {	\
-+		char comm[TASK_COMM_LEN];	\
-+		/* This will always be NUL terminated. */ \
-+		memcpy(comm, current->comm, sizeof(comm)); \
-+		printk_ratelimited(Level "coredump: %d(%*pE): " Format "\n",	\
-+			task_tgid_vnr(current), (int)strlen(comm), comm, ##__VA_ARGS__);	\
-+	} while (0)	\
-+
-+#define coredump_report(fmt, ...) __COREDUMP_PRINTK(KERN_INFO, fmt, ##__VA_ARGS__)
-+#define coredump_report_failure(fmt, ...) __COREDUMP_PRINTK(KERN_WARNING, fmt, ##__VA_ARGS__)
-+
- #else
-static inline void do_coredump(const kernel_siginfo_t *siginfo) {}
-+static inline void vfs_coredump(const kernel_siginfo_t *siginfo) {}
-+
-+#define coredump_report(...)
-+#define coredump_report_failure(...)
-+
- #endif
- 
- #if defined(CONFIG_COREDUMP) && defined(CONFIG_SYSCTL)
-diff --git a/include/linux/coresight-pmu.h b/include/linux/coresight-pmu.h
-index 6c2fd6cc5a98..89b0ac0014b0 100644
--- a/include/linux/coresight-pmu.h
-+++ b/include/linux/coresight-pmu.h
-@@ -7,8 +7,19 @@
- #ifndef _LINUX_CORESIGHT_PMU_H
- #define _LINUX_CORESIGHT_PMU_H
- 
-+#include <linux/bits.h>
-+
- #define CORESIGHT_ETM_PMU_NAME "cs_etm"
-#define CORESIGHT_ETM_PMU_SEED  0x10
-+
-+/*
-+ * The legacy Trace ID system based on fixed calculation from the cpu
-+ * number. This has been replaced by drivers using a dynamic allocation
-+ * system - but need to retain the legacy algorithm for backward comparibility
-+ * in certain situations:-
-+ * a) new perf running on older systems that generate the legacy mapping
-+ * b) older tools that may not update at the same time as the kernel.
-+ */
-+#define CORESIGHT_LEGACY_CPU_TRACE_ID(cpu)  (0x10 + (cpu * 2))
- 
- /*
-  * Below are the definition of bit offsets for perf option, and works as
-@@ -34,15 +45,25 @@
- #define ETM4_CFG_BIT_RETSTK	12
- #define ETM4_CFG_BIT_VMID_OPT	15
- 
-static inline int coresight_get_trace_id(int cpu)
-{
-	/*
-	 * A trace ID of value 0 is invalid, so let's start at some
-	 * random value that fits in 7 bits and go from there.  Since
-	 * the common convention is to have data trace IDs be I(N) + 1,
-	 * set instruction trace IDs as a function of the CPU number.
-	 */
-	return (CORESIGHT_ETM_PMU_SEED + (cpu * 2));
-}
-+/*
-+ * Interpretation of the PERF_RECORD_AUX_OUTPUT_HW_ID payload.
-+ * Used to associate a CPU with the CoreSight Trace ID.
-+ * [07:00] - Trace ID - uses 8 bits to make value easy to read in file.
-+ * [39:08] - Sink ID - as reported in /sys/bus/event_source/devices/cs_etm/sinks/
-+ *	      Added in minor version 1.
-+ * [55:40] - Unused (SBZ)
-+ * [59:56] - Minor Version - previously existing fields are compatible with
-+ *	      all minor versions.
-+ * [63:60] - Major Version - previously existing fields mean different things
-+ *	      in new major versions.
-+ */
-+#define CS_AUX_HW_ID_TRACE_ID_MASK	GENMASK_ULL(7, 0)
-+#define CS_AUX_HW_ID_SINK_ID_MASK	GENMASK_ULL(39, 8)
-+
-+#define CS_AUX_HW_ID_MINOR_VERSION_MASK	GENMASK_ULL(59, 56)
-+#define CS_AUX_HW_ID_MAJOR_VERSION_MASK	GENMASK_ULL(63, 60)
-+
-+#define CS_AUX_HW_ID_MAJOR_VERSION 0
-+#define CS_AUX_HW_ID_MINOR_VERSION 1
- 
- #endif
-diff --git a/include/linux/coresight.h b/include/linux/coresight.h
-index 9f445f09fcfe..2b48be97fcd0 100644
--- a/include/linux/coresight.h
-+++ b/include/linux/coresight.h
-@@ -6,10 +6,13 @@
- #ifndef _LINUX_CORESIGHT_H
- #define _LINUX_CORESIGHT_H
- 
-+#include <linux/amba/bus.h>
-+#include <linux/clk.h>
- #include <linux/device.h>
- #include <linux/io.h>
- #include <linux/perf_event.h>
- #include <linux/sched.h>
-+#include <linux/platform_device.h>
- 
- /* Peripheral id registers (0xFD0-0xFEC) */
- #define CORESIGHT_PERIPHIDR4	0xfd0
-@@ -33,7 +36,7 @@
- 
- #define CORESIGHT_UNLOCK	0xc5acce55
- 
-extern struct bus_type coresight_bustype;
-+extern const struct bus_type coresight_bustype;
- 
- enum coresight_dev_type {
- 	CORESIGHT_DEV_TYPE_SINK,
-@@ -41,10 +44,11 @@ enum coresight_dev_type {
- 	CORESIGHT_DEV_TYPE_LINKSINK,
- 	CORESIGHT_DEV_TYPE_SOURCE,
- 	CORESIGHT_DEV_TYPE_HELPER,
-	CORESIGHT_DEV_TYPE_ECT,
-+	CORESIGHT_DEV_TYPE_MAX
- };
- 
- enum coresight_dev_subtype_sink {
-+	CORESIGHT_DEV_SUBTYPE_SINK_DUMMY,
- 	CORESIGHT_DEV_SUBTYPE_SINK_PORT,
- 	CORESIGHT_DEV_SUBTYPE_SINK_BUFFER,
- 	CORESIGHT_DEV_SUBTYPE_SINK_SYSMEM,
-@@ -61,16 +65,14 @@ enum coresight_dev_subtype_source {
- 	CORESIGHT_DEV_SUBTYPE_SOURCE_PROC,
- 	CORESIGHT_DEV_SUBTYPE_SOURCE_BUS,
- 	CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE,
-+	CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM,
-+	CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS,
- };
- 
- enum coresight_dev_subtype_helper {
- 	CORESIGHT_DEV_SUBTYPE_HELPER_CATU,
-};
-
-/* Embedded Cross Trigger (ECT) sub-types */
-enum coresight_dev_subtype_ect {
-	CORESIGHT_DEV_SUBTYPE_ECT_NONE,
-	CORESIGHT_DEV_SUBTYPE_ECT_CTI,
-+	CORESIGHT_DEV_SUBTYPE_HELPER_ECT_CTI,
-+	CORESIGHT_DEV_SUBTYPE_HELPER_CTCU,
- };
- 
- /**
-@@ -83,8 +85,6 @@ enum coresight_dev_subtype_ect {
-  *			by @coresight_dev_subtype_source.
-  * @helper_subtype:	type of helper this component is, as defined
-  *			by @coresight_dev_subtype_helper.
- * @ect_subtype:        type of cross trigger this component is, as
- *			defined by @coresight_dev_subtype_ect
-  */
- union coresight_dev_subtype {
- 	/* We have some devices which acts as LINK and SINK */
-@@ -94,21 +94,25 @@ union coresight_dev_subtype {
- 	};
- 	enum coresight_dev_subtype_source source_subtype;
- 	enum coresight_dev_subtype_helper helper_subtype;
-	enum coresight_dev_subtype_ect ect_subtype;
- };
- 
- /**
-  * struct coresight_platform_data - data harvested from the firmware
-  * specification.
-  *
- * @nr_inport:	Number of elements for the input connections.
- * @nr_outport:	Number of elements for the output connections.
- * @conns:	Sparse array of nr_outport connections from this component.
-+ * @nr_inconns: Number of elements for the input connections.
-+ * @nr_outconns: Number of elements for the output connections.
-+ * @out_conns: Array of nr_outconns pointers to connections from this
-+ *	       component.
-+ * @in_conns: Sparse array of pointers to input connections. Sparse
-+ *            because the source device owns the connection so when it's
-+ *            unloaded the connection leaves an empty slot.
-  */
- struct coresight_platform_data {
-	int nr_inport;
-	int nr_outport;
-	struct coresight_connection *conns;
-+	int nr_inconns;
-+	int nr_outconns;
-+	struct coresight_connection **out_conns;
-+	struct coresight_connection **in_conns;
- };
- 
- /**
-@@ -163,19 +167,47 @@ struct coresight_desc {
- 
- /**
-  * struct coresight_connection - representation of a single connection
- * @outport:	a connection's output port number.
- * @child_port:	remote component's port number @output is connected to.
- * @chid_fwnode: remote component's fwnode handle.
- * @child_dev:	a @coresight_device representation of the component
-		connected to @outport.
-+ * @src_port:	a connection's output port number.
-+ * @dest_port:	destination's input port number @src_port is connected to.
-+ * @dest_fwnode: destination component's fwnode handle.
-+ * @dest_dev:	a @coresight_device representation of the component
-+		connected to @src_port. NULL until the device is created
-  * @link: Representation of the connection as a sysfs link.
-+ * @filter_src_fwnode: filter source component's fwnode handle.
-+ * @filter_src_dev: a @coresight_device representation of the component that
-+		needs to be filtered.
-+ *
-+ * The full connection structure looks like this, where in_conns store
-+ * references to same connection as the source device's out_conns.
-+ *
-+ * +-----------------------------+   +-----------------------------+
-+ * |coresight_device             |   |coresight_connection         |
-+ * |-----------------------------|   |-----------------------------|
-+ * |                             |   |                             |
-+ * |                             |   |                    dest_dev*|<--
-+ * |pdata->out_conns[nr_outconns]|<->|src_dev*                     |   |
-+ * |                             |   |                             |   |
-+ * +-----------------------------+   +-----------------------------+   |
-+ *                                                                     |
-+ *                                   +-----------------------------+   |
-+ *                                   |coresight_device             |   |
-+ *                                   |------------------------------   |
-+ *                                   |                             |   |
-+ *                                   |  pdata->in_conns[nr_inconns]|<--
-+ *                                   |                             |
-+ *                                   +-----------------------------+
-  */
- struct coresight_connection {
-	int outport;
-	int child_port;
-	struct fwnode_handle *child_fwnode;
-	struct coresight_device *child_dev;
-+	int src_port;
-+	int dest_port;
-+	struct fwnode_handle *dest_fwnode;
-+	struct coresight_device *dest_dev;
- 	struct coresight_sysfs_link *link;
-+	struct coresight_device *src_dev;
-+	struct fwnode_handle *filter_src_fwnode;
-+	struct coresight_device *filter_src_dev;
-+	int src_refcnt;
-+	int dest_refcnt;
- };
- 
- /**
-@@ -192,6 +224,24 @@ struct coresight_sysfs_link {
- 	const char *target_name;
- };
- 
-+/* architecturally we have 128 IDs some of which are reserved */
-+#define CORESIGHT_TRACE_IDS_MAX 128
-+
-+/**
-+ * Trace ID map.
-+ *
-+ * @used_ids:	Bitmap to register available (bit = 0) and in use (bit = 1) IDs.
-+ *		Initialised so that the reserved IDs are permanently marked as
-+ *		in use.
-+ * @perf_cs_etm_session_active: Number of Perf sessions using this ID map.
-+ */
-+struct coresight_trace_id_map {
-+	DECLARE_BITMAP(used_ids, CORESIGHT_TRACE_IDS_MAX);
-+	atomic_t __percpu *cpu_map;
-+	atomic_t perf_cs_etm_session_active;
-+	raw_spinlock_t lock;
-+};
-+
- /**
-  * struct coresight_device - representation of a device as used by the framework
-  * @pdata:	Platform data with device connections associated to this device.
-@@ -201,17 +251,24 @@ struct coresight_sysfs_link {
-  *		by @coresight_ops.
-  * @access:	Device i/o access abstraction for this device.
-  * @dev:	The device entity associated to this component.
- * @refcnt:	keep track of what is in use.
-+ * @mode:	The device mode, i.e sysFS, Perf or disabled. This is actually
-+ *		an 'enum cs_mode' but stored in an atomic type. Access is always
-+ *		through atomic APIs, ensuring SMP-safe synchronisation between
-+ *		racing from sysFS and Perf mode. A compare-and-exchange
-+ *		operation is done to atomically claim one mode or the other.
-+ * @refcnt:	keep track of what is in use. Only access this outside of the
-+ *		device's spinlock when the coresight_mutex held and mode ==
-+ *		CS_MODE_SYSFS. Otherwise it must be accessed from inside the
-+ *		spinlock.
-  * @orphan:	true if the component has connections that haven't been linked.
- * @enable:	'true' if component is currently part of an active path.
- * @activated:	'true' only if a _sink_ has been activated.  A sink can be
- *		activated but not yet enabled.  Enabling for a _sink_
- *		happens when a source has been selected and a path is enabled
- *		from source to that sink.
-+ * @sysfs_sink_activated: 'true' when a sink has been selected for use via sysfs
-+ *		by writing a 1 to the 'enable_sink' file.  A sink can be
-+ *		activated but not yet enabled.  Enabling for a _sink_ happens
-+ *		when a source has been selected and a path is enabled from
-+ *		source to that sink. A sink can also become enabled but not
-+ *		activated if it's used via Perf.
-  * @ea:		Device attribute for sink representation under PMU directory.
-  * @def_sink:	cached reference to default sink found for this device.
- * @ect_dev:	Associated cross trigger device. Not part of the trace data
- *		path or connections.
-  * @nr_links:   number of sysfs links created to other components from this
-  *		device. These will appear in the "connections" group.
-  * @has_conns_grp: Have added a "connections" group for sysfs links.
-@@ -227,23 +284,21 @@ struct coresight_device {
- 	const struct coresight_ops *ops;
- 	struct csdev_access access;
- 	struct device dev;
-	atomic_t *refcnt;
-+	atomic_t mode;
-+	int refcnt;
- 	bool orphan;
-	bool enable;	/* true only if configured as part of a path */
- 	/* sink specific fields */
-	bool activated;	/* true only if a sink is part of a path */
-+	bool sysfs_sink_activated;
- 	struct dev_ext_attribute *ea;
- 	struct coresight_device *def_sink;
-	/* cross trigger handling */
-	struct coresight_device *ect_dev;
-+	struct coresight_trace_id_map perf_sink_id_map;
- 	/* sysfs links between components */
- 	int nr_links;
- 	bool has_conns_grp;
-	bool ect_enabled; /* true only if associated ect device is enabled */
- 	/* system configuration and feature lists */
- 	struct list_head feature_csdev_list;
- 	struct list_head config_csdev_list;
-	spinlock_t cscfg_csdev_lock;
-+	raw_spinlock_t cscfg_csdev_lock;
- 	void *active_cscfg_ctxt;
- };
- 
-@@ -271,11 +326,31 @@ static struct coresight_dev_list (var) = {				\
- 
- #define to_coresight_device(d) container_of(d, struct coresight_device, dev)
- 
-+/**
-+ * struct coresight_path - data needed by enable/disable path
-+ * @path_list:		path from source to sink.
-+ * @trace_id:		trace_id of the whole path.
-+ * @handle:		handle of the aux_event.
-+ */
-+struct coresight_path {
-+	struct list_head		path_list;
-+	u8				trace_id;
-+	struct perf_output_handle	*handle;
-+};
-+
-+enum cs_mode {
-+	CS_MODE_DISABLED,
-+	CS_MODE_SYSFS,
-+	CS_MODE_PERF,
-+};
-+
-+#define coresight_ops(csdev)	csdev->ops
- #define source_ops(csdev)	csdev->ops->source_ops
- #define sink_ops(csdev)		csdev->ops->sink_ops
- #define link_ops(csdev)		csdev->ops->link_ops
- #define helper_ops(csdev)	csdev->ops->helper_ops
- #define ect_ops(csdev)		csdev->ops->ect_ops
-+#define panic_ops(csdev)	csdev->ops->panic_ops
- 
- /**
-  * struct coresight_ops_sink - basic operations for a sink
-@@ -287,7 +362,8 @@ static struct coresight_dev_list (var) = {				\
-  * @update_buffer:	update buffer pointers after a trace session.
-  */
- struct coresight_ops_sink {
-	int (*enable)(struct coresight_device *csdev, u32 mode, void *data);
-+	int (*enable)(struct coresight_device *csdev, enum cs_mode mode,
-+		      struct coresight_path *path);
- 	int (*disable)(struct coresight_device *csdev);
- 	void *(*alloc_buffer)(struct coresight_device *csdev,
- 			      struct perf_event *event, void **pages,
-@@ -305,8 +381,12 @@ struct coresight_ops_sink {
-  * @disable:	disables flow between iport and oport.
-  */
- struct coresight_ops_link {
-	int (*enable)(struct coresight_device *csdev, int iport, int oport);
-	void (*disable)(struct coresight_device *csdev, int iport, int oport);
-+	int (*enable)(struct coresight_device *csdev,
-+		      struct coresight_connection *in,
-+		      struct coresight_connection *out);
-+	void (*disable)(struct coresight_device *csdev,
-+			struct coresight_connection *in,
-+			struct coresight_connection *out);
- };
- 
- /**
-@@ -314,18 +394,19 @@ struct coresight_ops_link {
-  * Operations available for sources.
-  * @cpu_id:	returns the value of the CPU number this component
-  *		is associated to.
- * @trace_id:	returns the value of the component's trace ID as known
- *		to the HW.
-  * @enable:	enables tracing for a source.
-  * @disable:	disables tracing for a source.
-+ * @resume_perf: resumes tracing for a source in perf session.
-+ * @pause_perf:	pauses tracing for a source in perf session.
-  */
- struct coresight_ops_source {
- 	int (*cpu_id)(struct coresight_device *csdev);
-	int (*trace_id)(struct coresight_device *csdev);
-	int (*enable)(struct coresight_device *csdev,
-		      struct perf_event *event,  u32 mode);
-+	int (*enable)(struct coresight_device *csdev, struct perf_event *event,
-+		      enum cs_mode mode, struct coresight_path *path);
- 	void (*disable)(struct coresight_device *csdev,
- 			struct perf_event *event);
-+	int (*resume_perf)(struct coresight_device *csdev);
-+	void (*pause_perf)(struct coresight_device *csdev);
- };
- 
- /**
-@@ -338,31 +419,32 @@ struct coresight_ops_source {
-  * @disable	: Disable the device
-  */
- struct coresight_ops_helper {
-	int (*enable)(struct coresight_device *csdev, void *data);
-	int (*disable)(struct coresight_device *csdev, void *data);
-+	int (*enable)(struct coresight_device *csdev, enum cs_mode mode,
-+		      struct coresight_path *path);
-+	int (*disable)(struct coresight_device *csdev,
-+		       struct coresight_path *path);
- };
- 
-+
- /**
- * struct coresight_ops_ect - Ops for an embedded cross trigger device
-+ * struct coresight_ops_panic - Generic device ops for panic handing
-  *
- * @enable	: Enable the device
- * @disable	: Disable the device
-+ * @sync	: Sync the device register state/trace data
-  */
-struct coresight_ops_ect {
-	int (*enable)(struct coresight_device *csdev);
-	int (*disable)(struct coresight_device *csdev);
-+struct coresight_ops_panic {
-+	int (*sync)(struct coresight_device *csdev);
- };
- 
- struct coresight_ops {
-+	int (*trace_id)(struct coresight_device *csdev, enum cs_mode mode,
-+			struct coresight_device *sink);
- 	const struct coresight_ops_sink *sink_ops;
- 	const struct coresight_ops_link *link_ops;
- 	const struct coresight_ops_source *source_ops;
- 	const struct coresight_ops_helper *helper_ops;
-	const struct coresight_ops_ect *ect_ops;
-+	const struct coresight_ops_panic *panic_ops;
- };
- 
-#if IS_ENABLED(CONFIG_CORESIGHT)
-
- static inline u32 csdev_access_relaxed_read32(struct csdev_access *csa,
- 					      u32 offset)
- {
-@@ -372,6 +454,60 @@ static inline u32 csdev_access_relaxed_read32(struct csdev_access *csa,
- 	return csa->read(offset, true, false);
- }
- 
-+#define CORESIGHT_CIDRn(i)	(0xFF0 + ((i) * 4))
-+
-+static inline u32 coresight_get_cid(void __iomem *base)
-+{
-+	u32 i, cid = 0;
-+
-+	for (i = 0; i < 4; i++)
-+		cid |= readl(base + CORESIGHT_CIDRn(i)) << (i * 8);
-+
-+	return cid;
-+}
-+
-+static inline bool is_coresight_device(void __iomem *base)
-+{
-+	u32 cid = coresight_get_cid(base);
-+
-+	return cid == CORESIGHT_CID;
-+}
-+
-+#define CORESIGHT_PIDRn(i)	(0xFE0 + ((i) * 4))
-+
-+static inline u32 coresight_get_pid(struct csdev_access *csa)
-+{
-+	u32 i, pid = 0;
-+
-+	for (i = 0; i < 4; i++)
-+		pid |= csdev_access_relaxed_read32(csa, CORESIGHT_PIDRn(i)) << (i * 8);
-+
-+	return pid;
-+}
-+
-+static inline u64 csdev_access_relaxed_read_pair(struct csdev_access *csa,
-+						 u32 lo_offset, u32 hi_offset)
-+{
-+	if (likely(csa->io_mem)) {
-+		return readl_relaxed(csa->base + lo_offset) |
-+			((u64)readl_relaxed(csa->base + hi_offset) << 32);
-+	}
-+
-+	return csa->read(lo_offset, true, false) | (csa->read(hi_offset, true, false) << 32);
-+}
-+
-+static inline void csdev_access_relaxed_write_pair(struct csdev_access *csa, u64 val,
-+						   u32 lo_offset, u32 hi_offset)
-+{
-+	if (likely(csa->io_mem)) {
-+		writel_relaxed((u32)val, csa->base + lo_offset);
-+		writel_relaxed((u32)(val >> 32), csa->base + hi_offset);
-+	} else {
-+		csa->write((u32)val, lo_offset, true, false);
-+		csa->write((u32)(val >> 32), hi_offset, true, false);
-+	}
-+}
-+
- static inline u32 csdev_access_read32(struct csdev_access *csa, u32 offset)
- {
- 	if (likely(csa->io_mem))
-@@ -460,9 +596,14 @@ static inline void csdev_access_write64(struct csdev_access *csa, u64 val, u32 o
- }
- #endif	/* CONFIG_64BIT */
- 
-+static inline bool coresight_is_device_source(struct coresight_device *csdev)
-+{
-+	return csdev && (csdev->type == CORESIGHT_DEV_TYPE_SOURCE);
-+}
-+
- static inline bool coresight_is_percpu_source(struct coresight_device *csdev)
- {
-	return csdev && (csdev->type == CORESIGHT_DEV_TYPE_SOURCE) &&
-+	return csdev && coresight_is_device_source(csdev) &&
- 	       (csdev->subtype.source_subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_PROC);
- }
- 
-@@ -472,23 +613,60 @@ static inline bool coresight_is_percpu_sink(struct coresight_device *csdev)
- 	       (csdev->subtype.sink_subtype == CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM);
- }
- 
-extern struct coresight_device *
-coresight_register(struct coresight_desc *desc);
-extern void coresight_unregister(struct coresight_device *csdev);
-extern int coresight_enable(struct coresight_device *csdev);
-extern void coresight_disable(struct coresight_device *csdev);
-extern int coresight_timeout(struct csdev_access *csa, u32 offset,
-			     int position, int value);
-+/*
-+ * Atomically try to take the device and set a new mode. Returns true on
-+ * success, false if the device is already taken by someone else.
-+ */
-+static inline bool coresight_take_mode(struct coresight_device *csdev,
-+				       enum cs_mode new_mode)
-+{
-+	int curr = CS_MODE_DISABLED;
-+
-+	return atomic_try_cmpxchg_acquire(&csdev->mode, &curr, new_mode);
-+}
-+
-+static inline enum cs_mode coresight_get_mode(struct coresight_device *csdev)
-+{
-+	return atomic_read_acquire(&csdev->mode);
-+}
-+
-+static inline void coresight_set_mode(struct coresight_device *csdev,
-+				      enum cs_mode new_mode)
-+{
-+	enum cs_mode current_mode = coresight_get_mode(csdev);
-+
-+	/*
-+	 * Changing to a new mode must be done from an already disabled state
-+	 * unless it's synchronized with coresight_take_mode(). Otherwise the
-+	 * device is already in use and signifies a locking issue.
-+	 */
-+	WARN(new_mode != CS_MODE_DISABLED && current_mode != CS_MODE_DISABLED &&
-+	     current_mode != new_mode, "Device already in use\n");
- 
-extern int coresight_claim_device(struct coresight_device *csdev);
-extern int coresight_claim_device_unlocked(struct coresight_device *csdev);
-+	atomic_set_release(&csdev->mode, new_mode);
-+}
- 
-extern void coresight_disclaim_device(struct coresight_device *csdev);
-extern void coresight_disclaim_device_unlocked(struct coresight_device *csdev);
-extern char *coresight_alloc_device_name(struct coresight_dev_list *devs,
-+struct coresight_device *coresight_register(struct coresight_desc *desc);
-+void coresight_unregister(struct coresight_device *csdev);
-+int coresight_enable_sysfs(struct coresight_device *csdev);
-+void coresight_disable_sysfs(struct coresight_device *csdev);
-+int coresight_timeout(struct csdev_access *csa, u32 offset, int position, int value);
-+typedef void (*coresight_timeout_cb_t) (struct csdev_access *, u32, int, int);
-+int coresight_timeout_action(struct csdev_access *csa, u32 offset, int position, int value,
-+			     coresight_timeout_cb_t cb);
-+int coresight_claim_device(struct coresight_device *csdev);
-+int coresight_claim_device_unlocked(struct coresight_device *csdev);
-+
-+int coresight_claim_device(struct coresight_device *csdev);
-+int coresight_claim_device_unlocked(struct coresight_device *csdev);
-+void coresight_clear_self_claim_tag(struct csdev_access *csa);
-+void coresight_clear_self_claim_tag_unlocked(struct csdev_access *csa);
-+void coresight_disclaim_device(struct coresight_device *csdev);
-+void coresight_disclaim_device_unlocked(struct coresight_device *csdev);
-+char *coresight_alloc_device_name(struct coresight_dev_list *devs,
- 					 struct device *dev);
- 
-extern bool coresight_loses_context_with_cpu(struct device *dev);
-+bool coresight_loses_context_with_cpu(struct device *dev);
- 
- u32 coresight_relaxed_read32(struct coresight_device *csdev, u32 offset);
- u32 coresight_read32(struct coresight_device *csdev, u32 offset);
-@@ -501,85 +679,31 @@ void coresight_relaxed_write64(struct coresight_device *csdev,
- 			       u64 val, u32 offset);
- void coresight_write64(struct coresight_device *csdev, u64 val, u32 offset);
- 
-#else
-static inline struct coresight_device *
-coresight_register(struct coresight_desc *desc) { return NULL; }
-static inline void coresight_unregister(struct coresight_device *csdev) {}
-static inline int
-coresight_enable(struct coresight_device *csdev) { return -ENOSYS; }
-static inline void coresight_disable(struct coresight_device *csdev) {}
-
-static inline int coresight_timeout(struct csdev_access *csa, u32 offset,
-				    int position, int value)
-{
-	return 1;
-}
-
-static inline int coresight_claim_device_unlocked(struct coresight_device *csdev)
-{
-	return -EINVAL;
-}
-
-static inline int coresight_claim_device(struct coresight_device *csdev)
-{
-	return -EINVAL;
-}
-
-static inline void coresight_disclaim_device(struct coresight_device *csdev) {}
-static inline void coresight_disclaim_device_unlocked(struct coresight_device *csdev) {}
-
-static inline bool coresight_loses_context_with_cpu(struct device *dev)
-{
-	return false;
-}
-
-static inline u32 coresight_relaxed_read32(struct coresight_device *csdev, u32 offset)
-{
-	WARN_ON_ONCE(1);
-	return 0;
-}
-
-static inline u32 coresight_read32(struct coresight_device *csdev, u32 offset)
-{
-	WARN_ON_ONCE(1);
-	return 0;
-}
-
-static inline void coresight_write32(struct coresight_device *csdev, u32 val, u32 offset)
-{
-}
-
-static inline void coresight_relaxed_write32(struct coresight_device *csdev,
-					     u32 val, u32 offset)
-{
-}
-
-static inline u64 coresight_relaxed_read64(struct coresight_device *csdev,
-					   u32 offset)
-{
-	WARN_ON_ONCE(1);
-	return 0;
-}
-
-static inline u64 coresight_read64(struct coresight_device *csdev, u32 offset)
-{
-	WARN_ON_ONCE(1);
-	return 0;
-}
-
-static inline void coresight_relaxed_write64(struct coresight_device *csdev,
-					     u64 val, u32 offset)
-{
-}
-
-static inline void coresight_write64(struct coresight_device *csdev, u64 val, u32 offset)
-{
-}
-
-#endif		/* IS_ENABLED(CONFIG_CORESIGHT) */
-
-extern int coresight_get_cpu(struct device *dev);
-+int coresight_get_cpu(struct device *dev);
-+int coresight_get_static_trace_id(struct device *dev, u32 *id);
- 
- struct coresight_platform_data *coresight_get_platform_data(struct device *dev);
-
-+struct coresight_connection *
-+coresight_add_out_conn(struct device *dev,
-+		       struct coresight_platform_data *pdata,
-+		       const struct coresight_connection *new_conn);
-+int coresight_add_in_conn(struct coresight_connection *conn);
-+struct coresight_device *
-+coresight_find_input_type(struct coresight_platform_data *pdata,
-+			  enum coresight_dev_type type,
-+			  union coresight_dev_subtype subtype);
-+struct coresight_device *
-+coresight_find_output_type(struct coresight_platform_data *pdata,
-+			   enum coresight_dev_type type,
-+			   union coresight_dev_subtype subtype);
-+
-+int coresight_init_driver(const char *drv, struct amba_driver *amba_drv,
-+			  struct platform_driver *pdev_drv, struct module *owner);
-+
-+void coresight_remove_driver(struct amba_driver *amba_drv,
-+			     struct platform_driver *pdev_drv);
-+int coresight_etm_get_trace_id(struct coresight_device *csdev, enum cs_mode mode,
-+			       struct coresight_device *sink);
-+int coresight_get_enable_clocks(struct device *dev, struct clk **pclk,
-+				struct clk **atclk);
- #endif		/* _LINUX_COREISGHT_H */
-diff --git a/include/linux/counter.h b/include/linux/counter.h
-index 1fe17f5adb09..f208e867dd0f 100644
--- a/include/linux/counter.h
-+++ b/include/linux/counter.h
-@@ -6,14 +6,15 @@
- #ifndef _COUNTER_H_
- #define _COUNTER_H_
- 
-+#include <linux/array_size.h>
- #include <linux/cdev.h>
- #include <linux/device.h>
-#include <linux/kernel.h>
- #include <linux/kfifo.h>
- #include <linux/mutex.h>
- #include <linux/spinlock_types.h>
- #include <linux/types.h>
- #include <linux/wait.h>
-+
- #include <uapi/linux/counter.h>
- 
- struct counter_device;
-@@ -31,6 +32,8 @@ enum counter_comp_type {
- 	COUNTER_COMP_ENUM,
- 	COUNTER_COMP_COUNT_DIRECTION,
- 	COUNTER_COMP_COUNT_MODE,
-+	COUNTER_COMP_SIGNAL_POLARITY,
-+	COUNTER_COMP_ARRAY,
- };
- 
- /**
-@@ -38,66 +41,114 @@ enum counter_comp_type {
-  * @type:		Counter component data type
-  * @name:		device-specific component name
-  * @priv:		component-relevant data
- * @action_read:		Synapse action mode read callback. The read value of the
-+ * @action_read:	Synapse action mode read callback. The read value of the
-  *			respective Synapse action mode should be passed back via
-  *			the action parameter.
- * @device_u8_read:		Device u8 component read callback. The read value of the
-+ * @device_u8_read:	Device u8 component read callback. The read value of the
-  *			respective Device u8 component should be passed back via
-  *			the val parameter.
- * @count_u8_read:		Count u8 component read callback. The read value of the
-+ * @count_u8_read:	Count u8 component read callback. The read value of the
-  *			respective Count u8 component should be passed back via
-  *			the val parameter.
- * @signal_u8_read:		Signal u8 component read callback. The read value of the
-+ * @signal_u8_read:	Signal u8 component read callback. The read value of the
-  *			respective Signal u8 component should be passed back via
-  *			the val parameter.
- * @device_u32_read:		Device u32 component read callback. The read value of
-+ * @device_u32_read:	Device u32 component read callback. The read value of
-  *			the respective Device u32 component should be passed
-  *			back via the val parameter.
- * @count_u32_read:		Count u32 component read callback. The read value of the
-+ * @count_u32_read:	Count u32 component read callback. The read value of the
-  *			respective Count u32 component should be passed back via
-  *			the val parameter.
- * @signal_u32_read:		Signal u32 component read callback. The read value of
-+ * @signal_u32_read:	Signal u32 component read callback. The read value of
-  *			the respective Signal u32 component should be passed
-  *			back via the val parameter.
- * @device_u64_read:		Device u64 component read callback. The read value of
-+ * @device_u64_read:	Device u64 component read callback. The read value of
-  *			the respective Device u64 component should be passed
-  *			back via the val parameter.
- * @count_u64_read:		Count u64 component read callback. The read value of the
-+ * @count_u64_read:	Count u64 component read callback. The read value of the
-  *			respective Count u64 component should be passed back via
-  *			the val parameter.
- * @signal_u64_read:		Signal u64 component read callback. The read value of
-+ * @signal_u64_read:	Signal u64 component read callback. The read value of
-  *			the respective Signal u64 component should be passed
-  *			back via the val parameter.
- * @action_write:		Synapse action mode write callback. The write value of
-+ * @signal_array_u32_read:	Signal u32 array component read callback. The
-+ *				index of the respective Count u32 array
-+ *				component element is passed via the idx
-+ *				parameter. The read value of the respective
-+ *				Count u32 array component element should be
-+ *				passed back via the val parameter.
-+ * @device_array_u64_read:	Device u64 array component read callback. The
-+ *				index of the respective Device u64 array
-+ *				component element is passed via the idx
-+ *				parameter. The read value of the respective
-+ *				Device u64 array component element should be
-+ *				passed back via the val parameter.
-+ * @count_array_u64_read:	Count u64 array component read callback. The
-+ *				index of the respective Count u64 array
-+ *				component element is passed via the idx
-+ *				parameter. The read value of the respective
-+ *				Count u64 array component element should be
-+ *				passed back via the val parameter.
-+ * @signal_array_u64_read:	Signal u64 array component read callback. The
-+ *				index of the respective Count u64 array
-+ *				component element is passed via the idx
-+ *				parameter. The read value of the respective
-+ *				Count u64 array component element should be
-+ *				passed back via the val parameter.
-+ * @action_write:	Synapse action mode write callback. The write value of
-  *			the respective Synapse action mode is passed via the
-  *			action parameter.
- * @device_u8_write:		Device u8 component write callback. The write value of
-+ * @device_u8_write:	Device u8 component write callback. The write value of
-  *			the respective Device u8 component is passed via the val
-  *			parameter.
- * @count_u8_write:		Count u8 component write callback. The write value of
-+ * @count_u8_write:	Count u8 component write callback. The write value of
-  *			the respective Count u8 component is passed via the val
-  *			parameter.
- * @signal_u8_write:		Signal u8 component write callback. The write value of
-+ * @signal_u8_write:	Signal u8 component write callback. The write value of
-  *			the respective Signal u8 component is passed via the val
-  *			parameter.
- * @device_u32_write:		Device u32 component write callback. The write value of
-+ * @device_u32_write:	Device u32 component write callback. The write value of
-  *			the respective Device u32 component is passed via the
-  *			val parameter.
- * @count_u32_write:		Count u32 component write callback. The write value of
-+ * @count_u32_write:	Count u32 component write callback. The write value of
-  *			the respective Count u32 component is passed via the val
-  *			parameter.
- * @signal_u32_write:		Signal u32 component write callback. The write value of
-+ * @signal_u32_write:	Signal u32 component write callback. The write value of
-  *			the respective Signal u32 component is passed via the
-  *			val parameter.
- * @device_u64_write:		Device u64 component write callback. The write value of
-+ * @device_u64_write:	Device u64 component write callback. The write value of
-  *			the respective Device u64 component is passed via the
-  *			val parameter.
- * @count_u64_write:		Count u64 component write callback. The write value of
-+ * @count_u64_write:	Count u64 component write callback. The write value of
-  *			the respective Count u64 component is passed via the val
-  *			parameter.
- * @signal_u64_write:		Signal u64 component write callback. The write value of
-+ * @signal_u64_write:	Signal u64 component write callback. The write value of
-  *			the respective Signal u64 component is passed via the
-  *			val parameter.
-+ * @signal_array_u32_write:	Signal u32 array component write callback. The
-+ *				index of the respective Signal u32 array
-+ *				component element is passed via the idx
-+ *				parameter. The write value of the respective
-+ *				Signal u32 array component element is passed via
-+ *				the val parameter.
-+ * @device_array_u64_write:	Device u64 array component write callback. The
-+ *				index of the respective Device u64 array
-+ *				component element is passed via the idx
-+ *				parameter. The write value of the respective
-+ *				Device u64 array component element is passed via
-+ *				the val parameter.
-+ * @count_array_u64_write:	Count u64 array component write callback. The
-+ *				index of the respective Count u64 array
-+ *				component element is passed via the idx
-+ *				parameter. The write value of the respective
-+ *				Count u64 array component element is passed via
-+ *				the val parameter.
-+ * @signal_array_u64_write:	Signal u64 array component write callback. The
-+ *				index of the respective Signal u64 array
-+ *				component element is passed via the idx
-+ *				parameter. The write value of the respective
-+ *				Signal u64 array component element is passed via
-+ *				the val parameter.
-  */
- struct counter_comp {
- 	enum counter_comp_type type;
-@@ -125,6 +176,17 @@ struct counter_comp {
- 				      struct counter_count *count, u64 *val);
- 		int (*signal_u64_read)(struct counter_device *counter,
- 				       struct counter_signal *signal, u64 *val);
-+		int (*signal_array_u32_read)(struct counter_device *counter,
-+					     struct counter_signal *signal,
-+					     size_t idx, u32 *val);
-+		int (*device_array_u64_read)(struct counter_device *counter,
-+					     size_t idx, u64 *val);
-+		int (*count_array_u64_read)(struct counter_device *counter,
-+					    struct counter_count *count,
-+					    size_t idx, u64 *val);
-+		int (*signal_array_u64_read)(struct counter_device *counter,
-+					     struct counter_signal *signal,
-+					     size_t idx, u64 *val);
- 	};
- 	union {
- 		int (*action_write)(struct counter_device *counter,
-@@ -148,6 +210,17 @@ struct counter_comp {
- 				       struct counter_count *count, u64 val);
- 		int (*signal_u64_write)(struct counter_device *counter,
- 					struct counter_signal *signal, u64 val);
-+		int (*signal_array_u32_write)(struct counter_device *counter,
-+					      struct counter_signal *signal,
-+					      size_t idx, u32 val);
-+		int (*device_array_u64_write)(struct counter_device *counter,
-+					      size_t idx, u64 val);
-+		int (*count_array_u64_write)(struct counter_device *counter,
-+					     struct counter_count *count,
-+					     size_t idx, u64 val);
-+		int (*signal_array_u64_write)(struct counter_device *counter,
-+					      struct counter_signal *signal,
-+					      size_t idx, u64 val);
- 	};
- };
- 
-@@ -287,7 +360,6 @@ struct counter_ops {
-  * @num_counts:		number of Counts specified in @counts
-  * @ext:		optional array of Counter device extensions
-  * @num_ext:		number of Counter device extensions specified in @ext
- * @priv:		optional private data supplied by driver
-  * @dev:		internal device structure
-  * @chrdev:		internal character device structure
-  * @events_list:	list of current watching Counter events
-@@ -327,7 +399,7 @@ struct counter_device {
- 	struct mutex ops_exist_lock;
- };
- 
-void *counter_priv(const struct counter_device *const counter);
-+void *counter_priv(const struct counter_device *const counter) __attribute_const__;
- 
- struct counter_device *counter_alloc(size_t sizeof_priv);
- void counter_put(struct counter_device *const counter);
-@@ -452,9 +524,65 @@ struct counter_available {
- 	.priv = &(_available), \
- }
- 
-+struct counter_array {
-+	enum counter_comp_type type;
-+	const struct counter_available *avail;
-+	union {
-+		size_t length;
-+		size_t idx;
-+	};
-+};
-+
-+#define DEFINE_COUNTER_ARRAY_U64(_name, _length) \
-+	struct counter_array _name = { \
-+		.type = COUNTER_COMP_U64, \
-+		.length = (_length), \
-+	}
-+
-+#define DEFINE_COUNTER_ARRAY_CAPTURE(_name, _length) \
-+	DEFINE_COUNTER_ARRAY_U64(_name, _length)
-+
-+#define DEFINE_COUNTER_ARRAY_POLARITY(_name, _available, _length) \
-+	struct counter_array _name = { \
-+		.type = COUNTER_COMP_SIGNAL_POLARITY, \
-+		.avail = &(_available), \
-+		.length = (_length), \
-+	}
-+
-+#define COUNTER_COMP_DEVICE_ARRAY_U64(_name, _read, _write, _array) \
-+{ \
-+	.type = COUNTER_COMP_ARRAY, \
-+	.name = (_name), \
-+	.device_array_u64_read = (_read), \
-+	.device_array_u64_write = (_write), \
-+	.priv = &(_array), \
-+}
-+#define COUNTER_COMP_COUNT_ARRAY_U64(_name, _read, _write, _array) \
-+{ \
-+	.type = COUNTER_COMP_ARRAY, \
-+	.name = (_name), \
-+	.count_array_u64_read = (_read), \
-+	.count_array_u64_write = (_write), \
-+	.priv = &(_array), \
-+}
-+#define COUNTER_COMP_SIGNAL_ARRAY_U64(_name, _read, _write, _array) \
-+{ \
-+	.type = COUNTER_COMP_ARRAY, \
-+	.name = (_name), \
-+	.signal_array_u64_read = (_read), \
-+	.signal_array_u64_write = (_write), \
-+	.priv = &(_array), \
-+}
-+
-+#define COUNTER_COMP_CAPTURE(_read, _write) \
-+	COUNTER_COMP_COUNT_U64("capture", _read, _write)
-+
- #define COUNTER_COMP_CEILING(_read, _write) \
- 	COUNTER_COMP_COUNT_U64("ceiling", _read, _write)
- 
-+#define COUNTER_COMP_COMPARE(_read, _write) \
-+	COUNTER_COMP_COUNT_U64("compare", _read, _write)
-+
- #define COUNTER_COMP_COUNT_MODE(_read, _write, _available) \
- { \
- 	.type = COUNTER_COMP_COUNT_MODE, \
-@@ -477,10 +605,34 @@ struct counter_available {
- #define COUNTER_COMP_FLOOR(_read, _write) \
- 	COUNTER_COMP_COUNT_U64("floor", _read, _write)
- 
-+#define COUNTER_COMP_FREQUENCY(_read) \
-+	COUNTER_COMP_SIGNAL_U64("frequency", _read, NULL)
-+
-+#define COUNTER_COMP_POLARITY(_read, _write, _available) \
-+{ \
-+	.type = COUNTER_COMP_SIGNAL_POLARITY, \
-+	.name = "polarity", \
-+	.signal_u32_read = (_read), \
-+	.signal_u32_write = (_write), \
-+	.priv = &(_available), \
-+}
-+
- #define COUNTER_COMP_PRESET(_read, _write) \
- 	COUNTER_COMP_COUNT_U64("preset", _read, _write)
- 
- #define COUNTER_COMP_PRESET_ENABLE(_read, _write) \
- 	COUNTER_COMP_COUNT_BOOL("preset_enable", _read, _write)
- 
-+#define COUNTER_COMP_ARRAY_CAPTURE(_read, _write, _array) \
-+	COUNTER_COMP_COUNT_ARRAY_U64("capture", _read, _write, _array)
-+
-+#define COUNTER_COMP_ARRAY_POLARITY(_read, _write, _array) \
-+{ \
-+	.type = COUNTER_COMP_ARRAY, \
-+	.name = "polarity", \
-+	.signal_array_u32_read = (_read), \
-+	.signal_array_u32_write = (_write), \
-+	.priv = &(_array), \
-+}
-+
- #endif /* _COUNTER_H_ */
-diff --git a/include/linux/cper.h b/include/linux/cper.h
-index eacb7dd7b3af..5b1236d8c65b 100644
--- a/include/linux/cper.h
-+++ b/include/linux/cper.h
-@@ -89,6 +89,33 @@ enum {
- #define CPER_NOTIFY_DMAR						\
- 	GUID_INIT(0x667DD791, 0xC6B3, 0x4c27, 0x8A, 0x6B, 0x0F, 0x8E,	\
- 		  0x72, 0x2D, 0xEB, 0x41)
-+/* CXL Protocol Error Section */
-+#define CPER_SEC_CXL_PROT_ERR						\
-+	GUID_INIT(0x80B9EFB4, 0x52B5, 0x4DE3, 0xA7, 0x77, 0x68, 0x78,	\
-+		  0x4B, 0x77, 0x10, 0x48)
-+
-+/* CXL Event record UUIDs are formatted as GUIDs and reported in section type */
-+/*
-+ * General Media Event Record
-+ * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43
-+ */
-+#define CPER_SEC_CXL_GEN_MEDIA_GUID					\
-+	GUID_INIT(0xfbcd0a77, 0xc260, 0x417f,				\
-+		  0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6)
-+/*
-+ * DRAM Event Record
-+ * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44
-+ */
-+#define CPER_SEC_CXL_DRAM_GUID						\
-+	GUID_INIT(0x601dcbb3, 0x9c06, 0x4eab,				\
-+		  0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24)
-+/*
-+ * Memory Module Event Record
-+ * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45
-+ */
-+#define CPER_SEC_CXL_MEM_MODULE_GUID					\
-+	GUID_INIT(0xfe927475, 0xdd59, 0x4339,				\
-+		  0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74)
- 
- /*
-  * Flags bits definitions for flags in struct cper_record_header
-@@ -270,11 +297,11 @@ enum {
- #define CPER_ARM_INFO_FLAGS_PROPAGATED		BIT(2)
- #define CPER_ARM_INFO_FLAGS_OVERFLOW		BIT(3)
- 
-#define CPER_ARM_CACHE_ERROR			0
-#define CPER_ARM_TLB_ERROR			1
-#define CPER_ARM_BUS_ERROR			2
-#define CPER_ARM_VENDOR_ERROR			3
-#define CPER_ARM_MAX_TYPE			CPER_ARM_VENDOR_ERROR
-+#define CPER_ARM_ERR_TYPE_MASK			GENMASK(4,1)
-+#define CPER_ARM_CACHE_ERROR			BIT(1)
-+#define CPER_ARM_TLB_ERROR			BIT(2)
-+#define CPER_ARM_BUS_ERROR			BIT(3)
-+#define CPER_ARM_VENDOR_ERROR			BIT(4)
- 
- #define CPER_ARM_ERR_VALID_TRANSACTION_TYPE	BIT(0)
- #define CPER_ARM_ERR_VALID_OPERATION_TYPE	BIT(1)
-@@ -561,6 +588,8 @@ const char *cper_mem_err_type_str(unsigned int);
- const char *cper_mem_err_status_str(u64 status);
- void cper_print_bits(const char *prefix, unsigned int bits,
- 		     const char * const strs[], unsigned int strs_size);
-+int cper_bits_to_str(char *buf, int buf_size, unsigned long bits,
-+		     const char * const strs[], unsigned int strs_size);
- void cper_mem_err_pack(const struct cper_sec_mem_err *,
- 		       struct cper_mem_err_compact *);
- const char *cper_mem_err_unpack(struct trace_seq *,
-@@ -572,4 +601,14 @@ void cper_print_proc_ia(const char *pfx,
- int cper_mem_err_location(struct cper_mem_err_compact *mem, char *msg);
- int cper_dimm_err_location(struct cper_mem_err_compact *mem, char *msg);
- 
-+struct acpi_hest_generic_status;
-+void cper_estatus_print(const char *pfx,
-+			const struct acpi_hest_generic_status *estatus);
-+int cper_estatus_check_header(const struct acpi_hest_generic_status *estatus);
-+int cper_estatus_check(const struct acpi_hest_generic_status *estatus);
-+
-+struct cxl_cper_sec_prot_err;
-+void cxl_cper_print_prot_err(const char *pfx,
-+			     const struct cxl_cper_sec_prot_err *prot_err);
-+
- #endif
-diff --git a/include/linux/cpu.h b/include/linux/cpu.h
-index 314802f98b9d..487b3bf2e1ea 100644
--- a/include/linux/cpu.h
-+++ b/include/linux/cpu.h
-@@ -16,8 +16,9 @@
- 
- #include <linux/node.h>
- #include <linux/compiler.h>
-#include <linux/cpumask.h>
- #include <linux/cpuhotplug.h>
-+#include <linux/cpuhplock.h>
-+#include <linux/cpu_smt.h>
- 
- struct device;
- struct device_node;
-@@ -70,17 +71,37 @@ extern ssize_t cpu_show_mmio_stale_data(struct device *dev,
- 					char *buf);
- extern ssize_t cpu_show_retbleed(struct device *dev,
- 				 struct device_attribute *attr, char *buf);
-+extern ssize_t cpu_show_spec_rstack_overflow(struct device *dev,
-+					     struct device_attribute *attr, char *buf);
-+extern ssize_t cpu_show_gds(struct device *dev,
-+			    struct device_attribute *attr, char *buf);
-+extern ssize_t cpu_show_reg_file_data_sampling(struct device *dev,
-+					       struct device_attribute *attr, char *buf);
-+extern ssize_t cpu_show_ghostwrite(struct device *dev, struct device_attribute *attr, char *buf);
-+extern ssize_t cpu_show_old_microcode(struct device *dev,
-+				      struct device_attribute *attr, char *buf);
-+extern ssize_t cpu_show_indirect_target_selection(struct device *dev,
-+						  struct device_attribute *attr, char *buf);
-+extern ssize_t cpu_show_tsa(struct device *dev, struct device_attribute *attr, char *buf);
-+extern ssize_t cpu_show_vmscape(struct device *dev, struct device_attribute *attr, char *buf);
- 
- extern __printf(4, 5)
- struct device *cpu_device_create(struct device *parent, void *drvdata,
- 				 const struct attribute_group **groups,
- 				 const char *fmt, ...);
-+extern bool arch_cpu_is_hotpluggable(int cpu);
-+extern int arch_register_cpu(int cpu);
-+extern void arch_unregister_cpu(int cpu);
- #ifdef CONFIG_HOTPLUG_CPU
- extern void unregister_cpu(struct cpu *cpu);
- extern ssize_t arch_cpu_probe(const char *, size_t);
- extern ssize_t arch_cpu_release(const char *, size_t);
- #endif
- 
-+#ifdef CONFIG_GENERIC_CPU_DEVICES
-+DECLARE_PER_CPU(struct cpu, cpu_devices);
-+#endif
-+
- /*
-  * These states are not related to the core CPU hotplug mechanism. They are
-  * used by various (sub)architectures to track internal state
-@@ -100,7 +121,8 @@ void notify_cpu_starting(unsigned int cpu);
- extern void cpu_maps_update_begin(void);
- extern void cpu_maps_update_done(void);
- int bringup_hibernate_cpu(unsigned int sleep_cpu);
-void bringup_nonboot_cpus(unsigned int setup_max_cpus);
-+void bringup_nonboot_cpus(unsigned int max_cpus);
-+int arch_cpu_rescan_dead_smt_siblings(void);
- 
- #else	/* CONFIG_SMP */
- #define cpuhp_tasks_frozen	0
-@@ -115,38 +137,10 @@ static inline void cpu_maps_update_done(void)
- 
- static inline int add_cpu(unsigned int cpu) { return 0;}
- 
-#endif /* CONFIG_SMP */
-extern struct bus_type cpu_subsys;
-
-extern int lockdep_is_cpus_held(void);
-+static inline int arch_cpu_rescan_dead_smt_siblings(void) { return 0; }
- 
-#ifdef CONFIG_HOTPLUG_CPU
-extern void cpus_write_lock(void);
-extern void cpus_write_unlock(void);
-extern void cpus_read_lock(void);
-extern void cpus_read_unlock(void);
-extern int  cpus_read_trylock(void);
-extern void lockdep_assert_cpus_held(void);
-extern void cpu_hotplug_disable(void);
-extern void cpu_hotplug_enable(void);
-void clear_tasks_mm_cpumask(int cpu);
-int remove_cpu(unsigned int cpu);
-int cpu_device_down(struct device *dev);
-extern void smp_shutdown_nonboot_cpus(unsigned int primary_cpu);
-
-#else /* CONFIG_HOTPLUG_CPU */
-
-static inline void cpus_write_lock(void) { }
-static inline void cpus_write_unlock(void) { }
-static inline void cpus_read_lock(void) { }
-static inline void cpus_read_unlock(void) { }
-static inline int  cpus_read_trylock(void) { return true; }
-static inline void lockdep_assert_cpus_held(void) { }
-static inline void cpu_hotplug_disable(void) { }
-static inline void cpu_hotplug_enable(void) { }
-static inline int remove_cpu(unsigned int cpu) { return -EPERM; }
-static inline void smp_shutdown_nonboot_cpus(unsigned int primary_cpu) { }
-#endif	/* !CONFIG_HOTPLUG_CPU */
-+#endif /* CONFIG_SMP */
-+extern const struct bus_type cpu_subsys;
- 
- #ifdef CONFIG_PM_SLEEP_SMP
- extern int freeze_secondary_cpus(int primary);
-@@ -163,7 +157,7 @@ static inline int suspend_disable_secondary_cpus(void)
- }
- static inline void suspend_enable_secondary_cpus(void)
- {
-	return thaw_secondary_cpus();
-+	thaw_secondary_cpus();
- }
- 
- #else /* !CONFIG_PM_SLEEP_SMP */
-@@ -176,60 +170,63 @@ void __noreturn cpu_startup_entry(enum cpuhp_state state);
- 
- void cpu_idle_poll_ctrl(bool enable);
- 
-/* Attach to any functions which should be considered cpuidle. */
-#define __cpuidle	__section(".cpuidle.text")
-
- bool cpu_in_idle(unsigned long pc);
- 
- void arch_cpu_idle(void);
- void arch_cpu_idle_prepare(void);
- void arch_cpu_idle_enter(void);
- void arch_cpu_idle_exit(void);
-void arch_cpu_idle_dead(void);
-+void arch_tick_broadcast_enter(void);
-+void arch_tick_broadcast_exit(void);
-+void __noreturn arch_cpu_idle_dead(void);
- 
-int cpu_report_state(int cpu);
-int cpu_check_up_prepare(int cpu);
-void cpu_set_state_online(int cpu);
-void play_idle_precise(u64 duration_ns, u64 latency_ns);
-+#ifdef CONFIG_ARCH_HAS_CPU_FINALIZE_INIT
-+void arch_cpu_finalize_init(void);
-+#else
-+static inline void arch_cpu_finalize_init(void) { }
-+#endif
- 
-static inline void play_idle(unsigned long duration_us)
-{
-	play_idle_precise(duration_us * NSEC_PER_USEC, U64_MAX);
-}
-+void play_idle_precise(u64 duration_ns, u64 latency_ns);
- 
- #ifdef CONFIG_HOTPLUG_CPU
-bool cpu_wait_death(unsigned int cpu, int seconds);
-bool cpu_report_death(void);
- void cpuhp_report_idle_dead(void);
- #else
- static inline void cpuhp_report_idle_dead(void) { }
- #endif /* #ifdef CONFIG_HOTPLUG_CPU */
- 
-enum cpuhp_smt_control {
-	CPU_SMT_ENABLED,
-	CPU_SMT_DISABLED,
-	CPU_SMT_FORCE_DISABLED,
-	CPU_SMT_NOT_SUPPORTED,
-	CPU_SMT_NOT_IMPLEMENTED,
-+enum cpu_attack_vectors {
-+	CPU_MITIGATE_USER_KERNEL,
-+	CPU_MITIGATE_USER_USER,
-+	CPU_MITIGATE_GUEST_HOST,
-+	CPU_MITIGATE_GUEST_GUEST,
-+	NR_CPU_ATTACK_VECTORS,
- };
- 
-#if defined(CONFIG_SMP) && defined(CONFIG_HOTPLUG_SMT)
-extern enum cpuhp_smt_control cpu_smt_control;
-extern void cpu_smt_disable(bool force);
-extern void cpu_smt_check_topology(void);
-extern bool cpu_smt_possible(void);
-extern int cpuhp_smt_enable(void);
-extern int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval);
-#else
-# define cpu_smt_control		(CPU_SMT_NOT_IMPLEMENTED)
-static inline void cpu_smt_disable(bool force) { }
-static inline void cpu_smt_check_topology(void) { }
-static inline bool cpu_smt_possible(void) { return false; }
-static inline int cpuhp_smt_enable(void) { return 0; }
-static inline int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) { return 0; }
-#endif
-+enum smt_mitigations {
-+	SMT_MITIGATIONS_OFF,
-+	SMT_MITIGATIONS_AUTO,
-+	SMT_MITIGATIONS_ON,
-+};
- 
-+#ifdef CONFIG_CPU_MITIGATIONS
- extern bool cpu_mitigations_off(void);
- extern bool cpu_mitigations_auto_nosmt(void);
-+extern bool cpu_attack_vector_mitigated(enum cpu_attack_vectors v);
-+extern enum smt_mitigations smt_mitigations;
-+#else
-+static inline bool cpu_mitigations_off(void)
-+{
-+	return true;
-+}
-+static inline bool cpu_mitigations_auto_nosmt(void)
-+{
-+	return false;
-+}
-+static inline bool cpu_attack_vector_mitigated(enum cpu_attack_vectors v)
-+{
-+	return false;
-+}
-+#define smt_mitigations SMT_MITIGATIONS_OFF
-+#endif
- 
- #endif /* _LINUX_CPU_H_ */
-diff --git a/include/linux/cpu_cooling.h b/include/linux/cpu_cooling.h
-index a3bdc8a98f2c..2c774fb3c091 100644
--- a/include/linux/cpu_cooling.h
-+++ b/include/linux/cpu_cooling.h
-@@ -15,7 +15,6 @@
- 
- #include <linux/of.h>
- #include <linux/thermal.h>
-#include <linux/cpumask.h>
- 
- struct cpufreq_policy;
- 
-diff --git a/include/linux/cpu_rmap.h b/include/linux/cpu_rmap.h
-index be8aea04d023..2fd7ba75362a 100644
--- a/include/linux/cpu_rmap.h
-+++ b/include/linux/cpu_rmap.h
-@@ -7,7 +7,7 @@
-  * Copyright 2011 Solarflare Communications Inc.
-  */
- 
-#include <linux/cpumask.h>
-+#include <linux/cpumask_types.h>
- #include <linux/gfp.h>
- #include <linux/slab.h>
- #include <linux/kref.h>
-@@ -16,14 +16,13 @@
-  * struct cpu_rmap - CPU affinity reverse-map
-  * @refcount: kref for object
-  * @size: Number of objects to be reverse-mapped
- * @used: Number of objects added
-  * @obj: Pointer to array of object pointers
-  * @near: For each CPU, the index and distance to the nearest object,
-  *      based on affinity masks
-  */
- struct cpu_rmap {
- 	struct kref	refcount;
-	u16		size, used;
-+	u16		size;
- 	void		**obj;
- 	struct {
- 		u16	index;
-@@ -33,6 +32,7 @@ struct cpu_rmap {
- #define CPU_RMAP_DIST_INF 0xffff
- 
- extern struct cpu_rmap *alloc_cpu_rmap(unsigned int size, gfp_t flags);
-+extern void cpu_rmap_get(struct cpu_rmap *rmap);
- extern int cpu_rmap_put(struct cpu_rmap *rmap);
- 
- extern int cpu_rmap_add(struct cpu_rmap *rmap, void *obj);
-@@ -61,6 +61,7 @@ static inline struct cpu_rmap *alloc_irq_cpu_rmap(unsigned int size)
- }
- extern void free_irq_cpu_rmap(struct cpu_rmap *rmap);
- 
-+int irq_cpu_rmap_remove(struct cpu_rmap *rmap, int irq);
- extern int irq_cpu_rmap_add(struct cpu_rmap *rmap, int irq);
- 
- #endif /* __LINUX_CPU_RMAP_H */
-diff --git a/include/linux/cpu_smt.h b/include/linux/cpu_smt.h
-new file mode 100644
-index 000000000000..0c1664294b57
--- /dev/null
-+++ b/include/linux/cpu_smt.h
-@@ -0,0 +1,33 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_CPU_SMT_H_
-+#define _LINUX_CPU_SMT_H_
-+
-+enum cpuhp_smt_control {
-+	CPU_SMT_ENABLED,
-+	CPU_SMT_DISABLED,
-+	CPU_SMT_FORCE_DISABLED,
-+	CPU_SMT_NOT_SUPPORTED,
-+	CPU_SMT_NOT_IMPLEMENTED,
-+};
-+
-+#if defined(CONFIG_SMP) && defined(CONFIG_HOTPLUG_SMT)
-+extern enum cpuhp_smt_control cpu_smt_control;
-+extern unsigned int cpu_smt_num_threads;
-+extern void cpu_smt_disable(bool force);
-+extern void cpu_smt_set_num_threads(unsigned int num_threads,
-+				    unsigned int max_threads);
-+extern bool cpu_smt_possible(void);
-+extern int cpuhp_smt_enable(void);
-+extern int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval);
-+#else
-+# define cpu_smt_control               (CPU_SMT_NOT_IMPLEMENTED)
-+# define cpu_smt_num_threads 1
-+static inline void cpu_smt_disable(bool force) { }
-+static inline void cpu_smt_set_num_threads(unsigned int num_threads,
-+					   unsigned int max_threads) { }
-+static inline bool cpu_smt_possible(void) { return false; }
-+static inline int cpuhp_smt_enable(void) { return 0; }
-+static inline int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) { return 0; }
-+#endif
-+
-+#endif /* _LINUX_CPU_SMT_H_ */
-diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
-index d5595d57f4e5..0465d1e6f72a 100644
--- a/include/linux/cpufreq.h
-+++ b/include/linux/cpufreq.h
-@@ -15,23 +15,21 @@
- #include <linux/kobject.h>
- #include <linux/notifier.h>
- #include <linux/of.h>
-#include <linux/of_device.h>
- #include <linux/pm_opp.h>
- #include <linux/pm_qos.h>
- #include <linux/spinlock.h>
- #include <linux/sysfs.h>
-+#include <linux/minmax.h>
- 
- /*********************************************************************
-  *                        CPUFREQ INTERFACE                          *
-  *********************************************************************/
- /*
-  * Frequency values here are CPU kHz
- *
- * Maximum transition latency is in nanoseconds - if it's unknown,
- * CPUFREQ_ETERNAL shall be used.
-  */
- 
-#define CPUFREQ_ETERNAL			(-1)
-+#define CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS	NSEC_PER_MSEC
-+
- #define CPUFREQ_NAME_LEN		16
- /* Print length for names. Extra 1 space for accommodating '\n' in prints */
- #define CPUFREQ_NAME_PLEN		(CPUFREQ_NAME_LEN + 1)
-@@ -141,6 +139,12 @@ struct cpufreq_policy {
- 	 */
- 	bool			dvfs_possible_from_any_cpu;
- 
-+	/* Per policy boost enabled flag. */
-+	bool			boost_enabled;
-+
-+	/* Per policy boost supported flag. */
-+	bool			boost_supported;
-+
- 	 /* Cached frequency lookup from cpufreq_driver_resolve_freq. */
- 	unsigned int cached_target_freq;
- 	unsigned int cached_resolved_idx;
-@@ -164,6 +168,12 @@ struct cpufreq_policy {
- 	struct notifier_block nb_max;
- };
- 
-+DEFINE_GUARD(cpufreq_policy_write, struct cpufreq_policy *,
-+	     down_write(&_T->rwsem), up_write(&_T->rwsem))
-+
-+DEFINE_GUARD(cpufreq_policy_read, struct cpufreq_policy *,
-+	     down_read(&_T->rwsem), up_read(&_T->rwsem))
-+
- /*
-  * Used for passing new cpufreq policy data to the cpufreq driver's ->verify()
-  * callback for sanitization.  That callback is only expected to modify the min
-@@ -207,6 +217,9 @@ static inline struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
- static inline void cpufreq_cpu_put(struct cpufreq_policy *policy) { }
- #endif
- 
-+/* Scope based cleanup macro for cpufreq_policy kobject reference counting */
-+DEFINE_FREE(put_cpufreq_policy, struct cpufreq_policy *, if (_T) cpufreq_cpu_put(_T))
-+
- static inline bool policy_is_inactive(struct cpufreq_policy *policy)
- {
- 	return cpumask_empty(policy->cpus);
-@@ -226,9 +239,6 @@ void disable_cpufreq(void);
- 
- u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy);
- 
-struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu);
-void cpufreq_cpu_release(struct cpufreq_policy *policy);
-int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu);
- void refresh_frequency_limits(struct cpufreq_policy *policy);
- void cpufreq_update_policy(unsigned int cpu);
- void cpufreq_update_limits(unsigned int cpu);
-@@ -237,6 +247,13 @@ bool cpufreq_supports_freq_invariance(void);
- struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy);
- void cpufreq_enable_fast_switch(struct cpufreq_policy *policy);
- void cpufreq_disable_fast_switch(struct cpufreq_policy *policy);
-+bool has_target_index(void);
-+
-+DECLARE_PER_CPU(unsigned long, cpufreq_pressure);
-+static inline unsigned long cpufreq_get_pressure(int cpu)
-+{
-+	return READ_ONCE(per_cpu(cpufreq_pressure, cpu));
-+}
- #else
- static inline unsigned int cpufreq_get(unsigned int cpu)
- {
-@@ -259,6 +276,11 @@ static inline bool cpufreq_supports_freq_invariance(void)
- 	return false;
- }
- static inline void disable_cpufreq(void) { }
-+static inline void cpufreq_update_limits(unsigned int cpu) { }
-+static inline unsigned long cpufreq_get_pressure(int cpu)
-+{
-+	return 0;
-+}
- #endif
- 
- #ifdef CONFIG_CPU_FREQ_STAT
-@@ -340,7 +362,10 @@ struct cpufreq_driver {
- 	/*
- 	 * ->fast_switch() replacement for drivers that use an internal
- 	 * representation of performance levels and can pass hints other than
-	 * the target performance level to the hardware.
-+	 * the target performance level to the hardware. This can only be set
-+	 * if ->fast_switch is set too, because in those cases (under specific
-+	 * conditions) scale invariance can be disabled, which causes the
-+	 * schedutil governor to fall back to the latter.
- 	 */
- 	void		(*adjust_perf)(unsigned int cpu,
- 				       unsigned long min_perf,
-@@ -367,18 +392,18 @@ struct cpufreq_driver {
- 	int		(*target_intermediate)(struct cpufreq_policy *policy,
- 					       unsigned int index);
- 
-	/* should be defined, if possible */
-+	/* should be defined, if possible, return 0 on error */
- 	unsigned int	(*get)(unsigned int cpu);
- 
- 	/* Called to update policy limits on firmware notifications. */
-	void		(*update_limits)(unsigned int cpu);
-+	void		(*update_limits)(struct cpufreq_policy *policy);
- 
- 	/* optional */
- 	int		(*bios_limit)(int cpu, unsigned int *limit);
- 
- 	int		(*online)(struct cpufreq_policy *policy);
- 	int		(*offline)(struct cpufreq_policy *policy);
-	int		(*exit)(struct cpufreq_policy *policy);
-+	void		(*exit)(struct cpufreq_policy *policy);
- 	int		(*suspend)(struct cpufreq_policy *policy);
- 	int		(*resume)(struct cpufreq_policy *policy);
- 
-@@ -448,7 +473,7 @@ struct cpufreq_driver {
- #define CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING	BIT(6)
- 
- int cpufreq_register_driver(struct cpufreq_driver *driver_data);
-int cpufreq_unregister_driver(struct cpufreq_driver *driver_data);
-+void cpufreq_unregister_driver(struct cpufreq_driver *driver_data);
- 
- bool cpufreq_driver_test_flags(u16 flags);
- const char *cpufreq_get_current_driver(void);
-@@ -464,17 +489,8 @@ static inline void cpufreq_verify_within_limits(struct cpufreq_policy_data *poli
- 						unsigned int min,
- 						unsigned int max)
- {
-	if (policy->min < min)
-		policy->min = min;
-	if (policy->max < min)
-		policy->max = min;
-	if (policy->min > max)
-		policy->min = max;
-	if (policy->max > max)
-		policy->max = max;
-	if (policy->min > policy->max)
-		policy->min = policy->max;
-	return;
-+	policy->max = clamp(policy->max, min, max);
-+	policy->min = clamp(policy->min, min, policy->max);
- }
- 
- static inline void
-@@ -568,14 +584,6 @@ static inline unsigned long cpufreq_scale(unsigned long old, u_int div,
- #define CPUFREQ_POLICY_POWERSAVE	(1)
- #define CPUFREQ_POLICY_PERFORMANCE	(2)
- 
-/*
- * The polling frequency depends on the capability of the processor. Default
- * polling frequency is 1000 times the transition latency of the processor. The
- * ondemand governor will work on any processor with transition latency <= 10ms,
- * using appropriate sampling rate.
- */
-#define LATENCY_MULTIPLIER		(1000)
-
- struct cpufreq_governor {
- 	char	name[CPUFREQ_NAME_LEN];
- 	int	(*init)(struct cpufreq_policy *policy);
-@@ -640,6 +648,15 @@ module_exit(__governor##_exit)
- struct cpufreq_governor *cpufreq_default_governor(void);
- struct cpufreq_governor *cpufreq_fallback_governor(void);
- 
-+#ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL
-+bool sugov_is_governor(struct cpufreq_policy *policy);
-+#else
-+static inline bool sugov_is_governor(struct cpufreq_policy *policy)
-+{
-+	return false;
-+}
-+#endif
-+
- static inline void cpufreq_policy_apply_limits(struct cpufreq_policy *policy)
- {
- 	if (policy->max < policy->cur)
-@@ -696,26 +713,6 @@ struct cpufreq_frequency_table {
- 				    * order */
- };
- 
-#if defined(CONFIG_CPU_FREQ) && defined(CONFIG_PM_OPP)
-int dev_pm_opp_init_cpufreq_table(struct device *dev,
-				  struct cpufreq_frequency_table **table);
-void dev_pm_opp_free_cpufreq_table(struct device *dev,
-				   struct cpufreq_frequency_table **table);
-#else
-static inline int dev_pm_opp_init_cpufreq_table(struct device *dev,
-						struct cpufreq_frequency_table
-						**table)
-{
-	return -EINVAL;
-}
-
-static inline void dev_pm_opp_free_cpufreq_table(struct device *dev,
-						 struct cpufreq_frequency_table
-						 **table)
-{
-}
-#endif
-
- /*
-  * cpufreq_for_each_entry -	iterate over a cpufreq_frequency_table
-  * @pos:	the cpufreq_frequency_table * to use as a loop cursor.
-@@ -781,26 +778,23 @@ static inline void dev_pm_opp_free_cpufreq_table(struct device *dev,
- 		else
- 
- 
-int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy,
-				    struct cpufreq_frequency_table *table);
-+int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy);
-+
-+int cpufreq_frequency_table_verify(struct cpufreq_policy_data *policy);
- 
-int cpufreq_frequency_table_verify(struct cpufreq_policy_data *policy,
-				   struct cpufreq_frequency_table *table);
- int cpufreq_generic_frequency_table_verify(struct cpufreq_policy_data *policy);
- 
- int cpufreq_table_index_unsorted(struct cpufreq_policy *policy,
-				 unsigned int target_freq,
-				 unsigned int relation);
-+				 unsigned int target_freq, unsigned int min,
-+				 unsigned int max, unsigned int relation);
- int cpufreq_frequency_table_get_index(struct cpufreq_policy *policy,
- 		unsigned int freq);
- 
- ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf);
- 
- #ifdef CONFIG_CPU_FREQ
-int cpufreq_boost_trigger_state(int state);
-int cpufreq_boost_enabled(void);
-int cpufreq_enable_boost_support(void);
-bool policy_has_boost_freq(struct cpufreq_policy *policy);
-+bool cpufreq_boost_enabled(void);
-+int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state);
- 
- /* Find lowest freq at or above target in a table in ascending order */
- static inline int cpufreq_table_find_index_al(struct cpufreq_policy *policy,
-@@ -855,12 +849,12 @@ static inline int cpufreq_table_find_index_dl(struct cpufreq_policy *policy,
- 	return best;
- }
- 
-/* Works only on sorted freq-tables */
-static inline int cpufreq_table_find_index_l(struct cpufreq_policy *policy,
-					     unsigned int target_freq,
-					     bool efficiencies)
-+static inline int find_index_l(struct cpufreq_policy *policy,
-+			       unsigned int target_freq,
-+			       unsigned int min, unsigned int max,
-+			       bool efficiencies)
- {
-	target_freq = clamp_val(target_freq, policy->min, policy->max);
-+	target_freq = clamp_val(target_freq, min, max);
- 
- 	if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING)
- 		return cpufreq_table_find_index_al(policy, target_freq,
-@@ -870,6 +864,14 @@ static inline int cpufreq_table_find_index_l(struct cpufreq_policy *policy,
- 						   efficiencies);
- }
- 
-+/* Works only on sorted freq-tables */
-+static inline int cpufreq_table_find_index_l(struct cpufreq_policy *policy,
-+					     unsigned int target_freq,
-+					     bool efficiencies)
-+{
-+	return find_index_l(policy, target_freq, policy->min, policy->max, efficiencies);
-+}
-+
- /* Find highest freq at or below target in a table in ascending order */
- static inline int cpufreq_table_find_index_ah(struct cpufreq_policy *policy,
- 					      unsigned int target_freq,
-@@ -923,12 +925,12 @@ static inline int cpufreq_table_find_index_dh(struct cpufreq_policy *policy,
- 	return best;
- }
- 
-/* Works only on sorted freq-tables */
-static inline int cpufreq_table_find_index_h(struct cpufreq_policy *policy,
-					     unsigned int target_freq,
-					     bool efficiencies)
-+static inline int find_index_h(struct cpufreq_policy *policy,
-+			       unsigned int target_freq,
-+			       unsigned int min, unsigned int max,
-+			       bool efficiencies)
- {
-	target_freq = clamp_val(target_freq, policy->min, policy->max);
-+	target_freq = clamp_val(target_freq, min, max);
- 
- 	if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING)
- 		return cpufreq_table_find_index_ah(policy, target_freq,
-@@ -938,6 +940,14 @@ static inline int cpufreq_table_find_index_h(struct cpufreq_policy *policy,
- 						   efficiencies);
- }
- 
-+/* Works only on sorted freq-tables */
-+static inline int cpufreq_table_find_index_h(struct cpufreq_policy *policy,
-+					     unsigned int target_freq,
-+					     bool efficiencies)
-+{
-+	return find_index_h(policy, target_freq, policy->min, policy->max, efficiencies);
-+}
-+
- /* Find closest freq to target in a table in ascending order */
- static inline int cpufreq_table_find_index_ac(struct cpufreq_policy *policy,
- 					      unsigned int target_freq,
-@@ -1008,12 +1018,12 @@ static inline int cpufreq_table_find_index_dc(struct cpufreq_policy *policy,
- 	return best;
- }
- 
-/* Works only on sorted freq-tables */
-static inline int cpufreq_table_find_index_c(struct cpufreq_policy *policy,
-					     unsigned int target_freq,
-					     bool efficiencies)
-+static inline int find_index_c(struct cpufreq_policy *policy,
-+			       unsigned int target_freq,
-+			       unsigned int min, unsigned int max,
-+			       bool efficiencies)
- {
-	target_freq = clamp_val(target_freq, policy->min, policy->max);
-+	target_freq = clamp_val(target_freq, min, max);
- 
- 	if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING)
- 		return cpufreq_table_find_index_ac(policy, target_freq,
-@@ -1023,8 +1033,32 @@ static inline int cpufreq_table_find_index_c(struct cpufreq_policy *policy,
- 						   efficiencies);
- }
- 
-+/* Works only on sorted freq-tables */
-+static inline int cpufreq_table_find_index_c(struct cpufreq_policy *policy,
-+					     unsigned int target_freq,
-+					     bool efficiencies)
-+{
-+	return find_index_c(policy, target_freq, policy->min, policy->max, efficiencies);
-+}
-+
-+static inline bool cpufreq_is_in_limits(struct cpufreq_policy *policy,
-+					unsigned int min, unsigned int max,
-+					int idx)
-+{
-+	unsigned int freq;
-+
-+	if (idx < 0)
-+		return false;
-+
-+	freq = policy->freq_table[idx].frequency;
-+
-+	return freq == clamp_val(freq, min, max);
-+}
-+
- static inline int cpufreq_frequency_table_target(struct cpufreq_policy *policy,
- 						 unsigned int target_freq,
-+						 unsigned int min,
-+						 unsigned int max,
- 						 unsigned int relation)
- {
- 	bool efficiencies = policy->efficiencies_available &&
-@@ -1035,28 +1069,26 @@ static inline int cpufreq_frequency_table_target(struct cpufreq_policy *policy,
- 	relation &= ~CPUFREQ_RELATION_E;
- 
- 	if (unlikely(policy->freq_table_sorted == CPUFREQ_TABLE_UNSORTED))
-		return cpufreq_table_index_unsorted(policy, target_freq,
-						    relation);
-+		return cpufreq_table_index_unsorted(policy, target_freq, min,
-+						    max, relation);
- retry:
- 	switch (relation) {
- 	case CPUFREQ_RELATION_L:
-		idx = cpufreq_table_find_index_l(policy, target_freq,
-						 efficiencies);
-+		idx = find_index_l(policy, target_freq, min, max, efficiencies);
- 		break;
- 	case CPUFREQ_RELATION_H:
-		idx = cpufreq_table_find_index_h(policy, target_freq,
-						 efficiencies);
-+		idx = find_index_h(policy, target_freq, min, max, efficiencies);
- 		break;
- 	case CPUFREQ_RELATION_C:
-		idx = cpufreq_table_find_index_c(policy, target_freq,
-						 efficiencies);
-+		idx = find_index_c(policy, target_freq, min, max, efficiencies);
- 		break;
- 	default:
- 		WARN_ON_ONCE(1);
- 		return 0;
- 	}
- 
-	if (idx < 0 && efficiencies) {
-+	/* Limit frequency index to honor min and max */
-+	if (!cpufreq_is_in_limits(policy, min, max, idx) && efficiencies) {
- 		efficiencies = false;
- 		goto retry;
- 	}
-@@ -1110,71 +1142,60 @@ cpufreq_table_set_inefficient(struct cpufreq_policy *policy,
- }
- 
- static inline int parse_perf_domain(int cpu, const char *list_name,
-				    const char *cell_name)
-+				    const char *cell_name,
-+				    struct of_phandle_args *args)
- {
-	struct device_node *cpu_np;
-	struct of_phandle_args args;
- 	int ret;
- 
-	cpu_np = of_cpu_device_node_get(cpu);
-+	struct device_node *cpu_np __free(device_node) = of_cpu_device_node_get(cpu);
- 	if (!cpu_np)
- 		return -ENODEV;
- 
- 	ret = of_parse_phandle_with_args(cpu_np, list_name, cell_name, 0,
-					 &args);
-+					 args);
- 	if (ret < 0)
- 		return ret;
-
-	of_node_put(cpu_np);
-
-	return args.args[0];
-+	return 0;
- }
- 
- static inline int of_perf_domain_get_sharing_cpumask(int pcpu, const char *list_name,
-						     const char *cell_name, struct cpumask *cpumask)
-+						     const char *cell_name, struct cpumask *cpumask,
-+						     struct of_phandle_args *pargs)
- {
-	int target_idx;
- 	int cpu, ret;
-+	struct of_phandle_args args;
- 
-	ret = parse_perf_domain(pcpu, list_name, cell_name);
-+	ret = parse_perf_domain(pcpu, list_name, cell_name, pargs);
- 	if (ret < 0)
- 		return ret;
- 
-	target_idx = ret;
- 	cpumask_set_cpu(pcpu, cpumask);
- 
- 	for_each_possible_cpu(cpu) {
- 		if (cpu == pcpu)
- 			continue;
- 
-		ret = parse_perf_domain(cpu, list_name, cell_name);
-+		ret = parse_perf_domain(cpu, list_name, cell_name, &args);
- 		if (ret < 0)
- 			continue;
- 
-		if (target_idx == ret)
-+		if (of_phandle_args_equal(pargs, &args))
- 			cpumask_set_cpu(cpu, cpumask);
-+
-+		of_node_put(args.np);
- 	}
- 
-	return target_idx;
-}
-#else
-static inline int cpufreq_boost_trigger_state(int state)
-{
-	return 0;
-}
-static inline int cpufreq_boost_enabled(void)
-{
- 	return 0;
- }
-
-static inline int cpufreq_enable_boost_support(void)
-+#else
-+static inline bool cpufreq_boost_enabled(void)
- {
-	return -EINVAL;
-+	return false;
- }
- 
-static inline bool policy_has_boost_freq(struct cpufreq_policy *policy)
-+static inline int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
- {
-	return false;
-+	return -EOPNOTSUPP;
- }
- 
- static inline int
-@@ -1185,21 +1206,14 @@ cpufreq_table_set_inefficient(struct cpufreq_policy *policy,
- }
- 
- static inline int of_perf_domain_get_sharing_cpumask(int pcpu, const char *list_name,
-						     const char *cell_name, struct cpumask *cpumask)
-+						     const char *cell_name, struct cpumask *cpumask,
-+						     struct of_phandle_args *pargs)
- {
- 	return -EOPNOTSUPP;
- }
- #endif
- 
-#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
-void sched_cpufreq_governor_change(struct cpufreq_policy *policy,
-			struct cpufreq_governor *old_gov);
-#else
-static inline void sched_cpufreq_governor_change(struct cpufreq_policy *policy,
-			struct cpufreq_governor *old_gov) { }
-#endif
-
-extern unsigned int arch_freq_get_on_cpu(int cpu);
-+extern int arch_freq_get_on_cpu(int cpu);
- 
- #ifndef arch_set_freq_scale
- static __always_inline
-@@ -1209,10 +1223,10 @@ void arch_set_freq_scale(const struct cpumask *cpus,
- {
- }
- #endif
-+
- /* the following are really really optional */
- extern struct freq_attr cpufreq_freq_attr_scaling_available_freqs;
- extern struct freq_attr cpufreq_freq_attr_scaling_boost_freqs;
-extern struct freq_attr *cpufreq_generic_attr[];
- int cpufreq_table_validate_and_sort(struct cpufreq_policy *policy);
- 
- unsigned int cpufreq_generic_get(unsigned int cpu);
-@@ -1220,6 +1234,8 @@ void cpufreq_generic_init(struct cpufreq_policy *policy,
- 		struct cpufreq_frequency_table *table,
- 		unsigned int transition_latency);
- 
-+bool cpufreq_ready_for_eas(const struct cpumask *cpu_mask);
-+
- static inline void cpufreq_register_em_with_opp(struct cpufreq_policy *policy)
- {
- 	dev_pm_opp_of_register_em(get_cpu_device(policy->cpu),
-diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
-index f61447913db9..62cd7b35a29c 100644
--- a/include/linux/cpuhotplug.h
-+++ b/include/linux/cpuhotplug.h
-@@ -27,7 +27,7 @@
-  * startup callbacks sequentially from CPUHP_OFFLINE + 1 to CPUHP_ONLINE
-  * during a CPU online operation. During a CPU offline operation the
-  * installed teardown callbacks are invoked in the reverse order from
- * CPU_ONLINE - 1 down to CPUHP_OFFLINE.
-+ * CPUHP_ONLINE - 1 down to CPUHP_OFFLINE.
-  *
-  * The state space has three sections: PREPARE, STARTING and ONLINE.
-  *
-@@ -48,7 +48,7 @@
-  *    same section.
-  *
-  * If neither #1 nor #2 apply, please use the dynamic state space when
- * setting up a state by using CPUHP_PREPARE_DYN or CPUHP_PREPARE_ONLINE
-+ * setting up a state by using CPUHP_BP_PREPARE_DYN or CPUHP_AP_ONLINE_DYN
-  * for the @state argument of the setup function.
-  *
-  * See Documentation/core-api/cpu_hotplug.rst for further information and
-@@ -60,20 +60,17 @@ enum cpuhp_state {
- 	/* PREPARE section invoked on a control CPU */
- 	CPUHP_OFFLINE = 0,
- 	CPUHP_CREATE_THREADS,
-	CPUHP_PERF_PREPARE,
- 	CPUHP_PERF_X86_PREPARE,
- 	CPUHP_PERF_X86_AMD_UNCORE_PREP,
- 	CPUHP_PERF_POWER,
- 	CPUHP_PERF_SUPERH,
- 	CPUHP_X86_HPET_DEAD,
-	CPUHP_X86_APB_DEAD,
- 	CPUHP_X86_MCE_DEAD,
- 	CPUHP_VIRT_NET_DEAD,
-+	CPUHP_IBMVNIC_DEAD,
- 	CPUHP_SLUB_DEAD,
- 	CPUHP_DEBUG_OBJ_DEAD,
- 	CPUHP_MM_WRITEBACK_DEAD,
-	/* Must be after CPUHP_MM_VMSTAT_DEAD */
-	CPUHP_MM_DEMOTION_DEAD,
- 	CPUHP_MM_VMSTAT_DEAD,
- 	CPUHP_SOFTIRQ_DEAD,
- 	CPUHP_NET_MVNETA_DEAD,
-@@ -89,14 +86,11 @@ enum cpuhp_state {
- 	CPUHP_FS_BUFF_DEAD,
- 	CPUHP_PRINTK_DEAD,
- 	CPUHP_MM_MEMCQ_DEAD,
-	CPUHP_XFS_DEAD,
- 	CPUHP_PERCPU_CNT_DEAD,
- 	CPUHP_RADIX_DEAD,
- 	CPUHP_PAGE_ALLOC,
- 	CPUHP_NET_DEV_DEAD,
-	CPUHP_PCI_XGENE_DEAD,
- 	CPUHP_IOMMU_IOVA_DEAD,
-	CPUHP_LUSTRE_CFS_DEAD,
- 	CPUHP_AP_ARM_CACHE_B15_RAC_DEAD,
- 	CPUHP_PADATA_DEAD,
- 	CPUHP_AP_DTPM_CPU_DEAD,
-@@ -104,11 +98,9 @@ enum cpuhp_state {
- 	CPUHP_WORKQUEUE_PREP,
- 	CPUHP_POWER_NUMA_PREPARE,
- 	CPUHP_HRTIMERS_PREPARE,
-	CPUHP_PROFILE_PREPARE,
- 	CPUHP_X2APIC_PREPARE,
- 	CPUHP_SMPCFD_PREPARE,
- 	CPUHP_RELAY_PREPARE,
-	CPUHP_SLAB_PREPARE,
- 	CPUHP_MD_RAID5_PREPARE,
- 	CPUHP_RCUTREE_PREP,
- 	CPUHP_CPUIDLE_COUPLED_PREPARE,
-@@ -118,20 +110,19 @@ enum cpuhp_state {
- 	CPUHP_XEN_EVTCHN_PREPARE,
- 	CPUHP_ARM_SHMOBILE_SCU_PREPARE,
- 	CPUHP_SH_SH3X_PREPARE,
-	CPUHP_NET_FLOW_PREPARE,
- 	CPUHP_TOPOLOGY_PREPARE,
- 	CPUHP_NET_IUCV_PREPARE,
- 	CPUHP_ARM_BL_PREPARE,
- 	CPUHP_TRACE_RB_PREPARE,
-	CPUHP_MM_ZS_PREPARE,
-	CPUHP_MM_ZSWP_MEM_PREPARE,
- 	CPUHP_MM_ZSWP_POOL_PREPARE,
- 	CPUHP_KVM_PPC_BOOK3S_PREPARE,
- 	CPUHP_ZCOMP_PREPARE,
- 	CPUHP_TIMERS_PREPARE,
-+	CPUHP_TMIGR_PREPARE,
- 	CPUHP_MIPS_SOC_PREPARE,
- 	CPUHP_BP_PREPARE_DYN,
- 	CPUHP_BP_PREPARE_DYN_END		= CPUHP_BP_PREPARE_DYN + 20,
-+	CPUHP_BP_KICK_AP,
- 	CPUHP_BRINGUP_CPU,
- 
- 	/*
-@@ -140,6 +131,7 @@ enum cpuhp_state {
- 	 */
- 	CPUHP_AP_IDLE_DEAD,
- 	CPUHP_AP_OFFLINE,
-+	CPUHP_AP_CACHECTRL_STARTING,
- 	CPUHP_AP_SCHED_STARTING,
- 	CPUHP_AP_RCUTREE_DYING,
- 	CPUHP_AP_CPU_PM_STARTING,
-@@ -149,19 +141,17 @@ enum cpuhp_state {
- 	CPUHP_AP_IRQ_ARMADA_XP_STARTING,
- 	CPUHP_AP_IRQ_BCM2836_STARTING,
- 	CPUHP_AP_IRQ_MIPS_GIC_STARTING,
-	CPUHP_AP_IRQ_RISCV_STARTING,
-	CPUHP_AP_IRQ_LOONGARCH_STARTING,
-+	CPUHP_AP_IRQ_EIOINTC_STARTING,
-+	CPUHP_AP_IRQ_AVECINTC_STARTING,
- 	CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING,
-+	CPUHP_AP_IRQ_ACLINT_SSWI_STARTING,
-+	CPUHP_AP_IRQ_RISCV_IMSIC_STARTING,
-+	CPUHP_AP_IRQ_RISCV_SBI_IPI_STARTING,
- 	CPUHP_AP_ARM_MVEBU_COHERENCY,
-	CPUHP_AP_MICROCODE_LOADER,
- 	CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
- 	CPUHP_AP_PERF_X86_STARTING,
- 	CPUHP_AP_PERF_X86_AMD_IBS_STARTING,
-	CPUHP_AP_PERF_X86_CQM_STARTING,
-	CPUHP_AP_PERF_X86_CSTATE_STARTING,
- 	CPUHP_AP_PERF_XTENSA_STARTING,
-	CPUHP_AP_MIPS_OP_LOONGSON3_STARTING,
-	CPUHP_AP_ARM_SDEI_STARTING,
- 	CPUHP_AP_ARM_VFP_STARTING,
- 	CPUHP_AP_ARM64_DEBUG_MONITORS_STARTING,
- 	CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING,
-@@ -171,31 +161,32 @@ enum cpuhp_state {
- 	CPUHP_AP_ARM_L2X0_STARTING,
- 	CPUHP_AP_EXYNOS4_MCT_TIMER_STARTING,
- 	CPUHP_AP_ARM_ARCH_TIMER_STARTING,
-+	CPUHP_AP_ARM_ARCH_TIMER_EVTSTRM_STARTING,
- 	CPUHP_AP_ARM_GLOBAL_TIMER_STARTING,
- 	CPUHP_AP_JCORE_TIMER_STARTING,
- 	CPUHP_AP_ARM_TWD_STARTING,
- 	CPUHP_AP_QCOM_TIMER_STARTING,
- 	CPUHP_AP_TEGRA_TIMER_STARTING,
- 	CPUHP_AP_ARMADA_TIMER_STARTING,
-	CPUHP_AP_MARCO_TIMER_STARTING,
-+	CPUHP_AP_LOONGARCH_ARCH_TIMER_STARTING,
- 	CPUHP_AP_MIPS_GIC_TIMER_STARTING,
- 	CPUHP_AP_ARC_TIMER_STARTING,
-+	CPUHP_AP_REALTEK_TIMER_STARTING,
- 	CPUHP_AP_RISCV_TIMER_STARTING,
- 	CPUHP_AP_CLINT_TIMER_STARTING,
- 	CPUHP_AP_CSKY_TIMER_STARTING,
- 	CPUHP_AP_TI_GP_TIMER_STARTING,
- 	CPUHP_AP_HYPERV_TIMER_STARTING,
-	CPUHP_AP_KVM_STARTING,
-	CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING,
-	CPUHP_AP_KVM_ARM_VGIC_STARTING,
-	CPUHP_AP_KVM_ARM_TIMER_STARTING,
- 	/* Must be the last timer callback */
- 	CPUHP_AP_DUMMY_TIMER_STARTING,
- 	CPUHP_AP_ARM_XEN_STARTING,
-+	CPUHP_AP_ARM_XEN_RUNSTATE_STARTING,
- 	CPUHP_AP_ARM_CORESIGHT_STARTING,
- 	CPUHP_AP_ARM_CORESIGHT_CTI_STARTING,
- 	CPUHP_AP_ARM64_ISNDEP_STARTING,
- 	CPUHP_AP_SMPCFD_DYING,
-+	CPUHP_AP_HRTIMERS_DYING,
-+	CPUHP_AP_TICK_DYING,
- 	CPUHP_AP_X86_TBOOT_DYING,
- 	CPUHP_AP_ARM_CACHE_B15_RAC_DYING,
- 	CPUHP_AP_ONLINE,
-@@ -203,9 +194,10 @@ enum cpuhp_state {
- 
- 	/* Online section invoked on the hotplugged CPU from the hotplug thread */
- 	CPUHP_AP_ONLINE_IDLE,
-+	CPUHP_AP_HYPERV_ONLINE,
-+	CPUHP_AP_KVM_ONLINE,
- 	CPUHP_AP_SCHED_WAIT_EMPTY,
- 	CPUHP_AP_SMPBOOT_THREADS,
-	CPUHP_AP_X86_VDSO_VMA_ONLINE,
- 	CPUHP_AP_IRQ_AFFINITY_ONLINE,
- 	CPUHP_AP_BLK_MQ_ONLINE,
- 	CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS,
-@@ -215,10 +207,6 @@ enum cpuhp_state {
- 	CPUHP_AP_PERF_X86_UNCORE_ONLINE,
- 	CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
- 	CPUHP_AP_PERF_X86_AMD_POWER_ONLINE,
-	CPUHP_AP_PERF_X86_RAPL_ONLINE,
-	CPUHP_AP_PERF_X86_CQM_ONLINE,
-	CPUHP_AP_PERF_X86_CSTATE_ONLINE,
-	CPUHP_AP_PERF_X86_IDXD_ONLINE,
- 	CPUHP_AP_PERF_S390_CF_ONLINE,
- 	CPUHP_AP_PERF_S390_SF_ONLINE,
- 	CPUHP_AP_PERF_ARM_CCI_ONLINE,
-@@ -237,6 +225,7 @@ enum cpuhp_state {
- 	CPUHP_AP_PERF_ARM_APM_XGENE_ONLINE,
- 	CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE,
- 	CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
-+	CPUHP_AP_PERF_ARM_MRVL_PEM_ONLINE,
- 	CPUHP_AP_PERF_POWERPC_NEST_IMC_ONLINE,
- 	CPUHP_AP_PERF_POWERPC_CORE_IMC_ONLINE,
- 	CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE,
-@@ -244,15 +233,15 @@ enum cpuhp_state {
- 	CPUHP_AP_PERF_POWERPC_HV_24x7_ONLINE,
- 	CPUHP_AP_PERF_POWERPC_HV_GPCI_ONLINE,
- 	CPUHP_AP_PERF_CSKY_ONLINE,
-+	CPUHP_AP_TMIGR_ONLINE,
- 	CPUHP_AP_WATCHDOG_ONLINE,
- 	CPUHP_AP_WORKQUEUE_ONLINE,
- 	CPUHP_AP_RANDOM_ONLINE,
- 	CPUHP_AP_RCUTREE_ONLINE,
-+	CPUHP_AP_KTHREADS_ONLINE,
- 	CPUHP_AP_BASE_CACHEINFO_ONLINE,
- 	CPUHP_AP_ONLINE_DYN,
-	CPUHP_AP_ONLINE_DYN_END		= CPUHP_AP_ONLINE_DYN + 30,
-	/* Must be after CPUHP_AP_ONLINE_DYN for node_states[N_CPU] update */
-	CPUHP_AP_MM_DEMOTION_ONLINE,
-+	CPUHP_AP_ONLINE_DYN_END		= CPUHP_AP_ONLINE_DYN + 40,
- 	CPUHP_AP_X86_HPET_ONLINE,
- 	CPUHP_AP_X86_KVM_CLK_ONLINE,
- 	CPUHP_AP_ACTIVE,
-@@ -519,4 +508,20 @@ void cpuhp_online_idle(enum cpuhp_state state);
- static inline void cpuhp_online_idle(enum cpuhp_state state) { }
- #endif
- 
-+struct task_struct;
-+
-+void cpuhp_ap_sync_alive(void);
-+void arch_cpuhp_sync_state_poll(void);
-+void arch_cpuhp_cleanup_kick_cpu(unsigned int cpu);
-+int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle);
-+bool arch_cpuhp_init_parallel_bringup(void);
-+
-+#ifdef CONFIG_HOTPLUG_CORE_SYNC_DEAD
-+void cpuhp_ap_report_dead(void);
-+void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu);
-+#else
-+static inline void cpuhp_ap_report_dead(void) { }
-+static inline void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) { }
-+#endif
-+
- #endif
-diff --git a/include/linux/cpuhplock.h b/include/linux/cpuhplock.h
-new file mode 100644
-index 000000000000..f7aa20f62b87
--- /dev/null
-+++ b/include/linux/cpuhplock.h
-@@ -0,0 +1,49 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * include/linux/cpuhplock.h - CPU hotplug locking
-+ *
-+ * Locking functions for CPU hotplug.
-+ */
-+#ifndef _LINUX_CPUHPLOCK_H_
-+#define _LINUX_CPUHPLOCK_H_
-+
-+#include <linux/cleanup.h>
-+#include <linux/errno.h>
-+
-+struct device;
-+
-+extern int lockdep_is_cpus_held(void);
-+
-+#ifdef CONFIG_HOTPLUG_CPU
-+void cpus_write_lock(void);
-+void cpus_write_unlock(void);
-+void cpus_read_lock(void);
-+void cpus_read_unlock(void);
-+int  cpus_read_trylock(void);
-+void lockdep_assert_cpus_held(void);
-+void cpu_hotplug_disable_offlining(void);
-+void cpu_hotplug_disable(void);
-+void cpu_hotplug_enable(void);
-+void clear_tasks_mm_cpumask(int cpu);
-+int remove_cpu(unsigned int cpu);
-+int cpu_device_down(struct device *dev);
-+void smp_shutdown_nonboot_cpus(unsigned int primary_cpu);
-+
-+#else /* CONFIG_HOTPLUG_CPU */
-+
-+static inline void cpus_write_lock(void) { }
-+static inline void cpus_write_unlock(void) { }
-+static inline void cpus_read_lock(void) { }
-+static inline void cpus_read_unlock(void) { }
-+static inline int  cpus_read_trylock(void) { return true; }
-+static inline void lockdep_assert_cpus_held(void) { }
-+static inline void cpu_hotplug_disable_offlining(void) { }
-+static inline void cpu_hotplug_disable(void) { }
-+static inline void cpu_hotplug_enable(void) { }
-+static inline int remove_cpu(unsigned int cpu) { return -EPERM; }
-+static inline void smp_shutdown_nonboot_cpus(unsigned int primary_cpu) { }
-+#endif	/* !CONFIG_HOTPLUG_CPU */
-+
-+DEFINE_LOCK_GUARD_0(cpus_read_lock, cpus_read_lock(), cpus_read_unlock())
-+
-+#endif /* _LINUX_CPUHPLOCK_H_ */
-diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h
-index fce476275e16..4073690504a7 100644
--- a/include/linux/cpuidle.h
-+++ b/include/linux/cpuidle.h
-@@ -14,6 +14,7 @@
- #include <linux/percpu.h>
- #include <linux/list.h>
- #include <linux/hrtimer.h>
-+#include <linux/context_tracking.h>
- 
- #define CPUIDLE_STATE_MAX	10
- #define CPUIDLE_NAME_LEN	16
-@@ -60,7 +61,7 @@ struct cpuidle_state {
- 			struct cpuidle_driver *drv,
- 			int index);
- 
-	int (*enter_dead) (struct cpuidle_device *dev, int index);
-+	void (*enter_dead) (struct cpuidle_device *dev, int index);
- 
- 	/*
- 	 * CPUs execute ->enter_s2idle with the local tick or entire timekeeping
-@@ -115,6 +116,35 @@ struct cpuidle_device {
- DECLARE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
- DECLARE_PER_CPU(struct cpuidle_device, cpuidle_dev);
- 
-+static __always_inline void ct_cpuidle_enter(void)
-+{
-+	lockdep_assert_irqs_disabled();
-+	/*
-+	 * Idle is allowed to (temporary) enable IRQs. It
-+	 * will return with IRQs disabled.
-+	 *
-+	 * Trace IRQs enable here, then switch off RCU, and have
-+	 * arch_cpu_idle() use raw_local_irq_enable(). Note that
-+	 * ct_idle_enter() relies on lockdep IRQ state, so switch that
-+	 * last -- this is very similar to the entry code.
-+	 */
-+	trace_hardirqs_on_prepare();
-+	lockdep_hardirqs_on_prepare();
-+	instrumentation_end();
-+	ct_idle_enter();
-+	lockdep_hardirqs_on(_RET_IP_);
-+}
-+
-+static __always_inline void ct_cpuidle_exit(void)
-+{
-+	/*
-+	 * Carefully undo the above.
-+	 */
-+	lockdep_hardirqs_off(_RET_IP_);
-+	ct_idle_exit();
-+	instrumentation_begin();
-+}
-+
- /****************************
-  * CPUIDLE DRIVER INTERFACE *
-  ****************************/
-@@ -218,7 +248,8 @@ extern int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
- 				      struct cpuidle_device *dev,
- 				      u64 latency_limit_ns);
- extern int cpuidle_enter_s2idle(struct cpuidle_driver *drv,
-				struct cpuidle_device *dev);
-+				struct cpuidle_device *dev,
-+				u64 latency_limit_ns);
- extern void cpuidle_use_deepest_state(u64 latency_limit_ns);
- #else
- static inline int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
-@@ -226,7 +257,8 @@ static inline int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
- 					     u64 latency_limit_ns)
- {return -ENODEV; }
- static inline int cpuidle_enter_s2idle(struct cpuidle_driver *drv,
-				       struct cpuidle_device *dev)
-+				       struct cpuidle_device *dev,
-+				       u64 latency_limit_ns)
- {return -ENODEV; }
- static inline void cpuidle_use_deepest_state(u64 latency_limit_ns)
- {
-@@ -277,7 +309,7 @@ extern s64 cpuidle_governor_latency_req(unsigned int cpu);
- #define __CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter,			\
- 				idx,					\
- 				state,					\
-				is_retention)				\
-+				is_retention, is_rcu)			\
- ({									\
- 	int __ret = 0;							\
- 									\
-@@ -289,7 +321,11 @@ extern s64 cpuidle_governor_latency_req(unsigned int cpu);
- 	if (!is_retention)						\
- 		__ret =  cpu_pm_enter();				\
- 	if (!__ret) {							\
-+		if (!is_rcu)						\
-+			ct_cpuidle_enter();				\
- 		__ret = low_level_idle_enter(state);			\
-+		if (!is_rcu)						\
-+			ct_cpuidle_exit();				\
- 		if (!is_retention)					\
- 			cpu_pm_exit();					\
- 	}								\
-@@ -298,15 +334,21 @@ extern s64 cpuidle_governor_latency_req(unsigned int cpu);
- })
- 
- #define CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter, idx)	\
-	__CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter, idx, idx, 0)
-+	__CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter, idx, idx, 0, 0)
- 
- #define CPU_PM_CPU_IDLE_ENTER_RETENTION(low_level_idle_enter, idx)	\
-	__CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter, idx, idx, 1)
-+	__CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter, idx, idx, 1, 0)
- 
- #define CPU_PM_CPU_IDLE_ENTER_PARAM(low_level_idle_enter, idx, state)	\
-	__CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter, idx, state, 0)
-+	__CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter, idx, state, 0, 0)
-+
-+#define CPU_PM_CPU_IDLE_ENTER_PARAM_RCU(low_level_idle_enter, idx, state)	\
-+	__CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter, idx, state, 0, 1)
- 
- #define CPU_PM_CPU_IDLE_ENTER_RETENTION_PARAM(low_level_idle_enter, idx, state)	\
-	__CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter, idx, state, 1)
-+	__CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter, idx, state, 1, 0)
-+
-+#define CPU_PM_CPU_IDLE_ENTER_RETENTION_PARAM_RCU(low_level_idle_enter, idx, state)	\
-+	__CPU_PM_CPU_IDLE_ENTER(low_level_idle_enter, idx, state, 1, 1)
- 
- #endif /* _LINUX_CPUIDLE_H */
-diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
-index bd047864c7ac..80211900f373 100644
--- a/include/linux/cpumask.h
-+++ b/include/linux/cpumask.h
-@@ -4,28 +4,19 @@
- 
- /*
-  * Cpumasks provide a bitmap suitable for representing the
- * set of CPU's in a system, one bit position per CPU number.  In general,
-+ * set of CPUs in a system, one bit position per CPU number.  In general,
-  * only nr_cpu_ids (<= NR_CPUS) bits are valid.
-  */
-#include <linux/kernel.h>
-#include <linux/threads.h>
-#include <linux/bitmap.h>
- #include <linux/atomic.h>
-#include <linux/bug.h>
-+#include <linux/bitmap.h>
-+#include <linux/cleanup.h>
-+#include <linux/cpumask_types.h>
- #include <linux/gfp_types.h>
- #include <linux/numa.h>
-+#include <linux/threads.h>
-+#include <linux/types.h>
- 
-/* Don't assign or return these: may not be this big! */
-typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
-
-/**
- * cpumask_bits - get the bits in a cpumask
- * @maskp: the struct cpumask *
- *
- * You should only assume nr_cpu_ids bits of this mask are valid.  This is
- * a macro so it's const-correct.
- */
-#define cpumask_bits(maskp) ((maskp)->bits)
-+#include <asm/bug.h>
- 
- /**
-  * cpumask_pr_args - printf args to output a cpumask
-@@ -35,19 +26,56 @@ typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
-  */
- #define cpumask_pr_args(maskp)		nr_cpu_ids, cpumask_bits(maskp)
- 
-#if NR_CPUS == 1
-#define nr_cpu_ids		1U
-+#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
-+#define nr_cpu_ids ((unsigned int)NR_CPUS)
- #else
- extern unsigned int nr_cpu_ids;
- #endif
- 
-#ifdef CONFIG_CPUMASK_OFFSTACK
-/* Assuming NR_CPUS is huge, a runtime limit is more efficient.  Also,
- * not all bits may be allocated. */
-#define nr_cpumask_bits	nr_cpu_ids
-+static __always_inline void set_nr_cpu_ids(unsigned int nr)
-+{
-+#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
-+	WARN_ON(nr != nr_cpu_ids);
- #else
-#define nr_cpumask_bits	((unsigned int)NR_CPUS)
-+	nr_cpu_ids = nr;
- #endif
-+}
-+
-+/*
-+ * We have several different "preferred sizes" for the cpumask
-+ * operations, depending on operation.
-+ *
-+ * For example, the bitmap scanning and operating operations have
-+ * optimized routines that work for the single-word case, but only when
-+ * the size is constant. So if NR_CPUS fits in one single word, we are
-+ * better off using that small constant, in order to trigger the
-+ * optimized bit finding. That is 'small_cpumask_size'.
-+ *
-+ * The clearing and copying operations will similarly perform better
-+ * with a constant size, but we limit that size arbitrarily to four
-+ * words. We call this 'large_cpumask_size'.
-+ *
-+ * Finally, some operations just want the exact limit, either because
-+ * they set bits or just don't have any faster fixed-sized versions. We
-+ * call this just 'nr_cpumask_bits'.
-+ *
-+ * Note that these optional constants are always guaranteed to be at
-+ * least as big as 'nr_cpu_ids' itself is, and all our cpumask
-+ * allocations are at least that size (see cpumask_size()). The
-+ * optimization comes from being able to potentially use a compile-time
-+ * constant instead of a run-time generated exact number of CPUs.
-+ */
-+#if NR_CPUS <= BITS_PER_LONG
-+  #define small_cpumask_bits ((unsigned int)NR_CPUS)
-+  #define large_cpumask_bits ((unsigned int)NR_CPUS)
-+#elif NR_CPUS <= 4*BITS_PER_LONG
-+  #define small_cpumask_bits nr_cpu_ids
-+  #define large_cpumask_bits ((unsigned int)NR_CPUS)
-+#else
-+  #define small_cpumask_bits nr_cpu_ids
-+  #define large_cpumask_bits nr_cpu_ids
-+#endif
-+#define nr_cpumask_bits nr_cpu_ids
- 
- /*
-  * The following particular system cpumasks and operations manage
-@@ -55,22 +83,19 @@ extern unsigned int nr_cpu_ids;
-  *
-  *     cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
-  *     cpu_present_mask - has bit 'cpu' set iff cpu is populated
-+ *     cpu_enabled_mask - has bit 'cpu' set iff cpu can be brought online
-  *     cpu_online_mask  - has bit 'cpu' set iff cpu available to scheduler
-  *     cpu_active_mask  - has bit 'cpu' set iff cpu available to migration
-  *
-  *  If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
-  *
- *  The cpu_possible_mask is fixed at boot time, as the set of CPU id's
-+ *  The cpu_possible_mask is fixed at boot time, as the set of CPU IDs
-  *  that it is possible might ever be plugged in at anytime during the
-  *  life of that system boot.  The cpu_present_mask is dynamic(*),
-  *  representing which CPUs are currently plugged in.  And
-  *  cpu_online_mask is the dynamic subset of cpu_present_mask,
-  *  indicating those CPUs available for scheduling.
-  *
- *  If HOTPLUG is enabled, then cpu_possible_mask is forced to have
- *  all NR_CPUS bits set, otherwise it is just the set of CPUs that
- *  ACPI reports present at boot.
- *
-  *  If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
-  *  depending on what ACPI reports as currently plugged in, otherwise
-  *  cpu_present_mask is just a copy of cpu_possible_mask.
-@@ -79,7 +104,7 @@ extern unsigned int nr_cpu_ids;
-  *      hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
-  *
-  * Subtleties:
- * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
-+ * 1) UP ARCHes (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
-  *    assumption that their single CPU is online.  The UP
-  *    cpu_{online,possible,present}_masks are placebos.  Changing them
-  *    will have no useful affect on the following num_*_cpus()
-@@ -91,16 +116,19 @@ extern unsigned int nr_cpu_ids;
- 
- extern struct cpumask __cpu_possible_mask;
- extern struct cpumask __cpu_online_mask;
-+extern struct cpumask __cpu_enabled_mask;
- extern struct cpumask __cpu_present_mask;
- extern struct cpumask __cpu_active_mask;
- extern struct cpumask __cpu_dying_mask;
- #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
- #define cpu_online_mask   ((const struct cpumask *)&__cpu_online_mask)
-+#define cpu_enabled_mask   ((const struct cpumask *)&__cpu_enabled_mask)
- #define cpu_present_mask  ((const struct cpumask *)&__cpu_present_mask)
- #define cpu_active_mask   ((const struct cpumask *)&__cpu_active_mask)
- #define cpu_dying_mask    ((const struct cpumask *)&__cpu_dying_mask)
- 
- extern atomic_t __num_online_cpus;
-+extern unsigned int __num_possible_cpus;
- 
- extern cpumask_t cpus_booted_once_mask;
- 
-@@ -114,7 +142,7 @@ static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bit
- /* verify cpu argument to cpumask_* operators */
- static __always_inline unsigned int cpumask_check(unsigned int cpu)
- {
-	cpu_max_bits_warn(cpu, nr_cpumask_bits);
-+	cpu_max_bits_warn(cpu, small_cpumask_bits);
- 	return cpu;
- }
- 
-@@ -122,93 +150,127 @@ static __always_inline unsigned int cpumask_check(unsigned int cpu)
-  * cpumask_first - get the first cpu in a cpumask
-  * @srcp: the cpumask pointer
-  *
- * Returns >= nr_cpu_ids if no cpus set.
-+ * Return: >= nr_cpu_ids if no cpus set.
-  */
-static inline unsigned int cpumask_first(const struct cpumask *srcp)
-+static __always_inline unsigned int cpumask_first(const struct cpumask *srcp)
- {
-	return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
-+	return find_first_bit(cpumask_bits(srcp), small_cpumask_bits);
- }
- 
- /**
-  * cpumask_first_zero - get the first unset cpu in a cpumask
-  * @srcp: the cpumask pointer
-  *
- * Returns >= nr_cpu_ids if all cpus are set.
-+ * Return: >= nr_cpu_ids if all cpus are set.
-  */
-static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
-+static __always_inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
- {
-	return find_first_zero_bit(cpumask_bits(srcp), nr_cpumask_bits);
-+	return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits);
- }
- 
- /**
-  * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
- * @src1p: the first input
- * @src2p: the second input
-+ * @srcp1: the first input
-+ * @srcp2: the second input
-  *
- * Returns >= nr_cpu_ids if no cpus set in both.  See also cpumask_next_and().
-+ * Return: >= nr_cpu_ids if no cpus set in both.  See also cpumask_next_and().
-  */
-static inline
-+static __always_inline
- unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
- {
-	return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), nr_cpumask_bits);
-+	return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
-+}
-+
-+/**
-+ * cpumask_first_andnot - return the first cpu from *srcp1 & ~*srcp2
-+ * @srcp1: the first input
-+ * @srcp2: the second input
-+ *
-+ * Return: >= nr_cpu_ids if no such cpu found.
-+ */
-+static __always_inline
-+unsigned int cpumask_first_andnot(const struct cpumask *srcp1, const struct cpumask *srcp2)
-+{
-+	return find_first_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
-+}
-+
-+/**
-+ * cpumask_first_and_and - return the first cpu from *srcp1 & *srcp2 & *srcp3
-+ * @srcp1: the first input
-+ * @srcp2: the second input
-+ * @srcp3: the third input
-+ *
-+ * Return: >= nr_cpu_ids if no cpus set in all.
-+ */
-+static __always_inline
-+unsigned int cpumask_first_and_and(const struct cpumask *srcp1,
-+				   const struct cpumask *srcp2,
-+				   const struct cpumask *srcp3)
-+{
-+	return find_first_and_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
-+				      cpumask_bits(srcp3), small_cpumask_bits);
- }
- 
- /**
-  * cpumask_last - get the last CPU in a cpumask
-  * @srcp:	- the cpumask pointer
-  *
- * Returns	>= nr_cpumask_bits if no CPUs set.
-+ * Return:	>= nr_cpumask_bits if no CPUs set.
-  */
-static inline unsigned int cpumask_last(const struct cpumask *srcp)
-+static __always_inline unsigned int cpumask_last(const struct cpumask *srcp)
- {
-	return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits);
-+	return find_last_bit(cpumask_bits(srcp), small_cpumask_bits);
- }
- 
- /**
-  * cpumask_next - get the next cpu in a cpumask
- * @n: the cpu prior to the place to search (ie. return will be > @n)
-+ * @n: the cpu prior to the place to search (i.e. return will be > @n)
-  * @srcp: the cpumask pointer
-  *
- * Returns >= nr_cpu_ids if no further cpus set.
-+ * Return: >= nr_cpu_ids if no further cpus set.
-  */
-static inline
-+static __always_inline
- unsigned int cpumask_next(int n, const struct cpumask *srcp)
- {
- 	/* -1 is a legal arg here. */
- 	if (n != -1)
- 		cpumask_check(n);
-	return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1);
-+	return find_next_bit(cpumask_bits(srcp), small_cpumask_bits, n + 1);
- }
- 
- /**
-  * cpumask_next_zero - get the next unset cpu in a cpumask
- * @n: the cpu prior to the place to search (ie. return will be > @n)
-+ * @n: the cpu prior to the place to search (i.e. return will be > @n)
-  * @srcp: the cpumask pointer
-  *
- * Returns >= nr_cpu_ids if no further cpus unset.
-+ * Return: >= nr_cpu_ids if no further cpus unset.
-  */
-static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
-+static __always_inline
-+unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
- {
- 	/* -1 is a legal arg here. */
- 	if (n != -1)
- 		cpumask_check(n);
-	return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
-+	return find_next_zero_bit(cpumask_bits(srcp), small_cpumask_bits, n+1);
- }
- 
- #if NR_CPUS == 1
- /* Uniprocessor: there is only one valid CPU */
-static inline unsigned int cpumask_local_spread(unsigned int i, int node)
-+static __always_inline
-+unsigned int cpumask_local_spread(unsigned int i, int node)
- {
- 	return 0;
- }
- 
-static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
-						      const struct cpumask *src2p)
-+static __always_inline
-+unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
-+					const struct cpumask *src2p)
- {
- 	return cpumask_first_and(src1p, src2p);
- }
- 
-static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp)
-+static __always_inline
-+unsigned int cpumask_any_distribute(const struct cpumask *srcp)
- {
- 	return cpumask_first(srcp);
- }
-@@ -221,67 +283,101 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp);
- 
- /**
-  * cpumask_next_and - get the next cpu in *src1p & *src2p
- * @n: the cpu prior to the place to search (ie. return will be > @n)
-+ * @n: the cpu prior to the place to search (i.e. return will be > @n)
-  * @src1p: the first cpumask pointer
-  * @src2p: the second cpumask pointer
-  *
- * Returns >= nr_cpu_ids if no further cpus set in both.
-+ * Return: >= nr_cpu_ids if no further cpus set in both.
-  */
-static inline
-+static __always_inline
- unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
-		     const struct cpumask *src2p)
-+			      const struct cpumask *src2p)
- {
- 	/* -1 is a legal arg here. */
- 	if (n != -1)
- 		cpumask_check(n);
- 	return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p),
-		nr_cpumask_bits, n + 1);
-+		small_cpumask_bits, n + 1);
- }
- 
- /**
- * for_each_cpu - iterate over every cpu in a mask
- * @cpu: the (optionally unsigned) integer iterator
- * @mask: the cpumask pointer
-+ * cpumask_next_andnot - get the next cpu in *src1p & ~*src2p
-+ * @n: the cpu prior to the place to search (i.e. return will be > @n)
-+ * @src1p: the first cpumask pointer
-+ * @src2p: the second cpumask pointer
-  *
- * After the loop, cpu is >= nr_cpu_ids.
-+ * Return: >= nr_cpu_ids if no further cpus set in both.
-  */
-#define for_each_cpu(cpu, mask)				\
-	for ((cpu) = -1;				\
-		(cpu) = cpumask_next((cpu), (mask)),	\
-		(cpu) < nr_cpu_ids;)
-+static __always_inline
-+unsigned int cpumask_next_andnot(int n, const struct cpumask *src1p,
-+				 const struct cpumask *src2p)
-+{
-+	/* -1 is a legal arg here. */
-+	if (n != -1)
-+		cpumask_check(n);
-+	return find_next_andnot_bit(cpumask_bits(src1p), cpumask_bits(src2p),
-+		small_cpumask_bits, n + 1);
-+}
- 
- /**
- * for_each_cpu_not - iterate over every cpu in a complemented mask
- * @cpu: the (optionally unsigned) integer iterator
- * @mask: the cpumask pointer
-+ * cpumask_next_and_wrap - get the next cpu in *src1p & *src2p, starting from
-+ *			   @n+1. If nothing found, wrap around and start from
-+ *			   the beginning
-+ * @n: the cpu prior to the place to search (i.e. search starts from @n+1)
-+ * @src1p: the first cpumask pointer
-+ * @src2p: the second cpumask pointer
-  *
- * After the loop, cpu is >= nr_cpu_ids.
-+ * Return: next set bit, wrapped if needed, or >= nr_cpu_ids if @src1p & @src2p is empty.
-  */
-#define for_each_cpu_not(cpu, mask)				\
-	for ((cpu) = -1;					\
-		(cpu) = cpumask_next_zero((cpu), (mask)),	\
-		(cpu) < nr_cpu_ids;)
-
-#if NR_CPUS == 1
-static inline
-unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
-+static __always_inline
-+unsigned int cpumask_next_and_wrap(int n, const struct cpumask *src1p,
-+			      const struct cpumask *src2p)
- {
-	cpumask_check(start);
-+	/* -1 is a legal arg here. */
- 	if (n != -1)
- 		cpumask_check(n);
-+	return find_next_and_bit_wrap(cpumask_bits(src1p), cpumask_bits(src2p),
-+		small_cpumask_bits, n + 1);
-+}
- 
-	/*
-	 * Return the first available CPU when wrapping, or when starting before cpu0,
-	 * since there is only one valid option.
-	 */
-	if (wrap && n >= 0)
-		return nr_cpumask_bits;
-+/**
-+ * cpumask_next_wrap - get the next cpu in *src, starting from @n+1. If nothing
-+ *		       found, wrap around and start from the beginning
-+ * @n: the cpu prior to the place to search (i.e. search starts from @n+1)
-+ * @src: cpumask pointer
-+ *
-+ * Return: next set bit, wrapped if needed, or >= nr_cpu_ids if @src is empty.
-+ */
-+static __always_inline
-+unsigned int cpumask_next_wrap(int n, const struct cpumask *src)
-+{
-+	/* -1 is a legal arg here. */
-+	if (n != -1)
-+		cpumask_check(n);
-+	return find_next_bit_wrap(cpumask_bits(src), small_cpumask_bits, n + 1);
-+}
- 
-	return cpumask_first(mask);
-+/**
-+ * cpumask_random - get random cpu in *src.
-+ * @src: cpumask pointer
-+ *
-+ * Return: random set bit, or >= nr_cpu_ids if @src is empty.
-+ */
-+static __always_inline
-+unsigned int cpumask_random(const struct cpumask *src)
-+{
-+	return find_random_bit(cpumask_bits(src), nr_cpu_ids);
- }
-#else
-unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
-#endif
-+
-+/**
-+ * for_each_cpu - iterate over every cpu in a mask
-+ * @cpu: the (optionally unsigned) integer iterator
-+ * @mask: the cpumask pointer
-+ *
-+ * After the loop, cpu is >= nr_cpu_ids.
-+ */
-+#define for_each_cpu(cpu, mask)				\
-+	for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
- 
- /**
-  * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
-@@ -293,10 +389,8 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta
-  *
-  * After the loop, cpu is >= nr_cpu_ids.
-  */
-#define for_each_cpu_wrap(cpu, mask, start)					\
-	for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false);	\
-	     (cpu) < nr_cpumask_bits;						\
-	     (cpu) = cpumask_next_wrap((cpu), (mask), (start), true))
-+#define for_each_cpu_wrap(cpu, mask, start)				\
-+	for_each_set_bit_wrap(cpu, cpumask_bits(mask), small_cpumask_bits, start)
- 
- /**
-  * for_each_cpu_and - iterate over every cpu in both masks
-@@ -313,30 +407,180 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta
-  * After the loop, cpu is >= nr_cpu_ids.
-  */
- #define for_each_cpu_and(cpu, mask1, mask2)				\
-	for ((cpu) = -1;						\
-		(cpu) = cpumask_next_and((cpu), (mask1), (mask2)),	\
-		(cpu) < nr_cpu_ids;)
-+	for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
-+
-+/**
-+ * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
-+ *			 those present in another.
-+ * @cpu: the (optionally unsigned) integer iterator
-+ * @mask1: the first cpumask pointer
-+ * @mask2: the second cpumask pointer
-+ *
-+ * This saves a temporary CPU mask in many places.  It is equivalent to:
-+ *	struct cpumask tmp;
-+ *	cpumask_andnot(&tmp, &mask1, &mask2);
-+ *	for_each_cpu(cpu, &tmp)
-+ *		...
-+ *
-+ * After the loop, cpu is >= nr_cpu_ids.
-+ */
-+#define for_each_cpu_andnot(cpu, mask1, mask2)				\
-+	for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
- 
- /**
- * cpumask_any_but - return a "random" in a cpumask, but not this one.
-+ * for_each_cpu_or - iterate over every cpu present in either mask
-+ * @cpu: the (optionally unsigned) integer iterator
-+ * @mask1: the first cpumask pointer
-+ * @mask2: the second cpumask pointer
-+ *
-+ * This saves a temporary CPU mask in many places.  It is equivalent to:
-+ *	struct cpumask tmp;
-+ *	cpumask_or(&tmp, &mask1, &mask2);
-+ *	for_each_cpu(cpu, &tmp)
-+ *		...
-+ *
-+ * After the loop, cpu is >= nr_cpu_ids.
-+ */
-+#define for_each_cpu_or(cpu, mask1, mask2)				\
-+	for_each_or_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
-+
-+/**
-+ * for_each_cpu_from - iterate over CPUs present in @mask, from @cpu to the end of @mask.
-+ * @cpu: the (optionally unsigned) integer iterator
-+ * @mask: the cpumask pointer
-+ *
-+ * After the loop, cpu is >= nr_cpu_ids.
-+ */
-+#define for_each_cpu_from(cpu, mask)				\
-+	for_each_set_bit_from(cpu, cpumask_bits(mask), small_cpumask_bits)
-+
-+/**
-+ * cpumask_any_but - return an arbitrary cpu in a cpumask, but not this one.
-  * @mask: the cpumask to search
-  * @cpu: the cpu to ignore.
-  *
-  * Often used to find any cpu but smp_processor_id() in a mask.
- * Returns >= nr_cpu_ids if no cpus set.
-+ * If @cpu == -1, the function is equivalent to cpumask_any().
-+ * Return: >= nr_cpu_ids if no cpus set.
-  */
-static inline
-unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
-+static __always_inline
-+unsigned int cpumask_any_but(const struct cpumask *mask, int cpu)
- {
- 	unsigned int i;
- 
-	cpumask_check(cpu);
-+	/* -1 is a legal arg here. */
-+	if (cpu != -1)
-+		cpumask_check(cpu);
-+
- 	for_each_cpu(i, mask)
- 		if (i != cpu)
- 			break;
- 	return i;
- }
- 
-+/**
-+ * cpumask_any_and_but - pick an arbitrary cpu from *mask1 & *mask2, but not this one.
-+ * @mask1: the first input cpumask
-+ * @mask2: the second input cpumask
-+ * @cpu: the cpu to ignore
-+ *
-+ * If @cpu == -1, the function is equivalent to cpumask_any_and().
-+ * Returns >= nr_cpu_ids if no cpus set.
-+ */
-+static __always_inline
-+unsigned int cpumask_any_and_but(const struct cpumask *mask1,
-+				 const struct cpumask *mask2,
-+				 int cpu)
-+{
-+	unsigned int i;
-+
-+	/* -1 is a legal arg here. */
-+	if (cpu != -1)
-+		cpumask_check(cpu);
-+
-+	i = cpumask_first_and(mask1, mask2);
-+	if (i != cpu)
-+		return i;
-+
-+	return cpumask_next_and(cpu, mask1, mask2);
-+}
-+
-+/**
-+ * cpumask_any_andnot_but - pick an arbitrary cpu from *mask1 & ~*mask2, but not this one.
-+ * @mask1: the first input cpumask
-+ * @mask2: the second input cpumask
-+ * @cpu: the cpu to ignore
-+ *
-+ * If @cpu == -1, the function returns the first matching cpu.
-+ * Returns >= nr_cpu_ids if no cpus set.
-+ */
-+static __always_inline
-+unsigned int cpumask_any_andnot_but(const struct cpumask *mask1,
-+				    const struct cpumask *mask2,
-+				    int cpu)
-+{
-+	unsigned int i;
-+
-+	/* -1 is a legal arg here. */
-+	if (cpu != -1)
-+		cpumask_check(cpu);
-+
-+	i = cpumask_first_andnot(mask1, mask2);
-+	if (i != cpu)
-+		return i;
-+
-+	return cpumask_next_andnot(cpu, mask1, mask2);
-+}
-+
-+/**
-+ * cpumask_nth - get the Nth cpu in a cpumask
-+ * @srcp: the cpumask pointer
-+ * @cpu: the Nth cpu to find, starting from 0
-+ *
-+ * Return: >= nr_cpu_ids if such cpu doesn't exist.
-+ */
-+static __always_inline
-+unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)
-+{
-+	return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu));
-+}
-+
-+/**
-+ * cpumask_nth_and - get the Nth cpu in 2 cpumasks
-+ * @srcp1: the cpumask pointer
-+ * @srcp2: the cpumask pointer
-+ * @cpu: the Nth cpu to find, starting from 0
-+ *
-+ * Return: >= nr_cpu_ids if such cpu doesn't exist.
-+ */
-+static __always_inline
-+unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,
-+							const struct cpumask *srcp2)
-+{
-+	return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
-+				small_cpumask_bits, cpumask_check(cpu));
-+}
-+
-+/**
-+ * cpumask_nth_and_andnot - get the Nth cpu set in 1st and 2nd cpumask, and clear in 3rd.
-+ * @srcp1: the cpumask pointer
-+ * @srcp2: the cpumask pointer
-+ * @srcp3: the cpumask pointer
-+ * @cpu: the Nth cpu to find, starting from 0
-+ *
-+ * Return: >= nr_cpu_ids if such cpu doesn't exist.
-+ */
-+static __always_inline
-+unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1,
-+							const struct cpumask *srcp2,
-+							const struct cpumask *srcp3)
-+{
-+	return find_nth_and_andnot_bit(cpumask_bits(srcp1),
-+					cpumask_bits(srcp2),
-+					cpumask_bits(srcp3),
-+					small_cpumask_bits, cpumask_check(cpu));
-+}
-+
- #define CPU_BITS_NONE						\
- {								\
- 	[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL			\
-@@ -352,16 +596,30 @@ unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
-  * @cpu: cpu number (< nr_cpu_ids)
-  * @dstp: the cpumask pointer
-  */
-static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
-+static __always_inline
-+void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
- {
- 	set_bit(cpumask_check(cpu), cpumask_bits(dstp));
- }
- 
-static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
-+static __always_inline
-+void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
- {
- 	__set_bit(cpumask_check(cpu), cpumask_bits(dstp));
- }
- 
-+/**
-+ * cpumask_clear_cpus - clear cpus in a cpumask
-+ * @dstp:  the cpumask pointer
-+ * @cpu:   cpu number (< nr_cpu_ids)
-+ * @ncpus: number of cpus to clear (< nr_cpu_ids)
-+ */
-+static __always_inline void cpumask_clear_cpus(struct cpumask *dstp,
-+						unsigned int cpu, unsigned int ncpus)
-+{
-+	cpumask_check(cpu + ncpus - 1);
-+	bitmap_clear(cpumask_bits(dstp), cpumask_check(cpu), ncpus);
-+}
- 
- /**
-  * cpumask_clear_cpu - clear a cpu in a cpumask
-@@ -383,9 +641,10 @@ static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
-  * @cpu: cpu number (< nr_cpu_ids)
-  * @cpumask: the cpumask pointer
-  *
- * Returns true if @cpu is set in @cpumask, else returns false
-+ * Return: true if @cpu is set in @cpumask, else returns false
-  */
-static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
-+static __always_inline
-+bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
- {
- 	return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
- }
-@@ -395,11 +654,12 @@ static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpum
-  * @cpu: cpu number (< nr_cpu_ids)
-  * @cpumask: the cpumask pointer
-  *
- * Returns true if @cpu is set in old bitmap of @cpumask, else returns false
- *
-  * test_and_set_bit wrapper for cpumasks.
-+ *
-+ * Return: true if @cpu is set in old bitmap of @cpumask, else returns false
-  */
-static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
-+static __always_inline
-+bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
- {
- 	return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
- }
-@@ -409,11 +669,12 @@ static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cp
-  * @cpu: cpu number (< nr_cpu_ids)
-  * @cpumask: the cpumask pointer
-  *
- * Returns true if @cpu is set in old bitmap of @cpumask, else returns false
- *
-  * test_and_clear_bit wrapper for cpumasks.
-+ *
-+ * Return: true if @cpu is set in old bitmap of @cpumask, else returns false
-  */
-static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
-+static __always_inline
-+bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
- {
- 	return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
- }
-@@ -422,8 +683,12 @@ static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *
-  * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
-  * @dstp: the cpumask pointer
-  */
-static inline void cpumask_setall(struct cpumask *dstp)
-+static __always_inline void cpumask_setall(struct cpumask *dstp)
- {
-+	if (small_const_nbits(small_cpumask_bits)) {
-+		cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits);
-+		return;
-+	}
- 	bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
- }
- 
-@@ -431,9 +696,9 @@ static inline void cpumask_setall(struct cpumask *dstp)
-  * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
-  * @dstp: the cpumask pointer
-  */
-static inline void cpumask_clear(struct cpumask *dstp)
-+static __always_inline void cpumask_clear(struct cpumask *dstp)
- {
-	bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
-+	bitmap_zero(cpumask_bits(dstp), large_cpumask_bits);
- }
- 
- /**
-@@ -442,14 +707,14 @@ static inline void cpumask_clear(struct cpumask *dstp)
-  * @src1p: the first input
-  * @src2p: the second input
-  *
- * If *@dstp is empty, returns false, else returns true
-+ * Return: false if *@dstp is empty, else returns true
-  */
-static inline bool cpumask_and(struct cpumask *dstp,
-			       const struct cpumask *src1p,
-			       const struct cpumask *src2p)
-+static __always_inline
-+bool cpumask_and(struct cpumask *dstp, const struct cpumask *src1p,
-+		 const struct cpumask *src2p)
- {
- 	return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
-				       cpumask_bits(src2p), nr_cpumask_bits);
-+				       cpumask_bits(src2p), small_cpumask_bits);
- }
- 
- /**
-@@ -458,65 +723,72 @@ static inline bool cpumask_and(struct cpumask *dstp,
-  * @src1p: the first input
-  * @src2p: the second input
-  */
-static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
-			      const struct cpumask *src2p)
-+static __always_inline
-+void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
-+		const struct cpumask *src2p)
- {
- 	bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
-				      cpumask_bits(src2p), nr_cpumask_bits);
-+				      cpumask_bits(src2p), small_cpumask_bits);
- }
- 
- /**
- * cpumask_xor - *dstp = *src1p ^ *src2p
-+ * cpumask_weighted_or - *dstp = *src1p | *src2p and return the weight of the result
-  * @dstp: the cpumask result
-  * @src1p: the first input
-  * @src2p: the second input
-+ *
-+ * Return: The number of bits set in the resulting cpumask @dstp
-  */
-static inline void cpumask_xor(struct cpumask *dstp,
-			       const struct cpumask *src1p,
-			       const struct cpumask *src2p)
-+static __always_inline
-+unsigned int cpumask_weighted_or(struct cpumask *dstp, const struct cpumask *src1p,
-+				 const struct cpumask *src2p)
- {
-	bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
-				       cpumask_bits(src2p), nr_cpumask_bits);
-+	return bitmap_weighted_or(cpumask_bits(dstp), cpumask_bits(src1p),
-+				  cpumask_bits(src2p), small_cpumask_bits);
- }
- 
- /**
- * cpumask_andnot - *dstp = *src1p & ~*src2p
-+ * cpumask_xor - *dstp = *src1p ^ *src2p
-  * @dstp: the cpumask result
-  * @src1p: the first input
-  * @src2p: the second input
- *
- * If *@dstp is empty, returns false, else returns true
-  */
-static inline bool cpumask_andnot(struct cpumask *dstp,
-				  const struct cpumask *src1p,
-				  const struct cpumask *src2p)
-+static __always_inline
-+void cpumask_xor(struct cpumask *dstp, const struct cpumask *src1p,
-+		 const struct cpumask *src2p)
- {
-	return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
-					  cpumask_bits(src2p), nr_cpumask_bits);
-+	bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
-+				       cpumask_bits(src2p), small_cpumask_bits);
- }
- 
- /**
- * cpumask_complement - *dstp = ~*srcp
-+ * cpumask_andnot - *dstp = *src1p & ~*src2p
-  * @dstp: the cpumask result
- * @srcp: the input to invert
-+ * @src1p: the first input
-+ * @src2p: the second input
-+ *
-+ * Return: false if *@dstp is empty, else returns true
-  */
-static inline void cpumask_complement(struct cpumask *dstp,
-				      const struct cpumask *srcp)
-+static __always_inline
-+bool cpumask_andnot(struct cpumask *dstp, const struct cpumask *src1p,
-+		    const struct cpumask *src2p)
- {
-	bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
-					      nr_cpumask_bits);
-+	return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
-+					  cpumask_bits(src2p), small_cpumask_bits);
- }
- 
- /**
-  * cpumask_equal - *src1p == *src2p
-  * @src1p: the first input
-  * @src2p: the second input
-+ *
-+ * Return: true if the cpumasks are equal, false if not
-  */
-static inline bool cpumask_equal(const struct cpumask *src1p,
-				const struct cpumask *src2p)
-+static __always_inline
-+bool cpumask_equal(const struct cpumask *src1p, const struct cpumask *src2p)
- {
- 	return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
-						 nr_cpumask_bits);
-+						 small_cpumask_bits);
- }
- 
- /**
-@@ -524,25 +796,31 @@ static inline bool cpumask_equal(const struct cpumask *src1p,
-  * @src1p: the first input
-  * @src2p: the second input
-  * @src3p: the third input
-+ *
-+ * Return: true if first cpumask ORed with second cpumask == third cpumask,
-+ *	   otherwise false
-  */
-static inline bool cpumask_or_equal(const struct cpumask *src1p,
-				    const struct cpumask *src2p,
-				    const struct cpumask *src3p)
-+static __always_inline
-+bool cpumask_or_equal(const struct cpumask *src1p, const struct cpumask *src2p,
-+		      const struct cpumask *src3p)
- {
- 	return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
-			       cpumask_bits(src3p), nr_cpumask_bits);
-+			       cpumask_bits(src3p), small_cpumask_bits);
- }
- 
- /**
-  * cpumask_intersects - (*src1p & *src2p) != 0
-  * @src1p: the first input
-  * @src2p: the second input
-+ *
-+ * Return: true if first cpumask ANDed with second cpumask is non-empty,
-+ *	   otherwise false
-  */
-static inline bool cpumask_intersects(const struct cpumask *src1p,
-				     const struct cpumask *src2p)
-+static __always_inline
-+bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p)
- {
- 	return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
-						      nr_cpumask_bits);
-+						      small_cpumask_bits);
- }
- 
- /**
-@@ -550,29 +828,33 @@ static inline bool cpumask_intersects(const struct cpumask *src1p,
-  * @src1p: the first input
-  * @src2p: the second input
-  *
- * Returns true if *@src1p is a subset of *@src2p, else returns false
-+ * Return: true if *@src1p is a subset of *@src2p, else returns false
-  */
-static inline bool cpumask_subset(const struct cpumask *src1p,
-				 const struct cpumask *src2p)
-+static __always_inline
-+bool cpumask_subset(const struct cpumask *src1p, const struct cpumask *src2p)
- {
- 	return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
-						  nr_cpumask_bits);
-+						  small_cpumask_bits);
- }
- 
- /**
-  * cpumask_empty - *srcp == 0
-  * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
-+ *
-+ * Return: true if srcp is empty (has no bits set), else false
-  */
-static inline bool cpumask_empty(const struct cpumask *srcp)
-+static __always_inline bool cpumask_empty(const struct cpumask *srcp)
- {
-	return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
-+	return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits);
- }
- 
- /**
-  * cpumask_full - *srcp == 0xFFFFFFFF...
-  * @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
-+ *
-+ * Return: true if srcp is full (has all bits set), else false
-  */
-static inline bool cpumask_full(const struct cpumask *srcp)
-+static __always_inline bool cpumask_full(const struct cpumask *srcp)
- {
- 	return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
- }
-@@ -580,10 +862,39 @@ static inline bool cpumask_full(const struct cpumask *srcp)
- /**
-  * cpumask_weight - Count of bits in *srcp
-  * @srcp: the cpumask to count bits (< nr_cpu_ids) in.
-+ *
-+ * Return: count of bits set in *srcp
-+ */
-+static __always_inline unsigned int cpumask_weight(const struct cpumask *srcp)
-+{
-+	return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits);
-+}
-+
-+/**
-+ * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2)
-+ * @srcp1: the cpumask to count bits (< nr_cpu_ids) in.
-+ * @srcp2: the cpumask to count bits (< nr_cpu_ids) in.
-+ *
-+ * Return: count of bits set in both *srcp1 and *srcp2
-  */
-static inline unsigned int cpumask_weight(const struct cpumask *srcp)
-+static __always_inline
-+unsigned int cpumask_weight_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
- {
-	return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
-+	return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
-+}
-+
-+/**
-+ * cpumask_weight_andnot - Count of bits in (*srcp1 & ~*srcp2)
-+ * @srcp1: the cpumask to count bits (< nr_cpu_ids) in.
-+ * @srcp2: the cpumask to count bits (< nr_cpu_ids) in.
-+ *
-+ * Return: count of bits set in both *srcp1 and *srcp2
-+ */
-+static __always_inline
-+unsigned int cpumask_weight_andnot(const struct cpumask *srcp1,
-+				   const struct cpumask *srcp2)
-+{
-+	return bitmap_weight_andnot(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
- }
- 
- /**
-@@ -592,11 +903,11 @@ static inline unsigned int cpumask_weight(const struct cpumask *srcp)
-  * @srcp: the input to shift
-  * @n: the number of bits to shift by
-  */
-static inline void cpumask_shift_right(struct cpumask *dstp,
-				       const struct cpumask *srcp, int n)
-+static __always_inline
-+void cpumask_shift_right(struct cpumask *dstp, const struct cpumask *srcp, int n)
- {
- 	bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
-					       nr_cpumask_bits);
-+					       small_cpumask_bits);
- }
- 
- /**
-@@ -605,8 +916,8 @@ static inline void cpumask_shift_right(struct cpumask *dstp,
-  * @srcp: the input to shift
-  * @n: the number of bits to shift by
-  */
-static inline void cpumask_shift_left(struct cpumask *dstp,
-				      const struct cpumask *srcp, int n)
-+static __always_inline
-+void cpumask_shift_left(struct cpumask *dstp, const struct cpumask *srcp, int n)
- {
- 	bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
- 					      nr_cpumask_bits);
-@@ -617,26 +928,26 @@ static inline void cpumask_shift_left(struct cpumask *dstp,
-  * @dstp: the result
-  * @srcp: the input cpumask
-  */
-static inline void cpumask_copy(struct cpumask *dstp,
-				const struct cpumask *srcp)
-+static __always_inline
-+void cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp)
- {
-	bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
-+	bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits);
- }
- 
- /**
- * cpumask_any - pick a "random" cpu from *srcp
-+ * cpumask_any - pick an arbitrary cpu from *srcp
-  * @srcp: the input cpumask
-  *
- * Returns >= nr_cpu_ids if no cpus set.
-+ * Return: >= nr_cpu_ids if no cpus set.
-  */
- #define cpumask_any(srcp) cpumask_first(srcp)
- 
- /**
- * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
-+ * cpumask_any_and - pick an arbitrary cpu from *mask1 & *mask2
-  * @mask1: the first input cpumask
-  * @mask2: the second input cpumask
-  *
- * Returns >= nr_cpu_ids if no cpus set.
-+ * Return: >= nr_cpu_ids if no cpus set.
-  */
- #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
- 
-@@ -652,10 +963,10 @@ static inline void cpumask_copy(struct cpumask *dstp,
-  * @len: the length of the buffer
-  * @dstp: the cpumask to set.
-  *
- * Returns -errno, or 0 for success.
-+ * Return: -errno, or 0 for success.
-  */
-static inline int cpumask_parse_user(const char __user *buf, int len,
-				     struct cpumask *dstp)
-+static __always_inline
-+int cpumask_parse_user(const char __user *buf, int len, struct cpumask *dstp)
- {
- 	return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
- }
-@@ -666,10 +977,10 @@ static inline int cpumask_parse_user(const char __user *buf, int len,
-  * @len: the length of the buffer
-  * @dstp: the cpumask to set.
-  *
- * Returns -errno, or 0 for success.
-+ * Return: -errno, or 0 for success.
-  */
-static inline int cpumask_parselist_user(const char __user *buf, int len,
-				     struct cpumask *dstp)
-+static __always_inline
-+int cpumask_parselist_user(const char __user *buf, int len, struct cpumask *dstp)
- {
- 	return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
- 				     nr_cpumask_bits);
-@@ -680,9 +991,9 @@ static inline int cpumask_parselist_user(const char __user *buf, int len,
-  * @buf: the buffer to extract from
-  * @dstp: the cpumask to set.
-  *
- * Returns -errno, or 0 for success.
-+ * Return: -errno, or 0 for success.
-  */
-static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
-+static __always_inline int cpumask_parse(const char *buf, struct cpumask *dstp)
- {
- 	return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
- }
-@@ -692,70 +1003,32 @@ static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
-  * @buf: the buffer to extract from
-  * @dstp: the cpumask to set.
-  *
- * Returns -errno, or 0 for success.
-+ * Return: -errno, or 0 for success.
-  */
-static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
-+static __always_inline int cpulist_parse(const char *buf, struct cpumask *dstp)
- {
- 	return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
- }
- 
- /**
- * cpumask_size - size to allocate for a 'struct cpumask' in bytes
-+ * cpumask_size - calculate size to allocate for a 'struct cpumask' in bytes
-+ *
-+ * Return: size to allocate for a &struct cpumask in bytes
-  */
-static inline unsigned int cpumask_size(void)
-+static __always_inline unsigned int cpumask_size(void)
- {
-	return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
-+	return bitmap_size(large_cpumask_bits);
- }
- 
-/*
- * cpumask_var_t: struct cpumask for stack usage.
- *
- * Oh, the wicked games we play!  In order to make kernel coding a
- * little more difficult, we typedef cpumask_var_t to an array or a
- * pointer: doing &mask on an array is a noop, so it still works.
- *
- * ie.
- *	cpumask_var_t tmpmask;
- *	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
- *		return -ENOMEM;
- *
- *	  ... use 'tmpmask' like a normal struct cpumask * ...
- *
- *	free_cpumask_var(tmpmask);
- *
- *
- * However, one notable exception is there. alloc_cpumask_var() allocates
- * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
- * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
- *
- *	cpumask_var_t tmpmask;
- *	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
- *		return -ENOMEM;
- *
- *	var = *tmpmask;
- *
- * This code makes NR_CPUS length memcopy and brings to a memory corruption.
- * cpumask_copy() provide safe copy functionality.
- *
- * Note that there is another evil here: If you define a cpumask_var_t
- * as a percpu variable then the way to obtain the address of the cpumask
- * structure differently influences what this_cpu_* operation needs to be
- * used. Please use this_cpu_cpumask_var_t in those cases. The direct use
- * of this_cpu_ptr() or this_cpu_read() will lead to failures when the
- * other type of cpumask_var_t implementation is configured.
- *
- * Please also note that __cpumask_var_read_mostly can be used to declare
- * a cpumask_var_t variable itself (not its content) as read mostly.
- */
- #ifdef CONFIG_CPUMASK_OFFSTACK
-typedef struct cpumask *cpumask_var_t;
- 
- #define this_cpu_cpumask_var_ptr(x)	this_cpu_read(x)
- #define __cpumask_var_read_mostly	__read_mostly
-+#define CPUMASK_VAR_NULL		NULL
- 
- bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
- 
-static inline
-+static __always_inline
- bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
- {
- 	return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);
-@@ -770,14 +1043,16 @@ bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
-  * a nop returning a constant 1 (in <linux/cpumask.h>).
-  *
-  * See alloc_cpumask_var_node.
-+ *
-+ * Return: %true if allocation succeeded, %false if not
-  */
-static inline
-+static __always_inline
- bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
- {
- 	return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
- }
- 
-static inline
-+static __always_inline
- bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
- {
- 	return alloc_cpumask_var(mask, flags | __GFP_ZERO);
-@@ -787,59 +1062,61 @@ void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
- void free_cpumask_var(cpumask_var_t mask);
- void free_bootmem_cpumask_var(cpumask_var_t mask);
- 
-static inline bool cpumask_available(cpumask_var_t mask)
-+static __always_inline bool cpumask_available(cpumask_var_t mask)
- {
- 	return mask != NULL;
- }
- 
- #else
-typedef struct cpumask cpumask_var_t[1];
- 
- #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
- #define __cpumask_var_read_mostly
-+#define CPUMASK_VAR_NULL {}
- 
-static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
-+static __always_inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
- {
- 	return true;
- }
- 
-static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
-+static __always_inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
- 					  int node)
- {
- 	return true;
- }
- 
-static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
-+static __always_inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
- {
- 	cpumask_clear(*mask);
- 	return true;
- }
- 
-static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
-+static __always_inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
- 					  int node)
- {
- 	cpumask_clear(*mask);
- 	return true;
- }
- 
-static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
-+static __always_inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
- {
- }
- 
-static inline void free_cpumask_var(cpumask_var_t mask)
-+static __always_inline void free_cpumask_var(cpumask_var_t mask)
- {
- }
- 
-static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
-+static __always_inline void free_bootmem_cpumask_var(cpumask_var_t mask)
- {
- }
- 
-static inline bool cpumask_available(cpumask_var_t mask)
-+static __always_inline bool cpumask_available(cpumask_var_t mask)
- {
- 	return true;
- }
- #endif /* CONFIG_CPUMASK_OFFSTACK */
- 
-+DEFINE_FREE(free_cpumask_var, struct cpumask *, if (_T) free_cpumask_var(_T));
-+
- /* It's common to want to use cpu_all_mask in struct member initializers,
-  * so it has to refer to an address rather than a pointer. */
- extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
-@@ -853,62 +1130,43 @@ extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
- #define for_each_possible_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)
- #define for_each_online_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)
- #define for_each_present_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)
-+
-+#define for_each_possible_cpu_wrap(cpu, start)	\
-+	for ((void)(start), (cpu) = 0; (cpu) < 1; (cpu)++)
-+#define for_each_online_cpu_wrap(cpu, start)	\
-+	for ((void)(start), (cpu) = 0; (cpu) < 1; (cpu)++)
- #else
- #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
- #define for_each_online_cpu(cpu)   for_each_cpu((cpu), cpu_online_mask)
-+#define for_each_enabled_cpu(cpu)   for_each_cpu((cpu), cpu_enabled_mask)
- #define for_each_present_cpu(cpu)  for_each_cpu((cpu), cpu_present_mask)
-+
-+#define for_each_possible_cpu_wrap(cpu, start)	\
-+	for_each_cpu_wrap((cpu), cpu_possible_mask, (start))
-+#define for_each_online_cpu_wrap(cpu, start)	\
-+	for_each_cpu_wrap((cpu), cpu_online_mask, (start))
- #endif
- 
- /* Wrappers for arch boot code to manipulate normally-constant masks */
- void init_cpu_present(const struct cpumask *src);
- void init_cpu_possible(const struct cpumask *src);
-void init_cpu_online(const struct cpumask *src);
- 
-static inline void reset_cpu_possible_mask(void)
-{
-	bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
-}
-+#define assign_cpu(cpu, mask, val)	\
-+	assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
- 
-static inline void
-set_cpu_possible(unsigned int cpu, bool possible)
-{
-	if (possible)
-		cpumask_set_cpu(cpu, &__cpu_possible_mask);
-	else
-		cpumask_clear_cpu(cpu, &__cpu_possible_mask);
-}
-+#define __assign_cpu(cpu, mask, val)	\
-+	__assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
- 
-static inline void
-set_cpu_present(unsigned int cpu, bool present)
-{
-	if (present)
-		cpumask_set_cpu(cpu, &__cpu_present_mask);
-	else
-		cpumask_clear_cpu(cpu, &__cpu_present_mask);
-}
-+#define set_cpu_enabled(cpu, enabled)	assign_cpu((cpu), &__cpu_enabled_mask, (enabled))
-+#define set_cpu_present(cpu, present)	assign_cpu((cpu), &__cpu_present_mask, (present))
-+#define set_cpu_active(cpu, active)	assign_cpu((cpu), &__cpu_active_mask, (active))
-+#define set_cpu_dying(cpu, dying)	assign_cpu((cpu), &__cpu_dying_mask, (dying))
- 
- void set_cpu_online(unsigned int cpu, bool online);
-
-static inline void
-set_cpu_active(unsigned int cpu, bool active)
-{
-	if (active)
-		cpumask_set_cpu(cpu, &__cpu_active_mask);
-	else
-		cpumask_clear_cpu(cpu, &__cpu_active_mask);
-}
-
-static inline void
-set_cpu_dying(unsigned int cpu, bool dying)
-{
-	if (dying)
-		cpumask_set_cpu(cpu, &__cpu_dying_mask);
-	else
-		cpumask_clear_cpu(cpu, &__cpu_dying_mask);
-}
-+void set_cpu_possible(unsigned int cpu, bool possible);
- 
- /**
- * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
-+ * to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
-  * @bitmap: the bitmap
-  *
-  * There are a few places where cpumask_var_t isn't appropriate and
-@@ -921,7 +1179,7 @@ set_cpu_dying(unsigned int cpu, bool dying)
- 	((struct cpumask *)(1 ? (bitmap)				\
- 			    : (void *)sizeof(__check_is_bitmap(bitmap))))
- 
-static inline int __check_is_bitmap(const unsigned long *bitmap)
-+static __always_inline int __check_is_bitmap(const unsigned long *bitmap)
- {
- 	return 1;
- }
-@@ -936,7 +1194,7 @@ static inline int __check_is_bitmap(const unsigned long *bitmap)
- extern const unsigned long
- 	cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
- 
-static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
-+static __always_inline const struct cpumask *get_cpu_mask(unsigned int cpu)
- {
- 	const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
- 	p -= cpu / BITS_PER_LONG;
-@@ -951,36 +1209,49 @@ static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
-  * interface gives only a momentary snapshot and is not protected against
-  * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
-  * region.
-+ *
-+ * Return: momentary snapshot of the number of online CPUs
-  */
-static inline unsigned int num_online_cpus(void)
-+static __always_inline unsigned int num_online_cpus(void)
-+{
-+	return raw_atomic_read(&__num_online_cpus);
-+}
-+
-+static __always_inline unsigned int num_possible_cpus(void)
- {
-	return atomic_read(&__num_online_cpus);
-+	return __num_possible_cpus;
- }
-#define num_possible_cpus()	cpumask_weight(cpu_possible_mask)
-+
-+#define num_enabled_cpus()	cpumask_weight(cpu_enabled_mask)
- #define num_present_cpus()	cpumask_weight(cpu_present_mask)
- #define num_active_cpus()	cpumask_weight(cpu_active_mask)
- 
-static inline bool cpu_online(unsigned int cpu)
-+static __always_inline bool cpu_online(unsigned int cpu)
- {
- 	return cpumask_test_cpu(cpu, cpu_online_mask);
- }
- 
-static inline bool cpu_possible(unsigned int cpu)
-+static __always_inline bool cpu_enabled(unsigned int cpu)
-+{
-+	return cpumask_test_cpu(cpu, cpu_enabled_mask);
-+}
-+
-+static __always_inline bool cpu_possible(unsigned int cpu)
- {
- 	return cpumask_test_cpu(cpu, cpu_possible_mask);
- }
- 
-static inline bool cpu_present(unsigned int cpu)
-+static __always_inline bool cpu_present(unsigned int cpu)
- {
- 	return cpumask_test_cpu(cpu, cpu_present_mask);
- }
- 
-static inline bool cpu_active(unsigned int cpu)
-+static __always_inline bool cpu_active(unsigned int cpu)
- {
- 	return cpumask_test_cpu(cpu, cpu_active_mask);
- }
- 
-static inline bool cpu_dying(unsigned int cpu)
-+static __always_inline bool cpu_dying(unsigned int cpu)
- {
- 	return cpumask_test_cpu(cpu, cpu_dying_mask);
- }
-@@ -989,30 +1260,36 @@ static inline bool cpu_dying(unsigned int cpu)
- 
- #define num_online_cpus()	1U
- #define num_possible_cpus()	1U
-+#define num_enabled_cpus()	1U
- #define num_present_cpus()	1U
- #define num_active_cpus()	1U
- 
-static inline bool cpu_online(unsigned int cpu)
-+static __always_inline bool cpu_online(unsigned int cpu)
-+{
-+	return cpu == 0;
-+}
-+
-+static __always_inline bool cpu_possible(unsigned int cpu)
- {
- 	return cpu == 0;
- }
- 
-static inline bool cpu_possible(unsigned int cpu)
-+static __always_inline bool cpu_enabled(unsigned int cpu)
- {
- 	return cpu == 0;
- }
- 
-static inline bool cpu_present(unsigned int cpu)
-+static __always_inline bool cpu_present(unsigned int cpu)
- {
- 	return cpu == 0;
- }
- 
-static inline bool cpu_active(unsigned int cpu)
-+static __always_inline bool cpu_active(unsigned int cpu)
- {
- 	return cpu == 0;
- }
- 
-static inline bool cpu_dying(unsigned int cpu)
-+static __always_inline bool cpu_dying(unsigned int cpu)
- {
- 	return false;
- }
-@@ -1043,10 +1320,10 @@ static inline bool cpu_dying(unsigned int cpu)
-  * @mask: the cpumask to copy
-  * @buf: the buffer to copy into
-  *
- * Returns the length of the (null-terminated) @buf string, zero if
-+ * Return: the length of the (null-terminated) @buf string, zero if
-  * nothing is copied.
-  */
-static inline ssize_t
-+static __always_inline ssize_t
- cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
- {
- 	return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
-@@ -1066,12 +1343,12 @@ cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
-  * cpumask; Typically used by bin_attribute to export cpumask bitmask
-  * ABI.
-  *
- * Returns the length of how many bytes have been copied, excluding
-+ * Return: the length of how many bytes have been copied, excluding
-  * terminating '\0'.
-  */
-static inline ssize_t
-cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
-		loff_t off, size_t count)
-+static __always_inline
-+ssize_t cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
-+				    loff_t off, size_t count)
- {
- 	return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask),
- 				   nr_cpu_ids, off, count) - 1;
-@@ -1080,13 +1357,20 @@ cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
- /**
-  * cpumap_print_list_to_buf  - copies the cpumask into the buffer as
-  *	comma-separated list of cpus
-+ * @buf: the buffer to copy into
-+ * @mask: the cpumask to copy
-+ * @off: in the string from which we are copying, we copy to @buf
-+ * @count: the maximum number of bytes to print
-  *
-  * Everything is same with the above cpumap_print_bitmask_to_buf()
-  * except the print format.
-+ *
-+ * Return: the length of how many bytes have been copied, excluding
-+ * terminating '\0'.
-  */
-static inline ssize_t
-cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
-		loff_t off, size_t count)
-+static __always_inline
-+ssize_t cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
-+				 loff_t off, size_t count)
- {
- 	return bitmap_print_list_to_buf(buf, cpumask_bits(mask),
- 				   nr_cpu_ids, off, count) - 1;
-@@ -1127,9 +1411,10 @@ cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
-  * cover a worst-case of every other cpu being on one of two nodes for a
-  * very large NR_CPUS.
-  *
- *  Use PAGE_SIZE as a minimum for smaller configurations.
-+ *  Use PAGE_SIZE as a minimum for smaller configurations while avoiding
-+ *  unsigned comparison to -1.
-  */
-#define CPUMAP_FILE_MAX_BYTES  ((((NR_CPUS * 9)/32 - 1) > PAGE_SIZE) \
-+#define CPUMAP_FILE_MAX_BYTES  (((NR_CPUS * 9)/32 > PAGE_SIZE) \
- 					? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE)
- #define CPULIST_FILE_MAX_BYTES  (((NR_CPUS * 7)/2 > PAGE_SIZE) ? (NR_CPUS * 7)/2 : PAGE_SIZE)
- 
-diff --git a/include/linux/cpumask_types.h b/include/linux/cpumask_types.h
-new file mode 100644
-index 000000000000..461ed1b6bcdb
--- /dev/null
-+++ b/include/linux/cpumask_types.h
-@@ -0,0 +1,66 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef __LINUX_CPUMASK_TYPES_H
-+#define __LINUX_CPUMASK_TYPES_H
-+
-+#include <linux/bitops.h>
-+#include <linux/threads.h>
-+
-+/* Don't assign or return these: may not be this big! */
-+typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
-+
-+/**
-+ * cpumask_bits - get the bits in a cpumask
-+ * @maskp: the struct cpumask *
-+ *
-+ * You should only assume nr_cpu_ids bits of this mask are valid.  This is
-+ * a macro so it's const-correct.
-+ */
-+#define cpumask_bits(maskp) ((maskp)->bits)
-+
-+/*
-+ * cpumask_var_t: struct cpumask for stack usage.
-+ *
-+ * Oh, the wicked games we play!  In order to make kernel coding a
-+ * little more difficult, we typedef cpumask_var_t to an array or a
-+ * pointer: doing &mask on an array is a noop, so it still works.
-+ *
-+ * i.e.
-+ *	cpumask_var_t tmpmask;
-+ *	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
-+ *		return -ENOMEM;
-+ *
-+ *	  ... use 'tmpmask' like a normal struct cpumask * ...
-+ *
-+ *	free_cpumask_var(tmpmask);
-+ *
-+ *
-+ * However, one notable exception is there. alloc_cpumask_var() allocates
-+ * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
-+ * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
-+ *
-+ *	cpumask_var_t tmpmask;
-+ *	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
-+ *		return -ENOMEM;
-+ *
-+ *	var = *tmpmask;
-+ *
-+ * This code makes NR_CPUS length memcopy and brings to a memory corruption.
-+ * cpumask_copy() provide safe copy functionality.
-+ *
-+ * Note that there is another evil here: If you define a cpumask_var_t
-+ * as a percpu variable then the way to obtain the address of the cpumask
-+ * structure differently influences what this_cpu_* operation needs to be
-+ * used. Please use this_cpu_cpumask_var_t in those cases. The direct use
-+ * of this_cpu_ptr() or this_cpu_read() will lead to failures when the
-+ * other type of cpumask_var_t implementation is configured.
-+ *
-+ * Please also note that __cpumask_var_read_mostly can be used to declare
-+ * a cpumask_var_t variable itself (not its content) as read mostly.
-+ */
-+#ifdef CONFIG_CPUMASK_OFFSTACK
-+typedef struct cpumask *cpumask_var_t;
-+#else
-+typedef struct cpumask cpumask_var_t[1];
-+#endif /* CONFIG_CPUMASK_OFFSTACK */
-+
-+#endif /* __LINUX_CPUMASK_TYPES_H */
-diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h
-index d58e0476ee8e..a98d3330385c 100644
--- a/include/linux/cpuset.h
-+++ b/include/linux/cpuset.h
-@@ -70,28 +70,24 @@ extern int cpuset_init(void);
- extern void cpuset_init_smp(void);
- extern void cpuset_force_rebuild(void);
- extern void cpuset_update_active_cpus(void);
-extern void cpuset_wait_for_hotplug(void);
-extern void cpuset_read_lock(void);
-extern void cpuset_read_unlock(void);
-+extern void inc_dl_tasks_cs(struct task_struct *task);
-+extern void dec_dl_tasks_cs(struct task_struct *task);
-+extern void cpuset_lock(void);
-+extern void cpuset_unlock(void);
-+extern void cpuset_cpus_allowed_locked(struct task_struct *p, struct cpumask *mask);
- extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask);
- extern bool cpuset_cpus_allowed_fallback(struct task_struct *p);
-+extern bool cpuset_cpu_is_isolated(int cpu);
- extern nodemask_t cpuset_mems_allowed(struct task_struct *p);
- #define cpuset_current_mems_allowed (current->mems_allowed)
- void cpuset_init_current_mems_allowed(void);
- int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask);
- 
-extern bool __cpuset_node_allowed(int node, gfp_t gfp_mask);
-
-static inline bool cpuset_node_allowed(int node, gfp_t gfp_mask)
-{
-	if (cpusets_enabled())
-		return __cpuset_node_allowed(node, gfp_mask);
-	return true;
-}
-+extern bool cpuset_current_node_allowed(int node, gfp_t gfp_mask);
- 
- static inline bool __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask)
- {
-	return __cpuset_node_allowed(zone_to_nid(z), gfp_mask);
-+	return cpuset_current_node_allowed(zone_to_nid(z), gfp_mask);
- }
- 
- static inline bool cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask)
-@@ -104,6 +100,7 @@ static inline bool cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask)
- extern int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
- 					  const struct task_struct *tsk2);
- 
-+#ifdef CONFIG_CPUSETS_V1
- #define cpuset_memory_pressure_bump() 				\
- 	do {							\
- 		if (cpuset_memory_pressure_enabled)		\
-@@ -111,6 +108,9 @@ extern int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
- 	} while (0)
- extern int cpuset_memory_pressure_enabled;
- extern void __cpuset_memory_pressure_bump(void);
-+#else
-+static inline void cpuset_memory_pressure_bump(void) { }
-+#endif
- 
- extern void cpuset_task_status_allowed(struct seq_file *m,
- 					struct task_struct *task);
-@@ -118,23 +118,19 @@ extern int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns,
- 			    struct pid *pid, struct task_struct *tsk);
- 
- extern int cpuset_mem_spread_node(void);
-extern int cpuset_slab_spread_node(void);
- 
- static inline int cpuset_do_page_mem_spread(void)
- {
- 	return task_spread_page(current);
- }
- 
-static inline int cpuset_do_slab_mem_spread(void)
-{
-	return task_spread_slab(current);
-}
-
- extern bool current_cpuset_is_being_rebound(void);
- 
-+extern void dl_rebuild_rd_accounting(void);
- extern void rebuild_sched_domains(void);
- 
- extern void cpuset_print_current_mems_allowed(void);
-+extern void cpuset_reset_sched_domains(void);
- 
- /*
-  * read_mems_allowed_begin is required when making decisions involving
-@@ -178,6 +174,7 @@ static inline void set_mems_allowed(nodemask_t nodemask)
- 	task_unlock(current);
- }
- 
-+extern bool cpuset_node_allowed(struct cgroup *cgroup, int nid);
- #else /* !CONFIG_CPUSETS */
- 
- static inline bool cpusets_enabled(void) { return false; }
-@@ -194,15 +191,21 @@ static inline void cpuset_update_active_cpus(void)
- 	partition_sched_domains(1, NULL, NULL);
- }
- 
-static inline void cpuset_wait_for_hotplug(void) { }
-+static inline void inc_dl_tasks_cs(struct task_struct *task) { }
-+static inline void dec_dl_tasks_cs(struct task_struct *task) { }
-+static inline void cpuset_lock(void) { }
-+static inline void cpuset_unlock(void) { }
- 
-static inline void cpuset_read_lock(void) { }
-static inline void cpuset_read_unlock(void) { }
-+static inline void cpuset_cpus_allowed_locked(struct task_struct *p,
-+					struct cpumask *mask)
-+{
-+	cpumask_copy(mask, task_cpu_possible_mask(p));
-+}
- 
- static inline void cpuset_cpus_allowed(struct task_struct *p,
- 				       struct cpumask *mask)
- {
-	cpumask_copy(mask, task_cpu_possible_mask(p));
-+	cpuset_cpus_allowed_locked(p, mask);
- }
- 
- static inline bool cpuset_cpus_allowed_fallback(struct task_struct *p)
-@@ -210,6 +213,11 @@ static inline bool cpuset_cpus_allowed_fallback(struct task_struct *p)
- 	return false;
- }
- 
-+static inline bool cpuset_cpu_is_isolated(int cpu)
-+{
-+	return false;
-+}
-+
- static inline nodemask_t cpuset_mems_allowed(struct task_struct *p)
- {
- 	return node_possible_map;
-@@ -223,11 +231,6 @@ static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
- 	return 1;
- }
- 
-static inline bool cpuset_node_allowed(int node, gfp_t gfp_mask)
-{
-	return true;
-}
-
- static inline bool __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask)
- {
- 	return true;
-@@ -256,27 +259,26 @@ static inline int cpuset_mem_spread_node(void)
- 	return 0;
- }
- 
-static inline int cpuset_slab_spread_node(void)
-+static inline int cpuset_do_page_mem_spread(void)
- {
- 	return 0;
- }
- 
-static inline int cpuset_do_page_mem_spread(void)
-+static inline bool current_cpuset_is_being_rebound(void)
- {
-	return 0;
-+	return false;
- }
- 
-static inline int cpuset_do_slab_mem_spread(void)
-+static inline void dl_rebuild_rd_accounting(void)
- {
-	return 0;
- }
- 
-static inline bool current_cpuset_is_being_rebound(void)
-+static inline void rebuild_sched_domains(void)
- {
-	return false;
-+	partition_sched_domains(1, NULL, NULL);
- }
- 
-static inline void rebuild_sched_domains(void)
-+static inline void cpuset_reset_sched_domains(void)
- {
- 	partition_sched_domains(1, NULL, NULL);
- }
-@@ -299,6 +301,10 @@ static inline bool read_mems_allowed_retry(unsigned int seq)
- 	return false;
- }
- 
-+static inline bool cpuset_node_allowed(struct cgroup *cgroup, int nid)
-+{
-+	return true;
-+}
- #endif /* !CONFIG_CPUSETS */
- 
- #endif /* _LINUX_CPUSET_H */
-diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h
-index de62a722431e..d35726d6a415 100644
--- a/include/linux/crash_core.h
-+++ b/include/linux/crash_core.h
-@@ -6,82 +6,94 @@
- #include <linux/elfcore.h>
- #include <linux/elf.h>
- 
-#define CRASH_CORE_NOTE_NAME	   "CORE"
-#define CRASH_CORE_NOTE_HEAD_BYTES ALIGN(sizeof(struct elf_note), 4)
-#define CRASH_CORE_NOTE_NAME_BYTES ALIGN(sizeof(CRASH_CORE_NOTE_NAME), 4)
-#define CRASH_CORE_NOTE_DESC_BYTES ALIGN(sizeof(struct elf_prstatus), 4)
-+struct kimage;
- 
-+struct crash_mem {
-+	unsigned int max_nr_ranges;
-+	unsigned int nr_ranges;
-+	struct range ranges[] __counted_by(max_nr_ranges);
-+};
-+
-+#ifdef CONFIG_CRASH_DUMP
-+
-+int crash_shrink_memory(unsigned long new_size);
-+ssize_t crash_get_memory_size(void);
-+
-+#ifndef arch_kexec_protect_crashkres
- /*
- * The per-cpu notes area is a list of notes terminated by a "NULL"
- * note header.  For kdump, the code in vmcore.c runs in the context
- * of the second kernel to combine them into one note.
-+ * Protection mechanism for crashkernel reserved memory after
-+ * the kdump kernel is loaded.
-+ *
-+ * Provide an empty default implementation here -- architecture
-+ * code may override this
-  */
-#define CRASH_CORE_NOTE_BYTES	   ((CRASH_CORE_NOTE_HEAD_BYTES * 2) +	\
-				     CRASH_CORE_NOTE_NAME_BYTES +	\
-				     CRASH_CORE_NOTE_DESC_BYTES)
-
-#define VMCOREINFO_BYTES	   PAGE_SIZE
-#define VMCOREINFO_NOTE_NAME	   "VMCOREINFO"
-#define VMCOREINFO_NOTE_NAME_BYTES ALIGN(sizeof(VMCOREINFO_NOTE_NAME), 4)
-#define VMCOREINFO_NOTE_SIZE	   ((CRASH_CORE_NOTE_HEAD_BYTES * 2) +	\
-				     VMCOREINFO_NOTE_NAME_BYTES +	\
-				     VMCOREINFO_BYTES)
-
-typedef u32 note_buf_t[CRASH_CORE_NOTE_BYTES/4];
-
-void crash_update_vmcoreinfo_safecopy(void *ptr);
-void crash_save_vmcoreinfo(void);
-void arch_crash_save_vmcoreinfo(void);
-__printf(1, 2)
-void vmcoreinfo_append_str(const char *fmt, ...);
-phys_addr_t paddr_vmcoreinfo_note(void);
-
-#define VMCOREINFO_OSRELEASE(value) \
-	vmcoreinfo_append_str("OSRELEASE=%s\n", value)
-#define VMCOREINFO_BUILD_ID()						\
-	({								\
-		static_assert(sizeof(vmlinux_build_id) == 20);		\
-		vmcoreinfo_append_str("BUILD-ID=%20phN\n", vmlinux_build_id); \
-	})
-
-#define VMCOREINFO_PAGESIZE(value) \
-	vmcoreinfo_append_str("PAGESIZE=%ld\n", value)
-#define VMCOREINFO_SYMBOL(name) \
-	vmcoreinfo_append_str("SYMBOL(%s)=%lx\n", #name, (unsigned long)&name)
-#define VMCOREINFO_SYMBOL_ARRAY(name) \
-	vmcoreinfo_append_str("SYMBOL(%s)=%lx\n", #name, (unsigned long)name)
-#define VMCOREINFO_SIZE(name) \
-	vmcoreinfo_append_str("SIZE(%s)=%lu\n", #name, \
-			      (unsigned long)sizeof(name))
-#define VMCOREINFO_STRUCT_SIZE(name) \
-	vmcoreinfo_append_str("SIZE(%s)=%lu\n", #name, \
-			      (unsigned long)sizeof(struct name))
-#define VMCOREINFO_OFFSET(name, field) \
-	vmcoreinfo_append_str("OFFSET(%s.%s)=%lu\n", #name, #field, \
-			      (unsigned long)offsetof(struct name, field))
-#define VMCOREINFO_TYPE_OFFSET(name, field) \
-	vmcoreinfo_append_str("OFFSET(%s.%s)=%lu\n", #name, #field, \
-			      (unsigned long)offsetof(name, field))
-#define VMCOREINFO_LENGTH(name, value) \
-	vmcoreinfo_append_str("LENGTH(%s)=%lu\n", #name, (unsigned long)value)
-#define VMCOREINFO_NUMBER(name) \
-	vmcoreinfo_append_str("NUMBER(%s)=%ld\n", #name, (long)name)
-#define VMCOREINFO_CONFIG(name) \
-	vmcoreinfo_append_str("CONFIG_%s=y\n", #name)
-
-extern unsigned char *vmcoreinfo_data;
-extern size_t vmcoreinfo_size;
-extern u32 *vmcoreinfo_note;
-
-Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type,
-			  void *data, size_t data_len);
-void final_note(Elf_Word *buf);
-
-int __init parse_crashkernel(char *cmdline, unsigned long long system_ram,
-		unsigned long long *crash_size, unsigned long long *crash_base);
-int parse_crashkernel_high(char *cmdline, unsigned long long system_ram,
-		unsigned long long *crash_size, unsigned long long *crash_base);
-int parse_crashkernel_low(char *cmdline, unsigned long long system_ram,
-		unsigned long long *crash_size, unsigned long long *crash_base);
-+static inline void arch_kexec_protect_crashkres(void) { }
-+#endif
-+
-+#ifndef arch_kexec_unprotect_crashkres
-+static inline void arch_kexec_unprotect_crashkres(void) { }
-+#endif
-+
-+#ifdef CONFIG_CRASH_DM_CRYPT
-+int crash_load_dm_crypt_keys(struct kimage *image);
-+ssize_t dm_crypt_keys_read(char *buf, size_t count, u64 *ppos);
-+#else
-+static inline int crash_load_dm_crypt_keys(struct kimage *image) {return 0; }
-+#endif
-+
-+#ifndef arch_crash_handle_hotplug_event
-+static inline void arch_crash_handle_hotplug_event(struct kimage *image, void *arg) { }
-+#endif
-+
-+int crash_check_hotplug_support(void);
-+
-+#ifndef arch_crash_hotplug_support
-+static inline int arch_crash_hotplug_support(struct kimage *image, unsigned long kexec_flags)
-+{
-+	return 0;
-+}
-+#endif
-+
-+#ifndef crash_get_elfcorehdr_size
-+static inline unsigned int crash_get_elfcorehdr_size(void) { return 0; }
-+#endif
-+
-+/* Alignment required for elf header segment */
-+#define ELF_CORE_HEADER_ALIGN   4096
-+
-+extern int crash_exclude_mem_range(struct crash_mem *mem,
-+				   unsigned long long mstart,
-+				   unsigned long long mend);
-+extern int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map,
-+				       void **addr, unsigned long *sz);
-+
-+struct kimage;
-+struct kexec_segment;
-+
-+#define KEXEC_CRASH_HP_NONE			0
-+#define KEXEC_CRASH_HP_ADD_CPU			1
-+#define KEXEC_CRASH_HP_REMOVE_CPU		2
-+#define KEXEC_CRASH_HP_ADD_MEMORY		3
-+#define KEXEC_CRASH_HP_REMOVE_MEMORY		4
-+#define KEXEC_CRASH_HP_INVALID_CPU		-1U
-+
-+extern void __crash_kexec(struct pt_regs *regs);
-+extern void crash_kexec(struct pt_regs *regs);
-+int kexec_should_crash(struct task_struct *p);
-+int kexec_crash_loaded(void);
-+void crash_save_cpu(struct pt_regs *regs, int cpu);
-+extern int kimage_crash_copy_vmcoreinfo(struct kimage *image);
-+
-+#else /* !CONFIG_CRASH_DUMP*/
-+struct pt_regs;
-+struct task_struct;
-+struct kimage;
-+static inline void __crash_kexec(struct pt_regs *regs) { }
-+static inline void crash_kexec(struct pt_regs *regs) { }
-+static inline int kexec_should_crash(struct task_struct *p) { return 0; }
-+static inline int kexec_crash_loaded(void) { return 0; }
-+static inline void crash_save_cpu(struct pt_regs *regs, int cpu) {};
-+static inline int kimage_crash_copy_vmcoreinfo(struct kimage *image) { return 0; };
-+#endif /* CONFIG_CRASH_DUMP*/
- 
- #endif /* LINUX_CRASH_CORE_H */
-diff --git a/include/linux/crash_dump.h b/include/linux/crash_dump.h
-index 0f3a656293b0..dd6fc3b2133b 100644
--- a/include/linux/crash_dump.h
-+++ b/include/linux/crash_dump.h
-@@ -15,11 +15,15 @@
- extern unsigned long long elfcorehdr_addr;
- extern unsigned long long elfcorehdr_size;
- 
-+extern unsigned long long dm_crypt_keys_addr;
-+
- #ifdef CONFIG_CRASH_DUMP
- extern int elfcorehdr_alloc(unsigned long long *addr, unsigned long long *size);
- extern void elfcorehdr_free(unsigned long long addr);
- extern ssize_t elfcorehdr_read(char *buf, size_t count, u64 *ppos);
- extern ssize_t elfcorehdr_read_notes(char *buf, size_t count, u64 *ppos);
-+void elfcorehdr_fill_device_ram_ptload_elf64(Elf64_Phdr *phdr,
-+		unsigned long long paddr, unsigned long long size);
- extern int remap_oldmem_pfn_range(struct vm_area_struct *vma,
- 				  unsigned long from, unsigned long pfn,
- 				  unsigned long size, pgprot_t prot);
-@@ -50,6 +54,7 @@ void vmcore_cleanup(void);
- #define vmcore_elf64_check_arch(x) (elf_check_arch(x) || vmcore_elf_check_arch_cross(x))
- #endif
- 
-+#ifndef is_kdump_kernel
- /*
-  * is_kdump_kernel() checks whether this kernel is booting after a panic of
-  * previous kernel or not. This is determined by checking if previous kernel
-@@ -64,6 +69,7 @@ static inline bool is_kdump_kernel(void)
- {
- 	return elfcorehdr_addr != ELFCORE_ADDR_MAX;
- }
-+#endif
- 
- /* is_vmcore_usable() checks if the kernel is booting after a panic and
-  * the vmcore region is usable.
-@@ -75,7 +81,8 @@ static inline bool is_kdump_kernel(void)
- 
- static inline int is_vmcore_usable(void)
- {
-	return is_kdump_kernel() && elfcorehdr_addr != ELFCORE_ADDR_ERR ? 1 : 0;
-+	return elfcorehdr_addr != ELFCORE_ADDR_ERR &&
-+		elfcorehdr_addr != ELFCORE_ADDR_MAX ? 1 : 0;
- }
- 
- /* vmcore_unusable() marks the vmcore as unusable,
-@@ -84,8 +91,7 @@ static inline int is_vmcore_usable(void)
- 
- static inline void vmcore_unusable(void)
- {
-	if (is_kdump_kernel())
-		elfcorehdr_addr = ELFCORE_ADDR_ERR;
-+	elfcorehdr_addr = ELFCORE_ADDR_ERR;
- }
- 
- /**
-@@ -97,6 +103,12 @@ static inline void vmcore_unusable(void)
-  *              indicated in the vmcore instead. For example, a ballooned page
-  *              contains no data and reading from such a page will cause high
-  *              load in the hypervisor.
-+ * @get_device_ram: query RAM ranges that can only be detected by device
-+ *   drivers, such as the virtio-mem driver, so they can be included in
-+ *   the crash dump on architectures that allocate the elfcore hdr in the dump
-+ *   ("2nd") kernel. Indicated RAM ranges may contain holes to reduce the
-+ *   total number of ranges; such holes can be detected using the pfn_is_ram
-+ *   callback just like for other RAM.
-  * @next: List head to manage registered callbacks internally; initialized by
-  *        register_vmcore_cb().
-  *
-@@ -107,11 +119,44 @@ static inline void vmcore_unusable(void)
-  */
- struct vmcore_cb {
- 	bool (*pfn_is_ram)(struct vmcore_cb *cb, unsigned long pfn);
-+	int (*get_device_ram)(struct vmcore_cb *cb, struct list_head *list);
- 	struct list_head next;
- };
- extern void register_vmcore_cb(struct vmcore_cb *cb);
- extern void unregister_vmcore_cb(struct vmcore_cb *cb);
- 
-+struct vmcore_range {
-+	struct list_head list;
-+	unsigned long long paddr;
-+	unsigned long long size;
-+	loff_t offset;
-+};
-+
-+/* Allocate a vmcore range and add it to the list. */
-+static inline int vmcore_alloc_add_range(struct list_head *list,
-+		unsigned long long paddr, unsigned long long size)
-+{
-+	struct vmcore_range *m = kzalloc(sizeof(*m), GFP_KERNEL);
-+
-+	if (!m)
-+		return -ENOMEM;
-+	m->paddr = paddr;
-+	m->size = size;
-+	list_add_tail(&m->list, list);
-+	return 0;
-+}
-+
-+/* Free a list of vmcore ranges. */
-+static inline void vmcore_free_ranges(struct list_head *list)
-+{
-+	struct vmcore_range *m, *tmp;
-+
-+	list_for_each_entry_safe(m, tmp, list, list) {
-+		list_del(&m->list);
-+		kfree(m);
-+	}
-+}
-+
- #else /* !CONFIG_CRASH_DUMP */
- static inline bool is_kdump_kernel(void) { return false; }
- #endif /* CONFIG_CRASH_DUMP */
-diff --git a/include/linux/crash_reserve.h b/include/linux/crash_reserve.h
-new file mode 100644
-index 000000000000..f0dc03d94ca2
--- /dev/null
-+++ b/include/linux/crash_reserve.h
-@@ -0,0 +1,66 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef LINUX_CRASH_RESERVE_H
-+#define LINUX_CRASH_RESERVE_H
-+
-+#include <linux/linkage.h>
-+#include <linux/elfcore.h>
-+#include <linux/elf.h>
-+#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION
-+#include <asm/crash_reserve.h>
-+#endif
-+
-+/* Location of a reserved region to hold the crash kernel.
-+ */
-+extern struct resource crashk_res;
-+extern struct resource crashk_low_res;
-+extern struct range crashk_cma_ranges[];
-+#if defined(CONFIG_CMA) && defined(CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION)
-+#define CRASHKERNEL_CMA
-+#define CRASHKERNEL_CMA_RANGES_MAX 4
-+extern int crashk_cma_cnt;
-+#else
-+#define crashk_cma_cnt 0
-+#define CRASHKERNEL_CMA_RANGES_MAX 0
-+#endif
-+
-+
-+int __init parse_crashkernel(char *cmdline, unsigned long long system_ram,
-+		unsigned long long *crash_size, unsigned long long *crash_base,
-+		unsigned long long *low_size, unsigned long long *cma_size,
-+		bool *high);
-+
-+void __init reserve_crashkernel_cma(unsigned long long cma_size);
-+
-+#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION
-+#ifndef arch_add_crash_res_to_iomem
-+static inline bool arch_add_crash_res_to_iomem(void)
-+{
-+	return true;
-+}
-+#endif
-+#ifndef DEFAULT_CRASH_KERNEL_LOW_SIZE
-+#define DEFAULT_CRASH_KERNEL_LOW_SIZE	(128UL << 20)
-+#endif
-+#ifndef CRASH_ALIGN
-+#define CRASH_ALIGN			SZ_2M
-+#endif
-+#ifndef CRASH_ADDR_LOW_MAX
-+#define CRASH_ADDR_LOW_MAX		SZ_4G
-+#endif
-+#ifndef CRASH_ADDR_HIGH_MAX
-+#define CRASH_ADDR_HIGH_MAX		memblock_end_of_DRAM()
-+#endif
-+
-+void __init reserve_crashkernel_generic(unsigned long long crash_size,
-+					unsigned long long crash_base,
-+					unsigned long long crash_low_size,
-+					bool high);
-+#else
-+static inline void __init reserve_crashkernel_generic(
-+		unsigned long long crash_size,
-+		unsigned long long crash_base,
-+		unsigned long long crash_low_size,
-+		bool high)
-+{}
-+#endif
-+#endif /* LINUX_CRASH_RESERVE_H */
-diff --git a/include/linux/crc-ccitt.h b/include/linux/crc-ccitt.h
-index 72c92c396bb8..cd4f420231ba 100644
--- a/include/linux/crc-ccitt.h
-+++ b/include/linux/crc-ccitt.h
-@@ -5,19 +5,12 @@
- #include <linux/types.h>
- 
- extern u16 const crc_ccitt_table[256];
-extern u16 const crc_ccitt_false_table[256];
- 
- extern u16 crc_ccitt(u16 crc, const u8 *buffer, size_t len);
-extern u16 crc_ccitt_false(u16 crc, const u8 *buffer, size_t len);
- 
- static inline u16 crc_ccitt_byte(u16 crc, const u8 c)
- {
- 	return (crc >> 8) ^ crc_ccitt_table[(crc ^ c) & 0xff];
- }
- 
-static inline u16 crc_ccitt_false_byte(u16 crc, const u8 c)
-{
-    return (crc << 8) ^ crc_ccitt_false_table[(crc >> 8) ^ c];
-}
-
- #endif /* _LINUX_CRC_CCITT_H */
-diff --git a/include/linux/crc-t10dif.h b/include/linux/crc-t10dif.h
-index 6bb0c0bf357b..ecc8bc2dd7f4 100644
--- a/include/linux/crc-t10dif.h
-+++ b/include/linux/crc-t10dif.h
-@@ -4,13 +4,11 @@
- 
- #include <linux/types.h>
- 
-#define CRC_T10DIF_DIGEST_SIZE 2
-#define CRC_T10DIF_BLOCK_SIZE 1
-#define CRC_T10DIF_STRING "crct10dif"
-+u16 crc_t10dif_update(u16 crc, const u8 *p, size_t len);
- 
-extern __u16 crc_t10dif_generic(__u16 crc, const unsigned char *buffer,
-				size_t len);
-extern __u16 crc_t10dif(unsigned char const *, size_t);
-extern __u16 crc_t10dif_update(__u16 crc, unsigned char const *, size_t);
-+static inline u16 crc_t10dif(const u8 *p, size_t len)
-+{
-+	return crc_t10dif_update(0, p, len);
-+}
- 
- #endif
-diff --git a/include/linux/crc16.h b/include/linux/crc16.h
-index 9fa74529b317..b861d969b161 100644
--- a/include/linux/crc16.h
-+++ b/include/linux/crc16.h
-@@ -15,14 +15,7 @@
- 
- #include <linux/types.h>
- 
-extern u16 const crc16_table[256];
-
-extern u16 crc16(u16 crc, const u8 *buffer, size_t len);
-
-static inline u16 crc16_byte(u16 crc, const u8 data)
-{
-	return (crc >> 8) ^ crc16_table[(crc ^ data) & 0xff];
-}
-+u16 crc16(u16 crc, const u8 *p, size_t len);
- 
- #endif /* __CRC16_H */
- 
-diff --git a/include/linux/crc32.h b/include/linux/crc32.h
-index 9e8a032c1788..da78b215ff2e 100644
--- a/include/linux/crc32.h
-+++ b/include/linux/crc32.h
-@@ -1,69 +1,100 @@
-/*
- * crc32.h
- * See linux/lib/crc32.c for license and changes
- */
-+/* SPDX-License-Identifier: GPL-2.0-only */
- #ifndef _LINUX_CRC32_H
- #define _LINUX_CRC32_H
- 
- #include <linux/types.h>
- #include <linux/bitrev.h>
- 
-u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len);
-u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len);
-
- /**
- * crc32_le_combine - Combine two crc32 check values into one. For two
- * 		      sequences of bytes, seq1 and seq2 with lengths len1
- * 		      and len2, crc32_le() check values were calculated
- * 		      for each, crc1 and crc2.
-+ * crc32_le() - Compute least-significant-bit-first IEEE CRC-32
-+ * @crc: Initial CRC value.  ~0 (recommended) or 0 for a new CRC computation, or
-+ *	 the previous CRC value if computing incrementally.
-+ * @p: Pointer to the data buffer
-+ * @len: Length of data in bytes
-+ *
-+ * This implements the CRC variant that is often known as the IEEE CRC-32, or
-+ * simply CRC-32, and is widely used in Ethernet and other applications:
-+ *
-+ * - Polynomial: x^32 + x^26 + x^23 + x^22 + x^16 + x^12 + x^11 + x^10 + x^8 +
-+ *		 x^7 + x^5 + x^4 + x^2 + x^1 + x^0
-+ * - Bit order: Least-significant-bit-first
-+ * - Polynomial in integer form: 0xedb88320
-  *
- * @crc1: crc32 of the first block
- * @crc2: crc32 of the second block
- * @len2: length of the second block
-+ * This does *not* invert the CRC at the beginning or end.  The caller is
-+ * expected to do that if it needs to.  Inverting at both ends is recommended.
-  *
- * Return: The crc32_le() check value of seq1 and seq2 concatenated,
- * 	   requiring only crc1, crc2, and len2. Note: If seq_full denotes
- * 	   the concatenated memory area of seq1 with seq2, and crc_full
- * 	   the crc32_le() value of seq_full, then crc_full ==
- * 	   crc32_le_combine(crc1, crc2, len2) when crc_full was seeded
- * 	   with the same initializer as crc1, and crc2 seed was 0. See
- * 	   also crc32_combine_test().
-+ * For new applications, prefer to use CRC-32C instead.  See crc32c().
-+ *
-+ * Context: Any context
-+ * Return: The new CRC value
-  */
-u32 __attribute_const__ crc32_le_shift(u32 crc, size_t len);
-+u32 crc32_le(u32 crc, const void *p, size_t len);
- 
-static inline u32 crc32_le_combine(u32 crc1, u32 crc2, size_t len2)
-+/* This is just an alias for crc32_le(). */
-+static inline u32 crc32(u32 crc, const void *p, size_t len)
- {
-	return crc32_le_shift(crc1, len2) ^ crc2;
-+	return crc32_le(crc, p, len);
- }
- 
-u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len);
-
- /**
- * __crc32c_le_combine - Combine two crc32c check values into one. For two
- * 			 sequences of bytes, seq1 and seq2 with lengths len1
- * 			 and len2, __crc32c_le() check values were calculated
- * 			 for each, crc1 and crc2.
-+ * crc32_be() - Compute most-significant-bit-first IEEE CRC-32
-+ * @crc: Initial CRC value.  ~0 (recommended) or 0 for a new CRC computation, or
-+ *	 the previous CRC value if computing incrementally.
-+ * @p: Pointer to the data buffer
-+ * @len: Length of data in bytes
-  *
- * @crc1: crc32c of the first block
- * @crc2: crc32c of the second block
- * @len2: length of the second block
-+ * crc32_be() is the same as crc32_le() except that crc32_be() computes the
-+ * *most-significant-bit-first* variant of the CRC.  I.e., within each byte, the
-+ * most significant bit is processed first (treated as highest order polynomial
-+ * coefficient).  The same bit order is also used for the CRC value itself:
-  *
- * Return: The __crc32c_le() check value of seq1 and seq2 concatenated,
- * 	   requiring only crc1, crc2, and len2. Note: If seq_full denotes
- * 	   the concatenated memory area of seq1 with seq2, and crc_full
- * 	   the __crc32c_le() value of seq_full, then crc_full ==
- * 	   __crc32c_le_combine(crc1, crc2, len2) when crc_full was
- * 	   seeded with the same initializer as crc1, and crc2 seed
- * 	   was 0. See also crc32c_combine_test().
-+ * - Polynomial: x^32 + x^26 + x^23 + x^22 + x^16 + x^12 + x^11 + x^10 + x^8 +
-+ *		 x^7 + x^5 + x^4 + x^2 + x^1 + x^0
-+ * - Bit order: Most-significant-bit-first
-+ * - Polynomial in integer form: 0x04c11db7
-+ *
-+ * Context: Any context
-+ * Return: The new CRC value
-  */
-u32 __attribute_const__ __crc32c_le_shift(u32 crc, size_t len);
-+u32 crc32_be(u32 crc, const void *p, size_t len);
- 
-static inline u32 __crc32c_le_combine(u32 crc1, u32 crc2, size_t len2)
-{
-	return __crc32c_le_shift(crc1, len2) ^ crc2;
-}
-+/**
-+ * crc32c() - Compute CRC-32C
-+ * @crc: Initial CRC value.  ~0 (recommended) or 0 for a new CRC computation, or
-+ *	 the previous CRC value if computing incrementally.
-+ * @p: Pointer to the data buffer
-+ * @len: Length of data in bytes
-+ *
-+ * This implements CRC-32C, i.e. the Castagnoli CRC.  This is the recommended
-+ * CRC variant to use in new applications that want a 32-bit CRC.
-+ *
-+ * - Polynomial: x^32 + x^28 + x^27 + x^26 + x^25 + x^23 + x^22 + x^20 + x^19 +
-+ *		 x^18 + x^14 + x^13 + x^11 + x^10 + x^9 + x^8 + x^6 + x^0
-+ * - Bit order: Least-significant-bit-first
-+ * - Polynomial in integer form: 0x82f63b78
-+ *
-+ * This does *not* invert the CRC at the beginning or end.  The caller is
-+ * expected to do that if it needs to.  Inverting at both ends is recommended.
-+ *
-+ * Context: Any context
-+ * Return: The new CRC value
-+ */
-+u32 crc32c(u32 crc, const void *p, size_t len);
- 
-#define crc32(seed, data, length)  crc32_le(seed, (unsigned char const *)(data), length)
-+/*
-+ * crc32_optimizations() returns flags that indicate which CRC32 library
-+ * functions are using architecture-specific optimizations.  Unlike
-+ * IS_ENABLED(CONFIG_CRC32_ARCH) it takes into account the different CRC32
-+ * variants and also whether any needed CPU features are available at runtime.
-+ */
-+#define CRC32_LE_OPTIMIZATION	BIT(0) /* crc32_le() is optimized */
-+#define CRC32_BE_OPTIMIZATION	BIT(1) /* crc32_be() is optimized */
-+#define CRC32C_OPTIMIZATION	BIT(2) /* crc32c() is optimized */
-+#if IS_ENABLED(CONFIG_CRC32_ARCH)
-+u32 crc32_optimizations(void);
-+#else
-+static inline u32 crc32_optimizations(void) { return 0; }
-+#endif
- 
- /*
-  * Helpers for hash table generation of ethernet nics:
-diff --git a/include/linux/crc32c.h b/include/linux/crc32c.h
-index bd21af828ff6..b8cff2f4309a 100644
--- a/include/linux/crc32c.h
-+++ b/include/linux/crc32c.h
-@@ -2,12 +2,6 @@
- #ifndef _LINUX_CRC32C_H
- #define _LINUX_CRC32C_H
- 
-#include <linux/types.h>
-
-extern u32 crc32c(u32 crc, const void *address, unsigned int length);
-extern const char *crc32c_impl(void);
-
-/* This macro exists for backwards-compatibility. */
-#define crc32c_le crc32c
-+#include <linux/crc32.h>
- 
- #endif	/* _LINUX_CRC32C_H */
-diff --git a/include/linux/crc32poly.h b/include/linux/crc32poly.h
-index 62c4b7790a28..ccab711295fa 100644
--- a/include/linux/crc32poly.h
-+++ b/include/linux/crc32poly.h
-@@ -2,19 +2,13 @@
- #ifndef _LINUX_CRC32_POLY_H
- #define _LINUX_CRC32_POLY_H
- 
-/*
- * There are multiple 16-bit CRC polynomials in common use, but this is
- * *the* standard CRC-32 polynomial, first popularized by Ethernet.
- * x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x^1+x^0
- */
-+/* The polynomial used by crc32_le(), in integer form.  See crc32_le(). */
- #define CRC32_POLY_LE 0xedb88320
-+
-+/* The polynomial used by crc32_be(), in integer form.  See crc32_be(). */
- #define CRC32_POLY_BE 0x04c11db7
- 
-/*
- * This is the CRC32c polynomial, as outlined by Castagnoli.
- * x^32+x^28+x^27+x^26+x^25+x^23+x^22+x^20+x^19+x^18+x^14+x^13+x^11+x^10+x^9+
- * x^8+x^6+x^0
- */
-#define CRC32C_POLY_LE 0x82F63B78
-+/* The polynomial used by crc32c(), in integer form.  See crc32c(). */
-+#define CRC32C_POLY_LE 0x82f63b78
- 
- #endif /* _LINUX_CRC32_POLY_H */
-diff --git a/include/linux/crc64.h b/include/linux/crc64.h
-index e044c60d1e61..fc0c06ab1993 100644
--- a/include/linux/crc64.h
-+++ b/include/linux/crc64.h
-@@ -1,18 +1,28 @@
- /* SPDX-License-Identifier: GPL-2.0 */
-/*
- * See lib/crc64.c for the related specification and polynomial arithmetic.
- */
- #ifndef _LINUX_CRC64_H
- #define _LINUX_CRC64_H
- 
- #include <linux/types.h>
- 
-#define CRC64_ROCKSOFT_STRING "crc64-rocksoft"
-
-u64 __pure crc64_be(u64 crc, const void *p, size_t len);
-u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len);
-+/**
-+ * crc64_be - Calculate bitwise big-endian ECMA-182 CRC64
-+ * @crc: seed value for computation. 0 or (u64)~0 for a new CRC calculation,
-+ *       or the previous crc64 value if computing incrementally.
-+ * @p: pointer to buffer over which CRC64 is run
-+ * @len: length of buffer @p
-+ */
-+u64 crc64_be(u64 crc, const void *p, size_t len);
- 
-u64 crc64_rocksoft(const unsigned char *buffer, size_t len);
-u64 crc64_rocksoft_update(u64 crc, const unsigned char *buffer, size_t len);
-+/**
-+ * crc64_nvme - Calculate CRC64-NVME
-+ * @crc: seed value for computation. 0 for a new CRC calculation, or the
-+ *	 previous crc64 value if computing incrementally.
-+ * @p: pointer to buffer over which CRC64 is run
-+ * @len: length of buffer @p
-+ *
-+ * This computes the CRC64 defined in the NVME NVM Command Set Specification,
-+ * *including the bitwise inversion at the beginning and end*.
-+ */
-+u64 crc64_nvme(u64 crc, const void *p, size_t len);
- 
- #endif /* _LINUX_CRC64_H */
-diff --git a/include/linux/crc7.h b/include/linux/crc7.h
-index b462842f3c32..61d34749e437 100644
--- a/include/linux/crc7.h
-+++ b/include/linux/crc7.h
-@@ -3,13 +3,6 @@
- #define _LINUX_CRC7_H
- #include <linux/types.h>
- 
-extern const u8 crc7_be_syndrome_table[256];
-
-static inline u8 crc7_be_byte(u8 crc, u8 data)
-{
-	return crc7_be_syndrome_table[crc ^ data];
-}
-
- extern u8 crc7_be(u8 crc, const u8 *buffer, size_t len);
- 
- #endif
-diff --git a/include/linux/cred.h b/include/linux/cred.h
-index 9ed9232af934..343a140a6ba2 100644
--- a/include/linux/cred.h
-+++ b/include/linux/cred.h
-@@ -12,6 +12,7 @@
- #include <linux/init.h>
- #include <linux/key.h>
- #include <linux/atomic.h>
-+#include <linux/refcount.h>
- #include <linux/uidgid.h>
- #include <linux/sched.h>
- #include <linux/sched/user.h>
-@@ -19,11 +20,13 @@
- struct cred;
- struct inode;
- 
-+extern struct task_struct init_task;
-+
- /*
-  * COW Supplementary groups list
-  */
- struct group_info {
-	atomic_t	usage;
-+	refcount_t	usage;
- 	int		ngroups;
- 	kgid_t		gid[];
- } __randomize_layout;
-@@ -39,7 +42,7 @@ struct group_info {
-  */
- static inline struct group_info *get_group_info(struct group_info *gi)
- {
-	atomic_inc(&gi->usage);
-+	refcount_inc(&gi->usage);
- 	return gi;
- }
- 
-@@ -49,7 +52,7 @@ static inline struct group_info *get_group_info(struct group_info *gi)
-  */
- #define put_group_info(group_info)			\
- do {							\
-	if (atomic_dec_and_test(&(group_info)->usage))	\
-+	if (refcount_dec_and_test(&(group_info)->usage))	\
- 		groups_free(group_info);		\
- } while (0)
- 
-@@ -108,14 +111,7 @@ static inline int groups_search(const struct group_info *group_info, kgid_t grp)
-  * same context as task->real_cred.
-  */
- struct cred {
-	atomic_t	usage;
-#ifdef CONFIG_DEBUG_CREDENTIALS
-	atomic_t	subscribers;	/* number of processes subscribed */
-	void		*put_addr;
-	unsigned	magic;
-#define CRED_MAGIC	0x43736564
-#define CRED_MAGIC_DEAD	0x44656144
-#endif
-+	atomic_long_t	usage;
- 	kuid_t		uid;		/* real UID of the task */
- 	kgid_t		gid;		/* real GID of the task */
- 	kuid_t		suid;		/* saved UID of the task */
-@@ -154,17 +150,19 @@ struct cred {
- 
- extern void __put_cred(struct cred *);
- extern void exit_creds(struct task_struct *);
-extern int copy_creds(struct task_struct *, unsigned long);
-+extern int copy_creds(struct task_struct *, u64);
- extern const struct cred *get_task_cred(struct task_struct *);
- extern struct cred *cred_alloc_blank(void);
- extern struct cred *prepare_creds(void);
- extern struct cred *prepare_exec_creds(void);
- extern int commit_creds(struct cred *);
- extern void abort_creds(struct cred *);
-extern const struct cred *override_creds(const struct cred *);
-extern void revert_creds(const struct cred *);
- extern struct cred *prepare_kernel_cred(struct task_struct *);
-extern int change_create_files_as(struct cred *, struct inode *);
-+static inline const struct cred *kernel_cred(void)
-+{
-+	/* shut up sparse */
-+	return rcu_dereference_raw(init_task.cred);
-+}
- extern int set_security_override(struct cred *, u32);
- extern int set_security_override_from_ctx(struct cred *, const char *);
- extern int set_create_files_as(struct cred *, struct inode *);
-@@ -172,46 +170,6 @@ extern int cred_fscmp(const struct cred *, const struct cred *);
- extern void __init cred_init(void);
- extern int set_cred_ucounts(struct cred *);
- 
-/*
- * check for validity of credentials
- */
-#ifdef CONFIG_DEBUG_CREDENTIALS
-extern void __noreturn __invalid_creds(const struct cred *, const char *, unsigned);
-extern void __validate_process_creds(struct task_struct *,
-				     const char *, unsigned);
-
-extern bool creds_are_invalid(const struct cred *cred);
-
-static inline void __validate_creds(const struct cred *cred,
-				    const char *file, unsigned line)
-{
-	if (unlikely(creds_are_invalid(cred)))
-		__invalid_creds(cred, file, line);
-}
-
-#define validate_creds(cred)				\
-do {							\
-	__validate_creds((cred), __FILE__, __LINE__);	\
-} while(0)
-
-#define validate_process_creds()				\
-do {								\
-	__validate_process_creds(current, __FILE__, __LINE__);	\
-} while(0)
-
-extern void validate_creds_for_do_exit(struct task_struct *);
-#else
-static inline void validate_creds(const struct cred *cred)
-{
-}
-static inline void validate_creds_for_do_exit(struct task_struct *tsk)
-{
-}
-static inline void validate_process_creds(void)
-{
-}
-#endif
-
- static inline bool cap_ambient_invariant_ok(const struct cred *cred)
- {
- 	return cap_issubset(cred->cap_ambient,
-@@ -219,25 +177,33 @@ static inline bool cap_ambient_invariant_ok(const struct cred *cred)
- 					  cred->cap_inheritable));
- }
- 
-/**
- * get_new_cred - Get a reference on a new set of credentials
- * @cred: The new credentials to reference
- *
- * Get a reference on the specified set of new credentials.  The caller must
- * release the reference.
- */
-static inline struct cred *get_new_cred(struct cred *cred)
-+static inline const struct cred *override_creds(const struct cred *override_cred)
- {
-	atomic_inc(&cred->usage);
-	return cred;
-+	return rcu_replace_pointer(current->cred, override_cred, 1);
-+}
-+
-+static inline const struct cred *revert_creds(const struct cred *revert_cred)
-+{
-+	return rcu_replace_pointer(current->cred, revert_cred, 1);
- }
- 
-+DEFINE_CLASS(override_creds,
-+	     const struct cred *,
-+	     revert_creds(_T),
-+	     override_creds(override_cred), const struct cred *override_cred)
-+
-+#define scoped_with_creds(cred) \
-+	scoped_class(override_creds, __UNIQUE_ID(label), cred)
-+
-+#define scoped_with_kernel_creds() scoped_with_creds(kernel_cred())
-+
- /**
- * get_cred - Get a reference on a set of credentials
-+ * get_cred_many - Get references on a set of credentials
-  * @cred: The credentials to reference
-+ * @nr: Number of references to acquire
-  *
- * Get a reference on the specified set of credentials.  The caller must
- * release the reference.  If %NULL is passed, it is returned with no action.
-+ * Get references on the specified set of credentials.  The caller must release
-+ * all acquired reference.  If %NULL is passed, it is returned with no action.
-  *
-  * This is used to deal with a committed set of credentials.  Although the
-  * pointer is const, this will temporarily discard the const and increment the
-@@ -245,14 +211,28 @@ static inline struct cred *get_new_cred(struct cred *cred)
-  * accidental alteration of a set of credentials that should be considered
-  * immutable.
-  */
-static inline const struct cred *get_cred(const struct cred *cred)
-+static inline const struct cred *get_cred_many(const struct cred *cred, int nr)
- {
- 	struct cred *nonconst_cred = (struct cred *) cred;
- 	if (!cred)
- 		return cred;
-	validate_creds(cred);
- 	nonconst_cred->non_rcu = 0;
-	return get_new_cred(nonconst_cred);
-+	atomic_long_add(nr, &nonconst_cred->usage);
-+	return cred;
-+}
-+
-+/*
-+ * get_cred - Get a reference on a set of credentials
-+ * @cred: The credentials to reference
-+ *
-+ * Get a reference on the specified set of credentials.  The caller must
-+ * release the reference.  If %NULL is passed, it is returned with no action.
-+ *
-+ * This is used to deal with a committed set of credentials.
-+ */
-+static inline const struct cred *get_cred(const struct cred *cred)
-+{
-+	return get_cred_many(cred, 1);
- }
- 
- static inline const struct cred *get_cred_rcu(const struct cred *cred)
-@@ -260,9 +240,8 @@ static inline const struct cred *get_cred_rcu(const struct cred *cred)
- 	struct cred *nonconst_cred = (struct cred *) cred;
- 	if (!cred)
- 		return NULL;
-	if (!atomic_inc_not_zero(&nonconst_cred->usage))
-+	if (!atomic_long_inc_not_zero(&nonconst_cred->usage))
- 		return NULL;
-	validate_creds(cred);
- 	nonconst_cred->non_rcu = 0;
- 	return cred;
- }
-@@ -270,6 +249,7 @@ static inline const struct cred *get_cred_rcu(const struct cred *cred)
- /**
-  * put_cred - Release a reference to a set of credentials
-  * @cred: The credentials to release
-+ * @nr: Number of references to release
-  *
-  * Release a reference to a set of credentials, deleting them when the last ref
-  * is released.  If %NULL is passed, nothing is done.
-@@ -278,17 +258,35 @@ static inline const struct cred *get_cred_rcu(const struct cred *cred)
-  * on task_struct are attached by const pointers to prevent accidental
-  * alteration of otherwise immutable credential sets.
-  */
-static inline void put_cred(const struct cred *_cred)
-+static inline void put_cred_many(const struct cred *_cred, int nr)
- {
- 	struct cred *cred = (struct cred *) _cred;
- 
- 	if (cred) {
-		validate_creds(cred);
-		if (atomic_dec_and_test(&(cred)->usage))
-+		if (atomic_long_sub_and_test(nr, &cred->usage))
- 			__put_cred(cred);
- 	}
- }
- 
-+/*
-+ * put_cred - Release a reference to a set of credentials
-+ * @cred: The credentials to release
-+ *
-+ * Release a reference to a set of credentials, deleting them when the last ref
-+ * is released.  If %NULL is passed, nothing is done.
-+ */
-+static inline void put_cred(const struct cred *cred)
-+{
-+	put_cred_many(cred, 1);
-+}
-+
-+DEFINE_CLASS(prepare_creds,
-+	      struct cred *,
-+	      if (_T) put_cred(_T),
-+	      prepare_creds(), void)
-+
-+DEFINE_FREE(put_cred, struct cred *, if (!IS_ERR_OR_NULL(_T)) put_cred(_T))
-+
- /**
-  * current_cred - Access the current task's subjective credentials
-  *
-diff --git a/include/linux/crypto.h b/include/linux/crypto.h
-index 2324ab6f1846..a2137e19be7d 100644
--- a/include/linux/crypto.h
-+++ b/include/linux/crypto.h
-@@ -12,46 +12,30 @@
- #ifndef _LINUX_CRYPTO_H
- #define _LINUX_CRYPTO_H
- 
-#include <linux/atomic.h>
-#include <linux/kernel.h>
-#include <linux/list.h>
-#include <linux/bug.h>
-#include <linux/refcount.h>
-#include <linux/slab.h>
- #include <linux/completion.h>
-
-/*
- * Autoloaded crypto modules should only use a prefixed name to avoid allowing
- * arbitrary modules to be loaded. Loading from userspace may still need the
- * unprefixed names, so retains those aliases as well.
- * This uses __MODULE_INFO directly instead of MODULE_ALIAS because pre-4.3
- * gcc (e.g. avr32 toolchain) uses __LINE__ for uniqueness, and this macro
- * expands twice on the same line. Instead, use a separate base name for the
- * alias.
- */
-#define MODULE_ALIAS_CRYPTO(name)	\
-		__MODULE_INFO(alias, alias_userspace, name);	\
-		__MODULE_INFO(alias, alias_crypto, "crypto-" name)
-+#include <linux/errno.h>
-+#include <linux/refcount_types.h>
-+#include <linux/slab.h>
-+#include <linux/types.h>
- 
- /*
-  * Algorithm masks and types.
-  */
- #define CRYPTO_ALG_TYPE_MASK		0x0000000f
- #define CRYPTO_ALG_TYPE_CIPHER		0x00000001
-#define CRYPTO_ALG_TYPE_COMPRESS	0x00000002
- #define CRYPTO_ALG_TYPE_AEAD		0x00000003
-+#define CRYPTO_ALG_TYPE_LSKCIPHER	0x00000004
- #define CRYPTO_ALG_TYPE_SKCIPHER	0x00000005
-+#define CRYPTO_ALG_TYPE_AKCIPHER	0x00000006
-+#define CRYPTO_ALG_TYPE_SIG		0x00000007
- #define CRYPTO_ALG_TYPE_KPP		0x00000008
- #define CRYPTO_ALG_TYPE_ACOMPRESS	0x0000000a
- #define CRYPTO_ALG_TYPE_SCOMPRESS	0x0000000b
- #define CRYPTO_ALG_TYPE_RNG		0x0000000c
-#define CRYPTO_ALG_TYPE_AKCIPHER	0x0000000d
- #define CRYPTO_ALG_TYPE_HASH		0x0000000e
- #define CRYPTO_ALG_TYPE_SHASH		0x0000000e
- #define CRYPTO_ALG_TYPE_AHASH		0x0000000f
- 
-#define CRYPTO_ALG_TYPE_HASH_MASK	0x0000000e
-#define CRYPTO_ALG_TYPE_AHASH_MASK	0x0000000e
- #define CRYPTO_ALG_TYPE_ACOMPRESS_MASK	0x0000000e
- 
- #define CRYPTO_ALG_LARVAL		0x00000010
-@@ -66,6 +50,15 @@
- #define CRYPTO_ALG_NEED_FALLBACK	0x00000100
- 
- /*
-+ * Set if the algorithm data structure should be duplicated into
-+ * kmalloc memory before registration.  This is useful for hardware
-+ * that can be disconnected at will.  Do not use this if the data
-+ * structure is embedded into a bigger one.  Duplicate the overall
-+ * data structure in the driver in that case.
-+ */
-+#define CRYPTO_ALG_DUP_FIRST		0x00000200
-+
-+/*
-  * Set if the algorithm has passed automated run-time testing.  Note that
-  * if there is no run-time testing for a given algorithm it is considered
-  * to have passed.
-@@ -126,7 +119,6 @@
-  *	  crypto_aead_walksize() (with the remainder going at the end), no chunk
-  *	  can cross a page boundary or a scatterlist element boundary.
-  *    ahash:
- *	- The result buffer must be aligned to the algorithm's alignmask.
-  *	- crypto_ahash_finup() must not be used unless the algorithm implements
-  *	  ->finup() natively.
-  */
-@@ -141,6 +133,14 @@
-  */
- #define CRYPTO_ALG_FIPS_INTERNAL	0x00020000
- 
-+/* Set if the algorithm supports virtual addresses. */
-+#define CRYPTO_ALG_REQ_VIRT		0x00040000
-+
-+/* Set if the algorithm cannot have a fallback (e.g., phmac). */
-+#define CRYPTO_ALG_NO_FALLBACK		0x00080000
-+
-+/* The high bits 0xff000000 are reserved for type-specific flags. */
-+
- /*
-  * Transform masks and values (for crt_flags).
-  */
-@@ -150,6 +150,7 @@
- #define CRYPTO_TFM_REQ_FORBID_WEAK_KEYS	0x00000100
- #define CRYPTO_TFM_REQ_MAY_SLEEP	0x00000200
- #define CRYPTO_TFM_REQ_MAY_BACKLOG	0x00000400
-+#define CRYPTO_TFM_REQ_ON_STACK		0x00000800
- 
- /*
-  * Miscellaneous stuff.
-@@ -171,12 +172,11 @@
- 
- #define CRYPTO_MINALIGN_ATTR __attribute__ ((__aligned__(CRYPTO_MINALIGN)))
- 
-struct scatterlist;
-struct crypto_async_request;
- struct crypto_tfm;
- struct crypto_type;
-+struct module;
- 
-typedef void (*crypto_completion_t)(struct crypto_async_request *req, int err);
-+typedef void (*crypto_completion_t)(void *req, int err);
- 
- /**
-  * DOC: Block Cipher Context Data Structures
-@@ -257,136 +257,7 @@ struct cipher_alg {
- 	void (*cia_decrypt)(struct crypto_tfm *tfm, u8 *dst, const u8 *src);
- };
- 
-/**
- * struct compress_alg - compression/decompression algorithm
- * @coa_compress: Compress a buffer of specified length, storing the resulting
- *		  data in the specified buffer. Return the length of the
- *		  compressed data in dlen.
- * @coa_decompress: Decompress the source buffer, storing the uncompressed
- *		    data in the specified buffer. The length of the data is
- *		    returned in dlen.
- *
- * All fields are mandatory.
- */
-struct compress_alg {
-	int (*coa_compress)(struct crypto_tfm *tfm, const u8 *src,
-			    unsigned int slen, u8 *dst, unsigned int *dlen);
-	int (*coa_decompress)(struct crypto_tfm *tfm, const u8 *src,
-			      unsigned int slen, u8 *dst, unsigned int *dlen);
-};
-
-#ifdef CONFIG_CRYPTO_STATS
-/*
- * struct crypto_istat_aead - statistics for AEAD algorithm
- * @encrypt_cnt:	number of encrypt requests
- * @encrypt_tlen:	total data size handled by encrypt requests
- * @decrypt_cnt:	number of decrypt requests
- * @decrypt_tlen:	total data size handled by decrypt requests
- * @err_cnt:		number of error for AEAD requests
- */
-struct crypto_istat_aead {
-	atomic64_t encrypt_cnt;
-	atomic64_t encrypt_tlen;
-	atomic64_t decrypt_cnt;
-	atomic64_t decrypt_tlen;
-	atomic64_t err_cnt;
-};
-
-/*
- * struct crypto_istat_akcipher - statistics for akcipher algorithm
- * @encrypt_cnt:	number of encrypt requests
- * @encrypt_tlen:	total data size handled by encrypt requests
- * @decrypt_cnt:	number of decrypt requests
- * @decrypt_tlen:	total data size handled by decrypt requests
- * @verify_cnt:		number of verify operation
- * @sign_cnt:		number of sign requests
- * @err_cnt:		number of error for akcipher requests
- */
-struct crypto_istat_akcipher {
-	atomic64_t encrypt_cnt;
-	atomic64_t encrypt_tlen;
-	atomic64_t decrypt_cnt;
-	atomic64_t decrypt_tlen;
-	atomic64_t verify_cnt;
-	atomic64_t sign_cnt;
-	atomic64_t err_cnt;
-};
-
-/*
- * struct crypto_istat_cipher - statistics for cipher algorithm
- * @encrypt_cnt:	number of encrypt requests
- * @encrypt_tlen:	total data size handled by encrypt requests
- * @decrypt_cnt:	number of decrypt requests
- * @decrypt_tlen:	total data size handled by decrypt requests
- * @err_cnt:		number of error for cipher requests
- */
-struct crypto_istat_cipher {
-	atomic64_t encrypt_cnt;
-	atomic64_t encrypt_tlen;
-	atomic64_t decrypt_cnt;
-	atomic64_t decrypt_tlen;
-	atomic64_t err_cnt;
-};
-
-/*
- * struct crypto_istat_compress - statistics for compress algorithm
- * @compress_cnt:	number of compress requests
- * @compress_tlen:	total data size handled by compress requests
- * @decompress_cnt:	number of decompress requests
- * @decompress_tlen:	total data size handled by decompress requests
- * @err_cnt:		number of error for compress requests
- */
-struct crypto_istat_compress {
-	atomic64_t compress_cnt;
-	atomic64_t compress_tlen;
-	atomic64_t decompress_cnt;
-	atomic64_t decompress_tlen;
-	atomic64_t err_cnt;
-};
-
-/*
- * struct crypto_istat_hash - statistics for has algorithm
- * @hash_cnt:		number of hash requests
- * @hash_tlen:		total data size hashed
- * @err_cnt:		number of error for hash requests
- */
-struct crypto_istat_hash {
-	atomic64_t hash_cnt;
-	atomic64_t hash_tlen;
-	atomic64_t err_cnt;
-};
-
-/*
- * struct crypto_istat_kpp - statistics for KPP algorithm
- * @setsecret_cnt:		number of setsecrey operation
- * @generate_public_key_cnt:	number of generate_public_key operation
- * @compute_shared_secret_cnt:	number of compute_shared_secret operation
- * @err_cnt:			number of error for KPP requests
- */
-struct crypto_istat_kpp {
-	atomic64_t setsecret_cnt;
-	atomic64_t generate_public_key_cnt;
-	atomic64_t compute_shared_secret_cnt;
-	atomic64_t err_cnt;
-};
-
-/*
- * struct crypto_istat_rng: statistics for RNG algorithm
- * @generate_cnt:	number of RNG generate requests
- * @generate_tlen:	total data size of generated data by the RNG
- * @seed_cnt:		number of times the RNG was seeded
- * @err_cnt:		number of error for RNG requests
- */
-struct crypto_istat_rng {
-	atomic64_t generate_cnt;
-	atomic64_t generate_tlen;
-	atomic64_t seed_cnt;
-	atomic64_t err_cnt;
-};
-#endif /* CONFIG_CRYPTO_STATS */
-
- #define cra_cipher	cra_u.cipher
-#define cra_compress	cra_u.compress
- 
- /**
-  * struct crypto_alg - definition of a cryptograpic cipher algorithm
-@@ -405,18 +276,21 @@ struct crypto_istat_rng {
-  * @cra_ctxsize: Size of the operational context of the transformation. This
-  *		 value informs the kernel crypto API about the memory size
-  *		 needed to be allocated for the transformation context.
- * @cra_alignmask: Alignment mask for the input and output data buffer. The data
- *		   buffer containing the input data for the algorithm must be
- *		   aligned to this alignment mask. The data buffer for the
- *		   output data must be aligned to this alignment mask. Note that
- *		   the Crypto API will do the re-alignment in software, but
- *		   only under special conditions and there is a performance hit.
- *		   The re-alignment happens at these occasions for different
- *		   @cra_u types: cipher -- For both input data and output data
- *		   buffer; ahash -- For output hash destination buf; shash --
- *		   For output hash destination buf.
- *		   This is needed on hardware which is flawed by design and
- *		   cannot pick data from arbitrary addresses.
-+ * @cra_alignmask: For cipher, skcipher, lskcipher, and aead algorithms this is
-+ *		   1 less than the alignment, in bytes, that the algorithm
-+ *		   implementation requires for input and output buffers.  When
-+ *		   the crypto API is invoked with buffers that are not aligned
-+ *		   to this alignment, the crypto API automatically utilizes
-+ *		   appropriately aligned temporary buffers to comply with what
-+ *		   the algorithm needs.  (For scatterlists this happens only if
-+ *		   the algorithm uses the skcipher_walk helper functions.)  This
-+ *		   misalignment handling carries a performance penalty, so it is
-+ *		   preferred that algorithms do not set a nonzero alignmask.
-+ *		   Also, crypto API users may wish to allocate buffers aligned
-+ *		   to the alignmask of the algorithm being used, in order to
-+ *		   avoid the API having to realign them.  Note: the alignmask is
-+ *		   not supported for hash algorithms and is always 0 for them.
-+ * @cra_reqsize: Size of the request context for this algorithm.
-  * @cra_priority: Priority of this transformation implementation. In case
-  *		  multiple transformations with same @cra_name are available to
-  *		  the Crypto API, the kernel will use the one with highest
-@@ -435,42 +309,22 @@ struct crypto_istat_rng {
-  *	      transformation types. There are multiple options, such as
-  *	      &crypto_skcipher_type, &crypto_ahash_type, &crypto_rng_type.
-  *	      This field might be empty. In that case, there are no common
- *	      callbacks. This is the case for: cipher, compress, shash.
-+ *	      callbacks. This is the case for: cipher.
-  * @cra_u: Callbacks implementing the transformation. This is a union of
-  *	   multiple structures. Depending on the type of transformation selected
-  *	   by @cra_type and @cra_flags above, the associated structure must be
-  *	   filled with callbacks. This field might be empty. This is the case
-  *	   for ahash, shash.
- * @cra_init: Initialize the cryptographic transformation object. This function
- *	      is used to initialize the cryptographic transformation object.
- *	      This function is called only once at the instantiation time, right
- *	      after the transformation context was allocated. In case the
- *	      cryptographic hardware has some special requirements which need to
- *	      be handled by software, this function shall check for the precise
- *	      requirement of the transformation and put any software fallbacks
- *	      in place.
- * @cra_exit: Deinitialize the cryptographic transformation object. This is a
- *	      counterpart to @cra_init, used to remove various changes set in
- *	      @cra_init.
-+ * @cra_init: Deprecated, do not use.
-+ * @cra_exit: Deprecated, do not use.
-  * @cra_u.cipher: Union member which contains a single-block symmetric cipher
-  *		  definition. See @struct @cipher_alg.
- * @cra_u.compress: Union member which contains a (de)compression algorithm.
- *		    See @struct @compress_alg.
-  * @cra_module: Owner of this transformation implementation. Set to THIS_MODULE
-  * @cra_list: internally used
-  * @cra_users: internally used
-  * @cra_refcnt: internally used
-  * @cra_destroy: internally used
-  *
- * @stats: union of all possible crypto_istat_xxx structures
- * @stats.aead:		statistics for AEAD algorithm
- * @stats.akcipher:	statistics for akcipher algorithm
- * @stats.cipher:	statistics for cipher algorithm
- * @stats.compress:	statistics for compress algorithm
- * @stats.hash:		statistics for hash algorithm
- * @stats.rng:		statistics for rng algorithm
- * @stats.kpp:		statistics for KPP algorithm
- *
-  * The struct crypto_alg describes a generic Crypto API algorithm and is common
-  * for all of the transformations. Any variable not documented here shall not
-  * be used by a cipher implementation as it is internal to the Crypto API.
-@@ -483,6 +337,7 @@ struct crypto_alg {
- 	unsigned int cra_blocksize;
- 	unsigned int cra_ctxsize;
- 	unsigned int cra_alignmask;
-+	unsigned int cra_reqsize;
- 
- 	int cra_priority;
- 	refcount_t cra_refcnt;
-@@ -494,7 +349,6 @@ struct crypto_alg {
- 
- 	union {
- 		struct cipher_alg cipher;
-		struct compress_alg compress;
- 	} cra_u;
- 
- 	int (*cra_init)(struct crypto_tfm *tfm);
-@@ -502,81 +356,8 @@ struct crypto_alg {
- 	void (*cra_destroy)(struct crypto_alg *alg);
- 	
- 	struct module *cra_module;
-
-#ifdef CONFIG_CRYPTO_STATS
-	union {
-		struct crypto_istat_aead aead;
-		struct crypto_istat_akcipher akcipher;
-		struct crypto_istat_cipher cipher;
-		struct crypto_istat_compress compress;
-		struct crypto_istat_hash hash;
-		struct crypto_istat_rng rng;
-		struct crypto_istat_kpp kpp;
-	} stats;
-#endif /* CONFIG_CRYPTO_STATS */
-
- } CRYPTO_MINALIGN_ATTR;
- 
-#ifdef CONFIG_CRYPTO_STATS
-void crypto_stats_init(struct crypto_alg *alg);
-void crypto_stats_get(struct crypto_alg *alg);
-void crypto_stats_aead_encrypt(unsigned int cryptlen, struct crypto_alg *alg, int ret);
-void crypto_stats_aead_decrypt(unsigned int cryptlen, struct crypto_alg *alg, int ret);
-void crypto_stats_ahash_update(unsigned int nbytes, int ret, struct crypto_alg *alg);
-void crypto_stats_ahash_final(unsigned int nbytes, int ret, struct crypto_alg *alg);
-void crypto_stats_akcipher_encrypt(unsigned int src_len, int ret, struct crypto_alg *alg);
-void crypto_stats_akcipher_decrypt(unsigned int src_len, int ret, struct crypto_alg *alg);
-void crypto_stats_akcipher_sign(int ret, struct crypto_alg *alg);
-void crypto_stats_akcipher_verify(int ret, struct crypto_alg *alg);
-void crypto_stats_compress(unsigned int slen, int ret, struct crypto_alg *alg);
-void crypto_stats_decompress(unsigned int slen, int ret, struct crypto_alg *alg);
-void crypto_stats_kpp_set_secret(struct crypto_alg *alg, int ret);
-void crypto_stats_kpp_generate_public_key(struct crypto_alg *alg, int ret);
-void crypto_stats_kpp_compute_shared_secret(struct crypto_alg *alg, int ret);
-void crypto_stats_rng_seed(struct crypto_alg *alg, int ret);
-void crypto_stats_rng_generate(struct crypto_alg *alg, unsigned int dlen, int ret);
-void crypto_stats_skcipher_encrypt(unsigned int cryptlen, int ret, struct crypto_alg *alg);
-void crypto_stats_skcipher_decrypt(unsigned int cryptlen, int ret, struct crypto_alg *alg);
-#else
-static inline void crypto_stats_init(struct crypto_alg *alg)
-{}
-static inline void crypto_stats_get(struct crypto_alg *alg)
-{}
-static inline void crypto_stats_aead_encrypt(unsigned int cryptlen, struct crypto_alg *alg, int ret)
-{}
-static inline void crypto_stats_aead_decrypt(unsigned int cryptlen, struct crypto_alg *alg, int ret)
-{}
-static inline void crypto_stats_ahash_update(unsigned int nbytes, int ret, struct crypto_alg *alg)
-{}
-static inline void crypto_stats_ahash_final(unsigned int nbytes, int ret, struct crypto_alg *alg)
-{}
-static inline void crypto_stats_akcipher_encrypt(unsigned int src_len, int ret, struct crypto_alg *alg)
-{}
-static inline void crypto_stats_akcipher_decrypt(unsigned int src_len, int ret, struct crypto_alg *alg)
-{}
-static inline void crypto_stats_akcipher_sign(int ret, struct crypto_alg *alg)
-{}
-static inline void crypto_stats_akcipher_verify(int ret, struct crypto_alg *alg)
-{}
-static inline void crypto_stats_compress(unsigned int slen, int ret, struct crypto_alg *alg)
-{}
-static inline void crypto_stats_decompress(unsigned int slen, int ret, struct crypto_alg *alg)
-{}
-static inline void crypto_stats_kpp_set_secret(struct crypto_alg *alg, int ret)
-{}
-static inline void crypto_stats_kpp_generate_public_key(struct crypto_alg *alg, int ret)
-{}
-static inline void crypto_stats_kpp_compute_shared_secret(struct crypto_alg *alg, int ret)
-{}
-static inline void crypto_stats_rng_seed(struct crypto_alg *alg, int ret)
-{}
-static inline void crypto_stats_rng_generate(struct crypto_alg *alg, unsigned int dlen, int ret)
-{}
-static inline void crypto_stats_skcipher_encrypt(unsigned int cryptlen, int ret, struct crypto_alg *alg)
-{}
-static inline void crypto_stats_skcipher_decrypt(unsigned int cryptlen, int ret, struct crypto_alg *alg)
-{}
-#endif
- /*
-  * A helper struct for waiting for completion of async crypto ops
-  */
-@@ -595,7 +376,7 @@ struct crypto_wait {
- /*
-  * Async ops completion helper functioons
-  */
-void crypto_req_done(struct crypto_async_request *req, int err);
-+void crypto_req_done(void *req, int err);
- 
- static inline int crypto_wait_req(int err, struct crypto_wait *wait)
- {
-@@ -617,14 +398,6 @@ static inline void crypto_init_wait(struct crypto_wait *wait)
- }
- 
- /*
- * Algorithm registration interface.
- */
-int crypto_register_alg(struct crypto_alg *alg);
-void crypto_unregister_alg(struct crypto_alg *alg);
-int crypto_register_algs(struct crypto_alg *algs, int count);
-void crypto_unregister_algs(struct crypto_alg *algs, int count);
-
-/*
-  * Algorithm query interface.
-  */
- int crypto_has_alg(const char *name, u32 type, u32 mask);
-@@ -636,22 +409,21 @@ int crypto_has_alg(const char *name, u32 type, u32 mask);
-  */
- 
- struct crypto_tfm {
-+	refcount_t refcnt;
- 
- 	u32 crt_flags;
- 
- 	int node;
-	
-+
-+	struct crypto_tfm *fb;
-+
- 	void (*exit)(struct crypto_tfm *tfm);
-	
-+
- 	struct crypto_alg *__crt_alg;
- 
- 	void *__crt_ctx[] CRYPTO_MINALIGN_ATTR;
- };
- 
-struct crypto_comp {
-	struct crypto_tfm base;
-};
-
- /* 
-  * Transform user interface.
-  */
-@@ -664,8 +436,6 @@ static inline void crypto_free_tfm(struct crypto_tfm *tfm)
- 	return crypto_destroy_tfm(tfm, tfm);
- }
- 
-int alg_test(const char *driver, const char *alg, u32 type, u32 mask);
-
- /*
-  * Transform helpers which query the underlying algorithm.
-  */
-@@ -679,16 +449,6 @@ static inline const char *crypto_tfm_alg_driver_name(struct crypto_tfm *tfm)
- 	return tfm->__crt_alg->cra_driver_name;
- }
- 
-static inline int crypto_tfm_alg_priority(struct crypto_tfm *tfm)
-{
-	return tfm->__crt_alg->cra_priority;
-}
-
-static inline u32 crypto_tfm_alg_type(struct crypto_tfm *tfm)
-{
-	return tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK;
-}
-
- static inline unsigned int crypto_tfm_alg_blocksize(struct crypto_tfm *tfm)
- {
- 	return tfm->__crt_alg->cra_blocksize;
-@@ -699,6 +459,11 @@ static inline unsigned int crypto_tfm_alg_alignmask(struct crypto_tfm *tfm)
- 	return tfm->__crt_alg->cra_alignmask;
- }
- 
-+static inline unsigned int crypto_tfm_alg_reqsize(struct crypto_tfm *tfm)
-+{
-+	return tfm->__crt_alg->cra_reqsize;
-+}
-+
- static inline u32 crypto_tfm_get_flags(struct crypto_tfm *tfm)
- {
- 	return tfm->crt_flags;
-@@ -714,63 +479,51 @@ static inline void crypto_tfm_clear_flags(struct crypto_tfm *tfm, u32 flags)
- 	tfm->crt_flags &= ~flags;
- }
- 
-static inline void *crypto_tfm_ctx(struct crypto_tfm *tfm)
-{
-	return tfm->__crt_ctx;
-}
-
- static inline unsigned int crypto_tfm_ctx_alignment(void)
- {
- 	struct crypto_tfm *tfm;
- 	return __alignof__(tfm->__crt_ctx);
- }
- 
-static inline struct crypto_comp *__crypto_comp_cast(struct crypto_tfm *tfm)
-+static inline bool crypto_tfm_is_async(struct crypto_tfm *tfm)
- {
-	return (struct crypto_comp *)tfm;
-+	return tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC;
- }
- 
-static inline struct crypto_comp *crypto_alloc_comp(const char *alg_name,
-						    u32 type, u32 mask)
-+static inline bool crypto_req_on_stack(struct crypto_async_request *req)
- {
-	type &= ~CRYPTO_ALG_TYPE_MASK;
-	type |= CRYPTO_ALG_TYPE_COMPRESS;
-	mask |= CRYPTO_ALG_TYPE_MASK;
-
-	return __crypto_comp_cast(crypto_alloc_base(alg_name, type, mask));
-+	return req->flags & CRYPTO_TFM_REQ_ON_STACK;
- }
- 
-static inline struct crypto_tfm *crypto_comp_tfm(struct crypto_comp *tfm)
-+static inline void crypto_request_set_callback(
-+	struct crypto_async_request *req, u32 flags,
-+	crypto_completion_t compl, void *data)
- {
-	return &tfm->base;
-}
-+	u32 keep = CRYPTO_TFM_REQ_ON_STACK;
- 
-static inline void crypto_free_comp(struct crypto_comp *tfm)
-{
-	crypto_free_tfm(crypto_comp_tfm(tfm));
-+	req->complete = compl;
-+	req->data = data;
-+	req->flags &= keep;
-+	req->flags |= flags & ~keep;
- }
- 
-static inline int crypto_has_comp(const char *alg_name, u32 type, u32 mask)
-+static inline void crypto_request_set_tfm(struct crypto_async_request *req,
-+					  struct crypto_tfm *tfm)
- {
-	type &= ~CRYPTO_ALG_TYPE_MASK;
-	type |= CRYPTO_ALG_TYPE_COMPRESS;
-	mask |= CRYPTO_ALG_TYPE_MASK;
-
-	return crypto_has_alg(alg_name, type, mask);
-+	req->tfm = tfm;
-+	req->flags &= ~CRYPTO_TFM_REQ_ON_STACK;
- }
- 
-static inline const char *crypto_comp_name(struct crypto_comp *tfm)
-+struct crypto_async_request *crypto_request_clone(
-+	struct crypto_async_request *req, size_t total, gfp_t gfp);
-+
-+static inline void crypto_stack_request_init(struct crypto_async_request *req,
-+					     struct crypto_tfm *tfm)
- {
-	return crypto_tfm_alg_name(crypto_comp_tfm(tfm));
-+	req->flags = 0;
-+	crypto_request_set_tfm(req, tfm);
-+	req->flags |= CRYPTO_TFM_REQ_ON_STACK;
- }
- 
-int crypto_comp_compress(struct crypto_comp *tfm,
-			 const u8 *src, unsigned int slen,
-			 u8 *dst, unsigned int *dlen);
-
-int crypto_comp_decompress(struct crypto_comp *tfm,
-			   const u8 *src, unsigned int slen,
-			   u8 *dst, unsigned int *dlen);
-
- #endif	/* _LINUX_CRYPTO_H */
- 
-diff --git a/include/linux/damon.h b/include/linux/damon.h
-index 7b1f4a488230..3813373a9200 100644
--- a/include/linux/damon.h
-+++ b/include/linux/damon.h
-@@ -2,12 +2,13 @@
- /*
-  * DAMON api
-  *
- * Author: SeongJae Park <sjpark@amazon.de>
-+ * Author: SeongJae Park <sj@kernel.org>
-  */
- 
- #ifndef _DAMON_H_
- #define _DAMON_H_
- 
-+#include <linux/memcontrol.h>
- #include <linux/mutex.h>
- #include <linux/time64.h>
- #include <linux/types.h>
-@@ -21,7 +22,7 @@
- /* Get a random number in [l, r) */
- static inline unsigned long damon_rand(unsigned long l, unsigned long r)
- {
-	return l + prandom_u32_max(r - l);
-+	return l + get_random_u32_below(r - l);
- }
- 
- /**
-@@ -35,13 +36,38 @@ struct damon_addr_range {
- };
- 
- /**
-+ * struct damon_size_range - Represents size for filter to operate on [@min, @max].
-+ * @min:	Min size (inclusive).
-+ * @max:	Max size (inclusive).
-+ */
-+struct damon_size_range {
-+	unsigned long min;
-+	unsigned long max;
-+};
-+
-+/**
-  * struct damon_region - Represents a monitoring target region.
-  * @ar:			The address range of the region.
-  * @sampling_addr:	Address of the sample for the next access check.
-  * @nr_accesses:	Access frequency of this region.
-+ * @nr_accesses_bp:	@nr_accesses in basis point (0.01%) that updated for
-+ *			each sampling interval.
-  * @list:		List head for siblings.
-  * @age:		Age of this region.
-  *
-+ * @nr_accesses is reset to zero for every &damon_attrs->aggr_interval and be
-+ * increased for every &damon_attrs->sample_interval if an access to the region
-+ * during the last sampling interval is found.  The update of this field should
-+ * not be done with direct access but with the helper function,
-+ * damon_update_region_access_rate().
-+ *
-+ * @nr_accesses_bp is another representation of @nr_accesses in basis point
-+ * (1 in 10,000) that updated for every &damon_attrs->sample_interval in a
-+ * manner similar to moving sum.  By the algorithm, this value becomes
-+ * @nr_accesses * 10000 for every &struct damon_attrs->aggr_interval.  This can
-+ * be used when the aggregation interval is too huge and therefore cannot wait
-+ * for it before getting the access monitoring results.
-+ *
-  * @age is initially zero, increased for each aggregation interval, and reset
-  * to zero again if the access frequency is significantly changed.  If two
-  * regions are merged into a new region, both @nr_accesses and @age of the new
-@@ -51,6 +77,7 @@ struct damon_region {
- 	struct damon_addr_range ar;
- 	unsigned long sampling_addr;
- 	unsigned int nr_accesses;
-+	unsigned int nr_accesses_bp;
- 	struct list_head list;
- 
- 	unsigned int age;
-@@ -64,17 +91,23 @@ struct damon_region {
-  * @nr_regions:		Number of monitoring target regions of this target.
-  * @regions_list:	Head of the monitoring target regions of this target.
-  * @list:		List head for siblings.
-+ * @obsolete:		Whether the commit destination target is obsolete.
-  *
-  * Each monitoring context could have multiple targets.  For example, a context
-  * for virtual memory address spaces could have multiple target processes.  The
-  * @pid should be set for appropriate &struct damon_operations including the
-  * virtual address spaces monitoring operations.
-+ *
-+ * @obsolete is used only for damon_commit_targets() source targets, to specify
-+ * the matching destination targets are obsolete.  Read damon_commit_targets()
-+ * to see how it is handled.
-  */
- struct damon_target {
- 	struct pid *pid;
- 	unsigned int nr_regions;
- 	struct list_head regions_list;
- 	struct list_head list;
-+	bool obsolete;
- };
- 
- /**
-@@ -83,13 +116,21 @@ struct damon_target {
-  *
-  * @DAMOS_WILLNEED:	Call ``madvise()`` for the region with MADV_WILLNEED.
-  * @DAMOS_COLD:		Call ``madvise()`` for the region with MADV_COLD.
- * @DAMOS_PAGEOUT:	Call ``madvise()`` for the region with MADV_PAGEOUT.
-+ * @DAMOS_PAGEOUT:	Reclaim the region.
-  * @DAMOS_HUGEPAGE:	Call ``madvise()`` for the region with MADV_HUGEPAGE.
-  * @DAMOS_NOHUGEPAGE:	Call ``madvise()`` for the region with MADV_NOHUGEPAGE.
-  * @DAMOS_LRU_PRIO:	Prioritize the region on its LRU lists.
-  * @DAMOS_LRU_DEPRIO:	Deprioritize the region on its LRU lists.
-+ * @DAMOS_MIGRATE_HOT:  Migrate the regions prioritizing warmer regions.
-+ * @DAMOS_MIGRATE_COLD:	Migrate the regions prioritizing colder regions.
-  * @DAMOS_STAT:		Do nothing but count the stat.
-  * @NR_DAMOS_ACTIONS:	Total number of DAMOS actions
-+ *
-+ * The support of each action is up to running &struct damon_operations.
-+ * Refer to 'Operation Action' section of Documentation/mm/damon/design.rst for
-+ * status of the supports.
-+ *
-+ * Note that DAMOS_PAGEOUT doesn't trigger demotions.
-  */
- enum damos_action {
- 	DAMOS_WILLNEED,
-@@ -99,15 +140,82 @@ enum damos_action {
- 	DAMOS_NOHUGEPAGE,
- 	DAMOS_LRU_PRIO,
- 	DAMOS_LRU_DEPRIO,
-+	DAMOS_MIGRATE_HOT,
-+	DAMOS_MIGRATE_COLD,
- 	DAMOS_STAT,		/* Do nothing but only record the stat */
- 	NR_DAMOS_ACTIONS,
- };
- 
- /**
-+ * enum damos_quota_goal_metric - Represents the metric to be used as the goal
-+ *
-+ * @DAMOS_QUOTA_USER_INPUT:	User-input value.
-+ * @DAMOS_QUOTA_SOME_MEM_PSI_US:	System level some memory PSI in us.
-+ * @DAMOS_QUOTA_NODE_MEM_USED_BP:	MemUsed ratio of a node.
-+ * @DAMOS_QUOTA_NODE_MEM_FREE_BP:	MemFree ratio of a node.
-+ * @DAMOS_QUOTA_NODE_MEMCG_USED_BP:	MemUsed ratio of a node for a cgroup.
-+ * @DAMOS_QUOTA_NODE_MEMCG_FREE_BP:	MemFree ratio of a node for a cgroup.
-+ * @NR_DAMOS_QUOTA_GOAL_METRICS:	Number of DAMOS quota goal metrics.
-+ *
-+ * Metrics equal to larger than @NR_DAMOS_QUOTA_GOAL_METRICS are unsupported.
-+ */
-+enum damos_quota_goal_metric {
-+	DAMOS_QUOTA_USER_INPUT,
-+	DAMOS_QUOTA_SOME_MEM_PSI_US,
-+	DAMOS_QUOTA_NODE_MEM_USED_BP,
-+	DAMOS_QUOTA_NODE_MEM_FREE_BP,
-+	DAMOS_QUOTA_NODE_MEMCG_USED_BP,
-+	DAMOS_QUOTA_NODE_MEMCG_FREE_BP,
-+	NR_DAMOS_QUOTA_GOAL_METRICS,
-+};
-+
-+/**
-+ * struct damos_quota_goal - DAMOS scheme quota auto-tuning goal.
-+ * @metric:		Metric to be used for representing the goal.
-+ * @target_value:	Target value of @metric to achieve with the tuning.
-+ * @current_value:	Current value of @metric.
-+ * @last_psi_total:	Last measured total PSI
-+ * @nid:		Node id.
-+ * @memcg_id:		Memcg id.
-+ * @list:		List head for siblings.
-+ *
-+ * Data structure for getting the current score of the quota tuning goal.  The
-+ * score is calculated by how close @current_value and @target_value are.  Then
-+ * the score is entered to DAMON's internal feedback loop mechanism to get the
-+ * auto-tuned quota.
-+ *
-+ * If @metric is DAMOS_QUOTA_USER_INPUT, @current_value should be manually
-+ * entered by the user, probably inside the kdamond callbacks.  Otherwise,
-+ * DAMON sets @current_value with self-measured value of @metric.
-+ *
-+ * If @metric is DAMOS_QUOTA_NODE_MEM_{USED,FREE}_BP, @nid represents the node
-+ * id of the target node to account the used/free memory.
-+ *
-+ * If @metric is DAMOS_QUOTA_NODE_MEMCG_{USED,FREE}_BP, @nid and @memcg_id
-+ * represents the node id and the cgroup to account the used memory for.
-+ */
-+struct damos_quota_goal {
-+	enum damos_quota_goal_metric metric;
-+	unsigned long target_value;
-+	unsigned long current_value;
-+	/* metric-dependent fields */
-+	union {
-+		u64 last_psi_total;
-+		struct {
-+			int nid;
-+			unsigned short memcg_id;
-+		};
-+	};
-+	struct list_head list;
-+};
-+
-+/**
-  * struct damos_quota - Controls the aggressiveness of the given scheme.
-+ * @reset_interval:	Charge reset interval in milliseconds.
-  * @ms:			Maximum milliseconds that the scheme can use.
-  * @sz:			Maximum bytes of memory that the action can be applied.
- * @reset_interval:	Charge reset interval in milliseconds.
-+ * @goals:		Head of quota tuning goals (&damos_quota_goal) list.
-+ * @esz:		Effective size quota in bytes.
-  *
-  * @weight_sz:		Weight of the region's size for prioritization.
-  * @weight_nr_accesses:	Weight of the region's nr_accesses for prioritization.
-@@ -121,9 +229,21 @@ enum damos_action {
-  * size quota is set, DAMON tries to apply the action only up to &sz bytes
-  * within &reset_interval.
-  *
- * Internally, the time quota is transformed to a size quota using estimated
- * throughput of the scheme's action.  DAMON then compares it against &sz and
- * uses smaller one as the effective quota.
-+ * To convince the different types of quotas and goals, DAMON internally
-+ * converts those into one single size quota called "effective quota".  DAMON
-+ * internally uses it as the only one real quota.  The conversion is made as
-+ * follows.
-+ *
-+ * The time quota is transformed to a size quota using estimated throughput of
-+ * the scheme's action.  DAMON then compares it against &sz and uses smaller
-+ * one as the effective quota.
-+ *
-+ * If @goals is not empty, DAMON calculates yet another size quota based on the
-+ * goals using its internal feedback loop algorithm, for every @reset_interval.
-+ * Then, if the new size quota is smaller than the effective quota, it uses the
-+ * new size quota as the effective quota.
-+ *
-+ * The resulting effective size quota in bytes is set to @esz.
-  *
-  * For selecting regions within the quota, DAMON prioritizes current scheme's
-  * target memory regions using the &struct damon_operations->get_scheme_score.
-@@ -132,9 +252,11 @@ enum damos_action {
-  * encouraged to respect those.
-  */
- struct damos_quota {
-+	unsigned long reset_interval;
- 	unsigned long ms;
- 	unsigned long sz;
-	unsigned long reset_interval;
-+	struct list_head goals;
-+	unsigned long esz;
- 
- 	unsigned int weight_sz;
- 	unsigned int weight_nr_accesses;
-@@ -145,8 +267,6 @@ struct damos_quota {
- 	unsigned long total_charged_sz;
- 	unsigned long total_charged_ns;
- 
-	unsigned long esz;	/* Effective size quota in bytes */
-
- 	/* For charging the quota */
- 	unsigned long charged_sz;
- 	unsigned long charged_from;
-@@ -154,8 +274,10 @@ struct damos_quota {
- 	unsigned long charge_addr_from;
- 
- 	/* For prioritization */
-	unsigned long histogram[DAMOS_MAX_SCORE + 1];
- 	unsigned int min_score;
-+
-+	/* For feedback loop */
-+	unsigned long esz_bp;
- };
- 
- /**
-@@ -205,58 +327,244 @@ struct damos_watermarks {
-  * @sz_tried:	Total size of regions that the scheme is tried to be applied.
-  * @nr_applied:	Total number of regions that the scheme is applied.
-  * @sz_applied:	Total size of regions that the scheme is applied.
-+ * @sz_ops_filter_passed:
-+ *		Total bytes that passed ops layer-handled DAMOS filters.
-  * @qt_exceeds: Total number of times the quota of the scheme has exceeded.
-+ *
-+ * "Tried an action to a region" in this context means the DAMOS core logic
-+ * determined the region as eligible to apply the action.  The access pattern
-+ * (&struct damos_access_pattern), quotas (&struct damos_quota), watermarks
-+ * (&struct damos_watermarks) and filters (&struct damos_filter) that handled
-+ * on core logic can affect this.  The core logic asks the operation set
-+ * (&struct damon_operations) to apply the action to the region.
-+ *
-+ * "Applied an action to a region" in this context means the operation set
-+ * (&struct damon_operations) successfully applied the action to the region, at
-+ * least to a part of the region.  The filters (&struct damos_filter) that
-+ * handled on operation set layer and type of the action and pages of the
-+ * region can affect this.  For example, if a filter is set to exclude
-+ * anonymous pages and the region has only anonymous pages, the region will be
-+ * failed at applying the action.  If the action is &DAMOS_PAGEOUT and all
-+ * pages of the region are already paged out, the region will be failed at
-+ * applying the action.
-  */
- struct damos_stat {
- 	unsigned long nr_tried;
- 	unsigned long sz_tried;
- 	unsigned long nr_applied;
- 	unsigned long sz_applied;
-+	unsigned long sz_ops_filter_passed;
- 	unsigned long qt_exceeds;
- };
- 
- /**
- * struct damos - Represents a Data Access Monitoring-based Operation Scheme.
-+ * enum damos_filter_type - Type of memory for &struct damos_filter
-+ * @DAMOS_FILTER_TYPE_ANON:	Anonymous pages.
-+ * @DAMOS_FILTER_TYPE_ACTIVE:	Active pages.
-+ * @DAMOS_FILTER_TYPE_MEMCG:	Specific memcg's pages.
-+ * @DAMOS_FILTER_TYPE_YOUNG:	Recently accessed pages.
-+ * @DAMOS_FILTER_TYPE_HUGEPAGE_SIZE:	Page is part of a hugepage.
-+ * @DAMOS_FILTER_TYPE_UNMAPPED:	Unmapped pages.
-+ * @DAMOS_FILTER_TYPE_ADDR:	Address range.
-+ * @DAMOS_FILTER_TYPE_TARGET:	Data Access Monitoring target.
-+ * @NR_DAMOS_FILTER_TYPES:	Number of filter types.
-+ *
-+ * The anon pages type and memcg type filters are handled by underlying
-+ * &struct damon_operations as a part of scheme action trying, and therefore
-+ * accounted as 'tried'.  In contrast, other types are handled by core layer
-+ * before trying of the action and therefore not accounted as 'tried'.
-+ *
-+ * The support of the filters that handled by &struct damon_operations depend
-+ * on the running &struct damon_operations.
-+ * &enum DAMON_OPS_PADDR supports both anon pages type and memcg type filters,
-+ * while &enum DAMON_OPS_VADDR and &enum DAMON_OPS_FVADDR don't support any of
-+ * the two types.
-+ */
-+enum damos_filter_type {
-+	DAMOS_FILTER_TYPE_ANON,
-+	DAMOS_FILTER_TYPE_ACTIVE,
-+	DAMOS_FILTER_TYPE_MEMCG,
-+	DAMOS_FILTER_TYPE_YOUNG,
-+	DAMOS_FILTER_TYPE_HUGEPAGE_SIZE,
-+	DAMOS_FILTER_TYPE_UNMAPPED,
-+	DAMOS_FILTER_TYPE_ADDR,
-+	DAMOS_FILTER_TYPE_TARGET,
-+	NR_DAMOS_FILTER_TYPES,
-+};
-+
-+/**
-+ * struct damos_filter - DAMOS action target memory filter.
-+ * @type:	Type of the target memory.
-+ * @matching:	Whether this is for @type-matching memory.
-+ * @allow:	Whether to include or exclude the @matching memory.
-+ * @memcg_id:	Memcg id of the question if @type is DAMOS_FILTER_MEMCG.
-+ * @addr_range:	Address range if @type is DAMOS_FILTER_TYPE_ADDR.
-+ * @target_idx:	Index of the &struct damon_target of
-+ *		&damon_ctx->adaptive_targets if @type is
-+ *		DAMOS_FILTER_TYPE_TARGET.
-+ * @sz_range:	Size range if @type is DAMOS_FILTER_TYPE_HUGEPAGE_SIZE.
-+ * @list:	List head for siblings.
-+ *
-+ * Before applying the &damos->action to a memory region, DAMOS checks if each
-+ * byte of the region matches to this given condition and avoid applying the
-+ * action if so.  Support of each filter type depends on the running &struct
-+ * damon_operations and the type.  Refer to &enum damos_filter_type for more
-+ * details.
-+ */
-+struct damos_filter {
-+	enum damos_filter_type type;
-+	bool matching;
-+	bool allow;
-+	union {
-+		unsigned short memcg_id;
-+		struct damon_addr_range addr_range;
-+		int target_idx;
-+		struct damon_size_range sz_range;
-+	};
-+	struct list_head list;
-+};
-+
-+struct damon_ctx;
-+struct damos;
-+
-+/**
-+ * struct damos_walk_control - Control damos_walk().
-+ *
-+ * @walk_fn:	Function to be called back for each region.
-+ * @data:	Data that will be passed to walk functions.
-+ *
-+ * Control damos_walk(), which requests specific kdamond to invoke the given
-+ * function to each region that eligible to apply actions of the kdamond's
-+ * schemes.  Refer to damos_walk() for more details.
-+ */
-+struct damos_walk_control {
-+	void (*walk_fn)(void *data, struct damon_ctx *ctx,
-+			struct damon_target *t, struct damon_region *r,
-+			struct damos *s, unsigned long sz_filter_passed);
-+	void *data;
-+/* private: internal use only */
-+	/* informs if the kdamond finished handling of the walk request */
-+	struct completion completion;
-+	/* informs if the walk is canceled. */
-+	bool canceled;
-+};
-+
-+/**
-+ * struct damos_access_pattern - Target access pattern of the given scheme.
-  * @min_sz_region:	Minimum size of target regions.
-  * @max_sz_region:	Maximum size of target regions.
-  * @min_nr_accesses:	Minimum ``->nr_accesses`` of target regions.
-  * @max_nr_accesses:	Maximum ``->nr_accesses`` of target regions.
-  * @min_age_region:	Minimum age of target regions.
-  * @max_age_region:	Maximum age of target regions.
- * @action:		&damo_action to be applied to the target regions.
-+ */
-+struct damos_access_pattern {
-+	unsigned long min_sz_region;
-+	unsigned long max_sz_region;
-+	unsigned int min_nr_accesses;
-+	unsigned int max_nr_accesses;
-+	unsigned int min_age_region;
-+	unsigned int max_age_region;
-+};
-+
-+/**
-+ * struct damos_migrate_dests - Migration destination nodes and their weights.
-+ * @node_id_arr:	Array of migration destination node ids.
-+ * @weight_arr:		Array of migration weights for @node_id_arr.
-+ * @nr_dests:		Length of the @node_id_arr and @weight_arr arrays.
-+ *
-+ * @node_id_arr is an array of the ids of migration destination nodes.
-+ * @weight_arr is an array of the weights for those.  The weights in
-+ * @weight_arr are for nodes in @node_id_arr of same array index.
-+ */
-+struct damos_migrate_dests {
-+	unsigned int *node_id_arr;
-+	unsigned int *weight_arr;
-+	size_t nr_dests;
-+};
-+
-+/**
-+ * struct damos - Represents a Data Access Monitoring-based Operation Scheme.
-+ * @pattern:		Access pattern of target regions.
-+ * @action:		&damos_action to be applied to the target regions.
-+ * @apply_interval_us:	The time between applying the @action.
-  * @quota:		Control the aggressiveness of this scheme.
-  * @wmarks:		Watermarks for automated (in)activation of this scheme.
-+ * @migrate_dests:	Destination nodes if @action is "migrate_{hot,cold}".
-+ * @target_nid:		Destination node if @action is "migrate_{hot,cold}".
-+ * @core_filters:	Additional set of &struct damos_filter for &action.
-+ * @ops_filters:	ops layer handling &struct damos_filter objects list.
-+ * @last_applied:	Last @action applied ops-managing entity.
-  * @stat:		Statistics of this scheme.
-  * @list:		List head for siblings.
-  *
- * For each aggregation interval, DAMON finds regions which fit in the
- * condition (&min_sz_region, &max_sz_region, &min_nr_accesses,
- * &max_nr_accesses, &min_age_region, &max_age_region) and applies &action to
- * those.  To avoid consuming too much CPU time or IO resources for the
- * &action, &quota is used.
-+ * For each @apply_interval_us, DAMON finds regions which fit in the
-+ * &pattern and applies &action to those. To avoid consuming too much
-+ * CPU time or IO resources for the &action, &quota is used.
-+ *
-+ * If @apply_interval_us is zero, &damon_attrs->aggr_interval is used instead.
-  *
-  * To do the work only when needed, schemes can be activated for specific
-  * system situations using &wmarks.  If all schemes that registered to the
-  * monitoring context are inactive, DAMON stops monitoring either, and just
-  * repeatedly checks the watermarks.
-  *
- * If all schemes that registered to a &struct damon_ctx are inactive, DAMON
- * stops monitoring and just repeatedly checks the watermarks.
-+ * @migrate_dests specifies multiple migration target nodes with different
-+ * weights for migrate_hot or migrate_cold actions.  @target_nid is ignored if
-+ * this is set.
-+ *
-+ * @target_nid is used to set the migration target node for migrate_hot or
-+ * migrate_cold actions, and @migrate_dests is unset.
-+ *
-+ * Before applying the &action to a memory region, &struct damon_operations
-+ * implementation could check pages of the region and skip &action to respect
-+ * &core_filters
-+ *
-+ * The minimum entity that @action can be applied depends on the underlying
-+ * &struct damon_operations.  Since it may not be aligned with the core layer
-+ * abstract, namely &struct damon_region, &struct damon_operations could apply
-+ * @action to same entity multiple times.  Large folios that underlying on
-+ * multiple &struct damon region objects could be such examples.  The &struct
-+ * damon_operations can use @last_applied to avoid that.  DAMOS core logic
-+ * unsets @last_applied when each regions walking for applying the scheme is
-+ * finished.
-  *
-  * After applying the &action to each region, &stat_count and &stat_sz is
-  * updated to reflect the number of regions and total size of regions that the
-  * &action is applied.
-  */
- struct damos {
-	unsigned long min_sz_region;
-	unsigned long max_sz_region;
-	unsigned int min_nr_accesses;
-	unsigned int max_nr_accesses;
-	unsigned int min_age_region;
-	unsigned int max_age_region;
-+	struct damos_access_pattern pattern;
- 	enum damos_action action;
-+	unsigned long apply_interval_us;
-+/* private: internal use only */
-+	/*
-+	 * number of sample intervals that should be passed before applying
-+	 * @action
-+	 */
-+	unsigned long next_apply_sis;
-+	/* informs if ongoing DAMOS walk for this scheme is finished */
-+	bool walk_completed;
-+	/*
-+	 * If the current region in the filtering stage is allowed by core
-+	 * layer-handled filters.  If true, operations layer allows it, too.
-+	 */
-+	bool core_filters_allowed;
-+	/* whether to reject core/ops filters umatched regions */
-+	bool core_filters_default_reject;
-+	bool ops_filters_default_reject;
-+/* public: */
- 	struct damos_quota quota;
- 	struct damos_watermarks wmarks;
-+	union {
-+		struct {
-+			int target_nid;
-+			struct damos_migrate_dests migrate_dests;
-+		};
-+	};
-+	struct list_head core_filters;
-+	struct list_head ops_filters;
-+	void *last_applied;
- 	struct damos_stat stat;
- 	struct list_head list;
- };
-@@ -277,8 +585,6 @@ enum damon_ops_id {
- 	NR_DAMON_OPS,
- };
- 
-struct damon_ctx;
-
- /**
-  * struct damon_operations - Monitoring operations for given use cases.
-  *
-@@ -287,20 +593,19 @@ struct damon_ctx;
-  * @update:			Update operations-related data structures.
-  * @prepare_access_checks:	Prepare next access check of target regions.
-  * @check_accesses:		Check the accesses to target regions.
- * @reset_aggregated:		Reset aggregated accesses monitoring results.
-  * @get_scheme_score:		Get the score of a region for a scheme.
-  * @apply_scheme:		Apply a DAMON-based operation scheme.
-  * @target_valid:		Determine if the target is valid.
-+ * @cleanup_target:		Clean up each target before deallocation.
-  * @cleanup:			Clean up the context.
-  *
-  * DAMON can be extended for various address spaces and usages.  For this,
-  * users should register the low level operations for their target address
-  * space and usecase via the &damon_ctx.ops.  Then, the monitoring thread
-  * (&damon_ctx.kdamond) calls @init and @prepare_access_checks before starting
- * the monitoring, @update after each &damon_ctx.ops_update_interval, and
-+ * the monitoring, @update after each &damon_attrs.ops_update_interval, and
-  * @check_accesses, @target_valid and @prepare_access_checks after each
- * &damon_ctx.sample_interval.  Finally, @reset_aggregated is called after each
- * &damon_ctx.aggr_interval.
-+ * &damon_attrs.sample_interval.
-  *
-  * Each &struct damon_operations instance having valid @id can be registered
-  * via damon_register_ops() and selected by damon_select_ops() later.
-@@ -315,16 +620,16 @@ struct damon_ctx;
-  * last preparation and update the number of observed accesses of each region.
-  * It should also return max number of observed accesses that made as a result
-  * of its update.  The value will be used for regions adjustment threshold.
- * @reset_aggregated should reset the access monitoring results that aggregated
- * by @check_accesses.
-  * @get_scheme_score should return the priority score of a region for a scheme
-  * as an integer in [0, &DAMOS_MAX_SCORE].
-  * @apply_scheme is called from @kdamond when a region for user provided
-  * DAMON-based operation scheme is found.  It should apply the scheme's action
-  * to the region and return bytes of the region that the action is successfully
- * applied.
-+ * applied.  It should also report how many bytes of the region has passed
-+ * filters (&struct damos_filter) that handled by itself.
-  * @target_valid should check whether the target is still valid for the
-  * monitoring.
-+ * @cleanup_target is called before the target will be deallocated.
-  * @cleanup is called from @kdamond just before its termination.
-  */
- struct damon_operations {
-@@ -333,73 +638,117 @@ struct damon_operations {
- 	void (*update)(struct damon_ctx *context);
- 	void (*prepare_access_checks)(struct damon_ctx *context);
- 	unsigned int (*check_accesses)(struct damon_ctx *context);
-	void (*reset_aggregated)(struct damon_ctx *context);
- 	int (*get_scheme_score)(struct damon_ctx *context,
- 			struct damon_target *t, struct damon_region *r,
- 			struct damos *scheme);
- 	unsigned long (*apply_scheme)(struct damon_ctx *context,
- 			struct damon_target *t, struct damon_region *r,
-			struct damos *scheme);
-	bool (*target_valid)(void *target);
-+			struct damos *scheme, unsigned long *sz_filter_passed);
-+	bool (*target_valid)(struct damon_target *t);
-+	void (*cleanup_target)(struct damon_target *t);
- 	void (*cleanup)(struct damon_ctx *context);
- };
- 
-+/*
-+ * struct damon_call_control - Control damon_call().
-+ *
-+ * @fn:			Function to be called back.
-+ * @data:		Data that will be passed to @fn.
-+ * @repeat:		Repeat invocations.
-+ * @return_code:	Return code from @fn invocation.
-+ * @dealloc_on_cancel:	De-allocate when canceled.
-+ *
-+ * Control damon_call(), which requests specific kdamond to invoke a given
-+ * function.  Refer to damon_call() for more details.
-+ */
-+struct damon_call_control {
-+	int (*fn)(void *data);
-+	void *data;
-+	bool repeat;
-+	int return_code;
-+	bool dealloc_on_cancel;
-+/* private: internal use only */
-+	/* informs if the kdamond finished handling of the request */
-+	struct completion completion;
-+	/* informs if the kdamond canceled @fn infocation */
-+	bool canceled;
-+	/* List head for siblings. */
-+	struct list_head list;
-+};
-+
- /**
- * struct damon_callback - Monitoring events notification callbacks.
- *
- * @before_start:	Called before starting the monitoring.
- * @after_wmarks_check:	Called after each schemes' watermarks check.
- * @after_sampling:	Called after each sampling.
- * @after_aggregation:	Called after each aggregation.
- * @before_terminate:	Called before terminating the monitoring.
- * @private:		User private data.
- *
- * The monitoring thread (&damon_ctx.kdamond) calls @before_start and
- * @before_terminate just before starting and finishing the monitoring,
- * respectively.  Therefore, those are good places for installing and cleaning
- * @private.
- *
- * The monitoring thread calls @after_wmarks_check after each DAMON-based
- * operation schemes' watermarks check.  If users need to make changes to the
- * attributes of the monitoring context while it's deactivated due to the
- * watermarks, this is the good place to do.
- *
- * The monitoring thread calls @after_sampling and @after_aggregation for each
- * of the sampling intervals and aggregation intervals, respectively.
- * Therefore, users can safely access the monitoring results without additional
- * protection.  For the reason, users are recommended to use these callback for
- * the accesses to the results.
- *
- * If any callback returns non-zero, monitoring stops.
-+ * struct damon_intervals_goal - Monitoring intervals auto-tuning goal.
-+ *
-+ * @access_bp:		Access events observation ratio to achieve in bp.
-+ * @aggrs:		Number of aggregations to achieve @access_bp within.
-+ * @min_sample_us:	Minimum resulting sampling interval in microseconds.
-+ * @max_sample_us:	Maximum resulting sampling interval in microseconds.
-+ *
-+ * DAMON automatically tunes &damon_attrs->sample_interval and
-+ * &damon_attrs->aggr_interval aiming the ratio in bp (1/10,000) of
-+ * DAMON-observed access events to theoretical maximum amount within @aggrs
-+ * aggregations be same to @access_bp.  The logic increases
-+ * &damon_attrs->aggr_interval and &damon_attrs->sampling_interval in same
-+ * ratio if the current access events observation ratio is lower than the
-+ * target for each @aggrs aggregations, and vice versa.
-+ *
-+ * If @aggrs is zero, the tuning is disabled and hence this struct is ignored.
-  */
-struct damon_callback {
-	void *private;
-
-	int (*before_start)(struct damon_ctx *context);
-	int (*after_wmarks_check)(struct damon_ctx *context);
-	int (*after_sampling)(struct damon_ctx *context);
-	int (*after_aggregation)(struct damon_ctx *context);
-	void (*before_terminate)(struct damon_ctx *context);
-+struct damon_intervals_goal {
-+	unsigned long access_bp;
-+	unsigned long aggrs;
-+	unsigned long min_sample_us;
-+	unsigned long max_sample_us;
- };
- 
- /**
- * struct damon_ctx - Represents a context for each monitoring.  This is the
- * main interface that allows users to set the attributes and get the results
- * of the monitoring.
-+ * struct damon_attrs - Monitoring attributes for accuracy/overhead control.
-  *
-  * @sample_interval:		The time between access samplings.
-  * @aggr_interval:		The time between monitor results aggregations.
-  * @ops_update_interval:	The time between monitoring operations updates.
-+ * @intervals_goal:		Intervals auto-tuning goal.
-+ * @min_nr_regions:		The minimum number of adaptive monitoring
-+ *				regions.
-+ * @max_nr_regions:		The maximum number of adaptive monitoring
-+ *				regions.
-  *
-  * For each @sample_interval, DAMON checks whether each region is accessed or
- * not.  It aggregates and keeps the access information (number of accesses to
- * each region) for @aggr_interval time.  DAMON also checks whether the target
- * memory regions need update (e.g., by ``mmap()`` calls from the application,
- * in case of virtual memory monitoring) and applies the changes for each
- * @ops_update_interval.  All time intervals are in micro-seconds.
- * Please refer to &struct damon_operations and &struct damon_callback for more
- * detail.
-+ * not during the last @sample_interval.  If such access is found, DAMON
-+ * aggregates the information by increasing &damon_region->nr_accesses for
-+ * @aggr_interval time.  For each @aggr_interval, the count is reset.  DAMON
-+ * also checks whether the target memory regions need update (e.g., by
-+ * ``mmap()`` calls from the application, in case of virtual memory monitoring)
-+ * and applies the changes for each @ops_update_interval.  All time intervals
-+ * are in micro-seconds.  Please refer to &struct damon_operations and &struct
-+ * damon_call_control for more detail.
-+ */
-+struct damon_attrs {
-+	unsigned long sample_interval;
-+	unsigned long aggr_interval;
-+	unsigned long ops_update_interval;
-+	struct damon_intervals_goal intervals_goal;
-+	unsigned long min_nr_regions;
-+	unsigned long max_nr_regions;
-+/* private: internal use only */
-+	/*
-+	 * @aggr_interval to @sample_interval ratio.
-+	 * Core-external components call damon_set_attrs() with &damon_attrs
-+	 * that this field is unset.  In the case, damon_set_attrs() sets this
-+	 * field of resulting &damon_attrs.  Core-internal components such as
-+	 * kdamond_tune_intervals() calls damon_set_attrs() with &damon_attrs
-+	 * that this field is set.  In the case, damon_set_attrs() just keep
-+	 * it.
-+	 */
-+	unsigned long aggr_samples;
-+};
-+
-+/**
-+ * struct damon_ctx - Represents a context for each monitoring.  This is the
-+ * main interface that allows users to set the attributes and get the results
-+ * of the monitoring.
-  *
-+ * @attrs:		Monitoring attributes for accuracy/overhead control.
-  * @kdamond:		Kernel thread who does the monitoring.
-  * @kdamond_lock:	Mutex for the synchronizations with @kdamond.
-  *
-@@ -419,31 +768,52 @@ struct damon_callback {
-  * Accesses to other fields must be protected by themselves.
-  *
-  * @ops:	Set of monitoring operations for given use cases.
- * @callback:	Set of callbacks for monitoring events notifications.
- *
- * @min_nr_regions:	The minimum number of adaptive monitoring regions.
- * @max_nr_regions:	The maximum number of adaptive monitoring regions.
-+ * @addr_unit:	Scale factor for core to ops address conversion.
-+ * @min_sz_region:		Minimum region size.
-  * @adaptive_targets:	Head of monitoring targets (&damon_target) list.
-  * @schemes:		Head of schemes (&damos) list.
-  */
- struct damon_ctx {
-	unsigned long sample_interval;
-	unsigned long aggr_interval;
-	unsigned long ops_update_interval;
-+	struct damon_attrs attrs;
- 
- /* private: internal use only */
-	struct timespec64 last_aggregation;
-	struct timespec64 last_ops_update;
-+	/* number of sample intervals that passed since this context started */
-+	unsigned long passed_sample_intervals;
-+	/*
-+	 * number of sample intervals that should be passed before next
-+	 * aggregation
-+	 */
-+	unsigned long next_aggregation_sis;
-+	/*
-+	 * number of sample intervals that should be passed before next ops
-+	 * update
-+	 */
-+	unsigned long next_ops_update_sis;
-+	/*
-+	 * number of sample intervals that should be passed before next
-+	 * intervals tuning
-+	 */
-+	unsigned long next_intervals_tune_sis;
-+	/* for waiting until the execution of the kdamond_fn is started */
-+	struct completion kdamond_started;
-+	/* for scheme quotas prioritization */
-+	unsigned long *regions_score_histogram;
-+
-+	/* lists of &struct damon_call_control */
-+	struct list_head call_controls;
-+	struct mutex call_controls_lock;
-+
-+	struct damos_walk_control *walk_control;
-+	struct mutex walk_control_lock;
- 
- /* public: */
- 	struct task_struct *kdamond;
- 	struct mutex kdamond_lock;
- 
- 	struct damon_operations ops;
-	struct damon_callback callback;
-+	unsigned long addr_unit;
-+	unsigned long min_sz_region;
- 
-	unsigned long min_nr_regions;
-	unsigned long max_nr_regions;
- 	struct list_head adaptive_targets;
- 	struct list_head schemes;
- };
-@@ -463,9 +833,23 @@ static inline struct damon_region *damon_last_region(struct damon_target *t)
- 	return list_last_entry(&t->regions_list, struct damon_region, list);
- }
- 
-+static inline struct damon_region *damon_first_region(struct damon_target *t)
-+{
-+	return list_first_entry(&t->regions_list, struct damon_region, list);
-+}
-+
-+static inline unsigned long damon_sz_region(struct damon_region *r)
-+{
-+	return r->ar.end - r->ar.start;
-+}
-+
-+
- #define damon_for_each_region(r, t) \
- 	list_for_each_entry(r, &t->regions_list, list)
- 
-+#define damon_for_each_region_from(r, t) \
-+	list_for_each_entry_from(r, &t->regions_list, list)
-+
- #define damon_for_each_region_safe(r, next, t) \
- 	list_for_each_entry_safe(r, next, &t->regions_list, list)
- 
-@@ -481,6 +865,24 @@ static inline struct damon_region *damon_last_region(struct damon_target *t)
- #define damon_for_each_scheme_safe(s, next, ctx) \
- 	list_for_each_entry_safe(s, next, &(ctx)->schemes, list)
- 
-+#define damos_for_each_quota_goal(goal, quota) \
-+	list_for_each_entry(goal, &quota->goals, list)
-+
-+#define damos_for_each_quota_goal_safe(goal, next, quota) \
-+	list_for_each_entry_safe(goal, next, &(quota)->goals, list)
-+
-+#define damos_for_each_core_filter(f, scheme) \
-+	list_for_each_entry(f, &(scheme)->core_filters, list)
-+
-+#define damos_for_each_core_filter_safe(f, next, scheme) \
-+	list_for_each_entry_safe(f, next, &(scheme)->core_filters, list)
-+
-+#define damos_for_each_ops_filter(f, scheme) \
-+	list_for_each_entry(f, &(scheme)->ops_filters, list)
-+
-+#define damos_for_each_ops_filter_safe(f, next, scheme) \
-+	list_for_each_entry_safe(f, next, &(scheme)->ops_filters, list)
-+
- #ifdef CONFIG_DAMON
- 
- struct damon_region *damon_new_region(unsigned long start, unsigned long end);
-@@ -499,31 +901,45 @@ static inline void damon_insert_region(struct damon_region *r,
- void damon_add_region(struct damon_region *r, struct damon_target *t);
- void damon_destroy_region(struct damon_region *r, struct damon_target *t);
- int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges,
-		unsigned int nr_ranges);
-
-struct damos *damon_new_scheme(
-		unsigned long min_sz_region, unsigned long max_sz_region,
-		unsigned int min_nr_accesses, unsigned int max_nr_accesses,
-		unsigned int min_age_region, unsigned int max_age_region,
-		enum damos_action action, struct damos_quota *quota,
-		struct damos_watermarks *wmarks);
-+		unsigned int nr_ranges, unsigned long min_sz_region);
-+void damon_update_region_access_rate(struct damon_region *r, bool accessed,
-+		struct damon_attrs *attrs);
-+
-+struct damos_filter *damos_new_filter(enum damos_filter_type type,
-+		bool matching, bool allow);
-+void damos_add_filter(struct damos *s, struct damos_filter *f);
-+bool damos_filter_for_ops(enum damos_filter_type type);
-+void damos_destroy_filter(struct damos_filter *f);
-+
-+struct damos_quota_goal *damos_new_quota_goal(
-+		enum damos_quota_goal_metric metric,
-+		unsigned long target_value);
-+void damos_add_quota_goal(struct damos_quota *q, struct damos_quota_goal *g);
-+void damos_destroy_quota_goal(struct damos_quota_goal *goal);
-+
-+struct damos *damon_new_scheme(struct damos_access_pattern *pattern,
-+			enum damos_action action,
-+			unsigned long apply_interval_us,
-+			struct damos_quota *quota,
-+			struct damos_watermarks *wmarks,
-+			int target_nid);
- void damon_add_scheme(struct damon_ctx *ctx, struct damos *s);
- void damon_destroy_scheme(struct damos *s);
-+int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src);
- 
- struct damon_target *damon_new_target(void);
- void damon_add_target(struct damon_ctx *ctx, struct damon_target *t);
- bool damon_targets_empty(struct damon_ctx *ctx);
- void damon_free_target(struct damon_target *t);
-void damon_destroy_target(struct damon_target *t);
-+void damon_destroy_target(struct damon_target *t, struct damon_ctx *ctx);
- unsigned int damon_nr_regions(struct damon_target *t);
- 
- struct damon_ctx *damon_new_ctx(void);
- void damon_destroy_ctx(struct damon_ctx *ctx);
-int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int,
-		unsigned long aggr_int, unsigned long ops_upd_int,
-		unsigned long min_nr_reg, unsigned long max_nr_reg);
-int damon_set_schemes(struct damon_ctx *ctx,
-+int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs);
-+void damon_set_schemes(struct damon_ctx *ctx,
- 			struct damos **schemes, ssize_t nr_schemes);
-+int damon_commit_ctx(struct damon_ctx *old_ctx, struct damon_ctx *new_ctx);
- int damon_nr_running_ctxs(void);
- bool damon_is_registered_ops(enum damon_ops_id id);
- int damon_register_ops(struct damon_operations *ops);
-@@ -534,9 +950,25 @@ static inline bool damon_target_has_pid(const struct damon_ctx *ctx)
- 	return ctx->ops.id == DAMON_OPS_VADDR || ctx->ops.id == DAMON_OPS_FVADDR;
- }
- 
-+static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs)
-+{
-+	/* {aggr,sample}_interval are unsigned long, hence could overflow */
-+	return min(attrs->aggr_interval / attrs->sample_interval,
-+			(unsigned long)UINT_MAX);
-+}
-+
- 
-+bool damon_initialized(void);
- int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive);
- int damon_stop(struct damon_ctx **ctxs, int nr_ctxs);
-+bool damon_is_running(struct damon_ctx *ctx);
-+
-+int damon_call(struct damon_ctx *ctx, struct damon_call_control *control);
-+int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control);
-+
-+int damon_set_region_biggest_system_ram_default(struct damon_target *t,
-+				unsigned long *start, unsigned long *end,
-+				unsigned long min_sz_region);
- 
- #endif	/* CONFIG_DAMON */
- 
-diff --git a/include/linux/dax.h b/include/linux/dax.h
-index ba985333e26b..9d624f4d9df6 100644
--- a/include/linux/dax.h
-+++ b/include/linux/dax.h
-@@ -26,13 +26,7 @@ struct dax_operations {
- 	 * number of pages available for DAX at that pfn.
- 	 */
- 	long (*direct_access)(struct dax_device *, pgoff_t, long,
-			enum dax_access_mode, void **, pfn_t *);
-	/*
-	 * Validate whether this device is usable as an fsdax backing
-	 * device.
-	 */
-	bool (*dax_supported)(struct dax_device *, struct block_device *, int,
-			sector_t, sector_t);
-+			enum dax_access_mode, void **, unsigned long *);
- 	/* zero_page_range: required operation. Zero page range   */
- 	int (*zero_page_range)(struct dax_device *, pgoff_t, size_t);
- 	/*
-@@ -63,18 +57,21 @@ void kill_dax(struct dax_device *dax_dev);
- void dax_write_cache(struct dax_device *dax_dev, bool wc);
- bool dax_write_cache_enabled(struct dax_device *dax_dev);
- bool dax_synchronous(struct dax_device *dax_dev);
-+void set_dax_nocache(struct dax_device *dax_dev);
-+void set_dax_nomc(struct dax_device *dax_dev);
- void set_dax_synchronous(struct dax_device *dax_dev);
- size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
- 		void *addr, size_t bytes, struct iov_iter *i);
- /*
-  * Check if given mapping is supported by the file / underlying device.
-  */
-static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
-					     struct dax_device *dax_dev)
-+static inline bool daxdev_mapping_supported(vm_flags_t vm_flags,
-+					    const struct inode *inode,
-+					    struct dax_device *dax_dev)
- {
-	if (!(vma->vm_flags & VM_SYNC))
-+	if (!(vm_flags & VM_SYNC))
- 		return true;
-	if (!IS_DAX(file_inode(vma->vm_file)))
-+	if (!IS_DAX(inode))
- 		return false;
- 	return dax_synchronous(dax_dev);
- }
-@@ -86,11 +83,7 @@ static inline void *dax_holder(struct dax_device *dax_dev)
- static inline struct dax_device *alloc_dax(void *private,
- 		const struct dax_operations *ops)
- {
-	/*
-	 * Callers should check IS_ENABLED(CONFIG_DAX) to know if this
-	 * NULL is an error or expected.
-	 */
-	return NULL;
-+	return ERR_PTR(-EOPNOTSUPP);
- }
- static inline void put_dax(struct dax_device *dax_dev)
- {
-@@ -109,13 +102,20 @@ static inline bool dax_synchronous(struct dax_device *dax_dev)
- {
- 	return true;
- }
-+static inline void set_dax_nocache(struct dax_device *dax_dev)
-+{
-+}
-+static inline void set_dax_nomc(struct dax_device *dax_dev)
-+{
-+}
- static inline void set_dax_synchronous(struct dax_device *dax_dev)
- {
- }
-static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
-				struct dax_device *dax_dev)
-+static inline bool daxdev_mapping_supported(vm_flags_t vm_flags,
-+					    const struct inode *inode,
-+					    struct dax_device *dax_dev)
- {
-	return !(vma->vm_flags & VM_SYNC);
-+	return !(vm_flags & VM_SYNC);
- }
- static inline size_t dax_recovery_write(struct dax_device *dax_dev,
- 		pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
-@@ -124,9 +124,6 @@ static inline size_t dax_recovery_write(struct dax_device *dax_dev,
- }
- #endif
- 
-void set_dax_nocache(struct dax_device *dax_dev);
-void set_dax_nomc(struct dax_device *dax_dev);
-
- struct writeback_control;
- #if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)
- int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk);
-@@ -159,8 +156,8 @@ int dax_writeback_mapping_range(struct address_space *mapping,
- 
- struct page *dax_layout_busy_page(struct address_space *mapping);
- struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end);
-dax_entry_t dax_lock_page(struct page *page);
-void dax_unlock_page(struct page *page, dax_entry_t cookie);
-+dax_entry_t dax_lock_folio(struct folio *folio);
-+void dax_unlock_folio(struct folio *folio, dax_entry_t cookie);
- dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
- 		unsigned long index, struct page **page);
- void dax_unlock_mapping_entry(struct address_space *mapping,
-@@ -182,14 +179,14 @@ static inline int dax_writeback_mapping_range(struct address_space *mapping,
- 	return -EOPNOTSUPP;
- }
- 
-static inline dax_entry_t dax_lock_page(struct page *page)
-+static inline dax_entry_t dax_lock_folio(struct folio *folio)
- {
-	if (IS_DAX(page->mapping->host))
-+	if (IS_DAX(folio->mapping->host))
- 		return ~0UL;
- 	return 0;
- }
- 
-static inline void dax_unlock_page(struct page *page, dax_entry_t cookie)
-+static inline void dax_unlock_folio(struct folio *folio, dax_entry_t cookie)
- {
- }
- 
-@@ -205,11 +202,18 @@ static inline void dax_unlock_mapping_entry(struct address_space *mapping,
- }
- #endif
- 
-+int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len,
-+		const struct iomap_ops *ops);
- int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
- 		const struct iomap_ops *ops);
- int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
- 		const struct iomap_ops *ops);
- 
-+static inline bool dax_page_is_idle(struct page *page)
-+{
-+	return page && page_ref_count(page) == 0;
-+}
-+
- #if IS_ENABLED(CONFIG_DAX)
- int dax_read_lock(void);
- void dax_read_unlock(int id);
-@@ -223,10 +227,23 @@ static inline void dax_read_unlock(int id)
- {
- }
- #endif /* CONFIG_DAX */
-+
-+#if !IS_ENABLED(CONFIG_FS_DAX)
-+static inline int __must_check dax_break_layout(struct inode *inode,
-+			    loff_t start, loff_t end, void (cb)(struct inode *))
-+{
-+	return 0;
-+}
-+
-+static inline void dax_break_layout_final(struct inode *inode)
-+{
-+}
-+#endif
-+
- bool dax_alive(struct dax_device *dax_dev);
- void *dax_get_private(struct dax_device *dax_dev);
- long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
-		enum dax_access_mode mode, void **kaddr, pfn_t *pfn);
-+		enum dax_access_mode mode, void **kaddr, unsigned long *pfn);
- size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
- 		size_t bytes, struct iov_iter *i);
- size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
-@@ -239,13 +256,24 @@ void dax_flush(struct dax_device *dax_dev, void *addr, size_t size);
- 
- ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
- 		const struct iomap_ops *ops);
-vm_fault_t dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
-		    pfn_t *pfnp, int *errp, const struct iomap_ops *ops);
-+vm_fault_t dax_iomap_fault(struct vm_fault *vmf, unsigned int order,
-+			unsigned long *pfnp, int *errp,
-+			const struct iomap_ops *ops);
- vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
-		enum page_entry_size pe_size, pfn_t pfn);
-+		unsigned int order, unsigned long pfn);
- int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index);
-+void dax_delete_mapping_range(struct address_space *mapping,
-+				loff_t start, loff_t end);
- int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
- 				      pgoff_t index);
-+int __must_check dax_break_layout(struct inode *inode, loff_t start,
-+				loff_t end, void (cb)(struct inode *));
-+static inline int __must_check dax_break_layout_inode(struct inode *inode,
-+						void (cb)(struct inode *))
-+{
-+	return dax_break_layout(inode, 0, LLONG_MAX, cb);
-+}
-+void dax_break_layout_final(struct inode *inode);
- int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
- 				  struct inode *dest, loff_t destoff,
- 				  loff_t len, bool *is_same,
-@@ -259,12 +287,28 @@ static inline bool dax_mapping(struct address_space *mapping)
- 	return mapping->host && IS_DAX(mapping->host);
- }
- 
-+/*
-+ * Due to dax's memory and block duo personalities, hwpoison reporting
-+ * takes into consideration which personality is presently visible.
-+ * When dax acts like a block device, such as in block IO, an encounter of
-+ * dax hwpoison is reported as -EIO.
-+ * When dax acts like memory, such as in page fault, a detection of hwpoison
-+ * is reported as -EHWPOISON which leads to VM_FAULT_HWPOISON.
-+ */
-+static inline int dax_mem2blk_err(int err)
-+{
-+	return (err == -EHWPOISON) ? -EIO : err;
-+}
-+
- #ifdef CONFIG_DEV_DAX_HMEM_DEVICES
-void hmem_register_device(int target_nid, struct resource *r);
-+void hmem_register_resource(int target_nid, struct resource *r);
- #else
-static inline void hmem_register_device(int target_nid, struct resource *r)
-+static inline void hmem_register_resource(int target_nid, struct resource *r)
- {
- }
- #endif
- 
-+typedef int (*walk_hmem_fn)(struct device *dev, int target_nid,
-+			    const struct resource *res);
-+int walk_hmem_resources(struct device *dev, walk_hmem_fn fn);
- #endif
-diff --git a/include/linux/dcache.h b/include/linux/dcache.h
-index 92c78ed02b54..898c60d21c92 100644
--- a/include/linux/dcache.h
-+++ b/include/linux/dcache.h
-@@ -16,6 +16,7 @@
- #include <linux/wait.h>
- 
- struct path;
-+struct file;
- struct vfsmount;
- 
- /*
-@@ -56,6 +57,8 @@ struct qstr {
- };
- 
- #define QSTR_INIT(n,l) { { { .len = l } }, .name = n }
-+#define QSTR_LEN(n,l) (struct qstr)QSTR_INIT(n,l)
-+#define QSTR(n) QSTR_LEN(n, strlen(n))
- 
- extern const struct qstr empty_name;
- extern const struct qstr slash_name;
-@@ -67,16 +70,24 @@ extern const struct qstr dotdot_name;
-  * large memory footprint increase).
-  */
- #ifdef CONFIG_64BIT
-# define DNAME_INLINE_LEN 32 /* 192 bytes */
-+# define DNAME_INLINE_WORDS 5 /* 192 bytes */
- #else
- # ifdef CONFIG_SMP
-#  define DNAME_INLINE_LEN 36 /* 128 bytes */
-+#  define DNAME_INLINE_WORDS 9 /* 128 bytes */
- # else
-#  define DNAME_INLINE_LEN 40 /* 128 bytes */
-+#  define DNAME_INLINE_WORDS 11 /* 128 bytes */
- # endif
- #endif
- 
-+#define DNAME_INLINE_LEN (DNAME_INLINE_WORDS*sizeof(unsigned long))
-+
-+union shortname_store {
-+	unsigned char string[DNAME_INLINE_LEN];
-+	unsigned long words[DNAME_INLINE_WORDS];
-+};
-+
- #define d_lock	d_lockref.lock
-+#define d_iname d_shortname.string
- 
- struct dentry {
- 	/* RCU lookup touched fields */
-@@ -84,24 +95,32 @@ struct dentry {
- 	seqcount_spinlock_t d_seq;	/* per dentry seqlock */
- 	struct hlist_bl_node d_hash;	/* lookup hash list */
- 	struct dentry *d_parent;	/* parent directory */
-	struct qstr d_name;
-+	union {
-+	struct qstr __d_name;		/* for use ONLY in fs/dcache.c */
-+	const struct qstr d_name;
-+	};
- 	struct inode *d_inode;		/* Where the name belongs to - NULL is
- 					 * negative */
-	unsigned char d_iname[DNAME_INLINE_LEN];	/* small names */
-+	union shortname_store d_shortname;
-+	/* --- cacheline 1 boundary (64 bytes) was 32 bytes ago --- */
- 
- 	/* Ref lookup also touches following */
-	struct lockref d_lockref;	/* per-dentry lock and refcount */
- 	const struct dentry_operations *d_op;
- 	struct super_block *d_sb;	/* The root of the dentry tree */
- 	unsigned long d_time;		/* used by d_revalidate */
- 	void *d_fsdata;			/* fs-specific data */
-+	/* --- cacheline 2 boundary (128 bytes) --- */
-+	struct lockref d_lockref;	/* per-dentry lock and refcount
-+					 * keep separate from RCU lookup area if
-+					 * possible!
-+					 */
- 
- 	union {
- 		struct list_head d_lru;		/* LRU list */
- 		wait_queue_head_t *d_wait;	/* in-lookup ones only */
- 	};
-	struct list_head d_child;	/* child of parent list */
-	struct list_head d_subdirs;	/* our children */
-+	struct hlist_node d_sib;	/* child of parent list */
-+	struct hlist_head d_children;	/* our children */
- 	/*
- 	 * d_alias and d_rcu can share memory
- 	 */
-@@ -110,7 +129,7 @@ struct dentry {
- 		struct hlist_bl_node d_in_lookup_hash;	/* only for in-lookup ones */
- 	 	struct rcu_head d_rcu;
- 	} d_u;
-} __randomize_layout;
-+};
- 
- /*
-  * dentry->d_lock spinlock nesting subclasses:
-@@ -124,8 +143,14 @@ enum dentry_d_lock_class
- 	DENTRY_D_LOCK_NESTED
- };
- 
-+enum d_real_type {
-+	D_REAL_DATA,
-+	D_REAL_METADATA,
-+};
-+
- struct dentry_operations {
-	int (*d_revalidate)(struct dentry *, unsigned int);
-+	int (*d_revalidate)(struct inode *, const struct qstr *,
-+			    struct dentry *, unsigned int);
- 	int (*d_weak_revalidate)(struct dentry *, unsigned int);
- 	int (*d_hash)(const struct dentry *, struct qstr *);
- 	int (*d_compare)(const struct dentry *,
-@@ -138,7 +163,9 @@ struct dentry_operations {
- 	char *(*d_dname)(struct dentry *, char *, int);
- 	struct vfsmount *(*d_automount)(struct path *);
- 	int (*d_manage)(const struct path *, bool);
-	struct dentry *(*d_real)(struct dentry *, const struct inode *);
-+	struct dentry *(*d_real)(struct dentry *, enum d_real_type type);
-+	bool (*d_unalias_trylock)(const struct dentry *);
-+	void (*d_unalias_unlock)(const struct dentry *);
- } ____cacheline_aligned;
- 
- /*
-@@ -150,68 +177,59 @@ struct dentry_operations {
-  */
- 
- /* d_flags entries */
-#define DCACHE_OP_HASH			0x00000001
-#define DCACHE_OP_COMPARE		0x00000002
-#define DCACHE_OP_REVALIDATE		0x00000004
-#define DCACHE_OP_DELETE		0x00000008
-#define DCACHE_OP_PRUNE			0x00000010
-
-#define	DCACHE_DISCONNECTED		0x00000020
-     /* This dentry is possibly not currently connected to the dcache tree, in
-      * which case its parent will either be itself, or will have this flag as
-      * well.  nfsd will not use a dentry with this bit set, but will first
-      * endeavour to clear the bit either by discovering that it is connected,
-      * or by performing lookup operations.   Any filesystem which supports
-      * nfsd_operations MUST have a lookup function which, if it finds a
-      * directory inode with a DCACHE_DISCONNECTED dentry, will d_move that
-      * dentry into place and return that dentry rather than the passed one,
-      * typically using d_splice_alias. */
-
-#define DCACHE_REFERENCED		0x00000040 /* Recently used, don't discard. */
-
-#define DCACHE_DONTCACHE		0x00000080 /* Purge from memory on final dput() */
-
-#define DCACHE_CANT_MOUNT		0x00000100
-#define DCACHE_GENOCIDE			0x00000200
-#define DCACHE_SHRINK_LIST		0x00000400
-
-#define DCACHE_OP_WEAK_REVALIDATE	0x00000800
-
-#define DCACHE_NFSFS_RENAMED		0x00001000
-     /* this dentry has been "silly renamed" and has to be deleted on the last
-      * dput() */
-#define DCACHE_COOKIE			0x00002000 /* For use by dcookie subsystem */
-#define DCACHE_FSNOTIFY_PARENT_WATCHED	0x00004000
-     /* Parent inode is watched by some fsnotify listener */
-
-#define DCACHE_DENTRY_KILLED		0x00008000
-
-#define DCACHE_MOUNTED			0x00010000 /* is a mountpoint */
-#define DCACHE_NEED_AUTOMOUNT		0x00020000 /* handle automount on this dir */
-#define DCACHE_MANAGE_TRANSIT		0x00040000 /* manage transit from this dirent */
-+enum dentry_flags {
-+	DCACHE_OP_HASH			= BIT(0),
-+	DCACHE_OP_COMPARE		= BIT(1),
-+	DCACHE_OP_REVALIDATE		= BIT(2),
-+	DCACHE_OP_DELETE		= BIT(3),
-+	DCACHE_OP_PRUNE			= BIT(4),
-+	/*
-+	 * This dentry is possibly not currently connected to the dcache tree,
-+	 * in which case its parent will either be itself, or will have this
-+	 * flag as well.  nfsd will not use a dentry with this bit set, but will
-+	 * first endeavour to clear the bit either by discovering that it is
-+	 * connected, or by performing lookup operations.  Any filesystem which
-+	 * supports nfsd_operations MUST have a lookup function which, if it
-+	 * finds a directory inode with a DCACHE_DISCONNECTED dentry, will
-+	 * d_move that dentry into place and return that dentry rather than the
-+	 * passed one, typically using d_splice_alias.
-+	 */
-+	DCACHE_DISCONNECTED		= BIT(5),
-+	DCACHE_REFERENCED		= BIT(6),	/* Recently used, don't discard. */
-+	DCACHE_DONTCACHE		= BIT(7),	/* Purge from memory on final dput() */
-+	DCACHE_CANT_MOUNT		= BIT(8),
-+	DCACHE_SHRINK_LIST		= BIT(10),
-+	DCACHE_OP_WEAK_REVALIDATE	= BIT(11),
-+	/*
-+	 * this dentry has been "silly renamed" and has to be deleted on the
-+	 * last dput()
-+	 */
-+	DCACHE_NFSFS_RENAMED		= BIT(12),
-+	DCACHE_FSNOTIFY_PARENT_WATCHED	= BIT(13),	/* Parent inode is watched by some fsnotify listener */
-+	DCACHE_DENTRY_KILLED		= BIT(14),
-+	DCACHE_MOUNTED			= BIT(15),	/* is a mountpoint */
-+	DCACHE_NEED_AUTOMOUNT		= BIT(16),	/* handle automount on this dir */
-+	DCACHE_MANAGE_TRANSIT		= BIT(17),	/* manage transit from this dirent */
-+	DCACHE_LRU_LIST			= BIT(18),
-+	DCACHE_ENTRY_TYPE		= (7 << 19),	/* bits 19..21 are for storing type: */
-+	DCACHE_MISS_TYPE		= (0 << 19),	/* Negative dentry */
-+	DCACHE_WHITEOUT_TYPE		= (1 << 19),	/* Whiteout dentry (stop pathwalk) */
-+	DCACHE_DIRECTORY_TYPE		= (2 << 19),	/* Normal directory */
-+	DCACHE_AUTODIR_TYPE		= (3 << 19),	/* Lookupless directory (presumed automount) */
-+	DCACHE_REGULAR_TYPE		= (4 << 19),	/* Regular file type */
-+	DCACHE_SPECIAL_TYPE		= (5 << 19),	/* Other file type */
-+	DCACHE_SYMLINK_TYPE		= (6 << 19),	/* Symlink */
-+	DCACHE_NOKEY_NAME		= BIT(22),	/* Encrypted name encoded without key */
-+	DCACHE_OP_REAL			= BIT(23),
-+	DCACHE_PAR_LOOKUP		= BIT(24),	/* being looked up (with parent locked shared) */
-+	DCACHE_DENTRY_CURSOR		= BIT(25),
-+	DCACHE_NORCU			= BIT(26),	/* No RCU delay for freeing */
-+	DCACHE_PERSISTENT		= BIT(27)
-+};
-+
- #define DCACHE_MANAGED_DENTRY \
- 	(DCACHE_MOUNTED|DCACHE_NEED_AUTOMOUNT|DCACHE_MANAGE_TRANSIT)
- 
-#define DCACHE_LRU_LIST			0x00080000
-
-#define DCACHE_ENTRY_TYPE		0x00700000
-#define DCACHE_MISS_TYPE		0x00000000 /* Negative dentry (maybe fallthru to nowhere) */
-#define DCACHE_WHITEOUT_TYPE		0x00100000 /* Whiteout dentry (stop pathwalk) */
-#define DCACHE_DIRECTORY_TYPE		0x00200000 /* Normal directory */
-#define DCACHE_AUTODIR_TYPE		0x00300000 /* Lookupless directory (presumed automount) */
-#define DCACHE_REGULAR_TYPE		0x00400000 /* Regular file type (or fallthru to such) */
-#define DCACHE_SPECIAL_TYPE		0x00500000 /* Other file type (or fallthru to such) */
-#define DCACHE_SYMLINK_TYPE		0x00600000 /* Symlink (or fallthru to such) */
-
-#define DCACHE_MAY_FREE			0x00800000
-#define DCACHE_FALLTHRU			0x01000000 /* Fall through to lower layer */
-#define DCACHE_NOKEY_NAME		0x02000000 /* Encrypted name encoded without key */
-#define DCACHE_OP_REAL			0x04000000
-
-#define DCACHE_PAR_LOOKUP		0x10000000 /* being looked up (with parent locked shared) */
-#define DCACHE_DENTRY_CURSOR		0x20000000
-#define DCACHE_NORCU			0x40000000 /* No RCU delay for freeing */
-
- extern seqlock_t rename_lock;
- 
- /*
-@@ -219,12 +237,9 @@ extern seqlock_t rename_lock;
-  */
- extern void d_instantiate(struct dentry *, struct inode *);
- extern void d_instantiate_new(struct dentry *, struct inode *);
-extern struct dentry * d_instantiate_unique(struct dentry *, struct inode *);
-extern struct dentry * d_instantiate_anon(struct dentry *, struct inode *);
- extern void __d_drop(struct dentry *dentry);
- extern void d_drop(struct dentry *dentry);
- extern void d_delete(struct dentry *);
-extern void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op);
- 
- /* allocate/de-allocate */
- extern struct dentry * d_alloc(struct dentry *, const struct qstr *);
-@@ -232,28 +247,29 @@ extern struct dentry * d_alloc_anon(struct super_block *);
- extern struct dentry * d_alloc_parallel(struct dentry *, const struct qstr *,
- 					wait_queue_head_t *);
- extern struct dentry * d_splice_alias(struct inode *, struct dentry *);
-+/* weird procfs mess; *NOT* exported */
-+extern struct dentry * d_splice_alias_ops(struct inode *, struct dentry *,
-+					  const struct dentry_operations *);
- extern struct dentry * d_add_ci(struct dentry *, struct inode *, struct qstr *);
- extern bool d_same_name(const struct dentry *dentry, const struct dentry *parent,
- 			const struct qstr *name);
-extern struct dentry * d_exact_alias(struct dentry *, struct inode *);
- extern struct dentry *d_find_any_alias(struct inode *inode);
- extern struct dentry * d_obtain_alias(struct inode *);
- extern struct dentry * d_obtain_root(struct inode *);
- extern void shrink_dcache_sb(struct super_block *);
- extern void shrink_dcache_parent(struct dentry *);
-extern void shrink_dcache_for_umount(struct super_block *);
- extern void d_invalidate(struct dentry *);
- 
- /* only used at mount-time */
- extern struct dentry * d_make_root(struct inode *);
- 
-/* <clickety>-<click> the ramfs-type tree */
-extern void d_genocide(struct dentry *);
-
-extern void d_tmpfile(struct dentry *, struct inode *);
-+extern void d_mark_tmpfile(struct file *, struct inode *);
-+extern void d_tmpfile(struct file *, struct inode *);
- 
- extern struct dentry *d_find_alias(struct inode *);
- extern void d_prune_aliases(struct inode *);
-+extern void d_dispose_if_unused(struct dentry *, struct list_head *);
-+extern void shrink_dentry_list(struct list_head *);
- 
- extern struct dentry *d_find_alias_rcu(struct inode *);
- 
-@@ -272,23 +288,20 @@ extern void d_move(struct dentry *, struct dentry *);
- extern void d_exchange(struct dentry *, struct dentry *);
- extern struct dentry *d_ancestor(struct dentry *, struct dentry *);
- 
-/* appendix may either be NULL or be used for transname suffixes */
- extern struct dentry *d_lookup(const struct dentry *, const struct qstr *);
-extern struct dentry *d_hash_and_lookup(struct dentry *, struct qstr *);
-extern struct dentry *__d_lookup(const struct dentry *, const struct qstr *);
-extern struct dentry *__d_lookup_rcu(const struct dentry *parent,
-				const struct qstr *name, unsigned *seq);
- 
- static inline unsigned d_count(const struct dentry *dentry)
- {
- 	return dentry->d_lockref.count;
- }
- 
-+ino_t d_parent_ino(struct dentry *dentry);
-+
- /*
-  * helper function for dentry_operations.d_dname() members
-  */
-extern __printf(4, 5)
-char *dynamic_dname(struct dentry *, char *, int, const char *, ...);
-+extern __printf(3, 4)
-+char *dynamic_dname(char *, int, const char *, ...);
- 
- extern char *__d_path(const struct path *, const struct path *, char *, int);
- extern char *d_absolute_path(const struct path *, char *, int);
-@@ -299,20 +312,40 @@ extern char *dentry_path(const struct dentry *, char *, int);
- /* Allocation counts.. */
- 
- /**
- *	dget, dget_dlock -	get a reference to a dentry
- *	@dentry: dentry to get a reference to
-+ * dget_dlock -	get a reference to a dentry
-+ * @dentry: dentry to get a reference to
-  *
- *	Given a dentry or %NULL pointer increment the reference count
- *	if appropriate and return the dentry. A dentry will not be 
- *	destroyed when it has references.
-+ * Given a live dentry, increment the reference count and return the dentry.
-+ * Caller must hold @dentry->d_lock.  Making sure that dentry is alive is
-+ * caller's resonsibility.  There are many conditions sufficient to guarantee
-+ * that; e.g. anything with non-negative refcount is alive, so's anything
-+ * hashed, anything positive, anyone's parent, etc.
-  */
- static inline struct dentry *dget_dlock(struct dentry *dentry)
- {
-	if (dentry)
-		dentry->d_lockref.count++;
-+	dentry->d_lockref.count++;
- 	return dentry;
- }
- 
-+
-+/**
-+ * dget - get a reference to a dentry
-+ * @dentry: dentry to get a reference to
-+ *
-+ * Given a dentry or %NULL pointer increment the reference count
-+ * if appropriate and return the dentry.  A dentry will not be
-+ * destroyed when it has references.  Conversely, a dentry with
-+ * no references can disappear for any number of reasons, starting
-+ * with memory pressure.  In other words, that primitive is
-+ * used to clone an existing reference; using it on something with
-+ * zero refcount is a bug.
-+ *
-+ * NOTE: it will spin if @dentry->d_lock is held.  From the deadlock
-+ * avoidance point of view it is equivalent to spin_lock()/increment
-+ * refcount/spin_unlock(), so calling it under @dentry->d_lock is
-+ * always a bug; so's calling it under ->d_lock on any of its descendents.
-+ *
-+ */
- static inline struct dentry *dget(struct dentry *dentry)
- {
- 	if (dentry)
-@@ -323,12 +356,11 @@ static inline struct dentry *dget(struct dentry *dentry)
- extern struct dentry *dget_parent(struct dentry *dentry);
- 
- /**
- *	d_unhashed -	is dentry hashed
- *	@dentry: entry to check
-+ * d_unhashed - is dentry hashed
-+ * @dentry: entry to check
-  *
- *	Returns true if the dentry passed is not currently hashed.
-+ * Returns true if the dentry passed is not currently hashed.
-  */
- 
- static inline int d_unhashed(const struct dentry *dentry)
- {
- 	return hlist_bl_unhashed(&dentry->d_hash);
-@@ -488,20 +520,7 @@ static inline int simple_positive(const struct dentry *dentry)
- 	return d_really_is_positive(dentry) && !d_unhashed(dentry);
- }
- 
-extern void d_set_fallthru(struct dentry *dentry);
-
-static inline bool d_is_fallthru(const struct dentry *dentry)
-{
-	return dentry->d_flags & DCACHE_FALLTHRU;
-}
-
-
-extern int sysctl_vfs_cache_pressure;
-
-static inline unsigned long vfs_pressure_ratio(unsigned long val)
-{
-	return mult_frac(val, sysctl_vfs_cache_pressure, 100);
-}
-+unsigned long vfs_pressure_ratio(unsigned long val);
- 
- /**
-  * d_inode - Get the actual inode of this dentry
-@@ -545,41 +564,25 @@ static inline struct inode *d_backing_inode(const struct dentry *upper)
- }
- 
- /**
- * d_backing_dentry - Get upper or lower dentry we should be using
- * @upper: The upper layer
- *
- * This is the helper that should be used to get the dentry of the inode that
- * will be used if this dentry were opened as a file.  It may be the upper
- * dentry or it may be a lower dentry pinned by the upper.
- *
- * Normal filesystems should not use this to access their own dentries.
- */
-static inline struct dentry *d_backing_dentry(struct dentry *upper)
-{
-	return upper;
-}
-
-/**
-  * d_real - Return the real dentry
-  * @dentry: the dentry to query
- * @inode: inode to select the dentry from multiple layers (can be NULL)
-+ * @type: the type of real dentry (data or metadata)
-  *
-  * If dentry is on a union/overlay, then return the underlying, real dentry.
-  * Otherwise return the dentry itself.
-  *
-  * See also: Documentation/filesystems/vfs.rst
-  */
-static inline struct dentry *d_real(struct dentry *dentry,
-				    const struct inode *inode)
-+static inline struct dentry *d_real(struct dentry *dentry, enum d_real_type type)
- {
- 	if (unlikely(dentry->d_flags & DCACHE_OP_REAL))
-		return dentry->d_op->d_real(dentry, inode);
-+		return dentry->d_op->d_real(dentry, type);
- 	else
- 		return dentry;
- }
- 
- /**
- * d_real_inode - Return the real inode
-+ * d_real_inode - Return the real inode hosting the data
-  * @dentry: The dentry to query
-  *
-  * If dentry is on a union/overlay, then return the underlying, real inode.
-@@ -588,14 +591,28 @@ static inline struct dentry *d_real(struct dentry *dentry,
- static inline struct inode *d_real_inode(const struct dentry *dentry)
- {
- 	/* This usage of d_real() results in const dentry */
-	return d_backing_inode(d_real((struct dentry *) dentry, NULL));
-+	return d_inode(d_real((struct dentry *) dentry, D_REAL_DATA));
- }
- 
- struct name_snapshot {
- 	struct qstr name;
-	unsigned char inline_name[DNAME_INLINE_LEN];
-+	union shortname_store inline_name;
- };
- void take_dentry_name_snapshot(struct name_snapshot *, struct dentry *);
- void release_dentry_name_snapshot(struct name_snapshot *);
- 
-+static inline struct dentry *d_first_child(const struct dentry *dentry)
-+{
-+	return hlist_entry_safe(dentry->d_children.first, struct dentry, d_sib);
-+}
-+
-+static inline struct dentry *d_next_sibling(const struct dentry *dentry)
-+{
-+	return hlist_entry_safe(dentry->d_sib.next, struct dentry, d_sib);
-+}
-+
-+void set_default_d_op(struct super_block *, const struct dentry_operations *);
-+struct dentry *d_make_persistent(struct dentry *, struct inode *);
-+void d_make_discardable(struct dentry *dentry);
-+
- #endif	/* __LINUX_DCACHE_H */
-diff --git a/include/linux/dccp.h b/include/linux/dccp.h
-index 07e547c02fd8..0b61b8b996d4 100644
--- a/include/linux/dccp.h
-+++ b/include/linux/dccp.h
-@@ -2,79 +2,8 @@
- #ifndef _LINUX_DCCP_H
- #define _LINUX_DCCP_H
- 
-
-#include <linux/in.h>
-#include <linux/interrupt.h>
-#include <linux/ktime.h>
-#include <linux/list.h>
-#include <linux/uio.h>
-#include <linux/workqueue.h>
-
-#include <net/inet_connection_sock.h>
-#include <net/inet_sock.h>
-#include <net/inet_timewait_sock.h>
-#include <net/tcp_states.h>
- #include <uapi/linux/dccp.h>
- 
-enum dccp_state {
-	DCCP_OPEN	     = TCP_ESTABLISHED,
-	DCCP_REQUESTING	     = TCP_SYN_SENT,
-	DCCP_LISTEN	     = TCP_LISTEN,
-	DCCP_RESPOND	     = TCP_SYN_RECV,
-	/*
-	 * States involved in closing a DCCP connection:
-	 * 1) ACTIVE_CLOSEREQ is entered by a server sending a CloseReq.
-	 *
-	 * 2) CLOSING can have three different meanings (RFC 4340, 8.3):
-	 *  a. Client has performed active-close, has sent a Close to the server
-	 *     from state OPEN or PARTOPEN, and is waiting for the final Reset
-	 *     (in this case, SOCK_DONE == 1).
-	 *  b. Client is asked to perform passive-close, by receiving a CloseReq
-	 *     in (PART)OPEN state. It sends a Close and waits for final Reset
-	 *     (in this case, SOCK_DONE == 0).
-	 *  c. Server performs an active-close as in (a), keeps TIMEWAIT state.
-	 *
-	 * 3) The following intermediate states are employed to give passively
-	 *    closing nodes a chance to process their unread data:
-	 *    - PASSIVE_CLOSE    (from OPEN => CLOSED) and
-	 *    - PASSIVE_CLOSEREQ (from (PART)OPEN to CLOSING; case (b) above).
-	 */
-	DCCP_ACTIVE_CLOSEREQ = TCP_FIN_WAIT1,
-	DCCP_PASSIVE_CLOSE   = TCP_CLOSE_WAIT,	/* any node receiving a Close */
-	DCCP_CLOSING	     = TCP_CLOSING,
-	DCCP_TIME_WAIT	     = TCP_TIME_WAIT,
-	DCCP_CLOSED	     = TCP_CLOSE,
-	DCCP_NEW_SYN_RECV    = TCP_NEW_SYN_RECV,
-	DCCP_PARTOPEN	     = TCP_MAX_STATES,
-	DCCP_PASSIVE_CLOSEREQ,			/* clients receiving CloseReq */
-	DCCP_MAX_STATES
-};
-
-enum {
-	DCCPF_OPEN	      = TCPF_ESTABLISHED,
-	DCCPF_REQUESTING      = TCPF_SYN_SENT,
-	DCCPF_LISTEN	      = TCPF_LISTEN,
-	DCCPF_RESPOND	      = TCPF_SYN_RECV,
-	DCCPF_ACTIVE_CLOSEREQ = TCPF_FIN_WAIT1,
-	DCCPF_CLOSING	      = TCPF_CLOSING,
-	DCCPF_TIME_WAIT	      = TCPF_TIME_WAIT,
-	DCCPF_CLOSED	      = TCPF_CLOSE,
-	DCCPF_NEW_SYN_RECV    = TCPF_NEW_SYN_RECV,
-	DCCPF_PARTOPEN	      = (1 << DCCP_PARTOPEN),
-};
-
-static inline struct dccp_hdr *dccp_hdr(const struct sk_buff *skb)
-{
-	return (struct dccp_hdr *)skb_transport_header(skb);
-}
-
-static inline struct dccp_hdr *dccp_zeroed_hdr(struct sk_buff *skb, int headlen)
-{
-	skb_push(skb, headlen);
-	skb_reset_transport_header(skb);
-	return memset(skb_transport_header(skb), 0, headlen);
-}
-
- static inline struct dccp_hdr_ext *dccp_hdrx(const struct dccp_hdr *dh)
- {
- 	return (struct dccp_hdr_ext *)((unsigned char *)dh + sizeof(*dh));
-@@ -85,12 +14,6 @@ static inline unsigned int __dccp_basic_hdr_len(const struct dccp_hdr *dh)
- 	return sizeof(*dh) + (dh->dccph_x ? sizeof(struct dccp_hdr_ext) : 0);
- }
- 
-static inline unsigned int dccp_basic_hdr_len(const struct sk_buff *skb)
-{
-	const struct dccp_hdr *dh = dccp_hdr(skb);
-	return __dccp_basic_hdr_len(dh);
-}
-
- static inline __u64 dccp_hdr_seq(const struct dccp_hdr *dh)
- {
- 	__u64 seq_nr =  ntohs(dh->dccph_seq);
-@@ -103,224 +26,10 @@ static inline __u64 dccp_hdr_seq(const struct dccp_hdr *dh)
- 	return seq_nr;
- }
- 
-static inline struct dccp_hdr_request *dccp_hdr_request(struct sk_buff *skb)
-{
-	return (struct dccp_hdr_request *)(skb_transport_header(skb) +
-					   dccp_basic_hdr_len(skb));
-}
-
-static inline struct dccp_hdr_ack_bits *dccp_hdr_ack_bits(const struct sk_buff *skb)
-{
-	return (struct dccp_hdr_ack_bits *)(skb_transport_header(skb) +
-					    dccp_basic_hdr_len(skb));
-}
-
-static inline u64 dccp_hdr_ack_seq(const struct sk_buff *skb)
-{
-	const struct dccp_hdr_ack_bits *dhack = dccp_hdr_ack_bits(skb);
-	return ((u64)ntohs(dhack->dccph_ack_nr_high) << 32) + ntohl(dhack->dccph_ack_nr_low);
-}
-
-static inline struct dccp_hdr_response *dccp_hdr_response(struct sk_buff *skb)
-{
-	return (struct dccp_hdr_response *)(skb_transport_header(skb) +
-					    dccp_basic_hdr_len(skb));
-}
-
-static inline struct dccp_hdr_reset *dccp_hdr_reset(struct sk_buff *skb)
-{
-	return (struct dccp_hdr_reset *)(skb_transport_header(skb) +
-					 dccp_basic_hdr_len(skb));
-}
-
- static inline unsigned int __dccp_hdr_len(const struct dccp_hdr *dh)
- {
- 	return __dccp_basic_hdr_len(dh) +
- 	       dccp_packet_hdr_len(dh->dccph_type);
- }
- 
-static inline unsigned int dccp_hdr_len(const struct sk_buff *skb)
-{
-	return __dccp_hdr_len(dccp_hdr(skb));
-}
-
-/**
- * struct dccp_request_sock  -  represent DCCP-specific connection request
- * @dreq_inet_rsk: structure inherited from
- * @dreq_iss: initial sequence number, sent on the first Response (RFC 4340, 7.1)
- * @dreq_gss: greatest sequence number sent (for retransmitted Responses)
- * @dreq_isr: initial sequence number received in the first Request
- * @dreq_gsr: greatest sequence number received (for retransmitted Request(s))
- * @dreq_service: service code present on the Request (there is just one)
- * @dreq_featneg: feature negotiation options for this connection
- * The following two fields are analogous to the ones in dccp_sock:
- * @dreq_timestamp_echo: last received timestamp to echo (13.1)
- * @dreq_timestamp_echo: the time of receiving the last @dreq_timestamp_echo
- */
-struct dccp_request_sock {
-	struct inet_request_sock dreq_inet_rsk;
-	__u64			 dreq_iss;
-	__u64			 dreq_gss;
-	__u64			 dreq_isr;
-	__u64			 dreq_gsr;
-	__be32			 dreq_service;
-	spinlock_t		 dreq_lock;
-	struct list_head	 dreq_featneg;
-	__u32			 dreq_timestamp_echo;
-	__u32			 dreq_timestamp_time;
-};
-
-static inline struct dccp_request_sock *dccp_rsk(const struct request_sock *req)
-{
-	return (struct dccp_request_sock *)req;
-}
-
-extern struct inet_timewait_death_row dccp_death_row;
-
-extern int dccp_parse_options(struct sock *sk, struct dccp_request_sock *dreq,
-			      struct sk_buff *skb);
-
-struct dccp_options_received {
-	u64	dccpor_ndp:48;
-	u32	dccpor_timestamp;
-	u32	dccpor_timestamp_echo;
-	u32	dccpor_elapsed_time;
-};
-
-struct ccid;
-
-enum dccp_role {
-	DCCP_ROLE_UNDEFINED,
-	DCCP_ROLE_LISTEN,
-	DCCP_ROLE_CLIENT,
-	DCCP_ROLE_SERVER,
-};
-
-struct dccp_service_list {
-	__u32	dccpsl_nr;
-	__be32	dccpsl_list[];
-};
-
-#define DCCP_SERVICE_INVALID_VALUE htonl((__u32)-1)
-#define DCCP_SERVICE_CODE_IS_ABSENT		0
-
-static inline bool dccp_list_has_service(const struct dccp_service_list *sl,
-					const __be32 service)
-{
-	if (likely(sl != NULL)) {
-		u32 i = sl->dccpsl_nr;
-		while (i--)
-			if (sl->dccpsl_list[i] == service)
-				return true;
-	}
-	return false;
-}
-
-struct dccp_ackvec;
-
-/**
- * struct dccp_sock - DCCP socket state
- *
- * @dccps_swl - sequence number window low
- * @dccps_swh - sequence number window high
- * @dccps_awl - acknowledgement number window low
- * @dccps_awh - acknowledgement number window high
- * @dccps_iss - initial sequence number sent
- * @dccps_isr - initial sequence number received
- * @dccps_osr - first OPEN sequence number received
- * @dccps_gss - greatest sequence number sent
- * @dccps_gsr - greatest valid sequence number received
- * @dccps_gar - greatest valid ack number received on a non-Sync; initialized to %dccps_iss
- * @dccps_service - first (passive sock) or unique (active sock) service code
- * @dccps_service_list - second .. last service code on passive socket
- * @dccps_timestamp_echo - latest timestamp received on a TIMESTAMP option
- * @dccps_timestamp_time - time of receiving latest @dccps_timestamp_echo
- * @dccps_l_ack_ratio - feature-local Ack Ratio
- * @dccps_r_ack_ratio - feature-remote Ack Ratio
- * @dccps_l_seq_win - local Sequence Window (influences ack number validity)
- * @dccps_r_seq_win - remote Sequence Window (influences seq number validity)
- * @dccps_pcslen - sender   partial checksum coverage (via sockopt)
- * @dccps_pcrlen - receiver partial checksum coverage (via sockopt)
- * @dccps_send_ndp_count - local Send NDP Count feature (7.7.2)
- * @dccps_ndp_count - number of Non Data Packets since last data packet
- * @dccps_mss_cache - current value of MSS (path MTU minus header sizes)
- * @dccps_rate_last - timestamp for rate-limiting DCCP-Sync (RFC 4340, 7.5.4)
- * @dccps_featneg - tracks feature-negotiation state (mostly during handshake)
- * @dccps_hc_rx_ackvec - rx half connection ack vector
- * @dccps_hc_rx_ccid - CCID used for the receiver (or receiving half-connection)
- * @dccps_hc_tx_ccid - CCID used for the sender (or sending half-connection)
- * @dccps_options_received - parsed set of retrieved options
- * @dccps_qpolicy - TX dequeueing policy, one of %dccp_packet_dequeueing_policy
- * @dccps_tx_qlen - maximum length of the TX queue
- * @dccps_role - role of this sock, one of %dccp_role
- * @dccps_hc_rx_insert_options - receiver wants to add options when acking
- * @dccps_hc_tx_insert_options - sender wants to add options when sending
- * @dccps_server_timewait - server holds timewait state on close (RFC 4340, 8.3)
- * @dccps_sync_scheduled - flag which signals "send out-of-band message soon"
- * @dccps_xmitlet - tasklet scheduled by the TX CCID to dequeue data packets
- * @dccps_xmit_timer - used by the TX CCID to delay sending (rate-based pacing)
- * @dccps_syn_rtt - RTT sample from Request/Response exchange (in usecs)
- */
-struct dccp_sock {
-	/* inet_connection_sock has to be the first member of dccp_sock */
-	struct inet_connection_sock	dccps_inet_connection;
-#define dccps_syn_rtt			dccps_inet_connection.icsk_ack.lrcvtime
-	__u64				dccps_swl;
-	__u64				dccps_swh;
-	__u64				dccps_awl;
-	__u64				dccps_awh;
-	__u64				dccps_iss;
-	__u64				dccps_isr;
-	__u64				dccps_osr;
-	__u64				dccps_gss;
-	__u64				dccps_gsr;
-	__u64				dccps_gar;
-	__be32				dccps_service;
-	__u32				dccps_mss_cache;
-	struct dccp_service_list	*dccps_service_list;
-	__u32				dccps_timestamp_echo;
-	__u32				dccps_timestamp_time;
-	__u16				dccps_l_ack_ratio;
-	__u16				dccps_r_ack_ratio;
-	__u64				dccps_l_seq_win:48;
-	__u64				dccps_r_seq_win:48;
-	__u8				dccps_pcslen:4;
-	__u8				dccps_pcrlen:4;
-	__u8				dccps_send_ndp_count:1;
-	__u64				dccps_ndp_count:48;
-	unsigned long			dccps_rate_last;
-	struct list_head		dccps_featneg;
-	struct dccp_ackvec		*dccps_hc_rx_ackvec;
-	struct ccid			*dccps_hc_rx_ccid;
-	struct ccid			*dccps_hc_tx_ccid;
-	struct dccp_options_received	dccps_options_received;
-	__u8				dccps_qpolicy;
-	__u32				dccps_tx_qlen;
-	enum dccp_role			dccps_role:2;
-	__u8				dccps_hc_rx_insert_options:1;
-	__u8				dccps_hc_tx_insert_options:1;
-	__u8				dccps_server_timewait:1;
-	__u8				dccps_sync_scheduled:1;
-	struct tasklet_struct		dccps_xmitlet;
-	struct timer_list		dccps_xmit_timer;
-};
-
-static inline struct dccp_sock *dccp_sk(const struct sock *sk)
-{
-	return (struct dccp_sock *)sk;
-}
-
-static inline const char *dccp_role(const struct sock *sk)
-{
-	switch (dccp_sk(sk)->dccps_role) {
-	case DCCP_ROLE_UNDEFINED: return "undefined";
-	case DCCP_ROLE_LISTEN:	  return "listen";
-	case DCCP_ROLE_SERVER:	  return "server";
-	case DCCP_ROLE_CLIENT:	  return "client";
-	}
-	return NULL;
-}
-
-extern void dccp_syn_ack_timeout(const struct request_sock *req);
-
- #endif /* _LINUX_DCCP_H */
-diff --git a/include/linux/debugfs.h b/include/linux/debugfs.h
-index c869f1e73d75..7cecda29447e 100644
--- a/include/linux/debugfs.h
-+++ b/include/linux/debugfs.h
-@@ -45,7 +45,7 @@ struct debugfs_u32_array {
- 
- extern struct dentry *arch_debugfs_dir;
- 
-#define DEFINE_DEBUGFS_ATTRIBUTE(__fops, __get, __set, __fmt)		\
-+#define DEFINE_DEBUGFS_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, __is_signed)	\
- static int __fops ## _open(struct inode *inode, struct file *file)	\
- {									\
- 	__simple_attr_check_format(__fmt, 0ull);			\
-@@ -56,19 +56,88 @@ static const struct file_operations __fops = {				\
- 	.open	 = __fops ## _open,					\
- 	.release = simple_attr_release,					\
- 	.read	 = debugfs_attr_read,					\
-	.write	 = debugfs_attr_write,					\
-	.llseek  = no_llseek,						\
-+	.write	 = (__is_signed) ? debugfs_attr_write_signed : debugfs_attr_write,	\
- }
- 
-+#define DEFINE_DEBUGFS_ATTRIBUTE(__fops, __get, __set, __fmt)		\
-+	DEFINE_DEBUGFS_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, false)
-+
-+#define DEFINE_DEBUGFS_ATTRIBUTE_SIGNED(__fops, __get, __set, __fmt)	\
-+	DEFINE_DEBUGFS_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, true)
-+
- typedef struct vfsmount *(*debugfs_automount_t)(struct dentry *, void *);
- 
-+struct debugfs_short_fops {
-+	ssize_t (*read)(struct file *, char __user *, size_t, loff_t *);
-+	ssize_t (*write)(struct file *, const char __user *, size_t, loff_t *);
-+	loff_t (*llseek) (struct file *, loff_t, int);
-+};
-+
- #if defined(CONFIG_DEBUG_FS)
- 
- struct dentry *debugfs_lookup(const char *name, struct dentry *parent);
- 
-struct dentry *debugfs_create_file(const char *name, umode_t mode,
-				   struct dentry *parent, void *data,
-				   const struct file_operations *fops);
-+struct dentry *debugfs_create_file_full(const char *name, umode_t mode,
-+					struct dentry *parent, void *data,
-+					const void *aux,
-+					const struct file_operations *fops);
-+struct dentry *debugfs_create_file_short(const char *name, umode_t mode,
-+					 struct dentry *parent, void *data,
-+					 const void *aux,
-+					 const struct debugfs_short_fops *fops);
-+
-+/**
-+ * debugfs_create_file - create a file in the debugfs filesystem
-+ * @name: a pointer to a string containing the name of the file to create.
-+ * @mode: the permission that the file should have.
-+ * @parent: a pointer to the parent dentry for this file.  This should be a
-+ *          directory dentry if set.  If this parameter is NULL, then the
-+ *          file will be created in the root of the debugfs filesystem.
-+ * @data: a pointer to something that the caller will want to get to later
-+ *        on.  The inode.i_private pointer will point to this value on
-+ *        the open() call.
-+ * @fops: a pointer to a struct file_operations or struct debugfs_short_fops that
-+ *        should be used for this file.
-+ *
-+ * This is the basic "create a file" function for debugfs.  It allows for a
-+ * wide range of flexibility in creating a file, or a directory (if you want
-+ * to create a directory, the debugfs_create_dir() function is
-+ * recommended to be used instead.)
-+ *
-+ * This function will return a pointer to a dentry if it succeeds.  This
-+ * pointer must be passed to the debugfs_remove() function when the file is
-+ * to be removed (no automatic cleanup happens if your module is unloaded,
-+ * you are responsible here.)  If an error occurs, ERR_PTR(-ERROR) will be
-+ * returned.
-+ *
-+ * If debugfs is not enabled in the kernel, the value -%ENODEV will be
-+ * returned.
-+ *
-+ * If fops points to a struct debugfs_short_fops, then simple_open() will be
-+ * used for the open, and only read/write/llseek are supported and are proxied,
-+ * so no module reference or release are needed.
-+ *
-+ * NOTE: it's expected that most callers should _ignore_ the errors returned
-+ * by this function. Other debugfs functions handle the fact that the "dentry"
-+ * passed to them could be an error and they don't crash in that case.
-+ * Drivers should generally work fine even if debugfs fails to init anyway.
-+ */
-+#define debugfs_create_file(name, mode, parent, data, fops)			\
-+	_Generic(fops,								\
-+		 const struct file_operations *: debugfs_create_file_full,	\
-+		 const struct debugfs_short_fops *: debugfs_create_file_short,	\
-+		 struct file_operations *: debugfs_create_file_full,		\
-+		 struct debugfs_short_fops *: debugfs_create_file_short)	\
-+		(name, mode, parent, data, NULL, fops)
-+
-+#define debugfs_create_file_aux(name, mode, parent, data, aux, fops)		\
-+	_Generic(fops,								\
-+		 const struct file_operations *: debugfs_create_file_full,	\
-+		 const struct debugfs_short_fops *: debugfs_create_file_short,	\
-+		 struct file_operations *: debugfs_create_file_full,		\
-+		 struct debugfs_short_fops *: debugfs_create_file_short)	\
-+		(name, mode, parent, data, aux, fops)
-+
- struct dentry *debugfs_create_file_unsafe(const char *name, umode_t mode,
- 				   struct dentry *parent, void *data,
- 				   const struct file_operations *fops);
-@@ -91,7 +160,9 @@ struct dentry *debugfs_create_automount(const char *name,
- void debugfs_remove(struct dentry *dentry);
- #define debugfs_remove_recursive debugfs_remove
- 
-const struct file_operations *debugfs_real_fops(const struct file *filp);
-+void debugfs_lookup_and_remove(const char *name, struct dentry *parent);
-+
-+void *debugfs_get_aux(const struct file *file);
- 
- int debugfs_file_get(struct dentry *dentry);
- void debugfs_file_put(struct dentry *dentry);
-@@ -100,9 +171,10 @@ ssize_t debugfs_attr_read(struct file *file, char __user *buf,
- 			size_t len, loff_t *ppos);
- ssize_t debugfs_attr_write(struct file *file, const char __user *buf,
- 			size_t len, loff_t *ppos);
-+ssize_t debugfs_attr_write_signed(struct file *file, const char __user *buf,
-+			size_t len, loff_t *ppos);
- 
-struct dentry *debugfs_rename(struct dentry *old_dir, struct dentry *old_dentry,
-                struct dentry *new_dir, const char *new_name);
-+int debugfs_change_name(struct dentry *dentry, const char *fmt, ...) __printf(2, 3);
- 
- void debugfs_create_u8(const char *name, umode_t mode, struct dentry *parent,
- 		       u8 *value);
-@@ -161,6 +233,25 @@ ssize_t debugfs_write_file_bool(struct file *file, const char __user *user_buf,
- ssize_t debugfs_read_file_str(struct file *file, char __user *user_buf,
- 			      size_t count, loff_t *ppos);
- 
-+/**
-+ * struct debugfs_cancellation - cancellation data
-+ * @list: internal, for keeping track
-+ * @cancel: callback to call
-+ * @cancel_data: extra data for the callback to call
-+ */
-+struct debugfs_cancellation {
-+	struct list_head list;
-+	void (*cancel)(struct dentry *, void *);
-+	void *cancel_data;
-+};
-+
-+void __acquires(cancellation)
-+debugfs_enter_cancellation(struct file *file,
-+			   struct debugfs_cancellation *cancellation);
-+void __releases(cancellation)
-+debugfs_leave_cancellation(struct file *file,
-+			   struct debugfs_cancellation *cancellation);
-+
- #else
- 
- #include <linux/err.h>
-@@ -177,9 +268,17 @@ static inline struct dentry *debugfs_lookup(const char *name,
- 	return ERR_PTR(-ENODEV);
- }
- 
-+static inline struct dentry *debugfs_create_file_aux(const char *name,
-+					umode_t mode, struct dentry *parent,
-+					void *data, void *aux,
-+					const void *fops)
-+{
-+	return ERR_PTR(-ENODEV);
-+}
-+
- static inline struct dentry *debugfs_create_file(const char *name, umode_t mode,
- 					struct dentry *parent, void *data,
-					const struct file_operations *fops)
-+					const void *fops)
- {
- 	return ERR_PTR(-ENODEV);
- }
-@@ -225,7 +324,11 @@ static inline void debugfs_remove(struct dentry *dentry)
- static inline void debugfs_remove_recursive(struct dentry *dentry)
- { }
- 
-const struct file_operations *debugfs_real_fops(const struct file *filp);
-+static inline void debugfs_lookup_and_remove(const char *name,
-+					     struct dentry *parent)
-+{ }
-+
-+void *debugfs_get_aux(const struct file *file);
- 
- static inline int debugfs_file_get(struct dentry *dentry)
- {
-@@ -248,10 +351,17 @@ static inline ssize_t debugfs_attr_write(struct file *file,
- 	return -ENODEV;
- }
- 
-static inline struct dentry *debugfs_rename(struct dentry *old_dir, struct dentry *old_dentry,
-                struct dentry *new_dir, char *new_name)
-+static inline ssize_t debugfs_attr_write_signed(struct file *file,
-+					const char __user *buf,
-+					size_t len, loff_t *ppos)
- {
-	return ERR_PTR(-ENODEV);
-+	return -ENODEV;
-+}
-+
-+static inline int __printf(2, 3) debugfs_change_name(struct dentry *dentry,
-+					const char *fmt, ...)
-+{
-+	return -ENODEV;
- }
- 
- static inline void debugfs_create_u8(const char *name, umode_t mode,
-@@ -359,6 +469,11 @@ static inline ssize_t debugfs_read_file_str(struct file *file,
- 
- #endif
- 
-+#define debugfs_create_file_aux_num(name, mode, parent, data, n, fops) \
-+	debugfs_create_file_aux(name, mode, parent, data, \
-+				(void *)(unsigned long)n, fops)
-+#define debugfs_get_aux_num(f) (unsigned long)debugfs_get_aux(f)
-+
- /**
-  * debugfs_create_xul - create a debugfs file that is used to read and write an
-  * unsigned long value, formatted in hexadecimal
-diff --git a/include/linux/debugobjects.h b/include/linux/debugobjects.h
-index 32444686b6ff..8b95545e7924 100644
--- a/include/linux/debugobjects.h
-+++ b/include/linux/debugobjects.h
-@@ -23,13 +23,17 @@ struct debug_obj_descr;
-  * @state:	tracked object state
-  * @astate:	current active state
-  * @object:	pointer to the real object
-+ * @batch_last:	pointer to the last hlist node in a batch
-  * @descr:	pointer to an object type specific debug description structure
-  */
- struct debug_obj {
-	struct hlist_node	node;
-	enum debug_obj_state	state;
-	unsigned int		astate;
-	void			*object;
-+	struct hlist_node		node;
-+	enum debug_obj_state		state;
-+	unsigned int			astate;
-+	union {
-+		void			*object;
-+		struct hlist_node	*batch_last;
-+	};
- 	const struct debug_obj_descr *descr;
- };
- 
-diff --git a/include/linux/decompress/mm.h b/include/linux/decompress/mm.h
-index 9192986b1a73..ac862422df15 100644
--- a/include/linux/decompress/mm.h
-+++ b/include/linux/decompress/mm.h
-@@ -48,7 +48,7 @@ MALLOC_VISIBLE void *malloc(int size)
- 	if (!malloc_ptr)
- 		malloc_ptr = free_mem_ptr;
- 
-	malloc_ptr = (malloc_ptr + 3) & ~3;     /* Align */
-+	malloc_ptr = (malloc_ptr + 7) & ~7;     /* Align */
- 
- 	p = (void *)malloc_ptr;
- 	malloc_ptr += size;
-diff --git a/include/linux/decompress/unxz.h b/include/linux/decompress/unxz.h
-index f764e2a7201e..3dd2658a9dab 100644
--- a/include/linux/decompress/unxz.h
-+++ b/include/linux/decompress/unxz.h
-@@ -1,10 +1,9 @@
-+/* SPDX-License-Identifier: 0BSD */
-+
- /*
-  * Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd
-  *
-  * Author: Lasse Collin <lasse.collin@tukaani.org>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
-  */
- 
- #ifndef DECOMPRESS_UNXZ_H
-diff --git a/include/linux/delay.h b/include/linux/delay.h
-index 039e7e0c7378..46412c00033a 100644
--- a/include/linux/delay.h
-+++ b/include/linux/delay.h
-@@ -6,21 +6,12 @@
-  * Copyright (C) 1993 Linus Torvalds
-  *
-  * Delay routines, using a pre-computed "loops_per_jiffy" value.
- *
- * Please note that ndelay(), udelay() and mdelay() may return early for
- * several reasons:
- *  1. computed loops_per_jiffy too low (due to the time taken to
- *     execute the timer interrupt.)
- *  2. cache behaviour affecting the time it takes to execute the
- *     loop function.
- *  3. CPU clock rate changes.
- *
- * Please see this thread:
- *   https://lists.openwall.net/linux-kernel/2011/01/09/56
-+ * Sleep routines using timer list timers or hrtimers.
-  */
- 
- #include <linux/math.h>
- #include <linux/sched.h>
-+#include <linux/jiffies.h>
- 
- extern unsigned long loops_per_jiffy;
- 
-@@ -35,12 +26,21 @@ extern unsigned long loops_per_jiffy;
-  * The 2nd mdelay() definition ensures GCC will optimize away the 
-  * while loop for the common cases where n <= MAX_UDELAY_MS  --  Paul G.
-  */
-
- #ifndef MAX_UDELAY_MS
- #define MAX_UDELAY_MS	5
- #endif
- 
- #ifndef mdelay
-+/**
-+ * mdelay - Inserting a delay based on milliseconds with busy waiting
-+ * @n:	requested delay in milliseconds
-+ *
-+ * See udelay() for basic information about mdelay() and it's variants.
-+ *
-+ * Please double check, whether mdelay() is the right way to go or whether a
-+ * refactoring of the code is the better variant to be able to use msleep()
-+ * instead.
-+ */
- #define mdelay(n) (\
- 	(__builtin_constant_p(n) && (n)<=MAX_UDELAY_MS) ? udelay((n)*1000) : \
- 	({unsigned long __ms=(n); while (__ms--) udelay(1000);}))
-@@ -56,36 +56,82 @@ static inline void ndelay(unsigned long x)
- 
- extern unsigned long lpj_fine;
- void calibrate_delay(void);
-+unsigned long calibrate_delay_is_known(void);
- void __attribute__((weak)) calibration_delay_done(void);
- void msleep(unsigned int msecs);
- unsigned long msleep_interruptible(unsigned int msecs);
- void usleep_range_state(unsigned long min, unsigned long max,
- 			unsigned int state);
- 
-+/**
-+ * usleep_range - Sleep for an approximate time
-+ * @min:	Minimum time in microseconds to sleep
-+ * @max:	Maximum time in microseconds to sleep
-+ *
-+ * For basic information please refer to usleep_range_state().
-+ *
-+ * The task will be in the state TASK_UNINTERRUPTIBLE during the sleep.
-+ */
- static inline void usleep_range(unsigned long min, unsigned long max)
- {
- 	usleep_range_state(min, max, TASK_UNINTERRUPTIBLE);
- }
- 
-static inline void usleep_idle_range(unsigned long min, unsigned long max)
-+/**
-+ * usleep_range_idle - Sleep for an approximate time with idle time accounting
-+ * @min:	Minimum time in microseconds to sleep
-+ * @max:	Maximum time in microseconds to sleep
-+ *
-+ * For basic information please refer to usleep_range_state().
-+ *
-+ * The sleeping task has the state TASK_IDLE during the sleep to prevent
-+ * contribution to the load average.
-+ */
-+static inline void usleep_range_idle(unsigned long min, unsigned long max)
- {
- 	usleep_range_state(min, max, TASK_IDLE);
- }
- 
-+/**
-+ * ssleep - wrapper for seconds around msleep
-+ * @seconds:	Requested sleep duration in seconds
-+ *
-+ * Please refer to msleep() for detailed information.
-+ */
- static inline void ssleep(unsigned int seconds)
- {
- 	msleep(seconds * 1000);
- }
- 
-/* see Documentation/timers/timers-howto.rst for the thresholds */
-+static const unsigned int max_slack_shift = 2;
-+#define USLEEP_RANGE_UPPER_BOUND	((TICK_NSEC << max_slack_shift) / NSEC_PER_USEC)
-+
-+/**
-+ * fsleep - flexible sleep which autoselects the best mechanism
-+ * @usecs:	requested sleep duration in microseconds
-+ *
-+ * flseep() selects the best mechanism that will provide maximum 25% slack
-+ * to the requested sleep duration. Therefore it uses:
-+ *
-+ * * udelay() loop for sleep durations <= 10 microseconds to avoid hrtimer
-+ *   overhead for really short sleep durations.
-+ * * usleep_range() for sleep durations which would lead with the usage of
-+ *   msleep() to a slack larger than 25%. This depends on the granularity of
-+ *   jiffies.
-+ * * msleep() for all other sleep durations.
-+ *
-+ * Note: When %CONFIG_HIGH_RES_TIMERS is not set, all sleeps are processed with
-+ * the granularity of jiffies and the slack might exceed 25% especially for
-+ * short sleep durations.
-+ */
- static inline void fsleep(unsigned long usecs)
- {
- 	if (usecs <= 10)
- 		udelay(usecs);
-	else if (usecs <= 20000)
-		usleep_range(usecs, 2 * usecs);
-+	else if (usecs < USLEEP_RANGE_UPPER_BOUND)
-+		usleep_range(usecs, usecs + (usecs >> max_slack_shift));
- 	else
-		msleep(DIV_ROUND_UP(usecs, 1000));
-+		msleep(DIV_ROUND_UP(usecs, USEC_PER_MSEC));
- }
- 
- #endif /* defined(_LINUX_DELAY_H) */
-diff --git a/include/linux/delayacct.h b/include/linux/delayacct.h
-index 58aea2d7385c..800dcc360db2 100644
--- a/include/linux/delayacct.h
-+++ b/include/linux/delayacct.h
-@@ -29,29 +29,46 @@ struct task_delay_info {
- 	 * XXX_delay contains the accumulated delay time in nanoseconds.
- 	 */
- 	u64 blkio_start;
-+	u64 blkio_delay_max;
-+	u64 blkio_delay_min;
- 	u64 blkio_delay;	/* wait for sync block io completion */
- 	u64 swapin_start;
-+	u64 swapin_delay_max;
-+	u64 swapin_delay_min;
- 	u64 swapin_delay;	/* wait for swapin */
- 	u32 blkio_count;	/* total count of the number of sync block */
- 				/* io operations performed */
- 	u32 swapin_count;	/* total count of swapin */
- 
- 	u64 freepages_start;
-+	u64 freepages_delay_max;
-+	u64 freepages_delay_min;
- 	u64 freepages_delay;	/* wait for memory reclaim */
- 
- 	u64 thrashing_start;
-+	u64 thrashing_delay_max;
-+	u64 thrashing_delay_min;
- 	u64 thrashing_delay;	/* wait for thrashing page */
- 
- 	u64 compact_start;
-+	u64 compact_delay_max;
-+	u64 compact_delay_min;
- 	u64 compact_delay;	/* wait for memory compact */
- 
- 	u64 wpcopy_start;
-+	u64 wpcopy_delay_max;
-+	u64 wpcopy_delay_min;
- 	u64 wpcopy_delay;	/* wait for write-protect copy */
- 
-+	u64 irq_delay_max;
-+	u64 irq_delay_min;
-+	u64 irq_delay;	/* wait for IRQ/SOFTIRQ */
-+
- 	u32 freepages_count;	/* total count of memory reclaim */
- 	u32 thrashing_count;	/* total count of thrash waits */
- 	u32 compact_count;	/* total count of memory compact */
- 	u32 wpcopy_count;	/* total count of write-protect copy */
-+	u32 irq_count;	/* total count of IRQ/SOFTIRQ */
- };
- #endif
- 
-@@ -73,14 +90,15 @@ extern int delayacct_add_tsk(struct taskstats *, struct task_struct *);
- extern __u64 __delayacct_blkio_ticks(struct task_struct *);
- extern void __delayacct_freepages_start(void);
- extern void __delayacct_freepages_end(void);
-extern void __delayacct_thrashing_start(void);
-extern void __delayacct_thrashing_end(void);
-+extern void __delayacct_thrashing_start(bool *in_thrashing);
-+extern void __delayacct_thrashing_end(bool *in_thrashing);
- extern void __delayacct_swapin_start(void);
- extern void __delayacct_swapin_end(void);
- extern void __delayacct_compact_start(void);
- extern void __delayacct_compact_end(void);
- extern void __delayacct_wpcopy_start(void);
- extern void __delayacct_wpcopy_end(void);
-+extern void __delayacct_irq(struct task_struct *task, u32 delta);
- 
- static inline void delayacct_tsk_init(struct task_struct *tsk)
- {
-@@ -143,22 +161,22 @@ static inline void delayacct_freepages_end(void)
- 		__delayacct_freepages_end();
- }
- 
-static inline void delayacct_thrashing_start(void)
-+static inline void delayacct_thrashing_start(bool *in_thrashing)
- {
- 	if (!static_branch_unlikely(&delayacct_key))
- 		return;
- 
- 	if (current->delays)
-		__delayacct_thrashing_start();
-+		__delayacct_thrashing_start(in_thrashing);
- }
- 
-static inline void delayacct_thrashing_end(void)
-+static inline void delayacct_thrashing_end(bool *in_thrashing)
- {
- 	if (!static_branch_unlikely(&delayacct_key))
- 		return;
- 
- 	if (current->delays)
-		__delayacct_thrashing_end();
-+		__delayacct_thrashing_end(in_thrashing);
- }
- 
- static inline void delayacct_swapin_start(void)
-@@ -215,6 +233,15 @@ static inline void delayacct_wpcopy_end(void)
- 		__delayacct_wpcopy_end();
- }
- 
-+static inline void delayacct_irq(struct task_struct *task, u32 delta)
-+{
-+	if (!static_branch_unlikely(&delayacct_key))
-+		return;
-+
-+	if (task->delays)
-+		__delayacct_irq(task, delta);
-+}
-+
- #else
- static inline void delayacct_init(void)
- {}
-@@ -237,9 +264,9 @@ static inline void delayacct_freepages_start(void)
- {}
- static inline void delayacct_freepages_end(void)
- {}
-static inline void delayacct_thrashing_start(void)
-+static inline void delayacct_thrashing_start(bool *in_thrashing)
- {}
-static inline void delayacct_thrashing_end(void)
-+static inline void delayacct_thrashing_end(bool *in_thrashing)
- {}
- static inline void delayacct_swapin_start(void)
- {}
-@@ -253,6 +280,8 @@ static inline void delayacct_wpcopy_start(void)
- {}
- static inline void delayacct_wpcopy_end(void)
- {}
-+static inline void delayacct_irq(struct task_struct *task, u32 delta)
-+{}
- 
- #endif /* CONFIG_TASK_DELAY_ACCT */
- 
-diff --git a/include/linux/dev_printk.h b/include/linux/dev_printk.h
-index 8904063d4c9f..eb2094e43050 100644
--- a/include/linux/dev_printk.h
-+++ b/include/linux/dev_printk.h
-@@ -130,6 +130,16 @@ void _dev_info(const struct device *dev, const char *fmt, ...)
- 	})
- 
- /*
-+ * Dummy dev_printk for disabled debugging statements to use whilst maintaining
-+ * gcc's format checking.
-+ */
-+#define dev_no_printk(level, dev, fmt, ...)				\
-+	({								\
-+		if (0)							\
-+			_dev_printk(level, dev, fmt, ##__VA_ARGS__);	\
-+	})
-+
-+/*
-  * #defines for all the dev_<level> macros to prefix with whatever
-  * possible use of #define dev_fmt(fmt) ...
-  */
-@@ -158,10 +168,7 @@ void _dev_info(const struct device *dev, const char *fmt, ...)
- 	dev_printk(KERN_DEBUG, dev, dev_fmt(fmt), ##__VA_ARGS__)
- #else
- #define dev_dbg(dev, fmt, ...)						\
-({									\
-	if (0)								\
-		dev_printk(KERN_DEBUG, dev, dev_fmt(fmt), ##__VA_ARGS__); \
-})
-+	dev_no_printk(KERN_DEBUG, dev, dev_fmt(fmt), ##__VA_ARGS__)
- #endif
- 
- #ifdef CONFIG_PRINTK
-@@ -247,20 +254,14 @@ do {									\
- } while (0)
- #else
- #define dev_dbg_ratelimited(dev, fmt, ...)				\
-do {									\
-	if (0)								\
-		dev_printk(KERN_DEBUG, dev, dev_fmt(fmt), ##__VA_ARGS__); \
-} while (0)
-+	dev_no_printk(KERN_DEBUG, dev, dev_fmt(fmt), ##__VA_ARGS__)
- #endif
- 
- #ifdef VERBOSE_DEBUG
- #define dev_vdbg	dev_dbg
- #else
- #define dev_vdbg(dev, fmt, ...)						\
-({									\
-	if (0)								\
-		dev_printk(KERN_DEBUG, dev, dev_fmt(fmt), ##__VA_ARGS__); \
-})
-+	dev_no_printk(KERN_DEBUG, dev, dev_fmt(fmt), ##__VA_ARGS__)
- #endif
- 
- /*
-@@ -274,4 +275,15 @@ do {									\
- 	WARN_ONCE(condition, "%s %s: " format, \
- 			dev_driver_string(dev), dev_name(dev), ## arg)
- 
-+__printf(3, 4) int dev_err_probe(const struct device *dev, int err, const char *fmt, ...);
-+__printf(3, 4) int dev_warn_probe(const struct device *dev, int err, const char *fmt, ...);
-+
-+/* Simple helper for dev_err_probe() when ERR_PTR() is to be returned. */
-+#define dev_err_ptr_probe(dev, ___err, fmt, ...) \
-+	ERR_PTR(dev_err_probe(dev, ___err, fmt, ##__VA_ARGS__))
-+
-+/* Simple helper for dev_err_probe() when ERR_CAST() is to be returned. */
-+#define dev_err_cast_probe(dev, ___err_ptr, fmt, ...) \
-+	ERR_PTR(dev_err_probe(dev, PTR_ERR(___err_ptr), fmt, ##__VA_ARGS__))
-+
- #endif /* _DEVICE_PRINTK_H_ */
-diff --git a/include/linux/devcoredump.h b/include/linux/devcoredump.h
-index c008169ed2c6..377892604ff4 100644
--- a/include/linux/devcoredump.h
-+++ b/include/linux/devcoredump.h
-@@ -12,6 +12,9 @@
- #include <linux/scatterlist.h>
- #include <linux/slab.h>
- 
-+/* if data isn't read by userspace after 5 minutes then delete it */
-+#define DEVCD_TIMEOUT	(HZ * 60 * 5)
-+
- /*
-  * _devcd_free_sgtable - free all the memory of the given scatterlist table
-  * (i.e. both pages and scatterlist instances)
-@@ -50,19 +53,22 @@ static inline void _devcd_free_sgtable(struct scatterlist *table)
- 	kfree(delete_iter);
- }
- 
-
- #ifdef CONFIG_DEV_COREDUMP
- void dev_coredumpv(struct device *dev, void *data, size_t datalen,
- 		   gfp_t gfp);
- 
-void dev_coredumpm(struct device *dev, struct module *owner,
-		   void *data, size_t datalen, gfp_t gfp,
-		   ssize_t (*read)(char *buffer, loff_t offset, size_t count,
-				   void *data, size_t datalen),
-		   void (*free)(void *data));
-+void dev_coredumpm_timeout(struct device *dev, struct module *owner,
-+			   void *data, size_t datalen, gfp_t gfp,
-+			   ssize_t (*read)(char *buffer, loff_t offset,
-+					   size_t count, void *data,
-+					   size_t datalen),
-+			   void (*free)(void *data),
-+			   unsigned long timeout);
- 
- void dev_coredumpsg(struct device *dev, struct scatterlist *table,
- 		    size_t datalen, gfp_t gfp);
-+
-+void dev_coredump_put(struct device *dev);
- #else
- static inline void dev_coredumpv(struct device *dev, void *data,
- 				 size_t datalen, gfp_t gfp)
-@@ -71,11 +77,13 @@ static inline void dev_coredumpv(struct device *dev, void *data,
- }
- 
- static inline void
-dev_coredumpm(struct device *dev, struct module *owner,
-	      void *data, size_t datalen, gfp_t gfp,
-	      ssize_t (*read)(char *buffer, loff_t offset, size_t count,
-			      void *data, size_t datalen),
-	      void (*free)(void *data))
-+dev_coredumpm_timeout(struct device *dev, struct module *owner,
-+		      void *data, size_t datalen, gfp_t gfp,
-+		      ssize_t (*read)(char *buffer, loff_t offset,
-+				      size_t count, void *data,
-+				      size_t datalen),
-+		      void (*free)(void *data),
-+		      unsigned long timeout)
- {
- 	free(data);
- }
-@@ -85,6 +93,34 @@ static inline void dev_coredumpsg(struct device *dev, struct scatterlist *table,
- {
- 	_devcd_free_sgtable(table);
- }
-+static inline void dev_coredump_put(struct device *dev)
-+{
-+}
- #endif /* CONFIG_DEV_COREDUMP */
- 
-+/**
-+ * dev_coredumpm - create device coredump with read/free methods
-+ * @dev: the struct device for the crashed device
-+ * @owner: the module that contains the read/free functions, use %THIS_MODULE
-+ * @data: data cookie for the @read/@free functions
-+ * @datalen: length of the data
-+ * @gfp: allocation flags
-+ * @read: function to read from the given buffer
-+ * @free: function to free the given buffer
-+ *
-+ * Creates a new device coredump for the given device. If a previous one hasn't
-+ * been read yet, the new coredump is discarded. The data lifetime is determined
-+ * by the device coredump framework and when it is no longer needed the @free
-+ * function will be called to free the data.
-+ */
-+static inline void dev_coredumpm(struct device *dev, struct module *owner,
-+				 void *data, size_t datalen, gfp_t gfp,
-+				 ssize_t (*read)(char *buffer, loff_t offset, size_t count,
-+						 void *data, size_t datalen),
-+				void (*free)(void *data))
-+{
-+	dev_coredumpm_timeout(dev, owner, data, datalen, gfp, read, free,
-+			      DEVCD_TIMEOUT);
-+}
-+
- #endif /* __DEVCOREDUMP_H */
-diff --git a/include/linux/devfreq-governor.h b/include/linux/devfreq-governor.h
-new file mode 100644
-index 000000000000..dfdd0160a29f
--- /dev/null
-+++ b/include/linux/devfreq-governor.h
-@@ -0,0 +1,102 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * governor.h - internal header for devfreq governors.
-+ *
-+ * Copyright (C) 2011 Samsung Electronics
-+ *	MyungJoo Ham <myungjoo.ham@samsung.com>
-+ *
-+ * This header is for devfreq governors
-+ */
-+
-+#ifndef __LINUX_DEVFREQ_DEVFREQ_H__
-+#define __LINUX_DEVFREQ_DEVFREQ_H__
-+
-+#include <linux/devfreq.h>
-+
-+#define DEVFREQ_NAME_LEN			16
-+
-+#define to_devfreq(DEV)	container_of((DEV), struct devfreq, dev)
-+
-+/* Devfreq events */
-+#define DEVFREQ_GOV_START			0x1
-+#define DEVFREQ_GOV_STOP			0x2
-+#define DEVFREQ_GOV_UPDATE_INTERVAL		0x3
-+#define DEVFREQ_GOV_SUSPEND			0x4
-+#define DEVFREQ_GOV_RESUME			0x5
-+
-+#define DEVFREQ_MIN_FREQ			0
-+#define DEVFREQ_MAX_FREQ			ULONG_MAX
-+
-+/*
-+ * Definition of the governor feature flags
-+ * - DEVFREQ_GOV_FLAG_IMMUTABLE
-+ *   : This governor is never changeable to other governors.
-+ * - DEVFREQ_GOV_FLAG_IRQ_DRIVEN
-+ *   : The devfreq won't schedule the work for this governor.
-+ */
-+#define DEVFREQ_GOV_FLAG_IMMUTABLE			BIT(0)
-+#define DEVFREQ_GOV_FLAG_IRQ_DRIVEN			BIT(1)
-+
-+/*
-+ * Definition of governor attribute flags except for common sysfs attributes
-+ * - DEVFREQ_GOV_ATTR_POLLING_INTERVAL
-+ *   : Indicate polling_interval sysfs attribute
-+ * - DEVFREQ_GOV_ATTR_TIMER
-+ *   : Indicate timer sysfs attribute
-+ */
-+#define DEVFREQ_GOV_ATTR_POLLING_INTERVAL		BIT(0)
-+#define DEVFREQ_GOV_ATTR_TIMER				BIT(1)
-+
-+/**
-+ * struct devfreq_governor - Devfreq policy governor
-+ * @node:		list node - contains registered devfreq governors
-+ * @name:		Governor's name
-+ * @attrs:		Governor's sysfs attribute flags
-+ * @flags:		Governor's feature flags
-+ * @get_target_freq:	Returns desired operating frequency for the device.
-+ *			Basically, get_target_freq will run
-+ *			devfreq_dev_profile.get_dev_status() to get the
-+ *			status of the device (load = busy_time / total_time).
-+ * @event_handler:      Callback for devfreq core framework to notify events
-+ *                      to governors. Events include per device governor
-+ *                      init and exit, opp changes out of devfreq, suspend
-+ *                      and resume of per device devfreq during device idle.
-+ *
-+ * Note that the callbacks are called with devfreq->lock locked by devfreq.
-+ */
-+struct devfreq_governor {
-+	struct list_head node;
-+
-+	const char name[DEVFREQ_NAME_LEN];
-+	const u64 attrs;
-+	const u64 flags;
-+	int (*get_target_freq)(struct devfreq *this, unsigned long *freq);
-+	int (*event_handler)(struct devfreq *devfreq,
-+				unsigned int event, void *data);
-+};
-+
-+void devfreq_monitor_start(struct devfreq *devfreq);
-+void devfreq_monitor_stop(struct devfreq *devfreq);
-+void devfreq_monitor_suspend(struct devfreq *devfreq);
-+void devfreq_monitor_resume(struct devfreq *devfreq);
-+void devfreq_update_interval(struct devfreq *devfreq, unsigned int *delay);
-+
-+int devfreq_add_governor(struct devfreq_governor *governor);
-+int devfreq_remove_governor(struct devfreq_governor *governor);
-+
-+int devm_devfreq_add_governor(struct device *dev,
-+			      struct devfreq_governor *governor);
-+
-+int devfreq_update_status(struct devfreq *devfreq, unsigned long freq);
-+int devfreq_update_target(struct devfreq *devfreq, unsigned long freq);
-+void devfreq_get_freq_range(struct devfreq *devfreq, unsigned long *min_freq,
-+			    unsigned long *max_freq);
-+
-+static inline int devfreq_update_stats(struct devfreq *df)
-+{
-+	if (!df->profile->get_dev_status)
-+		return -EINVAL;
-+
-+	return df->profile->get_dev_status(df->dev.parent, &df->last_status);
-+}
-+#endif /* __LINUX_DEVFREQ_DEVFREQ_H__ */
-diff --git a/include/linux/devfreq.h b/include/linux/devfreq.h
-index 34aab4dd336c..dc1075dc3446 100644
--- a/include/linux/devfreq.h
-+++ b/include/linux/devfreq.h
-@@ -103,12 +103,13 @@ struct devfreq_dev_status {
-  *
-  * @is_cooling_device: A self-explanatory boolean giving the device a
-  *                     cooling effect property.
-+ * @dev_groups:		Optional device-specific sysfs attribute groups that to
-+ *			be attached to the devfreq device.
-  */
- struct devfreq_dev_profile {
- 	unsigned long initial_freq;
- 	unsigned int polling_ms;
- 	enum devfreq_timer timer;
-	bool is_cooling_device;
- 
- 	int (*target)(struct device *dev, unsigned long *freq, u32 flags);
- 	int (*get_dev_status)(struct device *dev,
-@@ -118,6 +119,10 @@ struct devfreq_dev_profile {
- 
- 	unsigned long *freq_table;
- 	unsigned int max_state;
-+
-+	bool is_cooling_device;
-+
-+	const struct attribute_group **dev_groups;
- };
- 
- /**
-@@ -152,8 +157,8 @@ struct devfreq_stats {
-  * @max_state:		count of entry present in the frequency table.
-  * @previous_freq:	previously configured frequency value.
-  * @last_status:	devfreq user device info, performance statistics
- * @data:	Private data of the governor. The devfreq framework does not
- *		touch this.
-+ * @data:	devfreq driver pass to governors, governor should not change it.
-+ * @governor_data:	private data for governors, devfreq core doesn't touch it.
-  * @user_min_freq_req:	PM QoS minimum frequency request from user (via sysfs)
-  * @user_max_freq_req:	PM QoS maximum frequency request from user (via sysfs)
-  * @scaling_min_freq:	Limit minimum frequency requested by OPP interface
-@@ -193,7 +198,8 @@ struct devfreq {
- 	unsigned long previous_freq;
- 	struct devfreq_dev_status last_status;
- 
-	void *data; /* private data for governors */
-+	void *data;
-+	void *governor_data;
- 
- 	struct dev_pm_qos_request user_min_freq_req;
- 	struct dev_pm_qos_request user_max_freq_req;
-@@ -272,8 +278,8 @@ void devm_devfreq_unregister_notifier(struct device *dev,
- struct devfreq *devfreq_get_devfreq_by_node(struct device_node *node);
- struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev,
- 				const char *phandle_name, int index);
-+#endif /* CONFIG_PM_DEVFREQ */
- 
-#if IS_ENABLED(CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND)
- /**
-  * struct devfreq_simple_ondemand_data - ``void *data`` fed to struct devfreq
-  *	and devfreq_add_device
-@@ -291,9 +297,7 @@ struct devfreq_simple_ondemand_data {
- 	unsigned int upthreshold;
- 	unsigned int downdifferential;
- };
-#endif
- 
-#if IS_ENABLED(CONFIG_DEVFREQ_GOV_PASSIVE)
- enum devfreq_parent_dev_type {
- 	DEVFREQ_PARENT_DEV,
- 	CPUFREQ_PARENT_DEV,
-@@ -336,9 +340,8 @@ struct devfreq_passive_data {
- 	struct notifier_block nb;
- 	struct list_head cpu_data_list;
- };
-#endif
- 
-#else /* !CONFIG_PM_DEVFREQ */
-+#if !defined(CONFIG_PM_DEVFREQ)
- static inline struct devfreq *devfreq_add_device(struct device *dev,
- 					struct devfreq_dev_profile *profile,
- 					const char *governor_name,
-diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h
-index 04c6acf7faaa..38f625af6ab4 100644
--- a/include/linux/device-mapper.h
-+++ b/include/linux/device-mapper.h
-@@ -1,3 +1,4 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
- /*
-  * Copyright (C) 2001 Sistina Software (UK) Limited.
-  * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
-@@ -87,12 +88,19 @@ typedef int (*dm_preresume_fn) (struct dm_target *ti);
- typedef void (*dm_resume_fn) (struct dm_target *ti);
- 
- typedef void (*dm_status_fn) (struct dm_target *ti, status_type_t status_type,
-			      unsigned status_flags, char *result, unsigned maxlen);
-+			      unsigned int status_flags, char *result, unsigned int maxlen);
- 
-typedef int (*dm_message_fn) (struct dm_target *ti, unsigned argc, char **argv,
-			      char *result, unsigned maxlen);
-+typedef int (*dm_message_fn) (struct dm_target *ti, unsigned int argc, char **argv,
-+			      char *result, unsigned int maxlen);
- 
-typedef int (*dm_prepare_ioctl_fn) (struct dm_target *ti, struct block_device **bdev);
-+/*
-+ * Called with *forward == true. If it remains true, the ioctl should be
-+ * forwarded to bdev. If it is reset to false, the target already fully handled
-+ * the ioctl and the return value is the return value for the whole ioctl.
-+ */
-+typedef int (*dm_prepare_ioctl_fn) (struct dm_target *ti, struct block_device **bdev,
-+				    unsigned int cmd, unsigned long arg,
-+				    bool *forward);
- 
- #ifdef CONFIG_BLK_DEV_ZONED
- typedef int (*dm_report_zones_fn) (struct dm_target *ti,
-@@ -148,7 +156,7 @@ typedef int (*dm_busy_fn) (struct dm_target *ti);
-  */
- typedef long (*dm_dax_direct_access_fn) (struct dm_target *ti, pgoff_t pgoff,
- 		long nr_pages, enum dax_access_mode node, void **kaddr,
-		pfn_t *pfn);
-+		unsigned long *pfn);
- typedef int (*dm_dax_zero_page_range_fn)(struct dm_target *ti, pgoff_t pgoff,
- 		size_t nr_pages);
- 
-@@ -164,22 +172,26 @@ void dm_error(const char *message);
- 
- struct dm_dev {
- 	struct block_device *bdev;
-+	struct file *bdev_file;
- 	struct dax_device *dax_dev;
-	fmode_t mode;
-+	blk_mode_t mode;
- 	char name[16];
- };
- 
-dev_t dm_get_dev_t(const char *path);
-
- /*
-  * Constructors should call these functions to ensure destination devices
-  * are opened/closed correctly.
-  */
-int dm_get_device(struct dm_target *ti, const char *path, fmode_t mode,
-+int dm_get_device(struct dm_target *ti, const char *path, blk_mode_t mode,
- 		  struct dm_dev **result);
- void dm_put_device(struct dm_target *ti, struct dm_dev *d);
- 
- /*
-+ * Helper function for getting devices
-+ */
-+int dm_devt_from_path(const char *path, dev_t *dev_p);
-+
-+/*
-  * Information about a target type
-  */
- 
-@@ -187,7 +199,7 @@ struct target_type {
- 	uint64_t features;
- 	const char *name;
- 	struct module *module;
-	unsigned version[3];
-+	unsigned int version[3];
- 	dm_ctr_fn ctr;
- 	dm_dtr_fn dtr;
- 	dm_map_fn map;
-@@ -294,6 +306,9 @@ struct target_type {
- #define dm_target_supports_mixed_zoned_model(type) (false)
- #endif
- 
-+#define DM_TARGET_ATOMIC_WRITES		0x00000400
-+#define dm_target_supports_atomic_writes(type) ((type)->features & DM_TARGET_ATOMIC_WRITES)
-+
- struct dm_target {
- 	struct dm_table *table;
- 	struct target_type *type;
-@@ -313,31 +328,31 @@ struct dm_target {
- 	 * It is a responsibility of the target driver to remap these bios
- 	 * to the real underlying devices.
- 	 */
-	unsigned num_flush_bios;
-+	unsigned int num_flush_bios;
- 
- 	/*
- 	 * The number of discard bios that will be submitted to the target.
- 	 * The bio number can be accessed with dm_bio_get_target_bio_nr.
- 	 */
-	unsigned num_discard_bios;
-+	unsigned int num_discard_bios;
- 
- 	/*
- 	 * The number of secure erase bios that will be submitted to the target.
- 	 * The bio number can be accessed with dm_bio_get_target_bio_nr.
- 	 */
-	unsigned num_secure_erase_bios;
-+	unsigned int num_secure_erase_bios;
- 
- 	/*
- 	 * The number of WRITE ZEROES bios that will be submitted to the target.
- 	 * The bio number can be accessed with dm_bio_get_target_bio_nr.
- 	 */
-	unsigned num_write_zeroes_bios;
-+	unsigned int num_write_zeroes_bios;
- 
- 	/*
- 	 * The minimum number of extra bytes allocated in each io for the
- 	 * target to use.
- 	 */
-	unsigned per_io_data_size;
-+	unsigned int per_io_data_size;
- 
- 	/* target specific data */
- 	void *private;
-@@ -358,6 +373,19 @@ struct dm_target {
- 	bool discards_supported:1;
- 
- 	/*
-+	 * Automatically set by dm-core if this target supports
-+	 * REQ_OP_ZONE_RESET_ALL. Otherwise, this operation will be emulated
-+	 * using REQ_OP_ZONE_RESET. Target drivers must not set this manually.
-+	 */
-+	bool zone_reset_all_supported:1;
-+
-+	/*
-+	 * Set if this target requires that discards be split on
-+	 * 'max_discard_sectors' boundaries.
-+	 */
-+	bool max_discard_granularity:1;
-+
-+	/*
- 	 * Set if we need to limit the number of in-flight bios when swapping.
- 	 */
- 	bool limit_swap_bios:1;
-@@ -379,11 +407,32 @@ struct dm_target {
- 	 * bio_set_dev(). NOTE: ideally a target should _not_ need this.
- 	 */
- 	bool needs_bio_set_dev:1;
-+
-+	/*
-+	 * Set if the target supports flush optimization. If all the targets in
-+	 * a table have flush_bypasses_map set, the dm core will not send
-+	 * flushes to the targets via a ->map method. It will iterate over
-+	 * dm_table->devices and send flushes to the devices directly. This
-+	 * optimization reduces the number of flushes being sent when multiple
-+	 * targets in a table use the same underlying device.
-+	 *
-+	 * This optimization may be enabled on targets that just pass the
-+	 * flushes to the underlying devices without performing any other
-+	 * actions on the flush request. Currently, dm-linear and dm-stripe
-+	 * support it.
-+	 */
-+	bool flush_bypasses_map:1;
-+
-+	/*
-+	 * Set if the target calls bio_integrity_alloc on bios received
-+	 * in the map method.
-+	 */
-+	bool mempool_needs_integrity:1;
- };
- 
- void *dm_per_bio_data(struct bio *bio, size_t data_size);
- struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size);
-unsigned dm_bio_get_target_bio_nr(const struct bio *bio);
-+unsigned int dm_bio_get_target_bio_nr(const struct bio *bio);
- 
- u64 dm_start_time_ns_from_clone(struct bio *bio);
- 
-@@ -394,7 +443,7 @@ void dm_unregister_target(struct target_type *t);
-  * Target argument parsing.
-  */
- struct dm_arg_set {
-	unsigned argc;
-+	unsigned int argc;
- 	char **argv;
- };
- 
-@@ -403,8 +452,8 @@ struct dm_arg_set {
-  * the error message to use if the number is found to be outside that range.
-  */
- struct dm_arg {
-	unsigned min;
-	unsigned max;
-+	unsigned int min;
-+	unsigned int max;
- 	char *error;
- };
- 
-@@ -413,7 +462,7 @@ struct dm_arg {
-  * returning -EINVAL and setting *error.
-  */
- int dm_read_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
-		unsigned *value, char **error);
-+		unsigned int *value, char **error);
- 
- /*
-  * Process the next argument as the start of a group containing between
-@@ -421,7 +470,7 @@ int dm_read_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
-  * *num_args or, if invalid, return -EINVAL and set *error.
-  */
- int dm_read_arg_group(const struct dm_arg *arg, struct dm_arg_set *arg_set,
-		      unsigned *num_args, char **error);
-+		      unsigned int *num_args, char **error);
- 
- /*
-  * Return the current argument and shift to the next.
-@@ -431,12 +480,14 @@ const char *dm_shift_arg(struct dm_arg_set *as);
- /*
-  * Move through num_args arguments.
-  */
-void dm_consume_args(struct dm_arg_set *as, unsigned num_args);
-+void dm_consume_args(struct dm_arg_set *as, unsigned int num_args);
- 
-/*-----------------------------------------------------------------
-+/*
-+ *----------------------------------------------------------------
-  * Functions for creating and manipulating mapped devices.
-  * Drop the reference with dm_put when you finish with the object.
- *---------------------------------------------------------------*/
-+ *----------------------------------------------------------------
-+ */
- 
- /*
-  * DM_ANY_MINOR chooses the next available minor number.
-@@ -461,7 +512,7 @@ void *dm_get_mdptr(struct mapped_device *md);
- /*
-  * A device can still be used while suspended, but I/O is deferred.
-  */
-int dm_suspend(struct mapped_device *md, unsigned suspend_flags);
-+int dm_suspend(struct mapped_device *md, unsigned int suspend_flags);
- int dm_resume(struct mapped_device *md);
- 
- /*
-@@ -481,19 +532,24 @@ struct gendisk *dm_disk(struct mapped_device *md);
- int dm_suspended(struct dm_target *ti);
- int dm_post_suspending(struct dm_target *ti);
- int dm_noflush_suspending(struct dm_target *ti);
-void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors);
-+void dm_accept_partial_bio(struct bio *bio, unsigned int n_sectors);
- void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone);
-union map_info *dm_get_rq_mapinfo(struct request *rq);
- 
- #ifdef CONFIG_BLK_DEV_ZONED
- struct dm_report_zones_args {
- 	struct dm_target *tgt;
-+	struct gendisk *disk;
- 	sector_t next_sector;
- 
-	void *orig_data;
-	report_zones_cb orig_cb;
- 	unsigned int zone_idx;
- 
-+	/* for block layer ->report_zones */
-+	struct blk_report_zones_args *rep_args;
-+
-+	/* for internal users */
-+	report_zones_cb cb;
-+	void *data;
-+
- 	/* must be filled by ->report_zones before calling dm_report_zones_cb */
- 	sector_t start;
- };
-@@ -509,23 +565,23 @@ int __init dm_early_create(struct dm_ioctl *dmi,
- 			   struct dm_target_spec **spec_array,
- 			   char **target_params_array);
- 
-struct queue_limits *dm_get_queue_limits(struct mapped_device *md);
-
- /*
-  * Geometry functions.
-  */
- int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo);
- int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo);
- 
-/*-----------------------------------------------------------------
-+/*
-+ *---------------------------------------------------------------
-  * Functions for manipulating device-mapper tables.
- *---------------------------------------------------------------*/
-+ *---------------------------------------------------------------
-+ */
- 
- /*
-  * First create an empty table.
-  */
-int dm_table_create(struct dm_table **result, fmode_t mode,
-		    unsigned num_targets, struct mapped_device *md);
-+int dm_table_create(struct dm_table **result, blk_mode_t mode,
-+		    unsigned int num_targets, struct mapped_device *md);
- 
- /*
-  * Then call this once for each target.
-@@ -567,7 +623,7 @@ void dm_sync_table(struct mapped_device *md);
-  * Queries
-  */
- sector_t dm_table_get_size(struct dm_table *t);
-fmode_t dm_table_get_mode(struct dm_table *t);
-+blk_mode_t dm_table_get_mode(struct dm_table *t);
- struct mapped_device *dm_table_get_md(struct dm_table *t);
- const char *dm_table_device_name(struct dm_table *t);
- 
-@@ -593,9 +649,11 @@ struct dm_table *dm_swap_table(struct mapped_device *md,
-  */
- void dm_destroy_crypto_profile(struct blk_crypto_profile *profile);
- 
-/*-----------------------------------------------------------------
-+/*
-+ *---------------------------------------------------------------
-  * Macros.
- *---------------------------------------------------------------*/
-+ *---------------------------------------------------------------
-+ */
- #define DM_NAME "device-mapper"
- 
- #define DM_FMT(fmt) DM_NAME ": " DM_MSG_PREFIX ": " fmt "\n"
-@@ -612,13 +670,32 @@ void dm_destroy_crypto_profile(struct blk_crypto_profile *profile);
- #define DMDEBUG(fmt, ...) pr_debug(DM_FMT(fmt), ##__VA_ARGS__)
- #define DMDEBUG_LIMIT(fmt, ...) pr_debug_ratelimited(DM_FMT(fmt), ##__VA_ARGS__)
- 
-#define DMEMIT(x...) sz += ((sz >= maxlen) ? \
-			  0 : scnprintf(result + sz, maxlen - sz, x))
-+#define DMEMIT(x...) (sz += ((sz >= maxlen) ? 0 : scnprintf(result + sz, maxlen - sz, x)))
- 
- #define DMEMIT_TARGET_NAME_VERSION(y) \
- 		DMEMIT("target_name=%s,target_version=%u.%u.%u", \
- 		       (y)->name, (y)->version[0], (y)->version[1], (y)->version[2])
- 
-+/**
-+ * module_dm() - Helper macro for DM targets that don't do anything
-+ * special in their module_init and module_exit.
-+ * Each module may only use this macro once, and calling it replaces
-+ * module_init() and module_exit().
-+ *
-+ * @name: DM target's name
-+ */
-+#define module_dm(name) \
-+static int __init dm_##name##_init(void) \
-+{ \
-+	return dm_register_target(&(name##_target)); \
-+} \
-+module_init(dm_##name##_init) \
-+static void __exit dm_##name##_exit(void) \
-+{ \
-+	dm_unregister_target(&(name##_target)); \
-+} \
-+module_exit(dm_##name##_exit)
-+
- /*
-  * Definitions of return values from target end_io function.
-  */
-diff --git a/include/linux/device.h b/include/linux/device.h
-index 424b55df0272..0be95294b6e6 100644
--- a/include/linux/device.h
-+++ b/include/linux/device.h
-@@ -26,10 +26,11 @@
- #include <linux/atomic.h>
- #include <linux/uidgid.h>
- #include <linux/gfp.h>
-#include <linux/overflow.h>
- #include <linux/device/bus.h>
- #include <linux/device/class.h>
-+#include <linux/device/devres.h>
- #include <linux/device/driver.h>
-+#include <linux/cleanup.h>
- #include <asm/device.h>
- 
- struct device;
-@@ -41,7 +42,6 @@ struct class;
- struct subsys_private;
- struct device_node;
- struct fwnode_handle;
-struct iommu_ops;
- struct iommu_group;
- struct dev_pin_info;
- struct dev_iommu;
-@@ -62,7 +62,7 @@ struct msi_device_data;
-  */
- struct subsys_interface {
- 	const char *name;
-	struct bus_type *subsys;
-+	const struct bus_type *subsys;
- 	struct list_head node;
- 	int (*add_dev)(struct device *dev, struct subsys_interface *sif);
- 	void (*remove_dev)(struct device *dev, struct subsys_interface *sif);
-@@ -71,9 +71,9 @@ struct subsys_interface {
- int subsys_interface_register(struct subsys_interface *sif);
- void subsys_interface_unregister(struct subsys_interface *sif);
- 
-int subsys_system_register(struct bus_type *subsys,
-+int subsys_system_register(const struct bus_type *subsys,
- 			   const struct attribute_group **groups);
-int subsys_virtual_register(struct bus_type *subsys,
-+int subsys_virtual_register(const struct bus_type *subsys,
- 			    const struct attribute_group **groups);
- 
- /*
-@@ -88,15 +88,20 @@ int subsys_virtual_register(struct bus_type *subsys,
- struct device_type {
- 	const char *name;
- 	const struct attribute_group **groups;
-	int (*uevent)(struct device *dev, struct kobj_uevent_env *env);
-	char *(*devnode)(struct device *dev, umode_t *mode,
-+	int (*uevent)(const struct device *dev, struct kobj_uevent_env *env);
-+	char *(*devnode)(const struct device *dev, umode_t *mode,
- 			 kuid_t *uid, kgid_t *gid);
- 	void (*release)(struct device *dev);
- 
- 	const struct dev_pm_ops *pm;
- };
- 
-/* interface for exporting device attributes */
-+/**
-+ * struct device_attribute - Interface for exporting device attributes.
-+ * @attr: sysfs attribute definition.
-+ * @show: Show handler.
-+ * @store: Store handler.
-+ */
- struct device_attribute {
- 	struct attribute	attr;
- 	ssize_t (*show)(struct device *dev, struct device_attribute *attr,
-@@ -105,6 +110,11 @@ struct device_attribute {
- 			 const char *buf, size_t count);
- };
- 
-+/**
-+ * struct dev_ext_attribute - Exported device attribute with extra context.
-+ * @attr: Exported device attribute.
-+ * @var: Pointer to context.
-+ */
- struct dev_ext_attribute {
- 	struct device_attribute attr;
- 	void *var;
-@@ -122,31 +132,140 @@ ssize_t device_show_bool(struct device *dev, struct device_attribute *attr,
- 			char *buf);
- ssize_t device_store_bool(struct device *dev, struct device_attribute *attr,
- 			 const char *buf, size_t count);
-+ssize_t device_show_string(struct device *dev, struct device_attribute *attr,
-+			   char *buf);
- 
-+/**
-+ * DEVICE_ATTR - Define a device attribute.
-+ * @_name: Attribute name.
-+ * @_mode: File mode.
-+ * @_show: Show handler. Optional, but mandatory if attribute is readable.
-+ * @_store: Store handler. Optional, but mandatory if attribute is writable.
-+ *
-+ * Convenience macro for defining a struct device_attribute.
-+ *
-+ * For example, ``DEVICE_ATTR(foo, 0644, foo_show, foo_store);`` expands to:
-+ *
-+ * .. code-block:: c
-+ *
-+ *	struct device_attribute dev_attr_foo = {
-+ *		.attr	= { .name = "foo", .mode = 0644 },
-+ *		.show	= foo_show,
-+ *		.store	= foo_store,
-+ *	};
-+ */
- #define DEVICE_ATTR(_name, _mode, _show, _store) \
- 	struct device_attribute dev_attr_##_name = __ATTR(_name, _mode, _show, _store)
-+
-+/**
-+ * DEVICE_ATTR_PREALLOC - Define a preallocated device attribute.
-+ * @_name: Attribute name.
-+ * @_mode: File mode.
-+ * @_show: Show handler. Optional, but mandatory if attribute is readable.
-+ * @_store: Store handler. Optional, but mandatory if attribute is writable.
-+ *
-+ * Like DEVICE_ATTR(), but ``SYSFS_PREALLOC`` is set on @_mode.
-+ */
- #define DEVICE_ATTR_PREALLOC(_name, _mode, _show, _store) \
- 	struct device_attribute dev_attr_##_name = \
- 		__ATTR_PREALLOC(_name, _mode, _show, _store)
-+
-+/**
-+ * DEVICE_ATTR_RW - Define a read-write device attribute.
-+ * @_name: Attribute name.
-+ *
-+ * Like DEVICE_ATTR(), but @_mode is 0644, @_show is <_name>_show,
-+ * and @_store is <_name>_store.
-+ */
- #define DEVICE_ATTR_RW(_name) \
- 	struct device_attribute dev_attr_##_name = __ATTR_RW(_name)
-+
-+/**
-+ * DEVICE_ATTR_ADMIN_RW - Define an admin-only read-write device attribute.
-+ * @_name: Attribute name.
-+ *
-+ * Like DEVICE_ATTR_RW(), but @_mode is 0600.
-+ */
- #define DEVICE_ATTR_ADMIN_RW(_name) \
- 	struct device_attribute dev_attr_##_name = __ATTR_RW_MODE(_name, 0600)
-+
-+/**
-+ * DEVICE_ATTR_RO - Define a readable device attribute.
-+ * @_name: Attribute name.
-+ *
-+ * Like DEVICE_ATTR(), but @_mode is 0444 and @_show is <_name>_show.
-+ */
- #define DEVICE_ATTR_RO(_name) \
- 	struct device_attribute dev_attr_##_name = __ATTR_RO(_name)
-+
-+/**
-+ * DEVICE_ATTR_ADMIN_RO - Define an admin-only readable device attribute.
-+ * @_name: Attribute name.
-+ *
-+ * Like DEVICE_ATTR_RO(), but @_mode is 0400.
-+ */
- #define DEVICE_ATTR_ADMIN_RO(_name) \
- 	struct device_attribute dev_attr_##_name = __ATTR_RO_MODE(_name, 0400)
-+
-+/**
-+ * DEVICE_ATTR_WO - Define an admin-only writable device attribute.
-+ * @_name: Attribute name.
-+ *
-+ * Like DEVICE_ATTR(), but @_mode is 0200 and @_store is <_name>_store.
-+ */
- #define DEVICE_ATTR_WO(_name) \
- 	struct device_attribute dev_attr_##_name = __ATTR_WO(_name)
-+
-+/**
-+ * DEVICE_ULONG_ATTR - Define a device attribute backed by an unsigned long.
-+ * @_name: Attribute name.
-+ * @_mode: File mode.
-+ * @_var: Identifier of unsigned long.
-+ *
-+ * Like DEVICE_ATTR(), but @_show and @_store are automatically provided
-+ * such that reads and writes to the attribute from userspace affect @_var.
-+ */
- #define DEVICE_ULONG_ATTR(_name, _mode, _var) \
- 	struct dev_ext_attribute dev_attr_##_name = \
- 		{ __ATTR(_name, _mode, device_show_ulong, device_store_ulong), &(_var) }
-+
-+/**
-+ * DEVICE_INT_ATTR - Define a device attribute backed by an int.
-+ * @_name: Attribute name.
-+ * @_mode: File mode.
-+ * @_var: Identifier of int.
-+ *
-+ * Like DEVICE_ULONG_ATTR(), but @_var is an int.
-+ */
- #define DEVICE_INT_ATTR(_name, _mode, _var) \
- 	struct dev_ext_attribute dev_attr_##_name = \
- 		{ __ATTR(_name, _mode, device_show_int, device_store_int), &(_var) }
-+
-+/**
-+ * DEVICE_BOOL_ATTR - Define a device attribute backed by a bool.
-+ * @_name: Attribute name.
-+ * @_mode: File mode.
-+ * @_var: Identifier of bool.
-+ *
-+ * Like DEVICE_ULONG_ATTR(), but @_var is a bool.
-+ */
- #define DEVICE_BOOL_ATTR(_name, _mode, _var) \
- 	struct dev_ext_attribute dev_attr_##_name = \
- 		{ __ATTR(_name, _mode, device_show_bool, device_store_bool), &(_var) }
-+
-+/**
-+ * DEVICE_STRING_ATTR_RO - Define a device attribute backed by a r/o string.
-+ * @_name: Attribute name.
-+ * @_mode: File mode.
-+ * @_var: Identifier of string.
-+ *
-+ * Like DEVICE_ULONG_ATTR(), but @_var is a string. Because the length of the
-+ * string allocation is unknown, the attribute must be read-only.
-+ */
-+#define DEVICE_STRING_ATTR_RO(_name, _mode, _var) \
-+	struct dev_ext_attribute dev_attr_##_name = \
-+		{ __ATTR(_name, (_mode) & ~0222, device_show_string, NULL), (_var) }
-+
- #define DEVICE_ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) \
- 	struct device_attribute dev_attr_##_name =		\
- 		__ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store)
-@@ -162,121 +281,6 @@ int __must_check device_create_bin_file(struct device *dev,
- void device_remove_bin_file(struct device *dev,
- 			    const struct bin_attribute *attr);
- 
-/* device resource management */
-typedef void (*dr_release_t)(struct device *dev, void *res);
-typedef int (*dr_match_t)(struct device *dev, void *res, void *match_data);
-
-void *__devres_alloc_node(dr_release_t release, size_t size, gfp_t gfp,
-			  int nid, const char *name) __malloc;
-#define devres_alloc(release, size, gfp) \
-	__devres_alloc_node(release, size, gfp, NUMA_NO_NODE, #release)
-#define devres_alloc_node(release, size, gfp, nid) \
-	__devres_alloc_node(release, size, gfp, nid, #release)
-
-void devres_for_each_res(struct device *dev, dr_release_t release,
-			 dr_match_t match, void *match_data,
-			 void (*fn)(struct device *, void *, void *),
-			 void *data);
-void devres_free(void *res);
-void devres_add(struct device *dev, void *res);
-void *devres_find(struct device *dev, dr_release_t release,
-		  dr_match_t match, void *match_data);
-void *devres_get(struct device *dev, void *new_res,
-		 dr_match_t match, void *match_data);
-void *devres_remove(struct device *dev, dr_release_t release,
-		    dr_match_t match, void *match_data);
-int devres_destroy(struct device *dev, dr_release_t release,
-		   dr_match_t match, void *match_data);
-int devres_release(struct device *dev, dr_release_t release,
-		   dr_match_t match, void *match_data);
-
-/* devres group */
-void * __must_check devres_open_group(struct device *dev, void *id, gfp_t gfp);
-void devres_close_group(struct device *dev, void *id);
-void devres_remove_group(struct device *dev, void *id);
-int devres_release_group(struct device *dev, void *id);
-
-/* managed devm_k.alloc/kfree for device drivers */
-void *devm_kmalloc(struct device *dev, size_t size, gfp_t gfp) __malloc;
-void *devm_krealloc(struct device *dev, void *ptr, size_t size,
-		    gfp_t gfp) __must_check;
-__printf(3, 0) char *devm_kvasprintf(struct device *dev, gfp_t gfp,
-				     const char *fmt, va_list ap) __malloc;
-__printf(3, 4) char *devm_kasprintf(struct device *dev, gfp_t gfp,
-				    const char *fmt, ...) __malloc;
-static inline void *devm_kzalloc(struct device *dev, size_t size, gfp_t gfp)
-{
-	return devm_kmalloc(dev, size, gfp | __GFP_ZERO);
-}
-static inline void *devm_kmalloc_array(struct device *dev,
-				       size_t n, size_t size, gfp_t flags)
-{
-	size_t bytes;
-
-	if (unlikely(check_mul_overflow(n, size, &bytes)))
-		return NULL;
-
-	return devm_kmalloc(dev, bytes, flags);
-}
-static inline void *devm_kcalloc(struct device *dev,
-				 size_t n, size_t size, gfp_t flags)
-{
-	return devm_kmalloc_array(dev, n, size, flags | __GFP_ZERO);
-}
-void devm_kfree(struct device *dev, const void *p);
-char *devm_kstrdup(struct device *dev, const char *s, gfp_t gfp) __malloc;
-const char *devm_kstrdup_const(struct device *dev, const char *s, gfp_t gfp);
-void *devm_kmemdup(struct device *dev, const void *src, size_t len, gfp_t gfp);
-
-unsigned long devm_get_free_pages(struct device *dev,
-				  gfp_t gfp_mask, unsigned int order);
-void devm_free_pages(struct device *dev, unsigned long addr);
-
-void __iomem *devm_ioremap_resource(struct device *dev,
-				    const struct resource *res);
-void __iomem *devm_ioremap_resource_wc(struct device *dev,
-				       const struct resource *res);
-
-void __iomem *devm_of_iomap(struct device *dev,
-			    struct device_node *node, int index,
-			    resource_size_t *size);
-
-/* allows to add/remove a custom action to devres stack */
-int devm_add_action(struct device *dev, void (*action)(void *), void *data);
-void devm_remove_action(struct device *dev, void (*action)(void *), void *data);
-void devm_release_action(struct device *dev, void (*action)(void *), void *data);
-
-static inline int devm_add_action_or_reset(struct device *dev,
-					   void (*action)(void *), void *data)
-{
-	int ret;
-
-	ret = devm_add_action(dev, action, data);
-	if (ret)
-		action(data);
-
-	return ret;
-}
-
-/**
- * devm_alloc_percpu - Resource-managed alloc_percpu
- * @dev: Device to allocate per-cpu memory for
- * @type: Type to allocate per-cpu memory for
- *
- * Managed alloc_percpu. Per-cpu memory allocated with this function is
- * automatically freed on driver detach.
- *
- * RETURNS:
- * Pointer to allocated memory on success, NULL on failure.
- */
-#define devm_alloc_percpu(dev, type)      \
-	((typeof(type) __percpu *)__devm_alloc_percpu((dev), sizeof(type), \
-						      __alignof__(type)))
-
-void __percpu *__devm_alloc_percpu(struct device *dev, size_t size,
-				   size_t align);
-void devm_free_percpu(struct device *dev, void __percpu *pdata);
-
- struct device_dma_parameters {
- 	/*
- 	 * a low level driver may set these to teach IOMMU code about
-@@ -327,6 +331,7 @@ enum device_link_state {
- #define DL_FLAG_MANAGED			BIT(6)
- #define DL_FLAG_SYNC_STATE_ONLY		BIT(7)
- #define DL_FLAG_INFERRED		BIT(8)
-+#define DL_FLAG_CYCLE			BIT(9)
- 
- /**
-  * enum dl_dev_state - Device driver presence tracking information.
-@@ -378,10 +383,8 @@ struct dev_links_info {
-  * @data:	Pointer to MSI device data
-  */
- struct dev_msi_info {
-#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
-	struct irq_domain	*domain;
-#endif
- #ifdef CONFIG_GENERIC_MSI_IRQ
-+	struct irq_domain	*domain;
- 	struct msi_device_data	*data;
- #endif
- };
-@@ -504,7 +507,10 @@ struct device_physical_location {
-  * @dma_pools:	Dma pools (if dma'ble device).
-  * @dma_mem:	Internal for coherent mem override.
-  * @cma_area:	Contiguous memory area for dma allocations
- * @dma_io_tlb_mem: Pointer to the swiotlb pool used.  Not for driver use.
-+ * @dma_io_tlb_mem: Software IO TLB allocator.  Not for driver use.
-+ * @dma_io_tlb_pools:	List of transient swiotlb memory pools.
-+ * @dma_io_tlb_lock:	Protects changes to the list of active pools.
-+ * @dma_uses_io_tlb: %true if device has used the software IO TLB.
-  * @archdata:	For arch-specific additions.
-  * @of_node:	Associated device tree node.
-  * @fwnode:	Associated device node supplied by platform firmware.
-@@ -512,7 +518,6 @@ struct device_physical_location {
-  * @id:		device instance
-  * @devres_lock: Spinlock to protect the resource of the device.
-  * @devres_head: The resources list of the device.
- * @knode_class: The node used to add the device to the class list.
-  * @class:	The class of the device.
-  * @groups:	Optional attribute groups.
-  * @release:	Callback to free the device after all references have
-@@ -543,6 +548,9 @@ struct device_physical_location {
-  *		and optionall (if the coherent mask is large enough) also
-  *		for dma allocations.  This flag is managed by the dma ops
-  *		instance from ->dma_supported.
-+ * @dma_skip_sync: DMA sync operations can be skipped for coherent buffers.
-+ * @dma_iommu: Device is using default IOMMU implementation for DMA and
-+ *		doesn't rely on dma_ops structure.
-  *
-  * At the lowest level, every device in a Linux system is represented by an
-  * instance of struct device. The device structure contains the information
-@@ -561,7 +569,7 @@ struct device {
- 	const char		*init_name; /* initial name of the device */
- 	const struct device_type *type;
- 
-	struct bus_type	*bus;		/* type of bus device is on */
-+	const struct bus_type	*bus;	/* type of bus device is on */
- 	struct device_driver *driver;	/* which driver has allocated this
- 					   device */
- 	void		*platform_data;	/* Platform specific data, device
-@@ -584,7 +592,7 @@ struct device {
- 	struct dev_pin_info	*pins;
- #endif
- 	struct dev_msi_info	msi;
-#ifdef CONFIG_DMA_OPS
-+#ifdef CONFIG_ARCH_HAS_DMA_OPS
- 	const struct dma_map_ops *dma_ops;
- #endif
- 	u64		*dma_mask;	/* dma mask (if dma'able device) */
-@@ -611,6 +619,11 @@ struct device {
- #ifdef CONFIG_SWIOTLB
- 	struct io_tlb_mem *dma_io_tlb_mem;
- #endif
-+#ifdef CONFIG_SWIOTLB_DYNAMIC
-+	struct list_head dma_io_tlb_pools;
-+	spinlock_t dma_io_tlb_lock;
-+	bool dma_uses_io_tlb;
-+#endif
- 	/* arch specific additions */
- 	struct dev_archdata	archdata;
- 
-@@ -626,7 +639,7 @@ struct device {
- 	spinlock_t		devres_lock;
- 	struct list_head	devres_head;
- 
-	struct class		*class;
-+	const struct class	*class;
- 	const struct attribute_group **groups;	/* optional groups */
- 
- 	void	(*release)(struct device *dev);
-@@ -650,6 +663,12 @@ struct device {
- #ifdef CONFIG_DMA_OPS_BYPASS
- 	bool			dma_ops_bypass : 1;
- #endif
-+#ifdef CONFIG_DMA_NEED_SYNC
-+	bool			dma_skip_sync:1;
-+#endif
-+#ifdef CONFIG_IOMMU_DMA
-+	bool			dma_iommu:1;
-+#endif
- };
- 
- /**
-@@ -680,10 +699,7 @@ struct device_link {
- 	bool supplier_preactivated; /* Owned by consumer probe. */
- };
- 
-static inline struct device *kobj_to_dev(struct kobject *kobj)
-{
-	return container_of(kobj, struct device, kobj);
-}
-+#define kobj_to_dev(__kobj)	container_of_const(__kobj, struct device, kobj)
- 
- /**
-  * device_iommu_mapped - Returns true when the device DMA is translated
-@@ -698,6 +714,11 @@ static inline bool device_iommu_mapped(struct device *dev)
- /* Get the wakeup routines, which depend on struct device */
- #include <linux/pm_wakeup.h>
- 
-+/**
-+ * dev_name - Return a device's name.
-+ * @dev: Device with name to get.
-+ * Return: The kobject name of the device, or its initial name if unavailable.
-+ */
- static inline const char *dev_name(const struct device *dev)
- {
- 	/* Use the init name until the kobject becomes available */
-@@ -742,7 +763,7 @@ static inline void set_dev_node(struct device *dev, int node)
- 
- static inline struct irq_domain *dev_get_msi_domain(const struct device *dev)
- {
-#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
-+#ifdef CONFIG_GENERIC_MSI_IRQ
- 	return dev->msi.domain;
- #else
- 	return NULL;
-@@ -751,7 +772,7 @@ static inline struct irq_domain *dev_get_msi_domain(const struct device *dev)
- 
- static inline void dev_set_msi_domain(struct device *dev, struct irq_domain *d)
- {
-#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
-+#ifdef CONFIG_GENERIC_MSI_IRQ
- 	dev->msi.domain = d;
- #endif
- }
-@@ -811,6 +832,9 @@ static inline bool device_pm_not_required(struct device *dev)
- static inline void device_set_pm_not_required(struct device *dev)
- {
- 	dev->power.no_pm = true;
-+#ifdef CONFIG_PM
-+	dev->power.no_callbacks = true;
-+#endif
- }
- 
- static inline void dev_pm_syscore_device(struct device *dev, bool val)
-@@ -830,6 +854,42 @@ static inline bool dev_pm_test_driver_flags(struct device *dev, u32 flags)
- 	return !!(dev->power.driver_flags & flags);
- }
- 
-+static inline bool dev_pm_smart_suspend(struct device *dev)
-+{
-+#ifdef CONFIG_PM_SLEEP
-+	return dev->power.smart_suspend;
-+#else
-+	return false;
-+#endif
-+}
-+
-+/*
-+ * dev_pm_set_strict_midlayer - Update the device's power.strict_midlayer flag
-+ * @dev: Target device.
-+ * @val: New flag value.
-+ *
-+ * When set, power.strict_midlayer means that the middle layer power management
-+ * code (typically, a bus type or a PM domain) does not expect its runtime PM
-+ * suspend callback to be invoked at all during system-wide PM transitions and
-+ * it does not expect its runtime PM resume callback to be invoked at any point
-+ * when runtime PM is disabled for the device during system-wide PM transitions.
-+ */
-+static inline void dev_pm_set_strict_midlayer(struct device *dev, bool val)
-+{
-+#ifdef CONFIG_PM_SLEEP
-+	dev->power.strict_midlayer = val;
-+#endif
-+}
-+
-+static inline bool dev_pm_strict_midlayer_is_set(struct device *dev)
-+{
-+#ifdef CONFIG_PM_SLEEP
-+	return dev->power.strict_midlayer;
-+#else
-+	return false;
-+#endif
-+}
-+
- static inline void device_lock(struct device *dev)
- {
- 	mutex_lock(&dev->mutex);
-@@ -850,18 +910,13 @@ static inline void device_unlock(struct device *dev)
- 	mutex_unlock(&dev->mutex);
- }
- 
-+DEFINE_GUARD(device, struct device *, device_lock(_T), device_unlock(_T))
-+
- static inline void device_lock_assert(struct device *dev)
- {
- 	lockdep_assert_held(&dev->mutex);
- }
- 
-static inline struct device_node *dev_of_node(struct device *dev)
-{
-	if (!IS_ENABLED(CONFIG_OF) || !dev)
-		return NULL;
-	return dev->of_node;
-}
-
- static inline bool dev_has_sync_state(struct device *dev)
- {
- 	if (!dev)
-@@ -873,6 +928,18 @@ static inline bool dev_has_sync_state(struct device *dev)
- 	return false;
- }
- 
-+static inline int dev_set_drv_sync_state(struct device *dev,
-+					 void (*fn)(struct device *dev))
-+{
-+	if (!dev || !dev->driver)
-+		return 0;
-+	if (dev->driver->sync_state && dev->driver->sync_state != fn)
-+		return -EBUSY;
-+	if (!dev->driver->sync_state)
-+		dev->driver->sync_state = fn;
-+	return 0;
-+}
-+
- static inline void dev_set_removable(struct device *dev,
- 				     enum device_removable removable)
- {
-@@ -897,23 +964,52 @@ void device_unregister(struct device *dev);
- void device_initialize(struct device *dev);
- int __must_check device_add(struct device *dev);
- void device_del(struct device *dev);
-int device_for_each_child(struct device *dev, void *data,
-			  int (*fn)(struct device *dev, void *data));
-int device_for_each_child_reverse(struct device *dev, void *data,
-				  int (*fn)(struct device *dev, void *data));
-struct device *device_find_child(struct device *dev, void *data,
-				 int (*match)(struct device *dev, void *data));
-struct device *device_find_child_by_name(struct device *parent,
-					 const char *name);
-struct device *device_find_any_child(struct device *parent);
-+
-+DEFINE_FREE(device_del, struct device *, if (_T) device_del(_T))
-+
-+int device_for_each_child(struct device *parent, void *data,
-+			  device_iter_t fn);
-+int device_for_each_child_reverse(struct device *parent, void *data,
-+				  device_iter_t fn);
-+int device_for_each_child_reverse_from(struct device *parent,
-+				       struct device *from, void *data,
-+				       device_iter_t fn);
-+struct device *device_find_child(struct device *parent, const void *data,
-+				 device_match_t match);
-+/**
-+ * device_find_child_by_name - device iterator for locating a child device.
-+ * @parent: parent struct device
-+ * @name: name of the child device
-+ *
-+ * This is similar to the device_find_child() function above, but it
-+ * returns a reference to a device that has the name @name.
-+ *
-+ * NOTE: you will need to drop the reference with put_device() after use.
-+ */
-+static inline struct device *device_find_child_by_name(struct device *parent,
-+						       const char *name)
-+{
-+	return device_find_child(parent, name, device_match_name);
-+}
-+
-+/**
-+ * device_find_any_child - device iterator for locating a child device, if any.
-+ * @parent: parent struct device
-+ *
-+ * This is similar to the device_find_child() function above, but it
-+ * returns a reference to a child device, if any.
-+ *
-+ * NOTE: you will need to drop the reference with put_device() after use.
-+ */
-+static inline struct device *device_find_any_child(struct device *parent)
-+{
-+	return device_find_child(parent, NULL, device_match_any);
-+}
- 
- int device_rename(struct device *dev, const char *new_name);
- int device_move(struct device *dev, struct device *new_parent,
- 		enum dpm_order dpm_order);
- int device_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid);
-const char *device_get_devnode(struct device *dev, umode_t *mode, kuid_t *uid,
-			       kgid_t *gid, const char **tmp);
-int device_is_dependent(struct device *dev, void *target);
- 
- static inline bool device_supports_offline(struct device *dev)
- {
-@@ -968,10 +1064,21 @@ void unlock_device_hotplug(void);
- int lock_device_hotplug_sysfs(void);
- int device_offline(struct device *dev);
- int device_online(struct device *dev);
-+
- void set_primary_fwnode(struct device *dev, struct fwnode_handle *fwnode);
- void set_secondary_fwnode(struct device *dev, struct fwnode_handle *fwnode);
-void device_set_of_node_from_dev(struct device *dev, const struct device *dev2);
- void device_set_node(struct device *dev, struct fwnode_handle *fwnode);
-+int device_add_of_node(struct device *dev, struct device_node *of_node);
-+void device_remove_of_node(struct device *dev);
-+void device_set_of_node_from_dev(struct device *dev, const struct device *dev2);
-+struct device *get_dev_from_fwnode(struct fwnode_handle *fwnode);
-+
-+static inline struct device_node *dev_of_node(struct device *dev)
-+{
-+	if (!IS_ENABLED(CONFIG_OF) || !dev)
-+		return NULL;
-+	return dev->of_node;
-+}
- 
- static inline int dev_num_vf(struct device *dev)
- {
-@@ -1000,12 +1107,12 @@ static inline void *dev_get_platdata(const struct device *dev)
-  * Manual binding of a device to driver. See drivers/base/bus.c
-  * for information on use.
-  */
-int __must_check device_driver_attach(struct device_driver *drv,
-+int __must_check device_driver_attach(const struct device_driver *drv,
- 				      struct device *dev);
- int __must_check device_bind_driver(struct device *dev);
- void device_release_driver(struct device *dev);
- int  __must_check device_attach(struct device *dev);
-int __must_check driver_attach(struct device_driver *drv);
-+int __must_check driver_attach(const struct device_driver *drv);
- void device_initial_probe(struct device *dev);
- int __must_check device_reprobe(struct device *dev);
- 
-@@ -1015,13 +1122,13 @@ bool device_is_bound(struct device *dev);
-  * Easy functions for dynamically creating devices on the fly
-  */
- __printf(5, 6) struct device *
-device_create(struct class *cls, struct device *parent, dev_t devt,
-+device_create(const struct class *cls, struct device *parent, dev_t devt,
- 	      void *drvdata, const char *fmt, ...);
- __printf(6, 7) struct device *
-device_create_with_groups(struct class *cls, struct device *parent, dev_t devt,
-+device_create_with_groups(const struct class *cls, struct device *parent, dev_t devt,
- 			  void *drvdata, const struct attribute_group **groups,
- 			  const char *fmt, ...);
-void device_destroy(struct class *cls, dev_t devt);
-+void device_destroy(const struct class *cls, dev_t devt);
- 
- int __must_check device_add_groups(struct device *dev,
- 				   const struct attribute_group **groups);
-@@ -1041,28 +1148,11 @@ static inline void device_remove_group(struct device *dev,
- {
- 	const struct attribute_group *groups[] = { grp, NULL };
- 
-	return device_remove_groups(dev, groups);
-+	device_remove_groups(dev, groups);
- }
- 
-int __must_check devm_device_add_groups(struct device *dev,
-					const struct attribute_group **groups);
-void devm_device_remove_groups(struct device *dev,
-			       const struct attribute_group **groups);
- int __must_check devm_device_add_group(struct device *dev,
- 				       const struct attribute_group *grp);
-void devm_device_remove_group(struct device *dev,
-			      const struct attribute_group *grp);
-
-/*
- * Platform "fixup" functions - allow the platform to have their say
- * about devices and actions that the general device layer doesn't
- * know about.
- */
-/* Notify platform of device discovery */
-extern int (*platform_notify)(struct device *dev);
-
-extern int (*platform_notify_remove)(struct device *dev);
-
- 
- /*
-  * get_device - atomically increment the reference count for the device.
-@@ -1070,6 +1160,9 @@ extern int (*platform_notify_remove)(struct device *dev);
-  */
- struct device *get_device(struct device *dev);
- void put_device(struct device *dev);
-+
-+DEFINE_FREE(put_device, struct device *, if (_T) put_device(_T))
-+
- bool kill_device(struct device *dev);
- 
- #ifdef CONFIG_DEVTMPFS
-@@ -1091,9 +1184,12 @@ void device_link_del(struct device_link *link);
- void device_link_remove(void *consumer, struct device *supplier);
- void device_links_supplier_sync_state_pause(void);
- void device_links_supplier_sync_state_resume(void);
-+void device_link_wait_removal(void);
- 
-extern __printf(3, 4)
-int dev_err_probe(const struct device *dev, int err, const char *fmt, ...);
-+static inline bool device_link_test(const struct device_link *link, u32 flags)
-+{
-+	return !!(link->flags & flags);
-+}
- 
- /* Create alias, so I can be autoloaded. */
- #define MODULE_ALIAS_CHARDEV(major,minor) \
-@@ -1101,10 +1197,4 @@ int dev_err_probe(const struct device *dev, int err, const char *fmt, ...);
- #define MODULE_ALIAS_CHARDEV_MAJOR(major) \
- 	MODULE_ALIAS("char-major-" __stringify(major) "-*")
- 
-#ifdef CONFIG_SYSFS_DEPRECATED
-extern long sysfs_deprecated;
-#else
-#define sysfs_deprecated 0
-#endif
-
- #endif /* _DEVICE_H_ */
-diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h
-index d8b29ccd07e5..99b1002b3e31 100644
--- a/include/linux/device/bus.h
-+++ b/include/linux/device/bus.h
-@@ -26,7 +26,6 @@ struct fwnode_handle;
-  *
-  * @name:	The name of the bus.
-  * @dev_name:	Used for subsystems to enumerate devices like ("foo%u", dev->id).
- * @dev_root:	Default device to use as the parent.
-  * @bus_groups:	Default attributes of the bus.
-  * @dev_groups:	Default attributes of the devices on the bus.
-  * @drv_groups: Default attributes of the device drivers on the bus.
-@@ -49,6 +48,7 @@ struct fwnode_handle;
-  *		will never get called until they do.
-  * @remove:	Called when a device removed from this bus.
-  * @shutdown:	Called at shut-down time to quiesce the device.
-+ * @irq_get_affinity:	Get IRQ affinity mask for the device on this bus.
-  *
-  * @online:	Called to put the device back online (after offlining it).
-  * @offline:	Called to put the device offline for hot-removal. May fail.
-@@ -63,12 +63,6 @@ struct fwnode_handle;
-  *			this bus.
-  * @pm:		Power management operations of this bus, callback the specific
-  *		device driver's pm-ops.
- * @iommu_ops:  IOMMU specific operations for this bus, used to attach IOMMU
- *              driver implementations to a bus and allow the driver to do
- *              bus-specific setup
- * @p:		The private data of the driver core, only the driver core can
- *		touch this.
- * @lock_key:	Lock class key for use by the lock validator
-  * @need_parent_lock:	When probing or removing a device on this bus, the
-  *			device core should lock the device's parent.
-  *
-@@ -84,17 +78,18 @@ struct fwnode_handle;
- struct bus_type {
- 	const char		*name;
- 	const char		*dev_name;
-	struct device		*dev_root;
- 	const struct attribute_group **bus_groups;
- 	const struct attribute_group **dev_groups;
- 	const struct attribute_group **drv_groups;
- 
-	int (*match)(struct device *dev, struct device_driver *drv);
-	int (*uevent)(struct device *dev, struct kobj_uevent_env *env);
-+	int (*match)(struct device *dev, const struct device_driver *drv);
-+	int (*uevent)(const struct device *dev, struct kobj_uevent_env *env);
- 	int (*probe)(struct device *dev);
- 	void (*sync_state)(struct device *dev);
- 	void (*remove)(struct device *dev);
- 	void (*shutdown)(struct device *dev);
-+	const struct cpumask *(*irq_get_affinity)(struct device *dev,
-+			unsigned int irq_vec);
- 
- 	int (*online)(struct device *dev);
- 	int (*offline)(struct device *dev);
-@@ -109,24 +104,19 @@ struct bus_type {
- 
- 	const struct dev_pm_ops *pm;
- 
-	const struct iommu_ops *iommu_ops;
-
-	struct subsys_private *p;
-	struct lock_class_key lock_key;
-
- 	bool need_parent_lock;
- };
- 
-extern int __must_check bus_register(struct bus_type *bus);
-+int __must_check bus_register(const struct bus_type *bus);
- 
-extern void bus_unregister(struct bus_type *bus);
-+void bus_unregister(const struct bus_type *bus);
- 
-extern int __must_check bus_rescan_devices(struct bus_type *bus);
-+int __must_check bus_rescan_devices(const struct bus_type *bus);
- 
- struct bus_attribute {
- 	struct attribute	attr;
-	ssize_t (*show)(struct bus_type *bus, char *buf);
-	ssize_t (*store)(struct bus_type *bus, const char *buf, size_t count);
-+	ssize_t (*show)(const struct bus_type *bus, char *buf);
-+	ssize_t (*store)(const struct bus_type *bus, const char *buf, size_t count);
- };
- 
- #define BUS_ATTR_RW(_name) \
-@@ -136,12 +126,15 @@ struct bus_attribute {
- #define BUS_ATTR_WO(_name) \
- 	struct bus_attribute bus_attr_##_name = __ATTR_WO(_name)
- 
-extern int __must_check bus_create_file(struct bus_type *,
-					struct bus_attribute *);
-extern void bus_remove_file(struct bus_type *, struct bus_attribute *);
-+int __must_check bus_create_file(const struct bus_type *bus, struct bus_attribute *attr);
-+void bus_remove_file(const struct bus_type *bus, struct bus_attribute *attr);
-+
-+/* Matching function type for drivers/base APIs to find a specific device */
-+typedef int (*device_match_t)(struct device *dev, const void *data);
- 
- /* Generic device matching functions that all busses can use to match with */
- int device_match_name(struct device *dev, const void *name);
-+int device_match_type(struct device *dev, const void *type);
- int device_match_of_node(struct device *dev, const void *np);
- int device_match_fwnode(struct device *dev, const void *fwnode);
- int device_match_devt(struct device *dev, const void *pdevt);
-@@ -149,23 +142,17 @@ int device_match_acpi_dev(struct device *dev, const void *adev);
- int device_match_acpi_handle(struct device *dev, const void *handle);
- int device_match_any(struct device *dev, const void *unused);
- 
-+/* Device iterating function type for various driver core for_each APIs */
-+typedef int (*device_iter_t)(struct device *dev, void *data);
-+
- /* iterator helpers for buses */
-struct subsys_dev_iter {
-	struct klist_iter		ki;
-	const struct device_type	*type;
-};
-void subsys_dev_iter_init(struct subsys_dev_iter *iter,
-			 struct bus_type *subsys,
-			 struct device *start,
-			 const struct device_type *type);
-struct device *subsys_dev_iter_next(struct subsys_dev_iter *iter);
-void subsys_dev_iter_exit(struct subsys_dev_iter *iter);
-
-int bus_for_each_dev(struct bus_type *bus, struct device *start, void *data,
-		     int (*fn)(struct device *dev, void *data));
-struct device *bus_find_device(struct bus_type *bus, struct device *start,
-			       const void *data,
-			       int (*match)(struct device *dev, const void *data));
-+int bus_for_each_dev(const struct bus_type *bus, struct device *start,
-+		     void *data, device_iter_t fn);
-+struct device *bus_find_device(const struct bus_type *bus, struct device *start,
-+			       const void *data, device_match_t match);
-+struct device *bus_find_device_reverse(const struct bus_type *bus,
-+				       struct device *start, const void *data,
-+				       device_match_t match);
- /**
-  * bus_find_device_by_name - device iterator for locating a particular device
-  * of a specific name.
-@@ -173,7 +160,7 @@ struct device *bus_find_device(struct bus_type *bus, struct device *start,
-  * @start: Device to begin with
-  * @name: name of the device to match
-  */
-static inline struct device *bus_find_device_by_name(struct bus_type *bus,
-+static inline struct device *bus_find_device_by_name(const struct bus_type *bus,
- 						     struct device *start,
- 						     const char *name)
- {
-@@ -187,7 +174,7 @@ static inline struct device *bus_find_device_by_name(struct bus_type *bus,
-  * @np: of_node of the device to match.
-  */
- static inline struct device *
-bus_find_device_by_of_node(struct bus_type *bus, const struct device_node *np)
-+bus_find_device_by_of_node(const struct bus_type *bus, const struct device_node *np)
- {
- 	return bus_find_device(bus, NULL, np, device_match_of_node);
- }
-@@ -199,7 +186,7 @@ bus_find_device_by_of_node(struct bus_type *bus, const struct device_node *np)
-  * @fwnode: fwnode of the device to match.
-  */
- static inline struct device *
-bus_find_device_by_fwnode(struct bus_type *bus, const struct fwnode_handle *fwnode)
-+bus_find_device_by_fwnode(const struct bus_type *bus, const struct fwnode_handle *fwnode)
- {
- 	return bus_find_device(bus, NULL, fwnode, device_match_fwnode);
- }
-@@ -210,7 +197,7 @@ bus_find_device_by_fwnode(struct bus_type *bus, const struct fwnode_handle *fwno
-  * @bus: bus type
-  * @devt: device type of the device to match.
-  */
-static inline struct device *bus_find_device_by_devt(struct bus_type *bus,
-+static inline struct device *bus_find_device_by_devt(const struct bus_type *bus,
- 						     dev_t devt)
- {
- 	return bus_find_device(bus, NULL, &devt, device_match_devt);
-@@ -223,7 +210,7 @@ static inline struct device *bus_find_device_by_devt(struct bus_type *bus,
-  * @cur: device to begin the search with.
-  */
- static inline struct device *
-bus_find_next_device(struct bus_type *bus,struct device *cur)
-+bus_find_next_device(const struct bus_type *bus,struct device *cur)
- {
- 	return bus_find_device(bus, cur, NULL, device_match_any);
- }
-@@ -238,23 +225,21 @@ struct acpi_device;
-  * @adev: ACPI COMPANION device to match.
-  */
- static inline struct device *
-bus_find_device_by_acpi_dev(struct bus_type *bus, const struct acpi_device *adev)
-+bus_find_device_by_acpi_dev(const struct bus_type *bus, const struct acpi_device *adev)
- {
- 	return bus_find_device(bus, NULL, adev, device_match_acpi_dev);
- }
- #else
- static inline struct device *
-bus_find_device_by_acpi_dev(struct bus_type *bus, const void *adev)
-+bus_find_device_by_acpi_dev(const struct bus_type *bus, const void *adev)
- {
- 	return NULL;
- }
- #endif
- 
-struct device *subsys_find_device_by_id(struct bus_type *bus, unsigned int id,
-					struct device *hint);
-int bus_for_each_drv(struct bus_type *bus, struct device_driver *start,
-+int bus_for_each_drv(const struct bus_type *bus, struct device_driver *start,
- 		     void *data, int (*fn)(struct device_driver *, void *));
-void bus_sort_breadthfirst(struct bus_type *bus,
-+void bus_sort_breadthfirst(const struct bus_type *bus,
- 			   int (*compare)(const struct device *a,
- 					  const struct device *b));
- /*
-@@ -265,28 +250,40 @@ void bus_sort_breadthfirst(struct bus_type *bus,
-  */
- struct notifier_block;
- 
-extern int bus_register_notifier(struct bus_type *bus,
-				 struct notifier_block *nb);
-extern int bus_unregister_notifier(struct bus_type *bus,
-				   struct notifier_block *nb);
-+int bus_register_notifier(const struct bus_type *bus, struct notifier_block *nb);
-+int bus_unregister_notifier(const struct bus_type *bus, struct notifier_block *nb);
- 
-/* All 4 notifers below get called with the target struct device *
- * as an argument. Note that those functions are likely to be called
- * with the device lock held in the core, so be careful.
-+/**
-+ * enum bus_notifier_event - Bus Notifier events that have happened
-+ * @BUS_NOTIFY_ADD_DEVICE: device is added to this bus
-+ * @BUS_NOTIFY_DEL_DEVICE: device is about to be removed from this bus
-+ * @BUS_NOTIFY_REMOVED_DEVICE: device is successfully removed from this bus
-+ * @BUS_NOTIFY_BIND_DRIVER: a driver is about to be bound to this device on this bus
-+ * @BUS_NOTIFY_BOUND_DRIVER: a driver is successfully bound to this device on this bus
-+ * @BUS_NOTIFY_UNBIND_DRIVER: a driver is about to be unbound from this device on this bus
-+ * @BUS_NOTIFY_UNBOUND_DRIVER: a driver is successfully unbound from this device on this bus
-+ * @BUS_NOTIFY_DRIVER_NOT_BOUND: a driver failed to be bound to this device on this bus
-+ *
-+ * These are the value passed to a bus notifier when a specific event happens.
-+ *
-+ * Note that bus notifiers are likely to be called with the device lock already
-+ * held by the driver core, so be careful in any notifier callback as to what
-+ * you do with the device structure.
-+ *
-+ * All bus notifiers are called with the target struct device * as an argument.
-  */
-#define BUS_NOTIFY_ADD_DEVICE		0x00000001 /* device added */
-#define BUS_NOTIFY_DEL_DEVICE		0x00000002 /* device to be removed */
-#define BUS_NOTIFY_REMOVED_DEVICE	0x00000003 /* device removed */
-#define BUS_NOTIFY_BIND_DRIVER		0x00000004 /* driver about to be
-						      bound */
-#define BUS_NOTIFY_BOUND_DRIVER		0x00000005 /* driver bound to device */
-#define BUS_NOTIFY_UNBIND_DRIVER	0x00000006 /* driver about to be
-						      unbound */
-#define BUS_NOTIFY_UNBOUND_DRIVER	0x00000007 /* driver is unbound
-						      from the device */
-#define BUS_NOTIFY_DRIVER_NOT_BOUND	0x00000008 /* driver fails to be bound */
-
-extern struct kset *bus_get_kset(struct bus_type *bus);
-extern struct klist *bus_get_device_klist(struct bus_type *bus);
-+enum bus_notifier_event {
-+	BUS_NOTIFY_ADD_DEVICE,
-+	BUS_NOTIFY_DEL_DEVICE,
-+	BUS_NOTIFY_REMOVED_DEVICE,
-+	BUS_NOTIFY_BIND_DRIVER,
-+	BUS_NOTIFY_BOUND_DRIVER,
-+	BUS_NOTIFY_UNBIND_DRIVER,
-+	BUS_NOTIFY_UNBOUND_DRIVER,
-+	BUS_NOTIFY_DRIVER_NOT_BOUND,
-+};
-+
-+struct kset *bus_get_kset(const struct bus_type *bus);
-+struct device *bus_get_dev_root(const struct bus_type *bus);
- 
- #endif
-diff --git a/include/linux/device/class.h b/include/linux/device/class.h
-index e61ec5502019..65880e60c720 100644
--- a/include/linux/device/class.h
-+++ b/include/linux/device/class.h
-@@ -25,10 +25,8 @@ struct fwnode_handle;
- /**
-  * struct class - device classes
-  * @name:	Name of the class.
- * @owner:	The module owner.
-  * @class_groups: Default attributes of this class.
-  * @dev_groups:	Default attributes of the devices that belong to the class.
- * @dev_kobj:	The kobject that represents this class and links it into the hierarchy.
-  * @dev_uevent:	Called when a device is added, removed from this class, or a
-  *		few other things that generate uevents to add the environment
-  *		variables.
-@@ -42,8 +40,6 @@ struct fwnode_handle;
-  *		for the devices belonging to the class. Usually tied to
-  *		device's namespace.
-  * @pm:		The default device power management operations of this class.
- * @p:		The private data of the driver core, no one other than the
- *		driver core can touch this.
-  *
-  * A class is a higher-level view of a device that abstracts out low-level
-  * implementation details. Drivers may see a SCSI disk or an ATA disk, but,
-@@ -53,70 +49,51 @@ struct fwnode_handle;
-  */
- struct class {
- 	const char		*name;
-	struct module		*owner;
- 
- 	const struct attribute_group	**class_groups;
- 	const struct attribute_group	**dev_groups;
-	struct kobject			*dev_kobj;
- 
-	int (*dev_uevent)(struct device *dev, struct kobj_uevent_env *env);
-	char *(*devnode)(struct device *dev, umode_t *mode);
-+	int (*dev_uevent)(const struct device *dev, struct kobj_uevent_env *env);
-+	char *(*devnode)(const struct device *dev, umode_t *mode);
- 
-	void (*class_release)(struct class *class);
-+	void (*class_release)(const struct class *class);
- 	void (*dev_release)(struct device *dev);
- 
- 	int (*shutdown_pre)(struct device *dev);
- 
- 	const struct kobj_ns_type_operations *ns_type;
-	const void *(*namespace)(struct device *dev);
-+	const void *(*namespace)(const struct device *dev);
- 
-	void (*get_ownership)(struct device *dev, kuid_t *uid, kgid_t *gid);
-+	void (*get_ownership)(const struct device *dev, kuid_t *uid, kgid_t *gid);
- 
- 	const struct dev_pm_ops *pm;
-
-	struct subsys_private *p;
- };
- 
- struct class_dev_iter {
- 	struct klist_iter		ki;
- 	const struct device_type	*type;
-+	struct subsys_private		*sp;
- };
- 
-extern struct kobject *sysfs_dev_block_kobj;
-extern struct kobject *sysfs_dev_char_kobj;
-extern int __must_check __class_register(struct class *class,
-					 struct lock_class_key *key);
-extern void class_unregister(struct class *class);
-
-/* This is a #define to keep the compiler from merging different
- * instances of the __key variable */
-#define class_register(class)			\
-({						\
-	static struct lock_class_key __key;	\
-	__class_register(class, &__key);	\
-})
-+int __must_check class_register(const struct class *class);
-+void class_unregister(const struct class *class);
-+bool class_is_registered(const struct class *class);
- 
- struct class_compat;
- struct class_compat *class_compat_register(const char *name);
- void class_compat_unregister(struct class_compat *cls);
-int class_compat_create_link(struct class_compat *cls, struct device *dev,
-			     struct device *device_link);
-void class_compat_remove_link(struct class_compat *cls, struct device *dev,
-			      struct device *device_link);
-
-extern void class_dev_iter_init(struct class_dev_iter *iter,
-				struct class *class,
-				struct device *start,
-				const struct device_type *type);
-extern struct device *class_dev_iter_next(struct class_dev_iter *iter);
-extern void class_dev_iter_exit(struct class_dev_iter *iter);
-
-extern int class_for_each_device(struct class *class, struct device *start,
-				 void *data,
-				 int (*fn)(struct device *dev, void *data));
-extern struct device *class_find_device(struct class *class,
-					struct device *start, const void *data,
-					int (*match)(struct device *, const void *));
-+int class_compat_create_link(struct class_compat *cls, struct device *dev);
-+void class_compat_remove_link(struct class_compat *cls, struct device *dev);
-+
-+void class_dev_iter_init(struct class_dev_iter *iter, const struct class *class,
-+			 const struct device *start, const struct device_type *type);
-+struct device *class_dev_iter_next(struct class_dev_iter *iter);
-+void class_dev_iter_exit(struct class_dev_iter *iter);
-+
-+int class_for_each_device(const struct class *class, const struct device *start,
-+			  void *data, device_iter_t fn);
-+struct device *class_find_device(const struct class *class, const struct device *start,
-+				 const void *data, device_match_t match);
- 
- /**
-  * class_find_device_by_name - device iterator for locating a particular device
-@@ -124,7 +101,7 @@ extern struct device *class_find_device(struct class *class,
-  * @class: class type
-  * @name: name of the device to match
-  */
-static inline struct device *class_find_device_by_name(struct class *class,
-+static inline struct device *class_find_device_by_name(const struct class *class,
- 						       const char *name)
- {
- 	return class_find_device(class, NULL, name, device_match_name);
-@@ -136,8 +113,8 @@ static inline struct device *class_find_device_by_name(struct class *class,
-  * @class: class type
-  * @np: of_node of the device to match.
-  */
-static inline struct device *
-class_find_device_by_of_node(struct class *class, const struct device_node *np)
-+static inline struct device *class_find_device_by_of_node(const struct class *class,
-+							  const struct device_node *np)
- {
- 	return class_find_device(class, NULL, np, device_match_of_node);
- }
-@@ -148,9 +125,8 @@ class_find_device_by_of_node(struct class *class, const struct device_node *np)
-  * @class: class type
-  * @fwnode: fwnode of the device to match.
-  */
-static inline struct device *
-class_find_device_by_fwnode(struct class *class,
-			    const struct fwnode_handle *fwnode)
-+static inline struct device *class_find_device_by_fwnode(const struct class *class,
-+							 const struct fwnode_handle *fwnode)
- {
- 	return class_find_device(class, NULL, fwnode, device_match_fwnode);
- }
-@@ -161,7 +137,7 @@ class_find_device_by_fwnode(struct class *class,
-  * @class: class type
-  * @devt: device type of the device to match.
-  */
-static inline struct device *class_find_device_by_devt(struct class *class,
-+static inline struct device *class_find_device_by_devt(const struct class *class,
- 						       dev_t devt)
- {
- 	return class_find_device(class, NULL, &devt, device_match_devt);
-@@ -175,14 +151,14 @@ struct acpi_device;
-  * @class: class type
-  * @adev: ACPI_COMPANION device to match.
-  */
-static inline struct device *
-class_find_device_by_acpi_dev(struct class *class, const struct acpi_device *adev)
-+static inline struct device *class_find_device_by_acpi_dev(const struct class *class,
-+							   const struct acpi_device *adev)
- {
- 	return class_find_device(class, NULL, adev, device_match_acpi_dev);
- }
- #else
-static inline struct device *
-class_find_device_by_acpi_dev(struct class *class, const void *adev)
-+static inline struct device *class_find_device_by_acpi_dev(const struct class *class,
-+							   const void *adev)
- {
- 	return NULL;
- }
-@@ -190,10 +166,10 @@ class_find_device_by_acpi_dev(struct class *class, const void *adev)
- 
- struct class_attribute {
- 	struct attribute attr;
-	ssize_t (*show)(struct class *class, struct class_attribute *attr,
-+	ssize_t (*show)(const struct class *class, const struct class_attribute *attr,
- 			char *buf);
-	ssize_t (*store)(struct class *class, struct class_attribute *attr,
-			const char *buf, size_t count);
-+	ssize_t (*store)(const struct class *class, const struct class_attribute *attr,
-+			 const char *buf, size_t count);
- };
- 
- #define CLASS_ATTR_RW(_name) \
-@@ -203,23 +179,21 @@ struct class_attribute {
- #define CLASS_ATTR_WO(_name) \
- 	struct class_attribute class_attr_##_name = __ATTR_WO(_name)
- 
-extern int __must_check class_create_file_ns(struct class *class,
-					     const struct class_attribute *attr,
-					     const void *ns);
-extern void class_remove_file_ns(struct class *class,
-				 const struct class_attribute *attr,
-				 const void *ns);
-+int __must_check class_create_file_ns(const struct class *class, const struct class_attribute *attr,
-+				      const void *ns);
-+void class_remove_file_ns(const struct class *class, const struct class_attribute *attr,
-+			  const void *ns);
- 
-static inline int __must_check class_create_file(struct class *class,
-					const struct class_attribute *attr)
-+static inline int __must_check class_create_file(const struct class *class,
-+						 const struct class_attribute *attr)
- {
- 	return class_create_file_ns(class, attr, NULL);
- }
- 
-static inline void class_remove_file(struct class *class,
-+static inline void class_remove_file(const struct class *class,
- 				     const struct class_attribute *attr)
- {
-	return class_remove_file_ns(class, attr, NULL);
-+	class_remove_file_ns(class, attr, NULL);
- }
- 
- /* Simple class attribute that is just a static string */
-@@ -235,46 +209,21 @@ struct class_attribute_string {
- 	struct class_attribute_string class_attr_##_name = \
- 		_CLASS_ATTR_STRING(_name, _mode, _str)
- 
-extern ssize_t show_class_attr_string(struct class *class, struct class_attribute *attr,
-                        char *buf);
-+ssize_t show_class_attr_string(const struct class *class, const struct class_attribute *attr,
-+			       char *buf);
- 
- struct class_interface {
- 	struct list_head	node;
-	struct class		*class;
-+	const struct class	*class;
- 
-	int (*add_dev)		(struct device *, struct class_interface *);
-	void (*remove_dev)	(struct device *, struct class_interface *);
-+	int (*add_dev)		(struct device *dev);
-+	void (*remove_dev)	(struct device *dev);
- };
- 
-extern int __must_check class_interface_register(struct class_interface *);
-extern void class_interface_unregister(struct class_interface *);
-
-extern struct class * __must_check __class_create(struct module *owner,
-						  const char *name,
-						  struct lock_class_key *key);
-extern void class_destroy(struct class *cls);
-
-/* This is a #define to keep the compiler from merging different
- * instances of the __key variable */
-
-/**
- * class_create - create a struct class structure
- * @owner: pointer to the module that is to "own" this struct class
- * @name: pointer to a string for the name of this class.
- *
- * This is used to create a struct class pointer that can then be used
- * in calls to device_create().
- *
- * Returns &struct class pointer on success, or ERR_PTR() on error.
- *
- * Note, the pointer created here is to be destroyed when finished by
- * making a call to class_destroy().
- */
-#define class_create(owner, name)		\
-({						\
-	static struct lock_class_key __key;	\
-	__class_create(owner, name, &__key);	\
-})
-+int __must_check class_interface_register(struct class_interface *);
-+void class_interface_unregister(struct class_interface *);
- 
-+struct class * __must_check class_create(const char *name);
-+void class_destroy(const struct class *cls);
- 
- #endif	/* _DEVICE_CLASS_H_ */
-diff --git a/include/linux/device/devres.h b/include/linux/device/devres.h
-new file mode 100644
-index 000000000000..9c1e3d643d69
--- /dev/null
-+++ b/include/linux/device/devres.h
-@@ -0,0 +1,189 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _DEVICE_DEVRES_H_
-+#define _DEVICE_DEVRES_H_
-+
-+#include <linux/err.h>
-+#include <linux/gfp_types.h>
-+#include <linux/numa.h>
-+#include <linux/overflow.h>
-+#include <linux/stdarg.h>
-+#include <linux/types.h>
-+#include <asm/bug.h>
-+#include <asm/percpu.h>
-+
-+struct device;
-+struct device_node;
-+struct resource;
-+
-+/* device resource management */
-+typedef void (*dr_release_t)(struct device *dev, void *res);
-+typedef int (*dr_match_t)(struct device *dev, void *res, void *match_data);
-+
-+void * __malloc
-+__devres_alloc_node(dr_release_t release, size_t size, gfp_t gfp, int nid, const char *name);
-+#define devres_alloc(release, size, gfp) \
-+	__devres_alloc_node(release, size, gfp, NUMA_NO_NODE, #release)
-+#define devres_alloc_node(release, size, gfp, nid) \
-+	__devres_alloc_node(release, size, gfp, nid, #release)
-+
-+void devres_for_each_res(struct device *dev, dr_release_t release,
-+			 dr_match_t match, void *match_data,
-+			 void (*fn)(struct device *, void *, void *),
-+			 void *data);
-+void devres_free(void *res);
-+void devres_add(struct device *dev, void *res);
-+void *devres_find(struct device *dev, dr_release_t release, dr_match_t match, void *match_data);
-+void *devres_get(struct device *dev, void *new_res, dr_match_t match, void *match_data);
-+void *devres_remove(struct device *dev, dr_release_t release, dr_match_t match, void *match_data);
-+int devres_destroy(struct device *dev, dr_release_t release, dr_match_t match, void *match_data);
-+int devres_release(struct device *dev, dr_release_t release, dr_match_t match, void *match_data);
-+
-+/* devres group */
-+void * __must_check devres_open_group(struct device *dev, void *id, gfp_t gfp);
-+void devres_close_group(struct device *dev, void *id);
-+void devres_remove_group(struct device *dev, void *id);
-+int devres_release_group(struct device *dev, void *id);
-+
-+/* managed devm_k.alloc/kfree for device drivers */
-+void * __alloc_size(2)
-+devm_kmalloc(struct device *dev, size_t size, gfp_t gfp);
-+void * __must_check __realloc_size(3)
-+devm_krealloc(struct device *dev, void *ptr, size_t size, gfp_t gfp);
-+static inline void *devm_kzalloc(struct device *dev, size_t size, gfp_t gfp)
-+{
-+	return devm_kmalloc(dev, size, gfp | __GFP_ZERO);
-+}
-+static inline void *devm_kmalloc_array(struct device *dev, size_t n, size_t size, gfp_t flags)
-+{
-+	size_t bytes;
-+
-+	if (unlikely(check_mul_overflow(n, size, &bytes)))
-+		return NULL;
-+
-+	return devm_kmalloc(dev, bytes, flags);
-+}
-+static inline void *devm_kcalloc(struct device *dev, size_t n, size_t size, gfp_t flags)
-+{
-+	return devm_kmalloc_array(dev, n, size, flags | __GFP_ZERO);
-+}
-+static inline __realloc_size(3, 4) void * __must_check
-+devm_krealloc_array(struct device *dev, void *p, size_t new_n, size_t new_size, gfp_t flags)
-+{
-+	size_t bytes;
-+
-+	if (unlikely(check_mul_overflow(new_n, new_size, &bytes)))
-+		return NULL;
-+
-+	return devm_krealloc(dev, p, bytes, flags);
-+}
-+
-+void devm_kfree(struct device *dev, const void *p);
-+
-+void * __realloc_size(3)
-+devm_kmemdup(struct device *dev, const void *src, size_t len, gfp_t gfp);
-+const void *
-+devm_kmemdup_const(struct device *dev, const void *src, size_t len, gfp_t gfp);
-+static inline void *devm_kmemdup_array(struct device *dev, const void *src,
-+				       size_t n, size_t size, gfp_t flags)
-+{
-+	return devm_kmemdup(dev, src, size_mul(size, n), flags);
-+}
-+
-+char * __malloc
-+devm_kstrdup(struct device *dev, const char *s, gfp_t gfp);
-+const char *devm_kstrdup_const(struct device *dev, const char *s, gfp_t gfp);
-+char * __printf(3, 0) __malloc
-+devm_kvasprintf(struct device *dev, gfp_t gfp, const char *fmt, va_list ap);
-+char * __printf(3, 4) __malloc
-+devm_kasprintf(struct device *dev, gfp_t gfp, const char *fmt, ...);
-+
-+/**
-+ * devm_alloc_percpu - Resource-managed alloc_percpu
-+ * @dev: Device to allocate per-cpu memory for
-+ * @type: Type to allocate per-cpu memory for
-+ *
-+ * Managed alloc_percpu. Per-cpu memory allocated with this function is
-+ * automatically freed on driver detach.
-+ *
-+ * RETURNS:
-+ * Pointer to allocated memory on success, NULL on failure.
-+ */
-+#define devm_alloc_percpu(dev, type)      \
-+	((typeof(type) __percpu *)__devm_alloc_percpu((dev), sizeof(type), __alignof__(type)))
-+
-+void __percpu *__devm_alloc_percpu(struct device *dev, size_t size, size_t align);
-+
-+unsigned long devm_get_free_pages(struct device *dev, gfp_t gfp_mask, unsigned int order);
-+void devm_free_pages(struct device *dev, unsigned long addr);
-+
-+#ifdef CONFIG_HAS_IOMEM
-+
-+void __iomem *devm_ioremap_resource(struct device *dev, const struct resource *res);
-+void __iomem *devm_ioremap_resource_wc(struct device *dev, const struct resource *res);
-+
-+void __iomem *devm_of_iomap(struct device *dev, struct device_node *node, int index,
-+			    resource_size_t *size);
-+#else
-+
-+static inline
-+void __iomem *devm_ioremap_resource(struct device *dev, const struct resource *res)
-+{
-+	return IOMEM_ERR_PTR(-EINVAL);
-+}
-+
-+static inline
-+void __iomem *devm_ioremap_resource_wc(struct device *dev, const struct resource *res)
-+{
-+	return IOMEM_ERR_PTR(-EINVAL);
-+}
-+
-+static inline
-+void __iomem *devm_of_iomap(struct device *dev, struct device_node *node, int index,
-+			    resource_size_t *size)
-+{
-+	return IOMEM_ERR_PTR(-EINVAL);
-+}
-+
-+#endif
-+
-+/* allows to add/remove a custom action to devres stack */
-+int devm_remove_action_nowarn(struct device *dev, void (*action)(void *), void *data);
-+
-+/**
-+ * devm_remove_action() - removes previously added custom action
-+ * @dev: Device that owns the action
-+ * @action: Function implementing the action
-+ * @data: Pointer to data passed to @action implementation
-+ *
-+ * Removes instance of @action previously added by devm_add_action().
-+ * Both action and data should match one of the existing entries.
-+ */
-+static inline
-+void devm_remove_action(struct device *dev, void (*action)(void *), void *data)
-+{
-+	WARN_ON(devm_remove_action_nowarn(dev, action, data));
-+}
-+
-+void devm_release_action(struct device *dev, void (*action)(void *), void *data);
-+
-+int __devm_add_action(struct device *dev, void (*action)(void *), void *data, const char *name);
-+#define devm_add_action(dev, action, data) \
-+	__devm_add_action(dev, action, data, #action)
-+
-+static inline int __devm_add_action_or_reset(struct device *dev, void (*action)(void *),
-+					     void *data, const char *name)
-+{
-+	int ret;
-+
-+	ret = __devm_add_action(dev, action, data, name);
-+	if (ret)
-+		action(data);
-+
-+	return ret;
-+}
-+#define devm_add_action_or_reset(dev, action, data) \
-+	__devm_add_action_or_reset(dev, action, data, #action)
-+
-+bool devm_is_action_added(struct device *dev, void (*action)(void *), void *data);
-+
-+#endif /* _DEVICE_DEVRES_H_ */
-diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h
-index 7acaabde5396..cd8e0f0a634b 100644
--- a/include/linux/device/driver.h
-+++ b/include/linux/device/driver.h
-@@ -95,7 +95,7 @@ enum probe_type {
-  */
- struct device_driver {
- 	const char		*name;
-	struct bus_type		*bus;
-+	const struct bus_type	*bus;
- 
- 	struct module		*owner;
- 	const char		*mod_name;	/* used for built-in modules */
-@@ -122,13 +122,12 @@ struct device_driver {
- };
- 
- 
-extern int __must_check driver_register(struct device_driver *drv);
-extern void driver_unregister(struct device_driver *drv);
-+int __must_check driver_register(struct device_driver *drv);
-+void driver_unregister(struct device_driver *drv);
- 
-extern struct device_driver *driver_find(const char *name,
-					 struct bus_type *bus);
-extern int driver_probe_done(void);
-extern void wait_for_device_probe(void);
-+struct device_driver *driver_find(const char *name, const struct bus_type *bus);
-+bool __init driver_probe_done(void);
-+void wait_for_device_probe(void);
- void __init wait_for_init_devices_probe(void);
- 
- /* sysfs interface for exporting driver attributes */
-@@ -147,21 +146,18 @@ struct driver_attribute {
- #define DRIVER_ATTR_WO(_name) \
- 	struct driver_attribute driver_attr_##_name = __ATTR_WO(_name)
- 
-extern int __must_check driver_create_file(struct device_driver *driver,
-					const struct driver_attribute *attr);
-extern void driver_remove_file(struct device_driver *driver,
-			       const struct driver_attribute *attr);
-+int __must_check driver_create_file(const struct device_driver *driver,
-+				    const struct driver_attribute *attr);
-+void driver_remove_file(const struct device_driver *driver,
-+			const struct driver_attribute *attr);
- 
- int driver_set_override(struct device *dev, const char **override,
- 			const char *s, size_t len);
-extern int __must_check driver_for_each_device(struct device_driver *drv,
-					       struct device *start,
-					       void *data,
-					       int (*fn)(struct device *dev,
-							 void *));
-struct device *driver_find_device(struct device_driver *drv,
-+int __must_check driver_for_each_device(struct device_driver *drv, struct device *start,
-+					void *data, device_iter_t fn);
-+struct device *driver_find_device(const struct device_driver *drv,
- 				  struct device *start, const void *data,
-				  int (*match)(struct device *dev, const void *data));
-+				  device_match_t match);
- 
- /**
-  * driver_find_device_by_name - device iterator for locating a particular device
-@@ -169,7 +165,7 @@ struct device *driver_find_device(struct device_driver *drv,
-  * @drv: the driver we're iterating
-  * @name: name of the device to match
-  */
-static inline struct device *driver_find_device_by_name(struct device_driver *drv,
-+static inline struct device *driver_find_device_by_name(const struct device_driver *drv,
- 							const char *name)
- {
- 	return driver_find_device(drv, NULL, name, device_match_name);
-@@ -182,7 +178,7 @@ static inline struct device *driver_find_device_by_name(struct device_driver *dr
-  * @np: of_node pointer to match.
-  */
- static inline struct device *
-driver_find_device_by_of_node(struct device_driver *drv,
-+driver_find_device_by_of_node(const struct device_driver *drv,
- 			      const struct device_node *np)
- {
- 	return driver_find_device(drv, NULL, np, device_match_of_node);
-@@ -207,13 +203,13 @@ driver_find_device_by_fwnode(struct device_driver *drv,
-  * @drv: the driver we're iterating
-  * @devt: devt pointer to match.
-  */
-static inline struct device *driver_find_device_by_devt(struct device_driver *drv,
-+static inline struct device *driver_find_device_by_devt(const struct device_driver *drv,
- 							dev_t devt)
- {
- 	return driver_find_device(drv, NULL, &devt, device_match_devt);
- }
- 
-static inline struct device *driver_find_next_device(struct device_driver *drv,
-+static inline struct device *driver_find_next_device(const struct device_driver *drv,
- 						     struct device *start)
- {
- 	return driver_find_device(drv, start, NULL, device_match_any);
-@@ -227,21 +223,21 @@ static inline struct device *driver_find_next_device(struct device_driver *drv,
-  * @adev: ACPI_COMPANION device to match.
-  */
- static inline struct device *
-driver_find_device_by_acpi_dev(struct device_driver *drv,
-+driver_find_device_by_acpi_dev(const struct device_driver *drv,
- 			       const struct acpi_device *adev)
- {
- 	return driver_find_device(drv, NULL, adev, device_match_acpi_dev);
- }
- #else
- static inline struct device *
-driver_find_device_by_acpi_dev(struct device_driver *drv, const void *adev)
-+driver_find_device_by_acpi_dev(const struct device_driver *drv, const void *adev)
- {
- 	return NULL;
- }
- #endif
- 
-extern int driver_deferred_probe_timeout;
- void driver_deferred_probe_add(struct device *dev);
-+int driver_deferred_probe_check_state(struct device *dev);
- void driver_init(void);
- 
- /**
-diff --git a/include/linux/device/faux.h b/include/linux/device/faux.h
-new file mode 100644
-index 000000000000..9f43c0e46aa4
--- /dev/null
-+++ b/include/linux/device/faux.h
-@@ -0,0 +1,69 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Copyright (c) 2025 Greg Kroah-Hartman <gregkh@linuxfoundation.org>
-+ * Copyright (c) 2025 The Linux Foundation
-+ *
-+ * A "simple" faux bus that allows devices to be created and added
-+ * automatically to it.  This is to be used whenever you need to create a
-+ * device that is not associated with any "real" system resources, and do
-+ * not want to have to deal with a bus/driver binding logic.  It is
-+ * intended to be very simple, with only a create and a destroy function
-+ * available.
-+ */
-+#ifndef _FAUX_DEVICE_H_
-+#define _FAUX_DEVICE_H_
-+
-+#include <linux/container_of.h>
-+#include <linux/device.h>
-+
-+/**
-+ * struct faux_device - a "faux" device
-+ * @dev:	internal struct device of the object
-+ *
-+ * A simple faux device that can be created/destroyed.  To be used when a
-+ * driver only needs to have a device to "hang" something off.  This can be
-+ * used for downloading firmware or other basic tasks.  Use this instead of
-+ * a struct platform_device if the device has no resources assigned to
-+ * it at all.
-+ */
-+struct faux_device {
-+	struct device dev;
-+};
-+#define to_faux_device(x) container_of_const((x), struct faux_device, dev)
-+
-+/**
-+ * struct faux_device_ops - a set of callbacks for a struct faux_device
-+ * @probe:	called when a faux device is probed by the driver core
-+ *		before the device is fully bound to the internal faux bus
-+ *		code.  If probe succeeds, return 0, otherwise return a
-+ *		negative error number to stop the probe sequence from
-+ *		succeeding.
-+ * @remove:	called when a faux device is removed from the system
-+ *
-+ * Both @probe and @remove are optional, if not needed, set to NULL.
-+ */
-+struct faux_device_ops {
-+	int (*probe)(struct faux_device *faux_dev);
-+	void (*remove)(struct faux_device *faux_dev);
-+};
-+
-+struct faux_device *faux_device_create(const char *name,
-+				       struct device *parent,
-+				       const struct faux_device_ops *faux_ops);
-+struct faux_device *faux_device_create_with_groups(const char *name,
-+						   struct device *parent,
-+						   const struct faux_device_ops *faux_ops,
-+						   const struct attribute_group **groups);
-+void faux_device_destroy(struct faux_device *faux_dev);
-+
-+static inline void *faux_device_get_drvdata(const struct faux_device *faux_dev)
-+{
-+	return dev_get_drvdata(&faux_dev->dev);
-+}
-+
-+static inline void faux_device_set_drvdata(struct faux_device *faux_dev, void *data)
-+{
-+	dev_set_drvdata(&faux_dev->dev, data);
-+}
-+
-+#endif /* _FAUX_DEVICE_H_ */
-diff --git a/include/linux/device_cgroup.h b/include/linux/device_cgroup.h
-index d02f32b7514e..0864773a57e8 100644
--- a/include/linux/device_cgroup.h
-+++ b/include/linux/device_cgroup.h
-@@ -18,15 +18,16 @@ static inline int devcgroup_inode_permission(struct inode *inode, int mask)
- {
- 	short type, access = 0;
- 
-+	if (likely(!S_ISBLK(inode->i_mode) && !S_ISCHR(inode->i_mode)))
-+		return 0;
-+
- 	if (likely(!inode->i_rdev))
- 		return 0;
- 
- 	if (S_ISBLK(inode->i_mode))
- 		type = DEVCG_DEV_BLOCK;
-	else if (S_ISCHR(inode->i_mode))
-+	else /* S_ISCHR by the test above */
- 		type = DEVCG_DEV_CHAR;
-	else
-		return 0;
- 
- 	if (mask & MAY_WRITE)
- 		access |= DEVCG_ACC_WRITE;
-diff --git a/include/linux/devm-helpers.h b/include/linux/devm-helpers.h
-index 74891802200d..708ca9131402 100644
--- a/include/linux/devm-helpers.h
-+++ b/include/linux/devm-helpers.h
-@@ -41,7 +41,7 @@ static inline void devm_delayed_work_drop(void *res)
-  * detached. A few drivers need delayed work which must be cancelled before
-  * driver is detached to avoid accessing removed resources.
-  * devm_delayed_work_autocancel() can be used to omit the explicit
- * cancelleation when driver is detached.
-+ * cancellation when driver is detached.
-  */
- static inline int devm_delayed_work_autocancel(struct device *dev,
- 					       struct delayed_work *w,
-@@ -66,7 +66,7 @@ static inline void devm_work_drop(void *res)
-  * A few drivers need to queue work which must be cancelled before driver
-  * is detached to avoid accessing removed resources.
-  * devm_work_autocancel() can be used to omit the explicit
- * cancelleation when driver is detached.
-+ * cancellation when driver is detached.
-  */
- static inline int devm_work_autocancel(struct device *dev,
- 				       struct work_struct *w,
-diff --git a/include/linux/dfl.h b/include/linux/dfl.h
-index 431636a0dc78..1f02db0c1897 100644
--- a/include/linux/dfl.h
-+++ b/include/linux/dfl.h
-@@ -27,11 +27,15 @@ enum dfl_id_type {
-  * @id: id of the dfl device.
-  * @type: type of DFL FIU of the device. See enum dfl_id_type.
-  * @feature_id: feature identifier local to its DFL FIU type.
-+ * @revision: revision of this dfl device feature.
-  * @mmio_res: mmio resource of this dfl device.
-  * @irqs: list of Linux IRQ numbers of this dfl device.
-  * @num_irqs: number of IRQs supported by this dfl device.
-  * @cdev: pointer to DFL FPGA container device this dfl device belongs to.
-  * @id_entry: matched id entry in dfl driver's id table.
-+ * @dfh_version: version of DFH for the device
-+ * @param_size: size of the block parameters in bytes
-+ * @params: pointer to block of parameters copied memory
-  */
- struct dfl_device {
- 	struct device dev;
-@@ -44,6 +48,9 @@ struct dfl_device {
- 	unsigned int num_irqs;
- 	struct dfl_fpga_cdev *cdev;
- 	const struct dfl_device_id *id_entry;
-+	u8 dfh_version;
-+	unsigned int param_size;
-+	void *params;
- };
- 
- /**
-@@ -64,7 +71,7 @@ struct dfl_driver {
- };
- 
- #define to_dfl_dev(d) container_of(d, struct dfl_device, dev)
-#define to_dfl_drv(d) container_of(d, struct dfl_driver, drv)
-+#define to_dfl_drv(d) container_of_const(d, struct dfl_driver, drv)
- 
- /*
-  * use a macro to avoid include chaining to get THIS_MODULE.
-@@ -84,4 +91,5 @@ void dfl_driver_unregister(struct dfl_driver *dfl_drv);
- 	module_driver(__dfl_driver, dfl_driver_register, \
- 		      dfl_driver_unregister)
- 
-+void *dfh_find_param(struct dfl_device *dfl_dev, int param_id, size_t *pcount);
- #endif /* __LINUX_DFL_H */
-diff --git a/include/linux/dibs.h b/include/linux/dibs.h
-new file mode 100644
-index 000000000000..c75607f8a5cf
--- /dev/null
-+++ b/include/linux/dibs.h
-@@ -0,0 +1,464 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ *  Direct Internal Buffer Sharing
-+ *
-+ *  Definitions for the DIBS module
-+ *
-+ *  Copyright IBM Corp. 2025
-+ */
-+#ifndef _DIBS_H
-+#define _DIBS_H
-+
-+#include <linux/device.h>
-+#include <linux/uuid.h>
-+
-+/* DIBS - Direct Internal Buffer Sharing - concept
-+ * -----------------------------------------------
-+ * In the case of multiple system sharing the same hardware, dibs fabrics can
-+ * provide dibs devices to these systems. The systems use dibs devices of the
-+ * same fabric to communicate via dmbs (Direct Memory Buffers). Each dmb has
-+ * exactly one owning local dibs device and one remote using dibs device, that
-+ * is authorized to write into this dmb. This access control is provided by the
-+ * dibs fabric.
-+ *
-+ * Because the access to the dmb is based on access to physical memory, it is
-+ * lossless and synchronous. The remote devices can directly access any offset
-+ * of the dmb.
-+ *
-+ * Dibs fabrics, dibs devices and dmbs are identified by tokens and ids.
-+ * Dibs fabric id is unique within the same hardware (with the exception of the
-+ * dibs loopback fabric), dmb token is unique within the same fabric, dibs
-+ * device gids are guaranteed to be unique within the same fabric and
-+ * statistically likely to be globally unique. The exchange of these tokens and
-+ * ids between the systems is not part of the dibs concept.
-+ *
-+ * The dibs layer provides an abstraction between dibs device drivers and dibs
-+ * clients.
-+ */
-+
-+/* DMB - Direct Memory Buffer
-+ * --------------------------
-+ * A dibs client provides a dmb as input buffer for a local receiving
-+ * dibs device for exactly one (remote) sending dibs device. Only this
-+ * sending device can send data into this dmb using move_data(). Sender
-+ * and receiver can be the same device. A dmb belongs to exactly one client.
-+ */
-+struct dibs_dmb {
-+	/* tok - Token for this dmb
-+	 * Used by remote and local devices and clients to address this dmb.
-+	 * Provided by dibs fabric. Unique per dibs fabric.
-+	 */
-+	u64 dmb_tok;
-+	/* rgid - GID of designated remote sending device */
-+	uuid_t rgid;
-+	/* cpu_addr - buffer address */
-+	void *cpu_addr;
-+	/* len - buffer length */
-+	u32 dmb_len;
-+	/* idx - Index of this DMB on this receiving device */
-+	u32 idx;
-+	/* VLAN support (deprecated)
-+	 * In order to write into a vlan-tagged dmb, the remote device needs
-+	 * to belong to the this vlan
-+	 */
-+	u32 vlan_valid;
-+	u32 vlan_id;
-+	/* optional, used by device driver */
-+	dma_addr_t dma_addr;
-+};
-+
-+/* DIBS events
-+ * -----------
-+ * Dibs devices can optionally notify dibs clients about events that happened
-+ * in the fabric or at the remote device or remote dmb.
-+ */
-+enum dibs_event_type {
-+	/* Buffer event, e.g. a remote dmb was unregistered */
-+	DIBS_BUF_EVENT,
-+	/* Device event, e.g. a remote dibs device was disabled */
-+	DIBS_DEV_EVENT,
-+	/* Software event, a dibs client can send an event signal to a
-+	 * remote dibs device.
-+	 */
-+	DIBS_SW_EVENT,
-+	DIBS_OTHER_TYPE };
-+
-+enum dibs_event_subtype {
-+	DIBS_BUF_UNREGISTERED,
-+	DIBS_DEV_DISABLED,
-+	DIBS_DEV_ERR_STATE,
-+	DIBS_OTHER_SUBTYPE
-+};
-+
-+struct dibs_event {
-+	u32 type;
-+	u32 subtype;
-+	/* uuid_null if invalid */
-+	uuid_t gid;
-+	/* zero if invalid */
-+	u64 buffer_tok;
-+	u64 time;
-+	/* additional data or zero */
-+	u64 data;
-+};
-+
-+struct dibs_dev;
-+
-+/* DIBS client
-+ * -----------
-+ */
-+#define MAX_DIBS_CLIENTS	8
-+#define NO_DIBS_CLIENT		0xff
-+/* All dibs clients have access to all dibs devices.
-+ * A dibs client provides the following functions to be called by dibs layer or
-+ * dibs device drivers:
-+ */
-+struct dibs_client_ops {
-+	/**
-+	 *  add_dev() - add a dibs device
-+	 *  @dev: device that was added
-+	 *
-+	 * Will be called during dibs_register_client() for all existing
-+	 * dibs devices and whenever a new dibs device is registered.
-+	 * dev is usable until dibs_client.remove() is called.
-+	 * *dev is protected by device refcounting.
-+	 */
-+	void (*add_dev)(struct dibs_dev *dev);
-+	/**
-+	 * del_dev() - remove a dibs device
-+	 * @dev: device to be removed
-+	 *
-+	 * Will be called whenever a dibs device is removed.
-+	 * Will be called during dibs_unregister_client() for all existing
-+	 * dibs devices and whenever a dibs device is unregistered.
-+	 * The device has already stopped initiative for this client:
-+	 * No new handlers will be started.
-+	 * The device is no longer usable by this client after this call.
-+	 */
-+	void (*del_dev)(struct dibs_dev *dev);
-+	/**
-+	 * handle_irq() - Handle signaling for a DMB
-+	 * @dev: device that owns the dmb
-+	 * @idx: Index of the dmb that got signalled
-+	 * @dmbemask: signaling mask of the dmb
-+	 *
-+	 * Handle signaling for a dmb that was registered by this client
-+	 * for this device.
-+	 * The dibs device can coalesce multiple signaling triggers into a
-+	 * single call of handle_irq(). dmbemask can be used to indicate
-+	 * different kinds of triggers.
-+	 *
-+	 * Context: Called in IRQ context by dibs device driver
-+	 */
-+	void (*handle_irq)(struct dibs_dev *dev, unsigned int idx,
-+			   u16 dmbemask);
-+	/**
-+	 * handle_event() - Handle control information sent by device
-+	 * @dev: device reporting the event
-+	 * @event: ism event structure
-+	 *
-+	 * * Context: Called in IRQ context by dibs device driver
-+	 */
-+	void (*handle_event)(struct dibs_dev *dev,
-+			     const struct dibs_event *event);
-+};
-+
-+struct dibs_client {
-+	/* client name for logging and debugging purposes */
-+	const char *name;
-+	const struct dibs_client_ops *ops;
-+	/* client index - provided and used by dibs layer */
-+	u8 id;
-+};
-+
-+/* Functions to be called by dibs clients:
-+ */
-+/**
-+ * dibs_register_client() - register a client with dibs layer
-+ * @client: this client
-+ *
-+ * Will call client->ops->add_dev() for all existing dibs devices.
-+ * Return: zero on success.
-+ */
-+int dibs_register_client(struct dibs_client *client);
-+/**
-+ * dibs_unregister_client() - unregister a client with dibs layer
-+ * @client: this client
-+ *
-+ * Will call client->ops->del_dev() for all existing dibs devices.
-+ * Return: zero on success.
-+ */
-+int dibs_unregister_client(struct dibs_client *client);
-+
-+/* dibs clients can call dibs device ops. */
-+
-+/* DIBS devices
-+ * ------------
-+ */
-+
-+/* Defined fabric id / CHID for all loopback devices:
-+ * All dibs loopback devices report this fabric id. In this case devices with
-+ * the same fabric id can NOT communicate via dibs. Only loopback devices with
-+ * the same dibs device gid can communicate (=same device with itself).
-+ */
-+#define DIBS_LOOPBACK_FABRIC	0xFFFF
-+
-+/* A dibs device provides the following functions to be called by dibs clients.
-+ * They are mandatory, unless marked 'optional'.
-+ */
-+struct dibs_dev_ops {
-+	/**
-+	 * get_fabric_id()
-+	 * @dev: local dibs device
-+	 *
-+	 * Only devices on the same dibs fabric can communicate. Fabric_id is
-+	 * unique inside the same HW system. Use fabric_id for fast negative
-+	 * checks, but only query_remote_gid() can give a reliable positive
-+	 * answer:
-+	 * Different fabric_id: dibs is not possible
-+	 * Same fabric_id: dibs may be possible or not
-+	 *		   (e.g. different HW systems)
-+	 * EXCEPTION: DIBS_LOOPBACK_FABRIC denotes an ism_loopback device
-+	 *	      that can only communicate with itself. Use dibs_dev.gid
-+	 *	      or query_remote_gid()to determine whether sender and
-+	 *	      receiver use the same ism_loopback device.
-+	 * Return: 2 byte dibs fabric id
-+	 */
-+	u16 (*get_fabric_id)(struct dibs_dev *dev);
-+	/**
-+	 * query_remote_gid()
-+	 * @dev: local dibs device
-+	 * @rgid: gid of remote dibs device
-+	 * @vid_valid: if zero, vid will be ignored;
-+	 *	       deprecated, ignored if device does not support vlan
-+	 * @vid: VLAN id; deprecated, ignored if device does not support vlan
-+	 *
-+	 * Query whether a remote dibs device is reachable via this local device
-+	 * and this vlan id.
-+	 * Return: 0 if remote gid is reachable.
-+	 */
-+	int (*query_remote_gid)(struct dibs_dev *dev, const uuid_t *rgid,
-+				u32 vid_valid, u32 vid);
-+	/**
-+	 * max_dmbs()
-+	 * Return: Max number of DMBs that can be registered for this kind of
-+	 *	   dibs_dev
-+	 */
-+	int (*max_dmbs)(void);
-+	/**
-+	 * register_dmb() - allocate and register a dmb
-+	 * @dev: dibs device
-+	 * @dmb: dmb struct to be registered
-+	 * @client: dibs client
-+	 * @vid: VLAN id; deprecated, ignored if device does not support vlan
-+	 *
-+	 * The following fields of dmb must provide valid input:
-+	 *	@rgid: gid of remote user device
-+	 *	@dmb_len: buffer length
-+	 *	@idx: Optionally:requested idx (if non-zero)
-+	 *	@vlan_valid: if zero, vlan_id will be ignored;
-+	 *		     deprecated, ignored if device does not support vlan
-+	 *	@vlan_id: deprecated, ignored if device does not support vlan
-+	 * Upon return in addition the following fields will be valid:
-+	 *	@dmb_tok: for usage by remote and local devices and clients
-+	 *	@cpu_addr: allocated buffer
-+	 *	@idx: dmb index, unique per dibs device
-+	 *	@dma_addr: to be used by device driver,if applicable
-+	 *
-+	 * Allocate a dmb buffer and register it with this device and for this
-+	 * client.
-+	 * Return: zero on success
-+	 */
-+	int (*register_dmb)(struct dibs_dev *dev, struct dibs_dmb *dmb,
-+			    struct dibs_client *client);
-+	/**
-+	 * unregister_dmb() - unregister and free a dmb
-+	 * @dev: dibs device
-+	 * @dmb: dmb struct to be unregistered
-+	 * The following fields of dmb must provide valid input:
-+	 *	@dmb_tok
-+	 *	@cpu_addr
-+	 *	@idx
-+	 *
-+	 * Free dmb.cpu_addr and unregister the dmb from this device.
-+	 * Return: zero on success
-+	 */
-+	int (*unregister_dmb)(struct dibs_dev *dev, struct dibs_dmb *dmb);
-+	/**
-+	 * move_data() - write into a remote dmb
-+	 * @dev: Local sending dibs device
-+	 * @dmb_tok: Token of the remote dmb
-+	 * @idx: signaling index in dmbemask
-+	 * @sf: signaling flag;
-+	 *      if true, idx will be turned on at target dmbemask mask
-+	 *      and target device will be signaled.
-+	 * @offset: offset within target dmb
-+	 * @data: pointer to data to be sent
-+	 * @size: length of data to be sent, can be zero.
-+	 *
-+	 * Use dev to write data of size at offset into a remote dmb
-+	 * identified by dmb_tok. Data is moved synchronously, *data can
-+	 * be freed when this function returns.
-+	 *
-+	 * If signaling flag (sf) is true, bit number idx bit will be turned
-+	 * on in the dmbemask mask when handle_irq() is called at the remote
-+	 * dibs client that owns the target dmb. The target device may chose
-+	 * to coalesce the signaling triggers of multiple move_data() calls
-+	 * to the same target dmb into a single handle_irq() call.
-+	 * Return: zero on success
-+	 */
-+	int (*move_data)(struct dibs_dev *dev, u64 dmb_tok, unsigned int idx,
-+			 bool sf, unsigned int offset, void *data,
-+			 unsigned int size);
-+	/**
-+	 * add_vlan_id() - add dibs device to vlan (optional, deprecated)
-+	 * @dev: dibs device
-+	 * @vlan_id: vlan id
-+	 *
-+	 * In order to write into a vlan-tagged dmb, the remote device needs
-+	 * to belong to the this vlan. A device can belong to more than 1 vlan.
-+	 * Any device can access an untagged dmb.
-+	 * Deprecated, only supported for backwards compatibility.
-+	 * Return: zero on success
-+	 */
-+	int (*add_vlan_id)(struct dibs_dev *dev, u64 vlan_id);
-+	/**
-+	 * del_vlan_id() - remove dibs device from vlan (optional, deprecated)
-+	 * @dev: dibs device
-+	 * @vlan_id: vlan id
-+	 * Return: zero on success
-+	 */
-+	int (*del_vlan_id)(struct dibs_dev *dev, u64 vlan_id);
-+	/**
-+	 * signal_event() - trigger an event at a remote dibs device (optional)
-+	 * @dev: local dibs device
-+	 * @rgid: gid of remote dibs device
-+	 * trigger_irq: zero: notification may be coalesced with other events
-+	 *		non-zero: notify immediately
-+	 * @subtype: 4 byte event code, meaning is defined by dibs client
-+	 * @data: 8 bytes of additional information,
-+	 *	  meaning is defined by dibs client
-+	 *
-+	 * dibs devices can offer support for sending a control event of type
-+	 * EVENT_SWR to a remote dibs device.
-+	 * NOTE: handle_event() will be called for all registered dibs clients
-+	 * at the remote device.
-+	 * Return: zero on success
-+	 */
-+	int (*signal_event)(struct dibs_dev *dev, const uuid_t *rgid,
-+			    u32 trigger_irq, u32 event_code, u64 info);
-+	/**
-+	 * support_mmapped_rdmb() - can this device provide memory mapped
-+	 *			    remote dmbs? (optional)
-+	 * @dev: dibs device
-+	 *
-+	 * A dibs device can provide a kernel address + length, that represent
-+	 * a remote target dmb (like MMIO). Alternatively to calling
-+	 * move_data(), a dibs client can write into such a ghost-send-buffer
-+	 * (= to this kernel address) and the data will automatically
-+	 * immediately appear in the target dmb, even without calling
-+	 * move_data().
-+	 *
-+	 * Either all 3 function pointers for support_dmb_nocopy(),
-+	 * attach_dmb() and detach_dmb() are defined, or all of them must
-+	 * be NULL.
-+	 *
-+	 * Return: non-zero, if memory mapped remote dmbs are supported.
-+	 */
-+	int (*support_mmapped_rdmb)(struct dibs_dev *dev);
-+	/**
-+	 * attach_dmb() - attach local memory to a remote dmb
-+	 * @dev: Local sending ism device
-+	 * @dmb: all other parameters are passed in the form of a
-+	 *	 dmb struct
-+	 *	 TODO: (THIS IS CONFUSING, should be changed)
-+	 *  dmb_tok: (in) Token of the remote dmb, we want to attach to
-+	 *  cpu_addr: (out) MMIO address
-+	 *  dma_addr: (out) MMIO address (if applicable, invalid otherwise)
-+	 *  dmb_len: (out) length of local MMIO region,
-+	 *           equal to length of remote DMB.
-+	 *  sba_idx: (out) index of remote dmb (NOT HELPFUL, should be removed)
-+	 *
-+	 * Provides a memory address to the sender that can be used to
-+	 * directly write into the remote dmb.
-+	 * Memory is available until detach_dmb is called
-+	 *
-+	 * Return: Zero upon success, Error code otherwise
-+	 */
-+	int (*attach_dmb)(struct dibs_dev *dev, struct dibs_dmb *dmb);
-+	/**
-+	 * detach_dmb() - Detach the ghost buffer from a remote dmb
-+	 * @dev: ism device
-+	 * @token: dmb token of the remote dmb
-+	 *
-+	 * No need to free cpu_addr.
-+	 *
-+	 * Return: Zero upon success, Error code otherwise
-+	 */
-+	int (*detach_dmb)(struct dibs_dev *dev, u64 token);
-+};
-+
-+struct dibs_dev {
-+	struct list_head list;
-+	struct device dev;
-+	/* To be filled by device driver, before calling dibs_dev_add(): */
-+	const struct dibs_dev_ops *ops;
-+	uuid_t gid;
-+	/* priv pointer for device driver */
-+	void *drv_priv;
-+
-+	/* priv pointer per client; for client usage only */
-+	void *priv[MAX_DIBS_CLIENTS];
-+
-+	/* get this lock before accessing any of the fields below */
-+	spinlock_t lock;
-+	/* array of client ids indexed by dmb idx;
-+	 * can be used as indices into priv and subs arrays
-+	 */
-+	u8 *dmb_clientid_arr;
-+	/* Sparse array of all ISM clients */
-+	struct dibs_client *subs[MAX_DIBS_CLIENTS];
-+};
-+
-+static inline void dibs_set_priv(struct dibs_dev *dev,
-+				 struct dibs_client *client, void *priv)
-+{
-+	dev->priv[client->id] = priv;
-+}
-+
-+static inline void *dibs_get_priv(struct dibs_dev *dev,
-+				  struct dibs_client *client)
-+{
-+	return dev->priv[client->id];
-+}
-+
-+/* ------- End of client-only functions ----------- */
-+
-+/* Functions to be called by dibs device drivers:
-+ */
-+/**
-+ * dibs_dev_alloc() - allocate and reference device structure
-+ *
-+ * The following fields will be valid upon successful return: dev
-+ * NOTE: Use put_device(dibs_get_dev(@dibs)) to give up your reference instead
-+ * of freeing @dibs @dev directly once you have successfully called this
-+ * function.
-+ * Return: Pointer to dibs device structure
-+ */
-+struct dibs_dev *dibs_dev_alloc(void);
-+/**
-+ * dibs_dev_add() - register with dibs layer and all clients
-+ * @dibs: dibs device
-+ *
-+ * The following fields must be valid upon entry: dev, ops, drv_priv
-+ * All fields will be valid upon successful return.
-+ * Return: zero on success
-+ */
-+int dibs_dev_add(struct dibs_dev *dibs);
-+/**
-+ * dibs_dev_del() - unregister from dibs layer and all clients
-+ * @dibs: dibs device
-+ */
-+void dibs_dev_del(struct dibs_dev *dibs);
-+
-+#endif	/* _DIBS_H */
-diff --git a/include/linux/dim.h b/include/linux/dim.h
-index 6c5733981563..06543fd40fcc 100644
--- a/include/linux/dim.h
-+++ b/include/linux/dim.h
-@@ -10,6 +10,15 @@
- #include <linux/types.h>
- #include <linux/workqueue.h>
- 
-+struct net_device;
-+
-+/* Number of DIM profiles and period mode. */
-+#define NET_DIM_PARAMS_NUM_PROFILES 5
-+#define NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE 256
-+#define NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE 128
-+#define NET_DIM_DEF_PROFILE_CQE 1
-+#define NET_DIM_DEF_PROFILE_EQE 1
-+
- /*
-  * Number of events between DIM iterations.
-  * Causes a moderation of the algorithm run.
-@@ -38,12 +47,45 @@
-  * @pkts: CQ packet counter suggestion (by DIM)
-  * @comps: Completion counter
-  * @cq_period_mode: CQ period count mode (from CQE/EQE)
-+ * @rcu: for asynchronous kfree_rcu
-  */
- struct dim_cq_moder {
- 	u16 usec;
- 	u16 pkts;
- 	u16 comps;
- 	u8 cq_period_mode;
-+	struct rcu_head rcu;
-+};
-+
-+#define DIM_PROFILE_RX		BIT(0)	/* support rx profile modification */
-+#define DIM_PROFILE_TX		BIT(1)	/* support tx profile modification */
-+
-+#define DIM_COALESCE_USEC	BIT(0)	/* support usec field modification */
-+#define DIM_COALESCE_PKTS	BIT(1)	/* support pkts field modification */
-+#define DIM_COALESCE_COMPS	BIT(2)	/* support comps field modification */
-+
-+/**
-+ * struct dim_irq_moder - Structure for irq moderation information.
-+ * Used to collect irq moderation related information.
-+ *
-+ * @profile_flags: DIM_PROFILE_*
-+ * @coal_flags: DIM_COALESCE_* for Rx and Tx
-+ * @dim_rx_mode: Rx DIM period count mode: CQE or EQE
-+ * @dim_tx_mode: Tx DIM period count mode: CQE or EQE
-+ * @rx_profile: DIM profile list for Rx
-+ * @tx_profile: DIM profile list for Tx
-+ * @rx_dim_work: Rx DIM worker scheduled by net_dim()
-+ * @tx_dim_work: Tx DIM worker scheduled by net_dim()
-+ */
-+struct dim_irq_moder {
-+	u8 profile_flags;
-+	u8 coal_flags;
-+	u8 dim_rx_mode;
-+	u8 dim_tx_mode;
-+	struct dim_cq_moder __rcu *rx_profile;
-+	struct dim_cq_moder __rcu *tx_profile;
-+	void (*rx_dim_work)(struct work_struct *work);
-+	void (*tx_dim_work)(struct work_struct *work);
- };
- 
- /**
-@@ -192,6 +234,77 @@ enum dim_step_result {
- };
- 
- /**
-+ * net_dim_init_irq_moder - collect information to initialize irq moderation
-+ * @dev: target network device
-+ * @profile_flags: Rx or Tx profile modification capability
-+ * @coal_flags: irq moderation params flags
-+ * @rx_mode: CQ period mode for Rx
-+ * @tx_mode: CQ period mode for Tx
-+ * @rx_dim_work: Rx worker called after dim decision
-+ * @tx_dim_work: Tx worker called after dim decision
-+ *
-+ * Return: 0 on success or a negative error code.
-+ */
-+int net_dim_init_irq_moder(struct net_device *dev, u8 profile_flags,
-+			   u8 coal_flags, u8 rx_mode, u8 tx_mode,
-+			   void (*rx_dim_work)(struct work_struct *work),
-+			   void (*tx_dim_work)(struct work_struct *work));
-+
-+/**
-+ * net_dim_free_irq_moder - free fields for irq moderation
-+ * @dev: target network device
-+ */
-+void net_dim_free_irq_moder(struct net_device *dev);
-+
-+/**
-+ * net_dim_setting - initialize DIM's cq mode and schedule worker
-+ * @dev: target network device
-+ * @dim: DIM context
-+ * @is_tx: true indicates the tx direction, false indicates the rx direction
-+ */
-+void net_dim_setting(struct net_device *dev, struct dim *dim, bool is_tx);
-+
-+/**
-+ * net_dim_work_cancel - synchronously cancel dim's worker
-+ * @dim: DIM context
-+ */
-+void net_dim_work_cancel(struct dim *dim);
-+
-+/**
-+ * net_dim_get_rx_irq_moder - get DIM rx results based on profile_ix
-+ * @dev: target network device
-+ * @dim: DIM context
-+ *
-+ * Return: DIM irq moderation
-+ */
-+struct dim_cq_moder
-+net_dim_get_rx_irq_moder(struct net_device *dev, struct dim *dim);
-+
-+/**
-+ * net_dim_get_tx_irq_moder - get DIM tx results based on profile_ix
-+ * @dev: target network device
-+ * @dim: DIM context
-+ *
-+ * Return: DIM irq moderation
-+ */
-+struct dim_cq_moder
-+net_dim_get_tx_irq_moder(struct net_device *dev, struct dim *dim);
-+
-+/**
-+ * net_dim_set_rx_mode - set DIM rx cq mode
-+ * @dev: target network device
-+ * @rx_mode: target rx cq mode
-+ */
-+void net_dim_set_rx_mode(struct net_device *dev, u8 rx_mode);
-+
-+/**
-+ * net_dim_set_tx_mode - set DIM tx cq mode
-+ * @dev: target network device
-+ * @tx_mode: target tx cq mode
-+ */
-+void net_dim_set_tx_mode(struct net_device *dev, u8 tx_mode);
-+
-+/**
-  *	dim_on_top - check if current state is a good place to stop (top location)
-  *	@dim: DIM context
-  *
-@@ -236,8 +349,10 @@ void dim_park_tired(struct dim *dim);
-  *
-  * Calculate the delta between two samples (in data rates).
-  * Takes into consideration counter wrap-around.
-+ * Returned boolean indicates whether curr_stats are reliable.
-  */
-void dim_calc_stats(struct dim_sample *start, struct dim_sample *end,
-+bool dim_calc_stats(const struct dim_sample *start,
-+		    const struct dim_sample *end,
- 		    struct dim_stats *curr_stats);
- 
- /**
-@@ -310,7 +425,7 @@ struct dim_cq_moder net_dim_get_def_tx_moderation(u8 cq_period_mode);
-  * This is the main logic of the algorithm, where data is processed in order
-  * to decide on next required action.
-  */
-void net_dim(struct dim *dim, struct dim_sample end_sample);
-+void net_dim(struct dim *dim, const struct dim_sample *end_sample);
- 
- /* RDMA DIM */
- 
-diff --git a/include/linux/dio.h b/include/linux/dio.h
-index 5abd07361eb5..464331c4c4a7 100644
--- a/include/linux/dio.h
-+++ b/include/linux/dio.h
-@@ -68,7 +68,7 @@ struct dio_bus {
- };
- 
- extern struct dio_bus dio_bus;      /* Single DIO bus */
-extern struct bus_type dio_bus_type;
-+extern const struct bus_type dio_bus_type;
- 
-     /*
-      *  DIO device IDs
-@@ -93,7 +93,7 @@ struct dio_driver {
- 	struct device_driver driver;
- };
- 
-#define to_dio_driver(drv)    container_of(drv, struct dio_driver, driver)
-+#define to_dio_driver(drv)    container_of_const(drv, struct dio_driver, driver)
- 
- /* DIO/DIO-II boards all have the following 8bit registers.
-  * These are offsets from the base of the device.
-diff --git a/include/linux/dlm.h b/include/linux/dlm.h
-index ff951e9f6f20..7e7b45b0d097 100644
--- a/include/linux/dlm.h
-+++ b/include/linux/dlm.h
-@@ -35,6 +35,9 @@ struct dlm_lockspace_ops {
- 			      int num_slots, int our_slot, uint32_t generation);
- };
- 
-+/* only relevant for kernel lockspaces, will be removed in future */
-+#define DLM_LSFL_SOFTIRQ __DLM_LSFL_RESERVED0
-+
- /*
-  * dlm_new_lockspace
-  *
-@@ -53,14 +56,13 @@ struct dlm_lockspace_ops {
-  *   The dlm should not use a resource directory, but statically assign
-  *   resource mastery to nodes based on the name hash that is otherwise
-  *   used to select the directory node.  Must be the same on all nodes.
- * DLM_LSFL_TIMEWARN
- *   The dlm should emit netlink messages if locks have been waiting
- *   for a configurable amount of time.  (Unused.)
- * DLM_LSFL_FS
- *   The lockspace user is in the kernel (i.e. filesystem).  Enables
- *   direct bast/cast callbacks.
-  * DLM_LSFL_NEWEXCL
-  *   dlm_new_lockspace() should return -EEXIST if the lockspace exists.
-+ * DLM_LSFL_SOFTIRQ
-+ *   dlm request callbacks (ast, bast) are softirq safe. Flag should be
-+ *   preferred by users. Will be default in some future. If set the
-+ *   strongest context for ast, bast callback is softirq as it avoids
-+ *   an additional context switch.
-  *
-  * lvblen: length of lvb in bytes.  Must be multiple of 8.
-  *   dlm_new_lockspace() returns an error if this does not match
-@@ -86,12 +88,43 @@ int dlm_new_lockspace(const char *name, const char *cluster,
- 		      int *ops_result, dlm_lockspace_t **lockspace);
- 
- /*
-+ * dlm_release_lockspace() release_option values:
-+ *
-+ * DLM_RELEASE_NO_LOCKS returns -EBUSY if any locks (lkb's)
-+ *   exist in the local lockspace.
-+ *
-+ * DLM_RELEASE_UNUSED previous value that is no longer used.
-+ *
-+ * DLM_RELEASE_NORMAL releases the lockspace regardless of any
-+ *   locks managed in the local lockspace.
-+ *
-+ * DLM_RELEASE_NO_EVENT release the lockspace regardless of any
-+ *   locks managed in the local lockspace, and does not submit
-+ *   a leave event to the cluster manager, so other nodes will
-+ *   not be notified that the node should be removed from the
-+ *   list of lockspace members.
-+ *
-+ * DLM_RELEASE_RECOVER like DLM_RELEASE_NORMAL, but the remaining
-+ *   nodes will handle the removal of the node as if the node
-+ *   had failed, e.g. the recover_slot() callback would be used.
-+ */
-+#define DLM_RELEASE_NO_LOCKS		0
-+#define DLM_RELEASE_UNUSED		1
-+#define DLM_RELEASE_NORMAL		2
-+#define DLM_RELEASE_NO_EVENT		3
-+#define DLM_RELEASE_RECOVER		4
-+#define __DLM_RELEASE_MAX		DLM_RELEASE_RECOVER
-+
-+/*
-  * dlm_release_lockspace
-  *
-  * Stop a lockspace.
-+ *
-+ * release_option: see DLM_RELEASE values above.
-  */
- 
-int dlm_release_lockspace(dlm_lockspace_t *lockspace, int force);
-+int dlm_release_lockspace(dlm_lockspace_t *lockspace,
-+			  unsigned int release_option);
- 
- /*
-  * dlm_lock
-@@ -127,14 +160,21 @@ int dlm_release_lockspace(dlm_lockspace_t *lockspace, int force);
-  * call.
-  *
-  * AST routines should not block (at least not for long), but may make
- * any locking calls they please.
-+ * any locking calls they please. If DLM_LSFL_SOFTIRQ for kernel
-+ * users of dlm_new_lockspace() is passed the ast and bast callbacks
-+ * can be processed in softirq context. Also some of the callback
-+ * contexts are in the same context as the DLM lock request API, users
-+ * must not hold locks while calling dlm lock request API and trying
-+ * to acquire this lock in the callback again, this will end in a
-+ * lock recursion. For newer implementation the DLM_LSFL_SOFTIRQ
-+ * should be used.
-  */
- 
- int dlm_lock(dlm_lockspace_t *lockspace,
- 	     int mode,
- 	     struct dlm_lksb *lksb,
- 	     uint32_t flags,
-	     void *name,
-+	     const void *name,
- 	     unsigned int namelen,
- 	     uint32_t parent_lkid,
- 	     void (*lockast) (void *astarg),
-diff --git a/include/linux/dlm_plock.h b/include/linux/dlm_plock.h
-index e6d76e8715a6..15fc856d198c 100644
--- a/include/linux/dlm_plock.h
-+++ b/include/linux/dlm_plock.h
-@@ -11,6 +11,8 @@ int dlm_posix_lock(dlm_lockspace_t *lockspace, u64 number, struct file *file,
- 		int cmd, struct file_lock *fl);
- int dlm_posix_unlock(dlm_lockspace_t *lockspace, u64 number, struct file *file,
- 		struct file_lock *fl);
-+int dlm_posix_cancel(dlm_lockspace_t *lockspace, u64 number, struct file *file,
-+		     struct file_lock *fl);
- int dlm_posix_get(dlm_lockspace_t *lockspace, u64 number, struct file *file,
- 		struct file_lock *fl);
- #endif
-diff --git a/include/linux/dm-bufio.h b/include/linux/dm-bufio.h
-index 15d9e15ca830..d1503b815a78 100644
--- a/include/linux/dm-bufio.h
-+++ b/include/linux/dm-bufio.h
-@@ -1,3 +1,4 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
- /*
-  * Copyright (C) 2009-2011 Red Hat, Inc.
-  *
-@@ -26,8 +27,8 @@ struct dm_buffer;
-  * Create a buffered IO cache on a given device
-  */
- struct dm_bufio_client *
-dm_bufio_client_create(struct block_device *bdev, unsigned block_size,
-		       unsigned reserved_buffers, unsigned aux_size,
-+dm_bufio_client_create(struct block_device *bdev, unsigned int block_size,
-+		       unsigned int reserved_buffers, unsigned int aux_size,
- 		       void (*alloc_callback)(struct dm_buffer *),
- 		       void (*write_callback)(struct dm_buffer *),
- 		       unsigned int flags);
-@@ -37,6 +38,8 @@ dm_bufio_client_create(struct block_device *bdev, unsigned block_size,
-  */
- void dm_bufio_client_destroy(struct dm_bufio_client *c);
- 
-+void dm_bufio_client_reset(struct dm_bufio_client *c);
-+
- /*
-  * Set the sector range.
-  * When this function is called, there must be no I/O in progress on the bufio
-@@ -61,6 +64,9 @@ void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start);
- void *dm_bufio_read(struct dm_bufio_client *c, sector_t block,
- 		    struct dm_buffer **bp);
- 
-+void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block,
-+				struct dm_buffer **bp, unsigned short ioprio);
-+
- /*
-  * Like dm_bufio_read, but return buffer from cache, don't read
-  * it. If the buffer is not in the cache, return NULL.
-@@ -81,7 +87,11 @@ void *dm_bufio_new(struct dm_bufio_client *c, sector_t block,
-  * I/O to finish.
-  */
- void dm_bufio_prefetch(struct dm_bufio_client *c,
-		       sector_t block, unsigned n_blocks);
-+		       sector_t block, unsigned int n_blocks);
-+
-+void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c,
-+				sector_t block, unsigned int n_blocks,
-+				unsigned short ioprio);
- 
- /*
-  * Release a reference obtained with dm_bufio_{read,get,new}. The data
-@@ -106,7 +116,7 @@ void dm_bufio_mark_buffer_dirty(struct dm_buffer *b);
-  * write the specified part of the buffer or it may write a larger superset.
-  */
- void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b,
-					unsigned start, unsigned end);
-+					unsigned int start, unsigned int end);
- 
- /*
-  * Initiate writing of dirty buffers, without waiting for completion.
-@@ -130,12 +140,6 @@ int dm_bufio_issue_flush(struct dm_bufio_client *c);
- int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t count);
- 
- /*
- * Like dm_bufio_release but also move the buffer to the new
- * block. dm_bufio_write_dirty_buffers is needed to commit the new block.
- */
-void dm_bufio_release_move(struct dm_buffer *b, sector_t new_block);
-
-/*
-  * Free the given buffer.
-  * This is just a hint, if the buffer is in use or dirty, this function
-  * does nothing.
-@@ -152,9 +156,9 @@ void dm_bufio_forget_buffers(struct dm_bufio_client *c, sector_t block, sector_t
- /*
-  * Set the minimum number of buffers before cleanup happens.
-  */
-void dm_bufio_set_minimum_buffers(struct dm_bufio_client *c, unsigned n);
-+void dm_bufio_set_minimum_buffers(struct dm_bufio_client *c, unsigned int n);
- 
-unsigned dm_bufio_get_block_size(struct dm_bufio_client *c);
-+unsigned int dm_bufio_get_block_size(struct dm_bufio_client *c);
- sector_t dm_bufio_get_device_size(struct dm_bufio_client *c);
- struct dm_io_client *dm_bufio_get_dm_io_client(struct dm_bufio_client *c);
- sector_t dm_bufio_get_block_number(struct dm_buffer *b);
-diff --git a/include/linux/dm-dirty-log.h b/include/linux/dm-dirty-log.h
-index 7084503c3405..0b10faedb26a 100644
--- a/include/linux/dm-dirty-log.h
-+++ b/include/linux/dm-dirty-log.h
-@@ -1,3 +1,4 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
- /*
-  * Copyright (C) 2003 Sistina Software
-  * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
-@@ -33,7 +34,7 @@ struct dm_dirty_log_type {
- 	struct list_head list;
- 
- 	int (*ctr)(struct dm_dirty_log *log, struct dm_target *ti,
-		   unsigned argc, char **argv);
-+		   unsigned int argc, char **argv);
- 	void (*dtr)(struct dm_dirty_log *log);
- 
- 	/*
-@@ -96,7 +97,7 @@ struct dm_dirty_log_type {
- 	 * Do not confuse this function with 'in_sync()', one
- 	 * tells you if an area is synchronised, the other
- 	 * assigns recovery work.
-	*/
-+	 */
- 	int (*get_resync_work)(struct dm_dirty_log *log, region_t *region);
- 
- 	/*
-@@ -116,7 +117,7 @@ struct dm_dirty_log_type {
- 	 * Support function for mirror status requests.
- 	 */
- 	int (*status)(struct dm_dirty_log *log, status_type_t status_type,
-		      char *result, unsigned maxlen);
-+		      char *result, unsigned int maxlen);
- 
- 	/*
- 	 * is_remote_recovering is necessary for cluster mirroring. It provides
-@@ -139,7 +140,7 @@ int dm_dirty_log_type_unregister(struct dm_dirty_log_type *type);
- struct dm_dirty_log *dm_dirty_log_create(const char *type_name,
- 			struct dm_target *ti,
- 			int (*flush_callback_fn)(struct dm_target *ti),
-			unsigned argc, char **argv);
-+			unsigned int argc, char **argv);
- void dm_dirty_log_destroy(struct dm_dirty_log *log);
- 
- #endif	/* __KERNEL__ */
-diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h
-index 8e1c4ab5df04..7b2968612b7e 100644
--- a/include/linux/dm-io.h
-+++ b/include/linux/dm-io.h
-@@ -1,3 +1,4 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
- /*
-  * Copyright (C) 2003 Sistina Software
-  * Copyright (C) 2004 - 2008 Red Hat, Inc. All rights reserved.
-@@ -26,7 +27,7 @@ struct page_list {
- 	struct page *page;
- };
- 
-typedef void (*io_notify_fn)(unsigned long error, void *context);
-+typedef void (*io_notify_fn)(unsigned int long error, void *context);
- 
- enum dm_io_mem_type {
- 	DM_IO_PAGE_LIST,/* Page list */
-@@ -38,7 +39,7 @@ enum dm_io_mem_type {
- struct dm_io_memory {
- 	enum dm_io_mem_type type;
- 
-	unsigned offset;
-+	unsigned int offset;
- 
- 	union {
- 		struct page_list *pl;
-@@ -78,8 +79,9 @@ void dm_io_client_destroy(struct dm_io_client *client);
-  * Each bit in the optional 'sync_error_bits' bitset indicates whether an
-  * error occurred doing io to the corresponding region.
-  */
-int dm_io(struct dm_io_request *io_req, unsigned num_regions,
-	  struct dm_io_region *region, unsigned long *sync_error_bits);
-+int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
-+	  struct dm_io_region *region, unsigned int long *sync_error_bits,
-+	  unsigned short ioprio);
- 
- #endif	/* __KERNEL__ */
- #endif	/* _LINUX_DM_IO_H */
-diff --git a/include/linux/dm-kcopyd.h b/include/linux/dm-kcopyd.h
-index c1707ee5b540..51fb1af0b63e 100644
--- a/include/linux/dm-kcopyd.h
-+++ b/include/linux/dm-kcopyd.h
-@@ -1,3 +1,4 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
- /*
-  * Copyright (C) 2001 - 2003 Sistina Software
-  * Copyright (C) 2004 - 2008 Red Hat, Inc. All rights reserved.
-@@ -23,11 +24,11 @@
- #define DM_KCOPYD_WRITE_SEQ    2
- 
- struct dm_kcopyd_throttle {
-	unsigned throttle;
-	unsigned num_io_jobs;
-	unsigned io_period;
-	unsigned total_period;
-	unsigned last_jiffies;
-+	unsigned int throttle;
-+	unsigned int num_io_jobs;
-+	unsigned int io_period;
-+	unsigned int total_period;
-+	unsigned int last_jiffies;
- };
- 
- /*
-@@ -60,12 +61,12 @@ void dm_kcopyd_client_flush(struct dm_kcopyd_client *kc);
-  * read_err is a boolean,
-  * write_err is a bitset, with 1 bit for each destination region
-  */
-typedef void (*dm_kcopyd_notify_fn)(int read_err, unsigned long write_err,
-+typedef void (*dm_kcopyd_notify_fn)(int read_err, unsigned int long write_err,
- 				    void *context);
- 
- void dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,
-		    unsigned num_dests, struct dm_io_region *dests,
-		    unsigned flags, dm_kcopyd_notify_fn fn, void *context);
-+		    unsigned int num_dests, struct dm_io_region *dests,
-+		    unsigned int flags, dm_kcopyd_notify_fn fn, void *context);
- 
- /*
-  * Prepare a callback and submit it via the kcopyd thread.
-@@ -80,11 +81,11 @@ void dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,
-  */
- void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc,
- 				 dm_kcopyd_notify_fn fn, void *context);
-void dm_kcopyd_do_callback(void *job, int read_err, unsigned long write_err);
-+void dm_kcopyd_do_callback(void *job, int read_err, unsigned int long write_err);
- 
- void dm_kcopyd_zero(struct dm_kcopyd_client *kc,
-		    unsigned num_dests, struct dm_io_region *dests,
-		    unsigned flags, dm_kcopyd_notify_fn fn, void *context);
-+		    unsigned int num_dests, struct dm_io_region *dests,
-+		    unsigned int flags, dm_kcopyd_notify_fn fn, void *context);
- 
- #endif	/* __KERNEL__ */
- #endif	/* _LINUX_DM_KCOPYD_H */
-diff --git a/include/linux/dm-region-hash.h b/include/linux/dm-region-hash.h
-index 9e2a7a401df5..3079ed93dd2d 100644
--- a/include/linux/dm-region-hash.h
-+++ b/include/linux/dm-region-hash.h
-@@ -1,3 +1,4 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
- /*
-  * Copyright (C) 2003 Sistina Software Limited.
-  * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
-@@ -12,9 +13,11 @@
- 
- #include <linux/dm-dirty-log.h>
- 
-/*-----------------------------------------------------------------
-+/*
-+ *----------------------------------------------------------------
-  * Region hash
- *----------------------------------------------------------------*/
-+ *----------------------------------------------------------------
-+ */
- struct dm_region_hash;
- struct dm_region;
- 
-@@ -37,7 +40,7 @@ struct dm_region_hash *dm_region_hash_create(
- 						     struct bio_list *bios),
- 		void (*wakeup_workers)(void *context),
- 		void (*wakeup_all_recovery_waiters)(void *context),
-		sector_t target_begin, unsigned max_recovery,
-+		sector_t target_begin, unsigned int max_recovery,
- 		struct dm_dirty_log *log, uint32_t region_size,
- 		region_t nr_regions);
- void dm_region_hash_destroy(struct dm_region_hash *rh);
-diff --git a/include/linux/dm-verity-loadpin.h b/include/linux/dm-verity-loadpin.h
-index 552b817ab102..3ac6dbaeaa37 100644
--- a/include/linux/dm-verity-loadpin.h
-+++ b/include/linux/dm-verity-loadpin.h
-@@ -12,7 +12,7 @@ extern struct list_head dm_verity_loadpin_trusted_root_digests;
- struct dm_verity_loadpin_trusted_root_digest {
- 	struct list_head node;
- 	unsigned int len;
-	u8 data[];
-+	u8 data[] __counted_by(len);
- };
- 
- #if IS_ENABLED(CONFIG_SECURITY_LOADPIN_VERITY)
-diff --git a/include/linux/dma-buf-mapping.h b/include/linux/dma-buf-mapping.h
-new file mode 100644
-index 000000000000..a3c0ce2d3a42
--- /dev/null
-+++ b/include/linux/dma-buf-mapping.h
-@@ -0,0 +1,17 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * DMA BUF Mapping Helpers
-+ *
-+ */
-+#ifndef __DMA_BUF_MAPPING_H__
-+#define __DMA_BUF_MAPPING_H__
-+#include <linux/dma-buf.h>
-+
-+struct sg_table *dma_buf_phys_vec_to_sgt(struct dma_buf_attachment *attach,
-+					 struct p2pdma_provider *provider,
-+					 struct dma_buf_phys_vec *phys_vec,
-+					 size_t nr_ranges, size_t size,
-+					 enum dma_data_direction dir);
-+void dma_buf_free_sgt(struct dma_buf_attachment *attach, struct sg_table *sgt,
-+		      enum dma_data_direction dir);
-+#endif
-diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
-index 71731796c8c3..0bc492090237 100644
--- a/include/linux/dma-buf.h
-+++ b/include/linux/dma-buf.h
-@@ -22,6 +22,7 @@
- #include <linux/fs.h>
- #include <linux/dma-fence.h>
- #include <linux/wait.h>
-+#include <linux/pci-p2pdma.h>
- 
- struct device;
- struct dma_buf;
-@@ -35,15 +36,6 @@ struct dma_buf_attachment;
-  */
- struct dma_buf_ops {
- 	/**
-	  * @cache_sgt_mapping:
-	  *
-	  * If true the framework will cache the first mapping made for each
-	  * attachment. This avoids creating mappings for attachments multiple
-	  * times.
-	  */
-	bool cache_sgt_mapping;
-
-	/**
- 	 * @attach:
- 	 *
- 	 * This is called from dma_buf_attach() to make sure that a given
-@@ -327,15 +319,6 @@ struct dma_buf {
- 	const struct dma_buf_ops *ops;
- 
- 	/**
-	 * @lock:
-	 *
-	 * Used internally to serialize list manipulation, attach/detach and
-	 * vmap/unmap. Note that in many cases this is superseeded by
-	 * dma_resv_lock() on @resv.
-	 */
-	struct mutex lock;
-
-	/**
- 	 * @vmapping_counter:
- 	 *
- 	 * Used internally to refcnt the vmaps returned by dma_buf_vmap().
-@@ -352,20 +335,23 @@ struct dma_buf {
- 	/**
- 	 * @exp_name:
- 	 *
-	 * Name of the exporter; useful for debugging. See the
-	 * DMA_BUF_SET_NAME IOCTL.
-+	 * Name of the exporter; useful for debugging. Must not be NULL
- 	 */
- 	const char *exp_name;
- 
- 	/**
- 	 * @name:
- 	 *
-	 * Userspace-provided name; useful for accounting and debugging,
-	 * protected by dma_resv_lock() on @resv and @name_lock for read access.
-+	 * Userspace-provided name. Default value is NULL. If not NULL,
-+	 * length cannot be longer than DMA_BUF_NAME_LEN, including NIL
-+	 * char. Useful for accounting and debugging. Read/Write accesses
-+	 * are protected by @name_lock
-+	 *
-+	 * See the IOCTLs DMA_BUF_SET_NAME or DMA_BUF_SET_NAME_A/B
- 	 */
- 	const char *name;
- 
-	/** @name_lock: Spinlock to protect name acces for read access. */
-+	/** @name_lock: Spinlock to protect name access for read access. */
- 	spinlock_t name_lock;
- 
- 	/**
-@@ -402,7 +388,7 @@ struct dma_buf {
- 	 *   anything the userspace API considers write access.
- 	 *
- 	 * - Drivers may just always add a write fence, since that only
-	 *   causes unecessarily synchronization, but no correctness issues.
-+	 *   causes unnecessary synchronization, but no correctness issues.
- 	 *
- 	 * - Some drivers only expose a synchronous userspace API with no
- 	 *   pipelining across drivers. These do not set any fences for their
-@@ -497,8 +483,6 @@ struct dma_buf_attach_ops {
-  * @dmabuf: buffer for this attachment.
-  * @dev: device attached to the buffer.
-  * @node: list of dma_buf_attachment, protected by dma_resv lock of the dmabuf.
- * @sgt: cached mapping.
- * @dir: direction of cached mapping.
-  * @peer2peer: true if the importer can handle peer resources without pages.
-  * @priv: exporter specific attachment data.
-  * @importer_ops: importer operations for this attachment, if provided
-@@ -518,8 +502,6 @@ struct dma_buf_attachment {
- 	struct dma_buf *dmabuf;
- 	struct device *dev;
- 	struct list_head node;
-	struct sg_table *sgt;
-	enum dma_data_direction dir;
- 	bool peer2peer;
- 	const struct dma_buf_attach_ops *importer_ops;
- 	void *importer_priv;
-@@ -550,6 +532,16 @@ struct dma_buf_export_info {
- };
- 
- /**
-+ * struct dma_buf_phys_vec - describe continuous chunk of memory
-+ * @paddr:   physical address of that chunk
-+ * @len:     Length of this chunk
-+ */
-+struct dma_buf_phys_vec {
-+	phys_addr_t paddr;
-+	size_t len;
-+};
-+
-+/**
-  * DEFINE_DMA_BUF_EXPORT_INFO - helper macro for exporters
-  * @name: export-info name
-  *
-@@ -587,20 +579,6 @@ static inline bool dma_buf_is_dynamic(struct dma_buf *dmabuf)
- 	return !!dmabuf->ops->pin;
- }
- 
-/**
- * dma_buf_attachment_is_dynamic - check if a DMA-buf attachment uses dynamic
- * mappings
- * @attach: the DMA-buf attachment to check
- *
- * Returns true if a DMA-buf importer wants to call the map/unmap functions with
- * the dma_resv lock held.
- */
-static inline bool
-dma_buf_attachment_is_dynamic(struct dma_buf_attachment *attach)
-{
-	return !!attach->importer_ops;
-}
-
- struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf,
- 					  struct device *dev);
- struct dma_buf_attachment *
-@@ -627,9 +605,19 @@ int dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
- 			     enum dma_data_direction dir);
- int dma_buf_end_cpu_access(struct dma_buf *dma_buf,
- 			   enum dma_data_direction dir);
-+struct sg_table *
-+dma_buf_map_attachment_unlocked(struct dma_buf_attachment *attach,
-+				enum dma_data_direction direction);
-+void dma_buf_unmap_attachment_unlocked(struct dma_buf_attachment *attach,
-+				       struct sg_table *sg_table,
-+				       enum dma_data_direction direction);
- 
- int dma_buf_mmap(struct dma_buf *, struct vm_area_struct *,
- 		 unsigned long);
- int dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map);
- void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map);
-+int dma_buf_vmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map);
-+void dma_buf_vunmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map);
-+struct dma_buf *dma_buf_iter_begin(void);
-+struct dma_buf *dma_buf_iter_next(struct dma_buf *dmbuf);
- #endif /* __DMA_BUF_H__ */
-diff --git a/include/linux/dma-buf/heaps/cma.h b/include/linux/dma-buf/heaps/cma.h
-new file mode 100644
-index 000000000000..e751479e21e7
--- /dev/null
-+++ b/include/linux/dma-buf/heaps/cma.h
-@@ -0,0 +1,16 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef DMA_BUF_HEAP_CMA_H_
-+#define DMA_BUF_HEAP_CMA_H_
-+
-+struct cma;
-+
-+#ifdef CONFIG_DMABUF_HEAPS_CMA
-+int dma_heap_cma_register_heap(struct cma *cma);
-+#else
-+static inline int dma_heap_cma_register_heap(struct cma *cma)
-+{
-+	return 0;
-+}
-+#endif // CONFIG_DMABUF_HEAPS_CMA
-+
-+#endif // DMA_BUF_HEAP_CMA_H_
-diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
-index 18aade195884..c249912456f9 100644
--- a/include/linux/dma-direct.h
-+++ b/include/linux/dma-direct.h
-@@ -12,7 +12,7 @@
- #include <linux/mem_encrypt.h>
- #include <linux/swiotlb.h>
- 
-extern unsigned int zone_dma_bits;
-+extern u64 zone_dma_limit;
- 
- /*
-  * Record the mapping of CPU physical to DMA addresses for a given region.
-@@ -21,7 +21,6 @@ struct bus_dma_region {
- 	phys_addr_t	cpu_start;
- 	dma_addr_t	dma_start;
- 	u64		size;
-	u64		offset;
- };
- 
- static inline dma_addr_t translate_phys_to_dma(struct device *dev,
-@@ -29,9 +28,12 @@ static inline dma_addr_t translate_phys_to_dma(struct device *dev,
- {
- 	const struct bus_dma_region *m;
- 
-	for (m = dev->dma_range_map; m->size; m++)
-		if (paddr >= m->cpu_start && paddr - m->cpu_start < m->size)
-			return (dma_addr_t)paddr - m->offset;
-+	for (m = dev->dma_range_map; m->size; m++) {
-+		u64 offset = paddr - m->cpu_start;
-+
-+		if (paddr >= m->cpu_start && offset < m->size)
-+			return m->dma_start + offset;
-+	}
- 
- 	/* make sure dma_capable fails when no translation is available */
- 	return DMA_MAPPING_ERROR;
-@@ -42,27 +44,52 @@ static inline phys_addr_t translate_dma_to_phys(struct device *dev,
- {
- 	const struct bus_dma_region *m;
- 
-	for (m = dev->dma_range_map; m->size; m++)
-		if (dma_addr >= m->dma_start && dma_addr - m->dma_start < m->size)
-			return (phys_addr_t)dma_addr + m->offset;
-+	for (m = dev->dma_range_map; m->size; m++) {
-+		u64 offset = dma_addr - m->dma_start;
-+
-+		if (dma_addr >= m->dma_start && offset < m->size)
-+			return m->cpu_start + offset;
-+	}
- 
- 	return (phys_addr_t)-1;
- }
- 
-+static inline dma_addr_t dma_range_map_min(const struct bus_dma_region *map)
-+{
-+	dma_addr_t ret = (dma_addr_t)U64_MAX;
-+
-+	for (; map->size; map++)
-+		ret = min(ret, map->dma_start);
-+	return ret;
-+}
-+
-+static inline dma_addr_t dma_range_map_max(const struct bus_dma_region *map)
-+{
-+	dma_addr_t ret = 0;
-+
-+	for (; map->size; map++)
-+		ret = max(ret, map->dma_start + map->size - 1);
-+	return ret;
-+}
-+
- #ifdef CONFIG_ARCH_HAS_PHYS_TO_DMA
- #include <asm/dma-direct.h>
- #ifndef phys_to_dma_unencrypted
- #define phys_to_dma_unencrypted		phys_to_dma
- #endif
- #else
-static inline dma_addr_t phys_to_dma_unencrypted(struct device *dev,
-		phys_addr_t paddr)
-+static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
- {
- 	if (dev->dma_range_map)
- 		return translate_phys_to_dma(dev, paddr);
- 	return paddr;
- }
- 
-+static inline dma_addr_t phys_to_dma_unencrypted(struct device *dev,
-+						phys_addr_t paddr)
-+{
-+	return dma_addr_unencrypted(__phys_to_dma(dev, paddr));
-+}
- /*
-  * If memory encryption is supported, phys_to_dma will set the memory encryption
-  * bit in the DMA address, and dma_to_phys will clear it.
-@@ -71,19 +98,20 @@ static inline dma_addr_t phys_to_dma_unencrypted(struct device *dev,
-  */
- static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
- {
-	return __sme_set(phys_to_dma_unencrypted(dev, paddr));
-+	return dma_addr_encrypted(__phys_to_dma(dev, paddr));
- }
- 
- static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dma_addr)
- {
- 	phys_addr_t paddr;
- 
-+	dma_addr = dma_addr_canonical(dma_addr);
- 	if (dev->dma_range_map)
- 		paddr = translate_dma_to_phys(dev, dma_addr);
- 	else
- 		paddr = dma_addr;
- 
-	return __sme_clr(paddr);
-+	return paddr;
- }
- #endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */
- 
-@@ -121,7 +149,5 @@ void dma_direct_free_pages(struct device *dev, size_t size,
- 		struct page *page, dma_addr_t dma_addr,
- 		enum dma_data_direction dir);
- int dma_direct_supported(struct device *dev, u64 mask);
-dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
-		size_t size, enum dma_data_direction dir, unsigned long attrs);
- 
- #endif /* _LINUX_DMA_DIRECT_H */
-diff --git a/include/linux/dma-fence-array.h b/include/linux/dma-fence-array.h
-index ec7f25def392..079b3dec0a16 100644
--- a/include/linux/dma-fence-array.h
-+++ b/include/linux/dma-fence-array.h
-@@ -33,6 +33,7 @@ struct dma_fence_array_cb {
-  * @num_pending: fences in the array still pending
-  * @fences: array of the fences
-  * @work: internal irq_work function
-+ * @callbacks: array of callback helpers
-  */
- struct dma_fence_array {
- 	struct dma_fence base;
-@@ -43,6 +44,8 @@ struct dma_fence_array {
- 	struct dma_fence **fences;
- 
- 	struct irq_work work;
-+
-+	struct dma_fence_array_cb callbacks[] __counted_by(num_fences);
- };
- 
- /**
-@@ -76,6 +79,12 @@ to_dma_fence_array(struct dma_fence *fence)
- 	for (index = 0, fence = dma_fence_array_first(head); fence;	\
- 	     ++(index), fence = dma_fence_array_next(head, index))
- 
-+struct dma_fence_array *dma_fence_array_alloc(int num_fences);
-+void dma_fence_array_init(struct dma_fence_array *array,
-+			  int num_fences, struct dma_fence **fences,
-+			  u64 context, unsigned seqno,
-+			  bool signal_on_any);
-+
- struct dma_fence_array *dma_fence_array_create(int num_fences,
- 					       struct dma_fence **fences,
- 					       u64 context, unsigned seqno,
-diff --git a/include/linux/dma-fence-chain.h b/include/linux/dma-fence-chain.h
-index 4bdf0b96da28..68c3c1e41014 100644
--- a/include/linux/dma-fence-chain.h
-+++ b/include/linux/dma-fence-chain.h
-@@ -85,11 +85,13 @@ dma_fence_chain_contained(struct dma_fence *fence)
-  * dma_fence_chain_alloc
-  *
-  * Returns a new struct dma_fence_chain object or NULL on failure.
-+ *
-+ * This specialized allocator has to be a macro for its allocations to be
-+ * accounted separately (to have a separate alloc_tag). The typecast is
-+ * intentional to enforce typesafety.
-  */
-static inline struct dma_fence_chain *dma_fence_chain_alloc(void)
-{
-	return kmalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
-};
-+#define dma_fence_chain_alloc()	\
-+		((struct dma_fence_chain *)kmalloc(sizeof(struct dma_fence_chain), GFP_KERNEL))
- 
- /**
-  * dma_fence_chain_free
-diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h
-index 66b1e56fbb81..62df222fe0f1 100644
--- a/include/linux/dma-fence-unwrap.h
-+++ b/include/linux/dma-fence-unwrap.h
-@@ -52,6 +52,8 @@ struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
- 					   struct dma_fence **fences,
- 					   struct dma_fence_unwrap *cursors);
- 
-+int dma_fence_dedup_array(struct dma_fence **array, int num_fences);
-+
- /**
-  * dma_fence_unwrap_merge - unwrap and merge fences
-  *
-diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h
-index 775cdc0b4f24..64639e104110 100644
--- a/include/linux/dma-fence.h
-+++ b/include/linux/dma-fence.h
-@@ -21,10 +21,12 @@
- #include <linux/sched.h>
- #include <linux/printk.h>
- #include <linux/rcupdate.h>
-+#include <linux/timekeeping.h>
- 
- struct dma_fence;
- struct dma_fence_ops;
- struct dma_fence_cb;
-+struct seq_file;
- 
- /**
-  * struct dma_fence - software synchronization primitive
-@@ -96,6 +98,7 @@ struct dma_fence {
- };
- 
- enum dma_fence_flag_bits {
-+	DMA_FENCE_FLAG_SEQNO64_BIT,
- 	DMA_FENCE_FLAG_SIGNALED_BIT,
- 	DMA_FENCE_FLAG_TIMESTAMP_BIT,
- 	DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
-@@ -124,14 +127,6 @@ struct dma_fence_cb {
-  */
- struct dma_fence_ops {
- 	/**
-	 * @use_64bit_seqno:
-	 *
-	 * True if this dma_fence implementation uses 64bit seqno, false
-	 * otherwise.
-	 */
-	bool use_64bit_seqno;
-
-	/**
- 	 * @get_driver_name:
- 	 *
- 	 * Returns the driver name. This is a callback to allow drivers to
-@@ -168,8 +163,8 @@ struct dma_fence_ops {
- 	 * implementation know that there is another driver waiting on the
- 	 * signal (ie. hw->sw case).
- 	 *
-	 * This function can be called from atomic context, but not
-	 * from irq context, so normal spinlocks can be used.
-+	 * This is called with irq's disabled, so only spinlocks which disable
-+	 * IRQ's can be used in the code outside of this callback.
- 	 *
- 	 * A return value of false indicates the fence already passed,
- 	 * or some failure occurred that made it impossible to enable
-@@ -238,30 +233,32 @@ struct dma_fence_ops {
- 	void (*release)(struct dma_fence *fence);
- 
- 	/**
-	 * @fence_value_str:
-+	 * @set_deadline:
- 	 *
-	 * Callback to fill in free-form debug info specific to this fence, like
-	 * the sequence number.
-+	 * Callback to allow a fence waiter to inform the fence signaler of
-+	 * an upcoming deadline, such as vblank, by which point the waiter
-+	 * would prefer the fence to be signaled by.  This is intended to
-+	 * give feedback to the fence signaler to aid in power management
-+	 * decisions, such as boosting GPU frequency.
- 	 *
-	 * This callback is optional.
-	 */
-	void (*fence_value_str)(struct dma_fence *fence, char *str, int size);
-
-	/**
-	 * @timeline_value_str:
-+	 * This is called without &dma_fence.lock held, it can be called
-+	 * multiple times and from any context.  Locking is up to the callee
-+	 * if it has some state to manage.  If multiple deadlines are set,
-+	 * the expectation is to track the soonest one.  If the deadline is
-+	 * before the current time, it should be interpreted as an immediate
-+	 * deadline.
- 	 *
-	 * Fills in the current value of the timeline as a string, like the
-	 * sequence number. Note that the specific fence passed to this function
-	 * should not matter, drivers should only use it to look up the
-	 * corresponding timeline structures.
-+	 * This callback is optional.
- 	 */
-	void (*timeline_value_str)(struct dma_fence *fence,
-				   char *str, int size);
-+	void (*set_deadline)(struct dma_fence *fence, ktime_t deadline);
- };
- 
- void dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
- 		    spinlock_t *lock, u64 context, u64 seqno);
- 
-+void dma_fence_init64(struct dma_fence *fence, const struct dma_fence_ops *ops,
-+		      spinlock_t *lock, u64 context, u64 seqno);
-+
- void dma_fence_release(struct kref *kref);
- void dma_fence_free(struct dma_fence *fence);
- void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq);
-@@ -382,6 +379,29 @@ bool dma_fence_remove_callback(struct dma_fence *fence,
- void dma_fence_enable_sw_signaling(struct dma_fence *fence);
- 
- /**
-+ * DOC: Safe external access to driver provided object members
-+ *
-+ * All data not stored directly in the dma-fence object, such as the
-+ * &dma_fence.lock and memory potentially accessed by functions in the
-+ * &dma_fence.ops table, MUST NOT be accessed after the fence has been signalled
-+ * because after that point drivers are allowed to free it.
-+ *
-+ * All code accessing that data via the dma-fence API (or directly, which is
-+ * discouraged), MUST make sure to contain the complete access within a
-+ * &rcu_read_lock and &rcu_read_unlock pair.
-+ *
-+ * Some dma-fence API handles this automatically, while other, as for example
-+ * &dma_fence_driver_name and &dma_fence_timeline_name, leave that
-+ * responsibility to the caller.
-+ *
-+ * To enable this scheme to work drivers MUST ensure a RCU grace period elapses
-+ * between signalling the fence and freeing the said data.
-+ *
-+ */
-+const char __rcu *dma_fence_driver_name(struct dma_fence *fence);
-+const char __rcu *dma_fence_timeline_name(struct dma_fence *fence);
-+
-+/**
-  * dma_fence_is_signaled_locked - Return an indication if the fence
-  *                                is signaled yet.
-  * @fence: the fence to check
-@@ -441,21 +461,20 @@ dma_fence_is_signaled(struct dma_fence *fence)
- 
- /**
-  * __dma_fence_is_later - return if f1 is chronologically later than f2
-+ * @fence: fence in whose context to do the comparison
-  * @f1: the first fence's seqno
-  * @f2: the second fence's seqno from the same context
- * @ops: dma_fence_ops associated with the seqno
-  *
-  * Returns true if f1 is chronologically later than f2. Both fences must be
-  * from the same context, since a seqno is not common across contexts.
-  */
-static inline bool __dma_fence_is_later(u64 f1, u64 f2,
-					const struct dma_fence_ops *ops)
-+static inline bool __dma_fence_is_later(struct dma_fence *fence, u64 f1, u64 f2)
- {
- 	/* This is for backward compatibility with drivers which can only handle
- 	 * 32bit sequence numbers. Use a 64bit compare when the driver says to
- 	 * do so.
- 	 */
-	if (ops->use_64bit_seqno)
-+	if (test_bit(DMA_FENCE_FLAG_SEQNO64_BIT, &fence->flags))
- 		return f1 > f2;
- 
- 	return (int)(lower_32_bits(f1) - lower_32_bits(f2)) > 0;
-@@ -475,7 +494,22 @@ static inline bool dma_fence_is_later(struct dma_fence *f1,
- 	if (WARN_ON(f1->context != f2->context))
- 		return false;
- 
-	return __dma_fence_is_later(f1->seqno, f2->seqno, f1->ops);
-+	return __dma_fence_is_later(f1, f1->seqno, f2->seqno);
-+}
-+
-+/**
-+ * dma_fence_is_later_or_same - return true if f1 is later or same as f2
-+ * @f1: the first fence from the same context
-+ * @f2: the second fence from the same context
-+ *
-+ * Returns true if f1 is chronologically later than f2 or the same fence. Both
-+ * fences must be from the same context, since a seqno is not re-used across
-+ * contexts.
-+ */
-+static inline bool dma_fence_is_later_or_same(struct dma_fence *f1,
-+					      struct dma_fence *f2)
-+{
-+	return f1 == f2 || dma_fence_is_later(f1, f2);
- }
- 
- /**
-@@ -538,6 +572,12 @@ int dma_fence_get_status(struct dma_fence *fence);
-  * rather than success. This must be set before signaling (so that the value
-  * is visible before any waiters on the signal callback are woken). This
-  * helper exists to help catching erroneous setting of #dma_fence.error.
-+ *
-+ * Examples of error codes which drivers should use:
-+ *
-+ * * %-ENODATA	 This operation produced no data, no other operation affected.
-+ * * %-ECANCELED All operations from the same context have been canceled.
-+ * * %-ETIME	 Operation caused a timeout and potentially device reset.
-  */
- static inline void dma_fence_set_error(struct dma_fence *fence,
- 				       int error)
-@@ -548,6 +588,25 @@ static inline void dma_fence_set_error(struct dma_fence *fence,
- 	fence->error = error;
- }
- 
-+/**
-+ * dma_fence_timestamp - helper to get the completion timestamp of a fence
-+ * @fence: fence to get the timestamp from.
-+ *
-+ * After a fence is signaled the timestamp is updated with the signaling time,
-+ * but setting the timestamp can race with tasks waiting for the signaling. This
-+ * helper busy waits for the correct timestamp to appear.
-+ */
-+static inline ktime_t dma_fence_timestamp(struct dma_fence *fence)
-+{
-+	if (WARN_ON(!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)))
-+		return ktime_get();
-+
-+	while (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags))
-+		cpu_relax();
-+
-+	return fence->timestamp;
-+}
-+
- signed long dma_fence_wait_timeout(struct dma_fence *,
- 				   bool intr, signed long timeout);
- signed long dma_fence_wait_any_timeout(struct dma_fence **fences,
-@@ -583,8 +642,10 @@ static inline signed long dma_fence_wait(struct dma_fence *fence, bool intr)
- 	return ret < 0 ? ret : 0;
- }
- 
-+void dma_fence_set_deadline(struct dma_fence *fence, ktime_t deadline);
-+
- struct dma_fence *dma_fence_get_stub(void);
-struct dma_fence *dma_fence_allocate_private_stub(void);
-+struct dma_fence *dma_fence_allocate_private_stub(ktime_t timestamp);
- u64 dma_fence_context_alloc(unsigned num);
- 
- extern const struct dma_fence_ops dma_fence_array_ops;
-diff --git a/include/linux/dma-heap.h b/include/linux/dma-heap.h
-index 0c05561cad6e..27d15f60950a 100644
--- a/include/linux/dma-heap.h
-+++ b/include/linux/dma-heap.h
-@@ -9,22 +9,21 @@
- #ifndef _DMA_HEAPS_H
- #define _DMA_HEAPS_H
- 
-#include <linux/cdev.h>
- #include <linux/types.h>
- 
- struct dma_heap;
- 
- /**
-  * struct dma_heap_ops - ops to operate on a given heap
- * @allocate:		allocate dmabuf and return struct dma_buf ptr
-+ * @allocate:	allocate dmabuf and return struct dma_buf ptr
-  *
-  * allocate returns dmabuf on success, ERR_PTR(-errno) on error.
-  */
- struct dma_heap_ops {
- 	struct dma_buf *(*allocate)(struct dma_heap *heap,
- 				    unsigned long len,
-				    unsigned long fd_flags,
-				    unsigned long heap_flags);
-+				    u32 fd_flags,
-+				    u64 heap_flags);
- };
- 
- /**
-@@ -41,28 +40,10 @@ struct dma_heap_export_info {
- 	void *priv;
- };
- 
-/**
- * dma_heap_get_drvdata() - get per-heap driver data
- * @heap: DMA-Heap to retrieve private data for
- *
- * Returns:
- * The per-heap data for the heap.
- */
- void *dma_heap_get_drvdata(struct dma_heap *heap);
- 
-/**
- * dma_heap_get_name() - get heap name
- * @heap: DMA-Heap to retrieve private data for
- *
- * Returns:
- * The char* for the heap name.
- */
- const char *dma_heap_get_name(struct dma_heap *heap);
- 
-/**
- * dma_heap_add - adds a heap to dmabuf heaps
- * @exp_info:		information needed to register this heap
- */
- struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info);
- 
- #endif /* _DMA_HEAPS_H */
-diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h
-deleted file mode 100644
-index 24607dc3c2ac..000000000000
--- a/include/linux/dma-iommu.h
-+++ /dev/null
-@@ -1,93 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2014-2015 ARM Ltd.
- */
-#ifndef __DMA_IOMMU_H
-#define __DMA_IOMMU_H
-
-#include <linux/errno.h>
-#include <linux/types.h>
-
-#ifdef CONFIG_IOMMU_DMA
-#include <linux/dma-mapping.h>
-#include <linux/iommu.h>
-#include <linux/msi.h>
-
-/* Domain management interface for IOMMU drivers */
-int iommu_get_dma_cookie(struct iommu_domain *domain);
-int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base);
-void iommu_put_dma_cookie(struct iommu_domain *domain);
-
-/* Setup call for arch DMA mapping code */
-void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit);
-int iommu_dma_init_fq(struct iommu_domain *domain);
-
-/* The DMA API isn't _quite_ the whole story, though... */
-/*
- * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU device
- *
- * The MSI page will be stored in @desc.
- *
- * Return: 0 on success otherwise an error describing the failure.
- */
-int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr);
-
-/* Update the MSI message if required. */
-void iommu_dma_compose_msi_msg(struct msi_desc *desc,
-			       struct msi_msg *msg);
-
-void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list);
-
-void iommu_dma_free_cpu_cached_iovas(unsigned int cpu,
-		struct iommu_domain *domain);
-
-extern bool iommu_dma_forcedac;
-
-#else /* CONFIG_IOMMU_DMA */
-
-struct iommu_domain;
-struct msi_desc;
-struct msi_msg;
-struct device;
-
-static inline void iommu_setup_dma_ops(struct device *dev, u64 dma_base,
-				       u64 dma_limit)
-{
-}
-
-static inline int iommu_dma_init_fq(struct iommu_domain *domain)
-{
-	return -EINVAL;
-}
-
-static inline int iommu_get_dma_cookie(struct iommu_domain *domain)
-{
-	return -ENODEV;
-}
-
-static inline int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)
-{
-	return -ENODEV;
-}
-
-static inline void iommu_put_dma_cookie(struct iommu_domain *domain)
-{
-}
-
-static inline int iommu_dma_prepare_msi(struct msi_desc *desc,
-					phys_addr_t msi_addr)
-{
-	return 0;
-}
-
-static inline void iommu_dma_compose_msi_msg(struct msi_desc *desc,
-					     struct msi_msg *msg)
-{
-}
-
-static inline void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list)
-{
-}
-
-#endif	/* CONFIG_IOMMU_DMA */
-#endif	/* __DMA_IOMMU_H */
-diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
-index d678afeb8a13..4809204c674c 100644
--- a/include/linux/dma-map-ops.h
-+++ b/include/linux/dma-map-ops.h
-@@ -8,35 +8,22 @@
- 
- #include <linux/dma-mapping.h>
- #include <linux/pgtable.h>
-+#include <linux/slab.h>
- 
- struct cma;
-
-/*
- * Values for struct dma_map_ops.flags:
- *
- * DMA_F_PCI_P2PDMA_SUPPORTED: Indicates the dma_map_ops implementation can
- * handle PCI P2PDMA pages in the map_sg/unmap_sg operation.
- */
-#define DMA_F_PCI_P2PDMA_SUPPORTED     (1 << 0)
-+struct iommu_ops;
- 
- struct dma_map_ops {
-	unsigned int flags;
-
- 	void *(*alloc)(struct device *dev, size_t size,
- 			dma_addr_t *dma_handle, gfp_t gfp,
- 			unsigned long attrs);
- 	void (*free)(struct device *dev, size_t size, void *vaddr,
- 			dma_addr_t dma_handle, unsigned long attrs);
-	struct page *(*alloc_pages)(struct device *dev, size_t size,
-+	struct page *(*alloc_pages_op)(struct device *dev, size_t size,
- 			dma_addr_t *dma_handle, enum dma_data_direction dir,
- 			gfp_t gfp);
- 	void (*free_pages)(struct device *dev, size_t size, struct page *vaddr,
- 			dma_addr_t dma_handle, enum dma_data_direction dir);
-	struct sg_table *(*alloc_noncontiguous)(struct device *dev, size_t size,
-			enum dma_data_direction dir, gfp_t gfp,
-			unsigned long attrs);
-	void (*free_noncontiguous)(struct device *dev, size_t size,
-			struct sg_table *sgt, enum dma_data_direction dir);
- 	int (*mmap)(struct device *, struct vm_area_struct *,
- 			void *, dma_addr_t, size_t, unsigned long attrs);
- 
-@@ -44,10 +31,10 @@ struct dma_map_ops {
- 			void *cpu_addr, dma_addr_t dma_addr, size_t size,
- 			unsigned long attrs);
- 
-	dma_addr_t (*map_page)(struct device *dev, struct page *page,
-			unsigned long offset, size_t size,
-			enum dma_data_direction dir, unsigned long attrs);
-	void (*unmap_page)(struct device *dev, dma_addr_t dma_handle,
-+	dma_addr_t (*map_phys)(struct device *dev, phys_addr_t phys,
-+			size_t size, enum dma_data_direction dir,
-+			unsigned long attrs);
-+	void (*unmap_phys)(struct device *dev, dma_addr_t dma_handle,
- 			size_t size, enum dma_data_direction dir,
- 			unsigned long attrs);
- 	/*
-@@ -59,12 +46,6 @@ struct dma_map_ops {
- 			enum dma_data_direction dir, unsigned long attrs);
- 	void (*unmap_sg)(struct device *dev, struct scatterlist *sg, int nents,
- 			enum dma_data_direction dir, unsigned long attrs);
-	dma_addr_t (*map_resource)(struct device *dev, phys_addr_t phys_addr,
-			size_t size, enum dma_data_direction dir,
-			unsigned long attrs);
-	void (*unmap_resource)(struct device *dev, dma_addr_t dma_handle,
-			size_t size, enum dma_data_direction dir,
-			unsigned long attrs);
- 	void (*sync_single_for_cpu)(struct device *dev, dma_addr_t dma_handle,
- 			size_t size, enum dma_data_direction dir);
- 	void (*sync_single_for_device)(struct device *dev,
-@@ -83,14 +64,14 @@ struct dma_map_ops {
- 	unsigned long (*get_merge_boundary)(struct device *dev);
- };
- 
-#ifdef CONFIG_DMA_OPS
-+#ifdef CONFIG_ARCH_HAS_DMA_OPS
- #include <asm/dma-mapping.h>
- 
- static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
- {
- 	if (dev->dma_ops)
- 		return dev->dma_ops;
-	return get_arch_dma_ops(dev->bus);
-+	return get_arch_dma_ops();
- }
- 
- static inline void set_dma_ops(struct device *dev,
-@@ -98,7 +79,7 @@ static inline void set_dma_ops(struct device *dev,
- {
- 	dev->dma_ops = dma_ops;
- }
-#else /* CONFIG_DMA_OPS */
-+#else /* CONFIG_ARCH_HAS_DMA_OPS */
- static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
- {
- 	return NULL;
-@@ -107,7 +88,7 @@ static inline void set_dma_ops(struct device *dev,
- 			       const struct dma_map_ops *dma_ops)
- {
- }
-#endif /* CONFIG_DMA_OPS */
-+#endif /* CONFIG_ARCH_HAS_DMA_OPS */
- 
- #ifdef CONFIG_DMA_CMA
- extern struct cma *dma_contiguous_default_area;
-@@ -166,14 +147,11 @@ static inline void dma_free_contiguous(struct device *dev, struct page *page,
- {
- 	__free_pages(page, get_order(size));
- }
-+static inline void dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
-+{
-+}
- #endif /* CONFIG_DMA_CMA*/
- 
-#ifdef CONFIG_DMA_PERNUMA_CMA
-void dma_pernuma_cma_reserve(void);
-#else
-static inline void dma_pernuma_cma_reserve(void) { }
-#endif /* CONFIG_DMA_PERNUMA_CMA */
-
- #ifdef CONFIG_DMA_DECLARE_COHERENT
- int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
- 		dma_addr_t device_addr, size_t size);
-@@ -220,20 +198,6 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma,
- }
- #endif /* CONFIG_DMA_GLOBAL_POOL */
- 
-/*
- * This is the actual return value from the ->alloc_noncontiguous method.
- * The users of the DMA API should only care about the sg_table, but to make
- * the DMA-API internal vmaping and freeing easier we stash away the page
- * array as well (except for the fallback case).  This can go away any time,
- * e.g. when a vmap-variant that takes a scatterlist comes along.
- */
-struct dma_sgt_handle {
-	struct sg_table sgt;
-	struct page **pages;
-};
-#define sgt_handle(sgt) \
-	container_of((sgt), struct dma_sgt_handle, sgt)
-
- int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
- 		void *cpu_addr, dma_addr_t dma_addr, size_t size,
- 		unsigned long attrs);
-@@ -269,17 +233,94 @@ static inline bool dev_is_dma_coherent(struct device *dev)
- 	return dev->dma_coherent;
- }
- #else
-+#define dma_default_coherent true
-+
- static inline bool dev_is_dma_coherent(struct device *dev)
- {
- 	return true;
- }
-#endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */
-+#endif
-+
-+static inline void dma_reset_need_sync(struct device *dev)
-+{
-+#ifdef CONFIG_DMA_NEED_SYNC
-+	/* Reset it only once so that the function can be called on hotpath */
-+	if (unlikely(dev->dma_skip_sync))
-+		dev->dma_skip_sync = false;
-+#endif
-+}
-+
-+/*
-+ * Check whether potential kmalloc() buffers are safe for non-coherent DMA.
-+ */
-+static inline bool dma_kmalloc_safe(struct device *dev,
-+				    enum dma_data_direction dir)
-+{
-+	/*
-+	 * If DMA bouncing of kmalloc() buffers is disabled, the kmalloc()
-+	 * caches have already been aligned to a DMA-safe size.
-+	 */
-+	if (!IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC))
-+		return true;
-+
-+	/*
-+	 * kmalloc() buffers are DMA-safe irrespective of size if the device
-+	 * is coherent or the direction is DMA_TO_DEVICE (non-desctructive
-+	 * cache maintenance and benign cache line evictions).
-+	 */
-+	if (dev_is_dma_coherent(dev) || dir == DMA_TO_DEVICE)
-+		return true;
-+
-+	return false;
-+}
-+
-+/*
-+ * Check whether the given size, assuming it is for a kmalloc()'ed buffer, is
-+ * sufficiently aligned for non-coherent DMA.
-+ */
-+static inline bool dma_kmalloc_size_aligned(size_t size)
-+{
-+	/*
-+	 * Larger kmalloc() sizes are guaranteed to be aligned to
-+	 * ARCH_DMA_MINALIGN.
-+	 */
-+	if (size >= 2 * ARCH_DMA_MINALIGN ||
-+	    IS_ALIGNED(kmalloc_size_roundup(size), dma_get_cache_alignment()))
-+		return true;
-+
-+	return false;
-+}
-+
-+/*
-+ * Check whether the given object size may have originated from a kmalloc()
-+ * buffer with a slab alignment below the DMA-safe alignment and needs
-+ * bouncing for non-coherent DMA. The pointer alignment is not considered and
-+ * in-structure DMA-safe offsets are the responsibility of the caller. Such
-+ * code should use the static ARCH_DMA_MINALIGN for compiler annotations.
-+ *
-+ * The heuristics can have false positives, bouncing unnecessarily, though the
-+ * buffers would be small. False negatives are theoretically possible if, for
-+ * example, multiple small kmalloc() buffers are coalesced into a larger
-+ * buffer that passes the alignment check. There are no such known constructs
-+ * in the kernel.
-+ */
-+static inline bool dma_kmalloc_needs_bounce(struct device *dev, size_t size,
-+					    enum dma_data_direction dir)
-+{
-+	return !dma_kmalloc_safe(dev, dir) && !dma_kmalloc_size_aligned(size);
-+}
- 
- void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
- 		gfp_t gfp, unsigned long attrs);
- void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
- 		dma_addr_t dma_addr, unsigned long attrs);
- 
-+#ifdef CONFIG_ARCH_HAS_DMA_SET_MASK
-+void arch_dma_set_mask(struct device *dev, u64 mask);
-+#else
-+#define arch_dma_set_mask(dev, mask)	do { } while (0)
-+#endif
-+
- #ifdef CONFIG_MMU
- /*
-  * Page protection so that devices that can't snoop CPU caches can use the
-@@ -348,25 +389,23 @@ void *arch_dma_set_uncached(void *addr, size_t size);
- void arch_dma_clear_uncached(void *addr, size_t size);
- 
- #ifdef CONFIG_ARCH_HAS_DMA_MAP_DIRECT
-bool arch_dma_map_page_direct(struct device *dev, phys_addr_t addr);
-bool arch_dma_unmap_page_direct(struct device *dev, dma_addr_t dma_handle);
-+bool arch_dma_map_phys_direct(struct device *dev, phys_addr_t addr);
-+bool arch_dma_unmap_phys_direct(struct device *dev, dma_addr_t dma_handle);
- bool arch_dma_map_sg_direct(struct device *dev, struct scatterlist *sg,
- 		int nents);
- bool arch_dma_unmap_sg_direct(struct device *dev, struct scatterlist *sg,
- 		int nents);
- #else
-#define arch_dma_map_page_direct(d, a)		(false)
-#define arch_dma_unmap_page_direct(d, a)	(false)
-+#define arch_dma_map_phys_direct(d, a)		(false)
-+#define arch_dma_unmap_phys_direct(d, a)	(false)
- #define arch_dma_map_sg_direct(d, s, n)		(false)
- #define arch_dma_unmap_sg_direct(d, s, n)	(false)
- #endif
- 
- #ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
-void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
-		const struct iommu_ops *iommu, bool coherent);
-+void arch_setup_dma_ops(struct device *dev, bool coherent);
- #else
-static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
-		u64 size, const struct iommu_ops *iommu, bool coherent)
-+static inline void arch_setup_dma_ops(struct device *dev, bool coherent)
- {
- }
- #endif /* CONFIG_ARCH_HAS_SETUP_DMA_OPS */
-@@ -380,10 +419,10 @@ static inline void arch_teardown_dma_ops(struct device *dev)
- #endif /* CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS */
- 
- #ifdef CONFIG_DMA_API_DEBUG
-void dma_debug_add_bus(struct bus_type *bus);
-+void dma_debug_add_bus(const struct bus_type *bus);
- void debug_dma_dump_mappings(struct device *dev);
- #else
-static inline void dma_debug_add_bus(struct bus_type *bus)
-+static inline void dma_debug_add_bus(const struct bus_type *bus)
- {
- }
- static inline void debug_dma_dump_mappings(struct device *dev)
-@@ -392,58 +431,4 @@ static inline void debug_dma_dump_mappings(struct device *dev)
- #endif /* CONFIG_DMA_API_DEBUG */
- 
- extern const struct dma_map_ops dma_dummy_ops;
-
-enum pci_p2pdma_map_type {
-	/*
-	 * PCI_P2PDMA_MAP_UNKNOWN: Used internally for indicating the mapping
-	 * type hasn't been calculated yet. Functions that return this enum
-	 * never return this value.
-	 */
-	PCI_P2PDMA_MAP_UNKNOWN = 0,
-
-	/*
-	 * PCI_P2PDMA_MAP_NOT_SUPPORTED: Indicates the transaction will
-	 * traverse the host bridge and the host bridge is not in the
-	 * allowlist. DMA Mapping routines should return an error when
-	 * this is returned.
-	 */
-	PCI_P2PDMA_MAP_NOT_SUPPORTED,
-
-	/*
-	 * PCI_P2PDMA_BUS_ADDR: Indicates that two devices can talk to
-	 * each other directly through a PCI switch and the transaction will
-	 * not traverse the host bridge. Such a mapping should program
-	 * the DMA engine with PCI bus addresses.
-	 */
-	PCI_P2PDMA_MAP_BUS_ADDR,
-
-	/*
-	 * PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: Indicates two devices can talk
-	 * to each other, but the transaction traverses a host bridge on the
-	 * allowlist. In this case, a normal mapping either with CPU physical
-	 * addresses (in the case of dma-direct) or IOVA addresses (in the
-	 * case of IOMMUs) should be used to program the DMA engine.
-	 */
-	PCI_P2PDMA_MAP_THRU_HOST_BRIDGE,
-};
-
-struct pci_p2pdma_map_state {
-	struct dev_pagemap *pgmap;
-	int map;
-	u64 bus_off;
-};
-
-#ifdef CONFIG_PCI_P2PDMA
-enum pci_p2pdma_map_type
-pci_p2pdma_map_segment(struct pci_p2pdma_map_state *state, struct device *dev,
-		       struct scatterlist *sg);
-#else /* CONFIG_PCI_P2PDMA */
-static inline enum pci_p2pdma_map_type
-pci_p2pdma_map_segment(struct pci_p2pdma_map_state *state, struct device *dev,
-		       struct scatterlist *sg)
-{
-	return PCI_P2PDMA_MAP_NOT_SUPPORTED;
-}
-#endif /* CONFIG_PCI_P2PDMA */
-
- #endif /* _LINUX_DMA_MAP_OPS_H */
-diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
-index 25a30906289d..2ceda49c609f 100644
--- a/include/linux/dma-mapping.h
-+++ b/include/linux/dma-mapping.h
-@@ -2,14 +2,11 @@
- #ifndef _LINUX_DMA_MAPPING_H
- #define _LINUX_DMA_MAPPING_H
- 
-#include <linux/sizes.h>
-#include <linux/string.h>
- #include <linux/device.h>
- #include <linux/err.h>
- #include <linux/dma-direction.h>
- #include <linux/scatterlist.h>
- #include <linux/bug.h>
-#include <linux/mem_encrypt.h>
- 
- /**
-  * List of possible attributes associated with a DMA mapping. The semantics
-@@ -62,6 +59,26 @@
- #define DMA_ATTR_PRIVILEGED		(1UL << 9)
- 
- /*
-+ * DMA_ATTR_MMIO - Indicates memory-mapped I/O (MMIO) region for DMA mapping
-+ *
-+ * This attribute indicates the physical address is not normal system
-+ * memory. It may not be used with kmap*()/phys_to_virt()/phys_to_page()
-+ * functions, it may not be cacheable, and access using CPU load/store
-+ * instructions may not be allowed.
-+ *
-+ * Usually this will be used to describe MMIO addresses, or other non-cacheable
-+ * register addresses. When DMA mapping this sort of address we call
-+ * the operation Peer to Peer as a one device is DMA'ing to another device.
-+ * For PCI devices the p2pdma APIs must be used to determine if DMA_ATTR_MMIO
-+ * is appropriate.
-+ *
-+ * For architectures that require cache flushing for DMA coherence
-+ * DMA_ATTR_MMIO will not perform any cache flushing. The address
-+ * provided must never be mapped cacheable into the CPU.
-+ */
-+#define DMA_ATTR_MMIO		(1UL << 10)
-+
-+/*
-  * A dma_addr_t can hold any valid DMA or bus address for the platform.  It can
-  * be given to a device to use as a DMA source or target.  It is specific to a
-  * given device and there may be a translation between the CPU physical address
-@@ -73,7 +90,23 @@
-  */
- #define DMA_MAPPING_ERROR		(~(dma_addr_t)0)
- 
-#define DMA_BIT_MASK(n)	(((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
-+#define DMA_BIT_MASK(n)	GENMASK_ULL(n - 1, 0)
-+
-+struct dma_iova_state {
-+	dma_addr_t addr;
-+	u64 __size;
-+};
-+
-+/*
-+ * Use the high bit to mark if we used swiotlb for one or more ranges.
-+ */
-+#define DMA_IOVA_USE_SWIOTLB		(1ULL << 63)
-+
-+static inline size_t dma_iova_size(struct dma_iova_state *state)
-+{
-+	/* Casting is needed for 32-bits systems */
-+	return (size_t)(state->__size & ~DMA_IOVA_USE_SWIOTLB);
-+}
- 
- #ifdef CONFIG_DMA_API_DEBUG
- void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr);
-@@ -105,6 +138,10 @@ dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
- 		unsigned long attrs);
- void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
- 		enum dma_data_direction dir, unsigned long attrs);
-+dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,
-+		enum dma_data_direction dir, unsigned long attrs);
-+void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size,
-+		enum dma_data_direction dir, unsigned long attrs);
- unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
- 		int nents, enum dma_data_direction dir, unsigned long attrs);
- void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
-@@ -116,14 +153,6 @@ dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
- 		size_t size, enum dma_data_direction dir, unsigned long attrs);
- void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
- 		enum dma_data_direction dir, unsigned long attrs);
-void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
-		enum dma_data_direction dir);
-void dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
-		size_t size, enum dma_data_direction dir);
-void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
-		    int nelems, enum dma_data_direction dir);
-void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
-		       int nelems, enum dma_data_direction dir);
- void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
- 		gfp_t flag, unsigned long attrs);
- void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
-@@ -139,14 +168,13 @@ int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
- 		void *cpu_addr, dma_addr_t dma_addr, size_t size,
- 		unsigned long attrs);
- bool dma_can_mmap(struct device *dev);
-int dma_supported(struct device *dev, u64 mask);
- bool dma_pci_p2pdma_supported(struct device *dev);
- int dma_set_mask(struct device *dev, u64 mask);
- int dma_set_coherent_mask(struct device *dev, u64 mask);
- u64 dma_get_required_mask(struct device *dev);
-+bool dma_addressing_limited(struct device *dev);
- size_t dma_max_mapping_size(struct device *dev);
- size_t dma_opt_mapping_size(struct device *dev);
-bool dma_need_sync(struct device *dev, dma_addr_t dma_addr);
- unsigned long dma_get_merge_boundary(struct device *dev);
- struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size,
- 		enum dma_data_direction dir, gfp_t gfp, unsigned long attrs);
-@@ -168,6 +196,15 @@ static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,
- 		size_t size, enum dma_data_direction dir, unsigned long attrs)
- {
- }
-+static inline dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys,
-+		size_t size, enum dma_data_direction dir, unsigned long attrs)
-+{
-+	return DMA_MAPPING_ERROR;
-+}
-+static inline void dma_unmap_phys(struct device *dev, dma_addr_t addr,
-+		size_t size, enum dma_data_direction dir, unsigned long attrs)
-+{
-+}
- static inline unsigned int dma_map_sg_attrs(struct device *dev,
- 		struct scatterlist *sg, int nents, enum dma_data_direction dir,
- 		unsigned long attrs)
-@@ -194,22 +231,6 @@ static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
- 		size_t size, enum dma_data_direction dir, unsigned long attrs)
- {
- }
-static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
-		size_t size, enum dma_data_direction dir)
-{
-}
-static inline void dma_sync_single_for_device(struct device *dev,
-		dma_addr_t addr, size_t size, enum dma_data_direction dir)
-{
-}
-static inline void dma_sync_sg_for_cpu(struct device *dev,
-		struct scatterlist *sg, int nelems, enum dma_data_direction dir)
-{
-}
-static inline void dma_sync_sg_for_device(struct device *dev,
-		struct scatterlist *sg, int nelems, enum dma_data_direction dir)
-{
-}
- static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
- {
- 	return -ENOMEM;
-@@ -248,10 +269,6 @@ static inline bool dma_can_mmap(struct device *dev)
- {
- 	return false;
- }
-static inline int dma_supported(struct device *dev, u64 mask)
-{
-	return 0;
-}
- static inline bool dma_pci_p2pdma_supported(struct device *dev)
- {
- 	return false;
-@@ -268,6 +285,10 @@ static inline u64 dma_get_required_mask(struct device *dev)
- {
- 	return 0;
- }
-+static inline bool dma_addressing_limited(struct device *dev)
-+{
-+	return false;
-+}
- static inline size_t dma_max_mapping_size(struct device *dev)
- {
- 	return 0;
-@@ -276,10 +297,6 @@ static inline size_t dma_opt_mapping_size(struct device *dev)
- {
- 	return 0;
- }
-static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr)
-{
-	return false;
-}
- static inline unsigned long dma_get_merge_boundary(struct device *dev)
- {
- 	return 0;
-@@ -309,6 +326,151 @@ static inline int dma_mmap_noncontiguous(struct device *dev,
- }
- #endif /* CONFIG_HAS_DMA */
- 
-+#ifdef CONFIG_IOMMU_DMA
-+/**
-+ * dma_use_iova - check if the IOVA API is used for this state
-+ * @state: IOVA state
-+ *
-+ * Return %true if the DMA transfers uses the dma_iova_*() calls or %false if
-+ * they can't be used.
-+ */
-+static inline bool dma_use_iova(struct dma_iova_state *state)
-+{
-+	return state->__size != 0;
-+}
-+
-+bool dma_iova_try_alloc(struct device *dev, struct dma_iova_state *state,
-+		phys_addr_t phys, size_t size);
-+void dma_iova_free(struct device *dev, struct dma_iova_state *state);
-+void dma_iova_destroy(struct device *dev, struct dma_iova_state *state,
-+		size_t mapped_len, enum dma_data_direction dir,
-+		unsigned long attrs);
-+int dma_iova_sync(struct device *dev, struct dma_iova_state *state,
-+		size_t offset, size_t size);
-+int dma_iova_link(struct device *dev, struct dma_iova_state *state,
-+		phys_addr_t phys, size_t offset, size_t size,
-+		enum dma_data_direction dir, unsigned long attrs);
-+void dma_iova_unlink(struct device *dev, struct dma_iova_state *state,
-+		size_t offset, size_t size, enum dma_data_direction dir,
-+		unsigned long attrs);
-+#else /* CONFIG_IOMMU_DMA */
-+static inline bool dma_use_iova(struct dma_iova_state *state)
-+{
-+	return false;
-+}
-+static inline bool dma_iova_try_alloc(struct device *dev,
-+		struct dma_iova_state *state, phys_addr_t phys, size_t size)
-+{
-+	return false;
-+}
-+static inline void dma_iova_free(struct device *dev,
-+		struct dma_iova_state *state)
-+{
-+}
-+static inline void dma_iova_destroy(struct device *dev,
-+		struct dma_iova_state *state, size_t mapped_len,
-+		enum dma_data_direction dir, unsigned long attrs)
-+{
-+}
-+static inline int dma_iova_sync(struct device *dev,
-+		struct dma_iova_state *state, size_t offset, size_t size)
-+{
-+	return -EOPNOTSUPP;
-+}
-+static inline int dma_iova_link(struct device *dev,
-+		struct dma_iova_state *state, phys_addr_t phys, size_t offset,
-+		size_t size, enum dma_data_direction dir, unsigned long attrs)
-+{
-+	return -EOPNOTSUPP;
-+}
-+static inline void dma_iova_unlink(struct device *dev,
-+		struct dma_iova_state *state, size_t offset, size_t size,
-+		enum dma_data_direction dir, unsigned long attrs)
-+{
-+}
-+#endif /* CONFIG_IOMMU_DMA */
-+
-+#if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC)
-+void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
-+		enum dma_data_direction dir);
-+void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
-+		size_t size, enum dma_data_direction dir);
-+void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
-+		int nelems, enum dma_data_direction dir);
-+void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
-+		int nelems, enum dma_data_direction dir);
-+bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr);
-+
-+static inline bool dma_dev_need_sync(const struct device *dev)
-+{
-+	/* Always call DMA sync operations when debugging is enabled */
-+	return !dev->dma_skip_sync || IS_ENABLED(CONFIG_DMA_API_DEBUG);
-+}
-+
-+static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
-+		size_t size, enum dma_data_direction dir)
-+{
-+	if (dma_dev_need_sync(dev))
-+		__dma_sync_single_for_cpu(dev, addr, size, dir);
-+}
-+
-+static inline void dma_sync_single_for_device(struct device *dev,
-+		dma_addr_t addr, size_t size, enum dma_data_direction dir)
-+{
-+	if (dma_dev_need_sync(dev))
-+		__dma_sync_single_for_device(dev, addr, size, dir);
-+}
-+
-+static inline void dma_sync_sg_for_cpu(struct device *dev,
-+		struct scatterlist *sg, int nelems, enum dma_data_direction dir)
-+{
-+	if (dma_dev_need_sync(dev))
-+		__dma_sync_sg_for_cpu(dev, sg, nelems, dir);
-+}
-+
-+static inline void dma_sync_sg_for_device(struct device *dev,
-+		struct scatterlist *sg, int nelems, enum dma_data_direction dir)
-+{
-+	if (dma_dev_need_sync(dev))
-+		__dma_sync_sg_for_device(dev, sg, nelems, dir);
-+}
-+
-+static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr)
-+{
-+	return dma_dev_need_sync(dev) ? __dma_need_sync(dev, dma_addr) : false;
-+}
-+bool dma_need_unmap(struct device *dev);
-+#else /* !CONFIG_HAS_DMA || !CONFIG_DMA_NEED_SYNC */
-+static inline bool dma_dev_need_sync(const struct device *dev)
-+{
-+	return false;
-+}
-+static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
-+		size_t size, enum dma_data_direction dir)
-+{
-+}
-+static inline void dma_sync_single_for_device(struct device *dev,
-+		dma_addr_t addr, size_t size, enum dma_data_direction dir)
-+{
-+}
-+static inline void dma_sync_sg_for_cpu(struct device *dev,
-+		struct scatterlist *sg, int nelems, enum dma_data_direction dir)
-+{
-+}
-+static inline void dma_sync_sg_for_device(struct device *dev,
-+		struct scatterlist *sg, int nelems, enum dma_data_direction dir)
-+{
-+}
-+static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr)
-+{
-+	return false;
-+}
-+static inline bool dma_need_unmap(struct device *dev)
-+{
-+	return false;
-+}
-+#endif /* !CONFIG_HAS_DMA || !CONFIG_DMA_NEED_SYNC */
-+
- struct page *dma_alloc_pages(struct device *dev, size_t size,
- 		dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp);
- void dma_free_pages(struct device *dev, size_t size, struct page *page,
-@@ -422,6 +584,8 @@ static inline void dma_sync_sgtable_for_device(struct device *dev,
- #define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0)
- #define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0)
- 
-+bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size);
-+
- static inline void *dma_alloc_coherent(struct device *dev, size_t size,
- 		dma_addr_t *dma_handle, gfp_t gfp)
- {
-@@ -467,20 +631,6 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
- 	return dma_set_mask_and_coherent(dev, mask);
- }
- 
-/**
- * dma_addressing_limited - return if the device is addressing limited
- * @dev:	device to check
- *
- * Return %true if the devices DMA mask is too small to address all memory in
- * the system, else %false.  Lack of addressing bits is the prime reason for
- * bounce buffering, but might not be the only one.
- */
-static inline bool dma_addressing_limited(struct device *dev)
-{
-	return min_not_zero(dma_get_mask(dev), dev->bus_dma_limit) <
-			    dma_get_required_mask(dev);
-}
-
- static inline unsigned int dma_get_max_seg_size(struct device *dev)
- {
- 	if (dev->dma_parms && dev->dma_parms->max_segment_size)
-@@ -488,13 +638,11 @@ static inline unsigned int dma_get_max_seg_size(struct device *dev)
- 	return SZ_64K;
- }
- 
-static inline int dma_set_max_seg_size(struct device *dev, unsigned int size)
-+static inline void dma_set_max_seg_size(struct device *dev, unsigned int size)
- {
-	if (dev->dma_parms) {
-		dev->dma_parms->max_segment_size = size;
-		return 0;
-	}
-	return -EIO;
-+	if (WARN_ON_ONCE(!dev->dma_parms))
-+		return;
-+	dev->dma_parms->max_segment_size = size;
- }
- 
- static inline unsigned long dma_get_seg_boundary(struct device *dev)
-@@ -523,13 +671,11 @@ static inline unsigned long dma_get_seg_boundary_nr_pages(struct device *dev,
- 	return (dma_get_seg_boundary(dev) >> page_shift) + 1;
- }
- 
-static inline int dma_set_seg_boundary(struct device *dev, unsigned long mask)
-+static inline void dma_set_seg_boundary(struct device *dev, unsigned long mask)
- {
-	if (dev->dma_parms) {
-		dev->dma_parms->segment_boundary_mask = mask;
-		return 0;
-	}
-	return -EIO;
-+	if (WARN_ON_ONCE(!dev->dma_parms))
-+		return;
-+	dev->dma_parms->segment_boundary_mask = mask;
- }
- 
- static inline unsigned int dma_get_min_align_mask(struct device *dev)
-@@ -539,22 +685,23 @@ static inline unsigned int dma_get_min_align_mask(struct device *dev)
- 	return 0;
- }
- 
-static inline int dma_set_min_align_mask(struct device *dev,
-+static inline void dma_set_min_align_mask(struct device *dev,
- 		unsigned int min_align_mask)
- {
- 	if (WARN_ON_ONCE(!dev->dma_parms))
-		return -EIO;
-+		return;
- 	dev->dma_parms->min_align_mask = min_align_mask;
-	return 0;
- }
- 
-+#ifndef dma_get_cache_alignment
- static inline int dma_get_cache_alignment(void)
- {
-#ifdef ARCH_DMA_MINALIGN
-+#ifdef ARCH_HAS_DMA_MINALIGN
- 	return ARCH_DMA_MINALIGN;
- #endif
- 	return 1;
- }
-+#endif
- 
- static inline void *dmam_alloc_coherent(struct device *dev, size_t size,
- 		dma_addr_t *dma_handle, gfp_t gfp)
-@@ -600,10 +747,14 @@ static inline int dma_mmap_wc(struct device *dev,
- #else
- #define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME)
- #define DEFINE_DMA_UNMAP_LEN(LEN_NAME)
-#define dma_unmap_addr(PTR, ADDR_NAME)           (0)
-#define dma_unmap_addr_set(PTR, ADDR_NAME, VAL)  do { } while (0)
-#define dma_unmap_len(PTR, LEN_NAME)             (0)
-#define dma_unmap_len_set(PTR, LEN_NAME, VAL)    do { } while (0)
-+#define dma_unmap_addr(PTR, ADDR_NAME)           \
-+	({ typeof(PTR) __p __maybe_unused = PTR; 0; })
-+#define dma_unmap_addr_set(PTR, ADDR_NAME, VAL)  \
-+	do { typeof(PTR) __p __maybe_unused = PTR; } while (0)
-+#define dma_unmap_len(PTR, LEN_NAME)             \
-+	({ typeof(PTR) __p __maybe_unused = PTR; 0; })
-+#define dma_unmap_len_set(PTR, LEN_NAME, VAL)    \
-+	do { typeof(PTR) __p __maybe_unused = PTR; } while (0)
- #endif
- 
- #endif /* _LINUX_DMA_MAPPING_H */
-diff --git a/include/linux/dma-resv.h b/include/linux/dma-resv.h
-index c8ccbc94d5d2..c5ab6fd9ebe8 100644
--- a/include/linux/dma-resv.h
-+++ b/include/linux/dma-resv.h
-@@ -62,6 +62,11 @@ struct dma_resv_list;
-  * For example when asking for WRITE fences then the KERNEL fences are returned
-  * as well. Similar when asked for READ fences then both WRITE and KERNEL
-  * fences are returned as well.
-+ *
-+ * Already used fences can be promoted in the sense that a fence with
-+ * DMA_RESV_USAGE_BOOKKEEP could become DMA_RESV_USAGE_READ by adding it again
-+ * with this usage. But fences can never be degraded in the sense that a fence
-+ * with DMA_RESV_USAGE_WRITE could become DMA_RESV_USAGE_READ.
-  */
- enum dma_resv_usage {
- 	/**
-@@ -98,10 +103,15 @@ enum dma_resv_usage {
- 	 * @DMA_RESV_USAGE_BOOKKEEP: No implicit sync.
- 	 *
- 	 * This should be used by submissions which don't want to participate in
-	 * implicit synchronization.
-+	 * any implicit synchronization.
-+	 *
-+	 * The most common cases are preemption fences, page table updates, TLB
-+	 * flushes as well as explicitly synced user submissions.
- 	 *
-	 * The most common case are preemption fences as well as page table
-	 * updates and their TLB flushes.
-+	 * Explicitly synced user submissions can be promoted to
-+	 * DMA_RESV_USAGE_READ or DMA_RESV_USAGE_WRITE as needed using
-+	 * dma_buf_import_sync_file() when implicit synchronization should
-+	 * become necessary after initial adding of the fence.
- 	 */
- 	DMA_RESV_USAGE_BOOKKEEP
- };
-@@ -469,6 +479,8 @@ int dma_resv_get_singleton(struct dma_resv *obj, enum dma_resv_usage usage,
- int dma_resv_copy_fences(struct dma_resv *dst, struct dma_resv *src);
- long dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage,
- 			   bool intr, unsigned long timeout);
-+void dma_resv_set_deadline(struct dma_resv *obj, enum dma_resv_usage usage,
-+			   ktime_t deadline);
- bool dma_resv_test_signaled(struct dma_resv *obj, enum dma_resv_usage usage);
- void dma_resv_describe(struct dma_resv *obj, struct seq_file *seq);
- 
-diff --git a/include/linux/dma/amd_xdma.h b/include/linux/dma/amd_xdma.h
-new file mode 100644
-index 000000000000..ceba69ed7cb4
--- /dev/null
-+++ b/include/linux/dma/amd_xdma.h
-@@ -0,0 +1,16 @@
-+/* SPDX-License-Identifier: GPL-2.0-or-later */
-+/*
-+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
-+ */
-+
-+#ifndef _DMAENGINE_AMD_XDMA_H
-+#define _DMAENGINE_AMD_XDMA_H
-+
-+#include <linux/interrupt.h>
-+#include <linux/platform_device.h>
-+
-+int xdma_enable_user_irq(struct platform_device *pdev, u32 irq_num);
-+void xdma_disable_user_irq(struct platform_device *pdev, u32 irq_num);
-+int xdma_get_user_irq(struct platform_device *pdev, u32 user_irq_index);
-+
-+#endif /* _DMAENGINE_AMD_XDMA_H */
-diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h
-index 7d8062e9c544..3080747689f6 100644
--- a/include/linux/dma/edma.h
-+++ b/include/linux/dma/edma.h
-@@ -18,19 +18,38 @@
- struct dw_edma;
- 
- struct dw_edma_region {
-	phys_addr_t	paddr;
-	void __iomem	*vaddr;
-+	u64		paddr;
-+	union {
-+		void		*mem;
-+		void __iomem	*io;
-+	} vaddr;
- 	size_t		sz;
- };
- 
-struct dw_edma_core_ops {
-+/**
-+ * struct dw_edma_core_ops - platform-specific eDMA methods
-+ * @irq_vector:		Get IRQ number of the passed eDMA channel. Note the
-+ *			method accepts the channel id in the end-to-end
-+ *			numbering with the eDMA write channels being placed
-+ *			first in the row.
-+ * @pci_address:	Get PCIe bus address corresponding to the passed CPU
-+ *			address. Note there is no need in specifying this
-+ *			function if the address translation is performed by
-+ *			the DW PCIe RP/EP controller with the DW eDMA device in
-+ *			subject and DMA_BYPASS isn't set for all the outbound
-+ *			iATU windows. That will be done by the controller
-+ *			automatically.
-+ */
-+struct dw_edma_plat_ops {
- 	int (*irq_vector)(struct device *dev, unsigned int nr);
-+	u64 (*pci_address)(struct device *dev, phys_addr_t cpu_addr);
- };
- 
- enum dw_edma_map_format {
- 	EDMA_MF_EDMA_LEGACY = 0x0,
- 	EDMA_MF_EDMA_UNROLL = 0x1,
-	EDMA_MF_HDMA_COMPAT = 0x5
-+	EDMA_MF_HDMA_COMPAT = 0x5,
-+	EDMA_MF_HDMA_NATIVE = 0x7,
- };
- 
- /**
-@@ -61,9 +80,8 @@ enum dw_edma_chip_flags {
-  */
- struct dw_edma_chip {
- 	struct device		*dev;
-	int			id;
- 	int			nr_irqs;
-	const struct dw_edma_core_ops   *ops;
-+	const struct dw_edma_plat_ops	*ops;
- 	u32			flags;
- 
- 	void __iomem		*reg_base;
-@@ -84,7 +102,7 @@ struct dw_edma_chip {
- };
- 
- /* Export to the platform drivers */
-#if IS_ENABLED(CONFIG_DW_EDMA)
-+#if IS_REACHABLE(CONFIG_DW_EDMA)
- int dw_edma_probe(struct dw_edma_chip *chip);
- int dw_edma_remove(struct dw_edma_chip *chip);
- #else
-diff --git a/include/linux/dma/hsu.h b/include/linux/dma/hsu.h
-index a6b7bc707356..77ea602c287c 100644
--- a/include/linux/dma/hsu.h
-+++ b/include/linux/dma/hsu.h
-@@ -8,11 +8,13 @@
- #ifndef _DMA_HSU_H
- #define _DMA_HSU_H
- 
-#include <linux/device.h>
-#include <linux/interrupt.h>
-+#include <linux/errno.h>
-+#include <linux/kconfig.h>
-+#include <linux/types.h>
- 
- #include <linux/platform_data/dma-hsu.h>
- 
-+struct device;
- struct hsu_dma;
- 
- /**
-diff --git a/include/linux/dma/imx-dma.h b/include/linux/dma/imx-dma.h
-index f487a4fa103a..76a8de9ae151 100644
--- a/include/linux/dma/imx-dma.h
-+++ b/include/linux/dma/imx-dma.h
-@@ -40,6 +40,8 @@ enum sdma_peripheral_type {
- 	IMX_DMATYPE_ASRC_SP,	/* Shared ASRC */
- 	IMX_DMATYPE_SAI,	/* SAI */
- 	IMX_DMATYPE_MULTI_SAI,	/* MULTI FIFOs For Audio */
-+	IMX_DMATYPE_HDMI,       /* HDMI Audio */
-+	IMX_DMATYPE_I2C,	/* I2C */
- };
- 
- enum imx_dma_prio {
-diff --git a/include/linux/dma/ipu-dma.h b/include/linux/dma/ipu-dma.h
-deleted file mode 100644
-index 6969391580d2..000000000000
--- a/include/linux/dma/ipu-dma.h
-+++ /dev/null
-@@ -1,174 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2008
- * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de>
- *
- * Copyright (C) 2005-2007 Freescale Semiconductor, Inc.
- */
-
-#ifndef __LINUX_DMA_IPU_DMA_H
-#define __LINUX_DMA_IPU_DMA_H
-
-#include <linux/types.h>
-#include <linux/dmaengine.h>
-
-/* IPU DMA Controller channel definitions. */
-enum ipu_channel {
-	IDMAC_IC_0 = 0,		/* IC (encoding task) to memory */
-	IDMAC_IC_1 = 1,		/* IC (viewfinder task) to memory */
-	IDMAC_ADC_0 = 1,
-	IDMAC_IC_2 = 2,
-	IDMAC_ADC_1 = 2,
-	IDMAC_IC_3 = 3,
-	IDMAC_IC_4 = 4,
-	IDMAC_IC_5 = 5,
-	IDMAC_IC_6 = 6,
-	IDMAC_IC_7 = 7,		/* IC (sensor data) to memory */
-	IDMAC_IC_8 = 8,
-	IDMAC_IC_9 = 9,
-	IDMAC_IC_10 = 10,
-	IDMAC_IC_11 = 11,
-	IDMAC_IC_12 = 12,
-	IDMAC_IC_13 = 13,
-	IDMAC_SDC_0 = 14,	/* Background synchronous display data */
-	IDMAC_SDC_1 = 15,	/* Foreground data (overlay) */
-	IDMAC_SDC_2 = 16,
-	IDMAC_SDC_3 = 17,
-	IDMAC_ADC_2 = 18,
-	IDMAC_ADC_3 = 19,
-	IDMAC_ADC_4 = 20,
-	IDMAC_ADC_5 = 21,
-	IDMAC_ADC_6 = 22,
-	IDMAC_ADC_7 = 23,
-	IDMAC_PF_0 = 24,
-	IDMAC_PF_1 = 25,
-	IDMAC_PF_2 = 26,
-	IDMAC_PF_3 = 27,
-	IDMAC_PF_4 = 28,
-	IDMAC_PF_5 = 29,
-	IDMAC_PF_6 = 30,
-	IDMAC_PF_7 = 31,
-};
-
-/* Order significant! */
-enum ipu_channel_status {
-	IPU_CHANNEL_FREE,
-	IPU_CHANNEL_INITIALIZED,
-	IPU_CHANNEL_READY,
-	IPU_CHANNEL_ENABLED,
-};
-
-#define IPU_CHANNELS_NUM 32
-
-enum pixel_fmt {
-	/* 1 byte */
-	IPU_PIX_FMT_GENERIC,
-	IPU_PIX_FMT_RGB332,
-	IPU_PIX_FMT_YUV420P,
-	IPU_PIX_FMT_YUV422P,
-	IPU_PIX_FMT_YUV420P2,
-	IPU_PIX_FMT_YVU422P,
-	/* 2 bytes */
-	IPU_PIX_FMT_RGB565,
-	IPU_PIX_FMT_RGB666,
-	IPU_PIX_FMT_BGR666,
-	IPU_PIX_FMT_YUYV,
-	IPU_PIX_FMT_UYVY,
-	/* 3 bytes */
-	IPU_PIX_FMT_RGB24,
-	IPU_PIX_FMT_BGR24,
-	/* 4 bytes */
-	IPU_PIX_FMT_GENERIC_32,
-	IPU_PIX_FMT_RGB32,
-	IPU_PIX_FMT_BGR32,
-	IPU_PIX_FMT_ABGR32,
-	IPU_PIX_FMT_BGRA32,
-	IPU_PIX_FMT_RGBA32,
-};
-
-enum ipu_color_space {
-	IPU_COLORSPACE_RGB,
-	IPU_COLORSPACE_YCBCR,
-	IPU_COLORSPACE_YUV
-};
-
-/*
- * Enumeration of IPU rotation modes
- */
-enum ipu_rotate_mode {
-	/* Note the enum values correspond to BAM value */
-	IPU_ROTATE_NONE = 0,
-	IPU_ROTATE_VERT_FLIP = 1,
-	IPU_ROTATE_HORIZ_FLIP = 2,
-	IPU_ROTATE_180 = 3,
-	IPU_ROTATE_90_RIGHT = 4,
-	IPU_ROTATE_90_RIGHT_VFLIP = 5,
-	IPU_ROTATE_90_RIGHT_HFLIP = 6,
-	IPU_ROTATE_90_LEFT = 7,
-};
-
-/*
- * Enumeration of DI ports for ADC.
- */
-enum display_port {
-	DISP0,
-	DISP1,
-	DISP2,
-	DISP3
-};
-
-struct idmac_video_param {
-	unsigned short		in_width;
-	unsigned short		in_height;
-	uint32_t		in_pixel_fmt;
-	unsigned short		out_width;
-	unsigned short		out_height;
-	uint32_t		out_pixel_fmt;
-	unsigned short		out_stride;
-	bool			graphics_combine_en;
-	bool			global_alpha_en;
-	bool			key_color_en;
-	enum display_port	disp;
-	unsigned short		out_left;
-	unsigned short		out_top;
-};
-
-/*
- * Union of initialization parameters for a logical channel. So far only video
- * parameters are used.
- */
-union ipu_channel_param {
-	struct idmac_video_param video;
-};
-
-struct idmac_tx_desc {
-	struct dma_async_tx_descriptor	txd;
-	struct scatterlist		*sg;	/* scatterlist for this */
-	unsigned int			sg_len;	/* tx-descriptor. */
-	struct list_head		list;
-};
-
-struct idmac_channel {
-	struct dma_chan		dma_chan;
-	dma_cookie_t		completed;	/* last completed cookie	   */
-	union ipu_channel_param	params;
-	enum ipu_channel	link;	/* input channel, linked to the output	   */
-	enum ipu_channel_status	status;
-	void			*client;	/* Only one client per channel	   */
-	unsigned int		n_tx_desc;
-	struct idmac_tx_desc	*desc;		/* allocated tx-descriptors	   */
-	struct scatterlist	*sg[2];	/* scatterlist elements in buffer-0 and -1 */
-	struct list_head	free_list;	/* free tx-descriptors		   */
-	struct list_head	queue;		/* queued tx-descriptors	   */
-	spinlock_t		lock;		/* protects sg[0,1], queue	   */
-	struct mutex		chan_mutex; /* protects status, cookie, free_list  */
-	bool			sec_chan_en;
-	int			active_buffer;
-	unsigned int		eof_irq;
-	char			eof_name[16];	/* EOF IRQ name for request_irq()  */
-};
-
-#define to_tx_desc(tx) container_of(tx, struct idmac_tx_desc, txd)
-#define to_idmac_chan(c) container_of(c, struct idmac_channel, dma_chan)
-
-#endif /* __LINUX_DMA_IPU_DMA_H */
-diff --git a/include/linux/dma/k3-udma-glue.h b/include/linux/dma/k3-udma-glue.h
-index e443be4d3b4b..5d43881e6fb7 100644
--- a/include/linux/dma/k3-udma-glue.h
-+++ b/include/linux/dma/k3-udma-glue.h
-@@ -26,6 +26,11 @@ struct k3_udma_glue_tx_channel;
- struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
- 		const char *name, struct k3_udma_glue_tx_channel_cfg *cfg);
- 
-+struct k3_udma_glue_tx_channel *
-+k3_udma_glue_request_tx_chn_for_thread_id(struct device *dev,
-+					  struct k3_udma_glue_tx_channel_cfg *cfg,
-+					  struct device_node *udmax_np, u32 thread_id);
-+
- void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn);
- int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
- 			     struct cppi5_host_desc_t *desc_tx,
-@@ -109,6 +114,11 @@ struct k3_udma_glue_rx_channel *k3_udma_glue_request_rx_chn(
- 		const char *name,
- 		struct k3_udma_glue_rx_channel_cfg *cfg);
- 
-+struct k3_udma_glue_rx_channel *
-+k3_udma_glue_request_remote_rx_chn_for_thread_id(struct device *dev,
-+						 struct k3_udma_glue_rx_channel_cfg *cfg,
-+						 struct device_node *udmax_np, u32 thread_id);
-+
- void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn);
- int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn);
- void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn);
-@@ -126,12 +136,9 @@ u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn,
- u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn);
- int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn,
- 			    u32 flow_num);
-void k3_udma_glue_rx_put_irq(struct k3_udma_glue_rx_channel *rx_chn,
-			     u32 flow_num);
- void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
- 		u32 flow_num, void *data,
-		void (*cleanup)(void *data, dma_addr_t desc_dma),
-		bool skip_fdq);
-+		void (*cleanup)(void *data, dma_addr_t desc_dma));
- int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn,
- 				u32 flow_idx);
- int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn,
-diff --git a/include/linux/dma/ti-cppi5.h b/include/linux/dma/ti-cppi5.h
-index efa2f0309f00..c53c0f6e3b1a 100644
--- a/include/linux/dma/ti-cppi5.h
-+++ b/include/linux/dma/ti-cppi5.h
-@@ -616,6 +616,7 @@ static inline void *cppi5_hdesc_get_swdata(struct cppi5_host_desc_t *desc)
- #define   CPPI5_TR_CSF_SUPR_EVT			BIT(2)
- #define   CPPI5_TR_CSF_EOL_ADV_SHIFT		(4U)
- #define   CPPI5_TR_CSF_EOL_ADV_MASK		GENMASK(6, 4)
-+#define   CPPI5_TR_CSF_EOL_ICNT0		BIT(4)
- #define   CPPI5_TR_CSF_EOP			BIT(7)
- 
- /**
-diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
-index c923f4e60f24..99efe2b9b4ea 100644
--- a/include/linux/dmaengine.h
-+++ b/include/linux/dmaengine.h
-@@ -84,7 +84,7 @@ enum dma_transfer_direction {
- 	DMA_TRANS_NONE,
- };
- 
-/**
-+/*
-  * Interleaved Transfer Request
-  * ----------------------------
-  * A chunk is collection of contiguous bytes to be transferred.
-@@ -161,6 +161,16 @@ struct dma_interleaved_template {
- };
- 
- /**
-+ * struct dma_vec - DMA vector
-+ * @addr: Bus address of the start of the vector
-+ * @len: Length in bytes of the DMA vector
-+ */
-+struct dma_vec {
-+	dma_addr_t addr;
-+	size_t len;
-+};
-+
-+/**
-  * enum dma_ctrl_flags - DMA flags to augment operation preparation,
-  *  control completion, and communicate status.
-  * @DMA_PREP_INTERRUPT - trigger an interrupt (callback) upon completion of
-@@ -213,7 +223,7 @@ enum sum_check_bits {
- };
- 
- /**
- * enum pq_check_flags - result of async_{xor,pq}_zero_sum operations
-+ * enum sum_check_flags - result of async_{xor,pq}_zero_sum operations
-  * @SUM_CHECK_P_RESULT - 1 if xor zero sum error, 0 otherwise
-  * @SUM_CHECK_Q_RESULT - 1 if reed-solomon zero sum error, 0 otherwise
-  */
-@@ -276,7 +286,7 @@ typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t;
-  *	pointer to the engine's metadata area
-  *   4. Read out the metadata from the pointer
-  *
- * Note: the two mode is not compatible and clients must use one mode for a
-+ * Warning: the two modes are not compatible and clients must use one mode for a
-  * descriptor.
-  */
- enum dma_desc_metadata_mode {
-@@ -394,7 +404,7 @@ enum dma_slave_buswidth {
-  * should be read (RX), if the source is memory this argument is
-  * ignored.
-  * @dst_addr: this is the physical address where DMA slave data
- * should be written (TX), if the source is memory this argument
-+ * should be written (TX), if the destination is memory this argument
-  * is ignored.
-  * @src_addr_width: this is the width in bytes of the source (RX)
-  * register where DMA data shall be read. If the source
-@@ -517,8 +527,6 @@ static inline const char *dma_chan_name(struct dma_chan *chan)
- 	return dev_name(&chan->dev->device);
- }
- 
-void dma_chan_cleanup(struct kref *kref);
-
- /**
-  * typedef dma_filter_fn - callback filter for dma_request_channel
-  * @chan: channel to be reviewed
-@@ -586,9 +594,13 @@ struct dma_descriptor_metadata_ops {
-  * @phys: physical address of the descriptor
-  * @chan: target channel for this operation
-  * @tx_submit: accept the descriptor, assign ordered cookie and mark the
- * descriptor pending. To be pushed on .issue_pending() call
-+ *	descriptor pending. To be pushed on .issue_pending() call
-+ * @desc_free: driver's callback function to free a resusable descriptor
-+ *	after completion
-  * @callback: routine to call after this operation is complete
-+ * @callback_result: error result from a DMA transaction
-  * @callback_param: general parameter to pass to the callback routine
-+ * @unmap: hook for generic DMA unmap data
-  * @desc_metadata_mode: core managed metadata mode to protect mixed use of
-  *	DESC_METADATA_CLIENT or DESC_METADATA_ENGINE. Otherwise
-  *	DESC_METADATA_NONE
-@@ -773,6 +785,7 @@ struct dma_filter {
- 
- /**
-  * struct dma_device - info on the entity supplying DMA services
-+ * @ref: reference is taken and put every time a channel is allocated or freed
-  * @chancnt: how many DMA channels are supported
-  * @privatecnt: how many DMA channels are requested by dma_request_channel
-  * @channels: the list of struct dma_chan
-@@ -789,6 +802,7 @@ struct dma_filter {
-  * @dev_id: unique device ID
-  * @dev: struct device reference for dma mapping api
-  * @owner: owner module (automatically set based on the provided dev)
-+ * @chan_ida: unique channel ID
-  * @src_addr_widths: bit mask of src addr widths the device supports
-  *	Width is specified in bytes, e.g. for a device supporting
-  *	a width of 4 the mask should have BIT(4) set.
-@@ -802,6 +816,7 @@ struct dma_filter {
-  * @max_sg_burst: max number of SG list entries executed in a single burst
-  *	DMA tansaction with no software intervention for reinitialization.
-  *	Zero value means unlimited number of entries.
-+ * @descriptor_reuse: a submitted transfer can be resubmitted after completion
-  * @residue_granularity: granularity of the transfer residue reported
-  *	by tx_status
-  * @device_alloc_chan_resources: allocate resources and return the
-@@ -816,12 +831,14 @@ struct dma_filter {
-  * @device_prep_dma_memset: prepares a memset operation
-  * @device_prep_dma_memset_sg: prepares a memset operation over a scatter list
-  * @device_prep_dma_interrupt: prepares an end of chain interrupt operation
-+ * @device_prep_peripheral_dma_vec: prepares a scatter-gather DMA transfer,
-+ *	where the address and size of each segment is located in one entry of
-+ *	the dma_vec array.
-  * @device_prep_slave_sg: prepares a slave dma operation
-  * @device_prep_dma_cyclic: prepare a cyclic dma operation suitable for audio.
-  *	The function takes a buffer of size buf_len. The callback function will
-  *	be called after period_len bytes have been transferred.
-  * @device_prep_interleaved_dma: Transfer expression in a generic way.
- * @device_prep_dma_imm_data: DMA's 8 byte immediate data to the dst address
-  * @device_caps: May be used to override the generic DMA slave capabilities
-  *	with per-channel specific ones
-  * @device_config: Pushes a new configuration to a channel, return 0 or an error
-@@ -839,7 +856,6 @@ struct dma_filter {
-  *	struct with auxiliary transfer status information, otherwise the call
-  *	will just return a simple status code
-  * @device_issue_pending: push pending transactions to hardware
- * @descriptor_reuse: a submitted transfer can be resubmitted after completion
-  * @device_release: called sometime atfer dma_async_device_unregister() is
-  *     called and there are no further references to this structure. This
-  *     must be implemented to free resources however many existing drivers
-@@ -847,6 +863,7 @@ struct dma_filter {
-  * @dbg_summary_show: optional routine to show contents in debugfs; default code
-  *     will be used when this is omitted, but custom code can show extra,
-  *     controller specific information.
-+ * @dbg_dev_root: the root folder in debugfs for this device
-  */
- struct dma_device {
- 	struct kref ref;
-@@ -855,7 +872,7 @@ struct dma_device {
- 	struct list_head channels;
- 	struct list_head global_node;
- 	struct dma_filter filter;
-	dma_cap_mask_t  cap_mask;
-+	dma_cap_mask_t cap_mask;
- 	enum dma_desc_metadata_mode desc_metadata_modes;
- 	unsigned short max_xor;
- 	unsigned short max_pq;
-@@ -909,6 +926,10 @@ struct dma_device {
- 	struct dma_async_tx_descriptor *(*device_prep_dma_interrupt)(
- 		struct dma_chan *chan, unsigned long flags);
- 
-+	struct dma_async_tx_descriptor *(*device_prep_peripheral_dma_vec)(
-+		struct dma_chan *chan, const struct dma_vec *vecs,
-+		size_t nents, enum dma_transfer_direction direction,
-+		unsigned long flags);
- 	struct dma_async_tx_descriptor *(*device_prep_slave_sg)(
- 		struct dma_chan *chan, struct scatterlist *sgl,
- 		unsigned int sg_len, enum dma_transfer_direction direction,
-@@ -920,14 +941,9 @@ struct dma_device {
- 	struct dma_async_tx_descriptor *(*device_prep_interleaved_dma)(
- 		struct dma_chan *chan, struct dma_interleaved_template *xt,
- 		unsigned long flags);
-	struct dma_async_tx_descriptor *(*device_prep_dma_imm_data)(
-		struct dma_chan *chan, dma_addr_t dst, u64 data,
-		unsigned long flags);
- 
-	void (*device_caps)(struct dma_chan *chan,
-			    struct dma_slave_caps *caps);
-	int (*device_config)(struct dma_chan *chan,
-			     struct dma_slave_config *config);
-+	void (*device_caps)(struct dma_chan *chan, struct dma_slave_caps *caps);
-+	int (*device_config)(struct dma_chan *chan, struct dma_slave_config *config);
- 	int (*device_pause)(struct dma_chan *chan);
- 	int (*device_resume)(struct dma_chan *chan);
- 	int (*device_terminate_all)(struct dma_chan *chan);
-@@ -954,7 +970,8 @@ static inline int dmaengine_slave_config(struct dma_chan *chan,
- 
- static inline bool is_slave_direction(enum dma_transfer_direction direction)
- {
-	return (direction == DMA_MEM_TO_DEV) || (direction == DMA_DEV_TO_MEM);
-+	return (direction == DMA_MEM_TO_DEV) || (direction == DMA_DEV_TO_MEM) ||
-+	       (direction == DMA_DEV_TO_DEV);
- }
- 
- static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_single(
-@@ -973,6 +990,25 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_single(
- 						  dir, flags, NULL);
- }
- 
-+/**
-+ * dmaengine_prep_peripheral_dma_vec() - Prepare a DMA scatter-gather descriptor
-+ * @chan: The channel to be used for this descriptor
-+ * @vecs: The array of DMA vectors that should be transferred
-+ * @nents: The number of DMA vectors in the array
-+ * @dir: Specifies the direction of the data transfer
-+ * @flags: DMA engine flags
-+ */
-+static inline struct dma_async_tx_descriptor *dmaengine_prep_peripheral_dma_vec(
-+	struct dma_chan *chan, const struct dma_vec *vecs, size_t nents,
-+	enum dma_transfer_direction dir, unsigned long flags)
-+{
-+	if (!chan || !chan->device || !chan->device->device_prep_peripheral_dma_vec)
-+		return NULL;
-+
-+	return chan->device->device_prep_peripheral_dma_vec(chan, vecs, nents,
-+							    dir, flags);
-+}
-+
- static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_sg(
- 	struct dma_chan *chan, struct scatterlist *sgl,	unsigned int sg_len,
- 	enum dma_transfer_direction dir, unsigned long flags)
-@@ -1488,6 +1524,7 @@ struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
- 
- struct dma_chan *dma_request_chan(struct device *dev, const char *name);
- struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
-+struct dma_chan *devm_dma_request_chan(struct device *dev, const char *name);
- 
- void dma_release_channel(struct dma_chan *chan);
- int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps);
-@@ -1524,6 +1561,12 @@ static inline struct dma_chan *dma_request_chan_by_mask(
- {
- 	return ERR_PTR(-ENODEV);
- }
-+
-+static inline struct dma_chan *devm_dma_request_chan(struct device *dev, const char *name)
-+{
-+	return ERR_PTR(-ENODEV);
-+}
-+
- static inline void dma_release_channel(struct dma_chan *chan)
- {
- }
-@@ -1575,7 +1618,8 @@ int dma_async_device_register(struct dma_device *device);
- int dmaenginem_async_device_register(struct dma_device *device);
- void dma_async_device_unregister(struct dma_device *device);
- int dma_async_device_channel_register(struct dma_device *device,
-				      struct dma_chan *chan);
-+				      struct dma_chan *chan,
-+				      const char *name);
- void dma_async_device_channel_unregister(struct dma_device *device,
- 					 struct dma_chan *chan);
- void dma_run_dependencies(struct dma_async_tx_descriptor *tx);
-@@ -1598,14 +1642,14 @@ static inline struct dma_chan
- {
- 	struct dma_chan *chan;
- 
-	chan = dma_request_slave_channel(dev, name);
-	if (chan)
-+	chan = dma_request_chan(dev, name);
-+	if (!IS_ERR(chan))
- 		return chan;
- 
- 	if (!fn || !fn_param)
- 		return NULL;
- 
-	return __dma_request_channel(&mask, fn, fn_param, NULL);
-+	return dma_request_channel(mask, fn, fn_param);
- }
- 
- static inline char *
-diff --git a/include/linux/dmapool.h b/include/linux/dmapool.h
-index f632ecfb4238..7d40b51933d1 100644
--- a/include/linux/dmapool.h
-+++ b/include/linux/dmapool.h
-@@ -11,6 +11,7 @@
- #ifndef LINUX_DMAPOOL_H
- #define	LINUX_DMAPOOL_H
- 
-+#include <linux/nodemask_types.h>
- #include <linux/scatterlist.h>
- #include <asm/io.h>
- 
-@@ -18,8 +19,8 @@ struct device;
- 
- #ifdef CONFIG_HAS_DMA
- 
-struct dma_pool *dma_pool_create(const char *name, struct device *dev, 
-			size_t size, size_t align, size_t allocation);
-+struct dma_pool *dma_pool_create_node(const char *name, struct device *dev,
-+		size_t size, size_t align, size_t boundary, int node);
- 
- void dma_pool_destroy(struct dma_pool *pool);
- 
-@@ -35,9 +36,12 @@ struct dma_pool *dmam_pool_create(const char *name, struct device *dev,
- void dmam_pool_destroy(struct dma_pool *pool);
- 
- #else /* !CONFIG_HAS_DMA */
-static inline struct dma_pool *dma_pool_create(const char *name,
-	struct device *dev, size_t size, size_t align, size_t allocation)
-{ return NULL; }
-+static inline struct dma_pool *dma_pool_create_node(const char *name,
-+		struct device *dev, size_t size, size_t align, size_t boundary,
-+		int node)
-+{
-+	return NULL;
-+}
- static inline void dma_pool_destroy(struct dma_pool *pool) { }
- static inline void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags,
- 				   dma_addr_t *handle) { return NULL; }
-@@ -49,6 +53,21 @@ static inline struct dma_pool *dmam_pool_create(const char *name,
- static inline void dmam_pool_destroy(struct dma_pool *pool) { }
- #endif /* !CONFIG_HAS_DMA */
- 
-+static inline struct dma_pool *dma_pool_create(const char *name,
-+		struct device *dev, size_t size, size_t align, size_t boundary)
-+{
-+	return dma_pool_create_node(name, dev, size, align, boundary,
-+				    NUMA_NO_NODE);
-+}
-+
-+/**
-+ * dma_pool_zalloc - Get a zero-initialized block of DMA coherent memory.
-+ * @pool: dma pool that will produce the block
-+ * @mem_flags: GFP_* bitmask
-+ * @handle: pointer to dma address of block
-+ *
-+ * Same as dma_pool_alloc(), but the returned memory is zeroed.
-+ */
- static inline void *dma_pool_zalloc(struct dma_pool *pool, gfp_t mem_flags,
- 				    dma_addr_t *handle)
- {
-diff --git a/include/linux/dmar.h b/include/linux/dmar.h
-index d81a51978d01..692b2b445761 100644
--- a/include/linux/dmar.h
-+++ b/include/linux/dmar.h
-@@ -39,6 +39,7 @@ struct dmar_drhd_unit {
- 	struct list_head list;		/* list of drhd units	*/
- 	struct  acpi_dmar_header *hdr;	/* ACPI header		*/
- 	u64	reg_base_addr;		/* register base address*/
-+	unsigned long reg_size;		/* size of register set */
- 	struct	dmar_dev_scope *devices;/* target device array	*/
- 	int	devices_cnt;		/* target device count	*/
- 	u16	segment;		/* PCI domain		*/
-@@ -105,8 +106,6 @@ static inline bool dmar_rcu_check(void)
- extern int dmar_table_init(void);
- extern int dmar_dev_scope_init(void);
- extern void dmar_register_bus_notifier(void);
-extern int dmar_parse_dev_scope(void *start, void *end, int *cnt,
-				struct dmar_dev_scope **devices, u16 segment);
- extern void *dmar_alloc_dev_scope(void *start, void *end, int *cnt);
- extern void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt);
- extern int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
-@@ -118,7 +117,7 @@ extern int dmar_remove_dev_scope(struct dmar_pci_notify_info *info,
- 				 int count);
- /* Intel IOMMU detection */
- void detect_intel_iommu(void);
-extern int enable_drhd_fault_handling(void);
-+extern int enable_drhd_fault_handling(unsigned int cpu);
- extern int dmar_device_add(acpi_handle handle);
- extern int dmar_device_remove(acpi_handle handle);
- 
-@@ -201,67 +200,74 @@ static inline void detect_intel_iommu(void)
- 
- struct irte {
- 	union {
-		/* Shared between remapped and posted mode*/
- 		struct {
-			__u64	present		: 1,  /*  0      */
-				fpd		: 1,  /*  1      */
-				__res0		: 6,  /*  2 -  6 */
-				avail		: 4,  /*  8 - 11 */
-				__res1		: 3,  /* 12 - 14 */
-				pst		: 1,  /* 15      */
-				vector		: 8,  /* 16 - 23 */
-				__res2		: 40; /* 24 - 63 */
-+			union {
-+				/* Shared between remapped and posted mode*/
-+				struct {
-+					__u64	present		: 1,  /*  0      */
-+						fpd		: 1,  /*  1      */
-+						__res0		: 6,  /*  2 -  6 */
-+						avail		: 4,  /*  8 - 11 */
-+						__res1		: 3,  /* 12 - 14 */
-+						pst		: 1,  /* 15      */
-+						vector		: 8,  /* 16 - 23 */
-+						__res2		: 40; /* 24 - 63 */
-+				};
-+
-+				/* Remapped mode */
-+				struct {
-+					__u64	r_present	: 1,  /*  0      */
-+						r_fpd		: 1,  /*  1      */
-+						dst_mode	: 1,  /*  2      */
-+						redir_hint	: 1,  /*  3      */
-+						trigger_mode	: 1,  /*  4      */
-+						dlvry_mode	: 3,  /*  5 -  7 */
-+						r_avail		: 4,  /*  8 - 11 */
-+						r_res0		: 4,  /* 12 - 15 */
-+						r_vector	: 8,  /* 16 - 23 */
-+						r_res1		: 8,  /* 24 - 31 */
-+						dest_id		: 32; /* 32 - 63 */
-+				};
-+
-+				/* Posted mode */
-+				struct {
-+					__u64	p_present	: 1,  /*  0      */
-+						p_fpd		: 1,  /*  1      */
-+						p_res0		: 6,  /*  2 -  7 */
-+						p_avail		: 4,  /*  8 - 11 */
-+						p_res1		: 2,  /* 12 - 13 */
-+						p_urgent	: 1,  /* 14      */
-+						p_pst		: 1,  /* 15      */
-+						p_vector	: 8,  /* 16 - 23 */
-+						p_res2		: 14, /* 24 - 37 */
-+						pda_l		: 26; /* 38 - 63 */
-+				};
-+				__u64 low;
-+			};
-+
-+			union {
-+				/* Shared between remapped and posted mode*/
-+				struct {
-+					__u64	sid		: 16,  /* 64 - 79  */
-+						sq		: 2,   /* 80 - 81  */
-+						svt		: 2,   /* 82 - 83  */
-+						__res3		: 44;  /* 84 - 127 */
-+				};
-+
-+				/* Posted mode*/
-+				struct {
-+					__u64	p_sid		: 16,  /* 64 - 79  */
-+						p_sq		: 2,   /* 80 - 81  */
-+						p_svt		: 2,   /* 82 - 83  */
-+						p_res3		: 12,  /* 84 - 95  */
-+						pda_h		: 32;  /* 96 - 127 */
-+				};
-+				__u64 high;
-+			};
- 		};
-
-		/* Remapped mode */
-		struct {
-			__u64	r_present	: 1,  /*  0      */
-				r_fpd		: 1,  /*  1      */
-				dst_mode	: 1,  /*  2      */
-				redir_hint	: 1,  /*  3      */
-				trigger_mode	: 1,  /*  4      */
-				dlvry_mode	: 3,  /*  5 -  7 */
-				r_avail		: 4,  /*  8 - 11 */
-				r_res0		: 4,  /* 12 - 15 */
-				r_vector	: 8,  /* 16 - 23 */
-				r_res1		: 8,  /* 24 - 31 */
-				dest_id		: 32; /* 32 - 63 */
-		};
-
-		/* Posted mode */
-		struct {
-			__u64	p_present	: 1,  /*  0      */
-				p_fpd		: 1,  /*  1      */
-				p_res0		: 6,  /*  2 -  7 */
-				p_avail		: 4,  /*  8 - 11 */
-				p_res1		: 2,  /* 12 - 13 */
-				p_urgent	: 1,  /* 14      */
-				p_pst		: 1,  /* 15      */
-				p_vector	: 8,  /* 16 - 23 */
-				p_res2		: 14, /* 24 - 37 */
-				pda_l		: 26; /* 38 - 63 */
-		};
-		__u64 low;
-	};
-
-	union {
-		/* Shared between remapped and posted mode*/
-		struct {
-			__u64	sid		: 16,  /* 64 - 79  */
-				sq		: 2,   /* 80 - 81  */
-				svt		: 2,   /* 82 - 83  */
-				__res3		: 44;  /* 84 - 127 */
-		};
-
-		/* Posted mode*/
-		struct {
-			__u64	p_sid		: 16,  /* 64 - 79  */
-				p_sq		: 2,   /* 80 - 81  */
-				p_svt		: 2,   /* 82 - 83  */
-				p_res3		: 12,  /* 84 - 95  */
-				pda_h		: 32;  /* 96 - 127 */
-		};
-		__u64 high;
-+#ifdef CONFIG_IRQ_REMAP
-+		__u128 irte;
-+#endif
- 	};
- };
- 
-@@ -286,7 +292,6 @@ static inline void dmar_copy_shared_irte(struct irte *dst, struct irte *src)
- struct irq_data;
- extern void dmar_msi_unmask(struct irq_data *data);
- extern void dmar_msi_mask(struct irq_data *data);
-extern void dmar_msi_read(int irq, struct msi_msg *msg);
- extern void dmar_msi_write(int irq, struct msi_msg *msg);
- extern int dmar_set_interrupt(struct intel_iommu *iommu);
- extern irqreturn_t dmar_fault(int irq, void *dev_id);
-diff --git a/include/linux/dnotify.h b/include/linux/dnotify.h
-index b1d26f9f1c9f..9f183a679277 100644
--- a/include/linux/dnotify.h
-+++ b/include/linux/dnotify.h
-@@ -30,7 +30,7 @@ struct dnotify_struct {
- 			    FS_MOVED_FROM | FS_MOVED_TO)
- 
- extern void dnotify_flush(struct file *, fl_owner_t);
-extern int fcntl_dirnotify(int, struct file *, unsigned long);
-+extern int fcntl_dirnotify(int, struct file *, unsigned int);
- 
- #else
- 
-@@ -38,7 +38,7 @@ static inline void dnotify_flush(struct file *filp, fl_owner_t id)
- {
- }
- 
-static inline int fcntl_dirnotify(int fd, struct file *filp, unsigned long arg)
-+static inline int fcntl_dirnotify(int fd, struct file *filp, unsigned int arg)
- {
- 	return -EINVAL;
- }
-diff --git a/include/linux/dpll.h b/include/linux/dpll.h
-new file mode 100644
-index 000000000000..562f520b23c2
--- /dev/null
-+++ b/include/linux/dpll.h
-@@ -0,0 +1,229 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ *  Copyright (c) 2023 Meta Platforms, Inc. and affiliates
-+ *  Copyright (c) 2023 Intel and affiliates
-+ */
-+
-+#ifndef __DPLL_H__
-+#define __DPLL_H__
-+
-+#include <uapi/linux/dpll.h>
-+#include <linux/device.h>
-+#include <linux/netlink.h>
-+#include <linux/netdevice.h>
-+#include <linux/rtnetlink.h>
-+
-+struct dpll_device;
-+struct dpll_pin;
-+struct dpll_pin_esync;
-+
-+struct dpll_device_ops {
-+	int (*mode_get)(const struct dpll_device *dpll, void *dpll_priv,
-+			enum dpll_mode *mode, struct netlink_ext_ack *extack);
-+	int (*lock_status_get)(const struct dpll_device *dpll, void *dpll_priv,
-+			       enum dpll_lock_status *status,
-+			       enum dpll_lock_status_error *status_error,
-+			       struct netlink_ext_ack *extack);
-+	int (*temp_get)(const struct dpll_device *dpll, void *dpll_priv,
-+			s32 *temp, struct netlink_ext_ack *extack);
-+	int (*clock_quality_level_get)(const struct dpll_device *dpll,
-+				       void *dpll_priv,
-+				       unsigned long *qls,
-+				       struct netlink_ext_ack *extack);
-+	int (*phase_offset_monitor_set)(const struct dpll_device *dpll,
-+					void *dpll_priv,
-+					enum dpll_feature_state state,
-+					struct netlink_ext_ack *extack);
-+	int (*phase_offset_monitor_get)(const struct dpll_device *dpll,
-+					void *dpll_priv,
-+					enum dpll_feature_state *state,
-+					struct netlink_ext_ack *extack);
-+	int (*phase_offset_avg_factor_set)(const struct dpll_device *dpll,
-+					   void *dpll_priv, u32 factor,
-+					   struct netlink_ext_ack *extack);
-+	int (*phase_offset_avg_factor_get)(const struct dpll_device *dpll,
-+					   void *dpll_priv, u32 *factor,
-+					   struct netlink_ext_ack *extack);
-+};
-+
-+struct dpll_pin_ops {
-+	int (*frequency_set)(const struct dpll_pin *pin, void *pin_priv,
-+			     const struct dpll_device *dpll, void *dpll_priv,
-+			     const u64 frequency,
-+			     struct netlink_ext_ack *extack);
-+	int (*frequency_get)(const struct dpll_pin *pin, void *pin_priv,
-+			     const struct dpll_device *dpll, void *dpll_priv,
-+			     u64 *frequency, struct netlink_ext_ack *extack);
-+	int (*direction_set)(const struct dpll_pin *pin, void *pin_priv,
-+			     const struct dpll_device *dpll, void *dpll_priv,
-+			     const enum dpll_pin_direction direction,
-+			     struct netlink_ext_ack *extack);
-+	int (*direction_get)(const struct dpll_pin *pin, void *pin_priv,
-+			     const struct dpll_device *dpll, void *dpll_priv,
-+			     enum dpll_pin_direction *direction,
-+			     struct netlink_ext_ack *extack);
-+	int (*state_on_pin_get)(const struct dpll_pin *pin, void *pin_priv,
-+				const struct dpll_pin *parent_pin,
-+				void *parent_pin_priv,
-+				enum dpll_pin_state *state,
-+				struct netlink_ext_ack *extack);
-+	int (*state_on_dpll_get)(const struct dpll_pin *pin, void *pin_priv,
-+				 const struct dpll_device *dpll,
-+				 void *dpll_priv, enum dpll_pin_state *state,
-+				 struct netlink_ext_ack *extack);
-+	int (*state_on_pin_set)(const struct dpll_pin *pin, void *pin_priv,
-+				const struct dpll_pin *parent_pin,
-+				void *parent_pin_priv,
-+				const enum dpll_pin_state state,
-+				struct netlink_ext_ack *extack);
-+	int (*state_on_dpll_set)(const struct dpll_pin *pin, void *pin_priv,
-+				 const struct dpll_device *dpll,
-+				 void *dpll_priv,
-+				 const enum dpll_pin_state state,
-+				 struct netlink_ext_ack *extack);
-+	int (*prio_get)(const struct dpll_pin *pin,  void *pin_priv,
-+			const struct dpll_device *dpll,  void *dpll_priv,
-+			u32 *prio, struct netlink_ext_ack *extack);
-+	int (*prio_set)(const struct dpll_pin *pin, void *pin_priv,
-+			const struct dpll_device *dpll, void *dpll_priv,
-+			const u32 prio, struct netlink_ext_ack *extack);
-+	int (*phase_offset_get)(const struct dpll_pin *pin, void *pin_priv,
-+				const struct dpll_device *dpll, void *dpll_priv,
-+				s64 *phase_offset,
-+				struct netlink_ext_ack *extack);
-+	int (*phase_adjust_get)(const struct dpll_pin *pin, void *pin_priv,
-+				const struct dpll_device *dpll, void *dpll_priv,
-+				s32 *phase_adjust,
-+				struct netlink_ext_ack *extack);
-+	int (*phase_adjust_set)(const struct dpll_pin *pin, void *pin_priv,
-+				const struct dpll_device *dpll, void *dpll_priv,
-+				const s32 phase_adjust,
-+				struct netlink_ext_ack *extack);
-+	int (*ffo_get)(const struct dpll_pin *pin, void *pin_priv,
-+		       const struct dpll_device *dpll, void *dpll_priv,
-+		       s64 *ffo, struct netlink_ext_ack *extack);
-+	int (*esync_set)(const struct dpll_pin *pin, void *pin_priv,
-+			 const struct dpll_device *dpll, void *dpll_priv,
-+			 u64 freq, struct netlink_ext_ack *extack);
-+	int (*esync_get)(const struct dpll_pin *pin, void *pin_priv,
-+			 const struct dpll_device *dpll, void *dpll_priv,
-+			 struct dpll_pin_esync *esync,
-+			 struct netlink_ext_ack *extack);
-+	int (*ref_sync_set)(const struct dpll_pin *pin, void *pin_priv,
-+			    const struct dpll_pin *ref_sync_pin,
-+			    void *ref_sync_pin_priv,
-+			    const enum dpll_pin_state state,
-+			    struct netlink_ext_ack *extack);
-+	int (*ref_sync_get)(const struct dpll_pin *pin, void *pin_priv,
-+			    const struct dpll_pin *ref_sync_pin,
-+			    void *ref_sync_pin_priv,
-+			    enum dpll_pin_state *state,
-+			    struct netlink_ext_ack *extack);
-+};
-+
-+struct dpll_pin_frequency {
-+	u64 min;
-+	u64 max;
-+};
-+
-+#define DPLL_PIN_FREQUENCY_RANGE(_min, _max)	\
-+	{					\
-+		.min = _min,			\
-+		.max = _max,			\
-+	}
-+
-+#define DPLL_PIN_FREQUENCY(_val) DPLL_PIN_FREQUENCY_RANGE(_val, _val)
-+#define DPLL_PIN_FREQUENCY_1PPS \
-+	DPLL_PIN_FREQUENCY(DPLL_PIN_FREQUENCY_1_HZ)
-+#define DPLL_PIN_FREQUENCY_10MHZ \
-+	DPLL_PIN_FREQUENCY(DPLL_PIN_FREQUENCY_10_MHZ)
-+#define DPLL_PIN_FREQUENCY_IRIG_B \
-+	DPLL_PIN_FREQUENCY(DPLL_PIN_FREQUENCY_10_KHZ)
-+#define DPLL_PIN_FREQUENCY_DCF77 \
-+	DPLL_PIN_FREQUENCY(DPLL_PIN_FREQUENCY_77_5_KHZ)
-+
-+struct dpll_pin_phase_adjust_range {
-+	s32 min;
-+	s32 max;
-+};
-+
-+struct dpll_pin_esync {
-+	u64 freq;
-+	const struct dpll_pin_frequency *range;
-+	u8 range_num;
-+	u8 pulse;
-+};
-+
-+struct dpll_pin_properties {
-+	const char *board_label;
-+	const char *panel_label;
-+	const char *package_label;
-+	enum dpll_pin_type type;
-+	unsigned long capabilities;
-+	u32 freq_supported_num;
-+	struct dpll_pin_frequency *freq_supported;
-+	struct dpll_pin_phase_adjust_range phase_range;
-+	u32 phase_gran;
-+};
-+
-+#if IS_ENABLED(CONFIG_DPLL)
-+void dpll_netdev_pin_set(struct net_device *dev, struct dpll_pin *dpll_pin);
-+void dpll_netdev_pin_clear(struct net_device *dev);
-+
-+size_t dpll_netdev_pin_handle_size(const struct net_device *dev);
-+int dpll_netdev_add_pin_handle(struct sk_buff *msg,
-+			       const struct net_device *dev);
-+#else
-+static inline void
-+dpll_netdev_pin_set(struct net_device *dev, struct dpll_pin *dpll_pin) { }
-+static inline void dpll_netdev_pin_clear(struct net_device *dev) { }
-+
-+static inline size_t dpll_netdev_pin_handle_size(const struct net_device *dev)
-+{
-+	return 0;
-+}
-+
-+static inline int
-+dpll_netdev_add_pin_handle(struct sk_buff *msg, const struct net_device *dev)
-+{
-+	return 0;
-+}
-+#endif
-+
-+struct dpll_device *
-+dpll_device_get(u64 clock_id, u32 dev_driver_id, struct module *module);
-+
-+void dpll_device_put(struct dpll_device *dpll);
-+
-+int dpll_device_register(struct dpll_device *dpll, enum dpll_type type,
-+			 const struct dpll_device_ops *ops, void *priv);
-+
-+void dpll_device_unregister(struct dpll_device *dpll,
-+			    const struct dpll_device_ops *ops, void *priv);
-+
-+struct dpll_pin *
-+dpll_pin_get(u64 clock_id, u32 dev_driver_id, struct module *module,
-+	     const struct dpll_pin_properties *prop);
-+
-+int dpll_pin_register(struct dpll_device *dpll, struct dpll_pin *pin,
-+		      const struct dpll_pin_ops *ops, void *priv);
-+
-+void dpll_pin_unregister(struct dpll_device *dpll, struct dpll_pin *pin,
-+			 const struct dpll_pin_ops *ops, void *priv);
-+
-+void dpll_pin_put(struct dpll_pin *pin);
-+
-+int dpll_pin_on_pin_register(struct dpll_pin *parent, struct dpll_pin *pin,
-+			     const struct dpll_pin_ops *ops, void *priv);
-+
-+void dpll_pin_on_pin_unregister(struct dpll_pin *parent, struct dpll_pin *pin,
-+				const struct dpll_pin_ops *ops, void *priv);
-+
-+int dpll_pin_ref_sync_pair_add(struct dpll_pin *pin,
-+			       struct dpll_pin *ref_sync_pin);
-+
-+int dpll_device_change_ntf(struct dpll_device *dpll);
-+
-+int dpll_pin_change_ntf(struct dpll_pin *pin);
-+
-+#endif
-diff --git a/include/linux/drbd.h b/include/linux/drbd.h
-index 5755537b51b1..5468a2399d48 100644
--- a/include/linux/drbd.h
-+++ b/include/linux/drbd.h
-@@ -38,13 +38,6 @@
- 
- #endif
- 
-extern const char *drbd_buildtag(void);
-#define REL_VERSION "8.4.11"
-#define API_VERSION 1
-#define PRO_VERSION_MIN 86
-#define PRO_VERSION_MAX 101
-
-
- enum drbd_io_error_p {
- 	EP_PASS_ON, /* FIXME should the better be named "Ignore"? */
- 	EP_CALL_HELPER,
-diff --git a/include/linux/drbd_config.h b/include/linux/drbd_config.h
-new file mode 100644
-index 000000000000..d215365c6bb1
--- /dev/null
-+++ b/include/linux/drbd_config.h
-@@ -0,0 +1,16 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * drbd_config.h
-+ * DRBD's compile time configuration.
-+ */
-+
-+#ifndef DRBD_CONFIG_H
-+#define DRBD_CONFIG_H
-+
-+extern const char *drbd_buildtag(void);
-+
-+#define REL_VERSION "8.4.11"
-+#define PRO_VERSION_MIN 86
-+#define PRO_VERSION_MAX 101
-+
-+#endif
-diff --git a/include/linux/drbd_genl_api.h b/include/linux/drbd_genl_api.h
-index bd62efc29002..70682c058027 100644
--- a/include/linux/drbd_genl_api.h
-+++ b/include/linux/drbd_genl_api.h
-@@ -47,7 +47,7 @@ enum drbd_state_info_bcast_reason {
- #undef linux
- 
- #include <linux/drbd.h>
-#define GENL_MAGIC_VERSION	API_VERSION
-+#define GENL_MAGIC_VERSION	1
- #define GENL_MAGIC_FAMILY	drbd
- #define GENL_MAGIC_FAMILY_HDRSZ	sizeof(struct drbd_genlmsghdr)
- #define GENL_MAGIC_INCLUDE_FILE <linux/drbd_genl.h>
-diff --git a/include/linux/drbd_limits.h b/include/linux/drbd_limits.h
-index 9e33f7038bea..5b042fb427e9 100644
--- a/include/linux/drbd_limits.h
-+++ b/include/linux/drbd_limits.h
-@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-+/* SPDX-License-Identifier: GPL-2.0-only */
- /*
-   drbd_limits.h
-   This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
-@@ -16,123 +16,123 @@
- 
- #define DEBUG_RANGE_CHECK 0
- 
-#define DRBD_MINOR_COUNT_MIN 1
-#define DRBD_MINOR_COUNT_MAX 255
-#define DRBD_MINOR_COUNT_DEF 32
-+#define DRBD_MINOR_COUNT_MIN 1U
-+#define DRBD_MINOR_COUNT_MAX 255U
-+#define DRBD_MINOR_COUNT_DEF 32U
- #define DRBD_MINOR_COUNT_SCALE '1'
- 
-#define DRBD_VOLUME_MAX 65535
-+#define DRBD_VOLUME_MAX 65534U
- 
-#define DRBD_DIALOG_REFRESH_MIN 0
-#define DRBD_DIALOG_REFRESH_MAX 600
-+#define DRBD_DIALOG_REFRESH_MIN 0U
-+#define DRBD_DIALOG_REFRESH_MAX 600U
- #define DRBD_DIALOG_REFRESH_SCALE '1'
- 
- /* valid port number */
-#define DRBD_PORT_MIN 1
-#define DRBD_PORT_MAX 0xffff
-+#define DRBD_PORT_MIN 1U
-+#define DRBD_PORT_MAX 0xffffU
- #define DRBD_PORT_SCALE '1'
- 
- /* startup { */
-   /* if you want more than 3.4 days, disable */
-#define DRBD_WFC_TIMEOUT_MIN 0
-#define DRBD_WFC_TIMEOUT_MAX 300000
-#define DRBD_WFC_TIMEOUT_DEF 0
-+#define DRBD_WFC_TIMEOUT_MIN 0U
-+#define DRBD_WFC_TIMEOUT_MAX 300000U
-+#define DRBD_WFC_TIMEOUT_DEF 0U
- #define DRBD_WFC_TIMEOUT_SCALE '1'
- 
-#define DRBD_DEGR_WFC_TIMEOUT_MIN 0
-#define DRBD_DEGR_WFC_TIMEOUT_MAX 300000
-#define DRBD_DEGR_WFC_TIMEOUT_DEF 0
-+#define DRBD_DEGR_WFC_TIMEOUT_MIN 0U
-+#define DRBD_DEGR_WFC_TIMEOUT_MAX 300000U
-+#define DRBD_DEGR_WFC_TIMEOUT_DEF 0U
- #define DRBD_DEGR_WFC_TIMEOUT_SCALE '1'
- 
-#define DRBD_OUTDATED_WFC_TIMEOUT_MIN 0
-#define DRBD_OUTDATED_WFC_TIMEOUT_MAX 300000
-#define DRBD_OUTDATED_WFC_TIMEOUT_DEF 0
-+#define DRBD_OUTDATED_WFC_TIMEOUT_MIN 0U
-+#define DRBD_OUTDATED_WFC_TIMEOUT_MAX 300000U
-+#define DRBD_OUTDATED_WFC_TIMEOUT_DEF 0U
- #define DRBD_OUTDATED_WFC_TIMEOUT_SCALE '1'
- /* }*/
- 
- /* net { */
-   /* timeout, unit centi seconds
-    * more than one minute timeout is not useful */
-#define DRBD_TIMEOUT_MIN 1
-#define DRBD_TIMEOUT_MAX 600
-#define DRBD_TIMEOUT_DEF 60       /* 6 seconds */
-+#define DRBD_TIMEOUT_MIN 1U
-+#define DRBD_TIMEOUT_MAX 600U
-+#define DRBD_TIMEOUT_DEF 60U       /* 6 seconds */
- #define DRBD_TIMEOUT_SCALE '1'
- 
-  /* If backing disk takes longer than disk_timeout, mark the disk as failed */
-#define DRBD_DISK_TIMEOUT_MIN 0    /* 0 = disabled */
-#define DRBD_DISK_TIMEOUT_MAX 6000 /* 10 Minutes */
-#define DRBD_DISK_TIMEOUT_DEF 0    /* disabled */
-+#define DRBD_DISK_TIMEOUT_MIN 0U    /* 0 = disabled */
-+#define DRBD_DISK_TIMEOUT_MAX 6000U /* 10 Minutes */
-+#define DRBD_DISK_TIMEOUT_DEF 0U    /* disabled */
- #define DRBD_DISK_TIMEOUT_SCALE '1'
- 
-   /* active connection retries when C_WF_CONNECTION */
-#define DRBD_CONNECT_INT_MIN 1
-#define DRBD_CONNECT_INT_MAX 120
-#define DRBD_CONNECT_INT_DEF 10   /* seconds */
-+#define DRBD_CONNECT_INT_MIN 1U
-+#define DRBD_CONNECT_INT_MAX 120U
-+#define DRBD_CONNECT_INT_DEF 10U   /* seconds */
- #define DRBD_CONNECT_INT_SCALE '1'
- 
-   /* keep-alive probes when idle */
-#define DRBD_PING_INT_MIN 1
-#define DRBD_PING_INT_MAX 120
-#define DRBD_PING_INT_DEF 10
-+#define DRBD_PING_INT_MIN 1U
-+#define DRBD_PING_INT_MAX 120U
-+#define DRBD_PING_INT_DEF 10U
- #define DRBD_PING_INT_SCALE '1'
- 
-  /* timeout for the ping packets.*/
-#define DRBD_PING_TIMEO_MIN  1
-#define DRBD_PING_TIMEO_MAX  300
-#define DRBD_PING_TIMEO_DEF  5
-+#define DRBD_PING_TIMEO_MIN  1U
-+#define DRBD_PING_TIMEO_MAX  300U
-+#define DRBD_PING_TIMEO_DEF  5U
- #define DRBD_PING_TIMEO_SCALE '1'
- 
-   /* max number of write requests between write barriers */
-#define DRBD_MAX_EPOCH_SIZE_MIN 1
-#define DRBD_MAX_EPOCH_SIZE_MAX 20000
-#define DRBD_MAX_EPOCH_SIZE_DEF 2048
-+#define DRBD_MAX_EPOCH_SIZE_MIN 1U
-+#define DRBD_MAX_EPOCH_SIZE_MAX 20000U
-+#define DRBD_MAX_EPOCH_SIZE_DEF 2048U
- #define DRBD_MAX_EPOCH_SIZE_SCALE '1'
- 
-   /* I don't think that a tcp send buffer of more than 10M is useful */
-#define DRBD_SNDBUF_SIZE_MIN  0
-#define DRBD_SNDBUF_SIZE_MAX  (10<<20)
-#define DRBD_SNDBUF_SIZE_DEF  0
-+#define DRBD_SNDBUF_SIZE_MIN  0U
-+#define DRBD_SNDBUF_SIZE_MAX  (10U<<20)
-+#define DRBD_SNDBUF_SIZE_DEF  0U
- #define DRBD_SNDBUF_SIZE_SCALE '1'
- 
-#define DRBD_RCVBUF_SIZE_MIN  0
-#define DRBD_RCVBUF_SIZE_MAX  (10<<20)
-#define DRBD_RCVBUF_SIZE_DEF  0
-+#define DRBD_RCVBUF_SIZE_MIN  0U
-+#define DRBD_RCVBUF_SIZE_MAX  (10U<<20)
-+#define DRBD_RCVBUF_SIZE_DEF  0U
- #define DRBD_RCVBUF_SIZE_SCALE '1'
- 
-   /* @4k PageSize -> 128kB - 512MB */
-#define DRBD_MAX_BUFFERS_MIN  32
-#define DRBD_MAX_BUFFERS_MAX  131072
-#define DRBD_MAX_BUFFERS_DEF  2048
-+#define DRBD_MAX_BUFFERS_MIN  32U
-+#define DRBD_MAX_BUFFERS_MAX  131072U
-+#define DRBD_MAX_BUFFERS_DEF  2048U
- #define DRBD_MAX_BUFFERS_SCALE '1'
- 
-   /* @4k PageSize -> 4kB - 512MB */
-#define DRBD_UNPLUG_WATERMARK_MIN  1
-#define DRBD_UNPLUG_WATERMARK_MAX  131072
-+#define DRBD_UNPLUG_WATERMARK_MIN  1U
-+#define DRBD_UNPLUG_WATERMARK_MAX  131072U
- #define DRBD_UNPLUG_WATERMARK_DEF (DRBD_MAX_BUFFERS_DEF/16)
- #define DRBD_UNPLUG_WATERMARK_SCALE '1'
- 
-   /* 0 is disabled.
-    * 200 should be more than enough even for very short timeouts */
-#define DRBD_KO_COUNT_MIN  0
-#define DRBD_KO_COUNT_MAX  200
-#define DRBD_KO_COUNT_DEF  7
-+#define DRBD_KO_COUNT_MIN  0U
-+#define DRBD_KO_COUNT_MAX  200U
-+#define DRBD_KO_COUNT_DEF  7U
- #define DRBD_KO_COUNT_SCALE '1'
- /* } */
- 
- /* syncer { */
-   /* FIXME allow rate to be zero? */
-#define DRBD_RESYNC_RATE_MIN 1
-+#define DRBD_RESYNC_RATE_MIN 1U
- /* channel bonding 10 GbE, or other hardware */
- #define DRBD_RESYNC_RATE_MAX (4 << 20)
-#define DRBD_RESYNC_RATE_DEF 250
-+#define DRBD_RESYNC_RATE_DEF 250U
- #define DRBD_RESYNC_RATE_SCALE 'k'  /* kilobytes */
- 
-#define DRBD_AL_EXTENTS_MIN  67
-+#define DRBD_AL_EXTENTS_MIN  67U
-   /* we use u16 as "slot number", (u16)~0 is "FREE".
-    * If you use >= 292 kB on-disk ring buffer,
-    * this is the maximum you can use: */
-#define DRBD_AL_EXTENTS_MAX  0xfffe
-#define DRBD_AL_EXTENTS_DEF  1237
-+#define DRBD_AL_EXTENTS_MAX  0xfffeU
-+#define DRBD_AL_EXTENTS_DEF  1237U
- #define DRBD_AL_EXTENTS_SCALE '1'
- 
- #define DRBD_MINOR_NUMBER_MIN  -1
-@@ -147,9 +147,9 @@
-  * the upper limit with 64bit kernel, enough ram and flexible meta data
-  * is 1 PiB, currently. */
- /* DRBD_MAX_SECTORS */
-#define DRBD_DISK_SIZE_MIN  0
-#define DRBD_DISK_SIZE_MAX  (1 * (2LLU << 40))
-#define DRBD_DISK_SIZE_DEF  0 /* = disabled = no user size... */
-+#define DRBD_DISK_SIZE_MIN  0LLU
-+#define DRBD_DISK_SIZE_MAX  (1LLU * (2LLU << 40))
-+#define DRBD_DISK_SIZE_DEF  0LLU /* = disabled = no user size... */
- #define DRBD_DISK_SIZE_SCALE 's'  /* sectors */
- 
- #define DRBD_ON_IO_ERROR_DEF EP_DETACH
-@@ -162,39 +162,39 @@
- #define DRBD_ON_CONGESTION_DEF OC_BLOCK
- #define DRBD_READ_BALANCING_DEF RB_PREFER_LOCAL
- 
-#define DRBD_MAX_BIO_BVECS_MIN 0
-#define DRBD_MAX_BIO_BVECS_MAX 128
-#define DRBD_MAX_BIO_BVECS_DEF 0
-+#define DRBD_MAX_BIO_BVECS_MIN 0U
-+#define DRBD_MAX_BIO_BVECS_MAX 128U
-+#define DRBD_MAX_BIO_BVECS_DEF 0U
- #define DRBD_MAX_BIO_BVECS_SCALE '1'
- 
-#define DRBD_C_PLAN_AHEAD_MIN  0
-#define DRBD_C_PLAN_AHEAD_MAX  300
-#define DRBD_C_PLAN_AHEAD_DEF  20
-+#define DRBD_C_PLAN_AHEAD_MIN  0U
-+#define DRBD_C_PLAN_AHEAD_MAX  300U
-+#define DRBD_C_PLAN_AHEAD_DEF  20U
- #define DRBD_C_PLAN_AHEAD_SCALE '1'
- 
-#define DRBD_C_DELAY_TARGET_MIN 1
-#define DRBD_C_DELAY_TARGET_MAX 100
-#define DRBD_C_DELAY_TARGET_DEF 10
-+#define DRBD_C_DELAY_TARGET_MIN 1U
-+#define DRBD_C_DELAY_TARGET_MAX 100U
-+#define DRBD_C_DELAY_TARGET_DEF 10U
- #define DRBD_C_DELAY_TARGET_SCALE '1'
- 
-#define DRBD_C_FILL_TARGET_MIN 0
-#define DRBD_C_FILL_TARGET_MAX (1<<20) /* 500MByte in sec */
-#define DRBD_C_FILL_TARGET_DEF 100 /* Try to place 50KiB in socket send buffer during resync */
-+#define DRBD_C_FILL_TARGET_MIN 0U
-+#define DRBD_C_FILL_TARGET_MAX (1U<<20) /* 500MByte in sec */
-+#define DRBD_C_FILL_TARGET_DEF 100U /* Try to place 50KiB in socket send buffer during resync */
- #define DRBD_C_FILL_TARGET_SCALE 's'  /* sectors */
- 
-#define DRBD_C_MAX_RATE_MIN     250
-#define DRBD_C_MAX_RATE_MAX     (4 << 20)
-#define DRBD_C_MAX_RATE_DEF     102400
-+#define DRBD_C_MAX_RATE_MIN     250U
-+#define DRBD_C_MAX_RATE_MAX     (4U << 20)
-+#define DRBD_C_MAX_RATE_DEF     102400U
- #define DRBD_C_MAX_RATE_SCALE	'k'  /* kilobytes */
- 
-#define DRBD_C_MIN_RATE_MIN     0
-#define DRBD_C_MIN_RATE_MAX     (4 << 20)
-#define DRBD_C_MIN_RATE_DEF     250
-+#define DRBD_C_MIN_RATE_MIN     0U
-+#define DRBD_C_MIN_RATE_MAX     (4U << 20)
-+#define DRBD_C_MIN_RATE_DEF     250U
- #define DRBD_C_MIN_RATE_SCALE	'k'  /* kilobytes */
- 
-#define DRBD_CONG_FILL_MIN	0
-#define DRBD_CONG_FILL_MAX	(10<<21) /* 10GByte in sectors */
-#define DRBD_CONG_FILL_DEF	0
-+#define DRBD_CONG_FILL_MIN	0U
-+#define DRBD_CONG_FILL_MAX	(10U<<21) /* 10GByte in sectors */
-+#define DRBD_CONG_FILL_DEF	0U
- #define DRBD_CONG_FILL_SCALE	's'  /* sectors */
- 
- #define DRBD_CONG_EXTENTS_MIN	DRBD_AL_EXTENTS_MIN
-@@ -204,48 +204,48 @@
- 
- #define DRBD_PROTOCOL_DEF DRBD_PROT_C
- 
-#define DRBD_DISK_BARRIER_DEF	0
-#define DRBD_DISK_FLUSHES_DEF	1
-#define DRBD_DISK_DRAIN_DEF	1
-#define DRBD_MD_FLUSHES_DEF	1
-#define DRBD_TCP_CORK_DEF	1
-#define DRBD_AL_UPDATES_DEF     1
-+#define DRBD_DISK_BARRIER_DEF	0U
-+#define DRBD_DISK_FLUSHES_DEF	1U
-+#define DRBD_DISK_DRAIN_DEF	1U
-+#define DRBD_MD_FLUSHES_DEF	1U
-+#define DRBD_TCP_CORK_DEF	1U
-+#define DRBD_AL_UPDATES_DEF     1U
- 
- /* We used to ignore the discard_zeroes_data setting.
-  * To not change established (and expected) behaviour,
-  * by default assume that, for discard_zeroes_data=0,
-  * we can make that an effective discard_zeroes_data=1,
-  * if we only explicitly zero-out unaligned partial chunks. */
-#define DRBD_DISCARD_ZEROES_IF_ALIGNED_DEF 1
-+#define DRBD_DISCARD_ZEROES_IF_ALIGNED_DEF 1U
- 
- /* Some backends pretend to support WRITE SAME,
-  * but fail such requests when they are actually submitted.
-  * This is to tell DRBD to not even try. */
-#define DRBD_DISABLE_WRITE_SAME_DEF 0
-+#define DRBD_DISABLE_WRITE_SAME_DEF 0U
- 
-#define DRBD_ALLOW_TWO_PRIMARIES_DEF	0
-#define DRBD_ALWAYS_ASBP_DEF	0
-#define DRBD_USE_RLE_DEF	1
-#define DRBD_CSUMS_AFTER_CRASH_ONLY_DEF 0
-+#define DRBD_ALLOW_TWO_PRIMARIES_DEF	0U
-+#define DRBD_ALWAYS_ASBP_DEF	0U
-+#define DRBD_USE_RLE_DEF	1U
-+#define DRBD_CSUMS_AFTER_CRASH_ONLY_DEF 0U
- 
-#define DRBD_AL_STRIPES_MIN     1
-#define DRBD_AL_STRIPES_MAX     1024
-#define DRBD_AL_STRIPES_DEF     1
-+#define DRBD_AL_STRIPES_MIN     1U
-+#define DRBD_AL_STRIPES_MAX     1024U
-+#define DRBD_AL_STRIPES_DEF     1U
- #define DRBD_AL_STRIPES_SCALE   '1'
- 
-#define DRBD_AL_STRIPE_SIZE_MIN   4
-#define DRBD_AL_STRIPE_SIZE_MAX   16777216
-#define DRBD_AL_STRIPE_SIZE_DEF   32
-+#define DRBD_AL_STRIPE_SIZE_MIN   4U
-+#define DRBD_AL_STRIPE_SIZE_MAX   16777216U
-+#define DRBD_AL_STRIPE_SIZE_DEF   32U
- #define DRBD_AL_STRIPE_SIZE_SCALE 'k' /* kilobytes */
- 
-#define DRBD_SOCKET_CHECK_TIMEO_MIN 0
-+#define DRBD_SOCKET_CHECK_TIMEO_MIN 0U
- #define DRBD_SOCKET_CHECK_TIMEO_MAX DRBD_PING_TIMEO_MAX
-#define DRBD_SOCKET_CHECK_TIMEO_DEF 0
-+#define DRBD_SOCKET_CHECK_TIMEO_DEF 0U
- #define DRBD_SOCKET_CHECK_TIMEO_SCALE '1'
- 
-#define DRBD_RS_DISCARD_GRANULARITY_MIN 0
-#define DRBD_RS_DISCARD_GRANULARITY_MAX (1<<20)  /* 1MiByte */
-#define DRBD_RS_DISCARD_GRANULARITY_DEF 0     /* disabled by default */
-+#define DRBD_RS_DISCARD_GRANULARITY_MIN 0U
-+#define DRBD_RS_DISCARD_GRANULARITY_MAX (1U<<20)  /* 1MiByte */
-+#define DRBD_RS_DISCARD_GRANULARITY_DEF 0U     /* disabled by default */
- #define DRBD_RS_DISCARD_GRANULARITY_SCALE '1' /* bytes */
- 
- #endif
-diff --git a/include/linux/dsa/8021q.h b/include/linux/dsa/8021q.h
-index 3ed117e299ec..d13aabdeb4b2 100644
--- a/include/linux/dsa/8021q.h
-+++ b/include/linux/dsa/8021q.h
-@@ -5,48 +5,25 @@
- #ifndef _NET_DSA_8021Q_H
- #define _NET_DSA_8021Q_H
- 
-#include <linux/refcount.h>
-#include <linux/types.h>
- #include <net/dsa.h>
-+#include <linux/types.h>
- 
-struct dsa_switch;
-struct dsa_port;
-struct sk_buff;
-struct net_device;
-
-struct dsa_tag_8021q_vlan {
-	struct list_head list;
-	int port;
-	u16 vid;
-	refcount_t refcount;
-};
-
-struct dsa_8021q_context {
-	struct dsa_switch *ds;
-	struct list_head vlans;
-	/* EtherType of RX VID, used for filtering on master interface */
-	__be16 proto;
-};
-+/* VBID is limited to three bits only and zero is reserved.
-+ * Only 7 bridges can be enumerated.
-+ */
-+#define DSA_TAG_8021Q_MAX_NUM_BRIDGES	7
- 
- int dsa_tag_8021q_register(struct dsa_switch *ds, __be16 proto);
- 
- void dsa_tag_8021q_unregister(struct dsa_switch *ds);
- 
- int dsa_tag_8021q_bridge_join(struct dsa_switch *ds, int port,
-			      struct dsa_bridge bridge);
-+			      struct dsa_bridge bridge, bool *tx_fwd_offload,
-+			      struct netlink_ext_ack *extack);
- 
- void dsa_tag_8021q_bridge_leave(struct dsa_switch *ds, int port,
- 				struct dsa_bridge bridge);
- 
-struct sk_buff *dsa_8021q_xmit(struct sk_buff *skb, struct net_device *netdev,
-			       u16 tpid, u16 tci);
-
-void dsa_8021q_rcv(struct sk_buff *skb, int *source_port, int *switch_id,
-		   int *vbid);
-
-struct net_device *dsa_tag_8021q_find_port_by_vbid(struct net_device *master,
-						   int vbid);
-
- u16 dsa_tag_8021q_bridge_vid(unsigned int bridge_num);
- 
- u16 dsa_tag_8021q_standalone_vid(const struct dsa_port *dp);
-diff --git a/include/linux/dsa/ksz_common.h b/include/linux/dsa/ksz_common.h
-new file mode 100644
-index 000000000000..576a99ca698d
--- /dev/null
-+++ b/include/linux/dsa/ksz_common.h
-@@ -0,0 +1,53 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/* Microchip switch tag common header
-+ *
-+ * Copyright (C) 2022 Microchip Technology Inc.
-+ */
-+
-+#ifndef _NET_DSA_KSZ_COMMON_H_
-+#define _NET_DSA_KSZ_COMMON_H_
-+
-+#include <net/dsa.h>
-+
-+/* All time stamps from the KSZ consist of 2 bits for seconds and 30 bits for
-+ * nanoseconds. This is NOT the same as 32 bits for nanoseconds.
-+ */
-+#define KSZ_TSTAMP_SEC_MASK  GENMASK(31, 30)
-+#define KSZ_TSTAMP_NSEC_MASK GENMASK(29, 0)
-+
-+static inline ktime_t ksz_decode_tstamp(u32 tstamp)
-+{
-+	u64 ns = FIELD_GET(KSZ_TSTAMP_SEC_MASK, tstamp) * NSEC_PER_SEC +
-+		 FIELD_GET(KSZ_TSTAMP_NSEC_MASK, tstamp);
-+
-+	return ns_to_ktime(ns);
-+}
-+
-+struct ksz_deferred_xmit_work {
-+	struct dsa_port *dp;
-+	struct sk_buff *skb;
-+	struct kthread_work work;
-+};
-+
-+struct ksz_tagger_data {
-+	void (*xmit_work_fn)(struct kthread_work *work);
-+	void (*hwtstamp_set_state)(struct dsa_switch *ds, bool on);
-+};
-+
-+struct ksz_skb_cb {
-+	struct sk_buff *clone;
-+	unsigned int ptp_type;
-+	bool update_correction;
-+	u32 tstamp;
-+};
-+
-+#define KSZ_SKB_CB(skb) \
-+	((struct ksz_skb_cb *)((skb)->cb))
-+
-+static inline struct ksz_tagger_data *
-+ksz_tagger_data(struct dsa_switch *ds)
-+{
-+	return ds->tagger_data;
-+}
-+
-+#endif /* _NET_DSA_KSZ_COMMON_H_ */
-diff --git a/include/linux/dsa/lan9303.h b/include/linux/dsa/lan9303.h
-index b4f22112ba75..3ce7cbcc37a3 100644
--- a/include/linux/dsa/lan9303.h
-+++ b/include/linux/dsa/lan9303.h
-@@ -5,8 +5,8 @@ struct lan9303;
- 
- struct lan9303_phy_ops {
- 	/* PHY 1 and 2 access*/
-	int	(*phy_read)(struct lan9303 *chip, int port, int regnum);
-	int	(*phy_write)(struct lan9303 *chip, int port,
-+	int	(*phy_read)(struct lan9303 *chip, int addr, int regnum);
-+	int	(*phy_write)(struct lan9303 *chip, int addr,
- 			     int regnum, u16 val);
- };
- 
-diff --git a/include/linux/dsa/ocelot.h b/include/linux/dsa/ocelot.h
-index dca2969015d8..620a3260fc08 100644
--- a/include/linux/dsa/ocelot.h
-+++ b/include/linux/dsa/ocelot.h
-@@ -5,6 +5,8 @@
- #ifndef _NET_DSA_TAG_OCELOT_H
- #define _NET_DSA_TAG_OCELOT_H
- 
-+#include <linux/if_bridge.h>
-+#include <linux/if_vlan.h>
- #include <linux/kthread.h>
- #include <linux/packing.h>
- #include <linux/skbuff.h>
-@@ -13,6 +15,7 @@
- struct ocelot_skb_cb {
- 	struct sk_buff *clone;
- 	unsigned int ptp_class; /* valid only for clones */
-+	unsigned long ptp_tx_time; /* valid only for clones */
- 	u32 tstamp_lo;
- 	u8 ptp_cmd;
- 	u8 ts_id;
-@@ -273,4 +276,49 @@ static inline u32 ocelot_ptp_rew_op(struct sk_buff *skb)
- 	return rew_op;
- }
- 
-+/**
-+ * ocelot_xmit_get_vlan_info: Determine VLAN_TCI and TAG_TYPE for injected frame
-+ * @skb: Pointer to socket buffer
-+ * @br: Pointer to bridge device that the port is under, if any
-+ * @vlan_tci:
-+ * @tag_type:
-+ *
-+ * If the port is under a VLAN-aware bridge, remove the VLAN header from the
-+ * payload and move it into the DSA tag, which will make the switch classify
-+ * the packet to the bridge VLAN. Otherwise, leave the classified VLAN at zero,
-+ * which is the pvid of standalone ports (OCELOT_STANDALONE_PVID), although not
-+ * of VLAN-unaware bridge ports (that would be ocelot_vlan_unaware_pvid()).
-+ * Anyway, VID 0 is fine because it is stripped on egress for these port modes,
-+ * and source address learning is not performed for packets injected from the
-+ * CPU anyway, so it doesn't matter that the VID is "wrong".
-+ */
-+static inline void ocelot_xmit_get_vlan_info(struct sk_buff *skb,
-+					     struct net_device *br,
-+					     u64 *vlan_tci, u64 *tag_type)
-+{
-+	struct vlan_ethhdr *hdr;
-+	u16 proto, tci;
-+
-+	if (!br || !br_vlan_enabled(br)) {
-+		*vlan_tci = 0;
-+		*tag_type = IFH_TAG_TYPE_C;
-+		return;
-+	}
-+
-+	hdr = (struct vlan_ethhdr *)skb_mac_header(skb);
-+	br_vlan_get_proto(br, &proto);
-+
-+	if (ntohs(hdr->h_vlan_proto) == proto) {
-+		vlan_remove_tag(skb, &tci);
-+		*vlan_tci = tci;
-+	} else {
-+		rcu_read_lock();
-+		br_vlan_get_pvid_rcu(br, &tci);
-+		rcu_read_unlock();
-+		*vlan_tci = tci;
-+	}
-+
-+	*tag_type = (proto != ETH_P_8021Q) ? IFH_TAG_TYPE_S : IFH_TAG_TYPE_C;
-+}
-+
- #endif
-diff --git a/include/linux/dsa/sja1105.h b/include/linux/dsa/sja1105.h
-index 159e43171ccc..b9dd35d4b8f5 100644
--- a/include/linux/dsa/sja1105.h
-+++ b/include/linux/dsa/sja1105.h
-@@ -28,7 +28,7 @@
- /* Source and Destination MAC of follow-up meta frames.
-  * Whereas the choice of SMAC only affects the unique identification of the
-  * switch as sender of meta frames, the DMAC must be an address that is present
- * in the DSA master port's multicast MAC filter.
-+ * in the DSA conduit port's multicast MAC filter.
-  * 01-80-C2-00-00-0E is a good choice for this, as all profiles of IEEE 1588
-  * over L2 use this address for some purpose already.
-  */
-@@ -48,13 +48,9 @@ struct sja1105_deferred_xmit_work {
- 
- /* Global tagger data */
- struct sja1105_tagger_data {
-	/* Tagger to switch */
- 	void (*xmit_work_fn)(struct kthread_work *work);
- 	void (*meta_tstamp_handler)(struct dsa_switch *ds, int port, u8 ts_id,
- 				    enum sja1110_meta_tstamp dir, u64 tstamp);
-	/* Switch to tagger */
-	bool (*rxtstamp_get_state)(struct dsa_switch *ds);
-	void (*rxtstamp_set_state)(struct dsa_switch *ds, bool on);
- };
- 
- struct sja1105_skb_cb {
-diff --git a/include/linux/dsa/tag_qca.h b/include/linux/dsa/tag_qca.h
-index 50be7cbd93a5..ee657452f122 100644
--- a/include/linux/dsa/tag_qca.h
-+++ b/include/linux/dsa/tag_qca.h
-@@ -45,8 +45,8 @@ struct sk_buff;
- 					QCA_HDR_MGMT_COMMAND_LEN + \
- 					QCA_HDR_MGMT_DATA1_LEN)
- 
-#define QCA_HDR_MGMT_DATA2_LEN		12 /* Other 12 byte for the mdio data */
-#define QCA_HDR_MGMT_PADDING_LEN	34 /* Padding to reach the min Ethernet packet */
-+#define QCA_HDR_MGMT_DATA2_LEN		28 /* Other 28 byte for the mdio data */
-+#define QCA_HDR_MGMT_PADDING_LEN	18 /* Padding to reach the min Ethernet packet */
- 
- #define QCA_HDR_MGMT_PKT_LEN		(QCA_HDR_MGMT_HEADER_LEN + \
- 					QCA_HDR_LEN + \
-@@ -61,9 +61,9 @@ struct sk_buff;
- 
- /* Special struct emulating a Ethernet header */
- struct qca_mgmt_ethhdr {
-	u32 command;		/* command bit 31:0 */
-	u32 seq;		/* seq 63:32 */
-	u32 mdio_data;		/* first 4byte mdio */
-+	__le32 command;		/* command bit 31:0 */
-+	__le32 seq;		/* seq 63:32 */
-+	__le32 mdio_data;		/* first 4byte mdio */
- 	__be16 hdr;		/* qca hdr */
- } __packed;
- 
-@@ -73,7 +73,7 @@ enum mdio_cmd {
- };
- 
- struct mib_ethhdr {
-	u32 data[3];		/* first 3 mib counter */
-+	__le32 data[3];		/* first 3 mib counter */
- 	__be16 hdr;		/* qca hdr */
- } __packed;
- 
-diff --git a/include/linux/dw_apb_timer.h b/include/linux/dw_apb_timer.h
-index 82ebf9223948..f8811c46b89e 100644
--- a/include/linux/dw_apb_timer.h
-+++ b/include/linux/dw_apb_timer.h
-@@ -34,9 +34,6 @@ struct dw_apb_clocksource {
- };
- 
- void dw_apb_clockevent_register(struct dw_apb_clock_event_device *dw_ced);
-void dw_apb_clockevent_pause(struct dw_apb_clock_event_device *dw_ced);
-void dw_apb_clockevent_resume(struct dw_apb_clock_event_device *dw_ced);
-void dw_apb_clockevent_stop(struct dw_apb_clock_event_device *dw_ced);
- 
- struct dw_apb_clock_event_device *
- dw_apb_clockevent_init(int cpu, const char *name, unsigned rating,
-diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h
-index dce631e678dd..05743900a116 100644
--- a/include/linux/dynamic_debug.h
-+++ b/include/linux/dynamic_debug.h
-@@ -6,6 +6,8 @@
- #include <linux/jump_label.h>
- #endif
- 
-+#include <linux/build_bug.h>
-+
- /*
-  * An instance of this structure is created in a special
-  * ELF section at every dynamic debug callsite.  At runtime,
-@@ -21,6 +23,9 @@ struct _ddebug {
- 	const char *filename;
- 	const char *format;
- 	unsigned int lineno:18;
-+#define CLS_BITS 6
-+	unsigned int class_id:CLS_BITS;
-+#define _DPRINTK_CLASS_DFLT		((1 << CLS_BITS) - 1)
- 	/*
- 	 * The flags field controls the behaviour at the callsite.
- 	 * The bits here are changed dynamically when the user
-@@ -32,10 +37,13 @@ struct _ddebug {
- #define _DPRINTK_FLAGS_INCL_FUNCNAME	(1<<2)
- #define _DPRINTK_FLAGS_INCL_LINENO	(1<<3)
- #define _DPRINTK_FLAGS_INCL_TID		(1<<4)
-+#define _DPRINTK_FLAGS_INCL_SOURCENAME	(1<<5)
-+#define _DPRINTK_FLAGS_INCL_STACK	(1<<6)
- 
- #define _DPRINTK_FLAGS_INCL_ANY		\
- 	(_DPRINTK_FLAGS_INCL_MODNAME | _DPRINTK_FLAGS_INCL_FUNCNAME |\
-	 _DPRINTK_FLAGS_INCL_LINENO  | _DPRINTK_FLAGS_INCL_TID)
-+	 _DPRINTK_FLAGS_INCL_LINENO  | _DPRINTK_FLAGS_INCL_TID |\
-+	 _DPRINTK_FLAGS_INCL_SOURCENAME | _DPRINTK_FLAGS_INCL_STACK)
- 
- #if defined DEBUG
- #define _DPRINTK_FLAGS_DEFAULT _DPRINTK_FLAGS_PRINT
-@@ -51,22 +59,88 @@ struct _ddebug {
- #endif
- } __attribute__((aligned(8)));
- 
-+enum class_map_type {
-+	DD_CLASS_TYPE_DISJOINT_BITS,
-+	/**
-+	 * DD_CLASS_TYPE_DISJOINT_BITS: classes are independent, one per bit.
-+	 * expecting hex input. Built for drm.debug, basis for other types.
-+	 */
-+	DD_CLASS_TYPE_LEVEL_NUM,
-+	/**
-+	 * DD_CLASS_TYPE_LEVEL_NUM: input is numeric level, 0-N.
-+	 * N turns on just bits N-1 .. 0, so N=0 turns all bits off.
-+	 */
-+	DD_CLASS_TYPE_DISJOINT_NAMES,
-+	/**
-+	 * DD_CLASS_TYPE_DISJOINT_NAMES: input is a CSV of [+-]CLASS_NAMES,
-+	 * classes are independent, like _DISJOINT_BITS.
-+	 */
-+	DD_CLASS_TYPE_LEVEL_NAMES,
-+	/**
-+	 * DD_CLASS_TYPE_LEVEL_NAMES: input is a CSV of [+-]CLASS_NAMES,
-+	 * intended for names like: INFO,DEBUG,TRACE, with a module prefix
-+	 * avoid EMERG,ALERT,CRIT,ERR,WARNING: they're not debug
-+	 */
-+};
-+
-+struct ddebug_class_map {
-+	struct list_head link;
-+	struct module *mod;
-+	const char *mod_name;	/* needed for builtins */
-+	const char **class_names;
-+	const int length;
-+	const int base;		/* index of 1st .class_id, allows split/shared space */
-+	enum class_map_type map_type;
-+};
-+
-+/**
-+ * DECLARE_DYNDBG_CLASSMAP - declare classnames known by a module
-+ * @_var:   a struct ddebug_class_map, passed to module_param_cb
-+ * @_type:  enum class_map_type, chooses bits/verbose, numeric/symbolic
-+ * @_base:  offset of 1st class-name. splits .class_id space
-+ * @classes: class-names used to control class'd prdbgs
-+ */
-+#define DECLARE_DYNDBG_CLASSMAP(_var, _maptype, _base, ...)		\
-+	static const char *_var##_classnames[] = { __VA_ARGS__ };	\
-+	static struct ddebug_class_map __aligned(8) __used		\
-+		__section("__dyndbg_classes") _var = {			\
-+		.mod = THIS_MODULE,					\
-+		.mod_name = KBUILD_MODNAME,				\
-+		.base = _base,						\
-+		.map_type = _maptype,					\
-+		.length = NUM_TYPE_ARGS(char*, __VA_ARGS__),		\
-+		.class_names = _var##_classnames,			\
-+	}
-+#define NUM_TYPE_ARGS(eltype, ...)				\
-+        (sizeof((eltype[]){__VA_ARGS__}) / sizeof(eltype))
-+
-+/* encapsulate linker provided built-in (or module) dyndbg data */
-+struct _ddebug_info {
-+	struct _ddebug *descs;
-+	struct ddebug_class_map *classes;
-+	unsigned int num_descs;
-+	unsigned int num_classes;
-+};
-+
-+struct ddebug_class_param {
-+	union {
-+		unsigned long *bits;
-+		unsigned int *lvl;
-+	};
-+	char flags[8];
-+	const struct ddebug_class_map *map;
-+};
- 
-+/*
-+ * pr_debug() and friends are globally enabled or modules have selectively
-+ * enabled them.
-+ */
-+#if defined(CONFIG_DYNAMIC_DEBUG) || \
-+	(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
- 
-#if defined(CONFIG_DYNAMIC_DEBUG_CORE)
-
-/* exported for module authors to exercise >control */
-int dynamic_debug_exec_queries(const char *query, const char *modname);
-
-int ddebug_add_module(struct _ddebug *tab, unsigned int n,
-				const char *modname);
-extern int ddebug_remove_module(const char *mod_name);
- extern __printf(2, 3)
- void __dynamic_pr_debug(struct _ddebug *descriptor, const char *fmt, ...);
- 
-extern int ddebug_dyndbg_module_param_cb(char *param, char *val,
-					const char *modname);
-
- struct device;
- 
- extern __printf(3, 4)
-@@ -87,7 +161,13 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor,
- 			 const struct ib_device *ibdev,
- 			 const char *fmt, ...);
- 
-#define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt)		\
-+#define __dynamic_dump_stack(desc)				\
-+{								\
-+	if (desc.flags & _DPRINTK_FLAGS_INCL_STACK)		\
-+		dump_stack();					\
-+}
-+
-+#define DEFINE_DYNAMIC_DEBUG_METADATA_CLS(name, cls, fmt)	\
- 	static struct _ddebug  __aligned(8)			\
- 	__section("__dyndbg") name = {				\
- 		.modname = KBUILD_MODNAME,			\
-@@ -96,8 +176,14 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor,
- 		.format = (fmt),				\
- 		.lineno = __LINE__,				\
- 		.flags = _DPRINTK_FLAGS_DEFAULT,		\
-+		.class_id = cls,				\
- 		_DPRINTK_KEY_INIT				\
-	}
-+	};							\
-+	BUILD_BUG_ON_MSG(cls > _DPRINTK_CLASS_DFLT,		\
-+			 "classid value overflow")
-+
-+#define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt)		\
-+	DEFINE_DYNAMIC_DEBUG_METADATA_CLS(name, _DPRINTK_CLASS_DFLT, fmt)
- 
- #ifdef CONFIG_JUMP_LABEL
- 
-@@ -128,17 +214,38 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor,
- 
- #endif /* CONFIG_JUMP_LABEL */
- 
-#define __dynamic_func_call(id, fmt, func, ...) do {	\
-	DEFINE_DYNAMIC_DEBUG_METADATA(id, fmt);		\
-	if (DYNAMIC_DEBUG_BRANCH(id))			\
-		func(&id, ##__VA_ARGS__);		\
-+/*
-+ * Factory macros: ($prefix)dynamic_func_call($suffix)
-+ *
-+ * Lower layer (with __ prefix) gets the callsite metadata, and wraps
-+ * the func inside a debug-branch/static-key construct.  Upper layer
-+ * (with _ prefix) does the UNIQUE_ID once, so that lower can ref the
-+ * name/label multiple times, and tie the elements together.
-+ * Multiple flavors:
-+ * (|_cls):	adds in _DPRINT_CLASS_DFLT as needed
-+ * (|_no_desc):	former gets callsite descriptor as 1st arg (for prdbgs)
-+ */
-+#define __dynamic_func_call_cls(id, cls, fmt, func, ...) do {	\
-+	DEFINE_DYNAMIC_DEBUG_METADATA_CLS(id, cls, fmt);	\
-+	if (DYNAMIC_DEBUG_BRANCH(id)) {				\
-+		func(&id, ##__VA_ARGS__);			\
-+		__dynamic_dump_stack(id);			\
-+	}							\
- } while (0)
-
-#define __dynamic_func_call_no_desc(id, fmt, func, ...) do {	\
-	DEFINE_DYNAMIC_DEBUG_METADATA(id, fmt);			\
-	if (DYNAMIC_DEBUG_BRANCH(id))				\
-		func(__VA_ARGS__);				\
-+#define __dynamic_func_call(id, fmt, func, ...)				\
-+	__dynamic_func_call_cls(id, _DPRINTK_CLASS_DFLT, fmt,		\
-+				func, ##__VA_ARGS__)
-+
-+#define __dynamic_func_call_cls_no_desc(id, cls, fmt, func, ...) do {	\
-+	DEFINE_DYNAMIC_DEBUG_METADATA_CLS(id, cls, fmt);		\
-+	if (DYNAMIC_DEBUG_BRANCH(id)) {					\
-+		func(__VA_ARGS__);					\
-+		__dynamic_dump_stack(id);				\
-+	}								\
- } while (0)
-+#define __dynamic_func_call_no_desc(id, fmt, func, ...)			\
-+	__dynamic_func_call_cls_no_desc(id, _DPRINTK_CLASS_DFLT,	\
-+					fmt, func, ##__VA_ARGS__)
- 
- /*
-  * "Factory macro" for generating a call to func, guarded by a
-@@ -148,22 +255,33 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor,
-  * the varargs. Note that fmt is repeated in invocations of this
-  * macro.
-  */
-+#define _dynamic_func_call_cls(cls, fmt, func, ...)			\
-+	__dynamic_func_call_cls(__UNIQUE_ID(ddebug), cls, fmt, func, ##__VA_ARGS__)
- #define _dynamic_func_call(fmt, func, ...)				\
-	__dynamic_func_call(__UNIQUE_ID(ddebug), fmt, func, ##__VA_ARGS__)
-+	_dynamic_func_call_cls(_DPRINTK_CLASS_DFLT, fmt, func, ##__VA_ARGS__)
-+
- /*
-  * A variant that does the same, except that the descriptor is not
-  * passed as the first argument to the function; it is only called
-  * with precisely the macro's varargs.
-  */
-#define _dynamic_func_call_no_desc(fmt, func, ...)	\
-	__dynamic_func_call_no_desc(__UNIQUE_ID(ddebug), fmt, func, ##__VA_ARGS__)
-+#define _dynamic_func_call_cls_no_desc(cls, fmt, func, ...)		\
-+	__dynamic_func_call_cls_no_desc(__UNIQUE_ID(ddebug), cls, fmt,	\
-+					func, ##__VA_ARGS__)
-+#define _dynamic_func_call_no_desc(fmt, func, ...)			\
-+	_dynamic_func_call_cls_no_desc(_DPRINTK_CLASS_DFLT, fmt,	\
-+				       func, ##__VA_ARGS__)
-+
-+#define dynamic_pr_debug_cls(cls, fmt, ...)				\
-+	_dynamic_func_call_cls(cls, fmt, __dynamic_pr_debug,		\
-+			   pr_fmt(fmt), ##__VA_ARGS__)
- 
- #define dynamic_pr_debug(fmt, ...)				\
-	_dynamic_func_call(fmt,	__dynamic_pr_debug,		\
-+	_dynamic_func_call(fmt, __dynamic_pr_debug,		\
- 			   pr_fmt(fmt), ##__VA_ARGS__)
- 
- #define dynamic_dev_dbg(dev, fmt, ...)				\
-	_dynamic_func_call(fmt,__dynamic_dev_dbg, 		\
-+	_dynamic_func_call(fmt, __dynamic_dev_dbg, 		\
- 			   dev, fmt, ##__VA_ARGS__)
- 
- #define dynamic_netdev_dbg(dev, fmt, ...)			\
-@@ -181,27 +299,50 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor,
- 				   KERN_DEBUG, prefix_str, prefix_type,	\
- 				   rowsize, groupsize, buf, len, ascii)
- 
-#else /* !CONFIG_DYNAMIC_DEBUG_CORE */
-+/* for test only, generally expect drm.debug style macro wrappers */
-+#define __pr_debug_cls(cls, fmt, ...) do {			\
-+	BUILD_BUG_ON_MSG(!__builtin_constant_p(cls),		\
-+			 "expecting constant class int/enum");	\
-+	dynamic_pr_debug_cls(cls, fmt, ##__VA_ARGS__);		\
-+	} while (0)
-+
-+#else /* !(CONFIG_DYNAMIC_DEBUG || (CONFIG_DYNAMIC_DEBUG_CORE && DYNAMIC_DEBUG_MODULE)) */
- 
- #include <linux/string.h>
- #include <linux/errno.h>
- #include <linux/printk.h>
- 
-static inline int ddebug_add_module(struct _ddebug *tab, unsigned int n,
-				    const char *modname)
-{
-	return 0;
-}
-+#define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt)
-+#define DYNAMIC_DEBUG_BRANCH(descriptor) false
- 
-static inline int ddebug_remove_module(const char *mod)
-{
-	return 0;
-}
-+#define dynamic_pr_debug(fmt, ...)					\
-+	no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
-+#define dynamic_dev_dbg(dev, fmt, ...)					\
-+	dev_no_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__)
-+#define dynamic_hex_dump(prefix_str, prefix_type, rowsize,		\
-+			 groupsize, buf, len, ascii)			\
-+	do { if (0)							\
-+		print_hex_dump(KERN_DEBUG, prefix_str, prefix_type,	\
-+				rowsize, groupsize, buf, len, ascii);	\
-+	} while (0)
-+
-+#endif /* CONFIG_DYNAMIC_DEBUG || (CONFIG_DYNAMIC_DEBUG_CORE && DYNAMIC_DEBUG_MODULE) */
-+
-+
-+#ifdef CONFIG_DYNAMIC_DEBUG_CORE
-+
-+extern int ddebug_dyndbg_module_param_cb(char *param, char *val,
-+					const char *modname);
-+struct kernel_param;
-+int param_set_dyndbg_classes(const char *instr, const struct kernel_param *kp);
-+int param_get_dyndbg_classes(char *buffer, const struct kernel_param *kp);
-+
-+#else
- 
- static inline int ddebug_dyndbg_module_param_cb(char *param, char *val,
- 						const char *modname)
- {
-	if (strstr(param, "dyndbg")) {
-+	if (!strcmp(param, "dyndbg")) {
- 		/* avoid pr_warn(), which wants pr_fmt() fully defined */
- 		printk(KERN_WARNING "dyndbg param is supported only in "
- 			"CONFIG_DYNAMIC_DEBUG builds\n");
-@@ -210,23 +351,15 @@ static inline int ddebug_dyndbg_module_param_cb(char *param, char *val,
- 	return -EINVAL;
- }
- 
-#define dynamic_pr_debug(fmt, ...)					\
-	do { if (0) printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); } while (0)
-#define dynamic_dev_dbg(dev, fmt, ...)					\
-	do { if (0) dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__); } while (0)
-#define dynamic_hex_dump(prefix_str, prefix_type, rowsize,		\
-			 groupsize, buf, len, ascii)			\
-	do { if (0)							\
-		print_hex_dump(KERN_DEBUG, prefix_str, prefix_type,	\
-				rowsize, groupsize, buf, len, ascii);	\
-	} while (0)
-+struct kernel_param;
-+static inline int param_set_dyndbg_classes(const char *instr, const struct kernel_param *kp)
-+{ return 0; }
-+static inline int param_get_dyndbg_classes(char *buffer, const struct kernel_param *kp)
-+{ return 0; }
- 
-static inline int dynamic_debug_exec_queries(const char *query, const char *modname)
-{
-	pr_warn("kernel not built with CONFIG_DYNAMIC_DEBUG_CORE\n");
-	return 0;
-}
-+#endif
- 
-#endif /* !CONFIG_DYNAMIC_DEBUG_CORE */
- 
-#endif
-+extern const struct kernel_param_ops param_ops_dyndbg_classes;
-+
-+#endif /* _DYNAMIC_DEBUG_H */
-diff --git a/include/linux/dynamic_queue_limits.h b/include/linux/dynamic_queue_limits.h
-index 407c2f281b64..808b1a5102e7 100644
--- a/include/linux/dynamic_queue_limits.h
-+++ b/include/linux/dynamic_queue_limits.h
-@@ -38,14 +38,25 @@
- 
- #ifdef __KERNEL__
- 
-+#include <linux/bitops.h>
- #include <asm/bug.h>
- 
-+#define DQL_HIST_LEN		4
-+#define DQL_HIST_ENT(dql, idx)	((dql)->history[(idx) % DQL_HIST_LEN])
-+
- struct dql {
- 	/* Fields accessed in enqueue path (dql_queued) */
- 	unsigned int	num_queued;		/* Total ever queued */
- 	unsigned int	adj_limit;		/* limit + num_completed */
- 	unsigned int	last_obj_cnt;		/* Count at last queuing */
- 
-+	/* Stall threshold (in jiffies), defined by user */
-+	unsigned short	stall_thrs;
-+
-+	unsigned long	history_head;		/* top 58 bits of jiffies */
-+	/* stall entries, a bit per entry */
-+	unsigned long	history[DQL_HIST_LEN];
-+
- 	/* Fields accessed only by completion path (dql_completed) */
- 
- 	unsigned int	limit ____cacheline_aligned_in_smp; /* Current limit */
-@@ -62,21 +73,61 @@ struct dql {
- 	unsigned int	max_limit;		/* Max limit */
- 	unsigned int	min_limit;		/* Minimum limit */
- 	unsigned int	slack_hold_time;	/* Time to measure slack */
-+
-+	/* Longest stall detected, reported to user */
-+	unsigned short	stall_max;
-+	unsigned long	last_reap;		/* Last reap (in jiffies) */
-+	unsigned long	stall_cnt;		/* Number of stalls */
- };
- 
- /* Set some static maximums */
- #define DQL_MAX_OBJECT (UINT_MAX / 16)
- #define DQL_MAX_LIMIT ((UINT_MAX / 2) - DQL_MAX_OBJECT)
- 
-+/* Populate the bitmap to be processed later in dql_check_stall() */
-+static inline void dql_queue_stall(struct dql *dql)
-+{
-+	unsigned long map, now, now_hi, i;
-+
-+	now = jiffies;
-+	now_hi = now / BITS_PER_LONG;
-+
-+	/* The following code set a bit in the ring buffer, where each
-+	 * bit trackes time the packet was queued. The dql->history buffer
-+	 * tracks DQL_HIST_LEN * BITS_PER_LONG time (jiffies) slot
-+	 */
-+	if (unlikely(now_hi != dql->history_head)) {
-+		/* About to reuse slots, clear them */
-+		for (i = 0; i < DQL_HIST_LEN; i++) {
-+			/* Multiplication masks high bits */
-+			if (now_hi * BITS_PER_LONG ==
-+			    (dql->history_head + i) * BITS_PER_LONG)
-+				break;
-+			DQL_HIST_ENT(dql, dql->history_head + i + 1) = 0;
-+		}
-+		/* pairs with smp_rmb() in dql_check_stall() */
-+		smp_wmb();
-+		WRITE_ONCE(dql->history_head, now_hi);
-+	}
-+
-+	/* __set_bit() does not guarantee WRITE_ONCE() semantics */
-+	map = DQL_HIST_ENT(dql, now_hi);
-+
-+	/* Populate the history with an entry (bit) per queued */
-+	if (!(map & BIT_MASK(now)))
-+		WRITE_ONCE(DQL_HIST_ENT(dql, now_hi), map | BIT_MASK(now));
-+}
-+
- /*
-  * Record number of objects queued. Assumes that caller has already checked
-  * availability in the queue with dql_avail.
-  */
- static inline void dql_queued(struct dql *dql, unsigned int count)
- {
-	BUG_ON(count > DQL_MAX_OBJECT);
-+	if (WARN_ON_ONCE(count > DQL_MAX_OBJECT))
-+		return;
- 
-	dql->last_obj_cnt = count;
-+	WRITE_ONCE(dql->last_obj_cnt, count);
- 
- 	/* We want to force a write first, so that cpu do not attempt
- 	 * to get cache line containing last_obj_cnt, num_queued, adj_limit
-@@ -86,6 +137,10 @@ static inline void dql_queued(struct dql *dql, unsigned int count)
- 	barrier();
- 
- 	dql->num_queued += count;
-+
-+	/* Only populate stall information if the threshold is set */
-+	if (READ_ONCE(dql->stall_thrs))
-+		dql_queue_stall(dql);
- }
- 
- /* Returns how many objects can be queued, < 0 indicates over limit. */
-diff --git a/include/linux/edac.h b/include/linux/edac.h
-index e730b3468719..fa32f2aca22f 100644
--- a/include/linux/edac.h
-+++ b/include/linux/edac.h
-@@ -30,7 +30,7 @@ struct device;
- 
- extern int edac_op_state;
- 
-struct bus_type *edac_get_sysfs_subsys(void);
-+const struct bus_type *edac_get_sysfs_subsys(void);
- 
- static inline void opstate_init(void)
- {
-@@ -187,6 +187,7 @@ static inline char *mc_event_error_type(const unsigned int err_type)
-  * @MEM_NVDIMM:		Non-volatile RAM
-  * @MEM_WIO2:		Wide I/O 2.
-  * @MEM_HBM2:		High bandwidth Memory Gen 2.
-+ * @MEM_HBM3:		High bandwidth Memory Gen 3.
-  */
- enum mem_type {
- 	MEM_EMPTY = 0,
-@@ -218,6 +219,7 @@ enum mem_type {
- 	MEM_NVDIMM,
- 	MEM_WIO2,
- 	MEM_HBM2,
-+	MEM_HBM3,
- };
- 
- #define MEM_FLAG_EMPTY		BIT(MEM_EMPTY)
-@@ -231,23 +233,24 @@ enum mem_type {
- #define MEM_FLAG_DDR		BIT(MEM_DDR)
- #define MEM_FLAG_RDDR		BIT(MEM_RDDR)
- #define MEM_FLAG_RMBS		BIT(MEM_RMBS)
-#define MEM_FLAG_DDR2           BIT(MEM_DDR2)
-#define MEM_FLAG_FB_DDR2        BIT(MEM_FB_DDR2)
-#define MEM_FLAG_RDDR2          BIT(MEM_RDDR2)
-#define MEM_FLAG_XDR            BIT(MEM_XDR)
-#define MEM_FLAG_DDR3           BIT(MEM_DDR3)
-#define MEM_FLAG_RDDR3          BIT(MEM_RDDR3)
-#define MEM_FLAG_LPDDR3         BIT(MEM_LPDDR3)
-#define MEM_FLAG_DDR4           BIT(MEM_DDR4)
-#define MEM_FLAG_RDDR4          BIT(MEM_RDDR4)
-#define MEM_FLAG_LRDDR4         BIT(MEM_LRDDR4)
-#define MEM_FLAG_LPDDR4         BIT(MEM_LPDDR4)
-#define MEM_FLAG_DDR5           BIT(MEM_DDR5)
-#define MEM_FLAG_RDDR5          BIT(MEM_RDDR5)
-#define MEM_FLAG_LRDDR5         BIT(MEM_LRDDR5)
-#define MEM_FLAG_NVDIMM         BIT(MEM_NVDIMM)
-+#define MEM_FLAG_DDR2		BIT(MEM_DDR2)
-+#define MEM_FLAG_FB_DDR2	BIT(MEM_FB_DDR2)
-+#define MEM_FLAG_RDDR2		BIT(MEM_RDDR2)
-+#define MEM_FLAG_XDR		BIT(MEM_XDR)
-+#define MEM_FLAG_DDR3		BIT(MEM_DDR3)
-+#define MEM_FLAG_RDDR3		BIT(MEM_RDDR3)
-+#define MEM_FLAG_LPDDR3		BIT(MEM_LPDDR3)
-+#define MEM_FLAG_DDR4		BIT(MEM_DDR4)
-+#define MEM_FLAG_RDDR4		BIT(MEM_RDDR4)
-+#define MEM_FLAG_LRDDR4		BIT(MEM_LRDDR4)
-+#define MEM_FLAG_LPDDR4		BIT(MEM_LPDDR4)
-+#define MEM_FLAG_DDR5		BIT(MEM_DDR5)
-+#define MEM_FLAG_RDDR5		BIT(MEM_RDDR5)
-+#define MEM_FLAG_LRDDR5		BIT(MEM_LRDDR5)
-+#define MEM_FLAG_NVDIMM		BIT(MEM_NVDIMM)
- #define MEM_FLAG_WIO2		BIT(MEM_WIO2)
- #define MEM_FLAG_HBM2		BIT(MEM_HBM2)
-+#define MEM_FLAG_HBM3		BIT(MEM_HBM3)
- 
- /**
-  * enum edac_type - Error Detection and Correction capabilities and mode
-@@ -492,7 +495,7 @@ struct edac_raw_error_desc {
-  */
- struct mem_ctl_info {
- 	struct device			dev;
-	struct bus_type			*bus;
-+	const struct bus_type		*bus;
- 
- 	struct list_head link;	/* for global list of mem_ctl_info structs */
- 
-@@ -658,4 +661,226 @@ static inline struct dimm_info *edac_get_dimm(struct mem_ctl_info *mci,
- 
- 	return mci->dimms[index];
- }
-+
-+#define EDAC_FEAT_NAME_LEN	128
-+
-+/* RAS feature type */
-+enum edac_dev_feat {
-+	RAS_FEAT_SCRUB,
-+	RAS_FEAT_ECS,
-+	RAS_FEAT_MEM_REPAIR,
-+	RAS_FEAT_MAX
-+};
-+
-+/**
-+ * struct edac_scrub_ops - scrub device operations (all elements optional)
-+ * @read_addr: read base address of scrubbing range.
-+ * @read_size: read offset of scrubbing range.
-+ * @write_addr: set base address of the scrubbing range.
-+ * @write_size: set offset of the scrubbing range.
-+ * @get_enabled_bg: check if currently performing background scrub.
-+ * @set_enabled_bg: start or stop a bg-scrub.
-+ * @get_min_cycle: get minimum supported scrub cycle duration in seconds.
-+ * @get_max_cycle: get maximum supported scrub cycle duration in seconds.
-+ * @get_cycle_duration: get current scrub cycle duration in seconds.
-+ * @set_cycle_duration: set current scrub cycle duration in seconds.
-+ */
-+struct edac_scrub_ops {
-+	int (*read_addr)(struct device *dev, void *drv_data, u64 *base);
-+	int (*read_size)(struct device *dev, void *drv_data, u64 *size);
-+	int (*write_addr)(struct device *dev, void *drv_data, u64 base);
-+	int (*write_size)(struct device *dev, void *drv_data, u64 size);
-+	int (*get_enabled_bg)(struct device *dev, void *drv_data, bool *enable);
-+	int (*set_enabled_bg)(struct device *dev, void *drv_data, bool enable);
-+	int (*get_min_cycle)(struct device *dev, void *drv_data,  u32 *min);
-+	int (*get_max_cycle)(struct device *dev, void *drv_data,  u32 *max);
-+	int (*get_cycle_duration)(struct device *dev, void *drv_data, u32 *cycle);
-+	int (*set_cycle_duration)(struct device *dev, void *drv_data, u32 cycle);
-+};
-+
-+#if IS_ENABLED(CONFIG_EDAC_SCRUB)
-+int edac_scrub_get_desc(struct device *scrub_dev,
-+			const struct attribute_group **attr_groups,
-+			u8 instance);
-+#else
-+static inline int edac_scrub_get_desc(struct device *scrub_dev,
-+				      const struct attribute_group **attr_groups,
-+				      u8 instance)
-+{ return -EOPNOTSUPP; }
-+#endif /* CONFIG_EDAC_SCRUB */
-+
-+/**
-+ * struct edac_ecs_ops - ECS device operations (all elements optional)
-+ * @get_log_entry_type: read the log entry type value.
-+ * @set_log_entry_type: set the log entry type value.
-+ * @get_mode: read the mode value.
-+ * @set_mode: set the mode value.
-+ * @reset: reset the ECS counter.
-+ * @get_threshold: read the threshold count per gigabits of memory cells.
-+ * @set_threshold: set the threshold count per gigabits of memory cells.
-+ */
-+struct edac_ecs_ops {
-+	int (*get_log_entry_type)(struct device *dev, void *drv_data, int fru_id, u32 *val);
-+	int (*set_log_entry_type)(struct device *dev, void *drv_data, int fru_id, u32 val);
-+	int (*get_mode)(struct device *dev, void *drv_data, int fru_id, u32 *val);
-+	int (*set_mode)(struct device *dev, void *drv_data, int fru_id, u32 val);
-+	int (*reset)(struct device *dev, void *drv_data, int fru_id, u32 val);
-+	int (*get_threshold)(struct device *dev, void *drv_data, int fru_id, u32 *threshold);
-+	int (*set_threshold)(struct device *dev, void *drv_data, int fru_id, u32 threshold);
-+};
-+
-+struct edac_ecs_ex_info {
-+	u16 num_media_frus;
-+};
-+
-+#if IS_ENABLED(CONFIG_EDAC_ECS)
-+int edac_ecs_get_desc(struct device *ecs_dev,
-+		      const struct attribute_group **attr_groups,
-+		      u16 num_media_frus);
-+#else
-+static inline int edac_ecs_get_desc(struct device *ecs_dev,
-+				    const struct attribute_group **attr_groups,
-+				    u16 num_media_frus)
-+{ return -EOPNOTSUPP; }
-+#endif /* CONFIG_EDAC_ECS */
-+
-+enum edac_mem_repair_type {
-+	EDAC_REPAIR_PPR,
-+	EDAC_REPAIR_CACHELINE_SPARING,
-+	EDAC_REPAIR_ROW_SPARING,
-+	EDAC_REPAIR_BANK_SPARING,
-+	EDAC_REPAIR_RANK_SPARING,
-+	EDAC_REPAIR_MAX
-+};
-+
-+extern const char * const edac_repair_type[];
-+
-+enum edac_mem_repair_cmd {
-+	EDAC_DO_MEM_REPAIR = 1,
-+};
-+
-+/**
-+ * struct edac_mem_repair_ops - memory repair operations
-+ * (all elements are optional except do_repair, set_hpa/set_dpa)
-+ * @get_repair_type: get the memory repair type, listed in
-+ *			 enum edac_mem_repair_function.
-+ * @get_persist_mode: get the current persist mode.
-+ *		      false - Soft repair type (temporary repair).
-+ *		      true - Hard memory repair type (permanent repair).
-+ * @set_persist_mode: set the persist mode of the memory repair instance.
-+ * @get_repair_safe_when_in_use: get whether memory media is accessible and
-+ *				 data is retained during repair operation.
-+ * @get_hpa: get current host physical address (HPA) of memory to repair.
-+ * @set_hpa: set host physical address (HPA) of memory to repair.
-+ * @get_min_hpa: get the minimum supported host physical address (HPA).
-+ * @get_max_hpa: get the maximum supported host physical address (HPA).
-+ * @get_dpa: get current device physical address (DPA) of memory to repair.
-+ * @set_dpa: set device physical address (DPA) of memory to repair.
-+ *	     In some states of system configuration (e.g. before address decoders
-+ *	     have been configured), memory devices (e.g. CXL) may not have an active
-+ *	     mapping in the host physical address map. As such, the memory
-+ *	     to repair must be identified by a device specific physical addressing
-+ *	     scheme using a device physical address(DPA). The DPA and other control
-+ *	     attributes to use for the repair operations will be presented in related
-+ *	     error records.
-+ * @get_min_dpa: get the minimum supported device physical address (DPA).
-+ * @get_max_dpa: get the maximum supported device physical address (DPA).
-+ * @get_nibble_mask: get current nibble mask of memory to repair.
-+ * @set_nibble_mask: set nibble mask of memory to repair.
-+ * @get_bank_group: get current bank group of memory to repair.
-+ * @set_bank_group: set bank group of memory to repair.
-+ * @get_bank: get current bank of memory to repair.
-+ * @set_bank: set bank of memory to repair.
-+ * @get_rank: get current rank of memory to repair.
-+ * @set_rank: set rank of memory to repair.
-+ * @get_row: get current row of memory to repair.
-+ * @set_row: set row of memory to repair.
-+ * @get_column: get current column of memory to repair.
-+ * @set_column: set column of memory to repair.
-+ * @get_channel: get current channel of memory to repair.
-+ * @set_channel: set channel of memory to repair.
-+ * @get_sub_channel: get current subchannel of memory to repair.
-+ * @set_sub_channel: set subchannel of memory to repair.
-+ * @do_repair: Issue memory repair operation for the HPA/DPA and
-+ *	       other control attributes set for the memory to repair.
-+ *
-+ * All elements are optional except do_repair and at least one of set_hpa/set_dpa.
-+ */
-+struct edac_mem_repair_ops {
-+	int (*get_repair_type)(struct device *dev, void *drv_data, const char **type);
-+	int (*get_persist_mode)(struct device *dev, void *drv_data, bool *persist);
-+	int (*set_persist_mode)(struct device *dev, void *drv_data, bool persist);
-+	int (*get_repair_safe_when_in_use)(struct device *dev, void *drv_data, bool *safe);
-+	int (*get_hpa)(struct device *dev, void *drv_data, u64 *hpa);
-+	int (*set_hpa)(struct device *dev, void *drv_data, u64 hpa);
-+	int (*get_min_hpa)(struct device *dev, void *drv_data, u64 *hpa);
-+	int (*get_max_hpa)(struct device *dev, void *drv_data, u64 *hpa);
-+	int (*get_dpa)(struct device *dev, void *drv_data, u64 *dpa);
-+	int (*set_dpa)(struct device *dev, void *drv_data, u64 dpa);
-+	int (*get_min_dpa)(struct device *dev, void *drv_data, u64 *dpa);
-+	int (*get_max_dpa)(struct device *dev, void *drv_data, u64 *dpa);
-+	int (*get_nibble_mask)(struct device *dev, void *drv_data, u32 *val);
-+	int (*set_nibble_mask)(struct device *dev, void *drv_data, u32 val);
-+	int (*get_bank_group)(struct device *dev, void *drv_data, u32 *val);
-+	int (*set_bank_group)(struct device *dev, void *drv_data, u32 val);
-+	int (*get_bank)(struct device *dev, void *drv_data, u32 *val);
-+	int (*set_bank)(struct device *dev, void *drv_data, u32 val);
-+	int (*get_rank)(struct device *dev, void *drv_data, u32 *val);
-+	int (*set_rank)(struct device *dev, void *drv_data, u32 val);
-+	int (*get_row)(struct device *dev, void *drv_data, u32 *val);
-+	int (*set_row)(struct device *dev, void *drv_data, u32 val);
-+	int (*get_column)(struct device *dev, void *drv_data, u32 *val);
-+	int (*set_column)(struct device *dev, void *drv_data, u32 val);
-+	int (*get_channel)(struct device *dev, void *drv_data, u32 *val);
-+	int (*set_channel)(struct device *dev, void *drv_data, u32 val);
-+	int (*get_sub_channel)(struct device *dev, void *drv_data, u32 *val);
-+	int (*set_sub_channel)(struct device *dev, void *drv_data, u32 val);
-+	int (*do_repair)(struct device *dev, void *drv_data, u32 val);
-+};
-+
-+#if IS_ENABLED(CONFIG_EDAC_MEM_REPAIR)
-+int edac_mem_repair_get_desc(struct device *dev,
-+			     const struct attribute_group **attr_groups,
-+			     u8 instance);
-+#else
-+static inline int edac_mem_repair_get_desc(struct device *dev,
-+					   const struct attribute_group **attr_groups,
-+					   u8 instance)
-+{ return -EOPNOTSUPP; }
-+#endif /* CONFIG_EDAC_MEM_REPAIR */
-+
-+/* EDAC device feature information structure */
-+struct edac_dev_data {
-+	union {
-+		const struct edac_scrub_ops *scrub_ops;
-+		const struct edac_ecs_ops *ecs_ops;
-+		const struct edac_mem_repair_ops *mem_repair_ops;
-+	};
-+	u8 instance;
-+	void *private;
-+};
-+
-+struct edac_dev_feat_ctx {
-+	struct device dev;
-+	void *private;
-+	struct edac_dev_data *scrub;
-+	struct edac_dev_data ecs;
-+	struct edac_dev_data *mem_repair;
-+};
-+
-+struct edac_dev_feature {
-+	enum edac_dev_feat ft_type;
-+	u8 instance;
-+	union {
-+		const struct edac_scrub_ops *scrub_ops;
-+		const struct edac_ecs_ops *ecs_ops;
-+		const struct edac_mem_repair_ops *mem_repair_ops;
-+	};
-+	void *ctx;
-+	struct edac_ecs_ex_info ecs_info;
-+};
-+
-+int edac_dev_register(struct device *parent, char *dev_name,
-+		      void *parent_pvt_data, int num_features,
-+		      const struct edac_dev_feature *ras_features);
- #endif /* _LINUX_EDAC_H_ */
-diff --git a/include/linux/eeprom_93cx6.h b/include/linux/eeprom_93cx6.h
-index c860c72a921d..3a485cc0e0fa 100644
--- a/include/linux/eeprom_93cx6.h
-+++ b/include/linux/eeprom_93cx6.h
-@@ -11,6 +11,8 @@
- 	Supported chipsets: 93c46, 93c56 and 93c66.
-  */
- 
-+#include <linux/bits.h>
-+
- /*
-  * EEPROM operation defines.
-  */
-@@ -34,6 +36,7 @@
-  * @register_write(struct eeprom_93cx6 *eeprom): handler to
-  * write to the eeprom register by using all reg_* fields.
-  * @width: eeprom width, should be one of the PCI_EEPROM_WIDTH_* defines
-+ * @quirks: eeprom or controller quirks
-  * @drive_data: Set if we're driving the data line.
-  * @reg_data_in: register field to indicate data input
-  * @reg_data_out: register field to indicate data output
-@@ -50,6 +53,9 @@ struct eeprom_93cx6 {
- 	void (*register_write)(struct eeprom_93cx6 *eeprom);
- 
- 	int width;
-+	unsigned int quirks;
-+/* Some EEPROMs require an extra clock cycle before reading */
-+#define PCI_EEPROM_QUIRK_EXTRA_READ_CYCLE	BIT(0)
- 
- 	char drive_data;
- 	char reg_data_in;
-@@ -71,3 +77,8 @@ extern void eeprom_93cx6_wren(struct eeprom_93cx6 *eeprom, bool enable);
- 
- extern void eeprom_93cx6_write(struct eeprom_93cx6 *eeprom,
- 			       u8 addr, u16 data);
-+
-+static inline bool has_quirk_extra_read_cycle(struct eeprom_93cx6 *eeprom)
-+{
-+	return eeprom->quirks & PCI_EEPROM_QUIRK_EXTRA_READ_CYCLE;
-+}
-diff --git a/include/linux/eeprom_93xx46.h b/include/linux/eeprom_93xx46.h
-deleted file mode 100644
-index 34c2175e6a1e..000000000000
--- a/include/linux/eeprom_93xx46.h
-+++ /dev/null
-@@ -1,32 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Module: eeprom_93xx46
- * platform description for 93xx46 EEPROMs.
- */
-#include <linux/gpio/consumer.h>
-
-struct eeprom_93xx46_platform_data {
-	unsigned char	flags;
-#define EE_ADDR8	0x01		/*  8 bit addr. cfg */
-#define EE_ADDR16	0x02		/* 16 bit addr. cfg */
-#define EE_READONLY	0x08		/* forbid writing */
-#define EE_SIZE1K	0x10		/* 1 kb of data, that is a 93xx46 */
-#define EE_SIZE2K	0x20		/* 2 kb of data, that is a 93xx56 */
-#define EE_SIZE4K	0x40		/* 4 kb of data, that is a 93xx66 */
-
-	unsigned int	quirks;
-/* Single word read transfers only; no sequential read. */
-#define EEPROM_93XX46_QUIRK_SINGLE_WORD_READ		(1 << 0)
-/* Instructions such as EWEN are (addrlen + 2) in length. */
-#define EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH		(1 << 1)
-/* Add extra cycle after address during a read */
-#define EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE		BIT(2)
-
-	/*
-	 * optional hooks to control additional logic
-	 * before and after spi transfer.
-	 */
-	void (*prepare)(void *);
-	void (*finish)(void *);
-	struct gpio_desc *select;
-};
-diff --git a/include/linux/efi.h b/include/linux/efi.h
-index d2b84c2fec39..2a43094e23f7 100644
--- a/include/linux/efi.h
-+++ b/include/linux/efi.h
-@@ -24,10 +24,11 @@
- #include <linux/range.h>
- #include <linux/reboot.h>
- #include <linux/uuid.h>
-#include <linux/screen_info.h>
- 
- #include <asm/page.h>
- 
-+struct screen_info;
-+
- #define EFI_SUCCESS		0
- #define EFI_LOAD_ERROR		( 1 | (1UL << (BITS_PER_LONG-1)))
- #define EFI_INVALID_PARAMETER	( 2 | (1UL << (BITS_PER_LONG-1)))
-@@ -39,6 +40,7 @@
- #define EFI_WRITE_PROTECTED	( 8 | (1UL << (BITS_PER_LONG-1)))
- #define EFI_OUT_OF_RESOURCES	( 9 | (1UL << (BITS_PER_LONG-1)))
- #define EFI_NOT_FOUND		(14 | (1UL << (BITS_PER_LONG-1)))
-+#define EFI_ACCESS_DENIED	(15 | (1UL << (BITS_PER_LONG-1)))
- #define EFI_TIMEOUT		(18 | (1UL << (BITS_PER_LONG-1)))
- #define EFI_ABORTED		(21 | (1UL << (BITS_PER_LONG-1)))
- #define EFI_SECURITY_VIOLATION	(26 | (1UL << (BITS_PER_LONG-1)))
-@@ -72,10 +74,10 @@ typedef void *efi_handle_t;
-  */
- typedef guid_t efi_guid_t __aligned(__alignof__(u32));
- 
-#define EFI_GUID(a, b, c, d...) (efi_guid_t){ {					\
-+#define EFI_GUID(a, b, c, d...) ((efi_guid_t){ {				\
- 	(a) & 0xff, ((a) >> 8) & 0xff, ((a) >> 16) & 0xff, ((a) >> 24) & 0xff,	\
- 	(b) & 0xff, ((b) >> 8) & 0xff,						\
-	(c) & 0xff, ((c) >> 8) & 0xff, d } }
-+	(c) & 0xff, ((c) >> 8) & 0xff, d } })
- 
- /*
-  * Generic EFI table header
-@@ -108,24 +110,26 @@ typedef	struct {
- #define EFI_MEMORY_MAPPED_IO_PORT_SPACE	12
- #define EFI_PAL_CODE			13
- #define EFI_PERSISTENT_MEMORY		14
-#define EFI_MAX_MEMORY_TYPE		15
-+#define EFI_UNACCEPTED_MEMORY		15
-+#define EFI_MAX_MEMORY_TYPE		16
- 
- /* Attribute values: */
-#define EFI_MEMORY_UC		((u64)0x0000000000000001ULL)	/* uncached */
-#define EFI_MEMORY_WC		((u64)0x0000000000000002ULL)	/* write-coalescing */
-#define EFI_MEMORY_WT		((u64)0x0000000000000004ULL)	/* write-through */
-#define EFI_MEMORY_WB		((u64)0x0000000000000008ULL)	/* write-back */
-#define EFI_MEMORY_UCE		((u64)0x0000000000000010ULL)	/* uncached, exported */
-#define EFI_MEMORY_WP		((u64)0x0000000000001000ULL)	/* write-protect */
-#define EFI_MEMORY_RP		((u64)0x0000000000002000ULL)	/* read-protect */
-#define EFI_MEMORY_XP		((u64)0x0000000000004000ULL)	/* execute-protect */
-#define EFI_MEMORY_NV		((u64)0x0000000000008000ULL)	/* non-volatile */
-#define EFI_MEMORY_MORE_RELIABLE \
-				((u64)0x0000000000010000ULL)	/* higher reliability */
-#define EFI_MEMORY_RO		((u64)0x0000000000020000ULL)	/* read-only */
-#define EFI_MEMORY_SP		((u64)0x0000000000040000ULL)	/* soft reserved */
-#define EFI_MEMORY_CPU_CRYPTO	((u64)0x0000000000080000ULL)	/* supports encryption */
-#define EFI_MEMORY_RUNTIME	((u64)0x8000000000000000ULL)	/* range requires runtime mapping */
-+#define EFI_MEMORY_UC			BIT_ULL(0)	/* uncached */
-+#define EFI_MEMORY_WC			BIT_ULL(1)	/* write-coalescing */
-+#define EFI_MEMORY_WT			BIT_ULL(2)	/* write-through */
-+#define EFI_MEMORY_WB			BIT_ULL(3)	/* write-back */
-+#define EFI_MEMORY_UCE			BIT_ULL(4)	/* uncached, exported */
-+#define EFI_MEMORY_WP			BIT_ULL(12)	/* write-protect */
-+#define EFI_MEMORY_RP			BIT_ULL(13)	/* read-protect */
-+#define EFI_MEMORY_XP			BIT_ULL(14)	/* execute-protect */
-+#define EFI_MEMORY_NV			BIT_ULL(15)	/* non-volatile */
-+#define EFI_MEMORY_MORE_RELIABLE	BIT_ULL(16)	/* higher reliability */
-+#define EFI_MEMORY_RO			BIT_ULL(17)	/* read-only */
-+#define EFI_MEMORY_SP			BIT_ULL(18)	/* soft reserved */
-+#define EFI_MEMORY_CPU_CRYPTO		BIT_ULL(19)	/* supports encryption */
-+#define EFI_MEMORY_HOT_PLUGGABLE	BIT_ULL(20)	/* supports unplugging at runtime */
-+#define EFI_MEMORY_RUNTIME		BIT_ULL(63)	/* range requires runtime mapping */
-+
- #define EFI_MEMORY_DESCRIPTOR_VERSION	1
- 
- #define EFI_PAGE_SHIFT		12
-@@ -286,7 +290,7 @@ typedef efi_status_t efi_get_variable_t (efi_char16_t *name, efi_guid_t *vendor,
- 					 unsigned long *data_size, void *data);
- typedef efi_status_t efi_get_next_variable_t (unsigned long *name_size, efi_char16_t *name,
- 					      efi_guid_t *vendor);
-typedef efi_status_t efi_set_variable_t (efi_char16_t *name, efi_guid_t *vendor, 
-+typedef efi_status_t efi_set_variable_t (efi_char16_t *name, efi_guid_t *vendor,
- 					 u32 attr, unsigned long data_size,
- 					 void *data);
- typedef efi_status_t efi_get_next_high_mono_count_t (u32 *count);
-@@ -356,36 +360,40 @@ void efi_native_runtime_setup(void);
-  * where the UEFI SPEC breaks the line.
-  */
- #define NULL_GUID				EFI_GUID(0x00000000, 0x0000, 0x0000,  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)
-#define MPS_TABLE_GUID				EFI_GUID(0xeb9d2d2f, 0x2d88, 0x11d3,  0x9a, 0x16, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d)
- #define ACPI_TABLE_GUID				EFI_GUID(0xeb9d2d30, 0x2d88, 0x11d3,  0x9a, 0x16, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d)
- #define ACPI_20_TABLE_GUID			EFI_GUID(0x8868e871, 0xe4f1, 0x11d3,  0xbc, 0x22, 0x00, 0x80, 0xc7, 0x3c, 0x88, 0x81)
- #define SMBIOS_TABLE_GUID			EFI_GUID(0xeb9d2d31, 0x2d88, 0x11d3,  0x9a, 0x16, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d)
- #define SMBIOS3_TABLE_GUID			EFI_GUID(0xf2fd1544, 0x9794, 0x4a2c,  0x99, 0x2e, 0xe5, 0xbb, 0xcf, 0x20, 0xe3, 0x94)
-#define SAL_SYSTEM_TABLE_GUID			EFI_GUID(0xeb9d2d32, 0x2d88, 0x11d3,  0x9a, 0x16, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d)
-#define HCDP_TABLE_GUID				EFI_GUID(0xf951938d, 0x620b, 0x42ef,  0x82, 0x79, 0xa8, 0x4b, 0x79, 0x61, 0x78, 0x98)
-#define UGA_IO_PROTOCOL_GUID			EFI_GUID(0x61a4d49e, 0x6f68, 0x4f1b,  0xb9, 0x22, 0xa8, 0x6e, 0xed, 0x0b, 0x07, 0xa2)
- #define EFI_GLOBAL_VARIABLE_GUID		EFI_GUID(0x8be4df61, 0x93ca, 0x11d2,  0xaa, 0x0d, 0x00, 0xe0, 0x98, 0x03, 0x2b, 0x8c)
- #define UV_SYSTEM_TABLE_GUID			EFI_GUID(0x3b13a7d4, 0x633e, 0x11dd,  0x93, 0xec, 0xda, 0x25, 0x56, 0xd8, 0x95, 0x93)
- #define LINUX_EFI_CRASH_GUID			EFI_GUID(0xcfc8fc79, 0xbe2e, 0x4ddc,  0x97, 0xf0, 0x9f, 0x98, 0xbf, 0xe2, 0x98, 0xa0)
- #define LOADED_IMAGE_PROTOCOL_GUID		EFI_GUID(0x5b1b31a1, 0x9562, 0x11d2,  0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b)
-+#define LOADED_IMAGE_DEVICE_PATH_PROTOCOL_GUID	EFI_GUID(0xbc62157e, 0x3e33, 0x4fec,  0x99, 0x20, 0x2d, 0x3b, 0x36, 0xd7, 0x50, 0xdf)
-+#define EFI_DEVICE_PATH_PROTOCOL_GUID		EFI_GUID(0x09576e91, 0x6d3f, 0x11d2,  0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b)
-+#define EFI_DEVICE_PATH_TO_TEXT_PROTOCOL_GUID	EFI_GUID(0x8b843e20, 0x8132, 0x4852,  0x90, 0xcc, 0x55, 0x1a, 0x4e, 0x4a, 0x7f, 0x1c)
-+#define EFI_DEVICE_PATH_FROM_TEXT_PROTOCOL_GUID	EFI_GUID(0x05c99a21, 0xc70f, 0x4ad2,  0x8a, 0x5f, 0x35, 0xdf, 0x33, 0x43, 0xf5, 0x1e)
- #define EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID	EFI_GUID(0x9042a9de, 0x23dc, 0x4a38,  0x96, 0xfb, 0x7a, 0xde, 0xd0, 0x80, 0x51, 0x6a)
-#define EFI_UGA_PROTOCOL_GUID			EFI_GUID(0x982c298b, 0xf4fa, 0x41cb,  0xb8, 0x38, 0x77, 0xaa, 0x68, 0x8f, 0xb8, 0x39)
-+#define EFI_EDID_DISCOVERED_PROTOCOL_GUID	EFI_GUID(0x1c0c34f6, 0xd380, 0x41fa,  0xa0, 0x49, 0x8a, 0xd0, 0x6c, 0x1a, 0x66, 0xaa)
-+#define EFI_EDID_ACTIVE_PROTOCOL_GUID		EFI_GUID(0xbd8c1056, 0x9f36, 0x44ec,  0x92, 0xa8, 0xa6, 0x33, 0x7f, 0x81, 0x79, 0x86)
- #define EFI_PCI_IO_PROTOCOL_GUID		EFI_GUID(0x4cf5b200, 0x68b8, 0x4ca5,  0x9e, 0xec, 0xb2, 0x3e, 0x3f, 0x50, 0x02, 0x9a)
- #define EFI_FILE_INFO_ID			EFI_GUID(0x09576e92, 0x6d3f, 0x11d2,  0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b)
- #define EFI_SYSTEM_RESOURCE_TABLE_GUID		EFI_GUID(0xb122a263, 0x3661, 0x4f68,  0x99, 0x29, 0x78, 0xf8, 0xb0, 0xd6, 0x21, 0x80)
- #define EFI_FILE_SYSTEM_GUID			EFI_GUID(0x964e5b22, 0x6459, 0x11d2,  0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b)
- #define DEVICE_TREE_GUID			EFI_GUID(0xb1b621d5, 0xf19c, 0x41a5,  0x83, 0x0b, 0xd9, 0x15, 0x2c, 0x69, 0xaa, 0xe0)
-#define EFI_PROPERTIES_TABLE_GUID		EFI_GUID(0x880aaca3, 0x4adc, 0x4a04,  0x90, 0x79, 0xb7, 0x47, 0x34, 0x08, 0x25, 0xe5)
- #define EFI_RNG_PROTOCOL_GUID			EFI_GUID(0x3152bca5, 0xeade, 0x433d,  0x86, 0x2e, 0xc0, 0x1c, 0xdc, 0x29, 0x1f, 0x44)
- #define EFI_RNG_ALGORITHM_RAW			EFI_GUID(0xe43176d7, 0xb6e8, 0x4827,  0xb7, 0x84, 0x7f, 0xfd, 0xc4, 0xb6, 0x85, 0x61)
- #define EFI_MEMORY_ATTRIBUTES_TABLE_GUID	EFI_GUID(0xdcfa911d, 0x26eb, 0x469f,  0xa2, 0x20, 0x38, 0xb7, 0xdc, 0x46, 0x12, 0x20)
- #define EFI_CONSOLE_OUT_DEVICE_GUID		EFI_GUID(0xd3b36f2c, 0xd551, 0x11d4,  0x9a, 0x46, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d)
- #define APPLE_PROPERTIES_PROTOCOL_GUID		EFI_GUID(0x91bd12fe, 0xf6c3, 0x44fb,  0xa5, 0xb7, 0x51, 0x22, 0xab, 0x30, 0x3a, 0xe0)
-+#define APPLE_SET_OS_PROTOCOL_GUID		EFI_GUID(0xc5c5da95, 0x7d5c, 0x45e6,  0xb2, 0xf1, 0x3f, 0xd5, 0x2b, 0xb1, 0x00, 0x77)
- #define EFI_TCG2_PROTOCOL_GUID			EFI_GUID(0x607f766c, 0x7455, 0x42be,  0x93, 0x0b, 0xe4, 0xd7, 0x6d, 0xb2, 0x72, 0x0f)
-+#define EFI_TCG2_FINAL_EVENTS_TABLE_GUID	EFI_GUID(0x1e2ed096, 0x30e2, 0x4254,  0xbd, 0x89, 0x86, 0x3b, 0xbe, 0xf8, 0x23, 0x25)
- #define EFI_LOAD_FILE_PROTOCOL_GUID		EFI_GUID(0x56ec3091, 0x954c, 0x11d2,  0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b)
- #define EFI_LOAD_FILE2_PROTOCOL_GUID		EFI_GUID(0x4006c0c1, 0xfcb3, 0x403e,  0x99, 0x6d, 0x4a, 0x6c, 0x87, 0x24, 0xe0, 0x6d)
- #define EFI_RT_PROPERTIES_TABLE_GUID		EFI_GUID(0xeb66918a, 0x7eef, 0x402a,  0x84, 0x2e, 0x93, 0x1d, 0x21, 0xc3, 0x8a, 0xe9)
- #define EFI_DXE_SERVICES_TABLE_GUID		EFI_GUID(0x05ad34ba, 0x6f02, 0x4214,  0x95, 0x2e, 0x4d, 0xa0, 0x39, 0x8e, 0x2b, 0xb9)
-+#define EFI_SMBIOS_PROTOCOL_GUID		EFI_GUID(0x03583ff6, 0xcb36, 0x4940,  0x94, 0x7e, 0xb9, 0xb3, 0x9f, 0x4a, 0xfa, 0xf7)
-+#define EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID	EFI_GUID(0xf4560cf6, 0x40ec, 0x4b4a,  0xa1, 0x92, 0xbf, 0x1d, 0x57, 0xd0, 0xb1, 0x89)
- 
- #define EFI_IMAGE_SECURITY_DATABASE_GUID	EFI_GUID(0xd719b2cb, 0x3d3a, 0x4596,  0xa3, 0xbc, 0xda, 0xd0, 0x0e, 0x67, 0x65, 0x6f)
- #define EFI_SHIM_LOCK_GUID			EFI_GUID(0x605dab50, 0xe046, 0x4300,  0xab, 0xb6, 0x3d, 0xd8, 0x10, 0xdd, 0x8b, 0x23)
-@@ -394,22 +402,25 @@ void efi_native_runtime_setup(void);
- #define EFI_CERT_X509_GUID			EFI_GUID(0xa5c059a1, 0x94e4, 0x4aa7, 0x87, 0xb5, 0xab, 0x15, 0x5c, 0x2b, 0xf0, 0x72)
- #define EFI_CERT_X509_SHA256_GUID		EFI_GUID(0x3bd2a492, 0x96c0, 0x4079, 0xb4, 0x20, 0xfc, 0xf9, 0x8e, 0xf1, 0x03, 0xed)
- #define EFI_CC_BLOB_GUID			EFI_GUID(0x067b1f5f, 0xcf26, 0x44c5, 0x85, 0x54, 0x93, 0xd7, 0x77, 0x91, 0x2d, 0x42)
-+#define EFI_CC_MEASUREMENT_PROTOCOL_GUID	EFI_GUID(0x96751a3d, 0x72f4, 0x41a6, 0xa7, 0x94, 0xed, 0x5d, 0x0e, 0x67, 0xae, 0x6b)
-+#define EFI_CC_FINAL_EVENTS_TABLE_GUID		EFI_GUID(0xdd4a4648, 0x2de7, 0x4665, 0x96, 0x4d, 0x21, 0xd9, 0xef, 0x5f, 0xb4, 0x46)
- 
- /*
-  * This GUID is used to pass to the kernel proper the struct screen_info
-  * structure that was populated by the stub based on the GOP protocol instance
-  * associated with ConOut
-  */
-#define LINUX_EFI_ARM_SCREEN_INFO_TABLE_GUID	EFI_GUID(0xe03fc20a, 0x85dc, 0x406e,  0xb9, 0x0e, 0x4a, 0xb5, 0x02, 0x37, 0x1d, 0x95)
-+#define LINUX_EFI_SCREEN_INFO_TABLE_GUID	EFI_GUID(0xe03fc20a, 0x85dc, 0x406e,  0xb9, 0x0e, 0x4a, 0xb5, 0x02, 0x37, 0x1d, 0x95)
- #define LINUX_EFI_ARM_CPU_STATE_TABLE_GUID	EFI_GUID(0xef79e4aa, 0x3c3d, 0x4989,  0xb9, 0x02, 0x07, 0xa9, 0x43, 0xe5, 0x50, 0xd2)
- #define LINUX_EFI_LOADER_ENTRY_GUID		EFI_GUID(0x4a67b082, 0x0a4c, 0x41cf,  0xb6, 0xc7, 0x44, 0x0b, 0x29, 0xbb, 0x8c, 0x4f)
- #define LINUX_EFI_RANDOM_SEED_TABLE_GUID	EFI_GUID(0x1ce1e5bc, 0x7ceb, 0x42f2,  0x81, 0xe5, 0x8a, 0xad, 0xf1, 0x80, 0xf5, 0x7b)
- #define LINUX_EFI_TPM_EVENT_LOG_GUID		EFI_GUID(0xb7799cb0, 0xeca2, 0x4943,  0x96, 0x67, 0x1f, 0xae, 0x07, 0xb7, 0x47, 0xfa)
-#define LINUX_EFI_TPM_FINAL_LOG_GUID		EFI_GUID(0x1e2ed096, 0x30e2, 0x4254,  0xbd, 0x89, 0x86, 0x3b, 0xbe, 0xf8, 0x23, 0x25)
- #define LINUX_EFI_MEMRESERVE_TABLE_GUID		EFI_GUID(0x888eb0c6, 0x8ede, 0x4ff5,  0xa8, 0xf0, 0x9a, 0xee, 0x5c, 0xb9, 0x77, 0xc2)
- #define LINUX_EFI_INITRD_MEDIA_GUID		EFI_GUID(0x5568e427, 0x68fc, 0x4f3d,  0xac, 0x74, 0xca, 0x55, 0x52, 0x31, 0xcc, 0x68)
- #define LINUX_EFI_MOK_VARIABLE_TABLE_GUID	EFI_GUID(0xc451ed2b, 0x9694, 0x45d3,  0xba, 0xba, 0xed, 0x9f, 0x89, 0x88, 0xa3, 0x89)
- #define LINUX_EFI_COCO_SECRET_AREA_GUID		EFI_GUID(0xadf956ad, 0xe98c, 0x484c,  0xae, 0x11, 0xb5, 0x1c, 0x7d, 0x33, 0x64, 0x47)
-+#define LINUX_EFI_BOOT_MEMMAP_GUID		EFI_GUID(0x800f683f, 0xd08b, 0x423a,  0xa2, 0x93, 0x96, 0x5c, 0x3c, 0x6f, 0xe2, 0xb4)
-+#define LINUX_EFI_UNACCEPTED_MEM_TABLE_GUID	EFI_GUID(0xd5d1de3c, 0x105c, 0x44f9,  0x9e, 0xa9, 0xbc, 0xef, 0x98, 0x12, 0x00, 0x31)
- 
- #define RISCV_EFI_BOOT_PROTOCOL_GUID		EFI_GUID(0xccd15fec, 0x6f73, 0x4eec,  0x83, 0x95, 0x3e, 0x69, 0xe4, 0xb9, 0x40, 0xbf)
- 
-@@ -428,6 +439,10 @@ void efi_native_runtime_setup(void);
- #define DELLEMC_EFI_RCI2_TABLE_GUID		EFI_GUID(0x2d9f28a2, 0xa886, 0x456a,  0x97, 0xa8, 0xf1, 0x1e, 0xf2, 0x4f, 0xf4, 0x55)
- #define AMD_SEV_MEM_ENCRYPT_GUID		EFI_GUID(0x0cf29b71, 0x9e51, 0x433a,  0xa3, 0xb7, 0x81, 0xf3, 0xab, 0x16, 0xb8, 0x75)
- 
-+/* OVMF protocol GUIDs */
-+#define OVMF_SEV_MEMORY_ACCEPTANCE_PROTOCOL_GUID	EFI_GUID(0xc5a010fe, 0x38a7, 0x4531,  0x8a, 0x4a, 0x05, 0x00, 0xd2, 0xfd, 0x16, 0x49)
-+#define OVMF_MEMORY_LOG_TABLE_GUID		EFI_GUID(0x95305139, 0xb20f, 0x4723,  0x84, 0x25, 0x62, 0x7c, 0x88, 0x8f, 0xf1, 0x21)
-+
- typedef struct {
- 	efi_guid_t guid;
- 	u64 table;
-@@ -518,6 +533,23 @@ typedef union {
- 	efi_system_table_32_t mixed_mode;
- } efi_system_table_t;
- 
-+struct efi_boot_memmap {
-+	unsigned long		map_size;
-+	unsigned long		desc_size;
-+	u32			desc_ver;
-+	unsigned long		map_key;
-+	unsigned long		buff_size;
-+	efi_memory_desc_t	map[];
-+};
-+
-+struct efi_unaccepted_memory {
-+	u32 version;
-+	u32 unit_size;
-+	u64 phys_base;
-+	u64 size;
-+	unsigned long bitmap[];
-+};
-+
- /*
-  * Architecture independent structure for describing a memory map for the
-  * benefit of efi_memmap_init_early(), and for passing context between
-@@ -550,15 +582,6 @@ struct efi_mem_range {
- };
- 
- typedef struct {
-	u32 version;
-	u32 length;
-	u64 memory_protection_attribute;
-} efi_properties_table_t;
-
-#define EFI_PROPERTIES_TABLE_VERSION	0x00010000
-#define EFI_PROPERTIES_RUNTIME_MEMORY_PROTECTION_NON_EXECUTABLE_PE_DATA	0x1
-
-typedef struct {
- 	u16 version;
- 	u16 length;
- 	u32 runtime_services_supported;
-@@ -568,12 +591,20 @@ typedef struct {
- 
- #define EFI_INVALID_TABLE_ADDR		(~0UL)
- 
-+// BIT0 implies that Runtime code includes the forward control flow guard
-+// instruction, such as X86 CET-IBT or ARM BTI.
-+#define EFI_MEMORY_ATTRIBUTES_FLAGS_RT_FORWARD_CONTROL_FLOW_GUARD	0x1
-+
- typedef struct {
- 	u32 version;
- 	u32 num_entries;
- 	u32 desc_size;
-	u32 reserved;
-	efi_memory_desc_t entry[0];
-+	u32 flags;
-+	/*
-+	 * There are @num_entries following, each of size @desc_size bytes,
-+	 * including an efi_memory_desc_t header. See efi_memdesc_ptr().
-+	 */
-+	efi_memory_desc_t entry[];
- } efi_memory_attributes_table_t;
- 
- typedef struct {
-@@ -614,8 +645,10 @@ extern struct efi {
- 	unsigned long			esrt;			/* ESRT table */
- 	unsigned long			tpm_log;		/* TPM2 Event Log table */
- 	unsigned long			tpm_final_log;		/* TPM2 Final Events Log table */
-+	unsigned long                   ovmf_debug_log;
- 	unsigned long			mokvar_table;		/* MOK variable config table */
- 	unsigned long			coco_secret;		/* Confidential computing secret table */
-+	unsigned long			unaccepted;		/* Unaccepted memory table */
- 
- 	efi_get_time_t			*get_time;
- 	efi_set_time_t			*set_time;
-@@ -653,11 +686,17 @@ extern struct efi {
- 
- #define EFI_RT_SUPPORTED_ALL					0x3fff
- 
-#define EFI_RT_SUPPORTED_TIME_SERVICES				0x000f
-+#define EFI_RT_SUPPORTED_TIME_SERVICES				0x0003
-+#define EFI_RT_SUPPORTED_WAKEUP_SERVICES			0x000c
- #define EFI_RT_SUPPORTED_VARIABLE_SERVICES			0x0070
- 
- extern struct mm_struct efi_mm;
- 
-+static inline bool mm_is_efi(struct mm_struct *mm)
-+{
-+	return IS_ENABLED(CONFIG_EFI) && mm == &efi_mm;
-+}
-+
- static inline int
- efi_guidcmp (efi_guid_t left, efi_guid_t right)
- {
-@@ -672,6 +711,7 @@ efi_guid_to_str(efi_guid_t *guid, char *out)
- }
- 
- extern void efi_init (void);
-+extern void efi_earlycon_reprobe(void);
- #ifdef CONFIG_EFI
- extern void efi_enter_virtual_mode (void);	/* switch EFI to virtual mode, if possible */
- #else
-@@ -690,20 +730,11 @@ static inline efi_status_t efi_query_variable_store(u32 attributes,
- 	return EFI_SUCCESS;
- }
- #endif
-extern void __iomem *efi_lookup_mapped_addr(u64 phys_addr);
- 
-extern int __init efi_memmap_alloc(unsigned int num_entries,
-				   struct efi_memory_map_data *data);
-extern void __efi_memmap_free(u64 phys, unsigned long size,
-			      unsigned long flags);
-+extern int __init __efi_memmap_init(struct efi_memory_map_data *data);
- extern int __init efi_memmap_init_early(struct efi_memory_map_data *data);
- extern int __init efi_memmap_init_late(phys_addr_t addr, unsigned long size);
- extern void __init efi_memmap_unmap(void);
-extern int __init efi_memmap_install(struct efi_memory_map_data *data);
-extern int __init efi_memmap_split_count(efi_memory_desc_t *md,
-					 struct range *range);
-extern void __init efi_memmap_insert(struct efi_memory_map *old_memmap,
-				     void *buf, struct efi_mem_range *mem);
- 
- #ifdef CONFIG_EFI_ESRT
- extern void __init efi_esrt_init(void);
-@@ -713,8 +744,7 @@ static inline void efi_esrt_init(void) { }
- extern int efi_config_parse_tables(const efi_config_table_t *config_tables,
- 				   int count,
- 				   const efi_config_table_type_t *arch_tables);
-extern int efi_systab_check_header(const efi_table_hdr_t *systab_hdr,
-				   int min_major_version);
-+extern int efi_systab_check_header(const efi_table_hdr_t *systab_hdr);
- extern void efi_systab_report_header(const efi_table_hdr_t *systab_hdr,
- 				     unsigned long fw_vendor);
- extern u64 efi_get_iobase (void);
-@@ -724,22 +754,15 @@ extern u64 efi_mem_attribute (unsigned long phys_addr, unsigned long size);
- extern int __init efi_uart_console_only (void);
- extern u64 efi_mem_desc_end(efi_memory_desc_t *md);
- extern int efi_mem_desc_lookup(u64 phys_addr, efi_memory_desc_t *out_md);
-+extern int __efi_mem_desc_lookup(u64 phys_addr, efi_memory_desc_t *out_md);
- extern void efi_mem_reserve(phys_addr_t addr, u64 size);
- extern int efi_mem_reserve_persistent(phys_addr_t addr, u64 size);
-extern void efi_initialize_iomem_resources(struct resource *code_resource,
-		struct resource *data_resource, struct resource *bss_resource);
- extern u64 efi_get_fdt_params(struct efi_memory_map_data *data);
- extern struct kobject *efi_kobj;
- 
- extern int efi_reboot_quirk_mode;
- extern bool efi_poweroff_required(void);
- 
-#ifdef CONFIG_EFI_FAKE_MEMMAP
-extern void __init efi_fake_memmap(void);
-#else
-static inline void efi_fake_memmap(void) { }
-#endif
-
- extern unsigned long efi_mem_attr_table;
- 
- /*
-@@ -749,14 +772,14 @@ extern unsigned long efi_mem_attr_table;
-  *                           argument in the page tables referred to by the
-  *                           first argument.
-  */
-typedef int (*efi_memattr_perm_setter)(struct mm_struct *, efi_memory_desc_t *);
-+typedef int (*efi_memattr_perm_setter)(struct mm_struct *, efi_memory_desc_t *, bool);
- 
-extern int efi_memattr_init(void);
-+extern void efi_memattr_init(void);
- extern int efi_memattr_apply_permissions(struct mm_struct *mm,
- 					 efi_memattr_perm_setter fn);
- 
- /*
- * efi_early_memdesc_ptr - get the n-th EFI memmap descriptor
-+ * efi_memdesc_ptr - get the n-th EFI memmap descriptor
-  * @map: the start of efi memmap
-  * @desc_size: the size of space for each EFI memmap descriptor
-  * @n: the index of efi memmap descriptor
-@@ -774,7 +797,7 @@ extern int efi_memattr_apply_permissions(struct mm_struct *mm,
-  * during bootup since for_each_efi_memory_desc_xxx() is available after the
-  * kernel initializes the EFI subsystem to set up struct efi_memory_map.
-  */
-#define efi_early_memdesc_ptr(map, desc_size, n)			\
-+#define efi_memdesc_ptr(map, desc_size, n)			\
- 	(efi_memory_desc_t *)((void *)(map) + ((n) * (desc_size)))
- 
- /* Iterate through an efi_memory_map */
-@@ -829,10 +852,6 @@ static inline int efi_range_is_wc(unsigned long start, unsigned long len)
- 	return 1;
- }
- 
-#ifdef CONFIG_EFI_PCDP
-extern int __init efi_setup_pcdp_console(char *);
-#endif
-
- /*
-  * We play games with efi_enabled so that the compiler will, if
-  * possible, remove EFI-related code altogether.
-@@ -845,10 +864,9 @@ extern int __init efi_setup_pcdp_console(char *);
- #define EFI_PARAVIRT		6	/* Access is via a paravirt interface */
- #define EFI_ARCH_1		7	/* First arch-specific bit */
- #define EFI_DBG			8	/* Print additional debug info at runtime */
-#define EFI_NX_PE_DATA		9	/* Can runtime data regions be mapped non-executable? */
-#define EFI_MEM_ATTR		10	/* Did firmware publish an EFI_MEMORY_ATTRIBUTES table? */
-#define EFI_MEM_NO_SOFT_RESERVE	11	/* Is the kernel configured to ignore soft reservations? */
-#define EFI_PRESERVE_BS_REGIONS	12	/* Are EFI boot-services memory segments available? */
-+#define EFI_MEM_ATTR		9	/* Did firmware publish an EFI_MEMORY_ATTRIBUTES table? */
-+#define EFI_MEM_NO_SOFT_RESERVE	10	/* Is the kernel configured to ignore soft reservations? */
-+#define EFI_PRESERVE_BS_REGIONS	11	/* Are EFI boot-services memory segments available? */
- 
- #ifdef CONFIG_EFI
- /*
-@@ -952,6 +970,7 @@ extern int efi_status_to_err(efi_status_t status);
- #define   EFI_DEV_MEDIA_VENDOR			 3
- #define   EFI_DEV_MEDIA_FILE			 4
- #define   EFI_DEV_MEDIA_PROTOCOL		 5
-+#define   EFI_DEV_MEDIA_REL_OFFSET		 8
- #define EFI_DEV_BIOS_BOOT		0x05
- #define EFI_DEV_END_PATH		0x7F
- #define EFI_DEV_END_PATH2		0xFF
-@@ -982,12 +1001,32 @@ struct efi_vendor_dev_path {
- 	u8				vendordata[];
- } __packed;
- 
-+struct efi_rel_offset_dev_path {
-+	struct efi_generic_dev_path	header;
-+	u32				reserved;
-+	u64				starting_offset;
-+	u64				ending_offset;
-+} __packed;
-+
-+struct efi_mem_mapped_dev_path {
-+	struct efi_generic_dev_path	header;
-+	u32				memory_type;
-+	u64				starting_addr;
-+	u64				ending_addr;
-+} __packed;
-+
-+struct efi_file_path_dev_path {
-+	struct efi_generic_dev_path	header;
-+	efi_char16_t			filename[];
-+} __packed;
-+
- struct efi_dev_path {
- 	union {
- 		struct efi_generic_dev_path	header;
- 		struct efi_acpi_dev_path	acpi;
- 		struct efi_pci_dev_path		pci;
- 		struct efi_vendor_dev_path	vendor;
-+		struct efi_rel_offset_dev_path	rel_offset;
- 	};
- } __packed;
- 
-@@ -1013,30 +1052,39 @@ struct efivar_operations {
- 	efi_set_variable_t *set_variable;
- 	efi_set_variable_t *set_variable_nonblocking;
- 	efi_query_variable_store_t *query_variable_store;
-+	efi_query_variable_info_t *query_variable_info;
- };
- 
- struct efivars {
- 	struct kset *kset;
-	struct kobject *kobject;
- 	const struct efivar_operations *ops;
- };
- 
-+#ifdef CONFIG_X86
-+u64 __attribute_const__ efivar_reserved_space(void);
-+#else
-+static inline u64 efivar_reserved_space(void) { return 0; }
-+#endif
-+
- /*
- * The maximum size of VariableName + Data = 1024
- * Therefore, it's reasonable to save that much
- * space in each part of the structure,
- * and we use a page for reading/writing.
-+ * There is no actual upper limit specified for the variable name size.
-+ *
-+ * This limit exists only for practical purposes, since name conversions
-+ * are bounds-checked and name data is occasionally stored in-line.
-  */
-
- #define EFI_VAR_NAME_LEN	1024
- 
- int efivars_register(struct efivars *efivars,
-		     const struct efivar_operations *ops,
-		     struct kobject *kobject);
-+		     const struct efivar_operations *ops);
- int efivars_unregister(struct efivars *efivars);
-struct kobject *efivars_kobject(void);
- 
-int efivar_supports_writes(void);
-+#ifdef CONFIG_EFI
-+bool efivar_is_available(void);
-+#else
-+static inline bool efivar_is_available(void) { return false; }
-+#endif
-+
-+bool efivar_supports_writes(void);
- 
- int efivar_lock(void);
- int efivar_trylock(void);
-@@ -1055,8 +1103,9 @@ efi_status_t efivar_set_variable_locked(efi_char16_t *name, efi_guid_t *vendor,
- efi_status_t efivar_set_variable(efi_char16_t *name, efi_guid_t *vendor,
- 				 u32 attr, unsigned long data_size, void *data);
- 
-efi_status_t check_var_size(u32 attributes, unsigned long size);
-efi_status_t check_var_size_nonblocking(u32 attributes, unsigned long size);
-+efi_status_t efivar_query_variable_info(u32 attr, u64 *storage_space,
-+					u64 *remaining_space,
-+					u64 *max_variable_size);
- 
- #if IS_ENABLED(CONFIG_EFI_CAPSULE_LOADER)
- extern bool efi_capsule_pending(int *reset_type);
-@@ -1070,43 +1119,17 @@ extern int efi_capsule_update(efi_capsule_header_t *capsule,
- static inline bool efi_capsule_pending(int *reset_type) { return false; }
- #endif
- 
-#ifdef CONFIG_EFI_RUNTIME_MAP
-int efi_runtime_map_init(struct kobject *);
-int efi_get_runtime_map_size(void);
-int efi_get_runtime_map_desc_size(void);
-int efi_runtime_map_copy(void *buf, size_t bufsz);
-#else
-static inline int efi_runtime_map_init(struct kobject *kobj)
-{
-	return 0;
-}
-
-static inline int efi_get_runtime_map_size(void)
-{
-	return 0;
-}
-
-static inline int efi_get_runtime_map_desc_size(void)
-{
-	return 0;
-}
-
-static inline int efi_runtime_map_copy(void *buf, size_t bufsz)
-{
-	return 0;
-}
-
-#endif
-
- #ifdef CONFIG_EFI
- extern bool efi_runtime_disabled(void);
- #else
- static inline bool efi_runtime_disabled(void) { return true; }
- #endif
- 
-extern void efi_call_virt_check_flags(unsigned long flags, const char *call);
-+extern void efi_call_virt_check_flags(unsigned long flags, const void *caller);
- extern unsigned long efi_call_virt_save_flags(void);
- 
-+void efi_runtime_assert_lock_held(void);
-+
- enum efi_secureboot_mode {
- 	efi_secureboot_mode_unset,
- 	efi_secureboot_mode_unknown,
-@@ -1142,13 +1165,10 @@ void efi_check_for_embedded_firmwares(void);
- static inline void efi_check_for_embedded_firmwares(void) { }
- #endif
- 
-efi_status_t efi_random_get_seed(void);
-
- #define arch_efi_call_virt(p, f, args...)	((p)->f(args))
- 
- /*
- * Arch code can implement the following three template macros, avoiding
- * reptition for the void/non-void return cases of {__,}efi_call_virt():
-+ * Arch code must implement the following three routines:
-  *
-  *  * arch_efi_call_virt_setup()
-  *
-@@ -1157,9 +1177,8 @@ efi_status_t efi_random_get_seed(void);
-  *
-  *  * arch_efi_call_virt()
-  *
- *    Performs the call. The last expression in the macro must be the call
- *    itself, allowing the logic to be shared by the void and non-void
- *    cases.
-+ *    Performs the call. This routine takes a variable number of arguments so
-+ *    it must be implemented as a variadic preprocessor macro.
-  *
-  *  * arch_efi_call_virt_teardown()
-  *
-@@ -1168,34 +1187,21 @@ efi_status_t efi_random_get_seed(void);
- 
- #define efi_call_virt_pointer(p, f, args...)				\
- ({									\
-	efi_status_t __s;						\
-+	typeof((p)->f(args)) __s;					\
- 	unsigned long __flags;						\
- 									\
- 	arch_efi_call_virt_setup();					\
- 									\
- 	__flags = efi_call_virt_save_flags();				\
- 	__s = arch_efi_call_virt(p, f, args);				\
-	efi_call_virt_check_flags(__flags, __stringify(f));		\
-+	efi_call_virt_check_flags(__flags, NULL);			\
- 									\
- 	arch_efi_call_virt_teardown();					\
- 									\
- 	__s;								\
- })
- 
-#define __efi_call_virt_pointer(p, f, args...)				\
-({									\
-	unsigned long __flags;						\
-									\
-	arch_efi_call_virt_setup();					\
-									\
-	__flags = efi_call_virt_save_flags();				\
-	arch_efi_call_virt(p, f, args);					\
-	efi_call_virt_check_flags(__flags, __stringify(f));		\
-									\
-	arch_efi_call_virt_teardown();					\
-})
-
-#define EFI_RANDOM_SEED_SIZE		64U
-+#define EFI_RANDOM_SEED_SIZE		32U // BLAKE2S_HASH_SIZE
- 
- struct linux_efi_random_seed {
- 	u32	size;
-@@ -1220,6 +1226,10 @@ extern int efi_tpm_final_log_size;
- 
- extern unsigned long rci2_table_phys;
- 
-+efi_status_t
-+efi_call_acpi_prm_handler(efi_status_t (__efiapi *handler_addr)(u64, void *),
-+			  u64 param_buffer_addr, void *context);
-+
- /*
-  * efi_runtime_service() function identifiers.
-  * "NONE" is used by efi_recover_from_page_fault() to check if the page
-@@ -1239,25 +1249,26 @@ enum efi_rts_ids {
- 	EFI_RESET_SYSTEM,
- 	EFI_UPDATE_CAPSULE,
- 	EFI_QUERY_CAPSULE_CAPS,
-+	EFI_ACPI_PRM_HANDLER,
- };
- 
-+union efi_rts_args;
-+
- /*
-  * efi_runtime_work:	Details of EFI Runtime Service work
- * @arg<1-5>:		EFI Runtime Service function arguments
-+ * @args:		Pointer to union describing the arguments
-  * @status:		Status of executing EFI Runtime Service
-  * @efi_rts_id:		EFI Runtime Service function identifier
-  * @efi_rts_comp:	Struct used for handling completions
-+ * @caller:		The caller of the runtime service
-  */
- struct efi_runtime_work {
-	void *arg1;
-	void *arg2;
-	void *arg3;
-	void *arg4;
-	void *arg5;
-	efi_status_t status;
-	struct work_struct work;
-	enum efi_rts_ids efi_rts_id;
-	struct completion efi_rts_comp;
-+	union efi_rts_args	*args;
-+	efi_status_t		status;
-+	struct work_struct	work;
-+	enum efi_rts_ids	efi_rts_id;
-+	struct completion	efi_rts_comp;
-+	const void		*caller;
- };
- 
- extern struct efi_runtime_work efi_rts_work;
-@@ -1280,8 +1291,6 @@ struct linux_efi_memreserve {
- 
- void __init efi_arch_mem_reserve(phys_addr_t addr, u64 size);
- 
-char *efi_systab_show_arch(char *str);
-
- /*
-  * The LINUX_EFI_MOK_VARIABLE_TABLE_GUID config table can be provided
-  * to the kernel by an EFI boot loader. The table contains a packed
-@@ -1321,7 +1330,37 @@ struct linux_efi_coco_secret_area {
- 	u64	size;
- };
- 
-+struct linux_efi_initrd {
-+	unsigned long	base;
-+	unsigned long	size;
-+};
-+
- /* Header of a populated EFI secret area */
- #define EFI_SECRET_TABLE_HEADER_GUID	EFI_GUID(0x1e74f542, 0x71dd, 0x4d66,  0x96, 0x3e, 0xef, 0x42, 0x87, 0xff, 0x17, 0x3b)
- 
-+bool xen_efi_config_table_is_usable(const efi_guid_t *guid, unsigned long table);
-+
-+static __always_inline
-+bool efi_config_table_is_usable(const efi_guid_t *guid, unsigned long table)
-+{
-+	if (!IS_ENABLED(CONFIG_XEN_EFI))
-+		return true;
-+	return xen_efi_config_table_is_usable(guid, table);
-+}
-+
-+umode_t efi_attr_is_visible(struct kobject *kobj, struct attribute *attr, int n);
-+
-+int ovmf_log_probe(unsigned long ovmf_debug_log_table);
-+
-+/*
-+ * efivar ops event type
-+ */
-+#define EFIVAR_OPS_RDONLY 0
-+#define EFIVAR_OPS_RDWR 1
-+
-+extern struct blocking_notifier_head efivar_ops_nh;
-+
-+void efivars_generic_ops_register(void);
-+void efivars_generic_ops_unregister(void);
-+
- #endif /* _LINUX_EFI_H */
-diff --git a/include/linux/ehl_pse_io_aux.h b/include/linux/ehl_pse_io_aux.h
-new file mode 100644
-index 000000000000..afb8587ee5fb
--- /dev/null
-+++ b/include/linux/ehl_pse_io_aux.h
-@@ -0,0 +1,24 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Intel Elkhart Lake PSE I/O Auxiliary Device
-+ *
-+ * Copyright (c) 2025 Intel Corporation.
-+ *
-+ * Author: Raag Jadav <raag.jadav@intel.com>
-+ */
-+
-+#ifndef _EHL_PSE_IO_AUX_H_
-+#define _EHL_PSE_IO_AUX_H_
-+
-+#include <linux/ioport.h>
-+
-+#define EHL_PSE_IO_NAME		"ehl_pse_io"
-+#define EHL_PSE_GPIO_NAME	"gpio"
-+#define EHL_PSE_TIO_NAME	"pps_tio"
-+
-+struct ehl_pse_io_data {
-+	struct resource mem;
-+	int irq;
-+};
-+
-+#endif /* _EHL_PSE_IO_AUX_H_ */
-diff --git a/include/linux/eisa.h b/include/linux/eisa.h
-index b012e30afebd..cf55630b595b 100644
--- a/include/linux/eisa.h
-+++ b/include/linux/eisa.h
-@@ -28,6 +28,9 @@
- #define EISA_CONFIG_ENABLED         1
- #define EISA_CONFIG_FORCED          2
- 
-+/* Chosen to hold the longest string in eisa.ids. */
-+#define EISA_DEVICE_INFO_NAME_SIZE 74
-+
- /* There is not much we can say about an EISA device, apart from
-  * signature, slot number, and base address. dma_mask is set by
-  * default to parent device mask..*/
-@@ -41,7 +44,7 @@ struct eisa_device {
- 	u64                   dma_mask;
- 	struct device         dev; /* generic device */
- #ifdef CONFIG_EISA_NAMES
-	char		      pretty_name[50];
-+	char		      pretty_name[EISA_DEVICE_INFO_NAME_SIZE];
- #endif
- };
- 
-@@ -60,12 +63,12 @@ struct eisa_driver {
- 	struct device_driver         driver;
- };
- 
-#define to_eisa_driver(drv) container_of(drv,struct eisa_driver, driver)
-+#define to_eisa_driver(drv) container_of_const(drv,struct eisa_driver, driver)
- 
- /* These external functions are only available when EISA support is enabled. */
- #ifdef CONFIG_EISA
- 
-extern struct bus_type eisa_bus_type;
-+extern const struct bus_type eisa_bus_type;
- int eisa_driver_register (struct eisa_driver *edrv);
- void eisa_driver_unregister (struct eisa_driver *edrv);
- 
-diff --git a/include/linux/elf-fdpic.h b/include/linux/elf-fdpic.h
-index 3bea95a1af53..e533f4513194 100644
--- a/include/linux/elf-fdpic.h
-+++ b/include/linux/elf-fdpic.h
-@@ -10,13 +10,25 @@
- 
- #include <uapi/linux/elf-fdpic.h>
- 
-+#if ELF_CLASS == ELFCLASS32
-+#define Elf_Sword			Elf32_Sword
-+#define elf_fdpic_loadseg		elf32_fdpic_loadseg
-+#define elf_fdpic_loadmap		elf32_fdpic_loadmap
-+#define ELF_FDPIC_LOADMAP_VERSION	ELF32_FDPIC_LOADMAP_VERSION
-+#else
-+#define Elf_Sword			Elf64_Sxword
-+#define elf_fdpic_loadmap		elf64_fdpic_loadmap
-+#define elf_fdpic_loadseg		elf64_fdpic_loadseg
-+#define ELF_FDPIC_LOADMAP_VERSION	ELF64_FDPIC_LOADMAP_VERSION
-+#endif
-+
- /*
-  * binfmt binary parameters structure
-  */
- struct elf_fdpic_params {
- 	struct elfhdr			hdr;		/* ref copy of ELF header */
- 	struct elf_phdr			*phdrs;		/* ref copy of PT_PHDR table */
-	struct elf32_fdpic_loadmap	*loadmap;	/* loadmap to be passed to userspace */
-+	struct elf_fdpic_loadmap	*loadmap;	/* loadmap to be passed to userspace */
- 	unsigned long			elfhdr_addr;	/* mapped ELF header user address */
- 	unsigned long			ph_addr;	/* mapped PT_PHDR user address */
- 	unsigned long			map_addr;	/* mapped loadmap user address */
-diff --git a/include/linux/elf.h b/include/linux/elf.h
-index c9a46c4e183b..5c402788da19 100644
--- a/include/linux/elf.h
-+++ b/include/linux/elf.h
-@@ -65,7 +65,7 @@ extern Elf64_Dyn _DYNAMIC [];
- struct file;
- struct coredump_params;
- 
-#ifndef ARCH_HAVE_EXTRA_ELF_NOTES
-+#ifndef CONFIG_ARCH_HAVE_EXTRA_ELF_NOTES
- static inline int elf_coredump_extra_notes_size(void) { return 0; }
- static inline int elf_coredump_extra_notes_write(struct coredump_params *cprm) { return 0; }
- #else
-diff --git a/include/linux/elfcore.h b/include/linux/elfcore.h
-index 346a8b56cdc8..bd5560542c79 100644
--- a/include/linux/elfcore.h
-+++ b/include/linux/elfcore.h
-@@ -88,22 +88,13 @@ static inline int elf_core_copy_task_regs(struct task_struct *t, elf_gregset_t*
- {
- #if defined (ELF_CORE_COPY_TASK_REGS)
- 	return ELF_CORE_COPY_TASK_REGS(t, elfregs);
-#elif defined (task_pt_regs)
-+#else
- 	elf_core_copy_regs(elfregs, task_pt_regs(t));
- #endif
- 	return 0;
- }
- 
-extern int dump_fpu (struct pt_regs *, elf_fpregset_t *);
-
-static inline int elf_core_copy_task_fpregs(struct task_struct *t, struct pt_regs *regs, elf_fpregset_t *fpu)
-{
-#ifdef ELF_CORE_COPY_FPREGS
-	return ELF_CORE_COPY_FPREGS(t, fpu);
-#else
-	return dump_fpu(regs, fpu);
-#endif
-}
-+int elf_core_copy_task_fpregs(struct task_struct *t, elf_fpregset_t *fpu);
- 
- #ifdef CONFIG_ARCH_BINFMT_ELF_EXTRA_PHDRS
- /*
-@@ -114,14 +105,14 @@ static inline int elf_core_copy_task_fpregs(struct task_struct *t, struct pt_reg
-  * Dumping its extra ELF program headers includes all the other information
-  * a debugger needs to easily find how the gate DSO was being used.
-  */
-extern Elf_Half elf_core_extra_phdrs(void);
-+extern Elf_Half elf_core_extra_phdrs(struct coredump_params *cprm);
- extern int
- elf_core_write_extra_phdrs(struct coredump_params *cprm, loff_t offset);
- extern int
- elf_core_write_extra_data(struct coredump_params *cprm);
-extern size_t elf_core_extra_data_size(void);
-+extern size_t elf_core_extra_data_size(struct coredump_params *cprm);
- #else
-static inline Elf_Half elf_core_extra_phdrs(void)
-+static inline Elf_Half elf_core_extra_phdrs(struct coredump_params *cprm)
- {
- 	return 0;
- }
-@@ -136,7 +127,7 @@ static inline int elf_core_write_extra_data(struct coredump_params *cprm)
- 	return 1;
- }
- 
-static inline size_t elf_core_extra_data_size(void)
-+static inline size_t elf_core_extra_data_size(struct coredump_params *cprm)
- {
- 	return 0;
- }
-diff --git a/include/linux/elfnote.h b/include/linux/elfnote.h
-index 69b136e4dd2b..bb3dcded055f 100644
--- a/include/linux/elfnote.h
-+++ b/include/linux/elfnote.h
-@@ -60,23 +60,21 @@
- 
- #else	/* !__ASSEMBLER__ */
- #include <uapi/linux/elf.h>
-+#include <linux/compiler.h>
- /*
-  * Use an anonymous structure which matches the shape of
-  * Elf{32,64}_Nhdr, but includes the name and desc data.  The size and
-  * type of name and desc depend on the macro arguments.  "name" must
- * be a literal string, and "desc" must be passed by value.  You may
- * only define one note per line, since __LINE__ is used to generate
- * unique symbols.
-+ * be a literal string, and "desc" must be passed by value.
-  */
-#define _ELFNOTE_PASTE(a,b)	a##b
-#define _ELFNOTE(size, name, unique, type, desc)			\
-+#define ELFNOTE(size, name, type, desc)					\
- 	static const struct {						\
- 		struct elf##size##_note _nhdr;				\
- 		unsigned char _name[sizeof(name)]			\
- 		__attribute__((aligned(sizeof(Elf##size##_Word))));	\
- 		typeof(desc) _desc					\
- 			     __attribute__((aligned(sizeof(Elf##size##_Word)))); \
-	} _ELFNOTE_PASTE(_note_, unique)				\
-+	} __UNIQUE_ID(note)						\
- 		__used							\
- 		__attribute__((section(".note." name),			\
- 			       aligned(sizeof(Elf##size##_Word)),	\
-@@ -89,11 +87,10 @@
- 		name,							\
- 		desc							\
- 	}
-#define ELFNOTE(size, name, type, desc)		\
-	_ELFNOTE(size, name, __LINE__, type, desc)
- 
- #define ELFNOTE32(name, type, desc) ELFNOTE(32, name, type, desc)
- #define ELFNOTE64(name, type, desc) ELFNOTE(64, name, type, desc)
-+
- #endif	/* __ASSEMBLER__ */
- 
- #endif /* _LINUX_ELFNOTE_H */
-diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h
-index b9caa01dfac4..43aa6153dc57 100644
--- a/include/linux/energy_model.h
-+++ b/include/linux/energy_model.h
-@@ -5,6 +5,7 @@
- #include <linux/device.h>
- #include <linux/jump_label.h>
- #include <linux/kobject.h>
-+#include <linux/kref.h>
- #include <linux/rcupdate.h>
- #include <linux/sched/cpufreq.h>
- #include <linux/sched/topology.h>
-@@ -12,6 +13,7 @@
- 
- /**
-  * struct em_perf_state - Performance state of a performance domain
-+ * @performance:	CPU performance (capacity) at a given frequency
-  * @frequency:	The frequency in KHz, for consistency with CPUFreq
-  * @power:	The power consumed at this level (by 1 CPU or by a registered
-  *		device). It can be a total power: static and dynamic.
-@@ -20,6 +22,7 @@
-  * @flags:	see "em_perf_state flags" description below.
-  */
- struct em_perf_state {
-+	unsigned long performance;
- 	unsigned long frequency;
- 	unsigned long power;
- 	unsigned long cost;
-@@ -37,9 +40,25 @@ struct em_perf_state {
- #define EM_PERF_STATE_INEFFICIENT BIT(0)
- 
- /**
-+ * struct em_perf_table - Performance states table
-+ * @rcu:	RCU used for safe access and destruction
-+ * @kref:	Reference counter to track the users
-+ * @state:	List of performance states, in ascending order
-+ */
-+struct em_perf_table {
-+	struct rcu_head rcu;
-+	struct kref kref;
-+	struct em_perf_state state[];
-+};
-+
-+/**
-  * struct em_perf_domain - Performance domain
- * @table:		List of performance states, in ascending order
-+ * @em_table:		Pointer to the runtime modifiable em_perf_table
-+ * @node:		node in	em_pd_list (in energy_model.c)
-+ * @id:			A unique ID number for each performance domain
-  * @nr_perf_states:	Number of performance states
-+ * @min_perf_state:	Minimum allowed Performance State index
-+ * @max_perf_state:	Maximum allowed Performance State index
-  * @flags:		See "em_perf_domain flags"
-  * @cpus:		Cpumask covering the CPUs of the domain. It's here
-  *			for performance reasons to avoid potential cache
-@@ -53,8 +72,12 @@ struct em_perf_state {
-  * field is unused.
-  */
- struct em_perf_domain {
-	struct em_perf_state *table;
-+	struct em_perf_table __rcu *em_table;
-+	struct list_head node;
-+	int id;
- 	int nr_perf_states;
-+	int min_perf_state;
-+	int max_perf_state;
- 	unsigned long flags;
- 	unsigned long cpus[];
- };
-@@ -98,27 +121,6 @@ struct em_perf_domain {
- #define EM_MAX_NUM_CPUS 16
- #endif
- 
-/*
- * To avoid an overflow on 32bit machines while calculating the energy
- * use a different order in the operation. First divide by the 'cpu_scale'
- * which would reduce big value stored in the 'cost' field, then multiply by
- * the 'sum_util'. This would allow to handle existing platforms, which have
- * e.g. power ~1.3 Watt at max freq, so the 'cost' value > 1mln micro-Watts.
- * In such scenario, where there are 4 CPUs in the Perf. Domain the 'sum_util'
- * could be 4096, then multiplication: 'cost' * 'sum_util'  would overflow.
- * This reordering of operations has some limitations, we lose small
- * precision in the estimation (comparing to 64bit platform w/o reordering).
- *
- * We are safe on 64bit machine.
- */
-#ifdef CONFIG_64BIT
-#define em_estimate_energy(cost, sum_util, scale_cpu) \
-	(((cost) * (sum_util)) / (scale_cpu))
-#else
-#define em_estimate_energy(cost, sum_util, scale_cpu) \
-	(((cost) / (scale_cpu)) * (sum_util))
-#endif
-
- struct em_data_callback {
- 	/**
- 	 * active_power() - Provide power at the next performance state of
-@@ -168,40 +170,58 @@ struct em_data_callback {
- 
- struct em_perf_domain *em_cpu_get(int cpu);
- struct em_perf_domain *em_pd_get(struct device *dev);
-+int em_dev_update_perf_domain(struct device *dev,
-+			      struct em_perf_table *new_table);
- int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states,
-				struct em_data_callback *cb, cpumask_t *span,
-				bool microwatts);
-+				const struct em_data_callback *cb,
-+				const cpumask_t *cpus, bool microwatts);
-+int em_dev_register_pd_no_update(struct device *dev, unsigned int nr_states,
-+				 const struct em_data_callback *cb,
-+				 const cpumask_t *cpus, bool microwatts);
- void em_dev_unregister_perf_domain(struct device *dev);
-+struct em_perf_table *em_table_alloc(struct em_perf_domain *pd);
-+void em_table_free(struct em_perf_table *table);
-+int em_dev_compute_costs(struct device *dev, struct em_perf_state *table,
-+			 int nr_states);
-+int em_dev_update_chip_binning(struct device *dev);
-+int em_update_performance_limits(struct em_perf_domain *pd,
-+		unsigned long freq_min_khz, unsigned long freq_max_khz);
-+void em_adjust_cpu_capacity(unsigned int cpu);
-+void em_rebuild_sched_domains(void);
- 
- /**
-  * em_pd_get_efficient_state() - Get an efficient performance state from the EM
- * @pd   : Performance domain for which we want an efficient frequency
- * @freq : Frequency to map with the EM
-+ * @table:		List of performance states, in ascending order
-+ * @pd:			performance domain for which this must be done
-+ * @max_util:		Max utilization to map with the EM
-  *
-  * It is called from the scheduler code quite frequently and as a consequence
-  * doesn't implement any check.
-  *
- * Return: An efficient performance state, high enough to meet @freq
-+ * Return: An efficient performance state id, high enough to meet @max_util
-  * requirement.
-  */
-static inline
-struct em_perf_state *em_pd_get_efficient_state(struct em_perf_domain *pd,
-						unsigned long freq)
-+static inline int
-+em_pd_get_efficient_state(struct em_perf_state *table,
-+			  struct em_perf_domain *pd, unsigned long max_util)
- {
-+	unsigned long pd_flags = pd->flags;
-+	int min_ps = pd->min_perf_state;
-+	int max_ps = pd->max_perf_state;
- 	struct em_perf_state *ps;
- 	int i;
- 
-	for (i = 0; i < pd->nr_perf_states; i++) {
-		ps = &pd->table[i];
-		if (ps->frequency >= freq) {
-			if (pd->flags & EM_PERF_DOMAIN_SKIP_INEFFICIENCIES &&
-+	for (i = min_ps; i <= max_ps; i++) {
-+		ps = &table[i];
-+		if (ps->performance >= max_util) {
-+			if (pd_flags & EM_PERF_DOMAIN_SKIP_INEFFICIENCIES &&
- 			    ps->flags & EM_PERF_STATE_INEFFICIENT)
- 				continue;
-			break;
-+			return i;
- 		}
- 	}
- 
-	return ps;
-+	return max_ps;
- }
- 
- /**
-@@ -224,9 +244,11 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd,
- 				unsigned long max_util, unsigned long sum_util,
- 				unsigned long allowed_cpu_cap)
- {
-	unsigned long freq, scale_cpu;
-+	struct em_perf_table *em_table;
- 	struct em_perf_state *ps;
-	int cpu;
-+	int i;
-+
-+	WARN_ONCE(!rcu_read_lock_held(), "EM: rcu read lock needed\n");
- 
- 	if (!sum_util)
- 		return 0;
-@@ -234,32 +256,28 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd,
- 	/*
- 	 * In order to predict the performance state, map the utilization of
- 	 * the most utilized CPU of the performance domain to a requested
-	 * frequency, like schedutil. Take also into account that the real
-	 * frequency might be set lower (due to thermal capping). Thus, clamp
-+	 * performance, like schedutil. Take also into account that the real
-+	 * performance might be set lower (due to thermal capping). Thus, clamp
- 	 * max utilization to the allowed CPU capacity before calculating
-	 * effective frequency.
-+	 * effective performance.
- 	 */
-	cpu = cpumask_first(to_cpumask(pd->cpus));
-	scale_cpu = arch_scale_cpu_capacity(cpu);
-	ps = &pd->table[pd->nr_perf_states - 1];
-
-	max_util = map_util_perf(max_util);
- 	max_util = min(max_util, allowed_cpu_cap);
-	freq = map_util_freq(max_util, ps->frequency, scale_cpu);
- 
- 	/*
- 	 * Find the lowest performance state of the Energy Model above the
-	 * requested frequency.
-+	 * requested performance.
- 	 */
-	ps = em_pd_get_efficient_state(pd, freq);
-+	em_table = rcu_dereference(pd->em_table);
-+	i = em_pd_get_efficient_state(em_table->state, pd, max_util);
-+	ps = &em_table->state[i];
- 
- 	/*
-	 * The capacity of a CPU in the domain at the performance state (ps)
-	 * can be computed as:
-+	 * The performance (capacity) of a CPU in the domain at the performance
-+	 * state (ps) can be computed as:
- 	 *
-	 *             ps->freq * scale_cpu
-	 *   ps->cap = --------------------                          (1)
-	 *                 cpu_max_freq
-+	 *                     ps->freq * scale_cpu
-+	 *   ps->performance = --------------------                  (1)
-+	 *                         cpu_max_freq
- 	 *
- 	 * So, ignoring the costs of idle states (which are not available in
- 	 * the EM), the energy consumed by this CPU at that performance state
-@@ -267,9 +285,10 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd,
- 	 *
- 	 *             ps->power * cpu_util
- 	 *   cpu_nrg = --------------------                          (2)
-	 *                   ps->cap
-+	 *               ps->performance
- 	 *
-	 * since 'cpu_util / ps->cap' represents its percentage of busy time.
-+	 * since 'cpu_util / ps->performance' represents its percentage of busy
-+	 * time.
- 	 *
- 	 *   NOTE: Although the result of this computation actually is in
- 	 *         units of power, it can be manipulated as an energy value
-@@ -279,9 +298,9 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd,
- 	 * By injecting (1) in (2), 'cpu_nrg' can be re-expressed as a product
- 	 * of two terms:
- 	 *
-	 *             ps->power * cpu_max_freq   cpu_util
-	 *   cpu_nrg = ------------------------ * ---------          (3)
-	 *                    ps->freq            scale_cpu
-+	 *             ps->power * cpu_max_freq
-+	 *   cpu_nrg = ------------------------ * cpu_util           (3)
-+	 *               ps->freq * scale_cpu
- 	 *
- 	 * The first term is static, and is stored in the em_perf_state struct
- 	 * as 'ps->cost'.
-@@ -291,11 +310,9 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd,
- 	 * total energy of the domain (which is the simple sum of the energy of
- 	 * all of its CPUs) can be factorized as:
- 	 *
-	 *            ps->cost * \Sum cpu_util
-	 *   pd_nrg = ------------------------                       (4)
-	 *                  scale_cpu
-+	 *   pd_nrg = ps->cost * \Sum cpu_util                       (4)
- 	 */
-	return em_estimate_energy(ps->cost, sum_util, scale_cpu);
-+	return ps->cost * sum_util;
- }
- 
- /**
-@@ -310,6 +327,23 @@ static inline int em_pd_nr_perf_states(struct em_perf_domain *pd)
- 	return pd->nr_perf_states;
- }
- 
-+/**
-+ * em_perf_state_from_pd() - Get the performance states table of perf.
-+ *				domain
-+ * @pd		: performance domain for which this must be done
-+ *
-+ * To use this function the rcu_read_lock() should be hold. After the usage
-+ * of the performance states table is finished, the rcu_read_unlock() should
-+ * be called.
-+ *
-+ * Return: the pointer to performance states table of the performance domain
-+ */
-+static inline
-+struct em_perf_state *em_perf_state_from_pd(struct em_perf_domain *pd)
-+{
-+	return rcu_dereference(pd->em_table)->state;
-+}
-+
- #else
- struct em_data_callback {};
- #define EM_ADV_DATA_CB(_active_power_cb, _cost_cb) { }
-@@ -318,8 +352,15 @@ struct em_data_callback {};
- 
- static inline
- int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states,
-				struct em_data_callback *cb, cpumask_t *span,
-				bool microwatts)
-+				const struct em_data_callback *cb,
-+				const cpumask_t *cpus, bool microwatts)
-+{
-+	return -EINVAL;
-+}
-+static inline
-+int em_dev_register_pd_no_update(struct device *dev, unsigned int nr_states,
-+				 const struct em_data_callback *cb,
-+				 const cpumask_t *cpus, bool microwatts)
- {
- 	return -EINVAL;
- }
-@@ -344,6 +385,41 @@ static inline int em_pd_nr_perf_states(struct em_perf_domain *pd)
- {
- 	return 0;
- }
-+static inline
-+struct em_perf_table *em_table_alloc(struct em_perf_domain *pd)
-+{
-+	return NULL;
-+}
-+static inline void em_table_free(struct em_perf_table *table) {}
-+static inline
-+int em_dev_update_perf_domain(struct device *dev,
-+			      struct em_perf_table *new_table)
-+{
-+	return -EINVAL;
-+}
-+static inline
-+struct em_perf_state *em_perf_state_from_pd(struct em_perf_domain *pd)
-+{
-+	return NULL;
-+}
-+static inline
-+int em_dev_compute_costs(struct device *dev, struct em_perf_state *table,
-+			 int nr_states)
-+{
-+	return -EINVAL;
-+}
-+static inline int em_dev_update_chip_binning(struct device *dev)
-+{
-+	return -EINVAL;
-+}
-+static inline
-+int em_update_performance_limits(struct em_perf_domain *pd,
-+		unsigned long freq_min_khz, unsigned long freq_max_khz)
-+{
-+	return -EINVAL;
-+}
-+static inline void em_adjust_cpu_capacity(unsigned int cpu) {}
-+static inline void em_rebuild_sched_domains(void) {}
- #endif
- 
- #endif
-diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h
-index 84a466b176cf..87efb38b7081 100644
--- a/include/linux/entry-common.h
-+++ b/include/linux/entry-common.h
-@@ -2,21 +2,15 @@
- #ifndef __LINUX_ENTRYCOMMON_H
- #define __LINUX_ENTRYCOMMON_H
- 
-#include <linux/static_call_types.h>
-+#include <linux/irq-entry-common.h>
-+#include <linux/livepatch.h>
- #include <linux/ptrace.h>
-#include <linux/syscalls.h>
-+#include <linux/resume_user_mode.h>
- #include <linux/seccomp.h>
- #include <linux/sched.h>
- 
- #include <asm/entry-common.h>
-
-/*
- * Define dummy _TIF work flags if not defined by the architecture or for
- * disabled functionality.
- */
-#ifndef _TIF_PATCH_PENDING
-# define _TIF_PATCH_PENDING		(0)
-#endif
-+#include <asm/syscall.h>
- 
- #ifndef _TIF_UPROBE
- # define _TIF_UPROBE			(0)
-@@ -43,6 +37,7 @@
- 				 SYSCALL_WORK_SYSCALL_AUDIT |		\
- 				 SYSCALL_WORK_SYSCALL_USER_DISPATCH |	\
- 				 ARCH_SYSCALL_WORK_ENTER)
-+
- #define SYSCALL_WORK_EXIT	(SYSCALL_WORK_SYSCALL_TRACEPOINT |	\
- 				 SYSCALL_WORK_SYSCALL_TRACE |		\
- 				 SYSCALL_WORK_SYSCALL_AUDIT |		\
-@@ -50,72 +45,7 @@
- 				 SYSCALL_WORK_SYSCALL_EXIT_TRAP	|	\
- 				 ARCH_SYSCALL_WORK_EXIT)
- 
-/*
- * TIF flags handled in exit_to_user_mode_loop()
- */
-#ifndef ARCH_EXIT_TO_USER_MODE_WORK
-# define ARCH_EXIT_TO_USER_MODE_WORK		(0)
-#endif
-
-#define EXIT_TO_USER_MODE_WORK						\
-	(_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE |		\
-	 _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL |	\
-	 ARCH_EXIT_TO_USER_MODE_WORK)
-
-/**
- * arch_enter_from_user_mode - Architecture specific sanity check for user mode regs
- * @regs:	Pointer to currents pt_regs
- *
- * Defaults to an empty implementation. Can be replaced by architecture
- * specific code.
- *
- * Invoked from syscall_enter_from_user_mode() in the non-instrumentable
- * section. Use __always_inline so the compiler cannot push it out of line
- * and make it instrumentable.
- */
-static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs);
-
-#ifndef arch_enter_from_user_mode
-static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs) {}
-#endif
-
-/**
- * enter_from_user_mode - Establish state when coming from user mode
- *
- * Syscall/interrupt entry disables interrupts, but user mode is traced as
- * interrupts enabled. Also with NO_HZ_FULL RCU might be idle.
- *
- * 1) Tell lockdep that interrupts are disabled
- * 2) Invoke context tracking if enabled to reactivate RCU
- * 3) Trace interrupts off state
- *
- * Invoked from architecture specific syscall entry code with interrupts
- * disabled. The calling code has to be non-instrumentable. When the
- * function returns all state is correct and interrupts are still
- * disabled. The subsequent functions can be instrumented.
- *
- * This is invoked when there is architecture specific functionality to be
- * done between establishing state and enabling interrupts. The caller must
- * enable interrupts before invoking syscall_enter_from_user_mode_work().
- */
-void enter_from_user_mode(struct pt_regs *regs);
-
-/**
- * syscall_enter_from_user_mode_prepare - Establish state and enable interrupts
- * @regs:	Pointer to currents pt_regs
- *
- * Invoked from architecture specific syscall entry code with interrupts
- * disabled. The calling code has to be non-instrumentable. When the
- * function returns all state is correct, interrupts are enabled and the
- * subsequent functions can be instrumented.
- *
- * This handles lockdep, RCU (context tracking) and tracing state, i.e.
- * the functionality provided by enter_from_user_mode().
- *
- * This is invoked when there is extra architecture specific functionality
- * to be done between establishing state and handling user mode entry work.
- */
-void syscall_enter_from_user_mode_prepare(struct pt_regs *regs);
-+long syscall_trace_enter(struct pt_regs *regs, long syscall, unsigned long work);
- 
- /**
-  * syscall_enter_from_user_mode_work - Check and handle work before invoking
-@@ -124,8 +54,8 @@ void syscall_enter_from_user_mode_prepare(struct pt_regs *regs);
-  * @syscall:	The syscall number
-  *
-  * Invoked from architecture specific syscall entry code with interrupts
- * enabled after invoking syscall_enter_from_user_mode_prepare() and extra
- * architecture specific work.
-+ * enabled after invoking enter_from_user_mode(), enabling interrupts and
-+ * extra architecture specific work.
-  *
-  * Returns: The original or a modified syscall number
-  *
-@@ -140,7 +70,15 @@ void syscall_enter_from_user_mode_prepare(struct pt_regs *regs);
-  *     ptrace_report_syscall_entry(), __secure_computing(), trace_sys_enter()
-  *  2) Invocation of audit_syscall_entry()
-  */
-long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall);
-+static __always_inline long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall)
-+{
-+	unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
-+
-+	if (work & SYSCALL_WORK_ENTER)
-+		syscall = syscall_trace_enter(regs, syscall, work);
-+
-+	return syscall;
-+}
- 
- /**
-  * syscall_enter_from_user_mode - Establish state and check and handle work
-@@ -153,131 +91,35 @@ long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall);
-  * function returns all state is correct, interrupts are enabled and the
-  * subsequent functions can be instrumented.
-  *
- * This is combination of syscall_enter_from_user_mode_prepare() and
- * syscall_enter_from_user_mode_work().
-+ * This is the combination of enter_from_user_mode() and
-+ * syscall_enter_from_user_mode_work() to be used when there is no
-+ * architecture specific work to be done between the two.
-  *
-  * Returns: The original or a modified syscall number. See
-  * syscall_enter_from_user_mode_work() for further explanation.
-  */
-long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall);
-
-/**
- * local_irq_enable_exit_to_user - Exit to user variant of local_irq_enable()
- * @ti_work:	Cached TIF flags gathered with interrupts disabled
- *
- * Defaults to local_irq_enable(). Can be supplied by architecture specific
- * code.
- */
-static inline void local_irq_enable_exit_to_user(unsigned long ti_work);
-
-#ifndef local_irq_enable_exit_to_user
-static inline void local_irq_enable_exit_to_user(unsigned long ti_work)
-+static __always_inline long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)
- {
-	local_irq_enable();
-}
-#endif
-+	long ret;
- 
-/**
- * local_irq_disable_exit_to_user - Exit to user variant of local_irq_disable()
- *
- * Defaults to local_irq_disable(). Can be supplied by architecture specific
- * code.
- */
-static inline void local_irq_disable_exit_to_user(void);
-+	enter_from_user_mode(regs);
- 
-#ifndef local_irq_disable_exit_to_user
-static inline void local_irq_disable_exit_to_user(void)
-{
-	local_irq_disable();
-}
-#endif
-
-/**
- * arch_exit_to_user_mode_work - Architecture specific TIF work for exit
- *				 to user mode.
- * @regs:	Pointer to currents pt_regs
- * @ti_work:	Cached TIF flags gathered with interrupts disabled
- *
- * Invoked from exit_to_user_mode_loop() with interrupt enabled
- *
- * Defaults to NOOP. Can be supplied by architecture specific code.
- */
-static inline void arch_exit_to_user_mode_work(struct pt_regs *regs,
-					       unsigned long ti_work);
-
-#ifndef arch_exit_to_user_mode_work
-static inline void arch_exit_to_user_mode_work(struct pt_regs *regs,
-					       unsigned long ti_work)
-{
-}
-#endif
-
-/**
- * arch_exit_to_user_mode_prepare - Architecture specific preparation for
- *				    exit to user mode.
- * @regs:	Pointer to currents pt_regs
- * @ti_work:	Cached TIF flags gathered with interrupts disabled
- *
- * Invoked from exit_to_user_mode_prepare() with interrupt disabled as the last
- * function before return. Defaults to NOOP.
- */
-static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
-						  unsigned long ti_work);
-+	instrumentation_begin();
-+	local_irq_enable();
-+	ret = syscall_enter_from_user_mode_work(regs, syscall);
-+	instrumentation_end();
- 
-#ifndef arch_exit_to_user_mode_prepare
-static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
-						  unsigned long ti_work)
-{
-+	return ret;
- }
-#endif
-
-/**
- * arch_exit_to_user_mode - Architecture specific final work before
- *			    exit to user mode.
- *
- * Invoked from exit_to_user_mode() with interrupt disabled as the last
- * function before return. Defaults to NOOP.
- *
- * This needs to be __always_inline because it is non-instrumentable code
- * invoked after context tracking switched to user mode.
- *
- * An architecture implementation must not do anything complex, no locking
- * etc. The main purpose is for speculation mitigations.
- */
-static __always_inline void arch_exit_to_user_mode(void);
-
-#ifndef arch_exit_to_user_mode
-static __always_inline void arch_exit_to_user_mode(void) { }
-#endif
- 
- /**
- * arch_do_signal_or_restart -  Architecture specific signal delivery function
- * @regs:	Pointer to currents pt_regs
- * @has_signal:	actual signal to handle
-+ * syscall_exit_work - Handle work before returning to user mode
-+ * @regs:	Pointer to current pt_regs
-+ * @work:	Current thread syscall work
-  *
- * Invoked from exit_to_user_mode_loop().
-+ * Do one-time syscall specific work.
-  */
-void arch_do_signal_or_restart(struct pt_regs *regs);
-
-/**
- * exit_to_user_mode - Fixup state when exiting to user mode
- *
- * Syscall/interrupt exit enables interrupts, but the kernel state is
- * interrupts disabled when this is invoked. Also tell RCU about it.
- *
- * 1) Trace interrupts on state
- * 2) Invoke context tracking if enabled to adjust RCU state
- * 3) Invoke architecture specific last minute exit code, e.g. speculation
- *    mitigations, etc.: arch_exit_to_user_mode()
- * 4) Tell lockdep that interrupts are enabled
- *
- * Invoked from architecture specific code when syscall_exit_to_user_mode()
- * is not suitable as the last step before returning to userspace. Must be
- * invoked with interrupts disabled and the caller must be
- * non-instrumentable.
- * The caller has to invoke syscall_exit_to_user_mode_work() before this.
- */
-void exit_to_user_mode(void);
-+void syscall_exit_work(struct pt_regs *regs, unsigned long work);
- 
- /**
-  * syscall_exit_to_user_mode_work - Handle work before returning to user mode
-@@ -292,7 +134,30 @@ void exit_to_user_mode(void);
-  * make the final state transitions. Interrupts must stay disabled between
-  * return from this function and the invocation of exit_to_user_mode().
-  */
-void syscall_exit_to_user_mode_work(struct pt_regs *regs);
-+static __always_inline void syscall_exit_to_user_mode_work(struct pt_regs *regs)
-+{
-+	unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
-+	unsigned long nr = syscall_get_nr(current, regs);
-+
-+	CT_WARN_ON(ct_state() != CT_STATE_KERNEL);
-+
-+	if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
-+		if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr))
-+			local_irq_enable();
-+	}
-+
-+	rseq_debug_syscall_return(regs);
-+
-+	/*
-+	 * Do one-time syscall specific work. If these work items are
-+	 * enabled, we want to run them exactly once per syscall exit with
-+	 * interrupts enabled.
-+	 */
-+	if (unlikely(work & SYSCALL_WORK_EXIT))
-+		syscall_exit_work(regs, work);
-+	local_irq_disable_exit_to_user();
-+	syscall_exit_to_user_mode_prepare(regs);
-+}
- 
- /**
-  * syscall_exit_to_user_mode - Handle work before returning to user mode
-@@ -323,147 +188,12 @@ void syscall_exit_to_user_mode_work(struct pt_regs *regs);
-  * exit_to_user_mode(). This function is preferred unless there is a
-  * compelling architectural reason to use the separate functions.
-  */
-void syscall_exit_to_user_mode(struct pt_regs *regs);
-
-/**
- * irqentry_enter_from_user_mode - Establish state before invoking the irq handler
- * @regs:	Pointer to currents pt_regs
- *
- * Invoked from architecture specific entry code with interrupts disabled.
- * Can only be called when the interrupt entry came from user mode. The
- * calling code must be non-instrumentable.  When the function returns all
- * state is correct and the subsequent functions can be instrumented.
- *
- * The function establishes state (lockdep, RCU (context tracking), tracing)
- */
-void irqentry_enter_from_user_mode(struct pt_regs *regs);
-
-/**
- * irqentry_exit_to_user_mode - Interrupt exit work
- * @regs:	Pointer to current's pt_regs
- *
- * Invoked with interrupts disabled and fully valid regs. Returns with all
- * work handled, interrupts disabled such that the caller can immediately
- * switch to user mode. Called from architecture specific interrupt
- * handling code.
- *
- * The call order is #2 and #3 as described in syscall_exit_to_user_mode().
- * Interrupt exit is not invoking #1 which is the syscall specific one time
- * work.
- */
-void irqentry_exit_to_user_mode(struct pt_regs *regs);
-
-#ifndef irqentry_state
-/**
- * struct irqentry_state - Opaque object for exception state storage
- * @exit_rcu: Used exclusively in the irqentry_*() calls; signals whether the
- *            exit path has to invoke ct_irq_exit().
- * @lockdep: Used exclusively in the irqentry_nmi_*() calls; ensures that
- *           lockdep state is restored correctly on exit from nmi.
- *
- * This opaque object is filled in by the irqentry_*_enter() functions and
- * must be passed back into the corresponding irqentry_*_exit() functions
- * when the exception is complete.
- *
- * Callers of irqentry_*_[enter|exit]() must consider this structure opaque
- * and all members private.  Descriptions of the members are provided to aid in
- * the maintenance of the irqentry_*() functions.
- */
-typedef struct irqentry_state {
-	union {
-		bool	exit_rcu;
-		bool	lockdep;
-	};
-} irqentry_state_t;
-#endif
-
-/**
- * irqentry_enter - Handle state tracking on ordinary interrupt entries
- * @regs:	Pointer to pt_regs of interrupted context
- *
- * Invokes:
- *  - lockdep irqflag state tracking as low level ASM entry disabled
- *    interrupts.
- *
- *  - Context tracking if the exception hit user mode.
- *
- *  - The hardirq tracer to keep the state consistent as low level ASM
- *    entry disabled interrupts.
- *
- * As a precondition, this requires that the entry came from user mode,
- * idle, or a kernel context in which RCU is watching.
- *
- * For kernel mode entries RCU handling is done conditional. If RCU is
- * watching then the only RCU requirement is to check whether the tick has
- * to be restarted. If RCU is not watching then ct_irq_enter() has to be
- * invoked on entry and ct_irq_exit() on exit.
- *
- * Avoiding the ct_irq_enter/exit() calls is an optimization but also
- * solves the problem of kernel mode pagefaults which can schedule, which
- * is not possible after invoking ct_irq_enter() without undoing it.
- *
- * For user mode entries irqentry_enter_from_user_mode() is invoked to
- * establish the proper context for NOHZ_FULL. Otherwise scheduling on exit
- * would not be possible.
- *
- * Returns: An opaque object that must be passed to idtentry_exit()
- */
-irqentry_state_t noinstr irqentry_enter(struct pt_regs *regs);
-
-/**
- * irqentry_exit_cond_resched - Conditionally reschedule on return from interrupt
- *
- * Conditional reschedule with additional sanity checks.
- */
-void raw_irqentry_exit_cond_resched(void);
-#ifdef CONFIG_PREEMPT_DYNAMIC
-#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
-#define irqentry_exit_cond_resched_dynamic_enabled	raw_irqentry_exit_cond_resched
-#define irqentry_exit_cond_resched_dynamic_disabled	NULL
-DECLARE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched);
-#define irqentry_exit_cond_resched()	static_call(irqentry_exit_cond_resched)()
-#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
-void dynamic_irqentry_exit_cond_resched(void);
-#define irqentry_exit_cond_resched()	dynamic_irqentry_exit_cond_resched()
-#endif
-#else /* CONFIG_PREEMPT_DYNAMIC */
-#define irqentry_exit_cond_resched()	raw_irqentry_exit_cond_resched()
-#endif /* CONFIG_PREEMPT_DYNAMIC */
-
-/**
- * irqentry_exit - Handle return from exception that used irqentry_enter()
- * @regs:	Pointer to pt_regs (exception entry regs)
- * @state:	Return value from matching call to irqentry_enter()
- *
- * Depending on the return target (kernel/user) this runs the necessary
- * preemption and work checks if possible and required and returns to
- * the caller with interrupts disabled and no further work pending.
- *
- * This is the last action before returning to the low level ASM code which
- * just needs to return to the appropriate context.
- *
- * Counterpart to irqentry_enter().
- */
-void noinstr irqentry_exit(struct pt_regs *regs, irqentry_state_t state);
-
-/**
- * irqentry_nmi_enter - Handle NMI entry
- * @regs:	Pointer to currents pt_regs
- *
- * Similar to irqentry_enter() but taking care of the NMI constraints.
- */
-irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs);
-
-/**
- * irqentry_nmi_exit - Handle return from NMI handling
- * @regs:	Pointer to pt_regs (NMI entry regs)
- * @irq_state:	Return value from matching call to irqentry_nmi_enter()
- *
- * Last action before returning to the low level assembly code.
- *
- * Counterpart to irqentry_nmi_enter().
- */
-void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state);
-+static __always_inline void syscall_exit_to_user_mode(struct pt_regs *regs)
-+{
-+	instrumentation_begin();
-+	syscall_exit_to_user_mode_work(regs);
-+	instrumentation_end();
-+	exit_to_user_mode();
-+}
- 
- #endif
-diff --git a/include/linux/entry-kvm.h b/include/linux/entry-virt.h
-index 6813171afccb..bfa767702d9a 100644
--- a/include/linux/entry-kvm.h
-+++ b/include/linux/entry-virt.h
-@@ -1,6 +1,6 @@
- /* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_ENTRYKVM_H
-#define __LINUX_ENTRYKVM_H
-+#ifndef __LINUX_ENTRYVIRT_H
-+#define __LINUX_ENTRYVIRT_H
- 
- #include <linux/static_call_types.h>
- #include <linux/resume_user_mode.h>
-@@ -10,17 +10,16 @@
- #include <linux/tick.h>
- 
- /* Transfer to guest mode work */
-#ifdef CONFIG_KVM_XFER_TO_GUEST_WORK
-+#ifdef CONFIG_VIRT_XFER_TO_GUEST_WORK
- 
- #ifndef ARCH_XFER_TO_GUEST_MODE_WORK
- # define ARCH_XFER_TO_GUEST_MODE_WORK	(0)
- #endif
- 
- #define XFER_TO_GUEST_MODE_WORK						\
-	(_TIF_NEED_RESCHED | _TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL |	\
-	 _TIF_NOTIFY_RESUME | ARCH_XFER_TO_GUEST_MODE_WORK)
-
-struct kvm_vcpu;
-+	(_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | _TIF_SIGPENDING | \
-+	 _TIF_NOTIFY_SIGNAL | _TIF_NOTIFY_RESUME |			\
-+	 ARCH_XFER_TO_GUEST_MODE_WORK)
- 
- /**
-  * arch_xfer_to_guest_mode_handle_work - Architecture specific xfer to guest
-@@ -31,12 +30,10 @@ struct kvm_vcpu;
-  * Invoked from xfer_to_guest_mode_handle_work(). Defaults to NOOP. Can be
-  * replaced by architecture specific code.
-  */
-static inline int arch_xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu,
-						      unsigned long ti_work);
-+static inline int arch_xfer_to_guest_mode_handle_work(unsigned long ti_work);
- 
-#ifndef arch_xfer_to_guest_mode_work
-static inline int arch_xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu,
-						      unsigned long ti_work)
-+#ifndef arch_xfer_to_guest_mode_handle_work
-+static inline int arch_xfer_to_guest_mode_handle_work(unsigned long ti_work)
- {
- 	return 0;
- }
-@@ -45,11 +42,10 @@ static inline int arch_xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu,
- /**
-  * xfer_to_guest_mode_handle_work - Check and handle pending work which needs
-  *				    to be handled before going to guest mode
- * @vcpu:	Pointer to current's VCPU data
-  *
-  * Returns: 0 or an error code
-  */
-int xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu);
-+int xfer_to_guest_mode_handle_work(void);
- 
- /**
-  * xfer_to_guest_mode_prepare - Perform last minute preparation work that
-@@ -94,6 +90,6 @@ static inline bool xfer_to_guest_mode_work_pending(void)
- 	lockdep_assert_irqs_disabled();
- 	return __xfer_to_guest_mode_work_pending();
- }
-#endif /* CONFIG_KVM_XFER_TO_GUEST_WORK */
-+#endif /* CONFIG_VIRT_XFER_TO_GUEST_WORK */
- 
- #endif
-diff --git a/include/linux/err.h b/include/linux/err.h
-index a139c64aef2a..8c37be0620ab 100644
--- a/include/linux/err.h
-+++ b/include/linux/err.h
-@@ -19,23 +19,74 @@
- 
- #ifndef __ASSEMBLY__
- 
-+/**
-+ * IS_ERR_VALUE - Detect an error pointer.
-+ * @x: The pointer to check.
-+ *
-+ * Like IS_ERR(), but does not generate a compiler warning if result is unused.
-+ */
- #define IS_ERR_VALUE(x) unlikely((unsigned long)(void *)(x) >= (unsigned long)-MAX_ERRNO)
- 
-+/**
-+ * ERR_PTR - Create an error pointer.
-+ * @error: A negative error code.
-+ *
-+ * Encodes @error into a pointer value. Users should consider the result
-+ * opaque and not assume anything about how the error is encoded.
-+ *
-+ * Return: A pointer with @error encoded within its value.
-+ */
- static inline void * __must_check ERR_PTR(long error)
- {
- 	return (void *) error;
- }
- 
-+/**
-+ * INIT_ERR_PTR - Init a const error pointer.
-+ * @error: A negative error code.
-+ *
-+ * Like ERR_PTR(), but usable to initialize static variables.
-+ */
-+#define INIT_ERR_PTR(error) ((void *)(error))
-+
-+/* Return the pointer in the percpu address space. */
-+#define ERR_PTR_PCPU(error) ((void __percpu *)(unsigned long)ERR_PTR(error))
-+
-+/* Cast an error pointer to __iomem. */
-+#define IOMEM_ERR_PTR(error) (__force void __iomem *)ERR_PTR(error)
-+
-+/**
-+ * PTR_ERR - Extract the error code from an error pointer.
-+ * @ptr: An error pointer.
-+ * Return: The error code within @ptr.
-+ */
- static inline long __must_check PTR_ERR(__force const void *ptr)
- {
- 	return (long) ptr;
- }
- 
-+/* Read an error pointer from the percpu address space. */
-+#define PTR_ERR_PCPU(ptr) (PTR_ERR((const void *)(__force const unsigned long)(ptr)))
-+
-+/**
-+ * IS_ERR - Detect an error pointer.
-+ * @ptr: The pointer to check.
-+ * Return: true if @ptr is an error pointer, false otherwise.
-+ */
- static inline bool __must_check IS_ERR(__force const void *ptr)
- {
- 	return IS_ERR_VALUE((unsigned long)ptr);
- }
- 
-+/* Read an error pointer from the percpu address space. */
-+#define IS_ERR_PCPU(ptr) (IS_ERR((const void *)(__force const unsigned long)(ptr)))
-+
-+/**
-+ * IS_ERR_OR_NULL - Detect an error pointer or a null pointer.
-+ * @ptr: The pointer to check.
-+ *
-+ * Like IS_ERR(), but also returns true for a null pointer.
-+ */
- static inline bool __must_check IS_ERR_OR_NULL(__force const void *ptr)
- {
- 	return unlikely(!ptr) || IS_ERR_VALUE((unsigned long)ptr);
-@@ -54,6 +105,23 @@ static inline void * __must_check ERR_CAST(__force const void *ptr)
- 	return (void *) ptr;
- }
- 
-+/**
-+ * PTR_ERR_OR_ZERO - Extract the error code from a pointer if it has one.
-+ * @ptr: A potential error pointer.
-+ *
-+ * Convenience function that can be used inside a function that returns
-+ * an error code to propagate errors received as error pointers.
-+ * For example, ``return PTR_ERR_OR_ZERO(ptr);`` replaces:
-+ *
-+ * .. code-block:: c
-+ *
-+ *	if (IS_ERR(ptr))
-+ *		return PTR_ERR(ptr);
-+ *	else
-+ *		return 0;
-+ *
-+ * Return: The error code within @ptr if it is an error pointer; 0 otherwise.
-+ */
- static inline int __must_check PTR_ERR_OR_ZERO(__force const void *ptr)
- {
- 	if (IS_ERR(ptr))
-diff --git a/include/linux/error-injection.h b/include/linux/error-injection.h
-index 635a95caf29f..20e738f4eae8 100644
--- a/include/linux/error-injection.h
-+++ b/include/linux/error-injection.h
-@@ -3,6 +3,7 @@
- #define _LINUX_ERROR_INJECTION_H
- 
- #include <linux/compiler.h>
-+#include <linux/errno.h>
- #include <asm-generic/error-injection.h>
- 
- #ifdef CONFIG_FUNCTION_ERROR_INJECTION
-@@ -19,7 +20,7 @@ static inline bool within_error_injection_list(unsigned long addr)
- 
- static inline int get_injectable_error_type(unsigned long addr)
- {
-	return EI_ETYPE_NONE;
-+	return -EOPNOTSUPP;
- }
- 
- #endif
-diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h
-index 92b10e67d5f8..9a1eacf35d37 100644
--- a/include/linux/etherdevice.h
-+++ b/include/linux/etherdevice.h
-@@ -21,7 +21,7 @@
- #include <linux/netdevice.h>
- #include <linux/random.h>
- #include <linux/crc32.h>
-#include <asm/unaligned.h>
-+#include <linux/unaligned.h>
- #include <asm/bitsperlong.h>
- 
- #ifdef __KERNEL__
-@@ -71,11 +71,17 @@ static const u8 eth_reserved_addr_base[ETH_ALEN] __aligned(2) =
- { 0x01, 0x80, 0xc2, 0x00, 0x00, 0x00 };
- #define eth_stp_addr eth_reserved_addr_base
- 
-+static const u8 eth_ipv4_mcast_addr_base[ETH_ALEN] __aligned(2) =
-+{ 0x01, 0x00, 0x5e, 0x00, 0x00, 0x00 };
-+
-+static const u8 eth_ipv6_mcast_addr_base[ETH_ALEN] __aligned(2) =
-+{ 0x33, 0x33, 0x00, 0x00, 0x00, 0x00 };
-+
- /**
-  * is_link_local_ether_addr - Determine if given Ethernet address is link-local
-  * @addr: Pointer to a six-byte array containing the Ethernet address
-  *
- * Return true if address is link local reserved addr (01:80:c2:00:00:0X) per
-+ * Return: true if address is link local reserved addr (01:80:c2:00:00:0X) per
-  * IEEE 802.1Q 8.6.3 Frame filtering.
-  *
-  * Please note: addr must be aligned to u16.
-@@ -98,7 +104,7 @@ static inline bool is_link_local_ether_addr(const u8 *addr)
-  * is_zero_ether_addr - Determine if give Ethernet address is all zeros.
-  * @addr: Pointer to a six-byte array containing the Ethernet address
-  *
- * Return true if the address is all zeroes.
-+ * Return: true if the address is all zeroes.
-  *
-  * Please note: addr must be aligned to u16.
-  */
-@@ -117,7 +123,7 @@ static inline bool is_zero_ether_addr(const u8 *addr)
-  * is_multicast_ether_addr - Determine if the Ethernet address is a multicast.
-  * @addr: Pointer to a six-byte array containing the Ethernet address
-  *
- * Return true if the address is a multicast address.
-+ * Return: true if the address is a multicast address.
-  * By definition the broadcast address is also a multicast address.
-  */
- static inline bool is_multicast_ether_addr(const u8 *addr)
-@@ -151,7 +157,7 @@ static inline bool is_multicast_ether_addr_64bits(const u8 *addr)
-  * is_local_ether_addr - Determine if the Ethernet address is locally-assigned one (IEEE 802).
-  * @addr: Pointer to a six-byte array containing the Ethernet address
-  *
- * Return true if the address is a local address.
-+ * Return: true if the address is a local address.
-  */
- static inline bool is_local_ether_addr(const u8 *addr)
- {
-@@ -162,7 +168,7 @@ static inline bool is_local_ether_addr(const u8 *addr)
-  * is_broadcast_ether_addr - Determine if the Ethernet address is broadcast
-  * @addr: Pointer to a six-byte array containing the Ethernet address
-  *
- * Return true if the address is the broadcast address.
-+ * Return: true if the address is the broadcast address.
-  *
-  * Please note: addr must be aligned to u16.
-  */
-@@ -177,7 +183,7 @@ static inline bool is_broadcast_ether_addr(const u8 *addr)
-  * is_unicast_ether_addr - Determine if the Ethernet address is unicast
-  * @addr: Pointer to a six-byte array containing the Ethernet address
-  *
- * Return true if the address is a unicast address.
-+ * Return: true if the address is a unicast address.
-  */
- static inline bool is_unicast_ether_addr(const u8 *addr)
- {
-@@ -191,7 +197,7 @@ static inline bool is_unicast_ether_addr(const u8 *addr)
-  * Check that the Ethernet address (MAC) is not 00:00:00:00:00:00, is not
-  * a multicast address, and is not FF:FF:FF:FF:FF:FF.
-  *
- * Return true if the address is valid.
-+ * Return: true if the address is valid.
-  *
-  * Please note: addr must be aligned to u16.
-  */
-@@ -208,7 +214,7 @@ static inline bool is_valid_ether_addr(const u8 *addr)
-  *
-  * Check that the value from the Ethertype/length field is a valid Ethertype.
-  *
- * Return true if the valid is an 802.3 supported Ethertype.
-+ * Return: true if the valid is an 802.3 supported Ethertype.
-  */
- static inline bool eth_proto_is_802_3(__be16 proto)
- {
-@@ -428,11 +434,31 @@ static inline bool ether_addr_equal_masked(const u8 *addr1, const u8 *addr2,
- 	return true;
- }
- 
-+static inline bool ether_addr_is_ipv4_mcast(const u8 *addr)
-+{
-+	u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0x80, 0x00, 0x00 };
-+
-+	return ether_addr_equal_masked(addr, eth_ipv4_mcast_addr_base, mask);
-+}
-+
-+static inline bool ether_addr_is_ipv6_mcast(const u8 *addr)
-+{
-+	u8 mask[ETH_ALEN] = { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 };
-+
-+	return ether_addr_equal_masked(addr, eth_ipv6_mcast_addr_base, mask);
-+}
-+
-+static inline bool ether_addr_is_ip_mcast(const u8 *addr)
-+{
-+	return ether_addr_is_ipv4_mcast(addr) ||
-+		ether_addr_is_ipv6_mcast(addr);
-+}
-+
- /**
-  * ether_addr_to_u64 - Convert an Ethernet address into a u64 value.
-  * @addr: Pointer to a six-byte array containing the Ethernet address
-  *
- * Return a u64 value of the address
-+ * Return: a u64 value of the address
-  */
- static inline u64 ether_addr_to_u64(const u8 *addr)
- {
-@@ -486,6 +512,20 @@ static inline void eth_addr_inc(u8 *addr)
- }
- 
- /**
-+ * eth_addr_add() - Add (or subtract) an offset to/from the given MAC address.
-+ *
-+ * @offset: Offset to add.
-+ * @addr: Pointer to a six-byte array containing Ethernet address to increment.
-+ */
-+static inline void eth_addr_add(u8 *addr, long offset)
-+{
-+	u64 u = ether_addr_to_u64(addr);
-+
-+	u += offset;
-+	u64_to_ether_addr(u, addr);
-+}
-+
-+/**
-  * is_etherdev_addr - Tell if given Ethernet address belongs to the device.
-  * @dev: Pointer to a device structure
-  * @addr: Pointer to a six-byte array containing the Ethernet address
-@@ -572,7 +612,40 @@ static inline void eth_hw_addr_gen(struct net_device *dev, const u8 *base_addr,
- }
- 
- /**
- * eth_skb_pad - Pad buffer to mininum number of octets for Ethernet frame
-+ * eth_skb_pkt_type - Assign packet type if destination address does not match
-+ * @skb: Assigned a packet type if address does not match @dev address
-+ * @dev: Network device used to compare packet address against
-+ *
-+ * If the destination MAC address of the packet does not match the network
-+ * device address, assign an appropriate packet type.
-+ */
-+static inline void eth_skb_pkt_type(struct sk_buff *skb,
-+				    const struct net_device *dev)
-+{
-+	const struct ethhdr *eth = eth_hdr(skb);
-+
-+	if (unlikely(!ether_addr_equal_64bits(eth->h_dest, dev->dev_addr))) {
-+		if (unlikely(is_multicast_ether_addr_64bits(eth->h_dest))) {
-+			if (ether_addr_equal_64bits(eth->h_dest, dev->broadcast))
-+				skb->pkt_type = PACKET_BROADCAST;
-+			else
-+				skb->pkt_type = PACKET_MULTICAST;
-+		} else {
-+			skb->pkt_type = PACKET_OTHERHOST;
-+		}
-+	}
-+}
-+
-+static inline struct ethhdr *eth_skb_pull_mac(struct sk_buff *skb)
-+{
-+	struct ethhdr *eth = (struct ethhdr *)skb->data;
-+
-+	skb_pull_inline(skb, ETH_HLEN);
-+	return eth;
-+}
-+
-+/**
-+ * eth_skb_pad - Pad buffer to minimum number of octets for Ethernet frame
-  * @skb: Buffer to pad
-  *
-  * An Ethernet frame should have a minimum size of 60 bytes.  This function
-diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h
-index 99dc7bfbcd3c..5c9162193d26 100644
--- a/include/linux/ethtool.h
-+++ b/include/linux/ethtool.h
-@@ -15,8 +15,15 @@
- 
- #include <linux/bitmap.h>
- #include <linux/compat.h>
-+#include <linux/if_ether.h>
- #include <linux/netlink.h>
-+#include <linux/timer_types.h>
- #include <uapi/linux/ethtool.h>
-+#include <uapi/linux/ethtool_netlink_generated.h>
-+#include <uapi/linux/net_tstamp.h>
-+
-+#define ETHTOOL_MM_MAX_VERIFY_TIME_MS		128
-+#define ETHTOOL_MM_MAX_VERIFY_RETRIES		3
- 
- struct compat_ethtool_rx_flow_spec {
- 	u32		flow_type;
-@@ -72,13 +79,24 @@ enum {
-  * @rx_buf_len: Current length of buffers on the rx ring.
-  * @tcp_data_split: Scatter packet headers and data to separate buffers
-  * @tx_push: The flag of tx push mode
-+ * @rx_push: The flag of rx push mode
-  * @cqe_size: Size of TX/RX completion queue event
-+ * @tx_push_buf_len: Size of TX push buffer
-+ * @tx_push_buf_max_len: Maximum allowed size of TX push buffer
-+ * @hds_thresh: Packet size threshold for header data split (HDS)
-+ * @hds_thresh_max: Maximum supported setting for @hds_threshold
-+ *
-  */
- struct kernel_ethtool_ringparam {
- 	u32	rx_buf_len;
- 	u8	tcp_data_split;
- 	u8	tx_push;
-+	u8	rx_push;
- 	u32	cqe_size;
-+	u32	tx_push_buf_len;
-+	u32	tx_push_buf_max_len;
-+	u32	hds_thresh;
-+	u32	hds_thresh_max;
- };
- 
- /**
-@@ -86,11 +104,19 @@ struct kernel_ethtool_ringparam {
-  * @ETHTOOL_RING_USE_RX_BUF_LEN: capture for setting rx_buf_len
-  * @ETHTOOL_RING_USE_CQE_SIZE: capture for setting cqe_size
-  * @ETHTOOL_RING_USE_TX_PUSH: capture for setting tx_push
-+ * @ETHTOOL_RING_USE_RX_PUSH: capture for setting rx_push
-+ * @ETHTOOL_RING_USE_TX_PUSH_BUF_LEN: capture for setting tx_push_buf_len
-+ * @ETHTOOL_RING_USE_TCP_DATA_SPLIT: capture for setting tcp_data_split
-+ * @ETHTOOL_RING_USE_HDS_THRS: capture for setting header-data-split-thresh
-  */
- enum ethtool_supported_ring_param {
-	ETHTOOL_RING_USE_RX_BUF_LEN = BIT(0),
-	ETHTOOL_RING_USE_CQE_SIZE   = BIT(1),
-	ETHTOOL_RING_USE_TX_PUSH    = BIT(2),
-+	ETHTOOL_RING_USE_RX_BUF_LEN		= BIT(0),
-+	ETHTOOL_RING_USE_CQE_SIZE		= BIT(1),
-+	ETHTOOL_RING_USE_TX_PUSH		= BIT(2),
-+	ETHTOOL_RING_USE_RX_PUSH		= BIT(3),
-+	ETHTOOL_RING_USE_TX_PUSH_BUF_LEN	= BIT(4),
-+	ETHTOOL_RING_USE_TCP_DATA_SPLIT		= BIT(5),
-+	ETHTOOL_RING_USE_HDS_THRS		= BIT(6),
- };
- 
- #define __ETH_RSS_HASH_BIT(bit)	((u32)1 << (bit))
-@@ -106,11 +132,6 @@ enum ethtool_supported_ring_param {
- struct net_device;
- struct netlink_ext_ack;
- 
-/* Some generic methods drivers may use in their ethtool_ops */
-u32 ethtool_op_get_link(struct net_device *dev);
-int ethtool_op_get_ts_info(struct net_device *dev, struct ethtool_ts_info *eti);
-
-
- /* Link extended state and substate. */
- struct ethtool_link_ext_state_info {
- 	enum ethtool_link_ext_state link_ext_state;
-@@ -125,6 +146,20 @@ struct ethtool_link_ext_state_info {
- 	};
- };
- 
-+struct ethtool_link_ext_stats {
-+	/* Custom Linux statistic for PHY level link down events.
-+	 * In a simpler world it should be equal to netdev->carrier_down_count
-+	 * unfortunately netdev also counts local reconfigurations which don't
-+	 * actually take the physical link down, not to mention NC-SI which,
-+	 * if present, keeps the link up regardless of host state.
-+	 * This statistic counts when PHY _actually_ went down, or lost link.
-+	 *
-+	 * Note that we need u64 for ethtool_stats_init() and comparisons
-+	 * to ETHTOOL_STAT_NOT_SET, but only u32 is exposed to the user.
-+	 */
-+	u64 link_down_events;
-+};
-+
- /**
-  * ethtool_rxfh_indir_default - get default value for RX flow hash indirection
-  * @index: Index in RX flow hash indirection table
-@@ -137,6 +172,57 @@ static inline u32 ethtool_rxfh_indir_default(u32 index, u32 n_rx_rings)
- 	return index % n_rx_rings;
- }
- 
-+/**
-+ * struct ethtool_rxfh_context - a custom RSS context configuration
-+ * @indir_size: Number of u32 entries in indirection table
-+ * @key_size: Size of hash key, in bytes
-+ * @priv_size: Size of driver private data, in bytes
-+ * @hfunc: RSS hash function identifier.  One of the %ETH_RSS_HASH_*
-+ * @input_xfrm: Defines how the input data is transformed. Valid values are one
-+ *	of %RXH_XFRM_*.
-+ * @indir_configured: indir has been specified (at create time or subsequently)
-+ * @key_configured: hkey has been specified (at create time or subsequently)
-+ */
-+struct ethtool_rxfh_context {
-+	u32 indir_size;
-+	u32 key_size;
-+	u16 priv_size;
-+	u8 hfunc;
-+	u8 input_xfrm;
-+	u8 indir_configured:1;
-+	u8 key_configured:1;
-+	/* private: driver private data, indirection table, and hash key are
-+	 * stored sequentially in @data area.  Use below helpers to access.
-+	 */
-+	u32 key_off;
-+	u8 data[] __aligned(sizeof(void *));
-+};
-+
-+static inline void *ethtool_rxfh_context_priv(struct ethtool_rxfh_context *ctx)
-+{
-+	return ctx->data;
-+}
-+
-+static inline u32 *ethtool_rxfh_context_indir(struct ethtool_rxfh_context *ctx)
-+{
-+	return (u32 *)(ctx->data + ALIGN(ctx->priv_size, sizeof(u32)));
-+}
-+
-+static inline u8 *ethtool_rxfh_context_key(struct ethtool_rxfh_context *ctx)
-+{
-+	return &ctx->data[ctx->key_off];
-+}
-+
-+void ethtool_rxfh_context_lost(struct net_device *dev, u32 context_id);
-+
-+struct link_mode_info {
-+	int                             speed;
-+	u8                              lanes;
-+	u8                              duplex;
-+};
-+
-+extern const struct link_mode_info link_mode_params[];
-+
- /* declare a link mode bitmap */
- #define __ETHTOOL_DECLARE_LINK_MODE_MASK(name)		\
- 	DECLARE_BITMAP(name, __ETHTOOL_LINK_MODE_MASK_NBITS)
-@@ -191,7 +277,7 @@ struct ethtool_link_ksettings {
-  *   @mode : one of the ETHTOOL_LINK_MODE_*_BIT
-  * (not atomic, no bound checking)
-  *
- * Returns true/false.
-+ * Returns: true/false.
-  */
- #define ethtool_link_ksettings_test_link_mode(ptr, name, mode)		\
- 	test_bit(ETHTOOL_LINK_MODE_ ## mode ## _BIT, (ptr)->link_modes.name)
-@@ -200,9 +286,22 @@ extern int
- __ethtool_get_link_ksettings(struct net_device *dev,
- 			     struct ethtool_link_ksettings *link_ksettings);
- 
-+struct ethtool_keee {
-+	__ETHTOOL_DECLARE_LINK_MODE_MASK(supported);
-+	__ETHTOOL_DECLARE_LINK_MODE_MASK(advertised);
-+	__ETHTOOL_DECLARE_LINK_MODE_MASK(lp_advertised);
-+	u32	tx_lpi_timer;
-+	bool	tx_lpi_enabled;
-+	bool	eee_active;
-+	bool	eee_enabled;
-+};
-+
- struct kernel_ethtool_coalesce {
- 	u8 use_cqe_mode_tx;
- 	u8 use_cqe_mode_rx;
-+	u32 tx_aggr_max_bytes;
-+	u32 tx_aggr_max_frames;
-+	u32 tx_aggr_time_usecs;
- };
- 
- /**
-@@ -246,7 +345,12 @@ bool ethtool_convert_link_mode_to_legacy_u32(u32 *legacy_u32,
- #define ETHTOOL_COALESCE_RATE_SAMPLE_INTERVAL	BIT(21)
- #define ETHTOOL_COALESCE_USE_CQE_RX		BIT(22)
- #define ETHTOOL_COALESCE_USE_CQE_TX		BIT(23)
-#define ETHTOOL_COALESCE_ALL_PARAMS		GENMASK(23, 0)
-+#define ETHTOOL_COALESCE_TX_AGGR_MAX_BYTES	BIT(24)
-+#define ETHTOOL_COALESCE_TX_AGGR_MAX_FRAMES	BIT(25)
-+#define ETHTOOL_COALESCE_TX_AGGR_TIME_USECS	BIT(26)
-+#define ETHTOOL_COALESCE_RX_PROFILE		BIT(27)
-+#define ETHTOOL_COALESCE_TX_PROFILE		BIT(28)
-+#define ETHTOOL_COALESCE_ALL_PARAMS		GENMASK(28, 0)
- 
- #define ETHTOOL_COALESCE_USECS						\
- 	(ETHTOOL_COALESCE_RX_USECS | ETHTOOL_COALESCE_TX_USECS)
-@@ -274,6 +378,10 @@ bool ethtool_convert_link_mode_to_legacy_u32(u32 *legacy_u32,
- 	 ETHTOOL_COALESCE_RATE_SAMPLE_INTERVAL)
- #define ETHTOOL_COALESCE_USE_CQE					\
- 	(ETHTOOL_COALESCE_USE_CQE_RX | ETHTOOL_COALESCE_USE_CQE_TX)
-+#define ETHTOOL_COALESCE_TX_AGGR		\
-+	(ETHTOOL_COALESCE_TX_AGGR_MAX_BYTES |	\
-+	 ETHTOOL_COALESCE_TX_AGGR_MAX_FRAMES |	\
-+	 ETHTOOL_COALESCE_TX_AGGR_TIME_USECS)
- 
- #define ETHTOOL_STAT_NOT_SET	(~0ULL)
- 
-@@ -287,48 +395,82 @@ static inline void ethtool_stats_init(u64 *stats, unsigned int n)
-  * via a more targeted API.
-  */
- struct ethtool_eth_mac_stats {
-	u64 FramesTransmittedOK;
-	u64 SingleCollisionFrames;
-	u64 MultipleCollisionFrames;
-	u64 FramesReceivedOK;
-	u64 FrameCheckSequenceErrors;
-	u64 AlignmentErrors;
-	u64 OctetsTransmittedOK;
-	u64 FramesWithDeferredXmissions;
-	u64 LateCollisions;
-	u64 FramesAbortedDueToXSColls;
-	u64 FramesLostDueToIntMACXmitError;
-	u64 CarrierSenseErrors;
-	u64 OctetsReceivedOK;
-	u64 FramesLostDueToIntMACRcvError;
-	u64 MulticastFramesXmittedOK;
-	u64 BroadcastFramesXmittedOK;
-	u64 FramesWithExcessiveDeferral;
-	u64 MulticastFramesReceivedOK;
-	u64 BroadcastFramesReceivedOK;
-	u64 InRangeLengthErrors;
-	u64 OutOfRangeLengthField;
-	u64 FrameTooLongErrors;
-+	enum ethtool_mac_stats_src src;
-+	struct_group(stats,
-+		u64 FramesTransmittedOK;
-+		u64 SingleCollisionFrames;
-+		u64 MultipleCollisionFrames;
-+		u64 FramesReceivedOK;
-+		u64 FrameCheckSequenceErrors;
-+		u64 AlignmentErrors;
-+		u64 OctetsTransmittedOK;
-+		u64 FramesWithDeferredXmissions;
-+		u64 LateCollisions;
-+		u64 FramesAbortedDueToXSColls;
-+		u64 FramesLostDueToIntMACXmitError;
-+		u64 CarrierSenseErrors;
-+		u64 OctetsReceivedOK;
-+		u64 FramesLostDueToIntMACRcvError;
-+		u64 MulticastFramesXmittedOK;
-+		u64 BroadcastFramesXmittedOK;
-+		u64 FramesWithExcessiveDeferral;
-+		u64 MulticastFramesReceivedOK;
-+		u64 BroadcastFramesReceivedOK;
-+		u64 InRangeLengthErrors;
-+		u64 OutOfRangeLengthField;
-+		u64 FrameTooLongErrors;
-+	);
- };
- 
- /* Basic IEEE 802.3 PHY statistics (30.3.2.1.*), not otherwise exposed
-  * via a more targeted API.
-  */
- struct ethtool_eth_phy_stats {
-	u64 SymbolErrorDuringCarrier;
-+	enum ethtool_mac_stats_src src;
-+	struct_group(stats,
-+		u64 SymbolErrorDuringCarrier;
-+	);
-+};
-+
-+/**
-+ * struct ethtool_phy_stats - PHY-level statistics counters
-+ * @rx_packets: Total successfully received frames
-+ * @rx_bytes: Total successfully received bytes
-+ * @rx_errors: Total received frames with errors (e.g., CRC errors)
-+ * @tx_packets: Total successfully transmitted frames
-+ * @tx_bytes: Total successfully transmitted bytes
-+ * @tx_errors: Total transmitted frames with errors
-+ *
-+ * This structure provides a standardized interface for reporting
-+ * PHY-level statistics counters. It is designed to expose statistics
-+ * commonly provided by PHYs but not explicitly defined in the IEEE
-+ * 802.3 standard.
-+ */
-+struct ethtool_phy_stats {
-+	u64 rx_packets;
-+	u64 rx_bytes;
-+	u64 rx_errors;
-+	u64 tx_packets;
-+	u64 tx_bytes;
-+	u64 tx_errors;
- };
- 
- /* Basic IEEE 802.3 MAC Ctrl statistics (30.3.3.*), not otherwise exposed
-  * via a more targeted API.
-  */
- struct ethtool_eth_ctrl_stats {
-	u64 MACControlFramesTransmitted;
-	u64 MACControlFramesReceived;
-	u64 UnsupportedOpcodesReceived;
-+	enum ethtool_mac_stats_src src;
-+	struct_group(stats,
-+		u64 MACControlFramesTransmitted;
-+		u64 MACControlFramesReceived;
-+		u64 UnsupportedOpcodesReceived;
-+	);
- };
- 
- /**
-  * struct ethtool_pause_stats - statistics for IEEE 802.3x pause frames
-+ * @src: input field denoting whether stats should be queried from the eMAC or
-+ *	pMAC (if the MM layer is supported). To be ignored otherwise.
-  * @tx_pause_frames: transmitted pause frame count. Reported to user space
-  *	as %ETHTOOL_A_PAUSE_STAT_TX_FRAMES.
-  *
-@@ -342,12 +484,37 @@ struct ethtool_eth_ctrl_stats {
-  *	from the standard.
-  */
- struct ethtool_pause_stats {
-	u64 tx_pause_frames;
-	u64 rx_pause_frames;
-+	enum ethtool_mac_stats_src src;
-+	struct_group(stats,
-+		u64 tx_pause_frames;
-+		u64 rx_pause_frames;
-+	);
- };
- 
- #define ETHTOOL_MAX_LANES	8
-+/*
-+ * IEEE 802.3ck/df defines 16 bins for FEC histogram plus one more for
-+ * the end-of-list marker, total 17 items
-+ */
-+#define ETHTOOL_FEC_HIST_MAX	17
-+/**
-+ * struct ethtool_fec_hist_range - error bits range for FEC histogram
-+ * statistics
-+ * @low: low bound of the bin (inclusive)
-+ * @high: high bound of the bin (inclusive)
-+ */
-+struct ethtool_fec_hist_range {
-+	u16 low;
-+	u16 high;
-+};
- 
-+struct ethtool_fec_hist {
-+	struct ethtool_fec_hist_value {
-+		u64 sum;
-+		u64 per_lane[ETHTOOL_MAX_LANES];
-+	} values[ETHTOOL_FEC_HIST_MAX];
-+	const struct ethtool_fec_hist_range *ranges;
-+};
- /**
-  * struct ethtool_fec_stats - statistics for IEEE 802.3 FEC
-  * @corrected_blocks: number of received blocks corrected by FEC
-@@ -365,8 +532,10 @@ struct ethtool_pause_stats {
-  *	not entire FEC data blocks. This is a non-standard statistic.
-  *	Reported to user space as %ETHTOOL_A_FEC_STAT_CORR_BITS.
-  *
- * @lane: per-lane/PCS-instance counts as defined by the standard
- * @total: error counts for the entire port, for drivers incapable of reporting
-+ * For each of the above fields, the two substructure members are:
-+ *
-+ * - @lanes: per-lane/PCS-instance counts as defined by the standard
-+ * - @total: error counts for the entire port, for drivers incapable of reporting
-  *	per-lane stats
-  *
-  * Drivers should fill in either only total or per-lane statistics, core
-@@ -389,10 +558,12 @@ struct ethtool_rmon_hist_range {
- 	u16 high;
- };
- 
-#define ETHTOOL_RMON_HIST_MAX	10
-+#define ETHTOOL_RMON_HIST_MAX	11
- 
- /**
-  * struct ethtool_rmon_stats - selected RMON (RFC 2819) statistics
-+ * @src: input field denoting whether stats should be queried from the eMAC or
-+ *	pMAC (if the MM layer is supported). To be ignored otherwise.
-  * @undersize_pkts: Equivalent to `etherStatsUndersizePkts` from the RFC.
-  * @oversize_pkts: Equivalent to `etherStatsOversizePkts` from the RFC.
-  * @fragments: Equivalent to `etherStatsFragments` from the RFC.
-@@ -408,30 +579,59 @@ struct ethtool_rmon_hist_range {
-  * ranges is left to the driver.
-  */
- struct ethtool_rmon_stats {
-	u64 undersize_pkts;
-	u64 oversize_pkts;
-	u64 fragments;
-	u64 jabbers;
-+	enum ethtool_mac_stats_src src;
-+	struct_group(stats,
-+		u64 undersize_pkts;
-+		u64 oversize_pkts;
-+		u64 fragments;
-+		u64 jabbers;
-+
-+		u64 hist[ETHTOOL_RMON_HIST_MAX];
-+		u64 hist_tx[ETHTOOL_RMON_HIST_MAX];
-+	);
-+};
- 
-	u64 hist[ETHTOOL_RMON_HIST_MAX];
-	u64 hist_tx[ETHTOOL_RMON_HIST_MAX];
-+/**
-+ * struct ethtool_ts_stats - HW timestamping statistics
-+ * @pkts: Number of packets successfully timestamped by the hardware.
-+ * @onestep_pkts_unconfirmed: Number of PTP packets with one-step TX
-+ *			      timestamping that were sent, but for which the
-+ *			      device offers no confirmation whether they made
-+ *			      it onto the wire and the timestamp was inserted
-+ *			      in the originTimestamp or correctionField, or
-+ *			      not.
-+ * @lost: Number of hardware timestamping requests where the timestamping
-+ *	information from the hardware never arrived for submission with
-+ *	the skb.
-+ * @err: Number of arbitrary timestamp generation error events that the
-+ *	hardware encountered, exclusive of @lost statistics. Cases such
-+ *	as resource exhaustion, unavailability, firmware errors, and
-+ *	detected illogical timestamp values not submitted with the skb
-+ *	are inclusive to this counter.
-+ */
-+struct ethtool_ts_stats {
-+	struct_group(tx_stats,
-+		u64 pkts;
-+		u64 onestep_pkts_unconfirmed;
-+		u64 lost;
-+		u64 err;
-+	);
- };
- 
- #define ETH_MODULE_EEPROM_PAGE_LEN	128
- #define ETH_MODULE_MAX_I2C_ADDRESS	0x7f
- 
- /**
- * struct ethtool_module_eeprom - EEPROM dump from specified page
- * @offset: Offset within the specified EEPROM page to begin read, in bytes.
- * @length: Number of bytes to read.
- * @page: Page number to read from.
- * @bank: Page bank number to read from, if applicable by EEPROM spec.
-+ * struct ethtool_module_eeprom - plug-in module EEPROM read / write parameters
-+ * @offset: When @offset is 0-127, it is used as an address to the Lower Memory
-+ *	(@page must be 0). Otherwise, it is used as an address to the
-+ *	Upper Memory.
-+ * @length: Number of bytes to read / write.
-+ * @page: Page number.
-+ * @bank: Bank number, if supported by EEPROM spec.
-  * @i2c_address: I2C address of a page. Value less than 0x7f expected. Most
-  *	EEPROMs use 0x50 or 0x51.
-  * @data: Pointer to buffer with EEPROM data of @length size.
- *
- * This can be used to manage pages during EEPROM dump in ethtool and pass
- * required information to the driver.
-  */
- struct ethtool_module_eeprom {
- 	u32	offset;
-@@ -454,15 +654,264 @@ struct ethtool_module_power_mode_params {
- };
- 
- /**
-+ * struct ethtool_mm_state - 802.3 MAC merge layer state
-+ * @verify_time:
-+ *	wait time between verification attempts in ms (according to clause
-+ *	30.14.1.6 aMACMergeVerifyTime)
-+ * @max_verify_time:
-+ *	maximum accepted value for the @verify_time variable in set requests
-+ * @verify_status:
-+ *	state of the verification state machine of the MM layer (according to
-+ *	clause 30.14.1.2 aMACMergeStatusVerify)
-+ * @tx_enabled:
-+ *	set if the MM layer is administratively enabled in the TX direction
-+ *	(according to clause 30.14.1.3 aMACMergeEnableTx)
-+ * @tx_active:
-+ *	set if the MM layer is enabled in the TX direction, which makes FP
-+ *	possible (according to 30.14.1.5 aMACMergeStatusTx). This should be
-+ *	true if MM is enabled, and the verification status is either verified,
-+ *	or disabled.
-+ * @pmac_enabled:
-+ *	set if the preemptible MAC is powered on and is able to receive
-+ *	preemptible packets and respond to verification frames.
-+ * @verify_enabled:
-+ *	set if the Verify function of the MM layer (which sends SMD-V
-+ *	verification requests) is administratively enabled (regardless of
-+ *	whether it is currently in the ETHTOOL_MM_VERIFY_STATUS_DISABLED state
-+ *	or not), according to clause 30.14.1.4 aMACMergeVerifyDisableTx (but
-+ *	using positive rather than negative logic). The device should always
-+ *	respond to received SMD-V requests as long as @pmac_enabled is set.
-+ * @tx_min_frag_size:
-+ *	the minimum size of non-final mPacket fragments that the link partner
-+ *	supports receiving, expressed in octets. Compared to the definition
-+ *	from clause 30.14.1.7 aMACMergeAddFragSize which is expressed in the
-+ *	range 0 to 3 (requiring a translation to the size in octets according
-+ *	to the formula 64 * (1 + addFragSize) - 4), a value in a continuous and
-+ *	unbounded range can be specified here.
-+ * @rx_min_frag_size:
-+ *	the minimum size of non-final mPacket fragments that this device
-+ *	supports receiving, expressed in octets.
-+ */
-+struct ethtool_mm_state {
-+	u32 verify_time;
-+	u32 max_verify_time;
-+	enum ethtool_mm_verify_status verify_status;
-+	bool tx_enabled;
-+	bool tx_active;
-+	bool pmac_enabled;
-+	bool verify_enabled;
-+	u32 tx_min_frag_size;
-+	u32 rx_min_frag_size;
-+};
-+
-+/**
-+ * struct ethtool_mm_cfg - 802.3 MAC merge layer configuration
-+ * @verify_time: see struct ethtool_mm_state
-+ * @verify_enabled: see struct ethtool_mm_state
-+ * @tx_enabled: see struct ethtool_mm_state
-+ * @pmac_enabled: see struct ethtool_mm_state
-+ * @tx_min_frag_size: see struct ethtool_mm_state
-+ */
-+struct ethtool_mm_cfg {
-+	u32 verify_time;
-+	bool verify_enabled;
-+	bool tx_enabled;
-+	bool pmac_enabled;
-+	u32 tx_min_frag_size;
-+};
-+
-+/**
-+ * struct ethtool_mm_stats - 802.3 MAC merge layer statistics
-+ * @MACMergeFrameAssErrorCount:
-+ *	received MAC frames with reassembly errors
-+ * @MACMergeFrameSmdErrorCount:
-+ *	received MAC frames/fragments rejected due to unknown or incorrect SMD
-+ * @MACMergeFrameAssOkCount:
-+ *	received MAC frames that were successfully reassembled and passed up
-+ * @MACMergeFragCountRx:
-+ *	number of additional correct SMD-C mPackets received due to preemption
-+ * @MACMergeFragCountTx:
-+ *	number of additional mPackets sent due to preemption
-+ * @MACMergeHoldCount:
-+ *	number of times the MM layer entered the HOLD state, which blocks
-+ *	transmission of preemptible traffic
-+ */
-+struct ethtool_mm_stats {
-+	u64 MACMergeFrameAssErrorCount;
-+	u64 MACMergeFrameSmdErrorCount;
-+	u64 MACMergeFrameAssOkCount;
-+	u64 MACMergeFragCountRx;
-+	u64 MACMergeFragCountTx;
-+	u64 MACMergeHoldCount;
-+};
-+
-+enum ethtool_mmsv_event {
-+	ETHTOOL_MMSV_LP_SENT_VERIFY_MPACKET,
-+	ETHTOOL_MMSV_LD_SENT_VERIFY_MPACKET,
-+	ETHTOOL_MMSV_LP_SENT_RESPONSE_MPACKET,
-+};
-+
-+/* MAC Merge verification mPacket type */
-+enum ethtool_mpacket {
-+	ETHTOOL_MPACKET_VERIFY,
-+	ETHTOOL_MPACKET_RESPONSE,
-+};
-+
-+struct ethtool_mmsv;
-+
-+/**
-+ * struct ethtool_mmsv_ops - Operations for MAC Merge Software Verification
-+ * @configure_tx: Driver callback for the event where the preemptible TX
-+ *		  becomes active or inactive. Preemptible traffic
-+ *		  classes must be committed to hardware only while
-+ *		  preemptible TX is active.
-+ * @configure_pmac: Driver callback for the event where the pMAC state
-+ *		    changes as result of an administrative setting
-+ *		    (ethtool) or a call to ethtool_mmsv_link_state_handle().
-+ * @send_mpacket: Driver-provided method for sending a Verify or a Response
-+ *		  mPacket.
-+ */
-+struct ethtool_mmsv_ops {
-+	void (*configure_tx)(struct ethtool_mmsv *mmsv, bool tx_active);
-+	void (*configure_pmac)(struct ethtool_mmsv *mmsv, bool pmac_enabled);
-+	void (*send_mpacket)(struct ethtool_mmsv *mmsv, enum ethtool_mpacket mpacket);
-+};
-+
-+/**
-+ * struct ethtool_mmsv - MAC Merge Software Verification
-+ * @ops: operations for MAC Merge Software Verification
-+ * @dev: pointer to net_device structure
-+ * @lock: serialize access to MAC Merge state between
-+ *	  ethtool requests and link state updates.
-+ * @status: current verification FSM state
-+ * @verify_timer: timer for verification in local TX direction
-+ * @verify_enabled: indicates if verification is enabled
-+ * @verify_retries: number of retries for verification
-+ * @pmac_enabled: indicates if the preemptible MAC is enabled
-+ * @verify_time: time for verification in milliseconds
-+ * @tx_enabled: indicates if transmission is enabled
-+ */
-+struct ethtool_mmsv {
-+	const struct ethtool_mmsv_ops *ops;
-+	struct net_device *dev;
-+	spinlock_t lock;
-+	enum ethtool_mm_verify_status status;
-+	struct timer_list verify_timer;
-+	bool verify_enabled;
-+	int verify_retries;
-+	bool pmac_enabled;
-+	u32 verify_time;
-+	bool tx_enabled;
-+};
-+
-+void ethtool_mmsv_stop(struct ethtool_mmsv *mmsv);
-+void ethtool_mmsv_link_state_handle(struct ethtool_mmsv *mmsv, bool up);
-+void ethtool_mmsv_event_handle(struct ethtool_mmsv *mmsv,
-+			       enum ethtool_mmsv_event event);
-+void ethtool_mmsv_get_mm(struct ethtool_mmsv *mmsv,
-+			 struct ethtool_mm_state *state);
-+void ethtool_mmsv_set_mm(struct ethtool_mmsv *mmsv, struct ethtool_mm_cfg *cfg);
-+void ethtool_mmsv_init(struct ethtool_mmsv *mmsv, struct net_device *dev,
-+		       const struct ethtool_mmsv_ops *ops);
-+
-+/**
-+ * struct ethtool_rxfh_param - RXFH (RSS) parameters
-+ * @hfunc: Defines the current RSS hash function used by HW (or to be set to).
-+ *	Valid values are one of the %ETH_RSS_HASH_*.
-+ * @indir_size: On SET, the array size of the user buffer for the
-+ *	indirection table, which may be zero, or
-+ *	%ETH_RXFH_INDIR_NO_CHANGE.  On GET (read from the driver),
-+ *	the array size of the hardware indirection table.
-+ * @indir: The indirection table of size @indir_size entries.
-+ * @key_size: On SET, the array size of the user buffer for the hash key,
-+ *	which may be zero.  On GET (read from the driver), the size of the
-+ *	hardware hash key.
-+ * @key: The hash key of size @key_size bytes.
-+ * @rss_context: RSS context identifier.  Context 0 is the default for normal
-+ *	traffic; other contexts can be referenced as the destination for RX flow
-+ *	classification rules.  On SET, %ETH_RXFH_CONTEXT_ALLOC is used
-+ *	to allocate a new RSS context; on return this field will
-+ *	contain the ID of the newly allocated context.
-+ * @rss_delete: Set to non-ZERO to remove the @rss_context context.
-+ * @input_xfrm: Defines how the input data is transformed. Valid values are one
-+ *	of %RXH_XFRM_*.
-+ */
-+struct ethtool_rxfh_param {
-+	u8	hfunc;
-+	u32	indir_size;
-+	u32	*indir;
-+	u32	key_size;
-+	u8	*key;
-+	u32	rss_context;
-+	u8	rss_delete;
-+	u8	input_xfrm;
-+};
-+
-+/**
-+ * struct ethtool_rxfh_fields - Rx Flow Hashing (RXFH) header field config
-+ * @data: which header fields are used for hashing, bitmask of RXH_* defines
-+ * @flow_type: L2-L4 network traffic flow type
-+ * @rss_context: RSS context, will only be used if rxfh_per_ctx_fields is
-+ *	set in struct ethtool_ops
-+ */
-+struct ethtool_rxfh_fields {
-+	u32 data;
-+	u32 flow_type;
-+	u32 rss_context;
-+};
-+
-+/**
-+ * struct kernel_ethtool_ts_info - kernel copy of struct ethtool_ts_info
-+ * @cmd: command number = %ETHTOOL_GET_TS_INFO
-+ * @so_timestamping: bit mask of the sum of the supported SO_TIMESTAMPING flags
-+ * @phc_index: device index of the associated PHC, or -1 if there is none
-+ * @phc_qualifier: qualifier of the associated PHC
-+ * @phc_source: source device of the associated PHC
-+ * @phc_phyindex: index of PHY device source of the associated PHC
-+ * @tx_types: bit mask of the supported hwtstamp_tx_types enumeration values
-+ * @rx_filters: bit mask of the supported hwtstamp_rx_filters enumeration values
-+ */
-+struct kernel_ethtool_ts_info {
-+	u32 cmd;
-+	u32 so_timestamping;
-+	int phc_index;
-+	enum hwtstamp_provider_qualifier phc_qualifier;
-+	enum hwtstamp_source phc_source;
-+	int phc_phyindex;
-+	u32 tx_types;
-+	u32 rx_filters;
-+};
-+
-+/**
-  * struct ethtool_ops - optional netdev operations
-+ * @supported_input_xfrm: supported types of input xfrm from %RXH_XFRM_*.
-  * @cap_link_lanes_supported: indicates if the driver supports lanes
-  *	parameter.
-+ * @rxfh_per_ctx_fields: device supports selecting different header fields
-+ *	for Rx hash calculation and RSS for each additional context.
-+ * @rxfh_per_ctx_key: device supports setting different RSS key for each
-+ *	additional context. Netlink API should report hfunc, key, and input_xfrm
-+ *	for every context, not just context 0.
-+ * @cap_rss_rxnfc_adds: device supports nonzero ring_cookie in filters with
-+ *	%FLOW_RSS flag; the queue ID from the filter is added to the value from
-+ *	the indirection table to determine the delivery queue.
-+ * @rxfh_indir_space: max size of RSS indirection tables, if indirection table
-+ *	size as returned by @get_rxfh_indir_size may change during lifetime
-+ *	of the device. Leave as 0 if the table size is constant.
-+ * @rxfh_key_space: same as @rxfh_indir_space, but for the key.
-+ * @rxfh_priv_size: size of the driver private data area the core should
-+ *	allocate for an RSS context (in &struct ethtool_rxfh_context).
-+ * @rxfh_max_num_contexts: maximum (exclusive) supported RSS context ID.
-+ *	If this is zero then the core may choose any (nonzero) ID, otherwise
-+ *	the core will only use IDs strictly less than this value, as the
-+ *	@rss_context argument to @create_rxfh_context and friends.
-  * @supported_coalesce_params: supported types of interrupt coalescing.
-  * @supported_ring_params: supported ring params.
- * @get_drvinfo: Report driver/device information.  Should only set the
- *	@driver, @version, @fw_version and @bus_info fields.  If not
- *	implemented, the @driver and @bus_info fields will be filled in
- *	according to the netdev's parent device.
-+ * @supported_hwtstamp_qualifiers: bitfield of supported hwtstamp qualifier.
-+ * @get_drvinfo: Report driver/device information. Modern drivers no
-+ *	longer have to implement this callback. Most fields are
-+ *	correctly filled in by the core using system information, or
-+ *	populated using other driver operations.
-  * @get_regs_len: Get buffer length required for @get_regs
-  * @get_regs: Get device registers
-  * @get_wol: Report whether Wake-on-Lan is enabled
-@@ -481,6 +930,7 @@ struct ethtool_module_power_mode_params {
-  *	do not attach ext_substate attribute to netlink message). If link_ext_state
-  *	and link_ext_substate are unknown, return -ENODATA. If not implemented,
-  *	link_ext_state and link_ext_substate will not be sent to userspace.
-+ * @get_link_ext_stats: Read extra link-related counters.
-  * @get_eeprom_len: Read range of EEPROM addresses for validation of
-  *	@get_eeprom and @set_eeprom requests.
-  *	Returns 0 if device does not support EEPROM access.
-@@ -540,6 +990,7 @@ struct ethtool_module_power_mode_params {
-  * @reset: Reset (part of) the device, as specified by a bitmask of
-  *	flags from &enum ethtool_reset_flags.  Returns a negative
-  *	error code or zero.
-+ * @get_rx_ring_count: Return the number of RX rings
-  * @get_rxfh_key_size: Get the size of the RX flow hash key.
-  *	Returns zero if not supported for this specific device.
-  * @get_rxfh_indir_size: Get the size of the RX flow hash indirection table.
-@@ -552,15 +1003,34 @@ struct ethtool_module_power_mode_params {
-  *	will remain unchanged.
-  *	Returns a negative error code or zero. An error code must be returned
-  *	if at least one unsupported change was requested.
- * @get_rxfh_context: Get the contents of the RX flow hash indirection table,
- *	hash key, and/or hash function assiciated to the given rss context.
-+ * @get_rxfh_fields: Get header fields used for flow hashing.
-+ * @set_rxfh_fields: Set header fields used for flow hashing.
-+ * @create_rxfh_context: Create a new RSS context with the specified RX flow
-+ *	hash indirection table, hash key, and hash function.
-+ *	The &struct ethtool_rxfh_context for this context is passed in @ctx;
-+ *	note that the indir table, hkey and hfunc are not yet populated as
-+ *	of this call.  The driver does not need to update these; the core
-+ *	will do so if this op succeeds.
-+ *	However, if @rxfh.indir is set to %NULL, the driver must update the
-+ *	indir table in @ctx with the (default or inherited) table actually in
-+ *	use; similarly, if @rxfh.key is %NULL, @rxfh.hfunc is
-+ *	%ETH_RSS_HASH_NO_CHANGE, or @rxfh.input_xfrm is %RXH_XFRM_NO_CHANGE,
-+ *	the driver should update the corresponding information in @ctx.
-+ *	If the driver provides this method, it must also provide
-+ *	@modify_rxfh_context and @remove_rxfh_context.
-  *	Returns a negative error code or zero.
- * @set_rxfh_context: Create, remove and configure RSS contexts. Allows setting
-+ * @modify_rxfh_context: Reconfigure the specified RSS context.  Allows setting
-  *	the contents of the RX flow hash indirection table, hash key, and/or
- *	hash function associated to the given context. Arguments which are set
- *	to %NULL or zero will remain unchanged.
-+ *	hash function associated with the given context.
-+ *	Parameters which are set to %NULL or zero will remain unchanged.
-+ *	The &struct ethtool_rxfh_context for this context is passed in @ctx;
-+ *	note that it will still contain the *old* settings.  The driver does
-+ *	not need to update these; the core will do so if this op succeeds.
-  *	Returns a negative error code or zero. An error code must be returned
-  *	if at least one unsupported change was requested.
-+ * @remove_rxfh_context: Remove the specified RSS context.
-+ *	The &struct ethtool_rxfh_context for this context is passed in @ctx.
-+ *	Returns a negative error code or zero.
-  * @get_channels: Get number of channels.
-  * @set_channels: Set number of channels.  Returns a negative error code or
-  *	zero.
-@@ -569,8 +1039,13 @@ struct ethtool_module_power_mode_params {
-  * @get_dump_data: Get dump data.
-  * @set_dump: Set dump specific flags to the device.
-  * @get_ts_info: Get the time stamping and PTP hardware clock capabilities.
-+ *	It may be called with RCU, or rtnl or reference on the device.
-  *	Drivers supporting transmit time stamps in software should set this to
-  *	ethtool_op_get_ts_info().
-+ * @get_ts_stats: Query the device hardware timestamping statistics. Drivers
-+ *	must not zero statistics which they don't report. The stats structure
-+ *	is initialized to ETHTOOL_STAT_NOT_SET indicating driver does not
-+ *	report statistics.
-  * @get_module_info: Get the size and type of the eeprom contained within
-  *	a plug-in module.
-  * @get_module_eeprom: Get the eeprom information from the plug-in module
-@@ -614,6 +1089,8 @@ struct ethtool_module_power_mode_params {
-  * @get_module_eeprom_by_page: Get a region of plug-in module EEPROM data from
-  *	specified page. Returns a negative error code or the amount of bytes
-  *	read.
-+ * @set_module_eeprom_by_page: Write to a region of plug-in module EEPROM,
-+ *	from kernel space only. Returns a negative error code or zero.
-  * @get_eth_phy_stats: Query some of the IEEE 802.3 PHY statistics.
-  * @get_eth_mac_stats: Query some of the IEEE 802.3 MAC statistics.
-  * @get_eth_ctrl_stats: Query some of the IEEE 802.3 MAC Ctrl statistics.
-@@ -624,6 +1101,9 @@ struct ethtool_module_power_mode_params {
-  *	plugged-in.
-  * @set_module_power_mode: Set the power mode policy for the plug-in module
-  *	used by the network device.
-+ * @get_mm: Query the 802.3 MAC Merge layer state.
-+ * @set_mm: Set the 802.3 MAC Merge layer parameters.
-+ * @get_mm_stats: Query the 802.3 MAC Merge layer statistics.
-  *
-  * All operations are optional (i.e. the function pointer may be set
-  * to %NULL) and callers must take this into account.  Callers must
-@@ -638,9 +1118,18 @@ struct ethtool_module_power_mode_params {
-  * of the generic netdev features interface.
-  */
- struct ethtool_ops {
-+	u32     supported_input_xfrm:8;
- 	u32     cap_link_lanes_supported:1;
-+	u32	rxfh_per_ctx_fields:1;
-+	u32	rxfh_per_ctx_key:1;
-+	u32	cap_rss_rxnfc_adds:1;
-+	u32	rxfh_indir_space;
-+	u16	rxfh_key_space;
-+	u16	rxfh_priv_size;
-+	u32	rxfh_max_num_contexts;
- 	u32	supported_coalesce_params;
- 	u32	supported_ring_params;
-+	u32	supported_hwtstamp_qualifiers;
- 	void	(*get_drvinfo)(struct net_device *, struct ethtool_drvinfo *);
- 	int	(*get_regs_len)(struct net_device *);
- 	void	(*get_regs)(struct net_device *, struct ethtool_regs *, void *);
-@@ -652,6 +1141,8 @@ struct ethtool_ops {
- 	u32	(*get_link)(struct net_device *);
- 	int	(*get_link_ext_state)(struct net_device *,
- 				      struct ethtool_link_ext_state_info *);
-+	void	(*get_link_ext_stats)(struct net_device *dev,
-+				      struct ethtool_link_ext_stats *stats);
- 	int	(*get_eeprom_len)(struct net_device *);
- 	int	(*get_eeprom)(struct net_device *,
- 			      struct ethtool_eeprom *, u8 *);
-@@ -694,30 +1185,44 @@ struct ethtool_ops {
- 	int	(*set_rxnfc)(struct net_device *, struct ethtool_rxnfc *);
- 	int	(*flash_device)(struct net_device *, struct ethtool_flash *);
- 	int	(*reset)(struct net_device *, u32 *);
-+	u32	(*get_rx_ring_count)(struct net_device *dev);
- 	u32	(*get_rxfh_key_size)(struct net_device *);
- 	u32	(*get_rxfh_indir_size)(struct net_device *);
-	int	(*get_rxfh)(struct net_device *, u32 *indir, u8 *key,
-			    u8 *hfunc);
-	int	(*set_rxfh)(struct net_device *, const u32 *indir,
-			    const u8 *key, const u8 hfunc);
-	int	(*get_rxfh_context)(struct net_device *, u32 *indir, u8 *key,
-				    u8 *hfunc, u32 rss_context);
-	int	(*set_rxfh_context)(struct net_device *, const u32 *indir,
-				    const u8 *key, const u8 hfunc,
-				    u32 *rss_context, bool delete);
-+	int	(*get_rxfh)(struct net_device *, struct ethtool_rxfh_param *);
-+	int	(*set_rxfh)(struct net_device *, struct ethtool_rxfh_param *,
-+			    struct netlink_ext_ack *extack);
-+	int	(*get_rxfh_fields)(struct net_device *,
-+				   struct ethtool_rxfh_fields *);
-+	int	(*set_rxfh_fields)(struct net_device *,
-+				   const struct ethtool_rxfh_fields *,
-+				   struct netlink_ext_ack *extack);
-+	int	(*create_rxfh_context)(struct net_device *,
-+				       struct ethtool_rxfh_context *ctx,
-+				       const struct ethtool_rxfh_param *rxfh,
-+				       struct netlink_ext_ack *extack);
-+	int	(*modify_rxfh_context)(struct net_device *,
-+				       struct ethtool_rxfh_context *ctx,
-+				       const struct ethtool_rxfh_param *rxfh,
-+				       struct netlink_ext_ack *extack);
-+	int	(*remove_rxfh_context)(struct net_device *,
-+				       struct ethtool_rxfh_context *ctx,
-+				       u32 rss_context,
-+				       struct netlink_ext_ack *extack);
- 	void	(*get_channels)(struct net_device *, struct ethtool_channels *);
- 	int	(*set_channels)(struct net_device *, struct ethtool_channels *);
- 	int	(*get_dump_flag)(struct net_device *, struct ethtool_dump *);
- 	int	(*get_dump_data)(struct net_device *,
- 				 struct ethtool_dump *, void *);
- 	int	(*set_dump)(struct net_device *, struct ethtool_dump *);
-	int	(*get_ts_info)(struct net_device *, struct ethtool_ts_info *);
-+	int	(*get_ts_info)(struct net_device *, struct kernel_ethtool_ts_info *);
-+	void	(*get_ts_stats)(struct net_device *dev,
-+				struct ethtool_ts_stats *ts_stats);
- 	int     (*get_module_info)(struct net_device *,
- 				   struct ethtool_modinfo *);
- 	int     (*get_module_eeprom)(struct net_device *,
- 				     struct ethtool_eeprom *, u8 *);
-	int	(*get_eee)(struct net_device *, struct ethtool_eee *);
-	int	(*set_eee)(struct net_device *, struct ethtool_eee *);
-+	int	(*get_eee)(struct net_device *dev, struct ethtool_keee *eee);
-+	int	(*set_eee)(struct net_device *dev, struct ethtool_keee *eee);
- 	int	(*get_tunable)(struct net_device *,
- 			       const struct ethtool_tunable *, void *);
- 	int	(*set_tunable)(struct net_device *,
-@@ -731,7 +1236,8 @@ struct ethtool_ops {
- 	int	(*set_link_ksettings)(struct net_device *,
- 				      const struct ethtool_link_ksettings *);
- 	void	(*get_fec_stats)(struct net_device *dev,
-				 struct ethtool_fec_stats *fec_stats);
-+				 struct ethtool_fec_stats *fec_stats,
-+				 struct ethtool_fec_hist *hist);
- 	int	(*get_fecparam)(struct net_device *,
- 				      struct ethtool_fecparam *);
- 	int	(*set_fecparam)(struct net_device *,
-@@ -745,6 +1251,9 @@ struct ethtool_ops {
- 	int	(*get_module_eeprom_by_page)(struct net_device *dev,
- 					     const struct ethtool_module_eeprom *page,
- 					     struct netlink_ext_ack *extack);
-+	int	(*set_module_eeprom_by_page)(struct net_device *dev,
-+					     const struct ethtool_module_eeprom *page,
-+					     struct netlink_ext_ack *extack);
- 	void	(*get_eth_phy_stats)(struct net_device *dev,
- 				     struct ethtool_eth_phy_stats *phy_stats);
- 	void	(*get_eth_mac_stats)(struct net_device *dev,
-@@ -760,6 +1269,10 @@ struct ethtool_ops {
- 	int	(*set_module_power_mode)(struct net_device *dev,
- 					 const struct ethtool_module_power_mode_params *params,
- 					 struct netlink_ext_ack *extack);
-+	int	(*get_mm)(struct net_device *dev, struct ethtool_mm_state *state);
-+	int	(*set_mm)(struct net_device *dev, struct ethtool_mm_cfg *cfg,
-+			  struct netlink_ext_ack *extack);
-+	void	(*get_mm_stats)(struct net_device *dev, struct ethtool_mm_stats *stats);
- };
- 
- int ethtool_check_ops(const struct ethtool_ops *ops);
-@@ -783,14 +1296,34 @@ int ethtool_virtdev_set_link_ksettings(struct net_device *dev,
- 				       const struct ethtool_link_ksettings *cmd,
- 				       u32 *dev_speed, u8 *dev_duplex);
- 
-+/**
-+ * struct ethtool_netdev_state - per-netdevice state for ethtool features
-+ * @rss_ctx:		XArray of custom RSS contexts
-+ * @rss_lock:		Protects entries in @rss_ctx.  May be taken from
-+ *			within RTNL.
-+ * @wol_enabled:	Wake-on-LAN is enabled
-+ * @module_fw_flash_in_progress: Module firmware flashing is in progress.
-+ */
-+struct ethtool_netdev_state {
-+	struct xarray		rss_ctx;
-+	struct mutex		rss_lock;
-+	unsigned		wol_enabled:1;
-+	unsigned		module_fw_flash_in_progress:1;
-+};
-+
- struct phy_device;
- struct phy_tdr_config;
-+struct phy_plca_cfg;
-+struct phy_plca_status;
- 
- /**
-  * struct ethtool_phy_ops - Optional PHY device options
-  * @get_sset_count: Get number of strings that @get_strings will write.
-  * @get_strings: Return a set of strings that describe the requested objects
-  * @get_stats: Return extended statistics about the PHY device.
-+ * @get_plca_cfg: Return PLCA configuration.
-+ * @set_plca_cfg: Set PLCA configuration.
-+ * @get_plca_status: Get PLCA configuration.
-  * @start_cable_test: Start a cable test
-  * @start_cable_test_tdr: Start a Time Domain Reflectometry cable test
-  *
-@@ -802,6 +1335,13 @@ struct ethtool_phy_ops {
- 	int (*get_strings)(struct phy_device *dev, u8 *data);
- 	int (*get_stats)(struct phy_device *dev,
- 			 struct ethtool_stats *stats, u64 *data);
-+	int (*get_plca_cfg)(struct phy_device *dev,
-+			    struct phy_plca_cfg *plca_cfg);
-+	int (*set_plca_cfg)(struct phy_device *dev,
-+			    const struct phy_plca_cfg *plca_cfg,
-+			    struct netlink_ext_ack *extack);
-+	int (*get_plca_status)(struct phy_device *dev,
-+			       struct phy_plca_status *plca_st);
- 	int (*start_cable_test)(struct phy_device *phydev,
- 				struct netlink_ext_ack *extack);
- 	int (*start_cable_test_tdr)(struct phy_device *phydev,
-@@ -830,17 +1370,115 @@ ethtool_params_from_link_mode(struct ethtool_link_ksettings *link_ksettings,
-  * @dev: pointer to net_device structure
-  * @vclock_index: pointer to pointer of vclock index
-  *
- * Return number of phc vclocks
-+ * Return: number of phc vclocks
-  */
- int ethtool_get_phc_vclocks(struct net_device *dev, int **vclock_index);
- 
-+/* Some generic methods drivers may use in their ethtool_ops */
-+u32 ethtool_op_get_link(struct net_device *dev);
-+int ethtool_op_get_ts_info(struct net_device *dev,
-+			   struct kernel_ethtool_ts_info *eti);
-+
-+/**
-+ * ethtool_mm_frag_size_add_to_min - Translate (standard) additional fragment
-+ *	size expressed as multiplier into (absolute) minimum fragment size
-+ *	value expressed in octets
-+ * @val_add: Value of addFragSize multiplier
-+ */
-+static inline u32 ethtool_mm_frag_size_add_to_min(u32 val_add)
-+{
-+	return (ETH_ZLEN + ETH_FCS_LEN) * (1 + val_add) - ETH_FCS_LEN;
-+}
-+
-+/**
-+ * ethtool_mm_frag_size_min_to_add - Translate (absolute) minimum fragment size
-+ *	expressed in octets into (standard) additional fragment size expressed
-+ *	as multiplier
-+ * @val_min: Value of addFragSize variable in octets
-+ * @val_add: Pointer where the standard addFragSize value is to be returned
-+ * @extack: Netlink extended ack
-+ *
-+ * Translate a value in octets to one of 0, 1, 2, 3 according to the reverse
-+ * application of the 802.3 formula 64 * (1 + addFragSize) - 4. To be called
-+ * by drivers which do not support programming the minimum fragment size to a
-+ * continuous range. Returns error on other fragment length values.
-+ */
-+static inline int ethtool_mm_frag_size_min_to_add(u32 val_min, u32 *val_add,
-+						  struct netlink_ext_ack *extack)
-+{
-+	u32 add_frag_size;
-+
-+	for (add_frag_size = 0; add_frag_size < 4; add_frag_size++) {
-+		if (ethtool_mm_frag_size_add_to_min(add_frag_size) == val_min) {
-+			*val_add = add_frag_size;
-+			return 0;
-+		}
-+	}
-+
-+	NL_SET_ERR_MSG_MOD(extack,
-+			   "minFragSize required to be one of 60, 124, 188 or 252");
-+	return -EINVAL;
-+}
-+
-+/**
-+ * ethtool_get_ts_info_by_layer - Obtains time stamping capabilities from the MAC or PHY layer.
-+ * @dev: pointer to net_device structure
-+ * @info: buffer to hold the result
-+ * Returns: zero on success, non-zero otherwise.
-+ */
-+int ethtool_get_ts_info_by_layer(struct net_device *dev,
-+				 struct kernel_ethtool_ts_info *info);
-+
- /**
-  * ethtool_sprintf - Write formatted string to ethtool string data
- * @data: Pointer to start of string to update
-+ * @data: Pointer to a pointer to the start of string to update
-  * @fmt: Format of string to write
-  *
- * Write formatted string to data. Update data to point at start of
-+ * Write formatted string to *data. Update *data to point at start of
-  * next string.
-  */
- extern __printf(2, 3) void ethtool_sprintf(u8 **data, const char *fmt, ...);
-+
-+/**
-+ * ethtool_puts - Write string to ethtool string data
-+ * @data: Pointer to a pointer to the start of string to update
-+ * @str: String to write
-+ *
-+ * Write string to *data without a trailing newline. Update *data
-+ * to point at start of next string.
-+ *
-+ * Prefer this function to ethtool_sprintf() when given only
-+ * two arguments or if @fmt is just "%s".
-+ */
-+extern void ethtool_puts(u8 **data, const char *str);
-+
-+/**
-+ * ethtool_cpy - Write possibly-not-NUL-terminated string to ethtool string data
-+ * @data: Pointer to a pointer to the start of string to write into
-+ * @str: NUL-byte padded char array of size ETH_GSTRING_LEN to copy from
-+ */
-+#define ethtool_cpy(data, str)	do {				\
-+	BUILD_BUG_ON(sizeof(str) != ETH_GSTRING_LEN);		\
-+	memcpy(*(data), str, ETH_GSTRING_LEN);			\
-+	*(data) += ETH_GSTRING_LEN;				\
-+} while (0)
-+
-+/* Link mode to forced speed capabilities maps */
-+struct ethtool_forced_speed_map {
-+	u32		speed;
-+	__ETHTOOL_DECLARE_LINK_MODE_MASK(caps);
-+
-+	const u32	*cap_arr;
-+	u32		arr_size;
-+};
-+
-+#define ETHTOOL_FORCED_SPEED_MAP(prefix, value)				\
-+{									\
-+	.speed		= SPEED_##value,				\
-+	.cap_arr	= prefix##_##value,				\
-+	.arr_size	= ARRAY_SIZE(prefix##_##value),			\
-+}
-+
-+void
-+ethtool_forced_speed_maps_init(struct ethtool_forced_speed_map *maps, u32 size);
- #endif /* _LINUX_ETHTOOL_H */
-diff --git a/include/linux/ethtool_netlink.h b/include/linux/ethtool_netlink.h
-index aba348d58ff6..39254b2726c0 100644
--- a/include/linux/ethtool_netlink.h
-+++ b/include/linux/ethtool_netlink.h
-@@ -23,12 +23,28 @@ struct phy_device;
- int ethnl_cable_test_alloc(struct phy_device *phydev, u8 cmd);
- void ethnl_cable_test_free(struct phy_device *phydev);
- void ethnl_cable_test_finished(struct phy_device *phydev);
-int ethnl_cable_test_result(struct phy_device *phydev, u8 pair, u8 result);
-int ethnl_cable_test_fault_length(struct phy_device *phydev, u8 pair, u32 cm);
-+int ethnl_cable_test_result_with_src(struct phy_device *phydev, u8 pair,
-+				     u8 result, u32 src);
-+int ethnl_cable_test_fault_length_with_src(struct phy_device *phydev, u8 pair,
-+					   u32 cm, u32 src);
- int ethnl_cable_test_amplitude(struct phy_device *phydev, u8 pair, s16 mV);
- int ethnl_cable_test_pulse(struct phy_device *phydev, u16 mV);
- int ethnl_cable_test_step(struct phy_device *phydev, u32 first, u32 last,
- 			  u32 step);
-+void ethtool_aggregate_mac_stats(struct net_device *dev,
-+				 struct ethtool_eth_mac_stats *mac_stats);
-+void ethtool_aggregate_phy_stats(struct net_device *dev,
-+				 struct ethtool_eth_phy_stats *phy_stats);
-+void ethtool_aggregate_ctrl_stats(struct net_device *dev,
-+				  struct ethtool_eth_ctrl_stats *ctrl_stats);
-+void ethtool_aggregate_pause_stats(struct net_device *dev,
-+				   struct ethtool_pause_stats *pause_stats);
-+void ethtool_aggregate_rmon_stats(struct net_device *dev,
-+				  struct ethtool_rmon_stats *rmon_stats);
-+bool ethtool_dev_mm_supported(struct net_device *dev);
-+
-+void ethnl_pse_send_ntf(struct net_device *netdev, unsigned long notif);
-+
- #else
- static inline int ethnl_cable_test_alloc(struct phy_device *phydev, u8 cmd)
- {
-@@ -42,14 +58,14 @@ static inline void ethnl_cable_test_free(struct phy_device *phydev)
- static inline void ethnl_cable_test_finished(struct phy_device *phydev)
- {
- }
-static inline int ethnl_cable_test_result(struct phy_device *phydev, u8 pair,
-					  u8 result)
-+static inline int ethnl_cable_test_result_with_src(struct phy_device *phydev,
-+						   u8 pair, u8 result, u32 src)
- {
- 	return -EOPNOTSUPP;
- }
- 
-static inline int ethnl_cable_test_fault_length(struct phy_device *phydev,
-						u8 pair, u32 cm)
-+static inline int ethnl_cable_test_fault_length_with_src(struct phy_device *phydev,
-+							 u8 pair, u32 cm, u32 src)
- {
- 	return -EOPNOTSUPP;
- }
-@@ -70,5 +86,61 @@ static inline int ethnl_cable_test_step(struct phy_device *phydev, u32 first,
- {
- 	return -EOPNOTSUPP;
- }
-+
-+static inline void
-+ethtool_aggregate_mac_stats(struct net_device *dev,
-+			    struct ethtool_eth_mac_stats *mac_stats)
-+{
-+}
-+
-+static inline void
-+ethtool_aggregate_phy_stats(struct net_device *dev,
-+			    struct ethtool_eth_phy_stats *phy_stats)
-+{
-+}
-+
-+static inline void
-+ethtool_aggregate_ctrl_stats(struct net_device *dev,
-+			     struct ethtool_eth_ctrl_stats *ctrl_stats)
-+{
-+}
-+
-+static inline void
-+ethtool_aggregate_pause_stats(struct net_device *dev,
-+			      struct ethtool_pause_stats *pause_stats)
-+{
-+}
-+
-+static inline void
-+ethtool_aggregate_rmon_stats(struct net_device *dev,
-+			     struct ethtool_rmon_stats *rmon_stats)
-+{
-+}
-+
-+static inline bool ethtool_dev_mm_supported(struct net_device *dev)
-+{
-+	return false;
-+}
-+
-+static inline void ethnl_pse_send_ntf(struct net_device *netdev,
-+				      unsigned long notif)
-+{
-+}
-+
- #endif /* IS_ENABLED(CONFIG_ETHTOOL_NETLINK) */
-+
-+static inline int ethnl_cable_test_result(struct phy_device *phydev, u8 pair,
-+					  u8 result)
-+{
-+	return ethnl_cable_test_result_with_src(phydev, pair, result,
-+						ETHTOOL_A_CABLE_INF_SRC_TDR);
-+}
-+
-+static inline int ethnl_cable_test_fault_length(struct phy_device *phydev,
-+						u8 pair, u32 cm)
-+{
-+	return ethnl_cable_test_fault_length_with_src(phydev, pair, cm,
-+						      ETHTOOL_A_CABLE_INF_SRC_TDR);
-+}
-+
- #endif /* _LINUX_ETHTOOL_NETLINK_H_ */
-diff --git a/include/linux/eventfd.h b/include/linux/eventfd.h
-index 305d5f19093b..e32bee4345fb 100644
--- a/include/linux/eventfd.h
-+++ b/include/linux/eventfd.h
-@@ -9,12 +9,12 @@
- #ifndef _LINUX_EVENTFD_H
- #define _LINUX_EVENTFD_H
- 
-#include <linux/fcntl.h>
- #include <linux/wait.h>
- #include <linux/err.h>
- #include <linux/percpu-defs.h>
- #include <linux/percpu.h>
- #include <linux/sched.h>
-+#include <uapi/linux/eventfd.h>
- 
- /*
-  * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining
-@@ -23,10 +23,6 @@
-  * from eventfd, in order to leave a free define-space for
-  * shared O_* flags.
-  */
-#define EFD_SEMAPHORE (1 << 0)
-#define EFD_CLOEXEC O_CLOEXEC
-#define EFD_NONBLOCK O_NONBLOCK
-
- #define EFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK)
- #define EFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS | EFD_SEMAPHORE)
- 
-@@ -39,14 +35,14 @@ void eventfd_ctx_put(struct eventfd_ctx *ctx);
- struct file *eventfd_fget(int fd);
- struct eventfd_ctx *eventfd_ctx_fdget(int fd);
- struct eventfd_ctx *eventfd_ctx_fileget(struct file *file);
-__u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n);
-+void eventfd_signal_mask(struct eventfd_ctx *ctx, __poll_t mask);
- int eventfd_ctx_remove_wait_queue(struct eventfd_ctx *ctx, wait_queue_entry_t *wait,
- 				  __u64 *cnt);
- void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt);
- 
- static inline bool eventfd_signal_allowed(void)
- {
-	return !current->in_eventfd_signal;
-+	return !current->in_eventfd;
- }
- 
- #else /* CONFIG_EVENTFD */
-@@ -61,9 +57,8 @@ static inline struct eventfd_ctx *eventfd_ctx_fdget(int fd)
- 	return ERR_PTR(-ENOSYS);
- }
- 
-static inline int eventfd_signal(struct eventfd_ctx *ctx, int n)
-+static inline void eventfd_signal_mask(struct eventfd_ctx *ctx, __poll_t mask)
- {
-	return -ENOSYS;
- }
- 
- static inline void eventfd_ctx_put(struct eventfd_ctx *ctx)
-@@ -89,5 +84,10 @@ static inline void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt)
- 
- #endif
- 
-+static inline void eventfd_signal(struct eventfd_ctx *ctx)
-+{
-+	eventfd_signal_mask(ctx, 0);
-+}
-+
- #endif /* _LINUX_EVENTFD_H */
- 
-diff --git a/include/linux/eventpoll.h b/include/linux/eventpoll.h
-index 3337745d81bd..ccb478eb174b 100644
--- a/include/linux/eventpoll.h
-+++ b/include/linux/eventpoll.h
-@@ -25,6 +25,10 @@ struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd, unsigned long t
- /* Used to release the epoll bits inside the "struct file" */
- void eventpoll_release_file(struct file *file);
- 
-+/* Copy ready events to userspace */
-+int epoll_sendevents(struct file *file, struct epoll_event __user *events,
-+		     int maxevents);
-+
- /*
-  * This is called from inside fs/file_table.c:__fput() to unlink files
-  * from the eventpoll interface. We need to have this facility to cleanup
-@@ -42,7 +46,7 @@ static inline void eventpoll_release(struct file *file)
- 	 * because the file in on the way to be removed and nobody ( but
- 	 * eventpoll ) has still a reference to this file.
- 	 */
-	if (likely(!file->f_ep))
-+	if (likely(!READ_ONCE(file->f_ep)))
- 		return;
- 
- 	/*
-diff --git a/include/linux/evm.h b/include/linux/evm.h
-index aa63e0b3c0a2..ddece4a6b25d 100644
--- a/include/linux/evm.h
-+++ b/include/linux/evm.h
-@@ -12,37 +12,22 @@
- #include <linux/integrity.h>
- #include <linux/xattr.h>
- 
-struct integrity_iint_cache;
-
- #ifdef CONFIG_EVM
- extern int evm_set_key(void *key, size_t keylen);
- extern enum integrity_status evm_verifyxattr(struct dentry *dentry,
- 					     const char *xattr_name,
- 					     void *xattr_value,
-					     size_t xattr_value_len,
-					     struct integrity_iint_cache *iint);
-extern int evm_inode_setattr(struct user_namespace *mnt_userns,
-			     struct dentry *dentry, struct iattr *attr);
-extern void evm_inode_post_setattr(struct dentry *dentry, int ia_valid);
-extern int evm_inode_setxattr(struct user_namespace *mnt_userns,
-			      struct dentry *dentry, const char *name,
-			      const void *value, size_t size);
-extern void evm_inode_post_setxattr(struct dentry *dentry,
-				    const char *xattr_name,
-				    const void *xattr_value,
-				    size_t xattr_value_len);
-extern int evm_inode_removexattr(struct user_namespace *mnt_userns,
-				 struct dentry *dentry, const char *xattr_name);
-extern void evm_inode_post_removexattr(struct dentry *dentry,
-				       const char *xattr_name);
-extern int evm_inode_init_security(struct inode *inode,
-				   const struct xattr *xattr_array,
-				   struct xattr *evm);
-+					     size_t xattr_value_len);
-+int evm_inode_init_security(struct inode *inode, struct inode *dir,
-+			    const struct qstr *qstr, struct xattr *xattrs,
-+			    int *xattr_count);
- extern bool evm_revalidate_status(const char *xattr_name);
- extern int evm_protected_xattr_if_enabled(const char *req_xattr_name);
- extern int evm_read_protected_xattrs(struct dentry *dentry, u8 *buffer,
- 				     int buffer_size, char type,
- 				     bool canonical_fmt);
-+extern bool evm_metadata_changed(struct inode *inode,
-+				 struct inode *metadata_inode);
- #ifdef CONFIG_FS_POSIX_ACL
- extern int posix_xattr_acl(const char *xattrname);
- #else
-@@ -62,55 +47,16 @@ static inline int evm_set_key(void *key, size_t keylen)
- static inline enum integrity_status evm_verifyxattr(struct dentry *dentry,
- 						    const char *xattr_name,
- 						    void *xattr_value,
-						    size_t xattr_value_len,
-					struct integrity_iint_cache *iint)
-+						    size_t xattr_value_len)
- {
- 	return INTEGRITY_UNKNOWN;
- }
- #endif
- 
-static inline int evm_inode_setattr(struct user_namespace *mnt_userns,
-				    struct dentry *dentry, struct iattr *attr)
-{
-	return 0;
-}
-
-static inline void evm_inode_post_setattr(struct dentry *dentry, int ia_valid)
-{
-	return;
-}
-
-static inline int evm_inode_setxattr(struct user_namespace *mnt_userns,
-				     struct dentry *dentry, const char *name,
-				     const void *value, size_t size)
-{
-	return 0;
-}
-
-static inline void evm_inode_post_setxattr(struct dentry *dentry,
-					   const char *xattr_name,
-					   const void *xattr_value,
-					   size_t xattr_value_len)
-{
-	return;
-}
-
-static inline int evm_inode_removexattr(struct user_namespace *mnt_userns,
-					struct dentry *dentry,
-					const char *xattr_name)
-{
-	return 0;
-}
-
-static inline void evm_inode_post_removexattr(struct dentry *dentry,
-					      const char *xattr_name)
-{
-	return;
-}
-
-static inline int evm_inode_init_security(struct inode *inode,
-					  const struct xattr *xattr_array,
-					  struct xattr *evm)
-+static inline int evm_inode_init_security(struct inode *inode, struct inode *dir,
-+					  const struct qstr *qstr,
-+					  struct xattr *xattrs,
-+					  int *xattr_count)
- {
- 	return 0;
- }
-@@ -132,5 +78,11 @@ static inline int evm_read_protected_xattrs(struct dentry *dentry, u8 *buffer,
- 	return -EOPNOTSUPP;
- }
- 
-+static inline bool evm_metadata_changed(struct inode *inode,
-+					struct inode *metadata_inode)
-+{
-+	return false;
-+}
-+
- #endif /* CONFIG_EVM */
- #endif /* LINUX_EVM_H */
-diff --git a/include/linux/execmem.h b/include/linux/execmem.h
-new file mode 100644
-index 000000000000..7de229134e30
--- /dev/null
-+++ b/include/linux/execmem.h
-@@ -0,0 +1,207 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_EXECMEM_ALLOC_H
-+#define _LINUX_EXECMEM_ALLOC_H
-+
-+#include <linux/types.h>
-+#include <linux/moduleloader.h>
-+#include <linux/cleanup.h>
-+
-+#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
-+		!defined(CONFIG_KASAN_VMALLOC)
-+#include <linux/kasan.h>
-+#define MODULE_ALIGN (PAGE_SIZE << KASAN_SHADOW_SCALE_SHIFT)
-+#else
-+#define MODULE_ALIGN PAGE_SIZE
-+#endif
-+
-+/**
-+ * enum execmem_type - types of executable memory ranges
-+ *
-+ * There are several subsystems that allocate executable memory.
-+ * Architectures define different restrictions on placement,
-+ * permissions, alignment and other parameters for memory that can be used
-+ * by these subsystems.
-+ * Types in this enum identify subsystems that allocate executable memory
-+ * and let architectures define parameters for ranges suitable for
-+ * allocations by each subsystem.
-+ *
-+ * @EXECMEM_DEFAULT: default parameters that would be used for types that
-+ * are not explicitly defined.
-+ * @EXECMEM_MODULE_TEXT: parameters for module text sections
-+ * @EXECMEM_KPROBES: parameters for kprobes
-+ * @EXECMEM_FTRACE: parameters for ftrace
-+ * @EXECMEM_BPF: parameters for BPF
-+ * @EXECMEM_MODULE_DATA: parameters for module data sections
-+ * @EXECMEM_TYPE_MAX:
-+ */
-+enum execmem_type {
-+	EXECMEM_DEFAULT,
-+	EXECMEM_MODULE_TEXT = EXECMEM_DEFAULT,
-+	EXECMEM_KPROBES,
-+	EXECMEM_FTRACE,
-+	EXECMEM_BPF,
-+	EXECMEM_MODULE_DATA,
-+	EXECMEM_TYPE_MAX,
-+};
-+
-+/**
-+ * enum execmem_range_flags - options for executable memory allocations
-+ * @EXECMEM_KASAN_SHADOW:	allocate kasan shadow
-+ * @EXECMEM_ROX_CACHE:		allocations should use ROX cache of huge pages
-+ */
-+enum execmem_range_flags {
-+	EXECMEM_KASAN_SHADOW	= (1 << 0),
-+	EXECMEM_ROX_CACHE	= (1 << 1),
-+};
-+
-+#ifdef CONFIG_ARCH_HAS_EXECMEM_ROX
-+/**
-+ * execmem_fill_trapping_insns - set memory to contain instructions that
-+ *				 will trap
-+ * @ptr:	pointer to memory to fill
-+ * @size:	size of the range to fill
-+ *
-+ * A hook for architecures to fill execmem ranges with invalid instructions.
-+ * Architectures that use EXECMEM_ROX_CACHE must implement this.
-+ */
-+void execmem_fill_trapping_insns(void *ptr, size_t size);
-+
-+/**
-+ * execmem_restore_rox - restore read-only-execute permissions
-+ * @ptr:	address of the region to remap
-+ * @size:	size of the region to remap
-+ *
-+ * Restores read-only-execute permissions on a range [@ptr, @ptr + @size)
-+ * after it was temporarily remapped as writable. Relies on architecture
-+ * implementation of set_memory_rox() to restore mapping using large pages.
-+ *
-+ * Return: 0 on success or negative error code on failure.
-+ */
-+int execmem_restore_rox(void *ptr, size_t size);
-+#else
-+static inline int execmem_restore_rox(void *ptr, size_t size) { return 0; }
-+#endif
-+
-+/**
-+ * struct execmem_range - definition of an address space suitable for code and
-+ *			  related data allocations
-+ * @start:	address space start
-+ * @end:	address space end (inclusive)
-+ * @fallback_start: start of the secondary address space range for fallback
-+ *                  allocations on architectures that require it
-+ * @fallback_end:   start of the secondary address space (inclusive)
-+ * @pgprot:	permissions for memory in this address space
-+ * @alignment:	alignment required for text allocations
-+ * @flags:	options for memory allocations for this range
-+ */
-+struct execmem_range {
-+	unsigned long   start;
-+	unsigned long   end;
-+	unsigned long   fallback_start;
-+	unsigned long   fallback_end;
-+	pgprot_t        pgprot;
-+	unsigned int	alignment;
-+	enum execmem_range_flags flags;
-+};
-+
-+/**
-+ * struct execmem_info - architecture parameters for code allocations
-+ * @ranges: array of parameter sets defining architecture specific
-+ * parameters for executable memory allocations. The ranges that are not
-+ * explicitly initialized by an architecture use parameters defined for
-+ * @EXECMEM_DEFAULT.
-+ */
-+struct execmem_info {
-+	struct execmem_range	ranges[EXECMEM_TYPE_MAX];
-+};
-+
-+/**
-+ * execmem_arch_setup - define parameters for allocations of executable memory
-+ *
-+ * A hook for architectures to define parameters for allocations of
-+ * executable memory. These parameters should be filled into the
-+ * @execmem_info structure.
-+ *
-+ * For architectures that do not implement this method a default set of
-+ * parameters will be used
-+ *
-+ * Return: a structure defining architecture parameters and restrictions
-+ * for allocations of executable memory
-+ */
-+struct execmem_info *execmem_arch_setup(void);
-+
-+/**
-+ * execmem_alloc - allocate executable memory
-+ * @type: type of the allocation
-+ * @size: how many bytes of memory are required
-+ *
-+ * Allocates memory that will contain executable code, either generated or
-+ * loaded from kernel modules.
-+ *
-+ * Allocates memory that will contain data coupled with executable code,
-+ * like data sections in kernel modules.
-+ *
-+ * The memory will have protections defined by architecture for executable
-+ * region of the @type.
-+ *
-+ * Return: a pointer to the allocated memory or %NULL
-+ */
-+void *execmem_alloc(enum execmem_type type, size_t size);
-+
-+/**
-+ * execmem_alloc_rw - allocate writable executable memory
-+ * @type: type of the allocation
-+ * @size: how many bytes of memory are required
-+ *
-+ * Allocates memory that will contain executable code, either generated or
-+ * loaded from kernel modules.
-+ *
-+ * Allocates memory that will contain data coupled with executable code,
-+ * like data sections in kernel modules.
-+ *
-+ * Forces writable permissions on the allocated memory and the caller is
-+ * responsible to manage the permissions afterwards.
-+ *
-+ * For architectures that use ROX cache the permissions will be set to R+W.
-+ * For architectures that don't use ROX cache the default permissions for @type
-+ * will be used as they must be writable.
-+ *
-+ * Return: a pointer to the allocated memory or %NULL
-+ */
-+void *execmem_alloc_rw(enum execmem_type type, size_t size);
-+
-+/**
-+ * execmem_free - free executable memory
-+ * @ptr: pointer to the memory that should be freed
-+ */
-+void execmem_free(void *ptr);
-+
-+DEFINE_FREE(execmem, void *, if (_T) execmem_free(_T));
-+
-+#ifdef CONFIG_MMU
-+/**
-+ * execmem_vmap - create virtual mapping for EXECMEM_MODULE_DATA memory
-+ * @size: size of the virtual mapping in bytes
-+ *
-+ * Maps virtually contiguous area in the range suitable for EXECMEM_MODULE_DATA.
-+ *
-+ * Return: the area descriptor on success or %NULL on failure.
-+ */
-+struct vm_struct *execmem_vmap(size_t size);
-+#endif
-+
-+/**
-+ * execmem_is_rox - check if execmem is read-only
-+ * @type - the execmem type to check
-+ *
-+ * Return: %true if the @type is read-only, %false if it's writable
-+ */
-+bool execmem_is_rox(enum execmem_type type);
-+
-+#if defined(CONFIG_EXECMEM) && !defined(CONFIG_ARCH_WANTS_EXECMEM_LATE)
-+void execmem_init(void);
-+#else
-+static inline void execmem_init(void) {}
-+#endif
-+
-+#endif /* _LINUX_EXECMEM_ALLOC_H */
-diff --git a/include/linux/export-internal.h b/include/linux/export-internal.h
-index c2b1d4fd5987..d445705ac13c 100644
--- a/include/linux/export-internal.h
-+++ b/include/linux/export-internal.h
-@@ -10,8 +10,63 @@
- #include <linux/compiler.h>
- #include <linux/types.h>
- 
-/* __used is needed to keep __crc_* for LTO */
-+#if defined(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)
-+/*
-+ * relative reference: this reduces the size by half on 64-bit architectures,
-+ * and eliminates the need for absolute relocations that require runtime
-+ * processing on relocatable kernels.
-+ */
-+#define __KSYM_ALIGN		".balign 4"
-+#define __KSYM_REF(sym)		".long " #sym "- ."
-+#elif defined(CONFIG_64BIT)
-+#define __KSYM_ALIGN		".balign 8"
-+#define __KSYM_REF(sym)		".quad " #sym
-+#else
-+#define __KSYM_ALIGN		".balign 4"
-+#define __KSYM_REF(sym)		".long " #sym
-+#endif
-+
-+/*
-+ * For every exported symbol, do the following:
-+ *
-+ * - Put the name of the symbol and namespace (empty string "" for none) in
-+ *   __ksymtab_strings.
-+ * - Place a struct kernel_symbol entry in the __ksymtab section.
-+ *
-+ * Note on .section use: we specify progbits since usage of the "M" (SHF_MERGE)
-+ * section flag requires it. Use '%progbits' instead of '@progbits' since the
-+ * former apparently works on all arches according to the binutils source.
-+ */
-+#define __KSYMTAB(name, sym, sec, ns)						\
-+	asm("	.section \"__ksymtab_strings\",\"aMS\",%progbits,1"	"\n"	\
-+	    "__kstrtab_" #name ":"					"\n"	\
-+	    "	.asciz \"" #name "\""					"\n"	\
-+	    "__kstrtabns_" #name ":"					"\n"	\
-+	    "	.asciz \"" ns "\""					"\n"	\
-+	    "	.previous"						"\n"	\
-+	    "	.section \"___ksymtab" sec "+" #name "\", \"a\""	"\n"	\
-+		__KSYM_ALIGN						"\n"	\
-+	    "__ksymtab_" #name ":"					"\n"	\
-+		__KSYM_REF(sym)						"\n"	\
-+		__KSYM_REF(__kstrtab_ ##name)				"\n"	\
-+		__KSYM_REF(__kstrtabns_ ##name)				"\n"	\
-+	    "	.previous"						"\n"	\
-+	)
-+
-+#if defined(CONFIG_PARISC) && defined(CONFIG_64BIT)
-+#define KSYM_FUNC(name)		P%name
-+#else
-+#define KSYM_FUNC(name)		name
-+#endif
-+
-+#define KSYMTAB_FUNC(name, sec, ns)	__KSYMTAB(name, KSYM_FUNC(name), sec, ns)
-+#define KSYMTAB_DATA(name, sec, ns)	__KSYMTAB(name, name, sec, ns)
-+
- #define SYMBOL_CRC(sym, crc, sec)   \
-	u32 __section("___kcrctab" sec "+" #sym) __used __crc_##sym = crc
-+	asm(".section \"___kcrctab" sec "+" #sym "\",\"a\""	"\n" \
-+	    ".balign 4"						"\n" \
-+	    "__crc_" #sym ":"					"\n" \
-+	    ".long " #crc					"\n" \
-+	    ".previous"						"\n")
- 
- #endif /* __LINUX_EXPORT_INTERNAL_H__ */
-diff --git a/include/linux/export.h b/include/linux/export.h
-index 3f31ced0d977..a686fd0ba406 100644
--- a/include/linux/export.h
-+++ b/include/linux/export.h
-@@ -2,18 +2,11 @@
- #ifndef _LINUX_EXPORT_H
- #define _LINUX_EXPORT_H
- 
-+#include <linux/compiler.h>
-+#include <linux/linkage.h>
- #include <linux/stringify.h>
- 
- /*
- * Export symbols from the kernel to modules.  Forked from module.h
- * to reduce the amount of pointless cruft we feed to gcc when only
- * exporting a simple symbol or two.
- *
- * Try not to add #includes here.  It slows compilation and makes kernel
- * hackers place grumpy comments in header files.
- */
-
-/*
-  * This comment block is used by fixdep. Please do not remove.
-  *
-  * When CONFIG_MODVERSIONS is changed from n to y, all source files having
-@@ -21,137 +14,83 @@
-  * side effect of the *.o build rule.
-  */
- 
-#ifndef __ASSEMBLY__
-#ifdef MODULE
-extern struct module __this_module;
-#define THIS_MODULE (&__this_module)
-+#ifdef CONFIG_64BIT
-+#define __EXPORT_SYMBOL_REF(sym)			\
-+	.balign 8				ASM_NL	\
-+	.quad sym
- #else
-#define THIS_MODULE ((struct module *)0)
-+#define __EXPORT_SYMBOL_REF(sym)			\
-+	.balign 4				ASM_NL	\
-+	.long sym
- #endif
- 
-#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
-#include <linux/compiler.h>
- /*
- * Emit the ksymtab entry as a pair of relative references: this reduces
- * the size by half on 64-bit architectures, and eliminates the need for
- * absolute relocations that require runtime processing on relocatable
- * kernels.
- */
-#define __KSYMTAB_ENTRY(sym, sec)					\
-	__ADDRESSABLE(sym)						\
-	asm("	.section \"___ksymtab" sec "+" #sym "\", \"a\"	\n"	\
-	    "	.balign	4					\n"	\
-	    "__ksymtab_" #sym ":				\n"	\
-	    "	.long	" #sym "- .				\n"	\
-	    "	.long	__kstrtab_" #sym "- .			\n"	\
-	    "	.long	__kstrtabns_" #sym "- .			\n"	\
-	    "	.previous					\n")
-
-struct kernel_symbol {
-	int value_offset;
-	int name_offset;
-	int namespace_offset;
-};
-#else
-#define __KSYMTAB_ENTRY(sym, sec)					\
-	static const struct kernel_symbol __ksymtab_##sym		\
-	__attribute__((section("___ksymtab" sec "+" #sym), used))	\
-	__aligned(sizeof(void *))					\
-	= { (unsigned long)&sym, __kstrtab_##sym, __kstrtabns_##sym }
-
-struct kernel_symbol {
-	unsigned long value;
-	const char *name;
-	const char *namespace;
-};
-#endif
-
-#ifdef __GENKSYMS__
-
-#define ___EXPORT_SYMBOL(sym, sec, ns)	__GENKSYMS_EXPORT_SYMBOL(sym)
-
-#else
-
-/*
- * For every exported symbol, do the following:
- *
- * - Put the name of the symbol and namespace (empty string "" for none) in
- *   __ksymtab_strings.
- * - Place a struct kernel_symbol entry in the __ksymtab section.
-+ * LLVM integrated assembler cam merge adjacent string literals (like
-+ * C and GNU-as) passed to '.ascii', but not to '.asciz' and chokes on:
-  *
- * note on .section use: we specify progbits since usage of the "M" (SHF_MERGE)
- * section flag requires it. Use '%progbits' instead of '@progbits' since the
- * former apparently works on all arches according to the binutils source.
-+ *   .asciz "MODULE_" "kvm" ;
-  */
-#define ___EXPORT_SYMBOL(sym, sec, ns)						\
-	extern typeof(sym) sym;							\
-	extern const char __kstrtab_##sym[];					\
-	extern const char __kstrtabns_##sym[];					\
-	asm("	.section \"__ksymtab_strings\",\"aMS\",%progbits,1	\n"	\
-	    "__kstrtab_" #sym ":					\n"	\
-	    "	.asciz 	\"" #sym "\"					\n"	\
-	    "__kstrtabns_" #sym ":					\n"	\
-	    "	.asciz 	\"" ns "\"					\n"	\
-	    "	.previous						\n");	\
-	__KSYMTAB_ENTRY(sym, sec)
-+#define ___EXPORT_SYMBOL(sym, license, ns...)		\
-+	.section ".export_symbol","a"		ASM_NL	\
-+	__export_symbol_##sym:			ASM_NL	\
-+		.asciz license			ASM_NL	\
-+		.ascii ns "\0"			ASM_NL	\
-+		__EXPORT_SYMBOL_REF(sym)	ASM_NL	\
-+	.previous
- 
-#endif
-
-#if !defined(CONFIG_MODULES) || defined(__DISABLE_EXPORTS)
-+#if defined(__DISABLE_EXPORTS)
- 
- /*
-  * Allow symbol exports to be disabled completely so that C code may
-  * be reused in other execution contexts such as the UEFI stub or the
-  * decompressor.
-  */
-#define __EXPORT_SYMBOL(sym, sec, ns)
-+#define __EXPORT_SYMBOL(sym, license, ns)
-+
-+#elif defined(__GENKSYMS__)
- 
-#elif defined(CONFIG_TRIM_UNUSED_KSYMS)
-+#define __EXPORT_SYMBOL(sym, license, ns)	__GENKSYMS_EXPORT_SYMBOL(sym)
- 
-#include <generated/autoksyms.h>
-+#elif defined(__ASSEMBLY__)
- 
-+#define __EXPORT_SYMBOL(sym, license, ns) \
-+	___EXPORT_SYMBOL(sym, license, ns)
-+
-+#else
-+
-+#ifdef CONFIG_GENDWARFKSYMS
- /*
- * For fine grained build dependencies, we want to tell the build system
- * about each possible exported symbol even if they're not actually exported.
- * We use a symbol pattern __ksym_marker_<symbol> that the build system filters
- * from the $(NM) output (see scripts/gen_ksymdeps.sh). These symbols are
- * discarded in the final link stage.
-+ * With CONFIG_GENDWARFKSYMS, ensure the compiler emits debugging
-+ * information for all exported symbols, including those defined in
-+ * different TUs, by adding a __gendwarfksyms_ptr_<symbol> pointer
-+ * that's discarded during the final link.
-  */
-#define __ksym_marker(sym)	\
-	static int __ksym_marker_##sym[0] __section(".discard.ksym") __used
-
-#define __EXPORT_SYMBOL(sym, sec, ns)					\
-	__ksym_marker(sym);						\
-	__cond_export_sym(sym, sec, ns, __is_defined(__KSYM_##sym))
-#define __cond_export_sym(sym, sec, ns, conf)				\
-	___cond_export_sym(sym, sec, ns, conf)
-#define ___cond_export_sym(sym, sec, ns, enabled)			\
-	__cond_export_sym_##enabled(sym, sec, ns)
-#define __cond_export_sym_1(sym, sec, ns) ___EXPORT_SYMBOL(sym, sec, ns)
-
-#ifdef __GENKSYMS__
-#define __cond_export_sym_0(sym, sec, ns) __GENKSYMS_EXPORT_SYMBOL(sym)
-+#define __GENDWARFKSYMS_EXPORT(sym)				\
-+	static typeof(sym) *__gendwarfksyms_ptr_##sym __used	\
-+		__section(".discard.gendwarfksyms") = &sym;
- #else
-#define __cond_export_sym_0(sym, sec, ns) /* nothing */
-+#define __GENDWARFKSYMS_EXPORT(sym)
- #endif
- 
-#else
-
-#define __EXPORT_SYMBOL(sym, sec, ns)	___EXPORT_SYMBOL(sym, sec, ns)
-+#define __EXPORT_SYMBOL(sym, license, ns)			\
-+	extern typeof(sym) sym;					\
-+	__ADDRESSABLE(sym)					\
-+	__GENDWARFKSYMS_EXPORT(sym)				\
-+	asm(__stringify(___EXPORT_SYMBOL(sym, license, ns)))
- 
-#endif /* CONFIG_MODULES */
-+#endif
- 
- #ifdef DEFAULT_SYMBOL_NAMESPACE
-#define _EXPORT_SYMBOL(sym, sec)	__EXPORT_SYMBOL(sym, sec, __stringify(DEFAULT_SYMBOL_NAMESPACE))
-+#define _EXPORT_SYMBOL(sym, license)	__EXPORT_SYMBOL(sym, license, DEFAULT_SYMBOL_NAMESPACE)
- #else
-#define _EXPORT_SYMBOL(sym, sec)	__EXPORT_SYMBOL(sym, sec, "")
-+#define _EXPORT_SYMBOL(sym, license)	__EXPORT_SYMBOL(sym, license, "")
- #endif
- 
- #define EXPORT_SYMBOL(sym)		_EXPORT_SYMBOL(sym, "")
-#define EXPORT_SYMBOL_GPL(sym)		_EXPORT_SYMBOL(sym, "_gpl")
-#define EXPORT_SYMBOL_NS(sym, ns)	__EXPORT_SYMBOL(sym, "", __stringify(ns))
-#define EXPORT_SYMBOL_NS_GPL(sym, ns)	__EXPORT_SYMBOL(sym, "_gpl", __stringify(ns))
-+#define EXPORT_SYMBOL_GPL(sym)		_EXPORT_SYMBOL(sym, "GPL")
-+#define EXPORT_SYMBOL_NS(sym, ns)	__EXPORT_SYMBOL(sym, "", ns)
-+#define EXPORT_SYMBOL_NS_GPL(sym, ns)	__EXPORT_SYMBOL(sym, "GPL", ns)
- 
-#endif /* !__ASSEMBLY__ */
-+#define EXPORT_SYMBOL_FOR_MODULES(sym, mods) __EXPORT_SYMBOL(sym, "GPL", "module:" mods)
- 
- #endif /* _LINUX_EXPORT_H */
-diff --git a/include/linux/exportfs.h b/include/linux/exportfs.h
-index fe848901fcc3..f0cf2714ec52 100644
--- a/include/linux/exportfs.h
-+++ b/include/linux/exportfs.h
-@@ -3,6 +3,7 @@
- #define LINUX_EXPORTFS_H 1
- 
- #include <linux/types.h>
-+#include <linux/path.h>
- 
- struct dentry;
- struct iattr;
-@@ -99,12 +100,35 @@ enum fid_type {
- 	FILEID_FAT_WITH_PARENT = 0x72,
- 
- 	/*
-+	 * 64 bit inode number, 32 bit generation number.
-+	 */
-+	FILEID_INO64_GEN = 0x81,
-+
-+	/*
-+	 * 64 bit inode number, 32 bit generation number,
-+	 * 64 bit parent inode number, 32 bit parent generation.
-+	 */
-+	FILEID_INO64_GEN_PARENT = 0x82,
-+
-+	/*
- 	 * 128 bit child FID (struct lu_fid)
- 	 * 128 bit parent FID (struct lu_fid)
- 	 */
- 	FILEID_LUSTRE = 0x97,
- 
- 	/*
-+	 * 64 bit inode number, 32 bit subvolume, 32 bit generation number:
-+	 */
-+	FILEID_BCACHEFS_WITHOUT_PARENT = 0xb1,
-+	FILEID_BCACHEFS_WITH_PARENT = 0xb2,
-+
-+	/*
-+	 *
-+	 * 64 bit namespace identifier, 32 bit namespace type, 32 bit inode number.
-+	 */
-+	FILEID_NSFS = 0xf1,
-+
-+	/*
- 	 * 64 bit unique kernfs id
- 	 */
- 	FILEID_KERNFS = 0xfe,
-@@ -123,7 +147,11 @@ struct fid {
- 			u32 parent_ino;
- 			u32 parent_gen;
- 		} i32;
- 		struct {
-+		struct {
-+			u64 ino;
-+			u32 gen;
-+		} __packed i64;
-+		struct {
-  			u32 block;
-  			u16 partref;
-  			u16 parent_partref;
-@@ -131,10 +159,38 @@ struct fid {
-  			u32 parent_block;
-  			u32 parent_generation;
-  		} udf;
-		__u32 raw[0];
-+		DECLARE_FLEX_ARRAY(__u32, raw);
- 	};
- };
- 
-+enum handle_to_path_flags {
-+	HANDLE_CHECK_PERMS   = (1 << 0),
-+	HANDLE_CHECK_SUBTREE = (1 << 1),
-+};
-+
-+struct handle_to_path_ctx {
-+	struct path root;
-+	enum handle_to_path_flags flags;
-+	unsigned int fh_flags;
-+};
-+
-+#define EXPORT_FH_CONNECTABLE	0x1 /* Encode file handle with parent */
-+#define EXPORT_FH_FID		0x2 /* File handle may be non-decodeable */
-+#define EXPORT_FH_DIR_ONLY	0x4 /* Only decode file handle for a directory */
-+
-+/*
-+ * Filesystems use only lower 8 bits of file_handle type for fid_type.
-+ * name_to_handle_at() uses upper 16 bits of type as user flags to be
-+ * interpreted by open_by_handle_at().
-+ */
-+#define FILEID_USER_FLAGS_MASK	0xffff0000
-+#define FILEID_USER_FLAGS(type) ((type) & FILEID_USER_FLAGS_MASK)
-+
-+/* Flags supported in encoded handle_type that is exported to user */
-+#define FILEID_IS_CONNECTABLE	0x10000
-+#define FILEID_IS_DIR		0x20000
-+#define FILEID_VALID_USER_FLAGS	(FILEID_IS_CONNECTABLE | FILEID_IS_DIR)
-+
- /**
-  * struct export_operations - for nfsd to communicate with file systems
-  * @encode_fh:      encode a file handle fragment from a dentry
-@@ -150,7 +206,7 @@ struct fid {
-  * encode_fh:
-  *    @encode_fh should store in the file handle fragment @fh (using at most
-  *    @max_len bytes) information that can be used by @decode_fh to recover the
- *    file referred to by the &struct dentry @de.  If the @connectable flag is
-+ *    file referred to by the &struct dentry @de.  If @flag has CONNECTABLE bit
-  *    set, the encode_fh() should store sufficient information so that a good
-  *    attempt can be made to find not only the file but also it's place in the
-  *    filesystem.   This typically means storing a reference to de->d_parent in
-@@ -180,18 +236,24 @@ struct fid {
-  *    directory.  The name should be stored in the @name (with the
-  *    understanding that it is already pointing to a %NAME_MAX+1 sized
-  *    buffer.   get_name() should return %0 on success, a negative error code
- *    or error.  @get_name will be called without @parent->i_mutex held.
-+ *    or error.  @get_name will be called without @parent->i_rwsem held.
-  *
-  * get_parent:
-  *    @get_parent should find the parent directory for the given @child which
-  *    is also a directory.  In the event that it cannot be found, or storage
-  *    space cannot be allocated, a %ERR_PTR should be returned.
-  *
-+ * permission:
-+ *    Allow filesystems to specify a custom permission function.
-+ *
-+ * open:
-+ *    Allow filesystems to specify a custom open function.
-+ *
-  * commit_metadata:
-  *    @commit_metadata should commit metadata changes to stable storage.
-  *
-  * Locking rules:
- *    get_parent is called with child->d_inode->i_mutex down
-+ *    get_parent is called with child->d_inode->i_rwsem down
-  *    get_name is not (which is possibly inconsistent)
-  */
- 
-@@ -213,7 +275,8 @@ struct export_operations {
- 			  bool write, u32 *device_generation);
- 	int (*commit_blocks)(struct inode *inode, struct iomap *iomaps,
- 			     int nr_iomaps, struct iattr *iattr);
-	u64 (*fetch_iversion)(struct inode *);
-+	int (*permission)(struct handle_to_path_ctx *ctx, unsigned int oflags);
-+	struct file * (*open)(const struct path *path, unsigned int oflags);
- #define	EXPORT_OP_NOWCC			(0x1) /* don't collect v3 wcc data */
- #define	EXPORT_OP_NOSUBTREECHK		(0x2) /* no subtree checking */
- #define	EXPORT_OP_CLOSE_BEFORE_UNLINK	(0x4) /* close files before unlink */
-@@ -221,16 +284,78 @@ struct export_operations {
- #define EXPORT_OP_NOATOMIC_ATTR		(0x10) /* Filesystem cannot supply
- 						  atomic attribute updates
- 						*/
-+#define EXPORT_OP_FLUSH_ON_CLOSE	(0x20) /* fs flushes file data on close */
-+#define EXPORT_OP_NOLOCKS		(0x40) /* no file locking support */
- 	unsigned long	flags;
- };
- 
-+/**
-+ * exportfs_cannot_lock() - check if export implements file locking
-+ * @export_ops:	the nfs export operations to check
-+ *
-+ * Returns true if the export does not support file locking.
-+ */
-+static inline bool
-+exportfs_cannot_lock(const struct export_operations *export_ops)
-+{
-+	return export_ops->flags & EXPORT_OP_NOLOCKS;
-+}
-+
- extern int exportfs_encode_inode_fh(struct inode *inode, struct fid *fid,
-				    int *max_len, struct inode *parent);
-+				    int *max_len, struct inode *parent,
-+				    int flags);
- extern int exportfs_encode_fh(struct dentry *dentry, struct fid *fid,
-	int *max_len, int connectable);
-+			      int *max_len, int flags);
-+
-+static inline bool exportfs_can_encode_fid(const struct export_operations *nop)
-+{
-+	return !nop || nop->encode_fh;
-+}
-+
-+static inline bool exportfs_can_decode_fh(const struct export_operations *nop)
-+{
-+	return nop && nop->fh_to_dentry;
-+}
-+
-+static inline bool exportfs_can_encode_fh(const struct export_operations *nop,
-+					  int fh_flags)
-+{
-+	/*
-+	 * If a non-decodeable file handle was requested, we only need to make
-+	 * sure that filesystem did not opt-out of encoding fid.
-+	 */
-+	if (fh_flags & EXPORT_FH_FID)
-+		return exportfs_can_encode_fid(nop);
-+
-+	/* Normal file handles cannot be created without export ops */
-+	if (!nop)
-+		return false;
-+
-+	/*
-+	 * If a connectable file handle was requested, we need to make sure that
-+	 * filesystem can also decode connected file handles.
-+	 */
-+	if ((fh_flags & EXPORT_FH_CONNECTABLE) && !nop->fh_to_parent)
-+		return false;
-+
-+	/*
-+	 * If a decodeable file handle was requested, we need to make sure that
-+	 * filesystem can also decode file handles.
-+	 */
-+	return exportfs_can_decode_fh(nop);
-+}
-+
-+static inline int exportfs_encode_fid(struct inode *inode, struct fid *fid,
-+				      int *max_len)
-+{
-+	return exportfs_encode_inode_fh(inode, fid, max_len, NULL,
-+					EXPORT_FH_FID);
-+}
-+
- extern struct dentry *exportfs_decode_fh_raw(struct vfsmount *mnt,
- 					     struct fid *fid, int fh_len,
- 					     int fileid_type,
-+					     unsigned int flags,
- 					     int (*acceptable)(void *, struct dentry *),
- 					     void *context);
- extern struct dentry *exportfs_decode_fh(struct vfsmount *mnt, struct fid *fid,
-@@ -240,10 +365,12 @@ extern struct dentry *exportfs_decode_fh(struct vfsmount *mnt, struct fid *fid,
- /*
-  * Generic helpers for filesystems.
-  */
-extern struct dentry *generic_fh_to_dentry(struct super_block *sb,
-+int generic_encode_ino32_fh(struct inode *inode, __u32 *fh, int *max_len,
-+			    struct inode *parent);
-+struct dentry *generic_fh_to_dentry(struct super_block *sb,
- 	struct fid *fid, int fh_len, int fh_type,
- 	struct inode *(*get_inode) (struct super_block *sb, u64 ino, u32 gen));
-extern struct dentry *generic_fh_to_parent(struct super_block *sb,
-+struct dentry *generic_fh_to_parent(struct super_block *sb,
- 	struct fid *fid, int fh_len, int fh_type,
- 	struct inode *(*get_inode) (struct super_block *sb, u64 ino, u32 gen));
- 
-diff --git a/include/linux/extcon.h b/include/linux/extcon.h
-index 3c45c3846fe9..e596a0abcb27 100644
--- a/include/linux/extcon.h
-+++ b/include/linux/extcon.h
-@@ -328,16 +328,4 @@ struct extcon_specific_cable_nb {
-        struct extcon_dev *edev;
-        unsigned long previous_value;
- };
-
-static inline int extcon_register_interest(struct extcon_specific_cable_nb *obj,
-				const char *extcon_name, const char *cable_name,
-				struct notifier_block *nb)
-{
-	return -EINVAL;
-}
-
-static inline int extcon_unregister_interest(struct extcon_specific_cable_nb *obj)
-{
-	return -EINVAL;
-}
- #endif /* __LINUX_EXTCON_H__ */
-diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h
-index d445150c5350..a7880787cad3 100644
--- a/include/linux/f2fs_fs.h
-+++ b/include/linux/f2fs_fs.h
-@@ -13,20 +13,23 @@
- 
- #define F2FS_SUPER_OFFSET		1024	/* byte-size offset */
- #define F2FS_MIN_LOG_SECTOR_SIZE	9	/* 9 bits for 512 bytes */
-#define F2FS_MAX_LOG_SECTOR_SIZE	12	/* 12 bits for 4096 bytes */
-#define F2FS_LOG_SECTORS_PER_BLOCK	3	/* log number for sector/blk */
-#define F2FS_BLKSIZE			4096	/* support only 4KB block */
-#define F2FS_BLKSIZE_BITS		12	/* bits for F2FS_BLKSIZE */
-+#define F2FS_MAX_LOG_SECTOR_SIZE	PAGE_SHIFT	/* Max is Block Size */
-+#define F2FS_LOG_SECTORS_PER_BLOCK	(PAGE_SHIFT - 9) /* log number for sector/blk */
-+#define F2FS_BLKSIZE			PAGE_SIZE /* support only block == page */
-+#define F2FS_BLKSIZE_BITS		PAGE_SHIFT /* bits for F2FS_BLKSIZE */
-+#define F2FS_SUM_BLKSIZE		4096	/* only support 4096 byte sum block */
- #define F2FS_MAX_EXTENSION		64	/* # of extension entries */
- #define F2FS_EXTENSION_LEN		8	/* max size of extension */
-#define F2FS_BLK_ALIGN(x)	(((x) + F2FS_BLKSIZE - 1) >> F2FS_BLKSIZE_BITS)
- 
- #define NULL_ADDR		((block_t)0)	/* used as block_t addresses */
- #define NEW_ADDR		((block_t)-1)	/* used as block_t addresses */
- #define COMPRESS_ADDR		((block_t)-2)	/* used as compressed data flag */
- 
-#define F2FS_BYTES_TO_BLK(bytes)	((bytes) >> F2FS_BLKSIZE_BITS)
-#define F2FS_BLK_TO_BYTES(blk)		((blk) << F2FS_BLKSIZE_BITS)
-+#define F2FS_BLKSIZE_MASK		(F2FS_BLKSIZE - 1)
-+#define F2FS_BYTES_TO_BLK(bytes)	((unsigned long long)(bytes) >> F2FS_BLKSIZE_BITS)
-+#define F2FS_BLK_TO_BYTES(blk)		((unsigned long long)(blk) << F2FS_BLKSIZE_BITS)
-+#define F2FS_BLK_END_BYTES(blk)		(F2FS_BLK_TO_BYTES(blk + 1) - 1)
-+#define F2FS_BLK_ALIGN(x)		(F2FS_BYTES_TO_BLK((x) + F2FS_BLKSIZE - 1))
- 
- /* 0, 1(node nid), 2(meta nid) are reserved node id */
- #define F2FS_RESERVED_NODE_NUM		3
-@@ -40,13 +43,6 @@
- 
- #define F2FS_ENC_UTF8_12_1	1
- 
-#define F2FS_IO_SIZE(sbi)	(1 << F2FS_OPTION(sbi).write_io_size_bits) /* Blocks */
-#define F2FS_IO_SIZE_KB(sbi)	(1 << (F2FS_OPTION(sbi).write_io_size_bits + 2)) /* KB */
-#define F2FS_IO_SIZE_BYTES(sbi)	(1 << (F2FS_OPTION(sbi).write_io_size_bits + 12)) /* B */
-#define F2FS_IO_SIZE_BITS(sbi)	(F2FS_OPTION(sbi).write_io_size_bits) /* power of 2 */
-#define F2FS_IO_SIZE_MASK(sbi)	(F2FS_IO_SIZE(sbi) - 1)
-#define F2FS_IO_ALIGNED(sbi)	(F2FS_IO_SIZE(sbi) > 1)
-
- /* This flag is used by node and meta inodes, and by recovery */
- #define GFP_F2FS_ZERO		(GFP_NOFS | __GFP_ZERO)
- 
-@@ -73,6 +69,47 @@ struct f2fs_device {
- 	__le32 total_segments;
- } __packed;
- 
-+/* reason of stop_checkpoint */
-+enum stop_cp_reason {
-+	STOP_CP_REASON_SHUTDOWN,
-+	STOP_CP_REASON_FAULT_INJECT,
-+	STOP_CP_REASON_META_PAGE,
-+	STOP_CP_REASON_WRITE_FAIL,
-+	STOP_CP_REASON_CORRUPTED_SUMMARY,
-+	STOP_CP_REASON_UPDATE_INODE,
-+	STOP_CP_REASON_FLUSH_FAIL,
-+	STOP_CP_REASON_NO_SEGMENT,
-+	STOP_CP_REASON_CORRUPTED_FREE_BITMAP,
-+	STOP_CP_REASON_CORRUPTED_NID,
-+	STOP_CP_REASON_MAX,
-+};
-+
-+#define	MAX_STOP_REASON			32
-+
-+/* detail reason for EFSCORRUPTED */
-+enum f2fs_error {
-+	ERROR_CORRUPTED_CLUSTER,
-+	ERROR_FAIL_DECOMPRESSION,
-+	ERROR_INVALID_BLKADDR,
-+	ERROR_CORRUPTED_DIRENT,
-+	ERROR_CORRUPTED_INODE,
-+	ERROR_INCONSISTENT_SUMMARY,
-+	ERROR_INCONSISTENT_FOOTER,
-+	ERROR_INCONSISTENT_SUM_TYPE,
-+	ERROR_CORRUPTED_JOURNAL,
-+	ERROR_INCONSISTENT_NODE_COUNT,
-+	ERROR_INCONSISTENT_BLOCK_COUNT,
-+	ERROR_INVALID_CURSEG,
-+	ERROR_INCONSISTENT_SIT,
-+	ERROR_CORRUPTED_VERITY_XATTR,
-+	ERROR_CORRUPTED_XATTR,
-+	ERROR_INVALID_NODE_REFERENCE,
-+	ERROR_INCONSISTENT_NAT,
-+	ERROR_MAX,
-+};
-+
-+#define MAX_F2FS_ERRORS			16
-+
- struct f2fs_super_block {
- 	__le32 magic;			/* Magic Number */
- 	__le16 major_ver;		/* Major Version */
-@@ -116,7 +153,9 @@ struct f2fs_super_block {
- 	__u8 hot_ext_count;		/* # of hot file extension */
- 	__le16  s_encoding;		/* Filename charset encoding */
- 	__le16  s_encoding_flags;	/* Filename charset encoding flags */
-	__u8 reserved[306];		/* valid reserved region */
-+	__u8 s_stop_reason[MAX_STOP_REASON];	/* stop checkpoint reason */
-+	__u8 s_errors[MAX_F2FS_ERRORS];		/* reason of image corrupts */
-+	__u8 reserved[258];		/* valid reserved region */
- 	__le32 crc;			/* checksum of superblock */
- } __packed;
- 
-@@ -172,14 +211,14 @@ struct f2fs_checkpoint {
- 	unsigned char sit_nat_version_bitmap[];
- } __packed;
- 
-#define CP_CHKSUM_OFFSET	4092	/* default chksum offset in checkpoint */
-+#define CP_CHKSUM_OFFSET	(F2FS_BLKSIZE - sizeof(__le32))	/* default chksum offset in checkpoint */
- #define CP_MIN_CHKSUM_OFFSET						\
- 	(offsetof(struct f2fs_checkpoint, sit_nat_version_bitmap))
- 
- /*
-  * For orphan inode management
-  */
-#define F2FS_ORPHANS_PER_BLOCK	1020
-+#define F2FS_ORPHANS_PER_BLOCK	((F2FS_BLKSIZE - 4 * sizeof(__le32)) / sizeof(__le32))
- 
- #define GET_ORPHAN_BLOCKS(n)	(((n) + F2FS_ORPHANS_PER_BLOCK - 1) / \
- 					F2FS_ORPHANS_PER_BLOCK)
-@@ -205,17 +244,33 @@ struct f2fs_extent {
- #define F2FS_NAME_LEN		255
- /* 200 bytes for inline xattrs by default */
- #define DEFAULT_INLINE_XATTR_ADDRS	50
-#define DEF_ADDRS_PER_INODE	923	/* Address Pointers in an Inode */
-+
-+#define OFFSET_OF_END_OF_I_EXT		360
-+#define SIZE_OF_I_NID			20
-+
-+struct node_footer {
-+	__le32 nid;		/* node id */
-+	__le32 ino;		/* inode number */
-+	__le32 flag;		/* include cold/fsync/dentry marks and offset */
-+	__le64 cp_ver;		/* checkpoint version */
-+	__le32 next_blkaddr;	/* next node page block address */
-+} __packed;
-+
-+/* Address Pointers in an Inode */
-+#define DEF_ADDRS_PER_INODE	((F2FS_BLKSIZE - OFFSET_OF_END_OF_I_EXT	\
-+					- SIZE_OF_I_NID	\
-+					- sizeof(struct node_footer)) / sizeof(__le32))
- #define CUR_ADDRS_PER_INODE(inode)	(DEF_ADDRS_PER_INODE - \
- 					get_extra_isize(inode))
- #define DEF_NIDS_PER_INODE	5	/* Node IDs in an Inode */
-#define ADDRS_PER_INODE(inode)	addrs_per_inode(inode)
-#define DEF_ADDRS_PER_BLOCK	1018	/* Address Pointers in a Direct Block */
-#define ADDRS_PER_BLOCK(inode)	addrs_per_block(inode)
-#define NIDS_PER_BLOCK		1018	/* Node IDs in an Indirect Block */
-+#define ADDRS_PER_INODE(inode)	addrs_per_page(inode, true)
-+/* Address Pointers in a Direct Block */
-+#define DEF_ADDRS_PER_BLOCK	((F2FS_BLKSIZE - sizeof(struct node_footer)) / sizeof(__le32))
-+#define ADDRS_PER_BLOCK(inode)	addrs_per_page(inode, false)
-+/* Node IDs in an Indirect Block */
-+#define NIDS_PER_BLOCK		((F2FS_BLKSIZE - sizeof(struct node_footer)) / sizeof(__le32))
- 
-#define ADDRS_PER_PAGE(page, inode)	\
-	(IS_INODE(page) ? ADDRS_PER_INODE(inode) : ADDRS_PER_BLOCK(inode))
-+#define ADDRS_PER_PAGE(folio, inode)	(addrs_per_page(inode, IS_INODE(folio)))
- 
- #define	NODE_DIR1_BLOCK		(DEF_ADDRS_PER_INODE + 1)
- #define	NODE_DIR2_BLOCK		(DEF_ADDRS_PER_INODE + 2)
-@@ -227,7 +282,7 @@ struct f2fs_extent {
- #define F2FS_INLINE_DATA	0x02	/* file inline data flag */
- #define F2FS_INLINE_DENTRY	0x04	/* file inline dentry flag */
- #define F2FS_DATA_EXIST		0x08	/* file inline data exist flag */
-#define F2FS_INLINE_DOTS	0x10	/* file having implicit dot dentries */
-+#define F2FS_INLINE_DOTS	0x10	/* file having implicit dot dentries (obsolete) */
- #define F2FS_EXTRA_ATTR		0x20	/* file having extra attribute */
- #define F2FS_PIN_FILE		0x40	/* file should not be gced */
- #define F2FS_COMPRESS_RELEASED	0x80	/* file released compressed blocks */
-@@ -277,7 +332,7 @@ struct f2fs_inode {
- 			__u8 i_log_cluster_size;	/* log of cluster size */
- 			__le16 i_compress_flag;		/* compress flag */
- 						/* 0 bit: chksum flag
-						 * [10,15] bits: compress level
-+						 * [8,15] bits: compress level
- 						 */
- 			__le32 i_extra_end[0];	/* for attribute size calculation */
- 		} __packed;
-@@ -302,15 +357,7 @@ enum {
- 	OFFSET_BIT_SHIFT
- };
- 
-#define OFFSET_BIT_MASK		(0x07)	/* (0x01 << OFFSET_BIT_SHIFT) - 1 */
-
-struct node_footer {
-	__le32 nid;		/* node id */
-	__le32 ino;		/* inode number */
-	__le32 flag;		/* include cold/fsync/dentry marks and offset */
-	__le64 cp_ver;		/* checkpoint version */
-	__le32 next_blkaddr;	/* next node page block address */
-} __packed;
-+#define OFFSET_BIT_MASK		GENMASK(OFFSET_BIT_SHIFT - 1, 0)
- 
- struct f2fs_node {
- 	/* can be one of three types: inode, direct, and indirect types */
-@@ -325,7 +372,7 @@ struct f2fs_node {
- /*
-  * For NAT entries
-  */
-#define NAT_ENTRY_PER_BLOCK (PAGE_SIZE / sizeof(struct f2fs_nat_entry))
-+#define NAT_ENTRY_PER_BLOCK (F2FS_BLKSIZE / sizeof(struct f2fs_nat_entry))
- 
- struct f2fs_nat_entry {
- 	__u8 version;		/* latest version of cached nat entry */
-@@ -340,16 +387,18 @@ struct f2fs_nat_block {
- /*
-  * For SIT entries
-  *
- * Each segment is 2MB in size by default so that a bitmap for validity of
- * there-in blocks should occupy 64 bytes, 512 bits.
-+ * A validity bitmap of 64 bytes covers 512 blocks of area. For a 4K page size,
-+ * this results in a segment size of 2MB. For 16k pages, the default segment size
-+ * is 8MB.
-  * Not allow to change this.
-  */
- #define SIT_VBLOCK_MAP_SIZE 64
-#define SIT_ENTRY_PER_BLOCK (PAGE_SIZE / sizeof(struct f2fs_sit_entry))
-+#define SIT_ENTRY_PER_BLOCK (F2FS_BLKSIZE / sizeof(struct f2fs_sit_entry))
- 
- /*
-  * F2FS uses 4 bytes to represent block address. As a result, supported size of
- * disk is 16 TB and it equals to 16 * 1024 * 1024 / 2 segments.
-+ * disk is 16 TB for a 4K page size and 64 TB for a 16K page size and it equals
-+ * to 16 * 1024 * 1024 / 2 segments.
-  */
- #define F2FS_MAX_SEGMENT       ((16 * 1024 * 1024) / 2)
- 
-@@ -379,8 +428,10 @@ struct f2fs_sit_block {
- /*
-  * For segment summary
-  *
- * One summary block contains exactly 512 summary entries, which represents
- * exactly 2MB segment by default. Not allow to change the basic units.
-+ * One summary block with 4KB size contains exactly 512 summary entries, which
-+ * represents exactly one segment with 2MB size.
-+ * Similarly, in the case of block with 16KB size, it represents one segment with 8MB size.
-+ * Not allow to change the basic units.
-  *
-  * NOTE: For initializing fields, you must use set_summary
-  *
-@@ -391,12 +442,12 @@ struct f2fs_sit_block {
-  * from node's page's beginning to get a data block address.
-  * ex) data_blkaddr = (block_t)(nodepage_start_address + ofs_in_node)
-  */
-#define ENTRIES_IN_SUM		512
-#define	SUMMARY_SIZE		(7)	/* sizeof(struct summary) */
-+#define ENTRIES_IN_SUM		(F2FS_SUM_BLKSIZE / 8)
-+#define	SUMMARY_SIZE		(7)	/* sizeof(struct f2fs_summary) */
- #define	SUM_FOOTER_SIZE		(5)	/* sizeof(struct summary_footer) */
- #define SUM_ENTRY_SIZE		(SUMMARY_SIZE * ENTRIES_IN_SUM)
- 
-/* a summary entry for a 4KB-sized block in a segment */
-+/* a summary entry for a block in a segment */
- struct f2fs_summary {
- 	__le32 nid;		/* parent node id */
- 	union {
-@@ -417,7 +468,7 @@ struct summary_footer {
- 	__le32 check_sum;		/* summary checksum */
- } __packed;
- 
-#define SUM_JOURNAL_SIZE	(F2FS_BLKSIZE - SUM_FOOTER_SIZE -\
-+#define SUM_JOURNAL_SIZE	(F2FS_SUM_BLKSIZE - SUM_FOOTER_SIZE -\
- 				SUM_ENTRY_SIZE)
- #define NAT_JOURNAL_ENTRIES	((SUM_JOURNAL_SIZE - 2) /\
- 				sizeof(struct nat_journal_entry))
-@@ -480,7 +531,7 @@ struct f2fs_journal {
- 	};
- } __packed;
- 
-/* 4KB-sized summary block structure */
-+/* Block-sized summary block structure */
- struct f2fs_summary_block {
- 	struct f2fs_summary entries[ENTRIES_IN_SUM];
- 	struct f2fs_journal journal;
-@@ -507,10 +558,11 @@ typedef __le32	f2fs_hash_t;
- #define MAX_DIR_HASH_DEPTH	63
- 
- /* MAX buckets in one level of dir */
-#define MAX_DIR_BUCKETS		(1 << ((MAX_DIR_HASH_DEPTH / 2) - 1))
-+#define MAX_DIR_BUCKETS		BIT((MAX_DIR_HASH_DEPTH / 2) - 1)
- 
- /*
-  * space utilization of regular dentry and inline dentry (w/o extra reservation)
-+ * when block size is 4KB.
-  *		regular dentry		inline dentry (def)	inline dentry (min)
-  * bitmap	1 * 27 = 27		1 * 23 = 23		1 * 1 = 1
-  * reserved	1 * 3 = 3		1 * 7 = 7		1 * 1 = 1
-@@ -521,11 +573,14 @@ typedef __le32	f2fs_hash_t;
-  * Note: there are more reserved space in inline dentry than in regular
-  * dentry, when converting inline dentry we should handle this carefully.
-  */
-#define NR_DENTRY_IN_BLOCK	214	/* the number of dentry in a block */
-+
-+/* the number of dentry in a block */
-+#define NR_DENTRY_IN_BLOCK	((BITS_PER_BYTE * F2FS_BLKSIZE) / \
-+					((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * BITS_PER_BYTE + 1))
- #define SIZE_OF_DIR_ENTRY	11	/* by byte */
- #define SIZE_OF_DENTRY_BITMAP	((NR_DENTRY_IN_BLOCK + BITS_PER_BYTE - 1) / \
- 					BITS_PER_BYTE)
-#define SIZE_OF_RESERVED	(PAGE_SIZE - ((SIZE_OF_DIR_ENTRY + \
-+#define SIZE_OF_RESERVED	(F2FS_BLKSIZE - ((SIZE_OF_DIR_ENTRY + \
- 				F2FS_SLOT_LEN) * \
- 				NR_DENTRY_IN_BLOCK + SIZE_OF_DENTRY_BITMAP))
- #define MIN_INLINE_DENTRY_SIZE		40	/* just include '.' and '..' entries */
-@@ -538,7 +593,7 @@ struct f2fs_dir_entry {
- 	__u8 file_type;		/* file type */
- } __packed;
- 
-/* 4KB-sized directory entry block */
-+/* Block-sized directory entry block */
- struct f2fs_dentry_block {
- 	/* validity bitmap for directory entries in each block */
- 	__u8 dentry_bitmap[SIZE_OF_DENTRY_BITMAP];
-@@ -547,21 +602,6 @@ struct f2fs_dentry_block {
- 	__u8 filename[NR_DENTRY_IN_BLOCK][F2FS_SLOT_LEN];
- } __packed;
- 
-/* file types used in inode_info->flags */
-enum {
-	F2FS_FT_UNKNOWN,
-	F2FS_FT_REG_FILE,
-	F2FS_FT_DIR,
-	F2FS_FT_CHRDEV,
-	F2FS_FT_BLKDEV,
-	F2FS_FT_FIFO,
-	F2FS_FT_SOCK,
-	F2FS_FT_SYMLINK,
-	F2FS_FT_MAX
-};
-
-#define S_SHIFT 12
-
- #define	F2FS_DEF_PROJID		0	/* default project ID */
- 
- #endif  /* _LINUX_F2FS_FS_H */
-diff --git a/include/linux/falloc.h b/include/linux/falloc.h
-index f3f0b97b1675..7c38c6b76b60 100644
--- a/include/linux/falloc.h
-+++ b/include/linux/falloc.h
-@@ -25,12 +25,19 @@ struct space_resv {
- #define FS_IOC_UNRESVSP64	_IOW('X', 43, struct space_resv)
- #define FS_IOC_ZERO_RANGE	_IOW('X', 57, struct space_resv)
- 
-#define	FALLOC_FL_SUPPORTED_MASK	(FALLOC_FL_KEEP_SIZE |		\
-					 FALLOC_FL_PUNCH_HOLE |		\
-					 FALLOC_FL_COLLAPSE_RANGE |	\
-					 FALLOC_FL_ZERO_RANGE |		\
-					 FALLOC_FL_INSERT_RANGE |	\
-					 FALLOC_FL_UNSHARE_RANGE)
-+/*
-+ * Mask of all supported fallocate modes.  Only one can be set at a time.
-+ *
-+ * In addition to the mode bit, the mode argument can also encode flags.
-+ * FALLOC_FL_KEEP_SIZE is the only supported flag so far.
-+ */
-+#define FALLOC_FL_MODE_MASK	(FALLOC_FL_ALLOCATE_RANGE |	\
-+				 FALLOC_FL_PUNCH_HOLE |		\
-+				 FALLOC_FL_COLLAPSE_RANGE |	\
-+				 FALLOC_FL_ZERO_RANGE |		\
-+				 FALLOC_FL_INSERT_RANGE |	\
-+				 FALLOC_FL_UNSHARE_RANGE |	\
-+				 FALLOC_FL_WRITE_ZEROES)
- 
- /* on ia32 l_start is on a 32-bit boundary */
- #if defined(CONFIG_X86_64)
-diff --git a/include/linux/fanotify.h b/include/linux/fanotify.h
-index 8ad743def6f3..879cff5eccd4 100644
--- a/include/linux/fanotify.h
-+++ b/include/linux/fanotify.h
-@@ -25,7 +25,7 @@
- 
- #define FANOTIFY_FID_BITS	(FAN_REPORT_DFID_NAME_TARGET)
- 
-#define FANOTIFY_INFO_MODES	(FANOTIFY_FID_BITS | FAN_REPORT_PIDFD)
-+#define FANOTIFY_INFO_MODES	(FANOTIFY_FID_BITS | FAN_REPORT_PIDFD | FAN_REPORT_MNT)
- 
- /*
-  * fanotify_init() flags that require CAP_SYS_ADMIN.
-@@ -36,6 +36,7 @@
- #define FANOTIFY_ADMIN_INIT_FLAGS	(FANOTIFY_PERM_CLASSES | \
- 					 FAN_REPORT_TID | \
- 					 FAN_REPORT_PIDFD | \
-+					 FAN_REPORT_FD_ERROR | \
- 					 FAN_UNLIMITED_QUEUE | \
- 					 FAN_UNLIMITED_MARKS)
- 
-@@ -46,7 +47,7 @@
-  * so one of the flags for reporting file handles is required.
-  */
- #define FANOTIFY_USER_INIT_FLAGS	(FAN_CLASS_NOTIF | \
-					 FANOTIFY_FID_BITS | \
-+					 FANOTIFY_FID_BITS | FAN_REPORT_MNT | \
- 					 FAN_CLOEXEC | FAN_NONBLOCK)
- 
- #define FANOTIFY_INIT_FLAGS	(FANOTIFY_ADMIN_INIT_FLAGS | \
-@@ -57,7 +58,7 @@
- #define FANOTIFY_INTERNAL_GROUP_FLAGS	(FANOTIFY_UNPRIV)
- 
- #define FANOTIFY_MARK_TYPE_BITS	(FAN_MARK_INODE | FAN_MARK_MOUNT | \
-				 FAN_MARK_FILESYSTEM)
-+				 FAN_MARK_FILESYSTEM | FAN_MARK_MNTNS)
- 
- #define FANOTIFY_MARK_CMD_BITS	(FAN_MARK_ADD | FAN_MARK_REMOVE | \
- 				 FAN_MARK_FLUSH)
-@@ -88,6 +89,16 @@
- #define FANOTIFY_DIRENT_EVENTS	(FAN_MOVE | FAN_CREATE | FAN_DELETE | \
- 				 FAN_RENAME)
- 
-+/* Content events can be used to inspect file content */
-+#define FANOTIFY_CONTENT_PERM_EVENTS (FAN_OPEN_PERM | FAN_OPEN_EXEC_PERM | \
-+				      FAN_ACCESS_PERM)
-+/* Pre-content events can be used to fill file content */
-+#define FANOTIFY_PRE_CONTENT_EVENTS  (FAN_PRE_ACCESS)
-+
-+/* Events that require a permission response from user */
-+#define FANOTIFY_PERM_EVENTS	(FANOTIFY_CONTENT_PERM_EVENTS | \
-+				 FANOTIFY_PRE_CONTENT_EVENTS)
-+
- /* Events that can be reported with event->fd */
- #define FANOTIFY_FD_EVENTS (FANOTIFY_PATH_EVENTS | FANOTIFY_PERM_EVENTS)
- 
-@@ -98,14 +109,13 @@
- /* Events that can only be reported with data type FSNOTIFY_EVENT_ERROR */
- #define FANOTIFY_ERROR_EVENTS	(FAN_FS_ERROR)
- 
-+#define FANOTIFY_MOUNT_EVENTS	(FAN_MNT_ATTACH | FAN_MNT_DETACH)
-+
- /* Events that user can request to be notified on */
- #define FANOTIFY_EVENTS		(FANOTIFY_PATH_EVENTS | \
- 				 FANOTIFY_INODE_EVENTS | \
-				 FANOTIFY_ERROR_EVENTS)
-
-/* Events that require a permission response from user */
-#define FANOTIFY_PERM_EVENTS	(FAN_OPEN_PERM | FAN_ACCESS_PERM | \
-				 FAN_OPEN_EXEC_PERM)
-+				 FANOTIFY_ERROR_EVENTS | \
-+				 FANOTIFY_MOUNT_EVENTS)
- 
- /* Extra flags that may be reported with event or control handling of events */
- #define FANOTIFY_EVENT_FLAGS	(FAN_EVENT_ON_CHILD | FAN_ONDIR)
-@@ -122,6 +132,13 @@
- #define ALL_FANOTIFY_EVENT_BITS		(FANOTIFY_OUTGOING_EVENTS | \
- 					 FANOTIFY_EVENT_FLAGS)
- 
-+/* These masks check for invalid bits in permission responses. */
-+#define FANOTIFY_RESPONSE_ACCESS (FAN_ALLOW | FAN_DENY)
-+#define FANOTIFY_RESPONSE_FLAGS (FAN_AUDIT | FAN_INFO)
-+#define FANOTIFY_RESPONSE_VALID_MASK \
-+	(FANOTIFY_RESPONSE_ACCESS | FANOTIFY_RESPONSE_FLAGS | \
-+	 (FAN_ERRNO_MASK << FAN_ERRNO_SHIFT))
-+
- /* Do not use these old uapi constants internally */
- #undef FAN_ALL_CLASS_BITS
- #undef FAN_ALL_INIT_FLAGS
-diff --git a/include/linux/fault-inject.h b/include/linux/fault-inject.h
-index 9f6e25467844..58fd14c82270 100644
--- a/include/linux/fault-inject.h
-+++ b/include/linux/fault-inject.h
-@@ -2,12 +2,21 @@
- #ifndef _LINUX_FAULT_INJECT_H
- #define _LINUX_FAULT_INJECT_H
- 
-+#include <linux/err.h>
-+#include <linux/types.h>
-+
-+struct dentry;
-+struct kmem_cache;
-+
-+enum fault_flags {
-+	FAULT_NOWARN =	1 << 0,
-+};
-+
- #ifdef CONFIG_FAULT_INJECTION
- 
-#include <linux/types.h>
-#include <linux/debugfs.h>
-#include <linux/ratelimit.h>
- #include <linux/atomic.h>
-+#include <linux/configfs.h>
-+#include <linux/ratelimit.h>
- 
- /*
-  * For explanation of the elements of this struct, see
-@@ -20,7 +29,6 @@ struct fault_attr {
- 	atomic_t space;
- 	unsigned long verbose;
- 	bool task_filter;
-	bool no_warn;
- 	unsigned long stacktrace_depth;
- 	unsigned long require_start;
- 	unsigned long require_end;
-@@ -40,13 +48,35 @@ struct fault_attr {
- 		.ratelimit_state = RATELIMIT_STATE_INIT_DISABLED,	\
- 		.verbose = 2,						\
- 		.dname = NULL,						\
-		.no_warn = false,					\
- 	}
- 
- #define DECLARE_FAULT_ATTR(name) struct fault_attr name = FAULT_ATTR_INITIALIZER
- int setup_fault_attr(struct fault_attr *attr, char *str);
-+bool should_fail_ex(struct fault_attr *attr, ssize_t size, int flags);
- bool should_fail(struct fault_attr *attr, ssize_t size);
- 
-+#else /* CONFIG_FAULT_INJECTION */
-+
-+struct fault_attr {
-+};
-+
-+#define DECLARE_FAULT_ATTR(name) struct fault_attr name = {}
-+
-+static inline int setup_fault_attr(struct fault_attr *attr, char *str)
-+{
-+	return 0; /* Note: 0 means error for __setup() handlers! */
-+}
-+static inline bool should_fail_ex(struct fault_attr *attr, ssize_t size, int flags)
-+{
-+	return false;
-+}
-+static inline bool should_fail(struct fault_attr *attr, ssize_t size)
-+{
-+	return false;
-+}
-+
-+#endif /* CONFIG_FAULT_INJECTION */
-+
- #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
- 
- struct dentry *fault_create_debugfs_attr(const char *name,
-@@ -62,17 +92,40 @@ static inline struct dentry *fault_create_debugfs_attr(const char *name,
- 
- #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
- 
-#endif /* CONFIG_FAULT_INJECTION */
-+#ifdef CONFIG_FAULT_INJECTION_CONFIGFS
- 
-struct kmem_cache;
-+struct fault_config {
-+	struct fault_attr attr;
-+	struct config_group group;
-+};
-+
-+void fault_config_init(struct fault_config *config, const char *name);
-+
-+#else /* CONFIG_FAULT_INJECTION_CONFIGFS */
-+
-+struct fault_config {
-+};
- 
-+static inline void fault_config_init(struct fault_config *config,
-+			const char *name)
-+{
-+}
-+
-+#endif /* CONFIG_FAULT_INJECTION_CONFIGFS */
-+
-+#ifdef CONFIG_FAIL_PAGE_ALLOC
- bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order);
-+#else
-+static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
-+{
-+	return false;
-+}
-+#endif /* CONFIG_FAIL_PAGE_ALLOC */
- 
-int should_failslab(struct kmem_cache *s, gfp_t gfpflags);
- #ifdef CONFIG_FAILSLAB
-extern bool __should_failslab(struct kmem_cache *s, gfp_t gfpflags);
-+int should_failslab(struct kmem_cache *s, gfp_t gfpflags);
- #else
-static inline bool __should_failslab(struct kmem_cache *s, gfp_t gfpflags)
-+static inline int should_failslab(struct kmem_cache *s, gfp_t gfpflags)
- {
- 	return false;
- }
-diff --git a/include/linux/fb.h b/include/linux/fb.h
-index 07fcd0e56682..05cc251035da 100644
--- a/include/linux/fb.h
-+++ b/include/linux/fb.h
-@@ -2,27 +2,31 @@
- #ifndef _LINUX_FB_H
- #define _LINUX_FB_H
- 
-#include <linux/refcount.h>
-#include <linux/kgdb.h>
- #include <uapi/linux/fb.h>
- 
- #define FBIO_CURSOR            _IOWR('F', 0x08, struct fb_cursor_user)
- 
-#include <linux/fs.h>
-#include <linux/init.h>
-+#include <linux/mutex.h>
-+#include <linux/printk.h>
-+#include <linux/refcount.h>
-+#include <linux/types.h>
- #include <linux/workqueue.h>
-#include <linux/notifier.h>
-#include <linux/list.h>
-#include <linux/backlight.h>
-#include <linux/slab.h>
-#include <asm/io.h>
- 
-struct vm_area_struct;
-struct fb_info;
-+#include <asm/video.h>
-+
-+struct backlight_device;
- struct device;
-+struct device_node;
-+struct fb_info;
- struct file;
-+struct i2c_adapter;
-+struct inode;
-+struct lcd_device;
-+struct module;
-+struct notifier_block;
-+struct page;
- struct videomode;
-struct device_node;
-+struct vm_area_struct;
- 
- /* Definitions below are used in the parsed monitor specs */
- #define FB_DPMS_ACTIVE_OFF	1
-@@ -125,26 +129,24 @@ struct fb_cursor_user {
-  * Register/unregister for framebuffer events
-  */
- 
-/*	The resolution of the passed in fb_info about to change */
-#define FB_EVENT_MODE_CHANGE		0x01
-
- #ifdef CONFIG_GUMSTIX_AM200EPD
- /* only used by mach-pxa/am200epd.c */
- #define FB_EVENT_FB_REGISTERED          0x05
- #define FB_EVENT_FB_UNREGISTERED        0x06
- #endif
- 
-/*      A display blank is requested       */
-#define FB_EVENT_BLANK                  0x09
-
- struct fb_event {
- 	struct fb_info *info;
- 	void *data;
- };
- 
-+/*	Enough for the VT console needs, see its max_font_width/height */
-+#define FB_MAX_BLIT_WIDTH	64
-+#define FB_MAX_BLIT_HEIGHT	128
-+
- struct fb_blit_caps {
-	u32 x;
-	u32 y;
-+	DECLARE_BITMAP(x, FB_MAX_BLIT_WIDTH);
-+	DECLARE_BITMAP(y, FB_MAX_BLIT_HEIGHT);
- 	u32 len;
- 	u32 flags;
- };
-@@ -191,10 +193,12 @@ struct fb_pixmap {
- 	u32 scan_align;		/* alignment per scanline		*/
- 	u32 access_align;	/* alignment per read/write (bits)	*/
- 	u32 flags;		/* see FB_PIXMAP_*			*/
-	u32 blit_x;             /* supported bit block dimensions (1-32)*/
-	u32 blit_y;             /* Format: blit_x = 1 << (width - 1)    */
-	                        /*         blit_y = 1 << (height - 1)   */
-	                        /* if 0, will be set to 0xffffffff (all)*/
-+				/* supported bit block dimensions	*/
-+				/* Format: test_bit(width - 1, blit_x)	*/
-+				/*	   test_bit(height - 1, blit_y)	*/
-+				/* if zero, will be set to full (all)	*/
-+	DECLARE_BITMAP(blit_x, FB_MAX_BLIT_WIDTH);
-+	DECLARE_BITMAP(blit_y, FB_MAX_BLIT_HEIGHT);
- 	/* access methods */
- 	void (*writeio)(struct fb_info *info, void __iomem *dst, void *src, unsigned int size);
- 	void (*readio) (struct fb_info *info, void *dst, void __iomem *src, unsigned int size);
-@@ -212,10 +216,12 @@ struct fb_deferred_io {
- 	/* delay between mkwrite and deferred handler */
- 	unsigned long delay;
- 	bool sort_pagereflist; /* sort pagelist by offset */
-+	int open_count; /* number of opened files; protected by fb_info lock */
- 	struct mutex lock; /* mutex that protects the pageref list */
- 	struct list_head pagereflist; /* list of pagerefs for touched pages */
-+	struct address_space *mapping; /* page cache object for fb device */
- 	/* callback */
-	void (*first_io)(struct fb_info *info);
-+	struct page *(*get_page)(struct fb_info *info, unsigned long offset);
- 	void (*deferred_io)(struct fb_info *info, struct list_head *pagelist);
- };
- #endif
-@@ -382,7 +388,6 @@ struct fb_tile_ops {
- #endif /* CONFIG_FB_TILEBLITTING */
- 
- /* FBINFO_* = fb_info.flags bit flags */
-#define FBINFO_DEFAULT		0
- #define FBINFO_HWACCEL_DISABLED	0x0002
- 	/* When FBINFO_HWACCEL_DISABLED is set:
- 	 *  Hardware acceleration is turned off.  Software implementations
-@@ -423,8 +428,6 @@ struct fb_tile_ops {
-  */
- #define FBINFO_MISC_ALWAYS_SETPAR   0x40000
- 
-/* where the fb is a firmware driver, and can be replaced with a proper one */
-#define FBINFO_MISC_FIRMWARE        0x80000
- /*
-  * Host and GPU endianness differ.
-  */
-@@ -463,6 +466,8 @@ struct fb_info {
- 	struct list_head modelist;      /* mode list */
- 	struct fb_videomode *mode;	/* current mode */
- 
-+	int blank; /* current blanking; see FB_BLANK_ constants */
-+
- #if IS_ENABLED(CONFIG_FB_BACKLIGHT)
- 	/* assigned backlight device */
- 	/* set before framebuffer registration,
-@@ -473,6 +478,13 @@ struct fb_info {
- 	struct mutex bl_curve_mutex;
- 	u8 bl_curve[FB_BACKLIGHT_LEVELS];
- #endif
-+
-+	/*
-+	 * Assigned LCD device; set before framebuffer
-+	 * registration, remove after unregister
-+	 */
-+	struct lcd_device *lcd_dev;
-+
- #ifdef CONFIG_FB_DEFERRED_IO
- 	struct delayed_work deferred_work;
- 	unsigned long npagerefs;
-@@ -482,7 +494,9 @@ struct fb_info {
- 
- 	const struct fb_ops *fbops;
- 	struct device *device;		/* This is the parent */
-+#if defined(CONFIG_FB_DEVICE)
- 	struct device *dev;		/* This is this fb device */
-+#endif
- 	int class_flag;                    /* private sysfs flags */
- #ifdef CONFIG_FB_TILEBLITTING
- 	struct fb_tile_ops *tileops;    /* Tile Blitting */
-@@ -499,32 +513,11 @@ struct fb_info {
- 	void *fbcon_par;                /* fbcon use-only private area */
- 	/* From here on everything is device dependent */
- 	void *par;
-	/* we need the PCI or similar aperture base/size not
-	   smem_start/size as smem_start may just be an object
-	   allocated inside the aperture so may not actually overlap */
-	struct apertures_struct {
-		unsigned int count;
-		struct aperture {
-			resource_size_t base;
-			resource_size_t size;
-		} ranges[0];
-	} *apertures;
- 
- 	bool skip_vt_switch; /* no VT switch on suspend/resume required */
-+	bool skip_panic; /* Do not write to the fb after a panic */
- };
- 
-static inline struct apertures_struct *alloc_apertures(unsigned int max_num) {
-	struct apertures_struct *a;
-
-	a = kzalloc(struct_size(a, ranges, max_num), GFP_KERNEL);
-	if (!a)
-		return NULL;
-	a->count = max_num;
-	return a;
-}
-
-#define FBINFO_FLAG_DEFAULT	FBINFO_DEFAULT
-
- /* This will go away
-  * fbset currently hacks in FB_ACCELF_TEXT into var.accel_flags
-  * when it wants to turn the acceleration engine on.  This is
-@@ -533,58 +526,6 @@ static inline struct apertures_struct *alloc_apertures(unsigned int max_num) {
-  */
- #define STUPID_ACCELF_TEXT_SHIT
- 
-// This will go away
-#if defined(__sparc__)
-
-/* We map all of our framebuffers such that big-endian accesses
- * are what we want, so the following is sufficient.
- */
-
-// This will go away
-#define fb_readb sbus_readb
-#define fb_readw sbus_readw
-#define fb_readl sbus_readl
-#define fb_readq sbus_readq
-#define fb_writeb sbus_writeb
-#define fb_writew sbus_writew
-#define fb_writel sbus_writel
-#define fb_writeq sbus_writeq
-#define fb_memset sbus_memset_io
-#define fb_memcpy_fromfb sbus_memcpy_fromio
-#define fb_memcpy_tofb sbus_memcpy_toio
-
-#elif defined(__i386__) || defined(__alpha__) || defined(__x86_64__) ||	\
-	defined(__hppa__) || defined(__sh__) || defined(__powerpc__) ||	\
-	defined(__arm__) || defined(__aarch64__)
-
-#define fb_readb __raw_readb
-#define fb_readw __raw_readw
-#define fb_readl __raw_readl
-#define fb_readq __raw_readq
-#define fb_writeb __raw_writeb
-#define fb_writew __raw_writew
-#define fb_writel __raw_writel
-#define fb_writeq __raw_writeq
-#define fb_memset memset_io
-#define fb_memcpy_fromfb memcpy_fromio
-#define fb_memcpy_tofb memcpy_toio
-
-#else
-
-#define fb_readb(addr) (*(volatile u8 *) (addr))
-#define fb_readw(addr) (*(volatile u16 *) (addr))
-#define fb_readl(addr) (*(volatile u32 *) (addr))
-#define fb_readq(addr) (*(volatile u64 *) (addr))
-#define fb_writeb(b,addr) (*(volatile u8 *) (addr) = (b))
-#define fb_writew(b,addr) (*(volatile u16 *) (addr) = (b))
-#define fb_writel(b,addr) (*(volatile u32 *) (addr) = (b))
-#define fb_writeq(b,addr) (*(volatile u64 *) (addr) = (b))
-#define fb_memset memset
-#define fb_memcpy_fromfb memcpy
-#define fb_memcpy_tofb memcpy
-
-#endif
-
- #define FB_LEFT_POS(p, bpp)          (fb_be_math(p) ? (32 - (bpp)) : 0)
- #define FB_SHIFT_HIGH(p, val, bits)  (fb_be_math(p) ? (val) >> (bits) : \
- 						      (val) << (bits))
-@@ -598,12 +539,41 @@ static inline struct apertures_struct *alloc_apertures(unsigned int max_num) {
- extern int fb_set_var(struct fb_info *info, struct fb_var_screeninfo *var);
- extern int fb_pan_display(struct fb_info *info, struct fb_var_screeninfo *var);
- extern int fb_blank(struct fb_info *info, int blank);
-+
-+/*
-+ * Helpers for framebuffers in I/O memory
-+ */
-+
- extern void cfb_fillrect(struct fb_info *info, const struct fb_fillrect *rect);
- extern void cfb_copyarea(struct fb_info *info, const struct fb_copyarea *area);
- extern void cfb_imageblit(struct fb_info *info, const struct fb_image *image);
-+extern ssize_t fb_io_read(struct fb_info *info, char __user *buf,
-+			  size_t count, loff_t *ppos);
-+extern ssize_t fb_io_write(struct fb_info *info, const char __user *buf,
-+			   size_t count, loff_t *ppos);
-+int fb_io_mmap(struct fb_info *info, struct vm_area_struct *vma);
-+
-+#define __FB_DEFAULT_IOMEM_OPS_RDWR \
-+	.fb_read	= fb_io_read, \
-+	.fb_write	= fb_io_write
-+
-+#define __FB_DEFAULT_IOMEM_OPS_DRAW \
-+	.fb_fillrect	= cfb_fillrect, \
-+	.fb_copyarea	= cfb_copyarea, \
-+	.fb_imageblit	= cfb_imageblit
-+
-+#define __FB_DEFAULT_IOMEM_OPS_MMAP \
-+	.fb_mmap	= fb_io_mmap
-+
-+#define FB_DEFAULT_IOMEM_OPS \
-+	__FB_DEFAULT_IOMEM_OPS_RDWR, \
-+	__FB_DEFAULT_IOMEM_OPS_DRAW, \
-+	__FB_DEFAULT_IOMEM_OPS_MMAP
-+
- /*
- * Drawing operations where framebuffer is in system RAM
-+ * Helpers for framebuffers in system memory
-  */
-+
- extern void sys_fillrect(struct fb_info *info, const struct fb_fillrect *rect);
- extern void sys_copyarea(struct fb_info *info, const struct fb_copyarea *area);
- extern void sys_imageblit(struct fb_info *info, const struct fb_image *image);
-@@ -612,15 +582,32 @@ extern ssize_t fb_sys_read(struct fb_info *info, char __user *buf,
- extern ssize_t fb_sys_write(struct fb_info *info, const char __user *buf,
- 			    size_t count, loff_t *ppos);
- 
-/* drivers/video/fbmem.c */
-+#define __FB_DEFAULT_SYSMEM_OPS_RDWR \
-+	.fb_read	= fb_sys_read, \
-+	.fb_write	= fb_sys_write
-+
-+#define __FB_DEFAULT_SYSMEM_OPS_DRAW \
-+	.fb_fillrect	= sys_fillrect, \
-+	.fb_copyarea	= sys_copyarea, \
-+	.fb_imageblit	= sys_imageblit
-+
-+/*
-+ * Helpers for framebuffers in DMA-able memory
-+ */
-+
-+#define __FB_DEFAULT_DMAMEM_OPS_RDWR \
-+	.fb_read	= fb_sys_read, \
-+	.fb_write	= fb_sys_write
-+
-+#define __FB_DEFAULT_DMAMEM_OPS_DRAW \
-+	.fb_fillrect	= sys_fillrect, \
-+	.fb_copyarea	= sys_copyarea, \
-+	.fb_imageblit	= sys_imageblit
-+
-+/* fbmem.c */
- extern int register_framebuffer(struct fb_info *fb_info);
- extern void unregister_framebuffer(struct fb_info *fb_info);
-extern int remove_conflicting_pci_framebuffers(struct pci_dev *pdev,
-					       const char *name);
-extern int remove_conflicting_framebuffers(struct apertures_struct *a,
-					   const char *name, bool primary);
-extern int fb_prepare_logo(struct fb_info *fb_info, int rotate);
-extern int fb_show_logo(struct fb_info *fb_info, int rotate);
-+extern int devm_register_framebuffer(struct device *dev, struct fb_info *fb_info);
- extern char* fb_get_buffer_offset(struct fb_info *info, struct fb_pixmap *buf, u32 size);
- extern void fb_pad_unaligned_buffer(u8 *dst, u32 d_pitch, u8 *src, u32 idx,
- 				u32 height, u32 shift_high, u32 shift_low, u32 mod);
-@@ -631,16 +618,6 @@ extern int fb_get_color_depth(struct fb_var_screeninfo *var,
- extern int fb_get_options(const char *name, char **option);
- extern int fb_new_modelist(struct fb_info *info);
- 
-extern struct fb_info *registered_fb[FB_MAX];
-extern int num_registered_fb;
-extern bool fb_center_logo;
-extern int fb_logo_count;
-extern struct class *fb_class;
-
-#define for_each_registered_fb(i)		\
-	for (i = 0; i < FB_MAX; i++)		\
-		if (!registered_fb[i]) {} else
-
- static inline void lock_fb_info(struct fb_info *info)
- {
- 	mutex_lock(&info->lock);
-@@ -666,16 +643,90 @@ static inline void __fb_pad_aligned_buffer(u8 *dst, u32 d_pitch,
- 	}
- }
- 
-/* drivers/video/fb_defio.c */
-+/* fb_defio.c */
- int fb_deferred_io_mmap(struct fb_info *info, struct vm_area_struct *vma);
- extern int  fb_deferred_io_init(struct fb_info *info);
- extern void fb_deferred_io_open(struct fb_info *info,
- 				struct inode *inode,
- 				struct file *file);
-+extern void fb_deferred_io_release(struct fb_info *info);
- extern void fb_deferred_io_cleanup(struct fb_info *info);
- extern int fb_deferred_io_fsync(struct file *file, loff_t start,
- 				loff_t end, int datasync);
- 
-+/*
-+ * Generate callbacks for deferred I/O
-+ */
-+
-+#define __FB_GEN_DEFAULT_DEFERRED_OPS_RDWR(__prefix, __damage_range, __mode) \
-+	static ssize_t __prefix ## _defio_read(struct fb_info *info, char __user *buf, \
-+					       size_t count, loff_t *ppos) \
-+	{ \
-+		return fb_ ## __mode ## _read(info, buf, count, ppos); \
-+	} \
-+	static ssize_t __prefix ## _defio_write(struct fb_info *info, const char __user *buf, \
-+						size_t count, loff_t *ppos) \
-+	{ \
-+		unsigned long offset = *ppos; \
-+		ssize_t ret = fb_ ## __mode ## _write(info, buf, count, ppos); \
-+		if (ret > 0) \
-+			__damage_range(info, offset, ret); \
-+		return ret; \
-+	}
-+
-+#define __FB_GEN_DEFAULT_DEFERRED_OPS_DRAW(__prefix, __damage_area, __mode) \
-+	static void __prefix ## _defio_fillrect(struct fb_info *info, \
-+						const struct fb_fillrect *rect) \
-+	{ \
-+		__mode ## _fillrect(info, rect); \
-+		__damage_area(info, rect->dx, rect->dy, rect->width, rect->height); \
-+	} \
-+	static void __prefix ## _defio_copyarea(struct fb_info *info, \
-+						const struct fb_copyarea *area) \
-+	{ \
-+		__mode ## _copyarea(info, area); \
-+		__damage_area(info, area->dx, area->dy, area->width, area->height); \
-+	} \
-+	static void __prefix ## _defio_imageblit(struct fb_info *info, \
-+						 const struct fb_image *image) \
-+	{ \
-+		__mode ## _imageblit(info, image); \
-+		__damage_area(info, image->dx, image->dy, image->width, image->height); \
-+	}
-+
-+#define FB_GEN_DEFAULT_DEFERRED_IOMEM_OPS(__prefix, __damage_range, __damage_area) \
-+	__FB_GEN_DEFAULT_DEFERRED_OPS_RDWR(__prefix, __damage_range, io) \
-+	__FB_GEN_DEFAULT_DEFERRED_OPS_DRAW(__prefix, __damage_area, cfb)
-+
-+#define FB_GEN_DEFAULT_DEFERRED_SYSMEM_OPS(__prefix, __damage_range, __damage_area) \
-+	__FB_GEN_DEFAULT_DEFERRED_OPS_RDWR(__prefix, __damage_range, sys) \
-+	__FB_GEN_DEFAULT_DEFERRED_OPS_DRAW(__prefix, __damage_area, sys)
-+
-+#define FB_GEN_DEFAULT_DEFERRED_DMAMEM_OPS(__prefix, __damage_range, __damage_area) \
-+	__FB_GEN_DEFAULT_DEFERRED_OPS_RDWR(__prefix, __damage_range, sys) \
-+	__FB_GEN_DEFAULT_DEFERRED_OPS_DRAW(__prefix, __damage_area, sys)
-+
-+/*
-+ * Initializes struct fb_ops for deferred I/O.
-+ */
-+
-+#define __FB_DEFAULT_DEFERRED_OPS_RDWR(__prefix) \
-+	.fb_read	= __prefix ## _defio_read, \
-+	.fb_write	= __prefix ## _defio_write
-+
-+#define __FB_DEFAULT_DEFERRED_OPS_DRAW(__prefix) \
-+	.fb_fillrect	= __prefix ## _defio_fillrect, \
-+	.fb_copyarea	= __prefix ## _defio_copyarea, \
-+	.fb_imageblit	= __prefix ## _defio_imageblit
-+
-+#define __FB_DEFAULT_DEFERRED_OPS_MMAP(__prefix) \
-+	.fb_mmap	= fb_deferred_io_mmap
-+
-+#define FB_DEFAULT_DEFERRED_OPS(__prefix) \
-+	__FB_DEFAULT_DEFERRED_OPS_RDWR(__prefix), \
-+	__FB_DEFAULT_DEFERRED_OPS_DRAW(__prefix), \
-+	__FB_DEFAULT_DEFERRED_OPS_MMAP(__prefix)
-+
- static inline bool fb_be_math(struct fb_info *info)
- {
- #ifdef CONFIG_FB_FOREIGN_ENDIAN
-@@ -695,14 +746,29 @@ static inline bool fb_be_math(struct fb_info *info)
- #endif /* CONFIG_FB_FOREIGN_ENDIAN */
- }
- 
-/* drivers/video/fbsysfs.c */
- extern struct fb_info *framebuffer_alloc(size_t size, struct device *dev);
- extern void framebuffer_release(struct fb_info *info);
-extern int fb_init_device(struct fb_info *fb_info);
-extern void fb_cleanup_device(struct fb_info *head);
- extern void fb_bl_default_curve(struct fb_info *fb_info, u8 off, u8 min, u8 max);
- 
-/* drivers/video/fbmon.c */
-+#if IS_ENABLED(CONFIG_FB_BACKLIGHT)
-+struct backlight_device *fb_bl_device(struct fb_info *info);
-+void fb_bl_notify_blank(struct fb_info *info, int old_blank);
-+#else
-+static inline struct backlight_device *fb_bl_device(struct fb_info *info)
-+{
-+	return NULL;
-+}
-+
-+static inline void fb_bl_notify_blank(struct fb_info *info, int old_blank)
-+{ }
-+#endif
-+
-+static inline struct lcd_device *fb_lcd_device(struct fb_info *info)
-+{
-+	return info->lcd_dev;
-+}
-+
-+/* fbmon.c */
- #define FB_MAXTIMINGS		0
- #define FB_VSYNCTIMINGS		1
- #define FB_HSYNCTIMINGS		2
-@@ -736,7 +802,7 @@ extern int of_get_fb_videomode(struct device_node *np,
- extern int fb_videomode_from_videomode(const struct videomode *vm,
- 				       struct fb_videomode *fbmode);
- 
-/* drivers/video/modedb.c */
-+/* modedb.c */
- #define VESA_MODEDB_SIZE 43
- #define DMT_SIZE 0x50
- 
-@@ -762,7 +828,7 @@ extern void fb_videomode_to_modelist(const struct fb_videomode *modedb, int num,
- extern const struct fb_videomode *fb_find_best_display(const struct fb_monspecs *specs,
- 						       struct list_head *head);
- 
-/* drivers/video/fbcmap.c */
-+/* fbcmap.c */
- extern int fb_alloc_cmap(struct fb_cmap *cmap, int len, int transp);
- extern int fb_alloc_cmap_gfp(struct fb_cmap *cmap, int len, int transp, gfp_t flags);
- extern void fb_dealloc_cmap(struct fb_cmap *cmap);
-@@ -797,7 +863,6 @@ struct dmt_videomode {
- 	const struct fb_videomode *mode;
- };
- 
-extern const char *fb_mode_option;
- extern const struct fb_videomode vesa_modes[];
- extern const struct dmt_videomode dmt_modes[];
- 
-@@ -813,7 +878,12 @@ extern int fb_find_mode(struct fb_var_screeninfo *var,
- 			const struct fb_videomode *default_mode,
- 			unsigned int default_bpp);
- 
-/* Convenience logging macros */
-+bool fb_modesetting_disabled(const char *drvname);
-+
-+/*
-+ * Convenience logging macros
-+ */
-+
- #define fb_err(fb_info, fmt, ...)					\
- 	pr_err("fb%d: " fmt, (fb_info)->node, ##__VA_ARGS__)
- #define fb_notice(info, fmt, ...)					\
-@@ -825,4 +895,12 @@ extern int fb_find_mode(struct fb_var_screeninfo *var,
- #define fb_dbg(fb_info, fmt, ...)					\
- 	pr_debug("fb%d: " fmt, (fb_info)->node, ##__VA_ARGS__)
- 
-+#define fb_warn_once(fb_info, fmt, ...)					\
-+	pr_warn_once("fb%d: " fmt, (fb_info)->node, ##__VA_ARGS__)
-+
-+#define fb_WARN_ONCE(fb_info, condition, fmt, ...) \
-+	WARN_ONCE(condition, "fb%d: " fmt, (fb_info)->node, ##__VA_ARGS__)
-+#define fb_WARN_ON_ONCE(fb_info, x) \
-+	fb_WARN_ONCE(fb_info, (x), "%s", "fb_WARN_ON_ONCE(" __stringify(x) ")")
-+
- #endif /* _LINUX_FB_H */
-diff --git a/include/linux/fbcon.h b/include/linux/fbcon.h
-index 2382dec6d6ab..f206370060e1 100644
--- a/include/linux/fbcon.h
-+++ b/include/linux/fbcon.h
-@@ -1,6 +1,13 @@
- #ifndef _LINUX_FBCON_H
- #define _LINUX_FBCON_H
- 
-+#include <linux/compiler_types.h>
-+
-+struct fb_blit_caps;
-+struct fb_info;
-+struct fb_var_screeninfo;
-+struct fb_videomode;
-+
- #ifdef CONFIG_FRAMEBUFFER_CONSOLE
- void __init fb_console_init(void);
- void __exit fb_console_exit(void);
-@@ -11,6 +18,7 @@ void fbcon_suspended(struct fb_info *info);
- void fbcon_resumed(struct fb_info *info);
- int fbcon_mode_deleted(struct fb_info *info,
- 		       struct fb_videomode *mode);
-+void fbcon_delete_modelist(struct list_head *head);
- void fbcon_new_modelist(struct fb_info *info);
- void fbcon_get_requirement(struct fb_info *info,
- 			   struct fb_blit_caps *caps);
-@@ -31,6 +39,7 @@ static inline void fbcon_suspended(struct fb_info *info) {}
- static inline void fbcon_resumed(struct fb_info *info) {}
- static inline int fbcon_mode_deleted(struct fb_info *info,
- 				     struct fb_videomode *mode) { return 0; }
-+static inline void fbcon_delete_modelist(struct list_head *head) {}
- static inline void fbcon_new_modelist(struct fb_info *info) {}
- static inline void fbcon_get_requirement(struct fb_info *info,
- 					 struct fb_blit_caps *caps) {}
-diff --git a/include/linux/fdtable.h b/include/linux/fdtable.h
-index e066816f3519..c45306a9f007 100644
--- a/include/linux/fdtable.h
-+++ b/include/linux/fdtable.h
-@@ -22,7 +22,6 @@
-  * as this is the granularity returned by copy_fdset().
-  */
- #define NR_OPEN_DEFAULT BITS_PER_LONG
-#define NR_OPEN_MAX ~0U
- 
- struct fdtable {
- 	unsigned int max_fds;
-@@ -33,16 +32,6 @@ struct fdtable {
- 	struct rcu_head rcu;
- };
- 
-static inline bool close_on_exec(unsigned int fd, const struct fdtable *fdt)
-{
-	return test_bit(fd, fdt->close_on_exec);
-}
-
-static inline bool fd_is_open(unsigned int fd, const struct fdtable *fdt)
-{
-	return test_bit(fd, fdt->open_fds);
-}
-
- /*
-  * Open file table structure
-  */
-@@ -83,12 +72,17 @@ struct dentry;
- static inline struct file *files_lookup_fd_raw(struct files_struct *files, unsigned int fd)
- {
- 	struct fdtable *fdt = rcu_dereference_raw(files->fdt);
-
-	if (fd < fdt->max_fds) {
-		fd = array_index_nospec(fd, fdt->max_fds);
-		return rcu_dereference_raw(fdt->fd[fd]);
-	}
-	return NULL;
-+	unsigned long mask = array_index_mask_nospec(fd, fdt->max_fds);
-+	struct file *needs_masking;
-+
-+	/*
-+	 * 'mask' is zero for an out-of-bounds fd, all ones for ok.
-+	 * 'fd&mask' is 'fd' for ok, or 0 for out of bounds.
-+	 *
-+	 * Accessing fdt->fd[0] is ok, but needs masking of the result.
-+	 */
-+	needs_masking = rcu_dereference_raw(fdt->fd[fd&mask]);
-+	return (struct file *)(mask & (unsigned long)needs_masking);
- }
- 
- static inline struct file *files_lookup_fd_locked(struct files_struct *files, unsigned int fd)
-@@ -98,36 +92,26 @@ static inline struct file *files_lookup_fd_locked(struct files_struct *files, un
- 	return files_lookup_fd_raw(files, fd);
- }
- 
-static inline struct file *files_lookup_fd_rcu(struct files_struct *files, unsigned int fd)
-+static inline bool close_on_exec(unsigned int fd, const struct files_struct *files)
- {
-	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
-			   "suspicious rcu_dereference_check() usage");
-	return files_lookup_fd_raw(files, fd);
-+	return test_bit(fd, files_fdtable(files)->close_on_exec);
- }
- 
-static inline struct file *lookup_fd_rcu(unsigned int fd)
-{
-	return files_lookup_fd_rcu(current->files, fd);
-}
-
-struct file *task_lookup_fd_rcu(struct task_struct *task, unsigned int fd);
-struct file *task_lookup_next_fd_rcu(struct task_struct *task, unsigned int *fd);
-
- struct task_struct;
- 
- void put_files_struct(struct files_struct *fs);
- int unshare_files(void);
-struct files_struct *dup_fd(struct files_struct *, unsigned, int *) __latent_entropy;
-+struct fd_range {
-+	unsigned int from, to;
-+};
-+struct files_struct *dup_fd(struct files_struct *, struct fd_range *) __latent_entropy;
- void do_close_on_exec(struct files_struct *);
- int iterate_fd(struct files_struct *, unsigned,
- 		int (*)(const void *, struct file *, unsigned),
- 		const void *);
- 
- extern int close_fd(unsigned int fd);
-extern int __close_range(unsigned int fd, unsigned int max_fd, unsigned int flags);
-extern struct file *close_fd_get_file(unsigned int fd);
-extern int unshare_fd(unsigned long unshare_flags, unsigned int max_fds,
-		      struct files_struct **new_fdp);
-+extern struct file *file_close_fd(unsigned int fd);
- 
- extern struct kmem_cache *files_cachep;
- 
-diff --git a/include/linux/fiemap.h b/include/linux/fiemap.h
-index c50882f19235..966092ffa89a 100644
--- a/include/linux/fiemap.h
-+++ b/include/linux/fiemap.h
-@@ -5,12 +5,18 @@
- #include <uapi/linux/fiemap.h>
- #include <linux/fs.h>
- 
-+/**
-+ * struct fiemap_extent_info - fiemap request to a filesystem
-+ * @fi_flags:		Flags as passed from user
-+ * @fi_extents_mapped:	Number of mapped extents
-+ * @fi_extents_max:	Size of fiemap_extent array
-+ * @fi_extents_start:	Start of fiemap_extent array
-+ */
- struct fiemap_extent_info {
-	unsigned int fi_flags;		/* Flags as passed from user */
-	unsigned int fi_extents_mapped;	/* Number of mapped extents */
-	unsigned int fi_extents_max;	/* Size of fiemap_extent array */
-	struct fiemap_extent __user *fi_extents_start; /* Start of
-							fiemap_extent array */
-+	unsigned int fi_flags;
-+	unsigned int fi_extents_mapped;
-+	unsigned int fi_extents_max;
-+	struct fiemap_extent __user *fi_extents_start;
- };
- 
- int fiemap_prep(struct inode *inode, struct fiemap_extent_info *fieinfo,
-diff --git a/include/linux/file.h b/include/linux/file.h
-index 39704eae83e2..cf389fde9bc2 100644
--- a/include/linux/file.h
-+++ b/include/linux/file.h
-@@ -10,6 +10,8 @@
- #include <linux/types.h>
- #include <linux/posix_types.h>
- #include <linux/errno.h>
-+#include <linux/cleanup.h>
-+#include <linux/err.h>
- 
- struct file;
- 
-@@ -23,63 +25,65 @@ struct inode;
- struct path;
- extern struct file *alloc_file_pseudo(struct inode *, struct vfsmount *,
- 	const char *, int flags, const struct file_operations *);
-+extern struct file *alloc_file_pseudo_noaccount(struct inode *, struct vfsmount *,
-+	const char *, int flags, const struct file_operations *);
- extern struct file *alloc_file_clone(struct file *, int flags,
- 	const struct file_operations *);
- 
-static inline void fput_light(struct file *file, int fput_needed)
-{
-	if (fput_needed)
-		fput(file);
-}
-
-+/* either a reference to struct file + flags
-+ * (cloned vs. borrowed, pos locked), with
-+ * flags stored in lower bits of value,
-+ * or empty (represented by 0).
-+ */
- struct fd {
-	struct file *file;
-	unsigned int flags;
-+	unsigned long word;
- };
- #define FDPUT_FPUT       1
- #define FDPUT_POS_UNLOCK 2
- 
-static inline void fdput(struct fd fd)
-+#define fd_file(f) ((struct file *)((f).word & ~(FDPUT_FPUT|FDPUT_POS_UNLOCK)))
-+static inline bool fd_empty(struct fd f)
- {
-	if (fd.flags & FDPUT_FPUT)
-		fput(fd.file);
-+	return unlikely(!f.word);
- }
- 
-extern struct file *fget(unsigned int fd);
-extern struct file *fget_raw(unsigned int fd);
-extern struct file *fget_task(struct task_struct *task, unsigned int fd);
-extern unsigned long __fdget(unsigned int fd);
-extern unsigned long __fdget_raw(unsigned int fd);
-extern unsigned long __fdget_pos(unsigned int fd);
-extern void __f_unlock_pos(struct file *);
-
-static inline struct fd __to_fd(unsigned long v)
-+#define EMPTY_FD (struct fd){0}
-+static inline struct fd BORROWED_FD(struct file *f)
- {
-	return (struct fd){(struct file *)(v & ~3),v & 3};
-+	return (struct fd){(unsigned long)f};
- }
-
-static inline struct fd fdget(unsigned int fd)
-+static inline struct fd CLONED_FD(struct file *f)
- {
-	return __to_fd(__fdget(fd));
-+	return (struct fd){(unsigned long)f | FDPUT_FPUT};
- }
- 
-static inline struct fd fdget_raw(unsigned int fd)
-+static inline void fdput(struct fd fd)
- {
-	return __to_fd(__fdget_raw(fd));
-+	if (unlikely(fd.word & FDPUT_FPUT))
-+		fput(fd_file(fd));
- }
- 
-static inline struct fd fdget_pos(int fd)
-{
-	return __to_fd(__fdget_pos(fd));
-}
-+extern struct file *fget(unsigned int fd);
-+extern struct file *fget_raw(unsigned int fd);
-+extern struct file *fget_task(struct task_struct *task, unsigned int fd);
-+extern struct file *fget_task_next(struct task_struct *task, unsigned int *fd);
-+extern void __f_unlock_pos(struct file *);
-+
-+struct fd fdget(unsigned int fd);
-+struct fd fdget_raw(unsigned int fd);
-+struct fd fdget_pos(unsigned int fd);
- 
- static inline void fdput_pos(struct fd f)
- {
-	if (f.flags & FDPUT_POS_UNLOCK)
-		__f_unlock_pos(f.file);
-+	if (f.word & FDPUT_POS_UNLOCK)
-+		__f_unlock_pos(fd_file(f));
- 	fdput(f);
- }
- 
-+DEFINE_CLASS(fd, struct fd, fdput(_T), fdget(fd), int fd)
-+DEFINE_CLASS(fd_raw, struct fd, fdput(_T), fdget_raw(fd), int fd)
-+DEFINE_CLASS(fd_pos, struct fd, fdput_pos(_T), fdget_pos(fd), int fd)
-+
- extern int f_dupfd(unsigned int from, struct file *file, unsigned flags);
- extern int replace_fd(unsigned fd, struct file *file, unsigned flags);
- extern void set_close_on_exec(unsigned int fd, int flag);
-@@ -88,20 +92,34 @@ extern int __get_unused_fd_flags(unsigned flags, unsigned long nofile);
- extern int get_unused_fd_flags(unsigned flags);
- extern void put_unused_fd(unsigned int fd);
- 
-extern void fd_install(unsigned int fd, struct file *file);
-+DEFINE_CLASS(get_unused_fd, int, if (_T >= 0) put_unused_fd(_T),
-+	     get_unused_fd_flags(flags), unsigned flags)
-+DEFINE_FREE(fput, struct file *, if (!IS_ERR_OR_NULL(_T)) fput(_T))
- 
-extern int __receive_fd(struct file *file, int __user *ufd,
-			unsigned int o_flags);
-+/*
-+ * take_fd() will take care to set @fd to -EBADF ensuring that
-+ * CLASS(get_unused_fd) won't call put_unused_fd(). This makes it
-+ * easier to rely on CLASS(get_unused_fd):
-+ *
-+ * struct file *f;
-+ *
-+ * CLASS(get_unused_fd, fd)(O_CLOEXEC);
-+ * if (fd < 0)
-+ *         return fd;
-+ *
-+ * f = dentry_open(&path, O_RDONLY, current_cred());
-+ * if (IS_ERR(f))
-+ *         return PTR_ERR(f);
-+ *
-+ * fd_install(fd, f);
-+ * return take_fd(fd);
-+ */
-+#define take_fd(fd) __get_and_null(fd, -EBADF)
- 
-extern int receive_fd(struct file *file, unsigned int o_flags);
-+extern void fd_install(unsigned int fd, struct file *file);
-+
-+int receive_fd(struct file *file, int __user *ufd, unsigned int o_flags);
- 
-static inline int receive_fd_user(struct file *file, int __user *ufd,
-				  unsigned int o_flags)
-{
-	if (ufd == NULL)
-		return -EFAULT;
-	return __receive_fd(file, ufd, o_flags);
-}
- int receive_fd_replace(int new_fd, struct file *file, unsigned int o_flags);
- 
- extern void flush_delayed_fput(void);
-@@ -109,4 +127,130 @@ extern void __fput_sync(struct file *);
- 
- extern unsigned int sysctl_nr_open_min, sysctl_nr_open_max;
- 
-+/*
-+ * fd_prepare: Combined fd + file allocation cleanup class.
-+ * @err: Error code to indicate if allocation succeeded.
-+ * @__fd: Allocated fd (may not be accessed directly)
-+ * @__file: Allocated struct file pointer (may not be accessed directly)
-+ *
-+ * Allocates an fd and a file together. On error paths, automatically cleans
-+ * up whichever resource was successfully allocated. Allows flexible file
-+ * allocation with different functions per usage.
-+ *
-+ * Do not use directly.
-+ */
-+struct fd_prepare {
-+	s32 err;
-+	s32 __fd; /* do not access directly */
-+	struct file *__file; /* do not access directly */
-+};
-+
-+/* Typedef for fd_prepare cleanup guards. */
-+typedef struct fd_prepare class_fd_prepare_t;
-+
-+/*
-+ * Accessors for fd_prepare class members.
-+ * _Generic() is used for zero-cost type safety.
-+ */
-+#define fd_prepare_fd(_fdf) \
-+	(_Generic((_fdf), struct fd_prepare: (_fdf).__fd))
-+
-+#define fd_prepare_file(_fdf) \
-+	(_Generic((_fdf), struct fd_prepare: (_fdf).__file))
-+
-+/* Do not use directly. */
-+static inline void class_fd_prepare_destructor(const struct fd_prepare *fdf)
-+{
-+	if (unlikely(fdf->err)) {
-+		if (likely(fdf->__fd >= 0))
-+			put_unused_fd(fdf->__fd);
-+		if (unlikely(!IS_ERR_OR_NULL(fdf->__file)))
-+			fput(fdf->__file);
-+	}
-+}
-+
-+/* Do not use directly. */
-+static inline int class_fd_prepare_lock_err(const struct fd_prepare *fdf)
-+{
-+	if (unlikely(fdf->err))
-+		return fdf->err;
-+	if (unlikely(fdf->__fd < 0))
-+		return fdf->__fd;
-+	if (unlikely(IS_ERR(fdf->__file)))
-+		return PTR_ERR(fdf->__file);
-+	if (unlikely(!fdf->__file))
-+		return -ENOMEM;
-+	return 0;
-+}
-+
-+/*
-+ * __FD_PREPARE_INIT - Helper to initialize fd_prepare class.
-+ * @_fd_flags: flags for get_unused_fd_flags()
-+ * @_file_owned: expression that returns struct file *
-+ *
-+ * Returns a struct fd_prepare with fd, file, and err set.
-+ * If fd allocation fails, fd will be negative and err will be set. If
-+ * fd succeeds but file_init_expr fails, file will be ERR_PTR and err
-+ * will be set. The err field is the single source of truth for error
-+ * checking.
-+ */
-+#define __FD_PREPARE_INIT(_fd_flags, _file_owned)                 \
-+	({                                                        \
-+		struct fd_prepare fdf = {                         \
-+			.__fd = get_unused_fd_flags((_fd_flags)), \
-+		};                                                \
-+		if (likely(fdf.__fd >= 0))                        \
-+			fdf.__file = (_file_owned);               \
-+		fdf.err = ACQUIRE_ERR(fd_prepare, &fdf);          \
-+		fdf;                                              \
-+	})
-+
-+/*
-+ * FD_PREPARE - Macro to declare and initialize an fd_prepare variable.
-+ *
-+ * Declares and initializes an fd_prepare variable with automatic
-+ * cleanup. No separate scope required - cleanup happens when variable
-+ * goes out of scope.
-+ *
-+ * @_fdf: name of struct fd_prepare variable to define
-+ * @_fd_flags: flags for get_unused_fd_flags()
-+ * @_file_owned: struct file to take ownership of (can be expression)
-+ */
-+#define FD_PREPARE(_fdf, _fd_flags, _file_owned) \
-+	CLASS_INIT(fd_prepare, _fdf, __FD_PREPARE_INIT(_fd_flags, _file_owned))
-+
-+/*
-+ * fd_publish - Publish prepared fd and file to the fd table.
-+ * @_fdf: struct fd_prepare variable
-+ */
-+#define fd_publish(_fdf)                                       \
-+	({                                                     \
-+		struct fd_prepare *fdp = &(_fdf);              \
-+		VFS_WARN_ON_ONCE(fdp->err);                    \
-+		VFS_WARN_ON_ONCE(fdp->__fd < 0);               \
-+		VFS_WARN_ON_ONCE(IS_ERR_OR_NULL(fdp->__file)); \
-+		fd_install(fdp->__fd, fdp->__file);            \
-+		fdp->__fd;                                     \
-+	})
-+
-+/* Do not use directly. */
-+#define __FD_ADD(_fdf, _fd_flags, _file_owned)            \
-+	({                                                \
-+		FD_PREPARE(_fdf, _fd_flags, _file_owned); \
-+		s32 ret = _fdf.err;                       \
-+		if (likely(!ret))                         \
-+			ret = fd_publish(_fdf);           \
-+		ret;                                      \
-+	})
-+
-+/*
-+ * FD_ADD - Allocate and install an fd and file in one step.
-+ * @_fd_flags: flags for get_unused_fd_flags()
-+ * @_file_owned: struct file to take ownership of
-+ *
-+ * Returns the allocated fd number, or negative error code on failure.
-+ */
-+#define FD_ADD(_fd_flags, _file_owned) \
-+	__FD_ADD(__UNIQUE_ID(fd_prepare), _fd_flags, _file_owned)
-+
- #endif /* __LINUX_FILE_H */
-diff --git a/include/linux/file_ref.h b/include/linux/file_ref.h
-new file mode 100644
-index 000000000000..31551e4cb8f3
--- /dev/null
-+++ b/include/linux/file_ref.h
-@@ -0,0 +1,218 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+#ifndef _LINUX_FILE_REF_H
-+#define _LINUX_FILE_REF_H
-+
-+#include <linux/atomic.h>
-+#include <linux/preempt.h>
-+#include <linux/types.h>
-+
-+/*
-+ * file_ref is a reference count implementation specifically for use by
-+ * files. It takes inspiration from rcuref but differs in key aspects
-+ * such as support for SLAB_TYPESAFE_BY_RCU type caches.
-+ *
-+ * FILE_REF_ONEREF                FILE_REF_MAXREF
-+ * 0x0000000000000000UL      0x7FFFFFFFFFFFFFFFUL
-+ * <-------------------valid ------------------->
-+ *
-+ *                       FILE_REF_SATURATED
-+ * 0x8000000000000000UL 0xA000000000000000UL 0xBFFFFFFFFFFFFFFFUL
-+ * <-----------------------saturation zone---------------------->
-+ *
-+ * FILE_REF_RELEASED                   FILE_REF_DEAD
-+ * 0xC000000000000000UL         0xE000000000000000UL
-+ * <-------------------dead zone------------------->
-+ *
-+ * FILE_REF_NOREF
-+ * 0xFFFFFFFFFFFFFFFFUL
-+ */
-+
-+#ifdef CONFIG_64BIT
-+#define FILE_REF_ONEREF		0x0000000000000000UL
-+#define FILE_REF_MAXREF		0x7FFFFFFFFFFFFFFFUL
-+#define FILE_REF_SATURATED	0xA000000000000000UL
-+#define FILE_REF_RELEASED	0xC000000000000000UL
-+#define FILE_REF_DEAD		0xE000000000000000UL
-+#define FILE_REF_NOREF		0xFFFFFFFFFFFFFFFFUL
-+#else
-+#define FILE_REF_ONEREF		0x00000000U
-+#define FILE_REF_MAXREF		0x7FFFFFFFU
-+#define FILE_REF_SATURATED	0xA0000000U
-+#define FILE_REF_RELEASED	0xC0000000U
-+#define FILE_REF_DEAD		0xE0000000U
-+#define FILE_REF_NOREF		0xFFFFFFFFU
-+#endif
-+
-+typedef struct {
-+#ifdef CONFIG_64BIT
-+	atomic64_t refcnt;
-+#else
-+	atomic_t refcnt;
-+#endif
-+} file_ref_t;
-+
-+/**
-+ * file_ref_init - Initialize a file reference count
-+ * @ref: Pointer to the reference count
-+ * @cnt: The initial reference count typically '1'
-+ */
-+static inline void file_ref_init(file_ref_t *ref, unsigned long cnt)
-+{
-+	atomic_long_set(&ref->refcnt, cnt - 1);
-+}
-+
-+bool __file_ref_put(file_ref_t *ref, unsigned long cnt);
-+
-+/**
-+ * file_ref_get - Acquire one reference on a file
-+ * @ref: Pointer to the reference count
-+ *
-+ * Similar to atomic_inc_not_zero() but saturates at FILE_REF_MAXREF.
-+ *
-+ * Provides full memory ordering.
-+ *
-+ * Return: False if the attempt to acquire a reference failed. This happens
-+ *         when the last reference has been put already. True if a reference
-+ *         was successfully acquired
-+ */
-+static __always_inline __must_check bool file_ref_get(file_ref_t *ref)
-+{
-+	/*
-+	 * Unconditionally increase the reference count with full
-+	 * ordering. The saturation and dead zones provide enough
-+	 * tolerance for this.
-+	 *
-+	 * If this indicates negative the file in question the fail can
-+	 * be freed and immediately reused due to SLAB_TYPSAFE_BY_RCU.
-+	 * Hence, unconditionally altering the file reference count to
-+	 * e.g., reset the file reference count back to the middle of
-+	 * the deadzone risk end up marking someone else's file as dead
-+	 * behind their back.
-+	 *
-+	 * It would be possible to do a careful:
-+	 *
-+	 * cnt = atomic_long_inc_return();
-+	 * if (likely(cnt >= 0))
-+	 *	return true;
-+	 *
-+	 * and then something like:
-+	 *
-+	 * if (cnt >= FILE_REF_RELEASE)
-+	 *	atomic_long_try_cmpxchg(&ref->refcnt, &cnt, FILE_REF_DEAD),
-+	 *
-+	 * to set the value back to the middle of the deadzone. But it's
-+	 * practically impossible to go from FILE_REF_DEAD to
-+	 * FILE_REF_ONEREF. It would need 2305843009213693952/2^61
-+	 * file_ref_get()s to resurrect such a dead file.
-+	 */
-+	return !atomic_long_add_negative(1, &ref->refcnt);
-+}
-+
-+/**
-+ * file_ref_inc - Acquire one reference on a file
-+ * @ref: Pointer to the reference count
-+ *
-+ * Acquire an additional reference on a file. Warns if the caller didn't
-+ * already hold a reference.
-+ */
-+static __always_inline void file_ref_inc(file_ref_t *ref)
-+{
-+	long prior = atomic_long_fetch_inc_relaxed(&ref->refcnt);
-+	WARN_ONCE(prior < 0, "file_ref_inc() on a released file reference");
-+}
-+
-+/**
-+ * file_ref_put -- Release a file reference
-+ * @ref:	Pointer to the reference count
-+ *
-+ * Provides release memory ordering, such that prior loads and stores
-+ * are done before, and provides an acquire ordering on success such
-+ * that free() must come after.
-+ *
-+ * Return: True if this was the last reference with no future references
-+ *         possible. This signals the caller that it can safely release
-+ *         the object which is protected by the reference counter.
-+ *         False if there are still active references or the put() raced
-+ *         with a concurrent get()/put() pair. Caller is not allowed to
-+ *         release the protected object.
-+ */
-+static __always_inline __must_check bool file_ref_put(file_ref_t *ref)
-+{
-+	long cnt;
-+
-+	/*
-+	 * While files are SLAB_TYPESAFE_BY_RCU and thus file_ref_put()
-+	 * calls don't risk UAFs when a file is recyclyed, it is still
-+	 * vulnerable to UAFs caused by freeing the whole slab page once
-+	 * it becomes unused. Prevent file_ref_put() from being
-+	 * preempted protects against this.
-+	 */
-+	guard(preempt)();
-+	/*
-+	 * Unconditionally decrease the reference count. The saturation
-+	 * and dead zones provide enough tolerance for this. If this
-+	 * fails then we need to handle the last reference drop and
-+	 * cases inside the saturation and dead zones.
-+	 */
-+	cnt = atomic_long_dec_return(&ref->refcnt);
-+	if (cnt >= 0)
-+		return false;
-+	return __file_ref_put(ref, cnt);
-+}
-+
-+/**
-+ * file_ref_put_close - drop a reference expecting it would transition to FILE_REF_NOREF
-+ * @ref:	Pointer to the reference count
-+ *
-+ * Semantically it is equivalent to calling file_ref_put(), but it trades lower
-+ * performance in face of other CPUs also modifying the refcount for higher
-+ * performance when this happens to be the last reference.
-+ *
-+ * For the last reference file_ref_put() issues 2 atomics. One to drop the
-+ * reference and another to transition it to FILE_REF_DEAD. This routine does
-+ * the work in one step, but in order to do it has to pre-read the variable which
-+ * decreases scalability.
-+ *
-+ * Use with close() et al, stick to file_ref_put() by default.
-+ */
-+static __always_inline __must_check bool file_ref_put_close(file_ref_t *ref)
-+{
-+	long old;
-+
-+	old = atomic_long_read(&ref->refcnt);
-+	if (likely(old == FILE_REF_ONEREF)) {
-+		if (likely(atomic_long_try_cmpxchg(&ref->refcnt, &old, FILE_REF_DEAD)))
-+			return true;
-+	}
-+	return file_ref_put(ref);
-+}
-+
-+/**
-+ * file_ref_read - Read the number of file references
-+ * @ref: Pointer to the reference count
-+ *
-+ * Return: The number of held references (0 ... N)
-+ */
-+static inline unsigned long file_ref_read(file_ref_t *ref)
-+{
-+	unsigned long c = atomic_long_read(&ref->refcnt);
-+
-+	/* Return 0 if within the DEAD zone. */
-+	return c >= FILE_REF_RELEASED ? 0 : c + 1;
-+}
-+
-+/*
-+ * __file_ref_read_raw - Return the value stored in ref->refcnt
-+ * @ref: Pointer to the reference count
-+ *
-+ * Return: The raw value found in the counter
-+ *
-+ * A hack for file_needs_f_pos_lock(), you probably want to use
-+ * file_ref_read() instead.
-+ */
-+static inline unsigned long __file_ref_read_raw(file_ref_t *ref)
-+{
-+	return atomic_long_read(&ref->refcnt);
-+}
-+
-+#endif
-diff --git a/include/linux/fileattr.h b/include/linux/fileattr.h
-index 9e37e063ac69..f89dcfad3f8f 100644
--- a/include/linux/fileattr.h
-+++ b/include/linux/fileattr.h
-@@ -14,13 +14,33 @@
- 	 FS_XFLAG_NODUMP | FS_XFLAG_NOATIME | FS_XFLAG_DAX | \
- 	 FS_XFLAG_PROJINHERIT)
- 
-+/* Read-only inode flags */
-+#define FS_XFLAG_RDONLY_MASK \
-+	(FS_XFLAG_PREALLOC | FS_XFLAG_HASATTR)
-+
-+/* Flags to indicate valid value of fsx_ fields */
-+#define FS_XFLAG_VALUES_MASK \
-+	(FS_XFLAG_EXTSIZE | FS_XFLAG_COWEXTSIZE)
-+
-+/* Flags for directories */
-+#define FS_XFLAG_DIRONLY_MASK \
-+	(FS_XFLAG_RTINHERIT | FS_XFLAG_NOSYMLINKS | FS_XFLAG_EXTSZINHERIT)
-+
-+/* Misc settable flags */
-+#define FS_XFLAG_MISC_MASK \
-+	(FS_XFLAG_REALTIME | FS_XFLAG_NODEFRAG | FS_XFLAG_FILESTREAM)
-+
-+#define FS_XFLAGS_MASK \
-+	(FS_XFLAG_COMMON | FS_XFLAG_RDONLY_MASK | FS_XFLAG_VALUES_MASK | \
-+	 FS_XFLAG_DIRONLY_MASK | FS_XFLAG_MISC_MASK)
-+
- /*
-  * Merged interface for miscellaneous file attributes.  'flags' originates from
-  * ext* and 'fsx_flags' from xfs.  There's some overlap between the two, which
-  * is handled by the VFS helpers, so filesystems are free to implement just one
-  * or both of these sub-interfaces.
-  */
-struct fileattr {
-+struct file_kattr {
- 	u32	flags;		/* flags (FS_IOC_GETFLAGS/FS_IOC_SETFLAGS) */
- 	/* struct fsxattr: */
- 	u32	fsx_xflags;	/* xflags field value (get/set) */
-@@ -33,10 +53,10 @@ struct fileattr {
- 	bool	fsx_valid:1;
- };
- 
-int copy_fsxattr_to_user(const struct fileattr *fa, struct fsxattr __user *ufa);
-+int copy_fsxattr_to_user(const struct file_kattr *fa, struct fsxattr __user *ufa);
- 
-void fileattr_fill_xflags(struct fileattr *fa, u32 xflags);
-void fileattr_fill_flags(struct fileattr *fa, u32 flags);
-+void fileattr_fill_xflags(struct file_kattr *fa, u32 xflags);
-+void fileattr_fill_flags(struct file_kattr *fa, u32 flags);
- 
- /**
-  * fileattr_has_fsx - check for extended flags/attributes
-@@ -45,15 +65,19 @@ void fileattr_fill_flags(struct fileattr *fa, u32 flags);
-  * Return: true if any attributes are present that are not represented in
-  * ->flags.
-  */
-static inline bool fileattr_has_fsx(const struct fileattr *fa)
-+static inline bool fileattr_has_fsx(const struct file_kattr *fa)
- {
- 	return fa->fsx_valid &&
- 		((fa->fsx_xflags & ~FS_XFLAG_COMMON) || fa->fsx_extsize != 0 ||
- 		 fa->fsx_projid != 0 ||	fa->fsx_cowextsize != 0);
- }
- 
-int vfs_fileattr_get(struct dentry *dentry, struct fileattr *fa);
-int vfs_fileattr_set(struct user_namespace *mnt_userns, struct dentry *dentry,
-		     struct fileattr *fa);
-+int vfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-+int vfs_fileattr_set(struct mnt_idmap *idmap, struct dentry *dentry,
-+		     struct file_kattr *fa);
-+int ioctl_getflags(struct file *file, unsigned int __user *argp);
-+int ioctl_setflags(struct file *file, unsigned int __user *argp);
-+int ioctl_fsgetxattr(struct file *file, void __user *argp);
-+int ioctl_fssetxattr(struct file *file, void __user *argp);
- 
- #endif /* _LINUX_FILEATTR_H */
-diff --git a/include/linux/filelock.h b/include/linux/filelock.h
-new file mode 100644
-index 000000000000..54b824c05299
--- /dev/null
-+++ b/include/linux/filelock.h
-@@ -0,0 +1,584 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_FILELOCK_H
-+#define _LINUX_FILELOCK_H
-+
-+#include <linux/fs.h>
-+
-+#define FL_POSIX	1
-+#define FL_FLOCK	2
-+#define FL_DELEG	4	/* NFSv4 delegation */
-+#define FL_ACCESS	8	/* not trying to lock, just looking */
-+#define FL_EXISTS	16	/* when unlocking, test for existence */
-+#define FL_LEASE	32	/* lease held on this file */
-+#define FL_CLOSE	64	/* unlock on close */
-+#define FL_SLEEP	128	/* A blocking lock */
-+#define FL_DOWNGRADE_PENDING	256 /* Lease is being downgraded */
-+#define FL_UNLOCK_PENDING	512 /* Lease is being broken */
-+#define FL_OFDLCK	1024	/* lock is "owned" by struct file */
-+#define FL_LAYOUT	2048	/* outstanding pNFS layout */
-+#define FL_RECLAIM	4096	/* reclaiming from a reboot server */
-+
-+#define FL_CLOSE_POSIX (FL_POSIX | FL_CLOSE)
-+
-+/*
-+ * Special return value from posix_lock_file() and vfs_lock_file() for
-+ * asynchronous locking.
-+ */
-+#define FILE_LOCK_DEFERRED 1
-+
-+struct file_lock;
-+struct file_lease;
-+
-+struct file_lock_operations {
-+	void (*fl_copy_lock)(struct file_lock *, struct file_lock *);
-+	void (*fl_release_private)(struct file_lock *);
-+};
-+
-+struct lock_manager_operations {
-+	void *lm_mod_owner;
-+	fl_owner_t (*lm_get_owner)(fl_owner_t);
-+	void (*lm_put_owner)(fl_owner_t);
-+	void (*lm_notify)(struct file_lock *);	/* unblock callback */
-+	int (*lm_grant)(struct file_lock *, int);
-+	bool (*lm_lock_expirable)(struct file_lock *cfl);
-+	void (*lm_expire_lock)(void);
-+};
-+
-+struct lease_manager_operations {
-+	bool (*lm_break)(struct file_lease *);
-+	int (*lm_change)(struct file_lease *, int, struct list_head *);
-+	void (*lm_setup)(struct file_lease *, void **);
-+	bool (*lm_breaker_owns_lease)(struct file_lease *);
-+};
-+
-+struct lock_manager {
-+	struct list_head list;
-+	/*
-+	 * NFSv4 and up also want opens blocked during the grace period;
-+	 * NLM doesn't care:
-+	 */
-+	bool block_opens;
-+};
-+
-+struct net;
-+void locks_start_grace(struct net *, struct lock_manager *);
-+void locks_end_grace(struct lock_manager *);
-+bool locks_in_grace(struct net *);
-+bool opens_in_grace(struct net *);
-+
-+/*
-+ * struct file_lock has a union that some filesystems use to track
-+ * their own private info. The NFS side of things is defined here:
-+ */
-+#include <linux/nfs_fs_i.h>
-+
-+/*
-+ * struct file_lock represents a generic "file lock". It's used to represent
-+ * POSIX byte range locks, BSD (flock) locks, and leases. It's important to
-+ * note that the same struct is used to represent both a request for a lock and
-+ * the lock itself, but the same object is never used for both.
-+ *
-+ * FIXME: should we create a separate "struct lock_request" to help distinguish
-+ * these two uses?
-+ *
-+ * The varous i_flctx lists are ordered by:
-+ *
-+ * 1) lock owner
-+ * 2) lock range start
-+ * 3) lock range end
-+ *
-+ * Obviously, the last two criteria only matter for POSIX locks.
-+ */
-+
-+struct file_lock_core {
-+	struct file_lock_core *flc_blocker;	/* The lock that is blocking us */
-+	struct list_head flc_list;	/* link into file_lock_context */
-+	struct hlist_node flc_link;	/* node in global lists */
-+	struct list_head flc_blocked_requests;	/* list of requests with
-+						 * ->fl_blocker pointing here
-+						 */
-+	struct list_head flc_blocked_member;	/* node in
-+						 * ->fl_blocker->fl_blocked_requests
-+						 */
-+	fl_owner_t flc_owner;
-+	unsigned int flc_flags;
-+	unsigned char flc_type;
-+	pid_t flc_pid;
-+	int flc_link_cpu;		/* what cpu's list is this on? */
-+	wait_queue_head_t flc_wait;
-+	struct file *flc_file;
-+};
-+
-+struct file_lock {
-+	struct file_lock_core c;
-+	loff_t fl_start;
-+	loff_t fl_end;
-+
-+	const struct file_lock_operations *fl_ops;	/* Callbacks for filesystems */
-+	const struct lock_manager_operations *fl_lmops;	/* Callbacks for lockmanagers */
-+	union {
-+		struct nfs_lock_info	nfs_fl;
-+		struct nfs4_lock_info	nfs4_fl;
-+		struct {
-+			struct list_head link;	/* link in AFS vnode's pending_locks list */
-+			int state;		/* state of grant or error if -ve */
-+			unsigned int	debug_id;
-+		} afs;
-+		struct {
-+			struct inode *inode;
-+		} ceph;
-+	} fl_u;
-+} __randomize_layout;
-+
-+struct file_lease {
-+	struct file_lock_core c;
-+	struct fasync_struct *	fl_fasync; /* for lease break notifications */
-+	/* for lease breaks: */
-+	unsigned long fl_break_time;
-+	unsigned long fl_downgrade_time;
-+	const struct lease_manager_operations *fl_lmops; /* Callbacks for lease managers */
-+} __randomize_layout;
-+
-+struct file_lock_context {
-+	spinlock_t		flc_lock;
-+	struct list_head	flc_flock;
-+	struct list_head	flc_posix;
-+	struct list_head	flc_lease;
-+};
-+
-+#ifdef CONFIG_FILE_LOCKING
-+int fcntl_getlk(struct file *, unsigned int, struct flock *);
-+int fcntl_setlk(unsigned int, struct file *, unsigned int,
-+			struct flock *);
-+
-+#if BITS_PER_LONG == 32
-+int fcntl_getlk64(struct file *, unsigned int, struct flock64 *);
-+int fcntl_setlk64(unsigned int, struct file *, unsigned int,
-+			struct flock64 *);
-+#endif
-+
-+int fcntl_setlease(unsigned int fd, struct file *filp, int arg);
-+int fcntl_getlease(struct file *filp);
-+int fcntl_setdeleg(unsigned int fd, struct file *filp, struct delegation *deleg);
-+int fcntl_getdeleg(struct file *filp, struct delegation *deleg);
-+
-+static inline bool lock_is_unlock(struct file_lock *fl)
-+{
-+	return fl->c.flc_type == F_UNLCK;
-+}
-+
-+static inline bool lock_is_read(struct file_lock *fl)
-+{
-+	return fl->c.flc_type == F_RDLCK;
-+}
-+
-+static inline bool lock_is_write(struct file_lock *fl)
-+{
-+	return fl->c.flc_type == F_WRLCK;
-+}
-+
-+static inline void locks_wake_up_waiter(struct file_lock_core *flc)
-+{
-+	wake_up(&flc->flc_wait);
-+}
-+
-+static inline void locks_wake_up(struct file_lock *fl)
-+{
-+	locks_wake_up_waiter(&fl->c);
-+}
-+
-+static inline bool locks_can_async_lock(const struct file_operations *fops)
-+{
-+	return !fops->lock || fops->fop_flags & FOP_ASYNC_LOCK;
-+}
-+
-+/* fs/locks.c */
-+void locks_free_lock_context(struct inode *inode);
-+void locks_free_lock(struct file_lock *fl);
-+void locks_init_lock(struct file_lock *);
-+struct file_lock *locks_alloc_lock(void);
-+void locks_copy_lock(struct file_lock *, struct file_lock *);
-+void locks_copy_conflock(struct file_lock *, struct file_lock *);
-+void locks_remove_posix(struct file *, fl_owner_t);
-+void locks_remove_file(struct file *);
-+void locks_release_private(struct file_lock *);
-+void posix_test_lock(struct file *, struct file_lock *);
-+int posix_lock_file(struct file *, struct file_lock *, struct file_lock *);
-+int locks_delete_block(struct file_lock *);
-+int vfs_test_lock(struct file *, struct file_lock *);
-+int vfs_lock_file(struct file *, unsigned int, struct file_lock *, struct file_lock *);
-+int vfs_cancel_lock(struct file *filp, struct file_lock *fl);
-+bool vfs_inode_has_locks(struct inode *inode);
-+int locks_lock_inode_wait(struct inode *inode, struct file_lock *fl);
-+
-+void locks_init_lease(struct file_lease *);
-+void locks_free_lease(struct file_lease *fl);
-+struct file_lease *locks_alloc_lease(void);
-+
-+#define LEASE_BREAK_LEASE		BIT(0)	// break leases and delegations
-+#define LEASE_BREAK_DELEG		BIT(1)	// break delegations only
-+#define LEASE_BREAK_LAYOUT		BIT(2)	// break layouts only
-+#define LEASE_BREAK_NONBLOCK		BIT(3)	// non-blocking break
-+#define LEASE_BREAK_OPEN_RDONLY		BIT(4)	// readonly open event
-+
-+int __break_lease(struct inode *inode, unsigned int flags);
-+void lease_get_mtime(struct inode *, struct timespec64 *time);
-+int generic_setlease(struct file *, int, struct file_lease **, void **priv);
-+int kernel_setlease(struct file *, int, struct file_lease **, void **);
-+int vfs_setlease(struct file *, int, struct file_lease **, void **);
-+int lease_modify(struct file_lease *, int, struct list_head *);
-+
-+struct notifier_block;
-+int lease_register_notifier(struct notifier_block *);
-+void lease_unregister_notifier(struct notifier_block *);
-+
-+struct files_struct;
-+void show_fd_locks(struct seq_file *f,
-+			 struct file *filp, struct files_struct *files);
-+bool locks_owner_has_blockers(struct file_lock_context *flctx,
-+			fl_owner_t owner);
-+
-+static inline struct file_lock_context *
-+locks_inode_context(const struct inode *inode)
-+{
-+	return smp_load_acquire(&inode->i_flctx);
-+}
-+
-+#else /* !CONFIG_FILE_LOCKING */
-+static inline int fcntl_getlk(struct file *file, unsigned int cmd,
-+			      struct flock __user *user)
-+{
-+	return -EINVAL;
-+}
-+
-+static inline int fcntl_setlk(unsigned int fd, struct file *file,
-+			      unsigned int cmd, struct flock __user *user)
-+{
-+	return -EACCES;
-+}
-+
-+#if BITS_PER_LONG == 32
-+static inline int fcntl_getlk64(struct file *file, unsigned int cmd,
-+				struct flock64 *user)
-+{
-+	return -EINVAL;
-+}
-+
-+static inline int fcntl_setlk64(unsigned int fd, struct file *file,
-+				unsigned int cmd, struct flock64 *user)
-+{
-+	return -EACCES;
-+}
-+#endif
-+static inline int fcntl_setlease(unsigned int fd, struct file *filp, int arg)
-+{
-+	return -EINVAL;
-+}
-+
-+static inline int fcntl_getlease(struct file *filp)
-+{
-+	return F_UNLCK;
-+}
-+
-+static inline int fcntl_setdeleg(unsigned int fd, struct file *filp, struct delegation *deleg)
-+{
-+	return -EINVAL;
-+}
-+
-+static inline int fcntl_getdeleg(struct file *filp, struct delegation *deleg)
-+{
-+	return -EINVAL;
-+}
-+
-+static inline bool lock_is_unlock(struct file_lock *fl)
-+{
-+	return false;
-+}
-+
-+static inline bool lock_is_read(struct file_lock *fl)
-+{
-+	return false;
-+}
-+
-+static inline bool lock_is_write(struct file_lock *fl)
-+{
-+	return false;
-+}
-+
-+static inline void locks_wake_up(struct file_lock *fl)
-+{
-+}
-+
-+static inline void
-+locks_free_lock_context(struct inode *inode)
-+{
-+}
-+
-+static inline void locks_init_lock(struct file_lock *fl)
-+{
-+	return;
-+}
-+
-+static inline void locks_init_lease(struct file_lease *fl)
-+{
-+	return;
-+}
-+
-+static inline void locks_copy_conflock(struct file_lock *new, struct file_lock *fl)
-+{
-+	return;
-+}
-+
-+static inline void locks_copy_lock(struct file_lock *new, struct file_lock *fl)
-+{
-+	return;
-+}
-+
-+static inline void locks_remove_posix(struct file *filp, fl_owner_t owner)
-+{
-+	return;
-+}
-+
-+static inline void locks_remove_file(struct file *filp)
-+{
-+	return;
-+}
-+
-+static inline void posix_test_lock(struct file *filp, struct file_lock *fl)
-+{
-+	return;
-+}
-+
-+static inline int posix_lock_file(struct file *filp, struct file_lock *fl,
-+				  struct file_lock *conflock)
-+{
-+	return -ENOLCK;
-+}
-+
-+static inline int locks_delete_block(struct file_lock *waiter)
-+{
-+	return -ENOENT;
-+}
-+
-+static inline int vfs_test_lock(struct file *filp, struct file_lock *fl)
-+{
-+	return 0;
-+}
-+
-+static inline int vfs_lock_file(struct file *filp, unsigned int cmd,
-+				struct file_lock *fl, struct file_lock *conf)
-+{
-+	return -ENOLCK;
-+}
-+
-+static inline int vfs_cancel_lock(struct file *filp, struct file_lock *fl)
-+{
-+	return 0;
-+}
-+
-+static inline bool vfs_inode_has_locks(struct inode *inode)
-+{
-+	return false;
-+}
-+
-+static inline int locks_lock_inode_wait(struct inode *inode, struct file_lock *fl)
-+{
-+	return -ENOLCK;
-+}
-+
-+static inline int __break_lease(struct inode *inode, unsigned int flags)
-+{
-+	return 0;
-+}
-+
-+static inline void lease_get_mtime(struct inode *inode,
-+				   struct timespec64 *time)
-+{
-+	return;
-+}
-+
-+static inline int generic_setlease(struct file *filp, int arg,
-+				    struct file_lease **flp, void **priv)
-+{
-+	return -EINVAL;
-+}
-+
-+static inline int kernel_setlease(struct file *filp, int arg,
-+			       struct file_lease **lease, void **priv)
-+{
-+	return -EINVAL;
-+}
-+
-+static inline int vfs_setlease(struct file *filp, int arg,
-+			       struct file_lease **lease, void **priv)
-+{
-+	return -EINVAL;
-+}
-+
-+static inline int lease_modify(struct file_lease *fl, int arg,
-+			       struct list_head *dispose)
-+{
-+	return -EINVAL;
-+}
-+
-+struct files_struct;
-+static inline void show_fd_locks(struct seq_file *f,
-+			struct file *filp, struct files_struct *files) {}
-+static inline bool locks_owner_has_blockers(struct file_lock_context *flctx,
-+			fl_owner_t owner)
-+{
-+	return false;
-+}
-+
-+static inline struct file_lock_context *
-+locks_inode_context(const struct inode *inode)
-+{
-+	return NULL;
-+}
-+
-+#endif /* !CONFIG_FILE_LOCKING */
-+
-+/* for walking lists of file_locks linked by fl_list */
-+#define for_each_file_lock(_fl, _head)	list_for_each_entry(_fl, _head, c.flc_list)
-+
-+static inline int locks_lock_file_wait(struct file *filp, struct file_lock *fl)
-+{
-+	return locks_lock_inode_wait(file_inode(filp), fl);
-+}
-+
-+#ifdef CONFIG_FILE_LOCKING
-+static inline unsigned int openmode_to_lease_flags(unsigned int mode)
-+{
-+	unsigned int flags = 0;
-+
-+	if ((mode & O_ACCMODE) == O_RDONLY)
-+		flags |= LEASE_BREAK_OPEN_RDONLY;
-+	if (mode & O_NONBLOCK)
-+		flags |= LEASE_BREAK_NONBLOCK;
-+	return flags;
-+}
-+
-+static inline int break_lease(struct inode *inode, unsigned int mode)
-+{
-+	struct file_lock_context *flctx;
-+
-+	/*
-+	 * Since this check is lockless, we must ensure that any refcounts
-+	 * taken are done before checking i_flctx->flc_lease. Otherwise, we
-+	 * could end up racing with tasks trying to set a new lease on this
-+	 * file.
-+	 */
-+	flctx = READ_ONCE(inode->i_flctx);
-+	if (!flctx)
-+		return 0;
-+	smp_mb();
-+	if (!list_empty_careful(&flctx->flc_lease))
-+		return __break_lease(inode, LEASE_BREAK_LEASE | openmode_to_lease_flags(mode));
-+	return 0;
-+}
-+
-+static inline int break_deleg(struct inode *inode, unsigned int flags)
-+{
-+	struct file_lock_context *flctx;
-+
-+	/*
-+	 * Since this check is lockless, we must ensure that any refcounts
-+	 * taken are done before checking i_flctx->flc_lease. Otherwise, we
-+	 * could end up racing with tasks trying to set a new lease on this
-+	 * file.
-+	 */
-+	flctx = READ_ONCE(inode->i_flctx);
-+	if (!flctx)
-+		return 0;
-+	smp_mb();
-+	if (!list_empty_careful(&flctx->flc_lease)) {
-+		flags |= LEASE_BREAK_DELEG;
-+		return __break_lease(inode, flags);
-+	}
-+	return 0;
-+}
-+
-+struct delegated_inode {
-+	struct inode *di_inode;
-+};
-+
-+static inline bool is_delegated(struct delegated_inode *di)
-+{
-+	return di->di_inode;
-+}
-+
-+static inline int try_break_deleg(struct inode *inode,
-+				  struct delegated_inode *di)
-+{
-+	int ret;
-+
-+	ret = break_deleg(inode, LEASE_BREAK_NONBLOCK);
-+	if (ret == -EWOULDBLOCK && di) {
-+		di->di_inode = inode;
-+		ihold(inode);
-+	}
-+	return ret;
-+}
-+
-+static inline int break_deleg_wait(struct delegated_inode *di)
-+{
-+	int ret;
-+
-+	ret = break_deleg(di->di_inode, 0);
-+	iput(di->di_inode);
-+	di->di_inode = NULL;
-+	return ret;
-+}
-+
-+static inline int break_layout(struct inode *inode, bool wait)
-+{
-+	smp_mb();
-+	if (inode->i_flctx && !list_empty_careful(&inode->i_flctx->flc_lease)) {
-+		unsigned int flags = LEASE_BREAK_LAYOUT;
-+
-+		if (!wait)
-+			flags |= LEASE_BREAK_NONBLOCK;
-+
-+		return __break_lease(inode, flags);
-+	}
-+	return 0;
-+}
-+
-+#else /* !CONFIG_FILE_LOCKING */
-+struct delegated_inode { };
-+
-+static inline bool is_delegated(struct delegated_inode *di)
-+{
-+	return false;
-+}
-+
-+static inline int break_lease(struct inode *inode, bool wait)
-+{
-+	return 0;
-+}
-+
-+static inline int break_deleg(struct inode *inode, unsigned int flags)
-+{
-+	return 0;
-+}
-+
-+static inline int try_break_deleg(struct inode *inode,
-+				  struct delegated_inode *delegated_inode)
-+{
-+	return 0;
-+}
-+
-+static inline int break_deleg_wait(struct delegated_inode *delegated_inode)
-+{
-+	BUG();
-+	return 0;
-+}
-+
-+static inline int break_layout(struct inode *inode, bool wait)
-+{
-+	return 0;
-+}
-+
-+#endif /* CONFIG_FILE_LOCKING */
-+
-+#endif /* _LINUX_FILELOCK_H */
-diff --git a/include/linux/filter.h b/include/linux/filter.h
-index a5f21dc3c432..fd54fed8f95f 100644
--- a/include/linux/filter.h
-+++ b/include/linux/filter.h
-@@ -14,6 +14,7 @@
- #include <linux/printk.h>
- #include <linux/workqueue.h>
- #include <linux/sched.h>
-+#include <linux/sched/clock.h>
- #include <linux/capability.h>
- #include <linux/set_memory.h>
- #include <linux/kallsyms.h>
-@@ -68,11 +69,23 @@ struct ctl_table_header;
- /* unused opcode to mark special load instruction. Same as BPF_ABS */
- #define BPF_PROBE_MEM	0x20
- 
-+/* unused opcode to mark special ldsx instruction. Same as BPF_IND */
-+#define BPF_PROBE_MEMSX	0x40
-+
-+/* unused opcode to mark special load instruction. Same as BPF_MSH */
-+#define BPF_PROBE_MEM32	0xa0
-+
-+/* unused opcode to mark special atomic instruction */
-+#define BPF_PROBE_ATOMIC 0xe0
-+
-+/* unused opcode to mark special ldsx instruction. Same as BPF_NOSPEC */
-+#define BPF_PROBE_MEM32SX 0xc0
-+
- /* unused opcode to mark call to interpreter with arguments */
- #define BPF_CALL_ARGS	0xe0
- 
- /* unused opcode to mark speculation barrier for mitigating
- * Speculative Store Bypass
-+ * Spectre v1 and v4
-  */
- #define BPF_NOSPEC	0xc0
- 
-@@ -89,39 +102,49 @@ struct ctl_table_header;
- 
- /* ALU ops on registers, bpf_add|sub|...: dst_reg += src_reg */
- 
-#define BPF_ALU64_REG(OP, DST, SRC)				\
-+#define BPF_ALU64_REG_OFF(OP, DST, SRC, OFF)			\
- 	((struct bpf_insn) {					\
- 		.code  = BPF_ALU64 | BPF_OP(OP) | BPF_X,	\
- 		.dst_reg = DST,					\
- 		.src_reg = SRC,					\
-		.off   = 0,					\
-+		.off   = OFF,					\
- 		.imm   = 0 })
- 
-#define BPF_ALU32_REG(OP, DST, SRC)				\
-+#define BPF_ALU64_REG(OP, DST, SRC)				\
-+	BPF_ALU64_REG_OFF(OP, DST, SRC, 0)
-+
-+#define BPF_ALU32_REG_OFF(OP, DST, SRC, OFF)			\
- 	((struct bpf_insn) {					\
- 		.code  = BPF_ALU | BPF_OP(OP) | BPF_X,		\
- 		.dst_reg = DST,					\
- 		.src_reg = SRC,					\
-		.off   = 0,					\
-+		.off   = OFF,					\
- 		.imm   = 0 })
- 
-+#define BPF_ALU32_REG(OP, DST, SRC)				\
-+	BPF_ALU32_REG_OFF(OP, DST, SRC, 0)
-+
- /* ALU ops on immediates, bpf_add|sub|...: dst_reg += imm32 */
- 
-#define BPF_ALU64_IMM(OP, DST, IMM)				\
-+#define BPF_ALU64_IMM_OFF(OP, DST, IMM, OFF)			\
- 	((struct bpf_insn) {					\
- 		.code  = BPF_ALU64 | BPF_OP(OP) | BPF_K,	\
- 		.dst_reg = DST,					\
- 		.src_reg = 0,					\
-		.off   = 0,					\
-+		.off   = OFF,					\
- 		.imm   = IMM })
-+#define BPF_ALU64_IMM(OP, DST, IMM)				\
-+	BPF_ALU64_IMM_OFF(OP, DST, IMM, 0)
- 
-#define BPF_ALU32_IMM(OP, DST, IMM)				\
-+#define BPF_ALU32_IMM_OFF(OP, DST, IMM, OFF)			\
- 	((struct bpf_insn) {					\
- 		.code  = BPF_ALU | BPF_OP(OP) | BPF_K,		\
- 		.dst_reg = DST,					\
- 		.src_reg = 0,					\
-		.off   = 0,					\
-+		.off   = OFF,					\
- 		.imm   = IMM })
-+#define BPF_ALU32_IMM(OP, DST, IMM)				\
-+	BPF_ALU32_IMM_OFF(OP, DST, IMM, 0)
- 
- /* Endianess conversion, cpu_to_{l,b}e(), {l,b}e_to_cpu() */
- 
-@@ -133,6 +156,16 @@ struct ctl_table_header;
- 		.off   = 0,					\
- 		.imm   = LEN })
- 
-+/* Byte Swap, bswap16/32/64 */
-+
-+#define BPF_BSWAP(DST, LEN)					\
-+	((struct bpf_insn) {					\
-+		.code  = BPF_ALU64 | BPF_END | BPF_SRC(BPF_TO_LE),	\
-+		.dst_reg = DST,					\
-+		.src_reg = 0,					\
-+		.off   = 0,					\
-+		.imm   = LEN })
-+
- /* Short form of mov, dst_reg = src_reg */
- 
- #define BPF_MOV64_REG(DST, SRC)					\
-@@ -151,6 +184,25 @@ struct ctl_table_header;
- 		.off   = 0,					\
- 		.imm   = 0 })
- 
-+/* Special (internal-only) form of mov, used to resolve per-CPU addrs:
-+ * dst_reg = src_reg + <percpu_base_off>
-+ * BPF_ADDR_PERCPU is used as a special insn->off value.
-+ */
-+#define BPF_ADDR_PERCPU	(-1)
-+
-+#define BPF_MOV64_PERCPU_REG(DST, SRC)				\
-+	((struct bpf_insn) {					\
-+		.code  = BPF_ALU64 | BPF_MOV | BPF_X,		\
-+		.dst_reg = DST,					\
-+		.src_reg = SRC,					\
-+		.off   = BPF_ADDR_PERCPU,			\
-+		.imm   = 0 })
-+
-+static inline bool insn_is_mov_percpu_addr(const struct bpf_insn *insn)
-+{
-+	return insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->off == BPF_ADDR_PERCPU;
-+}
-+
- /* Short form of mov, dst_reg = imm32 */
- 
- #define BPF_MOV64_IMM(DST, IMM)					\
-@@ -169,6 +221,24 @@ struct ctl_table_header;
- 		.off   = 0,					\
- 		.imm   = IMM })
- 
-+/* Short form of movsx, dst_reg = (s8,s16,s32)src_reg */
-+
-+#define BPF_MOVSX64_REG(DST, SRC, OFF)				\
-+	((struct bpf_insn) {					\
-+		.code  = BPF_ALU64 | BPF_MOV | BPF_X,		\
-+		.dst_reg = DST,					\
-+		.src_reg = SRC,					\
-+		.off   = OFF,					\
-+		.imm   = 0 })
-+
-+#define BPF_MOVSX32_REG(DST, SRC, OFF)				\
-+	((struct bpf_insn) {					\
-+		.code  = BPF_ALU | BPF_MOV | BPF_X,		\
-+		.dst_reg = DST,					\
-+		.src_reg = SRC,					\
-+		.off   = OFF,					\
-+		.imm   = 0 })
-+
- /* Special form of mov32, used for doing explicit zero extension on dst. */
- #define BPF_ZEXT_REG(DST)					\
- 	((struct bpf_insn) {					\
-@@ -183,6 +253,16 @@ static inline bool insn_is_zext(const struct bpf_insn *insn)
- 	return insn->code == (BPF_ALU | BPF_MOV | BPF_X) && insn->imm == 1;
- }
- 
-+/* addr_space_cast from as(0) to as(1) is for converting bpf arena pointers
-+ * to pointers in user vma.
-+ */
-+static inline bool insn_is_cast_user(const struct bpf_insn *insn)
-+{
-+	return insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) &&
-+			      insn->off == BPF_ADDR_SPACE_CAST &&
-+			      insn->imm == 1U << 16;
-+}
-+
- /* BPF_LD_IMM64 macro encodes single 'load 64-bit immediate' insn */
- #define BPF_LD_IMM64(DST, IMM)					\
- 	BPF_LD_IMM64_RAW(DST, 0, IMM)
-@@ -253,6 +333,16 @@ static inline bool insn_is_zext(const struct bpf_insn *insn)
- 		.off   = OFF,					\
- 		.imm   = 0 })
- 
-+/* Memory load, dst_reg = *(signed size *) (src_reg + off16) */
-+
-+#define BPF_LDX_MEMSX(SIZE, DST, SRC, OFF)			\
-+	((struct bpf_insn) {					\
-+		.code  = BPF_LDX | BPF_SIZE(SIZE) | BPF_MEMSX,	\
-+		.dst_reg = DST,					\
-+		.src_reg = SRC,					\
-+		.off   = OFF,					\
-+		.imm   = 0 })
-+
- /* Memory store, *(uint *) (dst_reg + off16) = src_reg */
- 
- #define BPF_STX_MEM(SIZE, DST, SRC, OFF)			\
-@@ -277,6 +367,8 @@ static inline bool insn_is_zext(const struct bpf_insn *insn)
-  *   BPF_XOR | BPF_FETCH      src_reg = atomic_fetch_xor(dst_reg + off16, src_reg);
-  *   BPF_XCHG                 src_reg = atomic_xchg(dst_reg + off16, src_reg)
-  *   BPF_CMPXCHG              r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg)
-+ *   BPF_LOAD_ACQ             dst_reg = smp_load_acquire(src_reg + off16)
-+ *   BPF_STORE_REL            smp_store_release(dst_reg + off16, src_reg)
-  */
- 
- #define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF)			\
-@@ -350,6 +442,16 @@ static inline bool insn_is_zext(const struct bpf_insn *insn)
- 		.off   = OFF,					\
- 		.imm   = 0 })
- 
-+/* Unconditional jumps, gotol pc + imm32 */
-+
-+#define BPF_JMP32_A(IMM)					\
-+	((struct bpf_insn) {					\
-+		.code  = BPF_JMP32 | BPF_JA,			\
-+		.dst_reg = 0,					\
-+		.src_reg = 0,					\
-+		.off   = 0,					\
-+		.imm   = IMM })
-+
- /* Relative call */
- 
- #define BPF_CALL_REL(TGT)					\
-@@ -372,6 +474,16 @@ static inline bool insn_is_zext(const struct bpf_insn *insn)
- 		.off   = 0,					\
- 		.imm   = BPF_CALL_IMM(FUNC) })
- 
-+/* Kfunc call */
-+
-+#define BPF_CALL_KFUNC(OFF, IMM)				\
-+	((struct bpf_insn) {					\
-+		.code  = BPF_JMP | BPF_CALL,			\
-+		.dst_reg = 0,					\
-+		.src_reg = BPF_PSEUDO_KFUNC_CALL,		\
-+		.off   = OFF,					\
-+		.imm   = IMM })
-+
- /* Raw code statement block */
- 
- #define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM)			\
-@@ -495,24 +607,27 @@ static inline bool insn_is_zext(const struct bpf_insn *insn)
- 	__BPF_MAP(n, __BPF_DECL_ARGS, __BPF_N, u64, __ur_1, u64, __ur_2,       \
- 		  u64, __ur_3, u64, __ur_4, u64, __ur_5)
- 
-#define BPF_CALL_x(x, name, ...)					       \
-+#define BPF_CALL_x(x, attr, name, ...)					       \
- 	static __always_inline						       \
- 	u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__));   \
- 	typedef u64 (*btf_##name)(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \
-	u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__));	       \
-	u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__))	       \
-+	attr u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__));    \
-+	attr u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__))     \
- 	{								       \
- 		return ((btf_##name)____##name)(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\
- 	}								       \
- 	static __always_inline						       \
- 	u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__))
- 
-#define BPF_CALL_0(name, ...)	BPF_CALL_x(0, name, __VA_ARGS__)
-#define BPF_CALL_1(name, ...)	BPF_CALL_x(1, name, __VA_ARGS__)
-#define BPF_CALL_2(name, ...)	BPF_CALL_x(2, name, __VA_ARGS__)
-#define BPF_CALL_3(name, ...)	BPF_CALL_x(3, name, __VA_ARGS__)
-#define BPF_CALL_4(name, ...)	BPF_CALL_x(4, name, __VA_ARGS__)
-#define BPF_CALL_5(name, ...)	BPF_CALL_x(5, name, __VA_ARGS__)
-+#define __NOATTR
-+#define BPF_CALL_0(name, ...)	BPF_CALL_x(0, __NOATTR, name, __VA_ARGS__)
-+#define BPF_CALL_1(name, ...)	BPF_CALL_x(1, __NOATTR, name, __VA_ARGS__)
-+#define BPF_CALL_2(name, ...)	BPF_CALL_x(2, __NOATTR, name, __VA_ARGS__)
-+#define BPF_CALL_3(name, ...)	BPF_CALL_x(3, __NOATTR, name, __VA_ARGS__)
-+#define BPF_CALL_4(name, ...)	BPF_CALL_x(4, __NOATTR, name, __VA_ARGS__)
-+#define BPF_CALL_5(name, ...)	BPF_CALL_x(5, __NOATTR, name, __VA_ARGS__)
-+
-+#define NOTRACE_BPF_CALL_1(name, ...)	BPF_CALL_x(1, notrace, name, __VA_ARGS__)
- 
- #define bpf_ctx_range(TYPE, MEMBER)						\
- 	offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1
-@@ -559,6 +674,11 @@ struct bpf_prog_stats {
- 	struct u64_stats_sync syncp;
- } __aligned(2 * sizeof(u64));
- 
-+struct bpf_timed_may_goto {
-+	u64 count;
-+	u64 timestamp;
-+};
-+
- struct sk_filter {
- 	refcount_t	refcnt;
- 	struct rcu_head	rcu;
-@@ -567,6 +687,11 @@ struct sk_filter {
- 
- DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key);
- 
-+extern struct mutex nf_conn_btf_access_lock;
-+extern int (*nfct_btf_struct_access)(struct bpf_verifier_log *log,
-+				     const struct bpf_reg_state *reg,
-+				     int off, int size);
-+
- typedef unsigned int (*bpf_dispatcher_fn)(const void *ctx,
- 					  const struct bpf_insn *insnsi,
- 					  unsigned int (*bpf_func)(const void *,
-@@ -581,15 +706,19 @@ static __always_inline u32 __bpf_prog_run(const struct bpf_prog *prog,
- 	cant_migrate();
- 	if (static_branch_unlikely(&bpf_stats_enabled_key)) {
- 		struct bpf_prog_stats *stats;
-		u64 start = sched_clock();
-+		u64 duration, start = sched_clock();
- 		unsigned long flags;
- 
- 		ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
-		stats = this_cpu_ptr(prog->stats);
-		flags = u64_stats_update_begin_irqsave(&stats->syncp);
-		u64_stats_inc(&stats->cnt);
-		u64_stats_add(&stats->nsecs, sched_clock() - start);
-		u64_stats_update_end_irqrestore(&stats->syncp, flags);
-+
-+		duration = sched_clock() - start;
-+		if (likely(prog->stats)) {
-+			stats = this_cpu_ptr(prog->stats);
-+			flags = u64_stats_update_begin_irqsave(&stats->syncp);
-+			u64_stats_inc(&stats->cnt);
-+			u64_stats_add(&stats->nsecs, duration);
-+			u64_stats_update_end_irqrestore(&stats->syncp, flags);
-+		}
- 	} else {
- 		ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
- 	}
-@@ -636,21 +765,128 @@ struct bpf_nh_params {
- 	};
- };
- 
-+/* flags for bpf_redirect_info kern_flags */
-+#define BPF_RI_F_RF_NO_DIRECT	BIT(0)	/* no napi_direct on return_frame */
-+#define BPF_RI_F_RI_INIT	BIT(1)
-+#define BPF_RI_F_CPU_MAP_INIT	BIT(2)
-+#define BPF_RI_F_DEV_MAP_INIT	BIT(3)
-+#define BPF_RI_F_XSK_MAP_INIT	BIT(4)
-+
- struct bpf_redirect_info {
-	u32 flags;
-	u32 tgt_index;
-+	u64 tgt_index;
- 	void *tgt_value;
- 	struct bpf_map *map;
-+	u32 flags;
- 	u32 map_id;
- 	enum bpf_map_type map_type;
-	u32 kern_flags;
- 	struct bpf_nh_params nh;
-+	u32 kern_flags;
- };
- 
-DECLARE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info);
-+struct bpf_net_context {
-+	struct bpf_redirect_info ri;
-+	struct list_head cpu_map_flush_list;
-+	struct list_head dev_map_flush_list;
-+	struct list_head xskmap_map_flush_list;
-+};
- 
-/* flags for bpf_redirect_info kern_flags */
-#define BPF_RI_F_RF_NO_DIRECT	BIT(0)	/* no napi_direct on return_frame */
-+static inline struct bpf_net_context *bpf_net_ctx_set(struct bpf_net_context *bpf_net_ctx)
-+{
-+	struct task_struct *tsk = current;
-+
-+	if (tsk->bpf_net_context != NULL)
-+		return NULL;
-+	bpf_net_ctx->ri.kern_flags = 0;
-+
-+	tsk->bpf_net_context = bpf_net_ctx;
-+	return bpf_net_ctx;
-+}
-+
-+static inline void bpf_net_ctx_clear(struct bpf_net_context *bpf_net_ctx)
-+{
-+	if (bpf_net_ctx)
-+		current->bpf_net_context = NULL;
-+}
-+
-+static inline struct bpf_net_context *bpf_net_ctx_get(void)
-+{
-+	return current->bpf_net_context;
-+}
-+
-+static inline struct bpf_redirect_info *bpf_net_ctx_get_ri(void)
-+{
-+	struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get();
-+
-+	if (!(bpf_net_ctx->ri.kern_flags & BPF_RI_F_RI_INIT)) {
-+		memset(&bpf_net_ctx->ri, 0, offsetof(struct bpf_net_context, ri.nh));
-+		bpf_net_ctx->ri.kern_flags |= BPF_RI_F_RI_INIT;
-+	}
-+
-+	return &bpf_net_ctx->ri;
-+}
-+
-+static inline struct list_head *bpf_net_ctx_get_cpu_map_flush_list(void)
-+{
-+	struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get();
-+
-+	if (!(bpf_net_ctx->ri.kern_flags & BPF_RI_F_CPU_MAP_INIT)) {
-+		INIT_LIST_HEAD(&bpf_net_ctx->cpu_map_flush_list);
-+		bpf_net_ctx->ri.kern_flags |= BPF_RI_F_CPU_MAP_INIT;
-+	}
-+
-+	return &bpf_net_ctx->cpu_map_flush_list;
-+}
-+
-+static inline struct list_head *bpf_net_ctx_get_dev_flush_list(void)
-+{
-+	struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get();
-+
-+	if (!(bpf_net_ctx->ri.kern_flags & BPF_RI_F_DEV_MAP_INIT)) {
-+		INIT_LIST_HEAD(&bpf_net_ctx->dev_map_flush_list);
-+		bpf_net_ctx->ri.kern_flags |= BPF_RI_F_DEV_MAP_INIT;
-+	}
-+
-+	return &bpf_net_ctx->dev_map_flush_list;
-+}
-+
-+static inline struct list_head *bpf_net_ctx_get_xskmap_flush_list(void)
-+{
-+	struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get();
-+
-+	if (!(bpf_net_ctx->ri.kern_flags & BPF_RI_F_XSK_MAP_INIT)) {
-+		INIT_LIST_HEAD(&bpf_net_ctx->xskmap_map_flush_list);
-+		bpf_net_ctx->ri.kern_flags |= BPF_RI_F_XSK_MAP_INIT;
-+	}
-+
-+	return &bpf_net_ctx->xskmap_map_flush_list;
-+}
-+
-+static inline void bpf_net_ctx_get_all_used_flush_lists(struct list_head **lh_map,
-+							struct list_head **lh_dev,
-+							struct list_head **lh_xsk)
-+{
-+	struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get();
-+	u32 kern_flags = bpf_net_ctx->ri.kern_flags;
-+	struct list_head *lh;
-+
-+	*lh_map = *lh_dev = *lh_xsk = NULL;
-+
-+	if (!IS_ENABLED(CONFIG_BPF_SYSCALL))
-+		return;
-+
-+	lh = &bpf_net_ctx->dev_map_flush_list;
-+	if (kern_flags & BPF_RI_F_DEV_MAP_INIT && !list_empty(lh))
-+		*lh_dev = lh;
-+
-+	lh = &bpf_net_ctx->cpu_map_flush_list;
-+	if (kern_flags & BPF_RI_F_CPU_MAP_INIT && !list_empty(lh))
-+		*lh_map = lh;
-+
-+	lh = &bpf_net_ctx->xskmap_map_flush_list;
-+	if (IS_ENABLED(CONFIG_XDP_SOCKETS) &&
-+	    kern_flags & BPF_RI_F_XSK_MAP_INIT && !list_empty(lh))
-+		*lh_xsk = lh;
-+}
- 
- /* Compute the linear packet data range [data, data_end) which
-  * will be accessed by various program types (cls_bpf, act_bpf,
-@@ -667,6 +903,26 @@ static inline void bpf_compute_data_pointers(struct sk_buff *skb)
- 	cb->data_end  = skb->data + skb_headlen(skb);
- }
- 
-+static inline int bpf_prog_run_data_pointers(
-+	const struct bpf_prog *prog,
-+	struct sk_buff *skb)
-+{
-+	struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb;
-+	void *save_data_meta, *save_data_end;
-+	int res;
-+
-+	save_data_meta = cb->data_meta;
-+	save_data_end = cb->data_end;
-+
-+	bpf_compute_data_pointers(skb);
-+	res = bpf_prog_run(prog, skb);
-+
-+	cb->data_meta = save_data_meta;
-+	cb->data_end = save_data_end;
-+
-+	return res;
-+}
-+
- /* Similar to bpf_compute_data_pointers(), except that save orginal
-  * data in cb->data and cb->meta_data for restore.
-  */
-@@ -679,7 +935,7 @@ static inline void bpf_compute_and_save_data_end(
- 	cb->data_end  = skb->data + skb_headlen(skb);
- }
- 
-/* Restore data saved by bpf_compute_data_pointers(). */
-+/* Restore data saved by bpf_compute_and_save_data_end(). */
- static inline void bpf_restore_data_end(
- 	struct sk_buff *skb, void *saved_data_end)
- {
-@@ -759,23 +1015,6 @@ DECLARE_STATIC_KEY_FALSE(bpf_master_redirect_enabled_key);
- 
- u32 xdp_master_redirect(struct xdp_buff *xdp);
- 
-static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
-					    struct xdp_buff *xdp)
-{
-	/* Driver XDP hooks are invoked within a single NAPI poll cycle and thus
-	 * under local_bh_disable(), which provides the needed RCU protection
-	 * for accessing map entries.
-	 */
-	u32 act = __bpf_prog_run(prog, xdp, BPF_DISPATCHER_FUNC(xdp));
-
-	if (static_branch_unlikely(&bpf_master_redirect_enabled_key)) {
-		if (act == XDP_TX && netif_is_bond_slave(xdp->rxq->dev))
-			act = xdp_master_redirect(xdp);
-	}
-
-	return act;
-}
-
- void bpf_prog_change_xdp(struct bpf_prog *prev_prog, struct bpf_prog *prog);
- 
- static inline u32 bpf_prog_insn_size(const struct bpf_prog *prog)
-@@ -783,12 +1022,6 @@ static inline u32 bpf_prog_insn_size(const struct bpf_prog *prog)
- 	return prog->len * sizeof(struct bpf_insn);
- }
- 
-static inline u32 bpf_prog_tag_scratch_size(const struct bpf_prog *prog)
-{
-	return round_up(bpf_prog_insn_size(prog) +
-			sizeof(__be64) + 1, SHA1_BLOCK_SIZE);
-}
-
- static inline unsigned int bpf_prog_size(unsigned int proglen)
- {
- 	return max(sizeof(struct bpf_prog),
-@@ -841,27 +1074,38 @@ bpf_ctx_narrow_access_offset(u32 off, u32 size, u32 size_default)
- 
- #define bpf_classic_proglen(fprog) (fprog->len * sizeof(fprog->filter[0]))
- 
-static inline void bpf_prog_lock_ro(struct bpf_prog *fp)
-+static inline int __must_check bpf_prog_lock_ro(struct bpf_prog *fp)
- {
- #ifndef CONFIG_BPF_JIT_ALWAYS_ON
- 	if (!fp->jited) {
- 		set_vm_flush_reset_perms(fp);
-		set_memory_ro((unsigned long)fp, fp->pages);
-+		return set_memory_ro((unsigned long)fp, fp->pages);
- 	}
- #endif
-+	return 0;
- }
- 
-static inline void bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr)
-+static inline int __must_check
-+bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr)
- {
- 	set_vm_flush_reset_perms(hdr);
-	set_memory_ro((unsigned long)hdr, hdr->size >> PAGE_SHIFT);
-	set_memory_x((unsigned long)hdr, hdr->size >> PAGE_SHIFT);
-+	return set_memory_rox((unsigned long)hdr, hdr->size >> PAGE_SHIFT);
- }
- 
-int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap);
-+int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap,
-+		       enum skb_drop_reason *reason);
-+
- static inline int sk_filter(struct sock *sk, struct sk_buff *skb)
- {
-	return sk_filter_trim_cap(sk, skb, 1);
-+	enum skb_drop_reason ignore_reason;
-+
-+	return sk_filter_trim_cap(sk, skb, 1, &ignore_reason);
-+}
-+
-+static inline int sk_filter_reason(struct sock *sk, struct sk_buff *skb,
-+				   enum skb_drop_reason *reason)
-+{
-+	return sk_filter_trim_cap(sk, skb, 1, reason);
- }
- 
- struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);
-@@ -869,7 +1113,6 @@ void bpf_prog_free(struct bpf_prog *fp);
- 
- bool bpf_opcode_in_insntable(u8 code);
- 
-void bpf_prog_free_linfo(struct bpf_prog *prog);
- void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,
- 			       const u32 *insn_to_jit_off);
- int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog);
-@@ -900,8 +1143,7 @@ int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk);
- int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk);
- void sk_reuseport_prog_free(struct bpf_prog *prog);
- int sk_detach_filter(struct sock *sk);
-int sk_get_filter(struct sock *sk, struct sock_filter __user *filter,
-		  unsigned int len);
-+int sk_get_filter(struct sock *sk, sockptr_t optval, unsigned int len);
- 
- bool sk_filter_charge(struct sock *sk, struct sk_filter *fp);
- void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
-@@ -914,9 +1156,22 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
- struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
- void bpf_jit_compile(struct bpf_prog *prog);
- bool bpf_jit_needs_zext(void);
-+bool bpf_jit_inlines_helper_call(s32 imm);
- bool bpf_jit_supports_subprog_tailcalls(void);
-+bool bpf_jit_supports_percpu_insn(void);
- bool bpf_jit_supports_kfunc_call(void);
-bool bpf_helper_changes_pkt_data(void *func);
-+bool bpf_jit_supports_far_kfunc_call(void);
-+bool bpf_jit_supports_exceptions(void);
-+bool bpf_jit_supports_ptr_xchg(void);
-+bool bpf_jit_supports_arena(void);
-+bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
-+bool bpf_jit_supports_private_stack(void);
-+bool bpf_jit_supports_timed_may_goto(void);
-+u64 bpf_arch_uaddress_limit(void);
-+void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
-+u64 arch_bpf_timed_may_goto(void);
-+u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *);
-+bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id);
- 
- static inline bool bpf_dump_raw_ok(const struct cred *cred)
- {
-@@ -930,25 +1185,23 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
- 				       const struct bpf_insn *patch, u32 len);
- int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt);
- 
-void bpf_clear_redirect_map(struct bpf_map *map);
-
- static inline bool xdp_return_frame_no_direct(void)
- {
-	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
- 
- 	return ri->kern_flags & BPF_RI_F_RF_NO_DIRECT;
- }
- 
- static inline void xdp_set_return_frame_no_direct(void)
- {
-	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
- 
- 	ri->kern_flags |= BPF_RI_F_RF_NO_DIRECT;
- }
- 
- static inline void xdp_clear_return_frame_no_direct(void)
- {
-	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
- 
- 	ri->kern_flags &= ~BPF_RI_F_RF_NO_DIRECT;
- }
-@@ -975,23 +1228,18 @@ static inline int xdp_ok_fwd_dev(const struct net_device *fwd,
-  * This does not appear to be a real limitation for existing software.
-  */
- int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
-			    struct xdp_buff *xdp, struct bpf_prog *prog);
-+			    struct xdp_buff *xdp, const struct bpf_prog *prog);
- int xdp_do_redirect(struct net_device *dev,
- 		    struct xdp_buff *xdp,
-		    struct bpf_prog *prog);
-+		    const struct bpf_prog *prog);
- int xdp_do_redirect_frame(struct net_device *dev,
- 			  struct xdp_buff *xdp,
- 			  struct xdp_frame *xdpf,
-			  struct bpf_prog *prog);
-+			  const struct bpf_prog *prog);
- void xdp_do_flush(void);
- 
-/* The xdp_do_flush_map() helper has been renamed to drop the _map suffix, as
- * it is no longer only flushing maps. Keep this define for compatibility
- * until all drivers are updated - do not use xdp_do_flush_map() in new code!
- */
-#define xdp_do_flush_map xdp_do_flush
-
-void bpf_warn_invalid_xdp_action(struct net_device *dev, struct bpf_prog *prog, u32 act);
-+void bpf_warn_invalid_xdp_action(const struct net_device *dev,
-+				 const struct bpf_prog *prog, u32 act);
- 
- #ifdef CONFIG_INET
- struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
-@@ -1018,6 +1266,8 @@ extern long bpf_jit_limit_max;
- 
- typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size);
- 
-+void bpf_jit_fill_hole_with_zero(void *area, unsigned int size);
-+
- struct bpf_binary_header *
- bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
- 		     unsigned int alignment,
-@@ -1030,6 +1280,9 @@ void bpf_jit_free(struct bpf_prog *fp);
- struct bpf_binary_header *
- bpf_jit_binary_pack_hdr(const struct bpf_prog *fp);
- 
-+void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns);
-+void bpf_prog_pack_free(void *ptr, u32 size);
-+
- static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)
- {
- 	return list_empty(&fp->aux->ksym.lnode) ||
-@@ -1042,8 +1295,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image,
- 			  struct bpf_binary_header **rw_hdr,
- 			  u8 **rw_image,
- 			  bpf_jit_fill_hole_t bpf_fill_ill_insns);
-int bpf_jit_binary_pack_finalize(struct bpf_prog *prog,
-				 struct bpf_binary_header *ro_header,
-+int bpf_jit_binary_pack_finalize(struct bpf_binary_header *ro_header,
- 				 struct bpf_binary_header *rw_header);
- void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header,
- 			      struct bpf_binary_header *rw_header);
-@@ -1055,13 +1307,15 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,
- 			  const struct bpf_insn *insn, bool extra_pass,
- 			  u64 *func_addr, bool *func_addr_fixed);
- 
-+const char *bpf_jit_get_prog_name(struct bpf_prog *prog);
-+
- struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp);
- void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other);
- 
- static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen,
- 				u32 pass, void *image)
- {
-	pr_err("flen=%u proglen=%u pass=%u image=%pK from=%s pid=%d\n", flen,
-+	pr_err("flen=%u proglen=%u pass=%u image=%p from=%s pid=%d\n", flen,
- 	       proglen, pass, image, current->comm, task_pid_nr(current));
- 
- 	if (image)
-@@ -1100,7 +1354,7 @@ static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog)
- 		return false;
- 	if (!bpf_jit_harden)
- 		return false;
-	if (bpf_jit_harden == 1 && capable(CAP_SYS_ADMIN))
-+	if (bpf_jit_harden == 1 && bpf_token_capable(prog->aux->token, CAP_BPF))
- 		return false;
- 
- 	return true;
-@@ -1121,17 +1375,18 @@ static inline bool bpf_jit_kallsyms_enabled(void)
- 	return false;
- }
- 
-const char *__bpf_address_lookup(unsigned long addr, unsigned long *size,
-+int __bpf_address_lookup(unsigned long addr, unsigned long *size,
- 				 unsigned long *off, char *sym);
- bool is_bpf_text_address(unsigned long addr);
- int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
- 		    char *sym);
-+struct bpf_prog *bpf_prog_ksym_find(unsigned long addr);
- 
-static inline const char *
-+static inline int
- bpf_address_lookup(unsigned long addr, unsigned long *size,
- 		   unsigned long *off, char **modname, char *sym)
- {
-	const char *ret = __bpf_address_lookup(addr, size, off, sym);
-+	int ret = __bpf_address_lookup(addr, size, off, sym);
- 
- 	if (ret && modname)
- 		*modname = NULL;
-@@ -1175,11 +1430,11 @@ static inline bool bpf_jit_kallsyms_enabled(void)
- 	return false;
- }
- 
-static inline const char *
-+static inline int
- __bpf_address_lookup(unsigned long addr, unsigned long *size,
- 		     unsigned long *off, char *sym)
- {
-	return NULL;
-+	return 0;
- }
- 
- static inline bool is_bpf_text_address(unsigned long addr)
-@@ -1193,11 +1448,16 @@ static inline int bpf_get_kallsym(unsigned int symnum, unsigned long *value,
- 	return -ERANGE;
- }
- 
-static inline const char *
-+static inline struct bpf_prog *bpf_prog_ksym_find(unsigned long addr)
-+{
-+	return NULL;
-+}
-+
-+static inline int
- bpf_address_lookup(unsigned long addr, unsigned long *size,
- 		   unsigned long *off, char **modname, char *sym)
- {
-	return NULL;
-+	return 0;
- }
- 
- static inline void bpf_prog_kallsyms_add(struct bpf_prog *fp)
-@@ -1277,13 +1537,14 @@ static inline int bpf_tell_extensions(void)
- 
- struct bpf_sock_addr_kern {
- 	struct sock *sk;
-	struct sockaddr *uaddr;
-+	struct sockaddr_unsized *uaddr;
- 	/* Temporary "register" to make indirect stores to nested structures
- 	 * defined above. We need three registers to make such a store, but
- 	 * only two (src and dst) are available at convert_ctx_access time
- 	 */
- 	u64 tmp_reg;
- 	void *t_ctx;	/* Attach type specific context. */
-+	u32 uaddrlen;
- };
- 
- struct bpf_sock_ops_kern {
-@@ -1298,6 +1559,7 @@ struct bpf_sock_ops_kern {
- 	void	*skb_data_end;
- 	u8	op;
- 	u8	is_fullsock;
-+	u8	is_locked_tcp_sock;
- 	u8	remaining_opt_len;
- 	u64	temp;			/* temp and everything after is not
- 					 * initialized to 0 before calling
-@@ -1312,7 +1574,7 @@ struct bpf_sock_ops_kern {
- 
- struct bpf_sysctl_kern {
- 	struct ctl_table_header *head;
-	struct ctl_table *table;
-+	const struct ctl_table *table;
- 	void *cur_val;
- 	size_t cur_len;
- 	void *new_val;
-@@ -1418,7 +1680,7 @@ extern struct static_key_false bpf_sk_lookup_enabled;
- 		_all_pass || _selected_sk ? SK_PASS : SK_DROP;		\
- 	 })
- 
-static inline bool bpf_sk_lookup_run_v4(struct net *net, int protocol,
-+static inline bool bpf_sk_lookup_run_v4(const struct net *net, int protocol,
- 					const __be32 saddr, const __be16 sport,
- 					const __be32 daddr, const u16 dport,
- 					const int ifindex, struct sock **psk)
-@@ -1455,7 +1717,7 @@ static inline bool bpf_sk_lookup_run_v4(struct net *net, int protocol,
- }
- 
- #if IS_ENABLED(CONFIG_IPV6)
-static inline bool bpf_sk_lookup_run_v6(struct net *net, int protocol,
-+static inline bool bpf_sk_lookup_run_v6(const struct net *net, int protocol,
- 					const struct in6_addr *saddr,
- 					const __be16 sport,
- 					const struct in6_addr *daddr,
-@@ -1494,18 +1756,18 @@ static inline bool bpf_sk_lookup_run_v6(struct net *net, int protocol,
- }
- #endif /* IS_ENABLED(CONFIG_IPV6) */
- 
-static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u32 ifindex,
-						  u64 flags, const u64 flag_mask,
-						  void *lookup_elem(struct bpf_map *map, u32 key))
-+static __always_inline long __bpf_xdp_redirect_map(struct bpf_map *map, u64 index,
-+						   u64 flags, const u64 flag_mask,
-+						   void *lookup_elem(struct bpf_map *map, u32 key))
- {
-	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
- 	const u64 action_mask = XDP_ABORTED | XDP_DROP | XDP_PASS | XDP_TX;
- 
- 	/* Lower bits of the flags are used as return code on lookup failure */
- 	if (unlikely(flags & ~(action_mask | flag_mask)))
- 		return XDP_ABORTED;
- 
-	ri->tgt_value = lookup_elem(map, ifindex);
-+	ri->tgt_value = lookup_elem(map, index);
- 	if (unlikely(!ri->tgt_value) && !(flags & BPF_F_BROADCAST)) {
- 		/* If the lookup fails we want to clear out the state in the
- 		 * redirect_info struct completely, so that if an eBPF program
-@@ -1517,7 +1779,7 @@ static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u32 ifind
- 		return flags & action_mask;
- 	}
- 
-	ri->tgt_index = ifindex;
-+	ri->tgt_index = index;
- 	ri->map_id = map->id;
- 	ri->map_type = map->map_type;
- 
-@@ -1532,4 +1794,64 @@ static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u32 ifind
- 	return XDP_REDIRECT;
- }
- 
-+#ifdef CONFIG_NET
-+int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len);
-+int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
-+			  u32 len, u64 flags);
-+int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len);
-+int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len);
-+void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len);
-+void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,
-+		      void *buf, unsigned long len, bool flush);
-+int __bpf_skb_meta_store_bytes(struct sk_buff *skb, u32 offset,
-+			       const void *from, u32 len, u64 flags);
-+void *bpf_skb_meta_pointer(struct sk_buff *skb, u32 offset);
-+#else /* CONFIG_NET */
-+static inline int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset,
-+				       void *to, u32 len)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset,
-+					const void *from, u32 len, u64 flags)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset,
-+				       void *buf, u32 len)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset,
-+					void *buf, u32 len)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len)
-+{
-+	return NULL;
-+}
-+
-+static inline void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off, void *buf,
-+				    unsigned long len, bool flush)
-+{
-+}
-+
-+static inline int __bpf_skb_meta_store_bytes(struct sk_buff *skb, u32 offset,
-+					     const void *from, u32 len,
-+					     u64 flags)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline void *bpf_skb_meta_pointer(struct sk_buff *skb, u32 offset)
-+{
-+	return ERR_PTR(-EOPNOTSUPP);
-+}
-+#endif /* CONFIG_NET */
-+
- #endif /* __LINUX_FILTER_H__ */
-diff --git a/include/linux/find.h b/include/linux/find.h
-index 424ef67d4a42..9d720ad92bc1 100644
--- a/include/linux/find.h
-+++ b/include/linux/find.h
-@@ -8,15 +8,44 @@
- 
- #include <linux/bitops.h>
- 
-extern unsigned long _find_next_bit(const unsigned long *addr1,
-		const unsigned long *addr2, unsigned long nbits,
-		unsigned long start, unsigned long invert, unsigned long le);
-+unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits,
-+				unsigned long start);
-+unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2,
-+					unsigned long nbits, unsigned long start);
-+unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
-+					unsigned long nbits, unsigned long start);
-+unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2,
-+					unsigned long nbits, unsigned long start);
-+unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
-+					 unsigned long start);
- extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size);
-+unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n);
-+unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
-+				unsigned long size, unsigned long n);
-+unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
-+					unsigned long size, unsigned long n);
-+unsigned long __find_nth_and_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
-+					const unsigned long *addr3, unsigned long size,
-+					unsigned long n);
- extern unsigned long _find_first_and_bit(const unsigned long *addr1,
- 					 const unsigned long *addr2, unsigned long size);
-+unsigned long _find_first_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
-+				 unsigned long size);
-+unsigned long _find_first_and_and_bit(const unsigned long *addr1, const unsigned long *addr2,
-+				      const unsigned long *addr3, unsigned long size);
- extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size);
- extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size);
- 
-+#ifdef __BIG_ENDIAN
-+unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size);
-+unsigned long _find_next_zero_bit_le(const  unsigned long *addr, unsigned
-+					long size, unsigned long offset);
-+unsigned long _find_next_bit_le(const unsigned long *addr, unsigned
-+				long size, unsigned long offset);
-+#endif
-+
-+unsigned long find_random_bit(const unsigned long *addr, unsigned long size);
-+
- #ifndef find_next_bit
- /**
-  * find_next_bit - find the next set bit in a memory region
-@@ -27,7 +56,7 @@ extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long siz
-  * Returns the bit number for the next set bit
-  * If no bits are set, returns @size.
-  */
-static inline
-+static __always_inline
- unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
- 			    unsigned long offset)
- {
-@@ -41,7 +70,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
- 		return val ? __ffs(val) : size;
- 	}
- 
-	return _find_next_bit(addr, NULL, size, offset, 0UL, 0);
-+	return _find_next_bit(addr, size, offset);
- }
- #endif
- 
-@@ -56,7 +85,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
-  * Returns the bit number for the next set bit
-  * If no bits are set, returns @size.
-  */
-static inline
-+static __always_inline
- unsigned long find_next_and_bit(const unsigned long *addr1,
- 		const unsigned long *addr2, unsigned long size,
- 		unsigned long offset)
-@@ -71,7 +100,68 @@ unsigned long find_next_and_bit(const unsigned long *addr1,
- 		return val ? __ffs(val) : size;
- 	}
- 
-	return _find_next_bit(addr1, addr2, size, offset, 0UL, 0);
-+	return _find_next_and_bit(addr1, addr2, size, offset);
-+}
-+#endif
-+
-+#ifndef find_next_andnot_bit
-+/**
-+ * find_next_andnot_bit - find the next set bit in *addr1 excluding all the bits
-+ *                        in *addr2
-+ * @addr1: The first address to base the search on
-+ * @addr2: The second address to base the search on
-+ * @size: The bitmap size in bits
-+ * @offset: The bitnumber to start searching at
-+ *
-+ * Returns the bit number for the next set bit
-+ * If no bits are set, returns @size.
-+ */
-+static __always_inline
-+unsigned long find_next_andnot_bit(const unsigned long *addr1,
-+		const unsigned long *addr2, unsigned long size,
-+		unsigned long offset)
-+{
-+	if (small_const_nbits(size)) {
-+		unsigned long val;
-+
-+		if (unlikely(offset >= size))
-+			return size;
-+
-+		val = *addr1 & ~*addr2 & GENMASK(size - 1, offset);
-+		return val ? __ffs(val) : size;
-+	}
-+
-+	return _find_next_andnot_bit(addr1, addr2, size, offset);
-+}
-+#endif
-+
-+#ifndef find_next_or_bit
-+/**
-+ * find_next_or_bit - find the next set bit in either memory regions
-+ * @addr1: The first address to base the search on
-+ * @addr2: The second address to base the search on
-+ * @size: The bitmap size in bits
-+ * @offset: The bitnumber to start searching at
-+ *
-+ * Returns the bit number for the next set bit
-+ * If no bits are set, returns @size.
-+ */
-+static __always_inline
-+unsigned long find_next_or_bit(const unsigned long *addr1,
-+		const unsigned long *addr2, unsigned long size,
-+		unsigned long offset)
-+{
-+	if (small_const_nbits(size)) {
-+		unsigned long val;
-+
-+		if (unlikely(offset >= size))
-+			return size;
-+
-+		val = (*addr1 | *addr2) & GENMASK(size - 1, offset);
-+		return val ? __ffs(val) : size;
-+	}
-+
-+	return _find_next_or_bit(addr1, addr2, size, offset);
- }
- #endif
- 
-@@ -85,7 +175,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1,
-  * Returns the bit number of the next zero bit
-  * If no bits are zero, returns @size.
-  */
-static inline
-+static __always_inline
- unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
- 				 unsigned long offset)
- {
-@@ -99,7 +189,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
- 		return val == ~0UL ? size : ffz(val);
- 	}
- 
-	return _find_next_bit(addr, NULL, size, offset, ~0UL, 0);
-+	return _find_next_zero_bit(addr, size, offset);
- }
- #endif
- 
-@@ -112,7 +202,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
-  * Returns the bit number of the first set bit.
-  * If no bits are set, returns @size.
-  */
-static inline
-+static __always_inline
- unsigned long find_first_bit(const unsigned long *addr, unsigned long size)
- {
- 	if (small_const_nbits(size)) {
-@@ -125,6 +215,90 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size)
- }
- #endif
- 
-+/**
-+ * find_nth_bit - find N'th set bit in a memory region
-+ * @addr: The address to start the search at
-+ * @size: The maximum number of bits to search
-+ * @n: The number of set bit, which position is needed, counting from 0
-+ *
-+ * The following is semantically equivalent:
-+ *	 idx = find_nth_bit(addr, size, 0);
-+ *	 idx = find_first_bit(addr, size);
-+ *
-+ * Returns the bit number of the N'th set bit.
-+ * If no such, returns >= @size.
-+ */
-+static __always_inline
-+unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n)
-+{
-+	if (n >= size)
-+		return size;
-+
-+	if (small_const_nbits(size)) {
-+		unsigned long val =  *addr & GENMASK(size - 1, 0);
-+
-+		return val ? fns(val, n) : size;
-+	}
-+
-+	return __find_nth_bit(addr, size, n);
-+}
-+
-+/**
-+ * find_nth_and_bit - find N'th set bit in 2 memory regions
-+ * @addr1: The 1st address to start the search at
-+ * @addr2: The 2nd address to start the search at
-+ * @size: The maximum number of bits to search
-+ * @n: The number of set bit, which position is needed, counting from 0
-+ *
-+ * Returns the bit number of the N'th set bit.
-+ * If no such, returns @size.
-+ */
-+static __always_inline
-+unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
-+				unsigned long size, unsigned long n)
-+{
-+	if (n >= size)
-+		return size;
-+
-+	if (small_const_nbits(size)) {
-+		unsigned long val =  *addr1 & *addr2 & GENMASK(size - 1, 0);
-+
-+		return val ? fns(val, n) : size;
-+	}
-+
-+	return __find_nth_and_bit(addr1, addr2, size, n);
-+}
-+
-+/**
-+ * find_nth_and_andnot_bit - find N'th set bit in 2 memory regions,
-+ *			     excluding those set in 3rd region
-+ * @addr1: The 1st address to start the search at
-+ * @addr2: The 2nd address to start the search at
-+ * @addr3: The 3rd address to start the search at
-+ * @size: The maximum number of bits to search
-+ * @n: The number of set bit, which position is needed, counting from 0
-+ *
-+ * Returns the bit number of the N'th set bit.
-+ * If no such, returns @size.
-+ */
-+static __always_inline
-+unsigned long find_nth_and_andnot_bit(const unsigned long *addr1,
-+					const unsigned long *addr2,
-+					const unsigned long *addr3,
-+					unsigned long size, unsigned long n)
-+{
-+	if (n >= size)
-+		return size;
-+
-+	if (small_const_nbits(size)) {
-+		unsigned long val =  *addr1 & *addr2 & (~*addr3) & GENMASK(size - 1, 0);
-+
-+		return val ? fns(val, n) : size;
-+	}
-+
-+	return __find_nth_and_andnot_bit(addr1, addr2, addr3, size, n);
-+}
-+
- #ifndef find_first_and_bit
- /**
-  * find_first_and_bit - find the first set bit in both memory regions
-@@ -135,7 +309,7 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size)
-  * Returns the bit number for the next set bit
-  * If no bits are set, returns @size.
-  */
-static inline
-+static __always_inline
- unsigned long find_first_and_bit(const unsigned long *addr1,
- 				 const unsigned long *addr2,
- 				 unsigned long size)
-@@ -150,6 +324,54 @@ unsigned long find_first_and_bit(const unsigned long *addr1,
- }
- #endif
- 
-+/**
-+ * find_first_andnot_bit - find the first bit set in 1st memory region and unset in 2nd
-+ * @addr1: The first address to base the search on
-+ * @addr2: The second address to base the search on
-+ * @size: The bitmap size in bits
-+ *
-+ * Returns the bit number for the first set bit
-+ * If no bits are set, returns >= @size.
-+ */
-+static __always_inline
-+unsigned long find_first_andnot_bit(const unsigned long *addr1,
-+				 const unsigned long *addr2,
-+				 unsigned long size)
-+{
-+	if (small_const_nbits(size)) {
-+		unsigned long val = *addr1 & (~*addr2) & GENMASK(size - 1, 0);
-+
-+		return val ? __ffs(val) : size;
-+	}
-+
-+	return _find_first_andnot_bit(addr1, addr2, size);
-+}
-+
-+/**
-+ * find_first_and_and_bit - find the first set bit in 3 memory regions
-+ * @addr1: The first address to base the search on
-+ * @addr2: The second address to base the search on
-+ * @addr3: The third address to base the search on
-+ * @size: The bitmap size in bits
-+ *
-+ * Returns the bit number for the first set bit
-+ * If no bits are set, returns @size.
-+ */
-+static __always_inline
-+unsigned long find_first_and_and_bit(const unsigned long *addr1,
-+				     const unsigned long *addr2,
-+				     const unsigned long *addr3,
-+				     unsigned long size)
-+{
-+	if (small_const_nbits(size)) {
-+		unsigned long val = *addr1 & *addr2 & *addr3 & GENMASK(size - 1, 0);
-+
-+		return val ? __ffs(val) : size;
-+	}
-+
-+	return _find_first_and_and_bit(addr1, addr2, addr3, size);
-+}
-+
- #ifndef find_first_zero_bit
- /**
-  * find_first_zero_bit - find the first cleared bit in a memory region
-@@ -159,7 +381,7 @@ unsigned long find_first_and_bit(const unsigned long *addr1,
-  * Returns the bit number of the first cleared bit.
-  * If no bits are zero, returns @size.
-  */
-static inline
-+static __always_inline
- unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size)
- {
- 	if (small_const_nbits(size)) {
-@@ -180,7 +402,7 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size)
-  *
-  * Returns the bit number of the last set bit, or size.
-  */
-static inline
-+static __always_inline
- unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
- {
- 	if (small_const_nbits(size)) {
-@@ -194,6 +416,78 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
- #endif
- 
- /**
-+ * find_next_and_bit_wrap - find the next set bit in both memory regions
-+ * @addr1: The first address to base the search on
-+ * @addr2: The second address to base the search on
-+ * @size: The bitmap size in bits
-+ * @offset: The bitnumber to start searching at
-+ *
-+ * Returns the bit number for the next set bit, or first set bit up to @offset
-+ * If no bits are set, returns @size.
-+ */
-+static __always_inline
-+unsigned long find_next_and_bit_wrap(const unsigned long *addr1,
-+					const unsigned long *addr2,
-+					unsigned long size, unsigned long offset)
-+{
-+	unsigned long bit = find_next_and_bit(addr1, addr2, size, offset);
-+
-+	if (bit < size || offset == 0)
-+		return bit;
-+
-+	bit = find_first_and_bit(addr1, addr2, offset);
-+	return bit < offset ? bit : size;
-+}
-+
-+/**
-+ * find_next_bit_wrap - find the next set bit in a memory region
-+ * @addr: The address to base the search on
-+ * @size: The bitmap size in bits
-+ * @offset: The bitnumber to start searching at
-+ *
-+ * Returns the bit number for the next set bit, or first set bit up to @offset
-+ * If no bits are set, returns @size.
-+ */
-+static __always_inline
-+unsigned long find_next_bit_wrap(const unsigned long *addr,
-+					unsigned long size, unsigned long offset)
-+{
-+	unsigned long bit = find_next_bit(addr, size, offset);
-+
-+	if (bit < size || offset == 0)
-+		return bit;
-+
-+	bit = find_first_bit(addr, offset);
-+	return bit < offset ? bit : size;
-+}
-+
-+/*
-+ * Helper for for_each_set_bit_wrap(). Make sure you're doing right thing
-+ * before using it alone.
-+ */
-+static __always_inline
-+unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size,
-+				 unsigned long start, unsigned long n)
-+{
-+	unsigned long bit;
-+
-+	/* If not wrapped around */
-+	if (n > start) {
-+		/* and have a bit, just return it. */
-+		bit = find_next_bit(bitmap, size, n);
-+		if (bit < size)
-+			return bit;
-+
-+		/* Otherwise, wrap around and ... */
-+		n = 0;
-+	}
-+
-+	/* Search the other part. */
-+	bit = find_next_bit(bitmap, start, n);
-+	return bit < start ? bit : size;
-+}
-+
-+/**
-  * find_next_clump8 - find next 8-bit clump with set bits in a memory region
-  * @clump: location to store copy of found clump
-  * @addr: address to base the search on
-@@ -212,20 +506,20 @@ extern unsigned long find_next_clump8(unsigned long *clump,
- 
- #if defined(__LITTLE_ENDIAN)
- 
-static inline unsigned long find_next_zero_bit_le(const void *addr,
-		unsigned long size, unsigned long offset)
-+static __always_inline
-+unsigned long find_next_zero_bit_le(const void *addr, unsigned long size, unsigned long offset)
- {
- 	return find_next_zero_bit(addr, size, offset);
- }
- 
-static inline unsigned long find_next_bit_le(const void *addr,
-		unsigned long size, unsigned long offset)
-+static __always_inline
-+unsigned long find_next_bit_le(const void *addr, unsigned long size, unsigned long offset)
- {
- 	return find_next_bit(addr, size, offset);
- }
- 
-static inline unsigned long find_first_zero_bit_le(const void *addr,
-		unsigned long size)
-+static __always_inline
-+unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)
- {
- 	return find_first_zero_bit(addr, size);
- }
-@@ -233,7 +527,7 @@ static inline unsigned long find_first_zero_bit_le(const void *addr,
- #elif defined(__BIG_ENDIAN)
- 
- #ifndef find_next_zero_bit_le
-static inline
-+static __always_inline
- unsigned long find_next_zero_bit_le(const void *addr, unsigned
- 		long size, unsigned long offset)
- {
-@@ -247,12 +541,26 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned
- 		return val == ~0UL ? size : ffz(val);
- 	}
- 
-	return _find_next_bit(addr, NULL, size, offset, ~0UL, 1);
-+	return _find_next_zero_bit_le(addr, size, offset);
-+}
-+#endif
-+
-+#ifndef find_first_zero_bit_le
-+static __always_inline
-+unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)
-+{
-+	if (small_const_nbits(size)) {
-+		unsigned long val = swab(*(const unsigned long *)addr) | ~GENMASK(size - 1, 0);
-+
-+		return val == ~0UL ? size : ffz(val);
-+	}
-+
-+	return _find_first_zero_bit_le(addr, size);
- }
- #endif
- 
- #ifndef find_next_bit_le
-static inline
-+static __always_inline
- unsigned long find_next_bit_le(const void *addr, unsigned
- 		long size, unsigned long offset)
- {
-@@ -266,40 +574,44 @@ unsigned long find_next_bit_le(const void *addr, unsigned
- 		return val ? __ffs(val) : size;
- 	}
- 
-	return _find_next_bit(addr, NULL, size, offset, 0UL, 1);
-+	return _find_next_bit_le(addr, size, offset);
- }
- #endif
- 
-#ifndef find_first_zero_bit_le
-#define find_first_zero_bit_le(addr, size) \
-	find_next_zero_bit_le((addr), (size), 0)
-#endif
-
- #else
- #error "Please fix <asm/byteorder.h>"
- #endif
- 
- #define for_each_set_bit(bit, addr, size) \
-	for ((bit) = find_next_bit((addr), (size), 0);		\
-	     (bit) < (size);					\
-	     (bit) = find_next_bit((addr), (size), (bit) + 1))
-+	for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
-+
-+#define for_each_and_bit(bit, addr1, addr2, size) \
-+	for ((bit) = 0;									\
-+	     (bit) = find_next_and_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
-+	     (bit)++)
-+
-+#define for_each_andnot_bit(bit, addr1, addr2, size) \
-+	for ((bit) = 0;									\
-+	     (bit) = find_next_andnot_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
-+	     (bit)++)
-+
-+#define for_each_or_bit(bit, addr1, addr2, size) \
-+	for ((bit) = 0;									\
-+	     (bit) = find_next_or_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
-+	     (bit)++)
- 
- /* same as for_each_set_bit() but use bit as value to start with */
- #define for_each_set_bit_from(bit, addr, size) \
-	for ((bit) = find_next_bit((addr), (size), (bit));	\
-	     (bit) < (size);					\
-	     (bit) = find_next_bit((addr), (size), (bit) + 1))
-+	for (; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
- 
- #define for_each_clear_bit(bit, addr, size) \
-	for ((bit) = find_next_zero_bit((addr), (size), 0);	\
-	     (bit) < (size);					\
-	     (bit) = find_next_zero_bit((addr), (size), (bit) + 1))
-+	for ((bit) = 0;									\
-+	     (bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size);		\
-+	     (bit)++)
- 
- /* same as for_each_clear_bit() but use bit as value to start with */
- #define for_each_clear_bit_from(bit, addr, size) \
-	for ((bit) = find_next_zero_bit((addr), (size), (bit));	\
-	     (bit) < (size);					\
-	     (bit) = find_next_zero_bit((addr), (size), (bit) + 1))
-+	for (; (bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
- 
- /**
-  * for_each_set_bitrange - iterate over all set bit ranges [b; e)
-@@ -309,11 +621,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned
-  * @size: bitmap size in number of bits
-  */
- #define for_each_set_bitrange(b, e, addr, size)			\
-	for ((b) = find_next_bit((addr), (size), 0),		\
-	     (e) = find_next_zero_bit((addr), (size), (b) + 1);	\
-+	for ((b) = 0;						\
-+	     (b) = find_next_bit((addr), (size), b),		\
-+	     (e) = find_next_zero_bit((addr), (size), (b) + 1),	\
- 	     (b) < (size);					\
-	     (b) = find_next_bit((addr), (size), (e) + 1),	\
-	     (e) = find_next_zero_bit((addr), (size), (b) + 1))
-+	     (b) = (e) + 1)
- 
- /**
-  * for_each_set_bitrange_from - iterate over all set bit ranges [b; e)
-@@ -323,11 +635,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned
-  * @size: bitmap size in number of bits
-  */
- #define for_each_set_bitrange_from(b, e, addr, size)		\
-	for ((b) = find_next_bit((addr), (size), (b)),		\
-	     (e) = find_next_zero_bit((addr), (size), (b) + 1);	\
-+	for (;							\
-+	     (b) = find_next_bit((addr), (size), (b)),		\
-+	     (e) = find_next_zero_bit((addr), (size), (b) + 1),	\
- 	     (b) < (size);					\
-	     (b) = find_next_bit((addr), (size), (e) + 1),	\
-	     (e) = find_next_zero_bit((addr), (size), (b) + 1))
-+	     (b) = (e) + 1)
- 
- /**
-  * for_each_clear_bitrange - iterate over all unset bit ranges [b; e)
-@@ -337,11 +649,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned
-  * @size: bitmap size in number of bits
-  */
- #define for_each_clear_bitrange(b, e, addr, size)		\
-	for ((b) = find_next_zero_bit((addr), (size), 0),	\
-	     (e) = find_next_bit((addr), (size), (b) + 1);	\
-+	for ((b) = 0;						\
-+	     (b) = find_next_zero_bit((addr), (size), (b)),	\
-+	     (e) = find_next_bit((addr), (size), (b) + 1),	\
- 	     (b) < (size);					\
-	     (b) = find_next_zero_bit((addr), (size), (e) + 1),	\
-	     (e) = find_next_bit((addr), (size), (b) + 1))
-+	     (b) = (e) + 1)
- 
- /**
-  * for_each_clear_bitrange_from - iterate over all unset bit ranges [b; e)
-@@ -351,11 +663,24 @@ unsigned long find_next_bit_le(const void *addr, unsigned
-  * @size: bitmap size in number of bits
-  */
- #define for_each_clear_bitrange_from(b, e, addr, size)		\
-	for ((b) = find_next_zero_bit((addr), (size), (b)),	\
-	     (e) = find_next_bit((addr), (size), (b) + 1);	\
-+	for (;							\
-+	     (b) = find_next_zero_bit((addr), (size), (b)),	\
-+	     (e) = find_next_bit((addr), (size), (b) + 1),	\
- 	     (b) < (size);					\
-	     (b) = find_next_zero_bit((addr), (size), (e) + 1),	\
-	     (e) = find_next_bit((addr), (size), (b) + 1))
-+	     (b) = (e) + 1)
-+
-+/**
-+ * for_each_set_bit_wrap - iterate over all set bits starting from @start, and
-+ * wrapping around the end of bitmap.
-+ * @bit: offset for current iteration
-+ * @addr: bitmap address to base the search on
-+ * @size: bitmap size in number of bits
-+ * @start: Starting bit for bitmap traversing, wrapping around the bitmap end
-+ */
-+#define for_each_set_bit_wrap(bit, addr, size, start) \
-+	for ((bit) = find_next_bit_wrap((addr), (size), (start));		\
-+	     (bit) < (size);							\
-+	     (bit) = __for_each_wrap((addr), (size), (start), (bit) + 1))
- 
- /**
-  * for_each_set_clump8 - iterate over bitmap for each 8-bit clump with set bits
-diff --git a/include/linux/firewire.h b/include/linux/firewire.h
-index 980019053e54..6143b7d28eac 100644
--- a/include/linux/firewire.h
-+++ b/include/linux/firewire.h
-@@ -75,7 +75,7 @@ void fw_csr_iterator_init(struct fw_csr_iterator *ci, const u32 *p);
- int fw_csr_iterator_next(struct fw_csr_iterator *ci, int *key, int *value);
- int fw_csr_string(const u32 *directory, int key, char *buf, size_t size);
- 
-extern struct bus_type fw_bus_type;
-+extern const struct bus_type fw_bus_type;
- 
- struct fw_card_driver;
- struct fw_node;
-@@ -88,23 +88,30 @@ struct fw_card {
- 
- 	int node_id;
- 	int generation;
-	int current_tlabel;
-	u64 tlabel_mask;
-	struct list_head transaction_list;
- 	u64 reset_jiffies;
- 
-	u32 split_timeout_hi;
-	u32 split_timeout_lo;
-	unsigned int split_timeout_cycles;
-	unsigned int split_timeout_jiffies;
-+	struct {
-+		int current_tlabel;
-+		u64 tlabel_mask;
-+		struct list_head list;
-+		spinlock_t lock;
-+	} transactions;
-+
-+	struct {
-+		u32 hi;
-+		u32 lo;
-+		unsigned int cycles;
-+		unsigned int jiffies;
-+		spinlock_t lock;
-+	} split_timeout;
- 
- 	unsigned long long guid;
- 	unsigned max_receive;
- 	int link_speed;
- 	int config_rom_generation;
- 
-	spinlock_t lock; /* Take this lock when handling the lists in
-			  * this struct. */
-+	spinlock_t lock;
-+
- 	struct fw_node *local_node;
- 	struct fw_node *root_node;
- 	struct fw_node *irm_node;
-@@ -115,8 +122,6 @@ struct fw_card {
- 	int index;
- 	struct list_head link;
- 
-	struct list_head phy_receiver_list;
-
- 	struct delayed_work br_work; /* bus reset job */
- 	bool br_short;
- 
-@@ -131,9 +136,16 @@ struct fw_card {
- 
- 	bool broadcast_channel_allocated;
- 	u32 broadcast_channel;
-	__be32 topology_map[(CSR_TOPOLOGY_MAP_END - CSR_TOPOLOGY_MAP) / 4];
-+
-+	struct {
-+		__be32 buffer[(CSR_TOPOLOGY_MAP_END - CSR_TOPOLOGY_MAP) / 4];
-+		spinlock_t lock;
-+	} topology_map;
- 
- 	__be32 maint_utility_register;
-+
-+	struct workqueue_struct *isoc_wq;
-+	struct workqueue_struct *async_wq;
- };
- 
- static inline struct fw_card *fw_card_get(struct fw_card *card)
-@@ -158,6 +170,20 @@ struct fw_attribute_group {
- 	struct attribute *attrs[13];
- };
- 
-+enum fw_device_quirk {
-+	// See afa1282a35d3 ("firewire: core: check for 1394a compliant IRM, fix inaccessibility of Sony camcorder").
-+	FW_DEVICE_QUIRK_IRM_IS_1394_1995_ONLY = BIT(0),
-+
-+	// See a509e43ff338 ("firewire: core: fix unstable I/O with Canon camcorder").
-+	FW_DEVICE_QUIRK_IRM_IGNORES_BUS_MANAGER = BIT(1),
-+
-+	// MOTU Audio Express transfers acknowledge packet with 0x10 for pending state.
-+	FW_DEVICE_QUIRK_ACK_PACKET_WITH_INVALID_PENDING_CODE = BIT(2),
-+
-+	// TASCAM FW-1082/FW-1804/FW-1884 often freezes when receiving S400 packets.
-+	FW_DEVICE_QUIRK_UNSTABLE_AT_S400 = BIT(3),
-+};
-+
- enum fw_device_state {
- 	FW_DEVICE_INITIALIZING,
- 	FW_DEVICE_RUNNING,
-@@ -191,6 +217,9 @@ struct fw_device {
- 	struct fw_card *card;
- 	struct device device;
- 
-+	// A set of enum fw_device_quirk.
-+	int quirks;
-+
- 	struct mutex client_list_mutex;
- 	struct list_head client_list;
- 
-@@ -208,10 +237,7 @@ struct fw_device {
- 	struct fw_attribute_group attribute_group;
- };
- 
-static inline struct fw_device *fw_device(struct device *dev)
-{
-	return container_of(dev, struct fw_device, device);
-}
-+#define fw_device(dev)	container_of_const(dev, struct fw_device, device)
- 
- static inline int fw_device_is_shutdown(struct fw_device *device)
- {
-@@ -229,10 +255,7 @@ struct fw_unit {
- 	struct fw_attribute_group attribute_group;
- };
- 
-static inline struct fw_unit *fw_unit(struct device *dev)
-{
-	return container_of(dev, struct fw_unit, device);
-}
-+#define fw_unit(dev)	container_of_const(dev, struct fw_unit, device)
- 
- static inline struct fw_unit *fw_unit_get(struct fw_unit *unit)
- {
-@@ -246,10 +269,7 @@ static inline void fw_unit_put(struct fw_unit *unit)
- 	put_device(&unit->device);
- }
- 
-static inline struct fw_device *fw_parent_device(struct fw_unit *unit)
-{
-	return fw_device(unit->device.parent);
-}
-+#define fw_parent_device(unit)	fw_device(unit->device.parent)
- 
- struct ieee1394_device_id;
- 
-@@ -270,6 +290,15 @@ typedef void (*fw_packet_callback_t)(struct fw_packet *packet,
- typedef void (*fw_transaction_callback_t)(struct fw_card *card, int rcode,
- 					  void *data, size_t length,
- 					  void *callback_data);
-+typedef void (*fw_transaction_callback_with_tstamp_t)(struct fw_card *card, int rcode,
-+					u32 request_tstamp, u32 response_tstamp, void *data,
-+					size_t length, void *callback_data);
-+
-+union fw_transaction_callback {
-+	fw_transaction_callback_t without_tstamp;
-+	fw_transaction_callback_with_tstamp_t with_tstamp;
-+};
-+
- /*
-  * This callback handles an inbound request subaction.  It is called in
-  * RCU read-side context, therefore must not sleep.
-@@ -278,9 +307,8 @@ typedef void (*fw_transaction_callback_t)(struct fw_card *card, int rcode,
-  * Otherwise there is a danger of recursion of inbound and outbound
-  * transactions from and to the local node.
-  *
- * The callback is responsible that either fw_send_response() or kfree()
- * is called on the @request, except for FCP registers for which the core
- * takes care of that.
-+ * The callback is responsible that fw_send_response() is called on the @request, except for FCP
-+ * registers for which the core takes care of that.
-  */
- typedef void (*fw_address_callback_t)(struct fw_card *card,
- 				      struct fw_request *request,
-@@ -306,8 +334,7 @@ struct fw_packet {
- 	 * For successful transmission, the status code is the ack received
- 	 * from the destination.  Otherwise it is one of the juju-specific
- 	 * rcodes:  RCODE_SEND_ERROR, _CANCELLED, _BUSY, _GENERATION, _NO_ACK.
-	 * The callback can be called from tasklet context and thus
-	 * must never block.
-+	 * The callback can be called from workqueue and thus must never block.
- 	 */
- 	fw_packet_callback_t callback;
- 	int ack;
-@@ -322,6 +349,7 @@ struct fw_transaction {
- 	struct fw_card *card;
- 	bool is_split_transaction;
- 	struct timer_list split_timeout_timer;
-+	u32 split_timeout_cycle;
- 
- 	struct fw_packet packet;
- 
-@@ -329,7 +357,8 @@ struct fw_transaction {
- 	 * The data passed to the callback is valid only during the
- 	 * callback.
- 	 */
-	fw_transaction_callback_t callback;
-+	union fw_transaction_callback callback;
-+	bool with_tstamp;
- 	void *callback_data;
- };
- 
-@@ -338,7 +367,11 @@ struct fw_address_handler {
- 	u64 length;
- 	fw_address_callback_t address_callback;
- 	void *callback_data;
-+
-+	// Only for core functions.
- 	struct list_head link;
-+	struct kref kref;
-+	struct completion done;
- };
- 
- struct fw_address_region {
-@@ -355,10 +388,79 @@ void fw_send_response(struct fw_card *card,
- 		      struct fw_request *request, int rcode);
- int fw_get_request_speed(struct fw_request *request);
- u32 fw_request_get_timestamp(const struct fw_request *request);
-void fw_send_request(struct fw_card *card, struct fw_transaction *t,
-		     int tcode, int destination_id, int generation, int speed,
-		     unsigned long long offset, void *payload, size_t length,
-		     fw_transaction_callback_t callback, void *callback_data);
-+
-+void __fw_send_request(struct fw_card *card, struct fw_transaction *t, int tcode,
-+		int destination_id, int generation, int speed, unsigned long long offset,
-+		void *payload, size_t length, union fw_transaction_callback callback,
-+		bool with_tstamp, void *callback_data);
-+
-+/**
-+ * fw_send_request() - submit a request packet for transmission to generate callback for response
-+ *		       subaction without time stamp.
-+ * @card:		interface to send the request at
-+ * @t:			transaction instance to which the request belongs
-+ * @tcode:		transaction code
-+ * @destination_id:	destination node ID, consisting of bus_ID and phy_ID
-+ * @generation:		bus generation in which request and response are valid
-+ * @speed:		transmission speed
-+ * @offset:		48bit wide offset into destination's address space
-+ * @payload:		data payload for the request subaction
-+ * @length:		length of the payload, in bytes
-+ * @callback:		function to be called when the transaction is completed
-+ * @callback_data:	data to be passed to the transaction completion callback
-+ *
-+ * A variation of __fw_send_request() to generate callback for response subaction without time
-+ * stamp.
-+ *
-+ * The callback is invoked in the workqueue context in most cases. However, if an error is detected
-+ * before queueing or the destination address refers to the local node, it is invoked in the
-+ * current context instead.
-+ */
-+static inline void fw_send_request(struct fw_card *card, struct fw_transaction *t, int tcode,
-+				   int destination_id, int generation, int speed,
-+				   unsigned long long offset, void *payload, size_t length,
-+				   fw_transaction_callback_t callback, void *callback_data)
-+{
-+	union fw_transaction_callback cb = {
-+		.without_tstamp = callback,
-+	};
-+	__fw_send_request(card, t, tcode, destination_id, generation, speed, offset, payload,
-+			  length, cb, false, callback_data);
-+}
-+
-+/**
-+ * fw_send_request_with_tstamp() - submit a request packet for transmission to generate callback for
-+ *				   response with time stamp.
-+ * @card:		interface to send the request at
-+ * @t:			transaction instance to which the request belongs
-+ * @tcode:		transaction code
-+ * @destination_id:	destination node ID, consisting of bus_ID and phy_ID
-+ * @generation:		bus generation in which request and response are valid
-+ * @speed:		transmission speed
-+ * @offset:		48bit wide offset into destination's address space
-+ * @payload:		data payload for the request subaction
-+ * @length:		length of the payload, in bytes
-+ * @callback:		function to be called when the transaction is completed
-+ * @callback_data:	data to be passed to the transaction completion callback
-+ *
-+ * A variation of __fw_send_request() to generate callback for response subaction with time stamp.
-+ *
-+ * The callback is invoked in the workqueue context in most cases. However, if an error is detected
-+ * before queueing or the destination address refers to the local node, it is invoked in the current
-+ * context instead.
-+ */
-+static inline void fw_send_request_with_tstamp(struct fw_card *card, struct fw_transaction *t,
-+	int tcode, int destination_id, int generation, int speed, unsigned long long offset,
-+	void *payload, size_t length, fw_transaction_callback_with_tstamp_t callback,
-+	void *callback_data)
-+{
-+	union fw_transaction_callback cb = {
-+		.with_tstamp = callback,
-+	};
-+	__fw_send_request(card, t, tcode, destination_id, generation, speed, offset, payload,
-+			  length, cb, true, callback_data);
-+}
-+
- int fw_cancel_transaction(struct fw_card *card,
- 			  struct fw_transaction *transaction);
- int fw_run_transaction(struct fw_card *card, int tcode, int destination_id,
-@@ -400,8 +502,8 @@ struct fw_iso_packet {
- 				/* rx: Sync bit, wait for matching sy	*/
- 	u32 tag:2;		/* tx: Tag in packet header		*/
- 	u32 sy:4;		/* tx: Sy in packet header		*/
-	u32 header_length:8;	/* Length of immediate header		*/
-	u32 header[0];		/* tx: Top of 1394 isoch. data_block	*/
-+	u32 header_length:8;	/* Size of immediate header		*/
-+	u32 header[];		/* tx: Top of 1394 isoch. data_block	*/
- };
- 
- #define FW_ISO_CONTEXT_TRANSMIT			0
-@@ -447,6 +549,7 @@ union fw_iso_callback {
- 
- struct fw_iso_context {
- 	struct fw_card *card;
-+	struct work_struct work;
- 	int type;
- 	int channel;
- 	int speed;
-@@ -466,6 +569,25 @@ int fw_iso_context_queue(struct fw_iso_context *ctx,
- 			 unsigned long payload);
- void fw_iso_context_queue_flush(struct fw_iso_context *ctx);
- int fw_iso_context_flush_completions(struct fw_iso_context *ctx);
-+
-+/**
-+ * fw_iso_context_schedule_flush_completions() - schedule work item to process isochronous context.
-+ * @ctx: the isochronous context
-+ *
-+ * Schedule a work item on workqueue to process the isochronous context. The registered callback
-+ * function is called by the worker when a queued packet buffer with the interrupt flag is
-+ * completed, either after transmission in the IT context or after being filled in the IR context.
-+ * The callback function is also called when the header buffer in the context becomes full, If it
-+ * is required to process the context in the current context, fw_iso_context_flush_completions() is
-+ * available instead.
-+ *
-+ * Context: Any context.
-+ */
-+static inline void fw_iso_context_schedule_flush_completions(struct fw_iso_context *ctx)
-+{
-+	queue_work(ctx->card->isoc_wq, &ctx->work);
-+}
-+
- int fw_iso_context_start(struct fw_iso_context *ctx,
- 			 int cycle, int sync, int tags);
- int fw_iso_context_stop(struct fw_iso_context *ctx);
-diff --git a/include/linux/firmware.h b/include/linux/firmware.h
-index de7fea3bca51..aae1b85ffc10 100644
--- a/include/linux/firmware.h
-+++ b/include/linux/firmware.h
-@@ -4,6 +4,7 @@
- 
- #include <linux/types.h>
- #include <linux/compiler.h>
-+#include <linux/cleanup.h>
- #include <linux/gfp.h>
- 
- #define FW_ACTION_NOUEVENT 0
-@@ -27,6 +28,7 @@ struct firmware {
-  * @FW_UPLOAD_ERR_INVALID_SIZE: invalid firmware image size
-  * @FW_UPLOAD_ERR_RW_ERROR: read or write to HW failed, see kernel log
-  * @FW_UPLOAD_ERR_WEAROUT: FLASH device is approaching wear-out, wait & retry
-+ * @FW_UPLOAD_ERR_FW_INVALID: invalid firmware file
-  * @FW_UPLOAD_ERR_MAX: Maximum error code marker
-  */
- enum fw_upload_err {
-@@ -38,6 +40,7 @@ enum fw_upload_err {
- 	FW_UPLOAD_ERR_INVALID_SIZE,
- 	FW_UPLOAD_ERR_RW_ERROR,
- 	FW_UPLOAD_ERR_WEAROUT,
-+	FW_UPLOAD_ERR_FW_INVALID,
- 	FW_UPLOAD_ERR_MAX
- };
- 
-@@ -95,6 +98,10 @@ static inline bool firmware_request_builtin(struct firmware *fw,
- #if IS_REACHABLE(CONFIG_FW_LOADER)
- int request_firmware(const struct firmware **fw, const char *name,
- 		     struct device *device);
-+int firmware_request_nowait_nowarn(
-+	struct module *module, const char *name,
-+	struct device *device, gfp_t gfp, void *context,
-+	void (*cont)(const struct firmware *fw, void *context));
- int firmware_request_nowarn(const struct firmware **fw, const char *name,
- 			    struct device *device);
- int firmware_request_platform(const struct firmware **fw, const char *name,
-@@ -120,6 +127,14 @@ static inline int request_firmware(const struct firmware **fw,
- 	return -EINVAL;
- }
- 
-+static inline int firmware_request_nowait_nowarn(
-+	struct module *module, const char *name,
-+	struct device *device, gfp_t gfp, void *context,
-+	void (*cont)(const struct firmware *fw, void *context))
-+{
-+	return -EINVAL;
-+}
-+
- static inline int firmware_request_nowarn(const struct firmware **fw,
- 					  const char *name,
- 					  struct device *device)
-@@ -196,4 +211,6 @@ static inline void firmware_upload_unregister(struct fw_upload *fw_upload)
- 
- int firmware_request_cache(struct device *device, const char *name);
- 
-+DEFINE_FREE(firmware, struct firmware *, release_firmware(_T))
-+
- #endif
-diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h
-index cad828e21c72..0ec1cdc5585d 100644
--- a/include/linux/firmware/cirrus/cs_dsp.h
-+++ b/include/linux/firmware/cirrus/cs_dsp.h
-@@ -42,6 +42,16 @@
- #define CS_DSP_ACKED_CTL_MIN_VALUE           0
- #define CS_DSP_ACKED_CTL_MAX_VALUE           0xFFFFFF
- 
-+/*
-+ * Write sequence operation codes
-+ */
-+#define CS_DSP_WSEQ_FULL	0x00
-+#define CS_DSP_WSEQ_ADDR8	0x02
-+#define CS_DSP_WSEQ_L16		0x04
-+#define CS_DSP_WSEQ_H16		0x05
-+#define CS_DSP_WSEQ_UNLOCK	0xFD
-+#define CS_DSP_WSEQ_END		0xFF
-+
- /**
-  * struct cs_dsp_region - Describes a logical memory region in DSP address space
-  * @type:	Memory region type
-@@ -54,14 +64,12 @@ struct cs_dsp_region {
- 
- /**
-  * struct cs_dsp_alg_region - Describes a logical algorithm region in DSP address space
- * @list:	List node for internal use
-  * @alg:	Algorithm id
-  * @ver:	Expected algorithm version
-  * @type:	Memory region type
-  * @base:	Address of region
-  */
- struct cs_dsp_alg_region {
-	struct list_head list;
- 	unsigned int alg;
- 	unsigned int ver;
- 	int type;
-@@ -94,7 +102,7 @@ struct cs_dsp_coeff_ctl {
- 	const char *subname;
- 	unsigned int subname_len;
- 	unsigned int offset;
-	size_t len;
-+	unsigned int len;
- 	unsigned int type;
- 	unsigned int flags;
- 	unsigned int set:1;
-@@ -123,7 +131,6 @@ struct cs_dsp_client_ops;
-  * @sysclk_mask:	Mask of frequency bits within sysclk register (ADSP1 only)
-  * @sysclk_shift:	Shift of frequency bits within sysclk register (ADSP1 only)
-  * @alg_regions:	List of currently loaded algorithm regions
- * @fw_file_name:	Filename of the current firmware
-  * @fw_name:		Name of the current firmware
-  * @fw_id:		ID of the current firmware, obtained from the wmfw
-  * @fw_id_version:	Version of the firmware, obtained from the wmfw
-@@ -156,6 +163,7 @@ struct cs_dsp {
- 	unsigned int sysclk_reg;
- 	unsigned int sysclk_mask;
- 	unsigned int sysclk_shift;
-+	bool no_core_startstop;
- 
- 	struct list_head alg_regions;
- 
-@@ -167,7 +175,7 @@ struct cs_dsp {
- 	const struct cs_dsp_region *mem;
- 	int num_mems;
- 
-	int fw_ver;
-+	int wmfw_ver;
- 
- 	bool booted;
- 	bool running;
-@@ -180,8 +188,8 @@ struct cs_dsp {
- 
- #ifdef CONFIG_DEBUG_FS
- 	struct dentry *debugfs_root;
-	char *wmfw_file_name;
-	char *bin_file_name;
-+	const char *wmfw_file_name;
-+	const char *bin_file_name;
- #endif
- };
- 
-@@ -213,13 +221,13 @@ int cs_dsp_adsp2_init(struct cs_dsp *dsp);
- int cs_dsp_halo_init(struct cs_dsp *dsp);
- 
- int cs_dsp_adsp1_power_up(struct cs_dsp *dsp,
-			  const struct firmware *wmfw_firmware, char *wmfw_filename,
-			  const struct firmware *coeff_firmware, char *coeff_filename,
-+			  const struct firmware *wmfw_firmware, const char *wmfw_filename,
-+			  const struct firmware *coeff_firmware, const char *coeff_filename,
- 			  const char *fw_name);
- void cs_dsp_adsp1_power_down(struct cs_dsp *dsp);
- int cs_dsp_power_up(struct cs_dsp *dsp,
-		    const struct firmware *wmfw_firmware, char *wmfw_filename,
-		    const struct firmware *coeff_firmware, char *coeff_filename,
-+		    const struct firmware *wmfw_firmware, const char *wmfw_filename,
-+		    const struct firmware *coeff_firmware, const char *coeff_filename,
- 		    const char *fw_name);
- void cs_dsp_power_down(struct cs_dsp *dsp);
- int cs_dsp_run(struct cs_dsp *dsp);
-@@ -238,8 +246,12 @@ void cs_dsp_cleanup_debugfs(struct cs_dsp *dsp);
- int cs_dsp_coeff_write_acked_control(struct cs_dsp_coeff_ctl *ctl, unsigned int event_id);
- int cs_dsp_coeff_write_ctrl(struct cs_dsp_coeff_ctl *ctl, unsigned int off,
- 			    const void *buf, size_t len);
-+int cs_dsp_coeff_lock_and_write_ctrl(struct cs_dsp_coeff_ctl *ctl, unsigned int off,
-+				     const void *buf, size_t len);
- int cs_dsp_coeff_read_ctrl(struct cs_dsp_coeff_ctl *ctl, unsigned int off,
- 			   void *buf, size_t len);
-+int cs_dsp_coeff_lock_and_read_ctrl(struct cs_dsp_coeff_ctl *ctl, unsigned int off,
-+				    void *buf, size_t len);
- struct cs_dsp_coeff_ctl *cs_dsp_get_ctl(struct cs_dsp *dsp, const char *name, int type,
- 					unsigned int alg);
- 
-@@ -255,6 +267,23 @@ struct cs_dsp_alg_region *cs_dsp_find_alg_region(struct cs_dsp *dsp,
- const char *cs_dsp_mem_region_name(unsigned int type);
- 
- /**
-+ * struct cs_dsp_wseq - Describes a write sequence
-+ * @ctl:	Write sequence cs_dsp control
-+ * @ops:	Operations contained within
-+ */
-+struct cs_dsp_wseq {
-+	struct cs_dsp_coeff_ctl *ctl;
-+	struct list_head ops;
-+};
-+
-+int cs_dsp_wseq_init(struct cs_dsp *dsp, struct cs_dsp_wseq *wseqs, unsigned int num_wseqs);
-+int cs_dsp_wseq_write(struct cs_dsp *dsp, struct cs_dsp_wseq *wseq, u32 addr, u32 data,
-+		      u8 op_code, bool update);
-+int cs_dsp_wseq_multi_write(struct cs_dsp *dsp, struct cs_dsp_wseq *wseq,
-+			    const struct reg_sequence *reg_seq, int num_regs,
-+			    u8 op_code, bool update);
-+
-+/**
-  * struct cs_dsp_chunk - Describes a buffer holding data formatted for the DSP
-  * @data:	Pointer to underlying buffer memory
-  * @max:	Pointer to end of the buffer memory
-diff --git a/include/linux/firmware/cirrus/cs_dsp_test_utils.h b/include/linux/firmware/cirrus/cs_dsp_test_utils.h
-new file mode 100644
-index 000000000000..1f97764fdfd7
--- /dev/null
-+++ b/include/linux/firmware/cirrus/cs_dsp_test_utils.h
-@@ -0,0 +1,159 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Support utilities for cs_dsp testing.
-+ *
-+ * Copyright (C) 2024 Cirrus Logic, Inc. and
-+ *                    Cirrus Logic International Semiconductor Ltd.
-+ */
-+
-+#include <linux/regmap.h>
-+#include <linux/firmware/cirrus/wmfw.h>
-+
-+struct kunit;
-+struct cs_dsp_test;
-+struct cs_dsp_test_local;
-+
-+/**
-+ * struct cs_dsp_test - base class for test utilities
-+ *
-+ * @test:	Pointer to struct kunit instance.
-+ * @dsp:	Pointer to struct cs_dsp instance.
-+ * @local:	Private data for each test suite.
-+ */
-+struct cs_dsp_test {
-+	struct kunit *test;
-+	struct cs_dsp *dsp;
-+
-+	struct cs_dsp_test_local *local;
-+
-+	/* private: Following members are private */
-+	bool saw_bus_write;
-+};
-+
-+/**
-+ * struct cs_dsp_mock_alg_def - Info for creating a mock algorithm entry.
-+ *
-+ * @id:		   Algorithm ID.
-+ * @ver:	   Algorithm version.
-+ * @xm_base_words: XM base address in DSP words.
-+ * @xm_size_words: XM size in DSP words.
-+ * @ym_base_words: YM base address in DSP words.
-+ * @ym_size_words: YM size in DSP words.
-+ * @zm_base_words: ZM base address in DSP words.
-+ * @zm_size_words: ZM size in DSP words.
-+ */
-+struct cs_dsp_mock_alg_def {
-+	unsigned int id;
-+	unsigned int ver;
-+	unsigned int xm_base_words;
-+	unsigned int xm_size_words;
-+	unsigned int ym_base_words;
-+	unsigned int ym_size_words;
-+	unsigned int zm_base_words;
-+	unsigned int zm_size_words;
-+};
-+
-+struct cs_dsp_mock_coeff_def {
-+	const char *shortname;
-+	const char *fullname;
-+	const char *description;
-+	u16 type;
-+	u16 flags;
-+	u16 mem_type;
-+	unsigned int offset_dsp_words;
-+	unsigned int length_bytes;
-+};
-+
-+/**
-+ * struct cs_dsp_mock_xm_header - XM header builder
-+ *
-+ * @test_priv:	     Pointer to the struct cs_dsp_test.
-+ * @blob_data:	     Pointer to the created blob data.
-+ * @blob_size_bytes: Size of the data at blob_data.
-+ */
-+struct cs_dsp_mock_xm_header {
-+	struct cs_dsp_test *test_priv;
-+	void *blob_data;
-+	size_t blob_size_bytes;
-+};
-+
-+struct cs_dsp_mock_wmfw_builder;
-+struct cs_dsp_mock_bin_builder;
-+
-+extern const unsigned int cs_dsp_mock_adsp2_32bit_sysbase;
-+extern const unsigned int cs_dsp_mock_adsp2_16bit_sysbase;
-+extern const unsigned int cs_dsp_mock_halo_core_base;
-+extern const unsigned int cs_dsp_mock_halo_sysinfo_base;
-+
-+extern const struct cs_dsp_region cs_dsp_mock_halo_dsp1_regions[];
-+extern const unsigned int cs_dsp_mock_halo_dsp1_region_sizes[];
-+extern const struct cs_dsp_region cs_dsp_mock_adsp2_32bit_dsp1_regions[];
-+extern const unsigned int cs_dsp_mock_adsp2_32bit_dsp1_region_sizes[];
-+extern const struct cs_dsp_region cs_dsp_mock_adsp2_16bit_dsp1_regions[];
-+extern const unsigned int cs_dsp_mock_adsp2_16bit_dsp1_region_sizes[];
-+int cs_dsp_mock_count_regions(const unsigned int *region_sizes);
-+unsigned int cs_dsp_mock_size_of_region(const struct cs_dsp *dsp, int mem_type);
-+unsigned int cs_dsp_mock_base_addr_for_mem(struct cs_dsp_test *priv, int mem_type);
-+unsigned int cs_dsp_mock_reg_addr_inc_per_unpacked_word(struct cs_dsp_test *priv);
-+unsigned int cs_dsp_mock_reg_block_length_bytes(struct cs_dsp_test *priv, int mem_type);
-+unsigned int cs_dsp_mock_reg_block_length_registers(struct cs_dsp_test *priv, int mem_type);
-+unsigned int cs_dsp_mock_reg_block_length_dsp_words(struct cs_dsp_test *priv, int mem_type);
-+bool cs_dsp_mock_has_zm(struct cs_dsp_test *priv);
-+int cs_dsp_mock_packed_to_unpacked_mem_type(int packed_mem_type);
-+unsigned int cs_dsp_mock_num_dsp_words_to_num_packed_regs(unsigned int num_dsp_words);
-+unsigned int cs_dsp_mock_xm_header_get_alg_base_in_words(struct cs_dsp_test *priv,
-+							 unsigned int alg_id,
-+							 int mem_type);
-+unsigned int cs_dsp_mock_xm_header_get_fw_version(struct cs_dsp_mock_xm_header *header);
-+void cs_dsp_mock_xm_header_drop_from_regmap_cache(struct cs_dsp_test *priv);
-+int cs_dsp_mock_xm_header_write_to_regmap(struct cs_dsp_mock_xm_header *header);
-+struct cs_dsp_mock_xm_header *cs_dsp_create_mock_xm_header(struct cs_dsp_test *priv,
-+							   const struct cs_dsp_mock_alg_def *algs,
-+							   size_t num_algs);
-+
-+int cs_dsp_mock_regmap_init(struct cs_dsp_test *priv);
-+void cs_dsp_mock_regmap_drop_range(struct cs_dsp_test *priv,
-+				   unsigned int first_reg, unsigned int last_reg);
-+void cs_dsp_mock_regmap_drop_regs(struct cs_dsp_test *priv,
-+				  unsigned int first_reg, size_t num_regs);
-+void cs_dsp_mock_regmap_drop_bytes(struct cs_dsp_test *priv,
-+				   unsigned int first_reg, size_t num_bytes);
-+void cs_dsp_mock_regmap_drop_system_regs(struct cs_dsp_test *priv);
-+bool cs_dsp_mock_regmap_is_dirty(struct cs_dsp_test *priv, bool drop_system_regs);
-+
-+struct cs_dsp_mock_bin_builder *cs_dsp_mock_bin_init(struct cs_dsp_test *priv,
-+						     int format_version,
-+						     unsigned int fw_version);
-+void cs_dsp_mock_bin_add_raw_block(struct cs_dsp_mock_bin_builder *builder,
-+				   unsigned int alg_id, unsigned int alg_ver,
-+				   int type, unsigned int offset,
-+				   const void *payload_data, size_t payload_len_bytes);
-+void cs_dsp_mock_bin_add_info(struct cs_dsp_mock_bin_builder *builder,
-+			      const char *info);
-+void cs_dsp_mock_bin_add_name(struct cs_dsp_mock_bin_builder *builder,
-+			      const char *name);
-+void cs_dsp_mock_bin_add_patch(struct cs_dsp_mock_bin_builder *builder,
-+			       unsigned int alg_id, unsigned int alg_ver,
-+			       int mem_region, unsigned int reg_addr_offset,
-+			       const void *payload_data, size_t payload_len_bytes);
-+struct firmware *cs_dsp_mock_bin_get_firmware(struct cs_dsp_mock_bin_builder *builder);
-+
-+struct cs_dsp_mock_wmfw_builder *cs_dsp_mock_wmfw_init(struct cs_dsp_test *priv,
-+						       int format_version);
-+void cs_dsp_mock_wmfw_add_raw_block(struct cs_dsp_mock_wmfw_builder *builder,
-+				    int mem_region, unsigned int mem_offset_dsp_words,
-+				    const void *payload_data, size_t payload_len_bytes);
-+void cs_dsp_mock_wmfw_add_info(struct cs_dsp_mock_wmfw_builder *builder,
-+			       const char *info);
-+void cs_dsp_mock_wmfw_add_data_block(struct cs_dsp_mock_wmfw_builder *builder,
-+				     int mem_region, unsigned int mem_offset_dsp_words,
-+				     const void *payload_data, size_t payload_len_bytes);
-+void cs_dsp_mock_wmfw_start_alg_info_block(struct cs_dsp_mock_wmfw_builder *builder,
-+					   unsigned int alg_id,
-+					   const char *name,
-+					   const char *description);
-+void cs_dsp_mock_wmfw_add_coeff_desc(struct cs_dsp_mock_wmfw_builder *builder,
-+				     const struct cs_dsp_mock_coeff_def *def);
-+void cs_dsp_mock_wmfw_end_alg_info_block(struct cs_dsp_mock_wmfw_builder *builder);
-+struct firmware *cs_dsp_mock_wmfw_get_firmware(struct cs_dsp_mock_wmfw_builder *builder);
-+int cs_dsp_mock_wmfw_format_version(struct cs_dsp_mock_wmfw_builder *builder);
-diff --git a/include/linux/firmware/imx/dsp.h b/include/linux/firmware/imx/dsp.h
-index 4f7895a3b73c..1f176a2683fe 100644
--- a/include/linux/firmware/imx/dsp.h
-+++ b/include/linux/firmware/imx/dsp.h
-@@ -37,17 +37,11 @@ struct imx_dsp_ipc {
- 
- static inline void imx_dsp_set_data(struct imx_dsp_ipc *ipc, void *data)
- {
-	if (!ipc)
-		return;
-
- 	ipc->private_data = data;
- }
- 
- static inline void *imx_dsp_get_data(struct imx_dsp_ipc *ipc)
- {
-	if (!ipc)
-		return NULL;
-
- 	return ipc->private_data;
- }
- 
-diff --git a/include/linux/firmware/imx/sci.h b/include/linux/firmware/imx/sci.h
-index 5cc63fe7e84d..df17196df5ff 100644
--- a/include/linux/firmware/imx/sci.h
-+++ b/include/linux/firmware/imx/sci.h
-@@ -21,31 +21,37 @@ int imx_scu_enable_general_irq_channel(struct device *dev);
- int imx_scu_irq_register_notifier(struct notifier_block *nb);
- int imx_scu_irq_unregister_notifier(struct notifier_block *nb);
- int imx_scu_irq_group_enable(u8 group, u32 mask, u8 enable);
-+int imx_scu_irq_get_status(u8 group, u32 *irq_status);
- int imx_scu_soc_init(struct device *dev);
- #else
- static inline int imx_scu_soc_init(struct device *dev)
- {
-	return -ENOTSUPP;
-+	return -EOPNOTSUPP;
- }
- 
- static inline int imx_scu_enable_general_irq_channel(struct device *dev)
- {
-	return -ENOTSUPP;
-+	return -EOPNOTSUPP;
- }
- 
- static inline int imx_scu_irq_register_notifier(struct notifier_block *nb)
- {
-	return -ENOTSUPP;
-+	return -EOPNOTSUPP;
- }
- 
- static inline int imx_scu_irq_unregister_notifier(struct notifier_block *nb)
- {
-	return -ENOTSUPP;
-+	return -EOPNOTSUPP;
- }
- 
- static inline int imx_scu_irq_group_enable(u8 group, u32 mask, u8 enable)
- {
-	return -ENOTSUPP;
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline int imx_scu_irq_get_status(u8 group, u32 *irq_status)
-+{
-+	return -EOPNOTSUPP;
- }
- #endif
- #endif /* _SC_SCI_H */
-diff --git a/include/linux/firmware/imx/sm.h b/include/linux/firmware/imx/sm.h
-new file mode 100644
-index 000000000000..a33b45027356
--- /dev/null
-+++ b/include/linux/firmware/imx/sm.h
-@@ -0,0 +1,97 @@
-+/* SPDX-License-Identifier: GPL-2.0+ */
-+/*
-+ * Copyright 2024 NXP
-+ */
-+
-+#ifndef _SCMI_IMX_H
-+#define _SCMI_IMX_H
-+
-+#include <linux/bitfield.h>
-+#include <linux/errno.h>
-+#include <linux/scmi_imx_protocol.h>
-+#include <linux/types.h>
-+
-+#define SCMI_IMX95_CTRL_PDM_CLK_SEL	0	/* AON PDM clock sel */
-+#define SCMI_IMX95_CTRL_MQS1_SETTINGS	1	/* AON MQS settings */
-+#define SCMI_IMX95_CTRL_SAI1_MCLK	2	/* AON SAI1 MCLK */
-+#define SCMI_IMX95_CTRL_SAI3_MCLK	3	/* WAKE SAI3 MCLK */
-+#define SCMI_IMX95_CTRL_SAI4_MCLK	4	/* WAKE SAI4 MCLK */
-+#define SCMI_IMX95_CTRL_SAI5_MCLK	5	/* WAKE SAI5 MCLK */
-+
-+#define SCMI_IMX94_CTRL_PDM_CLK_SEL	0U	/*!< AON PDM clock sel */
-+#define SCMI_IMX94_CTRL_MQS1_SETTINGS	1U	/*!< AON MQS settings */
-+#define SCMI_IMX94_CTRL_MQS2_SETTINGS	2U	/*!< WAKE MQS settings */
-+#define SCMI_IMX94_CTRL_SAI1_MCLK	3U	/*!< AON SAI1 MCLK */
-+#define SCMI_IMX94_CTRL_SAI2_MCLK	4U	/*!< WAKE SAI2 MCLK */
-+#define SCMI_IMX94_CTRL_SAI3_MCLK	5U	/*!< WAKE SAI3 MCLK */
-+#define SCMI_IMX94_CTRL_SAI4_MCLK	6U	/*!< WAKE SAI4 MCLK */
-+
-+#if IS_ENABLED(CONFIG_IMX_SCMI_MISC_DRV)
-+int scmi_imx_misc_ctrl_get(u32 id, u32 *num, u32 *val);
-+int scmi_imx_misc_ctrl_set(u32 id, u32 val);
-+#else
-+static inline int scmi_imx_misc_ctrl_get(u32 id, u32 *num, u32 *val)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline int scmi_imx_misc_ctrl_set(u32 id, u32 val)
-+{
-+	return -EOPNOTSUPP;
-+}
-+#endif
-+
-+#if IS_ENABLED(CONFIG_IMX_SCMI_CPU_DRV)
-+int scmi_imx_cpu_start(u32 cpuid, bool start);
-+int scmi_imx_cpu_started(u32 cpuid, bool *started);
-+int scmi_imx_cpu_reset_vector_set(u32 cpuid, u64 vector, bool start, bool boot,
-+				  bool resume);
-+#else
-+static inline int scmi_imx_cpu_start(u32 cpuid, bool start)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline int scmi_imx_cpu_started(u32 cpuid, bool *started)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline int scmi_imx_cpu_reset_vector_set(u32 cpuid, u64 vector, bool start,
-+						bool boot, bool resume)
-+{
-+	return -EOPNOTSUPP;
-+}
-+#endif
-+
-+enum scmi_imx_lmm_op {
-+	SCMI_IMX_LMM_BOOT,
-+	SCMI_IMX_LMM_POWER_ON,
-+	SCMI_IMX_LMM_SHUTDOWN,
-+};
-+
-+/* For shutdown pperation */
-+#define SCMI_IMX_LMM_OP_FORCEFUL	0
-+#define SCMI_IMX_LMM_OP_GRACEFUL	BIT(0)
-+
-+#if IS_ENABLED(CONFIG_IMX_SCMI_LMM_DRV)
-+int scmi_imx_lmm_operation(u32 lmid, enum scmi_imx_lmm_op op, u32 flags);
-+int scmi_imx_lmm_info(u32 lmid, struct scmi_imx_lmm_info *info);
-+int scmi_imx_lmm_reset_vector_set(u32 lmid, u32 cpuid, u32 flags, u64 vector);
-+#else
-+static inline int scmi_imx_lmm_operation(u32 lmid, enum scmi_imx_lmm_op op, u32 flags)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline int scmi_imx_lmm_info(u32 lmid, struct scmi_imx_lmm_info *info)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
-+static inline int scmi_imx_lmm_reset_vector_set(u32 lmid, u32 cpuid, u32 flags, u64 vector)
-+{
-+	return -EOPNOTSUPP;
-+}
-+#endif
-+#endif
-diff --git a/include/linux/firmware/intel/stratix10-smc.h b/include/linux/firmware/intel/stratix10-smc.h
-index a718f853d457..935dba3633b5 100644
--- a/include/linux/firmware/intel/stratix10-smc.h
-+++ b/include/linux/firmware/intel/stratix10-smc.h
-@@ -1,6 +1,7 @@
- /* SPDX-License-Identifier: GPL-2.0 */
- /*
-  * Copyright (C) 2017-2018, Intel Corporation
-+ * Copyright (C) 2025, Altera Corporation
-  */
- 
- #ifndef __STRATIX10_SMC_H
-@@ -47,6 +48,10 @@
- 	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, \
- 	ARM_SMCCC_OWNER_SIP, (func_num))
- 
-+#define INTEL_SIP_SMC_ASYNC_VAL(func_name)	\
-+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_STD_CALL, ARM_SMCCC_SMC_64, \
-+	ARM_SMCCC_OWNER_SIP, (func_name))
-+
- /**
-  * Return values in INTEL_SIP_SMC_* call
-  *
-@@ -467,6 +472,31 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE)
-         INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FIRMWARE_VERSION)
- 
- /**
-+ * SMC call protocol for Mailbox, starting FUNCID from 60
-+ *
-+ * Call register usage:
-+ * a0 INTEL_SIP_SMC_MBOX_SEND_CMD
-+ * a1 mailbox command code
-+ * a2 physical address that contain mailbox command data (not include header)
-+ * a3 mailbox command data size in word
-+ * a4 set to 0 for CASUAL, set to 1 for URGENT
-+ * a5 physical address for secure firmware to put response data
-+ *    (not include header)
-+ * a6 maximum size in word of physical address to store response data
-+ * a7 not used
-+ *
-+ * Return status
-+ * a0 INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_STATUS_REJECTED or
-+ *    INTEL_SIP_SMC_STATUS_ERROR
-+ * a1 mailbox error code
-+ * a2 response data length in word
-+ * a3 not used
-+ */
-+#define INTEL_SIP_SMC_FUNCID_MBOX_SEND_CMD 60
-+	#define INTEL_SIP_SMC_MBOX_SEND_CMD \
-+	INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_MBOX_SEND_CMD)
-+
-+/**
-  * Request INTEL_SIP_SMC_SVC_VERSION
-  *
-  * Sync call used to query the SIP SMC API Version
-@@ -595,4 +625,110 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE)
- #define INTEL_SIP_SMC_FCS_GET_PROVISION_DATA \
- 	INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FCS_GET_PROVISION_DATA)
- 
-+/**
-+ * Request INTEL_SIP_SMC_HWMON_READTEMP
-+ * Sync call to request temperature
-+ *
-+ * Call register usage:
-+ * a0 Temperature Channel
-+ * a1-a7 not used
-+ *
-+ * Return status
-+ * a0 INTEL_SIP_SMC_STATUS_OK
-+ * a1 Temperature Value
-+ * a2-a3 not used
-+ */
-+#define INTEL_SIP_SMC_FUNCID_HWMON_READTEMP 32
-+#define INTEL_SIP_SMC_HWMON_READTEMP \
-+	INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_HWMON_READTEMP)
-+
-+/**
-+ * Request INTEL_SIP_SMC_HWMON_READVOLT
-+ * Sync call to request voltage
-+ *
-+ * Call register usage:
-+ * a0 Voltage Channel
-+ * a1-a7 not used
-+ *
-+ * Return status
-+ * a0 INTEL_SIP_SMC_STATUS_OK
-+ * a1 Voltage Value
-+ * a2-a3 not used
-+ */
-+#define INTEL_SIP_SMC_FUNCID_HWMON_READVOLT 33
-+#define INTEL_SIP_SMC_HWMON_READVOLT \
-+	INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_HWMON_READVOLT)
-+
-+/**
-+ * Request INTEL_SIP_SMC_ASYNC_POLL
-+ * Async call used by service driver at EL1 to query mailbox response from SDM.
-+ *
-+ * Call register usage:
-+ * a0 INTEL_SIP_SMC_ASYNC_POLL
-+ * a1 transaction job id
-+ * a2-17 will be used to return the response data
-+ *
-+ * Return status
-+ * a0 INTEL_SIP_SMC_STATUS_OK
-+ * a1-17 will contain the response values from mailbox for the previous send
-+ * transaction
-+ * Or
-+ * a0 INTEL_SIP_SMC_STATUS_NO_RESPONSE
-+ * a1-17 not used
-+ */
-+#define INTEL_SIP_SMC_ASYNC_FUNC_ID_POLL (0xC8)
-+#define INTEL_SIP_SMC_ASYNC_POLL \
-+	INTEL_SIP_SMC_ASYNC_VAL(INTEL_SIP_SMC_ASYNC_FUNC_ID_POLL)
-+
-+/**
-+ * Request INTEL_SIP_SMC_ASYNC_RSU_GET_SPT
-+ * Async call to get RSU SPT from SDM.
-+ * Call register usage:
-+ * a0 INTEL_SIP_SMC_ASYNC_RSU_GET_SPT
-+ * a1 transaction job id
-+ * a2-a17 not used
-+ *
-+ * Return status:
-+ * a0 INTEL_SIP_SMC_STATUS_OK ,INTEL_SIP_SMC_STATUS_REJECTED
-+ * or INTEL_SIP_SMC_STATUS_BUSY
-+ * a1-a17 not used
-+ */
-+#define INTEL_SIP_SMC_ASYNC_FUNC_ID_RSU_GET_SPT (0xEA)
-+#define INTEL_SIP_SMC_ASYNC_RSU_GET_SPT \
-+	INTEL_SIP_SMC_ASYNC_VAL(INTEL_SIP_SMC_ASYNC_FUNC_ID_RSU_GET_SPT)
-+
-+/**
-+ * Request INTEL_SIP_SMC_ASYNC_RSU_GET_ERROR_STATUS
-+ * Async call to get RSU error status from SDM.
-+ * Call register usage:
-+ * a0 INTEL_SIP_SMC_ASYNC_RSU_GET_ERROR_STATUS
-+ * a1 transaction job id
-+ * a2-a17 not used
-+ *
-+ * Return status:
-+ * a0 INTEL_SIP_SMC_STATUS_OK ,INTEL_SIP_SMC_STATUS_REJECTED
-+ * or INTEL_SIP_SMC_STATUS_BUSY
-+ * a1-a17 not used
-+ */
-+#define INTEL_SIP_SMC_ASYNC_FUNC_ID_RSU_GET_ERROR_STATUS (0xEB)
-+#define INTEL_SIP_SMC_ASYNC_RSU_GET_ERROR_STATUS \
-+	INTEL_SIP_SMC_ASYNC_VAL(INTEL_SIP_SMC_ASYNC_FUNC_ID_RSU_GET_ERROR_STATUS)
-+
-+/**
-+ * Request INTEL_SIP_SMC_ASYNC_RSU_NOTIFY
-+ * Async call to send NOTIFY value to SDM.
-+ * Call register usage:
-+ * a0 INTEL_SIP_SMC_ASYNC_RSU_NOTIFY
-+ * a1 transaction job id
-+ * a2 notify value
-+ * a3-a17 not used
-+ *
-+ * Return status:
-+ * a0 INTEL_SIP_SMC_STATUS_OK ,INTEL_SIP_SMC_STATUS_REJECTED
-+ * or INTEL_SIP_SMC_STATUS_BUSY
-+ * a1-a17 not used
-+ */
-+#define INTEL_SIP_SMC_ASYNC_FUNC_ID_RSU_NOTIFY (0xEC)
-+#define INTEL_SIP_SMC_ASYNC_RSU_NOTIFY \
-+	INTEL_SIP_SMC_ASYNC_VAL(INTEL_SIP_SMC_ASYNC_FUNC_ID_RSU_NOTIFY)
- #endif
-diff --git a/include/linux/firmware/intel/stratix10-svc-client.h b/include/linux/firmware/intel/stratix10-svc-client.h
-index 0c16037fd08d..d290060f4c73 100644
--- a/include/linux/firmware/intel/stratix10-svc-client.h
-+++ b/include/linux/firmware/intel/stratix10-svc-client.h
-@@ -1,6 +1,7 @@
- /* SPDX-License-Identifier: GPL-2.0 */
- /*
-  * Copyright (C) 2017-2018, Intel Corporation
-+ * Copyright (C) 2025, Altera Corporation
-  */
- 
- #ifndef __STRATIX10_SVC_CLIENT_H
-@@ -11,10 +12,12 @@
-  *
-  * fpga: for FPGA configuration
-  * rsu: for remote status update
-+ * hwmon: for hardware monitoring (voltage and temperature)
-  */
- #define SVC_CLIENT_FPGA			"fpga"
- #define SVC_CLIENT_RSU			"rsu"
- #define SVC_CLIENT_FCS			"fcs"
-+#define SVC_CLIENT_HWMON		"hwmon"
- 
- /*
-  * Status of the sent command, in bit number
-@@ -70,6 +73,7 @@
- #define SVC_RSU_REQUEST_TIMEOUT_MS              300
- #define SVC_FCS_REQUEST_TIMEOUT_MS		2000
- #define SVC_COMPLETED_TIMEOUT_MS		30000
-+#define SVC_HWMON_REQUEST_TIMEOUT_MS		300
- 
- struct stratix10_svc_chan;
- 
-@@ -118,9 +122,15 @@ struct stratix10_svc_chan;
-  * @COMMAND_SMC_SVC_VERSION: Non-mailbox SMC SVC API Version,
-  * return status is SVC_STATUS_OK
-  *
-+ * @COMMAND_MBOX_SEND_CMD: send generic mailbox command, return status is
-+ * SVC_STATUS_OK or SVC_STATUS_ERROR
-+ *
-  * @COMMAND_RSU_DCMF_STATUS: query firmware for the DCMF status
-  * return status is SVC_STATUS_OK or SVC_STATUS_ERROR
-  *
-+ * @COMMAND_RSU_GET_SPT_TABLE: query firmware for SPT table
-+ * return status is SVC_STATUS_OK or SVC_STATUS_ERROR
-+ *
-  * @COMMAND_FCS_REQUEST_SERVICE: request validation of image from firmware,
-  * return status is SVC_STATUS_OK, SVC_STATUS_INVALID_PARAM
-  *
-@@ -138,6 +148,12 @@ struct stratix10_svc_chan;
-  *
-  * @COMMAND_FCS_RANDOM_NUMBER_GEN: generate a random number, return status
-  * is SVC_STATUS_OK, SVC_STATUS_ERROR
-+ *
-+ * @COMMAND_HWMON_READTEMP: query the temperature from the hardware monitor,
-+ * return status is SVC_STATUS_OK or SVC_STATUS_ERROR
-+ *
-+ * @COMMAND_HWMON_READVOLT: query the voltage from the hardware monitor,
-+ * return status is SVC_STATUS_OK or SVC_STATUS_ERROR
-  */
- enum stratix10_svc_command_code {
- 	/* for FPGA */
-@@ -155,6 +171,7 @@ enum stratix10_svc_command_code {
- 	COMMAND_RSU_DCMF_VERSION,
- 	COMMAND_RSU_DCMF_STATUS,
- 	COMMAND_FIRMWARE_VERSION,
-+	COMMAND_RSU_GET_SPT_TABLE,
- 	/* for FCS */
- 	COMMAND_FCS_REQUEST_SERVICE = 20,
- 	COMMAND_FCS_SEND_CERTIFICATE,
-@@ -164,8 +181,13 @@ enum stratix10_svc_command_code {
- 	COMMAND_FCS_RANDOM_NUMBER_GEN,
- 	/* for general status poll */
- 	COMMAND_POLL_SERVICE_STATUS = 40,
-+	/* for generic mailbox send command */
-+	COMMAND_MBOX_SEND_CMD = 100,
- 	/* Non-mailbox SMC Call */
- 	COMMAND_SMC_SVC_VERSION = 200,
-+	/* for HWMON */
-+	COMMAND_HWMON_READTEMP,
-+	COMMAND_HWMON_READVOLT
- };
- 
- /**
-@@ -279,5 +301,92 @@ int stratix10_svc_send(struct stratix10_svc_chan *chan, void *msg);
-  * request process.
-  */
- void stratix10_svc_done(struct stratix10_svc_chan *chan);
-+
-+/**
-+ * typedef async_callback_t - A type definition for an asynchronous callback function.
-+ *
-+ * This type defines a function pointer for an asynchronous callback.
-+ * The callback function takes a single argument, which is a pointer to
-+ * user-defined data.
-+ *
-+ * @cb_arg: Argument to be passed to the callback function.
-+ */
-+typedef void (*async_callback_t)(void *cb_arg);
-+
-+/**
-+ * stratix10_svc_add_async_client - Add an asynchronous client to a Stratix 10
-+ *                                  service channel.
-+ * @chan: Pointer to the Stratix 10 service channel structure.
-+ * @use_unique_clientid: Boolean flag indicating whether to use a unique client ID.
-+ *
-+ * This function registers an asynchronous client with the specified Stratix 10
-+ * service channel. If the use_unique_clientid flag is set to true, a unique client
-+ * ID will be assigned to the client.
-+ *
-+ * Return: 0 on success, or a negative error code on failure:
-+ *         -EINVAL if the channel is NULL or the async controller is not initialized.
-+ *         -EALREADY if the async channel is already allocated.
-+ *         -ENOMEM if memory allocation fails.
-+ *         Other negative values if ID allocation fails
-+ */
-+int stratix10_svc_add_async_client(struct stratix10_svc_chan *chan, bool use_unique_clientid);
-+
-+/**
-+ * stratix10_svc_remove_async_client - Remove an asynchronous client from the Stratix 10
-+ *                                     service channel.
-+ * @chan: Pointer to the Stratix 10 service channel structure.
-+ *
-+ * This function removes an asynchronous client from the specified Stratix 10 service channel.
-+ * It is typically used to clean up and release resources associated with the client.
-+ *
-+ * Return: 0 on success, -EINVAL if the channel or asynchronous channel is invalid.
-+ */
-+int stratix10_svc_remove_async_client(struct stratix10_svc_chan *chan);
-+
-+/**
-+ * stratix10_svc_async_send - Send an asynchronous message to the SDM mailbox
-+ *                            in EL3 secure firmware.
-+ * @chan: Pointer to the service channel structure.
-+ * @msg: Pointer to the message to be sent.
-+ * @handler: Pointer to the handler object used by caller to track the transaction.
-+ * @cb: Callback function to be called upon completion.
-+ * @cb_arg: Argument to be passed to the callback function.
-+ *
-+ * This function sends a message asynchronously to the SDM mailbox in EL3 secure firmware.
-+ * and registers a callback function to be invoked when the operation completes.
-+ *
-+ * Return: 0 on success,and negative error codes on failure.
-+ */
-+int stratix10_svc_async_send(struct stratix10_svc_chan *chan, void *msg, void **handler,
-+			     async_callback_t cb, void *cb_arg);
-+
-+/**
-+ * stratix10_svc_async_poll - Polls the status of an asynchronous service request.
-+ * @chan: Pointer to the service channel structure.
-+ * @tx_handle: Handle to the transaction being polled.
-+ * @data: Pointer to the callback data structure to be filled with the result.
-+ *
-+ * This function checks the status of an asynchronous service request
-+ * and fills the provided callback data structure with the result.
-+ *
-+ * Return: 0 on success, -EINVAL if any input parameter is invalid or if the
-+ *         async controller is not initialized, -EAGAIN if the transaction is
-+ *         still in progress, or other negative error codes on failure.
-+ */
-+int stratix10_svc_async_poll(struct stratix10_svc_chan *chan, void *tx_handle,
-+			     struct stratix10_svc_cb_data *data);
-+
-+/**
-+ * stratix10_svc_async_done - Complete an asynchronous transaction
-+ * @chan: Pointer to the service channel structure
-+ * @tx_handle: Pointer to the transaction handle
-+ *
-+ * This function completes an asynchronous transaction by removing the
-+ * transaction from the hash table and deallocating the associated resources.
-+ *
-+ * Return: 0 on success, -EINVAL on invalid input or errors.
-+ */
-+int stratix10_svc_async_done(struct stratix10_svc_chan *chan, void *tx_handle);
-+
- #endif
- 
-diff --git a/include/linux/firmware/mediatek/mtk-adsp-ipc.h b/include/linux/firmware/mediatek/mtk-adsp-ipc.h
-index 28fd313340b8..6e86799a7dc4 100644
--- a/include/linux/firmware/mediatek/mtk-adsp-ipc.h
-+++ b/include/linux/firmware/mediatek/mtk-adsp-ipc.h
-@@ -40,23 +40,17 @@ struct mtk_adsp_chan {
- struct mtk_adsp_ipc {
- 	struct mtk_adsp_chan chans[MTK_ADSP_MBOX_NUM];
- 	struct device *dev;
-	struct mtk_adsp_ipc_ops *ops;
-+	const struct mtk_adsp_ipc_ops *ops;
- 	void *private_data;
- };
- 
- static inline void mtk_adsp_ipc_set_data(struct mtk_adsp_ipc *ipc, void *data)
- {
-	if (!ipc)
-		return;
-
- 	ipc->private_data = data;
- }
- 
- static inline void *mtk_adsp_ipc_get_data(struct mtk_adsp_ipc *ipc)
- {
-	if (!ipc)
-		return NULL;
-
- 	return ipc->private_data;
- }
- 
-diff --git a/include/linux/firmware/meson/meson_sm.h b/include/linux/firmware/meson/meson_sm.h
-index 95b0da2326a9..8eaf8922ab02 100644
--- a/include/linux/firmware/meson/meson_sm.h
-+++ b/include/linux/firmware/meson/meson_sm.h
-@@ -19,7 +19,7 @@ enum {
- struct meson_sm_firmware;
- 
- int meson_sm_call(struct meson_sm_firmware *fw, unsigned int cmd_index,
-		  u32 *ret, u32 arg0, u32 arg1, u32 arg2, u32 arg3, u32 arg4);
-+		  s32 *ret, u32 arg0, u32 arg1, u32 arg2, u32 arg3, u32 arg4);
- int meson_sm_call_write(struct meson_sm_firmware *fw, void *buffer,
- 			unsigned int b_size, unsigned int cmd_index, u32 arg0,
- 			u32 arg1, u32 arg2, u32 arg3, u32 arg4);
-diff --git a/include/linux/firmware/qcom/qcom_qseecom.h b/include/linux/firmware/qcom/qcom_qseecom.h
-new file mode 100644
-index 000000000000..3387897bf368
--- /dev/null
-+++ b/include/linux/firmware/qcom/qcom_qseecom.h
-@@ -0,0 +1,54 @@
-+/* SPDX-License-Identifier: GPL-2.0-or-later */
-+/*
-+ * Driver for Qualcomm Secure Execution Environment (SEE) interface (QSEECOM).
-+ * Responsible for setting up and managing QSEECOM client devices.
-+ *
-+ * Copyright (C) 2023 Maximilian Luz <luzmaximilian@gmail.com>
-+ */
-+
-+#ifndef __QCOM_QSEECOM_H
-+#define __QCOM_QSEECOM_H
-+
-+#include <linux/auxiliary_bus.h>
-+#include <linux/dma-mapping.h>
-+#include <linux/types.h>
-+
-+#include <linux/firmware/qcom/qcom_scm.h>
-+
-+/**
-+ * struct qseecom_client - QSEECOM client device.
-+ * @aux_dev: Underlying auxiliary device.
-+ * @app_id: ID of the loaded application.
-+ */
-+struct qseecom_client {
-+	struct auxiliary_device aux_dev;
-+	u32 app_id;
-+};
-+
-+/**
-+ * qcom_qseecom_app_send() - Send to and receive data from a given QSEE app.
-+ * @client:   The QSEECOM client associated with the target app.
-+ * @req:      Request buffer sent to the app (must be TZ memory).
-+ * @req_size: Size of the request buffer.
-+ * @rsp:      Response buffer, written to by the app (must be TZ memory).
-+ * @rsp_size: Size of the response buffer.
-+ *
-+ * Sends a request to the QSEE app associated with the given client and read
-+ * back its response. The caller must provide two DMA memory regions, one for
-+ * the request and one for the response, and fill out the @req region with the
-+ * respective (app-specific) request data. The QSEE app reads this and returns
-+ * its response in the @rsp region.
-+ *
-+ * Note: This is a convenience wrapper around qcom_scm_qseecom_app_send().
-+ * Clients should prefer to use this wrapper.
-+ *
-+ * Return: Zero on success, nonzero on failure.
-+ */
-+static inline int qcom_qseecom_app_send(struct qseecom_client *client,
-+					void *req, size_t req_size,
-+					void *rsp, size_t rsp_size)
-+{
-+	return qcom_scm_qseecom_app_send(client->app_id, req, req_size, rsp, rsp_size);
-+}
-+
-+#endif /* __QCOM_QSEECOM_H */
-diff --git a/include/linux/firmware/qcom/qcom_scm.h b/include/linux/firmware/qcom/qcom_scm.h
-new file mode 100644
-index 000000000000..a55ca771286b
--- /dev/null
-+++ b/include/linux/firmware/qcom/qcom_scm.h
-@@ -0,0 +1,184 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/* Copyright (c) 2010-2015, 2018-2019 The Linux Foundation. All rights reserved.
-+ * Copyright (C) 2015 Linaro Ltd.
-+ */
-+#ifndef __QCOM_SCM_H
-+#define __QCOM_SCM_H
-+
-+#include <linux/err.h>
-+#include <linux/types.h>
-+#include <linux/cpumask.h>
-+
-+#include <dt-bindings/firmware/qcom,scm.h>
-+
-+#define QCOM_SCM_VERSION(major, minor)	(((major) << 16) | ((minor) & 0xFF))
-+#define QCOM_SCM_CPU_PWR_DOWN_L2_ON	0x0
-+#define QCOM_SCM_CPU_PWR_DOWN_L2_OFF	0x1
-+#define QCOM_SCM_HDCP_MAX_REQ_CNT	5
-+
-+struct qcom_scm_hdcp_req {
-+	u32 addr;
-+	u32 val;
-+};
-+
-+struct qcom_scm_vmperm {
-+	int vmid;
-+	int perm;
-+};
-+
-+enum qcom_scm_ocmem_client {
-+	QCOM_SCM_OCMEM_UNUSED_ID = 0x0,
-+	QCOM_SCM_OCMEM_GRAPHICS_ID,
-+	QCOM_SCM_OCMEM_VIDEO_ID,
-+	QCOM_SCM_OCMEM_LP_AUDIO_ID,
-+	QCOM_SCM_OCMEM_SENSORS_ID,
-+	QCOM_SCM_OCMEM_OTHER_OS_ID,
-+	QCOM_SCM_OCMEM_DEBUG_ID,
-+};
-+
-+enum qcom_scm_sec_dev_id {
-+	QCOM_SCM_MDSS_DEV_ID    = 1,
-+	QCOM_SCM_OCMEM_DEV_ID   = 5,
-+	QCOM_SCM_PCIE0_DEV_ID   = 11,
-+	QCOM_SCM_PCIE1_DEV_ID   = 12,
-+	QCOM_SCM_GFX_DEV_ID     = 18,
-+	QCOM_SCM_UFS_DEV_ID     = 19,
-+	QCOM_SCM_ICE_DEV_ID     = 20,
-+};
-+
-+enum qcom_scm_ice_cipher {
-+	QCOM_SCM_ICE_CIPHER_AES_128_XTS = 0,
-+	QCOM_SCM_ICE_CIPHER_AES_128_CBC = 1,
-+	QCOM_SCM_ICE_CIPHER_AES_256_XTS = 3,
-+	QCOM_SCM_ICE_CIPHER_AES_256_CBC = 4,
-+};
-+
-+#define QCOM_SCM_PERM_READ       0x4
-+#define QCOM_SCM_PERM_WRITE      0x2
-+#define QCOM_SCM_PERM_EXEC       0x1
-+#define QCOM_SCM_PERM_RW (QCOM_SCM_PERM_READ | QCOM_SCM_PERM_WRITE)
-+#define QCOM_SCM_PERM_RWX (QCOM_SCM_PERM_RW | QCOM_SCM_PERM_EXEC)
-+
-+bool qcom_scm_is_available(void);
-+
-+int qcom_scm_set_cold_boot_addr(void *entry);
-+int qcom_scm_set_warm_boot_addr(void *entry);
-+void qcom_scm_cpu_power_down(u32 flags);
-+int qcom_scm_set_remote_state(u32 state, u32 id);
-+
-+struct qcom_scm_pas_metadata {
-+	void *ptr;
-+	dma_addr_t phys;
-+	ssize_t size;
-+};
-+
-+int qcom_scm_pas_init_image(u32 peripheral, const void *metadata, size_t size,
-+			    struct qcom_scm_pas_metadata *ctx);
-+void qcom_scm_pas_metadata_release(struct qcom_scm_pas_metadata *ctx);
-+int qcom_scm_pas_mem_setup(u32 peripheral, phys_addr_t addr, phys_addr_t size);
-+int qcom_scm_pas_auth_and_reset(u32 peripheral);
-+int qcom_scm_pas_shutdown(u32 peripheral);
-+bool qcom_scm_pas_supported(u32 peripheral);
-+
-+int qcom_scm_io_readl(phys_addr_t addr, unsigned int *val);
-+int qcom_scm_io_writel(phys_addr_t addr, unsigned int val);
-+
-+bool qcom_scm_restore_sec_cfg_available(void);
-+int qcom_scm_restore_sec_cfg(u32 device_id, u32 spare);
-+int qcom_scm_set_gpu_smmu_aperture(unsigned int context_bank);
-+bool qcom_scm_set_gpu_smmu_aperture_is_available(void);
-+int qcom_scm_iommu_secure_ptbl_size(u32 spare, size_t *size);
-+int qcom_scm_iommu_secure_ptbl_init(u64 addr, u32 size, u32 spare);
-+int qcom_scm_iommu_set_cp_pool_size(u32 spare, u32 size);
-+int qcom_scm_mem_protect_video_var(u32 cp_start, u32 cp_size,
-+				   u32 cp_nonpixel_start, u32 cp_nonpixel_size);
-+int qcom_scm_assign_mem(phys_addr_t mem_addr, size_t mem_sz, u64 *src,
-+			const struct qcom_scm_vmperm *newvm,
-+			unsigned int dest_cnt);
-+
-+bool qcom_scm_ocmem_lock_available(void);
-+int qcom_scm_ocmem_lock(enum qcom_scm_ocmem_client id, u32 offset, u32 size,
-+			u32 mode);
-+int qcom_scm_ocmem_unlock(enum qcom_scm_ocmem_client id, u32 offset, u32 size);
-+
-+bool qcom_scm_ice_available(void);
-+int qcom_scm_ice_invalidate_key(u32 index);
-+int qcom_scm_ice_set_key(u32 index, const u8 *key, u32 key_size,
-+			 enum qcom_scm_ice_cipher cipher, u32 data_unit_size);
-+bool qcom_scm_has_wrapped_key_support(void);
-+int qcom_scm_derive_sw_secret(const u8 *eph_key, size_t eph_key_size,
-+			      u8 *sw_secret, size_t sw_secret_size);
-+int qcom_scm_generate_ice_key(u8 *lt_key, size_t lt_key_size);
-+int qcom_scm_prepare_ice_key(const u8 *lt_key, size_t lt_key_size,
-+			     u8 *eph_key, size_t eph_key_size);
-+int qcom_scm_import_ice_key(const u8 *raw_key, size_t raw_key_size,
-+			    u8 *lt_key, size_t lt_key_size);
-+
-+bool qcom_scm_hdcp_available(void);
-+int qcom_scm_hdcp_req(struct qcom_scm_hdcp_req *req, u32 req_cnt, u32 *resp);
-+
-+int qcom_scm_iommu_set_pt_format(u32 sec_id, u32 ctx_num, u32 pt_fmt);
-+int qcom_scm_qsmmu500_wait_safe_toggle(bool en);
-+
-+int qcom_scm_lmh_dcvsh(u32 payload_fn, u32 payload_reg, u32 payload_val,
-+		       u64 limit_node, u32 node_id, u64 version);
-+int qcom_scm_lmh_profile_change(u32 profile_id);
-+bool qcom_scm_lmh_dcvsh_available(void);
-+
-+/*
-+ * Request TZ to program set of access controlled registers necessary
-+ * irrespective of any features
-+ */
-+#define QCOM_SCM_GPU_ALWAYS_EN_REQ BIT(0)
-+/*
-+ * Request TZ to program BCL id to access controlled register when BCL is
-+ * enabled
-+ */
-+#define QCOM_SCM_GPU_BCL_EN_REQ BIT(1)
-+/*
-+ * Request TZ to program set of access controlled register for CLX feature
-+ * when enabled
-+ */
-+#define QCOM_SCM_GPU_CLX_EN_REQ BIT(2)
-+/*
-+ * Request TZ to program tsense ids to access controlled registers for reading
-+ * gpu temperature sensors
-+ */
-+#define QCOM_SCM_GPU_TSENSE_EN_REQ BIT(3)
-+
-+int qcom_scm_gpu_init_regs(u32 gpu_req);
-+
-+int qcom_scm_shm_bridge_create(u64 pfn_and_ns_perm_flags,
-+			       u64 ipfn_and_s_perm_flags, u64 size_and_flags,
-+			       u64 ns_vmids, u64 *handle);
-+int qcom_scm_shm_bridge_delete(u64 handle);
-+
-+#ifdef CONFIG_QCOM_QSEECOM
-+
-+int qcom_scm_qseecom_app_get_id(const char *app_name, u32 *app_id);
-+int qcom_scm_qseecom_app_send(u32 app_id, void *req, size_t req_size,
-+			      void *rsp, size_t rsp_size);
-+
-+#else /* CONFIG_QCOM_QSEECOM */
-+
-+static inline int qcom_scm_qseecom_app_get_id(const char *app_name, u32 *app_id)
-+{
-+	return -EINVAL;
-+}
-+
-+static inline int qcom_scm_qseecom_app_send(u32 app_id,
-+					    void *req, size_t req_size,
-+					    void *rsp, size_t rsp_size)
-+{
-+	return -EINVAL;
-+}
-+
-+#endif /* CONFIG_QCOM_QSEECOM */
-+
-+int qcom_scm_qtee_invoke_smc(phys_addr_t inbuf, size_t inbuf_size,
-+			     phys_addr_t outbuf, size_t outbuf_size,
-+			     u64 *result, u64 *response_type);
-+int qcom_scm_qtee_callback_response(phys_addr_t buf, size_t buf_size,
-+				    u64 *result, u64 *response_type);
-+
-+#endif
-diff --git a/include/linux/firmware/qcom/qcom_tzmem.h b/include/linux/firmware/qcom/qcom_tzmem.h
-new file mode 100644
-index 000000000000..23173e0c3ddd
--- /dev/null
-+++ b/include/linux/firmware/qcom/qcom_tzmem.h
-@@ -0,0 +1,80 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Copyright (C) 2023-2024 Linaro Ltd.
-+ */
-+
-+#ifndef __QCOM_TZMEM_H
-+#define __QCOM_TZMEM_H
-+
-+#include <linux/cleanup.h>
-+#include <linux/gfp.h>
-+#include <linux/types.h>
-+
-+struct device;
-+struct qcom_tzmem_pool;
-+
-+/**
-+ * enum qcom_tzmem_policy - Policy for pool growth.
-+ */
-+enum qcom_tzmem_policy {
-+	/**
-+	 * @QCOM_TZMEM_POLICY_STATIC: Static pool,
-+	 * never grow above initial size.
-+	 */
-+	QCOM_TZMEM_POLICY_STATIC = 1,
-+	/**
-+	 * @QCOM_TZMEM_POLICY_MULTIPLIER: When out of memory,
-+	 * add increment * current size of memory.
-+	 */
-+	QCOM_TZMEM_POLICY_MULTIPLIER,
-+	/**
-+	 * @QCOM_TZMEM_POLICY_ON_DEMAND: When out of memory
-+	 * add as much as is needed until max_size.
-+	 */
-+	QCOM_TZMEM_POLICY_ON_DEMAND,
-+};
-+
-+/**
-+ * struct qcom_tzmem_pool_config - TZ memory pool configuration.
-+ * @initial_size: Number of bytes to allocate for the pool during its creation.
-+ * @policy: Pool size growth policy.
-+ * @increment: Used with policies that allow pool growth.
-+ * @max_size: Size above which the pool will never grow.
-+ */
-+struct qcom_tzmem_pool_config {
-+	size_t initial_size;
-+	enum qcom_tzmem_policy policy;
-+	size_t increment;
-+	size_t max_size;
-+};
-+
-+struct qcom_tzmem_pool *
-+qcom_tzmem_pool_new(const struct qcom_tzmem_pool_config *config);
-+void qcom_tzmem_pool_free(struct qcom_tzmem_pool *pool);
-+struct qcom_tzmem_pool *
-+devm_qcom_tzmem_pool_new(struct device *dev,
-+			 const struct qcom_tzmem_pool_config *config);
-+
-+void *qcom_tzmem_alloc(struct qcom_tzmem_pool *pool, size_t size, gfp_t gfp);
-+void qcom_tzmem_free(void *ptr);
-+
-+DEFINE_FREE(qcom_tzmem, void *, if (_T) qcom_tzmem_free(_T))
-+
-+phys_addr_t qcom_tzmem_to_phys(void *ptr);
-+
-+#if IS_ENABLED(CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE)
-+int qcom_tzmem_shm_bridge_create(phys_addr_t paddr, size_t size, u64 *handle);
-+void qcom_tzmem_shm_bridge_delete(u64 handle);
-+#else
-+static inline int qcom_tzmem_shm_bridge_create(phys_addr_t paddr,
-+					       size_t size, u64 *handle)
-+{
-+	return 0;
-+}
-+
-+static inline void qcom_tzmem_shm_bridge_delete(u64 handle)
-+{
-+}
-+#endif
-+
-+#endif /* __QCOM_TZMEM */
-diff --git a/include/linux/firmware/samsung/exynos-acpm-protocol.h b/include/linux/firmware/samsung/exynos-acpm-protocol.h
-new file mode 100644
-index 000000000000..2091da965a5a
--- /dev/null
-+++ b/include/linux/firmware/samsung/exynos-acpm-protocol.h
-@@ -0,0 +1,70 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Copyright 2020 Samsung Electronics Co., Ltd.
-+ * Copyright 2020 Google LLC.
-+ * Copyright 2024 Linaro Ltd.
-+ */
-+
-+#ifndef __EXYNOS_ACPM_PROTOCOL_H
-+#define __EXYNOS_ACPM_PROTOCOL_H
-+
-+#include <linux/types.h>
-+
-+struct acpm_handle;
-+struct device_node;
-+
-+struct acpm_dvfs_ops {
-+	int (*set_rate)(const struct acpm_handle *handle,
-+			unsigned int acpm_chan_id, unsigned int clk_id,
-+			unsigned long rate);
-+	unsigned long (*get_rate)(const struct acpm_handle *handle,
-+				  unsigned int acpm_chan_id,
-+				  unsigned int clk_id);
-+};
-+
-+struct acpm_pmic_ops {
-+	int (*read_reg)(const struct acpm_handle *handle,
-+			unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
-+			u8 *buf);
-+	int (*bulk_read)(const struct acpm_handle *handle,
-+			 unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
-+			 u8 count, u8 *buf);
-+	int (*write_reg)(const struct acpm_handle *handle,
-+			 unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
-+			 u8 value);
-+	int (*bulk_write)(const struct acpm_handle *handle,
-+			  unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
-+			  u8 count, const u8 *buf);
-+	int (*update_reg)(const struct acpm_handle *handle,
-+			  unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
-+			  u8 value, u8 mask);
-+};
-+
-+struct acpm_ops {
-+	struct acpm_dvfs_ops dvfs_ops;
-+	struct acpm_pmic_ops pmic_ops;
-+};
-+
-+/**
-+ * struct acpm_handle - Reference to an initialized protocol instance
-+ * @ops:
-+ */
-+struct acpm_handle {
-+	struct acpm_ops ops;
-+};
-+
-+struct device;
-+
-+#if IS_ENABLED(CONFIG_EXYNOS_ACPM_PROTOCOL)
-+const struct acpm_handle *devm_acpm_get_by_node(struct device *dev,
-+						struct device_node *np);
-+#else
-+
-+static inline const struct acpm_handle *devm_acpm_get_by_node(struct device *dev,
-+							      struct device_node *np)
-+{
-+	return NULL;
-+}
-+#endif
-+
-+#endif /* __EXYNOS_ACPM_PROTOCOL_H */
-diff --git a/include/linux/firmware/thead/thead,th1520-aon.h b/include/linux/firmware/thead/thead,th1520-aon.h
-new file mode 100644
-index 000000000000..dae132b66873
--- /dev/null
-+++ b/include/linux/firmware/thead/thead,th1520-aon.h
-@@ -0,0 +1,200 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * Copyright (C) 2021 Alibaba Group Holding Limited.
-+ */
-+
-+#ifndef _THEAD_AON_H
-+#define _THEAD_AON_H
-+
-+#include <linux/device.h>
-+#include <linux/types.h>
-+
-+#define AON_RPC_MSG_MAGIC (0xef)
-+#define TH1520_AON_RPC_VERSION 2
-+#define TH1520_AON_RPC_MSG_NUM 7
-+
-+struct th1520_aon_chan;
-+
-+enum th1520_aon_rpc_svc {
-+	TH1520_AON_RPC_SVC_UNKNOWN = 0,
-+	TH1520_AON_RPC_SVC_PM = 1,
-+	TH1520_AON_RPC_SVC_MISC = 2,
-+	TH1520_AON_RPC_SVC_AVFS = 3,
-+	TH1520_AON_RPC_SVC_SYS = 4,
-+	TH1520_AON_RPC_SVC_WDG = 5,
-+	TH1520_AON_RPC_SVC_LPM = 6,
-+	TH1520_AON_RPC_SVC_MAX = 0x3F,
-+};
-+
-+enum th1520_aon_misc_func {
-+	TH1520_AON_MISC_FUNC_UNKNOWN = 0,
-+	TH1520_AON_MISC_FUNC_SET_CONTROL = 1,
-+	TH1520_AON_MISC_FUNC_GET_CONTROL = 2,
-+	TH1520_AON_MISC_FUNC_REGDUMP_CFG = 3,
-+};
-+
-+enum th1520_aon_wdg_func {
-+	TH1520_AON_WDG_FUNC_UNKNOWN = 0,
-+	TH1520_AON_WDG_FUNC_START = 1,
-+	TH1520_AON_WDG_FUNC_STOP = 2,
-+	TH1520_AON_WDG_FUNC_PING = 3,
-+	TH1520_AON_WDG_FUNC_TIMEOUTSET = 4,
-+	TH1520_AON_WDG_FUNC_RESTART = 5,
-+	TH1520_AON_WDG_FUNC_GET_STATE = 6,
-+	TH1520_AON_WDG_FUNC_POWER_OFF = 7,
-+	TH1520_AON_WDG_FUNC_AON_WDT_ON = 8,
-+	TH1520_AON_WDG_FUNC_AON_WDT_OFF = 9,
-+};
-+
-+enum th1520_aon_sys_func {
-+	TH1520_AON_SYS_FUNC_UNKNOWN = 0,
-+	TH1520_AON_SYS_FUNC_AON_RESERVE_MEM = 1,
-+};
-+
-+enum th1520_aon_lpm_func {
-+	TH1520_AON_LPM_FUNC_UNKNOWN = 0,
-+	TH1520_AON_LPM_FUNC_REQUIRE_STR = 1,
-+	TH1520_AON_LPM_FUNC_RESUME_STR = 2,
-+	TH1520_AON_LPM_FUNC_REQUIRE_STD = 3,
-+	TH1520_AON_LPM_FUNC_CPUHP = 4,
-+	TH1520_AON_LPM_FUNC_REGDUMP_CFG = 5,
-+};
-+
-+enum th1520_aon_pm_func {
-+	TH1520_AON_PM_FUNC_UNKNOWN = 0,
-+	TH1520_AON_PM_FUNC_SET_RESOURCE_REGULATOR = 1,
-+	TH1520_AON_PM_FUNC_GET_RESOURCE_REGULATOR = 2,
-+	TH1520_AON_PM_FUNC_SET_RESOURCE_POWER_MODE = 3,
-+	TH1520_AON_PM_FUNC_PWR_SET = 4,
-+	TH1520_AON_PM_FUNC_PWR_GET = 5,
-+	TH1520_AON_PM_FUNC_CHECK_FAULT = 6,
-+	TH1520_AON_PM_FUNC_GET_TEMPERATURE = 7,
-+};
-+
-+struct th1520_aon_rpc_msg_hdr {
-+	u8 ver; /* version of msg hdr */
-+	u8 size; /*  msg size ,uinit in bytes,the size includes rpc msg header self */
-+	u8 svc; /* rpc main service id */
-+	u8 func; /* rpc sub func id of specific service, sent by caller */
-+} __packed __aligned(1);
-+
-+struct th1520_aon_rpc_ack_common {
-+	struct th1520_aon_rpc_msg_hdr hdr;
-+	u8 err_code;
-+} __packed __aligned(1);
-+
-+#define RPC_SVC_MSG_TYPE_DATA 0
-+#define RPC_SVC_MSG_TYPE_ACK 1
-+#define RPC_SVC_MSG_NEED_ACK 0
-+#define RPC_SVC_MSG_NO_NEED_ACK 1
-+
-+#define RPC_GET_VER(MESG) ((MESG)->ver)
-+#define RPC_SET_VER(MESG, VER) ((MESG)->ver = (VER))
-+#define RPC_GET_SVC_ID(MESG) ((MESG)->svc & 0x3F)
-+#define RPC_SET_SVC_ID(MESG, ID) ((MESG)->svc |= 0x3F & (ID))
-+#define RPC_GET_SVC_FLAG_MSG_TYPE(MESG) (((MESG)->svc & 0x80) >> 7)
-+#define RPC_SET_SVC_FLAG_MSG_TYPE(MESG, TYPE) ((MESG)->svc |= (TYPE) << 7)
-+#define RPC_GET_SVC_FLAG_ACK_TYPE(MESG) (((MESG)->svc & 0x40) >> 6)
-+#define RPC_SET_SVC_FLAG_ACK_TYPE(MESG, ACK) ((MESG)->svc |= (ACK) << 6)
-+
-+#define RPC_SET_BE64(MESG, OFFSET, SET_DATA)                                \
-+	do {                                                                \
-+		u8 *data = (u8 *)(MESG);                                    \
-+		u64 _offset = (OFFSET);                                     \
-+		u64 _set_data = (SET_DATA);                                 \
-+		data[_offset + 7] = _set_data & 0xFF;                       \
-+		data[_offset + 6] = (_set_data & 0xFF00) >> 8;              \
-+		data[_offset + 5] = (_set_data & 0xFF0000) >> 16;           \
-+		data[_offset + 4] = (_set_data & 0xFF000000) >> 24;         \
-+		data[_offset + 3] = (_set_data & 0xFF00000000) >> 32;       \
-+		data[_offset + 2] = (_set_data & 0xFF0000000000) >> 40;     \
-+		data[_offset + 1] = (_set_data & 0xFF000000000000) >> 48;   \
-+		data[_offset + 0] = (_set_data & 0xFF00000000000000) >> 56; \
-+	} while (0)
-+
-+#define RPC_SET_BE32(MESG, OFFSET, SET_DATA)			    \
-+	do {							    \
-+		u8 *data = (u8 *)(MESG);			    \
-+		u64 _offset = (OFFSET);				    \
-+		u64 _set_data = (SET_DATA);			    \
-+		data[_offset + 3] = (_set_data) & 0xFF;		    \
-+		data[_offset + 2] = (_set_data & 0xFF00) >> 8;	    \
-+		data[_offset + 1] = (_set_data & 0xFF0000) >> 16;   \
-+		data[_offset + 0] = (_set_data & 0xFF000000) >> 24; \
-+	} while (0)
-+
-+#define RPC_SET_BE16(MESG, OFFSET, SET_DATA)		       \
-+	do {						       \
-+		u8 *data = (u8 *)(MESG);		       \
-+		u64 _offset = (OFFSET);			       \
-+		u64 _set_data = (SET_DATA);		       \
-+		data[_offset + 1] = (_set_data) & 0xFF;	       \
-+		data[_offset + 0] = (_set_data & 0xFF00) >> 8; \
-+	} while (0)
-+
-+#define RPC_SET_U8(MESG, OFFSET, SET_DATA)	  \
-+	do {					  \
-+		u8 *data = (u8 *)(MESG);	  \
-+		data[OFFSET] = (SET_DATA) & 0xFF; \
-+	} while (0)
-+
-+#define RPC_GET_BE64(MESG, OFFSET, PTR)                                      \
-+	do {                                                                 \
-+		u8 *data = (u8 *)(MESG);                                     \
-+		u64 _offset = (OFFSET);                                      \
-+		*(u32 *)(PTR) =                                              \
-+			(data[_offset + 7] | data[_offset + 6] << 8 |        \
-+			 data[_offset + 5] << 16 | data[_offset + 4] << 24 | \
-+			 data[_offset + 3] << 32 | data[_offset + 2] << 40 | \
-+			 data[_offset + 1] << 48 | data[_offset + 0] << 56); \
-+	} while (0)
-+
-+#define RPC_GET_BE32(MESG, OFFSET, PTR)                                      \
-+	do {                                                                 \
-+		u8 *data = (u8 *)(MESG);                                     \
-+		u64 _offset = (OFFSET);                                      \
-+		*(u32 *)(PTR) =                                              \
-+			(data[_offset + 3] | data[_offset + 2] << 8 |        \
-+			 data[_offset + 1] << 16 | data[_offset + 0] << 24); \
-+	} while (0)
-+
-+#define RPC_GET_BE16(MESG, OFFSET, PTR)                                       \
-+	do {                                                                  \
-+		u8 *data = (u8 *)(MESG);                                      \
-+		u64 _offset = (OFFSET);                                       \
-+		*(u16 *)(PTR) = (data[_offset + 1] | data[_offset + 0] << 8); \
-+	} while (0)
-+
-+#define RPC_GET_U8(MESG, OFFSET, PTR)          \
-+	do {                                   \
-+		u8 *data = (u8 *)(MESG);       \
-+		*(u8 *)(PTR) = (data[OFFSET]); \
-+	} while (0)
-+
-+/*
-+ * Defines for SC PM Power Mode
-+ */
-+#define TH1520_AON_PM_PW_MODE_OFF 0 /* Power off */
-+#define TH1520_AON_PM_PW_MODE_STBY 1 /* Power in standby */
-+#define TH1520_AON_PM_PW_MODE_LP 2 /* Power in low-power */
-+#define TH1520_AON_PM_PW_MODE_ON 3 /* Power on */
-+
-+/*
-+ * Defines for AON power islands
-+ */
-+#define TH1520_AON_AUDIO_PD 0
-+#define TH1520_AON_VDEC_PD 1
-+#define TH1520_AON_NPU_PD 2
-+#define TH1520_AON_VENC_PD 3
-+#define TH1520_AON_GPU_PD 4
-+#define TH1520_AON_DSP0_PD 5
-+#define TH1520_AON_DSP1_PD 6
-+
-+struct th1520_aon_chan *th1520_aon_init(struct device *dev);
-+void th1520_aon_deinit(struct th1520_aon_chan *aon_chan);
-+
-+int th1520_aon_call_rpc(struct th1520_aon_chan *aon_chan, void *msg);
-+int th1520_aon_power_update(struct th1520_aon_chan *aon_chan, u16 rsrc,
-+			    bool power_on);
-+
-+#endif /* _THEAD_AON_H */
-diff --git a/include/linux/firmware/xlnx-event-manager.h b/include/linux/firmware/xlnx-event-manager.h
-index 82e8254b0f80..645dd34155e6 100644
--- a/include/linux/firmware/xlnx-event-manager.h
-+++ b/include/linux/firmware/xlnx-event-manager.h
-@@ -1,4 +1,9 @@
- /* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * Xilinx Event Management Driver
-+ *
-+ * Copyright (C) 2024, Advanced Micro Devices, Inc.
-+ */
- 
- #ifndef _FIRMWARE_XLNX_EVENT_MANAGER_H_
- #define _FIRMWARE_XLNX_EVENT_MANAGER_H_
-@@ -7,6 +12,11 @@
- 
- #define CB_MAX_PAYLOAD_SIZE	(4U) /*In payload maximum 32bytes */
- 
-+#define EVENT_SUBSYSTEM_RESTART		(4U)
-+
-+#define PM_DEV_ACPU_0_0			(0x1810c0afU)
-+#define PM_DEV_ACPU_0			(0x1810c003U)
-+
- /************************** Exported Function *****************************/
- 
- typedef void (*event_cb_func_t)(const u32 *payload, void *data);
-diff --git a/include/linux/firmware/xlnx-zynqmp-ufs.h b/include/linux/firmware/xlnx-zynqmp-ufs.h
-new file mode 100644
-index 000000000000..d3538dd5822a
--- /dev/null
-+++ b/include/linux/firmware/xlnx-zynqmp-ufs.h
-@@ -0,0 +1,38 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * Firmware layer for UFS APIs.
-+ *
-+ * Copyright (c) 2025 Advanced Micro Devices, Inc.
-+ */
-+
-+#ifndef __FIRMWARE_XLNX_ZYNQMP_UFS_H__
-+#define __FIRMWARE_XLNX_ZYNQMP_UFS_H__
-+
-+#if IS_REACHABLE(CONFIG_ZYNQMP_FIRMWARE)
-+int zynqmp_pm_is_mphy_tx_rx_config_ready(bool *is_ready);
-+int zynqmp_pm_is_sram_init_done(bool *is_done);
-+int zynqmp_pm_set_sram_bypass(void);
-+int zynqmp_pm_get_ufs_calibration_values(u32 *val);
-+#else
-+static inline int zynqmp_pm_is_mphy_tx_rx_config_ready(bool *is_ready)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int zynqmp_pm_is_sram_init_done(bool *is_done)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int zynqmp_pm_set_sram_bypass(void)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int zynqmp_pm_get_ufs_calibration_values(u32 *val)
-+{
-+	return -ENODEV;
-+}
-+#endif
-+
-+#endif /* __FIRMWARE_XLNX_ZYNQMP_UFS_H__ */
-diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h
-index 9f50dacbf7d6..15fdbd089bbf 100644
--- a/include/linux/firmware/xlnx-zynqmp.h
-+++ b/include/linux/firmware/xlnx-zynqmp.h
-@@ -3,8 +3,9 @@
-  * Xilinx Zynq MPSoC Firmware layer
-  *
-  *  Copyright (C) 2014-2021 Xilinx
-+ *  Copyright (C) 2022 - 2025 Advanced Micro Devices, Inc.
-  *
- *  Michal Simek <michal.simek@xilinx.com>
-+ *  Michal Simek <michal.simek@amd.com>
-  *  Davorin Mista <davorin.mista@aggios.com>
-  *  Jolly Shah <jollys@xilinx.com>
-  *  Rajan Vaja <rajanv@xilinx.com>
-@@ -12,8 +13,10 @@
- 
- #ifndef __FIRMWARE_ZYNQMP_H__
- #define __FIRMWARE_ZYNQMP_H__
-+#include <linux/types.h>
- 
- #include <linux/err.h>
-+#include <linux/firmware/xlnx-zynqmp-ufs.h>
- 
- #define ZYNQMP_PM_VERSION_MAJOR	1
- #define ZYNQMP_PM_VERSION_MINOR	0
-@@ -30,9 +33,37 @@
- /* SMC SIP service Call Function Identifier Prefix */
- #define PM_SIP_SVC			0xC2000000
- 
-+/* SMC function ID to get SiP SVC version */
-+#define GET_SIP_SVC_VERSION	(0x8200ff03U)
-+
-+/* SiP Service Calls version numbers */
-+#define SIP_SVC_VERSION_MAJOR		(0U)
-+#define SIP_SVC_VERSION_MINOR		(2U)
-+
-+#define SIP_SVC_PASSTHROUGH_VERSION	((SIP_SVC_VERSION_MAJOR << 16) | \
-+					 SIP_SVC_VERSION_MINOR)
-+
-+/* Fixed ID for FW specific APIs */
-+#define PASS_THROUGH_FW_CMD_ID	GENMASK(11, 0)
-+
- /* PM API versions */
-+#define PM_API_VERSION_1	1
- #define PM_API_VERSION_2	2
- 
-+#define PM_PINCTRL_PARAM_SET_VERSION	2
-+
-+/* Family codes */
-+#define PM_ZYNQMP_FAMILY_CODE 0x1 /* ZynqMP family code */
-+#define PM_VERSAL_FAMILY_CODE 0x2 /* Versal family code */
-+#define PM_VERSAL_NET_FAMILY_CODE 0x3 /* Versal NET family code */
-+
-+#define API_ID_MASK		GENMASK(7, 0)
-+#define MODULE_ID_MASK		GENMASK(11, 8)
-+#define PLM_MODULE_ID_MASK	GENMASK(15, 8)
-+
-+/* Firmware feature check version mask */
-+#define FIRMWARE_VERSION_MASK		0xFFFFU
-+
- /* ATF only commands */
- #define TF_A_PM_REGISTER_SGI		0xa04
- #define PM_GET_TRUSTZONE_VERSION	0xa03
-@@ -40,7 +71,13 @@
- #define GET_CALLBACK_DATA		0xa01
- 
- /* Number of 32bits values in payload */
-#define PAYLOAD_ARG_CNT	4U
-+#define PAYLOAD_ARG_CNT	7U
-+
-+/* Number of 64bits arguments for SMC call */
-+#define SMC_ARG_CNT_64	8U
-+
-+/* Number of 32bits arguments for SMC call */
-+#define SMC_ARG_CNT_32	13U
- 
- /* Number of arguments for a callback */
- #define CB_ARG_CNT     4
-@@ -70,13 +107,48 @@
- #define XILINX_ZYNQMP_PM_FPGA_FULL	0x0U
- #define XILINX_ZYNQMP_PM_FPGA_PARTIAL	BIT(0)
- 
-+/* FPGA Status Reg */
-+#define XILINX_ZYNQMP_PM_FPGA_CONFIG_STAT_OFFSET	7U
-+#define XILINX_ZYNQMP_PM_FPGA_READ_CONFIG_REG		0U
-+
- /*
-  * Node IDs for the Error Events.
-  */
-#define EVENT_ERROR_PMC_ERR1	(0x28100000U)
-#define EVENT_ERROR_PMC_ERR2	(0x28104000U)
-#define EVENT_ERROR_PSM_ERR1	(0x28108000U)
-#define EVENT_ERROR_PSM_ERR2	(0x2810C000U)
-+#define VERSAL_EVENT_ERROR_PMC_ERR1	(0x28100000U)
-+#define VERSAL_EVENT_ERROR_PMC_ERR2	(0x28104000U)
-+#define VERSAL_EVENT_ERROR_PSM_ERR1	(0x28108000U)
-+#define VERSAL_EVENT_ERROR_PSM_ERR2	(0x2810C000U)
-+
-+#define VERSAL_NET_EVENT_ERROR_PMC_ERR1	(0x28100000U)
-+#define VERSAL_NET_EVENT_ERROR_PMC_ERR2	(0x28104000U)
-+#define VERSAL_NET_EVENT_ERROR_PMC_ERR3	(0x28108000U)
-+#define VERSAL_NET_EVENT_ERROR_PSM_ERR1	(0x2810C000U)
-+#define VERSAL_NET_EVENT_ERROR_PSM_ERR2	(0x28110000U)
-+#define VERSAL_NET_EVENT_ERROR_PSM_ERR3	(0x28114000U)
-+#define VERSAL_NET_EVENT_ERROR_PSM_ERR4	(0x28118000U)
-+
-+/* ZynqMP SD tap delay tuning */
-+#define SD_ITAPDLY	0xFF180314
-+#define SD_OTAPDLYSEL	0xFF180318
-+
-+/**
-+ * XPM_EVENT_ERROR_MASK_DDRMC_CR: Error event mask for DDRMC MC Correctable ECC Error.
-+ */
-+#define XPM_EVENT_ERROR_MASK_DDRMC_CR		BIT(18)
-+
-+/**
-+ * XPM_EVENT_ERROR_MASK_DDRMC_NCR: Error event mask for DDRMC MC Non-Correctable ECC Error.
-+ */
-+#define XPM_EVENT_ERROR_MASK_DDRMC_NCR		BIT(19)
-+#define XPM_EVENT_ERROR_MASK_NOC_NCR		BIT(13)
-+#define XPM_EVENT_ERROR_MASK_NOC_CR		BIT(12)
-+
-+enum pm_module_id {
-+	PM_MODULE_ID = 0x0,
-+	XPM_MODULE_ID = 0x2,
-+	XSEM_MODULE_ID = 0x3,
-+	TF_A_MODULE_ID = 0xa,
-+};
- 
- enum pm_api_cb_id {
- 	PM_INIT_SUSPEND_CB = 30,
-@@ -85,8 +157,12 @@ enum pm_api_cb_id {
- };
- 
- enum pm_api_id {
-+	PM_API_FEATURES = 0,
- 	PM_GET_API_VERSION = 1,
-+	PM_GET_NODE_STATUS = 3,
- 	PM_REGISTER_NOTIFIER = 5,
-+	PM_FORCE_POWERDOWN = 8,
-+	PM_REQUEST_WAKEUP = 10,
- 	PM_SYSTEM_SHUTDOWN = 12,
- 	PM_REQUEST_NODE = 13,
- 	PM_RELEASE_NODE = 14,
-@@ -102,7 +178,6 @@ enum pm_api_id {
- 	PM_SECURE_SHA = 26,
- 	PM_PINCTRL_REQUEST = 28,
- 	PM_PINCTRL_RELEASE = 29,
-	PM_PINCTRL_GET_FUNCTION = 30,
- 	PM_PINCTRL_SET_FUNCTION = 31,
- 	PM_PINCTRL_CONFIG_PARAM_GET = 32,
- 	PM_PINCTRL_CONFIG_PARAM_SET = 33,
-@@ -113,18 +188,20 @@ enum pm_api_id {
- 	PM_CLOCK_GETSTATE = 38,
- 	PM_CLOCK_SETDIVIDER = 39,
- 	PM_CLOCK_GETDIVIDER = 40,
-	PM_CLOCK_SETRATE = 41,
-	PM_CLOCK_GETRATE = 42,
- 	PM_CLOCK_SETPARENT = 43,
- 	PM_CLOCK_GETPARENT = 44,
-+	PM_FPGA_READ = 46,
- 	PM_SECURE_AES = 47,
-+	PM_EFUSE_ACCESS = 53,
- 	PM_FEATURE_CHECK = 63,
- };
- 
- /* PMU-FW return status codes */
- enum pm_ret_status {
- 	XST_PM_SUCCESS = 0,
-+	XST_PM_INVALID_VERSION = 4,
- 	XST_PM_NO_FEATURE = 19,
-+	XST_PM_INVALID_CRC = 301,
- 	XST_PM_INTERNAL = 2000,
- 	XST_PM_CONFLICT = 2001,
- 	XST_PM_NO_ACCESS = 2002,
-@@ -135,6 +212,11 @@ enum pm_ret_status {
- };
- 
- enum pm_ioctl_id {
-+	IOCTL_GET_RPU_OPER_MODE = 0,
-+	IOCTL_SET_RPU_OPER_MODE = 1,
-+	IOCTL_RPU_BOOT_ADDR_CONFIG = 2,
-+	IOCTL_TCM_COMB_CONFIG = 3,
-+	IOCTL_SET_TAPDELAY_BYPASS = 4,
- 	IOCTL_SD_DLL_RESET = 6,
- 	IOCTL_SET_SD_TAPDELAY = 7,
- 	IOCTL_SET_PLL_FRAC_MODE = 8,
-@@ -153,6 +235,14 @@ enum pm_ioctl_id {
- 	/* Runtime feature configuration */
- 	IOCTL_SET_FEATURE_CONFIG = 26,
- 	IOCTL_GET_FEATURE_CONFIG = 27,
-+	/* IOCTL for Secure Read/Write Interface */
-+	IOCTL_READ_REG = 28,
-+	IOCTL_MASK_WRITE_REG = 29,
-+	/* Dynamic SD/GEM configuration */
-+	IOCTL_SET_SD_CONFIG = 30,
-+	IOCTL_SET_GEM_CONFIG = 31,
-+	/* IOCTL to get default/current QoS */
-+	IOCTL_GET_QOS = 34,
- };
- 
- enum pm_query_id {
-@@ -170,6 +260,22 @@ enum pm_query_id {
- 	PM_QID_PINCTRL_GET_PIN_GROUPS = 11,
- 	PM_QID_CLOCK_GET_NUM_CLOCKS = 12,
- 	PM_QID_CLOCK_GET_MAX_DIVISOR = 13,
-+	PM_QID_PINCTRL_GET_ATTRIBUTES = 15,
-+};
-+
-+enum rpu_oper_mode {
-+	PM_RPU_MODE_LOCKSTEP = 0,
-+	PM_RPU_MODE_SPLIT = 1,
-+};
-+
-+enum rpu_boot_mem {
-+	PM_RPU_BOOTMEM_LOVEC = 0,
-+	PM_RPU_BOOTMEM_HIVEC = 1,
-+};
-+
-+enum rpu_tcm_comb {
-+	PM_RPU_TCM_SPLIT = 0,
-+	PM_RPU_TCM_COMB = 1,
- };
- 
- enum zynqmp_pm_reset_action {
-@@ -386,6 +492,18 @@ enum zynqmp_pm_shutdown_subtype {
- 	ZYNQMP_PM_SHUTDOWN_SUBTYPE_SYSTEM = 2,
- };
- 
-+enum tap_delay_signal_type {
-+	PM_TAPDELAY_NAND_DQS_IN = 0,
-+	PM_TAPDELAY_NAND_DQS_OUT = 1,
-+	PM_TAPDELAY_QSPI = 2,
-+	PM_TAPDELAY_MAX = 3,
-+};
-+
-+enum tap_delay_bypass_ctrl {
-+	PM_TAPDELAY_BYPASS_DISABLE = 0,
-+	PM_TAPDELAY_BYPASS_ENABLE = 1,
-+};
-+
- enum ospi_mux_select_type {
- 	PM_OSPI_MUX_SEL_DMA = 0,
- 	PM_OSPI_MUX_SEL_LINEAR = 1,
-@@ -400,6 +518,30 @@ enum pm_feature_config_id {
- };
- 
- /**
-+ * enum pm_sd_config_type - PM SD configuration.
-+ * @SD_CONFIG_EMMC_SEL: To set SD_EMMC_SEL in CTRL_REG_SD and SD_SLOTTYPE
-+ * @SD_CONFIG_BASECLK: To set SD_BASECLK in SD_CONFIG_REG1
-+ * @SD_CONFIG_8BIT: To set SD_8BIT in SD_CONFIG_REG2
-+ * @SD_CONFIG_FIXED: To set fixed config registers
-+ */
-+enum pm_sd_config_type {
-+	SD_CONFIG_EMMC_SEL = 1,
-+	SD_CONFIG_BASECLK = 2,
-+	SD_CONFIG_8BIT = 3,
-+	SD_CONFIG_FIXED = 4,
-+};
-+
-+/**
-+ * enum pm_gem_config_type - PM GEM configuration.
-+ * @GEM_CONFIG_SGMII_MODE: To set GEM_SGMII_MODE in GEM_CLK_CTRL register
-+ * @GEM_CONFIG_FIXED: To set fixed config registers
-+ */
-+enum pm_gem_config_type {
-+	GEM_CONFIG_SGMII_MODE = 1,
-+	GEM_CONFIG_FIXED = 2,
-+};
-+
-+/**
-  * struct zynqmp_pm_query_data - PM query data
-  * @qid:	query ID
-  * @arg1:	Argument 1 of query data
-@@ -413,20 +555,19 @@ struct zynqmp_pm_query_data {
- 	u32 arg3;
- };
- 
-int zynqmp_pm_invoke_fn(u32 pm_api_id, u32 arg0, u32 arg1,
-			u32 arg2, u32 arg3, u32 *ret_payload);
-+int zynqmp_pm_invoke_fn(u32 pm_api_id, u32 *ret_payload, u32 num_args, ...);
-+int zynqmp_pm_invoke_fw_fn(u32 pm_api_id, u32 *ret_payload, u32 num_args, ...);
- 
- #if IS_REACHABLE(CONFIG_ZYNQMP_FIRMWARE)
- int zynqmp_pm_get_api_version(u32 *version);
- int zynqmp_pm_get_chipid(u32 *idcode, u32 *version);
-+int zynqmp_pm_get_family_info(u32 *family);
- int zynqmp_pm_query_data(struct zynqmp_pm_query_data qdata, u32 *out);
- int zynqmp_pm_clock_enable(u32 clock_id);
- int zynqmp_pm_clock_disable(u32 clock_id);
- int zynqmp_pm_clock_getstate(u32 clock_id, u32 *state);
- int zynqmp_pm_clock_setdivider(u32 clock_id, u32 divider);
- int zynqmp_pm_clock_getdivider(u32 clock_id, u32 *divider);
-int zynqmp_pm_clock_setrate(u32 clock_id, u64 rate);
-int zynqmp_pm_clock_getrate(u32 clock_id, u64 *rate);
- int zynqmp_pm_clock_setparent(u32 clock_id, u32 parent_id);
- int zynqmp_pm_clock_getparent(u32 clock_id, u32 *parent_id);
- int zynqmp_pm_set_pll_frac_mode(u32 clk_id, u32 mode);
-@@ -436,12 +577,11 @@ int zynqmp_pm_get_pll_frac_data(u32 clk_id, u32 *data);
- int zynqmp_pm_set_sd_tapdelay(u32 node_id, u32 type, u32 value);
- int zynqmp_pm_sd_dll_reset(u32 node_id, u32 type);
- int zynqmp_pm_ospi_mux_select(u32 dev_id, u32 select);
-int zynqmp_pm_reset_assert(const enum zynqmp_pm_reset reset,
-+int zynqmp_pm_reset_assert(const u32 reset,
- 			   const enum zynqmp_pm_reset_action assert_flag);
-int zynqmp_pm_reset_get_status(const enum zynqmp_pm_reset reset, u32 *status);
-+int zynqmp_pm_reset_get_status(const u32 reset, u32 *status);
- unsigned int zynqmp_pm_bootmode_read(u32 *ps_mode);
- int zynqmp_pm_bootmode_write(u32 ps_mode);
-int zynqmp_pm_init_finalize(void);
- int zynqmp_pm_set_suspend_mode(u32 mode);
- int zynqmp_pm_request_node(const u32 node, const u32 capabilities,
- 			   const u32 qos, const enum zynqmp_pm_request_ack ack);
-@@ -450,18 +590,20 @@ int zynqmp_pm_set_requirement(const u32 node, const u32 capabilities,
- 			      const u32 qos,
- 			      const enum zynqmp_pm_request_ack ack);
- int zynqmp_pm_aes_engine(const u64 address, u32 *out);
-+int zynqmp_pm_efuse_access(const u64 address, u32 *out);
- int zynqmp_pm_sha_hash(const u64 address, const u32 size, const u32 flags);
- int zynqmp_pm_fpga_load(const u64 address, const u32 size, const u32 flags);
- int zynqmp_pm_fpga_get_status(u32 *value);
-+int zynqmp_pm_fpga_get_config_status(u32 *value);
- int zynqmp_pm_write_ggs(u32 index, u32 value);
- int zynqmp_pm_read_ggs(u32 index, u32 *value);
- int zynqmp_pm_write_pggs(u32 index, u32 value);
- int zynqmp_pm_read_pggs(u32 index, u32 *value);
-+int zynqmp_pm_set_tapdelay_bypass(u32 index, u32 value);
- int zynqmp_pm_system_shutdown(const u32 type, const u32 subtype);
- int zynqmp_pm_set_boot_health_status(u32 value);
- int zynqmp_pm_pinctrl_request(const u32 pin);
- int zynqmp_pm_pinctrl_release(const u32 pin);
-int zynqmp_pm_pinctrl_get_function(const u32 pin, u32 *id);
- int zynqmp_pm_pinctrl_set_function(const u32 pin, const u32 id);
- int zynqmp_pm_pinctrl_get_config(const u32 pin, const u32 param,
- 				 u32 *value);
-@@ -474,7 +616,24 @@ int zynqmp_pm_feature(const u32 api_id);
- int zynqmp_pm_is_function_supported(const u32 api_id, const u32 id);
- int zynqmp_pm_set_feature_config(enum pm_feature_config_id id, u32 value);
- int zynqmp_pm_get_feature_config(enum pm_feature_config_id id, u32 *payload);
-+int zynqmp_pm_sec_read_reg(u32 node_id, u32 offset, u32 *ret_value);
-+int zynqmp_pm_sec_mask_write_reg(const u32 node_id, const u32 offset,
-+				 u32 mask, u32 value);
- int zynqmp_pm_register_sgi(u32 sgi_num, u32 reset);
-+int zynqmp_pm_force_pwrdwn(const u32 target,
-+			   const enum zynqmp_pm_request_ack ack);
-+int zynqmp_pm_request_wake(const u32 node,
-+			   const bool set_addr,
-+			   const u64 address,
-+			   const enum zynqmp_pm_request_ack ack);
-+int zynqmp_pm_get_rpu_mode(u32 node_id, enum rpu_oper_mode *rpu_mode);
-+int zynqmp_pm_set_rpu_mode(u32 node_id, enum rpu_oper_mode rpu_mode);
-+int zynqmp_pm_set_tcm_config(u32 node_id, enum rpu_tcm_comb tcm_mode);
-+int zynqmp_pm_get_node_status(const u32 node, u32 *const status,
-+			      u32 *const requirements, u32 *const usage);
-+int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value);
-+int zynqmp_pm_set_gem_config(u32 node, enum pm_gem_config_type config,
-+			     u32 value);
- #else
- static inline int zynqmp_pm_get_api_version(u32 *version)
- {
-@@ -486,6 +645,11 @@ static inline int zynqmp_pm_get_chipid(u32 *idcode, u32 *version)
- 	return -ENODEV;
- }
- 
-+static inline int zynqmp_pm_get_family_info(u32 *family)
-+{
-+	return -ENODEV;
-+}
-+
- static inline int zynqmp_pm_query_data(struct zynqmp_pm_query_data qdata,
- 				       u32 *out)
- {
-@@ -517,16 +681,6 @@ static inline int zynqmp_pm_clock_getdivider(u32 clock_id, u32 *divider)
- 	return -ENODEV;
- }
- 
-static inline int zynqmp_pm_clock_setrate(u32 clock_id, u64 rate)
-{
-	return -ENODEV;
-}
-
-static inline int zynqmp_pm_clock_getrate(u32 clock_id, u64 *rate)
-{
-	return -ENODEV;
-}
-
- static inline int zynqmp_pm_clock_setparent(u32 clock_id, u32 parent_id)
- {
- 	return -ENODEV;
-@@ -572,14 +726,13 @@ static inline int zynqmp_pm_ospi_mux_select(u32 dev_id, u32 select)
- 	return -ENODEV;
- }
- 
-static inline int zynqmp_pm_reset_assert(const enum zynqmp_pm_reset reset,
-+static inline int zynqmp_pm_reset_assert(const u32 reset,
- 					 const enum zynqmp_pm_reset_action assert_flag)
- {
- 	return -ENODEV;
- }
- 
-static inline int zynqmp_pm_reset_get_status(const enum zynqmp_pm_reset reset,
-					     u32 *status)
-+static inline int zynqmp_pm_reset_get_status(const u32 reset, u32 *status)
- {
- 	return -ENODEV;
- }
-@@ -594,11 +747,6 @@ static inline int zynqmp_pm_bootmode_write(u32 ps_mode)
- 	return -ENODEV;
- }
- 
-static inline int zynqmp_pm_init_finalize(void)
-{
-	return -ENODEV;
-}
-
- static inline int zynqmp_pm_set_suspend_mode(u32 mode)
- {
- 	return -ENODEV;
-@@ -629,6 +777,11 @@ static inline int zynqmp_pm_aes_engine(const u64 address, u32 *out)
- 	return -ENODEV;
- }
- 
-+static inline int zynqmp_pm_efuse_access(const u64 address, u32 *out)
-+{
-+	return -ENODEV;
-+}
-+
- static inline int zynqmp_pm_sha_hash(const u64 address, const u32 size,
- 				     const u32 flags)
- {
-@@ -646,6 +799,11 @@ static inline int zynqmp_pm_fpga_get_status(u32 *value)
- 	return -ENODEV;
- }
- 
-+static inline int zynqmp_pm_fpga_get_config_status(u32 *value)
-+{
-+	return -ENODEV;
-+}
-+
- static inline int zynqmp_pm_write_ggs(u32 index, u32 value)
- {
- 	return -ENODEV;
-@@ -666,27 +824,27 @@ static inline int zynqmp_pm_read_pggs(u32 index, u32 *value)
- 	return -ENODEV;
- }
- 
-static inline int zynqmp_pm_system_shutdown(const u32 type, const u32 subtype)
-+static inline int zynqmp_pm_set_tapdelay_bypass(u32 index, u32 value)
- {
- 	return -ENODEV;
- }
- 
-static inline int zynqmp_pm_set_boot_health_status(u32 value)
-+static inline int zynqmp_pm_system_shutdown(const u32 type, const u32 subtype)
- {
- 	return -ENODEV;
- }
- 
-static inline int zynqmp_pm_pinctrl_request(const u32 pin)
-+static inline int zynqmp_pm_set_boot_health_status(u32 value)
- {
- 	return -ENODEV;
- }
- 
-static inline int zynqmp_pm_pinctrl_release(const u32 pin)
-+static inline int zynqmp_pm_pinctrl_request(const u32 pin)
- {
- 	return -ENODEV;
- }
- 
-static inline int zynqmp_pm_pinctrl_get_function(const u32 pin, u32 *id)
-+static inline int zynqmp_pm_pinctrl_release(const u32 pin)
- {
- 	return -ENODEV;
- }
-@@ -745,6 +903,68 @@ static inline int zynqmp_pm_register_sgi(u32 sgi_num, u32 reset)
- {
- 	return -ENODEV;
- }
-+
-+static inline int zynqmp_pm_force_pwrdwn(const u32 target,
-+					 const enum zynqmp_pm_request_ack ack)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int zynqmp_pm_request_wake(const u32 node,
-+					 const bool set_addr,
-+					 const u64 address,
-+					 const enum zynqmp_pm_request_ack ack)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int zynqmp_pm_sec_read_reg(u32 node_id, u32 offset, u32 *ret_value)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int zynqmp_pm_sec_mask_write_reg(const u32 node_id, const u32 offset,
-+					       u32 mask, u32 value)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int zynqmp_pm_get_rpu_mode(u32 node_id, enum rpu_oper_mode *rpu_mode)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int zynqmp_pm_set_rpu_mode(u32 node_id, enum rpu_oper_mode rpu_mode)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int zynqmp_pm_set_tcm_config(u32 node_id, enum rpu_tcm_comb tcm_mode)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int zynqmp_pm_get_node_status(const u32 node, u32 *const status,
-+					    u32 *const requirements,
-+					    u32 *const usage)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int zynqmp_pm_set_sd_config(u32 node,
-+					  enum pm_sd_config_type config,
-+					  u32 value)
-+{
-+	return -ENODEV;
-+}
-+
-+static inline int zynqmp_pm_set_gem_config(u32 node,
-+					   enum pm_gem_config_type config,
-+					   u32 value)
-+{
-+	return -ENODEV;
-+}
-+
- #endif
- 
- #endif /* __FIRMWARE_ZYNQMP_H__ */
-diff --git a/include/linux/fixp-arith.h b/include/linux/fixp-arith.h
-index 281cb4f83dbe..e485fb0c1201 100644
--- a/include/linux/fixp-arith.h
-+++ b/include/linux/fixp-arith.h
-@@ -2,6 +2,7 @@
- #ifndef _FIXP_ARITH_H
- #define _FIXP_ARITH_H
- 
-+#include <linux/bug.h>
- #include <linux/math64.h>
- 
- /*
-diff --git a/include/linux/flex_proportions.h b/include/linux/flex_proportions.h
-index 3e378b1fb0bc..e9a72fd0bfe7 100644
--- a/include/linux/flex_proportions.h
-+++ b/include/linux/flex_proportions.h
-@@ -39,38 +39,6 @@ void fprop_global_destroy(struct fprop_global *p);
- bool fprop_new_period(struct fprop_global *p, int periods);
- 
- /*
- *  ---- SINGLE ----
- */
-struct fprop_local_single {
-	/* the local events counter */
-	unsigned long events;
-	/* Period in which we last updated events */
-	unsigned int period;
-	raw_spinlock_t lock;	/* Protect period and numerator */
-};
-
-#define INIT_FPROP_LOCAL_SINGLE(name)			\
-{	.lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock),	\
-}
-
-int fprop_local_init_single(struct fprop_local_single *pl);
-void fprop_local_destroy_single(struct fprop_local_single *pl);
-void __fprop_inc_single(struct fprop_global *p, struct fprop_local_single *pl);
-void fprop_fraction_single(struct fprop_global *p,
-	struct fprop_local_single *pl, unsigned long *numerator,
-	unsigned long *denominator);
-
-static inline
-void fprop_inc_single(struct fprop_global *p, struct fprop_local_single *pl)
-{
-	unsigned long flags;
-
-	local_irq_save(flags);
-	__fprop_inc_single(p, pl);
-	local_irq_restore(flags);
-}
-
-/*
-  * ---- PERCPU ----
-  */
- struct fprop_local_percpu {
-diff --git a/include/linux/folio_queue.h b/include/linux/folio_queue.h
-new file mode 100644
-index 000000000000..adab609c972e
--- /dev/null
-+++ b/include/linux/folio_queue.h
-@@ -0,0 +1,282 @@
-+/* SPDX-License-Identifier: GPL-2.0-or-later */
-+/* Queue of folios definitions
-+ *
-+ * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
-+ * Written by David Howells (dhowells@redhat.com)
-+ *
-+ * See:
-+ *
-+ *	Documentation/core-api/folio_queue.rst
-+ *
-+ * for a description of the API.
-+ */
-+
-+#ifndef _LINUX_FOLIO_QUEUE_H
-+#define _LINUX_FOLIO_QUEUE_H
-+
-+#include <linux/pagevec.h>
-+#include <linux/mm.h>
-+
-+/*
-+ * Segment in a queue of running buffers.  Each segment can hold a number of
-+ * folios and a portion of the queue can be referenced with the ITER_FOLIOQ
-+ * iterator.  The possibility exists of inserting non-folio elements into the
-+ * queue (such as gaps).
-+ *
-+ * Explicit prev and next pointers are used instead of a list_head to make it
-+ * easier to add segments to tail and remove them from the head without the
-+ * need for a lock.
-+ */
-+struct folio_queue {
-+	struct folio_batch	vec;		/* Folios in the queue segment */
-+	u8			orders[PAGEVEC_SIZE]; /* Order of each folio */
-+	struct folio_queue	*next;		/* Next queue segment or NULL */
-+	struct folio_queue	*prev;		/* Previous queue segment of NULL */
-+	unsigned long		marks;		/* 1-bit mark per folio */
-+	unsigned long		marks2;		/* Second 1-bit mark per folio */
-+#if PAGEVEC_SIZE > BITS_PER_LONG
-+#error marks is not big enough
-+#endif
-+	unsigned int		rreq_id;
-+	unsigned int		debug_id;
-+};
-+
-+/**
-+ * folioq_init - Initialise a folio queue segment
-+ * @folioq: The segment to initialise
-+ * @rreq_id: The request identifier to use in tracelines.
-+ *
-+ * Initialise a folio queue segment and set an identifier to be used in traces.
-+ *
-+ * Note that the folio pointers are left uninitialised.
-+ */
-+static inline void folioq_init(struct folio_queue *folioq, unsigned int rreq_id)
-+{
-+	folio_batch_init(&folioq->vec);
-+	folioq->next = NULL;
-+	folioq->prev = NULL;
-+	folioq->marks = 0;
-+	folioq->marks2 = 0;
-+	folioq->rreq_id = rreq_id;
-+	folioq->debug_id = 0;
-+}
-+
-+/**
-+ * folioq_nr_slots: Query the capacity of a folio queue segment
-+ * @folioq: The segment to query
-+ *
-+ * Query the number of folios that a particular folio queue segment might hold.
-+ * [!] NOTE: This must not be assumed to be the same for every segment!
-+ */
-+static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq)
-+{
-+	return PAGEVEC_SIZE;
-+}
-+
-+/**
-+ * folioq_count: Query the occupancy of a folio queue segment
-+ * @folioq: The segment to query
-+ *
-+ * Query the number of folios that have been added to a folio queue segment.
-+ * Note that this is not decreased as folios are removed from a segment.
-+ */
-+static inline unsigned int folioq_count(struct folio_queue *folioq)
-+{
-+	return folio_batch_count(&folioq->vec);
-+}
-+
-+/**
-+ * folioq_full: Query if a folio queue segment is full
-+ * @folioq: The segment to query
-+ *
-+ * Query if a folio queue segment is fully occupied.  Note that this does not
-+ * change if folios are removed from a segment.
-+ */
-+static inline bool folioq_full(struct folio_queue *folioq)
-+{
-+	//return !folio_batch_space(&folioq->vec);
-+	return folioq_count(folioq) >= folioq_nr_slots(folioq);
-+}
-+
-+/**
-+ * folioq_is_marked: Check first folio mark in a folio queue segment
-+ * @folioq: The segment to query
-+ * @slot: The slot number of the folio to query
-+ *
-+ * Determine if the first mark is set for the folio in the specified slot in a
-+ * folio queue segment.
-+ */
-+static inline bool folioq_is_marked(const struct folio_queue *folioq, unsigned int slot)
-+{
-+	return test_bit(slot, &folioq->marks);
-+}
-+
-+/**
-+ * folioq_mark: Set the first mark on a folio in a folio queue segment
-+ * @folioq: The segment to modify
-+ * @slot: The slot number of the folio to modify
-+ *
-+ * Set the first mark for the folio in the specified slot in a folio queue
-+ * segment.
-+ */
-+static inline void folioq_mark(struct folio_queue *folioq, unsigned int slot)
-+{
-+	set_bit(slot, &folioq->marks);
-+}
-+
-+/**
-+ * folioq_unmark: Clear the first mark on a folio in a folio queue segment
-+ * @folioq: The segment to modify
-+ * @slot: The slot number of the folio to modify
-+ *
-+ * Clear the first mark for the folio in the specified slot in a folio queue
-+ * segment.
-+ */
-+static inline void folioq_unmark(struct folio_queue *folioq, unsigned int slot)
-+{
-+	clear_bit(slot, &folioq->marks);
-+}
-+
-+/**
-+ * folioq_is_marked2: Check second folio mark in a folio queue segment
-+ * @folioq: The segment to query
-+ * @slot: The slot number of the folio to query
-+ *
-+ * Determine if the second mark is set for the folio in the specified slot in a
-+ * folio queue segment.
-+ */
-+static inline bool folioq_is_marked2(const struct folio_queue *folioq, unsigned int slot)
-+{
-+	return test_bit(slot, &folioq->marks2);
-+}
-+
-+/**
-+ * folioq_mark2: Set the second mark on a folio in a folio queue segment
-+ * @folioq: The segment to modify
-+ * @slot: The slot number of the folio to modify
-+ *
-+ * Set the second mark for the folio in the specified slot in a folio queue
-+ * segment.
-+ */
-+static inline void folioq_mark2(struct folio_queue *folioq, unsigned int slot)
-+{
-+	set_bit(slot, &folioq->marks2);
-+}
-+
-+/**
-+ * folioq_unmark2: Clear the second mark on a folio in a folio queue segment
-+ * @folioq: The segment to modify
-+ * @slot: The slot number of the folio to modify
-+ *
-+ * Clear the second mark for the folio in the specified slot in a folio queue
-+ * segment.
-+ */
-+static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot)
-+{
-+	clear_bit(slot, &folioq->marks2);
-+}
-+
-+/**
-+ * folioq_append: Add a folio to a folio queue segment
-+ * @folioq: The segment to add to
-+ * @folio: The folio to add
-+ *
-+ * Add a folio to the tail of the sequence in a folio queue segment, increasing
-+ * the occupancy count and returning the slot number for the folio just added.
-+ * The folio size is extracted and stored in the queue and the marks are left
-+ * unmodified.
-+ *
-+ * Note that it's left up to the caller to check that the segment capacity will
-+ * not be exceeded and to extend the queue.
-+ */
-+static inline unsigned int folioq_append(struct folio_queue *folioq, struct folio *folio)
-+{
-+	unsigned int slot = folioq->vec.nr++;
-+
-+	folioq->vec.folios[slot] = folio;
-+	folioq->orders[slot] = folio_order(folio);
-+	return slot;
-+}
-+
-+/**
-+ * folioq_append_mark: Add a folio to a folio queue segment
-+ * @folioq: The segment to add to
-+ * @folio: The folio to add
-+ *
-+ * Add a folio to the tail of the sequence in a folio queue segment, increasing
-+ * the occupancy count and returning the slot number for the folio just added.
-+ * The folio size is extracted and stored in the queue, the first mark is set
-+ * and and the second and third marks are left unmodified.
-+ *
-+ * Note that it's left up to the caller to check that the segment capacity will
-+ * not be exceeded and to extend the queue.
-+ */
-+static inline unsigned int folioq_append_mark(struct folio_queue *folioq, struct folio *folio)
-+{
-+	unsigned int slot = folioq->vec.nr++;
-+
-+	folioq->vec.folios[slot] = folio;
-+	folioq->orders[slot] = folio_order(folio);
-+	folioq_mark(folioq, slot);
-+	return slot;
-+}
-+
-+/**
-+ * folioq_folio: Get a folio from a folio queue segment
-+ * @folioq: The segment to access
-+ * @slot: The folio slot to access
-+ *
-+ * Retrieve the folio in the specified slot from a folio queue segment.  Note
-+ * that no bounds check is made and if the slot hasn't been added into yet, the
-+ * pointer will be undefined.  If the slot has been cleared, NULL will be
-+ * returned.
-+ */
-+static inline struct folio *folioq_folio(const struct folio_queue *folioq, unsigned int slot)
-+{
-+	return folioq->vec.folios[slot];
-+}
-+
-+/**
-+ * folioq_folio_order: Get the order of a folio from a folio queue segment
-+ * @folioq: The segment to access
-+ * @slot: The folio slot to access
-+ *
-+ * Retrieve the order of the folio in the specified slot from a folio queue
-+ * segment.  Note that no bounds check is made and if the slot hasn't been
-+ * added into yet, the order returned will be 0.
-+ */
-+static inline unsigned int folioq_folio_order(const struct folio_queue *folioq, unsigned int slot)
-+{
-+	return folioq->orders[slot];
-+}
-+
-+/**
-+ * folioq_folio_size: Get the size of a folio from a folio queue segment
-+ * @folioq: The segment to access
-+ * @slot: The folio slot to access
-+ *
-+ * Retrieve the size of the folio in the specified slot from a folio queue
-+ * segment.  Note that no bounds check is made and if the slot hasn't been
-+ * added into yet, the size returned will be PAGE_SIZE.
-+ */
-+static inline size_t folioq_folio_size(const struct folio_queue *folioq, unsigned int slot)
-+{
-+	return PAGE_SIZE << folioq_folio_order(folioq, slot);
-+}
-+
-+/**
-+ * folioq_clear: Clear a folio from a folio queue segment
-+ * @folioq: The segment to clear
-+ * @slot: The folio slot to clear
-+ *
-+ * Clear a folio from a sequence in a folio queue segment and clear its marks.
-+ * The occupancy count is left unchanged.
-+ */
-+static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot)
-+{
-+	folioq->vec.folios[slot] = NULL;
-+	folioq_unmark(folioq, slot);
-+	folioq_unmark2(folioq, slot);
-+}
-+
-+#endif /* _LINUX_FOLIO_QUEUE_H */
-diff --git a/include/linux/font.h b/include/linux/font.h
-index abf1442ce719..fd8625cd76b2 100644
--- a/include/linux/font.h
-+++ b/include/linux/font.h
-@@ -35,6 +35,7 @@ struct font_desc {
- #define FONT6x10_IDX	10
- #define TER16x32_IDX	11
- #define FONT6x8_IDX	12
-+#define TER10x18_IDX	13
- 
- extern const struct font_desc	font_vga_8x8,
- 			font_vga_8x16,
-@@ -48,7 +49,8 @@ extern const struct font_desc	font_vga_8x8,
- 			font_mini_4x6,
- 			font_6x10,
- 			font_ter_16x32,
-			font_6x8;
-+			font_6x8,
-+			font_ter_10x18;
- 
- /* Find a font with a specific name */
- 
-@@ -57,7 +59,8 @@ extern const struct font_desc *find_font(const char *name);
- /* Get the default font for a specific screen size */
- 
- extern const struct font_desc *get_default_font(int xres, int yres,
-						u32 font_w, u32 font_h);
-+						unsigned long *font_w,
-+						unsigned long *font_h);
- 
- /* Max. length for the name of a predefined font */
- #define MAX_FONT_NAME	32
-diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h
-index 3b401fa0f374..b3b53f8c1b28 100644
--- a/include/linux/fortify-string.h
-+++ b/include/linux/fortify-string.h
-@@ -2,12 +2,58 @@
- #ifndef _LINUX_FORTIFY_STRING_H_
- #define _LINUX_FORTIFY_STRING_H_
- 
-+#include <linux/bitfield.h>
-+#include <linux/bug.h>
- #include <linux/const.h>
-+#include <linux/limits.h>
- 
- #define __FORTIFY_INLINE extern __always_inline __gnu_inline __overloadable
- #define __RENAME(x) __asm__(#x)
- 
-void fortify_panic(const char *name) __noreturn __cold;
-+#define FORTIFY_REASON_DIR(r)		FIELD_GET(BIT(0), r)
-+#define FORTIFY_REASON_FUNC(r)		FIELD_GET(GENMASK(7, 1), r)
-+#define FORTIFY_REASON(func, write)	(FIELD_PREP(BIT(0), write) | \
-+					 FIELD_PREP(GENMASK(7, 1), func))
-+
-+/* Overridden by KUnit tests. */
-+#ifndef fortify_panic
-+# define fortify_panic(func, write, avail, size, retfail)	\
-+	 __fortify_panic(FORTIFY_REASON(func, write), avail, size)
-+#endif
-+#ifndef fortify_warn_once
-+# define fortify_warn_once(x...)	WARN_ONCE(x)
-+#endif
-+
-+#define FORTIFY_READ		 0
-+#define FORTIFY_WRITE		 1
-+
-+#define EACH_FORTIFY_FUNC(macro)	\
-+	macro(strncpy),			\
-+	macro(strnlen),			\
-+	macro(strlen),			\
-+	macro(strscpy),			\
-+	macro(strlcat),			\
-+	macro(strcat),			\
-+	macro(strncat),			\
-+	macro(memset),			\
-+	macro(memcpy),			\
-+	macro(memmove),			\
-+	macro(memscan),			\
-+	macro(memcmp),			\
-+	macro(memchr),			\
-+	macro(memchr_inv),		\
-+	macro(kmemdup),			\
-+	macro(strcpy),			\
-+	macro(UNKNOWN),
-+
-+#define MAKE_FORTIFY_FUNC(func)	FORTIFY_FUNC_##func
-+
-+enum fortify_func {
-+	EACH_FORTIFY_FUNC(MAKE_FORTIFY_FUNC)
-+};
-+
-+void __fortify_report(const u8 reason, const size_t avail, const size_t size);
-+void __fortify_panic(const u8 reason, const size_t avail, const size_t size) __cold __noreturn;
- void __read_overflow(void) __compiletime_error("detected read beyond size of object (1st parameter)");
- void __read_overflow2(void) __compiletime_error("detected read beyond size of object (2nd parameter)");
- void __read_overflow2_field(size_t avail, size_t wanted) __compiletime_warning("detected read beyond size of field (2nd parameter); maybe use struct_group()?");
-@@ -16,10 +62,11 @@ void __write_overflow_field(size_t avail, size_t wanted) __compiletime_warning("
- 
- #define __compiletime_strlen(p)					\
- ({								\
-	unsigned char *__p = (unsigned char *)(p);		\
-	size_t __ret = (size_t)-1;				\
-	size_t __p_size = __builtin_object_size(p, 1);		\
-	if (__p_size != (size_t)-1) {				\
-+	char *__p = (char *)(p);				\
-+	size_t __ret = SIZE_MAX;				\
-+	const size_t __p_size = __member_size(p);		\
-+	if (__p_size != SIZE_MAX &&				\
-+	    __builtin_constant_p(*__p)) {			\
- 		size_t __p_len = __p_size - 1;			\
- 		if (__builtin_constant_p(__p[__p_len]) &&	\
- 		    __p[__p_len] == '\0')			\
-@@ -28,28 +75,55 @@ void __write_overflow_field(size_t avail, size_t wanted) __compiletime_warning("
- 	__ret;							\
- })
- 
-#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
-+#if defined(__SANITIZE_ADDRESS__)
-+
-+#if !defined(CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX) && !defined(CONFIG_GENERIC_ENTRY)
-+extern void *__underlying_memset(void *p, int c, __kernel_size_t size) __RENAME(memset);
-+extern void *__underlying_memmove(void *p, const void *q, __kernel_size_t size) __RENAME(memmove);
-+extern void *__underlying_memcpy(void *p, const void *q, __kernel_size_t size) __RENAME(memcpy);
-+#elif defined(CONFIG_KASAN_GENERIC)
-+extern void *__underlying_memset(void *p, int c, __kernel_size_t size) __RENAME(__asan_memset);
-+extern void *__underlying_memmove(void *p, const void *q, __kernel_size_t size) __RENAME(__asan_memmove);
-+extern void *__underlying_memcpy(void *p, const void *q, __kernel_size_t size) __RENAME(__asan_memcpy);
-+#else /* CONFIG_KASAN_SW_TAGS */
-+extern void *__underlying_memset(void *p, int c, __kernel_size_t size) __RENAME(__hwasan_memset);
-+extern void *__underlying_memmove(void *p, const void *q, __kernel_size_t size) __RENAME(__hwasan_memmove);
-+extern void *__underlying_memcpy(void *p, const void *q, __kernel_size_t size) __RENAME(__hwasan_memcpy);
-+#endif
-+
- extern void *__underlying_memchr(const void *p, int c, __kernel_size_t size) __RENAME(memchr);
- extern int __underlying_memcmp(const void *p, const void *q, __kernel_size_t size) __RENAME(memcmp);
-extern void *__underlying_memcpy(void *p, const void *q, __kernel_size_t size) __RENAME(memcpy);
-extern void *__underlying_memmove(void *p, const void *q, __kernel_size_t size) __RENAME(memmove);
-extern void *__underlying_memset(void *p, int c, __kernel_size_t size) __RENAME(memset);
- extern char *__underlying_strcat(char *p, const char *q) __RENAME(strcat);
- extern char *__underlying_strcpy(char *p, const char *q) __RENAME(strcpy);
- extern __kernel_size_t __underlying_strlen(const char *p) __RENAME(strlen);
- extern char *__underlying_strncat(char *p, const char *q, __kernel_size_t count) __RENAME(strncat);
- extern char *__underlying_strncpy(char *p, const char *q, __kernel_size_t size) __RENAME(strncpy);
-+
-+#else
-+
-+#if defined(__SANITIZE_MEMORY__)
-+/*
-+ * For KMSAN builds all memcpy/memset/memmove calls should be replaced by the
-+ * corresponding __msan_XXX functions.
-+ */
-+#include <linux/kmsan_string.h>
-+#define __underlying_memcpy	__msan_memcpy
-+#define __underlying_memmove	__msan_memmove
-+#define __underlying_memset	__msan_memset
- #else
-#define __underlying_memchr	__builtin_memchr
-#define __underlying_memcmp	__builtin_memcmp
- #define __underlying_memcpy	__builtin_memcpy
- #define __underlying_memmove	__builtin_memmove
- #define __underlying_memset	__builtin_memset
-+#endif
-+
-+#define __underlying_memchr	__builtin_memchr
-+#define __underlying_memcmp	__builtin_memcmp
- #define __underlying_strcat	__builtin_strcat
- #define __underlying_strcpy	__builtin_strcpy
- #define __underlying_strlen	__builtin_strlen
- #define __underlying_strncat	__builtin_strncat
- #define __underlying_strncpy	__builtin_strncpy
-+
- #endif
- 
- /**
-@@ -69,47 +143,87 @@ extern char *__underlying_strncpy(char *p, const char *q, __kernel_size_t size)
- 	__underlying_memcpy(dst, src, bytes)
- 
- /*
- * Clang's use of __builtin_object_size() within inlines needs hinting via
- * __pass_object_size(). The preference is to only ever use type 1 (member
-+ * Clang's use of __builtin_*object_size() within inlines needs hinting via
-+ * __pass_*object_size(). The preference is to only ever use type 1 (member
-  * size, rather than struct size), but there remain some stragglers using
-  * type 0 that will be converted in the future.
-  */
-#define POS	__pass_object_size(1)
-#define POS0	__pass_object_size(0)
-+#if __has_builtin(__builtin_dynamic_object_size)
-+#define POS			__pass_dynamic_object_size(1)
-+#define POS0			__pass_dynamic_object_size(0)
-+#else
-+#define POS			__pass_object_size(1)
-+#define POS0			__pass_object_size(0)
-+#endif
- 
-+#define __compiletime_lessthan(bounds, length)	(	\
-+	__builtin_constant_p((bounds) < (length)) &&	\
-+	(bounds) < (length)				\
-+)
-+
-+/**
-+ * strncpy - Copy a string to memory with non-guaranteed NUL padding
-+ *
-+ * @p: pointer to destination of copy
-+ * @q: pointer to NUL-terminated source string to copy
-+ * @size: bytes to write at @p
-+ *
-+ * If strlen(@q) >= @size, the copy of @q will stop after @size bytes,
-+ * and @p will NOT be NUL-terminated
-+ *
-+ * If strlen(@q) < @size, following the copy of @q, trailing NUL bytes
-+ * will be written to @p until @size total bytes have been written.
-+ *
-+ * Do not use this function. While FORTIFY_SOURCE tries to avoid
-+ * over-reads of @q, it cannot defend against writing unterminated
-+ * results to @p. Using strncpy() remains ambiguous and fragile.
-+ * Instead, please choose an alternative, so that the expectation
-+ * of @p's contents is unambiguous:
-+ *
-+ * +--------------------+--------------------+------------+
-+ * | **p** needs to be: | padded to **size** | not padded |
-+ * +====================+====================+============+
-+ * |     NUL-terminated | strscpy_pad()      | strscpy()  |
-+ * +--------------------+--------------------+------------+
-+ * | not NUL-terminated | strtomem_pad()     | strtomem() |
-+ * +--------------------+--------------------+------------+
-+ *
-+ * Note strscpy*()'s differing return values for detecting truncation,
-+ * and strtomem*()'s expectation that the destination is marked with
-+ * __nonstring when it is a character array.
-+ *
-+ */
- __FORTIFY_INLINE __diagnose_as(__builtin_strncpy, 1, 2, 3)
- char *strncpy(char * const POS p, const char *q, __kernel_size_t size)
- {
-	size_t p_size = __builtin_object_size(p, 1);
-+	const size_t p_size = __member_size(p);
- 
-	if (__builtin_constant_p(size) && p_size < size)
-+	if (__compiletime_lessthan(p_size, size))
- 		__write_overflow();
- 	if (p_size < size)
-		fortify_panic(__func__);
-+		fortify_panic(FORTIFY_FUNC_strncpy, FORTIFY_WRITE, p_size, size, p);
- 	return __underlying_strncpy(p, q, size);
- }
- 
-__FORTIFY_INLINE __diagnose_as(__builtin_strcat, 1, 2)
-char *strcat(char * const POS p, const char *q)
-{
-	size_t p_size = __builtin_object_size(p, 1);
-
-	if (p_size == (size_t)-1)
-		return __underlying_strcat(p, q);
-	if (strlcat(p, q, p_size) >= p_size)
-		fortify_panic(__func__);
-	return p;
-}
-
- extern __kernel_size_t __real_strnlen(const char *, __kernel_size_t) __RENAME(strnlen);
-+/**
-+ * strnlen - Return bounded count of characters in a NUL-terminated string
-+ *
-+ * @p: pointer to NUL-terminated string to count.
-+ * @maxlen: maximum number of characters to count.
-+ *
-+ * Returns number of characters in @p (NOT including the final NUL), or
-+ * @maxlen, if no NUL has been found up to there.
-+ *
-+ */
- __FORTIFY_INLINE __kernel_size_t strnlen(const char * const POS p, __kernel_size_t maxlen)
- {
-	size_t p_size = __builtin_object_size(p, 1);
-	size_t p_len = __compiletime_strlen(p);
-+	const size_t p_size = __member_size(p);
-+	const size_t p_len = __compiletime_strlen(p);
- 	size_t ret;
- 
- 	/* We can take compile-time actions when maxlen is const. */
-	if (__builtin_constant_p(maxlen) && p_len != (size_t)-1) {
-+	if (__builtin_constant_p(maxlen) && p_len != SIZE_MAX) {
- 		/* If p is const, we can use its compile-time-known len. */
- 		if (maxlen >= p_size)
- 			return p_len;
-@@ -118,7 +232,7 @@ __FORTIFY_INLINE __kernel_size_t strnlen(const char * const POS p, __kernel_size
- 	/* Do not check characters beyond the end of p. */
- 	ret = __real_strnlen(p, maxlen < p_size ? maxlen : p_size);
- 	if (p_size <= ret && maxlen != ret)
-		fortify_panic(__func__);
-+		fortify_panic(FORTIFY_FUNC_strnlen, FORTIFY_READ, p_size, ret + 1, ret);
- 	return ret;
- }
- 
-@@ -127,71 +241,67 @@ __FORTIFY_INLINE __kernel_size_t strnlen(const char * const POS p, __kernel_size
-  * possible for strlen() to be used on compile-time strings for use in
-  * static initializers (i.e. as a constant expression).
-  */
-+/**
-+ * strlen - Return count of characters in a NUL-terminated string
-+ *
-+ * @p: pointer to NUL-terminated string to count.
-+ *
-+ * Do not use this function unless the string length is known at
-+ * compile-time. When @p is unterminated, this function may crash
-+ * or return unexpected counts that could lead to memory content
-+ * exposures. Prefer strnlen().
-+ *
-+ * Returns number of characters in @p (NOT including the final NUL).
-+ *
-+ */
- #define strlen(p)							\
- 	__builtin_choose_expr(__is_constexpr(__builtin_strlen(p)),	\
- 		__builtin_strlen(p), __fortify_strlen(p))
- __FORTIFY_INLINE __diagnose_as(__builtin_strlen, 1)
- __kernel_size_t __fortify_strlen(const char * const POS p)
- {
-+	const size_t p_size = __member_size(p);
- 	__kernel_size_t ret;
-	size_t p_size = __builtin_object_size(p, 1);
- 
- 	/* Give up if we don't know how large p is. */
-	if (p_size == (size_t)-1)
-+	if (p_size == SIZE_MAX)
- 		return __underlying_strlen(p);
- 	ret = strnlen(p, p_size);
- 	if (p_size <= ret)
-		fortify_panic(__func__);
-+		fortify_panic(FORTIFY_FUNC_strlen, FORTIFY_READ, p_size, ret + 1, ret);
- 	return ret;
- }
- 
-/* defined after fortified strlen to reuse it */
-extern size_t __real_strlcpy(char *, const char *, size_t) __RENAME(strlcpy);
-__FORTIFY_INLINE size_t strlcpy(char * const POS p, const char * const POS q, size_t size)
-{
-	size_t p_size = __builtin_object_size(p, 1);
-	size_t q_size = __builtin_object_size(q, 1);
-	size_t q_len;	/* Full count of source string length. */
-	size_t len;	/* Count of characters going into destination. */
-
-	if (p_size == (size_t)-1 && q_size == (size_t)-1)
-		return __real_strlcpy(p, q, size);
-	q_len = strlen(q);
-	len = (q_len >= size) ? size - 1 : q_len;
-	if (__builtin_constant_p(size) && __builtin_constant_p(q_len) && size) {
-		/* Write size is always larger than destination. */
-		if (len >= p_size)
-			__write_overflow();
-	}
-	if (size) {
-		if (len >= p_size)
-			fortify_panic(__func__);
-		__underlying_memcpy(p, q, len);
-		p[len] = '\0';
-	}
-	return q_len;
-}
-
-/* defined after fortified strnlen to reuse it */
-extern ssize_t __real_strscpy(char *, const char *, size_t) __RENAME(strscpy);
-__FORTIFY_INLINE ssize_t strscpy(char * const POS p, const char * const POS q, size_t size)
-+/* Defined after fortified strnlen() to reuse it. */
-+extern ssize_t __real_strscpy(char *, const char *, size_t) __RENAME(sized_strscpy);
-+__FORTIFY_INLINE ssize_t sized_strscpy(char * const POS p, const char * const POS q, size_t size)
- {
-	size_t len;
- 	/* Use string size rather than possible enclosing struct size. */
-	size_t p_size = __builtin_object_size(p, 1);
-	size_t q_size = __builtin_object_size(q, 1);
-+	const size_t p_size = __member_size(p);
-+	const size_t q_size = __member_size(q);
-+	size_t len;
- 
- 	/* If we cannot get size of p and q default to call strscpy. */
-	if (p_size == (size_t) -1 && q_size == (size_t) -1)
-+	if (p_size == SIZE_MAX && q_size == SIZE_MAX)
- 		return __real_strscpy(p, q, size);
- 
- 	/*
- 	 * If size can be known at compile time and is greater than
- 	 * p_size, generate a compile time write overflow error.
- 	 */
-	if (__builtin_constant_p(size) && size > p_size)
-+	if (__compiletime_lessthan(p_size, size))
- 		__write_overflow();
- 
-+	/* Short-circuit for compile-time known-safe lengths. */
-+	if (__compiletime_lessthan(p_size, SIZE_MAX)) {
-+		len = __compiletime_strlen(q);
-+
-+		if (len < SIZE_MAX && __compiletime_lessthan(len, size)) {
-+			__underlying_memcpy(p, q, len + 1);
-+			return len;
-+		}
-+	}
-+
- 	/*
- 	 * This call protects from read overflow, because len will default to q
- 	 * length if it smaller than size.
-@@ -208,8 +318,8 @@ __FORTIFY_INLINE ssize_t strscpy(char * const POS p, const char * const POS q, s
- 	 * Generate a runtime write overflow error if len is greater than
- 	 * p_size.
- 	 */
-	if (len > p_size)
-		fortify_panic(__func__);
-+	if (p_size < len)
-+		fortify_panic(FORTIFY_FUNC_strscpy, FORTIFY_WRITE, p_size, len, -E2BIG);
- 
- 	/*
- 	 * We can now safely call vanilla strscpy because we are protected from:
-@@ -219,26 +329,137 @@ __FORTIFY_INLINE ssize_t strscpy(char * const POS p, const char * const POS q, s
- 	return __real_strscpy(p, q, len);
- }
- 
-/* defined after fortified strlen and strnlen to reuse them */
-+/* Defined after fortified strlen() to reuse it. */
-+extern size_t __real_strlcat(char *p, const char *q, size_t avail) __RENAME(strlcat);
-+/**
-+ * strlcat - Append a string to an existing string
-+ *
-+ * @p: pointer to %NUL-terminated string to append to
-+ * @q: pointer to %NUL-terminated string to append from
-+ * @avail: Maximum bytes available in @p
-+ *
-+ * Appends %NUL-terminated string @q after the %NUL-terminated
-+ * string at @p, but will not write beyond @avail bytes total,
-+ * potentially truncating the copy from @q. @p will stay
-+ * %NUL-terminated only if a %NUL already existed within
-+ * the @avail bytes of @p. If so, the resulting number of
-+ * bytes copied from @q will be at most "@avail - strlen(@p) - 1".
-+ *
-+ * Do not use this function. While FORTIFY_SOURCE tries to avoid
-+ * read and write overflows, this is only possible when the sizes
-+ * of @p and @q are known to the compiler. Prefer building the
-+ * string with formatting, via scnprintf(), seq_buf, or similar.
-+ *
-+ * Returns total bytes that _would_ have been contained by @p
-+ * regardless of truncation, similar to snprintf(). If return
-+ * value is >= @avail, the string has been truncated.
-+ *
-+ */
-+__FORTIFY_INLINE
-+size_t strlcat(char * const POS p, const char * const POS q, size_t avail)
-+{
-+	const size_t p_size = __member_size(p);
-+	const size_t q_size = __member_size(q);
-+	size_t p_len, copy_len;
-+	size_t actual, wanted;
-+
-+	/* Give up immediately if both buffer sizes are unknown. */
-+	if (p_size == SIZE_MAX && q_size == SIZE_MAX)
-+		return __real_strlcat(p, q, avail);
-+
-+	p_len = strnlen(p, avail);
-+	copy_len = strlen(q);
-+	wanted = actual = p_len + copy_len;
-+
-+	/* Cannot append any more: report truncation. */
-+	if (avail <= p_len)
-+		return wanted;
-+
-+	/* Give up if string is already overflowed. */
-+	if (p_size <= p_len)
-+		fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_READ, p_size, p_len + 1, wanted);
-+
-+	if (actual >= avail) {
-+		copy_len = avail - p_len - 1;
-+		actual = p_len + copy_len;
-+	}
-+
-+	/* Give up if copy will overflow. */
-+	if (p_size <= actual)
-+		fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_WRITE, p_size, actual + 1, wanted);
-+	__underlying_memcpy(p + p_len, q, copy_len);
-+	p[actual] = '\0';
-+
-+	return wanted;
-+}
-+
-+/* Defined after fortified strlcat() to reuse it. */
-+/**
-+ * strcat - Append a string to an existing string
-+ *
-+ * @p: pointer to NUL-terminated string to append to
-+ * @q: pointer to NUL-terminated source string to append from
-+ *
-+ * Do not use this function. While FORTIFY_SOURCE tries to avoid
-+ * read and write overflows, this is only possible when the
-+ * destination buffer size is known to the compiler. Prefer
-+ * building the string with formatting, via scnprintf() or similar.
-+ * At the very least, use strncat().
-+ *
-+ * Returns @p.
-+ *
-+ */
-+__FORTIFY_INLINE __diagnose_as(__builtin_strcat, 1, 2)
-+char *strcat(char * const POS p, const char *q)
-+{
-+	const size_t p_size = __member_size(p);
-+	const size_t wanted = strlcat(p, q, p_size);
-+
-+	if (p_size <= wanted)
-+		fortify_panic(FORTIFY_FUNC_strcat, FORTIFY_WRITE, p_size, wanted + 1, p);
-+	return p;
-+}
-+
-+/**
-+ * strncat - Append a string to an existing string
-+ *
-+ * @p: pointer to NUL-terminated string to append to
-+ * @q: pointer to source string to append from
-+ * @count: Maximum bytes to read from @q
-+ *
-+ * Appends at most @count bytes from @q (stopping at the first
-+ * NUL byte) after the NUL-terminated string at @p. @p will be
-+ * NUL-terminated.
-+ *
-+ * Do not use this function. While FORTIFY_SOURCE tries to avoid
-+ * read and write overflows, this is only possible when the sizes
-+ * of @p and @q are known to the compiler. Prefer building the
-+ * string with formatting, via scnprintf() or similar.
-+ *
-+ * Returns @p.
-+ *
-+ */
-+/* Defined after fortified strlen() and strnlen() to reuse them. */
- __FORTIFY_INLINE __diagnose_as(__builtin_strncat, 1, 2, 3)
- char *strncat(char * const POS p, const char * const POS q, __kernel_size_t count)
- {
-	size_t p_len, copy_len;
-	size_t p_size = __builtin_object_size(p, 1);
-	size_t q_size = __builtin_object_size(q, 1);
-+	const size_t p_size = __member_size(p);
-+	const size_t q_size = __member_size(q);
-+	size_t p_len, copy_len, total;
- 
-	if (p_size == (size_t)-1 && q_size == (size_t)-1)
-+	if (p_size == SIZE_MAX && q_size == SIZE_MAX)
- 		return __underlying_strncat(p, q, count);
- 	p_len = strlen(p);
- 	copy_len = strnlen(q, count);
-	if (p_size < p_len + copy_len + 1)
-		fortify_panic(__func__);
-+	total = p_len + copy_len + 1;
-+	if (p_size < total)
-+		fortify_panic(FORTIFY_FUNC_strncat, FORTIFY_WRITE, p_size, total, p);
- 	__underlying_memcpy(p + p_len, q, copy_len);
- 	p[p_len + copy_len] = '\0';
- 	return p;
- }
- 
-__FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size,
-+__FORTIFY_INLINE bool fortify_memset_chk(__kernel_size_t size,
- 					 const size_t p_size,
- 					 const size_t p_size_field)
- {
-@@ -246,15 +467,16 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size,
- 		/*
- 		 * Length argument is a constant expression, so we
- 		 * can perform compile-time bounds checking where
-		 * buffer sizes are known.
-+		 * buffer sizes are also known at compile time.
- 		 */
- 
- 		/* Error when size is larger than enclosing struct. */
-		if (p_size > p_size_field && p_size < size)
-+		if (__compiletime_lessthan(p_size_field, p_size) &&
-+		    __compiletime_lessthan(p_size, size))
- 			__write_overflow();
- 
- 		/* Warn when write size is larger than dest field. */
-		if (p_size_field < size)
-+		if (__compiletime_lessthan(p_size_field, size))
- 			__write_overflow_field(p_size_field, size);
- 	}
- 	/*
-@@ -268,11 +490,12 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size,
- 	/*
- 	 * Always stop accesses beyond the struct that contains the
- 	 * field, when the buffer's remaining size is known.
-	 * (The -1 test is to optimize away checks where the buffer
-+	 * (The SIZE_MAX test is to optimize away checks where the buffer
- 	 * lengths are unknown.)
- 	 */
-	if (p_size != (size_t)(-1) && p_size < size)
-		fortify_panic("memset");
-+	if (p_size != SIZE_MAX && p_size < size)
-+		fortify_panic(FORTIFY_FUNC_memset, FORTIFY_WRITE, p_size, size, true);
-+	return false;
- }
- 
- #define __fortify_memset_chk(p, c, size, p_size, p_size_field) ({	\
-@@ -282,11 +505,13 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size,
- })
- 
- /*
- * __builtin_object_size() must be captured here to avoid evaluating argument
- * side-effects further into the macro layers.
-+ * __struct_size() vs __member_size() must be captured here to avoid
-+ * evaluating argument side-effects further into the macro layers.
-  */
-+#ifndef CONFIG_KMSAN
- #define memset(p, c, s) __fortify_memset_chk(p, c, s,			\
-		__builtin_object_size(p, 0), __builtin_object_size(p, 1))
-+		__struct_size(p), __member_size(p))
-+#endif
- 
- /*
-  * To make sure the compiler can enforce protection against buffer overflows,
-@@ -319,36 +544,39 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size,
-  * V = vulnerable to run-time overflow (will need refactoring to solve)
-  *
-  */
-__FORTIFY_INLINE void fortify_memcpy_chk(__kernel_size_t size,
-+__FORTIFY_INLINE bool fortify_memcpy_chk(__kernel_size_t size,
- 					 const size_t p_size,
- 					 const size_t q_size,
- 					 const size_t p_size_field,
- 					 const size_t q_size_field,
-					 const char *func)
-+					 const u8 func)
- {
- 	if (__builtin_constant_p(size)) {
- 		/*
- 		 * Length argument is a constant expression, so we
- 		 * can perform compile-time bounds checking where
-		 * buffer sizes are known.
-+		 * buffer sizes are also known at compile time.
- 		 */
- 
- 		/* Error when size is larger than enclosing struct. */
-		if (p_size > p_size_field && p_size < size)
-+		if (__compiletime_lessthan(p_size_field, p_size) &&
-+		    __compiletime_lessthan(p_size, size))
- 			__write_overflow();
-		if (q_size > q_size_field && q_size < size)
-+		if (__compiletime_lessthan(q_size_field, q_size) &&
-+		    __compiletime_lessthan(q_size, size))
- 			__read_overflow2();
- 
- 		/* Warn when write size argument larger than dest field. */
-		if (p_size_field < size)
-+		if (__compiletime_lessthan(p_size_field, size))
- 			__write_overflow_field(p_size_field, size);
- 		/*
- 		 * Warn for source field over-read when building with W=1
- 		 * or when an over-write happened, so both can be fixed at
- 		 * the same time.
- 		 */
-		if ((IS_ENABLED(KBUILD_EXTRA_WARN1) || p_size_field < size) &&
-		    q_size_field < size)
-+		if ((IS_ENABLED(KBUILD_EXTRA_WARN1) ||
-+		     __compiletime_lessthan(p_size_field, size)) &&
-+		    __compiletime_lessthan(q_size_field, size))
- 			__read_overflow2_field(q_size_field, size);
- 	}
- 	/*
-@@ -362,118 +590,216 @@ __FORTIFY_INLINE void fortify_memcpy_chk(__kernel_size_t size,
- 	/*
- 	 * Always stop accesses beyond the struct that contains the
- 	 * field, when the buffer's remaining size is known.
-	 * (The -1 test is to optimize away checks where the buffer
-+	 * (The SIZE_MAX test is to optimize away checks where the buffer
- 	 * lengths are unknown.)
- 	 */
-	if ((p_size != (size_t)(-1) && p_size < size) ||
-	    (q_size != (size_t)(-1) && q_size < size))
-		fortify_panic(func);
-+	if (p_size != SIZE_MAX && p_size < size)
-+		fortify_panic(func, FORTIFY_WRITE, p_size, size, true);
-+	else if (q_size != SIZE_MAX && q_size < size)
-+		fortify_panic(func, FORTIFY_READ, q_size, size, true);
-+
-+	/*
-+	 * Warn when writing beyond destination field size.
-+	 *
-+	 * Note the implementation of __builtin_*object_size() behaves
-+	 * like sizeof() when not directly referencing a flexible
-+	 * array member, which means there will be many bounds checks
-+	 * that will appear at run-time, without a way for them to be
-+	 * detected at compile-time (as can be done when the destination
-+	 * is specifically the flexible array member).
-+	 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101832
-+	 */
-+	if (p_size_field != SIZE_MAX &&
-+	    p_size != p_size_field && p_size_field < size)
-+		return true;
-+
-+	return false;
- }
- 
-+/*
-+ * To work around what seems to be an optimizer bug, the macro arguments
-+ * need to have const copies or the values end up changed by the time they
-+ * reach fortify_warn_once(). See commit 6f7630b1b5bc ("fortify: Capture
-+ * __bos() results in const temp vars") for more details.
-+ */
- #define __fortify_memcpy_chk(p, q, size, p_size, q_size,		\
- 			     p_size_field, q_size_field, op) ({		\
-	size_t __fortify_size = (size_t)(size);				\
-	fortify_memcpy_chk(__fortify_size, p_size, q_size,		\
-			   p_size_field, q_size_field, #op);		\
-	__underlying_##op(p, q, __fortify_size);			\
-+	const size_t __fortify_size = (size_t)(size);			\
-+	const size_t __p_size = (p_size);				\
-+	const size_t __q_size = (q_size);				\
-+	const size_t __p_size_field = (p_size_field);			\
-+	const size_t __q_size_field = (q_size_field);			\
-+	/* Keep a mutable version of the size for the final copy. */	\
-+	size_t __copy_size = __fortify_size;				\
-+	fortify_warn_once(fortify_memcpy_chk(__fortify_size, __p_size,	\
-+				     __q_size, __p_size_field,		\
-+				     __q_size_field, FORTIFY_FUNC_ ##op), \
-+		  #op ": detected field-spanning write (size %zu) of single %s (size %zu)\n", \
-+		  __fortify_size,					\
-+		  "field \"" #p "\" at " FILE_LINE,			\
-+		  __p_size_field);					\
-+	/* Hide only the run-time size from value range tracking to */	\
-+	/* silence compile-time false positive bounds warnings. */	\
-+	if (!__builtin_constant_p(__copy_size))				\
-+		OPTIMIZER_HIDE_VAR(__copy_size);			\
-+	__underlying_##op(p, q, __copy_size);				\
- })
- 
- /*
- * __builtin_object_size() must be captured here to avoid evaluating argument
- * side-effects further into the macro layers.
-+ * Notes about compile-time buffer size detection:
-+ *
-+ * With these types...
-+ *
-+ *	struct middle {
-+ *		u16 a;
-+ *		u8 middle_buf[16];
-+ *		int b;
-+ *	};
-+ *	struct end {
-+ *		u16 a;
-+ *		u8 end_buf[16];
-+ *	};
-+ *	struct flex {
-+ *		int a;
-+ *		u8 flex_buf[];
-+ *	};
-+ *
-+ *	void func(TYPE *ptr) { ... }
-+ *
-+ * Cases where destination size cannot be currently detected:
-+ * - the size of ptr's object (seemingly by design, gcc & clang fail):
-+ *	__builtin_object_size(ptr, 1) == SIZE_MAX
-+ * - the size of flexible arrays in ptr's obj (by design, dynamic size):
-+ *	__builtin_object_size(ptr->flex_buf, 1) == SIZE_MAX
-+ * - the size of ANY array at the end of ptr's obj (gcc and clang bug):
-+ *	__builtin_object_size(ptr->end_buf, 1) == SIZE_MAX
-+ *	https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101836
-+ *
-+ * Cases where destination size is currently detected:
-+ * - the size of non-array members within ptr's object:
-+ *	__builtin_object_size(ptr->a, 1) == 2
-+ * - the size of non-flexible-array in the middle of ptr's obj:
-+ *	__builtin_object_size(ptr->middle_buf, 1) == 16
-+ *
-+ */
-+
-+/*
-+ * __struct_size() vs __member_size() must be captured here to avoid
-+ * evaluating argument side-effects further into the macro layers.
-  */
- #define memcpy(p, q, s)  __fortify_memcpy_chk(p, q, s,			\
-		__builtin_object_size(p, 0), __builtin_object_size(q, 0), \
-		__builtin_object_size(p, 1), __builtin_object_size(q, 1), \
-+		__struct_size(p), __struct_size(q),			\
-+		__member_size(p), __member_size(q),			\
- 		memcpy)
- #define memmove(p, q, s)  __fortify_memcpy_chk(p, q, s,			\
-		__builtin_object_size(p, 0), __builtin_object_size(q, 0), \
-		__builtin_object_size(p, 1), __builtin_object_size(q, 1), \
-+		__struct_size(p), __struct_size(q),			\
-+		__member_size(p), __member_size(q),			\
- 		memmove)
- 
- extern void *__real_memscan(void *, int, __kernel_size_t) __RENAME(memscan);
- __FORTIFY_INLINE void *memscan(void * const POS0 p, int c, __kernel_size_t size)
- {
-	size_t p_size = __builtin_object_size(p, 0);
-+	const size_t p_size = __struct_size(p);
- 
-	if (__builtin_constant_p(size) && p_size < size)
-+	if (__compiletime_lessthan(p_size, size))
- 		__read_overflow();
- 	if (p_size < size)
-		fortify_panic(__func__);
-+		fortify_panic(FORTIFY_FUNC_memscan, FORTIFY_READ, p_size, size, NULL);
- 	return __real_memscan(p, c, size);
- }
- 
- __FORTIFY_INLINE __diagnose_as(__builtin_memcmp, 1, 2, 3)
- int memcmp(const void * const POS0 p, const void * const POS0 q, __kernel_size_t size)
- {
-	size_t p_size = __builtin_object_size(p, 0);
-	size_t q_size = __builtin_object_size(q, 0);
-+	const size_t p_size = __struct_size(p);
-+	const size_t q_size = __struct_size(q);
- 
- 	if (__builtin_constant_p(size)) {
-		if (p_size < size)
-+		if (__compiletime_lessthan(p_size, size))
- 			__read_overflow();
-		if (q_size < size)
-+		if (__compiletime_lessthan(q_size, size))
- 			__read_overflow2();
- 	}
-	if (p_size < size || q_size < size)
-		fortify_panic(__func__);
-+	if (p_size < size)
-+		fortify_panic(FORTIFY_FUNC_memcmp, FORTIFY_READ, p_size, size, INT_MIN);
-+	else if (q_size < size)
-+		fortify_panic(FORTIFY_FUNC_memcmp, FORTIFY_READ, q_size, size, INT_MIN);
- 	return __underlying_memcmp(p, q, size);
- }
- 
- __FORTIFY_INLINE __diagnose_as(__builtin_memchr, 1, 2, 3)
- void *memchr(const void * const POS0 p, int c, __kernel_size_t size)
- {
-	size_t p_size = __builtin_object_size(p, 0);
-+	const size_t p_size = __struct_size(p);
- 
-	if (__builtin_constant_p(size) && p_size < size)
-+	if (__compiletime_lessthan(p_size, size))
- 		__read_overflow();
- 	if (p_size < size)
-		fortify_panic(__func__);
-+		fortify_panic(FORTIFY_FUNC_memchr, FORTIFY_READ, p_size, size, NULL);
- 	return __underlying_memchr(p, c, size);
- }
- 
- void *__real_memchr_inv(const void *s, int c, size_t n) __RENAME(memchr_inv);
- __FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size)
- {
-	size_t p_size = __builtin_object_size(p, 0);
-+	const size_t p_size = __struct_size(p);
- 
-	if (__builtin_constant_p(size) && p_size < size)
-+	if (__compiletime_lessthan(p_size, size))
- 		__read_overflow();
- 	if (p_size < size)
-		fortify_panic(__func__);
-+		fortify_panic(FORTIFY_FUNC_memchr_inv, FORTIFY_READ, p_size, size, NULL);
- 	return __real_memchr_inv(p, c, size);
- }
- 
-extern void *__real_kmemdup(const void *src, size_t len, gfp_t gfp) __RENAME(kmemdup);
-__FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp)
-+extern void *__real_kmemdup(const void *src, size_t len, gfp_t gfp) __RENAME(kmemdup_noprof)
-+								    __realloc_size(2);
-+__FORTIFY_INLINE void *kmemdup_noprof(const void * const POS0 p, size_t size, gfp_t gfp)
- {
-	size_t p_size = __builtin_object_size(p, 0);
-+	const size_t p_size = __struct_size(p);
- 
-	if (__builtin_constant_p(size) && p_size < size)
-+	if (__compiletime_lessthan(p_size, size))
- 		__read_overflow();
- 	if (p_size < size)
-		fortify_panic(__func__);
-+		fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ, p_size, size,
-+			      __real_kmemdup(p, 0, gfp));
- 	return __real_kmemdup(p, size, gfp);
- }
-+#define kmemdup(...)	alloc_hooks(kmemdup_noprof(__VA_ARGS__))
- 
-+/**
-+ * strcpy - Copy a string into another string buffer
-+ *
-+ * @p: pointer to destination of copy
-+ * @q: pointer to NUL-terminated source string to copy
-+ *
-+ * Do not use this function. While FORTIFY_SOURCE tries to avoid
-+ * overflows, this is only possible when the sizes of @q and @p are
-+ * known to the compiler. Prefer strscpy(), though note its different
-+ * return values for detecting truncation.
-+ *
-+ * Returns @p.
-+ *
-+ */
- /* Defined after fortified strlen to reuse it. */
- __FORTIFY_INLINE __diagnose_as(__builtin_strcpy, 1, 2)
- char *strcpy(char * const POS p, const char * const POS q)
- {
-	size_t p_size = __builtin_object_size(p, 1);
-	size_t q_size = __builtin_object_size(q, 1);
-+	const size_t p_size = __member_size(p);
-+	const size_t q_size = __member_size(q);
- 	size_t size;
- 
- 	/* If neither buffer size is known, immediately give up. */
-	if (p_size == (size_t)-1 && q_size == (size_t)-1)
-+	if (__builtin_constant_p(p_size) &&
-+	    __builtin_constant_p(q_size) &&
-+	    p_size == SIZE_MAX && q_size == SIZE_MAX)
- 		return __underlying_strcpy(p, q);
- 	size = strlen(q) + 1;
- 	/* Compile-time check for const size overflow. */
-	if (__builtin_constant_p(size) && p_size < size)
-+	if (__compiletime_lessthan(p_size, size))
- 		__write_overflow();
- 	/* Run-time check for dynamic size overflow. */
- 	if (p_size < size)
-		fortify_panic(__func__);
-+		fortify_panic(FORTIFY_FUNC_strcpy, FORTIFY_WRITE, p_size, size, p);
- 	__underlying_memcpy(p, q, size);
- 	return p;
- }
-diff --git a/include/linux/fpga/adi-axi-common.h b/include/linux/fpga/adi-axi-common.h
-deleted file mode 100644
-index 141ac3f251e6..000000000000
--- a/include/linux/fpga/adi-axi-common.h
-+++ /dev/null
-@@ -1,23 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Analog Devices AXI common registers & definitions
- *
- * Copyright 2019 Analog Devices Inc.
- *
- * https://wiki.analog.com/resources/fpga/docs/axi_ip
- * https://wiki.analog.com/resources/fpga/docs/hdl/regmap
- */
-
-#ifndef ADI_AXI_COMMON_H_
-#define ADI_AXI_COMMON_H_
-
-#define ADI_AXI_REG_VERSION			0x0000
-
-#define ADI_AXI_PCORE_VER(major, minor, patch)	\
-	(((major) << 16) | ((minor) << 8) | (patch))
-
-#define ADI_AXI_PCORE_VER_MAJOR(version)	(((version) >> 16) & 0xff)
-#define ADI_AXI_PCORE_VER_MINOR(version)	(((version) >> 8) & 0xff)
-#define ADI_AXI_PCORE_VER_PATCH(version)	((version) & 0xff)
-
-#endif /* ADI_AXI_COMMON_H_ */
-diff --git a/include/linux/fpga/fpga-bridge.h b/include/linux/fpga/fpga-bridge.h
-index 223da48a6d18..94c4edd047e5 100644
--- a/include/linux/fpga/fpga-bridge.h
-+++ b/include/linux/fpga/fpga-bridge.h
-@@ -45,6 +45,7 @@ struct fpga_bridge_info {
-  * @dev: FPGA bridge device
-  * @mutex: enforces exclusive reference to bridge
-  * @br_ops: pointer to struct of FPGA bridge ops
-+ * @br_ops_owner: module containing the br_ops
-  * @info: fpga image specific information
-  * @node: FPGA bridge list node
-  * @priv: low level driver private date
-@@ -54,6 +55,7 @@ struct fpga_bridge {
- 	struct device dev;
- 	struct mutex mutex; /* for exclusive reference to bridge */
- 	const struct fpga_bridge_ops *br_ops;
-+	struct module *br_ops_owner;
- 	struct fpga_image_info *info;
- 	struct list_head node;
- 	void *priv;
-@@ -79,10 +81,12 @@ int of_fpga_bridge_get_to_list(struct device_node *np,
- 			       struct fpga_image_info *info,
- 			       struct list_head *bridge_list);
- 
-+#define fpga_bridge_register(parent, name, br_ops, priv) \
-+	__fpga_bridge_register(parent, name, br_ops, priv, THIS_MODULE)
- struct fpga_bridge *
-fpga_bridge_register(struct device *parent, const char *name,
-		     const struct fpga_bridge_ops *br_ops,
-		     void *priv);
-+__fpga_bridge_register(struct device *parent, const char *name,
-+		       const struct fpga_bridge_ops *br_ops, void *priv,
-+		       struct module *owner);
- void fpga_bridge_unregister(struct fpga_bridge *br);
- 
- #endif /* _LINUX_FPGA_BRIDGE_H */
-diff --git a/include/linux/fpga/fpga-mgr.h b/include/linux/fpga/fpga-mgr.h
-index 54f63459efd6..0d4fe068f3d8 100644
--- a/include/linux/fpga/fpga-mgr.h
-+++ b/include/linux/fpga/fpga-mgr.h
-@@ -201,6 +201,7 @@ struct fpga_manager_ops {
-  * @state: state of fpga manager
-  * @compat_id: FPGA manager id for compatibility check.
-  * @mops: pointer to struct of fpga manager ops
-+ * @mops_owner: module containing the mops
-  * @priv: low level driver private date
-  */
- struct fpga_manager {
-@@ -210,6 +211,7 @@ struct fpga_manager {
- 	enum fpga_mgr_states state;
- 	struct fpga_compat_id *compat_id;
- 	const struct fpga_manager_ops *mops;
-+	struct module *mops_owner;
- 	void *priv;
- };
- 
-@@ -230,18 +232,30 @@ struct fpga_manager *fpga_mgr_get(struct device *dev);
- 
- void fpga_mgr_put(struct fpga_manager *mgr);
- 
-+#define fpga_mgr_register_full(parent, info) \
-+	__fpga_mgr_register_full(parent, info, THIS_MODULE)
- struct fpga_manager *
-fpga_mgr_register_full(struct device *parent, const struct fpga_manager_info *info);
-+__fpga_mgr_register_full(struct device *parent, const struct fpga_manager_info *info,
-+			 struct module *owner);
- 
-+#define fpga_mgr_register(parent, name, mops, priv) \
-+	__fpga_mgr_register(parent, name, mops, priv, THIS_MODULE)
- struct fpga_manager *
-fpga_mgr_register(struct device *parent, const char *name,
-		  const struct fpga_manager_ops *mops, void *priv);
-+__fpga_mgr_register(struct device *parent, const char *name,
-+		    const struct fpga_manager_ops *mops, void *priv, struct module *owner);
-+
- void fpga_mgr_unregister(struct fpga_manager *mgr);
- 
-+#define devm_fpga_mgr_register_full(parent, info) \
-+	__devm_fpga_mgr_register_full(parent, info, THIS_MODULE)
- struct fpga_manager *
-devm_fpga_mgr_register_full(struct device *parent, const struct fpga_manager_info *info);
-+__devm_fpga_mgr_register_full(struct device *parent, const struct fpga_manager_info *info,
-+			      struct module *owner);
-+#define devm_fpga_mgr_register(parent, name, mops, priv) \
-+	__devm_fpga_mgr_register(parent, name, mops, priv, THIS_MODULE)
- struct fpga_manager *
-devm_fpga_mgr_register(struct device *parent, const char *name,
-		       const struct fpga_manager_ops *mops, void *priv);
-+__devm_fpga_mgr_register(struct device *parent, const char *name,
-+			 const struct fpga_manager_ops *mops, void *priv,
-+			 struct module *owner);
- 
- #endif /*_LINUX_FPGA_MGR_H */
-diff --git a/include/linux/fpga/fpga-region.h b/include/linux/fpga/fpga-region.h
-index 9d4d32909340..5fbc05fe70a6 100644
--- a/include/linux/fpga/fpga-region.h
-+++ b/include/linux/fpga/fpga-region.h
-@@ -36,6 +36,7 @@ struct fpga_region_info {
-  * @mgr: FPGA manager
-  * @info: FPGA image info
-  * @compat_id: FPGA region id for compatibility check.
-+ * @ops_owner: module containing the get_bridges function
-  * @priv: private data
-  * @get_bridges: optional function to get bridges to a list
-  */
-@@ -46,6 +47,7 @@ struct fpga_region {
- 	struct fpga_manager *mgr;
- 	struct fpga_image_info *info;
- 	struct fpga_compat_id *compat_id;
-+	struct module *ops_owner;
- 	void *priv;
- 	int (*get_bridges)(struct fpga_region *region);
- };
-@@ -58,12 +60,17 @@ fpga_region_class_find(struct device *start, const void *data,
- 
- int fpga_region_program_fpga(struct fpga_region *region);
- 
-+#define fpga_region_register_full(parent, info) \
-+	__fpga_region_register_full(parent, info, THIS_MODULE)
- struct fpga_region *
-fpga_region_register_full(struct device *parent, const struct fpga_region_info *info);
-+__fpga_region_register_full(struct device *parent, const struct fpga_region_info *info,
-+			    struct module *owner);
- 
-+#define fpga_region_register(parent, mgr, get_bridges) \
-+	__fpga_region_register(parent, mgr, get_bridges, THIS_MODULE)
- struct fpga_region *
-fpga_region_register(struct device *parent, struct fpga_manager *mgr,
-		     int (*get_bridges)(struct fpga_region *));
-+__fpga_region_register(struct device *parent, struct fpga_manager *mgr,
-+		       int (*get_bridges)(struct fpga_region *), struct module *owner);
- void fpga_region_unregister(struct fpga_region *region);
- 
- #endif /* _FPGA_REGION_H */
-diff --git a/include/linux/fprobe.h b/include/linux/fprobe.h
-index 1c2bde0ead73..0a3bcd1718f3 100644
--- a/include/linux/fprobe.h
-+++ b/include/linux/fprobe.h
-@@ -5,33 +5,69 @@
- 
- #include <linux/compiler.h>
- #include <linux/ftrace.h>
-#include <linux/rethook.h>
-+#include <linux/rcupdate.h>
-+#include <linux/refcount.h>
-+#include <linux/rhashtable.h>
-+#include <linux/slab.h>
-+
-+struct fprobe;
-+typedef int (*fprobe_entry_cb)(struct fprobe *fp, unsigned long entry_ip,
-+			       unsigned long ret_ip, struct ftrace_regs *regs,
-+			       void *entry_data);
-+
-+typedef void (*fprobe_exit_cb)(struct fprobe *fp, unsigned long entry_ip,
-+			       unsigned long ret_ip, struct ftrace_regs *regs,
-+			       void *entry_data);
-+
-+/**
-+ * struct fprobe_hlist_node - address based hash list node for fprobe.
-+ *
-+ * @hlist: The hlist node for address search hash table.
-+ * @addr: One of the probing address of @fp.
-+ * @fp: The fprobe which owns this.
-+ */
-+struct fprobe_hlist_node {
-+	struct rhlist_head	hlist;
-+	unsigned long		addr;
-+	struct fprobe		*fp;
-+};
-+
-+/**
-+ * struct fprobe_hlist - hash list nodes for fprobe.
-+ *
-+ * @hlist: The hlist node for existence checking hash table.
-+ * @rcu: rcu_head for RCU deferred release.
-+ * @fp: The fprobe which owns this fprobe_hlist.
-+ * @size: The size of @array.
-+ * @array: The fprobe_hlist_node for each address to probe.
-+ */
-+struct fprobe_hlist {
-+	struct hlist_node		hlist;
-+	struct rcu_head			rcu;
-+	struct fprobe			*fp;
-+	int				size;
-+	struct fprobe_hlist_node	array[] __counted_by(size);
-+};
- 
- /**
-  * struct fprobe - ftrace based probe.
- * @ops: The ftrace_ops.
-+ *
-  * @nmissed: The counter for missing events.
-  * @flags: The status flag.
- * @rethook: The rethook data structure. (internal data)
-+ * @entry_data_size: The private data storage size.
-  * @entry_handler: The callback function for function entry.
-  * @exit_handler: The callback function for function exit.
-+ * @hlist_array: The fprobe_hlist for fprobe search from IP hash table.
-  */
- struct fprobe {
-#ifdef CONFIG_FUNCTION_TRACER
-	/*
-	 * If CONFIG_FUNCTION_TRACER is not set, CONFIG_FPROBE is disabled too.
-	 * But user of fprobe may keep embedding the struct fprobe on their own
-	 * code. To avoid build error, this will keep the fprobe data structure
-	 * defined here, but remove ftrace_ops data structure.
-	 */
-	struct ftrace_ops	ops;
-#endif
- 	unsigned long		nmissed;
- 	unsigned int		flags;
-	struct rethook		*rethook;
-+	size_t			entry_data_size;
- 
-	void (*entry_handler)(struct fprobe *fp, unsigned long entry_ip, struct pt_regs *regs);
-	void (*exit_handler)(struct fprobe *fp, unsigned long entry_ip, struct pt_regs *regs);
-+	fprobe_entry_cb entry_handler;
-+	fprobe_exit_cb  exit_handler;
-+
-+	struct fprobe_hlist	*hlist_array;
- };
- 
- /* This fprobe is soft-disabled. */
-@@ -58,6 +94,8 @@ int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter
- int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num);
- int register_fprobe_syms(struct fprobe *fp, const char **syms, int num);
- int unregister_fprobe(struct fprobe *fp);
-+bool fprobe_is_registered(struct fprobe *fp);
-+int fprobe_count_ips_from_filter(const char *filter, const char *notfilter);
- #else
- static inline int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)
- {
-@@ -75,6 +113,14 @@ static inline int unregister_fprobe(struct fprobe *fp)
- {
- 	return -EOPNOTSUPP;
- }
-+static inline bool fprobe_is_registered(struct fprobe *fp)
-+{
-+	return false;
-+}
-+static inline int fprobe_count_ips_from_filter(const char *filter, const char *notfilter)
-+{
-+	return -EOPNOTSUPP;
-+}
- #endif
- 
- /**
-@@ -102,4 +148,9 @@ static inline void enable_fprobe(struct fprobe *fp)
- 		fp->flags &= ~FPROBE_FL_DISABLED;
- }
- 
-+/* The entry data size is 4 bits (=16) * sizeof(long) in maximum */
-+#define FPROBE_DATA_SIZE_BITS		4
-+#define MAX_FPROBE_DATA_SIZE_WORD	((1L << FPROBE_DATA_SIZE_BITS) - 1)
-+#define MAX_FPROBE_DATA_SIZE		(MAX_FPROBE_DATA_SIZE_WORD * sizeof(long))
-+
- #endif
-diff --git a/include/linux/fpu.h b/include/linux/fpu.h
-new file mode 100644
-index 000000000000..2fb63e22913b
--- /dev/null
-+++ b/include/linux/fpu.h
-@@ -0,0 +1,12 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+
-+#ifndef _LINUX_FPU_H
-+#define _LINUX_FPU_H
-+
-+#ifdef _LINUX_FPU_COMPILATION_UNIT
-+#error FP code must be compiled separately. See Documentation/core-api/floating-point.rst.
-+#endif
-+
-+#include <asm/fpu.h>
-+
-+#endif
-diff --git a/include/linux/framer/framer-provider.h b/include/linux/framer/framer-provider.h
-new file mode 100644
-index 000000000000..9724d4b44b9c
--- /dev/null
-+++ b/include/linux/framer/framer-provider.h
-@@ -0,0 +1,193 @@
-+/* SPDX-License-Identifier: GPL-2.0-or-later */
-+/*
-+ * Generic framer profider header file
-+ *
-+ * Copyright 2023 CS GROUP France
-+ *
-+ * Author: Herve Codina <herve.codina@bootlin.com>
-+ */
-+
-+#ifndef __DRIVERS_PROVIDER_FRAMER_H
-+#define __DRIVERS_PROVIDER_FRAMER_H
-+
-+#include <linux/export.h>
-+#include <linux/framer/framer.h>
-+#include <linux/types.h>
-+
-+#define FRAMER_FLAG_POLL_STATUS  BIT(0)
-+
-+/**
-+ * struct framer_ops - set of function pointers for performing framer operations
-+ * @init: operation to be performed for initializing the framer
-+ * @exit: operation to be performed while exiting
-+ * @power_on: powering on the framer
-+ * @power_off: powering off the framer
-+ * @flags: OR-ed flags (FRAMER_FLAG_*) to ask for core functionality
-+ *          - @FRAMER_FLAG_POLL_STATUS:
-+ *            Ask the core to perform a polling to get the framer status and
-+ *            notify consumers on change.
-+ *            The framer should call @framer_notify_status_change() when it
-+ *            detects a status change. This is usually done using interrupts.
-+ *            If the framer cannot detect this change, it can ask the core for
-+ *            a status polling. The core will call @get_status() periodically
-+ *            and, on change detected, it will notify the consumer.
-+ *            the @get_status()
-+ * @owner: the module owner containing the ops
-+ */
-+struct framer_ops {
-+	int	(*init)(struct framer *framer);
-+	void	(*exit)(struct framer *framer);
-+	int	(*power_on)(struct framer *framer);
-+	int	(*power_off)(struct framer *framer);
-+
-+	/**
-+	 * @get_status:
-+	 *
-+	 * Optional.
-+	 *
-+	 * Used to get the framer status. framer_init() must have
-+	 * been called on the framer.
-+	 *
-+	 * Returns: 0 if successful, an negative error code otherwise
-+	 */
-+	int	(*get_status)(struct framer *framer, struct framer_status *status);
-+
-+	/**
-+	 * @set_config:
-+	 *
-+	 * Optional.
-+	 *
-+	 * Used to set the framer configuration. framer_init() must have
-+	 * been called on the framer.
-+	 *
-+	 * Returns: 0 if successful, an negative error code otherwise
-+	 */
-+	int	(*set_config)(struct framer *framer, const struct framer_config *config);
-+
-+	/**
-+	 * @get_config:
-+	 *
-+	 * Optional.
-+	 *
-+	 * Used to get the framer configuration. framer_init() must have
-+	 * been called on the framer.
-+	 *
-+	 * Returns: 0 if successful, an negative error code otherwise
-+	 */
-+	int	(*get_config)(struct framer *framer, struct framer_config *config);
-+
-+	u32 flags;
-+	struct module *owner;
-+};
-+
-+/**
-+ * struct framer_provider - represents the framer provider
-+ * @dev: framer provider device
-+ * @owner: the module owner having of_xlate
-+ * @list: to maintain a linked list of framer providers
-+ * @of_xlate: function pointer to obtain framer instance from framer pointer
-+ */
-+struct framer_provider {
-+	struct device		*dev;
-+	struct module		*owner;
-+	struct list_head	list;
-+	struct framer * (*of_xlate)(struct device *dev,
-+				    const struct of_phandle_args *args);
-+};
-+
-+static inline void framer_set_drvdata(struct framer *framer, void *data)
-+{
-+	dev_set_drvdata(&framer->dev, data);
-+}
-+
-+static inline void *framer_get_drvdata(struct framer *framer)
-+{
-+	return dev_get_drvdata(&framer->dev);
-+}
-+
-+#if IS_ENABLED(CONFIG_GENERIC_FRAMER)
-+
-+/* Create and destroy a framer */
-+struct framer *framer_create(struct device *dev, struct device_node *node,
-+			     const struct framer_ops *ops);
-+void framer_destroy(struct framer *framer);
-+
-+/* devm version */
-+struct framer *devm_framer_create(struct device *dev, struct device_node *node,
-+				  const struct framer_ops *ops);
-+
-+struct framer *framer_provider_simple_of_xlate(struct device *dev,
-+					       const struct of_phandle_args *args);
-+
-+struct framer_provider *
-+__framer_provider_of_register(struct device *dev, struct module *owner,
-+			      struct framer *(*of_xlate)(struct device *dev,
-+							 const struct of_phandle_args *args));
-+
-+void framer_provider_of_unregister(struct framer_provider *framer_provider);
-+
-+struct framer_provider *
-+__devm_framer_provider_of_register(struct device *dev, struct module *owner,
-+				   struct framer *(*of_xlate)(struct device *dev,
-+							      const struct of_phandle_args *args));
-+
-+void framer_notify_status_change(struct framer *framer);
-+
-+#else /* IS_ENABLED(CONFIG_GENERIC_FRAMER) */
-+
-+static inline struct framer *framer_create(struct device *dev, struct device_node *node,
-+					   const struct framer_ops *ops)
-+{
-+	return ERR_PTR(-ENOSYS);
-+}
-+
-+static inline void framer_destroy(struct framer *framer)
-+{
-+}
-+
-+/* devm version */
-+static inline struct framer *devm_framer_create(struct device *dev, struct device_node *node,
-+						const struct framer_ops *ops)
-+{
-+	return ERR_PTR(-ENOSYS);
-+}
-+
-+static inline struct framer *framer_provider_simple_of_xlate(struct device *dev,
-+							     const struct of_phandle_args *args)
-+{
-+	return ERR_PTR(-ENOSYS);
-+}
-+
-+static inline struct framer_provider *
-+__framer_provider_of_register(struct device *dev, struct module *owner,
-+			      struct framer *(*of_xlate)(struct device *dev,
-+							 const struct of_phandle_args *args))
-+{
-+	return ERR_PTR(-ENOSYS);
-+}
-+
-+void framer_provider_of_unregister(struct framer_provider *framer_provider)
-+{
-+}
-+
-+static inline struct framer_provider *
-+__devm_framer_provider_of_register(struct device *dev, struct module *owner,
-+				   struct framer *(*of_xlate)(struct device *dev,
-+							      const struct of_phandle_args *args))
-+{
-+	return ERR_PTR(-ENOSYS);
-+}
-+
-+void framer_notify_status_change(struct framer *framer)
-+{
-+}
-+
-+#endif /* IS_ENABLED(CONFIG_GENERIC_FRAMER) */
-+
-+#define framer_provider_of_register(dev, xlate)		\
-+	__framer_provider_of_register((dev), THIS_MODULE, (xlate))
-+
-+#define devm_framer_provider_of_register(dev, xlate)	\
-+	__devm_framer_provider_of_register((dev), THIS_MODULE, (xlate))
-+
-+#endif /* __DRIVERS_PROVIDER_FRAMER_H */
-diff --git a/include/linux/framer/framer.h b/include/linux/framer/framer.h
-new file mode 100644
-index 000000000000..2b85fe9e7f9a
--- /dev/null
-+++ b/include/linux/framer/framer.h
-@@ -0,0 +1,205 @@
-+/* SPDX-License-Identifier: GPL-2.0-or-later */
-+/*
-+ * Generic framer header file
-+ *
-+ * Copyright 2023 CS GROUP France
-+ *
-+ * Author: Herve Codina <herve.codina@bootlin.com>
-+ */
-+
-+#ifndef __DRIVERS_FRAMER_H
-+#define __DRIVERS_FRAMER_H
-+
-+#include <linux/err.h>
-+#include <linux/mutex.h>
-+#include <linux/notifier.h>
-+#include <linux/of.h>
-+#include <linux/device.h>
-+#include <linux/workqueue.h>
-+
-+/**
-+ * enum framer_iface - Framer interface
-+ * @FRAMER_IFACE_E1: E1 interface
-+ * @FRAMER_IFACE_T1: T1 interface
-+ */
-+enum framer_iface {
-+	FRAMER_IFACE_E1,
-+	FRAMER_IFACE_T1,
-+};
-+
-+/**
-+ * enum framer_clock_type - Framer clock type
-+ * @FRAMER_CLOCK_EXT: External clock
-+ * @FRAMER_CLOCK_INT: Internal clock
-+ */
-+enum framer_clock_type {
-+	FRAMER_CLOCK_EXT,
-+	FRAMER_CLOCK_INT,
-+};
-+
-+/**
-+ * struct framer_config - Framer configuration
-+ * @iface: Framer line interface
-+ * @clock_type: Framer clock type
-+ * @line_clock_rate: Framer line clock rate
-+ */
-+struct framer_config {
-+	enum framer_iface iface;
-+	enum framer_clock_type clock_type;
-+	unsigned long line_clock_rate;
-+};
-+
-+/**
-+ * struct framer_status - Framer status
-+ * @link_is_on: Framer link state. true, the link is on, false, the link is off.
-+ */
-+struct framer_status {
-+	bool link_is_on;
-+};
-+
-+/**
-+ * enum framer_event - Event available for notification
-+ * @FRAMER_EVENT_STATUS: Event notified on framer_status changes
-+ */
-+enum framer_event {
-+	FRAMER_EVENT_STATUS,
-+};
-+
-+/**
-+ * struct framer - represents the framer device
-+ * @dev: framer device
-+ * @id: id of the framer device
-+ * @ops: function pointers for performing framer operations
-+ * @mutex: mutex to protect framer_ops
-+ * @init_count: used to protect when the framer is used by multiple consumers
-+ * @power_count: used to protect when the framer is used by multiple consumers
-+ * @pwr: power regulator associated with the framer
-+ * @notify_status_work: work structure used for status notifications
-+ * @notifier_list: notifier list used for notifications
-+ * @polling_work: delayed work structure used for the polling task
-+ * @prev_status: previous read status used by the polling task to detect changes
-+ */
-+struct framer {
-+	struct device			dev;
-+	int				id;
-+	const struct framer_ops		*ops;
-+	struct mutex			mutex;	/* Protect framer */
-+	int				init_count;
-+	int				power_count;
-+	struct regulator		*pwr;
-+	struct work_struct		notify_status_work;
-+	struct blocking_notifier_head	notifier_list;
-+	struct delayed_work		polling_work;
-+	struct framer_status		prev_status;
-+};
-+
-+#if IS_ENABLED(CONFIG_GENERIC_FRAMER)
-+int framer_pm_runtime_get(struct framer *framer);
-+int framer_pm_runtime_get_sync(struct framer *framer);
-+int framer_pm_runtime_put(struct framer *framer);
-+int framer_pm_runtime_put_sync(struct framer *framer);
-+int framer_init(struct framer *framer);
-+int framer_exit(struct framer *framer);
-+int framer_power_on(struct framer *framer);
-+int framer_power_off(struct framer *framer);
-+int framer_get_status(struct framer *framer, struct framer_status *status);
-+int framer_get_config(struct framer *framer, struct framer_config *config);
-+int framer_set_config(struct framer *framer, const struct framer_config *config);
-+int framer_notifier_register(struct framer *framer, struct notifier_block *nb);
-+int framer_notifier_unregister(struct framer *framer, struct notifier_block *nb);
-+
-+struct framer *framer_get(struct device *dev, const char *con_id);
-+void framer_put(struct device *dev, struct framer *framer);
-+
-+struct framer *devm_framer_get(struct device *dev, const char *con_id);
-+struct framer *devm_framer_optional_get(struct device *dev, const char *con_id);
-+#else
-+static inline int framer_pm_runtime_get(struct framer *framer)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline int framer_pm_runtime_get_sync(struct framer *framer)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline int framer_pm_runtime_put(struct framer *framer)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline int framer_pm_runtime_put_sync(struct framer *framer)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline int framer_init(struct framer *framer)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline int framer_exit(struct framer *framer)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline int framer_power_on(struct framer *framer)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline int framer_power_off(struct framer *framer)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline int framer_get_status(struct framer *framer, struct framer_status *status)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline int framer_get_config(struct framer *framer, struct framer_config *config)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline int framer_set_config(struct framer *framer, const struct framer_config *config)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline int framer_notifier_register(struct framer *framer,
-+					   struct notifier_block *nb)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline int framer_notifier_unregister(struct framer *framer,
-+					     struct notifier_block *nb)
-+{
-+	return -ENOSYS;
-+}
-+
-+static inline struct framer *framer_get(struct device *dev, const char *con_id)
-+{
-+	return ERR_PTR(-ENOSYS);
-+}
-+
-+static inline void framer_put(struct device *dev, struct framer *framer)
-+{
-+}
-+
-+static inline struct framer *devm_framer_get(struct device *dev, const char *con_id)
-+{
-+	return ERR_PTR(-ENOSYS);
-+}
-+
-+static inline struct framer *devm_framer_optional_get(struct device *dev, const char *con_id)
-+{
-+	return NULL;
-+}
-+
-+#endif
-+
-+#endif /* __DRIVERS_FRAMER_H */
-diff --git a/include/linux/framer/pef2256.h b/include/linux/framer/pef2256.h
-new file mode 100644
-index 000000000000..71d80af58c40
--- /dev/null
-+++ b/include/linux/framer/pef2256.h
-@@ -0,0 +1,31 @@
-+/* SPDX-License-Identifier: GPL-2.0-or-later */
-+/*
-+ * PEF2256 consumer API
-+ *
-+ * Copyright 2023 CS GROUP France
-+ *
-+ * Author: Herve Codina <herve.codina@bootlin.com>
-+ */
-+#ifndef __PEF2256_H__
-+#define __PEF2256_H__
-+
-+#include <linux/types.h>
-+
-+struct pef2256;
-+struct regmap;
-+
-+/* Retrieve the PEF2256 regmap */
-+struct regmap *pef2256_get_regmap(struct pef2256 *pef2256);
-+
-+/* PEF2256 hardware versions */
-+enum pef2256_version {
-+	PEF2256_VERSION_UNKNOWN,
-+	PEF2256_VERSION_1_2,
-+	PEF2256_VERSION_2_1,
-+	PEF2256_VERSION_2_2,
-+};
-+
-+/* Get the PEF2256 hardware version */
-+enum pef2256_version pef2256_get_version(struct pef2256 *pef2256);
-+
-+#endif /* __PEF2256_H__ */
-diff --git a/include/linux/freelist.h b/include/linux/freelist.h
-deleted file mode 100644
-index fc1842b96469..000000000000
--- a/include/linux/freelist.h
-+++ /dev/null
-@@ -1,129 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
-#ifndef FREELIST_H
-#define FREELIST_H
-
-#include <linux/atomic.h>
-
-/*
- * Copyright: cameron@moodycamel.com
- *
- * A simple CAS-based lock-free free list. Not the fastest thing in the world
- * under heavy contention, but simple and correct (assuming nodes are never
- * freed until after the free list is destroyed), and fairly speedy under low
- * contention.
- *
- * Adapted from: https://moodycamel.com/blog/2014/solving-the-aba-problem-for-lock-free-free-lists
- */
-
-struct freelist_node {
-	atomic_t		refs;
-	struct freelist_node	*next;
-};
-
-struct freelist_head {
-	struct freelist_node	*head;
-};
-
-#define REFS_ON_FREELIST 0x80000000
-#define REFS_MASK	 0x7FFFFFFF
-
-static inline void __freelist_add(struct freelist_node *node, struct freelist_head *list)
-{
-	/*
-	 * Since the refcount is zero, and nobody can increase it once it's
-	 * zero (except us, and we run only one copy of this method per node at
-	 * a time, i.e. the single thread case), then we know we can safely
-	 * change the next pointer of the node; however, once the refcount is
-	 * back above zero, then other threads could increase it (happens under
-	 * heavy contention, when the refcount goes to zero in between a load
-	 * and a refcount increment of a node in try_get, then back up to
-	 * something non-zero, then the refcount increment is done by the other
-	 * thread) -- so if the CAS to add the node to the actual list fails,
-	 * decrese the refcount and leave the add operation to the next thread
-	 * who puts the refcount back to zero (which could be us, hence the
-	 * loop).
-	 */
-	struct freelist_node *head = READ_ONCE(list->head);
-
-	for (;;) {
-		WRITE_ONCE(node->next, head);
-		atomic_set_release(&node->refs, 1);
-
-		if (!try_cmpxchg_release(&list->head, &head, node)) {
-			/*
-			 * Hmm, the add failed, but we can only try again when
-			 * the refcount goes back to zero.
-			 */
-			if (atomic_fetch_add_release(REFS_ON_FREELIST - 1, &node->refs) == 1)
-				continue;
-		}
-		return;
-	}
-}
-
-static inline void freelist_add(struct freelist_node *node, struct freelist_head *list)
-{
-	/*
-	 * We know that the should-be-on-freelist bit is 0 at this point, so
-	 * it's safe to set it using a fetch_add.
-	 */
-	if (!atomic_fetch_add_release(REFS_ON_FREELIST, &node->refs)) {
-		/*
-		 * Oh look! We were the last ones referencing this node, and we
-		 * know we want to add it to the free list, so let's do it!
-		 */
-		__freelist_add(node, list);
-	}
-}
-
-static inline struct freelist_node *freelist_try_get(struct freelist_head *list)
-{
-	struct freelist_node *prev, *next, *head = smp_load_acquire(&list->head);
-	unsigned int refs;
-
-	while (head) {
-		prev = head;
-		refs = atomic_read(&head->refs);
-		if ((refs & REFS_MASK) == 0 ||
-		    !atomic_try_cmpxchg_acquire(&head->refs, &refs, refs+1)) {
-			head = smp_load_acquire(&list->head);
-			continue;
-		}
-
-		/*
-		 * Good, reference count has been incremented (it wasn't at
-		 * zero), which means we can read the next and not worry about
-		 * it changing between now and the time we do the CAS.
-		 */
-		next = READ_ONCE(head->next);
-		if (try_cmpxchg_acquire(&list->head, &head, next)) {
-			/*
-			 * Yay, got the node. This means it was on the list,
-			 * which means should-be-on-freelist must be false no
-			 * matter the refcount (because nobody else knows it's
-			 * been taken off yet, it can't have been put back on).
-			 */
-			WARN_ON_ONCE(atomic_read(&head->refs) & REFS_ON_FREELIST);
-
-			/*
-			 * Decrease refcount twice, once for our ref, and once
-			 * for the list's ref.
-			 */
-			atomic_fetch_add(-2, &head->refs);
-
-			return head;
-		}
-
-		/*
-		 * OK, the head must have changed on us, but we still need to decrement
-		 * the refcount we increased.
-		 */
-		refs = atomic_fetch_add(-1, &prev->refs);
-		if (refs == REFS_ON_FREELIST + 1)
-			__freelist_add(prev, list);
-	}
-
-	return NULL;
-}
-
-#endif /* FREELIST_H */
-diff --git a/include/linux/freezer.h b/include/linux/freezer.h
-index 0621c5f86c39..0a8c6c4d1a82 100644
--- a/include/linux/freezer.h
-+++ b/include/linux/freezer.h
-@@ -8,9 +8,11 @@
- #include <linux/sched.h>
- #include <linux/wait.h>
- #include <linux/atomic.h>
-+#include <linux/jump_label.h>
- 
- #ifdef CONFIG_FREEZER
-extern atomic_t system_freezing_cnt;	/* nr of freezing conds in effect */
-+DECLARE_STATIC_KEY_FALSE(freezer_active);
-+
- extern bool pm_freezing;		/* PM freezing in effect */
- extern bool pm_nosig_freezing;		/* PM nosig freezing in effect */
- 
-@@ -20,23 +22,25 @@ extern bool pm_nosig_freezing;		/* PM nosig freezing in effect */
- extern unsigned int freeze_timeout_msecs;
- 
- /*
- * Check if a process has been frozen
-+ * Check if a process has been frozen for PM or cgroup1 freezer. Note that
-+ * cgroup2 freezer uses the job control mechanism and does not interact with
-+ * the PM freezer.
-  */
-static inline bool frozen(struct task_struct *p)
-{
-	return p->flags & PF_FROZEN;
-}
-+extern bool frozen(struct task_struct *p);
- 
- extern bool freezing_slow_path(struct task_struct *p);
- 
- /*
- * Check if there is a request to freeze a process
-+ * Check if there is a request to freeze a task from PM or cgroup1 freezer.
-+ * Note that cgroup2 freezer uses the job control mechanism and does not
-+ * interact with the PM freezer.
-  */
- static inline bool freezing(struct task_struct *p)
- {
-	if (likely(!atomic_read(&system_freezing_cnt)))
-		return false;
-	return freezing_slow_path(p);
-+	if (static_branch_unlikely(&freezer_active))
-+		return freezing_slow_path(p);
-+
-+	return false;
- }
- 
- /* Takes and releases task alloc lock using task_lock() */
-@@ -47,227 +51,30 @@ extern int freeze_processes(void);
- extern int freeze_kernel_threads(void);
- extern void thaw_processes(void);
- extern void thaw_kernel_threads(void);
-+extern void thaw_process(struct task_struct *p);
- 
-/*
- * DO NOT ADD ANY NEW CALLERS OF THIS FUNCTION
- * If try_to_freeze causes a lockdep warning it means the caller may deadlock
- */
-static inline bool try_to_freeze_unsafe(void)
-+static inline bool try_to_freeze(void)
- {
- 	might_sleep();
- 	if (likely(!freezing(current)))
- 		return false;
-	return __refrigerator(false);
-}
-
-static inline bool try_to_freeze(void)
-{
- 	if (!(current->flags & PF_NOFREEZE))
- 		debug_check_no_locks_held();
-	return try_to_freeze_unsafe();
-+	return __refrigerator(false);
- }
- 
- extern bool freeze_task(struct task_struct *p);
- extern bool set_freezable(void);
- 
- #ifdef CONFIG_CGROUP_FREEZER
-extern bool cgroup_freezing(struct task_struct *task);
-+extern bool cgroup1_freezing(struct task_struct *task);
- #else /* !CONFIG_CGROUP_FREEZER */
-static inline bool cgroup_freezing(struct task_struct *task)
-+static inline bool cgroup1_freezing(struct task_struct *task)
- {
- 	return false;
- }
- #endif /* !CONFIG_CGROUP_FREEZER */
- 
-/*
- * The PF_FREEZER_SKIP flag should be set by a vfork parent right before it
- * calls wait_for_completion(&vfork) and reset right after it returns from this
- * function.  Next, the parent should call try_to_freeze() to freeze itself
- * appropriately in case the child has exited before the freezing of tasks is
- * complete.  However, we don't want kernel threads to be frozen in unexpected
- * places, so we allow them to block freeze_processes() instead or to set
- * PF_NOFREEZE if needed. Fortunately, in the ____call_usermodehelper() case the
- * parent won't really block freeze_processes(), since ____call_usermodehelper()
- * (the child) does a little before exec/exit and it can't be frozen before
- * waking up the parent.
- */
-
-
-/**
- * freezer_do_not_count - tell freezer to ignore %current
- *
- * Tell freezers to ignore the current task when determining whether the
- * target frozen state is reached.  IOW, the current task will be
- * considered frozen enough by freezers.
- *
- * The caller shouldn't do anything which isn't allowed for a frozen task
- * until freezer_cont() is called.  Usually, freezer[_do_not]_count() pair
- * wrap a scheduling operation and nothing much else.
- */
-static inline void freezer_do_not_count(void)
-{
-	current->flags |= PF_FREEZER_SKIP;
-}
-
-/**
- * freezer_count - tell freezer to stop ignoring %current
- *
- * Undo freezer_do_not_count().  It tells freezers that %current should be
- * considered again and tries to freeze if freezing condition is already in
- * effect.
- */
-static inline void freezer_count(void)
-{
-	current->flags &= ~PF_FREEZER_SKIP;
-	/*
-	 * If freezing is in progress, the following paired with smp_mb()
-	 * in freezer_should_skip() ensures that either we see %true
-	 * freezing() or freezer_should_skip() sees !PF_FREEZER_SKIP.
-	 */
-	smp_mb();
-	try_to_freeze();
-}
-
-/* DO NOT ADD ANY NEW CALLERS OF THIS FUNCTION */
-static inline void freezer_count_unsafe(void)
-{
-	current->flags &= ~PF_FREEZER_SKIP;
-	smp_mb();
-	try_to_freeze_unsafe();
-}
-
-/**
- * freezer_should_skip - whether to skip a task when determining frozen
- *			 state is reached
- * @p: task in quesion
- *
- * This function is used by freezers after establishing %true freezing() to
- * test whether a task should be skipped when determining the target frozen
- * state is reached.  IOW, if this function returns %true, @p is considered
- * frozen enough.
- */
-static inline bool freezer_should_skip(struct task_struct *p)
-{
-	/*
-	 * The following smp_mb() paired with the one in freezer_count()
-	 * ensures that either freezer_count() sees %true freezing() or we
-	 * see cleared %PF_FREEZER_SKIP and return %false.  This makes it
-	 * impossible for a task to slip frozen state testing after
-	 * clearing %PF_FREEZER_SKIP.
-	 */
-	smp_mb();
-	return p->flags & PF_FREEZER_SKIP;
-}
-
-/*
- * These functions are intended to be used whenever you want allow a sleeping
- * task to be frozen. Note that neither return any clear indication of
- * whether a freeze event happened while in this function.
- */
-
-/* Like schedule(), but should not block the freezer. */
-static inline void freezable_schedule(void)
-{
-	freezer_do_not_count();
-	schedule();
-	freezer_count();
-}
-
-/* DO NOT ADD ANY NEW CALLERS OF THIS FUNCTION */
-static inline void freezable_schedule_unsafe(void)
-{
-	freezer_do_not_count();
-	schedule();
-	freezer_count_unsafe();
-}
-
-/*
- * Like schedule_timeout(), but should not block the freezer.  Do not
- * call this with locks held.
- */
-static inline long freezable_schedule_timeout(long timeout)
-{
-	long __retval;
-	freezer_do_not_count();
-	__retval = schedule_timeout(timeout);
-	freezer_count();
-	return __retval;
-}
-
-/*
- * Like schedule_timeout_interruptible(), but should not block the freezer.  Do not
- * call this with locks held.
- */
-static inline long freezable_schedule_timeout_interruptible(long timeout)
-{
-	long __retval;
-	freezer_do_not_count();
-	__retval = schedule_timeout_interruptible(timeout);
-	freezer_count();
-	return __retval;
-}
-
-/* DO NOT ADD ANY NEW CALLERS OF THIS FUNCTION */
-static inline long freezable_schedule_timeout_interruptible_unsafe(long timeout)
-{
-	long __retval;
-
-	freezer_do_not_count();
-	__retval = schedule_timeout_interruptible(timeout);
-	freezer_count_unsafe();
-	return __retval;
-}
-
-/* Like schedule_timeout_killable(), but should not block the freezer. */
-static inline long freezable_schedule_timeout_killable(long timeout)
-{
-	long __retval;
-	freezer_do_not_count();
-	__retval = schedule_timeout_killable(timeout);
-	freezer_count();
-	return __retval;
-}
-
-/* DO NOT ADD ANY NEW CALLERS OF THIS FUNCTION */
-static inline long freezable_schedule_timeout_killable_unsafe(long timeout)
-{
-	long __retval;
-	freezer_do_not_count();
-	__retval = schedule_timeout_killable(timeout);
-	freezer_count_unsafe();
-	return __retval;
-}
-
-/*
- * Like schedule_hrtimeout_range(), but should not block the freezer.  Do not
- * call this with locks held.
- */
-static inline int freezable_schedule_hrtimeout_range(ktime_t *expires,
-		u64 delta, const enum hrtimer_mode mode)
-{
-	int __retval;
-	freezer_do_not_count();
-	__retval = schedule_hrtimeout_range(expires, delta, mode);
-	freezer_count();
-	return __retval;
-}
-
-/*
- * Freezer-friendly wrappers around wait_event_interruptible(),
- * wait_event_killable() and wait_event_interruptible_timeout(), originally
- * defined in <linux/wait.h>
- */
-
-/* DO NOT ADD ANY NEW CALLERS OF THIS FUNCTION */
-#define wait_event_freezekillable_unsafe(wq, condition)			\
-({									\
-	int __retval;							\
-	freezer_do_not_count();						\
-	__retval = wait_event_killable(wq, (condition));		\
-	freezer_count_unsafe();						\
-	__retval;							\
-})
-
- #else /* !CONFIG_FREEZER */
- static inline bool frozen(struct task_struct *p) { return false; }
- static inline bool freezing(struct task_struct *p) { return false; }
-@@ -278,38 +85,12 @@ static inline int freeze_processes(void) { return -ENOSYS; }
- static inline int freeze_kernel_threads(void) { return -ENOSYS; }
- static inline void thaw_processes(void) {}
- static inline void thaw_kernel_threads(void) {}
-+static inline void thaw_process(struct task_struct *p) {}
- 
- static inline bool try_to_freeze(void) { return false; }
- 
-static inline void freezer_do_not_count(void) {}
-static inline void freezer_count(void) {}
-static inline int freezer_should_skip(struct task_struct *p) { return 0; }
- static inline void set_freezable(void) {}
- 
-#define freezable_schedule()  schedule()
-
-#define freezable_schedule_unsafe()  schedule()
-
-#define freezable_schedule_timeout(timeout)  schedule_timeout(timeout)
-
-#define freezable_schedule_timeout_interruptible(timeout)		\
-	schedule_timeout_interruptible(timeout)
-
-#define freezable_schedule_timeout_interruptible_unsafe(timeout)	\
-	schedule_timeout_interruptible(timeout)
-
-#define freezable_schedule_timeout_killable(timeout)			\
-	schedule_timeout_killable(timeout)
-
-#define freezable_schedule_timeout_killable_unsafe(timeout)		\
-	schedule_timeout_killable(timeout)
-
-#define freezable_schedule_hrtimeout_range(expires, delta, mode)	\
-	schedule_hrtimeout_range(expires, delta, mode)
-
-#define wait_event_freezekillable_unsafe(wq, condition)			\
-		wait_event_killable(wq, condition)
-
- #endif /* !CONFIG_FREEZER */
- 
- #endif	/* FREEZER_H_INCLUDED */
-diff --git a/include/linux/frontswap.h b/include/linux/frontswap.h
-deleted file mode 100644
-index a631bac12220..000000000000
--- a/include/linux/frontswap.h
-+++ /dev/null
-@@ -1,91 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _LINUX_FRONTSWAP_H
-#define _LINUX_FRONTSWAP_H
-
-#include <linux/swap.h>
-#include <linux/mm.h>
-#include <linux/bitops.h>
-#include <linux/jump_label.h>
-
-struct frontswap_ops {
-	void (*init)(unsigned); /* this swap type was just swapon'ed */
-	int (*store)(unsigned, pgoff_t, struct page *); /* store a page */
-	int (*load)(unsigned, pgoff_t, struct page *); /* load a page */
-	void (*invalidate_page)(unsigned, pgoff_t); /* page no longer needed */
-	void (*invalidate_area)(unsigned); /* swap type just swapoff'ed */
-};
-
-int frontswap_register_ops(const struct frontswap_ops *ops);
-
-extern void frontswap_init(unsigned type, unsigned long *map);
-extern int __frontswap_store(struct page *page);
-extern int __frontswap_load(struct page *page);
-extern void __frontswap_invalidate_page(unsigned, pgoff_t);
-extern void __frontswap_invalidate_area(unsigned);
-
-#ifdef CONFIG_FRONTSWAP
-extern struct static_key_false frontswap_enabled_key;
-
-static inline bool frontswap_enabled(void)
-{
-	return static_branch_unlikely(&frontswap_enabled_key);
-}
-
-static inline void frontswap_map_set(struct swap_info_struct *p,
-				     unsigned long *map)
-{
-	p->frontswap_map = map;
-}
-
-static inline unsigned long *frontswap_map_get(struct swap_info_struct *p)
-{
-	return p->frontswap_map;
-}
-#else
-/* all inline routines become no-ops and all externs are ignored */
-
-static inline bool frontswap_enabled(void)
-{
-	return false;
-}
-
-static inline void frontswap_map_set(struct swap_info_struct *p,
-				     unsigned long *map)
-{
-}
-
-static inline unsigned long *frontswap_map_get(struct swap_info_struct *p)
-{
-	return NULL;
-}
-#endif
-
-static inline int frontswap_store(struct page *page)
-{
-	if (frontswap_enabled())
-		return __frontswap_store(page);
-
-	return -1;
-}
-
-static inline int frontswap_load(struct page *page)
-{
-	if (frontswap_enabled())
-		return __frontswap_load(page);
-
-	return -1;
-}
-
-static inline void frontswap_invalidate_page(unsigned type, pgoff_t offset)
-{
-	if (frontswap_enabled())
-		__frontswap_invalidate_page(type, offset);
-}
-
-static inline void frontswap_invalidate_area(unsigned type)
-{
-	if (frontswap_enabled())
-		__frontswap_invalidate_area(type);
-}
-
-#endif /* _LINUX_FRONTSWAP_H */
-diff --git a/include/linux/fs.h b/include/linux/fs.h
-index 9eced4cc286e..04ceeca12a0d 100644
--- a/include/linux/fs.h
-+++ b/include/linux/fs.h
-@@ -2,6 +2,8 @@
- #ifndef _LINUX_FS_H
- #define _LINUX_FS_H
- 
-+#include <linux/fs/super.h>
-+#include <linux/vfsdebug.h>
- #include <linux/linkage.h>
- #include <linux/wait_bit.h>
- #include <linux/kdev_t.h>
-@@ -10,7 +12,6 @@
- #include <linux/stat.h>
- #include <linux/cache.h>
- #include <linux/list.h>
-#include <linux/list_lru.h>
- #include <linux/llist.h>
- #include <linux/radix-tree.h>
- #include <linux/xarray.h>
-@@ -36,22 +37,23 @@
- #include <linux/uuid.h>
- #include <linux/errseq.h>
- #include <linux/ioprio.h>
-#include <linux/fs_types.h>
- #include <linux/build_bug.h>
- #include <linux/stddef.h>
- #include <linux/mount.h>
- #include <linux/cred.h>
- #include <linux/mnt_idmapping.h>
- #include <linux/slab.h>
-+#include <linux/maple_tree.h>
-+#include <linux/rw_hint.h>
-+#include <linux/file_ref.h>
-+#include <linux/unicode.h>
- 
- #include <asm/byteorder.h>
- #include <uapi/linux/fs.h>
- 
-struct backing_dev_info;
- struct bdi_writeback;
- struct bio;
- struct io_comp_batch;
-struct export_operations;
- struct fiemap_extent_info;
- struct hd_geometry;
- struct iovec;
-@@ -65,16 +67,13 @@ struct vfsmount;
- struct cred;
- struct swap_info_struct;
- struct seq_file;
-struct workqueue_struct;
- struct iov_iter;
-struct fscrypt_info;
-struct fscrypt_operations;
-struct fsverity_info;
-struct fsverity_operations;
-+struct fsnotify_mark_connector;
- struct fs_context;
- struct fs_parameter_spec;
-struct fileattr;
-+struct file_kattr;
- struct iomap_ops;
-+struct delegated_inode;
- 
- extern void __init inode_init(void);
- extern void __init inode_init_early(void);
-@@ -108,28 +107,28 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset,
-  */
- 
- /* file is open for reading */
-#define FMODE_READ		((__force fmode_t)0x1)
-+#define FMODE_READ		((__force fmode_t)(1 << 0))
- /* file is open for writing */
-#define FMODE_WRITE		((__force fmode_t)0x2)
-+#define FMODE_WRITE		((__force fmode_t)(1 << 1))
- /* file is seekable */
-#define FMODE_LSEEK		((__force fmode_t)0x4)
-+#define FMODE_LSEEK		((__force fmode_t)(1 << 2))
- /* file can be accessed using pread */
-#define FMODE_PREAD		((__force fmode_t)0x8)
-+#define FMODE_PREAD		((__force fmode_t)(1 << 3))
- /* file can be accessed using pwrite */
-#define FMODE_PWRITE		((__force fmode_t)0x10)
-+#define FMODE_PWRITE		((__force fmode_t)(1 << 4))
- /* File is opened for execution with sys_execve / sys_uselib */
-#define FMODE_EXEC		((__force fmode_t)0x20)
-/* File is opened with O_NDELAY (only set for block devices) */
-#define FMODE_NDELAY		((__force fmode_t)0x40)
-/* File is opened with O_EXCL (only set for block devices) */
-#define FMODE_EXCL		((__force fmode_t)0x80)
-/* File is opened using open(.., 3, ..) and is writeable only for ioctls
-   (specialy hack for floppy.c) */
-#define FMODE_WRITE_IOCTL	((__force fmode_t)0x100)
-+#define FMODE_EXEC		((__force fmode_t)(1 << 5))
-+/* File writes are restricted (block device specific) */
-+#define FMODE_WRITE_RESTRICTED	((__force fmode_t)(1 << 6))
-+/* File supports atomic writes */
-+#define FMODE_CAN_ATOMIC_WRITE	((__force fmode_t)(1 << 7))
-+
-+/* FMODE_* bit 8 */
-+
- /* 32bit hashes as llseek() offset (for directories) */
-#define FMODE_32BITHASH         ((__force fmode_t)0x200)
-+#define FMODE_32BITHASH         ((__force fmode_t)(1 << 9))
- /* 64bit hashes as llseek() offset (for directories) */
-#define FMODE_64BITHASH         ((__force fmode_t)0x400)
-+#define FMODE_64BITHASH         ((__force fmode_t)(1 << 10))
- 
- /*
-  * Don't update ctime and mtime.
-@@ -137,52 +136,85 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset,
-  * Currently a special hack for the XFS open_by_handle ioctl, but we'll
-  * hopefully graduate it to a proper O_CMTIME flag supported by open(2) soon.
-  */
-#define FMODE_NOCMTIME		((__force fmode_t)0x800)
-+#define FMODE_NOCMTIME		((__force fmode_t)(1 << 11))
- 
- /* Expect random access pattern */
-#define FMODE_RANDOM		((__force fmode_t)0x1000)
-+#define FMODE_RANDOM		((__force fmode_t)(1 << 12))
- 
-/* File is huge (eg. /dev/mem): treat loff_t as unsigned */
-#define FMODE_UNSIGNED_OFFSET	((__force fmode_t)0x2000)
-+/* Supports IOCB_HAS_METADATA */
-+#define FMODE_HAS_METADATA	((__force fmode_t)(1 << 13))
- 
- /* File is opened with O_PATH; almost nothing can be done with it */
-#define FMODE_PATH		((__force fmode_t)0x4000)
-+#define FMODE_PATH		((__force fmode_t)(1 << 14))
- 
- /* File needs atomic accesses to f_pos */
-#define FMODE_ATOMIC_POS	((__force fmode_t)0x8000)
-+#define FMODE_ATOMIC_POS	((__force fmode_t)(1 << 15))
- /* Write access to underlying fs */
-#define FMODE_WRITER		((__force fmode_t)0x10000)
-+#define FMODE_WRITER		((__force fmode_t)(1 << 16))
- /* Has read method(s) */
-#define FMODE_CAN_READ          ((__force fmode_t)0x20000)
-+#define FMODE_CAN_READ          ((__force fmode_t)(1 << 17))
- /* Has write method(s) */
-#define FMODE_CAN_WRITE         ((__force fmode_t)0x40000)
-+#define FMODE_CAN_WRITE         ((__force fmode_t)(1 << 18))
- 
-#define FMODE_OPENED		((__force fmode_t)0x80000)
-#define FMODE_CREATED		((__force fmode_t)0x100000)
-+#define FMODE_OPENED		((__force fmode_t)(1 << 19))
-+#define FMODE_CREATED		((__force fmode_t)(1 << 20))
- 
- /* File is stream-like */
-#define FMODE_STREAM		((__force fmode_t)0x200000)
-+#define FMODE_STREAM		((__force fmode_t)(1 << 21))
- 
- /* File supports DIRECT IO */
-#define	FMODE_CAN_ODIRECT	((__force fmode_t)0x400000)
-+#define	FMODE_CAN_ODIRECT	((__force fmode_t)(1 << 22))
-+
-+#define	FMODE_NOREUSE		((__force fmode_t)(1 << 23))
-+
-+/* File is embedded in backing_file object */
-+#define FMODE_BACKING		((__force fmode_t)(1 << 24))
-+
-+/*
-+ * Together with FMODE_NONOTIFY_PERM defines which fsnotify events shouldn't be
-+ * generated (see below)
-+ */
-+#define FMODE_NONOTIFY		((__force fmode_t)(1 << 25))
- 
-/* File was opened by fanotify and shouldn't generate fanotify events */
-#define FMODE_NONOTIFY		((__force fmode_t)0x4000000)
-+/*
-+ * Together with FMODE_NONOTIFY defines which fsnotify events shouldn't be
-+ * generated (see below)
-+ */
-+#define FMODE_NONOTIFY_PERM	((__force fmode_t)(1 << 26))
- 
- /* File is capable of returning -EAGAIN if I/O will block */
-#define FMODE_NOWAIT		((__force fmode_t)0x8000000)
-+#define FMODE_NOWAIT		((__force fmode_t)(1 << 27))
- 
- /* File represents mount that needs unmounting */
-#define FMODE_NEED_UNMOUNT	((__force fmode_t)0x10000000)
-+#define FMODE_NEED_UNMOUNT	((__force fmode_t)(1 << 28))
- 
- /* File does not contribute to nr_files count */
-#define FMODE_NOACCOUNT		((__force fmode_t)0x20000000)
-
-/* File supports async buffered reads */
-#define FMODE_BUF_RASYNC	((__force fmode_t)0x40000000)
-+#define FMODE_NOACCOUNT		((__force fmode_t)(1 << 29))
- 
-/* File supports async nowait buffered writes */
-#define FMODE_BUF_WASYNC	((__force fmode_t)0x80000000)
-+/*
-+ * The two FMODE_NONOTIFY* define which fsnotify events should not be generated
-+ * for an open file. These are the possible values of
-+ * (f->f_mode & FMODE_FSNOTIFY_MASK) and their meaning:
-+ *
-+ * FMODE_NONOTIFY - suppress all (incl. non-permission) events.
-+ * FMODE_NONOTIFY_PERM - suppress permission (incl. pre-content) events.
-+ * FMODE_NONOTIFY | FMODE_NONOTIFY_PERM - suppress only FAN_ACCESS_PERM.
-+ */
-+#define FMODE_FSNOTIFY_MASK \
-+	(FMODE_NONOTIFY | FMODE_NONOTIFY_PERM)
-+
-+#define FMODE_FSNOTIFY_NONE(mode) \
-+	((mode & FMODE_FSNOTIFY_MASK) == FMODE_NONOTIFY)
-+#ifdef CONFIG_FANOTIFY_ACCESS_PERMISSIONS
-+#define FMODE_FSNOTIFY_HSM(mode) \
-+	((mode & FMODE_FSNOTIFY_MASK) == 0 || \
-+	 (mode & FMODE_FSNOTIFY_MASK) == (FMODE_NONOTIFY | FMODE_NONOTIFY_PERM))
-+#define FMODE_FSNOTIFY_ACCESS_PERM(mode) \
-+	((mode & FMODE_FSNOTIFY_MASK) == 0)
-+#else
-+#define FMODE_FSNOTIFY_ACCESS_PERM(mode) 0
-+#define FMODE_FSNOTIFY_HSM(mode)	0
-+#endif
- 
- /*
-  * Attribute flags.  These should be or-ed together to figure out what
-@@ -198,6 +230,7 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset,
- #define ATTR_ATIME_SET	(1 << 7)
- #define ATTR_MTIME_SET	(1 << 8)
- #define ATTR_FORCE	(1 << 9) /* Not a change, but a change it */
-+#define ATTR_CTIME_SET	(1 << 10)
- #define ATTR_KILL_SUID	(1 << 11)
- #define ATTR_KILL_SGID	(1 << 12)
- #define ATTR_FILE	(1 << 13)
-@@ -205,6 +238,7 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset,
- #define ATTR_OPEN	(1 << 15) /* Truncating from open(O_TRUNC) */
- #define ATTR_TIMES_SET	(1 << 16)
- #define ATTR_TOUCH	(1 << 17)
-+#define ATTR_DELEG	(1 << 18) /* Delegated attrs. Don't break write delegations */
- 
- /*
-  * Whiteout is represented by a char device.  The following constants define the
-@@ -259,11 +293,6 @@ struct iattr {
- };
- 
- /*
- * Includes for diskquotas.
- */
-#include <linux/quota.h>
-
-/*
-  * Maximum number of layers of fs stack.  Needs to be limited to
-  * prevent kernel stack overflow
-  */
-@@ -308,25 +337,15 @@ struct address_space;
- struct writeback_control;
- struct readahead_control;
- 
-/*
- * Write life time hint values.
- * Stored in struct inode as u8.
- */
-enum rw_hint {
-	WRITE_LIFE_NOT_SET	= 0,
-	WRITE_LIFE_NONE		= RWH_WRITE_LIFE_NONE,
-	WRITE_LIFE_SHORT	= RWH_WRITE_LIFE_SHORT,
-	WRITE_LIFE_MEDIUM	= RWH_WRITE_LIFE_MEDIUM,
-	WRITE_LIFE_LONG		= RWH_WRITE_LIFE_LONG,
-	WRITE_LIFE_EXTREME	= RWH_WRITE_LIFE_EXTREME,
-};
-
- /* Match RWF_* bits to IOCB bits */
- #define IOCB_HIPRI		(__force int) RWF_HIPRI
- #define IOCB_DSYNC		(__force int) RWF_DSYNC
- #define IOCB_SYNC		(__force int) RWF_SYNC
- #define IOCB_NOWAIT		(__force int) RWF_NOWAIT
- #define IOCB_APPEND		(__force int) RWF_APPEND
-+#define IOCB_ATOMIC		(__force int) RWF_ATOMIC
-+#define IOCB_DONTCACHE		(__force int) RWF_DONTCACHE
-+#define IOCB_NOSIGNAL		(__force int) RWF_NOSIGNAL
- 
- /* non-RWF related bits - start at 16 */
- #define IOCB_EVENTFD		(1 << 16)
-@@ -337,6 +356,27 @@ enum rw_hint {
- #define IOCB_NOIO		(1 << 20)
- /* can use bio alloc cache */
- #define IOCB_ALLOC_CACHE	(1 << 21)
-+/* kiocb is a read or write operation submitted by fs/aio.c. */
-+#define IOCB_AIO_RW		(1 << 22)
-+#define IOCB_HAS_METADATA	(1 << 23)
-+
-+/* for use in trace events */
-+#define TRACE_IOCB_STRINGS \
-+	{ IOCB_HIPRI,		"HIPRI" }, \
-+	{ IOCB_DSYNC,		"DSYNC" }, \
-+	{ IOCB_SYNC,		"SYNC" }, \
-+	{ IOCB_NOWAIT,		"NOWAIT" }, \
-+	{ IOCB_APPEND,		"APPEND" }, \
-+	{ IOCB_ATOMIC,		"ATOMIC" }, \
-+	{ IOCB_DONTCACHE,	"DONTCACHE" }, \
-+	{ IOCB_EVENTFD,		"EVENTFD"}, \
-+	{ IOCB_DIRECT,		"DIRECT" }, \
-+	{ IOCB_WRITE,		"WRITE" }, \
-+	{ IOCB_WAITQ,		"WAITQ" }, \
-+	{ IOCB_NOIO,		"NOIO" }, \
-+	{ IOCB_ALLOC_CACHE,	"ALLOC_CACHE" }, \
-+	{ IOCB_AIO_RW,		"AIO_RW" }, \
-+	{ IOCB_HAS_METADATA,	"AIO_HAS_METADATA" }
- 
- struct kiocb {
- 	struct file		*ki_filp;
-@@ -345,7 +385,14 @@ struct kiocb {
- 	void			*private;
- 	int			ki_flags;
- 	u16			ki_ioprio; /* See linux/ioprio.h */
-	struct wait_page_queue	*ki_waitq; /* for async buffered IO */
-+	u8			ki_write_stream;
-+
-+	/*
-+	 * Only used for async buffered reads, where it denotes the page
-+	 * waitqueue associated with completing the read.
-+	 * Valid IFF IOCB_WAITQ is set.
-+	 */
-+	struct wait_page_queue	*ki_waitq;
- };
- 
- static inline bool is_sync_kiocb(struct kiocb *kiocb)
-@@ -354,7 +401,6 @@ static inline bool is_sync_kiocb(struct kiocb *kiocb)
- }
- 
- struct address_space_operations {
-	int (*writepage)(struct page *page, struct writeback_control *wbc);
- 	int (*read_folio)(struct file *, struct folio *);
- 
- 	/* Write back some dirty pages from this mapping. */
-@@ -365,12 +411,12 @@ struct address_space_operations {
- 
- 	void (*readahead)(struct readahead_control *);
- 
-	int (*write_begin)(struct file *, struct address_space *mapping,
-+	int (*write_begin)(const struct kiocb *, struct address_space *mapping,
- 				loff_t pos, unsigned len,
-				struct page **pagep, void **fsdata);
-	int (*write_end)(struct file *, struct address_space *mapping,
-+				struct folio **foliop, void **fsdata);
-+	int (*write_end)(const struct kiocb *, struct address_space *mapping,
- 				loff_t pos, unsigned len, unsigned copied,
-				struct page *page, void *fsdata);
-+				struct folio *folio, void *fsdata);
- 
- 	/* Unfortunately this kludge is needed for FIBMAP. Don't use it */
- 	sector_t (*bmap)(struct address_space *, sector_t);
-@@ -388,7 +434,7 @@ struct address_space_operations {
- 	bool (*is_partially_uptodate) (struct folio *, size_t from,
- 			size_t count);
- 	void (*is_dirty_writeback) (struct folio *, bool *dirty, bool *wb);
-	int (*error_remove_page)(struct address_space *, struct page *);
-+	int (*error_remove_folio)(struct address_space *, struct folio *);
- 
- 	/* swapfile support */
- 	int (*swap_activate)(struct swap_info_struct *sis, struct file *file,
-@@ -408,7 +454,7 @@ extern const struct address_space_operations empty_aops;
-  *   It is also used to block modification of page cache contents through
-  *   memory mappings.
-  * @gfp_mask: Memory allocation flags to use for allocating pages.
- * @i_mmap_writable: Number of VM_SHARED mappings.
-+ * @i_mmap_writable: Number of VM_SHARED, VM_MAYWRITE mappings.
-  * @nr_thps: Number of THPs in the pagecache (non-shmem only).
-  * @i_mmap: Tree of private and shared mappings.
-  * @i_mmap_rwsem: Protects @i_mmap and @i_mmap_writable.
-@@ -417,9 +463,9 @@ extern const struct address_space_operations empty_aops;
-  * @a_ops: Methods.
-  * @flags: Error bits and flags (AS_*).
-  * @wb_err: The most recent error which has occurred.
- * @private_lock: For use by the owner of the address_space.
- * @private_list: For use by the owner of the address_space.
- * @private_data: For use by the owner of the address_space.
-+ * @i_private_lock: For use by the owner of the address_space.
-+ * @i_private_list: For use by the owner of the address_space.
-+ * @i_private_data: For use by the owner of the address_space.
-  */
- struct address_space {
- 	struct inode		*host;
-@@ -432,20 +478,20 @@ struct address_space {
- 	atomic_t		nr_thps;
- #endif
- 	struct rb_root_cached	i_mmap;
-	struct rw_semaphore	i_mmap_rwsem;
- 	unsigned long		nrpages;
- 	pgoff_t			writeback_index;
- 	const struct address_space_operations *a_ops;
- 	unsigned long		flags;
- 	errseq_t		wb_err;
-	spinlock_t		private_lock;
-	struct list_head	private_list;
-	void			*private_data;
-+	spinlock_t		i_private_lock;
-+	struct list_head	i_private_list;
-+	struct rw_semaphore	i_mmap_rwsem;
-+	void *			i_private_data;
- } __attribute__((aligned(sizeof(long)))) __randomize_layout;
- 	/*
- 	 * On most architectures that alignment is already the case; but
- 	 * must be enforced here for CRIS, to let the least significant bit
-	 * of struct page's "mapping" pointer be used for PAGE_MAPPING_ANON.
-+	 * of struct folio's "mapping" pointer be used for FOLIO_MAPPING_ANON.
- 	 */
- 
- /* XArray tags, for tagging dirty and writeback pages in the pagecache. */
-@@ -456,7 +502,7 @@ struct address_space {
- /*
-  * Returns true if any of the pages in the mapping are marked with the tag.
-  */
-static inline bool mapping_tagged(struct address_space *mapping, xa_mark_t tag)
-+static inline bool mapping_tagged(const struct address_space *mapping, xa_mark_t tag)
- {
- 	return xa_marked(&mapping->i_pages, tag);
- }
-@@ -504,21 +550,21 @@ static inline void i_mmap_assert_write_locked(struct address_space *mapping)
- /*
-  * Might pages of this file be mapped into userspace?
-  */
-static inline int mapping_mapped(struct address_space *mapping)
-+static inline int mapping_mapped(const struct address_space *mapping)
- {
- 	return	!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root);
- }
- 
- /*
-  * Might pages of this file have been modified in userspace?
- * Note that i_mmap_writable counts all VM_SHARED vmas: do_mmap
-+ * Note that i_mmap_writable counts all VM_SHARED, VM_MAYWRITE vmas: do_mmap
-  * marks vma as VM_SHARED if it is shared, and the file was opened for
-  * writing i.e. vma may be mprotected writable even if now readonly.
-  *
-  * If i_mmap_writable is negative, no new writable mappings are allowed. You
-  * can only deny writable mappings, if none exists right now.
-  */
-static inline int mapping_writably_mapped(struct address_space *mapping)
-+static inline int mapping_writably_mapped(const struct address_space *mapping)
- {
- 	return atomic_read(&mapping->i_mmap_writable) > 0;
- }
-@@ -560,8 +606,8 @@ struct posix_acl;
- #define ACL_NOT_CACHED ((void *)(-1))
- /*
-  * ACL_DONT_CACHE is for stacked filesystems, that rely on underlying fs to
- * cache the ACL.  This also means that ->get_acl() can be called in RCU mode
- * with the LOOKUP_RCU flag.
-+ * cache the ACL.  This also means that ->get_inode_acl() can be called in RCU
-+ * mode with the LOOKUP_RCU flag.
-  */
- #define ACL_DONT_CACHE ((void *)(-3))
- 
-@@ -577,13 +623,139 @@ is_uncached_acl(struct posix_acl *acl)
- 	return (long)acl & 1;
- }
- 
-#define IOP_FASTPERM	0x0001
-#define IOP_LOOKUP	0x0002
-#define IOP_NOFOLLOW	0x0004
-#define IOP_XATTR	0x0008
-+#define IOP_FASTPERM		0x0001
-+#define IOP_LOOKUP		0x0002
-+#define IOP_NOFOLLOW		0x0004
-+#define IOP_XATTR		0x0008
- #define IOP_DEFAULT_READLINK	0x0010
-+#define IOP_MGTIME		0x0020
-+#define IOP_CACHED_LINK		0x0040
-+#define IOP_FASTPERM_MAY_EXEC	0x0080
- 
-struct fsnotify_mark_connector;
-+/*
-+ * Inode state bits.  Protected by inode->i_lock
-+ *
-+ * Four bits determine the dirty state of the inode: I_DIRTY_SYNC,
-+ * I_DIRTY_DATASYNC, I_DIRTY_PAGES, and I_DIRTY_TIME.
-+ *
-+ * Four bits define the lifetime of an inode.  Initially, inodes are I_NEW,
-+ * until that flag is cleared.  I_WILL_FREE, I_FREEING and I_CLEAR are set at
-+ * various stages of removing an inode.
-+ *
-+ * Two bits are used for locking and completion notification, I_NEW and I_SYNC.
-+ *
-+ * I_DIRTY_SYNC		Inode is dirty, but doesn't have to be written on
-+ *			fdatasync() (unless I_DIRTY_DATASYNC is also set).
-+ *			Timestamp updates are the usual cause.
-+ * I_DIRTY_DATASYNC	Data-related inode changes pending.  We keep track of
-+ *			these changes separately from I_DIRTY_SYNC so that we
-+ *			don't have to write inode on fdatasync() when only
-+ *			e.g. the timestamps have changed.
-+ * I_DIRTY_PAGES	Inode has dirty pages.  Inode itself may be clean.
-+ * I_DIRTY_TIME		The inode itself has dirty timestamps, and the
-+ *			lazytime mount option is enabled.  We keep track of this
-+ *			separately from I_DIRTY_SYNC in order to implement
-+ *			lazytime.  This gets cleared if I_DIRTY_INODE
-+ *			(I_DIRTY_SYNC and/or I_DIRTY_DATASYNC) gets set. But
-+ *			I_DIRTY_TIME can still be set if I_DIRTY_SYNC is already
-+ *			in place because writeback might already be in progress
-+ *			and we don't want to lose the time update
-+ * I_NEW		Serves as both a mutex and completion notification.
-+ *			New inodes set I_NEW.  If two processes both create
-+ *			the same inode, one of them will release its inode and
-+ *			wait for I_NEW to be released before returning.
-+ *			Inodes in I_WILL_FREE, I_FREEING or I_CLEAR state can
-+ *			also cause waiting on I_NEW, without I_NEW actually
-+ *			being set.  find_inode() uses this to prevent returning
-+ *			nearly-dead inodes.
-+ * I_WILL_FREE		Must be set when calling write_inode_now() if i_count
-+ *			is zero.  I_FREEING must be set when I_WILL_FREE is
-+ *			cleared.
-+ * I_FREEING		Set when inode is about to be freed but still has dirty
-+ *			pages or buffers attached or the inode itself is still
-+ *			dirty.
-+ * I_CLEAR		Added by clear_inode().  In this state the inode is
-+ *			clean and can be destroyed.  Inode keeps I_FREEING.
-+ *
-+ *			Inodes that are I_WILL_FREE, I_FREEING or I_CLEAR are
-+ *			prohibited for many purposes.  iget() must wait for
-+ *			the inode to be completely released, then create it
-+ *			anew.  Other functions will just ignore such inodes,
-+ *			if appropriate.  I_NEW is used for waiting.
-+ *
-+ * I_SYNC		Writeback of inode is running. The bit is set during
-+ *			data writeback, and cleared with a wakeup on the bit
-+ *			address once it is done. The bit is also used to pin
-+ *			the inode in memory for flusher thread.
-+ *
-+ * I_REFERENCED		Marks the inode as recently references on the LRU list.
-+ *
-+ * I_WB_SWITCH		Cgroup bdi_writeback switching in progress.  Used to
-+ *			synchronize competing switching instances and to tell
-+ *			wb stat updates to grab the i_pages lock.  See
-+ *			inode_switch_wbs_work_fn() for details.
-+ *
-+ * I_OVL_INUSE		Used by overlayfs to get exclusive ownership on upper
-+ *			and work dirs among overlayfs mounts.
-+ *
-+ * I_CREATING		New object's inode in the middle of setting up.
-+ *
-+ * I_DONTCACHE		Evict inode as soon as it is not used anymore.
-+ *
-+ * I_SYNC_QUEUED	Inode is queued in b_io or b_more_io writeback lists.
-+ *			Used to detect that mark_inode_dirty() should not move
-+ *			inode between dirty lists.
-+ *
-+ * I_PINNING_FSCACHE_WB	Inode is pinning an fscache object for writeback.
-+ *
-+ * I_LRU_ISOLATING	Inode is pinned being isolated from LRU without holding
-+ *			i_count.
-+ *
-+ * Q: What is the difference between I_WILL_FREE and I_FREEING?
-+ *
-+ * __I_{SYNC,NEW,LRU_ISOLATING} are used to derive unique addresses to wait
-+ * upon. There's one free address left.
-+ */
-+
-+enum inode_state_bits {
-+	__I_NEW			= 0U,
-+	__I_SYNC		= 1U,
-+	__I_LRU_ISOLATING	= 2U
-+	/* reserved wait address bit 3 */
-+};
-+
-+enum inode_state_flags_enum {
-+	I_NEW			= (1U << __I_NEW),
-+	I_SYNC			= (1U << __I_SYNC),
-+	I_LRU_ISOLATING         = (1U << __I_LRU_ISOLATING),
-+	/* reserved flag bit 3 */
-+	I_DIRTY_SYNC		= (1U << 4),
-+	I_DIRTY_DATASYNC	= (1U << 5),
-+	I_DIRTY_PAGES		= (1U << 6),
-+	I_WILL_FREE		= (1U << 7),
-+	I_FREEING		= (1U << 8),
-+	I_CLEAR			= (1U << 9),
-+	I_REFERENCED		= (1U << 10),
-+	I_LINKABLE		= (1U << 11),
-+	I_DIRTY_TIME		= (1U << 12),
-+	I_WB_SWITCH		= (1U << 13),
-+	I_OVL_INUSE		= (1U << 14),
-+	I_CREATING		= (1U << 15),
-+	I_DONTCACHE		= (1U << 16),
-+	I_SYNC_QUEUED		= (1U << 17),
-+	I_PINNING_NETFS_WB	= (1U << 18)
-+};
-+
-+#define I_DIRTY_INODE (I_DIRTY_SYNC | I_DIRTY_DATASYNC)
-+#define I_DIRTY (I_DIRTY_INODE | I_DIRTY_PAGES)
-+#define I_DIRTY_ALL (I_DIRTY | I_DIRTY_TIME)
-+
-+/*
-+ * Use inode_state_read() & friends to access.
-+ */
-+struct inode_state_flags {
-+	enum inode_state_flags_enum __state;
-+};
- 
- /*
-  * Keep mostly read-only and often accessed (especially for
-@@ -593,14 +765,13 @@ struct fsnotify_mark_connector;
- struct inode {
- 	umode_t			i_mode;
- 	unsigned short		i_opflags;
-	kuid_t			i_uid;
-	kgid_t			i_gid;
- 	unsigned int		i_flags;
-
- #ifdef CONFIG_FS_POSIX_ACL
- 	struct posix_acl	*i_acl;
- 	struct posix_acl	*i_default_acl;
- #endif
-+	kuid_t			i_uid;
-+	kgid_t			i_gid;
- 
- 	const struct inode_operations	*i_op;
- 	struct super_block	*i_sb;
-@@ -625,13 +796,17 @@ struct inode {
- 	};
- 	dev_t			i_rdev;
- 	loff_t			i_size;
-	struct timespec64	i_atime;
-	struct timespec64	i_mtime;
-	struct timespec64	i_ctime;
-+	time64_t		i_atime_sec;
-+	time64_t		i_mtime_sec;
-+	time64_t		i_ctime_sec;
-+	u32			i_atime_nsec;
-+	u32			i_mtime_nsec;
-+	u32			i_ctime_nsec;
-+	u32			i_generation;
- 	spinlock_t		i_lock;	/* i_blocks, i_bytes, maybe i_size */
- 	unsigned short          i_bytes;
- 	u8			i_blkbits;
-	u8			i_write_hint;
-+	enum rw_hint		i_write_hint;
- 	blkcnt_t		i_blocks;
- 
- #ifdef __NEED_I_SIZE_ORDERED
-@@ -639,7 +814,8 @@ struct inode {
- #endif
- 
- 	/* Misc */
-	unsigned long		i_state;
-+	struct inode_state_flags i_state;
-+	/* 32-bit hole */
- 	struct rw_semaphore	i_rwsem;
- 
- 	unsigned long		dirtied_when;	/* jiffies of first dirtying */
-@@ -676,7 +852,10 @@ struct inode {
- 	};
- 	struct file_lock_context	*i_flctx;
- 	struct address_space	i_data;
-	struct list_head	i_devices;
-+	union {
-+		struct list_head	i_devices;
-+		int			i_linklen;
-+	};
- 	union {
- 		struct pipe_inode_info	*i_pipe;
- 		struct cdev		*i_cdev;
-@@ -684,24 +863,114 @@ struct inode {
- 		unsigned		i_dir_seq;
- 	};
- 
-	__u32			i_generation;
- 
- #ifdef CONFIG_FSNOTIFY
- 	__u32			i_fsnotify_mask; /* all events this inode cares about */
-+	/* 32-bit hole reserved for expanding i_fsnotify_mask */
- 	struct fsnotify_mark_connector __rcu	*i_fsnotify_marks;
- #endif
- 
-#ifdef CONFIG_FS_ENCRYPTION
-	struct fscrypt_info	*i_crypt_info;
-#endif
-
-#ifdef CONFIG_FS_VERITY
-	struct fsverity_info	*i_verity_info;
-#endif
-
- 	void			*i_private; /* fs or device private pointer */
- } __randomize_layout;
- 
-+/*
-+ * i_state handling
-+ *
-+ * We hide all of it behind helpers so that we can validate consumers.
-+ */
-+static inline enum inode_state_flags_enum inode_state_read_once(struct inode *inode)
-+{
-+	return READ_ONCE(inode->i_state.__state);
-+}
-+
-+static inline enum inode_state_flags_enum inode_state_read(struct inode *inode)
-+{
-+	lockdep_assert_held(&inode->i_lock);
-+	return inode->i_state.__state;
-+}
-+
-+static inline void inode_state_set_raw(struct inode *inode,
-+				       enum inode_state_flags_enum flags)
-+{
-+	WRITE_ONCE(inode->i_state.__state, inode->i_state.__state | flags);
-+}
-+
-+static inline void inode_state_set(struct inode *inode,
-+				   enum inode_state_flags_enum flags)
-+{
-+	lockdep_assert_held(&inode->i_lock);
-+	inode_state_set_raw(inode, flags);
-+}
-+
-+static inline void inode_state_clear_raw(struct inode *inode,
-+					 enum inode_state_flags_enum flags)
-+{
-+	WRITE_ONCE(inode->i_state.__state, inode->i_state.__state & ~flags);
-+}
-+
-+static inline void inode_state_clear(struct inode *inode,
-+				     enum inode_state_flags_enum flags)
-+{
-+	lockdep_assert_held(&inode->i_lock);
-+	inode_state_clear_raw(inode, flags);
-+}
-+
-+static inline void inode_state_assign_raw(struct inode *inode,
-+					  enum inode_state_flags_enum flags)
-+{
-+	WRITE_ONCE(inode->i_state.__state, flags);
-+}
-+
-+static inline void inode_state_assign(struct inode *inode,
-+				      enum inode_state_flags_enum flags)
-+{
-+	lockdep_assert_held(&inode->i_lock);
-+	inode_state_assign_raw(inode, flags);
-+}
-+
-+static inline void inode_state_replace_raw(struct inode *inode,
-+					   enum inode_state_flags_enum clearflags,
-+					   enum inode_state_flags_enum setflags)
-+{
-+	enum inode_state_flags_enum flags;
-+	flags = inode->i_state.__state;
-+	flags &= ~clearflags;
-+	flags |= setflags;
-+	inode_state_assign_raw(inode, flags);
-+}
-+
-+static inline void inode_state_replace(struct inode *inode,
-+				       enum inode_state_flags_enum clearflags,
-+				       enum inode_state_flags_enum setflags)
-+{
-+	lockdep_assert_held(&inode->i_lock);
-+	inode_state_replace_raw(inode, clearflags, setflags);
-+}
-+
-+static inline void inode_set_cached_link(struct inode *inode, char *link, int linklen)
-+{
-+	VFS_WARN_ON_INODE(strlen(link) != linklen, inode);
-+	VFS_WARN_ON_INODE(inode->i_opflags & IOP_CACHED_LINK, inode);
-+	inode->i_link = link;
-+	inode->i_linklen = linklen;
-+	inode->i_opflags |= IOP_CACHED_LINK;
-+}
-+
-+/*
-+ * Get bit address from inode->i_state to use with wait_var_event()
-+ * infrastructre.
-+ */
-+#define inode_state_wait_address(inode, bit) ((char *)&(inode)->i_state + (bit))
-+
-+struct wait_queue_head *inode_bit_waitqueue(struct wait_bit_queue_entry *wqe,
-+					    struct inode *inode, u32 bit);
-+
-+static inline void inode_wake_up_bit(struct inode *inode, u32 bit)
-+{
-+	/* Caller is responsible for correct memory barriers. */
-+	wake_up_var(inode_state_wait_address(inode, bit));
-+}
-+
- struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode);
- 
- static inline unsigned int i_blocksize(const struct inode *node)
-@@ -725,8 +994,10 @@ static inline void inode_fake_hash(struct inode *inode)
- 	hlist_add_fake(&inode->i_hash);
- }
- 
-+void wait_on_new_inode(struct inode *inode);
-+
- /*
- * inode->i_mutex nesting subclasses for the lock validator:
-+ * inode->i_rwsem nesting subclasses for the lock validator:
-  *
-  * 0: the object of the current VFS operation
-  * 1: parent
-@@ -756,6 +1027,11 @@ static inline void inode_lock(struct inode *inode)
- 	down_write(&inode->i_rwsem);
- }
- 
-+static inline __must_check int inode_lock_killable(struct inode *inode)
-+{
-+	return down_write_killable(&inode->i_rwsem);
-+}
-+
- static inline void inode_unlock(struct inode *inode)
- {
- 	up_write(&inode->i_rwsem);
-@@ -766,6 +1042,11 @@ static inline void inode_lock_shared(struct inode *inode)
- 	down_read(&inode->i_rwsem);
- }
- 
-+static inline __must_check int inode_lock_shared_killable(struct inode *inode)
-+{
-+	return down_read_killable(&inode->i_rwsem);
-+}
-+
- static inline void inode_unlock_shared(struct inode *inode)
- {
- 	up_read(&inode->i_rwsem);
-@@ -861,13 +1142,14 @@ static inline loff_t i_size_read(const struct inode *inode)
- 	preempt_enable();
- 	return i_size;
- #else
-	return inode->i_size;
-+	/* Pairs with smp_store_release() in i_size_write() */
-+	return smp_load_acquire(&inode->i_size);
- #endif
- }
- 
- /*
-  * NOTE: unlike i_size_read(), i_size_write() does need locking around it
- * (normally i_mutex), otherwise on 32bit/SMP an update of i_size_seqcount
-+ * (normally i_rwsem), otherwise on 32bit/SMP an update of i_size_seqcount
-  * can be lost, resulting in subsequent i_size_read() calls spinning forever.
-  */
- static inline void i_size_write(struct inode *inode, loff_t i_size)
-@@ -883,7 +1165,12 @@ static inline void i_size_write(struct inode *inode, loff_t i_size)
- 	inode->i_size = i_size;
- 	preempt_enable();
- #else
-	inode->i_size = i_size;
-+	/*
-+	 * Pairs with smp_load_acquire() in i_size_read() to ensure
-+	 * changes related to inode size (such as page contents) are
-+	 * visible before we see the changed inode size.
-+	 */
-+	smp_store_release(&inode->i_size, i_size);
- #endif
- }
- 
-@@ -898,6 +1185,7 @@ static inline unsigned imajor(const struct inode *inode)
- }
- 
- struct fown_struct {
-+	struct file *file;	/* backpointer for security modules */
- 	rwlock_t lock;          /* protects pid, uid, euid fields */
- 	struct pid *pid;	/* pid or -pgrp where SIGIO should be sent */
- 	enum pid_type pid_type;	/* Kind of process group SIGIO should be sent to */
-@@ -913,6 +1201,7 @@ struct fown_struct {
-  *      and so were/are genuinely "ahead".  Start next readahead when
-  *      the first of these pages is accessed.
-  * @ra_pages: Maximum size of a readahead request, copied from the bdi.
-+ * @order: Preferred folio order used for most recent readahead.
-  * @mmap_miss: How many mmap accesses missed in the page cache.
-  * @prev_pos: The last byte in the most recent read request.
-  *
-@@ -924,7 +1213,8 @@ struct file_ra_state {
- 	unsigned int size;
- 	unsigned int async_size;
- 	unsigned int ra_pages;
-	unsigned int mmap_miss;
-+	unsigned short order;
-+	unsigned short mmap_miss;
- 	loff_t prev_pos;
- };
- 
-@@ -937,44 +1227,74 @@ static inline int ra_has_index(struct file_ra_state *ra, pgoff_t index)
- 		index <  ra->start + ra->size);
- }
- 
-+/**
-+ * struct file - Represents a file
-+ * @f_lock: Protects f_ep, f_flags. Must not be taken from IRQ context.
-+ * @f_mode: FMODE_* flags often used in hotpaths
-+ * @f_op: file operations
-+ * @f_mapping: Contents of a cacheable, mappable object.
-+ * @private_data: filesystem or driver specific data
-+ * @f_inode: cached inode
-+ * @f_flags: file flags
-+ * @f_iocb_flags: iocb flags
-+ * @f_cred: stashed credentials of creator/opener
-+ * @f_owner: file owner
-+ * @f_path: path of the file
-+ * @__f_path: writable alias for @f_path; *ONLY* for core VFS and only before
-+ *   the file gets open
-+ * @f_pos_lock: lock protecting file position
-+ * @f_pipe: specific to pipes
-+ * @f_pos: file position
-+ * @f_security: LSM security context of this file
-+ * @f_wb_err: writeback error
-+ * @f_sb_err: per sb writeback errors
-+ * @f_ep: link of all epoll hooks for this file
-+ * @f_task_work: task work entry point
-+ * @f_llist: work queue entrypoint
-+ * @f_ra: file's readahead state
-+ * @f_freeptr: Pointer used by SLAB_TYPESAFE_BY_RCU file cache (don't touch.)
-+ * @f_ref: reference count
-+ */
- struct file {
-+	spinlock_t			f_lock;
-+	fmode_t				f_mode;
-+	const struct file_operations	*f_op;
-+	struct address_space		*f_mapping;
-+	void				*private_data;
-+	struct inode			*f_inode;
-+	unsigned int			f_flags;
-+	unsigned int			f_iocb_flags;
-+	const struct cred		*f_cred;
-+	struct fown_struct		*f_owner;
-+	/* --- cacheline 1 boundary (64 bytes) --- */
- 	union {
-		struct llist_node	f_llist;
-		struct rcu_head 	f_rcuhead;
-		unsigned int 		f_iocb_flags;
-+		const struct path	f_path;
-+		struct path		__f_path;
- 	};
-	struct path		f_path;
-	struct inode		*f_inode;	/* cached value */
-	const struct file_operations	*f_op;
-
-	/*
-	 * Protects f_ep, f_flags.
-	 * Must not be taken from IRQ context.
-	 */
-	spinlock_t		f_lock;
-	atomic_long_t		f_count;
-	unsigned int 		f_flags;
-	fmode_t			f_mode;
-	struct mutex		f_pos_lock;
-	loff_t			f_pos;
-	struct fown_struct	f_owner;
-	const struct cred	*f_cred;
-	struct file_ra_state	f_ra;
-
-	u64			f_version;
-+	union {
-+		/* regular files (with FMODE_ATOMIC_POS) and directories */
-+		struct mutex		f_pos_lock;
-+		/* pipes */
-+		u64			f_pipe;
-+	};
-+	loff_t				f_pos;
- #ifdef CONFIG_SECURITY
-	void			*f_security;
-+	void				*f_security;
- #endif
-	/* needed for tty driver, and maybe others */
-	void			*private_data;
-
-+	/* --- cacheline 2 boundary (128 bytes) --- */
-+	errseq_t			f_wb_err;
-+	errseq_t			f_sb_err;
- #ifdef CONFIG_EPOLL
-	/* Used by fs/eventpoll.c to link all the hooks to this file */
-	struct hlist_head	*f_ep;
-#endif /* #ifdef CONFIG_EPOLL */
-	struct address_space	*f_mapping;
-	errseq_t		f_wb_err;
-	errseq_t		f_sb_err; /* for syncfs */
-+	struct hlist_head		*f_ep;
-+#endif
-+	union {
-+		struct callback_head	f_task_work;
-+		struct llist_node	f_llist;
-+		struct file_ra_state	f_ra;
-+		freeptr_t		f_freeptr;
-+	};
-+	file_ref_t			f_ref;
-+	/* --- cacheline 3 boundary (192 bytes) --- */
- } __randomize_layout
-   __attribute__((aligned(4)));	/* lest something weird decides that 2 is OK */
- 
-@@ -982,16 +1302,19 @@ struct file_handle {
- 	__u32 handle_bytes;
- 	int handle_type;
- 	/* file identifier */
-	unsigned char f_handle[];
-+	unsigned char f_handle[] __counted_by(handle_bytes);
- };
- 
- static inline struct file *get_file(struct file *f)
- {
-	atomic_long_inc(&f->f_count);
-+	file_ref_inc(&f->f_ref);
- 	return f;
- }
-#define get_file_rcu(x) atomic_long_inc_not_zero(&(x)->f_count)
-#define file_count(x)	atomic_long_read(&(x)->f_count)
-+
-+struct file *get_file_rcu(struct file __rcu **f);
-+struct file *get_file_active(struct file **f);
-+
-+#define file_count(f)	file_ref_read(&(f)->f_ref)
- 
- #define	MAX_NON_LFS	((1UL<<31) - 1)
- 
-@@ -1003,344 +1326,45 @@ static inline struct file *get_file(struct file *f)
- #define MAX_LFS_FILESIZE 	((loff_t)LLONG_MAX)
- #endif
- 
-#define FL_POSIX	1
-#define FL_FLOCK	2
-#define FL_DELEG	4	/* NFSv4 delegation */
-#define FL_ACCESS	8	/* not trying to lock, just looking */
-#define FL_EXISTS	16	/* when unlocking, test for existence */
-#define FL_LEASE	32	/* lease held on this file */
-#define FL_CLOSE	64	/* unlock on close */
-#define FL_SLEEP	128	/* A blocking lock */
-#define FL_DOWNGRADE_PENDING	256 /* Lease is being downgraded */
-#define FL_UNLOCK_PENDING	512 /* Lease is being broken */
-#define FL_OFDLCK	1024	/* lock is "owned" by struct file */
-#define FL_LAYOUT	2048	/* outstanding pNFS layout */
-#define FL_RECLAIM	4096	/* reclaiming from a reboot server */
-
-#define FL_CLOSE_POSIX (FL_POSIX | FL_CLOSE)
-
-/*
- * Special return value from posix_lock_file() and vfs_lock_file() for
- * asynchronous locking.
- */
-#define FILE_LOCK_DEFERRED 1
-
- /* legacy typedef, should eventually be removed */
- typedef void *fl_owner_t;
- 
- struct file_lock;
-
-struct file_lock_operations {
-	void (*fl_copy_lock)(struct file_lock *, struct file_lock *);
-	void (*fl_release_private)(struct file_lock *);
-};
-
-struct lock_manager_operations {
-	void *lm_mod_owner;
-	fl_owner_t (*lm_get_owner)(fl_owner_t);
-	void (*lm_put_owner)(fl_owner_t);
-	void (*lm_notify)(struct file_lock *);	/* unblock callback */
-	int (*lm_grant)(struct file_lock *, int);
-	bool (*lm_break)(struct file_lock *);
-	int (*lm_change)(struct file_lock *, int, struct list_head *);
-	void (*lm_setup)(struct file_lock *, void **);
-	bool (*lm_breaker_owns_lease)(struct file_lock *);
-	bool (*lm_lock_expirable)(struct file_lock *cfl);
-	void (*lm_expire_lock)(void);
-};
-
-struct lock_manager {
-	struct list_head list;
-	/*
-	 * NFSv4 and up also want opens blocked during the grace period;
-	 * NLM doesn't care:
-	 */
-	bool block_opens;
-};
-
-struct net;
-void locks_start_grace(struct net *, struct lock_manager *);
-void locks_end_grace(struct lock_manager *);
-bool locks_in_grace(struct net *);
-bool opens_in_grace(struct net *);
-
-/* that will die - we need it for nfs_lock_info */
-#include <linux/nfs_fs_i.h>
-
-/*
- * struct file_lock represents a generic "file lock". It's used to represent
- * POSIX byte range locks, BSD (flock) locks, and leases. It's important to
- * note that the same struct is used to represent both a request for a lock and
- * the lock itself, but the same object is never used for both.
- *
- * FIXME: should we create a separate "struct lock_request" to help distinguish
- * these two uses?
- *
- * The varous i_flctx lists are ordered by:
- *
- * 1) lock owner
- * 2) lock range start
- * 3) lock range end
- *
- * Obviously, the last two criteria only matter for POSIX locks.
- */
-struct file_lock {
-	struct file_lock *fl_blocker;	/* The lock, that is blocking us */
-	struct list_head fl_list;	/* link into file_lock_context */
-	struct hlist_node fl_link;	/* node in global lists */
-	struct list_head fl_blocked_requests;	/* list of requests with
-						 * ->fl_blocker pointing here
-						 */
-	struct list_head fl_blocked_member;	/* node in
-						 * ->fl_blocker->fl_blocked_requests
-						 */
-	fl_owner_t fl_owner;
-	unsigned int fl_flags;
-	unsigned char fl_type;
-	unsigned int fl_pid;
-	int fl_link_cpu;		/* what cpu's list is this on? */
-	wait_queue_head_t fl_wait;
-	struct file *fl_file;
-	loff_t fl_start;
-	loff_t fl_end;
-
-	struct fasync_struct *	fl_fasync; /* for lease break notifications */
-	/* for lease breaks: */
-	unsigned long fl_break_time;
-	unsigned long fl_downgrade_time;
-
-	const struct file_lock_operations *fl_ops;	/* Callbacks for filesystems */
-	const struct lock_manager_operations *fl_lmops;	/* Callbacks for lockmanagers */
-	union {
-		struct nfs_lock_info	nfs_fl;
-		struct nfs4_lock_info	nfs4_fl;
-		struct {
-			struct list_head link;	/* link in AFS vnode's pending_locks list */
-			int state;		/* state of grant or error if -ve */
-			unsigned int	debug_id;
-		} afs;
-	} fl_u;
-} __randomize_layout;
-
-struct file_lock_context {
-	spinlock_t		flc_lock;
-	struct list_head	flc_flock;
-	struct list_head	flc_posix;
-	struct list_head	flc_lease;
-};
-+struct file_lease;
- 
- /* The following constant reflects the upper bound of the file/locking space */
- #ifndef OFFSET_MAX
-#define INT_LIMIT(x)	(~((x)1 << (sizeof(x)*8 - 1)))
-#define OFFSET_MAX	INT_LIMIT(loff_t)
-#define OFFT_OFFSET_MAX	INT_LIMIT(off_t)
-#endif
-
-extern void send_sigio(struct fown_struct *fown, int fd, int band);
-
-#define locks_inode(f) file_inode(f)
-
-#ifdef CONFIG_FILE_LOCKING
-extern int fcntl_getlk(struct file *, unsigned int, struct flock *);
-extern int fcntl_setlk(unsigned int, struct file *, unsigned int,
-			struct flock *);
-
-#if BITS_PER_LONG == 32
-extern int fcntl_getlk64(struct file *, unsigned int, struct flock64 *);
-extern int fcntl_setlk64(unsigned int, struct file *, unsigned int,
-			struct flock64 *);
-#endif
-
-extern int fcntl_setlease(unsigned int fd, struct file *filp, long arg);
-extern int fcntl_getlease(struct file *filp);
-
-/* fs/locks.c */
-void locks_free_lock_context(struct inode *inode);
-void locks_free_lock(struct file_lock *fl);
-extern void locks_init_lock(struct file_lock *);
-extern struct file_lock * locks_alloc_lock(void);
-extern void locks_copy_lock(struct file_lock *, struct file_lock *);
-extern void locks_copy_conflock(struct file_lock *, struct file_lock *);
-extern void locks_remove_posix(struct file *, fl_owner_t);
-extern void locks_remove_file(struct file *);
-extern void locks_release_private(struct file_lock *);
-extern void posix_test_lock(struct file *, struct file_lock *);
-extern int posix_lock_file(struct file *, struct file_lock *, struct file_lock *);
-extern int locks_delete_block(struct file_lock *);
-extern int vfs_test_lock(struct file *, struct file_lock *);
-extern int vfs_lock_file(struct file *, unsigned int, struct file_lock *, struct file_lock *);
-extern int vfs_cancel_lock(struct file *filp, struct file_lock *fl);
-extern int locks_lock_inode_wait(struct inode *inode, struct file_lock *fl);
-extern int __break_lease(struct inode *inode, unsigned int flags, unsigned int type);
-extern void lease_get_mtime(struct inode *, struct timespec64 *time);
-extern int generic_setlease(struct file *, long, struct file_lock **, void **priv);
-extern int vfs_setlease(struct file *, long, struct file_lock **, void **);
-extern int lease_modify(struct file_lock *, int, struct list_head *);
-
-struct notifier_block;
-extern int lease_register_notifier(struct notifier_block *);
-extern void lease_unregister_notifier(struct notifier_block *);
-
-struct files_struct;
-extern void show_fd_locks(struct seq_file *f,
-			 struct file *filp, struct files_struct *files);
-extern bool locks_owner_has_blockers(struct file_lock_context *flctx,
-			fl_owner_t owner);
-#else /* !CONFIG_FILE_LOCKING */
-static inline int fcntl_getlk(struct file *file, unsigned int cmd,
-			      struct flock __user *user)
-{
-	return -EINVAL;
-}
-
-static inline int fcntl_setlk(unsigned int fd, struct file *file,
-			      unsigned int cmd, struct flock __user *user)
-{
-	return -EACCES;
-}
-
-#if BITS_PER_LONG == 32
-static inline int fcntl_getlk64(struct file *file, unsigned int cmd,
-				struct flock64 *user)
-{
-	return -EINVAL;
-}
-
-static inline int fcntl_setlk64(unsigned int fd, struct file *file,
-				unsigned int cmd, struct flock64 *user)
-{
-	return -EACCES;
-}
-+#define OFFSET_MAX	type_max(loff_t)
-+#define OFFT_OFFSET_MAX	type_max(off_t)
- #endif
-static inline int fcntl_setlease(unsigned int fd, struct file *filp, long arg)
-{
-	return -EINVAL;
-}
-
-static inline int fcntl_getlease(struct file *filp)
-{
-	return F_UNLCK;
-}
-
-static inline void
-locks_free_lock_context(struct inode *inode)
-{
-}
-
-static inline void locks_init_lock(struct file_lock *fl)
-{
-	return;
-}
-
-static inline void locks_copy_conflock(struct file_lock *new, struct file_lock *fl)
-{
-	return;
-}
-
-static inline void locks_copy_lock(struct file_lock *new, struct file_lock *fl)
-{
-	return;
-}
-
-static inline void locks_remove_posix(struct file *filp, fl_owner_t owner)
-{
-	return;
-}
-
-static inline void locks_remove_file(struct file *filp)
-{
-	return;
-}
-
-static inline void posix_test_lock(struct file *filp, struct file_lock *fl)
-{
-	return;
-}
-
-static inline int posix_lock_file(struct file *filp, struct file_lock *fl,
-				  struct file_lock *conflock)
-{
-	return -ENOLCK;
-}
-
-static inline int locks_delete_block(struct file_lock *waiter)
-{
-	return -ENOENT;
-}
-
-static inline int vfs_test_lock(struct file *filp, struct file_lock *fl)
-{
-	return 0;
-}
-
-static inline int vfs_lock_file(struct file *filp, unsigned int cmd,
-				struct file_lock *fl, struct file_lock *conf)
-{
-	return -ENOLCK;
-}
- 
-static inline int vfs_cancel_lock(struct file *filp, struct file_lock *fl)
-+int file_f_owner_allocate(struct file *file);
-+static inline struct fown_struct *file_f_owner(const struct file *file)
- {
-	return 0;
-}
-
-static inline int locks_lock_inode_wait(struct inode *inode, struct file_lock *fl)
-{
-	return -ENOLCK;
-}
-
-static inline int __break_lease(struct inode *inode, unsigned int mode, unsigned int type)
-{
-	return 0;
-}
-
-static inline void lease_get_mtime(struct inode *inode,
-				   struct timespec64 *time)
-{
-	return;
-}
-
-static inline int generic_setlease(struct file *filp, long arg,
-				    struct file_lock **flp, void **priv)
-{
-	return -EINVAL;
-}
-
-static inline int vfs_setlease(struct file *filp, long arg,
-			       struct file_lock **lease, void **priv)
-{
-	return -EINVAL;
-}
-
-static inline int lease_modify(struct file_lock *fl, int arg,
-			       struct list_head *dispose)
-{
-	return -EINVAL;
-+	return READ_ONCE(file->f_owner);
- }
- 
-struct files_struct;
-static inline void show_fd_locks(struct seq_file *f,
-			struct file *filp, struct files_struct *files) {}
-static inline bool locks_owner_has_blockers(struct file_lock_context *flctx,
-			fl_owner_t owner)
-{
-	return false;
-}
-#endif /* !CONFIG_FILE_LOCKING */
-+extern void send_sigio(struct fown_struct *fown, int fd, int band);
- 
- static inline struct inode *file_inode(const struct file *f)
- {
- 	return f->f_inode;
- }
- 
-+/*
-+ * file_dentry() is a relic from the days that overlayfs was using files with a
-+ * "fake" path, meaning, f_path on overlayfs and f_inode on underlying fs.
-+ * In those days, file_dentry() was needed to get the underlying fs dentry that
-+ * matches f_inode.
-+ * Files with "fake" path should not exist nowadays, so use an assertion to make
-+ * sure that file_dentry() was not papering over filesystem bugs.
-+ */
- static inline struct dentry *file_dentry(const struct file *file)
- {
-	return d_real(file->f_path.dentry, file_inode(file));
-}
-+	struct dentry *dentry = file->f_path.dentry;
- 
-static inline int locks_lock_file_wait(struct file *filp, struct file_lock *fl)
-{
-	return locks_lock_inode_wait(locks_inode(filp), fl);
-+	WARN_ON_ONCE(d_inode(dentry) != file_inode(file));
-+	return dentry;
- }
- 
- struct fasync_struct {
-@@ -1365,44 +1389,10 @@ extern void fasync_free(struct fasync_struct *);
- extern void kill_fasync(struct fasync_struct **, int, int);
- 
- extern void __f_setown(struct file *filp, struct pid *, enum pid_type, int force);
-extern int f_setown(struct file *filp, unsigned long arg, int force);
-+extern int f_setown(struct file *filp, int who, int force);
- extern void f_delown(struct file *filp);
- extern pid_t f_getown(struct file *filp);
-extern int send_sigurg(struct fown_struct *fown);
-
-/*
- * sb->s_flags.  Note that these mirror the equivalent MS_* flags where
- * represented in both.
- */
-#define SB_RDONLY	 1	/* Mount read-only */
-#define SB_NOSUID	 2	/* Ignore suid and sgid bits */
-#define SB_NODEV	 4	/* Disallow access to device special files */
-#define SB_NOEXEC	 8	/* Disallow program execution */
-#define SB_SYNCHRONOUS	16	/* Writes are synced at once */
-#define SB_MANDLOCK	64	/* Allow mandatory locks on an FS */
-#define SB_DIRSYNC	128	/* Directory modifications are synchronous */
-#define SB_NOATIME	1024	/* Do not update access times. */
-#define SB_NODIRATIME	2048	/* Do not update directory access times */
-#define SB_SILENT	32768
-#define SB_POSIXACL	(1<<16)	/* VFS does not apply the umask */
-#define SB_INLINECRYPT	(1<<17)	/* Use blk-crypto for encrypted files */
-#define SB_KERNMOUNT	(1<<22) /* this is a kern_mount call */
-#define SB_I_VERSION	(1<<23) /* Update inode I_version field */
-#define SB_LAZYTIME	(1<<25) /* Update the on-disk [acm]times lazily */
-
-/* These sb flags are internal to the kernel */
-#define SB_SUBMOUNT     (1<<26)
-#define SB_FORCE    	(1<<27)
-#define SB_NOSEC	(1<<28)
-#define SB_BORN		(1<<29)
-#define SB_ACTIVE	(1<<30)
-#define SB_NOUSER	(1<<31)
-
-/* These flags relate to encoding and casefolding */
-#define SB_ENC_STRICT_MODE_FL	(1 << 0)
-
-#define sb_has_strict_encoding(sb) \
-	(sb->s_encoding_flags & SB_ENC_STRICT_MODE_FL)
-+extern int send_sigurg(struct file *file);
- 
- /*
-  *	Umount options
-@@ -1414,174 +1404,6 @@ extern int send_sigurg(struct fown_struct *fown);
- #define UMOUNT_NOFOLLOW	0x00000008	/* Don't follow symlink on umount */
- #define UMOUNT_UNUSED	0x80000000	/* Flag guaranteed to be unused */
- 
-/* sb->s_iflags */
-#define SB_I_CGROUPWB	0x00000001	/* cgroup-aware writeback enabled */
-#define SB_I_NOEXEC	0x00000002	/* Ignore executables on this fs */
-#define SB_I_NODEV	0x00000004	/* Ignore devices on this fs */
-#define SB_I_STABLE_WRITES 0x00000008	/* don't modify blks until WB is done */
-
-/* sb->s_iflags to limit user namespace mounts */
-#define SB_I_USERNS_VISIBLE		0x00000010 /* fstype already mounted */
-#define SB_I_IMA_UNVERIFIABLE_SIGNATURE	0x00000020
-#define SB_I_UNTRUSTED_MOUNTER		0x00000040
-
-#define SB_I_SKIP_SYNC	0x00000100	/* Skip superblock at global sync */
-#define SB_I_PERSB_BDI	0x00000200	/* has a per-sb bdi */
-#define SB_I_TS_EXPIRY_WARNED 0x00000400 /* warned about timestamp range expiry */
-#define SB_I_RETIRED	0x00000800	/* superblock shouldn't be reused */
-
-/* Possible states of 'frozen' field */
-enum {
-	SB_UNFROZEN = 0,		/* FS is unfrozen */
-	SB_FREEZE_WRITE	= 1,		/* Writes, dir ops, ioctls frozen */
-	SB_FREEZE_PAGEFAULT = 2,	/* Page faults stopped as well */
-	SB_FREEZE_FS = 3,		/* For internal FS use (e.g. to stop
-					 * internal threads if needed) */
-	SB_FREEZE_COMPLETE = 4,		/* ->freeze_fs finished successfully */
-};
-
-#define SB_FREEZE_LEVELS (SB_FREEZE_COMPLETE - 1)
-
-struct sb_writers {
-	int				frozen;		/* Is sb frozen? */
-	wait_queue_head_t		wait_unfrozen;	/* wait for thaw */
-	struct percpu_rw_semaphore	rw_sem[SB_FREEZE_LEVELS];
-};
-
-struct super_block {
-	struct list_head	s_list;		/* Keep this first */
-	dev_t			s_dev;		/* search index; _not_ kdev_t */
-	unsigned char		s_blocksize_bits;
-	unsigned long		s_blocksize;
-	loff_t			s_maxbytes;	/* Max file size */
-	struct file_system_type	*s_type;
-	const struct super_operations	*s_op;
-	const struct dquot_operations	*dq_op;
-	const struct quotactl_ops	*s_qcop;
-	const struct export_operations *s_export_op;
-	unsigned long		s_flags;
-	unsigned long		s_iflags;	/* internal SB_I_* flags */
-	unsigned long		s_magic;
-	struct dentry		*s_root;
-	struct rw_semaphore	s_umount;
-	int			s_count;
-	atomic_t		s_active;
-#ifdef CONFIG_SECURITY
-	void                    *s_security;
-#endif
-	const struct xattr_handler **s_xattr;
-#ifdef CONFIG_FS_ENCRYPTION
-	const struct fscrypt_operations	*s_cop;
-	struct key		*s_master_keys; /* master crypto keys in use */
-#endif
-#ifdef CONFIG_FS_VERITY
-	const struct fsverity_operations *s_vop;
-#endif
-#if IS_ENABLED(CONFIG_UNICODE)
-	struct unicode_map *s_encoding;
-	__u16 s_encoding_flags;
-#endif
-	struct hlist_bl_head	s_roots;	/* alternate root dentries for NFS */
-	struct list_head	s_mounts;	/* list of mounts; _not_ for fs use */
-	struct block_device	*s_bdev;
-	struct backing_dev_info *s_bdi;
-	struct mtd_info		*s_mtd;
-	struct hlist_node	s_instances;
-	unsigned int		s_quota_types;	/* Bitmask of supported quota types */
-	struct quota_info	s_dquot;	/* Diskquota specific options */
-
-	struct sb_writers	s_writers;
-
-	/*
-	 * Keep s_fs_info, s_time_gran, s_fsnotify_mask, and
-	 * s_fsnotify_marks together for cache efficiency. They are frequently
-	 * accessed and rarely modified.
-	 */
-	void			*s_fs_info;	/* Filesystem private info */
-
-	/* Granularity of c/m/atime in ns (cannot be worse than a second) */
-	u32			s_time_gran;
-	/* Time limits for c/m/atime in seconds */
-	time64_t		   s_time_min;
-	time64_t		   s_time_max;
-#ifdef CONFIG_FSNOTIFY
-	__u32			s_fsnotify_mask;
-	struct fsnotify_mark_connector __rcu	*s_fsnotify_marks;
-#endif
-
-	char			s_id[32];	/* Informational name */
-	uuid_t			s_uuid;		/* UUID */
-
-	unsigned int		s_max_links;
-	fmode_t			s_mode;
-
-	/*
-	 * The next field is for VFS *only*. No filesystems have any business
-	 * even looking at it. You had been warned.
-	 */
-	struct mutex s_vfs_rename_mutex;	/* Kludge */
-
-	/*
-	 * Filesystem subtype.  If non-empty the filesystem type field
-	 * in /proc/mounts will be "type.subtype"
-	 */
-	const char *s_subtype;
-
-	const struct dentry_operations *s_d_op; /* default d_op for dentries */
-
-	struct shrinker s_shrink;	/* per-sb shrinker handle */
-
-	/* Number of inodes with nlink == 0 but still referenced */
-	atomic_long_t s_remove_count;
-
-	/*
-	 * Number of inode/mount/sb objects that are being watched, note that
-	 * inodes objects are currently double-accounted.
-	 */
-	atomic_long_t s_fsnotify_connectors;
-
-	/* Being remounted read-only */
-	int s_readonly_remount;
-
-	/* per-sb errseq_t for reporting writeback errors via syncfs */
-	errseq_t s_wb_err;
-
-	/* AIO completions deferred from interrupt context */
-	struct workqueue_struct *s_dio_done_wq;
-	struct hlist_head s_pins;
-
-	/*
-	 * Owning user namespace and default context in which to
-	 * interpret filesystem uids, gids, quotas, device nodes,
-	 * xattrs and security labels.
-	 */
-	struct user_namespace *s_user_ns;
-
-	/*
-	 * The list_lru structure is essentially just a pointer to a table
-	 * of per-node lru lists, each of which has its own spinlock.
-	 * There is no need to put them into separate cachelines.
-	 */
-	struct list_lru		s_dentry_lru;
-	struct list_lru		s_inode_lru;
-	struct rcu_head		rcu;
-	struct work_struct	destroy_work;
-
-	struct mutex		s_sync_lock;	/* sync serialisation lock */
-
-	/*
-	 * Indicates how deep in a filesystem stack this SB is
-	 */
-	int s_stack_depth;
-
-	/* s_inode_list_lock protects s_inodes */
-	spinlock_t		s_inode_list_lock ____cacheline_aligned_in_smp;
-	struct list_head	s_inodes;	/* all inodes */
-
-	spinlock_t		s_inode_wblist_lock;
-	struct list_head	s_inodes_wb;	/* writeback inodes */
-} __randomize_layout;
-
- static inline struct user_namespace *i_user_ns(const struct inode *inode)
- {
- 	return inode->i_sb->s_user_ns;
-@@ -1613,39 +1435,22 @@ static inline void i_gid_write(struct inode *inode, gid_t gid)
- }
- 
- /**
- * i_uid_into_mnt - map an inode's i_uid down into a mnt_userns
- * @mnt_userns: user namespace of the mount the inode was found from
-+ * i_uid_into_vfsuid - map an inode's i_uid down according to an idmapping
-+ * @idmap: idmap of the mount the inode was found from
-  * @inode: inode to map
-  *
- * Note, this will eventually be removed completely in favor of the type-safe
- * i_uid_into_vfsuid().
- *
- * Return: the inode's i_uid mapped down according to @mnt_userns.
- * If the inode's i_uid has no mapping INVALID_UID is returned.
- */
-static inline kuid_t i_uid_into_mnt(struct user_namespace *mnt_userns,
-				    const struct inode *inode)
-{
-	return AS_KUIDT(make_vfsuid(mnt_userns, i_user_ns(inode), inode->i_uid));
-}
-
-/**
- * i_uid_into_vfsuid - map an inode's i_uid down into a mnt_userns
- * @mnt_userns: user namespace of the mount the inode was found from
- * @inode: inode to map
- *
- * Return: whe inode's i_uid mapped down according to @mnt_userns.
-+ * Return: whe inode's i_uid mapped down according to @idmap.
-  * If the inode's i_uid has no mapping INVALID_VFSUID is returned.
-  */
-static inline vfsuid_t i_uid_into_vfsuid(struct user_namespace *mnt_userns,
-+static inline vfsuid_t i_uid_into_vfsuid(struct mnt_idmap *idmap,
- 					 const struct inode *inode)
- {
-	return make_vfsuid(mnt_userns, i_user_ns(inode), inode->i_uid);
-+	return make_vfsuid(idmap, i_user_ns(inode), inode->i_uid);
- }
- 
- /**
-  * i_uid_needs_update - check whether inode's i_uid needs to be updated
- * @mnt_userns: user namespace of the mount the inode was found from
-+ * @idmap: idmap of the mount the inode was found from
-  * @attr: the new attributes of @inode
-  * @inode: the inode to update
-  *
-@@ -1654,67 +1459,50 @@ static inline vfsuid_t i_uid_into_vfsuid(struct user_namespace *mnt_userns,
-  *
-  * Return: true if @inode's i_uid field needs to be updated, false if not.
-  */
-static inline bool i_uid_needs_update(struct user_namespace *mnt_userns,
-+static inline bool i_uid_needs_update(struct mnt_idmap *idmap,
- 				      const struct iattr *attr,
- 				      const struct inode *inode)
- {
- 	return ((attr->ia_valid & ATTR_UID) &&
- 		!vfsuid_eq(attr->ia_vfsuid,
-			   i_uid_into_vfsuid(mnt_userns, inode)));
-+			   i_uid_into_vfsuid(idmap, inode)));
- }
- 
- /**
-  * i_uid_update - update @inode's i_uid field
- * @mnt_userns: user namespace of the mount the inode was found from
-+ * @idmap: idmap of the mount the inode was found from
-  * @attr: the new attributes of @inode
-  * @inode: the inode to update
-  *
-  * Safely update @inode's i_uid field translating the vfsuid of any idmapped
-  * mount into the filesystem kuid.
-  */
-static inline void i_uid_update(struct user_namespace *mnt_userns,
-+static inline void i_uid_update(struct mnt_idmap *idmap,
- 				const struct iattr *attr,
- 				struct inode *inode)
- {
- 	if (attr->ia_valid & ATTR_UID)
-		inode->i_uid = from_vfsuid(mnt_userns, i_user_ns(inode),
-+		inode->i_uid = from_vfsuid(idmap, i_user_ns(inode),
- 					   attr->ia_vfsuid);
- }
- 
- /**
- * i_gid_into_mnt - map an inode's i_gid down into a mnt_userns
- * @mnt_userns: user namespace of the mount the inode was found from
- * @inode: inode to map
- *
- * Note, this will eventually be removed completely in favor of the type-safe
- * i_gid_into_vfsgid().
- *
- * Return: the inode's i_gid mapped down according to @mnt_userns.
- * If the inode's i_gid has no mapping INVALID_GID is returned.
- */
-static inline kgid_t i_gid_into_mnt(struct user_namespace *mnt_userns,
-				    const struct inode *inode)
-{
-	return AS_KGIDT(make_vfsgid(mnt_userns, i_user_ns(inode), inode->i_gid));
-}
-
-/**
- * i_gid_into_vfsgid - map an inode's i_gid down into a mnt_userns
- * @mnt_userns: user namespace of the mount the inode was found from
-+ * i_gid_into_vfsgid - map an inode's i_gid down according to an idmapping
-+ * @idmap: idmap of the mount the inode was found from
-  * @inode: inode to map
-  *
- * Return: the inode's i_gid mapped down according to @mnt_userns.
-+ * Return: the inode's i_gid mapped down according to @idmap.
-  * If the inode's i_gid has no mapping INVALID_VFSGID is returned.
-  */
-static inline vfsgid_t i_gid_into_vfsgid(struct user_namespace *mnt_userns,
-+static inline vfsgid_t i_gid_into_vfsgid(struct mnt_idmap *idmap,
- 					 const struct inode *inode)
- {
-	return make_vfsgid(mnt_userns, i_user_ns(inode), inode->i_gid);
-+	return make_vfsgid(idmap, i_user_ns(inode), inode->i_gid);
- }
- 
- /**
-  * i_gid_needs_update - check whether inode's i_gid needs to be updated
- * @mnt_userns: user namespace of the mount the inode was found from
-+ * @idmap: idmap of the mount the inode was found from
-  * @attr: the new attributes of @inode
-  * @inode: the inode to update
-  *
-@@ -1723,289 +1511,302 @@ static inline vfsgid_t i_gid_into_vfsgid(struct user_namespace *mnt_userns,
-  *
-  * Return: true if @inode's i_gid field needs to be updated, false if not.
-  */
-static inline bool i_gid_needs_update(struct user_namespace *mnt_userns,
-+static inline bool i_gid_needs_update(struct mnt_idmap *idmap,
- 				      const struct iattr *attr,
- 				      const struct inode *inode)
- {
- 	return ((attr->ia_valid & ATTR_GID) &&
- 		!vfsgid_eq(attr->ia_vfsgid,
-			   i_gid_into_vfsgid(mnt_userns, inode)));
-+			   i_gid_into_vfsgid(idmap, inode)));
- }
- 
- /**
-  * i_gid_update - update @inode's i_gid field
- * @mnt_userns: user namespace of the mount the inode was found from
-+ * @idmap: idmap of the mount the inode was found from
-  * @attr: the new attributes of @inode
-  * @inode: the inode to update
-  *
-  * Safely update @inode's i_gid field translating the vfsgid of any idmapped
-  * mount into the filesystem kgid.
-  */
-static inline void i_gid_update(struct user_namespace *mnt_userns,
-+static inline void i_gid_update(struct mnt_idmap *idmap,
- 				const struct iattr *attr,
- 				struct inode *inode)
- {
- 	if (attr->ia_valid & ATTR_GID)
-		inode->i_gid = from_vfsgid(mnt_userns, i_user_ns(inode),
-+		inode->i_gid = from_vfsgid(idmap, i_user_ns(inode),
- 					   attr->ia_vfsgid);
- }
- 
- /**
-  * inode_fsuid_set - initialize inode's i_uid field with callers fsuid
-  * @inode: inode to initialize
- * @mnt_userns: user namespace of the mount the inode was found from
-+ * @idmap: idmap of the mount the inode was found from
-  *
-  * Initialize the i_uid field of @inode. If the inode was found/created via
- * an idmapped mount map the caller's fsuid according to @mnt_users.
-+ * an idmapped mount map the caller's fsuid according to @idmap.
-  */
- static inline void inode_fsuid_set(struct inode *inode,
-				   struct user_namespace *mnt_userns)
-+				   struct mnt_idmap *idmap)
- {
-	inode->i_uid = mapped_fsuid(mnt_userns, i_user_ns(inode));
-+	inode->i_uid = mapped_fsuid(idmap, i_user_ns(inode));
- }
- 
- /**
-  * inode_fsgid_set - initialize inode's i_gid field with callers fsgid
-  * @inode: inode to initialize
- * @mnt_userns: user namespace of the mount the inode was found from
-+ * @idmap: idmap of the mount the inode was found from
-  *
-  * Initialize the i_gid field of @inode. If the inode was found/created via
- * an idmapped mount map the caller's fsgid according to @mnt_users.
-+ * an idmapped mount map the caller's fsgid according to @idmap.
-  */
- static inline void inode_fsgid_set(struct inode *inode,
-				   struct user_namespace *mnt_userns)
-+				   struct mnt_idmap *idmap)
- {
-	inode->i_gid = mapped_fsgid(mnt_userns, i_user_ns(inode));
-+	inode->i_gid = mapped_fsgid(idmap, i_user_ns(inode));
- }
- 
- /**
-  * fsuidgid_has_mapping() - check whether caller's fsuid/fsgid is mapped
-  * @sb: the superblock we want a mapping in
- * @mnt_userns: user namespace of the relevant mount
-+ * @idmap: idmap of the relevant mount
-  *
-  * Check whether the caller's fsuid and fsgid have a valid mapping in the
-  * s_user_ns of the superblock @sb. If the caller is on an idmapped mount map
- * the caller's fsuid and fsgid according to the @mnt_userns first.
-+ * the caller's fsuid and fsgid according to the @idmap first.
-  *
-  * Return: true if fsuid and fsgid is mapped, false if not.
-  */
- static inline bool fsuidgid_has_mapping(struct super_block *sb,
-					struct user_namespace *mnt_userns)
-+					struct mnt_idmap *idmap)
- {
- 	struct user_namespace *fs_userns = sb->s_user_ns;
- 	kuid_t kuid;
- 	kgid_t kgid;
- 
-	kuid = mapped_fsuid(mnt_userns, fs_userns);
-+	kuid = mapped_fsuid(idmap, fs_userns);
- 	if (!uid_valid(kuid))
- 		return false;
-	kgid = mapped_fsgid(mnt_userns, fs_userns);
-+	kgid = mapped_fsgid(idmap, fs_userns);
- 	if (!gid_valid(kgid))
- 		return false;
- 	return kuid_has_mapping(fs_userns, kuid) &&
- 	       kgid_has_mapping(fs_userns, kgid);
- }
- 
-extern struct timespec64 current_time(struct inode *inode);
-+struct timespec64 current_time(struct inode *inode);
-+struct timespec64 inode_set_ctime_current(struct inode *inode);
-+struct timespec64 inode_set_ctime_deleg(struct inode *inode,
-+					struct timespec64 update);
- 
-/*
- * Snapshotting support.
- */
-+static inline time64_t inode_get_atime_sec(const struct inode *inode)
-+{
-+	return inode->i_atime_sec;
-+}
- 
-/*
- * These are internal functions, please use sb_start_{write,pagefault,intwrite}
- * instead.
- */
-static inline void __sb_end_write(struct super_block *sb, int level)
-+static inline long inode_get_atime_nsec(const struct inode *inode)
- {
-	percpu_up_read(sb->s_writers.rw_sem + level-1);
-+	return inode->i_atime_nsec;
- }
- 
-static inline void __sb_start_write(struct super_block *sb, int level)
-+static inline struct timespec64 inode_get_atime(const struct inode *inode)
- {
-	percpu_down_read(sb->s_writers.rw_sem + level - 1);
-+	struct timespec64 ts = { .tv_sec  = inode_get_atime_sec(inode),
-+				 .tv_nsec = inode_get_atime_nsec(inode) };
-+
-+	return ts;
- }
- 
-static inline bool __sb_start_write_trylock(struct super_block *sb, int level)
-+static inline struct timespec64 inode_set_atime_to_ts(struct inode *inode,
-+						      struct timespec64 ts)
- {
-	return percpu_down_read_trylock(sb->s_writers.rw_sem + level - 1);
-+	inode->i_atime_sec = ts.tv_sec;
-+	inode->i_atime_nsec = ts.tv_nsec;
-+	return ts;
- }
- 
-#define __sb_writers_acquired(sb, lev)	\
-	percpu_rwsem_acquire(&(sb)->s_writers.rw_sem[(lev)-1], 1, _THIS_IP_)
-#define __sb_writers_release(sb, lev)	\
-	percpu_rwsem_release(&(sb)->s_writers.rw_sem[(lev)-1], 1, _THIS_IP_)
-+static inline struct timespec64 inode_set_atime(struct inode *inode,
-+						time64_t sec, long nsec)
-+{
-+	struct timespec64 ts = { .tv_sec  = sec,
-+				 .tv_nsec = nsec };
-+
-+	return inode_set_atime_to_ts(inode, ts);
-+}
- 
-static inline bool sb_write_started(const struct super_block *sb)
-+static inline time64_t inode_get_mtime_sec(const struct inode *inode)
- {
-	return lockdep_is_held_type(sb->s_writers.rw_sem + SB_FREEZE_WRITE - 1, 1);
-+	return inode->i_mtime_sec;
- }
- 
-/**
- * sb_end_write - drop write access to a superblock
- * @sb: the super we wrote to
- *
- * Decrement number of writers to the filesystem. Wake up possible waiters
- * wanting to freeze the filesystem.
- */
-static inline void sb_end_write(struct super_block *sb)
-+static inline long inode_get_mtime_nsec(const struct inode *inode)
- {
-	__sb_end_write(sb, SB_FREEZE_WRITE);
-+	return inode->i_mtime_nsec;
- }
- 
-/**
- * sb_end_pagefault - drop write access to a superblock from a page fault
- * @sb: the super we wrote to
- *
- * Decrement number of processes handling write page fault to the filesystem.
- * Wake up possible waiters wanting to freeze the filesystem.
- */
-static inline void sb_end_pagefault(struct super_block *sb)
-+static inline struct timespec64 inode_get_mtime(const struct inode *inode)
- {
-	__sb_end_write(sb, SB_FREEZE_PAGEFAULT);
-+	struct timespec64 ts = { .tv_sec  = inode_get_mtime_sec(inode),
-+				 .tv_nsec = inode_get_mtime_nsec(inode) };
-+	return ts;
- }
- 
-/**
- * sb_end_intwrite - drop write access to a superblock for internal fs purposes
- * @sb: the super we wrote to
- *
- * Decrement fs-internal number of writers to the filesystem.  Wake up possible
- * waiters wanting to freeze the filesystem.
- */
-static inline void sb_end_intwrite(struct super_block *sb)
-+static inline struct timespec64 inode_set_mtime_to_ts(struct inode *inode,
-+						      struct timespec64 ts)
- {
-	__sb_end_write(sb, SB_FREEZE_FS);
-+	inode->i_mtime_sec = ts.tv_sec;
-+	inode->i_mtime_nsec = ts.tv_nsec;
-+	return ts;
- }
- 
-/**
- * sb_start_write - get write access to a superblock
- * @sb: the super we write to
- *
- * When a process wants to write data or metadata to a file system (i.e. dirty
- * a page or an inode), it should embed the operation in a sb_start_write() -
- * sb_end_write() pair to get exclusion against file system freezing. This
- * function increments number of writers preventing freezing. If the file
- * system is already frozen, the function waits until the file system is
- * thawed.
- *
- * Since freeze protection behaves as a lock, users have to preserve
- * ordering of freeze protection and other filesystem locks. Generally,
- * freeze protection should be the outermost lock. In particular, we have:
-+static inline struct timespec64 inode_set_mtime(struct inode *inode,
-+						time64_t sec, long nsec)
-+{
-+	struct timespec64 ts = { .tv_sec  = sec,
-+				 .tv_nsec = nsec };
-+	return inode_set_mtime_to_ts(inode, ts);
-+}
-+
-+/*
-+ * Multigrain timestamps
-  *
- * sb_start_write
- *   -> i_mutex			(write path, truncate, directory ops, ...)
- *   -> s_umount		(freeze_super, thaw_super)
-+ * Conditionally use fine-grained ctime and mtime timestamps when there
-+ * are users actively observing them via getattr. The primary use-case
-+ * for this is NFS clients that use the ctime to distinguish between
-+ * different states of the file, and that are often fooled by multiple
-+ * operations that occur in the same coarse-grained timer tick.
-  */
-static inline void sb_start_write(struct super_block *sb)
-+#define I_CTIME_QUERIED		((u32)BIT(31))
-+
-+static inline time64_t inode_get_ctime_sec(const struct inode *inode)
-+{
-+	return inode->i_ctime_sec;
-+}
-+
-+static inline long inode_get_ctime_nsec(const struct inode *inode)
- {
-	__sb_start_write(sb, SB_FREEZE_WRITE);
-+	return inode->i_ctime_nsec & ~I_CTIME_QUERIED;
- }
- 
-static inline bool sb_start_write_trylock(struct super_block *sb)
-+static inline struct timespec64 inode_get_ctime(const struct inode *inode)
- {
-	return __sb_start_write_trylock(sb, SB_FREEZE_WRITE);
-+	struct timespec64 ts = { .tv_sec  = inode_get_ctime_sec(inode),
-+				 .tv_nsec = inode_get_ctime_nsec(inode) };
-+
-+	return ts;
- }
- 
-+struct timespec64 inode_set_ctime_to_ts(struct inode *inode, struct timespec64 ts);
-+
- /**
- * sb_start_pagefault - get write access to a superblock from a page fault
- * @sb: the super we write to
-+ * inode_set_ctime - set the ctime in the inode
-+ * @inode: inode in which to set the ctime
-+ * @sec: tv_sec value to set
-+ * @nsec: tv_nsec value to set
-  *
- * When a process starts handling write page fault, it should embed the
- * operation into sb_start_pagefault() - sb_end_pagefault() pair to get
- * exclusion against file system freezing. This is needed since the page fault
- * is going to dirty a page. This function increments number of running page
- * faults preventing freezing. If the file system is already frozen, the
- * function waits until the file system is thawed.
- *
- * Since page fault freeze protection behaves as a lock, users have to preserve
- * ordering of freeze protection and other filesystem locks. It is advised to
- * put sb_start_pagefault() close to mmap_lock in lock ordering. Page fault
- * handling code implies lock dependency:
- *
- * mmap_lock
- *   -> sb_start_pagefault
-+ * Set the ctime in @inode to { @sec, @nsec }
-  */
-static inline void sb_start_pagefault(struct super_block *sb)
-+static inline struct timespec64 inode_set_ctime(struct inode *inode,
-+						time64_t sec, long nsec)
- {
-	__sb_start_write(sb, SB_FREEZE_PAGEFAULT);
-+	struct timespec64 ts = { .tv_sec  = sec,
-+				 .tv_nsec = nsec };
-+
-+	return inode_set_ctime_to_ts(inode, ts);
- }
- 
-+struct timespec64 simple_inode_init_ts(struct inode *inode);
-+
-+/*
-+ * Snapshotting support.
-+ */
-+
- /**
- * sb_start_intwrite - get write access to a superblock for internal fs purposes
- * @sb: the super we write to
-+ * file_write_started - check if SB_FREEZE_WRITE is held
-+ * @file: the file we write to
-  *
- * This is the third level of protection against filesystem freezing. It is
- * free for use by a filesystem. The only requirement is that it must rank
- * below sb_start_pagefault.
- *
- * For example filesystem can call sb_start_intwrite() when starting a
- * transaction which somewhat eases handling of freezing for internal sources
- * of filesystem changes (internal fs threads, discarding preallocation on file
- * close, etc.).
-+ * May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
-+ * May be false positive with !S_ISREG, because file_start_write() has
-+ * no effect on !S_ISREG.
-  */
-static inline void sb_start_intwrite(struct super_block *sb)
-+static inline bool file_write_started(const struct file *file)
- {
-	__sb_start_write(sb, SB_FREEZE_FS);
-+	if (!S_ISREG(file_inode(file)->i_mode))
-+		return true;
-+	return sb_write_started(file_inode(file)->i_sb);
- }
- 
-static inline bool sb_start_intwrite_trylock(struct super_block *sb)
-+/**
-+ * file_write_not_started - check if SB_FREEZE_WRITE is not held
-+ * @file: the file we write to
-+ *
-+ * May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
-+ * May be false positive with !S_ISREG, because file_start_write() has
-+ * no effect on !S_ISREG.
-+ */
-+static inline bool file_write_not_started(const struct file *file)
- {
-	return __sb_start_write_trylock(sb, SB_FREEZE_FS);
-+	if (!S_ISREG(file_inode(file)->i_mode))
-+		return true;
-+	return sb_write_not_started(file_inode(file)->i_sb);
- }
- 
-bool inode_owner_or_capable(struct user_namespace *mnt_userns,
-+bool inode_owner_or_capable(struct mnt_idmap *idmap,
- 			    const struct inode *inode);
- 
- /*
-  * VFS helper functions..
-  */
-int vfs_create(struct user_namespace *, struct inode *,
-	       struct dentry *, umode_t, bool);
-int vfs_mkdir(struct user_namespace *, struct inode *,
-	      struct dentry *, umode_t);
-int vfs_mknod(struct user_namespace *, struct inode *, struct dentry *,
-              umode_t, dev_t);
-int vfs_symlink(struct user_namespace *, struct inode *,
-		struct dentry *, const char *);
-int vfs_link(struct dentry *, struct user_namespace *, struct inode *,
-	     struct dentry *, struct inode **);
-int vfs_rmdir(struct user_namespace *, struct inode *, struct dentry *);
-int vfs_unlink(struct user_namespace *, struct inode *, struct dentry *,
-	       struct inode **);
-+int vfs_create(struct mnt_idmap *, struct dentry *, umode_t,
-+	       struct delegated_inode *);
-+struct dentry *vfs_mkdir(struct mnt_idmap *, struct inode *,
-+			 struct dentry *, umode_t, struct delegated_inode *);
-+int vfs_mknod(struct mnt_idmap *, struct inode *, struct dentry *,
-+	      umode_t, dev_t, struct delegated_inode *);
-+int vfs_symlink(struct mnt_idmap *, struct inode *,
-+		struct dentry *, const char *, struct delegated_inode *);
-+int vfs_link(struct dentry *, struct mnt_idmap *, struct inode *,
-+	     struct dentry *, struct delegated_inode *);
-+int vfs_rmdir(struct mnt_idmap *, struct inode *, struct dentry *,
-+	      struct delegated_inode *);
-+int vfs_unlink(struct mnt_idmap *, struct inode *, struct dentry *,
-+	       struct delegated_inode *);
- 
- /**
-  * struct renamedata - contains all information required for renaming
- * @old_mnt_userns:    old user namespace of the mount the inode was found from
- * @old_dir:           parent of source
-+ * @mnt_idmap:     idmap of the mount in which the rename is happening.
-+ * @old_parent:        parent of source
-  * @old_dentry:                source
- * @new_mnt_userns:    new user namespace of the mount the inode was found from
- * @new_dir:           parent of destination
-+ * @new_parent:        parent of destination
-  * @new_dentry:                destination
-  * @delegated_inode:   returns an inode needing a delegation break
-  * @flags:             rename flags
-  */
- struct renamedata {
-	struct user_namespace *old_mnt_userns;
-	struct inode *old_dir;
-+	struct mnt_idmap *mnt_idmap;
-+	struct dentry *old_parent;
- 	struct dentry *old_dentry;
-	struct user_namespace *new_mnt_userns;
-	struct inode *new_dir;
-+	struct dentry *new_parent;
- 	struct dentry *new_dentry;
-	struct inode **delegated_inode;
-+	struct delegated_inode *delegated_inode;
- 	unsigned int flags;
- } __randomize_layout;
- 
- int vfs_rename(struct renamedata *);
- 
-static inline int vfs_whiteout(struct user_namespace *mnt_userns,
-+static inline int vfs_whiteout(struct mnt_idmap *idmap,
- 			       struct inode *dir, struct dentry *dentry)
- {
-	return vfs_mknod(mnt_userns, dir, dentry, S_IFCHR | WHITEOUT_MODE,
-			 WHITEOUT_DEV);
-+	return vfs_mknod(idmap, dir, dentry, S_IFCHR | WHITEOUT_MODE,
-+			 WHITEOUT_DEV, NULL);
- }
- 
-struct dentry *vfs_tmpfile(struct user_namespace *mnt_userns,
-			   struct dentry *dentry, umode_t mode, int open_flag);
-+struct file *kernel_tmpfile_open(struct mnt_idmap *idmap,
-+				 const struct path *parentpath,
-+				 umode_t mode, int open_flag,
-+				 const struct cred *cred);
-+struct file *kernel_file_open(const struct path *path, int flags,
-+			      const struct cred *cred);
- 
- int vfs_mkobj(struct dentry *, umode_t,
- 		int (*f)(struct dentry *, umode_t, void *),
-@@ -2015,8 +1816,6 @@ int vfs_fchown(struct file *file, uid_t user, gid_t group);
- int vfs_fchmod(struct file *file, umode_t mode);
- int vfs_utimes(const struct path *path, struct timespec64 *times);
- 
-extern long vfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
-
- #ifdef CONFIG_COMPAT
- extern long compat_ptr_ioctl(struct file *file, unsigned int cmd,
- 					unsigned long arg);
-@@ -2027,27 +1826,40 @@ extern long compat_ptr_ioctl(struct file *file, unsigned int cmd,
- /*
-  * VFS file helper functions.
-  */
-void inode_init_owner(struct user_namespace *mnt_userns, struct inode *inode,
-+void inode_init_owner(struct mnt_idmap *idmap, struct inode *inode,
- 		      const struct inode *dir, umode_t mode);
- extern bool may_open_dev(const struct path *path);
-umode_t mode_strip_sgid(struct user_namespace *mnt_userns,
-+umode_t mode_strip_sgid(struct mnt_idmap *idmap,
- 			const struct inode *dir, umode_t mode);
-+bool in_group_or_capable(struct mnt_idmap *idmap,
-+			 const struct inode *inode, vfsgid_t vfsgid);
- 
- /*
-  * This is the "filldir" function type, used by readdir() to let
-  * the kernel specify what kind of dirent layout it wants to have.
-  * This allows the kernel to read directories into kernel space or
-  * to have different dirent layouts depending on the binary type.
-+ * Return 'true' to keep going and 'false' if there are no more entries.
-  */
- struct dir_context;
-typedef int (*filldir_t)(struct dir_context *, const char *, int, loff_t, u64,
-+typedef bool (*filldir_t)(struct dir_context *, const char *, int, loff_t, u64,
- 			 unsigned);
- 
- struct dir_context {
- 	filldir_t actor;
- 	loff_t pos;
-+	/*
-+	 * Filesystems MUST NOT MODIFY count, but may use as a hint:
-+	 * 0	    unknown
-+	 * > 0      space in buffer (assume at least one entry)
-+	 * INT_MAX  unlimited
-+	 */
-+	int count;
- };
- 
-+/* If OR-ed with d_type, pending signals are not checked */
-+#define FILLDIR_FLAG_NOINTR	0x1000
-+
- /*
-  * These flags let !MMU mmap() govern direct device mapping vs immediate
-  * copying more easily for MAP_PRIVATE, especially for ROM filesystems.
-@@ -2087,11 +1899,23 @@ struct dir_context {
-  */
- #define REMAP_FILE_ADVISORY		(REMAP_FILE_CAN_SHORTEN)
- 
-+/*
-+ * These flags control the behavior of vfs_copy_file_range().
-+ * They are not available to the user via syscall.
-+ *
-+ * COPY_FILE_SPLICE: call splice direct instead of fs clone/copy ops
-+ */
-+#define COPY_FILE_SPLICE		(1 << 0)
-+
- struct iov_iter;
- struct io_uring_cmd;
-+struct offset_ctx;
-+
-+typedef unsigned int __bitwise fop_flags_t;
- 
- struct file_operations {
- 	struct module *owner;
-+	fop_flags_t fop_flags;
- 	loff_t (*llseek) (struct file *, loff_t, int);
- 	ssize_t (*read) (struct file *, char __user *, size_t, loff_t *);
- 	ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
-@@ -2099,26 +1923,24 @@ struct file_operations {
- 	ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
- 	int (*iopoll)(struct kiocb *kiocb, struct io_comp_batch *,
- 			unsigned int flags);
-	int (*iterate) (struct file *, struct dir_context *);
- 	int (*iterate_shared) (struct file *, struct dir_context *);
- 	__poll_t (*poll) (struct file *, struct poll_table_struct *);
- 	long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long);
- 	long (*compat_ioctl) (struct file *, unsigned int, unsigned long);
- 	int (*mmap) (struct file *, struct vm_area_struct *);
-	unsigned long mmap_supported_flags;
- 	int (*open) (struct inode *, struct file *);
- 	int (*flush) (struct file *, fl_owner_t id);
- 	int (*release) (struct inode *, struct file *);
- 	int (*fsync) (struct file *, loff_t, loff_t, int datasync);
- 	int (*fasync) (int, struct file *, int);
- 	int (*lock) (struct file *, int, struct file_lock *);
-	ssize_t (*sendpage) (struct file *, struct page *, int, size_t, loff_t *, int);
- 	unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
- 	int (*check_flags)(int);
- 	int (*flock) (struct file *, int, struct file_lock *);
- 	ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int);
- 	ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int);
-	int (*setlease)(struct file *, long, struct file_lock **, void **);
-+	void (*splice_eof)(struct file *file);
-+	int (*setlease)(struct file *, int, struct file_lease **, void **);
- 	long (*fallocate)(struct file *file, int mode, loff_t offset,
- 			  loff_t len);
- 	void (*show_fdinfo)(struct seq_file *m, struct file *f);
-@@ -2132,73 +1954,115 @@ struct file_operations {
- 				   loff_t len, unsigned int remap_flags);
- 	int (*fadvise)(struct file *, loff_t, loff_t, int);
- 	int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags);
-+	int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *,
-+				unsigned int poll_flags);
-+	int (*mmap_prepare)(struct vm_area_desc *);
- } __randomize_layout;
- 
-+/* Supports async buffered reads */
-+#define FOP_BUFFER_RASYNC	((__force fop_flags_t)(1 << 0))
-+/* Supports async buffered writes */
-+#define FOP_BUFFER_WASYNC	((__force fop_flags_t)(1 << 1))
-+/* Supports synchronous page faults for mappings */
-+#define FOP_MMAP_SYNC		((__force fop_flags_t)(1 << 2))
-+/* Supports non-exclusive O_DIRECT writes from multiple threads */
-+#define FOP_DIO_PARALLEL_WRITE	((__force fop_flags_t)(1 << 3))
-+/* Contains huge pages */
-+#define FOP_HUGE_PAGES		((__force fop_flags_t)(1 << 4))
-+/* Treat loff_t as unsigned (e.g., /dev/mem) */
-+#define FOP_UNSIGNED_OFFSET	((__force fop_flags_t)(1 << 5))
-+/* Supports asynchronous lock callbacks */
-+#define FOP_ASYNC_LOCK		((__force fop_flags_t)(1 << 6))
-+/* File system supports uncached read/write buffered IO */
-+#define FOP_DONTCACHE		((__force fop_flags_t)(1 << 7))
-+
-+/* Wrap a directory iterator that needs exclusive inode access */
-+int wrap_directory_iterator(struct file *, struct dir_context *,
-+			    int (*) (struct file *, struct dir_context *));
-+#define WRAP_DIR_ITER(x) \
-+	static int shared_##x(struct file *file , struct dir_context *ctx) \
-+	{ return wrap_directory_iterator(file, ctx, x); }
-+
- struct inode_operations {
- 	struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int);
- 	const char * (*get_link) (struct dentry *, struct inode *, struct delayed_call *);
-	int (*permission) (struct user_namespace *, struct inode *, int);
-	struct posix_acl * (*get_acl)(struct inode *, int, bool);
-+	int (*permission) (struct mnt_idmap *, struct inode *, int);
-+	struct posix_acl * (*get_inode_acl)(struct inode *, int, bool);
- 
- 	int (*readlink) (struct dentry *, char __user *,int);
- 
-	int (*create) (struct user_namespace *, struct inode *,struct dentry *,
-+	int (*create) (struct mnt_idmap *, struct inode *,struct dentry *,
- 		       umode_t, bool);
- 	int (*link) (struct dentry *,struct inode *,struct dentry *);
- 	int (*unlink) (struct inode *,struct dentry *);
-	int (*symlink) (struct user_namespace *, struct inode *,struct dentry *,
-+	int (*symlink) (struct mnt_idmap *, struct inode *,struct dentry *,
- 			const char *);
-	int (*mkdir) (struct user_namespace *, struct inode *,struct dentry *,
-		      umode_t);
-+	struct dentry *(*mkdir) (struct mnt_idmap *, struct inode *,
-+				 struct dentry *, umode_t);
- 	int (*rmdir) (struct inode *,struct dentry *);
-	int (*mknod) (struct user_namespace *, struct inode *,struct dentry *,
-+	int (*mknod) (struct mnt_idmap *, struct inode *,struct dentry *,
- 		      umode_t,dev_t);
-	int (*rename) (struct user_namespace *, struct inode *, struct dentry *,
-+	int (*rename) (struct mnt_idmap *, struct inode *, struct dentry *,
- 			struct inode *, struct dentry *, unsigned int);
-	int (*setattr) (struct user_namespace *, struct dentry *,
-			struct iattr *);
-	int (*getattr) (struct user_namespace *, const struct path *,
-+	int (*setattr) (struct mnt_idmap *, struct dentry *, struct iattr *);
-+	int (*getattr) (struct mnt_idmap *, const struct path *,
- 			struct kstat *, u32, unsigned int);
- 	ssize_t (*listxattr) (struct dentry *, char *, size_t);
- 	int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start,
- 		      u64 len);
-	int (*update_time)(struct inode *, struct timespec64 *, int);
-+	int (*update_time)(struct inode *, int);
- 	int (*atomic_open)(struct inode *, struct dentry *,
- 			   struct file *, unsigned open_flag,
- 			   umode_t create_mode);
-	int (*tmpfile) (struct user_namespace *, struct inode *,
-			struct dentry *, umode_t);
-	int (*set_acl)(struct user_namespace *, struct inode *,
-+	int (*tmpfile) (struct mnt_idmap *, struct inode *,
-+			struct file *, umode_t);
-+	struct posix_acl *(*get_acl)(struct mnt_idmap *, struct dentry *,
-+				     int);
-+	int (*set_acl)(struct mnt_idmap *, struct dentry *,
- 		       struct posix_acl *, int);
-	int (*fileattr_set)(struct user_namespace *mnt_userns,
-			    struct dentry *dentry, struct fileattr *fa);
-	int (*fileattr_get)(struct dentry *dentry, struct fileattr *fa);
-+	int (*fileattr_set)(struct mnt_idmap *idmap,
-+			    struct dentry *dentry, struct file_kattr *fa);
-+	int (*fileattr_get)(struct dentry *dentry, struct file_kattr *fa);
-+	struct offset_ctx *(*get_offset_ctx)(struct inode *inode);
- } ____cacheline_aligned;
- 
-static inline ssize_t call_read_iter(struct file *file, struct kiocb *kio,
-				     struct iov_iter *iter)
-+/* Did the driver provide valid mmap hook configuration? */
-+static inline bool can_mmap_file(struct file *file)
- {
-	return file->f_op->read_iter(kio, iter);
-+	bool has_mmap = file->f_op->mmap;
-+	bool has_mmap_prepare = file->f_op->mmap_prepare;
-+
-+	/* Hooks are mutually exclusive. */
-+	if (WARN_ON_ONCE(has_mmap && has_mmap_prepare))
-+		return false;
-+	if (!has_mmap && !has_mmap_prepare)
-+		return false;
-+
-+	return true;
- }
- 
-static inline ssize_t call_write_iter(struct file *file, struct kiocb *kio,
-				      struct iov_iter *iter)
-+int __compat_vma_mmap(const struct file_operations *f_op,
-+		struct file *file, struct vm_area_struct *vma);
-+int compat_vma_mmap(struct file *file, struct vm_area_struct *vma);
-+
-+static inline int vfs_mmap(struct file *file, struct vm_area_struct *vma)
- {
-	return file->f_op->write_iter(kio, iter);
-+	if (file->f_op->mmap_prepare)
-+		return compat_vma_mmap(file, vma);
-+
-+	return file->f_op->mmap(file, vma);
- }
- 
-static inline int call_mmap(struct file *file, struct vm_area_struct *vma)
-+static inline int vfs_mmap_prepare(struct file *file, struct vm_area_desc *desc)
- {
-	return file->f_op->mmap(file, vma);
-+	return file->f_op->mmap_prepare(desc);
- }
- 
- extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *);
- extern ssize_t vfs_write(struct file *, const char __user *, size_t, loff_t *);
- extern ssize_t vfs_copy_file_range(struct file *, loff_t , struct file *,
- 				   loff_t, size_t, unsigned int);
-extern ssize_t generic_copy_file_range(struct file *file_in, loff_t pos_in,
-				       struct file *file_out, loff_t pos_out,
-				       size_t len, unsigned int flags);
-+int remap_verify_area(struct file *file, loff_t pos, loff_t len, bool write);
- int __generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
- 				    struct file *file_out, loff_t pos_out,
- 				    loff_t *len, unsigned int remap_flags,
-@@ -2206,9 +2070,6 @@ int __generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
- int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
- 				  struct file *file_out, loff_t pos_out,
- 				  loff_t *count, unsigned int remap_flags);
-extern loff_t do_clone_file_range(struct file *file_in, loff_t pos_in,
-				  struct file *file_out, loff_t pos_out,
-				  loff_t len, unsigned int remap_flags);
- extern loff_t vfs_clone_file_range(struct file *file_in, loff_t pos_in,
- 				   struct file *file_out, loff_t pos_out,
- 				   loff_t len, unsigned int remap_flags);
-@@ -2218,41 +2079,6 @@ extern loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
- 					struct file *dst_file, loff_t dst_pos,
- 					loff_t len, unsigned int remap_flags);
- 
-
-struct super_operations {
-   	struct inode *(*alloc_inode)(struct super_block *sb);
-	void (*destroy_inode)(struct inode *);
-	void (*free_inode)(struct inode *);
-
-   	void (*dirty_inode) (struct inode *, int flags);
-	int (*write_inode) (struct inode *, struct writeback_control *wbc);
-	int (*drop_inode) (struct inode *);
-	void (*evict_inode) (struct inode *);
-	void (*put_super) (struct super_block *);
-	int (*sync_fs)(struct super_block *sb, int wait);
-	int (*freeze_super) (struct super_block *);
-	int (*freeze_fs) (struct super_block *);
-	int (*thaw_super) (struct super_block *);
-	int (*unfreeze_fs) (struct super_block *);
-	int (*statfs) (struct dentry *, struct kstatfs *);
-	int (*remount_fs) (struct super_block *, int *, char *);
-	void (*umount_begin) (struct super_block *);
-
-	int (*show_options)(struct seq_file *, struct dentry *);
-	int (*show_devname)(struct seq_file *, struct dentry *);
-	int (*show_path)(struct seq_file *, struct dentry *);
-	int (*show_stats)(struct seq_file *, struct dentry *);
-#ifdef CONFIG_QUOTA
-	ssize_t (*quota_read)(struct super_block *, int, char *, size_t, loff_t);
-	ssize_t (*quota_write)(struct super_block *, int, const char *, size_t, loff_t);
-	struct dquot **(*get_dquots)(struct inode *);
-#endif
-	long (*nr_cached_objects)(struct super_block *,
-				  struct shrink_control *);
-	long (*free_cached_objects)(struct super_block *,
-				    struct shrink_control *);
-};
-
- /*
-  * Inode flags - they have no relation to superblock flags now
-  */
-@@ -2278,6 +2104,7 @@ struct super_operations {
- #define S_CASEFOLD	(1 << 15) /* Casefolded file */
- #define S_VERITY	(1 << 16) /* Verity file (using fs/verity/) */
- #define S_KERNEL_FILE	(1 << 17) /* File is in use by the kernel (eg. fs/cachefiles) */
-+#define S_ANON_INODE	(1 << 19) /* Inode is an anonymous inode */
- 
- /*
-  * Note that nosuid etc flags are inode-specific: setting some file-system
-@@ -2294,7 +2121,6 @@ struct super_operations {
-  */
- #define __IS_FLG(inode, flg)	((inode)->i_sb->s_flags & (flg))
- 
-static inline bool sb_rdonly(const struct super_block *sb) { return sb->s_flags & SB_RDONLY; }
- #define IS_RDONLY(inode)	sb_rdonly((inode)->i_sb)
- #define IS_SYNC(inode)		(__IS_FLG(inode, SB_SYNCHRONOUS) || \
- 					((inode)->i_flags & S_SYNC))
-@@ -2307,11 +2133,22 @@ static inline bool sb_rdonly(const struct super_block *sb) { return sb->s_flags
- #define IS_NOQUOTA(inode)	((inode)->i_flags & S_NOQUOTA)
- #define IS_APPEND(inode)	((inode)->i_flags & S_APPEND)
- #define IS_IMMUTABLE(inode)	((inode)->i_flags & S_IMMUTABLE)
-+
-+#ifdef CONFIG_FS_POSIX_ACL
- #define IS_POSIXACL(inode)	__IS_FLG(inode, SB_POSIXACL)
-+#else
-+#define IS_POSIXACL(inode)	0
-+#endif
- 
- #define IS_DEADDIR(inode)	((inode)->i_flags & S_DEAD)
- #define IS_NOCMTIME(inode)	((inode)->i_flags & S_NOCMTIME)
-+
-+#ifdef CONFIG_SWAP
- #define IS_SWAPFILE(inode)	((inode)->i_flags & S_SWAPFILE)
-+#else
-+#define IS_SWAPFILE(inode)	((void)(inode), 0U)
-+#endif
-+
- #define IS_PRIVATE(inode)	((inode)->i_flags & S_PRIVATE)
- #define IS_IMA(inode)		((inode)->i_flags & S_IMA)
- #define IS_AUTOMOUNT(inode)	((inode)->i_flags & S_AUTOMOUNT)
-@@ -2323,12 +2160,13 @@ static inline bool sb_rdonly(const struct super_block *sb) { return sb->s_flags
- 
- #define IS_WHITEOUT(inode)	(S_ISCHR(inode->i_mode) && \
- 				 (inode)->i_rdev == WHITEOUT_DEV)
-+#define IS_ANON_FILE(inode)	((inode)->i_flags & S_ANON_INODE)
- 
-static inline bool HAS_UNMAPPED_ID(struct user_namespace *mnt_userns,
-+static inline bool HAS_UNMAPPED_ID(struct mnt_idmap *idmap,
- 				   struct inode *inode)
- {
-	return !vfsuid_valid(i_uid_into_vfsuid(mnt_userns, inode)) ||
-	       !vfsgid_valid(i_gid_into_vfsgid(mnt_userns, inode));
-+	return !vfsuid_valid(i_uid_into_vfsuid(idmap, inode)) ||
-+	       !vfsgid_valid(i_gid_into_vfsgid(idmap, inode));
- }
- 
- static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp)
-@@ -2351,111 +2189,6 @@ static inline void kiocb_clone(struct kiocb *kiocb, struct kiocb *kiocb_src,
- 	};
- }
- 
-/*
- * Inode state bits.  Protected by inode->i_lock
- *
- * Four bits determine the dirty state of the inode: I_DIRTY_SYNC,
- * I_DIRTY_DATASYNC, I_DIRTY_PAGES, and I_DIRTY_TIME.
- *
- * Four bits define the lifetime of an inode.  Initially, inodes are I_NEW,
- * until that flag is cleared.  I_WILL_FREE, I_FREEING and I_CLEAR are set at
- * various stages of removing an inode.
- *
- * Two bits are used for locking and completion notification, I_NEW and I_SYNC.
- *
- * I_DIRTY_SYNC		Inode is dirty, but doesn't have to be written on
- *			fdatasync() (unless I_DIRTY_DATASYNC is also set).
- *			Timestamp updates are the usual cause.
- * I_DIRTY_DATASYNC	Data-related inode changes pending.  We keep track of
- *			these changes separately from I_DIRTY_SYNC so that we
- *			don't have to write inode on fdatasync() when only
- *			e.g. the timestamps have changed.
- * I_DIRTY_PAGES	Inode has dirty pages.  Inode itself may be clean.
- * I_DIRTY_TIME		The inode itself only has dirty timestamps, and the
- *			lazytime mount option is enabled.  We keep track of this
- *			separately from I_DIRTY_SYNC in order to implement
- *			lazytime.  This gets cleared if I_DIRTY_INODE
- *			(I_DIRTY_SYNC and/or I_DIRTY_DATASYNC) gets set.  I.e.
- *			either I_DIRTY_TIME *or* I_DIRTY_INODE can be set in
- *			i_state, but not both.  I_DIRTY_PAGES may still be set.
- * I_NEW		Serves as both a mutex and completion notification.
- *			New inodes set I_NEW.  If two processes both create
- *			the same inode, one of them will release its inode and
- *			wait for I_NEW to be released before returning.
- *			Inodes in I_WILL_FREE, I_FREEING or I_CLEAR state can
- *			also cause waiting on I_NEW, without I_NEW actually
- *			being set.  find_inode() uses this to prevent returning
- *			nearly-dead inodes.
- * I_WILL_FREE		Must be set when calling write_inode_now() if i_count
- *			is zero.  I_FREEING must be set when I_WILL_FREE is
- *			cleared.
- * I_FREEING		Set when inode is about to be freed but still has dirty
- *			pages or buffers attached or the inode itself is still
- *			dirty.
- * I_CLEAR		Added by clear_inode().  In this state the inode is
- *			clean and can be destroyed.  Inode keeps I_FREEING.
- *
- *			Inodes that are I_WILL_FREE, I_FREEING or I_CLEAR are
- *			prohibited for many purposes.  iget() must wait for
- *			the inode to be completely released, then create it
- *			anew.  Other functions will just ignore such inodes,
- *			if appropriate.  I_NEW is used for waiting.
- *
- * I_SYNC		Writeback of inode is running. The bit is set during
- *			data writeback, and cleared with a wakeup on the bit
- *			address once it is done. The bit is also used to pin
- *			the inode in memory for flusher thread.
- *
- * I_REFERENCED		Marks the inode as recently references on the LRU list.
- *
- * I_DIO_WAKEUP		Never set.  Only used as a key for wait_on_bit().
- *
- * I_WB_SWITCH		Cgroup bdi_writeback switching in progress.  Used to
- *			synchronize competing switching instances and to tell
- *			wb stat updates to grab the i_pages lock.  See
- *			inode_switch_wbs_work_fn() for details.
- *
- * I_OVL_INUSE		Used by overlayfs to get exclusive ownership on upper
- *			and work dirs among overlayfs mounts.
- *
- * I_CREATING		New object's inode in the middle of setting up.
- *
- * I_DONTCACHE		Evict inode as soon as it is not used anymore.
- *
- * I_SYNC_QUEUED	Inode is queued in b_io or b_more_io writeback lists.
- *			Used to detect that mark_inode_dirty() should not move
- * 			inode between dirty lists.
- *
- * I_PINNING_FSCACHE_WB	Inode is pinning an fscache object for writeback.
- *
- * Q: What is the difference between I_WILL_FREE and I_FREEING?
- */
-#define I_DIRTY_SYNC		(1 << 0)
-#define I_DIRTY_DATASYNC	(1 << 1)
-#define I_DIRTY_PAGES		(1 << 2)
-#define __I_NEW			3
-#define I_NEW			(1 << __I_NEW)
-#define I_WILL_FREE		(1 << 4)
-#define I_FREEING		(1 << 5)
-#define I_CLEAR			(1 << 6)
-#define __I_SYNC		7
-#define I_SYNC			(1 << __I_SYNC)
-#define I_REFERENCED		(1 << 8)
-#define __I_DIO_WAKEUP		9
-#define I_DIO_WAKEUP		(1 << __I_DIO_WAKEUP)
-#define I_LINKABLE		(1 << 10)
-#define I_DIRTY_TIME		(1 << 11)
-#define I_WB_SWITCH		(1 << 13)
-#define I_OVL_INUSE		(1 << 14)
-#define I_CREATING		(1 << 15)
-#define I_DONTCACHE		(1 << 16)
-#define I_SYNC_QUEUED		(1 << 17)
-#define I_PINNING_FSCACHE_WB	(1 << 18)
-
-#define I_DIRTY_INODE (I_DIRTY_SYNC | I_DIRTY_DATASYNC)
-#define I_DIRTY (I_DIRTY_INODE | I_DIRTY_PAGES)
-#define I_DIRTY_ALL (I_DIRTY | I_DIRTY_TIME)
-
- extern void __mark_inode_dirty(struct inode *, int);
- static inline void mark_inode_dirty(struct inode *inode)
- {
-@@ -2467,6 +2200,11 @@ static inline void mark_inode_dirty_sync(struct inode *inode)
- 	__mark_inode_dirty(inode, I_DIRTY_SYNC);
- }
- 
-+static inline int icount_read(const struct inode *inode)
-+{
-+	return atomic_read(&inode->i_count);
-+}
-+
- /*
-  * Returns true if the given inode itself only has dirty timestamps (its pages
-  * may still be dirty) and isn't currently being allocated or freed.
-@@ -2478,8 +2216,8 @@ static inline void mark_inode_dirty_sync(struct inode *inode)
-  */
- static inline bool inode_is_dirtytime_only(struct inode *inode)
- {
-	return (inode->i_state & (I_DIRTY_TIME | I_NEW |
-				  I_FREEING | I_WILL_FREE)) == I_DIRTY_TIME;
-+	return (inode_state_read_once(inode) &
-+	       (I_DIRTY_TIME | I_NEW | I_FREEING | I_WILL_FREE)) == I_DIRTY_TIME;
- }
- 
- extern void inc_nlink(struct inode *inode);
-@@ -2508,7 +2246,7 @@ enum file_time_flags {
- 
- extern bool atime_needs_update(const struct path *, struct inode *);
- extern void touch_atime(const struct path *);
-int inode_update_time(struct inode *inode, struct timespec64 *time, int flags);
-+int inode_update_time(struct inode *inode, int flags);
- 
- static inline void file_accessed(struct file *file)
- {
-@@ -2530,6 +2268,9 @@ struct file_system_type {
- #define FS_USERNS_MOUNT		8	/* Can be mounted by userns root */
- #define FS_DISALLOW_NOTIFY_PERM	16	/* Disable fanotify permission events */
- #define FS_ALLOW_IDMAP         32      /* FS has been updated to handle vfs idmappings. */
-+#define FS_MGTIME		64	/* FS uses multigrain timestamps */
-+#define FS_LBS			128	/* FS supports LBS */
-+#define FS_POWER_FREEZE		256	/* Always freeze on suspend/hibernate */
- #define FS_RENAME_DOES_D_MOVE	32768	/* FS will handle d_move() during rename() internally. */
- 	int (*init_fs_context)(struct fs_context *);
- 	const struct fs_parameter_spec *parameters;
-@@ -2553,21 +2294,22 @@ struct file_system_type {
- 
- #define MODULE_ALIAS_FS(NAME) MODULE_ALIAS("fs-" NAME)
- 
-extern struct dentry *mount_bdev(struct file_system_type *fs_type,
-	int flags, const char *dev_name, void *data,
-	int (*fill_super)(struct super_block *, void *, int));
-extern struct dentry *mount_single(struct file_system_type *fs_type,
-	int flags, void *data,
-	int (*fill_super)(struct super_block *, void *, int));
-extern struct dentry *mount_nodev(struct file_system_type *fs_type,
-	int flags, void *data,
-	int (*fill_super)(struct super_block *, void *, int));
-+/**
-+ * is_mgtime: is this inode using multigrain timestamps
-+ * @inode: inode to test for multigrain timestamps
-+ *
-+ * Return true if the inode uses multigrain timestamps, false otherwise.
-+ */
-+static inline bool is_mgtime(const struct inode *inode)
-+{
-+	return inode->i_opflags & IOP_MGTIME;
-+}
-+
- extern struct dentry *mount_subtree(struct vfsmount *mnt, const char *path);
- void retire_super(struct super_block *sb);
- void generic_shutdown_super(struct super_block *sb);
- void kill_block_super(struct super_block *sb);
- void kill_anon_super(struct super_block *sb);
-void kill_litter_super(struct super_block *sb);
- void deactivate_super(struct super_block *sb);
- void deactivate_locked_super(struct super_block *sb);
- int set_anon_super(struct super_block *s, void *data);
-@@ -2581,12 +2323,20 @@ struct super_block *sget(struct file_system_type *type,
- 			int (*test)(struct super_block *,void *),
- 			int (*set)(struct super_block *,void *),
- 			int flags, void *data);
-+struct super_block *sget_dev(struct fs_context *fc, dev_t dev);
- 
- /* Alas, no aliases. Too much hassle with bringing module.h everywhere */
-#define fops_get(fops) \
-	(((fops) && try_module_get((fops)->owner) ? (fops) : NULL))
-#define fops_put(fops) \
-	do { if (fops) module_put((fops)->owner); } while(0)
-+#define fops_get(fops) ({						\
-+	const struct file_operations *_fops = (fops);			\
-+	(((_fops) && try_module_get((_fops)->owner) ? (_fops) : NULL));	\
-+})
-+
-+#define fops_put(fops) ({						\
-+	const struct file_operations *_fops = (fops);			\
-+	if (_fops)							\
-+		module_put((_fops)->owner);				\
-+})
-+
- /*
-  * This one is to be used *ONLY* from ->open() instances.
-  * fops must be non-NULL, pinned down *and* module dependencies
-@@ -2604,150 +2354,95 @@ extern int unregister_filesystem(struct file_system_type *);
- extern int vfs_statfs(const struct path *, struct kstatfs *);
- extern int user_statfs(const char __user *, struct kstatfs *);
- extern int fd_statfs(int, struct kstatfs *);
-extern int freeze_super(struct super_block *super);
-extern int thaw_super(struct super_block *super);
- extern __printf(2, 3)
- int super_setup_bdi_name(struct super_block *sb, char *fmt, ...);
- extern int super_setup_bdi(struct super_block *sb);
- 
-extern int current_umask(void);
-
-extern void ihold(struct inode * inode);
-extern void iput(struct inode *);
-extern int generic_update_time(struct inode *, struct timespec64 *, int);
-
-/* /sys/fs */
-extern struct kobject *fs_kobj;
-
-#define MAX_RW_COUNT (INT_MAX & PAGE_MASK)
-
-#ifdef CONFIG_FILE_LOCKING
-static inline int break_lease(struct inode *inode, unsigned int mode)
-{
-	/*
-	 * Since this check is lockless, we must ensure that any refcounts
-	 * taken are done before checking i_flctx->flc_lease. Otherwise, we
-	 * could end up racing with tasks trying to set a new lease on this
-	 * file.
-	 */
-	smp_mb();
-	if (inode->i_flctx && !list_empty_careful(&inode->i_flctx->flc_lease))
-		return __break_lease(inode, mode, FL_LEASE);
-	return 0;
-}
-
-static inline int break_deleg(struct inode *inode, unsigned int mode)
-{
-	/*
-	 * Since this check is lockless, we must ensure that any refcounts
-	 * taken are done before checking i_flctx->flc_lease. Otherwise, we
-	 * could end up racing with tasks trying to set a new lease on this
-	 * file.
-	 */
-	smp_mb();
-	if (inode->i_flctx && !list_empty_careful(&inode->i_flctx->flc_lease))
-		return __break_lease(inode, mode, FL_DELEG);
-	return 0;
-}
-
-static inline int try_break_deleg(struct inode *inode, struct inode **delegated_inode)
-+static inline void super_set_uuid(struct super_block *sb, const u8 *uuid, unsigned len)
- {
-	int ret;
-
-	ret = break_deleg(inode, O_WRONLY|O_NONBLOCK);
-	if (ret == -EWOULDBLOCK && delegated_inode) {
-		*delegated_inode = inode;
-		ihold(inode);
-	}
-	return ret;
-+	if (WARN_ON(len > sizeof(sb->s_uuid)))
-+		len = sizeof(sb->s_uuid);
-+	sb->s_uuid_len = len;
-+	memcpy(&sb->s_uuid, uuid, len);
- }
- 
-static inline int break_deleg_wait(struct inode **delegated_inode)
-+/* set sb sysfs name based on sb->s_bdev */
-+static inline void super_set_sysfs_name_bdev(struct super_block *sb)
- {
-	int ret;
-
-	ret = break_deleg(*delegated_inode, O_WRONLY);
-	iput(*delegated_inode);
-	*delegated_inode = NULL;
-	return ret;
-+	snprintf(sb->s_sysfs_name, sizeof(sb->s_sysfs_name), "%pg", sb->s_bdev);
- }
- 
-static inline int break_layout(struct inode *inode, bool wait)
-+/* set sb sysfs name based on sb->s_uuid */
-+static inline void super_set_sysfs_name_uuid(struct super_block *sb)
- {
-	smp_mb();
-	if (inode->i_flctx && !list_empty_careful(&inode->i_flctx->flc_lease))
-		return __break_lease(inode,
-				wait ? O_WRONLY : O_WRONLY | O_NONBLOCK,
-				FL_LAYOUT);
-	return 0;
-+	WARN_ON(sb->s_uuid_len != sizeof(sb->s_uuid));
-+	snprintf(sb->s_sysfs_name, sizeof(sb->s_sysfs_name), "%pU", sb->s_uuid.b);
- }
- 
-#else /* !CONFIG_FILE_LOCKING */
-static inline int break_lease(struct inode *inode, unsigned int mode)
-+/* set sb sysfs name based on sb->s_id */
-+static inline void super_set_sysfs_name_id(struct super_block *sb)
- {
-	return 0;
-+	strscpy(sb->s_sysfs_name, sb->s_id, sizeof(sb->s_sysfs_name));
- }
- 
-static inline int break_deleg(struct inode *inode, unsigned int mode)
-+/* try to use something standard before you use this */
-+__printf(2, 3)
-+static inline void super_set_sysfs_name_generic(struct super_block *sb, const char *fmt, ...)
- {
-	return 0;
-}
-+	va_list args;
- 
-static inline int try_break_deleg(struct inode *inode, struct inode **delegated_inode)
-{
-	return 0;
-+	va_start(args, fmt);
-+	vsnprintf(sb->s_sysfs_name, sizeof(sb->s_sysfs_name), fmt, args);
-+	va_end(args);
- }
- 
-static inline int break_deleg_wait(struct inode **delegated_inode)
-{
-	BUG();
-	return 0;
-}
-+extern void ihold(struct inode * inode);
-+extern void iput(struct inode *);
-+void iput_not_last(struct inode *);
-+int inode_update_timestamps(struct inode *inode, int flags);
-+int generic_update_time(struct inode *, int);
- 
-static inline int break_layout(struct inode *inode, bool wait)
-{
-	return 0;
-}
-+/* /sys/fs */
-+extern struct kobject *fs_kobj;
- 
-#endif /* CONFIG_FILE_LOCKING */
-+#define MAX_RW_COUNT (INT_MAX & PAGE_MASK)
- 
- /* fs/open.c */
- struct audit_names;
- struct filename {
- 	const char		*name;	/* pointer to actual string */
- 	const __user char	*uptr;	/* original userland pointer */
-	int			refcnt;
-+	atomic_t		refcnt;
- 	struct audit_names	*aname;
- 	const char		iname[];
- };
- static_assert(offsetof(struct filename, iname) % sizeof(long) == 0);
- 
-static inline struct user_namespace *file_mnt_user_ns(struct file *file)
-+static inline struct mnt_idmap *file_mnt_idmap(const struct file *file)
- {
-	return mnt_user_ns(file->f_path.mnt);
-+	return mnt_idmap(file->f_path.mnt);
- }
- 
- /**
-  * is_idmapped_mnt - check whether a mount is mapped
-  * @mnt: the mount to check
-  *
- * If @mnt has an idmapping attached different from the
- * filesystem's idmapping then @mnt is mapped.
-+ * If @mnt has an non @nop_mnt_idmap attached to it then @mnt is mapped.
-  *
-  * Return: true if mount is mapped, false if not.
-  */
- static inline bool is_idmapped_mnt(const struct vfsmount *mnt)
- {
-	return mnt_user_ns(mnt) != mnt->mnt_sb->s_user_ns;
-+	return mnt_idmap(mnt) != &nop_mnt_idmap;
- }
- 
-extern long vfs_truncate(const struct path *, loff_t);
-int do_truncate(struct user_namespace *, struct dentry *, loff_t start,
-+int vfs_truncate(const struct path *, loff_t);
-+int do_truncate(struct mnt_idmap *, struct dentry *, loff_t start,
- 		unsigned int time_attrs, struct file *filp);
- extern int vfs_fallocate(struct file *file, int mode, loff_t offset,
- 			loff_t len);
-extern long do_sys_open(int dfd, const char __user *filename, int flags,
-			umode_t mode);
-+int do_sys_open(int dfd, const char __user *filename, int flags,
-+		umode_t mode);
- extern struct file *file_open_name(struct filename *, int, umode_t);
- extern struct file *filp_open(const char *, int, umode_t);
- extern struct file *file_open_root(const struct path *,
-@@ -2758,27 +2453,83 @@ static inline struct file *file_open_root_mnt(struct vfsmount *mnt,
- 	return file_open_root(&(struct path){.mnt = mnt, .dentry = mnt->mnt_root},
- 			      name, flags, mode);
- }
-extern struct file * dentry_open(const struct path *, int, const struct cred *);
-extern struct file *dentry_create(const struct path *path, int flags,
-				  umode_t mode, const struct cred *cred);
-extern struct file * open_with_fake_path(const struct path *, int,
-					 struct inode*, const struct cred *);
-+struct file *dentry_open(const struct path *path, int flags,
-+			 const struct cred *creds);
-+struct file *dentry_open_nonotify(const struct path *path, int flags,
-+				  const struct cred *cred);
-+struct file *dentry_create(const struct path *path, int flags, umode_t mode,
-+			   const struct cred *cred);
-+const struct path *backing_file_user_path(const struct file *f);
-+
-+/*
-+ * When mmapping a file on a stackable filesystem (e.g., overlayfs), the file
-+ * stored in ->vm_file is a backing file whose f_inode is on the underlying
-+ * filesystem.  When the mapped file path and inode number are displayed to
-+ * user (e.g. via /proc/<pid>/maps), these helpers should be used to get the
-+ * path and inode number to display to the user, which is the path of the fd
-+ * that user has requested to map and the inode number that would be returned
-+ * by fstat() on that same fd.
-+ */
-+/* Get the path to display in /proc/<pid>/maps */
-+static inline const struct path *file_user_path(const struct file *f)
-+{
-+	if (unlikely(f->f_mode & FMODE_BACKING))
-+		return backing_file_user_path(f);
-+	return &f->f_path;
-+}
-+/* Get the inode whose inode number to display in /proc/<pid>/maps */
-+static inline const struct inode *file_user_inode(const struct file *f)
-+{
-+	if (unlikely(f->f_mode & FMODE_BACKING))
-+		return d_inode(backing_file_user_path(f)->dentry);
-+	return file_inode(f);
-+}
-+
- static inline struct file *file_clone_open(struct file *file)
- {
- 	return dentry_open(&file->f_path, file->f_flags, file->f_cred);
- }
- extern int filp_close(struct file *, fl_owner_t id);
- 
-extern struct filename *getname_flags(const char __user *, int, int *);
-+extern struct filename *getname_flags(const char __user *, int);
- extern struct filename *getname_uflags(const char __user *, int);
-extern struct filename *getname(const char __user *);
-+static inline struct filename *getname(const char __user *name)
-+{
-+	return getname_flags(name, 0);
-+}
- extern struct filename *getname_kernel(const char *);
-+extern struct filename *__getname_maybe_null(const char __user *);
-+static inline struct filename *getname_maybe_null(const char __user *name, int flags)
-+{
-+	if (!(flags & AT_EMPTY_PATH))
-+		return getname(name);
-+
-+	if (!name)
-+		return NULL;
-+	return __getname_maybe_null(name);
-+}
- extern void putname(struct filename *name);
-+DEFINE_FREE(putname, struct filename *, if (!IS_ERR_OR_NULL(_T)) putname(_T))
-+
-+static inline struct filename *refname(struct filename *name)
-+{
-+	atomic_inc(&name->refcnt);
-+	return name;
-+}
- 
- extern int finish_open(struct file *file, struct dentry *dentry,
- 			int (*open)(struct inode *, struct file *));
- extern int finish_no_open(struct file *file, struct dentry *dentry);
- 
-+/* Helper for the simple case when original dentry is used */
-+static inline int finish_open_simple(struct file *file, int error)
-+{
-+	if (error)
-+		return error;
-+
-+	return finish_open(file, file->f_path.dentry, NULL);
-+}
-+
- /* fs/dcache.c */
- extern void __init vfs_caches_init_early(void);
- extern void __init vfs_caches_init(void);
-@@ -2788,12 +2539,6 @@ extern struct kmem_cache *names_cachep;
- #define __getname()		kmem_cache_alloc(names_cachep, GFP_KERNEL)
- #define __putname(name)		kmem_cache_free(names_cachep, (void *)(name))
- 
-extern struct super_block *blockdev_superblock;
-static inline bool sb_is_blkdev_sb(struct super_block *sb)
-{
-	return IS_ENABLED(CONFIG_BLOCK) && sb == blockdev_superblock;
-}
-
- void emergency_thaw_all(void);
- extern int sync_filesystem(struct super_block *);
- extern const struct file_operations def_blk_fops;
-@@ -2839,6 +2584,8 @@ extern int __must_check file_fdatawait_range(struct file *file, loff_t lstart,
- extern int __must_check file_check_and_advance_wb_err(struct file *file);
- extern int __must_check file_write_and_wait_range(struct file *file,
- 						loff_t start, loff_t end);
-+int filemap_flush_range(struct address_space *mapping, loff_t start,
-+		loff_t end);
- 
- static inline int file_write_and_wait(struct file *file)
- {
-@@ -2871,6 +2618,11 @@ static inline ssize_t generic_write_sync(struct kiocb *iocb, ssize_t count)
- 				(iocb->ki_flags & IOCB_SYNC) ? 0 : 1);
- 		if (ret)
- 			return ret;
-+	} else if (iocb->ki_flags & IOCB_DONTCACHE) {
-+		struct address_space *mapping = iocb->ki_filp->f_mapping;
-+
-+		filemap_flush_range(mapping, iocb->ki_pos - count,
-+				iocb->ki_pos - 1);
- 	}
- 
- 	return count;
-@@ -2888,21 +2640,21 @@ static inline int bmap(struct inode *inode,  sector_t *block)
- }
- #endif
- 
-int notify_change(struct user_namespace *, struct dentry *,
-		  struct iattr *, struct inode **);
-int inode_permission(struct user_namespace *, struct inode *, int);
-int generic_permission(struct user_namespace *, struct inode *, int);
-+int notify_change(struct mnt_idmap *, struct dentry *,
-+		  struct iattr *, struct delegated_inode *);
-+int inode_permission(struct mnt_idmap *, struct inode *, int);
-+int generic_permission(struct mnt_idmap *, struct inode *, int);
- static inline int file_permission(struct file *file, int mask)
- {
-	return inode_permission(file_mnt_user_ns(file),
-+	return inode_permission(file_mnt_idmap(file),
- 				file_inode(file), mask);
- }
- static inline int path_permission(const struct path *path, int mask)
- {
-	return inode_permission(mnt_user_ns(path->mnt),
-+	return inode_permission(mnt_idmap(path->mnt),
- 				d_inode(path->dentry), mask);
- }
-int __check_sticky(struct user_namespace *mnt_userns, struct inode *dir,
-+int __check_sticky(struct mnt_idmap *idmap, struct inode *dir,
- 		   struct inode *inode);
- 
- static inline bool execute_ok(struct inode *inode)
-@@ -2915,6 +2667,13 @@ static inline bool inode_wrong_type(const struct inode *inode, umode_t mode)
- 	return (inode->i_mode ^ mode) & S_IFMT;
- }
- 
-+/**
-+ * file_start_write - get write access to a superblock for regular file io
-+ * @file: the file we want to write to
-+ *
-+ * This is a variant of sb_start_write() which is a noop on non-regular file.
-+ * Should be matched with a call to file_end_write().
-+ */
- static inline void file_start_write(struct file *file)
- {
- 	if (!S_ISREG(file_inode(file)->i_mode))
-@@ -2929,11 +2688,53 @@ static inline bool file_start_write_trylock(struct file *file)
- 	return sb_start_write_trylock(file_inode(file)->i_sb);
- }
- 
-+/**
-+ * file_end_write - drop write access to a superblock of a regular file
-+ * @file: the file we wrote to
-+ *
-+ * Should be matched with a call to file_start_write().
-+ */
- static inline void file_end_write(struct file *file)
- {
- 	if (!S_ISREG(file_inode(file)->i_mode))
- 		return;
-	__sb_end_write(file_inode(file)->i_sb, SB_FREEZE_WRITE);
-+	sb_end_write(file_inode(file)->i_sb);
-+}
-+
-+/**
-+ * kiocb_start_write - get write access to a superblock for async file io
-+ * @iocb: the io context we want to submit the write with
-+ *
-+ * This is a variant of sb_start_write() for async io submission.
-+ * Should be matched with a call to kiocb_end_write().
-+ */
-+static inline void kiocb_start_write(struct kiocb *iocb)
-+{
-+	struct inode *inode = file_inode(iocb->ki_filp);
-+
-+	sb_start_write(inode->i_sb);
-+	/*
-+	 * Fool lockdep by telling it the lock got released so that it
-+	 * doesn't complain about the held lock when we return to userspace.
-+	 */
-+	__sb_writers_release(inode->i_sb, SB_FREEZE_WRITE);
-+}
-+
-+/**
-+ * kiocb_end_write - drop write access to a superblock after async file io
-+ * @iocb: the io context we sumbitted the write with
-+ *
-+ * Should be matched with a call to kiocb_start_write().
-+ */
-+static inline void kiocb_end_write(struct kiocb *iocb)
-+{
-+	struct inode *inode = file_inode(iocb->ki_filp);
-+
-+	/*
-+	 * Tell lockdep we inherited freeze protection from submission thread.
-+	 */
-+	__sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
-+	sb_end_write(inode->i_sb);
- }
- 
- /*
-@@ -2975,6 +2776,34 @@ static inline void allow_write_access(struct file *file)
- 	if (file)
- 		atomic_inc(&file_inode(file)->i_writecount);
- }
-+
-+/*
-+ * Do not prevent write to executable file when watched by pre-content events.
-+ *
-+ * Note that FMODE_FSNOTIFY_HSM mode is set depending on pre-content watches at
-+ * the time of file open and remains constant for entire lifetime of the file,
-+ * so if pre-content watches are added post execution or removed before the end
-+ * of the execution, it will not cause i_writecount reference leak.
-+ */
-+static inline int exe_file_deny_write_access(struct file *exe_file)
-+{
-+	if (unlikely(FMODE_FSNOTIFY_HSM(exe_file->f_mode)))
-+		return 0;
-+	return deny_write_access(exe_file);
-+}
-+static inline void exe_file_allow_write_access(struct file *exe_file)
-+{
-+	if (unlikely(!exe_file || FMODE_FSNOTIFY_HSM(exe_file->f_mode)))
-+		return;
-+	allow_write_access(exe_file);
-+}
-+
-+static inline void file_set_fsnotify_mode(struct file *file, fmode_t mode)
-+{
-+	file->f_mode &= ~FMODE_FSNOTIFY_MASK;
-+	file->f_mode |= mode;
-+}
-+
- static inline bool inode_is_open_for_write(const struct inode *inode)
- {
- 	return atomic_read(&inode->i_writecount) > 0;
-@@ -2983,8 +2812,7 @@ static inline bool inode_is_open_for_write(const struct inode *inode)
- #if defined(CONFIG_IMA) || defined(CONFIG_FILE_LOCKING)
- static inline void i_readcount_dec(struct inode *inode)
- {
-	BUG_ON(!atomic_read(&inode->i_readcount));
-	atomic_dec(&inode->i_readcount);
-+	BUG_ON(atomic_dec_return(&inode->i_readcount) < 0);
- }
- static inline void i_readcount_inc(struct inode *inode)
- {
-@@ -3011,9 +2839,37 @@ extern struct file * open_exec(const char *);
- /* fs/dcache.c -- generic fs support functions */
- extern bool is_subdir(struct dentry *, struct dentry *);
- extern bool path_is_under(const struct path *, const struct path *);
-+u64 vfsmount_to_propagation_flags(struct vfsmount *mnt);
- 
- extern char *file_path(struct file *, char *, int);
- 
-+/**
-+ * is_dot_dotdot - returns true only if @name is "." or ".."
-+ * @name: file name to check
-+ * @len: length of file name, in bytes
-+ */
-+static inline bool is_dot_dotdot(const char *name, size_t len)
-+{
-+	return len && unlikely(name[0] == '.') &&
-+		(len == 1 || (len == 2 && name[1] == '.'));
-+}
-+
-+/**
-+ * name_contains_dotdot - check if a file name contains ".." path components
-+ * @name: File path string to check
-+ * Search for ".." surrounded by either '/' or start/end of string.
-+ */
-+static inline bool name_contains_dotdot(const char *name)
-+{
-+	size_t name_len;
-+
-+	name_len = strlen(name);
-+	return strcmp(name, "..") == 0 ||
-+	       strncmp(name, "../", 3) == 0 ||
-+	       strstr(name, "/../") != NULL ||
-+	       (name_len >= 3 && strcmp(name + name_len - 3, "/..") == 0);
-+}
-+
- #include <linux/err.h>
- 
- /* needed for stackable file system support */
-@@ -3021,14 +2877,19 @@ extern loff_t default_llseek(struct file *file, loff_t offset, int whence);
- 
- extern loff_t vfs_llseek(struct file *file, loff_t offset, int whence);
- 
-extern int inode_init_always(struct super_block *, struct inode *);
-+extern int inode_init_always_gfp(struct super_block *, struct inode *, gfp_t);
-+static inline int inode_init_always(struct super_block *sb, struct inode *inode)
-+{
-+	return inode_init_always_gfp(sb, inode, GFP_NOFS);
-+}
-+
- extern void inode_init_once(struct inode *);
- extern void address_space_init_once(struct address_space *mapping);
- extern struct inode * igrab(struct inode *);
- extern ino_t iunique(struct super_block *, ino_t);
- extern int inode_needs_sync(struct inode *inode);
-extern int generic_delete_inode(struct inode *inode);
-static inline int generic_drop_inode(struct inode *inode)
-+extern int inode_just_drop(struct inode *inode);
-+static inline int inode_generic_drop(struct inode *inode)
- {
- 	return !inode->i_nlink || inode_unhashed(inode);
- }
-@@ -3036,7 +2897,7 @@ extern void d_mark_dontcache(struct inode *inode);
- 
- extern struct inode *ilookup5_nowait(struct super_block *sb,
- 		unsigned long hashval, int (*test)(struct inode *, void *),
-		void *data);
-+		void *data, bool *isnew);
- extern struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
- 		int (*test)(struct inode *, void *), void *data);
- extern struct inode *ilookup(struct super_block *sb, unsigned long ino);
-@@ -3045,7 +2906,12 @@ extern struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
- 		int (*test)(struct inode *, void *),
- 		int (*set)(struct inode *, void *),
- 		void *data);
-extern struct inode * iget5_locked(struct super_block *, unsigned long, int (*test)(struct inode *, void *), int (*set)(struct inode *, void *), void *);
-+struct inode *iget5_locked(struct super_block *, unsigned long,
-+			   int (*test)(struct inode *, void *),
-+			   int (*set)(struct inode *, void *), void *);
-+struct inode *iget5_locked_rcu(struct super_block *, unsigned long,
-+			       int (*test)(struct inode *, void *),
-+			       int (*set)(struct inode *, void *), void *);
- extern struct inode * iget_locked(struct super_block *, unsigned long);
- extern struct inode *find_inode_nowait(struct super_block *,
- 				       unsigned long,
-@@ -3069,7 +2935,7 @@ extern void evict_inodes(struct super_block *sb);
- void dump_mapping(const struct address_space *);
- 
- /*
- * Userspace may rely on the the inode number being non-zero. For example, glibc
-+ * Userspace may rely on the inode number being non-zero. For example, glibc
-  * simply ignores files with zero i_ino in unlink() and other places.
-  *
-  * As an additional complication, if userspace was compiled with
-@@ -3083,25 +2949,32 @@ static inline bool is_zero_ino(ino_t ino)
- 	return (u32)ino == 0;
- }
- 
-extern void __iget(struct inode * inode);
-+static inline void __iget(struct inode *inode)
-+{
-+	lockdep_assert_held(&inode->i_lock);
-+	atomic_inc(&inode->i_count);
-+}
-+
- extern void iget_failed(struct inode *);
- extern void clear_inode(struct inode *);
- extern void __destroy_inode(struct inode *);
-extern struct inode *new_inode_pseudo(struct super_block *sb);
-+struct inode *alloc_inode(struct super_block *sb);
-+static inline struct inode *new_inode_pseudo(struct super_block *sb)
-+{
-+	return alloc_inode(sb);
-+}
- extern struct inode *new_inode(struct super_block *sb);
- extern void free_inode_nonrcu(struct inode *inode);
-extern int should_remove_suid(struct dentry *);
-+extern int setattr_should_drop_suidgid(struct mnt_idmap *, struct inode *);
- extern int file_remove_privs(struct file *);
-+int setattr_should_drop_sgid(struct mnt_idmap *idmap,
-+			     const struct inode *inode);
- 
- /*
-  * This must be used for allocating filesystems specific inodes to set
-  * up the inode reclaim context correctly.
-  */
-static inline void *
-alloc_inode_sb(struct super_block *sb, struct kmem_cache *cache, gfp_t gfp)
-{
-	return kmem_cache_alloc_lru(cache, &sb->s_inode_lru, gfp);
-}
-+#define alloc_inode_sb(_sb, _cache, _gfp) kmem_cache_alloc_lru(_cache, &_sb->s_inode_lru, _gfp)
- 
- extern void __insert_inode_hash(struct inode *, unsigned long hashval);
- static inline void insert_inode_hash(struct inode *inode)
-@@ -3117,13 +2990,12 @@ static inline void remove_inode_hash(struct inode *inode)
- }
- 
- extern void inode_sb_list_add(struct inode *inode);
-extern void inode_add_lru(struct inode *inode);
-
-extern int sb_set_blocksize(struct super_block *, int);
-extern int sb_min_blocksize(struct super_block *, int);
-+extern void inode_lru_list_add(struct inode *inode);
- 
-extern int generic_file_mmap(struct file *, struct vm_area_struct *);
-extern int generic_file_readonly_mmap(struct file *, struct vm_area_struct *);
-+int generic_file_mmap(struct file *, struct vm_area_struct *);
-+int generic_file_mmap_prepare(struct vm_area_desc *desc);
-+int generic_file_readonly_mmap(struct file *, struct vm_area_struct *);
-+int generic_file_readonly_mmap_prepare(struct vm_area_desc *desc);
- extern ssize_t generic_write_checks(struct kiocb *, struct iov_iter *);
- int generic_write_checks_count(struct kiocb *iocb, loff_t *count);
- extern int generic_write_check_limits(struct file *file, loff_t pos,
-@@ -3136,6 +3008,8 @@ extern ssize_t __generic_file_write_iter(struct kiocb *, struct iov_iter *);
- extern ssize_t generic_file_write_iter(struct kiocb *, struct iov_iter *);
- extern ssize_t generic_file_direct_write(struct kiocb *, struct iov_iter *);
- ssize_t generic_perform_write(struct kiocb *, struct iov_iter *);
-+ssize_t direct_write_fallback(struct kiocb *iocb, struct iov_iter *iter,
-+		ssize_t direct_written, ssize_t buffered_written);
- 
- ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
- 		rwf_t flags);
-@@ -3147,24 +3021,25 @@ ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb,
- 			    struct iov_iter *iter);
- 
- /* fs/splice.c */
-extern ssize_t generic_file_splice_read(struct file *, loff_t *,
-		struct pipe_inode_info *, size_t, unsigned int);
-+ssize_t filemap_splice_read(struct file *in, loff_t *ppos,
-+			    struct pipe_inode_info *pipe,
-+			    size_t len, unsigned int flags);
-+ssize_t copy_splice_read(struct file *in, loff_t *ppos,
-+			 struct pipe_inode_info *pipe,
-+			 size_t len, unsigned int flags);
- extern ssize_t iter_file_splice_write(struct pipe_inode_info *,
- 		struct file *, loff_t *, size_t, unsigned int);
-extern ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe,
-		struct file *out, loff_t *, size_t len, unsigned int flags);
-extern long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
-		loff_t *opos, size_t len, unsigned int flags);
- 
- 
- extern void
- file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping);
- extern loff_t noop_llseek(struct file *file, loff_t offset, int whence);
-#define no_llseek NULL
- extern loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize);
- extern loff_t generic_file_llseek(struct file *file, loff_t offset, int whence);
- extern loff_t generic_file_llseek_size(struct file *file, loff_t offset,
- 		int whence, loff_t maxsize, loff_t eof);
-+loff_t generic_llseek_cookie(struct file *file, loff_t offset, int whence,
-+			     u64 *cookie);
- extern loff_t fixed_size_llseek(struct file *file, loff_t offset,
- 		int whence, loff_t size);
- extern loff_t no_seek_end_llseek_size(struct file *, loff_t, int, loff_t);
-@@ -3189,7 +3064,7 @@ enum {
- ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
- 			     struct block_device *bdev, struct iov_iter *iter,
- 			     get_block_t get_block,
-			     dio_iodone_t end_io, dio_submit_t submit_io,
-+			     dio_iodone_t end_io,
- 			     int flags);
- 
- static inline ssize_t blockdev_direct_IO(struct kiocb *iocb,
-@@ -3198,11 +3073,13 @@ static inline ssize_t blockdev_direct_IO(struct kiocb *iocb,
- 					 get_block_t get_block)
- {
- 	return __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, iter,
-			get_block, NULL, NULL, DIO_LOCKING | DIO_SKIP_HOLES);
-+			get_block, NULL, DIO_LOCKING | DIO_SKIP_HOLES);
- }
- #endif
- 
-+bool inode_dio_finished(const struct inode *inode);
- void inode_dio_wait(struct inode *inode);
-+void inode_dio_wait_interruptible(struct inode *inode);
- 
- /**
-  * inode_dio_begin - signal start of a direct I/O requests
-@@ -3226,14 +3103,9 @@ static inline void inode_dio_begin(struct inode *inode)
- static inline void inode_dio_end(struct inode *inode)
- {
- 	if (atomic_dec_and_test(&inode->i_dio_count))
-		wake_up_bit(&inode->i_state, __I_DIO_WAKEUP);
-+		wake_up_var(&inode->i_dio_count);
- }
- 
-/*
- * Warn about a page cache invalidation failure diring a direct I/O write.
- */
-void dio_warn_stale_pagecache(struct file *filp);
-
- extern void inode_set_flags(struct inode *inode, unsigned int flags,
- 			    unsigned int mask);
- 
-@@ -3241,16 +3113,23 @@ extern const struct file_operations generic_ro_fops;
- 
- #define special_file(m) (S_ISCHR(m)||S_ISBLK(m)||S_ISFIFO(m)||S_ISSOCK(m))
- 
-extern int readlink_copy(char __user *, int, const char *);
-+extern int readlink_copy(char __user *, int, const char *, int);
- extern int page_readlink(struct dentry *, char __user *, int);
-+extern const char *page_get_link_raw(struct dentry *, struct inode *,
-+				     struct delayed_call *);
- extern const char *page_get_link(struct dentry *, struct inode *,
- 				 struct delayed_call *);
- extern void page_put_link(void *);
- extern int page_symlink(struct inode *inode, const char *symname, int len);
- extern const struct inode_operations page_symlink_inode_operations;
- extern void kfree_link(void *);
-void generic_fillattr(struct user_namespace *, struct inode *, struct kstat *);
-+void fill_mg_cmtime(struct kstat *stat, u32 request_mask, struct inode *inode);
-+void generic_fillattr(struct mnt_idmap *, u32, struct inode *, struct kstat *);
- void generic_fill_statx_attr(struct inode *inode, struct kstat *stat);
-+void generic_fill_statx_atomic_writes(struct kstat *stat,
-+				      unsigned int unit_min,
-+				      unsigned int unit_max,
-+				      unsigned int unit_max_opt);
- extern int vfs_getattr_nosec(const struct path *, struct kstat *, u32, unsigned int);
- extern int vfs_getattr(const struct path *, struct kstat *, u32, unsigned int);
- void __inode_add_bytes(struct inode *inode, loff_t bytes);
-@@ -3288,45 +3167,57 @@ extern int vfs_readlink(struct dentry *, char __user *, int);
- extern struct file_system_type *get_filesystem(struct file_system_type *fs);
- extern void put_filesystem(struct file_system_type *fs);
- extern struct file_system_type *get_fs_type(const char *name);
-extern struct super_block *get_super(struct block_device *);
-extern struct super_block *get_active_super(struct block_device *bdev);
- extern void drop_super(struct super_block *sb);
- extern void drop_super_exclusive(struct super_block *sb);
-extern void iterate_supers(void (*)(struct super_block *, void *), void *);
-+extern void iterate_supers(void (*f)(struct super_block *, void *), void *arg);
- extern void iterate_supers_type(struct file_system_type *,
- 			        void (*)(struct super_block *, void *), void *);
-+void filesystems_freeze(bool freeze_all);
-+void filesystems_thaw(void);
-+
-+void end_dirop(struct dentry *de);
- 
- extern int dcache_dir_open(struct inode *, struct file *);
- extern int dcache_dir_close(struct inode *, struct file *);
- extern loff_t dcache_dir_lseek(struct file *, loff_t, int);
- extern int dcache_readdir(struct file *, struct dir_context *);
-extern int simple_setattr(struct user_namespace *, struct dentry *,
-+extern int simple_setattr(struct mnt_idmap *, struct dentry *,
- 			  struct iattr *);
-extern int simple_getattr(struct user_namespace *, const struct path *,
-+extern int simple_getattr(struct mnt_idmap *, const struct path *,
- 			  struct kstat *, u32, unsigned int);
- extern int simple_statfs(struct dentry *, struct kstatfs *);
- extern int simple_open(struct inode *inode, struct file *file);
- extern int simple_link(struct dentry *, struct inode *, struct dentry *);
- extern int simple_unlink(struct inode *, struct dentry *);
- extern int simple_rmdir(struct inode *, struct dentry *);
-+extern void __simple_unlink(struct inode *, struct dentry *);
-+extern void __simple_rmdir(struct inode *, struct dentry *);
-+void simple_rename_timestamp(struct inode *old_dir, struct dentry *old_dentry,
-+			     struct inode *new_dir, struct dentry *new_dentry);
- extern int simple_rename_exchange(struct inode *old_dir, struct dentry *old_dentry,
- 				  struct inode *new_dir, struct dentry *new_dentry);
-extern int simple_rename(struct user_namespace *, struct inode *,
-+extern int simple_rename(struct mnt_idmap *, struct inode *,
- 			 struct dentry *, struct inode *, struct dentry *,
- 			 unsigned int);
- extern void simple_recursive_removal(struct dentry *,
-                               void (*callback)(struct dentry *));
-+extern void simple_remove_by_name(struct dentry *, const char *,
-+                              void (*callback)(struct dentry *));
-+extern void locked_recursive_removal(struct dentry *,
-+                              void (*callback)(struct dentry *));
- extern int noop_fsync(struct file *, loff_t, loff_t, int);
- extern ssize_t noop_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
- extern int simple_empty(struct dentry *);
-extern int simple_write_begin(struct file *file, struct address_space *mapping,
-			loff_t pos, unsigned len,
-			struct page **pagep, void **fsdata);
-+extern int simple_write_begin(const struct kiocb *iocb,
-+			      struct address_space *mapping,
-+			      loff_t pos, unsigned len,
-+			      struct folio **foliop, void **fsdata);
- extern const struct address_space_operations ram_aops;
- extern int always_delete_dentry(const struct dentry *);
- extern struct inode *alloc_anon_inode(struct super_block *);
-extern int simple_nosetlease(struct file *, long, struct file_lock **, void **);
-extern const struct dentry_operations simple_dentry_operations;
-+struct inode *anon_inode_make_secure_inode(struct super_block *sb, const char *name,
-+					   const struct inode *context_inode);
-+extern int simple_nosetlease(struct file *, int, struct file_lease **, void **);
- 
- extern struct dentry *simple_lookup(struct inode *, struct dentry *, unsigned int flags);
- extern ssize_t generic_read_dir(struct file *, char __user *, size_t, loff_t *);
-@@ -3340,31 +3231,110 @@ extern int simple_fill_super(struct super_block *, unsigned long,
- 			     const struct tree_descr *);
- extern int simple_pin_fs(struct file_system_type *, struct vfsmount **mount, int *count);
- extern void simple_release_fs(struct vfsmount **mount, int *count);
-+struct dentry *simple_start_creating(struct dentry *, const char *);
-+void simple_done_creating(struct dentry *);
- 
- extern ssize_t simple_read_from_buffer(void __user *to, size_t count,
- 			loff_t *ppos, const void *from, size_t available);
- extern ssize_t simple_write_to_buffer(void *to, size_t available, loff_t *ppos,
- 		const void __user *from, size_t count);
- 
-+struct offset_ctx {
-+	struct maple_tree	mt;
-+	unsigned long		next_offset;
-+};
-+
-+void simple_offset_init(struct offset_ctx *octx);
-+int simple_offset_add(struct offset_ctx *octx, struct dentry *dentry);
-+void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry);
-+int simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry,
-+			 struct inode *new_dir, struct dentry *new_dentry);
-+int simple_offset_rename_exchange(struct inode *old_dir,
-+				  struct dentry *old_dentry,
-+				  struct inode *new_dir,
-+				  struct dentry *new_dentry);
-+void simple_offset_destroy(struct offset_ctx *octx);
-+
-+extern const struct file_operations simple_offset_dir_operations;
-+
- extern int __generic_file_fsync(struct file *, loff_t, loff_t, int);
- extern int generic_file_fsync(struct file *, loff_t, loff_t, int);
- 
- extern int generic_check_addressable(unsigned, u64);
- 
-extern void generic_set_encrypted_ci_d_ops(struct dentry *dentry);
-+extern void generic_set_sb_d_ops(struct super_block *sb);
-+extern int generic_ci_match(const struct inode *parent,
-+			    const struct qstr *name,
-+			    const struct qstr *folded_name,
-+			    const u8 *de_name, u32 de_name_len);
- 
-int may_setattr(struct user_namespace *mnt_userns, struct inode *inode,
-+#if IS_ENABLED(CONFIG_UNICODE)
-+int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str);
-+int generic_ci_d_compare(const struct dentry *dentry, unsigned int len,
-+			 const char *str, const struct qstr *name);
-+
-+/**
-+ * generic_ci_validate_strict_name - Check if a given name is suitable
-+ * for a directory
-+ *
-+ * This functions checks if the proposed filename is valid for the
-+ * parent directory. That means that only valid UTF-8 filenames will be
-+ * accepted for casefold directories from filesystems created with the
-+ * strict encoding flag.  That also means that any name will be
-+ * accepted for directories that doesn't have casefold enabled, or
-+ * aren't being strict with the encoding.
-+ *
-+ * @dir: inode of the directory where the new file will be created
-+ * @name: name of the new file
-+ *
-+ * Return:
-+ * * True: if the filename is suitable for this directory. It can be
-+ *   true if a given name is not suitable for a strict encoding
-+ *   directory, but the directory being used isn't strict
-+ * * False if the filename isn't suitable for this directory. This only
-+ *   happens when a directory is casefolded and the filesystem is strict
-+ *   about its encoding.
-+ */
-+static inline bool generic_ci_validate_strict_name(struct inode *dir,
-+						   const struct qstr *name)
-+{
-+	if (!IS_CASEFOLDED(dir) || !sb_has_strict_encoding(dir->i_sb))
-+		return true;
-+
-+	/*
-+	 * A casefold dir must have a encoding set, unless the filesystem
-+	 * is corrupted
-+	 */
-+	if (WARN_ON_ONCE(!dir->i_sb->s_encoding))
-+		return true;
-+
-+	return !utf8_validate(dir->i_sb->s_encoding, name);
-+}
-+#else
-+static inline bool generic_ci_validate_strict_name(struct inode *dir,
-+						   const struct qstr *name)
-+{
-+	return true;
-+}
-+#endif
-+
-+int may_setattr(struct mnt_idmap *idmap, struct inode *inode,
- 		unsigned int ia_valid);
-int setattr_prepare(struct user_namespace *, struct dentry *, struct iattr *);
-+int setattr_prepare(struct mnt_idmap *, struct dentry *, struct iattr *);
- extern int inode_newsize_ok(const struct inode *, loff_t offset);
-void setattr_copy(struct user_namespace *, struct inode *inode,
-+void setattr_copy(struct mnt_idmap *, struct inode *inode,
- 		  const struct iattr *attr);
- 
- extern int file_update_time(struct file *file);
- 
-+static inline bool file_is_dax(const struct file *file)
-+{
-+	return file && IS_DAX(file->f_mapping->host);
-+}
-+
- static inline bool vma_is_dax(const struct vm_area_struct *vma)
- {
-	return vma->vm_file && IS_DAX(vma->vm_file->f_mapping->host);
-+	return file_is_dax(vma->vm_file);
- }
- 
- static inline bool vma_is_fsdax(struct vm_area_struct *vma)
-@@ -3395,7 +3365,8 @@ static inline int iocb_flags(struct file *file)
- 	return res;
- }
- 
-static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags)
-+static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags,
-+				     int rw_type)
- {
- 	int kiocb_flags = 0;
- 
-@@ -3406,34 +3377,41 @@ static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags)
- 		return 0;
- 	if (unlikely(flags & ~RWF_SUPPORTED))
- 		return -EOPNOTSUPP;
-+	if (unlikely((flags & RWF_APPEND) && (flags & RWF_NOAPPEND)))
-+		return -EINVAL;
- 
- 	if (flags & RWF_NOWAIT) {
- 		if (!(ki->ki_filp->f_mode & FMODE_NOWAIT))
- 			return -EOPNOTSUPP;
-		kiocb_flags |= IOCB_NOIO;
-+	}
-+	if (flags & RWF_ATOMIC) {
-+		if (rw_type != WRITE)
-+			return -EOPNOTSUPP;
-+		if (!(ki->ki_filp->f_mode & FMODE_CAN_ATOMIC_WRITE))
-+			return -EOPNOTSUPP;
-+	}
-+	if (flags & RWF_DONTCACHE) {
-+		/* file system must support it */
-+		if (!(ki->ki_filp->f_op->fop_flags & FOP_DONTCACHE))
-+			return -EOPNOTSUPP;
-+		/* DAX mappings not supported */
-+		if (IS_DAX(ki->ki_filp->f_mapping->host))
-+			return -EOPNOTSUPP;
- 	}
- 	kiocb_flags |= (__force int) (flags & RWF_SUPPORTED);
- 	if (flags & RWF_SYNC)
- 		kiocb_flags |= IOCB_DSYNC;
- 
-+	if ((flags & RWF_NOAPPEND) && (ki->ki_flags & IOCB_APPEND)) {
-+		if (IS_APPEND(file_inode(ki->ki_filp)))
-+			return -EPERM;
-+		ki->ki_flags &= ~IOCB_APPEND;
-+	}
-+
- 	ki->ki_flags |= kiocb_flags;
- 	return 0;
- }
- 
-static inline ino_t parent_ino(struct dentry *dentry)
-{
-	ino_t res;
-
-	/*
-	 * Don't strictly need d_lock here? If the parent ino could change
-	 * then surely we'd have a deeper race in the caller?
-	 */
-	spin_lock(&dentry->d_lock);
-	res = dentry->d_parent->d_inode->i_ino;
-	spin_unlock(&dentry->d_lock);
-	return res;
-}
-
- /* Transaction based IO helpers */
- 
- /*
-@@ -3471,7 +3449,7 @@ void simple_transaction_set(struct file *file, size_t n);
-  * All attributes contain a text representation of a numeric value
-  * that are accessed with the get() and set() functions.
-  */
-#define DEFINE_SIMPLE_ATTRIBUTE(__fops, __get, __set, __fmt)		\
-+#define DEFINE_SIMPLE_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, __is_signed)	\
- static int __fops ## _open(struct inode *inode, struct file *file)	\
- {									\
- 	__simple_attr_check_format(__fmt, 0ull);			\
-@@ -3482,10 +3460,16 @@ static const struct file_operations __fops = {				\
- 	.open	 = __fops ## _open,					\
- 	.release = simple_attr_release,					\
- 	.read	 = simple_attr_read,					\
-	.write	 = simple_attr_write,					\
-+	.write	 = (__is_signed) ? simple_attr_write_signed : simple_attr_write,	\
- 	.llseek	 = generic_file_llseek,					\
- }
- 
-+#define DEFINE_SIMPLE_ATTRIBUTE(__fops, __get, __set, __fmt)		\
-+	DEFINE_SIMPLE_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, false)
-+
-+#define DEFINE_SIMPLE_ATTRIBUTE_SIGNED(__fops, __get, __set, __fmt)	\
-+	DEFINE_SIMPLE_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, true)
-+
- static inline __printf(1, 2)
- void __simple_attr_check_format(const char *fmt, ...)
- {
-@@ -3500,29 +3484,29 @@ ssize_t simple_attr_read(struct file *file, char __user *buf,
- 			 size_t len, loff_t *ppos);
- ssize_t simple_attr_write(struct file *file, const char __user *buf,
- 			  size_t len, loff_t *ppos);
-+ssize_t simple_attr_write_signed(struct file *file, const char __user *buf,
-+				 size_t len, loff_t *ppos);
- 
- struct ctl_table;
- int __init list_bdev_fs_names(char *buf, size_t size);
- 
- #define __FMODE_EXEC		((__force int) FMODE_EXEC)
-#define __FMODE_NONOTIFY	((__force int) FMODE_NONOTIFY)
- 
- #define ACC_MODE(x) ("\004\002\006\006"[(x)&O_ACCMODE])
-#define OPEN_FMODE(flag) ((__force fmode_t)(((flag + 1) & O_ACCMODE) | \
-					    (flag & __FMODE_NONOTIFY)))
-+#define OPEN_FMODE(flag) ((__force fmode_t)((flag + 1) & O_ACCMODE))
- 
- static inline bool is_sxid(umode_t mode)
- {
-	return (mode & S_ISUID) || ((mode & S_ISGID) && (mode & S_IXGRP));
-+	return mode & (S_ISUID | S_ISGID);
- }
- 
-static inline int check_sticky(struct user_namespace *mnt_userns,
-+static inline int check_sticky(struct mnt_idmap *idmap,
- 			       struct inode *dir, struct inode *inode)
- {
- 	if (!(dir->i_mode & S_ISVTX))
- 		return 0;
- 
-	return __check_sticky(mnt_userns, dir, inode);
-+	return __check_sticky(idmap, dir, inode);
- }
- 
- static inline void inode_has_no_xattr(struct inode *inode)
-@@ -3540,17 +3524,17 @@ static inline bool dir_emit(struct dir_context *ctx,
- 			    const char *name, int namelen,
- 			    u64 ino, unsigned type)
- {
-	return ctx->actor(ctx, name, namelen, ctx->pos, ino, type) == 0;
-+	return ctx->actor(ctx, name, namelen, ctx->pos, ino, type);
- }
- static inline bool dir_emit_dot(struct file *file, struct dir_context *ctx)
- {
- 	return ctx->actor(ctx, ".", 1, ctx->pos,
-			  file->f_path.dentry->d_inode->i_ino, DT_DIR) == 0;
-+			  file->f_path.dentry->d_inode->i_ino, DT_DIR);
- }
- static inline bool dir_emit_dotdot(struct file *file, struct dir_context *ctx)
- {
- 	return ctx->actor(ctx, "..", 2, ctx->pos,
-			  parent_ino(file->f_path.dentry), DT_DIR) == 0;
-+			  d_parent_ino(file->f_path.dentry), DT_DIR);
- }
- static inline bool dir_emit_dots(struct file *file, struct dir_context *ctx)
- {
-@@ -3589,4 +3573,37 @@ extern int vfs_fadvise(struct file *file, loff_t offset, loff_t len,
- extern int generic_fadvise(struct file *file, loff_t offset, loff_t len,
- 			   int advice);
- 
-+static inline bool vfs_empty_path(int dfd, const char __user *path)
-+{
-+	char c;
-+
-+	if (dfd < 0)
-+		return false;
-+
-+	/* We now allow NULL to be used for empty path. */
-+	if (!path)
-+		return true;
-+
-+	if (unlikely(get_user(c, path)))
-+		return false;
-+
-+	return !c;
-+}
-+
-+int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter);
-+
-+static inline bool extensible_ioctl_valid(unsigned int cmd_a,
-+					  unsigned int cmd_b, size_t min_size)
-+{
-+	if (_IOC_DIR(cmd_a) != _IOC_DIR(cmd_b))
-+		return false;
-+	if (_IOC_TYPE(cmd_a) != _IOC_TYPE(cmd_b))
-+		return false;
-+	if (_IOC_NR(cmd_a) != _IOC_NR(cmd_b))
-+		return false;
-+	if (_IOC_SIZE(cmd_a) < min_size)
-+		return false;
-+	return true;
-+}
-+
- #endif /* _LINUX_FS_H */
-diff --git a/include/linux/fs/super.h b/include/linux/fs/super.h
-new file mode 100644
-index 000000000000..f21ffbb6dea5
--- /dev/null
-+++ b/include/linux/fs/super.h
-@@ -0,0 +1,238 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_FS_SUPER_H
-+#define _LINUX_FS_SUPER_H
-+
-+#include <linux/fs/super_types.h>
-+#include <linux/unicode.h>
-+
-+/*
-+ * These are internal functions, please use sb_start_{write,pagefault,intwrite}
-+ * instead.
-+ */
-+static inline void __sb_end_write(struct super_block *sb, int level)
-+{
-+	percpu_up_read(sb->s_writers.rw_sem + level - 1);
-+}
-+
-+static inline void __sb_start_write(struct super_block *sb, int level)
-+{
-+	percpu_down_read_freezable(sb->s_writers.rw_sem + level - 1, true);
-+}
-+
-+static inline bool __sb_start_write_trylock(struct super_block *sb, int level)
-+{
-+	return percpu_down_read_trylock(sb->s_writers.rw_sem + level - 1);
-+}
-+
-+#define __sb_writers_acquired(sb, lev) \
-+	percpu_rwsem_acquire(&(sb)->s_writers.rw_sem[(lev) - 1], 1, _THIS_IP_)
-+#define __sb_writers_release(sb, lev) \
-+	percpu_rwsem_release(&(sb)->s_writers.rw_sem[(lev) - 1], _THIS_IP_)
-+
-+/**
-+ * __sb_write_started - check if sb freeze level is held
-+ * @sb: the super we write to
-+ * @level: the freeze level
-+ *
-+ * * > 0 - sb freeze level is held
-+ * *   0 - sb freeze level is not held
-+ * * < 0 - !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN
-+ */
-+static inline int __sb_write_started(const struct super_block *sb, int level)
-+{
-+	return lockdep_is_held_type(sb->s_writers.rw_sem + level - 1, 1);
-+}
-+
-+/**
-+ * sb_write_started - check if SB_FREEZE_WRITE is held
-+ * @sb: the super we write to
-+ *
-+ * May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
-+ */
-+static inline bool sb_write_started(const struct super_block *sb)
-+{
-+	return __sb_write_started(sb, SB_FREEZE_WRITE);
-+}
-+
-+/**
-+ * sb_write_not_started - check if SB_FREEZE_WRITE is not held
-+ * @sb: the super we write to
-+ *
-+ * May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
-+ */
-+static inline bool sb_write_not_started(const struct super_block *sb)
-+{
-+	return __sb_write_started(sb, SB_FREEZE_WRITE) <= 0;
-+}
-+
-+/**
-+ * sb_end_write - drop write access to a superblock
-+ * @sb: the super we wrote to
-+ *
-+ * Decrement number of writers to the filesystem. Wake up possible waiters
-+ * wanting to freeze the filesystem.
-+ */
-+static inline void sb_end_write(struct super_block *sb)
-+{
-+	__sb_end_write(sb, SB_FREEZE_WRITE);
-+}
-+
-+/**
-+ * sb_end_pagefault - drop write access to a superblock from a page fault
-+ * @sb: the super we wrote to
-+ *
-+ * Decrement number of processes handling write page fault to the filesystem.
-+ * Wake up possible waiters wanting to freeze the filesystem.
-+ */
-+static inline void sb_end_pagefault(struct super_block *sb)
-+{
-+	__sb_end_write(sb, SB_FREEZE_PAGEFAULT);
-+}
-+
-+/**
-+ * sb_end_intwrite - drop write access to a superblock for internal fs purposes
-+ * @sb: the super we wrote to
-+ *
-+ * Decrement fs-internal number of writers to the filesystem.  Wake up possible
-+ * waiters wanting to freeze the filesystem.
-+ */
-+static inline void sb_end_intwrite(struct super_block *sb)
-+{
-+	__sb_end_write(sb, SB_FREEZE_FS);
-+}
-+
-+/**
-+ * sb_start_write - get write access to a superblock
-+ * @sb: the super we write to
-+ *
-+ * When a process wants to write data or metadata to a file system (i.e. dirty
-+ * a page or an inode), it should embed the operation in a sb_start_write() -
-+ * sb_end_write() pair to get exclusion against file system freezing. This
-+ * function increments number of writers preventing freezing. If the file
-+ * system is already frozen, the function waits until the file system is
-+ * thawed.
-+ *
-+ * Since freeze protection behaves as a lock, users have to preserve
-+ * ordering of freeze protection and other filesystem locks. Generally,
-+ * freeze protection should be the outermost lock. In particular, we have:
-+ *
-+ * sb_start_write
-+ *   -> i_rwsem			(write path, truncate, directory ops, ...)
-+ *   -> s_umount		(freeze_super, thaw_super)
-+ */
-+static inline void sb_start_write(struct super_block *sb)
-+{
-+	__sb_start_write(sb, SB_FREEZE_WRITE);
-+}
-+
-+DEFINE_GUARD(super_write,
-+	     struct super_block *,
-+	     sb_start_write(_T),
-+	     sb_end_write(_T))
-+
-+static inline bool sb_start_write_trylock(struct super_block *sb)
-+{
-+	return __sb_start_write_trylock(sb, SB_FREEZE_WRITE);
-+}
-+
-+/**
-+ * sb_start_pagefault - get write access to a superblock from a page fault
-+ * @sb: the super we write to
-+ *
-+ * When a process starts handling write page fault, it should embed the
-+ * operation into sb_start_pagefault() - sb_end_pagefault() pair to get
-+ * exclusion against file system freezing. This is needed since the page fault
-+ * is going to dirty a page. This function increments number of running page
-+ * faults preventing freezing. If the file system is already frozen, the
-+ * function waits until the file system is thawed.
-+ *
-+ * Since page fault freeze protection behaves as a lock, users have to preserve
-+ * ordering of freeze protection and other filesystem locks. It is advised to
-+ * put sb_start_pagefault() close to mmap_lock in lock ordering. Page fault
-+ * handling code implies lock dependency:
-+ *
-+ * mmap_lock
-+ *   -> sb_start_pagefault
-+ */
-+static inline void sb_start_pagefault(struct super_block *sb)
-+{
-+	__sb_start_write(sb, SB_FREEZE_PAGEFAULT);
-+}
-+
-+/**
-+ * sb_start_intwrite - get write access to a superblock for internal fs purposes
-+ * @sb: the super we write to
-+ *
-+ * This is the third level of protection against filesystem freezing. It is
-+ * free for use by a filesystem. The only requirement is that it must rank
-+ * below sb_start_pagefault.
-+ *
-+ * For example filesystem can call sb_start_intwrite() when starting a
-+ * transaction which somewhat eases handling of freezing for internal sources
-+ * of filesystem changes (internal fs threads, discarding preallocation on file
-+ * close, etc.).
-+ */
-+static inline void sb_start_intwrite(struct super_block *sb)
-+{
-+	__sb_start_write(sb, SB_FREEZE_FS);
-+}
-+
-+static inline bool sb_start_intwrite_trylock(struct super_block *sb)
-+{
-+	return __sb_start_write_trylock(sb, SB_FREEZE_FS);
-+}
-+
-+static inline bool sb_rdonly(const struct super_block *sb)
-+{
-+	return sb->s_flags & SB_RDONLY;
-+}
-+
-+static inline bool sb_is_blkdev_sb(struct super_block *sb)
-+{
-+	return IS_ENABLED(CONFIG_BLOCK) && sb == blockdev_superblock;
-+}
-+
-+#if IS_ENABLED(CONFIG_UNICODE)
-+static inline struct unicode_map *sb_encoding(const struct super_block *sb)
-+{
-+	return sb->s_encoding;
-+}
-+
-+/* Compare if two super blocks have the same encoding and flags */
-+static inline bool sb_same_encoding(const struct super_block *sb1,
-+				    const struct super_block *sb2)
-+{
-+	if (sb1->s_encoding == sb2->s_encoding)
-+		return true;
-+
-+	return (sb1->s_encoding && sb2->s_encoding &&
-+		(sb1->s_encoding->version == sb2->s_encoding->version) &&
-+		(sb1->s_encoding_flags == sb2->s_encoding_flags));
-+}
-+#else
-+static inline struct unicode_map *sb_encoding(const struct super_block *sb)
-+{
-+	return NULL;
-+}
-+
-+static inline bool sb_same_encoding(const struct super_block *sb1,
-+				    const struct super_block *sb2)
-+{
-+	return true;
-+}
-+#endif
-+
-+static inline bool sb_has_encoding(const struct super_block *sb)
-+{
-+	return !!sb_encoding(sb);
-+}
-+
-+int sb_set_blocksize(struct super_block *sb, int size);
-+int __must_check sb_min_blocksize(struct super_block *sb, int size);
-+
-+int freeze_super(struct super_block *super, enum freeze_holder who,
-+		 const void *freeze_owner);
-+int thaw_super(struct super_block *super, enum freeze_holder who,
-+	       const void *freeze_owner);
-+
-+#endif /* _LINUX_FS_SUPER_H */
-diff --git a/include/linux/fs/super_types.h b/include/linux/fs/super_types.h
-new file mode 100644
-index 000000000000..6bd3009e09b3
--- /dev/null
-+++ b/include/linux/fs/super_types.h
-@@ -0,0 +1,336 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_FS_SUPER_TYPES_H
-+#define _LINUX_FS_SUPER_TYPES_H
-+
-+#include <linux/fs_dirent.h>
-+#include <linux/errseq.h>
-+#include <linux/list_lru.h>
-+#include <linux/list.h>
-+#include <linux/list_bl.h>
-+#include <linux/llist.h>
-+#include <linux/uidgid.h>
-+#include <linux/uuid.h>
-+#include <linux/percpu-rwsem.h>
-+#include <linux/workqueue_types.h>
-+#include <linux/quota.h>
-+
-+struct backing_dev_info;
-+struct block_device;
-+struct dentry;
-+struct dentry_operations;
-+struct dquot_operations;
-+struct export_operations;
-+struct file;
-+struct file_system_type;
-+struct fscrypt_operations;
-+struct fsnotify_sb_info;
-+struct fsverity_operations;
-+struct kstatfs;
-+struct mount;
-+struct mtd_info;
-+struct quotactl_ops;
-+struct shrinker;
-+struct unicode_map;
-+struct user_namespace;
-+struct workqueue_struct;
-+struct writeback_control;
-+struct xattr_handler;
-+
-+extern struct super_block *blockdev_superblock;
-+
-+/* Possible states of 'frozen' field */
-+enum {
-+	SB_UNFROZEN		= 0,	/* FS is unfrozen */
-+	SB_FREEZE_WRITE		= 1,	/* Writes, dir ops, ioctls frozen */
-+	SB_FREEZE_PAGEFAULT	= 2,	/* Page faults stopped as well */
-+	SB_FREEZE_FS		= 3,	/* For internal FS use (e.g. to stop internal threads if needed) */
-+	SB_FREEZE_COMPLETE	= 4,	/* ->freeze_fs finished successfully */
-+};
-+
-+#define SB_FREEZE_LEVELS (SB_FREEZE_COMPLETE - 1)
-+
-+struct sb_writers {
-+	unsigned short			frozen;		/* Is sb frozen? */
-+	int				freeze_kcount;	/* How many kernel freeze requests? */
-+	int				freeze_ucount;	/* How many userspace freeze requests? */
-+	const void			*freeze_owner;	/* Owner of the freeze */
-+	struct percpu_rw_semaphore	rw_sem[SB_FREEZE_LEVELS];
-+};
-+
-+/**
-+ * enum freeze_holder - holder of the freeze
-+ * @FREEZE_HOLDER_KERNEL: kernel wants to freeze or thaw filesystem
-+ * @FREEZE_HOLDER_USERSPACE: userspace wants to freeze or thaw filesystem
-+ * @FREEZE_MAY_NEST: whether nesting freeze and thaw requests is allowed
-+ * @FREEZE_EXCL: a freeze that can only be undone by the owner
-+ *
-+ * Indicate who the owner of the freeze or thaw request is and whether
-+ * the freeze needs to be exclusive or can nest.
-+ * Without @FREEZE_MAY_NEST, multiple freeze and thaw requests from the
-+ * same holder aren't allowed. It is however allowed to hold a single
-+ * @FREEZE_HOLDER_USERSPACE and a single @FREEZE_HOLDER_KERNEL freeze at
-+ * the same time. This is relied upon by some filesystems during online
-+ * repair or similar.
-+ */
-+enum freeze_holder {
-+	FREEZE_HOLDER_KERNEL	= (1U << 0),
-+	FREEZE_HOLDER_USERSPACE	= (1U << 1),
-+	FREEZE_MAY_NEST		= (1U << 2),
-+	FREEZE_EXCL		= (1U << 3),
-+};
-+
-+struct super_operations {
-+	struct inode *(*alloc_inode)(struct super_block *sb);
-+	void (*destroy_inode)(struct inode *inode);
-+	void (*free_inode)(struct inode *inode);
-+	void (*dirty_inode)(struct inode *inode, int flags);
-+	int (*write_inode)(struct inode *inode, struct writeback_control *wbc);
-+	int (*drop_inode)(struct inode *inode);
-+	void (*evict_inode)(struct inode *inode);
-+	void (*put_super)(struct super_block *sb);
-+	int (*sync_fs)(struct super_block *sb, int wait);
-+	int (*freeze_super)(struct super_block *sb, enum freeze_holder who,
-+			    const void *owner);
-+	int (*freeze_fs)(struct super_block *sb);
-+	int (*thaw_super)(struct super_block *sb, enum freeze_holder who,
-+			  const void *owner);
-+	int (*unfreeze_fs)(struct super_block *sb);
-+	int (*statfs)(struct dentry *dentry, struct kstatfs *kstatfs);
-+	int (*remount_fs) (struct super_block *, int *, char *);
-+	void (*umount_begin)(struct super_block *sb);
-+
-+	int (*show_options)(struct seq_file *seq, struct dentry *dentry);
-+	int (*show_devname)(struct seq_file *seq, struct dentry *dentry);
-+	int (*show_path)(struct seq_file *seq, struct dentry *dentry);
-+	int (*show_stats)(struct seq_file *seq, struct dentry *dentry);
-+#ifdef CONFIG_QUOTA
-+	ssize_t (*quota_read)(struct super_block *sb, int type, char *data,
-+			      size_t len, loff_t off);
-+	ssize_t (*quota_write)(struct super_block *sb, int type,
-+			       const char *data, size_t len, loff_t off);
-+	struct dquot __rcu **(*get_dquots)(struct inode *inode);
-+#endif
-+	long (*nr_cached_objects)(struct super_block *sb,
-+				  struct shrink_control *sc);
-+	long (*free_cached_objects)(struct super_block *sb,
-+				    struct shrink_control *sc);
-+	/*
-+	 * If a filesystem can support graceful removal of a device and
-+	 * continue read-write operations, implement this callback.
-+	 *
-+	 * Return 0 if the filesystem can continue read-write.
-+	 * Non-zero return value or no such callback means the fs will be shutdown
-+	 * as usual.
-+	 */
-+	int (*remove_bdev)(struct super_block *sb, struct block_device *bdev);
-+	void (*shutdown)(struct super_block *sb);
-+};
-+
-+struct super_block {
-+	struct list_head			s_list;		/* Keep this first */
-+	dev_t					s_dev;		/* search index; _not_ kdev_t */
-+	unsigned char				s_blocksize_bits;
-+	unsigned long				s_blocksize;
-+	loff_t					s_maxbytes;	/* Max file size */
-+	struct file_system_type			*s_type;
-+	const struct super_operations		*s_op;
-+	const struct dquot_operations		*dq_op;
-+	const struct quotactl_ops		*s_qcop;
-+	const struct export_operations		*s_export_op;
-+	unsigned long				s_flags;
-+	unsigned long				s_iflags;	/* internal SB_I_* flags */
-+	unsigned long				s_magic;
-+	struct dentry				*s_root;
-+	struct rw_semaphore			s_umount;
-+	int					s_count;
-+	atomic_t				s_active;
-+#ifdef CONFIG_SECURITY
-+	void					*s_security;
-+#endif
-+	const struct xattr_handler		*const *s_xattr;
-+#ifdef CONFIG_FS_ENCRYPTION
-+	const struct fscrypt_operations		*s_cop;
-+	struct fscrypt_keyring			*s_master_keys; /* master crypto keys in use */
-+#endif
-+#ifdef CONFIG_FS_VERITY
-+	const struct fsverity_operations	*s_vop;
-+#endif
-+#if IS_ENABLED(CONFIG_UNICODE)
-+	struct unicode_map			*s_encoding;
-+	__u16					s_encoding_flags;
-+#endif
-+	struct hlist_bl_head			s_roots;	/* alternate root dentries for NFS */
-+	struct mount				*s_mounts;	/* list of mounts; _not_ for fs use */
-+	struct block_device			*s_bdev;	/* can go away once we use an accessor for @s_bdev_file */
-+	struct file				*s_bdev_file;
-+	struct backing_dev_info 		*s_bdi;
-+	struct mtd_info				*s_mtd;
-+	struct hlist_node			s_instances;
-+	unsigned int				s_quota_types;	/* Bitmask of supported quota types */
-+	struct quota_info			s_dquot;	/* Diskquota specific options */
-+
-+	struct sb_writers			s_writers;
-+
-+	/*
-+	 * Keep s_fs_info, s_time_gran, s_fsnotify_mask, and
-+	 * s_fsnotify_info together for cache efficiency. They are frequently
-+	 * accessed and rarely modified.
-+	 */
-+	void					*s_fs_info;	/* Filesystem private info */
-+
-+	/* Granularity of c/m/atime in ns (cannot be worse than a second) */
-+	u32					s_time_gran;
-+	/* Time limits for c/m/atime in seconds */
-+	time64_t				s_time_min;
-+	time64_t		   		s_time_max;
-+#ifdef CONFIG_FSNOTIFY
-+	u32					s_fsnotify_mask;
-+	struct fsnotify_sb_info			*s_fsnotify_info;
-+#endif
-+
-+	/*
-+	 * q: why are s_id and s_sysfs_name not the same? both are human
-+	 * readable strings that identify the filesystem
-+	 * a: s_id is allowed to change at runtime; it's used in log messages,
-+	 * and we want to when a device starts out as single device (s_id is dev
-+	 * name) but then a device is hot added and we have to switch to
-+	 * identifying it by UUID
-+	 * but s_sysfs_name is a handle for programmatic access, and can't
-+	 * change at runtime
-+	 */
-+	char					s_id[32];	/* Informational name */
-+	uuid_t					s_uuid;		/* UUID */
-+	u8					s_uuid_len;	/* Default 16, possibly smaller for weird filesystems */
-+
-+	/* if set, fs shows up under sysfs at /sys/fs/$FSTYP/s_sysfs_name */
-+	char					s_sysfs_name[UUID_STRING_LEN + 1];
-+
-+	unsigned int				s_max_links;
-+	unsigned int				s_d_flags;	/* default d_flags for dentries */
-+
-+	/*
-+	 * The next field is for VFS *only*. No filesystems have any business
-+	 * even looking at it. You had been warned.
-+	 */
-+	struct mutex				s_vfs_rename_mutex;	/* Kludge */
-+
-+	/*
-+	 * Filesystem subtype.  If non-empty the filesystem type field
-+	 * in /proc/mounts will be "type.subtype"
-+	 */
-+	const char				*s_subtype;
-+
-+	const struct dentry_operations		*__s_d_op; /* default d_op for dentries */
-+
-+	struct shrinker				*s_shrink;	/* per-sb shrinker handle */
-+
-+	/* Number of inodes with nlink == 0 but still referenced */
-+	atomic_long_t				s_remove_count;
-+
-+	/* Read-only state of the superblock is being changed */
-+	int					s_readonly_remount;
-+
-+	/* per-sb errseq_t for reporting writeback errors via syncfs */
-+	errseq_t s_wb_err;
-+
-+	/* AIO completions deferred from interrupt context */
-+	struct workqueue_struct			*s_dio_done_wq;
-+	struct hlist_head			s_pins;
-+
-+	/*
-+	 * Owning user namespace and default context in which to
-+	 * interpret filesystem uids, gids, quotas, device nodes,
-+	 * xattrs and security labels.
-+	 */
-+	struct user_namespace			*s_user_ns;
-+
-+	/*
-+	 * The list_lru structure is essentially just a pointer to a table
-+	 * of per-node lru lists, each of which has its own spinlock.
-+	 * There is no need to put them into separate cachelines.
-+	 */
-+	struct list_lru				s_dentry_lru;
-+	struct list_lru				s_inode_lru;
-+	struct rcu_head				rcu;
-+	struct work_struct			destroy_work;
-+
-+	struct mutex				s_sync_lock;	/* sync serialisation lock */
-+
-+	/*
-+	 * Indicates how deep in a filesystem stack this SB is
-+	 */
-+	int s_stack_depth;
-+
-+	/* s_inode_list_lock protects s_inodes */
-+	spinlock_t				s_inode_list_lock ____cacheline_aligned_in_smp;
-+	struct list_head			s_inodes;	/* all inodes */
-+
-+	spinlock_t				s_inode_wblist_lock;
-+	struct list_head			s_inodes_wb;	/* writeback inodes */
-+	long					s_min_writeback_pages;
-+} __randomize_layout;
-+
-+/*
-+ * sb->s_flags.  Note that these mirror the equivalent MS_* flags where
-+ * represented in both.
-+ */
-+#define SB_RDONLY       BIT(0)	/* Mount read-only */
-+#define SB_NOSUID       BIT(1)	/* Ignore suid and sgid bits */
-+#define SB_NODEV        BIT(2)	/* Disallow access to device special files */
-+#define SB_NOEXEC       BIT(3)	/* Disallow program execution */
-+#define SB_SYNCHRONOUS  BIT(4)	/* Writes are synced at once */
-+#define SB_MANDLOCK     BIT(6)	/* Allow mandatory locks on an FS */
-+#define SB_DIRSYNC      BIT(7)	/* Directory modifications are synchronous */
-+#define SB_NOATIME      BIT(10)	/* Do not update access times. */
-+#define SB_NODIRATIME   BIT(11)	/* Do not update directory access times */
-+#define SB_SILENT       BIT(15)
-+#define SB_POSIXACL     BIT(16)	/* Supports POSIX ACLs */
-+#define SB_INLINECRYPT  BIT(17)	/* Use blk-crypto for encrypted files */
-+#define SB_KERNMOUNT    BIT(22)	/* this is a kern_mount call */
-+#define SB_I_VERSION    BIT(23)	/* Update inode I_version field */
-+#define SB_LAZYTIME     BIT(25)	/* Update the on-disk [acm]times lazily */
-+
-+/* These sb flags are internal to the kernel */
-+#define SB_DEAD         BIT(21)
-+#define SB_DYING        BIT(24)
-+#define SB_FORCE        BIT(27)
-+#define SB_NOSEC        BIT(28)
-+#define SB_BORN         BIT(29)
-+#define SB_ACTIVE       BIT(30)
-+#define SB_NOUSER       BIT(31)
-+
-+/* These flags relate to encoding and casefolding */
-+#define SB_ENC_STRICT_MODE_FL		(1 << 0)
-+#define SB_ENC_NO_COMPAT_FALLBACK_FL	(1 << 1)
-+
-+#define sb_has_strict_encoding(sb) \
-+	(sb->s_encoding_flags & SB_ENC_STRICT_MODE_FL)
-+
-+#if IS_ENABLED(CONFIG_UNICODE)
-+#define sb_no_casefold_compat_fallback(sb) \
-+	(sb->s_encoding_flags & SB_ENC_NO_COMPAT_FALLBACK_FL)
-+#else
-+#define sb_no_casefold_compat_fallback(sb) (1)
-+#endif
-+
-+/* sb->s_iflags */
-+#define SB_I_CGROUPWB	0x00000001	/* cgroup-aware writeback enabled */
-+#define SB_I_NOEXEC	0x00000002	/* Ignore executables on this fs */
-+#define SB_I_NODEV	0x00000004	/* Ignore devices on this fs */
-+#define SB_I_STABLE_WRITES 0x00000008	/* don't modify blks until WB is done */
-+
-+/* sb->s_iflags to limit user namespace mounts */
-+#define SB_I_USERNS_VISIBLE		0x00000010 /* fstype already mounted */
-+#define SB_I_IMA_UNVERIFIABLE_SIGNATURE	0x00000020
-+#define SB_I_UNTRUSTED_MOUNTER		0x00000040
-+#define SB_I_EVM_HMAC_UNSUPPORTED	0x00000080
-+
-+#define SB_I_SKIP_SYNC	0x00000100	/* Skip superblock at global sync */
-+#define SB_I_PERSB_BDI	0x00000200	/* has a per-sb bdi */
-+#define SB_I_TS_EXPIRY_WARNED 0x00000400 /* warned about timestamp range expiry */
-+#define SB_I_RETIRED	0x00000800	/* superblock shouldn't be reused */
-+#define SB_I_NOUMASK	0x00001000	/* VFS does not apply umask */
-+#define SB_I_NOIDMAP	0x00002000	/* No idmapped mounts on this superblock */
-+#define SB_I_ALLOW_HSM	0x00004000	/* Allow HSM events on this superblock */
-+
-+#endif /* _LINUX_FS_SUPER_TYPES_H */
-diff --git a/include/linux/fs_context.h b/include/linux/fs_context.h
-index 13fa6f3df8e4..0d6c8a6d7be2 100644
--- a/include/linux/fs_context.h
-+++ b/include/linux/fs_context.h
-@@ -99,17 +99,17 @@ struct fs_context {
- 	const struct cred	*cred;		/* The mounter's credentials */
- 	struct p_log		log;		/* Logging buffer */
- 	const char		*source;	/* The source name (eg. dev path) */
-	void			*security;	/* Linux S&M options */
-+	void			*security;	/* LSM options */
- 	void			*s_fs_info;	/* Proposed s_fs_info */
- 	unsigned int		sb_flags;	/* Proposed superblock flags (SB_*) */
- 	unsigned int		sb_flags_mask;	/* Superblock flags that were changed */
- 	unsigned int		s_iflags;	/* OR'd with sb->s_iflags */
-	unsigned int		lsm_flags;	/* Information flags from the fs to the LSM */
- 	enum fs_context_purpose	purpose:8;
- 	enum fs_context_phase	phase:8;	/* The phase the context is in */
- 	bool			need_free:1;	/* Need to call ops->free() */
- 	bool			global:1;	/* Goes into &init_user_ns */
- 	bool			oldapi:1;	/* Coming from mount(2) */
-+	bool			exclusive:1;    /* create new superblock, reject existing one */
- };
- 
- struct fs_context_operations {
-@@ -134,30 +134,21 @@ extern struct fs_context *fs_context_for_submount(struct file_system_type *fs_ty
- 
- extern struct fs_context *vfs_dup_fs_context(struct fs_context *fc);
- extern int vfs_parse_fs_param(struct fs_context *fc, struct fs_parameter *param);
-extern int vfs_parse_fs_string(struct fs_context *fc, const char *key,
-			       const char *value, size_t v_size);
-+extern int vfs_parse_fs_qstr(struct fs_context *fc, const char *key,
-+				const struct qstr *value);
-+static inline int vfs_parse_fs_string(struct fs_context *fc, const char *key,
-+			       const char *value)
-+{
-+	return vfs_parse_fs_qstr(fc, key, value ? &QSTR(value) : NULL);
-+}
-+int vfs_parse_monolithic_sep(struct fs_context *fc, void *data,
-+			     char *(*sep)(char **));
- extern int generic_parse_monolithic(struct fs_context *fc, void *data);
- extern int vfs_get_tree(struct fs_context *fc);
- extern void put_fs_context(struct fs_context *fc);
- extern int vfs_parse_fs_param_source(struct fs_context *fc,
- 				     struct fs_parameter *param);
- extern void fc_drop_locked(struct fs_context *fc);
-int reconfigure_single(struct super_block *s,
-		       int flags, void *data);
-
-/*
- * sget() wrappers to be called from the ->get_tree() op.
- */
-enum vfs_get_super_keying {
-	vfs_get_single_super,	/* Only one such superblock may exist */
-	vfs_get_single_reconf_super, /* As above, but reconfigure if it exists */
-	vfs_get_keyed_super,	/* Superblocks with different s_fs_info keys may exist */
-	vfs_get_independent_super, /* Multiple independent superblocks may exist */
-};
-extern int vfs_get_super(struct fs_context *fc,
-			 enum vfs_get_super_keying keying,
-			 int (*fill_super)(struct super_block *sb,
-					   struct fs_context *fc));
- 
- extern int get_tree_nodev(struct fs_context *fc,
- 			 int (*fill_super)(struct super_block *sb,
-@@ -165,14 +156,19 @@ extern int get_tree_nodev(struct fs_context *fc,
- extern int get_tree_single(struct fs_context *fc,
- 			 int (*fill_super)(struct super_block *sb,
- 					   struct fs_context *fc));
-extern int get_tree_single_reconf(struct fs_context *fc,
-			 int (*fill_super)(struct super_block *sb,
-					   struct fs_context *fc));
- extern int get_tree_keyed(struct fs_context *fc,
- 			 int (*fill_super)(struct super_block *sb,
- 					   struct fs_context *fc),
- 			 void *key);
- 
-+int setup_bdev_super(struct super_block *sb, int sb_flags,
-+		struct fs_context *fc);
-+
-+#define GET_TREE_BDEV_QUIET_LOOKUP		0x0001
-+int get_tree_bdev_flags(struct fs_context *fc,
-+		int (*fill_super)(struct super_block *sb,
-+				  struct fs_context *fc), unsigned int flags);
-+
- extern int get_tree_bdev(struct fs_context *fc,
- 			       int (*fill_super)(struct super_block *sb,
- 						 struct fs_context *fc));
-@@ -195,10 +191,12 @@ struct fc_log {
- extern __attribute__((format(printf, 4, 5)))
- void logfc(struct fc_log *log, const char *prefix, char level, const char *fmt, ...);
- 
-#define __logfc(fc, l, fmt, ...) logfc((fc)->log.log, NULL, \
-					l, fmt, ## __VA_ARGS__)
-#define __plog(p, l, fmt, ...) logfc((p)->log, (p)->prefix, \
-					l, fmt, ## __VA_ARGS__)
-+#define __logfc(fc, l, fmt, ...) \
-+	logfc((fc)->log.log, NULL, (l), (fmt), ## __VA_ARGS__)
-+#define __plogp(p, prefix, l, fmt, ...) \
-+	logfc((p)->log, (prefix), (l), (fmt), ## __VA_ARGS__)
-+#define __plog(p, l, fmt, ...) __plogp(p, (p)->prefix, l, fmt, ## __VA_ARGS__)
-+
- /**
-  * infof - Store supplementary informational message
-  * @fc: The context in which to log the informational message
-@@ -209,7 +207,9 @@ void logfc(struct fc_log *log, const char *prefix, char level, const char *fmt,
-  */
- #define infof(fc, fmt, ...) __logfc(fc, 'i', fmt, ## __VA_ARGS__)
- #define info_plog(p, fmt, ...) __plog(p, 'i', fmt, ## __VA_ARGS__)
-#define infofc(p, fmt, ...) __plog((&(fc)->log), 'i', fmt, ## __VA_ARGS__)
-+#define infofc(fc, fmt, ...) __plog((&(fc)->log), 'i', fmt, ## __VA_ARGS__)
-+#define infofcp(fc, prefix, fmt, ...) \
-+	__plogp((&(fc)->log), prefix, 'i', fmt, ## __VA_ARGS__)
- 
- /**
-  * warnf - Store supplementary warning message
-@@ -222,6 +222,8 @@ void logfc(struct fc_log *log, const char *prefix, char level, const char *fmt,
- #define warnf(fc, fmt, ...) __logfc(fc, 'w', fmt, ## __VA_ARGS__)
- #define warn_plog(p, fmt, ...) __plog(p, 'w', fmt, ## __VA_ARGS__)
- #define warnfc(fc, fmt, ...) __plog((&(fc)->log), 'w', fmt, ## __VA_ARGS__)
-+#define warnfcp(fc, prefix, fmt, ...) \
-+	__plogp((&(fc)->log), prefix, 'w', fmt, ## __VA_ARGS__)
- 
- /**
-  * errorf - Store supplementary error message
-@@ -234,6 +236,8 @@ void logfc(struct fc_log *log, const char *prefix, char level, const char *fmt,
- #define errorf(fc, fmt, ...) __logfc(fc, 'e', fmt, ## __VA_ARGS__)
- #define error_plog(p, fmt, ...) __plog(p, 'e', fmt, ## __VA_ARGS__)
- #define errorfc(fc, fmt, ...) __plog((&(fc)->log), 'e', fmt, ## __VA_ARGS__)
-+#define errorfcp(fc, prefix, fmt, ...) \
-+	__plogp((&(fc)->log), prefix, 'e', fmt, ## __VA_ARGS__)
- 
- /**
-  * invalf - Store supplementary invalid argument error message
-@@ -246,5 +250,7 @@ void logfc(struct fc_log *log, const char *prefix, char level, const char *fmt,
- #define invalf(fc, fmt, ...) (errorf(fc, fmt, ## __VA_ARGS__), -EINVAL)
- #define inval_plog(p, fmt, ...) (error_plog(p, fmt, ## __VA_ARGS__), -EINVAL)
- #define invalfc(fc, fmt, ...) (errorfc(fc, fmt, ## __VA_ARGS__), -EINVAL)
-+#define invalfcp(fc, prefix, fmt, ...) \
-+	(errorfcp(fc, prefix, fmt, ## __VA_ARGS__), -EINVAL)
- 
- #endif /* _LINUX_FS_CONTEXT_H */
-diff --git a/include/linux/fs_types.h b/include/linux/fs_dirent.h
-index 54816791196f..92f75c5bac19 100644
--- a/include/linux/fs_types.h
-+++ b/include/linux/fs_dirent.h
-@@ -1,6 +1,9 @@
- /* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _LINUX_FS_TYPES_H
-#define _LINUX_FS_TYPES_H
-+#ifndef _LINUX_FS_DIRENT_H
-+#define _LINUX_FS_DIRENT_H
-+
-+#include <linux/stat.h>
-+#include <linux/types.h>
- 
- /*
-  * This is a header for the common implementation of dirent
-@@ -66,10 +69,10 @@
- 
- /*
-  * declarations for helper functions, accompanying implementation
- * is in fs/fs_types.c
-+ * is in fs/fs_dirent.c
-  */
- extern unsigned char fs_ftype_to_dtype(unsigned int filetype);
- extern unsigned char fs_umode_to_ftype(umode_t mode);
- extern unsigned char fs_umode_to_dtype(umode_t mode);
- 
-#endif
-+#endif /* _LINUX_FS_DIRENT_H */
-diff --git a/include/linux/fs_enet_pd.h b/include/linux/fs_enet_pd.h
-deleted file mode 100644
-index 77d783f71527..000000000000
--- a/include/linux/fs_enet_pd.h
-+++ /dev/null
-@@ -1,165 +0,0 @@
-/*
- * Platform information definitions for the
- * universal Freescale Ethernet driver.
- *
- * Copyright (c) 2003 Intracom S.A. 
- *  by Pantelis Antoniou <panto@intracom.gr>
- *
- * 2005 (c) MontaVista Software, Inc. 
- * Vitaly Bordug <vbordug@ru.mvista.com>
- *
- * This file is licensed under the terms of the GNU General Public License 
- * version 2. This program is licensed "as is" without any warranty of any 
- * kind, whether express or implied.
- */
-
-#ifndef FS_ENET_PD_H
-#define FS_ENET_PD_H
-
-#include <linux/clk.h>
-#include <linux/string.h>
-#include <linux/of_mdio.h>
-#include <linux/if_ether.h>
-#include <asm/types.h>
-
-#define FS_ENET_NAME	"fs_enet"
-
-enum fs_id {
-	fsid_fec1,
-	fsid_fec2,
-	fsid_fcc1,
-	fsid_fcc2,
-	fsid_fcc3,
-	fsid_scc1,
-	fsid_scc2,
-	fsid_scc3,
-	fsid_scc4,
-};
-
-#define FS_MAX_INDEX	9
-
-static inline int fs_get_fec_index(enum fs_id id)
-{
-	if (id >= fsid_fec1 && id <= fsid_fec2)
-		return id - fsid_fec1;
-	return -1;
-}
-
-static inline int fs_get_fcc_index(enum fs_id id)
-{
-	if (id >= fsid_fcc1 && id <= fsid_fcc3)
-		return id - fsid_fcc1;
-	return -1;
-}
-
-static inline int fs_get_scc_index(enum fs_id id)
-{
-	if (id >= fsid_scc1 && id <= fsid_scc4)
-		return id - fsid_scc1;
-	return -1;
-}
-
-static inline int fs_fec_index2id(int index)
-{
-	int id = fsid_fec1 + index - 1;
-	if (id >= fsid_fec1 && id <= fsid_fec2)
-		return id;
-	return FS_MAX_INDEX;
-		}
-
-static inline int fs_fcc_index2id(int index)
-{
-	int id = fsid_fcc1 + index - 1;
-	if (id >= fsid_fcc1 && id <= fsid_fcc3)
-		return id;
-	return FS_MAX_INDEX;
-}
-
-static inline int fs_scc_index2id(int index)
-{
-	int id = fsid_scc1 + index - 1;
-	if (id >= fsid_scc1 && id <= fsid_scc4)
-		return id;
-	return FS_MAX_INDEX;
-}
-
-enum fs_mii_method {
-	fsmii_fixed,
-	fsmii_fec,
-	fsmii_bitbang,
-};
-
-enum fs_ioport {
-	fsiop_porta,
-	fsiop_portb,
-	fsiop_portc,
-	fsiop_portd,
-	fsiop_porte,
-};
-
-struct fs_mii_bit {
-	u32 offset;
-	u8 bit;
-	u8 polarity;
-};
-struct fs_mii_bb_platform_info {
-	struct fs_mii_bit 	mdio_dir;
-	struct fs_mii_bit 	mdio_dat;
-	struct fs_mii_bit	mdc_dat;
-	int delay;	/* delay in us         */
-	int irq[32]; 	/* irqs per phy's */
-};
-
-struct fs_platform_info {
-
-	void(*init_ioports)(struct fs_platform_info *);
-	/* device specific information */
-	int fs_no;		/* controller index            */
-	char fs_type[4];	/* controller type             */
-
-	u32 cp_page;		/* CPM page */
-	u32 cp_block;		/* CPM sblock */
-	u32 cp_command;		/* CPM page/sblock/mcn */
-
-	u32 clk_trx;		/* some stuff for pins & mux configuration*/
-	u32 clk_rx;
-	u32 clk_tx;
-	u32 clk_route;
-	u32 clk_mask;
-
-	u32 mem_offset;
-	u32 dpram_offset;
-	u32 fcc_regs_c;
-	
-	u32 device_flags;
-
-	struct device_node *phy_node;
-	const struct fs_mii_bus_info *bus_info;
-
-	int rx_ring, tx_ring;	/* number of buffers on rx     */
-	__u8 macaddr[ETH_ALEN];	/* mac address                 */
-	int rx_copybreak;	/* limit we copy small frames  */
-	int napi_weight;	/* NAPI weight                 */
-
-	int use_rmii;		/* use RMII mode 	       */
-	int has_phy;            /* if the network is phy container as well...*/
-
-	struct clk *clk_per;	/* 'per' clock for register access */
-};
-struct fs_mii_fec_platform_info {
-	u32 irq[32];
-	u32 mii_speed;
-};
-
-static inline int fs_get_id(struct fs_platform_info *fpi)
-{
-	if(strstr(fpi->fs_type, "SCC"))
-		return fs_scc_index2id(fpi->fs_no);
-	if(strstr(fpi->fs_type, "FCC"))
-		return fs_fcc_index2id(fpi->fs_no);
-	if(strstr(fpi->fs_type, "FEC"))
-		return fs_fec_index2id(fpi->fs_no);
-	return fpi->fs_no;
-}
-
-#endif
-diff --git a/include/linux/fs_parser.h b/include/linux/fs_parser.h
-index f103c91139d4..5e8a3b546033 100644
--- a/include/linux/fs_parser.h
-+++ b/include/linux/fs_parser.h
-@@ -28,7 +28,8 @@ typedef int fs_param_type(struct p_log *,
-  */
- fs_param_type fs_param_is_bool, fs_param_is_u32, fs_param_is_s32, fs_param_is_u64,
- 	fs_param_is_enum, fs_param_is_string, fs_param_is_blob, fs_param_is_blockdev,
-	fs_param_is_path, fs_param_is_fd;
-+	fs_param_is_path, fs_param_is_fd, fs_param_is_uid, fs_param_is_gid,
-+	fs_param_is_file_or_string;
- 
- /*
-  * Specification of the type of value a parameter wants.
-@@ -57,6 +58,8 @@ struct fs_parse_result {
- 		int		int_32;		/* For spec_s32/spec_enum */
- 		unsigned int	uint_32;	/* For spec_u32{,_octal,_hex}/spec_enum */
- 		u64		uint_64;	/* For spec_u64 */
-+		kuid_t		uid;
-+		kgid_t		gid;
- 	};
- };
- 
-@@ -76,19 +79,17 @@ static inline int fs_parse(struct fs_context *fc,
- extern int fs_lookup_param(struct fs_context *fc,
- 			   struct fs_parameter *param,
- 			   bool want_bdev,
-+			   unsigned int flags,
- 			   struct path *_path);
- 
- extern int lookup_constant(const struct constant_table tbl[], const char *name, int not_found);
- 
-+extern const struct constant_table bool_names[];
-+
- #ifdef CONFIG_VALIDATE_FS_PARSER
-extern bool validate_constant_table(const struct constant_table *tbl, size_t tbl_size,
-				    int low, int high, int special);
- extern bool fs_validate_description(const char *name,
- 				    const struct fs_parameter_spec *desc);
- #else
-static inline bool validate_constant_table(const struct constant_table *tbl, size_t tbl_size,
-					   int low, int high, int special)
-{ return true; }
- static inline bool fs_validate_description(const char *name,
- 					   const struct fs_parameter_spec *desc)
- { return true; }
-@@ -120,7 +121,7 @@ static inline bool fs_validate_description(const char *name,
- #define fsparam_u32oct(NAME, OPT) \
- 			__fsparam(fs_param_is_u32, NAME, OPT, 0, (void *)8)
- #define fsparam_u32hex(NAME, OPT) \
-			__fsparam(fs_param_is_u32_hex, NAME, OPT, 0, (void *)16)
-+			__fsparam(fs_param_is_u32, NAME, OPT, 0, (void *)16)
- #define fsparam_s32(NAME, OPT)	__fsparam(fs_param_is_s32, NAME, OPT, 0, NULL)
- #define fsparam_u64(NAME, OPT)	__fsparam(fs_param_is_u64, NAME, OPT, 0, NULL)
- #define fsparam_enum(NAME, OPT, array)	__fsparam(fs_param_is_enum, NAME, OPT, 0, array)
-@@ -130,5 +131,13 @@ static inline bool fs_validate_description(const char *name,
- #define fsparam_bdev(NAME, OPT)	__fsparam(fs_param_is_blockdev, NAME, OPT, 0, NULL)
- #define fsparam_path(NAME, OPT)	__fsparam(fs_param_is_path, NAME, OPT, 0, NULL)
- #define fsparam_fd(NAME, OPT)	__fsparam(fs_param_is_fd, NAME, OPT, 0, NULL)
-+#define fsparam_file_or_string(NAME, OPT) \
-+				__fsparam(fs_param_is_file_or_string, NAME, OPT, 0, NULL)
-+#define fsparam_uid(NAME, OPT) __fsparam(fs_param_is_uid, NAME, OPT, 0, NULL)
-+#define fsparam_gid(NAME, OPT) __fsparam(fs_param_is_gid, NAME, OPT, 0, NULL)
-+
-+/* String parameter that allows empty argument */
-+#define fsparam_string_empty(NAME, OPT) \
-+	__fsparam(fs_param_is_string, NAME, OPT, fs_param_can_be_empty, NULL)
- 
- #endif /* _LINUX_FS_PARSER_H */
-diff --git a/include/linux/fs_stack.h b/include/linux/fs_stack.h
-index 54210a42c30d..0cc2fa283305 100644
--- a/include/linux/fs_stack.h
-+++ b/include/linux/fs_stack.h
-@@ -3,7 +3,7 @@
- #define _LINUX_FS_STACK_H
- 
- /* This file defines generic functions used primarily by stackable
- * filesystems; none of these functions require i_mutex to be held.
-+ * filesystems; none of these functions require i_rwsem to be held.
-  */
- 
- #include <linux/fs.h>
-@@ -16,15 +16,15 @@ extern void fsstack_copy_inode_size(struct inode *dst, struct inode *src);
- static inline void fsstack_copy_attr_atime(struct inode *dest,
- 					   const struct inode *src)
- {
-	dest->i_atime = src->i_atime;
-+	inode_set_atime_to_ts(dest, inode_get_atime(src));
- }
- 
- static inline void fsstack_copy_attr_times(struct inode *dest,
- 					   const struct inode *src)
- {
-	dest->i_atime = src->i_atime;
-	dest->i_mtime = src->i_mtime;
-	dest->i_ctime = src->i_ctime;
-+	inode_set_atime_to_ts(dest, inode_get_atime(src));
-+	inode_set_mtime_to_ts(dest, inode_get_mtime(src));
-+	inode_set_ctime_to_ts(dest, inode_get_ctime(src));
- }
- 
- #endif /* _LINUX_FS_STACK_H */
-diff --git a/include/linux/fs_struct.h b/include/linux/fs_struct.h
-index 783b48dedb72..0070764b790a 100644
--- a/include/linux/fs_struct.h
-+++ b/include/linux/fs_struct.h
-@@ -2,14 +2,14 @@
- #ifndef _LINUX_FS_STRUCT_H
- #define _LINUX_FS_STRUCT_H
- 
-+#include <linux/sched.h>
- #include <linux/path.h>
- #include <linux/spinlock.h>
- #include <linux/seqlock.h>
- 
- struct fs_struct {
- 	int users;
-	spinlock_t lock;
-	seqcount_spinlock_t seq;
-+	seqlock_t seq;
- 	int umask;
- 	int in_exec;
- 	struct path root, pwd;
-@@ -26,20 +26,25 @@ extern int unshare_fs_struct(void);
- 
- static inline void get_fs_root(struct fs_struct *fs, struct path *root)
- {
-	spin_lock(&fs->lock);
-+	read_seqlock_excl(&fs->seq);
- 	*root = fs->root;
- 	path_get(root);
-	spin_unlock(&fs->lock);
-+	read_sequnlock_excl(&fs->seq);
- }
- 
- static inline void get_fs_pwd(struct fs_struct *fs, struct path *pwd)
- {
-	spin_lock(&fs->lock);
-+	read_seqlock_excl(&fs->seq);
- 	*pwd = fs->pwd;
- 	path_get(pwd);
-	spin_unlock(&fs->lock);
-+	read_sequnlock_excl(&fs->seq);
- }
- 
- extern bool current_chrooted(void);
- 
-+static inline int current_umask(void)
-+{
-+	return current->fs->umask;
-+}
-+
- #endif /* _LINUX_FS_STRUCT_H */
-diff --git a/include/linux/fs_uart_pd.h b/include/linux/fs_uart_pd.h
-deleted file mode 100644
-index 36b61ff39277..000000000000
--- a/include/linux/fs_uart_pd.h
-+++ /dev/null
-@@ -1,71 +0,0 @@
-/*
- * Platform information definitions for the CPM Uart driver.
- *
- * 2006 (c) MontaVista Software, Inc.
- * Vitaly Bordug <vbordug@ru.mvista.com>
- *
- * This file is licensed under the terms of the GNU General Public License
- * version 2. This program is licensed "as is" without any warranty of any
- * kind, whether express or implied.
- */
-
-#ifndef FS_UART_PD_H
-#define FS_UART_PD_H
-
-#include <asm/types.h>
-
-enum fs_uart_id {
-	fsid_smc1_uart,
-	fsid_smc2_uart,
-	fsid_scc1_uart,
-	fsid_scc2_uart,
-	fsid_scc3_uart,
-	fsid_scc4_uart,
-	fs_uart_nr,
-};
-
-static inline int fs_uart_id_scc2fsid(int id)
-{
-    return fsid_scc1_uart + id - 1;
-}
-
-static inline int fs_uart_id_fsid2scc(int id)
-{
-    return id - fsid_scc1_uart + 1;
-}
-
-static inline int fs_uart_id_smc2fsid(int id)
-{
-    return fsid_smc1_uart + id - 1;
-}
-
-static inline int fs_uart_id_fsid2smc(int id)
-{
-    return id - fsid_smc1_uart + 1;
-}
-
-struct fs_uart_platform_info {
-        void(*init_ioports)(struct fs_uart_platform_info *);
-	/* device specific information */
-	int fs_no;		/* controller index */
-	char fs_type[4];        /* controller type  */
-	u32 uart_clk;
-	u8 tx_num_fifo;
-	u8 tx_buf_size;
-	u8 rx_num_fifo;
-	u8 rx_buf_size;
-	u8 brg;
-	u8 clk_rx;
-	u8 clk_tx;
-};
-
-static inline int fs_uart_get_id(struct fs_uart_platform_info *fpi)
-{
-        if(strstr(fpi->fs_type, "SMC"))
-                return fs_uart_id_smc2fsid(fpi->fs_no);
-        if(strstr(fpi->fs_type, "SCC"))
-                return fs_uart_id_scc2fsid(fpi->fs_no);
-        return fpi->fs_no;
-}
-
-#endif
-diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h
-index a174cedf4d90..4c91a019972b 100644
--- a/include/linux/fscache-cache.h
-+++ b/include/linux/fscache-cache.h
-@@ -19,6 +19,7 @@
- enum fscache_cache_trace;
- enum fscache_cookie_trace;
- enum fscache_access_trace;
-+enum fscache_volume_trace;
- 
- enum fscache_cache_state {
- 	FSCACHE_CACHE_IS_NOT_PRESENT,	/* No cache is present for this name */
-@@ -97,6 +98,11 @@ extern void fscache_withdraw_cookie(struct fscache_cookie *cookie);
- 
- extern void fscache_io_error(struct fscache_cache *cache);
- 
-+extern struct fscache_volume *
-+fscache_try_get_volume(struct fscache_volume *volume,
-+		       enum fscache_volume_trace where);
-+extern void fscache_put_volume(struct fscache_volume *volume,
-+			       enum fscache_volume_trace where);
- extern void fscache_end_volume_access(struct fscache_volume *volume,
- 				      struct fscache_cookie *cookie,
- 				      enum fscache_access_trace why);
-@@ -189,17 +195,20 @@ extern atomic_t fscache_n_write;
- extern atomic_t fscache_n_no_write_space;
- extern atomic_t fscache_n_no_create_space;
- extern atomic_t fscache_n_culled;
-+extern atomic_t fscache_n_dio_misfit;
- #define fscache_count_read() atomic_inc(&fscache_n_read)
- #define fscache_count_write() atomic_inc(&fscache_n_write)
- #define fscache_count_no_write_space() atomic_inc(&fscache_n_no_write_space)
- #define fscache_count_no_create_space() atomic_inc(&fscache_n_no_create_space)
- #define fscache_count_culled() atomic_inc(&fscache_n_culled)
-+#define fscache_count_dio_misfit() atomic_inc(&fscache_n_dio_misfit)
- #else
- #define fscache_count_read() do {} while(0)
- #define fscache_count_write() do {} while(0)
- #define fscache_count_no_write_space() do {} while(0)
- #define fscache_count_no_create_space() do {} while(0)
- #define fscache_count_culled() do {} while(0)
-+#define fscache_count_dio_misfit() do {} while(0)
- #endif
- 
- #endif /* _LINUX_FSCACHE_CACHE_H */
-diff --git a/include/linux/fscache.h b/include/linux/fscache.h
-index 720874e6ee94..58fdb9605425 100644
--- a/include/linux/fscache.h
-+++ b/include/linux/fscache.h
-@@ -75,7 +75,7 @@ struct fscache_volume {
- 	atomic_t			n_accesses;	/* Number of cache accesses in progress */
- 	unsigned int			debug_id;
- 	unsigned int			key_hash;	/* Hash of key string */
-	char				*key;		/* Volume ID, eg. "afs@example.com@1234" */
-+	u8				*key;		/* Volume ID, eg. "afs@example.com@1234" */
- 	struct list_head		proc_link;	/* Link in /proc/fs/fscache/volumes */
- 	struct hlist_bl_node		hash_link;	/* Link in hash table */
- 	struct work_struct		work;
-@@ -172,9 +172,12 @@ extern void __fscache_invalidate(struct fscache_cookie *, const void *, loff_t,
- extern int __fscache_begin_read_operation(struct netfs_cache_resources *, struct fscache_cookie *);
- extern int __fscache_begin_write_operation(struct netfs_cache_resources *, struct fscache_cookie *);
- 
-extern void __fscache_write_to_cache(struct fscache_cookie *, struct address_space *,
-				     loff_t, size_t, loff_t, netfs_io_terminated_t, void *,
-				     bool);
-+void __fscache_write_to_cache(struct fscache_cookie *cookie,
-+			      struct address_space *mapping,
-+			      loff_t start, size_t len, loff_t i_size,
-+			      netfs_io_terminated_t term_func,
-+			      void *term_func_priv,
-+			      bool using_pgpriv2, bool cond);
- extern void __fscache_clear_page_bits(struct address_space *, loff_t, size_t);
- 
- /**
-@@ -258,7 +261,7 @@ struct fscache_cookie *fscache_acquire_cookie(struct fscache_volume *volume,
- 
- /**
-  * fscache_use_cookie - Request usage of cookie attached to an object
- * @object: Object description
-+ * @cookie: The cookie representing the cache object
-  * @will_modify: If cache is expected to be modified locally
-  *
-  * Request usage of the cookie attached to an object.  The caller should tell
-@@ -274,7 +277,7 @@ static inline void fscache_use_cookie(struct fscache_cookie *cookie,
- 
- /**
-  * fscache_unuse_cookie - Cease usage of cookie attached to an object
- * @object: Object description
-+ * @cookie: The cookie representing the cache object
-  * @aux_data: Updated auxiliary data (or NULL)
-  * @object_size: Revised size of the object (or NULL)
-  *
-@@ -437,9 +440,6 @@ const struct netfs_cache_ops *fscache_operation_valid(const struct netfs_cache_r
-  * indicates the cache resources to which the operation state should be
-  * attached; @cookie indicates the cache object that will be accessed.
-  *
- * This is intended to be called from the ->begin_cache_operation() netfs lib
- * operation as implemented by the network filesystem.
- *
-  * @cres->inval_counter is set from @cookie->inval_counter for comparison at
-  * the end of the operation.  This allows invalidation during the operation to
-  * be detected by the caller.
-@@ -498,9 +498,6 @@ static inline void fscache_end_operation(struct netfs_cache_resources *cres)
-  *
-  *	NETFS_READ_HOLE_IGNORE - Just try to read (may return a short read).
-  *
- *	NETFS_READ_HOLE_CLEAR - Seek for data, clearing the part of the buffer
- *				skipped over, then do as for IGNORE.
- *
-  *	NETFS_READ_HOLE_FAIL - Give ENODATA if we encounter a hole.
-  */
- static inline
-@@ -600,7 +597,8 @@ static inline void fscache_clear_page_bits(struct address_space *mapping,
-  * @i_size: The new size of the inode
-  * @term_func: The function to call upon completion
-  * @term_func_priv: The private data for @term_func
- * @caching: If PG_fscache has been set
-+ * @using_pgpriv2: If we're using PG_private_2 to mark in-progress write
-+ * @caching: If we actually want to do the caching
-  *
-  * Helper function for a netfs to write dirty data from an inode into the cache
-  * object that's backing it.
-@@ -611,64 +609,24 @@ static inline void fscache_clear_page_bits(struct address_space *mapping,
-  * marked with PG_fscache.
-  *
-  * If given, @term_func will be called upon completion and supplied with
- * @term_func_priv.  Note that the PG_fscache flags will have been cleared by
- * this point, so the netfs must retain its own pin on the mapping.
-+ * @term_func_priv.  Note that if @using_pgpriv2 is set, the PG_private_2 flags
-+ * will have been cleared by this point, so the netfs must retain its own pin
-+ * on the mapping.
-  */
- static inline void fscache_write_to_cache(struct fscache_cookie *cookie,
- 					  struct address_space *mapping,
- 					  loff_t start, size_t len, loff_t i_size,
- 					  netfs_io_terminated_t term_func,
- 					  void *term_func_priv,
-					  bool caching)
-+					  bool using_pgpriv2, bool caching)
- {
- 	if (caching)
- 		__fscache_write_to_cache(cookie, mapping, start, len, i_size,
-					 term_func, term_func_priv, caching);
-+					 term_func, term_func_priv,
-+					 using_pgpriv2, caching);
- 	else if (term_func)
-		term_func(term_func_priv, -ENOBUFS, false);
-
-}
-
-#if __fscache_available
-bool fscache_dirty_folio(struct address_space *mapping, struct folio *folio,
-		struct fscache_cookie *cookie);
-#else
-#define fscache_dirty_folio(MAPPING, FOLIO, COOKIE) \
-		filemap_dirty_folio(MAPPING, FOLIO)
-#endif
-+		term_func(term_func_priv, -ENOBUFS);
- 
-/**
- * fscache_unpin_writeback - Unpin writeback resources
- * @wbc: The writeback control
- * @cookie: The cookie referring to the cache object
- *
- * Unpin the writeback resources pinned by fscache_dirty_folio().  This is
- * intended to be called by the netfs's ->write_inode() method.
- */
-static inline void fscache_unpin_writeback(struct writeback_control *wbc,
-					   struct fscache_cookie *cookie)
-{
-	if (wbc->unpinned_fscache_wb)
-		fscache_unuse_cookie(cookie, NULL, NULL);
-}
-
-/**
- * fscache_clear_inode_writeback - Clear writeback resources pinned by an inode
- * @cookie: The cookie referring to the cache object
- * @inode: The inode to clean up
- * @aux: Auxiliary data to apply to the inode
- *
- * Clear any writeback resources held by an inode when the inode is evicted.
- * This must be called before clear_inode() is called.
- */
-static inline void fscache_clear_inode_writeback(struct fscache_cookie *cookie,
-						 struct inode *inode,
-						 const void *aux)
-{
-	if (inode->i_state & I_PINNING_FSCACHE_WB) {
-		loff_t i_size = i_size_read(inode);
-		fscache_unuse_cookie(cookie, aux, &i_size);
-	}
- }
- 
- /**
-diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h
-index 7d2f1e0f23b1..516aba5b858b 100644
--- a/include/linux/fscrypt.h
-+++ b/include/linux/fscrypt.h
-@@ -31,7 +31,7 @@
- #define FSCRYPT_CONTENTS_ALIGNMENT 16
- 
- union fscrypt_policy;
-struct fscrypt_info;
-+struct fscrypt_inode_info;
- struct fs_parameter;
- struct seq_file;
- 
-@@ -59,26 +59,61 @@ struct fscrypt_name {
- 
- #ifdef CONFIG_FS_ENCRYPTION
- 
-/*
- * If set, the fscrypt bounce page pool won't be allocated (unless another
- * filesystem needs it).  Set this if the filesystem always uses its own bounce
- * pages for writes and therefore won't need the fscrypt bounce page pool.
- */
-#define FS_CFLG_OWN_PAGES (1U << 1)
-
- /* Crypto operations for filesystems */
- struct fscrypt_operations {
-+	/*
-+	 * The offset of the pointer to struct fscrypt_inode_info in the
-+	 * filesystem-specific part of the inode, relative to the beginning of
-+	 * the common part of the inode (the 'struct inode').
-+	 */
-+	ptrdiff_t inode_info_offs;
- 
-	/* Set of optional flags; see above for allowed flags */
-	unsigned int flags;
-+	/*
-+	 * If set, then fs/crypto/ will allocate a global bounce page pool the
-+	 * first time an encryption key is set up for a file.  The bounce page
-+	 * pool is required by the following functions:
-+	 *
-+	 * - fscrypt_encrypt_pagecache_blocks()
-+	 * - fscrypt_zeroout_range() for files not using inline crypto
-+	 *
-+	 * If the filesystem doesn't use those, it doesn't need to set this.
-+	 */
-+	unsigned int needs_bounce_pages : 1;
- 
- 	/*
-	 * If set, this is a filesystem-specific key description prefix that
-	 * will be accepted for "logon" keys for v1 fscrypt policies, in
-	 * addition to the generic prefix "fscrypt:".  This functionality is
-	 * deprecated, so new filesystems shouldn't set this field.
-+	 * If set, then fs/crypto/ will allow the use of encryption settings
-+	 * that assume inode numbers fit in 32 bits (i.e.
-+	 * FSCRYPT_POLICY_FLAG_IV_INO_LBLK_{32,64}), provided that the other
-+	 * prerequisites for these settings are also met.  This is only useful
-+	 * if the filesystem wants to support inline encryption hardware that is
-+	 * limited to 32-bit or 64-bit data unit numbers and where programming
-+	 * keyslots is very slow.
- 	 */
-	const char *key_prefix;
-+	unsigned int has_32bit_inodes : 1;
-+
-+	/*
-+	 * If set, then fs/crypto/ will allow users to select a crypto data unit
-+	 * size that is less than the filesystem block size.  This is done via
-+	 * the log2_data_unit_size field of the fscrypt policy.  This flag is
-+	 * not compatible with filesystems that encrypt variable-length blocks
-+	 * (i.e. blocks that aren't all equal to filesystem's block size), for
-+	 * example as a result of compression.  It's also not compatible with
-+	 * the fscrypt_encrypt_block_inplace() and
-+	 * fscrypt_decrypt_block_inplace() functions.
-+	 */
-+	unsigned int supports_subblock_data_units : 1;
-+
-+	/*
-+	 * This field exists only for backwards compatibility reasons and should
-+	 * only be set by the filesystems that are setting it already.  It
-+	 * contains the filesystem-specific key description prefix that is
-+	 * accepted for "logon" keys for v1 fscrypt policies.  This
-+	 * functionality is deprecated in favor of the generic prefix
-+	 * "fscrypt:", which itself is deprecated in favor of the filesystem
-+	 * keyring ioctls such as FS_IOC_ADD_ENCRYPTION_KEY.  Filesystems that
-+	 * are newly adding fscrypt support should not set this field.
-+	 */
-+	const char *legacy_key_prefix;
- 
- 	/*
- 	 * Get the fscrypt context of the given inode.
-@@ -146,50 +181,64 @@ struct fscrypt_operations {
- 	bool (*has_stable_inodes)(struct super_block *sb);
- 
- 	/*
-	 * Get the number of bits that the filesystem uses to represent inode
-	 * numbers and file logical block numbers.
-+	 * Return an array of pointers to the block devices to which the
-+	 * filesystem may write encrypted file contents, NULL if the filesystem
-+	 * only has a single such block device, or an ERR_PTR() on error.
- 	 *
-	 * By default, both of these are assumed to be 64-bit.  This function
-	 * can be implemented to declare that either or both of these numbers is
-	 * shorter, which may allow the use of the
-	 * FSCRYPT_POLICY_FLAG_IV_INO_LBLK_{32,64} flags and/or the use of
-	 * inline crypto hardware whose maximum DUN length is less than 64 bits
-	 * (e.g., eMMC v5.2 spec compliant hardware).  This function only needs
-	 * to be implemented if support for one of these features is needed.
-	 */
-	void (*get_ino_and_lblk_bits)(struct super_block *sb,
-				      int *ino_bits_ret, int *lblk_bits_ret);
-
-	/*
-	 * Return the number of block devices to which the filesystem may write
-	 * encrypted file contents.
-+	 * On successful non-NULL return, *num_devs is set to the number of
-+	 * devices in the returned array.  The caller must free the returned
-+	 * array using kfree().
- 	 *
- 	 * If the filesystem can use multiple block devices (other than block
- 	 * devices that aren't used for encrypted file contents, such as
- 	 * external journal devices), and wants to support inline encryption,
- 	 * then it must implement this function.  Otherwise it's not needed.
- 	 */
-	int (*get_num_devices)(struct super_block *sb);
-
-	/*
-	 * If ->get_num_devices() returns a value greater than 1, then this
-	 * function is called to get the array of request_queues that the
-	 * filesystem is using -- one per block device.  (There may be duplicate
-	 * entries in this array, as block devices can share a request_queue.)
-	 */
-	void (*get_devices)(struct super_block *sb,
-			    struct request_queue **devs);
-+	struct block_device **(*get_devices)(struct super_block *sb,
-+					     unsigned int *num_devs);
- };
- 
-static inline struct fscrypt_info *fscrypt_get_info(const struct inode *inode)
-+int fscrypt_d_revalidate(struct inode *dir, const struct qstr *name,
-+			 struct dentry *dentry, unsigned int flags);
-+
-+/*
-+ * Returns the address of the fscrypt info pointer within the
-+ * filesystem-specific part of the inode.  (To save memory on filesystems that
-+ * don't support fscrypt, a field in 'struct inode' itself is no longer used.)
-+ */
-+static inline struct fscrypt_inode_info **
-+fscrypt_inode_info_addr(const struct inode *inode)
-+{
-+	VFS_WARN_ON_ONCE(inode->i_sb->s_cop->inode_info_offs == 0);
-+	return (void *)inode + inode->i_sb->s_cop->inode_info_offs;
-+}
-+
-+/*
-+ * Load the inode's fscrypt info pointer, using a raw dereference.  Since this
-+ * uses a raw dereference with no memory barrier, it is appropriate to use only
-+ * when the caller knows the inode's key setup already happened, resulting in
-+ * non-NULL fscrypt info.  E.g., the file contents en/decryption functions use
-+ * this, since fscrypt_file_open() set up the key.
-+ */
-+static inline struct fscrypt_inode_info *
-+fscrypt_get_inode_info_raw(const struct inode *inode)
-+{
-+	struct fscrypt_inode_info *ci = *fscrypt_inode_info_addr(inode);
-+
-+	VFS_WARN_ON_ONCE(ci == NULL);
-+	return ci;
-+}
-+
-+static inline struct fscrypt_inode_info *
-+fscrypt_get_inode_info(const struct inode *inode)
- {
- 	/*
- 	 * Pairs with the cmpxchg_release() in fscrypt_setup_encryption_info().
-	 * I.e., another task may publish ->i_crypt_info concurrently, executing
-	 * a RELEASE barrier.  We need to use smp_load_acquire() here to safely
-+	 * I.e., another task may publish the fscrypt info concurrently,
-+	 * executing a RELEASE barrier.  Use smp_load_acquire() here to safely
- 	 * ACQUIRE the memory the other task published.
- 	 */
-	return smp_load_acquire(&inode->i_crypt_info);
-+	return smp_load_acquire(fscrypt_inode_info_addr(inode));
- }
- 
- /**
-@@ -209,15 +258,29 @@ static inline bool fscrypt_needs_contents_encryption(const struct inode *inode)
- }
- 
- /*
- * When d_splice_alias() moves a directory's no-key alias to its plaintext alias
- * as a result of the encryption key being added, DCACHE_NOKEY_NAME must be
- * cleared.  Note that we don't have to support arbitrary moves of this flag
- * because fscrypt doesn't allow no-key names to be the source or target of a
- * rename().
-+ * When d_splice_alias() moves a directory's no-key alias to its
-+ * plaintext alias as a result of the encryption key being added,
-+ * DCACHE_NOKEY_NAME must be cleared and there might be an opportunity
-+ * to disable d_revalidate.  Note that we don't have to support the
-+ * inverse operation because fscrypt doesn't allow no-key names to be
-+ * the source or target of a rename().
-  */
- static inline void fscrypt_handle_d_move(struct dentry *dentry)
- {
-	dentry->d_flags &= ~DCACHE_NOKEY_NAME;
-+	/*
-+	 * VFS calls fscrypt_handle_d_move even for non-fscrypt
-+	 * filesystems.
-+	 */
-+	if (dentry->d_flags & DCACHE_NOKEY_NAME) {
-+		dentry->d_flags &= ~DCACHE_NOKEY_NAME;
-+
-+		/*
-+		 * Other filesystem features might be handling dentry
-+		 * revalidation, in which case it cannot be disabled.
-+		 */
-+		if (dentry->d_op->d_revalidate == fscrypt_d_revalidate)
-+			dentry->d_flags &= ~DCACHE_OP_REVALIDATE;
-+	}
- }
- 
- /**
-@@ -249,19 +312,46 @@ static inline bool fscrypt_is_nokey_name(const struct dentry *dentry)
- 	return dentry->d_flags & DCACHE_NOKEY_NAME;
- }
- 
-+static inline void fscrypt_prepare_dentry(struct dentry *dentry,
-+					  bool is_nokey_name)
-+{
-+	/*
-+	 * This code tries to only take ->d_lock when necessary to write
-+	 * to ->d_flags.  We shouldn't be peeking on d_flags for
-+	 * DCACHE_OP_REVALIDATE unlocked, but in the unlikely case
-+	 * there is a race, the worst it can happen is that we fail to
-+	 * unset DCACHE_OP_REVALIDATE and pay the cost of an extra
-+	 * d_revalidate.
-+	 */
-+	if (is_nokey_name) {
-+		spin_lock(&dentry->d_lock);
-+		dentry->d_flags |= DCACHE_NOKEY_NAME;
-+		spin_unlock(&dentry->d_lock);
-+	} else if (dentry->d_flags & DCACHE_OP_REVALIDATE &&
-+		   dentry->d_op->d_revalidate == fscrypt_d_revalidate) {
-+		/*
-+		 * Unencrypted dentries and encrypted dentries where the
-+		 * key is available are always valid from fscrypt
-+		 * perspective. Avoid the cost of calling
-+		 * fscrypt_d_revalidate unnecessarily.
-+		 */
-+		spin_lock(&dentry->d_lock);
-+		dentry->d_flags &= ~DCACHE_OP_REVALIDATE;
-+		spin_unlock(&dentry->d_lock);
-+	}
-+}
-+
- /* crypto.c */
- void fscrypt_enqueue_decrypt_work(struct work_struct *);
- 
-struct page *fscrypt_encrypt_pagecache_blocks(struct page *page,
-					      unsigned int len,
-					      unsigned int offs,
-					      gfp_t gfp_flags);
-+struct page *fscrypt_encrypt_pagecache_blocks(struct folio *folio,
-+		size_t len, size_t offs, gfp_t gfp_flags);
- int fscrypt_encrypt_block_inplace(const struct inode *inode, struct page *page,
- 				  unsigned int len, unsigned int offs,
-				  u64 lblk_num, gfp_t gfp_flags);
-+				  u64 lblk_num);
- 
-int fscrypt_decrypt_pagecache_blocks(struct page *page, unsigned int len,
-				     unsigned int offs);
-+int fscrypt_decrypt_pagecache_blocks(struct folio *folio, size_t len,
-+				     size_t offs);
- int fscrypt_decrypt_block_inplace(const struct inode *inode, struct page *page,
- 				  unsigned int len, unsigned int offs,
- 				  u64 lblk_num);
-@@ -276,6 +366,17 @@ static inline struct page *fscrypt_pagecache_page(struct page *bounce_page)
- 	return (struct page *)page_private(bounce_page);
- }
- 
-+static inline bool fscrypt_is_bounce_folio(const struct folio *folio)
-+{
-+	return folio->mapping == NULL;
-+}
-+
-+static inline
-+struct folio *fscrypt_pagecache_folio(const struct folio *bounce_folio)
-+{
-+	return bounce_folio->private;
-+}
-+
- void fscrypt_free_bounce_page(struct page *bounce_page);
- 
- /* policy.c */
-@@ -295,8 +396,6 @@ int fscrypt_parse_test_dummy_encryption(const struct fs_parameter *param,
- 				    struct fscrypt_dummy_policy *dummy_policy);
- bool fscrypt_dummy_policies_equal(const struct fscrypt_dummy_policy *p1,
- 				  const struct fscrypt_dummy_policy *p2);
-int fscrypt_set_test_dummy_encryption(struct super_block *sb, const char *arg,
-				struct fscrypt_dummy_policy *dummy_policy);
- void fscrypt_show_test_dummy_encryption(struct seq_file *seq, char sep,
- 					struct super_block *sb);
- static inline bool
-@@ -312,10 +411,8 @@ fscrypt_free_dummy_policy(struct fscrypt_dummy_policy *dummy_policy)
- }
- 
- /* keyring.c */
-void fscrypt_sb_free(struct super_block *sb);
-+void fscrypt_destroy_keyring(struct super_block *sb);
- int fscrypt_ioctl_add_key(struct file *filp, void __user *arg);
-int fscrypt_add_test_dummy_key(struct super_block *sb,
-			       const struct fscrypt_dummy_policy *dummy_policy);
- int fscrypt_ioctl_remove_key(struct file *filp, void __user *arg);
- int fscrypt_ioctl_remove_key_all_users(struct file *filp, void __user *arg);
- int fscrypt_ioctl_get_key_status(struct file *filp, void __user *arg);
-@@ -350,10 +447,9 @@ int fscrypt_fname_disk_to_usr(const struct inode *inode,
- bool fscrypt_match_name(const struct fscrypt_name *fname,
- 			const u8 *de_name, u32 de_name_len);
- u64 fscrypt_fname_siphash(const struct inode *dir, const struct qstr *name);
-int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags);
- 
- /* bio.c */
-void fscrypt_decrypt_bio(struct bio *bio);
-+bool fscrypt_decrypt_bio(struct bio *bio);
- int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk,
- 			  sector_t pblk, unsigned int len);
- 
-@@ -366,6 +462,7 @@ int __fscrypt_prepare_rename(struct inode *old_dir, struct dentry *old_dentry,
- 			     unsigned int flags);
- int __fscrypt_prepare_lookup(struct inode *dir, struct dentry *dentry,
- 			     struct fscrypt_name *fname);
-+int fscrypt_prepare_lookup_partial(struct inode *dir, struct dentry *dentry);
- int __fscrypt_prepare_readdir(struct inode *dir);
- int __fscrypt_prepare_setattr(struct dentry *dentry, struct iattr *attr);
- int fscrypt_prepare_setflags(struct inode *inode,
-@@ -386,7 +483,8 @@ static inline void fscrypt_set_ops(struct super_block *sb,
- }
- #else  /* !CONFIG_FS_ENCRYPTION */
- 
-static inline struct fscrypt_info *fscrypt_get_info(const struct inode *inode)
-+static inline struct fscrypt_inode_info *
-+fscrypt_get_inode_info(const struct inode *inode)
- {
- 	return NULL;
- }
-@@ -405,15 +503,18 @@ static inline bool fscrypt_is_nokey_name(const struct dentry *dentry)
- 	return false;
- }
- 
-+static inline void fscrypt_prepare_dentry(struct dentry *dentry,
-+					  bool is_nokey_name)
-+{
-+}
-+
- /* crypto.c */
- static inline void fscrypt_enqueue_decrypt_work(struct work_struct *work)
- {
- }
- 
-static inline struct page *fscrypt_encrypt_pagecache_blocks(struct page *page,
-							    unsigned int len,
-							    unsigned int offs,
-							    gfp_t gfp_flags)
-+static inline struct page *fscrypt_encrypt_pagecache_blocks(struct folio *folio,
-+		size_t len, size_t offs, gfp_t gfp_flags)
- {
- 	return ERR_PTR(-EOPNOTSUPP);
- }
-@@ -421,15 +522,13 @@ static inline struct page *fscrypt_encrypt_pagecache_blocks(struct page *page,
- static inline int fscrypt_encrypt_block_inplace(const struct inode *inode,
- 						struct page *page,
- 						unsigned int len,
-						unsigned int offs, u64 lblk_num,
-						gfp_t gfp_flags)
-+						unsigned int offs, u64 lblk_num)
- {
- 	return -EOPNOTSUPP;
- }
- 
-static inline int fscrypt_decrypt_pagecache_blocks(struct page *page,
-						   unsigned int len,
-						   unsigned int offs)
-+static inline int fscrypt_decrypt_pagecache_blocks(struct folio *folio,
-+						   size_t len, size_t offs)
- {
- 	return -EOPNOTSUPP;
- }
-@@ -453,6 +552,18 @@ static inline struct page *fscrypt_pagecache_page(struct page *bounce_page)
- 	return ERR_PTR(-EINVAL);
- }
- 
-+static inline bool fscrypt_is_bounce_folio(const struct folio *folio)
-+{
-+	return false;
-+}
-+
-+static inline
-+struct folio *fscrypt_pagecache_folio(const struct folio *bounce_folio)
-+{
-+	WARN_ON_ONCE(1);
-+	return ERR_PTR(-EINVAL);
-+}
-+
- static inline void fscrypt_free_bounce_page(struct page *bounce_page)
- {
- }
-@@ -526,7 +637,7 @@ fscrypt_free_dummy_policy(struct fscrypt_dummy_policy *dummy_policy)
- }
- 
- /* keyring.c */
-static inline void fscrypt_sb_free(struct super_block *sb)
-+static inline void fscrypt_destroy_keyring(struct super_block *sb)
- {
- }
- 
-@@ -535,13 +646,6 @@ static inline int fscrypt_ioctl_add_key(struct file *filp, void __user *arg)
- 	return -EOPNOTSUPP;
- }
- 
-static inline int
-fscrypt_add_test_dummy_key(struct super_block *sb,
-			   const struct fscrypt_dummy_policy *dummy_policy)
-{
-	return 0;
-}
-
- static inline int fscrypt_ioctl_remove_key(struct file *filp, void __user *arg)
- {
- 	return -EOPNOTSUPP;
-@@ -639,15 +743,16 @@ static inline u64 fscrypt_fname_siphash(const struct inode *dir,
- 	return 0;
- }
- 
-static inline int fscrypt_d_revalidate(struct dentry *dentry,
-				       unsigned int flags)
-+static inline int fscrypt_d_revalidate(struct inode *dir, const struct qstr *name,
-+				       struct dentry *dentry, unsigned int flags)
- {
- 	return 1;
- }
- 
- /* bio.c */
-static inline void fscrypt_decrypt_bio(struct bio *bio)
-+static inline bool fscrypt_decrypt_bio(struct bio *bio)
- {
-+	return true;
- }
- 
- static inline int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk,
-@@ -687,6 +792,12 @@ static inline int __fscrypt_prepare_lookup(struct inode *dir,
- 	return -EOPNOTSUPP;
- }
- 
-+static inline int fscrypt_prepare_lookup_partial(struct inode *dir,
-+						 struct dentry *dentry)
-+{
-+	return -EOPNOTSUPP;
-+}
-+
- static inline int __fscrypt_prepare_readdir(struct inode *dir)
- {
- 	return -EOPNOTSUPP;
-@@ -768,7 +879,7 @@ bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode,
- bool fscrypt_mergeable_bio_bh(struct bio *bio,
- 			      const struct buffer_head *next_bh);
- 
-bool fscrypt_dio_supported(struct kiocb *iocb, struct iov_iter *iter);
-+bool fscrypt_dio_supported(struct inode *inode);
- 
- u64 fscrypt_limit_io_blocks(const struct inode *inode, u64 lblk, u64 nr_blocks);
- 
-@@ -801,11 +912,8 @@ static inline bool fscrypt_mergeable_bio_bh(struct bio *bio,
- 	return true;
- }
- 
-static inline bool fscrypt_dio_supported(struct kiocb *iocb,
-					 struct iov_iter *iter)
-+static inline bool fscrypt_dio_supported(struct inode *inode)
- {
-	const struct inode *inode = file_inode(iocb->ki_filp);
-
- 	return !fscrypt_needs_contents_encryption(inode);
- }
- 
-@@ -857,7 +965,7 @@ static inline bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode)
-  */
- static inline bool fscrypt_has_encryption_key(const struct inode *inode)
- {
-	return fscrypt_get_info(inode) != NULL;
-+	return fscrypt_get_inode_info(inode) != NULL;
- }
- 
- /**
-@@ -955,6 +1063,9 @@ static inline int fscrypt_prepare_lookup(struct inode *dir,
- 	fname->usr_fname = &dentry->d_name;
- 	fname->disk_name.name = (unsigned char *)dentry->d_name.name;
- 	fname->disk_name.len = dentry->d_name.len;
-+
-+	fscrypt_prepare_dentry(dentry, false);
-+
- 	return 0;
- }
- 
-diff --git a/include/linux/fsi.h b/include/linux/fsi.h
-index 3df8c54868df..adea1b432f2d 100644
--- a/include/linux/fsi.h
-+++ b/include/linux/fsi.h
-@@ -44,7 +44,7 @@ struct fsi_driver {
- };
- 
- #define to_fsi_dev(devp) container_of(devp, struct fsi_device, dev)
-#define to_fsi_drv(drvp) container_of(drvp, struct fsi_driver, drv)
-+#define to_fsi_drv(drvp) container_of_const(drvp, struct fsi_driver, drv)
- 
- extern int fsi_driver_register(struct fsi_driver *fsi_drv);
- extern void fsi_driver_unregister(struct fsi_driver *fsi_drv);
-@@ -68,7 +68,7 @@ extern int fsi_slave_read(struct fsi_slave *slave, uint32_t addr,
- extern int fsi_slave_write(struct fsi_slave *slave, uint32_t addr,
- 		const void *val, size_t size);
- 
-extern struct bus_type fsi_bus_type;
-+extern const struct bus_type fsi_bus_type;
- extern const struct device_type fsi_cdev_type;
- 
- enum fsi_dev_type {
-diff --git a/include/linux/fsl/enetc_mdio.h b/include/linux/fsl/enetc_mdio.h
-index 2d9203314865..623ccfcbf39c 100644
--- a/include/linux/fsl/enetc_mdio.h
-+++ b/include/linux/fsl/enetc_mdio.h
-@@ -37,18 +37,30 @@ struct enetc_mdio_priv {
- 
- #if IS_REACHABLE(CONFIG_FSL_ENETC_MDIO)
- 
-int enetc_mdio_read(struct mii_bus *bus, int phy_id, int regnum);
-int enetc_mdio_write(struct mii_bus *bus, int phy_id, int regnum, u16 value);
-+int enetc_mdio_read_c22(struct mii_bus *bus, int phy_id, int regnum);
-+int enetc_mdio_write_c22(struct mii_bus *bus, int phy_id, int regnum,
-+			 u16 value);
-+int enetc_mdio_read_c45(struct mii_bus *bus, int phy_id, int devad, int regnum);
-+int enetc_mdio_write_c45(struct mii_bus *bus, int phy_id, int devad, int regnum,
-+			 u16 value);
- struct enetc_hw *enetc_hw_alloc(struct device *dev, void __iomem *port_regs);
- 
- #else
- 
-static inline int enetc_mdio_read(struct mii_bus *bus, int phy_id, int regnum)
-+static inline int enetc_mdio_read_c22(struct mii_bus *bus, int phy_id,
-+				      int regnum)
- { return -EINVAL; }
-static inline int enetc_mdio_write(struct mii_bus *bus, int phy_id, int regnum,
-				   u16 value)
-+static inline int enetc_mdio_write_c22(struct mii_bus *bus, int phy_id,
-+				       int regnum, u16 value)
- { return -EINVAL; }
-struct enetc_hw *enetc_hw_alloc(struct device *dev, void __iomem *port_regs)
-+static inline int enetc_mdio_read_c45(struct mii_bus *bus, int phy_id,
-+				      int devad, int regnum)
-+{ return -EINVAL; }
-+static inline int enetc_mdio_write_c45(struct mii_bus *bus, int phy_id,
-+				       int devad, int regnum, u16 value)
-+{ return -EINVAL; }
-+static inline struct enetc_hw *enetc_hw_alloc(struct device *dev,
-+					      void __iomem *port_regs)
- { return ERR_PTR(-EINVAL); }
- 
- #endif
-diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h
-index a86115bc799c..897d6211c163 100644
--- a/include/linux/fsl/mc.h
-+++ b/include/linux/fsl/mc.h
-@@ -48,7 +48,7 @@ struct fsl_mc_driver {
- 	struct device_driver driver;
- 	const struct fsl_mc_device_id *match_id_table;
- 	int (*probe)(struct fsl_mc_device *dev);
-	int (*remove)(struct fsl_mc_device *dev);
-+	void (*remove)(struct fsl_mc_device *dev);
- 	void (*shutdown)(struct fsl_mc_device *dev);
- 	int (*suspend)(struct fsl_mc_device *dev, pm_message_t state);
- 	int (*resume)(struct fsl_mc_device *dev);
-@@ -56,7 +56,7 @@ struct fsl_mc_driver {
- };
- 
- #define to_fsl_mc_driver(_drv) \
-	container_of(_drv, struct fsl_mc_driver, driver)
-+	container_of_const(_drv, struct fsl_mc_driver, driver)
- 
- /**
-  * enum fsl_mc_pool_type - Types of allocatable MC bus resources
-@@ -417,8 +417,6 @@ int __must_check fsl_mc_portal_allocate(struct fsl_mc_device *mc_dev,
- 
- void fsl_mc_portal_free(struct fsl_mc_io *mc_io);
- 
-int fsl_mc_portal_reset(struct fsl_mc_io *mc_io);
-
- int __must_check fsl_mc_object_allocate(struct fsl_mc_device *mc_dev,
- 					enum fsl_mc_pool_type pool_type,
- 					struct fsl_mc_device **new_mc_adev);
-@@ -436,23 +434,23 @@ void fsl_mc_free_irqs(struct fsl_mc_device *mc_dev);
- struct fsl_mc_device *fsl_mc_get_endpoint(struct fsl_mc_device *mc_dev,
- 					  u16 if_id);
- 
-extern struct bus_type fsl_mc_bus_type;
-
-extern struct device_type fsl_mc_bus_dprc_type;
-extern struct device_type fsl_mc_bus_dpni_type;
-extern struct device_type fsl_mc_bus_dpio_type;
-extern struct device_type fsl_mc_bus_dpsw_type;
-extern struct device_type fsl_mc_bus_dpbp_type;
-extern struct device_type fsl_mc_bus_dpcon_type;
-extern struct device_type fsl_mc_bus_dpmcp_type;
-extern struct device_type fsl_mc_bus_dpmac_type;
-extern struct device_type fsl_mc_bus_dprtc_type;
-extern struct device_type fsl_mc_bus_dpseci_type;
-extern struct device_type fsl_mc_bus_dpdmux_type;
-extern struct device_type fsl_mc_bus_dpdcei_type;
-extern struct device_type fsl_mc_bus_dpaiop_type;
-extern struct device_type fsl_mc_bus_dpci_type;
-extern struct device_type fsl_mc_bus_dpdmai_type;
-+extern const struct bus_type fsl_mc_bus_type;
-+
-+extern const struct device_type fsl_mc_bus_dprc_type;
-+extern const struct device_type fsl_mc_bus_dpni_type;
-+extern const struct device_type fsl_mc_bus_dpio_type;
-+extern const struct device_type fsl_mc_bus_dpsw_type;
-+extern const struct device_type fsl_mc_bus_dpbp_type;
-+extern const struct device_type fsl_mc_bus_dpcon_type;
-+extern const struct device_type fsl_mc_bus_dpmcp_type;
-+extern const struct device_type fsl_mc_bus_dpmac_type;
-+extern const struct device_type fsl_mc_bus_dprtc_type;
-+extern const struct device_type fsl_mc_bus_dpseci_type;
-+extern const struct device_type fsl_mc_bus_dpdmux_type;
-+extern const struct device_type fsl_mc_bus_dpdcei_type;
-+extern const struct device_type fsl_mc_bus_dpaiop_type;
-+extern const struct device_type fsl_mc_bus_dpci_type;
-+extern const struct device_type fsl_mc_bus_dpdmai_type;
- 
- static inline bool is_fsl_mc_bus_dprc(const struct fsl_mc_device *mc_dev)
- {
-diff --git a/include/linux/fsl/netc_global.h b/include/linux/fsl/netc_global.h
-new file mode 100644
-index 000000000000..fdecca8c90f0
--- /dev/null
-+++ b/include/linux/fsl/netc_global.h
-@@ -0,0 +1,19 @@
-+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
-+/* Copyright 2024 NXP
-+ */
-+#ifndef __NETC_GLOBAL_H
-+#define __NETC_GLOBAL_H
-+
-+#include <linux/io.h>
-+
-+static inline u32 netc_read(void __iomem *reg)
-+{
-+	return ioread32(reg);
-+}
-+
-+static inline void netc_write(void __iomem *reg, u32 val)
-+{
-+	iowrite32(val, reg);
-+}
-+
-+#endif
-diff --git a/include/linux/fsl/ntmp.h b/include/linux/fsl/ntmp.h
-new file mode 100644
-index 000000000000..916dc4fe7de3
--- /dev/null
-+++ b/include/linux/fsl/ntmp.h
-@@ -0,0 +1,121 @@
-+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
-+/* Copyright 2025 NXP */
-+#ifndef __NETC_NTMP_H
-+#define __NETC_NTMP_H
-+
-+#include <linux/bitops.h>
-+#include <linux/if_ether.h>
-+
-+struct maft_keye_data {
-+	u8 mac_addr[ETH_ALEN];
-+	__le16 resv;
-+};
-+
-+struct maft_cfge_data {
-+	__le16 si_bitmap;
-+	__le16 resv;
-+};
-+
-+struct netc_cbdr_regs {
-+	void __iomem *pir;
-+	void __iomem *cir;
-+	void __iomem *mr;
-+
-+	void __iomem *bar0;
-+	void __iomem *bar1;
-+	void __iomem *lenr;
-+};
-+
-+struct netc_tbl_vers {
-+	u8 maft_ver;
-+	u8 rsst_ver;
-+};
-+
-+struct netc_cbdr {
-+	struct device *dev;
-+	struct netc_cbdr_regs regs;
-+
-+	int bd_num;
-+	int next_to_use;
-+	int next_to_clean;
-+
-+	int dma_size;
-+	void *addr_base;
-+	void *addr_base_align;
-+	dma_addr_t dma_base;
-+	dma_addr_t dma_base_align;
-+
-+	/* Serialize the order of command BD ring */
-+	spinlock_t ring_lock;
-+};
-+
-+struct ntmp_user {
-+	int cbdr_num;	/* number of control BD ring */
-+	struct device *dev;
-+	struct netc_cbdr *ring;
-+	struct netc_tbl_vers tbl;
-+};
-+
-+struct maft_entry_data {
-+	struct maft_keye_data keye;
-+	struct maft_cfge_data cfge;
-+};
-+
-+#if IS_ENABLED(CONFIG_NXP_NETC_LIB)
-+int ntmp_init_cbdr(struct netc_cbdr *cbdr, struct device *dev,
-+		   const struct netc_cbdr_regs *regs);
-+void ntmp_free_cbdr(struct netc_cbdr *cbdr);
-+
-+/* NTMP APIs */
-+int ntmp_maft_add_entry(struct ntmp_user *user, u32 entry_id,
-+			struct maft_entry_data *maft);
-+int ntmp_maft_query_entry(struct ntmp_user *user, u32 entry_id,
-+			  struct maft_entry_data *maft);
-+int ntmp_maft_delete_entry(struct ntmp_user *user, u32 entry_id);
-+int ntmp_rsst_update_entry(struct ntmp_user *user, const u32 *table,
-+			   int count);
-+int ntmp_rsst_query_entry(struct ntmp_user *user,
-+			  u32 *table, int count);
-+#else
-+static inline int ntmp_init_cbdr(struct netc_cbdr *cbdr, struct device *dev,
-+				 const struct netc_cbdr_regs *regs)
-+{
-+	return 0;
-+}
-+
-+static inline void ntmp_free_cbdr(struct netc_cbdr *cbdr)
-+{
-+}
-+
-+static inline int ntmp_maft_add_entry(struct ntmp_user *user, u32 entry_id,
-+				      struct maft_entry_data *maft)
-+{
-+	return 0;
-+}
-+
-+static inline int ntmp_maft_query_entry(struct ntmp_user *user, u32 entry_id,
-+					struct maft_entry_data *maft)
-+{
-+	return 0;
-+}
-+
-+static inline int ntmp_maft_delete_entry(struct ntmp_user *user, u32 entry_id)
-+{
-+	return 0;
-+}
-+
-+static inline int ntmp_rsst_update_entry(struct ntmp_user *user,
-+					 const u32 *table, int count)
-+{
-+	return 0;
-+}
-+
-+static inline int ntmp_rsst_query_entry(struct ntmp_user *user,
-+					u32 *table, int count)
-+{
-+	return 0;
-+}
-+
-+#endif
-+
-+#endif
-diff --git a/include/linux/fsl/ptp_qoriq.h b/include/linux/fsl/ptp_qoriq.h
-index 01acebe37fab..3601e25779ba 100644
--- a/include/linux/fsl/ptp_qoriq.h
-+++ b/include/linux/fsl/ptp_qoriq.h
-@@ -145,10 +145,10 @@ struct ptp_qoriq {
- 	struct ptp_clock *clock;
- 	struct ptp_clock_info caps;
- 	struct resource *rsrc;
-	struct dentry *debugfs_root;
- 	struct device *dev;
- 	bool extts_fifo_support;
- 	bool fiper3_support;
-+	bool etsec;
- 	int irq;
- 	int phc_index;
- 	u32 tclk_period;  /* nanoseconds */
-@@ -194,14 +194,5 @@ int ptp_qoriq_settime(struct ptp_clock_info *ptp,
- int ptp_qoriq_enable(struct ptp_clock_info *ptp,
- 		     struct ptp_clock_request *rq, int on);
- int extts_clean_up(struct ptp_qoriq *ptp_qoriq, int index, bool update_event);
-#ifdef CONFIG_DEBUG_FS
-void ptp_qoriq_create_debugfs(struct ptp_qoriq *ptp_qoriq);
-void ptp_qoriq_remove_debugfs(struct ptp_qoriq *ptp_qoriq);
-#else
-static inline void ptp_qoriq_create_debugfs(struct ptp_qoriq *ptp_qoriq)
-{ }
-static inline void ptp_qoriq_remove_debugfs(struct ptp_qoriq *ptp_qoriq)
-{ }
-#endif
- 
- #endif
-diff --git a/include/linux/fsl_devices.h b/include/linux/fsl_devices.h
-index 5d231ce8709b..49f20c2f99bf 100644
--- a/include/linux/fsl_devices.h
-+++ b/include/linux/fsl_devices.h
-@@ -118,7 +118,6 @@ struct fsl_usb2_platform_data {
- #define FSL_USB2_PORT0_ENABLED	0x00000001
- #define FSL_USB2_PORT1_ENABLED	0x00000002
- 
-#define FLS_USB2_WORKAROUND_ENGCM09152	(1 << 0)
- 
- struct spi_device;
- 
-diff --git a/include/linux/fsnotify.h b/include/linux/fsnotify.h
-index bb8467cd11ae..28a9cb13fbfa 100644
--- a/include/linux/fsnotify.h
-+++ b/include/linux/fsnotify.h
-@@ -17,6 +17,25 @@
- #include <linux/slab.h>
- #include <linux/bug.h>
- 
-+/* Are there any inode/mount/sb objects watched with priority prio or above? */
-+static inline bool fsnotify_sb_has_priority_watchers(struct super_block *sb,
-+						     int prio)
-+{
-+	struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb);
-+
-+	/* Were any marks ever added to any object on this sb? */
-+	if (!sbinfo)
-+		return false;
-+
-+	return atomic_long_read(&sbinfo->watched_objects[prio]);
-+}
-+
-+/* Are there any inode/mount/sb objects that are being watched at all? */
-+static inline bool fsnotify_sb_has_watchers(struct super_block *sb)
-+{
-+	return fsnotify_sb_has_priority_watchers(sb, 0);
-+}
-+
- /*
-  * Notify this @dir inode about a change in a child directory entry.
-  * The directory entry may have turned positive or negative or its inode may
-@@ -30,7 +49,7 @@ static inline int fsnotify_name(__u32 mask, const void *data, int data_type,
- 				struct inode *dir, const struct qstr *name,
- 				u32 cookie)
- {
-	if (atomic_long_read(&dir->i_sb->s_fsnotify_connectors) == 0)
-+	if (!fsnotify_sb_has_watchers(dir->i_sb))
- 		return 0;
- 
- 	return fsnotify(mask, data, data_type, dir, name, NULL, cookie);
-@@ -44,7 +63,7 @@ static inline void fsnotify_dirent(struct inode *dir, struct dentry *dentry,
- 
- static inline void fsnotify_inode(struct inode *inode, __u32 mask)
- {
-	if (atomic_long_read(&inode->i_sb->s_fsnotify_connectors) == 0)
-+	if (!fsnotify_sb_has_watchers(inode->i_sb))
- 		return;
- 
- 	if (S_ISDIR(inode->i_mode))
-@@ -59,7 +78,7 @@ static inline int fsnotify_parent(struct dentry *dentry, __u32 mask,
- {
- 	struct inode *inode = d_inode(dentry);
- 
-	if (atomic_long_read(&inode->i_sb->s_fsnotify_connectors) == 0)
-+	if (!fsnotify_sb_has_watchers(inode->i_sb))
- 		return 0;
- 
- 	if (S_ISDIR(inode->i_mode)) {
-@@ -89,40 +108,132 @@ static inline void fsnotify_dentry(struct dentry *dentry, __u32 mask)
- 	fsnotify_parent(dentry, mask, dentry, FSNOTIFY_EVENT_DENTRY);
- }
- 
-+static inline int fsnotify_path(const struct path *path, __u32 mask)
-+{
-+	return fsnotify_parent(path->dentry, mask, path, FSNOTIFY_EVENT_PATH);
-+}
-+
- static inline int fsnotify_file(struct file *file, __u32 mask)
- {
-	const struct path *path = &file->f_path;
-+	/*
-+	 * FMODE_NONOTIFY are fds generated by fanotify itself which should not
-+	 * generate new events. We also don't want to generate events for
-+	 * FMODE_PATH fds (involves open & close events) as they are just
-+	 * handle creation / destruction events and not "real" file events.
-+	 */
-+	if (FMODE_FSNOTIFY_NONE(file->f_mode))
-+		return 0;
-+
-+	return fsnotify_path(&file->f_path, mask);
-+}
-+
-+#ifdef CONFIG_FANOTIFY_ACCESS_PERMISSIONS
-+
-+int fsnotify_open_perm_and_set_mode(struct file *file);
- 
-	if (file->f_mode & FMODE_NONOTIFY)
-+/*
-+ * fsnotify_file_area_perm - permission hook before access to file range
-+ */
-+static inline int fsnotify_file_area_perm(struct file *file, int perm_mask,
-+					  const loff_t *ppos, size_t count)
-+{
-+	/*
-+	 * filesystem may be modified in the context of permission events
-+	 * (e.g. by HSM filling a file on access), so sb freeze protection
-+	 * must not be held.
-+	 */
-+	lockdep_assert_once(file_write_not_started(file));
-+
-+	if (!(perm_mask & (MAY_READ | MAY_WRITE | MAY_ACCESS)))
- 		return 0;
- 
-	return fsnotify_parent(path->dentry, mask, path, FSNOTIFY_EVENT_PATH);
-+	/*
-+	 * read()/write() and other types of access generate pre-content events.
-+	 */
-+	if (unlikely(FMODE_FSNOTIFY_HSM(file->f_mode))) {
-+		int ret = fsnotify_pre_content(&file->f_path, ppos, count);
-+
-+		if (ret)
-+			return ret;
-+	}
-+
-+	if (!(perm_mask & MAY_READ) ||
-+	    likely(!FMODE_FSNOTIFY_ACCESS_PERM(file->f_mode)))
-+		return 0;
-+
-+	/*
-+	 * read() also generates the legacy FS_ACCESS_PERM event, so content
-+	 * scanners can inspect the content filled by pre-content event.
-+	 */
-+	return fsnotify_path(&file->f_path, FS_ACCESS_PERM);
- }
- 
-/* Simple call site for access decisions */
-static inline int fsnotify_perm(struct file *file, int mask)
-+/*
-+ * fsnotify_mmap_perm - permission hook before mmap of file range
-+ */
-+static inline int fsnotify_mmap_perm(struct file *file, int prot,
-+				     const loff_t off, size_t len)
- {
-	int ret;
-	__u32 fsnotify_mask = 0;
-+	/*
-+	 * mmap() generates only pre-content events.
-+	 */
-+	if (!file || likely(!FMODE_FSNOTIFY_HSM(file->f_mode)))
-+		return 0;
- 
-	if (!(mask & (MAY_READ | MAY_OPEN)))
-+	return fsnotify_pre_content(&file->f_path, &off, len);
-+}
-+
-+/*
-+ * fsnotify_truncate_perm - permission hook before file truncate
-+ */
-+static inline int fsnotify_truncate_perm(const struct path *path, loff_t length)
-+{
-+	struct inode *inode = d_inode(path->dentry);
-+
-+	if (!(inode->i_sb->s_iflags & SB_I_ALLOW_HSM) ||
-+	    !fsnotify_sb_has_priority_watchers(inode->i_sb,
-+					       FSNOTIFY_PRIO_PRE_CONTENT))
- 		return 0;
- 
-	if (mask & MAY_OPEN) {
-		fsnotify_mask = FS_OPEN_PERM;
-+	return fsnotify_pre_content(path, &length, 0);
-+}
- 
-		if (file->f_flags & __FMODE_EXEC) {
-			ret = fsnotify_file(file, FS_OPEN_EXEC_PERM);
-+/*
-+ * fsnotify_file_perm - permission hook before file access (unknown range)
-+ */
-+static inline int fsnotify_file_perm(struct file *file, int perm_mask)
-+{
-+	return fsnotify_file_area_perm(file, perm_mask, NULL, 0);
-+}
- 
-			if (ret)
-				return ret;
-		}
-	} else if (mask & MAY_READ) {
-		fsnotify_mask = FS_ACCESS_PERM;
-	}
-+#else
-+static inline int fsnotify_open_perm_and_set_mode(struct file *file)
-+{
-+	return 0;
-+}
-+
-+static inline int fsnotify_file_area_perm(struct file *file, int perm_mask,
-+					  const loff_t *ppos, size_t count)
-+{
-+	return 0;
-+}
-+
-+static inline int fsnotify_mmap_perm(struct file *file, int prot,
-+				     const loff_t off, size_t len)
-+{
-+	return 0;
-+}
- 
-	return fsnotify_file(file, fsnotify_mask);
-+static inline int fsnotify_truncate_perm(const struct path *path, loff_t length)
-+{
-+	return 0;
-+}
-+
-+static inline int fsnotify_file_perm(struct file *file, int perm_mask)
-+{
-+	return 0;
- }
-+#endif
- 
- /*
-  * fsnotify_link_count - inode's link count changed
-@@ -184,6 +295,11 @@ static inline void fsnotify_vfsmount_delete(struct vfsmount *mnt)
- 	__fsnotify_vfsmount_delete(mnt);
- }
- 
-+static inline void fsnotify_mntns_delete(struct mnt_namespace *mntns)
-+{
-+	__fsnotify_mntns_delete(mntns);
-+}
-+
- /*
-  * fsnotify_inoderemove - an inode is going away
-  */
-@@ -392,4 +508,19 @@ static inline int fsnotify_sb_error(struct super_block *sb, struct inode *inode,
- 			NULL, NULL, NULL, 0);
- }
- 
-+static inline void fsnotify_mnt_attach(struct mnt_namespace *ns, struct vfsmount *mnt)
-+{
-+	fsnotify_mnt(FS_MNT_ATTACH, ns, mnt);
-+}
-+
-+static inline void fsnotify_mnt_detach(struct mnt_namespace *ns, struct vfsmount *mnt)
-+{
-+	fsnotify_mnt(FS_MNT_DETACH, ns, mnt);
-+}
-+
-+static inline void fsnotify_mnt_move(struct mnt_namespace *ns, struct vfsmount *mnt)
-+{
-+	fsnotify_mnt(FS_MNT_MOVE, ns, mnt);
-+}
-+
- #endif	/* _LINUX_FS_NOTIFY_H */
-diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h
-index d7d96c806bff..0d954ea7b179 100644
--- a/include/linux/fsnotify_backend.h
-+++ b/include/linux/fsnotify_backend.h
-@@ -31,8 +31,8 @@
- #define FS_ACCESS		0x00000001	/* File was accessed */
- #define FS_MODIFY		0x00000002	/* File was modified */
- #define FS_ATTRIB		0x00000004	/* Metadata changed */
-#define FS_CLOSE_WRITE		0x00000008	/* Writtable file was closed */
-#define FS_CLOSE_NOWRITE	0x00000010	/* Unwrittable file closed */
-+#define FS_CLOSE_WRITE		0x00000008	/* Writable file was closed */
-+#define FS_CLOSE_NOWRITE	0x00000010	/* Unwritable file closed */
- #define FS_OPEN			0x00000020	/* File was opened */
- #define FS_MOVED_FROM		0x00000040	/* File was moved from X */
- #define FS_MOVED_TO		0x00000080	/* File was moved to Y */
-@@ -55,6 +55,13 @@
- #define FS_OPEN_PERM		0x00010000	/* open event in an permission hook */
- #define FS_ACCESS_PERM		0x00020000	/* access event in a permissions hook */
- #define FS_OPEN_EXEC_PERM	0x00040000	/* open/exec event in a permission hook */
-+/* #define FS_DIR_MODIFY	0x00080000 */	/* Deprecated (reserved) */
-+
-+#define FS_PRE_ACCESS		0x00100000	/* Pre-content access hook */
-+
-+#define FS_MNT_ATTACH		0x01000000	/* Mount was attached */
-+#define FS_MNT_DETACH		0x02000000	/* Mount was detached */
-+#define FS_MNT_MOVE		(FS_MNT_ATTACH | FS_MNT_DETACH)
- 
- /*
-  * Set on inode mark that cares about things that happen to its children.
-@@ -77,8 +84,17 @@
-  */
- #define ALL_FSNOTIFY_DIRENT_EVENTS (FS_CREATE | FS_DELETE | FS_MOVE | FS_RENAME)
- 
-#define ALL_FSNOTIFY_PERM_EVENTS (FS_OPEN_PERM | FS_ACCESS_PERM | \
-				  FS_OPEN_EXEC_PERM)
-+/* Mount namespace events */
-+#define FSNOTIFY_MNT_EVENTS (FS_MNT_ATTACH | FS_MNT_DETACH)
-+
-+/* Content events can be used to inspect file content */
-+#define FSNOTIFY_CONTENT_PERM_EVENTS (FS_OPEN_PERM | FS_OPEN_EXEC_PERM | \
-+				      FS_ACCESS_PERM)
-+/* Pre-content events can be used to fill file content */
-+#define FSNOTIFY_PRE_CONTENT_EVENTS  (FS_PRE_ACCESS)
-+
-+#define ALL_FSNOTIFY_PERM_EVENTS (FSNOTIFY_CONTENT_PERM_EVENTS | \
-+				  FSNOTIFY_PRE_CONTENT_EVENTS)
- 
- /*
-  * This is a list of all events that may get sent to a parent that is watching
-@@ -99,6 +115,7 @@
- 
- /* Events that can be reported to backends */
- #define ALL_FSNOTIFY_EVENTS (ALL_FSNOTIFY_DIRENT_EVENTS | \
-+			     FSNOTIFY_MNT_EVENTS | \
- 			     FS_EVENTS_POSS_ON_CHILD | \
- 			     FS_DELETE_SELF | FS_MOVE_SELF | \
- 			     FS_UNMOUNT | FS_Q_OVERFLOW | FS_IN_IGNORED | \
-@@ -177,6 +194,17 @@ struct fsnotify_event {
- };
- 
- /*
-+ * fsnotify group priorities.
-+ * Events are sent in order from highest priority to lowest priority.
-+ */
-+enum fsnotify_group_prio {
-+	FSNOTIFY_PRIO_NORMAL = 0,	/* normal notifiers, no permissions */
-+	FSNOTIFY_PRIO_CONTENT,		/* fanotify permission events */
-+	FSNOTIFY_PRIO_PRE_CONTENT,	/* fanotify pre-content events */
-+	__FSNOTIFY_PRIO_NUM
-+};
-+
-+/*
-  * A group is a "thing" that wants to receive notification about filesystem
-  * events.  The mask holds the subset of event types this group cares about.
-  * refcnt on a group is up to the implementor and at any moment if it goes 0
-@@ -201,19 +229,11 @@ struct fsnotify_group {
- 	wait_queue_head_t notification_waitq;	/* read() on the notification file blocks on this waitq */
- 	unsigned int q_len;			/* events on the queue */
- 	unsigned int max_events;		/* maximum events allowed on the list */
-	/*
-	 * Valid fsnotify group priorities.  Events are send in order from highest
-	 * priority to lowest priority.  We default to the lowest priority.
-	 */
-	#define FS_PRIO_0	0 /* normal notifiers, no permissions */
-	#define FS_PRIO_1	1 /* fanotify content based access control */
-	#define FS_PRIO_2	2 /* fanotify pre-content access */
-	unsigned int priority;
-+	enum fsnotify_group_prio priority;	/* priority for sending events */
- 	bool shutdown;		/* group is being shut down, don't queue more events */
- 
- #define FSNOTIFY_GROUP_USER	0x01 /* user allocated group */
- #define FSNOTIFY_GROUP_DUPS	0x02 /* allow multiple marks per object */
-#define FSNOTIFY_GROUP_NOFS	0x04 /* group lock is not direct reclaim safe */
- 	int flags;
- 	unsigned int owner_flags;	/* stored flags of mark_mutex owner */
- 
-@@ -230,6 +250,7 @@ struct fsnotify_group {
- 						 * full */
- 
- 	struct mem_cgroup *memcg;	/* memcg to charge allocations */
-+	struct user_namespace *user_ns;	/* user ns where group was created */
- 
- 	/* groups can define private fields here or use the void *private */
- 	union {
-@@ -252,6 +273,8 @@ struct fsnotify_group {
- 			int f_flags; /* event_f_flags from fanotify_init() */
- 			struct ucounts *ucounts;
- 			mempool_t error_events_pool;
-+			/* chained on perm_group_list */
-+			struct list_head perm_grp_list;
- 		} fanotify_data;
- #endif /* CONFIG_FANOTIFY */
- 	};
-@@ -264,30 +287,29 @@ struct fsnotify_group {
- static inline void fsnotify_group_lock(struct fsnotify_group *group)
- {
- 	mutex_lock(&group->mark_mutex);
-	if (group->flags & FSNOTIFY_GROUP_NOFS)
-		group->owner_flags = memalloc_nofs_save();
-+	group->owner_flags = memalloc_nofs_save();
- }
- 
- static inline void fsnotify_group_unlock(struct fsnotify_group *group)
- {
-	if (group->flags & FSNOTIFY_GROUP_NOFS)
-		memalloc_nofs_restore(group->owner_flags);
-+	memalloc_nofs_restore(group->owner_flags);
- 	mutex_unlock(&group->mark_mutex);
- }
- 
- static inline void fsnotify_group_assert_locked(struct fsnotify_group *group)
- {
- 	WARN_ON_ONCE(!mutex_is_locked(&group->mark_mutex));
-	if (group->flags & FSNOTIFY_GROUP_NOFS)
-		WARN_ON_ONCE(!(current->flags & PF_MEMALLOC_NOFS));
-+	WARN_ON_ONCE(!(current->flags & PF_MEMALLOC_NOFS));
- }
- 
- /* When calling fsnotify tell it if the data is a path or inode */
- enum fsnotify_data_type {
- 	FSNOTIFY_EVENT_NONE,
-+	FSNOTIFY_EVENT_FILE_RANGE,
- 	FSNOTIFY_EVENT_PATH,
- 	FSNOTIFY_EVENT_INODE,
- 	FSNOTIFY_EVENT_DENTRY,
-+	FSNOTIFY_EVENT_MNT,
- 	FSNOTIFY_EVENT_ERROR,
- };
- 
-@@ -297,6 +319,22 @@ struct fs_error_report {
- 	struct super_block *sb;
- };
- 
-+struct file_range {
-+	const struct path *path;
-+	loff_t pos;
-+	size_t count;
-+};
-+
-+static inline const struct path *file_range_path(const struct file_range *range)
-+{
-+	return range->path;
-+}
-+
-+struct fsnotify_mnt {
-+	const struct mnt_namespace *ns;
-+	u64 mnt_id;
-+};
-+
- static inline struct inode *fsnotify_data_inode(const void *data, int data_type)
- {
- 	switch (data_type) {
-@@ -306,6 +344,8 @@ static inline struct inode *fsnotify_data_inode(const void *data, int data_type)
- 		return d_inode(data);
- 	case FSNOTIFY_EVENT_PATH:
- 		return d_inode(((const struct path *)data)->dentry);
-+	case FSNOTIFY_EVENT_FILE_RANGE:
-+		return d_inode(file_range_path(data)->dentry);
- 	case FSNOTIFY_EVENT_ERROR:
- 		return ((struct fs_error_report *)data)->inode;
- 	default:
-@@ -321,6 +361,8 @@ static inline struct dentry *fsnotify_data_dentry(const void *data, int data_typ
- 		return (struct dentry *)data;
- 	case FSNOTIFY_EVENT_PATH:
- 		return ((const struct path *)data)->dentry;
-+	case FSNOTIFY_EVENT_FILE_RANGE:
-+		return file_range_path(data)->dentry;
- 	default:
- 		return NULL;
- 	}
-@@ -332,6 +374,8 @@ static inline const struct path *fsnotify_data_path(const void *data,
- 	switch (data_type) {
- 	case FSNOTIFY_EVENT_PATH:
- 		return data;
-+	case FSNOTIFY_EVENT_FILE_RANGE:
-+		return file_range_path(data);
- 	default:
- 		return NULL;
- 	}
-@@ -347,6 +391,8 @@ static inline struct super_block *fsnotify_data_sb(const void *data,
- 		return ((struct dentry *)data)->d_sb;
- 	case FSNOTIFY_EVENT_PATH:
- 		return ((const struct path *)data)->dentry->d_sb;
-+	case FSNOTIFY_EVENT_FILE_RANGE:
-+		return file_range_path(data)->dentry->d_sb;
- 	case FSNOTIFY_EVENT_ERROR:
- 		return ((struct fs_error_report *) data)->sb;
- 	default:
-@@ -354,6 +400,24 @@ static inline struct super_block *fsnotify_data_sb(const void *data,
- 	}
- }
- 
-+static inline const struct fsnotify_mnt *fsnotify_data_mnt(const void *data,
-+							   int data_type)
-+{
-+	switch (data_type) {
-+	case FSNOTIFY_EVENT_MNT:
-+		return data;
-+	default:
-+		return NULL;
-+	}
-+}
-+
-+static inline u64 fsnotify_data_mnt_id(const void *data, int data_type)
-+{
-+	const struct fsnotify_mnt *mnt_data = fsnotify_data_mnt(data, data_type);
-+
-+	return mnt_data ? mnt_data->mnt_id : 0;
-+}
-+
- static inline struct fs_error_report *fsnotify_data_error_report(
- 							const void *data,
- 							int data_type)
-@@ -366,6 +430,18 @@ static inline struct fs_error_report *fsnotify_data_error_report(
- 	}
- }
- 
-+static inline const struct file_range *fsnotify_data_file_range(
-+							const void *data,
-+							int data_type)
-+{
-+	switch (data_type) {
-+	case FSNOTIFY_EVENT_FILE_RANGE:
-+		return (struct file_range *)data;
-+	default:
-+		return NULL;
-+	}
-+}
-+
- /*
-  * Index to merged marks iterator array that correlates to a type of watch.
-  * The type of watched object can be deduced from the iterator type, but not
-@@ -379,6 +455,7 @@ enum fsnotify_iter_type {
- 	FSNOTIFY_ITER_TYPE_SB,
- 	FSNOTIFY_ITER_TYPE_PARENT,
- 	FSNOTIFY_ITER_TYPE_INODE2,
-+	FSNOTIFY_ITER_TYPE_MNTNS,
- 	FSNOTIFY_ITER_TYPE_COUNT
- };
- 
-@@ -388,6 +465,7 @@ enum fsnotify_obj_type {
- 	FSNOTIFY_OBJ_TYPE_INODE,
- 	FSNOTIFY_OBJ_TYPE_VFSMOUNT,
- 	FSNOTIFY_OBJ_TYPE_SB,
-+	FSNOTIFY_OBJ_TYPE_MNTNS,
- 	FSNOTIFY_OBJ_TYPE_COUNT,
- 	FSNOTIFY_OBJ_TYPE_DETACHED = FSNOTIFY_OBJ_TYPE_COUNT
- };
-@@ -457,13 +535,6 @@ FSNOTIFY_ITER_FUNCS(sb, SB)
- 	     type++)
- 
- /*
- * fsnotify_connp_t is what we embed in objects which connector can be attached
- * to. fsnotify_connp_t * is how we refer from connector back to object.
- */
-struct fsnotify_mark_connector;
-typedef struct fsnotify_mark_connector __rcu *fsnotify_connp_t;
-
-/*
-  * Inode/vfsmount/sb point to this structure which tracks all marks attached to
-  * the inode/vfsmount/sb. The reference to inode/vfsmount/sb is held by this
-  * structure. We destroy this structure when there are no more marks attached
-@@ -471,14 +542,14 @@ typedef struct fsnotify_mark_connector __rcu *fsnotify_connp_t;
-  */
- struct fsnotify_mark_connector {
- 	spinlock_t lock;
-	unsigned short type;	/* Type of object [lock] */
-#define FSNOTIFY_CONN_FLAG_HAS_FSID	0x01
-+	unsigned char type;	/* Type of object [lock] */
-+	unsigned char prio;	/* Highest priority group */
-+#define FSNOTIFY_CONN_FLAG_IS_WATCHED	0x01
- #define FSNOTIFY_CONN_FLAG_HAS_IREF	0x02
- 	unsigned short flags;	/* flags [lock] */
-	__kernel_fsid_t fsid;	/* fsid of filesystem containing object */
- 	union {
- 		/* Object pointer [lock] */
-		fsnotify_connp_t *obj;
-+		void *obj;
- 		/* Used listing heads to free after srcu period expires */
- 		struct fsnotify_mark_connector *destroy_next;
- 	};
-@@ -486,6 +557,37 @@ struct fsnotify_mark_connector {
- };
- 
- /*
-+ * Container for per-sb fsnotify state (sb marks and more).
-+ * Attached lazily on first marked object on the sb and freed when killing sb.
-+ */
-+struct fsnotify_sb_info {
-+	struct fsnotify_mark_connector __rcu *sb_marks;
-+	/*
-+	 * Number of inode/mount/sb objects that are being watched in this sb.
-+	 * Note that inodes objects are currently double-accounted.
-+	 *
-+	 * The value in watched_objects[prio] is the number of objects that are
-+	 * watched by groups of priority >= prio, so watched_objects[0] is the
-+	 * total number of watched objects in this sb.
-+	 */
-+	atomic_long_t watched_objects[__FSNOTIFY_PRIO_NUM];
-+};
-+
-+static inline struct fsnotify_sb_info *fsnotify_sb_info(struct super_block *sb)
-+{
-+#ifdef CONFIG_FSNOTIFY
-+	return READ_ONCE(sb->s_fsnotify_info);
-+#else
-+	return NULL;
-+#endif
-+}
-+
-+static inline atomic_long_t *fsnotify_sb_watched_objects(struct super_block *sb)
-+{
-+	return &fsnotify_sb_info(sb)->watched_objects[0];
-+}
-+
-+/*
-  * A mark is simply an object attached to an in core inode which allows an
-  * fsnotify listener to indicate they are either no longer interested in events
-  * of a type matching mask or only interested in those events.
-@@ -530,6 +632,8 @@ struct fsnotify_mark {
- #define FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY	0x0100
- #define FSNOTIFY_MARK_FLAG_NO_IREF		0x0200
- #define FSNOTIFY_MARK_FLAG_HAS_IGNORE_FLAGS	0x0400
-+#define FSNOTIFY_MARK_FLAG_HAS_FSID		0x0800
-+#define FSNOTIFY_MARK_FLAG_WEAK_FSID		0x1000
- 	unsigned int flags;		/* flags [mark->lock] */
- };
- 
-@@ -546,7 +650,10 @@ extern int __fsnotify_parent(struct dentry *dentry, __u32 mask, const void *data
- extern void __fsnotify_inode_delete(struct inode *inode);
- extern void __fsnotify_vfsmount_delete(struct vfsmount *mnt);
- extern void fsnotify_sb_delete(struct super_block *sb);
-+extern void __fsnotify_mntns_delete(struct mnt_namespace *mntns);
-+extern void fsnotify_sb_free(struct super_block *sb);
- extern u32 fsnotify_get_cookie(void);
-+extern void fsnotify_mnt(__u32 mask, struct mnt_namespace *ns, struct vfsmount *mnt);
- 
- static inline __u32 fsnotify_parent_needed_mask(__u32 mask)
- {
-@@ -563,12 +670,14 @@ static inline __u32 fsnotify_parent_needed_mask(__u32 mask)
- 
- static inline int fsnotify_inode_watches_children(struct inode *inode)
- {
-+	__u32 parent_mask = READ_ONCE(inode->i_fsnotify_mask);
-+
- 	/* FS_EVENT_ON_CHILD is set if the inode may care */
-	if (!(inode->i_fsnotify_mask & FS_EVENT_ON_CHILD))
-+	if (!(parent_mask & FS_EVENT_ON_CHILD))
- 		return 0;
- 	/* this inode might care about child events, does it care about the
- 	 * specific set of events that can happen on a child? */
-	return inode->i_fsnotify_mask & FS_EVENTS_POSS_ON_CHILD;
-+	return parent_mask & FS_EVENTS_POSS_ON_CHILD;
- }
- 
- /*
-@@ -582,7 +691,7 @@ static inline void fsnotify_update_flags(struct dentry *dentry)
- 	/*
- 	 * Serialisation of setting PARENT_WATCHED on the dentries is provided
- 	 * by d_lock. If inotify_inode_watched changes after we have taken
-	 * d_lock, the following __fsnotify_update_child_dentry_flags call will
-+	 * d_lock, the following fsnotify_set_children_dentry_flags call will
- 	 * find our entry, so it will spin until we complete here, and update
- 	 * us with the new state.
- 	 */
-@@ -758,35 +867,35 @@ extern void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn);
- extern void fsnotify_init_mark(struct fsnotify_mark *mark,
- 			       struct fsnotify_group *group);
- /* Find mark belonging to given group in the list of marks */
-extern struct fsnotify_mark *fsnotify_find_mark(fsnotify_connp_t *connp,
-						struct fsnotify_group *group);
-/* Get cached fsid of filesystem containing object */
-extern int fsnotify_get_conn_fsid(const struct fsnotify_mark_connector *conn,
-				  __kernel_fsid_t *fsid);
-+struct fsnotify_mark *fsnotify_find_mark(void *obj, unsigned int obj_type,
-+					 struct fsnotify_group *group);
- /* attach the mark to the object */
-extern int fsnotify_add_mark(struct fsnotify_mark *mark,
-			     fsnotify_connp_t *connp, unsigned int obj_type,
-			     int add_flags, __kernel_fsid_t *fsid);
-extern int fsnotify_add_mark_locked(struct fsnotify_mark *mark,
-				    fsnotify_connp_t *connp,
-				    unsigned int obj_type, int add_flags,
-				    __kernel_fsid_t *fsid);
-+int fsnotify_add_mark(struct fsnotify_mark *mark, void *obj,
-+		      unsigned int obj_type, int add_flags);
-+int fsnotify_add_mark_locked(struct fsnotify_mark *mark, void *obj,
-+			     unsigned int obj_type, int add_flags);
- 
- /* attach the mark to the inode */
- static inline int fsnotify_add_inode_mark(struct fsnotify_mark *mark,
- 					  struct inode *inode,
- 					  int add_flags)
- {
-	return fsnotify_add_mark(mark, &inode->i_fsnotify_marks,
-				 FSNOTIFY_OBJ_TYPE_INODE, add_flags, NULL);
-+	return fsnotify_add_mark(mark, inode, FSNOTIFY_OBJ_TYPE_INODE,
-+				 add_flags);
- }
- static inline int fsnotify_add_inode_mark_locked(struct fsnotify_mark *mark,
- 						 struct inode *inode,
- 						 int add_flags)
- {
-	return fsnotify_add_mark_locked(mark, &inode->i_fsnotify_marks,
-					FSNOTIFY_OBJ_TYPE_INODE, add_flags,
-					NULL);
-+	return fsnotify_add_mark_locked(mark, inode, FSNOTIFY_OBJ_TYPE_INODE,
-+					add_flags);
-+}
-+
-+static inline struct fsnotify_mark *fsnotify_find_inode_mark(
-+						struct inode *inode,
-+						struct fsnotify_group *group)
-+{
-+	return fsnotify_find_mark(inode, FSNOTIFY_OBJ_TYPE_INODE, group);
- }
- 
- /* given a group and a mark, flag mark to be freed when all references are dropped */
-@@ -801,21 +910,6 @@ extern void fsnotify_wait_marks_destroyed(void);
- /* Clear all of the marks of a group attached to a given object type */
- extern void fsnotify_clear_marks_by_group(struct fsnotify_group *group,
- 					  unsigned int obj_type);
-/* run all the marks in a group, and clear all of the vfsmount marks */
-static inline void fsnotify_clear_vfsmount_marks_by_group(struct fsnotify_group *group)
-{
-	fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_VFSMOUNT);
-}
-/* run all the marks in a group, and clear all of the inode marks */
-static inline void fsnotify_clear_inode_marks_by_group(struct fsnotify_group *group)
-{
-	fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_INODE);
-}
-/* run all the marks in a group, and clear all of the sn marks */
-static inline void fsnotify_clear_sb_marks_by_group(struct fsnotify_group *group)
-{
-	fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_SB);
-}
- extern void fsnotify_get_mark(struct fsnotify_mark *mark);
- extern void fsnotify_put_mark(struct fsnotify_mark *mark);
- extern void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info);
-@@ -825,9 +919,17 @@ static inline void fsnotify_init_event(struct fsnotify_event *event)
- {
- 	INIT_LIST_HEAD(&event->list);
- }
-+int fsnotify_pre_content(const struct path *path, const loff_t *ppos,
-+			 size_t count);
- 
- #else
- 
-+static inline int fsnotify_pre_content(const struct path *path,
-+				       const loff_t *ppos, size_t count)
-+{
-+	return 0;
-+}
-+
- static inline int fsnotify(__u32 mask, const void *data, int data_type,
- 			   struct inode *dir, const struct qstr *name,
- 			   struct inode *inode, u32 cookie)
-@@ -850,6 +952,12 @@ static inline void __fsnotify_vfsmount_delete(struct vfsmount *mnt)
- static inline void fsnotify_sb_delete(struct super_block *sb)
- {}
- 
-+static inline void __fsnotify_mntns_delete(struct mnt_namespace *mntns)
-+{}
-+
-+static inline void fsnotify_sb_free(struct super_block *sb)
-+{}
-+
- static inline void fsnotify_update_flags(struct dentry *dentry)
- {}
- 
-@@ -861,6 +969,9 @@ static inline u32 fsnotify_get_cookie(void)
- static inline void fsnotify_unmount_inodes(struct super_block *sb)
- {}
- 
-+static inline void fsnotify_mnt(__u32 mask, struct mnt_namespace *ns, struct vfsmount *mnt)
-+{}
-+
- #endif	/* CONFIG_FSNOTIFY */
- 
- #endif	/* __KERNEL __ */
-diff --git a/include/linux/fsverity.h b/include/linux/fsverity.h
-index 7af030fa3c36..5bc7280425a7 100644
--- a/include/linux/fsverity.h
-+++ b/include/linux/fsverity.h
-@@ -12,6 +12,7 @@
- #define _LINUX_FSVERITY_H
- 
- #include <linux/fs.h>
-+#include <linux/mm.h>
- #include <crypto/hash_info.h>
- #include <crypto/sha2.h>
- #include <uapi/linux/fsverity.h>
-@@ -22,8 +23,19 @@
-  */
- #define FS_VERITY_MAX_DIGEST_SIZE	SHA512_DIGEST_SIZE
- 
-+/* Arbitrary limit to bound the kmalloc() size.  Can be changed. */
-+#define FS_VERITY_MAX_DESCRIPTOR_SIZE	16384
-+
-+struct fsverity_info;
-+
- /* Verity operations for filesystems */
- struct fsverity_operations {
-+	/**
-+	 * The offset of the pointer to struct fsverity_info in the
-+	 * filesystem-specific part of the inode, relative to the beginning of
-+	 * the common part of the inode (the 'struct inode').
-+	 */
-+	ptrdiff_t inode_info_offs;
- 
- 	/**
- 	 * Begin enabling verity on the given file.
-@@ -90,8 +102,7 @@ struct fsverity_operations {
- 	 *		  isn't already cached.  Implementations may ignore this
- 	 *		  argument; it's only a performance optimization.
- 	 *
-	 * This can be called at any time on an open verity file, as well as
-	 * between ->begin_enable_verity() and ->end_enable_verity().  It may be
-+	 * This can be called at any time on an open verity file.  It may be
- 	 * called by multiple processes concurrently, even with the same page.
- 	 *
- 	 * Note that this must retrieve a *page*, not necessarily a *block*.
-@@ -106,9 +117,9 @@ struct fsverity_operations {
- 	 * Write a Merkle tree block to the given inode.
- 	 *
- 	 * @inode: the inode for which the Merkle tree is being built
-	 * @buf: block to write
-	 * @index: 0-based index of the block within the Merkle tree
-	 * @log_blocksize: log base 2 of the Merkle tree block size
-+	 * @buf: the Merkle tree block to write
-+	 * @pos: the position of the block in the Merkle tree (in bytes)
-+	 * @size: the Merkle tree block size (in bytes)
- 	 *
- 	 * This is only called between ->begin_enable_verity() and
- 	 * ->end_enable_verity().
-@@ -116,20 +127,42 @@ struct fsverity_operations {
- 	 * Return: 0 on success, -errno on failure
- 	 */
- 	int (*write_merkle_tree_block)(struct inode *inode, const void *buf,
-				       u64 index, int log_blocksize);
-+				       u64 pos, unsigned int size);
- };
- 
- #ifdef CONFIG_FS_VERITY
- 
-+/*
-+ * Returns the address of the verity info pointer within the filesystem-specific
-+ * part of the inode.  (To save memory on filesystems that don't support
-+ * fsverity, a field in 'struct inode' itself is no longer used.)
-+ */
-+static inline struct fsverity_info **
-+fsverity_info_addr(const struct inode *inode)
-+{
-+	VFS_WARN_ON_ONCE(inode->i_sb->s_vop->inode_info_offs == 0);
-+	return (void *)inode + inode->i_sb->s_vop->inode_info_offs;
-+}
-+
- static inline struct fsverity_info *fsverity_get_info(const struct inode *inode)
- {
- 	/*
-	 * Pairs with the cmpxchg_release() in fsverity_set_info().
-	 * I.e., another task may publish ->i_verity_info concurrently,
-	 * executing a RELEASE barrier.  We need to use smp_load_acquire() here
-	 * to safely ACQUIRE the memory the other task published.
-+	 * Since this function can be called on inodes belonging to filesystems
-+	 * that don't support fsverity at all, and fsverity_info_addr() doesn't
-+	 * work on such filesystems, we have to start with an IS_VERITY() check.
-+	 * Checking IS_VERITY() here is also useful to minimize the overhead of
-+	 * fsverity_active() on non-verity files.
- 	 */
-	return smp_load_acquire(&inode->i_verity_info);
-+	if (!IS_VERITY(inode))
-+		return NULL;
-+
-+	/*
-+	 * Pairs with the cmpxchg_release() in fsverity_set_info().  I.e.,
-+	 * another task may publish the inode's verity info concurrently,
-+	 * executing a RELEASE barrier.  Use smp_load_acquire() here to safely
-+	 * ACQUIRE the memory the other task published.
-+	 */
-+	return smp_load_acquire(fsverity_info_addr(inode));
- }
- 
- /* enable.c */
-@@ -140,14 +173,33 @@ int fsverity_ioctl_enable(struct file *filp, const void __user *arg);
- 
- int fsverity_ioctl_measure(struct file *filp, void __user *arg);
- int fsverity_get_digest(struct inode *inode,
-			u8 digest[FS_VERITY_MAX_DIGEST_SIZE],
-			enum hash_algo *alg);
-+			u8 raw_digest[FS_VERITY_MAX_DIGEST_SIZE],
-+			u8 *alg, enum hash_algo *halg);
- 
- /* open.c */
- 
-int fsverity_file_open(struct inode *inode, struct file *filp);
-int fsverity_prepare_setattr(struct dentry *dentry, struct iattr *attr);
-void fsverity_cleanup_inode(struct inode *inode);
-+int __fsverity_file_open(struct inode *inode, struct file *filp);
-+int __fsverity_prepare_setattr(struct dentry *dentry, struct iattr *attr);
-+void __fsverity_cleanup_inode(struct inode *inode);
-+
-+/**
-+ * fsverity_cleanup_inode() - free the inode's verity info, if present
-+ * @inode: an inode being evicted
-+ *
-+ * Filesystems must call this on inode eviction to free the inode's verity info.
-+ */
-+static inline void fsverity_cleanup_inode(struct inode *inode)
-+{
-+	/*
-+	 * Only IS_VERITY() inodes can have verity info, so start by checking
-+	 * for IS_VERITY() (which is faster than retrieving the pointer to the
-+	 * verity info).  This minimizes overhead for non-verity inodes.
-+	 */
-+	if (IS_VERITY(inode))
-+		__fsverity_cleanup_inode(inode);
-+	else
-+		VFS_WARN_ON_ONCE(*fsverity_info_addr(inode) != NULL);
-+}
- 
- /* read_metadata.c */
- 
-@@ -155,7 +207,7 @@ int fsverity_ioctl_read_metadata(struct file *filp, const void __user *uarg);
- 
- /* verify.c */
- 
-bool fsverity_verify_page(struct page *page);
-+bool fsverity_verify_blocks(struct folio *folio, size_t len, size_t offset);
- void fsverity_verify_bio(struct bio *bio);
- void fsverity_enqueue_verify_work(struct work_struct *work);
- 
-@@ -182,23 +234,27 @@ static inline int fsverity_ioctl_measure(struct file *filp, void __user *arg)
- }
- 
- static inline int fsverity_get_digest(struct inode *inode,
-				      u8 digest[FS_VERITY_MAX_DIGEST_SIZE],
-				      enum hash_algo *alg)
-+				      u8 raw_digest[FS_VERITY_MAX_DIGEST_SIZE],
-+				      u8 *alg, enum hash_algo *halg)
- {
-	return -EOPNOTSUPP;
-+	/*
-+	 * fsverity is not enabled in the kernel configuration, so always report
-+	 * that the file doesn't have fsverity enabled (digest size 0).
-+	 */
-+	return 0;
- }
- 
- /* open.c */
- 
-static inline int fsverity_file_open(struct inode *inode, struct file *filp)
-+static inline int __fsverity_file_open(struct inode *inode, struct file *filp)
- {
-	return IS_VERITY(inode) ? -EOPNOTSUPP : 0;
-+	return -EOPNOTSUPP;
- }
- 
-static inline int fsverity_prepare_setattr(struct dentry *dentry,
-					   struct iattr *attr)
-+static inline int __fsverity_prepare_setattr(struct dentry *dentry,
-+					     struct iattr *attr)
- {
-	return IS_VERITY(d_inode(dentry)) ? -EOPNOTSUPP : 0;
-+	return -EOPNOTSUPP;
- }
- 
- static inline void fsverity_cleanup_inode(struct inode *inode)
-@@ -215,34 +271,45 @@ static inline int fsverity_ioctl_read_metadata(struct file *filp,
- 
- /* verify.c */
- 
-static inline bool fsverity_verify_page(struct page *page)
-+static inline bool fsverity_verify_blocks(struct folio *folio, size_t len,
-+					  size_t offset)
- {
-	WARN_ON(1);
-+	WARN_ON_ONCE(1);
- 	return false;
- }
- 
- static inline void fsverity_verify_bio(struct bio *bio)
- {
-	WARN_ON(1);
-+	WARN_ON_ONCE(1);
- }
- 
- static inline void fsverity_enqueue_verify_work(struct work_struct *work)
- {
-	WARN_ON(1);
-+	WARN_ON_ONCE(1);
- }
- 
- #endif	/* !CONFIG_FS_VERITY */
- 
-+static inline bool fsverity_verify_folio(struct folio *folio)
-+{
-+	return fsverity_verify_blocks(folio, folio_size(folio), 0);
-+}
-+
-+static inline bool fsverity_verify_page(struct page *page)
-+{
-+	return fsverity_verify_blocks(page_folio(page), PAGE_SIZE, 0);
-+}
-+
- /**
-  * fsverity_active() - do reads from the inode need to go through fs-verity?
-  * @inode: inode to check
-  *
- * This checks whether ->i_verity_info has been set.
-+ * This checks whether the inode's verity info has been set.
-  *
-  * Filesystems call this from ->readahead() to check whether the pages need to
-  * be verified or not.  Don't use IS_VERITY() for this purpose; it's subject to
-  * a race condition where the file is being read concurrently with
- * FS_IOC_ENABLE_VERITY completing.  (S_VERITY is set before ->i_verity_info.)
-+ * FS_IOC_ENABLE_VERITY completing.  (S_VERITY is set before the verity info.)
-  *
-  * Return: true if reads need to go through fs-verity, otherwise false
-  */
-@@ -251,4 +318,42 @@ static inline bool fsverity_active(const struct inode *inode)
- 	return fsverity_get_info(inode) != NULL;
- }
- 
-+/**
-+ * fsverity_file_open() - prepare to open a verity file
-+ * @inode: the inode being opened
-+ * @filp: the struct file being set up
-+ *
-+ * When opening a verity file, deny the open if it is for writing.  Otherwise,
-+ * set up the inode's verity info if not already done.
-+ *
-+ * When combined with fscrypt, this must be called after fscrypt_file_open().
-+ * Otherwise, we won't have the key set up to decrypt the verity metadata.
-+ *
-+ * Return: 0 on success, -errno on failure
-+ */
-+static inline int fsverity_file_open(struct inode *inode, struct file *filp)
-+{
-+	if (IS_VERITY(inode))
-+		return __fsverity_file_open(inode, filp);
-+	return 0;
-+}
-+
-+/**
-+ * fsverity_prepare_setattr() - prepare to change a verity inode's attributes
-+ * @dentry: dentry through which the inode is being changed
-+ * @attr: attributes to change
-+ *
-+ * Verity files are immutable, so deny truncates.  This isn't covered by the
-+ * open-time check because sys_truncate() takes a path, not a file descriptor.
-+ *
-+ * Return: 0 on success, -errno on failure
-+ */
-+static inline int fsverity_prepare_setattr(struct dentry *dentry,
-+					   struct iattr *attr)
-+{
-+	if (IS_VERITY(d_inode(dentry)))
-+		return __fsverity_prepare_setattr(dentry, attr);
-+	return 0;
-+}
-+
- #endif	/* _LINUX_FSVERITY_H */
-diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
-index 0b61371e287b..770f0dc993cc 100644
--- a/include/linux/ftrace.h
-+++ b/include/linux/ftrace.h
-@@ -37,9 +37,19 @@ extern void ftrace_boot_snapshot(void);
- static inline void ftrace_boot_snapshot(void) { }
- #endif
- 
-#ifdef CONFIG_FUNCTION_TRACER
- struct ftrace_ops;
- struct ftrace_regs;
-+struct dyn_ftrace;
-+
-+char *arch_ftrace_match_adjust(char *str, const char *search);
-+
-+#ifdef CONFIG_HAVE_FUNCTION_GRAPH_FREGS
-+unsigned long ftrace_return_to_handler(struct ftrace_regs *fregs);
-+#else
-+unsigned long ftrace_return_to_handler(unsigned long frame_pointer);
-+#endif
-+
-+#ifdef CONFIG_FUNCTION_TRACER
- /*
-  * If the arch's mcount caller does not support all of ftrace's
-  * features, then it must call an indirect function that
-@@ -56,6 +66,9 @@ void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip);
- void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
- 			       struct ftrace_ops *op, struct ftrace_regs *fregs);
- #endif
-+extern const struct ftrace_ops ftrace_nop_ops;
-+extern const struct ftrace_ops ftrace_list_ops;
-+struct ftrace_ops *ftrace_find_unique_ops(struct dyn_ftrace *rec);
- #endif /* CONFIG_FUNCTION_TRACER */
- 
- /* Main tracing buffer and events set up */
-@@ -69,19 +82,18 @@ static inline void early_trace_init(void) { }
- 
- struct module;
- struct ftrace_hash;
-struct ftrace_direct_func;
- 
- #if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_MODULES) && \
- 	defined(CONFIG_DYNAMIC_FTRACE)
-const char *
-+int
- ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
- 		   unsigned long *off, char **modname, char *sym);
- #else
-static inline const char *
-+static inline int
- ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
- 		   unsigned long *off, char **modname, char *sym)
- {
-	return NULL;
-+	return 0;
- }
- #endif
- 
-@@ -100,24 +112,76 @@ static inline int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *val
- 
- #ifdef CONFIG_FUNCTION_TRACER
- 
-extern int ftrace_enabled;
-+#include <linux/ftrace_regs.h>
- 
-#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
-+extern int ftrace_enabled;
- 
-+/**
-+ * ftrace_regs - ftrace partial/optimal register set
-+ *
-+ * ftrace_regs represents a group of registers which is used at the
-+ * function entry and exit. There are three types of registers.
-+ *
-+ * - Registers for passing the parameters to callee, including the stack
-+ *   pointer. (e.g. rcx, rdx, rdi, rsi, r8, r9 and rsp on x86_64)
-+ * - Registers for passing the return values to caller.
-+ *   (e.g. rax and rdx on x86_64)
-+ * - Registers for hooking the function call and return including the
-+ *   frame pointer (the frame pointer is architecture/config dependent)
-+ *   (e.g. rip, rbp and rsp for x86_64)
-+ *
-+ * Also, architecture dependent fields can be used for internal process.
-+ * (e.g. orig_ax on x86_64)
-+ *
-+ * Basically, ftrace_regs stores the registers related to the context.
-+ * On function entry, registers for function parameters and hooking the
-+ * function call are stored, and on function exit, registers for function
-+ * return value and frame pointers are stored.
-+ *
-+ * And also, it dpends on the context that which registers are restored
-+ * from the ftrace_regs.
-+ * On the function entry, those registers will be restored except for
-+ * the stack pointer, so that user can change the function parameters
-+ * and instruction pointer (e.g. live patching.)
-+ * On the function exit, only registers which is used for return values
-+ * are restored.
-+ *
-+ * NOTE: user *must not* access regs directly, only do it via APIs, because
-+ * the member can be changed according to the architecture.
-+ * This is why the structure is empty here, so that nothing accesses
-+ * the ftrace_regs directly.
-+ */
- struct ftrace_regs {
-	struct pt_regs		regs;
-+	/* Nothing to see here, use the accessor functions! */
- };
-#define arch_ftrace_get_regs(fregs) (&(fregs)->regs)
-+
-+#define ftrace_regs_size()	sizeof(struct __arch_ftrace_regs)
-+
-+#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
-+/*
-+ * Architectures that define HAVE_DYNAMIC_FTRACE_WITH_ARGS must define their own
-+ * arch_ftrace_get_regs() where it only returns pt_regs *if* it is fully
-+ * populated. It should return NULL otherwise.
-+ */
-+static inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs *fregs)
-+{
-+	return &arch_ftrace_regs(fregs)->regs;
-+}
- 
- /*
- * ftrace_instruction_pointer_set() is to be defined by the architecture
- * if to allow setting of the instruction pointer from the ftrace_regs
- * when HAVE_DYNAMIC_FTRACE_WITH_ARGS is set and it supports
- * live kernel patching.
-+ * ftrace_regs_set_instruction_pointer() is to be defined by the architecture
-+ * if to allow setting of the instruction pointer from the ftrace_regs when
-+ * HAVE_DYNAMIC_FTRACE_WITH_ARGS is set and it supports live kernel patching.
-  */
-#define ftrace_instruction_pointer_set(fregs, ip) do { } while (0)
-+#define ftrace_regs_set_instruction_pointer(fregs, ip) do { } while (0)
- #endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */
- 
-+#ifdef CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS
-+
-+static_assert(sizeof(struct pt_regs) == ftrace_regs_size());
-+
-+#endif /* CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS */
-+
- static __always_inline struct pt_regs *ftrace_get_regs(struct ftrace_regs *fregs)
- {
- 	if (!fregs)
-@@ -126,6 +190,91 @@ static __always_inline struct pt_regs *ftrace_get_regs(struct ftrace_regs *fregs
- 	return arch_ftrace_get_regs(fregs);
- }
- 
-+#if !defined(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS) || \
-+	defined(CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS)
-+
-+#ifndef arch_ftrace_partial_regs
-+#define arch_ftrace_partial_regs(regs) do {} while (0)
-+#endif
-+
-+static __always_inline struct pt_regs *
-+ftrace_partial_regs(struct ftrace_regs *fregs, struct pt_regs *regs)
-+{
-+	/*
-+	 * If CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS=y, ftrace_regs memory
-+	 * layout is including pt_regs. So always returns that address.
-+	 * Since arch_ftrace_get_regs() will check some members and may return
-+	 * NULL, we can not use it.
-+	 */
-+	regs = &arch_ftrace_regs(fregs)->regs;
-+
-+	/* Allow arch specific updates to regs. */
-+	arch_ftrace_partial_regs(regs);
-+	return regs;
-+}
-+
-+#endif /* !CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS || CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS */
-+
-+#ifdef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
-+
-+/*
-+ * Please define arch dependent pt_regs which compatible to the
-+ * perf_arch_fetch_caller_regs() but based on ftrace_regs.
-+ * This requires
-+ *   - user_mode(_regs) returns false (always kernel mode).
-+ *   - able to use the _regs for stack trace.
-+ */
-+#ifndef arch_ftrace_fill_perf_regs
-+/* As same as perf_arch_fetch_caller_regs(), do nothing by default */
-+#define arch_ftrace_fill_perf_regs(fregs, _regs) do {} while (0)
-+#endif
-+
-+static __always_inline struct pt_regs *
-+ftrace_fill_perf_regs(struct ftrace_regs *fregs, struct pt_regs *regs)
-+{
-+	arch_ftrace_fill_perf_regs(fregs, regs);
-+	return regs;
-+}
-+
-+#else /* !CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */
-+
-+static __always_inline struct pt_regs *
-+ftrace_fill_perf_regs(struct ftrace_regs *fregs, struct pt_regs *regs)
-+{
-+	return &arch_ftrace_regs(fregs)->regs;
-+}
-+
-+#endif
-+
-+/*
-+ * When true, the ftrace_regs_{get,set}_*() functions may be used on fregs.
-+ * Note: this can be true even when ftrace_get_regs() cannot provide a pt_regs.
-+ */
-+static __always_inline bool ftrace_regs_has_args(struct ftrace_regs *fregs)
-+{
-+	if (IS_ENABLED(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS))
-+		return true;
-+
-+	return ftrace_get_regs(fregs) != NULL;
-+}
-+
-+#ifdef CONFIG_HAVE_REGS_AND_STACK_ACCESS_API
-+static __always_inline unsigned long
-+ftrace_regs_get_kernel_stack_nth(struct ftrace_regs *fregs, unsigned int nth)
-+{
-+	unsigned long *stackp;
-+
-+	stackp = (unsigned long *)ftrace_regs_get_stack_pointer(fregs);
-+	if (((unsigned long)(stackp + nth) & ~(THREAD_SIZE - 1)) ==
-+	    ((unsigned long)stackp & ~(THREAD_SIZE - 1)))
-+		return *(stackp + nth);
-+
-+	return 0;
-+}
-+#else /* !CONFIG_HAVE_REGS_AND_STACK_ACCESS_API */
-+#define ftrace_regs_get_kernel_stack_nth(fregs, nth)	(0L)
-+#endif /* CONFIG_HAVE_REGS_AND_STACK_ACCESS_API */
-+
- typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip,
- 			      struct ftrace_ops *op, struct ftrace_regs *fregs);
- 
-@@ -186,6 +335,8 @@ ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops);
-  *             ftrace_enabled.
-  * DIRECT - Used by the direct ftrace_ops helper for direct functions
-  *            (internal ftrace only, should not be used by others)
-+ * SUBOP  - Is controlled by another op in field managed.
-+ * GRAPH  - Is a component of the fgraph_ops structure
-  */
- enum {
- 	FTRACE_OPS_FL_ENABLED			= BIT(0),
-@@ -206,8 +357,17 @@ enum {
- 	FTRACE_OPS_FL_TRACE_ARRAY		= BIT(15),
- 	FTRACE_OPS_FL_PERMANENT                 = BIT(16),
- 	FTRACE_OPS_FL_DIRECT			= BIT(17),
-+	FTRACE_OPS_FL_SUBOP			= BIT(18),
-+	FTRACE_OPS_FL_GRAPH			= BIT(19),
-+	FTRACE_OPS_FL_JMP			= BIT(20),
- };
- 
-+#ifndef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
-+#define FTRACE_OPS_FL_SAVE_ARGS                        FTRACE_OPS_FL_SAVE_REGS
-+#else
-+#define FTRACE_OPS_FL_SAVE_ARGS                        0
-+#endif
-+
- /*
-  * FTRACE_OPS_CMD_* commands allow the ftrace core logic to request changes
-  * to a ftrace_ops. Note, the requests may fail.
-@@ -287,7 +447,12 @@ struct ftrace_ops {
- 	unsigned long			trampoline;
- 	unsigned long			trampoline_size;
- 	struct list_head		list;
-+	struct list_head		subop_list;
- 	ftrace_ops_func_t		ops_func;
-+	struct ftrace_ops		*managed;
-+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
-+	unsigned long			direct_call;
-+#endif
- #endif
- };
- 
-@@ -362,74 +527,40 @@ struct ftrace_func_entry {
- 	unsigned long direct; /* for direct lookup only */
- };
- 
-struct dyn_ftrace;
-
- #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
-extern int ftrace_direct_func_count;
-int register_ftrace_direct(unsigned long ip, unsigned long addr);
-int unregister_ftrace_direct(unsigned long ip, unsigned long addr);
-int modify_ftrace_direct(unsigned long ip, unsigned long old_addr, unsigned long new_addr);
-struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr);
-int ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
-				struct dyn_ftrace *rec,
-				unsigned long old_addr,
-				unsigned long new_addr);
- unsigned long ftrace_find_rec_direct(unsigned long ip);
-int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr);
-int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr);
-int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr);
-int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr);
-+int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr);
-+int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
-+			     bool free_filters);
-+int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr);
-+int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr);
-+
-+void ftrace_stub_direct_tramp(void);
- 
- #else
- struct ftrace_ops;
-# define ftrace_direct_func_count 0
-static inline int register_ftrace_direct(unsigned long ip, unsigned long addr)
-{
-	return -ENOTSUPP;
-}
-static inline int unregister_ftrace_direct(unsigned long ip, unsigned long addr)
-{
-	return -ENOTSUPP;
-}
-static inline int modify_ftrace_direct(unsigned long ip,
-				       unsigned long old_addr, unsigned long new_addr)
-{
-	return -ENOTSUPP;
-}
-static inline struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr)
-{
-	return NULL;
-}
-static inline int ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
-					      struct dyn_ftrace *rec,
-					      unsigned long old_addr,
-					      unsigned long new_addr)
-{
-	return -ENODEV;
-}
- static inline unsigned long ftrace_find_rec_direct(unsigned long ip)
- {
- 	return 0;
- }
-static inline int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
-+static inline int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
- {
- 	return -ENODEV;
- }
-static inline int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
-+static inline int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
-+					   bool free_filters)
- {
- 	return -ENODEV;
- }
-static inline int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
-+static inline int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
- {
- 	return -ENODEV;
- }
-static inline int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr)
-+static inline int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr)
- {
- 	return -ENODEV;
- }
-#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
- 
-#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
- /*
-  * This must be implemented by the architecture.
-  * It is the way the ftrace direct_ops helper, when called
-@@ -443,15 +574,45 @@ static inline int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsi
-  * the return from the trampoline jump to the direct caller
-  * instead of going back to the function it just traced.
-  */
-static inline void arch_ftrace_set_direct_caller(struct pt_regs *regs,
-+static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs,
- 						 unsigned long addr) { }
-#endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
-+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
- 
-#ifdef CONFIG_STACK_TRACER
-+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP
-+static inline bool ftrace_is_jmp(unsigned long addr)
-+{
-+	return addr & 1;
-+}
- 
-extern int stack_tracer_enabled;
-+static inline unsigned long ftrace_jmp_set(unsigned long addr)
-+{
-+	return addr | 1UL;
-+}
-+
-+static inline unsigned long ftrace_jmp_get(unsigned long addr)
-+{
-+	return addr & ~1UL;
-+}
-+#else
-+static inline bool ftrace_is_jmp(unsigned long addr)
-+{
-+	return false;
-+}
-+
-+static inline unsigned long ftrace_jmp_set(unsigned long addr)
-+{
-+	return addr;
-+}
-+
-+static inline unsigned long ftrace_jmp_get(unsigned long addr)
-+{
-+	return addr;
-+}
-+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_JMP */
-+
-+#ifdef CONFIG_STACK_TRACER
- 
-int stack_trace_sysctl(struct ctl_table *table, int write, void *buffer,
-+int stack_trace_sysctl(const struct ctl_table *table, int write, void *buffer,
- 		       size_t *lenp, loff_t *ppos);
- 
- /* DO NOT MODIFY THIS VARIABLE DIRECTLY! */
-@@ -493,6 +654,30 @@ static inline void stack_tracer_disable(void) { }
- static inline void stack_tracer_enable(void) { }
- #endif
- 
-+enum {
-+	FTRACE_UPDATE_CALLS		= (1 << 0),
-+	FTRACE_DISABLE_CALLS		= (1 << 1),
-+	FTRACE_UPDATE_TRACE_FUNC	= (1 << 2),
-+	FTRACE_START_FUNC_RET		= (1 << 3),
-+	FTRACE_STOP_FUNC_RET		= (1 << 4),
-+	FTRACE_MAY_SLEEP		= (1 << 5),
-+};
-+
-+/* Arches can override ftrace_get_symaddr() to convert fentry_ip to symaddr. */
-+#ifndef ftrace_get_symaddr
-+/**
-+ * ftrace_get_symaddr - return the symbol address from fentry_ip
-+ * @fentry_ip: the address of ftrace location
-+ *
-+ * Get the symbol address from @fentry_ip (fast path). If there is no fast
-+ * search path, this returns 0.
-+ * User may need to use kallsyms API to find the symbol address.
-+ */
-+#define ftrace_get_symaddr(fentry_ip) (0)
-+#endif
-+
-+void ftrace_sync_ipi(void *data);
-+
- #ifdef CONFIG_DYNAMIC_FTRACE
- 
- void ftrace_arch_code_modify_prepare(void);
-@@ -536,6 +721,10 @@ bool is_ftrace_trampoline(unsigned long addr);
-  *  IPMODIFY - the record allows for the IP address to be changed.
-  *  DISABLED - the record is not ready to be touched yet
-  *  DIRECT   - there is a direct function to call
-+ *  CALL_OPS - the record can use callsite-specific ops
-+ *  CALL_OPS_EN - the function is set up to use callsite-specific ops
-+ *  TOUCHED  - A callback was added since boot up
-+ *  MODIFIED - The function had IPMODIFY or DIRECT attached to it
-  *
-  * When a new ftrace_ops is registered and wants a function to save
-  * pt_regs, the rec->flags REGS is set. When the function has been
-@@ -553,9 +742,13 @@ enum {
- 	FTRACE_FL_DISABLED	= (1UL << 25),
- 	FTRACE_FL_DIRECT	= (1UL << 24),
- 	FTRACE_FL_DIRECT_EN	= (1UL << 23),
-+	FTRACE_FL_CALL_OPS	= (1UL << 22),
-+	FTRACE_FL_CALL_OPS_EN	= (1UL << 21),
-+	FTRACE_FL_TOUCHED	= (1UL << 20),
-+	FTRACE_FL_MODIFIED	= (1UL << 19),
- };
- 
-#define FTRACE_REF_MAX_SHIFT	23
-+#define FTRACE_REF_MAX_SHIFT	19
- #define FTRACE_REF_MAX		((1UL << FTRACE_REF_MAX_SHIFT) - 1)
- 
- #define ftrace_rec_count(rec)	((rec)->flags & FTRACE_REF_MAX)
-@@ -579,15 +772,6 @@ void ftrace_set_global_notrace(unsigned char *buf, int len, int reset);
- void ftrace_free_filter(struct ftrace_ops *ops);
- void ftrace_ops_set_global_filter(struct ftrace_ops *ops);
- 
-enum {
-	FTRACE_UPDATE_CALLS		= (1 << 0),
-	FTRACE_DISABLE_CALLS		= (1 << 1),
-	FTRACE_UPDATE_TRACE_FUNC	= (1 << 2),
-	FTRACE_START_FUNC_RET		= (1 << 3),
-	FTRACE_STOP_FUNC_RET		= (1 << 4),
-	FTRACE_MAY_SLEEP		= (1 << 5),
-};
-
- /*
-  * The FTRACE_UPDATE_* enum is used to pass information back
-  * from the ftrace_update_record() and ftrace_test_record()
-@@ -614,6 +798,8 @@ enum {
- 	FTRACE_ITER_PROBE	= (1 << 4),
- 	FTRACE_ITER_MOD		= (1 << 5),
- 	FTRACE_ITER_ENABLED	= (1 << 6),
-+	FTRACE_ITER_TOUCHED	= (1 << 7),
-+	FTRACE_ITER_ADDRS	= (1 << 8),
- };
- 
- void arch_ftrace_update_code(int command);
-@@ -655,7 +841,6 @@ void __init
- ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable);
- 
- /* defined in arch */
-extern int ftrace_ip_converted(unsigned long ip);
- extern int ftrace_dyn_arch_init(void);
- extern void ftrace_replace_code(int enable);
- extern int ftrace_update_ftrace_func(ftrace_func_t func);
-@@ -793,7 +978,9 @@ static inline int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
-  */
- extern int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr);
- 
-#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
-+#if defined(CONFIG_DYNAMIC_FTRACE_WITH_REGS) || \
-+	defined(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS) || \
-+	defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS)
- /**
-  * ftrace_modify_call - convert from one addr to another (no nop)
-  * @rec: the call site record (e.g. mcount/fentry)
-@@ -806,6 +993,9 @@ extern int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr);
-  * what we expect it to be, and then on success of the compare,
-  * it should write to the location.
-  *
-+ * When using call ops, this is called when the associated ops change, even
-+ * when (addr == old_addr).
-+ *
-  * The code segment at @rec->ip should be a caller to @old_addr
-  *
-  * Return must be:
-@@ -826,20 +1016,12 @@ static inline int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_a
- }
- #endif
- 
-/* May be defined in arch */
-extern int ftrace_arch_read_dyn_info(char *buf, int size);
-
- extern int skip_trace(unsigned long ip);
- extern void ftrace_module_init(struct module *mod);
- extern void ftrace_module_enable(struct module *mod);
- extern void ftrace_release_mod(struct module *mod);
-
-extern void ftrace_disable_daemon(void);
-extern void ftrace_enable_daemon(void);
- #else /* CONFIG_DYNAMIC_FTRACE */
- static inline int skip_trace(unsigned long ip) { return 0; }
-static inline void ftrace_disable_daemon(void) { }
-static inline void ftrace_enable_daemon(void) { }
- static inline void ftrace_module_init(struct module *mod) { }
- static inline void ftrace_module_enable(struct module *mod) { }
- static inline void ftrace_release_mod(struct module *mod) { }
-@@ -943,7 +1125,7 @@ static inline void __ftrace_enabled_restore(int enabled)
- #define CALLER_ADDR5 ((unsigned long)ftrace_return_address(5))
- #define CALLER_ADDR6 ((unsigned long)ftrace_return_address(6))
- 
-static inline unsigned long get_lock_parent_ip(void)
-+static __always_inline unsigned long get_lock_parent_ip(void)
- {
- 	unsigned long addr = CALLER_ADDR0;
- 
-@@ -967,7 +1149,7 @@ static inline unsigned long get_lock_parent_ip(void)
- # define trace_preempt_off(a0, a1) do { } while (0)
- #endif
- 
-#ifdef CONFIG_FTRACE_MCOUNT_RECORD
-+#ifdef CONFIG_DYNAMIC_FTRACE
- extern void ftrace_init(void);
- #ifdef CC_USING_PATCHABLE_FUNCTION_ENTRY
- #define FTRACE_CALLSITE_SECTION	"__patchable_function_entries"
-@@ -985,7 +1167,15 @@ static inline void ftrace_init(void) { }
-  */
- struct ftrace_graph_ent {
- 	unsigned long func; /* Current function */
-	int depth;
-+	unsigned long depth;
-+} __packed;
-+
-+/*
-+ * Structure that defines an entry function trace with retaddr.
-+ */
-+struct fgraph_retaddr_ent {
-+	struct ftrace_graph_ent ent;
-+	unsigned long retaddr;  /* Return address */
- } __packed;
- 
- /*
-@@ -995,26 +1185,44 @@ struct ftrace_graph_ent {
-  */
- struct ftrace_graph_ret {
- 	unsigned long func; /* Current function */
-+#ifdef CONFIG_FUNCTION_GRAPH_RETVAL
-+	unsigned long retval;
-+#endif
- 	int depth;
- 	/* Number of functions that overran the depth limit for current task */
- 	unsigned int overrun;
-	unsigned long long calltime;
-	unsigned long long rettime;
- } __packed;
- 
-/* Type of the callback handlers for tracing function graph*/
-typedef void (*trace_func_graph_ret_t)(struct ftrace_graph_ret *); /* return */
-typedef int (*trace_func_graph_ent_t)(struct ftrace_graph_ent *); /* entry */
-+struct fgraph_ops;
- 
-extern int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace);
-+/* Type of the callback handlers for tracing function graph*/
-+typedef void (*trace_func_graph_ret_t)(struct ftrace_graph_ret *,
-+				       struct fgraph_ops *,
-+				       struct ftrace_regs *); /* return */
-+typedef int (*trace_func_graph_ent_t)(struct ftrace_graph_ent *,
-+				      struct fgraph_ops *,
-+				      struct ftrace_regs *); /* entry */
-+
-+extern int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace,
-+				   struct fgraph_ops *gops,
-+				   struct ftrace_regs *fregs);
-+bool ftrace_pids_enabled(struct ftrace_ops *ops);
- 
- #ifdef CONFIG_FUNCTION_GRAPH_TRACER
- 
- struct fgraph_ops {
- 	trace_func_graph_ent_t		entryfunc;
- 	trace_func_graph_ret_t		retfunc;
-+	struct ftrace_ops		ops; /* for the hash lists */
-+	void				*private;
-+	trace_func_graph_ent_t		saved_func;
-+	int				idx;
- };
- 
-+void *fgraph_reserve_data(int idx, int size_bytes);
-+void *fgraph_retrieve_data(int idx, int *size_bytes);
-+void *fgraph_retrieve_parent_data(int idx, int *size_bytes, int depth);
-+
- /*
-  * Stack of return addresses for functions
-  * of a thread.
-@@ -1023,16 +1231,10 @@ struct fgraph_ops {
- struct ftrace_ret_stack {
- 	unsigned long ret;
- 	unsigned long func;
-	unsigned long long calltime;
-#ifdef CONFIG_FUNCTION_PROFILER
-	unsigned long long subtime;
-#endif
- #ifdef HAVE_FUNCTION_GRAPH_FP_TEST
- 	unsigned long fp;
- #endif
-#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
- 	unsigned long *retp;
-#endif
- };
- 
- /*
-@@ -1043,14 +1245,23 @@ struct ftrace_ret_stack {
- extern void return_to_handler(void);
- 
- extern int
-function_graph_enter(unsigned long ret, unsigned long func,
-		     unsigned long frame_pointer, unsigned long *retp);
-+function_graph_enter_regs(unsigned long ret, unsigned long func,
-+			  unsigned long frame_pointer, unsigned long *retp,
-+			  struct ftrace_regs *fregs);
-+
-+static inline int function_graph_enter(unsigned long ret, unsigned long func,
-+				       unsigned long fp, unsigned long *retp)
-+{
-+	return function_graph_enter_regs(ret, func, fp, retp, NULL);
-+}
- 
- struct ftrace_ret_stack *
-ftrace_graph_get_ret_stack(struct task_struct *task, int idx);
-+ftrace_graph_get_ret_stack(struct task_struct *task, int skip);
-+unsigned long ftrace_graph_top_ret_addr(struct task_struct *task);
- 
- unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
- 				    unsigned long ret, unsigned long *retp);
-+unsigned long *fgraph_get_task_var(struct fgraph_ops *gops);
- 
- /*
-  * Sometimes we don't want to trace a function with the function
-@@ -1089,6 +1300,9 @@ extern void ftrace_graph_init_task(struct task_struct *t);
- extern void ftrace_graph_exit_task(struct task_struct *t);
- extern void ftrace_graph_init_idle_task(struct task_struct *t, int cpu);
- 
-+/* Used by assembly, but to quiet sparse warnings */
-+extern struct ftrace_ops *function_trace_op;
-+
- static inline void pause_graph_tracing(void)
- {
- 	atomic_inc(&current->tracing_graph_pause);
-@@ -1122,57 +1336,11 @@ static inline void unpause_graph_tracing(void) { }
- #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
- 
- #ifdef CONFIG_TRACING
-
-/* flags for current->trace */
-enum {
-	TSK_TRACE_FL_TRACE_BIT	= 0,
-	TSK_TRACE_FL_GRAPH_BIT	= 1,
-};
-enum {
-	TSK_TRACE_FL_TRACE	= 1 << TSK_TRACE_FL_TRACE_BIT,
-	TSK_TRACE_FL_GRAPH	= 1 << TSK_TRACE_FL_GRAPH_BIT,
-};
-
-static inline void set_tsk_trace_trace(struct task_struct *tsk)
-{
-	set_bit(TSK_TRACE_FL_TRACE_BIT, &tsk->trace);
-}
-
-static inline void clear_tsk_trace_trace(struct task_struct *tsk)
-{
-	clear_bit(TSK_TRACE_FL_TRACE_BIT, &tsk->trace);
-}
-
-static inline int test_tsk_trace_trace(struct task_struct *tsk)
-{
-	return tsk->trace & TSK_TRACE_FL_TRACE;
-}
-
-static inline void set_tsk_trace_graph(struct task_struct *tsk)
-{
-	set_bit(TSK_TRACE_FL_GRAPH_BIT, &tsk->trace);
-}
-
-static inline void clear_tsk_trace_graph(struct task_struct *tsk)
-{
-	clear_bit(TSK_TRACE_FL_GRAPH_BIT, &tsk->trace);
-}
-
-static inline int test_tsk_trace_graph(struct task_struct *tsk)
-{
-	return tsk->trace & TSK_TRACE_FL_GRAPH;
-}
-
- enum ftrace_dump_mode;
- 
-extern enum ftrace_dump_mode ftrace_dump_on_oops;
-extern int tracepoint_printk;
-+extern int ftrace_dump_on_oops_enabled(void);
- 
- extern void disable_trace_on_warning(void);
-extern int __disable_trace_on_warning;
-
-int tracepoint_printk_sysctl(struct ctl_table *table, int write,
-			     void *buffer, size_t *lenp, loff_t *ppos);
- 
- #else /* CONFIG_TRACING */
- static inline void  disable_trace_on_warning(void) { }
-diff --git a/include/linux/ftrace_regs.h b/include/linux/ftrace_regs.h
-new file mode 100644
-index 000000000000..15627ceea9bc
--- /dev/null
-+++ b/include/linux/ftrace_regs.h
-@@ -0,0 +1,43 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_FTRACE_REGS_H
-+#define _LINUX_FTRACE_REGS_H
-+
-+/*
-+ * For archs that just copy pt_regs in ftrace regs, it can use this default.
-+ * If an architecture does not use pt_regs, it must define all the below
-+ * accessor functions.
-+ */
-+#ifndef HAVE_ARCH_FTRACE_REGS
-+struct __arch_ftrace_regs {
-+	struct pt_regs		regs;
-+};
-+
-+#define arch_ftrace_regs(fregs) ((struct __arch_ftrace_regs *)(fregs))
-+
-+struct ftrace_regs;
-+
-+#define ftrace_regs_get_instruction_pointer(fregs) \
-+	instruction_pointer(&arch_ftrace_regs(fregs)->regs)
-+#define ftrace_regs_get_argument(fregs, n) \
-+	regs_get_kernel_argument(&arch_ftrace_regs(fregs)->regs, n)
-+#define ftrace_regs_get_stack_pointer(fregs) \
-+	kernel_stack_pointer(&arch_ftrace_regs(fregs)->regs)
-+#define ftrace_regs_get_return_value(fregs) \
-+	regs_return_value(&arch_ftrace_regs(fregs)->regs)
-+#define ftrace_regs_set_return_value(fregs, ret) \
-+	regs_set_return_value(&arch_ftrace_regs(fregs)->regs, ret)
-+#define ftrace_override_function_with_return(fregs) \
-+	override_function_with_return(&arch_ftrace_regs(fregs)->regs)
-+#define ftrace_regs_query_register_offset(name) \
-+	regs_query_register_offset(name)
-+#define ftrace_regs_get_frame_pointer(fregs) \
-+	frame_pointer(&arch_ftrace_regs(fregs)->regs)
-+
-+#endif /* HAVE_ARCH_FTRACE_REGS */
-+
-+/* This can be overridden by the architectures */
-+#ifndef FTRACE_REGS_MAX_ARGS
-+# define FTRACE_REGS_MAX_ARGS	6
-+#endif
-+
-+#endif /* _LINUX_FTRACE_REGS_H */
-diff --git a/include/linux/futex.h b/include/linux/futex.h
-index b70df27d7e85..9e9750f04980 100644
--- a/include/linux/futex.h
-+++ b/include/linux/futex.h
-@@ -4,11 +4,11 @@
- 
- #include <linux/sched.h>
- #include <linux/ktime.h>
-+#include <linux/mm_types.h>
- 
- #include <uapi/linux/futex.h>
- 
- struct inode;
-struct mm_struct;
- struct task_struct;
- 
- /*
-@@ -34,6 +34,7 @@ union futex_key {
- 		u64 i_seq;
- 		unsigned long pgoff;
- 		unsigned int offset;
-+		/* unsigned int node; */
- 	} shared;
- 	struct {
- 		union {
-@@ -42,11 +43,13 @@ union futex_key {
- 		};
- 		unsigned long address;
- 		unsigned int offset;
-+		/* unsigned int node; */
- 	} private;
- 	struct {
- 		u64 ptr;
- 		unsigned long word;
- 		unsigned int offset;
-+		unsigned int node;	/* NOT hashed! */
- 	} both;
- };
- 
-@@ -77,7 +80,20 @@ void futex_exec_release(struct task_struct *tsk);
- 
- long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
- 	      u32 __user *uaddr2, u32 val2, u32 val3);
-#else
-+int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4);
-+
-+#ifdef CONFIG_FUTEX_PRIVATE_HASH
-+int futex_hash_allocate_default(void);
-+void futex_hash_free(struct mm_struct *mm);
-+int futex_mm_init(struct mm_struct *mm);
-+
-+#else /* !CONFIG_FUTEX_PRIVATE_HASH */
-+static inline int futex_hash_allocate_default(void) { return 0; }
-+static inline int futex_hash_free(struct mm_struct *mm) { return 0; }
-+static inline int futex_mm_init(struct mm_struct *mm) { return 0; }
-+#endif /* CONFIG_FUTEX_PRIVATE_HASH */
-+
-+#else /* !CONFIG_FUTEX */
- static inline void futex_init_task(struct task_struct *tsk) { }
- static inline void futex_exit_recursive(struct task_struct *tsk) { }
- static inline void futex_exit_release(struct task_struct *tsk) { }
-@@ -88,6 +104,17 @@ static inline long do_futex(u32 __user *uaddr, int op, u32 val,
- {
- 	return -EINVAL;
- }
-+static inline int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4)
-+{
-+	return -EINVAL;
-+}
-+static inline int futex_hash_allocate_default(void)
-+{
-+	return 0;
-+}
-+static inline int futex_hash_free(struct mm_struct *mm) { return 0; }
-+static inline int futex_mm_init(struct mm_struct *mm) { return 0; }
-+
- #endif
- 
- #endif
-diff --git a/include/linux/fw_table.h b/include/linux/fw_table.h
-new file mode 100644
-index 000000000000..9bd605b87c4c
--- /dev/null
-+++ b/include/linux/fw_table.h
-@@ -0,0 +1,61 @@
-+/* SPDX-License-Identifier: GPL-2.0-or-later */
-+/*
-+ *  fw_tables.h - Parsing support for ACPI and ACPI-like tables provided by
-+ *                platform or device firmware
-+ *
-+ *  Copyright (C) 2001 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com>
-+ *  Copyright (C) 2023 Intel Corp.
-+ */
-+#ifndef _FW_TABLE_H_
-+#define _FW_TABLE_H_
-+
-+union acpi_subtable_headers;
-+
-+typedef int (*acpi_tbl_entry_handler)(union acpi_subtable_headers *header,
-+				      const unsigned long end);
-+
-+typedef int (*acpi_tbl_entry_handler_arg)(union acpi_subtable_headers *header,
-+					  void *arg, const unsigned long end);
-+
-+struct acpi_subtable_proc {
-+	int id;
-+	acpi_tbl_entry_handler handler;
-+	acpi_tbl_entry_handler_arg handler_arg;
-+	void *arg;
-+	int count;
-+};
-+
-+union fw_table_header {
-+	struct acpi_table_header acpi;
-+	struct acpi_table_cdat cdat;
-+};
-+
-+union acpi_subtable_headers {
-+	struct acpi_subtable_header common;
-+	struct acpi_hmat_structure hmat;
-+	struct acpi_prmt_module_header prmt;
-+	struct acpi_cedt_header cedt;
-+	struct acpi_cdat_header cdat;
-+};
-+
-+int acpi_parse_entries_array(char *id, unsigned long table_size,
-+			     union fw_table_header *table_header,
-+			     unsigned long max_length,
-+			     struct acpi_subtable_proc *proc,
-+			     int proc_num, unsigned int max_entries);
-+
-+int cdat_table_parse(enum acpi_cdat_type type,
-+		     acpi_tbl_entry_handler_arg handler_arg, void *arg,
-+		     struct acpi_table_cdat *table_header,
-+		     unsigned long length);
-+
-+/* CXL is the only non-ACPI consumer of the FIRMWARE_TABLE library */
-+#if IS_ENABLED(CONFIG_ACPI) && !IS_ENABLED(CONFIG_CXL_BUS)
-+#define EXPORT_SYMBOL_FWTBL_LIB(x) EXPORT_SYMBOL_ACPI_LIB(x)
-+#define __init_or_fwtbl_lib __init_or_acpilib
-+#else
-+#define EXPORT_SYMBOL_FWTBL_LIB(x) EXPORT_SYMBOL_NS_GPL(x, "CXL")
-+#define __init_or_fwtbl_lib
-+#endif
-+
-+#endif
-diff --git a/include/linux/fwctl.h b/include/linux/fwctl.h
-new file mode 100644
-index 000000000000..5d61fc8a6871
--- /dev/null
-+++ b/include/linux/fwctl.h
-@@ -0,0 +1,135 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES
-+ */
-+#ifndef __LINUX_FWCTL_H
-+#define __LINUX_FWCTL_H
-+#include <linux/device.h>
-+#include <linux/cdev.h>
-+#include <linux/cleanup.h>
-+#include <uapi/fwctl/fwctl.h>
-+
-+struct fwctl_device;
-+struct fwctl_uctx;
-+
-+/**
-+ * struct fwctl_ops - Driver provided operations
-+ *
-+ * fwctl_unregister() will wait until all excuting ops are completed before it
-+ * returns. Drivers should be mindful to not let their ops run for too long as
-+ * it will block device hot unplug and module unloading.
-+ */
-+struct fwctl_ops {
-+	/**
-+	 * @device_type: The drivers assigned device_type number. This is uABI.
-+	 */
-+	enum fwctl_device_type device_type;
-+	/**
-+	 * @uctx_size: The size of the fwctl_uctx struct to allocate. The first
-+	 * bytes of this memory will be a fwctl_uctx. The driver can use the
-+	 * remaining bytes as its private memory.
-+	 */
-+	size_t uctx_size;
-+	/**
-+	 * @open_uctx: Called when a file descriptor is opened before the uctx
-+	 * is ever used.
-+	 */
-+	int (*open_uctx)(struct fwctl_uctx *uctx);
-+	/**
-+	 * @close_uctx: Called when the uctx is destroyed, usually when the FD
-+	 * is closed.
-+	 */
-+	void (*close_uctx)(struct fwctl_uctx *uctx);
-+	/**
-+	 * @info: Implement FWCTL_INFO. Return a kmalloc() memory that is copied
-+	 * to out_device_data. On input length indicates the size of the user
-+	 * buffer on output it indicates the size of the memory. The driver can
-+	 * ignore length on input, the core code will handle everything.
-+	 */
-+	void *(*info)(struct fwctl_uctx *uctx, size_t *length);
-+	/**
-+	 * @fw_rpc: Implement FWCTL_RPC. Deliver rpc_in/in_len to the FW and
-+	 * return the response and set out_len. rpc_in can be returned as the
-+	 * response pointer. Otherwise the returned pointer is freed with
-+	 * kvfree().
-+	 */
-+	void *(*fw_rpc)(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope,
-+			void *rpc_in, size_t in_len, size_t *out_len);
-+};
-+
-+/**
-+ * struct fwctl_device - Per-driver registration struct
-+ * @dev: The sysfs (class/fwctl/fwctlXX) device
-+ *
-+ * Each driver instance will have one of these structs with the driver private
-+ * data following immediately after. This struct is refcounted, it is freed by
-+ * calling fwctl_put().
-+ */
-+struct fwctl_device {
-+	struct device dev;
-+	/* private: */
-+	struct cdev cdev;
-+
-+	/* Protect uctx_list */
-+	struct mutex uctx_list_lock;
-+	struct list_head uctx_list;
-+	/*
-+	 * Protect ops, held for write when ops becomes NULL during unregister,
-+	 * held for read whenever ops is loaded or an ops function is running.
-+	 */
-+	struct rw_semaphore registration_lock;
-+	const struct fwctl_ops *ops;
-+};
-+
-+struct fwctl_device *_fwctl_alloc_device(struct device *parent,
-+					 const struct fwctl_ops *ops,
-+					 size_t size);
-+/**
-+ * fwctl_alloc_device - Allocate a fwctl
-+ * @parent: Physical device that provides the FW interface
-+ * @ops: Driver ops to register
-+ * @drv_struct: 'struct driver_fwctl' that holds the struct fwctl_device
-+ * @member: Name of the struct fwctl_device in @drv_struct
-+ *
-+ * This allocates and initializes the fwctl_device embedded in the drv_struct.
-+ * Upon success the pointer must be freed via fwctl_put(). Returns a 'drv_struct
-+ * \*' on success, NULL on error.
-+ */
-+#define fwctl_alloc_device(parent, ops, drv_struct, member)               \
-+	({                                                                \
-+		static_assert(__same_type(struct fwctl_device,            \
-+					  ((drv_struct *)NULL)->member)); \
-+		static_assert(offsetof(drv_struct, member) == 0);         \
-+		(drv_struct *)_fwctl_alloc_device(parent, ops,            \
-+						  sizeof(drv_struct));    \
-+	})
-+
-+static inline struct fwctl_device *fwctl_get(struct fwctl_device *fwctl)
-+{
-+	get_device(&fwctl->dev);
-+	return fwctl;
-+}
-+static inline void fwctl_put(struct fwctl_device *fwctl)
-+{
-+	put_device(&fwctl->dev);
-+}
-+DEFINE_FREE(fwctl, struct fwctl_device *, if (_T) fwctl_put(_T));
-+
-+int fwctl_register(struct fwctl_device *fwctl);
-+void fwctl_unregister(struct fwctl_device *fwctl);
-+
-+/**
-+ * struct fwctl_uctx - Per user FD context
-+ * @fwctl: fwctl instance that owns the context
-+ *
-+ * Every FD opened by userspace will get a unique context allocation. Any driver
-+ * private data will follow immediately after.
-+ */
-+struct fwctl_uctx {
-+	struct fwctl_device *fwctl;
-+	/* private: */
-+	/* Head at fwctl_device::uctx_list */
-+	struct list_head uctx_list_entry;
-+};
-+
-+#endif
-diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h
-index 89b9bdfca925..097be89487bf 100644
--- a/include/linux/fwnode.h
-+++ b/include/linux/fwnode.h
-@@ -2,6 +2,11 @@
- /*
-  * fwnode.h - Firmware device node object handle type definition.
-  *
-+ * This header file provides low-level data types and definitions for firmware
-+ * and device property providers. The respective API header files supplied by
-+ * them should contain all of the requisite data types and definitions for end
-+ * users, so including it directly should not be necessary.
-+ *
-  * Copyright (C) 2015, Intel Corporation
-  * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
-  */
-@@ -9,16 +14,22 @@
- #ifndef _LINUX_FWNODE_H_
- #define _LINUX_FWNODE_H_
- 
-#include <linux/types.h>
-#include <linux/list.h>
- #include <linux/bits.h>
- #include <linux/err.h>
-+#include <linux/list.h>
-+#include <linux/types.h>
-+
-+enum dev_dma_attr {
-+	DEV_DMA_NOT_SUPPORTED,
-+	DEV_DMA_NON_COHERENT,
-+	DEV_DMA_COHERENT,
-+};
- 
- struct fwnode_operations;
- struct device;
- 
- /*
- * fwnode link flags
-+ * fwnode flags
-  *
-  * LINKS_ADDED:	The fwnode has already be parsed to add fwnode links.
-  * NOT_DEVICE:	The fwnode will never be populated as a struct device.
-@@ -36,21 +47,34 @@ struct device;
- #define FWNODE_FLAG_INITIALIZED			BIT(2)
- #define FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD	BIT(3)
- #define FWNODE_FLAG_BEST_EFFORT			BIT(4)
-+#define FWNODE_FLAG_VISITED			BIT(5)
- 
- struct fwnode_handle {
- 	struct fwnode_handle *secondary;
- 	const struct fwnode_operations *ops;
-+
-+	/* The below is used solely by device links, don't use otherwise */
- 	struct device *dev;
- 	struct list_head suppliers;
- 	struct list_head consumers;
- 	u8 flags;
- };
- 
-+/*
-+ * fwnode link flags
-+ *
-+ * CYCLE:	The fwnode link is part of a cycle. Don't defer probe.
-+ * IGNORE:	Completely ignore this link, even during cycle detection.
-+ */
-+#define FWLINK_FLAG_CYCLE			BIT(0)
-+#define FWLINK_FLAG_IGNORE			BIT(1)
-+
- struct fwnode_link {
- 	struct fwnode_handle *supplier;
- 	struct list_head s_hook;
- 	struct fwnode_handle *consumer;
- 	struct list_head c_hook;
-+	u8 flags;
- };
- 
- /**
-@@ -72,7 +96,7 @@ struct fwnode_endpoint {
- #define SWNODE_GRAPH_PORT_NAME_FMT		"port@%u"
- #define SWNODE_GRAPH_ENDPOINT_NAME_FMT		"endpoint@%u"
- 
-#define NR_FWNODE_REFERENCE_ARGS	8
-+#define NR_FWNODE_REFERENCE_ARGS	16
- 
- /**
-  * struct fwnode_reference_args - Fwnode reference with additional arguments
-@@ -93,6 +117,7 @@ struct fwnode_reference_args {
-  * @device_is_available: Return true if the device is available.
-  * @device_get_match_data: Return the device driver match data.
-  * @property_present: Return true if a property is present.
-+ * @property_read_bool: Return a boolean property value.
-  * @property_read_int_array: Read an array of integer properties. Return zero on
-  *			     success, a negative error code otherwise.
-  * @property_read_string_array: Read an array of string properties. Return zero
-@@ -122,6 +147,8 @@ struct fwnode_operations {
- 	(*device_get_dma_attr)(const struct fwnode_handle *fwnode);
- 	bool (*property_present)(const struct fwnode_handle *fwnode,
- 				 const char *propname);
-+	bool (*property_read_bool)(const struct fwnode_handle *fwnode,
-+				   const char *propname);
- 	int (*property_read_int_array)(const struct fwnode_handle *fwnode,
- 				       const char *propname,
- 				       unsigned int elem_size, void *val,
-@@ -176,7 +203,6 @@ struct fwnode_operations {
- 		if (fwnode_has_op(fwnode, op))				\
- 			(fwnode)->ops->op(fwnode, ## __VA_ARGS__);	\
- 	} while (false)
-#define get_dev_from_fwnode(fwnode)	get_device((fwnode)->dev)
- 
- static inline void fwnode_init(struct fwnode_handle *fwnode,
- 			       const struct fwnode_operations *ops)
-@@ -198,10 +224,10 @@ static inline void fwnode_dev_initialized(struct fwnode_handle *fwnode,
- 		fwnode->flags &= ~FWNODE_FLAG_INITIALIZED;
- }
- 
-extern u32 fw_devlink_get_flags(void);
-extern bool fw_devlink_is_strict(void);
-int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup);
-+int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup,
-+		    u8 flags);
- void fwnode_links_purge(struct fwnode_handle *fwnode);
- void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode);
-+bool fw_devlink_is_strict(void);
- 
- #endif
-diff --git a/include/linux/gameport.h b/include/linux/gameport.h
-index 69081d899492..86d62fdafd7a 100644
--- a/include/linux/gameport.h
-+++ b/include/linux/gameport.h
-@@ -5,7 +5,6 @@
- #ifndef _GAMEPORT_H
- #define _GAMEPORT_H
- 
-#include <asm/io.h>
- #include <linux/types.h>
- #include <linux/list.h>
- #include <linux/mutex.h>
-@@ -59,12 +58,12 @@ struct gameport_driver {
- 
- 	bool ignore;
- };
-#define to_gameport_driver(d)	container_of(d, struct gameport_driver, driver)
-+#define to_gameport_driver(d)	container_of_const(d, struct gameport_driver, driver)
- 
- int gameport_open(struct gameport *gameport, struct gameport_driver *drv, int mode);
- void gameport_close(struct gameport *gameport);
- 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
-+#if IS_REACHABLE(CONFIG_GAMEPORT)
- 
- void __gameport_register_port(struct gameport *gameport, struct module *owner);
- /* use a define to avoid include chaining to get THIS_MODULE */
-@@ -110,7 +109,7 @@ static inline void gameport_free_port(struct gameport *gameport)
- 
- static inline void gameport_set_name(struct gameport *gameport, const char *name)
- {
-	strlcpy(gameport->name, name, sizeof(gameport->name));
-+	strscpy(gameport->name, name, sizeof(gameport->name));
- }
- 
- /*
-@@ -165,18 +164,12 @@ void gameport_unregister_driver(struct gameport_driver *drv);
- 
- static inline void gameport_trigger(struct gameport *gameport)
- {
-	if (gameport->trigger)
-		gameport->trigger(gameport);
-	else
-		outb(0xff, gameport->io);
-+	gameport->trigger(gameport);
- }
- 
- static inline unsigned char gameport_read(struct gameport *gameport)
- {
-	if (gameport->read)
-		return gameport->read(gameport);
-	else
-		return inb(gameport->io);
-+	return gameport->read(gameport);
- }
- 
- static inline int gameport_cooked_read(struct gameport *gameport, int *axes, int *buttons)
-diff --git a/include/linux/gcd.h b/include/linux/gcd.h
-index cb572677fd7f..616e81a7f7e3 100644
--- a/include/linux/gcd.h
-+++ b/include/linux/gcd.h
-@@ -3,6 +3,9 @@
- #define _GCD_H
- 
- #include <linux/compiler.h>
-+#include <linux/jump_label.h>
-+
-+DECLARE_STATIC_KEY_TRUE(efficient_ffs_key);
- 
- unsigned long gcd(unsigned long a, unsigned long b) __attribute_const__;
- 
-diff --git a/include/linux/generic-radix-tree.h b/include/linux/generic-radix-tree.h
-index 107613f7d792..5b51c3d582d6 100644
--- a/include/linux/generic-radix-tree.h
-+++ b/include/linux/generic-radix-tree.h
-@@ -5,7 +5,7 @@
-  * DOC: Generic radix trees/sparse arrays
-  *
-  * Very simple and minimalistic, supporting arbitrary size entries up to
- * PAGE_SIZE.
-+ * GENRADIX_NODE_SIZE.
-  *
-  * A genradix is defined with the type it will store, like so:
-  *
-@@ -38,18 +38,76 @@
- 
- #include <asm/page.h>
- #include <linux/bug.h>
-+#include <linux/limits.h>
- #include <linux/log2.h>
- #include <linux/math.h>
-+#include <linux/slab.h>
- #include <linux/types.h>
- 
- struct genradix_root;
- 
-+#define GENRADIX_NODE_SHIFT	9
-+#define GENRADIX_NODE_SIZE	(1U << GENRADIX_NODE_SHIFT)
-+
-+#define GENRADIX_ARY		(GENRADIX_NODE_SIZE / sizeof(struct genradix_node *))
-+#define GENRADIX_ARY_SHIFT	ilog2(GENRADIX_ARY)
-+
-+/* depth that's needed for a genradix that can address up to ULONG_MAX: */
-+#define GENRADIX_MAX_DEPTH	\
-+	DIV_ROUND_UP(BITS_PER_LONG - GENRADIX_NODE_SHIFT, GENRADIX_ARY_SHIFT)
-+
-+#define GENRADIX_DEPTH_MASK				\
-+	((unsigned long) (roundup_pow_of_two(GENRADIX_MAX_DEPTH + 1) - 1))
-+
-+static inline int genradix_depth_shift(unsigned depth)
-+{
-+	return GENRADIX_NODE_SHIFT + GENRADIX_ARY_SHIFT * depth;
-+}
-+
-+/*
-+ * Returns size (of data, in bytes) that a tree of a given depth holds:
-+ */
-+static inline size_t genradix_depth_size(unsigned depth)
-+{
-+	return 1UL << genradix_depth_shift(depth);
-+}
-+
-+static inline unsigned genradix_root_to_depth(struct genradix_root *r)
-+{
-+	return (unsigned long) r & GENRADIX_DEPTH_MASK;
-+}
-+
-+static inline struct genradix_node *genradix_root_to_node(struct genradix_root *r)
-+{
-+	return (void *) ((unsigned long) r & ~GENRADIX_DEPTH_MASK);
-+}
-+
- struct __genradix {
- 	struct genradix_root		*root;
- };
- 
-+struct genradix_node {
-+	union {
-+		/* Interior node: */
-+		struct genradix_node	*children[GENRADIX_ARY];
-+
-+		/* Leaf: */
-+		u8			data[GENRADIX_NODE_SIZE];
-+	};
-+};
-+
-+static inline struct genradix_node *genradix_alloc_node(gfp_t gfp_mask)
-+{
-+	return kzalloc(GENRADIX_NODE_SIZE, gfp_mask);
-+}
-+
-+static inline void genradix_free_node(struct genradix_node *node)
-+{
-+	kfree(node);
-+}
-+
- /*
- * NOTE: currently, sizeof(_type) must not be larger than PAGE_SIZE:
-+ * NOTE: currently, sizeof(_type) must not be larger than GENRADIX_NODE_SIZE:
-  */
- 
- #define __GENRADIX_INITIALIZER					\
-@@ -100,14 +158,14 @@ void __genradix_free(struct __genradix *);
- static inline size_t __idx_to_offset(size_t idx, size_t obj_size)
- {
- 	if (__builtin_constant_p(obj_size))
-		BUILD_BUG_ON(obj_size > PAGE_SIZE);
-+		BUILD_BUG_ON(obj_size > GENRADIX_NODE_SIZE);
- 	else
-		BUG_ON(obj_size > PAGE_SIZE);
-+		BUG_ON(obj_size > GENRADIX_NODE_SIZE);
- 
- 	if (!is_power_of_2(obj_size)) {
-		size_t objs_per_page = PAGE_SIZE / obj_size;
-+		size_t objs_per_page = GENRADIX_NODE_SIZE / obj_size;
- 
-		return (idx / objs_per_page) * PAGE_SIZE +
-+		return (idx / objs_per_page) * GENRADIX_NODE_SIZE +
- 			(idx % objs_per_page) * obj_size;
- 	} else {
- 		return idx * obj_size;
-@@ -116,9 +174,38 @@ static inline size_t __idx_to_offset(size_t idx, size_t obj_size)
- 
- #define __genradix_cast(_radix)		(typeof((_radix)->type[0]) *)
- #define __genradix_obj_size(_radix)	sizeof((_radix)->type[0])
-+#define __genradix_objs_per_page(_radix)			\
-+	(GENRADIX_NODE_SIZE / sizeof((_radix)->type[0]))
-+#define __genradix_page_remainder(_radix)			\
-+	(GENRADIX_NODE_SIZE % sizeof((_radix)->type[0]))
-+
- #define __genradix_idx_to_offset(_radix, _idx)			\
- 	__idx_to_offset(_idx, __genradix_obj_size(_radix))
- 
-+static inline void *__genradix_ptr_inlined(struct __genradix *radix, size_t offset)
-+{
-+	struct genradix_root *r = READ_ONCE(radix->root);
-+	struct genradix_node *n = genradix_root_to_node(r);
-+	unsigned level		= genradix_root_to_depth(r);
-+	unsigned shift		= genradix_depth_shift(level);
-+
-+	if (unlikely(ilog2(offset) >= genradix_depth_shift(level)))
-+		return NULL;
-+
-+	while (n && shift > GENRADIX_NODE_SHIFT) {
-+		shift -= GENRADIX_ARY_SHIFT;
-+		n = n->children[offset >> shift];
-+		offset &= (1UL << shift) - 1;
-+	}
-+
-+	return n ? &n->data[offset] : NULL;
-+}
-+
-+#define genradix_ptr_inlined(_radix, _idx)			\
-+	(__genradix_cast(_radix)				\
-+	 __genradix_ptr_inlined(&(_radix)->tree,		\
-+			__genradix_idx_to_offset(_radix, _idx)))
-+
- void *__genradix_ptr(struct __genradix *, size_t);
- 
- /**
-@@ -133,7 +220,24 @@ void *__genradix_ptr(struct __genradix *, size_t);
- 	 __genradix_ptr(&(_radix)->tree,			\
- 			__genradix_idx_to_offset(_radix, _idx)))
- 
-void *__genradix_ptr_alloc(struct __genradix *, size_t, gfp_t);
-+void *__genradix_ptr_alloc(struct __genradix *, size_t,
-+			   struct genradix_node **, gfp_t);
-+
-+#define genradix_ptr_alloc_inlined(_radix, _idx, _gfp)			\
-+	(__genradix_cast(_radix)					\
-+	 (__genradix_ptr_inlined(&(_radix)->tree,			\
-+			__genradix_idx_to_offset(_radix, _idx)) ?:	\
-+	  __genradix_ptr_alloc(&(_radix)->tree,				\
-+			__genradix_idx_to_offset(_radix, _idx),		\
-+			NULL, _gfp)))
-+
-+#define genradix_ptr_alloc_preallocated_inlined(_radix, _idx, _new_node, _gfp)\
-+	(__genradix_cast(_radix)					\
-+	 (__genradix_ptr_inlined(&(_radix)->tree,			\
-+			__genradix_idx_to_offset(_radix, _idx)) ?:	\
-+	  __genradix_ptr_alloc(&(_radix)->tree,				\
-+			__genradix_idx_to_offset(_radix, _idx),		\
-+			_new_node, _gfp)))
- 
- /**
-  * genradix_ptr_alloc - get a pointer to a genradix entry, allocating it
-@@ -148,7 +252,13 @@ void *__genradix_ptr_alloc(struct __genradix *, size_t, gfp_t);
- 	(__genradix_cast(_radix)				\
- 	 __genradix_ptr_alloc(&(_radix)->tree,			\
- 			__genradix_idx_to_offset(_radix, _idx),	\
-			_gfp))
-+			NULL, _gfp))
-+
-+#define genradix_ptr_alloc_preallocated(_radix, _idx, _new_node, _gfp)\
-+	(__genradix_cast(_radix)				\
-+	 __genradix_ptr_alloc(&(_radix)->tree,			\
-+			__genradix_idx_to_offset(_radix, _idx),	\
-+			_new_node, _gfp))
- 
- struct genradix_iter {
- 	size_t			offset;
-@@ -179,16 +289,40 @@ void *__genradix_iter_peek(struct genradix_iter *, struct __genradix *, size_t);
- #define genradix_iter_peek(_iter, _radix)			\
- 	(__genradix_cast(_radix)				\
- 	 __genradix_iter_peek(_iter, &(_radix)->tree,		\
-			      PAGE_SIZE / __genradix_obj_size(_radix)))
-+			__genradix_objs_per_page(_radix)))
-+
-+void *__genradix_iter_peek_prev(struct genradix_iter *, struct __genradix *,
-+				size_t, size_t);
-+
-+/**
-+ * genradix_iter_peek_prev - get first entry at or below iterator's current
-+ *			     position
-+ * @_iter:	a genradix_iter
-+ * @_radix:	genradix being iterated over
-+ *
-+ * If no more entries exist at or below @_iter's current position, returns NULL
-+ */
-+#define genradix_iter_peek_prev(_iter, _radix)			\
-+	(__genradix_cast(_radix)				\
-+	 __genradix_iter_peek_prev(_iter, &(_radix)->tree,	\
-+			__genradix_objs_per_page(_radix),	\
-+			__genradix_obj_size(_radix) +		\
-+			__genradix_page_remainder(_radix)))
- 
- static inline void __genradix_iter_advance(struct genradix_iter *iter,
- 					   size_t obj_size)
- {
-+	if (iter->offset + obj_size < iter->offset) {
-+		iter->offset	= SIZE_MAX;
-+		iter->pos	= SIZE_MAX;
-+		return;
-+	}
-+
- 	iter->offset += obj_size;
- 
- 	if (!is_power_of_2(obj_size) &&
-	    (iter->offset & (PAGE_SIZE - 1)) + obj_size > PAGE_SIZE)
-		iter->offset = round_up(iter->offset, PAGE_SIZE);
-+	    (iter->offset & (GENRADIX_NODE_SIZE - 1)) + obj_size > GENRADIX_NODE_SIZE)
-+		iter->offset = round_up(iter->offset, GENRADIX_NODE_SIZE);
- 
- 	iter->pos++;
- }
-@@ -196,6 +330,25 @@ static inline void __genradix_iter_advance(struct genradix_iter *iter,
- #define genradix_iter_advance(_iter, _radix)			\
- 	__genradix_iter_advance(_iter, __genradix_obj_size(_radix))
- 
-+static inline void __genradix_iter_rewind(struct genradix_iter *iter,
-+					  size_t obj_size)
-+{
-+	if (iter->offset == 0 ||
-+	    iter->offset == SIZE_MAX) {
-+		iter->offset = SIZE_MAX;
-+		return;
-+	}
-+
-+	if ((iter->offset & (GENRADIX_NODE_SIZE - 1)) == 0)
-+		iter->offset -= GENRADIX_NODE_SIZE % obj_size;
-+
-+	iter->offset -= obj_size;
-+	iter->pos--;
-+}
-+
-+#define genradix_iter_rewind(_iter, _radix)			\
-+	__genradix_iter_rewind(_iter, __genradix_obj_size(_radix))
-+
- #define genradix_for_each_from(_radix, _iter, _p, _start)	\
- 	for (_iter = genradix_iter_init(_radix, _start);	\
- 	     (_p = genradix_iter_peek(&_iter, _radix)) != NULL;	\
-@@ -213,6 +366,23 @@ static inline void __genradix_iter_advance(struct genradix_iter *iter,
- #define genradix_for_each(_radix, _iter, _p)			\
- 	genradix_for_each_from(_radix, _iter, _p, 0)
- 
-+#define genradix_last_pos(_radix)				\
-+	(SIZE_MAX / GENRADIX_NODE_SIZE * __genradix_objs_per_page(_radix) - 1)
-+
-+/**
-+ * genradix_for_each_reverse - iterate over entry in a genradix, reverse order
-+ * @_radix:	genradix to iterate over
-+ * @_iter:	a genradix_iter to track current position
-+ * @_p:		pointer to genradix entry type
-+ *
-+ * On every iteration, @_p will point to the current entry, and @_iter.pos
-+ * will be the current entry's index.
-+ */
-+#define genradix_for_each_reverse(_radix, _iter, _p)		\
-+	for (_iter = genradix_iter_init(_radix,	genradix_last_pos(_radix));\
-+	     (_p = genradix_iter_peek_prev(&_iter, _radix)) != NULL;\
-+	     genradix_iter_rewind(&_iter, _radix))
-+
- int __genradix_prealloc(struct __genradix *, size_t, gfp_t);
- 
- /**
-diff --git a/include/linux/generic_pt/common.h b/include/linux/generic_pt/common.h
-new file mode 100644
-index 000000000000..6a9a1acb5aad
--- /dev/null
-+++ b/include/linux/generic_pt/common.h
-@@ -0,0 +1,191 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES
-+ */
-+#ifndef __GENERIC_PT_COMMON_H
-+#define __GENERIC_PT_COMMON_H
-+
-+#include <linux/types.h>
-+#include <linux/build_bug.h>
-+#include <linux/bits.h>
-+
-+/**
-+ * DOC: Generic Radix Page Table
-+ *
-+ * Generic Radix Page Table is a set of functions and helpers to efficiently
-+ * parse radix style page tables typically seen in HW implementations. The
-+ * interface is built to deliver similar code generation as the mm's pte/pmd/etc
-+ * system by fully inlining the exact code required to handle each table level.
-+ *
-+ * Like the mm subsystem each format contributes its parsing implementation
-+ * under common names and the common code implements the required algorithms.
-+ *
-+ * The system is divided into three logical levels:
-+ *
-+ *  - The page table format and its manipulation functions
-+ *  - Generic helpers to give a consistent API regardless of underlying format
-+ *  - An algorithm implementation (e.g. IOMMU/DRM/KVM/MM)
-+ *
-+ * Multiple implementations are supported. The intention is to have the generic
-+ * format code be re-usable for whatever specialized implementation is required.
-+ * The generic code is solely about the format of the radix tree; it does not
-+ * include memory allocation or higher level decisions that are left for the
-+ * implementation.
-+ *
-+ * The generic framework supports a superset of functions across many HW
-+ * implementations:
-+ *
-+ *  - Entries comprised of contiguous blocks of IO PTEs for larger page sizes
-+ *  - Multi-level tables, up to 6 levels. Runtime selected top level
-+ *  - Runtime variable table level size (ARM's concatenated tables)
-+ *  - Expandable top level allowing dynamic sizing of table levels
-+ *  - Optional leaf entries at any level
-+ *  - 32-bit/64-bit virtual and output addresses, using every address bit
-+ *  - Dirty tracking
-+ *  - Sign extended addressing
-+ */
-+
-+/**
-+ * struct pt_common - struct for all page table implementations
-+ */
-+struct pt_common {
-+	/**
-+	 * @top_of_table: Encodes the table top pointer and the top level in a
-+	 * single value. Must use READ_ONCE/WRITE_ONCE to access it. The lower
-+	 * bits of the aligned table pointer are used for the level.
-+	 */
-+	uintptr_t top_of_table;
-+	/**
-+	 * @max_oasz_lg2: Maximum number of bits the OA can contain. Upper bits
-+	 * must be zero. This may be less than what the page table format
-+	 * supports, but must not be more.
-+	 */
-+	u8 max_oasz_lg2;
-+	/**
-+	 * @max_vasz_lg2: Maximum number of bits the VA can contain. Upper bits
-+	 * are 0 or 1 depending on pt_full_va_prefix(). This may be less than
-+	 * what the page table format supports, but must not be more. When
-+	 * PT_FEAT_DYNAMIC_TOP is set this reflects the maximum VA capability.
-+	 */
-+	u8 max_vasz_lg2;
-+	/**
-+	 * @features: Bitmap of `enum pt_features`
-+	 */
-+	unsigned int features;
-+};
-+
-+/* Encoding parameters for top_of_table */
-+enum {
-+	PT_TOP_LEVEL_BITS = 3,
-+	PT_TOP_LEVEL_MASK = GENMASK(PT_TOP_LEVEL_BITS - 1, 0),
-+};
-+
-+/**
-+ * enum pt_features - Features turned on in the table. Each symbol is a bit
-+ * position.
-+ */
-+enum pt_features {
-+	/**
-+	 * @PT_FEAT_DMA_INCOHERENT: Cache flush page table memory before
-+	 * assuming the HW can read it. Otherwise a SMP release is sufficient
-+	 * for HW to read it.
-+	 */
-+	PT_FEAT_DMA_INCOHERENT,
-+	/**
-+	 * @PT_FEAT_FULL_VA: The table can span the full VA range from 0 to
-+	 * PT_VADDR_MAX.
-+	 */
-+	PT_FEAT_FULL_VA,
-+	/**
-+	 * @PT_FEAT_DYNAMIC_TOP: The table's top level can be increased
-+	 * dynamically during map. This requires HW support for atomically
-+	 * setting both the table top pointer and the starting table level.
-+	 */
-+	PT_FEAT_DYNAMIC_TOP,
-+	/**
-+	 * @PT_FEAT_SIGN_EXTEND: The top most bit of the valid VA range sign
-+	 * extends up to the full pt_vaddr_t. This divides the page table into
-+	 * three VA ranges::
-+	 *
-+	 *   0         -> 2^N - 1             Lower
-+	 *   2^N       -> (MAX - 2^N - 1)     Non-Canonical
-+	 *   MAX - 2^N -> MAX                 Upper
-+	 *
-+	 * In this mode pt_common::max_vasz_lg2 includes the sign bit and the
-+	 * upper bits that don't fall within the translation are just validated.
-+	 *
-+	 * If not set there is no sign extension and valid VA goes from 0 to 2^N
-+	 * - 1.
-+	 */
-+	PT_FEAT_SIGN_EXTEND,
-+	/**
-+	 * @PT_FEAT_FLUSH_RANGE: IOTLB maintenance is done by flushing IOVA
-+	 * ranges which will clean out any walk cache or any IOPTE fully
-+	 * contained by the range. The optimization objective is to minimize the
-+	 * number of flushes even if ranges include IOVA gaps that do not need
-+	 * to be flushed.
-+	 */
-+	PT_FEAT_FLUSH_RANGE,
-+	/**
-+	 * @PT_FEAT_FLUSH_RANGE_NO_GAPS: Like PT_FEAT_FLUSH_RANGE except that
-+	 * the optimization objective is to only flush IOVA that has been
-+	 * changed. This mode is suitable for cases like hypervisor shadowing
-+	 * where flushing unchanged ranges may cause the hypervisor to reparse
-+	 * significant amount of page table.
-+	 */
-+	PT_FEAT_FLUSH_RANGE_NO_GAPS,
-+	/* private: */
-+	PT_FEAT_FMT_START,
-+};
-+
-+struct pt_amdv1 {
-+	struct pt_common common;
-+};
-+
-+enum {
-+	/*
-+	 * The memory backing the tables is encrypted. Use __sme_set() to adjust
-+	 * the page table pointers in the tree. This only works with
-+	 * CONFIG_AMD_MEM_ENCRYPT.
-+	 */
-+	PT_FEAT_AMDV1_ENCRYPT_TABLES = PT_FEAT_FMT_START,
-+	/*
-+	 * The PTEs are set to prevent cache incoherent traffic, such as PCI no
-+	 * snoop. This is set either at creation time or before the first map
-+	 * operation.
-+	 */
-+	PT_FEAT_AMDV1_FORCE_COHERENCE,
-+};
-+
-+struct pt_vtdss {
-+	struct pt_common common;
-+};
-+
-+enum {
-+	/*
-+	 * The PTEs are set to prevent cache incoherent traffic, such as PCI no
-+	 * snoop. This is set either at creation time or before the first map
-+	 * operation.
-+	 */
-+	PT_FEAT_VTDSS_FORCE_COHERENCE = PT_FEAT_FMT_START,
-+	/*
-+	 * Prevent creating read-only PTEs. Used to work around HW errata
-+	 * ERRATA_772415_SPR17.
-+	 */
-+	PT_FEAT_VTDSS_FORCE_WRITEABLE,
-+};
-+
-+struct pt_x86_64 {
-+	struct pt_common common;
-+};
-+
-+enum {
-+	/*
-+	 * The memory backing the tables is encrypted. Use __sme_set() to adjust
-+	 * the page table pointers in the tree. This only works with
-+	 * CONFIG_AMD_MEM_ENCRYPT.
-+	 */
-+	PT_FEAT_X86_64_AMD_ENCRYPT_TABLES = PT_FEAT_FMT_START,
-+};
-+
-+#endif
-diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h
-new file mode 100644
-index 000000000000..9eefbb74efd0
--- /dev/null
-+++ b/include/linux/generic_pt/iommu.h
-@@ -0,0 +1,293 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES
-+ */
-+#ifndef __GENERIC_PT_IOMMU_H
-+#define __GENERIC_PT_IOMMU_H
-+
-+#include <linux/generic_pt/common.h>
-+#include <linux/iommu.h>
-+#include <linux/mm_types.h>
-+
-+struct iommu_iotlb_gather;
-+struct pt_iommu_ops;
-+struct pt_iommu_driver_ops;
-+struct iommu_dirty_bitmap;
-+
-+/**
-+ * DOC: IOMMU Radix Page Table
-+ *
-+ * The IOMMU implementation of the Generic Page Table provides an ops struct
-+ * that is useful to go with an iommu_domain to serve the DMA API, IOMMUFD and
-+ * the generic map/unmap interface.
-+ *
-+ * This interface uses a caller provided locking approach. The caller must have
-+ * a VA range lock concept that prevents concurrent threads from calling ops on
-+ * the same VA. Generally the range lock must be at least as large as a single
-+ * map call.
-+ */
-+
-+/**
-+ * struct pt_iommu - Base structure for IOMMU page tables
-+ *
-+ * The format-specific struct will include this as the first member.
-+ */
-+struct pt_iommu {
-+	/**
-+	 * @domain: The core IOMMU domain. The driver should use a union to
-+	 * overlay this memory with its previously existing domain struct to
-+	 * create an alias.
-+	 */
-+	struct iommu_domain domain;
-+
-+	/**
-+	 * @ops: Function pointers to access the API
-+	 */
-+	const struct pt_iommu_ops *ops;
-+
-+	/**
-+	 * @driver_ops: Function pointers provided by the HW driver to help
-+	 * manage HW details like caches.
-+	 */
-+	const struct pt_iommu_driver_ops *driver_ops;
-+
-+	/**
-+	 * @nid: Node ID to use for table memory allocations. The IOMMU driver
-+	 * may want to set the NID to the device's NID, if there are multiple
-+	 * table walkers.
-+	 */
-+	int nid;
-+
-+	/**
-+	 * @iommu_device: Device pointer used for any DMA cache flushing when
-+	 * PT_FEAT_DMA_INCOHERENT. This is the iommu device that created the
-+	 * page table which must have dma ops that perform cache flushing.
-+	 */
-+	struct device *iommu_device;
-+};
-+
-+/**
-+ * struct pt_iommu_info - Details about the IOMMU page table
-+ *
-+ * Returned from pt_iommu_ops->get_info()
-+ */
-+struct pt_iommu_info {
-+	/**
-+	 * @pgsize_bitmap: A bitmask where each set bit indicates
-+	 * a page size that can be natively stored in the page table.
-+	 */
-+	u64 pgsize_bitmap;
-+};
-+
-+struct pt_iommu_ops {
-+	/**
-+	 * @set_dirty: Make the iova write dirty
-+	 * @iommu_table: Table to manipulate
-+	 * @iova: IO virtual address to start
-+	 *
-+	 * This is only used by iommufd testing. It makes the iova dirty so that
-+	 * read_and_clear_dirty() will see it as dirty. Unlike all the other ops
-+	 * this one is safe to call without holding any locking. It may return
-+	 * -EAGAIN if there is a race.
-+	 */
-+	int (*set_dirty)(struct pt_iommu *iommu_table, dma_addr_t iova);
-+
-+	/**
-+	 * @get_info: Return the pt_iommu_info structure
-+	 * @iommu_table: Table to query
-+	 *
-+	 * Return some basic static information about the page table.
-+	 */
-+	void (*get_info)(struct pt_iommu *iommu_table,
-+			 struct pt_iommu_info *info);
-+
-+	/**
-+	 * @deinit: Undo a format specific init operation
-+	 * @iommu_table: Table to destroy
-+	 *
-+	 * Release all of the memory. The caller must have already removed the
-+	 * table from all HW access and all caches.
-+	 */
-+	void (*deinit)(struct pt_iommu *iommu_table);
-+};
-+
-+/**
-+ * struct pt_iommu_driver_ops - HW IOTLB cache flushing operations
-+ *
-+ * The IOMMU driver should implement these using container_of(iommu_table) to
-+ * get to it's iommu_domain derived structure. All ops can be called in atomic
-+ * contexts as they are buried under DMA API calls.
-+ */
-+struct pt_iommu_driver_ops {
-+	/**
-+	 * @change_top: Update the top of table pointer
-+	 * @iommu_table: Table to operate on
-+	 * @top_paddr: New CPU physical address of the top pointer
-+	 * @top_level: IOMMU PT level of the new top
-+	 *
-+	 * Called under the get_top_lock() spinlock. The driver must update all
-+	 * HW references to this domain with a new top address and
-+	 * configuration. On return mappings placed in the new top must be
-+	 * reachable by the HW.
-+	 *
-+	 * top_level encodes the level in IOMMU PT format, level 0 is the
-+	 * smallest page size increasing from there. This has to be translated
-+	 * to any HW specific format. During this call the new top will not be
-+	 * visible to any other API.
-+	 *
-+	 * This op is only used by PT_FEAT_DYNAMIC_TOP, and is required if
-+	 * enabled.
-+	 */
-+	void (*change_top)(struct pt_iommu *iommu_table, phys_addr_t top_paddr,
-+			   unsigned int top_level);
-+
-+	/**
-+	 * @get_top_lock: lock to hold when changing the table top
-+	 * @iommu_table: Table to operate on
-+	 *
-+	 * Return a lock to hold when changing the table top page table from
-+	 * being stored in HW. The lock will be held prior to calling
-+	 * change_top() and released once the top is fully visible.
-+	 *
-+	 * Typically this would be a lock that protects the iommu_domain's
-+	 * attachment list.
-+	 *
-+	 * This op is only used by PT_FEAT_DYNAMIC_TOP, and is required if
-+	 * enabled.
-+	 */
-+	spinlock_t *(*get_top_lock)(struct pt_iommu *iommu_table);
-+};
-+
-+static inline void pt_iommu_deinit(struct pt_iommu *iommu_table)
-+{
-+	/*
-+	 * It is safe to call pt_iommu_deinit() before an init, or if init
-+	 * fails. The ops pointer will only become non-NULL if deinit needs to be
-+	 * run.
-+	 */
-+	if (iommu_table->ops)
-+		iommu_table->ops->deinit(iommu_table);
-+}
-+
-+/**
-+ * struct pt_iommu_cfg - Common configuration values for all formats
-+ */
-+struct pt_iommu_cfg {
-+	/**
-+	 * @features: Features required. Only these features will be turned on.
-+	 * The feature list should reflect what the IOMMU HW is capable of.
-+	 */
-+	unsigned int features;
-+	/**
-+	 * @hw_max_vasz_lg2: Maximum VA the IOMMU HW can support. This will
-+	 * imply the top level of the table.
-+	 */
-+	u8 hw_max_vasz_lg2;
-+	/**
-+	 * @hw_max_oasz_lg2: Maximum OA the IOMMU HW can support. The format
-+	 * might select a lower maximum OA.
-+	 */
-+	u8 hw_max_oasz_lg2;
-+};
-+
-+/* Generate the exported function signatures from iommu_pt.h */
-+#define IOMMU_PROTOTYPES(fmt)                                                  \
-+	phys_addr_t pt_iommu_##fmt##_iova_to_phys(struct iommu_domain *domain, \
-+						  dma_addr_t iova);            \
-+	int pt_iommu_##fmt##_map_pages(struct iommu_domain *domain,            \
-+				       unsigned long iova, phys_addr_t paddr,  \
-+				       size_t pgsize, size_t pgcount,          \
-+				       int prot, gfp_t gfp, size_t *mapped);   \
-+	size_t pt_iommu_##fmt##_unmap_pages(                                   \
-+		struct iommu_domain *domain, unsigned long iova,               \
-+		size_t pgsize, size_t pgcount,                                 \
-+		struct iommu_iotlb_gather *iotlb_gather);                      \
-+	int pt_iommu_##fmt##_read_and_clear_dirty(                             \
-+		struct iommu_domain *domain, unsigned long iova, size_t size,  \
-+		unsigned long flags, struct iommu_dirty_bitmap *dirty);        \
-+	int pt_iommu_##fmt##_init(struct pt_iommu_##fmt *table,                \
-+				  const struct pt_iommu_##fmt##_cfg *cfg,      \
-+				  gfp_t gfp);                                  \
-+	void pt_iommu_##fmt##_hw_info(struct pt_iommu_##fmt *table,            \
-+				      struct pt_iommu_##fmt##_hw_info *info)
-+#define IOMMU_FORMAT(fmt, member)       \
-+	struct pt_iommu_##fmt {         \
-+		struct pt_iommu iommu;  \
-+		struct pt_##fmt member; \
-+	};                              \
-+	IOMMU_PROTOTYPES(fmt)
-+
-+/*
-+ * A driver uses IOMMU_PT_DOMAIN_OPS to populate the iommu_domain_ops for the
-+ * iommu_pt
-+ */
-+#define IOMMU_PT_DOMAIN_OPS(fmt)                        \
-+	.iova_to_phys = &pt_iommu_##fmt##_iova_to_phys, \
-+	.map_pages = &pt_iommu_##fmt##_map_pages,       \
-+	.unmap_pages = &pt_iommu_##fmt##_unmap_pages
-+#define IOMMU_PT_DIRTY_OPS(fmt) \
-+	.read_and_clear_dirty = &pt_iommu_##fmt##_read_and_clear_dirty
-+
-+/*
-+ * The driver should setup its domain struct like
-+ *	union {
-+ *		struct iommu_domain domain;
-+ *		struct pt_iommu_xxx xx;
-+ *	};
-+ * PT_IOMMU_CHECK_DOMAIN(struct mock_iommu_domain, xx.iommu, domain);
-+ *
-+ * Which creates an alias between driver_domain.domain and
-+ * driver_domain.xx.iommu.domain. This is to avoid a mass rename of existing
-+ * driver_domain.domain users.
-+ */
-+#define PT_IOMMU_CHECK_DOMAIN(s, pt_iommu_memb, domain_memb) \
-+	static_assert(offsetof(s, pt_iommu_memb.domain) ==   \
-+		      offsetof(s, domain_memb))
-+
-+struct pt_iommu_amdv1_cfg {
-+	struct pt_iommu_cfg common;
-+	unsigned int starting_level;
-+};
-+
-+struct pt_iommu_amdv1_hw_info {
-+	u64 host_pt_root;
-+	u8 mode;
-+};
-+
-+IOMMU_FORMAT(amdv1, amdpt);
-+
-+/* amdv1_mock is used by the iommufd selftest */
-+#define pt_iommu_amdv1_mock pt_iommu_amdv1
-+#define pt_iommu_amdv1_mock_cfg pt_iommu_amdv1_cfg
-+struct pt_iommu_amdv1_mock_hw_info;
-+IOMMU_PROTOTYPES(amdv1_mock);
-+
-+struct pt_iommu_vtdss_cfg {
-+	struct pt_iommu_cfg common;
-+	/* 4 is a 57 bit 5 level table */
-+	unsigned int top_level;
-+};
-+
-+struct pt_iommu_vtdss_hw_info {
-+	u64 ssptptr;
-+	u8 aw;
-+};
-+
-+IOMMU_FORMAT(vtdss, vtdss_pt);
-+
-+struct pt_iommu_x86_64_cfg {
-+	struct pt_iommu_cfg common;
-+	/* 4 is a 57 bit 5 level table */
-+	unsigned int top_level;
-+};
-+
-+struct pt_iommu_x86_64_hw_info {
-+	u64 gcr3_pt;
-+	u8 levels;
-+};
-+
-+IOMMU_FORMAT(x86_64, x86_64_pt);
-+
-+#undef IOMMU_PROTOTYPES
-+#undef IOMMU_FORMAT
-+#endif
-diff --git a/include/linux/genetlink.h b/include/linux/genetlink.h
-deleted file mode 100644
-index c285968e437a..000000000000
--- a/include/linux/genetlink.h
-+++ /dev/null
-@@ -1,19 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_GENERIC_NETLINK_H
-#define __LINUX_GENERIC_NETLINK_H
-
-#include <uapi/linux/genetlink.h>
-
-
-/* All generic netlink requests are serialized by a global lock.  */
-extern void genl_lock(void);
-extern void genl_unlock(void);
-
-/* for synchronisation between af_netlink and genetlink */
-extern atomic_t genl_sk_destructing_cnt;
-extern wait_queue_head_t genl_sk_destructing_waitq;
-
-#define MODULE_ALIAS_GENL_FAMILY(family)\
- MODULE_ALIAS_NET_PF_PROTO_NAME(PF_NETLINK, NETLINK_GENERIC, "-family-" family)
-
-#endif	/* __LINUX_GENERIC_NETLINK_H */
-diff --git a/include/linux/genl_magic_func.h b/include/linux/genl_magic_func.h
-index 939b1a8f571b..d4da060b7532 100644
--- a/include/linux/genl_magic_func.h
-+++ b/include/linux/genl_magic_func.h
-@@ -2,6 +2,7 @@
- #ifndef GENL_MAGIC_FUNC_H
- #define GENL_MAGIC_FUNC_H
- 
-+#include <linux/args.h>
- #include <linux/build_bug.h>
- #include <linux/genl_magic_struct.h>
- 
-@@ -23,7 +24,7 @@
- #define GENL_struct(tag_name, tag_number, s_name, s_fields)		\
- 	[tag_name] = { .type = NLA_NESTED },
- 
-static struct nla_policy CONCAT_(GENL_MAGIC_FAMILY, _tla_nl_policy)[] = {
-+static struct nla_policy CONCATENATE(GENL_MAGIC_FAMILY, _tla_nl_policy)[] = {
- #include GENL_MAGIC_INCLUDE_FILE
- };
- 
-@@ -209,7 +210,7 @@ static int s_name ## _from_attrs_for_change(struct s_name *s,		\
-  * Magic: define op number to op name mapping				{{{1
-  *									{{{2
-  */
-const char *CONCAT_(GENL_MAGIC_FAMILY, _genl_cmd_to_str)(__u8 cmd)
-+static const char *CONCATENATE(GENL_MAGIC_FAMILY, _genl_cmd_to_str)(__u8 cmd)
- {
- 	switch (cmd) {
- #undef GENL_op
-@@ -235,7 +236,7 @@ const char *CONCAT_(GENL_MAGIC_FAMILY, _genl_cmd_to_str)(__u8 cmd)
- 	.cmd = op_name,						\
- },
- 
-#define ZZZ_genl_ops		CONCAT_(GENL_MAGIC_FAMILY, _genl_ops)
-+#define ZZZ_genl_ops		CONCATENATE(GENL_MAGIC_FAMILY, _genl_ops)
- static struct genl_ops ZZZ_genl_ops[] __read_mostly = {
- #include GENL_MAGIC_INCLUDE_FILE
- };
-@@ -248,32 +249,32 @@ static struct genl_ops ZZZ_genl_ops[] __read_mostly = {
-  * and provide register/unregister functions.
-  *									{{{2
-  */
-#define ZZZ_genl_family		CONCAT_(GENL_MAGIC_FAMILY, _genl_family)
-+#define ZZZ_genl_family		CONCATENATE(GENL_MAGIC_FAMILY, _genl_family)
- static struct genl_family ZZZ_genl_family;
- /*
-  * Magic: define multicast groups
-  * Magic: define multicast group registration helper
-  */
-#define ZZZ_genl_mcgrps		CONCAT_(GENL_MAGIC_FAMILY, _genl_mcgrps)
-+#define ZZZ_genl_mcgrps		CONCATENATE(GENL_MAGIC_FAMILY, _genl_mcgrps)
- static const struct genl_multicast_group ZZZ_genl_mcgrps[] = {
- #undef GENL_mc_group
- #define GENL_mc_group(group) { .name = #group, },
- #include GENL_MAGIC_INCLUDE_FILE
- };
- 
-enum CONCAT_(GENL_MAGIC_FAMILY, group_ids) {
-+enum CONCATENATE(GENL_MAGIC_FAMILY, group_ids) {
- #undef GENL_mc_group
-#define GENL_mc_group(group) CONCAT_(GENL_MAGIC_FAMILY, _group_ ## group),
-+#define GENL_mc_group(group) CONCATENATE(GENL_MAGIC_FAMILY, _group_ ## group),
- #include GENL_MAGIC_INCLUDE_FILE
- };
- 
- #undef GENL_mc_group
- #define GENL_mc_group(group)						\
-static int CONCAT_(GENL_MAGIC_FAMILY, _genl_multicast_ ## group)(	\
-+static int CONCATENATE(GENL_MAGIC_FAMILY, _genl_multicast_ ## group)(	\
- 	struct sk_buff *skb, gfp_t flags)				\
- {									\
- 	unsigned int group_id =						\
-		CONCAT_(GENL_MAGIC_FAMILY, _group_ ## group);		\
-+		CONCATENATE(GENL_MAGIC_FAMILY, _group_ ## group);		\
- 	return genlmsg_multicast(&ZZZ_genl_family, skb, 0,		\
- 				 group_id, flags);			\
- }
-@@ -289,21 +290,22 @@ static struct genl_family ZZZ_genl_family __ro_after_init = {
- #ifdef GENL_MAGIC_FAMILY_HDRSZ
- 	.hdrsize = NLA_ALIGN(GENL_MAGIC_FAMILY_HDRSZ),
- #endif
-	.maxattr = ARRAY_SIZE(CONCAT_(GENL_MAGIC_FAMILY, _tla_nl_policy))-1,
-	.policy	= CONCAT_(GENL_MAGIC_FAMILY, _tla_nl_policy),
-+	.maxattr = ARRAY_SIZE(CONCATENATE(GENL_MAGIC_FAMILY, _tla_nl_policy))-1,
-+	.policy	= CONCATENATE(GENL_MAGIC_FAMILY, _tla_nl_policy),
- 	.ops = ZZZ_genl_ops,
- 	.n_ops = ARRAY_SIZE(ZZZ_genl_ops),
- 	.mcgrps = ZZZ_genl_mcgrps,
-+	.resv_start_op = 42, /* drbd is currently the only user */
- 	.n_mcgrps = ARRAY_SIZE(ZZZ_genl_mcgrps),
- 	.module = THIS_MODULE,
- };
- 
-int CONCAT_(GENL_MAGIC_FAMILY, _genl_register)(void)
-+int CONCATENATE(GENL_MAGIC_FAMILY, _genl_register)(void)
- {
- 	return genl_register_family(&ZZZ_genl_family);
- }
- 
-void CONCAT_(GENL_MAGIC_FAMILY, _genl_unregister)(void)
-+void CONCATENATE(GENL_MAGIC_FAMILY, _genl_unregister)(void)
- {
- 	genl_unregister_family(&ZZZ_genl_family);
- }
-diff --git a/include/linux/genl_magic_struct.h b/include/linux/genl_magic_struct.h
-index f81d48987528..621b87a87d74 100644
--- a/include/linux/genl_magic_struct.h
-+++ b/include/linux/genl_magic_struct.h
-@@ -14,14 +14,12 @@
- # error "you need to define GENL_MAGIC_INCLUDE_FILE before inclusion"
- #endif
- 
-#include <linux/genetlink.h>
-+#include <linux/args.h>
- #include <linux/types.h>
-+#include <net/genetlink.h>
- 
-#define CONCAT__(a,b)	a ## b
-#define CONCAT_(a,b)	CONCAT__(a,b)
-
-extern int CONCAT_(GENL_MAGIC_FAMILY, _genl_register)(void);
-extern void CONCAT_(GENL_MAGIC_FAMILY, _genl_unregister)(void);
-+extern int CONCATENATE(GENL_MAGIC_FAMILY, _genl_register)(void);
-+extern void CONCATENATE(GENL_MAGIC_FAMILY, _genl_unregister)(void);
- 
- /*
-  * Extension of genl attribute validation policies			{{{2
-diff --git a/include/linux/gfp.h b/include/linux/gfp.h
-index f314be58fa77..b155929af5b1 100644
--- a/include/linux/gfp.h
-+++ b/include/linux/gfp.h
-@@ -6,8 +6,12 @@
- 
- #include <linux/mmzone.h>
- #include <linux/topology.h>
-+#include <linux/alloc_tag.h>
-+#include <linux/cleanup.h>
-+#include <linux/sched.h>
- 
- struct vm_area_struct;
-+struct mempolicy;
- 
- /* Convert GFP flags to their corresponding migrate type */
- #define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE)
-@@ -18,6 +22,9 @@ static inline int gfp_migratetype(const gfp_t gfp_flags)
- 	VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK);
- 	BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE);
- 	BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE);
-+	BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE);
-+	BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >>
-+		      GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC);
- 
- 	if (unlikely(page_group_by_mobility_disabled))
- 		return MIGRATE_UNMOVABLE;
-@@ -33,27 +40,23 @@ static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags)
- 	return !!(gfp_flags & __GFP_DIRECT_RECLAIM);
- }
- 
-/**
- * gfpflags_normal_context - is gfp_flags a normal sleepable context?
- * @gfp_flags: gfp_flags to test
- *
- * Test whether @gfp_flags indicates that the allocation is from the
- * %current context and allowed to sleep.
- *
- * An allocation being allowed to block doesn't mean it owns the %current
- * context.  When direct reclaim path tries to allocate memory, the
- * allocation context is nested inside whatever %current was doing at the
- * time of the original allocation.  The nested allocation may be allowed
- * to block but modifying anything %current owns can corrupt the outer
- * context's expectations.
- *
- * %true result from this function indicates that the allocation context
- * can sleep and use anything that's associated with %current.
- */
-static inline bool gfpflags_normal_context(const gfp_t gfp_flags)
-+static inline bool gfpflags_allow_spinning(const gfp_t gfp_flags)
- {
-	return (gfp_flags & (__GFP_DIRECT_RECLAIM | __GFP_MEMALLOC)) ==
-		__GFP_DIRECT_RECLAIM;
-+	/*
-+	 * !__GFP_DIRECT_RECLAIM -> direct claim is not allowed.
-+	 * !__GFP_KSWAPD_RECLAIM -> it's not safe to wake up kswapd.
-+	 * All GFP_* flags including GFP_NOWAIT use one or both flags.
-+	 * alloc_pages_nolock() is the only API that doesn't specify either flag.
-+	 *
-+	 * This is stronger than GFP_NOWAIT or GFP_ATOMIC because
-+	 * those are guaranteed to never block on a sleeping lock.
-+	 * Here we are enforcing that the allocation doesn't ever spin
-+	 * on any locks (i.e. only trylocks). There is no high level
-+	 * GFP_$FOO flag for this use in alloc_pages_nolock() as the
-+	 * regular page allocator doesn't fully support this
-+	 * allocation mode.
-+	 */
-+	return !!(gfp_flags & __GFP_RECLAIM);
- }
- 
- #ifdef CONFIG_HIGHMEM
-@@ -174,6 +177,31 @@ static inline int gfp_zonelist(gfp_t flags)
- }
- 
- /*
-+ * gfp flag masking for nested internal allocations.
-+ *
-+ * For code that needs to do allocations inside the public allocation API (e.g.
-+ * memory allocation tracking code) the allocations need to obey the caller
-+ * allocation context constrains to prevent allocation context mismatches (e.g.
-+ * GFP_KERNEL allocations in GFP_NOFS contexts) from potential deadlock
-+ * situations.
-+ *
-+ * It is also assumed that these nested allocations are for internal kernel
-+ * object storage purposes only and are not going to be used for DMA, etc. Hence
-+ * we strip out all the zone information and leave just the context information
-+ * intact.
-+ *
-+ * Further, internal allocations must fail before the higher level allocation
-+ * can fail, so we must make them fail faster and fail silently. We also don't
-+ * want them to deplete emergency reserves.  Hence nested allocations must be
-+ * prepared for these allocations to fail.
-+ */
-+static inline gfp_t gfp_nested_mask(gfp_t flags)
-+{
-+	return ((flags & (GFP_KERNEL | GFP_ATOMIC | __GFP_NOLOCKDEP)) |
-+		(__GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN));
-+}
-+
-+/*
-  * We get the zone list from the current node and the gfp_mask.
-  * This zone list contains a maximum of MAX_NUMNODES*MAX_NR_ZONES zones.
-  * There are two zonelists per node, one for all zones with memory and
-@@ -194,40 +222,54 @@ static inline void arch_free_page(struct page *page, int order) { }
- static inline void arch_alloc_page(struct page *page, int order) { }
- #endif
- 
-struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid,
-+struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid,
- 		nodemask_t *nodemask);
-struct folio *__folio_alloc(gfp_t gfp, unsigned int order, int preferred_nid,
-+#define __alloc_pages(...)			alloc_hooks(__alloc_pages_noprof(__VA_ARGS__))
-+
-+struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid,
- 		nodemask_t *nodemask);
-+#define __folio_alloc(...)			alloc_hooks(__folio_alloc_noprof(__VA_ARGS__))
- 
-unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
-+unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
- 				nodemask_t *nodemask, int nr_pages,
-				struct list_head *page_list,
- 				struct page **page_array);
-+#define __alloc_pages_bulk(...)			alloc_hooks(alloc_pages_bulk_noprof(__VA_ARGS__))
- 
-unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp,
-+unsigned long alloc_pages_bulk_mempolicy_noprof(gfp_t gfp,
- 				unsigned long nr_pages,
- 				struct page **page_array);
-+#define  alloc_pages_bulk_mempolicy(...)				\
-+	alloc_hooks(alloc_pages_bulk_mempolicy_noprof(__VA_ARGS__))
- 
- /* Bulk allocate order-0 pages */
-static inline unsigned long
-alloc_pages_bulk_list(gfp_t gfp, unsigned long nr_pages, struct list_head *list)
-{
-	return __alloc_pages_bulk(gfp, numa_mem_id(), NULL, nr_pages, list, NULL);
-}
-+#define alloc_pages_bulk(_gfp, _nr_pages, _page_array)		\
-+	__alloc_pages_bulk(_gfp, numa_mem_id(), NULL, _nr_pages, _page_array)
- 
- static inline unsigned long
-alloc_pages_bulk_array(gfp_t gfp, unsigned long nr_pages, struct page **page_array)
-+alloc_pages_bulk_node_noprof(gfp_t gfp, int nid, unsigned long nr_pages,
-+				   struct page **page_array)
- {
-	return __alloc_pages_bulk(gfp, numa_mem_id(), NULL, nr_pages, NULL, page_array);
-+	if (nid == NUMA_NO_NODE)
-+		nid = numa_mem_id();
-+
-+	return alloc_pages_bulk_noprof(gfp, nid, NULL, nr_pages, page_array);
- }
- 
-static inline unsigned long
-alloc_pages_bulk_array_node(gfp_t gfp, int nid, unsigned long nr_pages, struct page **page_array)
-+#define alloc_pages_bulk_node(...)				\
-+	alloc_hooks(alloc_pages_bulk_node_noprof(__VA_ARGS__))
-+
-+static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask)
- {
-	if (nid == NUMA_NO_NODE)
-		nid = numa_mem_id();
-+	gfp_t warn_gfp = gfp_mask & (__GFP_THISNODE|__GFP_NOWARN);
-+
-+	if (warn_gfp != (__GFP_THISNODE|__GFP_NOWARN))
-+		return;
-+
-+	if (node_online(this_node))
-+		return;
- 
-	return __alloc_pages_bulk(gfp, nid, NULL, nr_pages, NULL, page_array);
-+	pr_warn("%pGg allocation from offline node %d\n", &gfp_mask, this_node);
-+	dump_stack();
- }
- 
- /*
-@@ -235,102 +277,123 @@ alloc_pages_bulk_array_node(gfp_t gfp, int nid, unsigned long nr_pages, struct p
-  * online. For more general interface, see alloc_pages_node().
-  */
- static inline struct page *
-__alloc_pages_node(int nid, gfp_t gfp_mask, unsigned int order)
-+__alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order)
- {
- 	VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES);
-	VM_WARN_ON((gfp_mask & __GFP_THISNODE) && !node_online(nid));
-+	warn_if_node_offline(nid, gfp_mask);
- 
-	return __alloc_pages(gfp_mask, order, nid, NULL);
-+	return __alloc_pages_noprof(gfp_mask, order, nid, NULL);
- }
- 
-+#define  __alloc_pages_node(...)		alloc_hooks(__alloc_pages_node_noprof(__VA_ARGS__))
-+
- static inline
-struct folio *__folio_alloc_node(gfp_t gfp, unsigned int order, int nid)
-+struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid)
- {
- 	VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES);
-	VM_WARN_ON((gfp & __GFP_THISNODE) && !node_online(nid));
-+	warn_if_node_offline(nid, gfp);
- 
-	return __folio_alloc(gfp, order, nid, NULL);
-+	return __folio_alloc_noprof(gfp, order, nid, NULL);
- }
- 
-+#define  __folio_alloc_node(...)		alloc_hooks(__folio_alloc_node_noprof(__VA_ARGS__))
-+
- /*
-  * Allocate pages, preferring the node given as nid. When nid == NUMA_NO_NODE,
-  * prefer the current CPU's closest node. Otherwise node must be valid and
-  * online.
-  */
-static inline struct page *alloc_pages_node(int nid, gfp_t gfp_mask,
-						unsigned int order)
-+static inline struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask,
-+						   unsigned int order)
- {
- 	if (nid == NUMA_NO_NODE)
- 		nid = numa_mem_id();
- 
-	return __alloc_pages_node(nid, gfp_mask, order);
-+	return __alloc_pages_node_noprof(nid, gfp_mask, order);
- }
- 
-+#define  alloc_pages_node(...)			alloc_hooks(alloc_pages_node_noprof(__VA_ARGS__))
-+
- #ifdef CONFIG_NUMA
-struct page *alloc_pages(gfp_t gfp, unsigned int order);
-struct folio *folio_alloc(gfp_t gfp, unsigned order);
-struct folio *vma_alloc_folio(gfp_t gfp, int order, struct vm_area_struct *vma,
-		unsigned long addr, bool hugepage);
-+struct page *alloc_pages_noprof(gfp_t gfp, unsigned int order);
-+struct folio *folio_alloc_noprof(gfp_t gfp, unsigned int order);
-+struct folio *folio_alloc_mpol_noprof(gfp_t gfp, unsigned int order,
-+		struct mempolicy *mpol, pgoff_t ilx, int nid);
-+struct folio *vma_alloc_folio_noprof(gfp_t gfp, int order, struct vm_area_struct *vma,
-+		unsigned long addr);
- #else
-static inline struct page *alloc_pages(gfp_t gfp_mask, unsigned int order)
-+static inline struct page *alloc_pages_noprof(gfp_t gfp_mask, unsigned int order)
-+{
-+	return alloc_pages_node_noprof(numa_node_id(), gfp_mask, order);
-+}
-+static inline struct folio *folio_alloc_noprof(gfp_t gfp, unsigned int order)
- {
-	return alloc_pages_node(numa_node_id(), gfp_mask, order);
-+	return __folio_alloc_node_noprof(gfp, order, numa_node_id());
- }
-static inline struct folio *folio_alloc(gfp_t gfp, unsigned int order)
-+static inline struct folio *folio_alloc_mpol_noprof(gfp_t gfp, unsigned int order,
-+		struct mempolicy *mpol, pgoff_t ilx, int nid)
- {
-	return __folio_alloc_node(gfp, order, numa_node_id());
-+	return folio_alloc_noprof(gfp, order);
- }
-#define vma_alloc_folio(gfp, order, vma, addr, hugepage)		\
-	folio_alloc(gfp, order)
-+#define vma_alloc_folio_noprof(gfp, order, vma, addr)		\
-+	folio_alloc_noprof(gfp, order)
- #endif
-+
-+#define alloc_pages(...)			alloc_hooks(alloc_pages_noprof(__VA_ARGS__))
-+#define folio_alloc(...)			alloc_hooks(folio_alloc_noprof(__VA_ARGS__))
-+#define folio_alloc_mpol(...)			alloc_hooks(folio_alloc_mpol_noprof(__VA_ARGS__))
-+#define vma_alloc_folio(...)			alloc_hooks(vma_alloc_folio_noprof(__VA_ARGS__))
-+
- #define alloc_page(gfp_mask) alloc_pages(gfp_mask, 0)
-static inline struct page *alloc_page_vma(gfp_t gfp,
-+
-+static inline struct page *alloc_page_vma_noprof(gfp_t gfp,
- 		struct vm_area_struct *vma, unsigned long addr)
- {
-	struct folio *folio = vma_alloc_folio(gfp, 0, vma, addr, false);
-+	struct folio *folio = vma_alloc_folio_noprof(gfp, 0, vma, addr);
- 
- 	return &folio->page;
- }
-+#define alloc_page_vma(...)			alloc_hooks(alloc_page_vma_noprof(__VA_ARGS__))
- 
-extern unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order);
-extern unsigned long get_zeroed_page(gfp_t gfp_mask);
-+struct page *alloc_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order);
-+#define alloc_pages_nolock(...)			alloc_hooks(alloc_pages_nolock_noprof(__VA_ARGS__))
- 
-void *alloc_pages_exact(size_t size, gfp_t gfp_mask) __alloc_size(1);
-void free_pages_exact(void *virt, size_t size);
-__meminit void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2);
-+extern unsigned long get_free_pages_noprof(gfp_t gfp_mask, unsigned int order);
-+#define __get_free_pages(...)			alloc_hooks(get_free_pages_noprof(__VA_ARGS__))
- 
-#define __get_free_page(gfp_mask) \
-		__get_free_pages((gfp_mask), 0)
-+extern unsigned long get_zeroed_page_noprof(gfp_t gfp_mask);
-+#define get_zeroed_page(...)			alloc_hooks(get_zeroed_page_noprof(__VA_ARGS__))
- 
-#define __get_dma_pages(gfp_mask, order) \
-		__get_free_pages((gfp_mask) | GFP_DMA, (order))
-+void *alloc_pages_exact_noprof(size_t size, gfp_t gfp_mask) __alloc_size(1);
-+#define alloc_pages_exact(...)			alloc_hooks(alloc_pages_exact_noprof(__VA_ARGS__))
- 
-extern void __free_pages(struct page *page, unsigned int order);
-extern void free_pages(unsigned long addr, unsigned int order);
-+void free_pages_exact(void *virt, size_t size);
- 
-struct page_frag_cache;
-extern void __page_frag_cache_drain(struct page *page, unsigned int count);
-extern void *page_frag_alloc_align(struct page_frag_cache *nc,
-				   unsigned int fragsz, gfp_t gfp_mask,
-				   unsigned int align_mask);
-+__meminit void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2);
-+#define alloc_pages_exact_nid(...)					\
-+	alloc_hooks(alloc_pages_exact_nid_noprof(__VA_ARGS__))
- 
-static inline void *page_frag_alloc(struct page_frag_cache *nc,
-			     unsigned int fragsz, gfp_t gfp_mask)
-{
-	return page_frag_alloc_align(nc, fragsz, gfp_mask, ~0u);
-}
-+#define __get_free_page(gfp_mask)					\
-+	__get_free_pages((gfp_mask), 0)
- 
-extern void page_frag_free(void *addr);
-+#define __get_dma_pages(gfp_mask, order)				\
-+	__get_free_pages((gfp_mask) | GFP_DMA, (order))
-+
-+extern void __free_pages(struct page *page, unsigned int order);
-+extern void free_pages_nolock(struct page *page, unsigned int order);
-+extern void free_pages(unsigned long addr, unsigned int order);
- 
- #define __free_page(page) __free_pages((page), 0)
- #define free_page(addr) free_pages((addr), 0)
- 
-void page_alloc_init(void);
-+void page_alloc_init_cpuhp(void);
-+bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp);
- void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp);
- void drain_all_pages(struct zone *zone);
- void drain_local_pages(struct zone *zone);
- 
- void page_alloc_init_late(void);
-+void setup_pcp_cacheinfo(unsigned int cpu);
- 
- /*
-  * gfp_allowed_mask is set to GFP_BOOT_MASK during early boot to restrict what
-@@ -344,32 +407,63 @@ extern gfp_t gfp_allowed_mask;
- /* Returns true if the gfp_mask allows use of ALLOC_NO_WATERMARK */
- bool gfp_pfmemalloc_allowed(gfp_t gfp_mask);
- 
-extern void pm_restrict_gfp_mask(void);
-extern void pm_restore_gfp_mask(void);
-
-extern gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma);
-+static inline bool gfp_has_io_fs(gfp_t gfp)
-+{
-+	return (gfp & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS);
-+}
- 
-#ifdef CONFIG_PM_SLEEP
-extern bool pm_suspended_storage(void);
-#else
-static inline bool pm_suspended_storage(void)
-+/*
-+ * Check if the gfp flags allow compaction - GFP_NOIO is a really
-+ * tricky context because the migration might require IO.
-+ */
-+static inline bool gfp_compaction_allowed(gfp_t gfp_mask)
- {
-	return false;
-+	return IS_ENABLED(CONFIG_COMPACTION) && (gfp_mask & __GFP_IO);
- }
-#endif /* CONFIG_PM_SLEEP */
-+
-+extern gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma);
- 
- #ifdef CONFIG_CONTIG_ALLOC
-+
-+typedef unsigned int __bitwise acr_flags_t;
-+#define ACR_FLAGS_NONE ((__force acr_flags_t)0) // ordinary allocation request
-+#define ACR_FLAGS_CMA ((__force acr_flags_t)BIT(0)) // allocate for CMA
-+
- /* The below functions must be run on a range from a single zone. */
-extern int alloc_contig_range(unsigned long start, unsigned long end,
-			      unsigned migratetype, gfp_t gfp_mask);
-extern struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
-				       int nid, nodemask_t *nodemask);
-+extern int alloc_contig_range_noprof(unsigned long start, unsigned long end,
-+				     acr_flags_t alloc_flags, gfp_t gfp_mask);
-+#define alloc_contig_range(...)			alloc_hooks(alloc_contig_range_noprof(__VA_ARGS__))
-+
-+extern struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask,
-+					      int nid, nodemask_t *nodemask);
-+#define alloc_contig_pages(...)			alloc_hooks(alloc_contig_pages_noprof(__VA_ARGS__))
-+
- #endif
- void free_contig_range(unsigned long pfn, unsigned long nr_pages);
- 
-#ifdef CONFIG_CMA
-/* CMA stuff */
-extern void init_cma_reserved_pageblock(struct page *page);
-+#ifdef CONFIG_CONTIG_ALLOC
-+static inline struct folio *folio_alloc_gigantic_noprof(int order, gfp_t gfp,
-+							int nid, nodemask_t *node)
-+{
-+	struct page *page;
-+
-+	if (WARN_ON(!order || !(gfp & __GFP_COMP)))
-+		return NULL;
-+
-+	page = alloc_contig_pages_noprof(1 << order, gfp, nid, node);
-+
-+	return page ? page_folio(page) : NULL;
-+}
-+#else
-+static inline struct folio *folio_alloc_gigantic_noprof(int order, gfp_t gfp,
-+							int nid, nodemask_t *node)
-+{
-+	return NULL;
-+}
- #endif
-+/* This should be paired with folio_put() rather than free_contig_range(). */
-+#define folio_alloc_gigantic(...) alloc_hooks(folio_alloc_gigantic_noprof(__VA_ARGS__))
-+
-+DEFINE_FREE(free_page, void *, free_page((unsigned long)_T))
- 
- #endif /* __LINUX_GFP_H */
-diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h
-index d88c46ca82e1..3de43b12209e 100644
--- a/include/linux/gfp_types.h
-+++ b/include/linux/gfp_types.h
-@@ -2,6 +2,8 @@
- #ifndef __LINUX_GFP_TYPES_H
- #define __LINUX_GFP_TYPES_H
- 
-+#include <linux/bits.h>
-+
- /* The typedef is in types.h but we want the documentation here */
- #if 0
- /**
-@@ -21,46 +23,80 @@ typedef unsigned int __bitwise gfp_t;
-  * include/trace/events/mmflags.h and tools/perf/builtin-kmem.c
-  */
- 
-+enum {
-+	___GFP_DMA_BIT,
-+	___GFP_HIGHMEM_BIT,
-+	___GFP_DMA32_BIT,
-+	___GFP_MOVABLE_BIT,
-+	___GFP_RECLAIMABLE_BIT,
-+	___GFP_HIGH_BIT,
-+	___GFP_IO_BIT,
-+	___GFP_FS_BIT,
-+	___GFP_ZERO_BIT,
-+	___GFP_UNUSED_BIT,	/* 0x200u unused */
-+	___GFP_DIRECT_RECLAIM_BIT,
-+	___GFP_KSWAPD_RECLAIM_BIT,
-+	___GFP_WRITE_BIT,
-+	___GFP_NOWARN_BIT,
-+	___GFP_RETRY_MAYFAIL_BIT,
-+	___GFP_NOFAIL_BIT,
-+	___GFP_NORETRY_BIT,
-+	___GFP_MEMALLOC_BIT,
-+	___GFP_COMP_BIT,
-+	___GFP_NOMEMALLOC_BIT,
-+	___GFP_HARDWALL_BIT,
-+	___GFP_THISNODE_BIT,
-+	___GFP_ACCOUNT_BIT,
-+	___GFP_ZEROTAGS_BIT,
-+#ifdef CONFIG_KASAN_HW_TAGS
-+	___GFP_SKIP_ZERO_BIT,
-+	___GFP_SKIP_KASAN_BIT,
-+#endif
-+#ifdef CONFIG_LOCKDEP
-+	___GFP_NOLOCKDEP_BIT,
-+#endif
-+	___GFP_NO_OBJ_EXT_BIT,
-+	___GFP_LAST_BIT
-+};
-+
- /* Plain integer GFP bitmasks. Do not use this directly. */
-#define ___GFP_DMA		0x01u
-#define ___GFP_HIGHMEM		0x02u
-#define ___GFP_DMA32		0x04u
-#define ___GFP_MOVABLE		0x08u
-#define ___GFP_RECLAIMABLE	0x10u
-#define ___GFP_HIGH		0x20u
-#define ___GFP_IO		0x40u
-#define ___GFP_FS		0x80u
-#define ___GFP_ZERO		0x100u
-#define ___GFP_ATOMIC		0x200u
-#define ___GFP_DIRECT_RECLAIM	0x400u
-#define ___GFP_KSWAPD_RECLAIM	0x800u
-#define ___GFP_WRITE		0x1000u
-#define ___GFP_NOWARN		0x2000u
-#define ___GFP_RETRY_MAYFAIL	0x4000u
-#define ___GFP_NOFAIL		0x8000u
-#define ___GFP_NORETRY		0x10000u
-#define ___GFP_MEMALLOC		0x20000u
-#define ___GFP_COMP		0x40000u
-#define ___GFP_NOMEMALLOC	0x80000u
-#define ___GFP_HARDWALL		0x100000u
-#define ___GFP_THISNODE		0x200000u
-#define ___GFP_ACCOUNT		0x400000u
-#define ___GFP_ZEROTAGS		0x800000u
-+#define ___GFP_DMA		BIT(___GFP_DMA_BIT)
-+#define ___GFP_HIGHMEM		BIT(___GFP_HIGHMEM_BIT)
-+#define ___GFP_DMA32		BIT(___GFP_DMA32_BIT)
-+#define ___GFP_MOVABLE		BIT(___GFP_MOVABLE_BIT)
-+#define ___GFP_RECLAIMABLE	BIT(___GFP_RECLAIMABLE_BIT)
-+#define ___GFP_HIGH		BIT(___GFP_HIGH_BIT)
-+#define ___GFP_IO		BIT(___GFP_IO_BIT)
-+#define ___GFP_FS		BIT(___GFP_FS_BIT)
-+#define ___GFP_ZERO		BIT(___GFP_ZERO_BIT)
-+/* 0x200u unused */
-+#define ___GFP_DIRECT_RECLAIM	BIT(___GFP_DIRECT_RECLAIM_BIT)
-+#define ___GFP_KSWAPD_RECLAIM	BIT(___GFP_KSWAPD_RECLAIM_BIT)
-+#define ___GFP_WRITE		BIT(___GFP_WRITE_BIT)
-+#define ___GFP_NOWARN		BIT(___GFP_NOWARN_BIT)
-+#define ___GFP_RETRY_MAYFAIL	BIT(___GFP_RETRY_MAYFAIL_BIT)
-+#define ___GFP_NOFAIL		BIT(___GFP_NOFAIL_BIT)
-+#define ___GFP_NORETRY		BIT(___GFP_NORETRY_BIT)
-+#define ___GFP_MEMALLOC		BIT(___GFP_MEMALLOC_BIT)
-+#define ___GFP_COMP		BIT(___GFP_COMP_BIT)
-+#define ___GFP_NOMEMALLOC	BIT(___GFP_NOMEMALLOC_BIT)
-+#define ___GFP_HARDWALL		BIT(___GFP_HARDWALL_BIT)
-+#define ___GFP_THISNODE		BIT(___GFP_THISNODE_BIT)
-+#define ___GFP_ACCOUNT		BIT(___GFP_ACCOUNT_BIT)
-+#define ___GFP_ZEROTAGS		BIT(___GFP_ZEROTAGS_BIT)
- #ifdef CONFIG_KASAN_HW_TAGS
-#define ___GFP_SKIP_ZERO		0x1000000u
-#define ___GFP_SKIP_KASAN_UNPOISON	0x2000000u
-#define ___GFP_SKIP_KASAN_POISON	0x4000000u
-+#define ___GFP_SKIP_ZERO	BIT(___GFP_SKIP_ZERO_BIT)
-+#define ___GFP_SKIP_KASAN	BIT(___GFP_SKIP_KASAN_BIT)
- #else
-#define ___GFP_SKIP_ZERO		0
-#define ___GFP_SKIP_KASAN_UNPOISON	0
-#define ___GFP_SKIP_KASAN_POISON	0
-+#define ___GFP_SKIP_ZERO	0
-+#define ___GFP_SKIP_KASAN	0
- #endif
- #ifdef CONFIG_LOCKDEP
-#define ___GFP_NOLOCKDEP	0x8000000u
-+#define ___GFP_NOLOCKDEP	BIT(___GFP_NOLOCKDEP_BIT)
- #else
- #define ___GFP_NOLOCKDEP	0
- #endif
-/* If the above are modified, __GFP_BITS_SHIFT may need updating */
-+#define ___GFP_NO_OBJ_EXT       BIT(___GFP_NO_OBJ_EXT_BIT)
- 
- /*
-  * Physical address zone modifiers (see linux/mmzone.h - low four bits)
-@@ -101,12 +137,15 @@ typedef unsigned int __bitwise gfp_t;
-  * node with no fallbacks or placement policy enforcements.
-  *
-  * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg.
-+ *
-+ * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension.
-  */
- #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE)
- #define __GFP_WRITE	((__force gfp_t)___GFP_WRITE)
- #define __GFP_HARDWALL   ((__force gfp_t)___GFP_HARDWALL)
- #define __GFP_THISNODE	((__force gfp_t)___GFP_THISNODE)
- #define __GFP_ACCOUNT	((__force gfp_t)___GFP_ACCOUNT)
-+#define __GFP_NO_OBJ_EXT   ((__force gfp_t)___GFP_NO_OBJ_EXT)
- 
- /**
-  * DOC: Watermark modifiers
-@@ -116,11 +155,8 @@ typedef unsigned int __bitwise gfp_t;
-  *
-  * %__GFP_HIGH indicates that the caller is high-priority and that granting
-  * the request is necessary before the system can make forward progress.
- * For example, creating an IO context to clean pages.
- *
- * %__GFP_ATOMIC indicates that the caller cannot reclaim or sleep and is
- * high priority. Users are typically interrupt handlers. This may be
- * used in conjunction with %__GFP_HIGH
-+ * For example creating an IO context to clean pages and requests
-+ * from atomic context.
-  *
-  * %__GFP_MEMALLOC allows access to all memory. This should only be used when
-  * the caller guarantees the allocation will allow more memory to be freed
-@@ -135,7 +171,6 @@ typedef unsigned int __bitwise gfp_t;
-  * %__GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves.
-  * This takes precedence over the %__GFP_MEMALLOC flag if both are set.
-  */
-#define __GFP_ATOMIC	((__force gfp_t)___GFP_ATOMIC)
- #define __GFP_HIGH	((__force gfp_t)___GFP_HIGH)
- #define __GFP_MEMALLOC	((__force gfp_t)___GFP_MEMALLOC)
- #define __GFP_NOMEMALLOC ((__force gfp_t)___GFP_NOMEMALLOC)
-@@ -168,25 +203,26 @@ typedef unsigned int __bitwise gfp_t;
-  * %__GFP_RECLAIM is shorthand to allow/forbid both direct and kswapd reclaim.
-  *
-  * The default allocator behavior depends on the request size. We have a concept
- * of so called costly allocations (with order > %PAGE_ALLOC_COSTLY_ORDER).
-+ * of so-called costly allocations (with order > %PAGE_ALLOC_COSTLY_ORDER).
-  * !costly allocations are too essential to fail so they are implicitly
-  * non-failing by default (with some exceptions like OOM victims might fail so
-  * the caller still has to check for failures) while costly requests try to be
-  * not disruptive and back off even without invoking the OOM killer.
-  * The following three modifiers might be used to override some of these
- * implicit rules
-+ * implicit rules. Please note that all of them must be used along with
-+ * %__GFP_DIRECT_RECLAIM flag.
-  *
-  * %__GFP_NORETRY: The VM implementation will try only very lightweight
-  * memory direct reclaim to get some memory under memory pressure (thus
-  * it can sleep). It will avoid disruptive actions like OOM killer. The
-  * caller must handle the failure which is quite likely to happen under
-  * heavy memory pressure. The flag is suitable when failure can easily be
- * handled at small cost, such as reduced throughput
-+ * handled at small cost, such as reduced throughput.
-  *
-  * %__GFP_RETRY_MAYFAIL: The VM implementation will retry memory reclaim
-  * procedures that have previously failed if there is some indication
- * that progress has been made else where.  It can wait for other
- * tasks to attempt high level approaches to freeing memory such as
-+ * that progress has been made elsewhere.  It can wait for other
-+ * tasks to attempt high-level approaches to freeing memory such as
-  * compaction (which removes fragmentation) and page-out.
-  * There is still a definite limit to the number of retries, but it is
-  * a larger limit than with %__GFP_NORETRY.
-@@ -205,11 +241,14 @@ typedef unsigned int __bitwise gfp_t;
-  * cannot handle allocation failures. The allocation could block
-  * indefinitely but will never return with failure. Testing for
-  * failure is pointless.
-+ * It _must_ be blockable and used together with __GFP_DIRECT_RECLAIM.
-+ * It should _never_ be used in non-sleepable contexts.
-  * New users should be evaluated carefully (and the flag should be
-  * used only when there is no reasonable failure policy) but it is
-  * definitely preferable to use the flag rather than opencode endless
-  * loop around allocator.
- * Using this flag for costly allocations is _highly_ discouraged.
-+ * Allocating pages from the buddy with __GFP_NOFAIL and order > 1 is
-+ * not supported. Please consider using kvmalloc() instead.
-  */
- #define __GFP_IO	((__force gfp_t)___GFP_IO)
- #define __GFP_FS	((__force gfp_t)___GFP_FS)
-@@ -236,27 +275,26 @@ typedef unsigned int __bitwise gfp_t;
-  * is being zeroed (either via __GFP_ZERO or via init_on_alloc, provided that
-  * __GFP_SKIP_ZERO is not set). This flag is intended for optimization: setting
-  * memory tags at the same time as zeroing memory has minimal additional
- * performace impact.
- *
- * %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation.
- * Only effective in HW_TAGS mode.
-+ * performance impact.
-  *
- * %__GFP_SKIP_KASAN_POISON makes KASAN skip poisoning on page deallocation.
- * Typically, used for userspace pages. Only effective in HW_TAGS mode.
-+ * %__GFP_SKIP_KASAN makes KASAN skip unpoisoning on page allocation.
-+ * Used for userspace and vmalloc pages; the latter are unpoisoned by
-+ * kasan_unpoison_vmalloc instead. For userspace pages, results in
-+ * poisoning being skipped as well, see should_skip_kasan_poison for
-+ * details. Only effective in HW_TAGS mode.
-  */
- #define __GFP_NOWARN	((__force gfp_t)___GFP_NOWARN)
- #define __GFP_COMP	((__force gfp_t)___GFP_COMP)
- #define __GFP_ZERO	((__force gfp_t)___GFP_ZERO)
- #define __GFP_ZEROTAGS	((__force gfp_t)___GFP_ZEROTAGS)
- #define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO)
-#define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON)
-#define __GFP_SKIP_KASAN_POISON   ((__force gfp_t)___GFP_SKIP_KASAN_POISON)
-+#define __GFP_SKIP_KASAN ((__force gfp_t)___GFP_SKIP_KASAN)
- 
- /* Disable lockdep for GFP context tracking */
- #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP)
- 
- /* Room for N __GFP_FOO bits */
-#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP))
-+#define __GFP_BITS_SHIFT ___GFP_LAST_BIT
- #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
- 
- /**
-@@ -281,7 +319,8 @@ typedef unsigned int __bitwise gfp_t;
-  * accounted to kmemcg.
-  *
-  * %GFP_NOWAIT is for kernel allocations that should not stall for direct
- * reclaim, start physical IO or use any filesystem callback.
-+ * reclaim, start physical IO or use any filesystem callback.  It is very
-+ * likely to fail to allocate memory, even for very small allocations.
-  *
-  * %GFP_NOIO will use direct reclaim to discard clean pages or slab pages
-  * that do not require the starting of any physical IO.
-@@ -329,18 +368,17 @@ typedef unsigned int __bitwise gfp_t;
-  * version does not attempt reclaim/compaction at all and is by default used
-  * in page fault path, while the non-light is used by khugepaged.
-  */
-#define GFP_ATOMIC	(__GFP_HIGH|__GFP_ATOMIC|__GFP_KSWAPD_RECLAIM)
-+#define GFP_ATOMIC	(__GFP_HIGH|__GFP_KSWAPD_RECLAIM)
- #define GFP_KERNEL	(__GFP_RECLAIM | __GFP_IO | __GFP_FS)
- #define GFP_KERNEL_ACCOUNT (GFP_KERNEL | __GFP_ACCOUNT)
-#define GFP_NOWAIT	(__GFP_KSWAPD_RECLAIM)
-+#define GFP_NOWAIT	(__GFP_KSWAPD_RECLAIM | __GFP_NOWARN)
- #define GFP_NOIO	(__GFP_RECLAIM)
- #define GFP_NOFS	(__GFP_RECLAIM | __GFP_IO)
- #define GFP_USER	(__GFP_RECLAIM | __GFP_IO | __GFP_FS | __GFP_HARDWALL)
- #define GFP_DMA		__GFP_DMA
- #define GFP_DMA32	__GFP_DMA32
- #define GFP_HIGHUSER	(GFP_USER | __GFP_HIGHMEM)
-#define GFP_HIGHUSER_MOVABLE	(GFP_HIGHUSER | __GFP_MOVABLE | \
-			 __GFP_SKIP_KASAN_POISON | __GFP_SKIP_KASAN_UNPOISON)
-+#define GFP_HIGHUSER_MOVABLE	(GFP_HIGHUSER | __GFP_MOVABLE | __GFP_SKIP_KASAN)
- #define GFP_TRANSHUGE_LIGHT	((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \
- 			 __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM)
- #define GFP_TRANSHUGE	(GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM)
-diff --git a/include/linux/gpio.h b/include/linux/gpio.h
-index a370387fa406..8f85ddb26429 100644
--- a/include/linux/gpio.h
-+++ b/include/linux/gpio.h
-@@ -1,6 +1,6 @@
- /* SPDX-License-Identifier: GPL-2.0 */
- /*
- * <linux/gpio.h>
-+ * NOTE: This header *must not* be included.
-  *
-  * This is the LEGACY GPIO bulk include file, including legacy APIs. It is
-  * used for GPIO drivers still referencing the global GPIO numberspace,
-@@ -12,98 +12,90 @@
- #ifndef __LINUX_GPIO_H
- #define __LINUX_GPIO_H
- 
-#include <linux/errno.h>
-
-/* see Documentation/driver-api/gpio/legacy.rst */
-
-/* make these flag values available regardless of GPIO kconfig options */
-#define GPIOF_DIR_OUT	(0 << 0)
-#define GPIOF_DIR_IN	(1 << 0)
-
-#define GPIOF_INIT_LOW	(0 << 1)
-#define GPIOF_INIT_HIGH	(1 << 1)
-
-#define GPIOF_IN		(GPIOF_DIR_IN)
-#define GPIOF_OUT_INIT_LOW	(GPIOF_DIR_OUT | GPIOF_INIT_LOW)
-#define GPIOF_OUT_INIT_HIGH	(GPIOF_DIR_OUT | GPIOF_INIT_HIGH)
-
-/* Gpio pin is active-low */
-#define GPIOF_ACTIVE_LOW        (1 << 2)
-
-/* Gpio pin is open drain */
-#define GPIOF_OPEN_DRAIN	(1 << 3)
-+#include <linux/types.h>
-+#ifdef CONFIG_GPIOLIB
-+#include <linux/gpio/consumer.h>
-+#endif
- 
-/* Gpio pin is open source */
-#define GPIOF_OPEN_SOURCE	(1 << 4)
-+#ifdef CONFIG_GPIOLIB_LEGACY
- 
-#define GPIOF_EXPORT		(1 << 5)
-#define GPIOF_EXPORT_CHANGEABLE	(1 << 6)
-#define GPIOF_EXPORT_DIR_FIXED	(GPIOF_EXPORT)
-#define GPIOF_EXPORT_DIR_CHANGEABLE (GPIOF_EXPORT | GPIOF_EXPORT_CHANGEABLE)
-+struct device;
- 
-/**
- * struct gpio - a structure describing a GPIO with configuration
- * @gpio:	the GPIO number
- * @flags:	GPIO configuration as specified by GPIOF_*
- * @label:	a literal description string of this GPIO
- */
-struct gpio {
-	unsigned	gpio;
-	unsigned long	flags;
-	const char	*label;
-};
-+/* make these flag values available regardless of GPIO kconfig options */
-+#define GPIOF_IN		((1 << 0))
-+#define GPIOF_OUT_INIT_LOW	((0 << 0) | (0 << 1))
-+#define GPIOF_OUT_INIT_HIGH	((0 << 0) | (1 << 1))
- 
- #ifdef CONFIG_GPIOLIB
-+/*
-+ * "valid" GPIO numbers are nonnegative and may be passed to
-+ * setup routines like gpio_request().  Only some valid numbers
-+ * can successfully be requested and used.
-+ *
-+ * Invalid GPIO numbers are useful for indicating no-such-GPIO in
-+ * platform data and other tables.
-+ */
-+static inline bool gpio_is_valid(int number)
-+{
-+	/* only non-negative numbers are valid */
-+	return number >= 0;
-+}
- 
-#ifdef CONFIG_ARCH_HAVE_CUSTOM_GPIO_H
-#include <asm/gpio.h>
-#else
-+/*
-+ * Platforms may implement their GPIO interface with library code,
-+ * at a small performance cost for non-inlined operations and some
-+ * extra memory (for code and for per-GPIO table entries).
-+ */
- 
-#include <asm-generic/gpio.h>
-+/* Always use the library code for GPIO management calls,
-+ * or when sleeping may be involved.
-+ */
-+int gpio_request(unsigned gpio, const char *label);
-+void gpio_free(unsigned gpio);
- 
-static inline int gpio_get_value(unsigned int gpio)
-+static inline int gpio_direction_input(unsigned gpio)
- {
-	return __gpio_get_value(gpio);
-+	return gpiod_direction_input(gpio_to_desc(gpio));
- }
-
-static inline void gpio_set_value(unsigned int gpio, int value)
-+static inline int gpio_direction_output(unsigned gpio, int value)
- {
-	__gpio_set_value(gpio, value);
-+	return gpiod_direction_output_raw(gpio_to_desc(gpio), value);
- }
- 
-static inline int gpio_cansleep(unsigned int gpio)
-+static inline int gpio_get_value_cansleep(unsigned gpio)
- {
-	return __gpio_cansleep(gpio);
-+	return gpiod_get_raw_value_cansleep(gpio_to_desc(gpio));
- }
-
-static inline int gpio_to_irq(unsigned int gpio)
-+static inline void gpio_set_value_cansleep(unsigned gpio, int value)
- {
-	return __gpio_to_irq(gpio);
-+	gpiod_set_raw_value_cansleep(gpio_to_desc(gpio), value);
- }
- 
-static inline int irq_to_gpio(unsigned int irq)
-+static inline int gpio_get_value(unsigned gpio)
- {
-	return -EINVAL;
-+	return gpiod_get_raw_value(gpio_to_desc(gpio));
-+}
-+static inline void gpio_set_value(unsigned gpio, int value)
-+{
-+	gpiod_set_raw_value(gpio_to_desc(gpio), value);
- }
- 
-#endif /* ! CONFIG_ARCH_HAVE_CUSTOM_GPIO_H */
-
-/* CONFIG_GPIOLIB: bindings for managed devices that want to request gpios */
-+static inline int gpio_to_irq(unsigned gpio)
-+{
-+	return gpiod_to_irq(gpio_to_desc(gpio));
-+}
- 
-struct device;
-+int gpio_request_one(unsigned gpio, unsigned long flags, const char *label);
- 
-int devm_gpio_request(struct device *dev, unsigned gpio, const char *label);
- int devm_gpio_request_one(struct device *dev, unsigned gpio,
- 			  unsigned long flags, const char *label);
- 
- #else /* ! CONFIG_GPIOLIB */
- 
- #include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/bug.h>
- 
-struct device;
-struct gpio_chip;
-+#include <asm/bug.h>
-+#include <asm/errno.h>
- 
- static inline bool gpio_is_valid(int number)
- {
-@@ -121,11 +113,6 @@ static inline int gpio_request_one(unsigned gpio,
- 	return -ENOSYS;
- }
- 
-static inline int gpio_request_array(const struct gpio *array, size_t num)
-{
-	return -ENOSYS;
-}
-
- static inline void gpio_free(unsigned gpio)
- {
- 	might_sleep();
-@@ -134,14 +121,6 @@ static inline void gpio_free(unsigned gpio)
- 	WARN_ON(1);
- }
- 
-static inline void gpio_free_array(const struct gpio *array, size_t num)
-{
-	might_sleep();
-
-	/* GPIO can never have been requested */
-	WARN_ON(1);
-}
-
- static inline int gpio_direction_input(unsigned gpio)
- {
- 	return -ENOSYS;
-@@ -152,11 +131,6 @@ static inline int gpio_direction_output(unsigned gpio, int value)
- 	return -ENOSYS;
- }
- 
-static inline int gpio_set_debounce(unsigned gpio, unsigned debounce)
-{
-	return -ENOSYS;
-}
-
- static inline int gpio_get_value(unsigned gpio)
- {
- 	/* GPIO can never have been requested or set as {in,out}put */
-@@ -170,13 +144,6 @@ static inline void gpio_set_value(unsigned gpio, int value)
- 	WARN_ON(1);
- }
- 
-static inline int gpio_cansleep(unsigned gpio)
-{
-	/* GPIO can never have been requested or set as {in,out}put */
-	WARN_ON(1);
-	return 0;
-}
-
- static inline int gpio_get_value_cansleep(unsigned gpio)
- {
- 	/* GPIO can never have been requested or set as {in,out}put */
-@@ -190,27 +157,6 @@ static inline void gpio_set_value_cansleep(unsigned gpio, int value)
- 	WARN_ON(1);
- }
- 
-static inline int gpio_export(unsigned gpio, bool direction_may_change)
-{
-	/* GPIO can never have been requested or set as {in,out}put */
-	WARN_ON(1);
-	return -EINVAL;
-}
-
-static inline int gpio_export_link(struct device *dev, const char *name,
-				unsigned gpio)
-{
-	/* GPIO can never have been exported */
-	WARN_ON(1);
-	return -EINVAL;
-}
-
-static inline void gpio_unexport(unsigned gpio)
-{
-	/* GPIO can never have been exported */
-	WARN_ON(1);
-}
-
- static inline int gpio_to_irq(unsigned gpio)
- {
- 	/* GPIO can never have been requested or set as input */
-@@ -218,20 +164,6 @@ static inline int gpio_to_irq(unsigned gpio)
- 	return -EINVAL;
- }
- 
-static inline int irq_to_gpio(unsigned irq)
-{
-	/* irq can never have been returned from gpio_to_irq() */
-	WARN_ON(1);
-	return -EINVAL;
-}
-
-static inline int devm_gpio_request(struct device *dev, unsigned gpio,
-				    const char *label)
-{
-	WARN_ON(1);
-	return -EINVAL;
-}
-
- static inline int devm_gpio_request_one(struct device *dev, unsigned gpio,
- 					unsigned long flags, const char *label)
- {
-@@ -240,5 +172,5 @@ static inline int devm_gpio_request_one(struct device *dev, unsigned gpio,
- }
- 
- #endif /* ! CONFIG_GPIOLIB */
-
-+#endif /* CONFIG_GPIOLIB_LEGACY */
- #endif /* __LINUX_GPIO_H */
-diff --git a/include/linux/gpio/aspeed.h b/include/linux/gpio/aspeed.h
-index 1bfb3cdc86d0..9a547e66c8c4 100644
--- a/include/linux/gpio/aspeed.h
-+++ b/include/linux/gpio/aspeed.h
-@@ -1,6 +1,10 @@
- #ifndef __GPIO_ASPEED_H
- #define __GPIO_ASPEED_H
- 
-+#include <linux/types.h>
-+
-+struct gpio_desc;
-+
- struct aspeed_gpio_copro_ops {
- 	int (*request_access)(void *data);
- 	int (*release_access)(void *data);
-diff --git a/include/linux/gpio/consumer.h b/include/linux/gpio/consumer.h
-index fe0f460d9a3b..cafeb7a40ad1 100644
--- a/include/linux/gpio/consumer.h
-+++ b/include/linux/gpio/consumer.h
-@@ -3,13 +3,15 @@
- #define __LINUX_GPIO_CONSUMER_H
- 
- #include <linux/bits.h>
-#include <linux/bug.h>
-#include <linux/compiler_types.h>
- #include <linux/err.h>
-+#include <linux/types.h>
- 
-+struct acpi_device;
- struct device;
-struct gpio_desc;
-+struct fwnode_handle;
-+
- struct gpio_array;
-+struct gpio_desc;
- 
- /**
-  * struct gpio_descs - Struct containing an array of descriptors that can be
-@@ -29,6 +31,7 @@ struct gpio_descs {
- #define GPIOD_FLAGS_BIT_DIR_OUT		BIT(1)
- #define GPIOD_FLAGS_BIT_DIR_VAL		BIT(2)
- #define GPIOD_FLAGS_BIT_OPEN_DRAIN	BIT(3)
-+/* GPIOD_FLAGS_BIT_NONEXCLUSIVE is DEPRECATED, don't use in new code. */
- #define GPIOD_FLAGS_BIT_NONEXCLUSIVE	BIT(4)
- 
- /**
-@@ -109,8 +112,6 @@ int gpiod_get_direction(struct gpio_desc *desc);
- int gpiod_direction_input(struct gpio_desc *desc);
- int gpiod_direction_output(struct gpio_desc *desc, int value);
- int gpiod_direction_output_raw(struct gpio_desc *desc, int value);
-int gpiod_enable_hw_timestamp_ns(struct gpio_desc *desc, unsigned long flags);
-int gpiod_disable_hw_timestamp_ns(struct gpio_desc *desc, unsigned long flags);
- 
- /* Value get/set from non-sleeping context */
- int gpiod_get_value(const struct gpio_desc *desc);
-@@ -118,7 +119,7 @@ int gpiod_get_array_value(unsigned int array_size,
- 			  struct gpio_desc **desc_array,
- 			  struct gpio_array *array_info,
- 			  unsigned long *value_bitmap);
-void gpiod_set_value(struct gpio_desc *desc, int value);
-+int gpiod_set_value(struct gpio_desc *desc, int value);
- int gpiod_set_array_value(unsigned int array_size,
- 			  struct gpio_desc **desc_array,
- 			  struct gpio_array *array_info,
-@@ -128,7 +129,7 @@ int gpiod_get_raw_array_value(unsigned int array_size,
- 			      struct gpio_desc **desc_array,
- 			      struct gpio_array *array_info,
- 			      unsigned long *value_bitmap);
-void gpiod_set_raw_value(struct gpio_desc *desc, int value);
-+int gpiod_set_raw_value(struct gpio_desc *desc, int value);
- int gpiod_set_raw_array_value(unsigned int array_size,
- 			      struct gpio_desc **desc_array,
- 			      struct gpio_array *array_info,
-@@ -140,7 +141,7 @@ int gpiod_get_array_value_cansleep(unsigned int array_size,
- 				   struct gpio_desc **desc_array,
- 				   struct gpio_array *array_info,
- 				   unsigned long *value_bitmap);
-void gpiod_set_value_cansleep(struct gpio_desc *desc, int value);
-+int gpiod_set_value_cansleep(struct gpio_desc *desc, int value);
- int gpiod_set_array_value_cansleep(unsigned int array_size,
- 				   struct gpio_desc **desc_array,
- 				   struct gpio_array *array_info,
-@@ -150,7 +151,7 @@ int gpiod_get_raw_array_value_cansleep(unsigned int array_size,
- 				       struct gpio_desc **desc_array,
- 				       struct gpio_array *array_info,
- 				       unsigned long *value_bitmap);
-void gpiod_set_raw_value_cansleep(struct gpio_desc *desc, int value);
-+int gpiod_set_raw_value_cansleep(struct gpio_desc *desc, int value);
- int gpiod_set_raw_array_value_cansleep(unsigned int array_size,
- 				       struct gpio_desc **desc_array,
- 				       struct gpio_array *array_info,
-@@ -158,7 +159,6 @@ int gpiod_set_raw_array_value_cansleep(unsigned int array_size,
- 
- int gpiod_set_config(struct gpio_desc *desc, unsigned long config);
- int gpiod_set_debounce(struct gpio_desc *desc, unsigned int debounce);
-int gpiod_set_transitory(struct gpio_desc *desc, bool transitory);
- void gpiod_toggle_active_low(struct gpio_desc *desc);
- 
- int gpiod_is_active_low(const struct gpio_desc *desc);
-@@ -167,17 +167,14 @@ int gpiod_cansleep(const struct gpio_desc *desc);
- int gpiod_to_irq(const struct gpio_desc *desc);
- int gpiod_set_consumer_name(struct gpio_desc *desc, const char *name);
- 
-+bool gpiod_is_shared(const struct gpio_desc *desc);
-+
- /* Convert between the old gpio_ and new gpiod_ interfaces */
- struct gpio_desc *gpio_to_desc(unsigned gpio);
- int desc_to_gpio(const struct gpio_desc *desc);
- 
-/* Child properties interface */
-struct fwnode_handle;
-+int gpiod_hwgpio(const struct gpio_desc *desc);
- 
-struct gpio_desc *fwnode_get_named_gpiod(struct fwnode_handle *fwnode,
-					 const char *propname, int index,
-					 enum gpiod_flags dflags,
-					 const char *label);
- struct gpio_desc *fwnode_gpiod_get_index(struct fwnode_handle *fwnode,
- 					 const char *con_id, int index,
- 					 enum gpiod_flags flags,
-@@ -188,8 +185,12 @@ struct gpio_desc *devm_fwnode_gpiod_get_index(struct device *dev,
- 					      enum gpiod_flags flags,
- 					      const char *label);
- 
-+bool gpiod_is_equal(const struct gpio_desc *desc,
-+		    const struct gpio_desc *other);
-+
- #else /* CONFIG_GPIOLIB */
- 
-+#include <linux/bug.h>
- #include <linux/kernel.h>
- 
- static inline int gpiod_count(struct device *dev, const char *con_id)
-@@ -352,18 +353,6 @@ static inline int gpiod_direction_output_raw(struct gpio_desc *desc, int value)
- 	WARN_ON(desc);
- 	return -ENOSYS;
- }
-static inline int gpiod_enable_hw_timestamp_ns(struct gpio_desc *desc,
-					       unsigned long flags)
-{
-	WARN_ON(desc);
-	return -ENOSYS;
-}
-static inline int gpiod_disable_hw_timestamp_ns(struct gpio_desc *desc,
-						unsigned long flags)
-{
-	WARN_ON(desc);
-	return -ENOSYS;
-}
- static inline int gpiod_get_value(const struct gpio_desc *desc)
- {
- 	/* GPIO can never have been requested */
-@@ -379,10 +368,11 @@ static inline int gpiod_get_array_value(unsigned int array_size,
- 	WARN_ON(desc_array);
- 	return 0;
- }
-static inline void gpiod_set_value(struct gpio_desc *desc, int value)
-+static inline int gpiod_set_value(struct gpio_desc *desc, int value)
- {
- 	/* GPIO can never have been requested */
- 	WARN_ON(desc);
-+	return 0;
- }
- static inline int gpiod_set_array_value(unsigned int array_size,
- 					struct gpio_desc **desc_array,
-@@ -408,10 +398,11 @@ static inline int gpiod_get_raw_array_value(unsigned int array_size,
- 	WARN_ON(desc_array);
- 	return 0;
- }
-static inline void gpiod_set_raw_value(struct gpio_desc *desc, int value)
-+static inline int gpiod_set_raw_value(struct gpio_desc *desc, int value)
- {
- 	/* GPIO can never have been requested */
- 	WARN_ON(desc);
-+	return 0;
- }
- static inline int gpiod_set_raw_array_value(unsigned int array_size,
- 					    struct gpio_desc **desc_array,
-@@ -438,10 +429,11 @@ static inline int gpiod_get_array_value_cansleep(unsigned int array_size,
- 	WARN_ON(desc_array);
- 	return 0;
- }
-static inline void gpiod_set_value_cansleep(struct gpio_desc *desc, int value)
-+static inline int gpiod_set_value_cansleep(struct gpio_desc *desc, int value)
- {
- 	/* GPIO can never have been requested */
- 	WARN_ON(desc);
-+	return 0;
- }
- static inline int gpiod_set_array_value_cansleep(unsigned int array_size,
- 					    struct gpio_desc **desc_array,
-@@ -467,11 +459,12 @@ static inline int gpiod_get_raw_array_value_cansleep(unsigned int array_size,
- 	WARN_ON(desc_array);
- 	return 0;
- }
-static inline void gpiod_set_raw_value_cansleep(struct gpio_desc *desc,
-						int value)
-+static inline int gpiod_set_raw_value_cansleep(struct gpio_desc *desc,
-+					       int value)
- {
- 	/* GPIO can never have been requested */
- 	WARN_ON(desc);
-+	return 0;
- }
- static inline int gpiod_set_raw_array_value_cansleep(unsigned int array_size,
- 						struct gpio_desc **desc_array,
-@@ -497,13 +490,6 @@ static inline int gpiod_set_debounce(struct gpio_desc *desc, unsigned int deboun
- 	return -ENOSYS;
- }
- 
-static inline int gpiod_set_transitory(struct gpio_desc *desc, bool transitory)
-{
-	/* GPIO can never have been requested */
-	WARN_ON(desc);
-	return -ENOSYS;
-}
-
- static inline void gpiod_toggle_active_low(struct gpio_desc *desc)
- {
- 	/* GPIO can never have been requested */
-@@ -538,6 +524,13 @@ static inline int gpiod_set_consumer_name(struct gpio_desc *desc,
- 	return -EINVAL;
- }
- 
-+static inline bool gpiod_is_shared(const struct gpio_desc *desc)
-+{
-+	/* GPIO can never have been requested */
-+	WARN_ON(desc);
-+	return false;
-+}
-+
- static inline struct gpio_desc *gpio_to_desc(unsigned gpio)
- {
- 	return NULL;
-@@ -550,18 +543,6 @@ static inline int desc_to_gpio(const struct gpio_desc *desc)
- 	return -EINVAL;
- }
- 
-/* Child properties interface */
-struct fwnode_handle;
-
-static inline
-struct gpio_desc *fwnode_get_named_gpiod(struct fwnode_handle *fwnode,
-					 const char *propname, int index,
-					 enum gpiod_flags dflags,
-					 const char *label)
-{
-	return ERR_PTR(-ENOSYS);
-}
-
- static inline
- struct gpio_desc *fwnode_gpiod_get_index(struct fwnode_handle *fwnode,
- 					 const char *con_id, int index,
-@@ -581,91 +562,54 @@ struct gpio_desc *devm_fwnode_gpiod_get_index(struct device *dev,
- 	return ERR_PTR(-ENOSYS);
- }
- 
-#endif /* CONFIG_GPIOLIB */
-
-static inline
-struct gpio_desc *devm_fwnode_gpiod_get(struct device *dev,
-					struct fwnode_handle *fwnode,
-					const char *con_id,
-					enum gpiod_flags flags,
-					const char *label)
-+static inline bool
-+gpiod_is_equal(const struct gpio_desc *desc, const struct gpio_desc *other)
- {
-	return devm_fwnode_gpiod_get_index(dev, fwnode, con_id, 0,
-					   flags, label);
-}
-
-static inline
-struct gpio_desc *devm_fwnode_get_index_gpiod_from_child(struct device *dev,
-						const char *con_id, int index,
-						struct fwnode_handle *child,
-						enum gpiod_flags flags,
-						const char *label)
-{
-	return devm_fwnode_gpiod_get_index(dev, child, con_id, index,
-					   flags, label);
-}
-
-static inline
-struct gpio_desc *devm_fwnode_get_gpiod_from_child(struct device *dev,
-						   const char *con_id,
-						   struct fwnode_handle *child,
-						   enum gpiod_flags flags,
-						   const char *label)
-{
-	return devm_fwnode_gpiod_get_index(dev, child, con_id, 0, flags, label);
-+	WARN_ON(desc || other);
-+	return false;
- }
- 
-#if IS_ENABLED(CONFIG_GPIOLIB) && IS_ENABLED(CONFIG_OF_GPIO)
-struct device_node;
-
-struct gpio_desc *gpiod_get_from_of_node(const struct device_node *node,
-					 const char *propname, int index,
-					 enum gpiod_flags dflags,
-					 const char *label);
-+#endif /* CONFIG_GPIOLIB */
- 
-#else  /* CONFIG_GPIOLIB && CONFIG_OF_GPIO */
-+#if IS_ENABLED(CONFIG_GPIOLIB) && IS_ENABLED(CONFIG_HTE)
-+int gpiod_enable_hw_timestamp_ns(struct gpio_desc *desc, unsigned long flags);
-+int gpiod_disable_hw_timestamp_ns(struct gpio_desc *desc, unsigned long flags);
-+#else
- 
-struct device_node;
-+#include <linux/bug.h>
- 
-static inline
-struct gpio_desc *gpiod_get_from_of_node(const struct device_node *node,
-					 const char *propname, int index,
-					 enum gpiod_flags dflags,
-					 const char *label)
-+static inline int gpiod_enable_hw_timestamp_ns(struct gpio_desc *desc,
-+					       unsigned long flags)
- {
-	return ERR_PTR(-ENOSYS);
-}
-
-#endif /* CONFIG_GPIOLIB && CONFIG_OF_GPIO */
-
-#ifdef CONFIG_GPIOLIB
-struct device_node;
-+	if (!IS_ENABLED(CONFIG_GPIOLIB))
-+		WARN_ON(desc);
- 
-struct gpio_desc *devm_gpiod_get_from_of_node(struct device *dev,
-					      const struct device_node *node,
-					      const char *propname, int index,
-					      enum gpiod_flags dflags,
-					      const char *label);
-
-#else  /* CONFIG_GPIOLIB */
-+	return -ENOSYS;
-+}
-+static inline int gpiod_disable_hw_timestamp_ns(struct gpio_desc *desc,
-+						unsigned long flags)
-+{
-+	if (!IS_ENABLED(CONFIG_GPIOLIB))
-+		WARN_ON(desc);
- 
-struct device_node;
-+	return -ENOSYS;
-+}
-+#endif /* CONFIG_GPIOLIB && CONFIG_HTE */
- 
- static inline
-struct gpio_desc *devm_gpiod_get_from_of_node(struct device *dev,
-					      const struct device_node *node,
-					      const char *propname, int index,
-					      enum gpiod_flags dflags,
-					      const char *label)
-+struct gpio_desc *devm_fwnode_gpiod_get(struct device *dev,
-+					struct fwnode_handle *fwnode,
-+					const char *con_id,
-+					enum gpiod_flags flags,
-+					const char *label)
- {
-	return ERR_PTR(-ENOSYS);
-+	return devm_fwnode_gpiod_get_index(dev, fwnode, con_id, 0,
-+					   flags, label);
- }
- 
-#endif /* CONFIG_GPIOLIB */
-
- struct acpi_gpio_params {
- 	unsigned int crs_entry_index;
-	unsigned int line_index;
-+	unsigned short line_index;
- 	bool active_low;
- };
- 
-@@ -689,8 +633,6 @@ struct acpi_gpio_mapping {
- 	unsigned int quirks;
- };
- 
-struct acpi_device;
-
- #if IS_ENABLED(CONFIG_GPIOLIB) && IS_ENABLED(CONFIG_ACPI)
- 
- int acpi_dev_add_driver_gpios(struct acpi_device *adev,
-@@ -700,8 +642,6 @@ void acpi_dev_remove_driver_gpios(struct acpi_device *adev);
- int devm_acpi_dev_add_driver_gpios(struct device *dev,
- 				   const struct acpi_gpio_mapping *gpios);
- 
-struct gpio_desc *acpi_get_and_request_gpiod(char *path, unsigned int pin, char *label);
-
- #else  /* CONFIG_GPIOLIB && CONFIG_ACPI */
- 
- static inline int acpi_dev_add_driver_gpios(struct acpi_device *adev,
-@@ -717,12 +657,6 @@ static inline int devm_acpi_dev_add_driver_gpios(struct device *dev,
- 	return -ENXIO;
- }
- 
-static inline struct gpio_desc *acpi_get_and_request_gpiod(char *path, unsigned int pin,
-							   char *label)
-{
-	return ERR_PTR(-ENOSYS);
-}
-
- #endif /* CONFIG_GPIOLIB && CONFIG_ACPI */
- 
- 
-@@ -753,4 +687,14 @@ static inline void gpiod_unexport(struct gpio_desc *desc)
- 
- #endif /* CONFIG_GPIOLIB && CONFIG_GPIO_SYSFS */
- 
-+static inline int gpiod_multi_set_value_cansleep(struct gpio_descs *descs,
-+						 unsigned long *value_bitmap)
-+{
-+	if (IS_ERR_OR_NULL(descs))
-+		return PTR_ERR_OR_ZERO(descs);
-+
-+	return gpiod_set_array_value_cansleep(descs->ndescs, descs->desc,
-+					      descs->info, value_bitmap);
-+}
-+
- #endif
-diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h
-index 6aeea1071b1b..fabe2baf7b50 100644
--- a/include/linux/gpio/driver.h
-+++ b/include/linux/gpio/driver.h
-@@ -2,32 +2,42 @@
- #ifndef __LINUX_GPIO_DRIVER_H
- #define __LINUX_GPIO_DRIVER_H
- 
-#include <linux/device.h>
-#include <linux/irq.h>
-+#include <linux/bits.h>
-+#include <linux/cleanup.h>
-+#include <linux/err.h>
- #include <linux/irqchip/chained_irq.h>
- #include <linux/irqdomain.h>
-+#include <linux/irqhandler.h>
- #include <linux/lockdep.h>
-#include <linux/pinctrl/pinctrl.h>
- #include <linux/pinctrl/pinconf-generic.h>
-+#include <linux/pinctrl/pinctrl.h>
- #include <linux/property.h>
-+#include <linux/spinlock_types.h>
- #include <linux/types.h>
-+#include <linux/util_macros.h>
- 
-+#ifdef CONFIG_GENERIC_MSI_IRQ
- #include <asm/msi.h>
-+#endif
- 
-struct gpio_desc;
-+struct device;
-+struct irq_chip;
-+struct irq_data;
-+struct module;
- struct of_phandle_args;
-struct device_node;
-+struct pinctrl_dev;
- struct seq_file;
-struct gpio_device;
-struct module;
-enum gpiod_flags;
-enum gpio_lookup_flags;
- 
- struct gpio_chip;
-+struct gpio_desc;
-+struct gpio_device;
-+
-+enum gpio_lookup_flags;
-+enum gpiod_flags;
- 
- union gpio_irq_fwspec {
- 	struct irq_fwspec	fwspec;
-#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
-+#ifdef CONFIG_GENERIC_MSI_IRQ
- 	msi_alloc_info_t	msiinfo;
- #endif
- };
-@@ -54,13 +64,6 @@ struct gpio_irq_chip {
- 	 */
- 	struct irq_domain *domain;
- 
-	/**
-	 * @domain_ops:
-	 *
-	 * Table of interrupt domain operations for this IRQ chip.
-	 */
-	const struct irq_domain_ops *domain_ops;
-
- #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
- 	/**
- 	 * @fwnode:
-@@ -245,6 +248,14 @@ struct gpio_irq_chip {
- 	bool initialized;
- 
- 	/**
-+	 * @domain_is_allocated_externally:
-+	 *
-+	 * True it the irq_domain was allocated outside of gpiolib, in which
-+	 * case gpiolib won't free the irq_domain itself.
-+	 */
-+	bool domain_is_allocated_externally;
-+
-+	/**
- 	 * @init_hw: optional routine to initialize hardware before
- 	 * an IRQ chip will be added. This is quite useful when
- 	 * a particular driver wants to clear IRQ related registers
-@@ -276,8 +287,9 @@ struct gpio_irq_chip {
- 	/**
- 	 * @first:
- 	 *
-	 * Required for static IRQ allocation. If set, irq_domain_add_simple()
-	 * will allocate and map all IRQs during initialization.
-+	 * Required for static IRQ allocation. If set,
-+	 * irq_domain_create_simple() will allocate and map all IRQs
-+	 * during initialization.
- 	 */
- 	unsigned int first;
- 
-@@ -318,25 +330,31 @@ struct gpio_irq_chip {
-  * @fwnode: optional fwnode providing this controller's properties
-  * @owner: helps prevent removal of modules exporting active GPIOs
-  * @request: optional hook for chip-specific activation, such as
- *	enabling module power and clock; may sleep
-+ *	enabling module power and clock; may sleep; must return 0 on success
-+ *	or negative error number on failure
-  * @free: optional hook for chip-specific deactivation, such as
-  *	disabling module power and clock; may sleep
-  * @get_direction: returns direction for signal "offset", 0=out, 1=in,
-  *	(same as GPIO_LINE_DIRECTION_OUT / GPIO_LINE_DIRECTION_IN),
-  *	or negative error. It is recommended to always implement this
-  *	function, even on input-only or output-only gpio chips.
- * @direction_input: configures signal "offset" as input, or returns error
- *	This can be omitted on input-only or output-only gpio chips.
- * @direction_output: configures signal "offset" as output, or returns error
- *	This can be omitted on input-only or output-only gpio chips.
-+ * @direction_input: configures signal "offset" as input, returns 0 on success
-+ *	or a negative error number. This can be omitted on input-only or
-+ *	output-only gpio chips.
-+ * @direction_output: configures signal "offset" as output, returns 0 on
-+ *	success or a negative error number. This can be omitted on input-only
-+ *	or output-only gpio chips.
-  * @get: returns value for signal "offset", 0=low, 1=high, or negative error
-  * @get_multiple: reads values for multiple signals defined by "mask" and
-  *	stores them in "bits", returns 0 on success or negative error
- * @set: assigns output value for signal "offset"
- * @set_multiple: assigns output values for multiple signals defined by "mask"
-+ * @set: assigns output value for signal "offset", returns 0 on success or
-+ *       negative error value
-+ * @set_multiple: assigns output values for multiple signals defined by
-+ *                "mask", returns 0 on success or negative error value
-  * @set_config: optional hook for all kinds of settings. Uses the same
- *	packed config format as generic pinconf.
- * @to_irq: optional hook supporting non-static gpio_to_irq() mappings;
-+ *	packed config format as generic pinconf. Must return 0 on success and
-+ *	a negative error number on failure.
-+ * @to_irq: optional hook supporting non-static gpiod_to_irq() mappings;
-  *	implementation may not sleep
-  * @dbg_show: optional routine to show contents in debugfs; default code
-  *	will be used when this is omitted, but custom code can show extra
-@@ -364,35 +382,12 @@ struct gpio_irq_chip {
-  * @names: if set, must be an array of strings to use as alternative
-  *      names for the GPIOs in this chip. Any entry in the array
-  *      may be NULL if there is no alias for the GPIO, however the
- *      array must be @ngpio entries long.  A name can include a single printk
- *      format specifier for an unsigned int.  It is substituted by the actual
- *      number of the gpio.
-+ *      array must be @ngpio entries long.
-  * @can_sleep: flag must be set iff get()/set() methods sleep, as they
-  *	must while accessing GPIO expander chips over I2C or SPI. This
-  *	implies that if the chip supports IRQs, these IRQs need to be threaded
-  *	as the chip access may sleep when e.g. reading out the IRQ status
-  *	registers.
- * @read_reg: reader function for generic GPIO
- * @write_reg: writer function for generic GPIO
- * @be_bits: if the generic GPIO has big endian bit order (bit 31 is representing
- *	line 0, bit 30 is line 1 ... bit 0 is line 31) this is set to true by the
- *	generic GPIO core. It is for internal housekeeping only.
- * @reg_dat: data (in) register for generic GPIO
- * @reg_set: output set register (out=high) for generic GPIO
- * @reg_clr: output clear register (out=low) for generic GPIO
- * @reg_dir_out: direction out setting register for generic GPIO
- * @reg_dir_in: direction in setting register for generic GPIO
- * @bgpio_dir_unreadable: indicates that the direction register(s) cannot
- *	be read and we need to rely on out internal state tracking.
- * @bgpio_bits: number of register bits used for a generic GPIO i.e.
- *	<register width> * 8
- * @bgpio_lock: used to lock chip->bgpio_data. Also, this is needed to keep
- *	shadowed and real data registers writes together.
- * @bgpio_data:	shadowed data register for generic GPIO to clear/set bits
- *	safely.
- * @bgpio_dir: shadowed direction register for generic GPIO to clear/set
- *	direction safely. A "1" in this word means the line is set as
- *	output.
-  *
-  * A gpio_chip can help platforms abstract various sources of GPIOs so
-  * they can all be accessed through a common programming interface.
-@@ -426,9 +421,9 @@ struct gpio_chip {
- 	int			(*get_multiple)(struct gpio_chip *gc,
- 						unsigned long *mask,
- 						unsigned long *bits);
-	void			(*set)(struct gpio_chip *gc,
-						unsigned int offset, int value);
-	void			(*set_multiple)(struct gpio_chip *gc,
-+	int			(*set)(struct gpio_chip *gc,
-+				       unsigned int offset, int value);
-+	int			(*set_multiple)(struct gpio_chip *gc,
- 						unsigned long *mask,
- 						unsigned long *bits);
- 	int			(*set_config)(struct gpio_chip *gc,
-@@ -458,22 +453,6 @@ struct gpio_chip {
- 	const char		*const *names;
- 	bool			can_sleep;
- 
-#if IS_ENABLED(CONFIG_GPIO_GENERIC)
-	unsigned long (*read_reg)(void __iomem *reg);
-	void (*write_reg)(void __iomem *reg, unsigned long data);
-	bool be_bits;
-	void __iomem *reg_dat;
-	void __iomem *reg_set;
-	void __iomem *reg_clr;
-	void __iomem *reg_dir_out;
-	void __iomem *reg_dir_in;
-	bool bgpio_dir_unreadable;
-	int bgpio_bits;
-	raw_spinlock_t bgpio_lock;
-	unsigned long bgpio_data;
-	unsigned long bgpio_dir;
-#endif /* CONFIG_GPIO_GENERIC */
-
- #ifdef CONFIG_GPIOLIB_IRQCHIP
- 	/*
- 	 * With CONFIG_GPIOLIB_IRQCHIP we get an irqchip inside the gpiolib
-@@ -489,14 +468,6 @@ struct gpio_chip {
- 	struct gpio_irq_chip irq;
- #endif /* CONFIG_GPIOLIB_IRQCHIP */
- 
-	/**
-	 * @valid_mask:
-	 *
-	 * If not %NULL, holds bitmask of GPIOs which are valid to be used
-	 * from the chip.
-	 */
-	unsigned long *valid_mask;
-
- #if defined(CONFIG_OF_GPIO)
- 	/*
- 	 * If CONFIG_OF_GPIO is enabled, then all GPIO controllers described in
-@@ -504,18 +475,33 @@ struct gpio_chip {
- 	 */
- 
- 	/**
-	 * @of_node:
-+	 * @of_gpio_n_cells:
-+	 *
-+	 * Number of cells used to form the GPIO specifier. The standard is 2
-+	 * cells:
-+	 *
-+	 * gpios = <&gpio offset flags>;
- 	 *
-	 * Pointer to a device tree node representing this GPIO controller.
-+	 * some complex GPIO controllers instantiate more than one chip per
-+	 * device tree node and have 3 cells:
-+	 *
-+	 * gpios = <&gpio instance offset flags>;
-+	 *
-+	 * Legacy GPIO controllers may even have 1 cell:
-+	 *
-+	 * gpios = <&gpio offset>;
- 	 */
-	struct device_node *of_node;
-+	unsigned int of_gpio_n_cells;
- 
- 	/**
-	 * @of_gpio_n_cells:
-+	 * @of_node_instance_match:
- 	 *
-	 * Number of cells used to form the GPIO specifier.
-+	 * Determine if a chip is the right instance. Must be implemented by
-+	 * any driver using more than one gpio_chip per device tree node.
-+	 * Returns true if gc is the instance indicated by i (which is the
-+	 * first cell in the phandles for GPIO lines and gpio-ranges).
- 	 */
-	unsigned int of_gpio_n_cells;
-+	bool (*of_node_instance_match)(struct gpio_chip *gc, unsigned int i);
- 
- 	/**
- 	 * @of_xlate:
-@@ -525,44 +511,72 @@ struct gpio_chip {
- 	 */
- 	int (*of_xlate)(struct gpio_chip *gc,
- 			const struct of_phandle_args *gpiospec, u32 *flags);
-+#endif /* CONFIG_OF_GPIO */
-+};
- 
-	/**
-	 * @of_gpio_ranges_fallback:
-	 *
-	 * Optional hook for the case that no gpio-ranges property is defined
-	 * within the device tree node "np" (usually DT before introduction
-	 * of gpio-ranges). So this callback is helpful to provide the
-	 * necessary backward compatibility for the pin ranges.
-	 */
-	int (*of_gpio_ranges_fallback)(struct gpio_chip *gc,
-				       struct device_node *np);
-+char *gpiochip_dup_line_label(struct gpio_chip *gc, unsigned int offset);
- 
-#endif /* CONFIG_OF_GPIO */
-+
-+struct _gpiochip_for_each_data {
-+	const char **label;
-+	unsigned int *i;
- };
- 
-extern const char *gpiochip_is_requested(struct gpio_chip *gc,
-			unsigned int offset);
-+DEFINE_CLASS(_gpiochip_for_each_data,
-+	     struct _gpiochip_for_each_data,
-+	     if (*_T.label) kfree(*_T.label),
-+	     ({
-+		struct _gpiochip_for_each_data _data = { label, i };
-+		*_data.i = 0;
-+		_data;
-+	     }),
-+	     const char **label, int *i)
-+
-+/**
-+ * for_each_hwgpio_in_range - Iterates over all GPIOs in a given range
-+ * @_chip: Chip to iterate over.
-+ * @_i: Loop counter.
-+ * @_base: First GPIO in the ranger.
-+ * @_size: Amount of GPIOs to check starting from @base.
-+ * @_label: Place to store the address of the label if the GPIO is requested.
-+ *          Set to NULL for unused GPIOs.
-+ */
-+#define for_each_hwgpio_in_range(_chip, _i, _base, _size, _label)			\
-+	for (CLASS(_gpiochip_for_each_data, _data)(&_label, &_i);			\
-+	     _i < _size;								\
-+	     _i++, kfree(_label), _label = NULL)					\
-+		for_each_if(!IS_ERR(_label = gpiochip_dup_line_label(_chip, _base + _i)))
-+
-+/**
-+ * for_each_hwgpio - Iterates over all GPIOs for given chip.
-+ * @_chip: Chip to iterate over.
-+ * @_i: Loop counter.
-+ * @_label: Place to store the address of the label if the GPIO is requested.
-+ *          Set to NULL for unused GPIOs.
-+ */
-+#define for_each_hwgpio(_chip, _i, _label)						\
-+	for_each_hwgpio_in_range(_chip, _i, 0, _chip->ngpio, _label)
- 
- /**
-  * for_each_requested_gpio_in_range - iterates over requested GPIOs in a given range
- * @chip:	the chip to query
- * @i:		loop variable
- * @base:	first GPIO in the range
- * @size:	amount of GPIOs to check starting from @base
- * @label:	label of current GPIO
-+ * @_chip:	the chip to query
-+ * @_i:		loop variable
-+ * @_base:	first GPIO in the range
-+ * @_size:	amount of GPIOs to check starting from @base
-+ * @_label:	label of current GPIO
-  */
-#define for_each_requested_gpio_in_range(chip, i, base, size, label)			\
-	for (i = 0; i < size; i++)							\
-		if ((label = gpiochip_is_requested(chip, base + i)) == NULL) {} else
-+#define for_each_requested_gpio_in_range(_chip, _i, _base, _size, _label)		\
-+	for_each_hwgpio_in_range(_chip, _i, _base, _size, _label)			\
-+		for_each_if(_label)
- 
- /* Iterates over all requested GPIO of the given @chip */
- #define for_each_requested_gpio(chip, i, label)						\
- 	for_each_requested_gpio_in_range(chip, i, 0, chip->ngpio, label)
- 
- /* add/remove chips */
-extern int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
-				      struct lock_class_key *lock_key,
-				      struct lock_class_key *request_key);
-+int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
-+			       struct lock_class_key *lock_key,
-+			       struct lock_class_key *request_key);
- 
- /**
-  * gpiochip_add_data() - register a gpio_chip
-@@ -606,17 +620,22 @@ extern int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
- 	devm_gpiochip_add_data_with_key(dev, gc, data, NULL, NULL)
- #endif /* CONFIG_LOCKDEP */
- 
-static inline int gpiochip_add(struct gpio_chip *gc)
-{
-	return gpiochip_add_data(gc, NULL);
-}
-extern void gpiochip_remove(struct gpio_chip *gc);
-extern int devm_gpiochip_add_data_with_key(struct device *dev, struct gpio_chip *gc, void *data,
-					   struct lock_class_key *lock_key,
-					   struct lock_class_key *request_key);
-+void gpiochip_remove(struct gpio_chip *gc);
-+int devm_gpiochip_add_data_with_key(struct device *dev, struct gpio_chip *gc,
-+				    void *data, struct lock_class_key *lock_key,
-+				    struct lock_class_key *request_key);
-+
-+struct gpio_device *gpio_device_find(const void *data,
-+				int (*match)(struct gpio_chip *gc,
-+					     const void *data));
- 
-extern struct gpio_chip *gpiochip_find(void *data,
-			      int (*match)(struct gpio_chip *gc, void *data));
-+struct gpio_device *gpio_device_get(struct gpio_device *gdev);
-+void gpio_device_put(struct gpio_device *gdev);
-+
-+DEFINE_FREE(gpio_device_put, struct gpio_device *,
-+	    if (!IS_ERR_OR_NULL(_T)) gpio_device_put(_T))
-+
-+struct device *gpio_device_to_device(struct gpio_device *gdev);
- 
- bool gpiochip_line_is_irq(struct gpio_chip *gc, unsigned int offset);
- int gpiochip_reqres_irq(struct gpio_chip *gc, unsigned int offset);
-@@ -647,16 +666,11 @@ bool gpiochip_line_is_open_source(struct gpio_chip *gc, unsigned int offset);
- /* Sleep persistence inquiry for drivers */
- bool gpiochip_line_is_persistent(struct gpio_chip *gc, unsigned int offset);
- bool gpiochip_line_is_valid(const struct gpio_chip *gc, unsigned int offset);
-+const unsigned long *gpiochip_query_valid_mask(const struct gpio_chip *gc);
- 
- /* get driver data */
- void *gpiochip_get_data(struct gpio_chip *gc);
- 
-struct bgpio_pdata {
-	const char *label;
-	int base;
-	int ngpio;
-};
-
- #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
- 
- int gpiochip_populate_parent_fwspec_twocell(struct gpio_chip *gc,
-@@ -670,35 +684,13 @@ int gpiochip_populate_parent_fwspec_fourcell(struct gpio_chip *gc,
- 
- #endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */
- 
-int bgpio_init(struct gpio_chip *gc, struct device *dev,
-	       unsigned long sz, void __iomem *dat, void __iomem *set,
-	       void __iomem *clr, void __iomem *dirout, void __iomem *dirin,
-	       unsigned long flags);
-
-#define BGPIOF_BIG_ENDIAN		BIT(0)
-#define BGPIOF_UNREADABLE_REG_SET	BIT(1) /* reg_set is unreadable */
-#define BGPIOF_UNREADABLE_REG_DIR	BIT(2) /* reg_dir is unreadable */
-#define BGPIOF_BIG_ENDIAN_BYTE_ORDER	BIT(3)
-#define BGPIOF_READ_OUTPUT_REG_SET	BIT(4) /* reg_set stores output value */
-#define BGPIOF_NO_OUTPUT		BIT(5) /* only input */
-#define BGPIOF_NO_SET_ON_INPUT		BIT(6)
-
-int gpiochip_irq_map(struct irq_domain *d, unsigned int irq,
-		     irq_hw_number_t hwirq);
-void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq);
-
-int gpiochip_irq_domain_activate(struct irq_domain *domain,
-				 struct irq_data *data, bool reserve);
-void gpiochip_irq_domain_deactivate(struct irq_domain *domain,
-				    struct irq_data *data);
-
-bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc,
-				unsigned int offset);
-
- #ifdef CONFIG_GPIOLIB_IRQCHIP
- int gpiochip_irqchip_add_domain(struct gpio_chip *gc,
- 				struct irq_domain *domain);
- #else
-+
-+#include <asm/bug.h>
-+
- static inline int gpiochip_irqchip_add_domain(struct gpio_chip *gc,
- 					      struct irq_domain *domain)
- {
-@@ -726,23 +718,68 @@ struct gpio_pin_range {
- 
- #ifdef CONFIG_PINCTRL
- 
-int gpiochip_add_pin_range(struct gpio_chip *gc, const char *pinctl_name,
-			   unsigned int gpio_offset, unsigned int pin_offset,
-			   unsigned int npins);
-+int gpiochip_add_pin_range_with_pins(struct gpio_chip *gc,
-+				     const char *pinctl_name,
-+				     unsigned int gpio_offset,
-+				     unsigned int pin_offset,
-+				     unsigned int const *pins,
-+				     unsigned int npins);
- int gpiochip_add_pingroup_range(struct gpio_chip *gc,
- 			struct pinctrl_dev *pctldev,
- 			unsigned int gpio_offset, const char *pin_group);
- void gpiochip_remove_pin_ranges(struct gpio_chip *gc);
- 
-+static inline int
-+gpiochip_add_pin_range(struct gpio_chip *gc,
-+		       const char *pinctl_name,
-+		       unsigned int gpio_offset,
-+		       unsigned int pin_offset,
-+		       unsigned int npins)
-+{
-+	return gpiochip_add_pin_range_with_pins(gc, pinctl_name, gpio_offset,
-+						pin_offset, NULL, npins);
-+}
-+
-+static inline int
-+gpiochip_add_sparse_pin_range(struct gpio_chip *gc,
-+			      const char *pinctl_name,
-+			      unsigned int gpio_offset,
-+			      unsigned int const *pins,
-+			      unsigned int npins)
-+{
-+	return gpiochip_add_pin_range_with_pins(gc, pinctl_name, gpio_offset, 0,
-+						pins, npins);
-+}
- #else /* ! CONFIG_PINCTRL */
- 
- static inline int
-+gpiochip_add_pin_range_with_pins(struct gpio_chip *gc,
-+				 const char *pinctl_name,
-+				 unsigned int gpio_offset,
-+				 unsigned int pin_offset,
-+				 unsigned int npins)
-+{
-+	return 0;
-+}
-+
-+static inline int
- gpiochip_add_pin_range(struct gpio_chip *gc, const char *pinctl_name,
- 		       unsigned int gpio_offset, unsigned int pin_offset,
- 		       unsigned int npins)
- {
- 	return 0;
- }
-+
-+static inline int
-+gpiochip_add_sparse_pin_range(struct gpio_chip *gc,
-+			      const char *pinctl_name,
-+			      unsigned int gpio_offset,
-+			      unsigned int const *pins,
-+			      unsigned int npins)
-+{
-+	return 0;
-+}
-+
- static inline int
- gpiochip_add_pingroup_range(struct gpio_chip *gc,
- 			struct pinctrl_dev *pctldev,
-@@ -765,17 +802,31 @@ struct gpio_desc *gpiochip_request_own_desc(struct gpio_chip *gc,
- 					    enum gpiod_flags dflags);
- void gpiochip_free_own_desc(struct gpio_desc *desc);
- 
-+struct gpio_desc *
-+gpio_device_get_desc(struct gpio_device *gdev, unsigned int hwnum);
-+
-+struct gpio_chip *gpio_device_get_chip(struct gpio_device *gdev);
-+
- #ifdef CONFIG_GPIOLIB
- 
- /* lock/unlock as IRQ */
- int gpiochip_lock_as_irq(struct gpio_chip *gc, unsigned int offset);
- void gpiochip_unlock_as_irq(struct gpio_chip *gc, unsigned int offset);
- 
-
- struct gpio_chip *gpiod_to_chip(const struct gpio_desc *desc);
-+struct gpio_device *gpiod_to_gpio_device(struct gpio_desc *desc);
-+
-+/* struct gpio_device getters */
-+int gpio_device_get_base(struct gpio_device *gdev);
-+const char *gpio_device_get_label(struct gpio_device *gdev);
-+
-+struct gpio_device *gpio_device_find_by_label(const char *label);
-+struct gpio_device *gpio_device_find_by_fwnode(const struct fwnode_handle *fwnode);
- 
- #else /* CONFIG_GPIOLIB */
- 
-+#include <asm/bug.h>
-+
- static inline struct gpio_chip *gpiod_to_chip(const struct gpio_desc *desc)
- {
- 	/* GPIO can never have been requested */
-@@ -783,6 +834,36 @@ static inline struct gpio_chip *gpiod_to_chip(const struct gpio_desc *desc)
- 	return ERR_PTR(-ENODEV);
- }
- 
-+static inline struct gpio_device *gpiod_to_gpio_device(struct gpio_desc *desc)
-+{
-+	WARN_ON(1);
-+	return ERR_PTR(-ENODEV);
-+}
-+
-+static inline int gpio_device_get_base(struct gpio_device *gdev)
-+{
-+	WARN_ON(1);
-+	return -ENODEV;
-+}
-+
-+static inline const char *gpio_device_get_label(struct gpio_device *gdev)
-+{
-+	WARN_ON(1);
-+	return NULL;
-+}
-+
-+static inline struct gpio_device *gpio_device_find_by_label(const char *label)
-+{
-+	WARN_ON(1);
-+	return NULL;
-+}
-+
-+static inline struct gpio_device *gpio_device_find_by_fwnode(const struct fwnode_handle *fwnode)
-+{
-+	WARN_ON(1);
-+	return NULL;
-+}
-+
- static inline int gpiochip_lock_as_irq(struct gpio_chip *gc,
- 				       unsigned int offset)
- {
-@@ -799,7 +880,7 @@ static inline void gpiochip_unlock_as_irq(struct gpio_chip *gc,
- 
- #define for_each_gpiochip_node(dev, child)					\
- 	device_for_each_child_node(dev, child)					\
-		if (!fwnode_property_present(child, "gpio-controller")) {} else
-+		for_each_if(fwnode_property_present(child, "gpio-controller"))
- 
- static inline unsigned int gpiochip_node_count(struct device *dev)
- {
-diff --git a/include/linux/gpio/forwarder.h b/include/linux/gpio/forwarder.h
-new file mode 100644
-index 000000000000..ee5d8355f735
--- /dev/null
-+++ b/include/linux/gpio/forwarder.h
-@@ -0,0 +1,41 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef __LINUX_GPIO_FORWARDER_H
-+#define __LINUX_GPIO_FORWARDER_H
-+
-+struct gpio_desc;
-+struct gpio_chip;
-+struct gpiochip_fwd;
-+
-+struct gpiochip_fwd *devm_gpiochip_fwd_alloc(struct device *dev,
-+					     unsigned int ngpios);
-+int gpiochip_fwd_desc_add(struct gpiochip_fwd *fwd,
-+			  struct gpio_desc *desc, unsigned int offset);
-+void gpiochip_fwd_desc_free(struct gpiochip_fwd *fwd, unsigned int offset);
-+int gpiochip_fwd_register(struct gpiochip_fwd *fwd, void *data);
-+
-+struct gpio_chip *gpiochip_fwd_get_gpiochip(struct gpiochip_fwd *fwd);
-+
-+void *gpiochip_fwd_get_data(struct gpiochip_fwd *fwd);
-+
-+int gpiochip_fwd_gpio_request(struct gpiochip_fwd *fwd, unsigned int offset);
-+int gpiochip_fwd_gpio_get_direction(struct gpiochip_fwd *fwd,
-+				    unsigned int offset);
-+int gpiochip_fwd_gpio_direction_input(struct gpiochip_fwd *fwd,
-+				      unsigned int offset);
-+int gpiochip_fwd_gpio_direction_output(struct gpiochip_fwd *fwd,
-+				       unsigned int offset,
-+				       int value);
-+int gpiochip_fwd_gpio_get(struct gpiochip_fwd *fwd, unsigned int offset);
-+int gpiochip_fwd_gpio_get_multiple(struct gpiochip_fwd *fwd,
-+				   unsigned long *mask,
-+				   unsigned long *bits);
-+int gpiochip_fwd_gpio_set(struct gpiochip_fwd *fwd, unsigned int offset,
-+			  int value);
-+int gpiochip_fwd_gpio_set_multiple(struct gpiochip_fwd *fwd,
-+				   unsigned long *mask,
-+				   unsigned long *bits);
-+int gpiochip_fwd_gpio_set_config(struct gpiochip_fwd *fwd, unsigned int offset,
-+				 unsigned long config);
-+int gpiochip_fwd_gpio_to_irq(struct gpiochip_fwd *fwd, unsigned int offset);
-+
-+#endif
-diff --git a/include/linux/gpio/generic.h b/include/linux/gpio/generic.h
-new file mode 100644
-index 000000000000..ff566dc9c3cb
--- /dev/null
-+++ b/include/linux/gpio/generic.h
-@@ -0,0 +1,190 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+
-+#ifndef __LINUX_GPIO_GENERIC_H
-+#define __LINUX_GPIO_GENERIC_H
-+
-+#include <linux/cleanup.h>
-+#include <linux/gpio/driver.h>
-+#include <linux/spinlock.h>
-+
-+struct device;
-+
-+#define GPIO_GENERIC_BIG_ENDIAN			BIT(0)
-+#define GPIO_GENERIC_UNREADABLE_REG_SET		BIT(1) /* reg_set is unreadable */
-+#define GPIO_GENERIC_UNREADABLE_REG_DIR		BIT(2) /* reg_dir is unreadable */
-+#define GPIO_GENERIC_BIG_ENDIAN_BYTE_ORDER	BIT(3)
-+#define GPIO_GENERIC_READ_OUTPUT_REG_SET	BIT(4) /* reg_set stores output value */
-+#define GPIO_GENERIC_NO_OUTPUT			BIT(5) /* only input */
-+#define GPIO_GENERIC_NO_SET_ON_INPUT		BIT(6)
-+#define GPIO_GENERIC_PINCTRL_BACKEND		BIT(7) /* Call pinctrl direction setters */
-+#define GPIO_GENERIC_NO_INPUT			BIT(8) /* only output */
-+
-+/**
-+ * struct gpio_generic_chip_config - Generic GPIO chip configuration data
-+ * @dev: Parent device of the new GPIO chip (compulsory).
-+ * @sz: Size (width) of the MMIO registers in bytes, typically 1, 2 or 4.
-+ * @dat: MMIO address for the register to READ the value of the GPIO lines, it
-+ *       is expected that a 1 in the corresponding bit in this register means
-+ *       the line is asserted.
-+ * @set: MMIO address for the register to SET the value of the GPIO lines, it
-+ *       is expected that we write the line with 1 in this register to drive
-+ *       the GPIO line high.
-+ * @clr: MMIO address for the register to CLEAR the value of the GPIO lines,
-+ *       it is expected that we write the line with 1 in this register to
-+ *       drive the GPIO line low. It is allowed to leave this address as NULL,
-+ *       in that case the SET register will be assumed to also clear the GPIO
-+ *       lines, by actively writing the line with 0.
-+ * @dirout: MMIO address for the register to set the line as OUTPUT. It is
-+ *          assumed that setting a line to 1 in this register will turn that
-+ *          line into an output line. Conversely, setting the line to 0 will
-+ *          turn that line into an input.
-+ * @dirin: MMIO address for the register to set this line as INPUT. It is
-+ *         assumed that setting a line to 1 in this register will turn that
-+ *         line into an input line. Conversely, setting the line to 0 will
-+ *         turn that line into an output.
-+ * @flags: Different flags that will affect the behaviour of the device, such
-+ *         as endianness etc.
-+ */
-+struct gpio_generic_chip_config {
-+	struct device *dev;
-+	unsigned long sz;
-+	void __iomem *dat;
-+	void __iomem *set;
-+	void __iomem *clr;
-+	void __iomem *dirout;
-+	void __iomem *dirin;
-+	unsigned long flags;
-+};
-+
-+/**
-+ * struct gpio_generic_chip - Generic GPIO chip implementation.
-+ * @gc: The underlying struct gpio_chip object, implementing low-level GPIO
-+ *      chip routines.
-+ * @read_reg: reader function for generic GPIO
-+ * @write_reg: writer function for generic GPIO
-+ * @be_bits: if the generic GPIO has big endian bit order (bit 31 is
-+ *           representing line 0, bit 30 is line 1 ... bit 0 is line 31) this
-+ *           is set to true by the generic GPIO core. It is for internal
-+ *           housekeeping only.
-+ * @reg_dat: data (in) register for generic GPIO
-+ * @reg_set: output set register (out=high) for generic GPIO
-+ * @reg_clr: output clear register (out=low) for generic GPIO
-+ * @reg_dir_out: direction out setting register for generic GPIO
-+ * @reg_dir_in: direction in setting register for generic GPIO
-+ * @dir_unreadable: indicates that the direction register(s) cannot be read and
-+ *                  we need to rely on out internal state tracking.
-+ * @pinctrl: the generic GPIO uses a pin control backend.
-+ * @bits: number of register bits used for a generic GPIO
-+ *        i.e. <register width> * 8
-+ * @lock: used to lock chip->sdata. Also, this is needed to keep
-+ *        shadowed and real data registers writes together.
-+ * @sdata: shadowed data register for generic GPIO to clear/set bits safely.
-+ * @sdir: shadowed direction register for generic GPIO to clear/set direction
-+ *        safely. A "1" in this word means the line is set as output.
-+ */
-+struct gpio_generic_chip {
-+	struct gpio_chip gc;
-+	unsigned long (*read_reg)(void __iomem *reg);
-+	void (*write_reg)(void __iomem *reg, unsigned long data);
-+	bool be_bits;
-+	void __iomem *reg_dat;
-+	void __iomem *reg_set;
-+	void __iomem *reg_clr;
-+	void __iomem *reg_dir_out;
-+	void __iomem *reg_dir_in;
-+	bool dir_unreadable;
-+	bool pinctrl;
-+	int bits;
-+	raw_spinlock_t lock;
-+	unsigned long sdata;
-+	unsigned long sdir;
-+};
-+
-+static inline struct gpio_generic_chip *
-+to_gpio_generic_chip(struct gpio_chip *gc)
-+{
-+	return container_of(gc, struct gpio_generic_chip, gc);
-+}
-+
-+int gpio_generic_chip_init(struct gpio_generic_chip *chip,
-+			   const struct gpio_generic_chip_config *cfg);
-+
-+/**
-+ * gpio_generic_chip_set() - Set the GPIO line value of the generic GPIO chip.
-+ * @chip: Generic GPIO chip to use.
-+ * @offset: Hardware offset of the line to set.
-+ * @value: New GPIO line value.
-+ *
-+ * Some modules using the generic GPIO chip, need to set line values in their
-+ * direction setters but they don't have access to the gpio-mmio symbols so
-+ * they use the function pointer in struct gpio_chip directly. This is not
-+ * optimal and can lead to crashes at run-time in some instances. This wrapper
-+ * provides a safe interface for users.
-+ *
-+ * Returns: 0 on success, negative error number of failure.
-+ */
-+static inline int
-+gpio_generic_chip_set(struct gpio_generic_chip *chip, unsigned int offset,
-+		      int value)
-+{
-+	if (WARN_ON(!chip->gc.set))
-+		return -EOPNOTSUPP;
-+
-+	return chip->gc.set(&chip->gc, offset, value);
-+}
-+
-+/**
-+ * gpio_generic_read_reg() - Read a register using the underlying callback.
-+ * @chip: Generic GPIO chip to use.
-+ * @reg: Register to read.
-+ *
-+ * Returns: value read from register.
-+ */
-+static inline unsigned long
-+gpio_generic_read_reg(struct gpio_generic_chip *chip, void __iomem *reg)
-+{
-+	if (WARN_ON(!chip->read_reg))
-+		return 0;
-+
-+	return chip->read_reg(reg);
-+}
-+
-+/**
-+ * gpio_generic_write_reg() - Write a register using the underlying callback.
-+ * @chip: Generic GPIO chip to use.
-+ * @reg: Register to write to.
-+ * @val: New value to write.
-+ */
-+static inline void gpio_generic_write_reg(struct gpio_generic_chip *chip,
-+					  void __iomem *reg, unsigned long val)
-+{
-+	if (WARN_ON(!chip->write_reg))
-+		return;
-+
-+	chip->write_reg(reg, val);
-+}
-+
-+#define gpio_generic_chip_lock(gen_gc) \
-+	raw_spin_lock(&(gen_gc)->lock)
-+
-+#define gpio_generic_chip_unlock(gen_gc) \
-+	raw_spin_unlock(&(gen_gc)->lock)
-+
-+#define gpio_generic_chip_lock_irqsave(gen_gc, flags) \
-+	raw_spin_lock_irqsave(&(gen_gc)->lock, flags)
-+
-+#define gpio_generic_chip_unlock_irqrestore(gen_gc, flags) \
-+	raw_spin_unlock_irqrestore(&(gen_gc)->lock, flags)
-+
-+DEFINE_LOCK_GUARD_1(gpio_generic_lock,
-+		    struct gpio_generic_chip,
-+		    gpio_generic_chip_lock(_T->lock),
-+		    gpio_generic_chip_unlock(_T->lock))
-+
-+DEFINE_LOCK_GUARD_1(gpio_generic_lock_irqsave,
-+		    struct gpio_generic_chip,
-+		    gpio_generic_chip_lock_irqsave(_T->lock, _T->flags),
-+		    gpio_generic_chip_unlock_irqrestore(_T->lock, _T->flags),
-+		    unsigned long flags)
-+
-+#endif /* __LINUX_GPIO_GENERIC_H */
-diff --git a/include/linux/gpio/gpio-nomadik.h b/include/linux/gpio/gpio-nomadik.h
-new file mode 100644
-index 000000000000..592a774a53cd
--- /dev/null
-+++ b/include/linux/gpio/gpio-nomadik.h
-@@ -0,0 +1,292 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef __LINUX_GPIO_NOMADIK_H
-+#define __LINUX_GPIO_NOMADIK_H
-+
-+struct fwnode_handle;
-+
-+/* Package definitions */
-+#define PINCTRL_NMK_STN8815	0
-+#define PINCTRL_NMK_DB8500	1
-+
-+#define GPIO_BLOCK_SHIFT 5
-+#define NMK_GPIO_PER_CHIP BIT(GPIO_BLOCK_SHIFT)
-+#define NMK_MAX_BANKS DIV_ROUND_UP(512, NMK_GPIO_PER_CHIP)
-+
-+/* Register in the logic block */
-+#define NMK_GPIO_DAT	0x00
-+#define NMK_GPIO_DATS	0x04
-+#define NMK_GPIO_DATC	0x08
-+#define NMK_GPIO_PDIS	0x0c
-+#define NMK_GPIO_DIR	0x10
-+#define NMK_GPIO_DIRS	0x14
-+#define NMK_GPIO_DIRC	0x18
-+#define NMK_GPIO_SLPC	0x1c
-+#define NMK_GPIO_AFSLA	0x20
-+#define NMK_GPIO_AFSLB	0x24
-+#define NMK_GPIO_LOWEMI	0x28
-+
-+#define NMK_GPIO_RIMSC	0x40
-+#define NMK_GPIO_FIMSC	0x44
-+#define NMK_GPIO_IS	0x48
-+#define NMK_GPIO_IC	0x4c
-+#define NMK_GPIO_RWIMSC	0x50
-+#define NMK_GPIO_FWIMSC	0x54
-+#define NMK_GPIO_WKS	0x58
-+/* These appear in DB8540 and later ASICs */
-+#define NMK_GPIO_EDGELEVEL 0x5C
-+#define NMK_GPIO_LEVEL	0x60
-+
-+/* Pull up/down values */
-+enum nmk_gpio_pull {
-+	NMK_GPIO_PULL_NONE,
-+	NMK_GPIO_PULL_UP,
-+	NMK_GPIO_PULL_DOWN,
-+};
-+
-+/* Sleep mode */
-+enum nmk_gpio_slpm {
-+	NMK_GPIO_SLPM_INPUT,
-+	NMK_GPIO_SLPM_WAKEUP_ENABLE = NMK_GPIO_SLPM_INPUT,
-+	NMK_GPIO_SLPM_NOCHANGE,
-+	NMK_GPIO_SLPM_WAKEUP_DISABLE = NMK_GPIO_SLPM_NOCHANGE,
-+};
-+
-+struct nmk_gpio_chip {
-+	struct gpio_chip chip;
-+	void __iomem *addr;
-+	struct clk *clk;
-+	unsigned int bank;
-+	void (*set_ioforce)(bool enable);
-+	spinlock_t lock;
-+	bool sleepmode;
-+	bool is_mobileye_soc;
-+	/* Keep track of configured edges */
-+	u32 edge_rising;
-+	u32 edge_falling;
-+	u32 real_wake;
-+	u32 rwimsc;
-+	u32 fwimsc;
-+	u32 rimsc;
-+	u32 fimsc;
-+	u32 pull_up;
-+	u32 lowemi;
-+};
-+
-+/* Alternate functions: function C is set in hw by setting both A and B */
-+#define NMK_GPIO_ALT_GPIO	0
-+#define NMK_GPIO_ALT_A	1
-+#define NMK_GPIO_ALT_B	2
-+#define NMK_GPIO_ALT_C	(NMK_GPIO_ALT_A | NMK_GPIO_ALT_B)
-+
-+#define NMK_GPIO_ALT_CX_SHIFT 2
-+#define NMK_GPIO_ALT_C1	((1<<NMK_GPIO_ALT_CX_SHIFT) | NMK_GPIO_ALT_C)
-+#define NMK_GPIO_ALT_C2	((2<<NMK_GPIO_ALT_CX_SHIFT) | NMK_GPIO_ALT_C)
-+#define NMK_GPIO_ALT_C3	((3<<NMK_GPIO_ALT_CX_SHIFT) | NMK_GPIO_ALT_C)
-+#define NMK_GPIO_ALT_C4	((4<<NMK_GPIO_ALT_CX_SHIFT) | NMK_GPIO_ALT_C)
-+
-+#define PRCM_GPIOCR_ALTCX(pin_num,\
-+	altc1_used, altc1_ri, altc1_cb,\
-+	altc2_used, altc2_ri, altc2_cb,\
-+	altc3_used, altc3_ri, altc3_cb,\
-+	altc4_used, altc4_ri, altc4_cb)\
-+{\
-+	.pin = pin_num,\
-+	.altcx[PRCM_IDX_GPIOCR_ALTC1] = {\
-+		.used = altc1_used,\
-+		.reg_index = altc1_ri,\
-+		.control_bit = altc1_cb\
-+	},\
-+	.altcx[PRCM_IDX_GPIOCR_ALTC2] = {\
-+		.used = altc2_used,\
-+		.reg_index = altc2_ri,\
-+		.control_bit = altc2_cb\
-+	},\
-+	.altcx[PRCM_IDX_GPIOCR_ALTC3] = {\
-+		.used = altc3_used,\
-+		.reg_index = altc3_ri,\
-+		.control_bit = altc3_cb\
-+	},\
-+	.altcx[PRCM_IDX_GPIOCR_ALTC4] = {\
-+		.used = altc4_used,\
-+		.reg_index = altc4_ri,\
-+		.control_bit = altc4_cb\
-+	},\
-+}
-+
-+/**
-+ * enum prcm_gpiocr_reg_index
-+ * Used to reference an PRCM GPIOCR register address.
-+ */
-+enum prcm_gpiocr_reg_index {
-+	PRCM_IDX_GPIOCR1,
-+	PRCM_IDX_GPIOCR2,
-+	PRCM_IDX_GPIOCR3
-+};
-+/**
-+ * enum prcm_gpiocr_altcx_index
-+ * Used to reference an Other alternate-C function.
-+ */
-+enum prcm_gpiocr_altcx_index {
-+	PRCM_IDX_GPIOCR_ALTC1,
-+	PRCM_IDX_GPIOCR_ALTC2,
-+	PRCM_IDX_GPIOCR_ALTC3,
-+	PRCM_IDX_GPIOCR_ALTC4,
-+	PRCM_IDX_GPIOCR_ALTC_MAX,
-+};
-+
-+/**
-+ * struct prcm_gpio_altcx - Other alternate-C function
-+ * @used: other alternate-C function availability
-+ * @reg_index: PRCM GPIOCR register index used to control the function
-+ * @control_bit: PRCM GPIOCR bit used to control the function
-+ */
-+struct prcm_gpiocr_altcx {
-+	bool used:1;
-+	u8 reg_index:2;
-+	u8 control_bit:5;
-+} __packed;
-+
-+/**
-+ * struct prcm_gpio_altcx_pin_desc - Other alternate-C pin
-+ * @pin: The pin number
-+ * @altcx: array of other alternate-C[1-4] functions
-+ */
-+struct prcm_gpiocr_altcx_pin_desc {
-+	unsigned short pin;
-+	struct prcm_gpiocr_altcx altcx[PRCM_IDX_GPIOCR_ALTC_MAX];
-+};
-+
-+/**
-+ * struct nmk_function - Nomadik pinctrl mux function
-+ * @name: The name of the function, exported to pinctrl core.
-+ * @groups: An array of pin groups that may select this function.
-+ * @ngroups: The number of entries in @groups.
-+ */
-+struct nmk_function {
-+	const char *name;
-+	const char * const *groups;
-+	unsigned int ngroups;
-+};
-+
-+/**
-+ * struct nmk_pingroup - describes a Nomadik pin group
-+ * @grp: Generic data of the pin group (name and pins)
-+ * @altsetting: the altsetting to apply to all pins in this group to
-+ *	configure them to be used by a function
-+ */
-+struct nmk_pingroup {
-+	struct pingroup grp;
-+	int altsetting;
-+};
-+
-+#define NMK_PIN_GROUP(a, b)							\
-+	{									\
-+		.grp = PINCTRL_PINGROUP(#a, a##_pins, ARRAY_SIZE(a##_pins)),	\
-+		.altsetting = b,						\
-+	}
-+
-+/**
-+ * struct nmk_pinctrl_soc_data - Nomadik pin controller per-SoC configuration
-+ * @pins:	An array describing all pins the pin controller affects.
-+ *		All pins which are also GPIOs must be listed first within the
-+ *		array, and be numbered identically to the GPIO controller's
-+ *		numbering.
-+ * @npins:	The number of entries in @pins.
-+ * @functions:	The functions supported on this SoC.
-+ * @nfunction:	The number of entries in @functions.
-+ * @groups:	An array describing all pin groups the pin SoC supports.
-+ * @ngroups:	The number of entries in @groups.
-+ * @altcx_pins:	The pins that support Other alternate-C function on this SoC
-+ * @npins_altcx: The number of Other alternate-C pins
-+ * @prcm_gpiocr_registers: The array of PRCM GPIOCR registers on this SoC
-+ */
-+struct nmk_pinctrl_soc_data {
-+	const struct pinctrl_pin_desc *pins;
-+	unsigned int npins;
-+	const struct nmk_function *functions;
-+	unsigned int nfunctions;
-+	const struct nmk_pingroup *groups;
-+	unsigned int ngroups;
-+	const struct prcm_gpiocr_altcx_pin_desc *altcx_pins;
-+	unsigned int npins_altcx;
-+	const u16 *prcm_gpiocr_registers;
-+};
-+
-+#ifdef CONFIG_PINCTRL_STN8815
-+
-+void nmk_pinctrl_stn8815_init(const struct nmk_pinctrl_soc_data **soc);
-+
-+#else
-+
-+static inline void
-+nmk_pinctrl_stn8815_init(const struct nmk_pinctrl_soc_data **soc)
-+{
-+}
-+
-+#endif
-+
-+#ifdef CONFIG_PINCTRL_DB8500
-+
-+void nmk_pinctrl_db8500_init(const struct nmk_pinctrl_soc_data **soc);
-+
-+#else
-+
-+static inline void
-+nmk_pinctrl_db8500_init(const struct nmk_pinctrl_soc_data **soc)
-+{
-+}
-+
-+#endif
-+
-+#ifdef CONFIG_PINCTRL_DB8540
-+
-+void nmk_pinctrl_db8540_init(const struct nmk_pinctrl_soc_data **soc);
-+
-+#else
-+
-+static inline void
-+nmk_pinctrl_db8540_init(const struct nmk_pinctrl_soc_data **soc)
-+{
-+}
-+
-+#endif
-+
-+struct platform_device;
-+
-+#ifdef CONFIG_DEBUG_FS
-+
-+/*
-+ * Symbols declared in gpio-nomadik used by pinctrl-nomadik. If pinctrl-nomadik
-+ * is enabled, then gpio-nomadik is enabled as well; the reverse if not always
-+ * true.
-+ */
-+void nmk_gpio_dbg_show_one(struct seq_file *s, struct pinctrl_dev *pctldev,
-+			   struct gpio_chip *chip, unsigned int offset);
-+
-+#else
-+
-+static inline void nmk_gpio_dbg_show_one(struct seq_file *s,
-+					 struct pinctrl_dev *pctldev,
-+					 struct gpio_chip *chip,
-+					 unsigned int offset)
-+{
-+}
-+
-+#endif
-+
-+void __nmk_gpio_make_output(struct nmk_gpio_chip *nmk_chip,
-+			    unsigned int offset, int val);
-+void __nmk_gpio_set_slpm(struct nmk_gpio_chip *nmk_chip, unsigned int offset,
-+			 enum nmk_gpio_slpm mode);
-+struct nmk_gpio_chip *nmk_gpio_populate_chip(struct fwnode_handle *fwnode,
-+					     struct platform_device *pdev);
-+
-+/* Symbols declared in pinctrl-nomadik used by gpio-nomadik. */
-+#ifdef CONFIG_PINCTRL_NOMADIK
-+extern struct nmk_gpio_chip *nmk_gpio_chips[NMK_MAX_BANKS];
-+extern spinlock_t nmk_gpio_slpm_lock;
-+int __maybe_unused nmk_prcm_gpiocr_get_mode(struct pinctrl_dev *pctldev,
-+					    int gpio);
-+#endif
-+
-+#endif /* __LINUX_GPIO_NOMADIK_H */
-diff --git a/include/linux/gpio/gpio-reg.h b/include/linux/gpio/gpio-reg.h
-index 39b888c40b39..3913b6660ed1 100644
--- a/include/linux/gpio/gpio-reg.h
-+++ b/include/linux/gpio/gpio-reg.h
-@@ -2,9 +2,13 @@
- #ifndef GPIO_REG_H
- #define GPIO_REG_H
- 
-+#include <linux/types.h>
-+
- struct device;
- struct irq_domain;
- 
-+struct gpio_chip;
-+
- struct gpio_chip *gpio_reg_init(struct device *dev, void __iomem *reg,
- 	int base, int num, const char *label, u32 direction, u32 def_out,
- 	const char *const *names, struct irq_domain *irqdom, const int *irqs);
-diff --git a/include/linux/gpio/machine.h b/include/linux/gpio/machine.h
-index 0b619eb7ae83..44e5f162973e 100644
--- a/include/linux/gpio/machine.h
-+++ b/include/linux/gpio/machine.h
-@@ -3,7 +3,6 @@
- #define __LINUX_GPIO_MACHINE_H
- 
- #include <linux/types.h>
-#include <linux/list.h>
- 
- enum gpio_lookup_flags {
- 	GPIO_ACTIVE_HIGH		= (0 << 0),
-diff --git a/include/linux/gpio/property.h b/include/linux/gpio/property.h
-new file mode 100644
-index 000000000000..0d2209308002
--- /dev/null
-+++ b/include/linux/gpio/property.h
-@@ -0,0 +1,14 @@
-+// SPDX-License-Identifier: GPL-2.0+
-+#ifndef __LINUX_GPIO_PROPERTY_H
-+#define __LINUX_GPIO_PROPERTY_H
-+
-+#include <linux/property.h>
-+
-+struct software_node;
-+
-+#define PROPERTY_ENTRY_GPIO(_name_, _chip_node_, _idx_, _flags_) \
-+	PROPERTY_ENTRY_REF(_name_, _chip_node_, _idx_, _flags_)
-+
-+extern const struct software_node swnode_gpio_undefined;
-+
-+#endif /* __LINUX_GPIO_PROPERTY_H */
-diff --git a/include/linux/gpio/regmap.h b/include/linux/gpio/regmap.h
-index a9f7b7faf57b..12d154732ca9 100644
--- a/include/linux/gpio/regmap.h
-+++ b/include/linux/gpio/regmap.h
-@@ -6,6 +6,7 @@
- struct device;
- struct fwnode_handle;
- struct gpio_regmap;
-+struct gpio_chip;
- struct irq_domain;
- struct regmap;
- 
-@@ -21,7 +22,7 @@ struct regmap;
-  *			If not given, the fwnode of the parent is used.
-  * @label:		(Optional) Descriptive name for GPIO controller.
-  *			If not given, the name of the device is used.
- * @ngpio:		Number of GPIOs
-+ * @ngpio:		(Optional) Number of GPIOs
-  * @names:		(Optional) Array of names for gpios
-  * @reg_dat_base:	(Optional) (in) register base address
-  * @reg_set_base:	(Optional) set register base address
-@@ -30,16 +31,27 @@ struct regmap;
-  * @reg_dir_out_base:	(Optional) out setting register base address
-  * @reg_stride:		(Optional) May be set if the registers (of the
-  *			same type, dat, set, etc) are not consecutive.
- * @ngpio_per_reg:	Number of GPIOs per register
-+ * @ngpio_per_reg:	(Optional) Number of GPIOs per register
-  * @irq_domain:		(Optional) IRQ domain if the controller is
-  *			interrupt-capable
-  * @reg_mask_xlate:     (Optional) Translates base address and GPIO
-  *			offset to a register/bitmask pair. If not
-  *			given the default gpio_regmap_simple_xlate()
-  *			is used.
-+ * @fixed_direction_output:
-+ *			(Optional) Bitmap representing the fixed direction of
-+ *			the GPIO lines. Useful when there are GPIO lines with a
-+ *			fixed direction mixed together in the same register.
-  * @drvdata:		(Optional) Pointer to driver specific data which is
-  *			not used by gpio-remap but is provided "as is" to the
-  *			driver callback(s).
-+ * @init_valid_mask:	(Optional) Routine to initialize @valid_mask, to be used
-+ *			if not all GPIOs are valid.
-+ * @regmap_irq_chip:	(Optional) Pointer on an regmap_irq_chip structure. If
-+ *			set, a regmap-irq device will be created and the IRQ
-+ *			domain will be set accordingly.
-+ * @regmap_irq_line:	(Optional) The IRQ the device uses to signal interrupts.
-+ * @regmap_irq_flags:	(Optional) The IRQF_ flags to use for the interrupt.
-  *
-  * The ->reg_mask_xlate translates a given base address and GPIO offset to
-  * register and mask pair. The base address is one of the given register
-@@ -77,11 +89,22 @@ struct gpio_regmap_config {
- 	int reg_stride;
- 	int ngpio_per_reg;
- 	struct irq_domain *irq_domain;
-+	unsigned long *fixed_direction_output;
-+
-+#ifdef CONFIG_REGMAP_IRQ
-+	struct regmap_irq_chip *regmap_irq_chip;
-+	int regmap_irq_line;
-+	unsigned long regmap_irq_flags;
-+#endif
- 
- 	int (*reg_mask_xlate)(struct gpio_regmap *gpio, unsigned int base,
- 			      unsigned int offset, unsigned int *reg,
- 			      unsigned int *mask);
- 
-+	int (*init_valid_mask)(struct gpio_chip *gc,
-+			       unsigned long *valid_mask,
-+			       unsigned int ngpios);
-+
- 	void *drvdata;
- };
- 
-diff --git a/include/linux/gpio_keys.h b/include/linux/gpio_keys.h
-index 3f84aeb81e48..80fa930b04c6 100644
--- a/include/linux/gpio_keys.h
-+++ b/include/linux/gpio_keys.h
-@@ -21,6 +21,7 @@ struct device;
-  *			disable button via sysfs
-  * @value:		axis value for %EV_ABS
-  * @irq:		Irq number in case of interrupt keys
-+ * @wakeirq:		Optional dedicated wake-up interrupt
-  */
- struct gpio_keys_button {
- 	unsigned int code;
-@@ -34,6 +35,7 @@ struct gpio_keys_button {
- 	bool can_disable;
- 	int value;
- 	unsigned int irq;
-+	unsigned int wakeirq;
- };
- 
- /**
-diff --git a/include/linux/greybus.h b/include/linux/greybus.h
-index 18c0fb958b74..4d58e27ceaf6 100644
--- a/include/linux/greybus.h
-+++ b/include/linux/greybus.h
-@@ -64,7 +64,7 @@ struct greybus_driver {
- 
- 	struct device_driver driver;
- };
-#define to_greybus_driver(d) container_of(d, struct greybus_driver, driver)
-+#define to_greybus_driver(d) container_of_const(d, struct greybus_driver, driver)
- 
- static inline void greybus_set_drvdata(struct gb_bundle *bundle, void *data)
- {
-@@ -104,44 +104,14 @@ void gb_debugfs_init(void);
- void gb_debugfs_cleanup(void);
- struct dentry *gb_debugfs_get(void);
- 
-extern struct bus_type greybus_bus_type;
-+extern const struct bus_type greybus_bus_type;
- 
-extern struct device_type greybus_hd_type;
-extern struct device_type greybus_module_type;
-extern struct device_type greybus_interface_type;
-extern struct device_type greybus_control_type;
-extern struct device_type greybus_bundle_type;
-extern struct device_type greybus_svc_type;
-
-static inline int is_gb_host_device(const struct device *dev)
-{
-	return dev->type == &greybus_hd_type;
-}
-
-static inline int is_gb_module(const struct device *dev)
-{
-	return dev->type == &greybus_module_type;
-}
-
-static inline int is_gb_interface(const struct device *dev)
-{
-	return dev->type == &greybus_interface_type;
-}
-
-static inline int is_gb_control(const struct device *dev)
-{
-	return dev->type == &greybus_control_type;
-}
-
-static inline int is_gb_bundle(const struct device *dev)
-{
-	return dev->type == &greybus_bundle_type;
-}
-
-static inline int is_gb_svc(const struct device *dev)
-{
-	return dev->type == &greybus_svc_type;
-}
-+extern const struct device_type greybus_hd_type;
-+extern const struct device_type greybus_module_type;
-+extern const struct device_type greybus_interface_type;
-+extern const struct device_type greybus_control_type;
-+extern const struct device_type greybus_bundle_type;
-+extern const struct device_type greybus_svc_type;
- 
- static inline bool cport_id_valid(struct gb_host_device *hd, u16 cport_id)
- {
-diff --git a/include/linux/greybus/greybus_protocols.h b/include/linux/greybus/greybus_protocols.h
-index aeb8f9243545..820134b0105c 100644
--- a/include/linux/greybus/greybus_protocols.h
-+++ b/include/linux/greybus/greybus_protocols.h
-@@ -232,9 +232,7 @@ struct gb_fw_download_fetch_firmware_request {
- 	__le32			size;
- } __packed;
- 
-struct gb_fw_download_fetch_firmware_response {
-	__u8			data[0];
-} __packed;
-+/* gb_fw_download_fetch_firmware_response contains no other data */
- 
- /* firmware download release firmware request */
- struct gb_fw_download_release_firmware_request {
-@@ -414,9 +412,7 @@ struct gb_bootrom_get_firmware_request {
- 	__le32			size;
- } __packed;
- 
-struct gb_bootrom_get_firmware_response {
-	__u8			data[0];
-} __packed;
-+/* gb_bootrom_get_firmware_response contains no other data */
- 
- /* Bootrom protocol Ready to boot request */
- struct gb_bootrom_ready_to_boot_request {
-diff --git a/include/linux/greybus/svc.h b/include/linux/greybus/svc.h
-index 5afaf5f06856..da547fb9071b 100644
--- a/include/linux/greybus/svc.h
-+++ b/include/linux/greybus/svc.h
-@@ -100,7 +100,4 @@ bool gb_svc_watchdog_enabled(struct gb_svc *svc);
- int gb_svc_watchdog_enable(struct gb_svc *svc);
- int gb_svc_watchdog_disable(struct gb_svc *svc);
- 
-int gb_svc_protocol_init(void);
-void gb_svc_protocol_exit(void);
-
- #endif /* __SVC_H */
-diff --git a/include/linux/group_cpus.h b/include/linux/group_cpus.h
-new file mode 100644
-index 000000000000..9d4e5ab6c314
--- /dev/null
-+++ b/include/linux/group_cpus.h
-@@ -0,0 +1,14 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Copyright (C) 2016 Thomas Gleixner.
-+ * Copyright (C) 2016-2017 Christoph Hellwig.
-+ */
-+
-+#ifndef __LINUX_GROUP_CPUS_H
-+#define __LINUX_GROUP_CPUS_H
-+#include <linux/kernel.h>
-+#include <linux/cpu.h>
-+
-+struct cpumask *group_cpus_evenly(unsigned int numgrps, unsigned int *nummasks);
-+
-+#endif
-diff --git a/include/linux/habanalabs/cpucp_if.h b/include/linux/habanalabs/cpucp_if.h
-new file mode 100644
-index 000000000000..45f181bcf890
--- /dev/null
-+++ b/include/linux/habanalabs/cpucp_if.h
-@@ -0,0 +1,1437 @@
-+/* SPDX-License-Identifier: GPL-2.0
-+ *
-+ * Copyright 2020-2023 HabanaLabs, Ltd.
-+ * All Rights Reserved.
-+ *
-+ */
-+
-+#ifndef CPUCP_IF_H
-+#define CPUCP_IF_H
-+
-+#include <linux/types.h>
-+#include <linux/if_ether.h>
-+
-+#include "hl_boot_if.h"
-+
-+#define NUM_HBM_PSEUDO_CH				2
-+#define NUM_HBM_CH_PER_DEV				8
-+#define CPUCP_PKT_HBM_ECC_INFO_WR_PAR_SHIFT		0
-+#define CPUCP_PKT_HBM_ECC_INFO_WR_PAR_MASK		0x00000001
-+#define CPUCP_PKT_HBM_ECC_INFO_RD_PAR_SHIFT		1
-+#define CPUCP_PKT_HBM_ECC_INFO_RD_PAR_MASK		0x00000002
-+#define CPUCP_PKT_HBM_ECC_INFO_CA_PAR_SHIFT		2
-+#define CPUCP_PKT_HBM_ECC_INFO_CA_PAR_MASK		0x00000004
-+#define CPUCP_PKT_HBM_ECC_INFO_DERR_SHIFT		3
-+#define CPUCP_PKT_HBM_ECC_INFO_DERR_MASK		0x00000008
-+#define CPUCP_PKT_HBM_ECC_INFO_SERR_SHIFT		4
-+#define CPUCP_PKT_HBM_ECC_INFO_SERR_MASK		0x00000010
-+#define CPUCP_PKT_HBM_ECC_INFO_TYPE_SHIFT		5
-+#define CPUCP_PKT_HBM_ECC_INFO_TYPE_MASK		0x00000020
-+#define CPUCP_PKT_HBM_ECC_INFO_HBM_CH_SHIFT		6
-+#define CPUCP_PKT_HBM_ECC_INFO_HBM_CH_MASK		0x000007C0
-+
-+#define PLL_MAP_MAX_BITS	128
-+#define PLL_MAP_LEN		(PLL_MAP_MAX_BITS / 8)
-+
-+enum eq_event_id {
-+	EQ_EVENT_NIC_STS_REQUEST = 0,
-+	EQ_EVENT_PWR_MODE_0,
-+	EQ_EVENT_PWR_MODE_1,
-+	EQ_EVENT_PWR_MODE_2,
-+	EQ_EVENT_PWR_MODE_3,
-+	EQ_EVENT_PWR_BRK_ENTRY,
-+	EQ_EVENT_PWR_BRK_EXIT,
-+	EQ_EVENT_HEARTBEAT,
-+	EQ_EVENT_CPLD_RESET_REASON,
-+	EQ_EVENT_CPLD_SHUTDOWN,
-+	EQ_EVENT_POWER_EVT_START,
-+	EQ_EVENT_POWER_EVT_END,
-+	EQ_EVENT_THERMAL_EVT_START,
-+	EQ_EVENT_THERMAL_EVT_END,
-+};
-+
-+/*
-+ * info of the pkt queue pointers in the first async occurrence
-+ */
-+struct cpucp_pkt_sync_err {
-+	__le32 pi;
-+	__le32 ci;
-+};
-+
-+struct hl_eq_hbm_ecc_data {
-+	/* SERR counter */
-+	__le32 sec_cnt;
-+	/* DERR counter */
-+	__le32 dec_cnt;
-+	/* Supplemental Information according to the mask bits */
-+	__le32 hbm_ecc_info;
-+	/* Address in hbm where the ecc happened */
-+	__le32 first_addr;
-+	/* SERR continuous address counter */
-+	__le32 sec_cont_cnt;
-+	__le32 pad;
-+};
-+
-+/*
-+ * EVENT QUEUE
-+ */
-+
-+struct hl_eq_header {
-+	__le32 reserved;
-+	__le32 ctl;
-+};
-+
-+struct hl_eq_ecc_data {
-+	__le64 ecc_address;
-+	__le64 ecc_syndrom;
-+	__u8 memory_wrapper_idx;
-+	__u8 is_critical;
-+	__le16 block_id;
-+	__u8 pad[4];
-+};
-+
-+enum hl_sm_sei_cause {
-+	SM_SEI_SO_OVERFLOW,
-+	SM_SEI_LBW_4B_UNALIGNED,
-+	SM_SEI_AXI_RESPONSE_ERR
-+};
-+
-+struct hl_eq_sm_sei_data {
-+	__le32 sei_log;
-+	/* enum hl_sm_sei_cause */
-+	__u8 sei_cause;
-+	__u8 pad[3];
-+};
-+
-+enum hl_fw_alive_severity {
-+	FW_ALIVE_SEVERITY_MINOR,
-+	FW_ALIVE_SEVERITY_CRITICAL
-+};
-+
-+struct hl_eq_fw_alive {
-+	__le64 uptime_seconds;
-+	__le32 process_id;
-+	__le32 thread_id;
-+	/* enum hl_fw_alive_severity */
-+	__u8 severity;
-+	__u8 pad[7];
-+};
-+
-+struct hl_eq_intr_cause {
-+	__le64 intr_cause_data;
-+};
-+
-+struct hl_eq_pcie_drain_ind_data {
-+	struct hl_eq_intr_cause intr_cause;
-+	__le64 drain_wr_addr_lbw;
-+	__le64 drain_rd_addr_lbw;
-+	__le64 drain_wr_addr_hbw;
-+	__le64 drain_rd_addr_hbw;
-+};
-+
-+struct hl_eq_razwi_lbw_info_regs {
-+	__le32 rr_aw_razwi_reg;
-+	__le32 rr_aw_razwi_id_reg;
-+	__le32 rr_ar_razwi_reg;
-+	__le32 rr_ar_razwi_id_reg;
-+};
-+
-+struct hl_eq_razwi_hbw_info_regs {
-+	__le32 rr_aw_razwi_hi_reg;
-+	__le32 rr_aw_razwi_lo_reg;
-+	__le32 rr_aw_razwi_id_reg;
-+	__le32 rr_ar_razwi_hi_reg;
-+	__le32 rr_ar_razwi_lo_reg;
-+	__le32 rr_ar_razwi_id_reg;
-+};
-+
-+/* razwi_happened masks */
-+#define RAZWI_HAPPENED_HBW	0x1
-+#define RAZWI_HAPPENED_LBW	0x2
-+#define RAZWI_HAPPENED_AW	0x4
-+#define RAZWI_HAPPENED_AR	0x8
-+
-+struct hl_eq_razwi_info {
-+	__le32 razwi_happened_mask;
-+	union {
-+		struct hl_eq_razwi_lbw_info_regs lbw;
-+		struct hl_eq_razwi_hbw_info_regs hbw;
-+	};
-+	__le32 pad;
-+};
-+
-+struct hl_eq_razwi_with_intr_cause {
-+	struct hl_eq_razwi_info razwi_info;
-+	struct hl_eq_intr_cause intr_cause;
-+};
-+
-+#define HBM_CA_ERR_CMD_LIFO_LEN		8
-+#define HBM_RD_ERR_DATA_LIFO_LEN	8
-+#define HBM_WR_PAR_CMD_LIFO_LEN		11
-+
-+enum hl_hbm_sei_cause {
-+	/* Command/address parity error event is split into 2 events due to
-+	 * size limitation: ODD suffix for odd HBM CK_t cycles and EVEN  suffix
-+	 * for even HBM CK_t cycles
-+	 */
-+	HBM_SEI_CMD_PARITY_EVEN,
-+	HBM_SEI_CMD_PARITY_ODD,
-+	/* Read errors can be reflected as a combination of SERR/DERR/parity
-+	 * errors. Therefore, we define one event for all read error types.
-+	 * LKD will perform further proccessing.
-+	 */
-+	HBM_SEI_READ_ERR,
-+	HBM_SEI_WRITE_DATA_PARITY_ERR,
-+	HBM_SEI_CATTRIP,
-+	HBM_SEI_MEM_BIST_FAIL,
-+	HBM_SEI_DFI,
-+	HBM_SEI_INV_TEMP_READ_OUT,
-+	HBM_SEI_BIST_FAIL,
-+};
-+
-+/* Masks for parsing hl_hbm_sei_headr fields */
-+#define HBM_ECC_SERR_CNTR_MASK		0xFF
-+#define HBM_ECC_DERR_CNTR_MASK		0xFF00
-+#define HBM_RD_PARITY_CNTR_MASK		0xFF0000
-+
-+/* HBM index and MC index are known by the event_id */
-+struct hl_hbm_sei_header {
-+	union {
-+		/* relevant only in case of HBM read error */
-+		struct {
-+			__u8 ecc_serr_cnt;
-+			__u8 ecc_derr_cnt;
-+			__u8 read_par_cnt;
-+			__u8 reserved;
-+		};
-+		/* All other cases */
-+		__le32 cnt;
-+	};
-+	__u8 sei_cause;		/* enum hl_hbm_sei_cause */
-+	__u8 mc_channel;		/* range: 0-3 */
-+	__u8 mc_pseudo_channel;	/* range: 0-7 */
-+	__u8 is_critical;
-+};
-+
-+#define HBM_RD_ADDR_SID_SHIFT		0
-+#define HBM_RD_ADDR_SID_MASK		0x1
-+#define HBM_RD_ADDR_BG_SHIFT		1
-+#define HBM_RD_ADDR_BG_MASK		0x6
-+#define HBM_RD_ADDR_BA_SHIFT		3
-+#define HBM_RD_ADDR_BA_MASK		0x18
-+#define HBM_RD_ADDR_COL_SHIFT		5
-+#define HBM_RD_ADDR_COL_MASK		0x7E0
-+#define HBM_RD_ADDR_ROW_SHIFT		11
-+#define HBM_RD_ADDR_ROW_MASK		0x3FFF800
-+
-+struct hbm_rd_addr {
-+	union {
-+		/* bit fields are only for FW use */
-+		struct {
-+			u32 dbg_rd_err_addr_sid:1;
-+			u32 dbg_rd_err_addr_bg:2;
-+			u32 dbg_rd_err_addr_ba:2;
-+			u32 dbg_rd_err_addr_col:6;
-+			u32 dbg_rd_err_addr_row:15;
-+			u32 reserved:6;
-+		};
-+		__le32 rd_addr_val;
-+	};
-+};
-+
-+#define HBM_RD_ERR_BEAT_SHIFT		2
-+/* dbg_rd_err_misc fields: */
-+/* Read parity is calculated per DW on every beat */
-+#define HBM_RD_ERR_PAR_ERR_BEAT0_SHIFT	0
-+#define HBM_RD_ERR_PAR_ERR_BEAT0_MASK	0x3
-+#define HBM_RD_ERR_PAR_DATA_BEAT0_SHIFT	8
-+#define HBM_RD_ERR_PAR_DATA_BEAT0_MASK	0x300
-+/* ECC is calculated per PC on every beat */
-+#define HBM_RD_ERR_SERR_BEAT0_SHIFT	16
-+#define HBM_RD_ERR_SERR_BEAT0_MASK	0x10000
-+#define HBM_RD_ERR_DERR_BEAT0_SHIFT	24
-+#define HBM_RD_ERR_DERR_BEAT0_MASK	0x100000
-+
-+struct hl_eq_hbm_sei_read_err_intr_info {
-+	/* DFI_RD_ERR_REP_ADDR */
-+	struct hbm_rd_addr dbg_rd_err_addr;
-+	/* DFI_RD_ERR_REP_ERR */
-+	union {
-+		struct {
-+			/* bit fields are only for FW use */
-+			u32 dbg_rd_err_par:8;
-+			u32 dbg_rd_err_par_data:8;
-+			u32 dbg_rd_err_serr:4;
-+			u32 dbg_rd_err_derr:4;
-+			u32 reserved:8;
-+		};
-+		__le32 dbg_rd_err_misc;
-+	};
-+	/* DFI_RD_ERR_REP_DM */
-+	__le32 dbg_rd_err_dm;
-+	/* DFI_RD_ERR_REP_SYNDROME */
-+	__le32 dbg_rd_err_syndrome;
-+	/* DFI_RD_ERR_REP_DATA */
-+	__le32 dbg_rd_err_data[HBM_RD_ERR_DATA_LIFO_LEN];
-+};
-+
-+struct hl_eq_hbm_sei_ca_par_intr_info {
-+	/* 14 LSBs */
-+	__le16 dbg_row[HBM_CA_ERR_CMD_LIFO_LEN];
-+	/* 18 LSBs */
-+	__le32 dbg_col[HBM_CA_ERR_CMD_LIFO_LEN];
-+};
-+
-+#define WR_PAR_LAST_CMD_COL_SHIFT	0
-+#define WR_PAR_LAST_CMD_COL_MASK	0x3F
-+#define WR_PAR_LAST_CMD_BG_SHIFT	6
-+#define WR_PAR_LAST_CMD_BG_MASK		0xC0
-+#define WR_PAR_LAST_CMD_BA_SHIFT	8
-+#define WR_PAR_LAST_CMD_BA_MASK		0x300
-+#define WR_PAR_LAST_CMD_SID_SHIFT	10
-+#define WR_PAR_LAST_CMD_SID_MASK	0x400
-+
-+/* Row address isn't latched */
-+struct hbm_sei_wr_cmd_address {
-+	/* DFI_DERR_LAST_CMD */
-+	union {
-+		struct {
-+			/* bit fields are only for FW use */
-+			u32 col:6;
-+			u32 bg:2;
-+			u32 ba:2;
-+			u32 sid:1;
-+			u32 reserved:21;
-+		};
-+		__le32 dbg_wr_cmd_addr;
-+	};
-+};
-+
-+struct hl_eq_hbm_sei_wr_par_intr_info {
-+	/* entry 0: WR command address from the 1st cycle prior to the error
-+	 * entry 1: WR command address from the 2nd cycle prior to the error
-+	 * and so on...
-+	 */
-+	struct hbm_sei_wr_cmd_address dbg_last_wr_cmds[HBM_WR_PAR_CMD_LIFO_LEN];
-+	/* derr[0:1] - 1st HBM cycle DERR output
-+	 * derr[2:3] - 2nd HBM cycle DERR output
-+	 */
-+	__u8 dbg_derr;
-+	/* extend to reach 8B */
-+	__u8 pad[3];
-+};
-+
-+/*
-+ * this struct represents the following sei causes:
-+ * command parity, ECC double error, ECC single error, dfi error, cattrip,
-+ * temperature read-out, read parity error and write parity error.
-+ * some only use the header while some have extra data.
-+ */
-+struct hl_eq_hbm_sei_data {
-+	struct hl_hbm_sei_header hdr;
-+	union {
-+		struct hl_eq_hbm_sei_ca_par_intr_info ca_parity_even_info;
-+		struct hl_eq_hbm_sei_ca_par_intr_info ca_parity_odd_info;
-+		struct hl_eq_hbm_sei_read_err_intr_info read_err_info;
-+		struct hl_eq_hbm_sei_wr_par_intr_info wr_parity_info;
-+	};
-+};
-+
-+/* Engine/farm arc interrupt type */
-+enum hl_engine_arc_interrupt_type {
-+	/* Qman/farm ARC DCCM QUEUE FULL interrupt type */
-+	ENGINE_ARC_DCCM_QUEUE_FULL_IRQ = 1
-+};
-+
-+/* Data structure specifies details of payload of DCCM QUEUE FULL interrupt */
-+struct hl_engine_arc_dccm_queue_full_irq {
-+	/* Queue index value which caused DCCM QUEUE FULL */
-+	__le32 queue_index;
-+	__le32 pad;
-+};
-+
-+/* Data structure specifies details of QM/FARM ARC interrupt */
-+struct hl_eq_engine_arc_intr_data {
-+	/* ARC engine id e.g.  DCORE0_TPC0_QM_ARC, DCORE0_TCP1_QM_ARC */
-+	__le32 engine_id;
-+	__le32 intr_type; /* enum hl_engine_arc_interrupt_type */
-+	/* More info related to the interrupt e.g. queue index
-+	 * incase of DCCM_QUEUE_FULL interrupt.
-+	 */
-+	__le64 payload;
-+	__le64 pad[5];
-+};
-+
-+#define ADDR_DEC_ADDRESS_COUNT_MAX 4
-+
-+/* Data structure specifies details of ADDR_DEC interrupt */
-+struct hl_eq_addr_dec_intr_data {
-+	struct hl_eq_intr_cause intr_cause;
-+	__le64 addr[ADDR_DEC_ADDRESS_COUNT_MAX];
-+	__u8 addr_cnt;
-+	__u8 pad[7];
-+};
-+
-+struct hl_eq_entry {
-+	struct hl_eq_header hdr;
-+	union {
-+		__le64 data_placeholder;
-+		struct hl_eq_ecc_data ecc_data;
-+		struct hl_eq_hbm_ecc_data hbm_ecc_data;	/* Obsolete */
-+		struct hl_eq_sm_sei_data sm_sei_data;
-+		struct cpucp_pkt_sync_err pkt_sync_err;
-+		struct hl_eq_fw_alive fw_alive;
-+		struct hl_eq_intr_cause intr_cause;
-+		struct hl_eq_pcie_drain_ind_data pcie_drain_ind_data;
-+		struct hl_eq_razwi_info razwi_info;
-+		struct hl_eq_razwi_with_intr_cause razwi_with_intr_cause;
-+		struct hl_eq_hbm_sei_data sei_data;	/* Gaudi2 HBM */
-+		struct hl_eq_engine_arc_intr_data arc_data;
-+		struct hl_eq_addr_dec_intr_data addr_dec;
-+		__le64 data[7];
-+	};
-+};
-+
-+#define HL_EQ_ENTRY_SIZE		sizeof(struct hl_eq_entry)
-+
-+#define EQ_CTL_READY_SHIFT		31
-+#define EQ_CTL_READY_MASK		0x80000000
-+
-+#define EQ_CTL_EVENT_MODE_SHIFT		28
-+#define EQ_CTL_EVENT_MODE_MASK		0x70000000
-+
-+#define EQ_CTL_EVENT_TYPE_SHIFT		16
-+#define EQ_CTL_EVENT_TYPE_MASK		0x0FFF0000
-+
-+#define EQ_CTL_INDEX_SHIFT		0
-+#define EQ_CTL_INDEX_MASK		0x0000FFFF
-+
-+enum pq_init_status {
-+	PQ_INIT_STATUS_NA = 0,
-+	PQ_INIT_STATUS_READY_FOR_CP,
-+	PQ_INIT_STATUS_READY_FOR_HOST,
-+	PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI,
-+	PQ_INIT_STATUS_LEN_NOT_POWER_OF_TWO_ERR,
-+	PQ_INIT_STATUS_ILLEGAL_Q_ADDR_ERR
-+};
-+
-+/*
-+ * CpuCP Primary Queue Packets
-+ *
-+ * During normal operation, the host's kernel driver needs to send various
-+ * messages to CpuCP, usually either to SET some value into a H/W periphery or
-+ * to GET the current value of some H/W periphery. For example, SET the
-+ * frequency of MME/TPC and GET the value of the thermal sensor.
-+ *
-+ * These messages can be initiated either by the User application or by the
-+ * host's driver itself, e.g. power management code. In either case, the
-+ * communication from the host's driver to CpuCP will *always* be in
-+ * synchronous mode, meaning that the host will send a single message and poll
-+ * until the message was acknowledged and the results are ready (if results are
-+ * needed).
-+ *
-+ * This means that only a single message can be sent at a time and the host's
-+ * driver must wait for its result before sending the next message. Having said
-+ * that, because these are control messages which are sent in a relatively low
-+ * frequency, this limitation seems acceptable. It's important to note that
-+ * in case of multiple devices, messages to different devices *can* be sent
-+ * at the same time.
-+ *
-+ * The message, inputs/outputs (if relevant) and fence object will be located
-+ * on the device DDR at an address that will be determined by the host's driver.
-+ * During device initialization phase, the host will pass to CpuCP that address.
-+ * Most of the message types will contain inputs/outputs inside the message
-+ * itself. The common part of each message will contain the opcode of the
-+ * message (its type) and a field representing a fence object.
-+ *
-+ * When the host's driver wishes to send a message to CPU CP, it will write the
-+ * message contents to the device DDR, clear the fence object and then write to
-+ * the PSOC_ARC1_AUX_SW_INTR, to issue interrupt 121 to ARC Management CPU.
-+ *
-+ * Upon receiving the interrupt (#121), CpuCP will read the message from the
-+ * DDR. In case the message is a SET operation, CpuCP will first perform the
-+ * operation and then write to the fence object on the device DDR. In case the
-+ * message is a GET operation, CpuCP will first fill the results section on the
-+ * device DDR and then write to the fence object. If an error occurred, CpuCP
-+ * will fill the rc field with the right error code.
-+ *
-+ * In the meantime, the host's driver will poll on the fence object. Once the
-+ * host sees that the fence object is signaled, it will read the results from
-+ * the device DDR (if relevant) and resume the code execution in the host's
-+ * driver.
-+ *
-+ * To use QMAN packets, the opcode must be the QMAN opcode, shifted by 8
-+ * so the value being put by the host's driver matches the value read by CpuCP
-+ *
-+ * Non-QMAN packets should be limited to values 1 through (2^8 - 1)
-+ *
-+ * Detailed description:
-+ *
-+ * CPUCP_PACKET_DISABLE_PCI_ACCESS -
-+ *       After receiving this packet the embedded CPU must NOT issue PCI
-+ *       transactions (read/write) towards the Host CPU. This also include
-+ *       sending MSI-X interrupts.
-+ *       This packet is usually sent before the device is moved to D3Hot state.
-+ *
-+ * CPUCP_PACKET_ENABLE_PCI_ACCESS -
-+ *       After receiving this packet the embedded CPU is allowed to issue PCI
-+ *       transactions towards the Host CPU, including sending MSI-X interrupts.
-+ *       This packet is usually send after the device is moved to D0 state.
-+ *
-+ * CPUCP_PACKET_TEMPERATURE_GET -
-+ *       Fetch the current temperature / Max / Max Hyst / Critical /
-+ *       Critical Hyst of a specified thermal sensor. The packet's
-+ *       arguments specify the desired sensor and the field to get.
-+ *
-+ * CPUCP_PACKET_VOLTAGE_GET -
-+ *       Fetch the voltage / Max / Min of a specified sensor. The packet's
-+ *       arguments specify the sensor and type.
-+ *
-+ * CPUCP_PACKET_CURRENT_GET -
-+ *       Fetch the current / Max / Min of a specified sensor. The packet's
-+ *       arguments specify the sensor and type.
-+ *
-+ * CPUCP_PACKET_FAN_SPEED_GET -
-+ *       Fetch the speed / Max / Min of a specified fan. The packet's
-+ *       arguments specify the sensor and type.
-+ *
-+ * CPUCP_PACKET_PWM_GET -
-+ *       Fetch the pwm value / mode of a specified pwm. The packet's
-+ *       arguments specify the sensor and type.
-+ *
-+ * CPUCP_PACKET_PWM_SET -
-+ *       Set the pwm value / mode of a specified pwm. The packet's
-+ *       arguments specify the sensor, type and value.
-+ *
-+ * CPUCP_PACKET_FREQUENCY_SET -
-+ *       Set the frequency of a specified PLL. The packet's arguments specify
-+ *       the PLL and the desired frequency. The actual frequency in the device
-+ *       might differ from the requested frequency.
-+ *
-+ * CPUCP_PACKET_FREQUENCY_GET -
-+ *       Fetch the frequency of a specified PLL. The packet's arguments specify
-+ *       the PLL.
-+ *
-+ * CPUCP_PACKET_LED_SET -
-+ *       Set the state of a specified led. The packet's arguments
-+ *       specify the led and the desired state.
-+ *
-+ * CPUCP_PACKET_I2C_WR -
-+ *       Write 32-bit value to I2C device. The packet's arguments specify the
-+ *       I2C bus, address and value.
-+ *
-+ * CPUCP_PACKET_I2C_RD -
-+ *       Read 32-bit value from I2C device. The packet's arguments specify the
-+ *       I2C bus and address.
-+ *
-+ * CPUCP_PACKET_INFO_GET -
-+ *       Fetch information from the device as specified in the packet's
-+ *       structure. The host's driver passes the max size it allows the CpuCP to
-+ *       write to the structure, to prevent data corruption in case of
-+ *       mismatched driver/FW versions.
-+ *
-+ * CPUCP_PACKET_FLASH_PROGRAM_REMOVED - this packet was removed
-+ *
-+ * CPUCP_PACKET_UNMASK_RAZWI_IRQ -
-+ *       Unmask the given IRQ. The IRQ number is specified in the value field.
-+ *       The packet is sent after receiving an interrupt and printing its
-+ *       relevant information.
-+ *
-+ * CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY -
-+ *       Unmask the given IRQs. The IRQs numbers are specified in an array right
-+ *       after the cpucp_packet structure, where its first element is the array
-+ *       length. The packet is sent after a soft reset was done in order to
-+ *       handle any interrupts that were sent during the reset process.
-+ *
-+ * CPUCP_PACKET_TEST -
-+ *       Test packet for CpuCP connectivity. The CPU will put the fence value
-+ *       in the result field.
-+ *
-+ * CPUCP_PACKET_FREQUENCY_CURR_GET -
-+ *       Fetch the current frequency of a specified PLL. The packet's arguments
-+ *       specify the PLL.
-+ *
-+ * CPUCP_PACKET_MAX_POWER_GET -
-+ *       Fetch the maximal power of the device.
-+ *
-+ * CPUCP_PACKET_MAX_POWER_SET -
-+ *       Set the maximal power of the device. The packet's arguments specify
-+ *       the power.
-+ *
-+ * CPUCP_PACKET_EEPROM_DATA_GET -
-+ *       Get EEPROM data from the CpuCP kernel. The buffer is specified in the
-+ *       addr field. The CPU will put the returned data size in the result
-+ *       field. In addition, the host's driver passes the max size it allows the
-+ *       CpuCP to write to the structure, to prevent data corruption in case of
-+ *       mismatched driver/FW versions.
-+ *
-+ * CPUCP_PACKET_NIC_INFO_GET -
-+ *       Fetch information from the device regarding the NIC. the host's driver
-+ *       passes the max size it allows the CpuCP to write to the structure, to
-+ *       prevent data corruption in case of mismatched driver/FW versions.
-+ *
-+ * CPUCP_PACKET_TEMPERATURE_SET -
-+ *       Set the value of the offset property of a specified thermal sensor.
-+ *       The packet's arguments specify the desired sensor and the field to
-+ *       set.
-+ *
-+ * CPUCP_PACKET_VOLTAGE_SET -
-+ *       Trigger the reset_history property of a specified voltage sensor.
-+ *       The packet's arguments specify the desired sensor and the field to
-+ *       set.
-+ *
-+ * CPUCP_PACKET_CURRENT_SET -
-+ *       Trigger the reset_history property of a specified current sensor.
-+ *       The packet's arguments specify the desired sensor and the field to
-+ *       set.
-+ *
-+ * CPUCP_PACKET_PCIE_THROUGHPUT_GET -
-+ *       Get throughput of PCIe.
-+ *       The packet's arguments specify the transaction direction (TX/RX).
-+ *       The window measurement is 10[msec], and the return value is in KB/sec.
-+ *
-+ * CPUCP_PACKET_PCIE_REPLAY_CNT_GET
-+ *       Replay count measures number of "replay" events, which is basicly
-+ *       number of retries done by PCIe.
-+ *
-+ * CPUCP_PACKET_TOTAL_ENERGY_GET -
-+ *       Total Energy is measurement of energy from the time FW Linux
-+ *       is loaded. It is calculated by multiplying the average power
-+ *       by time (passed from armcp start). The units are in MilliJouls.
-+ *
-+ * CPUCP_PACKET_PLL_INFO_GET -
-+ *       Fetch frequencies of PLL from the required PLL IP.
-+ *       The packet's arguments specify the device PLL type
-+ *       Pll type is the PLL from device pll_index enum.
-+ *       The result is composed of 4 outputs, each is 16-bit
-+ *       frequency in MHz.
-+ *
-+ * CPUCP_PACKET_POWER_GET -
-+ *       Fetch the present power consumption of the device (Current * Voltage).
-+ *
-+ * CPUCP_PACKET_NIC_PFC_SET -
-+ *       Enable/Disable the NIC PFC feature. The packet's arguments specify the
-+ *       NIC port, relevant lanes to configure and one bit indication for
-+ *       enable/disable.
-+ *
-+ * CPUCP_PACKET_NIC_FAULT_GET -
-+ *       Fetch the current indication for local/remote faults from the NIC MAC.
-+ *       The result is 32-bit value of the relevant register.
-+ *
-+ * CPUCP_PACKET_NIC_LPBK_SET -
-+ *       Enable/Disable the MAC loopback feature. The packet's arguments specify
-+ *       the NIC port, relevant lanes to configure and one bit indication for
-+ *       enable/disable.
-+ *
-+ * CPUCP_PACKET_NIC_MAC_INIT -
-+ *       Configure the NIC MAC channels. The packet's arguments specify the
-+ *       NIC port and the speed.
-+ *
-+ * CPUCP_PACKET_MSI_INFO_SET -
-+ *       set the index number for each supported msi type going from
-+ *       host to device
-+ *
-+ * CPUCP_PACKET_NIC_XPCS91_REGS_GET -
-+ *       Fetch the un/correctable counters values from the NIC MAC.
-+ *
-+ * CPUCP_PACKET_NIC_STAT_REGS_GET -
-+ *       Fetch various NIC MAC counters from the NIC STAT.
-+ *
-+ * CPUCP_PACKET_NIC_STAT_REGS_CLR -
-+ *       Clear the various NIC MAC counters in the NIC STAT.
-+ *
-+ * CPUCP_PACKET_NIC_STAT_REGS_ALL_GET -
-+ *       Fetch all NIC MAC counters from the NIC STAT.
-+ *
-+ * CPUCP_PACKET_IS_IDLE_CHECK -
-+ *       Check if the device is IDLE in regard to the DMA/compute engines
-+ *       and QMANs. The f/w will return a bitmask where each bit represents
-+ *       a different engine or QMAN according to enum cpucp_idle_mask.
-+ *       The bit will be 1 if the engine is NOT idle.
-+ *
-+ * CPUCP_PACKET_HBM_REPLACED_ROWS_INFO_GET -
-+ *       Fetch all HBM replaced-rows and prending to be replaced rows data.
-+ *
-+ * CPUCP_PACKET_HBM_PENDING_ROWS_STATUS -
-+ *       Fetch status of HBM rows pending replacement and need a reboot to
-+ *       be replaced.
-+ *
-+ * CPUCP_PACKET_POWER_SET -
-+ *       Resets power history of device to 0
-+ *
-+ * CPUCP_PACKET_ENGINE_CORE_ASID_SET -
-+ *       Packet to perform engine core ASID configuration
-+ *
-+ * CPUCP_PACKET_SEC_ATTEST_GET -
-+ *       Get the attestaion data that is collected during various stages of the
-+ *       boot sequence. the attestation data is also hashed with some unique
-+ *       number (nonce) provided by the host to prevent replay attacks.
-+ *       public key and certificate also provided as part of the FW response.
-+ *
-+ * CPUCP_PACKET_INFO_SIGNED_GET -
-+ *       Get the device information signed by the Trusted Platform device.
-+ *       device info data is also hashed with some unique number (nonce) provided
-+ *       by the host to prevent replay attacks. public key and certificate also
-+ *       provided as part of the FW response.
-+ *
-+ * CPUCP_PACKET_MONITOR_DUMP_GET -
-+ *       Get monitors registers dump from the CpuCP kernel.
-+ *       The CPU will put the registers dump in the a buffer allocated by the driver
-+ *       which address is passed via the CpuCp packet. In addition, the host's driver
-+ *       passes the max size it allows the CpuCP to write to the structure, to prevent
-+ *       data corruption in case of mismatched driver/FW versions.
-+ *       Obsolete.
-+ *
-+ * CPUCP_PACKET_GENERIC_PASSTHROUGH -
-+ *       Generic opcode for all firmware info that is only passed to host
-+ *       through the LKD, without getting parsed there.
-+ *
-+ * CPUCP_PACKET_ACTIVE_STATUS_SET -
-+ *       LKD sends FW indication whether device is free or in use, this indication is reported
-+ *       also to the BMC.
-+ *
-+ * CPUCP_PACKET_SOFT_RESET -
-+ *       Packet to perform soft-reset.
-+ *
-+ * CPUCP_PACKET_INTS_REGISTER -
-+ *       Packet to inform FW that queues have been established and LKD is ready to receive
-+ *       EQ events.
-+ */
-+
-+enum cpucp_packet_id {
-+	CPUCP_PACKET_DISABLE_PCI_ACCESS = 1,	/* internal */
-+	CPUCP_PACKET_ENABLE_PCI_ACCESS,		/* internal */
-+	CPUCP_PACKET_TEMPERATURE_GET,		/* sysfs */
-+	CPUCP_PACKET_VOLTAGE_GET,		/* sysfs */
-+	CPUCP_PACKET_CURRENT_GET,		/* sysfs */
-+	CPUCP_PACKET_FAN_SPEED_GET,		/* sysfs */
-+	CPUCP_PACKET_PWM_GET,			/* sysfs */
-+	CPUCP_PACKET_PWM_SET,			/* sysfs */
-+	CPUCP_PACKET_FREQUENCY_SET,		/* sysfs */
-+	CPUCP_PACKET_FREQUENCY_GET,		/* sysfs */
-+	CPUCP_PACKET_LED_SET,			/* debugfs */
-+	CPUCP_PACKET_I2C_WR,			/* debugfs */
-+	CPUCP_PACKET_I2C_RD,			/* debugfs */
-+	CPUCP_PACKET_INFO_GET,			/* IOCTL */
-+	CPUCP_PACKET_FLASH_PROGRAM_REMOVED,
-+	CPUCP_PACKET_UNMASK_RAZWI_IRQ,		/* internal */
-+	CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY,	/* internal */
-+	CPUCP_PACKET_TEST,			/* internal */
-+	CPUCP_PACKET_FREQUENCY_CURR_GET,	/* sysfs */
-+	CPUCP_PACKET_MAX_POWER_GET,		/* sysfs */
-+	CPUCP_PACKET_MAX_POWER_SET,		/* sysfs */
-+	CPUCP_PACKET_EEPROM_DATA_GET,		/* sysfs */
-+	CPUCP_PACKET_NIC_INFO_GET,		/* internal */
-+	CPUCP_PACKET_TEMPERATURE_SET,		/* sysfs */
-+	CPUCP_PACKET_VOLTAGE_SET,		/* sysfs */
-+	CPUCP_PACKET_CURRENT_SET,		/* sysfs */
-+	CPUCP_PACKET_PCIE_THROUGHPUT_GET,	/* internal */
-+	CPUCP_PACKET_PCIE_REPLAY_CNT_GET,	/* internal */
-+	CPUCP_PACKET_TOTAL_ENERGY_GET,		/* internal */
-+	CPUCP_PACKET_PLL_INFO_GET,		/* internal */
-+	CPUCP_PACKET_NIC_STATUS,		/* internal */
-+	CPUCP_PACKET_POWER_GET,			/* internal */
-+	CPUCP_PACKET_NIC_PFC_SET,		/* internal */
-+	CPUCP_PACKET_NIC_FAULT_GET,		/* internal */
-+	CPUCP_PACKET_NIC_LPBK_SET,		/* internal */
-+	CPUCP_PACKET_NIC_MAC_CFG,		/* internal */
-+	CPUCP_PACKET_MSI_INFO_SET,		/* internal */
-+	CPUCP_PACKET_NIC_XPCS91_REGS_GET,	/* internal */
-+	CPUCP_PACKET_NIC_STAT_REGS_GET,		/* internal */
-+	CPUCP_PACKET_NIC_STAT_REGS_CLR,		/* internal */
-+	CPUCP_PACKET_NIC_STAT_REGS_ALL_GET,	/* internal */
-+	CPUCP_PACKET_IS_IDLE_CHECK,		/* internal */
-+	CPUCP_PACKET_HBM_REPLACED_ROWS_INFO_GET,/* internal */
-+	CPUCP_PACKET_HBM_PENDING_ROWS_STATUS,	/* internal */
-+	CPUCP_PACKET_POWER_SET,			/* internal */
-+	CPUCP_PACKET_RESERVED,			/* not used */
-+	CPUCP_PACKET_ENGINE_CORE_ASID_SET,	/* internal */
-+	CPUCP_PACKET_RESERVED2,			/* not used */
-+	CPUCP_PACKET_SEC_ATTEST_GET,		/* internal */
-+	CPUCP_PACKET_INFO_SIGNED_GET,		/* internal */
-+	CPUCP_PACKET_RESERVED4,			/* not used */
-+	CPUCP_PACKET_MONITOR_DUMP_GET,		/* debugfs */
-+	CPUCP_PACKET_RESERVED5,			/* not used */
-+	CPUCP_PACKET_RESERVED6,			/* not used */
-+	CPUCP_PACKET_RESERVED7,			/* not used */
-+	CPUCP_PACKET_GENERIC_PASSTHROUGH,	/* IOCTL */
-+	CPUCP_PACKET_RESERVED8,			/* not used */
-+	CPUCP_PACKET_ACTIVE_STATUS_SET,		/* internal */
-+	CPUCP_PACKET_RESERVED9,			/* not used */
-+	CPUCP_PACKET_RESERVED10,		/* not used */
-+	CPUCP_PACKET_RESERVED11,		/* not used */
-+	CPUCP_PACKET_RESERVED12,		/* internal */
-+	CPUCP_PACKET_RESERVED13,                /* internal */
-+	CPUCP_PACKET_SOFT_RESET,		/* internal */
-+	CPUCP_PACKET_INTS_REGISTER,		/* internal */
-+	CPUCP_PACKET_ID_MAX			/* must be last */
-+};
-+
-+#define CPUCP_PACKET_FENCE_VAL	0xFE8CE7A5
-+
-+#define CPUCP_PKT_CTL_RC_SHIFT		12
-+#define CPUCP_PKT_CTL_RC_MASK		0x0000F000
-+
-+#define CPUCP_PKT_CTL_OPCODE_SHIFT	16
-+#define CPUCP_PKT_CTL_OPCODE_MASK	0x1FFF0000
-+
-+#define CPUCP_PKT_RES_PLL_OUT0_SHIFT	0
-+#define CPUCP_PKT_RES_PLL_OUT0_MASK	0x000000000000FFFFull
-+#define CPUCP_PKT_RES_PLL_OUT1_SHIFT	16
-+#define CPUCP_PKT_RES_PLL_OUT1_MASK	0x00000000FFFF0000ull
-+#define CPUCP_PKT_RES_PLL_OUT2_SHIFT	32
-+#define CPUCP_PKT_RES_PLL_OUT2_MASK	0x0000FFFF00000000ull
-+#define CPUCP_PKT_RES_PLL_OUT3_SHIFT	48
-+#define CPUCP_PKT_RES_PLL_OUT3_MASK	0xFFFF000000000000ull
-+
-+#define CPUCP_PKT_RES_EEPROM_OUT0_SHIFT	0
-+#define CPUCP_PKT_RES_EEPROM_OUT0_MASK	0x000000000000FFFFull
-+#define CPUCP_PKT_RES_EEPROM_OUT1_SHIFT	16
-+#define CPUCP_PKT_RES_EEPROM_OUT1_MASK	0x0000000000FF0000ull
-+
-+#define CPUCP_PKT_VAL_PFC_IN1_SHIFT	0
-+#define CPUCP_PKT_VAL_PFC_IN1_MASK	0x0000000000000001ull
-+#define CPUCP_PKT_VAL_PFC_IN2_SHIFT	1
-+#define CPUCP_PKT_VAL_PFC_IN2_MASK	0x000000000000001Eull
-+
-+#define CPUCP_PKT_VAL_LPBK_IN1_SHIFT	0
-+#define CPUCP_PKT_VAL_LPBK_IN1_MASK	0x0000000000000001ull
-+#define CPUCP_PKT_VAL_LPBK_IN2_SHIFT	1
-+#define CPUCP_PKT_VAL_LPBK_IN2_MASK	0x000000000000001Eull
-+
-+#define CPUCP_PKT_VAL_MAC_CNT_IN1_SHIFT	0
-+#define CPUCP_PKT_VAL_MAC_CNT_IN1_MASK	0x0000000000000001ull
-+#define CPUCP_PKT_VAL_MAC_CNT_IN2_SHIFT	1
-+#define CPUCP_PKT_VAL_MAC_CNT_IN2_MASK	0x00000000FFFFFFFEull
-+
-+/* heartbeat status bits */
-+#define CPUCP_PKT_HB_STATUS_EQ_FAULT_SHIFT		0
-+#define CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK		0x00000001
-+
-+struct cpucp_packet {
-+	union {
-+		__le64 value;	/* For SET packets */
-+		__le64 result;	/* For GET packets */
-+		__le64 addr;	/* For PQ */
-+	};
-+
-+	__le32 ctl;
-+
-+	__le32 fence;		/* Signal to host that message is completed */
-+
-+	union {
-+		struct {/* For temperature/current/voltage/fan/pwm get/set */
-+			__le16 sensor_index;
-+			__le16 type;
-+		};
-+
-+		struct {	/* For I2C read/write */
-+			__u8 i2c_bus;
-+			__u8 i2c_addr;
-+			__u8 i2c_reg;
-+			/*
-+			 * In legacy implemetations, i2c_len was not present,
-+			 * was unused and just added as pad.
-+			 * So if i2c_len is 0, it is treated as legacy
-+			 * and r/w 1 Byte, else if i2c_len is specified,
-+			 * its treated as new multibyte r/w support.
-+			 */
-+			__u8 i2c_len;
-+		};
-+
-+		struct {/* For PLL info fetch */
-+			__le16 pll_type;
-+			/* TODO pll_reg is kept temporary before removal */
-+			__le16 pll_reg;
-+		};
-+
-+		/* For any general request */
-+		__le32 index;
-+
-+		/* For frequency get/set */
-+		__le32 pll_index;
-+
-+		/* For led set */
-+		__le32 led_index;
-+
-+		/* For get CpuCP info/EEPROM data/NIC info */
-+		__le32 data_max_size;
-+
-+		/*
-+		 * For any general status bitmask. Shall be used whenever the
-+		 * result cannot be used to hold general purpose data.
-+		 */
-+		__le32 status_mask;
-+	};
-+
-+	union {
-+		/* For NIC requests */
-+		__le32 port_index;
-+
-+		/* For Generic packet sub index */
-+		__le32 pkt_subidx;
-+
-+		/* random, used once number, for security packets */
-+		__le32 nonce;
-+	};
-+};
-+
-+struct cpucp_unmask_irq_arr_packet {
-+	struct cpucp_packet cpucp_pkt;
-+	__le32 length;
-+	__le32 irqs[];
-+};
-+
-+struct cpucp_nic_status_packet {
-+	struct cpucp_packet cpucp_pkt;
-+	__le32 length;
-+	__le32 data[];
-+};
-+
-+struct cpucp_array_data_packet {
-+	struct cpucp_packet cpucp_pkt;
-+	__le32 length;
-+	__le32 data[];
-+};
-+
-+enum cpucp_led_index {
-+	CPUCP_LED0_INDEX = 0,
-+	CPUCP_LED1_INDEX,
-+	CPUCP_LED2_INDEX,
-+	CPUCP_LED_MAX_INDEX = CPUCP_LED2_INDEX
-+};
-+
-+/*
-+ * enum cpucp_packet_rc - Error return code
-+ * @cpucp_packet_success	-> in case of success.
-+ * @cpucp_packet_invalid	-> this is to support first generation platforms.
-+ * @cpucp_packet_fault		-> in case of processing error like failing to
-+ *                                 get device binding or semaphore etc.
-+ * @cpucp_packet_invalid_pkt	-> when cpucp packet is un-supported.
-+ * @cpucp_packet_invalid_params	-> when checking parameter like length of buffer
-+ *				   or attribute value etc.
-+ * @cpucp_packet_rc_max		-> It indicates size of enum so should be at last.
-+ */
-+enum cpucp_packet_rc {
-+	cpucp_packet_success,
-+	cpucp_packet_invalid,
-+	cpucp_packet_fault,
-+	cpucp_packet_invalid_pkt,
-+	cpucp_packet_invalid_params,
-+	cpucp_packet_rc_max
-+};
-+
-+/*
-+ * cpucp_temp_type should adhere to hwmon_temp_attributes
-+ * defined in Linux kernel hwmon.h file
-+ */
-+enum cpucp_temp_type {
-+	cpucp_temp_input,
-+	cpucp_temp_min = 4,
-+	cpucp_temp_min_hyst,
-+	cpucp_temp_max = 6,
-+	cpucp_temp_max_hyst,
-+	cpucp_temp_crit,
-+	cpucp_temp_crit_hyst,
-+	cpucp_temp_offset = 19,
-+	cpucp_temp_lowest = 21,
-+	cpucp_temp_highest = 22,
-+	cpucp_temp_reset_history = 23,
-+	cpucp_temp_warn = 24,
-+	cpucp_temp_max_crit = 25,
-+	cpucp_temp_max_warn = 26,
-+};
-+
-+enum cpucp_in_attributes {
-+	cpucp_in_input,
-+	cpucp_in_min,
-+	cpucp_in_max,
-+	cpucp_in_lowest = 6,
-+	cpucp_in_highest = 7,
-+	cpucp_in_reset_history,
-+	cpucp_in_intr_alarm_a,
-+	cpucp_in_intr_alarm_b,
-+};
-+
-+enum cpucp_curr_attributes {
-+	cpucp_curr_input,
-+	cpucp_curr_min,
-+	cpucp_curr_max,
-+	cpucp_curr_lowest = 6,
-+	cpucp_curr_highest = 7,
-+	cpucp_curr_reset_history
-+};
-+
-+enum cpucp_fan_attributes {
-+	cpucp_fan_input,
-+	cpucp_fan_min = 2,
-+	cpucp_fan_max
-+};
-+
-+enum cpucp_pwm_attributes {
-+	cpucp_pwm_input,
-+	cpucp_pwm_enable
-+};
-+
-+enum cpucp_pcie_throughput_attributes {
-+	cpucp_pcie_throughput_tx,
-+	cpucp_pcie_throughput_rx
-+};
-+
-+/* TODO temporary kept before removal */
-+enum cpucp_pll_reg_attributes {
-+	cpucp_pll_nr_reg,
-+	cpucp_pll_nf_reg,
-+	cpucp_pll_od_reg,
-+	cpucp_pll_div_factor_reg,
-+	cpucp_pll_div_sel_reg
-+};
-+
-+/* TODO temporary kept before removal */
-+enum cpucp_pll_type_attributes {
-+	cpucp_pll_cpu,
-+	cpucp_pll_pci,
-+};
-+
-+/*
-+ * cpucp_power_type aligns with hwmon_power_attributes
-+ * defined in Linux kernel hwmon.h file
-+ */
-+enum cpucp_power_type {
-+	CPUCP_POWER_INPUT = 8,
-+	CPUCP_POWER_INPUT_HIGHEST = 9,
-+	CPUCP_POWER_RESET_INPUT_HISTORY = 11
-+};
-+
-+/*
-+ * MSI type enumeration table for all ASICs and future SW versions.
-+ * For future ASIC-LKD compatibility, we can only add new enumerations.
-+ * at the end of the table (before CPUCP_NUM_OF_MSI_TYPES).
-+ * Changing the order of entries or removing entries is not allowed.
-+ */
-+enum cpucp_msi_type {
-+	CPUCP_EVENT_QUEUE_MSI_TYPE,
-+	CPUCP_NIC_PORT1_MSI_TYPE,
-+	CPUCP_NIC_PORT3_MSI_TYPE,
-+	CPUCP_NIC_PORT5_MSI_TYPE,
-+	CPUCP_NIC_PORT7_MSI_TYPE,
-+	CPUCP_NIC_PORT9_MSI_TYPE,
-+	CPUCP_EVENT_QUEUE_ERR_MSI_TYPE,
-+	CPUCP_NUM_OF_MSI_TYPES
-+};
-+
-+/*
-+ * PLL enumeration table used for all ASICs and future SW versions.
-+ * For future ASIC-LKD compatibility, we can only add new enumerations.
-+ * at the end of the table.
-+ * Changing the order of entries or removing entries is not allowed.
-+ */
-+enum pll_index {
-+	CPU_PLL = 0,
-+	PCI_PLL = 1,
-+	NIC_PLL = 2,
-+	DMA_PLL = 3,
-+	MESH_PLL = 4,
-+	MME_PLL = 5,
-+	TPC_PLL = 6,
-+	IF_PLL = 7,
-+	SRAM_PLL = 8,
-+	NS_PLL = 9,
-+	HBM_PLL = 10,
-+	MSS_PLL = 11,
-+	DDR_PLL = 12,
-+	VID_PLL = 13,
-+	BANK_PLL = 14,
-+	MMU_PLL = 15,
-+	IC_PLL = 16,
-+	MC_PLL = 17,
-+	EMMC_PLL = 18,
-+	D2D_PLL = 19,
-+	CS_PLL = 20,
-+	C2C_PLL = 21,
-+	NCH_PLL = 22,
-+	C2M_PLL = 23,
-+	PLL_MAX
-+};
-+
-+enum rl_index {
-+	TPC_RL = 0,
-+	MME_RL,
-+	EDMA_RL,
-+};
-+
-+enum pvt_index {
-+	PVT_SW,
-+	PVT_SE,
-+	PVT_NW,
-+	PVT_NE
-+};
-+
-+/* Event Queue Packets */
-+
-+struct eq_generic_event {
-+	__le64 data[7];
-+};
-+
-+/*
-+ * CpuCP info
-+ */
-+
-+#define CARD_NAME_MAX_LEN		16
-+#define CPUCP_MAX_SENSORS		128
-+#define CPUCP_MAX_NICS			128
-+#define CPUCP_LANES_PER_NIC		4
-+#define CPUCP_NIC_QSFP_EEPROM_MAX_LEN	1024
-+#define CPUCP_MAX_NIC_LANES		(CPUCP_MAX_NICS * CPUCP_LANES_PER_NIC)
-+#define CPUCP_NIC_MASK_ARR_LEN		((CPUCP_MAX_NICS + 63) / 64)
-+#define CPUCP_NIC_POLARITY_ARR_LEN	((CPUCP_MAX_NIC_LANES + 63) / 64)
-+#define CPUCP_HBM_ROW_REPLACE_MAX	32
-+
-+struct cpucp_sensor {
-+	__le32 type;
-+	__le32 flags;
-+};
-+
-+/**
-+ * struct cpucp_card_types - ASIC card type.
-+ * @cpucp_card_type_pci: PCI card.
-+ * @cpucp_card_type_pmc: PCI Mezzanine Card.
-+ */
-+enum cpucp_card_types {
-+	cpucp_card_type_pci,
-+	cpucp_card_type_pmc
-+};
-+
-+#define CPUCP_SEC_CONF_ENABLED_SHIFT	0
-+#define CPUCP_SEC_CONF_ENABLED_MASK	0x00000001
-+
-+#define CPUCP_SEC_CONF_FLASH_WP_SHIFT	1
-+#define CPUCP_SEC_CONF_FLASH_WP_MASK	0x00000002
-+
-+#define CPUCP_SEC_CONF_EEPROM_WP_SHIFT	2
-+#define CPUCP_SEC_CONF_EEPROM_WP_MASK	0x00000004
-+
-+/**
-+ * struct cpucp_security_info - Security information.
-+ * @config: configuration bit field
-+ * @keys_num: number of stored keys
-+ * @revoked_keys: revoked keys bit field
-+ * @min_svn: minimal security version
-+ */
-+struct cpucp_security_info {
-+	__u8 config;
-+	__u8 keys_num;
-+	__u8 revoked_keys;
-+	__u8 min_svn;
-+};
-+
-+/**
-+ * struct cpucp_info - Info from CpuCP that is necessary to the host's driver
-+ * @sensors: available sensors description.
-+ * @kernel_version: CpuCP linux kernel version.
-+ * @reserved: reserved field.
-+ * @card_type: card configuration type.
-+ * @card_location: in a server, each card has different connections topology
-+ *                 depending on its location (relevant for PMC card type)
-+ * @cpld_version: CPLD programmed F/W version.
-+ * @infineon_version: Infineon main DC-DC version.
-+ * @fuse_version: silicon production FUSE information.
-+ * @thermal_version: thermald S/W version.
-+ * @cpucp_version: CpuCP S/W version.
-+ * @infineon_second_stage_version: Infineon 2nd stage DC-DC version.
-+ * @dram_size: available DRAM size.
-+ * @card_name: card name that will be displayed in HWMON subsystem on the host
-+ * @tpc_binning_mask: TPC binning mask, 1 bit per TPC instance
-+ *                    (0 = functional, 1 = binned)
-+ * @decoder_binning_mask: Decoder binning mask, 1 bit per decoder instance
-+ *                        (0 = functional, 1 = binned), maximum 1 per dcore
-+ * @sram_binning: Categorize SRAM functionality
-+ *                (0 = fully functional, 1 = lower-half is not functional,
-+ *                 2 = upper-half is not functional)
-+ * @sec_info: security information
-+ * @cpld_timestamp: CPLD programmed F/W timestamp.
-+ * @pll_map: Bit map of supported PLLs for current ASIC version.
-+ * @mme_binning_mask: MME binning mask,
-+ *                    bits [0:6]   <==> dcore0 mme fma
-+ *                    bits [7:13]  <==> dcore1 mme fma
-+ *                    bits [14:20] <==> dcore0 mme ima
-+ *                    bits [21:27] <==> dcore1 mme ima
-+ *                    For each group, if the 6th bit is set then first 5 bits
-+ *                    represent the col's idx [0-31], otherwise these bits are
-+ *                    ignored, and col idx 32 is binned. 7th bit is don't care.
-+ * @dram_binning_mask: DRAM binning mask, 1 bit per dram instance
-+ *                     (0 = functional 1 = binned)
-+ * @memory_repair_flag: eFuse flag indicating memory repair
-+ * @edma_binning_mask: EDMA binning mask, 1 bit per EDMA instance
-+ *                     (0 = functional 1 = binned)
-+ * @xbar_binning_mask: Xbar binning mask, 1 bit per Xbar instance
-+ *                     (0 = functional 1 = binned)
-+ * @interposer_version: Interposer version programmed in eFuse
-+ * @substrate_version: Substrate version programmed in eFuse
-+ * @eq_health_check_supported: eq health check feature supported in FW.
-+ * @fw_hbm_region_size: Size in bytes of FW reserved region in HBM.
-+ * @fw_os_version: Firmware OS Version
-+ */
-+struct cpucp_info {
-+	struct cpucp_sensor sensors[CPUCP_MAX_SENSORS];
-+	__u8 kernel_version[VERSION_MAX_LEN];
-+	__le32 reserved1;
-+	__le32 card_type;
-+	__le32 card_location;
-+	__le32 cpld_version;
-+	__le32 infineon_version;
-+	__u8 fuse_version[VERSION_MAX_LEN];
-+	__u8 thermal_version[VERSION_MAX_LEN];
-+	__u8 cpucp_version[VERSION_MAX_LEN];
-+	__le32 infineon_second_stage_version;
-+	__le64 dram_size;
-+	char card_name[CARD_NAME_MAX_LEN];
-+	__le64 tpc_binning_mask;
-+	__le64 decoder_binning_mask;
-+	__u8 sram_binning;
-+	__u8 dram_binning_mask;
-+	__u8 memory_repair_flag;
-+	__u8 edma_binning_mask;
-+	__u8 xbar_binning_mask;
-+	__u8 interposer_version;
-+	__u8 substrate_version;
-+	__u8 eq_health_check_supported;
-+	struct cpucp_security_info sec_info;
-+	__le32 cpld_timestamp;
-+	__u8 pll_map[PLL_MAP_LEN];
-+	__le64 mme_binning_mask;
-+	__u8 fw_os_version[VERSION_MAX_LEN];
-+};
-+
-+struct cpucp_mac_addr {
-+	__u8 mac_addr[ETH_ALEN];
-+};
-+
-+enum cpucp_serdes_type {
-+	TYPE_1_SERDES_TYPE,
-+	TYPE_2_SERDES_TYPE,
-+	HLS1_SERDES_TYPE,
-+	HLS1H_SERDES_TYPE,
-+	HLS2_SERDES_TYPE,
-+	HLS2_TYPE_1_SERDES_TYPE,
-+	MAX_NUM_SERDES_TYPE,		/* number of types */
-+	UNKNOWN_SERDES_TYPE = 0xFFFF	/* serdes_type is u16 */
-+};
-+
-+struct cpucp_nic_info {
-+	struct cpucp_mac_addr mac_addrs[CPUCP_MAX_NICS];
-+	__le64 link_mask[CPUCP_NIC_MASK_ARR_LEN];
-+	__le64 pol_tx_mask[CPUCP_NIC_POLARITY_ARR_LEN];
-+	__le64 pol_rx_mask[CPUCP_NIC_POLARITY_ARR_LEN];
-+	__le64 link_ext_mask[CPUCP_NIC_MASK_ARR_LEN];
-+	__u8 qsfp_eeprom[CPUCP_NIC_QSFP_EEPROM_MAX_LEN];
-+	__le64 auto_neg_mask[CPUCP_NIC_MASK_ARR_LEN];
-+	__le16 serdes_type; /* enum cpucp_serdes_type */
-+	__le16 tx_swap_map[CPUCP_MAX_NICS];
-+	__u8 reserved[6];
-+};
-+
-+#define PAGE_DISCARD_MAX	64
-+
-+struct page_discard_info {
-+	__u8 num_entries;
-+	__u8 reserved[7];
-+	__le32 mmu_page_idx[PAGE_DISCARD_MAX];
-+};
-+
-+/*
-+ * struct frac_val - fracture value represented by "integer.frac".
-+ * @integer: the integer part of the fracture value;
-+ * @frac: the fracture part of the fracture value.
-+ */
-+struct frac_val {
-+	union {
-+		struct {
-+			__le16 integer;
-+			__le16 frac;
-+		};
-+		__le32 val;
-+	};
-+};
-+
-+/*
-+ * struct ser_val - the SER (symbol error rate) value is represented by "integer * 10 ^ -exp".
-+ * @integer: the integer part of the SER value;
-+ * @exp: the exponent part of the SER value.
-+ */
-+struct ser_val {
-+	__le16 integer;
-+	__le16 exp;
-+};
-+
-+/*
-+ * struct cpucp_nic_status - describes the status of a NIC port.
-+ * @port: NIC port index.
-+ * @bad_format_cnt: e.g. CRC.
-+ * @responder_out_of_sequence_psn_cnt: e.g NAK.
-+ * @high_ber_reinit_cnt: link reinit due to high BER.
-+ * @correctable_err_cnt: e.g. bit-flip.
-+ * @uncorrectable_err_cnt: e.g. MAC errors.
-+ * @retraining_cnt: re-training counter.
-+ * @up: is port up.
-+ * @pcs_link: has PCS link.
-+ * @phy_ready: is PHY ready.
-+ * @auto_neg: is Autoneg enabled.
-+ * @timeout_retransmission_cnt: timeout retransmission events.
-+ * @high_ber_cnt: high ber events.
-+ * @pre_fec_ser: pre FEC SER value.
-+ * @post_fec_ser: post FEC SER value.
-+ * @throughput: measured throughput.
-+ * @latency: measured latency.
-+ */
-+struct cpucp_nic_status {
-+	__le32 port;
-+	__le32 bad_format_cnt;
-+	__le32 responder_out_of_sequence_psn_cnt;
-+	__le32 high_ber_reinit;
-+	__le32 correctable_err_cnt;
-+	__le32 uncorrectable_err_cnt;
-+	__le32 retraining_cnt;
-+	__u8 up;
-+	__u8 pcs_link;
-+	__u8 phy_ready;
-+	__u8 auto_neg;
-+	__le32 timeout_retransmission_cnt;
-+	__le32 high_ber_cnt;
-+	struct ser_val pre_fec_ser;
-+	struct ser_val post_fec_ser;
-+	struct frac_val bandwidth;
-+	struct frac_val lat;
-+};
-+
-+enum cpucp_hbm_row_replace_cause {
-+	REPLACE_CAUSE_DOUBLE_ECC_ERR,
-+	REPLACE_CAUSE_MULTI_SINGLE_ECC_ERR,
-+};
-+
-+struct cpucp_hbm_row_info {
-+	__u8 hbm_idx;
-+	__u8 pc;
-+	__u8 sid;
-+	__u8 bank_idx;
-+	__le16 row_addr;
-+	__u8 replaced_row_cause; /* enum cpucp_hbm_row_replace_cause */
-+	__u8 pad;
-+};
-+
-+struct cpucp_hbm_row_replaced_rows_info {
-+	__le16 num_replaced_rows;
-+	__u8 pad[6];
-+	struct cpucp_hbm_row_info replaced_rows[CPUCP_HBM_ROW_REPLACE_MAX];
-+};
-+
-+enum cpu_reset_status {
-+	CPU_RST_STATUS_NA = 0,
-+	CPU_RST_STATUS_SOFT_RST_DONE = 1,
-+};
-+
-+#define SEC_PCR_DATA_BUF_SZ	256
-+#define SEC_PCR_QUOTE_BUF_SZ	510	/* (512 - 2) 2 bytes used for size */
-+#define SEC_SIGNATURE_BUF_SZ	255	/* (256 - 1) 1 byte used for size */
-+#define SEC_PUB_DATA_BUF_SZ	510	/* (512 - 2) 2 bytes used for size */
-+#define SEC_CERTIFICATE_BUF_SZ	2046	/* (2048 - 2) 2 bytes used for size */
-+
-+/*
-+ * struct cpucp_sec_attest_info - attestation report of the boot
-+ * @pcr_data: raw values of the PCR registers
-+ * @pcr_num_reg: number of PCR registers in the pcr_data array
-+ * @pcr_reg_len: length of each PCR register in the pcr_data array (bytes)
-+ * @nonce: number only used once. random number provided by host. this also
-+ *	    passed to the quote command as a qualifying data.
-+ * @pcr_quote_len: length of the attestation quote data (bytes)
-+ * @pcr_quote: attestation report data structure
-+ * @quote_sig_len: length of the attestation report signature (bytes)
-+ * @quote_sig: signature structure of the attestation report
-+ * @pub_data_len: length of the public data (bytes)
-+ * @public_data: public key for the signed attestation
-+ *		 (outPublic + name + qualifiedName)
-+ * @certificate_len: length of the certificate (bytes)
-+ * @certificate: certificate for the attestation signing key
-+ */
-+struct cpucp_sec_attest_info {
-+	__u8 pcr_data[SEC_PCR_DATA_BUF_SZ];
-+	__u8 pcr_num_reg;
-+	__u8 pcr_reg_len;
-+	__le16 pad0;
-+	__le32 nonce;
-+	__le16 pcr_quote_len;
-+	__u8 pcr_quote[SEC_PCR_QUOTE_BUF_SZ];
-+	__u8 quote_sig_len;
-+	__u8 quote_sig[SEC_SIGNATURE_BUF_SZ];
-+	__le16 pub_data_len;
-+	__u8 public_data[SEC_PUB_DATA_BUF_SZ];
-+	__le16 certificate_len;
-+	__u8 certificate[SEC_CERTIFICATE_BUF_SZ];
-+};
-+
-+/*
-+ * struct cpucp_dev_info_signed - device information signed by a secured device
-+ * @info: device information structure as defined above
-+ * @nonce: number only used once. random number provided by host. this number is
-+ *	   hashed and signed along with the device information.
-+ * @info_sig_len: length of the attestation signature (bytes)
-+ * @info_sig: signature of the info + nonce data.
-+ * @pub_data_len: length of the public data (bytes)
-+ * @public_data: public key info signed info data
-+ *		 (outPublic + name + qualifiedName)
-+ * @certificate_len: length of the certificate (bytes)
-+ * @certificate: certificate for the signing key
-+ */
-+struct cpucp_dev_info_signed {
-+	struct cpucp_info info;	/* assumed to be 64bit aligned */
-+	__le32 nonce;
-+	__le32 pad0;
-+	__u8 info_sig_len;
-+	__u8 info_sig[SEC_SIGNATURE_BUF_SZ];
-+	__le16 pub_data_len;
-+	__u8 public_data[SEC_PUB_DATA_BUF_SZ];
-+	__le16 certificate_len;
-+	__u8 certificate[SEC_CERTIFICATE_BUF_SZ];
-+};
-+
-+#define DCORE_MON_REGS_SZ	512
-+/*
-+ * struct dcore_monitor_regs_data - DCORE monitor regs data.
-+ * the structure follows sync manager block layout. Obsolete.
-+ * @mon_pay_addrl: array of payload address low bits.
-+ * @mon_pay_addrh: array of payload address high bits.
-+ * @mon_pay_data: array of payload data.
-+ * @mon_arm: array of monitor arm.
-+ * @mon_status: array of monitor status.
-+ */
-+struct dcore_monitor_regs_data {
-+	__le32 mon_pay_addrl[DCORE_MON_REGS_SZ];
-+	__le32 mon_pay_addrh[DCORE_MON_REGS_SZ];
-+	__le32 mon_pay_data[DCORE_MON_REGS_SZ];
-+	__le32 mon_arm[DCORE_MON_REGS_SZ];
-+	__le32 mon_status[DCORE_MON_REGS_SZ];
-+};
-+
-+/* contains SM data for each SYNC_MNGR (Obsolete) */
-+struct cpucp_monitor_dump {
-+	struct dcore_monitor_regs_data sync_mngr_w_s;
-+	struct dcore_monitor_regs_data sync_mngr_e_s;
-+	struct dcore_monitor_regs_data sync_mngr_w_n;
-+	struct dcore_monitor_regs_data sync_mngr_e_n;
-+};
-+
-+/*
-+ * The Type of the generic request (and other input arguments) will be fetched from user by reading
-+ * from "pkt_subidx" field in struct cpucp_packet.
-+ *
-+ * HL_PASSTHROUGHT_VERSIONS	- Fetch all firmware versions.
-+ * HL_GET_ERR_COUNTERS_CMD	- Command to get error counters
-+ * HL_GET_P_STATE		- get performance state
-+ */
-+enum hl_passthrough_type {
-+	HL_PASSTHROUGH_VERSIONS,
-+	HL_GET_ERR_COUNTERS_CMD,
-+	HL_GET_P_STATE,
-+};
-+
-+#endif /* CPUCP_IF_H */
-diff --git a/include/linux/habanalabs/hl_boot_if.h b/include/linux/habanalabs/hl_boot_if.h
-new file mode 100644
-index 000000000000..af5fb4ad77eb
--- /dev/null
-+++ b/include/linux/habanalabs/hl_boot_if.h
-@@ -0,0 +1,807 @@
-+/* SPDX-License-Identifier: GPL-2.0
-+ *
-+ * Copyright 2018-2023 HabanaLabs, Ltd.
-+ * All Rights Reserved.
-+ *
-+ */
-+
-+#ifndef HL_BOOT_IF_H
-+#define HL_BOOT_IF_H
-+
-+#define LKD_HARD_RESET_MAGIC		0xED7BD694 /* deprecated - do not use */
-+#define HL_POWER9_HOST_MAGIC		0x1DA30009
-+
-+#define BOOT_FIT_SRAM_OFFSET		0x200000
-+
-+#define VERSION_MAX_LEN			128
-+
-+enum cpu_boot_err {
-+	CPU_BOOT_ERR_DRAM_INIT_FAIL = 0,
-+	CPU_BOOT_ERR_FIT_CORRUPTED = 1,
-+	CPU_BOOT_ERR_TS_INIT_FAIL = 2,
-+	CPU_BOOT_ERR_DRAM_SKIPPED = 3,
-+	CPU_BOOT_ERR_BMC_WAIT_SKIPPED = 4,
-+	CPU_BOOT_ERR_NIC_DATA_NOT_RDY = 5,
-+	CPU_BOOT_ERR_NIC_FW_FAIL = 6,
-+	CPU_BOOT_ERR_SECURITY_NOT_RDY = 7,
-+	CPU_BOOT_ERR_SECURITY_FAIL = 8,
-+	CPU_BOOT_ERR_EFUSE_FAIL = 9,
-+	CPU_BOOT_ERR_PRI_IMG_VER_FAIL = 10,
-+	CPU_BOOT_ERR_SEC_IMG_VER_FAIL = 11,
-+	CPU_BOOT_ERR_PLL_FAIL = 12,
-+	CPU_BOOT_ERR_DEVICE_UNUSABLE_FAIL = 13,
-+	CPU_BOOT_ERR_BOOT_FW_CRIT_ERR = 18,
-+	CPU_BOOT_ERR_BINNING_FAIL = 19,
-+	CPU_BOOT_ERR_TPM_FAIL = 20,
-+	CPU_BOOT_ERR_TMP_THRESH_INIT_FAIL = 21,
-+	CPU_BOOT_ERR_EEPROM_FAIL = 22,
-+	CPU_BOOT_ERR_ENG_ARC_MEM_SCRUB_FAIL = 23,
-+	CPU_BOOT_ERR_ENABLED = 31,
-+	CPU_BOOT_ERR_SCND_EN = 63,
-+	CPU_BOOT_ERR_LAST = 64 /* we have 2 registers of 32 bits */
-+};
-+
-+/*
-+ * Mask for fatal failures
-+ * This mask contains all possible fatal failures, and a dynamic code
-+ * will clear the non-relevant ones.
-+ */
-+#define CPU_BOOT_ERR_FATAL_MASK					\
-+		((1 << CPU_BOOT_ERR_DRAM_INIT_FAIL) |		\
-+		 (1 << CPU_BOOT_ERR_PLL_FAIL) |			\
-+		 (1 << CPU_BOOT_ERR_BINNING_FAIL) |		\
-+		 (1 << CPU_BOOT_ERR_DRAM_SKIPPED) |		\
-+		 (1 << CPU_BOOT_ERR_ENG_ARC_MEM_SCRUB_FAIL) |	\
-+		 (1 << CPU_BOOT_ERR_EEPROM_FAIL))
-+
-+/*
-+ * CPU error bits in BOOT_ERROR registers
-+ *
-+ * CPU_BOOT_ERR0_DRAM_INIT_FAIL		DRAM initialization failed.
-+ *					DRAM is not reliable to use.
-+ *
-+ * CPU_BOOT_ERR0_FIT_CORRUPTED		FIT data integrity verification of the
-+ *					image provided by the host has failed.
-+ *
-+ * CPU_BOOT_ERR0_TS_INIT_FAIL		Thermal Sensor initialization failed.
-+ *					Boot continues as usual, but keep in
-+ *					mind this is a warning.
-+ *
-+ * CPU_BOOT_ERR0_DRAM_SKIPPED		DRAM initialization has been skipped.
-+ *					Skipping DRAM initialization has been
-+ *					requested (e.g. strap, command, etc.)
-+ *					and FW skipped the DRAM initialization.
-+ *					Host can initialize the DRAM.
-+ *
-+ * CPU_BOOT_ERR0_BMC_WAIT_SKIPPED	Waiting for BMC data will be skipped.
-+ *					Meaning the BMC data might not be
-+ *					available until reset.
-+ *
-+ * CPU_BOOT_ERR0_NIC_DATA_NOT_RDY	NIC data from BMC is not ready.
-+ *					BMC has not provided the NIC data yet.
-+ *					Once provided this bit will be cleared.
-+ *
-+ * CPU_BOOT_ERR0_NIC_FW_FAIL		NIC FW loading failed.
-+ *					The NIC FW loading and initialization
-+ *					failed. This means NICs are not usable.
-+ *
-+ * CPU_BOOT_ERR0_SECURITY_NOT_RDY	Chip security initialization has been
-+ *					started, but is not ready yet - chip
-+ *					cannot be accessed.
-+ *
-+ * CPU_BOOT_ERR0_SECURITY_FAIL		Security related tasks have failed.
-+ *					The tasks are security init (root of
-+ *					trust), boot authentication (chain of
-+ *					trust), data packets authentication.
-+ *
-+ * CPU_BOOT_ERR0_EFUSE_FAIL		Reading from eFuse failed.
-+ *					The PCI device ID might be wrong.
-+ *
-+ * CPU_BOOT_ERR0_PRI_IMG_VER_FAIL	Verification of primary image failed.
-+ *					It mean that ppboot checksum
-+ *					verification for the preboot primary
-+ *					image has failed to match expected
-+ *					checksum. Trying to program image again
-+ *					might solve this.
-+ *
-+ * CPU_BOOT_ERR0_SEC_IMG_VER_FAIL	Verification of secondary image failed.
-+ *					It mean that ppboot checksum
-+ *					verification for the preboot secondary
-+ *					image has failed to match expected
-+ *					checksum. Trying to program image again
-+ *					might solve this.
-+ *
-+ * CPU_BOOT_ERR0_PLL_FAIL		PLL settings failed, meaning that one
-+ *					of the PLLs remains in REF_CLK
-+ *
-+ * CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL	Device is unusable and customer support
-+ *					should be contacted.
-+ *
-+ * CPU_BOOT_ERR0_BOOT_FW_CRIT_ERR	Critical error was detected during
-+ *					the execution of ppboot or preboot.
-+ *					for example: stack overflow.
-+ *
-+ * CPU_BOOT_ERR0_BINNING_FAIL		Binning settings failed, meaning
-+ *					malfunctioning components might still be
-+ *					in use.
-+ *
-+ * CPU_BOOT_ERR0_TPM_FAIL		TPM verification flow failed.
-+ *
-+ * CPU_BOOT_ERR0_TMP_THRESH_INIT_FAIL	Failed to set threshold for tmperature
-+ *					sensor.
-+ *
-+ * CPU_BOOT_ERR_EEPROM_FAIL		Failed reading EEPROM data. Defaults
-+ *					are used.
-+ *
-+ * CPU_BOOT_ERR_ENG_ARC_MEM_SCRUB_FAIL	Failed scrubbing the Engines/ARCFarm
-+ *					memories. Boot disabled until reset.
-+ *
-+ * CPU_BOOT_ERR0_ENABLED		Error registers enabled.
-+ *					This is a main indication that the
-+ *					running FW populates the error
-+ *					registers. Meaning the error bits are
-+ *					not garbage, but actual error statuses.
-+ */
-+#define CPU_BOOT_ERR0_DRAM_INIT_FAIL		(1 << CPU_BOOT_ERR_DRAM_INIT_FAIL)
-+#define CPU_BOOT_ERR0_FIT_CORRUPTED		(1 << CPU_BOOT_ERR_FIT_CORRUPTED)
-+#define CPU_BOOT_ERR0_TS_INIT_FAIL		(1 << CPU_BOOT_ERR_TS_INIT_FAIL)
-+#define CPU_BOOT_ERR0_DRAM_SKIPPED		(1 << CPU_BOOT_ERR_DRAM_SKIPPED)
-+#define CPU_BOOT_ERR0_BMC_WAIT_SKIPPED		(1 << CPU_BOOT_ERR_BMC_WAIT_SKIPPED)
-+#define CPU_BOOT_ERR0_NIC_DATA_NOT_RDY		(1 << CPU_BOOT_ERR_NIC_DATA_NOT_RDY)
-+#define CPU_BOOT_ERR0_NIC_FW_FAIL		(1 << CPU_BOOT_ERR_NIC_FW_FAIL)
-+#define CPU_BOOT_ERR0_SECURITY_NOT_RDY		(1 << CPU_BOOT_ERR_SECURITY_NOT_RDY)
-+#define CPU_BOOT_ERR0_SECURITY_FAIL		(1 << CPU_BOOT_ERR_SECURITY_FAIL)
-+#define CPU_BOOT_ERR0_EFUSE_FAIL		(1 << CPU_BOOT_ERR_EFUSE_FAIL)
-+#define CPU_BOOT_ERR0_PRI_IMG_VER_FAIL		(1 << CPU_BOOT_ERR_PRI_IMG_VER_FAIL)
-+#define CPU_BOOT_ERR0_SEC_IMG_VER_FAIL		(1 << CPU_BOOT_ERR_SEC_IMG_VER_FAIL)
-+#define CPU_BOOT_ERR0_PLL_FAIL			(1 << CPU_BOOT_ERR_PLL_FAIL)
-+#define CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL	(1 << CPU_BOOT_ERR_DEVICE_UNUSABLE_FAIL)
-+#define CPU_BOOT_ERR0_BOOT_FW_CRIT_ERR		(1 << CPU_BOOT_ERR_BOOT_FW_CRIT_ERR)
-+#define CPU_BOOT_ERR0_BINNING_FAIL		(1 << CPU_BOOT_ERR_BINNING_FAIL)
-+#define CPU_BOOT_ERR0_TPM_FAIL			(1 << CPU_BOOT_ERR_TPM_FAIL)
-+#define CPU_BOOT_ERR0_TMP_THRESH_INIT_FAIL	(1 << CPU_BOOT_ERR_TMP_THRESH_INIT_FAIL)
-+#define CPU_BOOT_ERR0_EEPROM_FAIL		(1 << CPU_BOOT_ERR_EEPROM_FAIL)
-+#define CPU_BOOT_ERR0_ENG_ARC_MEM_SCRUB_FAIL	(1 << CPU_BOOT_ERR_ENG_ARC_MEM_SCRUB_FAIL)
-+#define CPU_BOOT_ERR0_ENABLED			(1 << CPU_BOOT_ERR_ENABLED)
-+#define CPU_BOOT_ERR1_ENABLED			(1 << CPU_BOOT_ERR_ENABLED)
-+
-+enum cpu_boot_dev_sts {
-+	CPU_BOOT_DEV_STS_SECURITY_EN = 0,
-+	CPU_BOOT_DEV_STS_DEBUG_EN = 1,
-+	CPU_BOOT_DEV_STS_WATCHDOG_EN = 2,
-+	CPU_BOOT_DEV_STS_DRAM_INIT_EN = 3,
-+	CPU_BOOT_DEV_STS_BMC_WAIT_EN = 4,
-+	CPU_BOOT_DEV_STS_E2E_CRED_EN = 5,
-+	CPU_BOOT_DEV_STS_HBM_CRED_EN = 6,
-+	CPU_BOOT_DEV_STS_RL_EN = 7,
-+	CPU_BOOT_DEV_STS_SRAM_SCR_EN = 8,
-+	CPU_BOOT_DEV_STS_DRAM_SCR_EN = 9,
-+	CPU_BOOT_DEV_STS_FW_HARD_RST_EN = 10,
-+	CPU_BOOT_DEV_STS_PLL_INFO_EN = 11,
-+	CPU_BOOT_DEV_STS_SP_SRAM_EN = 12,
-+	CPU_BOOT_DEV_STS_CLK_GATE_EN = 13,
-+	CPU_BOOT_DEV_STS_HBM_ECC_EN = 14,
-+	CPU_BOOT_DEV_STS_PKT_PI_ACK_EN = 15,
-+	CPU_BOOT_DEV_STS_FW_LD_COM_EN = 16,
-+	CPU_BOOT_DEV_STS_FW_IATU_CONF_EN = 17,
-+	CPU_BOOT_DEV_STS_FW_NIC_MAC_EN = 18,
-+	CPU_BOOT_DEV_STS_DYN_PLL_EN = 19,
-+	CPU_BOOT_DEV_STS_GIC_PRIVILEGED_EN = 20,
-+	CPU_BOOT_DEV_STS_EQ_INDEX_EN = 21,
-+	CPU_BOOT_DEV_STS_MULTI_IRQ_POLL_EN = 22,
-+	CPU_BOOT_DEV_STS_FW_NIC_STAT_XPCS91_EN = 23,
-+	CPU_BOOT_DEV_STS_FW_NIC_STAT_EXT_EN = 24,
-+	CPU_BOOT_DEV_STS_IS_IDLE_CHECK_EN = 25,
-+	CPU_BOOT_DEV_STS_MAP_HWMON_EN = 26,
-+	CPU_BOOT_DEV_STS_NIC_MEM_CLEAR_EN = 27,
-+	CPU_BOOT_DEV_STS_MMU_PGTBL_DRAM_EN = 28,
-+	CPU_BOOT_DEV_STS_ENABLED = 31,
-+	CPU_BOOT_DEV_STS_SCND_EN = 63,
-+	CPU_BOOT_DEV_STS_LAST = 64 /* we have 2 registers of 32 bits */
-+};
-+
-+/*
-+ * BOOT DEVICE STATUS bits in BOOT_DEVICE_STS registers
-+ *
-+ * CPU_BOOT_DEV_STS0_SECURITY_EN	Security is Enabled.
-+ *					This is an indication for security
-+ *					enabled in FW, which means that
-+ *					all conditions for security are met:
-+ *					device is indicated as security enabled,
-+ *					registers are protected, and device
-+ *					uses keys for image verification.
-+ *					Initialized in: preboot
-+ *
-+ * CPU_BOOT_DEV_STS0_DEBUG_EN		Debug is enabled.
-+ *					Enabled when JTAG or DEBUG is enabled
-+ *					in FW.
-+ *					Initialized in: preboot
-+ *
-+ * CPU_BOOT_DEV_STS0_WATCHDOG_EN	Watchdog is enabled.
-+ *					Watchdog is enabled in FW.
-+ *					Initialized in: preboot
-+ *
-+ * CPU_BOOT_DEV_STS0_DRAM_INIT_EN	DRAM initialization is enabled.
-+ *					DRAM initialization has been done in FW.
-+ *					Initialized in: u-boot
-+ *
-+ * CPU_BOOT_DEV_STS0_BMC_WAIT_EN	Waiting for BMC data enabled.
-+ *					If set, it means that during boot,
-+ *					FW waited for BMC data.
-+ *					Initialized in: u-boot
-+ *
-+ * CPU_BOOT_DEV_STS0_E2E_CRED_EN	E2E credits initialized.
-+ *					FW initialized E2E credits.
-+ *					Initialized in: u-boot
-+ *
-+ * CPU_BOOT_DEV_STS0_HBM_CRED_EN	HBM credits initialized.
-+ *					FW initialized HBM credits.
-+ *					Initialized in: u-boot
-+ *
-+ * CPU_BOOT_DEV_STS0_RL_EN		Rate limiter initialized.
-+ *					FW initialized rate limiter.
-+ *					Initialized in: u-boot
-+ *
-+ * CPU_BOOT_DEV_STS0_SRAM_SCR_EN	SRAM scrambler enabled.
-+ *					FW initialized SRAM scrambler.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_DRAM_SCR_EN	DRAM scrambler enabled.
-+ *					FW initialized DRAM scrambler.
-+ *					Initialized in: u-boot
-+ *
-+ * CPU_BOOT_DEV_STS0_FW_HARD_RST_EN	FW hard reset procedure is enabled.
-+ *					FW has the hard reset procedure
-+ *					implemented. This means that FW will
-+ *					perform hard reset procedure on
-+ *					receiving the halt-machine event.
-+ *					Initialized in: preboot, u-boot, linux
-+ *
-+ * CPU_BOOT_DEV_STS0_PLL_INFO_EN	FW retrieval of PLL info is enabled.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_SP_SRAM_EN		SP SRAM is initialized and available
-+ *					for use.
-+ *					Initialized in: preboot
-+ *
-+ * CPU_BOOT_DEV_STS0_CLK_GATE_EN	Clock Gating enabled.
-+ *					FW initialized Clock Gating.
-+ *					Initialized in: preboot
-+ *
-+ * CPU_BOOT_DEV_STS0_HBM_ECC_EN		HBM ECC handling Enabled.
-+ *					FW handles HBM ECC indications.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN	Packets ack value used in the armcpd
-+ *					is set to the PI counter.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_FW_LD_COM_EN	Flexible FW loading communication
-+ *					protocol is enabled.
-+ *					Initialized in: preboot
-+ *
-+ * CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN	FW iATU configuration is enabled.
-+ *					This bit if set, means the iATU has been
-+ *					configured and is ready for use.
-+ *					Initialized in: ppboot
-+ *
-+ * CPU_BOOT_DEV_STS0_FW_NIC_MAC_EN	NIC MAC channels init is done by FW and
-+ *					any access to them is done via the FW.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_DYN_PLL_EN		Dynamic PLL configuration is enabled.
-+ *					FW sends to host a bitmap of supported
-+ *					PLLs.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN	GIC access permission only from
-+ *					privileged entity. FW sets this status
-+ *					bit for host. If this bit is set then
-+ *					GIC can not be accessed from host.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_EQ_INDEX_EN	Event Queue (EQ) index is a running
-+ *					index for each new event sent to host.
-+ *					This is used as a method in host to
-+ *					identify that the waiting event in
-+ *					queue is actually a new event which
-+ *					was not served before.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN	Use multiple scratchpad interfaces to
-+ *					prevent IRQs overriding each other.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_FW_NIC_STAT_XPCS91_EN
-+ *					NIC STAT and XPCS91 access is restricted
-+ *					and is done via FW only.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_FW_NIC_STAT_EXT_EN
-+ *					NIC STAT get all is supported.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_IS_IDLE_CHECK_EN
-+ *					F/W checks if the device is idle by reading defined set
-+ *					of registers. It returns a bitmask of all the engines,
-+ *					where a bit is set if the engine is not idle.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_MAP_HWMON_EN
-+ *					If set, means f/w supports proprietary
-+ *					HWMON enum mapping to cpucp enums.
-+ *					Initialized in: linux
-+ *
-+ * CPU_BOOT_DEV_STS0_NIC_MEM_CLEAR_EN
-+ *					If set, means f/w supports nic hbm memory clear and
-+ *					tmr,txs hbm memory init.
-+ *					Initialized in: zephyr-mgmt
-+ *
-+ * CPU_BOOT_DEV_STS_MMU_PGTBL_DRAM_EN
-+ *					MMU page tables are located in DRAM.
-+ *					F/W initializes security settings for MMU
-+ *					page tables to reside in DRAM.
-+ *					Initialized in: zephyr-mgmt
-+ *
-+ * CPU_BOOT_DEV_STS0_ENABLED		Device status register enabled.
-+ *					This is a main indication that the
-+ *					running FW populates the device status
-+ *					register. Meaning the device status
-+ *					bits are not garbage, but actual
-+ *					statuses.
-+ *					Initialized in: preboot
-+ *
-+ */
-+#define CPU_BOOT_DEV_STS0_SECURITY_EN		(1 << CPU_BOOT_DEV_STS_SECURITY_EN)
-+#define CPU_BOOT_DEV_STS0_DEBUG_EN		(1 << CPU_BOOT_DEV_STS_DEBUG_EN)
-+#define CPU_BOOT_DEV_STS0_WATCHDOG_EN		(1 << CPU_BOOT_DEV_STS_WATCHDOG_EN)
-+#define CPU_BOOT_DEV_STS0_DRAM_INIT_EN		(1 << CPU_BOOT_DEV_STS_DRAM_INIT_EN)
-+#define CPU_BOOT_DEV_STS0_BMC_WAIT_EN		(1 << CPU_BOOT_DEV_STS_BMC_WAIT_EN)
-+#define CPU_BOOT_DEV_STS0_E2E_CRED_EN		(1 << CPU_BOOT_DEV_STS_E2E_CRED_EN)
-+#define CPU_BOOT_DEV_STS0_HBM_CRED_EN		(1 << CPU_BOOT_DEV_STS_HBM_CRED_EN)
-+#define CPU_BOOT_DEV_STS0_RL_EN			(1 << CPU_BOOT_DEV_STS_RL_EN)
-+#define CPU_BOOT_DEV_STS0_SRAM_SCR_EN		(1 << CPU_BOOT_DEV_STS_SRAM_SCR_EN)
-+#define CPU_BOOT_DEV_STS0_DRAM_SCR_EN		(1 << CPU_BOOT_DEV_STS_DRAM_SCR_EN)
-+#define CPU_BOOT_DEV_STS0_FW_HARD_RST_EN	(1 << CPU_BOOT_DEV_STS_FW_HARD_RST_EN)
-+#define CPU_BOOT_DEV_STS0_PLL_INFO_EN		(1 << CPU_BOOT_DEV_STS_PLL_INFO_EN)
-+#define CPU_BOOT_DEV_STS0_SP_SRAM_EN		(1 << CPU_BOOT_DEV_STS_SP_SRAM_EN)
-+#define CPU_BOOT_DEV_STS0_CLK_GATE_EN		(1 << CPU_BOOT_DEV_STS_CLK_GATE_EN)
-+#define CPU_BOOT_DEV_STS0_HBM_ECC_EN		(1 << CPU_BOOT_DEV_STS_HBM_ECC_EN)
-+#define CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN		(1 << CPU_BOOT_DEV_STS_PKT_PI_ACK_EN)
-+#define CPU_BOOT_DEV_STS0_FW_LD_COM_EN		(1 << CPU_BOOT_DEV_STS_FW_LD_COM_EN)
-+#define CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN	(1 << CPU_BOOT_DEV_STS_FW_IATU_CONF_EN)
-+#define CPU_BOOT_DEV_STS0_FW_NIC_MAC_EN		(1 << CPU_BOOT_DEV_STS_FW_NIC_MAC_EN)
-+#define CPU_BOOT_DEV_STS0_DYN_PLL_EN		(1 << CPU_BOOT_DEV_STS_DYN_PLL_EN)
-+#define CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN	(1 << CPU_BOOT_DEV_STS_GIC_PRIVILEGED_EN)
-+#define CPU_BOOT_DEV_STS0_EQ_INDEX_EN		(1 << CPU_BOOT_DEV_STS_EQ_INDEX_EN)
-+#define CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN	(1 << CPU_BOOT_DEV_STS_MULTI_IRQ_POLL_EN)
-+#define CPU_BOOT_DEV_STS0_FW_NIC_STAT_XPCS91_EN	(1 << CPU_BOOT_DEV_STS_FW_NIC_STAT_XPCS91_EN)
-+#define CPU_BOOT_DEV_STS0_FW_NIC_STAT_EXT_EN	(1 << CPU_BOOT_DEV_STS_FW_NIC_STAT_EXT_EN)
-+#define CPU_BOOT_DEV_STS0_IS_IDLE_CHECK_EN	(1 << CPU_BOOT_DEV_STS_IS_IDLE_CHECK_EN)
-+#define CPU_BOOT_DEV_STS0_MAP_HWMON_EN		(1 << CPU_BOOT_DEV_STS_MAP_HWMON_EN)
-+#define CPU_BOOT_DEV_STS0_NIC_MEM_CLEAR_EN	(1 << CPU_BOOT_DEV_STS_NIC_MEM_CLEAR_EN)
-+#define CPU_BOOT_DEV_STS0_MMU_PGTBL_DRAM_EN	(1 << CPU_BOOT_DEV_STS_MMU_PGTBL_DRAM_EN)
-+#define CPU_BOOT_DEV_STS0_ENABLED		(1 << CPU_BOOT_DEV_STS_ENABLED)
-+#define CPU_BOOT_DEV_STS1_ENABLED		(1 << CPU_BOOT_DEV_STS_ENABLED)
-+
-+enum cpu_boot_status {
-+	CPU_BOOT_STATUS_NA = 0,		/* Default value after reset of chip */
-+	CPU_BOOT_STATUS_IN_WFE = 1,
-+	CPU_BOOT_STATUS_DRAM_RDY = 2,
-+	CPU_BOOT_STATUS_SRAM_AVAIL = 3,
-+	CPU_BOOT_STATUS_IN_BTL = 4,	/* BTL is H/W FSM */
-+	CPU_BOOT_STATUS_IN_PREBOOT = 5,
-+	CPU_BOOT_STATUS_IN_SPL,		/* deprecated - not reported */
-+	CPU_BOOT_STATUS_IN_UBOOT = 7,
-+	CPU_BOOT_STATUS_DRAM_INIT_FAIL,	/* deprecated - will be removed */
-+	CPU_BOOT_STATUS_FIT_CORRUPTED,	/* deprecated - will be removed */
-+	/* U-Boot console prompt activated, commands are not processed */
-+	CPU_BOOT_STATUS_UBOOT_NOT_READY = 10,
-+	/* Finished NICs init, reported after DRAM and NICs */
-+	CPU_BOOT_STATUS_NIC_FW_RDY = 11,
-+	CPU_BOOT_STATUS_TS_INIT_FAIL,	/* deprecated - will be removed */
-+	CPU_BOOT_STATUS_DRAM_SKIPPED,	/* deprecated - will be removed */
-+	CPU_BOOT_STATUS_BMC_WAITING_SKIPPED, /* deprecated - will be removed */
-+	/* Last boot loader progress status, ready to receive commands */
-+	CPU_BOOT_STATUS_READY_TO_BOOT = 15,
-+	/* Internal Boot finished, ready for boot-fit */
-+	CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT = 16,
-+	/* Internal Security has been initialized, device can be accessed */
-+	CPU_BOOT_STATUS_SECURITY_READY = 17,
-+	/* FW component is preparing to shutdown and communication with host is not available */
-+	CPU_BOOT_STATUS_FW_SHUTDOWN_PREP = 18,
-+};
-+
-+enum kmd_msg {
-+	KMD_MSG_NA = 0,
-+	KMD_MSG_GOTO_WFE,
-+	KMD_MSG_FIT_RDY,
-+	KMD_MSG_SKIP_BMC,
-+	RESERVED,
-+	KMD_MSG_RST_DEV,
-+	KMD_MSG_LAST
-+};
-+
-+enum cpu_msg_status {
-+	CPU_MSG_CLR = 0,
-+	CPU_MSG_OK,
-+	CPU_MSG_ERR,
-+};
-+
-+/* communication registers mapping - consider ABI when changing */
-+struct cpu_dyn_regs {
-+	__le32 cpu_pq_base_addr_low;
-+	__le32 cpu_pq_base_addr_high;
-+	__le32 cpu_pq_length;
-+	__le32 cpu_pq_init_status;
-+	__le32 cpu_eq_base_addr_low;
-+	__le32 cpu_eq_base_addr_high;
-+	__le32 cpu_eq_length;
-+	__le32 cpu_eq_ci;
-+	__le32 cpu_cq_base_addr_low;
-+	__le32 cpu_cq_base_addr_high;
-+	__le32 cpu_cq_length;
-+	__le32 cpu_pf_pq_pi;
-+	__le32 cpu_boot_dev_sts0;
-+	__le32 cpu_boot_dev_sts1;
-+	__le32 cpu_boot_err0;
-+	__le32 cpu_boot_err1;
-+	__le32 cpu_boot_status;
-+	__le32 fw_upd_sts;
-+	__le32 fw_upd_cmd;
-+	__le32 fw_upd_pending_sts;
-+	__le32 fuse_ver_offset;
-+	__le32 preboot_ver_offset;
-+	__le32 uboot_ver_offset;
-+	__le32 hw_state;
-+	__le32 kmd_msg_to_cpu;
-+	__le32 cpu_cmd_status_to_host;
-+	__le32 gic_host_pi_upd_irq;
-+	__le32 gic_tpc_qm_irq_ctrl;
-+	__le32 gic_mme_qm_irq_ctrl;
-+	__le32 gic_dma_qm_irq_ctrl;
-+	__le32 gic_nic_qm_irq_ctrl;
-+	__le32 gic_dma_core_irq_ctrl;
-+	__le32 gic_host_halt_irq;
-+	__le32 gic_host_ints_irq;
-+	__le32 reserved0;
-+	__le32 gic_rot_qm_irq_ctrl;
-+	__le32 reserved1;
-+	__le32 eng_arc_irq_ctrl;
-+	__le32 reserved2[20];		/* reserve for future use */
-+};
-+
-+/* TODO: remove the desc magic after the code is updated to use message */
-+/* HCDM - Habana Communications Descriptor Magic */
-+#define HL_COMMS_DESC_MAGIC	0x4843444D
-+#define HL_COMMS_DESC_VER	3
-+
-+/* HCMv - Habana Communications Message + header version */
-+#define HL_COMMS_MSG_MAGIC_VALUE	0x48434D00
-+#define HL_COMMS_MSG_MAGIC_MASK		0xFFFFFF00
-+#define HL_COMMS_MSG_MAGIC_VER_MASK	0xFF
-+
-+#define HL_COMMS_MSG_MAGIC_VER(ver)	(HL_COMMS_MSG_MAGIC_VALUE |	\
-+					((ver) & HL_COMMS_MSG_MAGIC_VER_MASK))
-+#define HL_COMMS_MSG_MAGIC_V0		HL_COMMS_DESC_MAGIC
-+#define HL_COMMS_MSG_MAGIC_V1		HL_COMMS_MSG_MAGIC_VER(1)
-+#define HL_COMMS_MSG_MAGIC_V2		HL_COMMS_MSG_MAGIC_VER(2)
-+#define HL_COMMS_MSG_MAGIC_V3		HL_COMMS_MSG_MAGIC_VER(3)
-+
-+#define HL_COMMS_MSG_MAGIC		HL_COMMS_MSG_MAGIC_V3
-+
-+#define HL_COMMS_MSG_MAGIC_VALIDATE_MAGIC(magic)			\
-+		(((magic) & HL_COMMS_MSG_MAGIC_MASK) ==			\
-+		HL_COMMS_MSG_MAGIC_VALUE)
-+
-+#define HL_COMMS_MSG_MAGIC_VALIDATE_VERSION(magic, ver)			\
-+		(((magic) & HL_COMMS_MSG_MAGIC_VER_MASK) >=		\
-+		((ver) & HL_COMMS_MSG_MAGIC_VER_MASK))
-+
-+#define HL_COMMS_MSG_MAGIC_VALIDATE(magic, ver)				\
-+		(HL_COMMS_MSG_MAGIC_VALIDATE_MAGIC((magic)) &&		\
-+		HL_COMMS_MSG_MAGIC_VALIDATE_VERSION((magic), (ver)))
-+
-+enum comms_msg_type {
-+	HL_COMMS_DESC_TYPE = 0,
-+	HL_COMMS_RESET_CAUSE_TYPE = 1,
-+	HL_COMMS_FW_CFG_SKIP_TYPE = 2,
-+	HL_COMMS_BINNING_CONF_TYPE = 3,
-+};
-+
-+/*
-+ * Binning information shared between LKD and FW
-+ * @tpc_mask_l - TPC binning information lower 64 bit
-+ * @dec_mask - Decoder binning information
-+ * @dram_mask - DRAM binning information
-+ * @edma_mask - EDMA binning information
-+ * @mme_mask_l - MME binning information lower 32
-+ * @mme_mask_h - MME binning information upper 32
-+ * @rot_mask - Rotator binning information
-+ * @xbar_mask - xBAR binning information
-+ * @reserved - reserved field for future binning info w/o ABI change
-+ * @tpc_mask_h - TPC binning information upper 64 bit
-+ * @nic_mask - NIC binning information
-+ */
-+struct lkd_fw_binning_info {
-+	__le64 tpc_mask_l;
-+	__le32 dec_mask;
-+	__le32 dram_mask;
-+	__le32 edma_mask;
-+	__le32 mme_mask_l;
-+	__le32 mme_mask_h;
-+	__le32 rot_mask;
-+	__le32 xbar_mask;
-+	__le32 reserved0;
-+	__le64 tpc_mask_h;
-+	__le64 nic_mask;
-+	__le32 reserved1[8];
-+};
-+
-+/* TODO: remove this struct after the code is updated to use message */
-+/* this is the comms descriptor header - meta data */
-+struct comms_desc_header {
-+	__le32 magic;		/* magic for validation */
-+	__le32 crc32;		/* CRC32 of the descriptor w/o header */
-+	__le16 size;		/* size of the descriptor w/o header */
-+	__u8 version;	/* descriptor version */
-+	__u8 reserved[5];	/* pad to 64 bit */
-+};
-+
-+/* this is the comms message header - meta data */
-+struct comms_msg_header {
-+	__le32 magic;		/* magic for validation */
-+	__le32 crc32;		/* CRC32 of the message w/o header */
-+	__le16 size;		/* size of the message w/o header */
-+	__u8 version;	/* message payload version */
-+	__u8 type;		/* message type */
-+	__u8 reserved[4];	/* pad to 64 bit */
-+};
-+
-+enum lkd_fw_ascii_msg_lvls {
-+	LKD_FW_ASCII_MSG_ERR = 0,
-+	LKD_FW_ASCII_MSG_WRN = 1,
-+	LKD_FW_ASCII_MSG_INF = 2,
-+	LKD_FW_ASCII_MSG_DBG = 3,
-+};
-+
-+#define LKD_FW_ASCII_MSG_MAX_LEN		128
-+#define LKD_FW_ASCII_MSG_MAX			4	/* consider ABI when changing */
-+#define LKD_FW_ASCII_MSG_MIN_DESC_VERSION	3
-+
-+struct lkd_fw_ascii_msg {
-+	__u8 valid;
-+	__u8 msg_lvl;
-+	__u8 reserved[6];
-+	char msg[LKD_FW_ASCII_MSG_MAX_LEN];
-+};
-+
-+/* this is the main FW descriptor - consider ABI when changing */
-+struct lkd_fw_comms_desc {
-+	struct comms_desc_header header;
-+	struct cpu_dyn_regs cpu_dyn_regs;
-+	char fuse_ver[VERSION_MAX_LEN];
-+	char cur_fw_ver[VERSION_MAX_LEN];
-+	/* can be used for 1 more version w/o ABI change */
-+	char reserved0[VERSION_MAX_LEN];
-+	__le64 img_addr;	/* address for next FW component load */
-+	struct lkd_fw_binning_info binning_info;
-+	struct lkd_fw_ascii_msg ascii_msg[LKD_FW_ASCII_MSG_MAX];
-+	__le32 rsvd_mem_size_mb; /* reserved memory size [MB] for FW/SVE */
-+	char reserved1[4];
-+};
-+
-+enum comms_reset_cause {
-+	HL_RESET_CAUSE_UNKNOWN = 0,
-+	HL_RESET_CAUSE_HEARTBEAT = 1,
-+	HL_RESET_CAUSE_TDR = 2,
-+};
-+
-+/* TODO: remove define after struct name is aligned on all projects */
-+#define lkd_msg_comms lkd_fw_comms_msg
-+
-+/* this is the comms message descriptor */
-+struct lkd_fw_comms_msg {
-+	struct comms_msg_header header;
-+	/* union for future expantions of new messages */
-+	union {
-+		struct {
-+			struct cpu_dyn_regs cpu_dyn_regs;
-+			char fuse_ver[VERSION_MAX_LEN];
-+			char cur_fw_ver[VERSION_MAX_LEN];
-+			/* can be used for 1 more version w/o ABI change */
-+			char reserved0[VERSION_MAX_LEN];
-+			/* address for next FW component load */
-+			__le64 img_addr;
-+			struct lkd_fw_binning_info binning_info;
-+			struct lkd_fw_ascii_msg ascii_msg[LKD_FW_ASCII_MSG_MAX];
-+			/* reserved memory size [MB] for FW/SVE */
-+			__le32 rsvd_mem_size_mb;
-+			char reserved1[4];
-+		};
-+		struct {
-+			__u8 reset_cause;
-+		};
-+		struct {
-+			__u8 fw_cfg_skip; /* 1 - skip, 0 - don't skip */
-+		};
-+		struct lkd_fw_binning_info binning_conf;
-+	};
-+};
-+
-+/*
-+ * LKD commands:
-+ *
-+ * COMMS_NOOP			Used to clear the command register and no actual
-+ *				command is send.
-+ *
-+ * COMMS_CLR_STS		Clear status command - FW should clear the
-+ *				status register. Used for synchronization
-+ *				between the commands as part of the race free
-+ *				protocol.
-+ *
-+ * COMMS_RST_STATE		Reset the current communication state which is
-+ *				kept by FW for proper responses.
-+ *				Should be used in the beginning of the
-+ *				communication cycle to clean any leftovers from
-+ *				previous communication attempts.
-+ *
-+ * COMMS_PREP_DESC		Prepare descriptor for setting up the
-+ *				communication and other dynamic data:
-+ *				struct lkd_fw_comms_desc.
-+ *				This command has a parameter stating the next FW
-+ *				component size, so the FW can actually prepare a
-+ *				space for it and in the status response provide
-+ *				the descriptor offset. The Offset of the next FW
-+ *				data component is a part of the descriptor
-+ *				structure.
-+ *
-+ * COMMS_DATA_RDY		The FW data has been uploaded and is ready for
-+ *				validation.
-+ *
-+ * COMMS_EXEC			Execute the next FW component.
-+ *
-+ * COMMS_RST_DEV		Reset the device.
-+ *
-+ * COMMS_GOTO_WFE		Execute WFE command. Allowed only on non-secure
-+ *				devices.
-+ *
-+ * COMMS_SKIP_BMC		Perform actions required for BMC-less servers.
-+ *				Do not wait for BMC response.
-+ *
-+ * COMMS_PREP_DESC_ELBI		Same as COMMS_PREP_DESC only that the memory
-+ *				space is allocated in a ELBI access only
-+ *				address range.
-+ *
-+ */
-+enum comms_cmd {
-+	COMMS_NOOP = 0,
-+	COMMS_CLR_STS = 1,
-+	COMMS_RST_STATE = 2,
-+	COMMS_PREP_DESC = 3,
-+	COMMS_DATA_RDY = 4,
-+	COMMS_EXEC = 5,
-+	COMMS_RST_DEV = 6,
-+	COMMS_GOTO_WFE = 7,
-+	COMMS_SKIP_BMC = 8,
-+	COMMS_PREP_DESC_ELBI = 10,
-+	COMMS_INVLD_LAST
-+};
-+
-+#define COMMS_COMMAND_SIZE_SHIFT	0
-+#define COMMS_COMMAND_SIZE_MASK		0x1FFFFFF
-+#define COMMS_COMMAND_CMD_SHIFT		27
-+#define COMMS_COMMAND_CMD_MASK		0xF8000000
-+
-+/*
-+ * LKD command to FW register structure
-+ * @size	- FW component size
-+ * @cmd		- command from enum comms_cmd
-+ */
-+struct comms_command {
-+	union {		/* bit fields are only for FW use */
-+		struct {
-+			u32 size :25;		/* 32MB max. */
-+			u32 reserved :2;
-+			enum comms_cmd cmd :5;		/* 32 commands */
-+		};
-+		__le32 val;
-+	};
-+};
-+
-+/*
-+ * FW status
-+ *
-+ * COMMS_STS_NOOP		Used to clear the status register and no actual
-+ *				status is provided.
-+ *
-+ * COMMS_STS_ACK		Command has been received and recognized.
-+ *
-+ * COMMS_STS_OK			Command execution has finished successfully.
-+ *
-+ * COMMS_STS_ERR		Command execution was unsuccessful and resulted
-+ *				in error.
-+ *
-+ * COMMS_STS_VALID_ERR		FW validation has failed.
-+ *
-+ * COMMS_STS_TIMEOUT_ERR	Command execution has timed out.
-+ */
-+enum comms_sts {
-+	COMMS_STS_NOOP = 0,
-+	COMMS_STS_ACK = 1,
-+	COMMS_STS_OK = 2,
-+	COMMS_STS_ERR = 3,
-+	COMMS_STS_VALID_ERR = 4,
-+	COMMS_STS_TIMEOUT_ERR = 5,
-+	COMMS_STS_INVLD_LAST
-+};
-+
-+/* RAM types for FW components loading - defines the base address */
-+enum comms_ram_types {
-+	COMMS_SRAM = 0,
-+	COMMS_DRAM = 1,
-+};
-+
-+#define COMMS_STATUS_OFFSET_SHIFT	0
-+#define COMMS_STATUS_OFFSET_MASK	0x03FFFFFF
-+#define COMMS_STATUS_OFFSET_ALIGN_SHIFT	2
-+#define COMMS_STATUS_RAM_TYPE_SHIFT	26
-+#define COMMS_STATUS_RAM_TYPE_MASK	0x0C000000
-+#define COMMS_STATUS_STATUS_SHIFT	28
-+#define COMMS_STATUS_STATUS_MASK	0xF0000000
-+
-+/*
-+ * FW status to LKD register structure
-+ * @offset	- an offset from the base of the ram_type shifted right by
-+ *		  2 bits (always aligned to 32 bits).
-+ *		  Allows a maximum addressable offset of 256MB from RAM base.
-+ *		  Example: for real offset in RAM of 0x800000 (8MB), the value
-+ *		  in offset field is (0x800000 >> 2) = 0x200000.
-+ * @ram_type	- the RAM type that should be used for offset from
-+ *		  enum comms_ram_types
-+ * @status	- status from enum comms_sts
-+ */
-+struct comms_status {
-+	union {		/* bit fields are only for FW use */
-+		struct {
-+			u32 offset :26;
-+			enum comms_ram_types ram_type :2;
-+			enum comms_sts status :4;	/* 16 statuses */
-+		};
-+		__le32 val;
-+	};
-+};
-+
-+#define NAME_MAX_LEN	32 /* bytes */
-+struct hl_module_data {
-+	__u8 name[NAME_MAX_LEN];
-+	__u8 version[VERSION_MAX_LEN];
-+};
-+
-+/**
-+ * struct hl_component_versions - versions associated with hl component.
-+ * @struct_size: size of all the struct (including dynamic size of modules).
-+ * @modules_offset: offset of the modules field in this struct.
-+ * @component: version of the component itself.
-+ * @fw_os: Firmware OS Version.
-+ * @comp_name: Name of the component.
-+ * @modules_counter: number of set bits in modules_mask.
-+ * @reserved: reserved for future use.
-+ * @modules: versions of the component's modules. Elborated explanation in
-+ *              struct cpucp_versions.
-+ */
-+struct hl_component_versions {
-+	__le16 struct_size;
-+	__le16 modules_offset;
-+	__u8 component[VERSION_MAX_LEN];
-+	__u8 fw_os[VERSION_MAX_LEN];
-+	__u8 comp_name[NAME_MAX_LEN];
-+	__u8 modules_counter;
-+	__u8 reserved[3];
-+	struct hl_module_data modules[];
-+};
-+
-+/* Max size of fit size */
-+#define HL_FW_VERSIONS_FIT_SIZE	4096
-+
-+#endif /* HL_BOOT_IF_H */
-diff --git a/include/linux/hdmi.h b/include/linux/hdmi.h
-index c8ec982ff498..96bda41d9148 100644
--- a/include/linux/hdmi.h
-+++ b/include/linux/hdmi.h
-@@ -59,6 +59,15 @@ enum hdmi_infoframe_type {
- #define HDMI_DRM_INFOFRAME_SIZE    26
- #define HDMI_VENDOR_INFOFRAME_SIZE  4
- 
-+/*
-+ * HDMI 1.3a table 5-14 states that the largest InfoFrame_length is 27,
-+ * not including the packet header or checksum byte. We include the
-+ * checksum byte in HDMI_INFOFRAME_HEADER_SIZE, so this should allow
-+ * HDMI_INFOFRAME_SIZE(MAX) to be the largest buffer we could ever need
-+ * for any HDMI infoframe.
-+ */
-+#define HDMI_MAX_INFOFRAME_SIZE    27
-+
- #define HDMI_INFOFRAME_SIZE(type)	\
- 	(HDMI_INFOFRAME_HEADER_SIZE + HDMI_ ## type ## _INFOFRAME_SIZE)
- 
-@@ -170,19 +179,19 @@ struct hdmi_avi_infoframe {
- 	enum hdmi_infoframe_type type;
- 	unsigned char version;
- 	unsigned char length;
-+	bool itc;
-+	unsigned char pixel_repeat;
- 	enum hdmi_colorspace colorspace;
- 	enum hdmi_scan_mode scan_mode;
- 	enum hdmi_colorimetry colorimetry;
- 	enum hdmi_picture_aspect picture_aspect;
- 	enum hdmi_active_aspect active_aspect;
-	bool itc;
- 	enum hdmi_extended_colorimetry extended_colorimetry;
- 	enum hdmi_quantization_range quantization_range;
- 	enum hdmi_nups nups;
- 	unsigned char video_code;
- 	enum hdmi_ycc_quantization_range ycc_quantization_range;
- 	enum hdmi_content_type content_type;
-	unsigned char pixel_repeat;
- 	unsigned short top_bar;
- 	unsigned short bottom_bar;
- 	unsigned short left_bar;
-@@ -336,7 +345,12 @@ ssize_t hdmi_audio_infoframe_pack(struct hdmi_audio_infoframe *frame,
- 				  void *buffer, size_t size);
- ssize_t hdmi_audio_infoframe_pack_only(const struct hdmi_audio_infoframe *frame,
- 				       void *buffer, size_t size);
-int hdmi_audio_infoframe_check(struct hdmi_audio_infoframe *frame);
-+int hdmi_audio_infoframe_check(const struct hdmi_audio_infoframe *frame);
-+
-+struct dp_sdp;
-+ssize_t
-+hdmi_audio_infoframe_pack_for_dp(const struct hdmi_audio_infoframe *frame,
-+				 struct dp_sdp *sdp, u8 dp_version);
- 
- enum hdmi_3d_structure {
- 	HDMI_3D_STRUCTURE_INVALID = -1,
-@@ -431,7 +445,6 @@ ssize_t hdmi_infoframe_pack(union hdmi_infoframe *frame, void *buffer,
- 			    size_t size);
- ssize_t hdmi_infoframe_pack_only(const union hdmi_infoframe *frame,
- 				 void *buffer, size_t size);
-int hdmi_infoframe_check(union hdmi_infoframe *frame);
- int hdmi_infoframe_unpack(union hdmi_infoframe *frame,
- 			  const void *buffer, size_t size);
- void hdmi_infoframe_log(const char *level, struct device *dev,
-diff --git a/include/linux/hex.h b/include/linux/hex.h
-new file mode 100644
-index 000000000000..2618382e5b0c
--- /dev/null
-+++ b/include/linux/hex.h
-@@ -0,0 +1,35 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_HEX_H
-+#define _LINUX_HEX_H
-+
-+#include <linux/types.h>
-+
-+extern const char hex_asc[];
-+#define hex_asc_lo(x)	hex_asc[((x) & 0x0f)]
-+#define hex_asc_hi(x)	hex_asc[((x) & 0xf0) >> 4]
-+
-+static inline char *hex_byte_pack(char *buf, u8 byte)
-+{
-+	*buf++ = hex_asc_hi(byte);
-+	*buf++ = hex_asc_lo(byte);
-+	return buf;
-+}
-+
-+extern const char hex_asc_upper[];
-+#define hex_asc_upper_lo(x)	hex_asc_upper[((x) & 0x0f)]
-+#define hex_asc_upper_hi(x)	hex_asc_upper[((x) & 0xf0) >> 4]
-+
-+static inline char *hex_byte_pack_upper(char *buf, u8 byte)
-+{
-+	*buf++ = hex_asc_upper_hi(byte);
-+	*buf++ = hex_asc_upper_lo(byte);
-+	return buf;
-+}
-+
-+extern int hex_to_bin(unsigned char ch);
-+extern int __must_check hex2bin(u8 *dst, const char *src, size_t count);
-+extern char *bin2hex(char *dst, const void *src, size_t count);
-+
-+bool mac_pton(const char *s, u8 *mac);
-+
-+#endif
-diff --git a/include/linux/hfs_common.h b/include/linux/hfs_common.h
-new file mode 100644
-index 000000000000..dadb5e0aa8a3
--- /dev/null
-+++ b/include/linux/hfs_common.h
-@@ -0,0 +1,653 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * HFS/HFS+ common definitions, inline functions,
-+ * and shared functionality.
-+ */
-+
-+#ifndef _HFS_COMMON_H_
-+#define _HFS_COMMON_H_
-+
-+#ifdef pr_fmt
-+#undef pr_fmt
-+#endif
-+
-+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-+
-+#define hfs_dbg(fmt, ...)							\
-+	pr_debug("pid %d:%s:%d %s(): " fmt,					\
-+		 current->pid, __FILE__, __LINE__, __func__, ##__VA_ARGS__)	\
-+
-+/*
-+ * Format of structures on disk
-+ * Information taken from Apple Technote #1150 (HFS Plus Volume Format)
-+ */
-+
-+/* offsets to various blocks */
-+#define HFS_DD_BLK			0	/* Driver Descriptor block */
-+#define HFS_PMAP_BLK			1	/* First block of partition map */
-+#define HFS_MDB_BLK			2	/* Block (w/i partition) of MDB */
-+
-+/* magic numbers for various disk blocks */
-+#define HFS_DRVR_DESC_MAGIC		0x4552	/* "ER": driver descriptor map */
-+#define HFS_OLD_PMAP_MAGIC		0x5453	/* "TS": old-type partition map */
-+#define HFS_NEW_PMAP_MAGIC		0x504D	/* "PM": new-type partition map */
-+#define HFS_SUPER_MAGIC			0x4244	/* "BD": HFS MDB (super block) */
-+#define HFS_MFS_SUPER_MAGIC		0xD2D7	/* MFS MDB (super block) */
-+
-+#define HFSPLUS_VOLHEAD_SIG		0x482b
-+#define HFSPLUS_VOLHEAD_SIGX		0x4858
-+#define HFSPLUS_SUPER_MAGIC		0x482b
-+
-+#define HFSP_WRAP_MAGIC			0x4244
-+#define HFSP_WRAP_ATTRIB_SLOCK		0x8000
-+#define HFSP_WRAP_ATTRIB_SPARED		0x0200
-+
-+#define HFSP_WRAPOFF_SIG		0x00
-+#define HFSP_WRAPOFF_ATTRIB		0x0A
-+#define HFSP_WRAPOFF_ABLKSIZE		0x14
-+#define HFSP_WRAPOFF_ABLKSTART		0x1C
-+#define HFSP_WRAPOFF_EMBEDSIG		0x7C
-+#define HFSP_WRAPOFF_EMBEDEXT		0x7E
-+
-+#define HFSP_HARDLINK_TYPE		0x686c6e6b	/* 'hlnk' */
-+#define HFSP_HFSPLUS_CREATOR		0x6866732b	/* 'hfs+' */
-+
-+#define HFSP_SYMLINK_TYPE		0x736c6e6b	/* 'slnk' */
-+#define HFSP_SYMLINK_CREATOR		0x72686170	/* 'rhap' */
-+
-+#define HFSP_MOUNT_VERSION		0x482b4c78	/* 'H+Lx' */
-+
-+#define HFSP_HIDDENDIR_NAME \
-+	"\xe2\x90\x80\xe2\x90\x80\xe2\x90\x80\xe2\x90\x80HFS+ Private Data"
-+
-+/* various FIXED size parameters */
-+#define HFS_SECTOR_SIZE			512	/* size of an HFS sector */
-+#define HFS_SECTOR_SIZE_BITS		9	/* log_2(HFS_SECTOR_SIZE) */
-+#define HFS_MAX_VALENCE			32767U
-+
-+#define HFSPLUS_SECTOR_SIZE		HFS_SECTOR_SIZE
-+#define HFSPLUS_SECTOR_SHIFT		HFS_SECTOR_SIZE_BITS
-+#define HFSPLUS_VOLHEAD_SECTOR		2
-+#define HFSPLUS_MIN_VERSION		4
-+#define HFSPLUS_CURRENT_VERSION		5
-+
-+#define HFS_NAMELEN			31	/* maximum length of an HFS filename */
-+#define HFS_MAX_NAMELEN			128
-+
-+#define HFSPLUS_MAX_STRLEN		255
-+#define HFSPLUS_ATTR_MAX_STRLEN		127
-+
-+/* Meanings of the drAtrb field of the MDB,
-+ * Reference: _Inside Macintosh: Files_ p. 2-61
-+ */
-+#define HFS_SB_ATTRIB_HLOCK	(1 << 7)
-+#define HFS_SB_ATTRIB_UNMNT	(1 << 8)
-+#define HFS_SB_ATTRIB_SPARED	(1 << 9)
-+#define HFS_SB_ATTRIB_INCNSTNT	(1 << 11)
-+#define HFS_SB_ATTRIB_SLOCK	(1 << 15)
-+
-+/* values for hfs_cat_rec.cdrType */
-+#define HFS_CDR_DIR		0x01	/* folder (directory) */
-+#define HFS_CDR_FIL		0x02	/* file */
-+#define HFS_CDR_THD		0x03	/* folder (directory) thread */
-+#define HFS_CDR_FTH		0x04	/* file thread */
-+
-+/* legal values for hfs_ext_key.FkType and hfs_file.fork */
-+#define HFS_FK_DATA		0x00
-+#define HFS_FK_RSRC		0xFF
-+
-+/* bits in hfs_fil_entry.Flags */
-+#define HFS_FIL_LOCK		0x01	/* locked */
-+#define HFS_FIL_THD		0x02	/* file thread */
-+#define HFS_FIL_DOPEN		0x04	/* data fork open */
-+#define HFS_FIL_ROPEN		0x08	/* resource fork open */
-+#define HFS_FIL_DIR		0x10	/* directory (always clear) */
-+#define HFS_FIL_NOCOPY		0x40	/* copy-protected file */
-+#define HFS_FIL_USED		0x80	/* open */
-+
-+/* bits in hfs_dir_entry.Flags. dirflags is 16 bits. */
-+#define HFS_DIR_LOCK		0x01	/* locked */
-+#define HFS_DIR_THD		0x02	/* directory thread */
-+#define HFS_DIR_INEXPFOLDER	0x04	/* in a shared area */
-+#define HFS_DIR_MOUNTED		0x08	/* mounted */
-+#define HFS_DIR_DIR		0x10	/* directory (always set) */
-+#define HFS_DIR_EXPFOLDER	0x20	/* share point */
-+
-+/* bits hfs_finfo.fdFlags */
-+#define HFS_FLG_INITED		0x0100
-+#define HFS_FLG_LOCKED		0x1000
-+#define HFS_FLG_INVISIBLE	0x4000
-+
-+/* Some special File ID numbers */
-+#define HFS_POR_CNID		1	/* Parent Of the Root */
-+#define HFSPLUS_POR_CNID	HFS_POR_CNID
-+#define HFS_ROOT_CNID		2	/* ROOT directory */
-+#define HFSPLUS_ROOT_CNID	HFS_ROOT_CNID
-+#define HFS_EXT_CNID		3	/* EXTents B-tree */
-+#define HFSPLUS_EXT_CNID	HFS_EXT_CNID
-+#define HFS_CAT_CNID		4	/* CATalog B-tree */
-+#define HFSPLUS_CAT_CNID	HFS_CAT_CNID
-+#define HFS_BAD_CNID		5	/* BAD blocks file */
-+#define HFSPLUS_BAD_CNID	HFS_BAD_CNID
-+#define HFS_ALLOC_CNID		6	/* ALLOCation file (HFS+) */
-+#define HFSPLUS_ALLOC_CNID	HFS_ALLOC_CNID
-+#define HFS_START_CNID		7	/* STARTup file (HFS+) */
-+#define HFSPLUS_START_CNID	HFS_START_CNID
-+#define HFS_ATTR_CNID		8	/* ATTRibutes file (HFS+) */
-+#define HFSPLUS_ATTR_CNID	HFS_ATTR_CNID
-+#define HFS_EXCH_CNID		15	/* ExchangeFiles temp id */
-+#define HFSPLUS_EXCH_CNID	HFS_EXCH_CNID
-+#define HFS_FIRSTUSER_CNID	16	/* first available user id */
-+#define HFSPLUS_FIRSTUSER_CNID	HFS_FIRSTUSER_CNID
-+
-+/*======== HFS/HFS+ structures as they appear on the disk ========*/
-+
-+typedef __be32 hfsplus_cnid;
-+typedef __be16 hfsplus_unichr;
-+
-+/* Pascal-style string of up to 31 characters */
-+struct hfs_name {
-+	u8 len;
-+	u8 name[HFS_NAMELEN];
-+} __packed;
-+
-+/* A "string" as used in filenames, etc. */
-+struct hfsplus_unistr {
-+	__be16 length;
-+	hfsplus_unichr unicode[HFSPLUS_MAX_STRLEN];
-+} __packed;
-+
-+/*
-+ * A "string" is used in attributes file
-+ * for name of extended attribute
-+ */
-+struct hfsplus_attr_unistr {
-+	__be16 length;
-+	hfsplus_unichr unicode[HFSPLUS_ATTR_MAX_STRLEN];
-+} __packed;
-+
-+struct hfs_extent {
-+	__be16 block;
-+	__be16 count;
-+};
-+typedef struct hfs_extent hfs_extent_rec[3];
-+
-+/* A single contiguous area of a file */
-+struct hfsplus_extent {
-+	__be32 start_block;
-+	__be32 block_count;
-+} __packed;
-+typedef struct hfsplus_extent hfsplus_extent_rec[8];
-+
-+/* Information for a "Fork" in a file */
-+struct hfsplus_fork_raw {
-+	__be64 total_size;
-+	__be32 clump_size;
-+	__be32 total_blocks;
-+	hfsplus_extent_rec extents;
-+} __packed;
-+
-+struct hfs_mdb {
-+	__be16 drSigWord;		/* Signature word indicating fs type */
-+	__be32 drCrDate;		/* fs creation date/time */
-+	__be32 drLsMod;			/* fs modification date/time */
-+	__be16 drAtrb;			/* fs attributes */
-+	__be16 drNmFls;			/* number of files in root directory */
-+	__be16 drVBMSt;			/* location (in 512-byte blocks)
-+					   of the volume bitmap */
-+	__be16 drAllocPtr;		/* location (in allocation blocks)
-+					   to begin next allocation search */
-+	__be16 drNmAlBlks;		/* number of allocation blocks */
-+	__be32 drAlBlkSiz;		/* bytes in an allocation block */
-+	__be32 drClpSiz;		/* clumpsize, the number of bytes to
-+					   allocate when extending a file */
-+	__be16 drAlBlSt;		/* location (in 512-byte blocks)
-+					   of the first allocation block */
-+	__be32 drNxtCNID;		/* CNID to assign to the next
-+					   file or directory created */
-+	__be16 drFreeBks;		/* number of free allocation blocks */
-+	u8 drVN[28];			/* the volume label */
-+	__be32 drVolBkUp;		/* fs backup date/time */
-+	__be16 drVSeqNum;		/* backup sequence number */
-+	__be32 drWrCnt;			/* fs write count */
-+	__be32 drXTClpSiz;		/* clumpsize for the extents B-tree */
-+	__be32 drCTClpSiz;		/* clumpsize for the catalog B-tree */
-+	__be16 drNmRtDirs;		/* number of directories in
-+					   the root directory */
-+	__be32 drFilCnt;		/* number of files in the fs */
-+	__be32 drDirCnt;		/* number of directories in the fs */
-+	u8 drFndrInfo[32];		/* data used by the Finder */
-+	__be16 drEmbedSigWord;		/* embedded volume signature */
-+	__be32 drEmbedExtent;		/* starting block number (xdrStABN)
-+					   and number of allocation blocks
-+					   (xdrNumABlks) occupied by embedded
-+					   volume */
-+	__be32 drXTFlSize;		/* bytes in the extents B-tree */
-+	hfs_extent_rec drXTExtRec;	/* extents B-tree's first 3 extents */
-+	__be32 drCTFlSize;		/* bytes in the catalog B-tree */
-+	hfs_extent_rec drCTExtRec;	/* catalog B-tree's first 3 extents */
-+} __packed;
-+
-+/* HFS+ Volume Header */
-+struct hfsplus_vh {
-+	__be16 signature;
-+	__be16 version;
-+	__be32 attributes;
-+	__be32 last_mount_vers;
-+	u32 reserved;
-+
-+	__be32 create_date;
-+	__be32 modify_date;
-+	__be32 backup_date;
-+	__be32 checked_date;
-+
-+	__be32 file_count;
-+	__be32 folder_count;
-+
-+	__be32 blocksize;
-+	__be32 total_blocks;
-+	__be32 free_blocks;
-+
-+	__be32 next_alloc;
-+	__be32 rsrc_clump_sz;
-+	__be32 data_clump_sz;
-+	hfsplus_cnid next_cnid;
-+
-+	__be32 write_count;
-+	__be64 encodings_bmp;
-+
-+	u32 finder_info[8];
-+
-+	struct hfsplus_fork_raw alloc_file;
-+	struct hfsplus_fork_raw ext_file;
-+	struct hfsplus_fork_raw cat_file;
-+	struct hfsplus_fork_raw attr_file;
-+	struct hfsplus_fork_raw start_file;
-+} __packed;
-+
-+/* HFS+ volume attributes */
-+#define HFSPLUS_VOL_UNMNT		(1 << 8)
-+#define HFSPLUS_VOL_SPARE_BLK		(1 << 9)
-+#define HFSPLUS_VOL_NOCACHE		(1 << 10)
-+#define HFSPLUS_VOL_INCNSTNT		(1 << 11)
-+#define HFSPLUS_VOL_NODEID_REUSED	(1 << 12)
-+#define HFSPLUS_VOL_JOURNALED		(1 << 13)
-+#define HFSPLUS_VOL_SOFTLOCK		(1 << 15)
-+#define HFSPLUS_VOL_UNUSED_NODE_FIX	(1 << 31)
-+
-+struct hfs_point {
-+	__be16 v;
-+	__be16 h;
-+} __packed;
-+
-+typedef struct hfs_point hfsp_point;
-+
-+struct hfs_rect {
-+	__be16 top;
-+	__be16 left;
-+	__be16 bottom;
-+	__be16 right;
-+} __packed;
-+
-+typedef struct hfs_rect hfsp_rect;
-+
-+struct hfs_finfo {
-+	__be32 fdType;
-+	__be32 fdCreator;
-+	__be16 fdFlags;
-+	struct hfs_point fdLocation;
-+	__be16 fdFldr;
-+} __packed;
-+
-+typedef struct hfs_finfo FInfo;
-+
-+struct hfs_fxinfo {
-+	__be16 fdIconID;
-+	u8 fdUnused[8];
-+	__be16 fdComment;
-+	__be32 fdPutAway;
-+} __packed;
-+
-+typedef struct hfs_fxinfo FXInfo;
-+
-+struct hfs_dinfo {
-+	struct hfs_rect frRect;
-+	__be16 frFlags;
-+	struct hfs_point frLocation;
-+	__be16 frView;
-+} __packed;
-+
-+typedef struct hfs_dinfo DInfo;
-+
-+struct hfs_dxinfo {
-+	struct hfs_point frScroll;
-+	__be32 frOpenChain;
-+	__be16 frUnused;
-+	__be16 frComment;
-+	__be32 frPutAway;
-+} __packed;
-+
-+typedef struct hfs_dxinfo DXInfo;
-+
-+union hfs_finder_info {
-+	struct {
-+		struct hfs_finfo finfo;
-+		struct hfs_fxinfo fxinfo;
-+	} file;
-+	struct {
-+		struct hfs_dinfo dinfo;
-+		struct hfs_dxinfo dxinfo;
-+	} dir;
-+} __packed;
-+
-+/* The key used in the catalog b-tree: */
-+struct hfs_cat_key {
-+	u8 key_len;		/* number of bytes in the key */
-+	u8 reserved;		/* padding */
-+	__be32 ParID;		/* CNID of the parent dir */
-+	struct hfs_name	CName;	/* The filename of the entry */
-+} __packed;
-+
-+/* HFS+ catalog entry key */
-+struct hfsplus_cat_key {
-+	__be16 key_len;
-+	hfsplus_cnid parent;
-+	struct hfsplus_unistr name;
-+} __packed;
-+
-+#define HFSPLUS_CAT_KEYLEN	(sizeof(struct hfsplus_cat_key))
-+
-+/* The key used in the extents b-tree: */
-+struct hfs_ext_key {
-+	u8 key_len;		/* number of bytes in the key */
-+	u8 FkType;		/* HFS_FK_{DATA,RSRC} */
-+	__be32 FNum;		/* The File ID of the file */
-+	__be16 FABN;		/* allocation blocks number*/
-+} __packed;
-+
-+/* HFS+ extents tree key */
-+struct hfsplus_ext_key {
-+	__be16 key_len;
-+	u8 fork_type;
-+	u8 pad;
-+	hfsplus_cnid cnid;
-+	__be32 start_block;
-+} __packed;
-+
-+#define HFSPLUS_EXT_KEYLEN	sizeof(struct hfsplus_ext_key)
-+
-+typedef union hfs_btree_key {
-+	u8 key_len;			/* number of bytes in the key */
-+	struct hfs_cat_key cat;
-+	struct hfs_ext_key ext;
-+} hfs_btree_key;
-+
-+#define HFS_MAX_CAT_KEYLEN	(sizeof(struct hfs_cat_key) - sizeof(u8))
-+#define HFS_MAX_EXT_KEYLEN	(sizeof(struct hfs_ext_key) - sizeof(u8))
-+
-+typedef union hfs_btree_key btree_key;
-+
-+/* The catalog record for a file */
-+struct hfs_cat_file {
-+	s8 type;			/* The type of entry */
-+	u8 reserved;
-+	u8 Flags;			/* Flags such as read-only */
-+	s8 Typ;				/* file version number = 0 */
-+	struct hfs_finfo UsrWds;	/* data used by the Finder */
-+	__be32 FlNum;			/* The CNID */
-+	__be16 StBlk;			/* obsolete */
-+	__be32 LgLen;			/* The logical EOF of the data fork*/
-+	__be32 PyLen;			/* The physical EOF of the data fork */
-+	__be16 RStBlk;			/* obsolete */
-+	__be32 RLgLen;			/* The logical EOF of the rsrc fork */
-+	__be32 RPyLen;			/* The physical EOF of the rsrc fork */
-+	__be32 CrDat;			/* The creation date */
-+	__be32 MdDat;			/* The modified date */
-+	__be32 BkDat;			/* The last backup date */
-+	struct hfs_fxinfo FndrInfo;	/* more data for the Finder */
-+	__be16 ClpSize;			/* number of bytes to allocate
-+					   when extending files */
-+	hfs_extent_rec ExtRec;		/* first extent record
-+					   for the data fork */
-+	hfs_extent_rec RExtRec;		/* first extent record
-+					   for the resource fork */
-+	u32 Resrv;			/* reserved by Apple */
-+} __packed;
-+
-+/* the catalog record for a directory */
-+struct hfs_cat_dir {
-+	s8 type;			/* The type of entry */
-+	u8 reserved;
-+	__be16 Flags;			/* flags */
-+	__be16 Val;			/* Valence: number of files and
-+					   dirs in the directory */
-+	__be32 DirID;			/* The CNID */
-+	__be32 CrDat;			/* The creation date */
-+	__be32 MdDat;			/* The modification date */
-+	__be32 BkDat;			/* The last backup date */
-+	struct hfs_dinfo UsrInfo;	/* data used by the Finder */
-+	struct hfs_dxinfo FndrInfo;	/* more data used by Finder */
-+	u8 Resrv[16];			/* reserved by Apple */
-+} __packed;
-+
-+/* the catalog record for a thread */
-+struct hfs_cat_thread {
-+	s8 type;			/* The type of entry */
-+	u8 reserved[9];			/* reserved by Apple */
-+	__be32 ParID;			/* CNID of parent directory */
-+	struct hfs_name CName;		/* The name of this entry */
-+}  __packed;
-+
-+/* A catalog tree record */
-+typedef union hfs_cat_rec {
-+	s8 type;			/* The type of entry */
-+	struct hfs_cat_file file;
-+	struct hfs_cat_dir dir;
-+	struct hfs_cat_thread thread;
-+} hfs_cat_rec;
-+
-+/* POSIX permissions */
-+struct hfsplus_perm {
-+	__be32 owner;
-+	__be32 group;
-+	u8  rootflags;
-+	u8  userflags;
-+	__be16 mode;
-+	__be32 dev;
-+} __packed;
-+
-+#define HFSPLUS_FLG_NODUMP	0x01
-+#define HFSPLUS_FLG_IMMUTABLE	0x02
-+#define HFSPLUS_FLG_APPEND	0x04
-+
-+/* HFS/HFS+ BTree node descriptor */
-+struct hfs_bnode_desc {
-+	__be32 next;		/* (V) Number of the next node at this level */
-+	__be32 prev;		/* (V) Number of the prev node at this level */
-+	u8 type;		/* (F) The type of node */
-+	u8 height;		/* (F) The level of this node (leaves=1) */
-+	__be16 num_recs;	/* (V) The number of records in this node */
-+	u16 reserved;
-+} __packed;
-+
-+/* HFS/HFS+ BTree node types */
-+#define HFS_NODE_INDEX	0x00	/* An internal (index) node */
-+#define HFS_NODE_HEADER	0x01	/* The tree header node (node 0) */
-+#define HFS_NODE_MAP	0x02	/* Holds part of the bitmap of used nodes */
-+#define HFS_NODE_LEAF	0xFF	/* A leaf (ndNHeight==1) node */
-+
-+/* HFS/HFS+ BTree header */
-+struct hfs_btree_header_rec {
-+	__be16 depth;		/* (V) The number of levels in this B-tree */
-+	__be32 root;		/* (V) The node number of the root node */
-+	__be32 leaf_count;	/* (V) The number of leaf records */
-+	__be32 leaf_head;	/* (V) The number of the first leaf node */
-+	__be32 leaf_tail;	/* (V) The number of the last leaf node */
-+	__be16 node_size;	/* (F) The number of bytes in a node (=512) */
-+	__be16 max_key_len;	/* (F) The length of a key in an index node */
-+	__be32 node_count;	/* (V) The total number of nodes */
-+	__be32 free_nodes;	/* (V) The number of unused nodes */
-+	u16 reserved1;
-+	__be32 clump_size;	/* (F) clump size. not usually used. */
-+	u8 btree_type;		/* (F) BTree type */
-+	u8 key_type;
-+	__be32 attributes;	/* (F) attributes */
-+	u32 reserved3[16];
-+} __packed;
-+
-+/* BTree attributes */
-+#define BTREE_ATTR_BADCLOSE	0x00000001	/* b-tree not closed properly. not
-+						   used by hfsplus. */
-+#define HFS_TREE_BIGKEYS	0x00000002	/* key length is u16 instead of u8.
-+						   used by hfsplus. */
-+#define HFS_TREE_VARIDXKEYS	0x00000004	/* variable key length instead of
-+						   max key length. use din catalog
-+						   b-tree but not in extents
-+						   b-tree (hfsplus). */
-+
-+/* HFS+ BTree misc info */
-+#define HFSPLUS_TREE_HEAD			0
-+#define HFSPLUS_NODE_MXSZ			32768
-+#define HFSPLUS_ATTR_TREE_NODE_SIZE		8192
-+#define HFSPLUS_BTREE_HDR_NODE_RECS_COUNT	3
-+#define HFSPLUS_BTREE_HDR_USER_BYTES		128
-+
-+/* btree key type */
-+#define HFSPLUS_KEY_CASEFOLDING		0xCF	/* case-insensitive */
-+#define HFSPLUS_KEY_BINARY		0xBC	/* case-sensitive */
-+
-+/* HFS+ folder data (part of an hfsplus_cat_entry) */
-+struct hfsplus_cat_folder {
-+	__be16 type;
-+	__be16 flags;
-+	__be32 valence;
-+	hfsplus_cnid id;
-+	__be32 create_date;
-+	__be32 content_mod_date;
-+	__be32 attribute_mod_date;
-+	__be32 access_date;
-+	__be32 backup_date;
-+	struct hfsplus_perm permissions;
-+	struct_group_attr(info, __packed,
-+		DInfo user_info;
-+		DXInfo finder_info;
-+	);
-+	__be32 text_encoding;
-+	__be32 subfolders;	/* Subfolder count in HFSX. Reserved in HFS+. */
-+} __packed;
-+
-+/* HFS+ file data (part of a cat_entry) */
-+struct hfsplus_cat_file {
-+	__be16 type;
-+	__be16 flags;
-+	u32 reserved1;
-+	hfsplus_cnid id;
-+	__be32 create_date;
-+	__be32 content_mod_date;
-+	__be32 attribute_mod_date;
-+	__be32 access_date;
-+	__be32 backup_date;
-+	struct hfsplus_perm permissions;
-+	struct_group_attr(info, __packed,
-+		FInfo user_info;
-+		FXInfo finder_info;
-+	);
-+	__be32 text_encoding;
-+	u32 reserved2;
-+
-+	struct hfsplus_fork_raw data_fork;
-+	struct hfsplus_fork_raw rsrc_fork;
-+} __packed;
-+
-+/* File and folder flag bits */
-+#define HFSPLUS_FILE_LOCKED		0x0001
-+#define HFSPLUS_FILE_THREAD_EXISTS	0x0002
-+#define HFSPLUS_XATTR_EXISTS		0x0004
-+#define HFSPLUS_ACL_EXISTS		0x0008
-+#define HFSPLUS_HAS_FOLDER_COUNT	0x0010	/* Folder has subfolder count
-+						 * (HFSX only) */
-+
-+/* HFS+ catalog thread (part of a cat_entry) */
-+struct hfsplus_cat_thread {
-+	__be16 type;
-+	s16 reserved;
-+	hfsplus_cnid parentID;
-+	struct hfsplus_unistr nodeName;
-+} __packed;
-+
-+#define HFSPLUS_MIN_THREAD_SZ		10
-+
-+/* A data record in the catalog tree */
-+typedef union {
-+	__be16 type;
-+	struct hfsplus_cat_folder folder;
-+	struct hfsplus_cat_file file;
-+	struct hfsplus_cat_thread thread;
-+} __packed hfsplus_cat_entry;
-+
-+/* HFS+ catalog entry type */
-+#define HFSPLUS_FOLDER		0x0001
-+#define HFSPLUS_FILE		0x0002
-+#define HFSPLUS_FOLDER_THREAD	0x0003
-+#define HFSPLUS_FILE_THREAD	0x0004
-+
-+#define HFSPLUS_XATTR_FINDER_INFO_NAME	"com.apple.FinderInfo"
-+#define HFSPLUS_XATTR_ACL_NAME		"com.apple.system.Security"
-+
-+#define HFSPLUS_ATTR_INLINE_DATA	0x10
-+#define HFSPLUS_ATTR_FORK_DATA		0x20
-+#define HFSPLUS_ATTR_EXTENTS		0x30
-+
-+/* HFS+ attributes tree key */
-+struct hfsplus_attr_key {
-+	__be16 key_len;
-+	__be16 pad;
-+	hfsplus_cnid cnid;
-+	__be32 start_block;
-+	struct hfsplus_attr_unistr key_name;
-+} __packed;
-+
-+#define HFSPLUS_ATTR_KEYLEN	sizeof(struct hfsplus_attr_key)
-+
-+/* HFS+ fork data attribute */
-+struct hfsplus_attr_fork_data {
-+	__be32 record_type;
-+	__be32 reserved;
-+	struct hfsplus_fork_raw the_fork;
-+} __packed;
-+
-+/* HFS+ extension attribute */
-+struct hfsplus_attr_extents {
-+	__be32 record_type;
-+	__be32 reserved;
-+	struct hfsplus_extent extents;
-+} __packed;
-+
-+#define HFSPLUS_MAX_INLINE_DATA_SIZE	3802
-+
-+/* HFS+ attribute inline data */
-+struct hfsplus_attr_inline_data {
-+	__be32 record_type;
-+	__be32 reserved1;
-+	u8 reserved2[6];
-+	__be16 length;
-+	u8 raw_bytes[HFSPLUS_MAX_INLINE_DATA_SIZE];
-+} __packed;
-+
-+/* A data record in the attributes tree */
-+typedef union {
-+	__be32 record_type;
-+	struct hfsplus_attr_fork_data fork_data;
-+	struct hfsplus_attr_extents extents;
-+	struct hfsplus_attr_inline_data inline_data;
-+} __packed hfsplus_attr_entry;
-+
-+/* HFS+ generic BTree key */
-+typedef union {
-+	__be16 key_len;
-+	struct hfsplus_cat_key cat;
-+	struct hfsplus_ext_key ext;
-+	struct hfsplus_attr_key attr;
-+} __packed hfsplus_btree_key;
-+
-+#endif /* _HFS_COMMON_H_ */
-diff --git a/include/linux/hid-over-i2c.h b/include/linux/hid-over-i2c.h
-new file mode 100644
-index 000000000000..3b1a0208a6b8
--- /dev/null
-+++ b/include/linux/hid-over-i2c.h
-@@ -0,0 +1,117 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/* Copyright 2024 Intel Corporation */
-+
-+#include <linux/bits.h>
-+
-+#ifndef _HID_OVER_I2C_H_
-+#define _HID_OVER_I2C_H_
-+
-+#define HIDI2C_REG_LEN		sizeof(__le16)
-+
-+/* Input report type definition in HIDI2C protocol */
-+enum hidi2c_report_type {
-+	HIDI2C_RESERVED = 0,
-+	HIDI2C_INPUT,
-+	HIDI2C_OUTPUT,
-+	HIDI2C_FEATURE,
-+};
-+
-+/* Power state type definition in HIDI2C protocol */
-+enum hidi2c_power_state {
-+	HIDI2C_ON,
-+	HIDI2C_SLEEP,
-+};
-+
-+/* Opcode type definition in HIDI2C protocol */
-+enum hidi2c_opcode {
-+	HIDI2C_RESET = 1,
-+	HIDI2C_GET_REPORT,
-+	HIDI2C_SET_REPORT,
-+	HIDI2C_GET_IDLE,
-+	HIDI2C_SET_IDLE,
-+	HIDI2C_GET_PROTOCOL,
-+	HIDI2C_SET_PROTOCOL,
-+	HIDI2C_SET_POWER,
-+};
-+
-+/**
-+ * struct hidi2c_report_packet - Report packet definition in HIDI2C protocol
-+ * @len: data field length
-+ * @data: HIDI2C report packet data
-+ */
-+struct hidi2c_report_packet {
-+	__le16 len;
-+	u8 data[];
-+} __packed;
-+
-+#define HIDI2C_LENGTH_LEN	sizeof(__le16)
-+
-+#define HIDI2C_PACKET_LEN(data_len)	((data_len) + HIDI2C_LENGTH_LEN)
-+#define HIDI2C_DATA_LEN(pkt_len)	((pkt_len) - HIDI2C_LENGTH_LEN)
-+
-+#define HIDI2C_CMD_MAX_RI	0x0F
-+
-+/**
-+ * HIDI2C command data packet - Command packet definition in HIDI2C protocol
-+ * @report_id:		[0:3] report id (<15) for features or output reports
-+ * @report_type:	[4:5] indicate report type, reference to hidi2c_report_type
-+ * @reserved0:		[6:7] reserved bits
-+ * @opcode:		[8:11] command operation code, reference to hidi2c_opcode
-+ * @reserved1:		[12:15] reserved bits
-+ * @report_id_optional: [23:16] appended 3rd byte.
-+ *                      If the report_id in the low byte is set to the
-+ *                      sentinel value (HIDI2C_CMD_MAX_RI), then this
-+ *                      optional third byte represents the report id (>=15)
-+ *                      Otherwise, not this 3rd byte.
-+ */
-+
-+#define HIDI2C_CMD_LEN			sizeof(__le16)
-+#define HIDI2C_CMD_LEN_OPT		(sizeof(__le16) + 1)
-+#define HIDI2C_CMD_REPORT_ID		GENMASK(3, 0)
-+#define HIDI2C_CMD_REPORT_TYPE		GENMASK(5, 4)
-+#define HIDI2C_CMD_OPCODE		GENMASK(11, 8)
-+#define HIDI2C_CMD_OPCODE		GENMASK(11, 8)
-+#define HIDI2C_CMD_3RD_BYTE		GENMASK(23, 16)
-+
-+#define HIDI2C_HID_DESC_BCDVERSION	0x100
-+
-+/**
-+ * struct hidi2c_dev_descriptor - HIDI2C device descriptor definition
-+ * @dev_desc_len: The length of the complete device descriptor, fixed to 0x1E (30).
-+ * @bcd_ver: The version number of the HIDI2C protocol supported.
-+ *           In binary coded decimal (BCD) format.
-+ * @report_desc_len: The length of the report descriptor
-+ * @report_desc_reg: The register address to retrieve report descriptor
-+ * @input_reg: the register address to retrieve input report
-+ * @max_input_len: The length of the largest possible HID input (or feature) report
-+ * @output_reg: the register address to send output report
-+ * @max_output_len: The length of the largest output (or feature) report
-+ * @cmd_reg: the register address to send command
-+ * @data_reg: the register address to send command data
-+ * @vendor_id: Device manufacturers vendor ID
-+ * @product_id: Device unique model/product ID
-+ * @version_id: Device’s unique version
-+ * @reserved0: Reserved and should be 0
-+ * @reserved1: Reserved and should be 0
-+ */
-+struct hidi2c_dev_descriptor {
-+	__le16 dev_desc_len;
-+	__le16 bcd_ver;
-+	__le16 report_desc_len;
-+	__le16 report_desc_reg;
-+	__le16 input_reg;
-+	__le16 max_input_len;
-+	__le16 output_reg;
-+	__le16 max_output_len;
-+	__le16 cmd_reg;
-+	__le16 data_reg;
-+	__le16 vendor_id;
-+	__le16 product_id;
-+	__le16 version_id;
-+	__le16 reserved0;
-+	__le16 reserved1;
-+} __packed;
-+
-+#define HIDI2C_DEV_DESC_LEN		sizeof(struct hidi2c_dev_descriptor)
-+
-+#endif /* _HID_OVER_I2C_H_ */
-diff --git a/include/linux/hid-over-spi.h b/include/linux/hid-over-spi.h
-new file mode 100644
-index 000000000000..da5a14b5e89b
--- /dev/null
-+++ b/include/linux/hid-over-spi.h
-@@ -0,0 +1,155 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/* Copyright 2024 Intel Corporation */
-+
-+#ifndef _HID_OVER_SPI_H_
-+#define _HID_OVER_SPI_H_
-+
-+#include <linux/bits.h>
-+#include <linux/types.h>
-+
-+/* Input report type definition in HIDSPI protocol */
-+enum input_report_type {
-+	INVALID_INPUT_REPORT_TYPE_0	= 0,
-+	DATA				= 1,
-+	INVALID_TYPE_2			= 2,
-+	RESET_RESPONSE			= 3,
-+	COMMAND_RESPONSE		= 4,
-+	GET_FEATURE_RESPONSE		= 5,
-+	INVALID_TYPE_6			= 6,
-+	DEVICE_DESCRIPTOR_RESPONSE	= 7,
-+	REPORT_DESCRIPTOR_RESPONSE	= 8,
-+	SET_FEATURE_RESPONSE		= 9,
-+	OUTPUT_REPORT_RESPONSE		= 10,
-+	GET_INPUT_REPORT_RESPONSE	= 11,
-+	INVALID_INPUT_REPORT_TYPE	= 0xF,
-+};
-+
-+/* Output report type definition in HIDSPI protocol */
-+enum output_report_type {
-+	INVALID_OUTPUT_REPORT_TYPE_0	= 0,
-+	DEVICE_DESCRIPTOR		= 1,
-+	REPORT_DESCRIPTOR		= 2,
-+	SET_FEATURE			= 3,
-+	GET_FEATURE			= 4,
-+	OUTPUT_REPORT			= 5,
-+	GET_INPUT_REPORT		= 6,
-+	COMMAND_CONTENT			= 7,
-+};
-+
-+/* Set power command ID for output report */
-+#define HIDSPI_SET_POWER_CMD_ID  1
-+
-+/* Power state definition in HIDSPI protocol */
-+enum hidspi_power_state {
-+	HIDSPI_ON	= 1,
-+	HIDSPI_SLEEP	= 2,
-+	HIDSPI_OFF	= 3,
-+};
-+
-+/**
-+ * Input report header definition in HIDSPI protocol
-+ * Report header size is 32bits, it includes:
-+ * protocol_ver:     [0:3] Current supported HIDSPI protocol version, must be 0x3
-+ * reserved0:        [4:7] Reserved bits
-+ * input_report_len: [8:21] Input report length in number bytes divided by 4
-+ * last_frag_flag:   [22]Indicate if this packet is last fragment.
-+ *                       1 - indicates last fragment
-+ *                       0 - indicates additional fragments
-+ * reserved1:        [23] Reserved bits
-+ * @sync_const:      [24:31] Used to validate input report header, must be 0x5A
-+ */
-+#define HIDSPI_INPUT_HEADER_SIZE		sizeof(u32)
-+#define HIDSPI_INPUT_HEADER_VER			GENMASK(3, 0)
-+#define HIDSPI_INPUT_HEADER_REPORT_LEN		GENMASK(21, 8)
-+#define HIDSPI_INPUT_HEADER_LAST_FLAG		BIT(22)
-+#define HIDSPI_INPUT_HEADER_SYNC		GENMASK(31, 24)
-+
-+/**
-+ * struct input_report_body_header - Input report body header definition in HIDSPI protocol
-+ * @input_report_type: indicate input report type, reference to enum input_report_type
-+ * @content_len: this input report body packet length
-+ * @content_id: indicate this input report's report id
-+ */
-+struct input_report_body_header {
-+	u8 input_report_type;
-+	__le16 content_len;
-+	u8 content_id;
-+} __packed;
-+
-+#define HIDSPI_INPUT_BODY_HEADER_SIZE	sizeof(struct input_report_body_header)
-+
-+/**
-+ * struct input_report_body - Input report body definition in HIDSPI protocol
-+ * @body_hdr: input report body header
-+ * @content: input report body content
-+ */
-+struct input_report_body {
-+	struct input_report_body_header body_hdr;
-+	u8 content[];
-+} __packed;
-+
-+#define HIDSPI_INPUT_BODY_SIZE(content_len)	((content_len) + HIDSPI_INPUT_BODY_HEADER_SIZE)
-+
-+/**
-+ * struct output_report_header - Output report header definition in HIDSPI protocol
-+ * @report_type: output report type, reference to enum output_report_type
-+ * @content_len: length of content
-+ * @content_id: 0x00 - descriptors
-+ *              report id - Set/Feature feature or Input/Output Reports
-+ *              command opcode - for commands
-+ */
-+struct output_report_header {
-+	u8 report_type;
-+	__le16 content_len;
-+	u8 content_id;
-+} __packed;
-+
-+#define HIDSPI_OUTPUT_REPORT_HEADER_SIZE	sizeof(struct output_report_header)
-+
-+/**
-+ * struct output_report - Output report definition in HIDSPI protocol
-+ * @output_hdr: output report header
-+ * @content: output report content
-+ */
-+struct output_report {
-+	struct output_report_header output_hdr;
-+	u8 content[];
-+} __packed;
-+
-+#define HIDSPI_OUTPUT_REPORT_SIZE(content_len)	((content_len) + HIDSPI_OUTPUT_REPORT_HEADER_SIZE)
-+
-+/**
-+ * struct hidspi_dev_descriptor - HIDSPI device descriptor definition
-+ * @dev_desc_len: The length of the complete device descriptor, fixed to 0x18 (24).
-+ * @bcd_ver: The version number of the HIDSPI protocol supported.
-+ *           In binary coded decimal (BCD) format. Must be fixed to 0x0300.
-+ * @rep_desc_len: The length of the report descriptor
-+ * @max_input_len: The length of the largest possible HID input (or feature) report
-+ * @max_output_len: The length of the largest output (or feature) report
-+ * @max_frag_len: The length of the largest fragment, where a fragment represents
-+ *                the body of an input report.
-+ * @vendor_id: Device manufacturers vendor ID
-+ * @product_id: Device unique model/product ID
-+ * @version_id: Device’s unique version
-+ * @flags: Specify flags for the device’s operation
-+ * @reserved: Reserved and should be 0
-+ */
-+struct hidspi_dev_descriptor {
-+	__le16 dev_desc_len;
-+	__le16 bcd_ver;
-+	__le16 rep_desc_len;
-+	__le16 max_input_len;
-+	__le16 max_output_len;
-+	__le16 max_frag_len;
-+	__le16 vendor_id;
-+	__le16 product_id;
-+	__le16 version_id;
-+	__le16 flags;
-+	__le32 reserved;
-+};
-+
-+#define HIDSPI_DEVICE_DESCRIPTOR_SIZE		sizeof(struct hidspi_dev_descriptor)
-+#define HIDSPI_INPUT_DEVICE_DESCRIPTOR_SIZE	\
-+	(HIDSPI_INPUT_BODY_HEADER_SIZE + HIDSPI_DEVICE_DESCRIPTOR_SIZE)
-+
-+#endif /* _HID_OVER_SPI_H_ */
-diff --git a/include/linux/hid-roccat.h b/include/linux/hid-roccat.h
-index 3214fb0815fc..753654fff07f 100644
--- a/include/linux/hid-roccat.h
-+++ b/include/linux/hid-roccat.h
-@@ -16,7 +16,7 @@
- 
- #ifdef __KERNEL__
- 
-int roccat_connect(struct class *klass, struct hid_device *hid,
-+int roccat_connect(const struct class *klass, struct hid_device *hid,
- 		int report_size);
- void roccat_disconnect(int minor);
- int roccat_report_event(int minor, u8 const *data);
-diff --git a/include/linux/hid-sensor-hub.h b/include/linux/hid-sensor-hub.h
-index c27329e2a5ad..e71056553108 100644
--- a/include/linux/hid-sensor-hub.h
-+++ b/include/linux/hid-sensor-hub.h
-@@ -17,7 +17,7 @@
-  * @attrib_id:		Attribute id for this attribute.
-  * @report_id:		Report id in which this information resides.
-  * @index:		Field index in the report.
- * @units:		Measurment unit for this attribute.
-+ * @units:		Measurement unit for this attribute.
-  * @unit_expo:		Exponent used in the data.
-  * @size:		Size in bytes for data size.
-  * @logical_minimum:	Logical minimum value for this attribute.
-@@ -39,8 +39,8 @@ struct hid_sensor_hub_attribute_info {
-  * struct sensor_hub_pending - Synchronous read pending information
-  * @status:		Pending status true/false.
-  * @ready:		Completion synchronization data.
- * @usage_id:		Usage id for physical device, E.g. Gyro usage id.
- * @attr_usage_id:	Usage Id of a field, E.g. X-AXIS for a gyro.
-+ * @usage_id:		Usage id for physical device, e.g. gyro usage id.
-+ * @attr_usage_id:	Usage Id of a field, e.g. X-axis for a gyro.
-  * @raw_size:		Response size for a read request.
-  * @raw_data:		Place holder for received response.
-  */
-@@ -104,10 +104,10 @@ struct hid_sensor_hub_callbacks {
- int sensor_hub_device_open(struct hid_sensor_hub_device *hsdev);
- 
- /**
-* sensor_hub_device_clode() - Close hub device
-+* sensor_hub_device_close() - Close hub device
- * @hsdev:	Hub device instance.
- *
-* Used to clode hid device for sensor hub.
-+* Used to close hid device for sensor hub.
- */
- void sensor_hub_device_close(struct hid_sensor_hub_device *hsdev);
- 
-@@ -128,12 +128,13 @@ int sensor_hub_register_callback(struct hid_sensor_hub_device *hsdev,
- 			struct hid_sensor_hub_callbacks *usage_callback);
- 
- /**
-* sensor_hub_remove_callback() - Remove client callbacks
-+* sensor_hub_remove_callback() - Remove client callback
- * @hsdev:	Hub device instance.
-* @usage_id:	Usage id of the client (E.g. 0x200076 for Gyro).
-+* @usage_id:	Usage id of the client (e.g. 0x200076 for gyro).
- *
-* If there is a callback registred, this call will remove that
-* callbacks, so that it will stop data and event notifications.
-+* Removes a previously registered callback for the given usage_id
-+* and hsdev. Once removed, the client will no longer receive data or
-+* event notifications.
- */
- int sensor_hub_remove_callback(struct hid_sensor_hub_device *hsdev,
- 			u32 usage_id);
-diff --git a/include/linux/hid-sensor-ids.h b/include/linux/hid-sensor-ids.h
-index ac631159403a..8a03d9696b1c 100644
--- a/include/linux/hid-sensor-ids.h
-+++ b/include/linux/hid-sensor-ids.h
-@@ -21,11 +21,17 @@
- #define HID_USAGE_SENSOR_ALS					0x200041
- #define HID_USAGE_SENSOR_DATA_LIGHT				0x2004d0
- #define HID_USAGE_SENSOR_LIGHT_ILLUM				0x2004d1
-+#define HID_USAGE_SENSOR_LIGHT_COLOR_TEMPERATURE		0x2004d2
-+#define HID_USAGE_SENSOR_LIGHT_CHROMATICITY			0x2004d3
-+#define HID_USAGE_SENSOR_LIGHT_CHROMATICITY_X			0x2004d4
-+#define HID_USAGE_SENSOR_LIGHT_CHROMATICITY_Y			0x2004d5
- 
- /* PROX (200011) */
- #define HID_USAGE_SENSOR_PROX                                   0x200011
- #define HID_USAGE_SENSOR_DATA_PRESENCE                          0x2004b0
- #define HID_USAGE_SENSOR_HUMAN_PRESENCE                         0x2004b1
-+#define HID_USAGE_SENSOR_HUMAN_PROXIMITY                        0x2004b2
-+#define HID_USAGE_SENSOR_HUMAN_ATTENTION                        0x2004bd
- 
- /* Pressure (200031) */
- #define HID_USAGE_SENSOR_PRESSURE                               0x200031
-@@ -132,6 +138,7 @@
- #define HID_USAGE_SENSOR_PROP_FRIENDLY_NAME			0x200301
- #define HID_USAGE_SENSOR_PROP_SERIAL_NUM			0x200307
- #define HID_USAGE_SENSOR_PROP_MANUFACTURER			0x200305
-+#define HID_USAGE_SENSOR_PROP_MODEL				0x200306
- #define HID_USAGE_SENSOR_PROP_REPORT_INTERVAL			0x20030E
- #define HID_USAGE_SENSOR_PROP_SENSITIVITY_ABS			0x20030F
- #define HID_USAGE_SENSOR_PROP_SENSITIVITY_RANGE_PCT		0x200310
-diff --git a/include/linux/hid.h b/include/linux/hid.h
-index 4363a63b9775..dce862cafbbd 100644
--- a/include/linux/hid.h
-+++ b/include/linux/hid.h
-@@ -26,6 +26,7 @@
- #include <linux/mutex.h>
- #include <linux/power_supply.h>
- #include <uapi/linux/hid.h>
-+#include <linux/hid_bpf.h>
- 
- /*
-  * We parse each description item into this structure. Short items data
-@@ -45,7 +46,7 @@ struct hid_item {
- 	    __s16  s16;
- 	    __u32  u32;
- 	    __s32  s32;
-	    __u8  *longdata;
-+	    const __u8  *longdata;
- 	} data;
- };
- 
-@@ -80,6 +81,8 @@ struct hid_item {
- #define HID_MAIN_ITEM_TAG_FEATURE		11
- #define HID_MAIN_ITEM_TAG_BEGIN_COLLECTION	10
- #define HID_MAIN_ITEM_TAG_END_COLLECTION	12
-+#define HID_MAIN_ITEM_TAG_RESERVED_MIN		13
-+#define HID_MAIN_ITEM_TAG_RESERVED_MAX		15
- 
- /*
-  * HID report descriptor main item contents
-@@ -153,8 +156,10 @@ struct hid_item {
- #define HID_UP_TELEPHONY	0x000b0000
- #define HID_UP_CONSUMER		0x000c0000
- #define HID_UP_DIGITIZER	0x000d0000
-+#define HID_UP_HAPTIC		0x000e0000
- #define HID_UP_PID		0x000f0000
- #define HID_UP_BATTERY		0x00850000
-+#define HID_UP_CAMERA		0x00900000
- #define HID_UP_HPVENDOR         0xff7f0000
- #define HID_UP_HPVENDOR2        0xff010000
- #define HID_UP_MSVENDOR		0xff000000
-@@ -216,6 +221,7 @@ struct hid_item {
- #define HID_GD_DOWN		0x00010091
- #define HID_GD_RIGHT		0x00010092
- #define HID_GD_LEFT		0x00010093
-+#define HID_GD_DO_NOT_DISTURB	0x0001009b
- /* Microsoft Win8 Wireless Radio Controls CA usage codes */
- #define HID_GD_RFKILL_BTN	0x000100c6
- #define HID_GD_RFKILL_LED	0x000100c7
-@@ -311,18 +317,32 @@ struct hid_item {
- #define HID_DG_TOOLSERIALNUMBER	0x000d005b
- #define HID_DG_LATENCYMODE	0x000d0060
- 
-+#define HID_HP_SIMPLECONTROLLER	0x000e0001
-+#define HID_HP_WAVEFORMLIST	0x000e0010
-+#define HID_HP_DURATIONLIST	0x000e0011
-+#define HID_HP_AUTOTRIGGER	0x000e0020
-+#define HID_HP_MANUALTRIGGER	0x000e0021
-+#define HID_HP_AUTOTRIGGERASSOCIATEDCONTROL 0x000e0022
-+#define HID_HP_INTENSITY	0x000e0023
-+#define HID_HP_REPEATCOUNT	0x000e0024
-+#define HID_HP_RETRIGGERPERIOD	0x000e0025
-+#define HID_HP_WAVEFORMVENDORPAGE	0x000e0026
-+#define HID_HP_WAVEFORMVENDORID	0x000e0027
-+#define HID_HP_WAVEFORMCUTOFFTIME	0x000e0028
-+#define HID_HP_WAVEFORMNONE	0x000e1001
-+#define HID_HP_WAVEFORMSTOP	0x000e1002
-+#define HID_HP_WAVEFORMCLICK	0x000e1003
-+#define HID_HP_WAVEFORMBUZZCONTINUOUS	0x000e1004
-+#define HID_HP_WAVEFORMRUMBLECONTINUOUS	0x000e1005
-+#define HID_HP_WAVEFORMPRESS	0x000e1006
-+#define HID_HP_WAVEFORMRELEASE	0x000e1007
-+#define HID_HP_VENDORWAVEFORMMIN	0x000e2001
-+#define HID_HP_VENDORWAVEFORMMAX	0x000e2fff
-+
- #define HID_BAT_ABSOLUTESTATEOFCHARGE	0x00850065
-+#define HID_BAT_CHARGING		0x00850044
- 
- #define HID_VD_ASUS_CUSTOM_MEDIA_KEYS	0xff310076
-/*
- * HID report types --- Ouch! HID spec says 1 2 3!
- */
-
-#define HID_INPUT_REPORT	0
-#define HID_OUTPUT_REPORT	1
-#define HID_FEATURE_REPORT	2
-
-#define HID_REPORT_TYPES	3
- 
- /*
-  * HID connect requests
-@@ -347,6 +367,32 @@ struct hid_item {
-  */
- #define MAX_USBHID_BOOT_QUIRKS 4
- 
-+/**
-+ * DOC: HID quirks
-+ * | @HID_QUIRK_NOTOUCH:
-+ * | @HID_QUIRK_IGNORE: ignore this device
-+ * | @HID_QUIRK_NOGET:
-+ * | @HID_QUIRK_HIDDEV_FORCE:
-+ * | @HID_QUIRK_BADPAD:
-+ * | @HID_QUIRK_MULTI_INPUT:
-+ * | @HID_QUIRK_HIDINPUT_FORCE:
-+ * | @HID_QUIRK_ALWAYS_POLL:
-+ * | @HID_QUIRK_INPUT_PER_APP:
-+ * | @HID_QUIRK_X_INVERT:
-+ * | @HID_QUIRK_Y_INVERT:
-+ * | @HID_QUIRK_IGNORE_MOUSE:
-+ * | @HID_QUIRK_SKIP_OUTPUT_REPORTS:
-+ * | @HID_QUIRK_SKIP_OUTPUT_REPORT_ID:
-+ * | @HID_QUIRK_NO_OUTPUT_REPORTS_ON_INTR_EP:
-+ * | @HID_QUIRK_HAVE_SPECIAL_DRIVER:
-+ * | @HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE:
-+ * | @HID_QUIRK_IGNORE_SPECIAL_DRIVER
-+ * | @HID_QUIRK_POWER_ON_AFTER_BACKLIGHT
-+ * | @HID_QUIRK_FULLSPEED_INTERVAL:
-+ * | @HID_QUIRK_NO_INIT_REPORTS:
-+ * | @HID_QUIRK_NO_IGNORE:
-+ * | @HID_QUIRK_NO_INPUT_SYNC:
-+ */
- /* BIT(0) reserved for backward compatibility, was HID_QUIRK_INVERT */
- #define HID_QUIRK_NOTOUCH			BIT(1)
- #define HID_QUIRK_IGNORE			BIT(2)
-@@ -361,11 +407,15 @@ struct hid_item {
- #define HID_QUIRK_INPUT_PER_APP			BIT(11)
- #define HID_QUIRK_X_INVERT			BIT(12)
- #define HID_QUIRK_Y_INVERT			BIT(13)
-+#define HID_QUIRK_IGNORE_MOUSE			BIT(14)
- #define HID_QUIRK_SKIP_OUTPUT_REPORTS		BIT(16)
- #define HID_QUIRK_SKIP_OUTPUT_REPORT_ID		BIT(17)
- #define HID_QUIRK_NO_OUTPUT_REPORTS_ON_INTR_EP	BIT(18)
- #define HID_QUIRK_HAVE_SPECIAL_DRIVER		BIT(19)
- #define HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE	BIT(20)
-+#define HID_QUIRK_NOINVERT			BIT(21)
-+#define HID_QUIRK_IGNORE_SPECIAL_DRIVER		BIT(22)
-+#define HID_QUIRK_POWER_ON_AFTER_BACKLIGHT	BIT(23)
- #define HID_QUIRK_FULLSPEED_INTERVAL		BIT(28)
- #define HID_QUIRK_NO_INIT_REPORTS		BIT(29)
- #define HID_QUIRK_NO_IGNORE			BIT(30)
-@@ -399,6 +449,12 @@ struct hid_item {
- #define HID_BOOT_PROTOCOL	0
- 
- /*
-+ * HID units
-+ */
-+#define HID_UNIT_GRAM		0x0101
-+#define HID_UNIT_NEWTON		0xe111
-+
-+/*
-  * This is the global environment of the parser. This information is
-  * persistent for main-items. The global environment can be saved and
-  * restored with PUSH/POP statements.
-@@ -456,9 +512,9 @@ struct hid_usage {
- 	__s8	  wheel_factor;		/* 120/resolution_multiplier */
- 	__u16     code;			/* input driver code */
- 	__u8      type;			/* input driver type */
-	__s8	  hat_min;		/* hat switch fun */
-	__s8	  hat_max;		/* ditto */
-	__s8	  hat_dir;		/* ditto */
-+	__s16	  hat_min;		/* hat switch fun */
-+	__s16	  hat_max;		/* ditto */
-+	__s16	  hat_dir;		/* ditto */
- 	__s16	  wheel_accumulated;	/* hi-res wheel */
- };
- 
-@@ -509,7 +565,7 @@ struct hid_report {
- 	struct list_head hidinput_list;
- 	struct list_head field_entry_list;		/* ordered list of input fields */
- 	unsigned int id;				/* id of this report */
-	unsigned int type;				/* report type */
-+	enum hid_report_type type;			/* report type */
- 	unsigned int application;			/* application usage for this report */
- 	struct hid_field *field[HID_MAX_FIELDS];	/* fields of the report */
- 	struct hid_field_entry *field_entries;		/* allocated memory of input field_entry */
-@@ -561,9 +617,9 @@ struct hid_input {
- 	struct hid_report *report;
- 	struct input_dev *input;
- 	const char *name;
-	bool registered;
- 	struct list_head reports;	/* the list of reports */
- 	unsigned int application;	/* application usage for this input */
-+	bool registered;
- };
- 
- enum hid_type {
-@@ -581,15 +637,17 @@ enum hid_battery_status {
- struct hid_driver;
- struct hid_ll_driver;
- 
-struct hid_device {							/* device report descriptor */
-	__u8 *dev_rdesc;
-	unsigned dev_rsize;
-	__u8 *rdesc;
-	unsigned rsize;
-+struct hid_device {
-+	const __u8 *dev_rdesc;						/* device report descriptor */
-+	const __u8 *bpf_rdesc;						/* bpf modified report descriptor, if any */
-+	const __u8 *rdesc;						/* currently used report descriptor */
-+	unsigned int dev_rsize;
-+	unsigned int bpf_rsize;
-+	unsigned int rsize;
-+	unsigned int collection_size;					/* Number of allocated hid_collections */
- 	struct hid_collection *collection;				/* List of HID collections */
-	unsigned collection_size;					/* Number of allocated hid_collections */
-	unsigned maxcollection;						/* Number of parsed collections */
-	unsigned maxapplication;					/* Number of applications */
-+	unsigned int maxcollection;						/* Number of parsed collections */
-+	unsigned int maxapplication;					/* Number of applications */
- 	__u16 bus;							/* BUS ID */
- 	__u16 group;							/* Report group */
- 	__u32 vendor;							/* Vendor ID */
-@@ -603,8 +661,9 @@ struct hid_device {							/* device report descriptor */
- 	struct semaphore driver_input_lock;				/* protects the current driver */
- 	struct device dev;						/* device */
- 	struct hid_driver *driver;
-+	void *devres_group_id;						/* ID of probe devres group	*/
- 
-	struct hid_ll_driver *ll_driver;
-+	const struct hid_ll_driver *ll_driver;
- 	struct mutex ll_open_lock;
- 	unsigned int ll_open_count;
- 
-@@ -620,6 +679,7 @@ struct hid_device {							/* device report descriptor */
- 	__s32 battery_max;
- 	__s32 battery_report_type;
- 	__s32 battery_report_id;
-+	__s32 battery_charge_status;
- 	enum hid_battery_status battery_status;
- 	bool battery_avoid_query;
- 	ktime_t battery_ratelimit_time;
-@@ -628,6 +688,7 @@ struct hid_device {							/* device report descriptor */
- 	unsigned long status;						/* see STAT flags above */
- 	unsigned claimed;						/* Claimed by hidinput, hiddev? */
- 	unsigned quirks;						/* Various quirks the device can pull on us */
-+	unsigned initial_quirks;					/* Initial set of quirks supplied when creating device */
- 	bool io_started;						/* If IO has started */
- 
- 	struct list_head inputs;					/* The list of inputs */
-@@ -658,8 +719,17 @@ struct hid_device {							/* device report descriptor */
- 	struct list_head debug_list;
- 	spinlock_t  debug_list_lock;
- 	wait_queue_head_t debug_wait;
-+	struct kref			ref;
-+
-+	unsigned int id;						/* system unique id */
-+
-+#ifdef CONFIG_HID_BPF
-+	struct hid_bpf bpf;						/* hid-bpf data */
-+#endif /* CONFIG_HID_BPF */
- };
- 
-+void hiddev_free(struct kref *ref);
-+
- #define to_hid_device(pdev) \
- 	container_of(pdev, struct hid_device, dev)
- 
-@@ -703,8 +773,9 @@ struct hid_descriptor {
- 	__le16 bcdHID;
- 	__u8  bCountryCode;
- 	__u8  bNumDescriptors;
-+	struct hid_class_descriptor rpt_desc;
- 
-	struct hid_class_descriptor desc[1];
-+	struct hid_class_descriptor opt_descs[];
- } __attribute__ ((packed));
- 
- #define HID_DEVICE(b, g, ven, prod)					\
-@@ -755,6 +826,8 @@ struct hid_usage_id {
-  * @suspend: invoked on suspend (NULL means nop)
-  * @resume: invoked on resume if device was not reset (NULL means nop)
-  * @reset_resume: invoked on resume if device was reset (NULL means nop)
-+ * @on_hid_hw_open: invoked when hid core opens first instance (NULL means nop)
-+ * @on_hid_hw_close: invoked when hid core closes last instance (NULL means nop)
-  *
-  * probe should return -errno on error, or 0 on success. During probe,
-  * input will not be passed to raw_event unless hid_device_io_start is
-@@ -774,7 +847,7 @@ struct hid_usage_id {
-  * zero from them.
-  */
- struct hid_driver {
-	char *name;
-+	const char *name;
- 	const struct hid_device_id *id_table;
- 
- 	struct list_head dyn_list;
-@@ -792,7 +865,7 @@ struct hid_driver {
- 			struct hid_usage *usage, __s32 value);
- 	void (*report)(struct hid_device *hdev, struct hid_report *report);
- 
-	__u8 *(*report_fixup)(struct hid_device *hdev, __u8 *buf,
-+	const __u8 *(*report_fixup)(struct hid_device *hdev, __u8 *buf,
- 			unsigned int *size);
- 
- 	int (*input_mapping)(struct hid_device *hdev,
-@@ -806,11 +879,13 @@ struct hid_driver {
- 	void (*feature_mapping)(struct hid_device *hdev,
- 			struct hid_field *field,
- 			struct hid_usage *usage);
-#ifdef CONFIG_PM
-+
- 	int (*suspend)(struct hid_device *hdev, pm_message_t message);
- 	int (*resume)(struct hid_device *hdev);
- 	int (*reset_resume)(struct hid_device *hdev);
-#endif
-+	void (*on_hid_hw_open)(struct hid_device *hdev);
-+	void (*on_hid_hw_close)(struct hid_device *hdev);
-+
- /* private: */
- 	struct device_driver driver;
- };
-@@ -833,6 +908,7 @@ struct hid_driver {
-  * @output_report: send output report to device
-  * @idle: send idle request to device
-  * @may_wakeup: return if device may act as a wakeup source during system-suspend
-+ * @max_buffer_size: over-ride maximum data buffer size (default: HID_MAX_BUFFER_SIZE)
-  */
- struct hid_ll_driver {
- 	int (*start)(struct hid_device *hdev);
-@@ -858,23 +934,11 @@ struct hid_ll_driver {
- 
- 	int (*idle)(struct hid_device *hdev, int report, int idle, int reqtype);
- 	bool (*may_wakeup)(struct hid_device *hdev);
-};
- 
-extern struct hid_ll_driver i2c_hid_ll_driver;
-extern struct hid_ll_driver hidp_hid_driver;
-extern struct hid_ll_driver uhid_hid_driver;
-extern struct hid_ll_driver usb_hid_driver;
-
-static inline bool hid_is_using_ll_driver(struct hid_device *hdev,
-		struct hid_ll_driver *driver)
-{
-	return hdev->ll_driver == driver;
-}
-+	unsigned int max_buffer_size;
-+};
- 
-static inline bool hid_is_usb(struct hid_device *hdev)
-{
-	return hid_is_using_ll_driver(hdev, &usb_hid_driver);
-}
-+extern bool hid_is_usb(const struct hid_device *hdev);
- 
- #define	PM_HINT_FULLON	1<<5
- #define PM_HINT_NORMAL	1<<1
-@@ -883,19 +947,17 @@ static inline bool hid_is_usb(struct hid_device *hdev)
- /* We ignore a few input applications that are not widely used */
- #define IS_INPUT_APPLICATION(a) \
- 		(((a >= HID_UP_GENDESK) && (a <= HID_GD_MULTIAXIS)) \
-		|| ((a >= HID_DG_PEN) && (a <= HID_DG_WHITEBOARD)) \
-+		|| ((a >= HID_DG_DIGITIZER) && (a <= HID_DG_WHITEBOARD)) \
- 		|| (a == HID_GD_SYSTEM_CONTROL) || (a == HID_CP_CONSUMER_CONTROL) \
- 		|| (a == HID_GD_WIRELESS_RADIO_CTLS))
- 
- /* HID core API */
- 
-extern int hid_debug;
-
- extern bool hid_ignore(struct hid_device *);
- extern int hid_add_device(struct hid_device *);
- extern void hid_destroy_device(struct hid_device *);
- 
-extern struct bus_type hid_bus_type;
-+extern const struct bus_type hid_bus_type;
- 
- extern int __must_check __hid_register_driver(struct hid_driver *,
- 		struct module *, const char *mod_name);
-@@ -922,22 +984,26 @@ extern void hidinput_hid_event(struct hid_device *, struct hid_field *, struct h
- extern void hidinput_report_event(struct hid_device *hid, struct hid_report *report);
- extern int hidinput_connect(struct hid_device *hid, unsigned int force);
- extern void hidinput_disconnect(struct hid_device *);
-+void hidinput_reset_resume(struct hid_device *hid);
- 
-+struct hid_field *hid_find_field(struct hid_device *hdev, unsigned int report_type,
-+				 unsigned int application, unsigned int usage);
- int hid_set_field(struct hid_field *, unsigned, __s32);
-int hid_input_report(struct hid_device *, int type, u8 *, u32, int);
-+int hid_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size,
-+		     int interrupt);
- struct hid_field *hidinput_get_led_field(struct hid_device *hid);
- unsigned int hidinput_count_leds(struct hid_device *hid);
- __s32 hidinput_calc_abs_res(const struct hid_field *field, __u16 code);
- void hid_output_report(struct hid_report *report, __u8 *data);
-int __hid_request(struct hid_device *hid, struct hid_report *rep, int reqtype);
-+int __hid_request(struct hid_device *hid, struct hid_report *rep, enum hid_class_request reqtype);
- u8 *hid_alloc_report_buf(struct hid_report *report, gfp_t flags);
- struct hid_device *hid_allocate_device(void);
- struct hid_report *hid_register_report(struct hid_device *device,
-				       unsigned int type, unsigned int id,
-+				       enum hid_report_type type, unsigned int id,
- 				       unsigned int application);
-int hid_parse_report(struct hid_device *hid, __u8 *start, unsigned size);
-+int hid_parse_report(struct hid_device *hid, const __u8 *start, unsigned size);
- struct hid_report *hid_validate_values(struct hid_device *hid,
-				       unsigned int type, unsigned int id,
-+				       enum hid_report_type type, unsigned int id,
- 				       unsigned int field_index,
- 				       unsigned int report_counts);
- 
-@@ -954,7 +1020,6 @@ const struct hid_device_id *hid_match_device(struct hid_device *hdev,
- 					     struct hid_driver *hdrv);
- bool hid_compare_device_paths(struct hid_device *hdev_a,
- 			      struct hid_device *hdev_b, char separator);
-s32 hid_snto32(__u32 value, unsigned n);
- __u32 hid_field_extract(const struct hid_device *hid, __u8 *report,
- 		     unsigned offset, unsigned n);
- 
-@@ -1106,10 +1171,18 @@ void hid_hw_stop(struct hid_device *hdev);
- int __must_check hid_hw_open(struct hid_device *hdev);
- void hid_hw_close(struct hid_device *hdev);
- void hid_hw_request(struct hid_device *hdev,
-		    struct hid_report *report, int reqtype);
-+		    struct hid_report *report, enum hid_class_request reqtype);
-+int __hid_hw_raw_request(struct hid_device *hdev,
-+			 unsigned char reportnum, __u8 *buf,
-+			 size_t len, enum hid_report_type rtype,
-+			 enum hid_class_request reqtype,
-+			 __u64 source, bool from_bpf);
-+int __hid_hw_output_report(struct hid_device *hdev, __u8 *buf, size_t len, __u64 source,
-+			   bool from_bpf);
- int hid_hw_raw_request(struct hid_device *hdev,
- 		       unsigned char reportnum, __u8 *buf,
-		       size_t len, unsigned char rtype, int reqtype);
-+		       size_t len, enum hid_report_type rtype,
-+		       enum hid_class_request reqtype);
- int hid_hw_output_report(struct hid_device *hdev, __u8 *buf, size_t len);
- 
- /**
-@@ -1137,7 +1210,7 @@ static inline int hid_hw_power(struct hid_device *hdev, int level)
-  * @reqtype: hid request type
-  */
- static inline int hid_hw_idle(struct hid_device *hdev, int report, int idle,
-		int reqtype)
-+		enum hid_class_request reqtype)
- {
- 	if (hdev->ll_driver->idle)
- 		return hdev->ll_driver->idle(hdev, report, idle, reqtype);
-@@ -1175,32 +1248,26 @@ static inline void hid_hw_wait(struct hid_device *hdev)
- /**
-  * hid_report_len - calculate the report length
-  *
- * @report: the report we want to know the length
-+ * @report: the report whose length we want to know
-+ *
-+ * The length counts the report ID byte, but only if the ID is nonzero
-+ * and therefore is included in the report.  Reports whose ID is zero
-+ * never include an ID byte.
-  */
- static inline u32 hid_report_len(struct hid_report *report)
- {
- 	return DIV_ROUND_UP(report->size, 8) + (report->id > 0);
- }
- 
-int hid_report_raw_event(struct hid_device *hid, int type, u8 *data, u32 size,
-		int interrupt);
-+int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size,
-+			 int interrupt);
- 
- /* HID quirks API */
- unsigned long hid_lookup_quirk(const struct hid_device *hdev);
- int hid_quirks_init(char **quirks_param, __u16 bus, int count);
- void hid_quirks_exit(__u16 bus);
- 
-#ifdef CONFIG_HID_PID
-int hid_pidff_init(struct hid_device *hid);
-#else
-#define hid_pidff_init NULL
-#endif
-
-#define dbg_hid(fmt, ...)						\
-do {									\
-	if (hid_debug)							\
-		printk(KERN_DEBUG "%s: " fmt, __FILE__, ##__VA_ARGS__);	\
-} while (0)
-+#define dbg_hid(fmt, ...) pr_debug("%s: " fmt, __FILE__, ##__VA_ARGS__)
- 
- #define hid_err(hid, fmt, ...)				\
- 	dev_err(&(hid)->dev, fmt, ##__VA_ARGS__)
-@@ -1208,6 +1275,8 @@ do {									\
- 	dev_notice(&(hid)->dev, fmt, ##__VA_ARGS__)
- #define hid_warn(hid, fmt, ...)				\
- 	dev_warn(&(hid)->dev, fmt, ##__VA_ARGS__)
-+#define hid_warn_ratelimited(hid, fmt, ...)				\
-+	dev_warn_ratelimited(&(hid)->dev, fmt, ##__VA_ARGS__)
- #define hid_info(hid, fmt, ...)				\
- 	dev_info(&(hid)->dev, fmt, ##__VA_ARGS__)
- #define hid_dbg(hid, fmt, ...)				\
-@@ -1224,4 +1293,15 @@ do {									\
- #define hid_dbg_once(hid, fmt, ...)			\
- 	dev_dbg_once(&(hid)->dev, fmt, ##__VA_ARGS__)
- 
-+#define hid_err_ratelimited(hid, fmt, ...)			\
-+	dev_err_ratelimited(&(hid)->dev, fmt, ##__VA_ARGS__)
-+#define hid_notice_ratelimited(hid, fmt, ...)			\
-+	dev_notice_ratelimited(&(hid)->dev, fmt, ##__VA_ARGS__)
-+#define hid_warn_ratelimited(hid, fmt, ...)			\
-+	dev_warn_ratelimited(&(hid)->dev, fmt, ##__VA_ARGS__)
-+#define hid_info_ratelimited(hid, fmt, ...)			\
-+	dev_info_ratelimited(&(hid)->dev, fmt, ##__VA_ARGS__)
-+#define hid_dbg_ratelimited(hid, fmt, ...)			\
-+	dev_dbg_ratelimited(&(hid)->dev, fmt, ##__VA_ARGS__)
-+
- #endif
-diff --git a/include/linux/hid_bpf.h b/include/linux/hid_bpf.h
-new file mode 100644
-index 000000000000..a2e47dbcf82c
--- /dev/null
-+++ b/include/linux/hid_bpf.h
-@@ -0,0 +1,236 @@
-+/* SPDX-License-Identifier: GPL-2.0+ */
-+
-+#ifndef __HID_BPF_H
-+#define __HID_BPF_H
-+
-+#include <linux/bpf.h>
-+#include <linux/mutex.h>
-+#include <linux/srcu.h>
-+#include <uapi/linux/hid.h>
-+
-+struct hid_device;
-+
-+/*
-+ * The following is the user facing HID BPF API.
-+ *
-+ * Extra care should be taken when editing this part, as
-+ * it might break existing out of the tree bpf programs.
-+ */
-+
-+/**
-+ * struct hid_bpf_ctx - User accessible data for all HID programs
-+ *
-+ * ``data`` is not directly accessible from the context. We need to issue
-+ * a call to hid_bpf_get_data() in order to get a pointer to that field.
-+ *
-+ * @hid: the &struct hid_device representing the device itself
-+ * @allocated_size: Allocated size of data.
-+ *
-+ *                  This is how much memory is available and can be requested
-+ *                  by the HID program.
-+ *                  Note that for ``HID_BPF_RDESC_FIXUP``, that memory is set to
-+ *                  ``4096`` (4 KB)
-+ * @size: Valid data in the data field.
-+ *
-+ *        Programs can get the available valid size in data by fetching this field.
-+ *        Programs can also change this value by returning a positive number in the
-+ *        program.
-+ *        To discard the event, return a negative error code.
-+ *
-+ *        ``size`` must always be less or equal than ``allocated_size`` (it is enforced
-+ *        once all BPF programs have been run).
-+ * @retval: Return value of the previous program.
-+ *
-+ * ``hid`` and ``allocated_size`` are read-only, ``size`` and ``retval`` are read-write.
-+ */
-+struct hid_bpf_ctx {
-+	struct hid_device *hid;
-+	__u32 allocated_size;
-+	union {
-+		__s32 retval;
-+		__s32 size;
-+	};
-+};
-+
-+/*
-+ * Below is HID internal
-+ */
-+
-+#define HID_BPF_MAX_PROGS_PER_DEV 64
-+#define HID_BPF_FLAG_MASK (((HID_BPF_FLAG_MAX - 1) << 1) - 1)
-+
-+
-+struct hid_report_enum;
-+
-+struct hid_ops {
-+	struct hid_report *(*hid_get_report)(struct hid_report_enum *report_enum, const u8 *data);
-+	int (*hid_hw_raw_request)(struct hid_device *hdev,
-+				  unsigned char reportnum, __u8 *buf,
-+				  size_t len, enum hid_report_type rtype,
-+				  enum hid_class_request reqtype,
-+				  u64 source, bool from_bpf);
-+	int (*hid_hw_output_report)(struct hid_device *hdev, __u8 *buf, size_t len,
-+				    u64 source, bool from_bpf);
-+	int (*hid_input_report)(struct hid_device *hid, enum hid_report_type type,
-+				u8 *data, u32 size, int interrupt, u64 source, bool from_bpf,
-+				bool lock_already_taken);
-+	struct module *owner;
-+	const struct bus_type *bus_type;
-+};
-+
-+extern const struct hid_ops *hid_ops;
-+
-+/**
-+ * struct hid_bpf_ops - A BPF struct_ops of callbacks allowing to attach HID-BPF
-+ *			programs to a HID device
-+ * @hid_id: the HID uniq ID to attach to. This is writeable before ``load()``, and
-+ *	    cannot be changed after
-+ * @flags: flags used while attaching the struct_ops to the device. Currently only
-+ *	   available value is %0 or ``BPF_F_BEFORE``.
-+ *	   Writeable only before ``load()``
-+ */
-+struct hid_bpf_ops {
-+	/* hid_id needs to stay first so we can easily change it
-+	 * from userspace.
-+	 */
-+	int			hid_id;
-+	u32			flags;
-+
-+	/* private: do not show up in the docs */
-+	struct list_head	list;
-+
-+	/* public: rest should show up in the docs */
-+
-+	/**
-+	 * @hid_device_event: called whenever an event is coming in from the device
-+	 *
-+	 * It has the following arguments:
-+	 *
-+	 * ``ctx``: The HID-BPF context as &struct hid_bpf_ctx
-+	 *
-+	 * Return: %0 on success and keep processing; a positive
-+	 * value to change the incoming size buffer; a negative
-+	 * error code to interrupt the processing of this event
-+	 *
-+	 * Context: Interrupt context.
-+	 */
-+	int (*hid_device_event)(struct hid_bpf_ctx *ctx, enum hid_report_type report_type,
-+				u64 source);
-+
-+	/**
-+	 * @hid_rdesc_fixup: called when the probe function parses the report descriptor
-+	 * of the HID device
-+	 *
-+	 * It has the following arguments:
-+	 *
-+	 * ``ctx``: The HID-BPF context as &struct hid_bpf_ctx
-+	 *
-+	 * Return: %0 on success and keep processing; a positive
-+	 * value to change the incoming size buffer; a negative
-+	 * error code to interrupt the processing of this device
-+	 */
-+	int (*hid_rdesc_fixup)(struct hid_bpf_ctx *ctx);
-+
-+	/**
-+	 * @hid_hw_request: called whenever a hid_hw_raw_request() call is emitted
-+	 * on the HID device
-+	 *
-+	 * It has the following arguments:
-+	 *
-+	 * ``ctx``: The HID-BPF context as &struct hid_bpf_ctx
-+	 *
-+	 * ``reportnum``: the report number, as in hid_hw_raw_request()
-+	 *
-+	 * ``rtype``: the report type (``HID_INPUT_REPORT``, ``HID_FEATURE_REPORT``,
-+	 * ``HID_OUTPUT_REPORT``)
-+	 *
-+	 * ``reqtype``: the request
-+	 *
-+	 * ``source``: a u64 referring to a uniq but identifiable source. If %0, the
-+	 * kernel itself emitted that call. For hidraw, ``source`` is set
-+	 * to the associated ``struct file *``.
-+	 *
-+	 * Return: %0 to keep processing the request by hid-core; any other value
-+	 * stops hid-core from processing that event. A positive value should be
-+	 * returned with the number of bytes returned in the incoming buffer; a
-+	 * negative error code interrupts the processing of this call.
-+	 */
-+	int (*hid_hw_request)(struct hid_bpf_ctx *ctx, unsigned char reportnum,
-+			       enum hid_report_type rtype, enum hid_class_request reqtype,
-+			       u64 source);
-+
-+	/**
-+	 * @hid_hw_output_report: called whenever a hid_hw_output_report() call is emitted
-+	 * on the HID device
-+	 *
-+	 * It has the following arguments:
-+	 *
-+	 * ``ctx``: The HID-BPF context as &struct hid_bpf_ctx
-+	 *
-+	 * ``source``: a u64 referring to a uniq but identifiable source. If %0, the
-+	 * kernel itself emitted that call. For hidraw, ``source`` is set
-+	 * to the associated ``struct file *``.
-+	 *
-+	 * Return: %0 to keep processing the request by hid-core; any other value
-+	 * stops hid-core from processing that event. A positive value should be
-+	 * returned with the number of bytes written to the device; a negative error
-+	 * code interrupts the processing of this call.
-+	 */
-+	int (*hid_hw_output_report)(struct hid_bpf_ctx *ctx, u64 source);
-+
-+
-+	/* private: do not show up in the docs */
-+	struct hid_device *hdev;
-+};
-+
-+/* stored in each device */
-+struct hid_bpf {
-+	u8 *device_data;		/* allocated when a bpf program of type
-+					 * SEC(f.../hid_bpf_device_event) has been attached
-+					 * to this HID device
-+					 */
-+	u32 allocated_data;
-+	bool destroyed;			/* prevents the assignment of any progs */
-+
-+	struct hid_bpf_ops *rdesc_ops;
-+	struct list_head prog_list;
-+	struct mutex prog_list_lock;	/* protects prog_list update */
-+	struct srcu_struct srcu;	/* protects prog_list read-only access */
-+};
-+
-+#ifdef CONFIG_HID_BPF
-+u8 *dispatch_hid_bpf_device_event(struct hid_device *hid, enum hid_report_type type, u8 *data,
-+				  u32 *size, int interrupt, u64 source, bool from_bpf);
-+int dispatch_hid_bpf_raw_requests(struct hid_device *hdev,
-+				  unsigned char reportnum, __u8 *buf,
-+				  u32 size, enum hid_report_type rtype,
-+				  enum hid_class_request reqtype,
-+				  u64 source, bool from_bpf);
-+int dispatch_hid_bpf_output_report(struct hid_device *hdev, __u8 *buf, u32 size,
-+				   u64 source, bool from_bpf);
-+int hid_bpf_connect_device(struct hid_device *hdev);
-+void hid_bpf_disconnect_device(struct hid_device *hdev);
-+void hid_bpf_destroy_device(struct hid_device *hid);
-+int hid_bpf_device_init(struct hid_device *hid);
-+const u8 *call_hid_bpf_rdesc_fixup(struct hid_device *hdev, const u8 *rdesc, unsigned int *size);
-+#else /* CONFIG_HID_BPF */
-+static inline u8 *dispatch_hid_bpf_device_event(struct hid_device *hid, enum hid_report_type type,
-+						u8 *data, u32 *size, int interrupt,
-+						u64 source, bool from_bpf) { return data; }
-+static inline int dispatch_hid_bpf_raw_requests(struct hid_device *hdev,
-+						unsigned char reportnum, u8 *buf,
-+						u32 size, enum hid_report_type rtype,
-+						enum hid_class_request reqtype,
-+						u64 source, bool from_bpf) { return 0; }
-+static inline int dispatch_hid_bpf_output_report(struct hid_device *hdev, __u8 *buf, u32 size,
-+						 u64 source, bool from_bpf) { return 0; }
-+static inline int hid_bpf_connect_device(struct hid_device *hdev) { return 0; }
-+static inline void hid_bpf_disconnect_device(struct hid_device *hdev) {}
-+static inline void hid_bpf_destroy_device(struct hid_device *hid) {}
-+static inline int hid_bpf_device_init(struct hid_device *hid) { return 0; }
-+static inline const u8 *call_hid_bpf_rdesc_fixup(struct hid_device *hdev, const u8 *rdesc,
-+						 unsigned int *size) { return rdesc; }
-+
-+#endif /* CONFIG_HID_BPF */
-+
-+#endif /* __HID_BPF_H */
-diff --git a/include/linux/hidraw.h b/include/linux/hidraw.h
-index cd67f4ca5599..18fd30a288de 100644
--- a/include/linux/hidraw.h
-+++ b/include/linux/hidraw.h
-@@ -32,6 +32,7 @@ struct hidraw_list {
- 	struct hidraw *hidraw;
- 	struct list_head node;
- 	struct mutex read_mutex;
-+	bool revoked;
- };
- 
- #ifdef CONFIG_HIDRAW
-diff --git a/include/linux/highmem-internal.h b/include/linux/highmem-internal.h
-index 034b1106d022..0574c21ca45d 100644
--- a/include/linux/highmem-internal.h
-+++ b/include/linux/highmem-internal.h
-@@ -7,7 +7,7 @@
-  */
- #ifdef CONFIG_KMAP_LOCAL
- void *__kmap_local_pfn_prot(unsigned long pfn, pgprot_t prot);
-void *__kmap_local_page_prot(struct page *page, pgprot_t prot);
-+void *__kmap_local_page_prot(const struct page *page, pgprot_t prot);
- void kunmap_local_indexed(const void *vaddr);
- void kmap_local_fork(struct task_struct *tsk);
- void __kmap_local_sched_out(void);
-@@ -33,7 +33,7 @@ static inline void kmap_flush_tlb(unsigned long addr) { }
- #endif
- 
- void *kmap_high(struct page *page);
-void kunmap_high(struct page *page);
-+void kunmap_high(const struct page *page);
- void __kmap_flush_unused(void);
- struct page *__kmap_to_page(void *addr);
- 
-@@ -50,7 +50,7 @@ static inline void *kmap(struct page *page)
- 	return addr;
- }
- 
-static inline void kunmap(struct page *page)
-+static inline void kunmap(const struct page *page)
- {
- 	might_sleep();
- 	if (!PageHighMem(page))
-@@ -68,18 +68,26 @@ static inline void kmap_flush_unused(void)
- 	__kmap_flush_unused();
- }
- 
-static inline void *kmap_local_page(struct page *page)
-+static inline void *kmap_local_page(const struct page *page)
- {
- 	return __kmap_local_page_prot(page, kmap_prot);
- }
- 
-static inline void *kmap_local_folio(struct folio *folio, size_t offset)
-+static inline void *kmap_local_page_try_from_panic(const struct page *page)
- {
-	struct page *page = folio_page(folio, offset / PAGE_SIZE);
-+	if (!PageHighMem(page))
-+		return page_address(page);
-+	/* If the page is in HighMem, it's not safe to kmap it.*/
-+	return NULL;
-+}
-+
-+static inline void *kmap_local_folio(const struct folio *folio, size_t offset)
-+{
-+	const struct page *page = folio_page(folio, offset / PAGE_SIZE);
- 	return __kmap_local_page_prot(page, kmap_prot) + offset % PAGE_SIZE;
- }
- 
-static inline void *kmap_local_page_prot(struct page *page, pgprot_t prot)
-+static inline void *kmap_local_page_prot(const struct page *page, pgprot_t prot)
- {
- 	return __kmap_local_page_prot(page, prot);
- }
-@@ -94,7 +102,7 @@ static inline void __kunmap_local(const void *vaddr)
- 	kunmap_local_indexed(vaddr);
- }
- 
-static inline void *kmap_atomic_prot(struct page *page, pgprot_t prot)
-+static inline void *kmap_atomic_prot(const struct page *page, pgprot_t prot)
- {
- 	if (IS_ENABLED(CONFIG_PREEMPT_RT))
- 		migrate_disable();
-@@ -105,7 +113,7 @@ static inline void *kmap_atomic_prot(struct page *page, pgprot_t prot)
- 	return __kmap_local_page_prot(page, prot);
- }
- 
-static inline void *kmap_atomic(struct page *page)
-+static inline void *kmap_atomic(const struct page *page)
- {
- 	return kmap_atomic_prot(page, kmap_prot);
- }
-@@ -131,28 +139,26 @@ static inline void __kunmap_atomic(const void *addr)
- 		preempt_enable();
- }
- 
-unsigned int __nr_free_highpages(void);
-extern atomic_long_t _totalhigh_pages;
-+unsigned long __nr_free_highpages(void);
-+unsigned long __totalhigh_pages(void);
- 
-static inline unsigned int nr_free_highpages(void)
-+static inline unsigned long nr_free_highpages(void)
- {
- 	return __nr_free_highpages();
- }
- 
- static inline unsigned long totalhigh_pages(void)
- {
-	return (unsigned long)atomic_long_read(&_totalhigh_pages);
-}
-
-static inline void totalhigh_pages_add(long count)
-{
-	atomic_long_add(count, &_totalhigh_pages);
-+	return __totalhigh_pages();
- }
- 
- static inline bool is_kmap_addr(const void *x)
- {
- 	unsigned long addr = (unsigned long)x;
-	return addr >= PKMAP_ADDR(0) && addr < PKMAP_ADDR(LAST_PKMAP);
-+
-+	return (addr >= PKMAP_ADDR(0) && addr < PKMAP_ADDR(LAST_PKMAP)) ||
-+		(addr >= __fix_to_virt(FIX_KMAP_END) &&
-+		 addr < __fix_to_virt(FIX_KMAP_BEGIN));
- }
- #else /* CONFIG_HIGHMEM */
- 
-@@ -167,27 +173,32 @@ static inline void *kmap(struct page *page)
- 	return page_address(page);
- }
- 
-static inline void kunmap_high(struct page *page) { }
-+static inline void kunmap_high(const struct page *page) { }
- static inline void kmap_flush_unused(void) { }
- 
-static inline void kunmap(struct page *page)
-+static inline void kunmap(const struct page *page)
- {
- #ifdef ARCH_HAS_FLUSH_ON_KUNMAP
- 	kunmap_flush_on_unmap(page_address(page));
- #endif
- }
- 
-static inline void *kmap_local_page(struct page *page)
-+static inline void *kmap_local_page(const struct page *page)
-+{
-+	return page_address(page);
-+}
-+
-+static inline void *kmap_local_page_try_from_panic(const struct page *page)
- {
- 	return page_address(page);
- }
- 
-static inline void *kmap_local_folio(struct folio *folio, size_t offset)
-+static inline void *kmap_local_folio(const struct folio *folio, size_t offset)
- {
-	return page_address(&folio->page) + offset;
-+	return folio_address(folio) + offset;
- }
- 
-static inline void *kmap_local_page_prot(struct page *page, pgprot_t prot)
-+static inline void *kmap_local_page_prot(const struct page *page, pgprot_t prot)
- {
- 	return kmap_local_page(page);
- }
-@@ -200,11 +211,11 @@ static inline void *kmap_local_pfn(unsigned long pfn)
- static inline void __kunmap_local(const void *addr)
- {
- #ifdef ARCH_HAS_FLUSH_ON_KUNMAP
-	kunmap_flush_on_unmap(addr);
-+	kunmap_flush_on_unmap(PTR_ALIGN_DOWN(addr, PAGE_SIZE));
- #endif
- }
- 
-static inline void *kmap_atomic(struct page *page)
-+static inline void *kmap_atomic(const struct page *page)
- {
- 	if (IS_ENABLED(CONFIG_PREEMPT_RT))
- 		migrate_disable();
-@@ -214,7 +225,7 @@ static inline void *kmap_atomic(struct page *page)
- 	return page_address(page);
- }
- 
-static inline void *kmap_atomic_prot(struct page *page, pgprot_t prot)
-+static inline void *kmap_atomic_prot(const struct page *page, pgprot_t prot)
- {
- 	return kmap_atomic(page);
- }
-@@ -227,7 +238,7 @@ static inline void *kmap_atomic_pfn(unsigned long pfn)
- static inline void __kunmap_atomic(const void *addr)
- {
- #ifdef ARCH_HAS_FLUSH_ON_KUNMAP
-	kunmap_flush_on_unmap(addr);
-+	kunmap_flush_on_unmap(PTR_ALIGN_DOWN(addr, PAGE_SIZE));
- #endif
- 	pagefault_enable();
- 	if (IS_ENABLED(CONFIG_PREEMPT_RT))
-@@ -236,8 +247,8 @@ static inline void __kunmap_atomic(const void *addr)
- 		preempt_enable();
- }
- 
-static inline unsigned int nr_free_highpages(void) { return 0; }
-static inline unsigned long totalhigh_pages(void) { return 0UL; }
-+static inline unsigned long nr_free_highpages(void) { return 0; }
-+static inline unsigned long totalhigh_pages(void) { return 0; }
- 
- static inline bool is_kmap_addr(const void *x)
- {
-diff --git a/include/linux/highmem.h b/include/linux/highmem.h
-index 25679035ca28..abc20f9810fd 100644
--- a/include/linux/highmem.h
-+++ b/include/linux/highmem.h
-@@ -6,6 +6,7 @@
- #include <linux/kernel.h>
- #include <linux/bug.h>
- #include <linux/cacheflush.h>
-+#include <linux/kmsan.h>
- #include <linux/mm.h>
- #include <linux/uaccess.h>
- #include <linux/hardirq.h>
-@@ -42,7 +43,7 @@ static inline void *kmap(struct page *page);
-  * Counterpart to kmap(). A NOOP for CONFIG_HIGHMEM=n and for mappings of
-  * pages in the low memory area.
-  */
-static inline void kunmap(struct page *page);
-+static inline void kunmap(const struct page *page);
- 
- /**
-  * kmap_to_page - Get the page for a kmap'ed address
-@@ -85,14 +86,14 @@ static inline void kmap_flush_unused(void);
-  * virtual address of the direct mapping. Only real highmem pages are
-  * temporarily mapped.
-  *
- * While it is significantly faster than kmap() for the higmem case it
- * comes with restrictions about the pointer validity.
-+ * While kmap_local_page() is significantly faster than kmap() for the highmem
-+ * case it comes with restrictions about the pointer validity.
-  *
-  * On HIGHMEM enabled systems mapping a highmem page has the side effect of
-  * disabling migration in order to keep the virtual address stable across
-  * preemption. No caller of kmap_local_page() can rely on this side effect.
-  */
-static inline void *kmap_local_page(struct page *page);
-+static inline void *kmap_local_page(const struct page *page);
- 
- /**
-  * kmap_local_folio - Map a page in this folio for temporary usage
-@@ -118,9 +119,8 @@ static inline void *kmap_local_page(struct page *page);
-  * virtual address of the direct mapping. Only real highmem pages are
-  * temporarily mapped.
-  *
- * While it is significantly faster than kmap() for the higmem case it
- * comes with restrictions about the pointer validity. Only use when really
- * necessary.
-+ * While it is significantly faster than kmap() for the highmem case it
-+ * comes with restrictions about the pointer validity.
-  *
-  * On HIGHMEM enabled systems mapping a highmem page has the side effect of
-  * disabling migration in order to keep the virtual address stable across
-@@ -129,7 +129,7 @@ static inline void *kmap_local_page(struct page *page);
-  * Context: Can be invoked from any context.
-  * Return: The virtual address of @offset.
-  */
-static inline void *kmap_local_folio(struct folio *folio, size_t offset);
-+static inline void *kmap_local_folio(const struct folio *folio, size_t offset);
- 
- /**
-  * kmap_atomic - Atomically map a page for temporary usage - Deprecated!
-@@ -176,10 +176,10 @@ static inline void *kmap_local_folio(struct folio *folio, size_t offset);
-  * kunmap_atomic(vaddr2);
-  * kunmap_atomic(vaddr1);
-  */
-static inline void *kmap_atomic(struct page *page);
-+static inline void *kmap_atomic(const struct page *page);
- 
- /* Highmem related interfaces for management code */
-static inline unsigned int nr_free_highpages(void);
-+static inline unsigned long nr_free_highpages(void);
- static inline unsigned long totalhigh_pages(void);
- 
- #ifndef ARCH_HAS_FLUSH_ANON_PAGE
-@@ -207,31 +207,30 @@ static inline void clear_user_highpage(struct page *page, unsigned long vaddr)
- }
- #endif
- 
-#ifndef __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE
-+#ifndef vma_alloc_zeroed_movable_folio
- /**
- * alloc_zeroed_user_highpage_movable - Allocate a zeroed HIGHMEM page for a VMA that the caller knows can move
- * @vma: The VMA the page is to be allocated for
- * @vaddr: The virtual address the page will be inserted into
- *
- * Returns: The allocated and zeroed HIGHMEM page
-+ * vma_alloc_zeroed_movable_folio - Allocate a zeroed page for a VMA.
-+ * @vma: The VMA the page is to be allocated for.
-+ * @vaddr: The virtual address the page will be inserted into.
-  *
- * This function will allocate a page for a VMA that the caller knows will
- * be able to migrate in the future using move_pages() or reclaimed
-+ * This function will allocate a page suitable for inserting into this
-+ * VMA at this virtual address.  It may be allocated from highmem or
-+ * the movable zone.  An architecture may provide its own implementation.
-  *
- * An architecture may override this function by defining
- * __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE and providing their own
- * implementation.
-+ * Return: A folio containing one allocated and zeroed page or NULL if
-+ * we are out of memory.
-  */
-static inline struct page *
-alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,
-+static inline
-+struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma,
- 				   unsigned long vaddr)
- {
-	struct page *page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
-+	struct folio *folio;
- 
-	if (page)
-		clear_user_highpage(page, vaddr);
-+	folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, vaddr);
-+	if (folio && user_alloc_needs_zeroing())
-+		clear_user_highpage(&folio->page, vaddr);
- 
-	return page;
-+	return folio;
- }
- #endif
- 
-@@ -244,18 +243,18 @@ static inline void clear_highpage(struct page *page)
- 
- static inline void clear_highpage_kasan_tagged(struct page *page)
- {
-	u8 tag;
-+	void *kaddr = kmap_local_page(page);
- 
-	tag = page_kasan_tag(page);
-	page_kasan_tag_reset(page);
-	clear_highpage(page);
-	page_kasan_tag_set(page, tag);
-+	clear_page(kasan_reset_tag(kaddr));
-+	kunmap_local(kaddr);
- }
- 
-#ifndef __HAVE_ARCH_TAG_CLEAR_HIGHPAGE
-+#ifndef __HAVE_ARCH_TAG_CLEAR_HIGHPAGES
- 
-static inline void tag_clear_highpage(struct page *page)
-+/* Return false to let people know we did not initialize the pages */
-+static inline bool tag_clear_highpages(struct page *page, int numpages)
- {
-+	return false;
- }
- 
- #endif
-@@ -295,12 +294,6 @@ static inline void zero_user_segment(struct page *page,
- 	zero_user_segments(page, start, end, 0, 0);
- }
- 
-static inline void zero_user(struct page *page,
-	unsigned start, unsigned size)
-{
-	zero_user_segments(page, start, start + size, 0, 0);
-}
-
- #ifndef __HAVE_ARCH_COPY_USER_HIGHPAGE
- 
- static inline void copy_user_highpage(struct page *to, struct page *from,
-@@ -311,6 +304,7 @@ static inline void copy_user_highpage(struct page *to, struct page *from,
- 	vfrom = kmap_local_page(from);
- 	vto = kmap_local_page(to);
- 	copy_user_page(vto, vfrom, vaddr, to);
-+	kmsan_unpoison_memory(page_address(to), PAGE_SIZE);
- 	kunmap_local(vto);
- 	kunmap_local(vfrom);
- }
-@@ -326,12 +320,73 @@ static inline void copy_highpage(struct page *to, struct page *from)
- 	vfrom = kmap_local_page(from);
- 	vto = kmap_local_page(to);
- 	copy_page(vto, vfrom);
-+	kmsan_copy_page_meta(to, from);
- 	kunmap_local(vto);
- 	kunmap_local(vfrom);
- }
- 
- #endif
- 
-+#ifdef copy_mc_to_kernel
-+/*
-+ * If architecture supports machine check exception handling, define the
-+ * #MC versions of copy_user_highpage and copy_highpage. They copy a memory
-+ * page with #MC in source page (@from) handled, and return the number
-+ * of bytes not copied if there was a #MC, otherwise 0 for success.
-+ */
-+static inline int copy_mc_user_highpage(struct page *to, struct page *from,
-+					unsigned long vaddr, struct vm_area_struct *vma)
-+{
-+	unsigned long ret;
-+	char *vfrom, *vto;
-+
-+	vfrom = kmap_local_page(from);
-+	vto = kmap_local_page(to);
-+	ret = copy_mc_to_kernel(vto, vfrom, PAGE_SIZE);
-+	if (!ret)
-+		kmsan_unpoison_memory(page_address(to), PAGE_SIZE);
-+	kunmap_local(vto);
-+	kunmap_local(vfrom);
-+
-+	if (ret)
-+		memory_failure_queue(page_to_pfn(from), 0);
-+
-+	return ret;
-+}
-+
-+static inline int copy_mc_highpage(struct page *to, struct page *from)
-+{
-+	unsigned long ret;
-+	char *vfrom, *vto;
-+
-+	vfrom = kmap_local_page(from);
-+	vto = kmap_local_page(to);
-+	ret = copy_mc_to_kernel(vto, vfrom, PAGE_SIZE);
-+	if (!ret)
-+		kmsan_copy_page_meta(to, from);
-+	kunmap_local(vto);
-+	kunmap_local(vfrom);
-+
-+	if (ret)
-+		memory_failure_queue(page_to_pfn(from), 0);
-+
-+	return ret;
-+}
-+#else
-+static inline int copy_mc_user_highpage(struct page *to, struct page *from,
-+					unsigned long vaddr, struct vm_area_struct *vma)
-+{
-+	copy_user_highpage(to, from, vaddr, vma);
-+	return 0;
-+}
-+
-+static inline int copy_mc_highpage(struct page *to, struct page *from)
-+{
-+	copy_highpage(to, from);
-+	return 0;
-+}
-+#endif
-+
- static inline void memcpy_page(struct page *dst_page, size_t dst_off,
- 			       struct page *src_page, size_t src_off,
- 			       size_t len)
-@@ -345,6 +400,33 @@ static inline void memcpy_page(struct page *dst_page, size_t dst_off,
- 	kunmap_local(dst);
- }
- 
-+static inline void memcpy_folio(struct folio *dst_folio, size_t dst_off,
-+		struct folio *src_folio, size_t src_off, size_t len)
-+{
-+	VM_BUG_ON(dst_off + len > folio_size(dst_folio));
-+	VM_BUG_ON(src_off + len > folio_size(src_folio));
-+
-+	do {
-+		char *dst = kmap_local_folio(dst_folio, dst_off);
-+		const char *src = kmap_local_folio(src_folio, src_off);
-+		size_t chunk = len;
-+
-+		if (folio_test_highmem(dst_folio) &&
-+		    chunk > PAGE_SIZE - offset_in_page(dst_off))
-+			chunk = PAGE_SIZE - offset_in_page(dst_off);
-+		if (folio_test_highmem(src_folio) &&
-+		    chunk > PAGE_SIZE - offset_in_page(src_off))
-+			chunk = PAGE_SIZE - offset_in_page(src_off);
-+		memcpy(dst, src, chunk);
-+		kunmap_local(src);
-+		kunmap_local(dst);
-+
-+		dst_off += chunk;
-+		src_off += chunk;
-+		len -= chunk;
-+	} while (len > 0);
-+}
-+
- static inline void memset_page(struct page *page, size_t offset, int val,
- 			       size_t len)
- {
-@@ -387,6 +469,170 @@ static inline void memzero_page(struct page *page, size_t offset, size_t len)
- }
- 
- /**
-+ * memcpy_from_folio - Copy a range of bytes from a folio.
-+ * @to: The memory to copy to.
-+ * @folio: The folio to read from.
-+ * @offset: The first byte in the folio to read.
-+ * @len: The number of bytes to copy.
-+ */
-+static inline void memcpy_from_folio(char *to, struct folio *folio,
-+		size_t offset, size_t len)
-+{
-+	VM_BUG_ON(offset + len > folio_size(folio));
-+
-+	do {
-+		const char *from = kmap_local_folio(folio, offset);
-+		size_t chunk = len;
-+
-+		if (folio_test_partial_kmap(folio) &&
-+		    chunk > PAGE_SIZE - offset_in_page(offset))
-+			chunk = PAGE_SIZE - offset_in_page(offset);
-+		memcpy(to, from, chunk);
-+		kunmap_local(from);
-+
-+		to += chunk;
-+		offset += chunk;
-+		len -= chunk;
-+	} while (len > 0);
-+}
-+
-+/**
-+ * memcpy_to_folio - Copy a range of bytes to a folio.
-+ * @folio: The folio to write to.
-+ * @offset: The first byte in the folio to store to.
-+ * @from: The memory to copy from.
-+ * @len: The number of bytes to copy.
-+ */
-+static inline void memcpy_to_folio(struct folio *folio, size_t offset,
-+		const char *from, size_t len)
-+{
-+	VM_BUG_ON(offset + len > folio_size(folio));
-+
-+	do {
-+		char *to = kmap_local_folio(folio, offset);
-+		size_t chunk = len;
-+
-+		if (folio_test_partial_kmap(folio) &&
-+		    chunk > PAGE_SIZE - offset_in_page(offset))
-+			chunk = PAGE_SIZE - offset_in_page(offset);
-+		memcpy(to, from, chunk);
-+		kunmap_local(to);
-+
-+		from += chunk;
-+		offset += chunk;
-+		len -= chunk;
-+	} while (len > 0);
-+
-+	flush_dcache_folio(folio);
-+}
-+
-+/**
-+ * folio_zero_tail - Zero the tail of a folio.
-+ * @folio: The folio to zero.
-+ * @offset: The byte offset in the folio to start zeroing at.
-+ * @kaddr: The address the folio is currently mapped to.
-+ *
-+ * If you have already used kmap_local_folio() to map a folio, written
-+ * some data to it and now need to zero the end of the folio (and flush
-+ * the dcache), you can use this function.  If you do not have the
-+ * folio kmapped (eg the folio has been partially populated by DMA),
-+ * use folio_zero_range() or folio_zero_segment() instead.
-+ *
-+ * Return: An address which can be passed to kunmap_local().
-+ */
-+static inline __must_check void *folio_zero_tail(struct folio *folio,
-+		size_t offset, void *kaddr)
-+{
-+	size_t len = folio_size(folio) - offset;
-+
-+	if (folio_test_partial_kmap(folio)) {
-+		size_t max = PAGE_SIZE - offset_in_page(offset);
-+
-+		while (len > max) {
-+			memset(kaddr, 0, max);
-+			kunmap_local(kaddr);
-+			len -= max;
-+			offset += max;
-+			max = PAGE_SIZE;
-+			kaddr = kmap_local_folio(folio, offset);
-+		}
-+	}
-+
-+	memset(kaddr, 0, len);
-+	flush_dcache_folio(folio);
-+
-+	return kaddr;
-+}
-+
-+/**
-+ * folio_fill_tail - Copy some data to a folio and pad with zeroes.
-+ * @folio: The destination folio.
-+ * @offset: The offset into @folio at which to start copying.
-+ * @from: The data to copy.
-+ * @len: How many bytes of data to copy.
-+ *
-+ * This function is most useful for filesystems which support inline data.
-+ * When they want to copy data from the inode into the page cache, this
-+ * function does everything for them.  It supports large folios even on
-+ * HIGHMEM configurations.
-+ */
-+static inline void folio_fill_tail(struct folio *folio, size_t offset,
-+		const char *from, size_t len)
-+{
-+	char *to = kmap_local_folio(folio, offset);
-+
-+	VM_BUG_ON(offset + len > folio_size(folio));
-+
-+	if (folio_test_partial_kmap(folio)) {
-+		size_t max = PAGE_SIZE - offset_in_page(offset);
-+
-+		while (len > max) {
-+			memcpy(to, from, max);
-+			kunmap_local(to);
-+			len -= max;
-+			from += max;
-+			offset += max;
-+			max = PAGE_SIZE;
-+			to = kmap_local_folio(folio, offset);
-+		}
-+	}
-+
-+	memcpy(to, from, len);
-+	to = folio_zero_tail(folio, offset + len, to + len);
-+	kunmap_local(to);
-+}
-+
-+/**
-+ * memcpy_from_file_folio - Copy some bytes from a file folio.
-+ * @to: The destination buffer.
-+ * @folio: The folio to copy from.
-+ * @pos: The position in the file.
-+ * @len: The maximum number of bytes to copy.
-+ *
-+ * Copy up to @len bytes from this folio.  This may be limited by PAGE_SIZE
-+ * if the folio comes from HIGHMEM, and by the size of the folio.
-+ *
-+ * Return: The number of bytes copied from the folio.
-+ */
-+static inline size_t memcpy_from_file_folio(char *to, struct folio *folio,
-+		loff_t pos, size_t len)
-+{
-+	size_t offset = offset_in_folio(folio, pos);
-+	char *from = kmap_local_folio(folio, offset);
-+
-+	if (folio_test_partial_kmap(folio)) {
-+		offset = offset_in_page(offset);
-+		len = min_t(size_t, len, PAGE_SIZE - offset);
-+	} else
-+		len = min(len, folio_size(folio) - offset);
-+
-+	memcpy(to, from, len);
-+	kunmap_local(from);
-+
-+	return len;
-+}
-+
-+/**
-  * folio_zero_segments() - Zero two byte ranges in a folio.
-  * @folio: The folio to write to.
-  * @start1: The first byte to zero.
-@@ -424,4 +670,18 @@ static inline void folio_zero_range(struct folio *folio,
- 	zero_user_segments(&folio->page, start, start + length, 0, 0);
- }
- 
-+/**
-+ * folio_release_kmap - Unmap a folio and drop a refcount.
-+ * @folio: The folio to release.
-+ * @addr: The address previously returned by a call to kmap_local_folio().
-+ *
-+ * It is common, eg in directory handling to kmap a folio.  This function
-+ * unmaps the folio and drops the refcount that was being held to keep the
-+ * folio alive while we accessed it.
-+ */
-+static inline void folio_release_kmap(struct folio *folio, void *addr)
-+{
-+	kunmap_local(addr);
-+	folio_put(folio);
-+}
- #endif /* _LINUX_HIGHMEM_H */
-diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h
-index 116e8bd68c99..ca1ec437a3ca 100644
--- a/include/linux/hisi_acc_qm.h
-+++ b/include/linux/hisi_acc_qm.h
-@@ -43,6 +43,7 @@
- #define QM_MB_CMD_CQC_BT                0x5
- #define QM_MB_CMD_SQC_VFT_V2            0x6
- #define QM_MB_CMD_STOP_QP               0x8
-+#define QM_MB_CMD_FLUSH_QM		0x9
- #define QM_MB_CMD_SRC                   0xc
- #define QM_MB_CMD_DST                   0xd
- 
-@@ -87,29 +88,6 @@
- #define PEH_AXUSER_CFG			0x401001
- #define PEH_AXUSER_CFG_ENABLE		0xffffffff
- 
-#define QM_AXI_RRESP			BIT(0)
-#define QM_AXI_BRESP			BIT(1)
-#define QM_ECC_MBIT			BIT(2)
-#define QM_ECC_1BIT			BIT(3)
-#define QM_ACC_GET_TASK_TIMEOUT		BIT(4)
-#define QM_ACC_DO_TASK_TIMEOUT		BIT(5)
-#define QM_ACC_WB_NOT_READY_TIMEOUT	BIT(6)
-#define QM_SQ_CQ_VF_INVALID		BIT(7)
-#define QM_CQ_VF_INVALID		BIT(8)
-#define QM_SQ_VF_INVALID		BIT(9)
-#define QM_DB_TIMEOUT			BIT(10)
-#define QM_OF_FIFO_OF			BIT(11)
-#define QM_DB_RANDOM_INVALID		BIT(12)
-#define QM_MAILBOX_TIMEOUT		BIT(13)
-#define QM_FLR_TIMEOUT			BIT(14)
-
-#define QM_BASE_NFE	(QM_AXI_RRESP | QM_AXI_BRESP | QM_ECC_MBIT | \
-			 QM_ACC_GET_TASK_TIMEOUT | QM_DB_TIMEOUT | \
-			 QM_OF_FIFO_OF | QM_DB_RANDOM_INVALID | \
-			 QM_MAILBOX_TIMEOUT | QM_FLR_TIMEOUT)
-#define QM_BASE_CE			QM_ECC_1BIT
-
-#define QM_Q_DEPTH			1024
- #define QM_MIN_QNUM                     2
- #define HISI_ACC_SGL_SGE_NR_MAX		255
- #define QM_SHAPER_CFG			0x100164
-@@ -119,36 +97,40 @@
- /* page number for queue file region */
- #define QM_DOORBELL_PAGE_NR		1
- 
-+#define QM_DEV_ALG_MAX_LEN		256
-+
-+#define QM_MIG_REGION_SEL		0x100198
-+#define QM_MIG_REGION_EN		BIT(0)
-+
- /* uacce mode of the driver */
- #define UACCE_MODE_NOUACCE		0 /* don't use uacce */
- #define UACCE_MODE_SVA			1 /* use uacce sva mode */
- #define UACCE_MODE_DESC	"0(default) means only register to crypto, 1 means both register to crypto and uacce"
- 
-+#define QM_ECC_MBIT			BIT(2)
-+
- enum qm_stop_reason {
- 	QM_NORMAL,
- 	QM_SOFT_RESET,
-	QM_FLR,
-+	QM_DOWN,
- };
- 
- enum qm_state {
-	QM_INIT = 0,
-	QM_START,
-	QM_CLOSE,
-+	QM_WORK = 0,
- 	QM_STOP,
- };
- 
- enum qp_state {
-	QP_INIT = 1,
-	QP_START,
-+	QP_START = 1,
- 	QP_STOP,
-	QP_CLOSE,
- };
- 
- enum qm_hw_ver {
-	QM_HW_UNKNOWN = -1,
- 	QM_HW_V1 = 0x20,
- 	QM_HW_V2 = 0x21,
- 	QM_HW_V3 = 0x30,
-+	QM_HW_V4 = 0x50,
-+	QM_HW_V5 = 0x51,
- };
- 
- enum qm_fun_type {
-@@ -168,6 +150,34 @@ enum qm_vf_state {
- 	QM_NOT_READY,
- };
- 
-+enum qm_misc_ctl_bits {
-+	QM_DRIVER_REMOVING = 0x0,
-+	QM_RST_SCHED,
-+	QM_RESETTING,
-+	QM_MODULE_PARAM,
-+};
-+
-+enum qm_cap_bits {
-+	QM_SUPPORT_DB_ISOLATION = 0x0,
-+	QM_SUPPORT_FUNC_QOS,
-+	QM_SUPPORT_STOP_QP,
-+	QM_SUPPORT_STOP_FUNC,
-+	QM_SUPPORT_MB_COMMAND,
-+	QM_SUPPORT_SVA_PREFETCH,
-+	QM_SUPPORT_RPM,
-+	QM_SUPPORT_DAE,
-+};
-+
-+struct qm_dev_alg {
-+	u64 alg_msk;
-+	const char *alg;
-+};
-+
-+struct qm_dev_dfx {
-+	u32 dev_state;
-+	u32 dev_timeout;
-+};
-+
- struct dfx_diff_registers {
- 	u32 *regs;
- 	u32 reg_offset;
-@@ -196,6 +206,7 @@ struct qm_debug {
- 	struct dentry *debug_root;
- 	struct dentry *qm_d;
- 	struct debugfs_file files[DEBUG_FILE_NUM];
-+	struct qm_dev_dfx dev_dfx;
- 	unsigned int *qm_last_words;
- 	/* ACC engines recoreding last regs */
- 	unsigned int *last_words;
-@@ -228,16 +239,28 @@ struct hisi_qm_status {
- 
- struct hisi_qm;
- 
-struct hisi_qm_err_info {
-	char *acpi_rst;
-	u32 msi_wr_port;
-+enum acc_err_result {
-+	ACC_ERR_NONE,
-+	ACC_ERR_NEED_RESET,
-+	ACC_ERR_RECOVERED,
-+};
-+
-+struct hisi_qm_err_mask {
- 	u32 ecc_2bits_mask;
-	u32 dev_ce_mask;
-+	u32 shutdown_mask;
-+	u32 reset_mask;
- 	u32 ce;
- 	u32 nfe;
- 	u32 fe;
- };
- 
-+struct hisi_qm_err_info {
-+	char *acpi_rst;
-+	u32 msi_wr_port;
-+	struct hisi_qm_err_mask qm_err;
-+	struct hisi_qm_err_mask dev_err;
-+};
-+
- struct hisi_qm_err_status {
- 	u32 is_qm_ecc_mbit;
- 	u32 is_dev_ecc_mbit;
-@@ -253,9 +276,47 @@ struct hisi_qm_err_ini {
- 	void (*close_axi_master_ooo)(struct hisi_qm *qm);
- 	void (*open_sva_prefetch)(struct hisi_qm *qm);
- 	void (*close_sva_prefetch)(struct hisi_qm *qm);
-	void (*log_dev_hw_err)(struct hisi_qm *qm, u32 err_sts);
- 	void (*show_last_dfx_regs)(struct hisi_qm *qm);
- 	void (*err_info_init)(struct hisi_qm *qm);
-+	enum acc_err_result (*get_err_result)(struct hisi_qm *qm);
-+	bool (*dev_is_abnormal)(struct hisi_qm *qm);
-+	int (*set_priv_status)(struct hisi_qm *qm);
-+	void (*disable_axi_error)(struct hisi_qm *qm);
-+	void (*enable_axi_error)(struct hisi_qm *qm);
-+};
-+
-+struct hisi_qm_cap_info {
-+	u32 type;
-+	/* Register offset */
-+	u32 offset;
-+	/* Bit offset in register */
-+	u32 shift;
-+	u32 mask;
-+	u32 v1_val;
-+	u32 v2_val;
-+	u32 v3_val;
-+};
-+
-+struct hisi_qm_cap_query_info {
-+	u32 type;
-+	const char *name;
-+	u32 offset;
-+	u32 v1_val;
-+	u32 v2_val;
-+	u32 v3_val;
-+};
-+
-+struct hisi_qm_cap_record {
-+	u32 type;
-+	const char *name;
-+	u32 cap_val;
-+};
-+
-+struct hisi_qm_cap_tables {
-+	u32 qm_cap_size;
-+	struct hisi_qm_cap_record *qm_cap_table;
-+	u32 dev_cap_size;
-+	struct hisi_qm_cap_record *dev_cap_table;
- };
- 
- struct hisi_qm_list {
-@@ -269,6 +330,33 @@ struct hisi_qm_poll_data {
- 	struct hisi_qm *qm;
- 	struct work_struct work;
- 	u16 *qp_finish_id;
-+	u16 eqe_num;
-+};
-+
-+/**
-+ * struct qm_err_isolate
-+ * @isolate_lock: protects device error log
-+ * @err_threshold: user config error threshold which triggers isolation
-+ * @is_isolate: device isolation state
-+ * @uacce_hw_errs: index into qm device error list
-+ */
-+struct qm_err_isolate {
-+	struct mutex isolate_lock;
-+	u32 err_threshold;
-+	bool is_isolate;
-+	struct list_head qm_hw_errs;
-+};
-+
-+struct qm_rsv_buf {
-+	struct qm_sqc *sqc;
-+	struct qm_cqc *cqc;
-+	struct qm_eqc *eqc;
-+	struct qm_aeqc *aeqc;
-+	dma_addr_t sqc_dma;
-+	dma_addr_t cqc_dma;
-+	dma_addr_t eqc_dma;
-+	dma_addr_t aeqc_dma;
-+	struct qm_dma qcdma;
- };
- 
- struct hisi_qm {
-@@ -278,6 +366,9 @@ struct hisi_qm {
- 	struct pci_dev *pdev;
- 	void __iomem *io_base;
- 	void __iomem *db_io_base;
-+
-+	/* Capbility version, 0: not supports */
-+	u32 cap_ver;
- 	u32 sqe_size;
- 	u32 qp_base;
- 	u32 qp_num;
-@@ -286,6 +377,8 @@ struct hisi_qm {
- 	u32 max_qp_num;
- 	u32 vfs_num;
- 	u32 db_interval;
-+	u16 eq_depth;
-+	u16 aeq_depth;
- 	struct list_head list;
- 	struct hisi_qm_list *qm_list;
- 
-@@ -298,12 +391,16 @@ struct hisi_qm {
- 	dma_addr_t cqc_dma;
- 	dma_addr_t eqe_dma;
- 	dma_addr_t aeqe_dma;
-+	struct qm_rsv_buf xqc_buf;
- 
- 	struct hisi_qm_status status;
- 	const struct hisi_qm_err_ini *err_ini;
- 	struct hisi_qm_err_info err_info;
- 	struct hisi_qm_err_status err_status;
-	unsigned long misc_ctl; /* driver removing and reset sched */
-+	/* driver removing and reset sched */
-+	unsigned long misc_ctl;
-+	/* Device capability bit */
-+	unsigned long caps;
- 
- 	struct rw_semaphore qps_lock;
- 	struct idr qp_idr;
-@@ -312,6 +409,8 @@ struct hisi_qm {
- 
- 	struct mutex mailbox_lock;
- 
-+	struct mutex ifc_lock;
-+
- 	const struct hisi_qm_hw_ops *ops;
- 
- 	struct qm_debug debug;
-@@ -322,12 +421,8 @@ struct hisi_qm {
- 	struct work_struct rst_work;
- 	struct work_struct cmd_process;
- 
-	const char *algs;
- 	bool use_sva;
-	bool is_frozen;
- 
-	/* doorbell isolation enable */
-	bool use_db_isolation;
- 	resource_size_t phys_base;
- 	resource_size_t db_phys_base;
- 	struct uacce_device *uacce;
-@@ -335,6 +430,9 @@ struct hisi_qm {
- 	struct qm_shaper_factor *factor;
- 	u32 mb_qos;
- 	u32 type_rate;
-+	struct qm_err_isolate isolate_data;
-+
-+	struct hisi_qm_cap_tables cap_tables;
- };
- 
- struct hisi_qp_status {
-@@ -351,6 +449,8 @@ struct hisi_qp_ops {
- 
- struct hisi_qp {
- 	u32 qp_id;
-+	u16 sq_depth;
-+	u16 cq_depth;
- 	u8 alg_type;
- 	u8 req_type;
- 
-@@ -373,35 +473,6 @@ struct hisi_qp {
- 	struct uacce_queue *uacce_q;
- };
- 
-static inline int q_num_set(const char *val, const struct kernel_param *kp,
-			    unsigned int device)
-{
-	struct pci_dev *pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI,
-					      device, NULL);
-	u32 n, q_num;
-	int ret;
-
-	if (!val)
-		return -EINVAL;
-
-	if (!pdev) {
-		q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2);
-		pr_info("No device found currently, suppose queue number is %u\n",
-			q_num);
-	} else {
-		if (pdev->revision == QM_HW_V1)
-			q_num = QM_QNUM_V1;
-		else
-			q_num = QM_QNUM_V2;
-	}
-
-	ret = kstrtou32(val, 10, &n);
-	if (ret || n < QM_MIN_QNUM || n > q_num)
-		return -EINVAL;
-
-	return param_set_int(val, kp);
-}
-
- static inline int vfs_num_set(const char *val, const struct kernel_param *kp)
- {
- 	u32 n;
-@@ -447,12 +518,28 @@ static inline void hisi_qm_init_list(struct hisi_qm_list *qm_list)
- 	mutex_init(&qm_list->lock);
- }
- 
-+static inline void hisi_qm_add_list(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
-+{
-+	mutex_lock(&qm_list->lock);
-+	list_add_tail(&qm->list, &qm_list->list);
-+	mutex_unlock(&qm_list->lock);
-+}
-+
-+static inline void hisi_qm_del_list(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
-+{
-+	mutex_lock(&qm_list->lock);
-+	list_del(&qm->list);
-+	mutex_unlock(&qm_list->lock);
-+}
-+
-+int hisi_qm_q_num_set(const char *val, const struct kernel_param *kp,
-+		      unsigned int device);
- int hisi_qm_init(struct hisi_qm *qm);
- void hisi_qm_uninit(struct hisi_qm *qm);
- int hisi_qm_start(struct hisi_qm *qm);
- int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r);
- int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg);
-int hisi_qm_stop_qp(struct hisi_qp *qp);
-+void hisi_qm_stop_qp(struct hisi_qp *qp);
- int hisi_qp_send(struct hisi_qp *qp, const void *msg);
- void hisi_qm_debug_init(struct hisi_qm *qm);
- void hisi_qm_debug_regs_clear(struct hisi_qm *qm);
-@@ -461,11 +548,11 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen);
- int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs);
- void hisi_qm_dev_err_init(struct hisi_qm *qm);
- void hisi_qm_dev_err_uninit(struct hisi_qm *qm);
-int hisi_qm_diff_regs_init(struct hisi_qm *qm,
-		struct dfx_diff_registers *dregs, int reg_len);
-void hisi_qm_diff_regs_uninit(struct hisi_qm *qm, int reg_len);
-+int hisi_qm_regs_debugfs_init(struct hisi_qm *qm,
-+			  struct dfx_diff_registers *dregs, u32 reg_len);
-+void hisi_qm_regs_debugfs_uninit(struct hisi_qm *qm, u32 reg_len);
- void hisi_qm_acc_diff_regs_dump(struct hisi_qm *qm, struct seq_file *s,
-		struct dfx_diff_registers *dregs, int regs_len);
-+				struct dfx_diff_registers *dregs, u32 regs_len);
- 
- pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev,
- 					  pci_channel_state_t state);
-@@ -480,9 +567,9 @@ int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,
- struct hisi_acc_sgl_pool;
- struct hisi_acc_hw_sgl *hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
- 	struct scatterlist *sgl, struct hisi_acc_sgl_pool *pool,
-	u32 index, dma_addr_t *hw_sgl_dma);
-+	u32 index, dma_addr_t *hw_sgl_dma, enum dma_data_direction dir);
- void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
-			   struct hisi_acc_hw_sgl *hw_sgl);
-+			   struct hisi_acc_hw_sgl *hw_sgl, enum dma_data_direction dir);
- struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev,
- 						   u32 count, u32 sge_nr);
- void hisi_acc_free_sgl_pool(struct device *dev,
-@@ -492,8 +579,8 @@ int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num,
- void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num);
- void hisi_qm_dev_shutdown(struct pci_dev *pdev);
- void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list);
-int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list);
-void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list);
-+int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard);
-+void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard);
- int hisi_qm_resume(struct device *dev);
- int hisi_qm_suspend(struct device *dev);
- void hisi_qm_pm_uninit(struct hisi_qm *qm);
-@@ -501,6 +588,14 @@ void hisi_qm_pm_init(struct hisi_qm *qm);
- int hisi_qm_get_dfx_access(struct hisi_qm *qm);
- void hisi_qm_put_dfx_access(struct hisi_qm *qm);
- void hisi_qm_regs_dump(struct seq_file *s, struct debugfs_regset32 *regset);
-+u32 hisi_qm_get_hw_info(struct hisi_qm *qm,
-+			const struct hisi_qm_cap_info *info_table,
-+			u32 index, bool is_read);
-+u32 hisi_qm_get_cap_value(struct hisi_qm *qm,
-+			const struct hisi_qm_cap_query_info *info_table,
-+			u32 index, bool is_read);
-+int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs,
-+		     u32 dev_algs_size);
- 
- /* Used by VFIO ACC live migration driver */
- struct pci_driver *hisi_sec_get_pf_driver(void);
-diff --git a/include/linux/hmm-dma.h b/include/linux/hmm-dma.h
-new file mode 100644
-index 000000000000..f58b9fc71999
--- /dev/null
-+++ b/include/linux/hmm-dma.h
-@@ -0,0 +1,33 @@
-+/* SPDX-License-Identifier: GPL-2.0-or-later */
-+/* Copyright (c) 2024 NVIDIA Corporation & Affiliates */
-+#ifndef LINUX_HMM_DMA_H
-+#define LINUX_HMM_DMA_H
-+
-+#include <linux/dma-mapping.h>
-+
-+struct dma_iova_state;
-+struct pci_p2pdma_map_state;
-+
-+/*
-+ * struct hmm_dma_map - array of PFNs and DMA addresses
-+ *
-+ * @state: DMA IOVA state
-+ * @pfns: array of PFNs
-+ * @dma_list: array of DMA addresses
-+ * @dma_entry_size: size of each DMA entry in the array
-+ */
-+struct hmm_dma_map {
-+	struct dma_iova_state state;
-+	unsigned long *pfn_list;
-+	dma_addr_t *dma_list;
-+	size_t dma_entry_size;
-+};
-+
-+int hmm_dma_map_alloc(struct device *dev, struct hmm_dma_map *map,
-+		      size_t nr_entries, size_t dma_entry_size);
-+void hmm_dma_map_free(struct device *dev, struct hmm_dma_map *map);
-+dma_addr_t hmm_dma_map_pfn(struct device *dev, struct hmm_dma_map *map,
-+			   size_t idx,
-+			   struct pci_p2pdma_map_state *p2pdma_state);
-+bool hmm_dma_unmap_pfn(struct device *dev, struct hmm_dma_map *map, size_t idx);
-+#endif /* LINUX_HMM_DMA_H */
-diff --git a/include/linux/hmm.h b/include/linux/hmm.h
-index 126a36571667..db75ffc949a7 100644
--- a/include/linux/hmm.h
-+++ b/include/linux/hmm.h
-@@ -23,6 +23,10 @@ struct mmu_interval_notifier;
-  * HMM_PFN_WRITE - if the page memory can be written to (requires HMM_PFN_VALID)
-  * HMM_PFN_ERROR - accessing the pfn is impossible and the device should
-  *                 fail. ie poisoned memory, special pages, no vma, etc
-+ * HMM_PFN_P2PDMA - P2P page
-+ * HMM_PFN_P2PDMA_BUS - Bus mapped P2P transfer
-+ * HMM_PFN_DMA_MAPPED - Flag preserved on input-to-output transformation
-+ *                      to mark that page is already DMA mapped
-  *
-  * On input:
-  * 0                 - Return the current state of the page, do not fault it.
-@@ -36,13 +40,21 @@ enum hmm_pfn_flags {
- 	HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1),
- 	HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2),
- 	HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3),
-	HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8),
-+	/*
-+	 * Sticky flags, carried from input to output,
-+	 * don't forget to update HMM_PFN_INOUT_FLAGS
-+	 */
-+	HMM_PFN_DMA_MAPPED = 1UL << (BITS_PER_LONG - 4),
-+	HMM_PFN_P2PDMA     = 1UL << (BITS_PER_LONG - 5),
-+	HMM_PFN_P2PDMA_BUS = 1UL << (BITS_PER_LONG - 6),
-+
-+	HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 11),
- 
- 	/* Input flags */
- 	HMM_PFN_REQ_FAULT = HMM_PFN_VALID,
- 	HMM_PFN_REQ_WRITE = HMM_PFN_WRITE,
- 
-	HMM_PFN_FLAGS = 0xFFUL << HMM_PFN_ORDER_SHIFT,
-+	HMM_PFN_FLAGS = ~((1UL << HMM_PFN_ORDER_SHIFT) - 1),
- };
- 
- /*
-@@ -58,6 +70,14 @@ static inline struct page *hmm_pfn_to_page(unsigned long hmm_pfn)
- }
- 
- /*
-+ * hmm_pfn_to_phys() - return physical address pointed to by a device entry
-+ */
-+static inline phys_addr_t hmm_pfn_to_phys(unsigned long hmm_pfn)
-+{
-+	return __pfn_to_phys(hmm_pfn & ~HMM_PFN_FLAGS);
-+}
-+
-+/*
-  * hmm_pfn_to_map_order() - return the CPU mapping size order
-  *
-  * This is optionally useful to optimize processing of the pfn result
-diff --git a/include/linux/host1x.h b/include/linux/host1x.h
-index cb2100d9b0ff..9fa9c30a34e6 100644
--- a/include/linux/host1x.h
-+++ b/include/linux/host1x.h
-@@ -8,17 +8,23 @@
- 
- #include <linux/device.h>
- #include <linux/dma-direction.h>
-+#include <linux/dma-fence.h>
- #include <linux/spinlock.h>
- #include <linux/types.h>
- 
- enum host1x_class {
- 	HOST1X_CLASS_HOST1X = 0x1,
-+	HOST1X_CLASS_NVJPG1 = 0x7,
-+	HOST1X_CLASS_NVENC = 0x21,
-+	HOST1X_CLASS_NVENC1 = 0x22,
- 	HOST1X_CLASS_GR2D = 0x51,
- 	HOST1X_CLASS_GR2D_SB = 0x52,
- 	HOST1X_CLASS_VIC = 0x5D,
- 	HOST1X_CLASS_GR3D = 0x60,
-+	HOST1X_CLASS_NVJPG = 0xC0,
- 	HOST1X_CLASS_NVDEC = 0xF0,
- 	HOST1X_CLASS_NVDEC1 = 0xF5,
-+	HOST1X_CLASS_OFA = 0xF8,
- };
- 
- struct host1x;
-@@ -221,7 +227,9 @@ u32 host1x_syncpt_base_id(struct host1x_syncpt_base *base);
- void host1x_syncpt_release_vblank_reservation(struct host1x_client *client,
- 					      u32 syncpt_id);
- 
-struct dma_fence *host1x_fence_create(struct host1x_syncpt *sp, u32 threshold);
-+struct dma_fence *host1x_fence_create(struct host1x_syncpt *sp, u32 threshold,
-+				      bool timeout);
-+void host1x_fence_cancel(struct dma_fence *fence);
- 
- /*
-  * host1x channel
-@@ -288,8 +296,9 @@ struct host1x_job {
- 	u32 syncpt_incrs;
- 	u32 syncpt_end;
- 
-	/* Completion waiter ref */
-	void *waiter;
-+	/* Completion fence for job tracking */
-+	struct dma_fence *fence;
-+	struct dma_fence_cb fence_cb;
- 
- 	/* Maximum time to wait for this job */
- 	unsigned int timeout;
-@@ -439,7 +448,7 @@ int __host1x_client_register(struct host1x_client *client);
- 		__host1x_client_register(client);	\
- 	})
- 
-int host1x_client_unregister(struct host1x_client *client);
-+void host1x_client_unregister(struct host1x_client *client);
- 
- int host1x_client_suspend(struct host1x_client *client);
- int host1x_client_resume(struct host1x_client *client);
-@@ -462,6 +471,7 @@ struct host1x_memory_context {
- 	refcount_t ref;
- 	struct pid *owner;
- 
-+	struct device_dma_parameters dma_parms;
- 	struct device dev;
- 	u64 dma_mask;
- 	u32 stream_id;
-@@ -469,11 +479,13 @@ struct host1x_memory_context {
- 
- #ifdef CONFIG_IOMMU_API
- struct host1x_memory_context *host1x_memory_context_alloc(struct host1x *host1x,
-+							  struct device *dev,
- 							  struct pid *pid);
- void host1x_memory_context_get(struct host1x_memory_context *cd);
- void host1x_memory_context_put(struct host1x_memory_context *cd);
- #else
- static inline struct host1x_memory_context *host1x_memory_context_alloc(struct host1x *host1x,
-+									struct device *dev,
- 									struct pid *pid)
- {
- 	return NULL;
-diff --git a/include/linux/host1x_context_bus.h b/include/linux/host1x_context_bus.h
-index 72462737a6db..c928cb432680 100644
--- a/include/linux/host1x_context_bus.h
-+++ b/include/linux/host1x_context_bus.h
-@@ -9,7 +9,7 @@
- #include <linux/device.h>
- 
- #ifdef CONFIG_TEGRA_HOST1X_CONTEXT_BUS
-extern struct bus_type host1x_context_device_bus_type;
-+extern const struct bus_type host1x_context_device_bus_type;
- #endif
- 
- #endif
-diff --git a/include/linux/hp_sdc.h b/include/linux/hp_sdc.h
-index 6f1dee7e67e0..9be8704e2d38 100644
--- a/include/linux/hp_sdc.h
-+++ b/include/linux/hp_sdc.h
-@@ -180,7 +180,7 @@ switch (val) {						\
- 
- #define HP_SDC_CMD_SET_IM	0x40    /* 010xxxxx == set irq mask */
- 
-/* The documents provided do not explicitly state that all registers betweem 
-+/* The documents provided do not explicitly state that all registers between
-  * 0x01 and 0x1f inclusive can be read by sending their register index as a 
-  * command, but this is implied and appears to be the case.
-  */
-diff --git a/include/linux/hpet.h b/include/linux/hpet.h
-index 8604564b985d..21e69eaf7a36 100644
--- a/include/linux/hpet.h
-+++ b/include/linux/hpet.h
-@@ -30,7 +30,7 @@ struct hpet {
- 			unsigned long _hpet_compare;
- 		} _u1;
- 		u64 hpet_fsb[2];	/* FSB route */
-	} hpet_timers[1];
-+	} hpet_timers[];
- };
- 
- #define	hpet_mc		_u0._hpet_mc
-diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
-index 0ee140176f10..2cf1bf65b225 100644
--- a/include/linux/hrtimer.h
-+++ b/include/linux/hrtimer.h
-@@ -13,16 +13,12 @@
- #define _LINUX_HRTIMER_H
- 
- #include <linux/hrtimer_defs.h>
-#include <linux/rbtree.h>
-+#include <linux/hrtimer_types.h>
- #include <linux/init.h>
- #include <linux/list.h>
-#include <linux/percpu.h>
-#include <linux/seqlock.h>
-+#include <linux/percpu-defs.h>
-+#include <linux/rbtree.h>
- #include <linux/timer.h>
-#include <linux/timerqueue.h>
-
-struct hrtimer_clock_base;
-struct hrtimer_cpu_base;
- 
- /*
-  * Mode arguments of xxx_hrtimer functions:
-@@ -60,14 +56,6 @@ enum hrtimer_mode {
- };
- 
- /*
- * Return values for the callback function
- */
-enum hrtimer_restart {
-	HRTIMER_NORESTART,	/* Timer is not restarted */
-	HRTIMER_RESTART,	/* Timer must be restarted */
-};
-
-/*
-  * Values to track state of the timer
-  *
-  * Possible states:
-@@ -95,38 +83,6 @@ enum hrtimer_restart {
- #define HRTIMER_STATE_ENQUEUED	0x01
- 
- /**
- * struct hrtimer - the basic hrtimer structure
- * @node:	timerqueue node, which also manages node.expires,
- *		the absolute expiry time in the hrtimers internal
- *		representation. The time is related to the clock on
- *		which the timer is based. Is setup by adding
- *		slack to the _softexpires value. For non range timers
- *		identical to _softexpires.
- * @_softexpires: the absolute earliest expiry time of the hrtimer.
- *		The time which was given as expiry time when the timer
- *		was armed.
- * @function:	timer expiry callback function
- * @base:	pointer to the timer base (per cpu and per clock)
- * @state:	state information (See bit values above)
- * @is_rel:	Set if the timer was armed relative
- * @is_soft:	Set if hrtimer will be expired in soft interrupt context.
- * @is_hard:	Set if hrtimer will be expired in hard interrupt context
- *		even on RT.
- *
- * The hrtimer structure must be initialized by hrtimer_init()
- */
-struct hrtimer {
-	struct timerqueue_node		node;
-	ktime_t				_softexpires;
-	enum hrtimer_restart		(*function)(struct hrtimer *);
-	struct hrtimer_clock_base	*base;
-	u8				state;
-	u8				is_rel;
-	u8				is_soft;
-	u8				is_hard;
-};
-
-/**
-  * struct hrtimer_sleeper - simple sleeper structure
-  * @timer:	embedded timer structure
-  * @task:	task to wake up
-@@ -138,105 +94,6 @@ struct hrtimer_sleeper {
- 	struct task_struct *task;
- };
- 
-#ifdef CONFIG_64BIT
-# define __hrtimer_clock_base_align	____cacheline_aligned
-#else
-# define __hrtimer_clock_base_align
-#endif
-
-/**
- * struct hrtimer_clock_base - the timer base for a specific clock
- * @cpu_base:		per cpu clock base
- * @index:		clock type index for per_cpu support when moving a
- *			timer to a base on another cpu.
- * @clockid:		clock id for per_cpu support
- * @seq:		seqcount around __run_hrtimer
- * @running:		pointer to the currently running hrtimer
- * @active:		red black tree root node for the active timers
- * @get_time:		function to retrieve the current time of the clock
- * @offset:		offset of this clock to the monotonic base
- */
-struct hrtimer_clock_base {
-	struct hrtimer_cpu_base	*cpu_base;
-	unsigned int		index;
-	clockid_t		clockid;
-	seqcount_raw_spinlock_t	seq;
-	struct hrtimer		*running;
-	struct timerqueue_head	active;
-	ktime_t			(*get_time)(void);
-	ktime_t			offset;
-} __hrtimer_clock_base_align;
-
-enum  hrtimer_base_type {
-	HRTIMER_BASE_MONOTONIC,
-	HRTIMER_BASE_REALTIME,
-	HRTIMER_BASE_BOOTTIME,
-	HRTIMER_BASE_TAI,
-	HRTIMER_BASE_MONOTONIC_SOFT,
-	HRTIMER_BASE_REALTIME_SOFT,
-	HRTIMER_BASE_BOOTTIME_SOFT,
-	HRTIMER_BASE_TAI_SOFT,
-	HRTIMER_MAX_CLOCK_BASES,
-};
-
-/**
- * struct hrtimer_cpu_base - the per cpu clock bases
- * @lock:		lock protecting the base and associated clock bases
- *			and timers
- * @cpu:		cpu number
- * @active_bases:	Bitfield to mark bases with active timers
- * @clock_was_set_seq:	Sequence counter of clock was set events
- * @hres_active:	State of high resolution mode
- * @in_hrtirq:		hrtimer_interrupt() is currently executing
- * @hang_detected:	The last hrtimer interrupt detected a hang
- * @softirq_activated:	displays, if the softirq is raised - update of softirq
- *			related settings is not required then.
- * @nr_events:		Total number of hrtimer interrupt events
- * @nr_retries:		Total number of hrtimer interrupt retries
- * @nr_hangs:		Total number of hrtimer interrupt hangs
- * @max_hang_time:	Maximum time spent in hrtimer_interrupt
- * @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are
- *			 expired
- * @timer_waiters:	A hrtimer_cancel() invocation waits for the timer
- *			callback to finish.
- * @expires_next:	absolute time of the next event, is required for remote
- *			hrtimer enqueue; it is the total first expiry time (hard
- *			and soft hrtimer are taken into account)
- * @next_timer:		Pointer to the first expiring timer
- * @softirq_expires_next: Time to check, if soft queues needs also to be expired
- * @softirq_next_timer: Pointer to the first expiring softirq based timer
- * @clock_base:		array of clock bases for this cpu
- *
- * Note: next_timer is just an optimization for __remove_hrtimer().
- *	 Do not dereference the pointer because it is not reliable on
- *	 cross cpu removals.
- */
-struct hrtimer_cpu_base {
-	raw_spinlock_t			lock;
-	unsigned int			cpu;
-	unsigned int			active_bases;
-	unsigned int			clock_was_set_seq;
-	unsigned int			hres_active		: 1,
-					in_hrtirq		: 1,
-					hang_detected		: 1,
-					softirq_activated       : 1;
-#ifdef CONFIG_HIGH_RES_TIMERS
-	unsigned int			nr_events;
-	unsigned short			nr_retries;
-	unsigned short			nr_hangs;
-	unsigned int			max_hang_time;
-#endif
-#ifdef CONFIG_PREEMPT_RT
-	spinlock_t			softirq_expiry_lock;
-	atomic_t			timer_waiters;
-#endif
-	ktime_t				expires_next;
-	struct hrtimer			*next_timer;
-	ktime_t				softirq_expires_next;
-	struct hrtimer			*softirq_next_timer;
-	struct hrtimer_clock_base	clock_base[HRTIMER_MAX_CLOCK_BASES];
-} ____cacheline_aligned;
-
- static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time)
- {
- 	timer->node.expires = time;
-@@ -297,14 +154,11 @@ static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
- 	return ktime_to_ns(timer->node.expires);
- }
- 
-static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
-{
-	return ktime_sub(timer->node.expires, timer->base->get_time());
-}
-+ktime_t hrtimer_cb_get_time(const struct hrtimer *timer);
- 
-static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer)
-+static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
- {
-	return timer->base->get_time();
-+	return ktime_sub(timer->node.expires, hrtimer_cb_get_time(timer));
- }
- 
- static inline int hrtimer_is_hres_active(struct hrtimer *timer)
-@@ -343,8 +197,7 @@ __hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now)
- static inline ktime_t
- hrtimer_expires_remaining_adjusted(const struct hrtimer *timer)
- {
-	return __hrtimer_expires_remaining_adjusted(timer,
-						    timer->base->get_time());
-+	return __hrtimer_expires_remaining_adjusted(timer, hrtimer_cb_get_time(timer));
- }
- 
- #ifdef CONFIG_TIMERFD
-@@ -366,37 +219,25 @@ static inline void hrtimer_cancel_wait_running(struct hrtimer *timer)
- }
- #endif
- 
-+static inline enum hrtimer_restart hrtimer_dummy_timeout(struct hrtimer *unused)
-+{
-+	return HRTIMER_NORESTART;
-+}
-+
- /* Exported timer functions: */
- 
- /* Initialize timers: */
-extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock,
-			 enum hrtimer_mode mode);
-extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
-				 enum hrtimer_mode mode);
-+extern void hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*function)(struct hrtimer *),
-+			  clockid_t clock_id, enum hrtimer_mode mode);
-+extern void hrtimer_setup_on_stack(struct hrtimer *timer,
-+				   enum hrtimer_restart (*function)(struct hrtimer *),
-+				   clockid_t clock_id, enum hrtimer_mode mode);
-+extern void hrtimer_setup_sleeper_on_stack(struct hrtimer_sleeper *sl, clockid_t clock_id,
-+					   enum hrtimer_mode mode);
- 
- #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
-extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock,
-				  enum hrtimer_mode mode);
-extern void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
-					  clockid_t clock_id,
-					  enum hrtimer_mode mode);
-
- extern void destroy_hrtimer_on_stack(struct hrtimer *timer);
- #else
-static inline void hrtimer_init_on_stack(struct hrtimer *timer,
-					 clockid_t which_clock,
-					 enum hrtimer_mode mode)
-{
-	hrtimer_init(timer, which_clock, mode);
-}
-
-static inline void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
-						 clockid_t clock_id,
-						 enum hrtimer_mode mode)
-{
-	hrtimer_init_sleeper(sl, clock_id, mode);
-}
-
- static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
- #endif
- 
-@@ -480,30 +321,45 @@ static inline int hrtimer_callback_running(struct hrtimer *timer)
- 	return timer->base->running == timer;
- }
- 
-+/**
-+ * hrtimer_update_function - Update the timer's callback function
-+ * @timer:	Timer to update
-+ * @function:	New callback function
-+ *
-+ * Only safe to call if the timer is not enqueued. Can be called in the callback function if the
-+ * timer is not enqueued at the same time (see the comments above HRTIMER_STATE_ENQUEUED).
-+ */
-+static inline void hrtimer_update_function(struct hrtimer *timer,
-+					   enum hrtimer_restart (*function)(struct hrtimer *))
-+{
-+#ifdef CONFIG_PROVE_LOCKING
-+	guard(raw_spinlock_irqsave)(&timer->base->cpu_base->lock);
-+
-+	if (WARN_ON_ONCE(hrtimer_is_queued(timer)))
-+		return;
-+
-+	if (WARN_ON_ONCE(!function))
-+		return;
-+#endif
-+	ACCESS_PRIVATE(timer, function) = function;
-+}
-+
- /* Forward a hrtimer so it expires after now: */
- extern u64
- hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);
- 
- /**
- * hrtimer_forward_now - forward the timer expiry so it expires after now
-+ * hrtimer_forward_now() - forward the timer expiry so it expires after now
-  * @timer:	hrtimer to forward
-  * @interval:	the interval to forward
-  *
- * Forward the timer expiry so it will expire after the current time
- * of the hrtimer clock base. Returns the number of overruns.
- *
- * Can be safely called from the callback function of @timer. If
- * called from other contexts @timer must neither be enqueued nor
- * running the callback and the caller needs to take care of
- * serialization.
- *
- * Note: This only updates the timer expiry value and does not requeue
- * the timer.
-+ * It is a variant of hrtimer_forward(). The timer will expire after the current
-+ * time of the hrtimer clock base. See hrtimer_forward() for details.
-  */
- static inline u64 hrtimer_forward_now(struct hrtimer *timer,
- 				      ktime_t interval)
- {
-	return hrtimer_forward(timer, timer->base->get_time(), interval);
-+	return hrtimer_forward(timer, hrtimer_cb_get_time(timer), interval);
- }
- 
- /* Precise sleep: */
-@@ -530,10 +386,11 @@ extern void __init hrtimers_init(void);
- extern void sysrq_timer_list_show(void);
- 
- int hrtimers_prepare_cpu(unsigned int cpu);
-+int hrtimers_cpu_starting(unsigned int cpu);
- #ifdef CONFIG_HOTPLUG_CPU
-int hrtimers_dead_cpu(unsigned int cpu);
-+int hrtimers_cpu_dying(unsigned int cpu);
- #else
-#define hrtimers_dead_cpu	NULL
-+#define hrtimers_cpu_dying	NULL
- #endif
- 
- #endif
-diff --git a/include/linux/hrtimer_defs.h b/include/linux/hrtimer_defs.h
-index 2d3e3c5fb946..aa49ffa130e5 100644
--- a/include/linux/hrtimer_defs.h
-+++ b/include/linux/hrtimer_defs.h
-@@ -3,6 +3,8 @@
- #define _LINUX_HRTIMER_DEFS_H
- 
- #include <linux/ktime.h>
-+#include <linux/timerqueue.h>
-+#include <linux/seqlock.h>
- 
- #ifdef CONFIG_HIGH_RES_TIMERS
- 
-@@ -24,4 +26,105 @@
- 
- #endif
- 
-+#ifdef CONFIG_64BIT
-+# define __hrtimer_clock_base_align	____cacheline_aligned
-+#else
-+# define __hrtimer_clock_base_align
-+#endif
-+
-+/**
-+ * struct hrtimer_clock_base - the timer base for a specific clock
-+ * @cpu_base:		per cpu clock base
-+ * @index:		clock type index for per_cpu support when moving a
-+ *			timer to a base on another cpu.
-+ * @clockid:		clock id for per_cpu support
-+ * @seq:		seqcount around __run_hrtimer
-+ * @running:		pointer to the currently running hrtimer
-+ * @active:		red black tree root node for the active timers
-+ * @offset:		offset of this clock to the monotonic base
-+ */
-+struct hrtimer_clock_base {
-+	struct hrtimer_cpu_base	*cpu_base;
-+	unsigned int		index;
-+	clockid_t		clockid;
-+	seqcount_raw_spinlock_t	seq;
-+	struct hrtimer		*running;
-+	struct timerqueue_head	active;
-+	ktime_t			offset;
-+} __hrtimer_clock_base_align;
-+
-+enum  hrtimer_base_type {
-+	HRTIMER_BASE_MONOTONIC,
-+	HRTIMER_BASE_REALTIME,
-+	HRTIMER_BASE_BOOTTIME,
-+	HRTIMER_BASE_TAI,
-+	HRTIMER_BASE_MONOTONIC_SOFT,
-+	HRTIMER_BASE_REALTIME_SOFT,
-+	HRTIMER_BASE_BOOTTIME_SOFT,
-+	HRTIMER_BASE_TAI_SOFT,
-+	HRTIMER_MAX_CLOCK_BASES,
-+};
-+
-+/**
-+ * struct hrtimer_cpu_base - the per cpu clock bases
-+ * @lock:		lock protecting the base and associated clock bases
-+ *			and timers
-+ * @cpu:		cpu number
-+ * @active_bases:	Bitfield to mark bases with active timers
-+ * @clock_was_set_seq:	Sequence counter of clock was set events
-+ * @hres_active:	State of high resolution mode
-+ * @in_hrtirq:		hrtimer_interrupt() is currently executing
-+ * @hang_detected:	The last hrtimer interrupt detected a hang
-+ * @softirq_activated:	displays, if the softirq is raised - update of softirq
-+ *			related settings is not required then.
-+ * @nr_events:		Total number of hrtimer interrupt events
-+ * @nr_retries:		Total number of hrtimer interrupt retries
-+ * @nr_hangs:		Total number of hrtimer interrupt hangs
-+ * @max_hang_time:	Maximum time spent in hrtimer_interrupt
-+ * @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are
-+ *			 expired
-+ * @online:		CPU is online from an hrtimers point of view
-+ * @timer_waiters:	A hrtimer_cancel() invocation waits for the timer
-+ *			callback to finish.
-+ * @expires_next:	absolute time of the next event, is required for remote
-+ *			hrtimer enqueue; it is the total first expiry time (hard
-+ *			and soft hrtimer are taken into account)
-+ * @next_timer:		Pointer to the first expiring timer
-+ * @softirq_expires_next: Time to check, if soft queues needs also to be expired
-+ * @softirq_next_timer: Pointer to the first expiring softirq based timer
-+ * @clock_base:		array of clock bases for this cpu
-+ *
-+ * Note: next_timer is just an optimization for __remove_hrtimer().
-+ *	 Do not dereference the pointer because it is not reliable on
-+ *	 cross cpu removals.
-+ */
-+struct hrtimer_cpu_base {
-+	raw_spinlock_t			lock;
-+	unsigned int			cpu;
-+	unsigned int			active_bases;
-+	unsigned int			clock_was_set_seq;
-+	unsigned int			hres_active		: 1,
-+					in_hrtirq		: 1,
-+					hang_detected		: 1,
-+					softirq_activated       : 1,
-+					online			: 1;
-+#ifdef CONFIG_HIGH_RES_TIMERS
-+	unsigned int			nr_events;
-+	unsigned short			nr_retries;
-+	unsigned short			nr_hangs;
-+	unsigned int			max_hang_time;
-+#endif
-+#ifdef CONFIG_PREEMPT_RT
-+	spinlock_t			softirq_expiry_lock;
-+	atomic_t			timer_waiters;
-+#endif
-+	ktime_t				expires_next;
-+	struct hrtimer			*next_timer;
-+	ktime_t				softirq_expires_next;
-+	struct hrtimer			*softirq_next_timer;
-+	struct hrtimer_clock_base	clock_base[HRTIMER_MAX_CLOCK_BASES];
-+	call_single_data_t		csd;
-+} ____cacheline_aligned;
-+
-+
- #endif
-diff --git a/include/linux/hrtimer_types.h b/include/linux/hrtimer_types.h
-new file mode 100644
-index 000000000000..8fbbb6bdf7a1
--- /dev/null
-+++ b/include/linux/hrtimer_types.h
-@@ -0,0 +1,50 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+#ifndef _LINUX_HRTIMER_TYPES_H
-+#define _LINUX_HRTIMER_TYPES_H
-+
-+#include <linux/types.h>
-+#include <linux/timerqueue_types.h>
-+
-+struct hrtimer_clock_base;
-+
-+/*
-+ * Return values for the callback function
-+ */
-+enum hrtimer_restart {
-+	HRTIMER_NORESTART,	/* Timer is not restarted */
-+	HRTIMER_RESTART,	/* Timer must be restarted */
-+};
-+
-+/**
-+ * struct hrtimer - the basic hrtimer structure
-+ * @node:	timerqueue node, which also manages node.expires,
-+ *		the absolute expiry time in the hrtimers internal
-+ *		representation. The time is related to the clock on
-+ *		which the timer is based. Is setup by adding
-+ *		slack to the _softexpires value. For non range timers
-+ *		identical to _softexpires.
-+ * @_softexpires: the absolute earliest expiry time of the hrtimer.
-+ *		The time which was given as expiry time when the timer
-+ *		was armed.
-+ * @function:	timer expiry callback function
-+ * @base:	pointer to the timer base (per cpu and per clock)
-+ * @state:	state information (See bit values above)
-+ * @is_rel:	Set if the timer was armed relative
-+ * @is_soft:	Set if hrtimer will be expired in soft interrupt context.
-+ * @is_hard:	Set if hrtimer will be expired in hard interrupt context
-+ *		even on RT.
-+ *
-+ * The hrtimer structure must be initialized by hrtimer_setup()
-+ */
-+struct hrtimer {
-+	struct timerqueue_node		node;
-+	ktime_t				_softexpires;
-+	enum hrtimer_restart		(*__private function)(struct hrtimer *);
-+	struct hrtimer_clock_base	*base;
-+	u8				state;
-+	u8				is_rel;
-+	u8				is_soft;
-+	u8				is_hard;
-+};
-+
-+#endif /* _LINUX_HRTIMER_TYPES_H */
-diff --git a/include/linux/hsi/ssi_protocol.h b/include/linux/hsi/ssi_protocol.h
-index 2d6f3cfa7dea..972434daa000 100644
--- a/include/linux/hsi/ssi_protocol.h
-+++ b/include/linux/hsi/ssi_protocol.h
-@@ -24,6 +24,7 @@ int ssip_slave_stop_tx(struct hsi_client *master);
- void ssip_reset_event(struct hsi_client *master);
- 
- int ssip_slave_running(struct hsi_client *master);
-+void ssi_waketest(struct hsi_client *cl, unsigned int enable);
- 
- #endif /* __LINUX_SSIP_SLAVE_H__ */
- 
-diff --git a/include/linux/htcpld.h b/include/linux/htcpld.h
-deleted file mode 100644
-index 842fce69ac06..000000000000
--- a/include/linux/htcpld.h
-+++ /dev/null
-@@ -1,25 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_HTCPLD_H
-#define __LINUX_HTCPLD_H
-
-struct htcpld_chip_platform_data {
-	unsigned int addr;
-	unsigned int reset;
-	unsigned int num_gpios;
-	unsigned int gpio_out_base;
-	unsigned int gpio_in_base;
-	unsigned int irq_base;
-	unsigned int num_irqs;
-};
-
-struct htcpld_core_platform_data {
-	unsigned int                      int_reset_gpio_hi;
-	unsigned int                      int_reset_gpio_lo;
-	unsigned int                      i2c_adapter_id;
-
-	struct htcpld_chip_platform_data  *chip;
-	unsigned int                      num_chip;
-};
-
-#endif /* __LINUX_HTCPLD_H */
-
-diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
-index 768e5261fdae..ae7f21aad0ac 100644
--- a/include/linux/huge_mm.h
-+++ b/include/linux/huge_mm.h
-@@ -2,16 +2,16 @@
- #ifndef _LINUX_HUGE_MM_H
- #define _LINUX_HUGE_MM_H
- 
-#include <linux/sched/coredump.h>
- #include <linux/mm_types.h>
- 
- #include <linux/fs.h> /* only for vma_is_dax() */
-+#include <linux/kobject.h>
- 
- vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf);
- int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
- 		  pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
- 		  struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);
-void huge_pmd_set_accessed(struct vm_fault *vmf);
-+bool huge_pmd_set_accessed(struct vm_fault *vmf);
- int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
- 		  pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
- 		  struct vm_area_struct *vma);
-@@ -25,9 +25,6 @@ static inline void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
- #endif
- 
- vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf);
-struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
-				   unsigned long addr, pmd_t *pmd,
-				   unsigned int flags);
- bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
- 			   pmd_t *pmd, unsigned long addr, unsigned long next);
- int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd,
-@@ -39,47 +36,18 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
- int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
- 		    pmd_t *pmd, unsigned long addr, pgprot_t newprot,
- 		    unsigned long cp_flags);
-vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn,
-				   pgprot_t pgprot, bool write);
- 
-/**
- * vmf_insert_pfn_pmd - insert a pmd size pfn
- * @vmf: Structure describing the fault
- * @pfn: pfn to insert
- * @pgprot: page protection to use
- * @write: whether it's a write fault
- *
- * Insert a pmd size pfn. See vmf_insert_pfn() for additional info.
- *
- * Return: vm_fault_t value.
- */
-static inline vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn,
-					    bool write)
-{
-	return vmf_insert_pfn_pmd_prot(vmf, pfn, vmf->vma->vm_page_prot, write);
-}
-vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn,
-				   pgprot_t pgprot, bool write);
-
-/**
- * vmf_insert_pfn_pud - insert a pud size pfn
- * @vmf: Structure describing the fault
- * @pfn: pfn to insert
- * @pgprot: page protection to use
- * @write: whether it's a write fault
- *
- * Insert a pud size pfn. See vmf_insert_pfn() for additional info.
- *
- * Return: vm_fault_t value.
- */
-static inline vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn,
-					    bool write)
-{
-	return vmf_insert_pfn_pud_prot(vmf, pfn, vmf->vma->vm_page_prot, write);
-}
-+vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn,
-+			      bool write);
-+vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, unsigned long pfn,
-+			      bool write);
-+vm_fault_t vmf_insert_folio_pmd(struct vm_fault *vmf, struct folio *folio,
-+				bool write);
-+vm_fault_t vmf_insert_folio_pud(struct vm_fault *vmf, struct folio *folio,
-+				bool write);
- 
- enum transparent_hugepage_flag {
-	TRANSPARENT_HUGEPAGE_NEVER_DAX,
-+	TRANSPARENT_HUGEPAGE_UNSUPPORTED,
- 	TRANSPARENT_HUGEPAGE_FLAG,
- 	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
- 	TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
-@@ -101,147 +69,440 @@ ssize_t single_hugepage_flag_show(struct kobject *kobj,
- 				  struct kobj_attribute *attr, char *buf,
- 				  enum transparent_hugepage_flag flag);
- extern struct kobj_attribute shmem_enabled_attr;
-+extern struct kobj_attribute thpsize_shmem_enabled_attr;
- 
-#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
-#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
-+/*
-+ * Mask of all large folio orders supported for anonymous THP; all orders up to
-+ * and including PMD_ORDER, except order-0 (which is not "huge") and order-1
-+ * (which is a limitation of the THP implementation).
-+ */
-+#define THP_ORDERS_ALL_ANON	((BIT(PMD_ORDER + 1) - 1) & ~(BIT(0) | BIT(1)))
- 
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-+/*
-+ * Mask of all large folio orders supported for file THP. Folios in a DAX
-+ * file is never split and the MAX_PAGECACHE_ORDER limit does not apply to
-+ * it.  Same to PFNMAPs where there's neither page* nor pagecache.
-+ */
-+#define THP_ORDERS_ALL_SPECIAL		\
-+	(BIT(PMD_ORDER) | BIT(PUD_ORDER))
-+#define THP_ORDERS_ALL_FILE_DEFAULT	\
-+	((BIT(MAX_PAGECACHE_ORDER + 1) - 1) & ~BIT(0))
-+
-+/*
-+ * Mask of all large folio orders supported for THP.
-+ */
-+#define THP_ORDERS_ALL	\
-+	(THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL | THP_ORDERS_ALL_FILE_DEFAULT)
-+
-+enum tva_type {
-+	TVA_SMAPS,		/* Exposing "THPeligible:" in smaps. */
-+	TVA_PAGEFAULT,		/* Serving a page fault. */
-+	TVA_KHUGEPAGED,		/* Khugepaged collapse. */
-+	TVA_FORCED_COLLAPSE,	/* Forced collapse (e.g. MADV_COLLAPSE). */
-+};
-+
-+#define thp_vma_allowable_order(vma, vm_flags, type, order) \
-+	(!!thp_vma_allowable_orders(vma, vm_flags, type, BIT(order)))
-+
-+#define split_folio(f) split_folio_to_list(f, NULL)
-+
-+#ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES
- #define HPAGE_PMD_SHIFT PMD_SHIFT
-#define HPAGE_PMD_SIZE	((1UL) << HPAGE_PMD_SHIFT)
-+#define HPAGE_PUD_SHIFT PUD_SHIFT
-+#else
-+#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
-+#define HPAGE_PUD_SHIFT ({ BUILD_BUG(); 0; })
-+#endif
-+
-+#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
-+#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
- #define HPAGE_PMD_MASK	(~(HPAGE_PMD_SIZE - 1))
-+#define HPAGE_PMD_SIZE	((1UL) << HPAGE_PMD_SHIFT)
- 
-#define HPAGE_PUD_SHIFT PUD_SHIFT
-#define HPAGE_PUD_SIZE	((1UL) << HPAGE_PUD_SHIFT)
-+#define HPAGE_PUD_ORDER (HPAGE_PUD_SHIFT-PAGE_SHIFT)
-+#define HPAGE_PUD_NR (1<<HPAGE_PUD_ORDER)
- #define HPAGE_PUD_MASK	(~(HPAGE_PUD_SIZE - 1))
-+#define HPAGE_PUD_SIZE	((1UL) << HPAGE_PUD_SHIFT)
-+
-+enum mthp_stat_item {
-+	MTHP_STAT_ANON_FAULT_ALLOC,
-+	MTHP_STAT_ANON_FAULT_FALLBACK,
-+	MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE,
-+	MTHP_STAT_ZSWPOUT,
-+	MTHP_STAT_SWPIN,
-+	MTHP_STAT_SWPIN_FALLBACK,
-+	MTHP_STAT_SWPIN_FALLBACK_CHARGE,
-+	MTHP_STAT_SWPOUT,
-+	MTHP_STAT_SWPOUT_FALLBACK,
-+	MTHP_STAT_SHMEM_ALLOC,
-+	MTHP_STAT_SHMEM_FALLBACK,
-+	MTHP_STAT_SHMEM_FALLBACK_CHARGE,
-+	MTHP_STAT_SPLIT,
-+	MTHP_STAT_SPLIT_FAILED,
-+	MTHP_STAT_SPLIT_DEFERRED,
-+	MTHP_STAT_NR_ANON,
-+	MTHP_STAT_NR_ANON_PARTIALLY_MAPPED,
-+	__MTHP_STAT_COUNT
-+};
-+
-+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
-+struct mthp_stat {
-+	unsigned long stats[ilog2(MAX_PTRS_PER_PTE) + 1][__MTHP_STAT_COUNT];
-+};
-+
-+DECLARE_PER_CPU(struct mthp_stat, mthp_stats);
-+
-+static inline void mod_mthp_stat(int order, enum mthp_stat_item item, int delta)
-+{
-+	if (order <= 0 || order > PMD_ORDER)
-+		return;
-+
-+	this_cpu_add(mthp_stats.stats[order][item], delta);
-+}
-+
-+static inline void count_mthp_stat(int order, enum mthp_stat_item item)
-+{
-+	mod_mthp_stat(order, item, 1);
-+}
-+
-+#else
-+static inline void mod_mthp_stat(int order, enum mthp_stat_item item, int delta)
-+{
-+}
-+
-+static inline void count_mthp_stat(int order, enum mthp_stat_item item)
-+{
-+}
-+#endif
-+
-+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
- 
- extern unsigned long transparent_hugepage_flags;
-+extern unsigned long huge_anon_orders_always;
-+extern unsigned long huge_anon_orders_madvise;
-+extern unsigned long huge_anon_orders_inherit;
-+
-+static inline bool hugepage_global_enabled(void)
-+{
-+	return transparent_hugepage_flags &
-+			((1<<TRANSPARENT_HUGEPAGE_FLAG) |
-+			(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG));
-+}
- 
-#define hugepage_flags_enabled()					       \
-	(transparent_hugepage_flags &				       \
-	 ((1<<TRANSPARENT_HUGEPAGE_FLAG) |		       \
-	  (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)))
-#define hugepage_flags_always()				\
-	(transparent_hugepage_flags &			\
-	 (1<<TRANSPARENT_HUGEPAGE_FLAG))
-+static inline bool hugepage_global_always(void)
-+{
-+	return transparent_hugepage_flags &
-+			(1<<TRANSPARENT_HUGEPAGE_FLAG);
-+}
-+
-+static inline int highest_order(unsigned long orders)
-+{
-+	return fls_long(orders) - 1;
-+}
-+
-+static inline int next_order(unsigned long *orders, int prev)
-+{
-+	*orders &= ~BIT(prev);
-+	return highest_order(*orders);
-+}
- 
- /*
-  * Do the below checks:
-  *   - For file vma, check if the linear page offset of vma is
- *     HPAGE_PMD_NR aligned within the file.  The hugepage is
- *     guaranteed to be hugepage-aligned within the file, but we must
- *     check that the PMD-aligned addresses in the VMA map to
- *     PMD-aligned offsets within the file, else the hugepage will
- *     not be PMD-mappable.
- *   - For all vmas, check if the haddr is in an aligned HPAGE_PMD_SIZE
-+ *     order-aligned within the file.  The hugepage is
-+ *     guaranteed to be order-aligned within the file, but we must
-+ *     check that the order-aligned addresses in the VMA map to
-+ *     order-aligned offsets within the file, else the hugepage will
-+ *     not be mappable.
-+ *   - For all vmas, check if the haddr is in an aligned hugepage
-  *     area.
-  */
-static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
-		unsigned long addr)
-+static inline bool thp_vma_suitable_order(struct vm_area_struct *vma,
-+		unsigned long addr, int order)
- {
-+	unsigned long hpage_size = PAGE_SIZE << order;
- 	unsigned long haddr;
- 
- 	/* Don't have to check pgoff for anonymous vma */
- 	if (!vma_is_anonymous(vma)) {
- 		if (!IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,
-				HPAGE_PMD_NR))
-+				hpage_size >> PAGE_SHIFT))
- 			return false;
- 	}
- 
-	haddr = addr & HPAGE_PMD_MASK;
-+	haddr = ALIGN_DOWN(addr, hpage_size);
- 
-	if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
-+	if (haddr < vma->vm_start || haddr + hpage_size > vma->vm_end)
- 		return false;
- 	return true;
- }
- 
-static inline bool file_thp_enabled(struct vm_area_struct *vma)
-+/*
-+ * Filter the bitfield of input orders to the ones suitable for use in the vma.
-+ * See thp_vma_suitable_order().
-+ * All orders that pass the checks are returned as a bitfield.
-+ */
-+static inline unsigned long thp_vma_suitable_orders(struct vm_area_struct *vma,
-+		unsigned long addr, unsigned long orders)
- {
-	struct inode *inode;
-+	int order;
- 
-	if (!vma->vm_file)
-		return false;
-+	/*
-+	 * Iterate over orders, highest to lowest, removing orders that don't
-+	 * meet alignment requirements from the set. Exit loop at first order
-+	 * that meets requirements, since all lower orders must also meet
-+	 * requirements.
-+	 */
-+
-+	order = highest_order(orders);
- 
-	inode = vma->vm_file->f_inode;
-+	while (orders) {
-+		if (thp_vma_suitable_order(vma, addr, order))
-+			break;
-+		order = next_order(&orders, order);
-+	}
- 
-	return (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) &&
-	       (vma->vm_flags & VM_EXEC) &&
-	       !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
-+	return orders;
- }
- 
-bool hugepage_vma_check(struct vm_area_struct *vma,
-			unsigned long vm_flags,
-			bool smaps, bool in_pf);
-+unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
-+					 vm_flags_t vm_flags,
-+					 enum tva_type type,
-+					 unsigned long orders);
-+
-+/**
-+ * thp_vma_allowable_orders - determine hugepage orders that are allowed for vma
-+ * @vma:  the vm area to check
-+ * @vm_flags: use these vm_flags instead of vma->vm_flags
-+ * @type: TVA type
-+ * @orders: bitfield of all orders to consider
-+ *
-+ * Calculates the intersection of the requested hugepage orders and the allowed
-+ * hugepage orders for the provided vma. Permitted orders are encoded as a set
-+ * bit at the corresponding bit position (bit-2 corresponds to order-2, bit-3
-+ * corresponds to order-3, etc). Order-0 is never considered a hugepage order.
-+ *
-+ * Return: bitfield of orders allowed for hugepage in the vma. 0 if no hugepage
-+ * orders are allowed.
-+ */
-+static inline
-+unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma,
-+				       vm_flags_t vm_flags,
-+				       enum tva_type type,
-+				       unsigned long orders)
-+{
-+	/*
-+	 * Optimization to check if required orders are enabled early. Only
-+	 * forced collapse ignores sysfs configs.
-+	 */
-+	if (type != TVA_FORCED_COLLAPSE && vma_is_anonymous(vma)) {
-+		unsigned long mask = READ_ONCE(huge_anon_orders_always);
-+
-+		if (vm_flags & VM_HUGEPAGE)
-+			mask |= READ_ONCE(huge_anon_orders_madvise);
-+		if (hugepage_global_always() ||
-+		    ((vm_flags & VM_HUGEPAGE) && hugepage_global_enabled()))
-+			mask |= READ_ONCE(huge_anon_orders_inherit);
-+
-+		orders &= mask;
-+		if (!orders)
-+			return 0;
-+	}
-+
-+	return __thp_vma_allowable_orders(vma, vm_flags, type, orders);
-+}
-+
-+struct thpsize {
-+	struct kobject kobj;
-+	struct list_head node;
-+	int order;
-+};
-+
-+#define to_thpsize(kobj) container_of(kobj, struct thpsize, kobj)
- 
- #define transparent_hugepage_use_zero_page()				\
- 	(transparent_hugepage_flags &					\
- 	 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
- 
-+/*
-+ * Check whether THPs are explicitly disabled for this VMA, for example,
-+ * through madvise or prctl.
-+ */
-+static inline bool vma_thp_disabled(struct vm_area_struct *vma,
-+		vm_flags_t vm_flags, bool forced_collapse)
-+{
-+	/* Are THPs disabled for this VMA? */
-+	if (vm_flags & VM_NOHUGEPAGE)
-+		return true;
-+	/* Are THPs disabled for all VMAs in the whole process? */
-+	if (mm_flags_test(MMF_DISABLE_THP_COMPLETELY, vma->vm_mm))
-+		return true;
-+	/*
-+	 * Are THPs disabled only for VMAs where we didn't get an explicit
-+	 * advise to use them?
-+	 */
-+	if (vm_flags & VM_HUGEPAGE)
-+		return false;
-+	/*
-+	 * Forcing a collapse (e.g., madv_collapse), is a clear advice to
-+	 * use THPs.
-+	 */
-+	if (forced_collapse)
-+		return false;
-+	return mm_flags_test(MMF_DISABLE_THP_EXCEPT_ADVISED, vma->vm_mm);
-+}
-+
-+static inline bool thp_disabled_by_hw(void)
-+{
-+	/* If the hardware/firmware marked hugepage support disabled. */
-+	return transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED);
-+}
-+
- unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
- 		unsigned long len, unsigned long pgoff, unsigned long flags);
-+unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
-+		unsigned long len, unsigned long pgoff, unsigned long flags,
-+		vm_flags_t vm_flags);
-+
-+enum split_type {
-+	SPLIT_TYPE_UNIFORM,
-+	SPLIT_TYPE_NON_UNIFORM,
-+};
- 
-void prep_transhuge_page(struct page *page);
-void free_transhuge_page(struct page *page);
-+bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins);
-+int __split_huge_page_to_list_to_order(struct page *page, struct list_head *list,
-+		unsigned int new_order);
-+int folio_split_unmapped(struct folio *folio, unsigned int new_order);
-+int min_order_for_split(struct folio *folio);
-+int split_folio_to_list(struct folio *folio, struct list_head *list);
-+bool folio_split_supported(struct folio *folio, unsigned int new_order,
-+		enum split_type split_type, bool warns);
-+int folio_split(struct folio *folio, unsigned int new_order, struct page *page,
-+		struct list_head *list);
-+
-+static inline int split_huge_page_to_list_to_order(struct page *page, struct list_head *list,
-+		unsigned int new_order)
-+{
-+	return __split_huge_page_to_list_to_order(page, list, new_order);
-+}
-+static inline int split_huge_page_to_order(struct page *page, unsigned int new_order)
-+{
-+	return split_huge_page_to_list_to_order(page, NULL, new_order);
-+}
- 
-bool can_split_folio(struct folio *folio, int *pextra_pins);
-int split_huge_page_to_list(struct page *page, struct list_head *list);
-+/**
-+ * try_folio_split_to_order() - try to split a @folio at @page to @new_order
-+ * using non uniform split.
-+ * @folio: folio to be split
-+ * @page: split to @new_order at the given page
-+ * @new_order: the target split order
-+ *
-+ * Try to split a @folio at @page using non uniform split to @new_order, if
-+ * non uniform split is not supported, fall back to uniform split. After-split
-+ * folios are put back to LRU list. Use min_order_for_split() to get the lower
-+ * bound of @new_order.
-+ *
-+ * Return: 0 - split is successful, otherwise split failed.
-+ */
-+static inline int try_folio_split_to_order(struct folio *folio,
-+		struct page *page, unsigned int new_order)
-+{
-+	if (!folio_split_supported(folio, new_order, SPLIT_TYPE_NON_UNIFORM, /* warns= */ false))
-+		return split_huge_page_to_order(&folio->page, new_order);
-+	return folio_split(folio, new_order, page, NULL);
-+}
- static inline int split_huge_page(struct page *page)
- {
-	return split_huge_page_to_list(page, NULL);
-+	return split_huge_page_to_list_to_order(page, NULL, 0);
- }
-void deferred_split_huge_page(struct page *page);
-+void deferred_split_folio(struct folio *folio, bool partially_mapped);
-+#ifdef CONFIG_MEMCG
-+void reparent_deferred_split_queue(struct mem_cgroup *memcg);
-+#endif
- 
- void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
-		unsigned long address, bool freeze, struct folio *folio);
-+		unsigned long address, bool freeze);
-+
-+/**
-+ * pmd_is_huge() - Is this PMD either a huge PMD entry or a software leaf entry?
-+ * @pmd: The PMD to check.
-+ *
-+ * A huge PMD entry is a non-empty entry which is present and marked huge or a
-+ * software leaf entry. This check be performed without the appropriate locks
-+ * held, in which case the condition should be rechecked after they are
-+ * acquired.
-+ *
-+ * Returns: true if this PMD is huge, false otherwise.
-+ */
-+static inline bool pmd_is_huge(pmd_t pmd)
-+{
-+	if (pmd_present(pmd)) {
-+		return pmd_trans_huge(pmd);
-+	} else if (!pmd_none(pmd)) {
-+		/*
-+		 * Non-present PMDs must be valid huge non-present entries. We
-+		 * cannot assert that here due to header dependency issues.
-+		 */
-+		return true;
-+	}
-+
-+	return false;
-+}
- 
- #define split_huge_pmd(__vma, __pmd, __address)				\
- 	do {								\
- 		pmd_t *____pmd = (__pmd);				\
-		if (is_swap_pmd(*____pmd) || pmd_trans_huge(*____pmd)	\
-					|| pmd_devmap(*____pmd))	\
-+		if (pmd_is_huge(*____pmd))				\
- 			__split_huge_pmd(__vma, __pmd, __address,	\
-						false, NULL);		\
-+					 false);			\
- 	}  while (0)
- 
-
- void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
-		bool freeze, struct folio *folio);
-+		bool freeze);
- 
- void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
- 		unsigned long address);
- 
-+#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
-+int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
-+		    pud_t *pudp, unsigned long addr, pgprot_t newprot,
-+		    unsigned long cp_flags);
-+#else
-+static inline int
-+change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
-+		pud_t *pudp, unsigned long addr, pgprot_t newprot,
-+		unsigned long cp_flags) { return 0; }
-+#endif
-+
- #define split_huge_pud(__vma, __pud, __address)				\
- 	do {								\
- 		pud_t *____pud = (__pud);				\
-		if (pud_trans_huge(*____pud)				\
-					|| pud_devmap(*____pud))	\
-+		if (pud_trans_huge(*____pud))				\
- 			__split_huge_pud(__vma, __pud, __address);	\
- 	}  while (0)
- 
-int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags,
-+int hugepage_madvise(struct vm_area_struct *vma, vm_flags_t *vm_flags,
- 		     int advice);
-+int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
-+		     unsigned long end, bool *lock_dropped);
- void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start,
-			   unsigned long end, long adjust_next);
-+			   unsigned long end, struct vm_area_struct *next);
- spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma);
- spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma);
- 
-static inline int is_swap_pmd(pmd_t pmd)
-{
-	return !pmd_none(pmd) && !pmd_present(pmd);
-}
-
- /* mmap_lock must be held on entry */
- static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
- 		struct vm_area_struct *vma)
- {
-	if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd))
-+	if (pmd_is_huge(*pmd))
- 		return __pmd_trans_huge_lock(pmd, vma);
-	else
-		return NULL;
-+
-+	return NULL;
- }
- static inline spinlock_t *pud_trans_huge_lock(pud_t *pud,
- 		struct vm_area_struct *vma)
- {
-	if (pud_trans_huge(*pud) || pud_devmap(*pud))
-+	if (pud_trans_huge(*pud))
- 		return __pud_trans_huge_lock(pud, vma);
- 	else
- 		return NULL;
-@@ -250,130 +511,186 @@ static inline spinlock_t *pud_trans_huge_lock(pud_t *pud,
- /**
-  * folio_test_pmd_mappable - Can we map this folio with a PMD?
-  * @folio: The folio to test
-+ *
-+ * Return: true - @folio can be mapped, false - @folio cannot be mapped.
-  */
- static inline bool folio_test_pmd_mappable(struct folio *folio)
- {
- 	return folio_order(folio) >= HPAGE_PMD_ORDER;
- }
- 
-struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
-		pmd_t *pmd, int flags, struct dev_pagemap **pgmap);
-struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
-		pud_t *pud, int flags, struct dev_pagemap **pgmap);
-
- vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf);
- 
-extern struct page *huge_zero_page;
-+vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf);
-+
-+extern struct folio *huge_zero_folio;
- extern unsigned long huge_zero_pfn;
- 
-static inline bool is_huge_zero_page(struct page *page)
-+static inline bool is_huge_zero_folio(const struct folio *folio)
- {
-	return READ_ONCE(huge_zero_page) == page;
-+	VM_WARN_ON_ONCE(!folio);
-+
-+	return READ_ONCE(huge_zero_folio) == folio;
- }
- 
-static inline bool is_huge_zero_pmd(pmd_t pmd)
-+static inline bool is_huge_zero_pfn(unsigned long pfn)
- {
-	return pmd_present(pmd) && READ_ONCE(huge_zero_pfn) == pmd_pfn(pmd);
-+	return READ_ONCE(huge_zero_pfn) == (pfn & ~(HPAGE_PMD_NR - 1));
- }
- 
-static inline bool is_huge_zero_pud(pud_t pud)
-+static inline bool is_huge_zero_pmd(pmd_t pmd)
- {
-	return false;
-+	return pmd_present(pmd) && is_huge_zero_pfn(pmd_pfn(pmd));
- }
- 
-struct page *mm_get_huge_zero_page(struct mm_struct *mm);
-void mm_put_huge_zero_page(struct mm_struct *mm);
-+struct folio *mm_get_huge_zero_folio(struct mm_struct *mm);
-+void mm_put_huge_zero_folio(struct mm_struct *mm);
-+
-+static inline struct folio *get_persistent_huge_zero_folio(void)
-+{
-+	if (!IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO))
-+		return NULL;
-+
-+	if (unlikely(!huge_zero_folio))
-+		return NULL;
- 
-#define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot))
-+	return huge_zero_folio;
-+}
- 
- static inline bool thp_migration_supported(void)
- {
- 	return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION);
- }
- 
-static inline struct list_head *page_deferred_list(struct page *page)
-{
-	/*
-	 * See organization of tail pages of compound page in
-	 * "struct page" definition.
-	 */
-	return &page[2].deferred_list;
-}
-+void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address,
-+			   pmd_t *pmd, bool freeze);
-+bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr,
-+			   pmd_t *pmdp, struct folio *folio);
-+void map_anon_folio_pmd_nopf(struct folio *folio, pmd_t *pmd,
-+		struct vm_area_struct *vma, unsigned long haddr);
- 
- #else /* CONFIG_TRANSPARENT_HUGEPAGE */
-#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
-#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
-#define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })
-
-#define HPAGE_PUD_SHIFT ({ BUILD_BUG(); 0; })
-#define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
-#define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; })
- 
- static inline bool folio_test_pmd_mappable(struct folio *folio)
- {
- 	return false;
- }
- 
-static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
-		unsigned long addr)
-+static inline bool thp_vma_suitable_order(struct vm_area_struct *vma,
-+		unsigned long addr, int order)
- {
- 	return false;
- }
- 
-static inline bool hugepage_vma_check(struct vm_area_struct *vma,
-				       unsigned long vm_flags,
-				       bool smaps, bool in_pf)
-+static inline unsigned long thp_vma_suitable_orders(struct vm_area_struct *vma,
-+		unsigned long addr, unsigned long orders)
- {
-	return false;
-+	return 0;
- }
- 
-static inline void prep_transhuge_page(struct page *page) {}
-+static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma,
-+					vm_flags_t vm_flags,
-+					enum tva_type type,
-+					unsigned long orders)
-+{
-+	return 0;
-+}
- 
- #define transparent_hugepage_flags 0UL
- 
- #define thp_get_unmapped_area	NULL
- 
-+static inline unsigned long
-+thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
-+			      unsigned long len, unsigned long pgoff,
-+			      unsigned long flags, vm_flags_t vm_flags)
-+{
-+	return 0;
-+}
-+
- static inline bool
-can_split_folio(struct folio *folio, int *pextra_pins)
-+can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins)
- {
- 	return false;
- }
- static inline int
-split_huge_page_to_list(struct page *page, struct list_head *list)
-+split_huge_page_to_list_to_order(struct page *page, struct list_head *list,
-+		unsigned int new_order)
- {
-	return 0;
-+	VM_WARN_ON_ONCE_PAGE(1, page);
-+	return -EINVAL;
-+}
-+static inline int split_huge_page_to_order(struct page *page, unsigned int new_order)
-+{
-+	VM_WARN_ON_ONCE_PAGE(1, page);
-+	return -EINVAL;
- }
- static inline int split_huge_page(struct page *page)
- {
-	return 0;
-+	VM_WARN_ON_ONCE_PAGE(1, page);
-+	return -EINVAL;
-+}
-+
-+static inline int min_order_for_split(struct folio *folio)
-+{
-+	VM_WARN_ON_ONCE_FOLIO(1, folio);
-+	return -EINVAL;
-+}
-+
-+static inline int split_folio_to_list(struct folio *folio, struct list_head *list)
-+{
-+	VM_WARN_ON_ONCE_FOLIO(1, folio);
-+	return -EINVAL;
-+}
-+
-+static inline int try_folio_split_to_order(struct folio *folio,
-+		struct page *page, unsigned int new_order)
-+{
-+	VM_WARN_ON_ONCE_FOLIO(1, folio);
-+	return -EINVAL;
- }
-static inline void deferred_split_huge_page(struct page *page) {}
-+
-+static inline void deferred_split_folio(struct folio *folio, bool partially_mapped) {}
-+static inline void reparent_deferred_split_queue(struct mem_cgroup *memcg) {}
- #define split_huge_pmd(__vma, __pmd, __address)	\
- 	do { } while (0)
- 
- static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
-		unsigned long address, bool freeze, struct folio *folio) {}
-+		unsigned long address, bool freeze) {}
- static inline void split_huge_pmd_address(struct vm_area_struct *vma,
-		unsigned long address, bool freeze, struct folio *folio) {}
-+		unsigned long address, bool freeze) {}
-+static inline void split_huge_pmd_locked(struct vm_area_struct *vma,
-+					 unsigned long address, pmd_t *pmd,
-+					 bool freeze) {}
-+
-+static inline bool unmap_huge_pmd_locked(struct vm_area_struct *vma,
-+					 unsigned long addr, pmd_t *pmdp,
-+					 struct folio *folio)
-+{
-+	return false;
-+}
- 
- #define split_huge_pud(__vma, __pmd, __address)	\
- 	do { } while (0)
- 
- static inline int hugepage_madvise(struct vm_area_struct *vma,
-				   unsigned long *vm_flags, int advice)
-+				   vm_flags_t *vm_flags, int advice)
- {
-	BUG();
-	return 0;
-+	return -EINVAL;
-+}
-+
-+static inline int madvise_collapse(struct vm_area_struct *vma,
-+				   unsigned long start,
-+				   unsigned long end, bool *lock_dropped)
-+{
-+	return -EINVAL;
- }
-+
- static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
- 					 unsigned long start,
- 					 unsigned long end,
-					 long adjust_next)
-{
-}
-static inline int is_swap_pmd(pmd_t pmd)
-+					 struct vm_area_struct *next)
- {
-	return 0;
- }
- static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
- 		struct vm_area_struct *vma)
-@@ -391,60 +708,101 @@ static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
- 	return 0;
- }
- 
-static inline bool is_huge_zero_page(struct page *page)
-+static inline vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf)
-+{
-+	return 0;
-+}
-+
-+static inline bool is_huge_zero_folio(const struct folio *folio)
- {
- 	return false;
- }
- 
-static inline bool is_huge_zero_pmd(pmd_t pmd)
-+static inline bool is_huge_zero_pfn(unsigned long pfn)
- {
- 	return false;
- }
- 
-static inline bool is_huge_zero_pud(pud_t pud)
-+static inline bool is_huge_zero_pmd(pmd_t pmd)
- {
- 	return false;
- }
- 
-static inline void mm_put_huge_zero_page(struct mm_struct *mm)
-+static inline void mm_put_huge_zero_folio(struct mm_struct *mm)
- {
- 	return;
- }
- 
-static inline struct page *follow_devmap_pmd(struct vm_area_struct *vma,
-	unsigned long addr, pmd_t *pmd, int flags, struct dev_pagemap **pgmap)
-+static inline bool thp_migration_supported(void)
- {
-	return NULL;
-+	return false;
- }
- 
-static inline struct page *follow_devmap_pud(struct vm_area_struct *vma,
-	unsigned long addr, pud_t *pud, int flags, struct dev_pagemap **pgmap)
-+static inline int highest_order(unsigned long orders)
-+{
-+	return 0;
-+}
-+
-+static inline int next_order(unsigned long *orders, int prev)
-+{
-+	return 0;
-+}
-+
-+static inline void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
-+				    unsigned long address)
-+{
-+}
-+
-+static inline int change_huge_pud(struct mmu_gather *tlb,
-+				  struct vm_area_struct *vma, pud_t *pudp,
-+				  unsigned long addr, pgprot_t newprot,
-+				  unsigned long cp_flags)
-+{
-+	return 0;
-+}
-+
-+static inline struct folio *get_persistent_huge_zero_folio(void)
- {
- 	return NULL;
- }
- 
-static inline bool thp_migration_supported(void)
-+static inline bool pmd_is_huge(pmd_t pmd)
- {
- 	return false;
- }
- #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
- 
-static inline int split_folio_to_list(struct folio *folio,
-		struct list_head *list)
-+static inline int split_folio_to_list_to_order(struct folio *folio,
-+		struct list_head *list, int new_order)
- {
-	return split_huge_page_to_list(&folio->page, list);
-+	return split_huge_page_to_list_to_order(&folio->page, list, new_order);
- }
- 
-/*
- * archs that select ARCH_WANTS_THP_SWAP but don't support THP_SWP due to
- * limitations in the implementation like arm64 MTE can override this to
- * false
- */
-#ifndef arch_thp_swp_supported
-static inline bool arch_thp_swp_supported(void)
-+static inline int split_folio_to_order(struct folio *folio, int new_order)
- {
-	return true;
-+	return split_folio_to_list_to_order(folio, NULL, new_order);
- }
-#endif
- 
-+/**
-+ * largest_zero_folio - Get the largest zero size folio available
-+ *
-+ * This function shall be used when mm_get_huge_zero_folio() cannot be
-+ * used as there is no appropriate mm lifetime to tie the huge zero folio
-+ * from the caller.
-+ *
-+ * Deduce the size of the folio with folio_size instead of assuming the
-+ * folio size.
-+ *
-+ * Return: pointer to PMD sized zero folio if CONFIG_PERSISTENT_HUGE_ZERO_FOLIO
-+ * is enabled or a single page sized zero folio
-+ */
-+static inline struct folio *largest_zero_folio(void)
-+{
-+	struct folio *folio = get_persistent_huge_zero_folio();
-+
-+	if (folio)
-+		return folio;
-+
-+	return page_folio(ZERO_PAGE(0));
-+}
- #endif /* _LINUX_HUGE_MM_H */
-diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
-index 3ec981a0d8b3..019a1c5281e4 100644
--- a/include/linux/hugetlb.h
-+++ b/include/linux/hugetlb.h
-@@ -2,52 +2,39 @@
- #ifndef _LINUX_HUGETLB_H
- #define _LINUX_HUGETLB_H
- 
-+#include <linux/mm.h>
- #include <linux/mm_types.h>
- #include <linux/mmdebug.h>
- #include <linux/fs.h>
- #include <linux/hugetlb_inline.h>
- #include <linux/cgroup.h>
-+#include <linux/page_ref.h>
- #include <linux/list.h>
- #include <linux/kref.h>
- #include <linux/pgtable.h>
- #include <linux/gfp.h>
- #include <linux/userfaultfd_k.h>
-+#include <linux/nodemask.h>
- 
- struct ctl_table;
- struct user_struct;
- struct mmu_gather;
-+struct node;
- 
-#ifndef is_hugepd
-typedef struct { unsigned long pd; } hugepd_t;
-#define is_hugepd(hugepd) (0)
-#define __hugepd(x) ((hugepd_t) { (x) })
-#endif
-+void free_huge_folio(struct folio *folio);
- 
- #ifdef CONFIG_HUGETLB_PAGE
- 
-#include <linux/mempolicy.h>
-+#include <linux/pagemap.h>
- #include <linux/shm.h>
- #include <asm/tlbflush.h>
- 
- /*
-  * For HugeTLB page, there are more metadata to save in the struct page. But
-  * the head struct page cannot meet our needs, so we have to abuse other tail
- * struct page to store the metadata. In order to avoid conflicts caused by
- * subsequent use of more tail struct pages, we gather these discrete indexes
- * of tail struct page here.
-+ * struct page to store the metadata.
-  */
-enum {
-	SUBPAGE_INDEX_SUBPOOL = 1,	/* reuse page->private */
-#ifdef CONFIG_CGROUP_HUGETLB
-	SUBPAGE_INDEX_CGROUP,		/* reuse page->private */
-	SUBPAGE_INDEX_CGROUP_RSVD,	/* reuse page->private */
-	__MAX_CGROUP_SUBPAGE_INDEX = SUBPAGE_INDEX_CGROUP_RSVD,
-#endif
-#ifdef CONFIG_MEMORY_FAILURE
-	SUBPAGE_INDEX_HWPOISON,
-#endif
-	__NR_USED_SUBPAGE,
-};
-+#define __NR_USED_SUBPAGE 3
- 
- struct hugepage_subpool {
- 	spinlock_t lock;
-@@ -68,6 +55,7 @@ struct resv_map {
- 	long adds_in_progress;
- 	struct list_head region_cache;
- 	long region_cache_count;
-+	struct rw_semaphore rw_sema;
- #ifdef CONFIG_CGROUP_HUGETLB
- 	/*
- 	 * On private mappings, the counter to uncharge reservations is stored
-@@ -114,6 +102,12 @@ struct file_region {
- #endif
- };
- 
-+struct hugetlb_vma_lock {
-+	struct kref refs;
-+	struct rw_semaphore rw_sema;
-+	struct vm_area_struct *vma;
-+};
-+
- extern struct resv_map *resv_map_alloc(void);
- void resv_map_release(struct kref *ref);
- 
-@@ -126,33 +120,21 @@ struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
- 						long min_hpages);
- void hugepage_put_subpool(struct hugepage_subpool *spool);
- 
-void reset_vma_resv_huge_pages(struct vm_area_struct *vma);
-+void hugetlb_dup_vma_private(struct vm_area_struct *vma);
- void clear_vma_resv_huge_pages(struct vm_area_struct *vma);
-int hugetlb_sysctl_handler(struct ctl_table *, int, void *, size_t *, loff_t *);
-int hugetlb_overcommit_handler(struct ctl_table *, int, void *, size_t *,
-		loff_t *);
-int hugetlb_treat_movable_handler(struct ctl_table *, int, void *, size_t *,
-		loff_t *);
-int hugetlb_mempolicy_sysctl_handler(struct ctl_table *, int, void *, size_t *,
-		loff_t *);
-
- int move_hugetlb_page_tables(struct vm_area_struct *vma,
- 			     struct vm_area_struct *new_vma,
- 			     unsigned long old_addr, unsigned long new_addr,
- 			     unsigned long len);
- int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *,
- 			    struct vm_area_struct *, struct vm_area_struct *);
-long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *,
-			 struct page **, struct vm_area_struct **,
-			 unsigned long *, unsigned long *, long, unsigned int,
-			 int *);
- void unmap_hugepage_range(struct vm_area_struct *,
-			  unsigned long, unsigned long, struct page *,
-			  zap_flags_t);
-void __unmap_hugepage_range_final(struct mmu_gather *tlb,
-+			  unsigned long start, unsigned long end,
-+			  struct folio *, zap_flags_t);
-+void __unmap_hugepage_range(struct mmu_gather *tlb,
- 			  struct vm_area_struct *vma,
- 			  unsigned long start, unsigned long end,
-			  struct page *ref_page, zap_flags_t zap_flags);
-+			  struct folio *, zap_flags_t zap_flags);
- void hugetlb_report_meminfo(struct seq_file *);
- int hugetlb_report_node_meminfo(char *buf, int len, int nid);
- void hugetlb_show_meminfo_node(int nid);
-@@ -160,41 +142,101 @@ unsigned long hugetlb_total_pages(void);
- vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
- 			unsigned long address, unsigned int flags);
- #ifdef CONFIG_USERFAULTFD
-int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte,
-				struct vm_area_struct *dst_vma,
-				unsigned long dst_addr,
-				unsigned long src_addr,
-				enum mcopy_atomic_mode mode,
-				struct page **pagep,
-				bool wp_copy);
-+int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
-+			     struct vm_area_struct *dst_vma,
-+			     unsigned long dst_addr,
-+			     unsigned long src_addr,
-+			     uffd_flags_t flags,
-+			     struct folio **foliop);
- #endif /* CONFIG_USERFAULTFD */
-bool hugetlb_reserve_pages(struct inode *inode, long from, long to,
-						struct vm_area_struct *vma,
-						vm_flags_t vm_flags);
-+long hugetlb_reserve_pages(struct inode *inode, long from, long to,
-+			   struct vm_area_desc *desc, vm_flags_t vm_flags);
- long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
- 						long freed);
-int isolate_hugetlb(struct page *page, struct list_head *list);
-int get_hwpoison_huge_page(struct page *page, bool *hugetlb);
-int get_huge_page_for_hwpoison(unsigned long pfn, int flags);
-void putback_active_hugepage(struct page *page);
-void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason);
-void free_huge_page(struct page *page);
-+bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list);
-+int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison);
-+int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
-+				bool *migratable_cleared);
-+void folio_putback_hugetlb(struct folio *folio);
-+void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason);
- void hugetlb_fix_reserve_counts(struct inode *inode);
- extern struct mutex *hugetlb_fault_mutex_table;
- u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx);
- 
- pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
- 		      unsigned long addr, pud_t *pud);
-+bool hugetlbfs_pagecache_present(struct hstate *h,
-+				 struct vm_area_struct *vma,
-+				 unsigned long address);
- 
-struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage);
-+struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio);
- 
-extern int sysctl_hugetlb_shm_group;
-extern struct list_head huge_boot_pages;
-+extern int sysctl_hugetlb_shm_group __read_mostly;
-+extern struct list_head huge_boot_pages[MAX_NUMNODES];
-+
-+void hugetlb_bootmem_alloc(void);
-+bool hugetlb_bootmem_allocated(void);
-+extern nodemask_t hugetlb_bootmem_nodes;
-+void hugetlb_bootmem_set_nodes(void);
- 
- /* arch callbacks */
- 
-+#ifndef CONFIG_HIGHPTE
-+/*
-+ * pte_offset_huge() and pte_alloc_huge() are helpers for those architectures
-+ * which may go down to the lowest PTE level in their huge_pte_offset() and
-+ * huge_pte_alloc(): to avoid reliance on pte_offset_map() without pte_unmap().
-+ */
-+static inline pte_t *pte_offset_huge(pmd_t *pmd, unsigned long address)
-+{
-+	return pte_offset_kernel(pmd, address);
-+}
-+static inline pte_t *pte_alloc_huge(struct mm_struct *mm, pmd_t *pmd,
-+				    unsigned long address)
-+{
-+	return pte_alloc(mm, pmd) ? NULL : pte_offset_huge(pmd, address);
-+}
-+#endif
-+
- pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
- 			unsigned long addr, unsigned long sz);
-+/*
-+ * huge_pte_offset(): Walk the hugetlb pgtable until the last level PTE.
-+ * Returns the pte_t* if found, or NULL if the address is not mapped.
-+ *
-+ * IMPORTANT: we should normally not directly call this function, instead
-+ * this is only a common interface to implement arch-specific
-+ * walker. Please use hugetlb_walk() instead, because that will attempt to
-+ * verify the locking for you.
-+ *
-+ * Since this function will walk all the pgtable pages (including not only
-+ * high-level pgtable page, but also PUD entry that can be unshared
-+ * concurrently for VM_SHARED), the caller of this function should be
-+ * responsible of its thread safety.  One can follow this rule:
-+ *
-+ *  (1) For private mappings: pmd unsharing is not possible, so holding the
-+ *      mmap_lock for either read or write is sufficient. Most callers
-+ *      already hold the mmap_lock, so normally, no special action is
-+ *      required.
-+ *
-+ *  (2) For shared mappings: pmd unsharing is possible (so the PUD-ranged
-+ *      pgtable page can go away from under us!  It can be done by a pmd
-+ *      unshare with a follow up munmap() on the other process), then we
-+ *      need either:
-+ *
-+ *     (2.1) hugetlb vma lock read or write held, to make sure pmd unshare
-+ *           won't happen upon the range (it also makes sure the pte_t we
-+ *           read is the right and stable one), or,
-+ *
-+ *     (2.2) hugetlb mapping i_mmap_rwsem lock held read or write, to make
-+ *           sure even if unshare happened the racy unmap() will wait until
-+ *           i_mmap_rwsem is released.
-+ *
-+ * Option (2.1) is the safest, which guarantees pte stability from pmd
-+ * sharing pov, until the vma lock released.  Option (2.2) doesn't protect
-+ * a concurrent pmd unshare, but it makes sure the pgtable page is safe to
-+ * access.
-+ */
- pte_t *huge_pte_offset(struct mm_struct *mm,
- 		       unsigned long addr, unsigned long sz);
- unsigned long hugetlb_mask_last_page(struct hstate *h);
-@@ -202,30 +244,44 @@ int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
- 				unsigned long addr, pte_t *ptep);
- void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
- 				unsigned long *start, unsigned long *end);
-struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
-			      int write);
-struct page *follow_huge_pd(struct vm_area_struct *vma,
-			    unsigned long address, hugepd_t hpd,
-			    int flags, int pdshift);
-struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
-				pmd_t *pmd, int flags);
-struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
-				pud_t *pud, int flags);
-struct page *follow_huge_pgd(struct mm_struct *mm, unsigned long address,
-			     pgd_t *pgd, int flags);
-
-int pmd_huge(pmd_t pmd);
-int pud_huge(pud_t pud);
-unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
-+
-+extern void __hugetlb_zap_begin(struct vm_area_struct *vma,
-+				unsigned long *begin, unsigned long *end);
-+extern void __hugetlb_zap_end(struct vm_area_struct *vma,
-+			      struct zap_details *details);
-+
-+static inline void hugetlb_zap_begin(struct vm_area_struct *vma,
-+				     unsigned long *start, unsigned long *end)
-+{
-+	if (is_vm_hugetlb_page(vma))
-+		__hugetlb_zap_begin(vma, start, end);
-+}
-+
-+static inline void hugetlb_zap_end(struct vm_area_struct *vma,
-+				   struct zap_details *details)
-+{
-+	if (is_vm_hugetlb_page(vma))
-+		__hugetlb_zap_end(vma, details);
-+}
-+
-+void hugetlb_vma_lock_read(struct vm_area_struct *vma);
-+void hugetlb_vma_unlock_read(struct vm_area_struct *vma);
-+void hugetlb_vma_lock_write(struct vm_area_struct *vma);
-+void hugetlb_vma_unlock_write(struct vm_area_struct *vma);
-+int hugetlb_vma_trylock_write(struct vm_area_struct *vma);
-+void hugetlb_vma_assert_locked(struct vm_area_struct *vma);
-+void hugetlb_vma_lock_release(struct kref *kref);
-+long hugetlb_change_protection(struct vm_area_struct *vma,
- 		unsigned long address, unsigned long end, pgprot_t newprot,
- 		unsigned long cp_flags);
-
-bool is_hugetlb_entry_migration(pte_t pte);
- void hugetlb_unshare_all_pmds(struct vm_area_struct *vma);
-+void fixup_hugetlb_reservations(struct vm_area_struct *vma);
-+void hugetlb_split(struct vm_area_struct *vma, unsigned long addr);
-+int hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
- 
- #else /* !CONFIG_HUGETLB_PAGE */
- 
-static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
-+static inline void hugetlb_dup_vma_private(struct vm_area_struct *vma)
- {
- }
- 
-@@ -238,8 +294,8 @@ static inline unsigned long hugetlb_total_pages(void)
- 	return 0;
- }
- 
-static inline struct address_space *hugetlb_page_mapping_lock_write(
-							struct page *hpage)
-+static inline struct address_space *hugetlb_folio_mapping_lock_write(
-+							struct folio *folio)
- {
- 	return NULL;
- }
-@@ -257,20 +313,16 @@ static inline void adjust_range_if_pmd_sharing_possible(
- {
- }
- 
-static inline long follow_hugetlb_page(struct mm_struct *mm,
-			struct vm_area_struct *vma, struct page **pages,
-			struct vm_area_struct **vmas, unsigned long *position,
-			unsigned long *nr_pages, long i, unsigned int flags,
-			int *nonblocking)
-+static inline void hugetlb_zap_begin(
-+				struct vm_area_struct *vma,
-+				unsigned long *start, unsigned long *end)
- {
-	BUG();
-	return 0;
- }
- 
-static inline struct page *follow_huge_addr(struct mm_struct *mm,
-					unsigned long address, int write)
-+static inline void hugetlb_zap_end(
-+				struct vm_area_struct *vma,
-+				struct zap_details *details)
- {
-	return ERR_PTR(-EINVAL);
- }
- 
- static inline int copy_hugetlb_page_range(struct mm_struct *dst,
-@@ -305,45 +357,29 @@ static inline void hugetlb_show_meminfo_node(int nid)
- {
- }
- 
-static inline struct page *follow_huge_pd(struct vm_area_struct *vma,
-				unsigned long address, hugepd_t hpd, int flags,
-				int pdshift)
-+static inline void hugetlb_vma_lock_read(struct vm_area_struct *vma)
- {
-	return NULL;
- }
- 
-static inline struct page *follow_huge_pmd(struct mm_struct *mm,
-				unsigned long address, pmd_t *pmd, int flags)
-+static inline void hugetlb_vma_unlock_read(struct vm_area_struct *vma)
- {
-	return NULL;
- }
- 
-static inline struct page *follow_huge_pud(struct mm_struct *mm,
-				unsigned long address, pud_t *pud, int flags)
-+static inline void hugetlb_vma_lock_write(struct vm_area_struct *vma)
- {
-	return NULL;
-}
-
-static inline struct page *follow_huge_pgd(struct mm_struct *mm,
-				unsigned long address, pgd_t *pgd, int flags)
-{
-	return NULL;
- }
- 
-static inline int prepare_hugepage_range(struct file *file,
-				unsigned long addr, unsigned long len)
-+static inline void hugetlb_vma_unlock_write(struct vm_area_struct *vma)
- {
-	return -EINVAL;
- }
- 
-static inline int pmd_huge(pmd_t pmd)
-+static inline int hugetlb_vma_trylock_write(struct vm_area_struct *vma)
- {
-	return 0;
-+	return 1;
- }
- 
-static inline int pud_huge(pud_t pud)
-+static inline void hugetlb_vma_assert_locked(struct vm_area_struct *vma)
- {
-	return 0;
- }
- 
- static inline int is_hugepage_only_range(struct mm_struct *mm,
-@@ -352,22 +388,13 @@ static inline int is_hugepage_only_range(struct mm_struct *mm,
- 	return 0;
- }
- 
-static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb,
-				unsigned long addr, unsigned long end,
-				unsigned long floor, unsigned long ceiling)
-{
-	BUG();
-}
-
- #ifdef CONFIG_USERFAULTFD
-static inline int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
-						pte_t *dst_pte,
-						struct vm_area_struct *dst_vma,
-						unsigned long dst_addr,
-						unsigned long src_addr,
-						enum mcopy_atomic_mode mode,
-						struct page **pagep,
-						bool wp_copy)
-+static inline int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
-+					   struct vm_area_struct *dst_vma,
-+					   unsigned long dst_addr,
-+					   unsigned long src_addr,
-+					   uffd_flags_t flags,
-+					   struct folio **foliop)
- {
- 	BUG();
- 	return 0;
-@@ -380,31 +407,32 @@ static inline pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr,
- 	return NULL;
- }
- 
-static inline int isolate_hugetlb(struct page *page, struct list_head *list)
-+static inline bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list)
- {
-	return -EBUSY;
-+	return false;
- }
- 
-static inline int get_hwpoison_huge_page(struct page *page, bool *hugetlb)
-+static inline int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison)
- {
- 	return 0;
- }
- 
-static inline int get_huge_page_for_hwpoison(unsigned long pfn, int flags)
-+static inline int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
-+					bool *migratable_cleared)
- {
- 	return 0;
- }
- 
-static inline void putback_active_hugepage(struct page *page)
-+static inline void folio_putback_hugetlb(struct folio *folio)
- {
- }
- 
-static inline void move_hugetlb_state(struct page *oldpage,
-					struct page *newpage, int reason)
-+static inline void move_hugetlb_state(struct folio *old_folio,
-+					struct folio *new_folio, int reason)
- {
- }
- 
-static inline unsigned long hugetlb_change_protection(
-+static inline long hugetlb_change_protection(
- 			struct vm_area_struct *vma, unsigned long address,
- 			unsigned long end, pgprot_t newprot,
- 			unsigned long cp_flags)
-@@ -412,9 +440,9 @@ static inline unsigned long hugetlb_change_protection(
- 	return 0;
- }
- 
-static inline void __unmap_hugepage_range_final(struct mmu_gather *tlb,
-+static inline void __unmap_hugepage_range(struct mmu_gather *tlb,
- 			struct vm_area_struct *vma, unsigned long start,
-			unsigned long end, struct page *ref_page,
-+			unsigned long end, struct folio *folio,
- 			zap_flags_t zap_flags)
- {
- 	BUG();
-@@ -430,17 +458,18 @@ static inline vm_fault_t hugetlb_fault(struct mm_struct *mm,
- 
- static inline void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) { }
- 
-+static inline void fixup_hugetlb_reservations(struct vm_area_struct *vma)
-+{
-+}
-+
-+static inline void hugetlb_split(struct vm_area_struct *vma, unsigned long addr) {}
-+
-+static inline int hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
-+{
-+	return 0;
-+}
-+
- #endif /* !CONFIG_HUGETLB_PAGE */
-/*
- * hugepages at page global directory. If arch support
- * hugepages at pgd level, they need to define this.
- */
-#ifndef pgd_huge
-#define pgd_huge(x)	0
-#endif
-#ifndef p4d_huge
-#define p4d_huge(x)	0
-#endif
- 
- #ifndef pgd_write
- static inline int pgd_write(pgd_t pgd)
-@@ -483,7 +512,6 @@ static inline struct hugetlbfs_sb_info *HUGETLBFS_SB(struct super_block *sb)
- }
- 
- struct hugetlbfs_inode_info {
-	struct shared_policy policy;
- 	struct inode vfs_inode;
- 	unsigned int seals;
- };
-@@ -493,17 +521,13 @@ static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
- 	return container_of(inode, struct hugetlbfs_inode_info, vfs_inode);
- }
- 
-extern const struct file_operations hugetlbfs_file_operations;
- extern const struct vm_operations_struct hugetlb_vm_ops;
- struct file *hugetlb_file_setup(const char *name, size_t size, vm_flags_t acct,
- 				int creat_flags, int page_size_log);
- 
-static inline bool is_file_hugepages(struct file *file)
-+static inline bool is_file_hugepages(const struct file *file)
- {
-	if (file->f_op == &hugetlbfs_file_operations)
-		return true;
-
-	return is_file_shm_hugepages(file);
-+	return file->f_op->fop_flags & FOP_HUGE_PAGES;
- }
- 
- static inline struct hstate *hstate_inode(struct inode *i)
-@@ -526,16 +550,10 @@ static inline struct hstate *hstate_inode(struct inode *i)
- }
- #endif /* !CONFIG_HUGETLBFS */
- 
-#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
-unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
-					unsigned long len, unsigned long pgoff,
-					unsigned long flags);
-#endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */
-
- unsigned long
-generic_hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
-				  unsigned long len, unsigned long pgoff,
-				  unsigned long flags);
-+hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
-+				    unsigned long len, unsigned long pgoff,
-+				    unsigned long flags);
- 
- /*
-  * huegtlb page specific state flags.  These flags are located in page.private
-@@ -574,6 +592,7 @@ enum hugetlb_page_flags {
- 	HPG_freed,
- 	HPG_vmemmap_optimized,
- 	HPG_raw_hwp_unreliable,
-+	HPG_cma,
- 	__NR_HPAGEFLAGS,
- };
- 
-@@ -583,27 +602,39 @@ enum hugetlb_page_flags {
-  */
- #ifdef CONFIG_HUGETLB_PAGE
- #define TESTHPAGEFLAG(uname, flname)				\
-static inline int HPage##uname(struct page *page)		\
-	{ return test_bit(HPG_##flname, &(page->private)); }
-+static __always_inline						\
-+bool folio_test_hugetlb_##flname(struct folio *folio)		\
-+	{	void *private = &folio->private;		\
-+		return test_bit(HPG_##flname, private);		\
-+	}
- 
- #define SETHPAGEFLAG(uname, flname)				\
-static inline void SetHPage##uname(struct page *page)		\
-	{ set_bit(HPG_##flname, &(page->private)); }
-+static __always_inline						\
-+void folio_set_hugetlb_##flname(struct folio *folio)		\
-+	{	void *private = &folio->private;		\
-+		set_bit(HPG_##flname, private);			\
-+	}
- 
- #define CLEARHPAGEFLAG(uname, flname)				\
-static inline void ClearHPage##uname(struct page *page)		\
-	{ clear_bit(HPG_##flname, &(page->private)); }
-+static __always_inline						\
-+void folio_clear_hugetlb_##flname(struct folio *folio)		\
-+	{	void *private = &folio->private;		\
-+		clear_bit(HPG_##flname, private);		\
-+	}
- #else
- #define TESTHPAGEFLAG(uname, flname)				\
-static inline int HPage##uname(struct page *page)		\
-+static inline bool						\
-+folio_test_hugetlb_##flname(struct folio *folio)		\
- 	{ return 0; }
- 
- #define SETHPAGEFLAG(uname, flname)				\
-static inline void SetHPage##uname(struct page *page)		\
-+static inline void						\
-+folio_set_hugetlb_##flname(struct folio *folio) 		\
- 	{ }
- 
- #define CLEARHPAGEFLAG(uname, flname)				\
-static inline void ClearHPage##uname(struct page *page)		\
-+static inline void						\
-+folio_clear_hugetlb_##flname(struct folio *folio)		\
- 	{ }
- #endif
- 
-@@ -621,6 +652,7 @@ HPAGEFLAG(Temporary, temporary)
- HPAGEFLAG(Freed, freed)
- HPAGEFLAG(VmemmapOptimized, vmemmap_optimized)
- HPAGEFLAG(RawHwpUnreliable, raw_hwp_unreliable)
-+HPAGEFLAG(Cma, cma)
- 
- #ifdef CONFIG_HUGETLB_PAGE
- 
-@@ -628,6 +660,7 @@ HPAGEFLAG(RawHwpUnreliable, raw_hwp_unreliable)
- /* Defines one hugetlb page size */
- struct hstate {
- 	struct mutex resize_lock;
-+	struct lock_class_key resize_key;
- 	int next_nid_to_alloc;
- 	int next_nid_to_free;
- 	unsigned int order;
-@@ -645,30 +678,39 @@ struct hstate {
- 	unsigned int nr_huge_pages_node[MAX_NUMNODES];
- 	unsigned int free_huge_pages_node[MAX_NUMNODES];
- 	unsigned int surplus_huge_pages_node[MAX_NUMNODES];
-#ifdef CONFIG_CGROUP_HUGETLB
-	/* cgroup control files */
-	struct cftype cgroup_files_dfl[8];
-	struct cftype cgroup_files_legacy[10];
-#endif
- 	char name[HSTATE_NAME_LEN];
- };
- 
-+struct cma;
-+
- struct huge_bootmem_page {
- 	struct list_head list;
- 	struct hstate *hstate;
-+	unsigned long flags;
-+	struct cma *cma;
- };
- 
-int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list);
-struct page *alloc_huge_page(struct vm_area_struct *vma,
-				unsigned long addr, int avoid_reserve);
-struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
-				nodemask_t *nmask, gfp_t gfp_mask);
-struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
-				unsigned long address);
-int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
-+#define HUGE_BOOTMEM_HVO		0x0001
-+#define HUGE_BOOTMEM_ZONES_VALID	0x0002
-+#define HUGE_BOOTMEM_CMA		0x0004
-+
-+bool hugetlb_bootmem_page_zones_valid(int nid, struct huge_bootmem_page *m);
-+
-+int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list);
-+int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn);
-+void wait_for_freed_hugetlb_folios(void);
-+struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
-+				unsigned long addr, bool cow_from_owner);
-+struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
-+				nodemask_t *nmask, gfp_t gfp_mask,
-+				bool allow_alloc_fallback);
-+struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid,
-+					  nodemask_t *nmask, gfp_t gfp_mask);
-+
-+int hugetlb_add_to_page_cache(struct folio *folio, struct address_space *mapping,
- 			pgoff_t idx);
- void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
-				unsigned long address, struct page *page);
-+				unsigned long address, struct folio *folio);
- 
- /* arch callback */
- int __init __alloc_bootmem_huge_page(struct hstate *h, int nid);
-@@ -688,18 +730,20 @@ extern unsigned int default_hstate_idx;
- 
- #define default_hstate (hstates[default_hstate_idx])
- 
-/*
- * hugetlb page subpool pointer located in hpage[1].private
- */
-static inline struct hugepage_subpool *hugetlb_page_subpool(struct page *hpage)
-+static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
-+{
-+	return HUGETLBFS_SB(inode->i_sb)->spool;
-+}
-+
-+static inline struct hugepage_subpool *hugetlb_folio_subpool(struct folio *folio)
- {
-	return (void *)page_private(hpage + SUBPAGE_INDEX_SUBPOOL);
-+	return folio->_hugetlb_subpool;
- }
- 
-static inline void hugetlb_set_page_subpool(struct page *hpage,
-+static inline void hugetlb_set_folio_subpool(struct folio *folio,
- 					struct hugepage_subpool *subpool)
- {
-	set_page_private(hpage + SUBPAGE_INDEX_SUBPOOL, (unsigned long)subpool);
-+	folio->_hugetlb_subpool = subpool;
- }
- 
- static inline struct hstate *hstate_file(struct file *f)
-@@ -712,7 +756,10 @@ static inline struct hstate *hstate_sizelog(int page_size_log)
- 	if (!page_size_log)
- 		return &default_hstate;
- 
-	return size_to_hstate(1UL << page_size_log);
-+	if (page_size_log < BITS_PER_LONG)
-+		return size_to_hstate(1UL << page_size_log);
-+
-+	return NULL;
- }
- 
- static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
-@@ -744,9 +791,14 @@ static inline unsigned huge_page_shift(struct hstate *h)
- 	return h->order + PAGE_SHIFT;
- }
- 
-+static inline bool order_is_gigantic(unsigned int order)
-+{
-+	return order > MAX_PAGE_ORDER;
-+}
-+
- static inline bool hstate_is_gigantic(struct hstate *h)
- {
-	return huge_page_order(h) >= MAX_ORDER;
-+	return order_is_gigantic(huge_page_order(h));
- }
- 
- static inline unsigned int pages_per_huge_page(const struct hstate *h)
-@@ -759,6 +811,12 @@ static inline unsigned int blocks_per_huge_page(struct hstate *h)
- 	return huge_page_size(h) / 512;
- }
- 
-+static inline struct folio *filemap_lock_hugetlb_folio(struct hstate *h,
-+				struct address_space *mapping, pgoff_t idx)
-+{
-+	return filemap_lock_folio(mapping, idx << huge_page_order(h));
-+}
-+
- #include <asm/hugetlb.h>
- 
- #ifndef is_hugepage_only_range
-@@ -770,9 +828,9 @@ static inline int is_hugepage_only_range(struct mm_struct *mm,
- #define is_hugepage_only_range is_hugepage_only_range
- #endif
- 
-#ifndef arch_clear_hugepage_flags
-static inline void arch_clear_hugepage_flags(struct page *page) { }
-#define arch_clear_hugepage_flags arch_clear_hugepage_flags
-+#ifndef arch_clear_hugetlb_flags
-+static inline void arch_clear_hugetlb_flags(struct folio *folio) { }
-+#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags
- #endif
- 
- #ifndef arch_make_huge_pte
-@@ -783,10 +841,21 @@ static inline pte_t arch_make_huge_pte(pte_t entry, unsigned int shift,
- }
- #endif
- 
-static inline struct hstate *page_hstate(struct page *page)
-+#ifndef arch_has_huge_bootmem_alloc
-+/*
-+ * Some architectures do their own bootmem allocation, so they can't use
-+ * early CMA allocation.
-+ */
-+static inline bool arch_has_huge_bootmem_alloc(void)
-+{
-+	return false;
-+}
-+#endif
-+
-+static inline struct hstate *folio_hstate(struct folio *folio)
- {
-	VM_BUG_ON_PAGE(!PageHuge(page), page);
-	return size_to_hstate(page_size(page));
-+	VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio);
-+	return size_to_hstate(folio_size(folio));
- }
- 
- static inline unsigned hstate_index_to_shift(unsigned index)
-@@ -799,14 +868,14 @@ static inline int hstate_index(struct hstate *h)
- 	return h - hstates;
- }
- 
-extern int dissolve_free_huge_page(struct page *page);
-extern int dissolve_free_huge_pages(unsigned long start_pfn,
-+int dissolve_free_hugetlb_folio(struct folio *folio);
-+int dissolve_free_hugetlb_folios(unsigned long start_pfn,
- 				    unsigned long end_pfn);
- 
- #ifdef CONFIG_MEMORY_FAILURE
-extern void hugetlb_clear_page_hwpoison(struct page *hpage);
-+extern void folio_clear_hugetlb_hwpoison(struct folio *folio);
- #else
-static inline void hugetlb_clear_page_hwpoison(struct page *hpage)
-+static inline void folio_clear_hugetlb_hwpoison(struct folio *folio)
- {
- }
- #endif
-@@ -863,10 +932,11 @@ static inline bool hugepage_movable_supported(struct hstate *h)
- /* Movability of hugepages depends on migration support. */
- static inline gfp_t htlb_alloc_mask(struct hstate *h)
- {
-	if (hugepage_movable_supported(h))
-		return GFP_HIGHUSER_MOVABLE;
-	else
-		return GFP_HIGHUSER;
-+	gfp_t gfp = __GFP_COMP | __GFP_NOWARN;
-+
-+	gfp |= hugepage_movable_supported(h) ? GFP_HIGHUSER_MOVABLE : GFP_HIGHUSER;
-+
-+	return gfp;
- }
- 
- static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask)
-@@ -881,13 +951,64 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask)
- 	return modified_mask;
- }
- 
-+static inline bool htlb_allow_alloc_fallback(int reason)
-+{
-+	bool allowed_fallback = false;
-+
-+	/*
-+	 * Note: the memory offline, memory failure and migration syscalls will
-+	 * be allowed to fallback to other nodes due to lack of a better chioce,
-+	 * that might break the per-node hugetlb pool. While other cases will
-+	 * set the __GFP_THISNODE to avoid breaking the per-node hugetlb pool.
-+	 */
-+	switch (reason) {
-+	case MR_MEMORY_HOTPLUG:
-+	case MR_MEMORY_FAILURE:
-+	case MR_SYSCALL:
-+	case MR_MEMPOLICY_MBIND:
-+		allowed_fallback = true;
-+		break;
-+	default:
-+		break;
-+	}
-+
-+	return allowed_fallback;
-+}
-+
- static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
- 					   struct mm_struct *mm, pte_t *pte)
- {
-	if (huge_page_size(h) == PMD_SIZE)
-+	const unsigned long size = huge_page_size(h);
-+
-+	VM_WARN_ON(size == PAGE_SIZE);
-+
-+	/*
-+	 * hugetlb must use the exact same PT locks as core-mm page table
-+	 * walkers would. When modifying a PTE table, hugetlb must take the
-+	 * PTE PT lock, when modifying a PMD table, hugetlb must take the PMD
-+	 * PT lock etc.
-+	 *
-+	 * The expectation is that any hugetlb folio smaller than a PMD is
-+	 * always mapped into a single PTE table and that any hugetlb folio
-+	 * smaller than a PUD (but at least as big as a PMD) is always mapped
-+	 * into a single PMD table.
-+	 *
-+	 * If that does not hold for an architecture, then that architecture
-+	 * must disable split PT locks such that all *_lockptr() functions
-+	 * will give us the same result: the per-MM PT lock.
-+	 *
-+	 * Note that with e.g., CONFIG_PGTABLE_LEVELS=2 where
-+	 * PGDIR_SIZE==P4D_SIZE==PUD_SIZE==PMD_SIZE, we'd use pud_lockptr()
-+	 * and core-mm would use pmd_lockptr(). However, in such configurations
-+	 * split PMD locks are disabled -- they don't make sense on a single
-+	 * PGDIR page table -- and the end result is the same.
-+	 */
-+	if (size >= PUD_SIZE)
-+		return pud_lockptr(mm, (pud_t *) pte);
-+	else if (size >= PMD_SIZE || IS_ENABLED(CONFIG_HIGHPTE))
- 		return pmd_lockptr(mm, (pmd_t *) pte);
-	VM_BUG_ON(huge_page_size(h) == PAGE_SIZE);
-	return &mm->page_table_lock;
-+	/* pte_alloc_huge() only applies with !CONFIG_HIGHPTE */
-+	return ptep_lockptr(mm, pte);
- }
- 
- #ifndef hugepages_supported
-@@ -921,7 +1042,9 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
- static inline pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
- 						unsigned long addr, pte_t *ptep)
- {
-	return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
-+	unsigned long psize = huge_page_size(hstate_vma(vma));
-+
-+	return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, psize);
- }
- #endif
- 
-@@ -931,41 +1054,80 @@ static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
- 						unsigned long addr, pte_t *ptep,
- 						pte_t old_pte, pte_t pte)
- {
-	set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
-+	unsigned long psize = huge_page_size(hstate_vma(vma));
-+
-+	set_huge_pte_at(vma->vm_mm, addr, ptep, pte, psize);
- }
- #endif
- 
-+#ifdef CONFIG_NUMA
-+void hugetlb_register_node(struct node *node);
-+void hugetlb_unregister_node(struct node *node);
-+#endif
-+
-+/*
-+ * Check if a given raw @page in a hugepage is HWPOISON.
-+ */
-+bool is_raw_hwpoison_page_in_hugepage(struct page *page);
-+
-+static inline unsigned long huge_page_mask_align(struct file *file)
-+{
-+	return PAGE_MASK & ~huge_page_mask(hstate_file(file));
-+}
-+
- #else	/* CONFIG_HUGETLB_PAGE */
- struct hstate {};
- 
-static inline struct hugepage_subpool *hugetlb_page_subpool(struct page *hpage)
-+static inline unsigned long huge_page_mask_align(struct file *file)
-+{
-+	return 0;
-+}
-+
-+static inline struct hugepage_subpool *hugetlb_folio_subpool(struct folio *folio)
- {
- 	return NULL;
- }
- 
-static inline int isolate_or_dissolve_huge_page(struct page *page,
-+static inline struct folio *filemap_lock_hugetlb_folio(struct hstate *h,
-+				struct address_space *mapping, pgoff_t idx)
-+{
-+	return NULL;
-+}
-+
-+static inline int isolate_or_dissolve_huge_folio(struct folio *folio,
- 						struct list_head *list)
- {
- 	return -ENOMEM;
- }
- 
-static inline struct page *alloc_huge_page(struct vm_area_struct *vma,
-+static inline int replace_free_hugepage_folios(unsigned long start_pfn,
-+		unsigned long end_pfn)
-+{
-+	return 0;
-+}
-+
-+static inline void wait_for_freed_hugetlb_folios(void)
-+{
-+}
-+
-+static inline struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
- 					   unsigned long addr,
-					   int avoid_reserve)
-+					   bool cow_from_owner)
- {
- 	return NULL;
- }
- 
-static inline struct page *
-alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
-			nodemask_t *nmask, gfp_t gfp_mask)
-+static inline struct folio *
-+alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid,
-+			    nodemask_t *nmask, gfp_t gfp_mask)
- {
- 	return NULL;
- }
- 
-static inline struct page *alloc_huge_page_vma(struct hstate *h,
-					       struct vm_area_struct *vma,
-					       unsigned long address)
-+static inline struct folio *
-+alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
-+			nodemask_t *nmask, gfp_t gfp_mask,
-+			bool allow_alloc_fallback)
- {
- 	return NULL;
- }
-@@ -990,7 +1152,7 @@ static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
- 	return NULL;
- }
- 
-static inline struct hstate *page_hstate(struct page *page)
-+static inline struct hstate *folio_hstate(struct folio *folio)
- {
- 	return NULL;
- }
-@@ -1050,12 +1212,12 @@ static inline int hstate_index(struct hstate *h)
- 	return 0;
- }
- 
-static inline int dissolve_free_huge_page(struct page *page)
-+static inline int dissolve_free_hugetlb_folio(struct folio *folio)
- {
- 	return 0;
- }
- 
-static inline int dissolve_free_huge_pages(unsigned long start_pfn,
-+static inline int dissolve_free_hugetlb_folios(unsigned long start_pfn,
- 					   unsigned long end_pfn)
- {
- 	return 0;
-@@ -1081,6 +1243,11 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask)
- 	return 0;
- }
- 
-+static inline bool htlb_allow_alloc_fallback(int reason)
-+{
-+	return false;
-+}
-+
- static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
- 					   struct mm_struct *mm, pte_t *pte)
- {
-@@ -1102,13 +1269,40 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
- static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
- 					  unsigned long addr, pte_t *ptep)
- {
-+#ifdef CONFIG_MMU
-+	return ptep_get(ptep);
-+#else
- 	return *ptep;
-+#endif
- }
- 
- static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
-				   pte_t *ptep, pte_t pte)
-+				   pte_t *ptep, pte_t pte, unsigned long sz)
-+{
-+}
-+
-+static inline void hugetlb_register_node(struct node *node)
- {
- }
-+
-+static inline void hugetlb_unregister_node(struct node *node)
-+{
-+}
-+
-+static inline bool hugetlbfs_pagecache_present(
-+    struct hstate *h, struct vm_area_struct *vma, unsigned long address)
-+{
-+	return false;
-+}
-+
-+static inline void hugetlb_bootmem_alloc(void)
-+{
-+}
-+
-+static inline bool hugetlb_bootmem_allocated(void)
-+{
-+	return false;
-+}
- #endif	/* CONFIG_HUGETLB_PAGE */
- 
- static inline spinlock_t *huge_pte_lock(struct hstate *h,
-@@ -1123,13 +1317,21 @@ static inline spinlock_t *huge_pte_lock(struct hstate *h,
- 
- #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA)
- extern void __init hugetlb_cma_reserve(int order);
-extern void __init hugetlb_cma_check(void);
- #else
- static inline __init void hugetlb_cma_reserve(int order)
- {
- }
-static inline __init void hugetlb_cma_check(void)
-+#endif
-+
-+#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
-+static inline bool hugetlb_pmd_shared(pte_t *pte)
- {
-+	return page_count(virt_to_page(pte)) > 1;
-+}
-+#else
-+static inline bool hugetlb_pmd_shared(pte_t *pte)
-+{
-+	return false;
- }
- #endif
- 
-@@ -1143,4 +1345,36 @@ bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr);
- #define flush_hugetlb_tlb_range(vma, addr, end)	flush_tlb_range(vma, addr, end)
- #endif
- 
-+static inline bool __vma_shareable_lock(struct vm_area_struct *vma)
-+{
-+	return (vma->vm_flags & VM_MAYSHARE) && vma->vm_private_data;
-+}
-+
-+bool __vma_private_lock(struct vm_area_struct *vma);
-+
-+/*
-+ * Safe version of huge_pte_offset() to check the locks.  See comments
-+ * above huge_pte_offset().
-+ */
-+static inline pte_t *
-+hugetlb_walk(struct vm_area_struct *vma, unsigned long addr, unsigned long sz)
-+{
-+#if defined(CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING) && defined(CONFIG_LOCKDEP)
-+	struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
-+
-+	/*
-+	 * If pmd sharing possible, locking needed to safely walk the
-+	 * hugetlb pgtables.  More information can be found at the comment
-+	 * above huge_pte_offset() in the same file.
-+	 *
-+	 * NOTE: lockdep_is_held() is only defined with CONFIG_LOCKDEP.
-+	 */
-+	if (__vma_shareable_lock(vma))
-+		WARN_ON_ONCE(!lockdep_is_held(&vma_lock->rw_sema) &&
-+			     !lockdep_is_held(
-+				 &vma->vm_file->f_mapping->i_mmap_rwsem));
-+#endif
-+	return huge_pte_offset(vma->vm_mm, addr, sz);
-+}
-+
- #endif /* _LINUX_HUGETLB_H */
-diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h
-index 379344828e78..e5d64b8b59c2 100644
--- a/include/linux/hugetlb_cgroup.h
-+++ b/include/linux/hugetlb_cgroup.h
-@@ -22,15 +22,6 @@ struct resv_map;
- struct file_region;
- 
- #ifdef CONFIG_CGROUP_HUGETLB
-/*
- * Minimum page order trackable by hugetlb cgroup.
- * At least 4 pages are necessary for all the tracking information.
- * The second tail page (hpage[SUBPAGE_INDEX_CGROUP]) is the fault
- * usage cgroup. The third tail page (hpage[SUBPAGE_INDEX_CGROUP_RSVD])
- * is the reservation usage cgroup.
- */
-#define HUGETLB_CGROUP_MIN_ORDER order_base_2(__MAX_CGROUP_SUBPAGE_INDEX + 1)
-
- enum hugetlb_memory_event {
- 	HUGETLB_MAX,
- 	HUGETLB_NR_MEMORY_EVENTS,
-@@ -67,55 +58,46 @@ struct hugetlb_cgroup {
- };
- 
- static inline struct hugetlb_cgroup *
-__hugetlb_cgroup_from_page(struct page *page, bool rsvd)
-+__hugetlb_cgroup_from_folio(struct folio *folio, bool rsvd)
- {
-	VM_BUG_ON_PAGE(!PageHuge(page), page);
-
-	if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER)
-		return NULL;
-+	VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio);
- 	if (rsvd)
-		return (void *)page_private(page + SUBPAGE_INDEX_CGROUP_RSVD);
-+		return folio->_hugetlb_cgroup_rsvd;
- 	else
-		return (void *)page_private(page + SUBPAGE_INDEX_CGROUP);
-+		return folio->_hugetlb_cgroup;
- }
- 
-static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page)
-+static inline struct hugetlb_cgroup *hugetlb_cgroup_from_folio(struct folio *folio)
- {
-	return __hugetlb_cgroup_from_page(page, false);
-+	return __hugetlb_cgroup_from_folio(folio, false);
- }
- 
- static inline struct hugetlb_cgroup *
-hugetlb_cgroup_from_page_rsvd(struct page *page)
-+hugetlb_cgroup_from_folio_rsvd(struct folio *folio)
- {
-	return __hugetlb_cgroup_from_page(page, true);
-+	return __hugetlb_cgroup_from_folio(folio, true);
- }
- 
-static inline int __set_hugetlb_cgroup(struct page *page,
-+static inline void __set_hugetlb_cgroup(struct folio *folio,
- 				       struct hugetlb_cgroup *h_cg, bool rsvd)
- {
-	VM_BUG_ON_PAGE(!PageHuge(page), page);
-
-	if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER)
-		return -1;
-+	VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio);
- 	if (rsvd)
-		set_page_private(page + SUBPAGE_INDEX_CGROUP_RSVD,
-				 (unsigned long)h_cg);
-+		folio->_hugetlb_cgroup_rsvd = h_cg;
- 	else
-		set_page_private(page + SUBPAGE_INDEX_CGROUP,
-				 (unsigned long)h_cg);
-	return 0;
-+		folio->_hugetlb_cgroup = h_cg;
- }
- 
-static inline int set_hugetlb_cgroup(struct page *page,
-+static inline void set_hugetlb_cgroup(struct folio *folio,
- 				     struct hugetlb_cgroup *h_cg)
- {
-	return __set_hugetlb_cgroup(page, h_cg, false);
-+	__set_hugetlb_cgroup(folio, h_cg, false);
- }
- 
-static inline int set_hugetlb_cgroup_rsvd(struct page *page,
-+static inline void set_hugetlb_cgroup_rsvd(struct folio *folio,
- 					  struct hugetlb_cgroup *h_cg)
- {
-	return __set_hugetlb_cgroup(page, h_cg, true);
-+	__set_hugetlb_cgroup(folio, h_cg, true);
- }
- 
- static inline bool hugetlb_cgroup_disabled(void)
-@@ -148,14 +130,14 @@ extern int hugetlb_cgroup_charge_cgroup_rsvd(int idx, unsigned long nr_pages,
- 					     struct hugetlb_cgroup **ptr);
- extern void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,
- 					 struct hugetlb_cgroup *h_cg,
-					 struct page *page);
-+					 struct folio *folio);
- extern void hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages,
- 					      struct hugetlb_cgroup *h_cg,
-					      struct page *page);
-extern void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages,
-					 struct page *page);
-extern void hugetlb_cgroup_uncharge_page_rsvd(int idx, unsigned long nr_pages,
-					      struct page *page);
-+					      struct folio *folio);
-+extern void hugetlb_cgroup_uncharge_folio(int idx, unsigned long nr_pages,
-+					 struct folio *folio);
-+extern void hugetlb_cgroup_uncharge_folio_rsvd(int idx, unsigned long nr_pages,
-+					      struct folio *folio);
- 
- extern void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
- 					   struct hugetlb_cgroup *h_cg);
-@@ -171,8 +153,8 @@ extern void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv,
- 						bool region_del);
- 
- extern void hugetlb_cgroup_file_init(void) __init;
-extern void hugetlb_cgroup_migrate(struct page *oldhpage,
-				   struct page *newhpage);
-+extern void hugetlb_cgroup_migrate(struct folio *old_folio,
-+				   struct folio *new_folio);
- 
- #else
- static inline void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv,
-@@ -182,33 +164,25 @@ static inline void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv,
- {
- }
- 
-static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page)
-+static inline struct hugetlb_cgroup *hugetlb_cgroup_from_folio(struct folio *folio)
- {
- 	return NULL;
- }
- 
- static inline struct hugetlb_cgroup *
-hugetlb_cgroup_from_page_resv(struct page *page)
-+hugetlb_cgroup_from_folio_rsvd(struct folio *folio)
- {
- 	return NULL;
- }
- 
-static inline struct hugetlb_cgroup *
-hugetlb_cgroup_from_page_rsvd(struct page *page)
-{
-	return NULL;
-}
-
-static inline int set_hugetlb_cgroup(struct page *page,
-+static inline void set_hugetlb_cgroup(struct folio *folio,
- 				     struct hugetlb_cgroup *h_cg)
- {
-	return 0;
- }
- 
-static inline int set_hugetlb_cgroup_rsvd(struct page *page,
-+static inline void set_hugetlb_cgroup_rsvd(struct folio *folio,
- 					  struct hugetlb_cgroup *h_cg)
- {
-	return 0;
- }
- 
- static inline bool hugetlb_cgroup_disabled(void)
-@@ -245,25 +219,25 @@ static inline int hugetlb_cgroup_charge_cgroup_rsvd(int idx,
- 
- static inline void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,
- 						struct hugetlb_cgroup *h_cg,
-						struct page *page)
-+						struct folio *folio)
- {
- }
- 
- static inline void
- hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages,
- 				  struct hugetlb_cgroup *h_cg,
-				  struct page *page)
-+				  struct folio *folio)
- {
- }
- 
-static inline void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages,
-						struct page *page)
-+static inline void hugetlb_cgroup_uncharge_folio(int idx, unsigned long nr_pages,
-+						struct folio *folio)
- {
- }
- 
-static inline void hugetlb_cgroup_uncharge_page_rsvd(int idx,
-+static inline void hugetlb_cgroup_uncharge_folio_rsvd(int idx,
- 						     unsigned long nr_pages,
-						     struct page *page)
-+						     struct folio *folio)
- {
- }
- static inline void hugetlb_cgroup_uncharge_cgroup(int idx,
-@@ -288,8 +262,8 @@ static inline void hugetlb_cgroup_file_init(void)
- {
- }
- 
-static inline void hugetlb_cgroup_migrate(struct page *oldhpage,
-					  struct page *newhpage)
-+static inline void hugetlb_cgroup_migrate(struct folio *old_folio,
-+					  struct folio *new_folio)
- {
- }
- 
-diff --git a/include/linux/hugetlb_inline.h b/include/linux/hugetlb_inline.h
-index 0660a03d37d9..a27aa0162918 100644
--- a/include/linux/hugetlb_inline.h
-+++ b/include/linux/hugetlb_inline.h
-@@ -2,22 +2,27 @@
- #ifndef _LINUX_HUGETLB_INLINE_H
- #define _LINUX_HUGETLB_INLINE_H
- 
-#ifdef CONFIG_HUGETLB_PAGE
-
- #include <linux/mm.h>
- 
-static inline bool is_vm_hugetlb_page(struct vm_area_struct *vma)
-+#ifdef CONFIG_HUGETLB_PAGE
-+
-+static inline bool is_vm_hugetlb_flags(vm_flags_t vm_flags)
- {
-	return !!(vma->vm_flags & VM_HUGETLB);
-+	return !!(vm_flags & VM_HUGETLB);
- }
- 
- #else
- 
-static inline bool is_vm_hugetlb_page(struct vm_area_struct *vma)
-+static inline bool is_vm_hugetlb_flags(vm_flags_t vm_flags)
- {
- 	return false;
- }
- 
- #endif
- 
-+static inline bool is_vm_hugetlb_page(struct vm_area_struct *vma)
-+{
-+	return is_vm_hugetlb_flags(vma->vm_flags);
-+}
-+
- #endif
-diff --git a/include/linux/hung_task.h b/include/linux/hung_task.h
-new file mode 100644
-index 000000000000..c4403eeb7144
--- /dev/null
-+++ b/include/linux/hung_task.h
-@@ -0,0 +1,101 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * Detect Hung Task: detecting tasks stuck in D state
-+ *
-+ * Copyright (C) 2025 Tongcheng Travel (www.ly.com)
-+ * Author: Lance Yang <mingzhe.yang@ly.com>
-+ */
-+#ifndef __LINUX_HUNG_TASK_H
-+#define __LINUX_HUNG_TASK_H
-+
-+#include <linux/bug.h>
-+#include <linux/sched.h>
-+#include <linux/compiler.h>
-+
-+/*
-+ * @blocker: Combines lock address and blocking type.
-+ *
-+ * Since lock pointers are at least 4-byte aligned(32-bit) or 8-byte
-+ * aligned(64-bit). This leaves the 2 least bits (LSBs) of the pointer
-+ * always zero. So we can use these bits to encode the specific blocking
-+ * type.
-+ *
-+ * Note that on architectures where this is not guaranteed, or for any
-+ * unaligned lock, this tracking mechanism is silently skipped for that
-+ * lock.
-+ *
-+ * Type encoding:
-+ * 00 - Blocked on mutex			(BLOCKER_TYPE_MUTEX)
-+ * 01 - Blocked on semaphore			(BLOCKER_TYPE_SEM)
-+ * 10 - Blocked on rw-semaphore as READER	(BLOCKER_TYPE_RWSEM_READER)
-+ * 11 - Blocked on rw-semaphore as WRITER	(BLOCKER_TYPE_RWSEM_WRITER)
-+ */
-+#define BLOCKER_TYPE_MUTEX		0x00UL
-+#define BLOCKER_TYPE_SEM		0x01UL
-+#define BLOCKER_TYPE_RWSEM_READER	0x02UL
-+#define BLOCKER_TYPE_RWSEM_WRITER	0x03UL
-+
-+#define BLOCKER_TYPE_MASK		0x03UL
-+
-+#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
-+static inline void hung_task_set_blocker(void *lock, unsigned long type)
-+{
-+	unsigned long lock_ptr = (unsigned long)lock;
-+
-+	WARN_ON_ONCE(!lock_ptr);
-+	WARN_ON_ONCE(READ_ONCE(current->blocker));
-+
-+	/*
-+	 * If the lock pointer matches the BLOCKER_TYPE_MASK, return
-+	 * without writing anything.
-+	 */
-+	if (lock_ptr & BLOCKER_TYPE_MASK)
-+		return;
-+
-+	WRITE_ONCE(current->blocker, lock_ptr | type);
-+}
-+
-+static inline void hung_task_clear_blocker(void)
-+{
-+	WRITE_ONCE(current->blocker, 0UL);
-+}
-+
-+/*
-+ * hung_task_get_blocker_type - Extracts blocker type from encoded blocker
-+ * address.
-+ *
-+ * @blocker: Blocker pointer with encoded type (via LSB bits)
-+ *
-+ * Returns: BLOCKER_TYPE_MUTEX, BLOCKER_TYPE_SEM, etc.
-+ */
-+static inline unsigned long hung_task_get_blocker_type(unsigned long blocker)
-+{
-+	WARN_ON_ONCE(!blocker);
-+
-+	return blocker & BLOCKER_TYPE_MASK;
-+}
-+
-+static inline void *hung_task_blocker_to_lock(unsigned long blocker)
-+{
-+	WARN_ON_ONCE(!blocker);
-+
-+	return (void *)(blocker & ~BLOCKER_TYPE_MASK);
-+}
-+#else
-+static inline void hung_task_set_blocker(void *lock, unsigned long type)
-+{
-+}
-+static inline void hung_task_clear_blocker(void)
-+{
-+}
-+static inline unsigned long hung_task_get_blocker_type(unsigned long blocker)
-+{
-+	return 0UL;
-+}
-+static inline void *hung_task_blocker_to_lock(unsigned long blocker)
-+{
-+	return NULL;
-+}
-+#endif
-+
-+#endif /* __LINUX_HUNG_TASK_H */
-diff --git a/include/linux/hw_bitfield.h b/include/linux/hw_bitfield.h
-new file mode 100644
-index 000000000000..df202e167ce4
--- /dev/null
-+++ b/include/linux/hw_bitfield.h
-@@ -0,0 +1,62 @@
-+/* SPDX-License-Identifier: GPL-2.0-or-later */
-+/*
-+ * Copyright (C) 2025, Collabora Ltd.
-+ */
-+
-+#ifndef _LINUX_HW_BITFIELD_H
-+#define _LINUX_HW_BITFIELD_H
-+
-+#include <linux/bitfield.h>
-+#include <linux/build_bug.h>
-+#include <linux/limits.h>
-+
-+/**
-+ * FIELD_PREP_WM16() - prepare a bitfield element with a mask in the upper half
-+ * @_mask: shifted mask defining the field's length and position
-+ * @_val:  value to put in the field
-+ *
-+ * FIELD_PREP_WM16() masks and shifts up the value, as well as bitwise ORs the
-+ * result with the mask shifted up by 16.
-+ *
-+ * This is useful for a common design of hardware registers where the upper
-+ * 16-bit half of a 32-bit register is used as a write-enable mask. In such a
-+ * register, a bit in the lower half is only updated if the corresponding bit
-+ * in the upper half is high.
-+ */
-+#define FIELD_PREP_WM16(_mask, _val)					     \
-+	({								     \
-+		typeof(_val) __val = _val;				     \
-+		typeof(_mask) __mask = _mask;				     \
-+		__BF_FIELD_CHECK(__mask, ((u16)0U), __val,		     \
-+				 "HWORD_UPDATE: ");			     \
-+		(((typeof(__mask))(__val) << __bf_shf(__mask)) & (__mask)) | \
-+		((__mask) << 16);					     \
-+	})
-+
-+/**
-+ * FIELD_PREP_WM16_CONST() - prepare a constant bitfield element with a mask in
-+ *                           the upper half
-+ * @_mask: shifted mask defining the field's length and position
-+ * @_val:  value to put in the field
-+ *
-+ * FIELD_PREP_WM16_CONST() masks and shifts up the value, as well as bitwise ORs
-+ * the result with the mask shifted up by 16.
-+ *
-+ * This is useful for a common design of hardware registers where the upper
-+ * 16-bit half of a 32-bit register is used as a write-enable mask. In such a
-+ * register, a bit in the lower half is only updated if the corresponding bit
-+ * in the upper half is high.
-+ *
-+ * Unlike FIELD_PREP_WM16(), this is a constant expression and can therefore
-+ * be used in initializers. Error checking is less comfortable for this
-+ * version.
-+ */
-+#define FIELD_PREP_WM16_CONST(_mask, _val)				 \
-+	(								 \
-+		FIELD_PREP_CONST(_mask, _val) |				 \
-+		(BUILD_BUG_ON_ZERO(const_true((u64)(_mask) > U16_MAX)) + \
-+		 ((_mask) << 16))					 \
-+	)
-+
-+
-+#endif /* _LINUX_HW_BITFIELD_H */
-diff --git a/include/linux/hw_breakpoint.h b/include/linux/hw_breakpoint.h
-index 78dd7035d1e5..db199d653dd1 100644
--- a/include/linux/hw_breakpoint.h
-+++ b/include/linux/hw_breakpoint.h
-@@ -7,6 +7,16 @@
- 
- #ifdef CONFIG_HAVE_HW_BREAKPOINT
- 
-+enum bp_type_idx {
-+	TYPE_INST	= 0,
-+#if defined(CONFIG_HAVE_MIXED_BREAKPOINTS_REGS)
-+	TYPE_DATA	= 0,
-+#else
-+	TYPE_DATA	= 1,
-+#endif
-+	TYPE_MAX
-+};
-+
- extern int __init init_hw_breakpoint(void);
- 
- static inline void hw_breakpoint_init(struct perf_event_attr *attr)
-@@ -74,15 +84,12 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,
- extern int register_perf_hw_breakpoint(struct perf_event *bp);
- extern void unregister_hw_breakpoint(struct perf_event *bp);
- extern void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events);
-+extern bool hw_breakpoint_is_used(void);
- 
- extern int dbg_reserve_bp_slot(struct perf_event *bp);
- extern int dbg_release_bp_slot(struct perf_event *bp);
- extern int reserve_bp_slot(struct perf_event *bp);
- extern void release_bp_slot(struct perf_event *bp);
-int hw_breakpoint_weight(struct perf_event *bp);
-int arch_reserve_bp_slot(struct perf_event *bp);
-void arch_release_bp_slot(struct perf_event *bp);
-void arch_unregister_hw_breakpoint(struct perf_event *bp);
- 
- extern void flush_ptrace_hw_breakpoint(struct task_struct *tsk);
- 
-@@ -121,6 +128,8 @@ register_perf_hw_breakpoint(struct perf_event *bp)	{ return -ENOSYS; }
- static inline void unregister_hw_breakpoint(struct perf_event *bp)	{ }
- static inline void
- unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events)	{ }
-+static inline bool hw_breakpoint_is_used(void)		{ return false; }
-+
- static inline int
- reserve_bp_slot(struct perf_event *bp)			{return -ENOSYS; }
- static inline void release_bp_slot(struct perf_event *bp) 		{ }
-diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h
-index aa1d4da03538..b424555753b1 100644
--- a/include/linux/hw_random.h
-+++ b/include/linux/hw_random.h
-@@ -13,9 +13,8 @@
- #define LINUX_HWRANDOM_H_
- 
- #include <linux/completion.h>
-#include <linux/types.h>
-#include <linux/list.h>
- #include <linux/kref.h>
-+#include <linux/types.h>
- 
- /**
-  * struct hwrng - Hardware Random Number Generator driver
-@@ -34,7 +33,7 @@
-  * @priv:		Private data, for use by the RNG driver.
-  * @quality:		Estimation of true entropy in RNG's bitstream
-  *			(in bits of entropy per 1024 bits of input;
- *			valid values: 1 to 1024, or 0 for unknown).
-+ *			valid values: 1 to 1024, or 0 for maximum).
-  */
- struct hwrng {
- 	const char *name;
-@@ -50,6 +49,7 @@ struct hwrng {
- 	struct list_head list;
- 	struct kref ref;
- 	struct completion cleanup_done;
-+	struct completion dying;
- };
- 
- struct device;
-@@ -61,4 +61,7 @@ extern int devm_hwrng_register(struct device *dev, struct hwrng *rng);
- extern void hwrng_unregister(struct hwrng *rng);
- extern void devm_hwrng_unregister(struct device *dve, struct hwrng *rng);
- 
-+extern long hwrng_msleep(struct hwrng *rng, unsigned int msecs);
-+extern long hwrng_yield(struct hwrng *rng);
-+
- #endif /* LINUX_HWRANDOM_H_ */
-diff --git a/include/linux/hwmon-sysfs.h b/include/linux/hwmon-sysfs.h
-index cb26d02f52f3..d896713359cd 100644
--- a/include/linux/hwmon-sysfs.h
-+++ b/include/linux/hwmon-sysfs.h
-@@ -8,6 +8,7 @@
- #define _LINUX_HWMON_SYSFS_H
- 
- #include <linux/device.h>
-+#include <linux/kstrtox.h>
- 
- struct sensor_device_attribute{
- 	struct device_attribute dev_attr;
-diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h
-index 14325f93c6b2..301a83afbd66 100644
--- a/include/linux/hwmon.h
-+++ b/include/linux/hwmon.h
-@@ -24,6 +24,7 @@ enum hwmon_sensor_types {
- 	hwmon_curr,
- 	hwmon_power,
- 	hwmon_energy,
-+	hwmon_energy64,
- 	hwmon_humidity,
- 	hwmon_fan,
- 	hwmon_pwm,
-@@ -44,6 +45,8 @@ enum hwmon_chip_attributes {
- 	hwmon_chip_in_samples,
- 	hwmon_chip_power_samples,
- 	hwmon_chip_temp_samples,
-+	hwmon_chip_beep_enable,
-+	hwmon_chip_pec,
- };
- 
- #define HWMON_C_TEMP_RESET_HISTORY	BIT(hwmon_chip_temp_reset_history)
-@@ -58,6 +61,8 @@ enum hwmon_chip_attributes {
- #define HWMON_C_IN_SAMPLES		BIT(hwmon_chip_in_samples)
- #define HWMON_C_POWER_SAMPLES		BIT(hwmon_chip_power_samples)
- #define HWMON_C_TEMP_SAMPLES		BIT(hwmon_chip_temp_samples)
-+#define HWMON_C_BEEP_ENABLE		BIT(hwmon_chip_beep_enable)
-+#define HWMON_C_PEC			BIT(hwmon_chip_pec)
- 
- enum hwmon_temp_attributes {
- 	hwmon_temp_enable,
-@@ -87,6 +92,7 @@ enum hwmon_temp_attributes {
- 	hwmon_temp_reset_history,
- 	hwmon_temp_rated_min,
- 	hwmon_temp_rated_max,
-+	hwmon_temp_beep,
- };
- 
- #define HWMON_T_ENABLE		BIT(hwmon_temp_enable)
-@@ -116,6 +122,7 @@ enum hwmon_temp_attributes {
- #define HWMON_T_RESET_HISTORY	BIT(hwmon_temp_reset_history)
- #define HWMON_T_RATED_MIN	BIT(hwmon_temp_rated_min)
- #define HWMON_T_RATED_MAX	BIT(hwmon_temp_rated_max)
-+#define HWMON_T_BEEP		BIT(hwmon_temp_beep)
- 
- enum hwmon_in_attributes {
- 	hwmon_in_enable,
-@@ -136,6 +143,8 @@ enum hwmon_in_attributes {
- 	hwmon_in_crit_alarm,
- 	hwmon_in_rated_min,
- 	hwmon_in_rated_max,
-+	hwmon_in_beep,
-+	hwmon_in_fault,
- };
- 
- #define HWMON_I_ENABLE		BIT(hwmon_in_enable)
-@@ -156,6 +165,8 @@ enum hwmon_in_attributes {
- #define HWMON_I_CRIT_ALARM	BIT(hwmon_in_crit_alarm)
- #define HWMON_I_RATED_MIN	BIT(hwmon_in_rated_min)
- #define HWMON_I_RATED_MAX	BIT(hwmon_in_rated_max)
-+#define HWMON_I_BEEP		BIT(hwmon_in_beep)
-+#define HWMON_I_FAULT		BIT(hwmon_in_fault)
- 
- enum hwmon_curr_attributes {
- 	hwmon_curr_enable,
-@@ -176,6 +187,7 @@ enum hwmon_curr_attributes {
- 	hwmon_curr_crit_alarm,
- 	hwmon_curr_rated_min,
- 	hwmon_curr_rated_max,
-+	hwmon_curr_beep,
- };
- 
- #define HWMON_C_ENABLE		BIT(hwmon_curr_enable)
-@@ -196,6 +208,7 @@ enum hwmon_curr_attributes {
- #define HWMON_C_CRIT_ALARM	BIT(hwmon_curr_crit_alarm)
- #define HWMON_C_RATED_MIN	BIT(hwmon_curr_rated_min)
- #define HWMON_C_RATED_MAX	BIT(hwmon_curr_rated_max)
-+#define HWMON_C_BEEP		BIT(hwmon_curr_beep)
- 
- enum hwmon_power_attributes {
- 	hwmon_power_enable,
-@@ -285,6 +298,8 @@ enum hwmon_humidity_attributes {
- 	hwmon_humidity_fault,
- 	hwmon_humidity_rated_min,
- 	hwmon_humidity_rated_max,
-+	hwmon_humidity_min_alarm,
-+	hwmon_humidity_max_alarm,
- };
- 
- #define HWMON_H_ENABLE			BIT(hwmon_humidity_enable)
-@@ -298,6 +313,8 @@ enum hwmon_humidity_attributes {
- #define HWMON_H_FAULT			BIT(hwmon_humidity_fault)
- #define HWMON_H_RATED_MIN		BIT(hwmon_humidity_rated_min)
- #define HWMON_H_RATED_MAX		BIT(hwmon_humidity_rated_max)
-+#define HWMON_H_MIN_ALARM		BIT(hwmon_humidity_min_alarm)
-+#define HWMON_H_MAX_ALARM		BIT(hwmon_humidity_max_alarm)
- 
- enum hwmon_fan_attributes {
- 	hwmon_fan_enable,
-@@ -312,6 +329,7 @@ enum hwmon_fan_attributes {
- 	hwmon_fan_min_alarm,
- 	hwmon_fan_max_alarm,
- 	hwmon_fan_fault,
-+	hwmon_fan_beep,
- };
- 
- #define HWMON_F_ENABLE			BIT(hwmon_fan_enable)
-@@ -326,6 +344,7 @@ enum hwmon_fan_attributes {
- #define HWMON_F_MIN_ALARM		BIT(hwmon_fan_min_alarm)
- #define HWMON_F_MAX_ALARM		BIT(hwmon_fan_max_alarm)
- #define HWMON_F_FAULT			BIT(hwmon_fan_fault)
-+#define HWMON_F_BEEP			BIT(hwmon_fan_beep)
- 
- enum hwmon_pwm_attributes {
- 	hwmon_pwm_input,
-@@ -350,7 +369,9 @@ enum hwmon_intrusion_attributes {
- 
- /**
-  * struct hwmon_ops - hwmon device operations
- * @is_visible: Callback to return attribute visibility. Mandatory.
-+ * @visible:	Static visibility. If non-zero, 'is_visible' is ignored.
-+ * @is_visible: Callback to return attribute visibility. Mandatory unless
-+ *		'visible' is non-zero.
-  *		Parameters are:
-  *		@const void *drvdata:
-  *			Pointer to driver-private data structure passed
-@@ -394,6 +415,7 @@ enum hwmon_intrusion_attributes {
-  *		The function returns 0 on success or a negative error number.
-  */
- struct hwmon_ops {
-+	umode_t visible;
- 	umode_t (*is_visible)(const void *drvdata, enum hwmon_sensor_types type,
- 			      u32 attr, int channel);
- 	int (*read)(struct device *dev, enum hwmon_sensor_types type,
-@@ -415,12 +437,12 @@ struct hwmon_channel_info {
- 	const u32 *config;
- };
- 
-#define HWMON_CHANNEL_INFO(stype, ...)	\
-	(&(struct hwmon_channel_info) {	\
-		.type = hwmon_##stype,	\
-		.config = (u32 []) {	\
-			__VA_ARGS__, 0	\
-		}			\
-+#define HWMON_CHANNEL_INFO(stype, ...)		\
-+	(&(const struct hwmon_channel_info) {	\
-+		.type = hwmon_##stype,		\
-+		.config = (const u32 []) {	\
-+			__VA_ARGS__, 0		\
-+		}				\
- 	})
- 
- /**
-@@ -430,12 +452,16 @@ struct hwmon_channel_info {
-  */
- struct hwmon_chip_info {
- 	const struct hwmon_ops *ops;
-	const struct hwmon_channel_info **info;
-+	const struct hwmon_channel_info * const *info;
- };
- 
- /* hwmon_device_register() is deprecated */
- struct device *hwmon_device_register(struct device *dev);
- 
-+/*
-+ * hwmon_device_register_with_groups() and
-+ * devm_hwmon_device_register_with_groups() are deprecated.
-+ */
- struct device *
- hwmon_device_register_with_groups(struct device *dev, const char *name,
- 				  void *drvdata,
-@@ -459,7 +485,6 @@ devm_hwmon_device_register_with_info(struct device *dev,
- 				const struct attribute_group **extra_groups);
- 
- void hwmon_device_unregister(struct device *dev);
-void devm_hwmon_device_unregister(struct device *dev);
- 
- int hwmon_notify_event(struct device *dev, enum hwmon_sensor_types type,
- 		       u32 attr, int channel);
-@@ -467,6 +492,9 @@ int hwmon_notify_event(struct device *dev, enum hwmon_sensor_types type,
- char *hwmon_sanitize_name(const char *name);
- char *devm_hwmon_sanitize_name(struct device *dev, const char *name);
- 
-+void hwmon_lock(struct device *dev);
-+void hwmon_unlock(struct device *dev);
-+
- /**
-  * hwmon_is_bad_char - Is the char invalid in a hwmon name
-  * @ch: the char to be considered
-diff --git a/include/linux/hwspinlock.h b/include/linux/hwspinlock.h
-index bfe7c1f1ac6d..f35b42e8c5de 100644
--- a/include/linux/hwspinlock.h
-+++ b/include/linux/hwspinlock.h
-@@ -58,18 +58,16 @@ struct hwspinlock_pdata {
- int hwspin_lock_register(struct hwspinlock_device *bank, struct device *dev,
- 		const struct hwspinlock_ops *ops, int base_id, int num_locks);
- int hwspin_lock_unregister(struct hwspinlock_device *bank);
-struct hwspinlock *hwspin_lock_request(void);
- struct hwspinlock *hwspin_lock_request_specific(unsigned int id);
- int hwspin_lock_free(struct hwspinlock *hwlock);
- int of_hwspin_lock_get_id(struct device_node *np, int index);
-int hwspin_lock_get_id(struct hwspinlock *hwlock);
- int __hwspin_lock_timeout(struct hwspinlock *, unsigned int, int,
- 							unsigned long *);
- int __hwspin_trylock(struct hwspinlock *, int, unsigned long *);
- void __hwspin_unlock(struct hwspinlock *, int, unsigned long *);
- int of_hwspin_lock_get_id_byname(struct device_node *np, const char *name);
-+int hwspin_lock_bust(struct hwspinlock *hwlock, unsigned int id);
- int devm_hwspin_lock_free(struct device *dev, struct hwspinlock *hwlock);
-struct hwspinlock *devm_hwspin_lock_request(struct device *dev);
- struct hwspinlock *devm_hwspin_lock_request_specific(struct device *dev,
- 						     unsigned int id);
- int devm_hwspin_lock_unregister(struct device *dev,
-@@ -94,11 +92,6 @@ int devm_hwspin_lock_register(struct device *dev,
-  * Note: ERR_PTR(-ENODEV) will still be considered a success for NULL-checking
-  * users. Others, which care, can still check this with IS_ERR.
-  */
-static inline struct hwspinlock *hwspin_lock_request(void)
-{
-	return ERR_PTR(-ENODEV);
-}
-
- static inline struct hwspinlock *hwspin_lock_request_specific(unsigned int id)
- {
- 	return ERR_PTR(-ENODEV);
-@@ -127,12 +120,12 @@ void __hwspin_unlock(struct hwspinlock *hwlock, int mode, unsigned long *flags)
- {
- }
- 
-static inline int of_hwspin_lock_get_id(struct device_node *np, int index)
-+static inline int hwspin_lock_bust(struct hwspinlock *hwlock, unsigned int id)
- {
- 	return 0;
- }
- 
-static inline int hwspin_lock_get_id(struct hwspinlock *hwlock)
-+static inline int of_hwspin_lock_get_id(struct device_node *np, int index)
- {
- 	return 0;
- }
-@@ -149,11 +142,6 @@ int devm_hwspin_lock_free(struct device *dev, struct hwspinlock *hwlock)
- 	return 0;
- }
- 
-static inline struct hwspinlock *devm_hwspin_lock_request(struct device *dev)
-{
-	return ERR_PTR(-ENODEV);
-}
-
- static inline
- struct hwspinlock *devm_hwspin_lock_request_specific(struct device *dev,
- 						     unsigned int id)
-diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h
-index 3b42264333ef..dfc516c1c719 100644
--- a/include/linux/hyperv.h
-+++ b/include/linux/hyperv.h
-@@ -24,7 +24,7 @@
- #include <linux/mod_devicetable.h>
- #include <linux/interrupt.h>
- #include <linux/reciprocal_div.h>
-#include <asm/hyperv-tlfs.h>
-+#include <hyperv/hvhdk.h>
- 
- #define MAX_PAGE_BUFFER_COUNT				32
- #define MAX_MULTIPAGE_BUFFER_COUNT			32 /* 128K */
-@@ -164,8 +164,28 @@ struct hv_ring_buffer {
- 	u8 buffer[];
- } __packed;
- 
-+
-+/*
-+ * If the requested ring buffer size is at least 8 times the size of the
-+ * header, steal space from the ring buffer for the header. Otherwise, add
-+ * space for the header so that is doesn't take too much of the ring buffer
-+ * space.
-+ *
-+ * The factor of 8 is somewhat arbitrary. The goal is to prevent adding a
-+ * relatively small header (4 Kbytes on x86) to a large-ish power-of-2 ring
-+ * buffer size (such as 128 Kbytes) and so end up making a nearly twice as
-+ * large allocation that will be almost half wasted. As a contrasting example,
-+ * on ARM64 with 64 Kbyte page size, we don't want to take 64 Kbytes for the
-+ * header from a 128 Kbyte allocation, leaving only 64 Kbytes for the ring.
-+ * In this latter case, we must add 64 Kbytes for the header and not worry
-+ * about what's wasted.
-+ */
-+#define VMBUS_HEADER_ADJ(payload_sz) \
-+	((payload_sz) >=  8 * sizeof(struct hv_ring_buffer) ? \
-+	0 : sizeof(struct hv_ring_buffer))
-+
- /* Calculate the proper size of a ringbuffer, it must be page-aligned */
-#define VMBUS_RING_SIZE(payload_sz) PAGE_ALIGN(sizeof(struct hv_ring_buffer) + \
-+#define VMBUS_RING_SIZE(payload_sz) PAGE_ALIGN(VMBUS_HEADER_ADJ(payload_sz) + \
- 					       (payload_sz))
- 
- struct hv_ring_buffer_info {
-@@ -245,16 +265,18 @@ static inline u32 hv_get_avail_to_write_percent(
-  * Linux kernel.
-  */
- 
-#define VERSION_WS2008  ((0 << 16) | (13))
-#define VERSION_WIN7    ((1 << 16) | (1))
-#define VERSION_WIN8    ((2 << 16) | (4))
-#define VERSION_WIN8_1    ((3 << 16) | (0))
-#define VERSION_WIN10 ((4 << 16) | (0))
-#define VERSION_WIN10_V4_1 ((4 << 16) | (1))
-#define VERSION_WIN10_V5 ((5 << 16) | (0))
-#define VERSION_WIN10_V5_1 ((5 << 16) | (1))
-#define VERSION_WIN10_V5_2 ((5 << 16) | (2))
-#define VERSION_WIN10_V5_3 ((5 << 16) | (3))
-+#define VMBUS_MAKE_VERSION(MAJ, MIN)	((((u32)MAJ) << 16) | (MIN))
-+#define VERSION_WS2008					VMBUS_MAKE_VERSION(0, 13)
-+#define VERSION_WIN7					VMBUS_MAKE_VERSION(1, 1)
-+#define VERSION_WIN8					VMBUS_MAKE_VERSION(2, 4)
-+#define VERSION_WIN8_1					VMBUS_MAKE_VERSION(3, 0)
-+#define VERSION_WIN10					VMBUS_MAKE_VERSION(4, 0)
-+#define VERSION_WIN10_V4_1				VMBUS_MAKE_VERSION(4, 1)
-+#define VERSION_WIN10_V5				VMBUS_MAKE_VERSION(5, 0)
-+#define VERSION_WIN10_V5_1				VMBUS_MAKE_VERSION(5, 1)
-+#define VERSION_WIN10_V5_2				VMBUS_MAKE_VERSION(5, 2)
-+#define VERSION_WIN10_V5_3				VMBUS_MAKE_VERSION(5, 3)
-+#define VERSION_WIN10_V6_0				VMBUS_MAKE_VERSION(6, 0)
- 
- /* Make maximum size of pipe payload of 16K */
- #define MAX_PIPE_DATA_PAYLOAD		(sizeof(u8) * 16384)
-@@ -315,14 +337,22 @@ struct vmbus_channel_offer {
- } __packed;
- 
- /* Server Flags */
-#define VMBUS_CHANNEL_ENUMERATE_DEVICE_INTERFACE	1
-#define VMBUS_CHANNEL_SERVER_SUPPORTS_TRANSFER_PAGES	2
-#define VMBUS_CHANNEL_SERVER_SUPPORTS_GPADLS		4
-#define VMBUS_CHANNEL_NAMED_PIPE_MODE			0x10
-#define VMBUS_CHANNEL_LOOPBACK_OFFER			0x100
-#define VMBUS_CHANNEL_PARENT_OFFER			0x200
-#define VMBUS_CHANNEL_REQUEST_MONITORED_NOTIFICATION	0x400
-#define VMBUS_CHANNEL_TLNPI_PROVIDER_OFFER		0x2000
-+#define VMBUS_CHANNEL_ENUMERATE_DEVICE_INTERFACE		0x0001
-+/*
-+ * This flag indicates that the channel is offered by the paravisor, and must
-+ * use encrypted memory for the channel ring buffer.
-+ */
-+#define VMBUS_CHANNEL_CONFIDENTIAL_RING_BUFFER			0x0002
-+/*
-+ * This flag indicates that the channel is offered by the paravisor, and must
-+ * use encrypted memory for GPA direct packets and additional GPADLs.
-+ */
-+#define VMBUS_CHANNEL_CONFIDENTIAL_EXTERNAL_MEMORY		0x0004
-+#define VMBUS_CHANNEL_NAMED_PIPE_MODE					0x0010
-+#define VMBUS_CHANNEL_LOOPBACK_OFFER					0x0100
-+#define VMBUS_CHANNEL_PARENT_OFFER						0x0200
-+#define VMBUS_CHANNEL_REQUEST_MONITORED_NOTIFICATION	0x0400
-+#define VMBUS_CHANNEL_TLNPI_PROVIDER_OFFER				0x2000
- 
- struct vmpacket_descriptor {
- 	u16 type;
-@@ -348,20 +378,7 @@ struct vmtransfer_page_packet_header {
- 	u8  sender_owns_set;
- 	u8 reserved;
- 	u32 range_cnt;
-	struct vmtransfer_page_range ranges[1];
-} __packed;
-
-struct vmgpadl_packet_header {
-	struct vmpacket_descriptor d;
-	u32 gpadl;
-	u32 reserved;
-} __packed;
-
-struct vmadd_remove_transfer_page_set {
-	struct vmpacket_descriptor d;
-	u32 gpadl;
-	u16 xfer_pageset_id;
-	u16 reserved;
-+	struct vmtransfer_page_range ranges[];
- } __packed;
- 
- /*
-@@ -375,30 +392,6 @@ struct gpa_range {
- };
- 
- /*
- * This is the format for an Establish Gpadl packet, which contains a handle by
- * which this GPADL will be known and a set of GPA ranges associated with it.
- * This can be converted to a MDL by the guest OS.  If there are multiple GPA
- * ranges, then the resulting MDL will be "chained," representing multiple VA
- * ranges.
- */
-struct vmestablish_gpadl {
-	struct vmpacket_descriptor d;
-	u32 gpadl;
-	u32 range_cnt;
-	struct gpa_range range[1];
-} __packed;
-
-/*
- * This is the format for a Teardown Gpadl packet, which indicates that the
- * GPADL handle in the Establish Gpadl packet will never be referenced again.
- */
-struct vmteardown_gpadl {
-	struct vmpacket_descriptor d;
-	u32 gpadl;
-	u32 reserved;	/* for alignment to a 8-byte boundary */
-} __packed;
-
-/*
-  * This is the format for a GPA-Direct packet, which contains a set of GPA
-  * ranges, in addition to commands and/or data.
-  */
-@@ -409,25 +402,6 @@ struct vmdata_gpa_direct {
- 	struct gpa_range range[1];
- } __packed;
- 
-/* This is the format for a Additional Data Packet. */
-struct vmadditional_data {
-	struct vmpacket_descriptor d;
-	u64 total_bytes;
-	u32 offset;
-	u32 byte_cnt;
-	unsigned char data[1];
-} __packed;
-
-union vmpacket_largest_possible_header {
-	struct vmpacket_descriptor simple_hdr;
-	struct vmtransfer_page_packet_header xfer_page_hdr;
-	struct vmgpadl_packet_header gpadl_hdr;
-	struct vmadd_remove_transfer_page_set add_rm_xfer_page_hdr;
-	struct vmestablish_gpadl establish_gpadl_hdr;
-	struct vmteardown_gpadl teardown_gpadl_hdr;
-	struct vmdata_gpa_direct data_gpa_direct_hdr;
-};
-
- #define VMPACKET_DATA_START_ADDRESS(__packet)	\
- 	(void *)(((unsigned char *)__packet) +	\
- 	 ((struct vmpacket_descriptor)__packet)->offset8 * 8)
-@@ -657,6 +631,12 @@ struct vmbus_channel_relid_released {
- 	u32 child_relid;
- } __packed;
- 
-+/*
-+ * Used by the paravisor only, means that the encrypted ring buffers and
-+ * the encrypted external memory are supported
-+ */
-+#define VMBUS_FEATURE_FLAG_CONFIDENTIAL_CHANNELS	0x10
-+
- struct vmbus_channel_initiate_contact {
- 	struct vmbus_channel_message_header header;
- 	u32 vmbus_version_requested;
-@@ -665,8 +645,9 @@ struct vmbus_channel_initiate_contact {
- 		u64 interrupt_page;
- 		struct {
- 			u8	msg_sint;
-			u8	padding1[3];
-			u32	padding2;
-+			u8	msg_vtl;
-+			u8	reserved[2];
-+			u32 feature_flags; /* VMBus version 6.0 */
- 		};
- 	};
- 	u64 monitor_page1;
-@@ -743,20 +724,6 @@ struct vmbus_channel_msginfo {
- 	unsigned char msg[];
- };
- 
-struct vmbus_close_msg {
-	struct vmbus_channel_msginfo info;
-	struct vmbus_channel_close_channel msg;
-};
-
-/* Define connection identifier type. */
-union hv_connection_id {
-	u32 asu32;
-	struct {
-		u32 id:24;
-		u32 reserved:8;
-	} u;
-};
-
- enum vmbus_device_type {
- 	HV_IDE = 0,
- 	HV_SCSI,
-@@ -800,6 +767,8 @@ struct vmbus_requestor {
- #define VMBUS_RQST_RESET (U64_MAX - 3)
- 
- struct vmbus_device {
-+	/* preferred ring buffer size in KB, 0 means no preferred size for this device */
-+	size_t pref_ring_size;
- 	u16  dev_type;
- 	guid_t guid;
- 	bool perf_device;
-@@ -812,6 +781,7 @@ struct vmbus_gpadl {
- 	u32 gpadl_handle;
- 	u32 size;
- 	void *buffer;
-+	bool decrypted;
- };
- 
- struct vmbus_channel {
-@@ -842,7 +812,7 @@ struct vmbus_channel {
- 	struct hv_ring_buffer_info outbound;	/* send to parent */
- 	struct hv_ring_buffer_info inbound;	/* receive from parent */
- 
-	struct vmbus_close_msg close_msg;
-+	struct vmbus_channel_close_channel close_msg;
- 
- 	/* Statistics */
- 	u64	interrupts;	/* Host to Guest interrupts */
-@@ -969,7 +939,7 @@ struct vmbus_channel {
- 	 * mechanism improves throughput by:
- 	 *
- 	 * A) Making the host more efficient - each time it wakes up,
-	 *    potentially it will process morev number of packets. The
-+	 *    potentially it will process more number of packets. The
- 	 *    monitor latency allows a batch to build up.
- 	 * B) By deferring the hypercall to signal, we will also minimize
- 	 *    the interrupts.
-@@ -1044,6 +1014,16 @@ struct vmbus_channel {
- 
- 	/* The max size of a packet on this channel */
- 	u32 max_pkt_size;
-+
-+	/* function to mmap ring buffer memory to the channel's sysfs ring attribute */
-+	int (*mmap_ring_buffer)(struct vmbus_channel *channel, struct vm_area_struct *vma);
-+
-+	/* boolean to control visibility of sysfs for ring buffer */
-+	bool ring_sysfs_visible;
-+	/* The ring buffer is encrypted */
-+	bool co_ring_buffer;
-+	/* The external memory is encrypted */
-+	bool co_external_memory;
- };
- 
- #define lock_requestor(channel, flags)					\
-@@ -1068,6 +1048,16 @@ u64 vmbus_request_addr_match(struct vmbus_channel *channel, u64 trans_id,
- 			     u64 rqst_addr);
- u64 vmbus_request_addr(struct vmbus_channel *channel, u64 trans_id);
- 
-+static inline bool is_co_ring_buffer(const struct vmbus_channel_offer_channel *o)
-+{
-+	return !!(o->offer.chn_flags & VMBUS_CHANNEL_CONFIDENTIAL_RING_BUFFER);
-+}
-+
-+static inline bool is_co_external_memory(const struct vmbus_channel_offer_channel *o)
-+{
-+	return !!(o->offer.chn_flags & VMBUS_CHANNEL_CONFIDENTIAL_EXTERNAL_MEMORY);
-+}
-+
- static inline bool is_hvsock_offer(const struct vmbus_channel_offer_channel *o)
- {
- 	return !!(o->offer.chn_flags & VMBUS_CHANNEL_TLNPI_PROVIDER_OFFER);
-@@ -1203,13 +1193,6 @@ extern int vmbus_sendpacket(struct vmbus_channel *channel,
- 				  enum vmbus_packet_type type,
- 				  u32 flags);
- 
-extern int vmbus_sendpacket_pagebuffer(struct vmbus_channel *channel,
-					    struct hv_page_buffer pagebuffers[],
-					    u32 pagecount,
-					    void *buffer,
-					    u32 bufferlen,
-					    u64 requestid);
-
- extern int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel,
- 				     struct vmbus_packet_mpb_array *mpb,
- 				     u32 desc_size,
-@@ -1239,9 +1222,6 @@ extern int vmbus_recvpacket_raw(struct vmbus_channel *channel,
- 				     u32 *buffer_actual_len,
- 				     u64 *requestid);
- 
-
-extern void vmbus_ontimer(unsigned long data);
-
- /* Base driver object */
- struct hv_driver {
- 	const char *name;
-@@ -1273,7 +1253,7 @@ struct hv_driver {
- 	} dynids;
- 
- 	int (*probe)(struct hv_device *, const struct hv_vmbus_device_id *);
-	int (*remove)(struct hv_device *);
-+	void (*remove)(struct hv_device *dev);
- 	void (*shutdown)(struct hv_device *);
- 
- 	int (*suspend)(struct hv_device *);
-@@ -1309,15 +1289,8 @@ struct hv_device {
- };
- 
- 
-static inline struct hv_device *device_to_hv_device(struct device *d)
-{
-	return container_of(d, struct hv_device, device);
-}
-
-static inline struct hv_driver *drv_to_hv_drv(struct device_driver *d)
-{
-	return container_of(d, struct hv_driver, driver);
-}
-+#define device_to_hv_device(d)	container_of_const(d, struct hv_device, device)
-+#define drv_to_hv_drv(d)	container_of_const(d, struct hv_driver, driver)
- 
- static inline void hv_set_drvdata(struct hv_device *dev, void *data)
- {
-@@ -1329,6 +1302,8 @@ static inline void *hv_get_drvdata(struct hv_device *dev)
- 	return dev_get_drvdata(&dev->device);
- }
- 
-+struct device *hv_get_vmbus_root_device(void);
-+
- struct hv_ring_buffer_debug_info {
- 	u32 current_interrupt_mask;
- 	u32 current_read_index;
-@@ -1341,6 +1316,8 @@ struct hv_ring_buffer_debug_info {
- int hv_ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info,
- 				struct hv_ring_buffer_debug_info *debug_info);
- 
-+bool hv_ringbuffer_spinlock_busy(struct vmbus_channel *channel);
-+
- /* Vmbus interface */
- #define vmbus_driver_register(driver)	\
- 	__vmbus_driver_register(driver, THIS_MODULE, KBUILD_MODNAME)
-@@ -1544,6 +1521,7 @@ struct hv_util_service {
- 	void *channel;
- 	void (*util_cb)(void *);
- 	int (*util_init)(struct hv_util_service *);
-+	int (*util_init_transport)(void);
- 	void (*util_deinit)(void);
- 	int (*util_pre_suspend)(void);
- 	int (*util_pre_resume)(void);
-@@ -1654,6 +1632,7 @@ int vmbus_send_tl_connect_request(const guid_t *shv_guest_servie_id,
- 				  const guid_t *shv_host_servie_id);
- int vmbus_send_modifychannel(struct vmbus_channel *channel, u32 target_vp);
- void vmbus_set_event(struct vmbus_channel *channel);
-+int vmbus_channel_set_cpu(struct vmbus_channel *channel, u32 target_cpu);
- 
- /* Get the start of the ring buffer. */
- static inline void *
-diff --git a/include/linux/hypervisor.h b/include/linux/hypervisor.h
-index 9efbc54e35e5..be5417303ecf 100644
--- a/include/linux/hypervisor.h
-+++ b/include/linux/hypervisor.h
-@@ -37,6 +37,9 @@ static inline bool hypervisor_isolated_pci_functions(void)
- 	if (IS_ENABLED(CONFIG_S390))
- 		return true;
- 
-+	if (IS_ENABLED(CONFIG_LOONGARCH))
-+		return true;
-+
- 	return jailhouse_paravirt();
- }
- 
-diff --git a/include/linux/i2c-algo-pca.h b/include/linux/i2c-algo-pca.h
-index 7c522fdd9ea7..e305bf32e40a 100644
--- a/include/linux/i2c-algo-pca.h
-+++ b/include/linux/i2c-algo-pca.h
-@@ -71,7 +71,7 @@ struct i2c_algo_pca_data {
- 	void 				*data;	/* private low level data */
- 	void (*write_byte)		(void *data, int reg, int val);
- 	int  (*read_byte)		(void *data, int reg);
-	int  (*wait_for_completion)	(void *data);
-+	int  (*wait_for_completion_cb)	(void *data);
- 	void (*reset_chip)		(void *data);
- 	/* For PCA9564, use one of the predefined frequencies:
- 	 * 330000, 288000, 217000, 146000, 88000, 59000, 44000, 36000
-diff --git a/include/linux/i2c-atr.h b/include/linux/i2c-atr.h
-new file mode 100644
-index 000000000000..2bb54dc87c8e
--- /dev/null
-+++ b/include/linux/i2c-atr.h
-@@ -0,0 +1,149 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/*
-+ * I2C Address Translator
-+ *
-+ * Copyright (c) 2019,2022 Luca Ceresoli <luca@lucaceresoli.net>
-+ * Copyright (c) 2022,2023 Tomi Valkeinen <tomi.valkeinen@ideasonboard.com>
-+ *
-+ * Based on i2c-mux.h
-+ */
-+
-+#ifndef _LINUX_I2C_ATR_H
-+#define _LINUX_I2C_ATR_H
-+
-+#include <linux/i2c.h>
-+#include <linux/types.h>
-+
-+struct device;
-+struct fwnode_handle;
-+struct i2c_atr;
-+
-+/**
-+ * enum i2c_atr_flags - Flags for an I2C ATR driver
-+ *
-+ * @I2C_ATR_F_STATIC: ATR does not support dynamic mapping, use static mapping.
-+ *                    Mappings will only be added or removed as a result of
-+ *                    devices being added or removed from a child bus.
-+ *                    The ATR pool will have to be big enough to accomodate all
-+ *                    devices expected to be added to the child buses.
-+ * @I2C_ATR_F_PASSTHROUGH: Allow unmapped incoming addresses to pass through
-+ */
-+enum i2c_atr_flags {
-+	I2C_ATR_F_STATIC = BIT(0),
-+	I2C_ATR_F_PASSTHROUGH = BIT(1),
-+};
-+
-+/**
-+ * struct i2c_atr_ops - Callbacks from ATR to the device driver.
-+ * @attach_addr: Notify the driver of a new device connected on a child
-+ *               bus, with the alias assigned to it. The driver must
-+ *               configure the hardware to use the alias.
-+ * @detach_addr: Notify the driver of a device getting disconnected. The
-+ *               driver must configure the hardware to stop using the
-+ *               alias.
-+ *
-+ * All these functions return 0 on success, a negative error code otherwise.
-+ */
-+struct i2c_atr_ops {
-+	int (*attach_addr)(struct i2c_atr *atr, u32 chan_id,
-+			   u16 addr, u16 alias);
-+	void (*detach_addr)(struct i2c_atr *atr, u32 chan_id,
-+			    u16 addr);
-+};
-+
-+/**
-+ * struct i2c_atr_adap_desc - An ATR downstream bus descriptor
-+ * @chan_id:        Index of the new adapter (0 .. max_adapters-1).  This value is
-+ *                  passed to the callbacks in `struct i2c_atr_ops`.
-+ * @parent:         The device used as the parent of the new i2c adapter, or NULL
-+ *                  to use the i2c-atr device as the parent.
-+ * @bus_handle:     The fwnode handle that points to the adapter's i2c
-+ *                  peripherals, or NULL.
-+ * @num_aliases:    The number of aliases in this adapter's private alias pool. Set
-+ *                  to zero if this adapter uses the ATR's global alias pool.
-+ * @aliases:        An optional array of private aliases used by the adapter
-+ *                  instead of the ATR's global pool of aliases. Must contain
-+ *                  exactly num_aliases entries if num_aliases > 0, is ignored
-+ *                  otherwise.
-+ */
-+struct i2c_atr_adap_desc {
-+	u32 chan_id;
-+	struct device *parent;
-+	struct fwnode_handle *bus_handle;
-+	size_t num_aliases;
-+	u16 *aliases;
-+};
-+
-+/**
-+ * i2c_atr_new() - Allocate and initialize an I2C ATR helper.
-+ * @parent:       The parent (upstream) adapter
-+ * @dev:          The device acting as an ATR
-+ * @ops:          Driver-specific callbacks
-+ * @max_adapters: Maximum number of child adapters
-+ * @flags:        Flags for ATR
-+ *
-+ * The new ATR helper is connected to the parent adapter but has no child
-+ * adapters. Call i2c_atr_add_adapter() to add some.
-+ *
-+ * Call i2c_atr_delete() to remove.
-+ *
-+ * Return: pointer to the new ATR helper object, or ERR_PTR
-+ */
-+struct i2c_atr *i2c_atr_new(struct i2c_adapter *parent, struct device *dev,
-+			    const struct i2c_atr_ops *ops, int max_adapters,
-+			    u32 flags);
-+
-+/**
-+ * i2c_atr_delete - Delete an I2C ATR helper.
-+ * @atr: I2C ATR helper to be deleted.
-+ *
-+ * Precondition: all the adapters added with i2c_atr_add_adapter() must be
-+ * removed by calling i2c_atr_del_adapter().
-+ */
-+void i2c_atr_delete(struct i2c_atr *atr);
-+
-+/**
-+ * i2c_atr_add_adapter - Create a child ("downstream") I2C bus.
-+ * @atr:        The I2C ATR
-+ * @desc:       An ATR adapter descriptor
-+ *
-+ * After calling this function a new i2c bus will appear. Adding and removing
-+ * devices on the downstream bus will result in calls to the
-+ * &i2c_atr_ops->attach_client and &i2c_atr_ops->detach_client callbacks for the
-+ * driver to assign an alias to the device.
-+ *
-+ * The adapter's fwnode is set to @bus_handle, or if @bus_handle is NULL the
-+ * function looks for a child node whose 'reg' property matches the chan_id
-+ * under the i2c-atr device's 'i2c-atr' node.
-+ *
-+ * Call i2c_atr_del_adapter() to remove the adapter.
-+ *
-+ * Return: 0 on success, a negative error code otherwise.
-+ */
-+int i2c_atr_add_adapter(struct i2c_atr *atr, struct i2c_atr_adap_desc *desc);
-+
-+/**
-+ * i2c_atr_del_adapter - Remove a child ("downstream") I2C bus added by
-+ *                       i2c_atr_add_adapter(). If no I2C bus has been added
-+ *                       this function is a no-op.
-+ * @atr:     The I2C ATR
-+ * @chan_id: Index of the adapter to be removed (0 .. max_adapters-1)
-+ */
-+void i2c_atr_del_adapter(struct i2c_atr *atr, u32 chan_id);
-+
-+/**
-+ * i2c_atr_set_driver_data - Set private driver data to the i2c-atr instance.
-+ * @atr:  The I2C ATR
-+ * @data: Pointer to the data to store
-+ */
-+void i2c_atr_set_driver_data(struct i2c_atr *atr, void *data);
-+
-+/**
-+ * i2c_atr_get_driver_data - Get the stored drive data.
-+ * @atr:     The I2C ATR
-+ *
-+ * Return: Pointer to the stored data
-+ */
-+void *i2c_atr_get_driver_data(struct i2c_atr *atr);
-+
-+#endif /* _LINUX_I2C_ATR_H */
-diff --git a/include/linux/i2c-mux.h b/include/linux/i2c-mux.h
-index 98ef73b7c8fd..1784ac7afb11 100644
--- a/include/linux/i2c-mux.h
-+++ b/include/linux/i2c-mux.h
-@@ -56,8 +56,7 @@ struct i2c_adapter *i2c_root_adapter(struct device *dev);
-  * callback functions to perform hardware-specific mux control.
-  */
- int i2c_mux_add_adapter(struct i2c_mux_core *muxc,
-			u32 force_nr, u32 chan_id,
-			unsigned int class);
-+			u32 force_nr, u32 chan_id);
- 
- void i2c_mux_del_adapters(struct i2c_mux_core *muxc);
- 
-diff --git a/include/linux/i2c-of-prober.h b/include/linux/i2c-of-prober.h
-new file mode 100644
-index 000000000000..bb6d47f50ee5
--- /dev/null
-+++ b/include/linux/i2c-of-prober.h
-@@ -0,0 +1,140 @@
-+/* SPDX-License-Identifier: GPL-2.0-or-later */
-+/*
-+ * Definitions for the Linux I2C OF component prober
-+ *
-+ * Copyright (C) 2024 Google LLC
-+ */
-+
-+#ifndef _LINUX_I2C_OF_PROBER_H
-+#define _LINUX_I2C_OF_PROBER_H
-+
-+#include <linux/kconfig.h>
-+#include <linux/types.h>
-+
-+struct device;
-+struct device_node;
-+
-+/**
-+ * struct i2c_of_probe_ops - I2C OF component prober callbacks
-+ *
-+ * A set of callbacks to be used by i2c_of_probe_component().
-+ *
-+ * All callbacks are optional. Callbacks are called only once per run, and are
-+ * used in the order they are defined in this structure.
-+ *
-+ * All callbacks that have return values shall return %0 on success,
-+ * or a negative error number on failure.
-+ *
-+ * The @dev parameter passed to the callbacks is the same as @dev passed to
-+ * i2c_of_probe_component(). It should only be used for dev_printk() calls
-+ * and nothing else, especially not managed device resource (devres) APIs.
-+ */
-+struct i2c_of_probe_ops {
-+	/**
-+	 * @enable: Retrieve and enable resources so that the components respond to probes.
-+	 *
-+	 * It is OK for this callback to return -EPROBE_DEFER since the intended use includes
-+	 * retrieving resources and enables them. Resources should be reverted to their initial
-+	 * state and released before returning if this fails.
-+	 */
-+	int (*enable)(struct device *dev, struct device_node *bus_node, void *data);
-+
-+	/**
-+	 * @cleanup_early: Release exclusive resources prior to calling probe() on a
-+	 *		   detected component.
-+	 *
-+	 * Only called if a matching component is actually found. If none are found,
-+	 * resources that would have been released in this callback should be released in
-+	 * @free_resourcs_late instead.
-+	 */
-+	void (*cleanup_early)(struct device *dev, void *data);
-+
-+	/**
-+	 * @cleanup: Opposite of @enable to balance refcounts and free resources after probing.
-+	 *
-+	 * Should check if resources were already freed by @cleanup_early.
-+	 */
-+	void (*cleanup)(struct device *dev, void *data);
-+};
-+
-+/**
-+ * struct i2c_of_probe_cfg - I2C OF component prober configuration
-+ * @ops: Callbacks for the prober to use.
-+ * @type: A string to match the device node name prefix to probe for.
-+ */
-+struct i2c_of_probe_cfg {
-+	const struct i2c_of_probe_ops *ops;
-+	const char *type;
-+};
-+
-+#if IS_ENABLED(CONFIG_OF_DYNAMIC)
-+
-+int i2c_of_probe_component(struct device *dev, const struct i2c_of_probe_cfg *cfg, void *ctx);
-+
-+/**
-+ * DOC: I2C OF component prober simple helpers
-+ *
-+ * Components such as trackpads are commonly connected to a devices baseboard
-+ * with a 6-pin ribbon cable. That gives at most one voltage supply and one
-+ * GPIO (commonly a "enable" or "reset" line) besides the I2C bus, interrupt
-+ * pin, and common ground. Touchscreens, while integrated into the display
-+ * panel's connection, typically have the same set of connections.
-+ *
-+ * A simple set of helpers are provided here for use with the I2C OF component
-+ * prober. This implementation targets such components, allowing for at most
-+ * one regulator supply.
-+ *
-+ * The following helpers are provided:
-+ * * i2c_of_probe_simple_enable()
-+ * * i2c_of_probe_simple_cleanup_early()
-+ * * i2c_of_probe_simple_cleanup()
-+ */
-+
-+/**
-+ * struct i2c_of_probe_simple_opts - Options for simple I2C component prober callbacks
-+ * @res_node_compatible: Compatible string of device node to retrieve resources from.
-+ * @supply_name: Name of regulator supply.
-+ * @gpio_name: Name of GPIO. NULL if no GPIO line is used. Empty string ("") if GPIO
-+ *	       line is unnamed.
-+ * @post_power_on_delay_ms: Delay after regulators are powered on. Passed to msleep().
-+ * @post_gpio_config_delay_ms: Delay after GPIO is configured. Passed to msleep().
-+ * @gpio_assert_to_enable: %true if GPIO should be asserted, i.e. set to logical high,
-+ *			   to enable the component.
-+ *
-+ * This describes power sequences common for the class of components supported by the
-+ * simple component prober:
-+ * * @gpio_name is configured to the non-active setting according to @gpio_assert_to_enable.
-+ * * @supply_name regulator supply is enabled.
-+ * * Wait for @post_power_on_delay_ms to pass.
-+ * * @gpio_name is configured to the active setting according to @gpio_assert_to_enable.
-+ * * Wait for @post_gpio_config_delay_ms to pass.
-+ */
-+struct i2c_of_probe_simple_opts {
-+	const char *res_node_compatible;
-+	const char *supply_name;
-+	const char *gpio_name;
-+	unsigned int post_power_on_delay_ms;
-+	unsigned int post_gpio_config_delay_ms;
-+	bool gpio_assert_to_enable;
-+};
-+
-+struct gpio_desc;
-+struct regulator;
-+
-+struct i2c_of_probe_simple_ctx {
-+	/* public: provided by user before helpers are used. */
-+	const struct i2c_of_probe_simple_opts *opts;
-+	/* private: internal fields for helpers. */
-+	struct regulator *supply;
-+	struct gpio_desc *gpiod;
-+};
-+
-+int i2c_of_probe_simple_enable(struct device *dev, struct device_node *bus_node, void *data);
-+void i2c_of_probe_simple_cleanup_early(struct device *dev, void *data);
-+void i2c_of_probe_simple_cleanup(struct device *dev, void *data);
-+
-+extern struct i2c_of_probe_ops i2c_of_probe_simple_ops;
-+
-+#endif /* IS_ENABLED(CONFIG_OF_DYNAMIC) */
-+
-+#endif /* _LINUX_I2C_OF_PROBER_H */
-diff --git a/include/linux/i2c-smbus.h b/include/linux/i2c-smbus.h
-index ced1c6ead52a..dc1bd2ab4c13 100644
--- a/include/linux/i2c-smbus.h
-+++ b/include/linux/i2c-smbus.h
-@@ -44,9 +44,11 @@ static inline void i2c_free_slave_host_notify_device(struct i2c_client *client)
- #endif
- 
- #if IS_ENABLED(CONFIG_I2C_SMBUS) && IS_ENABLED(CONFIG_DMI)
-void i2c_register_spd(struct i2c_adapter *adap);
-+void i2c_register_spd_write_disable(struct i2c_adapter *adap);
-+void i2c_register_spd_write_enable(struct i2c_adapter *adap);
- #else
-static inline void i2c_register_spd(struct i2c_adapter *adap) { }
-+static inline void i2c_register_spd_write_disable(struct i2c_adapter *adap) { }
-+static inline void i2c_register_spd_write_enable(struct i2c_adapter *adap) { }
- #endif
- 
- #endif /* _LINUX_I2C_SMBUS_H */
-diff --git a/include/linux/i2c.h b/include/linux/i2c.h
-index 8eab5017bff3..20fd41b51d5c 100644
--- a/include/linux/i2c.h
-+++ b/include/linux/i2c.h
-@@ -23,14 +23,13 @@
- #include <linux/swab.h>		/* for swab16 */
- #include <uapi/linux/i2c.h>
- 
-extern struct bus_type i2c_bus_type;
-extern struct device_type i2c_adapter_type;
-extern struct device_type i2c_client_type;
-+extern const struct bus_type i2c_bus_type;
-+extern const struct device_type i2c_adapter_type;
-+extern const struct device_type i2c_client_type;
- 
- /* --- General options ------------------------------------------------	*/
- 
- struct i2c_msg;
-struct i2c_algorithm;
- struct i2c_adapter;
- struct i2c_client;
- struct i2c_driver;
-@@ -189,6 +188,7 @@ s32 i2c_smbus_read_i2c_block_data_or_emulated(const struct i2c_client *client,
- 					      u8 *values);
- int i2c_get_device_id(const struct i2c_client *client,
- 		      struct i2c_device_identity *id);
-+const struct i2c_device_id *i2c_client_get_device_id(const struct i2c_client *client);
- #endif /* I2C */
- 
- /**
-@@ -235,8 +235,7 @@ enum i2c_driver_flags {
- /**
-  * struct i2c_driver - represent an I2C device driver
-  * @class: What kind of i2c device we instantiate (for detect)
- * @probe: Callback for device binding - soon to be deprecated
- * @probe_new: New callback for device binding
-+ * @probe: Callback for device binding
-  * @remove: Callback for device unbinding
-  * @shutdown: Callback for device shutdown
-  * @alert: Alert callback, for example for the SMBus alert protocol
-@@ -272,13 +271,9 @@ struct i2c_driver {
- 	unsigned int class;
- 
- 	/* Standard driver model interfaces */
-	int (*probe)(struct i2c_client *client, const struct i2c_device_id *id);
-	int (*remove)(struct i2c_client *client);
-+	int (*probe)(struct i2c_client *client);
-+	void (*remove)(struct i2c_client *client);
- 
-	/* New driver model interface to aid the seamless removal of the
-	 * current probe()'s, more commonly unused than used second parameter.
-	 */
-	int (*probe_new)(struct i2c_client *client);
- 
- 	/* driver model interfaces that don't relate to enumeration  */
- 	void (*shutdown)(struct i2c_client *client);
-@@ -308,7 +303,7 @@ struct i2c_driver {
- 
- 	u32 flags;
- };
-#define to_i2c_driver(d) container_of(d, struct i2c_driver, driver)
-+#define to_i2c_driver(d) container_of_const(d, struct i2c_driver, driver)
- 
- /**
-  * struct i2c_client - represent an I2C slave device
-@@ -326,6 +321,8 @@ struct i2c_driver {
-  *	calls it to pass on slave events to the slave driver.
-  * @devres_group_id: id of the devres group that will be created for resources
-  *	acquired when probing this device.
-+ * @debugfs: pointer to the debugfs subdirectory which the I2C core created
-+ *	for this client.
-  *
-  * An i2c_client identifies a single device (i.e. chip) connected to an
-  * i2c bus. The behaviour exposed to Linux is defined by the driver
-@@ -355,6 +352,7 @@ struct i2c_client {
- 	i2c_slave_cb_t slave_cb;	/* callback for slave mode	*/
- #endif
- 	void *devres_group_id;		/* ID of probe devres group	*/
-+	struct dentry *debugfs;		/* per-client debugfs dir	*/
- };
- #define to_i2c_client(d) container_of(d, struct i2c_client, dev)
- 
-@@ -362,6 +360,8 @@ struct i2c_adapter *i2c_verify_adapter(struct device *dev);
- const struct i2c_device_id *i2c_match_id(const struct i2c_device_id *id,
- 					 const struct i2c_client *client);
- 
-+const void *i2c_get_match_data(const struct i2c_client *client);
-+
- static inline struct i2c_client *kobj_to_i2c_client(struct kobject *kobj)
- {
- 	struct device * const dev = kobj_to_dev(kobj);
-@@ -380,7 +380,6 @@ static inline void i2c_set_clientdata(struct i2c_client *client, void *data)
- 
- /* I2C slave support */
- 
-#if IS_ENABLED(CONFIG_I2C_SLAVE)
- enum i2c_slave_event {
- 	I2C_SLAVE_READ_REQUESTED,
- 	I2C_SLAVE_WRITE_REQUESTED,
-@@ -391,9 +390,10 @@ enum i2c_slave_event {
- 
- int i2c_slave_register(struct i2c_client *client, i2c_slave_cb_t slave_cb);
- int i2c_slave_unregister(struct i2c_client *client);
-bool i2c_detect_slave_mode(struct device *dev);
- int i2c_slave_event(struct i2c_client *client,
- 		    enum i2c_slave_event event, u8 *val);
-+#if IS_ENABLED(CONFIG_I2C_SLAVE)
-+bool i2c_detect_slave_mode(struct device *dev);
- #else
- static inline bool i2c_detect_slave_mode(struct device *dev) { return false; }
- #endif
-@@ -405,7 +405,6 @@ static inline bool i2c_detect_slave_mode(struct device *dev) { return false; }
-  * @addr: stored in i2c_client.addr
-  * @dev_name: Overrides the default <busnr>-<addr> dev_name if set
-  * @platform_data: stored in i2c_client.dev.platform_data
- * @of_node: pointer to OpenFirmware device node
-  * @fwnode: device node supplied by the platform firmware
-  * @swnode: software node for the device
-  * @resources: resources associated with the device
-@@ -429,7 +428,6 @@ struct i2c_board_info {
- 	unsigned short	addr;
- 	const char	*dev_name;
- 	void		*platform_data;
-	struct device_node *of_node;
- 	struct fwnode_handle *fwnode;
- 	const struct software_node *swnode;
- 	const struct resource *resources;
-@@ -514,46 +512,54 @@ i2c_register_board_info(int busnum, struct i2c_board_info const *info,
- #endif /* I2C_BOARDINFO */
- 
- /**
- * struct i2c_algorithm - represent I2C transfer method
- * @master_xfer: Issue a set of i2c transactions to the given I2C adapter
- *   defined by the msgs array, with num messages available to transfer via
- *   the adapter specified by adap.
- * @master_xfer_atomic: same as @master_xfer. Yet, only using atomic context
- *   so e.g. PMICs can be accessed very late before shutdown. Optional.
- * @smbus_xfer: Issue smbus transactions to the given I2C adapter. If this
-+ * struct i2c_algorithm - represent I2C transfer methods
-+ * @xfer: Transfer a given number of messages defined by the msgs array via
-+ *   the specified adapter.
-+ * @xfer_atomic: Same as @xfer. Yet, only using atomic context so e.g. PMICs
-+ *   can be accessed very late before shutdown. Optional.
-+ * @smbus_xfer: Issue SMBus transactions to the given I2C adapter. If this
-  *   is not present, then the bus layer will try and convert the SMBus calls
-  *   into I2C transfers instead.
- * @smbus_xfer_atomic: same as @smbus_xfer. Yet, only using atomic context
-+ * @smbus_xfer_atomic: Same as @smbus_xfer. Yet, only using atomic context
-  *   so e.g. PMICs can be accessed very late before shutdown. Optional.
-  * @functionality: Return the flags that this algorithm/adapter pair supports
-  *   from the ``I2C_FUNC_*`` flags.
- * @reg_slave: Register given client to I2C slave mode of this adapter
- * @unreg_slave: Unregister given client from I2C slave mode of this adapter
-+ * @reg_target: Register given client to local target mode of this adapter
-+ * @unreg_target: Unregister given client from local target mode of this adapter
-+ *
-+ * @master_xfer: deprecated, use @xfer
-+ * @master_xfer_atomic: deprecated, use @xfer_atomic
-+ * @reg_slave: deprecated, use @reg_target
-+ * @unreg_slave: deprecated, use @unreg_target
-  *
- * The following structs are for those who like to implement new bus drivers:
-  * i2c_algorithm is the interface to a class of hardware solutions which can
-  * be addressed using the same bus algorithms - i.e. bit-banging or the PCF8584
-  * to name two of the most common.
-  *
- * The return codes from the ``master_xfer{_atomic}`` fields should indicate the
-+ * The return codes from the ``xfer{_atomic}`` fields should indicate the
-  * type of error code that occurred during the transfer, as documented in the
-  * Kernel Documentation file Documentation/i2c/fault-codes.rst. Otherwise, the
-  * number of messages executed should be returned.
-  */
- struct i2c_algorithm {
- 	/*
-	 * If an adapter algorithm can't do I2C-level access, set master_xfer
-+	 * If an adapter algorithm can't do I2C-level access, set xfer
- 	 * to NULL. If an adapter algorithm can do SMBus access, set
- 	 * smbus_xfer. If set to NULL, the SMBus protocol is simulated
- 	 * using common I2C messages.
-	 *
-	 * master_xfer should return the number of messages successfully
-	 * processed, or a negative value on error
- 	 */
-	int (*master_xfer)(struct i2c_adapter *adap, struct i2c_msg *msgs,
-			   int num);
-	int (*master_xfer_atomic)(struct i2c_adapter *adap,
-+	union {
-+		int (*xfer)(struct i2c_adapter *adap, struct i2c_msg *msgs,
-+			    int num);
-+		int (*master_xfer)(struct i2c_adapter *adap, struct i2c_msg *msgs,
-+				   int num);
-+	};
-+	union {
-+		int (*xfer_atomic)(struct i2c_adapter *adap,
- 				   struct i2c_msg *msgs, int num);
-+		int (*master_xfer_atomic)(struct i2c_adapter *adap,
-+					   struct i2c_msg *msgs, int num);
-+	};
- 	int (*smbus_xfer)(struct i2c_adapter *adap, u16 addr,
- 			  unsigned short flags, char read_write,
- 			  u8 command, int size, union i2c_smbus_data *data);
-@@ -565,8 +571,14 @@ struct i2c_algorithm {
- 	u32 (*functionality)(struct i2c_adapter *adap);
- 
- #if IS_ENABLED(CONFIG_I2C_SLAVE)
-	int (*reg_slave)(struct i2c_client *client);
-	int (*unreg_slave)(struct i2c_client *client);
-+	union {
-+		int (*reg_target)(struct i2c_client *client);
-+		int (*reg_slave)(struct i2c_client *client);
-+	};
-+	union {
-+		int (*unreg_target)(struct i2c_client *client);
-+		int (*unreg_slave)(struct i2c_client *client);
-+	};
- #endif
- };
- 
-@@ -748,6 +760,11 @@ struct i2c_adapter {
- 
- 	struct irq_domain *host_notify_domain;
- 	struct regulator *bus_regulator;
-+
-+	struct dentry *debugfs;
-+
-+	/* 7bit address space */
-+	DECLARE_BITMAP(addrs_in_instantiation, 1 << 7);
- };
- #define to_i2c_adapter(d) container_of(d, struct i2c_adapter, dev)
- 
-@@ -852,8 +869,6 @@ static inline void i2c_mark_adapter_resumed(struct i2c_adapter *adap)
- 
- /* i2c adapter classes (bitmask) */
- #define I2C_CLASS_HWMON		(1<<0)	/* lm_sensors, ... */
-#define I2C_CLASS_DDC		(1<<3)	/* DDC bus on graphics adapters */
-#define I2C_CLASS_SPD		(1<<7)	/* Memory modules */
- /* Warn users that the adapter doesn't support classes anymore */
- #define I2C_CLASS_DEPRECATED	(1<<8)
- 
-@@ -932,7 +947,22 @@ static inline int i2c_adapter_id(struct i2c_adapter *adap)
- 
- static inline u8 i2c_8bit_addr_from_msg(const struct i2c_msg *msg)
- {
-	return (msg->addr << 1) | (msg->flags & I2C_M_RD ? 1 : 0);
-+	return (msg->addr << 1) | (msg->flags & I2C_M_RD);
-+}
-+
-+/*
-+ * 10-bit address
-+ *   addr_1: 5'b11110 | addr[9:8] | (R/nW)
-+ *   addr_2: addr[7:0]
-+ */
-+static inline u8 i2c_10bit_addr_hi_from_msg(const struct i2c_msg *msg)
-+{
-+	return 0xf0 | ((msg->addr & GENMASK(9, 8)) >> 7) | (msg->flags & I2C_M_RD);
-+}
-+
-+static inline u8 i2c_10bit_addr_lo_from_msg(const struct i2c_msg *msg)
-+{
-+	return msg->addr & GENMASK(7, 0);
- }
- 
- u8 *i2c_get_dma_safe_msg_buf(struct i2c_msg *msg, unsigned int threshold);
-@@ -962,21 +992,55 @@ int i2c_handle_smbus_host_notify(struct i2c_adapter *adap, unsigned short addr);
- #define builtin_i2c_driver(__i2c_driver) \
- 	builtin_driver(__i2c_driver, i2c_add_driver)
- 
-#endif /* I2C */
-+/* must call put_device() when done with returned i2c_client device */
-+struct i2c_client *i2c_find_device_by_fwnode(struct fwnode_handle *fwnode);
-+
-+/* must call put_device() when done with returned i2c_adapter device */
-+struct i2c_adapter *i2c_find_adapter_by_fwnode(struct fwnode_handle *fwnode);
-+
-+/* must call i2c_put_adapter() when done with returned i2c_adapter device */
-+struct i2c_adapter *i2c_get_adapter_by_fwnode(struct fwnode_handle *fwnode);
-+
-+#else /* I2C */
-+
-+static inline struct i2c_client *
-+i2c_find_device_by_fwnode(struct fwnode_handle *fwnode)
-+{
-+	return NULL;
-+}
-+
-+static inline struct i2c_adapter *
-+i2c_find_adapter_by_fwnode(struct fwnode_handle *fwnode)
-+{
-+	return NULL;
-+}
-+
-+static inline struct i2c_adapter *
-+i2c_get_adapter_by_fwnode(struct fwnode_handle *fwnode)
-+{
-+	return NULL;
-+}
-+
-+#endif /* !I2C */
- 
- #if IS_ENABLED(CONFIG_OF)
- /* must call put_device() when done with returned i2c_client device */
-struct i2c_client *of_find_i2c_device_by_node(struct device_node *node);
-+static inline struct i2c_client *of_find_i2c_device_by_node(struct device_node *node)
-+{
-+	return i2c_find_device_by_fwnode(of_fwnode_handle(node));
-+}
- 
- /* must call put_device() when done with returned i2c_adapter device */
-struct i2c_adapter *of_find_i2c_adapter_by_node(struct device_node *node);
-+static inline struct i2c_adapter *of_find_i2c_adapter_by_node(struct device_node *node)
-+{
-+	return i2c_find_adapter_by_fwnode(of_fwnode_handle(node));
-+}
- 
- /* must call i2c_put_adapter() when done with returned i2c_adapter device */
-struct i2c_adapter *of_get_i2c_adapter_by_node(struct device_node *node);
-
-const struct of_device_id
-*i2c_of_match_device(const struct of_device_id *matches,
-		     struct i2c_client *client);
-+static inline struct i2c_adapter *of_get_i2c_adapter_by_node(struct device_node *node)
-+{
-+	return i2c_get_adapter_by_fwnode(of_fwnode_handle(node));
-+}
- 
- int of_i2c_get_board_info(struct device *dev, struct device_node *node,
- 			  struct i2c_board_info *info);
-@@ -998,13 +1062,6 @@ static inline struct i2c_adapter *of_get_i2c_adapter_by_node(struct device_node
- 	return NULL;
- }
- 
-static inline const struct of_device_id
-*i2c_of_match_device(const struct of_device_id *matches,
-		     struct i2c_client *client)
-{
-	return NULL;
-}
-
- static inline int of_i2c_get_board_info(struct device *dev,
- 					struct device_node *node,
- 					struct i2c_board_info *info)
-@@ -1017,7 +1074,7 @@ static inline int of_i2c_get_board_info(struct device *dev,
- struct acpi_resource;
- struct acpi_resource_i2c_serialbus;
- 
-#if IS_ENABLED(CONFIG_ACPI)
-+#if IS_REACHABLE(CONFIG_ACPI) && IS_REACHABLE(CONFIG_I2C)
- bool i2c_acpi_get_i2c_resource(struct acpi_resource *ares,
- 			       struct acpi_resource_i2c_serialbus **i2c);
- int i2c_acpi_client_count(struct acpi_device *adev);
-diff --git a/include/linux/i3c/device.h b/include/linux/i3c/device.h
-index 8242e13e7b0b..9fcb6410a584 100644
--- a/include/linux/i3c/device.h
-+++ b/include/linux/i3c/device.h
-@@ -18,17 +18,18 @@
- /**
-  * enum i3c_error_code - I3C error codes
-  *
- * These are the standard error codes as defined by the I3C specification.
- * When -EIO is returned by the i3c_device_do_priv_xfers() or
- * i3c_device_send_hdr_cmds() one can check the error code in
- * &struct_i3c_priv_xfer.err or &struct i3c_hdr_cmd.err to get a better idea of
- * what went wrong.
- *
-  * @I3C_ERROR_UNKNOWN: unknown error, usually means the error is not I3C
-  *		       related
-  * @I3C_ERROR_M0: M0 error
-  * @I3C_ERROR_M1: M1 error
-  * @I3C_ERROR_M2: M2 error
-+ *
-+ * These are the standard error codes as defined by the I3C specification.
-+ * When -EIO is returned by the i3c_device_do_priv_xfers() or
-+ * i3c_device_send_hdr_cmds() one can check the error code in
-+ * &struct_i3c_xfer.err or &struct i3c_hdr_cmd.err to get a better idea of
-+ * what went wrong.
-+ *
-  */
- enum i3c_error_code {
- 	I3C_ERROR_UNKNOWN = 0,
-@@ -38,29 +39,39 @@ enum i3c_error_code {
- };
- 
- /**
- * enum i3c_hdr_mode - HDR mode ids
-+ * enum i3c_xfer_mode - I3C xfer mode ids
-  * @I3C_HDR_DDR: DDR mode
-  * @I3C_HDR_TSP: TSP mode
-  * @I3C_HDR_TSL: TSL mode
-+ * @I3C_SDR: SDR mode (NOT HDR mode)
-  */
-enum i3c_hdr_mode {
-	I3C_HDR_DDR,
-	I3C_HDR_TSP,
-	I3C_HDR_TSL,
-+enum i3c_xfer_mode {
-+	/* The below 3 value (I3C_HDR*) must match GETCAP1 Byte bit position */
-+	I3C_HDR_DDR = 0,
-+	I3C_HDR_TSP = 1,
-+	I3C_HDR_TSL = 2,
-+	/* Use for default SDR transfer mode */
-+	I3C_SDR = 31,
- };
- 
- /**
- * struct i3c_priv_xfer - I3C SDR private transfer
-+ * struct i3c_xfer - I3C data transfer
-  * @rnw: encodes the transfer direction. true for a read, false for a write
-+ * @cmd: Read/Write command in HDR mode, read: 0x80 - 0xff, write: 0x00 - 0x7f
-  * @len: transfer length in bytes of the transfer
-+ * @actual_len: actual length in bytes are transferred by the controller
-  * @data: input/output buffer
-  * @data.in: input buffer. Must point to a DMA-able buffer
-  * @data.out: output buffer. Must point to a DMA-able buffer
-  * @err: I3C error code
-  */
-struct i3c_priv_xfer {
-	u8 rnw;
-+struct i3c_xfer {
-+	union {
-+		u8 rnw;
-+		u8 cmd;
-+	};
- 	u16 len;
-+	u16 actual_len;
- 	union {
- 		void *in;
- 		const void *out;
-@@ -68,6 +79,9 @@ struct i3c_priv_xfer {
- 	enum i3c_error_code err;
- };
- 
-+/* keep back compatible */
-+#define i3c_priv_xfer i3c_xfer
-+
- /**
-  * enum i3c_dcr - I3C DCR values
-  * @I3C_DCR_GENERIC_DEVICE: generic I3C device
-@@ -95,7 +109,7 @@ enum i3c_dcr {
- 
- /**
-  * struct i3c_device_info - I3C device information
- * @pid: Provisional ID
-+ * @pid: Provisioned ID
-  * @bcr: Bus Characteristic Register
-  * @dcr: Device Characteristic Register
-  * @static_addr: static/I2C address
-@@ -180,13 +194,17 @@ struct i3c_driver {
- 	const struct i3c_device_id *id_table;
- };
- 
-static inline struct i3c_driver *drv_to_i3cdrv(struct device_driver *drv)
-{
-	return container_of(drv, struct i3c_driver, driver);
-}
-+#define drv_to_i3cdrv(__drv)	container_of_const(__drv, struct i3c_driver, driver)
- 
- struct device *i3cdev_to_dev(struct i3c_device *i3cdev);
-struct i3c_device *dev_to_i3cdev(struct device *dev);
-+
-+/**
-+ * dev_to_i3cdev() - Returns the I3C device containing @dev
-+ * @__dev: device object
-+ *
-+ * Return: a pointer to an I3C device object.
-+ */
-+#define dev_to_i3cdev(__dev)	container_of_const(__dev, struct i3c_device, dev)
- 
- const struct i3c_device_id *
- i3c_device_match_id(struct i3c_device *i3cdev,
-@@ -238,7 +256,7 @@ void i3c_driver_unregister(struct i3c_driver *drv);
-  *
-  * Return: 0 if both registrations succeeds, a negative error code otherwise.
-  */
-static inline int i3c_i2c_driver_register(struct i3c_driver *i3cdrv,
-+static __always_inline int i3c_i2c_driver_register(struct i3c_driver *i3cdrv,
- 					  struct i2c_driver *i2cdrv)
- {
- 	int ret;
-@@ -263,7 +281,7 @@ static inline int i3c_i2c_driver_register(struct i3c_driver *i3cdrv,
-  * Note that when CONFIG_I3C is not enabled, this function only unregisters the
-  * @i2cdrv.
-  */
-static inline void i3c_i2c_driver_unregister(struct i3c_driver *i3cdrv,
-+static __always_inline void i3c_i2c_driver_unregister(struct i3c_driver *i3cdrv,
- 					     struct i2c_driver *i2cdrv)
- {
- 	if (IS_ENABLED(CONFIG_I3C))
-@@ -276,7 +294,7 @@ static inline void i3c_i2c_driver_unregister(struct i3c_driver *i3cdrv,
-  * module_i3c_i2c_driver() - Register a module providing an I3C and an I2C
-  *			     driver
-  * @__i3cdrv: the I3C driver to register
- * @__i2cdrv: the I3C driver to register
-+ * @__i2cdrv: the I2C driver to register
-  *
-  * Provide generic init/exit functions that simply register/unregister an I3C
-  * and an I2C driver.
-@@ -287,13 +305,22 @@ static inline void i3c_i2c_driver_unregister(struct i3c_driver *i3cdrv,
- #define module_i3c_i2c_driver(__i3cdrv, __i2cdrv)	\
- 	module_driver(__i3cdrv,				\
- 		      i3c_i2c_driver_register,		\
-		      i3c_i2c_driver_unregister)
-+		      i3c_i2c_driver_unregister,	\
-+		      __i2cdrv)
-+
-+int i3c_device_do_xfers(struct i3c_device *dev, struct i3c_xfer *xfers,
-+			int nxfers, enum i3c_xfer_mode mode);
-+
-+static inline int i3c_device_do_priv_xfers(struct i3c_device *dev,
-+					   struct i3c_xfer *xfers,
-+					   int nxfers)
-+{
-+	return i3c_device_do_xfers(dev, xfers, nxfers, I3C_SDR);
-+}
- 
-int i3c_device_do_priv_xfers(struct i3c_device *dev,
-			     struct i3c_priv_xfer *xfers,
-			     int nxfers);
-+int i3c_device_do_setdasa(struct i3c_device *dev);
- 
-void i3c_device_get_info(struct i3c_device *dev, struct i3c_device_info *info);
-+void i3c_device_get_info(const struct i3c_device *dev, struct i3c_device_info *info);
- 
- struct i3c_ibi_payload {
- 	unsigned int len;
-@@ -331,5 +358,6 @@ int i3c_device_request_ibi(struct i3c_device *dev,
- void i3c_device_free_ibi(struct i3c_device *dev);
- int i3c_device_enable_ibi(struct i3c_device *dev);
- int i3c_device_disable_ibi(struct i3c_device *dev);
-+u32 i3c_device_get_supported_xfer_mode(struct i3c_device *dev);
- 
- #endif /* I3C_DEV_H */
-diff --git a/include/linux/i3c/master.h b/include/linux/i3c/master.h
-index 604a126b78c8..2fd850f4678b 100644
--- a/include/linux/i3c/master.h
-+++ b/include/linux/i3c/master.h
-@@ -22,10 +22,18 @@
- #define I3C_BROADCAST_ADDR		0x7e
- #define I3C_MAX_ADDR			GENMASK(6, 0)
- 
-+struct i2c_client;
-+
-+/* notifier actions. notifier call data is the struct i3c_bus */
-+enum {
-+	I3C_NOTIFY_BUS_ADD,
-+	I3C_NOTIFY_BUS_REMOVE,
-+};
-+
- struct i3c_master_controller;
- struct i3c_bus;
-struct i2c_device;
- struct i3c_device;
-+extern const struct bus_type i3c_bus_type;
- 
- /**
-  * struct i3c_i2c_dev_desc - Common part of the I3C/I2C device descriptor
-@@ -69,7 +77,6 @@ struct i2c_dev_boardinfo {
- /**
-  * struct i2c_dev_desc - I2C device descriptor
-  * @common: common part of the I2C device descriptor
- * @boardinfo: pointer to the boardinfo attached to this I2C device
-  * @dev: I2C device object registered to the I2C framework
-  * @addr: I2C device address
-  * @lvr: LVR (Legacy Virtual Register) needed by the I3C core to know about
-@@ -128,6 +135,7 @@ struct i3c_ibi_slot {
-  *		     rejected by the master
-  * @num_slots: number of IBI slots reserved for this device
-  * @enabled: reflect the IBI status
-+ * @wq: workqueue used to execute IBI handlers.
-  * @handler: IBI handler specified at i3c_device_request_ibi() call time. This
-  *	     handler will be called from the controller workqueue, and as such
-  *	     is allowed to sleep (though it is recommended to process the IBI
-@@ -150,6 +158,7 @@ struct i3c_device_ibi_info {
- 	unsigned int max_payload_len;
- 	unsigned int num_slots;
- 	unsigned int enabled;
-+	struct workqueue_struct *wq;
- 	void (*handler)(struct i3c_device *dev,
- 			const struct i3c_ibi_payload *payload);
- };
-@@ -165,7 +174,7 @@ struct i3c_device_ibi_info {
-  *		 assigned a dynamic address by the master. Will be used during
-  *		 bus initialization to assign it a specific dynamic address
-  *		 before starting DAA (Dynamic Address Assignment)
- * @pid: I3C Provisional ID exposed by the device. This is a unique identifier
-+ * @pid: I3C Provisioned ID exposed by the device. This is a unique identifier
-  *	 that may be used to attach boardinfo to i3c_dev_desc when the device
-  *	 does not have a static address
-  * @of_node: optional DT node in case the device has been described in the DT
-@@ -240,10 +249,15 @@ struct i3c_device {
-  */
- #define I3C_BUS_MAX_DEVS		11
- 
-#define I3C_BUS_MAX_I3C_SCL_RATE	12900000
-#define I3C_BUS_TYP_I3C_SCL_RATE	12500000
-#define I3C_BUS_I2C_FM_PLUS_SCL_RATE	1000000
-#define I3C_BUS_I2C_FM_SCL_RATE		400000
-+/* Taken from the I3C Spec V1.1.1, chapter 6.2. "Timing specification" */
-+#define I3C_BUS_I2C_FM_PLUS_SCL_MAX_RATE	1000000
-+#define I3C_BUS_I2C_FM_SCL_MAX_RATE		400000
-+#define I3C_BUS_I3C_SCL_MAX_RATE	12900000
-+#define I3C_BUS_I3C_SCL_TYP_RATE	12500000
-+#define I3C_BUS_TAVAL_MIN_NS		1000
-+#define I3C_BUS_TBUF_MIXED_FM_MIN_NS	1300
-+#define I3C_BUS_THIGH_MIXED_MAX_NS	41
-+#define I3C_BUS_TIDLE_MIN_NS		200000
- #define I3C_BUS_TLOW_OD_MIN_NS		200
- 
- /**
-@@ -269,13 +283,29 @@ enum i3c_bus_mode {
- };
- 
- /**
-+ * enum i3c_open_drain_speed - I3C open-drain speed
-+ * @I3C_OPEN_DRAIN_SLOW_SPEED: Slow open-drain speed for sending the first
-+ *				broadcast address. The first broadcast address at this speed
-+ *				will be visible to all devices on the I3C bus. I3C devices
-+ *				working in I2C mode will turn off their spike filter when
-+ *				switching into I3C mode.
-+ * @I3C_OPEN_DRAIN_NORMAL_SPEED: Normal open-drain speed in I3C bus mode.
-+ */
-+enum i3c_open_drain_speed {
-+	I3C_OPEN_DRAIN_SLOW_SPEED,
-+	I3C_OPEN_DRAIN_NORMAL_SPEED,
-+};
-+
-+/**
-  * enum i3c_addr_slot_status - I3C address slot status
-  * @I3C_ADDR_SLOT_FREE: address is free
-  * @I3C_ADDR_SLOT_RSVD: address is reserved
-  * @I3C_ADDR_SLOT_I2C_DEV: address is assigned to an I2C device
-  * @I3C_ADDR_SLOT_I3C_DEV: address is assigned to an I3C device
-  * @I3C_ADDR_SLOT_STATUS_MASK: address slot mask
- *
-+ * @I3C_ADDR_SLOT_EXT_STATUS_MASK: address slot mask with extended information
-+ * @I3C_ADDR_SLOT_EXT_DESIRED: the bitmask represents addresses that are preferred by some devices,
-+ *			       such as the "assigned-address" property in a device tree source.
-  * On an I3C bus, addresses are assigned dynamically, and we need to know which
-  * addresses are free to use and which ones are already assigned.
-  *
-@@ -288,8 +318,12 @@ enum i3c_addr_slot_status {
- 	I3C_ADDR_SLOT_I2C_DEV,
- 	I3C_ADDR_SLOT_I3C_DEV,
- 	I3C_ADDR_SLOT_STATUS_MASK = 3,
-+	I3C_ADDR_SLOT_EXT_STATUS_MASK = 7,
-+	I3C_ADDR_SLOT_EXT_DESIRED = BIT(2),
- };
- 
-+#define I3C_ADDR_SLOT_STATUS_BITS 4
-+
- /**
-  * struct i3c_bus - I3C bus object
-  * @cur_master: I3C master currently driving the bus. Since I3C is multi-master
-@@ -331,7 +365,7 @@ enum i3c_addr_slot_status {
- struct i3c_bus {
- 	struct i3c_dev_desc *cur_master;
- 	int id;
-	unsigned long addrslots[((I2C_MAX_ADDR + 1) * 2) / BITS_PER_LONG];
-+	unsigned long addrslots[((I2C_MAX_ADDR + 1) * I3C_ADDR_SLOT_STATUS_BITS) / BITS_PER_LONG];
- 	enum i3c_bus_mode mode;
- 	struct {
- 		unsigned long i3c;
-@@ -384,7 +418,11 @@ struct i3c_bus {
-  * @send_ccc_cmd: send a CCC command
-  *		  This method is mandatory.
-  * @priv_xfers: do one or several private I3C SDR transfers
- *		This method is mandatory.
-+ *		This method is mandatory when i3c_xfers is not implemented. It
-+ *		is deprecated.
-+ * @i3c_xfers: do one or several I3C SDR or HDR transfers
-+ *	       This method is mandatory when priv_xfers is not implemented but
-+ *	       should be implemented instead of priv_xfers.
-  * @attach_i2c_dev: called every time an I2C device is attached to the bus.
-  *		    This is a good place to attach master controller specific
-  *		    data to I2C devices.
-@@ -425,6 +463,9 @@ struct i3c_bus {
-  *		      for a future IBI
-  *		      This method is mandatory only if ->request_ibi is not
-  *		      NULL.
-+ * @enable_hotjoin: enable hot join event detect.
-+ * @disable_hotjoin: disable hot join event detect.
-+ * @set_speed: adjust I3C open drain mode timing.
-  */
- struct i3c_master_controller_ops {
- 	int (*bus_init)(struct i3c_master_controller *master);
-@@ -437,13 +478,17 @@ struct i3c_master_controller_ops {
- 				 const struct i3c_ccc_cmd *cmd);
- 	int (*send_ccc_cmd)(struct i3c_master_controller *master,
- 			    struct i3c_ccc_cmd *cmd);
-+	/* Deprecated, please use i3c_xfers() */
- 	int (*priv_xfers)(struct i3c_dev_desc *dev,
- 			  struct i3c_priv_xfer *xfers,
- 			  int nxfers);
-+	int (*i3c_xfers)(struct i3c_dev_desc *dev,
-+			 struct i3c_xfer *xfers,
-+			 int nxfers, enum i3c_xfer_mode mode);
- 	int (*attach_i2c_dev)(struct i2c_dev_desc *dev);
- 	void (*detach_i2c_dev)(struct i2c_dev_desc *dev);
- 	int (*i2c_xfers)(struct i2c_dev_desc *dev,
-			 const struct i2c_msg *xfers, int nxfers);
-+			 struct i2c_msg *xfers, int nxfers);
- 	int (*request_ibi)(struct i3c_dev_desc *dev,
- 			   const struct i3c_ibi_setup *req);
- 	void (*free_ibi)(struct i3c_dev_desc *dev);
-@@ -451,6 +496,9 @@ struct i3c_master_controller_ops {
- 	int (*disable_ibi)(struct i3c_dev_desc *dev);
- 	void (*recycle_ibi_slot)(struct i3c_dev_desc *dev,
- 				 struct i3c_ibi_slot *slot);
-+	int (*enable_hotjoin)(struct i3c_master_controller *master);
-+	int (*disable_hotjoin)(struct i3c_master_controller *master);
-+	int (*set_speed)(struct i3c_master_controller *master, enum i3c_open_drain_speed speed);
- };
- 
- /**
-@@ -464,11 +512,12 @@ struct i3c_master_controller_ops {
-  * @ops: master operations. See &struct i3c_master_controller_ops
-  * @secondary: true if the master is a secondary master
-  * @init_done: true when the bus initialization is done
-+ * @hotjoin: true if the master support hotjoin
-  * @boardinfo.i3c: list of I3C  boardinfo objects
-  * @boardinfo.i2c: list of I2C boardinfo objects
-  * @boardinfo: board-level information attached to devices connected on the bus
-  * @bus: I3C bus exposed by this master
- * @wq: workqueue used to execute IBI handlers. Can also be used by master
-+ * @wq: workqueue which can be used by master
-  *	drivers if they need to postpone operations that need to take place
-  *	in a thread context. Typical examples are Hot Join processing which
-  *	requires taking the bus lock in maintenance, which in turn, can only
-@@ -486,6 +535,7 @@ struct i3c_master_controller {
- 	const struct i3c_master_controller_ops *ops;
- 	unsigned int secondary : 1;
- 	unsigned int init_done : 1;
-+	unsigned int hotjoin: 1;
- 	struct {
- 		struct list_head i3c;
- 		struct list_head i2c;
-@@ -516,6 +566,26 @@ struct i3c_master_controller {
- #define i3c_bus_for_each_i3cdev(bus, dev)				\
- 	list_for_each_entry(dev, &(bus)->devs.i3c, common.node)
- 
-+/**
-+ * struct i3c_dma - DMA transfer and mapping descriptor
-+ * @dev: device object of a device doing DMA
-+ * @buf: destination/source buffer for DMA
-+ * @len: length of transfer
-+ * @map_len: length of DMA mapping
-+ * @addr: mapped DMA address for a Host Controller Driver
-+ * @dir: DMA direction
-+ * @bounce_buf: an allocated bounce buffer if transfer needs it or NULL
-+ */
-+struct i3c_dma {
-+	struct device *dev;
-+	void *buf;
-+	size_t len;
-+	size_t map_len;
-+	dma_addr_t addr;
-+	enum dma_data_direction dir;
-+	void *bounce_buf;
-+};
-+
- int i3c_master_do_i2c_xfers(struct i3c_master_controller *master,
- 			    const struct i2c_msg *xfers,
- 			    int nxfers);
-@@ -533,6 +603,12 @@ int i3c_master_get_free_addr(struct i3c_master_controller *master,
- int i3c_master_add_i3c_dev_locked(struct i3c_master_controller *master,
- 				  u8 addr);
- int i3c_master_do_daa(struct i3c_master_controller *master);
-+struct i3c_dma *i3c_master_dma_map_single(struct device *dev, void *ptr,
-+					  size_t len, bool force_bounce,
-+					  enum dma_data_direction dir);
-+void i3c_master_dma_unmap_single(struct i3c_dma *dma_xfer);
-+DEFINE_FREE(i3c_master_dma_unmap_single, void *,
-+	    if (_T) i3c_master_dma_unmap_single(_T))
- 
- int i3c_master_set_info(struct i3c_master_controller *master,
- 			const struct i3c_device_info *info);
-@@ -541,7 +617,9 @@ int i3c_master_register(struct i3c_master_controller *master,
- 			struct device *parent,
- 			const struct i3c_master_controller_ops *ops,
- 			bool secondary);
-int i3c_master_unregister(struct i3c_master_controller *master);
-+void i3c_master_unregister(struct i3c_master_controller *master);
-+int i3c_master_enable_hotjoin(struct i3c_master_controller *master);
-+int i3c_master_disable_hotjoin(struct i3c_master_controller *master);
- 
- /**
-  * i3c_dev_get_master_data() - get master private data attached to an I3C
-@@ -651,4 +729,9 @@ void i3c_master_queue_ibi(struct i3c_dev_desc *dev, struct i3c_ibi_slot *slot);
- 
- struct i3c_ibi_slot *i3c_master_get_free_ibi_slot(struct i3c_dev_desc *dev);
- 
-+void i3c_for_each_bus_locked(int (*fn)(struct i3c_bus *bus, void *data),
-+			     void *data);
-+int i3c_register_notifier(struct notifier_block *nb);
-+int i3c_unregister_notifier(struct notifier_block *nb);
-+
- #endif /* I3C_MASTER_H */
-diff --git a/include/linux/i8042.h b/include/linux/i8042.h
-index 0261e2fb3636..00037c13abc8 100644
--- a/include/linux/i8042.h
-+++ b/include/linux/i8042.h
-@@ -3,6 +3,7 @@
- #define _LINUX_I8042_H
- 
- 
-+#include <linux/errno.h>
- #include <linux/types.h>
- 
- /*
-@@ -53,15 +54,29 @@
- 
- struct serio;
- 
-+/**
-+ * typedef i8042_filter_t - i8042 filter callback
-+ * @data: Data received by the i8042 controller
-+ * @str: Status register of the i8042 controller
-+ * @serio: Serio of the i8042 controller
-+ * @context: Context pointer associated with this callback
-+ *
-+ * This represents a i8042 filter callback which can be used with i8042_install_filter()
-+ * and i8042_remove_filter() to filter the i8042 input for platform-specific key codes.
-+ *
-+ * Context: Interrupt context.
-+ * Returns: true if the data should be filtered out, false if otherwise.
-+ */
-+typedef bool (*i8042_filter_t)(unsigned char data, unsigned char str, struct serio *serio,
-+			       void *context);
-+
- #if defined(CONFIG_SERIO_I8042) || defined(CONFIG_SERIO_I8042_MODULE)
- 
- void i8042_lock_chip(void);
- void i8042_unlock_chip(void);
- int i8042_command(unsigned char *param, int command);
-int i8042_install_filter(bool (*filter)(unsigned char data, unsigned char str,
-					struct serio *serio));
-int i8042_remove_filter(bool (*filter)(unsigned char data, unsigned char str,
-				       struct serio *serio));
-+int i8042_install_filter(i8042_filter_t filter, void *context);
-+int i8042_remove_filter(i8042_filter_t filter);
- 
- #else
- 
-@@ -78,14 +93,12 @@ static inline int i8042_command(unsigned char *param, int command)
- 	return -ENODEV;
- }
- 
-static inline int i8042_install_filter(bool (*filter)(unsigned char data, unsigned char str,
-					struct serio *serio))
-+static inline int i8042_install_filter(i8042_filter_t filter, void *context)
- {
- 	return -ENODEV;
- }
- 
-static inline int i8042_remove_filter(bool (*filter)(unsigned char data, unsigned char str,
-				       struct serio *serio))
-+static inline int i8042_remove_filter(i8042_filter_t filter)
- {
- 	return -ENODEV;
- }
-diff --git a/include/linux/i8253.h b/include/linux/i8253.h
-index 8336b2f6f834..56c280eb2d4f 100644
--- a/include/linux/i8253.h
-+++ b/include/linux/i8253.h
-@@ -21,9 +21,9 @@
- #define PIT_LATCH	((PIT_TICK_RATE + HZ/2) / HZ)
- 
- extern raw_spinlock_t i8253_lock;
-extern bool i8253_clear_counter_on_shutdown;
- extern struct clock_event_device i8253_clockevent;
- extern void clockevent_i8253_init(bool oneshot);
-+extern void clockevent_i8253_disable(void);
- 
- extern void setup_pit_timer(void);
- 
-diff --git a/include/linux/i8254.h b/include/linux/i8254.h
-new file mode 100644
-index 000000000000..a675c309232b
--- /dev/null
-+++ b/include/linux/i8254.h
-@@ -0,0 +1,21 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/* Copyright (C) William Breathitt Gray */
-+#ifndef _I8254_H_
-+#define _I8254_H_
-+
-+struct device;
-+struct regmap;
-+
-+/**
-+ * struct i8254_regmap_config - Configuration for the register map of an i8254
-+ * @parent:	parent device
-+ * @map:	regmap for the i8254
-+ */
-+struct i8254_regmap_config {
-+	struct device *parent;
-+	struct regmap *map;
-+};
-+
-+int devm_i8254_regmap_register(struct device *dev, const struct i8254_regmap_config *config);
-+
-+#endif /* _I8254_H_ */
-diff --git a/include/linux/icmp.h b/include/linux/icmp.h
-index 0af4d210ee31..043ec5d9c882 100644
--- a/include/linux/icmp.h
-+++ b/include/linux/icmp.h
-@@ -40,4 +40,36 @@ void ip_icmp_error_rfc4884(const struct sk_buff *skb,
- 			   struct sock_ee_data_rfc4884 *out,
- 			   int thlen, int off);
- 
-+/* RFC 4884 */
-+#define ICMP_EXT_ORIG_DGRAM_MIN_LEN	128
-+#define ICMP_EXT_VERSION_2		2
-+
-+/* ICMP Extension Object Classes */
-+#define ICMP_EXT_OBJ_CLASS_IIO		2	/* RFC 5837 */
-+
-+/* Interface Information Object - RFC 5837 */
-+enum {
-+	ICMP_EXT_CTYPE_IIO_ROLE_IIF,
-+};
-+
-+#define ICMP_EXT_CTYPE_IIO_ROLE(ROLE)	((ROLE) << 6)
-+#define ICMP_EXT_CTYPE_IIO_MTU		BIT(0)
-+#define ICMP_EXT_CTYPE_IIO_NAME		BIT(1)
-+#define ICMP_EXT_CTYPE_IIO_IPADDR	BIT(2)
-+#define ICMP_EXT_CTYPE_IIO_IFINDEX	BIT(3)
-+
-+struct icmp_ext_iio_name_subobj {
-+	u8 len;
-+	char name[IFNAMSIZ];
-+};
-+
-+enum {
-+	/* RFC 5837 - Incoming IP Interface Role */
-+	ICMP_ERR_EXT_IIO_IIF,
-+	/* Add new constants above. Used by "icmp_errors_extension_mask"
-+	 * sysctl.
-+	 */
-+	ICMP_ERR_EXT_COUNT,
-+};
-+
- #endif	/* _LINUX_ICMP_H */
-diff --git a/include/linux/icmpv6.h b/include/linux/icmpv6.h
-index db0f4fcfdaf4..e3b3b0fa2a8f 100644
--- a/include/linux/icmpv6.h
-+++ b/include/linux/icmpv6.h
-@@ -85,12 +85,10 @@ extern void				icmpv6_param_prob_reason(struct sk_buff *skb,
- 
- struct flowi6;
- struct in6_addr;
-extern void				icmpv6_flow_init(struct sock *sk,
-							 struct flowi6 *fl6,
-							 u8 type,
-							 const struct in6_addr *saddr,
-							 const struct in6_addr *daddr,
-							 int oif);
-+
-+void icmpv6_flow_init(const struct sock *sk, struct flowi6 *fl6, u8 type,
-+		      const struct in6_addr *saddr,
-+		      const struct in6_addr *daddr, int oif);
- 
- static inline void icmpv6_param_prob(struct sk_buff *skb, u8 code, int pos)
- {
-diff --git a/include/linux/idle_inject.h b/include/linux/idle_inject.h
-index fb88e23a99d3..a85d5dd40f72 100644
--- a/include/linux/idle_inject.h
-+++ b/include/linux/idle_inject.h
-@@ -13,6 +13,9 @@ struct idle_inject_device;
- 
- struct idle_inject_device *idle_inject_register(struct cpumask *cpumask);
- 
-+struct idle_inject_device *idle_inject_register_full(struct cpumask *cpumask,
-+						     bool (*update)(void));
-+
- void idle_inject_unregister(struct idle_inject_device *ii_dev);
- 
- int idle_inject_start(struct idle_inject_device *ii_dev);
-diff --git a/include/linux/idr.h b/include/linux/idr.h
-index a0dce14090a9..789e23e67444 100644
--- a/include/linux/idr.h
-+++ b/include/linux/idr.h
-@@ -15,6 +15,7 @@
- #include <linux/radix-tree.h>
- #include <linux/gfp.h>
- #include <linux/percpu.h>
-+#include <linux/cleanup.h>
- 
- struct idr {
- 	struct radix_tree_root	idr_rt;
-@@ -124,6 +125,22 @@ void *idr_get_next_ul(struct idr *, unsigned long *nextid);
- void *idr_replace(struct idr *, void *, unsigned long id);
- void idr_destroy(struct idr *);
- 
-+struct __class_idr {
-+	struct idr *idr;
-+	int id;
-+};
-+
-+#define idr_null ((struct __class_idr){ NULL, -1 })
-+#define take_idr_id(id) __get_and_null(id, idr_null)
-+
-+DEFINE_CLASS(idr_alloc, struct __class_idr,
-+	     if (_T.id >= 0) idr_remove(_T.idr, _T.id),
-+	     ((struct __class_idr){
-+	     	.idr = idr,
-+		.id = idr_alloc(idr, ptr, start, end, gfp),
-+	     }),
-+	     struct idr *idr, void *ptr, int start, int end, gfp_t gfp);
-+
- /**
-  * idr_init_base() - Initialise an IDR.
-  * @idr: IDR handle.
-@@ -200,7 +217,7 @@ static inline void idr_preload_end(void)
-  */
- #define idr_for_each_entry_ul(idr, entry, tmp, id)			\
- 	for (tmp = 0, id = 0;						\
-	     tmp <= id && ((entry) = idr_get_next_ul(idr, &(id))) != NULL; \
-+	     ((entry) = tmp <= id ? idr_get_next_ul(idr, &(id)) : NULL) != NULL; \
- 	     tmp = id, ++id)
- 
- /**
-@@ -224,10 +241,12 @@ static inline void idr_preload_end(void)
-  * @id: Entry ID.
-  *
-  * Continue to iterate over entries, continuing after the current position.
-+ * After normal termination @entry is left with the value NULL.  This
-+ * is convenient for a "not found" value.
-  */
- #define idr_for_each_entry_continue_ul(idr, entry, tmp, id)		\
- 	for (tmp = id;							\
-	     tmp <= id && ((entry) = idr_get_next_ul(idr, &(id))) != NULL; \
-+	     ((entry) = tmp <= id ? idr_get_next_ul(idr, &(id)) : NULL) != NULL; \
- 	     tmp = id, ++id)
- 
- /*
-@@ -255,6 +274,7 @@ struct ida {
- int ida_alloc_range(struct ida *, unsigned int min, unsigned int max, gfp_t);
- void ida_free(struct ida *, unsigned int id);
- void ida_destroy(struct ida *ida);
-+int ida_find_first_range(struct ida *ida, unsigned int min, unsigned int max);
- 
- /**
-  * ida_alloc() - Allocate an unused ID.
-@@ -314,16 +334,18 @@ static inline void ida_init(struct ida *ida)
- 	xa_init_flags(&ida->xa, IDA_INIT_FLAGS);
- }
- 
-/*
- * ida_simple_get() and ida_simple_remove() are deprecated. Use
- * ida_alloc() and ida_free() instead respectively.
- */
-#define ida_simple_get(ida, start, end, gfp)	\
-			ida_alloc_range(ida, start, (end) - 1, gfp)
-#define ida_simple_remove(ida, id)	ida_free(ida, id)
-
- static inline bool ida_is_empty(const struct ida *ida)
- {
- 	return xa_empty(&ida->xa);
- }
-+
-+static inline bool ida_exists(struct ida *ida, unsigned int id)
-+{
-+	return ida_find_first_range(ida, id, id) == id;
-+}
-+
-+static inline int ida_find_first(struct ida *ida)
-+{
-+	return ida_find_first_range(ida, 0, ~0);
-+}
- #endif /* __IDR_H__ */
-diff --git a/include/linux/ieee80211-eht.h b/include/linux/ieee80211-eht.h
-new file mode 100644
-index 000000000000..f9782e46c5e5
--- /dev/null
-+++ b/include/linux/ieee80211-eht.h
-@@ -0,0 +1,1182 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * IEEE 802.11 EHT definitions
-+ *
-+ * Copyright (c) 2001-2002, SSH Communications Security Corp and Jouni Malinen
-+ * <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2002-2003, Jouni Malinen <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2005, Devicescape Software, Inc.
-+ * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net>
-+ * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH
-+ * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
-+ * Copyright (c) 2018 - 2025 Intel Corporation
-+ */
-+
-+#ifndef LINUX_IEEE80211_EHT_H
-+#define LINUX_IEEE80211_EHT_H
-+
-+#include <linux/types.h>
-+#include <linux/if_ether.h>
-+/* need HE definitions for the inlines here */
-+#include <linux/ieee80211-he.h>
-+
-+#define IEEE80211_TTLM_MAX_CNT				2
-+#define IEEE80211_TTLM_CONTROL_DIRECTION		0x03
-+#define IEEE80211_TTLM_CONTROL_DEF_LINK_MAP		0x04
-+#define IEEE80211_TTLM_CONTROL_SWITCH_TIME_PRESENT	0x08
-+#define IEEE80211_TTLM_CONTROL_EXPECTED_DUR_PRESENT	0x10
-+#define IEEE80211_TTLM_CONTROL_LINK_MAP_SIZE		0x20
-+
-+#define IEEE80211_TTLM_DIRECTION_DOWN		0
-+#define IEEE80211_TTLM_DIRECTION_UP		1
-+#define IEEE80211_TTLM_DIRECTION_BOTH		2
-+
-+/**
-+ * struct ieee80211_ttlm_elem - TID-To-Link Mapping element
-+ *
-+ * Defined in section 9.4.2.314 in P802.11be_D4
-+ *
-+ * @control: the first part of control field
-+ * @optional: the second part of control field
-+ */
-+struct ieee80211_ttlm_elem {
-+	u8 control;
-+	u8 optional[];
-+} __packed;
-+
-+#define IEEE80211_EHT_MCS_NSS_RX 0x0f
-+#define IEEE80211_EHT_MCS_NSS_TX 0xf0
-+
-+/**
-+ * struct ieee80211_eht_mcs_nss_supp_20mhz_only - EHT 20MHz only station max
-+ * supported NSS for per MCS.
-+ *
-+ * For each field below, bits 0 - 3 indicate the maximal number of spatial
-+ * streams for Rx, and bits 4 - 7 indicate the maximal number of spatial streams
-+ * for Tx.
-+ *
-+ * @rx_tx_mcs7_max_nss: indicates the maximum number of spatial streams
-+ *     supported for reception and the maximum number of spatial streams
-+ *     supported for transmission for MCS 0 - 7.
-+ * @rx_tx_mcs9_max_nss: indicates the maximum number of spatial streams
-+ *     supported for reception and the maximum number of spatial streams
-+ *     supported for transmission for MCS 8 - 9.
-+ * @rx_tx_mcs11_max_nss: indicates the maximum number of spatial streams
-+ *     supported for reception and the maximum number of spatial streams
-+ *     supported for transmission for MCS 10 - 11.
-+ * @rx_tx_mcs13_max_nss: indicates the maximum number of spatial streams
-+ *     supported for reception and the maximum number of spatial streams
-+ *     supported for transmission for MCS 12 - 13.
-+ * @rx_tx_max_nss: array of the previous fields for easier loop access
-+ */
-+struct ieee80211_eht_mcs_nss_supp_20mhz_only {
-+	union {
-+		struct {
-+			u8 rx_tx_mcs7_max_nss;
-+			u8 rx_tx_mcs9_max_nss;
-+			u8 rx_tx_mcs11_max_nss;
-+			u8 rx_tx_mcs13_max_nss;
-+		};
-+		u8 rx_tx_max_nss[4];
-+	};
-+};
-+
-+/**
-+ * struct ieee80211_eht_mcs_nss_supp_bw - EHT max supported NSS per MCS (except
-+ * 20MHz only stations).
-+ *
-+ * For each field below, bits 0 - 3 indicate the maximal number of spatial
-+ * streams for Rx, and bits 4 - 7 indicate the maximal number of spatial streams
-+ * for Tx.
-+ *
-+ * @rx_tx_mcs9_max_nss: indicates the maximum number of spatial streams
-+ *     supported for reception and the maximum number of spatial streams
-+ *     supported for transmission for MCS 0 - 9.
-+ * @rx_tx_mcs11_max_nss: indicates the maximum number of spatial streams
-+ *     supported for reception and the maximum number of spatial streams
-+ *     supported for transmission for MCS 10 - 11.
-+ * @rx_tx_mcs13_max_nss: indicates the maximum number of spatial streams
-+ *     supported for reception and the maximum number of spatial streams
-+ *     supported for transmission for MCS 12 - 13.
-+ * @rx_tx_max_nss: array of the previous fields for easier loop access
-+ */
-+struct ieee80211_eht_mcs_nss_supp_bw {
-+	union {
-+		struct {
-+			u8 rx_tx_mcs9_max_nss;
-+			u8 rx_tx_mcs11_max_nss;
-+			u8 rx_tx_mcs13_max_nss;
-+		};
-+		u8 rx_tx_max_nss[3];
-+	};
-+};
-+
-+/**
-+ * struct ieee80211_eht_cap_elem_fixed - EHT capabilities fixed data
-+ *
-+ * This structure is the "EHT Capabilities element" fixed fields as
-+ * described in P802.11be_D2.0 section 9.4.2.313.
-+ *
-+ * @mac_cap_info: MAC capabilities, see IEEE80211_EHT_MAC_CAP*
-+ * @phy_cap_info: PHY capabilities, see IEEE80211_EHT_PHY_CAP*
-+ */
-+struct ieee80211_eht_cap_elem_fixed {
-+	u8 mac_cap_info[2];
-+	u8 phy_cap_info[9];
-+} __packed;
-+
-+/**
-+ * struct ieee80211_eht_cap_elem - EHT capabilities element
-+ * @fixed: fixed parts, see &ieee80211_eht_cap_elem_fixed
-+ * @optional: optional parts
-+ */
-+struct ieee80211_eht_cap_elem {
-+	struct ieee80211_eht_cap_elem_fixed fixed;
-+
-+	/*
-+	 * Followed by:
-+	 * Supported EHT-MCS And NSS Set field: 4, 3, 6 or 9 octets.
-+	 * EHT PPE Thresholds field: variable length.
-+	 */
-+	u8 optional[];
-+} __packed;
-+
-+#define IEEE80211_EHT_OPER_INFO_PRESENT	                        0x01
-+#define IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT	0x02
-+#define IEEE80211_EHT_OPER_EHT_DEF_PE_DURATION	                0x04
-+#define IEEE80211_EHT_OPER_GROUP_ADDRESSED_BU_IND_LIMIT         0x08
-+#define IEEE80211_EHT_OPER_GROUP_ADDRESSED_BU_IND_EXP_MASK      0x30
-+#define IEEE80211_EHT_OPER_MCS15_DISABLE                        0x40
-+
-+/**
-+ * struct ieee80211_eht_operation - eht operation element
-+ *
-+ * This structure is the "EHT Operation Element" fields as
-+ * described in P802.11be_D2.0 section 9.4.2.311
-+ *
-+ * @params: EHT operation element parameters. See &IEEE80211_EHT_OPER_*
-+ * @basic_mcs_nss: indicates the EHT-MCSs for each number of spatial streams in
-+ *     EHT PPDUs that are supported by all EHT STAs in the BSS in transmit and
-+ *     receive.
-+ * @optional: optional parts
-+ */
-+struct ieee80211_eht_operation {
-+	u8 params;
-+	struct ieee80211_eht_mcs_nss_supp_20mhz_only basic_mcs_nss;
-+	u8 optional[];
-+} __packed;
-+
-+/**
-+ * struct ieee80211_eht_operation_info - eht operation information
-+ *
-+ * @control: EHT operation information control.
-+ * @ccfs0: defines a channel center frequency for a 20, 40, 80, 160, or 320 MHz
-+ *     EHT BSS.
-+ * @ccfs1: defines a channel center frequency for a 160 or 320 MHz EHT BSS.
-+ * @optional: optional parts
-+ */
-+struct ieee80211_eht_operation_info {
-+	u8 control;
-+	u8 ccfs0;
-+	u8 ccfs1;
-+	u8 optional[];
-+} __packed;
-+
-+/* EHT MAC capabilities as defined in P802.11be_D2.0 section 9.4.2.313.2 */
-+#define IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS			0x01
-+#define IEEE80211_EHT_MAC_CAP0_OM_CONTROL			0x02
-+#define IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1		0x04
-+#define IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE2		0x08
-+#define IEEE80211_EHT_MAC_CAP0_RESTRICTED_TWT			0x10
-+#define IEEE80211_EHT_MAC_CAP0_SCS_TRAFFIC_DESC			0x20
-+#define IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_MASK		0xc0
-+#define	IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_3895	        0
-+#define	IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_7991	        1
-+#define	IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_11454	        2
-+
-+#define IEEE80211_EHT_MAC_CAP1_MAX_AMPDU_LEN_MASK		0x01
-+#define IEEE80211_EHT_MAC_CAP1_EHT_TRS				0x02
-+#define IEEE80211_EHT_MAC_CAP1_TXOP_RET				0x04
-+#define IEEE80211_EHT_MAC_CAP1_TWO_BQRS				0x08
-+#define IEEE80211_EHT_MAC_CAP1_EHT_LINK_ADAPT_MASK		0x30
-+#define IEEE80211_EHT_MAC_CAP1_UNSOL_EPCS_PRIO_ACCESS		0x40
-+
-+/* EHT PHY capabilities as defined in P802.11be_D2.0 section 9.4.2.313.3 */
-+#define IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ			0x02
-+#define IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ		0x04
-+#define IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI		0x08
-+#define IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO		0x10
-+#define IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER			0x20
-+#define IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE			0x40
-+
-+/* EHT beamformee number of spatial streams <= 80MHz is split */
-+#define IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK		0x80
-+#define IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK		0x03
-+
-+#define IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK	0x1c
-+#define IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK	0xe0
-+
-+#define IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK		0x07
-+#define IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK		0x38
-+
-+/* EHT number of sounding dimensions for 320MHz is split */
-+#define IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK		0xc0
-+#define IEEE80211_EHT_PHY_CAP3_SOUNDING_DIM_320MHZ_MASK		0x01
-+#define IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK		0x02
-+#define IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK		0x04
-+#define IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK		0x08
-+#define IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK		0x10
-+#define IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK			0x20
-+#define IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK		0x40
-+#define IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK			0x80
-+
-+#define IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO		0x01
-+#define IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP			0x02
-+#define IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP		0x04
-+#define IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI	0x08
-+#define IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK			0xf0
-+
-+#define IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK		0x01
-+#define IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP		0x02
-+#define IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP		0x04
-+#define IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT		0x08
-+#define IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK	0x30
-+#define   IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_0US	0
-+#define   IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_8US	1
-+#define   IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_16US	2
-+#define   IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_20US	3
-+
-+/* Maximum number of supported EHT LTF is split */
-+#define IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK	0xc0
-+#define IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF		0x40
-+#define IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK	0x07
-+
-+#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ			0x08
-+#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ		0x30
-+#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_320MHZ		0x40
-+#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK			0x78
-+#define IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP		0x80
-+
-+#define IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW	0x01
-+#define IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ	0x02
-+#define IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ	0x04
-+#define IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_320MHZ	0x08
-+#define IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ		0x10
-+#define IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ		0x20
-+#define IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_320MHZ		0x40
-+#define IEEE80211_EHT_PHY_CAP7_TB_SOUNDING_FDBK_RATE_LIMIT	0x80
-+
-+#define IEEE80211_EHT_PHY_CAP8_RX_1024QAM_WIDER_BW_DL_OFDMA	0x01
-+#define IEEE80211_EHT_PHY_CAP8_RX_4096QAM_WIDER_BW_DL_OFDMA	0x02
-+
-+/*
-+ * EHT operation channel width as defined in P802.11be_D2.0 section 9.4.2.311
-+ */
-+#define IEEE80211_EHT_OPER_CHAN_WIDTH		0x7
-+#define IEEE80211_EHT_OPER_CHAN_WIDTH_20MHZ	0
-+#define IEEE80211_EHT_OPER_CHAN_WIDTH_40MHZ	1
-+#define IEEE80211_EHT_OPER_CHAN_WIDTH_80MHZ	2
-+#define IEEE80211_EHT_OPER_CHAN_WIDTH_160MHZ	3
-+#define IEEE80211_EHT_OPER_CHAN_WIDTH_320MHZ	4
-+
-+/* Calculate 802.11be EHT capabilities IE Tx/Rx EHT MCS NSS Support Field size */
-+static inline u8
-+ieee80211_eht_mcs_nss_size(const struct ieee80211_he_cap_elem *he_cap,
-+			   const struct ieee80211_eht_cap_elem_fixed *eht_cap,
-+			   bool from_ap)
-+{
-+	u8 count = 0;
-+
-+	/* on 2.4 GHz, if it supports 40 MHz, the result is 3 */
-+	if (he_cap->phy_cap_info[0] &
-+	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G)
-+		return 3;
-+
-+	/* on 2.4 GHz, these three bits are reserved, so should be 0 */
-+	if (he_cap->phy_cap_info[0] &
-+	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G)
-+		count += 3;
-+
-+	if (he_cap->phy_cap_info[0] &
-+	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G)
-+		count += 3;
-+
-+	if (eht_cap->phy_cap_info[0] & IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ)
-+		count += 3;
-+
-+	if (count)
-+		return count;
-+
-+	return from_ap ? 3 : 4;
-+}
-+
-+/* 802.11be EHT PPE Thresholds */
-+#define IEEE80211_EHT_PPE_THRES_NSS_POS			0
-+#define IEEE80211_EHT_PPE_THRES_NSS_MASK		0xf
-+#define IEEE80211_EHT_PPE_THRES_RU_INDEX_BITMASK_MASK	0x1f0
-+#define IEEE80211_EHT_PPE_THRES_INFO_PPET_SIZE		3
-+#define IEEE80211_EHT_PPE_THRES_INFO_HEADER_SIZE	9
-+
-+/*
-+ * Calculate 802.11be EHT capabilities IE EHT field size
-+ */
-+static inline u8
-+ieee80211_eht_ppe_size(u16 ppe_thres_hdr, const u8 *phy_cap_info)
-+{
-+	u32 n;
-+
-+	if (!(phy_cap_info[5] &
-+	      IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT))
-+		return 0;
-+
-+	n = hweight16(ppe_thres_hdr &
-+		      IEEE80211_EHT_PPE_THRES_RU_INDEX_BITMASK_MASK);
-+	n *= 1 + u16_get_bits(ppe_thres_hdr, IEEE80211_EHT_PPE_THRES_NSS_MASK);
-+
-+	/*
-+	 * Each pair is 6 bits, and we need to add the 9 "header" bits to the
-+	 * total size.
-+	 */
-+	n = n * IEEE80211_EHT_PPE_THRES_INFO_PPET_SIZE * 2 +
-+	    IEEE80211_EHT_PPE_THRES_INFO_HEADER_SIZE;
-+	return DIV_ROUND_UP(n, 8);
-+}
-+
-+static inline bool
-+ieee80211_eht_capa_size_ok(const u8 *he_capa, const u8 *data, u8 len,
-+			   bool from_ap)
-+{
-+	const struct ieee80211_eht_cap_elem_fixed *elem = (const void *)data;
-+	u8 needed = sizeof(struct ieee80211_eht_cap_elem_fixed);
-+
-+	if (len < needed || !he_capa)
-+		return false;
-+
-+	needed += ieee80211_eht_mcs_nss_size((const void *)he_capa,
-+					     (const void *)data,
-+					     from_ap);
-+	if (len < needed)
-+		return false;
-+
-+	if (elem->phy_cap_info[5] &
-+			IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT) {
-+		u16 ppe_thres_hdr;
-+
-+		if (len < needed + sizeof(ppe_thres_hdr))
-+			return false;
-+
-+		ppe_thres_hdr = get_unaligned_le16(data + needed);
-+		needed += ieee80211_eht_ppe_size(ppe_thres_hdr,
-+						 elem->phy_cap_info);
-+	}
-+
-+	return len >= needed;
-+}
-+
-+static inline bool
-+ieee80211_eht_oper_size_ok(const u8 *data, u8 len)
-+{
-+	const struct ieee80211_eht_operation *elem = (const void *)data;
-+	u8 needed = sizeof(*elem);
-+
-+	if (len < needed)
-+		return false;
-+
-+	if (elem->params & IEEE80211_EHT_OPER_INFO_PRESENT) {
-+		needed += 3;
-+
-+		if (elem->params &
-+		    IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT)
-+			needed += 2;
-+	}
-+
-+	return len >= needed;
-+}
-+
-+/* must validate ieee80211_eht_oper_size_ok() first */
-+static inline u16
-+ieee80211_eht_oper_dis_subchan_bitmap(const struct ieee80211_eht_operation *eht_oper)
-+{
-+	const struct ieee80211_eht_operation_info *info =
-+		(const void *)eht_oper->optional;
-+
-+	if (!(eht_oper->params & IEEE80211_EHT_OPER_INFO_PRESENT))
-+		return 0;
-+
-+	if (!(eht_oper->params & IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT))
-+		return 0;
-+
-+	return get_unaligned_le16(info->optional);
-+}
-+
-+#define IEEE80211_BW_IND_DIS_SUBCH_PRESENT	BIT(1)
-+
-+struct ieee80211_bandwidth_indication {
-+	u8 params;
-+	struct ieee80211_eht_operation_info info;
-+} __packed;
-+
-+static inline bool
-+ieee80211_bandwidth_indication_size_ok(const u8 *data, u8 len)
-+{
-+	const struct ieee80211_bandwidth_indication *bwi = (const void *)data;
-+
-+	if (len < sizeof(*bwi))
-+		return false;
-+
-+	if (bwi->params & IEEE80211_BW_IND_DIS_SUBCH_PRESENT &&
-+	    len < sizeof(*bwi) + 2)
-+		return false;
-+
-+	return true;
-+}
-+
-+/* Protected EHT action codes */
-+enum ieee80211_protected_eht_actioncode {
-+	WLAN_PROTECTED_EHT_ACTION_TTLM_REQ = 0,
-+	WLAN_PROTECTED_EHT_ACTION_TTLM_RES = 1,
-+	WLAN_PROTECTED_EHT_ACTION_TTLM_TEARDOWN = 2,
-+	WLAN_PROTECTED_EHT_ACTION_EPCS_ENABLE_REQ = 3,
-+	WLAN_PROTECTED_EHT_ACTION_EPCS_ENABLE_RESP = 4,
-+	WLAN_PROTECTED_EHT_ACTION_EPCS_ENABLE_TEARDOWN = 5,
-+	WLAN_PROTECTED_EHT_ACTION_EML_OP_MODE_NOTIF = 6,
-+	WLAN_PROTECTED_EHT_ACTION_LINK_RECOMMEND = 7,
-+	WLAN_PROTECTED_EHT_ACTION_ML_OP_UPDATE_REQ = 8,
-+	WLAN_PROTECTED_EHT_ACTION_ML_OP_UPDATE_RESP = 9,
-+	WLAN_PROTECTED_EHT_ACTION_LINK_RECONFIG_NOTIF = 10,
-+	WLAN_PROTECTED_EHT_ACTION_LINK_RECONFIG_REQ = 11,
-+	WLAN_PROTECTED_EHT_ACTION_LINK_RECONFIG_RESP = 12,
-+};
-+
-+/* multi-link device */
-+#define IEEE80211_MLD_MAX_NUM_LINKS	15
-+
-+#define IEEE80211_ML_CONTROL_TYPE			0x0007
-+#define IEEE80211_ML_CONTROL_TYPE_BASIC			0
-+#define IEEE80211_ML_CONTROL_TYPE_PREQ			1
-+#define IEEE80211_ML_CONTROL_TYPE_RECONF		2
-+#define IEEE80211_ML_CONTROL_TYPE_TDLS			3
-+#define IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS		4
-+#define IEEE80211_ML_CONTROL_PRESENCE_MASK		0xfff0
-+
-+struct ieee80211_multi_link_elem {
-+	__le16 control;
-+	u8 variable[];
-+} __packed;
-+
-+#define IEEE80211_MLC_BASIC_PRES_LINK_ID		0x0010
-+#define IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT	0x0020
-+#define IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY		0x0040
-+#define IEEE80211_MLC_BASIC_PRES_EML_CAPA		0x0080
-+#define IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP		0x0100
-+#define IEEE80211_MLC_BASIC_PRES_MLD_ID			0x0200
-+#define IEEE80211_MLC_BASIC_PRES_EXT_MLD_CAPA_OP	0x0400
-+
-+#define IEEE80211_MED_SYNC_DELAY_DURATION		0x00ff
-+#define IEEE80211_MED_SYNC_DELAY_SYNC_OFDM_ED_THRESH	0x0f00
-+#define IEEE80211_MED_SYNC_DELAY_SYNC_MAX_NUM_TXOPS	0xf000
-+
-+/*
-+ * Described in P802.11be_D3.0
-+ * dot11MSDTimerDuration should default to 5484 (i.e. 171.375)
-+ * dot11MSDOFDMEDthreshold defaults to -72 (i.e. 0)
-+ * dot11MSDTXOPMAX defaults to 1
-+ */
-+#define IEEE80211_MED_SYNC_DELAY_DEFAULT		0x10ac
-+
-+#define IEEE80211_EML_CAP_EMLSR_SUPP			0x0001
-+#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY		0x000e
-+#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_0US		0
-+#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_32US		1
-+#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_64US		2
-+#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_128US		3
-+#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_256US		4
-+#define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY	0x0070
-+#define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_0US		0
-+#define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_16US		1
-+#define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_32US		2
-+#define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_64US		3
-+#define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_128US		4
-+#define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_256US		5
-+#define IEEE80211_EML_CAP_EMLMR_SUPPORT			0x0080
-+#define IEEE80211_EML_CAP_EMLMR_DELAY			0x0700
-+#define  IEEE80211_EML_CAP_EMLMR_DELAY_0US			0
-+#define  IEEE80211_EML_CAP_EMLMR_DELAY_32US			1
-+#define  IEEE80211_EML_CAP_EMLMR_DELAY_64US			2
-+#define  IEEE80211_EML_CAP_EMLMR_DELAY_128US			3
-+#define  IEEE80211_EML_CAP_EMLMR_DELAY_256US			4
-+#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT		0x7800
-+#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_0			0
-+#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128US		1
-+#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_256US		2
-+#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_512US		3
-+#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_1TU		4
-+#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_2TU		5
-+#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_4TU		6
-+#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_8TU		7
-+#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_16TU		8
-+#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_32TU		9
-+#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_64TU		10
-+#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128TU		11
-+
-+#define IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS		0x000f
-+#define IEEE80211_MLD_CAP_OP_SRS_SUPPORT		0x0010
-+#define IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP	0x0060
-+#define IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_NO_SUPP	0
-+#define IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP_SAME	1
-+#define IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_RESERVED	2
-+#define IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP_DIFF	3
-+#define IEEE80211_MLD_CAP_OP_FREQ_SEP_TYPE_IND		0x0f80
-+#define IEEE80211_MLD_CAP_OP_AAR_SUPPORT		0x1000
-+#define IEEE80211_MLD_CAP_OP_LINK_RECONF_SUPPORT	0x2000
-+#define IEEE80211_MLD_CAP_OP_ALIGNED_TWT_SUPPORT	0x4000
-+
-+struct ieee80211_mle_basic_common_info {
-+	u8 len;
-+	u8 mld_mac_addr[ETH_ALEN];
-+	u8 variable[];
-+} __packed;
-+
-+#define IEEE80211_MLC_PREQ_PRES_MLD_ID			0x0010
-+
-+struct ieee80211_mle_preq_common_info {
-+	u8 len;
-+	u8 variable[];
-+} __packed;
-+
-+#define IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR		0x0010
-+#define IEEE80211_MLC_RECONF_PRES_EML_CAPA		0x0020
-+#define IEEE80211_MLC_RECONF_PRES_MLD_CAPA_OP		0x0040
-+#define IEEE80211_MLC_RECONF_PRES_EXT_MLD_CAPA_OP	0x0080
-+
-+/* no fixed fields in RECONF */
-+
-+struct ieee80211_mle_tdls_common_info {
-+	u8 len;
-+	u8 ap_mld_mac_addr[ETH_ALEN];
-+} __packed;
-+
-+#define IEEE80211_MLC_PRIO_ACCESS_PRES_AP_MLD_MAC_ADDR	0x0010
-+
-+/* no fixed fields in PRIO_ACCESS */
-+
-+/**
-+ * ieee80211_mle_common_size - check multi-link element common size
-+ * @data: multi-link element, must already be checked for size using
-+ *	ieee80211_mle_size_ok()
-+ * Return: the size of the multi-link element's "common" subfield 
-+ */
-+static inline u8 ieee80211_mle_common_size(const u8 *data)
-+{
-+	const struct ieee80211_multi_link_elem *mle = (const void *)data;
-+	u16 control = le16_to_cpu(mle->control);
-+
-+	switch (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE)) {
-+	case IEEE80211_ML_CONTROL_TYPE_BASIC:
-+	case IEEE80211_ML_CONTROL_TYPE_PREQ:
-+	case IEEE80211_ML_CONTROL_TYPE_TDLS:
-+	case IEEE80211_ML_CONTROL_TYPE_RECONF:
-+	case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS:
-+		/*
-+		 * The length is the first octet pointed by mle->variable so no
-+		 * need to add anything
-+		 */
-+		break;
-+	default:
-+		WARN_ON(1);
-+		return 0;
-+	}
-+
-+	return sizeof(*mle) + mle->variable[0];
-+}
-+
-+/**
-+ * ieee80211_mle_get_link_id - returns the link ID
-+ * @data: the basic multi link element
-+ * Return: the link ID, or -1 if not present
-+ *
-+ * The element is assumed to be of the correct type (BASIC) and big enough,
-+ * this must be checked using ieee80211_mle_type_ok().
-+ */
-+static inline int ieee80211_mle_get_link_id(const u8 *data)
-+{
-+	const struct ieee80211_multi_link_elem *mle = (const void *)data;
-+	u16 control = le16_to_cpu(mle->control);
-+	const u8 *common = mle->variable;
-+
-+	/* common points now at the beginning of ieee80211_mle_basic_common_info */
-+	common += sizeof(struct ieee80211_mle_basic_common_info);
-+
-+	if (!(control & IEEE80211_MLC_BASIC_PRES_LINK_ID))
-+		return -1;
-+
-+	return *common;
-+}
-+
-+/**
-+ * ieee80211_mle_get_bss_param_ch_cnt - returns the BSS parameter change count
-+ * @data: pointer to the basic multi link element
-+ * Return: the BSS Parameter Change Count field value, or -1 if not present
-+ *
-+ * The element is assumed to be of the correct type (BASIC) and big enough,
-+ * this must be checked using ieee80211_mle_type_ok().
-+ */
-+static inline int
-+ieee80211_mle_get_bss_param_ch_cnt(const u8 *data)
-+{
-+	const struct ieee80211_multi_link_elem *mle = (const void *)data;
-+	u16 control = le16_to_cpu(mle->control);
-+	const u8 *common = mle->variable;
-+
-+	/* common points now at the beginning of ieee80211_mle_basic_common_info */
-+	common += sizeof(struct ieee80211_mle_basic_common_info);
-+
-+	if (!(control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT))
-+		return -1;
-+
-+	if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID)
-+		common += 1;
-+
-+	return *common;
-+}
-+
-+/**
-+ * ieee80211_mle_get_eml_med_sync_delay - returns the medium sync delay
-+ * @data: pointer to the multi-link element
-+ * Return: the medium synchronization delay field value from the multi-link
-+ *	element, or the default value (%IEEE80211_MED_SYNC_DELAY_DEFAULT)
-+ *	if not present
-+ *
-+ * The element is assumed to be of the correct type (BASIC) and big enough,
-+ * this must be checked using ieee80211_mle_type_ok().
-+ */
-+static inline u16 ieee80211_mle_get_eml_med_sync_delay(const u8 *data)
-+{
-+	const struct ieee80211_multi_link_elem *mle = (const void *)data;
-+	u16 control = le16_to_cpu(mle->control);
-+	const u8 *common = mle->variable;
-+
-+	/* common points now at the beginning of ieee80211_mle_basic_common_info */
-+	common += sizeof(struct ieee80211_mle_basic_common_info);
-+
-+	if (!(control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY))
-+		return IEEE80211_MED_SYNC_DELAY_DEFAULT;
-+
-+	if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID)
-+		common += 1;
-+	if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT)
-+		common += 1;
-+
-+	return get_unaligned_le16(common);
-+}
-+
-+/**
-+ * ieee80211_mle_get_eml_cap - returns the EML capability
-+ * @data: pointer to the multi-link element
-+ * Return: the EML capability field value from the multi-link element,
-+ *	or 0 if not present
-+ *
-+ * The element is assumed to be of the correct type (BASIC) and big enough,
-+ * this must be checked using ieee80211_mle_type_ok().
-+ */
-+static inline u16 ieee80211_mle_get_eml_cap(const u8 *data)
-+{
-+	const struct ieee80211_multi_link_elem *mle = (const void *)data;
-+	u16 control = le16_to_cpu(mle->control);
-+	const u8 *common = mle->variable;
-+
-+	/* common points now at the beginning of ieee80211_mle_basic_common_info */
-+	common += sizeof(struct ieee80211_mle_basic_common_info);
-+
-+	if (!(control & IEEE80211_MLC_BASIC_PRES_EML_CAPA))
-+		return 0;
-+
-+	if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID)
-+		common += 1;
-+	if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT)
-+		common += 1;
-+	if (control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY)
-+		common += 2;
-+
-+	return get_unaligned_le16(common);
-+}
-+
-+/**
-+ * ieee80211_mle_get_mld_capa_op - returns the MLD capabilities and operations.
-+ * @data: pointer to the multi-link element
-+ * Return: the MLD capabilities and operations field value from the multi-link
-+ *	element, or 0 if not present
-+ *
-+ * The element is assumed to be of the correct type (BASIC) and big enough,
-+ * this must be checked using ieee80211_mle_type_ok().
-+ */
-+static inline u16 ieee80211_mle_get_mld_capa_op(const u8 *data)
-+{
-+	const struct ieee80211_multi_link_elem *mle = (const void *)data;
-+	u16 control = le16_to_cpu(mle->control);
-+	const u8 *common = mle->variable;
-+
-+	/*
-+	 * common points now at the beginning of
-+	 * ieee80211_mle_basic_common_info
-+	 */
-+	common += sizeof(struct ieee80211_mle_basic_common_info);
-+
-+	if (!(control & IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP))
-+		return 0;
-+
-+	if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID)
-+		common += 1;
-+	if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT)
-+		common += 1;
-+	if (control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY)
-+		common += 2;
-+	if (control & IEEE80211_MLC_BASIC_PRES_EML_CAPA)
-+		common += 2;
-+
-+	return get_unaligned_le16(common);
-+}
-+
-+/* Defined in Figure 9-1074t in P802.11be_D7.0 */
-+#define IEEE80211_EHT_ML_EXT_MLD_CAPA_OP_PARAM_UPDATE           0x0001
-+#define IEEE80211_EHT_ML_EXT_MLD_CAPA_OP_RECO_MAX_LINKS_MASK    0x001e
-+#define IEEE80211_EHT_ML_EXT_MLD_CAPA_NSTR_UPDATE               0x0020
-+#define IEEE80211_EHT_ML_EXT_MLD_CAPA_EMLSR_ENA_ON_ONE_LINK     0x0040
-+#define IEEE80211_EHT_ML_EXT_MLD_CAPA_BTM_MLD_RECO_MULTI_AP     0x0080
-+
-+/**
-+ * ieee80211_mle_get_ext_mld_capa_op - returns the extended MLD capabilities
-+ *	and operations.
-+ * @data: pointer to the multi-link element
-+ * Return: the extended MLD capabilities and operations field value from
-+ *	the multi-link element, or 0 if not present
-+ *
-+ * The element is assumed to be of the correct type (BASIC) and big enough,
-+ * this must be checked using ieee80211_mle_type_ok().
-+ */
-+static inline u16 ieee80211_mle_get_ext_mld_capa_op(const u8 *data)
-+{
-+	const struct ieee80211_multi_link_elem *mle = (const void *)data;
-+	u16 control = le16_to_cpu(mle->control);
-+	const u8 *common = mle->variable;
-+
-+	/*
-+	 * common points now at the beginning of
-+	 * ieee80211_mle_basic_common_info
-+	 */
-+	common += sizeof(struct ieee80211_mle_basic_common_info);
-+
-+	if (!(control & IEEE80211_MLC_BASIC_PRES_EXT_MLD_CAPA_OP))
-+		return 0;
-+
-+	if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID)
-+		common += 1;
-+	if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT)
-+		common += 1;
-+	if (control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY)
-+		common += 2;
-+	if (control & IEEE80211_MLC_BASIC_PRES_EML_CAPA)
-+		common += 2;
-+	if (control & IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP)
-+		common += 2;
-+	if (control & IEEE80211_MLC_BASIC_PRES_MLD_ID)
-+		common += 1;
-+
-+	return get_unaligned_le16(common);
-+}
-+
-+/**
-+ * ieee80211_mle_get_mld_id - returns the MLD ID
-+ * @data: pointer to the multi-link element
-+ * Return: The MLD ID in the given multi-link element, or 0 if not present
-+ *
-+ * The element is assumed to be of the correct type (BASIC) and big enough,
-+ * this must be checked using ieee80211_mle_type_ok().
-+ */
-+static inline u8 ieee80211_mle_get_mld_id(const u8 *data)
-+{
-+	const struct ieee80211_multi_link_elem *mle = (const void *)data;
-+	u16 control = le16_to_cpu(mle->control);
-+	const u8 *common = mle->variable;
-+
-+	/*
-+	 * common points now at the beginning of
-+	 * ieee80211_mle_basic_common_info
-+	 */
-+	common += sizeof(struct ieee80211_mle_basic_common_info);
-+
-+	if (!(control & IEEE80211_MLC_BASIC_PRES_MLD_ID))
-+		return 0;
-+
-+	if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID)
-+		common += 1;
-+	if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT)
-+		common += 1;
-+	if (control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY)
-+		common += 2;
-+	if (control & IEEE80211_MLC_BASIC_PRES_EML_CAPA)
-+		common += 2;
-+	if (control & IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP)
-+		common += 2;
-+
-+	return *common;
-+}
-+
-+/**
-+ * ieee80211_mle_size_ok - validate multi-link element size
-+ * @data: pointer to the element data
-+ * @len: length of the containing element
-+ * Return: whether or not the multi-link element size is OK
-+ */
-+static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len)
-+{
-+	const struct ieee80211_multi_link_elem *mle = (const void *)data;
-+	u8 fixed = sizeof(*mle);
-+	u8 common = 0;
-+	bool check_common_len = false;
-+	u16 control;
-+
-+	if (!data || len < fixed)
-+		return false;
-+
-+	control = le16_to_cpu(mle->control);
-+
-+	switch (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE)) {
-+	case IEEE80211_ML_CONTROL_TYPE_BASIC:
-+		common += sizeof(struct ieee80211_mle_basic_common_info);
-+		check_common_len = true;
-+		if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID)
-+			common += 1;
-+		if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT)
-+			common += 1;
-+		if (control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY)
-+			common += 2;
-+		if (control & IEEE80211_MLC_BASIC_PRES_EML_CAPA)
-+			common += 2;
-+		if (control & IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP)
-+			common += 2;
-+		if (control & IEEE80211_MLC_BASIC_PRES_MLD_ID)
-+			common += 1;
-+		if (control & IEEE80211_MLC_BASIC_PRES_EXT_MLD_CAPA_OP)
-+			common += 2;
-+		break;
-+	case IEEE80211_ML_CONTROL_TYPE_PREQ:
-+		common += sizeof(struct ieee80211_mle_preq_common_info);
-+		if (control & IEEE80211_MLC_PREQ_PRES_MLD_ID)
-+			common += 1;
-+		check_common_len = true;
-+		break;
-+	case IEEE80211_ML_CONTROL_TYPE_RECONF:
-+		if (control & IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR)
-+			common += ETH_ALEN;
-+		if (control & IEEE80211_MLC_RECONF_PRES_EML_CAPA)
-+			common += 2;
-+		if (control & IEEE80211_MLC_RECONF_PRES_MLD_CAPA_OP)
-+			common += 2;
-+		if (control & IEEE80211_MLC_RECONF_PRES_EXT_MLD_CAPA_OP)
-+			common += 2;
-+		break;
-+	case IEEE80211_ML_CONTROL_TYPE_TDLS:
-+		common += sizeof(struct ieee80211_mle_tdls_common_info);
-+		check_common_len = true;
-+		break;
-+	case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS:
-+		common = ETH_ALEN + 1;
-+		break;
-+	default:
-+		/* we don't know this type */
-+		return true;
-+	}
-+
-+	if (len < fixed + common)
-+		return false;
-+
-+	if (!check_common_len)
-+		return true;
-+
-+	/* if present, common length is the first octet there */
-+	return mle->variable[0] >= common;
-+}
-+
-+/**
-+ * ieee80211_mle_type_ok - validate multi-link element type and size
-+ * @data: pointer to the element data
-+ * @type: expected type of the element
-+ * @len: length of the containing element
-+ * Return: whether or not the multi-link element type matches and size is OK
-+ */
-+static inline bool ieee80211_mle_type_ok(const u8 *data, u8 type, size_t len)
-+{
-+	const struct ieee80211_multi_link_elem *mle = (const void *)data;
-+	u16 control;
-+
-+	if (!ieee80211_mle_size_ok(data, len))
-+		return false;
-+
-+	control = le16_to_cpu(mle->control);
-+
-+	if (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE) == type)
-+		return true;
-+
-+	return false;
-+}
-+
-+enum ieee80211_mle_subelems {
-+	IEEE80211_MLE_SUBELEM_PER_STA_PROFILE		= 0,
-+	IEEE80211_MLE_SUBELEM_FRAGMENT		        = 254,
-+};
-+
-+#define IEEE80211_MLE_STA_CONTROL_LINK_ID			0x000f
-+#define IEEE80211_MLE_STA_CONTROL_COMPLETE_PROFILE		0x0010
-+#define IEEE80211_MLE_STA_CONTROL_STA_MAC_ADDR_PRESENT		0x0020
-+#define IEEE80211_MLE_STA_CONTROL_BEACON_INT_PRESENT		0x0040
-+#define IEEE80211_MLE_STA_CONTROL_TSF_OFFS_PRESENT		0x0080
-+#define IEEE80211_MLE_STA_CONTROL_DTIM_INFO_PRESENT		0x0100
-+#define IEEE80211_MLE_STA_CONTROL_NSTR_LINK_PAIR_PRESENT	0x0200
-+#define IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE		0x0400
-+#define IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT	0x0800
-+
-+struct ieee80211_mle_per_sta_profile {
-+	__le16 control;
-+	u8 sta_info_len;
-+	u8 variable[];
-+} __packed;
-+
-+/**
-+ * ieee80211_mle_basic_sta_prof_size_ok - validate basic multi-link element sta
-+ *	profile size
-+ * @data: pointer to the sub element data
-+ * @len: length of the containing sub element
-+ * Return: %true if the STA profile is large enough, %false otherwise
-+ */
-+static inline bool ieee80211_mle_basic_sta_prof_size_ok(const u8 *data,
-+							size_t len)
-+{
-+	const struct ieee80211_mle_per_sta_profile *prof = (const void *)data;
-+	u16 control;
-+	u8 fixed = sizeof(*prof);
-+	u8 info_len = 1;
-+
-+	if (len < fixed)
-+		return false;
-+
-+	control = le16_to_cpu(prof->control);
-+
-+	if (control & IEEE80211_MLE_STA_CONTROL_STA_MAC_ADDR_PRESENT)
-+		info_len += 6;
-+	if (control & IEEE80211_MLE_STA_CONTROL_BEACON_INT_PRESENT)
-+		info_len += 2;
-+	if (control & IEEE80211_MLE_STA_CONTROL_TSF_OFFS_PRESENT)
-+		info_len += 8;
-+	if (control & IEEE80211_MLE_STA_CONTROL_DTIM_INFO_PRESENT)
-+		info_len += 2;
-+	if (control & IEEE80211_MLE_STA_CONTROL_COMPLETE_PROFILE &&
-+	    control & IEEE80211_MLE_STA_CONTROL_NSTR_LINK_PAIR_PRESENT) {
-+		if (control & IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE)
-+			info_len += 2;
-+		else
-+			info_len += 1;
-+	}
-+	if (control & IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT)
-+		info_len += 1;
-+
-+	return prof->sta_info_len >= info_len &&
-+	       fixed + prof->sta_info_len - 1 <= len;
-+}
-+
-+/**
-+ * ieee80211_mle_basic_sta_prof_bss_param_ch_cnt - get per-STA profile BSS
-+ *	parameter change count
-+ * @prof: the per-STA profile, having been checked with
-+ *	ieee80211_mle_basic_sta_prof_size_ok() for the correct length
-+ *
-+ * Return: The BSS parameter change count value if present, 0 otherwise.
-+ */
-+static inline u8
-+ieee80211_mle_basic_sta_prof_bss_param_ch_cnt(const struct ieee80211_mle_per_sta_profile *prof)
-+{
-+	u16 control = le16_to_cpu(prof->control);
-+	const u8 *pos = prof->variable;
-+
-+	if (!(control & IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT))
-+		return 0;
-+
-+	if (control & IEEE80211_MLE_STA_CONTROL_STA_MAC_ADDR_PRESENT)
-+		pos += 6;
-+	if (control & IEEE80211_MLE_STA_CONTROL_BEACON_INT_PRESENT)
-+		pos += 2;
-+	if (control & IEEE80211_MLE_STA_CONTROL_TSF_OFFS_PRESENT)
-+		pos += 8;
-+	if (control & IEEE80211_MLE_STA_CONTROL_DTIM_INFO_PRESENT)
-+		pos += 2;
-+	if (control & IEEE80211_MLE_STA_CONTROL_COMPLETE_PROFILE &&
-+	    control & IEEE80211_MLE_STA_CONTROL_NSTR_LINK_PAIR_PRESENT) {
-+		if (control & IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE)
-+			pos += 2;
-+		else
-+			pos += 1;
-+	}
-+
-+	return *pos;
-+}
-+
-+#define IEEE80211_MLE_STA_RECONF_CONTROL_LINK_ID			0x000f
-+#define IEEE80211_MLE_STA_RECONF_CONTROL_COMPLETE_PROFILE		0x0010
-+#define IEEE80211_MLE_STA_RECONF_CONTROL_STA_MAC_ADDR_PRESENT		0x0020
-+#define IEEE80211_MLE_STA_RECONF_CONTROL_AP_REM_TIMER_PRESENT		0x0040
-+#define	IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE                 0x0780
-+#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_AP_REM          0
-+#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_OP_PARAM_UPDATE 1
-+#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_ADD_LINK        2
-+#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_DEL_LINK        3
-+#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_NSTR_STATUS     4
-+#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_PARAMS_PRESENT       0x0800
-+
-+/**
-+ * ieee80211_mle_reconf_sta_prof_size_ok - validate reconfiguration multi-link
-+ *	element sta profile size.
-+ * @data: pointer to the sub element data
-+ * @len: length of the containing sub element
-+ * Return: %true if the STA profile is large enough, %false otherwise
-+ */
-+static inline bool ieee80211_mle_reconf_sta_prof_size_ok(const u8 *data,
-+							 size_t len)
-+{
-+	const struct ieee80211_mle_per_sta_profile *prof = (const void *)data;
-+	u16 control;
-+	u8 fixed = sizeof(*prof);
-+	u8 info_len = 1;
-+
-+	if (len < fixed)
-+		return false;
-+
-+	control = le16_to_cpu(prof->control);
-+
-+	if (control & IEEE80211_MLE_STA_RECONF_CONTROL_STA_MAC_ADDR_PRESENT)
-+		info_len += ETH_ALEN;
-+	if (control & IEEE80211_MLE_STA_RECONF_CONTROL_AP_REM_TIMER_PRESENT)
-+		info_len += 2;
-+	if (control & IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_PARAMS_PRESENT)
-+		info_len += 2;
-+
-+	return prof->sta_info_len >= info_len &&
-+	       fixed + prof->sta_info_len - 1 <= len;
-+}
-+
-+#define IEEE80211_MLE_STA_EPCS_CONTROL_LINK_ID			0x000f
-+#define IEEE80211_EPCS_ENA_RESP_BODY_LEN                        3
-+
-+static inline bool ieee80211_tid_to_link_map_size_ok(const u8 *data, size_t len)
-+{
-+	const struct ieee80211_ttlm_elem *t2l = (const void *)data;
-+	u8 control, fixed = sizeof(*t2l), elem_len = 0;
-+
-+	if (len < fixed)
-+		return false;
-+
-+	control = t2l->control;
-+
-+	if (control & IEEE80211_TTLM_CONTROL_SWITCH_TIME_PRESENT)
-+		elem_len += 2;
-+	if (control & IEEE80211_TTLM_CONTROL_EXPECTED_DUR_PRESENT)
-+		elem_len += 3;
-+
-+	if (!(control & IEEE80211_TTLM_CONTROL_DEF_LINK_MAP)) {
-+		u8 bm_size;
-+
-+		elem_len += 1;
-+		if (len < fixed + elem_len)
-+			return false;
-+
-+		if (control & IEEE80211_TTLM_CONTROL_LINK_MAP_SIZE)
-+			bm_size = 1;
-+		else
-+			bm_size = 2;
-+
-+		elem_len += hweight8(t2l->optional[0]) * bm_size;
-+	}
-+
-+	return len >= fixed + elem_len;
-+}
-+
-+/**
-+ * ieee80211_emlsr_pad_delay_in_us - Fetch the EMLSR Padding delay
-+ *	in microseconds
-+ * @eml_cap: EML capabilities field value from common info field of
-+ *	the Multi-link element
-+ * Return: the EMLSR Padding delay (in microseconds) encoded in the
-+ *	EML Capabilities field
-+ */
-+
-+static inline u32 ieee80211_emlsr_pad_delay_in_us(u16 eml_cap)
-+{
-+	/* IEEE Std 802.11be-2024 Table 9-417i—Encoding of the EMLSR
-+	 * Padding Delay subfield.
-+	 */
-+	u32 pad_delay = u16_get_bits(eml_cap,
-+				     IEEE80211_EML_CAP_EMLSR_PADDING_DELAY);
-+
-+	if (!pad_delay ||
-+	    pad_delay > IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_256US)
-+		return 0;
-+
-+	return 32 * (1 << (pad_delay - 1));
-+}
-+
-+/**
-+ * ieee80211_emlsr_trans_delay_in_us - Fetch the EMLSR Transition
-+ *	delay in microseconds
-+ * @eml_cap: EML capabilities field value from common info field of
-+ *	the Multi-link element
-+ * Return: the EMLSR Transition delay (in microseconds) encoded in the
-+ *	EML Capabilities field
-+ */
-+
-+static inline u32 ieee80211_emlsr_trans_delay_in_us(u16 eml_cap)
-+{
-+	/* IEEE Std 802.11be-2024 Table 9-417j—Encoding of the EMLSR
-+	 * Transition Delay subfield.
-+	 */
-+	u32 trans_delay =
-+		u16_get_bits(eml_cap,
-+			     IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY);
-+
-+	/* invalid values also just use 0 */
-+	if (!trans_delay ||
-+	    trans_delay > IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_256US)
-+		return 0;
-+
-+	return 16 * (1 << (trans_delay - 1));
-+}
-+
-+/**
-+ * ieee80211_eml_trans_timeout_in_us - Fetch the EMLSR Transition
-+ *	timeout value in microseconds
-+ * @eml_cap: EML capabilities field value from common info field of
-+ *	the Multi-link element
-+ * Return: the EMLSR Transition timeout (in microseconds) encoded in
-+ *	the EML Capabilities field
-+ */
-+
-+static inline u32 ieee80211_eml_trans_timeout_in_us(u16 eml_cap)
-+{
-+	/* IEEE Std 802.11be-2024 Table 9-417m—Encoding of the
-+	 * Transition Timeout subfield.
-+	 */
-+	u8 timeout = u16_get_bits(eml_cap,
-+				  IEEE80211_EML_CAP_TRANSITION_TIMEOUT);
-+
-+	/* invalid values also just use 0 */
-+	if (!timeout || timeout > IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128TU)
-+		return 0;
-+
-+	return 128 * (1 << (timeout - 1));
-+}
-+
-+#define for_each_mle_subelement(_elem, _data, _len)			\
-+	if (ieee80211_mle_size_ok(_data, _len))				\
-+		for_each_element(_elem,					\
-+				 _data + ieee80211_mle_common_size(_data),\
-+				 _len - ieee80211_mle_common_size(_data))
-+
-+#endif /* LINUX_IEEE80211_H */
-diff --git a/include/linux/ieee80211-he.h b/include/linux/ieee80211-he.h
-new file mode 100644
-index 000000000000..a08c446fbb04
--- /dev/null
-+++ b/include/linux/ieee80211-he.h
-@@ -0,0 +1,825 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * IEEE 802.11 HE definitions
-+ *
-+ * Copyright (c) 2001-2002, SSH Communications Security Corp and Jouni Malinen
-+ * <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2002-2003, Jouni Malinen <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2005, Devicescape Software, Inc.
-+ * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net>
-+ * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH
-+ * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
-+ * Copyright (c) 2018 - 2025 Intel Corporation
-+ */
-+
-+#ifndef LINUX_IEEE80211_HE_H
-+#define LINUX_IEEE80211_HE_H
-+
-+#include <linux/types.h>
-+#include <linux/if_ether.h>
-+
-+#define IEEE80211_TWT_CONTROL_NDP			BIT(0)
-+#define IEEE80211_TWT_CONTROL_RESP_MODE			BIT(1)
-+#define IEEE80211_TWT_CONTROL_NEG_TYPE_BROADCAST	BIT(3)
-+#define IEEE80211_TWT_CONTROL_RX_DISABLED		BIT(4)
-+#define IEEE80211_TWT_CONTROL_WAKE_DUR_UNIT		BIT(5)
-+
-+#define IEEE80211_TWT_REQTYPE_REQUEST			BIT(0)
-+#define IEEE80211_TWT_REQTYPE_SETUP_CMD			GENMASK(3, 1)
-+#define IEEE80211_TWT_REQTYPE_TRIGGER			BIT(4)
-+#define IEEE80211_TWT_REQTYPE_IMPLICIT			BIT(5)
-+#define IEEE80211_TWT_REQTYPE_FLOWTYPE			BIT(6)
-+#define IEEE80211_TWT_REQTYPE_FLOWID			GENMASK(9, 7)
-+#define IEEE80211_TWT_REQTYPE_WAKE_INT_EXP		GENMASK(14, 10)
-+#define IEEE80211_TWT_REQTYPE_PROTECTION		BIT(15)
-+
-+enum ieee80211_twt_setup_cmd {
-+	TWT_SETUP_CMD_REQUEST,
-+	TWT_SETUP_CMD_SUGGEST,
-+	TWT_SETUP_CMD_DEMAND,
-+	TWT_SETUP_CMD_GROUPING,
-+	TWT_SETUP_CMD_ACCEPT,
-+	TWT_SETUP_CMD_ALTERNATE,
-+	TWT_SETUP_CMD_DICTATE,
-+	TWT_SETUP_CMD_REJECT,
-+};
-+
-+struct ieee80211_twt_params {
-+	__le16 req_type;
-+	__le64 twt;
-+	u8 min_twt_dur;
-+	__le16 mantissa;
-+	u8 channel;
-+} __packed;
-+
-+struct ieee80211_twt_setup {
-+	u8 dialog_token;
-+	u8 element_id;
-+	u8 length;
-+	u8 control;
-+	u8 params[];
-+} __packed;
-+
-+/**
-+ * struct ieee80211_he_cap_elem - HE capabilities element
-+ * @mac_cap_info: HE MAC Capabilities Information
-+ * @phy_cap_info: HE PHY Capabilities Information
-+ *
-+ * This structure represents the fixed fields of the payload of the
-+ * "HE capabilities element" as described in IEEE Std 802.11ax-2021
-+ * sections 9.4.2.248.2 and 9.4.2.248.3.
-+ */
-+struct ieee80211_he_cap_elem {
-+	u8 mac_cap_info[6];
-+	u8 phy_cap_info[11];
-+} __packed;
-+
-+#define IEEE80211_TX_RX_MCS_NSS_DESC_MAX_LEN	5
-+
-+/**
-+ * enum ieee80211_he_mcs_support - HE MCS support definitions
-+ * @IEEE80211_HE_MCS_SUPPORT_0_7: MCSes 0-7 are supported for the
-+ *	number of streams
-+ * @IEEE80211_HE_MCS_SUPPORT_0_9: MCSes 0-9 are supported
-+ * @IEEE80211_HE_MCS_SUPPORT_0_11: MCSes 0-11 are supported
-+ * @IEEE80211_HE_MCS_NOT_SUPPORTED: This number of streams isn't supported
-+ *
-+ * These definitions are used in each 2-bit subfield of the rx_mcs_*
-+ * and tx_mcs_* fields of &struct ieee80211_he_mcs_nss_supp, which are
-+ * both split into 8 subfields by number of streams. These values indicate
-+ * which MCSes are supported for the number of streams the value appears
-+ * for.
-+ */
-+enum ieee80211_he_mcs_support {
-+	IEEE80211_HE_MCS_SUPPORT_0_7	= 0,
-+	IEEE80211_HE_MCS_SUPPORT_0_9	= 1,
-+	IEEE80211_HE_MCS_SUPPORT_0_11	= 2,
-+	IEEE80211_HE_MCS_NOT_SUPPORTED	= 3,
-+};
-+
-+/**
-+ * struct ieee80211_he_mcs_nss_supp - HE Tx/Rx HE MCS NSS Support Field
-+ *
-+ * This structure holds the data required for the Tx/Rx HE MCS NSS Support Field
-+ * described in P802.11ax_D2.0 section 9.4.2.237.4
-+ *
-+ * @rx_mcs_80: Rx MCS map 2 bits for each stream, total 8 streams, for channel
-+ *     widths less than 80MHz.
-+ * @tx_mcs_80: Tx MCS map 2 bits for each stream, total 8 streams, for channel
-+ *     widths less than 80MHz.
-+ * @rx_mcs_160: Rx MCS map 2 bits for each stream, total 8 streams, for channel
-+ *     width 160MHz.
-+ * @tx_mcs_160: Tx MCS map 2 bits for each stream, total 8 streams, for channel
-+ *     width 160MHz.
-+ * @rx_mcs_80p80: Rx MCS map 2 bits for each stream, total 8 streams, for
-+ *     channel width 80p80MHz.
-+ * @tx_mcs_80p80: Tx MCS map 2 bits for each stream, total 8 streams, for
-+ *     channel width 80p80MHz.
-+ */
-+struct ieee80211_he_mcs_nss_supp {
-+	__le16 rx_mcs_80;
-+	__le16 tx_mcs_80;
-+	__le16 rx_mcs_160;
-+	__le16 tx_mcs_160;
-+	__le16 rx_mcs_80p80;
-+	__le16 tx_mcs_80p80;
-+} __packed;
-+
-+/**
-+ * struct ieee80211_he_operation - HE Operation element
-+ * @he_oper_params: HE Operation Parameters + BSS Color Information
-+ * @he_mcs_nss_set: Basic HE-MCS And NSS Set
-+ * @optional: Optional fields VHT Operation Information, Max Co-Hosted
-+ *            BSSID Indicator, and 6 GHz Operation Information
-+ *
-+ * This structure represents the payload of the "HE Operation
-+ * element" as described in IEEE Std 802.11ax-2021 section 9.4.2.249.
-+ */
-+struct ieee80211_he_operation {
-+	__le32 he_oper_params;
-+	__le16 he_mcs_nss_set;
-+	u8 optional[];
-+} __packed;
-+
-+/**
-+ * struct ieee80211_he_spr - Spatial Reuse Parameter Set element
-+ * @he_sr_control: SR Control
-+ * @optional: Optional fields Non-SRG OBSS PD Max Offset, SRG OBSS PD
-+ *            Min Offset, SRG OBSS PD Max Offset, SRG BSS Color
-+ *            Bitmap, and SRG Partial BSSID Bitmap
-+ *
-+ * This structure represents the payload of the "Spatial Reuse
-+ * Parameter Set element" as described in IEEE Std 802.11ax-2021
-+ * section 9.4.2.252.
-+ */
-+struct ieee80211_he_spr {
-+	u8 he_sr_control;
-+	u8 optional[];
-+} __packed;
-+
-+/**
-+ * struct ieee80211_he_mu_edca_param_ac_rec - MU AC Parameter Record field
-+ * @aifsn: ACI/AIFSN
-+ * @ecw_min_max: ECWmin/ECWmax
-+ * @mu_edca_timer: MU EDCA Timer
-+ *
-+ * This structure represents the "MU AC Parameter Record" as described
-+ * in IEEE Std 802.11ax-2021 section 9.4.2.251, Figure 9-788p.
-+ */
-+struct ieee80211_he_mu_edca_param_ac_rec {
-+	u8 aifsn;
-+	u8 ecw_min_max;
-+	u8 mu_edca_timer;
-+} __packed;
-+
-+/**
-+ * struct ieee80211_mu_edca_param_set - MU EDCA Parameter Set element
-+ * @mu_qos_info: QoS Info
-+ * @ac_be: MU AC_BE Parameter Record
-+ * @ac_bk: MU AC_BK Parameter Record
-+ * @ac_vi: MU AC_VI Parameter Record
-+ * @ac_vo: MU AC_VO Parameter Record
-+ *
-+ * This structure represents the payload of the "MU EDCA Parameter Set
-+ * element" as described in IEEE Std 802.11ax-2021 section 9.4.2.251.
-+ */
-+struct ieee80211_mu_edca_param_set {
-+	u8 mu_qos_info;
-+	struct ieee80211_he_mu_edca_param_ac_rec ac_be;
-+	struct ieee80211_he_mu_edca_param_ac_rec ac_bk;
-+	struct ieee80211_he_mu_edca_param_ac_rec ac_vi;
-+	struct ieee80211_he_mu_edca_param_ac_rec ac_vo;
-+} __packed;
-+
-+/* 802.11ax HE MAC capabilities */
-+#define IEEE80211_HE_MAC_CAP0_HTC_HE				0x01
-+#define IEEE80211_HE_MAC_CAP0_TWT_REQ				0x02
-+#define IEEE80211_HE_MAC_CAP0_TWT_RES				0x04
-+#define IEEE80211_HE_MAC_CAP0_DYNAMIC_FRAG_NOT_SUPP		0x00
-+#define IEEE80211_HE_MAC_CAP0_DYNAMIC_FRAG_LEVEL_1		0x08
-+#define IEEE80211_HE_MAC_CAP0_DYNAMIC_FRAG_LEVEL_2		0x10
-+#define IEEE80211_HE_MAC_CAP0_DYNAMIC_FRAG_LEVEL_3		0x18
-+#define IEEE80211_HE_MAC_CAP0_DYNAMIC_FRAG_MASK			0x18
-+#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_1		0x00
-+#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_2		0x20
-+#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_4		0x40
-+#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_8		0x60
-+#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_16		0x80
-+#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_32		0xa0
-+#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_64		0xc0
-+#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_UNLIMITED	0xe0
-+#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_MASK		0xe0
-+
-+#define IEEE80211_HE_MAC_CAP1_MIN_FRAG_SIZE_UNLIMITED		0x00
-+#define IEEE80211_HE_MAC_CAP1_MIN_FRAG_SIZE_128			0x01
-+#define IEEE80211_HE_MAC_CAP1_MIN_FRAG_SIZE_256			0x02
-+#define IEEE80211_HE_MAC_CAP1_MIN_FRAG_SIZE_512			0x03
-+#define IEEE80211_HE_MAC_CAP1_MIN_FRAG_SIZE_MASK		0x03
-+#define IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_0US		0x00
-+#define IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_8US		0x04
-+#define IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US		0x08
-+#define IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_MASK		0x0c
-+#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_1		0x00
-+#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_2		0x10
-+#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_3		0x20
-+#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_4		0x30
-+#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_5		0x40
-+#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_6		0x50
-+#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_7		0x60
-+#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8		0x70
-+#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_MASK		0x70
-+
-+/* Link adaptation is split between byte HE_MAC_CAP1 and
-+ * HE_MAC_CAP2. It should be set only if IEEE80211_HE_MAC_CAP0_HTC_HE
-+ * in which case the following values apply:
-+ * 0 = No feedback.
-+ * 1 = reserved.
-+ * 2 = Unsolicited feedback.
-+ * 3 = both
-+ */
-+#define IEEE80211_HE_MAC_CAP1_LINK_ADAPTATION			0x80
-+
-+#define IEEE80211_HE_MAC_CAP2_LINK_ADAPTATION			0x01
-+#define IEEE80211_HE_MAC_CAP2_ALL_ACK				0x02
-+#define IEEE80211_HE_MAC_CAP2_TRS				0x04
-+#define IEEE80211_HE_MAC_CAP2_BSR				0x08
-+#define IEEE80211_HE_MAC_CAP2_BCAST_TWT				0x10
-+#define IEEE80211_HE_MAC_CAP2_32BIT_BA_BITMAP			0x20
-+#define IEEE80211_HE_MAC_CAP2_MU_CASCADING			0x40
-+#define IEEE80211_HE_MAC_CAP2_ACK_EN				0x80
-+
-+#define IEEE80211_HE_MAC_CAP3_OMI_CONTROL			0x02
-+#define IEEE80211_HE_MAC_CAP3_OFDMA_RA				0x04
-+
-+/* The maximum length of an A-MDPU is defined by the combination of the Maximum
-+ * A-MDPU Length Exponent field in the HT capabilities, VHT capabilities and the
-+ * same field in the HE capabilities.
-+ */
-+#define IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_0		0x00
-+#define IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_1		0x08
-+#define IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_2		0x10
-+#define IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3		0x18
-+#define IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_MASK		0x18
-+#define IEEE80211_HE_MAC_CAP3_AMSDU_FRAG			0x20
-+#define IEEE80211_HE_MAC_CAP3_FLEX_TWT_SCHED			0x40
-+#define IEEE80211_HE_MAC_CAP3_RX_CTRL_FRAME_TO_MULTIBSS		0x80
-+
-+#define IEEE80211_HE_MAC_CAP4_BSRP_BQRP_A_MPDU_AGG		0x01
-+#define IEEE80211_HE_MAC_CAP4_QTP				0x02
-+#define IEEE80211_HE_MAC_CAP4_BQR				0x04
-+#define IEEE80211_HE_MAC_CAP4_PSR_RESP				0x08
-+#define IEEE80211_HE_MAC_CAP4_NDP_FB_REP			0x10
-+#define IEEE80211_HE_MAC_CAP4_OPS				0x20
-+#define IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU			0x40
-+/* Multi TID agg TX is split between byte #4 and #5
-+ * The value is a combination of B39,B40,B41
-+ */
-+#define IEEE80211_HE_MAC_CAP4_MULTI_TID_AGG_TX_QOS_B39		0x80
-+
-+#define IEEE80211_HE_MAC_CAP5_MULTI_TID_AGG_TX_QOS_B40		0x01
-+#define IEEE80211_HE_MAC_CAP5_MULTI_TID_AGG_TX_QOS_B41		0x02
-+#define IEEE80211_HE_MAC_CAP5_SUBCHAN_SELECTIVE_TRANSMISSION	0x04
-+#define IEEE80211_HE_MAC_CAP5_UL_2x996_TONE_RU			0x08
-+#define IEEE80211_HE_MAC_CAP5_OM_CTRL_UL_MU_DATA_DIS_RX		0x10
-+#define IEEE80211_HE_MAC_CAP5_HE_DYNAMIC_SM_PS			0x20
-+#define IEEE80211_HE_MAC_CAP5_PUNCTURED_SOUNDING		0x40
-+#define IEEE80211_HE_MAC_CAP5_HT_VHT_TRIG_FRAME_RX		0x80
-+
-+#define IEEE80211_HE_VHT_MAX_AMPDU_FACTOR	20
-+#define IEEE80211_HE_HT_MAX_AMPDU_FACTOR	16
-+#define IEEE80211_HE_6GHZ_MAX_AMPDU_FACTOR	13
-+
-+/* 802.11ax HE PHY capabilities */
-+#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G		0x02
-+#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G	0x04
-+#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G		0x08
-+#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G	0x10
-+#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL		0x1e
-+
-+#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_RU_MAPPING_IN_2G	0x20
-+#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_RU_MAPPING_IN_5G	0x40
-+#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK			0xfe
-+
-+#define IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_80MHZ_ONLY_SECOND_20MHZ	0x01
-+#define IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_80MHZ_ONLY_SECOND_40MHZ	0x02
-+#define IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_160MHZ_ONLY_SECOND_20MHZ	0x04
-+#define IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_160MHZ_ONLY_SECOND_40MHZ	0x08
-+#define IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK			0x0f
-+#define IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A				0x10
-+#define IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD			0x20
-+#define IEEE80211_HE_PHY_CAP1_HE_LTF_AND_GI_FOR_HE_PPDUS_0_8US		0x40
-+/* Midamble RX/TX Max NSTS is split between byte #2 and byte #3 */
-+#define IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS			0x80
-+
-+#define IEEE80211_HE_PHY_CAP2_MIDAMBLE_RX_TX_MAX_NSTS			0x01
-+#define IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US			0x02
-+#define IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ			0x04
-+#define IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ			0x08
-+#define IEEE80211_HE_PHY_CAP2_DOPPLER_TX				0x10
-+#define IEEE80211_HE_PHY_CAP2_DOPPLER_RX				0x20
-+
-+/* Note that the meaning of UL MU below is different between an AP and a non-AP
-+ * sta, where in the AP case it indicates support for Rx and in the non-AP sta
-+ * case it indicates support for Tx.
-+ */
-+#define IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO			0x40
-+#define IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO			0x80
-+
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_NO_DCM			0x00
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_BPSK			0x01
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_QPSK			0x02
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_16_QAM			0x03
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_MASK			0x03
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_TX_NSS_1				0x00
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_TX_NSS_2				0x04
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_NO_DCM			0x00
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_BPSK			0x08
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_QPSK			0x10
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_16_QAM			0x18
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_MASK			0x18
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_RX_NSS_1				0x00
-+#define IEEE80211_HE_PHY_CAP3_DCM_MAX_RX_NSS_2				0x20
-+#define IEEE80211_HE_PHY_CAP3_RX_PARTIAL_BW_SU_IN_20MHZ_MU		0x40
-+#define IEEE80211_HE_PHY_CAP3_SU_BEAMFORMER				0x80
-+
-+#define IEEE80211_HE_PHY_CAP4_SU_BEAMFORMEE				0x01
-+#define IEEE80211_HE_PHY_CAP4_MU_BEAMFORMER				0x02
-+
-+/* Minimal allowed value of Max STS under 80MHz is 3 */
-+#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_4		0x0c
-+#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_5		0x10
-+#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_6		0x14
-+#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_7		0x18
-+#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_8		0x1c
-+#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_MASK	0x1c
-+
-+/* Minimal allowed value of Max STS above 80MHz is 3 */
-+#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4		0x60
-+#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_5		0x80
-+#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_6		0xa0
-+#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_7		0xc0
-+#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_8		0xe0
-+#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_MASK	0xe0
-+
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_1	0x00
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_2	0x01
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_3	0x02
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_4	0x03
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_5	0x04
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_6	0x05
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_7	0x06
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_8	0x07
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_MASK	0x07
-+
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_1	0x00
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_2	0x08
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_3	0x10
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_4	0x18
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_5	0x20
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_6	0x28
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_7	0x30
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_8	0x38
-+#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_MASK	0x38
-+
-+#define IEEE80211_HE_PHY_CAP5_NG16_SU_FEEDBACK				0x40
-+#define IEEE80211_HE_PHY_CAP5_NG16_MU_FEEDBACK				0x80
-+
-+#define IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_42_SU			0x01
-+#define IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_75_MU			0x02
-+#define IEEE80211_HE_PHY_CAP6_TRIG_SU_BEAMFORMING_FB			0x04
-+#define IEEE80211_HE_PHY_CAP6_TRIG_MU_BEAMFORMING_PARTIAL_BW_FB		0x08
-+#define IEEE80211_HE_PHY_CAP6_TRIG_CQI_FB				0x10
-+#define IEEE80211_HE_PHY_CAP6_PARTIAL_BW_EXT_RANGE			0x20
-+#define IEEE80211_HE_PHY_CAP6_PARTIAL_BANDWIDTH_DL_MUMIMO		0x40
-+#define IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT			0x80
-+
-+#define IEEE80211_HE_PHY_CAP7_PSR_BASED_SR				0x01
-+#define IEEE80211_HE_PHY_CAP7_POWER_BOOST_FACTOR_SUPP			0x02
-+#define IEEE80211_HE_PHY_CAP7_HE_SU_MU_PPDU_4XLTF_AND_08_US_GI		0x04
-+#define IEEE80211_HE_PHY_CAP7_MAX_NC_1					0x08
-+#define IEEE80211_HE_PHY_CAP7_MAX_NC_2					0x10
-+#define IEEE80211_HE_PHY_CAP7_MAX_NC_3					0x18
-+#define IEEE80211_HE_PHY_CAP7_MAX_NC_4					0x20
-+#define IEEE80211_HE_PHY_CAP7_MAX_NC_5					0x28
-+#define IEEE80211_HE_PHY_CAP7_MAX_NC_6					0x30
-+#define IEEE80211_HE_PHY_CAP7_MAX_NC_7					0x38
-+#define IEEE80211_HE_PHY_CAP7_MAX_NC_MASK				0x38
-+#define IEEE80211_HE_PHY_CAP7_STBC_TX_ABOVE_80MHZ			0x40
-+#define IEEE80211_HE_PHY_CAP7_STBC_RX_ABOVE_80MHZ			0x80
-+
-+#define IEEE80211_HE_PHY_CAP8_HE_ER_SU_PPDU_4XLTF_AND_08_US_GI		0x01
-+#define IEEE80211_HE_PHY_CAP8_20MHZ_IN_40MHZ_HE_PPDU_IN_2G		0x02
-+#define IEEE80211_HE_PHY_CAP8_20MHZ_IN_160MHZ_HE_PPDU			0x04
-+#define IEEE80211_HE_PHY_CAP8_80MHZ_IN_160MHZ_HE_PPDU			0x08
-+#define IEEE80211_HE_PHY_CAP8_HE_ER_SU_1XLTF_AND_08_US_GI		0x10
-+#define IEEE80211_HE_PHY_CAP8_MIDAMBLE_RX_TX_2X_AND_1XLTF		0x20
-+#define IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_242				0x00
-+#define IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_484				0x40
-+#define IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_996				0x80
-+#define IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_2x996				0xc0
-+#define IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_MASK				0xc0
-+
-+#define IEEE80211_HE_PHY_CAP9_LONGER_THAN_16_SIGB_OFDM_SYM		0x01
-+#define IEEE80211_HE_PHY_CAP9_NON_TRIGGERED_CQI_FEEDBACK		0x02
-+#define IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU		0x04
-+#define IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU		0x08
-+#define IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_COMP_SIGB	0x10
-+#define IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_NON_COMP_SIGB	0x20
-+#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_0US			0x0
-+#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_8US			0x1
-+#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US			0x2
-+#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED		0x3
-+#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_POS			6
-+#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK			0xc0
-+
-+#define IEEE80211_HE_PHY_CAP10_HE_MU_M1RU_MAX_LTF			0x01
-+
-+/* 802.11ax HE TX/RX MCS NSS Support  */
-+#define IEEE80211_TX_RX_MCS_NSS_SUPP_HIGHEST_MCS_POS			(3)
-+#define IEEE80211_TX_RX_MCS_NSS_SUPP_TX_BITMAP_POS			(6)
-+#define IEEE80211_TX_RX_MCS_NSS_SUPP_RX_BITMAP_POS			(11)
-+#define IEEE80211_TX_RX_MCS_NSS_SUPP_TX_BITMAP_MASK			0x07c0
-+#define IEEE80211_TX_RX_MCS_NSS_SUPP_RX_BITMAP_MASK			0xf800
-+
-+/* TX/RX HE MCS Support field Highest MCS subfield encoding */
-+enum ieee80211_he_highest_mcs_supported_subfield_enc {
-+	HIGHEST_MCS_SUPPORTED_MCS7 = 0,
-+	HIGHEST_MCS_SUPPORTED_MCS8,
-+	HIGHEST_MCS_SUPPORTED_MCS9,
-+	HIGHEST_MCS_SUPPORTED_MCS10,
-+	HIGHEST_MCS_SUPPORTED_MCS11,
-+};
-+
-+/* Calculate 802.11ax HE capabilities IE Tx/Rx HE MCS NSS Support Field size */
-+static inline u8
-+ieee80211_he_mcs_nss_size(const struct ieee80211_he_cap_elem *he_cap)
-+{
-+	u8 count = 4;
-+
-+	if (he_cap->phy_cap_info[0] &
-+	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G)
-+		count += 4;
-+
-+	if (he_cap->phy_cap_info[0] &
-+	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G)
-+		count += 4;
-+
-+	return count;
-+}
-+
-+/* 802.11ax HE PPE Thresholds */
-+#define IEEE80211_PPE_THRES_NSS_SUPPORT_2NSS			(1)
-+#define IEEE80211_PPE_THRES_NSS_POS				(0)
-+#define IEEE80211_PPE_THRES_NSS_MASK				(7)
-+#define IEEE80211_PPE_THRES_RU_INDEX_BITMASK_2x966_AND_966_RU	\
-+	(BIT(5) | BIT(6))
-+#define IEEE80211_PPE_THRES_RU_INDEX_BITMASK_MASK		0x78
-+#define IEEE80211_PPE_THRES_RU_INDEX_BITMASK_POS		(3)
-+#define IEEE80211_PPE_THRES_INFO_PPET_SIZE			(3)
-+#define IEEE80211_HE_PPE_THRES_INFO_HEADER_SIZE			(7)
-+
-+/*
-+ * Calculate 802.11ax HE capabilities IE PPE field size
-+ * Input: Header byte of ppe_thres (first byte), and HE capa IE's PHY cap u8*
-+ */
-+static inline u8
-+ieee80211_he_ppe_size(u8 ppe_thres_hdr, const u8 *phy_cap_info)
-+{
-+	u8 n;
-+
-+	if ((phy_cap_info[6] &
-+	     IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) == 0)
-+		return 0;
-+
-+	n = hweight8(ppe_thres_hdr &
-+		     IEEE80211_PPE_THRES_RU_INDEX_BITMASK_MASK);
-+	n *= (1 + ((ppe_thres_hdr & IEEE80211_PPE_THRES_NSS_MASK) >>
-+		   IEEE80211_PPE_THRES_NSS_POS));
-+
-+	/*
-+	 * Each pair is 6 bits, and we need to add the 7 "header" bits to the
-+	 * total size.
-+	 */
-+	n = (n * IEEE80211_PPE_THRES_INFO_PPET_SIZE * 2) + 7;
-+	n = DIV_ROUND_UP(n, 8);
-+
-+	return n;
-+}
-+
-+static inline bool ieee80211_he_capa_size_ok(const u8 *data, u8 len)
-+{
-+	const struct ieee80211_he_cap_elem *he_cap_ie_elem = (const void *)data;
-+	u8 needed = sizeof(*he_cap_ie_elem);
-+
-+	if (len < needed)
-+		return false;
-+
-+	needed += ieee80211_he_mcs_nss_size(he_cap_ie_elem);
-+	if (len < needed)
-+		return false;
-+
-+	if (he_cap_ie_elem->phy_cap_info[6] &
-+			IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) {
-+		if (len < needed + 1)
-+			return false;
-+		needed += ieee80211_he_ppe_size(data[needed],
-+						he_cap_ie_elem->phy_cap_info);
-+	}
-+
-+	return len >= needed;
-+}
-+
-+/* HE Operation defines */
-+#define IEEE80211_HE_OPERATION_DFLT_PE_DURATION_MASK		0x00000007
-+#define IEEE80211_HE_OPERATION_TWT_REQUIRED			0x00000008
-+#define IEEE80211_HE_OPERATION_RTS_THRESHOLD_MASK		0x00003ff0
-+#define IEEE80211_HE_OPERATION_RTS_THRESHOLD_OFFSET		4
-+#define IEEE80211_HE_OPERATION_VHT_OPER_INFO			0x00004000
-+#define IEEE80211_HE_OPERATION_CO_HOSTED_BSS			0x00008000
-+#define IEEE80211_HE_OPERATION_ER_SU_DISABLE			0x00010000
-+#define IEEE80211_HE_OPERATION_6GHZ_OP_INFO			0x00020000
-+#define IEEE80211_HE_OPERATION_BSS_COLOR_MASK			0x3f000000
-+#define IEEE80211_HE_OPERATION_BSS_COLOR_OFFSET			24
-+#define IEEE80211_HE_OPERATION_PARTIAL_BSS_COLOR		0x40000000
-+#define IEEE80211_HE_OPERATION_BSS_COLOR_DISABLED		0x80000000
-+
-+#define IEEE80211_6GHZ_CTRL_REG_LPI_AP			0
-+#define IEEE80211_6GHZ_CTRL_REG_SP_AP			1
-+#define IEEE80211_6GHZ_CTRL_REG_VLP_AP			2
-+#define IEEE80211_6GHZ_CTRL_REG_INDOOR_LPI_AP		3
-+#define IEEE80211_6GHZ_CTRL_REG_INDOOR_SP_AP_OLD	4
-+#define IEEE80211_6GHZ_CTRL_REG_AP_ROLE_NOT_RELEVANT	7
-+#define IEEE80211_6GHZ_CTRL_REG_INDOOR_SP_AP		8
-+
-+/**
-+ * struct ieee80211_he_6ghz_oper - HE 6 GHz operation Information field
-+ * @primary: primary channel
-+ * @control: control flags
-+ * @ccfs0: channel center frequency segment 0
-+ * @ccfs1: channel center frequency segment 1
-+ * @minrate: minimum rate (in 1 Mbps units)
-+ */
-+struct ieee80211_he_6ghz_oper {
-+	u8 primary;
-+#define IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH	0x3
-+#define		IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_20MHZ	0
-+#define		IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_40MHZ	1
-+#define		IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_80MHZ	2
-+#define		IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_160MHZ	3
-+#define IEEE80211_HE_6GHZ_OPER_CTRL_DUP_BEACON	0x4
-+#define IEEE80211_HE_6GHZ_OPER_CTRL_REG_INFO	0x78
-+	u8 control;
-+	u8 ccfs0;
-+	u8 ccfs1;
-+	u8 minrate;
-+} __packed;
-+
-+/**
-+ * enum ieee80211_reg_conn_bits - represents Regulatory connectivity field bits.
-+ *
-+ * This enumeration defines bit flags used to represent regulatory connectivity
-+ * field bits.
-+ *
-+ * @IEEE80211_REG_CONN_LPI_VALID: Indicates whether the LPI bit is valid.
-+ * @IEEE80211_REG_CONN_LPI_VALUE: Represents the value of the LPI bit.
-+ * @IEEE80211_REG_CONN_SP_VALID: Indicates whether the SP bit is valid.
-+ * @IEEE80211_REG_CONN_SP_VALUE: Represents the value of the SP bit.
-+ */
-+enum ieee80211_reg_conn_bits {
-+	IEEE80211_REG_CONN_LPI_VALID = BIT(0),
-+	IEEE80211_REG_CONN_LPI_VALUE = BIT(1),
-+	IEEE80211_REG_CONN_SP_VALID = BIT(2),
-+	IEEE80211_REG_CONN_SP_VALUE = BIT(3),
-+};
-+
-+/* transmit power interpretation type of transmit power envelope element */
-+enum ieee80211_tx_power_intrpt_type {
-+	IEEE80211_TPE_LOCAL_EIRP,
-+	IEEE80211_TPE_LOCAL_EIRP_PSD,
-+	IEEE80211_TPE_REG_CLIENT_EIRP,
-+	IEEE80211_TPE_REG_CLIENT_EIRP_PSD,
-+};
-+
-+/* category type of transmit power envelope element */
-+enum ieee80211_tx_power_category_6ghz {
-+	IEEE80211_TPE_CAT_6GHZ_DEFAULT = 0,
-+	IEEE80211_TPE_CAT_6GHZ_SUBORDINATE = 1,
-+};
-+
-+/*
-+ * For IEEE80211_TPE_LOCAL_EIRP / IEEE80211_TPE_REG_CLIENT_EIRP,
-+ * setting to 63.5 dBm means no constraint.
-+ */
-+#define IEEE80211_TPE_MAX_TX_PWR_NO_CONSTRAINT	127
-+
-+/*
-+ * For IEEE80211_TPE_LOCAL_EIRP_PSD / IEEE80211_TPE_REG_CLIENT_EIRP_PSD,
-+ * setting to 127 indicates no PSD limit for the 20 MHz channel.
-+ */
-+#define IEEE80211_TPE_PSD_NO_LIMIT		127
-+
-+/**
-+ * struct ieee80211_tx_pwr_env - Transmit Power Envelope
-+ * @info: Transmit Power Information field
-+ * @variable: Maximum Transmit Power field
-+ *
-+ * This structure represents the payload of the "Transmit Power
-+ * Envelope element" as described in IEEE Std 802.11ax-2021 section
-+ * 9.4.2.161
-+ */
-+struct ieee80211_tx_pwr_env {
-+	u8 info;
-+	u8 variable[];
-+} __packed;
-+
-+#define IEEE80211_TX_PWR_ENV_INFO_COUNT 0x7
-+#define IEEE80211_TX_PWR_ENV_INFO_INTERPRET 0x38
-+#define IEEE80211_TX_PWR_ENV_INFO_CATEGORY 0xC0
-+
-+#define IEEE80211_TX_PWR_ENV_EXT_COUNT	0xF
-+
-+static inline bool ieee80211_valid_tpe_element(const u8 *data, u8 len)
-+{
-+	const struct ieee80211_tx_pwr_env *env = (const void *)data;
-+	u8 count, interpret, category;
-+	u8 needed = sizeof(*env);
-+	u8 N; /* also called N in the spec */
-+
-+	if (len < needed)
-+		return false;
-+
-+	count = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_COUNT);
-+	interpret = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_INTERPRET);
-+	category = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_CATEGORY);
-+
-+	switch (category) {
-+	case IEEE80211_TPE_CAT_6GHZ_DEFAULT:
-+	case IEEE80211_TPE_CAT_6GHZ_SUBORDINATE:
-+		break;
-+	default:
-+		return false;
-+	}
-+
-+	switch (interpret) {
-+	case IEEE80211_TPE_LOCAL_EIRP:
-+	case IEEE80211_TPE_REG_CLIENT_EIRP:
-+		if (count > 3)
-+			return false;
-+
-+		/* count == 0 encodes 1 value for 20 MHz, etc. */
-+		needed += count + 1;
-+
-+		if (len < needed)
-+			return false;
-+
-+		/* there can be extension fields not accounted for in 'count' */
-+
-+		return true;
-+	case IEEE80211_TPE_LOCAL_EIRP_PSD:
-+	case IEEE80211_TPE_REG_CLIENT_EIRP_PSD:
-+		if (count > 4)
-+			return false;
-+
-+		N = count ? 1 << (count - 1) : 1;
-+		needed += N;
-+
-+		if (len < needed)
-+			return false;
-+
-+		if (len > needed) {
-+			u8 K = u8_get_bits(env->variable[N],
-+					   IEEE80211_TX_PWR_ENV_EXT_COUNT);
-+
-+			needed += 1 + K;
-+			if (len < needed)
-+				return false;
-+		}
-+
-+		return true;
-+	}
-+
-+	return false;
-+}
-+
-+/*
-+ * ieee80211_he_oper_size - calculate 802.11ax HE Operations IE size
-+ * @he_oper_ie: byte data of the He Operations IE, stating from the byte
-+ *	after the ext ID byte. It is assumed that he_oper_ie has at least
-+ *	sizeof(struct ieee80211_he_operation) bytes, the caller must have
-+ *	validated this.
-+ * @return the actual size of the IE data (not including header), or 0 on error
-+ */
-+static inline u8
-+ieee80211_he_oper_size(const u8 *he_oper_ie)
-+{
-+	const struct ieee80211_he_operation *he_oper = (const void *)he_oper_ie;
-+	u8 oper_len = sizeof(struct ieee80211_he_operation);
-+	u32 he_oper_params;
-+
-+	/* Make sure the input is not NULL */
-+	if (!he_oper_ie)
-+		return 0;
-+
-+	/* Calc required length */
-+	he_oper_params = le32_to_cpu(he_oper->he_oper_params);
-+	if (he_oper_params & IEEE80211_HE_OPERATION_VHT_OPER_INFO)
-+		oper_len += 3;
-+	if (he_oper_params & IEEE80211_HE_OPERATION_CO_HOSTED_BSS)
-+		oper_len++;
-+	if (he_oper_params & IEEE80211_HE_OPERATION_6GHZ_OP_INFO)
-+		oper_len += sizeof(struct ieee80211_he_6ghz_oper);
-+
-+	/* Add the first byte (extension ID) to the total length */
-+	oper_len++;
-+
-+	return oper_len;
-+}
-+
-+/**
-+ * ieee80211_he_6ghz_oper - obtain 6 GHz operation field
-+ * @he_oper: HE operation element (must be pre-validated for size)
-+ *	but may be %NULL
-+ *
-+ * Return: a pointer to the 6 GHz operation field, or %NULL
-+ */
-+static inline const struct ieee80211_he_6ghz_oper *
-+ieee80211_he_6ghz_oper(const struct ieee80211_he_operation *he_oper)
-+{
-+	const u8 *ret;
-+	u32 he_oper_params;
-+
-+	if (!he_oper)
-+		return NULL;
-+
-+	ret = (const void *)&he_oper->optional;
-+
-+	he_oper_params = le32_to_cpu(he_oper->he_oper_params);
-+
-+	if (!(he_oper_params & IEEE80211_HE_OPERATION_6GHZ_OP_INFO))
-+		return NULL;
-+	if (he_oper_params & IEEE80211_HE_OPERATION_VHT_OPER_INFO)
-+		ret += 3;
-+	if (he_oper_params & IEEE80211_HE_OPERATION_CO_HOSTED_BSS)
-+		ret++;
-+
-+	return (const void *)ret;
-+}
-+
-+/* HE Spatial Reuse defines */
-+#define IEEE80211_HE_SPR_PSR_DISALLOWED				BIT(0)
-+#define IEEE80211_HE_SPR_NON_SRG_OBSS_PD_SR_DISALLOWED		BIT(1)
-+#define IEEE80211_HE_SPR_NON_SRG_OFFSET_PRESENT			BIT(2)
-+#define IEEE80211_HE_SPR_SRG_INFORMATION_PRESENT		BIT(3)
-+#define IEEE80211_HE_SPR_HESIGA_SR_VAL15_ALLOWED		BIT(4)
-+
-+/*
-+ * ieee80211_he_spr_size - calculate 802.11ax HE Spatial Reuse IE size
-+ * @he_spr_ie: byte data of the He Spatial Reuse IE, stating from the byte
-+ *	after the ext ID byte. It is assumed that he_spr_ie has at least
-+ *	sizeof(struct ieee80211_he_spr) bytes, the caller must have validated
-+ *	this
-+ * @return the actual size of the IE data (not including header), or 0 on error
-+ */
-+static inline u8
-+ieee80211_he_spr_size(const u8 *he_spr_ie)
-+{
-+	const struct ieee80211_he_spr *he_spr = (const void *)he_spr_ie;
-+	u8 spr_len = sizeof(struct ieee80211_he_spr);
-+	u8 he_spr_params;
-+
-+	/* Make sure the input is not NULL */
-+	if (!he_spr_ie)
-+		return 0;
-+
-+	/* Calc required length */
-+	he_spr_params = he_spr->he_sr_control;
-+	if (he_spr_params & IEEE80211_HE_SPR_NON_SRG_OFFSET_PRESENT)
-+		spr_len++;
-+	if (he_spr_params & IEEE80211_HE_SPR_SRG_INFORMATION_PRESENT)
-+		spr_len += 18;
-+
-+	/* Add the first byte (extension ID) to the total length */
-+	spr_len++;
-+
-+	return spr_len;
-+}
-+
-+struct ieee80211_he_6ghz_capa {
-+	/* uses IEEE80211_HE_6GHZ_CAP_* below */
-+	__le16 capa;
-+} __packed;
-+
-+/* HE 6 GHz band capabilities */
-+/* uses enum ieee80211_min_mpdu_spacing values */
-+#define IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START	0x0007
-+/* uses enum ieee80211_vht_max_ampdu_length_exp values */
-+#define IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP	0x0038
-+/* uses IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_* values */
-+#define IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN	0x00c0
-+/* WLAN_HT_CAP_SM_PS_* values */
-+#define IEEE80211_HE_6GHZ_CAP_SM_PS		0x0600
-+#define IEEE80211_HE_6GHZ_CAP_RD_RESPONDER	0x0800
-+#define IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS	0x1000
-+#define IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS	0x2000
-+
-+#endif /* LINUX_IEEE80211_HE_H */
-diff --git a/include/linux/ieee80211-ht.h b/include/linux/ieee80211-ht.h
-new file mode 100644
-index 000000000000..21bbf470540f
--- /dev/null
-+++ b/include/linux/ieee80211-ht.h
-@@ -0,0 +1,292 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * IEEE 802.11 HT definitions
-+ *
-+ * Copyright (c) 2001-2002, SSH Communications Security Corp and Jouni Malinen
-+ * <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2002-2003, Jouni Malinen <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2005, Devicescape Software, Inc.
-+ * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net>
-+ * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH
-+ * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
-+ * Copyright (c) 2018 - 2025 Intel Corporation
-+ */
-+
-+#ifndef LINUX_IEEE80211_HT_H
-+#define LINUX_IEEE80211_HT_H
-+
-+#include <linux/types.h>
-+#include <linux/if_ether.h>
-+
-+/* Maximal size of an A-MSDU that can be transported in a HT BA session */
-+#define IEEE80211_MAX_MPDU_LEN_HT_BA		4095
-+
-+/* Maximal size of an A-MSDU */
-+#define IEEE80211_MAX_MPDU_LEN_HT_3839		3839
-+#define IEEE80211_MAX_MPDU_LEN_HT_7935		7935
-+
-+#define IEEE80211_HT_CTL_LEN		4
-+
-+enum ieee80211_ht_chanwidth_values {
-+	IEEE80211_HT_CHANWIDTH_20MHZ = 0,
-+	IEEE80211_HT_CHANWIDTH_ANY = 1,
-+};
-+
-+/**
-+ * struct ieee80211_bar - Block Ack Request frame format
-+ * @frame_control: Frame Control
-+ * @duration: Duration
-+ * @ra: RA
-+ * @ta: TA
-+ * @control: BAR Control
-+ * @start_seq_num: Starting Sequence Number (see Figure 9-37)
-+ *
-+ * This structure represents the "BlockAckReq frame format"
-+ * as described in IEEE Std 802.11-2020 section 9.3.1.7.
-+*/
-+struct ieee80211_bar {
-+	__le16 frame_control;
-+	__le16 duration;
-+	__u8 ra[ETH_ALEN];
-+	__u8 ta[ETH_ALEN];
-+	__le16 control;
-+	__le16 start_seq_num;
-+} __packed;
-+
-+/* 802.11 BAR control masks */
-+#define IEEE80211_BAR_CTRL_ACK_POLICY_NORMAL	0x0000
-+#define IEEE80211_BAR_CTRL_MULTI_TID		0x0002
-+#define IEEE80211_BAR_CTRL_CBMTID_COMPRESSED_BA	0x0004
-+#define IEEE80211_BAR_CTRL_TID_INFO_MASK	0xf000
-+#define IEEE80211_BAR_CTRL_TID_INFO_SHIFT	12
-+
-+#define IEEE80211_HT_MCS_MASK_LEN		10
-+
-+/**
-+ * struct ieee80211_mcs_info - Supported MCS Set field
-+ * @rx_mask: RX mask
-+ * @rx_highest: highest supported RX rate. If set represents
-+ *	the highest supported RX data rate in units of 1 Mbps.
-+ *	If this field is 0 this value should not be used to
-+ *	consider the highest RX data rate supported.
-+ * @tx_params: TX parameters
-+ * @reserved: Reserved bits
-+ *
-+ * This structure represents the "Supported MCS Set field" as
-+ * described in IEEE Std 802.11-2020 section 9.4.2.55.4.
-+ */
-+struct ieee80211_mcs_info {
-+	u8 rx_mask[IEEE80211_HT_MCS_MASK_LEN];
-+	__le16 rx_highest;
-+	u8 tx_params;
-+	u8 reserved[3];
-+} __packed;
-+
-+/* 802.11n HT capability MSC set */
-+#define IEEE80211_HT_MCS_RX_HIGHEST_MASK	0x3ff
-+#define IEEE80211_HT_MCS_TX_DEFINED		0x01
-+#define IEEE80211_HT_MCS_TX_RX_DIFF		0x02
-+/* value 0 == 1 stream etc */
-+#define IEEE80211_HT_MCS_TX_MAX_STREAMS_MASK	0x0C
-+#define IEEE80211_HT_MCS_TX_MAX_STREAMS_SHIFT	2
-+#define		IEEE80211_HT_MCS_TX_MAX_STREAMS	4
-+#define IEEE80211_HT_MCS_TX_UNEQUAL_MODULATION	0x10
-+
-+#define IEEE80211_HT_MCS_CHAINS(mcs) ((mcs) == 32 ? 1 : (1 + ((mcs) >> 3)))
-+
-+/*
-+ * 802.11n D5.0 20.3.5 / 20.6 says:
-+ * - indices 0 to 7 and 32 are single spatial stream
-+ * - 8 to 31 are multiple spatial streams using equal modulation
-+ *   [8..15 for two streams, 16..23 for three and 24..31 for four]
-+ * - remainder are multiple spatial streams using unequal modulation
-+ */
-+#define IEEE80211_HT_MCS_UNEQUAL_MODULATION_START 33
-+#define IEEE80211_HT_MCS_UNEQUAL_MODULATION_START_BYTE \
-+	(IEEE80211_HT_MCS_UNEQUAL_MODULATION_START / 8)
-+
-+/**
-+ * struct ieee80211_ht_cap - HT capabilities element
-+ * @cap_info: HT Capability Information
-+ * @ampdu_params_info: A-MPDU Parameters
-+ * @mcs: Supported MCS Set
-+ * @extended_ht_cap_info: HT Extended Capabilities
-+ * @tx_BF_cap_info: Transmit Beamforming Capabilities
-+ * @antenna_selection_info: ASEL Capability
-+ *
-+ * This structure represents the payload of the "HT Capabilities
-+ * element" as described in IEEE Std 802.11-2020 section 9.4.2.55.
-+ */
-+struct ieee80211_ht_cap {
-+	__le16 cap_info;
-+	u8 ampdu_params_info;
-+
-+	/* 16 bytes MCS information */
-+	struct ieee80211_mcs_info mcs;
-+
-+	__le16 extended_ht_cap_info;
-+	__le32 tx_BF_cap_info;
-+	u8 antenna_selection_info;
-+} __packed;
-+
-+/* 802.11n HT capabilities masks (for cap_info) */
-+#define IEEE80211_HT_CAP_LDPC_CODING		0x0001
-+#define IEEE80211_HT_CAP_SUP_WIDTH_20_40	0x0002
-+#define IEEE80211_HT_CAP_SM_PS			0x000C
-+#define		IEEE80211_HT_CAP_SM_PS_SHIFT	2
-+#define IEEE80211_HT_CAP_GRN_FLD		0x0010
-+#define IEEE80211_HT_CAP_SGI_20			0x0020
-+#define IEEE80211_HT_CAP_SGI_40			0x0040
-+#define IEEE80211_HT_CAP_TX_STBC		0x0080
-+#define IEEE80211_HT_CAP_RX_STBC		0x0300
-+#define		IEEE80211_HT_CAP_RX_STBC_SHIFT	8
-+#define IEEE80211_HT_CAP_DELAY_BA		0x0400
-+#define IEEE80211_HT_CAP_MAX_AMSDU		0x0800
-+#define IEEE80211_HT_CAP_DSSSCCK40		0x1000
-+#define IEEE80211_HT_CAP_RESERVED		0x2000
-+#define IEEE80211_HT_CAP_40MHZ_INTOLERANT	0x4000
-+#define IEEE80211_HT_CAP_LSIG_TXOP_PROT		0x8000
-+
-+/* 802.11n HT extended capabilities masks (for extended_ht_cap_info) */
-+#define IEEE80211_HT_EXT_CAP_PCO		0x0001
-+#define IEEE80211_HT_EXT_CAP_PCO_TIME		0x0006
-+#define		IEEE80211_HT_EXT_CAP_PCO_TIME_SHIFT	1
-+#define IEEE80211_HT_EXT_CAP_MCS_FB		0x0300
-+#define		IEEE80211_HT_EXT_CAP_MCS_FB_SHIFT	8
-+#define IEEE80211_HT_EXT_CAP_HTC_SUP		0x0400
-+#define IEEE80211_HT_EXT_CAP_RD_RESPONDER	0x0800
-+
-+/* 802.11n HT capability AMPDU settings (for ampdu_params_info) */
-+#define IEEE80211_HT_AMPDU_PARM_FACTOR		0x03
-+#define IEEE80211_HT_AMPDU_PARM_DENSITY		0x1C
-+#define		IEEE80211_HT_AMPDU_PARM_DENSITY_SHIFT	2
-+
-+/*
-+ * Maximum length of AMPDU that the STA can receive in high-throughput (HT).
-+ * Length = 2 ^ (13 + max_ampdu_length_exp) - 1 (octets)
-+ */
-+enum ieee80211_max_ampdu_length_exp {
-+	IEEE80211_HT_MAX_AMPDU_8K = 0,
-+	IEEE80211_HT_MAX_AMPDU_16K = 1,
-+	IEEE80211_HT_MAX_AMPDU_32K = 2,
-+	IEEE80211_HT_MAX_AMPDU_64K = 3
-+};
-+
-+#define IEEE80211_HT_MAX_AMPDU_FACTOR 13
-+
-+/* Minimum MPDU start spacing */
-+enum ieee80211_min_mpdu_spacing {
-+	IEEE80211_HT_MPDU_DENSITY_NONE = 0,	/* No restriction */
-+	IEEE80211_HT_MPDU_DENSITY_0_25 = 1,	/* 1/4 usec */
-+	IEEE80211_HT_MPDU_DENSITY_0_5 = 2,	/* 1/2 usec */
-+	IEEE80211_HT_MPDU_DENSITY_1 = 3,	/* 1 usec */
-+	IEEE80211_HT_MPDU_DENSITY_2 = 4,	/* 2 usec */
-+	IEEE80211_HT_MPDU_DENSITY_4 = 5,	/* 4 usec */
-+	IEEE80211_HT_MPDU_DENSITY_8 = 6,	/* 8 usec */
-+	IEEE80211_HT_MPDU_DENSITY_16 = 7	/* 16 usec */
-+};
-+
-+/**
-+ * struct ieee80211_ht_operation - HT operation IE
-+ * @primary_chan: Primary Channel
-+ * @ht_param: HT Operation Information parameters
-+ * @operation_mode: HT Operation Information operation mode
-+ * @stbc_param: HT Operation Information STBC params
-+ * @basic_set: Basic HT-MCS Set
-+ *
-+ * This structure represents the payload of the "HT Operation
-+ * element" as described in IEEE Std 802.11-2020 section 9.4.2.56.
-+ */
-+struct ieee80211_ht_operation {
-+	u8 primary_chan;
-+	u8 ht_param;
-+	__le16 operation_mode;
-+	__le16 stbc_param;
-+	u8 basic_set[16];
-+} __packed;
-+
-+/* for ht_param */
-+#define IEEE80211_HT_PARAM_CHA_SEC_OFFSET		0x03
-+#define		IEEE80211_HT_PARAM_CHA_SEC_NONE		0x00
-+#define		IEEE80211_HT_PARAM_CHA_SEC_ABOVE	0x01
-+#define		IEEE80211_HT_PARAM_CHA_SEC_BELOW	0x03
-+#define IEEE80211_HT_PARAM_CHAN_WIDTH_ANY		0x04
-+#define IEEE80211_HT_PARAM_RIFS_MODE			0x08
-+
-+/* for operation_mode */
-+#define IEEE80211_HT_OP_MODE_PROTECTION			0x0003
-+#define		IEEE80211_HT_OP_MODE_PROTECTION_NONE		0
-+#define		IEEE80211_HT_OP_MODE_PROTECTION_NONMEMBER	1
-+#define		IEEE80211_HT_OP_MODE_PROTECTION_20MHZ		2
-+#define		IEEE80211_HT_OP_MODE_PROTECTION_NONHT_MIXED	3
-+#define IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT		0x0004
-+#define IEEE80211_HT_OP_MODE_NON_HT_STA_PRSNT		0x0010
-+#define IEEE80211_HT_OP_MODE_CCFS2_SHIFT		5
-+#define IEEE80211_HT_OP_MODE_CCFS2_MASK			0x1fe0
-+
-+/* for stbc_param */
-+#define IEEE80211_HT_STBC_PARAM_DUAL_BEACON		0x0040
-+#define IEEE80211_HT_STBC_PARAM_DUAL_CTS_PROT		0x0080
-+#define IEEE80211_HT_STBC_PARAM_STBC_BEACON		0x0100
-+#define IEEE80211_HT_STBC_PARAM_LSIG_TXOP_FULLPROT	0x0200
-+#define IEEE80211_HT_STBC_PARAM_PCO_ACTIVE		0x0400
-+#define IEEE80211_HT_STBC_PARAM_PCO_PHASE		0x0800
-+
-+
-+/* block-ack parameters */
-+#define IEEE80211_ADDBA_PARAM_AMSDU_MASK 0x0001
-+#define IEEE80211_ADDBA_PARAM_POLICY_MASK 0x0002
-+#define IEEE80211_ADDBA_PARAM_TID_MASK 0x003C
-+#define IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK 0xFFC0
-+#define IEEE80211_DELBA_PARAM_TID_MASK 0xF000
-+#define IEEE80211_DELBA_PARAM_INITIATOR_MASK 0x0800
-+
-+/*
-+ * A-MPDU buffer sizes
-+ * According to HT size varies from 8 to 64 frames
-+ * HE adds the ability to have up to 256 frames.
-+ * EHT adds the ability to have up to 1K frames.
-+ */
-+#define IEEE80211_MIN_AMPDU_BUF		0x8
-+#define IEEE80211_MAX_AMPDU_BUF_HT	0x40
-+#define IEEE80211_MAX_AMPDU_BUF_HE	0x100
-+#define IEEE80211_MAX_AMPDU_BUF_EHT	0x400
-+
-+
-+/* Spatial Multiplexing Power Save Modes (for capability) */
-+#define WLAN_HT_CAP_SM_PS_STATIC	0
-+#define WLAN_HT_CAP_SM_PS_DYNAMIC	1
-+#define WLAN_HT_CAP_SM_PS_INVALID	2
-+#define WLAN_HT_CAP_SM_PS_DISABLED	3
-+
-+/* for SM power control field lower two bits */
-+#define WLAN_HT_SMPS_CONTROL_DISABLED	0
-+#define WLAN_HT_SMPS_CONTROL_STATIC	1
-+#define WLAN_HT_SMPS_CONTROL_DYNAMIC	3
-+
-+/* HT action codes */
-+enum ieee80211_ht_actioncode {
-+	WLAN_HT_ACTION_NOTIFY_CHANWIDTH = 0,
-+	WLAN_HT_ACTION_SMPS = 1,
-+	WLAN_HT_ACTION_PSMP = 2,
-+	WLAN_HT_ACTION_PCO_PHASE = 3,
-+	WLAN_HT_ACTION_CSI = 4,
-+	WLAN_HT_ACTION_NONCOMPRESSED_BF = 5,
-+	WLAN_HT_ACTION_COMPRESSED_BF = 6,
-+	WLAN_HT_ACTION_ASEL_IDX_FEEDBACK = 7,
-+};
-+
-+/* BACK action code */
-+enum ieee80211_back_actioncode {
-+	WLAN_ACTION_ADDBA_REQ = 0,
-+	WLAN_ACTION_ADDBA_RESP = 1,
-+	WLAN_ACTION_DELBA = 2,
-+};
-+
-+/* BACK (block-ack) parties */
-+enum ieee80211_back_parties {
-+	WLAN_BACK_RECIPIENT = 0,
-+	WLAN_BACK_INITIATOR = 1,
-+};
-+
-+#endif /* LINUX_IEEE80211_HT_H */
-diff --git a/include/linux/ieee80211-mesh.h b/include/linux/ieee80211-mesh.h
-new file mode 100644
-index 000000000000..4b829bcb38b6
--- /dev/null
-+++ b/include/linux/ieee80211-mesh.h
-@@ -0,0 +1,230 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * IEEE 802.11 mesh definitions
-+ *
-+ * Copyright (c) 2001-2002, SSH Communications Security Corp and Jouni Malinen
-+ * <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2002-2003, Jouni Malinen <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2005, Devicescape Software, Inc.
-+ * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net>
-+ * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH
-+ * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
-+ * Copyright (c) 2018 - 2025 Intel Corporation
-+ */
-+
-+#ifndef LINUX_IEEE80211_MESH_H
-+#define LINUX_IEEE80211_MESH_H
-+
-+#include <linux/types.h>
-+#include <linux/if_ether.h>
-+
-+#define IEEE80211_MAX_MESH_ID_LEN	32
-+
-+struct ieee80211s_hdr {
-+	u8 flags;
-+	u8 ttl;
-+	__le32 seqnum;
-+	u8 eaddr1[ETH_ALEN];
-+	u8 eaddr2[ETH_ALEN];
-+} __packed __aligned(2);
-+
-+/* Mesh flags */
-+#define MESH_FLAGS_AE_A4 	0x1
-+#define MESH_FLAGS_AE_A5_A6	0x2
-+#define MESH_FLAGS_AE		0x3
-+#define MESH_FLAGS_PS_DEEP	0x4
-+
-+/**
-+ * enum ieee80211_preq_flags - mesh PREQ element flags
-+ *
-+ * @IEEE80211_PREQ_PROACTIVE_PREP_FLAG: proactive PREP subfield
-+ */
-+enum ieee80211_preq_flags {
-+	IEEE80211_PREQ_PROACTIVE_PREP_FLAG	= 1<<2,
-+};
-+
-+/**
-+ * enum ieee80211_preq_target_flags - mesh PREQ element per target flags
-+ *
-+ * @IEEE80211_PREQ_TO_FLAG: target only subfield
-+ * @IEEE80211_PREQ_USN_FLAG: unknown target HWMP sequence number subfield
-+ */
-+enum ieee80211_preq_target_flags {
-+	IEEE80211_PREQ_TO_FLAG	= 1<<0,
-+	IEEE80211_PREQ_USN_FLAG	= 1<<2,
-+};
-+
-+/**
-+ * struct ieee80211_mesh_chansw_params_ie - mesh channel switch parameters IE
-+ * @mesh_ttl: Time To Live
-+ * @mesh_flags: Flags
-+ * @mesh_reason: Reason Code
-+ * @mesh_pre_value: Precedence Value
-+ *
-+ * This structure represents the payload of the "Mesh Channel Switch
-+ * Parameters element" as described in IEEE Std 802.11-2020 section
-+ * 9.4.2.102.
-+ */
-+struct ieee80211_mesh_chansw_params_ie {
-+	u8 mesh_ttl;
-+	u8 mesh_flags;
-+	__le16 mesh_reason;
-+	__le16 mesh_pre_value;
-+} __packed;
-+
-+/**
-+ * struct ieee80211_meshconf_ie - Mesh Configuration element
-+ * @meshconf_psel: Active Path Selection Protocol Identifier
-+ * @meshconf_pmetric: Active Path Selection Metric Identifier
-+ * @meshconf_congest: Congestion Control Mode Identifier
-+ * @meshconf_synch: Synchronization Method Identifier
-+ * @meshconf_auth: Authentication Protocol Identifier
-+ * @meshconf_form: Mesh Formation Info
-+ * @meshconf_cap: Mesh Capability (see &enum mesh_config_capab_flags)
-+ *
-+ * This structure represents the payload of the "Mesh Configuration
-+ * element" as described in IEEE Std 802.11-2020 section 9.4.2.97.
-+ */
-+struct ieee80211_meshconf_ie {
-+	u8 meshconf_psel;
-+	u8 meshconf_pmetric;
-+	u8 meshconf_congest;
-+	u8 meshconf_synch;
-+	u8 meshconf_auth;
-+	u8 meshconf_form;
-+	u8 meshconf_cap;
-+} __packed;
-+
-+/**
-+ * enum mesh_config_capab_flags - Mesh Configuration IE capability field flags
-+ *
-+ * @IEEE80211_MESHCONF_CAPAB_ACCEPT_PLINKS: STA is willing to establish
-+ *	additional mesh peerings with other mesh STAs
-+ * @IEEE80211_MESHCONF_CAPAB_FORWARDING: the STA forwards MSDUs
-+ * @IEEE80211_MESHCONF_CAPAB_TBTT_ADJUSTING: TBTT adjustment procedure
-+ *	is ongoing
-+ * @IEEE80211_MESHCONF_CAPAB_POWER_SAVE_LEVEL: STA is in deep sleep mode or has
-+ *	neighbors in deep sleep mode
-+ *
-+ * Enumerates the "Mesh Capability" as described in IEEE Std
-+ * 802.11-2020 section 9.4.2.97.7.
-+ */
-+enum mesh_config_capab_flags {
-+	IEEE80211_MESHCONF_CAPAB_ACCEPT_PLINKS		= 0x01,
-+	IEEE80211_MESHCONF_CAPAB_FORWARDING		= 0x08,
-+	IEEE80211_MESHCONF_CAPAB_TBTT_ADJUSTING		= 0x20,
-+	IEEE80211_MESHCONF_CAPAB_POWER_SAVE_LEVEL	= 0x40,
-+};
-+
-+#define IEEE80211_MESHCONF_FORM_CONNECTED_TO_GATE 0x1
-+
-+/*
-+ * mesh channel switch parameters element's flag indicator
-+ *
-+ */
-+#define WLAN_EID_CHAN_SWITCH_PARAM_TX_RESTRICT BIT(0)
-+#define WLAN_EID_CHAN_SWITCH_PARAM_INITIATOR BIT(1)
-+#define WLAN_EID_CHAN_SWITCH_PARAM_REASON BIT(2)
-+
-+/**
-+ * struct ieee80211_rann_ie - RANN (root announcement) element
-+ * @rann_flags: Flags
-+ * @rann_hopcount: Hop Count
-+ * @rann_ttl: Element TTL
-+ * @rann_addr: Root Mesh STA Address
-+ * @rann_seq: HWMP Sequence Number
-+ * @rann_interval: Interval
-+ * @rann_metric: Metric
-+ *
-+ * This structure represents the payload of the "RANN element" as
-+ * described in IEEE Std 802.11-2020 section 9.4.2.111.
-+ */
-+struct ieee80211_rann_ie {
-+	u8 rann_flags;
-+	u8 rann_hopcount;
-+	u8 rann_ttl;
-+	u8 rann_addr[ETH_ALEN];
-+	__le32 rann_seq;
-+	__le32 rann_interval;
-+	__le32 rann_metric;
-+} __packed;
-+
-+enum ieee80211_rann_flags {
-+	RANN_FLAG_IS_GATE = 1 << 0,
-+};
-+
-+/* Mesh action codes */
-+enum ieee80211_mesh_actioncode {
-+	WLAN_MESH_ACTION_LINK_METRIC_REPORT,
-+	WLAN_MESH_ACTION_HWMP_PATH_SELECTION,
-+	WLAN_MESH_ACTION_GATE_ANNOUNCEMENT,
-+	WLAN_MESH_ACTION_CONGESTION_CONTROL_NOTIFICATION,
-+	WLAN_MESH_ACTION_MCCA_SETUP_REQUEST,
-+	WLAN_MESH_ACTION_MCCA_SETUP_REPLY,
-+	WLAN_MESH_ACTION_MCCA_ADVERTISEMENT_REQUEST,
-+	WLAN_MESH_ACTION_MCCA_ADVERTISEMENT,
-+	WLAN_MESH_ACTION_MCCA_TEARDOWN,
-+	WLAN_MESH_ACTION_TBTT_ADJUSTMENT_REQUEST,
-+	WLAN_MESH_ACTION_TBTT_ADJUSTMENT_RESPONSE,
-+};
-+
-+/**
-+ * enum ieee80211_mesh_sync_method - mesh synchronization method identifier
-+ *
-+ * @IEEE80211_SYNC_METHOD_NEIGHBOR_OFFSET: the default synchronization method
-+ * @IEEE80211_SYNC_METHOD_VENDOR: a vendor specific synchronization method
-+ *	that will be specified in a vendor specific information element
-+ */
-+enum ieee80211_mesh_sync_method {
-+	IEEE80211_SYNC_METHOD_NEIGHBOR_OFFSET = 1,
-+	IEEE80211_SYNC_METHOD_VENDOR = 255,
-+};
-+
-+/**
-+ * enum ieee80211_mesh_path_protocol - mesh path selection protocol identifier
-+ *
-+ * @IEEE80211_PATH_PROTOCOL_HWMP: the default path selection protocol
-+ * @IEEE80211_PATH_PROTOCOL_VENDOR: a vendor specific protocol that will
-+ *	be specified in a vendor specific information element
-+ */
-+enum ieee80211_mesh_path_protocol {
-+	IEEE80211_PATH_PROTOCOL_HWMP = 1,
-+	IEEE80211_PATH_PROTOCOL_VENDOR = 255,
-+};
-+
-+/**
-+ * enum ieee80211_mesh_path_metric - mesh path selection metric identifier
-+ *
-+ * @IEEE80211_PATH_METRIC_AIRTIME: the default path selection metric
-+ * @IEEE80211_PATH_METRIC_VENDOR: a vendor specific metric that will be
-+ *	specified in a vendor specific information element
-+ */
-+enum ieee80211_mesh_path_metric {
-+	IEEE80211_PATH_METRIC_AIRTIME = 1,
-+	IEEE80211_PATH_METRIC_VENDOR = 255,
-+};
-+
-+/**
-+ * enum ieee80211_root_mode_identifier - root mesh STA mode identifier
-+ *
-+ * These attribute are used by dot11MeshHWMPRootMode to set root mesh STA mode
-+ *
-+ * @IEEE80211_ROOTMODE_NO_ROOT: the mesh STA is not a root mesh STA (default)
-+ * @IEEE80211_ROOTMODE_ROOT: the mesh STA is a root mesh STA if greater than
-+ *	this value
-+ * @IEEE80211_PROACTIVE_PREQ_NO_PREP: the mesh STA is a root mesh STA supports
-+ *	the proactive PREQ with proactive PREP subfield set to 0
-+ * @IEEE80211_PROACTIVE_PREQ_WITH_PREP: the mesh STA is a root mesh STA
-+ *	supports the proactive PREQ with proactive PREP subfield set to 1
-+ * @IEEE80211_PROACTIVE_RANN: the mesh STA is a root mesh STA supports
-+ *	the proactive RANN
-+ */
-+enum ieee80211_root_mode_identifier {
-+	IEEE80211_ROOTMODE_NO_ROOT = 0,
-+	IEEE80211_ROOTMODE_ROOT = 1,
-+	IEEE80211_PROACTIVE_PREQ_NO_PREP = 2,
-+	IEEE80211_PROACTIVE_PREQ_WITH_PREP = 3,
-+	IEEE80211_PROACTIVE_RANN = 4,
-+};
-+
-+#endif /* LINUX_IEEE80211_MESH_H */
-diff --git a/include/linux/ieee80211-nan.h b/include/linux/ieee80211-nan.h
-new file mode 100644
-index 000000000000..d07959bf8a90
--- /dev/null
-+++ b/include/linux/ieee80211-nan.h
-@@ -0,0 +1,35 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * WFA NAN definitions
-+ *
-+ * Copyright (c) 2001-2002, SSH Communications Security Corp and Jouni Malinen
-+ * <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2002-2003, Jouni Malinen <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2005, Devicescape Software, Inc.
-+ * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net>
-+ * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH
-+ * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
-+ * Copyright (c) 2018 - 2025 Intel Corporation
-+ */
-+
-+#ifndef LINUX_IEEE80211_NAN_H
-+#define LINUX_IEEE80211_NAN_H
-+
-+/* NAN operation mode, as defined in Wi-Fi Aware (TM) specification Table 81 */
-+#define NAN_OP_MODE_PHY_MODE_VHT	0x01
-+#define NAN_OP_MODE_PHY_MODE_HE		0x10
-+#define NAN_OP_MODE_PHY_MODE_MASK	0x11
-+#define NAN_OP_MODE_80P80MHZ		0x02
-+#define NAN_OP_MODE_160MHZ		0x04
-+#define NAN_OP_MODE_PNDL_SUPPRTED	0x08
-+
-+/* NAN Device capabilities, as defined in Wi-Fi Aware (TM) specification
-+ * Table 79
-+ */
-+#define NAN_DEV_CAPA_DFS_OWNER			0x01
-+#define NAN_DEV_CAPA_EXT_KEY_ID_SUPPORTED	0x02
-+#define NAN_DEV_CAPA_SIM_NDP_RX_SUPPORTED	0x04
-+#define NAN_DEV_CAPA_NDPE_SUPPORTED		0x08
-+#define NAN_DEV_CAPA_S3_SUPPORTED		0x10
-+
-+#endif /* LINUX_IEEE80211_NAN_H */
-diff --git a/include/linux/ieee80211-p2p.h b/include/linux/ieee80211-p2p.h
-new file mode 100644
-index 000000000000..180891c11f08
--- /dev/null
-+++ b/include/linux/ieee80211-p2p.h
-@@ -0,0 +1,71 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * WFA P2P definitions
-+ *
-+ * Copyright (c) 2001-2002, SSH Communications Security Corp and Jouni Malinen
-+ * <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2002-2003, Jouni Malinen <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2005, Devicescape Software, Inc.
-+ * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net>
-+ * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH
-+ * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
-+ * Copyright (c) 2018 - 2025 Intel Corporation
-+ */
-+
-+#ifndef LINUX_IEEE80211_P2P_H
-+#define LINUX_IEEE80211_P2P_H
-+
-+#include <linux/types.h>
-+/*
-+ * Peer-to-Peer IE attribute related definitions.
-+ */
-+/*
-+ * enum ieee80211_p2p_attr_id - identifies type of peer-to-peer attribute.
-+ */
-+enum ieee80211_p2p_attr_id {
-+	IEEE80211_P2P_ATTR_STATUS = 0,
-+	IEEE80211_P2P_ATTR_MINOR_REASON,
-+	IEEE80211_P2P_ATTR_CAPABILITY,
-+	IEEE80211_P2P_ATTR_DEVICE_ID,
-+	IEEE80211_P2P_ATTR_GO_INTENT,
-+	IEEE80211_P2P_ATTR_GO_CONFIG_TIMEOUT,
-+	IEEE80211_P2P_ATTR_LISTEN_CHANNEL,
-+	IEEE80211_P2P_ATTR_GROUP_BSSID,
-+	IEEE80211_P2P_ATTR_EXT_LISTEN_TIMING,
-+	IEEE80211_P2P_ATTR_INTENDED_IFACE_ADDR,
-+	IEEE80211_P2P_ATTR_MANAGABILITY,
-+	IEEE80211_P2P_ATTR_CHANNEL_LIST,
-+	IEEE80211_P2P_ATTR_ABSENCE_NOTICE,
-+	IEEE80211_P2P_ATTR_DEVICE_INFO,
-+	IEEE80211_P2P_ATTR_GROUP_INFO,
-+	IEEE80211_P2P_ATTR_GROUP_ID,
-+	IEEE80211_P2P_ATTR_INTERFACE,
-+	IEEE80211_P2P_ATTR_OPER_CHANNEL,
-+	IEEE80211_P2P_ATTR_INVITE_FLAGS,
-+	/* 19 - 220: Reserved */
-+	IEEE80211_P2P_ATTR_VENDOR_SPECIFIC = 221,
-+
-+	IEEE80211_P2P_ATTR_MAX
-+};
-+
-+/* Notice of Absence attribute - described in P2P spec 4.1.14 */
-+/* Typical max value used here */
-+#define IEEE80211_P2P_NOA_DESC_MAX	4
-+
-+struct ieee80211_p2p_noa_desc {
-+	u8 count;
-+	__le32 duration;
-+	__le32 interval;
-+	__le32 start_time;
-+} __packed;
-+
-+struct ieee80211_p2p_noa_attr {
-+	u8 index;
-+	u8 oppps_ctwindow;
-+	struct ieee80211_p2p_noa_desc desc[IEEE80211_P2P_NOA_DESC_MAX];
-+} __packed;
-+
-+#define IEEE80211_P2P_OPPPS_ENABLE_BIT		BIT(7)
-+#define IEEE80211_P2P_OPPPS_CTWINDOW_MASK	0x7F
-+
-+#endif /* LINUX_IEEE80211_P2P_H */
-diff --git a/include/linux/ieee80211-s1g.h b/include/linux/ieee80211-s1g.h
-new file mode 100644
-index 000000000000..5b9ed2dcc00e
--- /dev/null
-+++ b/include/linux/ieee80211-s1g.h
-@@ -0,0 +1,575 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * IEEE 802.11 S1G definitions
-+ *
-+ * Copyright (c) 2001-2002, SSH Communications Security Corp and Jouni Malinen
-+ * <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2002-2003, Jouni Malinen <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2005, Devicescape Software, Inc.
-+ * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net>
-+ * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH
-+ * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
-+ * Copyright (c) 2018 - 2025 Intel Corporation
-+ */
-+
-+#ifndef LINUX_IEEE80211_S1G_H
-+#define LINUX_IEEE80211_S1G_H
-+
-+#include <linux/types.h>
-+#include <linux/if_ether.h>
-+
-+/* bits unique to S1G beacon frame control */
-+#define IEEE80211_S1G_BCN_NEXT_TBTT	0x100
-+#define IEEE80211_S1G_BCN_CSSID		0x200
-+#define IEEE80211_S1G_BCN_ANO		0x400
-+
-+/* see 802.11ah-2016 9.9 NDP CMAC frames */
-+#define IEEE80211_S1G_1MHZ_NDP_BITS	25
-+#define IEEE80211_S1G_1MHZ_NDP_BYTES	4
-+#define IEEE80211_S1G_2MHZ_NDP_BITS	37
-+#define IEEE80211_S1G_2MHZ_NDP_BYTES	5
-+
-+/**
-+ * ieee80211_is_s1g_beacon - check if IEEE80211_FTYPE_EXT &&
-+ * IEEE80211_STYPE_S1G_BEACON
-+ * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is an S1G beacon
-+ */
-+static inline bool ieee80211_is_s1g_beacon(__le16 fc)
-+{
-+	return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE |
-+				 IEEE80211_FCTL_STYPE)) ==
-+	       cpu_to_le16(IEEE80211_FTYPE_EXT | IEEE80211_STYPE_S1G_BEACON);
-+}
-+
-+/**
-+ * ieee80211_s1g_has_next_tbtt - check if IEEE80211_S1G_BCN_NEXT_TBTT
-+ * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame contains the variable-length
-+ *	next TBTT field
-+ */
-+static inline bool ieee80211_s1g_has_next_tbtt(__le16 fc)
-+{
-+	return ieee80211_is_s1g_beacon(fc) &&
-+		(fc & cpu_to_le16(IEEE80211_S1G_BCN_NEXT_TBTT));
-+}
-+
-+/**
-+ * ieee80211_s1g_has_ano - check if IEEE80211_S1G_BCN_ANO
-+ * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame contains the variable-length
-+ *	ANO field
-+ */
-+static inline bool ieee80211_s1g_has_ano(__le16 fc)
-+{
-+	return ieee80211_is_s1g_beacon(fc) &&
-+		(fc & cpu_to_le16(IEEE80211_S1G_BCN_ANO));
-+}
-+
-+/**
-+ * ieee80211_s1g_has_cssid - check if IEEE80211_S1G_BCN_CSSID
-+ * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame contains the variable-length
-+ *	compressed SSID field
-+ */
-+static inline bool ieee80211_s1g_has_cssid(__le16 fc)
-+{
-+	return ieee80211_is_s1g_beacon(fc) &&
-+		(fc & cpu_to_le16(IEEE80211_S1G_BCN_CSSID));
-+}
-+
-+/**
-+ * enum ieee80211_s1g_chanwidth - S1G channel widths
-+ * These are defined in IEEE802.11-2016ah Table 10-20
-+ * as BSS Channel Width
-+ *
-+ * @IEEE80211_S1G_CHANWIDTH_1MHZ: 1MHz operating channel
-+ * @IEEE80211_S1G_CHANWIDTH_2MHZ: 2MHz operating channel
-+ * @IEEE80211_S1G_CHANWIDTH_4MHZ: 4MHz operating channel
-+ * @IEEE80211_S1G_CHANWIDTH_8MHZ: 8MHz operating channel
-+ * @IEEE80211_S1G_CHANWIDTH_16MHZ: 16MHz operating channel
-+ */
-+enum ieee80211_s1g_chanwidth {
-+	IEEE80211_S1G_CHANWIDTH_1MHZ = 0,
-+	IEEE80211_S1G_CHANWIDTH_2MHZ = 1,
-+	IEEE80211_S1G_CHANWIDTH_4MHZ = 3,
-+	IEEE80211_S1G_CHANWIDTH_8MHZ = 7,
-+	IEEE80211_S1G_CHANWIDTH_16MHZ = 15,
-+};
-+
-+/**
-+ * enum ieee80211_s1g_pri_chanwidth - S1G primary channel widths
-+ *	described in IEEE80211-2024 Table 10-39.
-+ *
-+ * @IEEE80211_S1G_PRI_CHANWIDTH_2MHZ: 2MHz primary channel
-+ * @IEEE80211_S1G_PRI_CHANWIDTH_1MHZ: 1MHz primary channel
-+ */
-+enum ieee80211_s1g_pri_chanwidth {
-+	IEEE80211_S1G_PRI_CHANWIDTH_2MHZ = 0,
-+	IEEE80211_S1G_PRI_CHANWIDTH_1MHZ = 1,
-+};
-+
-+/**
-+ * struct ieee80211_s1g_bcn_compat_ie - S1G Beacon Compatibility element
-+ * @compat_info: Compatibility Information
-+ * @beacon_int: Beacon Interval
-+ * @tsf_completion: TSF Completion
-+ *
-+ * This structure represents the payload of the "S1G Beacon
-+ * Compatibility element" as described in IEEE Std 802.11-2020 section
-+ * 9.4.2.196.
-+ */
-+struct ieee80211_s1g_bcn_compat_ie {
-+	__le16 compat_info;
-+	__le16 beacon_int;
-+	__le32 tsf_completion;
-+} __packed;
-+
-+/**
-+ * struct ieee80211_s1g_oper_ie - S1G Operation element
-+ * @ch_width: S1G Operation Information Channel Width
-+ * @oper_class: S1G Operation Information Operating Class
-+ * @primary_ch: S1G Operation Information Primary Channel Number
-+ * @oper_ch: S1G Operation Information  Channel Center Frequency
-+ * @basic_mcs_nss: Basic S1G-MCS and NSS Set
-+ *
-+ * This structure represents the payload of the "S1G Operation
-+ * element" as described in IEEE Std 802.11-2020 section 9.4.2.212.
-+ */
-+struct ieee80211_s1g_oper_ie {
-+	u8 ch_width;
-+	u8 oper_class;
-+	u8 primary_ch;
-+	u8 oper_ch;
-+	__le16 basic_mcs_nss;
-+} __packed;
-+
-+/**
-+ * struct ieee80211_aid_response_ie - AID Response element
-+ * @aid: AID/Group AID
-+ * @switch_count: AID Switch Count
-+ * @response_int: AID Response Interval
-+ *
-+ * This structure represents the payload of the "AID Response element"
-+ * as described in IEEE Std 802.11-2020 section 9.4.2.194.
-+ */
-+struct ieee80211_aid_response_ie {
-+	__le16 aid;
-+	u8 switch_count;
-+	__le16 response_int;
-+} __packed;
-+
-+struct ieee80211_s1g_cap {
-+	u8 capab_info[10];
-+	u8 supp_mcs_nss[5];
-+} __packed;
-+
-+/**
-+ * ieee80211_s1g_optional_len - determine length of optional S1G beacon fields
-+ * @fc: frame control bytes in little-endian byteorder
-+ * Return: total length in bytes of the optional fixed-length fields
-+ *
-+ * S1G beacons may contain up to three optional fixed-length fields that
-+ * precede the variable-length elements. Whether these fields are present
-+ * is indicated by flags in the frame control field.
-+ *
-+ * From IEEE 802.11-2024 section 9.3.4.3:
-+ *  - Next TBTT field may be 0 or 3 bytes
-+ *  - Short SSID field may be 0 or 4 bytes
-+ *  - Access Network Options (ANO) field may be 0 or 1 byte
-+ */
-+static inline size_t
-+ieee80211_s1g_optional_len(__le16 fc)
-+{
-+	size_t len = 0;
-+
-+	if (ieee80211_s1g_has_next_tbtt(fc))
-+		len += 3;
-+
-+	if (ieee80211_s1g_has_cssid(fc))
-+		len += 4;
-+
-+	if (ieee80211_s1g_has_ano(fc))
-+		len += 1;
-+
-+	return len;
-+}
-+
-+/* S1G Capabilities Information field */
-+#define IEEE80211_S1G_CAPABILITY_LEN	15
-+
-+#define S1G_CAP0_S1G_LONG	BIT(0)
-+#define S1G_CAP0_SGI_1MHZ	BIT(1)
-+#define S1G_CAP0_SGI_2MHZ	BIT(2)
-+#define S1G_CAP0_SGI_4MHZ	BIT(3)
-+#define S1G_CAP0_SGI_8MHZ	BIT(4)
-+#define S1G_CAP0_SGI_16MHZ	BIT(5)
-+#define S1G_CAP0_SUPP_CH_WIDTH	GENMASK(7, 6)
-+
-+#define S1G_SUPP_CH_WIDTH_2	0
-+#define S1G_SUPP_CH_WIDTH_4	1
-+#define S1G_SUPP_CH_WIDTH_8	2
-+#define S1G_SUPP_CH_WIDTH_16	3
-+#define S1G_SUPP_CH_WIDTH_MAX(cap) ((1 << FIELD_GET(S1G_CAP0_SUPP_CH_WIDTH, \
-+						    cap[0])) << 1)
-+
-+#define S1G_CAP1_RX_LDPC	BIT(0)
-+#define S1G_CAP1_TX_STBC	BIT(1)
-+#define S1G_CAP1_RX_STBC	BIT(2)
-+#define S1G_CAP1_SU_BFER	BIT(3)
-+#define S1G_CAP1_SU_BFEE	BIT(4)
-+#define S1G_CAP1_BFEE_STS	GENMASK(7, 5)
-+
-+#define S1G_CAP2_SOUNDING_DIMENSIONS	GENMASK(2, 0)
-+#define S1G_CAP2_MU_BFER		BIT(3)
-+#define S1G_CAP2_MU_BFEE		BIT(4)
-+#define S1G_CAP2_PLUS_HTC_VHT		BIT(5)
-+#define S1G_CAP2_TRAVELING_PILOT	GENMASK(7, 6)
-+
-+#define S1G_CAP3_RD_RESPONDER		BIT(0)
-+#define S1G_CAP3_HT_DELAYED_BA		BIT(1)
-+#define S1G_CAP3_MAX_MPDU_LEN		BIT(2)
-+#define S1G_CAP3_MAX_AMPDU_LEN_EXP	GENMASK(4, 3)
-+#define S1G_CAP3_MIN_MPDU_START		GENMASK(7, 5)
-+
-+#define S1G_CAP4_UPLINK_SYNC	BIT(0)
-+#define S1G_CAP4_DYNAMIC_AID	BIT(1)
-+#define S1G_CAP4_BAT		BIT(2)
-+#define S1G_CAP4_TIME_ADE	BIT(3)
-+#define S1G_CAP4_NON_TIM	BIT(4)
-+#define S1G_CAP4_GROUP_AID	BIT(5)
-+#define S1G_CAP4_STA_TYPE	GENMASK(7, 6)
-+
-+#define S1G_CAP5_CENT_AUTH_CONTROL	BIT(0)
-+#define S1G_CAP5_DIST_AUTH_CONTROL	BIT(1)
-+#define S1G_CAP5_AMSDU			BIT(2)
-+#define S1G_CAP5_AMPDU			BIT(3)
-+#define S1G_CAP5_ASYMMETRIC_BA		BIT(4)
-+#define S1G_CAP5_FLOW_CONTROL		BIT(5)
-+#define S1G_CAP5_SECTORIZED_BEAM	GENMASK(7, 6)
-+
-+#define S1G_CAP6_OBSS_MITIGATION	BIT(0)
-+#define S1G_CAP6_FRAGMENT_BA		BIT(1)
-+#define S1G_CAP6_NDP_PS_POLL		BIT(2)
-+#define S1G_CAP6_RAW_OPERATION		BIT(3)
-+#define S1G_CAP6_PAGE_SLICING		BIT(4)
-+#define S1G_CAP6_TXOP_SHARING_IMP_ACK	BIT(5)
-+#define S1G_CAP6_VHT_LINK_ADAPT		GENMASK(7, 6)
-+
-+#define S1G_CAP7_TACK_AS_PS_POLL		BIT(0)
-+#define S1G_CAP7_DUP_1MHZ			BIT(1)
-+#define S1G_CAP7_MCS_NEGOTIATION		BIT(2)
-+#define S1G_CAP7_1MHZ_CTL_RESPONSE_PREAMBLE	BIT(3)
-+#define S1G_CAP7_NDP_BFING_REPORT_POLL		BIT(4)
-+#define S1G_CAP7_UNSOLICITED_DYN_AID		BIT(5)
-+#define S1G_CAP7_SECTOR_TRAINING_OPERATION	BIT(6)
-+#define S1G_CAP7_TEMP_PS_MODE_SWITCH		BIT(7)
-+
-+#define S1G_CAP8_TWT_GROUPING	BIT(0)
-+#define S1G_CAP8_BDT		BIT(1)
-+#define S1G_CAP8_COLOR		GENMASK(4, 2)
-+#define S1G_CAP8_TWT_REQUEST	BIT(5)
-+#define S1G_CAP8_TWT_RESPOND	BIT(6)
-+#define S1G_CAP8_PV1_FRAME	BIT(7)
-+
-+#define S1G_CAP9_LINK_ADAPT_PER_CONTROL_RESPONSE BIT(0)
-+
-+#define S1G_OPER_CH_WIDTH_PRIMARY	BIT(0)
-+#define S1G_OPER_CH_WIDTH_OPER		GENMASK(4, 1)
-+#define S1G_OPER_CH_PRIMARY_LOCATION	BIT(5)
-+
-+#define S1G_2M_PRIMARY_LOCATION_LOWER	0
-+#define S1G_2M_PRIMARY_LOCATION_UPPER	1
-+
-+#define LISTEN_INT_USF	GENMASK(15, 14)
-+#define LISTEN_INT_UI	GENMASK(13, 0)
-+
-+#define IEEE80211_MAX_USF	FIELD_MAX(LISTEN_INT_USF)
-+#define IEEE80211_MAX_UI	FIELD_MAX(LISTEN_INT_UI)
-+
-+/* S1G encoding types */
-+#define IEEE80211_S1G_TIM_ENC_MODE_BLOCK	0
-+#define IEEE80211_S1G_TIM_ENC_MODE_SINGLE	1
-+#define IEEE80211_S1G_TIM_ENC_MODE_OLB		2
-+
-+enum ieee80211_s1g_actioncode {
-+	WLAN_S1G_AID_SWITCH_REQUEST,
-+	WLAN_S1G_AID_SWITCH_RESPONSE,
-+	WLAN_S1G_SYNC_CONTROL,
-+	WLAN_S1G_STA_INFO_ANNOUNCE,
-+	WLAN_S1G_EDCA_PARAM_SET,
-+	WLAN_S1G_EL_OPERATION,
-+	WLAN_S1G_TWT_SETUP,
-+	WLAN_S1G_TWT_TEARDOWN,
-+	WLAN_S1G_SECT_GROUP_ID_LIST,
-+	WLAN_S1G_SECT_ID_FEEDBACK,
-+	WLAN_S1G_TWT_INFORMATION = 11,
-+};
-+
-+/**
-+ * ieee80211_is_s1g_short_beacon - check if frame is an S1G short beacon
-+ * @fc: frame control bytes in little-endian byteorder
-+ * @variable: pointer to the beacon frame elements
-+ * @variable_len: length of the frame elements
-+ * Return: whether or not the frame is an S1G short beacon. As per
-+ *	IEEE80211-2024 11.1.3.10.1, The S1G beacon compatibility element shall
-+ *	always be present as the first element in beacon frames generated at a
-+ *	TBTT (Target Beacon Transmission Time), so any frame not containing
-+ *	this element must have been generated at a TSBTT (Target Short Beacon
-+ *	Transmission Time) that is not a TBTT. Additionally, short beacons are
-+ *	prohibited from containing the S1G beacon compatibility element as per
-+ *	IEEE80211-2024 9.3.4.3 Table 9-76, so if we have an S1G beacon with
-+ *	either no elements or the first element is not the beacon compatibility
-+ *	element, we have a short beacon.
-+ */
-+static inline bool ieee80211_is_s1g_short_beacon(__le16 fc, const u8 *variable,
-+						 size_t variable_len)
-+{
-+	if (!ieee80211_is_s1g_beacon(fc))
-+		return false;
-+
-+	/*
-+	 * If the frame does not contain at least 1 element (this is perfectly
-+	 * valid in a short beacon) and is an S1G beacon, we have a short
-+	 * beacon.
-+	 */
-+	if (variable_len < 2)
-+		return true;
-+
-+	return variable[0] != WLAN_EID_S1G_BCN_COMPAT;
-+}
-+
-+struct s1g_tim_aid {
-+	u16 aid;
-+	u8 target_blk; /* Target block index */
-+	u8 target_subblk; /* Target subblock index */
-+	u8 target_subblk_bit; /* Target subblock bit */
-+};
-+
-+struct s1g_tim_enc_block {
-+	u8 enc_mode;
-+	bool inverse;
-+	const u8 *ptr;
-+	u8 len;
-+
-+	/*
-+	 * For an OLB encoded block that spans multiple blocks, this
-+	 * is the offset into the span described by that encoded block.
-+	 */
-+	u8 olb_blk_offset;
-+};
-+
-+/*
-+ * Helper routines to quickly extract the length of an encoded block. Validation
-+ * is also performed to ensure the length extracted lies within the TIM.
-+ */
-+
-+static inline int ieee80211_s1g_len_bitmap(const u8 *ptr, const u8 *end)
-+{
-+	u8 blkmap;
-+	u8 n_subblks;
-+
-+	if (ptr >= end)
-+		return -EINVAL;
-+
-+	blkmap = *ptr;
-+	n_subblks = hweight8(blkmap);
-+
-+	if (ptr + 1 + n_subblks > end)
-+		return -EINVAL;
-+
-+	return 1 + n_subblks;
-+}
-+
-+static inline int ieee80211_s1g_len_single(const u8 *ptr, const u8 *end)
-+{
-+	return (ptr + 1 > end) ? -EINVAL : 1;
-+}
-+
-+static inline int ieee80211_s1g_len_olb(const u8 *ptr, const u8 *end)
-+{
-+	if (ptr >= end)
-+		return -EINVAL;
-+
-+	return (ptr + 1 + *ptr > end) ? -EINVAL : 1 + *ptr;
-+}
-+
-+/*
-+ * Enumerate all encoded blocks until we find the encoded block that describes
-+ * our target AID. OLB is a special case as a single encoded block can describe
-+ * multiple blocks as a single encoded block.
-+ */
-+static inline int ieee80211_s1g_find_target_block(struct s1g_tim_enc_block *enc,
-+						  const struct s1g_tim_aid *aid,
-+						  const u8 *ptr, const u8 *end)
-+{
-+	/* need at least block-control octet */
-+	while (ptr + 1 <= end) {
-+		u8 ctrl = *ptr++;
-+		u8 mode = ctrl & 0x03;
-+		bool contains, inverse = ctrl & BIT(2);
-+		u8 span, blk_off = ctrl >> 3;
-+		int len;
-+
-+		switch (mode) {
-+		case IEEE80211_S1G_TIM_ENC_MODE_BLOCK:
-+			len = ieee80211_s1g_len_bitmap(ptr, end);
-+			contains = blk_off == aid->target_blk;
-+			break;
-+		case IEEE80211_S1G_TIM_ENC_MODE_SINGLE:
-+			len = ieee80211_s1g_len_single(ptr, end);
-+			contains = blk_off == aid->target_blk;
-+			break;
-+		case IEEE80211_S1G_TIM_ENC_MODE_OLB:
-+			len = ieee80211_s1g_len_olb(ptr, end);
-+			/*
-+			 * An OLB encoded block can describe more then one
-+			 * block, meaning an encoded OLB block can span more
-+			 * then a single block.
-+			 */
-+			if (len > 0) {
-+				/* Minus one for the length octet */
-+				span = DIV_ROUND_UP(len - 1, 8);
-+				/*
-+				 * Check if our target block lies within the
-+				 * block span described by this encoded block.
-+				 */
-+				contains = (aid->target_blk >= blk_off) &&
-+					   (aid->target_blk < blk_off + span);
-+			}
-+			break;
-+		default:
-+			return -EOPNOTSUPP;
-+		}
-+
-+		if (len < 0)
-+			return len;
-+
-+		if (contains) {
-+			enc->enc_mode = mode;
-+			enc->inverse = inverse;
-+			enc->ptr = ptr;
-+			enc->len = (u8)len;
-+			enc->olb_blk_offset = blk_off;
-+			return 0;
-+		}
-+
-+		ptr += len;
-+	}
-+
-+	return -ENOENT;
-+}
-+
-+static inline bool ieee80211_s1g_parse_bitmap(struct s1g_tim_enc_block *enc,
-+					      struct s1g_tim_aid *aid)
-+{
-+	const u8 *ptr = enc->ptr;
-+	u8 blkmap = *ptr++;
-+
-+	/*
-+	 * If our block bitmap does not contain a set bit that corresponds
-+	 * to our AID, it could mean a variety of things depending on if
-+	 * the encoding mode is inverted or not.
-+	 *
-+	 * 1. If inverted, it means the entire subblock is present and hence
-+	 *    our AID has been set.
-+	 * 2. If not inverted, it means our subblock is not present and hence
-+	 *    it is all zero meaning our AID is not set.
-+	 */
-+	if (!(blkmap & BIT(aid->target_subblk)))
-+		return enc->inverse;
-+
-+	/*
-+	 * Increment ptr by the number of set subblocks that appear before our
-+	 * target subblock. If our target subblock is 0, do nothing as ptr
-+	 * already points to our target subblock.
-+	 */
-+	if (aid->target_subblk)
-+		ptr += hweight8(blkmap & GENMASK(aid->target_subblk - 1, 0));
-+
-+	return !!(*ptr & BIT(aid->target_subblk_bit)) ^ enc->inverse;
-+}
-+
-+static inline bool ieee80211_s1g_parse_single(struct s1g_tim_enc_block *enc,
-+					      struct s1g_tim_aid *aid)
-+{
-+	/*
-+	 * Single AID mode describes, as the name suggests, a single AID
-+	 * within the block described by the encoded block. The octet
-+	 * contains the 6 LSBs of the AID described in the block. The other
-+	 * 2 bits are reserved. When inversed, every single AID described
-+	 * by the current block have buffered traffic except for the AID
-+	 * described in the single AID octet.
-+	 */
-+	return ((*enc->ptr & 0x3f) == (aid->aid & 0x3f)) ^ enc->inverse;
-+}
-+
-+static inline bool ieee80211_s1g_parse_olb(struct s1g_tim_enc_block *enc,
-+					   struct s1g_tim_aid *aid)
-+{
-+	const u8 *ptr = enc->ptr;
-+	u8 blk_len = *ptr++;
-+	/*
-+	 * Given an OLB encoded block that describes multiple blocks,
-+	 * calculate the offset into the span. Then calculate the
-+	 * subblock location normally.
-+	 */
-+	u16 span_offset = aid->target_blk - enc->olb_blk_offset;
-+	u16 subblk_idx = span_offset * 8 + aid->target_subblk;
-+
-+	if (subblk_idx >= blk_len)
-+		return enc->inverse;
-+
-+	return !!(ptr[subblk_idx] & BIT(aid->target_subblk_bit)) ^ enc->inverse;
-+}
-+
-+/*
-+ * An S1G PVB has 3 non optional encoding types, each that can be inverted.
-+ * An S1G PVB is constructed with zero or more encoded block subfields. Each
-+ * encoded block represents a single "block" of AIDs (64), and each encoded
-+ * block can contain one of the 3 encoding types alongside a single bit for
-+ * whether the bits should be inverted.
-+ *
-+ * As the standard makes no guarantee about the ordering of encoded blocks,
-+ * we must parse every encoded block in the worst case scenario given an
-+ * AID that lies within the last block.
-+ */
-+static inline bool ieee80211_s1g_check_tim(const struct ieee80211_tim_ie *tim,
-+					   u8 tim_len, u16 aid)
-+{
-+	int err;
-+	struct s1g_tim_aid target_aid;
-+	struct s1g_tim_enc_block enc_blk;
-+
-+	if (tim_len < 3)
-+		return false;
-+
-+	target_aid.aid = aid;
-+	target_aid.target_blk = (aid >> 6) & 0x1f;
-+	target_aid.target_subblk = (aid >> 3) & 0x7;
-+	target_aid.target_subblk_bit = aid & 0x7;
-+
-+	/*
-+	 * Find our AIDs target encoded block and fill &enc_blk with the
-+	 * encoded blocks information. If no entry is found or an error
-+	 * occurs return false.
-+	 */
-+	err = ieee80211_s1g_find_target_block(&enc_blk, &target_aid,
-+					      tim->virtual_map,
-+					      (const u8 *)tim + tim_len + 2);
-+	if (err)
-+		return false;
-+
-+	switch (enc_blk.enc_mode) {
-+	case IEEE80211_S1G_TIM_ENC_MODE_BLOCK:
-+		return ieee80211_s1g_parse_bitmap(&enc_blk, &target_aid);
-+	case IEEE80211_S1G_TIM_ENC_MODE_SINGLE:
-+		return ieee80211_s1g_parse_single(&enc_blk, &target_aid);
-+	case IEEE80211_S1G_TIM_ENC_MODE_OLB:
-+		return ieee80211_s1g_parse_olb(&enc_blk, &target_aid);
-+	default:
-+		return false;
-+	}
-+}
-+
-+#endif /* LINUX_IEEE80211_H */
-diff --git a/include/linux/ieee80211-vht.h b/include/linux/ieee80211-vht.h
-new file mode 100644
-index 000000000000..898dfb561fef
--- /dev/null
-+++ b/include/linux/ieee80211-vht.h
-@@ -0,0 +1,236 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+/*
-+ * IEEE 802.11 VHT definitions
-+ *
-+ * Copyright (c) 2001-2002, SSH Communications Security Corp and Jouni Malinen
-+ * <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2002-2003, Jouni Malinen <jkmaline@cc.hut.fi>
-+ * Copyright (c) 2005, Devicescape Software, Inc.
-+ * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net>
-+ * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH
-+ * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
-+ * Copyright (c) 2018 - 2025 Intel Corporation
-+ */
-+
-+#ifndef LINUX_IEEE80211_VHT_H
-+#define LINUX_IEEE80211_VHT_H
-+
-+#include <linux/types.h>
-+#include <linux/if_ether.h>
-+
-+#define IEEE80211_MAX_MPDU_LEN_VHT_3895		3895
-+#define IEEE80211_MAX_MPDU_LEN_VHT_7991		7991
-+#define IEEE80211_MAX_MPDU_LEN_VHT_11454	11454
-+
-+/**
-+ * enum ieee80211_vht_opmode_bits - VHT operating mode field bits
-+ * @IEEE80211_OPMODE_NOTIF_CHANWIDTH_MASK: channel width mask
-+ * @IEEE80211_OPMODE_NOTIF_CHANWIDTH_20MHZ: 20 MHz channel width
-+ * @IEEE80211_OPMODE_NOTIF_CHANWIDTH_40MHZ: 40 MHz channel width
-+ * @IEEE80211_OPMODE_NOTIF_CHANWIDTH_80MHZ: 80 MHz channel width
-+ * @IEEE80211_OPMODE_NOTIF_CHANWIDTH_160MHZ: 160 MHz or 80+80 MHz channel width
-+ * @IEEE80211_OPMODE_NOTIF_BW_160_80P80: 160 / 80+80 MHz indicator flag
-+ * @IEEE80211_OPMODE_NOTIF_RX_NSS_MASK: number of spatial streams mask
-+ *	(the NSS value is the value of this field + 1)
-+ * @IEEE80211_OPMODE_NOTIF_RX_NSS_SHIFT: number of spatial streams shift
-+ * @IEEE80211_OPMODE_NOTIF_RX_NSS_TYPE_BF: indicates streams in SU-MIMO PPDU
-+ *	using a beamforming steering matrix
-+ */
-+enum ieee80211_vht_opmode_bits {
-+	IEEE80211_OPMODE_NOTIF_CHANWIDTH_MASK	= 0x03,
-+	IEEE80211_OPMODE_NOTIF_CHANWIDTH_20MHZ	= 0,
-+	IEEE80211_OPMODE_NOTIF_CHANWIDTH_40MHZ	= 1,
-+	IEEE80211_OPMODE_NOTIF_CHANWIDTH_80MHZ	= 2,
-+	IEEE80211_OPMODE_NOTIF_CHANWIDTH_160MHZ	= 3,
-+	IEEE80211_OPMODE_NOTIF_BW_160_80P80	= 0x04,
-+	IEEE80211_OPMODE_NOTIF_RX_NSS_MASK	= 0x70,
-+	IEEE80211_OPMODE_NOTIF_RX_NSS_SHIFT	= 4,
-+	IEEE80211_OPMODE_NOTIF_RX_NSS_TYPE_BF	= 0x80,
-+};
-+
-+/*
-+ * Maximum length of AMPDU that the STA can receive in VHT.
-+ * Length = 2 ^ (13 + max_ampdu_length_exp) - 1 (octets)
-+ */
-+enum ieee80211_vht_max_ampdu_length_exp {
-+	IEEE80211_VHT_MAX_AMPDU_8K = 0,
-+	IEEE80211_VHT_MAX_AMPDU_16K = 1,
-+	IEEE80211_VHT_MAX_AMPDU_32K = 2,
-+	IEEE80211_VHT_MAX_AMPDU_64K = 3,
-+	IEEE80211_VHT_MAX_AMPDU_128K = 4,
-+	IEEE80211_VHT_MAX_AMPDU_256K = 5,
-+	IEEE80211_VHT_MAX_AMPDU_512K = 6,
-+	IEEE80211_VHT_MAX_AMPDU_1024K = 7
-+};
-+
-+/**
-+ * struct ieee80211_vht_mcs_info - VHT MCS information
-+ * @rx_mcs_map: RX MCS map 2 bits for each stream, total 8 streams
-+ * @rx_highest: Indicates highest long GI VHT PPDU data rate
-+ *	STA can receive. Rate expressed in units of 1 Mbps.
-+ *	If this field is 0 this value should not be used to
-+ *	consider the highest RX data rate supported.
-+ *	The top 3 bits of this field indicate the Maximum NSTS,total
-+ *	(a beamformee capability.)
-+ * @tx_mcs_map: TX MCS map 2 bits for each stream, total 8 streams
-+ * @tx_highest: Indicates highest long GI VHT PPDU data rate
-+ *	STA can transmit. Rate expressed in units of 1 Mbps.
-+ *	If this field is 0 this value should not be used to
-+ *	consider the highest TX data rate supported.
-+ *	The top 2 bits of this field are reserved, the
-+ *	3rd bit from the top indiciates VHT Extended NSS BW
-+ *	Capability.
-+ */
-+struct ieee80211_vht_mcs_info {
-+	__le16 rx_mcs_map;
-+	__le16 rx_highest;
-+	__le16 tx_mcs_map;
-+	__le16 tx_highest;
-+} __packed;
-+
-+/* for rx_highest */
-+#define IEEE80211_VHT_MAX_NSTS_TOTAL_SHIFT	13
-+#define IEEE80211_VHT_MAX_NSTS_TOTAL_MASK	(7 << IEEE80211_VHT_MAX_NSTS_TOTAL_SHIFT)
-+
-+/* for tx_highest */
-+#define IEEE80211_VHT_EXT_NSS_BW_CAPABLE	(1 << 13)
-+
-+/**
-+ * enum ieee80211_vht_mcs_support - VHT MCS support definitions
-+ * @IEEE80211_VHT_MCS_SUPPORT_0_7: MCSes 0-7 are supported for the
-+ *	number of streams
-+ * @IEEE80211_VHT_MCS_SUPPORT_0_8: MCSes 0-8 are supported
-+ * @IEEE80211_VHT_MCS_SUPPORT_0_9: MCSes 0-9 are supported
-+ * @IEEE80211_VHT_MCS_NOT_SUPPORTED: This number of streams isn't supported
-+ *
-+ * These definitions are used in each 2-bit subfield of the @rx_mcs_map
-+ * and @tx_mcs_map fields of &struct ieee80211_vht_mcs_info, which are
-+ * both split into 8 subfields by number of streams. These values indicate
-+ * which MCSes are supported for the number of streams the value appears
-+ * for.
-+ */
-+enum ieee80211_vht_mcs_support {
-+	IEEE80211_VHT_MCS_SUPPORT_0_7	= 0,
-+	IEEE80211_VHT_MCS_SUPPORT_0_8	= 1,
-+	IEEE80211_VHT_MCS_SUPPORT_0_9	= 2,
-+	IEEE80211_VHT_MCS_NOT_SUPPORTED	= 3,
-+};
-+
-+/**
-+ * struct ieee80211_vht_cap - VHT capabilities
-+ *
-+ * This structure is the "VHT capabilities element" as
-+ * described in 802.11ac D3.0 8.4.2.160
-+ * @vht_cap_info: VHT capability info
-+ * @supp_mcs: VHT MCS supported rates
-+ */
-+struct ieee80211_vht_cap {
-+	__le32 vht_cap_info;
-+	struct ieee80211_vht_mcs_info supp_mcs;
-+} __packed;
-+
-+/**
-+ * enum ieee80211_vht_chanwidth - VHT channel width
-+ * @IEEE80211_VHT_CHANWIDTH_USE_HT: use the HT operation IE to
-+ *	determine the channel width (20 or 40 MHz)
-+ * @IEEE80211_VHT_CHANWIDTH_80MHZ: 80 MHz bandwidth
-+ * @IEEE80211_VHT_CHANWIDTH_160MHZ: 160 MHz bandwidth
-+ * @IEEE80211_VHT_CHANWIDTH_80P80MHZ: 80+80 MHz bandwidth
-+ */
-+enum ieee80211_vht_chanwidth {
-+	IEEE80211_VHT_CHANWIDTH_USE_HT		= 0,
-+	IEEE80211_VHT_CHANWIDTH_80MHZ		= 1,
-+	IEEE80211_VHT_CHANWIDTH_160MHZ		= 2,
-+	IEEE80211_VHT_CHANWIDTH_80P80MHZ	= 3,
-+};
-+
-+/**
-+ * struct ieee80211_vht_operation - VHT operation IE
-+ *
-+ * This structure is the "VHT operation element" as
-+ * described in 802.11ac D3.0 8.4.2.161
-+ * @chan_width: Operating channel width
-+ * @center_freq_seg0_idx: center freq segment 0 index
-+ * @center_freq_seg1_idx: center freq segment 1 index
-+ * @basic_mcs_set: VHT Basic MCS rate set
-+ */
-+struct ieee80211_vht_operation {
-+	u8 chan_width;
-+	u8 center_freq_seg0_idx;
-+	u8 center_freq_seg1_idx;
-+	__le16 basic_mcs_set;
-+} __packed;
-+
-+/* 802.11ac VHT Capabilities */
-+#define IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895			0x00000000
-+#define IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_7991			0x00000001
-+#define IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454			0x00000002
-+#define IEEE80211_VHT_CAP_MAX_MPDU_MASK				0x00000003
-+#define IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ		0x00000004
-+#define IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ	0x00000008
-+#define IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK			0x0000000C
-+#define IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_SHIFT			2
-+#define IEEE80211_VHT_CAP_RXLDPC				0x00000010
-+#define IEEE80211_VHT_CAP_SHORT_GI_80				0x00000020
-+#define IEEE80211_VHT_CAP_SHORT_GI_160				0x00000040
-+#define IEEE80211_VHT_CAP_TXSTBC				0x00000080
-+#define IEEE80211_VHT_CAP_RXSTBC_1				0x00000100
-+#define IEEE80211_VHT_CAP_RXSTBC_2				0x00000200
-+#define IEEE80211_VHT_CAP_RXSTBC_3				0x00000300
-+#define IEEE80211_VHT_CAP_RXSTBC_4				0x00000400
-+#define IEEE80211_VHT_CAP_RXSTBC_MASK				0x00000700
-+#define IEEE80211_VHT_CAP_RXSTBC_SHIFT				8
-+#define IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE			0x00000800
-+#define IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE			0x00001000
-+#define IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT                  13
-+#define IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK			\
-+		(7 << IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT)
-+#define IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT		16
-+#define IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK		\
-+		(7 << IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT)
-+#define IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE			0x00080000
-+#define IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE			0x00100000
-+#define IEEE80211_VHT_CAP_VHT_TXOP_PS				0x00200000
-+#define IEEE80211_VHT_CAP_HTC_VHT				0x00400000
-+#define IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT	23
-+#define IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK	\
-+		(7 << IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT)
-+#define IEEE80211_VHT_CAP_VHT_LINK_ADAPTATION_VHT_UNSOL_MFB	0x08000000
-+#define IEEE80211_VHT_CAP_VHT_LINK_ADAPTATION_VHT_MRQ_MFB	0x0c000000
-+#define IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN			0x10000000
-+#define IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN			0x20000000
-+#define IEEE80211_VHT_CAP_EXT_NSS_BW_SHIFT			30
-+#define IEEE80211_VHT_CAP_EXT_NSS_BW_MASK			0xc0000000
-+
-+/**
-+ * ieee80211_get_vht_max_nss - return max NSS for a given bandwidth/MCS
-+ * @cap: VHT capabilities of the peer
-+ * @bw: bandwidth to use
-+ * @mcs: MCS index to use
-+ * @ext_nss_bw_capable: indicates whether or not the local transmitter
-+ *	(rate scaling algorithm) can deal with the new logic
-+ *	(dot11VHTExtendedNSSBWCapable)
-+ * @max_vht_nss: current maximum NSS as advertised by the STA in
-+ *	operating mode notification, can be 0 in which case the
-+ *	capability data will be used to derive this (from MCS support)
-+ * Return: The maximum NSS that can be used for the given bandwidth/MCS
-+ *	combination
-+ *
-+ * Due to the VHT Extended NSS Bandwidth Support, the maximum NSS can
-+ * vary for a given BW/MCS. This function parses the data.
-+ *
-+ * Note: This function is exported by cfg80211.
-+ */
-+int ieee80211_get_vht_max_nss(struct ieee80211_vht_cap *cap,
-+			      enum ieee80211_vht_chanwidth bw,
-+			      int mcs, bool ext_nss_bw_capable,
-+			      unsigned int max_vht_nss);
-+
-+/* VHT action codes */
-+enum ieee80211_vht_actioncode {
-+	WLAN_VHT_ACTION_COMPRESSED_BF = 0,
-+	WLAN_VHT_ACTION_GROUPID_MGMT = 1,
-+	WLAN_VHT_ACTION_OPMODE_NOTIF = 2,
-+};
-+
-+#endif /* LINUX_IEEE80211_VHT_H */
-diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h
-index 55e6f4ad0ca6..96439de55f07 100644
--- a/include/linux/ieee80211.h
-+++ b/include/linux/ieee80211.h
-@@ -9,7 +9,7 @@
-  * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net>
-  * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH
-  * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
- * Copyright (c) 2018 - 2022 Intel Corporation
-+ * Copyright (c) 2018 - 2025 Intel Corporation
-  */
- 
- #ifndef LINUX_IEEE80211_H
-@@ -20,7 +20,7 @@
- #include <linux/etherdevice.h>
- #include <linux/bitfield.h>
- #include <asm/byteorder.h>
-#include <asm/unaligned.h>
-+#include <linux/unaligned.h>
- 
- /*
-  * DS bit usage
-@@ -43,6 +43,7 @@
- #define IEEE80211_FCTL_VERS		0x0003
- #define IEEE80211_FCTL_FTYPE		0x000c
- #define IEEE80211_FCTL_STYPE		0x00f0
-+#define IEEE80211_FCTL_TYPE		(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)
- #define IEEE80211_FCTL_TODS		0x0100
- #define IEEE80211_FCTL_FROMDS		0x0200
- #define IEEE80211_FCTL_MOREFRAGS	0x0400
-@@ -109,15 +110,6 @@
- #define IEEE80211_STYPE_DMG_BEACON		0x0000
- #define IEEE80211_STYPE_S1G_BEACON		0x0010
- 
-/* bits unique to S1G beacon */
-#define IEEE80211_S1G_BCN_NEXT_TBTT	0x100
-
-/* see 802.11ah-2016 9.9 NDP CMAC frames */
-#define IEEE80211_S1G_1MHZ_NDP_BITS	25
-#define IEEE80211_S1G_1MHZ_NDP_BYTES	4
-#define IEEE80211_S1G_2MHZ_NDP_BITS	37
-#define IEEE80211_S1G_2MHZ_NDP_BYTES	5
-
- #define IEEE80211_NDP_FTYPE_CTS			0
- #define IEEE80211_NDP_FTYPE_CF_END		0
- #define IEEE80211_NDP_FTYPE_PS_POLL		1
-@@ -153,9 +145,6 @@
- 
- #define IEEE80211_ANO_NETTYPE_WILD              15
- 
-/* bits unique to S1G beacon */
-#define IEEE80211_S1G_BCN_NEXT_TBTT    0x100
-
- /* control extension - for IEEE80211_FTYPE_CTL | IEEE80211_STYPE_CTL_EXT */
- #define IEEE80211_CTL_EXT_POLL		0x2000
- #define IEEE80211_CTL_EXT_SPR		0x3000
-@@ -172,11 +161,11 @@
- #define IEEE80211_SN_MODULO		(IEEE80211_MAX_SN + 1)
- 
- 
-/* PV1 Layout 11ah 9.8.3.1 */
-+/* PV1 Layout IEEE 802.11-2020 9.8.3.1 */
- #define IEEE80211_PV1_FCTL_VERS		0x0003
- #define IEEE80211_PV1_FCTL_FTYPE	0x001c
- #define IEEE80211_PV1_FCTL_STYPE	0x00e0
-#define IEEE80211_PV1_FCTL_TODS		0x0100
-+#define IEEE80211_PV1_FCTL_FROMDS		0x0100
- #define IEEE80211_PV1_FCTL_MOREFRAGS	0x0200
- #define IEEE80211_PV1_FCTL_PM		0x0400
- #define IEEE80211_PV1_FCTL_MOREDATA	0x0800
-@@ -191,6 +180,11 @@ static inline bool ieee80211_sn_less(u16 sn1, u16 sn2)
- 	return ((sn1 - sn2) & IEEE80211_SN_MASK) > (IEEE80211_SN_MODULO >> 1);
- }
- 
-+static inline bool ieee80211_sn_less_eq(u16 sn1, u16 sn2)
-+{
-+	return ((sn2 - sn1) & IEEE80211_SN_MASK) <= (IEEE80211_SN_MODULO >> 1);
-+}
-+
- static inline u16 ieee80211_sn_add(u16 sn1, u16 sn2)
- {
- 	return (sn1 + sn2) & IEEE80211_SN_MASK;
-@@ -216,6 +210,7 @@ static inline u16 ieee80211_sn_sub(u16 sn1, u16 sn2)
- #define IEEE80211_MAX_AID_S1G		8191
- #define IEEE80211_MAX_TIM_LEN		251
- #define IEEE80211_MAX_MESH_PEERINGS	63
-+
- /* Maximum size for the MA-UNITDATA primitive, 802.11 standard section
-    6.2.1.1.2.
- 
-@@ -229,21 +224,8 @@ static inline u16 ieee80211_sn_sub(u16 sn1, u16 sn2)
- /* 30 byte 4 addr hdr, 2 byte QoS, 2304 byte MSDU, 12 byte crypt, 4 byte FCS */
- #define IEEE80211_MAX_FRAME_LEN		2352
- 
-/* Maximal size of an A-MSDU that can be transported in a HT BA session */
-#define IEEE80211_MAX_MPDU_LEN_HT_BA		4095
-
-/* Maximal size of an A-MSDU */
-#define IEEE80211_MAX_MPDU_LEN_HT_3839		3839
-#define IEEE80211_MAX_MPDU_LEN_HT_7935		7935
-
-#define IEEE80211_MAX_MPDU_LEN_VHT_3895		3895
-#define IEEE80211_MAX_MPDU_LEN_VHT_7991		7991
-#define IEEE80211_MAX_MPDU_LEN_VHT_11454	11454
-
- #define IEEE80211_MAX_SSID_LEN		32
- 
-#define IEEE80211_MAX_MESH_ID_LEN	32
-
- #define IEEE80211_FIRST_TSPEC_TSID	8
- #define IEEE80211_NUM_TIDS		16
- 
-@@ -294,8 +276,6 @@ static inline u16 ieee80211_sn_sub(u16 sn1, u16 sn2)
- #define IEEE80211_WMM_IE_STA_QOSINFO_SP_MASK	0x03
- #define IEEE80211_WMM_IE_STA_QOSINFO_SP_SHIFT	5
- 
-#define IEEE80211_HT_CTL_LEN		4
-
- /* trigger type within common_info of trigger frame */
- #define IEEE80211_TRIGGER_TYPE_MASK		0xf
- #define IEEE80211_TRIGGER_TYPE_BASIC		0x0
-@@ -307,12 +287,21 @@ static inline u16 ieee80211_sn_sub(u16 sn1, u16 sn2)
- #define IEEE80211_TRIGGER_TYPE_BQRP		0x6
- #define IEEE80211_TRIGGER_TYPE_NFRP		0x7
- 
-+/* UL-bandwidth within common_info of trigger frame */
-+#define IEEE80211_TRIGGER_ULBW_MASK		0xc0000
-+#define IEEE80211_TRIGGER_ULBW_20MHZ		0x0
-+#define IEEE80211_TRIGGER_ULBW_40MHZ		0x1
-+#define IEEE80211_TRIGGER_ULBW_80MHZ		0x2
-+#define IEEE80211_TRIGGER_ULBW_160_80P80MHZ	0x3
-+
- struct ieee80211_hdr {
- 	__le16 frame_control;
- 	__le16 duration_id;
-	u8 addr1[ETH_ALEN];
-	u8 addr2[ETH_ALEN];
-	u8 addr3[ETH_ALEN];
-+	struct_group(addrs,
-+		u8 addr1[ETH_ALEN];
-+		u8 addr2[ETH_ALEN];
-+		u8 addr3[ETH_ALEN];
-+	);
- 	__le16 seq_ctrl;
- 	u8 addr4[ETH_ALEN];
- } __packed __aligned(2);
-@@ -336,6 +325,17 @@ struct ieee80211_qos_hdr {
- 	__le16 qos_ctrl;
- } __packed __aligned(2);
- 
-+struct ieee80211_qos_hdr_4addr {
-+	__le16 frame_control;
-+	__le16 duration_id;
-+	u8 addr1[ETH_ALEN];
-+	u8 addr2[ETH_ALEN];
-+	u8 addr3[ETH_ALEN];
-+	__le16 seq_ctrl;
-+	u8 addr4[ETH_ALEN];
-+	__le16 qos_ctrl;
-+} __packed __aligned(2);
-+
- struct ieee80211_trigger {
- 	__le16 frame_control;
- 	__le16 duration;
-@@ -348,6 +348,7 @@ struct ieee80211_trigger {
- /**
-  * ieee80211_has_tods - check if IEEE80211_FCTL_TODS is set
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame has to-DS set
-  */
- static inline bool ieee80211_has_tods(__le16 fc)
- {
-@@ -357,6 +358,7 @@ static inline bool ieee80211_has_tods(__le16 fc)
- /**
-  * ieee80211_has_fromds - check if IEEE80211_FCTL_FROMDS is set
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame has from-DS set
-  */
- static inline bool ieee80211_has_fromds(__le16 fc)
- {
-@@ -366,6 +368,7 @@ static inline bool ieee80211_has_fromds(__le16 fc)
- /**
-  * ieee80211_has_a4 - check if IEEE80211_FCTL_TODS and IEEE80211_FCTL_FROMDS are set
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not it's a 4-address frame (from-DS and to-DS set)
-  */
- static inline bool ieee80211_has_a4(__le16 fc)
- {
-@@ -376,6 +379,7 @@ static inline bool ieee80211_has_a4(__le16 fc)
- /**
-  * ieee80211_has_morefrags - check if IEEE80211_FCTL_MOREFRAGS is set
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame has more fragments (more frags bit set)
-  */
- static inline bool ieee80211_has_morefrags(__le16 fc)
- {
-@@ -385,6 +389,7 @@ static inline bool ieee80211_has_morefrags(__le16 fc)
- /**
-  * ieee80211_has_retry - check if IEEE80211_FCTL_RETRY is set
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the retry flag is set
-  */
- static inline bool ieee80211_has_retry(__le16 fc)
- {
-@@ -394,6 +399,7 @@ static inline bool ieee80211_has_retry(__le16 fc)
- /**
-  * ieee80211_has_pm - check if IEEE80211_FCTL_PM is set
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the power management flag is set
-  */
- static inline bool ieee80211_has_pm(__le16 fc)
- {
-@@ -403,6 +409,7 @@ static inline bool ieee80211_has_pm(__le16 fc)
- /**
-  * ieee80211_has_moredata - check if IEEE80211_FCTL_MOREDATA is set
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the more data flag is set
-  */
- static inline bool ieee80211_has_moredata(__le16 fc)
- {
-@@ -412,6 +419,7 @@ static inline bool ieee80211_has_moredata(__le16 fc)
- /**
-  * ieee80211_has_protected - check if IEEE80211_FCTL_PROTECTED is set
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the protected flag is set
-  */
- static inline bool ieee80211_has_protected(__le16 fc)
- {
-@@ -421,6 +429,7 @@ static inline bool ieee80211_has_protected(__le16 fc)
- /**
-  * ieee80211_has_order - check if IEEE80211_FCTL_ORDER is set
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the order flag is set
-  */
- static inline bool ieee80211_has_order(__le16 fc)
- {
-@@ -430,6 +439,7 @@ static inline bool ieee80211_has_order(__le16 fc)
- /**
-  * ieee80211_is_mgmt - check if type is IEEE80211_FTYPE_MGMT
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame type is management
-  */
- static inline bool ieee80211_is_mgmt(__le16 fc)
- {
-@@ -440,6 +450,7 @@ static inline bool ieee80211_is_mgmt(__le16 fc)
- /**
-  * ieee80211_is_ctl - check if type is IEEE80211_FTYPE_CTL
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame type is control
-  */
- static inline bool ieee80211_is_ctl(__le16 fc)
- {
-@@ -450,6 +461,7 @@ static inline bool ieee80211_is_ctl(__le16 fc)
- /**
-  * ieee80211_is_data - check if type is IEEE80211_FTYPE_DATA
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a data frame
-  */
- static inline bool ieee80211_is_data(__le16 fc)
- {
-@@ -460,6 +472,7 @@ static inline bool ieee80211_is_data(__le16 fc)
- /**
-  * ieee80211_is_ext - check if type is IEEE80211_FTYPE_EXT
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame type is extended
-  */
- static inline bool ieee80211_is_ext(__le16 fc)
- {
-@@ -471,6 +484,7 @@ static inline bool ieee80211_is_ext(__le16 fc)
- /**
-  * ieee80211_is_data_qos - check if type is IEEE80211_FTYPE_DATA and IEEE80211_STYPE_QOS_DATA is set
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a QoS data frame
-  */
- static inline bool ieee80211_is_data_qos(__le16 fc)
- {
-@@ -485,6 +499,8 @@ static inline bool ieee80211_is_data_qos(__le16 fc)
- /**
-  * ieee80211_is_data_present - check if type is IEEE80211_FTYPE_DATA and has data
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a QoS data frame that has data
-+ *	(i.e. is not null data)
-  */
- static inline bool ieee80211_is_data_present(__le16 fc)
- {
-@@ -499,6 +515,7 @@ static inline bool ieee80211_is_data_present(__le16 fc)
- /**
-  * ieee80211_is_assoc_req - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_ASSOC_REQ
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is an association request
-  */
- static inline bool ieee80211_is_assoc_req(__le16 fc)
- {
-@@ -509,6 +526,7 @@ static inline bool ieee80211_is_assoc_req(__le16 fc)
- /**
-  * ieee80211_is_assoc_resp - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_ASSOC_RESP
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is an association response
-  */
- static inline bool ieee80211_is_assoc_resp(__le16 fc)
- {
-@@ -519,6 +537,7 @@ static inline bool ieee80211_is_assoc_resp(__le16 fc)
- /**
-  * ieee80211_is_reassoc_req - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_REASSOC_REQ
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a reassociation request
-  */
- static inline bool ieee80211_is_reassoc_req(__le16 fc)
- {
-@@ -529,6 +548,7 @@ static inline bool ieee80211_is_reassoc_req(__le16 fc)
- /**
-  * ieee80211_is_reassoc_resp - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_REASSOC_RESP
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a reassociation response
-  */
- static inline bool ieee80211_is_reassoc_resp(__le16 fc)
- {
-@@ -539,6 +559,7 @@ static inline bool ieee80211_is_reassoc_resp(__le16 fc)
- /**
-  * ieee80211_is_probe_req - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_PROBE_REQ
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a probe request
-  */
- static inline bool ieee80211_is_probe_req(__le16 fc)
- {
-@@ -549,6 +570,7 @@ static inline bool ieee80211_is_probe_req(__le16 fc)
- /**
-  * ieee80211_is_probe_resp - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_PROBE_RESP
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a probe response
-  */
- static inline bool ieee80211_is_probe_resp(__le16 fc)
- {
-@@ -559,6 +581,7 @@ static inline bool ieee80211_is_probe_resp(__le16 fc)
- /**
-  * ieee80211_is_beacon - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_BEACON
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a (regular, not S1G) beacon
-  */
- static inline bool ieee80211_is_beacon(__le16 fc)
- {
-@@ -567,42 +590,9 @@ static inline bool ieee80211_is_beacon(__le16 fc)
- }
- 
- /**
- * ieee80211_is_s1g_beacon - check if IEEE80211_FTYPE_EXT &&
- * IEEE80211_STYPE_S1G_BEACON
- * @fc: frame control bytes in little-endian byteorder
- */
-static inline bool ieee80211_is_s1g_beacon(__le16 fc)
-{
-	return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE |
-				 IEEE80211_FCTL_STYPE)) ==
-	       cpu_to_le16(IEEE80211_FTYPE_EXT | IEEE80211_STYPE_S1G_BEACON);
-}
-
-/**
- * ieee80211_next_tbtt_present - check if IEEE80211_FTYPE_EXT &&
- * IEEE80211_STYPE_S1G_BEACON && IEEE80211_S1G_BCN_NEXT_TBTT
- * @fc: frame control bytes in little-endian byteorder
- */
-static inline bool ieee80211_next_tbtt_present(__le16 fc)
-{
-	return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
-	       cpu_to_le16(IEEE80211_FTYPE_EXT | IEEE80211_STYPE_S1G_BEACON) &&
-	       fc & cpu_to_le16(IEEE80211_S1G_BCN_NEXT_TBTT);
-}
-
-/**
- * ieee80211_is_s1g_short_beacon - check if next tbtt present bit is set. Only
- * true for S1G beacons when they're short.
- * @fc: frame control bytes in little-endian byteorder
- */
-static inline bool ieee80211_is_s1g_short_beacon(__le16 fc)
-{
-	return ieee80211_is_s1g_beacon(fc) && ieee80211_next_tbtt_present(fc);
-}
-
-/**
-  * ieee80211_is_atim - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_ATIM
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is an ATIM frame
-  */
- static inline bool ieee80211_is_atim(__le16 fc)
- {
-@@ -613,6 +603,7 @@ static inline bool ieee80211_is_atim(__le16 fc)
- /**
-  * ieee80211_is_disassoc - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_DISASSOC
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a disassociation frame
-  */
- static inline bool ieee80211_is_disassoc(__le16 fc)
- {
-@@ -623,6 +614,7 @@ static inline bool ieee80211_is_disassoc(__le16 fc)
- /**
-  * ieee80211_is_auth - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_AUTH
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is an authentication frame
-  */
- static inline bool ieee80211_is_auth(__le16 fc)
- {
-@@ -633,6 +625,7 @@ static inline bool ieee80211_is_auth(__le16 fc)
- /**
-  * ieee80211_is_deauth - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_DEAUTH
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a deauthentication frame
-  */
- static inline bool ieee80211_is_deauth(__le16 fc)
- {
-@@ -643,6 +636,7 @@ static inline bool ieee80211_is_deauth(__le16 fc)
- /**
-  * ieee80211_is_action - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_ACTION
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is an action frame
-  */
- static inline bool ieee80211_is_action(__le16 fc)
- {
-@@ -653,6 +647,7 @@ static inline bool ieee80211_is_action(__le16 fc)
- /**
-  * ieee80211_is_back_req - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_BACK_REQ
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a block-ACK request frame
-  */
- static inline bool ieee80211_is_back_req(__le16 fc)
- {
-@@ -663,6 +658,7 @@ static inline bool ieee80211_is_back_req(__le16 fc)
- /**
-  * ieee80211_is_back - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_BACK
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a block-ACK frame
-  */
- static inline bool ieee80211_is_back(__le16 fc)
- {
-@@ -673,6 +669,7 @@ static inline bool ieee80211_is_back(__le16 fc)
- /**
-  * ieee80211_is_pspoll - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_PSPOLL
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a PS-poll frame
-  */
- static inline bool ieee80211_is_pspoll(__le16 fc)
- {
-@@ -683,6 +680,7 @@ static inline bool ieee80211_is_pspoll(__le16 fc)
- /**
-  * ieee80211_is_rts - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_RTS
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is an RTS frame
-  */
- static inline bool ieee80211_is_rts(__le16 fc)
- {
-@@ -693,6 +691,7 @@ static inline bool ieee80211_is_rts(__le16 fc)
- /**
-  * ieee80211_is_cts - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_CTS
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a CTS frame
-  */
- static inline bool ieee80211_is_cts(__le16 fc)
- {
-@@ -703,6 +702,7 @@ static inline bool ieee80211_is_cts(__le16 fc)
- /**
-  * ieee80211_is_ack - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_ACK
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is an ACK frame
-  */
- static inline bool ieee80211_is_ack(__le16 fc)
- {
-@@ -713,6 +713,7 @@ static inline bool ieee80211_is_ack(__le16 fc)
- /**
-  * ieee80211_is_cfend - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_CFEND
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a CF-end frame
-  */
- static inline bool ieee80211_is_cfend(__le16 fc)
- {
-@@ -723,6 +724,7 @@ static inline bool ieee80211_is_cfend(__le16 fc)
- /**
-  * ieee80211_is_cfendack - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_CFENDACK
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a CF-end-ack frame
-  */
- static inline bool ieee80211_is_cfendack(__le16 fc)
- {
-@@ -733,6 +735,7 @@ static inline bool ieee80211_is_cfendack(__le16 fc)
- /**
-  * ieee80211_is_nullfunc - check if frame is a regular (non-QoS) nullfunc frame
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a nullfunc frame
-  */
- static inline bool ieee80211_is_nullfunc(__le16 fc)
- {
-@@ -743,6 +746,7 @@ static inline bool ieee80211_is_nullfunc(__le16 fc)
- /**
-  * ieee80211_is_qos_nullfunc - check if frame is a QoS nullfunc frame
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a QoS nullfunc frame
-  */
- static inline bool ieee80211_is_qos_nullfunc(__le16 fc)
- {
-@@ -753,6 +757,7 @@ static inline bool ieee80211_is_qos_nullfunc(__le16 fc)
- /**
-  * ieee80211_is_trigger - check if frame is trigger frame
-  * @fc: frame control field in little-endian byteorder
-+ * Return: whether or not the frame is a trigger frame
-  */
- static inline bool ieee80211_is_trigger(__le16 fc)
- {
-@@ -763,6 +768,7 @@ static inline bool ieee80211_is_trigger(__le16 fc)
- /**
-  * ieee80211_is_any_nullfunc - check if frame is regular or QoS nullfunc frame
-  * @fc: frame control bytes in little-endian byteorder
-+ * Return: whether or not the frame is a nullfunc or QoS nullfunc frame
-  */
- static inline bool ieee80211_is_any_nullfunc(__le16 fc)
- {
-@@ -770,22 +776,10 @@ static inline bool ieee80211_is_any_nullfunc(__le16 fc)
- }
- 
- /**
- * ieee80211_is_bufferable_mmpdu - check if frame is bufferable MMPDU
- * @fc: frame control field in little-endian byteorder
- */
-static inline bool ieee80211_is_bufferable_mmpdu(__le16 fc)
-{
-	/* IEEE 802.11-2012, definition of "bufferable management frame";
-	 * note that this ignores the IBSS special case. */
-	return ieee80211_is_mgmt(fc) &&
-	       (ieee80211_is_action(fc) ||
-		ieee80211_is_disassoc(fc) ||
-		ieee80211_is_deauth(fc));
-}
-
-/**
-  * ieee80211_is_first_frag - check if IEEE80211_SCTL_FRAG is not set
-  * @seq_ctrl: frame sequence control bytes in little-endian byteorder
-+ * Return: whether or not the frame is the first fragment (also true if
-+ *	it's not fragmented at all)
-  */
- static inline bool ieee80211_is_first_frag(__le16 seq_ctrl)
- {
-@@ -795,6 +789,7 @@ static inline bool ieee80211_is_first_frag(__le16 seq_ctrl)
- /**
-  * ieee80211_is_frag - check if a frame is a fragment
-  * @hdr: 802.11 header of the frame
-+ * Return: whether or not the frame is a fragment
-  */
- static inline bool ieee80211_is_frag(struct ieee80211_hdr *hdr)
- {
-@@ -802,44 +797,20 @@ static inline bool ieee80211_is_frag(struct ieee80211_hdr *hdr)
- 	       hdr->seq_ctrl & cpu_to_le16(IEEE80211_SCTL_FRAG);
- }
- 
-struct ieee80211s_hdr {
-	u8 flags;
-	u8 ttl;
-	__le32 seqnum;
-	u8 eaddr1[ETH_ALEN];
-	u8 eaddr2[ETH_ALEN];
-} __packed __aligned(2);
-
-/* Mesh flags */
-#define MESH_FLAGS_AE_A4 	0x1
-#define MESH_FLAGS_AE_A5_A6	0x2
-#define MESH_FLAGS_AE		0x3
-#define MESH_FLAGS_PS_DEEP	0x4
-
-/**
- * enum ieee80211_preq_flags - mesh PREQ element flags
- *
- * @IEEE80211_PREQ_PROACTIVE_PREP_FLAG: proactive PREP subfield
- */
-enum ieee80211_preq_flags {
-	IEEE80211_PREQ_PROACTIVE_PREP_FLAG	= 1<<2,
-};
-
-/**
- * enum ieee80211_preq_target_flags - mesh PREQ element per target flags
- *
- * @IEEE80211_PREQ_TO_FLAG: target only subfield
- * @IEEE80211_PREQ_USN_FLAG: unknown target HWMP sequence number subfield
- */
-enum ieee80211_preq_target_flags {
-	IEEE80211_PREQ_TO_FLAG	= 1<<0,
-	IEEE80211_PREQ_USN_FLAG	= 1<<2,
-};
-+static inline u16 ieee80211_get_sn(struct ieee80211_hdr *hdr)
-+{
-+	return le16_get_bits(hdr->seq_ctrl, IEEE80211_SCTL_SEQ);
-+}
- 
- /**
- * struct ieee80211_quiet_ie
-+ * struct ieee80211_quiet_ie - Quiet element
-+ * @count: Quiet Count
-+ * @period: Quiet Period
-+ * @duration: Quiet Duration
-+ * @offset: Quiet Offset
-  *
- * This structure refers to "Quiet information element"
-+ * This structure represents the payload of the "Quiet element" as
-+ * described in IEEE Std 802.11-2020 section 9.4.2.22.
-  */
- struct ieee80211_quiet_ie {
- 	u8 count;
-@@ -849,9 +820,15 @@ struct ieee80211_quiet_ie {
- } __packed;
- 
- /**
- * struct ieee80211_msrment_ie
-+ * struct ieee80211_msrment_ie - Measurement element
-+ * @token: Measurement Token
-+ * @mode: Measurement Report Mode
-+ * @type: Measurement Type
-+ * @request: Measurement Request or Measurement Report
-  *
- * This structure refers to "Measurement Request/Report information element"
-+ * This structure represents the payload of both the "Measurement
-+ * Request element" and the "Measurement Report element" as described
-+ * in IEEE Std 802.11-2020 sections 9.4.2.20 and 9.4.2.21.
-  */
- struct ieee80211_msrment_ie {
- 	u8 token;
-@@ -861,9 +838,14 @@ struct ieee80211_msrment_ie {
- } __packed;
- 
- /**
- * struct ieee80211_channel_sw_ie
-+ * struct ieee80211_channel_sw_ie - Channel Switch Announcement element
-+ * @mode: Channel Switch Mode
-+ * @new_ch_num: New Channel Number
-+ * @count: Channel Switch Count
-  *
- * This structure refers to "Channel Switch Announcement information element"
-+ * This structure represents the payload of the "Channel Switch
-+ * Announcement element" as described in IEEE Std 802.11-2020 section
-+ * 9.4.2.18.
-  */
- struct ieee80211_channel_sw_ie {
- 	u8 mode;
-@@ -872,9 +854,14 @@ struct ieee80211_channel_sw_ie {
- } __packed;
- 
- /**
- * struct ieee80211_ext_chansw_ie
-+ * struct ieee80211_ext_chansw_ie - Extended Channel Switch Announcement element
-+ * @mode: Channel Switch Mode
-+ * @new_operating_class: New Operating Class
-+ * @new_ch_num: New Channel Number
-+ * @count: Channel Switch Count
-  *
- * This structure represents the "Extended Channel Switch Announcement element"
-+ * This structure represents the "Extended Channel Switch Announcement
-+ * element" as described in IEEE Std 802.11-2020 section 9.4.2.52.
-  */
- struct ieee80211_ext_chansw_ie {
- 	u8 mode;
-@@ -894,19 +881,14 @@ struct ieee80211_sec_chan_offs_ie {
- } __packed;
- 
- /**
- * struct ieee80211_mesh_chansw_params_ie - mesh channel switch parameters IE
- *
- * This structure represents the "Mesh Channel Switch Paramters element"
- */
-struct ieee80211_mesh_chansw_params_ie {
-	u8 mesh_ttl;
-	u8 mesh_flags;
-	__le16 mesh_reason;
-	__le16 mesh_pre_value;
-} __packed;
-
-/**
-  * struct ieee80211_wide_bw_chansw_ie - wide bandwidth channel switch IE
-+ * @new_channel_width: New Channel Width
-+ * @new_center_freq_seg0: New Channel Center Frequency Segment 0
-+ * @new_center_freq_seg1: New Channel Center Frequency Segment 1
-+ *
-+ * This structure represents the payload of the "Wide Bandwidth
-+ * Channel Switch element" as described in IEEE Std 802.11-2020
-+ * section 9.4.2.160.
-  */
- struct ieee80211_wide_bw_chansw_ie {
- 	u8 new_channel_width;
-@@ -914,138 +896,41 @@ struct ieee80211_wide_bw_chansw_ie {
- } __packed;
- 
- /**
- * struct ieee80211_tim
-+ * struct ieee80211_tim_ie - Traffic Indication Map information element
-+ * @dtim_count: DTIM Count
-+ * @dtim_period: DTIM Period
-+ * @bitmap_ctrl: Bitmap Control
-+ * @required_octet: "Syntatic sugar" to force the struct size to the
-+ *                  minimum valid size when carried in a non-S1G PPDU
-+ * @virtual_map: Partial Virtual Bitmap
-  *
- * This structure refers to "Traffic Indication Map information element"
-+ * This structure represents the payload of the "TIM element" as
-+ * described in IEEE Std 802.11-2020 section 9.4.2.5. Note that this
-+ * definition is only applicable when the element is carried in a
-+ * non-S1G PPDU. When the TIM is carried in an S1G PPDU, the Bitmap
-+ * Control and Partial Virtual Bitmap may not be present.
-  */
- struct ieee80211_tim_ie {
- 	u8 dtim_count;
- 	u8 dtim_period;
- 	u8 bitmap_ctrl;
-	/* variable size: 1 - 251 bytes */
-	u8 virtual_map[1];
-} __packed;
-
-/**
- * struct ieee80211_meshconf_ie
- *
- * This structure refers to "Mesh Configuration information element"
- */
-struct ieee80211_meshconf_ie {
-	u8 meshconf_psel;
-	u8 meshconf_pmetric;
-	u8 meshconf_congest;
-	u8 meshconf_synch;
-	u8 meshconf_auth;
-	u8 meshconf_form;
-	u8 meshconf_cap;
-} __packed;
-
-/**
- * enum mesh_config_capab_flags - Mesh Configuration IE capability field flags
- *
- * @IEEE80211_MESHCONF_CAPAB_ACCEPT_PLINKS: STA is willing to establish
- *	additional mesh peerings with other mesh STAs
- * @IEEE80211_MESHCONF_CAPAB_FORWARDING: the STA forwards MSDUs
- * @IEEE80211_MESHCONF_CAPAB_TBTT_ADJUSTING: TBTT adjustment procedure
- *	is ongoing
- * @IEEE80211_MESHCONF_CAPAB_POWER_SAVE_LEVEL: STA is in deep sleep mode or has
- *	neighbors in deep sleep mode
- */
-enum mesh_config_capab_flags {
-	IEEE80211_MESHCONF_CAPAB_ACCEPT_PLINKS		= 0x01,
-	IEEE80211_MESHCONF_CAPAB_FORWARDING		= 0x08,
-	IEEE80211_MESHCONF_CAPAB_TBTT_ADJUSTING		= 0x20,
-	IEEE80211_MESHCONF_CAPAB_POWER_SAVE_LEVEL	= 0x40,
-};
-
-#define IEEE80211_MESHCONF_FORM_CONNECTED_TO_GATE 0x1
-
-/**
- * mesh channel switch parameters element's flag indicator
- *
- */
-#define WLAN_EID_CHAN_SWITCH_PARAM_TX_RESTRICT BIT(0)
-#define WLAN_EID_CHAN_SWITCH_PARAM_INITIATOR BIT(1)
-#define WLAN_EID_CHAN_SWITCH_PARAM_REASON BIT(2)
-
-/**
- * struct ieee80211_rann_ie
- *
- * This structure refers to "Root Announcement information element"
- */
-struct ieee80211_rann_ie {
-	u8 rann_flags;
-	u8 rann_hopcount;
-	u8 rann_ttl;
-	u8 rann_addr[ETH_ALEN];
-	__le32 rann_seq;
-	__le32 rann_interval;
-	__le32 rann_metric;
-+	union {
-+		u8 required_octet;
-+		DECLARE_FLEX_ARRAY(u8, virtual_map);
-+	};
- } __packed;
- 
-enum ieee80211_rann_flags {
-	RANN_FLAG_IS_GATE = 1 << 0,
-};
-
-enum ieee80211_ht_chanwidth_values {
-	IEEE80211_HT_CHANWIDTH_20MHZ = 0,
-	IEEE80211_HT_CHANWIDTH_ANY = 1,
-};
-
-/**
- * enum ieee80211_opmode_bits - VHT operating mode field bits
- * @IEEE80211_OPMODE_NOTIF_CHANWIDTH_MASK: channel width mask
- * @IEEE80211_OPMODE_NOTIF_CHANWIDTH_20MHZ: 20 MHz channel width
- * @IEEE80211_OPMODE_NOTIF_CHANWIDTH_40MHZ: 40 MHz channel width
- * @IEEE80211_OPMODE_NOTIF_CHANWIDTH_80MHZ: 80 MHz channel width
- * @IEEE80211_OPMODE_NOTIF_CHANWIDTH_160MHZ: 160 MHz or 80+80 MHz channel width
- * @IEEE80211_OPMODE_NOTIF_BW_160_80P80: 160 / 80+80 MHz indicator flag
- * @IEEE80211_OPMODE_NOTIF_RX_NSS_MASK: number of spatial streams mask
- *	(the NSS value is the value of this field + 1)
- * @IEEE80211_OPMODE_NOTIF_RX_NSS_SHIFT: number of spatial streams shift
- * @IEEE80211_OPMODE_NOTIF_RX_NSS_TYPE_BF: indicates streams in SU-MIMO PPDU
- *	using a beamforming steering matrix
- */
-enum ieee80211_vht_opmode_bits {
-	IEEE80211_OPMODE_NOTIF_CHANWIDTH_MASK	= 0x03,
-	IEEE80211_OPMODE_NOTIF_CHANWIDTH_20MHZ	= 0,
-	IEEE80211_OPMODE_NOTIF_CHANWIDTH_40MHZ	= 1,
-	IEEE80211_OPMODE_NOTIF_CHANWIDTH_80MHZ	= 2,
-	IEEE80211_OPMODE_NOTIF_CHANWIDTH_160MHZ	= 3,
-	IEEE80211_OPMODE_NOTIF_BW_160_80P80	= 0x04,
-	IEEE80211_OPMODE_NOTIF_RX_NSS_MASK	= 0x70,
-	IEEE80211_OPMODE_NOTIF_RX_NSS_SHIFT	= 4,
-	IEEE80211_OPMODE_NOTIF_RX_NSS_TYPE_BF	= 0x80,
-};
-
-/**
- * enum ieee80211_s1g_chanwidth
- * These are defined in IEEE802.11-2016ah Table 10-20
- * as BSS Channel Width
- *
- * @IEEE80211_S1G_CHANWIDTH_1MHZ: 1MHz operating channel
- * @IEEE80211_S1G_CHANWIDTH_2MHZ: 2MHz operating channel
- * @IEEE80211_S1G_CHANWIDTH_4MHZ: 4MHz operating channel
- * @IEEE80211_S1G_CHANWIDTH_8MHZ: 8MHz operating channel
- * @IEEE80211_S1G_CHANWIDTH_16MHZ: 16MHz operating channel
- */
-enum ieee80211_s1g_chanwidth {
-	IEEE80211_S1G_CHANWIDTH_1MHZ = 0,
-	IEEE80211_S1G_CHANWIDTH_2MHZ = 1,
-	IEEE80211_S1G_CHANWIDTH_4MHZ = 3,
-	IEEE80211_S1G_CHANWIDTH_8MHZ = 7,
-	IEEE80211_S1G_CHANWIDTH_16MHZ = 15,
-};
-
- #define WLAN_SA_QUERY_TR_ID_LEN 2
- #define WLAN_MEMBERSHIP_LEN 8
- #define WLAN_USER_POSITION_LEN 16
- 
- /**
- * struct ieee80211_tpc_report_ie
-+ * struct ieee80211_tpc_report_ie - TPC Report element
-+ * @tx_power: Transmit Power
-+ * @link_margin: Link Margin
-  *
- * This structure refers to "TPC Report element"
-+ * This structure represents the payload of the "TPC Report element" as
-+ * described in IEEE Std 802.11-2020 section 9.4.2.16.
-  */
- struct ieee80211_tpc_report_ie {
- 	u8 tx_power;
-@@ -1062,46 +947,6 @@ struct ieee80211_addba_ext_ie {
- 	u8 data;
- } __packed;
- 
-/**
- * struct ieee80211_s1g_bcn_compat_ie
- *
- * S1G Beacon Compatibility element
- */
-struct ieee80211_s1g_bcn_compat_ie {
-	__le16 compat_info;
-	__le16 beacon_int;
-	__le32 tsf_completion;
-} __packed;
-
-/**
- * struct ieee80211_s1g_oper_ie
- *
- * S1G Operation element
- */
-struct ieee80211_s1g_oper_ie {
-	u8 ch_width;
-	u8 oper_class;
-	u8 primary_ch;
-	u8 oper_ch;
-	__le16 basic_mcs_nss;
-} __packed;
-
-/**
- * struct ieee80211_aid_response_ie
- *
- * AID Response element
- */
-struct ieee80211_aid_response_ie {
-	__le16 aid;
-	u8 switch_count;
-	__le16 response_int;
-} __packed;
-
-struct ieee80211_s1g_cap {
-	u8 capab_info[10];
-	u8 supp_mcs_nss[5];
-} __packed;
-
- struct ieee80211_ext {
- 	__le16 frame_control;
- 	__le16 duration;
-@@ -1110,58 +955,27 @@ struct ieee80211_ext {
- 			u8 sa[ETH_ALEN];
- 			__le32 timestamp;
- 			u8 change_seq;
-			u8 variable[0];
-+			u8 variable[];
- 		} __packed s1g_beacon;
-		struct {
-			u8 sa[ETH_ALEN];
-			__le32 timestamp;
-			u8 change_seq;
-			u8 next_tbtt[3];
-			u8 variable[0];
-		} __packed s1g_short_beacon;
- 	} u;
- } __packed __aligned(2);
- 
-#define IEEE80211_TWT_CONTROL_NDP			BIT(0)
-#define IEEE80211_TWT_CONTROL_RESP_MODE			BIT(1)
-#define IEEE80211_TWT_CONTROL_NEG_TYPE_BROADCAST	BIT(3)
-#define IEEE80211_TWT_CONTROL_RX_DISABLED		BIT(4)
-#define IEEE80211_TWT_CONTROL_WAKE_DUR_UNIT		BIT(5)
-
-#define IEEE80211_TWT_REQTYPE_REQUEST			BIT(0)
-#define IEEE80211_TWT_REQTYPE_SETUP_CMD			GENMASK(3, 1)
-#define IEEE80211_TWT_REQTYPE_TRIGGER			BIT(4)
-#define IEEE80211_TWT_REQTYPE_IMPLICIT			BIT(5)
-#define IEEE80211_TWT_REQTYPE_FLOWTYPE			BIT(6)
-#define IEEE80211_TWT_REQTYPE_FLOWID			GENMASK(9, 7)
-#define IEEE80211_TWT_REQTYPE_WAKE_INT_EXP		GENMASK(14, 10)
-#define IEEE80211_TWT_REQTYPE_PROTECTION		BIT(15)
-
-enum ieee80211_twt_setup_cmd {
-	TWT_SETUP_CMD_REQUEST,
-	TWT_SETUP_CMD_SUGGEST,
-	TWT_SETUP_CMD_DEMAND,
-	TWT_SETUP_CMD_GROUPING,
-	TWT_SETUP_CMD_ACCEPT,
-	TWT_SETUP_CMD_ALTERNATE,
-	TWT_SETUP_CMD_DICTATE,
-	TWT_SETUP_CMD_REJECT,
-};
-
-struct ieee80211_twt_params {
-	__le16 req_type;
-	__le64 twt;
-	u8 min_twt_dur;
-	__le16 mantissa;
-	u8 channel;
-} __packed;
-
-struct ieee80211_twt_setup {
-	u8 dialog_token;
-	u8 element_id;
-	u8 length;
-	u8 control;
-	u8 params[];
-+/**
-+ * struct ieee80211_bss_load_elem - BSS Load elemen
-+ *
-+ * Defined in section 9.4.2.26 in IEEE 802.11-REVme D4.1
-+ *
-+ * @sta_count: total number of STAs currently associated with the AP.
-+ * @channel_util: Percentage of time that the access point sensed the channel
-+ *	was busy. This value is in range [0, 255], the highest value means
-+ *	100% busy.
-+ * @avail_admission_capa: remaining amount of medium time used for admission
-+ *	control.
-+ */
-+struct ieee80211_bss_load_elem {
-+	__le16 sta_count;
-+	u8 channel_util;
-+	__le16 avail_admission_capa;
- } __packed;
- 
- struct ieee80211_mgmt {
-@@ -1270,6 +1084,8 @@ struct ieee80211_mgmt {
- 					__le16 status;
- 					__le16 capab;
- 					__le16 timeout;
-+					/* followed by BA Extension */
-+					u8 variable[];
- 				} __packed addba_resp;
- 				struct{
- 					u8 action_code;
-@@ -1300,7 +1116,7 @@ struct ieee80211_mgmt {
- 					u8 action_code;
- 					u8 dialog_token;
- 					__le16 capability;
-					u8 variable[0];
-+					u8 variable[];
- 				} __packed tdls_discover_resp;
- 				struct {
- 					u8 action_code;
-@@ -1341,22 +1157,57 @@ struct ieee80211_mgmt {
- 					u8 max_tod_error;
- 					u8 max_toa_error;
- 				} __packed wnm_timing_msr;
-+				struct {
-+					u8 action_code;
-+					u8 dialog_token;
-+					u8 variable[];
-+				} __packed ttlm_req;
-+				struct {
-+					u8 action_code;
-+					u8 dialog_token;
-+					__le16 status_code;
-+					u8 variable[];
-+				} __packed ttlm_res;
-+				struct {
-+					u8 action_code;
-+				} __packed ttlm_tear_down;
-+				struct {
-+					u8 action_code;
-+					u8 dialog_token;
-+					u8 variable[];
-+				} __packed ml_reconf_req;
-+				struct {
-+					u8 action_code;
-+					u8 dialog_token;
-+					u8 count;
-+					u8 variable[];
-+				} __packed ml_reconf_resp;
-+				struct {
-+					u8 action_code;
-+					u8 variable[];
-+				} __packed epcs;
- 			} u;
- 		} __packed action;
-+		DECLARE_FLEX_ARRAY(u8, body); /* Generic frame body */
- 	} u;
- } __packed __aligned(2);
- 
- /* Supported rates membership selectors */
- #define BSS_MEMBERSHIP_SELECTOR_HT_PHY	127
- #define BSS_MEMBERSHIP_SELECTOR_VHT_PHY	126
-#define BSS_MEMBERSHIP_SELECTOR_HE_PHY	122
-+#define BSS_MEMBERSHIP_SELECTOR_GLK	125
-+#define BSS_MEMBERSHIP_SELECTOR_EPD	124
- #define BSS_MEMBERSHIP_SELECTOR_SAE_H2E 123
-+#define BSS_MEMBERSHIP_SELECTOR_HE_PHY	122
-+#define BSS_MEMBERSHIP_SELECTOR_EHT_PHY	121
-+
-+#define BSS_MEMBERSHIP_SELECTOR_MIN	BSS_MEMBERSHIP_SELECTOR_EHT_PHY
- 
- /* mgmt header + 1 byte category code */
- #define IEEE80211_MIN_ACTION_SIZE offsetof(struct ieee80211_mgmt, u.action.u)
- 
- 
-/* Management MIC information element (IEEE 802.11w) */
-+/* Management MIC information element (IEEE 802.11w) for CMAC */
- struct ieee80211_mmie {
- 	u8 element_id;
- 	u8 length;
-@@ -1374,6 +1225,15 @@ struct ieee80211_mmie_16 {
- 	u8 mic[16];
- } __packed;
- 
-+/* Management MIC information element (IEEE 802.11w) for all variants */
-+struct ieee80211_mmie_var {
-+	u8 element_id;
-+	u8 length;
-+	__le16 key_id;
-+	u8 sequence_number[6];
-+	u8 mic[]; /* 8 or 16 bytes */
-+} __packed;
-+
- struct ieee80211_vendor_ie {
- 	u8 element_id;
- 	u8 len;
-@@ -1450,1553 +1310,39 @@ struct ieee80211_tdls_data {
- 		struct {
- 			u8 dialog_token;
- 			__le16 capability;
-			u8 variable[0];
-+			u8 variable[];
- 		} __packed setup_req;
- 		struct {
- 			__le16 status_code;
- 			u8 dialog_token;
- 			__le16 capability;
-			u8 variable[0];
-+			u8 variable[];
- 		} __packed setup_resp;
- 		struct {
- 			__le16 status_code;
- 			u8 dialog_token;
-			u8 variable[0];
-+			u8 variable[];
- 		} __packed setup_cfm;
- 		struct {
- 			__le16 reason_code;
-			u8 variable[0];
-+			u8 variable[];
- 		} __packed teardown;
- 		struct {
- 			u8 dialog_token;
-			u8 variable[0];
-+			u8 variable[];
- 		} __packed discover_req;
- 		struct {
- 			u8 target_channel;
- 			u8 oper_class;
-			u8 variable[0];
-+			u8 variable[];
- 		} __packed chan_switch_req;
- 		struct {
- 			__le16 status_code;
-			u8 variable[0];
-+			u8 variable[];
- 		} __packed chan_switch_resp;
- 	} u;
- } __packed;
- 
-/*
- * Peer-to-Peer IE attribute related definitions.
- */
-/**
- * enum ieee80211_p2p_attr_id - identifies type of peer-to-peer attribute.
- */
-enum ieee80211_p2p_attr_id {
-	IEEE80211_P2P_ATTR_STATUS = 0,
-	IEEE80211_P2P_ATTR_MINOR_REASON,
-	IEEE80211_P2P_ATTR_CAPABILITY,
-	IEEE80211_P2P_ATTR_DEVICE_ID,
-	IEEE80211_P2P_ATTR_GO_INTENT,
-	IEEE80211_P2P_ATTR_GO_CONFIG_TIMEOUT,
-	IEEE80211_P2P_ATTR_LISTEN_CHANNEL,
-	IEEE80211_P2P_ATTR_GROUP_BSSID,
-	IEEE80211_P2P_ATTR_EXT_LISTEN_TIMING,
-	IEEE80211_P2P_ATTR_INTENDED_IFACE_ADDR,
-	IEEE80211_P2P_ATTR_MANAGABILITY,
-	IEEE80211_P2P_ATTR_CHANNEL_LIST,
-	IEEE80211_P2P_ATTR_ABSENCE_NOTICE,
-	IEEE80211_P2P_ATTR_DEVICE_INFO,
-	IEEE80211_P2P_ATTR_GROUP_INFO,
-	IEEE80211_P2P_ATTR_GROUP_ID,
-	IEEE80211_P2P_ATTR_INTERFACE,
-	IEEE80211_P2P_ATTR_OPER_CHANNEL,
-	IEEE80211_P2P_ATTR_INVITE_FLAGS,
-	/* 19 - 220: Reserved */
-	IEEE80211_P2P_ATTR_VENDOR_SPECIFIC = 221,
-
-	IEEE80211_P2P_ATTR_MAX
-};
-
-/* Notice of Absence attribute - described in P2P spec 4.1.14 */
-/* Typical max value used here */
-#define IEEE80211_P2P_NOA_DESC_MAX	4
-
-struct ieee80211_p2p_noa_desc {
-	u8 count;
-	__le32 duration;
-	__le32 interval;
-	__le32 start_time;
-} __packed;
-
-struct ieee80211_p2p_noa_attr {
-	u8 index;
-	u8 oppps_ctwindow;
-	struct ieee80211_p2p_noa_desc desc[IEEE80211_P2P_NOA_DESC_MAX];
-} __packed;
-
-#define IEEE80211_P2P_OPPPS_ENABLE_BIT		BIT(7)
-#define IEEE80211_P2P_OPPPS_CTWINDOW_MASK	0x7F
-
-/**
- * struct ieee80211_bar - HT Block Ack Request
- *
- * This structure refers to "HT BlockAckReq" as
- * described in 802.11n draft section 7.2.1.7.1
- */
-struct ieee80211_bar {
-	__le16 frame_control;
-	__le16 duration;
-	__u8 ra[ETH_ALEN];
-	__u8 ta[ETH_ALEN];
-	__le16 control;
-	__le16 start_seq_num;
-} __packed;
-
-/* 802.11 BAR control masks */
-#define IEEE80211_BAR_CTRL_ACK_POLICY_NORMAL	0x0000
-#define IEEE80211_BAR_CTRL_MULTI_TID		0x0002
-#define IEEE80211_BAR_CTRL_CBMTID_COMPRESSED_BA	0x0004
-#define IEEE80211_BAR_CTRL_TID_INFO_MASK	0xf000
-#define IEEE80211_BAR_CTRL_TID_INFO_SHIFT	12
-
-#define IEEE80211_HT_MCS_MASK_LEN		10
-
-/**
- * struct ieee80211_mcs_info - MCS information
- * @rx_mask: RX mask
- * @rx_highest: highest supported RX rate. If set represents
- *	the highest supported RX data rate in units of 1 Mbps.
- *	If this field is 0 this value should not be used to
- *	consider the highest RX data rate supported.
- * @tx_params: TX parameters
- */
-struct ieee80211_mcs_info {
-	u8 rx_mask[IEEE80211_HT_MCS_MASK_LEN];
-	__le16 rx_highest;
-	u8 tx_params;
-	u8 reserved[3];
-} __packed;
-
-/* 802.11n HT capability MSC set */
-#define IEEE80211_HT_MCS_RX_HIGHEST_MASK	0x3ff
-#define IEEE80211_HT_MCS_TX_DEFINED		0x01
-#define IEEE80211_HT_MCS_TX_RX_DIFF		0x02
-/* value 0 == 1 stream etc */
-#define IEEE80211_HT_MCS_TX_MAX_STREAMS_MASK	0x0C
-#define IEEE80211_HT_MCS_TX_MAX_STREAMS_SHIFT	2
-#define		IEEE80211_HT_MCS_TX_MAX_STREAMS	4
-#define IEEE80211_HT_MCS_TX_UNEQUAL_MODULATION	0x10
-
-/*
- * 802.11n D5.0 20.3.5 / 20.6 says:
- * - indices 0 to 7 and 32 are single spatial stream
- * - 8 to 31 are multiple spatial streams using equal modulation
- *   [8..15 for two streams, 16..23 for three and 24..31 for four]
- * - remainder are multiple spatial streams using unequal modulation
- */
-#define IEEE80211_HT_MCS_UNEQUAL_MODULATION_START 33
-#define IEEE80211_HT_MCS_UNEQUAL_MODULATION_START_BYTE \
-	(IEEE80211_HT_MCS_UNEQUAL_MODULATION_START / 8)
-
-/**
- * struct ieee80211_ht_cap - HT capabilities
- *
- * This structure is the "HT capabilities element" as
- * described in 802.11n D5.0 7.3.2.57
- */
-struct ieee80211_ht_cap {
-	__le16 cap_info;
-	u8 ampdu_params_info;
-
-	/* 16 bytes MCS information */
-	struct ieee80211_mcs_info mcs;
-
-	__le16 extended_ht_cap_info;
-	__le32 tx_BF_cap_info;
-	u8 antenna_selection_info;
-} __packed;
-
-/* 802.11n HT capabilities masks (for cap_info) */
-#define IEEE80211_HT_CAP_LDPC_CODING		0x0001
-#define IEEE80211_HT_CAP_SUP_WIDTH_20_40	0x0002
-#define IEEE80211_HT_CAP_SM_PS			0x000C
-#define		IEEE80211_HT_CAP_SM_PS_SHIFT	2
-#define IEEE80211_HT_CAP_GRN_FLD		0x0010
-#define IEEE80211_HT_CAP_SGI_20			0x0020
-#define IEEE80211_HT_CAP_SGI_40			0x0040
-#define IEEE80211_HT_CAP_TX_STBC		0x0080
-#define IEEE80211_HT_CAP_RX_STBC		0x0300
-#define		IEEE80211_HT_CAP_RX_STBC_SHIFT	8
-#define IEEE80211_HT_CAP_DELAY_BA		0x0400
-#define IEEE80211_HT_CAP_MAX_AMSDU		0x0800
-#define IEEE80211_HT_CAP_DSSSCCK40		0x1000
-#define IEEE80211_HT_CAP_RESERVED		0x2000
-#define IEEE80211_HT_CAP_40MHZ_INTOLERANT	0x4000
-#define IEEE80211_HT_CAP_LSIG_TXOP_PROT		0x8000
-
-/* 802.11n HT extended capabilities masks (for extended_ht_cap_info) */
-#define IEEE80211_HT_EXT_CAP_PCO		0x0001
-#define IEEE80211_HT_EXT_CAP_PCO_TIME		0x0006
-#define		IEEE80211_HT_EXT_CAP_PCO_TIME_SHIFT	1
-#define IEEE80211_HT_EXT_CAP_MCS_FB		0x0300
-#define		IEEE80211_HT_EXT_CAP_MCS_FB_SHIFT	8
-#define IEEE80211_HT_EXT_CAP_HTC_SUP		0x0400
-#define IEEE80211_HT_EXT_CAP_RD_RESPONDER	0x0800
-
-/* 802.11n HT capability AMPDU settings (for ampdu_params_info) */
-#define IEEE80211_HT_AMPDU_PARM_FACTOR		0x03
-#define IEEE80211_HT_AMPDU_PARM_DENSITY		0x1C
-#define		IEEE80211_HT_AMPDU_PARM_DENSITY_SHIFT	2
-
-/*
- * Maximum length of AMPDU that the STA can receive in high-throughput (HT).
- * Length = 2 ^ (13 + max_ampdu_length_exp) - 1 (octets)
- */
-enum ieee80211_max_ampdu_length_exp {
-	IEEE80211_HT_MAX_AMPDU_8K = 0,
-	IEEE80211_HT_MAX_AMPDU_16K = 1,
-	IEEE80211_HT_MAX_AMPDU_32K = 2,
-	IEEE80211_HT_MAX_AMPDU_64K = 3
-};
-
-/*
- * Maximum length of AMPDU that the STA can receive in VHT.
- * Length = 2 ^ (13 + max_ampdu_length_exp) - 1 (octets)
- */
-enum ieee80211_vht_max_ampdu_length_exp {
-	IEEE80211_VHT_MAX_AMPDU_8K = 0,
-	IEEE80211_VHT_MAX_AMPDU_16K = 1,
-	IEEE80211_VHT_MAX_AMPDU_32K = 2,
-	IEEE80211_VHT_MAX_AMPDU_64K = 3,
-	IEEE80211_VHT_MAX_AMPDU_128K = 4,
-	IEEE80211_VHT_MAX_AMPDU_256K = 5,
-	IEEE80211_VHT_MAX_AMPDU_512K = 6,
-	IEEE80211_VHT_MAX_AMPDU_1024K = 7
-};
-
-#define IEEE80211_HT_MAX_AMPDU_FACTOR 13
-
-/* Minimum MPDU start spacing */
-enum ieee80211_min_mpdu_spacing {
-	IEEE80211_HT_MPDU_DENSITY_NONE = 0,	/* No restriction */
-	IEEE80211_HT_MPDU_DENSITY_0_25 = 1,	/* 1/4 usec */
-	IEEE80211_HT_MPDU_DENSITY_0_5 = 2,	/* 1/2 usec */
-	IEEE80211_HT_MPDU_DENSITY_1 = 3,	/* 1 usec */
-	IEEE80211_HT_MPDU_DENSITY_2 = 4,	/* 2 usec */
-	IEEE80211_HT_MPDU_DENSITY_4 = 5,	/* 4 usec */
-	IEEE80211_HT_MPDU_DENSITY_8 = 6,	/* 8 usec */
-	IEEE80211_HT_MPDU_DENSITY_16 = 7	/* 16 usec */
-};
-
-/**
- * struct ieee80211_ht_operation - HT operation IE
- *
- * This structure is the "HT operation element" as
- * described in 802.11n-2009 7.3.2.57
- */
-struct ieee80211_ht_operation {
-	u8 primary_chan;
-	u8 ht_param;
-	__le16 operation_mode;
-	__le16 stbc_param;
-	u8 basic_set[16];
-} __packed;
-
-/* for ht_param */
-#define IEEE80211_HT_PARAM_CHA_SEC_OFFSET		0x03
-#define		IEEE80211_HT_PARAM_CHA_SEC_NONE		0x00
-#define		IEEE80211_HT_PARAM_CHA_SEC_ABOVE	0x01
-#define		IEEE80211_HT_PARAM_CHA_SEC_BELOW	0x03
-#define IEEE80211_HT_PARAM_CHAN_WIDTH_ANY		0x04
-#define IEEE80211_HT_PARAM_RIFS_MODE			0x08
-
-/* for operation_mode */
-#define IEEE80211_HT_OP_MODE_PROTECTION			0x0003
-#define		IEEE80211_HT_OP_MODE_PROTECTION_NONE		0
-#define		IEEE80211_HT_OP_MODE_PROTECTION_NONMEMBER	1
-#define		IEEE80211_HT_OP_MODE_PROTECTION_20MHZ		2
-#define		IEEE80211_HT_OP_MODE_PROTECTION_NONHT_MIXED	3
-#define IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT		0x0004
-#define IEEE80211_HT_OP_MODE_NON_HT_STA_PRSNT		0x0010
-#define IEEE80211_HT_OP_MODE_CCFS2_SHIFT		5
-#define IEEE80211_HT_OP_MODE_CCFS2_MASK			0x1fe0
-
-/* for stbc_param */
-#define IEEE80211_HT_STBC_PARAM_DUAL_BEACON		0x0040
-#define IEEE80211_HT_STBC_PARAM_DUAL_CTS_PROT		0x0080
-#define IEEE80211_HT_STBC_PARAM_STBC_BEACON		0x0100
-#define IEEE80211_HT_STBC_PARAM_LSIG_TXOP_FULLPROT	0x0200
-#define IEEE80211_HT_STBC_PARAM_PCO_ACTIVE		0x0400
-#define IEEE80211_HT_STBC_PARAM_PCO_PHASE		0x0800
-
-
-/* block-ack parameters */
-#define IEEE80211_ADDBA_PARAM_AMSDU_MASK 0x0001
-#define IEEE80211_ADDBA_PARAM_POLICY_MASK 0x0002
-#define IEEE80211_ADDBA_PARAM_TID_MASK 0x003C
-#define IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK 0xFFC0
-#define IEEE80211_DELBA_PARAM_TID_MASK 0xF000
-#define IEEE80211_DELBA_PARAM_INITIATOR_MASK 0x0800
-
-/*
- * A-MPDU buffer sizes
- * According to HT size varies from 8 to 64 frames
- * HE adds the ability to have up to 256 frames.
- * EHT adds the ability to have up to 1K frames.
- */
-#define IEEE80211_MIN_AMPDU_BUF		0x8
-#define IEEE80211_MAX_AMPDU_BUF_HT	0x40
-#define IEEE80211_MAX_AMPDU_BUF_HE	0x100
-#define IEEE80211_MAX_AMPDU_BUF_EHT	0x400
-
-
-/* Spatial Multiplexing Power Save Modes (for capability) */
-#define WLAN_HT_CAP_SM_PS_STATIC	0
-#define WLAN_HT_CAP_SM_PS_DYNAMIC	1
-#define WLAN_HT_CAP_SM_PS_INVALID	2
-#define WLAN_HT_CAP_SM_PS_DISABLED	3
-
-/* for SM power control field lower two bits */
-#define WLAN_HT_SMPS_CONTROL_DISABLED	0
-#define WLAN_HT_SMPS_CONTROL_STATIC	1
-#define WLAN_HT_SMPS_CONTROL_DYNAMIC	3
-
-/**
- * struct ieee80211_vht_mcs_info - VHT MCS information
- * @rx_mcs_map: RX MCS map 2 bits for each stream, total 8 streams
- * @rx_highest: Indicates highest long GI VHT PPDU data rate
- *	STA can receive. Rate expressed in units of 1 Mbps.
- *	If this field is 0 this value should not be used to
- *	consider the highest RX data rate supported.
- *	The top 3 bits of this field indicate the Maximum NSTS,total
- *	(a beamformee capability.)
- * @tx_mcs_map: TX MCS map 2 bits for each stream, total 8 streams
- * @tx_highest: Indicates highest long GI VHT PPDU data rate
- *	STA can transmit. Rate expressed in units of 1 Mbps.
- *	If this field is 0 this value should not be used to
- *	consider the highest TX data rate supported.
- *	The top 2 bits of this field are reserved, the
- *	3rd bit from the top indiciates VHT Extended NSS BW
- *	Capability.
- */
-struct ieee80211_vht_mcs_info {
-	__le16 rx_mcs_map;
-	__le16 rx_highest;
-	__le16 tx_mcs_map;
-	__le16 tx_highest;
-} __packed;
-
-/* for rx_highest */
-#define IEEE80211_VHT_MAX_NSTS_TOTAL_SHIFT	13
-#define IEEE80211_VHT_MAX_NSTS_TOTAL_MASK	(7 << IEEE80211_VHT_MAX_NSTS_TOTAL_SHIFT)
-
-/* for tx_highest */
-#define IEEE80211_VHT_EXT_NSS_BW_CAPABLE	(1 << 13)
-
-/**
- * enum ieee80211_vht_mcs_support - VHT MCS support definitions
- * @IEEE80211_VHT_MCS_SUPPORT_0_7: MCSes 0-7 are supported for the
- *	number of streams
- * @IEEE80211_VHT_MCS_SUPPORT_0_8: MCSes 0-8 are supported
- * @IEEE80211_VHT_MCS_SUPPORT_0_9: MCSes 0-9 are supported
- * @IEEE80211_VHT_MCS_NOT_SUPPORTED: This number of streams isn't supported
- *
- * These definitions are used in each 2-bit subfield of the @rx_mcs_map
- * and @tx_mcs_map fields of &struct ieee80211_vht_mcs_info, which are
- * both split into 8 subfields by number of streams. These values indicate
- * which MCSes are supported for the number of streams the value appears
- * for.
- */
-enum ieee80211_vht_mcs_support {
-	IEEE80211_VHT_MCS_SUPPORT_0_7	= 0,
-	IEEE80211_VHT_MCS_SUPPORT_0_8	= 1,
-	IEEE80211_VHT_MCS_SUPPORT_0_9	= 2,
-	IEEE80211_VHT_MCS_NOT_SUPPORTED	= 3,
-};
-
-/**
- * struct ieee80211_vht_cap - VHT capabilities
- *
- * This structure is the "VHT capabilities element" as
- * described in 802.11ac D3.0 8.4.2.160
- * @vht_cap_info: VHT capability info
- * @supp_mcs: VHT MCS supported rates
- */
-struct ieee80211_vht_cap {
-	__le32 vht_cap_info;
-	struct ieee80211_vht_mcs_info supp_mcs;
-} __packed;
-
-/**
- * enum ieee80211_vht_chanwidth - VHT channel width
- * @IEEE80211_VHT_CHANWIDTH_USE_HT: use the HT operation IE to
- *	determine the channel width (20 or 40 MHz)
- * @IEEE80211_VHT_CHANWIDTH_80MHZ: 80 MHz bandwidth
- * @IEEE80211_VHT_CHANWIDTH_160MHZ: 160 MHz bandwidth
- * @IEEE80211_VHT_CHANWIDTH_80P80MHZ: 80+80 MHz bandwidth
- */
-enum ieee80211_vht_chanwidth {
-	IEEE80211_VHT_CHANWIDTH_USE_HT		= 0,
-	IEEE80211_VHT_CHANWIDTH_80MHZ		= 1,
-	IEEE80211_VHT_CHANWIDTH_160MHZ		= 2,
-	IEEE80211_VHT_CHANWIDTH_80P80MHZ	= 3,
-};
-
-/**
- * struct ieee80211_vht_operation - VHT operation IE
- *
- * This structure is the "VHT operation element" as
- * described in 802.11ac D3.0 8.4.2.161
- * @chan_width: Operating channel width
- * @center_freq_seg0_idx: center freq segment 0 index
- * @center_freq_seg1_idx: center freq segment 1 index
- * @basic_mcs_set: VHT Basic MCS rate set
- */
-struct ieee80211_vht_operation {
-	u8 chan_width;
-	u8 center_freq_seg0_idx;
-	u8 center_freq_seg1_idx;
-	__le16 basic_mcs_set;
-} __packed;
-
-/**
- * struct ieee80211_he_cap_elem - HE capabilities element
- *
- * This structure is the "HE capabilities element" fixed fields as
- * described in P802.11ax_D4.0 section 9.4.2.242.2 and 9.4.2.242.3
- */
-struct ieee80211_he_cap_elem {
-	u8 mac_cap_info[6];
-	u8 phy_cap_info[11];
-} __packed;
-
-#define IEEE80211_TX_RX_MCS_NSS_DESC_MAX_LEN	5
-
-/**
- * enum ieee80211_he_mcs_support - HE MCS support definitions
- * @IEEE80211_HE_MCS_SUPPORT_0_7: MCSes 0-7 are supported for the
- *	number of streams
- * @IEEE80211_HE_MCS_SUPPORT_0_9: MCSes 0-9 are supported
- * @IEEE80211_HE_MCS_SUPPORT_0_11: MCSes 0-11 are supported
- * @IEEE80211_HE_MCS_NOT_SUPPORTED: This number of streams isn't supported
- *
- * These definitions are used in each 2-bit subfield of the rx_mcs_*
- * and tx_mcs_* fields of &struct ieee80211_he_mcs_nss_supp, which are
- * both split into 8 subfields by number of streams. These values indicate
- * which MCSes are supported for the number of streams the value appears
- * for.
- */
-enum ieee80211_he_mcs_support {
-	IEEE80211_HE_MCS_SUPPORT_0_7	= 0,
-	IEEE80211_HE_MCS_SUPPORT_0_9	= 1,
-	IEEE80211_HE_MCS_SUPPORT_0_11	= 2,
-	IEEE80211_HE_MCS_NOT_SUPPORTED	= 3,
-};
-
-/**
- * struct ieee80211_he_mcs_nss_supp - HE Tx/Rx HE MCS NSS Support Field
- *
- * This structure holds the data required for the Tx/Rx HE MCS NSS Support Field
- * described in P802.11ax_D2.0 section 9.4.2.237.4
- *
- * @rx_mcs_80: Rx MCS map 2 bits for each stream, total 8 streams, for channel
- *     widths less than 80MHz.
- * @tx_mcs_80: Tx MCS map 2 bits for each stream, total 8 streams, for channel
- *     widths less than 80MHz.
- * @rx_mcs_160: Rx MCS map 2 bits for each stream, total 8 streams, for channel
- *     width 160MHz.
- * @tx_mcs_160: Tx MCS map 2 bits for each stream, total 8 streams, for channel
- *     width 160MHz.
- * @rx_mcs_80p80: Rx MCS map 2 bits for each stream, total 8 streams, for
- *     channel width 80p80MHz.
- * @tx_mcs_80p80: Tx MCS map 2 bits for each stream, total 8 streams, for
- *     channel width 80p80MHz.
- */
-struct ieee80211_he_mcs_nss_supp {
-	__le16 rx_mcs_80;
-	__le16 tx_mcs_80;
-	__le16 rx_mcs_160;
-	__le16 tx_mcs_160;
-	__le16 rx_mcs_80p80;
-	__le16 tx_mcs_80p80;
-} __packed;
-
-/**
- * struct ieee80211_he_operation - HE capabilities element
- *
- * This structure is the "HE operation element" fields as
- * described in P802.11ax_D4.0 section 9.4.2.243
- */
-struct ieee80211_he_operation {
-	__le32 he_oper_params;
-	__le16 he_mcs_nss_set;
-	/* Optional 0,1,3,4,5,7 or 8 bytes: depends on @he_oper_params */
-	u8 optional[];
-} __packed;
-
-/**
- * struct ieee80211_he_spr - HE spatial reuse element
- *
- * This structure is the "HE spatial reuse element" element as
- * described in P802.11ax_D4.0 section 9.4.2.241
- */
-struct ieee80211_he_spr {
-	u8 he_sr_control;
-	/* Optional 0 to 19 bytes: depends on @he_sr_control */
-	u8 optional[];
-} __packed;
-
-/**
- * struct ieee80211_he_mu_edca_param_ac_rec - MU AC Parameter Record field
- *
- * This structure is the "MU AC Parameter Record" fields as
- * described in P802.11ax_D4.0 section 9.4.2.245
- */
-struct ieee80211_he_mu_edca_param_ac_rec {
-	u8 aifsn;
-	u8 ecw_min_max;
-	u8 mu_edca_timer;
-} __packed;
-
-/**
- * struct ieee80211_mu_edca_param_set - MU EDCA Parameter Set element
- *
- * This structure is the "MU EDCA Parameter Set element" fields as
- * described in P802.11ax_D4.0 section 9.4.2.245
- */
-struct ieee80211_mu_edca_param_set {
-	u8 mu_qos_info;
-	struct ieee80211_he_mu_edca_param_ac_rec ac_be;
-	struct ieee80211_he_mu_edca_param_ac_rec ac_bk;
-	struct ieee80211_he_mu_edca_param_ac_rec ac_vi;
-	struct ieee80211_he_mu_edca_param_ac_rec ac_vo;
-} __packed;
-
-#define IEEE80211_EHT_MCS_NSS_RX 0x0f
-#define IEEE80211_EHT_MCS_NSS_TX 0xf0
-
-/**
- * struct ieee80211_eht_mcs_nss_supp_20mhz_only - EHT 20MHz only station max
- * supported NSS for per MCS.
- *
- * For each field below, bits 0 - 3 indicate the maximal number of spatial
- * streams for Rx, and bits 4 - 7 indicate the maximal number of spatial streams
- * for Tx.
- *
- * @rx_tx_mcs7_max_nss: indicates the maximum number of spatial streams
- *     supported for reception and the maximum number of spatial streams
- *     supported for transmission for MCS 0 - 7.
- * @rx_tx_mcs9_max_nss: indicates the maximum number of spatial streams
- *     supported for reception and the maximum number of spatial streams
- *     supported for transmission for MCS 8 - 9.
- * @rx_tx_mcs11_max_nss: indicates the maximum number of spatial streams
- *     supported for reception and the maximum number of spatial streams
- *     supported for transmission for MCS 10 - 11.
- * @rx_tx_mcs13_max_nss: indicates the maximum number of spatial streams
- *     supported for reception and the maximum number of spatial streams
- *     supported for transmission for MCS 12 - 13.
- */
-struct ieee80211_eht_mcs_nss_supp_20mhz_only {
-	u8 rx_tx_mcs7_max_nss;
-	u8 rx_tx_mcs9_max_nss;
-	u8 rx_tx_mcs11_max_nss;
-	u8 rx_tx_mcs13_max_nss;
-};
-
-/**
- * struct ieee80211_eht_mcs_nss_supp_bw - EHT max supported NSS per MCS (except
- * 20MHz only stations).
- *
- * For each field below, bits 0 - 3 indicate the maximal number of spatial
- * streams for Rx, and bits 4 - 7 indicate the maximal number of spatial streams
- * for Tx.
- *
- * @rx_tx_mcs9_max_nss: indicates the maximum number of spatial streams
- *     supported for reception and the maximum number of spatial streams
- *     supported for transmission for MCS 0 - 9.
- * @rx_tx_mcs11_max_nss: indicates the maximum number of spatial streams
- *     supported for reception and the maximum number of spatial streams
- *     supported for transmission for MCS 10 - 11.
- * @rx_tx_mcs13_max_nss: indicates the maximum number of spatial streams
- *     supported for reception and the maximum number of spatial streams
- *     supported for transmission for MCS 12 - 13.
- */
-struct ieee80211_eht_mcs_nss_supp_bw {
-	u8 rx_tx_mcs9_max_nss;
-	u8 rx_tx_mcs11_max_nss;
-	u8 rx_tx_mcs13_max_nss;
-};
-
-/**
- * struct ieee80211_eht_cap_elem_fixed - EHT capabilities fixed data
- *
- * This structure is the "EHT Capabilities element" fixed fields as
- * described in P802.11be_D2.0 section 9.4.2.313.
- *
- * @mac_cap_info: MAC capabilities, see IEEE80211_EHT_MAC_CAP*
- * @phy_cap_info: PHY capabilities, see IEEE80211_EHT_PHY_CAP*
- */
-struct ieee80211_eht_cap_elem_fixed {
-	u8 mac_cap_info[2];
-	u8 phy_cap_info[9];
-} __packed;
-
-/**
- * struct ieee80211_eht_cap_elem - EHT capabilities element
- * @fixed: fixed parts, see &ieee80211_eht_cap_elem_fixed
- * @optional: optional parts
- */
-struct ieee80211_eht_cap_elem {
-	struct ieee80211_eht_cap_elem_fixed fixed;
-
-	/*
-	 * Followed by:
-	 * Supported EHT-MCS And NSS Set field: 4, 3, 6 or 9 octets.
-	 * EHT PPE Thresholds field: variable length.
-	 */
-	u8 optional[];
-} __packed;
-
-#define IEEE80211_EHT_OPER_INFO_PRESENT	                        0x01
-#define IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT	0x02
-#define IEEE80211_EHT_OPER_EHT_DEF_PE_DURATION	                0x04
-#define IEEE80211_EHT_OPER_GROUP_ADDRESSED_BU_IND_LIMIT         0x08
-#define IEEE80211_EHT_OPER_GROUP_ADDRESSED_BU_IND_EXP_MASK      0x30
-
-/**
- * struct ieee80211_eht_operation - eht operation element
- *
- * This structure is the "EHT Operation Element" fields as
- * described in P802.11be_D2.0 section 9.4.2.311
- *
- * @params: EHT operation element parameters. See &IEEE80211_EHT_OPER_*
- * @basic_mcs_nss: indicates the EHT-MCSs for each number of spatial streams in
- *     EHT PPDUs that are supported by all EHT STAs in the BSS in transmit and
- *     receive.
- * @optional: optional parts
- */
-struct ieee80211_eht_operation {
-	u8 params;
-	__le32 basic_mcs_nss;
-	u8 optional[];
-} __packed;
-
-/**
- * struct ieee80211_eht_operation_info - eht operation information
- *
- * @control: EHT operation information control.
- * @ccfs0: defines a channel center frequency for a 20, 40, 80, 160, or 320 MHz
- *     EHT BSS.
- * @ccfs1: defines a channel center frequency for a 160 or 320 MHz EHT BSS.
- * @optional: optional parts
- */
-struct ieee80211_eht_operation_info {
-	u8 control;
-	u8 ccfs0;
-	u8 ccfs1;
-	u8 optional[];
-} __packed;
-
-/* 802.11ac VHT Capabilities */
-#define IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895			0x00000000
-#define IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_7991			0x00000001
-#define IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454			0x00000002
-#define IEEE80211_VHT_CAP_MAX_MPDU_MASK				0x00000003
-#define IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ		0x00000004
-#define IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ	0x00000008
-#define IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK			0x0000000C
-#define IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_SHIFT			2
-#define IEEE80211_VHT_CAP_RXLDPC				0x00000010
-#define IEEE80211_VHT_CAP_SHORT_GI_80				0x00000020
-#define IEEE80211_VHT_CAP_SHORT_GI_160				0x00000040
-#define IEEE80211_VHT_CAP_TXSTBC				0x00000080
-#define IEEE80211_VHT_CAP_RXSTBC_1				0x00000100
-#define IEEE80211_VHT_CAP_RXSTBC_2				0x00000200
-#define IEEE80211_VHT_CAP_RXSTBC_3				0x00000300
-#define IEEE80211_VHT_CAP_RXSTBC_4				0x00000400
-#define IEEE80211_VHT_CAP_RXSTBC_MASK				0x00000700
-#define IEEE80211_VHT_CAP_RXSTBC_SHIFT				8
-#define IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE			0x00000800
-#define IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE			0x00001000
-#define IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT                  13
-#define IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK			\
-		(7 << IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT)
-#define IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT		16
-#define IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK		\
-		(7 << IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT)
-#define IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE			0x00080000
-#define IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE			0x00100000
-#define IEEE80211_VHT_CAP_VHT_TXOP_PS				0x00200000
-#define IEEE80211_VHT_CAP_HTC_VHT				0x00400000
-#define IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT	23
-#define IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK	\
-		(7 << IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT)
-#define IEEE80211_VHT_CAP_VHT_LINK_ADAPTATION_VHT_UNSOL_MFB	0x08000000
-#define IEEE80211_VHT_CAP_VHT_LINK_ADAPTATION_VHT_MRQ_MFB	0x0c000000
-#define IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN			0x10000000
-#define IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN			0x20000000
-#define IEEE80211_VHT_CAP_EXT_NSS_BW_SHIFT			30
-#define IEEE80211_VHT_CAP_EXT_NSS_BW_MASK			0xc0000000
-
-/**
- * ieee80211_get_vht_max_nss - return max NSS for a given bandwidth/MCS
- * @cap: VHT capabilities of the peer
- * @bw: bandwidth to use
- * @mcs: MCS index to use
- * @ext_nss_bw_capable: indicates whether or not the local transmitter
- *	(rate scaling algorithm) can deal with the new logic
- *	(dot11VHTExtendedNSSBWCapable)
- * @max_vht_nss: current maximum NSS as advertised by the STA in
- *	operating mode notification, can be 0 in which case the
- *	capability data will be used to derive this (from MCS support)
- *
- * Due to the VHT Extended NSS Bandwidth Support, the maximum NSS can
- * vary for a given BW/MCS. This function parses the data.
- *
- * Note: This function is exported by cfg80211.
- */
-int ieee80211_get_vht_max_nss(struct ieee80211_vht_cap *cap,
-			      enum ieee80211_vht_chanwidth bw,
-			      int mcs, bool ext_nss_bw_capable,
-			      unsigned int max_vht_nss);
-
-/**
- * enum ieee80211_ap_reg_power - regulatory power for a Access Point
- *
- * @IEEE80211_REG_UNSET_AP: Access Point has no regulatory power mode
- * @IEEE80211_REG_LPI: Indoor Access Point
- * @IEEE80211_REG_SP: Standard power Access Point
- * @IEEE80211_REG_VLP: Very low power Access Point
- * @IEEE80211_REG_AP_POWER_AFTER_LAST: internal
- * @IEEE80211_REG_AP_POWER_MAX: maximum value
- */
-enum ieee80211_ap_reg_power {
-	IEEE80211_REG_UNSET_AP,
-	IEEE80211_REG_LPI_AP,
-	IEEE80211_REG_SP_AP,
-	IEEE80211_REG_VLP_AP,
-	IEEE80211_REG_AP_POWER_AFTER_LAST,
-	IEEE80211_REG_AP_POWER_MAX =
-		IEEE80211_REG_AP_POWER_AFTER_LAST - 1,
-};
-
-/**
- * enum ieee80211_client_reg_power - regulatory power for a client
- *
- * @IEEE80211_REG_UNSET_CLIENT: Client has no regulatory power mode
- * @IEEE80211_REG_DEFAULT_CLIENT: Default Client
- * @IEEE80211_REG_SUBORDINATE_CLIENT: Subordinate Client
- * @IEEE80211_REG_CLIENT_POWER_AFTER_LAST: internal
- * @IEEE80211_REG_CLIENT_POWER_MAX: maximum value
- */
-enum ieee80211_client_reg_power {
-	IEEE80211_REG_UNSET_CLIENT,
-	IEEE80211_REG_DEFAULT_CLIENT,
-	IEEE80211_REG_SUBORDINATE_CLIENT,
-	IEEE80211_REG_CLIENT_POWER_AFTER_LAST,
-	IEEE80211_REG_CLIENT_POWER_MAX =
-		IEEE80211_REG_CLIENT_POWER_AFTER_LAST - 1,
-};
-
-/* 802.11ax HE MAC capabilities */
-#define IEEE80211_HE_MAC_CAP0_HTC_HE				0x01
-#define IEEE80211_HE_MAC_CAP0_TWT_REQ				0x02
-#define IEEE80211_HE_MAC_CAP0_TWT_RES				0x04
-#define IEEE80211_HE_MAC_CAP0_DYNAMIC_FRAG_NOT_SUPP		0x00
-#define IEEE80211_HE_MAC_CAP0_DYNAMIC_FRAG_LEVEL_1		0x08
-#define IEEE80211_HE_MAC_CAP0_DYNAMIC_FRAG_LEVEL_2		0x10
-#define IEEE80211_HE_MAC_CAP0_DYNAMIC_FRAG_LEVEL_3		0x18
-#define IEEE80211_HE_MAC_CAP0_DYNAMIC_FRAG_MASK			0x18
-#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_1		0x00
-#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_2		0x20
-#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_4		0x40
-#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_8		0x60
-#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_16		0x80
-#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_32		0xa0
-#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_64		0xc0
-#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_UNLIMITED	0xe0
-#define IEEE80211_HE_MAC_CAP0_MAX_NUM_FRAG_MSDU_MASK		0xe0
-
-#define IEEE80211_HE_MAC_CAP1_MIN_FRAG_SIZE_UNLIMITED		0x00
-#define IEEE80211_HE_MAC_CAP1_MIN_FRAG_SIZE_128			0x01
-#define IEEE80211_HE_MAC_CAP1_MIN_FRAG_SIZE_256			0x02
-#define IEEE80211_HE_MAC_CAP1_MIN_FRAG_SIZE_512			0x03
-#define IEEE80211_HE_MAC_CAP1_MIN_FRAG_SIZE_MASK		0x03
-#define IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_0US		0x00
-#define IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_8US		0x04
-#define IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US		0x08
-#define IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_MASK		0x0c
-#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_1		0x00
-#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_2		0x10
-#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_3		0x20
-#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_4		0x30
-#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_5		0x40
-#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_6		0x50
-#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_7		0x60
-#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8		0x70
-#define IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_MASK		0x70
-
-/* Link adaptation is split between byte HE_MAC_CAP1 and
- * HE_MAC_CAP2. It should be set only if IEEE80211_HE_MAC_CAP0_HTC_HE
- * in which case the following values apply:
- * 0 = No feedback.
- * 1 = reserved.
- * 2 = Unsolicited feedback.
- * 3 = both
- */
-#define IEEE80211_HE_MAC_CAP1_LINK_ADAPTATION			0x80
-
-#define IEEE80211_HE_MAC_CAP2_LINK_ADAPTATION			0x01
-#define IEEE80211_HE_MAC_CAP2_ALL_ACK				0x02
-#define IEEE80211_HE_MAC_CAP2_TRS				0x04
-#define IEEE80211_HE_MAC_CAP2_BSR				0x08
-#define IEEE80211_HE_MAC_CAP2_BCAST_TWT				0x10
-#define IEEE80211_HE_MAC_CAP2_32BIT_BA_BITMAP			0x20
-#define IEEE80211_HE_MAC_CAP2_MU_CASCADING			0x40
-#define IEEE80211_HE_MAC_CAP2_ACK_EN				0x80
-
-#define IEEE80211_HE_MAC_CAP3_OMI_CONTROL			0x02
-#define IEEE80211_HE_MAC_CAP3_OFDMA_RA				0x04
-
-/* The maximum length of an A-MDPU is defined by the combination of the Maximum
- * A-MDPU Length Exponent field in the HT capabilities, VHT capabilities and the
- * same field in the HE capabilities.
- */
-#define IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_0		0x00
-#define IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_1		0x08
-#define IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_2		0x10
-#define IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3		0x18
-#define IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_MASK		0x18
-#define IEEE80211_HE_MAC_CAP3_AMSDU_FRAG			0x20
-#define IEEE80211_HE_MAC_CAP3_FLEX_TWT_SCHED			0x40
-#define IEEE80211_HE_MAC_CAP3_RX_CTRL_FRAME_TO_MULTIBSS		0x80
-
-#define IEEE80211_HE_MAC_CAP4_BSRP_BQRP_A_MPDU_AGG		0x01
-#define IEEE80211_HE_MAC_CAP4_QTP				0x02
-#define IEEE80211_HE_MAC_CAP4_BQR				0x04
-#define IEEE80211_HE_MAC_CAP4_PSR_RESP				0x08
-#define IEEE80211_HE_MAC_CAP4_NDP_FB_REP			0x10
-#define IEEE80211_HE_MAC_CAP4_OPS				0x20
-#define IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU			0x40
-/* Multi TID agg TX is split between byte #4 and #5
- * The value is a combination of B39,B40,B41
- */
-#define IEEE80211_HE_MAC_CAP4_MULTI_TID_AGG_TX_QOS_B39		0x80
-
-#define IEEE80211_HE_MAC_CAP5_MULTI_TID_AGG_TX_QOS_B40		0x01
-#define IEEE80211_HE_MAC_CAP5_MULTI_TID_AGG_TX_QOS_B41		0x02
-#define IEEE80211_HE_MAC_CAP5_SUBCHAN_SELECTIVE_TRANSMISSION	0x04
-#define IEEE80211_HE_MAC_CAP5_UL_2x996_TONE_RU			0x08
-#define IEEE80211_HE_MAC_CAP5_OM_CTRL_UL_MU_DATA_DIS_RX		0x10
-#define IEEE80211_HE_MAC_CAP5_HE_DYNAMIC_SM_PS			0x20
-#define IEEE80211_HE_MAC_CAP5_PUNCTURED_SOUNDING		0x40
-#define IEEE80211_HE_MAC_CAP5_HT_VHT_TRIG_FRAME_RX		0x80
-
-#define IEEE80211_HE_VHT_MAX_AMPDU_FACTOR	20
-#define IEEE80211_HE_HT_MAX_AMPDU_FACTOR	16
-#define IEEE80211_HE_6GHZ_MAX_AMPDU_FACTOR	13
-
-/* 802.11ax HE PHY capabilities */
-#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G		0x02
-#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G	0x04
-#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G		0x08
-#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G	0x10
-#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL		0x1e
-
-#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_RU_MAPPING_IN_2G	0x20
-#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_RU_MAPPING_IN_5G	0x40
-#define IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK			0xfe
-
-#define IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_80MHZ_ONLY_SECOND_20MHZ	0x01
-#define IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_80MHZ_ONLY_SECOND_40MHZ	0x02
-#define IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_160MHZ_ONLY_SECOND_20MHZ	0x04
-#define IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_160MHZ_ONLY_SECOND_40MHZ	0x08
-#define IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK			0x0f
-#define IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A				0x10
-#define IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD			0x20
-#define IEEE80211_HE_PHY_CAP1_HE_LTF_AND_GI_FOR_HE_PPDUS_0_8US		0x40
-/* Midamble RX/TX Max NSTS is split between byte #2 and byte #3 */
-#define IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS			0x80
-
-#define IEEE80211_HE_PHY_CAP2_MIDAMBLE_RX_TX_MAX_NSTS			0x01
-#define IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US			0x02
-#define IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ			0x04
-#define IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ			0x08
-#define IEEE80211_HE_PHY_CAP2_DOPPLER_TX				0x10
-#define IEEE80211_HE_PHY_CAP2_DOPPLER_RX				0x20
-
-/* Note that the meaning of UL MU below is different between an AP and a non-AP
- * sta, where in the AP case it indicates support for Rx and in the non-AP sta
- * case it indicates support for Tx.
- */
-#define IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO			0x40
-#define IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO			0x80
-
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_NO_DCM			0x00
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_BPSK			0x01
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_QPSK			0x02
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_16_QAM			0x03
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_MASK			0x03
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_TX_NSS_1				0x00
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_TX_NSS_2				0x04
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_NO_DCM			0x00
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_BPSK			0x08
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_QPSK			0x10
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_16_QAM			0x18
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_MASK			0x18
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_RX_NSS_1				0x00
-#define IEEE80211_HE_PHY_CAP3_DCM_MAX_RX_NSS_2				0x20
-#define IEEE80211_HE_PHY_CAP3_RX_PARTIAL_BW_SU_IN_20MHZ_MU		0x40
-#define IEEE80211_HE_PHY_CAP3_SU_BEAMFORMER				0x80
-
-#define IEEE80211_HE_PHY_CAP4_SU_BEAMFORMEE				0x01
-#define IEEE80211_HE_PHY_CAP4_MU_BEAMFORMER				0x02
-
-/* Minimal allowed value of Max STS under 80MHz is 3 */
-#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_4		0x0c
-#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_5		0x10
-#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_6		0x14
-#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_7		0x18
-#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_8		0x1c
-#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_MASK	0x1c
-
-/* Minimal allowed value of Max STS above 80MHz is 3 */
-#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4		0x60
-#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_5		0x80
-#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_6		0xa0
-#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_7		0xc0
-#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_8		0xe0
-#define IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_MASK	0xe0
-
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_1	0x00
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_2	0x01
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_3	0x02
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_4	0x03
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_5	0x04
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_6	0x05
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_7	0x06
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_8	0x07
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_MASK	0x07
-
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_1	0x00
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_2	0x08
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_3	0x10
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_4	0x18
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_5	0x20
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_6	0x28
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_7	0x30
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_8	0x38
-#define IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_MASK	0x38
-
-#define IEEE80211_HE_PHY_CAP5_NG16_SU_FEEDBACK				0x40
-#define IEEE80211_HE_PHY_CAP5_NG16_MU_FEEDBACK				0x80
-
-#define IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_42_SU			0x01
-#define IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_75_MU			0x02
-#define IEEE80211_HE_PHY_CAP6_TRIG_SU_BEAMFORMING_FB			0x04
-#define IEEE80211_HE_PHY_CAP6_TRIG_MU_BEAMFORMING_PARTIAL_BW_FB		0x08
-#define IEEE80211_HE_PHY_CAP6_TRIG_CQI_FB				0x10
-#define IEEE80211_HE_PHY_CAP6_PARTIAL_BW_EXT_RANGE			0x20
-#define IEEE80211_HE_PHY_CAP6_PARTIAL_BANDWIDTH_DL_MUMIMO		0x40
-#define IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT			0x80
-
-#define IEEE80211_HE_PHY_CAP7_PSR_BASED_SR				0x01
-#define IEEE80211_HE_PHY_CAP7_POWER_BOOST_FACTOR_SUPP			0x02
-#define IEEE80211_HE_PHY_CAP7_HE_SU_MU_PPDU_4XLTF_AND_08_US_GI		0x04
-#define IEEE80211_HE_PHY_CAP7_MAX_NC_1					0x08
-#define IEEE80211_HE_PHY_CAP7_MAX_NC_2					0x10
-#define IEEE80211_HE_PHY_CAP7_MAX_NC_3					0x18
-#define IEEE80211_HE_PHY_CAP7_MAX_NC_4					0x20
-#define IEEE80211_HE_PHY_CAP7_MAX_NC_5					0x28
-#define IEEE80211_HE_PHY_CAP7_MAX_NC_6					0x30
-#define IEEE80211_HE_PHY_CAP7_MAX_NC_7					0x38
-#define IEEE80211_HE_PHY_CAP7_MAX_NC_MASK				0x38
-#define IEEE80211_HE_PHY_CAP7_STBC_TX_ABOVE_80MHZ			0x40
-#define IEEE80211_HE_PHY_CAP7_STBC_RX_ABOVE_80MHZ			0x80
-
-#define IEEE80211_HE_PHY_CAP8_HE_ER_SU_PPDU_4XLTF_AND_08_US_GI		0x01
-#define IEEE80211_HE_PHY_CAP8_20MHZ_IN_40MHZ_HE_PPDU_IN_2G		0x02
-#define IEEE80211_HE_PHY_CAP8_20MHZ_IN_160MHZ_HE_PPDU			0x04
-#define IEEE80211_HE_PHY_CAP8_80MHZ_IN_160MHZ_HE_PPDU			0x08
-#define IEEE80211_HE_PHY_CAP8_HE_ER_SU_1XLTF_AND_08_US_GI		0x10
-#define IEEE80211_HE_PHY_CAP8_MIDAMBLE_RX_TX_2X_AND_1XLTF		0x20
-#define IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_242				0x00
-#define IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_484				0x40
-#define IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_996				0x80
-#define IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_2x996				0xc0
-#define IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_MASK				0xc0
-
-#define IEEE80211_HE_PHY_CAP9_LONGER_THAN_16_SIGB_OFDM_SYM		0x01
-#define IEEE80211_HE_PHY_CAP9_NON_TRIGGERED_CQI_FEEDBACK		0x02
-#define IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU		0x04
-#define IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU		0x08
-#define IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_COMP_SIGB	0x10
-#define IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_NON_COMP_SIGB	0x20
-#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_0US			0x0
-#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_8US			0x1
-#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US			0x2
-#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED		0x3
-#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_POS			6
-#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK			0xc0
-
-#define IEEE80211_HE_PHY_CAP10_HE_MU_M1RU_MAX_LTF			0x01
-
-/* 802.11ax HE TX/RX MCS NSS Support  */
-#define IEEE80211_TX_RX_MCS_NSS_SUPP_HIGHEST_MCS_POS			(3)
-#define IEEE80211_TX_RX_MCS_NSS_SUPP_TX_BITMAP_POS			(6)
-#define IEEE80211_TX_RX_MCS_NSS_SUPP_RX_BITMAP_POS			(11)
-#define IEEE80211_TX_RX_MCS_NSS_SUPP_TX_BITMAP_MASK			0x07c0
-#define IEEE80211_TX_RX_MCS_NSS_SUPP_RX_BITMAP_MASK			0xf800
-
-/* TX/RX HE MCS Support field Highest MCS subfield encoding */
-enum ieee80211_he_highest_mcs_supported_subfield_enc {
-	HIGHEST_MCS_SUPPORTED_MCS7 = 0,
-	HIGHEST_MCS_SUPPORTED_MCS8,
-	HIGHEST_MCS_SUPPORTED_MCS9,
-	HIGHEST_MCS_SUPPORTED_MCS10,
-	HIGHEST_MCS_SUPPORTED_MCS11,
-};
-
-/* Calculate 802.11ax HE capabilities IE Tx/Rx HE MCS NSS Support Field size */
-static inline u8
-ieee80211_he_mcs_nss_size(const struct ieee80211_he_cap_elem *he_cap)
-{
-	u8 count = 4;
-
-	if (he_cap->phy_cap_info[0] &
-	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G)
-		count += 4;
-
-	if (he_cap->phy_cap_info[0] &
-	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G)
-		count += 4;
-
-	return count;
-}
-
-/* 802.11ax HE PPE Thresholds */
-#define IEEE80211_PPE_THRES_NSS_SUPPORT_2NSS			(1)
-#define IEEE80211_PPE_THRES_NSS_POS				(0)
-#define IEEE80211_PPE_THRES_NSS_MASK				(7)
-#define IEEE80211_PPE_THRES_RU_INDEX_BITMASK_2x966_AND_966_RU	\
-	(BIT(5) | BIT(6))
-#define IEEE80211_PPE_THRES_RU_INDEX_BITMASK_MASK		0x78
-#define IEEE80211_PPE_THRES_RU_INDEX_BITMASK_POS		(3)
-#define IEEE80211_PPE_THRES_INFO_PPET_SIZE			(3)
-#define IEEE80211_HE_PPE_THRES_INFO_HEADER_SIZE			(7)
-
-/*
- * Calculate 802.11ax HE capabilities IE PPE field size
- * Input: Header byte of ppe_thres (first byte), and HE capa IE's PHY cap u8*
- */
-static inline u8
-ieee80211_he_ppe_size(u8 ppe_thres_hdr, const u8 *phy_cap_info)
-{
-	u8 n;
-
-	if ((phy_cap_info[6] &
-	     IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) == 0)
-		return 0;
-
-	n = hweight8(ppe_thres_hdr &
-		     IEEE80211_PPE_THRES_RU_INDEX_BITMASK_MASK);
-	n *= (1 + ((ppe_thres_hdr & IEEE80211_PPE_THRES_NSS_MASK) >>
-		   IEEE80211_PPE_THRES_NSS_POS));
-
-	/*
-	 * Each pair is 6 bits, and we need to add the 7 "header" bits to the
-	 * total size.
-	 */
-	n = (n * IEEE80211_PPE_THRES_INFO_PPET_SIZE * 2) + 7;
-	n = DIV_ROUND_UP(n, 8);
-
-	return n;
-}
-
-static inline bool ieee80211_he_capa_size_ok(const u8 *data, u8 len)
-{
-	const struct ieee80211_he_cap_elem *he_cap_ie_elem = (const void *)data;
-	u8 needed = sizeof(*he_cap_ie_elem);
-
-	if (len < needed)
-		return false;
-
-	needed += ieee80211_he_mcs_nss_size(he_cap_ie_elem);
-	if (len < needed)
-		return false;
-
-	if (he_cap_ie_elem->phy_cap_info[6] &
-			IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) {
-		if (len < needed + 1)
-			return false;
-		needed += ieee80211_he_ppe_size(data[needed],
-						he_cap_ie_elem->phy_cap_info);
-	}
-
-	return len >= needed;
-}
-
-/* HE Operation defines */
-#define IEEE80211_HE_OPERATION_DFLT_PE_DURATION_MASK		0x00000007
-#define IEEE80211_HE_OPERATION_TWT_REQUIRED			0x00000008
-#define IEEE80211_HE_OPERATION_RTS_THRESHOLD_MASK		0x00003ff0
-#define IEEE80211_HE_OPERATION_RTS_THRESHOLD_OFFSET		4
-#define IEEE80211_HE_OPERATION_VHT_OPER_INFO			0x00004000
-#define IEEE80211_HE_OPERATION_CO_HOSTED_BSS			0x00008000
-#define IEEE80211_HE_OPERATION_ER_SU_DISABLE			0x00010000
-#define IEEE80211_HE_OPERATION_6GHZ_OP_INFO			0x00020000
-#define IEEE80211_HE_OPERATION_BSS_COLOR_MASK			0x3f000000
-#define IEEE80211_HE_OPERATION_BSS_COLOR_OFFSET			24
-#define IEEE80211_HE_OPERATION_PARTIAL_BSS_COLOR		0x40000000
-#define IEEE80211_HE_OPERATION_BSS_COLOR_DISABLED		0x80000000
-
-#define IEEE80211_6GHZ_CTRL_REG_LPI_AP	0
-#define IEEE80211_6GHZ_CTRL_REG_SP_AP	1
-
-/**
- * ieee80211_he_6ghz_oper - HE 6 GHz operation Information field
- * @primary: primary channel
- * @control: control flags
- * @ccfs0: channel center frequency segment 0
- * @ccfs1: channel center frequency segment 1
- * @minrate: minimum rate (in 1 Mbps units)
- */
-struct ieee80211_he_6ghz_oper {
-	u8 primary;
-#define IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH	0x3
-#define		IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_20MHZ	0
-#define		IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_40MHZ	1
-#define		IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_80MHZ	2
-#define		IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_160MHZ	3
-#define IEEE80211_HE_6GHZ_OPER_CTRL_DUP_BEACON	0x4
-#define IEEE80211_HE_6GHZ_OPER_CTRL_REG_INFO	0x38
-	u8 control;
-	u8 ccfs0;
-	u8 ccfs1;
-	u8 minrate;
-} __packed;
-
-/*
- * In "9.4.2.161 Transmit Power Envelope element" of "IEEE Std 802.11ax-2021",
- * it show four types in "Table 9-275a-Maximum Transmit Power Interpretation
- * subfield encoding", and two category for each type in "Table E-12-Regulatory
- * Info subfield encoding in the United States".
- * So it it totally max 8 Transmit Power Envelope element.
- */
-#define IEEE80211_TPE_MAX_IE_COUNT	8
-/*
- * In "Table 9-277—Meaning of Maximum Transmit Power Count subfield"
- * of "IEEE Std 802.11ax™‐2021", the max power level is 8.
- */
-#define IEEE80211_MAX_NUM_PWR_LEVEL	8
-
-#define IEEE80211_TPE_MAX_POWER_COUNT	8
-
-/* transmit power interpretation type of transmit power envelope element */
-enum ieee80211_tx_power_intrpt_type {
-	IEEE80211_TPE_LOCAL_EIRP,
-	IEEE80211_TPE_LOCAL_EIRP_PSD,
-	IEEE80211_TPE_REG_CLIENT_EIRP,
-	IEEE80211_TPE_REG_CLIENT_EIRP_PSD,
-};
-
-/**
- * struct ieee80211_tx_pwr_env
- *
- * This structure represents the "Transmit Power Envelope element"
- */
-struct ieee80211_tx_pwr_env {
-	u8 tx_power_info;
-	s8 tx_power[IEEE80211_TPE_MAX_POWER_COUNT];
-} __packed;
-
-#define IEEE80211_TX_PWR_ENV_INFO_COUNT 0x7
-#define IEEE80211_TX_PWR_ENV_INFO_INTERPRET 0x38
-#define IEEE80211_TX_PWR_ENV_INFO_CATEGORY 0xC0
-
-/*
- * ieee80211_he_oper_size - calculate 802.11ax HE Operations IE size
- * @he_oper_ie: byte data of the He Operations IE, stating from the byte
- *	after the ext ID byte. It is assumed that he_oper_ie has at least
- *	sizeof(struct ieee80211_he_operation) bytes, the caller must have
- *	validated this.
- * @return the actual size of the IE data (not including header), or 0 on error
- */
-static inline u8
-ieee80211_he_oper_size(const u8 *he_oper_ie)
-{
-	const struct ieee80211_he_operation *he_oper = (const void *)he_oper_ie;
-	u8 oper_len = sizeof(struct ieee80211_he_operation);
-	u32 he_oper_params;
-
-	/* Make sure the input is not NULL */
-	if (!he_oper_ie)
-		return 0;
-
-	/* Calc required length */
-	he_oper_params = le32_to_cpu(he_oper->he_oper_params);
-	if (he_oper_params & IEEE80211_HE_OPERATION_VHT_OPER_INFO)
-		oper_len += 3;
-	if (he_oper_params & IEEE80211_HE_OPERATION_CO_HOSTED_BSS)
-		oper_len++;
-	if (he_oper_params & IEEE80211_HE_OPERATION_6GHZ_OP_INFO)
-		oper_len += sizeof(struct ieee80211_he_6ghz_oper);
-
-	/* Add the first byte (extension ID) to the total length */
-	oper_len++;
-
-	return oper_len;
-}
-
-/**
- * ieee80211_he_6ghz_oper - obtain 6 GHz operation field
- * @he_oper: HE operation element (must be pre-validated for size)
- *	but may be %NULL
- *
- * Return: a pointer to the 6 GHz operation field, or %NULL
- */
-static inline const struct ieee80211_he_6ghz_oper *
-ieee80211_he_6ghz_oper(const struct ieee80211_he_operation *he_oper)
-{
-	const u8 *ret = (const void *)&he_oper->optional;
-	u32 he_oper_params;
-
-	if (!he_oper)
-		return NULL;
-
-	he_oper_params = le32_to_cpu(he_oper->he_oper_params);
-
-	if (!(he_oper_params & IEEE80211_HE_OPERATION_6GHZ_OP_INFO))
-		return NULL;
-	if (he_oper_params & IEEE80211_HE_OPERATION_VHT_OPER_INFO)
-		ret += 3;
-	if (he_oper_params & IEEE80211_HE_OPERATION_CO_HOSTED_BSS)
-		ret++;
-
-	return (const void *)ret;
-}
-
-/* HE Spatial Reuse defines */
-#define IEEE80211_HE_SPR_PSR_DISALLOWED				BIT(0)
-#define IEEE80211_HE_SPR_NON_SRG_OBSS_PD_SR_DISALLOWED		BIT(1)
-#define IEEE80211_HE_SPR_NON_SRG_OFFSET_PRESENT			BIT(2)
-#define IEEE80211_HE_SPR_SRG_INFORMATION_PRESENT		BIT(3)
-#define IEEE80211_HE_SPR_HESIGA_SR_VAL15_ALLOWED		BIT(4)
-
-/*
- * ieee80211_he_spr_size - calculate 802.11ax HE Spatial Reuse IE size
- * @he_spr_ie: byte data of the He Spatial Reuse IE, stating from the byte
- *	after the ext ID byte. It is assumed that he_spr_ie has at least
- *	sizeof(struct ieee80211_he_spr) bytes, the caller must have validated
- *	this
- * @return the actual size of the IE data (not including header), or 0 on error
- */
-static inline u8
-ieee80211_he_spr_size(const u8 *he_spr_ie)
-{
-	const struct ieee80211_he_spr *he_spr = (const void *)he_spr_ie;
-	u8 spr_len = sizeof(struct ieee80211_he_spr);
-	u8 he_spr_params;
-
-	/* Make sure the input is not NULL */
-	if (!he_spr_ie)
-		return 0;
-
-	/* Calc required length */
-	he_spr_params = he_spr->he_sr_control;
-	if (he_spr_params & IEEE80211_HE_SPR_NON_SRG_OFFSET_PRESENT)
-		spr_len++;
-	if (he_spr_params & IEEE80211_HE_SPR_SRG_INFORMATION_PRESENT)
-		spr_len += 18;
-
-	/* Add the first byte (extension ID) to the total length */
-	spr_len++;
-
-	return spr_len;
-}
-
-/* S1G Capabilities Information field */
-#define IEEE80211_S1G_CAPABILITY_LEN	15
-
-#define S1G_CAP0_S1G_LONG	BIT(0)
-#define S1G_CAP0_SGI_1MHZ	BIT(1)
-#define S1G_CAP0_SGI_2MHZ	BIT(2)
-#define S1G_CAP0_SGI_4MHZ	BIT(3)
-#define S1G_CAP0_SGI_8MHZ	BIT(4)
-#define S1G_CAP0_SGI_16MHZ	BIT(5)
-#define S1G_CAP0_SUPP_CH_WIDTH	GENMASK(7, 6)
-
-#define S1G_SUPP_CH_WIDTH_2	0
-#define S1G_SUPP_CH_WIDTH_4	1
-#define S1G_SUPP_CH_WIDTH_8	2
-#define S1G_SUPP_CH_WIDTH_16	3
-#define S1G_SUPP_CH_WIDTH_MAX(cap) ((1 << FIELD_GET(S1G_CAP0_SUPP_CH_WIDTH, \
-						    cap[0])) << 1)
-
-#define S1G_CAP1_RX_LDPC	BIT(0)
-#define S1G_CAP1_TX_STBC	BIT(1)
-#define S1G_CAP1_RX_STBC	BIT(2)
-#define S1G_CAP1_SU_BFER	BIT(3)
-#define S1G_CAP1_SU_BFEE	BIT(4)
-#define S1G_CAP1_BFEE_STS	GENMASK(7, 5)
-
-#define S1G_CAP2_SOUNDING_DIMENSIONS	GENMASK(2, 0)
-#define S1G_CAP2_MU_BFER		BIT(3)
-#define S1G_CAP2_MU_BFEE		BIT(4)
-#define S1G_CAP2_PLUS_HTC_VHT		BIT(5)
-#define S1G_CAP2_TRAVELING_PILOT	GENMASK(7, 6)
-
-#define S1G_CAP3_RD_RESPONDER		BIT(0)
-#define S1G_CAP3_HT_DELAYED_BA		BIT(1)
-#define S1G_CAP3_MAX_MPDU_LEN		BIT(2)
-#define S1G_CAP3_MAX_AMPDU_LEN_EXP	GENMASK(4, 3)
-#define S1G_CAP3_MIN_MPDU_START		GENMASK(7, 5)
-
-#define S1G_CAP4_UPLINK_SYNC	BIT(0)
-#define S1G_CAP4_DYNAMIC_AID	BIT(1)
-#define S1G_CAP4_BAT		BIT(2)
-#define S1G_CAP4_TIME_ADE	BIT(3)
-#define S1G_CAP4_NON_TIM	BIT(4)
-#define S1G_CAP4_GROUP_AID	BIT(5)
-#define S1G_CAP4_STA_TYPE	GENMASK(7, 6)
-
-#define S1G_CAP5_CENT_AUTH_CONTROL	BIT(0)
-#define S1G_CAP5_DIST_AUTH_CONTROL	BIT(1)
-#define S1G_CAP5_AMSDU			BIT(2)
-#define S1G_CAP5_AMPDU			BIT(3)
-#define S1G_CAP5_ASYMMETRIC_BA		BIT(4)
-#define S1G_CAP5_FLOW_CONTROL		BIT(5)
-#define S1G_CAP5_SECTORIZED_BEAM	GENMASK(7, 6)
-
-#define S1G_CAP6_OBSS_MITIGATION	BIT(0)
-#define S1G_CAP6_FRAGMENT_BA		BIT(1)
-#define S1G_CAP6_NDP_PS_POLL		BIT(2)
-#define S1G_CAP6_RAW_OPERATION		BIT(3)
-#define S1G_CAP6_PAGE_SLICING		BIT(4)
-#define S1G_CAP6_TXOP_SHARING_IMP_ACK	BIT(5)
-#define S1G_CAP6_VHT_LINK_ADAPT		GENMASK(7, 6)
-
-#define S1G_CAP7_TACK_AS_PS_POLL		BIT(0)
-#define S1G_CAP7_DUP_1MHZ			BIT(1)
-#define S1G_CAP7_MCS_NEGOTIATION		BIT(2)
-#define S1G_CAP7_1MHZ_CTL_RESPONSE_PREAMBLE	BIT(3)
-#define S1G_CAP7_NDP_BFING_REPORT_POLL		BIT(4)
-#define S1G_CAP7_UNSOLICITED_DYN_AID		BIT(5)
-#define S1G_CAP7_SECTOR_TRAINING_OPERATION	BIT(6)
-#define S1G_CAP7_TEMP_PS_MODE_SWITCH		BIT(7)
-
-#define S1G_CAP8_TWT_GROUPING	BIT(0)
-#define S1G_CAP8_BDT		BIT(1)
-#define S1G_CAP8_COLOR		GENMASK(4, 2)
-#define S1G_CAP8_TWT_REQUEST	BIT(5)
-#define S1G_CAP8_TWT_RESPOND	BIT(6)
-#define S1G_CAP8_PV1_FRAME	BIT(7)
-
-#define S1G_CAP9_LINK_ADAPT_PER_CONTROL_RESPONSE BIT(0)
-
-#define S1G_OPER_CH_WIDTH_PRIMARY_1MHZ	BIT(0)
-#define S1G_OPER_CH_WIDTH_OPER		GENMASK(4, 1)
-
-/* EHT MAC capabilities as defined in P802.11be_D2.0 section 9.4.2.313.2 */
-#define IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS			0x01
-#define IEEE80211_EHT_MAC_CAP0_OM_CONTROL			0x02
-#define IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1		0x04
-#define IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE2		0x08
-#define IEEE80211_EHT_MAC_CAP0_RESTRICTED_TWT			0x10
-#define IEEE80211_EHT_MAC_CAP0_SCS_TRAFFIC_DESC			0x20
-#define IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_MASK		0xc0
-#define	IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_3895	        0
-#define	IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_7991	        1
-#define	IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_11454	        2
-
-#define IEEE80211_EHT_MAC_CAP1_MAX_AMPDU_LEN_MASK		0x01
-
-/* EHT PHY capabilities as defined in P802.11be_D2.0 section 9.4.2.313.3 */
-#define IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ			0x02
-#define IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ		0x04
-#define IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI		0x08
-#define IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO		0x10
-#define IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER			0x20
-#define IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE			0x40
-
-/* EHT beamformee number of spatial streams <= 80MHz is split */
-#define IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK		0x80
-#define IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK		0x03
-
-#define IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK	0x1c
-#define IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK	0xe0
-
-#define IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK		0x07
-#define IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK		0x38
-
-/* EHT number of sounding dimensions for 320MHz is split */
-#define IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK		0xc0
-#define IEEE80211_EHT_PHY_CAP3_SOUNDING_DIM_320MHZ_MASK		0x01
-#define IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK		0x02
-#define IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK		0x04
-#define IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK		0x08
-#define IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK		0x10
-#define IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK			0x20
-#define IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK		0x40
-#define IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK			0x80
-
-#define IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO		0x01
-#define IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP			0x02
-#define IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP		0x04
-#define IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI	0x08
-#define IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK			0xf0
-
-#define IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK		0x01
-#define IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP		0x02
-#define IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP		0x04
-#define IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT		0x08
-#define IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK	0x30
-#define   IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_0US	0
-#define   IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_8US	1
-#define   IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_16US	2
-#define   IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_20US	3
-
-/* Maximum number of supported EHT LTF is split */
-#define IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK	0xc0
-#define IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK	0x07
-
-#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK			0x78
-#define IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP		0x80
-
-#define IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW	0x01
-#define IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ	0x02
-#define IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ	0x04
-#define IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_320MHZ	0x08
-#define IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ		0x10
-#define IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ		0x20
-#define IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_320MHZ		0x40
-#define IEEE80211_EHT_PHY_CAP7_TB_SOUNDING_FDBK_RATE_LIMIT	0x80
-
-#define IEEE80211_EHT_PHY_CAP8_RX_1024QAM_WIDER_BW_DL_OFDMA	0x01
-#define IEEE80211_EHT_PHY_CAP8_RX_4096QAM_WIDER_BW_DL_OFDMA	0x02
-
-/*
- * EHT operation channel width as defined in P802.11be_D2.0 section 9.4.2.311
- */
-#define IEEE80211_EHT_OPER_CHAN_WIDTH		0x7
-#define IEEE80211_EHT_OPER_CHAN_WIDTH_20MHZ	0
-#define IEEE80211_EHT_OPER_CHAN_WIDTH_40MHZ	1
-#define IEEE80211_EHT_OPER_CHAN_WIDTH_80MHZ	2
-#define IEEE80211_EHT_OPER_CHAN_WIDTH_160MHZ	3
-#define IEEE80211_EHT_OPER_CHAN_WIDTH_320MHZ	4
-
-/* Calculate 802.11be EHT capabilities IE Tx/Rx EHT MCS NSS Support Field size */
-static inline u8
-ieee80211_eht_mcs_nss_size(const struct ieee80211_he_cap_elem *he_cap,
-			   const struct ieee80211_eht_cap_elem_fixed *eht_cap)
-{
-	u8 count = 0;
-
-	/* on 2.4 GHz, if it supports 40 MHz, the result is 3 */
-	if (he_cap->phy_cap_info[0] &
-	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G)
-		return 3;
-
-	/* on 2.4 GHz, these three bits are reserved, so should be 0 */
-	if (he_cap->phy_cap_info[0] &
-	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G)
-		count += 3;
-
-	if (he_cap->phy_cap_info[0] &
-	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G)
-		count += 3;
-
-	if (eht_cap->phy_cap_info[0] & IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ)
-		count += 3;
-
-	return count ? count : 4;
-}
-
-/* 802.11be EHT PPE Thresholds */
-#define IEEE80211_EHT_PPE_THRES_NSS_POS			0
-#define IEEE80211_EHT_PPE_THRES_NSS_MASK		0xf
-#define IEEE80211_EHT_PPE_THRES_RU_INDEX_BITMASK_MASK	0x1f0
-#define IEEE80211_EHT_PPE_THRES_INFO_PPET_SIZE		3
-#define IEEE80211_EHT_PPE_THRES_INFO_HEADER_SIZE	9
-
-/*
- * Calculate 802.11be EHT capabilities IE EHT field size
- */
-static inline u8
-ieee80211_eht_ppe_size(u16 ppe_thres_hdr, const u8 *phy_cap_info)
-{
-	u32 n;
-
-	if (!(phy_cap_info[5] &
-	      IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT))
-		return 0;
-
-	n = hweight16(ppe_thres_hdr &
-		      IEEE80211_EHT_PPE_THRES_RU_INDEX_BITMASK_MASK);
-	n *= 1 + u16_get_bits(ppe_thres_hdr, IEEE80211_EHT_PPE_THRES_NSS_MASK);
-
-	/*
-	 * Each pair is 6 bits, and we need to add the 9 "header" bits to the
-	 * total size.
-	 */
-	n = n * IEEE80211_EHT_PPE_THRES_INFO_PPET_SIZE * 2 +
-	    IEEE80211_EHT_PPE_THRES_INFO_HEADER_SIZE;
-	return DIV_ROUND_UP(n, 8);
-}
-
-static inline bool
-ieee80211_eht_capa_size_ok(const u8 *he_capa, const u8 *data, u8 len)
-{
-	const struct ieee80211_eht_cap_elem_fixed *elem = (const void *)data;
-	u8 needed = sizeof(struct ieee80211_eht_cap_elem_fixed);
-
-	if (len < needed || !he_capa)
-		return false;
-
-	needed += ieee80211_eht_mcs_nss_size((const void *)he_capa,
-					     (const void *)data);
-	if (len < needed)
-		return false;
-
-	if (elem->phy_cap_info[5] &
-			IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT) {
-		u16 ppe_thres_hdr;
-
-		if (len < needed + sizeof(ppe_thres_hdr))
-			return false;
-
-		ppe_thres_hdr = get_unaligned_le16(data + needed);
-		needed += ieee80211_eht_ppe_size(ppe_thres_hdr,
-						 elem->phy_cap_info);
-	}
-
-	return len >= needed;
-}
-
-static inline bool
-ieee80211_eht_oper_size_ok(const u8 *data, u8 len)
-{
-	const struct ieee80211_eht_operation *elem = (const void *)data;
-	u8 needed = sizeof(*elem);
-
-	if (len < needed)
-		return false;
-
-	if (elem->params & IEEE80211_EHT_OPER_INFO_PRESENT) {
-		needed += 3;
-
-		if (elem->params &
-		    IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT)
-			needed += 2;
-	}
-
-	return len >= needed;
-}
-
-#define LISTEN_INT_USF	GENMASK(15, 14)
-#define LISTEN_INT_UI	GENMASK(13, 0)
-
-#define IEEE80211_MAX_USF	FIELD_MAX(LISTEN_INT_USF)
-#define IEEE80211_MAX_UI	FIELD_MAX(LISTEN_INT_UI)
-
- /* Authentication algorithms */
- #define WLAN_AUTH_OPEN 0
- #define WLAN_AUTH_SHARED_KEY 1
-@@ -3147,10 +1493,12 @@ enum ieee80211_statuscode {
- 	WLAN_STATUS_DENIED_WITH_SUGGESTED_BAND_AND_CHANNEL = 99,
- 	WLAN_STATUS_DENIED_DUE_TO_SPECTRUM_MANAGEMENT = 103,
- 	/* 802.11ai */
-	WLAN_STATUS_FILS_AUTHENTICATION_FAILURE = 108,
-	WLAN_STATUS_UNKNOWN_AUTHENTICATION_SERVER = 109,
-+	WLAN_STATUS_FILS_AUTHENTICATION_FAILURE = 112,
-+	WLAN_STATUS_UNKNOWN_AUTHENTICATION_SERVER = 113,
- 	WLAN_STATUS_SAE_HASH_TO_ELEMENT = 126,
- 	WLAN_STATUS_SAE_PK = 127,
-+	WLAN_STATUS_DENIED_TID_TO_LINK_MAPPING = 133,
-+	WLAN_STATUS_PREF_TID_TO_LINK_MAPPING_SUGGESTED = 134,
- };
- 
- 
-@@ -3427,6 +1775,7 @@ enum ieee80211_eid_ext {
- 	WLAN_EID_EXT_FILS_PUBLIC_KEY = 12,
- 	WLAN_EID_EXT_FILS_NONCE = 13,
- 	WLAN_EID_EXT_FUTURE_CHAN_GUIDANCE = 14,
-+	WLAN_EID_EXT_DH_PARAMETER = 32,
- 	WLAN_EID_EXT_HE_CAPABILITY = 35,
- 	WLAN_EID_EXT_HE_OPERATION = 36,
- 	WLAN_EID_EXT_UORA = 37,
-@@ -3448,6 +1797,10 @@ enum ieee80211_eid_ext {
- 	WLAN_EID_EXT_EHT_OPERATION = 106,
- 	WLAN_EID_EXT_EHT_MULTI_LINK = 107,
- 	WLAN_EID_EXT_EHT_CAPABILITY = 108,
-+	WLAN_EID_EXT_TID_TO_LINK_MAPPING = 109,
-+	WLAN_EID_EXT_BANDWIDTH_INDICATION = 135,
-+	WLAN_EID_EXT_KNOWN_STA_IDENTIFCATION = 136,
-+	WLAN_EID_EXT_NON_AP_STA_REG_CON = 137,
- };
- 
- /* Action category code */
-@@ -3474,6 +1827,7 @@ enum ieee80211_category {
- 	WLAN_CATEGORY_UNPROT_DMG = 20,
- 	WLAN_CATEGORY_VHT = 21,
- 	WLAN_CATEGORY_S1G = 22,
-+	WLAN_CATEGORY_PROTECTED_EHT = 37,
- 	WLAN_CATEGORY_VENDOR_SPECIFIC_PROTECTED = 126,
- 	WLAN_CATEGORY_VENDOR_SPECIFIC = 127,
- };
-@@ -3487,25 +1841,6 @@ enum ieee80211_spectrum_mgmt_actioncode {
- 	WLAN_ACTION_SPCT_CHL_SWITCH = 4,
- };
- 
-/* HT action codes */
-enum ieee80211_ht_actioncode {
-	WLAN_HT_ACTION_NOTIFY_CHANWIDTH = 0,
-	WLAN_HT_ACTION_SMPS = 1,
-	WLAN_HT_ACTION_PSMP = 2,
-	WLAN_HT_ACTION_PCO_PHASE = 3,
-	WLAN_HT_ACTION_CSI = 4,
-	WLAN_HT_ACTION_NONCOMPRESSED_BF = 5,
-	WLAN_HT_ACTION_COMPRESSED_BF = 6,
-	WLAN_HT_ACTION_ASEL_IDX_FEEDBACK = 7,
-};
-
-/* VHT action codes */
-enum ieee80211_vht_actioncode {
-	WLAN_VHT_ACTION_COMPRESSED_BF = 0,
-	WLAN_VHT_ACTION_GROUPID_MGMT = 1,
-	WLAN_VHT_ACTION_OPMODE_NOTIF = 2,
-};
-
- /* Self Protected Action codes */
- enum ieee80211_self_protected_actioncode {
- 	WLAN_SP_RESERVED = 0,
-@@ -3516,32 +1851,12 @@ enum ieee80211_self_protected_actioncode {
- 	WLAN_SP_MGK_ACK = 5,
- };
- 
-/* Mesh action codes */
-enum ieee80211_mesh_actioncode {
-	WLAN_MESH_ACTION_LINK_METRIC_REPORT,
-	WLAN_MESH_ACTION_HWMP_PATH_SELECTION,
-	WLAN_MESH_ACTION_GATE_ANNOUNCEMENT,
-	WLAN_MESH_ACTION_CONGESTION_CONTROL_NOTIFICATION,
-	WLAN_MESH_ACTION_MCCA_SETUP_REQUEST,
-	WLAN_MESH_ACTION_MCCA_SETUP_REPLY,
-	WLAN_MESH_ACTION_MCCA_ADVERTISEMENT_REQUEST,
-	WLAN_MESH_ACTION_MCCA_ADVERTISEMENT,
-	WLAN_MESH_ACTION_MCCA_TEARDOWN,
-	WLAN_MESH_ACTION_TBTT_ADJUSTMENT_REQUEST,
-	WLAN_MESH_ACTION_TBTT_ADJUSTMENT_RESPONSE,
-};
-
- /* Unprotected WNM action codes */
- enum ieee80211_unprotected_wnm_actioncode {
- 	WLAN_UNPROTECTED_WNM_ACTION_TIM = 0,
- 	WLAN_UNPROTECTED_WNM_ACTION_TIMING_MEASUREMENT_RESPONSE = 1,
- };
- 
-/* Public action codes */
-enum ieee80211_public_actioncode {
-	WLAN_PUBLIC_ACTION_FTM_RESPONSE = 33,
-};
-
- /* Security key length */
- enum ieee80211_key_len {
- 	WLAN_KEY_LEN_WEP40 = 5,
-@@ -3558,18 +1873,14 @@ enum ieee80211_key_len {
- 	WLAN_KEY_LEN_BIP_GMAC_256 = 32,
- };
- 
-enum ieee80211_s1g_actioncode {
-	WLAN_S1G_AID_SWITCH_REQUEST,
-	WLAN_S1G_AID_SWITCH_RESPONSE,
-	WLAN_S1G_SYNC_CONTROL,
-	WLAN_S1G_STA_INFO_ANNOUNCE,
-	WLAN_S1G_EDCA_PARAM_SET,
-	WLAN_S1G_EL_OPERATION,
-	WLAN_S1G_TWT_SETUP,
-	WLAN_S1G_TWT_TEARDOWN,
-	WLAN_S1G_SECT_GROUP_ID_LIST,
-	WLAN_S1G_SECT_ID_FEEDBACK,
-	WLAN_S1G_TWT_INFORMATION = 11,
-+/* Radio measurement action codes as defined in IEEE 802.11-2024 - Table 9-470 */
-+enum ieee80211_radio_measurement_actioncode {
-+	WLAN_RM_ACTION_RADIO_MEASUREMENT_REQUEST = 0,
-+	WLAN_RM_ACTION_RADIO_MEASUREMENT_REPORT  = 1,
-+	WLAN_RM_ACTION_LINK_MEASUREMENT_REQUEST  = 2,
-+	WLAN_RM_ACTION_LINK_MEASUREMENT_REPORT   = 3,
-+	WLAN_RM_ACTION_NEIGHBOR_REPORT_REQUEST   = 4,
-+	WLAN_RM_ACTION_NEIGHBOR_REPORT_RESPONSE  = 5,
- };
- 
- #define IEEE80211_WEP_IV_LEN		4
-@@ -3587,6 +1898,9 @@ enum ieee80211_s1g_actioncode {
- #define IEEE80211_GCMP_HDR_LEN		8
- #define IEEE80211_GCMP_MIC_LEN		16
- #define IEEE80211_GCMP_PN_LEN		6
-+#define IEEE80211_CMAC_128_MIC_LEN	8
-+#define IEEE80211_CMAC_256_MIC_LEN	16
-+#define IEEE80211_GMAC_MIC_LEN		16
- 
- #define FILS_NONCE_LEN			16
- #define FILS_MAX_KEK_LEN		64
-@@ -3633,7 +1947,7 @@ enum ieee80211_pub_actioncode {
- 	WLAN_PUB_ACTION_NETWORK_CHANNEL_CONTROL = 30,
- 	WLAN_PUB_ACTION_WHITE_SPACE_MAP_ANN = 31,
- 	WLAN_PUB_ACTION_FTM_REQUEST = 32,
-	WLAN_PUB_ACTION_FTM = 33,
-+	WLAN_PUB_ACTION_FTM_RESPONSE = 33,
- 	WLAN_PUB_ACTION_FILS_DISCOVERY = 34,
- };
- 
-@@ -3712,71 +2026,15 @@ enum ieee80211_tdls_actioncode {
- /* Defines support for enhanced multi-bssid advertisement*/
- #define WLAN_EXT_CAPA11_EMA_SUPPORT	BIT(3)
- 
-+/* Enable Beacon Protection */
-+#define WLAN_EXT_CAPA11_BCN_PROTECT	BIT(4)
-+
- /* TDLS specific payload type in the LLC/SNAP header */
- #define WLAN_TDLS_SNAP_RFTYPE	0x2
- 
- /* BSS Coex IE information field bits */
- #define WLAN_BSS_COEX_INFORMATION_REQUEST	BIT(0)
- 
-/**
- * enum ieee80211_mesh_sync_method - mesh synchronization method identifier
- *
- * @IEEE80211_SYNC_METHOD_NEIGHBOR_OFFSET: the default synchronization method
- * @IEEE80211_SYNC_METHOD_VENDOR: a vendor specific synchronization method
- *	that will be specified in a vendor specific information element
- */
-enum ieee80211_mesh_sync_method {
-	IEEE80211_SYNC_METHOD_NEIGHBOR_OFFSET = 1,
-	IEEE80211_SYNC_METHOD_VENDOR = 255,
-};
-
-/**
- * enum ieee80211_mesh_path_protocol - mesh path selection protocol identifier
- *
- * @IEEE80211_PATH_PROTOCOL_HWMP: the default path selection protocol
- * @IEEE80211_PATH_PROTOCOL_VENDOR: a vendor specific protocol that will
- *	be specified in a vendor specific information element
- */
-enum ieee80211_mesh_path_protocol {
-	IEEE80211_PATH_PROTOCOL_HWMP = 1,
-	IEEE80211_PATH_PROTOCOL_VENDOR = 255,
-};
-
-/**
- * enum ieee80211_mesh_path_metric - mesh path selection metric identifier
- *
- * @IEEE80211_PATH_METRIC_AIRTIME: the default path selection metric
- * @IEEE80211_PATH_METRIC_VENDOR: a vendor specific metric that will be
- *	specified in a vendor specific information element
- */
-enum ieee80211_mesh_path_metric {
-	IEEE80211_PATH_METRIC_AIRTIME = 1,
-	IEEE80211_PATH_METRIC_VENDOR = 255,
-};
-
-/**
- * enum ieee80211_root_mode_identifier - root mesh STA mode identifier
- *
- * These attribute are used by dot11MeshHWMPRootMode to set root mesh STA mode
- *
- * @IEEE80211_ROOTMODE_NO_ROOT: the mesh STA is not a root mesh STA (default)
- * @IEEE80211_ROOTMODE_ROOT: the mesh STA is a root mesh STA if greater than
- *	this value
- * @IEEE80211_PROACTIVE_PREQ_NO_PREP: the mesh STA is a root mesh STA supports
- *	the proactive PREQ with proactive PREP subfield set to 0
- * @IEEE80211_PROACTIVE_PREQ_WITH_PREP: the mesh STA is a root mesh STA
- *	supports the proactive PREQ with proactive PREP subfield set to 1
- * @IEEE80211_PROACTIVE_RANN: the mesh STA is a root mesh STA supports
- *	the proactive RANN
- */
-enum ieee80211_root_mode_identifier {
-	IEEE80211_ROOTMODE_NO_ROOT = 0,
-	IEEE80211_ROOTMODE_ROOT = 1,
-	IEEE80211_PROACTIVE_PREQ_NO_PREP = 2,
-	IEEE80211_PROACTIVE_PREQ_WITH_PREP = 3,
-	IEEE80211_PROACTIVE_RANN = 4,
-};
-
- /*
-  * IEEE 802.11-2007 7.3.2.9 Country information element
-  *
-@@ -3869,7 +2127,7 @@ enum ieee80211_idle_options {
- };
- 
- /**
- * struct ieee80211_bss_max_idle_period_ie
-+ * struct ieee80211_bss_max_idle_period_ie - BSS max idle period element struct
-  *
-  * This structure refers to "BSS Max idle period element"
-  *
-@@ -3884,19 +2142,6 @@ struct ieee80211_bss_max_idle_period_ie {
- 	u8 idle_options;
- } __packed;
- 
-/* BACK action code */
-enum ieee80211_back_actioncode {
-	WLAN_ACTION_ADDBA_REQ = 0,
-	WLAN_ACTION_ADDBA_RESP = 1,
-	WLAN_ACTION_DELBA = 2,
-};
-
-/* BACK (block-ack) parties */
-enum ieee80211_back_parties {
-	WLAN_BACK_RECIPIENT = 0,
-	WLAN_BACK_INITIATOR = 1,
-};
-
- /* SA Query action */
- enum ieee80211_sa_query_action {
- 	WLAN_ACTION_SA_QUERY_REQUEST = 0,
-@@ -3904,7 +2149,7 @@ enum ieee80211_sa_query_action {
- };
- 
- /**
- * struct ieee80211_bssid_index
-+ * struct ieee80211_bssid_index - multiple BSSID index element structure
-  *
-  * This structure refers to "Multiple BSSID-index element"
-  *
-@@ -3919,7 +2164,8 @@ struct ieee80211_bssid_index {
- };
- 
- /**
- * struct ieee80211_multiple_bssid_configuration
-+ * struct ieee80211_multiple_bssid_configuration - multiple BSSID configuration
-+ *	element structure
-  *
-  * This structure refers to "Multiple BSSID Configuration element"
-  *
-@@ -4029,44 +2275,33 @@ struct ieee80211_tspec_ie {
- 	__le16 medium_time;
- } __packed;
- 
-struct ieee80211_he_6ghz_capa {
-	/* uses IEEE80211_HE_6GHZ_CAP_* below */
-	__le16 capa;
-} __packed;
-
-/* HE 6 GHz band capabilities */
-/* uses enum ieee80211_min_mpdu_spacing values */
-#define IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START	0x0007
-/* uses enum ieee80211_vht_max_ampdu_length_exp values */
-#define IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP	0x0038
-/* uses IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_* values */
-#define IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN	0x00c0
-/* WLAN_HT_CAP_SM_PS_* values */
-#define IEEE80211_HE_6GHZ_CAP_SM_PS		0x0600
-#define IEEE80211_HE_6GHZ_CAP_RD_RESPONDER	0x0800
-#define IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS	0x1000
-#define IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS	0x2000
-
- /**
-  * ieee80211_get_qos_ctl - get pointer to qos control bytes
-  * @hdr: the frame
-+ * Return: a pointer to the QoS control field in the frame header
-  *
-  * The qos ctrl bytes come after the frame_control, duration, seq_num
- * and 3 or 4 addresses of length ETH_ALEN.
- * 3 addr: 2 + 2 + 2 + 3*6 = 24
- * 4 addr: 2 + 2 + 2 + 4*6 = 30
-+ * and 3 or 4 addresses of length ETH_ALEN. Checks frame_control to choose
-+ * between struct ieee80211_qos_hdr_4addr and struct ieee80211_qos_hdr.
-  */
- static inline u8 *ieee80211_get_qos_ctl(struct ieee80211_hdr *hdr)
- {
-	if (ieee80211_has_a4(hdr->frame_control))
-		return (u8 *)hdr + 30;
-+	union {
-+		struct ieee80211_qos_hdr	addr3;
-+		struct ieee80211_qos_hdr_4addr	addr4;
-+	} *qos;
-+
-+	qos = (void *)hdr;
-+	if (ieee80211_has_a4(qos->addr3.frame_control))
-+		return (u8 *)&qos->addr4.qos_ctrl;
- 	else
-		return (u8 *)hdr + 24;
-+		return (u8 *)&qos->addr3.qos_ctrl;
- }
- 
- /**
-  * ieee80211_get_tid - get qos TID
-  * @hdr: the frame
-+ * Return: the TID from the QoS control field
-  */
- static inline u8 ieee80211_get_tid(struct ieee80211_hdr *hdr)
- {
-@@ -4078,6 +2313,7 @@ static inline u8 ieee80211_get_tid(struct ieee80211_hdr *hdr)
- /**
-  * ieee80211_get_SA - get pointer to SA
-  * @hdr: the frame
-+ * Return: a pointer to the source address (SA)
-  *
-  * Given an 802.11 frame, this function returns the offset
-  * to the source address (SA). It does not verify that the
-@@ -4097,6 +2333,7 @@ static inline u8 *ieee80211_get_SA(struct ieee80211_hdr *hdr)
- /**
-  * ieee80211_get_DA - get pointer to DA
-  * @hdr: the frame
-+ * Return: a pointer to the destination address (DA)
-  *
-  * Given an 802.11 frame, this function returns the offset
-  * to the destination address (DA). It does not verify that
-@@ -4113,8 +2350,48 @@ static inline u8 *ieee80211_get_DA(struct ieee80211_hdr *hdr)
- }
- 
- /**
-+ * ieee80211_is_bufferable_mmpdu - check if frame is bufferable MMPDU
-+ * @skb: the skb to check, starting with the 802.11 header
-+ * Return: whether or not the MMPDU is bufferable
-+ */
-+static inline bool ieee80211_is_bufferable_mmpdu(struct sk_buff *skb)
-+{
-+	struct ieee80211_mgmt *mgmt = (void *)skb->data;
-+	__le16 fc = mgmt->frame_control;
-+
-+	/*
-+	 * IEEE 802.11 REVme D2.0 definition of bufferable MMPDU;
-+	 * note that this ignores the IBSS special case.
-+	 */
-+	if (!ieee80211_is_mgmt(fc))
-+		return false;
-+
-+	if (ieee80211_is_disassoc(fc) || ieee80211_is_deauth(fc))
-+		return true;
-+
-+	if (!ieee80211_is_action(fc))
-+		return false;
-+
-+	if (skb->len < offsetofend(typeof(*mgmt), u.action.u.ftm.action_code))
-+		return true;
-+
-+	/* action frame - additionally check for non-bufferable FTM */
-+
-+	if (mgmt->u.action.category != WLAN_CATEGORY_PUBLIC &&
-+	    mgmt->u.action.category != WLAN_CATEGORY_PROTECTED_DUAL_OF_ACTION)
-+		return true;
-+
-+	if (mgmt->u.action.u.ftm.action_code == WLAN_PUB_ACTION_FTM_REQUEST ||
-+	    mgmt->u.action.u.ftm.action_code == WLAN_PUB_ACTION_FTM_RESPONSE)
-+		return false;
-+
-+	return true;
-+}
-+
-+/**
-  * _ieee80211_is_robust_mgmt_frame - check if frame is a robust management frame
-  * @hdr: the frame (buffer must include at least the first octet of payload)
-+ * Return: whether or not the frame is a robust management frame
-  */
- static inline bool _ieee80211_is_robust_mgmt_frame(struct ieee80211_hdr *hdr)
- {
-@@ -4151,6 +2428,7 @@ static inline bool _ieee80211_is_robust_mgmt_frame(struct ieee80211_hdr *hdr)
- /**
-  * ieee80211_is_robust_mgmt_frame - check if skb contains a robust mgmt frame
-  * @skb: the skb containing the frame, length will be checked
-+ * Return: whether or not the frame is a robust management frame
-  */
- static inline bool ieee80211_is_robust_mgmt_frame(struct sk_buff *skb)
- {
-@@ -4163,6 +2441,7 @@ static inline bool ieee80211_is_robust_mgmt_frame(struct sk_buff *skb)
-  * ieee80211_is_public_action - check if frame is a public action frame
-  * @hdr: the frame
-  * @len: length of the frame
-+ * Return: whether or not the frame is a public action frame
-  */
- static inline bool ieee80211_is_public_action(struct ieee80211_hdr *hdr,
- 					      size_t len)
-@@ -4177,9 +2456,40 @@ static inline bool ieee80211_is_public_action(struct ieee80211_hdr *hdr,
- }
- 
- /**
-+ * ieee80211_is_protected_dual_of_public_action - check if skb contains a
-+ * protected dual of public action management frame
-+ * @skb: the skb containing the frame, length will be checked
-+ *
-+ * Return: true if the skb contains a protected dual of public action
-+ * management frame, false otherwise.
-+ */
-+static inline bool
-+ieee80211_is_protected_dual_of_public_action(struct sk_buff *skb)
-+{
-+	u8 action;
-+
-+	if (!ieee80211_is_public_action((void *)skb->data, skb->len) ||
-+	    skb->len < IEEE80211_MIN_ACTION_SIZE + 1)
-+		return false;
-+
-+	action = *(u8 *)(skb->data + IEEE80211_MIN_ACTION_SIZE);
-+
-+	return action != WLAN_PUB_ACTION_20_40_BSS_COEX &&
-+		action != WLAN_PUB_ACTION_DSE_REG_LOC_ANN &&
-+		action != WLAN_PUB_ACTION_MSMT_PILOT &&
-+		action != WLAN_PUB_ACTION_TDLS_DISCOVER_RES &&
-+		action != WLAN_PUB_ACTION_LOC_TRACK_NOTI &&
-+		action != WLAN_PUB_ACTION_FTM_REQUEST &&
-+		action != WLAN_PUB_ACTION_FTM_RESPONSE &&
-+		action != WLAN_PUB_ACTION_FILS_DISCOVERY &&
-+		action != WLAN_PUB_ACTION_VENDOR_SPECIFIC;
-+}
-+
-+/**
-  * _ieee80211_is_group_privacy_action - check if frame is a group addressed
- * privacy action frame
-+ *	privacy action frame
-  * @hdr: the frame
-+ * Return: whether or not the frame is a group addressed privacy action frame
-  */
- static inline bool _ieee80211_is_group_privacy_action(struct ieee80211_hdr *hdr)
- {
-@@ -4195,8 +2505,9 @@ static inline bool _ieee80211_is_group_privacy_action(struct ieee80211_hdr *hdr)
- 
- /**
-  * ieee80211_is_group_privacy_action - check if frame is a group addressed
- * privacy action frame
-+ *	privacy action frame
-  * @skb: the skb containing the frame, length will be checked
-+ * Return: whether or not the frame is a group addressed privacy action frame
-  */
- static inline bool ieee80211_is_group_privacy_action(struct sk_buff *skb)
- {
-@@ -4208,20 +2519,15 @@ static inline bool ieee80211_is_group_privacy_action(struct sk_buff *skb)
- /**
-  * ieee80211_tu_to_usec - convert time units (TU) to microseconds
-  * @tu: the TUs
-+ * Return: the time value converted to microseconds
-  */
- static inline unsigned long ieee80211_tu_to_usec(unsigned long tu)
- {
- 	return 1024 * tu;
- }
- 
-/**
- * ieee80211_check_tim - check if AID bit is set in TIM
- * @tim: the TIM IE
- * @tim_len: length of the TIM IE
- * @aid: the AID to look for
- */
-static inline bool ieee80211_check_tim(const struct ieee80211_tim_ie *tim,
-				       u8 tim_len, u16 aid)
-+static inline bool __ieee80211_check_tim(const struct ieee80211_tim_ie *tim,
-+					 u8 tim_len, u16 aid)
- {
- 	u8 mask;
- 	u8 index, indexn1, indexn2;
-@@ -4245,14 +2551,15 @@ static inline bool ieee80211_check_tim(const struct ieee80211_tim_ie *tim,
- }
- 
- /**
- * ieee80211_get_tdls_action - get tdls packet action (or -1, if not tdls packet)
-+ * ieee80211_get_tdls_action - get TDLS action code
-  * @skb: the skb containing the frame, length will not be checked
- * @hdr_size: the size of the ieee80211_hdr that starts at skb->data
-+ * Return: the TDLS action code, or -1 if it's not an encapsulated TDLS action
-+ *	frame
-  *
-  * This function assumes the frame is a data frame, and that the network header
-  * is in the correct place.
-  */
-static inline int ieee80211_get_tdls_action(struct sk_buff *skb, u32 hdr_size)
-+static inline int ieee80211_get_tdls_action(struct sk_buff *skb)
- {
- 	if (!skb_is_nonlinear(skb) &&
- 	    skb->len > (skb_network_offset(skb) + 2)) {
-@@ -4287,6 +2594,7 @@ static inline int ieee80211_get_tdls_action(struct sk_buff *skb, u32 hdr_size)
- /**
-  * ieee80211_action_contains_tpc - checks if the frame contains TPC element
-  * @skb: the skb containing the frame, length will be checked
-+ * Return: %true if the frame contains a TPC element, %false otherwise
-  *
-  * This function checks if it's either TPC report action frame or Link
-  * Measurement report action frame as defined in IEEE Std. 802.11-2012 8.5.2.5
-@@ -4331,6 +2639,11 @@ static inline bool ieee80211_action_contains_tpc(struct sk_buff *skb)
- 	return true;
- }
- 
-+/**
-+ * ieee80211_is_timing_measurement - check if frame is timing measurement response
-+ * @skb: the SKB to check
-+ * Return: whether or not the frame is a valid timing measurement response
-+ */
- static inline bool ieee80211_is_timing_measurement(struct sk_buff *skb)
- {
- 	struct ieee80211_mgmt *mgmt = (void *)skb->data;
-@@ -4350,6 +2663,11 @@ static inline bool ieee80211_is_timing_measurement(struct sk_buff *skb)
- 	return false;
- }
- 
-+/**
-+ * ieee80211_is_ftm - check if frame is FTM response
-+ * @skb: the SKB to check
-+ * Return: whether or not the frame is a valid FTM response action frame
-+ */
- static inline bool ieee80211_is_ftm(struct sk_buff *skb)
- {
- 	struct ieee80211_mgmt *mgmt = (void *)skb->data;
-@@ -4358,7 +2676,7 @@ static inline bool ieee80211_is_ftm(struct sk_buff *skb)
- 		return false;
- 
- 	if (mgmt->u.action.u.ftm.action_code ==
-		WLAN_PUBLIC_ACTION_FTM_RESPONSE &&
-+		WLAN_PUB_ACTION_FTM_RESPONSE &&
- 	    skb->len >= offsetofend(typeof(*mgmt), u.action.u.ftm))
- 		return true;
- 
-@@ -4404,6 +2722,7 @@ struct element {
-  * @element: element pointer after for_each_element() or friends
-  * @data: same data pointer as passed to for_each_element() or friends
-  * @datalen: same data length as passed to for_each_element() or friends
-+ * Return: %true if all elements were iterated, %false otherwise; see notes
-  *
-  * This function returns %true if all the data was parsed or considered
-  * while walking the elements. Only use this if your for_each_element()
-@@ -4418,7 +2737,7 @@ static inline bool for_each_element_completed(const struct element *element,
- 	return (const u8 *)element == (const u8 *)data + datalen;
- }
- 
-/**
-+/*
-  * RSNX Capabilities:
-  * bits 0-3: Field length (n-1)
-  */
-@@ -4433,8 +2752,8 @@ static inline bool for_each_element_completed(const struct element *element,
- #define IEEE80211_AP_INFO_TBTT_HDR_FILTERED			0x04
- #define IEEE80211_AP_INFO_TBTT_HDR_COLOC			0x08
- #define IEEE80211_AP_INFO_TBTT_HDR_COUNT			0xF0
-#define IEEE80211_TBTT_INFO_OFFSET_BSSID_BSS_PARAM		9
-#define IEEE80211_TBTT_INFO_OFFSET_BSSID_SSSID_BSS_PARAM	13
-+#define IEEE80211_TBTT_INFO_TYPE_TBTT				0
-+#define IEEE80211_TBTT_INFO_TYPE_MLD				1
- 
- #define IEEE80211_RNR_TBTT_PARAMS_OCT_RECOMMENDED		0x01
- #define IEEE80211_RNR_TBTT_PARAMS_SAME_SSID			0x02
-@@ -4444,6 +2763,9 @@ static inline bool for_each_element_completed(const struct element *element,
- #define IEEE80211_RNR_TBTT_PARAMS_PROBE_ACTIVE			0x20
- #define IEEE80211_RNR_TBTT_PARAMS_COLOC_AP			0x40
- 
-+#define IEEE80211_RNR_TBTT_PARAMS_PSD_NO_LIMIT			127
-+#define IEEE80211_RNR_TBTT_PARAMS_PSD_RESERVED			-128
-+
- struct ieee80211_neighbor_ap_info {
- 	u8 tbtt_info_hdr;
- 	u8 tbtt_info_len;
-@@ -4458,229 +2780,64 @@ enum ieee80211_range_params_max_total_ltf {
- 	IEEE80211_RANGE_PARAMS_MAX_TOTAL_LTF_UNSPECIFIED,
- };
- 
-/* multi-link device */
-#define IEEE80211_MLD_MAX_NUM_LINKS	15
-
-#define IEEE80211_ML_CONTROL_TYPE			0x0007
-#define IEEE80211_ML_CONTROL_TYPE_BASIC			0
-#define IEEE80211_ML_CONTROL_TYPE_PREQ			1
-#define IEEE80211_ML_CONTROL_TYPE_RECONF		2
-#define IEEE80211_ML_CONTROL_TYPE_TDLS			3
-#define IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS		4
-#define IEEE80211_ML_CONTROL_PRESENCE_MASK		0xfff0
-
-struct ieee80211_multi_link_elem {
-	__le16 control;
-	u8 variable[];
-+/*
-+ * reduced neighbor report, based on Draft P802.11be_D3.0,
-+ * section 9.4.2.170.2.
-+ */
-+struct ieee80211_rnr_mld_params {
-+	u8 mld_id;
-+	__le16 params;
- } __packed;
- 
-#define IEEE80211_MLC_BASIC_PRES_LINK_ID		0x0010
-#define IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT	0x0020
-#define IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY		0x0040
-#define IEEE80211_MLC_BASIC_PRES_EML_CAPA		0x0080
-#define IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP		0x0100
-#define IEEE80211_MLC_BASIC_PRES_MLD_ID			0x0200
-
-#define IEEE80211_MED_SYNC_DELAY_DURATION		0x00ff
-#define IEEE80211_MED_SYNC_DELAY_SYNC_OFDM_ED_THRESH	0x0f00
-#define IEEE80211_MED_SYNC_DELAY_SYNC_MAX_NUM_TXOPS	0xf000
-
-#define IEEE80211_EML_CAP_EMLSR_SUPP			0x0001
-#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY		0x000e
-#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_0US		0
-#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_32US		1
-#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_64US		2
-#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_128US		3
-#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_256US		4
-#define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY	0x0070
-#define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_0US		0
-#define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_16US		1
-#define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_32US		2
-#define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_64US		3
-#define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_128US		4
-#define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_256US		5
-#define IEEE80211_EML_CAP_EMLMR_SUPPORT			0x0080
-#define IEEE80211_EML_CAP_EMLMR_DELAY			0x0700
-#define  IEEE80211_EML_CAP_EMLMR_DELAY_0US			0
-#define  IEEE80211_EML_CAP_EMLMR_DELAY_32US			1
-#define  IEEE80211_EML_CAP_EMLMR_DELAY_64US			2
-#define  IEEE80211_EML_CAP_EMLMR_DELAY_128US			3
-#define  IEEE80211_EML_CAP_EMLMR_DELAY_256US			4
-#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT		0x7800
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_0			0
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128US		1
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_256US		2
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_512US		3
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_1TU		4
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_2TU		5
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_4TU		6
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_8TU		7
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_16TU		8
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_32TU		9
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_64TU		10
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128TU		11
-
-#define IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS		0x000f
-#define IEEE80211_MLD_CAP_OP_SRS_SUPPORT		0x0010
-#define IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP	0x0060
-#define IEEE80211_MLD_CAP_OP_FREQ_SEP_TYPE_IND		0x0f80
-#define IEEE80211_MLD_CAP_OP_AAR_SUPPORT		0x1000
-
-struct ieee80211_mle_basic_common_info {
-	u8 len;
-	u8 mld_mac_addr[ETH_ALEN];
-	u8 variable[];
-} __packed;
-+#define IEEE80211_RNR_MLD_PARAMS_LINK_ID			0x000F
-+#define IEEE80211_RNR_MLD_PARAMS_BSS_CHANGE_COUNT		0x0FF0
-+#define IEEE80211_RNR_MLD_PARAMS_UPDATES_INCLUDED		0x1000
-+#define IEEE80211_RNR_MLD_PARAMS_DISABLED_LINK			0x2000
- 
-#define IEEE80211_MLC_PREQ_PRES_MLD_ID			0x0010
-+/* Format of the TBTT information element if it has 7, 8 or 9 bytes */
-+struct ieee80211_tbtt_info_7_8_9 {
-+	u8 tbtt_offset;
-+	u8 bssid[ETH_ALEN];
- 
-struct ieee80211_mle_preq_common_info {
-	u8 len;
-	u8 variable[];
-+	/* The following element is optional, structure may not grow */
-+	u8 bss_params;
-+	s8 psd_20;
- } __packed;
- 
-#define IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR		0x0010
-
-/* no fixed fields in RECONF */
-+/* Format of the TBTT information element if it has >= 11 bytes */
-+struct ieee80211_tbtt_info_ge_11 {
-+	u8 tbtt_offset;
-+	u8 bssid[ETH_ALEN];
-+	__le32 short_ssid;
- 
-struct ieee80211_mle_tdls_common_info {
-	u8 len;
-	u8 ap_mld_mac_addr[ETH_ALEN];
-+	/* The following elements are optional, structure may grow */
-+	u8 bss_params;
-+	s8 psd_20;
-+	struct ieee80211_rnr_mld_params mld_params;
- } __packed;
- 
-#define IEEE80211_MLC_PRIO_ACCESS_PRES_AP_MLD_MAC_ADDR	0x0010
-
-/* no fixed fields in PRIO_ACCESS */
-+#include "ieee80211-ht.h"
-+#include "ieee80211-vht.h"
-+#include "ieee80211-he.h"
-+#include "ieee80211-eht.h"
-+#include "ieee80211-mesh.h"
-+#include "ieee80211-s1g.h"
-+#include "ieee80211-p2p.h"
-+#include "ieee80211-nan.h"
- 
- /**
- * ieee80211_mle_common_size - check multi-link element common size
- * @data: multi-link element, must already be checked for size using
- *	ieee80211_mle_size_ok()
- */
-static inline u8 ieee80211_mle_common_size(const u8 *data)
-{
-	const struct ieee80211_multi_link_elem *mle = (const void *)data;
-	u16 control = le16_to_cpu(mle->control);
-	u8 common = 0;
-
-	switch (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE)) {
-	case IEEE80211_ML_CONTROL_TYPE_BASIC:
-		common += sizeof(struct ieee80211_mle_basic_common_info);
-		break;
-	case IEEE80211_ML_CONTROL_TYPE_PREQ:
-		common += sizeof(struct ieee80211_mle_preq_common_info);
-		break;
-	case IEEE80211_ML_CONTROL_TYPE_RECONF:
-		if (control & IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR)
-			common += ETH_ALEN;
-		return common;
-	case IEEE80211_ML_CONTROL_TYPE_TDLS:
-		common += sizeof(struct ieee80211_mle_tdls_common_info);
-		break;
-	case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS:
-		if (control & IEEE80211_MLC_PRIO_ACCESS_PRES_AP_MLD_MAC_ADDR)
-			common += ETH_ALEN;
-		return common;
-	default:
-		WARN_ON(1);
-		return 0;
-	}
-
-	return common + mle->variable[0];
-}
-
-/**
- * ieee80211_mle_size_ok - validate multi-link element size
- * @data: pointer to the element data
- * @len: length of the containing element
-+ * ieee80211_check_tim - check if AID bit is set in TIM
-+ * @tim: the TIM IE
-+ * @tim_len: length of the TIM IE
-+ * @aid: the AID to look for
-+ * @s1g: whether the TIM is from an S1G PPDU
-+ * Return: whether or not traffic is indicated in the TIM for the given AID
-  */
-static inline bool ieee80211_mle_size_ok(const u8 *data, u8 len)
-+static inline bool ieee80211_check_tim(const struct ieee80211_tim_ie *tim,
-+				       u8 tim_len, u16 aid, bool s1g)
- {
-	const struct ieee80211_multi_link_elem *mle = (const void *)data;
-	u8 fixed = sizeof(*mle);
-	u8 common = 0;
-	bool check_common_len = false;
-	u16 control;
-
-	if (len < fixed)
-		return false;
-
-	control = le16_to_cpu(mle->control);
-
-	switch (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE)) {
-	case IEEE80211_ML_CONTROL_TYPE_BASIC:
-		common += sizeof(struct ieee80211_mle_basic_common_info);
-		check_common_len = true;
-		if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID)
-			common += 1;
-		if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT)
-			common += 1;
-		if (control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY)
-			common += 2;
-		if (control & IEEE80211_MLC_BASIC_PRES_EML_CAPA)
-			common += 2;
-		if (control & IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP)
-			common += 2;
-		if (control & IEEE80211_MLC_BASIC_PRES_MLD_ID)
-			common += 1;
-		break;
-	case IEEE80211_ML_CONTROL_TYPE_PREQ:
-		common += sizeof(struct ieee80211_mle_preq_common_info);
-		if (control & IEEE80211_MLC_PREQ_PRES_MLD_ID)
-			common += 1;
-		check_common_len = true;
-		break;
-	case IEEE80211_ML_CONTROL_TYPE_RECONF:
-		if (control & IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR)
-			common += ETH_ALEN;
-		break;
-	case IEEE80211_ML_CONTROL_TYPE_TDLS:
-		common += sizeof(struct ieee80211_mle_tdls_common_info);
-		check_common_len = true;
-		break;
-	case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS:
-		if (control & IEEE80211_MLC_PRIO_ACCESS_PRES_AP_MLD_MAC_ADDR)
-			common += ETH_ALEN;
-		break;
-	default:
-		/* we don't know this type */
-		return true;
-	}
-
-	if (len < fixed + common)
-		return false;
-
-	if (!check_common_len)
-		return true;
-
-	/* if present, common length is the first octet there */
-	return mle->variable[0] >= common;
-+	return s1g ? ieee80211_s1g_check_tim(tim, tim_len, aid) :
-+		     __ieee80211_check_tim(tim, tim_len, aid);
- }
- 
-enum ieee80211_mle_subelems {
-	IEEE80211_MLE_SUBELEM_PER_STA_PROFILE		= 0,
-};
-
-#define IEEE80211_MLE_STA_CONTROL_LINK_ID			0x000f
-#define IEEE80211_MLE_STA_CONTROL_COMPLETE_PROFILE		0x0010
-#define IEEE80211_MLE_STA_CONTROL_STA_MAC_ADDR_PRESENT		0x0020
-#define IEEE80211_MLE_STA_CONTROL_BEACON_INT_PRESENT		0x0040
-#define IEEE80211_MLE_STA_CONTROL_TSF_OFFS_PRESENT		0x0080
-#define IEEE80211_MLE_STA_CONTROL_DTIM_INFO_PRESENT		0x0100
-#define IEEE80211_MLE_STA_CONTROL_NSTR_LINK_PAIR_PRESENT	0x0200
-#define IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE		0x0400
-#define IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT	0x0800
-
-struct ieee80211_mle_per_sta_profile {
-	__le16 control;
-	u8 sta_info_len;
-	u8 variable[];
-} __packed;
-
-#define for_each_mle_subelement(_elem, _data, _len)			\
-	if (ieee80211_mle_size_ok(_data, _len))				\
-		for_each_element(_elem,					\
-				 _data + ieee80211_mle_common_size(_data),\
-				 _len - ieee80211_mle_common_size(_data))
-
- #endif /* LINUX_IEEE80211_H */
-diff --git a/include/linux/ieee802154.h b/include/linux/ieee802154.h
-index f1f9412b6ac6..140f61ec0f5f 100644
--- a/include/linux/ieee802154.h
-+++ b/include/linux/ieee802154.h
-@@ -44,6 +44,13 @@
- #define IEEE802154_SHORT_ADDR_LEN	2
- #define IEEE802154_PAN_ID_LEN		2
- 
-+/* Duration in superframe order */
-+#define IEEE802154_MAX_SCAN_DURATION	14
-+#define IEEE802154_ACTIVE_SCAN_DURATION	15
-+/* Superframe duration in slots */
-+#define IEEE802154_SUPERFRAME_PERIOD	16
-+/* Various periods expressed in symbols */
-+#define IEEE802154_SLOT_PERIOD		60
- #define IEEE802154_LIFS_PERIOD		40
- #define IEEE802154_SIFS_PERIOD		12
- #define IEEE802154_MAX_SIFS_FRAME_SIZE	18
-@@ -276,6 +283,30 @@ enum {
- 	IEEE802154_SYSTEM_ERROR = 0xff,
- };
- 
-+/**
-+ * enum ieee802154_filtering_level - Filtering levels applicable to a PHY
-+ *
-+ * @IEEE802154_FILTERING_NONE: No filtering at all, what is received is
-+ *	forwarded to the softMAC
-+ * @IEEE802154_FILTERING_1_FCS: First filtering level, frames with an invalid
-+ *	FCS should be dropped
-+ * @IEEE802154_FILTERING_2_PROMISCUOUS: Second filtering level, promiscuous
-+ *	mode as described in the spec, identical in terms of filtering to the
-+ *	level one on PHY side, but at the MAC level the frame should be
-+ *	forwarded to the upper layer directly
-+ * @IEEE802154_FILTERING_3_SCAN: Third filtering level, scan related, where
-+ *	only beacons must be processed, all remaining traffic gets dropped
-+ * @IEEE802154_FILTERING_4_FRAME_FIELDS: Fourth filtering level actually
-+ *	enforcing the validity of the content of the frame with various checks
-+ */
-+enum ieee802154_filtering_level {
-+	IEEE802154_FILTERING_NONE,
-+	IEEE802154_FILTERING_1_FCS,
-+	IEEE802154_FILTERING_2_PROMISCUOUS,
-+	IEEE802154_FILTERING_3_SCAN,
-+	IEEE802154_FILTERING_4_FRAME_FIELDS,
-+};
-+
- /* frame control handling */
- #define IEEE802154_FCTL_FTYPE		0x0003
- #define IEEE802154_FCTL_ACKREQ		0x0020
-diff --git a/include/linux/if_arp.h b/include/linux/if_arp.h
-index 1ed52441972f..10a1e81434cb 100644
--- a/include/linux/if_arp.h
-+++ b/include/linux/if_arp.h
-@@ -53,6 +53,10 @@ static inline bool dev_is_mac_header_xmit(const struct net_device *dev)
- 	case ARPHRD_NONE:
- 	case ARPHRD_RAWIP:
- 	case ARPHRD_PIMREG:
-+	/* PPP adds its l2 header automatically in ppp_start_xmit().
-+	 * This makes it look like an l3 device to __bpf_redirect() and tcf_mirred_init().
-+	 */
-+	case ARPHRD_PPP:
- 		return false;
- 	default:
- 		return true;
-diff --git a/include/linux/if_bridge.h b/include/linux/if_bridge.h
-index d62ef428e3aa..c5fe3b2a53e8 100644
--- a/include/linux/if_bridge.h
-+++ b/include/linux/if_bridge.h
-@@ -59,15 +59,15 @@ struct br_ip_list {
- #define BR_MRP_LOST_IN_CONT	BIT(19)
- #define BR_TX_FWD_OFFLOAD	BIT(20)
- #define BR_PORT_LOCKED		BIT(21)
-+#define BR_PORT_MAB		BIT(22)
-+#define BR_NEIGH_VLAN_SUPPRESS	BIT(23)
- 
- #define BR_DEFAULT_AGEING_TIME	(300 * HZ)
- 
- struct net_bridge;
-void brioctl_set(int (*hook)(struct net *net, struct net_bridge *br,
-			     unsigned int cmd, struct ifreq *ifr,
-+void brioctl_set(int (*hook)(struct net *net, unsigned int cmd,
- 			     void __user *uarg));
-int br_ioctl_call(struct net *net, struct net_bridge *br, unsigned int cmd,
-		  struct ifreq *ifr, void __user *uarg);
-+int br_ioctl_call(struct net *net, unsigned int cmd, void __user *uarg);
- 
- #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_BRIDGE_IGMP_SNOOPING)
- int br_multicast_list_adjacent(struct net_device *dev,
-diff --git a/include/linux/if_ether.h b/include/linux/if_ether.h
-index 8a9792a6427a..61b7335aa037 100644
--- a/include/linux/if_ether.h
-+++ b/include/linux/if_ether.h
-@@ -19,6 +19,9 @@
- #include <linux/skbuff.h>
- #include <uapi/linux/if_ether.h>
- 
-+/* XX:XX:XX:XX:XX:XX */
-+#define MAC_ADDR_STR_LEN (3 * ETH_ALEN - 1)
-+
- static inline struct ethhdr *eth_hdr(const struct sk_buff *skb)
- {
- 	return (struct ethhdr *)skb_mac_header(skb);
-diff --git a/include/linux/if_hsr.h b/include/linux/if_hsr.h
-index 0404f5bf4f30..f4cf2dd36d19 100644
--- a/include/linux/if_hsr.h
-+++ b/include/linux/if_hsr.h
-@@ -13,6 +13,15 @@ enum hsr_version {
- 	PRP_V1,
- };
- 
-+enum hsr_port_type {
-+	HSR_PT_NONE = 0,	/* Must be 0, used by framereg */
-+	HSR_PT_SLAVE_A,
-+	HSR_PT_SLAVE_B,
-+	HSR_PT_INTERLINK,
-+	HSR_PT_MASTER,
-+	HSR_PT_PORTS,	/* This must be the last item in the enum */
-+};
-+
- /* HSR Tag.
-  * As defined in IEC-62439-3:2010, the HSR tag is really { ethertype = 0x88FB,
-  * path, LSDU_size, sequence Nr }. But we let eth_header() create { h_dest,
-@@ -32,6 +41,10 @@ struct hsr_tag {
- #if IS_ENABLED(CONFIG_HSR)
- extern bool is_hsr_master(struct net_device *dev);
- extern int hsr_get_version(struct net_device *dev, enum hsr_version *ver);
-+struct net_device *hsr_get_port_ndev(struct net_device *ndev,
-+				     enum hsr_port_type pt);
-+int hsr_get_port_type(struct net_device *hsr_dev, struct net_device *dev,
-+		      enum hsr_port_type *type);
- #else
- static inline bool is_hsr_master(struct net_device *dev)
- {
-@@ -42,6 +55,19 @@ static inline int hsr_get_version(struct net_device *dev,
- {
- 	return -EINVAL;
- }
-+
-+static inline struct net_device *hsr_get_port_ndev(struct net_device *ndev,
-+						   enum hsr_port_type pt)
-+{
-+	return ERR_PTR(-EINVAL);
-+}
-+
-+static inline int hsr_get_port_type(struct net_device *hsr_dev,
-+				    struct net_device *dev,
-+				    enum hsr_port_type *type)
-+{
-+	return -EINVAL;
-+}
- #endif /* CONFIG_HSR */
- 
- #endif /*_LINUX_IF_HSR_H_*/
-diff --git a/include/linux/if_ltalk.h b/include/linux/if_ltalk.h
-deleted file mode 100644
-index 4cc1c0b77870..000000000000
--- a/include/linux/if_ltalk.h
-+++ /dev/null
-@@ -1,8 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_LTALK_H
-#define __LINUX_LTALK_H
-
-#include <uapi/linux/if_ltalk.h>
-
-extern struct net_device *alloc_ltalkdev(int sizeof_priv);
-#endif
-diff --git a/include/linux/if_macvlan.h b/include/linux/if_macvlan.h
-index 523025106a64..0f7281e3e448 100644
--- a/include/linux/if_macvlan.h
-+++ b/include/linux/if_macvlan.h
-@@ -59,8 +59,10 @@ static inline void macvlan_count_rx(const struct macvlan_dev *vlan,
- 
- extern void macvlan_common_setup(struct net_device *dev);
- 
-extern int macvlan_common_newlink(struct net *src_net, struct net_device *dev,
-				  struct nlattr *tb[], struct nlattr *data[],
-+struct rtnl_newlink_params;
-+
-+extern int macvlan_common_newlink(struct net_device *dev,
-+				  struct rtnl_newlink_params *params,
- 				  struct netlink_ext_ack *extack);
- 
- extern void macvlan_dellink(struct net_device *dev, struct list_head *head);
-diff --git a/include/linux/if_pppol2tp.h b/include/linux/if_pppol2tp.h
-index 96d40942e5a3..c87efd333faa 100644
--- a/include/linux/if_pppol2tp.h
-+++ b/include/linux/if_pppol2tp.h
-@@ -4,8 +4,6 @@
-  *
-  * This file supplies definitions required by the PPP over L2TP driver
-  * (l2tp_ppp.c).  All version information wrt this file is located in l2tp_ppp.c
- *
- * License:
-  */
- #ifndef __LINUX_IF_PPPOL2TP_H
- #define __LINUX_IF_PPPOL2TP_H
-diff --git a/include/linux/if_pppox.h b/include/linux/if_pppox.h
-index 69e813bcb947..db45d6f1c4f4 100644
--- a/include/linux/if_pppox.h
-+++ b/include/linux/if_pppox.h
-@@ -5,8 +5,6 @@
-  *
-  * This file supplies definitions required by the PPP over Ethernet driver
-  * (pppox.c).  All version information wrt this file is located in pppox.c
- *
- * License:
-  */
- #ifndef __LINUX_IF_PPPOX_H
- #define __LINUX_IF_PPPOX_H
-@@ -45,7 +43,7 @@ struct pppox_sock {
- 	/* struct sock must be the first member of pppox_sock */
- 	struct sock sk;
- 	struct ppp_channel chan;
-	struct pppox_sock	*next;	  /* for hash table */
-+	struct pppox_sock __rcu	*next;	  /* for hash table */
- 	union {
- 		struct pppoe_opt pppoe;
- 		struct pptp_opt  pptp;
-diff --git a/include/linux/if_rmnet.h b/include/linux/if_rmnet.h
-index 839d1e48b85e..c44bf6e80ecb 100644
--- a/include/linux/if_rmnet.h
-+++ b/include/linux/if_rmnet.h
-@@ -42,7 +42,7 @@ struct rmnet_map_ul_csum_header {
- 
- /* csum_info field:
-  *  OFFSET:	where (offset in bytes) to insert computed checksum
- *  UDP:	1 = UDP checksum (zero checkum means no checksum)
-+ *  UDP:	1 = UDP checksum (zero checksum means no checksum)
-  *  ENABLED:	1 = checksum computation requested
-  */
- #define MAP_CSUM_UL_OFFSET_MASK		GENMASK(13, 0)
-diff --git a/include/linux/if_team.h b/include/linux/if_team.h
-index fc985e5c739d..ce97d891cf72 100644
--- a/include/linux/if_team.h
-+++ b/include/linux/if_team.h
-@@ -162,8 +162,8 @@ struct team_option {
- 	bool per_port;
- 	unsigned int array_size; /* != 0 means the option is array */
- 	enum team_option_type type;
-	int (*init)(struct team *team, struct team_option_inst_info *info);
-	int (*getter)(struct team *team, struct team_gsetter_ctx *ctx);
-+	void (*init)(struct team *team, struct team_option_inst_info *info);
-+	void (*getter)(struct team *team, struct team_gsetter_ctx *ctx);
- 	int (*setter)(struct team *team, struct team_gsetter_ctx *ctx);
- };
- 
-@@ -189,7 +189,7 @@ struct team {
- 	struct net_device *dev; /* associated netdevice */
- 	struct team_pcpu_stats __percpu *pcpu_stats;
- 
-	struct mutex lock; /* used for overall locking, e.g. port lists write */
-+	const struct header_ops *header_ops_cache;
- 
- 	/*
- 	 * List of enabled ports and their count
-@@ -208,6 +208,7 @@ struct team {
- 	bool queue_override_enabled;
- 	struct list_head *qom_lists; /* array of queue override mapping lists */
- 	bool port_mtu_change_allowed;
-+	bool notifier_ctx;
- 	struct {
- 		unsigned int count;
- 		unsigned int interval; /* in ms */
-@@ -220,7 +221,6 @@ struct team {
- 		atomic_t count_pending;
- 		struct delayed_work dw;
- 	} mcast_rejoin;
-	struct lock_class_key team_lock_key;
- 	long mode_priv[TEAM_MODE_PRIV_LONGS];
- };
- 
-diff --git a/include/linux/if_tun.h b/include/linux/if_tun.h
-index 2a7660843444..80166eb62f41 100644
--- a/include/linux/if_tun.h
-+++ b/include/linux/if_tun.h
-@@ -19,52 +19,57 @@ struct tun_msg_ctl {
- 	void *ptr;
- };
- 
-struct tun_xdp_hdr {
-	int buflen;
-	struct virtio_net_hdr gso;
-};
-
- #if defined(CONFIG_TUN) || defined(CONFIG_TUN_MODULE)
- struct socket *tun_get_socket(struct file *);
- struct ptr_ring *tun_get_tx_ring(struct file *file);
-+
- static inline bool tun_is_xdp_frame(void *ptr)
- {
-       return (unsigned long)ptr & TUN_XDP_FLAG;
-+	return (unsigned long)ptr & TUN_XDP_FLAG;
- }
-+
- static inline void *tun_xdp_to_ptr(struct xdp_frame *xdp)
- {
-       return (void *)((unsigned long)xdp | TUN_XDP_FLAG);
-+	return (void *)((unsigned long)xdp | TUN_XDP_FLAG);
- }
-+
- static inline struct xdp_frame *tun_ptr_to_xdp(void *ptr)
- {
-       return (void *)((unsigned long)ptr & ~TUN_XDP_FLAG);
-+	return (void *)((unsigned long)ptr & ~TUN_XDP_FLAG);
- }
-+
- void tun_ptr_free(void *ptr);
- #else
- #include <linux/err.h>
- #include <linux/errno.h>
- struct file;
- struct socket;
-+
- static inline struct socket *tun_get_socket(struct file *f)
- {
- 	return ERR_PTR(-EINVAL);
- }
-+
- static inline struct ptr_ring *tun_get_tx_ring(struct file *f)
- {
- 	return ERR_PTR(-EINVAL);
- }
-+
- static inline bool tun_is_xdp_frame(void *ptr)
- {
- 	return false;
- }
-+
- static inline void *tun_xdp_to_ptr(struct xdp_frame *xdp)
- {
- 	return NULL;
- }
-+
- static inline struct xdp_frame *tun_ptr_to_xdp(void *ptr)
- {
- 	return NULL;
- }
-+
- static inline void tun_ptr_free(void *ptr)
- {
- }
-diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h
-index e00c4ee81ff7..f7f34eb15e06 100644
--- a/include/linux/if_vlan.h
-+++ b/include/linux/if_vlan.h
-@@ -62,6 +62,14 @@ static inline struct vlan_ethhdr *vlan_eth_hdr(const struct sk_buff *skb)
- 	return (struct vlan_ethhdr *)skb_mac_header(skb);
- }
- 
-+/* Prefer this version in TX path, instead of
-+ * skb_reset_mac_header() + vlan_eth_hdr()
-+ */
-+static inline struct vlan_ethhdr *skb_vlan_eth_hdr(const struct sk_buff *skb)
-+{
-+	return (struct vlan_ethhdr *)skb->data;
-+}
-+
- #define VLAN_PRIO_MASK		0xe000 /* Priority Code Point */
- #define VLAN_PRIO_SHIFT		13
- #define VLAN_CFI_MASK		0x1000 /* Canonical Format Indicator / Drop Eligible Indicator */
-@@ -71,12 +79,7 @@ static inline struct vlan_ethhdr *vlan_eth_hdr(const struct sk_buff *skb)
- /* found in socket.c */
- extern void vlan_ioctl_set(int (*hook)(struct net *, void __user *));
- 
-static inline bool is_vlan_dev(const struct net_device *dev)
-{
-        return dev->priv_flags & IFF_802_1Q_VLAN;
-}
-
-#define skb_vlan_tag_present(__skb)	((__skb)->vlan_present)
-+#define skb_vlan_tag_present(__skb)	(!!(__skb)->vlan_all)
- #define skb_vlan_tag_get(__skb)		((__skb)->vlan_tci)
- #define skb_vlan_tag_get_id(__skb)	((__skb)->vlan_tci & VLAN_VID_MASK)
- #define skb_vlan_tag_get_cfi(__skb)	(!!((__skb)->vlan_tci & VLAN_CFI_MASK))
-@@ -128,7 +131,7 @@ struct vlan_pcpu_stats {
- 	u32			tx_dropped;
- };
- 
-#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
-+#if IS_ENABLED(CONFIG_VLAN_8021Q)
- 
- extern struct net_device *__vlan_find_dev_deep_rcu(struct net_device *real_dev,
- 					       __be16 vlan_proto, u16 vlan_id);
-@@ -168,6 +171,7 @@ struct netpoll;
-  *	@real_dev_addr: address of underlying netdevice
-  *	@dent: proc dir entry
-  *	@vlan_pcpu_stats: ptr to percpu rx stats
-+ *	@netpoll: netpoll instance "propagated" down to @real_dev
-  */
- struct vlan_dev_priv {
- 	unsigned int				nr_ingress_mappings;
-@@ -191,6 +195,11 @@ struct vlan_dev_priv {
- #endif
- };
- 
-+static inline bool is_vlan_dev(const struct net_device *dev)
-+{
-+	return dev->priv_flags & IFF_802_1Q_VLAN;
-+}
-+
- static inline struct vlan_dev_priv *vlan_dev_priv(const struct net_device *dev)
- {
- 	return netdev_priv(dev);
-@@ -228,6 +237,11 @@ extern void vlan_vids_del_by_dev(struct net_device *dev,
- extern bool vlan_uses_dev(const struct net_device *dev);
- 
- #else
-+static inline bool is_vlan_dev(const struct net_device *dev)
-+{
-+	return false;
-+}
-+
- static inline struct net_device *
- __vlan_find_dev_deep_rcu(struct net_device *real_dev,
- 		     __be16 vlan_proto, u16 vlan_id)
-@@ -245,19 +259,19 @@ vlan_for_each(struct net_device *dev,
- 
- static inline struct net_device *vlan_dev_real_dev(const struct net_device *dev)
- {
-	BUG();
-+	WARN_ON_ONCE(1);
- 	return NULL;
- }
- 
- static inline u16 vlan_dev_vlan_id(const struct net_device *dev)
- {
-	BUG();
-+	WARN_ON_ONCE(1);
- 	return 0;
- }
- 
- static inline __be16 vlan_dev_vlan_proto(const struct net_device *dev)
- {
-	BUG();
-+	WARN_ON_ONCE(1);
- 	return 0;
- }
- 
-@@ -302,7 +316,7 @@ static inline bool vlan_uses_dev(const struct net_device *dev)
-  * eth_type_vlan - check for valid vlan ether type.
-  * @ethertype: ether type to check
-  *
- * Returns true if the ether type is a vlan ether type.
-+ * Returns: true if the ether type is a vlan ether type.
-  */
- static inline bool eth_type_vlan(__be16 ethertype)
- {
-@@ -333,25 +347,27 @@ static inline bool vlan_hw_offload_capable(netdev_features_t features,
-  * @mac_len: MAC header length including outer vlan headers
-  *
-  * Inserts the VLAN tag into @skb as part of the payload at offset mac_len
- * Returns error if skb_cow_head fails.
- *
-  * Does not change skb->protocol so this function can be used during receive.
-+ *
-+ * Returns: error if skb_cow_head fails.
-  */
- static inline int __vlan_insert_inner_tag(struct sk_buff *skb,
- 					  __be16 vlan_proto, u16 vlan_tci,
- 					  unsigned int mac_len)
- {
-+	const u8 meta_len = mac_len > ETH_TLEN ? skb_metadata_len(skb) : 0;
- 	struct vlan_ethhdr *veth;
- 
-	if (skb_cow_head(skb, VLAN_HLEN) < 0)
-+	if (skb_cow_head(skb, meta_len + VLAN_HLEN) < 0)
- 		return -ENOMEM;
- 
- 	skb_push(skb, VLAN_HLEN);
- 
- 	/* Move the mac header sans proto to the beginning of the new header. */
- 	if (likely(mac_len > ETH_TLEN))
-		memmove(skb->data, skb->data + VLAN_HLEN, mac_len - ETH_TLEN);
-	skb->mac_header -= VLAN_HLEN;
-+		skb_postpush_data_move(skb, VLAN_HLEN, mac_len - ETH_TLEN);
-+	if (skb_mac_header_was_set(skb))
-+		skb->mac_header -= VLAN_HLEN;
- 
- 	veth = (struct vlan_ethhdr *)(skb->data + mac_len - ETH_HLEN);
- 
-@@ -381,9 +397,9 @@ static inline int __vlan_insert_inner_tag(struct sk_buff *skb,
-  * @vlan_tci: VLAN TCI to insert
-  *
-  * Inserts the VLAN tag into @skb as part of the payload
- * Returns error if skb_cow_head fails.
- *
-  * Does not change skb->protocol so this function can be used during receive.
-+ *
-+ * Returns: error if skb_cow_head fails.
-  */
- static inline int __vlan_insert_tag(struct sk_buff *skb,
- 				    __be16 vlan_proto, u16 vlan_tci)
-@@ -399,12 +415,14 @@ static inline int __vlan_insert_tag(struct sk_buff *skb,
-  * @mac_len: MAC header length including outer vlan headers
-  *
-  * Inserts the VLAN tag into @skb as part of the payload at offset mac_len
- * Returns a VLAN tagged skb. If a new skb is created, @skb is freed.
-+ * Returns a VLAN tagged skb. This might change skb->head.
-  *
-  * Following the skb_unshare() example, in case of error, the calling function
-  * doesn't have to worry about freeing the original skb.
-  *
-  * Does not change skb->protocol so this function can be used during receive.
-+ *
-+ * Return: modified @skb on success, NULL on error (@skb is freed).
-  */
- static inline struct sk_buff *vlan_insert_inner_tag(struct sk_buff *skb,
- 						    __be16 vlan_proto,
-@@ -428,12 +446,14 @@ static inline struct sk_buff *vlan_insert_inner_tag(struct sk_buff *skb,
-  * @vlan_tci: VLAN TCI to insert
-  *
-  * Inserts the VLAN tag into @skb as part of the payload
- * Returns a VLAN tagged skb. If a new skb is created, @skb is freed.
-+ * Returns a VLAN tagged skb. This might change skb->head.
-  *
-  * Following the skb_unshare() example, in case of error, the calling function
-  * doesn't have to worry about freeing the original skb.
-  *
-  * Does not change skb->protocol so this function can be used during receive.
-+ *
-+ * Return: modified @skb on success, NULL on error (@skb is freed).
-  */
- static inline struct sk_buff *vlan_insert_tag(struct sk_buff *skb,
- 					      __be16 vlan_proto, u16 vlan_tci)
-@@ -448,10 +468,12 @@ static inline struct sk_buff *vlan_insert_tag(struct sk_buff *skb,
-  * @vlan_tci: VLAN TCI to insert
-  *
-  * Inserts the VLAN tag into @skb as part of the payload
- * Returns a VLAN tagged skb. If a new skb is created, @skb is freed.
-+ * Returns a VLAN tagged skb. This might change skb->head.
-  *
-  * Following the skb_unshare() example, in case of error, the calling function
-  * doesn't have to worry about freeing the original skb.
-+ *
-+ * Return: modified @skb on success, NULL on error (@skb is freed).
-  */
- static inline struct sk_buff *vlan_insert_tag_set_proto(struct sk_buff *skb,
- 							__be16 vlan_proto,
-@@ -471,7 +493,7 @@ static inline struct sk_buff *vlan_insert_tag_set_proto(struct sk_buff *skb,
-  */
- static inline void __vlan_hwaccel_clear_tag(struct sk_buff *skb)
- {
-	skb->vlan_present = 0;
-+	skb->vlan_all = 0;
- }
- 
- /**
-@@ -483,9 +505,7 @@ static inline void __vlan_hwaccel_clear_tag(struct sk_buff *skb)
-  */
- static inline void __vlan_hwaccel_copy_tag(struct sk_buff *dst, const struct sk_buff *src)
- {
-	dst->vlan_present = src->vlan_present;
-	dst->vlan_proto = src->vlan_proto;
-	dst->vlan_tci = src->vlan_tci;
-+	dst->vlan_all = src->vlan_all;
- }
- 
- /*
-@@ -519,7 +539,6 @@ static inline void __vlan_hwaccel_put_tag(struct sk_buff *skb,
- {
- 	skb->vlan_proto = vlan_proto;
- 	skb->vlan_tci = vlan_tci;
-	skb->vlan_present = 1;
- }
- 
- /**
-@@ -527,14 +546,14 @@ static inline void __vlan_hwaccel_put_tag(struct sk_buff *skb,
-  * @skb: skbuff to query
-  * @vlan_tci: buffer to store value
-  *
- * Returns error if the skb is not of VLAN type
-+ * Returns: error if the skb is not of VLAN type
-  */
- static inline int __vlan_get_tag(const struct sk_buff *skb, u16 *vlan_tci)
- {
-	struct vlan_ethhdr *veth = (struct vlan_ethhdr *)skb->data;
-+	struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb);
- 
- 	if (!eth_type_vlan(veth->h_vlan_proto))
-		return -EINVAL;
-+		return -ENODATA;
- 
- 	*vlan_tci = ntohs(veth->h_vlan_TCI);
- 	return 0;
-@@ -545,7 +564,7 @@ static inline int __vlan_get_tag(const struct sk_buff *skb, u16 *vlan_tci)
-  * @skb: skbuff to query
-  * @vlan_tci: buffer to store value
-  *
- * Returns error if @skb->vlan_tci is not set correctly
-+ * Returns: error if @skb->vlan_tci is not set correctly
-  */
- static inline int __vlan_hwaccel_get_tag(const struct sk_buff *skb,
- 					 u16 *vlan_tci)
-@@ -555,7 +574,7 @@ static inline int __vlan_hwaccel_get_tag(const struct sk_buff *skb,
- 		return 0;
- 	} else {
- 		*vlan_tci = 0;
-		return -EINVAL;
-+		return -ENODATA;
- 	}
- }
- 
-@@ -564,7 +583,7 @@ static inline int __vlan_hwaccel_get_tag(const struct sk_buff *skb,
-  * @skb: skbuff to query
-  * @vlan_tci: buffer to store value
-  *
- * Returns error if the skb is not VLAN tagged
-+ * Returns: error if the skb is not VLAN tagged
-  */
- static inline int vlan_get_tag(const struct sk_buff *skb, u16 *vlan_tci)
- {
-@@ -576,16 +595,19 @@ static inline int vlan_get_tag(const struct sk_buff *skb, u16 *vlan_tci)
- }
- 
- /**
- * vlan_get_protocol - get protocol EtherType.
-+ * __vlan_get_protocol_offset() - get protocol EtherType.
-  * @skb: skbuff to query
-  * @type: first vlan protocol
-+ * @mac_offset: MAC offset
-  * @depth: buffer to store length of eth and vlan tags in bytes
-  *
- * Returns the EtherType of the packet, regardless of whether it is
-+ * Returns: the EtherType of the packet, regardless of whether it is
-  * vlan encapsulated (normal or hardware accelerated) or not.
-  */
-static inline __be16 __vlan_get_protocol(const struct sk_buff *skb, __be16 type,
-					 int *depth)
-+static inline __be16 __vlan_get_protocol_offset(const struct sk_buff *skb,
-+						__be16 type,
-+						int mac_offset,
-+						int *depth)
- {
- 	unsigned int vlan_depth = skb->mac_len, parse_depth = VLAN_MAX_DEPTH;
- 
-@@ -604,7 +626,8 @@ static inline __be16 __vlan_get_protocol(const struct sk_buff *skb, __be16 type,
- 		do {
- 			struct vlan_hdr vhdr, *vh;
- 
-			vh = skb_header_pointer(skb, vlan_depth, sizeof(vhdr), &vhdr);
-+			vh = skb_header_pointer(skb, mac_offset + vlan_depth,
-+						sizeof(vhdr), &vhdr);
- 			if (unlikely(!vh || !--parse_depth))
- 				return 0;
- 
-@@ -619,11 +642,17 @@ static inline __be16 __vlan_get_protocol(const struct sk_buff *skb, __be16 type,
- 	return type;
- }
- 
-+static inline __be16 __vlan_get_protocol(const struct sk_buff *skb, __be16 type,
-+					 int *depth)
-+{
-+	return __vlan_get_protocol_offset(skb, type, 0, depth);
-+}
-+
- /**
-  * vlan_get_protocol - get protocol EtherType.
-  * @skb: skbuff to query
-  *
- * Returns the EtherType of the packet, regardless of whether it is
-+ * Returns: the EtherType of the packet, regardless of whether it is
-  * vlan encapsulated (normal or hardware accelerated) or not.
-  */
- static inline __be16 vlan_get_protocol(const struct sk_buff *skb)
-@@ -631,6 +660,23 @@ static inline __be16 vlan_get_protocol(const struct sk_buff *skb)
- 	return __vlan_get_protocol(skb, skb->protocol, NULL);
- }
- 
-+/* This version of __vlan_get_protocol() also pulls mac header in skb->head */
-+static inline __be16 vlan_get_protocol_and_depth(struct sk_buff *skb,
-+						 __be16 type, int *depth)
-+{
-+	int maclen;
-+
-+	type = __vlan_get_protocol(skb, type, &maclen);
-+
-+	if (type) {
-+		if (!pskb_may_pull(skb, maclen))
-+			type = 0;
-+		else if (depth)
-+			*depth = maclen;
-+	}
-+	return type;
-+}
-+
- /* A getter for the SKB protocol field which will handle VLAN tags consistently
-  * whether VLAN acceleration is enabled or not.
-  */
-@@ -680,10 +726,29 @@ static inline void vlan_set_encap_proto(struct sk_buff *skb,
- }
- 
- /**
-+ * vlan_remove_tag - remove outer VLAN tag from payload
-+ * @skb: skbuff to remove tag from
-+ * @vlan_tci: buffer to store value
-+ *
-+ * Expects the skb to contain a VLAN tag in the payload, and to have skb->data
-+ * pointing at the MAC header.
-+ */
-+static inline void vlan_remove_tag(struct sk_buff *skb, u16 *vlan_tci)
-+{
-+	struct vlan_hdr *vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
-+
-+	*vlan_tci = ntohs(vhdr->h_vlan_TCI);
-+
-+	vlan_set_encap_proto(skb, vhdr);
-+	__skb_pull(skb, VLAN_HLEN);
-+	skb_postpull_data_move(skb, VLAN_HLEN, 2 * ETH_ALEN);
-+}
-+
-+/**
-  * skb_vlan_tagged - check if skb is vlan tagged.
-  * @skb: skbuff to query
-  *
- * Returns true if the skb is tagged, regardless of whether it is hardware
-+ * Returns: true if the skb is tagged, regardless of whether it is hardware
-  * accelerated or not.
-  */
- static inline bool skb_vlan_tagged(const struct sk_buff *skb)
-@@ -699,7 +764,7 @@ static inline bool skb_vlan_tagged(const struct sk_buff *skb)
-  * skb_vlan_tagged_multi - check if skb is vlan tagged with multiple headers.
-  * @skb: skbuff to query
-  *
- * Returns true if the skb is tagged with multiple vlan headers, regardless
-+ * Returns: true if the skb is tagged with multiple vlan headers, regardless
-  * of whether it is hardware accelerated or not.
-  */
- static inline bool skb_vlan_tagged_multi(struct sk_buff *skb)
-@@ -715,7 +780,7 @@ static inline bool skb_vlan_tagged_multi(struct sk_buff *skb)
- 		if (unlikely(!pskb_may_pull(skb, VLAN_ETH_HLEN)))
- 			return false;
- 
-		veh = (struct vlan_ethhdr *)skb->data;
-+		veh = skb_vlan_eth_hdr(skb);
- 		protocol = veh->h_vlan_encapsulated_proto;
- 	}
- 
-@@ -730,7 +795,7 @@ static inline bool skb_vlan_tagged_multi(struct sk_buff *skb)
-  * @skb: skbuff to query
-  * @features: features to be checked
-  *
- * Returns features without unsafe ones if the skb has multiple tags.
-+ * Returns: features without unsafe ones if the skb has multiple tags.
-  */
- static inline netdev_features_t vlan_features_check(struct sk_buff *skb,
- 						    netdev_features_t features)
-@@ -754,9 +819,11 @@ static inline netdev_features_t vlan_features_check(struct sk_buff *skb,
-  * @h1: Pointer to vlan header
-  * @h2: Pointer to vlan header
-  *
- * Compare two vlan headers, returns 0 if equal.
-+ * Compare two vlan headers.
-  *
-  * Please note that alignment of h1 & h2 are only guaranteed to be 16 bits.
-+ *
-+ * Return: 0 if equal, arbitrary non-zero value if not equal.
-  */
- static inline unsigned long compare_vlan_header(const struct vlan_hdr *h1,
- 						const struct vlan_hdr *h2)
-diff --git a/include/linux/igmp.h b/include/linux/igmp.h
-index 93c262ecbdc9..073b30a9b850 100644
--- a/include/linux/igmp.h
-+++ b/include/linux/igmp.h
-@@ -15,6 +15,7 @@
- #include <linux/in.h>
- #include <linux/ip.h>
- #include <linux/refcount.h>
-+#include <linux/sockptr.h>
- #include <uapi/linux/igmp.h>
- 
- static inline struct igmphdr *igmp_hdr(const struct sk_buff *skb)
-@@ -38,7 +39,7 @@ struct ip_sf_socklist {
- 	unsigned int		sl_max;
- 	unsigned int		sl_count;
- 	struct rcu_head		rcu;
-	__be32			sl_addr[];
-+	__be32			sl_addr[] __counted_by(sl_max);
- };
- 
- #define IP_SFBLOCK	10	/* allocate this many at once */
-@@ -86,6 +87,8 @@ struct ip_mc_list {
- 	char			loaded;
- 	unsigned char		gsquery;	/* check source marks? */
- 	unsigned char		crcount;
-+	unsigned long		mca_cstamp;
-+	unsigned long		mca_tstamp;
- 	struct rcu_head		rcu;
- };
- 
-@@ -118,10 +121,10 @@ extern int ip_mc_source(int add, int omode, struct sock *sk,
- 		struct ip_mreq_source *mreqs, int ifindex);
- extern int ip_mc_msfilter(struct sock *sk, struct ip_msfilter *msf,int ifindex);
- extern int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf,
-		struct ip_msfilter __user *optval, int __user *optlen);
-+			sockptr_t optval, sockptr_t optlen);
- extern int ip_mc_gsfget(struct sock *sk, struct group_filter *gsf,
-			struct sockaddr_storage __user *p);
-extern int ip_mc_sf_allow(struct sock *sk, __be32 local, __be32 rmt,
-+			sockptr_t optval, size_t offset);
-+extern int ip_mc_sf_allow(const struct sock *sk, __be32 local, __be32 rmt,
- 			  int dif, int sdif);
- extern void ip_mc_init_dev(struct in_device *);
- extern void ip_mc_destroy_dev(struct in_device *);
-diff --git a/include/linux/iio/adc-helpers.h b/include/linux/iio/adc-helpers.h
-new file mode 100644
-index 000000000000..56b092a2a4c4
--- /dev/null
-+++ b/include/linux/iio/adc-helpers.h
-@@ -0,0 +1,27 @@
-+/* SPDX-License-Identifier: GPL-2.0-only */
-+
-+/*
-+ * The industrial I/O ADC firmware property parsing helpers
-+ *
-+ * Copyright (c) 2025 Matti Vaittinen <mazziesaccount@gmail.com>
-+ */
-+
-+#ifndef _INDUSTRIAL_IO_ADC_HELPERS_H_
-+#define _INDUSTRIAL_IO_ADC_HELPERS_H_
-+
-+#include <linux/property.h>
-+
-+struct device;
-+struct iio_chan_spec;
-+
-+static inline int iio_adc_device_num_channels(struct device *dev)
-+{
-+	return device_get_named_child_node_count(dev, "channel");
-+}
-+
-+int devm_iio_adc_device_alloc_chaninfo_se(struct device *dev,
-+					  const struct iio_chan_spec *template,
-+					  int max_chan_id,
-+					  struct iio_chan_spec **cs);
-+
-+#endif /* _INDUSTRIAL_IO_ADC_HELPERS_H_ */
-diff --git a/include/linux/iio/adc/ad_sigma_delta.h b/include/linux/iio/adc/ad_sigma_delta.h
-index 7852f6c9a714..6e70a412e218 100644
--- a/include/linux/iio/adc/ad_sigma_delta.h
-+++ b/include/linux/iio/adc/ad_sigma_delta.h
-@@ -8,6 +8,8 @@
- #ifndef __AD_SIGMA_DELTA_H__
- #define __AD_SIGMA_DELTA_H__
- 
-+#include <linux/iio/iio.h>
-+
- enum ad_sigma_delta_mode {
- 	AD_SD_MODE_CONTINUOUS = 0,
- 	AD_SD_MODE_SINGLE = 1,
-@@ -27,7 +29,10 @@ struct ad_sd_calib_data {
- 
- struct ad_sigma_delta;
- struct device;
-+struct gpio_desc;
- struct iio_dev;
-+struct spi_offload;
-+struct spi_offload_trigger;
- 
- /**
-  * struct ad_sigma_delta_info - Sigma Delta driver specific callbacks and options
-@@ -35,10 +40,19 @@ struct iio_dev;
-  * @append_status: Will be called to enable status append at the end of the sample, may be NULL.
-  * @set_mode: Will be called to select the current mode, may be NULL.
-  * @disable_all: Will be called to disable all channels, may be NULL.
-+ * @disable_one: Will be called to disable a single channel after
-+ *		ad_sigma_delta_single_conversion(), may be NULL.
-+ *		Usage of this callback expects iio_chan_spec.address to contain
-+ *		the value required for the driver to identify the channel.
-  * @postprocess_sample: Is called for each sampled data word, can be used to
-  *		modify or drop the sample data, it, may be NULL.
-  * @has_registers: true if the device has writable and readable registers, false
-  *		if there is just one read-only sample data shift register.
-+ * @has_named_irqs: Set to true if there is more than one IRQ line.
-+ * @supports_spi_offload: Set to true if the driver supports SPI offload. Often
-+ *		special considerations are needed for scan_type and other channel
-+ *		info, so individual drivers have to set this to let the core
-+ *		code know that it can use SPI offload if it is available.
-  * @addr_shift: Shift of the register address in the communications register.
-  * @read_mask: Mask for the communications register having the read bit set.
-  * @status_ch_mask: Mask for the channel number stored in status register.
-@@ -46,20 +60,25 @@ struct iio_dev;
-  *   be used.
-  * @irq_flags: flags for the interrupt used by the triggered buffer
-  * @num_slots: Number of sequencer slots
-+ * @num_resetclks: Number of SPI clk cycles with MOSI=1 to reset the chip.
-  */
- struct ad_sigma_delta_info {
- 	int (*set_channel)(struct ad_sigma_delta *, unsigned int channel);
- 	int (*append_status)(struct ad_sigma_delta *, bool append);
- 	int (*set_mode)(struct ad_sigma_delta *, enum ad_sigma_delta_mode mode);
- 	int (*disable_all)(struct ad_sigma_delta *);
-+	int (*disable_one)(struct ad_sigma_delta *, unsigned int chan);
- 	int (*postprocess_sample)(struct ad_sigma_delta *, unsigned int raw_sample);
- 	bool has_registers;
-+	bool has_named_irqs;
-+	bool supports_spi_offload;
- 	unsigned int addr_shift;
- 	unsigned int read_mask;
- 	unsigned int status_ch_mask;
- 	unsigned int data_reg;
- 	unsigned long irq_flags;
- 	unsigned int num_slots;
-+	unsigned int num_resetclks;
- };
- 
- /**
-@@ -76,21 +95,28 @@ struct ad_sigma_delta {
- 
- /* private: */
- 	struct completion	completion;
-+	spinlock_t		irq_lock; /* protects .irq_dis and irq en/disable state */
- 	bool			irq_dis;
- 
- 	bool			bus_locked;
- 	bool			keep_cs_asserted;
- 
-	uint8_t			comm;
-+	u8			comm;
- 
- 	const struct ad_sigma_delta_info *info;
- 	unsigned int		active_slots;
- 	unsigned int		current_slot;
- 	unsigned int		num_slots;
-+	struct gpio_desc	*rdy_gpiod;
-+	int			irq_line;
- 	bool			status_appended;
- 	/* map slots to channels in order to know what to expect from devices */
- 	unsigned int		*slots;
-	uint8_t			*samples_buf;
-+	struct spi_message	sample_msg;
-+	struct spi_transfer	sample_xfer[2];
-+	u8			*samples_buf;
-+	struct spi_offload	*offload;
-+	struct spi_offload_trigger *offload_trigger;
- 
- 	/*
- 	 * DMA (thus cache coherency maintenance) requires the
-@@ -99,10 +125,16 @@ struct ad_sigma_delta {
- 	 * 'rx_buf' is up to 32 bits per sample + 64 bit timestamp,
- 	 * rounded to 16 bytes to take into account padding.
- 	 */
-	uint8_t				tx_buf[4] ____cacheline_aligned;
-	uint8_t				rx_buf[16] __aligned(8);
-+	u8				tx_buf[4] __aligned(IIO_DMA_MINALIGN);
-+	u8				rx_buf[16] __aligned(8);
-+	u8				sample_addr;
- };
- 
-+static inline bool ad_sigma_delta_has_spi_offload(struct ad_sigma_delta *sd)
-+{
-+	return sd->offload != NULL;
-+}
-+
- static inline int ad_sigma_delta_set_channel(struct ad_sigma_delta *sd,
- 	unsigned int channel)
- {
-@@ -135,6 +167,15 @@ static inline int ad_sigma_delta_disable_all(struct ad_sigma_delta *sd)
- 	return 0;
- }
- 
-+static inline int ad_sigma_delta_disable_one(struct ad_sigma_delta *sd,
-+					     unsigned int chan)
-+{
-+	if (sd->info->disable_one)
-+		return sd->info->disable_one(sd, chan);
-+
-+	return 0;
-+}
-+
- static inline int ad_sigma_delta_set_mode(struct ad_sigma_delta *sd,
- 	unsigned int mode)
- {
-@@ -153,14 +194,13 @@ static inline int ad_sigma_delta_postprocess_sample(struct ad_sigma_delta *sd,
- 	return 0;
- }
- 
-void ad_sd_set_comm(struct ad_sigma_delta *sigma_delta, uint8_t comm);
-+void ad_sd_set_comm(struct ad_sigma_delta *sigma_delta, u8 comm);
- int ad_sd_write_reg(struct ad_sigma_delta *sigma_delta, unsigned int reg,
- 	unsigned int size, unsigned int val);
- int ad_sd_read_reg(struct ad_sigma_delta *sigma_delta, unsigned int reg,
- 	unsigned int size, unsigned int *val);
- 
-int ad_sd_reset(struct ad_sigma_delta *sigma_delta,
-	unsigned int reset_length);
-+int ad_sd_reset(struct ad_sigma_delta *sigma_delta);
- 
- int ad_sigma_delta_single_conversion(struct iio_dev *indio_dev,
- 	const struct iio_chan_spec *chan, int *val);
-diff --git a/include/linux/iio/adc/adi-axi-adc.h b/include/linux/iio/adc/adi-axi-adc.h
-deleted file mode 100644
-index 52620e5b8052..000000000000
--- a/include/linux/iio/adc/adi-axi-adc.h
-+++ /dev/null
-@@ -1,64 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Analog Devices Generic AXI ADC IP core driver/library
- * Link: https://wiki.analog.com/resources/fpga/docs/axi_adc_ip
- *
- * Copyright 2012-2020 Analog Devices Inc.
- */
-#ifndef __ADI_AXI_ADC_H__
-#define __ADI_AXI_ADC_H__
-
-struct device;
-struct iio_chan_spec;
-
-/**
- * struct adi_axi_adc_chip_info - Chip specific information
- * @name		Chip name
- * @id			Chip ID (usually product ID)
- * @channels		Channel specifications of type @struct iio_chan_spec
- * @num_channels	Number of @channels
- * @scale_table		Supported scales by the chip; tuples of 2 ints
- * @num_scales		Number of scales in the table
- * @max_rate		Maximum sampling rate supported by the device
- */
-struct adi_axi_adc_chip_info {
-	const char			*name;
-	unsigned int			id;
-
-	const struct iio_chan_spec	*channels;
-	unsigned int			num_channels;
-
-	const unsigned int		(*scale_table)[2];
-	int				num_scales;
-
-	unsigned long			max_rate;
-};
-
-/**
- * struct adi_axi_adc_conv - data of the ADC attached to the AXI ADC
- * @chip_info		chip info details for the client ADC
- * @preenable_setup	op to run in the client before enabling the AXI ADC
- * @reg_access		IIO debugfs_reg_access hook for the client ADC
- * @read_raw		IIO read_raw hook for the client ADC
- * @write_raw		IIO write_raw hook for the client ADC
- */
-struct adi_axi_adc_conv {
-	const struct adi_axi_adc_chip_info		*chip_info;
-
-	int (*preenable_setup)(struct adi_axi_adc_conv *conv);
-	int (*reg_access)(struct adi_axi_adc_conv *conv, unsigned int reg,
-			  unsigned int writeval, unsigned int *readval);
-	int (*read_raw)(struct adi_axi_adc_conv *conv,
-			struct iio_chan_spec const *chan,
-			int *val, int *val2, long mask);
-	int (*write_raw)(struct adi_axi_adc_conv *conv,
-			 struct iio_chan_spec const *chan,
-			 int val, int val2, long mask);
-};
-
-struct adi_axi_adc_conv *devm_adi_axi_adc_conv_register(struct device *dev,
-							size_t sizeof_priv);
-
-void *adi_axi_adc_conv_priv(struct adi_axi_adc_conv *conv);
-
-#endif
-diff --git a/include/linux/iio/adc/qcom-vadc-common.h b/include/linux/iio/adc/qcom-vadc-common.h
-index aa21b032e861..3bf4c49726a7 100644
--- a/include/linux/iio/adc/qcom-vadc-common.h
-+++ b/include/linux/iio/adc/qcom-vadc-common.h
-@@ -83,27 +83,27 @@ struct vadc_linear_graph {
- /**
-  * enum vadc_scale_fn_type - Scaling function to convert ADC code to
-  *				physical scaled units for the channel.
- * SCALE_DEFAULT: Default scaling to convert raw adc code to voltage (uV).
- * SCALE_THERM_100K_PULLUP: Returns temperature in millidegC.
-+ * @SCALE_DEFAULT: Default scaling to convert raw adc code to voltage (uV).
-+ * @SCALE_THERM_100K_PULLUP: Returns temperature in millidegC.
-  *				 Uses a mapping table with 100K pullup.
- * SCALE_PMIC_THERM: Returns result in milli degree's Centigrade.
- * SCALE_XOTHERM: Returns XO thermistor voltage in millidegC.
- * SCALE_PMI_CHG_TEMP: Conversion for PMI CHG temp
- * SCALE_HW_CALIB_DEFAULT: Default scaling to convert raw adc code to
-+ * @SCALE_PMIC_THERM: Returns result in milli degree's Centigrade.
-+ * @SCALE_XOTHERM: Returns XO thermistor voltage in millidegC.
-+ * @SCALE_PMI_CHG_TEMP: Conversion for PMI CHG temp
-+ * @SCALE_HW_CALIB_DEFAULT: Default scaling to convert raw adc code to
-  *	voltage (uV) with hardware applied offset/slope values to adc code.
- * SCALE_HW_CALIB_THERM_100K_PULLUP: Returns temperature in millidegC using
-+ * @SCALE_HW_CALIB_THERM_100K_PULLUP: Returns temperature in millidegC using
-  *	lookup table. The hardware applies offset/slope to adc code.
- * SCALE_HW_CALIB_XOTHERM: Returns XO thermistor voltage in millidegC using
-+ * @SCALE_HW_CALIB_XOTHERM: Returns XO thermistor voltage in millidegC using
-  *	100k pullup. The hardware applies offset/slope to adc code.
- * SCALE_HW_CALIB_THERM_100K_PU_PM7: Returns temperature in millidegC using
-+ * @SCALE_HW_CALIB_THERM_100K_PU_PM7: Returns temperature in millidegC using
-  *	lookup table for PMIC7. The hardware applies offset/slope to adc code.
- * SCALE_HW_CALIB_PMIC_THERM: Returns result in milli degree's Centigrade.
-+ * @SCALE_HW_CALIB_PMIC_THERM: Returns result in milli degree's Centigrade.
-  *	The hardware applies offset/slope to adc code.
- * SCALE_HW_CALIB_PMIC_THERM: Returns result in milli degree's Centigrade.
-+ * @SCALE_HW_CALIB_PMIC_THERM: Returns result in milli degree's Centigrade.
-  *	The hardware applies offset/slope to adc code. This is for PMIC7.
- * SCALE_HW_CALIB_PM5_CHG_TEMP: Returns result in millidegrees for PMIC5
-+ * @SCALE_HW_CALIB_PM5_CHG_TEMP: Returns result in millidegrees for PMIC5
-  *	charger temperature.
- * SCALE_HW_CALIB_PM5_SMB_TEMP: Returns result in millidegrees for PMIC5
-+ * @SCALE_HW_CALIB_PM5_SMB_TEMP: Returns result in millidegrees for PMIC5
-  *	SMB1390 temperature.
-  */
- enum vadc_scale_fn_type {
-@@ -120,6 +120,7 @@ enum vadc_scale_fn_type {
- 	SCALE_HW_CALIB_PMIC_THERM_PM7,
- 	SCALE_HW_CALIB_PM5_CHG_TEMP,
- 	SCALE_HW_CALIB_PM5_SMB_TEMP,
-+	/* private: */
- 	SCALE_HW_CALIB_INVALID,
- };
- 
-diff --git a/include/linux/iio/backend.h b/include/linux/iio/backend.h
-new file mode 100644
-index 000000000000..7f815f3fed6a
--- /dev/null
-+++ b/include/linux/iio/backend.h
-@@ -0,0 +1,270 @@
-+/* SPDX-License-Identifier: GPL-2.0-or-later */
-+#ifndef _IIO_BACKEND_H_
-+#define _IIO_BACKEND_H_
-+
-+#include <linux/types.h>
-+#include <linux/iio/iio.h>
-+
-+struct iio_chan_spec;
-+struct fwnode_handle;
-+struct iio_backend;
-+struct device;
-+struct iio_dev;
-+
-+enum iio_backend_data_type {
-+	IIO_BACKEND_TWOS_COMPLEMENT,
-+	IIO_BACKEND_OFFSET_BINARY,
-+	IIO_BACKEND_DATA_UNSIGNED,
-+	IIO_BACKEND_DATA_TYPE_MAX
-+};
-+
-+enum iio_backend_data_source {
-+	IIO_BACKEND_INTERNAL_CONTINUOUS_WAVE,
-+	IIO_BACKEND_EXTERNAL,
-+	IIO_BACKEND_INTERNAL_RAMP_16BIT,
-+	IIO_BACKEND_DATA_SOURCE_MAX
-+};
-+
-+#define iio_backend_debugfs_ptr(ptr)	PTR_IF(IS_ENABLED(CONFIG_DEBUG_FS), ptr)
-+
-+/**
-+ * IIO_BACKEND_EX_INFO - Helper for an IIO extended channel attribute
-+ * @_name: Attribute name
-+ * @_shared: Whether the attribute is shared between all channels
-+ * @_what: Data private to the driver
-+ */
-+#define IIO_BACKEND_EX_INFO(_name, _shared, _what) {	\
-+	.name = (_name),				\
-+	.shared = (_shared),				\
-+	.read =  iio_backend_ext_info_get,		\
-+	.write = iio_backend_ext_info_set,		\
-+	.private = (_what),				\
-+}
-+
-+/**
-+ * struct iio_backend_data_fmt - Backend data format
-+ * @type: Data type.
-+ * @sign_extend: Bool to tell if the data is sign extended.
-+ * @enable: Enable/Disable the data format module. If disabled,
-+ *	    not formatting will happen.
-+ */
-+struct iio_backend_data_fmt {
-+	enum iio_backend_data_type type;
-+	bool sign_extend;
-+	bool enable;
-+};
-+
-+/* vendor specific from 32 */
-+enum iio_backend_test_pattern {
-+	IIO_BACKEND_NO_TEST_PATTERN,
-+	/* modified prbs9 */
-+	IIO_BACKEND_ADI_PRBS_9A = 32,
-+	/* modified prbs23 */
-+	IIO_BACKEND_ADI_PRBS_23A,
-+	IIO_BACKEND_TEST_PATTERN_MAX
-+};
-+
-+enum iio_backend_sample_trigger {
-+	IIO_BACKEND_SAMPLE_TRIGGER_EDGE_FALLING,
-+	IIO_BACKEND_SAMPLE_TRIGGER_EDGE_RISING,
-+	IIO_BACKEND_SAMPLE_TRIGGER_MAX
-+};
-+
-+enum iio_backend_interface_type {
-+	IIO_BACKEND_INTERFACE_SERIAL_LVDS,
-+	IIO_BACKEND_INTERFACE_SERIAL_CMOS,
-+	IIO_BACKEND_INTERFACE_MAX
-+};
-+
-+enum iio_backend_filter_type {
-+	IIO_BACKEND_FILTER_TYPE_DISABLED,
-+	IIO_BACKEND_FILTER_TYPE_SINC1,
-+	IIO_BACKEND_FILTER_TYPE_SINC5,
-+	IIO_BACKEND_FILTER_TYPE_SINC5_PLUS_COMP,
-+	IIO_BACKEND_FILTER_TYPE_MAX
-+};
-+
-+/**
-+ * struct iio_backend_ops - operations structure for an iio_backend
-+ * @enable: Enable backend.
-+ * @disable: Disable backend.
-+ * @chan_enable: Enable one channel.
-+ * @chan_disable: Disable one channel.
-+ * @data_format_set: Configure the data format for a specific channel.
-+ * @data_source_set: Configure the data source for a specific channel.
-+ * @data_source_get: Data source getter for a specific channel.
-+ * @set_sample_rate: Configure the sampling rate for a specific channel.
-+ * @test_pattern_set: Configure a test pattern.
-+ * @chan_status: Get the channel status.
-+ * @iodelay_set: Set digital I/O delay.
-+ * @data_sample_trigger: Control when to sample data.
-+ * @request_buffer: Request an IIO buffer.
-+ * @free_buffer: Free an IIO buffer.
-+ * @extend_chan_spec: Extend an IIO channel.
-+ * @ext_info_set: Extended info setter.
-+ * @ext_info_get: Extended info getter.
-+ * @interface_type_get: Interface type.
-+ * @data_size_set: Data size.
-+ * @oversampling_ratio_set: Set Oversampling ratio.
-+ * @read_raw: Read a channel attribute from a backend device
-+ * @debugfs_print_chan_status: Print channel status into a buffer.
-+ * @debugfs_reg_access: Read or write register value of backend.
-+ * @filter_type_set: Set filter type.
-+ * @interface_data_align: Perform the data alignment process.
-+ * @num_lanes_set: Set the number of lanes enabled.
-+ * @ddr_enable: Enable interface DDR (Double Data Rate) mode.
-+ * @ddr_disable: Disable interface DDR (Double Data Rate) mode.
-+ * @data_stream_enable: Enable data stream.
author	Hobin Woo <hobin.woo@samsung.com>	2024-07-05 12:27:25 +0900
committer	Steve French <stfrench@microsoft.com>	2024-07-05 09:56:13 -0500
commit	e2e33caa5dc2eae7bddf88b22ce11ec3d760e5cd (patch)
tree	4384112736f21d7b9116ee907d048d300f7451f3
parent	25a6e135569b3901452e4863c94560df7c11c492 (diff)